2023-09-23T22:31:56.772 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-09-23T22:31:56.772 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401893 --name teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi --verbose --description upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401893/orig.config.yaml 2023-09-23T22:31:56.831 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T22:31:56.919 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401893 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7401893' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 4997 sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 targets: smithi110.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi138.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-23_14:23:10 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-09-23T22:31:56.919 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa; will attempt to use it 2023-09-23T22:31:56.920 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks 2023-09-23T22:31:56.920 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-09-23T22:31:56.922 INFO:teuthology.task.internal:Checking packages... 2023-09-23T22:31:56.943 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'bb0daeff97f020f686596fca6eba20c310e26fb7' 2023-09-23T22:31:56.943 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T22:31:56.943 INFO:teuthology.packaging:ref: None 2023-09-23T22:31:56.943 INFO:teuthology.packaging:tag: None 2023-09-23T22:31:56.943 INFO:teuthology.packaging:branch: reef 2023-09-23T22:31:56.943 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:31:56.944 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-09-23T22:31:57.342 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-462.gceea2059 2023-09-23T22:31:57.342 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-09-23T22:31:57.349 INFO:teuthology.task.internal:no buildpackages task found 2023-09-23T22:31:57.349 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-09-23T22:31:57.364 INFO:teuthology.task.internal:Saving configuration 2023-09-23T22:31:57.387 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-09-23T22:31:57.394 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T22:31:57.411 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi110.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401893', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 22:26:55.528142', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T22:31:57.428 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi138.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401893', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 22:26:55.529538', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T22:31:57.428 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-09-23T22:31:57.433 INFO:teuthology.task.internal:roles: ubuntu@smithi110.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-09-23T22:31:57.434 INFO:teuthology.task.internal:roles: ubuntu@smithi138.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-09-23T22:31:57.434 INFO:teuthology.run_tasks:Running task console_log... 2023-09-23T22:31:57.493 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f0115d02af0>, signals=[15]) 2023-09-23T22:31:57.494 INFO:teuthology.run_tasks:Running task internal.connect... 2023-09-23T22:31:57.500 INFO:teuthology.task.internal:Opening connections... 2023-09-23T22:31:57.500 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi110.front.sepia.ceph.com 2023-09-23T22:31:57.501 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi110.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T22:31:57.568 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi138.front.sepia.ceph.com 2023-09-23T22:31:57.569 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T22:31:57.635 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-09-23T22:31:57.642 DEBUG:teuthology.orchestra.run.smithi110:> uname -m 2023-09-23T22:31:57.661 INFO:teuthology.orchestra.run.smithi110.stdout:x86_64 2023-09-23T22:31:57.661 DEBUG:teuthology.orchestra.run.smithi110:> cat /etc/os-release 2023-09-23T22:31:57.719 INFO:teuthology.orchestra.run.smithi110.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T22:31:57.719 INFO:teuthology.orchestra.run.smithi110.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T22:31:57.719 INFO:teuthology.orchestra.run.smithi110.stdout:ID="rhel" 2023-09-23T22:31:57.719 INFO:teuthology.orchestra.run.smithi110.stdout:ID_LIKE="fedora" 2023-09-23T22:31:57.719 INFO:teuthology.orchestra.run.smithi110.stdout:VERSION_ID="8.6" 2023-09-23T22:31:57.719 INFO:teuthology.orchestra.run.smithi110.stdout:PLATFORM_ID="platform:el8" 2023-09-23T22:31:57.719 INFO:teuthology.orchestra.run.smithi110.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T22:31:57.719 INFO:teuthology.orchestra.run.smithi110.stdout:ANSI_COLOR="0;31" 2023-09-23T22:31:57.719 INFO:teuthology.orchestra.run.smithi110.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T22:31:57.719 INFO:teuthology.orchestra.run.smithi110.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T22:31:57.720 INFO:teuthology.orchestra.run.smithi110.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T22:31:57.720 INFO:teuthology.orchestra.run.smithi110.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T22:31:57.720 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:31:57.720 INFO:teuthology.orchestra.run.smithi110.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T22:31:57.720 INFO:teuthology.orchestra.run.smithi110.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T22:31:57.720 INFO:teuthology.orchestra.run.smithi110.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T22:31:57.720 INFO:teuthology.orchestra.run.smithi110.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T22:31:57.720 INFO:teuthology.lock.ops:Updating smithi110.front.sepia.ceph.com on lock server 2023-09-23T22:31:57.738 DEBUG:teuthology.orchestra.run.smithi138:> uname -m 2023-09-23T22:31:57.757 INFO:teuthology.orchestra.run.smithi138.stdout:x86_64 2023-09-23T22:31:57.758 DEBUG:teuthology.orchestra.run.smithi138:> cat /etc/os-release 2023-09-23T22:31:57.816 INFO:teuthology.orchestra.run.smithi138.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T22:31:57.817 INFO:teuthology.orchestra.run.smithi138.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T22:31:57.817 INFO:teuthology.orchestra.run.smithi138.stdout:ID="rhel" 2023-09-23T22:31:57.817 INFO:teuthology.orchestra.run.smithi138.stdout:ID_LIKE="fedora" 2023-09-23T22:31:57.817 INFO:teuthology.orchestra.run.smithi138.stdout:VERSION_ID="8.6" 2023-09-23T22:31:57.817 INFO:teuthology.orchestra.run.smithi138.stdout:PLATFORM_ID="platform:el8" 2023-09-23T22:31:57.817 INFO:teuthology.orchestra.run.smithi138.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T22:31:57.817 INFO:teuthology.orchestra.run.smithi138.stdout:ANSI_COLOR="0;31" 2023-09-23T22:31:57.817 INFO:teuthology.orchestra.run.smithi138.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T22:31:57.817 INFO:teuthology.orchestra.run.smithi138.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T22:31:57.817 INFO:teuthology.orchestra.run.smithi138.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T22:31:57.817 INFO:teuthology.orchestra.run.smithi138.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T22:31:57.817 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:31:57.818 INFO:teuthology.orchestra.run.smithi138.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T22:31:57.818 INFO:teuthology.orchestra.run.smithi138.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T22:31:57.818 INFO:teuthology.orchestra.run.smithi138.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T22:31:57.818 INFO:teuthology.orchestra.run.smithi138.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T22:31:57.818 INFO:teuthology.lock.ops:Updating smithi138.front.sepia.ceph.com on lock server 2023-09-23T22:31:57.837 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-09-23T22:31:57.846 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-09-23T22:31:57.862 INFO:teuthology.task.internal:Checking for old test directory... 2023-09-23T22:31:57.862 DEBUG:teuthology.orchestra.run.smithi110:> test '!' -e /home/ubuntu/cephtest 2023-09-23T22:31:57.864 DEBUG:teuthology.orchestra.run.smithi138:> test '!' -e /home/ubuntu/cephtest 2023-09-23T22:31:57.883 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-09-23T22:31:57.888 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-09-23T22:31:57.888 DEBUG:teuthology.orchestra.run.smithi110:> test -z $(ls -A /var/lib/ceph) 2023-09-23T22:31:57.922 DEBUG:teuthology.orchestra.run.smithi138:> test -z $(ls -A /var/lib/ceph) 2023-09-23T22:31:57.953 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-09-23T22:31:58.007 INFO:teuthology.run_tasks:Running task kernel... 2023-09-23T22:31:58.020 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-09-23T22:31:58.020 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-09-23T22:31:58.021 DEBUG:teuthology.orchestra.run.smithi110:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T22:31:58.021 DEBUG:teuthology.orchestra.run.smithi138:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T22:31:58.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T22:31:58.038 DEBUG:teuthology.orchestra.run.smithi110:> uname -r 2023-09-23T22:31:58.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T22:31:58.040 DEBUG:teuthology.orchestra.run.smithi138:> uname -r 2023-09-23T22:31:58.096 INFO:teuthology.orchestra.run.smithi110.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T22:31:58.096 INFO:teuthology.task.kernel:Running kernel on smithi110: 4.18.0-372.9.1.el8.x86_64 2023-09-23T22:31:58.097 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum install -y kernel 2023-09-23T22:31:58.098 INFO:teuthology.orchestra.run.smithi138.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T22:31:58.098 INFO:teuthology.task.kernel:Running kernel on smithi138: 4.18.0-372.9.1.el8.x86_64 2023-09-23T22:31:58.098 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum install -y kernel 2023-09-23T22:31:59.283 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:31:59.284 INFO:teuthology.orchestra.run.smithi138.stdout:Unable to read consumer identity 2023-09-23T22:31:59.348 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:31:59.348 INFO:teuthology.orchestra.run.smithi110.stdout:Unable to read consumer identity 2023-09-23T22:32:00.784 INFO:teuthology.orchestra.run.smithi138.stdout:Copr repo for python3-asyncssh owned by ceph 20 kB/s | 3.5 kB 00:00 2023-09-23T22:32:00.830 INFO:teuthology.orchestra.run.smithi110.stdout:Copr repo for python3-asyncssh owned by ceph 18 kB/s | 3.5 kB 00:00 2023-09-23T22:32:01.676 INFO:teuthology.orchestra.run.smithi138.stdout:Extra Packages for Enterprise Linux 18 MB/s | 16 MB 00:00 2023-09-23T22:32:01.997 INFO:teuthology.orchestra.run.smithi110.stdout:Extra Packages for Enterprise Linux 14 MB/s | 16 MB 00:01 2023-09-23T22:32:05.168 INFO:teuthology.orchestra.run.smithi138.stdout:lab-extras 404 kB/s | 24 kB 00:00 2023-09-23T22:32:05.464 INFO:teuthology.orchestra.run.smithi110.stdout:lab-extras 384 kB/s | 24 kB 00:00 2023-09-23T22:32:05.870 INFO:teuthology.orchestra.run.smithi138.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T22:32:05.922 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:32:05.923 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-09-23T22:32:05.923 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:32:06.051 DEBUG:teuthology.orchestra.run.smithi138:> echo no | sudo yum reinstall kernel || true 2023-09-23T22:32:06.158 INFO:teuthology.orchestra.run.smithi110.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T22:32:06.206 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:32:06.207 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-09-23T22:32:06.207 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:32:06.313 DEBUG:teuthology.orchestra.run.smithi110:> echo no | sudo yum reinstall kernel || true 2023-09-23T22:32:06.381 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:32:06.381 INFO:teuthology.orchestra.run.smithi138.stdout:Unable to read consumer identity 2023-09-23T22:32:06.643 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:00:01 ago on Sat 23 Sep 2023 10:32:05 PM UTC. 2023-09-23T22:32:06.645 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:32:06.645 INFO:teuthology.orchestra.run.smithi110.stdout:Unable to read consumer identity 2023-09-23T22:32:06.684 INFO:teuthology.orchestra.run.smithi138.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T22:32:06.727 INFO:teuthology.orchestra.run.smithi138.stderr:Error: No packages marked for reinstall. 2023-09-23T22:32:06.774 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum reinstall -y kernel || true 2023-09-23T22:32:06.904 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:00:01 ago on Sat 23 Sep 2023 10:32:05 PM UTC. 2023-09-23T22:32:06.944 INFO:teuthology.orchestra.run.smithi110.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T22:32:06.975 INFO:teuthology.orchestra.run.smithi110.stderr:Error: No packages marked for reinstall. 2023-09-23T22:32:07.020 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum reinstall -y kernel || true 2023-09-23T22:32:07.103 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:32:07.103 INFO:teuthology.orchestra.run.smithi138.stdout:Unable to read consumer identity 2023-09-23T22:32:07.347 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:32:07.347 INFO:teuthology.orchestra.run.smithi110.stdout:Unable to read consumer identity 2023-09-23T22:32:07.359 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 10:32:05 PM UTC. 2023-09-23T22:32:07.401 INFO:teuthology.orchestra.run.smithi138.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T22:32:07.449 INFO:teuthology.orchestra.run.smithi138.stderr:Error: No packages marked for reinstall. 2023-09-23T22:32:07.495 DEBUG:teuthology.orchestra.run.smithi138:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T22:32:07.575 INFO:teuthology.orchestra.run.smithi138.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T22:32:07.575 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T22:32:07.575 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T22:32:07.576 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-09-23T22:32:07.576 DEBUG:teuthology.orchestra.run.smithi138:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T22:32:07.603 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 10:32:05 PM UTC. 2023-09-23T22:32:07.605 INFO:teuthology.orchestra.run.smithi138.stdout:ttyS1 2023-09-23T22:32:07.617 DEBUG:teuthology.parallel:result is None 2023-09-23T22:32:07.643 INFO:teuthology.orchestra.run.smithi110.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T22:32:07.680 INFO:teuthology.orchestra.run.smithi110.stderr:Error: No packages marked for reinstall. 2023-09-23T22:32:07.727 DEBUG:teuthology.orchestra.run.smithi110:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T22:32:07.806 INFO:teuthology.orchestra.run.smithi110.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T22:32:07.807 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T22:32:07.807 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T22:32:07.807 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-09-23T22:32:07.807 DEBUG:teuthology.orchestra.run.smithi110:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T22:32:07.841 INFO:teuthology.orchestra.run.smithi110.stdout:ttyS1 2023-09-23T22:32:07.853 DEBUG:teuthology.parallel:result is None 2023-09-23T22:32:07.853 INFO:teuthology.run_tasks:Running task internal.base... 2023-09-23T22:32:07.861 INFO:teuthology.task.internal:Creating test directory... 2023-09-23T22:32:07.861 DEBUG:teuthology.orchestra.run.smithi110:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T22:32:07.895 DEBUG:teuthology.orchestra.run.smithi138:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T22:32:07.912 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-09-23T22:32:07.918 INFO:teuthology.run_tasks:Running task internal.archive... 2023-09-23T22:32:07.924 INFO:teuthology.task.internal:Creating archive directory... 2023-09-23T22:32:07.924 DEBUG:teuthology.orchestra.run.smithi110:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T22:32:07.952 DEBUG:teuthology.orchestra.run.smithi138:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T22:32:07.975 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-09-23T22:32:07.981 INFO:teuthology.task.internal:Enabling coredump saving... 2023-09-23T22:32:07.981 DEBUG:teuthology.orchestra.run.smithi110:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T22:32:08.009 DEBUG:teuthology.orchestra.run.smithi138:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T22:32:08.053 INFO:teuthology.orchestra.run.smithi138.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T22:32:08.064 INFO:teuthology.orchestra.run.smithi138.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T22:32:08.068 INFO:teuthology.orchestra.run.smithi110.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T22:32:08.078 INFO:teuthology.orchestra.run.smithi110.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T22:32:08.079 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-09-23T22:32:08.086 INFO:teuthology.task.internal:Configuring sudo... 2023-09-23T22:32:08.086 DEBUG:teuthology.orchestra.run.smithi110:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T22:32:08.088 DEBUG:teuthology.orchestra.run.smithi138:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T22:32:08.132 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-09-23T22:32:08.139 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-09-23T22:32:08.140 DEBUG:teuthology.orchestra.run.smithi110:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T22:32:08.155 DEBUG:teuthology.orchestra.run.smithi138:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T22:32:08.189 DEBUG:teuthology.orchestra.run.smithi110:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T22:32:08.252 DEBUG:teuthology.orchestra.run.smithi110:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T22:32:08.309 DEBUG:teuthology.orchestra.run.smithi110:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T22:32:08.391 DEBUG:teuthology.orchestra.run.smithi110:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T22:32:08.434 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-09-23T22:32:08.434 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T22:32:08.500 DEBUG:teuthology.orchestra.run.smithi138:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T22:32:08.544 DEBUG:teuthology.orchestra.run.smithi138:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T22:32:08.589 DEBUG:teuthology.orchestra.run.smithi138:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T22:32:08.673 DEBUG:teuthology.orchestra.run.smithi138:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T22:32:08.714 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-09-23T22:32:08.714 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T22:32:08.781 DEBUG:teuthology.orchestra.run.smithi110:> sudo service rsyslog restart 2023-09-23T22:32:08.783 DEBUG:teuthology.orchestra.run.smithi138:> sudo service rsyslog restart 2023-09-23T22:32:08.842 INFO:teuthology.orchestra.run.smithi110.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T22:32:08.906 INFO:teuthology.orchestra.run.smithi138.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T22:32:09.211 INFO:teuthology.run_tasks:Running task internal.timer... 2023-09-23T22:32:09.219 INFO:teuthology.task.internal:Starting timer... 2023-09-23T22:32:09.219 INFO:teuthology.run_tasks:Running task pcp... 2023-09-23T22:32:09.227 INFO:teuthology.run_tasks:Running task selinux... 2023-09-23T22:32:09.234 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-09-23T22:32:09.235 DEBUG:teuthology.orchestra.run.smithi110:> sudo service auditd rotate 2023-09-23T22:32:09.296 INFO:teuthology.orchestra.run.smithi110.stdout:Rotating logs: 2023-09-23T22:32:09.297 DEBUG:teuthology.orchestra.run.smithi138:> sudo service auditd rotate 2023-09-23T22:32:09.354 INFO:teuthology.orchestra.run.smithi138.stdout:Rotating logs: 2023-09-23T22:32:09.356 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-09-23T22:32:09.356 DEBUG:teuthology.orchestra.run.smithi110:> /usr/sbin/getenforce 2023-09-23T22:32:09.379 INFO:teuthology.orchestra.run.smithi110.stdout:Permissive 2023-09-23T22:32:09.379 DEBUG:teuthology.orchestra.run.smithi138:> /usr/sbin/getenforce 2023-09-23T22:32:09.402 INFO:teuthology.orchestra.run.smithi138.stdout:Permissive 2023-09-23T22:32:09.402 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi110.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi138.front.sepia.ceph.com': 'permissive'} 2023-09-23T22:32:09.403 DEBUG:teuthology.orchestra.run.smithi110:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T22:32:09.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T22:32:09.445 DEBUG:teuthology.orchestra.run.smithi138:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T22:32:09.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T22:32:09.473 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-09-23T22:32:09.473 DEBUG:teuthology.orchestra.run.smithi110:> sudo /usr/sbin/setenforce permissive 2023-09-23T22:32:09.511 DEBUG:teuthology.orchestra.run.smithi138:> sudo /usr/sbin/setenforce permissive 2023-09-23T22:32:09.541 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-09-23T22:32:09.550 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-09-23T22:32:09.555 INFO:teuthology.repo_utils:Fetching main from origin 2023-09-23T22:32:09.672 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-09-23T22:32:09.688 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-09-23T22:32:09.689 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi110.front.sepia.ceph.com,smithi138.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-09-23T22:40:38.958 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi110.front.sepia.ceph.com'), Remote(name='ubuntu@smithi138.front.sepia.ceph.com')] 2023-09-23T22:40:38.959 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi110.front.sepia.ceph.com' 2023-09-23T22:40:38.959 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi110.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T22:40:39.029 DEBUG:teuthology.orchestra.run.smithi110:> true 2023-09-23T22:40:39.101 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi110.front.sepia.ceph.com' 2023-09-23T22:40:39.102 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi138.front.sepia.ceph.com' 2023-09-23T22:40:39.102 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T22:40:39.170 DEBUG:teuthology.orchestra.run.smithi138:> true 2023-09-23T22:40:39.195 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi138.front.sepia.ceph.com' 2023-09-23T22:40:39.195 INFO:teuthology.run_tasks:Running task clock... 2023-09-23T22:40:39.206 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-09-23T22:40:39.206 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T22:40:39.207 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T22:40:39.209 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T22:40:39.209 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T22:40:39.239 INFO:teuthology.orchestra.run.smithi110.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T22:40:39.254 INFO:teuthology.orchestra.run.smithi110.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T22:40:39.269 INFO:teuthology.orchestra.run.smithi138.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T22:40:39.283 INFO:teuthology.orchestra.run.smithi138.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T22:40:39.323 INFO:teuthology.orchestra.run.smithi110.stderr:sudo: ntpd: command not found 2023-09-23T22:40:39.335 INFO:teuthology.orchestra.run.smithi110.stdout:506 Cannot talk to daemon 2023-09-23T22:40:39.350 INFO:teuthology.orchestra.run.smithi110.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T22:40:39.362 INFO:teuthology.orchestra.run.smithi138.stderr:sudo: ntpd: command not found 2023-09-23T22:40:39.363 INFO:teuthology.orchestra.run.smithi110.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T22:40:39.373 INFO:teuthology.orchestra.run.smithi138.stdout:506 Cannot talk to daemon 2023-09-23T22:40:39.388 INFO:teuthology.orchestra.run.smithi138.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T22:40:39.402 INFO:teuthology.orchestra.run.smithi138.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T22:40:39.439 INFO:teuthology.orchestra.run.smithi110.stderr:bash: ntpq: command not found 2023-09-23T22:40:39.441 INFO:teuthology.orchestra.run.smithi110.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T22:40:39.442 INFO:teuthology.orchestra.run.smithi110.stdout:=============================================================================== 2023-09-23T22:40:39.442 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T22:40:39.442 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T22:40:39.442 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T22:40:39.442 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T22:40:39.473 INFO:teuthology.orchestra.run.smithi138.stderr:bash: ntpq: command not found 2023-09-23T22:40:39.476 INFO:teuthology.orchestra.run.smithi138.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T22:40:39.476 INFO:teuthology.orchestra.run.smithi138.stdout:=============================================================================== 2023-09-23T22:40:39.476 INFO:teuthology.orchestra.run.smithi138.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T22:40:39.477 INFO:teuthology.orchestra.run.smithi138.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T22:40:39.477 INFO:teuthology.orchestra.run.smithi138.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T22:40:39.477 INFO:teuthology.orchestra.run.smithi138.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T22:40:39.478 INFO:teuthology.run_tasks:Running task pexec... 2023-09-23T22:40:39.488 INFO:teuthology.task.pexec:Executing custom commands... 2023-09-23T22:40:39.488 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi110.front.sepia.ceph.com 2023-09-23T22:40:39.488 DEBUG:teuthology.orchestra.run.smithi110:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T22:40:39.489 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi138.front.sepia.ceph.com 2023-09-23T22:40:39.489 DEBUG:teuthology.orchestra.run.smithi138:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T22:40:39.859 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:40:39.890 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:40:40.657 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:22 ago on Sat 23 Sep 2023 10:39:18 PM UTC. 2023-09-23T22:40:40.679 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:23 ago on Sat 23 Sep 2023 10:39:17 PM UTC. 2023-09-23T22:40:42.371 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:40:42.372 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:40:42.372 INFO:teuthology.orchestra.run.smithi110.stdout: Package Architecture Version Repository Size 2023-09-23T22:40:42.372 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:40:42.372 INFO:teuthology.orchestra.run.smithi110.stdout:Resetting modules: 2023-09-23T22:40:42.372 INFO:teuthology.orchestra.run.smithi110.stdout: container-tools 2023-09-23T22:40:42.372 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:40:42.372 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-09-23T22:40:42.372 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:40:42.372 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:40:42.399 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:40:42.400 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:40:42.400 INFO:teuthology.orchestra.run.smithi138.stdout: Package Architecture Version Repository Size 2023-09-23T22:40:42.400 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:40:42.400 INFO:teuthology.orchestra.run.smithi138.stdout:Resetting modules: 2023-09-23T22:40:42.400 INFO:teuthology.orchestra.run.smithi138.stdout: container-tools 2023-09-23T22:40:42.401 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:40:42.401 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-09-23T22:40:42.401 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:40:42.401 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:40:43.361 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:25 ago on Sat 23 Sep 2023 10:39:17 PM UTC. 2023-09-23T22:40:43.368 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:24 ago on Sat 23 Sep 2023 10:39:18 PM UTC. 2023-09-23T22:40:46.041 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:40:46.049 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:40:46.497 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:40:46.504 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:40:47.295 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:29 ago on Sat 23 Sep 2023 10:39:17 PM UTC. 2023-09-23T22:40:47.303 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:28 ago on Sat 23 Sep 2023 10:39:18 PM UTC. 2023-09-23T22:40:49.013 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:40:49.106 INFO:teuthology.orchestra.run.smithi110.stdout:========================================================================================================================== 2023-09-23T22:40:49.106 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-09-23T22:40:49.106 INFO:teuthology.orchestra.run.smithi110.stdout:========================================================================================================================== 2023-09-23T22:40:49.106 INFO:teuthology.orchestra.run.smithi110.stdout:Installing group/module packages: 2023-09-23T22:40:49.106 INFO:teuthology.orchestra.run.smithi110.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-09-23T22:40:49.107 INFO:teuthology.orchestra.run.smithi110.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-09-23T22:40:49.107 INFO:teuthology.orchestra.run.smithi110.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-09-23T22:40:49.107 INFO:teuthology.orchestra.run.smithi110.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-09-23T22:40:49.107 INFO:teuthology.orchestra.run.smithi110.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T22:40:49.107 INFO:teuthology.orchestra.run.smithi110.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T22:40:49.107 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-09-23T22:40:49.107 INFO:teuthology.orchestra.run.smithi110.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T22:40:49.107 INFO:teuthology.orchestra.run.smithi110.stdout:Downgrading: 2023-09-23T22:40:49.107 INFO:teuthology.orchestra.run.smithi110.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T22:40:49.107 INFO:teuthology.orchestra.run.smithi110.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-09-23T22:40:49.107 INFO:teuthology.orchestra.run.smithi110.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-09-23T22:40:49.108 INFO:teuthology.orchestra.run.smithi110.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-09-23T22:40:49.108 INFO:teuthology.orchestra.run.smithi110.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-09-23T22:40:49.108 INFO:teuthology.orchestra.run.smithi110.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-09-23T22:40:49.108 INFO:teuthology.orchestra.run.smithi110.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-09-23T22:40:49.108 INFO:teuthology.orchestra.run.smithi110.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-09-23T22:40:49.108 INFO:teuthology.orchestra.run.smithi110.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-09-23T22:40:49.108 INFO:teuthology.orchestra.run.smithi110.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-09-23T22:40:49.108 INFO:teuthology.orchestra.run.smithi110.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-09-23T22:40:49.108 INFO:teuthology.orchestra.run.smithi110.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T22:40:49.108 INFO:teuthology.orchestra.run.smithi110.stdout:Installing module profiles: 2023-09-23T22:40:49.108 INFO:teuthology.orchestra.run.smithi110.stdout: container-tools/common 2023-09-23T22:40:49.108 INFO:teuthology.orchestra.run.smithi110.stdout:Enabling module streams: 2023-09-23T22:40:49.108 INFO:teuthology.orchestra.run.smithi110.stdout: container-tools 3.0 2023-09-23T22:40:49.109 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:40:49.109 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-09-23T22:40:49.109 INFO:teuthology.orchestra.run.smithi110.stdout:========================================================================================================================== 2023-09-23T22:40:49.109 INFO:teuthology.orchestra.run.smithi110.stdout:Install 7 Packages 2023-09-23T22:40:49.109 INFO:teuthology.orchestra.run.smithi110.stdout:Downgrade 12 Packages 2023-09-23T22:40:49.109 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:40:49.109 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 51 M 2023-09-23T22:40:49.109 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-09-23T22:40:49.120 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:40:49.169 INFO:teuthology.orchestra.run.smithi110.stdout:(1/19): containers-common-1.2.4-1.module+el8.6. 1.5 MB/s | 91 kB 00:00 2023-09-23T22:40:49.185 INFO:teuthology.orchestra.run.smithi110.stdout:(2/19): container-selinux-2.178.0-2.module+el8. 701 kB/s | 53 kB 00:00 2023-09-23T22:40:49.202 INFO:teuthology.orchestra.run.smithi110.stdout:(3/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 556 kB/s | 51 kB 00:00 2023-09-23T22:40:49.210 INFO:teuthology.orchestra.run.smithi138.stdout:========================================================================================================================== 2023-09-23T22:40:49.210 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-09-23T22:40:49.210 INFO:teuthology.orchestra.run.smithi138.stdout:========================================================================================================================== 2023-09-23T22:40:49.210 INFO:teuthology.orchestra.run.smithi138.stdout:Installing group/module packages: 2023-09-23T22:40:49.211 INFO:teuthology.orchestra.run.smithi138.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-09-23T22:40:49.211 INFO:teuthology.orchestra.run.smithi138.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-09-23T22:40:49.211 INFO:teuthology.orchestra.run.smithi138.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-09-23T22:40:49.211 INFO:teuthology.orchestra.run.smithi138.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-09-23T22:40:49.211 INFO:teuthology.orchestra.run.smithi138.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T22:40:49.211 INFO:teuthology.orchestra.run.smithi138.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T22:40:49.211 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-09-23T22:40:49.211 INFO:teuthology.orchestra.run.smithi138.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T22:40:49.211 INFO:teuthology.orchestra.run.smithi138.stdout:Downgrading: 2023-09-23T22:40:49.211 INFO:teuthology.orchestra.run.smithi138.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T22:40:49.211 INFO:teuthology.orchestra.run.smithi138.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-09-23T22:40:49.211 INFO:teuthology.orchestra.run.smithi138.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-09-23T22:40:49.212 INFO:teuthology.orchestra.run.smithi138.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-09-23T22:40:49.212 INFO:teuthology.orchestra.run.smithi138.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-09-23T22:40:49.212 INFO:teuthology.orchestra.run.smithi138.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-09-23T22:40:49.212 INFO:teuthology.orchestra.run.smithi138.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-09-23T22:40:49.212 INFO:teuthology.orchestra.run.smithi138.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-09-23T22:40:49.212 INFO:teuthology.orchestra.run.smithi138.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-09-23T22:40:49.212 INFO:teuthology.orchestra.run.smithi138.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-09-23T22:40:49.212 INFO:teuthology.orchestra.run.smithi138.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-09-23T22:40:49.212 INFO:teuthology.orchestra.run.smithi138.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T22:40:49.212 INFO:teuthology.orchestra.run.smithi138.stdout:Installing module profiles: 2023-09-23T22:40:49.212 INFO:teuthology.orchestra.run.smithi138.stdout: container-tools/common 2023-09-23T22:40:49.212 INFO:teuthology.orchestra.run.smithi138.stdout:Enabling module streams: 2023-09-23T22:40:49.212 INFO:teuthology.orchestra.run.smithi138.stdout: container-tools 3.0 2023-09-23T22:40:49.213 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:40:49.213 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-09-23T22:40:49.213 INFO:teuthology.orchestra.run.smithi138.stdout:========================================================================================================================== 2023-09-23T22:40:49.213 INFO:teuthology.orchestra.run.smithi138.stdout:Install 7 Packages 2023-09-23T22:40:49.213 INFO:teuthology.orchestra.run.smithi138.stdout:Downgrade 12 Packages 2023-09-23T22:40:49.213 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:40:49.213 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 51 M 2023-09-23T22:40:49.213 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-09-23T22:40:49.244 INFO:teuthology.orchestra.run.smithi110.stdout:(4/19): criu-3.15-1.module+el8.6.0+14874+644362 6.6 MB/s | 511 kB 00:00 2023-09-23T22:40:49.275 INFO:teuthology.orchestra.run.smithi138.stdout:(1/19): container-selinux-2.178.0-2.module+el8. 878 kB/s | 53 kB 00:00 2023-09-23T22:40:49.278 INFO:teuthology.orchestra.run.smithi110.stdout:(5/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 915 kB/s | 69 kB 00:00 2023-09-23T22:40:49.300 INFO:teuthology.orchestra.run.smithi138.stdout:(2/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 601 kB/s | 51 kB 00:00 2023-09-23T22:40:49.303 INFO:teuthology.orchestra.run.smithi110.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 1.2 MB/s | 72 kB 00:00 2023-09-23T22:40:49.325 INFO:teuthology.orchestra.run.smithi138.stdout:(3/19): containers-common-1.2.4-1.module+el8.6. 821 kB/s | 91 kB 00:00 2023-09-23T22:40:49.337 INFO:teuthology.orchestra.run.smithi110.stdout:(7/19): podman-docker-3.0.1-9.module+el8.6.0+14 1.6 MB/s | 56 kB 00:00 2023-09-23T22:40:49.351 INFO:teuthology.orchestra.run.smithi138.stdout:(4/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 1.3 MB/s | 69 kB 00:00 2023-09-23T22:40:49.384 INFO:teuthology.orchestra.run.smithi138.stdout:(5/19): criu-3.15-1.module+el8.6.0+14874+644362 4.6 MB/s | 511 kB 00:00 2023-09-23T22:40:49.409 INFO:teuthology.orchestra.run.smithi138.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 1.2 MB/s | 72 kB 00:00 2023-09-23T22:40:49.437 INFO:teuthology.orchestra.run.smithi110.stdout:(8/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 12 MB/s | 2.9 MB 00:00 2023-09-23T22:40:49.451 INFO:teuthology.orchestra.run.smithi138.stdout:(7/19): podman-docker-3.0.1-9.module+el8.6.0+14 1.3 MB/s | 56 kB 00:00 2023-09-23T22:40:49.462 INFO:teuthology.orchestra.run.smithi110.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 407 kB/s | 51 kB 00:00 2023-09-23T22:40:49.504 INFO:teuthology.orchestra.run.smithi110.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 7.5 MB/s | 322 kB 00:00 2023-09-23T22:40:49.518 INFO:teuthology.orchestra.run.smithi138.stdout:(8/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 15 MB/s | 2.9 MB 00:00 2023-09-23T22:40:49.543 INFO:teuthology.orchestra.run.smithi138.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 554 kB/s | 51 kB 00:00 2023-09-23T22:40:49.546 INFO:teuthology.orchestra.run.smithi110.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 979 kB/s | 41 kB 00:00 2023-09-23T22:40:49.594 INFO:teuthology.orchestra.run.smithi138.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 6.3 MB/s | 322 kB 00:00 2023-09-23T22:40:49.627 INFO:teuthology.orchestra.run.smithi138.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 1.2 MB/s | 41 kB 00:00 2023-09-23T22:40:49.638 INFO:teuthology.orchestra.run.smithi110.stdout:(12/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 24 MB/s | 2.2 MB 00:00 2023-09-23T22:40:49.719 INFO:teuthology.orchestra.run.smithi138.stdout:(12/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 24 MB/s | 2.2 MB 00:00 2023-09-23T22:40:49.789 INFO:teuthology.orchestra.run.smithi110.stdout:(13/19): podman-3.0.1-9.module+el8.6.0+14874+64 21 MB/s | 11 MB 00:00 2023-09-23T22:40:49.862 INFO:teuthology.orchestra.run.smithi138.stdout:(13/19): podman-3.0.1-9.module+el8.6.0+14874+64 23 MB/s | 11 MB 00:00 2023-09-23T22:40:49.948 INFO:teuthology.orchestra.run.smithi110.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 21 MB/s | 6.4 MB 00:00 2023-09-23T22:40:50.020 INFO:teuthology.orchestra.run.smithi138.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 21 MB/s | 6.4 MB 00:00 2023-09-23T22:40:50.064 INFO:teuthology.orchestra.run.smithi110.stdout:(15/19): buildah-1.19.9-3.module+el8.6.0+14874+ 25 MB/s | 6.8 MB 00:00 2023-09-23T22:40:50.106 INFO:teuthology.orchestra.run.smithi110.stdout:(16/19): cockpit-podman-29-2.module+el8.6.0+148 6.7 MB/s | 1.1 MB 00:00 2023-09-23T22:40:50.120 INFO:teuthology.orchestra.run.smithi138.stdout:(15/19): buildah-1.19.9-3.module+el8.6.0+14874+ 26 MB/s | 6.8 MB 00:00 2023-09-23T22:40:50.123 INFO:teuthology.orchestra.run.smithi110.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 3.1 MB/s | 185 kB 00:00 2023-09-23T22:40:50.163 INFO:teuthology.orchestra.run.smithi138.stdout:(16/19): cockpit-podman-29-2.module+el8.6.0+148 7.5 MB/s | 1.1 MB 00:00 2023-09-23T22:40:50.173 INFO:teuthology.orchestra.run.smithi110.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 756 kB/s | 51 kB 00:00 2023-09-23T22:40:50.188 INFO:teuthology.orchestra.run.smithi138.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 2.7 MB/s | 185 kB 00:00 2023-09-23T22:40:50.204 INFO:teuthology.orchestra.run.smithi138.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 1.2 MB/s | 51 kB 00:00 2023-09-23T22:40:50.357 INFO:teuthology.orchestra.run.smithi110.stdout:(19/19): containernetworking-plugins-0.9.1-1.mo 21 MB/s | 19 MB 00:00 2023-09-23T22:40:50.358 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:40:50.358 INFO:teuthology.orchestra.run.smithi110.stdout:Total 41 MB/s | 51 MB 00:01 2023-09-23T22:40:50.396 INFO:teuthology.orchestra.run.smithi138.stdout:(19/19): containernetworking-plugins-0.9.1-1.mo 22 MB/s | 19 MB 00:00 2023-09-23T22:40:50.397 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:40:50.397 INFO:teuthology.orchestra.run.smithi138.stdout:Total 43 MB/s | 51 MB 00:01 2023-09-23T22:40:50.835 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-09-23T22:40:50.865 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-09-23T22:40:50.878 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-09-23T22:40:50.878 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-09-23T22:40:50.907 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-09-23T22:40:50.907 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-09-23T22:40:51.316 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-09-23T22:40:51.320 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-09-23T22:40:51.336 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-09-23T22:40:51.340 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-09-23T22:40:52.504 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-09-23T22:40:52.511 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-09-23T22:40:52.559 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-09-23T22:40:52.560 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-09-23T22:40:52.741 INFO:teuthology.orchestra.run.smithi110.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T22:40:52.750 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T22:40:52.765 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T22:40:52.767 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T22:40:52.775 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T22:40:52.795 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T22:40:52.973 INFO:teuthology.orchestra.run.smithi110.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T22:40:52.998 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T22:41:09.875 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T22:41:09.890 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T22:41:11.663 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-09-23T22:41:11.688 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-09-23T22:41:11.895 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-09-23T22:41:11.930 INFO:teuthology.orchestra.run.smithi110.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-09-23T22:41:12.082 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-09-23T22:41:12.126 INFO:teuthology.orchestra.run.smithi110.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-09-23T22:41:12.260 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-09-23T22:41:12.295 INFO:teuthology.orchestra.run.smithi110.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-09-23T22:41:12.447 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-09-23T22:41:12.447 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-09-23T22:41:12.447 INFO:teuthology.orchestra.run.smithi138.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-09-23T22:41:12.447 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:41:12.488 INFO:teuthology.orchestra.run.smithi110.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-09-23T22:41:12.488 INFO:teuthology.orchestra.run.smithi110.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-09-23T22:41:12.488 INFO:teuthology.orchestra.run.smithi110.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-09-23T22:41:12.488 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:41:13.288 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-09-23T22:41:13.378 INFO:teuthology.orchestra.run.smithi110.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-09-23T22:41:13.512 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-09-23T22:41:13.613 INFO:teuthology.orchestra.run.smithi110.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-09-23T22:41:14.975 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-09-23T22:41:15.108 INFO:teuthology.orchestra.run.smithi110.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-09-23T22:41:15.280 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-09-23T22:41:15.448 INFO:teuthology.orchestra.run.smithi110.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-09-23T22:41:15.662 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-09-23T22:41:15.848 INFO:teuthology.orchestra.run.smithi110.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-09-23T22:41:15.935 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-09-23T22:41:16.161 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-09-23T22:41:16.661 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-09-23T22:41:16.886 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-09-23T22:41:17.391 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-09-23T22:41:17.590 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-09-23T22:41:17.648 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-09-23T22:41:17.836 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-09-23T22:41:17.852 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-09-23T22:41:18.053 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-09-23T22:41:18.065 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-09-23T22:41:18.192 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-09-23T22:41:18.193 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T22:41:18.254 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-09-23T22:41:18.426 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-09-23T22:41:18.427 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T22:41:18.497 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T22:41:18.662 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-09-23T22:41:18.663 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T22:41:18.746 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T22:41:18.812 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T22:41:18.929 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-09-23T22:41:18.929 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T22:41:18.978 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T22:41:19.115 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T22:41:19.163 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-09-23T22:41:19.238 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T22:41:19.337 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-09-23T22:41:19.405 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-09-23T22:41:19.529 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-09-23T22:41:19.588 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-09-23T22:41:19.687 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-09-23T22:41:19.762 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-09-23T22:41:19.836 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-09-23T22:41:19.929 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-09-23T22:41:20.006 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-09-23T22:41:20.007 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T22:41:20.079 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-09-23T22:41:20.178 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T22:41:20.248 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-09-23T22:41:20.248 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T22:41:20.437 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T22:41:20.710 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-09-23T22:41:20.966 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-09-23T22:41:21.072 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T22:41:21.072 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-09-23T22:41:21.073 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-09-23T22:41:21.073 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-09-23T22:41:21.073 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-09-23T22:41:21.073 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-09-23T22:41:21.073 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-09-23T22:41:21.073 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-09-23T22:41:21.073 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-09-23T22:41:21.073 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-09-23T22:41:21.073 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-09-23T22:41:21.073 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-09-23T22:41:21.073 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-09-23T22:41:21.073 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-09-23T22:41:21.073 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-09-23T22:41:21.074 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-09-23T22:41:21.074 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-09-23T22:41:21.074 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-09-23T22:41:21.074 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-09-23T22:41:21.074 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-09-23T22:41:21.074 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-09-23T22:41:21.074 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-09-23T22:41:21.074 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-09-23T22:41:21.074 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-09-23T22:41:21.074 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T22:41:21.074 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-09-23T22:41:21.074 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-09-23T22:41:21.075 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-09-23T22:41:21.075 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-09-23T22:41:21.075 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-09-23T22:41:21.075 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-09-23T22:41:21.382 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T22:41:21.382 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-09-23T22:41:21.382 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-09-23T22:41:21.382 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-09-23T22:41:21.382 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-09-23T22:41:21.382 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-09-23T22:41:21.383 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-09-23T22:41:21.383 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-09-23T22:41:21.383 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-09-23T22:41:21.383 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-09-23T22:41:21.383 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-09-23T22:41:21.383 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-09-23T22:41:21.383 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-09-23T22:41:21.383 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-09-23T22:41:21.383 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-09-23T22:41:21.383 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-09-23T22:41:21.383 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-09-23T22:41:21.383 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-09-23T22:41:21.384 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-09-23T22:41:21.384 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-09-23T22:41:21.384 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-09-23T22:41:21.384 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-09-23T22:41:21.384 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-09-23T22:41:21.384 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-09-23T22:41:21.384 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T22:41:21.384 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-09-23T22:41:21.384 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-09-23T22:41:21.384 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-09-23T22:41:21.384 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-09-23T22:41:21.384 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-09-23T22:41:21.385 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-09-23T22:41:22.493 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-09-23T22:41:22.493 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:05 ago on Sat 23 Sep 2023 10:39:17 PM UTC. 2023-09-23T22:41:22.859 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-09-23T22:41:22.860 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:04 ago on Sat 23 Sep 2023 10:39:18 PM UTC. 2023-09-23T22:41:25.188 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-09-23T22:41:25.383 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:41:25.384 INFO:teuthology.orchestra.run.smithi138.stdout:Downgraded: 2023-09-23T22:41:25.384 INFO:teuthology.orchestra.run.smithi138.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.384 INFO:teuthology.orchestra.run.smithi138.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T22:41:25.384 INFO:teuthology.orchestra.run.smithi138.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.384 INFO:teuthology.orchestra.run.smithi138.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.384 INFO:teuthology.orchestra.run.smithi138.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.384 INFO:teuthology.orchestra.run.smithi138.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.384 INFO:teuthology.orchestra.run.smithi138.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.384 INFO:teuthology.orchestra.run.smithi138.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.384 INFO:teuthology.orchestra.run.smithi138.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.384 INFO:teuthology.orchestra.run.smithi138.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-09-23T22:41:25.385 INFO:teuthology.orchestra.run.smithi138.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.385 INFO:teuthology.orchestra.run.smithi138.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.385 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-09-23T22:41:25.385 INFO:teuthology.orchestra.run.smithi138.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.385 INFO:teuthology.orchestra.run.smithi138.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T22:41:25.385 INFO:teuthology.orchestra.run.smithi138.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.385 INFO:teuthology.orchestra.run.smithi138.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.385 INFO:teuthology.orchestra.run.smithi138.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.385 INFO:teuthology.orchestra.run.smithi138.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-09-23T22:41:25.385 INFO:teuthology.orchestra.run.smithi138.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T22:41:25.385 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:41:25.385 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:41:25.423 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-09-23T22:41:25.567 DEBUG:teuthology.parallel:result is None 2023-09-23T22:41:25.621 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:41:25.621 INFO:teuthology.orchestra.run.smithi110.stdout:Downgraded: 2023-09-23T22:41:25.621 INFO:teuthology.orchestra.run.smithi110.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.621 INFO:teuthology.orchestra.run.smithi110.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T22:41:25.621 INFO:teuthology.orchestra.run.smithi110.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.621 INFO:teuthology.orchestra.run.smithi110.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.622 INFO:teuthology.orchestra.run.smithi110.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.622 INFO:teuthology.orchestra.run.smithi110.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.622 INFO:teuthology.orchestra.run.smithi110.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.622 INFO:teuthology.orchestra.run.smithi110.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.622 INFO:teuthology.orchestra.run.smithi110.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.622 INFO:teuthology.orchestra.run.smithi110.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-09-23T22:41:25.622 INFO:teuthology.orchestra.run.smithi110.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.622 INFO:teuthology.orchestra.run.smithi110.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.622 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-09-23T22:41:25.622 INFO:teuthology.orchestra.run.smithi110.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.622 INFO:teuthology.orchestra.run.smithi110.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T22:41:25.623 INFO:teuthology.orchestra.run.smithi110.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.623 INFO:teuthology.orchestra.run.smithi110.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.623 INFO:teuthology.orchestra.run.smithi110.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:41:25.623 INFO:teuthology.orchestra.run.smithi110.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-09-23T22:41:25.623 INFO:teuthology.orchestra.run.smithi110.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T22:41:25.623 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:41:25.623 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:41:25.803 DEBUG:teuthology.parallel:result is None 2023-09-23T22:41:25.804 INFO:teuthology.run_tasks:Running task install... 2023-09-23T22:41:25.813 DEBUG:teuthology.task.install:project ceph 2023-09-23T22:41:25.813 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T22:41:25.813 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T22:41:25.814 INFO:teuthology.task.install:Using flavor: default 2023-09-23T22:41:25.819 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-09-23T22:41:25.820 INFO:teuthology.task.install:extra packages: [] 2023-09-23T22:41:25.820 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T22:41:25.820 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T22:41:25.820 INFO:teuthology.packaging:ref: None 2023-09-23T22:41:25.820 INFO:teuthology.packaging:tag: None 2023-09-23T22:41:25.820 INFO:teuthology.packaging:branch: quincy 2023-09-23T22:41:25.821 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:41:25.821 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T22:41:25.823 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T22:41:25.823 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T22:41:25.823 INFO:teuthology.packaging:ref: None 2023-09-23T22:41:25.823 INFO:teuthology.packaging:tag: None 2023-09-23T22:41:25.823 INFO:teuthology.packaging:branch: quincy 2023-09-23T22:41:25.823 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:41:25.823 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T22:41:26.078 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f/centos/8/flavors/default/ 2023-09-23T22:41:26.078 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1125.gc9bb1bcf 2023-09-23T22:41:26.084 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f/centos/8/flavors/default/ 2023-09-23T22:41:26.084 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1125.gc9bb1bcf 2023-09-23T22:41:26.190 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T22:41:26.190 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-09-23T22:41:26.191 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T22:41:26.195 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T22:41:26.196 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-09-23T22:41:26.196 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T22:41:26.219 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T22:41:26.220 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T22:41:26.220 INFO:teuthology.packaging:ref: None 2023-09-23T22:41:26.220 INFO:teuthology.packaging:tag: None 2023-09-23T22:41:26.220 INFO:teuthology.packaging:branch: quincy 2023-09-23T22:41:26.220 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:41:26.220 DEBUG:teuthology.orchestra.run.smithi138:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T22:41:26.223 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T22:41:26.223 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T22:41:26.223 INFO:teuthology.packaging:ref: None 2023-09-23T22:41:26.223 INFO:teuthology.packaging:tag: None 2023-09-23T22:41:26.223 INFO:teuthology.packaging:branch: quincy 2023-09-23T22:41:26.223 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:41:26.223 DEBUG:teuthology.orchestra.run.smithi110:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T22:41:26.289 DEBUG:teuthology.orchestra.run.smithi138:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T22:41:26.290 DEBUG:teuthology.orchestra.run.smithi110:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T22:41:26.367 DEBUG:teuthology.orchestra.run.smithi110:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T22:41:26.368 DEBUG:teuthology.orchestra.run.smithi138:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T22:41:26.433 INFO:teuthology.orchestra.run.smithi110.stdout:check_obsoletes = 1 2023-09-23T22:41:26.434 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum clean all 2023-09-23T22:41:26.435 INFO:teuthology.orchestra.run.smithi138.stdout:check_obsoletes = 1 2023-09-23T22:41:26.436 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum clean all 2023-09-23T22:41:26.803 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:41:26.806 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:41:27.072 INFO:teuthology.orchestra.run.smithi110.stdout:44 files removed 2023-09-23T22:41:27.080 INFO:teuthology.orchestra.run.smithi138.stdout:44 files removed 2023-09-23T22:41:27.119 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-radosgw 2023-09-23T22:41:27.127 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-radosgw 2023-09-23T22:41:27.449 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:41:27.455 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:41:28.188 INFO:teuthology.orchestra.run.smithi110.stdout:ceph packages for x86_64 206 kB/s | 78 kB 00:00 2023-09-23T22:41:28.195 INFO:teuthology.orchestra.run.smithi138.stdout:ceph packages for x86_64 207 kB/s | 78 kB 00:00 2023-09-23T22:41:28.619 INFO:teuthology.orchestra.run.smithi110.stdout:ceph noarch packages 23 kB/s | 9.1 kB 00:00 2023-09-23T22:41:28.704 INFO:teuthology.orchestra.run.smithi138.stdout:ceph noarch packages 19 kB/s | 9.1 kB 00:00 2023-09-23T22:41:28.880 INFO:teuthology.orchestra.run.smithi110.stdout:ceph source packages 6.0 kB/s | 1.4 kB 00:00 2023-09-23T22:41:29.106 INFO:teuthology.orchestra.run.smithi110.stdout:Copr repo for python3-asyncssh owned by ceph 17 kB/s | 3.5 kB 00:00 2023-09-23T22:41:29.113 INFO:teuthology.orchestra.run.smithi138.stdout:ceph source packages 3.7 kB/s | 1.4 kB 00:00 2023-09-23T22:41:29.297 INFO:teuthology.orchestra.run.smithi138.stdout:Copr repo for python3-asyncssh owned by ceph 21 kB/s | 3.5 kB 00:00 2023-09-23T22:41:30.898 INFO:teuthology.orchestra.run.smithi138.stdout:Extra Packages for Enterprise Linux 10 MB/s | 16 MB 00:01 2023-09-23T22:41:34.456 INFO:teuthology.orchestra.run.smithi138.stdout:lab-extras 371 kB/s | 24 kB 00:00 2023-09-23T22:41:35.433 INFO:teuthology.orchestra.run.smithi138.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 59 MB/s | 53 MB 00:00 2023-09-23T22:41:37.200 INFO:teuthology.orchestra.run.smithi110.stdout:Extra Packages for Enterprise Linux 2.0 MB/s | 16 MB 00:08 2023-09-23T22:41:40.748 INFO:teuthology.orchestra.run.smithi110.stdout:lab-extras 553 kB/s | 24 kB 00:00 2023-09-23T22:41:41.675 INFO:teuthology.orchestra.run.smithi110.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 62 MB/s | 53 MB 00:00 2023-09-23T22:41:43.592 INFO:teuthology.orchestra.run.smithi138.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 56 MB/s | 47 MB 00:00 2023-09-23T22:41:49.810 INFO:teuthology.orchestra.run.smithi110.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 55 MB/s | 47 MB 00:00 2023-09-23T22:41:54.662 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:41:54.664 INFO:teuthology.orchestra.run.smithi138.stdout:==================================================================================================== 2023-09-23T22:41:54.664 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-09-23T22:41:54.665 INFO:teuthology.orchestra.run.smithi138.stdout:==================================================================================================== 2023-09-23T22:41:54.665 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-09-23T22:41:54.665 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-radosgw x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 17 M 2023-09-23T22:41:54.665 INFO:teuthology.orchestra.run.smithi138.stdout:Upgrading: 2023-09-23T22:41:54.665 INFO:teuthology.orchestra.run.smithi138.stdout: librados2 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 3.5 M 2023-09-23T22:41:54.665 INFO:teuthology.orchestra.run.smithi138.stdout: librbd1 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 3.6 M 2023-09-23T22:41:54.665 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-09-23T22:41:54.665 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-base x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 5.2 M 2023-09-23T22:41:54.665 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-common x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 23 M 2023-09-23T22:41:54.665 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-selinux x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 24 k 2023-09-23T22:41:54.665 INFO:teuthology.orchestra.run.smithi138.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T22:41:54.666 INFO:teuthology.orchestra.run.smithi138.stdout: libcephfs2 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 795 k 2023-09-23T22:41:54.666 INFO:teuthology.orchestra.run.smithi138.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T22:41:54.666 INFO:teuthology.orchestra.run.smithi138.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T22:41:54.666 INFO:teuthology.orchestra.run.smithi138.stdout: libradosstriper1 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 506 k 2023-09-23T22:41:54.666 INFO:teuthology.orchestra.run.smithi138.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T22:41:54.666 INFO:teuthology.orchestra.run.smithi138.stdout: librgw2 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 5.6 M 2023-09-23T22:41:54.666 INFO:teuthology.orchestra.run.smithi138.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T22:41:54.666 INFO:teuthology.orchestra.run.smithi138.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T22:41:54.666 INFO:teuthology.orchestra.run.smithi138.stdout: python3-ceph-argparse x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 45 k 2023-09-23T22:41:54.666 INFO:teuthology.orchestra.run.smithi138.stdout: python3-ceph-common x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 115 k 2023-09-23T22:41:54.666 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cephfs x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 190 k 2023-09-23T22:41:54.667 INFO:teuthology.orchestra.run.smithi138.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T22:41:54.667 INFO:teuthology.orchestra.run.smithi138.stdout: python3-rados x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 372 k 2023-09-23T22:41:54.667 INFO:teuthology.orchestra.run.smithi138.stdout: python3-rbd x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 358 k 2023-09-23T22:41:54.667 INFO:teuthology.orchestra.run.smithi138.stdout: python3-rgw x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 106 k 2023-09-23T22:41:54.667 INFO:teuthology.orchestra.run.smithi138.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-23T22:41:54.667 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:41:54.667 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-09-23T22:41:54.667 INFO:teuthology.orchestra.run.smithi138.stdout:==================================================================================================== 2023-09-23T22:41:54.667 INFO:teuthology.orchestra.run.smithi138.stdout:Install 21 Packages 2023-09-23T22:41:54.667 INFO:teuthology.orchestra.run.smithi138.stdout:Upgrade 2 Packages 2023-09-23T22:41:54.667 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:41:54.668 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 63 M 2023-09-23T22:41:54.668 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-09-23T22:41:56.243 INFO:teuthology.orchestra.run.smithi138.stdout:(1/23): ceph-base-17.2.6-1125.gc9bb1bcf.el8.x86 3.8 MB/s | 5.2 MB 00:01 2023-09-23T22:41:56.335 INFO:teuthology.orchestra.run.smithi138.stdout:(2/23): ceph-selinux-17.2.6-1125.gc9bb1bcf.el8. 266 kB/s | 24 kB 00:00 2023-09-23T22:41:56.477 INFO:teuthology.orchestra.run.smithi138.stdout:(3/23): libcephfs2-17.2.6-1125.gc9bb1bcf.el8.x8 5.5 MB/s | 795 kB 00:00 2023-09-23T22:41:56.635 INFO:teuthology.orchestra.run.smithi138.stdout:(4/23): libradosstriper1-17.2.6-1125.gc9bb1bcf. 3.1 MB/s | 506 kB 00:00 2023-09-23T22:41:57.502 INFO:teuthology.orchestra.run.smithi138.stdout:(5/23): librgw2-17.2.6-1125.gc9bb1bcf.el8.x86_6 6.4 MB/s | 5.6 MB 00:00 2023-09-23T22:41:57.669 INFO:teuthology.orchestra.run.smithi138.stdout:(6/23): ceph-radosgw-17.2.6-1125.gc9bb1bcf.el8. 6.0 MB/s | 17 MB 00:02 2023-09-23T22:41:57.694 INFO:teuthology.orchestra.run.smithi138.stdout:(7/23): python3-ceph-argparse-17.2.6-1125.gc9bb 232 kB/s | 45 kB 00:00 2023-09-23T22:41:57.765 INFO:teuthology.orchestra.run.smithi138.stdout:(8/23): python3-ceph-common-17.2.6-1125.gc9bb1b 1.2 MB/s | 115 kB 00:00 2023-09-23T22:41:57.795 INFO:teuthology.orchestra.run.smithi138.stdout:(9/23): python3-cephfs-17.2.6-1125.gc9bb1bcf.el 1.8 MB/s | 190 kB 00:00 2023-09-23T22:41:57.862 INFO:teuthology.orchestra.run.smithi138.stdout:(10/23): python3-rados-17.2.6-1125.gc9bb1bcf.el 3.8 MB/s | 372 kB 00:00 2023-09-23T22:41:58.087 INFO:teuthology.orchestra.run.smithi138.stdout:(11/23): ceph-common-17.2.6-1125.gc9bb1bcf.el8. 7.3 MB/s | 23 MB 00:03 2023-09-23T22:41:58.113 INFO:teuthology.orchestra.run.smithi138.stdout:(12/23): python3-rgw-17.2.6-1125.gc9bb1bcf.el8. 425 kB/s | 106 kB 00:00 2023-09-23T22:41:58.138 INFO:teuthology.orchestra.run.smithi138.stdout:(13/23): python3-rbd-17.2.6-1125.gc9bb1bcf.el8. 1.0 MB/s | 358 kB 00:00 2023-09-23T22:41:58.372 INFO:teuthology.orchestra.run.smithi138.stdout:(14/23): liboath-2.6.2-3.el8.x86_64.rpm 227 kB/s | 59 kB 00:00 2023-09-23T22:41:58.397 INFO:teuthology.orchestra.run.smithi138.stdout:(15/23): libunwind-1.3.1-3.el8.x86_64.rpm 290 kB/s | 75 kB 00:00 2023-09-23T22:41:58.447 INFO:teuthology.orchestra.run.smithi138.stdout:(16/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 939 kB/s | 47 kB 00:00 2023-09-23T22:41:58.473 INFO:teuthology.orchestra.run.smithi138.stdout:(17/23): gperftools-libs-2.7-9.el8.x86_64.rpm 795 kB/s | 306 kB 00:00 2023-09-23T22:41:58.498 INFO:teuthology.orchestra.run.smithi138.stdout:(18/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 5.0 MB/s | 259 kB 00:00 2023-09-23T22:41:58.548 INFO:teuthology.orchestra.run.smithi138.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 585 kB/s | 44 kB 00:00 2023-09-23T22:41:58.574 INFO:teuthology.orchestra.run.smithi138.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 5.0 MB/s | 354 kB 00:00 2023-09-23T22:41:58.682 INFO:teuthology.orchestra.run.smithi138.stdout:(21/23): thrift-0.13.0-2.el8.x86_64.rpm 5.6 MB/s | 1.7 MB 00:00 2023-09-23T22:41:59.241 INFO:teuthology.orchestra.run.smithi138.stdout:(22/23): librbd1-17.2.6-1125.gc9bb1bcf.el8.x86_ 5.4 MB/s | 3.6 MB 00:00 2023-09-23T22:41:59.641 INFO:teuthology.orchestra.run.smithi138.stdout:(23/23): librados2-17.2.6-1125.gc9bb1bcf.el8.x8 3.2 MB/s | 3.5 MB 00:01 2023-09-23T22:41:59.646 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:41:59.646 INFO:teuthology.orchestra.run.smithi138.stdout:Total 13 MB/s | 63 MB 00:04 2023-09-23T22:41:59.692 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-09-23T22:41:59.737 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-09-23T22:41:59.737 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-09-23T22:42:00.081 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-09-23T22:42:00.084 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-09-23T22:42:00.704 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:42:00.707 INFO:teuthology.orchestra.run.smithi110.stdout:==================================================================================================== 2023-09-23T22:42:00.707 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-09-23T22:42:00.707 INFO:teuthology.orchestra.run.smithi110.stdout:==================================================================================================== 2023-09-23T22:42:00.707 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-09-23T22:42:00.707 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-radosgw x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 17 M 2023-09-23T22:42:00.707 INFO:teuthology.orchestra.run.smithi110.stdout:Upgrading: 2023-09-23T22:42:00.708 INFO:teuthology.orchestra.run.smithi110.stdout: librados2 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 3.5 M 2023-09-23T22:42:00.708 INFO:teuthology.orchestra.run.smithi110.stdout: librbd1 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 3.6 M 2023-09-23T22:42:00.708 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-09-23T22:42:00.708 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-base x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 5.2 M 2023-09-23T22:42:00.708 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-common x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 23 M 2023-09-23T22:42:00.708 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-selinux x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 24 k 2023-09-23T22:42:00.708 INFO:teuthology.orchestra.run.smithi110.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T22:42:00.708 INFO:teuthology.orchestra.run.smithi110.stdout: libcephfs2 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 795 k 2023-09-23T22:42:00.708 INFO:teuthology.orchestra.run.smithi110.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T22:42:00.708 INFO:teuthology.orchestra.run.smithi110.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T22:42:00.708 INFO:teuthology.orchestra.run.smithi110.stdout: libradosstriper1 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 506 k 2023-09-23T22:42:00.708 INFO:teuthology.orchestra.run.smithi110.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T22:42:00.708 INFO:teuthology.orchestra.run.smithi110.stdout: librgw2 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 5.6 M 2023-09-23T22:42:00.709 INFO:teuthology.orchestra.run.smithi110.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T22:42:00.709 INFO:teuthology.orchestra.run.smithi110.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T22:42:00.709 INFO:teuthology.orchestra.run.smithi110.stdout: python3-ceph-argparse x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 45 k 2023-09-23T22:42:00.709 INFO:teuthology.orchestra.run.smithi110.stdout: python3-ceph-common x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 115 k 2023-09-23T22:42:00.709 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cephfs x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 190 k 2023-09-23T22:42:00.709 INFO:teuthology.orchestra.run.smithi110.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T22:42:00.709 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rados x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 372 k 2023-09-23T22:42:00.709 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rbd x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 358 k 2023-09-23T22:42:00.709 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rgw x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 106 k 2023-09-23T22:42:00.709 INFO:teuthology.orchestra.run.smithi110.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-23T22:42:00.709 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:42:00.709 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-09-23T22:42:00.709 INFO:teuthology.orchestra.run.smithi110.stdout:==================================================================================================== 2023-09-23T22:42:00.709 INFO:teuthology.orchestra.run.smithi110.stdout:Install 21 Packages 2023-09-23T22:42:00.710 INFO:teuthology.orchestra.run.smithi110.stdout:Upgrade 2 Packages 2023-09-23T22:42:00.710 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:42:00.710 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 63 M 2023-09-23T22:42:00.710 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-09-23T22:42:01.140 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-09-23T22:42:01.454 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T22:42:01.544 INFO:teuthology.orchestra.run.smithi110.stdout:(1/23): ceph-base-17.2.6-1125.gc9bb1bcf.el8.x86 6.6 MB/s | 5.2 MB 00:00 2023-09-23T22:42:01.611 INFO:teuthology.orchestra.run.smithi110.stdout:(2/23): ceph-selinux-17.2.6-1125.gc9bb1bcf.el8. 366 kB/s | 24 kB 00:00 2023-09-23T22:42:01.778 INFO:teuthology.orchestra.run.smithi110.stdout:(3/23): libcephfs2-17.2.6-1125.gc9bb1bcf.el8.x8 4.7 MB/s | 795 kB 00:00 2023-09-23T22:42:01.836 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T22:42:01.910 INFO:teuthology.orchestra.run.smithi110.stdout:(4/23): libradosstriper1-17.2.6-1125.gc9bb1bcf. 3.8 MB/s | 506 kB 00:00 2023-09-23T22:42:02.021 INFO:teuthology.orchestra.run.smithi138.stdout: Upgrading : librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/25 2023-09-23T22:42:02.098 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/25 2023-09-23T22:42:02.310 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-rados-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 3/25 2023-09-23T22:42:02.554 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T22:42:02.611 INFO:teuthology.orchestra.run.smithi110.stdout:(5/23): ceph-radosgw-17.2.6-1125.gc9bb1bcf.el8. 9.1 MB/s | 17 MB 00:01 2023-09-23T22:42:02.704 INFO:teuthology.orchestra.run.smithi110.stdout:(6/23): python3-ceph-argparse-17.2.6-1125.gc9bb 487 kB/s | 45 kB 00:00 2023-09-23T22:42:02.746 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/25 2023-09-23T22:42:02.787 INFO:teuthology.orchestra.run.smithi110.stdout:(7/23): librgw2-17.2.6-1125.gc9bb1bcf.el8.x86_6 6.3 MB/s | 5.6 MB 00:00 2023-09-23T22:42:02.812 INFO:teuthology.orchestra.run.smithi110.stdout:(8/23): python3-ceph-common-17.2.6-1125.gc9bb1b 1.0 MB/s | 115 kB 00:00 2023-09-23T22:42:02.846 INFO:teuthology.orchestra.run.smithi110.stdout:(9/23): python3-cephfs-17.2.6-1125.gc9bb1bcf.el 3.2 MB/s | 190 kB 00:00 2023-09-23T22:42:03.096 INFO:teuthology.orchestra.run.smithi110.stdout:(10/23): ceph-common-17.2.6-1125.gc9bb1bcf.el8. 10 MB/s | 23 MB 00:02 2023-09-23T22:42:03.121 INFO:teuthology.orchestra.run.smithi110.stdout:(11/23): python3-rados-17.2.6-1125.gc9bb1bcf.el 1.2 MB/s | 372 kB 00:00 2023-09-23T22:42:03.141 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/25 2023-09-23T22:42:03.147 INFO:teuthology.orchestra.run.smithi110.stdout:(12/23): python3-rbd-17.2.6-1125.gc9bb1bcf.el8. 1.2 MB/s | 358 kB 00:00 2023-09-23T22:42:03.172 INFO:teuthology.orchestra.run.smithi110.stdout:(13/23): python3-rgw-17.2.6-1125.gc9bb1bcf.el8. 1.4 MB/s | 106 kB 00:00 2023-09-23T22:42:03.329 INFO:teuthology.orchestra.run.smithi138.stdout: Upgrading : librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 6/25 2023-09-23T22:42:03.429 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 6/25 2023-09-23T22:42:03.596 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T22:42:03.623 INFO:teuthology.orchestra.run.smithi110.stdout:(14/23): liboath-2.6.2-3.el8.x86_64.rpm 124 kB/s | 59 kB 00:00 2023-09-23T22:42:03.648 INFO:teuthology.orchestra.run.smithi110.stdout:(15/23): libunwind-1.3.1-3.el8.x86_64.rpm 158 kB/s | 75 kB 00:00 2023-09-23T22:42:03.665 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T22:42:03.698 INFO:teuthology.orchestra.run.smithi110.stdout:(16/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 936 kB/s | 47 kB 00:00 2023-09-23T22:42:03.740 INFO:teuthology.orchestra.run.smithi110.stdout:(17/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 6.1 MB/s | 259 kB 00:00 2023-09-23T22:42:03.774 INFO:teuthology.orchestra.run.smithi110.stdout:(18/23): python3-prettytable-0.7.2-14.el8.noarc 1.3 MB/s | 44 kB 00:00 2023-09-23T22:42:03.799 INFO:teuthology.orchestra.run.smithi110.stdout:(19/23): gperftools-libs-2.7-9.el8.x86_64.rpm 452 kB/s | 306 kB 00:00 2023-09-23T22:42:03.866 INFO:teuthology.orchestra.run.smithi110.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 3.8 MB/s | 354 kB 00:00 2023-09-23T22:42:03.990 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-09-23T22:42:04.125 INFO:teuthology.orchestra.run.smithi110.stdout:(21/23): librados2-17.2.6-1125.gc9bb1bcf.el8.x8 11 MB/s | 3.5 MB 00:00 2023-09-23T22:42:04.167 INFO:teuthology.orchestra.run.smithi110.stdout:(22/23): thrift-0.13.0-2.el8.x86_64.rpm 3.2 MB/s | 1.7 MB 00:00 2023-09-23T22:42:04.249 INFO:teuthology.orchestra.run.smithi110.stdout:(23/23): librbd1-17.2.6-1125.gc9bb1bcf.el8.x86_ 9.4 MB/s | 3.6 MB 00:00 2023-09-23T22:42:04.253 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:42:04.254 INFO:teuthology.orchestra.run.smithi110.stdout:Total 18 MB/s | 63 MB 00:03 2023-09-23T22:42:04.298 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-09-23T22:42:04.345 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-09-23T22:42:04.345 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-09-23T22:42:04.652 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-23T22:42:04.694 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-09-23T22:42:04.698 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-09-23T22:42:04.821 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : librgw2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 10/25 2023-09-23T22:42:04.885 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: librgw2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 10/25 2023-09-23T22:42:05.006 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-ceph-argparse-2:17.2.6-1125.gc9bb1bcf.el8. 11/25 2023-09-23T22:42:05.215 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-cephfs-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 12/25 2023-09-23T22:42:05.413 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-rgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 13/25 2023-09-23T22:42:05.624 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-rbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 14/25 2023-09-23T22:42:05.773 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-09-23T22:42:05.778 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : libradosstriper1-2:17.2.6-1125.gc9bb1bcf.el8.x86_6 15/25 2023-09-23T22:42:05.849 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1125.gc9bb1bcf.el8.x86_6 15/25 2023-09-23T22:42:05.989 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T22:42:06.005 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T22:42:06.202 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T22:42:06.369 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T22:42:06.573 INFO:teuthology.orchestra.run.smithi110.stdout: Upgrading : librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/25 2023-09-23T22:42:06.641 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T22:42:06.666 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/25 2023-09-23T22:42:06.821 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x8 19/25 2023-09-23T22:42:06.849 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-rados-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 3/25 2023-09-23T22:42:07.113 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T22:42:07.331 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/25 2023-09-23T22:42:07.709 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/25 2023-09-23T22:42:07.905 INFO:teuthology.orchestra.run.smithi110.stdout: Upgrading : librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 6/25 2023-09-23T22:42:08.016 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 6/25 2023-09-23T22:42:08.222 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T22:42:08.290 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T22:42:08.630 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-09-23T22:42:09.337 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-23T22:42:09.514 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : librgw2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 10/25 2023-09-23T22:42:09.620 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librgw2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 10/25 2023-09-23T22:42:09.741 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-ceph-argparse-2:17.2.6-1125.gc9bb1bcf.el8. 11/25 2023-09-23T22:42:09.899 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cephfs-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 12/25 2023-09-23T22:42:10.114 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-rgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 13/25 2023-09-23T22:42:10.292 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-rbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 14/25 2023-09-23T22:42:10.472 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libradosstriper1-2:17.2.6-1125.gc9bb1bcf.el8.x86_6 15/25 2023-09-23T22:42:10.533 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1125.gc9bb1bcf.el8.x86_6 15/25 2023-09-23T22:42:10.541 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 20/25 2023-09-23T22:42:10.656 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T22:42:10.745 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 20/25 2023-09-23T22:42:10.885 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T22:42:11.096 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T22:42:11.319 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 20/25 2023-09-23T22:42:11.978 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 21/25 2023-09-23T22:42:12.009 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x8 19/25 2023-09-23T22:42:12.344 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 21/25 2023-09-23T22:42:12.511 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-selinux-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 22/25 2023-09-23T22:42:15.834 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 20/25 2023-09-23T22:42:16.097 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 20/25 2023-09-23T22:42:16.669 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 20/25 2023-09-23T22:42:16.920 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 21/25 2023-09-23T22:42:17.040 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 21/25 2023-09-23T22:42:17.204 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-selinux-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 22/25 2023-09-23T22:42:31.053 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 22/25 2023-09-23T22:42:31.054 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /sys 2023-09-23T22:42:31.054 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /proc 2023-09-23T22:42:31.054 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /dev 2023-09-23T22:42:31.054 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /run 2023-09-23T22:42:31.054 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /mnt 2023-09-23T22:42:31.054 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /var/tmp 2023-09-23T22:42:31.054 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /home 2023-09-23T22:42:31.054 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /tmp 2023-09-23T22:42:31.054 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /dev 2023-09-23T22:42:31.055 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:42:32.725 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-radosgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 23/25 2023-09-23T22:42:32.800 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 23/25 2023-09-23T22:42:32.801 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T22:42:33.001 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T22:42:33.002 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T22:42:33.606 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T22:42:33.606 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/25 2023-09-23T22:42:33.606 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/25 2023-09-23T22:42:33.606 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-radosgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 3/25 2023-09-23T22:42:33.606 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-selinux-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 4/25 2023-09-23T22:42:33.606 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/25 2023-09-23T22:42:33.606 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libradosstriper1-2:17.2.6-1125.gc9bb1bcf.el8.x86_6 6/25 2023-09-23T22:42:33.607 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : librgw2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 7/25 2023-09-23T22:42:33.607 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1125.gc9bb1bcf.el8. 8/25 2023-09-23T22:42:33.607 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x8 9/25 2023-09-23T22:42:33.607 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-cephfs-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 10/25 2023-09-23T22:42:33.607 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-rados-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 11/25 2023-09-23T22:42:33.607 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-rbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 12/25 2023-09-23T22:42:33.607 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-rgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 13/25 2023-09-23T22:42:33.607 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T22:42:33.607 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-23T22:42:33.608 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-23T22:42:33.608 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-23T22:42:33.608 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-09-23T22:42:33.608 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-09-23T22:42:33.608 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-09-23T22:42:33.608 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-09-23T22:42:33.608 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 22/25 2023-09-23T22:42:33.608 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T22:42:33.609 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 24/25 2023-09-23T22:42:34.942 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T22:42:34.942 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:00:51 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:42:35.833 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 22/25 2023-09-23T22:42:35.833 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /sys 2023-09-23T22:42:35.833 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /proc 2023-09-23T22:42:35.833 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /dev 2023-09-23T22:42:35.833 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /run 2023-09-23T22:42:35.834 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /mnt 2023-09-23T22:42:35.834 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /var/tmp 2023-09-23T22:42:35.834 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /home 2023-09-23T22:42:35.834 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /tmp 2023-09-23T22:42:35.834 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /dev 2023-09-23T22:42:35.834 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:42:37.559 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-radosgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 23/25 2023-09-23T22:42:37.641 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 23/25 2023-09-23T22:42:37.641 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T22:42:37.870 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T22:42:37.871 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T22:42:37.975 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-09-23T22:42:38.162 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:42:38.163 INFO:teuthology.orchestra.run.smithi138.stdout:Upgraded: 2023-09-23T22:42:38.163 INFO:teuthology.orchestra.run.smithi138.stdout: librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:38.163 INFO:teuthology.orchestra.run.smithi138.stdout: librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:38.163 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-09-23T22:42:38.163 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:38.163 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:38.163 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-radosgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:38.163 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-selinux-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:38.163 INFO:teuthology.orchestra.run.smithi138.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T22:42:38.163 INFO:teuthology.orchestra.run.smithi138.stdout: libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:38.163 INFO:teuthology.orchestra.run.smithi138.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T22:42:38.164 INFO:teuthology.orchestra.run.smithi138.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T22:42:38.164 INFO:teuthology.orchestra.run.smithi138.stdout: libradosstriper1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:38.164 INFO:teuthology.orchestra.run.smithi138.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T22:42:38.164 INFO:teuthology.orchestra.run.smithi138.stdout: librgw2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:38.164 INFO:teuthology.orchestra.run.smithi138.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T22:42:38.164 INFO:teuthology.orchestra.run.smithi138.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T22:42:38.164 INFO:teuthology.orchestra.run.smithi138.stdout: python3-ceph-argparse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:38.164 INFO:teuthology.orchestra.run.smithi138.stdout: python3-ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:38.164 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cephfs-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:38.164 INFO:teuthology.orchestra.run.smithi138.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T22:42:38.165 INFO:teuthology.orchestra.run.smithi138.stdout: python3-rados-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:38.165 INFO:teuthology.orchestra.run.smithi138.stdout: python3-rbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:38.165 INFO:teuthology.orchestra.run.smithi138.stdout: python3-rgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:38.165 INFO:teuthology.orchestra.run.smithi138.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-23T22:42:38.165 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:42:38.165 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:42:38.325 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-test 2023-09-23T22:42:38.459 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T22:42:38.459 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/25 2023-09-23T22:42:38.459 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/25 2023-09-23T22:42:38.459 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-radosgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 3/25 2023-09-23T22:42:38.460 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-selinux-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 4/25 2023-09-23T22:42:38.460 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/25 2023-09-23T22:42:38.460 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libradosstriper1-2:17.2.6-1125.gc9bb1bcf.el8.x86_6 6/25 2023-09-23T22:42:38.460 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librgw2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 7/25 2023-09-23T22:42:38.460 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1125.gc9bb1bcf.el8. 8/25 2023-09-23T22:42:38.460 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x8 9/25 2023-09-23T22:42:38.460 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cephfs-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 10/25 2023-09-23T22:42:38.460 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-rados-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 11/25 2023-09-23T22:42:38.460 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-rbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 12/25 2023-09-23T22:42:38.460 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-rgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 13/25 2023-09-23T22:42:38.460 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T22:42:38.460 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-23T22:42:38.461 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-23T22:42:38.461 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-23T22:42:38.461 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-09-23T22:42:38.461 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-09-23T22:42:38.461 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-09-23T22:42:38.461 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-09-23T22:42:38.461 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 22/25 2023-09-23T22:42:38.461 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T22:42:38.461 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 24/25 2023-09-23T22:42:38.674 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:42:39.565 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:00:56 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:42:39.850 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T22:42:39.851 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:00:50 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:42:40.999 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:42:41.000 INFO:teuthology.orchestra.run.smithi138.stdout:============================================================================================ 2023-09-23T22:42:41.000 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-09-23T22:42:41.000 INFO:teuthology.orchestra.run.smithi138.stdout:============================================================================================ 2023-09-23T22:42:41.001 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-09-23T22:42:41.001 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-test x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 52 M 2023-09-23T22:42:41.001 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-09-23T22:42:41.001 INFO:teuthology.orchestra.run.smithi138.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T22:42:41.001 INFO:teuthology.orchestra.run.smithi138.stdout: libcephsqlite x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 174 k 2023-09-23T22:42:41.001 INFO:teuthology.orchestra.run.smithi138.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T22:42:41.001 INFO:teuthology.orchestra.run.smithi138.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T22:42:41.001 INFO:teuthology.orchestra.run.smithi138.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T22:42:41.001 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:42:41.001 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-09-23T22:42:41.001 INFO:teuthology.orchestra.run.smithi138.stdout:============================================================================================ 2023-09-23T22:42:41.001 INFO:teuthology.orchestra.run.smithi138.stdout:Install 6 Packages 2023-09-23T22:42:41.002 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:42:41.002 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 53 M 2023-09-23T22:42:41.002 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 233 M 2023-09-23T22:42:41.002 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-09-23T22:42:41.045 INFO:teuthology.orchestra.run.smithi138.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.6 MB/s | 70 kB 00:00 2023-09-23T22:42:41.120 INFO:teuthology.orchestra.run.smithi138.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.7 MB/s | 323 kB 00:00 2023-09-23T22:42:41.162 INFO:teuthology.orchestra.run.smithi138.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.4 MB/s | 187 kB 00:00 2023-09-23T22:42:41.204 INFO:teuthology.orchestra.run.smithi138.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.7 MB/s | 202 kB 00:00 2023-09-23T22:42:41.429 INFO:teuthology.orchestra.run.smithi138.stdout:(5/6): libcephsqlite-17.2.6-1125.gc9bb1bcf.el8. 409 kB/s | 174 kB 00:00 2023-09-23T22:42:42.712 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-09-23T22:42:42.896 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:42:42.896 INFO:teuthology.orchestra.run.smithi110.stdout:Upgraded: 2023-09-23T22:42:42.897 INFO:teuthology.orchestra.run.smithi110.stdout: librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:42.897 INFO:teuthology.orchestra.run.smithi110.stdout: librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:42.897 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-09-23T22:42:42.897 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:42.897 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:42.897 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-radosgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:42.897 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-selinux-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:42.897 INFO:teuthology.orchestra.run.smithi110.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T22:42:42.897 INFO:teuthology.orchestra.run.smithi110.stdout: libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:42.897 INFO:teuthology.orchestra.run.smithi110.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T22:42:42.897 INFO:teuthology.orchestra.run.smithi110.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T22:42:42.897 INFO:teuthology.orchestra.run.smithi110.stdout: libradosstriper1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:42.897 INFO:teuthology.orchestra.run.smithi110.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T22:42:42.898 INFO:teuthology.orchestra.run.smithi110.stdout: librgw2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:42.898 INFO:teuthology.orchestra.run.smithi110.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T22:42:42.898 INFO:teuthology.orchestra.run.smithi110.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T22:42:42.898 INFO:teuthology.orchestra.run.smithi110.stdout: python3-ceph-argparse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:42.898 INFO:teuthology.orchestra.run.smithi110.stdout: python3-ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:42.898 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cephfs-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:42.898 INFO:teuthology.orchestra.run.smithi110.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T22:42:42.898 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rados-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:42.898 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:42.898 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:42.898 INFO:teuthology.orchestra.run.smithi110.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-23T22:42:42.898 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:42:42.899 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:42:43.063 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-test 2023-09-23T22:42:43.408 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:42:43.471 INFO:teuthology.orchestra.run.smithi138.stdout:(6/6): ceph-test-17.2.6-1125.gc9bb1bcf.el8.x86_ 21 MB/s | 52 MB 00:02 2023-09-23T22:42:43.473 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:42:43.473 INFO:teuthology.orchestra.run.smithi138.stdout:Total 21 MB/s | 53 MB 00:02 2023-09-23T22:42:43.510 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-09-23T22:42:43.531 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-09-23T22:42:43.531 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-09-23T22:42:43.809 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-09-23T22:42:43.812 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-09-23T22:42:44.268 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-09-23T22:42:44.422 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T22:42:44.501 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T22:42:44.718 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T22:42:44.835 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:00:55 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:42:44.889 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T22:42:45.046 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T22:42:45.180 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : libcephsqlite-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/6 2023-09-23T22:42:46.189 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:42:46.190 INFO:teuthology.orchestra.run.smithi110.stdout:============================================================================================ 2023-09-23T22:42:46.190 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-09-23T22:42:46.190 INFO:teuthology.orchestra.run.smithi110.stdout:============================================================================================ 2023-09-23T22:42:46.190 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-09-23T22:42:46.190 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-test x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 52 M 2023-09-23T22:42:46.190 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-09-23T22:42:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T22:42:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: libcephsqlite x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 174 k 2023-09-23T22:42:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T22:42:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T22:42:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T22:42:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:42:46.191 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-09-23T22:42:46.191 INFO:teuthology.orchestra.run.smithi110.stdout:============================================================================================ 2023-09-23T22:42:46.191 INFO:teuthology.orchestra.run.smithi110.stdout:Install 6 Packages 2023-09-23T22:42:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:42:46.192 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 53 M 2023-09-23T22:42:46.192 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 233 M 2023-09-23T22:42:46.192 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-09-23T22:42:46.233 INFO:teuthology.orchestra.run.smithi110.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.7 MB/s | 70 kB 00:00 2023-09-23T22:42:46.400 INFO:teuthology.orchestra.run.smithi110.stdout:(2/6): libcephsqlite-17.2.6-1125.gc9bb1bcf.el8. 838 kB/s | 174 kB 00:00 2023-09-23T22:42:46.434 INFO:teuthology.orchestra.run.smithi110.stdout:(3/6): socat-1.7.4.1-1.el8.x86_64.rpm 1.6 MB/s | 323 kB 00:00 2023-09-23T22:42:46.567 INFO:teuthology.orchestra.run.smithi110.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 1.5 MB/s | 202 kB 00:00 2023-09-23T22:42:46.601 INFO:teuthology.orchestra.run.smithi110.stdout:(5/6): oniguruma-6.8.2-2.el8.x86_64.rpm 935 kB/s | 187 kB 00:00 2023-09-23T22:42:48.668 INFO:teuthology.orchestra.run.smithi110.stdout:(6/6): ceph-test-17.2.6-1125.gc9bb1bcf.el8.x86_ 21 MB/s | 52 MB 00:02 2023-09-23T22:42:48.670 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:42:48.670 INFO:teuthology.orchestra.run.smithi110.stdout:Total 21 MB/s | 53 MB 00:02 2023-09-23T22:42:48.708 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-09-23T22:42:48.729 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-09-23T22:42:48.729 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-09-23T22:42:49.007 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-09-23T22:42:49.011 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-09-23T22:42:49.476 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-09-23T22:42:49.617 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T22:42:49.685 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T22:42:49.912 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T22:42:50.084 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T22:42:50.139 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/6 2023-09-23T22:42:50.298 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T22:42:50.393 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-test-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 6/6 2023-09-23T22:42:50.466 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libcephsqlite-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/6 2023-09-23T22:42:50.611 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-test-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 6/6 2023-09-23T22:42:50.611 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-test-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/6 2023-09-23T22:42:50.611 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libcephsqlite-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/6 2023-09-23T22:42:50.611 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T22:42:50.611 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T22:42:50.611 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T22:42:51.665 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T22:42:51.666 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:08 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:42:55.005 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-09-23T22:42:55.186 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:42:55.187 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-09-23T22:42:55.187 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-test-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:55.187 INFO:teuthology.orchestra.run.smithi138.stdout: jq-1.6-3.el8.x86_64 2023-09-23T22:42:55.187 INFO:teuthology.orchestra.run.smithi138.stdout: libcephsqlite-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:55.187 INFO:teuthology.orchestra.run.smithi138.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T22:42:55.187 INFO:teuthology.orchestra.run.smithi138.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T22:42:55.187 INFO:teuthology.orchestra.run.smithi138.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T22:42:55.187 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:42:55.187 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:42:55.325 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph 2023-09-23T22:42:55.429 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/6 2023-09-23T22:42:55.659 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:42:55.704 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-test-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 6/6 2023-09-23T22:42:55.916 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-test-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 6/6 2023-09-23T22:42:55.917 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-test-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/6 2023-09-23T22:42:55.917 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libcephsqlite-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/6 2023-09-23T22:42:55.917 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T22:42:55.917 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T22:42:55.917 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T22:42:56.532 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:13 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:42:57.021 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T22:42:57.021 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:08 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:42:57.888 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:42:57.892 INFO:teuthology.orchestra.run.smithi138.stdout:======================================================================================================= 2023-09-23T22:42:57.892 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-09-23T22:42:57.892 INFO:teuthology.orchestra.run.smithi138.stdout:======================================================================================================= 2023-09-23T22:42:57.892 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-09-23T22:42:57.892 INFO:teuthology.orchestra.run.smithi138.stdout: ceph x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 6.4 k 2023-09-23T22:42:57.892 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-09-23T22:42:57.892 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mds x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 2.3 M 2023-09-23T22:42:57.892 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 1.6 M 2023-09-23T22:42:57.892 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 238 k 2023-09-23T22:42:57.892 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mon x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 3.9 M 2023-09-23T22:42:57.892 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-osd x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 17 M 2023-09-23T22:42:57.893 INFO:teuthology.orchestra.run.smithi138.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T22:42:57.893 INFO:teuthology.orchestra.run.smithi138.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T22:42:57.893 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T22:42:57.893 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T22:42:57.893 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T22:42:57.893 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T22:42:57.893 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T22:42:57.893 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T22:42:57.893 INFO:teuthology.orchestra.run.smithi138.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T22:42:57.893 INFO:teuthology.orchestra.run.smithi138.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T22:42:57.893 INFO:teuthology.orchestra.run.smithi138.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T22:42:57.893 INFO:teuthology.orchestra.run.smithi138.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T22:42:57.893 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T22:42:57.894 INFO:teuthology.orchestra.run.smithi138.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T22:42:57.894 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T22:42:57.894 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T22:42:57.894 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T22:42:57.894 INFO:teuthology.orchestra.run.smithi138.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T22:42:57.894 INFO:teuthology.orchestra.run.smithi138.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T22:42:57.894 INFO:teuthology.orchestra.run.smithi138.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T22:42:57.894 INFO:teuthology.orchestra.run.smithi138.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T22:42:57.894 INFO:teuthology.orchestra.run.smithi138.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T22:42:57.894 INFO:teuthology.orchestra.run.smithi138.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T22:42:57.894 INFO:teuthology.orchestra.run.smithi138.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T22:42:57.894 INFO:teuthology.orchestra.run.smithi138.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T22:42:57.894 INFO:teuthology.orchestra.run.smithi138.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T22:42:57.895 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:42:57.895 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-09-23T22:42:57.895 INFO:teuthology.orchestra.run.smithi138.stdout:======================================================================================================= 2023-09-23T22:42:57.895 INFO:teuthology.orchestra.run.smithi138.stdout:Install 32 Packages 2023-09-23T22:42:57.895 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:42:57.895 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 28 M 2023-09-23T22:42:57.895 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 110 M 2023-09-23T22:42:57.895 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-09-23T22:42:58.188 INFO:teuthology.orchestra.run.smithi138.stdout:(1/32): ceph-17.2.6-1125.gc9bb1bcf.el8.x86_64.r 25 kB/s | 6.4 kB 00:00 2023-09-23T22:42:58.618 INFO:teuthology.orchestra.run.smithi138.stdout:(2/32): ceph-mds-17.2.6-1125.gc9bb1bcf.el8.x86_ 3.3 MB/s | 2.3 MB 00:00 2023-09-23T22:42:58.655 INFO:teuthology.orchestra.run.smithi138.stdout:(3/32): ceph-mgr-17.2.6-1125.gc9bb1bcf.el8.x86_ 2.2 MB/s | 1.6 MB 00:00 2023-09-23T22:42:58.773 INFO:teuthology.orchestra.run.smithi138.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1125.gc9bb 2.0 MB/s | 238 kB 00:00 2023-09-23T22:42:58.956 INFO:teuthology.orchestra.run.smithi138.stdout:(5/32): ceph-mon-17.2.6-1125.gc9bb1bcf.el8.x86_ 5.1 MB/s | 3.9 MB 00:00 2023-09-23T22:42:58.973 INFO:teuthology.orchestra.run.smithi138.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 218 kB/s | 44 kB 00:00 2023-09-23T22:42:59.057 INFO:teuthology.orchestra.run.smithi138.stdout:(7/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 1.8 MB/s | 185 kB 00:00 2023-09-23T22:42:59.190 INFO:teuthology.orchestra.run.smithi138.stdout:(8/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 2.8 MB/s | 384 kB 00:00 2023-09-23T22:42:59.215 INFO:teuthology.orchestra.run.smithi138.stdout:(9/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 712 kB/s | 173 kB 00:00 2023-09-23T22:42:59.241 INFO:teuthology.orchestra.run.smithi138.stdout:(10/32): python3-jaraco-6.2-6.el8.noarch.rpm 221 kB/s | 11 kB 00:00 2023-09-23T22:42:59.266 INFO:teuthology.orchestra.run.smithi138.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 351 kB/s | 18 kB 00:00 2023-09-23T22:42:59.308 INFO:teuthology.orchestra.run.smithi138.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 724 kB/s | 49 kB 00:00 2023-09-23T22:42:59.334 INFO:teuthology.orchestra.run.smithi138.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 873 kB/s | 59 kB 00:00 2023-09-23T22:42:59.384 INFO:teuthology.orchestra.run.smithi138.stdout:(14/32): python3-portend-2.6-1.el8.noarch.rpm 312 kB/s | 16 kB 00:00 2023-09-23T22:42:59.551 INFO:teuthology.orchestra.run.smithi138.stdout:(15/32): ceph-osd-17.2.6-1125.gc9bb1bcf.el8.x86 18 MB/s | 17 MB 00:00 2023-09-23T22:42:59.576 INFO:teuthology.orchestra.run.smithi138.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 97 kB/s | 19 kB 00:00 2023-09-23T22:42:59.601 INFO:teuthology.orchestra.run.smithi138.stdout:(17/32): python3-pecan-1.3.2-9.el8.noarch.rpm 967 kB/s | 283 kB 00:00 2023-09-23T22:42:59.627 INFO:teuthology.orchestra.run.smithi138.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 583 kB/s | 29 kB 00:00 2023-09-23T22:42:59.652 INFO:teuthology.orchestra.run.smithi138.stdout:(19/32): python3-singledispatch-3.4.0.3-18.el8. 238 kB/s | 24 kB 00:00 2023-09-23T22:42:59.685 INFO:teuthology.orchestra.run.smithi138.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 328 kB/s | 27 kB 00:00 2023-09-23T22:42:59.711 INFO:teuthology.orchestra.run.smithi138.stdout:(21/32): python3-waitress-1.4.3-1.el8.noarch.rp 3.0 MB/s | 255 kB 00:00 2023-09-23T22:42:59.736 INFO:teuthology.orchestra.run.smithi138.stdout:(22/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.9 MB/s | 251 kB 00:00 2023-09-23T22:42:59.761 INFO:teuthology.orchestra.run.smithi138.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 456 kB/s | 23 kB 00:00 2023-09-23T22:42:59.787 INFO:teuthology.orchestra.run.smithi138.stdout:(24/32): python3-webtest-2.0.33-1.el8.noarch.rp 847 kB/s | 85 kB 00:00 2023-09-23T22:42:59.797 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-09-23T22:42:59.879 INFO:teuthology.orchestra.run.smithi138.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 768 kB/s | 109 kB 00:00 2023-09-23T22:42:59.912 INFO:teuthology.orchestra.run.smithi138.stdout:(26/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 2.0 MB/s | 238 kB 00:00 2023-09-23T22:42:59.937 INFO:teuthology.orchestra.run.smithi138.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 926 kB/s | 54 kB 00:00 2023-09-23T22:42:59.971 INFO:teuthology.orchestra.run.smithi138.stdout:(28/32): python3-cryptography-3.2.1-5.el8.x86_6 2.6 MB/s | 559 kB 00:00 2023-09-23T22:42:59.977 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:42:59.977 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-09-23T22:42:59.977 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-test-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:59.977 INFO:teuthology.orchestra.run.smithi110.stdout: jq-1.6-3.el8.x86_64 2023-09-23T22:42:59.977 INFO:teuthology.orchestra.run.smithi110.stdout: libcephsqlite-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:42:59.977 INFO:teuthology.orchestra.run.smithi110.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T22:42:59.977 INFO:teuthology.orchestra.run.smithi110.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T22:42:59.977 INFO:teuthology.orchestra.run.smithi110.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T22:42:59.977 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:42:59.977 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:42:59.996 INFO:teuthology.orchestra.run.smithi138.stdout:(29/32): python3-markupsafe-0.23-19.el8.x86_64. 463 kB/s | 39 kB 00:00 2023-09-23T22:43:00.022 INFO:teuthology.orchestra.run.smithi138.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 1.8 MB/s | 157 kB 00:00 2023-09-23T22:43:00.047 INFO:teuthology.orchestra.run.smithi138.stdout:(31/32): python3-werkzeug-0.12.2-4.el8.noarch.r 5.9 MB/s | 457 kB 00:00 2023-09-23T22:43:00.064 INFO:teuthology.orchestra.run.smithi138.stdout:(32/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 1.5 MB/s | 103 kB 00:00 2023-09-23T22:43:00.065 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:43:00.065 INFO:teuthology.orchestra.run.smithi138.stdout:Total 13 MB/s | 28 MB 00:02 2023-09-23T22:43:00.119 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph 2023-09-23T22:43:00.157 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-09-23T22:43:00.194 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-09-23T22:43:00.194 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-09-23T22:43:00.397 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-09-23T22:43:00.401 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-09-23T22:43:00.452 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:43:01.324 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:12 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:43:01.548 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-09-23T22:43:01.767 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T22:43:02.045 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T22:43:02.290 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-09-23T22:43:02.538 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-09-23T22:43:02.702 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:43:02.705 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================================= 2023-09-23T22:43:02.705 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-09-23T22:43:02.705 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================================= 2023-09-23T22:43:02.705 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-09-23T22:43:02.705 INFO:teuthology.orchestra.run.smithi110.stdout: ceph x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 6.4 k 2023-09-23T22:43:02.705 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-09-23T22:43:02.706 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mds x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 2.3 M 2023-09-23T22:43:02.706 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 1.6 M 2023-09-23T22:43:02.706 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 238 k 2023-09-23T22:43:02.706 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mon x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 3.9 M 2023-09-23T22:43:02.706 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-osd x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 17 M 2023-09-23T22:43:02.706 INFO:teuthology.orchestra.run.smithi110.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T22:43:02.706 INFO:teuthology.orchestra.run.smithi110.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T22:43:02.706 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T22:43:02.706 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T22:43:02.706 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T22:43:02.706 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T22:43:02.706 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T22:43:02.707 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T22:43:02.707 INFO:teuthology.orchestra.run.smithi110.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T22:43:02.707 INFO:teuthology.orchestra.run.smithi110.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T22:43:02.707 INFO:teuthology.orchestra.run.smithi110.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T22:43:02.707 INFO:teuthology.orchestra.run.smithi110.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T22:43:02.707 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T22:43:02.707 INFO:teuthology.orchestra.run.smithi110.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T22:43:02.707 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T22:43:02.707 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T22:43:02.707 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T22:43:02.707 INFO:teuthology.orchestra.run.smithi110.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T22:43:02.707 INFO:teuthology.orchestra.run.smithi110.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T22:43:02.708 INFO:teuthology.orchestra.run.smithi110.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T22:43:02.708 INFO:teuthology.orchestra.run.smithi110.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T22:43:02.708 INFO:teuthology.orchestra.run.smithi110.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T22:43:02.708 INFO:teuthology.orchestra.run.smithi110.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T22:43:02.708 INFO:teuthology.orchestra.run.smithi110.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T22:43:02.708 INFO:teuthology.orchestra.run.smithi110.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T22:43:02.708 INFO:teuthology.orchestra.run.smithi110.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T22:43:02.708 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:43:02.708 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-09-23T22:43:02.708 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================================= 2023-09-23T22:43:02.708 INFO:teuthology.orchestra.run.smithi110.stdout:Install 32 Packages 2023-09-23T22:43:02.708 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:43:02.709 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 28 M 2023-09-23T22:43:02.709 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 110 M 2023-09-23T22:43:02.709 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-09-23T22:43:02.784 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-09-23T22:43:02.990 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-09-23T22:43:03.103 INFO:teuthology.orchestra.run.smithi110.stdout:(1/32): ceph-17.2.6-1125.gc9bb1bcf.el8.x86_64.r 31 kB/s | 6.4 kB 00:00 2023-09-23T22:43:03.237 INFO:teuthology.orchestra.run.smithi110.stdout:(2/32): ceph-mds-17.2.6-1125.gc9bb1bcf.el8.x86_ 6.8 MB/s | 2.3 MB 00:00 2023-09-23T22:43:03.239 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-09-23T22:43:03.601 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-09-23T22:43:03.604 INFO:teuthology.orchestra.run.smithi110.stdout:(3/32): ceph-mgr-17.2.6-1125.gc9bb1bcf.el8.x86_ 2.2 MB/s | 1.6 MB 00:00 2023-09-23T22:43:03.704 INFO:teuthology.orchestra.run.smithi110.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1125.gc9bb 2.3 MB/s | 238 kB 00:00 2023-09-23T22:43:03.864 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-09-23T22:43:04.019 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-09-23T22:43:04.079 INFO:teuthology.orchestra.run.smithi110.stdout:(5/32): ceph-osd-17.2.6-1125.gc9bb1bcf.el8.x86_ 20 MB/s | 17 MB 00:00 2023-09-23T22:43:04.096 INFO:teuthology.orchestra.run.smithi110.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 111 kB/s | 44 kB 00:00 2023-09-23T22:43:04.191 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-09-23T22:43:04.280 INFO:teuthology.orchestra.run.smithi110.stdout:(7/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 940 kB/s | 173 kB 00:00 2023-09-23T22:43:04.394 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-09-23T22:43:04.422 INFO:teuthology.orchestra.run.smithi110.stdout:(8/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 541 kB/s | 185 kB 00:00 2023-09-23T22:43:04.447 INFO:teuthology.orchestra.run.smithi110.stdout:(9/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 2.3 MB/s | 384 kB 00:00 2023-09-23T22:43:04.489 INFO:teuthology.orchestra.run.smithi110.stdout:(10/32): python3-jaraco-6.2-6.el8.noarch.rpm 166 kB/s | 11 kB 00:00 2023-09-23T22:43:04.540 INFO:teuthology.orchestra.run.smithi110.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 191 kB/s | 18 kB 00:00 2023-09-23T22:43:04.573 INFO:teuthology.orchestra.run.smithi110.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 582 kB/s | 49 kB 00:00 2023-09-23T22:43:04.604 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T22:43:04.615 INFO:teuthology.orchestra.run.smithi110.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 782 kB/s | 59 kB 00:00 2023-09-23T22:43:04.732 INFO:teuthology.orchestra.run.smithi110.stdout:(14/32): ceph-mon-17.2.6-1125.gc9bb1bcf.el8.x86 2.4 MB/s | 3.9 MB 00:01 2023-09-23T22:43:04.757 INFO:teuthology.orchestra.run.smithi110.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 110 kB/s | 16 kB 00:00 2023-09-23T22:43:04.782 INFO:teuthology.orchestra.run.smithi110.stdout:(16/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.3 MB/s | 283 kB 00:00 2023-09-23T22:43:04.808 INFO:teuthology.orchestra.run.smithi110.stdout:(17/32): python3-simplegeneric-0.8.1-17.el8.noa 248 kB/s | 19 kB 00:00 2023-09-23T22:43:04.817 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T22:43:04.859 INFO:teuthology.orchestra.run.smithi110.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 386 kB/s | 29 kB 00:00 2023-09-23T22:43:04.883 INFO:teuthology.orchestra.run.smithi110.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 389 kB/s | 27 kB 00:00 2023-09-23T22:43:04.967 INFO:teuthology.orchestra.run.smithi110.stdout:(20/32): python3-waitress-1.4.3-1.el8.noarch.rp 2.3 MB/s | 255 kB 00:00 2023-09-23T22:43:04.992 INFO:teuthology.orchestra.run.smithi110.stdout:(21/32): python3-singledispatch-3.4.0.3-18.el8. 102 kB/s | 24 kB 00:00 2023-09-23T22:43:05.017 INFO:teuthology.orchestra.run.smithi110.stdout:(22/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.8 MB/s | 251 kB 00:00 2023-09-23T22:43:05.025 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T22:43:05.051 INFO:teuthology.orchestra.run.smithi110.stdout:(23/32): python3-webtest-2.0.33-1.el8.noarch.rp 1.0 MB/s | 85 kB 00:00 2023-09-23T22:43:05.085 INFO:teuthology.orchestra.run.smithi110.stdout:(24/32): python3-zc-lockfile-2.0-2.el8.noarch.r 248 kB/s | 23 kB 00:00 2023-09-23T22:43:05.119 INFO:teuthology.orchestra.run.smithi110.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 1.1 MB/s | 109 kB 00:00 2023-09-23T22:43:05.177 INFO:teuthology.orchestra.run.smithi110.stdout:(26/32): python3-cryptography-3.2.1-5.el8.x86_6 4.5 MB/s | 559 kB 00:00 2023-09-23T22:43:05.189 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T22:43:05.203 INFO:teuthology.orchestra.run.smithi110.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 693 kB/s | 54 kB 00:00 2023-09-23T22:43:05.229 INFO:teuthology.orchestra.run.smithi110.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 773 kB/s | 39 kB 00:00 2023-09-23T22:43:05.253 INFO:teuthology.orchestra.run.smithi110.stdout:(29/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.4 MB/s | 238 kB 00:00 2023-09-23T22:43:05.314 INFO:teuthology.orchestra.run.smithi110.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 1.4 MB/s | 157 kB 00:00 2023-09-23T22:43:05.345 INFO:teuthology.orchestra.run.smithi110.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 1.1 MB/s | 103 kB 00:00 2023-09-23T22:43:05.364 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T22:43:05.379 INFO:teuthology.orchestra.run.smithi110.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 3.0 MB/s | 457 kB 00:00 2023-09-23T22:43:05.383 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:43:05.383 INFO:teuthology.orchestra.run.smithi110.stdout:Total 11 MB/s | 28 MB 00:02 2023-09-23T22:43:05.472 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-09-23T22:43:05.508 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-09-23T22:43:05.508 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-09-23T22:43:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T22:43:05.711 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-09-23T22:43:05.715 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-09-23T22:43:05.740 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T22:43:06.665 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T22:43:06.844 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-09-23T22:43:06.858 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T22:43:07.053 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T22:43:07.114 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T22:43:07.365 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T22:43:07.392 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T22:43:07.576 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-09-23T22:43:07.628 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T22:43:07.824 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-09-23T22:43:07.923 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T22:43:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-09-23T22:43:08.257 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T22:43:08.276 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-09-23T22:43:08.524 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-09-23T22:43:08.615 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1125.gc9bb1bcf.el8. 27/32 2023-09-23T22:43:08.778 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 28/32 2023-09-23T22:43:08.903 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-09-23T22:43:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-09-23T22:43:09.363 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-09-23T22:43:09.591 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-09-23T22:43:09.789 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-09-23T22:43:10.055 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T22:43:10.229 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T22:43:10.251 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 28/32 2023-09-23T22:43:10.435 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T22:43:10.452 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-osd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 29/32 2023-09-23T22:43:10.846 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-osd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 29/32 2023-09-23T22:43:11.018 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-mon-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 30/32 2023-09-23T22:43:11.225 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T22:43:11.243 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-mon-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 30/32 2023-09-23T22:43:11.375 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T22:43:11.402 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-mds-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 31/32 2023-09-23T22:43:11.419 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-mds-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 31/32 2023-09-23T22:43:11.503 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T22:43:11.575 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 32/32 2023-09-23T22:43:11.750 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T22:43:11.916 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 32/32 2023-09-23T22:43:11.916 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/32 2023-09-23T22:43:11.916 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-mds-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/32 2023-09-23T22:43:11.917 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 3/32 2023-09-23T22:43:11.917 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-mon-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 4/32 2023-09-23T22:43:11.917 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-osd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/32 2023-09-23T22:43:11.917 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1125.gc9bb1bcf.el8. 6/32 2023-09-23T22:43:11.917 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T22:43:11.917 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T22:43:11.917 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T22:43:11.917 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T22:43:11.917 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T22:43:11.917 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T22:43:11.917 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T22:43:11.918 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T22:43:11.918 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T22:43:11.918 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T22:43:11.918 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T22:43:11.918 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T22:43:11.918 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T22:43:11.918 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T22:43:11.918 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T22:43:11.918 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T22:43:11.918 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T22:43:11.918 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T22:43:11.918 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-09-23T22:43:11.918 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-09-23T22:43:11.919 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-09-23T22:43:11.919 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-09-23T22:43:11.919 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-09-23T22:43:11.919 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-09-23T22:43:11.919 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-09-23T22:43:12.034 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T22:43:12.219 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T22:43:12.492 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T22:43:12.769 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T22:43:12.948 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T22:43:13.288 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T22:43:13.477 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-09-23T22:43:13.477 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:30 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:43:13.618 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T22:43:13.951 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1125.gc9bb1bcf.el8. 27/32 2023-09-23T22:43:14.113 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 28/32 2023-09-23T22:43:15.584 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 28/32 2023-09-23T22:43:15.773 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-09-23T22:43:15.813 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-osd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 29/32 2023-09-23T22:43:15.962 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:43:15.962 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-09-23T22:43:15.962 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:43:15.962 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mds-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:43:15.962 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:43:15.962 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-modules-core-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T22:43:15.963 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mon-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:43:15.963 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-osd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:43:15.963 INFO:teuthology.orchestra.run.smithi138.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T22:43:15.963 INFO:teuthology.orchestra.run.smithi138.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T22:43:15.963 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T22:43:15.963 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T22:43:15.963 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T22:43:15.963 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T22:43:15.963 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T22:43:15.963 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T22:43:15.964 INFO:teuthology.orchestra.run.smithi138.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T22:43:15.964 INFO:teuthology.orchestra.run.smithi138.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T22:43:15.964 INFO:teuthology.orchestra.run.smithi138.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T22:43:15.964 INFO:teuthology.orchestra.run.smithi138.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T22:43:15.964 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T22:43:15.964 INFO:teuthology.orchestra.run.smithi138.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T22:43:15.964 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T22:43:15.964 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T22:43:15.965 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T22:43:15.965 INFO:teuthology.orchestra.run.smithi138.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T22:43:15.965 INFO:teuthology.orchestra.run.smithi138.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T22:43:15.965 INFO:teuthology.orchestra.run.smithi138.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T22:43:15.965 INFO:teuthology.orchestra.run.smithi138.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T22:43:15.965 INFO:teuthology.orchestra.run.smithi138.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T22:43:15.965 INFO:teuthology.orchestra.run.smithi138.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T22:43:15.965 INFO:teuthology.orchestra.run.smithi138.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T22:43:15.966 INFO:teuthology.orchestra.run.smithi138.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T22:43:15.966 INFO:teuthology.orchestra.run.smithi138.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T22:43:15.966 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:43:15.966 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:43:16.125 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-base 2023-09-23T22:43:16.206 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-osd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 29/32 2023-09-23T22:43:16.477 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:43:16.588 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mon-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 30/32 2023-09-23T22:43:16.821 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mon-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 30/32 2023-09-23T22:43:16.997 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mds-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 31/32 2023-09-23T22:43:17.014 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mds-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 31/32 2023-09-23T22:43:17.136 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 32/32 2023-09-23T22:43:17.242 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:34 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:43:17.492 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 32/32 2023-09-23T22:43:17.492 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/32 2023-09-23T22:43:17.492 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mds-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/32 2023-09-23T22:43:17.492 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 3/32 2023-09-23T22:43:17.492 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mon-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 4/32 2023-09-23T22:43:17.492 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-osd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/32 2023-09-23T22:43:17.492 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1125.gc9bb1bcf.el8. 6/32 2023-09-23T22:43:17.492 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T22:43:17.493 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T22:43:17.493 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T22:43:17.493 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T22:43:17.493 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T22:43:17.493 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T22:43:17.493 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T22:43:17.493 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T22:43:17.493 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T22:43:17.494 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T22:43:17.494 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T22:43:17.494 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T22:43:17.494 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T22:43:17.494 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T22:43:17.494 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T22:43:17.494 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T22:43:17.494 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T22:43:17.494 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T22:43:17.494 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-09-23T22:43:17.494 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-09-23T22:43:17.495 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-09-23T22:43:17.495 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-09-23T22:43:17.497 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-09-23T22:43:17.497 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-09-23T22:43:17.497 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-09-23T22:43:18.494 INFO:teuthology.orchestra.run.smithi138.stdout:Package ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:43:18.602 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:43:18.603 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-09-23T22:43:18.603 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:43:18.797 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install cephadm 2023-09-23T22:43:18.935 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-09-23T22:43:18.935 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:29 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:43:19.142 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:43:20.084 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:37 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:43:21.330 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-09-23T22:43:21.459 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:43:21.460 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:43:21.460 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-09-23T22:43:21.460 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:43:21.460 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-09-23T22:43:21.460 INFO:teuthology.orchestra.run.smithi138.stdout: cephadm noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 80 k 2023-09-23T22:43:21.460 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:43:21.461 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-09-23T22:43:21.461 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:43:21.461 INFO:teuthology.orchestra.run.smithi138.stdout:Install 1 Package 2023-09-23T22:43:21.461 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:43:21.461 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 80 k 2023-09-23T22:43:21.461 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 363 k 2023-09-23T22:43:21.461 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-09-23T22:43:21.521 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:43:21.521 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-09-23T22:43:21.522 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:43:21.522 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mds-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:43:21.522 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:43:21.522 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-modules-core-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T22:43:21.522 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mon-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:43:21.522 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-osd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:43:21.522 INFO:teuthology.orchestra.run.smithi110.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T22:43:21.522 INFO:teuthology.orchestra.run.smithi110.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T22:43:21.523 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T22:43:21.523 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T22:43:21.523 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T22:43:21.523 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T22:43:21.523 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T22:43:21.523 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T22:43:21.524 INFO:teuthology.orchestra.run.smithi110.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T22:43:21.524 INFO:teuthology.orchestra.run.smithi110.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T22:43:21.524 INFO:teuthology.orchestra.run.smithi110.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T22:43:21.524 INFO:teuthology.orchestra.run.smithi110.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T22:43:21.524 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T22:43:21.525 INFO:teuthology.orchestra.run.smithi110.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T22:43:21.525 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T22:43:21.525 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T22:43:21.525 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T22:43:21.525 INFO:teuthology.orchestra.run.smithi110.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T22:43:21.525 INFO:teuthology.orchestra.run.smithi110.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T22:43:21.525 INFO:teuthology.orchestra.run.smithi110.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T22:43:21.525 INFO:teuthology.orchestra.run.smithi110.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T22:43:21.525 INFO:teuthology.orchestra.run.smithi110.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T22:43:21.526 INFO:teuthology.orchestra.run.smithi110.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T22:43:21.526 INFO:teuthology.orchestra.run.smithi110.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T22:43:21.526 INFO:teuthology.orchestra.run.smithi110.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T22:43:21.526 INFO:teuthology.orchestra.run.smithi110.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T22:43:21.526 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:43:21.526 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:43:21.677 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-base 2023-09-23T22:43:21.762 INFO:teuthology.orchestra.run.smithi138.stdout:cephadm-17.2.6-1125.gc9bb1bcf.el8.noarch.rpm 266 kB/s | 80 kB 00:00 2023-09-23T22:43:21.764 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:43:21.764 INFO:teuthology.orchestra.run.smithi138.stdout:Total 265 kB/s | 80 kB 00:00 2023-09-23T22:43:21.764 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-09-23T22:43:21.771 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-09-23T22:43:21.771 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-09-23T22:43:21.773 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-09-23T22:43:21.776 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-09-23T22:43:21.818 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-09-23T22:43:22.025 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:43:22.896 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:33 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:43:23.510 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/1 2023-09-23T22:43:23.670 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/1 2023-09-23T22:43:23.835 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/1 2023-09-23T22:43:24.161 INFO:teuthology.orchestra.run.smithi110.stdout:Package ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:43:24.272 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:43:24.273 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-09-23T22:43:24.273 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:43:24.385 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install cephadm 2023-09-23T22:43:24.727 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:43:25.431 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/1 2023-09-23T22:43:25.431 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:42 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:43:25.590 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:36 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:43:26.972 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:43:26.973 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:43:26.973 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-09-23T22:43:26.974 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:43:26.974 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-09-23T22:43:26.974 INFO:teuthology.orchestra.run.smithi110.stdout: cephadm noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 80 k 2023-09-23T22:43:26.974 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:43:26.974 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-09-23T22:43:26.974 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:43:26.974 INFO:teuthology.orchestra.run.smithi110.stdout:Install 1 Package 2023-09-23T22:43:26.974 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:43:26.974 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 80 k 2023-09-23T22:43:26.974 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 363 k 2023-09-23T22:43:26.974 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-09-23T22:43:27.128 INFO:teuthology.orchestra.run.smithi110.stdout:cephadm-17.2.6-1125.gc9bb1bcf.el8.noarch.rpm 520 kB/s | 80 kB 00:00 2023-09-23T22:43:27.129 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:43:27.129 INFO:teuthology.orchestra.run.smithi110.stdout:Total 516 kB/s | 80 kB 00:00 2023-09-23T22:43:27.129 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-09-23T22:43:27.137 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-09-23T22:43:27.137 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-09-23T22:43:27.139 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-09-23T22:43:27.143 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-09-23T22:43:27.188 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-09-23T22:43:28.443 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-09-23T22:43:28.622 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:43:28.622 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-09-23T22:43:28.622 INFO:teuthology.orchestra.run.smithi138.stdout: cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T22:43:28.622 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:43:28.622 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:43:28.770 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T22:43:29.125 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:43:29.925 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/1 2023-09-23T22:43:30.019 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:47 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:43:30.095 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/1 2023-09-23T22:43:30.261 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/1 2023-09-23T22:43:31.306 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/1 2023-09-23T22:43:31.306 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:42 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:43:31.385 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:43:31.386 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:43:31.386 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repo Size 2023-09-23T22:43:31.386 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:43:31.386 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-09-23T22:43:31.386 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 162 k 2023-09-23T22:43:31.386 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:43:31.387 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-09-23T22:43:31.387 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:43:31.387 INFO:teuthology.orchestra.run.smithi138.stdout:Install 1 Package 2023-09-23T22:43:31.387 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:43:31.387 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 162 k 2023-09-23T22:43:31.388 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 490 k 2023-09-23T22:43:31.388 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-09-23T22:43:31.567 INFO:teuthology.orchestra.run.smithi138.stdout:ceph-immutable-object-cache-17.2.6-1125.gc9bb1b 903 kB/s | 162 kB 00:00 2023-09-23T22:43:31.568 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:43:31.568 INFO:teuthology.orchestra.run.smithi138.stdout:Total 897 kB/s | 162 kB 00:00 2023-09-23T22:43:31.568 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-09-23T22:43:31.578 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-09-23T22:43:31.579 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-09-23T22:43:31.629 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-09-23T22:43:31.632 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-09-23T22:43:31.751 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-09-23T22:43:31.923 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1125.gc9bb1bcf. 1/1 2023-09-23T22:43:32.281 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1125.gc9bb1bcf. 1/1 2023-09-23T22:43:33.341 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1125.gc9bb1bcf. 1/1 2023-09-23T22:43:33.341 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:50 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:43:33.938 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-09-23T22:43:34.117 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:43:34.117 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-09-23T22:43:34.117 INFO:teuthology.orchestra.run.smithi110.stdout: cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T22:43:34.117 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:43:34.117 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:43:34.248 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T22:43:34.606 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:43:35.794 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:46 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:43:36.766 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-09-23T22:43:36.943 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:43:36.943 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-09-23T22:43:36.943 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-immutable-object-cache-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:43:36.943 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:43:36.943 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:43:37.077 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-mgr 2023-09-23T22:43:37.166 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:43:37.166 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:43:37.166 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repo Size 2023-09-23T22:43:37.167 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:43:37.167 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-09-23T22:43:37.167 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 162 k 2023-09-23T22:43:37.167 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:43:37.167 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-09-23T22:43:37.167 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:43:37.167 INFO:teuthology.orchestra.run.smithi110.stdout:Install 1 Package 2023-09-23T22:43:37.167 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:43:37.168 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 162 k 2023-09-23T22:43:37.168 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 490 k 2023-09-23T22:43:37.168 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-09-23T22:43:37.423 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:43:37.468 INFO:teuthology.orchestra.run.smithi110.stdout:ceph-immutable-object-cache-17.2.6-1125.gc9bb1b 541 kB/s | 162 kB 00:00 2023-09-23T22:43:37.468 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:43:37.468 INFO:teuthology.orchestra.run.smithi110.stdout:Total 538 kB/s | 162 kB 00:00 2023-09-23T22:43:37.469 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-09-23T22:43:37.479 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-09-23T22:43:37.479 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-09-23T22:43:37.530 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-09-23T22:43:37.534 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-09-23T22:43:37.645 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-09-23T22:43:37.877 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1125.gc9bb1bcf. 1/1 2023-09-23T22:43:38.238 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1125.gc9bb1bcf. 1/1 2023-09-23T22:43:38.994 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:55 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:43:39.256 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1125.gc9bb1bcf. 1/1 2023-09-23T22:43:39.256 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:50 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:43:40.249 INFO:teuthology.orchestra.run.smithi138.stdout:Package ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:43:40.360 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:43:40.361 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-09-23T22:43:40.361 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:43:40.813 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T22:43:41.155 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:43:42.760 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:59 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:43:44.117 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:43:44.118 INFO:teuthology.orchestra.run.smithi138.stdout:=================================================================================================== 2023-09-23T22:43:44.118 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-09-23T22:43:44.118 INFO:teuthology.orchestra.run.smithi138.stdout:=================================================================================================== 2023-09-23T22:43:44.118 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-09-23T22:43:44.118 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 1.7 M 2023-09-23T22:43:44.119 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-09-23T22:43:44.119 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 23 k 2023-09-23T22:43:44.119 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 14 k 2023-09-23T22:43:44.119 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T22:43:44.119 INFO:teuthology.orchestra.run.smithi138.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T22:43:44.119 INFO:teuthology.orchestra.run.smithi138.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T22:43:44.119 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:43:44.119 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-09-23T22:43:44.119 INFO:teuthology.orchestra.run.smithi138.stdout:=================================================================================================== 2023-09-23T22:43:44.119 INFO:teuthology.orchestra.run.smithi138.stdout:Install 6 Packages 2023-09-23T22:43:44.119 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:43:44.120 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 2.0 M 2023-09-23T22:43:44.120 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 8.0 M 2023-09-23T22:43:44.120 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-09-23T22:43:44.640 INFO:teuthology.orchestra.run.smithi138.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1125.gc9b 159 kB/s | 23 kB 00:00 2023-09-23T22:43:44.665 INFO:teuthology.orchestra.run.smithi138.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1125.gc9bb 85 kB/s | 14 kB 00:00 2023-09-23T22:43:44.807 INFO:teuthology.orchestra.run.smithi138.stdout:(3/6): ceph-mgr-dashboard-17.2.6-1125.gc9bb1bcf 5.4 MB/s | 1.7 MB 00:00 2023-09-23T22:43:44.865 INFO:teuthology.orchestra.run.smithi138.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 732 kB/s | 43 kB 00:00 2023-09-23T22:43:44.949 INFO:teuthology.orchestra.run.smithi138.stdout:(5/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 109 kB/s | 34 kB 00:00 2023-09-23T22:43:45.032 INFO:teuthology.orchestra.run.smithi138.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 535 kB/s | 196 kB 00:00 2023-09-23T22:43:45.034 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:43:45.034 INFO:teuthology.orchestra.run.smithi138.stdout:Total 2.2 MB/s | 2.0 MB 00:00 2023-09-23T22:43:45.046 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-09-23T22:43:45.057 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-09-23T22:43:45.057 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-09-23T22:43:45.114 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-09-23T22:43:45.117 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-09-23T22:43:45.414 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-09-23T22:43:45.572 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T22:43:45.768 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T22:43:45.948 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T22:43:46.060 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1125.gc9bb1bcf.el8.n 4/6 2023-09-23T22:43:46.327 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-09-23T22:43:46.381 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1125.gc9bb1bcf.el8. 5/6 2023-09-23T22:43:46.507 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:43:46.507 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-09-23T22:43:46.507 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-immutable-object-cache-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:43:46.507 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:43:46.507 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:43:46.584 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1125.gc9bb1bcf.el8.noarc 6/6 2023-09-23T22:43:46.664 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-mgr 2023-09-23T22:43:46.823 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1125.gc9bb1bcf.el8.noarc 6/6 2023-09-23T22:43:46.823 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1125.gc9bb1bcf.el8. 1/6 2023-09-23T22:43:46.823 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1125.gc9bb1bcf.el8.noarc 2/6 2023-09-23T22:43:46.823 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1125.gc9bb1bcf.el8.n 3/6 2023-09-23T22:43:46.823 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T22:43:46.823 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T22:43:47.008 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:43:47.854 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T22:43:47.854 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:04 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:43:48.633 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:59 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:43:49.955 INFO:teuthology.orchestra.run.smithi110.stdout:Package ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:43:50.069 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:43:50.070 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-09-23T22:43:50.070 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:43:50.192 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T22:43:50.534 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:43:52.257 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:03 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:43:53.627 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:43:53.628 INFO:teuthology.orchestra.run.smithi110.stdout:=================================================================================================== 2023-09-23T22:43:53.628 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-09-23T22:43:53.628 INFO:teuthology.orchestra.run.smithi110.stdout:=================================================================================================== 2023-09-23T22:43:53.628 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-09-23T22:43:53.628 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 1.7 M 2023-09-23T22:43:53.628 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-09-23T22:43:53.628 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 23 k 2023-09-23T22:43:53.628 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 14 k 2023-09-23T22:43:53.629 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T22:43:53.629 INFO:teuthology.orchestra.run.smithi110.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T22:43:53.629 INFO:teuthology.orchestra.run.smithi110.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T22:43:53.629 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:43:53.629 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-09-23T22:43:53.629 INFO:teuthology.orchestra.run.smithi110.stdout:=================================================================================================== 2023-09-23T22:43:53.629 INFO:teuthology.orchestra.run.smithi110.stdout:Install 6 Packages 2023-09-23T22:43:53.629 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:43:53.630 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 2.0 M 2023-09-23T22:43:53.630 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 8.0 M 2023-09-23T22:43:53.630 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-09-23T22:43:53.816 INFO:teuthology.orchestra.run.smithi110.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1125.gc9b 154 kB/s | 23 kB 00:00 2023-09-23T22:43:53.841 INFO:teuthology.orchestra.run.smithi110.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1125.gc9bb 83 kB/s | 14 kB 00:00 2023-09-23T22:43:53.900 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-09-23T22:43:54.081 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:43:54.081 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-09-23T22:43:54.082 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-grafana-dashboards-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T22:43:54.082 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-dashboard-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T22:43:54.082 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-prometheus-alerts-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T22:43:54.082 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T22:43:54.082 INFO:teuthology.orchestra.run.smithi138.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T22:43:54.082 INFO:teuthology.orchestra.run.smithi138.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T22:43:54.082 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:43:54.082 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:43:54.205 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T22:43:54.225 INFO:teuthology.orchestra.run.smithi110.stdout:(3/6): ceph-mgr-dashboard-17.2.6-1125.gc9bb1bcf 3.0 MB/s | 1.7 MB 00:00 2023-09-23T22:43:54.292 INFO:teuthology.orchestra.run.smithi110.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 638 kB/s | 43 kB 00:00 2023-09-23T22:43:54.317 INFO:teuthology.orchestra.run.smithi110.stdout:(5/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 67 kB/s | 34 kB 00:00 2023-09-23T22:43:54.534 INFO:teuthology.orchestra.run.smithi110.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 284 kB/s | 196 kB 00:00 2023-09-23T22:43:54.537 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:43:54.537 INFO:teuthology.orchestra.run.smithi110.stdout:Total 2.2 MB/s | 2.0 MB 00:00 2023-09-23T22:43:54.548 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-09-23T22:43:54.551 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:43:54.557 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-09-23T22:43:54.558 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-09-23T22:43:54.614 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-09-23T22:43:54.618 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-09-23T22:43:54.889 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-09-23T22:43:55.076 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T22:43:55.255 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T22:43:55.401 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:12 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:43:55.436 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T22:43:55.556 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1125.gc9bb1bcf.el8.n 4/6 2023-09-23T22:43:55.892 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1125.gc9bb1bcf.el8. 5/6 2023-09-23T22:43:56.129 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1125.gc9bb1bcf.el8.noarc 6/6 2023-09-23T22:43:56.362 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1125.gc9bb1bcf.el8.noarc 6/6 2023-09-23T22:43:56.362 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1125.gc9bb1bcf.el8. 1/6 2023-09-23T22:43:56.363 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1125.gc9bb1bcf.el8.noarc 2/6 2023-09-23T22:43:56.363 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1125.gc9bb1bcf.el8.n 3/6 2023-09-23T22:43:56.363 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T22:43:56.363 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T22:43:56.775 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:43:56.776 INFO:teuthology.orchestra.run.smithi138.stdout:======================================================================================================================= 2023-09-23T22:43:56.776 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-09-23T22:43:56.777 INFO:teuthology.orchestra.run.smithi138.stdout:======================================================================================================================= 2023-09-23T22:43:56.777 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-09-23T22:43:56.777 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 7.4 M 2023-09-23T22:43:56.777 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-09-23T22:43:56.777 INFO:teuthology.orchestra.run.smithi138.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T22:43:56.777 INFO:teuthology.orchestra.run.smithi138.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T22:43:56.777 INFO:teuthology.orchestra.run.smithi138.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T22:43:56.777 INFO:teuthology.orchestra.run.smithi138.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T22:43:56.777 INFO:teuthology.orchestra.run.smithi138.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T22:43:56.777 INFO:teuthology.orchestra.run.smithi138.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T22:43:56.777 INFO:teuthology.orchestra.run.smithi138.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T22:43:56.777 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:43:56.778 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-09-23T22:43:56.778 INFO:teuthology.orchestra.run.smithi138.stdout:======================================================================================================================= 2023-09-23T22:43:56.778 INFO:teuthology.orchestra.run.smithi138.stdout:Install 8 Packages 2023-09-23T22:43:56.778 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:43:56.778 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 35 M 2023-09-23T22:43:56.778 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 210 M 2023-09-23T22:43:56.779 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-09-23T22:43:57.071 INFO:teuthology.orchestra.run.smithi138.stdout:(1/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 2.2 MB/s | 644 kB 00:00 2023-09-23T22:43:57.096 INFO:teuthology.orchestra.run.smithi138.stdout:(2/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 540 kB/s | 171 kB 00:00 2023-09-23T22:43:57.279 INFO:teuthology.orchestra.run.smithi138.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 20 MB/s | 3.7 MB 00:00 2023-09-23T22:43:57.421 INFO:teuthology.orchestra.run.smithi138.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 1.6 MB/s | 225 kB 00:00 2023-09-23T22:43:57.431 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T22:43:57.431 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:08 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:43:57.572 INFO:teuthology.orchestra.run.smithi138.stdout:(5/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 32 MB/s | 4.9 MB 00:00 2023-09-23T22:43:57.772 INFO:teuthology.orchestra.run.smithi138.stdout:(6/8): python3-scipy-1.0.0-21.module+el8.5.0+10 20 MB/s | 14 MB 00:00 2023-09-23T22:43:57.864 INFO:teuthology.orchestra.run.smithi138.stdout:(7/8): ceph-mgr-diskprediction-local-17.2.6-112 6.8 MB/s | 7.4 MB 00:01 2023-09-23T22:43:58.039 INFO:teuthology.orchestra.run.smithi138.stdout:(8/8): openblas-0.3.15-3.el8.x86_64.rpm 10 MB/s | 4.7 MB 00:00 2023-09-23T22:43:58.040 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:43:58.040 INFO:teuthology.orchestra.run.smithi138.stdout:Total 28 MB/s | 35 MB 00:01 2023-09-23T22:43:58.278 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-09-23T22:43:58.296 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-09-23T22:43:58.296 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-09-23T22:43:58.588 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-09-23T22:43:58.592 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-09-23T22:43:59.141 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-09-23T22:43:59.260 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T22:43:59.374 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T22:43:59.494 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T22:44:00.096 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T22:44:00.277 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T22:44:00.871 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T22:44:01.060 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T22:44:01.730 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T22:44:02.030 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T22:44:02.107 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-09-23T22:44:02.235 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T22:44:02.293 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:02.293 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-09-23T22:44:02.293 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-grafana-dashboards-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T22:44:02.293 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-dashboard-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T22:44:02.293 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-prometheus-alerts-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T22:44:02.294 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T22:44:02.294 INFO:teuthology.orchestra.run.smithi110.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T22:44:02.294 INFO:teuthology.orchestra.run.smithi110.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T22:44:02.294 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:02.294 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:44:02.425 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T22:44:02.769 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:44:03.648 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:14 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:44:03.986 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T22:44:05.053 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:44:05.054 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================================================= 2023-09-23T22:44:05.054 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-09-23T22:44:05.054 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================================================= 2023-09-23T22:44:05.054 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-09-23T22:44:05.054 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 7.4 M 2023-09-23T22:44:05.054 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-09-23T22:44:05.054 INFO:teuthology.orchestra.run.smithi110.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T22:44:05.054 INFO:teuthology.orchestra.run.smithi110.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T22:44:05.054 INFO:teuthology.orchestra.run.smithi110.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T22:44:05.054 INFO:teuthology.orchestra.run.smithi110.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T22:44:05.055 INFO:teuthology.orchestra.run.smithi110.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T22:44:05.055 INFO:teuthology.orchestra.run.smithi110.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T22:44:05.055 INFO:teuthology.orchestra.run.smithi110.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T22:44:05.055 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:05.055 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-09-23T22:44:05.055 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================================================= 2023-09-23T22:44:05.055 INFO:teuthology.orchestra.run.smithi110.stdout:Install 8 Packages 2023-09-23T22:44:05.055 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:05.056 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 35 M 2023-09-23T22:44:05.056 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 210 M 2023-09-23T22:44:05.056 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-09-23T22:44:05.131 INFO:teuthology.orchestra.run.smithi110.stdout:(1/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 2.2 MB/s | 171 kB 00:00 2023-09-23T22:44:05.156 INFO:teuthology.orchestra.run.smithi110.stdout:(2/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 6.3 MB/s | 644 kB 00:00 2023-09-23T22:44:05.200 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T22:44:05.282 INFO:teuthology.orchestra.run.smithi110.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 30 MB/s | 3.7 MB 00:00 2023-09-23T22:44:05.332 INFO:teuthology.orchestra.run.smithi110.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 4.4 MB/s | 225 kB 00:00 2023-09-23T22:44:05.384 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1125.gc9bb1bc 8/8 2023-09-23T22:44:05.532 INFO:teuthology.orchestra.run.smithi110.stdout:(5/8): python3-scipy-1.0.0-21.module+el8.5.0+10 34 MB/s | 14 MB 00:00 2023-09-23T22:44:05.624 INFO:teuthology.orchestra.run.smithi110.stdout:(6/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 17 MB/s | 4.9 MB 00:00 2023-09-23T22:44:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1125.gc9bb1bc 8/8 2023-09-23T22:44:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1125.gc9bb1bc 1/8 2023-09-23T22:44:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T22:44:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-09-23T22:44:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-09-23T22:44:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T22:44:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T22:44:05.635 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-09-23T22:44:05.716 INFO:teuthology.orchestra.run.smithi110.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 26 MB/s | 4.7 MB 00:00 2023-09-23T22:44:05.893 INFO:teuthology.orchestra.run.smithi110.stdout:(8/8): ceph-mgr-diskprediction-local-17.2.6-112 8.8 MB/s | 7.4 MB 00:00 2023-09-23T22:44:05.894 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:44:05.894 INFO:teuthology.orchestra.run.smithi110.stdout:Total 42 MB/s | 35 MB 00:00 2023-09-23T22:44:06.131 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-09-23T22:44:06.149 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-09-23T22:44:06.149 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-09-23T22:44:06.444 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-09-23T22:44:06.448 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-09-23T22:44:06.692 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-09-23T22:44:06.692 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:23 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:44:06.990 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-09-23T22:44:07.156 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T22:44:07.287 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T22:44:07.423 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T22:44:08.057 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T22:44:08.256 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T22:44:08.840 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T22:44:09.073 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T22:44:09.287 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-09-23T22:44:09.476 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:09.476 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-09-23T22:44:09.476 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T22:44:09.476 INFO:teuthology.orchestra.run.smithi138.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T22:44:09.476 INFO:teuthology.orchestra.run.smithi138.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T22:44:09.476 INFO:teuthology.orchestra.run.smithi138.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T22:44:09.476 INFO:teuthology.orchestra.run.smithi138.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T22:44:09.476 INFO:teuthology.orchestra.run.smithi138.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T22:44:09.477 INFO:teuthology.orchestra.run.smithi138.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T22:44:09.477 INFO:teuthology.orchestra.run.smithi138.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T22:44:09.477 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:09.477 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:44:09.637 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-mgr-rook 2023-09-23T22:44:09.748 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T22:44:09.985 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:44:10.050 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T22:44:10.256 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T22:44:10.934 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:27 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:44:12.010 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T22:44:12.315 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:44:12.317 INFO:teuthology.orchestra.run.smithi138.stdout:======================================================================================================== 2023-09-23T22:44:12.317 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-09-23T22:44:12.317 INFO:teuthology.orchestra.run.smithi138.stdout:======================================================================================================== 2023-09-23T22:44:12.317 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-09-23T22:44:12.318 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-rook noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 47 k 2023-09-23T22:44:12.318 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-09-23T22:44:12.318 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T22:44:12.318 INFO:teuthology.orchestra.run.smithi138.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T22:44:12.318 INFO:teuthology.orchestra.run.smithi138.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T22:44:12.318 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T22:44:12.318 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T22:44:12.318 INFO:teuthology.orchestra.run.smithi138.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T22:44:12.318 INFO:teuthology.orchestra.run.smithi138.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T22:44:12.318 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T22:44:12.318 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T22:44:12.319 INFO:teuthology.orchestra.run.smithi138.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T22:44:12.319 INFO:teuthology.orchestra.run.smithi138.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T22:44:12.319 INFO:teuthology.orchestra.run.smithi138.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T22:44:12.319 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:12.319 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-09-23T22:44:12.319 INFO:teuthology.orchestra.run.smithi138.stdout:======================================================================================================== 2023-09-23T22:44:12.319 INFO:teuthology.orchestra.run.smithi138.stdout:Install 13 Packages 2023-09-23T22:44:12.319 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:12.320 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 1.9 M 2023-09-23T22:44:12.320 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 25 M 2023-09-23T22:44:12.320 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-09-23T22:44:12.778 INFO:teuthology.orchestra.run.smithi138.stdout:(1/13): ceph-mgr-rook-17.2.6-1125.gc9bb1bcf.el8 162 kB/s | 47 kB 00:00 2023-09-23T22:44:12.862 INFO:teuthology.orchestra.run.smithi138.stdout:(2/13): python3-certifi-2018.10.15-7.el8.noarch 43 kB/s | 16 kB 00:00 2023-09-23T22:44:12.887 INFO:teuthology.orchestra.run.smithi138.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 82 kB/s | 33 kB 00:00 2023-09-23T22:44:13.087 INFO:teuthology.orchestra.run.smithi138.stdout:(4/13): python3-rsa-4.9-2.el8.noarch.rpm 328 kB/s | 64 kB 00:00 2023-09-23T22:44:13.220 INFO:teuthology.orchestra.run.smithi138.stdout:(5/13): python3-websocket-client-0.56.0-5.el8.n 459 kB/s | 61 kB 00:00 2023-09-23T22:44:13.252 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T22:44:13.271 INFO:teuthology.orchestra.run.smithi138.stdout:(6/13): python3-google-auth-1.1.1-10.el8.noarch 165 kB/s | 81 kB 00:00 2023-09-23T22:44:13.296 INFO:teuthology.orchestra.run.smithi138.stdout:(7/13): python3-requests-oauthlib-1.0.0-1.el8.n 567 kB/s | 43 kB 00:00 2023-09-23T22:44:13.355 INFO:teuthology.orchestra.run.smithi138.stdout:(8/13): python3-pyasn1-modules-0.3.7-6.el8.noar 2.0 MB/s | 110 kB 00:00 2023-09-23T22:44:13.380 INFO:teuthology.orchestra.run.smithi138.stdout:(9/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 1.4 MB/s | 155 kB 00:00 2023-09-23T22:44:13.405 INFO:teuthology.orchestra.run.smithi138.stdout:(10/13): python3-jsonpatch-1.21-2.el8.noarch.rp 528 kB/s | 27 kB 00:00 2023-09-23T22:44:13.405 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1125.gc9bb1bc 8/8 2023-09-23T22:44:13.431 INFO:teuthology.orchestra.run.smithi138.stdout:(11/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 2.4 MB/s | 126 kB 00:00 2023-09-23T22:44:13.447 INFO:teuthology.orchestra.run.smithi138.stdout:(12/13): python3-jsonpointer-1.10-11.el8.noarch 476 kB/s | 20 kB 00:00 2023-09-23T22:44:13.481 INFO:teuthology.orchestra.run.smithi138.stdout:(13/13): python3-kubernetes-11.0.0-6.el8.noarch 1.9 MB/s | 1.2 MB 00:00 2023-09-23T22:44:13.483 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:44:13.483 INFO:teuthology.orchestra.run.smithi138.stdout:Total 1.7 MB/s | 1.9 MB 00:01 2023-09-23T22:44:13.543 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-09-23T22:44:13.558 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-09-23T22:44:13.558 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-09-23T22:44:13.629 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-09-23T22:44:13.632 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-09-23T22:44:13.654 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1125.gc9bb1bc 8/8 2023-09-23T22:44:13.654 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1125.gc9bb1bc 1/8 2023-09-23T22:44:13.654 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T22:44:13.654 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-09-23T22:44:13.654 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-09-23T22:44:13.655 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T22:44:13.655 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T22:44:13.655 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-09-23T22:44:14.099 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-09-23T22:44:14.332 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T22:44:14.568 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T22:44:14.742 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-09-23T22:44:14.742 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:25 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:44:14.747 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T22:44:14.915 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-09-23T22:44:15.127 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-09-23T22:44:15.345 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-23T22:44:15.577 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-23T22:44:15.765 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T22:44:15.930 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T22:44:16.137 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T22:44:17.121 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-09-23T22:44:17.215 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T22:44:17.308 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:17.308 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-09-23T22:44:17.308 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T22:44:17.308 INFO:teuthology.orchestra.run.smithi110.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T22:44:17.308 INFO:teuthology.orchestra.run.smithi110.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T22:44:17.308 INFO:teuthology.orchestra.run.smithi110.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T22:44:17.308 INFO:teuthology.orchestra.run.smithi110.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T22:44:17.308 INFO:teuthology.orchestra.run.smithi110.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T22:44:17.308 INFO:teuthology.orchestra.run.smithi110.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T22:44:17.308 INFO:teuthology.orchestra.run.smithi110.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T22:44:17.309 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:17.309 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:44:17.475 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-mgr-rook 2023-09-23T22:44:17.573 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T22:44:17.724 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-mgr-rook-2:17.2.6-1125.gc9bb1bcf.el8.noarch 13/13 2023-09-23T22:44:17.822 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:44:17.963 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1125.gc9bb1bcf.el8.noarch 13/13 2023-09-23T22:44:17.964 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/13 2023-09-23T22:44:17.964 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T22:44:17.964 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T22:44:17.964 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T22:44:17.964 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T22:44:17.964 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T22:44:17.964 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T22:44:17.964 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-09-23T22:44:17.964 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-09-23T22:44:17.965 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-09-23T22:44:17.965 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-09-23T22:44:17.965 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-09-23T22:44:18.636 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:29 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:44:19.052 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-09-23T22:44:19.052 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:36 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:44:20.071 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:44:20.073 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================================== 2023-09-23T22:44:20.073 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-09-23T22:44:20.073 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================================== 2023-09-23T22:44:20.073 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-09-23T22:44:20.073 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-rook noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 47 k 2023-09-23T22:44:20.073 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-09-23T22:44:20.073 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T22:44:20.073 INFO:teuthology.orchestra.run.smithi110.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T22:44:20.073 INFO:teuthology.orchestra.run.smithi110.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T22:44:20.073 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T22:44:20.074 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T22:44:20.074 INFO:teuthology.orchestra.run.smithi110.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T22:44:20.074 INFO:teuthology.orchestra.run.smithi110.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T22:44:20.074 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T22:44:20.074 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T22:44:20.074 INFO:teuthology.orchestra.run.smithi110.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T22:44:20.074 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T22:44:20.074 INFO:teuthology.orchestra.run.smithi110.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T22:44:20.074 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:20.074 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-09-23T22:44:20.074 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================================== 2023-09-23T22:44:20.074 INFO:teuthology.orchestra.run.smithi110.stdout:Install 13 Packages 2023-09-23T22:44:20.075 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:20.075 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 1.9 M 2023-09-23T22:44:20.075 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 25 M 2023-09-23T22:44:20.075 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-09-23T22:44:20.273 INFO:teuthology.orchestra.run.smithi110.stdout:(1/13): ceph-mgr-rook-17.2.6-1125.gc9bb1bcf.el8 282 kB/s | 47 kB 00:00 2023-09-23T22:44:20.314 INFO:teuthology.orchestra.run.smithi110.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 157 kB/s | 33 kB 00:00 2023-09-23T22:44:20.340 INFO:teuthology.orchestra.run.smithi110.stdout:(3/13): python3-certifi-2018.10.15-7.el8.noarch 69 kB/s | 16 kB 00:00 2023-09-23T22:44:20.448 INFO:teuthology.orchestra.run.smithi110.stdout:(4/13): python3-rsa-4.9-2.el8.noarch.rpm 592 kB/s | 64 kB 00:00 2023-09-23T22:44:20.474 INFO:teuthology.orchestra.run.smithi110.stdout:(5/13): python3-google-auth-1.1.1-10.el8.noarch 405 kB/s | 81 kB 00:00 2023-09-23T22:44:20.516 INFO:teuthology.orchestra.run.smithi110.stdout:(6/13): python3-websocket-client-0.56.0-5.el8.n 914 kB/s | 61 kB 00:00 2023-09-23T22:44:20.541 INFO:teuthology.orchestra.run.smithi110.stdout:(7/13): python3-requests-oauthlib-1.0.0-1.el8.n 637 kB/s | 43 kB 00:00 2023-09-23T22:44:20.608 INFO:teuthology.orchestra.run.smithi110.stdout:(8/13): python3-pyasn1-modules-0.3.7-6.el8.noar 1.8 MB/s | 110 kB 00:00 2023-09-23T22:44:20.641 INFO:teuthology.orchestra.run.smithi110.stdout:(9/13): python3-kubernetes-11.0.0-6.el8.noarch. 3.6 MB/s | 1.2 MB 00:00 2023-09-23T22:44:20.667 INFO:teuthology.orchestra.run.smithi110.stdout:(10/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 1.0 MB/s | 155 kB 00:00 2023-09-23T22:44:20.700 INFO:teuthology.orchestra.run.smithi110.stdout:(11/13): python3-jsonpatch-1.21-2.el8.noarch.rp 287 kB/s | 27 kB 00:00 2023-09-23T22:44:20.725 INFO:teuthology.orchestra.run.smithi110.stdout:(12/13): python3-jsonpointer-1.10-11.el8.noarch 378 kB/s | 20 kB 00:00 2023-09-23T22:44:20.775 INFO:teuthology.orchestra.run.smithi110.stdout:(13/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 943 kB/s | 126 kB 00:00 2023-09-23T22:44:20.776 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:44:20.776 INFO:teuthology.orchestra.run.smithi110.stdout:Total 2.8 MB/s | 1.9 MB 00:00 2023-09-23T22:44:20.838 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-09-23T22:44:20.850 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-09-23T22:44:20.850 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-09-23T22:44:20.920 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-09-23T22:44:20.923 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-09-23T22:44:21.427 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-09-23T22:44:21.460 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-09-23T22:44:21.647 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:21.647 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-09-23T22:44:21.647 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-rook-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T22:44:21.647 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T22:44:21.647 INFO:teuthology.orchestra.run.smithi138.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T22:44:21.647 INFO:teuthology.orchestra.run.smithi138.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T22:44:21.647 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T22:44:21.648 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T22:44:21.648 INFO:teuthology.orchestra.run.smithi138.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T22:44:21.648 INFO:teuthology.orchestra.run.smithi138.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T22:44:21.648 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T22:44:21.648 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T22:44:21.648 INFO:teuthology.orchestra.run.smithi138.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T22:44:21.648 INFO:teuthology.orchestra.run.smithi138.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T22:44:21.648 INFO:teuthology.orchestra.run.smithi138.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T22:44:21.648 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:21.648 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:44:21.663 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T22:44:21.793 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T22:44:21.916 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T22:44:22.119 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T22:44:22.143 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:44:22.278 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-09-23T22:44:22.534 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-09-23T22:44:22.776 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-23T22:44:22.959 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:39 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:44:22.966 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-23T22:44:23.178 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T22:44:23.328 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T22:44:23.551 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T22:44:24.393 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:44:24.394 INFO:teuthology.orchestra.run.smithi138.stdout:=============================================================================================== 2023-09-23T22:44:24.394 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-09-23T22:44:24.394 INFO:teuthology.orchestra.run.smithi138.stdout:=============================================================================================== 2023-09-23T22:44:24.394 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-09-23T22:44:24.394 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 121 k 2023-09-23T22:44:24.395 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-09-23T22:44:24.395 INFO:teuthology.orchestra.run.smithi138.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-23T22:44:24.395 INFO:teuthology.orchestra.run.smithi138.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T22:44:24.395 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T22:44:24.395 INFO:teuthology.orchestra.run.smithi138.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-23T22:44:24.395 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:24.395 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-09-23T22:44:24.395 INFO:teuthology.orchestra.run.smithi138.stdout:=============================================================================================== 2023-09-23T22:44:24.395 INFO:teuthology.orchestra.run.smithi138.stdout:Install 5 Packages 2023-09-23T22:44:24.395 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:24.396 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 6.0 M 2023-09-23T22:44:24.396 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 26 M 2023-09-23T22:44:24.396 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-09-23T22:44:24.613 INFO:teuthology.orchestra.run.smithi138.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1125.gc9bb1bcf.e 646 kB/s | 121 kB 00:00 2023-09-23T22:44:24.620 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T22:44:24.688 INFO:teuthology.orchestra.run.smithi138.stdout:(2/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 7.1 MB/s | 538 kB 00:00 2023-09-23T22:44:24.814 INFO:teuthology.orchestra.run.smithi138.stdout:(3/5): python3-babel-2.5.1-7.el8.noarch.rpm 38 MB/s | 4.8 MB 00:00 2023-09-23T22:44:24.839 INFO:teuthology.orchestra.run.smithi138.stdout:(4/5): python3-natsort-7.1.1-2.el8.noarch.rpm 146 kB/s | 60 kB 00:00 2023-09-23T22:44:24.846 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T22:44:24.998 INFO:teuthology.orchestra.run.smithi138.stdout:(5/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 832 kB/s | 476 kB 00:00 2023-09-23T22:44:25.000 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:44:25.000 INFO:teuthology.orchestra.run.smithi138.stdout:Total 9.8 MB/s | 6.0 MB 00:00 2023-09-23T22:44:25.029 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-rook-2:17.2.6-1125.gc9bb1bcf.el8.noarch 13/13 2023-09-23T22:44:25.053 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-09-23T22:44:25.065 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-09-23T22:44:25.065 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-09-23T22:44:25.144 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-09-23T22:44:25.148 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-09-23T22:44:25.249 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1125.gc9bb1bcf.el8.noarch 13/13 2023-09-23T22:44:25.249 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/13 2023-09-23T22:44:25.249 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T22:44:25.249 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T22:44:25.249 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T22:44:25.249 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T22:44:25.249 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T22:44:25.249 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T22:44:25.249 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-09-23T22:44:25.249 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-09-23T22:44:25.250 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-09-23T22:44:25.250 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-09-23T22:44:25.250 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-09-23T22:44:26.101 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-09-23T22:44:26.329 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-09-23T22:44:26.330 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:37 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:44:26.484 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-23T22:44:26.724 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-23T22:44:27.060 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T22:44:27.333 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-23T22:44:27.506 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 5/5 2023-09-23T22:44:27.769 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 5/5 2023-09-23T22:44:27.769 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/5 2023-09-23T22:44:27.769 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-23T22:44:27.769 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T22:44:27.769 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-23T22:44:28.894 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-09-23T22:44:29.076 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-23T22:44:29.076 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:46 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:44:29.079 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:29.079 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-09-23T22:44:29.079 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-rook-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T22:44:29.080 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T22:44:29.080 INFO:teuthology.orchestra.run.smithi110.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T22:44:29.080 INFO:teuthology.orchestra.run.smithi110.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T22:44:29.080 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T22:44:29.080 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T22:44:29.080 INFO:teuthology.orchestra.run.smithi110.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T22:44:29.080 INFO:teuthology.orchestra.run.smithi110.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T22:44:29.080 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T22:44:29.080 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T22:44:29.080 INFO:teuthology.orchestra.run.smithi110.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T22:44:29.081 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T22:44:29.081 INFO:teuthology.orchestra.run.smithi110.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T22:44:29.081 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:29.081 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:44:29.215 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T22:44:29.565 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:44:30.465 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:41 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:44:31.520 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-09-23T22:44:31.703 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:31.703 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-09-23T22:44:31.703 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T22:44:31.703 INFO:teuthology.orchestra.run.smithi138.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-23T22:44:31.704 INFO:teuthology.orchestra.run.smithi138.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T22:44:31.704 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T22:44:31.704 INFO:teuthology.orchestra.run.smithi138.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-23T22:44:31.704 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:31.704 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:44:31.824 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:44:31.825 INFO:teuthology.orchestra.run.smithi110.stdout:=============================================================================================== 2023-09-23T22:44:31.825 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-09-23T22:44:31.825 INFO:teuthology.orchestra.run.smithi110.stdout:=============================================================================================== 2023-09-23T22:44:31.825 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-09-23T22:44:31.825 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 121 k 2023-09-23T22:44:31.825 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-09-23T22:44:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-23T22:44:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T22:44:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T22:44:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-23T22:44:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:31.826 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-09-23T22:44:31.826 INFO:teuthology.orchestra.run.smithi110.stdout:=============================================================================================== 2023-09-23T22:44:31.826 INFO:teuthology.orchestra.run.smithi110.stdout:Install 5 Packages 2023-09-23T22:44:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:31.827 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 6.0 M 2023-09-23T22:44:31.827 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 26 M 2023-09-23T22:44:31.827 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-09-23T22:44:31.851 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-fuse 2023-09-23T22:44:31.961 INFO:teuthology.orchestra.run.smithi110.stdout:(1/5): python3-natsort-7.1.1-2.el8.noarch.rpm 580 kB/s | 60 kB 00:00 2023-09-23T22:44:32.019 INFO:teuthology.orchestra.run.smithi110.stdout:(2/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 9.1 MB/s | 538 kB 00:00 2023-09-23T22:44:32.044 INFO:teuthology.orchestra.run.smithi110.stdout:(3/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 2.5 MB/s | 476 kB 00:00 2023-09-23T22:44:32.145 INFO:teuthology.orchestra.run.smithi110.stdout:(4/5): python3-babel-2.5.1-7.el8.noarch.rpm 38 MB/s | 4.8 MB 00:00 2023-09-23T22:44:32.203 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:44:32.220 INFO:teuthology.orchestra.run.smithi110.stdout:(5/5): ceph-mgr-cephadm-17.2.6-1125.gc9bb1bcf.e 333 kB/s | 121 kB 00:00 2023-09-23T22:44:32.221 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:44:32.221 INFO:teuthology.orchestra.run.smithi110.stdout:Total 15 MB/s | 6.0 MB 00:00 2023-09-23T22:44:32.273 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-09-23T22:44:32.284 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-09-23T22:44:32.285 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-09-23T22:44:32.362 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-09-23T22:44:32.366 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-09-23T22:44:33.045 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:50 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:44:33.299 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-09-23T22:44:33.658 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-23T22:44:33.923 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-23T22:44:34.257 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T22:44:34.449 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:44:34.449 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:44:34.449 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-09-23T22:44:34.449 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:44:34.449 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-09-23T22:44:34.450 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-fuse 2023-09-23T22:44:34.450 INFO:teuthology.orchestra.run.smithi138.stdout: x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 874 k 2023-09-23T22:44:34.450 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-09-23T22:44:34.450 INFO:teuthology.orchestra.run.smithi138.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T22:44:34.450 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:34.450 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-09-23T22:44:34.450 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:44:34.450 INFO:teuthology.orchestra.run.smithi138.stdout:Install 2 Packages 2023-09-23T22:44:34.450 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:34.451 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 957 k 2023-09-23T22:44:34.451 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 3.0 M 2023-09-23T22:44:34.451 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-09-23T22:44:34.546 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-23T22:44:34.668 INFO:teuthology.orchestra.run.smithi138.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 382 kB/s | 83 kB 00:00 2023-09-23T22:44:34.737 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 5/5 2023-09-23T22:44:34.954 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 5/5 2023-09-23T22:44:34.954 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/5 2023-09-23T22:44:34.954 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-23T22:44:34.954 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T22:44:34.954 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-23T22:44:35.284 INFO:teuthology.orchestra.run.smithi138.stdout:(2/2): ceph-fuse-17.2.6-1125.gc9bb1bcf.el8.x86_ 1.0 MB/s | 874 kB 00:00 2023-09-23T22:44:35.285 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:44:35.285 INFO:teuthology.orchestra.run.smithi138.stdout:Total 1.1 MB/s | 957 kB 00:00 2023-09-23T22:44:35.297 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-09-23T22:44:35.318 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-09-23T22:44:35.318 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-09-23T22:44:35.393 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-09-23T22:44:35.397 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-09-23T22:44:35.562 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-09-23T22:44:35.839 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T22:44:36.011 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/2 2023-09-23T22:44:36.028 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-23T22:44:36.028 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:47 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:44:36.334 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/2 2023-09-23T22:44:36.334 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/2 2023-09-23T22:44:37.404 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T22:44:37.404 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:54 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:44:40.002 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-09-23T22:44:40.184 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:40.184 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-09-23T22:44:40.184 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T22:44:40.184 INFO:teuthology.orchestra.run.smithi110.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-23T22:44:40.185 INFO:teuthology.orchestra.run.smithi110.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T22:44:40.185 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T22:44:40.185 INFO:teuthology.orchestra.run.smithi110.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-23T22:44:40.185 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:40.185 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:44:40.323 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-fuse 2023-09-23T22:44:40.677 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:44:41.321 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-09-23T22:44:41.503 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:41.503 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-09-23T22:44:41.503 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T22:44:41.503 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:41.503 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:44:41.609 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:52 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:44:41.656 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install librados-devel 2023-09-23T22:44:42.015 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:44:43.051 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:44:43.052 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:44:43.052 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-09-23T22:44:43.052 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:44:43.052 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-09-23T22:44:43.052 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-fuse 2023-09-23T22:44:43.052 INFO:teuthology.orchestra.run.smithi110.stdout: x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 874 k 2023-09-23T22:44:43.052 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-09-23T22:44:43.052 INFO:teuthology.orchestra.run.smithi110.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T22:44:43.052 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:43.053 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-09-23T22:44:43.053 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:44:43.053 INFO:teuthology.orchestra.run.smithi110.stdout:Install 2 Packages 2023-09-23T22:44:43.053 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:43.053 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 957 k 2023-09-23T22:44:43.053 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 3.0 M 2023-09-23T22:44:43.054 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-09-23T22:44:43.261 INFO:teuthology.orchestra.run.smithi110.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 398 kB/s | 83 kB 00:00 2023-09-23T22:44:43.591 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:00 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:44:44.415 INFO:teuthology.orchestra.run.smithi110.stdout:(2/2): ceph-fuse-17.2.6-1125.gc9bb1bcf.el8.x86_ 642 kB/s | 874 kB 00:01 2023-09-23T22:44:44.416 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:44:44.417 INFO:teuthology.orchestra.run.smithi110.stdout:Total 702 kB/s | 957 kB 00:01 2023-09-23T22:44:44.430 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-09-23T22:44:44.452 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-09-23T22:44:44.453 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-09-23T22:44:44.530 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-09-23T22:44:44.534 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-09-23T22:44:44.881 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-09-23T22:44:45.019 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:44:45.020 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:44:45.020 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repo Size 2023-09-23T22:44:45.020 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:44:45.020 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-09-23T22:44:45.021 INFO:teuthology.orchestra.run.smithi138.stdout: librados-devel x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 130 k 2023-09-23T22:44:45.021 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:45.021 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-09-23T22:44:45.021 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:44:45.021 INFO:teuthology.orchestra.run.smithi138.stdout:Install 1 Package 2023-09-23T22:44:45.021 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:45.021 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 130 k 2023-09-23T22:44:45.022 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 464 k 2023-09-23T22:44:45.022 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-09-23T22:44:45.136 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T22:44:45.239 INFO:teuthology.orchestra.run.smithi138.stdout:librados-devel-17.2.6-1125.gc9bb1bcf.el8.x86_64 598 kB/s | 130 kB 00:00 2023-09-23T22:44:45.239 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:44:45.239 INFO:teuthology.orchestra.run.smithi138.stdout:Total 594 kB/s | 130 kB 00:00 2023-09-23T22:44:45.240 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-09-23T22:44:45.245 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-09-23T22:44:45.246 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-09-23T22:44:45.299 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-09-23T22:44:45.302 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-09-23T22:44:45.334 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/2 2023-09-23T22:44:45.419 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-09-23T22:44:45.609 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : librados-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:44:45.657 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/2 2023-09-23T22:44:45.657 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/2 2023-09-23T22:44:45.829 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: librados-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:44:46.683 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T22:44:46.683 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:57 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:44:46.878 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : librados-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:44:46.879 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:03 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:44:51.022 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-09-23T22:44:51.172 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-09-23T22:44:51.203 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:51.203 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-09-23T22:44:51.203 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T22:44:51.203 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:51.203 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:44:51.321 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install librados-devel 2023-09-23T22:44:51.352 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:51.353 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-09-23T22:44:51.353 INFO:teuthology.orchestra.run.smithi138.stdout: librados-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:44:51.353 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:51.353 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:44:51.494 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install libcephfs2 2023-09-23T22:44:51.676 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:44:51.848 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:44:52.549 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:03 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:44:52.700 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:09 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:44:53.906 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:44:53.907 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:44:53.907 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repo Size 2023-09-23T22:44:53.908 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:44:53.908 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-09-23T22:44:53.908 INFO:teuthology.orchestra.run.smithi110.stdout: librados-devel x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 130 k 2023-09-23T22:44:53.908 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:53.908 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-09-23T22:44:53.908 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:44:53.908 INFO:teuthology.orchestra.run.smithi110.stdout:Install 1 Package 2023-09-23T22:44:53.908 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:53.908 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 130 k 2023-09-23T22:44:53.908 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 464 k 2023-09-23T22:44:53.908 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-09-23T22:44:53.946 INFO:teuthology.orchestra.run.smithi138.stdout:Package libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:44:54.056 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:44:54.057 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-09-23T22:44:54.057 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:44:54.176 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install libcephfs-devel 2023-09-23T22:44:54.226 INFO:teuthology.orchestra.run.smithi110.stdout:librados-devel-17.2.6-1125.gc9bb1bcf.el8.x86_64 408 kB/s | 130 kB 00:00 2023-09-23T22:44:54.227 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:44:54.227 INFO:teuthology.orchestra.run.smithi110.stdout:Total 406 kB/s | 130 kB 00:00 2023-09-23T22:44:54.227 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-09-23T22:44:54.233 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-09-23T22:44:54.233 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-09-23T22:44:54.286 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-09-23T22:44:54.290 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-09-23T22:44:54.407 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-09-23T22:44:54.527 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:44:54.581 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : librados-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:44:54.837 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librados-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:44:56.035 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:13 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:44:56.068 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librados-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:44:56.068 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:07 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:44:57.457 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:44:57.458 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:44:57.458 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repo Size 2023-09-23T22:44:57.458 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:44:57.458 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-09-23T22:44:57.458 INFO:teuthology.orchestra.run.smithi138.stdout: libcephfs-devel x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 25 k 2023-09-23T22:44:57.459 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:57.459 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-09-23T22:44:57.459 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:44:57.459 INFO:teuthology.orchestra.run.smithi138.stdout:Install 1 Package 2023-09-23T22:44:57.459 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:44:57.459 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 25 k 2023-09-23T22:44:57.459 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 109 k 2023-09-23T22:44:57.459 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-09-23T22:44:57.594 INFO:teuthology.orchestra.run.smithi138.stdout:libcephfs-devel-17.2.6-1125.gc9bb1bcf.el8.x86_6 184 kB/s | 25 kB 00:00 2023-09-23T22:44:57.595 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:44:57.595 INFO:teuthology.orchestra.run.smithi138.stdout:Total 182 kB/s | 25 kB 00:00 2023-09-23T22:44:57.595 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-09-23T22:44:57.599 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-09-23T22:44:57.599 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-09-23T22:44:57.601 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-09-23T22:44:57.604 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-09-23T22:44:57.656 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-09-23T22:44:57.832 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : libcephfs-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:44:58.049 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:44:59.023 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libcephfs-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:44:59.023 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:16 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:44:59.483 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-09-23T22:44:59.665 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:59.665 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-09-23T22:44:59.665 INFO:teuthology.orchestra.run.smithi110.stdout: librados-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:44:59.665 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:44:59.665 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:44:59.816 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install libcephfs2 2023-09-23T22:45:00.171 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:01.431 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:12 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:45:02.765 INFO:teuthology.orchestra.run.smithi110.stdout:Package libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:45:02.874 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:45:02.875 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-09-23T22:45:02.875 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:45:03.002 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install libcephfs-devel 2023-09-23T22:45:03.356 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:04.902 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:15 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:45:05.940 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-09-23T22:45:06.121 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:45:06.121 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-09-23T22:45:06.121 INFO:teuthology.orchestra.run.smithi138.stdout: libcephfs-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:45:06.121 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:45:06.121 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:45:06.265 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install librados2 2023-09-23T22:45:06.276 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:45:06.276 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:45:06.276 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repo Size 2023-09-23T22:45:06.276 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:45:06.277 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-09-23T22:45:06.277 INFO:teuthology.orchestra.run.smithi110.stdout: libcephfs-devel x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 25 k 2023-09-23T22:45:06.277 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:45:06.277 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-09-23T22:45:06.277 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:45:06.277 INFO:teuthology.orchestra.run.smithi110.stdout:Install 1 Package 2023-09-23T22:45:06.277 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:45:06.278 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 25 k 2023-09-23T22:45:06.278 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 109 k 2023-09-23T22:45:06.278 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-09-23T22:45:06.418 INFO:teuthology.orchestra.run.smithi110.stdout:libcephfs-devel-17.2.6-1125.gc9bb1bcf.el8.x86_6 177 kB/s | 25 kB 00:00 2023-09-23T22:45:06.418 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:45:06.418 INFO:teuthology.orchestra.run.smithi110.stdout:Total 176 kB/s | 25 kB 00:00 2023-09-23T22:45:06.419 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-09-23T22:45:06.422 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-09-23T22:45:06.423 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-09-23T22:45:06.424 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-09-23T22:45:06.428 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-09-23T22:45:06.478 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-09-23T22:45:06.617 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:06.671 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libcephfs-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:06.887 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:07.884 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libcephfs-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:07.884 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:18 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:45:07.981 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:24 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:45:09.230 INFO:teuthology.orchestra.run.smithi138.stdout:Package librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:45:09.339 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:45:09.340 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-09-23T22:45:09.341 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:45:09.464 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install librbd1 2023-09-23T22:45:09.818 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:11.350 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:28 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:45:12.609 INFO:teuthology.orchestra.run.smithi138.stdout:Package librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:45:12.719 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:45:12.721 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-09-23T22:45:12.721 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:45:12.848 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install python3-rados 2023-09-23T22:45:13.200 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:13.465 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-09-23T22:45:13.643 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:45:13.643 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-09-23T22:45:13.643 INFO:teuthology.orchestra.run.smithi110.stdout: libcephfs-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:45:13.643 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:45:13.643 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:45:13.773 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install librados2 2023-09-23T22:45:14.124 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:14.213 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:31 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:45:15.464 INFO:teuthology.orchestra.run.smithi138.stdout:Package python3-rados-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:45:15.576 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:45:15.578 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-09-23T22:45:15.578 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:45:15.603 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:26 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:45:15.711 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install python3-rgw 2023-09-23T22:45:16.063 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:16.854 INFO:teuthology.orchestra.run.smithi110.stdout:Package librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:45:16.962 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:45:16.963 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-09-23T22:45:16.963 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:45:17.087 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install librbd1 2023-09-23T22:45:17.439 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:17.584 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:34 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:45:18.835 INFO:teuthology.orchestra.run.smithi138.stdout:Package python3-rgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:45:18.944 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:45:18.945 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-09-23T22:45:18.945 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:45:18.978 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:29 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:45:19.058 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install python3-cephfs 2023-09-23T22:45:19.410 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:20.233 INFO:teuthology.orchestra.run.smithi110.stdout:Package librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:45:20.341 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:45:20.342 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-09-23T22:45:20.342 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:45:20.463 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install python3-rados 2023-09-23T22:45:20.815 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:21.076 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:38 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:45:22.343 INFO:teuthology.orchestra.run.smithi138.stdout:Package python3-cephfs-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:45:22.373 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:33 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:45:22.452 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:45:22.453 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-09-23T22:45:22.453 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:45:22.586 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install python3-rbd 2023-09-23T22:45:22.940 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:23.626 INFO:teuthology.orchestra.run.smithi110.stdout:Package python3-rados-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:45:23.735 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:45:23.737 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-09-23T22:45:23.737 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:45:23.852 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install python3-rgw 2023-09-23T22:45:24.204 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:24.223 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:41 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:45:25.164 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:36 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:45:25.556 INFO:teuthology.orchestra.run.smithi138.stdout:Package python3-rbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:45:25.663 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:45:25.665 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-09-23T22:45:25.665 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:45:25.791 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install rbd-fuse 2023-09-23T22:45:26.143 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:26.442 INFO:teuthology.orchestra.run.smithi110.stdout:Package python3-rgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:45:26.552 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:45:26.553 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-09-23T22:45:26.553 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:45:26.693 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install python3-cephfs 2023-09-23T22:45:26.958 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:43 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:45:27.045 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:27.881 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:38 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:45:28.392 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:45:28.392 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:45:28.393 INFO:teuthology.orchestra.run.smithi138.stdout: Package Architecture Version Repository Size 2023-09-23T22:45:28.393 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:45:28.393 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-09-23T22:45:28.393 INFO:teuthology.orchestra.run.smithi138.stdout: rbd-fuse x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 90 k 2023-09-23T22:45:28.393 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:45:28.393 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-09-23T22:45:28.393 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:45:28.393 INFO:teuthology.orchestra.run.smithi138.stdout:Install 1 Package 2023-09-23T22:45:28.393 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:45:28.394 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 90 k 2023-09-23T22:45:28.394 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 264 k 2023-09-23T22:45:28.394 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-09-23T22:45:28.738 INFO:teuthology.orchestra.run.smithi138.stdout:rbd-fuse-17.2.6-1125.gc9bb1bcf.el8.x86_64.rpm 262 kB/s | 90 kB 00:00 2023-09-23T22:45:28.738 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:45:28.739 INFO:teuthology.orchestra.run.smithi138.stdout:Total 261 kB/s | 90 kB 00:00 2023-09-23T22:45:28.739 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-09-23T22:45:28.749 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-09-23T22:45:28.749 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-09-23T22:45:28.801 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-09-23T22:45:28.805 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-09-23T22:45:28.913 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-09-23T22:45:29.068 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : rbd-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:29.126 INFO:teuthology.orchestra.run.smithi110.stdout:Package python3-cephfs-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:45:29.233 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:45:29.234 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-09-23T22:45:29.235 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:45:29.303 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:29.342 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install python3-rbd 2023-09-23T22:45:29.693 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:30.340 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : rbd-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:30.340 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:47 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:45:30.492 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:41 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:45:31.820 INFO:teuthology.orchestra.run.smithi110.stdout:Package python3-rbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T22:45:31.931 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:45:31.932 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-09-23T22:45:31.932 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:45:32.045 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install rbd-fuse 2023-09-23T22:45:32.396 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:32.951 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-09-23T22:45:33.132 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:45:33.132 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-09-23T22:45:33.132 INFO:teuthology.orchestra.run.smithi138.stdout: rbd-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:45:33.132 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:45:33.132 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:45:33.269 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install rbd-mirror 2023-09-23T22:45:33.283 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:44 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:45:33.624 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:34.465 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:51 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:45:34.713 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:45:34.713 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:45:34.714 INFO:teuthology.orchestra.run.smithi110.stdout: Package Architecture Version Repository Size 2023-09-23T22:45:34.714 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:45:34.714 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-09-23T22:45:34.714 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-fuse x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 90 k 2023-09-23T22:45:34.714 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:45:34.714 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-09-23T22:45:34.714 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:45:34.715 INFO:teuthology.orchestra.run.smithi110.stdout:Install 1 Package 2023-09-23T22:45:34.715 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:45:34.715 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 90 k 2023-09-23T22:45:34.715 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 264 k 2023-09-23T22:45:34.716 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-09-23T22:45:34.911 INFO:teuthology.orchestra.run.smithi110.stdout:rbd-fuse-17.2.6-1125.gc9bb1bcf.el8.x86_64.rpm 460 kB/s | 90 kB 00:00 2023-09-23T22:45:34.912 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:45:34.912 INFO:teuthology.orchestra.run.smithi110.stdout:Total 457 kB/s | 90 kB 00:00 2023-09-23T22:45:34.912 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-09-23T22:45:34.922 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-09-23T22:45:34.922 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-09-23T22:45:34.975 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-09-23T22:45:34.979 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-09-23T22:45:35.101 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-09-23T22:45:35.309 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : rbd-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:35.539 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:35.824 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:45:35.825 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:45:35.825 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repo Size 2023-09-23T22:45:35.825 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:45:35.826 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-09-23T22:45:35.826 INFO:teuthology.orchestra.run.smithi138.stdout: rbd-mirror x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 3.5 M 2023-09-23T22:45:35.826 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:45:35.826 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-09-23T22:45:35.826 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:45:35.826 INFO:teuthology.orchestra.run.smithi138.stdout:Install 1 Package 2023-09-23T22:45:35.826 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:45:35.826 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 3.5 M 2023-09-23T22:45:35.826 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 16 M 2023-09-23T22:45:35.826 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-09-23T22:45:36.483 INFO:teuthology.orchestra.run.smithi138.stdout:rbd-mirror-17.2.6-1125.gc9bb1bcf.el8.x86_64.rpm 5.3 MB/s | 3.5 MB 00:00 2023-09-23T22:45:36.483 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:45:36.484 INFO:teuthology.orchestra.run.smithi138.stdout:Total 5.3 MB/s | 3.5 MB 00:00 2023-09-23T22:45:36.484 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-09-23T22:45:36.496 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-09-23T22:45:36.496 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-09-23T22:45:36.567 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-09-23T22:45:36.571 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-09-23T22:45:36.608 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : rbd-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:36.608 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:47 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:45:37.016 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-09-23T22:45:37.170 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : rbd-mirror-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:37.526 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:38.583 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : rbd-mirror-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:38.583 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:55 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:45:39.044 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-09-23T22:45:39.222 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:45:39.223 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-09-23T22:45:39.223 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:45:39.223 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:45:39.223 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:45:39.357 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install rbd-mirror 2023-09-23T22:45:39.710 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:40.517 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:51 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:45:41.169 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-09-23T22:45:41.351 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:45:41.351 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-09-23T22:45:41.351 INFO:teuthology.orchestra.run.smithi138.stdout: rbd-mirror-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:45:41.351 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:45:41.352 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:45:41.492 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install rbd-nbd 2023-09-23T22:45:41.847 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:41.952 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:45:41.953 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:45:41.953 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repo Size 2023-09-23T22:45:41.953 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:45:41.953 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-09-23T22:45:41.953 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-mirror x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 3.5 M 2023-09-23T22:45:41.953 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:45:41.953 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-09-23T22:45:41.954 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:45:41.954 INFO:teuthology.orchestra.run.smithi110.stdout:Install 1 Package 2023-09-23T22:45:41.954 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:45:41.954 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 3.5 M 2023-09-23T22:45:41.954 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 16 M 2023-09-23T22:45:41.955 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-09-23T22:45:42.539 INFO:teuthology.orchestra.run.smithi110.stdout:rbd-mirror-17.2.6-1125.gc9bb1bcf.el8.x86_64.rpm 5.9 MB/s | 3.5 MB 00:00 2023-09-23T22:45:42.540 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:45:42.540 INFO:teuthology.orchestra.run.smithi110.stdout:Total 5.9 MB/s | 3.5 MB 00:00 2023-09-23T22:45:42.541 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-09-23T22:45:42.552 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-09-23T22:45:42.552 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-09-23T22:45:42.619 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-09-23T22:45:42.623 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-09-23T22:45:42.685 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:59 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:45:43.064 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-09-23T22:45:43.294 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : rbd-mirror-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:43.640 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:44.043 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-09-23T22:45:44.044 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:45:44.044 INFO:teuthology.orchestra.run.smithi138.stdout: Package Architecture Version Repository Size 2023-09-23T22:45:44.044 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:45:44.044 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-09-23T22:45:44.045 INFO:teuthology.orchestra.run.smithi138.stdout: rbd-nbd x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 173 k 2023-09-23T22:45:44.045 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:45:44.045 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-09-23T22:45:44.045 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-09-23T22:45:44.045 INFO:teuthology.orchestra.run.smithi138.stdout:Install 1 Package 2023-09-23T22:45:44.045 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:45:44.046 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 173 k 2023-09-23T22:45:44.046 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 552 k 2023-09-23T22:45:44.046 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-09-23T22:45:44.394 INFO:teuthology.orchestra.run.smithi138.stdout:rbd-nbd-17.2.6-1125.gc9bb1bcf.el8.x86_64.rpm 496 kB/s | 173 kB 00:00 2023-09-23T22:45:44.394 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:45:44.395 INFO:teuthology.orchestra.run.smithi138.stdout:Total 494 kB/s | 173 kB 00:00 2023-09-23T22:45:44.395 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-09-23T22:45:44.405 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-09-23T22:45:44.406 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-09-23T22:45:44.460 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-09-23T22:45:44.464 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-09-23T22:45:44.581 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-09-23T22:45:44.676 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : rbd-mirror-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:44.676 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:55 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:45:44.724 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : rbd-nbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:44.951 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:45.977 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : rbd-nbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:45.977 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:04:02 ago on Sat 23 Sep 2023 10:41:43 PM UTC. 2023-09-23T22:45:47.301 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-09-23T22:45:47.480 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:45:47.480 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-09-23T22:45:47.480 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-mirror-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:45:47.480 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:45:47.481 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:45:47.613 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install rbd-nbd 2023-09-23T22:45:47.969 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-09-23T22:45:48.852 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:59 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:45:48.877 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-09-23T22:45:49.060 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:45:49.060 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-09-23T22:45:49.060 INFO:teuthology.orchestra.run.smithi138.stdout: rbd-nbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:45:49.060 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:45:49.060 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-09-23T22:45:49.200 DEBUG:teuthology.parallel:result is None 2023-09-23T22:45:50.214 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-09-23T22:45:50.214 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:45:50.214 INFO:teuthology.orchestra.run.smithi110.stdout: Package Architecture Version Repository Size 2023-09-23T22:45:50.214 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:45:50.214 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-09-23T22:45:50.215 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-nbd x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 173 k 2023-09-23T22:45:50.215 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:45:50.215 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-09-23T22:45:50.215 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-09-23T22:45:50.215 INFO:teuthology.orchestra.run.smithi110.stdout:Install 1 Package 2023-09-23T22:45:50.215 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:45:50.215 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 173 k 2023-09-23T22:45:50.216 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 552 k 2023-09-23T22:45:50.216 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-09-23T22:45:50.559 INFO:teuthology.orchestra.run.smithi110.stdout:rbd-nbd-17.2.6-1125.gc9bb1bcf.el8.x86_64.rpm 503 kB/s | 173 kB 00:00 2023-09-23T22:45:50.560 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:45:50.560 INFO:teuthology.orchestra.run.smithi110.stdout:Total 501 kB/s | 173 kB 00:00 2023-09-23T22:45:50.560 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-09-23T22:45:50.570 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-09-23T22:45:50.571 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-09-23T22:45:50.624 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-09-23T22:45:50.628 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-09-23T22:45:50.738 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-09-23T22:45:50.949 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : rbd-nbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:51.163 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : rbd-nbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T22:45:52.226 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:04:03 ago on Sat 23 Sep 2023 10:41:49 PM UTC. 2023-09-23T22:45:55.550 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-09-23T22:45:55.736 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:45:55.736 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-09-23T22:45:55.736 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-nbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T22:45:55.736 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:45:55.736 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-09-23T22:45:55.885 DEBUG:teuthology.parallel:result is None 2023-09-23T22:45:55.886 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T22:45:55.886 INFO:teuthology.packaging:ref: None 2023-09-23T22:45:55.886 INFO:teuthology.packaging:tag: None 2023-09-23T22:45:55.886 INFO:teuthology.packaging:branch: quincy 2023-09-23T22:45:55.886 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:45:55.886 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T22:45:56.308 DEBUG:teuthology.orchestra.run.smithi110:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T22:45:56.330 INFO:teuthology.orchestra.run.smithi110.stdout:17.2.6-1125.gc9bb1bcf.el8 2023-09-23T22:45:56.331 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1125.gc9bb1bcf.el8 2023-09-23T22:45:56.331 INFO:teuthology.task.install:The correct ceph version 17.2.6-1125.gc9bb1bcf is installed. 2023-09-23T22:45:56.331 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T22:45:56.331 INFO:teuthology.packaging:ref: None 2023-09-23T22:45:56.331 INFO:teuthology.packaging:tag: None 2023-09-23T22:45:56.331 INFO:teuthology.packaging:branch: quincy 2023-09-23T22:45:56.331 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:45:56.331 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T22:45:56.712 DEBUG:teuthology.orchestra.run.smithi138:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T22:45:56.739 INFO:teuthology.orchestra.run.smithi138.stdout:17.2.6-1125.gc9bb1bcf.el8 2023-09-23T22:45:56.739 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1125.gc9bb1bcf.el8 2023-09-23T22:45:56.739 INFO:teuthology.task.install:The correct ceph version 17.2.6-1125.gc9bb1bcf is installed. 2023-09-23T22:45:56.739 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-09-23T22:45:56.740 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-09-23T22:45:56.740 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T22:45:56.769 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-09-23T22:45:56.769 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T22:45:56.811 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-09-23T22:45:56.811 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-09-23T22:45:56.811 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T22:45:56.840 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T22:45:56.908 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-09-23T22:45:56.908 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T22:45:56.937 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T22:45:57.006 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-09-23T22:45:57.006 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-09-23T22:45:57.006 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T22:45:57.034 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T22:45:57.101 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-09-23T22:45:57.101 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T22:45:57.129 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T22:45:57.196 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-09-23T22:45:57.196 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-09-23T22:45:57.196 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T22:45:57.222 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T22:45:57.288 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-09-23T22:45:57.288 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T22:45:57.315 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T22:45:57.382 INFO:teuthology.run_tasks:Running task cephadm... 2023-09-23T22:45:57.451 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T22:45:57.452 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2023-09-23T22:45:57.452 INFO:tasks.cephadm:Cluster fsid is f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:45:57.452 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-09-23T22:45:57.453 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.110', 'mon.c': '[v2:172.21.15.110:3301,v1:172.21.15.110:6790]', 'mon.b': '172.21.15.138'} 2023-09-23T22:45:57.453 INFO:tasks.cephadm:First mon is mon.a on smithi110 2023-09-23T22:45:57.453 INFO:tasks.cephadm:First mgr is y 2023-09-23T22:45:57.453 INFO:tasks.cephadm:Normalizing hostnames... 2023-09-23T22:45:57.453 DEBUG:teuthology.orchestra.run.smithi110:> sudo hostname $(hostname -s) 2023-09-23T22:45:57.479 DEBUG:teuthology.orchestra.run.smithi138:> sudo hostname $(hostname -s) 2023-09-23T22:45:57.506 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2023-09-23T22:45:57.506 DEBUG:teuthology.orchestra.run.smithi110:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T22:45:57.738 INFO:teuthology.orchestra.run.smithi110.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 23 22:45 /home/ubuntu/cephtest/cephadm 2023-09-23T22:45:57.738 DEBUG:teuthology.orchestra.run.smithi138:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T22:45:58.075 INFO:teuthology.orchestra.run.smithi138.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 23 22:45 /home/ubuntu/cephtest/cephadm 2023-09-23T22:45:58.075 DEBUG:teuthology.orchestra.run.smithi110:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T22:45:58.093 DEBUG:teuthology.orchestra.run.smithi138:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T22:45:58.122 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2023-09-23T22:45:58.122 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-23T22:45:58.135 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-23T22:45:59.279 INFO:teuthology.orchestra.run.smithi138.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T22:45:59.388 INFO:teuthology.orchestra.run.smithi110.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T22:46:38.617 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T22:46:38.617 INFO:teuthology.orchestra.run.smithi138.stdout: "ceph_version": "ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)", 2023-09-23T22:46:38.617 INFO:teuthology.orchestra.run.smithi138.stdout: "image_id": "444539aec5bf9e20782edb464496ac68eeb42e917d810f31b62a4e32fd3e4cab", 2023-09-23T22:46:38.617 INFO:teuthology.orchestra.run.smithi138.stdout: "repo_digests": [ 2023-09-23T22:46:38.617 INFO:teuthology.orchestra.run.smithi138.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:3bb80add7917e9e7cd2f8b41f950baba963e834e98b4566c8403c083fc4f1c17" 2023-09-23T22:46:38.617 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T22:46:38.617 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T22:46:39.953 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T22:46:39.953 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph_version": "ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)", 2023-09-23T22:46:39.953 INFO:teuthology.orchestra.run.smithi110.stdout: "image_id": "444539aec5bf9e20782edb464496ac68eeb42e917d810f31b62a4e32fd3e4cab", 2023-09-23T22:46:39.953 INFO:teuthology.orchestra.run.smithi110.stdout: "repo_digests": [ 2023-09-23T22:46:39.953 INFO:teuthology.orchestra.run.smithi110.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:3bb80add7917e9e7cd2f8b41f950baba963e834e98b4566c8403c083fc4f1c17" 2023-09-23T22:46:39.954 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T22:46:39.954 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T22:46:39.979 DEBUG:teuthology.orchestra.run.smithi110:> sudo mkdir -p /etc/ceph 2023-09-23T22:46:40.011 DEBUG:teuthology.orchestra.run.smithi138:> sudo mkdir -p /etc/ceph 2023-09-23T22:46:40.048 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod 777 /etc/ceph 2023-09-23T22:46:40.080 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod 777 /etc/ceph 2023-09-23T22:46:40.117 INFO:tasks.cephadm:Writing seed config... 2023-09-23T22:46:40.118 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-09-23T22:46:40.118 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-09-23T22:46:40.118 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-09-23T22:46:40.118 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-09-23T22:46:40.118 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-09-23T22:46:40.118 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-09-23T22:46:40.119 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-09-23T22:46:40.119 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-09-23T22:46:40.119 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-09-23T22:46:40.119 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-09-23T22:46:40.119 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-09-23T22:46:40.119 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-09-23T22:46:40.119 DEBUG:teuthology.orchestra.run.smithi110:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-09-23T22:46:40.137 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = f5f05148-5a62-11ee-8db4-212e2dc638e7 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-09-23T22:46:40.137 DEBUG:teuthology.orchestra.run.smithi110:mon.a> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mon.a.service 2023-09-23T22:46:40.179 DEBUG:teuthology.orchestra.run.smithi110:mgr.y> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mgr.y.service 2023-09-23T22:46:40.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:-- Logs begin at Sat 2023-09-23 22:30:57 UTC. -- 2023-09-23T22:46:40.208 INFO:tasks.cephadm:Bootstrapping... 2023-09-23T22:46:40.208 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.110 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-09-23T22:46:40.234 INFO:journalctl@ceph.mgr.y.smithi110.stdout:-- Logs begin at Sat 2023-09-23 22:30:57 UTC. -- 2023-09-23T22:46:40.408 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:46:40.408 INFO:teuthology.orchestra.run.smithi110.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'f5f05148-5a62-11ee-8db4-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.110', '--skip-admin-label'] 2023-09-23T22:46:40.470 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stdout 3.0.1 2023-09-23T22:46:40.470 INFO:teuthology.orchestra.run.smithi110.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-09-23T22:46:40.470 INFO:teuthology.orchestra.run.smithi110.stdout:Verifying podman|docker is present... 2023-09-23T22:46:40.522 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stdout 3.0.1 2023-09-23T22:46:40.522 INFO:teuthology.orchestra.run.smithi110.stdout:Verifying lvm2 is present... 2023-09-23T22:46:40.522 INFO:teuthology.orchestra.run.smithi110.stdout:Verifying time synchronization is in place... 2023-09-23T22:46:40.529 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-23T22:46:40.529 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T22:46:40.535 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-23T22:46:40.535 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout inactive 2023-09-23T22:46:40.540 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout enabled 2023-09-23T22:46:40.546 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout active 2023-09-23T22:46:40.546 INFO:teuthology.orchestra.run.smithi110.stdout:Unit chronyd.service is enabled and running 2023-09-23T22:46:40.546 INFO:teuthology.orchestra.run.smithi110.stdout:Repeating the final host check... 2023-09-23T22:46:40.607 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stdout 3.0.1 2023-09-23T22:46:40.607 INFO:teuthology.orchestra.run.smithi110.stdout:podman (/bin/podman) version 3.0.1 is present 2023-09-23T22:46:40.607 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl is present 2023-09-23T22:46:40.607 INFO:teuthology.orchestra.run.smithi110.stdout:lvcreate is present 2023-09-23T22:46:40.613 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-23T22:46:40.613 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T22:46:40.619 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-23T22:46:40.619 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout inactive 2023-09-23T22:46:40.624 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout enabled 2023-09-23T22:46:40.630 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout active 2023-09-23T22:46:40.630 INFO:teuthology.orchestra.run.smithi110.stdout:Unit chronyd.service is enabled and running 2023-09-23T22:46:40.631 INFO:teuthology.orchestra.run.smithi110.stdout:Host looks OK 2023-09-23T22:46:40.631 INFO:teuthology.orchestra.run.smithi110.stdout:Cluster fsid: f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:46:40.631 INFO:teuthology.orchestra.run.smithi110.stdout:Acquiring lock 140629460167648 on /run/cephadm/f5f05148-5a62-11ee-8db4-212e2dc638e7.lock 2023-09-23T22:46:40.631 INFO:teuthology.orchestra.run.smithi110.stdout:Lock 140629460167648 acquired on /run/cephadm/f5f05148-5a62-11ee-8db4-212e2dc638e7.lock 2023-09-23T22:46:40.631 INFO:teuthology.orchestra.run.smithi110.stdout:Verifying IP 172.21.15.110 port 3300 ... 2023-09-23T22:46:40.632 INFO:teuthology.orchestra.run.smithi110.stdout:Verifying IP 172.21.15.110 port 6789 ... 2023-09-23T22:46:40.632 INFO:teuthology.orchestra.run.smithi110.stdout:Base mon IP(s) is [172.21.15.110:3300, 172.21.15.110:6789], mon addrv is [v2:172.21.15.110:3300,v1:172.21.15.110:6789] 2023-09-23T22:46:40.635 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp metric 100 2023-09-23T22:46:40.635 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.110 metric 100 2023-09-23T22:46:40.637 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-09-23T22:46:40.638 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2023-09-23T22:46:40.638 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2023-09-23T22:46:40.640 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-09-23T22:46:40.640 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-09-23T22:46:40.640 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-23T22:46:40.640 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-09-23T22:46:40.640 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout inet6 fe80::d2ee:be1a:ef6e:23ea/64 scope link noprefixroute 2023-09-23T22:46:40.640 INFO:teuthology.orchestra.run.smithi110.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-23T22:46:40.641 INFO:teuthology.orchestra.run.smithi110.stdout:Mon IP `172.21.15.110` is in CIDR network `172.21.0.0/20` 2023-09-23T22:46:40.641 INFO:teuthology.orchestra.run.smithi110.stdout:Mon IP `172.21.15.110` is in CIDR network `172.21.0.0/20` 2023-09-23T22:46:40.641 INFO:teuthology.orchestra.run.smithi110.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-09-23T22:46:40.641 INFO:teuthology.orchestra.run.smithi110.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-09-23T22:46:40.642 INFO:teuthology.orchestra.run.smithi110.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T22:46:41.162 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stdout 444539aec5bf9e20782edb464496ac68eeb42e917d810f31b62a4e32fd3e4cab 2023-09-23T22:46:41.162 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T22:46:41.162 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stderr Getting image source signatures 2023-09-23T22:46:41.163 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stderr Copying blob sha256:ce16db1fb95f1913bf433ce46f6a040afd03f13a267a85c9c5302717cf75a00a 2023-09-23T22:46:41.163 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stderr Copying blob sha256:3f4fd5c3c800c6da9ef94691df203a05a5bbbf32c483d09d620ec06eef42ea2b 2023-09-23T22:46:41.163 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stderr Copying config sha256:444539aec5bf9e20782edb464496ac68eeb42e917d810f31b62a4e32fd3e4cab 2023-09-23T22:46:41.163 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stderr Writing manifest to image destination 2023-09-23T22:46:41.163 INFO:teuthology.orchestra.run.smithi110.stdout:/bin/podman: stderr Storing signatures 2023-09-23T22:46:42.242 INFO:teuthology.orchestra.run.smithi110.stdout:ceph: stdout ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable) 2023-09-23T22:46:42.242 INFO:teuthology.orchestra.run.smithi110.stdout:Ceph version: ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable) 2023-09-23T22:46:42.242 INFO:teuthology.orchestra.run.smithi110.stdout:Extracting ceph user uid/gid from container image... 2023-09-23T22:46:43.242 INFO:teuthology.orchestra.run.smithi110.stdout:stat: stdout 167 167 2023-09-23T22:46:43.243 INFO:teuthology.orchestra.run.smithi110.stdout:Creating initial keys... 2023-09-23T22:46:44.139 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph-authtool: stdout AQDTag9lLNNMIhAAHrzrHxEiLBSd9O8K+AIkYw== 2023-09-23T22:46:45.135 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph-authtool: stdout AQDUag9lTHpEHxAAWSq8VQkqztmoGVJEFDAwmw== 2023-09-23T22:46:46.060 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph-authtool: stdout AQDVag9l4bV6HhAAys1z/zkjdMK5PuikAh7Dyg== 2023-09-23T22:46:46.061 INFO:teuthology.orchestra.run.smithi110.stdout:Creating initial monmap... 2023-09-23T22:46:47.002 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T22:46:47.002 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2023-09-23T22:46:47.002 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:46:47.002 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T22:46:47.003 INFO:teuthology.orchestra.run.smithi110.stdout:monmaptool for a [v2:172.21.15.110:3300,v1:172.21.15.110:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T22:46:47.003 INFO:teuthology.orchestra.run.smithi110.stdout:setting min_mon_release = octopus 2023-09-23T22:46:47.003 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/monmaptool: set fsid to f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:46:47.003 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T22:46:47.003 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:46:47.003 INFO:teuthology.orchestra.run.smithi110.stdout:Creating mon... 2023-09-23T22:46:47.975 INFO:teuthology.orchestra.run.smithi110.stdout:create mon.a on 2023-09-23T22:46:48.073 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-09-23T22:46:48.158 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-09-23T22:46:48.246 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7.target. 2023-09-23T22:46:48.246 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7.target. 2023-09-23T22:46:48.371 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T22:46:48.372 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Failed to reset failed state of unit ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mon.a.service: Unit ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mon.a.service not loaded. 2023-09-23T22:46:48.456 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7.target.wants/ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@.service. 2023-09-23T22:46:48.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:48 smithi110 systemd[1]: Starting Ceph mon.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:46:49.016 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T22:46:49.016 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout disabled 2023-09-23T22:46:49.024 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T22:46:49.024 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout inactive 2023-09-23T22:46:49.024 INFO:teuthology.orchestra.run.smithi110.stdout:firewalld.service is not enabled 2023-09-23T22:46:49.024 INFO:teuthology.orchestra.run.smithi110.stdout:Not possible to enable service . firewalld.service is not available 2023-09-23T22:46:49.024 INFO:teuthology.orchestra.run.smithi110.stdout:Waiting for mon to start... 2023-09-23T22:46:49.025 INFO:teuthology.orchestra.run.smithi110.stdout:Waiting for mon... 2023-09-23T22:46:49.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:48 smithi110 bash[99229]: a0a8fef3a87a2a3403221c139e190138bd51e9875f564fdb51bd9c11d82c39af 2023-09-23T22:46:49.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:49 smithi110 systemd[1]: Started Ceph mon.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:46:49.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:49 smithi110 ceph-mon[99293]: mkfs f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:46:49.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:49 smithi110 ceph-mon[99293]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T22:46:50.309 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout cluster: 2023-09-23T22:46:50.309 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout id: f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:46:50.309 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-09-23T22:46:50.309 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2023-09-23T22:46:50.309 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout services: 2023-09-23T22:46:50.310 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.524422s) 2023-09-23T22:46:50.310 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-09-23T22:46:50.310 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-09-23T22:46:50.310 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2023-09-23T22:46:50.310 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout data: 2023-09-23T22:46:50.310 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-09-23T22:46:50.310 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-09-23T22:46:50.310 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-09-23T22:46:50.310 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout pgs: 2023-09-23T22:46:50.310 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2023-09-23T22:46:50.310 INFO:teuthology.orchestra.run.smithi110.stdout:mon is available 2023-09-23T22:46:50.310 INFO:teuthology.orchestra.run.smithi110.stdout:Assimilating anything we can from ceph.conf... 2023-09-23T22:46:50.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:50 smithi110 ceph-mon[99293]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T22:46:50.316 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:50 smithi110 ceph-mon[99293]: monmap e1: 1 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0]} removed_ranks: {} 2023-09-23T22:46:50.316 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:50 smithi110 ceph-mon[99293]: fsmap 2023-09-23T22:46:50.316 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:50 smithi110 ceph-mon[99293]: osdmap e1: 0 total, 0 up, 0 in 2023-09-23T22:46:50.316 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:50 smithi110 ceph-mon[99293]: mgrmap e1: no daemons active 2023-09-23T22:46:50.317 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:50 smithi110 ceph-mon[99293]: from='client.? 172.21.15.110:0/1654453733' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-09-23T22:46:51.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:51 smithi110 ceph-mon[99293]: from='client.? 172.21.15.110:0/1308904574' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T22:46:51.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:51 smithi110 ceph-mon[99293]: from='client.? 172.21.15.110:0/1308904574' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-09-23T22:46:51.418 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2023-09-23T22:46:51.418 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout [global] 2023-09-23T22:46:51.418 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout fsid = f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:46:51.418 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.110:3300,v1:172.21.15.110:6789] 2023-09-23T22:46:51.418 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-23T22:46:51.418 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-23T22:46:51.418 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-23T22:46:51.418 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-23T22:46:51.418 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2023-09-23T22:46:51.419 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-23T22:46:51.419 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-23T22:46:51.419 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2023-09-23T22:46:51.419 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout [osd] 2023-09-23T22:46:51.419 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-23T22:46:51.419 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-23T22:46:51.419 INFO:teuthology.orchestra.run.smithi110.stdout:Generating new minimal ceph.conf... 2023-09-23T22:46:52.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:52 smithi110 ceph-mon[99293]: from='client.? 172.21.15.110:0/39462290' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:46:52.458 INFO:teuthology.orchestra.run.smithi110.stdout:Restarting the monitor... 2023-09-23T22:46:52.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:52 smithi110 systemd[1]: Stopping Ceph mon.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:46:52.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:52 smithi110 conmon[99269]: 2023-09-23T22:46:52.686+0000 7eff76018700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:46:52.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:52 smithi110 conmon[99269]: 2023-09-23T22:46:52.686+0000 7eff76018700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-09-23T22:46:53.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:53 smithi110 bash[99847]: a0a8fef3a87a2a3403221c139e190138bd51e9875f564fdb51bd9c11d82c39af 2023-09-23T22:46:53.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:53 smithi110 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T22:46:53.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:53 smithi110 systemd[1]: Stopped Ceph mon.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:46:53.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:53 smithi110 systemd[1]: Starting Ceph mon.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:46:54.053 INFO:teuthology.orchestra.run.smithi110.stdout:Setting mon public_network to 172.21.0.0/20 2023-09-23T22:46:54.298 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T22:46:54.298 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable), process ceph-mon, pid 7 2023-09-23T22:46:54.298 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: pidfile_write: ignore empty --pid-file 2023-09-23T22:46:54.298 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 bash[100178]: c610848cb8f9c5edd35dd43d44bf10952759704924b6a402d58220de14c415d2 2023-09-23T22:46:54.298 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: load: jerasure load: lrc 2023-09-23T22:46:54.298 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: RocksDB version: 6.15.5 2023-09-23T22:46:54.298 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T22:46:54.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Compile date Sep 23 2023 2023-09-23T22:46:54.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: DB SUMMARY 2023-09-23T22:46:54.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: DB Session ID: GEOHWCKLD920E4812HFL 2023-09-23T22:46:54.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: CURRENT file: CURRENT 2023-09-23T22:46:54.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T22:46:54.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2023-09-23T22:46:54.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-09-23T22:46:54.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83609 ; 2023-09-23T22:46:54.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.error_if_exists: 0 2023-09-23T22:46:54.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.create_if_missing: 0 2023-09-23T22:46:54.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.paranoid_checks: 1 2023-09-23T22:46:54.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T22:46:54.300 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.env: 0x55eff1b7ba80 2023-09-23T22:46:54.300 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.fs: Posix File System 2023-09-23T22:46:54.300 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.info_log: 0x55eff28aa5a0 2023-09-23T22:46:54.300 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T22:46:54.300 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.statistics: (nil) 2023-09-23T22:46:54.300 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.use_fsync: 0 2023-09-23T22:46:54.300 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_log_file_size: 0 2023-09-23T22:46:54.300 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T22:46:54.300 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T22:46:54.300 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T22:46:54.300 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T22:46:54.300 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.allow_fallocate: 1 2023-09-23T22:46:54.300 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T22:46:54.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T22:46:54.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.use_direct_reads: 0 2023-09-23T22:46:54.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T22:46:54.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T22:46:54.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.db_log_dir: 2023-09-23T22:46:54.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-09-23T22:46:54.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T22:46:54.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T22:46:54.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T22:46:54.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T22:46:54.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T22:46:54.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T22:46:54.302 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T22:46:54.302 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T22:46:54.302 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.write_buffer_manager: 0x55eff361c900 2023-09-23T22:46:54.302 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T22:46:54.302 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T22:46:54.302 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T22:46:54.302 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T22:46:54.302 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T22:46:54.302 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T22:46:54.302 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T22:46:54.302 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T22:46:54.302 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T22:46:54.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.unordered_write: 0 2023-09-23T22:46:54.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T22:46:54.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T22:46:54.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T22:46:54.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T22:46:54.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.row_cache: None 2023-09-23T22:46:54.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.wal_filter: None 2023-09-23T22:46:54.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T22:46:54.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T22:46:54.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.preserve_deletes: 0 2023-09-23T22:46:54.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.two_write_queues: 0 2023-09-23T22:46:54.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T22:46:54.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.atomic_flush: 0 2023-09-23T22:46:54.304 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T22:46:54.304 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T22:46:54.304 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T22:46:54.304 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.log_readahead_size: 0 2023-09-23T22:46:54.304 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T22:46:54.304 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T22:46:54.304 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T22:46:54.304 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T22:46:54.304 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T22:46:54.304 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T22:46:54.304 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_background_jobs: 2 2023-09-23T22:46:54.304 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_background_compactions: -1 2023-09-23T22:46:54.305 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_subcompactions: 1 2023-09-23T22:46:54.305 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T22:46:54.305 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T22:46:54.305 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T22:46:54.305 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T22:46:54.305 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T22:46:54.305 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T22:46:54.305 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T22:46:54.305 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T22:46:54.305 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_open_files: -1 2023-09-23T22:46:54.305 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T22:46:54.305 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T22:46:54.305 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T22:46:54.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T22:46:54.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_background_flushes: -1 2023-09-23T22:46:54.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Compression algorithms supported: 2023-09-23T22:46:54.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T22:46:54.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: kZSTD supported: 0 2023-09-23T22:46:54.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: kXpressCompression supported: 0 2023-09-23T22:46:54.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T22:46:54.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: kZlibCompression supported: 1 2023-09-23T22:46:54.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: kSnappyCompression supported: 1 2023-09-23T22:46:54.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: kLZ4Compression supported: 1 2023-09-23T22:46:54.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: kBZip2Compression supported: 0 2023-09-23T22:46:54.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T22:46:54.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2023-09-23T22:46:54.307 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T22:46:54.307 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T22:46:54.307 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.merge_operator: 2023-09-23T22:46:54.307 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compaction_filter: None 2023-09-23T22:46:54.307 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T22:46:54.307 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T22:46:54.307 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T22:46:54.307 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T22:46:54.307 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55eff28fcfb8) 2023-09-23T22:46:54.307 INFO:journalctl@ceph.mon.a.smithi110.stdout: cache_index_and_filter_blocks: 1 2023-09-23T22:46:54.307 INFO:journalctl@ceph.mon.a.smithi110.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T22:46:54.307 INFO:journalctl@ceph.mon.a.smithi110.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T22:46:54.307 INFO:journalctl@ceph.mon.a.smithi110.stdout: pin_top_level_index_and_filter: 1 2023-09-23T22:46:54.308 INFO:journalctl@ceph.mon.a.smithi110.stdout: index_type: 0 2023-09-23T22:46:54.308 INFO:journalctl@ceph.mon.a.smithi110.stdout: data_block_index_type: 0 2023-09-23T22:46:54.308 INFO:journalctl@ceph.mon.a.smithi110.stdout: index_shortening: 1 2023-09-23T22:46:54.308 INFO:journalctl@ceph.mon.a.smithi110.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T22:46:54.308 INFO:journalctl@ceph.mon.a.smithi110.stdout: hash_index_allow_collision: 1 2023-09-23T22:46:54.308 INFO:journalctl@ceph.mon.a.smithi110.stdout: checksum: 1 2023-09-23T22:46:54.308 INFO:journalctl@ceph.mon.a.smithi110.stdout: no_block_cache: 0 2023-09-23T22:46:54.308 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_cache: 0x55eff292f090 2023-09-23T22:46:54.308 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_cache_name: BinnedLRUCache 2023-09-23T22:46:54.308 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_cache_options: 2023-09-23T22:46:54.308 INFO:journalctl@ceph.mon.a.smithi110.stdout: capacity : 536870912 2023-09-23T22:46:54.308 INFO:journalctl@ceph.mon.a.smithi110.stdout: num_shard_bits : 4 2023-09-23T22:46:54.309 INFO:journalctl@ceph.mon.a.smithi110.stdout: strict_capacity_limit : 0 2023-09-23T22:46:54.309 INFO:journalctl@ceph.mon.a.smithi110.stdout: high_pri_pool_ratio: 0.000 2023-09-23T22:46:54.309 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_cache_compressed: (nil) 2023-09-23T22:46:54.309 INFO:journalctl@ceph.mon.a.smithi110.stdout: persistent_cache: (nil) 2023-09-23T22:46:54.309 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_size: 4096 2023-09-23T22:46:54.309 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_size_deviation: 10 2023-09-23T22:46:54.309 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_restart_interval: 16 2023-09-23T22:46:54.309 INFO:journalctl@ceph.mon.a.smithi110.stdout: index_block_restart_interval: 1 2023-09-23T22:46:54.309 INFO:journalctl@ceph.mon.a.smithi110.stdout: metadata_block_size: 4096 2023-09-23T22:46:54.309 INFO:journalctl@ceph.mon.a.smithi110.stdout: partition_filters: 0 2023-09-23T22:46:54.309 INFO:journalctl@ceph.mon.a.smithi110.stdout: use_delta_encoding: 1 2023-09-23T22:46:54.309 INFO:journalctl@ceph.mon.a.smithi110.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T22:46:54.309 INFO:journalctl@ceph.mon.a.smithi110.stdout: whole_key_filtering: 1 2023-09-23T22:46:54.309 INFO:journalctl@ceph.mon.a.smithi110.stdout: verify_compression: 0 2023-09-23T22:46:54.310 INFO:journalctl@ceph.mon.a.smithi110.stdout: read_amp_bytes_per_bit: 0 2023-09-23T22:46:54.310 INFO:journalctl@ceph.mon.a.smithi110.stdout: format_version: 4 2023-09-23T22:46:54.310 INFO:journalctl@ceph.mon.a.smithi110.stdout: enable_index_compression: 1 2023-09-23T22:46:54.310 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_align: 0 2023-09-23T22:46:54.310 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T22:46:54.310 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T22:46:54.310 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compression: NoCompression 2023-09-23T22:46:54.310 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T22:46:54.310 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T22:46:54.310 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T22:46:54.310 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.num_levels: 7 2023-09-23T22:46:54.310 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T22:46:54.311 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T22:46:54.311 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T22:46:54.311 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T22:46:54.311 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T22:46:54.311 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T22:46:54.311 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T22:46:54.311 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:46:54.311 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T22:46:54.311 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T22:46:54.311 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T22:46:54.311 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T22:46:54.311 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T22:46:54.311 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T22:46:54.312 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:46:54.312 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T22:46:54.312 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T22:46:54.312 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T22:46:54.312 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T22:46:54.312 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T22:46:54.312 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T22:46:54.312 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T22:46:54.312 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T22:46:54.312 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T22:46:54.312 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T22:46:54.312 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T22:46:54.312 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T22:46:54.313 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T22:46:54.313 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T22:46:54.313 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T22:46:54.313 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T22:46:54.313 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T22:46:54.313 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T22:46:54.313 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T22:46:54.313 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T22:46:54.313 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T22:46:54.313 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T22:46:54.313 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T22:46:54.313 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T22:46:54.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T22:46:54.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T22:46:54.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T22:46:54.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T22:46:54.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T22:46:54.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T22:46:54.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T22:46:54.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T22:46:54.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T22:46:54.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T22:46:54.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.table_properties_collectors: 2023-09-23T22:46:54.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.inplace_update_support: 0 2023-09-23T22:46:54.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T22:46:54.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T22:46:54.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T22:46:54.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T22:46:54.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.bloom_locality: 0 2023-09-23T22:46:54.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.max_successive_merges: 0 2023-09-23T22:46:54.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T22:46:54.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T22:46:54.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T22:46:54.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T22:46:54.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.ttl: 2592000 2023-09-23T22:46:54.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T22:46:54.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.enable_blob_files: false 2023-09-23T22:46:54.316 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.min_blob_size: 0 2023-09-23T22:46:54.316 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T22:46:54.316 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T22:46:54.316 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T22:46:54.316 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T22:46:54.316 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T22:46:54.316 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2023-09-23T22:46:54.316 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2023-09-23T22:46:54.316 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695509214049722, "job": 1, "event": "recovery_started", "wal_files": [10]} 2023-09-23T22:46:54.316 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2023-09-23T22:46:54.316 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T22:46:54.316 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695509214050378, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80156, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78424, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10136, "raw_average_key_size": 47, "raw_value_size": 72632, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695509214, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "3dfe1224-496e-4c8d-a221-ef6994288fd5", "db_session_id": "GEOHWCKLD920E4812HFL"}} 2023-09-23T22:46:54.317 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2023-09-23T22:46:54.317 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695509214050719, "job": 1, "event": "recovery_finished"} 2023-09-23T22:46:54.317 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T22:46:54.317 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55eff28c3180 2023-09-23T22:46:54.317 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: DB pointer 0x55eff2998000 2023-09-23T22:46:54.317 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T22:46:54.317 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T22:46:54.317 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** DB Stats ** 2023-09-23T22:46:54.317 INFO:journalctl@ceph.mon.a.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:46:54.317 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T22:46:54.317 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:46:54.317 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:46:54.317 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T22:46:54.318 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T22:46:54.318 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:46:54.318 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2023-09-23T22:46:54.318 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** Compaction Stats [default] ** 2023-09-23T22:46:54.318 INFO:journalctl@ceph.mon.a.smithi110.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:46:54.318 INFO:journalctl@ceph.mon.a.smithi110.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:46:54.318 INFO:journalctl@ceph.mon.a.smithi110.stdout: L0 2/0 80.04 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 177.4 0.00 0.00 1 0.000 0 0 2023-09-23T22:46:54.318 INFO:journalctl@ceph.mon.a.smithi110.stdout: Sum 2/0 80.04 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 177.4 0.00 0.00 1 0.000 0 0 2023-09-23T22:46:54.318 INFO:journalctl@ceph.mon.a.smithi110.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 177.4 0.00 0.00 1 0.000 0 0 2023-09-23T22:46:54.318 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2023-09-23T22:46:54.318 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** Compaction Stats [default] ** 2023-09-23T22:46:54.318 INFO:journalctl@ceph.mon.a.smithi110.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:46:54.318 INFO:journalctl@ceph.mon.a.smithi110.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:46:54.319 INFO:journalctl@ceph.mon.a.smithi110.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 177.4 0.00 0.00 1 0.000 0 0 2023-09-23T22:46:54.319 INFO:journalctl@ceph.mon.a.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:46:54.319 INFO:journalctl@ceph.mon.a.smithi110.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T22:46:54.319 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:46:54.319 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:46:54.319 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:46:54.319 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:46:54.319 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative compaction: 0.00 GB write, 26.81 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:46:54.319 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval compaction: 0.00 GB write, 26.81 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:46:54.319 INFO:journalctl@ceph.mon.a.smithi110.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:46:54.319 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2023-09-23T22:46:54.319 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:46:54.319 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2023-09-23T22:46:54.320 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** Compaction Stats [default] ** 2023-09-23T22:46:54.320 INFO:journalctl@ceph.mon.a.smithi110.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:46:54.320 INFO:journalctl@ceph.mon.a.smithi110.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:46:54.320 INFO:journalctl@ceph.mon.a.smithi110.stdout: L0 2/0 80.04 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 177.4 0.00 0.00 1 0.000 0 0 2023-09-23T22:46:54.320 INFO:journalctl@ceph.mon.a.smithi110.stdout: Sum 2/0 80.04 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 177.4 0.00 0.00 1 0.000 0 0 2023-09-23T22:46:54.320 INFO:journalctl@ceph.mon.a.smithi110.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T22:46:54.320 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2023-09-23T22:46:54.320 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** Compaction Stats [default] ** 2023-09-23T22:46:54.320 INFO:journalctl@ceph.mon.a.smithi110.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:46:54.320 INFO:journalctl@ceph.mon.a.smithi110.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:46:54.320 INFO:journalctl@ceph.mon.a.smithi110.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 177.4 0.00 0.00 1 0.000 0 0 2023-09-23T22:46:54.320 INFO:journalctl@ceph.mon.a.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:46:54.321 INFO:journalctl@ceph.mon.a.smithi110.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T22:46:54.321 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:46:54.321 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:46:54.321 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:46:54.321 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:46:54.321 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative compaction: 0.00 GB write, 26.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:46:54.321 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:46:54.321 INFO:journalctl@ceph.mon.a.smithi110.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:46:54.321 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2023-09-23T22:46:54.321 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:46:54.321 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: starting mon.a rank 0 at public addrs [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] at bind addrs [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:46:54.321 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 systemd[1]: Started Ceph mon.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:46:54.321 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: mon.a@-1(???) e1 preinit fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:46:54.322 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: mon.a@-1(???).mds e1 new map 2023-09-23T22:46:54.322 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: mon.a@-1(???).mds e1 print_map 2023-09-23T22:46:54.322 INFO:journalctl@ceph.mon.a.smithi110.stdout: e1 2023-09-23T22:46:54.322 INFO:journalctl@ceph.mon.a.smithi110.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T22:46:54.322 INFO:journalctl@ceph.mon.a.smithi110.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:46:54.322 INFO:journalctl@ceph.mon.a.smithi110.stdout: legacy client fscid: -1 2023-09-23T22:46:54.322 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2023-09-23T22:46:54.322 INFO:journalctl@ceph.mon.a.smithi110.stdout: No filesystems configured 2023-09-23T22:46:54.322 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T22:46:54.322 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:46:54.322 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:46:54.322 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:46:54.323 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-09-23T22:46:54.323 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T22:46:54.323 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: monmap e1: 1 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0]} removed_ranks: {} 2023-09-23T22:46:54.323 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: fsmap 2023-09-23T22:46:54.323 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: osdmap e1: 0 total, 0 up, 0 in 2023-09-23T22:46:54.323 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:54 smithi110 ceph-mon[100243]: mgrmap e1: no daemons active 2023-09-23T22:46:55.208 INFO:teuthology.orchestra.run.smithi110.stdout:Wrote config to /etc/ceph/ceph.conf 2023-09-23T22:46:55.208 INFO:teuthology.orchestra.run.smithi110.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-09-23T22:46:55.209 INFO:teuthology.orchestra.run.smithi110.stdout:Creating mgr... 2023-09-23T22:46:55.209 INFO:teuthology.orchestra.run.smithi110.stdout:Verifying port 9283 ... 2023-09-23T22:46:55.338 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T22:46:55.338 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Failed to reset failed state of unit ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mgr.y.service: Unit ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mgr.y.service not loaded. 2023-09-23T22:46:55.427 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7.target.wants/ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@.service. 2023-09-23T22:46:55.566 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:55 smithi110 systemd[1]: Starting Ceph mgr.y for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:46:55.876 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:55 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3383851195' entity='client.admin' 2023-09-23T22:46:56.025 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T22:46:56.025 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout disabled 2023-09-23T22:46:56.032 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T22:46:56.033 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout inactive 2023-09-23T22:46:56.033 INFO:teuthology.orchestra.run.smithi110.stdout:firewalld.service is not enabled 2023-09-23T22:46:56.033 INFO:teuthology.orchestra.run.smithi110.stdout:Not possible to enable service . firewalld.service is not available 2023-09-23T22:46:56.041 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T22:46:56.041 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout disabled 2023-09-23T22:46:56.048 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T22:46:56.048 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout inactive 2023-09-23T22:46:56.049 INFO:teuthology.orchestra.run.smithi110.stdout:firewalld.service is not enabled 2023-09-23T22:46:56.049 INFO:teuthology.orchestra.run.smithi110.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2023-09-23T22:46:56.049 INFO:teuthology.orchestra.run.smithi110.stdout:Waiting for mgr to start... 2023-09-23T22:46:56.049 INFO:teuthology.orchestra.run.smithi110.stdout:Waiting for mgr... 2023-09-23T22:46:56.274 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:56 smithi110 bash[100625]: 906aab6b0363265390686c1b9fd3a07c494d97774e487c71bf99f1449c54ef9f 2023-09-23T22:46:56.274 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:56 smithi110 systemd[1]: Started Ceph mgr.y for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:46:56.274 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:56 smithi110 conmon[100666]: 2023-09-23T22:46:56.136+0000 7f44fe3861c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:46:56.274 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:56 smithi110 conmon[100666]: 2023-09-23T22:46:56.214+0000 7f44fe3861c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:46:56.579 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:56 smithi110 conmon[100666]: 2023-09-23T22:46:56.312+0000 7f44fe3861c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:46:56.579 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:56 smithi110 conmon[100666]: 2023-09-23T22:46:56.405+0000 7f44fe3861c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:46:56.579 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:56 smithi110 conmon[100666]: 2023-09-23T22:46:56.578+0000 7f44fe3861c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:46:56.906 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:46:56 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/97271297' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T22:46:57.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:56 smithi110 conmon[100666]: 2023-09-23T22:46:56.905+0000 7f44fe3861c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:46:57.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:56 smithi110 conmon[100666]: 2023-09-23T22:46:56.992+0000 7f44fe3861c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:46:57.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:57 smithi110 conmon[100666]: 2023-09-23T22:46:57.117+0000 7f44fe3861c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:46:57.226 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2023-09-23T22:46:57.226 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout { 2023-09-23T22:46:57.226 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "fsid": "f5f05148-5a62-11ee-8db4-212e2dc638e7", 2023-09-23T22:46:57.227 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T22:46:57.227 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T22:46:57.227 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T22:46:57.227 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T22:46:57.227 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:46:57.227 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T22:46:57.227 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T22:46:57.227 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 0 2023-09-23T22:46:57.227 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2023-09-23T22:46:57.227 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T22:46:57.227 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T22:46:57.227 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2023-09-23T22:46:57.228 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2023-09-23T22:46:57.228 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T22:46:57.228 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T22:46:57.228 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T22:46:57.228 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T22:46:57.228 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:46:57.228 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T22:46:57.228 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T22:46:57.228 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T22:46:57.228 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T22:46:57.228 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T22:46:57.228 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T22:46:57.228 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T22:46:57.229 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T22:46:57.231 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:46:57.232 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T22:46:57.232 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T22:46:57.232 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T22:46:57.232 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T22:46:57.232 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T22:46:57.232 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T22:46:57.232 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T22:46:57.232 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T22:46:57.232 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T22:46:57.232 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:46:57.233 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T22:46:57.233 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T22:46:57.233 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T22:46:57.233 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T22:46:57.233 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:46:57.233 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T22:46:57.233 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T22:46:57.233 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T22:46:57.233 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T22:46:57.233 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T22:46:57.233 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T22:46:57.233 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T22:46:57.233 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2023-09-23T22:46:57.234 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T22:46:57.234 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:46:57.234 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T22:46:57.234 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T22:46:57.234 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T22:46:49.032823+0000", 2023-09-23T22:46:57.234 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T22:46:57.234 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:46:57.234 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T22:46:57.234 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout } 2023-09-23T22:46:57.234 INFO:teuthology.orchestra.run.smithi110.stdout:mgr not available, waiting (1/15)... 2023-09-23T22:46:57.625 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:57 smithi110 conmon[100666]: 2023-09-23T22:46:57.210+0000 7f44fe3861c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:46:57.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:57 smithi110 conmon[100666]: 2023-09-23T22:46:57.624+0000 7f44fe3861c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:46:58.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:58 smithi110 conmon[100666]: 2023-09-23T22:46:58.186+0000 7f44fe3861c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:46:58.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:58 smithi110 conmon[100666]: 2023-09-23T22:46:58.352+0000 7f44fe3861c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:46:59.358 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:59 smithi110 conmon[100666]: 2023-09-23T22:46:59.096+0000 7f44fe3861c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:46:59.358 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:59 smithi110 conmon[100666]: 2023-09-23T22:46:59.182+0000 7f44fe3861c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:46:59.645 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:59 smithi110 conmon[100666]: 2023-09-23T22:46:59.357+0000 7f44fe3861c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:46:59.914 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:59 smithi110 conmon[100666]: 2023-09-23T22:46:59.645+0000 7f44fe3861c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:46:59.914 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:46:59 smithi110 conmon[100666]: 2023-09-23T22:46:59.735+0000 7f44fe3861c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:47:00.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:00 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2575381555' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T22:47:00.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:00 smithi110 conmon[100666]: 2023-09-23T22:47:00.034+0000 7f44fe3861c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:47:00.290 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2023-09-23T22:47:00.292 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout { 2023-09-23T22:47:00.293 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "fsid": "f5f05148-5a62-11ee-8db4-212e2dc638e7", 2023-09-23T22:47:00.293 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T22:47:00.293 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T22:47:00.293 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T22:47:00.293 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T22:47:00.293 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:00.293 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T22:47:00.293 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T22:47:00.293 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 0 2023-09-23T22:47:00.293 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2023-09-23T22:47:00.293 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T22:47:00.294 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T22:47:00.294 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2023-09-23T22:47:00.294 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2023-09-23T22:47:00.294 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T22:47:00.294 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T22:47:00.294 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T22:47:00.294 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T22:47:00.294 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:00.294 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T22:47:00.294 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T22:47:00.294 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T22:47:00.294 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T22:47:00.295 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T22:47:00.295 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T22:47:00.295 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T22:47:00.295 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T22:47:00.298 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:00.298 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T22:47:00.298 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T22:47:00.298 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T22:47:00.298 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T22:47:00.298 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T22:47:00.298 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T22:47:00.298 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T22:47:00.298 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T22:47:00.298 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T22:47:00.298 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:00.299 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T22:47:00.299 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T22:47:00.299 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T22:47:00.299 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T22:47:00.299 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:00.299 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T22:47:00.299 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T22:47:00.299 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T22:47:00.299 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T22:47:00.299 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T22:47:00.299 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T22:47:00.299 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T22:47:00.300 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2023-09-23T22:47:00.300 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T22:47:00.300 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:00.300 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T22:47:00.300 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T22:47:00.300 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T22:46:49.032823+0000", 2023-09-23T22:47:00.300 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T22:47:00.300 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:00.300 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T22:47:00.300 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout } 2023-09-23T22:47:00.300 INFO:teuthology.orchestra.run.smithi110.stdout:mgr not available, waiting (2/15)... 2023-09-23T22:47:00.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:00 smithi110 conmon[100666]: 2023-09-23T22:47:00.294+0000 7f44fe3861c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:47:01.326 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:01 smithi110 conmon[100666]: 2023-09-23T22:47:01.010+0000 7f44fe3861c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:47:01.327 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:01 smithi110 conmon[100666]: 2023-09-23T22:47:01.140+0000 7f44fe3861c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:47:01.648 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:01 smithi110 conmon[100666]: 2023-09-23T22:47:01.326+0000 7f44fe3861c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:47:01.648 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:01 smithi110 conmon[100666]: 2023-09-23T22:47:01.437+0000 7f44fe3861c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:47:01.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:01 smithi110 conmon[100666]: 2023-09-23T22:47:01.648+0000 7f44fe3861c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:47:02.850 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:02 smithi110 conmon[100666]: 2023-09-23T22:47:02.523+0000 7f44fe3861c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:47:02.850 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:02 smithi110 ceph-mon[100243]: Activating manager daemon y 2023-09-23T22:47:02.850 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:02 smithi110 ceph-mon[100243]: mgrmap e2: y(active, starting, since 0.00301477s) 2023-09-23T22:47:02.850 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:02 smithi110 ceph-mon[100243]: from='mgr.14100 172.21.15.110:0/3736858241' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:47:02.850 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:02 smithi110 ceph-mon[100243]: from='mgr.14100 172.21.15.110:0/3736858241' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:47:02.850 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:02 smithi110 ceph-mon[100243]: from='mgr.14100 172.21.15.110:0/3736858241' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:47:02.850 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:02 smithi110 ceph-mon[100243]: from='mgr.14100 172.21.15.110:0/3736858241' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:47:02.851 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:02 smithi110 ceph-mon[100243]: from='mgr.14100 172.21.15.110:0/3736858241' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:47:02.851 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:02 smithi110 ceph-mon[100243]: Manager daemon y is now available 2023-09-23T22:47:02.851 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:02 smithi110 ceph-mon[100243]: from='mgr.14100 172.21.15.110:0/3736858241' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:47:02.851 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:02 smithi110 ceph-mon[100243]: from='mgr.14100 172.21.15.110:0/3736858241' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:47:02.851 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:02 smithi110 ceph-mon[100243]: from='mgr.14100 172.21.15.110:0/3736858241' entity='mgr.y' 2023-09-23T22:47:02.851 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:02 smithi110 ceph-mon[100243]: from='mgr.14100 172.21.15.110:0/3736858241' entity='mgr.y' 2023-09-23T22:47:02.851 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:02 smithi110 ceph-mon[100243]: from='mgr.14100 172.21.15.110:0/3736858241' entity='mgr.y' 2023-09-23T22:47:03.433 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2023-09-23T22:47:03.434 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout { 2023-09-23T22:47:03.434 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "fsid": "f5f05148-5a62-11ee-8db4-212e2dc638e7", 2023-09-23T22:47:03.434 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T22:47:03.434 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T22:47:03.434 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T22:47:03.434 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T22:47:03.434 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:03.434 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T22:47:03.434 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T22:47:03.434 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 0 2023-09-23T22:47:03.435 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2023-09-23T22:47:03.435 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T22:47:03.435 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T22:47:03.435 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2023-09-23T22:47:03.435 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum_age": 8, 2023-09-23T22:47:03.435 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T22:47:03.435 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T22:47:03.435 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T22:47:03.435 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T22:47:03.435 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:03.435 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T22:47:03.435 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T22:47:03.435 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T22:47:03.436 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T22:47:03.436 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T22:47:03.436 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T22:47:03.436 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T22:47:03.436 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T22:47:03.439 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:03.439 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T22:47:03.439 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T22:47:03.439 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T22:47:03.439 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T22:47:03.439 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T22:47:03.440 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T22:47:03.440 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T22:47:03.440 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T22:47:03.440 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T22:47:03.440 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:03.440 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T22:47:03.440 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T22:47:03.440 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T22:47:03.440 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T22:47:03.440 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:03.440 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T22:47:03.441 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T22:47:03.441 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T22:47:03.441 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T22:47:03.441 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T22:47:03.441 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T22:47:03.441 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T22:47:03.441 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2023-09-23T22:47:03.441 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T22:47:03.441 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:03.441 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T22:47:03.441 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T22:47:03.441 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T22:46:49.032823+0000", 2023-09-23T22:47:03.442 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T22:47:03.442 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:03.442 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T22:47:03.442 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout } 2023-09-23T22:47:03.442 INFO:teuthology.orchestra.run.smithi110.stdout:mgr not available, waiting (3/15)... 2023-09-23T22:47:03.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:03 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2737283361' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T22:47:03.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:03 smithi110 ceph-mon[100243]: mgrmap e3: y(active, since 1.00564s) 2023-09-23T22:47:06.322 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:06 smithi110 ceph-mon[100243]: mgrmap e4: y(active, since 2s) 2023-09-23T22:47:06.720 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2023-09-23T22:47:06.720 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout { 2023-09-23T22:47:06.720 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "fsid": "f5f05148-5a62-11ee-8db4-212e2dc638e7", 2023-09-23T22:47:06.720 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T22:47:06.720 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T22:47:06.720 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T22:47:06.720 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T22:47:06.720 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:06.721 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T22:47:06.721 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T22:47:06.721 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 0 2023-09-23T22:47:06.721 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2023-09-23T22:47:06.721 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T22:47:06.721 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T22:47:06.721 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2023-09-23T22:47:06.721 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "quorum_age": 12, 2023-09-23T22:47:06.721 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T22:47:06.721 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T22:47:06.721 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T22:47:06.721 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T22:47:06.721 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:06.722 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T22:47:06.722 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T22:47:06.722 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T22:47:06.722 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T22:47:06.722 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T22:47:06.722 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T22:47:06.722 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T22:47:06.722 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T22:47:06.722 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:06.722 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T22:47:06.722 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T22:47:06.722 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T22:47:06.722 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T22:47:06.723 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T22:47:06.723 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T22:47:06.723 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T22:47:06.723 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T22:47:06.723 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T22:47:06.723 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:06.723 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T22:47:06.723 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T22:47:06.723 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T22:47:06.723 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T22:47:06.723 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:06.723 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T22:47:06.723 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T22:47:06.724 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T22:47:06.724 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T22:47:06.724 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T22:47:06.724 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T22:47:06.724 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T22:47:06.724 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ], 2023-09-23T22:47:06.724 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T22:47:06.724 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:06.724 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T22:47:06.724 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T22:47:06.724 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T22:46:49.032823+0000", 2023-09-23T22:47:06.724 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T22:47:06.725 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout }, 2023-09-23T22:47:06.725 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T22:47:06.725 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout } 2023-09-23T22:47:06.725 INFO:teuthology.orchestra.run.smithi110.stdout:mgr is available 2023-09-23T22:47:07.394 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:07 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2715551439' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T22:47:07.982 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2023-09-23T22:47:07.982 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout [global] 2023-09-23T22:47:07.982 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout fsid = f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:47:07.982 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-23T22:47:07.982 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-23T22:47:07.982 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-23T22:47:07.982 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-23T22:47:07.982 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2023-09-23T22:47:07.982 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-23T22:47:07.983 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-23T22:47:07.983 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 2023-09-23T22:47:07.983 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout [osd] 2023-09-23T22:47:07.983 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-23T22:47:07.983 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-23T22:47:07.983 INFO:teuthology.orchestra.run.smithi110.stdout:Enabling cephadm module... 2023-09-23T22:47:08.243 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:08 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2777243065' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T22:47:09.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:09 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/366762677' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-09-23T22:47:09.390 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:09 smithi110 conmon[100666]: ignoring --setuser ceph since I am not root 2023-09-23T22:47:09.391 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:09 smithi110 conmon[100666]: ignoring --setgroup ceph since I am not root 2023-09-23T22:47:09.391 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:09 smithi110 conmon[100666]: 2023-09-23T22:47:09.217+0000 7f329ad0b1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:47:09.391 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:09 smithi110 conmon[100666]: 2023-09-23T22:47:09.299+0000 7f329ad0b1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:47:09.647 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:09 smithi110 conmon[100666]: 2023-09-23T22:47:09.388+0000 7f329ad0b1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:47:09.647 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:09 smithi110 conmon[100666]: 2023-09-23T22:47:09.478+0000 7f329ad0b1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:47:09.901 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:09 smithi110 conmon[100666]: 2023-09-23T22:47:09.646+0000 7f329ad0b1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:47:10.194 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:10 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/366762677' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-09-23T22:47:10.195 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:10 smithi110 ceph-mon[100243]: mgrmap e5: y(active, since 6s) 2023-09-23T22:47:10.195 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:09 smithi110 conmon[100666]: 2023-09-23T22:47:09.970+0000 7f329ad0b1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:47:10.195 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:10 smithi110 conmon[100666]: 2023-09-23T22:47:10.059+0000 7f329ad0b1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:47:10.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:10 smithi110 conmon[100666]: 2023-09-23T22:47:10.192+0000 7f329ad0b1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:47:10.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:10 smithi110 conmon[100666]: 2023-09-23T22:47:10.287+0000 7f329ad0b1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:47:10.930 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout { 2023-09-23T22:47:10.931 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-09-23T22:47:10.931 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T22:47:10.931 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-23T22:47:10.932 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-23T22:47:10.932 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout } 2023-09-23T22:47:10.932 INFO:teuthology.orchestra.run.smithi110.stdout:Waiting for the mgr to restart... 2023-09-23T22:47:10.932 INFO:teuthology.orchestra.run.smithi110.stdout:Waiting for mgr epoch 5... 2023-09-23T22:47:10.936 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:10 smithi110 conmon[100666]: 2023-09-23T22:47:10.722+0000 7f329ad0b1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:47:11.195 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:11 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2153267180' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T22:47:11.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:11 smithi110 conmon[100666]: 2023-09-23T22:47:11.304+0000 7f329ad0b1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:47:11.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:11 smithi110 conmon[100666]: 2023-09-23T22:47:11.474+0000 7f329ad0b1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:47:12.473 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:12 smithi110 conmon[100666]: 2023-09-23T22:47:12.220+0000 7f329ad0b1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:47:12.474 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:12 smithi110 conmon[100666]: 2023-09-23T22:47:12.308+0000 7f329ad0b1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:47:12.742 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:12 smithi110 conmon[100666]: 2023-09-23T22:47:12.472+0000 7f329ad0b1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:47:13.108 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:12 smithi110 conmon[100666]: 2023-09-23T22:47:12.740+0000 7f329ad0b1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:47:13.108 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:12 smithi110 conmon[100666]: 2023-09-23T22:47:12.825+0000 7f329ad0b1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:47:13.360 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:13 smithi110 conmon[100666]: 2023-09-23T22:47:13.107+0000 7f329ad0b1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:47:13.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:13 smithi110 conmon[100666]: 2023-09-23T22:47:13.359+0000 7f329ad0b1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:47:14.360 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:14 smithi110 conmon[100666]: 2023-09-23T22:47:14.047+0000 7f329ad0b1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:47:14.360 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:14 smithi110 conmon[100666]: 2023-09-23T22:47:14.176+0000 7f329ad0b1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:47:14.680 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:14 smithi110 conmon[100666]: 2023-09-23T22:47:14.358+0000 7f329ad0b1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:47:14.680 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:14 smithi110 conmon[100666]: 2023-09-23T22:47:14.471+0000 7f329ad0b1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:47:14.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:14 smithi110 conmon[100666]: 2023-09-23T22:47:14.679+0000 7f329ad0b1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:47:15.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:15 smithi110 conmon[100666]: 2023-09-23T22:47:15.514+0000 7f329ad0b1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:47:15.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:15 smithi110 ceph-mon[100243]: Active manager daemon y restarted 2023-09-23T22:47:15.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:15 smithi110 ceph-mon[100243]: Activating manager daemon y 2023-09-23T22:47:15.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:15 smithi110 ceph-mon[100243]: osdmap e2: 0 total, 0 up, 0 in 2023-09-23T22:47:15.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:15 smithi110 ceph-mon[100243]: mgrmap e6: y(active, starting, since 0.00335836s) 2023-09-23T22:47:15.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:15 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:47:15.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:15 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:47:15.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:15 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:47:15.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:15 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:47:15.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:15 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:47:15.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:15 smithi110 ceph-mon[100243]: Manager daemon y is now available 2023-09-23T22:47:15.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:15 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' 2023-09-23T22:47:15.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:15 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' 2023-09-23T22:47:15.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:15 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:47:15.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:15 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:47:15.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:15 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:47:16.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:16 smithi110 ceph-mon[100243]: Found migration_current of "None". Setting to last migration. 2023-09-23T22:47:16.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:16 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:47:16.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:16 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:47:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:16 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' 2023-09-23T22:47:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:16 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:47:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:16 smithi110 ceph-mon[100243]: mgrmap e7: y(active, since 1.00525s) 2023-09-23T22:47:17.155 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout { 2023-09-23T22:47:17.155 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-09-23T22:47:17.155 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-23T22:47:17.155 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout } 2023-09-23T22:47:17.155 INFO:teuthology.orchestra.run.smithi110.stdout:mgr epoch 5 is available 2023-09-23T22:47:17.155 INFO:teuthology.orchestra.run.smithi110.stdout:Setting orchestrator backend to cephadm... 2023-09-23T22:47:17.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:17 smithi110 ceph-mon[100243]: [23/Sep/2023:22:47:16] ENGINE Bus STARTING 2023-09-23T22:47:17.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:17 smithi110 ceph-mon[100243]: [23/Sep/2023:22:47:16] ENGINE Serving on https://172.21.15.110:7150 2023-09-23T22:47:17.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:17 smithi110 ceph-mon[100243]: [23/Sep/2023:22:47:16] ENGINE Bus STARTED 2023-09-23T22:47:17.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:17 smithi110 ceph-mon[100243]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T22:47:17.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:17 smithi110 ceph-mon[100243]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T22:47:18.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:18 smithi110 ceph-mon[100243]: mgrmap e8: y(active, since 2s) 2023-09-23T22:47:18.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:18 smithi110 ceph-mon[100243]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:47:18.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:18 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' 2023-09-23T22:47:18.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:18 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:47:19.572 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout value unchanged 2023-09-23T22:47:19.572 INFO:teuthology.orchestra.run.smithi110.stdout:Generating ssh key... 2023-09-23T22:47:20.343 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:20 smithi110 ceph-mon[100243]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:47:20.343 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:20 smithi110 conmon[100666]: Generating public/private rsa key pair. 2023-09-23T22:47:20.343 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:20 smithi110 conmon[100666]: Your identification has been saved in /tmp/tmpnvg0949_/key. 2023-09-23T22:47:20.343 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:20 smithi110 conmon[100666]: Your public key has been saved in /tmp/tmpnvg0949_/key.pub. 2023-09-23T22:47:20.343 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:20 smithi110 conmon[100666]: The key fingerprint is: 2023-09-23T22:47:20.343 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:20 smithi110 conmon[100666]: SHA256:4PQORxtMXjv9EUBaQTNsdp1lKVGNQF1VbJDh8TrVr6U ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:47:20.343 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:20 smithi110 conmon[100666]: The key's randomart image is: 2023-09-23T22:47:20.343 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:20 smithi110 conmon[100666]: +---[RSA 3072]----+ 2023-09-23T22:47:20.344 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:20 smithi110 conmon[100666]: | . .=@*BX&| 2023-09-23T22:47:20.344 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:20 smithi110 conmon[100666]: | + . =+==B*| 2023-09-23T22:47:20.344 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:20 smithi110 conmon[100666]: | o = +o..+.+| 2023-09-23T22:47:20.344 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:20 smithi110 conmon[100666]: | o + o . . +.| 2023-09-23T22:47:20.344 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:20 smithi110 conmon[100666]: | o S + o| 2023-09-23T22:47:20.344 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:20 smithi110 conmon[100666]: | + = | 2023-09-23T22:47:20.344 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:20 smithi110 conmon[100666]: | . E | 2023-09-23T22:47:20.344 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:20 smithi110 conmon[100666]: | | 2023-09-23T22:47:20.344 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:20 smithi110 conmon[100666]: | | 2023-09-23T22:47:20.344 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:20 smithi110 conmon[100666]: +----[SHA256]-----+ 2023-09-23T22:47:21.574 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:21 smithi110 ceph-mon[100243]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:47:21.574 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:21 smithi110 ceph-mon[100243]: Generating ssh key... 2023-09-23T22:47:21.574 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:21 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' 2023-09-23T22:47:21.574 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:21 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' 2023-09-23T22:47:22.088 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDVfezf4kSZXhPVbcVMnyOVtduQ0iVH8WG25+fQhJox3eeya8ua2iwKnPQHWviT+6Bv4bBwDZBwxQ9oAySuJqfPzLHwx/yylZjQQ7kNLeayePSz9jDVu/h8MYg5puwR9Y2O/xzyyAQo4hpwcRGnVkQFRYEbfVZGpcHDDMizmlhOKOhVsBwpsBaDtCSSpB7rF74JlF3jUoVagWTRCjCVMITF4vPdwRqmuQBhtskYOLyNixXCq0Jm25og/SlBibbqsHdV3/LIYOyV4BT4SaBEccg7L0s4zdtQKxxrIJ1rCngDwPdTXMYHPANXEuXobKC9Rnd7i+/OzIRnT0S+JJL4cOZlvyv/OAX2aOn4HTNRKragaFReTBQQmc0Q6r0SZPNk/yEONtr2BTu5rDd3+lkOLV7I1nfEwXzs9NmTZI2cDQgIEbnY0O65OlaNxjLdufsRdeA6EwhY3uhwdl9gORTxTadTue1Kl4UBND/nB/wmOmhvTA6y+W1oE/IpKHo4WLjOL+k= ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:47:22.088 INFO:teuthology.orchestra.run.smithi110.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-09-23T22:47:22.088 INFO:teuthology.orchestra.run.smithi110.stdout:Adding key to root@localhost authorized_keys... 2023-09-23T22:47:22.089 INFO:teuthology.orchestra.run.smithi110.stdout:Adding host smithi110... 2023-09-23T22:47:22.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:22 smithi110 ceph-mon[100243]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:47:23.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:23 smithi110 ceph-mon[100243]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi110", "addr": "172.21.15.110", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:47:25.079 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout Added host 'smithi110' with addr '172.21.15.110' 2023-09-23T22:47:25.079 INFO:teuthology.orchestra.run.smithi110.stdout:Deploying unmanaged mon service... 2023-09-23T22:47:25.318 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:25 smithi110 ceph-mon[100243]: Deploying cephadm binary to smithi110 2023-09-23T22:47:25.318 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:25 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' 2023-09-23T22:47:25.318 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:25 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:47:26.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:26 smithi110 ceph-mon[100243]: Added host smithi110 2023-09-23T22:47:26.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:26 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' 2023-09-23T22:47:26.903 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-09-23T22:47:26.904 INFO:teuthology.orchestra.run.smithi110.stdout:Deploying unmanaged mgr service... 2023-09-23T22:47:27.116 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:27 smithi110 ceph-mon[100243]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:47:27.116 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:27 smithi110 ceph-mon[100243]: Saving service mon spec with placement count:5 2023-09-23T22:47:27.116 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:27 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' 2023-09-23T22:47:28.212 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-09-23T22:47:28.513 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:28 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' 2023-09-23T22:47:28.514 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:28 smithi110 ceph-mon[100243]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:47:28.514 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:28 smithi110 ceph-mon[100243]: Saving service mgr spec with placement count:2 2023-09-23T22:47:28.514 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:28 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' 2023-09-23T22:47:30.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:29 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' 2023-09-23T22:47:30.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:29 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/582286356' entity='client.admin' 2023-09-23T22:47:30.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:29 smithi110 ceph-mon[100243]: from='mgr.14120 172.21.15.110:0/3250132864' entity='mgr.y' 2023-09-23T22:47:31.396 INFO:teuthology.orchestra.run.smithi110.stdout:Enabling the dashboard module... 2023-09-23T22:47:31.651 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:31 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1055637076' entity='client.admin' 2023-09-23T22:47:32.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:32 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2294915072' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-09-23T22:47:32.674 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:32 smithi110 conmon[100666]: ignoring --setuser ceph since I am not root 2023-09-23T22:47:32.675 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:32 smithi110 conmon[100666]: ignoring --setgroup ceph since I am not root 2023-09-23T22:47:32.675 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:32 smithi110 conmon[100666]: 2023-09-23T22:47:32.591+0000 7f6d3f24b1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:47:32.675 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:32 smithi110 conmon[100666]: 2023-09-23T22:47:32.673+0000 7f6d3f24b1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:47:32.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:32 smithi110 conmon[100666]: 2023-09-23T22:47:32.761+0000 7f6d3f24b1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:47:32.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:32 smithi110 conmon[100666]: 2023-09-23T22:47:32.852+0000 7f6d3f24b1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:47:33.285 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:33 smithi110 conmon[100666]: 2023-09-23T22:47:33.018+0000 7f6d3f24b1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:47:33.578 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:33 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2294915072' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-09-23T22:47:33.578 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:33 smithi110 ceph-mon[100243]: mgrmap e9: y(active, since 16s) 2023-09-23T22:47:33.579 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:33 smithi110 conmon[100666]: 2023-09-23T22:47:33.351+0000 7f6d3f24b1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:47:33.579 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:33 smithi110 conmon[100666]: 2023-09-23T22:47:33.444+0000 7f6d3f24b1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:47:33.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:33 smithi110 conmon[100666]: 2023-09-23T22:47:33.578+0000 7f6d3f24b1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:47:33.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:33 smithi110 conmon[100666]: 2023-09-23T22:47:33.672+0000 7f6d3f24b1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:47:34.386 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout { 2023-09-23T22:47:34.386 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-09-23T22:47:34.386 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T22:47:34.386 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-23T22:47:34.387 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-23T22:47:34.387 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout } 2023-09-23T22:47:34.387 INFO:teuthology.orchestra.run.smithi110.stdout:Waiting for the mgr to restart... 2023-09-23T22:47:34.387 INFO:teuthology.orchestra.run.smithi110.stdout:Waiting for mgr epoch 9... 2023-09-23T22:47:34.391 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:34 smithi110 conmon[100666]: 2023-09-23T22:47:34.105+0000 7f6d3f24b1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:47:34.643 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:34 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3494115903' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T22:47:34.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:34 smithi110 conmon[100666]: 2023-09-23T22:47:34.683+0000 7f6d3f24b1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:47:34.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:34 smithi110 conmon[100666]: 2023-09-23T22:47:34.857+0000 7f6d3f24b1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:47:35.864 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:35 smithi110 conmon[100666]: 2023-09-23T22:47:35.609+0000 7f6d3f24b1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:47:35.864 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:35 smithi110 conmon[100666]: 2023-09-23T22:47:35.697+0000 7f6d3f24b1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:47:36.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:35 smithi110 conmon[100666]: 2023-09-23T22:47:35.864+0000 7f6d3f24b1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:47:36.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:36 smithi110 conmon[100666]: 2023-09-23T22:47:36.133+0000 7f6d3f24b1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:47:36.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:36 smithi110 conmon[100666]: 2023-09-23T22:47:36.217+0000 7f6d3f24b1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:47:36.757 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:36 smithi110 conmon[100666]: 2023-09-23T22:47:36.500+0000 7f6d3f24b1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:47:37.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:36 smithi110 conmon[100666]: 2023-09-23T22:47:36.756+0000 7f6d3f24b1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:47:37.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:37 smithi110 conmon[100666]: 2023-09-23T22:47:37.446+0000 7f6d3f24b1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:47:37.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:37 smithi110 conmon[100666]: 2023-09-23T22:47:37.574+0000 7f6d3f24b1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:47:38.082 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:37 smithi110 conmon[100666]: 2023-09-23T22:47:37.760+0000 7f6d3f24b1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:47:38.082 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:37 smithi110 conmon[100666]: 2023-09-23T22:47:37.872+0000 7f6d3f24b1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:47:38.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:38 smithi110 conmon[100666]: 2023-09-23T22:47:38.081+0000 7f6d3f24b1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:47:39.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:47:38 smithi110 conmon[100666]: 2023-09-23T22:47:38.922+0000 7f6d3f24b1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:47:39.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:38 smithi110 ceph-mon[100243]: Active manager daemon y restarted 2023-09-23T22:47:39.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:38 smithi110 ceph-mon[100243]: Activating manager daemon y 2023-09-23T22:47:39.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:38 smithi110 ceph-mon[100243]: osdmap e3: 0 total, 0 up, 0 in 2023-09-23T22:47:39.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:38 smithi110 ceph-mon[100243]: mgrmap e10: y(active, starting, since 0.00303666s) 2023-09-23T22:47:39.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:47:39.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:47:39.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:47:39.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:47:39.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:47:39.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:38 smithi110 ceph-mon[100243]: Manager daemon y is now available 2023-09-23T22:47:39.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:47:39.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:47:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:39 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:47:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:39 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:47:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:39 smithi110 ceph-mon[100243]: [23/Sep/2023:22:47:39] ENGINE Bus STARTING 2023-09-23T22:47:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:39 smithi110 ceph-mon[100243]: [23/Sep/2023:22:47:39] ENGINE Serving on https://172.21.15.110:7150 2023-09-23T22:47:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:39 smithi110 ceph-mon[100243]: [23/Sep/2023:22:47:39] ENGINE Bus STARTED 2023-09-23T22:47:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:39 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:47:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:39 smithi110 ceph-mon[100243]: mgrmap e11: y(active, since 1.00519s) 2023-09-23T22:47:40.587 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout { 2023-09-23T22:47:40.587 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-09-23T22:47:40.587 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-23T22:47:40.587 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout } 2023-09-23T22:47:40.587 INFO:teuthology.orchestra.run.smithi110.stdout:mgr epoch 9 is available 2023-09-23T22:47:40.588 INFO:teuthology.orchestra.run.smithi110.stdout:Generating a dashboard self-signed certificate... 2023-09-23T22:47:41.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:41 smithi110 ceph-mon[100243]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T22:47:41.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:41 smithi110 ceph-mon[100243]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T22:47:41.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:41 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:47:41.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:41 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:47:42.346 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-09-23T22:47:42.346 INFO:teuthology.orchestra.run.smithi110.stdout:Creating initial admin user... 2023-09-23T22:47:42.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:42 smithi110 ceph-mon[100243]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:47:42.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:42 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:47:42.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:42 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:47:42.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:42 smithi110 ceph-mon[100243]: mgrmap e12: y(active, since 2s) 2023-09-23T22:47:44.051 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$ZQDf6W.GjTbXAMd4vPpbYe9G./5uMB4EBE33rzsNnDRTtSONrU9Zi", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1695509263, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-09-23T22:47:44.051 INFO:teuthology.orchestra.run.smithi110.stdout:Fetching dashboard port number... 2023-09-23T22:47:44.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:44 smithi110 ceph-mon[100243]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:47:44.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:44 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:47:45.318 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stdout 8443 2023-09-23T22:47:45.327 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T22:47:45.327 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout disabled 2023-09-23T22:47:45.334 INFO:teuthology.orchestra.run.smithi110.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T22:47:45.335 INFO:teuthology.orchestra.run.smithi110.stdout:systemctl: stdout inactive 2023-09-23T22:47:45.335 INFO:teuthology.orchestra.run.smithi110.stdout:firewalld.service is not enabled 2023-09-23T22:47:45.335 INFO:teuthology.orchestra.run.smithi110.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-09-23T22:47:45.336 INFO:teuthology.orchestra.run.smithi110.stdout:Ceph Dashboard is now available at: 2023-09-23T22:47:45.336 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:47:45.337 INFO:teuthology.orchestra.run.smithi110.stdout: URL: https://smithi110:8443/ 2023-09-23T22:47:45.337 INFO:teuthology.orchestra.run.smithi110.stdout: User: admin 2023-09-23T22:47:45.337 INFO:teuthology.orchestra.run.smithi110.stdout: Password: sx9n3gleei 2023-09-23T22:47:45.337 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:47:45.337 INFO:teuthology.orchestra.run.smithi110.stdout:Saving cluster configuration to /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config directory 2023-09-23T22:47:45.337 INFO:teuthology.orchestra.run.smithi110.stdout:Enabling autotune for osd_memory_target 2023-09-23T22:47:45.589 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:45 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/17127171' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-09-23T22:47:46.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:46 smithi110 ceph-mon[100243]: mgrmap e13: y(active, since 6s) 2023-09-23T22:47:48.051 INFO:teuthology.orchestra.run.smithi110.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-09-23T22:47:48.052 INFO:teuthology.orchestra.run.smithi110.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-09-23T22:47:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:47:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-09-23T22:47:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:47:48.052 INFO:teuthology.orchestra.run.smithi110.stdout:Or, if you are only running a single cluster on this host: 2023-09-23T22:47:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:47:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-09-23T22:47:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:47:48.052 INFO:teuthology.orchestra.run.smithi110.stdout:Please consider enabling telemetry to help improve Ceph: 2023-09-23T22:47:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:47:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: ceph telemetry on 2023-09-23T22:47:48.053 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:47:48.053 INFO:teuthology.orchestra.run.smithi110.stdout:For more information see: 2023-09-23T22:47:48.053 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:47:48.053 INFO:teuthology.orchestra.run.smithi110.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2023-09-23T22:47:48.053 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:47:48.053 INFO:teuthology.orchestra.run.smithi110.stdout:Bootstrap complete. 2023-09-23T22:47:48.094 INFO:tasks.cephadm:Fetching config... 2023-09-23T22:47:48.094 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-09-23T22:47:48.094 DEBUG:teuthology.orchestra.run.smithi110:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-09-23T22:47:48.111 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-09-23T22:47:48.112 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-09-23T22:47:48.112 DEBUG:teuthology.orchestra.run.smithi110:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-09-23T22:47:48.168 INFO:tasks.cephadm:Fetching mon keyring... 2023-09-23T22:47:48.168 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-09-23T22:47:48.168 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-09-23T22:47:48.237 INFO:tasks.cephadm:Fetching pub ssh key... 2023-09-23T22:47:48.237 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-09-23T22:47:48.237 DEBUG:teuthology.orchestra.run.smithi110:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-09-23T22:47:48.294 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-09-23T22:47:48.294 DEBUG:teuthology.orchestra.run.smithi110:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDVfezf4kSZXhPVbcVMnyOVtduQ0iVH8WG25+fQhJox3eeya8ua2iwKnPQHWviT+6Bv4bBwDZBwxQ9oAySuJqfPzLHwx/yylZjQQ7kNLeayePSz9jDVu/h8MYg5puwR9Y2O/xzyyAQo4hpwcRGnVkQFRYEbfVZGpcHDDMizmlhOKOhVsBwpsBaDtCSSpB7rF74JlF3jUoVagWTRCjCVMITF4vPdwRqmuQBhtskYOLyNixXCq0Jm25og/SlBibbqsHdV3/LIYOyV4BT4SaBEccg7L0s4zdtQKxxrIJ1rCngDwPdTXMYHPANXEuXobKC9Rnd7i+/OzIRnT0S+JJL4cOZlvyv/OAX2aOn4HTNRKragaFReTBQQmc0Q6r0SZPNk/yEONtr2BTu5rDd3+lkOLV7I1nfEwXzs9NmTZI2cDQgIEbnY0O65OlaNxjLdufsRdeA6EwhY3uhwdl9gORTxTadTue1Kl4UBND/nB/wmOmhvTA6y+W1oE/IpKHo4WLjOL+k= ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T22:47:48.373 INFO:teuthology.orchestra.run.smithi110.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDVfezf4kSZXhPVbcVMnyOVtduQ0iVH8WG25+fQhJox3eeya8ua2iwKnPQHWviT+6Bv4bBwDZBwxQ9oAySuJqfPzLHwx/yylZjQQ7kNLeayePSz9jDVu/h8MYg5puwR9Y2O/xzyyAQo4hpwcRGnVkQFRYEbfVZGpcHDDMizmlhOKOhVsBwpsBaDtCSSpB7rF74JlF3jUoVagWTRCjCVMITF4vPdwRqmuQBhtskYOLyNixXCq0Jm25og/SlBibbqsHdV3/LIYOyV4BT4SaBEccg7L0s4zdtQKxxrIJ1rCngDwPdTXMYHPANXEuXobKC9Rnd7i+/OzIRnT0S+JJL4cOZlvyv/OAX2aOn4HTNRKragaFReTBQQmc0Q6r0SZPNk/yEONtr2BTu5rDd3+lkOLV7I1nfEwXzs9NmTZI2cDQgIEbnY0O65OlaNxjLdufsRdeA6EwhY3uhwdl9gORTxTadTue1Kl4UBND/nB/wmOmhvTA6y+W1oE/IpKHo4WLjOL+k= ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:47:48.385 DEBUG:teuthology.orchestra.run.smithi138:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDVfezf4kSZXhPVbcVMnyOVtduQ0iVH8WG25+fQhJox3eeya8ua2iwKnPQHWviT+6Bv4bBwDZBwxQ9oAySuJqfPzLHwx/yylZjQQ7kNLeayePSz9jDVu/h8MYg5puwR9Y2O/xzyyAQo4hpwcRGnVkQFRYEbfVZGpcHDDMizmlhOKOhVsBwpsBaDtCSSpB7rF74JlF3jUoVagWTRCjCVMITF4vPdwRqmuQBhtskYOLyNixXCq0Jm25og/SlBibbqsHdV3/LIYOyV4BT4SaBEccg7L0s4zdtQKxxrIJ1rCngDwPdTXMYHPANXEuXobKC9Rnd7i+/OzIRnT0S+JJL4cOZlvyv/OAX2aOn4HTNRKragaFReTBQQmc0Q6r0SZPNk/yEONtr2BTu5rDd3+lkOLV7I1nfEwXzs9NmTZI2cDQgIEbnY0O65OlaNxjLdufsRdeA6EwhY3uhwdl9gORTxTadTue1Kl4UBND/nB/wmOmhvTA6y+W1oE/IpKHo4WLjOL+k= ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T22:47:48.430 INFO:teuthology.orchestra.run.smithi138.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDVfezf4kSZXhPVbcVMnyOVtduQ0iVH8WG25+fQhJox3eeya8ua2iwKnPQHWviT+6Bv4bBwDZBwxQ9oAySuJqfPzLHwx/yylZjQQ7kNLeayePSz9jDVu/h8MYg5puwR9Y2O/xzyyAQo4hpwcRGnVkQFRYEbfVZGpcHDDMizmlhOKOhVsBwpsBaDtCSSpB7rF74JlF3jUoVagWTRCjCVMITF4vPdwRqmuQBhtskYOLyNixXCq0Jm25og/SlBibbqsHdV3/LIYOyV4BT4SaBEccg7L0s4zdtQKxxrIJ1rCngDwPdTXMYHPANXEuXobKC9Rnd7i+/OzIRnT0S+JJL4cOZlvyv/OAX2aOn4HTNRKragaFReTBQQmc0Q6r0SZPNk/yEONtr2BTu5rDd3+lkOLV7I1nfEwXzs9NmTZI2cDQgIEbnY0O65OlaNxjLdufsRdeA6EwhY3uhwdl9gORTxTadTue1Kl4UBND/nB/wmOmhvTA6y+W1oE/IpKHo4WLjOL+k= ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:47:48.443 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-09-23T22:47:48.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:48 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3111127628' entity='client.admin' 2023-09-23T22:47:48.776 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:51.457 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-09-23T22:47:51.458 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-09-23T22:47:52.195 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:51 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/906069993' entity='client.admin' 2023-09-23T22:47:52.253 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:53.523 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:47:53.523 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:47:53.523 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi110", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:47:53.523 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:47:53.523 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:47:53.523 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:47:54.684 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi138 2023-09-23T22:47:54.684 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-09-23T22:47:54.684 DEBUG:teuthology.orchestra.run.smithi138:> dd of=/etc/ceph/ceph.conf 2023-09-23T22:47:54.704 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-09-23T22:47:54.704 DEBUG:teuthology.orchestra.run.smithi138:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:47:54.764 INFO:tasks.cephadm:Adding host smithi138 to orchestrator... 2023-09-23T22:47:54.764 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph orch host add smithi138 2023-09-23T22:47:55.140 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:55.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:54 smithi110 ceph-mon[100243]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:47:55.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:54 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:47:55.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:54 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:47:55.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:54 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:47:55.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:54 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:47:55.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:54 smithi110 ceph-mon[100243]: Updating smithi110:/etc/ceph/ceph.conf 2023-09-23T22:47:55.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:54 smithi110 ceph-mon[100243]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:47:56.399 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:56 smithi110 ceph-mon[100243]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:47:56.400 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:56 smithi110 ceph-mon[100243]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:47:56.400 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:56 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:47:56.400 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:56 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:47:56.400 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:56 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:47:57.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:57 smithi110 ceph-mon[100243]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi138", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:47:57.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:57 smithi110 ceph-mon[100243]: Deploying cephadm binary to smithi138 2023-09-23T22:47:58.531 INFO:teuthology.orchestra.run.smithi110.stdout:Added host 'smithi138' with addr '172.21.15.138' 2023-09-23T22:47:59.205 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph orch host ls --format=json 2023-09-23T22:47:59.539 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:59.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:59 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:47:59.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:59 smithi110 ceph-mon[100243]: Added host smithi138 2023-09-23T22:47:59.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:59 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:47:59.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:59 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:47:59.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:59 smithi110 ceph-mon[100243]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:47:59.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:47:59 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:01.170 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:48:01.170 INFO:teuthology.orchestra.run.smithi110.stdout:[{"addr": "172.21.15.110", "hostname": "smithi110", "labels": [], "status": ""}, {"addr": "172.21.15.138", "hostname": "smithi138", "labels": [], "status": ""}] 2023-09-23T22:48:01.397 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:01 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:01.821 INFO:tasks.cephadm:Setting crush tunables to default 2023-09-23T22:48:01.821 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd crush tunables default 2023-09-23T22:48:02.150 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:02.402 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:02 smithi110 ceph-mon[100243]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:02.402 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:02 smithi110 ceph-mon[100243]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:48:04.364 INFO:teuthology.orchestra.run.smithi110.stderr:adjusted tunables profile to default 2023-09-23T22:48:04.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:04 smithi110 ceph-mon[100243]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:04.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:04 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:04.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:04 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:04.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:04 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:04.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:04 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:04.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:04 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi138", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:48:04.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:04 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:04.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:04 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:04.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:04 smithi110 ceph-mon[100243]: Updating smithi138:/etc/ceph/ceph.conf 2023-09-23T22:48:04.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:04 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2292132868' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-23T22:48:04.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:04 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:04.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:04 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:04.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:04 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:05.012 INFO:tasks.cephadm:Adding mon.a on smithi110 2023-09-23T22:48:05.012 INFO:tasks.cephadm:Adding mon.c on smithi110 2023-09-23T22:48:05.012 INFO:tasks.cephadm:Adding mon.b on smithi138 2023-09-23T22:48:05.012 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph orch apply mon '3;smithi110:172.21.15.110=a;smithi110:[v2:172.21.15.110:3301,v1:172.21.15.110:6790]=c;smithi138:172.21.15.138=b' 2023-09-23T22:48:05.377 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:48:05.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:05 smithi110 ceph-mon[100243]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:48:05.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:05 smithi110 ceph-mon[100243]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:48:05.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:05 smithi110 ceph-mon[100243]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:48:05.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:05 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2292132868' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T22:48:05.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:05 smithi110 ceph-mon[100243]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:48:06.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:06 smithi110 ceph-mon[100243]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:07.058 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled mon update... 2023-09-23T22:48:07.687 DEBUG:teuthology.orchestra.run.smithi110:mon.c> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mon.c.service 2023-09-23T22:48:07.690 DEBUG:teuthology.orchestra.run.smithi138:mon.b> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mon.b.service 2023-09-23T22:48:07.693 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T22:48:07.693 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T22:48:07.723 INFO:journalctl@ceph.mon.b.smithi138.stdout:-- Logs begin at Sat 2023-09-23 22:31:02 UTC. -- 2023-09-23T22:48:07.725 INFO:journalctl@ceph.mon.c.smithi110.stdout:-- Logs begin at Sat 2023-09-23 22:30:57 UTC. -- 2023-09-23T22:48:08.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:08 smithi110 ceph-mon[100243]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:08.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:08 smithi110 ceph-mon[100243]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi110:172.21.15.110=a;smithi110:[v2:172.21.15.110:3301,v1:172.21.15.110:6790]=c;smithi138:172.21.15.138=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:48:08.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:08 smithi110 ceph-mon[100243]: Saving service mon spec with placement smithi110:172.21.15.110=a;smithi110:[v2:172.21.15.110:3301,v1:172.21.15.110:6790]=c;smithi138:172.21.15.138=b;count:3 2023-09-23T22:48:08.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:08 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:08.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:08 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:08.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:08 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:08.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:08 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:08.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:08 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:08.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:08 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:48:08.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:08 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:08.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:08 smithi110 ceph-mon[100243]: Deploying daemon mon.b on smithi138 2023-09-23T22:48:08.628 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:48:10.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:10 smithi110 ceph-mon[100243]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:11.131 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:48:11.131 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":1,"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","modified":"2023-09-23T22:46:46.458336Z","created":"2023-09-23T22:46:46.458336Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:3300","nonce":0},{"type":"v1","addr":"172.21.15.110:6789","nonce":0}]},"addr":"172.21.15.110:6789/0","public_addr":"172.21.15.110:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-09-23T22:48:11.132 INFO:teuthology.orchestra.run.smithi138.stderr:dumped monmap epoch 1 2023-09-23T22:48:11.161 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:10 smithi138 systemd[1]: Starting Ceph mon.b for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:48:11.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T22:48:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable), process ceph-mon, pid 7 2023-09-23T22:48:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: pidfile_write: ignore empty --pid-file 2023-09-23T22:48:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: load: jerasure load: lrc 2023-09-23T22:48:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: RocksDB version: 6.15.5 2023-09-23T22:48:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T22:48:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Compile date Sep 23 2023 2023-09-23T22:48:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: DB SUMMARY 2023-09-23T22:48:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: DB Session ID: 256PZ4SRSE856YP99G27 2023-09-23T22:48:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: CURRENT file: CURRENT 2023-09-23T22:48:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T22:48:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-09-23T22:48:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-09-23T22:48:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2023-09-23T22:48:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.error_if_exists: 0 2023-09-23T22:48:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.create_if_missing: 0 2023-09-23T22:48:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.paranoid_checks: 1 2023-09-23T22:48:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T22:48:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.env: 0x55dfb2771a80 2023-09-23T22:48:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.fs: Posix File System 2023-09-23T22:48:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.info_log: 0x55dfb48c65a0 2023-09-23T22:48:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T22:48:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.statistics: (nil) 2023-09-23T22:48:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.use_fsync: 0 2023-09-23T22:48:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_log_file_size: 0 2023-09-23T22:48:11.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T22:48:11.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T22:48:11.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T22:48:11.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T22:48:11.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.allow_fallocate: 1 2023-09-23T22:48:11.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T22:48:11.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T22:48:11.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.use_direct_reads: 0 2023-09-23T22:48:11.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T22:48:11.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T22:48:11.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.db_log_dir: 2023-09-23T22:48:11.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-09-23T22:48:11.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T22:48:11.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T22:48:11.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T22:48:11.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T22:48:11.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T22:48:11.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T22:48:11.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T22:48:11.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T22:48:11.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.write_buffer_manager: 0x55dfb5638900 2023-09-23T22:48:11.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T22:48:11.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T22:48:11.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T22:48:11.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T22:48:11.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T22:48:11.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T22:48:11.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T22:48:11.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T22:48:11.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T22:48:11.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.unordered_write: 0 2023-09-23T22:48:11.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T22:48:11.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T22:48:11.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T22:48:11.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T22:48:11.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.row_cache: None 2023-09-23T22:48:11.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.wal_filter: None 2023-09-23T22:48:11.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T22:48:11.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T22:48:11.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.preserve_deletes: 0 2023-09-23T22:48:11.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.two_write_queues: 0 2023-09-23T22:48:11.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T22:48:11.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.atomic_flush: 0 2023-09-23T22:48:11.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T22:48:11.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T22:48:11.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T22:48:11.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.log_readahead_size: 0 2023-09-23T22:48:11.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T22:48:11.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T22:48:11.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T22:48:11.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T22:48:11.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T22:48:11.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T22:48:11.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_background_jobs: 2 2023-09-23T22:48:11.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_background_compactions: -1 2023-09-23T22:48:11.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_subcompactions: 1 2023-09-23T22:48:11.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T22:48:11.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T22:48:11.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T22:48:11.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T22:48:11.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T22:48:11.949 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T22:48:11.949 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T22:48:11.949 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T22:48:11.949 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_open_files: -1 2023-09-23T22:48:11.949 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T22:48:11.949 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T22:48:11.949 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T22:48:11.949 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T22:48:11.949 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_background_flushes: -1 2023-09-23T22:48:11.949 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Compression algorithms supported: 2023-09-23T22:48:11.949 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T22:48:11.949 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: kZSTD supported: 0 2023-09-23T22:48:11.949 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: kXpressCompression supported: 0 2023-09-23T22:48:11.950 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T22:48:11.950 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: kZlibCompression supported: 1 2023-09-23T22:48:11.950 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: kSnappyCompression supported: 1 2023-09-23T22:48:11.950 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: kLZ4Compression supported: 1 2023-09-23T22:48:11.950 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: kBZip2Compression supported: 0 2023-09-23T22:48:11.950 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T22:48:11.950 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2023-09-23T22:48:11.950 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T22:48:11.950 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T22:48:11.950 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.merge_operator: 2023-09-23T22:48:11.950 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compaction_filter: None 2023-09-23T22:48:11.950 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T22:48:11.951 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T22:48:11.951 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T22:48:11.951 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T22:48:11.951 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55dfb4918fb8) 2023-09-23T22:48:11.951 INFO:journalctl@ceph.mon.b.smithi138.stdout: cache_index_and_filter_blocks: 1 2023-09-23T22:48:11.951 INFO:journalctl@ceph.mon.b.smithi138.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T22:48:11.951 INFO:journalctl@ceph.mon.b.smithi138.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T22:48:11.951 INFO:journalctl@ceph.mon.b.smithi138.stdout: pin_top_level_index_and_filter: 1 2023-09-23T22:48:11.951 INFO:journalctl@ceph.mon.b.smithi138.stdout: index_type: 0 2023-09-23T22:48:11.951 INFO:journalctl@ceph.mon.b.smithi138.stdout: data_block_index_type: 0 2023-09-23T22:48:11.951 INFO:journalctl@ceph.mon.b.smithi138.stdout: index_shortening: 1 2023-09-23T22:48:11.951 INFO:journalctl@ceph.mon.b.smithi138.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T22:48:11.951 INFO:journalctl@ceph.mon.b.smithi138.stdout: hash_index_allow_collision: 1 2023-09-23T22:48:11.952 INFO:journalctl@ceph.mon.b.smithi138.stdout: checksum: 1 2023-09-23T22:48:11.952 INFO:journalctl@ceph.mon.b.smithi138.stdout: no_block_cache: 0 2023-09-23T22:48:11.952 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_cache: 0x55dfb494b090 2023-09-23T22:48:11.952 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_cache_name: BinnedLRUCache 2023-09-23T22:48:11.952 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_cache_options: 2023-09-23T22:48:11.952 INFO:journalctl@ceph.mon.b.smithi138.stdout: capacity : 536870912 2023-09-23T22:48:11.952 INFO:journalctl@ceph.mon.b.smithi138.stdout: num_shard_bits : 4 2023-09-23T22:48:11.952 INFO:journalctl@ceph.mon.b.smithi138.stdout: strict_capacity_limit : 0 2023-09-23T22:48:11.952 INFO:journalctl@ceph.mon.b.smithi138.stdout: high_pri_pool_ratio: 0.000 2023-09-23T22:48:11.952 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_cache_compressed: (nil) 2023-09-23T22:48:11.952 INFO:journalctl@ceph.mon.b.smithi138.stdout: persistent_cache: (nil) 2023-09-23T22:48:11.952 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_size: 4096 2023-09-23T22:48:11.952 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_size_deviation: 10 2023-09-23T22:48:11.953 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_restart_interval: 16 2023-09-23T22:48:11.953 INFO:journalctl@ceph.mon.b.smithi138.stdout: index_block_restart_interval: 1 2023-09-23T22:48:11.953 INFO:journalctl@ceph.mon.b.smithi138.stdout: metadata_block_size: 4096 2023-09-23T22:48:11.953 INFO:journalctl@ceph.mon.b.smithi138.stdout: partition_filters: 0 2023-09-23T22:48:11.953 INFO:journalctl@ceph.mon.b.smithi138.stdout: use_delta_encoding: 1 2023-09-23T22:48:11.953 INFO:journalctl@ceph.mon.b.smithi138.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T22:48:11.953 INFO:journalctl@ceph.mon.b.smithi138.stdout: whole_key_filtering: 1 2023-09-23T22:48:11.953 INFO:journalctl@ceph.mon.b.smithi138.stdout: verify_compression: 0 2023-09-23T22:48:11.953 INFO:journalctl@ceph.mon.b.smithi138.stdout: read_amp_bytes_per_bit: 0 2023-09-23T22:48:11.953 INFO:journalctl@ceph.mon.b.smithi138.stdout: format_version: 4 2023-09-23T22:48:11.953 INFO:journalctl@ceph.mon.b.smithi138.stdout: enable_index_compression: 1 2023-09-23T22:48:11.953 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_align: 0 2023-09-23T22:48:11.953 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T22:48:11.954 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T22:48:11.954 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compression: NoCompression 2023-09-23T22:48:11.954 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T22:48:11.954 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T22:48:11.954 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T22:48:11.954 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.num_levels: 7 2023-09-23T22:48:11.954 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T22:48:11.954 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T22:48:11.954 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T22:48:11.954 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T22:48:11.954 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T22:48:11.954 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T22:48:11.955 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T22:48:11.955 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:48:11.955 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T22:48:11.955 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T22:48:11.955 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T22:48:11.956 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T22:48:11.956 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T22:48:11.956 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T22:48:11.956 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:48:11.956 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T22:48:11.956 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T22:48:11.956 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T22:48:11.956 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T22:48:11.956 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T22:48:11.957 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T22:48:11.957 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T22:48:11.957 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T22:48:11.957 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T22:48:11.957 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T22:48:11.957 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T22:48:11.957 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T22:48:11.957 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T22:48:11.957 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T22:48:11.957 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T22:48:11.957 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T22:48:11.957 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T22:48:11.957 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T22:48:11.958 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T22:48:11.958 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T22:48:11.958 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T22:48:11.958 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T22:48:11.958 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T22:48:11.958 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T22:48:11.958 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T22:48:11.958 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T22:48:11.958 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T22:48:11.958 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T22:48:11.958 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T22:48:11.958 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T22:48:11.959 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T22:48:11.959 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T22:48:11.959 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T22:48:11.959 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T22:48:11.959 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.table_properties_collectors: 2023-09-23T22:48:11.959 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.inplace_update_support: 0 2023-09-23T22:48:11.959 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T22:48:11.959 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T22:48:11.959 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T22:48:11.959 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T22:48:11.959 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.bloom_locality: 0 2023-09-23T22:48:11.959 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.max_successive_merges: 0 2023-09-23T22:48:11.960 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T22:48:11.960 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T22:48:11.960 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T22:48:11.960 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T22:48:11.960 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.ttl: 2592000 2023-09-23T22:48:11.960 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T22:48:11.960 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.enable_blob_files: false 2023-09-23T22:48:11.960 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.min_blob_size: 0 2023-09-23T22:48:11.960 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T22:48:11.960 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T22:48:11.960 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T22:48:11.960 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T22:48:11.960 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T22:48:11.961 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-09-23T22:48:11.961 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-09-23T22:48:11.961 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695509291555788, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-09-23T22:48:11.961 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-09-23T22:48:11.961 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T22:48:11.961 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695509291556050, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695509291, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "3f235c90-35dc-4a63-b01f-cd2acfe6f82c", "db_session_id": "256PZ4SRSE856YP99G27"}} 2023-09-23T22:48:11.961 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-09-23T22:48:11.961 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695509291556365, "job": 1, "event": "recovery_finished"} 2023-09-23T22:48:11.961 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T22:48:11.961 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55dfb48df180 2023-09-23T22:48:11.961 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: DB pointer 0x55dfb49b4000 2023-09-23T22:48:11.961 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T22:48:11.962 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T22:48:11.962 INFO:journalctl@ceph.mon.b.smithi138.stdout: ** DB Stats ** 2023-09-23T22:48:11.962 INFO:journalctl@ceph.mon.b.smithi138.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:48:11.962 INFO:journalctl@ceph.mon.b.smithi138.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T22:48:11.962 INFO:journalctl@ceph.mon.b.smithi138.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:48:11.962 INFO:journalctl@ceph.mon.b.smithi138.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:48:11.962 INFO:journalctl@ceph.mon.b.smithi138.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T22:48:11.962 INFO:journalctl@ceph.mon.b.smithi138.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T22:48:11.962 INFO:journalctl@ceph.mon.b.smithi138.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:48:11.962 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2023-09-23T22:48:11.962 INFO:journalctl@ceph.mon.b.smithi138.stdout: ** Compaction Stats [default] ** 2023-09-23T22:48:11.962 INFO:journalctl@ceph.mon.b.smithi138.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:48:11.963 INFO:journalctl@ceph.mon.b.smithi138.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:48:11.963 INFO:journalctl@ceph.mon.b.smithi138.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T22:48:11.963 INFO:journalctl@ceph.mon.b.smithi138.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T22:48:11.963 INFO:journalctl@ceph.mon.b.smithi138.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T22:48:11.963 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2023-09-23T22:48:11.963 INFO:journalctl@ceph.mon.b.smithi138.stdout: ** Compaction Stats [default] ** 2023-09-23T22:48:11.963 INFO:journalctl@ceph.mon.b.smithi138.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:48:11.963 INFO:journalctl@ceph.mon.b.smithi138.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:48:11.963 INFO:journalctl@ceph.mon.b.smithi138.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T22:48:11.963 INFO:journalctl@ceph.mon.b.smithi138.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:48:11.963 INFO:journalctl@ceph.mon.b.smithi138.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T22:48:11.963 INFO:journalctl@ceph.mon.b.smithi138.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:48:11.964 INFO:journalctl@ceph.mon.b.smithi138.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:48:11.964 INFO:journalctl@ceph.mon.b.smithi138.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:48:11.964 INFO:journalctl@ceph.mon.b.smithi138.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:48:11.964 INFO:journalctl@ceph.mon.b.smithi138.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:48:11.964 INFO:journalctl@ceph.mon.b.smithi138.stdout: Interval compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:48:11.964 INFO:journalctl@ceph.mon.b.smithi138.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:48:11.964 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2023-09-23T22:48:11.964 INFO:journalctl@ceph.mon.b.smithi138.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:48:11.964 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2023-09-23T22:48:11.964 INFO:journalctl@ceph.mon.b.smithi138.stdout: ** Compaction Stats [default] ** 2023-09-23T22:48:11.964 INFO:journalctl@ceph.mon.b.smithi138.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:48:11.964 INFO:journalctl@ceph.mon.b.smithi138.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:48:11.964 INFO:journalctl@ceph.mon.b.smithi138.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T22:48:11.965 INFO:journalctl@ceph.mon.b.smithi138.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T22:48:11.965 INFO:journalctl@ceph.mon.b.smithi138.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T22:48:11.965 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2023-09-23T22:48:11.965 INFO:journalctl@ceph.mon.b.smithi138.stdout: ** Compaction Stats [default] ** 2023-09-23T22:48:11.965 INFO:journalctl@ceph.mon.b.smithi138.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:48:11.965 INFO:journalctl@ceph.mon.b.smithi138.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:48:11.965 INFO:journalctl@ceph.mon.b.smithi138.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T22:48:11.965 INFO:journalctl@ceph.mon.b.smithi138.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:48:11.965 INFO:journalctl@ceph.mon.b.smithi138.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T22:48:11.965 INFO:journalctl@ceph.mon.b.smithi138.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:48:11.965 INFO:journalctl@ceph.mon.b.smithi138.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:48:11.965 INFO:journalctl@ceph.mon.b.smithi138.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:48:11.966 INFO:journalctl@ceph.mon.b.smithi138.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:48:11.966 INFO:journalctl@ceph.mon.b.smithi138.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:48:11.966 INFO:journalctl@ceph.mon.b.smithi138.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:48:11.966 INFO:journalctl@ceph.mon.b.smithi138.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:48:11.966 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2023-09-23T22:48:11.966 INFO:journalctl@ceph.mon.b.smithi138.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:48:11.966 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: mon.b does not exist in monmap, will attempt to join an existing cluster 2023-09-23T22:48:11.966 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: using public_addr v2:172.21.15.138:0/0 -> [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] 2023-09-23T22:48:11.966 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: starting mon.b rank -1 at public addrs [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] at bind addrs [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:48:11.966 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: mon.b@-1(???) e0 preinit fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:48:11.966 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: mon.b@-1(synchronizing).mds e1 new map 2023-09-23T22:48:11.966 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: mon.b@-1(synchronizing).mds e1 print_map 2023-09-23T22:48:11.966 INFO:journalctl@ceph.mon.b.smithi138.stdout: e1 2023-09-23T22:48:11.967 INFO:journalctl@ceph.mon.b.smithi138.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T22:48:11.967 INFO:journalctl@ceph.mon.b.smithi138.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:48:11.967 INFO:journalctl@ceph.mon.b.smithi138.stdout: legacy client fscid: -1 2023-09-23T22:48:11.967 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2023-09-23T22:48:11.967 INFO:journalctl@ceph.mon.b.smithi138.stdout: No filesystems configured 2023-09-23T22:48:11.967 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-23T22:48:11.967 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-23T22:48:11.967 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-23T22:48:11.967 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-23T22:48:11.967 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-23T22:48:11.967 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-23T22:48:11.967 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T22:48:11.967 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:48:11.968 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:48:11.968 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:48:11.968 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:48:11.968 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:48:11.968 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:48:11.968 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2292132868' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T22:48:11.968 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:48:11.968 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:11.968 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:11.968 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi110:172.21.15.110=a;smithi110:[v2:172.21.15.110:3301,v1:172.21.15.110:6790]=c;smithi138:172.21.15.138=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:48:11.968 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: Saving service mon spec with placement smithi110:172.21.15.110=a;smithi110:[v2:172.21.15.110:3301,v1:172.21.15.110:6790]=c;smithi138:172.21.15.138=b;count:3 2023-09-23T22:48:11.968 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:11.968 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:11.969 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:11.969 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:11.969 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:11.969 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:48:11.969 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:11.969 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: Deploying daemon mon.b on smithi138 2023-09-23T22:48:11.969 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:11.969 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T22:48:11.969 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 ceph-mon[104418]: mon.b@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-09-23T22:48:11.969 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 bash[104309]: 522bb9710d62f912e2235cecae3d663239cb620a3ec9cb74177a6270ab70d783 2023-09-23T22:48:11.969 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:11 smithi138 systemd[1]: Started Ceph mon.b for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:48:13.017 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T22:48:13.017 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T22:48:13.362 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:48:16.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:48:16.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:16.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: mon.a calling monitor election 2023-09-23T22:48:16.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:16.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: mon.b calling monitor election 2023-09-23T22:48:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T22:48:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: monmap e2: 2 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0]} removed_ranks: {} 2023-09-23T22:48:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: fsmap 2023-09-23T22:48:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:48:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: mgrmap e13: y(active, since 37s) 2023-09-23T22:48:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: overall HEALTH_OK 2023-09-23T22:48:16.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:16.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:16.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:16.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:48:16.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:16.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:48:16.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:16.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: mon.a calling monitor election 2023-09-23T22:48:16.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:16.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:16.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:16.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: mon.b calling monitor election 2023-09-23T22:48:16.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:16.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:16.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T22:48:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: monmap e2: 2 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0]} removed_ranks: {} 2023-09-23T22:48:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: fsmap 2023-09-23T22:48:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:48:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: mgrmap e13: y(active, since 37s) 2023-09-23T22:48:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: overall HEALTH_OK 2023-09-23T22:48:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:48:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:17.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:17 smithi138 ceph-mon[104418]: Deploying daemon mon.c on smithi110 2023-09-23T22:48:17.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:17 smithi138 ceph-mon[104418]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:17.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:17 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:17.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:17 smithi110 ceph-mon[100243]: Deploying daemon mon.c on smithi110 2023-09-23T22:48:17.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:17 smithi110 ceph-mon[100243]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:17.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:17 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:18.045 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:48:18.045 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":2,"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","modified":"2023-09-23T22:48:11.575897Z","created":"2023-09-23T22:46:46.458336Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:3300","nonce":0},{"type":"v1","addr":"172.21.15.110:6789","nonce":0}]},"addr":"172.21.15.110:6789/0","public_addr":"172.21.15.110:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:3300","nonce":0},{"type":"v1","addr":"172.21.15.138:6789","nonce":0}]},"addr":"172.21.15.138:6789/0","public_addr":"172.21.15.138:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-09-23T22:48:18.047 INFO:teuthology.orchestra.run.smithi138.stderr:dumped monmap epoch 2 2023-09-23T22:48:18.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:18 smithi138 ceph-mon[104418]: from='client.? 172.21.15.138:0/1601517244' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:48:18.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:18 smithi110 ceph-mon[100243]: from='client.? 172.21.15.138:0/1601517244' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:48:19.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 systemd[1]: Starting Ceph mon.c for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:48:19.682 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T22:48:19.683 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T22:48:20.031 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:48:20.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T22:48:20.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable), process ceph-mon, pid 7 2023-09-23T22:48:20.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: pidfile_write: ignore empty --pid-file 2023-09-23T22:48:20.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: load: jerasure load: lrc 2023-09-23T22:48:20.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: RocksDB version: 6.15.5 2023-09-23T22:48:20.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T22:48:20.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Compile date Sep 23 2023 2023-09-23T22:48:20.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: DB SUMMARY 2023-09-23T22:48:20.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: DB Session ID: URB56ND7FQYYNJ2DAIDL 2023-09-23T22:48:20.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: CURRENT file: CURRENT 2023-09-23T22:48:20.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T22:48:20.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-09-23T22:48:20.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-09-23T22:48:20.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2023-09-23T22:48:20.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.error_if_exists: 0 2023-09-23T22:48:20.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.create_if_missing: 0 2023-09-23T22:48:20.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.paranoid_checks: 1 2023-09-23T22:48:20.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T22:48:20.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.env: 0x561032658a80 2023-09-23T22:48:20.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.fs: Posix File System 2023-09-23T22:48:20.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.info_log: 0x561034e405a0 2023-09-23T22:48:20.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T22:48:20.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.statistics: (nil) 2023-09-23T22:48:20.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.use_fsync: 0 2023-09-23T22:48:20.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_log_file_size: 0 2023-09-23T22:48:20.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T22:48:20.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T22:48:20.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T22:48:20.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T22:48:20.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.allow_fallocate: 1 2023-09-23T22:48:20.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T22:48:20.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T22:48:20.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.use_direct_reads: 0 2023-09-23T22:48:20.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T22:48:20.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T22:48:20.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.db_log_dir: 2023-09-23T22:48:20.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-09-23T22:48:20.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T22:48:20.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T22:48:20.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T22:48:20.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T22:48:20.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T22:48:20.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T22:48:20.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T22:48:20.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T22:48:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.write_buffer_manager: 0x561035bb2930 2023-09-23T22:48:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T22:48:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T22:48:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T22:48:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T22:48:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T22:48:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T22:48:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T22:48:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T22:48:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T22:48:20.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.unordered_write: 0 2023-09-23T22:48:20.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T22:48:20.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T22:48:20.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T22:48:20.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T22:48:20.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.row_cache: None 2023-09-23T22:48:20.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.wal_filter: None 2023-09-23T22:48:20.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T22:48:20.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T22:48:20.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.preserve_deletes: 0 2023-09-23T22:48:20.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.two_write_queues: 0 2023-09-23T22:48:20.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T22:48:20.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.atomic_flush: 0 2023-09-23T22:48:20.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T22:48:20.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T22:48:20.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T22:48:20.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.log_readahead_size: 0 2023-09-23T22:48:20.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T22:48:20.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T22:48:20.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T22:48:20.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T22:48:20.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T22:48:20.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T22:48:20.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_background_jobs: 2 2023-09-23T22:48:20.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_background_compactions: -1 2023-09-23T22:48:20.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_subcompactions: 1 2023-09-23T22:48:20.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T22:48:20.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T22:48:20.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T22:48:20.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T22:48:20.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T22:48:20.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T22:48:20.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T22:48:20.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T22:48:20.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_open_files: -1 2023-09-23T22:48:20.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T22:48:20.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T22:48:20.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T22:48:20.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T22:48:20.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_background_flushes: -1 2023-09-23T22:48:20.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Compression algorithms supported: 2023-09-23T22:48:20.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T22:48:20.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: kZSTD supported: 0 2023-09-23T22:48:20.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: kXpressCompression supported: 0 2023-09-23T22:48:20.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T22:48:20.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: kZlibCompression supported: 1 2023-09-23T22:48:20.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: kSnappyCompression supported: 1 2023-09-23T22:48:20.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: kLZ4Compression supported: 1 2023-09-23T22:48:20.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: kBZip2Compression supported: 0 2023-09-23T22:48:20.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T22:48:20.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2023-09-23T22:48:20.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T22:48:20.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T22:48:20.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.merge_operator: 2023-09-23T22:48:20.213 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compaction_filter: None 2023-09-23T22:48:20.213 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T22:48:20.213 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T22:48:20.213 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T22:48:20.213 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T22:48:20.213 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561034e92fb8) 2023-09-23T22:48:20.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: cache_index_and_filter_blocks: 1 2023-09-23T22:48:20.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T22:48:20.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T22:48:20.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: pin_top_level_index_and_filter: 1 2023-09-23T22:48:20.215 INFO:journalctl@ceph.mon.c.smithi110.stdout: index_type: 0 2023-09-23T22:48:20.215 INFO:journalctl@ceph.mon.c.smithi110.stdout: data_block_index_type: 0 2023-09-23T22:48:20.215 INFO:journalctl@ceph.mon.c.smithi110.stdout: index_shortening: 1 2023-09-23T22:48:20.215 INFO:journalctl@ceph.mon.c.smithi110.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T22:48:20.216 INFO:journalctl@ceph.mon.c.smithi110.stdout: hash_index_allow_collision: 1 2023-09-23T22:48:20.216 INFO:journalctl@ceph.mon.c.smithi110.stdout: checksum: 1 2023-09-23T22:48:20.216 INFO:journalctl@ceph.mon.c.smithi110.stdout: no_block_cache: 0 2023-09-23T22:48:20.216 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_cache: 0x561034ec5090 2023-09-23T22:48:20.216 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_cache_name: BinnedLRUCache 2023-09-23T22:48:20.216 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_cache_options: 2023-09-23T22:48:20.216 INFO:journalctl@ceph.mon.c.smithi110.stdout: capacity : 536870912 2023-09-23T22:48:20.217 INFO:journalctl@ceph.mon.c.smithi110.stdout: num_shard_bits : 4 2023-09-23T22:48:20.217 INFO:journalctl@ceph.mon.c.smithi110.stdout: strict_capacity_limit : 0 2023-09-23T22:48:20.217 INFO:journalctl@ceph.mon.c.smithi110.stdout: high_pri_pool_ratio: 0.000 2023-09-23T22:48:20.217 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_cache_compressed: (nil) 2023-09-23T22:48:20.217 INFO:journalctl@ceph.mon.c.smithi110.stdout: persistent_cache: (nil) 2023-09-23T22:48:20.217 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_size: 4096 2023-09-23T22:48:20.217 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_size_deviation: 10 2023-09-23T22:48:20.217 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_restart_interval: 16 2023-09-23T22:48:20.217 INFO:journalctl@ceph.mon.c.smithi110.stdout: index_block_restart_interval: 1 2023-09-23T22:48:20.217 INFO:journalctl@ceph.mon.c.smithi110.stdout: metadata_block_size: 4096 2023-09-23T22:48:20.218 INFO:journalctl@ceph.mon.c.smithi110.stdout: partition_filters: 0 2023-09-23T22:48:20.218 INFO:journalctl@ceph.mon.c.smithi110.stdout: use_delta_encoding: 1 2023-09-23T22:48:20.218 INFO:journalctl@ceph.mon.c.smithi110.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T22:48:20.218 INFO:journalctl@ceph.mon.c.smithi110.stdout: whole_key_filtering: 1 2023-09-23T22:48:20.218 INFO:journalctl@ceph.mon.c.smithi110.stdout: verify_compression: 0 2023-09-23T22:48:20.218 INFO:journalctl@ceph.mon.c.smithi110.stdout: read_amp_bytes_per_bit: 0 2023-09-23T22:48:20.218 INFO:journalctl@ceph.mon.c.smithi110.stdout: format_version: 4 2023-09-23T22:48:20.218 INFO:journalctl@ceph.mon.c.smithi110.stdout: enable_index_compression: 1 2023-09-23T22:48:20.218 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_align: 0 2023-09-23T22:48:20.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T22:48:20.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T22:48:20.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compression: NoCompression 2023-09-23T22:48:20.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T22:48:20.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T22:48:20.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T22:48:20.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.num_levels: 7 2023-09-23T22:48:20.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T22:48:20.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T22:48:20.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T22:48:20.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T22:48:20.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T22:48:20.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T22:48:20.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T22:48:20.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:48:20.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T22:48:20.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T22:48:20.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T22:48:20.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T22:48:20.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T22:48:20.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T22:48:20.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:48:20.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T22:48:20.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T22:48:20.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T22:48:20.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T22:48:20.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T22:48:20.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T22:48:20.222 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T22:48:20.222 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T22:48:20.222 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T22:48:20.222 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T22:48:20.222 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T22:48:20.222 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T22:48:20.222 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T22:48:20.222 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T22:48:20.222 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T22:48:20.222 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T22:48:20.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T22:48:20.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T22:48:20.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T22:48:20.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T22:48:20.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T22:48:20.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T22:48:20.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T22:48:20.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T22:48:20.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T22:48:20.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T22:48:20.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T22:48:20.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T22:48:20.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T22:48:20.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T22:48:20.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T22:48:20.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T22:48:20.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T22:48:20.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T22:48:20.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.table_properties_collectors: 2023-09-23T22:48:20.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.inplace_update_support: 0 2023-09-23T22:48:20.225 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T22:48:20.225 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T22:48:20.225 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T22:48:20.225 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T22:48:20.225 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.bloom_locality: 0 2023-09-23T22:48:20.225 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.max_successive_merges: 0 2023-09-23T22:48:20.225 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T22:48:20.225 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T22:48:20.226 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T22:48:20.226 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T22:48:20.226 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.ttl: 2592000 2023-09-23T22:48:20.226 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T22:48:20.226 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.enable_blob_files: false 2023-09-23T22:48:20.226 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.min_blob_size: 0 2023-09-23T22:48:20.226 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T22:48:20.226 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T22:48:20.227 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T22:48:20.227 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T22:48:20.227 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T22:48:20.227 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-09-23T22:48:20.227 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-09-23T22:48:20.227 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695509299763909, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-09-23T22:48:20.227 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-09-23T22:48:20.227 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T22:48:20.228 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695509299764171, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695509299, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "4ff42085-b787-466c-b926-baa3cbd57f9e", "db_session_id": "URB56ND7FQYYNJ2DAIDL"}} 2023-09-23T22:48:20.228 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-09-23T22:48:20.228 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695509299764521, "job": 1, "event": "recovery_finished"} 2023-09-23T22:48:20.228 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T22:48:20.228 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x561034e59180 2023-09-23T22:48:20.228 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: DB pointer 0x561034f2e000 2023-09-23T22:48:20.228 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T22:48:20.228 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T22:48:20.228 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** DB Stats ** 2023-09-23T22:48:20.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:48:20.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T22:48:20.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:48:20.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:48:20.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T22:48:20.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T22:48:20.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:48:20.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2023-09-23T22:48:20.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** Compaction Stats [default] ** 2023-09-23T22:48:20.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:48:20.229 INFO:journalctl@ceph.mon.c.smithi110.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:48:20.230 INFO:journalctl@ceph.mon.c.smithi110.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-09-23T22:48:20.230 INFO:journalctl@ceph.mon.c.smithi110.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-09-23T22:48:20.230 INFO:journalctl@ceph.mon.c.smithi110.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-09-23T22:48:20.230 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2023-09-23T22:48:20.230 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** Compaction Stats [default] ** 2023-09-23T22:48:20.230 INFO:journalctl@ceph.mon.c.smithi110.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:48:20.230 INFO:journalctl@ceph.mon.c.smithi110.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:48:20.230 INFO:journalctl@ceph.mon.c.smithi110.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-09-23T22:48:20.230 INFO:journalctl@ceph.mon.c.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:48:20.230 INFO:journalctl@ceph.mon.c.smithi110.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T22:48:20.231 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:48:20.231 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:48:20.231 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:48:20.231 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:48:20.231 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative compaction: 0.00 GB write, 0.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:48:20.231 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval compaction: 0.00 GB write, 0.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:48:20.231 INFO:journalctl@ceph.mon.c.smithi110.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:48:20.231 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2023-09-23T22:48:20.231 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:48:20.231 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2023-09-23T22:48:20.232 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** Compaction Stats [default] ** 2023-09-23T22:48:20.232 INFO:journalctl@ceph.mon.c.smithi110.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:48:20.232 INFO:journalctl@ceph.mon.c.smithi110.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:48:20.232 INFO:journalctl@ceph.mon.c.smithi110.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-09-23T22:48:20.232 INFO:journalctl@ceph.mon.c.smithi110.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-09-23T22:48:20.232 INFO:journalctl@ceph.mon.c.smithi110.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T22:48:20.232 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2023-09-23T22:48:20.232 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** Compaction Stats [default] ** 2023-09-23T22:48:20.232 INFO:journalctl@ceph.mon.c.smithi110.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:48:20.233 INFO:journalctl@ceph.mon.c.smithi110.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:48:20.233 INFO:journalctl@ceph.mon.c.smithi110.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-09-23T22:48:20.233 INFO:journalctl@ceph.mon.c.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:48:20.233 INFO:journalctl@ceph.mon.c.smithi110.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T22:48:20.233 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:48:20.233 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:48:20.233 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:48:20.233 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:48:20.234 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:48:20.234 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:48:20.234 INFO:journalctl@ceph.mon.c.smithi110.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:48:20.234 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2023-09-23T22:48:20.234 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:48:20.234 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-09-23T22:48:20.234 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: using public_addrv [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] 2023-09-23T22:48:20.234 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: starting mon.c rank -1 at public addrs [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] at bind addrs [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:48:20.234 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.c@-1(???) e0 preinit fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:48:20.234 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 bash[118755]: aaa9613ef02dcdbf270885a3af9c77bd8d0ecf9c99c4176b4a9523d8189835a7 2023-09-23T22:48:20.235 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 systemd[1]: Started Ceph mon.c for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:48:20.235 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.c@-1(synchronizing).mds e1 new map 2023-09-23T22:48:20.235 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.c@-1(synchronizing).mds e1 print_map 2023-09-23T22:48:20.235 INFO:journalctl@ceph.mon.c.smithi110.stdout: e1 2023-09-23T22:48:20.235 INFO:journalctl@ceph.mon.c.smithi110.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T22:48:20.235 INFO:journalctl@ceph.mon.c.smithi110.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:48:20.235 INFO:journalctl@ceph.mon.c.smithi110.stdout: legacy client fscid: -1 2023-09-23T22:48:20.235 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2023-09-23T22:48:20.235 INFO:journalctl@ceph.mon.c.smithi110.stdout: No filesystems configured 2023-09-23T22:48:20.235 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-23T22:48:20.236 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-23T22:48:20.236 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-23T22:48:20.236 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-23T22:48:20.236 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-23T22:48:20.236 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-23T22:48:20.236 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T22:48:20.236 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:48:20.236 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:48:20.236 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:48:20.236 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:48:20.236 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:48:20.236 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:48:20.237 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2292132868' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T22:48:20.237 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:48:20.237 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:20.237 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:20.237 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi110:172.21.15.110=a;smithi110:[v2:172.21.15.110:3301,v1:172.21.15.110:6790]=c;smithi138:172.21.15.138=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:48:20.237 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: Saving service mon spec with placement smithi110:172.21.15.110=a;smithi110:[v2:172.21.15.110:3301,v1:172.21.15.110:6790]=c;smithi138:172.21.15.138=b;count:3 2023-09-23T22:48:20.237 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:20.237 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:20.237 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:20.237 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:20.238 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:20.238 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:48:20.238 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:20.238 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: Deploying daemon mon.b on smithi138 2023-09-23T22:48:20.238 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:20.238 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:48:20.238 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:20.238 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.a calling monitor election 2023-09-23T22:48:20.238 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:20.239 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:20.239 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:20.239 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.b calling monitor election 2023-09-23T22:48:20.239 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:20.239 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:20.239 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:20.239 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:20.239 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T22:48:20.240 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: monmap e2: 2 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0]} removed_ranks: {} 2023-09-23T22:48:20.240 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: fsmap 2023-09-23T22:48:20.240 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:48:20.240 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mgrmap e13: y(active, since 37s) 2023-09-23T22:48:20.240 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: overall HEALTH_OK 2023-09-23T22:48:20.240 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:20.240 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:20.241 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:20.241 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:48:20.241 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:20.241 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: Deploying daemon mon.c on smithi110 2023-09-23T22:48:20.241 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:20.241 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:20.241 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: from='client.? 172.21.15.138:0/1601517244' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:48:20.241 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T22:48:20.241 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:19 smithi110 ceph-mon[118821]: mon.c@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-09-23T22:48:25.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:48:25.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: mon.a calling monitor election 2023-09-23T22:48:25.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: mon.b calling monitor election 2023-09-23T22:48:25.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:25.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: mon.c calling monitor election 2023-09-23T22:48:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:48:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: monmap e3: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} 2023-09-23T22:48:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: fsmap 2023-09-23T22:48:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:48:25.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: mgrmap e13: y(active, since 45s) 2023-09-23T22:48:25.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: overall HEALTH_OK 2023-09-23T22:48:25.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:25.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:25.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:25.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:25.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:48:25.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: mon.a calling monitor election 2023-09-23T22:48:25.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: mon.b calling monitor election 2023-09-23T22:48:25.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:25.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: mon.c calling monitor election 2023-09-23T22:48:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:48:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: monmap e3: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} 2023-09-23T22:48:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: fsmap 2023-09-23T22:48:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:48:25.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: mgrmap e13: y(active, since 45s) 2023-09-23T22:48:25.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: overall HEALTH_OK 2023-09-23T22:48:25.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:25.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:25.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:25.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:25.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:48:25.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: mon.a calling monitor election 2023-09-23T22:48:25.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: mon.b calling monitor election 2023-09-23T22:48:25.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:48:25.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:25.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: mon.c calling monitor election 2023-09-23T22:48:25.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:25.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:25.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:48:25.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: monmap e3: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} 2023-09-23T22:48:25.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: fsmap 2023-09-23T22:48:25.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:48:25.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: mgrmap e13: y(active, since 45s) 2023-09-23T22:48:25.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: overall HEALTH_OK 2023-09-23T22:48:25.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:25.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:25.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:25.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:25.445 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-09-23T22:48:25.445 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":3,"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","modified":"2023-09-23T22:48:19.784822Z","created":"2023-09-23T22:46:46.458336Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:3300","nonce":0},{"type":"v1","addr":"172.21.15.110:6789","nonce":0}]},"addr":"172.21.15.110:6789/0","public_addr":"172.21.15.110:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:3300","nonce":0},{"type":"v1","addr":"172.21.15.138:6789","nonce":0}]},"addr":"172.21.15.138:6789/0","public_addr":"172.21.15.138:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:3301","nonce":0},{"type":"v1","addr":"172.21.15.110:6790","nonce":0}]},"addr":"172.21.15.110:6790/0","public_addr":"172.21.15.110:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-09-23T22:48:25.447 INFO:teuthology.orchestra.run.smithi138.stderr:dumped monmap epoch 3 2023-09-23T22:48:26.146 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:25 smithi138 ceph-mon[104418]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:26.146 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:25 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:26.146 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:25 smithi138 ceph-mon[104418]: from='client.? 172.21.15.138:0/2239066760' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:48:26.146 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:25 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:26.164 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-09-23T22:48:26.164 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph config generate-minimal-conf 2023-09-23T22:48:26.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:25 smithi110 ceph-mon[100243]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:26.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:25 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:26.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:25 smithi110 ceph-mon[100243]: from='client.? 172.21.15.138:0/2239066760' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:48:26.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:25 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:26.191 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:25 smithi110 ceph-mon[118821]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:26.191 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:25 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:26.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:25 smithi110 ceph-mon[118821]: from='client.? 172.21.15.138:0/2239066760' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:48:26.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:25 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:48:26.513 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:27.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:27.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:27.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:27 smithi138 ceph-mon[104418]: Updating smithi110:/etc/ceph/ceph.conf 2023-09-23T22:48:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:27 smithi138 ceph-mon[104418]: Updating smithi138:/etc/ceph/ceph.conf 2023-09-23T22:48:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:48:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:48:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:27.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:27.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:27.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[100243]: Updating smithi110:/etc/ceph/ceph.conf 2023-09-23T22:48:27.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[100243]: Updating smithi138:/etc/ceph/ceph.conf 2023-09-23T22:48:27.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:48:27.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:48:27.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:27.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:27.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:27.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[118821]: Updating smithi110:/etc/ceph/ceph.conf 2023-09-23T22:48:27.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[118821]: Updating smithi138:/etc/ceph/ceph.conf 2023-09-23T22:48:27.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:27.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:48:27.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:48:27.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:28 smithi138 ceph-mon[104418]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:48:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:28 smithi138 ceph-mon[104418]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:48:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:28 smithi138 ceph-mon[104418]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:28 smithi138 ceph-mon[104418]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T22:48:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:28 smithi138 ceph-mon[104418]: Reconfiguring daemon mon.a on smithi110 2023-09-23T22:48:28.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:28 smithi110 ceph-mon[100243]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:48:28.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:28 smithi110 ceph-mon[100243]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:48:28.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:28 smithi110 ceph-mon[100243]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:28.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:28 smithi110 ceph-mon[100243]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T22:48:28.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:28 smithi110 ceph-mon[100243]: Reconfiguring daemon mon.a on smithi110 2023-09-23T22:48:28.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:28 smithi110 ceph-mon[118821]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:48:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:28 smithi110 ceph-mon[118821]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:48:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:28 smithi110 ceph-mon[118821]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:28 smithi110 ceph-mon[118821]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T22:48:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:28 smithi110 ceph-mon[118821]: Reconfiguring daemon mon.a on smithi110 2023-09-23T22:48:28.590 INFO:teuthology.orchestra.run.smithi110.stdout:# minimal ceph.conf for f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:48:28.590 INFO:teuthology.orchestra.run.smithi110.stdout:[global] 2023-09-23T22:48:28.590 INFO:teuthology.orchestra.run.smithi110.stdout: fsid = f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:48:28.591 INFO:teuthology.orchestra.run.smithi110.stdout: mon_host = [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] 2023-09-23T22:48:29.310 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:29 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/4245154293' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:29.311 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:29 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:29.311 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:29 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/4245154293' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:29.311 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:29 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:29.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:29 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/4245154293' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:29.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:29 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:29.539 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-09-23T22:48:29.540 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-09-23T22:48:29.540 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T22:48:29.608 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-09-23T22:48:29.608 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:48:29.676 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-09-23T22:48:29.676 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T22:48:29.725 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-09-23T22:48:29.725 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:48:29.795 INFO:tasks.cephadm:Adding mgr.y on smithi110 2023-09-23T22:48:29.796 INFO:tasks.cephadm:Adding mgr.x on smithi138 2023-09-23T22:48:29.796 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph orch apply mgr '2;smithi110=y;smithi138=x' 2023-09-23T22:48:30.163 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:48:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:30 smithi138 ceph-mon[104418]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:30 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:30 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:30 smithi138 ceph-mon[104418]: Reconfiguring mon.c (monmap changed)... 2023-09-23T22:48:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:30 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:48:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:30 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:48:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:30 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:30 smithi138 ceph-mon[104418]: Reconfiguring daemon mon.c on smithi110 2023-09-23T22:48:30.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:30 smithi110 ceph-mon[100243]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:30.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:30 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:30.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:30 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:30.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:30 smithi110 ceph-mon[100243]: Reconfiguring mon.c (monmap changed)... 2023-09-23T22:48:30.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:30 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:48:30.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:30 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:48:30.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:30 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:30.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:30 smithi110 ceph-mon[100243]: Reconfiguring daemon mon.c on smithi110 2023-09-23T22:48:30.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:30 smithi110 ceph-mon[118821]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:30.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:30 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:30.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:30 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:30.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:30 smithi110 ceph-mon[118821]: Reconfiguring mon.c (monmap changed)... 2023-09-23T22:48:30.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:30 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:48:30.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:30 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:48:30.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:30 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:30.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:30 smithi110 ceph-mon[118821]: Reconfiguring daemon mon.c on smithi110 2023-09-23T22:48:31.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:31 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:31.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:31 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:31.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:31 smithi138 ceph-mon[104418]: Reconfiguring mon.b (monmap changed)... 2023-09-23T22:48:31.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:31 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:48:31.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:31 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:48:31.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:31 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:31.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:31 smithi138 ceph-mon[104418]: Reconfiguring daemon mon.b on smithi138 2023-09-23T22:48:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:31 smithi138 ceph-mon[104418]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:32.188 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled mgr update... 2023-09-23T22:48:32.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:31 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:32.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:31 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:32.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:31 smithi110 ceph-mon[100243]: Reconfiguring mon.b (monmap changed)... 2023-09-23T22:48:32.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:31 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:48:32.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:31 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:48:32.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:31 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:32.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:31 smithi110 ceph-mon[100243]: Reconfiguring daemon mon.b on smithi138 2023-09-23T22:48:32.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:31 smithi110 ceph-mon[100243]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:32.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:31 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:32.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:31 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:32.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:31 smithi110 ceph-mon[118821]: Reconfiguring mon.b (monmap changed)... 2023-09-23T22:48:32.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:31 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:48:32.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:31 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:48:32.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:31 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:32.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:31 smithi110 ceph-mon[118821]: Reconfiguring daemon mon.b on smithi138 2023-09-23T22:48:32.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:31 smithi110 ceph-mon[118821]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:33.078 DEBUG:teuthology.orchestra.run.smithi138:mgr.x> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mgr.x.service 2023-09-23T22:48:33.080 INFO:tasks.cephadm:Deploying OSDs... 2023-09-23T22:48:33.080 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-09-23T22:48:33.081 DEBUG:teuthology.orchestra.run.smithi110:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T22:48:33.102 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T22:48:33.102 DEBUG:teuthology.orchestra.run.smithi110:> stat /dev/vg_nvme/lv_1 2023-09-23T22:48:33.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:-- Logs begin at Sat 2023-09-23 22:31:02 UTC. -- 2023-09-23T22:48:33.162 INFO:teuthology.orchestra.run.smithi110.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T22:48:33.162 INFO:teuthology.orchestra.run.smithi110.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:48:33.162 INFO:teuthology.orchestra.run.smithi110.stdout:Device: 6h/6d Inode: 387574 Links: 1 2023-09-23T22:48:33.162 INFO:teuthology.orchestra.run.smithi110.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:48:33.162 INFO:teuthology.orchestra.run.smithi110.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:48:33.162 INFO:teuthology.orchestra.run.smithi110.stdout:Access: 2023-09-23 22:48:19.176719884 +0000 2023-09-23T22:48:33.162 INFO:teuthology.orchestra.run.smithi110.stdout:Modify: 2023-09-23 22:40:11.322359742 +0000 2023-09-23T22:48:33.162 INFO:teuthology.orchestra.run.smithi110.stdout:Change: 2023-09-23 22:40:11.322359742 +0000 2023-09-23T22:48:33.163 INFO:teuthology.orchestra.run.smithi110.stdout: Birth: - 2023-09-23T22:48:33.163 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T22:48:33.232 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records in 2023-09-23T22:48:33.232 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records out 2023-09-23T22:48:33.233 INFO:teuthology.orchestra.run.smithi110.stderr:512 bytes copied, 0.000120124 s, 4.3 MB/s 2023-09-23T22:48:33.234 DEBUG:teuthology.orchestra.run.smithi110:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T22:48:33.294 DEBUG:teuthology.orchestra.run.smithi110:> stat /dev/vg_nvme/lv_2 2023-09-23T22:48:33.353 INFO:teuthology.orchestra.run.smithi110.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T22:48:33.353 INFO:teuthology.orchestra.run.smithi110.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:48:33.353 INFO:teuthology.orchestra.run.smithi110.stdout:Device: 6h/6d Inode: 391314 Links: 1 2023-09-23T22:48:33.353 INFO:teuthology.orchestra.run.smithi110.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:48:33.353 INFO:teuthology.orchestra.run.smithi110.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:48:33.353 INFO:teuthology.orchestra.run.smithi110.stdout:Access: 2023-09-23 22:48:19.176719884 +0000 2023-09-23T22:48:33.353 INFO:teuthology.orchestra.run.smithi110.stdout:Modify: 2023-09-23 22:40:11.806351613 +0000 2023-09-23T22:48:33.354 INFO:teuthology.orchestra.run.smithi110.stdout:Change: 2023-09-23 22:40:11.806351613 +0000 2023-09-23T22:48:33.354 INFO:teuthology.orchestra.run.smithi110.stdout: Birth: - 2023-09-23T22:48:33.354 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T22:48:33.421 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records in 2023-09-23T22:48:33.421 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records out 2023-09-23T22:48:33.421 INFO:teuthology.orchestra.run.smithi110.stderr:512 bytes copied, 0.000143306 s, 3.6 MB/s 2023-09-23T22:48:33.422 DEBUG:teuthology.orchestra.run.smithi110:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T22:48:33.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:33 smithi138 ceph-mon[104418]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi110=y;smithi138=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:48:33.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:33 smithi138 ceph-mon[104418]: Saving service mgr spec with placement smithi110=y;smithi138=x;count:2 2023-09-23T22:48:33.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:33 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:33.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:33 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:33 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:33 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:33 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:33 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:33 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:33 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:48:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:33 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T22:48:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:33 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:48:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:33 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:33.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[100243]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi110=y;smithi138=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:48:33.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[100243]: Saving service mgr spec with placement smithi110=y;smithi138=x;count:2 2023-09-23T22:48:33.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:33.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:33.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:33.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:33.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:33.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:33.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:33.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:48:33.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T22:48:33.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:48:33.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:33.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[118821]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi110=y;smithi138=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:48:33.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[118821]: Saving service mgr spec with placement smithi110=y;smithi138=x;count:2 2023-09-23T22:48:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:48:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T22:48:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:48:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:33 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:33.472 DEBUG:teuthology.orchestra.run.smithi110:> stat /dev/vg_nvme/lv_3 2023-09-23T22:48:33.530 INFO:teuthology.orchestra.run.smithi110.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T22:48:33.530 INFO:teuthology.orchestra.run.smithi110.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:48:33.530 INFO:teuthology.orchestra.run.smithi110.stdout:Device: 6h/6d Inode: 391487 Links: 1 2023-09-23T22:48:33.530 INFO:teuthology.orchestra.run.smithi110.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:48:33.530 INFO:teuthology.orchestra.run.smithi110.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:48:33.530 INFO:teuthology.orchestra.run.smithi110.stdout:Access: 2023-09-23 22:48:19.176719884 +0000 2023-09-23T22:48:33.530 INFO:teuthology.orchestra.run.smithi110.stdout:Modify: 2023-09-23 22:40:12.301343299 +0000 2023-09-23T22:48:33.530 INFO:teuthology.orchestra.run.smithi110.stdout:Change: 2023-09-23 22:40:12.301343299 +0000 2023-09-23T22:48:33.531 INFO:teuthology.orchestra.run.smithi110.stdout: Birth: - 2023-09-23T22:48:33.531 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T22:48:33.596 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records in 2023-09-23T22:48:33.596 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records out 2023-09-23T22:48:33.596 INFO:teuthology.orchestra.run.smithi110.stderr:512 bytes copied, 9.66e-05 s, 5.3 MB/s 2023-09-23T22:48:33.597 DEBUG:teuthology.orchestra.run.smithi110:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T22:48:33.655 DEBUG:teuthology.orchestra.run.smithi110:> stat /dev/vg_nvme/lv_4 2023-09-23T22:48:33.712 INFO:teuthology.orchestra.run.smithi110.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T22:48:33.712 INFO:teuthology.orchestra.run.smithi110.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:48:33.712 INFO:teuthology.orchestra.run.smithi110.stdout:Device: 6h/6d Inode: 388588 Links: 1 2023-09-23T22:48:33.712 INFO:teuthology.orchestra.run.smithi110.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:48:33.712 INFO:teuthology.orchestra.run.smithi110.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:48:33.712 INFO:teuthology.orchestra.run.smithi110.stdout:Access: 2023-09-23 22:48:19.176719884 +0000 2023-09-23T22:48:33.712 INFO:teuthology.orchestra.run.smithi110.stdout:Modify: 2023-09-23 22:40:12.792335052 +0000 2023-09-23T22:48:33.712 INFO:teuthology.orchestra.run.smithi110.stdout:Change: 2023-09-23 22:40:12.792335052 +0000 2023-09-23T22:48:33.713 INFO:teuthology.orchestra.run.smithi110.stdout: Birth: - 2023-09-23T22:48:33.713 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T22:48:33.777 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records in 2023-09-23T22:48:33.777 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records out 2023-09-23T22:48:33.778 INFO:teuthology.orchestra.run.smithi110.stderr:512 bytes copied, 0.000127681 s, 4.0 MB/s 2023-09-23T22:48:33.778 DEBUG:teuthology.orchestra.run.smithi110:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T22:48:33.836 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-09-23T22:48:33.837 DEBUG:teuthology.orchestra.run.smithi138:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T22:48:33.855 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T22:48:33.855 DEBUG:teuthology.orchestra.run.smithi138:> stat /dev/vg_nvme/lv_1 2023-09-23T22:48:33.911 INFO:teuthology.orchestra.run.smithi138.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T22:48:33.912 INFO:teuthology.orchestra.run.smithi138.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:48:33.912 INFO:teuthology.orchestra.run.smithi138.stdout:Device: 6h/6d Inode: 386701 Links: 1 2023-09-23T22:48:33.912 INFO:teuthology.orchestra.run.smithi138.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:48:33.912 INFO:teuthology.orchestra.run.smithi138.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:48:33.912 INFO:teuthology.orchestra.run.smithi138.stdout:Access: 2023-09-23 22:48:10.742916342 +0000 2023-09-23T22:48:33.912 INFO:teuthology.orchestra.run.smithi138.stdout:Modify: 2023-09-23 22:40:11.342069661 +0000 2023-09-23T22:48:33.912 INFO:teuthology.orchestra.run.smithi138.stdout:Change: 2023-09-23 22:40:11.342069661 +0000 2023-09-23T22:48:33.912 INFO:teuthology.orchestra.run.smithi138.stdout: Birth: - 2023-09-23T22:48:33.912 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T22:48:33.979 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records in 2023-09-23T22:48:33.979 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records out 2023-09-23T22:48:33.979 INFO:teuthology.orchestra.run.smithi138.stderr:512 bytes copied, 0.00023724 s, 2.2 MB/s 2023-09-23T22:48:33.980 DEBUG:teuthology.orchestra.run.smithi138:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T22:48:34.038 DEBUG:teuthology.orchestra.run.smithi138:> stat /dev/vg_nvme/lv_2 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi138.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi138.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi138.stdout:Device: 6h/6d Inode: 386770 Links: 1 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi138.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi138.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi138.stdout:Access: 2023-09-23 22:48:10.742916342 +0000 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi138.stdout:Modify: 2023-09-23 22:40:11.846059464 +0000 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi138.stdout:Change: 2023-09-23 22:40:11.846059464 +0000 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi138.stdout: Birth: - 2023-09-23T22:48:34.096 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T22:48:34.159 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records in 2023-09-23T22:48:34.160 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records out 2023-09-23T22:48:34.160 INFO:teuthology.orchestra.run.smithi138.stderr:512 bytes copied, 0.000247677 s, 2.1 MB/s 2023-09-23T22:48:34.161 DEBUG:teuthology.orchestra.run.smithi138:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T22:48:34.218 DEBUG:teuthology.orchestra.run.smithi138:> stat /dev/vg_nvme/lv_3 2023-09-23T22:48:34.274 INFO:teuthology.orchestra.run.smithi138.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T22:48:34.274 INFO:teuthology.orchestra.run.smithi138.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:48:34.274 INFO:teuthology.orchestra.run.smithi138.stdout:Device: 6h/6d Inode: 381933 Links: 1 2023-09-23T22:48:34.275 INFO:teuthology.orchestra.run.smithi138.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:48:34.275 INFO:teuthology.orchestra.run.smithi138.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:48:34.275 INFO:teuthology.orchestra.run.smithi138.stdout:Access: 2023-09-23 22:48:10.742916342 +0000 2023-09-23T22:48:34.275 INFO:teuthology.orchestra.run.smithi138.stdout:Modify: 2023-09-23 22:40:12.351049246 +0000 2023-09-23T22:48:34.275 INFO:teuthology.orchestra.run.smithi138.stdout:Change: 2023-09-23 22:40:12.351049246 +0000 2023-09-23T22:48:34.275 INFO:teuthology.orchestra.run.smithi138.stdout: Birth: - 2023-09-23T22:48:34.275 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T22:48:34.338 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:34 smithi138 ceph-mon[104418]: Deploying daemon mgr.x on smithi138 2023-09-23T22:48:34.338 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:34 smithi138 ceph-mon[104418]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:34.341 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records in 2023-09-23T22:48:34.341 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records out 2023-09-23T22:48:34.342 INFO:teuthology.orchestra.run.smithi138.stderr:512 bytes copied, 0.000270625 s, 1.9 MB/s 2023-09-23T22:48:34.343 DEBUG:teuthology.orchestra.run.smithi138:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T22:48:34.400 DEBUG:teuthology.orchestra.run.smithi138:> stat /dev/vg_nvme/lv_4 2023-09-23T22:48:34.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:34 smithi110 ceph-mon[118821]: Deploying daemon mgr.x on smithi138 2023-09-23T22:48:34.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:34 smithi110 ceph-mon[118821]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:34.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:34 smithi110 ceph-mon[100243]: Deploying daemon mgr.x on smithi138 2023-09-23T22:48:34.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:34 smithi110 ceph-mon[100243]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:34.458 INFO:teuthology.orchestra.run.smithi138.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T22:48:34.458 INFO:teuthology.orchestra.run.smithi138.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:48:34.458 INFO:teuthology.orchestra.run.smithi138.stdout:Device: 6h/6d Inode: 385995 Links: 1 2023-09-23T22:48:34.458 INFO:teuthology.orchestra.run.smithi138.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:48:34.458 INFO:teuthology.orchestra.run.smithi138.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:48:34.458 INFO:teuthology.orchestra.run.smithi138.stdout:Access: 2023-09-23 22:48:34.393435015 +0000 2023-09-23T22:48:34.458 INFO:teuthology.orchestra.run.smithi138.stdout:Modify: 2023-09-23 22:40:12.840039353 +0000 2023-09-23T22:48:34.458 INFO:teuthology.orchestra.run.smithi138.stdout:Change: 2023-09-23 22:40:12.840039353 +0000 2023-09-23T22:48:34.458 INFO:teuthology.orchestra.run.smithi138.stdout: Birth: - 2023-09-23T22:48:34.459 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T22:48:34.572 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records in 2023-09-23T22:48:34.572 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records out 2023-09-23T22:48:34.572 INFO:teuthology.orchestra.run.smithi138.stderr:512 bytes copied, 0.000200044 s, 2.6 MB/s 2023-09-23T22:48:34.573 DEBUG:teuthology.orchestra.run.smithi138:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T22:48:34.592 INFO:tasks.cephadm:Deploying osd.0 on smithi110 with /dev/vg_nvme/lv_4... 2023-09-23T22:48:34.592 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T22:48:34.691 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:34 smithi138 systemd[1]: Starting Ceph mgr.x for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:48:34.930 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:35.259 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:35 smithi138 ceph-mgr[107617]: -- 172.21.15.138:0/3086539715 <== mon.1 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c338a809c0 con 0x55c338a58800 2023-09-23T22:48:35.259 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:35 smithi138 bash[107552]: 5ce725349f64fe7a5c9c896f585d50b761caeda9fd631a56bc5aa749f3936f12 2023-09-23T22:48:35.260 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:35 smithi138 systemd[1]: Started Ceph mgr.x for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:48:35.527 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:35 smithi138 conmon[107593]: 2023-09-23T22:48:35.258+0000 7f5ff77d11c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:48:35.527 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:35 smithi138 conmon[107593]: 2023-09-23T22:48:35.338+0000 7f5ff77d11c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:48:35.527 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:35 smithi138 conmon[107593]: 2023-09-23T22:48:35.430+0000 7f5ff77d11c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:48:35.527 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:35 smithi138 conmon[107593]: 2023-09-23T22:48:35.526+0000 7f5ff77d11c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:48:35.940 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:35 smithi138 conmon[107593]: 2023-09-23T22:48:35.730+0000 7f5ff77d11c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:48:36.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:36 smithi110 ceph-mon[100243]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:36.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:36 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:36.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:36 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:36.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:36 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:36.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:36 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:36.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:36 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:36.304 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:36 smithi110 ceph-mon[118821]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:36.304 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:36 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:36.304 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:36 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:36.304 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:36 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:36.304 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:36 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:36.304 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:36 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:36.406 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:36 smithi138 conmon[107593]: 2023-09-23T22:48:36.080+0000 7f5ff77d11c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:48:36.407 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:36 smithi138 conmon[107593]: 2023-09-23T22:48:36.171+0000 7f5ff77d11c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:48:36.407 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:36 smithi138 conmon[107593]: 2023-09-23T22:48:36.311+0000 7f5ff77d11c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:48:36.407 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:36 smithi138 ceph-mon[104418]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:36.407 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:36 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:36.407 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:36 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:36.407 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:36 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:36.407 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:36 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:36.407 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:36 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:36.796 INFO:teuthology.orchestra.run.smithi110.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T22:48:36.796 INFO:teuthology.orchestra.run.smithi110.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T22:48:36.796 INFO:teuthology.orchestra.run.smithi110.stderr: stderr: 10+0 records in 2023-09-23T22:48:36.796 INFO:teuthology.orchestra.run.smithi110.stderr:10+0 records out 2023-09-23T22:48:36.796 INFO:teuthology.orchestra.run.smithi110.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00843448 s, 1.2 GB/s 2023-09-23T22:48:36.796 INFO:teuthology.orchestra.run.smithi110.stderr:--> Zapping successful for: 2023-09-23T22:48:37.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:36 smithi138 conmon[107593]: 2023-09-23T22:48:36.405+0000 7f5ff77d11c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:48:37.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:36 smithi138 conmon[107593]: 2023-09-23T22:48:36.821+0000 7f5ff77d11c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:48:37.643 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:37.643 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:37.643 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:37.643 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:37.644 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:37.644 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[118821]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T22:48:37.644 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:48:37.644 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:48:37.644 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:37.644 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[118821]: Reconfiguring daemon mgr.y on smithi110 2023-09-23T22:48:37.644 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:37.644 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:37.644 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:37.644 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:37.645 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:37.645 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[100243]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T22:48:37.645 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:48:37.645 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:48:37.645 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:37.645 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:37 smithi110 ceph-mon[100243]: Reconfiguring daemon mgr.y on smithi110 2023-09-23T22:48:37.691 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:37 smithi138 conmon[107593]: 2023-09-23T22:48:37.383+0000 7f5ff77d11c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:48:37.691 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:37 smithi138 conmon[107593]: 2023-09-23T22:48:37.552+0000 7f5ff77d11c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:48:37.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:37 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:37.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:37 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:37.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:37 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:37.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:37 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:37.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:37 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:37.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:37 smithi138 ceph-mon[104418]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T22:48:37.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:37 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:48:37.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:37 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:48:37.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:37 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:37.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:37 smithi138 ceph-mon[104418]: Reconfiguring daemon mgr.y on smithi110 2023-09-23T22:48:38.091 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi110:vg_nvme/lv_4 2023-09-23T22:48:38.448 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:38.560 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:38 smithi138 conmon[107593]: 2023-09-23T22:48:38.304+0000 7f5ff77d11c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:48:38.560 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:38 smithi138 conmon[107593]: 2023-09-23T22:48:38.393+0000 7f5ff77d11c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:48:38.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:38 smithi138 ceph-mon[104418]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:38.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:38 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:38.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:38 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:38.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:38 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:38.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:38 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:38.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:38 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:38.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:38 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:38.684 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:38 smithi110 ceph-mon[118821]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:38.684 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:38 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:38.684 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:38 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:38.685 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:38 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:38.685 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:38 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:38.685 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:38 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:38.685 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:38 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:38.685 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:38 smithi110 ceph-mon[100243]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:38.685 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:38.685 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:38.685 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:38.685 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:38.685 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:38.685 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:38.828 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:38 smithi138 conmon[107593]: 2023-09-23T22:48:38.559+0000 7f5ff77d11c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:48:39.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:38 smithi138 conmon[107593]: 2023-09-23T22:48:38.827+0000 7f5ff77d11c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:48:39.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:38 smithi138 conmon[107593]: 2023-09-23T22:48:38.911+0000 7f5ff77d11c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:48:39.450 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:39 smithi138 conmon[107593]: 2023-09-23T22:48:39.198+0000 7f5ff77d11c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:48:39.941 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:39 smithi138 conmon[107593]: 2023-09-23T22:48:39.449+0000 7f5ff77d11c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:48:40.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:39 smithi110 ceph-mon[100243]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:40.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:39 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:40.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:39 smithi110 ceph-mon[118821]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:40.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:39 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:40.270 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:40 smithi138 conmon[107593]: 2023-09-23T22:48:40.141+0000 7f5ff77d11c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:48:40.271 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:39 smithi138 ceph-mon[104418]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:40.271 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:39 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:40.565 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:40 smithi138 conmon[107593]: 2023-09-23T22:48:40.269+0000 7f5ff77d11c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:48:40.566 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:40 smithi138 conmon[107593]: 2023-09-23T22:48:40.453+0000 7f5ff77d11c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:48:40.941 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:40 smithi138 conmon[107593]: 2023-09-23T22:48:40.564+0000 7f5ff77d11c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:48:40.941 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:40 smithi138 conmon[107593]: 2023-09-23T22:48:40.772+0000 7f5ff77d11c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:48:41.151 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:40 smithi110 ceph-mon[100243]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:48:41.151 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:40 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:48:41.152 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:40 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:48:41.152 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:40 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:41.152 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:40 smithi110 ceph-mon[118821]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:48:41.152 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:40 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:48:41.152 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:40 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:48:41.152 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:40 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:41.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:40 smithi138 ceph-mon[104418]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:48:41.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:40 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:48:41.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:40 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:48:41.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:40 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:41.941 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:48:41 smithi138 conmon[107593]: 2023-09-23T22:48:41.612+0000 7f5ff77d11c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:48:42.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:41 smithi110 ceph-mon[100243]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:42.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:41 smithi110 ceph-mon[100243]: Standby manager daemon x started 2023-09-23T22:48:42.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:41 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/1188686036' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:48:42.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:41 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/1188686036' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:48:42.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:41 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/1188686036' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:48:42.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:41 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/1188686036' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:48:42.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:41 smithi110 ceph-mon[118821]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:42.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:41 smithi110 ceph-mon[118821]: Standby manager daemon x started 2023-09-23T22:48:42.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:41 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/1188686036' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:48:42.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:41 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/1188686036' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:48:42.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:41 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/1188686036' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:48:42.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:41 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/1188686036' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:48:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:41 smithi138 ceph-mon[104418]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:41 smithi138 ceph-mon[104418]: Standby manager daemon x started 2023-09-23T22:48:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:41 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/1188686036' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:48:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:41 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/1188686036' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:48:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:41 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/1188686036' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:48:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:41 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/1188686036' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:48:43.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:43 smithi138 ceph-mon[104418]: mgrmap e14: y(active, since 63s), standbys: x 2023-09-23T22:48:43.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:43 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:48:43.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:43 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2358494072' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "78791acb-c93c-4e4f-bf04-ea3eee46408d"}]: dispatch 2023-09-23T22:48:43.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:43 smithi138 ceph-mon[104418]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "78791acb-c93c-4e4f-bf04-ea3eee46408d"}]: dispatch 2023-09-23T22:48:43.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:43 smithi138 ceph-mon[104418]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "78791acb-c93c-4e4f-bf04-ea3eee46408d"}]': finished 2023-09-23T22:48:43.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:43 smithi138 ceph-mon[104418]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T22:48:43.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:43 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:48:43.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:43 smithi110 ceph-mon[100243]: mgrmap e14: y(active, since 63s), standbys: x 2023-09-23T22:48:43.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:43 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:48:43.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:43 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2358494072' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "78791acb-c93c-4e4f-bf04-ea3eee46408d"}]: dispatch 2023-09-23T22:48:43.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:43 smithi110 ceph-mon[100243]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "78791acb-c93c-4e4f-bf04-ea3eee46408d"}]: dispatch 2023-09-23T22:48:43.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:43 smithi110 ceph-mon[100243]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "78791acb-c93c-4e4f-bf04-ea3eee46408d"}]': finished 2023-09-23T22:48:43.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:43 smithi110 ceph-mon[100243]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T22:48:43.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:43 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:48:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:43 smithi110 ceph-mon[118821]: mgrmap e14: y(active, since 63s), standbys: x 2023-09-23T22:48:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:43 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:48:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:43 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2358494072' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "78791acb-c93c-4e4f-bf04-ea3eee46408d"}]: dispatch 2023-09-23T22:48:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:43 smithi110 ceph-mon[118821]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "78791acb-c93c-4e4f-bf04-ea3eee46408d"}]: dispatch 2023-09-23T22:48:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:43 smithi110 ceph-mon[118821]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "78791acb-c93c-4e4f-bf04-ea3eee46408d"}]': finished 2023-09-23T22:48:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:43 smithi110 ceph-mon[118821]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T22:48:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:43 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:48:44.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:44 smithi138 ceph-mon[104418]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:44.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:44 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1731625171' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:48:44.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:44 smithi110 ceph-mon[100243]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:44.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:44 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1731625171' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:48:44.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:44 smithi110 ceph-mon[118821]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:44.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:44 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1731625171' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:48:46.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:46 smithi138 ceph-mon[104418]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:46.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:46 smithi110 ceph-mon[100243]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:46.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:46 smithi110 ceph-mon[118821]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:48.429 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:48 smithi110 ceph-mon[100243]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:48.430 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:48 smithi110 ceph-mon[118821]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:48.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:48 smithi138 ceph-mon[104418]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:49.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:49 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:48:49.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:49 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:49.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:49 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:48:49.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:49 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:49.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:49 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:48:49.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:49 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:50.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:50 smithi138 ceph-mon[104418]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:50.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:50 smithi138 ceph-mon[104418]: Deploying daemon osd.0 on smithi110 2023-09-23T22:48:50.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:50 smithi110 ceph-mon[100243]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:50.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:50 smithi110 ceph-mon[100243]: Deploying daemon osd.0 on smithi110 2023-09-23T22:48:50.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:50 smithi110 ceph-mon[118821]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:50.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:50 smithi110 ceph-mon[118821]: Deploying daemon osd.0 on smithi110 2023-09-23T22:48:52.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:52 smithi138 ceph-mon[104418]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:52.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:52 smithi110 ceph-mon[100243]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:52.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:52 smithi110 ceph-mon[118821]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:54.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:54 smithi138 ceph-mon[104418]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:54.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:54 smithi110 ceph-mon[100243]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:54.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:54 smithi110 ceph-mon[118821]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:55.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:55 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:55.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:55 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:55.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:55 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:55.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:55 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:55.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:55 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:55.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:55 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:55.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:55 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:55.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:55 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:55.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:55 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:56.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:56 smithi138 ceph-mon[104418]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:56.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:56 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:56.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:56 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:56.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:56 smithi110 ceph-mon[100243]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:56.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:56 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:56.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:56 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:56.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:56 smithi110 ceph-mon[118821]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:56.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:56 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:56.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:56 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:57.470 INFO:teuthology.orchestra.run.smithi110.stdout:Created osd(s) 0 on host 'smithi110' 2023-09-23T22:48:57.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:57 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:57.675 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:57 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:57.675 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:57 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:57.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:57 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:57.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:57 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:57.676 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:57 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:57.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:57 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:57.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:57 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:57.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:57 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:58.137 DEBUG:teuthology.orchestra.run.smithi110:osd.0> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.0.service 2023-09-23T22:48:58.139 INFO:tasks.cephadm:Deploying osd.1 on smithi110 with /dev/vg_nvme/lv_3... 2023-09-23T22:48:58.140 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T22:48:58.168 INFO:journalctl@ceph.osd.0.smithi110.stdout:-- Logs begin at Sat 2023-09-23 22:30:57 UTC. -- 2023-09-23T22:48:58.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:58 smithi110 ceph-mon[100243]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:58.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:58 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:58.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:58 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:58.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:58 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:58.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:58 smithi110 ceph-mon[100243]: from='osd.0 [v2:172.21.15.110:6802/2453789689,v1:172.21.15.110:6803/2453789689]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:48:58.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:58 smithi110 ceph-mon[100243]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:48:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:58 smithi110 ceph-mon[118821]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:58 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:58 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:58 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:58 smithi110 ceph-mon[118821]: from='osd.0 [v2:172.21.15.110:6802/2453789689,v1:172.21.15.110:6803/2453789689]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:48:58.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:58 smithi110 ceph-mon[118821]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:48:58.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:58 smithi138 ceph-mon[104418]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:48:58.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:58 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:58.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:58 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:58.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:58 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:48:58.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:58 smithi138 ceph-mon[104418]: from='osd.0 [v2:172.21.15.110:6802/2453789689,v1:172.21.15.110:6803/2453789689]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:48:58.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:58 smithi138 ceph-mon[104418]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:48:59.373 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:59.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:59 smithi110 ceph-mon[100243]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:48:59.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:59 smithi110 ceph-mon[100243]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T22:48:59.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:59 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:48:59.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:59 smithi110 ceph-mon[100243]: from='osd.0 [v2:172.21.15.110:6802/2453789689,v1:172.21.15.110:6803/2453789689]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T22:48:59.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:48:59 smithi110 ceph-mon[100243]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T22:48:59.617 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:59 smithi110 ceph-mon[118821]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:48:59.617 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:59 smithi110 ceph-mon[118821]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T22:48:59.617 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:59 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:48:59.617 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:59 smithi110 ceph-mon[118821]: from='osd.0 [v2:172.21.15.110:6802/2453789689,v1:172.21.15.110:6803/2453789689]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T22:48:59.617 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:48:59 smithi110 ceph-mon[118821]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T22:48:59.618 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 22:48:59 smithi110 conmon[125066]: 2023-09-23T22:48:59.459+0000 7f587d93f700 -1 osd.0 0 waiting for initial osdmap 2023-09-23T22:48:59.618 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 22:48:59 smithi110 conmon[125066]: 2023-09-23T22:48:59.461+0000 7f587973c700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:48:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:59 smithi138 ceph-mon[104418]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:48:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:59 smithi138 ceph-mon[104418]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T22:48:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:59 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:48:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:59 smithi138 ceph-mon[104418]: from='osd.0 [v2:172.21.15.110:6802/2453789689,v1:172.21.15.110:6803/2453789689]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T22:48:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:48:59 smithi138 ceph-mon[104418]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T22:49:00.463 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:00 smithi110 ceph-mon[118821]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:49:00.463 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:00 smithi110 ceph-mon[118821]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-09-23T22:49:00.463 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:00 smithi110 ceph-mon[118821]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T22:49:00.464 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:00 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:49:00.464 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:00 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:49:00.464 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:00 smithi110 ceph-mon[100243]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:49:00.464 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:00 smithi110 ceph-mon[100243]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-09-23T22:49:00.464 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:00 smithi110 ceph-mon[100243]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T22:49:00.464 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:00 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:49:00.464 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:00 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:49:00.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:00 smithi138 ceph-mon[104418]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:49:00.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:00 smithi138 ceph-mon[104418]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-09-23T22:49:00.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:00 smithi138 ceph-mon[104418]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T22:49:00.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:00 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:49:00.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:00 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:49:01.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[100243]: purged_snaps scrub starts 2023-09-23T22:49:01.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[100243]: purged_snaps scrub ok 2023-09-23T22:49:01.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[100243]: osd.0 [v2:172.21.15.110:6802/2453789689,v1:172.21.15.110:6803/2453789689] boot 2023-09-23T22:49:01.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[100243]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T22:49:01.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:49:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[118821]: purged_snaps scrub starts 2023-09-23T22:49:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[118821]: purged_snaps scrub ok 2023-09-23T22:49:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[118821]: osd.0 [v2:172.21.15.110:6802/2453789689,v1:172.21.15.110:6803/2453789689] boot 2023-09-23T22:49:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[118821]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T22:49:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:49:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:01 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:01.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:01 smithi138 ceph-mon[104418]: purged_snaps scrub starts 2023-09-23T22:49:01.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:01 smithi138 ceph-mon[104418]: purged_snaps scrub ok 2023-09-23T22:49:01.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:01 smithi138 ceph-mon[104418]: osd.0 [v2:172.21.15.110:6802/2453789689,v1:172.21.15.110:6803/2453789689] boot 2023-09-23T22:49:01.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:01 smithi138 ceph-mon[104418]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T22:49:01.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:01 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:49:01.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:01 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:01.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:01 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:01.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:01 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:01.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:01 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:01.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:01 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:01.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:01 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:01.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:01 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:01.950 INFO:teuthology.orchestra.run.smithi110.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T22:49:01.950 INFO:teuthology.orchestra.run.smithi110.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T22:49:01.950 INFO:teuthology.orchestra.run.smithi110.stderr: stderr: 10+0 records in 2023-09-23T22:49:01.950 INFO:teuthology.orchestra.run.smithi110.stderr:10+0 records out 2023-09-23T22:49:01.950 INFO:teuthology.orchestra.run.smithi110.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00794276 s, 1.3 GB/s 2023-09-23T22:49:01.950 INFO:teuthology.orchestra.run.smithi110.stderr:--> Zapping successful for: 2023-09-23T22:49:02.721 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:02 smithi110 ceph-mon[118821]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:49:02.722 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:02 smithi110 ceph-mon[118821]: Detected new or changed devices on smithi110 2023-09-23T22:49:02.722 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:02 smithi110 ceph-mon[118821]: Adjusting osd_memory_target on smithi110 to 16144M 2023-09-23T22:49:02.722 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:02 smithi110 ceph-mon[118821]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T22:49:02.722 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:02 smithi110 ceph-mon[100243]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:49:02.723 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:02 smithi110 ceph-mon[100243]: Detected new or changed devices on smithi110 2023-09-23T22:49:02.723 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:02 smithi110 ceph-mon[100243]: Adjusting osd_memory_target on smithi110 to 16144M 2023-09-23T22:49:02.723 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:02 smithi110 ceph-mon[100243]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T22:49:02.724 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi110:vg_nvme/lv_3 2023-09-23T22:49:02.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:02 smithi138 ceph-mon[104418]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:49:02.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:02 smithi138 ceph-mon[104418]: Detected new or changed devices on smithi110 2023-09-23T22:49:02.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:02 smithi138 ceph-mon[104418]: Adjusting osd_memory_target on smithi110 to 16144M 2023-09-23T22:49:02.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:02 smithi138 ceph-mon[104418]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T22:49:03.056 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:04.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:04 smithi110 ceph-mon[100243]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:04.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:04 smithi110 ceph-mon[118821]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:04.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:04 smithi138 ceph-mon[104418]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:05.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:05 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:49:05.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:05 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:49:05.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:05 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:05.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:05 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:49:05.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:05 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:49:05.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:05 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:05.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:05 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:49:05.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:05 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:49:05.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:05 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:06.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:06 smithi138 ceph-mon[104418]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:49:06.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:06 smithi138 ceph-mon[104418]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:06.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:06 smithi110 ceph-mon[100243]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:49:06.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:06 smithi110 ceph-mon[100243]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:06.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:06 smithi110 ceph-mon[118821]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:49:06.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:06 smithi110 ceph-mon[118821]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:07.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:07 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3221290687' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6793178f-2ef1-4c48-927c-831a083a3356"}]: dispatch 2023-09-23T22:49:07.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:07 smithi138 ceph-mon[104418]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6793178f-2ef1-4c48-927c-831a083a3356"}]: dispatch 2023-09-23T22:49:07.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:07 smithi138 ceph-mon[104418]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6793178f-2ef1-4c48-927c-831a083a3356"}]': finished 2023-09-23T22:49:07.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:07 smithi138 ceph-mon[104418]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T22:49:07.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:07 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:07.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:07 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3221290687' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6793178f-2ef1-4c48-927c-831a083a3356"}]: dispatch 2023-09-23T22:49:07.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:07 smithi110 ceph-mon[100243]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6793178f-2ef1-4c48-927c-831a083a3356"}]: dispatch 2023-09-23T22:49:07.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:07 smithi110 ceph-mon[100243]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6793178f-2ef1-4c48-927c-831a083a3356"}]': finished 2023-09-23T22:49:07.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:07 smithi110 ceph-mon[100243]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T22:49:07.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:07 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:07.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:07 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3221290687' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6793178f-2ef1-4c48-927c-831a083a3356"}]: dispatch 2023-09-23T22:49:07.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:07 smithi110 ceph-mon[118821]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6793178f-2ef1-4c48-927c-831a083a3356"}]: dispatch 2023-09-23T22:49:07.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:07 smithi110 ceph-mon[118821]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6793178f-2ef1-4c48-927c-831a083a3356"}]': finished 2023-09-23T22:49:07.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:07 smithi110 ceph-mon[118821]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T22:49:07.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:07 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:08 smithi138 ceph-mon[104418]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:08 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/546066744' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:49:08.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:08 smithi110 ceph-mon[100243]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:08.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:08 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/546066744' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:49:08.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:08 smithi110 ceph-mon[118821]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:08.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:08 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/546066744' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:49:10.803 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:10 smithi110 ceph-mon[100243]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:10.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:10 smithi110 ceph-mon[118821]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:10.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:10 smithi138 ceph-mon[104418]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:12.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:12 smithi138 ceph-mon[104418]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:12.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:12 smithi110 ceph-mon[100243]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:12.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:12 smithi110 ceph-mon[118821]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:14.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:14 smithi138 ceph-mon[104418]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:14.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:14 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:49:14.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:14 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:14.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:14 smithi110 ceph-mon[118821]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:14.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:14 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:49:14.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:14 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:14.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:14 smithi110 ceph-mon[100243]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:14.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:14 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:49:14.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:14 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:15.653 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:15 smithi110 ceph-mon[100243]: Deploying daemon osd.1 on smithi110 2023-09-23T22:49:15.653 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:15 smithi110 ceph-mon[118821]: Deploying daemon osd.1 on smithi110 2023-09-23T22:49:15.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:15 smithi138 ceph-mon[104418]: Deploying daemon osd.1 on smithi110 2023-09-23T22:49:16.612 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:16 smithi110 ceph-mon[100243]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:16.613 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:16 smithi110 ceph-mon[118821]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:16.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:16 smithi138 ceph-mon[104418]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:17.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:17 smithi110 ceph-mon[100243]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:17.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:17 smithi110 ceph-mon[118821]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:17.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:17 smithi138 ceph-mon[104418]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:19.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:19.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:49:19.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:19.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:19 smithi110 ceph-mon[118821]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:19.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:19 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:19.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:19 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:49:19.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:19 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:19.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:19 smithi110 ceph-mon[100243]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:20.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:19 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:20.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:19 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:49:20.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:19 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:20.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:19 smithi138 ceph-mon[104418]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:22.008 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:21 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:22.008 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:21 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:22.008 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:21 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:22.009 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:21 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:22.009 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:21 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:22.009 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:21 smithi110 ceph-mon[100243]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:22.009 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:21 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:22.009 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:21 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:22.009 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:21 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:22.009 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:21 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:22.009 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:21 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:22.010 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:21 smithi110 ceph-mon[118821]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:22.017 INFO:teuthology.orchestra.run.smithi110.stdout:Created osd(s) 1 on host 'smithi110' 2023-09-23T22:49:22.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:21 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:22.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:21 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:22.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:21 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:22.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:21 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:22.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:21 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:22.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:21 smithi138 ceph-mon[104418]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:22.701 DEBUG:teuthology.orchestra.run.smithi110:osd.1> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T22:49:22.703 INFO:tasks.cephadm:Deploying osd.2 on smithi110 with /dev/vg_nvme/lv_2... 2023-09-23T22:49:22.704 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T22:49:22.734 INFO:journalctl@ceph.osd.1.smithi110.stdout:-- Logs begin at Sat 2023-09-23 22:30:57 UTC. -- 2023-09-23T22:49:23.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:23 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:23.346 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:23 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:23.346 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:23 smithi110 ceph-mon[100243]: from='osd.1 [v2:172.21.15.110:6810/3697882113,v1:172.21.15.110:6811/3697882113]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:49:23.346 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:23 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:49:23.346 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:23 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:23.346 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:23 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:23.346 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:23 smithi110 ceph-mon[118821]: from='osd.1 [v2:172.21.15.110:6810/3697882113,v1:172.21.15.110:6811/3697882113]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:49:23.346 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:23 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:49:23.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:23 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:23.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:23 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:23.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:23 smithi138 ceph-mon[104418]: from='osd.1 [v2:172.21.15.110:6810/3697882113,v1:172.21.15.110:6811/3697882113]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:49:23.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:23 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:49:23.774 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:24.091 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:24 smithi110 ceph-mon[100243]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:24.091 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:24 smithi110 ceph-mon[100243]: from='osd.1 [v2:172.21.15.110:6810/3697882113,v1:172.21.15.110:6811/3697882113]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:49:24.091 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:24 smithi110 ceph-mon[100243]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T22:49:24.091 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:24.091 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:24 smithi110 ceph-mon[100243]: from='osd.1 [v2:172.21.15.110:6810/3697882113,v1:172.21.15.110:6811/3697882113]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T22:49:24.091 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:24 smithi110 ceph-mon[118821]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:24.091 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:24 smithi110 ceph-mon[118821]: from='osd.1 [v2:172.21.15.110:6810/3697882113,v1:172.21.15.110:6811/3697882113]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:49:24.092 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:24 smithi110 ceph-mon[118821]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T22:49:24.092 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:24.092 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:24 smithi110 ceph-mon[118821]: from='osd.1 [v2:172.21.15.110:6810/3697882113,v1:172.21.15.110:6811/3697882113]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T22:49:24.092 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 22:49:24 smithi110 conmon[134020]: 2023-09-23T22:49:24.018+0000 7f941d4a6700 -1 osd.1 0 waiting for initial osdmap 2023-09-23T22:49:24.092 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 22:49:24 smithi110 conmon[134020]: 2023-09-23T22:49:24.024+0000 7f94182a1700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:49:24.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:24 smithi138 ceph-mon[104418]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:24.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:24 smithi138 ceph-mon[104418]: from='osd.1 [v2:172.21.15.110:6810/3697882113,v1:172.21.15.110:6811/3697882113]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:49:24.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:24 smithi138 ceph-mon[104418]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T22:49:24.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:24.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:24 smithi138 ceph-mon[104418]: from='osd.1 [v2:172.21.15.110:6810/3697882113,v1:172.21.15.110:6811/3697882113]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T22:49:25.264 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:25 smithi110 ceph-mon[100243]: purged_snaps scrub starts 2023-09-23T22:49:25.264 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:25 smithi110 ceph-mon[100243]: purged_snaps scrub ok 2023-09-23T22:49:25.264 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:25 smithi110 ceph-mon[100243]: from='osd.1 [v2:172.21.15.110:6810/3697882113,v1:172.21.15.110:6811/3697882113]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-09-23T22:49:25.265 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:25 smithi110 ceph-mon[100243]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T22:49:25.265 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:25 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:25.265 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:25 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:25.265 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:25 smithi110 ceph-mon[118821]: purged_snaps scrub starts 2023-09-23T22:49:25.265 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:25 smithi110 ceph-mon[118821]: purged_snaps scrub ok 2023-09-23T22:49:25.265 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:25 smithi110 ceph-mon[118821]: from='osd.1 [v2:172.21.15.110:6810/3697882113,v1:172.21.15.110:6811/3697882113]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-09-23T22:49:25.266 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:25 smithi110 ceph-mon[118821]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T22:49:25.266 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:25 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:25.266 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:25 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:25.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:25 smithi138 ceph-mon[104418]: purged_snaps scrub starts 2023-09-23T22:49:25.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:25 smithi138 ceph-mon[104418]: purged_snaps scrub ok 2023-09-23T22:49:25.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:25 smithi138 ceph-mon[104418]: from='osd.1 [v2:172.21.15.110:6810/3697882113,v1:172.21.15.110:6811/3697882113]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-09-23T22:49:25.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:25 smithi138 ceph-mon[104418]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T22:49:25.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:25 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:25.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:25 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:25.848 INFO:teuthology.orchestra.run.smithi110.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T22:49:25.849 INFO:teuthology.orchestra.run.smithi110.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T22:49:25.849 INFO:teuthology.orchestra.run.smithi110.stderr: stderr: 10+0 records in 2023-09-23T22:49:25.849 INFO:teuthology.orchestra.run.smithi110.stderr:10+0 records out 2023-09-23T22:49:25.849 INFO:teuthology.orchestra.run.smithi110.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00826784 s, 1.3 GB/s 2023-09-23T22:49:25.849 INFO:teuthology.orchestra.run.smithi110.stderr:--> Zapping successful for: 2023-09-23T22:49:26.179 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:26 smithi110 ceph-mon[100243]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:26.179 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:26 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:26.179 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:26 smithi110 ceph-mon[100243]: osd.1 [v2:172.21.15.110:6810/3697882113,v1:172.21.15.110:6811/3697882113] boot 2023-09-23T22:49:26.180 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:26 smithi110 ceph-mon[100243]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T22:49:26.180 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:26 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:26.180 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:26 smithi110 ceph-mon[118821]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:26.180 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:26 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:26.181 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:26 smithi110 ceph-mon[118821]: osd.1 [v2:172.21.15.110:6810/3697882113,v1:172.21.15.110:6811/3697882113] boot 2023-09-23T22:49:26.181 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:26 smithi110 ceph-mon[118821]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T22:49:26.181 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:26 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:26.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:26 smithi138 ceph-mon[104418]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:49:26.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:26 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:26.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:26 smithi138 ceph-mon[104418]: osd.1 [v2:172.21.15.110:6810/3697882113,v1:172.21.15.110:6811/3697882113] boot 2023-09-23T22:49:26.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:26 smithi138 ceph-mon[104418]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T22:49:26.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:26 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:49:26.593 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi110:vg_nvme/lv_2 2023-09-23T22:49:26.928 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:27.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[100243]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T22:49:27.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[100243]: Detected new or changed devices on smithi110 2023-09-23T22:49:27.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:27.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:27.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:27.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:27.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[100243]: Adjusting osd_memory_target on smithi110 to 8072M 2023-09-23T22:49:27.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:27.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:27.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:27.358 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:27.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[118821]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T22:49:27.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[118821]: Detected new or changed devices on smithi110 2023-09-23T22:49:27.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:27.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:27.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:27.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:27.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[118821]: Adjusting osd_memory_target on smithi110 to 8072M 2023-09-23T22:49:27.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:27.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:27.359 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:27.359 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:27 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:27.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:27 smithi138 ceph-mon[104418]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T22:49:27.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:27 smithi138 ceph-mon[104418]: Detected new or changed devices on smithi110 2023-09-23T22:49:27.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:27.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:27.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:27.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:27.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:27 smithi138 ceph-mon[104418]: Adjusting osd_memory_target on smithi110 to 8072M 2023-09-23T22:49:27.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:27 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:28.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:28 smithi110 ceph-mon[100243]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:28.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:28 smithi110 ceph-mon[118821]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:28.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:28 smithi138 ceph-mon[104418]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:29.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:29 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:49:29.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:29 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:49:29.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:29 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:29.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:29 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:49:29.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:29 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:49:29.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:29 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:29.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:29 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:49:29.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:29 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:49:29.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:29 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:30.279 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:30 smithi110 ceph-mon[100243]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:49:30.279 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:30 smithi110 ceph-mon[100243]: pgmap v59: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:30.280 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:30 smithi110 ceph-mon[118821]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:49:30.280 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:30 smithi110 ceph-mon[118821]: pgmap v59: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:30 smithi138 ceph-mon[104418]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:49:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:30 smithi138 ceph-mon[104418]: pgmap v59: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:31.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:31 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1812749042' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "146fbd28-a365-4545-8e97-d629c6ddba21"}]: dispatch 2023-09-23T22:49:31.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:31 smithi138 ceph-mon[104418]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "146fbd28-a365-4545-8e97-d629c6ddba21"}]: dispatch 2023-09-23T22:49:31.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:31 smithi138 ceph-mon[104418]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "146fbd28-a365-4545-8e97-d629c6ddba21"}]': finished 2023-09-23T22:49:31.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:31 smithi138 ceph-mon[104418]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T22:49:31.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:31 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:49:31.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:31 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1812749042' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "146fbd28-a365-4545-8e97-d629c6ddba21"}]: dispatch 2023-09-23T22:49:31.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:31 smithi110 ceph-mon[100243]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "146fbd28-a365-4545-8e97-d629c6ddba21"}]: dispatch 2023-09-23T22:49:31.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:31 smithi110 ceph-mon[100243]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "146fbd28-a365-4545-8e97-d629c6ddba21"}]': finished 2023-09-23T22:49:31.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:31 smithi110 ceph-mon[100243]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T22:49:31.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:31 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:49:31.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:31 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1812749042' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "146fbd28-a365-4545-8e97-d629c6ddba21"}]: dispatch 2023-09-23T22:49:31.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:31 smithi110 ceph-mon[118821]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "146fbd28-a365-4545-8e97-d629c6ddba21"}]: dispatch 2023-09-23T22:49:31.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:31 smithi110 ceph-mon[118821]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "146fbd28-a365-4545-8e97-d629c6ddba21"}]': finished 2023-09-23T22:49:31.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:31 smithi110 ceph-mon[118821]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T22:49:31.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:31 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:49:32.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:32 smithi138 ceph-mon[104418]: pgmap v60: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:32.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:32 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1457758216' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:49:32.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:32 smithi110 ceph-mon[100243]: pgmap v60: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:32.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:32 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1457758216' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:49:32.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:32 smithi110 ceph-mon[118821]: pgmap v60: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:32.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:32 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1457758216' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:49:34.641 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:34 smithi110 ceph-mon[100243]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:34.642 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:34 smithi110 ceph-mon[118821]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:34.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:34 smithi138 ceph-mon[104418]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:36.307 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:36 smithi110 ceph-mon[100243]: pgmap v63: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:36.307 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:36 smithi110 ceph-mon[118821]: pgmap v63: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:36.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:36 smithi138 ceph-mon[104418]: pgmap v63: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:38.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:38 smithi138 ceph-mon[104418]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:38.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:38 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:49:38.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:38 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:38.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:38 smithi138 ceph-mon[104418]: Deploying daemon osd.2 on smithi110 2023-09-23T22:49:38.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:38 smithi110 ceph-mon[100243]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:38.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:49:38.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:38.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:38 smithi110 ceph-mon[100243]: Deploying daemon osd.2 on smithi110 2023-09-23T22:49:38.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:38 smithi110 ceph-mon[118821]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:38.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:38 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:49:38.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:38 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:38.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:38 smithi110 ceph-mon[118821]: Deploying daemon osd.2 on smithi110 2023-09-23T22:49:40.296 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:40 smithi110 ceph-mon[100243]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:40.296 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:40 smithi110 ceph-mon[118821]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:40.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:40 smithi138 ceph-mon[104418]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:42.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:42 smithi138 ceph-mon[104418]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:42.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:42 smithi110 ceph-mon[100243]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:42.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:42 smithi110 ceph-mon[118821]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:43.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:43 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:43.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:43 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:43.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:43 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:49:43.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:43 smithi110 ceph-mon[100243]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:43.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:43 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:43.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:43 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:43.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:43 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:49:43.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:43 smithi110 ceph-mon[118821]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:43.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:43 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:43.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:43 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:43.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:43 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:49:43.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:43 smithi138 ceph-mon[104418]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:45.533 INFO:teuthology.orchestra.run.smithi110.stdout:Created osd(s) 2 on host 'smithi110' 2023-09-23T22:49:45.751 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:45 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:45.752 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:45 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:45.752 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:45 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:45.752 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:45 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:45.752 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:45 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:45.752 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:45 smithi110 ceph-mon[118821]: pgmap v68: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:45.752 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:45 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:45.752 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:45 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:45.752 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:45 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:45.753 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:45 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:45.753 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:45 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:45.753 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:45 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:45.753 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:45 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:45.753 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:45 smithi110 ceph-mon[100243]: pgmap v68: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:45.753 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:45 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:45.753 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:45 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:45.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:45 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:45.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:45 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:45.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:45 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:45.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:45 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:45.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:45 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:45.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:45 smithi138 ceph-mon[104418]: pgmap v68: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:45.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:45 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:45.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:45 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:46.172 DEBUG:teuthology.orchestra.run.smithi110:osd.2> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T22:49:46.174 INFO:tasks.cephadm:Deploying osd.3 on smithi110 with /dev/vg_nvme/lv_1... 2023-09-23T22:49:46.175 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T22:49:46.205 INFO:journalctl@ceph.osd.2.smithi110.stdout:-- Logs begin at Sat 2023-09-23 22:30:57 UTC. -- 2023-09-23T22:49:46.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:46 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:49:46.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:46 smithi110 ceph-mon[118821]: from='osd.2 [v2:172.21.15.110:6818/2231416166,v1:172.21.15.110:6819/2231416166]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:49:46.884 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:46 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:49:46.884 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:46 smithi110 ceph-mon[100243]: from='osd.2 [v2:172.21.15.110:6818/2231416166,v1:172.21.15.110:6819/2231416166]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:49:46.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:46 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:49:46.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:46 smithi138 ceph-mon[104418]: from='osd.2 [v2:172.21.15.110:6818/2231416166,v1:172.21.15.110:6819/2231416166]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:49:47.350 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:47.547 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:47 smithi110 ceph-mon[100243]: from='osd.2 [v2:172.21.15.110:6818/2231416166,v1:172.21.15.110:6819/2231416166]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:49:47.548 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:47 smithi110 ceph-mon[100243]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T22:49:47.548 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:47 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:49:47.548 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:47 smithi110 ceph-mon[100243]: from='osd.2 [v2:172.21.15.110:6818/2231416166,v1:172.21.15.110:6819/2231416166]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T22:49:47.548 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:47 smithi110 ceph-mon[100243]: pgmap v70: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:47.548 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 22:49:47 smithi110 conmon[142903]: 2023-09-23T22:49:47.545+0000 7f9aea1ca700 -1 osd.2 0 waiting for initial osdmap 2023-09-23T22:49:47.548 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:47 smithi110 ceph-mon[118821]: from='osd.2 [v2:172.21.15.110:6818/2231416166,v1:172.21.15.110:6819/2231416166]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:49:47.549 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:47 smithi110 ceph-mon[118821]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T22:49:47.549 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:47 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:49:47.549 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:47 smithi110 ceph-mon[118821]: from='osd.2 [v2:172.21.15.110:6818/2231416166,v1:172.21.15.110:6819/2231416166]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T22:49:47.549 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:47 smithi110 ceph-mon[118821]: pgmap v70: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:47.818 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 22:49:47 smithi110 conmon[142903]: 2023-09-23T22:49:47.549+0000 7f9ae5fc7700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:49:47.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:47 smithi138 ceph-mon[104418]: from='osd.2 [v2:172.21.15.110:6818/2231416166,v1:172.21.15.110:6819/2231416166]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:49:47.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:47 smithi138 ceph-mon[104418]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T22:49:47.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:47 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:49:47.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:47 smithi138 ceph-mon[104418]: from='osd.2 [v2:172.21.15.110:6818/2231416166,v1:172.21.15.110:6819/2231416166]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T22:49:47.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:47 smithi138 ceph-mon[104418]: pgmap v70: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:49:48.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:48 smithi110 ceph-mon[100243]: from='osd.2 [v2:172.21.15.110:6818/2231416166,v1:172.21.15.110:6819/2231416166]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-09-23T22:49:48.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:48 smithi110 ceph-mon[100243]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T22:49:48.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:48 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:49:48.769 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:48 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:49:48.769 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:48 smithi110 ceph-mon[118821]: from='osd.2 [v2:172.21.15.110:6818/2231416166,v1:172.21.15.110:6819/2231416166]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-09-23T22:49:48.769 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:48 smithi110 ceph-mon[118821]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T22:49:48.769 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:48 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:49:48.769 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:48 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:49:48.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:48 smithi138 ceph-mon[104418]: from='osd.2 [v2:172.21.15.110:6818/2231416166,v1:172.21.15.110:6819/2231416166]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-09-23T22:49:48.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:48 smithi138 ceph-mon[104418]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T22:49:48.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:48 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:49:48.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:48 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:49:49.554 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: purged_snaps scrub starts 2023-09-23T22:49:49.555 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: purged_snaps scrub ok 2023-09-23T22:49:49.555 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: osd.2 [v2:172.21.15.110:6818/2231416166,v1:172.21.15.110:6819/2231416166] boot 2023-09-23T22:49:49.555 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T22:49:49.555 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:49:49.555 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:49:49.555 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T22:49:49.555 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: Detected new or changed devices on smithi110 2023-09-23T22:49:49.555 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:49.555 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: purged_snaps scrub starts 2023-09-23T22:49:49.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: purged_snaps scrub ok 2023-09-23T22:49:49.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: osd.2 [v2:172.21.15.110:6818/2231416166,v1:172.21.15.110:6819/2231416166] boot 2023-09-23T22:49:49.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T22:49:49.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:49:49.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:49:49.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T22:49:49.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: Detected new or changed devices on smithi110 2023-09-23T22:49:49.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:49.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:49.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:49.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:49.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:49.885 INFO:teuthology.orchestra.run.smithi110.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T22:49:49.885 INFO:teuthology.orchestra.run.smithi110.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T22:49:49.885 INFO:teuthology.orchestra.run.smithi110.stderr: stderr: 10+0 records in 2023-09-23T22:49:49.885 INFO:teuthology.orchestra.run.smithi110.stderr:10+0 records out 2023-09-23T22:49:49.885 INFO:teuthology.orchestra.run.smithi110.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00865671 s, 1.2 GB/s 2023-09-23T22:49:49.885 INFO:teuthology.orchestra.run.smithi110.stderr:--> Zapping successful for: 2023-09-23T22:49:49.923 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: Adjusting osd_memory_target on smithi110 to 5381M 2023-09-23T22:49:49.923 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:49.923 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:49.924 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:49.924 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:49.924 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:49.924 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:49.924 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:49.924 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:49.924 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: Adjusting osd_memory_target on smithi110 to 5381M 2023-09-23T22:49:49.924 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:49.924 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:49.925 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:49.925 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:49 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:49.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: purged_snaps scrub starts 2023-09-23T22:49:49.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: purged_snaps scrub ok 2023-09-23T22:49:49.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: osd.2 [v2:172.21.15.110:6818/2231416166,v1:172.21.15.110:6819/2231416166] boot 2023-09-23T22:49:49.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T22:49:49.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:49:49.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:49:49.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T22:49:49.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: Detected new or changed devices on smithi110 2023-09-23T22:49:49.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:49.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:49.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:49.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:49.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:49:49.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: Adjusting osd_memory_target on smithi110 to 5381M 2023-09-23T22:49:49.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:49.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:49.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:49.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:49 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:50.666 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi110:vg_nvme/lv_1 2023-09-23T22:49:50.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:50 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T22:49:50.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:50 smithi138 ceph-mon[104418]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T22:49:50.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:50 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:49:50.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:50 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T22:49:50.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:50 smithi110 ceph-mon[100243]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T22:49:50.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:50 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:49:50.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:50 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T22:49:50.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:50 smithi110 ceph-mon[118821]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T22:49:50.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:50 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:49:51.028 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:51.305 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 22:49:51 smithi110 sudo[148670]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T22:49:51.305 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 22:49:51 smithi110 sudo[148670]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:49:51.306 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 22:49:51 smithi110 sudo[148670]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:49:51.306 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 22:49:51 smithi110 sudo[148670]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:49:51.567 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 22:49:51 smithi110 sudo[148727]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T22:49:51.567 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 22:49:51 smithi110 sudo[148727]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:49:51.567 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 22:49:51 smithi110 sudo[148727]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:49:51.567 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 22:49:51 smithi110 sudo[148727]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:49:51.568 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:51 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:49:51.568 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:51 smithi110 ceph-mon[100243]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T22:49:51.568 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:51 smithi110 ceph-mon[100243]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:49:51.568 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:51 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:51.568 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:51 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:49:51.568 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:51 smithi110 ceph-mon[118821]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T22:49:51.568 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:51 smithi110 ceph-mon[118821]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:49:51.569 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:51 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:51.851 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 22:49:51 smithi110 sudo[148756]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T22:49:51.852 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 22:49:51 smithi110 sudo[148756]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:49:51.852 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 22:49:51 smithi110 sudo[148756]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:49:51.852 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 22:49:51 smithi110 sudo[148756]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:49:51.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:51 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:49:51.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:51 smithi138 ceph-mon[104418]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T22:49:51.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:51 smithi138 ceph-mon[104418]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:49:51.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:51 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:52.113 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 22:49:51 smithi110 sudo[148785]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T22:49:52.113 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 22:49:51 smithi110 sudo[148785]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:49:52.113 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 22:49:51 smithi110 sudo[148785]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:49:52.113 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 22:49:51 smithi110 sudo[148785]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:49:52.113 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 22:49:52 smithi110 sudo[148798]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T22:49:52.113 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 22:49:52 smithi110 sudo[148798]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:49:52.113 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 22:49:52 smithi110 sudo[148798]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:49:52.113 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 22:49:52 smithi110 sudo[148798]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:49:52.626 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 22:49:52 smithi110 sudo[148846]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T22:49:52.627 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 22:49:52 smithi110 sudo[148846]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:49:52.627 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 22:49:52 smithi110 sudo[148846]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:49:52.627 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 22:49:52 smithi110 sudo[148846]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:49:52.884 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:52 smithi110 sudo[148897]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T22:49:52.884 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:52 smithi110 sudo[148897]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:49:52.884 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:52 smithi110 sudo[148897]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:49:52.884 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:52 smithi110 sudo[148897]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:49:53.146 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:52 smithi138 ceph-mon[104418]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T22:49:53.146 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:52 smithi138 ceph-mon[104418]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T22:49:53.146 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:52 smithi138 ceph-mon[104418]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:49:53.146 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 sudo[108492]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T22:49:53.146 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 sudo[108492]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:49:53.147 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 sudo[108492]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:49:53.147 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 sudo[108492]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:49:53.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:52 smithi110 ceph-mon[118821]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T22:49:53.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:52 smithi110 ceph-mon[118821]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T22:49:53.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:52 smithi110 ceph-mon[118821]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:49:53.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:52 smithi110 sudo[148901]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T22:49:53.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:52 smithi110 sudo[148901]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:49:53.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:52 smithi110 sudo[148901]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:49:53.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:52 smithi110 sudo[148901]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:49:53.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:52 smithi110 ceph-mon[100243]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T22:49:53.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:52 smithi110 ceph-mon[100243]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T22:49:53.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:52 smithi110 ceph-mon[100243]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:49:53.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 sudo[108495]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T22:49:53.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 sudo[108495]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:49:53.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 sudo[108495]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:49:53.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 sudo[108495]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:49:53.825 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 sudo[149149]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T22:49:53.825 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 sudo[149149]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:49:53.825 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 sudo[149149]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:49:53.826 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 sudo[149149]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:49:54.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:49:54.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:49:54.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:49:54.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:49:54.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:49:54.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:49:54.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:49:54.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:49:54.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:49:54.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:49:54.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:49:54.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:54.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: pgmap v78: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:49:54.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:49:54.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:54.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:49:54.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:49:54.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:53 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:49:54.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:49:54.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:49:54.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:49:54.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:49:54.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:49:54.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:49:54.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:49:54.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:49:54.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:49:54.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:49:54.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:49:54.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:54.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: pgmap v78: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:49:54.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:49:54.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:54.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:49:54.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:49:54.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:49:54.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 sudo[149198]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T22:49:54.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 sudo[149198]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:49:54.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 sudo[149198]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:49:54.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 sudo[149198]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:49:54.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:49:54.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:49:54.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:49:54.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:49:54.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:49:54.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:49:54.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:49:54.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:49:54.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:49:54.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:49:54.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:49:54.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:54.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: pgmap v78: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:49:54.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:49:54.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:49:54.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:49:54.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:49:54.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:53 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:49:55.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:54 smithi138 ceph-mon[104418]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:49:55.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:54 smithi138 ceph-mon[104418]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:49:55.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:54 smithi110 ceph-mon[100243]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:49:55.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:54 smithi110 ceph-mon[100243]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:49:55.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:54 smithi110 ceph-mon[118821]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:49:55.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:54 smithi110 ceph-mon[118821]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:49:56.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:55 smithi138 ceph-mon[104418]: pgmap v79: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:49:56.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:55 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2520537936' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6fcf80d5-6977-4929-b7ba-0e9ab65df0a8"}]: dispatch 2023-09-23T22:49:56.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:55 smithi138 ceph-mon[104418]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6fcf80d5-6977-4929-b7ba-0e9ab65df0a8"}]: dispatch 2023-09-23T22:49:56.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:55 smithi138 ceph-mon[104418]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6fcf80d5-6977-4929-b7ba-0e9ab65df0a8"}]': finished 2023-09-23T22:49:56.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:55 smithi138 ceph-mon[104418]: osdmap e22: 4 total, 3 up, 4 in 2023-09-23T22:49:56.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:55 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:49:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:55 smithi110 ceph-mon[118821]: pgmap v79: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:49:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:55 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2520537936' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6fcf80d5-6977-4929-b7ba-0e9ab65df0a8"}]: dispatch 2023-09-23T22:49:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:55 smithi110 ceph-mon[118821]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6fcf80d5-6977-4929-b7ba-0e9ab65df0a8"}]: dispatch 2023-09-23T22:49:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:55 smithi110 ceph-mon[118821]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6fcf80d5-6977-4929-b7ba-0e9ab65df0a8"}]': finished 2023-09-23T22:49:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:55 smithi110 ceph-mon[118821]: osdmap e22: 4 total, 3 up, 4 in 2023-09-23T22:49:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:55 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:49:56.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:55 smithi110 ceph-mon[100243]: pgmap v79: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:49:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:55 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2520537936' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6fcf80d5-6977-4929-b7ba-0e9ab65df0a8"}]: dispatch 2023-09-23T22:49:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:55 smithi110 ceph-mon[100243]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6fcf80d5-6977-4929-b7ba-0e9ab65df0a8"}]: dispatch 2023-09-23T22:49:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:55 smithi110 ceph-mon[100243]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6fcf80d5-6977-4929-b7ba-0e9ab65df0a8"}]': finished 2023-09-23T22:49:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:55 smithi110 ceph-mon[100243]: osdmap e22: 4 total, 3 up, 4 in 2023-09-23T22:49:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:55 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:49:57.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:56 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3674595407' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:49:57.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:56 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3674595407' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:49:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:56 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3674595407' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:49:58.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:49:57 smithi110 ceph-mon[118821]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 5.0 KiB/s rd, 242 KiB/s wr, 16 op/s 2023-09-23T22:49:58.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:49:57 smithi110 ceph-mon[100243]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 5.0 KiB/s rd, 242 KiB/s wr, 16 op/s 2023-09-23T22:49:58.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:49:57 smithi138 ceph-mon[104418]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 5.0 KiB/s rd, 242 KiB/s wr, 16 op/s 2023-09-23T22:50:00.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:00 smithi138 ceph-mon[104418]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.7 KiB/s rd, 177 KiB/s wr, 12 op/s 2023-09-23T22:50:00.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:00 smithi138 ceph-mon[104418]: overall HEALTH_OK 2023-09-23T22:50:00.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:00 smithi110 ceph-mon[100243]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.7 KiB/s rd, 177 KiB/s wr, 12 op/s 2023-09-23T22:50:00.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:00 smithi110 ceph-mon[100243]: overall HEALTH_OK 2023-09-23T22:50:00.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:00 smithi110 ceph-mon[118821]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.7 KiB/s rd, 177 KiB/s wr, 12 op/s 2023-09-23T22:50:00.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:00 smithi110 ceph-mon[118821]: overall HEALTH_OK 2023-09-23T22:50:02.604 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:02 smithi110 ceph-mon[100243]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.9 KiB/s rd, 139 KiB/s wr, 9 op/s 2023-09-23T22:50:02.604 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:02 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:50:02.604 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:02 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:02.604 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:02 smithi110 ceph-mon[118821]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.9 KiB/s rd, 139 KiB/s wr, 9 op/s 2023-09-23T22:50:02.605 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:02 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:50:02.605 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:02 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:02.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:02 smithi138 ceph-mon[104418]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.9 KiB/s rd, 139 KiB/s wr, 9 op/s 2023-09-23T22:50:02.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:02 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:50:02.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:02 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:03.599 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:03 smithi110 ceph-mon[100243]: Deploying daemon osd.3 on smithi110 2023-09-23T22:50:03.600 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:03 smithi110 ceph-mon[118821]: Deploying daemon osd.3 on smithi110 2023-09-23T22:50:03.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:03 smithi138 ceph-mon[104418]: Deploying daemon osd.3 on smithi110 2023-09-23T22:50:04.537 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:04 smithi110 ceph-mon[100243]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2023-09-23T22:50:04.537 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:04 smithi110 ceph-mon[118821]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2023-09-23T22:50:04.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:04 smithi138 ceph-mon[104418]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2023-09-23T22:50:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:06 smithi138 ceph-mon[104418]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2023-09-23T22:50:06.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:06 smithi110 ceph-mon[118821]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2023-09-23T22:50:06.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:06 smithi110 ceph-mon[100243]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2023-09-23T22:50:08.237 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:08 smithi110 ceph-mon[100243]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.3 KiB/s rd, 113 KiB/s wr, 7 op/s 2023-09-23T22:50:08.238 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:08 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:08.238 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:08 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:08.238 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:08 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:08.238 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:08 smithi110 ceph-mon[118821]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.3 KiB/s rd, 113 KiB/s wr, 7 op/s 2023-09-23T22:50:08.238 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:08 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:08.238 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:08 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:08.238 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:08 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:08.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:08 smithi138 ceph-mon[104418]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.3 KiB/s rd, 113 KiB/s wr, 7 op/s 2023-09-23T22:50:08.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:08 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:08.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:08 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:08.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:08 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:10.188 INFO:teuthology.orchestra.run.smithi110.stdout:Created osd(s) 3 on host 'smithi110' 2023-09-23T22:50:10.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[118821]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:50:10.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:10.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:10.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:50:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[118821]: from='osd.3 [v2:172.21.15.110:6826/427497470,v1:172.21.15.110:6827/427497470]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:50:10.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[100243]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:50:10.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:10.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:10.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:10.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:50:10.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:10.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:10.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:10.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:10.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:10 smithi110 ceph-mon[100243]: from='osd.3 [v2:172.21.15.110:6826/427497470,v1:172.21.15.110:6827/427497470]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:50:10.869 DEBUG:teuthology.orchestra.run.smithi110:osd.3> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.3.service 2023-09-23T22:50:10.871 INFO:tasks.cephadm:Deploying osd.4 on smithi138 with /dev/vg_nvme/lv_4... 2023-09-23T22:50:10.871 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T22:50:10.899 INFO:journalctl@ceph.osd.3.smithi110.stdout:-- Logs begin at Sat 2023-09-23 22:30:57 UTC. -- 2023-09-23T22:50:10.901 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:10 smithi138 ceph-mon[104418]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:50:10.901 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:10 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:10.901 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:10 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:10.901 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:10 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:10.901 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:10 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:50:10.901 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:10 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:10.901 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:10 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:10.901 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:10 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:10.902 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:10 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:10.902 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:10 smithi138 ceph-mon[104418]: from='osd.3 [v2:172.21.15.110:6826/427497470,v1:172.21.15.110:6827/427497470]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:50:11.223 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:50:12.275 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 22:50:12 smithi110 conmon[151896]: 2023-09-23T22:50:12.166+0000 7fb85a92e700 -1 osd.3 0 waiting for initial osdmap 2023-09-23T22:50:12.275 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 22:50:12 smithi110 conmon[151896]: 2023-09-23T22:50:12.172+0000 7fb85772d700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:50:12.276 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:12 smithi110 ceph-mon[100243]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:50:12.276 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:12 smithi110 ceph-mon[100243]: from='osd.3 [v2:172.21.15.110:6826/427497470,v1:172.21.15.110:6827/427497470]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:50:12.276 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:12 smithi110 ceph-mon[100243]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T22:50:12.276 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:12 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:50:12.276 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:12 smithi110 ceph-mon[100243]: from='osd.3 [v2:172.21.15.110:6826/427497470,v1:172.21.15.110:6827/427497470]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T22:50:12.276 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:12 smithi110 ceph-mon[118821]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:50:12.276 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:12 smithi110 ceph-mon[118821]: from='osd.3 [v2:172.21.15.110:6826/427497470,v1:172.21.15.110:6827/427497470]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:50:12.276 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:12 smithi110 ceph-mon[118821]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T22:50:12.276 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:12 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:50:12.277 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:12 smithi110 ceph-mon[118821]: from='osd.3 [v2:172.21.15.110:6826/427497470,v1:172.21.15.110:6827/427497470]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T22:50:12.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:12 smithi138 ceph-mon[104418]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:50:12.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:12 smithi138 ceph-mon[104418]: from='osd.3 [v2:172.21.15.110:6826/427497470,v1:172.21.15.110:6827/427497470]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:50:12.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:12 smithi138 ceph-mon[104418]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T22:50:12.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:12 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:50:12.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:12 smithi138 ceph-mon[104418]: from='osd.3 [v2:172.21.15.110:6826/427497470,v1:172.21.15.110:6827/427497470]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T22:50:12.962 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T22:50:13.003 INFO:teuthology.orchestra.run.smithi138.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T22:50:13.004 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10+0 records in 2023-09-23T22:50:13.004 INFO:teuthology.orchestra.run.smithi138.stderr:10+0 records out 2023-09-23T22:50:13.004 INFO:teuthology.orchestra.run.smithi138.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0110015 s, 953 MB/s 2023-09-23T22:50:13.004 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 2023-09-23T22:50:13.004 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping successful for: 2023-09-23T22:50:13.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:13 smithi138 ceph-mon[104418]: from='osd.3 [v2:172.21.15.110:6826/427497470,v1:172.21.15.110:6827/427497470]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-09-23T22:50:13.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:13 smithi138 ceph-mon[104418]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T22:50:13.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:13 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:50:13.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:13 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:50:13.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:13 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:50:13.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:13 smithi110 ceph-mon[100243]: from='osd.3 [v2:172.21.15.110:6826/427497470,v1:172.21.15.110:6827/427497470]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-09-23T22:50:13.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:13 smithi110 ceph-mon[100243]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T22:50:13.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:13 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:50:13.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:13 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:50:13.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:13 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:50:13.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:13 smithi110 ceph-mon[118821]: from='osd.3 [v2:172.21.15.110:6826/427497470,v1:172.21.15.110:6827/427497470]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-09-23T22:50:13.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:13 smithi110 ceph-mon[118821]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T22:50:13.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:13 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:50:13.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:13 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:50:13.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:13 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:50:13.549 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi138:vg_nvme/lv_4 2023-09-23T22:50:13.924 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:50:14.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:14 smithi138 ceph-mon[104418]: purged_snaps scrub starts 2023-09-23T22:50:14.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:14 smithi138 ceph-mon[104418]: purged_snaps scrub ok 2023-09-23T22:50:14.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:14 smithi138 ceph-mon[104418]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:50:14.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:14 smithi138 ceph-mon[104418]: osd.3 [v2:172.21.15.110:6826/427497470,v1:172.21.15.110:6827/427497470] boot 2023-09-23T22:50:14.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:14 smithi138 ceph-mon[104418]: osdmap e25: 4 total, 4 up, 4 in 2023-09-23T22:50:14.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:14 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:50:14.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:14 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:14.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:14 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:14 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:50:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:14 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:50:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:14 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:50:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:14 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:50:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:14 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:14 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:14 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:50:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:14 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:14.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[118821]: purged_snaps scrub starts 2023-09-23T22:50:14.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[118821]: purged_snaps scrub ok 2023-09-23T22:50:14.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[118821]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:50:14.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[118821]: osd.3 [v2:172.21.15.110:6826/427497470,v1:172.21.15.110:6827/427497470] boot 2023-09-23T22:50:14.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[118821]: osdmap e25: 4 total, 4 up, 4 in 2023-09-23T22:50:14.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:50:14.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:14.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:14.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:50:14.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:50:14.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:50:14.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:50:14.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:14.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:14.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:50:14.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:14.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[100243]: purged_snaps scrub starts 2023-09-23T22:50:14.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[100243]: purged_snaps scrub ok 2023-09-23T22:50:14.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[100243]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:50:14.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[100243]: osd.3 [v2:172.21.15.110:6826/427497470,v1:172.21.15.110:6827/427497470] boot 2023-09-23T22:50:14.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[100243]: osdmap e25: 4 total, 4 up, 4 in 2023-09-23T22:50:14.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:50:14.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:14.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:14.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:50:14.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:50:14.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:50:14.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:50:14.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:14.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:14.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:50:14.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:14 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:15.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:15 smithi138 ceph-mon[104418]: Detected new or changed devices on smithi110 2023-09-23T22:50:15.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:15 smithi138 ceph-mon[104418]: Adjusting osd_memory_target on smithi110 to 4036M 2023-09-23T22:50:15.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:15 smithi138 ceph-mon[104418]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T22:50:15.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:15 smithi110 ceph-mon[100243]: Detected new or changed devices on smithi110 2023-09-23T22:50:15.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:15 smithi110 ceph-mon[100243]: Adjusting osd_memory_target on smithi110 to 4036M 2023-09-23T22:50:15.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:15 smithi110 ceph-mon[100243]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T22:50:15.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:15 smithi110 ceph-mon[118821]: Detected new or changed devices on smithi110 2023-09-23T22:50:15.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:15 smithi110 ceph-mon[118821]: Adjusting osd_memory_target on smithi110 to 4036M 2023-09-23T22:50:15.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:15 smithi110 ceph-mon[118821]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T22:50:16.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:16 smithi138 ceph-mon[104418]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:16.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:50:16.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:50:16.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:16.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:16 smithi110 ceph-mon[118821]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:16.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:16 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:50:16.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:16 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:50:16.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:16 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:16.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:16 smithi110 ceph-mon[100243]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:16.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:50:16.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:50:16.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:17.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:17 smithi138 ceph-mon[104418]: from='client.24190 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:50:17.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:17 smithi110 ceph-mon[100243]: from='client.24190 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:50:17.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:17 smithi110 ceph-mon[118821]: from='client.24190 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:50:18.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:18 smithi138 ceph-mon[104418]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:18.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:18 smithi138 ceph-mon[104418]: from='client.? 172.21.15.138:0/199580067' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "131d0749-c46c-44f0-ab7f-03f8ad6ed502"}]: dispatch 2023-09-23T22:50:18.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:18 smithi138 ceph-mon[104418]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "131d0749-c46c-44f0-ab7f-03f8ad6ed502"}]: dispatch 2023-09-23T22:50:18.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:18 smithi138 ceph-mon[104418]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "131d0749-c46c-44f0-ab7f-03f8ad6ed502"}]': finished 2023-09-23T22:50:18.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:18 smithi138 ceph-mon[104418]: osdmap e27: 5 total, 4 up, 5 in 2023-09-23T22:50:18.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:18 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:50:18.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:18 smithi110 ceph-mon[100243]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:18.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:18 smithi110 ceph-mon[100243]: from='client.? 172.21.15.138:0/199580067' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "131d0749-c46c-44f0-ab7f-03f8ad6ed502"}]: dispatch 2023-09-23T22:50:18.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:18 smithi110 ceph-mon[100243]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "131d0749-c46c-44f0-ab7f-03f8ad6ed502"}]: dispatch 2023-09-23T22:50:18.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:18 smithi110 ceph-mon[100243]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "131d0749-c46c-44f0-ab7f-03f8ad6ed502"}]': finished 2023-09-23T22:50:18.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:18 smithi110 ceph-mon[100243]: osdmap e27: 5 total, 4 up, 5 in 2023-09-23T22:50:18.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:18 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:50:18.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:18 smithi110 ceph-mon[118821]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:18.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:18 smithi110 ceph-mon[118821]: from='client.? 172.21.15.138:0/199580067' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "131d0749-c46c-44f0-ab7f-03f8ad6ed502"}]: dispatch 2023-09-23T22:50:18.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:18 smithi110 ceph-mon[118821]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "131d0749-c46c-44f0-ab7f-03f8ad6ed502"}]: dispatch 2023-09-23T22:50:18.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:18 smithi110 ceph-mon[118821]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "131d0749-c46c-44f0-ab7f-03f8ad6ed502"}]': finished 2023-09-23T22:50:18.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:18 smithi110 ceph-mon[118821]: osdmap e27: 5 total, 4 up, 5 in 2023-09-23T22:50:18.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:18 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:50:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:19 smithi138 ceph-mon[104418]: from='client.? 172.21.15.138:0/1481144376' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:50:19.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:19 smithi110 ceph-mon[100243]: from='client.? 172.21.15.138:0/1481144376' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:50:19.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:19 smithi110 ceph-mon[118821]: from='client.? 172.21.15.138:0/1481144376' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:50:20.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:20 smithi138 ceph-mon[104418]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:20.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:20 smithi110 ceph-mon[100243]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:20.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:20 smithi110 ceph-mon[118821]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:22.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:22 smithi138 ceph-mon[104418]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:22.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:22 smithi110 ceph-mon[100243]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:22.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:22 smithi110 ceph-mon[118821]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:24.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:24 smithi138 ceph-mon[104418]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:24.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T22:50:24.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:24.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:24 smithi110 ceph-mon[100243]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:24.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T22:50:24.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:24.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:24 smithi110 ceph-mon[118821]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:24.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T22:50:24.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:25.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:25 smithi138 ceph-mon[104418]: Deploying daemon osd.4 on smithi138 2023-09-23T22:50:25.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:25 smithi110 ceph-mon[100243]: Deploying daemon osd.4 on smithi138 2023-09-23T22:50:25.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:25 smithi110 ceph-mon[118821]: Deploying daemon osd.4 on smithi138 2023-09-23T22:50:26.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:26 smithi138 ceph-mon[104418]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:26.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:26 smithi110 ceph-mon[100243]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:26.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:26 smithi110 ceph-mon[118821]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:28.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:28 smithi138 ceph-mon[104418]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:28.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:28 smithi110 ceph-mon[100243]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:28.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:28 smithi110 ceph-mon[118821]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:30 smithi138 ceph-mon[104418]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:30 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:30 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:30 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:30.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:30 smithi110 ceph-mon[100243]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:30.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:30 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:30.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:30 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:30.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:30 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:30.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:30 smithi110 ceph-mon[118821]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:30.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:30 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:30.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:30 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:30.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:30 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:32.361 INFO:teuthology.orchestra.run.smithi138.stdout:Created osd(s) 4 on host 'smithi138' 2023-09-23T22:50:32.521 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:32 smithi138 ceph-mon[104418]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:32.521 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:32 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:32.521 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:32 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:32.521 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:32 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:32.521 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:32 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:50:32.521 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:32 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:32.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:32 smithi110 ceph-mon[100243]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:32.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:32 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:32.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:32 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:32.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:32 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:32.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:32 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:50:32.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:32 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:32.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:32 smithi110 ceph-mon[118821]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:32.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:32 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:32.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:32 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:32.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:32 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:32.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:32 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:50:32.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:32 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:32.984 DEBUG:teuthology.orchestra.run.smithi138:osd.4> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.4.service 2023-09-23T22:50:32.986 INFO:tasks.cephadm:Deploying osd.5 on smithi138 with /dev/vg_nvme/lv_3... 2023-09-23T22:50:32.986 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T22:50:33.017 INFO:journalctl@ceph.osd.4.smithi138.stdout:-- Logs begin at Sat 2023-09-23 22:31:02 UTC. -- 2023-09-23T22:50:33.546 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:33 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:33.546 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:33 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:33.546 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:33 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:33.546 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:33 smithi138 ceph-mon[104418]: from='osd.4 [v2:172.21.15.138:6800/941787557,v1:172.21.15.138:6801/941787557]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:50:33.546 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:33 smithi138 ceph-mon[104418]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:50:33.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:33 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:33.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:33 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:33.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:33 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:33.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:33 smithi110 ceph-mon[100243]: from='osd.4 [v2:172.21.15.138:6800/941787557,v1:172.21.15.138:6801/941787557]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:50:33.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:33 smithi110 ceph-mon[100243]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:50:33.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:33 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:33.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:33 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:33.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:33 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:33.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:33 smithi110 ceph-mon[118821]: from='osd.4 [v2:172.21.15.138:6800/941787557,v1:172.21.15.138:6801/941787557]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:50:33.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:33 smithi110 ceph-mon[118821]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:50:34.059 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:50:34.529 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:34 smithi138 ceph-mon[104418]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:34.529 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:34 smithi138 ceph-mon[104418]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T22:50:34.529 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:34 smithi138 ceph-mon[104418]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T22:50:34.529 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:34 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:50:34.529 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:34 smithi138 ceph-mon[104418]: from='osd.4 [v2:172.21.15.138:6800/941787557,v1:172.21.15.138:6801/941787557]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:50:34.529 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:34 smithi138 ceph-mon[104418]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:50:34.530 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 22:50:34 smithi138 conmon[112407]: 2023-09-23T22:50:34.369+0000 7f933ccba700 -1 osd.4 0 waiting for initial osdmap 2023-09-23T22:50:34.530 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 22:50:34 smithi138 conmon[112407]: 2023-09-23T22:50:34.373+0000 7f9337ab5700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:50:34.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:34 smithi110 ceph-mon[100243]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:34.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:34 smithi110 ceph-mon[100243]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T22:50:34.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:34 smithi110 ceph-mon[100243]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T22:50:34.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:34 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:50:34.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:34 smithi110 ceph-mon[100243]: from='osd.4 [v2:172.21.15.138:6800/941787557,v1:172.21.15.138:6801/941787557]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:50:34.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:34 smithi110 ceph-mon[100243]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:50:34.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:34 smithi110 ceph-mon[118821]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:34.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:34 smithi110 ceph-mon[118821]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T22:50:34.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:34 smithi110 ceph-mon[118821]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T22:50:34.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:34 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:50:34.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:34 smithi110 ceph-mon[118821]: from='osd.4 [v2:172.21.15.138:6800/941787557,v1:172.21.15.138:6801/941787557]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:50:34.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:34 smithi110 ceph-mon[118821]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:50:35.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:35 smithi138 ceph-mon[104418]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-09-23T22:50:35.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:35 smithi138 ceph-mon[104418]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T22:50:35.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:35 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:50:35.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:35 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:50:35.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:35 smithi110 ceph-mon[100243]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-09-23T22:50:35.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:35 smithi110 ceph-mon[100243]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T22:50:35.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:35 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:50:35.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:35 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:50:35.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:35 smithi110 ceph-mon[118821]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-09-23T22:50:35.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:35 smithi110 ceph-mon[118821]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T22:50:35.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:35 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:50:35.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:35 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:50:36.353 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T22:50:36.353 INFO:teuthology.orchestra.run.smithi138.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T22:50:36.354 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10+0 records in 2023-09-23T22:50:36.354 INFO:teuthology.orchestra.run.smithi138.stderr:10+0 records out 2023-09-23T22:50:36.354 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0109368 s, 959 MB/s 2023-09-23T22:50:36.354 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping successful for: 2023-09-23T22:50:36.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:36 smithi138 ceph-mon[104418]: purged_snaps scrub starts 2023-09-23T22:50:36.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:36 smithi138 ceph-mon[104418]: purged_snaps scrub ok 2023-09-23T22:50:36.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:36 smithi138 ceph-mon[104418]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:36.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:36 smithi138 ceph-mon[104418]: osd.4 [v2:172.21.15.138:6800/941787557,v1:172.21.15.138:6801/941787557] boot 2023-09-23T22:50:36.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:36 smithi138 ceph-mon[104418]: osdmap e30: 5 total, 5 up, 5 in 2023-09-23T22:50:36.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:36 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:50:36.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:36 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:36 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:36 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:50:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:36 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:36 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:36 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:50:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:36 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:36 smithi138 ceph-mon[104418]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T22:50:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[100243]: purged_snaps scrub starts 2023-09-23T22:50:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[100243]: purged_snaps scrub ok 2023-09-23T22:50:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[100243]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[100243]: osd.4 [v2:172.21.15.138:6800/941787557,v1:172.21.15.138:6801/941787557] boot 2023-09-23T22:50:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[100243]: osdmap e30: 5 total, 5 up, 5 in 2023-09-23T22:50:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:50:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:36.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:36.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:50:36.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:36.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:36.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:50:36.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:36.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[100243]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T22:50:36.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[118821]: purged_snaps scrub starts 2023-09-23T22:50:36.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[118821]: purged_snaps scrub ok 2023-09-23T22:50:36.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[118821]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:50:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[118821]: osd.4 [v2:172.21.15.138:6800/941787557,v1:172.21.15.138:6801/941787557] boot 2023-09-23T22:50:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[118821]: osdmap e30: 5 total, 5 up, 5 in 2023-09-23T22:50:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:50:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:50:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:50:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:36 smithi110 ceph-mon[118821]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T22:50:36.918 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi138:vg_nvme/lv_3 2023-09-23T22:50:37.251 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:50:37.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:37 smithi138 ceph-mon[104418]: Detected new or changed devices on smithi138 2023-09-23T22:50:37.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:37 smithi138 ceph-mon[104418]: Adjusting osd_memory_target on smithi138 to 17168M 2023-09-23T22:50:37.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:37 smithi138 ceph-mon[104418]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T22:50:37.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:37 smithi110 ceph-mon[100243]: Detected new or changed devices on smithi138 2023-09-23T22:50:37.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:37 smithi110 ceph-mon[100243]: Adjusting osd_memory_target on smithi138 to 17168M 2023-09-23T22:50:37.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:37 smithi110 ceph-mon[100243]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T22:50:37.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:37 smithi110 ceph-mon[118821]: Detected new or changed devices on smithi138 2023-09-23T22:50:37.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:37 smithi110 ceph-mon[118821]: Adjusting osd_memory_target on smithi138 to 17168M 2023-09-23T22:50:37.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:37 smithi110 ceph-mon[118821]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T22:50:38.414 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:38 smithi138 ceph-mon[104418]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:38.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:38 smithi110 ceph-mon[100243]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:38.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:38 smithi110 ceph-mon[118821]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:39.644 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:39 smithi138 ceph-mon[104418]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T22:50:39.644 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:39 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:50:39.644 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:39 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:50:39.645 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:39 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:39.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:39 smithi110 ceph-mon[100243]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T22:50:39.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:39 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:50:39.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:39 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:50:39.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:39 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:39.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:39 smithi110 ceph-mon[118821]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T22:50:39.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:39 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:50:39.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:39 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:50:39.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:39 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:40.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:40 smithi138 ceph-mon[104418]: from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:50:40.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:40 smithi138 ceph-mon[104418]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:40.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:40 smithi138 ceph-mon[104418]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T22:50:40.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:40 smithi110 ceph-mon[100243]: from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:50:40.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:40 smithi110 ceph-mon[100243]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:40.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:40 smithi110 ceph-mon[100243]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T22:50:40.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:40 smithi110 ceph-mon[118821]: from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:50:40.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:40 smithi110 ceph-mon[118821]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:40.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:40 smithi110 ceph-mon[118821]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T22:50:41.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:41 smithi138 ceph-mon[104418]: from='client.? 172.21.15.138:0/1016913269' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8cb496e5-d242-4e41-b279-e8d5ecea2996"}]: dispatch 2023-09-23T22:50:41.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:41 smithi138 ceph-mon[104418]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8cb496e5-d242-4e41-b279-e8d5ecea2996"}]: dispatch 2023-09-23T22:50:41.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:41 smithi138 ceph-mon[104418]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8cb496e5-d242-4e41-b279-e8d5ecea2996"}]': finished 2023-09-23T22:50:41.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:41 smithi138 ceph-mon[104418]: osdmap e35: 6 total, 5 up, 6 in 2023-09-23T22:50:41.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:41 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:50:41.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:41 smithi110 ceph-mon[100243]: from='client.? 172.21.15.138:0/1016913269' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8cb496e5-d242-4e41-b279-e8d5ecea2996"}]: dispatch 2023-09-23T22:50:41.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:41 smithi110 ceph-mon[100243]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8cb496e5-d242-4e41-b279-e8d5ecea2996"}]: dispatch 2023-09-23T22:50:41.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:41 smithi110 ceph-mon[100243]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8cb496e5-d242-4e41-b279-e8d5ecea2996"}]': finished 2023-09-23T22:50:41.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:41 smithi110 ceph-mon[100243]: osdmap e35: 6 total, 5 up, 6 in 2023-09-23T22:50:41.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:41 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:50:41.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:41 smithi110 ceph-mon[118821]: from='client.? 172.21.15.138:0/1016913269' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8cb496e5-d242-4e41-b279-e8d5ecea2996"}]: dispatch 2023-09-23T22:50:41.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:41 smithi110 ceph-mon[118821]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8cb496e5-d242-4e41-b279-e8d5ecea2996"}]: dispatch 2023-09-23T22:50:41.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:41 smithi110 ceph-mon[118821]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8cb496e5-d242-4e41-b279-e8d5ecea2996"}]': finished 2023-09-23T22:50:41.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:41 smithi110 ceph-mon[118821]: osdmap e35: 6 total, 5 up, 6 in 2023-09-23T22:50:41.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:41 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:50:42.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:42 smithi138 ceph-mon[104418]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:42.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:42 smithi138 ceph-mon[104418]: from='client.? 172.21.15.138:0/68504750' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:50:42.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:42 smithi110 ceph-mon[100243]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:42.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:42 smithi110 ceph-mon[100243]: from='client.? 172.21.15.138:0/68504750' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:50:42.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:42 smithi110 ceph-mon[118821]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:42.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:42 smithi110 ceph-mon[118821]: from='client.? 172.21.15.138:0/68504750' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:50:44.656 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:44 smithi138 ceph-mon[104418]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-09-23T22:50:44.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:44 smithi110 ceph-mon[100243]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-09-23T22:50:44.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:44 smithi110 ceph-mon[118821]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-09-23T22:50:46.663 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:46 smithi138 ceph-mon[104418]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-09-23T22:50:46.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:46 smithi110 ceph-mon[100243]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-09-23T22:50:46.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:46 smithi110 ceph-mon[118821]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-09-23T22:50:47.690 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:47 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T22:50:47.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:47 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:47.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:47 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T22:50:47.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:47 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:47.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:47 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T22:50:47.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:47 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:48.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:48 smithi138 ceph-mon[104418]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:50:48.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:48 smithi138 ceph-mon[104418]: Deploying daemon osd.5 on smithi138 2023-09-23T22:50:48.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:48 smithi110 ceph-mon[100243]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:50:48.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:48 smithi110 ceph-mon[100243]: Deploying daemon osd.5 on smithi138 2023-09-23T22:50:48.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:48 smithi110 ceph-mon[118821]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:50:48.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:48 smithi110 ceph-mon[118821]: Deploying daemon osd.5 on smithi138 2023-09-23T22:50:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:50 smithi138 ceph-mon[104418]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-09-23T22:50:50.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:50 smithi110 ceph-mon[100243]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-09-23T22:50:50.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:50 smithi110 ceph-mon[118821]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-09-23T22:50:52.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:52 smithi138 ceph-mon[104418]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:50:52.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:52 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:52.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:52 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:52.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:52 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:52.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:52 smithi110 ceph-mon[100243]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:50:52.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:52 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:52.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:52 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:52.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:52 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:52.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:52 smithi110 ceph-mon[118821]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:50:52.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:52 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:52.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:52 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:52.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:52 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:54.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:54 smithi138 ceph-mon[104418]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:54.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:54 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:54.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:54 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:54.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:54 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:54.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:54 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:50:54.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:54 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:54.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:54 smithi110 ceph-mon[100243]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:54.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:54 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:54.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:54 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:54.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:54 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:54.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:54 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:50:54.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:54 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:54.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:54 smithi110 ceph-mon[118821]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:54.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:54 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:54.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:54 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:54.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:54 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:50:54.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:54 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:50:54.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:54 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:54.809 INFO:teuthology.orchestra.run.smithi138.stdout:Created osd(s) 5 on host 'smithi138' 2023-09-23T22:50:55.439 DEBUG:teuthology.orchestra.run.smithi138:osd.5> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.5.service 2023-09-23T22:50:55.442 INFO:tasks.cephadm:Deploying osd.6 on smithi138 with /dev/vg_nvme/lv_2... 2023-09-23T22:50:55.442 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T22:50:55.472 INFO:journalctl@ceph.osd.5.smithi138.stdout:-- Logs begin at Sat 2023-09-23 22:31:02 UTC. -- 2023-09-23T22:50:56.117 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:55 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:56.117 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:55 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:56.117 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:55 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:56.117 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:55 smithi138 ceph-mon[104418]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:56.117 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:55 smithi138 ceph-mon[104418]: from='osd.5 [v2:172.21.15.138:6808/3098195887,v1:172.21.15.138:6809/3098195887]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T22:50:56.117 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:55 smithi138 ceph-mon[104418]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T22:50:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:55 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:55 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:55 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:56.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:55 smithi110 ceph-mon[118821]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:55 smithi110 ceph-mon[118821]: from='osd.5 [v2:172.21.15.138:6808/3098195887,v1:172.21.15.138:6809/3098195887]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T22:50:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:55 smithi110 ceph-mon[118821]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T22:50:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:55 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:55 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:55 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:50:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:55 smithi110 ceph-mon[100243]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:55 smithi110 ceph-mon[100243]: from='osd.5 [v2:172.21.15.138:6808/3098195887,v1:172.21.15.138:6809/3098195887]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T22:50:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:55 smithi110 ceph-mon[100243]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T22:50:56.549 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:50:57.191 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 22:50:56 smithi138 conmon[122074]: 2023-09-23T22:50:56.807+0000 7fb5dd0a4700 -1 osd.5 0 waiting for initial osdmap 2023-09-23T22:50:57.191 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 22:50:56 smithi138 conmon[122074]: 2023-09-23T22:50:56.814+0000 7fb5d769e700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:50:57.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:56 smithi138 ceph-mon[104418]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T22:50:57.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:56 smithi138 ceph-mon[104418]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T22:50:57.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:56 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:50:57.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:56 smithi138 ceph-mon[104418]: from='osd.5 [v2:172.21.15.138:6808/3098195887,v1:172.21.15.138:6809/3098195887]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:50:57.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:56 smithi138 ceph-mon[104418]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:50:57.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:56 smithi110 ceph-mon[100243]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T22:50:57.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:56 smithi110 ceph-mon[100243]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T22:50:57.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:56 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:50:57.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:56 smithi110 ceph-mon[100243]: from='osd.5 [v2:172.21.15.138:6808/3098195887,v1:172.21.15.138:6809/3098195887]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:50:57.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:56 smithi110 ceph-mon[100243]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:50:57.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:56 smithi110 ceph-mon[118821]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T22:50:57.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:56 smithi110 ceph-mon[118821]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T22:50:57.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:56 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:50:57.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:56 smithi110 ceph-mon[118821]: from='osd.5 [v2:172.21.15.138:6808/3098195887,v1:172.21.15.138:6809/3098195887]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:50:57.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:56 smithi110 ceph-mon[118821]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:50:58.027 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:57 smithi138 ceph-mon[104418]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-09-23T22:50:58.027 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:57 smithi138 ceph-mon[104418]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T22:50:58.028 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:57 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:50:58.028 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:57 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:50:58.028 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:57 smithi138 ceph-mon[104418]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:58.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:57 smithi110 ceph-mon[100243]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-09-23T22:50:58.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:57 smithi110 ceph-mon[100243]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T22:50:58.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:57 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:50:58.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:57 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:50:58.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:57 smithi110 ceph-mon[100243]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:58.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:57 smithi110 ceph-mon[118821]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-09-23T22:50:58.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:57 smithi110 ceph-mon[118821]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T22:50:58.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:57 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:50:58.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:57 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:50:58.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:57 smithi110 ceph-mon[118821]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:50:58.530 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T22:50:58.530 INFO:teuthology.orchestra.run.smithi138.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T22:50:58.530 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10+0 records in 2023-09-23T22:50:58.530 INFO:teuthology.orchestra.run.smithi138.stderr:10+0 records out 2023-09-23T22:50:58.530 INFO:teuthology.orchestra.run.smithi138.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0109372 s, 959 MB/s 2023-09-23T22:50:58.531 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 2023-09-23T22:50:58.531 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping successful for: 2023-09-23T22:50:59.124 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:58 smithi138 ceph-mon[104418]: purged_snaps scrub starts 2023-09-23T22:50:59.125 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:58 smithi138 ceph-mon[104418]: purged_snaps scrub ok 2023-09-23T22:50:59.125 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:58 smithi138 ceph-mon[104418]: osd.5 [v2:172.21.15.138:6808/3098195887,v1:172.21.15.138:6809/3098195887] boot 2023-09-23T22:50:59.125 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:58 smithi138 ceph-mon[104418]: osdmap e38: 6 total, 6 up, 6 in 2023-09-23T22:50:59.125 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:58 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:50:59.125 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:58 smithi138 ceph-mon[104418]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T22:50:59.139 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi138:vg_nvme/lv_2 2023-09-23T22:50:59.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:58 smithi110 ceph-mon[100243]: purged_snaps scrub starts 2023-09-23T22:50:59.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:58 smithi110 ceph-mon[100243]: purged_snaps scrub ok 2023-09-23T22:50:59.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:58 smithi110 ceph-mon[100243]: osd.5 [v2:172.21.15.138:6808/3098195887,v1:172.21.15.138:6809/3098195887] boot 2023-09-23T22:50:59.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:58 smithi110 ceph-mon[100243]: osdmap e38: 6 total, 6 up, 6 in 2023-09-23T22:50:59.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:58 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:50:59.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:58 smithi110 ceph-mon[100243]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T22:50:59.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:58 smithi110 ceph-mon[118821]: purged_snaps scrub starts 2023-09-23T22:50:59.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:58 smithi110 ceph-mon[118821]: purged_snaps scrub ok 2023-09-23T22:50:59.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:58 smithi110 ceph-mon[118821]: osd.5 [v2:172.21.15.138:6808/3098195887,v1:172.21.15.138:6809/3098195887] boot 2023-09-23T22:50:59.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:58 smithi110 ceph-mon[118821]: osdmap e38: 6 total, 6 up, 6 in 2023-09-23T22:50:59.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:58 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:50:59.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:58 smithi110 ceph-mon[118821]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T22:50:59.477 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:51:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:59 smithi138 ceph-mon[104418]: Detected new or changed devices on smithi138 2023-09-23T22:51:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:59 smithi138 ceph-mon[104418]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:59 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:59 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:59 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:59 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:59 smithi138 ceph-mon[104418]: Adjusting osd_memory_target on smithi138 to 8584M 2023-09-23T22:51:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:59 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:59 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:59 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:59 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:50:59 smithi138 ceph-mon[104418]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T22:51:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[100243]: Detected new or changed devices on smithi138 2023-09-23T22:51:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[100243]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[100243]: Adjusting osd_memory_target on smithi138 to 8584M 2023-09-23T22:51:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[100243]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T22:51:00.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[118821]: Detected new or changed devices on smithi138 2023-09-23T22:51:00.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[118821]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:00.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:00.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:00.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:00.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:00.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[118821]: Adjusting osd_memory_target on smithi138 to 8584M 2023-09-23T22:51:00.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:00.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:00.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:00.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:00.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:50:59 smithi110 ceph-mon[118821]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T22:51:02.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:01 smithi138 ceph-mon[104418]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:02.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:01 smithi138 ceph-mon[104418]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T22:51:02.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:01 smithi138 ceph-mon[104418]: from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:51:02.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:01 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:51:02.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:01 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:51:02.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:01 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:02.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:01 smithi110 ceph-mon[100243]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:02.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:01 smithi110 ceph-mon[100243]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T22:51:02.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:01 smithi110 ceph-mon[100243]: from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:51:02.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:01 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:51:02.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:01 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:51:02.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:01 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:02.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:01 smithi110 ceph-mon[118821]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:02.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:01 smithi110 ceph-mon[118821]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T22:51:02.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:01 smithi110 ceph-mon[118821]: from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:51:02.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:01 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:51:02.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:01 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:51:02.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:01 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:03.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:02 smithi138 ceph-mon[104418]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T22:51:03.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:02 smithi110 ceph-mon[100243]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T22:51:03.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:02 smithi110 ceph-mon[118821]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T22:51:04.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:03 smithi138 ceph-mon[104418]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:04.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:03 smithi138 ceph-mon[104418]: from='client.? 172.21.15.138:0/3599291413' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c709e629-85ab-4273-9ca8-f2c4e8fc9074"}]: dispatch 2023-09-23T22:51:04.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:03 smithi138 ceph-mon[104418]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c709e629-85ab-4273-9ca8-f2c4e8fc9074"}]: dispatch 2023-09-23T22:51:04.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:03 smithi138 ceph-mon[104418]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c709e629-85ab-4273-9ca8-f2c4e8fc9074"}]': finished 2023-09-23T22:51:04.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:03 smithi138 ceph-mon[104418]: osdmap e43: 7 total, 6 up, 7 in 2023-09-23T22:51:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:03 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:51:04.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:03 smithi110 ceph-mon[100243]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:04.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:03 smithi110 ceph-mon[100243]: from='client.? 172.21.15.138:0/3599291413' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c709e629-85ab-4273-9ca8-f2c4e8fc9074"}]: dispatch 2023-09-23T22:51:04.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:03 smithi110 ceph-mon[100243]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c709e629-85ab-4273-9ca8-f2c4e8fc9074"}]: dispatch 2023-09-23T22:51:04.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:03 smithi110 ceph-mon[100243]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c709e629-85ab-4273-9ca8-f2c4e8fc9074"}]': finished 2023-09-23T22:51:04.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:03 smithi110 ceph-mon[100243]: osdmap e43: 7 total, 6 up, 7 in 2023-09-23T22:51:04.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:03 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:51:04.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:03 smithi110 ceph-mon[118821]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:04.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:03 smithi110 ceph-mon[118821]: from='client.? 172.21.15.138:0/3599291413' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c709e629-85ab-4273-9ca8-f2c4e8fc9074"}]: dispatch 2023-09-23T22:51:04.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:03 smithi110 ceph-mon[118821]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c709e629-85ab-4273-9ca8-f2c4e8fc9074"}]: dispatch 2023-09-23T22:51:04.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:03 smithi110 ceph-mon[118821]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c709e629-85ab-4273-9ca8-f2c4e8fc9074"}]': finished 2023-09-23T22:51:04.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:03 smithi110 ceph-mon[118821]: osdmap e43: 7 total, 6 up, 7 in 2023-09-23T22:51:04.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:03 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:51:05.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:04 smithi138 ceph-mon[104418]: from='client.? 172.21.15.138:0/1625070428' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:51:05.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:04 smithi110 ceph-mon[100243]: from='client.? 172.21.15.138:0/1625070428' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:51:05.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:04 smithi110 ceph-mon[118821]: from='client.? 172.21.15.138:0/1625070428' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:51:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:06 smithi138 ceph-mon[104418]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:06.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:06 smithi110 ceph-mon[100243]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:06.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:06 smithi110 ceph-mon[118821]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:08.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:08 smithi138 ceph-mon[104418]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:08.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:08 smithi110 ceph-mon[100243]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:08.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:08 smithi110 ceph-mon[118821]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:10.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:10 smithi138 ceph-mon[104418]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:10.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:10 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T22:51:10.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:10 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:10.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:10 smithi138 ceph-mon[104418]: Deploying daemon osd.6 on smithi138 2023-09-23T22:51:10.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:10 smithi110 ceph-mon[100243]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:10.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:10 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T22:51:10.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:10 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:10.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:10 smithi110 ceph-mon[100243]: Deploying daemon osd.6 on smithi138 2023-09-23T22:51:10.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:10 smithi110 ceph-mon[118821]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:10.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:10 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T22:51:10.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:10 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:10.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:10 smithi110 ceph-mon[118821]: Deploying daemon osd.6 on smithi138 2023-09-23T22:51:12.619 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:12 smithi138 ceph-mon[104418]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:12.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:12 smithi110 ceph-mon[100243]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:12.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:12 smithi110 ceph-mon[118821]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:14.653 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:14 smithi138 ceph-mon[104418]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:14.653 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:14 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:14.653 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:14 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:14.653 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:14 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:51:14.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:14 smithi110 ceph-mon[100243]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:15.258 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:14 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:15.258 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:14 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:15.258 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:14 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:51:15.258 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:14 smithi110 ceph-mon[118821]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:15.259 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:14 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:15.259 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:14 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:15.259 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:14 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:51:16.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:16 smithi138 ceph-mon[104418]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:16.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:16.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:16.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:16.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:16.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:16 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:16.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:16 smithi110 ceph-mon[100243]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:16.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:16.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:16.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:16.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:16.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:16 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:16 smithi110 ceph-mon[118821]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:16 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:16 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:16 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:16 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:16 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:17.090 INFO:teuthology.orchestra.run.smithi138.stdout:Created osd(s) 6 on host 'smithi138' 2023-09-23T22:51:17.669 DEBUG:teuthology.orchestra.run.smithi138:osd.6> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-23T22:51:17.672 INFO:tasks.cephadm:Deploying osd.7 on smithi138 with /dev/vg_nvme/lv_1... 2023-09-23T22:51:17.672 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T22:51:17.704 INFO:journalctl@ceph.osd.6.smithi138.stdout:-- Logs begin at Sat 2023-09-23 22:31:02 UTC. -- 2023-09-23T22:51:18.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:18 smithi138 ceph-mon[104418]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:18.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:18 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:18.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:18 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:18.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:18 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:51:18.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:18 smithi138 ceph-mon[104418]: from='osd.6 [v2:172.21.15.138:6816/187413403,v1:172.21.15.138:6817/187413403]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:51:18.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:18 smithi138 ceph-mon[104418]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:51:18.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:18 smithi110 ceph-mon[100243]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:18.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:18 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:18.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:18 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:18.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:18 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:51:18.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:18 smithi110 ceph-mon[100243]: from='osd.6 [v2:172.21.15.138:6816/187413403,v1:172.21.15.138:6817/187413403]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:51:18.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:18 smithi110 ceph-mon[100243]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:51:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:18 smithi110 ceph-mon[118821]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:18 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:18 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:18 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:51:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:18 smithi110 ceph-mon[118821]: from='osd.6 [v2:172.21.15.138:6816/187413403,v1:172.21.15.138:6817/187413403]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:51:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:18 smithi110 ceph-mon[118821]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:51:18.782 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:51:19.441 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 22:51:19 smithi138 conmon[131081]: 2023-09-23T22:51:19.092+0000 7fb610132700 -1 osd.6 0 waiting for initial osdmap 2023-09-23T22:51:19.441 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 22:51:19 smithi138 conmon[131081]: 2023-09-23T22:51:19.099+0000 7fb608f29700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:51:19.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:19 smithi138 ceph-mon[104418]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T22:51:19.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:19 smithi138 ceph-mon[104418]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T22:51:19.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:19 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:51:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:19 smithi138 ceph-mon[104418]: from='osd.6 [v2:172.21.15.138:6816/187413403,v1:172.21.15.138:6817/187413403]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:51:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:19 smithi138 ceph-mon[104418]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:51:19.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:19 smithi110 ceph-mon[100243]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T22:51:19.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:19 smithi110 ceph-mon[100243]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T22:51:19.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:19 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:51:19.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:19 smithi110 ceph-mon[100243]: from='osd.6 [v2:172.21.15.138:6816/187413403,v1:172.21.15.138:6817/187413403]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:51:19.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:19 smithi110 ceph-mon[100243]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:51:19.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:19 smithi110 ceph-mon[118821]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T22:51:19.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:19 smithi110 ceph-mon[118821]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T22:51:19.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:19 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:51:19.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:19 smithi110 ceph-mon[118821]: from='osd.6 [v2:172.21.15.138:6816/187413403,v1:172.21.15.138:6817/187413403]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:51:19.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:19 smithi110 ceph-mon[118821]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:51:20.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:20 smithi138 ceph-mon[104418]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:20.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:20 smithi138 ceph-mon[104418]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-09-23T22:51:20.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:20 smithi138 ceph-mon[104418]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T22:51:20.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:20 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:51:20.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:20 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:51:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:20 smithi138 ceph-mon[104418]: osd.6 [v2:172.21.15.138:6816/187413403,v1:172.21.15.138:6817/187413403] boot 2023-09-23T22:51:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:20 smithi138 ceph-mon[104418]: osdmap e46: 7 total, 7 up, 7 in 2023-09-23T22:51:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:20 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:51:20.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:20 smithi110 ceph-mon[100243]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:20.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:20 smithi110 ceph-mon[100243]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-09-23T22:51:20.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:20 smithi110 ceph-mon[100243]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T22:51:20.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:20 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:51:20.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:20 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:51:20.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:20 smithi110 ceph-mon[100243]: osd.6 [v2:172.21.15.138:6816/187413403,v1:172.21.15.138:6817/187413403] boot 2023-09-23T22:51:20.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:20 smithi110 ceph-mon[100243]: osdmap e46: 7 total, 7 up, 7 in 2023-09-23T22:51:20.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:20 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:51:20.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:20 smithi110 ceph-mon[118821]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:51:20.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:20 smithi110 ceph-mon[118821]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-09-23T22:51:20.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:20 smithi110 ceph-mon[118821]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T22:51:20.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:20 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:51:20.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:20 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:51:20.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:20 smithi110 ceph-mon[118821]: osd.6 [v2:172.21.15.138:6816/187413403,v1:172.21.15.138:6817/187413403] boot 2023-09-23T22:51:20.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:20 smithi110 ceph-mon[118821]: osdmap e46: 7 total, 7 up, 7 in 2023-09-23T22:51:20.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:20 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:51:20.694 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T22:51:20.694 INFO:teuthology.orchestra.run.smithi138.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T22:51:20.694 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10+0 records in 2023-09-23T22:51:20.695 INFO:teuthology.orchestra.run.smithi138.stderr:10+0 records out 2023-09-23T22:51:20.695 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.014744 s, 711 MB/s 2023-09-23T22:51:20.695 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping successful for: 2023-09-23T22:51:21.388 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:21 smithi138 ceph-mon[104418]: purged_snaps scrub starts 2023-09-23T22:51:21.389 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:21 smithi138 ceph-mon[104418]: purged_snaps scrub ok 2023-09-23T22:51:21.389 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:21 smithi138 ceph-mon[104418]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T22:51:21.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:21 smithi110 ceph-mon[100243]: purged_snaps scrub starts 2023-09-23T22:51:21.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:21 smithi110 ceph-mon[100243]: purged_snaps scrub ok 2023-09-23T22:51:21.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:21 smithi110 ceph-mon[100243]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T22:51:21.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:21 smithi110 ceph-mon[118821]: purged_snaps scrub starts 2023-09-23T22:51:21.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:21 smithi110 ceph-mon[118821]: purged_snaps scrub ok 2023-09-23T22:51:21.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:21 smithi110 ceph-mon[118821]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T22:51:21.761 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi138:vg_nvme/lv_1 2023-09-23T22:51:22.104 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:51:22.646 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:22 smithi138 ceph-mon[104418]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:51:22.646 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:22 smithi138 ceph-mon[104418]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T22:51:22.646 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:22 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:22.646 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:22 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:22.646 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:22 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:22.647 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:22 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:22.647 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:22 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:22.647 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:22 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:22.647 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:22 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:22.647 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:22 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:22.647 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:22 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:22.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[100243]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:51:22.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[100243]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T22:51:22.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:22.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:22.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:22.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:22.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:22.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:22.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:22.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:22.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:22.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[118821]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:51:22.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[118821]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T22:51:22.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:22 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:23.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:23 smithi138 ceph-mon[104418]: Detected new or changed devices on smithi138 2023-09-23T22:51:23.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:23 smithi138 ceph-mon[104418]: Adjusting osd_memory_target on smithi138 to 5722M 2023-09-23T22:51:23.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:23 smithi138 ceph-mon[104418]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T22:51:23.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:23 smithi110 ceph-mon[100243]: Detected new or changed devices on smithi138 2023-09-23T22:51:23.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:23 smithi110 ceph-mon[100243]: Adjusting osd_memory_target on smithi138 to 5722M 2023-09-23T22:51:23.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:23 smithi110 ceph-mon[100243]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T22:51:23.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:23 smithi110 ceph-mon[118821]: Detected new or changed devices on smithi138 2023-09-23T22:51:23.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:23 smithi110 ceph-mon[118821]: Adjusting osd_memory_target on smithi138 to 5722M 2023-09-23T22:51:23.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:23 smithi110 ceph-mon[118821]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T22:51:24.670 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:24 smithi138 ceph-mon[104418]: pgmap v151: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2023-09-23T22:51:24.671 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:51:24.671 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:51:24.671 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:24 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:24.671 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:24 smithi138 ceph-mon[104418]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:51:24.671 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:24 smithi138 ceph-mon[104418]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T22:51:24.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:24 smithi110 ceph-mon[118821]: pgmap v151: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2023-09-23T22:51:24.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:51:24.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:51:24.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:24 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:24.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:24 smithi110 ceph-mon[118821]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:51:24.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:24 smithi110 ceph-mon[118821]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T22:51:24.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:24 smithi110 ceph-mon[100243]: pgmap v151: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2023-09-23T22:51:24.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:51:24.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:51:24.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:24 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:24.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:24 smithi110 ceph-mon[100243]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:51:24.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:24 smithi110 ceph-mon[100243]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T22:51:25.389 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:25 smithi138 ceph-mon[104418]: from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:51:25.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:25 smithi110 ceph-mon[100243]: from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:51:25.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:25 smithi110 ceph-mon[118821]: from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:51:26.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:26 smithi138 ceph-mon[104418]: pgmap v153: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2023-09-23T22:51:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:26 smithi138 ceph-mon[104418]: from='client.? 172.21.15.138:0/2291524830' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed15f3b0-cd2e-43b7-b803-e45252b62a45"}]: dispatch 2023-09-23T22:51:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:26 smithi138 ceph-mon[104418]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed15f3b0-cd2e-43b7-b803-e45252b62a45"}]: dispatch 2023-09-23T22:51:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:26 smithi138 ceph-mon[104418]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ed15f3b0-cd2e-43b7-b803-e45252b62a45"}]': finished 2023-09-23T22:51:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:26 smithi138 ceph-mon[104418]: osdmap e51: 8 total, 7 up, 8 in 2023-09-23T22:51:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:26 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:26.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:26 smithi110 ceph-mon[100243]: pgmap v153: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2023-09-23T22:51:26.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:26 smithi110 ceph-mon[100243]: from='client.? 172.21.15.138:0/2291524830' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed15f3b0-cd2e-43b7-b803-e45252b62a45"}]: dispatch 2023-09-23T22:51:26.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:26 smithi110 ceph-mon[100243]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed15f3b0-cd2e-43b7-b803-e45252b62a45"}]: dispatch 2023-09-23T22:51:26.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:26 smithi110 ceph-mon[100243]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ed15f3b0-cd2e-43b7-b803-e45252b62a45"}]': finished 2023-09-23T22:51:26.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:26 smithi110 ceph-mon[100243]: osdmap e51: 8 total, 7 up, 8 in 2023-09-23T22:51:26.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:26 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:26.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:26 smithi110 ceph-mon[118821]: pgmap v153: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2023-09-23T22:51:26.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:26 smithi110 ceph-mon[118821]: from='client.? 172.21.15.138:0/2291524830' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed15f3b0-cd2e-43b7-b803-e45252b62a45"}]: dispatch 2023-09-23T22:51:26.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:26 smithi110 ceph-mon[118821]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed15f3b0-cd2e-43b7-b803-e45252b62a45"}]: dispatch 2023-09-23T22:51:26.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:26 smithi110 ceph-mon[118821]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ed15f3b0-cd2e-43b7-b803-e45252b62a45"}]': finished 2023-09-23T22:51:26.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:26 smithi110 ceph-mon[118821]: osdmap e51: 8 total, 7 up, 8 in 2023-09-23T22:51:26.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:26 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:27.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:27 smithi138 ceph-mon[104418]: from='client.? 172.21.15.138:0/199076814' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:51:27.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:27 smithi110 ceph-mon[100243]: from='client.? 172.21.15.138:0/199076814' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:51:27.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:27 smithi110 ceph-mon[118821]: from='client.? 172.21.15.138:0/199076814' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:51:28.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:28 smithi138 ceph-mon[104418]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 101 KiB/s, 0 objects/s recovering 2023-09-23T22:51:28.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:28 smithi138 ceph-mon[104418]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2023-09-23T22:51:28.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:28 smithi138 ceph-mon[104418]: Cluster is now healthy 2023-09-23T22:51:28.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:28 smithi110 ceph-mon[100243]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 101 KiB/s, 0 objects/s recovering 2023-09-23T22:51:28.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:28 smithi110 ceph-mon[100243]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2023-09-23T22:51:28.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:28 smithi110 ceph-mon[100243]: Cluster is now healthy 2023-09-23T22:51:28.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:28 smithi110 ceph-mon[118821]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 101 KiB/s, 0 objects/s recovering 2023-09-23T22:51:28.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:28 smithi110 ceph-mon[118821]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2023-09-23T22:51:28.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:28 smithi110 ceph-mon[118821]: Cluster is now healthy 2023-09-23T22:51:30.595 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:30 smithi138 ceph-mon[104418]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 94 KiB/s, 0 objects/s recovering 2023-09-23T22:51:30.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:30 smithi110 ceph-mon[100243]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 94 KiB/s, 0 objects/s recovering 2023-09-23T22:51:30.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:30 smithi110 ceph-mon[118821]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 94 KiB/s, 0 objects/s recovering 2023-09-23T22:51:32.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:32 smithi138 ceph-mon[104418]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:51:32.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:32 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T22:51:32.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:32 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:32.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:32 smithi110 ceph-mon[100243]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:51:32.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:32 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T22:51:32.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:32 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:32.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:32 smithi110 ceph-mon[118821]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:51:32.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:32 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T22:51:32.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:32 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:33.399 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:33 smithi138 ceph-mon[104418]: Deploying daemon osd.7 on smithi138 2023-09-23T22:51:33.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:33 smithi110 ceph-mon[100243]: Deploying daemon osd.7 on smithi138 2023-09-23T22:51:33.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:33 smithi110 ceph-mon[118821]: Deploying daemon osd.7 on smithi138 2023-09-23T22:51:34.657 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:34 smithi138 ceph-mon[104418]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-09-23T22:51:34.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:34 smithi110 ceph-mon[118821]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-09-23T22:51:34.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:34 smithi110 ceph-mon[100243]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-09-23T22:51:36.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:36 smithi138 ceph-mon[104418]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:51:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:36 smithi110 ceph-mon[100243]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:51:36.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:36 smithi110 ceph-mon[118821]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:51:38.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:38 smithi138 ceph-mon[104418]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:51:38.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:38 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:38.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:38 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:38.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:38 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:51:38.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:38 smithi110 ceph-mon[100243]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:51:38.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:38.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:38.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:38 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:51:38.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:38 smithi110 ceph-mon[118821]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:51:38.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:38 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:38.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:38 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:38.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:38 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:51:39.960 INFO:teuthology.orchestra.run.smithi138.stdout:Created osd(s) 7 on host 'smithi138' 2023-09-23T22:51:40.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:40 smithi138 ceph-mon[104418]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:51:40.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:40 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:40.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:40 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:40.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:40 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:40.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:40 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:40.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:40 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:40.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:40 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:40.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:40 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:40.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:40 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:51:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[100243]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:51:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:40.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:40.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:51:40.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[118821]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:51:40.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:40.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:40.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:40.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:40.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:40.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:40.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:40.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:40 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:51:40.541 DEBUG:teuthology.orchestra.run.smithi138:osd.7> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.7.service 2023-09-23T22:51:40.544 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-09-23T22:51:40.544 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T22:51:40.574 INFO:journalctl@ceph.osd.7.smithi138.stdout:-- Logs begin at Sat 2023-09-23 22:31:02 UTC. -- 2023-09-23T22:51:40.894 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:41.411 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:41 smithi110 ceph-mon[100243]: from='osd.7 [v2:172.21.15.138:6824/676244673,v1:172.21.15.138:6825/676244673]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:51:41.411 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:41 smithi110 ceph-mon[100243]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:51:41.411 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:41 smithi110 ceph-mon[118821]: from='osd.7 [v2:172.21.15.138:6824/676244673,v1:172.21.15.138:6825/676244673]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:51:41.412 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:41 smithi110 ceph-mon[118821]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:51:41.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:41 smithi138 ceph-mon[104418]: from='osd.7 [v2:172.21.15.138:6824/676244673,v1:172.21.15.138:6825/676244673]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:51:41.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:41 smithi138 ceph-mon[104418]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:51:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:42 smithi138 ceph-mon[104418]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:51:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:42 smithi138 ceph-mon[104418]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T22:51:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:42 smithi138 ceph-mon[104418]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T22:51:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:42 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:42.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:42 smithi138 ceph-mon[104418]: from='osd.7 [v2:172.21.15.138:6824/676244673,v1:172.21.15.138:6825/676244673]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:51:42.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:42 smithi138 ceph-mon[104418]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:51:42.442 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 22:51:42 smithi138 conmon[139964]: 2023-09-23T22:51:42.193+0000 7fb42c7b0700 -1 osd.7 0 waiting for initial osdmap 2023-09-23T22:51:42.442 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 22:51:42 smithi138 conmon[139964]: 2023-09-23T22:51:42.200+0000 7fb4255a7700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:51:42.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:42 smithi110 ceph-mon[118821]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:51:42.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:42 smithi110 ceph-mon[118821]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T22:51:42.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:42 smithi110 ceph-mon[118821]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T22:51:42.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:42 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:42.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:42 smithi110 ceph-mon[118821]: from='osd.7 [v2:172.21.15.138:6824/676244673,v1:172.21.15.138:6825/676244673]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:51:42.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:42 smithi110 ceph-mon[118821]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:51:42.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:42 smithi110 ceph-mon[100243]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:51:42.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:42 smithi110 ceph-mon[100243]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T22:51:42.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:42 smithi110 ceph-mon[100243]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T22:51:42.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:42 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:42.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:42 smithi110 ceph-mon[100243]: from='osd.7 [v2:172.21.15.138:6824/676244673,v1:172.21.15.138:6825/676244673]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:51:42.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:42 smithi110 ceph-mon[100243]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T22:51:42.565 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:51:43.342 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":53,"num_osds":8,"num_up_osds":7,"osd_up_since":1695509479,"num_in_osds":8,"osd_in_since":1695509486,"num_remapped_pgs":0} 2023-09-23T22:51:43.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:43 smithi110 ceph-mon[118821]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-09-23T22:51:43.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:43 smithi110 ceph-mon[118821]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T22:51:43.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:43 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:43.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:43 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:43 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/776643601' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:51:43.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:43 smithi138 ceph-mon[104418]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-09-23T22:51:43.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:43 smithi138 ceph-mon[104418]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T22:51:43.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:43 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:43.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:43 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:43.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:43 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/776643601' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:51:43.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:43 smithi110 ceph-mon[100243]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-09-23T22:51:43.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:43 smithi110 ceph-mon[100243]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T22:51:43.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:43 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:43.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:43 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:43.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:43 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/776643601' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:51:44.343 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T22:51:44.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: purged_snaps scrub starts 2023-09-23T22:51:44.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: purged_snaps scrub ok 2023-09-23T22:51:44.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:51:44.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: osd.7 [v2:172.21.15.138:6824/676244673,v1:172.21.15.138:6825/676244673] boot 2023-09-23T22:51:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: osdmap e54: 8 total, 8 up, 8 in 2023-09-23T22:51:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: Detected new or changed devices on smithi138 2023-09-23T22:51:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:44.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:44.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:44.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:44.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:44.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:44.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: purged_snaps scrub starts 2023-09-23T22:51:44.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: purged_snaps scrub ok 2023-09-23T22:51:44.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:51:44.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:44.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: osd.7 [v2:172.21.15.138:6824/676244673,v1:172.21.15.138:6825/676244673] boot 2023-09-23T22:51:44.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: osdmap e54: 8 total, 8 up, 8 in 2023-09-23T22:51:44.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:44.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: Detected new or changed devices on smithi138 2023-09-23T22:51:44.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:44.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:44.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:44.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:44.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:44.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:44.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:44.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:44.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:44.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:44 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: purged_snaps scrub starts 2023-09-23T22:51:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: purged_snaps scrub ok 2023-09-23T22:51:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:51:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:44.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: osd.7 [v2:172.21.15.138:6824/676244673,v1:172.21.15.138:6825/676244673] boot 2023-09-23T22:51:44.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: osdmap e54: 8 total, 8 up, 8 in 2023-09-23T22:51:44.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:51:44.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: Detected new or changed devices on smithi138 2023-09-23T22:51:44.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:44.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:44.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:44.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:44.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:44.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:51:44.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:44.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:44.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:44.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:44 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:44.696 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:45.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:45 smithi138 ceph-mon[104418]: Adjusting osd_memory_target on smithi138 to 4292M 2023-09-23T22:51:45.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:45 smithi138 ceph-mon[104418]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T22:51:45.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:45 smithi110 ceph-mon[118821]: Adjusting osd_memory_target on smithi138 to 4292M 2023-09-23T22:51:45.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:45 smithi110 ceph-mon[118821]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T22:51:45.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:45 smithi110 ceph-mon[100243]: Adjusting osd_memory_target on smithi138 to 4292M 2023-09-23T22:51:45.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:45 smithi110 ceph-mon[100243]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T22:51:46.382 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:51:46.409 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:46 smithi110 ceph-mon[100243]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:51:46.409 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:46 smithi110 ceph-mon[100243]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T22:51:46.409 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:46 smithi110 ceph-mon[118821]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:51:46.409 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:46 smithi110 ceph-mon[118821]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T22:51:46.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:46 smithi138 ceph-mon[104418]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:51:46.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:46 smithi138 ceph-mon[104418]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T22:51:47.108 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":57,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509503,"num_in_osds":8,"osd_in_since":1695509486,"num_remapped_pgs":0} 2023-09-23T22:51:47.108 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:51:47.447 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:47.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:47 smithi110 ceph-mon[100243]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T22:51:47.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:47 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3582895112' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:51:47.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:47 smithi110 ceph-mon[118821]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T22:51:47.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:47 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3582895112' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:51:47.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:47 smithi138 ceph-mon[104418]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T22:51:47.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:47 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3582895112' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:51:48.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:48 smithi110 ceph-mon[100243]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:51:48.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:48 smithi110 ceph-mon[100243]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T22:51:48.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:48 smithi110 ceph-mon[118821]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:51:48.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:48 smithi110 ceph-mon[118821]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T22:51:48.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:48 smithi138 ceph-mon[104418]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:51:48.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:48 smithi138 ceph-mon[104418]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T22:51:49.151 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:51:49.151 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":58,"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:46:49.032269+0000","modified":"2023-09-23T22:51:47.215226+0000","last_up_change":"2023-09-23T22:51:43.190723+0000","last_in_change":"2023-09-23T22:51:26.130429+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:49:49.263399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"78791acb-c93c-4e4f-bf04-ea3eee46408d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6803","nonce":2453789689}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6805","nonce":2453789689}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6809","nonce":2453789689}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6807","nonce":2453789689}]},"public_addr":"172.21.15.110:6803/2453789689","cluster_addr":"172.21.15.110:6805/2453789689","heartbeat_back_addr":"172.21.15.110:6809/2453789689","heartbeat_front_addr":"172.21.15.110:6807/2453789689","state":["exists","up"]},{"osd":1,"uuid":"6793178f-2ef1-4c48-927c-831a083a3356","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6811","nonce":3697882113}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6813","nonce":3697882113}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6817","nonce":3697882113}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6815","nonce":3697882113}]},"public_addr":"172.21.15.110:6811/3697882113","cluster_addr":"172.21.15.110:6813/3697882113","heartbeat_back_addr":"172.21.15.110:6817/3697882113","heartbeat_front_addr":"172.21.15.110:6815/3697882113","state":["exists","up"]},{"osd":2,"uuid":"146fbd28-a365-4545-8e97-d629c6ddba21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6819","nonce":2231416166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6821","nonce":2231416166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6825","nonce":2231416166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6823","nonce":2231416166}]},"public_addr":"172.21.15.110:6819/2231416166","cluster_addr":"172.21.15.110:6821/2231416166","heartbeat_back_addr":"172.21.15.110:6825/2231416166","heartbeat_front_addr":"172.21.15.110:6823/2231416166","state":["exists","up"]},{"osd":3,"uuid":"6fcf80d5-6977-4929-b7ba-0e9ab65df0a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6827","nonce":427497470}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6829","nonce":427497470}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6833","nonce":427497470}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6831","nonce":427497470}]},"public_addr":"172.21.15.110:6827/427497470","cluster_addr":"172.21.15.110:6829/427497470","heartbeat_back_addr":"172.21.15.110:6833/427497470","heartbeat_front_addr":"172.21.15.110:6831/427497470","state":["exists","up"]},{"osd":4,"uuid":"131d0749-c46c-44f0-ab7f-03f8ad6ed502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6801","nonce":941787557}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6803","nonce":941787557}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6807","nonce":941787557}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6805","nonce":941787557}]},"public_addr":"172.21.15.138:6801/941787557","cluster_addr":"172.21.15.138:6803/941787557","heartbeat_back_addr":"172.21.15.138:6807/941787557","heartbeat_front_addr":"172.21.15.138:6805/941787557","state":["exists","up"]},{"osd":5,"uuid":"8cb496e5-d242-4e41-b279-e8d5ecea2996","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6809","nonce":3098195887}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6811","nonce":3098195887}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6815","nonce":3098195887}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6813","nonce":3098195887}]},"public_addr":"172.21.15.138:6809/3098195887","cluster_addr":"172.21.15.138:6811/3098195887","heartbeat_back_addr":"172.21.15.138:6815/3098195887","heartbeat_front_addr":"172.21.15.138:6813/3098195887","state":["exists","up"]},{"osd":6,"uuid":"c709e629-85ab-4273-9ca8-f2c4e8fc9074","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6817","nonce":187413403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6819","nonce":187413403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6823","nonce":187413403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6821","nonce":187413403}]},"public_addr":"172.21.15.138:6817/187413403","cluster_addr":"172.21.15.138:6819/187413403","heartbeat_back_addr":"172.21.15.138:6823/187413403","heartbeat_front_addr":"172.21.15.138:6821/187413403","state":["exists","up"]},{"osd":7,"uuid":"ed15f3b0-cd2e-43b7-b803-e45252b62a45","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6825","nonce":676244673}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6827","nonce":676244673}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6831","nonce":676244673}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6829","nonce":676244673}]},"public_addr":"172.21.15.138:6825/676244673","cluster_addr":"172.21.15.138:6827/676244673","heartbeat_back_addr":"172.21.15.138:6831/676244673","heartbeat_front_addr":"172.21.15.138:6829/676244673","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:48:58.523930+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:22.993201+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:46.747264+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:11.206564+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:33.511236+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:56.364297+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:18.141750+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:41.242882+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:0/3412430555":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1649304640":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1326657364":"2023-09-24T22:47:38.935467+0000","172.21.15.110:6801/4228126008":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1731974664":"2023-09-24T22:47:15.528545+0000","172.21.15.110:6801/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/1506915348":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/3250066431":"2023-09-24T22:47:15.528545+0000","172.21.15.110:6800/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.110:6800/4228126008":"2023-09-24T22:47:38.935467+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:51:49.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:49 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2962832523' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:51:49.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:49 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2962832523' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:51:49.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:49 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2962832523' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:51:49.883 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-09-23T22:49:49.263399+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2023-09-23T22:51:49.883 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool get .mgr pg_num 2023-09-23T22:51:50.221 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:50.629 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:50 smithi110 ceph-mon[100243]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:51:50.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:50 smithi110 ceph-mon[118821]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:51:50.690 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:50 smithi138 ceph-mon[104418]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:51:52.001 INFO:teuthology.orchestra.run.smithi110.stdout:pg_num: 1 2023-09-23T22:51:52.296 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:52 smithi110 ceph-mon[118821]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T22:51:52.296 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:52 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/228524153' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T22:51:52.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:52 smithi138 ceph-mon[104418]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T22:51:52.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:52 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/228524153' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T22:51:52.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:52 smithi110 ceph-mon[100243]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T22:51:52.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:52 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/228524153' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T22:51:52.736 INFO:tasks.cephadm:Adding prometheus.a on smithi138 2023-09-23T22:51:52.736 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph orch apply prometheus '1;smithi138=a' 2023-09-23T22:51:53.085 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:51:54.417 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:54 smithi138 ceph-mon[104418]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2023-09-23T22:51:54.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:54 smithi110 ceph-mon[100243]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2023-09-23T22:51:54.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:54 smithi110 ceph-mon[118821]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2023-09-23T22:51:54.867 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled prometheus update... 2023-09-23T22:51:55.519 DEBUG:teuthology.orchestra.run.smithi138:prometheus.a> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@prometheus.a.service 2023-09-23T22:51:55.521 INFO:tasks.cephadm:Adding node-exporter.a on smithi110 2023-09-23T22:51:55.521 INFO:tasks.cephadm:Adding node-exporter.b on smithi138 2023-09-23T22:51:55.521 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi110=a;smithi138=b' 2023-09-23T22:51:55.551 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:-- Logs begin at Sat 2023-09-23 22:31:02 UTC. -- 2023-09-23T22:51:55.883 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:51:56.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:55 smithi138 ceph-mon[104418]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:51:56.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:55 smithi138 ceph-mon[104418]: Saving service prometheus spec with placement smithi138=a;count:1 2023-09-23T22:51:56.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:55 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:56.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:55 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:51:56.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:55 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:56.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:55 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:56.124 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:55 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:56.124 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:55 smithi138 ceph-mon[104418]: Deploying daemon prometheus.a on smithi138 2023-09-23T22:51:56.124 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:55 smithi138 ceph-mon[104418]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-09-23T22:51:56.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[100243]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:51:56.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[100243]: Saving service prometheus spec with placement smithi138=a;count:1 2023-09-23T22:51:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:51:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[100243]: Deploying daemon prometheus.a on smithi138 2023-09-23T22:51:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[100243]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-09-23T22:51:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[118821]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:51:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[118821]: Saving service prometheus spec with placement smithi138=a;count:1 2023-09-23T22:51:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:51:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[118821]: Deploying daemon prometheus.a on smithi138 2023-09-23T22:51:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:55 smithi110 ceph-mon[118821]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-09-23T22:51:57.790 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled node-exporter update... 2023-09-23T22:51:58.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:58 smithi138 ceph-mon[104418]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:51:58.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:58 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:58.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:58 smithi110 ceph-mon[100243]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:51:58.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:58 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:58.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:58 smithi110 ceph-mon[118821]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:51:58.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:58 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:51:58.755 DEBUG:teuthology.orchestra.run.smithi110:node-exporter.a> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@node-exporter.a.service 2023-09-23T22:51:58.758 DEBUG:teuthology.orchestra.run.smithi138:node-exporter.b> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@node-exporter.b.service 2023-09-23T22:51:58.760 INFO:tasks.cephadm:Adding alertmanager.a on smithi110 2023-09-23T22:51:58.760 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi110=a' 2023-09-23T22:51:58.789 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:-- Logs begin at Sat 2023-09-23 22:31:02 UTC. -- 2023-09-23T22:51:58.793 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:-- Logs begin at Sat 2023-09-23 22:30:57 UTC. -- 2023-09-23T22:51:59.046 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:59 smithi138 ceph-mon[104418]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi110=a;smithi138=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:51:59.046 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:51:59 smithi138 ceph-mon[104418]: Saving service node-exporter spec with placement smithi110=a;smithi138=b;count:2 2023-09-23T22:51:59.155 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:51:59.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:59 smithi110 ceph-mon[100243]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi110=a;smithi138=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:51:59.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:51:59 smithi110 ceph-mon[100243]: Saving service node-exporter spec with placement smithi110=a;smithi138=b;count:2 2023-09-23T22:51:59.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:59 smithi110 ceph-mon[118821]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi110=a;smithi138=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:51:59.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:51:59 smithi110 ceph-mon[118821]: Saving service node-exporter spec with placement smithi110=a;smithi138=b;count:2 2023-09-23T22:52:00.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:00 smithi138 ceph-mon[104418]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2023-09-23T22:52:00.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:00 smithi110 ceph-mon[100243]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2023-09-23T22:52:00.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:00 smithi110 ceph-mon[118821]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2023-09-23T22:52:02.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:02 smithi138 ceph-mon[104418]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T22:52:02.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:02 smithi110 ceph-mon[100243]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T22:52:02.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:02 smithi110 ceph-mon[118821]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T22:52:04.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:04 smithi138 ceph-mon[104418]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:04.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:04 smithi110 ceph-mon[100243]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:04.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:04 smithi110 ceph-mon[118821]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:05.025 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled alertmanager update... 2023-09-23T22:52:06.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:06 smithi138 ceph-mon[104418]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:06.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:06 smithi138 ceph-mon[104418]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi110=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:52:06.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:06 smithi138 ceph-mon[104418]: Saving service alertmanager spec with placement smithi110=a;count:1 2023-09-23T22:52:06.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:06 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:52:06.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:06 smithi110 ceph-mon[118821]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:06.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:06 smithi110 ceph-mon[118821]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi110=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:52:06.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:06 smithi110 ceph-mon[118821]: Saving service alertmanager spec with placement smithi110=a;count:1 2023-09-23T22:52:06.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:06 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:52:06.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:06 smithi110 ceph-mon[100243]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:06.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:06 smithi110 ceph-mon[100243]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi110=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:52:06.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:06 smithi110 ceph-mon[100243]: Saving service alertmanager spec with placement smithi110=a;count:1 2023-09-23T22:52:06.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:06 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:52:07.424 DEBUG:teuthology.orchestra.run.smithi110:alertmanager.a> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@alertmanager.a.service 2023-09-23T22:52:07.427 INFO:tasks.cephadm:Adding grafana.a on smithi138 2023-09-23T22:52:07.428 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph orch apply grafana '1;smithi138=a' 2023-09-23T22:52:07.462 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:-- Logs begin at Sat 2023-09-23 22:30:57 UTC. -- 2023-09-23T22:52:08.027 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:08 smithi138 ceph-mon[104418]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:08.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:08 smithi110 ceph-mon[100243]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:08.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:08 smithi110 ceph-mon[118821]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:08.568 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:52:08.829 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:08 smithi138 systemd[1]: Starting Ceph prometheus.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:52:10.183 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:09 smithi138 conmon[148324]: ts=2023-09-23T22:52:09.995Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T22:52:10.184 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:09 smithi138 conmon[148324]: ts=2023-09-23T22:52:09.996Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T22:52:10.184 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:09 smithi138 conmon[148324]: ts=2023-09-23T22:52:09.996Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi138 (none))" 2023-09-23T22:52:10.184 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:09 smithi138 conmon[148324]: ts=2023-09-23T22:52:09.996Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T22:52:10.184 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:09 smithi138 conmon[148324]: ts=2023-09-23T22:52:09.996Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T22:52:10.184 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:09 smithi138 conmon[148324]: ts=2023-09-23T22:52:09.997Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T22:52:10.184 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:09 smithi138 conmon[148324]: ts=2023-09-23T22:52:09.997Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T22:52:10.184 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:09 smithi138 conmon[148324]: ts=2023-09-23T22:52:09.998Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T22:52:10.184 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:09 smithi138 conmon[148324]: ts=2023-09-23T22:52:09.998Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T22:52:10.184 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:10 smithi138 conmon[148324]: ts=2023-09-23T22:52:10.000Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T22:52:10.184 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:10 smithi138 conmon[148324]: ts=2023-09-23T22:52:10.000Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.799µs 2023-09-23T22:52:10.184 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:10 smithi138 conmon[148324]: ts=2023-09-23T22:52:10.000Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T22:52:10.185 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:10 smithi138 conmon[148324]: ts=2023-09-23T22:52:10.001Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-09-23T22:52:10.185 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:10 smithi138 conmon[148324]: ts=2023-09-23T22:52:10.001Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.641µs wal_replay_duration=170.222µs wbl_replay_duration=106ns total_replay_duration=206.907µs 2023-09-23T22:52:10.185 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:10 smithi138 conmon[148324]: ts=2023-09-23T22:52:10.002Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T22:52:10.185 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:10 smithi138 conmon[148324]: ts=2023-09-23T22:52:10.002Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T22:52:10.185 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:10 smithi138 conmon[148324]: ts=2023-09-23T22:52:10.002Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T22:52:10.185 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:10 smithi138 bash[148239]: 8abf0df36f5d39b5b94930a0bfaf093badca5f31cffa56d8665bb02b97187cc6 2023-09-23T22:52:10.185 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:10 smithi138 systemd[1]: Started Ceph prometheus.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:52:10.185 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:10 smithi138 conmon[148324]: ts=2023-09-23T22:52:10.020Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.984524ms db_storage=781ns remote_storage=1.338µs web_handler=330ns query_engine=686ns scrape=451.642µs scrape_sd=17.54µs notify=872ns notify_sd=1.997µs rules=17.185507ms tracing=7.7µs 2023-09-23T22:52:10.185 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:10 smithi138 conmon[148324]: ts=2023-09-23T22:52:10.020Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T22:52:10.185 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:52:10 smithi138 conmon[148324]: ts=2023-09-23T22:52:10.020Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T22:52:10.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:10 smithi138 ceph-mon[104418]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:10.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:10 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:52:10.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:10 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:52:10.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:10 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:52:10.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:10 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T22:52:10.585 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled grafana update... 2023-09-23T22:52:10.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:10 smithi110 ceph-mon[100243]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:10.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:10 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:52:10.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:10 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:52:10.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:10 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:52:10.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:10 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T22:52:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:10 smithi110 ceph-mon[118821]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:10 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:52:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:10 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:52:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:10 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:52:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:10 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T22:52:11.272 DEBUG:teuthology.orchestra.run.smithi138:grafana.a> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@grafana.a.service 2023-09-23T22:52:11.275 INFO:tasks.cephadm:Setting up client nodes... 2023-09-23T22:52:11.275 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T22:52:11.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:-- Logs begin at Sat 2023-09-23 22:31:02 UTC. -- 2023-09-23T22:52:11.434 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:11 smithi138 conmon[107593]: ignoring --setuser ceph since I am not root 2023-09-23T22:52:11.434 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:11 smithi138 conmon[107593]: ignoring --setgroup ceph since I am not root 2023-09-23T22:52:11.434 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:11 smithi138 ceph-mgr[107617]: -- 172.21.15.138:0/3454670605 <== mon.1 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5573a4dc09c0 con 0x5573a4d99000 2023-09-23T22:52:11.434 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:11 smithi138 conmon[107593]: 2023-09-23T22:52:11.261+0000 7f5820d671c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:52:11.434 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:11 smithi138 conmon[107593]: 2023-09-23T22:52:11.346+0000 7f5820d671c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:52:11.434 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:11 smithi138 conmon[107593]: 2023-09-23T22:52:11.432+0000 7f5820d671c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:52:11.437 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:11 smithi110 conmon[100666]: ignoring --setuser ceph since I am not root 2023-09-23T22:52:11.437 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:11 smithi110 conmon[100666]: ignoring --setgroup ceph since I am not root 2023-09-23T22:52:11.437 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:11 smithi110 ceph-mgr[100691]: -- 172.21.15.110:0/4075840424 <== mon.2 v2:172.21.15.110:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559a81c609c0 con 0x559a81c38c00 2023-09-23T22:52:11.438 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:11 smithi110 conmon[100666]: 2023-09-23T22:52:11.260+0000 7f05ef8781c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:52:11.438 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:11 smithi110 conmon[100666]: 2023-09-23T22:52:11.345+0000 7f05ef8781c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:52:11.626 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:11.691 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:11 smithi138 conmon[107593]: 2023-09-23T22:52:11.524+0000 7f5820d671c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:52:11.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:11 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:52:11.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:11 smithi138 ceph-mon[104418]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T22:52:11.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:11 smithi138 ceph-mon[104418]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T22:52:11.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:11 smithi110 conmon[100666]: 2023-09-23T22:52:11.436+0000 7f05ef8781c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:52:11.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:11 smithi110 conmon[100666]: 2023-09-23T22:52:11.538+0000 7f05ef8781c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:52:11.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:11 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:52:11.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:11 smithi110 ceph-mon[100243]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T22:52:11.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:11 smithi110 ceph-mon[100243]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T22:52:11.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:11 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' 2023-09-23T22:52:11.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:11 smithi110 ceph-mon[118821]: from='mgr.14152 172.21.15.110:0/3198264640' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T22:52:11.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:11 smithi110 ceph-mon[118821]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T22:52:11.993 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:11 smithi138 conmon[107593]: 2023-09-23T22:52:11.690+0000 7f5820d671c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:52:12.057 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:11 smithi110 conmon[100666]: 2023-09-23T22:52:11.732+0000 7f05ef8781c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:52:12.298 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:11 smithi138 conmon[107593]: 2023-09-23T22:52:11.992+0000 7f5820d671c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:52:12.298 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:12 smithi138 conmon[107593]: 2023-09-23T22:52:12.078+0000 7f5820d671c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:52:12.298 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:12 smithi138 conmon[107593]: 2023-09-23T22:52:12.205+0000 7f5820d671c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:52:12.375 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:12 smithi110 conmon[100666]: 2023-09-23T22:52:12.056+0000 7f05ef8781c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:52:12.375 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:12 smithi110 conmon[100666]: 2023-09-23T22:52:12.148+0000 7f05ef8781c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:52:12.375 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:12 smithi110 conmon[100666]: 2023-09-23T22:52:12.281+0000 7f05ef8781c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:52:12.691 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:12 smithi138 conmon[107593]: 2023-09-23T22:52:12.297+0000 7f5820d671c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:52:12.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:12 smithi110 conmon[100666]: 2023-09-23T22:52:12.373+0000 7f05ef8781c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:52:13.058 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:12 smithi110 conmon[100666]: 2023-09-23T22:52:12.797+0000 7f05ef8781c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:52:13.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:12 smithi138 conmon[107593]: 2023-09-23T22:52:12.711+0000 7f5820d671c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:52:13.578 INFO:teuthology.orchestra.run.smithi110.stdout:[client.0] 2023-09-23T22:52:13.578 INFO:teuthology.orchestra.run.smithi110.stdout: key = AQAdbA9lhh4+IhAAFDswnaBg7gKYftvPJkhg+w== 2023-09-23T22:52:13.628 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:13 smithi138 conmon[107593]: 2023-09-23T22:52:13.266+0000 7f5820d671c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:52:13.628 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:13 smithi138 conmon[107593]: 2023-09-23T22:52:13.435+0000 7f5820d671c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:52:13.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:13 smithi110 conmon[100666]: 2023-09-23T22:52:13.402+0000 7f05ef8781c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:52:13.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:13 smithi110 conmon[100666]: 2023-09-23T22:52:13.574+0000 7f05ef8781c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:52:13.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:13 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/4037708391' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:52:13.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:13 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/4037708391' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:52:13.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:13 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/4037708391' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:52:13.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:13 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/4037708391' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:52:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:13 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/4037708391' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:52:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:13 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/4037708391' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:52:14.353 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-09-23T22:52:14.353 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-09-23T22:52:14.353 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-09-23T22:52:14.394 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T22:52:14.456 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:14 smithi138 conmon[107593]: 2023-09-23T22:52:14.193+0000 7f5820d671c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:52:14.456 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:14 smithi138 conmon[107593]: 2023-09-23T22:52:14.282+0000 7f5820d671c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:52:14.630 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:14 smithi110 conmon[100666]: 2023-09-23T22:52:14.367+0000 7f05ef8781c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:52:14.630 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:14 smithi110 conmon[100666]: 2023-09-23T22:52:14.461+0000 7f05ef8781c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:52:14.630 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:14 smithi110 conmon[100666]: 2023-09-23T22:52:14.628+0000 7f05ef8781c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:52:14.751 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:52:15.147 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:14 smithi138 conmon[107593]: 2023-09-23T22:52:14.455+0000 7f5820d671c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:52:15.147 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:14 smithi138 conmon[107593]: 2023-09-23T22:52:14.763+0000 7f5820d671c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:52:15.147 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:14 smithi138 conmon[107593]: 2023-09-23T22:52:14.858+0000 7f5820d671c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:52:15.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:14 smithi110 conmon[100666]: 2023-09-23T22:52:14.901+0000 7f05ef8781c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:52:15.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:14 smithi110 conmon[100666]: 2023-09-23T22:52:14.988+0000 7f05ef8781c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:52:15.439 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:15 smithi138 conmon[107593]: 2023-09-23T22:52:15.163+0000 7f5820d671c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:52:15.530 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:15 smithi110 conmon[100666]: 2023-09-23T22:52:15.272+0000 7f05ef8781c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:52:15.691 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:15 smithi138 conmon[107593]: 2023-09-23T22:52:15.436+0000 7f5820d671c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:52:15.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:15 smithi110 conmon[100666]: 2023-09-23T22:52:15.529+0000 7f05ef8781c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:52:16.317 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:16 smithi138 conmon[107593]: 2023-09-23T22:52:16.182+0000 7f5820d671c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:52:16.318 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:16 smithi138 conmon[107593]: 2023-09-23T22:52:16.316+0000 7f5820d671c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:52:16.447 INFO:teuthology.orchestra.run.smithi138.stdout:[client.1] 2023-09-23T22:52:16.447 INFO:teuthology.orchestra.run.smithi138.stdout: key = AQAgbA9lLY10GhAAZASjxGP9/oLo7XDKg3+apQ== 2023-09-23T22:52:16.494 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:16 smithi110 conmon[100666]: 2023-09-23T22:52:16.226+0000 7f05ef8781c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:52:16.494 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:16 smithi110 conmon[100666]: 2023-09-23T22:52:16.355+0000 7f05ef8781c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:52:16.639 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:16 smithi138 conmon[107593]: 2023-09-23T22:52:16.511+0000 7f5820d671c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:52:16.639 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:16 smithi138 ceph-mon[104418]: from='client.? 172.21.15.138:0/2497808114' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:52:16.639 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:16 smithi138 ceph-mon[104418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:52:16.639 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:16 smithi138 ceph-mon[104418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:52:16.872 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:16 smithi110 conmon[100666]: 2023-09-23T22:52:16.544+0000 7f05ef8781c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:52:16.872 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:16 smithi110 conmon[100666]: 2023-09-23T22:52:16.657+0000 7f05ef8781c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:52:16.873 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:16 smithi110 ceph-mon[100243]: from='client.? 172.21.15.138:0/2497808114' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:52:16.873 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:16 smithi110 ceph-mon[100243]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:52:16.873 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:16 smithi110 ceph-mon[100243]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:52:16.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:16 smithi110 ceph-mon[118821]: from='client.? 172.21.15.138:0/2497808114' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:52:16.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:16 smithi110 ceph-mon[118821]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:52:16.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:16 smithi110 ceph-mon[118821]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:52:16.940 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:16 smithi138 conmon[107593]: 2023-09-23T22:52:16.637+0000 7f5820d671c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:52:16.941 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:16 smithi138 conmon[107593]: 2023-09-23T22:52:16.861+0000 7f5820d671c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:52:17.078 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-09-23T22:52:17.078 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-09-23T22:52:17.078 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-09-23T22:52:17.119 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-09-23T22:52:17.119 DEBUG:teuthology.orchestra.run.smithi110:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-23T22:52:17.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:16 smithi110 conmon[100666]: 2023-09-23T22:52:16.870+0000 7f05ef8781c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:52:17.445 INFO:teuthology.misc.health.smithi110.stdout: 2023-09-23T22:52:17.445 INFO:teuthology.misc.health.smithi110.stdout:{"epoch":58,"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:46:49.032269+0000","modified":"2023-09-23T22:51:47.215226+0000","last_up_change":"2023-09-23T22:51:43.190723+0000","last_in_change":"2023-09-23T22:51:26.130429+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:49:49.263399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"78791acb-c93c-4e4f-bf04-ea3eee46408d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6803","nonce":2453789689}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6805","nonce":2453789689}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6809","nonce":2453789689}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6807","nonce":2453789689}]},"public_addr":"172.21.15.110:6803/2453789689","cluster_addr":"172.21.15.110:6805/2453789689","heartbeat_back_addr":"172.21.15.110:6809/2453789689","heartbeat_front_addr":"172.21.15.110:6807/2453789689","state":["exists","up"]},{"osd":1,"uuid":"6793178f-2ef1-4c48-927c-831a083a3356","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6811","nonce":3697882113}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6813","nonce":3697882113}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6817","nonce":3697882113}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6815","nonce":3697882113}]},"public_addr":"172.21.15.110:6811/3697882113","cluster_addr":"172.21.15.110:6813/3697882113","heartbeat_back_addr":"172.21.15.110:6817/3697882113","heartbeat_front_addr":"172.21.15.110:6815/3697882113","state":["exists","up"]},{"osd":2,"uuid":"146fbd28-a365-4545-8e97-d629c6ddba21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6819","nonce":2231416166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6821","nonce":2231416166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6825","nonce":2231416166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6823","nonce":2231416166}]},"public_addr":"172.21.15.110:6819/2231416166","cluster_addr":"172.21.15.110:6821/2231416166","heartbeat_back_addr":"172.21.15.110:6825/2231416166","heartbeat_front_addr":"172.21.15.110:6823/2231416166","state":["exists","up"]},{"osd":3,"uuid":"6fcf80d5-6977-4929-b7ba-0e9ab65df0a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6827","nonce":427497470}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6829","nonce":427497470}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6833","nonce":427497470}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6831","nonce":427497470}]},"public_addr":"172.21.15.110:6827/427497470","cluster_addr":"172.21.15.110:6829/427497470","heartbeat_back_addr":"172.21.15.110:6833/427497470","heartbeat_front_addr":"172.21.15.110:6831/427497470","state":["exists","up"]},{"osd":4,"uuid":"131d0749-c46c-44f0-ab7f-03f8ad6ed502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6801","nonce":941787557}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6803","nonce":941787557}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6807","nonce":941787557}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6805","nonce":941787557}]},"public_addr":"172.21.15.138:6801/941787557","cluster_addr":"172.21.15.138:6803/941787557","heartbeat_back_addr":"172.21.15.138:6807/941787557","heartbeat_front_addr":"172.21.15.138:6805/941787557","state":["exists","up"]},{"osd":5,"uuid":"8cb496e5-d242-4e41-b279-e8d5ecea2996","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6809","nonce":3098195887}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6811","nonce":3098195887}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6815","nonce":3098195887}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6813","nonce":3098195887}]},"public_addr":"172.21.15.138:6809/3098195887","cluster_addr":"172.21.15.138:6811/3098195887","heartbeat_back_addr":"172.21.15.138:6815/3098195887","heartbeat_front_addr":"172.21.15.138:6813/3098195887","state":["exists","up"]},{"osd":6,"uuid":"c709e629-85ab-4273-9ca8-f2c4e8fc9074","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6817","nonce":187413403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6819","nonce":187413403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6823","nonce":187413403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6821","nonce":187413403}]},"public_addr":"172.21.15.138:6817/187413403","cluster_addr":"172.21.15.138:6819/187413403","heartbeat_back_addr":"172.21.15.138:6823/187413403","heartbeat_front_addr":"172.21.15.138:6821/187413403","state":["exists","up"]},{"osd":7,"uuid":"ed15f3b0-cd2e-43b7-b803-e45252b62a45","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6825","nonce":676244673}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6827","nonce":676244673}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6831","nonce":676244673}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6829","nonce":676244673}]},"public_addr":"172.21.15.138:6825/676244673","cluster_addr":"172.21.15.138:6827/676244673","heartbeat_back_addr":"172.21.15.138:6831/676244673","heartbeat_front_addr":"172.21.15.138:6829/676244673","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:48:58.523930+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:22.993201+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:46.747264+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:11.206564+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:33.511236+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:56.364297+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:18.141750+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:41.242882+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:0/3412430555":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1649304640":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1326657364":"2023-09-24T22:47:38.935467+0000","172.21.15.110:6801/4228126008":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1731974664":"2023-09-24T22:47:15.528545+0000","172.21.15.110:6801/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/1506915348":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/3250066431":"2023-09-24T22:47:15.528545+0000","172.21.15.110:6800/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.110:6800/4228126008":"2023-09-24T22:47:38.935467+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:52:17.460 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-09-23T22:52:17.460 INFO:tasks.cephadm:Creating RBD pool 2023-09-23T22:52:17.460 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-09-23T22:52:17.762 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:17 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/4170539936' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:52:17.762 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:17 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/4170539936' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:52:17.810 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:17.844 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:17 smithi138 conmon[107593]: 2023-09-23T22:52:17.723+0000 7f5820d671c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:52:17.844 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:17 smithi138 conmon[107593]: [23/Sep/2023:22:52:17] ENGINE Bus STARTING 2023-09-23T22:52:17.844 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:17 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/4170539936' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:52:18.044 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:17 smithi110 conmon[100666]: 2023-09-23T22:52:17.760+0000 7f05ef8781c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:52:18.044 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:17 smithi110 conmon[100666]: [23/Sep/2023:22:52:17] ENGINE Bus STARTING 2023-09-23T22:52:18.044 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:18 smithi110 conmon[100666]: CherryPy Checker: 2023-09-23T22:52:18.044 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:18 smithi110 conmon[100666]: The Application mounted at '' has an empty config. 2023-09-23T22:52:18.044 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:18 smithi110 conmon[100666]: 2023-09-23T22:52:18.044 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:18 smithi110 conmon[100666]: [23/Sep/2023:22:52:18] ENGINE Serving on http://:::9283 2023-09-23T22:52:18.044 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:18 smithi110 conmon[100666]: [23/Sep/2023:22:52:18] ENGINE Bus STARTED 2023-09-23T22:52:18.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:17 smithi138 conmon[107593]: CherryPy Checker: 2023-09-23T22:52:18.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:17 smithi138 conmon[107593]: The Application mounted at '' has an empty config. 2023-09-23T22:52:18.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:17 smithi138 conmon[107593]: 2023-09-23T22:52:18.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:17 smithi138 conmon[107593]: [23/Sep/2023:22:52:17] ENGINE Serving on http://:::9283 2023-09-23T22:52:18.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:17 smithi138 conmon[107593]: [23/Sep/2023:22:52:17] ENGINE Bus STARTED 2023-09-23T22:52:18.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: Standby manager daemon x restarted 2023-09-23T22:52:18.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: Standby manager daemon x started 2023-09-23T22:52:18.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/600217638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:52:18.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/600217638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:52:18.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/600217638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:52:18.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/600217638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:52:18.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: Active manager daemon y restarted 2023-09-23T22:52:18.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: Activating manager daemon y 2023-09-23T22:52:18.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T22:52:18.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: mgrmap e17: y(active, starting, since 0.00733499s), standbys: x 2023-09-23T22:52:18.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:52:18.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:52:18.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:52:18.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:52:18.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:52:18.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:52:18.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:52:18.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:52:18.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:52:18.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:52:18.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:52:18.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:52:18.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:52:18.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:52:18.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:52:18.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:52:18.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: Manager daemon y is now available 2023-09-23T22:52:18.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:52:18.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:52:18.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:18.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:18.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:52:18.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:52:18.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:18.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: Standby manager daemon x restarted 2023-09-23T22:52:18.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: Standby manager daemon x started 2023-09-23T22:52:18.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/600217638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:52:18.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/600217638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:52:18.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/600217638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:52:18.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/600217638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:52:18.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: Active manager daemon y restarted 2023-09-23T22:52:18.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: Activating manager daemon y 2023-09-23T22:52:18.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T22:52:18.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: mgrmap e17: y(active, starting, since 0.00733499s), standbys: x 2023-09-23T22:52:18.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:52:18.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:52:18.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:52:18.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:52:18.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:52:18.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:52:18.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:52:18.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:52:18.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:52:18.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:52:18.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:52:18.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:52:18.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:52:18.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:52:18.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:52:18.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:52:18.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: Manager daemon y is now available 2023-09-23T22:52:18.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:52:18.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:52:18.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:18.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:18.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:52:18.709 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:52:18.709 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:18.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: Standby manager daemon x restarted 2023-09-23T22:52:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: Standby manager daemon x started 2023-09-23T22:52:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/600217638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:52:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/600217638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:52:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/600217638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:52:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/600217638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:52:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: Active manager daemon y restarted 2023-09-23T22:52:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: Activating manager daemon y 2023-09-23T22:52:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T22:52:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: mgrmap e17: y(active, starting, since 0.00733499s), standbys: x 2023-09-23T22:52:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:52:18.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:52:18.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:52:18.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:52:18.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:52:18.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:52:18.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:52:18.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:52:18.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:52:18.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:52:18.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:52:18.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:52:18.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:52:18.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:52:18.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:52:18.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:52:18.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: Manager daemon y is now available 2023-09-23T22:52:18.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:52:18.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:52:18.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:18.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:18.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:52:18.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:52:18.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:20.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:19 smithi138 ceph-mon[104418]: [23/Sep/2023:22:52:18] ENGINE Bus STARTING 2023-09-23T22:52:20.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:19 smithi138 ceph-mon[104418]: [23/Sep/2023:22:52:18] ENGINE Serving on https://172.21.15.110:7150 2023-09-23T22:52:20.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:19 smithi138 ceph-mon[104418]: [23/Sep/2023:22:52:18] ENGINE Bus STARTED 2023-09-23T22:52:20.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:19 smithi138 ceph-mon[104418]: mgrmap e18: y(active, since 1.01007s), standbys: x 2023-09-23T22:52:20.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:19 smithi138 ceph-mon[104418]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:20.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:19 smithi110 ceph-mon[100243]: [23/Sep/2023:22:52:18] ENGINE Bus STARTING 2023-09-23T22:52:20.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:19 smithi110 ceph-mon[100243]: [23/Sep/2023:22:52:18] ENGINE Serving on https://172.21.15.110:7150 2023-09-23T22:52:20.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:19 smithi110 ceph-mon[100243]: [23/Sep/2023:22:52:18] ENGINE Bus STARTED 2023-09-23T22:52:20.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:19 smithi110 ceph-mon[100243]: mgrmap e18: y(active, since 1.01007s), standbys: x 2023-09-23T22:52:20.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:19 smithi110 ceph-mon[100243]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:20.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:19 smithi110 ceph-mon[118821]: [23/Sep/2023:22:52:18] ENGINE Bus STARTING 2023-09-23T22:52:20.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:19 smithi110 ceph-mon[118821]: [23/Sep/2023:22:52:18] ENGINE Serving on https://172.21.15.110:7150 2023-09-23T22:52:20.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:19 smithi110 ceph-mon[118821]: [23/Sep/2023:22:52:18] ENGINE Bus STARTED 2023-09-23T22:52:20.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:19 smithi110 ceph-mon[118821]: mgrmap e18: y(active, since 1.01007s), standbys: x 2023-09-23T22:52:20.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:19 smithi110 ceph-mon[118821]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:20.313 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'rbd' created 2023-09-23T22:52:21.072 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-09-23T22:52:21.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:20 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2808223735' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:52:21.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:20 smithi138 ceph-mon[104418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:52:21.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:20 smithi138 ceph-mon[104418]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:21.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:20 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:20 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:20 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:20 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:20 smithi138 ceph-mon[104418]: mgrmap e19: y(active, since 2s), standbys: x 2023-09-23T22:52:21.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:20 smithi138 ceph-mon[104418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T22:52:21.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:20 smithi138 ceph-mon[104418]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T22:52:21.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:20 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:20 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:20 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:52:21.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:20 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:52:21.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:20 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:52:21.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:20 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:52:21.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:20 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2808223735' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:52:21.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[118821]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:52:21.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[118821]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:21.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[118821]: mgrmap e19: y(active, since 2s), standbys: x 2023-09-23T22:52:21.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[118821]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T22:52:21.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[118821]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T22:52:21.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:52:21.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:52:21.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:52:21.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:52:21.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2808223735' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:52:21.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[100243]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:52:21.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[100243]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:21.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[100243]: mgrmap e19: y(active, since 2s), standbys: x 2023-09-23T22:52:21.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[100243]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T22:52:21.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[100243]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T22:52:21.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:52:21.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:52:21.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:52:21.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:52:21.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:20 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:21.442 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:22.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:21 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:52:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:52:22.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:21 smithi138 ceph-mon[104418]: Adjusting osd_memory_target on smithi138 to 4036M 2023-09-23T22:52:22.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:21 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:22.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:21 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:22.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:21 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:52:22.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:21 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:52:22.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:21 smithi138 ceph-mon[104418]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T22:52:22.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:21 smithi110 ceph-mon[118821]: Adjusting osd_memory_target on smithi138 to 4036M 2023-09-23T22:52:22.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:21 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:22.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:21 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:22.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:21 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:52:22.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:21 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:52:22.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:21 smithi110 ceph-mon[118821]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T22:52:22.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:21 smithi110 ceph-mon[100243]: Adjusting osd_memory_target on smithi138 to 4036M 2023-09-23T22:52:22.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:21 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:22.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:21 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:22.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:21 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:52:22.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:21 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:52:22.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:21 smithi110 ceph-mon[100243]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T22:52:23.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:22 smithi138 ceph-mon[104418]: Updating smithi110:/etc/ceph/ceph.conf 2023-09-23T22:52:23.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:22 smithi138 ceph-mon[104418]: Updating smithi138:/etc/ceph/ceph.conf 2023-09-23T22:52:23.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:22 smithi138 ceph-mon[104418]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:52:23.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:22 smithi138 ceph-mon[104418]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:52:23.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:22 smithi138 ceph-mon[104418]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:52:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:22 smithi138 ceph-mon[104418]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:22 smithi138 ceph-mon[104418]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:52:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:22 smithi138 ceph-mon[104418]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:52:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:22 smithi138 ceph-mon[104418]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:52:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:22 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:22 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:22 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:22 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:22 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:22 smithi138 ceph-mon[104418]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T22:52:23.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[100243]: Updating smithi110:/etc/ceph/ceph.conf 2023-09-23T22:52:23.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[100243]: Updating smithi138:/etc/ceph/ceph.conf 2023-09-23T22:52:23.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[100243]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:52:23.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[100243]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:52:23.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[100243]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:52:23.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[100243]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:23.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[100243]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:52:23.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[100243]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:52:23.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[100243]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:52:23.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:23.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:23.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:23.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:23.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:23.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[100243]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T22:52:23.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[118821]: Updating smithi110:/etc/ceph/ceph.conf 2023-09-23T22:52:23.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[118821]: Updating smithi138:/etc/ceph/ceph.conf 2023-09-23T22:52:23.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[118821]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:52:23.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[118821]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:52:23.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[118821]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:52:23.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[118821]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:23.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[118821]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:52:23.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[118821]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:52:23.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[118821]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:52:23.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:23.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:23.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:23.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:23.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:23.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:22 smithi110 ceph-mon[118821]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T22:52:23.927 INFO:teuthology.orchestra.run.smithi110.stderr:enabled application 'rbd' on pool 'rbd' 2023-09-23T22:52:24.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:23 smithi138 ceph-mon[104418]: Deploying daemon alertmanager.a on smithi110 2023-09-23T22:52:24.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:23 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2805366496' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:52:24.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:23 smithi110 ceph-mon[100243]: Deploying daemon alertmanager.a on smithi110 2023-09-23T22:52:24.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:23 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2805366496' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:52:24.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:23 smithi110 ceph-mon[118821]: Deploying daemon alertmanager.a on smithi110 2023-09-23T22:52:24.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:23 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2805366496' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:52:24.887 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T22:52:24.887 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T22:52:24.888 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T22:52:25.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:24 smithi138 ceph-mon[104418]: pgmap v9: 9 pgs: 5 active+clean, 4 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:25.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:24 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2805366496' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:52:25.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:24 smithi138 ceph-mon[104418]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T22:52:25.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:24 smithi110 ceph-mon[100243]: pgmap v9: 9 pgs: 5 active+clean, 4 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:25.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:24 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2805366496' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:52:25.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:24 smithi110 ceph-mon[100243]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T22:52:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:24 smithi110 ceph-mon[118821]: pgmap v9: 9 pgs: 5 active+clean, 4 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:24 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2805366496' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:52:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:24 smithi110 ceph-mon[118821]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T22:52:25.234 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:27.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:26 smithi138 ceph-mon[104418]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:27.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:26 smithi110 ceph-mon[100243]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:27.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:26 smithi110 ceph-mon[118821]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:29.054 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:28 smithi110 ceph-mon[100243]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:29.055 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:28 smithi110 ceph-mon[118821]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:29.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:28 smithi138 ceph-mon[104418]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:29.300 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:52:30.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:29 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3614393147' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:52:30.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:29 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3614393147' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:52:30.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:29 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3614393147' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:52:30.462 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":19,"active_gid":24347,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6800","nonce":576619896},{"type":"v1","addr":"172.21.15.110:6801","nonce":576619896}]},"active_addr":"172.21.15.110:6801/576619896","active_change":"2023-09-23T22:52:17.775258+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24340,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.110:8443/","prometheus":"http://172.21.15.110:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":1399491671}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":3464832160}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":2850586742}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":380987145}]}]} 2023-09-23T22:52:30.466 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T22:52:30.467 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T22:52:30.467 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:52:30.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:30 smithi110 ceph-mon[100243]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:30.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:30 smithi110 ceph-mon[118821]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:31.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:30 smithi138 ceph-mon[104418]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:31.508 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:31.759 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:52:31 smithi110 systemd[1]: Starting Ceph alertmanager.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:52:32.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:31 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:52:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:52:32.988 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:32 smithi110 ceph-mon[118821]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:32.988 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:32 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:32.988 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:32 smithi110 ceph-mon[100243]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:32.988 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:32 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:32.989 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:52:32 smithi110 conmon[163775]: ts=2023-09-23T22:52:32.725Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T22:52:32.989 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:52:32 smithi110 conmon[163775]: ts=2023-09-23T22:52:32.725Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T22:52:32.989 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:52:32 smithi110 conmon[163775]: ts=2023-09-23T22:52:32.726Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.110 port=9094 2023-09-23T22:52:32.989 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:52:32 smithi110 conmon[163775]: ts=2023-09-23T22:52:32.727Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T22:52:32.989 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:52:32 smithi110 conmon[163775]: ts=2023-09-23T22:52:32.750Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T22:52:32.989 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:52:32 smithi110 conmon[163775]: ts=2023-09-23T22:52:32.751Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T22:52:32.989 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:52:32 smithi110 conmon[163775]: ts=2023-09-23T22:52:32.753Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T22:52:32.989 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:52:32 smithi110 conmon[163775]: ts=2023-09-23T22:52:32.753Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T22:52:32.989 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:52:32 smithi110 bash[163715]: 655e561a5462c36cea589b5aa97c4845044bb3b49bcd3e7b66146922ac3c6045 2023-09-23T22:52:32.989 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:52:32 smithi110 systemd[1]: Started Ceph alertmanager.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:52:33.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:32 smithi138 ceph-mon[104418]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:33.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:32 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:33.268 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:33 smithi110 conmon[100666]: [23/Sep/2023:22:52:33] ENGINE Bus STOPPING 2023-09-23T22:52:33.268 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:33 smithi110 conmon[100666]: [23/Sep/2023:22:52:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:52:33.268 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:33 smithi110 conmon[100666]: [23/Sep/2023:22:52:33] ENGINE Bus STOPPED 2023-09-23T22:52:33.268 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:33 smithi110 conmon[100666]: [23/Sep/2023:22:52:33] ENGINE Bus STARTING 2023-09-23T22:52:33.269 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:33 smithi110 conmon[100666]: [23/Sep/2023:22:52:33] ENGINE Serving on http://:::9283 2023-09-23T22:52:33.659 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:52:33.659 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":63,"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:46:49.032269+0000","modified":"2023-09-23T22:52:23.917579+0000","last_up_change":"2023-09-23T22:51:43.190723+0000","last_in_change":"2023-09-23T22:51:26.130429+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:49:49.263399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:52:19.908332+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"78791acb-c93c-4e4f-bf04-ea3eee46408d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6803","nonce":2453789689}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6805","nonce":2453789689}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6809","nonce":2453789689}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6807","nonce":2453789689}]},"public_addr":"172.21.15.110:6803/2453789689","cluster_addr":"172.21.15.110:6805/2453789689","heartbeat_back_addr":"172.21.15.110:6809/2453789689","heartbeat_front_addr":"172.21.15.110:6807/2453789689","state":["exists","up"]},{"osd":1,"uuid":"6793178f-2ef1-4c48-927c-831a083a3356","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6811","nonce":3697882113}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6813","nonce":3697882113}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6817","nonce":3697882113}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6815","nonce":3697882113}]},"public_addr":"172.21.15.110:6811/3697882113","cluster_addr":"172.21.15.110:6813/3697882113","heartbeat_back_addr":"172.21.15.110:6817/3697882113","heartbeat_front_addr":"172.21.15.110:6815/3697882113","state":["exists","up"]},{"osd":2,"uuid":"146fbd28-a365-4545-8e97-d629c6ddba21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6819","nonce":2231416166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6821","nonce":2231416166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6825","nonce":2231416166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6823","nonce":2231416166}]},"public_addr":"172.21.15.110:6819/2231416166","cluster_addr":"172.21.15.110:6821/2231416166","heartbeat_back_addr":"172.21.15.110:6825/2231416166","heartbeat_front_addr":"172.21.15.110:6823/2231416166","state":["exists","up"]},{"osd":3,"uuid":"6fcf80d5-6977-4929-b7ba-0e9ab65df0a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6827","nonce":427497470}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6829","nonce":427497470}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6833","nonce":427497470}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6831","nonce":427497470}]},"public_addr":"172.21.15.110:6827/427497470","cluster_addr":"172.21.15.110:6829/427497470","heartbeat_back_addr":"172.21.15.110:6833/427497470","heartbeat_front_addr":"172.21.15.110:6831/427497470","state":["exists","up"]},{"osd":4,"uuid":"131d0749-c46c-44f0-ab7f-03f8ad6ed502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6801","nonce":941787557}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6803","nonce":941787557}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6807","nonce":941787557}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6805","nonce":941787557}]},"public_addr":"172.21.15.138:6801/941787557","cluster_addr":"172.21.15.138:6803/941787557","heartbeat_back_addr":"172.21.15.138:6807/941787557","heartbeat_front_addr":"172.21.15.138:6805/941787557","state":["exists","up"]},{"osd":5,"uuid":"8cb496e5-d242-4e41-b279-e8d5ecea2996","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6809","nonce":3098195887}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6811","nonce":3098195887}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6815","nonce":3098195887}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6813","nonce":3098195887}]},"public_addr":"172.21.15.138:6809/3098195887","cluster_addr":"172.21.15.138:6811/3098195887","heartbeat_back_addr":"172.21.15.138:6815/3098195887","heartbeat_front_addr":"172.21.15.138:6813/3098195887","state":["exists","up"]},{"osd":6,"uuid":"c709e629-85ab-4273-9ca8-f2c4e8fc9074","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6817","nonce":187413403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6819","nonce":187413403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6823","nonce":187413403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6821","nonce":187413403}]},"public_addr":"172.21.15.138:6817/187413403","cluster_addr":"172.21.15.138:6819/187413403","heartbeat_back_addr":"172.21.15.138:6823/187413403","heartbeat_front_addr":"172.21.15.138:6821/187413403","state":["exists","up"]},{"osd":7,"uuid":"ed15f3b0-cd2e-43b7-b803-e45252b62a45","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6825","nonce":676244673}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6827","nonce":676244673}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6831","nonce":676244673}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6829","nonce":676244673}]},"public_addr":"172.21.15.138:6825/676244673","cluster_addr":"172.21.15.138:6827/676244673","heartbeat_back_addr":"172.21.15.138:6831/676244673","heartbeat_front_addr":"172.21.15.138:6829/676244673","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:48:58.523930+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:22.993201+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:46.747264+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:11.206564+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:33.511236+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:56.364297+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:18.141750+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:41.242882+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:6801/2914393040":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6800/2914393040":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/3674528218":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/3412430555":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1649304640":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1326657364":"2023-09-24T22:47:38.935467+0000","172.21.15.110:6801/4228126008":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/2523943302":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/1731974664":"2023-09-24T22:47:15.528545+0000","172.21.15.110:6801/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/1506915348":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/3250066431":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/410644117":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/657635177":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6800/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.110:6800/4228126008":"2023-09-24T22:47:38.935467+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:52:33.688 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:52:33 smithi110 conmon[100666]: [23/Sep/2023:22:52:33] ENGINE Bus STARTED 2023-09-23T22:52:34.403 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.403 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.403 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.403 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.403 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.403 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.404 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:52:34.404 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[100243]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:52:34.404 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.404 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2010888421' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:52:34.404 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.404 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.405 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.405 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.405 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.405 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.405 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:52:34.405 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[118821]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:52:34.405 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.405 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:33 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2010888421' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:52:34.422 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T22:52:34.422 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:52:34.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:33 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:33 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:33 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:33 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:33 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:33 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:33 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:52:34.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:33 smithi138 ceph-mon[104418]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:52:34.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:33 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:34.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:33 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2010888421' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:52:34.760 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:35.012 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:52:34 smithi110 conmon[163775]: ts=2023-09-23T22:52:34.727Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000371814s 2023-09-23T22:52:35.413 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:35 smithi110 ceph-mon[100243]: Deploying daemon grafana.a on smithi138 2023-09-23T22:52:35.413 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:35 smithi110 ceph-mon[100243]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:35.413 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:35 smithi110 ceph-mon[118821]: Deploying daemon grafana.a on smithi138 2023-09-23T22:52:35.413 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:35 smithi110 ceph-mon[118821]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:35.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:35 smithi138 ceph-mon[104418]: Deploying daemon grafana.a on smithi138 2023-09-23T22:52:35.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:35 smithi138 ceph-mon[104418]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:36.569 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:52:36.569 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":63,"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:46:49.032269+0000","modified":"2023-09-23T22:52:23.917579+0000","last_up_change":"2023-09-23T22:51:43.190723+0000","last_in_change":"2023-09-23T22:51:26.130429+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:49:49.263399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:52:19.908332+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"78791acb-c93c-4e4f-bf04-ea3eee46408d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6803","nonce":2453789689}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6805","nonce":2453789689}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6809","nonce":2453789689}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6807","nonce":2453789689}]},"public_addr":"172.21.15.110:6803/2453789689","cluster_addr":"172.21.15.110:6805/2453789689","heartbeat_back_addr":"172.21.15.110:6809/2453789689","heartbeat_front_addr":"172.21.15.110:6807/2453789689","state":["exists","up"]},{"osd":1,"uuid":"6793178f-2ef1-4c48-927c-831a083a3356","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6811","nonce":3697882113}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6813","nonce":3697882113}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6817","nonce":3697882113}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6815","nonce":3697882113}]},"public_addr":"172.21.15.110:6811/3697882113","cluster_addr":"172.21.15.110:6813/3697882113","heartbeat_back_addr":"172.21.15.110:6817/3697882113","heartbeat_front_addr":"172.21.15.110:6815/3697882113","state":["exists","up"]},{"osd":2,"uuid":"146fbd28-a365-4545-8e97-d629c6ddba21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6819","nonce":2231416166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6821","nonce":2231416166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6825","nonce":2231416166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6823","nonce":2231416166}]},"public_addr":"172.21.15.110:6819/2231416166","cluster_addr":"172.21.15.110:6821/2231416166","heartbeat_back_addr":"172.21.15.110:6825/2231416166","heartbeat_front_addr":"172.21.15.110:6823/2231416166","state":["exists","up"]},{"osd":3,"uuid":"6fcf80d5-6977-4929-b7ba-0e9ab65df0a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6827","nonce":427497470}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6829","nonce":427497470}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6833","nonce":427497470}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6831","nonce":427497470}]},"public_addr":"172.21.15.110:6827/427497470","cluster_addr":"172.21.15.110:6829/427497470","heartbeat_back_addr":"172.21.15.110:6833/427497470","heartbeat_front_addr":"172.21.15.110:6831/427497470","state":["exists","up"]},{"osd":4,"uuid":"131d0749-c46c-44f0-ab7f-03f8ad6ed502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6801","nonce":941787557}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6803","nonce":941787557}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6807","nonce":941787557}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6805","nonce":941787557}]},"public_addr":"172.21.15.138:6801/941787557","cluster_addr":"172.21.15.138:6803/941787557","heartbeat_back_addr":"172.21.15.138:6807/941787557","heartbeat_front_addr":"172.21.15.138:6805/941787557","state":["exists","up"]},{"osd":5,"uuid":"8cb496e5-d242-4e41-b279-e8d5ecea2996","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6809","nonce":3098195887}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6811","nonce":3098195887}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6815","nonce":3098195887}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6813","nonce":3098195887}]},"public_addr":"172.21.15.138:6809/3098195887","cluster_addr":"172.21.15.138:6811/3098195887","heartbeat_back_addr":"172.21.15.138:6815/3098195887","heartbeat_front_addr":"172.21.15.138:6813/3098195887","state":["exists","up"]},{"osd":6,"uuid":"c709e629-85ab-4273-9ca8-f2c4e8fc9074","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6817","nonce":187413403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6819","nonce":187413403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6823","nonce":187413403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6821","nonce":187413403}]},"public_addr":"172.21.15.138:6817/187413403","cluster_addr":"172.21.15.138:6819/187413403","heartbeat_back_addr":"172.21.15.138:6823/187413403","heartbeat_front_addr":"172.21.15.138:6821/187413403","state":["exists","up"]},{"osd":7,"uuid":"ed15f3b0-cd2e-43b7-b803-e45252b62a45","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6825","nonce":676244673}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6827","nonce":676244673}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6831","nonce":676244673}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6829","nonce":676244673}]},"public_addr":"172.21.15.138:6825/676244673","cluster_addr":"172.21.15.138:6827/676244673","heartbeat_back_addr":"172.21.15.138:6831/676244673","heartbeat_front_addr":"172.21.15.138:6829/676244673","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:48:58.523930+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:22.993201+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:46.747264+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:11.206564+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:33.511236+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:56.364297+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:18.141750+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:41.242882+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:6801/2914393040":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6800/2914393040":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/3674528218":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/3412430555":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1649304640":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1326657364":"2023-09-24T22:47:38.935467+0000","172.21.15.110:6801/4228126008":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/2523943302":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/1731974664":"2023-09-24T22:47:15.528545+0000","172.21.15.110:6801/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/1506915348":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/3250066431":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/410644117":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/657635177":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6800/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.110:6800/4228126008":"2023-09-24T22:47:38.935467+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:52:37.328 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T22:52:37.329 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T22:52:37.329 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T22:52:37.330 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T22:52:37.330 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T22:52:37.331 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T22:52:37.331 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T22:52:37.333 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T22:52:37.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:37 smithi138 ceph-mon[104418]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:37.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:37 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2450884532' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:52:37.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:37 smithi110 ceph-mon[118821]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:37.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:37 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2450884532' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:52:37.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:37 smithi110 ceph-mon[100243]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:37.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:37 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2450884532' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:52:38.848 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:38.899 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:38.900 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:38.963 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:39.100 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:39.112 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:38 smithi110 ceph-mon[100243]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:39.112 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:38 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:39.112 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:38 smithi110 ceph-mon[118821]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:39.112 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:38 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:39.158 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:39.190 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:38 smithi138 ceph-mon[104418]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:39.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:38 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:39.259 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:39.419 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:40.574 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:40 smithi110 ceph-mon[100243]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:40.574 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:40 smithi110 ceph-mon[118821]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:40.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:40 smithi138 ceph-mon[104418]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:42.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:41 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:52:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:52:43.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:42 smithi138 ceph-mon[104418]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:43.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:42 smithi110 ceph-mon[100243]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:43.201 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:52:42 smithi110 conmon[163775]: ts=2023-09-23T22:52:42.730Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.00335331s 2023-09-23T22:52:43.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:42 smithi110 ceph-mon[118821]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:45.115 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:44 smithi110 ceph-mon[100243]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:45.115 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:44 smithi110 ceph-mon[118821]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:45.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:44 smithi138 ceph-mon[104418]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:45.927 INFO:teuthology.orchestra.run.smithi110.stdout:107374182431 2023-09-23T22:52:45.927 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T22:52:46.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:46 smithi110 ceph-mon[100243]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:46.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:46 smithi110 ceph-mon[118821]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:47.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:46 smithi138 ceph-mon[104418]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:47.600 INFO:teuthology.orchestra.run.smithi110.stdout:55834574890 2023-09-23T22:52:47.600 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T22:52:48.685 INFO:teuthology.orchestra.run.smithi110.stdout:34359738414 2023-09-23T22:52:48.685 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T22:52:48.825 INFO:teuthology.orchestra.run.smithi110.stdout:197568495635 2023-09-23T22:52:48.826 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T22:52:48.854 INFO:teuthology.orchestra.run.smithi110.stdout:128849018908 2023-09-23T22:52:48.854 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T22:52:49.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:48 smithi138 ceph-mon[104418]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:49.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:48 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:49.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:48 smithi110 ceph-mon[118821]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:49.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:48 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:49.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:48 smithi110 ceph-mon[100243]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:49.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:48 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:49.819 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:50.043 INFO:teuthology.orchestra.run.smithi110.stdout:231928233999 2023-09-23T22:52:50.043 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T22:52:50.144 INFO:teuthology.orchestra.run.smithi110.stdout:163208757271 2023-09-23T22:52:50.145 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T22:52:50.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:50 smithi110 ceph-mon[100243]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:50.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:50 smithi110 ceph-mon[118821]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:50 smithi138 ceph-mon[104418]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:50.832 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:50.854 INFO:teuthology.orchestra.run.smithi110.stdout:77309411365 2023-09-23T22:52:50.854 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T22:52:50.884 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:51.328 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:52.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:52:51 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:52:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:52:53.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:52 smithi138 ceph-mon[104418]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:53.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:52 smithi110 ceph-mon[100243]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:53.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:52 smithi110 ceph-mon[118821]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:53.765 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:53.822 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:54.373 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:54.422 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:54.426 INFO:teuthology.orchestra.run.smithi110.stdout:128849018910 2023-09-23T22:52:54.875 INFO:teuthology.orchestra.run.smithi110.stdout:55834574892 2023-09-23T22:52:55.029 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:54 smithi110 ceph-mon[100243]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:55.029 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:54 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2513561513' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:52:55.029 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:54 smithi110 ceph-mon[118821]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:55.029 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:54 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2513561513' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:52:55.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:54 smithi138 ceph-mon[104418]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:55.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:54 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2513561513' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:52:55.733 INFO:teuthology.orchestra.run.smithi110.stdout:163208757273 2023-09-23T22:52:56.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:55 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3972945302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:52:56.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:55 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/933231849' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:52:56.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:55 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/949516726' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:52:56.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:55 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3972945302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:52:56.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:55 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/933231849' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:52:56.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:55 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/949516726' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:52:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:55 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3972945302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:52:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:55 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/933231849' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:52:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:55 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/949516726' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:52:56.291 INFO:teuthology.orchestra.run.smithi110.stdout:107374182434 2023-09-23T22:52:56.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:56 smithi110 ceph-mon[100243]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:56.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:56 smithi110 ceph-mon[118821]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:57.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:56 smithi138 ceph-mon[104418]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:57.192 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:56 smithi138 systemd[1]: Starting Ceph grafana.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:52:57.613 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574890 got 55834574892 for osd.1 2023-09-23T22:52:57.614 DEBUG:teuthology.parallel:result is None 2023-09-23T22:52:57.851 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 bash[151443]: e7b0f17c56818bb7d07e9d5f51916ba0c0ae4489236e9e042590e19cacf8dc9f 2023-09-23T22:52:57.851 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 systemd[1]: Started Ceph grafana.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:52:57.851 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=settings t=2023-09-23T22:52:57.705960129Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-23T22:52:57.851 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=settings t=2023-09-23T22:52:57.706138584Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-23T22:52:57.851 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=settings t=2023-09-23T22:52:57.706157043Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-23T22:52:57.851 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=settings t=2023-09-23T22:52:57.70616394Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-23T22:52:57.851 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=settings t=2023-09-23T22:52:57.70616993Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T22:52:57.851 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=settings t=2023-09-23T22:52:57.70617579Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T22:52:57.852 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=settings t=2023-09-23T22:52:57.706181232Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T22:52:57.852 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=settings t=2023-09-23T22:52:57.706186952Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T22:52:57.852 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=settings t=2023-09-23T22:52:57.706192819Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-23T22:52:57.852 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=settings t=2023-09-23T22:52:57.706199652Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-23T22:52:57.852 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=settings t=2023-09-23T22:52:57.706205093Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-23T22:52:57.852 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=settings t=2023-09-23T22:52:57.706210602Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-23T22:52:57.852 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=settings t=2023-09-23T22:52:57.706216076Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-23T22:52:57.852 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=settings t=2023-09-23T22:52:57.706221682Z level=info msg="App mode production" 2023-09-23T22:52:57.852 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=sqlstore t=2023-09-23T22:52:57.70627074Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-23T22:52:57.852 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=sqlstore t=2023-09-23T22:52:57.706287061Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T22:52:57.852 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:57.707089165Z level=info msg="Starting DB migrations" 2023-09-23T22:52:57.852 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:57.707167974Z level=info msg="Executing migration" id="create migration_log table" 2023-09-23T22:52:57.852 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:57.733491924Z level=info msg="Executing migration" id="create user table" 2023-09-23T22:52:57.853 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:57.75857222Z level=info msg="Executing migration" id="add unique index user.login" 2023-09-23T22:52:57.853 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:57.783799697Z level=info msg="Executing migration" id="add unique index user.email" 2023-09-23T22:52:57.853 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:57.825817489Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2023-09-23T22:52:57.853 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:57.850816412Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2023-09-23T22:52:57.994 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757271 got 163208757273 for osd.5 2023-09-23T22:52:57.994 DEBUG:teuthology.parallel:result is None 2023-09-23T22:52:58.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:57.901838033Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2023-09-23T22:52:58.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:57.943668035Z level=info msg="Executing migration" id="create user table v2" 2023-09-23T22:52:58.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:57 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:57.985627904Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2023-09-23T22:52:58.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.010909574Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2023-09-23T22:52:58.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.061093507Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2023-09-23T22:52:58.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.086331319Z level=info msg="Executing migration" id="Drop old table user_v1" 2023-09-23T22:52:58.112 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.111454237Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2023-09-23T22:52:58.363 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.153430745Z level=info msg="Executing migration" id="Update user table charset" 2023-09-23T22:52:58.363 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.195413459Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2023-09-23T22:52:58.363 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.237251182Z level=info msg="Executing migration" id="Add missing user data" 2023-09-23T22:52:58.363 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.26240532Z level=info msg="Executing migration" id="Add is_disabled column to user" 2023-09-23T22:52:58.363 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.329366298Z level=info msg="Executing migration" id="Add index user.login/user.email" 2023-09-23T22:52:58.619 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.363101842Z level=info msg="Executing migration" id="Add is_service_account column to user" 2023-09-23T22:52:58.619 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.388524232Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2023-09-23T22:52:58.619 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.430313114Z level=info msg="Executing migration" id="create temp user table v1-7" 2023-09-23T22:52:58.619 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.44719834Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2023-09-23T22:52:58.620 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.480914037Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2023-09-23T22:52:58.620 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.497678967Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2023-09-23T22:52:58.620 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.514444869Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2023-09-23T22:52:58.620 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.531233827Z level=info msg="Executing migration" id="Update temp_user table charset" 2023-09-23T22:52:58.620 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.548256322Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2023-09-23T22:52:58.621 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.565145908Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2023-09-23T22:52:58.621 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.581812781Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2023-09-23T22:52:58.621 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.598909289Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2023-09-23T22:52:58.621 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.615514493Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-09-23T22:52:58.868 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:58 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:58.868 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:58 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:58.868 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:58 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:58.868 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:58 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:58.868 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:58 smithi138 ceph-mon[104418]: Deploying daemon node-exporter.a on smithi110 2023-09-23T22:52:58.868 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:58 smithi138 ceph-mon[104418]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:58.868 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:52:58 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:58.869 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.641024532Z level=info msg="Executing migration" id="create temp_user v2" 2023-09-23T22:52:58.869 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.674649731Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2023-09-23T22:52:58.869 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.691374811Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2023-09-23T22:52:58.869 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.724925265Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2023-09-23T22:52:58.869 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.741873601Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2023-09-23T22:52:58.869 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.75883185Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2023-09-23T22:52:58.869 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.775533089Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2023-09-23T22:52:58.869 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.809095297Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2023-09-23T22:52:58.869 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.82608629Z level=info msg="Executing migration" id="create star table" 2023-09-23T22:52:58.869 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.842869772Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2023-09-23T22:52:58.870 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.868153798Z level=info msg="Executing migration" id="create org table v1" 2023-09-23T22:52:58.911 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:58 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:58.911 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:58 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:58.911 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:58 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:58.911 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:58 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:58.911 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:58 smithi110 ceph-mon[100243]: Deploying daemon node-exporter.a on smithi110 2023-09-23T22:52:58.911 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:58 smithi110 ceph-mon[100243]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:58.911 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:52:58 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:58.912 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:58 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:58.912 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:58 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:58.912 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:58 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:58.912 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:58 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:58.912 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:58 smithi110 ceph-mon[118821]: Deploying daemon node-exporter.a on smithi110 2023-09-23T22:52:58.912 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:58 smithi110 ceph-mon[118821]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:52:58.912 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:52:58 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:52:59.141 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.912610167Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2023-09-23T22:52:59.142 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.954668563Z level=info msg="Executing migration" id="create org_user table v1" 2023-09-23T22:52:59.142 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:58 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:58.979642384Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2023-09-23T22:52:59.142 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.004857722Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2023-09-23T22:52:59.142 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.030167766Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2023-09-23T22:52:59.142 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.072083216Z level=info msg="Executing migration" id="Update org table charset" 2023-09-23T22:52:59.142 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.097316496Z level=info msg="Executing migration" id="Update org_user table charset" 2023-09-23T22:52:59.142 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.13928217Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2023-09-23T22:52:59.391 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.190003665Z level=info msg="Executing migration" id="create dashboard table" 2023-09-23T22:52:59.391 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.215122737Z level=info msg="Executing migration" id="add index dashboard.account_id" 2023-09-23T22:52:59.391 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.257084343Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2023-09-23T22:52:59.391 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.282237087Z level=info msg="Executing migration" id="create dashboard_tag table" 2023-09-23T22:52:59.391 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.324278861Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2023-09-23T22:52:59.392 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.366280856Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-09-23T22:52:59.643 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.391333303Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2023-09-23T22:52:59.643 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.433495648Z level=info msg="Executing migration" id="create dashboard v2" 2023-09-23T22:52:59.643 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.458657022Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2023-09-23T22:52:59.643 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.525610519Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2023-09-23T22:52:59.643 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.56762488Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2023-09-23T22:52:59.643 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.592608036Z level=info msg="Executing migration" id="drop table dashboard_v1" 2023-09-23T22:52:59.643 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.617855243Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2023-09-23T22:52:59.694 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018908 got 128849018910 for osd.4 2023-09-23T22:52:59.694 DEBUG:teuthology.parallel:result is None 2023-09-23T22:52:59.903 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.643076779Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2023-09-23T22:52:59.903 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.685056485Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2023-09-23T22:52:59.903 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.710328513Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2023-09-23T22:52:59.903 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.735362988Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2023-09-23T22:52:59.903 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.760676988Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2023-09-23T22:52:59.903 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.78584845Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2023-09-23T22:52:59.904 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.827696349Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2023-09-23T22:52:59.904 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.852897113Z level=info msg="Executing migration" id="Update dashboard table charset" 2023-09-23T22:52:59.904 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.878295626Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2023-09-23T22:52:59.904 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.903474678Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2023-09-23T22:52:59.995 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182431 got 107374182434 for osd.3 2023-09-23T22:52:59.995 DEBUG:teuthology.parallel:result is None 2023-09-23T22:53:00.172 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:52:59 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:52:59.962482484Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2023-09-23T22:53:00.172 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.004449028Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2023-09-23T22:53:00.172 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.063062844Z level=info msg="Executing migration" id="Add column uid in dashboard" 2023-09-23T22:53:00.172 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.088229233Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2023-09-23T22:53:00.172 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.130167125Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2023-09-23T22:53:00.172 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.172191242Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2023-09-23T22:53:00.182 INFO:teuthology.orchestra.run.smithi110.stdout:197568495637 2023-09-23T22:53:00.353 INFO:teuthology.orchestra.run.smithi110.stdout:231928234002 2023-09-23T22:53:00.424 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:00 smithi138 ceph-mon[104418]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:00.424 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:00 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1321664253' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:53:00.424 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.222641364Z level=info msg="Executing migration" id="Update dashboard title length" 2023-09-23T22:53:00.424 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.247644406Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2023-09-23T22:53:00.424 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.272864161Z level=info msg="Executing migration" id="create dashboard_provisioning" 2023-09-23T22:53:00.424 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.314890224Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-09-23T22:53:00.425 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.356853386Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2023-09-23T22:53:00.425 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.390435819Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-09-23T22:53:00.425 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.407398134Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-09-23T22:53:00.425 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.424316615Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2023-09-23T22:53:00.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:00 smithi110 ceph-mon[100243]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:00.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:00 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1321664253' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:53:00.577 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:00 smithi110 ceph-mon[118821]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:00.577 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:00 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1321664253' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:53:00.677 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.466478399Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2023-09-23T22:53:00.677 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.483304025Z level=info msg="Executing migration" id="Add check_sum column" 2023-09-23T22:53:00.677 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.500237531Z level=info msg="Executing migration" id="Add index for dashboard_title" 2023-09-23T22:53:00.677 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.517110681Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2023-09-23T22:53:00.677 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.550695637Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2023-09-23T22:53:00.677 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.567406745Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2023-09-23T22:53:00.677 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.584425938Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2023-09-23T22:53:00.677 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.60123313Z level=info msg="Executing migration" id="create data_source table" 2023-09-23T22:53:00.677 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.618065773Z level=info msg="Executing migration" id="add index data_source.account_id" 2023-09-23T22:53:00.677 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.643411551Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2023-09-23T22:53:00.677 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.676933857Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2023-09-23T22:53:00.858 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:00 smithi110 systemd[1]: Starting Ceph node-exporter.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:53:00.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.760789059Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2023-09-23T22:53:00.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.785923761Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2023-09-23T22:53:00.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.811217563Z level=info msg="Executing migration" id="create data_source table v2" 2023-09-23T22:53:00.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.836407314Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2023-09-23T22:53:00.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.86162934Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2023-09-23T22:53:00.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.886651241Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2023-09-23T22:53:00.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.912053194Z level=info msg="Executing migration" id="Add column with_credentials" 2023-09-23T22:53:01.222 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:00 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:00.963460578Z level=info msg="Executing migration" id="Add secure json data column" 2023-09-23T22:53:01.223 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.012956375Z level=info msg="Executing migration" id="Update data_source table charset" 2023-09-23T22:53:01.223 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.038253931Z level=info msg="Executing migration" id="Update initial version to 1" 2023-09-23T22:53:01.223 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.063423436Z level=info msg="Executing migration" id="Add read_only data column" 2023-09-23T22:53:01.223 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.105163176Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2023-09-23T22:53:01.223 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.130440079Z level=info msg="Executing migration" id="Update json_data with nulls" 2023-09-23T22:53:01.223 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.155661884Z level=info msg="Executing migration" id="Add uid column" 2023-09-23T22:53:01.223 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.18073733Z level=info msg="Executing migration" id="Update uid value" 2023-09-23T22:53:01.223 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.222689227Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2023-09-23T22:53:01.491 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:01 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3296262258' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:53:01.491 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.264727219Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2023-09-23T22:53:01.491 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.290101323Z level=info msg="Executing migration" id="create api_key table" 2023-09-23T22:53:01.491 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.331728778Z level=info msg="Executing migration" id="add index api_key.account_id" 2023-09-23T22:53:01.491 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.373760789Z level=info msg="Executing migration" id="add index api_key.key" 2023-09-23T22:53:01.491 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.399008344Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2023-09-23T22:53:01.492 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.424213488Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2023-09-23T22:53:01.492 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.449566436Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2023-09-23T22:53:01.492 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.491282098Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2023-09-23T22:53:01.558 INFO:teuthology.orchestra.run.smithi110.stdout:77309411367 2023-09-23T22:53:01.583 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:01 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3296262258' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:53:01.583 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:01 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3296262258' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:53:01.680 INFO:teuthology.orchestra.run.smithi110.stdout:34359738417 2023-09-23T22:53:01.727 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928233999 got 231928234002 for osd.7 2023-09-23T22:53:01.727 DEBUG:teuthology.parallel:result is None 2023-09-23T22:53:01.760 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.525078971Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2023-09-23T22:53:01.760 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.550260405Z level=info msg="Executing migration" id="create api_key table v2" 2023-09-23T22:53:01.760 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.575497276Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2023-09-23T22:53:01.760 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.617441418Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2023-09-23T22:53:01.760 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.64261379Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2023-09-23T22:53:01.760 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.684699554Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2023-09-23T22:53:01.760 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.709655143Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2023-09-23T22:53:01.760 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.735003185Z level=info msg="Executing migration" id="Update api_key table charset" 2023-09-23T22:53:01.760 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.760205994Z level=info msg="Executing migration" id="Add expires to api_key table" 2023-09-23T22:53:01.778 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495635 got 197568495637 for osd.6 2023-09-23T22:53:01.779 DEBUG:teuthology.parallel:result is None 2023-09-23T22:53:02.020 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:53:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:53:02.020 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.785625673Z level=info msg="Executing migration" id="Add service account foreign key" 2023-09-23T22:53:02.020 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.827452802Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2023-09-23T22:53:02.020 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.869466143Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2023-09-23T22:53:02.020 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.911348816Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2023-09-23T22:53:02.020 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:01 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:01.9783426Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2023-09-23T22:53:02.021 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.02024883Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2023-09-23T22:53:02.289 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.062690378Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2023-09-23T22:53:02.289 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.087992897Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2023-09-23T22:53:02.289 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.113102246Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-09-23T22:53:02.289 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.15506157Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2023-09-23T22:53:02.289 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.180368004Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2023-09-23T22:53:02.289 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.205289721Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2023-09-23T22:53:02.289 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.247360146Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2023-09-23T22:53:02.290 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.28926743Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2023-09-23T22:53:02.397 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738414 got 34359738417 for osd.0 2023-09-23T22:53:02.397 DEBUG:teuthology.parallel:result is None 2023-09-23T22:53:02.541 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:02 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3377286018' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:53:02.541 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:02 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1566673626' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:53:02.541 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:02 smithi138 ceph-mon[104418]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:02.541 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.339710529Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-09-23T22:53:02.541 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.381639638Z level=info msg="Executing migration" id="create quota table v1" 2023-09-23T22:53:02.541 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.423697082Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2023-09-23T22:53:02.542 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.448833805Z level=info msg="Executing migration" id="Update quota table charset" 2023-09-23T22:53:02.542 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.490645474Z level=info msg="Executing migration" id="create plugin_setting table" 2023-09-23T22:53:02.542 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.516039376Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-09-23T22:53:02.542 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.541052202Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2023-09-23T22:53:02.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:02 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3377286018' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:53:02.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:02 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1566673626' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:53:02.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:02 smithi110 ceph-mon[100243]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:02.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:02 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3377286018' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:53:02.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:02 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1566673626' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:53:02.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:02 smithi110 ceph-mon[118821]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:02.809 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.574874501Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2023-09-23T22:53:02.810 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.600120071Z level=info msg="Executing migration" id="create session table" 2023-09-23T22:53:02.810 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.625284283Z level=info msg="Executing migration" id="Drop old table playlist table" 2023-09-23T22:53:02.810 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.650550463Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2023-09-23T22:53:02.810 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.692797372Z level=info msg="Executing migration" id="create playlist table v2" 2023-09-23T22:53:02.810 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.717661564Z level=info msg="Executing migration" id="create playlist item table v2" 2023-09-23T22:53:02.810 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.759617783Z level=info msg="Executing migration" id="Update playlist table charset" 2023-09-23T22:53:02.810 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.784751425Z level=info msg="Executing migration" id="Update playlist_item table charset" 2023-09-23T22:53:02.870 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411365 got 77309411367 for osd.2 2023-09-23T22:53:02.870 DEBUG:teuthology.parallel:result is None 2023-09-23T22:53:02.871 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T22:53:02.871 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:53:03.078 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.809892669Z level=info msg="Executing migration" id="drop preferences table v2" 2023-09-23T22:53:03.079 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.860544649Z level=info msg="Executing migration" id="drop preferences table v3" 2023-09-23T22:53:03.079 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.885612444Z level=info msg="Executing migration" id="create preferences table v3" 2023-09-23T22:53:03.079 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.910838688Z level=info msg="Executing migration" id="Update preferences table charset" 2023-09-23T22:53:03.079 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.952850843Z level=info msg="Executing migration" id="Add column team_id in preferences" 2023-09-23T22:53:03.079 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:02 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:02.977979842Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2023-09-23T22:53:03.079 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.003277414Z level=info msg="Executing migration" id="Add column week_start in preferences" 2023-09-23T22:53:03.079 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.028453474Z level=info msg="Executing migration" id="Add column preferences.json_data" 2023-09-23T22:53:03.079 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.045404635Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2023-09-23T22:53:03.079 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.078881852Z level=info msg="Executing migration" id="Add preferences index org_id" 2023-09-23T22:53:03.201 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:03 smithi110 bash[171339]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-23T22:53:03.236 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:03.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:03 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:03.348 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.129754396Z level=info msg="Executing migration" id="Add preferences index user_id" 2023-09-23T22:53:03.348 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.171802028Z level=info msg="Executing migration" id="create alert table v1" 2023-09-23T22:53:03.348 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.196860257Z level=info msg="Executing migration" id="add index alert org_id & id " 2023-09-23T22:53:03.348 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.238751017Z level=info msg="Executing migration" id="add index alert state" 2023-09-23T22:53:03.348 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.264013275Z level=info msg="Executing migration" id="add index alert dashboard_id" 2023-09-23T22:53:03.348 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.305872103Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2023-09-23T22:53:03.348 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.347921812Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-09-23T22:53:03.616 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.373074451Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-09-23T22:53:03.616 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.414937079Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-09-23T22:53:03.616 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.44041362Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2023-09-23T22:53:03.617 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.457247461Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-09-23T22:53:03.617 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.473945482Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2023-09-23T22:53:03.617 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.490866877Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2023-09-23T22:53:03.617 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.524546254Z level=info msg="Executing migration" id="create alert_notification table v1" 2023-09-23T22:53:03.617 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.566360744Z level=info msg="Executing migration" id="Add column is_default" 2023-09-23T22:53:03.617 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.591598817Z level=info msg="Executing migration" id="Add column frequency" 2023-09-23T22:53:03.617 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.616703643Z level=info msg="Executing migration" id="Add column send_reminder" 2023-09-23T22:53:03.647 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:03 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:03.647 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:03 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:03.648 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:03 smithi110 bash[171339]: Getting image source signatures 2023-09-23T22:53:03.876 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.650544537Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2023-09-23T22:53:03.877 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.692470781Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2023-09-23T22:53:03.877 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.717539476Z level=info msg="Executing migration" id="Update alert table charset" 2023-09-23T22:53:03.877 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.7426451Z level=info msg="Executing migration" id="Update alert_notification table charset" 2023-09-23T22:53:03.877 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.784789432Z level=info msg="Executing migration" id="create notification_journal table v1" 2023-09-23T22:53:03.877 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.826756907Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2023-09-23T22:53:03.877 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.851792813Z level=info msg="Executing migration" id="drop alert_notification_journal" 2023-09-23T22:53:03.877 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.876842942Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2023-09-23T22:53:03.934 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:03 smithi110 bash[171339]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T22:53:03.935 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:03 smithi110 bash[171339]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T22:53:03.935 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:03 smithi110 bash[171339]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T22:53:04.137 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.910753696Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-09-23T22:53:04.137 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.952460627Z level=info msg="Executing migration" id="Add for to alert table" 2023-09-23T22:53:04.137 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:03 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:03.977592745Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2023-09-23T22:53:04.137 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.019602969Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2023-09-23T22:53:04.137 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.061464588Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2023-09-23T22:53:04.137 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.086743532Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2023-09-23T22:53:04.137 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.111917853Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2023-09-23T22:53:04.405 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:04 smithi138 ceph-mon[104418]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:04.406 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.137182667Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2023-09-23T22:53:04.406 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.179551067Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2023-09-23T22:53:04.406 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.204695743Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2023-09-23T22:53:04.406 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.246675303Z level=info msg="Executing migration" id="Drop old annotation table v4" 2023-09-23T22:53:04.406 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.271912602Z level=info msg="Executing migration" id="create annotation table v5" 2023-09-23T22:53:04.406 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.313815978Z level=info msg="Executing migration" id="add index annotation 0 v3" 2023-09-23T22:53:04.407 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.374739967Z level=info msg="Executing migration" id="add index annotation 1 v3" 2023-09-23T22:53:04.407 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.405823188Z level=info msg="Executing migration" id="add index annotation 2 v3" 2023-09-23T22:53:04.613 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:04 smithi110 ceph-mon[118821]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:04.613 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:04 smithi110 ceph-mon[100243]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:04.674 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.456442095Z level=info msg="Executing migration" id="add index annotation 3 v3" 2023-09-23T22:53:04.674 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.506592118Z level=info msg="Executing migration" id="add index annotation 4 v3" 2023-09-23T22:53:04.674 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.540222215Z level=info msg="Executing migration" id="Update annotation table charset" 2023-09-23T22:53:04.674 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.573844098Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2023-09-23T22:53:04.674 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.607287029Z level=info msg="Executing migration" id="Drop category_id index" 2023-09-23T22:53:04.674 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.632505473Z level=info msg="Executing migration" id="Add column tags to annotation table" 2023-09-23T22:53:04.674 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.674337778Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2023-09-23T22:53:04.926 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.699569705Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2023-09-23T22:53:04.926 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.724687698Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-09-23T22:53:04.927 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.76662778Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-09-23T22:53:04.927 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.792000537Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2023-09-23T22:53:04.927 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.817170412Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-09-23T22:53:04.927 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.84241333Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2023-09-23T22:53:04.927 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.859327457Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2023-09-23T22:53:04.927 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.876032698Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2023-09-23T22:53:04.927 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.892851455Z level=info msg="Executing migration" id="Add created time to annotation table" 2023-09-23T22:53:04.927 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.909726776Z level=info msg="Executing migration" id="Add updated time to annotation table" 2023-09-23T22:53:04.927 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.926586811Z level=info msg="Executing migration" id="Add index for created in annotation table" 2023-09-23T22:53:05.171 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:05 smithi110 bash[171339]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-23T22:53:05.171 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:05 smithi110 bash[171339]: Writing manifest to image destination 2023-09-23T22:53:05.171 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:05 smithi110 bash[171339]: Storing signatures 2023-09-23T22:53:05.191 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.968945752Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2023-09-23T22:53:05.191 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:04 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:04.985986693Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2023-09-23T22:53:05.191 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.002759083Z level=info msg="Executing migration" id="Add epoch_end column" 2023-09-23T22:53:05.191 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.019525539Z level=info msg="Executing migration" id="Add index for epoch_end" 2023-09-23T22:53:05.191 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.036430103Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2023-09-23T22:53:05.191 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.053388039Z level=info msg="Executing migration" id="Move region to single row" 2023-09-23T22:53:05.191 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.086832123Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2023-09-23T22:53:05.191 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.112268793Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-09-23T22:53:05.191 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.154195042Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-09-23T22:53:05.343 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:53:05.344 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T22:53:05.456 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.19624396Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2023-09-23T22:53:05.456 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.221444367Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2023-09-23T22:53:05.456 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.246657616Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2023-09-23T22:53:05.456 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.288604073Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2023-09-23T22:53:05.457 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.313980865Z level=info msg="Executing migration" id="create test_data table" 2023-09-23T22:53:05.457 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.355920153Z level=info msg="Executing migration" id="create dashboard_version table v1" 2023-09-23T22:53:05.457 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.381254261Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2023-09-23T22:53:05.457 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.406261209Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-09-23T22:53:05.457 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.431564547Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2023-09-23T22:53:05.457 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.456666448Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2023-09-23T22:53:05.716 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.507246034Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2023-09-23T22:53:05.717 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.532515792Z level=info msg="Executing migration" id="create team table" 2023-09-23T22:53:05.717 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.557669103Z level=info msg="Executing migration" id="add index team.org_id" 2023-09-23T22:53:05.717 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.624669576Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2023-09-23T22:53:05.717 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.649747165Z level=info msg="Executing migration" id="create team member table" 2023-09-23T22:53:05.717 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.691667636Z level=info msg="Executing migration" id="add index team_member.org_id" 2023-09-23T22:53:05.717 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.716899334Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2023-09-23T22:53:05.977 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.750658451Z level=info msg="Executing migration" id="add index team_member.team_id" 2023-09-23T22:53:05.977 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.775811212Z level=info msg="Executing migration" id="Add column email to team table" 2023-09-23T22:53:05.977 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.826212022Z level=info msg="Executing migration" id="Add column external to team_member table" 2023-09-23T22:53:05.977 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.868039004Z level=info msg="Executing migration" id="Add column permission to team_member table" 2023-09-23T22:53:05.977 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.893387197Z level=info msg="Executing migration" id="create dashboard acl table" 2023-09-23T22:53:05.977 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.918510632Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2023-09-23T22:53:05.978 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.96051786Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2023-09-23T22:53:05.978 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:05 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:05.977328407Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2023-09-23T22:53:06.255 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.002551168Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2023-09-23T22:53:06.255 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.036312557Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2023-09-23T22:53:06.255 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.061470959Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2023-09-23T22:53:06.255 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.103442316Z level=info msg="Executing migration" id="add index dashboard_permission" 2023-09-23T22:53:06.255 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.120380385Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2023-09-23T22:53:06.255 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.137225572Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2023-09-23T22:53:06.255 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.153929441Z level=info msg="Executing migration" id="create tag table" 2023-09-23T22:53:06.255 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.17082911Z level=info msg="Executing migration" id="add index tag.key_value" 2023-09-23T22:53:06.256 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.204816103Z level=info msg="Executing migration" id="create login attempt table" 2023-09-23T22:53:06.256 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.221476614Z level=info msg="Executing migration" id="add index login_attempt.username" 2023-09-23T22:53:06.256 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.255196525Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2023-09-23T22:53:06.507 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.297391036Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-09-23T22:53:06.507 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.330914415Z level=info msg="Executing migration" id="create login_attempt v2" 2023-09-23T22:53:06.507 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.347829261Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2023-09-23T22:53:06.508 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.364763102Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2023-09-23T22:53:06.508 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.390004744Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2023-09-23T22:53:06.508 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.415197569Z level=info msg="Executing migration" id="create user auth table" 2023-09-23T22:53:06.508 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.432143902Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-09-23T22:53:06.508 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.465608014Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2023-09-23T22:53:06.508 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.5075713Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2023-09-23T22:53:06.767 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.541328073Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2023-09-23T22:53:06.768 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.566441934Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2023-09-23T22:53:06.768 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.60855052Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2023-09-23T22:53:06.768 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.650236931Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2023-09-23T22:53:06.768 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.692233286Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2023-09-23T22:53:06.768 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.717438877Z level=info msg="Executing migration" id="create server_lock table" 2023-09-23T22:53:06.768 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.742540088Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2023-09-23T22:53:06.768 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.767742673Z level=info msg="Executing migration" id="create user auth token table" 2023-09-23T22:53:06.912 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":30,"stamp":"2023-09-23T22:53:03.796633+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50360,"kb_used_data":6648,"kb_used_omap":0,"kb_used_meta":43648,"kb_avail":749910856,"statfs":{"total":767960285184,"available":767908716544,"internally_reserved":0,"allocated":6807552,"data_stored":3875051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44695552},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002085"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:23.589888+0000","last_change":"2023-09-23T22:52:21.315891+0000","last_active":"2023-09-23T22:52:23.589888+0000","last_peered":"2023-09-23T22:52:23.589888+0000","last_clean":"2023-09-23T22:52:23.589888+0000","last_became_active":"2023-09-23T22:52:21.315580+0000","last_became_peered":"2023-09-23T22:52:21.315580+0000","last_unstale":"2023-09-23T22:52:23.589888+0000","last_undegraded":"2023-09-23T22:52:23.589888+0000","last_fullsized":"2023-09-23T22:52:23.589888+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:33.692738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:23.429011+0000","last_change":"2023-09-23T22:52:21.890478+0000","last_active":"2023-09-23T22:52:23.429011+0000","last_peered":"2023-09-23T22:52:23.429011+0000","last_clean":"2023-09-23T22:52:23.429011+0000","last_became_active":"2023-09-23T22:52:21.890133+0000","last_became_peered":"2023-09-23T22:52:21.890133+0000","last_unstale":"2023-09-23T22:52:23.429011+0000","last_undegraded":"2023-09-23T22:52:23.429011+0000","last_fullsized":"2023-09-23T22:52:23.429011+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:34:57.292871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:22.313391+0000","last_change":"2023-09-23T22:52:21.890419+0000","last_active":"2023-09-23T22:52:22.313391+0000","last_peered":"2023-09-23T22:52:22.313391+0000","last_clean":"2023-09-23T22:52:22.313391+0000","last_became_active":"2023-09-23T22:52:21.890113+0000","last_became_peered":"2023-09-23T22:52:21.890113+0000","last_unstale":"2023-09-23T22:52:22.313391+0000","last_undegraded":"2023-09-23T22:52:22.313391+0000","last_fullsized":"2023-09-23T22:52:22.313391+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:23.384783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:22.313392+0000","last_change":"2023-09-23T22:52:21.315773+0000","last_active":"2023-09-23T22:52:22.313392+0000","last_peered":"2023-09-23T22:52:22.313392+0000","last_clean":"2023-09-23T22:52:22.313392+0000","last_became_active":"2023-09-23T22:52:21.315503+0000","last_became_peered":"2023-09-23T22:52:21.315503+0000","last_unstale":"2023-09-23T22:52:22.313392+0000","last_undegraded":"2023-09-23T22:52:22.313392+0000","last_fullsized":"2023-09-23T22:52:22.313392+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:39:58.756695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":270,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:22.313446+0000","last_change":"2023-09-23T22:51:47.222693+0000","last_active":"2023-09-23T22:52:22.313446+0000","last_peered":"2023-09-23T22:52:22.313446+0000","last_clean":"2023-09-23T22:52:22.313446+0000","last_became_active":"2023-09-23T22:51:47.222168+0000","last_became_peered":"2023-09-23T22:51:47.222168+0000","last_unstale":"2023-09-23T22:52:22.313446+0000","last_undegraded":"2023-09-23T22:52:22.313446+0000","last_fullsized":"2023-09-23T22:52:22.313446+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T22:49:49.548021+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:16.935480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:23.589818+0000","last_change":"2023-09-23T22:52:21.315119+0000","last_active":"2023-09-23T22:52:23.589818+0000","last_peered":"2023-09-23T22:52:23.589818+0000","last_clean":"2023-09-23T22:52:23.589818+0000","last_became_active":"2023-09-23T22:52:21.314817+0000","last_became_peered":"2023-09-23T22:52:21.314817+0000","last_unstale":"2023-09-23T22:52:23.589818+0000","last_undegraded":"2023-09-23T22:52:23.589818+0000","last_fullsized":"2023-09-23T22:52:23.589818+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:22.207385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:22.312277+0000","last_change":"2023-09-23T22:52:21.890238+0000","last_active":"2023-09-23T22:52:22.312277+0000","last_peered":"2023-09-23T22:52:22.312277+0000","last_clean":"2023-09-23T22:52:22.312277+0000","last_became_active":"2023-09-23T22:52:21.889979+0000","last_became_peered":"2023-09-23T22:52:21.889979+0000","last_unstale":"2023-09-23T22:52:22.312277+0000","last_undegraded":"2023-09-23T22:52:22.312277+0000","last_fullsized":"2023-09-23T22:52:22.312277+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:25.487367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:23.589734+0000","last_change":"2023-09-23T22:52:21.314044+0000","last_active":"2023-09-23T22:52:23.589734+0000","last_peered":"2023-09-23T22:52:23.589734+0000","last_clean":"2023-09-23T22:52:23.589734+0000","last_became_active":"2023-09-23T22:52:21.313723+0000","last_became_peered":"2023-09-23T22:52:21.313723+0000","last_unstale":"2023-09-23T22:52:23.589734+0000","last_undegraded":"2023-09-23T22:52:23.589734+0000","last_fullsized":"2023-09-23T22:52:23.589734+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:31.447410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:23.429013+0000","last_change":"2023-09-23T22:52:21.313895+0000","last_active":"2023-09-23T22:52:23.429013+0000","last_peered":"2023-09-23T22:52:23.429013+0000","last_clean":"2023-09-23T22:52:23.429013+0000","last_became_active":"2023-09-23T22:52:21.313631+0000","last_became_peered":"2023-09-23T22:52:21.313631+0000","last_unstale":"2023-09-23T22:52:23.429013+0000","last_undegraded":"2023-09-23T22:52:23.429013+0000","last_fullsized":"2023-09-23T22:52:23.429013+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:41.617398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234003,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6320,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738832,"statfs":{"total":95995035648,"available":95988563968,"internally_reserved":0,"allocated":1220608,"data_stored":853268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":5.1849999999999996,"5min":5.1849999999999996,"15min":5.1849999999999996},"last":0.63400000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.903,"5min":2.903,"15min":2.903},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.0659999999999998,"5min":5.0659999999999998,"15min":5.0659999999999998},"last":0.73699999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.0259999999999998,"5min":3.0259999999999998,"15min":3.0259999999999998},"last":0.626},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":2.649,"5min":2.649,"15min":2.649},"last":0.59199999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":4.9539999999999997,"5min":4.9539999999999997,"15min":4.9539999999999997},"last":0.70299999999999996}]},{"osd":4,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.442}]},{"osd":5,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.498}]},{"osd":6,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.57399999999999995}]}]},{"osd":6,"up_from":46,"seq":197568495639,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6380,"kb_used_data":1188,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738772,"statfs":{"total":95995035648,"available":95988502528,"internally_reserved":0,"allocated":1216512,"data_stored":853003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.53600000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.65200000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.627}]},{"osd":3,"last update":"Sat Sep 23 22:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.60399999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.70699999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.51200000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:52:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.73299999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574894,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6512,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738640,"statfs":{"total":95995035648,"available":95988367360,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.33800000000000002},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.375,"5min":0.35899999999999999,"15min":0.34499999999999997},"min":{"1min":0.26300000000000001,"5min":0.218,"15min":0.218},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.29599999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:52:53 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.41099999999999998,"15min":0.40300000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.317},{"interface":"front","average":{"1min":0.436,"5min":0.42799999999999999,"15min":0.41799999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.34799999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.41599999999999998,"15min":0.41999999999999998},"min":{"1min":0.32900000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.45700000000000002,"15min":0.46300000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.35899999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.61099999999999999,"15min":0.57599999999999996},"min":{"1min":0.38800000000000001,"5min":0.379,"15min":0.379},"max":{"1min":5.3710000000000004,"5min":5.3710000000000004,"15min":5.3710000000000004},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.56499999999999995,"15min":0.55900000000000005},"min":{"1min":0.45200000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.90200000000000002,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.53500000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:53:00 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.56399999999999995,"15min":0.56699999999999995},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.85399999999999998,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.59199999999999997,"15min":0.59799999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.94699999999999995,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.54200000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.61599999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:52:45 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":5.3390000000000004,"5min":5.3390000000000004,"15min":5.3390000000000004},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.51600000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738418,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7156,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93737996,"statfs":{"total":95995035648,"available":95987707904,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:52:28 2023","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":0.35099999999999998,"15min":0.34699999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.57399999999999995,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.36399999999999999,"15min":0.35599999999999998},"min":{"1min":0.23499999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.26900000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:52:58 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.40899999999999997,"15min":0.41699999999999998},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":0.59299999999999997,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.371,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.28699999999999998,"5min":0.249,"15min":0.249},"max":{"1min":0.621,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.29099999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:52:18 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.47299999999999998,"15min":0.47899999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.317},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.46200000000000002,"15min":0.46899999999999997},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.245}]},{"osd":4,"last update":"Sat Sep 23 22:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.56299999999999994,"15min":0.56000000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.58499999999999996,"15min":0.58799999999999997},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.36499999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:51:59 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.44800000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.438}]},{"osd":7,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.46700000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411368,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6444,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738708,"statfs":{"total":95995035648,"available":95988436992,"internally_reserved":0,"allocated":626688,"data_stored":262562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:59 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.35299999999999998,"15min":0.34200000000000003},"min":{"1min":0.24099999999999999,"5min":0.161,"15min":0.161},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.376,"5min":0.377,"15min":0.36399999999999999},"min":{"1min":0.23799999999999999,"5min":0.188,"15min":0.188},"max":{"1min":0.61899999999999999,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.25700000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:52:59 2023","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.38200000000000001,"15min":0.36699999999999999},"min":{"1min":0.24299999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.38100000000000001,"15min":0.35999999999999999},"min":{"1min":0.27700000000000002,"5min":0.22,"15min":0.22},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.40300000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.437,"15min":0.436},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.443,"15min":0.441},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.309}]},{"osd":4,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.59599999999999997,"15min":0.58499999999999996},"min":{"1min":0.44700000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.60099999999999998,"15min":0.58899999999999997},"min":{"1min":0.39700000000000002,"5min":0.379,"15min":0.379},"max":{"1min":1.05,"5min":1.05,"15min":1.05},"last":0.49199999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.42499999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59199999999999997,"15min":0.59099999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.56299999999999994}]},{"osd":6,"last update":"Sat Sep 23 22:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.55100000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:52:45 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":1.042,"5min":1.042,"15min":1.042},"last":0.54100000000000004}]}]},{"osd":3,"up_from":25,"seq":107374182436,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5872,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739280,"statfs":{"total":95995035648,"available":95989022720,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.35299999999999998,"15min":0.35099999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.51600000000000001,"5min":0.621,"15min":0.621},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.438,"15min":0.44600000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.58099999999999996,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.28999999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.42299999999999999,"15min":0.42399999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.374},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.43099999999999999,"15min":0.433},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.27800000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.45600000000000002,"15min":0.45800000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.61799999999999999,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.308},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.33500000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:52:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.57399999999999995,"15min":0.57799999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:53:01 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.57599999999999996,"15min":0.58599999999999997},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.59599999999999997,"15min":0.60199999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.52200000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:52:24 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.48099999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:52:45 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.44800000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018912,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5872,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739280,"statfs":{"total":95995035648,"available":95989022720,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.501,"15min":0.501},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.75900000000000001,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.51400000000000001,"15min":0.51800000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.55900000000000005}]},{"osd":1,"last update":"Sat Sep 23 22:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.498,"15min":0.5},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.63200000000000001,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.505},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.51300000000000001,"15min":0.50800000000000001},"min":{"1min":0.39800000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.71699999999999997,"5min":0.752,"15min":0.752},"last":0.57399999999999995}]},{"osd":2,"last update":"Sat Sep 23 22:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.502,"15min":0.502},"min":{"1min":0.37,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.66900000000000004,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.53500000000000003,"15min":0.53800000000000003},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69699999999999995,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.52000000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.51600000000000001,"15min":0.52000000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.623},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.52900000000000003,"15min":0.52700000000000002},"min":{"1min":0.35299999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.75,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.58299999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:53:03 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.44800000000000001,"15min":0.44},"min":{"1min":0.29799999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.44400000000000001,"15min":0.436},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.67700000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.45400000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:52:24 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.48899999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:52:46 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.53100000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757275,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5804,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739348,"statfs":{"total":95995035648,"available":95989092352,"internally_reserved":0,"allocated":626688,"data_stored":262958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.56699999999999995,"15min":0.57299999999999995},"min":{"1min":0.373,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79400000000000004,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.48899999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.55200000000000005,"15min":0.55700000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.70499999999999996,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.502,"5min":0.52700000000000002,"15min":0.53100000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.44700000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.53900000000000003,"15min":0.54500000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.64200000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.55600000000000005,"15min":0.55700000000000005},"min":{"1min":0.39700000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.56100000000000005,"15min":0.55900000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.67800000000000005}]},{"osd":4,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.39200000000000002,"15min":0.38300000000000001},"min":{"1min":0.318,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.505,"5min":0.439,"15min":0.42799999999999999},"min":{"1min":0.30599999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.625}]},{"osd":6,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.72099999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:52:46 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.73499999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:53:06.915 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:53:07.036 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:06 smithi138 ceph-mon[104418]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:07.036 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:06 smithi138 ceph-mon[104418]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:07.036 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.851517533Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2023-09-23T22:53:07.037 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.893634616Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2023-09-23T22:53:07.037 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.918664857Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2023-09-23T22:53:07.037 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.960706509Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2023-09-23T22:53:07.037 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:06 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:06.985987867Z level=info msg="Executing migration" id="create cache_data table" 2023-09-23T22:53:07.037 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.011262687Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2023-09-23T22:53:07.037 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.036448017Z level=info msg="Executing migration" id="create short_url table v1" 2023-09-23T22:53:07.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:06 smithi110 ceph-mon[100243]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:07.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:06 smithi110 ceph-mon[100243]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:07.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:06 smithi110 ceph-mon[118821]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:07.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:06 smithi110 ceph-mon[118821]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:07.313 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.08699825Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2023-09-23T22:53:07.313 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.112162344Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2023-09-23T22:53:07.313 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.154011133Z level=info msg="Executing migration" id="delete alert_definition table" 2023-09-23T22:53:07.313 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.179386857Z level=info msg="Executing migration" id="recreate alert_definition table" 2023-09-23T22:53:07.314 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.229884798Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2023-09-23T22:53:07.314 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.271743792Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2023-09-23T22:53:07.314 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.313656192Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2023-09-23T22:53:07.446 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:07.582 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.355674087Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2023-09-23T22:53:07.582 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.380797319Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2023-09-23T22:53:07.582 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.422792004Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2023-09-23T22:53:07.582 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.464827917Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2023-09-23T22:53:07.582 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.490067564Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2023-09-23T22:53:07.582 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.540188016Z level=info msg="Executing migration" id="drop alert_definition table" 2023-09-23T22:53:07.659 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.354Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T22:53:07.659 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.354Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T22:53:07.659 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T22:53:07.659 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T22:53:07.659 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T22:53:07.659 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T22:53:07.660 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T22:53:07.660 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T22:53:07.660 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T22:53:07.660 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T22:53:07.660 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T22:53:07.660 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T22:53:07.660 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T22:53:07.660 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T22:53:07.660 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T22:53:07.660 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T22:53:07.660 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T22:53:07.660 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T22:53:07.661 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T22:53:07.661 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T22:53:07.661 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T22:53:07.661 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T22:53:07.661 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T22:53:07.661 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T22:53:07.661 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T22:53:07.661 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T22:53:07.661 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T22:53:07.661 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T22:53:07.661 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T22:53:07.661 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T22:53:07.661 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T22:53:07.662 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T22:53:07.662 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T22:53:07.662 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T22:53:07.662 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T22:53:07.662 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T22:53:07.662 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T22:53:07.662 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T22:53:07.662 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T22:53:07.662 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T22:53:07.662 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T22:53:07.662 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T22:53:07.662 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T22:53:07.663 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T22:53:07.663 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T22:53:07.663 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T22:53:07.663 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.355Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T22:53:07.663 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.356Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T22:53:07.663 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.356Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T22:53:07.663 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.356Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T22:53:07.663 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.356Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T22:53:07.663 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.356Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T22:53:07.663 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 conmon[171849]: ts=2023-09-23T22:53:07.356Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T22:53:07.663 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 bash[171339]: 9d830b481d84cb61e561254f5a2226b49f7826684400f028a1603d16ffee0673 2023-09-23T22:53:07.663 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:53:07 smithi110 systemd[1]: Started Ceph node-exporter.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:53:07.834 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.582188707Z level=info msg="Executing migration" id="delete alert_definition_version table" 2023-09-23T22:53:07.834 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.632710578Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2023-09-23T22:53:07.834 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.67459408Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-09-23T22:53:07.834 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.699776554Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-09-23T22:53:07.834 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.741727438Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2023-09-23T22:53:07.834 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.783672256Z level=info msg="Executing migration" id="drop alert_definition_version table" 2023-09-23T22:53:07.834 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.800567691Z level=info msg="Executing migration" id="create alert_instance table" 2023-09-23T22:53:07.835 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.83415984Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-09-23T22:53:08.120 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.884648537Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2023-09-23T22:53:08.120 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.926884601Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2023-09-23T22:53:08.120 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.968790964Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-09-23T22:53:08.120 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:07 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:07.994067631Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2023-09-23T22:53:08.120 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.06970668Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2023-09-23T22:53:08.120 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.119802828Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2023-09-23T22:53:08.404 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:08 smithi138 systemd[1]: Starting Ceph node-exporter.b for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:53:08.404 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.170355488Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-09-23T22:53:08.404 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.195520548Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2023-09-23T22:53:08.404 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.237606581Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2023-09-23T22:53:08.404 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.279387263Z level=info msg="Executing migration" id="create alert_rule table" 2023-09-23T22:53:08.405 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.313420073Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2023-09-23T22:53:08.405 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.330142756Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2023-09-23T22:53:08.405 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.363672194Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-09-23T22:53:08.682 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:08 smithi138 bash[151909]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-23T22:53:08.683 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:08 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:08.683 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:08 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:08.683 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:08 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:08.683 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:08 smithi138 ceph-mon[104418]: Deploying daemon node-exporter.b on smithi138 2023-09-23T22:53:08.683 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:08 smithi138 ceph-mon[104418]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:08.684 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.405837274Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2023-09-23T22:53:08.684 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.431047163Z level=info msg="Executing migration" id="add column for to alert_rule" 2023-09-23T22:53:08.684 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.472997465Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2023-09-23T22:53:08.684 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.514774434Z level=info msg="Executing migration" id="add column labels to alert_rule" 2023-09-23T22:53:08.684 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.540183246Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2023-09-23T22:53:08.684 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.565202375Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-09-23T22:53:08.684 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.590359432Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2023-09-23T22:53:08.684 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.615723516Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2023-09-23T22:53:08.684 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.640979003Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-09-23T22:53:08.685 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.682747215Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2023-09-23T22:53:08.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:08 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:08.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:08 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:08.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:08 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:08.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:08 smithi110 ceph-mon[100243]: Deploying daemon node-exporter.b on smithi138 2023-09-23T22:53:08.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:08 smithi110 ceph-mon[100243]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:08.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:08 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:08.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:08 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:08.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:08 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:08.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:08 smithi110 ceph-mon[118821]: Deploying daemon node-exporter.b on smithi138 2023-09-23T22:53:08.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:08 smithi110 ceph-mon[118821]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:08.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.716623257Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2023-09-23T22:53:08.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.741594931Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2023-09-23T22:53:08.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.783718697Z level=info msg="Executing migration" id="create alert_rule_version table" 2023-09-23T22:53:08.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.825543097Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-09-23T22:53:08.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.867440888Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-09-23T22:53:08.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.892767088Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2023-09-23T22:53:08.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.918105399Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2023-09-23T22:53:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:53:09.192 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T22:53:09.286 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:09 smithi138 bash[151909]: Getting image source signatures 2023-09-23T22:53:09.287 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.968350962Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2023-09-23T22:53:09.287 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:08 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:08.993549275Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2023-09-23T22:53:09.287 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.035519229Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2023-09-23T22:53:09.287 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.060622279Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2023-09-23T22:53:09.287 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.086016312Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2023-09-23T22:53:09.287 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.111055178Z level=info msg="Executing migration" id=create_alert_configuration_table 2023-09-23T22:53:09.287 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.153055612Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2023-09-23T22:53:09.287 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.195015404Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-09-23T22:53:09.572 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:09 smithi138 bash[151909]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T22:53:09.572 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:09 smithi138 bash[151909]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T22:53:09.572 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:09 smithi138 bash[151909]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T22:53:09.572 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.28695365Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2023-09-23T22:53:09.572 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.404321053Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2023-09-23T22:53:09.572 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.488883974Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2023-09-23T22:53:09.572 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.572030022Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2023-09-23T22:53:09.823 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.614039862Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2023-09-23T22:53:09.823 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.6893956Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2023-09-23T22:53:09.823 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.781397473Z level=info msg="Executing migration" id="create provenance_type table" 2023-09-23T22:53:09.823 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.823478193Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2023-09-23T22:53:09.885 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":32,"stamp":"2023-09-23T22:53:07.797454+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50360,"kb_used_data":6648,"kb_used_omap":0,"kb_used_meta":43648,"kb_avail":749910856,"statfs":{"total":767960285184,"available":767908716544,"internally_reserved":0,"allocated":6807552,"data_stored":3875051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44695552},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002210"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:23.589888+0000","last_change":"2023-09-23T22:52:21.315891+0000","last_active":"2023-09-23T22:52:23.589888+0000","last_peered":"2023-09-23T22:52:23.589888+0000","last_clean":"2023-09-23T22:52:23.589888+0000","last_became_active":"2023-09-23T22:52:21.315580+0000","last_became_peered":"2023-09-23T22:52:21.315580+0000","last_unstale":"2023-09-23T22:52:23.589888+0000","last_undegraded":"2023-09-23T22:52:23.589888+0000","last_fullsized":"2023-09-23T22:52:23.589888+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:33.692738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:23.429011+0000","last_change":"2023-09-23T22:52:21.890478+0000","last_active":"2023-09-23T22:52:23.429011+0000","last_peered":"2023-09-23T22:52:23.429011+0000","last_clean":"2023-09-23T22:52:23.429011+0000","last_became_active":"2023-09-23T22:52:21.890133+0000","last_became_peered":"2023-09-23T22:52:21.890133+0000","last_unstale":"2023-09-23T22:52:23.429011+0000","last_undegraded":"2023-09-23T22:52:23.429011+0000","last_fullsized":"2023-09-23T22:52:23.429011+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:34:57.292871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:22.313391+0000","last_change":"2023-09-23T22:52:21.890419+0000","last_active":"2023-09-23T22:52:22.313391+0000","last_peered":"2023-09-23T22:52:22.313391+0000","last_clean":"2023-09-23T22:52:22.313391+0000","last_became_active":"2023-09-23T22:52:21.890113+0000","last_became_peered":"2023-09-23T22:52:21.890113+0000","last_unstale":"2023-09-23T22:52:22.313391+0000","last_undegraded":"2023-09-23T22:52:22.313391+0000","last_fullsized":"2023-09-23T22:52:22.313391+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:23.384783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:22.313392+0000","last_change":"2023-09-23T22:52:21.315773+0000","last_active":"2023-09-23T22:52:22.313392+0000","last_peered":"2023-09-23T22:52:22.313392+0000","last_clean":"2023-09-23T22:52:22.313392+0000","last_became_active":"2023-09-23T22:52:21.315503+0000","last_became_peered":"2023-09-23T22:52:21.315503+0000","last_unstale":"2023-09-23T22:52:22.313392+0000","last_undegraded":"2023-09-23T22:52:22.313392+0000","last_fullsized":"2023-09-23T22:52:22.313392+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:39:58.756695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":270,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:22.313446+0000","last_change":"2023-09-23T22:51:47.222693+0000","last_active":"2023-09-23T22:52:22.313446+0000","last_peered":"2023-09-23T22:52:22.313446+0000","last_clean":"2023-09-23T22:52:22.313446+0000","last_became_active":"2023-09-23T22:51:47.222168+0000","last_became_peered":"2023-09-23T22:51:47.222168+0000","last_unstale":"2023-09-23T22:52:22.313446+0000","last_undegraded":"2023-09-23T22:52:22.313446+0000","last_fullsized":"2023-09-23T22:52:22.313446+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T22:49:49.548021+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:16.935480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:23.589818+0000","last_change":"2023-09-23T22:52:21.315119+0000","last_active":"2023-09-23T22:52:23.589818+0000","last_peered":"2023-09-23T22:52:23.589818+0000","last_clean":"2023-09-23T22:52:23.589818+0000","last_became_active":"2023-09-23T22:52:21.314817+0000","last_became_peered":"2023-09-23T22:52:21.314817+0000","last_unstale":"2023-09-23T22:52:23.589818+0000","last_undegraded":"2023-09-23T22:52:23.589818+0000","last_fullsized":"2023-09-23T22:52:23.589818+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:22.207385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:22.312277+0000","last_change":"2023-09-23T22:52:21.890238+0000","last_active":"2023-09-23T22:52:22.312277+0000","last_peered":"2023-09-23T22:52:22.312277+0000","last_clean":"2023-09-23T22:52:22.312277+0000","last_became_active":"2023-09-23T22:52:21.889979+0000","last_became_peered":"2023-09-23T22:52:21.889979+0000","last_unstale":"2023-09-23T22:52:22.312277+0000","last_undegraded":"2023-09-23T22:52:22.312277+0000","last_fullsized":"2023-09-23T22:52:22.312277+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:25.487367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:23.589734+0000","last_change":"2023-09-23T22:52:21.314044+0000","last_active":"2023-09-23T22:52:23.589734+0000","last_peered":"2023-09-23T22:52:23.589734+0000","last_clean":"2023-09-23T22:52:23.589734+0000","last_became_active":"2023-09-23T22:52:21.313723+0000","last_became_peered":"2023-09-23T22:52:21.313723+0000","last_unstale":"2023-09-23T22:52:23.589734+0000","last_undegraded":"2023-09-23T22:52:23.589734+0000","last_fullsized":"2023-09-23T22:52:23.589734+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:31.447410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:52:23.429013+0000","last_change":"2023-09-23T22:52:21.313895+0000","last_active":"2023-09-23T22:52:23.429013+0000","last_peered":"2023-09-23T22:52:23.429013+0000","last_clean":"2023-09-23T22:52:23.429013+0000","last_became_active":"2023-09-23T22:52:21.313631+0000","last_became_peered":"2023-09-23T22:52:21.313631+0000","last_unstale":"2023-09-23T22:52:23.429013+0000","last_undegraded":"2023-09-23T22:52:23.429013+0000","last_fullsized":"2023-09-23T22:52:23.429013+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:41.617398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234003,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6320,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738832,"statfs":{"total":95995035648,"available":95988563968,"internally_reserved":0,"allocated":1220608,"data_stored":853268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":5.1849999999999996,"5min":5.1849999999999996,"15min":5.1849999999999996},"last":0.63400000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.903,"5min":2.903,"15min":2.903},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.0659999999999998,"5min":5.0659999999999998,"15min":5.0659999999999998},"last":0.73699999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.0259999999999998,"5min":3.0259999999999998,"15min":3.0259999999999998},"last":0.626},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":2.649,"5min":2.649,"15min":2.649},"last":0.59199999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":4.9539999999999997,"5min":4.9539999999999997,"15min":4.9539999999999997},"last":0.70299999999999996}]},{"osd":4,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.442}]},{"osd":5,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.498}]},{"osd":6,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.57399999999999995}]}]},{"osd":6,"up_from":46,"seq":197568495639,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6380,"kb_used_data":1188,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738772,"statfs":{"total":95995035648,"available":95988502528,"internally_reserved":0,"allocated":1216512,"data_stored":853003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.53600000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.65200000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.627}]},{"osd":3,"last update":"Sat Sep 23 22:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.60399999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.70699999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.51200000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:52:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.73299999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574894,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6512,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738640,"statfs":{"total":95995035648,"available":95988367360,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.33800000000000002},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.375,"5min":0.35899999999999999,"15min":0.34499999999999997},"min":{"1min":0.26300000000000001,"5min":0.218,"15min":0.218},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.29599999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:52:53 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.41099999999999998,"15min":0.40300000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.317},{"interface":"front","average":{"1min":0.436,"5min":0.42799999999999999,"15min":0.41799999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.34799999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.41599999999999998,"15min":0.41999999999999998},"min":{"1min":0.32900000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.45700000000000002,"15min":0.46300000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.35899999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.61099999999999999,"15min":0.57599999999999996},"min":{"1min":0.38800000000000001,"5min":0.379,"15min":0.379},"max":{"1min":5.3710000000000004,"5min":5.3710000000000004,"15min":5.3710000000000004},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.56499999999999995,"15min":0.55900000000000005},"min":{"1min":0.45200000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.90200000000000002,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.53500000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:53:00 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.56399999999999995,"15min":0.56699999999999995},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.85399999999999998,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.59199999999999997,"15min":0.59799999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.94699999999999995,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.54200000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.61599999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:52:45 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":5.3390000000000004,"5min":5.3390000000000004,"15min":5.3390000000000004},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.51600000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738418,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7156,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93737996,"statfs":{"total":95995035648,"available":95987707904,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:52:28 2023","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":0.35099999999999998,"15min":0.34699999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.57399999999999995,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.36399999999999999,"15min":0.35599999999999998},"min":{"1min":0.23499999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.26900000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:52:58 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.40899999999999997,"15min":0.41699999999999998},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":0.59299999999999997,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.371,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.28699999999999998,"5min":0.249,"15min":0.249},"max":{"1min":0.621,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.29099999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:52:18 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.47299999999999998,"15min":0.47899999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.317},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.46200000000000002,"15min":0.46899999999999997},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.245}]},{"osd":4,"last update":"Sat Sep 23 22:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.56299999999999994,"15min":0.56000000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.58499999999999996,"15min":0.58799999999999997},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.36499999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:51:59 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.44800000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.438}]},{"osd":7,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.46700000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411368,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6444,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738708,"statfs":{"total":95995035648,"available":95988436992,"internally_reserved":0,"allocated":626688,"data_stored":262562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:59 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.35299999999999998,"15min":0.34200000000000003},"min":{"1min":0.24099999999999999,"5min":0.161,"15min":0.161},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.376,"5min":0.377,"15min":0.36399999999999999},"min":{"1min":0.23799999999999999,"5min":0.188,"15min":0.188},"max":{"1min":0.61899999999999999,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.25700000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:52:59 2023","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.38200000000000001,"15min":0.36699999999999999},"min":{"1min":0.24299999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.38100000000000001,"15min":0.35999999999999999},"min":{"1min":0.27700000000000002,"5min":0.22,"15min":0.22},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.40300000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.437,"15min":0.436},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.443,"15min":0.441},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.309}]},{"osd":4,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.59599999999999997,"15min":0.58499999999999996},"min":{"1min":0.44700000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.60099999999999998,"15min":0.58899999999999997},"min":{"1min":0.39700000000000002,"5min":0.379,"15min":0.379},"max":{"1min":1.05,"5min":1.05,"15min":1.05},"last":0.49199999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.42499999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59199999999999997,"15min":0.59099999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.56299999999999994}]},{"osd":6,"last update":"Sat Sep 23 22:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.55100000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:52:45 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":1.042,"5min":1.042,"15min":1.042},"last":0.54100000000000004}]}]},{"osd":3,"up_from":25,"seq":107374182436,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5872,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739280,"statfs":{"total":95995035648,"available":95989022720,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.35299999999999998,"15min":0.35099999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.51600000000000001,"5min":0.621,"15min":0.621},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.438,"15min":0.44600000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.58099999999999996,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.28999999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.42299999999999999,"15min":0.42399999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.374},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.43099999999999999,"15min":0.433},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.27800000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.45600000000000002,"15min":0.45800000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.61799999999999999,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.308},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.33500000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:52:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.57399999999999995,"15min":0.57799999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:53:01 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.57599999999999996,"15min":0.58599999999999997},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.59599999999999997,"15min":0.60199999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.52200000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:52:24 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.48099999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:52:45 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.44800000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018912,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5872,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739280,"statfs":{"total":95995035648,"available":95989022720,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.501,"15min":0.501},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.75900000000000001,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.51400000000000001,"15min":0.51800000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.55900000000000005}]},{"osd":1,"last update":"Sat Sep 23 22:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.498,"15min":0.5},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.63200000000000001,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.505},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.51300000000000001,"15min":0.50800000000000001},"min":{"1min":0.39800000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.71699999999999997,"5min":0.752,"15min":0.752},"last":0.57399999999999995}]},{"osd":2,"last update":"Sat Sep 23 22:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.502,"15min":0.502},"min":{"1min":0.37,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.66900000000000004,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.53500000000000003,"15min":0.53800000000000003},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69699999999999995,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.52000000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.51600000000000001,"15min":0.52000000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.623},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.52900000000000003,"15min":0.52700000000000002},"min":{"1min":0.35299999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.75,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.58299999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:53:03 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.44800000000000001,"15min":0.44},"min":{"1min":0.29799999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.44400000000000001,"15min":0.436},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.67700000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.45400000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:52:24 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.48899999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:52:46 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.53100000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757275,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5804,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739348,"statfs":{"total":95995035648,"available":95989092352,"internally_reserved":0,"allocated":626688,"data_stored":262958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.56699999999999995,"15min":0.57299999999999995},"min":{"1min":0.373,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79400000000000004,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.48899999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.55200000000000005,"15min":0.55700000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.70499999999999996,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.502,"5min":0.52700000000000002,"15min":0.53100000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.44700000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.53900000000000003,"15min":0.54500000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.64200000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.55600000000000005,"15min":0.55700000000000005},"min":{"1min":0.39700000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.56100000000000005,"15min":0.55900000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.67800000000000005}]},{"osd":4,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.39200000000000002,"15min":0.38300000000000001},"min":{"1min":0.318,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.505,"5min":0.439,"15min":0.42799999999999999},"min":{"1min":0.30599999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.625}]},{"osd":6,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.72099999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:52:46 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.73499999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:53:09.888 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T22:53:09.888 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T22:53:09.888 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T22:53:09.888 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T22:53:10.109 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.86551752Z level=info msg="Executing migration" id="create alert_image table" 2023-09-23T22:53:10.109 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.907470228Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2023-09-23T22:53:10.109 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.949434434Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2023-09-23T22:53:10.109 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:09 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:09.991429952Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2023-09-23T22:53:10.109 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.025054958Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2023-09-23T22:53:10.109 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.067227453Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2023-09-23T22:53:10.109 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.067333842Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2023-09-23T22:53:10.109 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.108971805Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2023-09-23T22:53:10.233 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:10.402 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:10 smithi138 ceph-mon[104418]: from='client.24524 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:10.402 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:10 smithi138 ceph-mon[104418]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:10.403 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.150962352Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2023-09-23T22:53:10.403 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.192941414Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2023-09-23T22:53:10.403 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.193090794Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2023-09-23T22:53:10.403 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.193225082Z level=info msg="alerts found to migrate" alerts=0 2023-09-23T22:53:10.403 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.234815174Z level=info msg="Executing migration" id="create library_element table v1" 2023-09-23T22:53:10.403 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.276832189Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2023-09-23T22:53:10.403 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.318728153Z level=info msg="Executing migration" id="create library_element_connection table v1" 2023-09-23T22:53:10.403 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.402651263Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2023-09-23T22:53:10.633 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:10 smithi110 ceph-mon[100243]: from='client.24524 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:10.633 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:10 smithi110 ceph-mon[100243]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:10.633 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:10 smithi110 ceph-mon[118821]: from='client.24524 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:10.633 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:10 smithi110 ceph-mon[118821]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:10.670 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:10 smithi138 bash[151909]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-23T22:53:10.671 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.522036139Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2023-09-23T22:53:10.671 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.605344652Z level=info msg="Executing migration" id="increase max description length to 2048" 2023-09-23T22:53:10.932 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:10 smithi138 bash[151909]: Writing manifest to image destination 2023-09-23T22:53:10.932 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:10 smithi138 bash[151909]: Storing signatures 2023-09-23T22:53:10.932 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.670701547Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2023-09-23T22:53:10.932 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.750450953Z level=info msg="Executing migration" id="create data_keys table" 2023-09-23T22:53:10.932 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.813981582Z level=info msg="Executing migration" id="create secrets table" 2023-09-23T22:53:10.933 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.856574504Z level=info msg="Executing migration" id="rename data_keys name column to id" 2023-09-23T22:53:10.933 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:10 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:10.932277217Z level=info msg="Executing migration" id="add name column into data_keys" 2023-09-23T22:53:11.191 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:11 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:11.016678356Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2023-09-23T22:53:11.191 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:11 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:11.05895085Z level=info msg="Executing migration" id="rename data_keys name column to label" 2023-09-23T22:53:11.191 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:11 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:11.102158385Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2023-09-23T22:53:11.191 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:11 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:11.156959223Z level=info msg="Executing migration" id="create kv_store table v1" 2023-09-23T22:53:11.537 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:11 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:11.250402011Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2023-09-23T22:53:11.537 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:11 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:11.309581293Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2023-09-23T22:53:11.538 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:11 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:11.368638318Z level=info msg="Executing migration" id="create permission table" 2023-09-23T22:53:11.538 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:11 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:11.410748981Z level=info msg="Executing migration" id="add unique index permission.role_id" 2023-09-23T22:53:11.538 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:11 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:11.452762189Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2023-09-23T22:53:11.538 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:11 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:11.494767013Z level=info msg="Executing migration" id="create role table" 2023-09-23T22:53:11.806 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:11 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:11.537039421Z level=info msg="Executing migration" id="add column display_name" 2023-09-23T22:53:11.806 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:11 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:11.621337082Z level=info msg="Executing migration" id="add column group_name" 2023-09-23T22:53:11.806 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:11 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:11.730535145Z level=info msg="Executing migration" id="add index role.org_id" 2023-09-23T22:53:11.959 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:53:11.959 INFO:teuthology.orchestra.run.smithi110.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T22:53:12.059 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:53:11 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:53:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:53:12.059 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:11 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:11.806362237Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2023-09-23T22:53:12.059 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:11 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:11.890608153Z level=info msg="Executing migration" id="add index role_org_id_uid" 2023-09-23T22:53:12.059 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:11 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:11.974756892Z level=info msg="Executing migration" id="create team role table" 2023-09-23T22:53:12.379 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.058997959Z level=info msg="Executing migration" id="add index team_role.org_id" 2023-09-23T22:53:12.379 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.126195686Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2023-09-23T22:53:12.380 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.210777449Z level=info msg="Executing migration" id="add index team_role.team_id" 2023-09-23T22:53:12.380 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.23612658Z level=info msg="Executing migration" id="create user role table" 2023-09-23T22:53:12.380 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.298598815Z level=info msg="Executing migration" id="add index user_role.org_id" 2023-09-23T22:53:12.380 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.379314215Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2023-09-23T22:53:12.589 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T22:53:12.589 INFO:tasks.cephadm:Setup complete, yielding 2023-09-23T22:53:12.590 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-09-23T22:53:12.603 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi110.front.sepia.ceph.com 2023-09-23T22:53:12.603 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-09-23T22:53:12.665 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.429860618Z level=info msg="Executing migration" id="add index user_role.user_id" 2023-09-23T22:53:12.665 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.472465766Z level=info msg="Executing migration" id="create builtin role table" 2023-09-23T22:53:12.665 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.497468482Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2023-09-23T22:53:12.665 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.547856617Z level=info msg="Executing migration" id="add index builtin_role.name" 2023-09-23T22:53:12.665 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.619620759Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2023-09-23T22:53:12.666 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.570Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T22:53:12.666 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.570Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T22:53:12.666 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.570Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T22:53:12.666 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.570Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T22:53:12.666 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T22:53:12.666 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T22:53:12.667 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T22:53:12.667 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T22:53:12.667 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T22:53:12.667 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T22:53:12.667 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T22:53:12.667 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T22:53:12.667 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T22:53:12.667 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T22:53:12.667 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T22:53:12.667 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T22:53:12.667 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T22:53:12.668 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T22:53:12.668 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T22:53:12.668 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T22:53:12.668 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T22:53:12.668 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T22:53:12.668 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T22:53:12.668 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T22:53:12.668 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T22:53:12.668 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T22:53:12.668 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T22:53:12.668 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T22:53:12.669 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T22:53:12.669 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T22:53:12.669 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T22:53:12.669 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T22:53:12.669 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T22:53:12.669 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T22:53:12.669 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T22:53:12.669 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T22:53:12.669 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T22:53:12.669 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T22:53:12.669 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T22:53:12.669 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T22:53:12.669 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T22:53:12.670 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T22:53:12.670 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T22:53:12.670 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T22:53:12.670 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T22:53:12.670 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T22:53:12.670 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T22:53:12.670 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T22:53:12.670 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T22:53:12.670 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T22:53:12.670 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.571Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T22:53:12.670 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.572Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T22:53:12.670 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151991]: ts=2023-09-23T22:53:12.572Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T22:53:12.670 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 bash[151909]: 68cfd3f377d7fbf19f8732cd5fc5cf1f6d25743ca7dee624bd5c2494cfb0d89e 2023-09-23T22:53:12.671 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:53:12 smithi138 systemd[1]: Started Ceph node-exporter.b for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:53:12.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:12 smithi138 ceph-mon[104418]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:12.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:12 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3331015390' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:53:12.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:12 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:12.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:12 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:12.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:12 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:12.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:12 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:12.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:12 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:12.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.665328625Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2023-09-23T22:53:12.942 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.741160485Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2023-09-23T22:53:12.942 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.766414671Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2023-09-23T22:53:12.942 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.791696914Z level=info msg="Executing migration" id="add unique index role.uid" 2023-09-23T22:53:12.942 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.833422967Z level=info msg="Executing migration" id="create seed assignment table" 2023-09-23T22:53:12.942 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.908748836Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2023-09-23T22:53:12.965 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:13.148 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:12 smithi110 ceph-mon[100243]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:13.148 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:12 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3331015390' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:53:13.148 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:12 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:13.148 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:12 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:13.148 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:12 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:13.148 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:12 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:13.149 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:12 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:13.149 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:12 smithi110 ceph-mon[118821]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:13.149 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:12 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3331015390' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:53:13.149 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:12 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:13.149 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:12 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:13.149 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:12 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:13.149 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:12 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:13.149 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:12 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:13.245 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.955889168Z level=info msg="Executing migration" id="add column hidden to role table" 2023-09-23T22:53:13.245 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:12 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:12.984623054Z level=info msg="Executing migration" id="create query_history table v1" 2023-09-23T22:53:13.245 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.026523628Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2023-09-23T22:53:13.245 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.076751405Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2023-09-23T22:53:13.245 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.118765203Z level=info msg="Executing migration" id="teams permissions migration" 2023-09-23T22:53:13.245 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.152373178Z level=info msg="Executing migration" id="dashboard permissions" 2023-09-23T22:53:13.245 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.177690781Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2023-09-23T22:53:13.245 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.202817875Z level=info msg="Executing migration" id="drop managed folder create actions" 2023-09-23T22:53:13.245 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.24481053Z level=info msg="Executing migration" id="alerting notification permissions" 2023-09-23T22:53:13.536 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.295346948Z level=info msg="Executing migration" id="create query_history_star table v1" 2023-09-23T22:53:13.536 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.320605792Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2023-09-23T22:53:13.536 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.345909464Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2023-09-23T22:53:13.536 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.38771149Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2023-09-23T22:53:13.536 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.454894713Z level=info msg="Executing migration" id="create correlation table v1" 2023-09-23T22:53:13.850 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.536103261Z level=info msg="Executing migration" id="add index correlations.uid" 2023-09-23T22:53:13.850 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.598412292Z level=info msg="Executing migration" id="add index correlations.source_uid" 2023-09-23T22:53:13.850 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.673294011Z level=info msg="Executing migration" id="add correlation config column" 2023-09-23T22:53:13.850 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.698375152Z level=info msg="Executing migration" id="create entity_events table" 2023-09-23T22:53:13.850 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.765609297Z level=info msg="Executing migration" id="create dashboard public config v1" 2023-09-23T22:53:13.850 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.849916413Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-23T22:53:13.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:13 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:53:13.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:13 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:13.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:13 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:53:13.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:13 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:14.118 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:13 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:53:14.118 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:13 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:14.118 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.850032316Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-23T22:53:14.118 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.899933025Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T22:53:14.118 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.900115658Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T22:53:14.118 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.941870213Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2023-09-23T22:53:14.118 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.967145724Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2023-09-23T22:53:14.119 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:13 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:13.992353311Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2023-09-23T22:53:14.119 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.042603592Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T22:53:14.378 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.117914233Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2023-09-23T22:53:14.378 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.168505735Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-23T22:53:14.378 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.193834679Z level=info msg="Executing migration" id="Drop public config table" 2023-09-23T22:53:14.378 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.260538555Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2023-09-23T22:53:14.379 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.28581895Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2023-09-23T22:53:14.379 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.32771378Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-23T22:53:14.379 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.344641723Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2023-09-23T22:53:14.680 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.37853009Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2023-09-23T22:53:14.680 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.472164021Z level=info msg="Executing migration" id="add annotations_enabled column" 2023-09-23T22:53:14.680 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.495921781Z level=info msg="Executing migration" id="add time_selection_enabled column" 2023-09-23T22:53:14.681 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.54637897Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2023-09-23T22:53:14.681 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.596942417Z level=info msg="Executing migration" id="add share column" 2023-09-23T22:53:14.681 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.680826335Z level=info msg="Executing migration" id="create default alerting folders" 2023-09-23T22:53:14.932 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.762539165Z level=info msg="Executing migration" id="create file table" 2023-09-23T22:53:14.933 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.798350136Z level=info msg="Executing migration" id="file table idx: path natural pk" 2023-09-23T22:53:14.933 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.840276705Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2023-09-23T22:53:14.933 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.882278445Z level=info msg="Executing migration" id="create file_meta table" 2023-09-23T22:53:14.933 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.907446682Z level=info msg="Executing migration" id="file table idx: path key" 2023-09-23T22:53:14.933 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:14 smithi138 ceph-mon[104418]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:15.191 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:14 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.932873917Z level=info msg="Executing migration" id="set path collation in file table" 2023-09-23T22:53:15.191 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:14.999988777Z level=info msg="Executing migration" id="managed permissions migration" 2023-09-23T22:53:15.191 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.075266748Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2023-09-23T22:53:15.191 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.117440144Z level=info msg="Executing migration" id="RBAC action name migrator" 2023-09-23T22:53:15.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:14 smithi110 ceph-mon[100243]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:15.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:14 smithi110 ceph-mon[118821]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:15.511 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.202496745Z level=info msg="Executing migration" id="Add UID column to playlist" 2023-09-23T22:53:15.512 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.243486223Z level=info msg="Executing migration" id="Update uid column values in playlist" 2023-09-23T22:53:15.512 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.2855666Z level=info msg="Executing migration" id="Add index for uid in playlist" 2023-09-23T22:53:15.512 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.377555622Z level=info msg="Executing migration" id="update group index for alert rules" 2023-09-23T22:53:15.512 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.43573011Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2023-09-23T22:53:15.512 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.511872444Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2023-09-23T22:53:15.791 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.537225357Z level=info msg="Executing migration" id="add action column to seed_assignment" 2023-09-23T22:53:15.791 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.570920992Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2023-09-23T22:53:15.791 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.587808442Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2023-09-23T22:53:15.791 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.613057627Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2023-09-23T22:53:15.791 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.646701128Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2023-09-23T22:53:15.791 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.705329102Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2023-09-23T22:53:15.792 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.789977367Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2023-09-23T22:53:16.077 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:15 smithi138 ceph-mon[104418]: from='client.24536 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:53:16.077 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:15 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T22:53:16.077 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.840089961Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2023-09-23T22:53:16.078 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.890410517Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2023-09-23T22:53:16.078 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:15 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:15.94917111Z level=info msg="Executing migration" id="create folder table" 2023-09-23T22:53:16.078 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:16 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:16.019083201Z level=info msg="Executing migration" id="Add index for parent_uid" 2023-09-23T22:53:16.078 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:16 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:16.077434889Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2023-09-23T22:53:16.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:15 smithi110 ceph-mon[100243]: from='client.24536 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:53:16.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:15 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T22:53:16.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:15 smithi110 ceph-mon[118821]: from='client.24536 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:53:16.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:15 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T22:53:16.351 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:16 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:16.158820469Z level=info msg="Executing migration" id="Update folder title length" 2023-09-23T22:53:16.351 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:16 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:16.200883572Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2023-09-23T22:53:16.351 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:16 smithi138 conmon[151484]: logger=migrator t=2023-09-23T22:53:16.226082169Z level=info msg="migrations completed" performed=485 skipped=0 duration=18.51893584s 2023-09-23T22:53:16.351 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:16 smithi138 conmon[151484]: logger=sqlstore t=2023-09-23T22:53:16.226623488Z level=info msg="Created default organization" 2023-09-23T22:53:16.351 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:16 smithi138 conmon[151484]: logger=plugin.loader t=2023-09-23T22:53:16.324518865Z level=info msg="Plugin registered" pluginID=input 2023-09-23T22:53:16.351 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:16 smithi138 conmon[151484]: logger=plugin.loader t=2023-09-23T22:53:16.351351828Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-23T22:53:16.352 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:16 smithi138 conmon[151484]: logger=plugin.loader t=2023-09-23T22:53:16.351371376Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-23T22:53:16.691 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:16 smithi138 conmon[151484]: logger=secrets t=2023-09-23T22:53:16.351421184Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-23T22:53:16.691 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:16 smithi138 conmon[151484]: logger=query_data t=2023-09-23T22:53:16.353089036Z level=info msg="Query Service initialization" 2023-09-23T22:53:16.691 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:16 smithi138 conmon[151484]: logger=live.push_http t=2023-09-23T22:53:16.355732282Z level=info msg="Live Push Gateway initialization" 2023-09-23T22:53:17.099 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[100243]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:17.099 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T22:53:17.099 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[100243]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T22:53:17.099 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T22:53:17.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:53:17.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:53:17.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:16 smithi110 conmon[100219]: 2023-09-23T22:53:16.880+0000 7fbe0a999700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T22:53:17.100 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[118821]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:17.100 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T22:53:17.100 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[118821]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T22:53:17.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T22:53:17.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:53:17.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:53:17.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:16 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.156 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:17 smithi138 conmon[151484]: logger=infra.usagestats.collector t=2023-09-23T22:53:17.123915866Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-23T22:53:17.156 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:17 smithi138 conmon[151484]: logger=provisioning.datasources t=2023-09-23T22:53:17.12428833Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-09-23T22:53:17.157 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:17 smithi138 conmon[151484]: logger=provisioning.datasources t=2023-09-23T22:53:17.156818144Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T22:53:17.157 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:16 smithi138 ceph-mon[104418]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:17.157 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:16 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T22:53:17.157 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:16 smithi138 ceph-mon[104418]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T22:53:17.157 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:16 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T22:53:17.157 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:16 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.157 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:16 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.157 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:16 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.157 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:16 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.157 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:16 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:53:17.158 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:16 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:53:17.158 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:16 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.441 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:17 smithi138 conmon[151484]: logger=provisioning.datasources t=2023-09-23T22:53:17.213001377Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2023-09-23T22:53:17.441 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:17 smithi138 conmon[151484]: logger=provisioning.alerting t=2023-09-23T22:53:17.2782632Z level=info msg="starting to provision alerting" 2023-09-23T22:53:17.441 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:17 smithi138 conmon[151484]: logger=provisioning.alerting t=2023-09-23T22:53:17.278283909Z level=info msg="finished to provision alerting" 2023-09-23T22:53:17.441 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:17 smithi138 conmon[151484]: logger=grafanaStorageLogger t=2023-09-23T22:53:17.278447106Z level=info msg="storage starting" 2023-09-23T22:53:17.442 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:17 smithi138 conmon[151484]: logger=ngalert.state.manager t=2023-09-23T22:53:17.278517334Z level=info msg="Warming state cache for startup" 2023-09-23T22:53:17.442 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:17 smithi138 conmon[151484]: logger=http.server t=2023-09-23T22:53:17.282026157Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T22:53:17.442 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:17 smithi138 conmon[151484]: logger=sqlstore.transactions t=2023-09-23T22:53:17.288690456Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T22:53:17.442 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:17 smithi138 conmon[151484]: logger=ngalert.state.manager t=2023-09-23T22:53:17.326454186Z level=info msg="State cache has been initialized" states=0 duration=47.935995ms 2023-09-23T22:53:17.442 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:17 smithi138 conmon[151484]: logger=ticker t=2023-09-23T22:53:17.326511939Z level=info msg=starting first_tick=2023-09-23T22:53:20Z 2023-09-23T22:53:17.442 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:53:17 smithi138 conmon[151484]: logger=ngalert.multiorg.alertmanager t=2023-09-23T22:53:17.32651454Z level=info msg="starting MultiOrg Alertmanager" 2023-09-23T22:53:17.687 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-09-23T22:53:17.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[100243]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:53:17.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[100243]: Reconfiguring daemon alertmanager.a on smithi110 2023-09-23T22:53:17.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T22:53:17.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[100243]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T22:53:17.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T22:53:17.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[100243]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T22:53:17.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[100243]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T22:53:17.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[100243]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T22:53:17.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T22:53:17.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[100243]: fsmap foo:0 2023-09-23T22:53:17.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[100243]: Saving service mds.foo spec with placement count:2 2023-09-23T22:53:17.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:17.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[118821]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:53:17.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[118821]: Reconfiguring daemon alertmanager.a on smithi110 2023-09-23T22:53:17.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T22:53:17.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[118821]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T22:53:17.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T22:53:17.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[118821]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T22:53:17.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[118821]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T22:53:17.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[118821]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T22:53:17.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T22:53:17.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[118821]: fsmap foo:0 2023-09-23T22:53:17.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[118821]: Saving service mds.foo spec with placement count:2 2023-09-23T22:53:17.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:17.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:18.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:17 smithi138 ceph-mon[104418]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:53:18.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:17 smithi138 ceph-mon[104418]: Reconfiguring daemon alertmanager.a on smithi110 2023-09-23T22:53:18.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T22:53:18.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:17 smithi138 ceph-mon[104418]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T22:53:18.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T22:53:18.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:17 smithi138 ceph-mon[104418]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T22:53:18.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:17 smithi138 ceph-mon[104418]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T22:53:18.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:17 smithi138 ceph-mon[104418]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T22:53:18.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T22:53:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:17 smithi138 ceph-mon[104418]: fsmap foo:0 2023-09-23T22:53:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:17 smithi138 ceph-mon[104418]: Saving service mds.foo spec with placement count:2 2023-09-23T22:53:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:18.381 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:19.165 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:18 smithi110 ceph-mon[100243]: pgmap v40: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:19.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:18 smithi110 ceph-mon[100243]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T22:53:19.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:18 smithi110 ceph-mon[118821]: pgmap v40: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:19.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:18 smithi110 ceph-mon[118821]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T22:53:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:18 smithi138 ceph-mon[104418]: pgmap v40: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:18 smithi138 ceph-mon[104418]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T22:53:20.053 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:19 smithi110 ceph-mon[100243]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T22:53:20.054 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:19 smithi110 ceph-mon[118821]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T22:53:20.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:19 smithi138 ceph-mon[104418]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T22:53:20.446 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:20 smithi110 systemd[1]: Stopping Ceph alertmanager.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:53:20.701 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:20 smithi110 conmon[163775]: ts=2023-09-23T22:53:20.444Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T22:53:20.701 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:20 smithi110 bash[174775]: 655e561a5462c36cea589b5aa97c4845044bb3b49bcd3e7b66146922ac3c6045 2023-09-23T22:53:21.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:20 smithi138 ceph-mon[104418]: pgmap v43: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:21.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:20 smithi110 ceph-mon[100243]: pgmap v43: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:21.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:20 smithi110 ceph-mon[118821]: pgmap v43: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:21.780 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-09-23T22:53:22.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:53:21 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:53:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:53:22.201 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:21 smithi110 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T22:53:22.201 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:21 smithi110 systemd[1]: Stopped Ceph alertmanager.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:53:22.201 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:21 smithi110 systemd[1]: Starting Ceph alertmanager.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:53:22.503 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:22.701 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:22 smithi110 conmon[175324]: ts=2023-09-23T22:53:22.418Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T22:53:22.701 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:22 smithi110 conmon[175324]: ts=2023-09-23T22:53:22.418Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T22:53:22.701 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:22 smithi110 conmon[175324]: ts=2023-09-23T22:53:22.420Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.110 port=9094 2023-09-23T22:53:22.701 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:22 smithi110 conmon[175324]: ts=2023-09-23T22:53:22.420Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T22:53:22.702 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:22 smithi110 bash[175233]: b148630013bf1353fac8665603dc6f07b49c8c1f7abca5c6e9340e89e4c3d8a5 2023-09-23T22:53:22.702 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:22 smithi110 systemd[1]: Started Ceph alertmanager.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:53:22.702 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:22 smithi110 conmon[175324]: ts=2023-09-23T22:53:22.447Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T22:53:22.702 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:22 smithi110 conmon[175324]: ts=2023-09-23T22:53:22.447Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T22:53:22.702 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:22 smithi110 conmon[175324]: ts=2023-09-23T22:53:22.450Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T22:53:22.702 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:22 smithi110 conmon[175324]: ts=2023-09-23T22:53:22.450Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T22:53:23.182 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:22 smithi110 ceph-mon[100243]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:23.183 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:22 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:23.183 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:22 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:23.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:22 smithi110 ceph-mon[118821]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:23.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:22 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:23.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:22 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:23.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:22 smithi138 ceph-mon[104418]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:23.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:22 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:23.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:22 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:23.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:23 smithi110 ceph-mon[100243]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:53:23.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:23 smithi110 ceph-mon[100243]: Reconfiguring daemon prometheus.a on smithi138 2023-09-23T22:53:23.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:23 smithi110 ceph-mon[118821]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:53:23.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:23 smithi110 ceph-mon[118821]: Reconfiguring daemon prometheus.a on smithi138 2023-09-23T22:53:24.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:23 smithi138 ceph-mon[104418]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:53:24.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:23 smithi138 ceph-mon[104418]: Reconfiguring daemon prometheus.a on smithi138 2023-09-23T22:53:24.701 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:24 smithi110 conmon[175324]: ts=2023-09-23T22:53:24.421Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000205829s 2023-09-23T22:53:25.013 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T22:53:25.023 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T22:53:25.023 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T22:53:25.023 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T22:53:25.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:24 smithi138 ceph-mon[104418]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:25.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:24 smithi110 ceph-mon[100243]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:25.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:24 smithi110 ceph-mon[118821]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:25.366 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:25.657 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:25 smithi138 systemd[1]: Stopping Ceph prometheus.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:53:25.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:25 smithi138 conmon[148324]: ts=2023-09-23T22:53:25.656Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T22:53:25.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:25 smithi138 conmon[148324]: ts=2023-09-23T22:53:25.656Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T22:53:25.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:25 smithi138 conmon[148324]: ts=2023-09-23T22:53:25.656Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T22:53:25.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:25 smithi138 conmon[148324]: ts=2023-09-23T22:53:25.656Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T22:53:25.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:25 smithi138 conmon[148324]: ts=2023-09-23T22:53:25.656Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T22:53:25.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:25 smithi138 conmon[148324]: ts=2023-09-23T22:53:25.657Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T22:53:25.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:25 smithi138 conmon[148324]: ts=2023-09-23T22:53:25.657Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T22:53:25.942 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:25 smithi138 conmon[148324]: ts=2023-09-23T22:53:25.657Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T22:53:25.942 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:25 smithi138 conmon[148324]: ts=2023-09-23T22:53:25.657Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T22:53:25.942 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:25 smithi138 conmon[148324]: ts=2023-09-23T22:53:25.657Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T22:53:25.942 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:25 smithi138 conmon[148324]: ts=2023-09-23T22:53:25.657Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T22:53:25.942 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:25 smithi138 conmon[148324]: ts=2023-09-23T22:53:25.657Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T22:53:26.441 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 bash[153530]: 8abf0df36f5d39b5b94930a0bfaf093badca5f31cffa56d8665bb02b97187cc6 2023-09-23T22:53:26.441 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T22:53:26.441 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 systemd[1]: Stopped Ceph prometheus.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:53:26.441 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 systemd[1]: Starting Ceph prometheus.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:53:27.044 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:26 smithi110 ceph-mon[100243]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:27.045 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:26 smithi110 ceph-mon[118821]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:27.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:26 smithi138 ceph-mon[104418]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:27.191 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 bash[153857]: 685848669b55d0cac80f930f30221ba0f5d2c60b26ff81b83b006d5ec64a07b0 2023-09-23T22:53:27.191 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.942Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T22:53:27.191 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.943Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T22:53:27.191 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.943Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi138 (none))" 2023-09-23T22:53:27.191 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.943Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T22:53:27.192 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.943Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T22:53:27.192 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.944Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T22:53:27.192 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.945Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T22:53:27.192 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 systemd[1]: Started Ceph prometheus.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:53:27.192 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.946Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T22:53:27.192 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.946Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T22:53:27.192 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.947Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T22:53:27.192 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.947Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.47µs 2023-09-23T22:53:27.192 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.947Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T22:53:27.192 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.949Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-09-23T22:53:27.192 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.949Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-09-23T22:53:27.193 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.949Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.475µs wal_replay_duration=2.089721ms wbl_replay_duration=134ns total_replay_duration=2.128284ms 2023-09-23T22:53:27.193 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.950Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T22:53:27.193 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.950Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T22:53:27.193 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.950Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T22:53:27.193 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.965Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.016712ms db_storage=872ns remote_storage=1.498µs web_handler=392ns query_engine=595ns scrape=486.447µs scrape_sd=28.924µs notify=20.982µs notify_sd=5.667µs rules=14.175583ms tracing=6.441µs 2023-09-23T22:53:27.193 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.965Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T22:53:27.193 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:53:26 smithi138 conmon[153898]: ts=2023-09-23T22:53:26.965Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T22:53:27.254 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:53:27.390 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE Bus STOPPING 2023-09-23T22:53:27.390 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:53:27.390 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE Bus STOPPED 2023-09-23T22:53:27.391 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE Bus STARTING 2023-09-23T22:53:27.391 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE Serving on http://:::9283 2023-09-23T22:53:27.391 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE Bus STARTED 2023-09-23T22:53:27.391 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE Bus STOPPING 2023-09-23T22:53:27.391 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:53:27.391 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE Bus STOPPED 2023-09-23T22:53:27.391 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE Bus STARTING 2023-09-23T22:53:27.391 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE Serving on http://:::9283 2023-09-23T22:53:27.391 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE Bus STARTED 2023-09-23T22:53:27.391 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE Bus STOPPING 2023-09-23T22:53:27.391 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:53:27.391 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE Bus STOPPED 2023-09-23T22:53:27.392 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE Bus STARTING 2023-09-23T22:53:27.392 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE Serving on http://:::9283 2023-09-23T22:53:27.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:27 smithi110 conmon[100666]: [23/Sep/2023:22:53:27] ENGINE Bus STARTED 2023-09-23T22:53:28.007 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":19,"active_gid":24347,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6800","nonce":576619896},{"type":"v1","addr":"172.21.15.110:6801","nonce":576619896}]},"active_addr":"172.21.15.110:6801/576619896","active_change":"2023-09-23T22:52:17.775258+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24340,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.110:8443/","prometheus":"http://172.21.15.110:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":1399491671}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":3464832160}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":2850586742}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":380987145}]}]} 2023-09-23T22:53:28.012 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T22:53:28.012 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T22:53:28.013 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:53:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:53:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:53:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110:9093"}]: dispatch 2023-09-23T22:53:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110:9093"}]: dispatch 2023-09-23T22:53:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:53:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:53:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:53:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:53:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:53:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:53:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:53:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:53:28.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:28.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:53:28.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:27 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1010432505' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:53:28.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:28.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:28.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:53:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:53:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110:9093"}]: dispatch 2023-09-23T22:53:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110:9093"}]: dispatch 2023-09-23T22:53:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:53:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:53:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:53:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:53:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:53:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:53:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:53:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:53:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:53:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1010432505' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:53:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:53:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:53:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110:9093"}]: dispatch 2023-09-23T22:53:28.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110:9093"}]: dispatch 2023-09-23T22:53:28.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:28.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:53:28.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:53:28.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:53:28.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:53:28.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:28.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:53:28.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:53:28.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:53:28.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:53:28.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:28.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:53:28.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:27 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1010432505' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:53:28.640 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:29.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:29 smithi110 ceph-mon[100243]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:29.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:29 smithi110 ceph-mon[118821]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:29.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:29 smithi138 ceph-mon[104418]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:30.520 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:53:30.520 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":68,"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:46:49.032269+0000","modified":"2023-09-23T22:53:18.898237+0000","last_up_change":"2023-09-23T22:51:43.190723+0000","last_in_change":"2023-09-23T22:51:26.130429+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:49:49.263399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:52:19.908332+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:53:15.155816+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:53:16.112056+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"78791acb-c93c-4e4f-bf04-ea3eee46408d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6803","nonce":2453789689}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6805","nonce":2453789689}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6809","nonce":2453789689}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6807","nonce":2453789689}]},"public_addr":"172.21.15.110:6803/2453789689","cluster_addr":"172.21.15.110:6805/2453789689","heartbeat_back_addr":"172.21.15.110:6809/2453789689","heartbeat_front_addr":"172.21.15.110:6807/2453789689","state":["exists","up"]},{"osd":1,"uuid":"6793178f-2ef1-4c48-927c-831a083a3356","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6811","nonce":3697882113}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6813","nonce":3697882113}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6817","nonce":3697882113}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6815","nonce":3697882113}]},"public_addr":"172.21.15.110:6811/3697882113","cluster_addr":"172.21.15.110:6813/3697882113","heartbeat_back_addr":"172.21.15.110:6817/3697882113","heartbeat_front_addr":"172.21.15.110:6815/3697882113","state":["exists","up"]},{"osd":2,"uuid":"146fbd28-a365-4545-8e97-d629c6ddba21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6819","nonce":2231416166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6821","nonce":2231416166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6825","nonce":2231416166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6823","nonce":2231416166}]},"public_addr":"172.21.15.110:6819/2231416166","cluster_addr":"172.21.15.110:6821/2231416166","heartbeat_back_addr":"172.21.15.110:6825/2231416166","heartbeat_front_addr":"172.21.15.110:6823/2231416166","state":["exists","up"]},{"osd":3,"uuid":"6fcf80d5-6977-4929-b7ba-0e9ab65df0a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6827","nonce":427497470}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6829","nonce":427497470}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6833","nonce":427497470}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6831","nonce":427497470}]},"public_addr":"172.21.15.110:6827/427497470","cluster_addr":"172.21.15.110:6829/427497470","heartbeat_back_addr":"172.21.15.110:6833/427497470","heartbeat_front_addr":"172.21.15.110:6831/427497470","state":["exists","up"]},{"osd":4,"uuid":"131d0749-c46c-44f0-ab7f-03f8ad6ed502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6801","nonce":941787557}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6803","nonce":941787557}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6807","nonce":941787557}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6805","nonce":941787557}]},"public_addr":"172.21.15.138:6801/941787557","cluster_addr":"172.21.15.138:6803/941787557","heartbeat_back_addr":"172.21.15.138:6807/941787557","heartbeat_front_addr":"172.21.15.138:6805/941787557","state":["exists","up"]},{"osd":5,"uuid":"8cb496e5-d242-4e41-b279-e8d5ecea2996","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6809","nonce":3098195887}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6811","nonce":3098195887}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6815","nonce":3098195887}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6813","nonce":3098195887}]},"public_addr":"172.21.15.138:6809/3098195887","cluster_addr":"172.21.15.138:6811/3098195887","heartbeat_back_addr":"172.21.15.138:6815/3098195887","heartbeat_front_addr":"172.21.15.138:6813/3098195887","state":["exists","up"]},{"osd":6,"uuid":"c709e629-85ab-4273-9ca8-f2c4e8fc9074","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6817","nonce":187413403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6819","nonce":187413403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6823","nonce":187413403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6821","nonce":187413403}]},"public_addr":"172.21.15.138:6817/187413403","cluster_addr":"172.21.15.138:6819/187413403","heartbeat_back_addr":"172.21.15.138:6823/187413403","heartbeat_front_addr":"172.21.15.138:6821/187413403","state":["exists","up"]},{"osd":7,"uuid":"ed15f3b0-cd2e-43b7-b803-e45252b62a45","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6825","nonce":676244673}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6827","nonce":676244673}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6831","nonce":676244673}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6829","nonce":676244673}]},"public_addr":"172.21.15.138:6825/676244673","cluster_addr":"172.21.15.138:6827/676244673","heartbeat_back_addr":"172.21.15.138:6831/676244673","heartbeat_front_addr":"172.21.15.138:6829/676244673","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:48:58.523930+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:22.993201+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:46.747264+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:11.206564+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:33.511236+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:56.364297+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:18.141750+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:41.242882+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:6801/2914393040":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6800/2914393040":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/3674528218":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/3412430555":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1649304640":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1326657364":"2023-09-24T22:47:38.935467+0000","172.21.15.110:6801/4228126008":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/2523943302":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/1731974664":"2023-09-24T22:47:15.528545+0000","172.21.15.110:6801/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/1506915348":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/3250066431":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/410644117":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/657635177":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6800/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.110:6800/4228126008":"2023-09-24T22:47:38.935467+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:53:31.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:30 smithi138 ceph-mon[104418]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:31.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:30 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:31.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:30 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:31.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:30 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/602606846' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:53:31.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:30 smithi110 ceph-mon[100243]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:31.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:30 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:31.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:30 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:31.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:30 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/602606846' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:53:31.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:30 smithi110 ceph-mon[118821]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:31.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:30 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:31.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:30 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:31.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:30 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/602606846' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:53:31.412 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T22:53:31.412 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:53:31.774 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:32.701 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:53:32 smithi110 conmon[175324]: ts=2023-09-23T22:53:32.423Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002226225s 2023-09-23T22:53:32.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:32.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:32.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[100243]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:32.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:53:32.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:53:32.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:32.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.fuglpi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:53:32.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.fuglpi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:53:32.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:53:32.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[100243]: Deploying daemon mds.foo.smithi110.fuglpi on smithi110 2023-09-23T22:53:32.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:32.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:32.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[118821]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:32.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:53:32.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:53:32.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:32.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.fuglpi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:53:32.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.fuglpi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:53:32.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:53:32.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:32 smithi110 ceph-mon[118821]: Deploying daemon mds.foo.smithi110.fuglpi on smithi110 2023-09-23T22:53:32.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:32 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:32.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:32 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:32.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:32 smithi138 ceph-mon[104418]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:32.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:32 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:53:32.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:32 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:53:32.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:32 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:32.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:32 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.fuglpi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:53:32.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:32 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.fuglpi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:53:32.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:32 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:53:32.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:32 smithi138 ceph-mon[104418]: Deploying daemon mds.foo.smithi110.fuglpi on smithi110 2023-09-23T22:53:33.527 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:53:33.528 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":68,"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:46:49.032269+0000","modified":"2023-09-23T22:53:18.898237+0000","last_up_change":"2023-09-23T22:51:43.190723+0000","last_in_change":"2023-09-23T22:51:26.130429+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:49:49.263399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:52:19.908332+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:53:15.155816+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:53:16.112056+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"78791acb-c93c-4e4f-bf04-ea3eee46408d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6803","nonce":2453789689}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6805","nonce":2453789689}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6809","nonce":2453789689}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":2453789689},{"type":"v1","addr":"172.21.15.110:6807","nonce":2453789689}]},"public_addr":"172.21.15.110:6803/2453789689","cluster_addr":"172.21.15.110:6805/2453789689","heartbeat_back_addr":"172.21.15.110:6809/2453789689","heartbeat_front_addr":"172.21.15.110:6807/2453789689","state":["exists","up"]},{"osd":1,"uuid":"6793178f-2ef1-4c48-927c-831a083a3356","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6811","nonce":3697882113}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6813","nonce":3697882113}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6817","nonce":3697882113}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3697882113},{"type":"v1","addr":"172.21.15.110:6815","nonce":3697882113}]},"public_addr":"172.21.15.110:6811/3697882113","cluster_addr":"172.21.15.110:6813/3697882113","heartbeat_back_addr":"172.21.15.110:6817/3697882113","heartbeat_front_addr":"172.21.15.110:6815/3697882113","state":["exists","up"]},{"osd":2,"uuid":"146fbd28-a365-4545-8e97-d629c6ddba21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6819","nonce":2231416166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6821","nonce":2231416166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6825","nonce":2231416166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":2231416166},{"type":"v1","addr":"172.21.15.110:6823","nonce":2231416166}]},"public_addr":"172.21.15.110:6819/2231416166","cluster_addr":"172.21.15.110:6821/2231416166","heartbeat_back_addr":"172.21.15.110:6825/2231416166","heartbeat_front_addr":"172.21.15.110:6823/2231416166","state":["exists","up"]},{"osd":3,"uuid":"6fcf80d5-6977-4929-b7ba-0e9ab65df0a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6827","nonce":427497470}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6829","nonce":427497470}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6833","nonce":427497470}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":427497470},{"type":"v1","addr":"172.21.15.110:6831","nonce":427497470}]},"public_addr":"172.21.15.110:6827/427497470","cluster_addr":"172.21.15.110:6829/427497470","heartbeat_back_addr":"172.21.15.110:6833/427497470","heartbeat_front_addr":"172.21.15.110:6831/427497470","state":["exists","up"]},{"osd":4,"uuid":"131d0749-c46c-44f0-ab7f-03f8ad6ed502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6801","nonce":941787557}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6803","nonce":941787557}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6807","nonce":941787557}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6805","nonce":941787557}]},"public_addr":"172.21.15.138:6801/941787557","cluster_addr":"172.21.15.138:6803/941787557","heartbeat_back_addr":"172.21.15.138:6807/941787557","heartbeat_front_addr":"172.21.15.138:6805/941787557","state":["exists","up"]},{"osd":5,"uuid":"8cb496e5-d242-4e41-b279-e8d5ecea2996","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6809","nonce":3098195887}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6811","nonce":3098195887}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6815","nonce":3098195887}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6813","nonce":3098195887}]},"public_addr":"172.21.15.138:6809/3098195887","cluster_addr":"172.21.15.138:6811/3098195887","heartbeat_back_addr":"172.21.15.138:6815/3098195887","heartbeat_front_addr":"172.21.15.138:6813/3098195887","state":["exists","up"]},{"osd":6,"uuid":"c709e629-85ab-4273-9ca8-f2c4e8fc9074","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6817","nonce":187413403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6819","nonce":187413403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6823","nonce":187413403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6821","nonce":187413403}]},"public_addr":"172.21.15.138:6817/187413403","cluster_addr":"172.21.15.138:6819/187413403","heartbeat_back_addr":"172.21.15.138:6823/187413403","heartbeat_front_addr":"172.21.15.138:6821/187413403","state":["exists","up"]},{"osd":7,"uuid":"ed15f3b0-cd2e-43b7-b803-e45252b62a45","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6825","nonce":676244673}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6827","nonce":676244673}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6831","nonce":676244673}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6829","nonce":676244673}]},"public_addr":"172.21.15.138:6825/676244673","cluster_addr":"172.21.15.138:6827/676244673","heartbeat_back_addr":"172.21.15.138:6831/676244673","heartbeat_front_addr":"172.21.15.138:6829/676244673","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:48:58.523930+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:22.993201+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:46.747264+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:11.206564+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:33.511236+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:56.364297+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:18.141750+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:41.242882+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:6801/2914393040":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6800/2914393040":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/3674528218":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/3412430555":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1649304640":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1326657364":"2023-09-24T22:47:38.935467+0000","172.21.15.110:6801/4228126008":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/2523943302":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/1731974664":"2023-09-24T22:47:15.528545+0000","172.21.15.110:6801/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/1506915348":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/3250066431":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/410644117":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/657635177":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6800/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.110:6800/4228126008":"2023-09-24T22:47:38.935467+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:53:33.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:33 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:33.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:33 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/391997305' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:53:33.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:33 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:33.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:33 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/391997305' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:53:33.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:33 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:33.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:33 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/391997305' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:53:34.622 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T22:53:34.622 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T22:53:34.623 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T22:53:34.623 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T22:53:34.623 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T22:53:34.624 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T22:53:34.624 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T22:53:34.625 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T22:53:34.843 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:34 smithi110 ceph-mon[100243]: pgmap v50: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:34.843 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:34 smithi110 ceph-mon[118821]: pgmap v50: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:34.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:34 smithi138 ceph-mon[104418]: pgmap v50: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:35.988 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:36.192 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:36.291 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:36.348 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:37.079 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:37.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:36 smithi138 ceph-mon[104418]: pgmap v51: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:37.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:36 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:37.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:36 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:37.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:36 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:37.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:36 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.xrewco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:53:37.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:36 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.xrewco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:53:37.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:36 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:53:37.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:36 smithi110 ceph-mon[100243]: pgmap v51: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:37.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:36 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:37.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:36 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:37.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:36 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:37.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:36 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.xrewco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:53:37.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:36 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.xrewco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:53:37.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:36 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:53:37.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:36 smithi110 ceph-mon[118821]: pgmap v51: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:37.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:36 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:37.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:36 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:37.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:36 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:37.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:36 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.xrewco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:53:37.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:36 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.xrewco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:53:37.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:36 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:53:38.003 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:38.004 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:38.125 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[118821]: Deploying daemon mds.foo.smithi138.xrewco on smithi138 2023-09-23T22:53:38.125 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[118821]: mds.? [v2:172.21.15.110:6834/1876054795,v1:172.21.15.110:6835/1876054795] up:boot 2023-09-23T22:53:38.125 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[118821]: daemon mds.foo.smithi110.fuglpi assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T22:53:38.125 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[118821]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T22:53:38.125 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[118821]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T22:53:38.125 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[118821]: Cluster is now healthy 2023-09-23T22:53:38.125 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[118821]: fsmap foo:0 1 up:standby 2023-09-23T22:53:38.125 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.fuglpi"}]: dispatch 2023-09-23T22:53:38.126 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[118821]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:creating} 2023-09-23T22:53:38.126 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[118821]: daemon mds.foo.smithi110.fuglpi is now active in filesystem foo as rank 0 2023-09-23T22:53:38.126 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[100243]: Deploying daemon mds.foo.smithi138.xrewco on smithi138 2023-09-23T22:53:38.126 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[100243]: mds.? [v2:172.21.15.110:6834/1876054795,v1:172.21.15.110:6835/1876054795] up:boot 2023-09-23T22:53:38.126 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[100243]: daemon mds.foo.smithi110.fuglpi assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T22:53:38.126 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[100243]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T22:53:38.126 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[100243]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T22:53:38.126 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[100243]: Cluster is now healthy 2023-09-23T22:53:38.126 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[100243]: fsmap foo:0 1 up:standby 2023-09-23T22:53:38.126 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.fuglpi"}]: dispatch 2023-09-23T22:53:38.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[100243]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:creating} 2023-09-23T22:53:38.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:37 smithi110 ceph-mon[100243]: daemon mds.foo.smithi110.fuglpi is now active in filesystem foo as rank 0 2023-09-23T22:53:38.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:37 smithi138 ceph-mon[104418]: Deploying daemon mds.foo.smithi138.xrewco on smithi138 2023-09-23T22:53:38.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:37 smithi138 ceph-mon[104418]: mds.? [v2:172.21.15.110:6834/1876054795,v1:172.21.15.110:6835/1876054795] up:boot 2023-09-23T22:53:38.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:37 smithi138 ceph-mon[104418]: daemon mds.foo.smithi110.fuglpi assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T22:53:38.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:37 smithi138 ceph-mon[104418]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T22:53:38.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:37 smithi138 ceph-mon[104418]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T22:53:38.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:37 smithi138 ceph-mon[104418]: Cluster is now healthy 2023-09-23T22:53:38.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:37 smithi138 ceph-mon[104418]: fsmap foo:0 1 up:standby 2023-09-23T22:53:38.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:37 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.fuglpi"}]: dispatch 2023-09-23T22:53:38.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:37 smithi138 ceph-mon[104418]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:creating} 2023-09-23T22:53:38.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:37 smithi138 ceph-mon[104418]: daemon mds.foo.smithi110.fuglpi is now active in filesystem foo as rank 0 2023-09-23T22:53:38.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[100243]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:38.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[100243]: mds.? [v2:172.21.15.110:6834/1876054795,v1:172.21.15.110:6835/1876054795] up:active 2023-09-23T22:53:38.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[100243]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 2023-09-23T22:53:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:53:38.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[118821]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:38.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[118821]: mds.? [v2:172.21.15.110:6834/1876054795,v1:172.21.15.110:6835/1876054795] up:active 2023-09-23T22:53:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[118821]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 2023-09-23T22:53:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:38 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:53:39.072 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:39.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:38 smithi138 ceph-mon[104418]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:39.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:38 smithi138 ceph-mon[104418]: mds.? [v2:172.21.15.110:6834/1876054795,v1:172.21.15.110:6835/1876054795] up:active 2023-09-23T22:53:39.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:38 smithi138 ceph-mon[104418]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 2023-09-23T22:53:39.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:38 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:39.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:38 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:39.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:38 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:39.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:38 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:39.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:38 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:39.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:38 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:53:40.062 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:40 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:53:40] "GET /metrics HTTP/1.1" 200 208050 "" "Prometheus/2.43.0" 2023-09-23T22:53:40.063 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:39 smithi110 ceph-mon[118821]: mds.? [v2:172.21.15.138:6832/276794957,v1:172.21.15.138:6833/276794957] up:boot 2023-09-23T22:53:40.063 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:39 smithi110 ceph-mon[118821]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T22:53:40.063 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:39 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.xrewco"}]: dispatch 2023-09-23T22:53:40.063 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:39 smithi110 ceph-mon[118821]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T22:53:40.063 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:39 smithi110 ceph-mon[100243]: mds.? [v2:172.21.15.138:6832/276794957,v1:172.21.15.138:6833/276794957] up:boot 2023-09-23T22:53:40.063 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:39 smithi110 ceph-mon[100243]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T22:53:40.064 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:39 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.xrewco"}]: dispatch 2023-09-23T22:53:40.064 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:39 smithi110 ceph-mon[100243]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T22:53:40.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:39 smithi138 ceph-mon[104418]: mds.? [v2:172.21.15.138:6832/276794957,v1:172.21.15.138:6833/276794957] up:boot 2023-09-23T22:53:40.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:39 smithi138 ceph-mon[104418]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T22:53:40.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:39 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.xrewco"}]: dispatch 2023-09-23T22:53:40.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:39 smithi138 ceph-mon[104418]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T22:53:41.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:40 smithi138 ceph-mon[104418]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:41.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:40 smithi110 ceph-mon[100243]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:41.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:40 smithi110 ceph-mon[118821]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:41.345 INFO:teuthology.orchestra.run.smithi110.stdout:128849018920 2023-09-23T22:53:41.345 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T22:53:41.602 INFO:teuthology.orchestra.run.smithi110.stdout:163208757283 2023-09-23T22:53:41.603 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T22:53:42.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:53:41 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:53:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:53:42.318 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:42 smithi110 ceph-mon[118821]: mds.? [v2:172.21.15.110:6834/1876054795,v1:172.21.15.110:6835/1876054795] up:active 2023-09-23T22:53:42.318 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:42 smithi110 ceph-mon[118821]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T22:53:42.319 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:42 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:42.319 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:42 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:42.319 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:42 smithi110 ceph-mon[118821]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:42.654 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:42 smithi110 ceph-mon[100243]: mds.? [v2:172.21.15.110:6834/1876054795,v1:172.21.15.110:6835/1876054795] up:active 2023-09-23T22:53:42.654 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:42 smithi110 ceph-mon[100243]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T22:53:42.654 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:42 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:42.655 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:42 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:42.655 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:42 smithi110 ceph-mon[100243]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:42.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:42 smithi138 ceph-mon[104418]: mds.? [v2:172.21.15.110:6834/1876054795,v1:172.21.15.110:6835/1876054795] up:active 2023-09-23T22:53:42.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:42 smithi138 ceph-mon[104418]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T22:53:42.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:42 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:42.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:42 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:42.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:42 smithi138 ceph-mon[104418]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:44.186 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:43 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:44.186 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:43 smithi110 ceph-mon[100243]: mds.? [v2:172.21.15.138:6832/276794957,v1:172.21.15.138:6833/276794957] up:standby 2023-09-23T22:53:44.186 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:43 smithi110 ceph-mon[100243]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T22:53:44.186 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:43 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:44.186 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:43 smithi110 ceph-mon[118821]: mds.? [v2:172.21.15.138:6832/276794957,v1:172.21.15.138:6833/276794957] up:standby 2023-09-23T22:53:44.186 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:43 smithi110 ceph-mon[118821]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T22:53:44.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:43 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:44.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:43 smithi138 ceph-mon[104418]: mds.? [v2:172.21.15.138:6832/276794957,v1:172.21.15.138:6833/276794957] up:standby 2023-09-23T22:53:44.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:43 smithi138 ceph-mon[104418]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T22:53:45.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:44 smithi138 ceph-mon[104418]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:45.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:44 smithi110 ceph-mon[100243]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:45.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:44 smithi110 ceph-mon[118821]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:45.426 INFO:teuthology.orchestra.run.smithi110.stdout:107374182445 2023-09-23T22:53:45.426 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T22:53:45.753 INFO:teuthology.orchestra.run.smithi110.stdout:77309411376 2023-09-23T22:53:45.754 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T22:53:46.396 INFO:teuthology.orchestra.run.smithi110.stdout:197568495648 2023-09-23T22:53:46.397 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T22:53:47.169 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:46 smithi110 ceph-mon[100243]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:47.170 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:46 smithi110 ceph-mon[118821]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:47.190 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:46 smithi138 ceph-mon[104418]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:48.168 INFO:teuthology.orchestra.run.smithi110.stdout:34359738427 2023-09-23T22:53:48.168 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T22:53:48.201 INFO:teuthology.orchestra.run.smithi110.stdout:231928234012 2023-09-23T22:53:48.202 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T22:53:48.747 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:48.817 INFO:teuthology.orchestra.run.smithi110.stdout:55834574903 2023-09-23T22:53:48.818 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T22:53:48.854 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:49.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:48 smithi138 ceph-mon[104418]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:49.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:48 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:49.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:48 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:49.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:48 smithi110 ceph-mon[100243]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:49.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:48 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:49.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:48 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:49.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:48 smithi110 ceph-mon[118821]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:49.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:48 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:53:49.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:48 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:49.503 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:50.320 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:53:50 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:53:50] "GET /metrics HTTP/1.1" 200 238793 "" "Prometheus/2.43.0" 2023-09-23T22:53:50.320 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:50 smithi110 ceph-mon[118821]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:50.423 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:50.424 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:50.424 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:50.427 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:50.571 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:50 smithi110 ceph-mon[100243]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:50 smithi138 ceph-mon[104418]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:53:51.390 INFO:teuthology.orchestra.run.smithi110.stdout:128849018922 2023-09-23T22:53:51.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:51 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/254980592' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:53:51.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:51 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/254980592' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:53:51.917 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:51 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/254980592' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:53:52.101 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:52.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:53:51 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:53:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:53:52.636 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:52 smithi110 ceph-mon[100243]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:53:52.636 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:52 smithi110 ceph-mon[118821]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:53:52.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:52 smithi138 ceph-mon[104418]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:53:53.868 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018920 got 128849018922 for osd.4 2023-09-23T22:53:53.869 DEBUG:teuthology.parallel:result is None 2023-09-23T22:53:55.154 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:54 smithi110 ceph-mon[118821]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:53:55.155 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:54 smithi110 ceph-mon[100243]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:53:55.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:54 smithi138 ceph-mon[104418]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:53:56.442 INFO:teuthology.orchestra.run.smithi110.stdout:77309411379 2023-09-23T22:53:56.921 INFO:teuthology.orchestra.run.smithi110.stdout:163208757286 2023-09-23T22:53:57.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:56 smithi138 ceph-mon[104418]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:57.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:56 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2376823060' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:53:57.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:56 smithi110 ceph-mon[100243]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:57.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:56 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2376823060' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:53:57.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:56 smithi110 ceph-mon[118821]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:57.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:56 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2376823060' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:53:57.739 INFO:teuthology.orchestra.run.smithi110.stdout:197568495650 2023-09-23T22:53:58.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:57 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2988582186' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:53:58.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:57 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/642298740' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:53:58.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:57 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3521486676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:53:58.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:57 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2988582186' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:53:58.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:57 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/642298740' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:53:58.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:57 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3521486676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:53:58.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:57 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2988582186' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:53:58.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:57 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/642298740' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:53:58.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:57 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3521486676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:53:58.266 INFO:teuthology.orchestra.run.smithi110.stdout:107374182447 2023-09-23T22:53:58.452 INFO:teuthology.orchestra.run.smithi110.stdout:34359738429 2023-09-23T22:53:59.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:58 smithi110 ceph-mon[118821]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:59.184 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:53:58 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1488474016' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:53:59.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:58 smithi110 ceph-mon[100243]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:59.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:53:58 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1488474016' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:53:59.190 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:58 smithi138 ceph-mon[104418]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:53:59.190 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:53:58 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1488474016' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:53:59.417 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757283 got 163208757286 for osd.5 2023-09-23T22:53:59.417 DEBUG:teuthology.parallel:result is None 2023-09-23T22:54:00.290 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:54:00 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:54:00] "GET /metrics HTTP/1.1" 200 238793 "" "Prometheus/2.43.0" 2023-09-23T22:54:00.311 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182445 got 107374182447 for osd.3 2023-09-23T22:54:00.311 DEBUG:teuthology.parallel:result is None 2023-09-23T22:54:00.356 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738427 got 34359738429 for osd.0 2023-09-23T22:54:00.356 DEBUG:teuthology.parallel:result is None 2023-09-23T22:54:00.523 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495648 got 197568495650 for osd.6 2023-09-23T22:54:00.523 DEBUG:teuthology.parallel:result is None 2023-09-23T22:54:00.548 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:00 smithi110 ceph-mon[100243]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:00.549 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:00 smithi110 ceph-mon[118821]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:00.571 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411376 got 77309411379 for osd.2 2023-09-23T22:54:00.571 DEBUG:teuthology.parallel:result is None 2023-09-23T22:54:00.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:00 smithi138 ceph-mon[104418]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:01.041 INFO:teuthology.orchestra.run.smithi110.stdout:55834574906 2023-09-23T22:54:01.419 INFO:teuthology.orchestra.run.smithi110.stdout:231928234015 2023-09-23T22:54:01.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:01 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/964302014' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:54:01.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:01 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/964302014' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:54:01.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:01 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/964302014' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:54:02.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:54:01 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:54:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:54:02.248 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574903 got 55834574906 for osd.1 2023-09-23T22:54:02.249 DEBUG:teuthology.parallel:result is None 2023-09-23T22:54:02.509 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:02 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1074327492' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:54:02.509 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:02 smithi110 ceph-mon[100243]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:02.509 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:02 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1074327492' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:54:02.509 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:02 smithi110 ceph-mon[118821]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:02.526 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234012 got 231928234015 for osd.7 2023-09-23T22:54:02.527 DEBUG:teuthology.parallel:result is None 2023-09-23T22:54:02.527 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T22:54:02.527 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:54:02.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:02 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1074327492' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:54:02.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:02 smithi138 ceph-mon[104418]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:02.888 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:03 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:03 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:03 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:54:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:03 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:54:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:03 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:03 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:54:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:03 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:03.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:03 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:03.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:03 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:03.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:03 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:54:03.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:03 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:54:03.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:03 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:03.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:03 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:54:03.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:03 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:03.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:03 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:03.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:03 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:03.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:03 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:54:03.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:03 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:54:03.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:03 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:03.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:03 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:54:03.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:03 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:04.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:04 smithi110 ceph-mon[100243]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:04.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:04 smithi110 ceph-mon[118821]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:04.827 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:54:04.829 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T22:54:04.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:04 smithi138 ceph-mon[104418]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:05.628 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":65,"stamp":"2023-09-23T22:54:03.809706+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52408,"kb_used_data":7224,"kb_used_omap":0,"kb_used_meta":45120,"kb_avail":749908808,"statfs":{"total":767960285184,"available":767906619392,"internally_reserved":0,"allocated":7397376,"data_stored":4172641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46202880},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002554"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.139602+0000","last_change":"2023-09-23T22:53:18.138105+0000","last_active":"2023-09-23T22:53:18.139602+0000","last_peered":"2023-09-23T22:53:18.139602+0000","last_clean":"2023-09-23T22:53:18.139602+0000","last_became_active":"2023-09-23T22:53:18.137820+0000","last_became_peered":"2023-09-23T22:53:18.137820+0000","last_unstale":"2023-09-23T22:53:18.139602+0000","last_undegraded":"2023-09-23T22:53:18.139602+0000","last_fullsized":"2023-09-23T22:53:18.139602+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:11.526142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.877077+0000","last_change":"2023-09-23T22:53:16.886024+0000","last_active":"2023-09-23T22:53:36.877077+0000","last_peered":"2023-09-23T22:53:36.877077+0000","last_clean":"2023-09-23T22:53:36.877077+0000","last_became_active":"2023-09-23T22:53:16.885284+0000","last_became_peered":"2023-09-23T22:53:16.885284+0000","last_unstale":"2023-09-23T22:53:36.877077+0000","last_undegraded":"2023-09-23T22:53:36.877077+0000","last_fullsized":"2023-09-23T22:53:36.877077+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:38.524289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.906673+0000","last_change":"2023-09-23T22:53:17.906381+0000","last_active":"2023-09-23T22:53:17.906673+0000","last_peered":"2023-09-23T22:53:17.906673+0000","last_clean":"2023-09-23T22:53:17.906673+0000","last_became_active":"2023-09-23T22:53:17.906010+0000","last_became_peered":"2023-09-23T22:53:17.906010+0000","last_unstale":"2023-09-23T22:53:17.906673+0000","last_undegraded":"2023-09-23T22:53:17.906673+0000","last_fullsized":"2023-09-23T22:53:17.906673+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:39.381896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:20.246164+0000","last_change":"2023-09-23T22:53:16.889809+0000","last_active":"2023-09-23T22:53:20.246164+0000","last_peered":"2023-09-23T22:53:20.246164+0000","last_clean":"2023-09-23T22:53:20.246164+0000","last_became_active":"2023-09-23T22:53:16.889533+0000","last_became_peered":"2023-09-23T22:53:16.889533+0000","last_unstale":"2023-09-23T22:53:20.246164+0000","last_undegraded":"2023-09-23T22:53:20.246164+0000","last_fullsized":"2023-09-23T22:53:20.246164+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:48.318427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.906517+0000","last_change":"2023-09-23T22:53:17.905814+0000","last_active":"2023-09-23T22:53:17.906517+0000","last_peered":"2023-09-23T22:53:17.906517+0000","last_clean":"2023-09-23T22:53:17.906517+0000","last_became_active":"2023-09-23T22:53:17.905541+0000","last_became_peered":"2023-09-23T22:53:17.905541+0000","last_unstale":"2023-09-23T22:53:17.906517+0000","last_undegraded":"2023-09-23T22:53:17.906517+0000","last_fullsized":"2023-09-23T22:53:17.906517+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:38:22.986263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.876830+0000","last_change":"2023-09-23T22:53:16.891297+0000","last_active":"2023-09-23T22:53:36.876830+0000","last_peered":"2023-09-23T22:53:36.876830+0000","last_clean":"2023-09-23T22:53:36.876830+0000","last_became_active":"2023-09-23T22:53:16.891025+0000","last_became_peered":"2023-09-23T22:53:16.891025+0000","last_unstale":"2023-09-23T22:53:36.876830+0000","last_undegraded":"2023-09-23T22:53:36.876830+0000","last_fullsized":"2023-09-23T22:53:36.876830+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:40:50.608339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.139686+0000","last_change":"2023-09-23T22:53:18.138626+0000","last_active":"2023-09-23T22:53:18.139686+0000","last_peered":"2023-09-23T22:53:18.139686+0000","last_clean":"2023-09-23T22:53:18.139686+0000","last_became_active":"2023-09-23T22:53:18.138148+0000","last_became_peered":"2023-09-23T22:53:18.138148+0000","last_unstale":"2023-09-23T22:53:18.139686+0000","last_undegraded":"2023-09-23T22:53:18.139686+0000","last_fullsized":"2023-09-23T22:53:18.139686+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:57.413976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.876758+0000","last_change":"2023-09-23T22:53:16.894996+0000","last_active":"2023-09-23T22:53:36.876758+0000","last_peered":"2023-09-23T22:53:36.876758+0000","last_clean":"2023-09-23T22:53:36.876758+0000","last_became_active":"2023-09-23T22:53:16.894734+0000","last_became_peered":"2023-09-23T22:53:16.894734+0000","last_unstale":"2023-09-23T22:53:36.876758+0000","last_undegraded":"2023-09-23T22:53:36.876758+0000","last_fullsized":"2023-09-23T22:53:36.876758+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:51.380926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.899794+0000","last_change":"2023-09-23T22:53:16.999836+0000","last_active":"2023-09-23T22:53:17.899794+0000","last_peered":"2023-09-23T22:53:17.899794+0000","last_clean":"2023-09-23T22:53:17.899794+0000","last_became_active":"2023-09-23T22:53:16.999448+0000","last_became_peered":"2023-09-23T22:53:16.999448+0000","last_unstale":"2023-09-23T22:53:17.899794+0000","last_undegraded":"2023-09-23T22:53:17.899794+0000","last_fullsized":"2023-09-23T22:53:17.899794+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:21.113817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.904433+0000","last_change":"2023-09-23T22:53:16.895075+0000","last_active":"2023-09-23T22:53:18.904433+0000","last_peered":"2023-09-23T22:53:18.904433+0000","last_clean":"2023-09-23T22:53:18.904433+0000","last_became_active":"2023-09-23T22:53:16.894785+0000","last_became_peered":"2023-09-23T22:53:16.894785+0000","last_unstale":"2023-09-23T22:53:18.904433+0000","last_undegraded":"2023-09-23T22:53:18.904433+0000","last_fullsized":"2023-09-23T22:53:18.904433+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:45.846898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.906502+0000","last_change":"2023-09-23T22:53:17.905706+0000","last_active":"2023-09-23T22:53:17.906502+0000","last_peered":"2023-09-23T22:53:17.906502+0000","last_clean":"2023-09-23T22:53:17.906502+0000","last_became_active":"2023-09-23T22:53:17.905309+0000","last_became_peered":"2023-09-23T22:53:17.905309+0000","last_unstale":"2023-09-23T22:53:17.906502+0000","last_undegraded":"2023-09-23T22:53:17.906502+0000","last_fullsized":"2023-09-23T22:53:17.906502+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:06.105426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.904722+0000","last_change":"2023-09-23T22:53:16.894451+0000","last_active":"2023-09-23T22:53:18.904722+0000","last_peered":"2023-09-23T22:53:18.904722+0000","last_clean":"2023-09-23T22:53:18.904722+0000","last_became_active":"2023-09-23T22:53:16.894169+0000","last_became_peered":"2023-09-23T22:53:16.894169+0000","last_unstale":"2023-09-23T22:53:18.904722+0000","last_undegraded":"2023-09-23T22:53:18.904722+0000","last_fullsized":"2023-09-23T22:53:18.904722+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:20.585622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.906216+0000","last_change":"2023-09-23T22:52:21.315891+0000","last_active":"2023-09-23T22:53:17.906216+0000","last_peered":"2023-09-23T22:53:17.906216+0000","last_clean":"2023-09-23T22:53:17.906216+0000","last_became_active":"2023-09-23T22:52:21.315580+0000","last_became_peered":"2023-09-23T22:52:21.315580+0000","last_unstale":"2023-09-23T22:53:17.906216+0000","last_undegraded":"2023-09-23T22:53:17.906216+0000","last_fullsized":"2023-09-23T22:53:17.906216+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:33.692738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.075643+0000","last_change":"2023-09-23T22:53:18.075389+0000","last_active":"2023-09-23T22:53:18.075643+0000","last_peered":"2023-09-23T22:53:18.075643+0000","last_clean":"2023-09-23T22:53:18.075643+0000","last_became_active":"2023-09-23T22:53:18.075127+0000","last_became_peered":"2023-09-23T22:53:18.075127+0000","last_unstale":"2023-09-23T22:53:18.075643+0000","last_undegraded":"2023-09-23T22:53:18.075643+0000","last_fullsized":"2023-09-23T22:53:18.075643+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:07.596409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138550+0000","last_change":"2023-09-23T22:52:21.890478+0000","last_active":"2023-09-23T22:53:18.138550+0000","last_peered":"2023-09-23T22:53:18.138550+0000","last_clean":"2023-09-23T22:53:18.138550+0000","last_became_active":"2023-09-23T22:52:21.890133+0000","last_became_peered":"2023-09-23T22:52:21.890133+0000","last_unstale":"2023-09-23T22:53:18.138550+0000","last_undegraded":"2023-09-23T22:53:18.138550+0000","last_fullsized":"2023-09-23T22:53:18.138550+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:34:57.292871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.877565+0000","last_change":"2023-09-23T22:53:16.890252+0000","last_active":"2023-09-23T22:53:36.877565+0000","last_peered":"2023-09-23T22:53:36.877565+0000","last_clean":"2023-09-23T22:53:36.877565+0000","last_became_active":"2023-09-23T22:53:16.889999+0000","last_became_peered":"2023-09-23T22:53:16.889999+0000","last_unstale":"2023-09-23T22:53:36.877565+0000","last_undegraded":"2023-09-23T22:53:36.877565+0000","last_fullsized":"2023-09-23T22:53:36.877565+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:32.424686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.900875+0000","last_change":"2023-09-23T22:53:17.000093+0000","last_active":"2023-09-23T22:53:17.900875+0000","last_peered":"2023-09-23T22:53:17.900875+0000","last_clean":"2023-09-23T22:53:17.900875+0000","last_became_active":"2023-09-23T22:53:16.999800+0000","last_became_peered":"2023-09-23T22:53:16.999800+0000","last_unstale":"2023-09-23T22:53:17.900875+0000","last_undegraded":"2023-09-23T22:53:17.900875+0000","last_fullsized":"2023-09-23T22:53:17.900875+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:48.627741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138936+0000","last_change":"2023-09-23T22:53:16.893176+0000","last_active":"2023-09-23T22:53:18.138936+0000","last_peered":"2023-09-23T22:53:18.138936+0000","last_clean":"2023-09-23T22:53:18.138936+0000","last_became_active":"2023-09-23T22:53:16.892857+0000","last_became_peered":"2023-09-23T22:53:16.892857+0000","last_unstale":"2023-09-23T22:53:18.138936+0000","last_undegraded":"2023-09-23T22:53:18.138936+0000","last_fullsized":"2023-09-23T22:53:18.138936+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:34.831545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.074684+0000","last_change":"2023-09-23T22:52:21.890419+0000","last_active":"2023-09-23T22:53:18.074684+0000","last_peered":"2023-09-23T22:53:18.074684+0000","last_clean":"2023-09-23T22:53:18.074684+0000","last_became_active":"2023-09-23T22:52:21.890113+0000","last_became_peered":"2023-09-23T22:52:21.890113+0000","last_unstale":"2023-09-23T22:53:18.074684+0000","last_undegraded":"2023-09-23T22:53:18.074684+0000","last_fullsized":"2023-09-23T22:53:18.074684+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:23.384783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.901427+0000","last_change":"2023-09-23T22:53:16.905062+0000","last_active":"2023-09-23T22:53:17.901427+0000","last_peered":"2023-09-23T22:53:17.901427+0000","last_clean":"2023-09-23T22:53:17.901427+0000","last_became_active":"2023-09-23T22:53:16.904812+0000","last_became_peered":"2023-09-23T22:53:16.904812+0000","last_unstale":"2023-09-23T22:53:17.901427+0000","last_undegraded":"2023-09-23T22:53:17.901427+0000","last_fullsized":"2023-09-23T22:53:17.901427+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:44.859759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.900447+0000","last_change":"2023-09-23T22:53:16.888998+0000","last_active":"2023-09-23T22:53:17.900447+0000","last_peered":"2023-09-23T22:53:17.900447+0000","last_clean":"2023-09-23T22:53:17.900447+0000","last_became_active":"2023-09-23T22:53:16.888784+0000","last_became_peered":"2023-09-23T22:53:16.888784+0000","last_unstale":"2023-09-23T22:53:17.900447+0000","last_undegraded":"2023-09-23T22:53:17.900447+0000","last_fullsized":"2023-09-23T22:53:17.900447+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:18.878414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.074616+0000","last_change":"2023-09-23T22:52:21.315773+0000","last_active":"2023-09-23T22:53:18.074616+0000","last_peered":"2023-09-23T22:53:18.074616+0000","last_clean":"2023-09-23T22:53:18.074616+0000","last_became_active":"2023-09-23T22:52:21.315503+0000","last_became_peered":"2023-09-23T22:52:21.315503+0000","last_unstale":"2023-09-23T22:53:18.074616+0000","last_undegraded":"2023-09-23T22:53:18.074616+0000","last_fullsized":"2023-09-23T22:53:18.074616+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:39:58.756695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.076496+0000","last_change":"2023-09-23T22:53:18.075922+0000","last_active":"2023-09-23T22:53:18.076496+0000","last_peered":"2023-09-23T22:53:18.076496+0000","last_clean":"2023-09-23T22:53:18.076496+0000","last_became_active":"2023-09-23T22:53:18.075621+0000","last_became_peered":"2023-09-23T22:53:18.075621+0000","last_unstale":"2023-09-23T22:53:18.076496+0000","last_undegraded":"2023-09-23T22:53:18.076496+0000","last_fullsized":"2023-09-23T22:53:18.076496+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:01.490333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.902804+0000","last_change":"2023-09-23T22:53:16.884368+0000","last_active":"2023-09-23T22:53:17.902804+0000","last_peered":"2023-09-23T22:53:17.902804+0000","last_clean":"2023-09-23T22:53:17.902804+0000","last_became_active":"2023-09-23T22:53:16.884081+0000","last_became_peered":"2023-09-23T22:53:16.884081+0000","last_unstale":"2023-09-23T22:53:17.902804+0000","last_undegraded":"2023-09-23T22:53:17.902804+0000","last_fullsized":"2023-09-23T22:53:17.902804+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:45.320726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":275,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.074538+0000","last_change":"2023-09-23T22:51:47.222693+0000","last_active":"2023-09-23T22:53:18.074538+0000","last_peered":"2023-09-23T22:53:18.074538+0000","last_clean":"2023-09-23T22:53:18.074538+0000","last_became_active":"2023-09-23T22:51:47.222168+0000","last_became_peered":"2023-09-23T22:51:47.222168+0000","last_unstale":"2023-09-23T22:53:18.074538+0000","last_undegraded":"2023-09-23T22:53:18.074538+0000","last_fullsized":"2023-09-23T22:53:18.074538+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T22:49:49.548021+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:16.935480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.899375+0000","last_change":"2023-09-23T22:52:21.315119+0000","last_active":"2023-09-23T22:53:17.899375+0000","last_peered":"2023-09-23T22:53:17.899375+0000","last_clean":"2023-09-23T22:53:17.899375+0000","last_became_active":"2023-09-23T22:52:21.314817+0000","last_became_peered":"2023-09-23T22:52:21.314817+0000","last_unstale":"2023-09-23T22:53:17.899375+0000","last_undegraded":"2023-09-23T22:53:17.899375+0000","last_fullsized":"2023-09-23T22:53:17.899375+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:22.207385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138454+0000","last_change":"2023-09-23T22:53:17.000507+0000","last_active":"2023-09-23T22:53:18.138454+0000","last_peered":"2023-09-23T22:53:18.138454+0000","last_clean":"2023-09-23T22:53:18.138454+0000","last_became_active":"2023-09-23T22:53:17.000238+0000","last_became_peered":"2023-09-23T22:53:17.000238+0000","last_unstale":"2023-09-23T22:53:18.138454+0000","last_undegraded":"2023-09-23T22:53:18.138454+0000","last_fullsized":"2023-09-23T22:53:18.138454+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.070327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.137964+0000","last_change":"2023-09-23T22:53:16.892741+0000","last_active":"2023-09-23T22:53:18.137964+0000","last_peered":"2023-09-23T22:53:18.137964+0000","last_clean":"2023-09-23T22:53:18.137964+0000","last_became_active":"2023-09-23T22:53:16.887836+0000","last_became_peered":"2023-09-23T22:53:16.887836+0000","last_unstale":"2023-09-23T22:53:18.137964+0000","last_undegraded":"2023-09-23T22:53:18.137964+0000","last_fullsized":"2023-09-23T22:53:18.137964+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:21.750136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.899282+0000","last_change":"2023-09-23T22:52:21.890238+0000","last_active":"2023-09-23T22:53:17.899282+0000","last_peered":"2023-09-23T22:53:17.899282+0000","last_clean":"2023-09-23T22:53:17.899282+0000","last_became_active":"2023-09-23T22:52:21.889979+0000","last_became_peered":"2023-09-23T22:52:21.889979+0000","last_unstale":"2023-09-23T22:53:17.899282+0000","last_undegraded":"2023-09-23T22:53:17.899282+0000","last_fullsized":"2023-09-23T22:53:17.899282+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:25.487367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138854+0000","last_change":"2023-09-23T22:53:16.999411+0000","last_active":"2023-09-23T22:53:18.138854+0000","last_peered":"2023-09-23T22:53:18.138854+0000","last_clean":"2023-09-23T22:53:18.138854+0000","last_became_active":"2023-09-23T22:53:16.999141+0000","last_became_peered":"2023-09-23T22:53:16.999141+0000","last_unstale":"2023-09-23T22:53:18.138854+0000","last_undegraded":"2023-09-23T22:53:18.138854+0000","last_fullsized":"2023-09-23T22:53:18.138854+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:54.430172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.904826+0000","last_change":"2023-09-23T22:53:16.894735+0000","last_active":"2023-09-23T22:53:17.904826+0000","last_peered":"2023-09-23T22:53:17.904826+0000","last_clean":"2023-09-23T22:53:17.904826+0000","last_became_active":"2023-09-23T22:53:16.894479+0000","last_became_peered":"2023-09-23T22:53:16.894479+0000","last_unstale":"2023-09-23T22:53:17.904826+0000","last_undegraded":"2023-09-23T22:53:17.904826+0000","last_fullsized":"2023-09-23T22:53:17.904826+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:45:29.733182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.899344+0000","last_change":"2023-09-23T22:52:21.314044+0000","last_active":"2023-09-23T22:53:17.899344+0000","last_peered":"2023-09-23T22:53:17.899344+0000","last_clean":"2023-09-23T22:53:17.899344+0000","last_became_active":"2023-09-23T22:52:21.313723+0000","last_became_peered":"2023-09-23T22:52:21.313723+0000","last_unstale":"2023-09-23T22:53:17.899344+0000","last_undegraded":"2023-09-23T22:53:17.899344+0000","last_fullsized":"2023-09-23T22:53:17.899344+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:31.447410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138032+0000","last_change":"2023-09-23T22:53:17.000045+0000","last_active":"2023-09-23T22:53:18.138032+0000","last_peered":"2023-09-23T22:53:18.138032+0000","last_clean":"2023-09-23T22:53:18.138032+0000","last_became_active":"2023-09-23T22:53:16.999783+0000","last_became_peered":"2023-09-23T22:53:16.999783+0000","last_unstale":"2023-09-23T22:53:18.138032+0000","last_undegraded":"2023-09-23T22:53:18.138032+0000","last_fullsized":"2023-09-23T22:53:18.138032+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:58.830318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.899759+0000","last_change":"2023-09-23T22:53:16.886475+0000","last_active":"2023-09-23T22:53:17.899759+0000","last_peered":"2023-09-23T22:53:17.899759+0000","last_clean":"2023-09-23T22:53:17.899759+0000","last_became_active":"2023-09-23T22:53:16.886243+0000","last_became_peered":"2023-09-23T22:53:16.886243+0000","last_unstale":"2023-09-23T22:53:17.899759+0000","last_undegraded":"2023-09-23T22:53:17.899759+0000","last_fullsized":"2023-09-23T22:53:17.899759+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:16.683185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138791+0000","last_change":"2023-09-23T22:52:21.313895+0000","last_active":"2023-09-23T22:53:18.138791+0000","last_peered":"2023-09-23T22:53:18.138791+0000","last_clean":"2023-09-23T22:53:18.138791+0000","last_became_active":"2023-09-23T22:52:21.313631+0000","last_became_peered":"2023-09-23T22:52:21.313631+0000","last_unstale":"2023-09-23T22:53:18.138791+0000","last_undegraded":"2023-09-23T22:53:18.138791+0000","last_fullsized":"2023-09-23T22:53:18.138791+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:41.617398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138532+0000","last_change":"2023-09-23T22:53:16.998598+0000","last_active":"2023-09-23T22:53:18.138532+0000","last_peered":"2023-09-23T22:53:18.138532+0000","last_clean":"2023-09-23T22:53:18.138532+0000","last_became_active":"2023-09-23T22:53:16.998354+0000","last_became_peered":"2023-09-23T22:53:16.998354+0000","last_unstale":"2023-09-23T22:53:18.138532+0000","last_undegraded":"2023-09-23T22:53:18.138532+0000","last_fullsized":"2023-09-23T22:53:18.138532+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:29.811991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.902415+0000","last_change":"2023-09-23T22:53:16.894485+0000","last_active":"2023-09-23T22:53:17.902415+0000","last_peered":"2023-09-23T22:53:17.902415+0000","last_clean":"2023-09-23T22:53:17.902415+0000","last_became_active":"2023-09-23T22:53:16.894025+0000","last_became_peered":"2023-09-23T22:53:16.894025+0000","last_unstale":"2023-09-23T22:53:17.902415+0000","last_undegraded":"2023-09-23T22:53:17.902415+0000","last_fullsized":"2023-09-23T22:53:17.902415+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:26.374772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.139679+0000","last_change":"2023-09-23T22:53:18.138464+0000","last_active":"2023-09-23T22:53:18.139679+0000","last_peered":"2023-09-23T22:53:18.139679+0000","last_clean":"2023-09-23T22:53:18.139679+0000","last_became_active":"2023-09-23T22:53:18.137635+0000","last_became_peered":"2023-09-23T22:53:18.137635+0000","last_unstale":"2023-09-23T22:53:18.139679+0000","last_undegraded":"2023-09-23T22:53:18.139679+0000","last_fullsized":"2023-09-23T22:53:18.139679+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:57.384430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.902689+0000","last_change":"2023-09-23T22:53:16.893337+0000","last_active":"2023-09-23T22:53:17.902689+0000","last_peered":"2023-09-23T22:53:17.902689+0000","last_clean":"2023-09-23T22:53:17.902689+0000","last_became_active":"2023-09-23T22:53:16.891945+0000","last_became_peered":"2023-09-23T22:53:16.891945+0000","last_unstale":"2023-09-23T22:53:17.902689+0000","last_undegraded":"2023-09-23T22:53:17.902689+0000","last_fullsized":"2023-09-23T22:53:17.902689+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:49.534479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.140875+0000","last_change":"2023-09-23T22:53:18.140633+0000","last_active":"2023-09-23T22:53:18.140875+0000","last_peered":"2023-09-23T22:53:18.140875+0000","last_clean":"2023-09-23T22:53:18.140875+0000","last_became_active":"2023-09-23T22:53:18.140386+0000","last_became_peered":"2023-09-23T22:53:18.140386+0000","last_unstale":"2023-09-23T22:53:18.140875+0000","last_undegraded":"2023-09-23T22:53:18.140875+0000","last_fullsized":"2023-09-23T22:53:18.140875+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:16:51.208509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.905087+0000","last_change":"2023-09-23T22:53:16.895307+0000","last_active":"2023-09-23T22:53:17.905087+0000","last_peered":"2023-09-23T22:53:17.905087+0000","last_clean":"2023-09-23T22:53:17.905087+0000","last_became_active":"2023-09-23T22:53:16.895026+0000","last_became_peered":"2023-09-23T22:53:16.895026+0000","last_unstale":"2023-09-23T22:53:17.905087+0000","last_undegraded":"2023-09-23T22:53:17.905087+0000","last_fullsized":"2023-09-23T22:53:17.905087+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:44.078221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.907777+0000","last_change":"2023-09-23T22:53:17.907538+0000","last_active":"2023-09-23T22:53:17.907777+0000","last_peered":"2023-09-23T22:53:17.907777+0000","last_clean":"2023-09-23T22:53:17.907777+0000","last_became_active":"2023-09-23T22:53:17.907261+0000","last_became_peered":"2023-09-23T22:53:17.907261+0000","last_unstale":"2023-09-23T22:53:17.907777+0000","last_undegraded":"2023-09-23T22:53:17.907777+0000","last_fullsized":"2023-09-23T22:53:17.907777+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:36.671603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.875959+0000","last_change":"2023-09-23T22:53:16.895707+0000","last_active":"2023-09-23T22:53:36.875959+0000","last_peered":"2023-09-23T22:53:36.875959+0000","last_clean":"2023-09-23T22:53:36.875959+0000","last_became_active":"2023-09-23T22:53:16.895451+0000","last_became_peered":"2023-09-23T22:53:16.895451+0000","last_unstale":"2023-09-23T22:53:36.875959+0000","last_undegraded":"2023-09-23T22:53:36.875959+0000","last_fullsized":"2023-09-23T22:53:36.875959+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:29.222424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.900713+0000","last_change":"2023-09-23T22:53:16.903902+0000","last_active":"2023-09-23T22:53:17.900713+0000","last_peered":"2023-09-23T22:53:17.900713+0000","last_clean":"2023-09-23T22:53:17.900713+0000","last_became_active":"2023-09-23T22:53:16.903613+0000","last_became_peered":"2023-09-23T22:53:16.903613+0000","last_unstale":"2023-09-23T22:53:17.900713+0000","last_undegraded":"2023-09-23T22:53:17.900713+0000","last_fullsized":"2023-09-23T22:53:17.900713+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:45.743285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.899489+0000","last_change":"2023-09-23T22:53:16.888352+0000","last_active":"2023-09-23T22:53:17.899489+0000","last_peered":"2023-09-23T22:53:17.899489+0000","last_clean":"2023-09-23T22:53:17.899489+0000","last_became_active":"2023-09-23T22:53:16.888074+0000","last_became_peered":"2023-09-23T22:53:16.888074+0000","last_unstale":"2023-09-23T22:53:17.899489+0000","last_undegraded":"2023-09-23T22:53:17.899489+0000","last_fullsized":"2023-09-23T22:53:17.899489+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:08.674783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138598+0000","last_change":"2023-09-23T22:53:18.137978+0000","last_active":"2023-09-23T22:53:18.138598+0000","last_peered":"2023-09-23T22:53:18.138598+0000","last_clean":"2023-09-23T22:53:18.138598+0000","last_became_active":"2023-09-23T22:53:18.137679+0000","last_became_peered":"2023-09-23T22:53:18.137679+0000","last_unstale":"2023-09-23T22:53:18.138598+0000","last_undegraded":"2023-09-23T22:53:18.138598+0000","last_fullsized":"2023-09-23T22:53:18.138598+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:17.681479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:20.393182+0000","last_change":"2023-09-23T22:53:16.887968+0000","last_active":"2023-09-23T22:53:20.393182+0000","last_peered":"2023-09-23T22:53:20.393182+0000","last_clean":"2023-09-23T22:53:20.393182+0000","last_became_active":"2023-09-23T22:53:16.887692+0000","last_became_peered":"2023-09-23T22:53:16.887692+0000","last_unstale":"2023-09-23T22:53:20.393182+0000","last_undegraded":"2023-09-23T22:53:20.393182+0000","last_fullsized":"2023-09-23T22:53:20.393182+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:24.662748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138490+0000","last_change":"2023-09-23T22:53:18.137944+0000","last_active":"2023-09-23T22:53:18.138490+0000","last_peered":"2023-09-23T22:53:18.138490+0000","last_clean":"2023-09-23T22:53:18.138490+0000","last_became_active":"2023-09-23T22:53:18.137652+0000","last_became_peered":"2023-09-23T22:53:18.137652+0000","last_unstale":"2023-09-23T22:53:18.138490+0000","last_undegraded":"2023-09-23T22:53:18.138490+0000","last_fullsized":"2023-09-23T22:53:18.138490+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:25:39.443733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.877265+0000","last_change":"2023-09-23T22:53:16.895724+0000","last_active":"2023-09-23T22:53:36.877265+0000","last_peered":"2023-09-23T22:53:36.877265+0000","last_clean":"2023-09-23T22:53:36.877265+0000","last_became_active":"2023-09-23T22:53:16.895451+0000","last_became_peered":"2023-09-23T22:53:16.895451+0000","last_unstale":"2023-09-23T22:53:36.877265+0000","last_undegraded":"2023-09-23T22:53:36.877265+0000","last_fullsized":"2023-09-23T22:53:36.877265+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:07.343242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.076154+0000","last_change":"2023-09-23T22:53:18.075506+0000","last_active":"2023-09-23T22:53:18.076154+0000","last_peered":"2023-09-23T22:53:18.076154+0000","last_clean":"2023-09-23T22:53:18.076154+0000","last_became_active":"2023-09-23T22:53:18.075206+0000","last_became_peered":"2023-09-23T22:53:18.075206+0000","last_unstale":"2023-09-23T22:53:18.076154+0000","last_undegraded":"2023-09-23T22:53:18.076154+0000","last_fullsized":"2023-09-23T22:53:18.076154+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:32:16.168188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:20.393228+0000","last_change":"2023-09-23T22:53:16.888803+0000","last_active":"2023-09-23T22:53:20.393228+0000","last_peered":"2023-09-23T22:53:20.393228+0000","last_clean":"2023-09-23T22:53:20.393228+0000","last_became_active":"2023-09-23T22:53:16.888513+0000","last_became_peered":"2023-09-23T22:53:16.888513+0000","last_unstale":"2023-09-23T22:53:20.393228+0000","last_undegraded":"2023-09-23T22:53:20.393228+0000","last_fullsized":"2023-09-23T22:53:20.393228+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:57.392303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.139499+0000","last_change":"2023-09-23T22:53:18.138127+0000","last_active":"2023-09-23T22:53:18.139499+0000","last_peered":"2023-09-23T22:53:18.139499+0000","last_clean":"2023-09-23T22:53:18.139499+0000","last_became_active":"2023-09-23T22:53:18.137851+0000","last_became_peered":"2023-09-23T22:53:18.137851+0000","last_unstale":"2023-09-23T22:53:18.139499+0000","last_undegraded":"2023-09-23T22:53:18.139499+0000","last_fullsized":"2023-09-23T22:53:18.139499+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:34.946793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.877654+0000","last_change":"2023-09-23T22:53:16.897754+0000","last_active":"2023-09-23T22:53:36.877654+0000","last_peered":"2023-09-23T22:53:36.877654+0000","last_clean":"2023-09-23T22:53:36.877654+0000","last_became_active":"2023-09-23T22:53:16.897488+0000","last_became_peered":"2023-09-23T22:53:16.897488+0000","last_unstale":"2023-09-23T22:53:36.877654+0000","last_undegraded":"2023-09-23T22:53:36.877654+0000","last_fullsized":"2023-09-23T22:53:36.877654+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:50:46.440416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.901201+0000","last_change":"2023-09-23T22:53:17.000324+0000","last_active":"2023-09-23T22:53:17.901201+0000","last_peered":"2023-09-23T22:53:17.901201+0000","last_clean":"2023-09-23T22:53:17.901201+0000","last_became_active":"2023-09-23T22:53:17.000070+0000","last_became_peered":"2023-09-23T22:53:17.000070+0000","last_unstale":"2023-09-23T22:53:17.901201+0000","last_undegraded":"2023-09-23T22:53:17.901201+0000","last_fullsized":"2023-09-23T22:53:17.901201+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:14.840209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.074491+0000","last_change":"2023-09-23T22:53:16.889362+0000","last_active":"2023-09-23T22:53:18.074491+0000","last_peered":"2023-09-23T22:53:18.074491+0000","last_clean":"2023-09-23T22:53:18.074491+0000","last_became_active":"2023-09-23T22:53:16.889081+0000","last_became_peered":"2023-09-23T22:53:16.889081+0000","last_unstale":"2023-09-23T22:53:18.074491+0000","last_undegraded":"2023-09-23T22:53:18.074491+0000","last_fullsized":"2023-09-23T22:53:18.074491+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:11.321151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.076163+0000","last_change":"2023-09-23T22:53:18.075841+0000","last_active":"2023-09-23T22:53:18.076163+0000","last_peered":"2023-09-23T22:53:18.076163+0000","last_clean":"2023-09-23T22:53:18.076163+0000","last_became_active":"2023-09-23T22:53:18.075550+0000","last_became_peered":"2023-09-23T22:53:18.075550+0000","last_unstale":"2023-09-23T22:53:18.076163+0000","last_undegraded":"2023-09-23T22:53:18.076163+0000","last_fullsized":"2023-09-23T22:53:18.076163+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:40.805558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.900561+0000","last_change":"2023-09-23T22:53:16.882261+0000","last_active":"2023-09-23T22:53:17.900561+0000","last_peered":"2023-09-23T22:53:17.900561+0000","last_clean":"2023-09-23T22:53:17.900561+0000","last_became_active":"2023-09-23T22:53:16.881924+0000","last_became_peered":"2023-09-23T22:53:16.881924+0000","last_unstale":"2023-09-23T22:53:17.900561+0000","last_undegraded":"2023-09-23T22:53:17.900561+0000","last_fullsized":"2023-09-23T22:53:17.900561+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:48.503506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.139383+0000","last_change":"2023-09-23T22:53:18.137940+0000","last_active":"2023-09-23T22:53:18.139383+0000","last_peered":"2023-09-23T22:53:18.139383+0000","last_clean":"2023-09-23T22:53:18.139383+0000","last_became_active":"2023-09-23T22:53:18.137643+0000","last_became_peered":"2023-09-23T22:53:18.137643+0000","last_unstale":"2023-09-23T22:53:18.139383+0000","last_undegraded":"2023-09-23T22:53:18.139383+0000","last_fullsized":"2023-09-23T22:53:18.139383+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:02.763345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.877260+0000","last_change":"2023-09-23T22:53:16.895361+0000","last_active":"2023-09-23T22:53:36.877260+0000","last_peered":"2023-09-23T22:53:36.877260+0000","last_clean":"2023-09-23T22:53:36.877260+0000","last_became_active":"2023-09-23T22:53:16.894995+0000","last_became_peered":"2023-09-23T22:53:16.894995+0000","last_unstale":"2023-09-23T22:53:36.877260+0000","last_undegraded":"2023-09-23T22:53:36.877260+0000","last_fullsized":"2023-09-23T22:53:36.877260+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:37.468777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.139665+0000","last_change":"2023-09-23T22:53:18.138452+0000","last_active":"2023-09-23T22:53:18.139665+0000","last_peered":"2023-09-23T22:53:18.139665+0000","last_clean":"2023-09-23T22:53:18.139665+0000","last_became_active":"2023-09-23T22:53:18.137756+0000","last_became_peered":"2023-09-23T22:53:18.137756+0000","last_unstale":"2023-09-23T22:53:18.139665+0000","last_undegraded":"2023-09-23T22:53:18.139665+0000","last_fullsized":"2023-09-23T22:53:18.139665+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:42.057582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.878271+0000","last_change":"2023-09-23T22:53:16.891930+0000","last_active":"2023-09-23T22:53:36.878271+0000","last_peered":"2023-09-23T22:53:36.878271+0000","last_clean":"2023-09-23T22:53:36.878271+0000","last_became_active":"2023-09-23T22:53:16.891658+0000","last_became_peered":"2023-09-23T22:53:16.891658+0000","last_unstale":"2023-09-23T22:53:36.878271+0000","last_undegraded":"2023-09-23T22:53:36.878271+0000","last_fullsized":"2023-09-23T22:53:36.878271+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:19.762775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.137911+0000","last_change":"2023-09-23T22:53:17.000151+0000","last_active":"2023-09-23T22:53:18.137911+0000","last_peered":"2023-09-23T22:53:18.137911+0000","last_clean":"2023-09-23T22:53:18.137911+0000","last_became_active":"2023-09-23T22:53:16.999582+0000","last_became_peered":"2023-09-23T22:53:16.999582+0000","last_unstale":"2023-09-23T22:53:18.137911+0000","last_undegraded":"2023-09-23T22:53:18.137911+0000","last_fullsized":"2023-09-23T22:53:18.137911+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:16.874918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.877557+0000","last_change":"2023-09-23T22:53:16.895495+0000","last_active":"2023-09-23T22:53:36.877557+0000","last_peered":"2023-09-23T22:53:36.877557+0000","last_clean":"2023-09-23T22:53:36.877557+0000","last_became_active":"2023-09-23T22:53:16.895192+0000","last_became_peered":"2023-09-23T22:53:16.895192+0000","last_unstale":"2023-09-23T22:53:36.877557+0000","last_undegraded":"2023-09-23T22:53:36.877557+0000","last_fullsized":"2023-09-23T22:53:36.877557+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:41.114595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.075640+0000","last_change":"2023-09-23T22:53:18.075428+0000","last_active":"2023-09-23T22:53:18.075640+0000","last_peered":"2023-09-23T22:53:18.075640+0000","last_clean":"2023-09-23T22:53:18.075640+0000","last_became_active":"2023-09-23T22:53:18.075112+0000","last_became_peered":"2023-09-23T22:53:18.075112+0000","last_unstale":"2023-09-23T22:53:18.075640+0000","last_undegraded":"2023-09-23T22:53:18.075640+0000","last_fullsized":"2023-09-23T22:53:18.075640+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:49.645031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.899387+0000","last_change":"2023-09-23T22:53:16.893248+0000","last_active":"2023-09-23T22:53:17.899387+0000","last_peered":"2023-09-23T22:53:17.899387+0000","last_clean":"2023-09-23T22:53:17.899387+0000","last_became_active":"2023-09-23T22:53:16.892954+0000","last_became_peered":"2023-09-23T22:53:16.892954+0000","last_unstale":"2023-09-23T22:53:17.899387+0000","last_undegraded":"2023-09-23T22:53:17.899387+0000","last_fullsized":"2023-09-23T22:53:17.899387+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:22.235094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.139416+0000","last_change":"2023-09-23T22:53:18.137964+0000","last_active":"2023-09-23T22:53:18.139416+0000","last_peered":"2023-09-23T22:53:18.139416+0000","last_clean":"2023-09-23T22:53:18.139416+0000","last_became_active":"2023-09-23T22:53:18.137683+0000","last_became_peered":"2023-09-23T22:53:18.137683+0000","last_unstale":"2023-09-23T22:53:18.139416+0000","last_undegraded":"2023-09-23T22:53:18.139416+0000","last_fullsized":"2023-09-23T22:53:18.139416+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:51.842249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.900576+0000","last_change":"2023-09-23T22:53:16.886203+0000","last_active":"2023-09-23T22:53:17.900576+0000","last_peered":"2023-09-23T22:53:17.900576+0000","last_clean":"2023-09-23T22:53:17.900576+0000","last_became_active":"2023-09-23T22:53:16.885705+0000","last_became_peered":"2023-09-23T22:53:16.885705+0000","last_unstale":"2023-09-23T22:53:17.900576+0000","last_undegraded":"2023-09-23T22:53:17.900576+0000","last_fullsized":"2023-09-23T22:53:17.900576+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:08.071819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.139272+0000","last_change":"2023-09-23T22:53:18.138026+0000","last_active":"2023-09-23T22:53:18.139272+0000","last_peered":"2023-09-23T22:53:18.139272+0000","last_clean":"2023-09-23T22:53:18.139272+0000","last_became_active":"2023-09-23T22:53:18.137722+0000","last_became_peered":"2023-09-23T22:53:18.137722+0000","last_unstale":"2023-09-23T22:53:18.139272+0000","last_undegraded":"2023-09-23T22:53:18.139272+0000","last_fullsized":"2023-09-23T22:53:18.139272+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:13.377862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.902745+0000","last_change":"2023-09-23T22:53:16.894299+0000","last_active":"2023-09-23T22:53:17.902745+0000","last_peered":"2023-09-23T22:53:17.902745+0000","last_clean":"2023-09-23T22:53:17.902745+0000","last_became_active":"2023-09-23T22:53:16.893621+0000","last_became_peered":"2023-09-23T22:53:16.893621+0000","last_unstale":"2023-09-23T22:53:17.902745+0000","last_undegraded":"2023-09-23T22:53:17.902745+0000","last_fullsized":"2023-09-23T22:53:17.902745+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:55.765968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.901274+0000","last_change":"2023-09-23T22:53:17.000931+0000","last_active":"2023-09-23T22:53:17.901274+0000","last_peered":"2023-09-23T22:53:17.901274+0000","last_clean":"2023-09-23T22:53:17.901274+0000","last_became_active":"2023-09-23T22:53:17.000684+0000","last_became_peered":"2023-09-23T22:53:17.000684+0000","last_unstale":"2023-09-23T22:53:17.901274+0000","last_undegraded":"2023-09-23T22:53:17.901274+0000","last_fullsized":"2023-09-23T22:53:17.901274+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:58.542147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138704+0000","last_change":"2023-09-23T22:53:16.887441+0000","last_active":"2023-09-23T22:53:18.138704+0000","last_peered":"2023-09-23T22:53:18.138704+0000","last_clean":"2023-09-23T22:53:18.138704+0000","last_became_active":"2023-09-23T22:53:16.887111+0000","last_became_peered":"2023-09-23T22:53:16.887111+0000","last_unstale":"2023-09-23T22:53:18.138704+0000","last_undegraded":"2023-09-23T22:53:18.138704+0000","last_fullsized":"2023-09-23T22:53:18.138704+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:40.227618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.140865+0000","last_change":"2023-09-23T22:53:18.140550+0000","last_active":"2023-09-23T22:53:18.140865+0000","last_peered":"2023-09-23T22:53:18.140865+0000","last_clean":"2023-09-23T22:53:18.140865+0000","last_became_active":"2023-09-23T22:53:18.140252+0000","last_became_peered":"2023-09-23T22:53:18.140252+0000","last_unstale":"2023-09-23T22:53:18.140865+0000","last_undegraded":"2023-09-23T22:53:18.140865+0000","last_fullsized":"2023-09-23T22:53:18.140865+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:16.122288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:20.498118+0000","last_change":"2023-09-23T22:53:16.893423+0000","last_active":"2023-09-23T22:53:20.498118+0000","last_peered":"2023-09-23T22:53:20.498118+0000","last_clean":"2023-09-23T22:53:20.498118+0000","last_became_active":"2023-09-23T22:53:16.893144+0000","last_became_peered":"2023-09-23T22:53:16.893144+0000","last_unstale":"2023-09-23T22:53:20.498118+0000","last_undegraded":"2023-09-23T22:53:20.498118+0000","last_fullsized":"2023-09-23T22:53:20.498118+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:50.803912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234016,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6600,"kb_used_data":1280,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738552,"statfs":{"total":95995035648,"available":95988277248,"internally_reserved":0,"allocated":1310720,"data_stored":891354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.53100000000000003,"15min":0.54000000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.38},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.748,"15min":0.78600000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.79700000000000004,"5min":5.1849999999999996,"15min":5.1849999999999996},"last":0.42599999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.64800000000000002,"15min":0.66800000000000004},"min":{"1min":0.38700000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.88100000000000001,"5min":2.903,"15min":2.903},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.85699999999999998,"15min":0.92000000000000004},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.73699999999999999,"5min":5.0659999999999998,"15min":5.0659999999999998},"last":0.46600000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.76600000000000001,"15min":0.80400000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.81299999999999994,"5min":3.0259999999999998,"15min":3.0259999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.65400000000000003,"15min":0.67200000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.88400000000000001,"5min":2.649,"15min":2.649},"last":0.57099999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.55000000000000004,"15min":0.55300000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.85999999999999999,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.79700000000000004,"15min":0.83499999999999996},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.90800000000000003,"5min":4.9539999999999997,"15min":4.9539999999999997},"last":0.65000000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.48099999999999998,"15min":0.47899999999999998},"min":{"1min":0.34599999999999997,"5min":0.308,"15min":0.308},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.495,"15min":0.49099999999999999},"min":{"1min":0.33700000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.66600000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.51500000000000001,"15min":0.51100000000000001},"min":{"1min":0.36099999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.501,"15min":0.498},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.55100000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.54300000000000004,"15min":0.54400000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.628},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.56299999999999994,"15min":0.56100000000000005},"min":{"1min":0.438,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.70899999999999996}]}]},{"osd":6,"up_from":46,"seq":197568495652,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6584,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738568,"statfs":{"total":95995035648,"available":95988293632,"internally_reserved":0,"allocated":1294336,"data_stored":890943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.39300000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.51000000000000001,"15min":0.51500000000000001},"min":{"1min":0.35499999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.66800000000000004,"5min":0.751,"15min":0.751},"last":0.51400000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.52100000000000002,"15min":0.51700000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56499999999999995,"15min":0.56599999999999995},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.748,"5min":0.753,"15min":0.753},"last":0.60499999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.68999999999999995,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.48499999999999999,"15min":0.48099999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.41199999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.52200000000000002,"15min":0.52500000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.68899999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.41699999999999998,"5min":0.38,"15min":0.38},"max":{"1min":0.78700000000000003,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.64400000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.43099999999999999,"15min":0.42699999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.5},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.46800000000000003,"15min":0.46200000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.53400000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.44700000000000001,"15min":0.439},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.497,"15min":0.495},"min":{"1min":0.33700000000000002,"5min":0.316,"15min":0.316},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.627}]},{"osd":7,"last update":"Sat Sep 23 22:53:49 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.54400000000000004,"15min":0.54100000000000004},"min":{"1min":0.41399999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.55000000000000004,"15min":0.54700000000000004},"min":{"1min":0.35899999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.73299999999999998,"5min":0.879,"15min":0.879},"last":0.66900000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574907,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6764,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93738388,"statfs":{"total":95995035648,"available":95988109312,"internally_reserved":0,"allocated":692224,"data_stored":299557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:32 2023","interfaces":[{"interface":"back","average":{"1min":0.315,"5min":0.34499999999999997,"15min":0.33700000000000002},"min":{"1min":0.20499999999999999,"5min":0.191,"15min":0.191},"max":{"1min":0.58999999999999997,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.36099999999999999,"15min":0.34499999999999997},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.57399999999999995,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.32400000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:53:56 2023","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":0.39600000000000002,"15min":0.39800000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.56000000000000005,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.42399999999999999,"15min":0.41599999999999998},"min":{"1min":0.307,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.61599999999999999,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.38800000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:53:23 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.40999999999999998,"15min":0.41799999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.44400000000000001,"15min":0.45800000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.70899999999999996,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.34699999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:53:40 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.59899999999999998,"15min":0.57199999999999995},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.73399999999999999,"5min":5.3710000000000004,"15min":5.3710000000000004},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.505,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.753,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.83399999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.56000000000000005,"15min":0.56599999999999995},"min":{"1min":0.34799999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.79200000000000004,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59399999999999997,"15min":0.59799999999999998},"min":{"1min":0.40999999999999998,"5min":0.38,"15min":0.38},"max":{"1min":0.92100000000000004,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.84599999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:53:23 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.62,"15min":0.59699999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":5.2999999999999998,"5min":5.2999999999999998,"15min":5.2999999999999998},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.61799999999999999,"15min":0.627},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.64400000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:53:46 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.80800000000000005,"15min":0.84699999999999998},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.81499999999999995,"5min":5.3390000000000004,"15min":5.3390000000000004},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.59499999999999997,"15min":0.60699999999999998},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.68500000000000005,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.57799999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738431,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7416,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93737736,"statfs":{"total":95995035648,"available":95987441664,"internally_reserved":0,"allocated":1294336,"data_stored":890501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:53:29 2023","interfaces":[{"interface":"back","average":{"1min":0.312,"5min":0.34499999999999997,"15min":0.34499999999999997},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.45600000000000002,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.33000000000000002,"5min":0.35999999999999999,"15min":0.35499999999999998},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.29099999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:53:58 2023","interfaces":[{"interface":"back","average":{"1min":0.32100000000000001,"5min":0.38900000000000001,"15min":0.41099999999999998},"min":{"1min":0.23100000000000001,"5min":0.188,"15min":0.188},"max":{"1min":0.47599999999999998,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.378,"5min":0.40400000000000003,"15min":0.40799999999999997},"min":{"1min":0.29399999999999998,"5min":0.249,"15min":0.249},"max":{"1min":0.56699999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.33400000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.443,"15min":0.46899999999999997},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.59299999999999997,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.441,"15min":0.46200000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.58599999999999997,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.316}]},{"osd":4,"last update":"Sat Sep 23 22:53:43 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.55200000000000005,"15min":0.55600000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.56799999999999995,"15min":0.58199999999999996},"min":{"1min":0.36499999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.63300000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.59199999999999997,"15min":0.59799999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.56899999999999995,"15min":0.57099999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.58799999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.59099999999999997,"15min":0.59999999999999998},"min":{"1min":0.41299999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.81299999999999994,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.58299999999999996,"15min":0.58799999999999997},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.77900000000000003,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.57499999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:53:46 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.57499999999999996,"15min":0.58399999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.69899999999999995,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.57799999999999996,"15min":0.58699999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.71699999999999997,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.59899999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411381,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6640,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93738512,"statfs":{"total":95995035648,"available":95988236288,"internally_reserved":0,"allocated":696320,"data_stored":299484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:59 2023","interfaces":[{"interface":"back","average":{"1min":0.308,"5min":0.34699999999999998,"15min":0.34000000000000002},"min":{"1min":0.24299999999999999,"5min":0.161,"15min":0.161},"max":{"1min":0.46300000000000002,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.32800000000000001,"5min":0.371,"15min":0.36199999999999999},"min":{"1min":0.24299999999999999,"5min":0.188,"15min":0.188},"max":{"1min":0.434,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.34499999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:53:59 2023","interfaces":[{"interface":"back","average":{"1min":0.30099999999999999,"5min":0.37,"15min":0.36299999999999999},"min":{"1min":0.23000000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.379,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.33500000000000002,"5min":0.378,"15min":0.35899999999999999},"min":{"1min":0.24299999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.41699999999999998,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.39200000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:53:21 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.42299999999999999,"15min":0.43099999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.437,"15min":0.439},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.98199999999999998,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.28100000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.59099999999999997,"15min":0.58299999999999996},"min":{"1min":0.441,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.79100000000000004,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.59199999999999997,"15min":0.58599999999999997},"min":{"1min":0.40200000000000002,"5min":0.379,"15min":0.379},"max":{"1min":0.81999999999999995,"5min":1.05,"15min":1.05},"last":0.60599999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.42499999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59199999999999997,"15min":0.59099999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.52300000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:53:21 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.63100000000000001,"15min":0.63600000000000001},"min":{"1min":0.45700000000000002,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.502},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.63900000000000001,"15min":0.64500000000000002},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"last":0.57199999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:53:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.63300000000000001,"15min":0.64700000000000002},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.74299999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.66500000000000004,"15min":0.68300000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.69299999999999995,"5min":1.042,"15min":1.042},"last":0.54300000000000004}]}]},{"osd":3,"up_from":25,"seq":107374182449,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6128,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739024,"statfs":{"total":95995035648,"available":95988760576,"internally_reserved":0,"allocated":696320,"data_stored":299647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.29599999999999999,"5min":0.34200000000000003,"15min":0.34799999999999998},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.433,"5min":0.621,"15min":0.621},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.34000000000000002,"5min":0.41599999999999998,"15min":0.438},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.50700000000000001,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.81100000000000005}]},{"osd":1,"last update":"Sat Sep 23 22:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.40799999999999997,"15min":0.41899999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.496,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":1.7749999999999999},{"interface":"front","average":{"1min":0.32500000000000001,"5min":0.40899999999999997,"15min":0.42599999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.51700000000000002,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.41699999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.436,"15min":0.45100000000000001},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.51800000000000002,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.37,"5min":0.441,"15min":0.45300000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.52900000000000003,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.622}]},{"osd":4,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.54500000000000004,"15min":0.55300000000000005},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.143,"5min":1.143,"15min":1.143},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.56699999999999995,"15min":0.57599999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.64800000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:53:01 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.57599999999999996,"15min":0.58599999999999997},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.59599999999999997,"15min":0.60199999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.77700000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.58299999999999996,"15min":0.59399999999999997},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.754,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.57999999999999996,"15min":0.58899999999999997},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72799999999999998,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.752}]},{"osd":7,"last update":"Sat Sep 23 22:53:48 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.56999999999999995,"15min":0.57299999999999995},"min":{"1min":0.37,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.171,"5min":1.171,"15min":1.171},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.374,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.71799999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018925,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6212,"kb_used_data":700,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738940,"statfs":{"total":95995035648,"available":95988674560,"internally_reserved":0,"allocated":716800,"data_stored":301239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:39 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.496,"15min":0.5},"min":{"1min":0.27800000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.68600000000000005,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.57999999999999996,"15min":0.54000000000000004},"min":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.9510000000000005,"5min":8.9510000000000005,"15min":8.9510000000000005},"last":0.46400000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:53:39 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.57299999999999995,"15min":0.52500000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.2089999999999996,"5min":9.2089999999999996,"15min":9.2089999999999996},"last":0.44},{"interface":"front","average":{"1min":0.501,"5min":0.51200000000000001,"15min":0.50800000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.626,"5min":0.752,"15min":0.752},"last":0.40899999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:53:39 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.49099999999999999,"15min":0.499},"min":{"1min":0.33100000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.57499999999999996,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.628},{"interface":"front","average":{"1min":0.88,"5min":0.60299999999999998,"15min":0.56100000000000005},"min":{"1min":0.32600000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.0429999999999993,"5min":9.0429999999999993,"15min":9.0429999999999993},"last":0.505}]},{"osd":3,"last update":"Sat Sep 23 22:53:39 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.51000000000000001,"15min":0.51800000000000002},"min":{"1min":0.372,"5min":0.314,"15min":0.314},"max":{"1min":0.63900000000000001,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.51900000000000002,"15min":0.52400000000000002},"min":{"1min":0.38200000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.60599999999999998,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.51300000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:53:03 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.44800000000000001,"15min":0.44},"min":{"1min":0.29799999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.44400000000000001,"15min":0.436},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.67700000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.58399999999999996}]},{"osd":6,"last update":"Sat Sep 23 22:53:25 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.45700000000000002,"15min":0.45100000000000001},"min":{"1min":0.34100000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.66600000000000004,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.51300000000000001,"15min":0.504},"min":{"1min":0.33000000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.72799999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.68999999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:53:52 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.58599999999999997,"15min":0.58399999999999996},"min":{"1min":0.375,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.56899999999999995,"15min":0.57099999999999995},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.752,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.60299999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757288,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6064,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739088,"statfs":{"total":95995035648,"available":95988826112,"internally_reserved":0,"allocated":696320,"data_stored":299916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.56699999999999995,"15min":0.57299999999999995},"min":{"1min":0.373,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79400000000000004,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.48799999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.55200000000000005,"15min":0.55700000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.70499999999999996,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.52700000000000002,"15min":0.53100000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.55000000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.53900000000000003,"15min":0.54500000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.68300000000000005}]},{"osd":3,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.55600000000000005,"15min":0.55700000000000005},"min":{"1min":0.39700000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.56100000000000005,"15min":0.55900000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.73199999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.39200000000000002,"15min":0.38300000000000001},"min":{"1min":0.318,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.505,"5min":0.439,"15min":0.42799999999999999},"min":{"1min":0.30599999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.747}]},{"osd":6,"last update":"Sat Sep 23 22:53:26 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.46400000000000002,"15min":0.45400000000000001},"min":{"1min":0.35099999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.48399999999999999,"15min":0.47299999999999998},"min":{"1min":0.33600000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.76300000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:53:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.55900000000000005,"15min":0.55400000000000005},"min":{"1min":0.36499999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.52200000000000002,"15min":0.51800000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.77000000000000002,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.70299999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:54:05.634 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:54:05.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:05 smithi138 ceph-mon[104418]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:05.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:05 smithi110 ceph-mon[100243]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:05.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:05 smithi110 ceph-mon[118821]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:06.328 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:06 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:06 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:06 smithi138 ceph-mon[104418]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:06.755 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:06 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:06.755 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:06 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:06.755 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:06 smithi110 ceph-mon[100243]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:06.755 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:06 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:06.756 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:06 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:06.756 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:06 smithi110 ceph-mon[118821]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:08.128 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:54:08.129 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T22:54:08.807 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":67,"stamp":"2023-09-23T22:54:07.810476+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52408,"kb_used_data":7224,"kb_used_omap":0,"kb_used_meta":45120,"kb_avail":749908808,"statfs":{"total":767960285184,"available":767906619392,"internally_reserved":0,"allocated":7397376,"data_stored":4172641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46202880},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002534"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.139602+0000","last_change":"2023-09-23T22:53:18.138105+0000","last_active":"2023-09-23T22:53:18.139602+0000","last_peered":"2023-09-23T22:53:18.139602+0000","last_clean":"2023-09-23T22:53:18.139602+0000","last_became_active":"2023-09-23T22:53:18.137820+0000","last_became_peered":"2023-09-23T22:53:18.137820+0000","last_unstale":"2023-09-23T22:53:18.139602+0000","last_undegraded":"2023-09-23T22:53:18.139602+0000","last_fullsized":"2023-09-23T22:53:18.139602+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:11.526142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.877077+0000","last_change":"2023-09-23T22:53:16.886024+0000","last_active":"2023-09-23T22:53:36.877077+0000","last_peered":"2023-09-23T22:53:36.877077+0000","last_clean":"2023-09-23T22:53:36.877077+0000","last_became_active":"2023-09-23T22:53:16.885284+0000","last_became_peered":"2023-09-23T22:53:16.885284+0000","last_unstale":"2023-09-23T22:53:36.877077+0000","last_undegraded":"2023-09-23T22:53:36.877077+0000","last_fullsized":"2023-09-23T22:53:36.877077+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:38.524289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.906673+0000","last_change":"2023-09-23T22:53:17.906381+0000","last_active":"2023-09-23T22:53:17.906673+0000","last_peered":"2023-09-23T22:53:17.906673+0000","last_clean":"2023-09-23T22:53:17.906673+0000","last_became_active":"2023-09-23T22:53:17.906010+0000","last_became_peered":"2023-09-23T22:53:17.906010+0000","last_unstale":"2023-09-23T22:53:17.906673+0000","last_undegraded":"2023-09-23T22:53:17.906673+0000","last_fullsized":"2023-09-23T22:53:17.906673+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:39.381896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:20.246164+0000","last_change":"2023-09-23T22:53:16.889809+0000","last_active":"2023-09-23T22:53:20.246164+0000","last_peered":"2023-09-23T22:53:20.246164+0000","last_clean":"2023-09-23T22:53:20.246164+0000","last_became_active":"2023-09-23T22:53:16.889533+0000","last_became_peered":"2023-09-23T22:53:16.889533+0000","last_unstale":"2023-09-23T22:53:20.246164+0000","last_undegraded":"2023-09-23T22:53:20.246164+0000","last_fullsized":"2023-09-23T22:53:20.246164+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:48.318427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.906517+0000","last_change":"2023-09-23T22:53:17.905814+0000","last_active":"2023-09-23T22:53:17.906517+0000","last_peered":"2023-09-23T22:53:17.906517+0000","last_clean":"2023-09-23T22:53:17.906517+0000","last_became_active":"2023-09-23T22:53:17.905541+0000","last_became_peered":"2023-09-23T22:53:17.905541+0000","last_unstale":"2023-09-23T22:53:17.906517+0000","last_undegraded":"2023-09-23T22:53:17.906517+0000","last_fullsized":"2023-09-23T22:53:17.906517+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:38:22.986263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.876830+0000","last_change":"2023-09-23T22:53:16.891297+0000","last_active":"2023-09-23T22:53:36.876830+0000","last_peered":"2023-09-23T22:53:36.876830+0000","last_clean":"2023-09-23T22:53:36.876830+0000","last_became_active":"2023-09-23T22:53:16.891025+0000","last_became_peered":"2023-09-23T22:53:16.891025+0000","last_unstale":"2023-09-23T22:53:36.876830+0000","last_undegraded":"2023-09-23T22:53:36.876830+0000","last_fullsized":"2023-09-23T22:53:36.876830+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:40:50.608339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.139686+0000","last_change":"2023-09-23T22:53:18.138626+0000","last_active":"2023-09-23T22:53:18.139686+0000","last_peered":"2023-09-23T22:53:18.139686+0000","last_clean":"2023-09-23T22:53:18.139686+0000","last_became_active":"2023-09-23T22:53:18.138148+0000","last_became_peered":"2023-09-23T22:53:18.138148+0000","last_unstale":"2023-09-23T22:53:18.139686+0000","last_undegraded":"2023-09-23T22:53:18.139686+0000","last_fullsized":"2023-09-23T22:53:18.139686+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:57.413976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.876758+0000","last_change":"2023-09-23T22:53:16.894996+0000","last_active":"2023-09-23T22:53:36.876758+0000","last_peered":"2023-09-23T22:53:36.876758+0000","last_clean":"2023-09-23T22:53:36.876758+0000","last_became_active":"2023-09-23T22:53:16.894734+0000","last_became_peered":"2023-09-23T22:53:16.894734+0000","last_unstale":"2023-09-23T22:53:36.876758+0000","last_undegraded":"2023-09-23T22:53:36.876758+0000","last_fullsized":"2023-09-23T22:53:36.876758+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:51.380926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.899794+0000","last_change":"2023-09-23T22:53:16.999836+0000","last_active":"2023-09-23T22:53:17.899794+0000","last_peered":"2023-09-23T22:53:17.899794+0000","last_clean":"2023-09-23T22:53:17.899794+0000","last_became_active":"2023-09-23T22:53:16.999448+0000","last_became_peered":"2023-09-23T22:53:16.999448+0000","last_unstale":"2023-09-23T22:53:17.899794+0000","last_undegraded":"2023-09-23T22:53:17.899794+0000","last_fullsized":"2023-09-23T22:53:17.899794+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:21.113817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.904433+0000","last_change":"2023-09-23T22:53:16.895075+0000","last_active":"2023-09-23T22:53:18.904433+0000","last_peered":"2023-09-23T22:53:18.904433+0000","last_clean":"2023-09-23T22:53:18.904433+0000","last_became_active":"2023-09-23T22:53:16.894785+0000","last_became_peered":"2023-09-23T22:53:16.894785+0000","last_unstale":"2023-09-23T22:53:18.904433+0000","last_undegraded":"2023-09-23T22:53:18.904433+0000","last_fullsized":"2023-09-23T22:53:18.904433+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:45.846898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.906502+0000","last_change":"2023-09-23T22:53:17.905706+0000","last_active":"2023-09-23T22:53:17.906502+0000","last_peered":"2023-09-23T22:53:17.906502+0000","last_clean":"2023-09-23T22:53:17.906502+0000","last_became_active":"2023-09-23T22:53:17.905309+0000","last_became_peered":"2023-09-23T22:53:17.905309+0000","last_unstale":"2023-09-23T22:53:17.906502+0000","last_undegraded":"2023-09-23T22:53:17.906502+0000","last_fullsized":"2023-09-23T22:53:17.906502+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:06.105426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.904722+0000","last_change":"2023-09-23T22:53:16.894451+0000","last_active":"2023-09-23T22:53:18.904722+0000","last_peered":"2023-09-23T22:53:18.904722+0000","last_clean":"2023-09-23T22:53:18.904722+0000","last_became_active":"2023-09-23T22:53:16.894169+0000","last_became_peered":"2023-09-23T22:53:16.894169+0000","last_unstale":"2023-09-23T22:53:18.904722+0000","last_undegraded":"2023-09-23T22:53:18.904722+0000","last_fullsized":"2023-09-23T22:53:18.904722+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:20.585622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.906216+0000","last_change":"2023-09-23T22:52:21.315891+0000","last_active":"2023-09-23T22:53:17.906216+0000","last_peered":"2023-09-23T22:53:17.906216+0000","last_clean":"2023-09-23T22:53:17.906216+0000","last_became_active":"2023-09-23T22:52:21.315580+0000","last_became_peered":"2023-09-23T22:52:21.315580+0000","last_unstale":"2023-09-23T22:53:17.906216+0000","last_undegraded":"2023-09-23T22:53:17.906216+0000","last_fullsized":"2023-09-23T22:53:17.906216+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:33.692738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.075643+0000","last_change":"2023-09-23T22:53:18.075389+0000","last_active":"2023-09-23T22:53:18.075643+0000","last_peered":"2023-09-23T22:53:18.075643+0000","last_clean":"2023-09-23T22:53:18.075643+0000","last_became_active":"2023-09-23T22:53:18.075127+0000","last_became_peered":"2023-09-23T22:53:18.075127+0000","last_unstale":"2023-09-23T22:53:18.075643+0000","last_undegraded":"2023-09-23T22:53:18.075643+0000","last_fullsized":"2023-09-23T22:53:18.075643+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:07.596409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138550+0000","last_change":"2023-09-23T22:52:21.890478+0000","last_active":"2023-09-23T22:53:18.138550+0000","last_peered":"2023-09-23T22:53:18.138550+0000","last_clean":"2023-09-23T22:53:18.138550+0000","last_became_active":"2023-09-23T22:52:21.890133+0000","last_became_peered":"2023-09-23T22:52:21.890133+0000","last_unstale":"2023-09-23T22:53:18.138550+0000","last_undegraded":"2023-09-23T22:53:18.138550+0000","last_fullsized":"2023-09-23T22:53:18.138550+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:34:57.292871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.877565+0000","last_change":"2023-09-23T22:53:16.890252+0000","last_active":"2023-09-23T22:53:36.877565+0000","last_peered":"2023-09-23T22:53:36.877565+0000","last_clean":"2023-09-23T22:53:36.877565+0000","last_became_active":"2023-09-23T22:53:16.889999+0000","last_became_peered":"2023-09-23T22:53:16.889999+0000","last_unstale":"2023-09-23T22:53:36.877565+0000","last_undegraded":"2023-09-23T22:53:36.877565+0000","last_fullsized":"2023-09-23T22:53:36.877565+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:32.424686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.900875+0000","last_change":"2023-09-23T22:53:17.000093+0000","last_active":"2023-09-23T22:53:17.900875+0000","last_peered":"2023-09-23T22:53:17.900875+0000","last_clean":"2023-09-23T22:53:17.900875+0000","last_became_active":"2023-09-23T22:53:16.999800+0000","last_became_peered":"2023-09-23T22:53:16.999800+0000","last_unstale":"2023-09-23T22:53:17.900875+0000","last_undegraded":"2023-09-23T22:53:17.900875+0000","last_fullsized":"2023-09-23T22:53:17.900875+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:48.627741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138936+0000","last_change":"2023-09-23T22:53:16.893176+0000","last_active":"2023-09-23T22:53:18.138936+0000","last_peered":"2023-09-23T22:53:18.138936+0000","last_clean":"2023-09-23T22:53:18.138936+0000","last_became_active":"2023-09-23T22:53:16.892857+0000","last_became_peered":"2023-09-23T22:53:16.892857+0000","last_unstale":"2023-09-23T22:53:18.138936+0000","last_undegraded":"2023-09-23T22:53:18.138936+0000","last_fullsized":"2023-09-23T22:53:18.138936+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:34.831545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.074684+0000","last_change":"2023-09-23T22:52:21.890419+0000","last_active":"2023-09-23T22:53:18.074684+0000","last_peered":"2023-09-23T22:53:18.074684+0000","last_clean":"2023-09-23T22:53:18.074684+0000","last_became_active":"2023-09-23T22:52:21.890113+0000","last_became_peered":"2023-09-23T22:52:21.890113+0000","last_unstale":"2023-09-23T22:53:18.074684+0000","last_undegraded":"2023-09-23T22:53:18.074684+0000","last_fullsized":"2023-09-23T22:53:18.074684+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:23.384783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.901427+0000","last_change":"2023-09-23T22:53:16.905062+0000","last_active":"2023-09-23T22:53:17.901427+0000","last_peered":"2023-09-23T22:53:17.901427+0000","last_clean":"2023-09-23T22:53:17.901427+0000","last_became_active":"2023-09-23T22:53:16.904812+0000","last_became_peered":"2023-09-23T22:53:16.904812+0000","last_unstale":"2023-09-23T22:53:17.901427+0000","last_undegraded":"2023-09-23T22:53:17.901427+0000","last_fullsized":"2023-09-23T22:53:17.901427+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:44.859759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.900447+0000","last_change":"2023-09-23T22:53:16.888998+0000","last_active":"2023-09-23T22:53:17.900447+0000","last_peered":"2023-09-23T22:53:17.900447+0000","last_clean":"2023-09-23T22:53:17.900447+0000","last_became_active":"2023-09-23T22:53:16.888784+0000","last_became_peered":"2023-09-23T22:53:16.888784+0000","last_unstale":"2023-09-23T22:53:17.900447+0000","last_undegraded":"2023-09-23T22:53:17.900447+0000","last_fullsized":"2023-09-23T22:53:17.900447+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:18.878414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.074616+0000","last_change":"2023-09-23T22:52:21.315773+0000","last_active":"2023-09-23T22:53:18.074616+0000","last_peered":"2023-09-23T22:53:18.074616+0000","last_clean":"2023-09-23T22:53:18.074616+0000","last_became_active":"2023-09-23T22:52:21.315503+0000","last_became_peered":"2023-09-23T22:52:21.315503+0000","last_unstale":"2023-09-23T22:53:18.074616+0000","last_undegraded":"2023-09-23T22:53:18.074616+0000","last_fullsized":"2023-09-23T22:53:18.074616+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:39:58.756695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.076496+0000","last_change":"2023-09-23T22:53:18.075922+0000","last_active":"2023-09-23T22:53:18.076496+0000","last_peered":"2023-09-23T22:53:18.076496+0000","last_clean":"2023-09-23T22:53:18.076496+0000","last_became_active":"2023-09-23T22:53:18.075621+0000","last_became_peered":"2023-09-23T22:53:18.075621+0000","last_unstale":"2023-09-23T22:53:18.076496+0000","last_undegraded":"2023-09-23T22:53:18.076496+0000","last_fullsized":"2023-09-23T22:53:18.076496+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:01.490333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.902804+0000","last_change":"2023-09-23T22:53:16.884368+0000","last_active":"2023-09-23T22:53:17.902804+0000","last_peered":"2023-09-23T22:53:17.902804+0000","last_clean":"2023-09-23T22:53:17.902804+0000","last_became_active":"2023-09-23T22:53:16.884081+0000","last_became_peered":"2023-09-23T22:53:16.884081+0000","last_unstale":"2023-09-23T22:53:17.902804+0000","last_undegraded":"2023-09-23T22:53:17.902804+0000","last_fullsized":"2023-09-23T22:53:17.902804+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:45.320726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":275,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.074538+0000","last_change":"2023-09-23T22:51:47.222693+0000","last_active":"2023-09-23T22:53:18.074538+0000","last_peered":"2023-09-23T22:53:18.074538+0000","last_clean":"2023-09-23T22:53:18.074538+0000","last_became_active":"2023-09-23T22:51:47.222168+0000","last_became_peered":"2023-09-23T22:51:47.222168+0000","last_unstale":"2023-09-23T22:53:18.074538+0000","last_undegraded":"2023-09-23T22:53:18.074538+0000","last_fullsized":"2023-09-23T22:53:18.074538+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T22:49:49.548021+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:16.935480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.899375+0000","last_change":"2023-09-23T22:52:21.315119+0000","last_active":"2023-09-23T22:53:17.899375+0000","last_peered":"2023-09-23T22:53:17.899375+0000","last_clean":"2023-09-23T22:53:17.899375+0000","last_became_active":"2023-09-23T22:52:21.314817+0000","last_became_peered":"2023-09-23T22:52:21.314817+0000","last_unstale":"2023-09-23T22:53:17.899375+0000","last_undegraded":"2023-09-23T22:53:17.899375+0000","last_fullsized":"2023-09-23T22:53:17.899375+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:22.207385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138454+0000","last_change":"2023-09-23T22:53:17.000507+0000","last_active":"2023-09-23T22:53:18.138454+0000","last_peered":"2023-09-23T22:53:18.138454+0000","last_clean":"2023-09-23T22:53:18.138454+0000","last_became_active":"2023-09-23T22:53:17.000238+0000","last_became_peered":"2023-09-23T22:53:17.000238+0000","last_unstale":"2023-09-23T22:53:18.138454+0000","last_undegraded":"2023-09-23T22:53:18.138454+0000","last_fullsized":"2023-09-23T22:53:18.138454+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:26.070327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.137964+0000","last_change":"2023-09-23T22:53:16.892741+0000","last_active":"2023-09-23T22:53:18.137964+0000","last_peered":"2023-09-23T22:53:18.137964+0000","last_clean":"2023-09-23T22:53:18.137964+0000","last_became_active":"2023-09-23T22:53:16.887836+0000","last_became_peered":"2023-09-23T22:53:16.887836+0000","last_unstale":"2023-09-23T22:53:18.137964+0000","last_undegraded":"2023-09-23T22:53:18.137964+0000","last_fullsized":"2023-09-23T22:53:18.137964+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:21.750136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.899282+0000","last_change":"2023-09-23T22:52:21.890238+0000","last_active":"2023-09-23T22:53:17.899282+0000","last_peered":"2023-09-23T22:53:17.899282+0000","last_clean":"2023-09-23T22:53:17.899282+0000","last_became_active":"2023-09-23T22:52:21.889979+0000","last_became_peered":"2023-09-23T22:52:21.889979+0000","last_unstale":"2023-09-23T22:53:17.899282+0000","last_undegraded":"2023-09-23T22:53:17.899282+0000","last_fullsized":"2023-09-23T22:53:17.899282+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:25.487367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138854+0000","last_change":"2023-09-23T22:53:16.999411+0000","last_active":"2023-09-23T22:53:18.138854+0000","last_peered":"2023-09-23T22:53:18.138854+0000","last_clean":"2023-09-23T22:53:18.138854+0000","last_became_active":"2023-09-23T22:53:16.999141+0000","last_became_peered":"2023-09-23T22:53:16.999141+0000","last_unstale":"2023-09-23T22:53:18.138854+0000","last_undegraded":"2023-09-23T22:53:18.138854+0000","last_fullsized":"2023-09-23T22:53:18.138854+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:54.430172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.904826+0000","last_change":"2023-09-23T22:53:16.894735+0000","last_active":"2023-09-23T22:53:17.904826+0000","last_peered":"2023-09-23T22:53:17.904826+0000","last_clean":"2023-09-23T22:53:17.904826+0000","last_became_active":"2023-09-23T22:53:16.894479+0000","last_became_peered":"2023-09-23T22:53:16.894479+0000","last_unstale":"2023-09-23T22:53:17.904826+0000","last_undegraded":"2023-09-23T22:53:17.904826+0000","last_fullsized":"2023-09-23T22:53:17.904826+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:45:29.733182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.899344+0000","last_change":"2023-09-23T22:52:21.314044+0000","last_active":"2023-09-23T22:53:17.899344+0000","last_peered":"2023-09-23T22:53:17.899344+0000","last_clean":"2023-09-23T22:53:17.899344+0000","last_became_active":"2023-09-23T22:52:21.313723+0000","last_became_peered":"2023-09-23T22:52:21.313723+0000","last_unstale":"2023-09-23T22:53:17.899344+0000","last_undegraded":"2023-09-23T22:53:17.899344+0000","last_fullsized":"2023-09-23T22:53:17.899344+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:31.447410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138032+0000","last_change":"2023-09-23T22:53:17.000045+0000","last_active":"2023-09-23T22:53:18.138032+0000","last_peered":"2023-09-23T22:53:18.138032+0000","last_clean":"2023-09-23T22:53:18.138032+0000","last_became_active":"2023-09-23T22:53:16.999783+0000","last_became_peered":"2023-09-23T22:53:16.999783+0000","last_unstale":"2023-09-23T22:53:18.138032+0000","last_undegraded":"2023-09-23T22:53:18.138032+0000","last_fullsized":"2023-09-23T22:53:18.138032+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:58.830318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.899759+0000","last_change":"2023-09-23T22:53:16.886475+0000","last_active":"2023-09-23T22:53:17.899759+0000","last_peered":"2023-09-23T22:53:17.899759+0000","last_clean":"2023-09-23T22:53:17.899759+0000","last_became_active":"2023-09-23T22:53:16.886243+0000","last_became_peered":"2023-09-23T22:53:16.886243+0000","last_unstale":"2023-09-23T22:53:17.899759+0000","last_undegraded":"2023-09-23T22:53:17.899759+0000","last_fullsized":"2023-09-23T22:53:17.899759+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:16.683185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138791+0000","last_change":"2023-09-23T22:52:21.313895+0000","last_active":"2023-09-23T22:53:18.138791+0000","last_peered":"2023-09-23T22:53:18.138791+0000","last_clean":"2023-09-23T22:53:18.138791+0000","last_became_active":"2023-09-23T22:52:21.313631+0000","last_became_peered":"2023-09-23T22:52:21.313631+0000","last_unstale":"2023-09-23T22:53:18.138791+0000","last_undegraded":"2023-09-23T22:53:18.138791+0000","last_fullsized":"2023-09-23T22:53:18.138791+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:41.617398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138532+0000","last_change":"2023-09-23T22:53:16.998598+0000","last_active":"2023-09-23T22:53:18.138532+0000","last_peered":"2023-09-23T22:53:18.138532+0000","last_clean":"2023-09-23T22:53:18.138532+0000","last_became_active":"2023-09-23T22:53:16.998354+0000","last_became_peered":"2023-09-23T22:53:16.998354+0000","last_unstale":"2023-09-23T22:53:18.138532+0000","last_undegraded":"2023-09-23T22:53:18.138532+0000","last_fullsized":"2023-09-23T22:53:18.138532+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:29.811991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.902415+0000","last_change":"2023-09-23T22:53:16.894485+0000","last_active":"2023-09-23T22:53:17.902415+0000","last_peered":"2023-09-23T22:53:17.902415+0000","last_clean":"2023-09-23T22:53:17.902415+0000","last_became_active":"2023-09-23T22:53:16.894025+0000","last_became_peered":"2023-09-23T22:53:16.894025+0000","last_unstale":"2023-09-23T22:53:17.902415+0000","last_undegraded":"2023-09-23T22:53:17.902415+0000","last_fullsized":"2023-09-23T22:53:17.902415+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:26.374772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.139679+0000","last_change":"2023-09-23T22:53:18.138464+0000","last_active":"2023-09-23T22:53:18.139679+0000","last_peered":"2023-09-23T22:53:18.139679+0000","last_clean":"2023-09-23T22:53:18.139679+0000","last_became_active":"2023-09-23T22:53:18.137635+0000","last_became_peered":"2023-09-23T22:53:18.137635+0000","last_unstale":"2023-09-23T22:53:18.139679+0000","last_undegraded":"2023-09-23T22:53:18.139679+0000","last_fullsized":"2023-09-23T22:53:18.139679+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:57.384430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.902689+0000","last_change":"2023-09-23T22:53:16.893337+0000","last_active":"2023-09-23T22:53:17.902689+0000","last_peered":"2023-09-23T22:53:17.902689+0000","last_clean":"2023-09-23T22:53:17.902689+0000","last_became_active":"2023-09-23T22:53:16.891945+0000","last_became_peered":"2023-09-23T22:53:16.891945+0000","last_unstale":"2023-09-23T22:53:17.902689+0000","last_undegraded":"2023-09-23T22:53:17.902689+0000","last_fullsized":"2023-09-23T22:53:17.902689+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:49.534479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.140875+0000","last_change":"2023-09-23T22:53:18.140633+0000","last_active":"2023-09-23T22:53:18.140875+0000","last_peered":"2023-09-23T22:53:18.140875+0000","last_clean":"2023-09-23T22:53:18.140875+0000","last_became_active":"2023-09-23T22:53:18.140386+0000","last_became_peered":"2023-09-23T22:53:18.140386+0000","last_unstale":"2023-09-23T22:53:18.140875+0000","last_undegraded":"2023-09-23T22:53:18.140875+0000","last_fullsized":"2023-09-23T22:53:18.140875+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:16:51.208509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.905087+0000","last_change":"2023-09-23T22:53:16.895307+0000","last_active":"2023-09-23T22:53:17.905087+0000","last_peered":"2023-09-23T22:53:17.905087+0000","last_clean":"2023-09-23T22:53:17.905087+0000","last_became_active":"2023-09-23T22:53:16.895026+0000","last_became_peered":"2023-09-23T22:53:16.895026+0000","last_unstale":"2023-09-23T22:53:17.905087+0000","last_undegraded":"2023-09-23T22:53:17.905087+0000","last_fullsized":"2023-09-23T22:53:17.905087+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:44.078221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.907777+0000","last_change":"2023-09-23T22:53:17.907538+0000","last_active":"2023-09-23T22:53:17.907777+0000","last_peered":"2023-09-23T22:53:17.907777+0000","last_clean":"2023-09-23T22:53:17.907777+0000","last_became_active":"2023-09-23T22:53:17.907261+0000","last_became_peered":"2023-09-23T22:53:17.907261+0000","last_unstale":"2023-09-23T22:53:17.907777+0000","last_undegraded":"2023-09-23T22:53:17.907777+0000","last_fullsized":"2023-09-23T22:53:17.907777+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:36.671603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.875959+0000","last_change":"2023-09-23T22:53:16.895707+0000","last_active":"2023-09-23T22:53:36.875959+0000","last_peered":"2023-09-23T22:53:36.875959+0000","last_clean":"2023-09-23T22:53:36.875959+0000","last_became_active":"2023-09-23T22:53:16.895451+0000","last_became_peered":"2023-09-23T22:53:16.895451+0000","last_unstale":"2023-09-23T22:53:36.875959+0000","last_undegraded":"2023-09-23T22:53:36.875959+0000","last_fullsized":"2023-09-23T22:53:36.875959+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:29.222424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.900713+0000","last_change":"2023-09-23T22:53:16.903902+0000","last_active":"2023-09-23T22:53:17.900713+0000","last_peered":"2023-09-23T22:53:17.900713+0000","last_clean":"2023-09-23T22:53:17.900713+0000","last_became_active":"2023-09-23T22:53:16.903613+0000","last_became_peered":"2023-09-23T22:53:16.903613+0000","last_unstale":"2023-09-23T22:53:17.900713+0000","last_undegraded":"2023-09-23T22:53:17.900713+0000","last_fullsized":"2023-09-23T22:53:17.900713+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:45.743285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.899489+0000","last_change":"2023-09-23T22:53:16.888352+0000","last_active":"2023-09-23T22:53:17.899489+0000","last_peered":"2023-09-23T22:53:17.899489+0000","last_clean":"2023-09-23T22:53:17.899489+0000","last_became_active":"2023-09-23T22:53:16.888074+0000","last_became_peered":"2023-09-23T22:53:16.888074+0000","last_unstale":"2023-09-23T22:53:17.899489+0000","last_undegraded":"2023-09-23T22:53:17.899489+0000","last_fullsized":"2023-09-23T22:53:17.899489+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:08.674783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138598+0000","last_change":"2023-09-23T22:53:18.137978+0000","last_active":"2023-09-23T22:53:18.138598+0000","last_peered":"2023-09-23T22:53:18.138598+0000","last_clean":"2023-09-23T22:53:18.138598+0000","last_became_active":"2023-09-23T22:53:18.137679+0000","last_became_peered":"2023-09-23T22:53:18.137679+0000","last_unstale":"2023-09-23T22:53:18.138598+0000","last_undegraded":"2023-09-23T22:53:18.138598+0000","last_fullsized":"2023-09-23T22:53:18.138598+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:17.681479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:20.393182+0000","last_change":"2023-09-23T22:53:16.887968+0000","last_active":"2023-09-23T22:53:20.393182+0000","last_peered":"2023-09-23T22:53:20.393182+0000","last_clean":"2023-09-23T22:53:20.393182+0000","last_became_active":"2023-09-23T22:53:16.887692+0000","last_became_peered":"2023-09-23T22:53:16.887692+0000","last_unstale":"2023-09-23T22:53:20.393182+0000","last_undegraded":"2023-09-23T22:53:20.393182+0000","last_fullsized":"2023-09-23T22:53:20.393182+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:24.662748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138490+0000","last_change":"2023-09-23T22:53:18.137944+0000","last_active":"2023-09-23T22:53:18.138490+0000","last_peered":"2023-09-23T22:53:18.138490+0000","last_clean":"2023-09-23T22:53:18.138490+0000","last_became_active":"2023-09-23T22:53:18.137652+0000","last_became_peered":"2023-09-23T22:53:18.137652+0000","last_unstale":"2023-09-23T22:53:18.138490+0000","last_undegraded":"2023-09-23T22:53:18.138490+0000","last_fullsized":"2023-09-23T22:53:18.138490+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:25:39.443733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.877265+0000","last_change":"2023-09-23T22:53:16.895724+0000","last_active":"2023-09-23T22:53:36.877265+0000","last_peered":"2023-09-23T22:53:36.877265+0000","last_clean":"2023-09-23T22:53:36.877265+0000","last_became_active":"2023-09-23T22:53:16.895451+0000","last_became_peered":"2023-09-23T22:53:16.895451+0000","last_unstale":"2023-09-23T22:53:36.877265+0000","last_undegraded":"2023-09-23T22:53:36.877265+0000","last_fullsized":"2023-09-23T22:53:36.877265+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:07.343242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.076154+0000","last_change":"2023-09-23T22:53:18.075506+0000","last_active":"2023-09-23T22:53:18.076154+0000","last_peered":"2023-09-23T22:53:18.076154+0000","last_clean":"2023-09-23T22:53:18.076154+0000","last_became_active":"2023-09-23T22:53:18.075206+0000","last_became_peered":"2023-09-23T22:53:18.075206+0000","last_unstale":"2023-09-23T22:53:18.076154+0000","last_undegraded":"2023-09-23T22:53:18.076154+0000","last_fullsized":"2023-09-23T22:53:18.076154+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:32:16.168188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:20.393228+0000","last_change":"2023-09-23T22:53:16.888803+0000","last_active":"2023-09-23T22:53:20.393228+0000","last_peered":"2023-09-23T22:53:20.393228+0000","last_clean":"2023-09-23T22:53:20.393228+0000","last_became_active":"2023-09-23T22:53:16.888513+0000","last_became_peered":"2023-09-23T22:53:16.888513+0000","last_unstale":"2023-09-23T22:53:20.393228+0000","last_undegraded":"2023-09-23T22:53:20.393228+0000","last_fullsized":"2023-09-23T22:53:20.393228+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:57.392303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.139499+0000","last_change":"2023-09-23T22:53:18.138127+0000","last_active":"2023-09-23T22:53:18.139499+0000","last_peered":"2023-09-23T22:53:18.139499+0000","last_clean":"2023-09-23T22:53:18.139499+0000","last_became_active":"2023-09-23T22:53:18.137851+0000","last_became_peered":"2023-09-23T22:53:18.137851+0000","last_unstale":"2023-09-23T22:53:18.139499+0000","last_undegraded":"2023-09-23T22:53:18.139499+0000","last_fullsized":"2023-09-23T22:53:18.139499+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:34.946793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.877654+0000","last_change":"2023-09-23T22:53:16.897754+0000","last_active":"2023-09-23T22:53:36.877654+0000","last_peered":"2023-09-23T22:53:36.877654+0000","last_clean":"2023-09-23T22:53:36.877654+0000","last_became_active":"2023-09-23T22:53:16.897488+0000","last_became_peered":"2023-09-23T22:53:16.897488+0000","last_unstale":"2023-09-23T22:53:36.877654+0000","last_undegraded":"2023-09-23T22:53:36.877654+0000","last_fullsized":"2023-09-23T22:53:36.877654+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:50:46.440416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.901201+0000","last_change":"2023-09-23T22:53:17.000324+0000","last_active":"2023-09-23T22:53:17.901201+0000","last_peered":"2023-09-23T22:53:17.901201+0000","last_clean":"2023-09-23T22:53:17.901201+0000","last_became_active":"2023-09-23T22:53:17.000070+0000","last_became_peered":"2023-09-23T22:53:17.000070+0000","last_unstale":"2023-09-23T22:53:17.901201+0000","last_undegraded":"2023-09-23T22:53:17.901201+0000","last_fullsized":"2023-09-23T22:53:17.901201+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:14.840209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.074491+0000","last_change":"2023-09-23T22:53:16.889362+0000","last_active":"2023-09-23T22:53:18.074491+0000","last_peered":"2023-09-23T22:53:18.074491+0000","last_clean":"2023-09-23T22:53:18.074491+0000","last_became_active":"2023-09-23T22:53:16.889081+0000","last_became_peered":"2023-09-23T22:53:16.889081+0000","last_unstale":"2023-09-23T22:53:18.074491+0000","last_undegraded":"2023-09-23T22:53:18.074491+0000","last_fullsized":"2023-09-23T22:53:18.074491+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:11.321151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.076163+0000","last_change":"2023-09-23T22:53:18.075841+0000","last_active":"2023-09-23T22:53:18.076163+0000","last_peered":"2023-09-23T22:53:18.076163+0000","last_clean":"2023-09-23T22:53:18.076163+0000","last_became_active":"2023-09-23T22:53:18.075550+0000","last_became_peered":"2023-09-23T22:53:18.075550+0000","last_unstale":"2023-09-23T22:53:18.076163+0000","last_undegraded":"2023-09-23T22:53:18.076163+0000","last_fullsized":"2023-09-23T22:53:18.076163+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:40.805558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.900561+0000","last_change":"2023-09-23T22:53:16.882261+0000","last_active":"2023-09-23T22:53:17.900561+0000","last_peered":"2023-09-23T22:53:17.900561+0000","last_clean":"2023-09-23T22:53:17.900561+0000","last_became_active":"2023-09-23T22:53:16.881924+0000","last_became_peered":"2023-09-23T22:53:16.881924+0000","last_unstale":"2023-09-23T22:53:17.900561+0000","last_undegraded":"2023-09-23T22:53:17.900561+0000","last_fullsized":"2023-09-23T22:53:17.900561+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:48.503506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.139383+0000","last_change":"2023-09-23T22:53:18.137940+0000","last_active":"2023-09-23T22:53:18.139383+0000","last_peered":"2023-09-23T22:53:18.139383+0000","last_clean":"2023-09-23T22:53:18.139383+0000","last_became_active":"2023-09-23T22:53:18.137643+0000","last_became_peered":"2023-09-23T22:53:18.137643+0000","last_unstale":"2023-09-23T22:53:18.139383+0000","last_undegraded":"2023-09-23T22:53:18.139383+0000","last_fullsized":"2023-09-23T22:53:18.139383+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:02.763345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.877260+0000","last_change":"2023-09-23T22:53:16.895361+0000","last_active":"2023-09-23T22:53:36.877260+0000","last_peered":"2023-09-23T22:53:36.877260+0000","last_clean":"2023-09-23T22:53:36.877260+0000","last_became_active":"2023-09-23T22:53:16.894995+0000","last_became_peered":"2023-09-23T22:53:16.894995+0000","last_unstale":"2023-09-23T22:53:36.877260+0000","last_undegraded":"2023-09-23T22:53:36.877260+0000","last_fullsized":"2023-09-23T22:53:36.877260+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:37.468777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.139665+0000","last_change":"2023-09-23T22:53:18.138452+0000","last_active":"2023-09-23T22:53:18.139665+0000","last_peered":"2023-09-23T22:53:18.139665+0000","last_clean":"2023-09-23T22:53:18.139665+0000","last_became_active":"2023-09-23T22:53:18.137756+0000","last_became_peered":"2023-09-23T22:53:18.137756+0000","last_unstale":"2023-09-23T22:53:18.139665+0000","last_undegraded":"2023-09-23T22:53:18.139665+0000","last_fullsized":"2023-09-23T22:53:18.139665+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:42.057582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.878271+0000","last_change":"2023-09-23T22:53:16.891930+0000","last_active":"2023-09-23T22:53:36.878271+0000","last_peered":"2023-09-23T22:53:36.878271+0000","last_clean":"2023-09-23T22:53:36.878271+0000","last_became_active":"2023-09-23T22:53:16.891658+0000","last_became_peered":"2023-09-23T22:53:16.891658+0000","last_unstale":"2023-09-23T22:53:36.878271+0000","last_undegraded":"2023-09-23T22:53:36.878271+0000","last_fullsized":"2023-09-23T22:53:36.878271+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:19.762775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.137911+0000","last_change":"2023-09-23T22:53:17.000151+0000","last_active":"2023-09-23T22:53:18.137911+0000","last_peered":"2023-09-23T22:53:18.137911+0000","last_clean":"2023-09-23T22:53:18.137911+0000","last_became_active":"2023-09-23T22:53:16.999582+0000","last_became_peered":"2023-09-23T22:53:16.999582+0000","last_unstale":"2023-09-23T22:53:18.137911+0000","last_undegraded":"2023-09-23T22:53:18.137911+0000","last_fullsized":"2023-09-23T22:53:18.137911+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:16.874918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:36.877557+0000","last_change":"2023-09-23T22:53:16.895495+0000","last_active":"2023-09-23T22:53:36.877557+0000","last_peered":"2023-09-23T22:53:36.877557+0000","last_clean":"2023-09-23T22:53:36.877557+0000","last_became_active":"2023-09-23T22:53:16.895192+0000","last_became_peered":"2023-09-23T22:53:16.895192+0000","last_unstale":"2023-09-23T22:53:36.877557+0000","last_undegraded":"2023-09-23T22:53:36.877557+0000","last_fullsized":"2023-09-23T22:53:36.877557+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:41.114595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.075640+0000","last_change":"2023-09-23T22:53:18.075428+0000","last_active":"2023-09-23T22:53:18.075640+0000","last_peered":"2023-09-23T22:53:18.075640+0000","last_clean":"2023-09-23T22:53:18.075640+0000","last_became_active":"2023-09-23T22:53:18.075112+0000","last_became_peered":"2023-09-23T22:53:18.075112+0000","last_unstale":"2023-09-23T22:53:18.075640+0000","last_undegraded":"2023-09-23T22:53:18.075640+0000","last_fullsized":"2023-09-23T22:53:18.075640+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:49.645031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.899387+0000","last_change":"2023-09-23T22:53:16.893248+0000","last_active":"2023-09-23T22:53:17.899387+0000","last_peered":"2023-09-23T22:53:17.899387+0000","last_clean":"2023-09-23T22:53:17.899387+0000","last_became_active":"2023-09-23T22:53:16.892954+0000","last_became_peered":"2023-09-23T22:53:16.892954+0000","last_unstale":"2023-09-23T22:53:17.899387+0000","last_undegraded":"2023-09-23T22:53:17.899387+0000","last_fullsized":"2023-09-23T22:53:17.899387+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:22.235094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.139416+0000","last_change":"2023-09-23T22:53:18.137964+0000","last_active":"2023-09-23T22:53:18.139416+0000","last_peered":"2023-09-23T22:53:18.139416+0000","last_clean":"2023-09-23T22:53:18.139416+0000","last_became_active":"2023-09-23T22:53:18.137683+0000","last_became_peered":"2023-09-23T22:53:18.137683+0000","last_unstale":"2023-09-23T22:53:18.139416+0000","last_undegraded":"2023-09-23T22:53:18.139416+0000","last_fullsized":"2023-09-23T22:53:18.139416+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:51.842249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.900576+0000","last_change":"2023-09-23T22:53:16.886203+0000","last_active":"2023-09-23T22:53:17.900576+0000","last_peered":"2023-09-23T22:53:17.900576+0000","last_clean":"2023-09-23T22:53:17.900576+0000","last_became_active":"2023-09-23T22:53:16.885705+0000","last_became_peered":"2023-09-23T22:53:16.885705+0000","last_unstale":"2023-09-23T22:53:17.900576+0000","last_undegraded":"2023-09-23T22:53:17.900576+0000","last_fullsized":"2023-09-23T22:53:17.900576+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:08.071819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.139272+0000","last_change":"2023-09-23T22:53:18.138026+0000","last_active":"2023-09-23T22:53:18.139272+0000","last_peered":"2023-09-23T22:53:18.139272+0000","last_clean":"2023-09-23T22:53:18.139272+0000","last_became_active":"2023-09-23T22:53:18.137722+0000","last_became_peered":"2023-09-23T22:53:18.137722+0000","last_unstale":"2023-09-23T22:53:18.139272+0000","last_undegraded":"2023-09-23T22:53:18.139272+0000","last_fullsized":"2023-09-23T22:53:18.139272+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:13.377862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.902745+0000","last_change":"2023-09-23T22:53:16.894299+0000","last_active":"2023-09-23T22:53:17.902745+0000","last_peered":"2023-09-23T22:53:17.902745+0000","last_clean":"2023-09-23T22:53:17.902745+0000","last_became_active":"2023-09-23T22:53:16.893621+0000","last_became_peered":"2023-09-23T22:53:16.893621+0000","last_unstale":"2023-09-23T22:53:17.902745+0000","last_undegraded":"2023-09-23T22:53:17.902745+0000","last_fullsized":"2023-09-23T22:53:17.902745+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:55.765968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:17.901274+0000","last_change":"2023-09-23T22:53:17.000931+0000","last_active":"2023-09-23T22:53:17.901274+0000","last_peered":"2023-09-23T22:53:17.901274+0000","last_clean":"2023-09-23T22:53:17.901274+0000","last_became_active":"2023-09-23T22:53:17.000684+0000","last_became_peered":"2023-09-23T22:53:17.000684+0000","last_unstale":"2023-09-23T22:53:17.901274+0000","last_undegraded":"2023-09-23T22:53:17.901274+0000","last_fullsized":"2023-09-23T22:53:17.901274+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:58.542147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.138704+0000","last_change":"2023-09-23T22:53:16.887441+0000","last_active":"2023-09-23T22:53:18.138704+0000","last_peered":"2023-09-23T22:53:18.138704+0000","last_clean":"2023-09-23T22:53:18.138704+0000","last_became_active":"2023-09-23T22:53:16.887111+0000","last_became_peered":"2023-09-23T22:53:16.887111+0000","last_unstale":"2023-09-23T22:53:18.138704+0000","last_undegraded":"2023-09-23T22:53:18.138704+0000","last_fullsized":"2023-09-23T22:53:18.138704+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:40.227618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.140865+0000","last_change":"2023-09-23T22:53:18.140550+0000","last_active":"2023-09-23T22:53:18.140865+0000","last_peered":"2023-09-23T22:53:18.140865+0000","last_clean":"2023-09-23T22:53:18.140865+0000","last_became_active":"2023-09-23T22:53:18.140252+0000","last_became_peered":"2023-09-23T22:53:18.140252+0000","last_unstale":"2023-09-23T22:53:18.140865+0000","last_undegraded":"2023-09-23T22:53:18.140865+0000","last_fullsized":"2023-09-23T22:53:18.140865+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:16.122288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T22:53:20.498118+0000","last_change":"2023-09-23T22:53:16.893423+0000","last_active":"2023-09-23T22:53:20.498118+0000","last_peered":"2023-09-23T22:53:20.498118+0000","last_clean":"2023-09-23T22:53:20.498118+0000","last_became_active":"2023-09-23T22:53:16.893144+0000","last_became_peered":"2023-09-23T22:53:16.893144+0000","last_unstale":"2023-09-23T22:53:20.498118+0000","last_undegraded":"2023-09-23T22:53:20.498118+0000","last_fullsized":"2023-09-23T22:53:20.498118+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:50.803912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234016,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6600,"kb_used_data":1280,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738552,"statfs":{"total":95995035648,"available":95988277248,"internally_reserved":0,"allocated":1310720,"data_stored":891354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.53100000000000003,"15min":0.54000000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.38},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.748,"15min":0.78600000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.79700000000000004,"5min":5.1849999999999996,"15min":5.1849999999999996},"last":0.42599999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.64800000000000002,"15min":0.66800000000000004},"min":{"1min":0.38700000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.88100000000000001,"5min":2.903,"15min":2.903},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.85699999999999998,"15min":0.92000000000000004},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.73699999999999999,"5min":5.0659999999999998,"15min":5.0659999999999998},"last":0.46600000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.76600000000000001,"15min":0.80400000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.81299999999999994,"5min":3.0259999999999998,"15min":3.0259999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.65400000000000003,"15min":0.67200000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.88400000000000001,"5min":2.649,"15min":2.649},"last":0.57099999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.55000000000000004,"15min":0.55300000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.85999999999999999,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.79700000000000004,"15min":0.83499999999999996},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.90800000000000003,"5min":4.9539999999999997,"15min":4.9539999999999997},"last":0.65000000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.48099999999999998,"15min":0.47899999999999998},"min":{"1min":0.34599999999999997,"5min":0.308,"15min":0.308},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.495,"15min":0.49099999999999999},"min":{"1min":0.33700000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.66600000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.51500000000000001,"15min":0.51100000000000001},"min":{"1min":0.36099999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.501,"15min":0.498},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.55100000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.54300000000000004,"15min":0.54400000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.628},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.56299999999999994,"15min":0.56100000000000005},"min":{"1min":0.438,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.70899999999999996}]}]},{"osd":6,"up_from":46,"seq":197568495652,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6584,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738568,"statfs":{"total":95995035648,"available":95988293632,"internally_reserved":0,"allocated":1294336,"data_stored":890943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.39300000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.51000000000000001,"15min":0.51500000000000001},"min":{"1min":0.35499999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.66800000000000004,"5min":0.751,"15min":0.751},"last":0.51400000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.52100000000000002,"15min":0.51700000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56499999999999995,"15min":0.56599999999999995},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.748,"5min":0.753,"15min":0.753},"last":0.60499999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.68999999999999995,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.48499999999999999,"15min":0.48099999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.41199999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.52200000000000002,"15min":0.52500000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.68899999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.41699999999999998,"5min":0.38,"15min":0.38},"max":{"1min":0.78700000000000003,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.64400000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.43099999999999999,"15min":0.42699999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.5},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.46800000000000003,"15min":0.46200000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.53400000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.44700000000000001,"15min":0.439},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.497,"15min":0.495},"min":{"1min":0.33700000000000002,"5min":0.316,"15min":0.316},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.627}]},{"osd":7,"last update":"Sat Sep 23 22:53:49 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.54400000000000004,"15min":0.54100000000000004},"min":{"1min":0.41399999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.55000000000000004,"15min":0.54700000000000004},"min":{"1min":0.35899999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.73299999999999998,"5min":0.879,"15min":0.879},"last":0.66900000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574907,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6764,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93738388,"statfs":{"total":95995035648,"available":95988109312,"internally_reserved":0,"allocated":692224,"data_stored":299557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:32 2023","interfaces":[{"interface":"back","average":{"1min":0.315,"5min":0.34499999999999997,"15min":0.33700000000000002},"min":{"1min":0.20499999999999999,"5min":0.191,"15min":0.191},"max":{"1min":0.58999999999999997,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.36099999999999999,"15min":0.34499999999999997},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.57399999999999995,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.32400000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:53:56 2023","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":0.39600000000000002,"15min":0.39800000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.56000000000000005,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.42399999999999999,"15min":0.41599999999999998},"min":{"1min":0.307,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.61599999999999999,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.38800000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:53:23 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.40999999999999998,"15min":0.41799999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.44400000000000001,"15min":0.45800000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.70899999999999996,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.34699999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:53:40 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.59899999999999998,"15min":0.57199999999999995},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.73399999999999999,"5min":5.3710000000000004,"15min":5.3710000000000004},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.505,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.753,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.83399999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.56000000000000005,"15min":0.56599999999999995},"min":{"1min":0.34799999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.79200000000000004,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59399999999999997,"15min":0.59799999999999998},"min":{"1min":0.40999999999999998,"5min":0.38,"15min":0.38},"max":{"1min":0.92100000000000004,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.84599999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:53:23 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.62,"15min":0.59699999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":5.2999999999999998,"5min":5.2999999999999998,"15min":5.2999999999999998},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.61799999999999999,"15min":0.627},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.64400000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:53:46 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.80800000000000005,"15min":0.84699999999999998},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.81499999999999995,"5min":5.3390000000000004,"15min":5.3390000000000004},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.59499999999999997,"15min":0.60699999999999998},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.68500000000000005,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.57799999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738431,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7416,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93737736,"statfs":{"total":95995035648,"available":95987441664,"internally_reserved":0,"allocated":1294336,"data_stored":890501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:53:29 2023","interfaces":[{"interface":"back","average":{"1min":0.312,"5min":0.34499999999999997,"15min":0.34499999999999997},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.45600000000000002,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.33000000000000002,"5min":0.35999999999999999,"15min":0.35499999999999998},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.29099999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:53:58 2023","interfaces":[{"interface":"back","average":{"1min":0.32100000000000001,"5min":0.38900000000000001,"15min":0.41099999999999998},"min":{"1min":0.23100000000000001,"5min":0.188,"15min":0.188},"max":{"1min":0.47599999999999998,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.378,"5min":0.40400000000000003,"15min":0.40799999999999997},"min":{"1min":0.29399999999999998,"5min":0.249,"15min":0.249},"max":{"1min":0.56699999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.33400000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.443,"15min":0.46899999999999997},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.59299999999999997,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.441,"15min":0.46200000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.58599999999999997,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.316}]},{"osd":4,"last update":"Sat Sep 23 22:53:43 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.55200000000000005,"15min":0.55600000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.56799999999999995,"15min":0.58199999999999996},"min":{"1min":0.36499999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.63300000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.59199999999999997,"15min":0.59799999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.56899999999999995,"15min":0.57099999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.58799999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.59099999999999997,"15min":0.59999999999999998},"min":{"1min":0.41299999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.81299999999999994,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.58299999999999996,"15min":0.58799999999999997},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.77900000000000003,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.57499999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:53:46 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.57499999999999996,"15min":0.58399999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.69899999999999995,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.57799999999999996,"15min":0.58699999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.71699999999999997,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.59899999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411381,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6640,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93738512,"statfs":{"total":95995035648,"available":95988236288,"internally_reserved":0,"allocated":696320,"data_stored":299484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:59 2023","interfaces":[{"interface":"back","average":{"1min":0.308,"5min":0.34699999999999998,"15min":0.34000000000000002},"min":{"1min":0.24299999999999999,"5min":0.161,"15min":0.161},"max":{"1min":0.46300000000000002,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.32800000000000001,"5min":0.371,"15min":0.36199999999999999},"min":{"1min":0.24299999999999999,"5min":0.188,"15min":0.188},"max":{"1min":0.434,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.34499999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:53:59 2023","interfaces":[{"interface":"back","average":{"1min":0.30099999999999999,"5min":0.37,"15min":0.36299999999999999},"min":{"1min":0.23000000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.379,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.33500000000000002,"5min":0.378,"15min":0.35899999999999999},"min":{"1min":0.24299999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.41699999999999998,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.39200000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:53:21 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.42299999999999999,"15min":0.43099999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.437,"15min":0.439},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.98199999999999998,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.28100000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.59099999999999997,"15min":0.58299999999999996},"min":{"1min":0.441,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.79100000000000004,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.59199999999999997,"15min":0.58599999999999997},"min":{"1min":0.40200000000000002,"5min":0.379,"15min":0.379},"max":{"1min":0.81999999999999995,"5min":1.05,"15min":1.05},"last":0.60599999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.42499999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59199999999999997,"15min":0.59099999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.52300000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:53:21 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.63100000000000001,"15min":0.63600000000000001},"min":{"1min":0.45700000000000002,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.502},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.63900000000000001,"15min":0.64500000000000002},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"last":0.57199999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:53:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.63300000000000001,"15min":0.64700000000000002},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.74299999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.66500000000000004,"15min":0.68300000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.69299999999999995,"5min":1.042,"15min":1.042},"last":0.54300000000000004}]}]},{"osd":3,"up_from":25,"seq":107374182449,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6128,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739024,"statfs":{"total":95995035648,"available":95988760576,"internally_reserved":0,"allocated":696320,"data_stored":299647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.29599999999999999,"5min":0.34200000000000003,"15min":0.34799999999999998},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.433,"5min":0.621,"15min":0.621},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.34000000000000002,"5min":0.41599999999999998,"15min":0.438},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.50700000000000001,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.81100000000000005}]},{"osd":1,"last update":"Sat Sep 23 22:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.40799999999999997,"15min":0.41899999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.496,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":1.7749999999999999},{"interface":"front","average":{"1min":0.32500000000000001,"5min":0.40899999999999997,"15min":0.42599999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.51700000000000002,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.41699999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.436,"15min":0.45100000000000001},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.51800000000000002,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.37,"5min":0.441,"15min":0.45300000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.52900000000000003,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.622}]},{"osd":4,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.54500000000000004,"15min":0.55300000000000005},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.143,"5min":1.143,"15min":1.143},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.56699999999999995,"15min":0.57599999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.64800000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:53:01 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.57599999999999996,"15min":0.58599999999999997},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.59599999999999997,"15min":0.60199999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.77700000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.58299999999999996,"15min":0.59399999999999997},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.754,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.57999999999999996,"15min":0.58899999999999997},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72799999999999998,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.752}]},{"osd":7,"last update":"Sat Sep 23 22:53:48 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.56999999999999995,"15min":0.57299999999999995},"min":{"1min":0.37,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.171,"5min":1.171,"15min":1.171},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.374,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.71799999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018925,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6212,"kb_used_data":700,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738940,"statfs":{"total":95995035648,"available":95988674560,"internally_reserved":0,"allocated":716800,"data_stored":301239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:39 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.496,"15min":0.5},"min":{"1min":0.27800000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.68600000000000005,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.57999999999999996,"15min":0.54000000000000004},"min":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.9510000000000005,"5min":8.9510000000000005,"15min":8.9510000000000005},"last":0.46400000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:53:39 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.57299999999999995,"15min":0.52500000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.2089999999999996,"5min":9.2089999999999996,"15min":9.2089999999999996},"last":0.44},{"interface":"front","average":{"1min":0.501,"5min":0.51200000000000001,"15min":0.50800000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.626,"5min":0.752,"15min":0.752},"last":0.40899999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:53:39 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.49099999999999999,"15min":0.499},"min":{"1min":0.33100000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.57499999999999996,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.628},{"interface":"front","average":{"1min":0.88,"5min":0.60299999999999998,"15min":0.56100000000000005},"min":{"1min":0.32600000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.0429999999999993,"5min":9.0429999999999993,"15min":9.0429999999999993},"last":0.505}]},{"osd":3,"last update":"Sat Sep 23 22:53:39 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.51000000000000001,"15min":0.51800000000000002},"min":{"1min":0.372,"5min":0.314,"15min":0.314},"max":{"1min":0.63900000000000001,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.51900000000000002,"15min":0.52400000000000002},"min":{"1min":0.38200000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.60599999999999998,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.51300000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:53:03 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.44800000000000001,"15min":0.44},"min":{"1min":0.29799999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.44400000000000001,"15min":0.436},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.67700000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.58399999999999996}]},{"osd":6,"last update":"Sat Sep 23 22:53:25 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.45700000000000002,"15min":0.45100000000000001},"min":{"1min":0.34100000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.66600000000000004,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.51300000000000001,"15min":0.504},"min":{"1min":0.33000000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.72799999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.68999999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:53:52 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.58599999999999997,"15min":0.58399999999999996},"min":{"1min":0.375,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.56899999999999995,"15min":0.57099999999999995},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.752,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.60299999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757288,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6064,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739088,"statfs":{"total":95995035648,"available":95988826112,"internally_reserved":0,"allocated":696320,"data_stored":299916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.56699999999999995,"15min":0.57299999999999995},"min":{"1min":0.373,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79400000000000004,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.48799999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.55200000000000005,"15min":0.55700000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.70499999999999996,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.52700000000000002,"15min":0.53100000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.55000000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.53900000000000003,"15min":0.54500000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.68300000000000005}]},{"osd":3,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.55600000000000005,"15min":0.55700000000000005},"min":{"1min":0.39700000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.56100000000000005,"15min":0.55900000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.73199999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.39200000000000002,"15min":0.38300000000000001},"min":{"1min":0.318,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.505,"5min":0.439,"15min":0.42799999999999999},"min":{"1min":0.30599999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.747}]},{"osd":6,"last update":"Sat Sep 23 22:53:26 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.46400000000000002,"15min":0.45400000000000001},"min":{"1min":0.35099999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.48399999999999999,"15min":0.47299999999999998},"min":{"1min":0.33600000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.76300000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:53:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.55900000000000005,"15min":0.55400000000000005},"min":{"1min":0.36499999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.52200000000000002,"15min":0.51800000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.77000000000000002,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.70299999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:54:08.813 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T22:54:08.813 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T22:54:08.813 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T22:54:08.813 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T22:54:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:08 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:08 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:08 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:54:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:08 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:54:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:08 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:08 smithi138 ceph-mon[104418]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:08 smithi138 ceph-mon[104418]: from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:08.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:08 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:08.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:08 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:08.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:08 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:54:08.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:08 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:54:08.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:08 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:08.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:08 smithi110 ceph-mon[100243]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:08.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:08 smithi110 ceph-mon[100243]: from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:08.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:08 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:08.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:08 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:08.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:08 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:54:08.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:08 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:54:08.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:08 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:08.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:08 smithi110 ceph-mon[118821]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:08.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:08 smithi110 ceph-mon[118821]: from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:09.157 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:10.323 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:54:10 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:54:10] "GET /metrics HTTP/1.1" 200 238769 "" "Prometheus/2.43.0" 2023-09-23T22:54:10.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:10 smithi138 ceph-mon[104418]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:10.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:10 smithi110 ceph-mon[100243]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:10.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:10 smithi110 ceph-mon[118821]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:10.910 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T22:54:10.910 INFO:teuthology.orchestra.run.smithi110.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T22:54:11.615 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:11 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/151785794' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:54:11.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:11 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/151785794' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:54:11.632 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T22:54:11.632 INFO:teuthology.run_tasks:Running task print... 2023-09-23T22:54:11.643 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-09-23T22:54:11.643 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T22:54:11.651 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T22:54:11.652 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T22:54:11.652 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-09-23T22:54:11.653 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T22:54:11.654 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T22:54:11.654 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-09-23T22:54:11.655 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-09-23T22:54:11.655 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T22:54:11.655 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-09-23T22:54:11.655 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi110.front.sepia.ceph.com 2023-09-23T22:54:11.656 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-09-23T22:54:11.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:11 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/151785794' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:54:12.112 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:12.163 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:12.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:54:11 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:54:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:54:12.642 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:12 smithi138 ceph-mon[104418]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:12.700 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:12 smithi110 ceph-mon[100243]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:12.700 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:12 smithi110 ceph-mon[118821]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:14 smithi138 ceph-mon[104418]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:14 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/533248228' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T22:54:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:14 smithi138 ceph-mon[104418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T22:54:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:14 smithi138 ceph-mon[104418]: from='client.? ' entity='client.admin' 2023-09-23T22:54:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:14 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:54:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:14 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:54:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:14 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:54:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:14 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:15.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:14 smithi110 ceph-mon[118821]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:15.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:14 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/533248228' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T22:54:15.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:14 smithi110 ceph-mon[118821]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T22:54:15.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:14 smithi110 ceph-mon[118821]: from='client.? ' entity='client.admin' 2023-09-23T22:54:15.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:14 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:54:15.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:14 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:54:15.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:14 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:54:15.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:14 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:14 smithi110 ceph-mon[100243]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:14 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/533248228' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T22:54:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:14 smithi110 ceph-mon[100243]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T22:54:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:14 smithi110 ceph-mon[100243]: from='client.? ' entity='client.admin' 2023-09-23T22:54:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:14 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:54:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:14 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:54:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:14 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:54:15.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:14 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:15.203 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:54:14 smithi110 conmon[100666]: [23/Sep/2023:22:54:14] ENGINE Bus STOPPING 2023-09-23T22:54:15.203 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:54:14 smithi110 conmon[100666]: [23/Sep/2023:22:54:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:54:15.203 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:54:14 smithi110 conmon[100666]: [23/Sep/2023:22:54:14] ENGINE Bus STOPPED 2023-09-23T22:54:15.203 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:54:14 smithi110 conmon[100666]: [23/Sep/2023:22:54:14] ENGINE Bus STARTING 2023-09-23T22:54:15.203 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:54:14 smithi110 conmon[100666]: [23/Sep/2023:22:54:14] ENGINE Serving on http://:::9283 2023-09-23T22:54:15.203 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:54:14 smithi110 conmon[100666]: [23/Sep/2023:22:54:14] ENGINE Bus STARTED 2023-09-23T22:54:15.333 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_0' created 2023-09-23T22:54:15.530 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set global log_to_journald false --force' 2023-09-23T22:54:16.088 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-09-23T22:54:16.130 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:16.330 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:16 smithi110 ceph-mon[100243]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T22:54:16.330 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:16 smithi110 ceph-mon[100243]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T22:54:16.330 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:16 smithi110 ceph-mon[100243]: pgmap v72: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:16.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:16 smithi138 ceph-mon[104418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T22:54:16.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:16 smithi138 ceph-mon[104418]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T22:54:16.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:16 smithi138 ceph-mon[104418]: pgmap v72: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:16.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:16 smithi110 ceph-mon[118821]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T22:54:16.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:16 smithi110 ceph-mon[118821]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T22:54:16.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:16 smithi110 ceph-mon[118821]: pgmap v72: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:17.237 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:17.573 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:17 smithi110 ceph-mon[100243]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T22:54:17.574 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:17 smithi110 ceph-mon[118821]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T22:54:17.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:17 smithi138 ceph-mon[104418]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T22:54:18.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:18 smithi110 ceph-mon[100243]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T22:54:18.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:18 smithi110 ceph-mon[100243]: pgmap v75: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:18.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:18 smithi110 ceph-mon[118821]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T22:54:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:18 smithi110 ceph-mon[118821]: pgmap v75: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:18.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:18 smithi138 ceph-mon[104418]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T22:54:18.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:18 smithi138 ceph-mon[104418]: pgmap v75: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:18.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:19.078 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-09-23T22:54:19.490 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:20.323 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:54:20 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:54:20] "GET /metrics HTTP/1.1" 200 241019 "" "Prometheus/2.43.0" 2023-09-23T22:54:20.333 INFO:teuthology.orchestra.run.smithi110.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-09-23T22:54:20.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:20 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/4250570138' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:54:20.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:20 smithi138 ceph-mon[104418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:54:20.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:20 smithi138 ceph-mon[104418]: pgmap v76: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:20.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:20 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/4250570138' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:54:20.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:20 smithi110 ceph-mon[100243]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:54:20.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:20 smithi110 ceph-mon[100243]: pgmap v76: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:20.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:20 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/4250570138' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:54:20.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:20 smithi110 ceph-mon[118821]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:54:20.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:20 smithi110 ceph-mon[118821]: pgmap v76: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:21.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:21 smithi138 ceph-mon[104418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:54:21.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:21 smithi138 ceph-mon[104418]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T22:54:21.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:21 smithi110 ceph-mon[100243]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:54:21.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:21 smithi110 ceph-mon[100243]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T22:54:21.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:21 smithi110 ceph-mon[118821]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:54:21.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:21 smithi110 ceph-mon[118821]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T22:54:21.747 INFO:teuthology.orchestra.run.smithi110.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:54:22.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:54:21 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:54:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:54:22.222 DEBUG:teuthology.orchestra.run.smithi110:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-09-23T22:54:22.225 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T22:54:22.225 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T22:54:22.271 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:hints = 1 2023-09-23T22:54:22.271 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T22:54:22.272 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object prefix: benchmark_data_smithi110_190492 2023-09-23T22:54:22.272 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:54:22.272 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 0 0 0 0 0 0 - 0 2023-09-23T22:54:22.536 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T22:54:22.752 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[100243]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:54:22.752 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[100243]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:54:22.752 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:22.752 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:54:22.753 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:54:22.753 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:54:22.753 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:22.753 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[100243]: pgmap v78: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:22.753 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[100243]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:54:23.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:22 smithi138 ceph-mon[104418]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:54:23.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:22 smithi138 ceph-mon[104418]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:54:23.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:22 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:23.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:22 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:54:23.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:22 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:54:23.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:22 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:54:23.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:22 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:23.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:22 smithi138 ceph-mon[104418]: pgmap v78: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:23.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:22 smithi138 ceph-mon[104418]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:54:23.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[118821]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:54:23.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[118821]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:54:23.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:23.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:54:23.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:54:23.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:54:23.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:54:23.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[118821]: pgmap v78: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:54:23.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:22 smithi110 ceph-mon[118821]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:54:23.267 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:23.275 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 1 16 3675 3659 227.923 228.688 0.00397668 0.00437127 2023-09-23T22:54:24.275 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 2 16 6785 6769 211.153 194.375 0.00851305 0.0047252 2023-09-23T22:54:25.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:24 smithi138 ceph-mon[104418]: pgmap v79: 89 pgs: 89 active+clean; 187 MiB data, 483 MiB used, 715 GiB / 715 GiB avail; 25 MiB/s wr, 397 op/s 2023-09-23T22:54:25.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:24 smithi110 ceph-mon[118821]: pgmap v79: 89 pgs: 89 active+clean; 187 MiB data, 483 MiB used, 715 GiB / 715 GiB avail; 25 MiB/s wr, 397 op/s 2023-09-23T22:54:25.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:24 smithi110 ceph-mon[100243]: pgmap v79: 89 pgs: 89 active+clean; 187 MiB data, 483 MiB used, 715 GiB / 715 GiB avail; 25 MiB/s wr, 397 op/s 2023-09-23T22:54:25.275 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 3 16 10047 10031 208.725 203.875 0.0023359 0.00478289 2023-09-23T22:54:26.276 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 4 16 13149 13133 204.988 193.875 0.0033581 0.00487124 2023-09-23T22:54:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:26 smithi138 ceph-mon[104418]: pgmap v80: 89 pgs: 89 active+clean; 332 MiB data, 714 MiB used, 715 GiB / 715 GiB avail; 39 MiB/s wr, 625 op/s 2023-09-23T22:54:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:26 smithi110 ceph-mon[100243]: pgmap v80: 89 pgs: 89 active+clean; 332 MiB data, 714 MiB used, 715 GiB / 715 GiB avail; 39 MiB/s wr, 625 op/s 2023-09-23T22:54:27.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:26 smithi110 ceph-mon[118821]: pgmap v80: 89 pgs: 89 active+clean; 332 MiB data, 714 MiB used, 715 GiB / 715 GiB avail; 39 MiB/s wr, 625 op/s 2023-09-23T22:54:27.276 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 5 16 15693 15677 195.794 159 0.00761506 0.00510168 2023-09-23T22:54:27.473 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T22:54:27.473 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (65s) 19s ago 114s 22.7M - 0.25.0 c8568f914cd2 b148630013bf 2023-09-23T22:54:27.474 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi138 *:3000 running (90s) 21s ago 89s 89.8M - 9.4.7 2c41d148cca3 e7b0f17c5681 2023-09-23T22:54:27.474 INFO:teuthology.orchestra.run.smithi110.stdout:mds.foo.smithi110.fuglpi smithi110 running (51s) 19s ago 51s 16.4M - 17.2.6-1125-gc9bb1bcf 444539aec5bf f59eb5601951 2023-09-23T22:54:27.474 INFO:teuthology.orchestra.run.smithi110.stdout:mds.foo.smithi138.xrewco smithi138 running (49s) 21s ago 48s 13.0M - 17.2.6-1125-gc9bb1bcf 444539aec5bf cf02b45fd734 2023-09-23T22:54:27.474 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi138 *:8443 running (5m) 21s ago 5m 439M - 17.2.6-1125-gc9bb1bcf 444539aec5bf 5ce725349f64 2023-09-23T22:54:27.474 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:9283 running (7m) 19s ago 7m 495M - 17.2.6-1125-gc9bb1bcf 444539aec5bf 906aab6b0363 2023-09-23T22:54:27.474 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (7m) 19s ago 7m 82.6M 2048M 17.2.6-1125-gc9bb1bcf 444539aec5bf c610848cb8f9 2023-09-23T22:54:27.474 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi138 running (6m) 21s ago 6m 55.2M 2048M 17.2.6-1125-gc9bb1bcf 444539aec5bf 522bb9710d62 2023-09-23T22:54:27.474 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (6m) 19s ago 6m 57.2M 2048M 17.2.6-1125-gc9bb1bcf 444539aec5bf aaa9613ef02d 2023-09-23T22:54:27.474 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (80s) 19s ago 80s 16.2M - 1.5.0 0da6a335fe13 9d830b481d84 2023-09-23T22:54:27.474 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi138 *:9100 running (75s) 21s ago 74s 8786k - 1.5.0 0da6a335fe13 68cfd3f377d7 2023-09-23T22:54:27.475 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (5m) 19s ago 5m 57.0M 4036M 17.2.6-1125-gc9bb1bcf 444539aec5bf d7807df7af7a 2023-09-23T22:54:27.475 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (5m) 19s ago 5m 59.0M 4036M 17.2.6-1125-gc9bb1bcf 444539aec5bf 9cdbba4d4209 2023-09-23T22:54:27.475 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (4m) 19s ago 4m 51.2M 4036M 17.2.6-1125-gc9bb1bcf 444539aec5bf 8350abe92da2 2023-09-23T22:54:27.475 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (4m) 19s ago 4m 52.4M 4036M 17.2.6-1125-gc9bb1bcf 444539aec5bf d34afe3cf9ea 2023-09-23T22:54:27.475 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi138 running (3m) 21s ago 3m 54.8M 4036M 17.2.6-1125-gc9bb1bcf 444539aec5bf 47f7252bf067 2023-09-23T22:54:27.475 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi138 running (3m) 21s ago 3m 50.2M 4036M 17.2.6-1125-gc9bb1bcf 444539aec5bf fb3decabc2c4 2023-09-23T22:54:27.475 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi138 running (3m) 21s ago 3m 49.4M 4036M 17.2.6-1125-gc9bb1bcf 444539aec5bf c3f9d2352b7c 2023-09-23T22:54:27.475 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi138 running (2m) 21s ago 2m 51.5M 4036M 17.2.6-1125-gc9bb1bcf 444539aec5bf 161d1d2d0445 2023-09-23T22:54:27.475 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi138 *:9095 running (60s) 21s ago 2m 37.4M - 2.43.0 a07b618ecd1d 685848669b55 2023-09-23T22:54:28.276 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 6 16 18358 18342 190.922 166.562 0.00327019 0.00523071 2023-09-23T22:54:29.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:28 smithi138 ceph-mon[104418]: from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:54:29.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:28 smithi138 ceph-mon[104418]: pgmap v81: 89 pgs: 89 active+clean; 332 MiB data, 714 MiB used, 715 GiB / 715 GiB avail; 33 MiB/s wr, 529 op/s 2023-09-23T22:54:29.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:28 smithi110 ceph-mon[100243]: from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:54:29.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:28 smithi110 ceph-mon[100243]: pgmap v81: 89 pgs: 89 active+clean; 332 MiB data, 714 MiB used, 715 GiB / 715 GiB avail; 33 MiB/s wr, 529 op/s 2023-09-23T22:54:29.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:28 smithi110 ceph-mon[118821]: from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:54:29.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:28 smithi110 ceph-mon[118821]: pgmap v81: 89 pgs: 89 active+clean; 332 MiB data, 714 MiB used, 715 GiB / 715 GiB avail; 33 MiB/s wr, 529 op/s 2023-09-23T22:54:29.277 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 7 16 20896 20880 186.292 158.625 0.00279436 0.00536279 2023-09-23T22:54:30.277 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 8 16 23292 23276 181.725 149.75 0.00339625 0.00549606 2023-09-23T22:54:30.347 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:54:30 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:54:30] "GET /metrics HTTP/1.1" 200 241019 "" "Prometheus/2.43.0" 2023-09-23T22:54:30.353 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:30 smithi110 ceph-mon[100243]: pgmap v82: 89 pgs: 89 active+clean; 1.2 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T22:54:30.356 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:30 smithi110 ceph-mon[118821]: pgmap v82: 89 pgs: 89 active+clean; 1.2 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T22:54:30.400 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for minority of mons to upgrade' 2023-09-23T22:54:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:30 smithi138 ceph-mon[104418]: pgmap v82: 89 pgs: 89 active+clean; 1.2 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T22:54:31.062 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:31.277 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 9 16 25419 25403 176.306 132.938 0.00837289 0.00566684 2023-09-23T22:54:32.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:54:31 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:54:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:54:32.277 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 10 16 27596 27580 172.282 136.062 0.00555861 0.00579917 2023-09-23T22:54:33.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:32 smithi138 ceph-mon[104418]: pgmap v83: 89 pgs: 89 active+clean; 1.2 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-09-23T22:54:33.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:32 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:33.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:32 smithi110 ceph-mon[100243]: pgmap v83: 89 pgs: 89 active+clean; 1.2 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-09-23T22:54:33.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:32 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:33.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:32 smithi110 ceph-mon[118821]: pgmap v83: 89 pgs: 89 active+clean; 1.2 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-09-23T22:54:33.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:32 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:33.277 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 11 16 29084 29068 165.077 93 0.00816747 0.00605232 2023-09-23T22:54:34.277 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 12 16 31191 31175 162.295 131.688 0.0161797 0.00615577 2023-09-23T22:54:34.537 INFO:teuthology.orchestra.run.smithi110.stdout:wait for minority of mons to upgrade 2023-09-23T22:54:35.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:34 smithi138 ceph-mon[104418]: pgmap v84: 89 pgs: 89 active+clean; 1.6 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-09-23T22:54:35.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:34 smithi110 ceph-mon[100243]: pgmap v84: 89 pgs: 89 active+clean; 1.6 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-09-23T22:54:35.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:34 smithi110 ceph-mon[118821]: pgmap v84: 89 pgs: 89 active+clean; 1.6 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-09-23T22:54:35.277 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 13 16 33147 33131 159.215 122.25 0.00958722 0.00627497 2023-09-23T22:54:36.277 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 14 16 34728 34712 154.901 98.8125 0.0121528 0.00644952 2023-09-23T22:54:36.324 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-09-23T22:54:37.032 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:37.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:36 smithi138 ceph-mon[104418]: pgmap v85: 89 pgs: 89 active+clean; 1.8 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-09-23T22:54:37.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:36 smithi110 ceph-mon[100243]: pgmap v85: 89 pgs: 89 active+clean; 1.8 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-09-23T22:54:37.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:36 smithi110 ceph-mon[118821]: pgmap v85: 89 pgs: 89 active+clean; 1.8 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-09-23T22:54:37.278 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 15 16 36518 36502 152.025 111.875 0.00945049 0.00657304 2023-09-23T22:54:38.278 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 16 16 38319 38303 149.559 112.562 0.00830088 0.00668129 2023-09-23T22:54:39.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:38 smithi138 ceph-mon[104418]: pgmap v86: 89 pgs: 89 active+clean; 1.8 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T22:54:39.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:38 smithi110 ceph-mon[100243]: pgmap v86: 89 pgs: 89 active+clean; 1.8 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T22:54:39.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:38 smithi110 ceph-mon[118821]: pgmap v86: 89 pgs: 89 active+clean; 1.8 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T22:54:39.279 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 17 16 39272 39256 144.262 59.5625 0.122887 0.00690329 2023-09-23T22:54:40.279 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 18 16 41149 41133 142.765 117.312 0.00618309 0.00699966 2023-09-23T22:54:40.344 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:54:40 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:54:40] "GET /metrics HTTP/1.1" 200 242000 "" "Prometheus/2.43.0" 2023-09-23T22:54:40.344 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:40 smithi110 ceph-mon[100243]: pgmap v87: 89 pgs: 89 active+clean; 2.4 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 177 MiB/s wr, 2.83k op/s 2023-09-23T22:54:40.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:40 smithi138 ceph-mon[104418]: pgmap v87: 89 pgs: 89 active+clean; 2.4 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 177 MiB/s wr, 2.83k op/s 2023-09-23T22:54:40.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:40 smithi110 ceph-mon[118821]: pgmap v87: 89 pgs: 89 active+clean; 2.4 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 177 MiB/s wr, 2.83k op/s 2023-09-23T22:54:41.280 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 19 16 42892 42876 140.978 108.938 0.00779725 0.00708772 2023-09-23T22:54:42.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:54:41 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:54:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:54:42.280 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T22:54:42.280441+0000 min lat: 0.0011204 max lat: 0.355939 avg lat: 0.00718094 2023-09-23T22:54:42.280 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:54:42.280 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 20 16 44563 44547 139.151 104.438 0.0108731 0.00718094 2023-09-23T22:54:43.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:42 smithi138 ceph-mon[104418]: pgmap v88: 89 pgs: 89 active+clean; 2.4 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T22:54:43.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:42 smithi110 ceph-mon[118821]: pgmap v88: 89 pgs: 89 active+clean; 2.4 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T22:54:43.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:42 smithi110 ceph-mon[100243]: pgmap v88: 89 pgs: 89 active+clean; 2.4 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T22:54:43.284 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 21 16 46158 46142 137.246 99.6875 0.0112536 0.00728051 2023-09-23T22:54:44.285 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 22 16 47655 47639 135.256 93.5625 0.010598 0.0073889 2023-09-23T22:54:45.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:44 smithi138 ceph-mon[104418]: pgmap v89: 89 pgs: 89 active+clean; 2.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-09-23T22:54:45.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:44 smithi110 ceph-mon[100243]: pgmap v89: 89 pgs: 89 active+clean; 2.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-09-23T22:54:45.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:44 smithi110 ceph-mon[118821]: pgmap v89: 89 pgs: 89 active+clean; 2.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-09-23T22:54:45.285 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 23 16 49232 49216 133.661 98.5625 0.00943968 0.00747749 2023-09-23T22:54:46.285 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 24 16 50735 50719 132.006 93.9375 0.0119457 0.00757088 2023-09-23T22:54:47.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:46 smithi138 ceph-mon[104418]: pgmap v90: 89 pgs: 89 active+clean; 2.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T22:54:47.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:46 smithi110 ceph-mon[118821]: pgmap v90: 89 pgs: 89 active+clean; 2.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T22:54:47.213 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:46 smithi110 ceph-mon[100243]: pgmap v90: 89 pgs: 89 active+clean; 2.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T22:54:47.285 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 25 16 52253 52237 130.521 94.875 0.00906672 0.0076554 2023-09-23T22:54:48.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:47 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:48.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:47 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:48.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:47 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:48.285 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 26 16 53608 53592 128.759 84.6875 0.0171998 0.00776182 2023-09-23T22:54:49.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:48 smithi138 ceph-mon[104418]: pgmap v91: 89 pgs: 89 active+clean; 2.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T22:54:49.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:48 smithi110 ceph-mon[100243]: pgmap v91: 89 pgs: 89 active+clean; 2.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T22:54:49.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:48 smithi110 ceph-mon[118821]: pgmap v91: 89 pgs: 89 active+clean; 2.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T22:54:49.285 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 27 16 55054 55038 127.338 90.375 0.0147764 0.00784823 2023-09-23T22:54:50.285 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 28 16 56478 56462 125.969 89 0.0171248 0.00793418 2023-09-23T22:54:50.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:50 smithi110 ceph-mon[100243]: pgmap v92: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-09-23T22:54:50.357 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:54:50 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:54:50] "GET /metrics HTTP/1.1" 200 242202 "" "Prometheus/2.43.0" 2023-09-23T22:54:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:50 smithi138 ceph-mon[104418]: pgmap v92: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-09-23T22:54:50.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:50 smithi110 ceph-mon[118821]: pgmap v92: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-09-23T22:54:51.285 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 29 16 57848 57832 124.578 85.625 0.0113747 0.00802272 2023-09-23T22:54:52.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:54:51 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:54:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:54:52.286 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 30 16 59368 59352 123.592 95 0.0139168 0.00808725 2023-09-23T22:54:53.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:52 smithi138 ceph-mon[104418]: pgmap v93: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-09-23T22:54:53.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:52 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/589818057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:54:53.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:52 smithi110 ceph-mon[100243]: pgmap v93: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-09-23T22:54:53.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:52 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/589818057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:54:53.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:52 smithi110 ceph-mon[118821]: pgmap v93: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-09-23T22:54:53.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:52 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/589818057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:54:53.286 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 31 16 60846 60830 122.585 92.375 0.0126943 0.00815375 2023-09-23T22:54:54.286 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 32 16 62395 62379 121.78 96.8125 0.00737819 0.00820765 2023-09-23T22:54:55.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:54 smithi138 ceph-mon[104418]: pgmap v94: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-09-23T22:54:55.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:54 smithi110 ceph-mon[100243]: pgmap v94: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-09-23T22:54:55.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:54 smithi110 ceph-mon[118821]: pgmap v94: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-09-23T22:54:55.286 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 33 16 63855 63839 120.855 91.25 0.00900671 0.0082697 2023-09-23T22:54:56.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:55 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1377010792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:54:56.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:55 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1377010792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:54:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:55 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1377010792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:54:56.286 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 34 16 65353 65337 120.054 93.625 0.0112541 0.00832515 2023-09-23T22:54:57.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:56 smithi138 ceph-mon[104418]: pgmap v95: 89 pgs: 89 active+clean; 3.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T22:54:57.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:56 smithi110 ceph-mon[118821]: pgmap v95: 89 pgs: 89 active+clean; 3.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T22:54:57.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:56 smithi110 ceph-mon[100243]: pgmap v95: 89 pgs: 89 active+clean; 3.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T22:54:57.286 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 35 16 66849 66833 119.295 93.5 0.0109939 0.00837744 2023-09-23T22:54:58.286 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 36 16 68222 68206 118.365 85.8125 0.0108212 0.0084433 2023-09-23T22:54:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:58 smithi138 ceph-mon[104418]: pgmap v96: 89 pgs: 89 active+clean; 3.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T22:54:59.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:54:58 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/475076312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:54:59.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:58 smithi110 ceph-mon[100243]: pgmap v96: 89 pgs: 89 active+clean; 3.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T22:54:59.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:54:58 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/475076312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:54:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:58 smithi110 ceph-mon[118821]: pgmap v96: 89 pgs: 89 active+clean; 3.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T22:54:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:54:58 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/475076312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:54:59.286 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 37 16 69646 69630 117.572 89 0.0193936 0.00850142 2023-09-23T22:55:00.286 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 38 16 70753 70737 116.299 69.1875 0.0156764 0.00859412 2023-09-23T22:55:00.365 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:55:00 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:55:00] "GET /metrics HTTP/1.1" 200 242202 "" "Prometheus/2.43.0" 2023-09-23T22:55:00.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:00 smithi110 ceph-mon[100243]: pgmap v97: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-09-23T22:55:00.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:00 smithi110 ceph-mon[118821]: pgmap v97: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-09-23T22:55:00.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:00 smithi138 ceph-mon[104418]: pgmap v97: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-09-23T22:55:01.286 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 39 16 72116 72100 115.501 85.1875 0.00428743 0.0086528 2023-09-23T22:55:01.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:01 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1057934975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:01 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1057934975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:01.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:01 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1057934975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:02.193 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:01 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:55:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:55:02.286 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T22:55:02.286982+0000 min lat: 0.0011204 max lat: 0.355939 avg lat: 0.00871101 2023-09-23T22:55:02.287 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:55:02.287 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 40 15 73479 73464 114.745 85.25 0.00610552 0.00871101 2023-09-23T22:55:02.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:02 smithi138 ceph-mon[104418]: pgmap v98: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T22:55:02.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:02 smithi110 ceph-mon[100243]: pgmap v98: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T22:55:02.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:02 smithi110 ceph-mon[118821]: pgmap v98: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T22:55:03.286 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 41 16 74793 74777 113.948 82.0625 0.0224125 0.00877161 2023-09-23T22:55:03.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:03 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:03.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:03 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3574874502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:03.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:03 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:03.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:03 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3574874502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:03.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:03 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:03.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:03 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3574874502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:04.287 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 42 16 76145 76129 113.247 84.5 0.00935635 0.00882549 2023-09-23T22:55:04.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:04 smithi138 ceph-mon[104418]: pgmap v99: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T22:55:04.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:04 smithi110 ceph-mon[100243]: pgmap v99: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T22:55:04.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:04 smithi110 ceph-mon[118821]: pgmap v99: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T22:55:05.287 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 43 16 77476 77460 112.548 83.1875 0.00578016 0.00888008 2023-09-23T22:55:06.287 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 44 16 78737 78721 111.781 78.8125 0.0129339 0.00894167 2023-09-23T22:55:07.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:06 smithi138 ceph-mon[104418]: pgmap v100: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T22:55:07.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:06 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2042610454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:07.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:06 smithi110 ceph-mon[118821]: pgmap v100: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T22:55:07.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:06 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2042610454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:07.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:06 smithi110 ceph-mon[100243]: pgmap v100: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T22:55:07.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:06 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2042610454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:07.287 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 45 16 80001 79985 111.052 79 0.0139305 0.00899973 2023-09-23T22:55:08.287 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 46 16 81170 81154 110.227 73.0625 0.00934134 0.00906742 2023-09-23T22:55:08.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:08 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:08.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:08 smithi138 ceph-mon[104418]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T22:55:08.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:08 smithi138 ceph-mon[104418]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T22:55:08.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:08 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:55:08.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:08 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:08.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:08 smithi138 ceph-mon[104418]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T22:55:08.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:08 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:08.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:08 smithi110 ceph-mon[100243]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T22:55:08.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:08 smithi110 ceph-mon[100243]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T22:55:08.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:08 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:55:08.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:08 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:08.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:08 smithi110 ceph-mon[100243]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T22:55:08.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:08 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:08.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:08 smithi110 ceph-mon[118821]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T22:55:08.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:08 smithi110 ceph-mon[118821]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T22:55:08.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:08 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:55:08.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:08 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:08.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:08 smithi110 ceph-mon[118821]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T22:55:09.290 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 47 16 82301 82285 109.378 70.6875 0.0151052 0.00913802 2023-09-23T22:55:09.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:09 smithi138 ceph-mon[104418]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi138 2023-09-23T22:55:09.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:09 smithi138 ceph-mon[104418]: pgmap v101: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T22:55:09.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:09 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/4202401559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:09.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:09 smithi110 ceph-mon[100243]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi138 2023-09-23T22:55:09.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:09 smithi110 ceph-mon[100243]: pgmap v101: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T22:55:09.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:09 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/4202401559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:09.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:09 smithi110 ceph-mon[118821]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi138 2023-09-23T22:55:09.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:09 smithi110 ceph-mon[118821]: pgmap v101: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T22:55:09.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:09 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/4202401559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:10.290 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 48 16 83434 83418 108.574 70.8125 0.0115281 0.00920481 2023-09-23T22:55:10.359 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:55:10 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:55:10] "GET /metrics HTTP/1.1" 200 242288 "" "Prometheus/2.43.0" 2023-09-23T22:55:10.361 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:10 smithi110 ceph-mon[100243]: pgmap v102: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-09-23T22:55:10.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:10 smithi138 ceph-mon[104418]: pgmap v102: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-09-23T22:55:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:10 smithi110 ceph-mon[118821]: pgmap v102: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-09-23T22:55:11.291 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 49 16 84521 84505 107.745 67.9375 0.0109479 0.00927695 2023-09-23T22:55:11.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:11 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3023612947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:11.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:11 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3023612947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:11.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:11 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3023612947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:12.202 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:11 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:55:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:55:12.291 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 50 16 85633 85617 106.98 69.5 0.00496784 0.00934297 2023-09-23T22:55:12.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:12 smithi138 ceph-mon[104418]: pgmap v103: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T22:55:12.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:12 smithi110 ceph-mon[100243]: pgmap v103: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T22:55:12.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:12 smithi110 ceph-mon[118821]: pgmap v103: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T22:55:13.294 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 51 16 86697 86681 106.18 66.5 0.00959566 0.00941339 2023-09-23T22:55:14.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:13 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/120722660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:14.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:13 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/120722660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:14.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:13 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/120722660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:14.294 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 52 16 87752 87736 105.406 65.9375 0.0200255 0.00948105 2023-09-23T22:55:15.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:14 smithi110 ceph-mon[100243]: pgmap v104: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-09-23T22:55:15.201 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:14 smithi138 ceph-mon[104418]: pgmap v104: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-09-23T22:55:15.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:14 smithi110 ceph-mon[118821]: pgmap v104: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-09-23T22:55:15.294 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 53 16 88790 88774 104.642 64.875 0.0139076 0.00955203 2023-09-23T22:55:16.294 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 54 16 89802 89786 103.875 63.25 0.0129393 0.00962188 2023-09-23T22:55:17.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:16 smithi138 ceph-mon[104418]: pgmap v105: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-23T22:55:17.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:16 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2540948700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:17.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:16 smithi110 ceph-mon[100243]: pgmap v105: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-23T22:55:17.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:16 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2540948700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:16 smithi110 ceph-mon[118821]: pgmap v105: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-23T22:55:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:16 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2540948700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:17.294 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 55 16 90846 90830 103.173 65.25 0.0127891 0.00968828 2023-09-23T22:55:18.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T22:55:18.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2023-09-23T22:55:18.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T22:55:18.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:18.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T22:55:18.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2023-09-23T22:55:18.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T22:55:18.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:18.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T22:55:18.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2023-09-23T22:55:18.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T22:55:18.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:18.294 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 56 16 91854 91838 102.456 63 0.00996454 0.00975593 2023-09-23T22:55:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:18 smithi138 ceph-mon[104418]: pgmap v106: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2023-09-23T22:55:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-09-23T22:55:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2023-09-23T22:55:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:18 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-23T22:55:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:18 smithi138 ceph-mon[104418]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T22:55:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:18 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3062020620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:19.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:18 smithi110 ceph-mon[118821]: pgmap v106: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2023-09-23T22:55:19.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-09-23T22:55:19.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2023-09-23T22:55:19.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:18 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-23T22:55:19.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:18 smithi110 ceph-mon[118821]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T22:55:19.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:18 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3062020620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:19.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:18 smithi110 ceph-mon[100243]: pgmap v106: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2023-09-23T22:55:19.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-09-23T22:55:19.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2023-09-23T22:55:19.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:18 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-23T22:55:19.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:18 smithi110 ceph-mon[100243]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T22:55:19.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:18 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3062020620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:19.294 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 57 16 92845 92829 101.745 61.9375 0.00701262 0.00982325 2023-09-23T22:55:20.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:19 smithi138 ceph-mon[104418]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T22:55:20.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:19 smithi110 ceph-mon[100243]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T22:55:20.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:19 smithi110 ceph-mon[118821]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T22:55:20.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:55:20 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:55:20] "GET /metrics HTTP/1.1" 200 242346 "" "Prometheus/2.43.0" 2023-09-23T22:55:20.294 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 58 16 93856 93840 101.081 63.1875 0.0177679 0.00988861 2023-09-23T22:55:21.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:20 smithi138 ceph-mon[104418]: pgmap v109: 89 pgs: 3 peering, 86 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-09-23T22:55:21.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:20 smithi110 ceph-mon[118821]: pgmap v109: 89 pgs: 3 peering, 86 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-09-23T22:55:21.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:20 smithi110 ceph-mon[100243]: pgmap v109: 89 pgs: 3 peering, 86 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-09-23T22:55:21.297 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 59 16 94849 94833 100.416 62.0625 0.0211731 0.00995438 2023-09-23T22:55:22.193 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:21 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:55:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:55:22.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:21 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2442757019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:22.297 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T22:55:22.297477+0000 min lat: 0.0011204 max lat: 0.355939 avg lat: 0.010017 2023-09-23T22:55:22.297 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:55:22.297 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 60 16 95838 95822 99.7724 61.8125 0.0236694 0.010017 2023-09-23T22:55:22.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:21 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2442757019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:22.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:21 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2442757019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:23.297 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 61 16 96832 96816 99.1554 62.125 0.0130223 0.0100799 2023-09-23T22:55:23.448 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:22 smithi138 ceph-mon[104418]: pgmap v110: 89 pgs: 3 peering, 86 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2023-09-23T22:55:23.448 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:22 smithi138 ceph-mon[104418]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:55:23.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:22 smithi110 ceph-mon[118821]: pgmap v110: 89 pgs: 3 peering, 86 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2023-09-23T22:55:23.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:22 smithi110 ceph-mon[118821]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:55:23.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:22 smithi110 ceph-mon[100243]: pgmap v110: 89 pgs: 3 peering, 86 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2023-09-23T22:55:23.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:22 smithi110 ceph-mon[100243]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:55:24.297 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 62 16 97787 97771 98.5189 59.6875 0.015133 0.0101441 2023-09-23T22:55:25.297 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 63 16 98765 98749 97.9255 61.125 0.0163192 0.0102076 2023-09-23T22:55:25.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:24 smithi138 ceph-mon[104418]: pgmap v111: 89 pgs: 1 peering, 88 active+clean; 5.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s 2023-09-23T22:55:25.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:24 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3978139963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:25.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:24 smithi110 ceph-mon[100243]: pgmap v111: 89 pgs: 1 peering, 88 active+clean; 5.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s 2023-09-23T22:55:25.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:24 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3978139963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:25.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:24 smithi110 ceph-mon[118821]: pgmap v111: 89 pgs: 1 peering, 88 active+clean; 5.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s 2023-09-23T22:55:25.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:24 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3978139963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:26.297 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 64 16 99700 99684 97.3086 58.4375 0.0199484 0.0102716 2023-09-23T22:55:26.448 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:26 smithi138 ceph-mon[104418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T22:55:26.448 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:26 smithi138 ceph-mon[104418]: Cluster is now healthy 2023-09-23T22:55:26.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:26 smithi110 ceph-mon[100243]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T22:55:26.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:26 smithi110 ceph-mon[100243]: Cluster is now healthy 2023-09-23T22:55:26.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:26 smithi110 ceph-mon[118821]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T22:55:26.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:26 smithi110 ceph-mon[118821]: Cluster is now healthy 2023-09-23T22:55:27.298 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 65 16 100591 100575 96.6675 55.6875 0.00704692 0.0103399 2023-09-23T22:55:27.449 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:27 smithi138 ceph-mon[104418]: pgmap v112: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T22:55:27.449 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:27 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3727781301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:27.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:27 smithi110 ceph-mon[100243]: pgmap v112: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T22:55:27.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:27 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3727781301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:27.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:27 smithi110 ceph-mon[118821]: pgmap v112: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T22:55:27.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:27 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3727781301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:28.298 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 66 16 101478 101462 96.043 55.4375 0.0156465 0.0104076 2023-09-23T22:55:29.300 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 67 16 102368 102352 95.4374 55.625 0.0115084 0.0104741 2023-09-23T22:55:29.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:29 smithi138 ceph-mon[104418]: pgmap v113: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T22:55:29.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:29 smithi110 ceph-mon[100243]: pgmap v113: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T22:55:29.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:29 smithi110 ceph-mon[118821]: pgmap v113: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T22:55:30.300 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 68 16 103267 103251 94.8602 56.1875 0.011293 0.0105371 2023-09-23T22:55:30.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:30 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/406994572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:30.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:55:30 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:55:30] "GET /metrics HTTP/1.1" 200 242346 "" "Prometheus/2.43.0" 2023-09-23T22:55:30.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:30 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/406994572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:30.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:30 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/406994572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:31.300 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 69 16 104178 104162 94.3108 56.9375 0.0125603 0.010597 2023-09-23T22:55:31.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:31 smithi138 ceph-mon[104418]: pgmap v114: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T22:55:31.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:31 smithi110 ceph-mon[118821]: pgmap v114: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T22:55:31.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:31 smithi110 ceph-mon[100243]: pgmap v114: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T22:55:32.194 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:31 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:55:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:55:32.196 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:32 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1973584195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:32.300 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 70 16 104986 104970 93.685 50.5 0.0117098 0.0106688 2023-09-23T22:55:32.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:32 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1973584195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:32.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:32 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1973584195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:33.300 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 71 16 105854 105838 93.1297 54.25 0.0104008 0.0107332 2023-09-23T22:55:33.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:33 smithi138 ceph-mon[104418]: pgmap v115: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s 2023-09-23T22:55:33.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:33 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:33.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:33 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:33.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:33 smithi110 ceph-mon[118821]: pgmap v115: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s 2023-09-23T22:55:33.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:33 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:33.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:33 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:33.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:33 smithi110 ceph-mon[100243]: pgmap v115: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s 2023-09-23T22:55:33.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:33 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:33.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:33 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:34.300 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 72 16 106718 106702 92.5864 54 0.0145182 0.0107961 2023-09-23T22:55:35.300 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 73 16 107557 107541 92.0365 52.4375 0.0218402 0.010861 2023-09-23T22:55:35.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:35 smithi138 ceph-mon[104418]: pgmap v116: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2023-09-23T22:55:35.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:35 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2711425484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:35.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:35 smithi110 ceph-mon[100243]: pgmap v116: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2023-09-23T22:55:35.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:35 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2711425484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:35.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:35 smithi110 ceph-mon[118821]: pgmap v116: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2023-09-23T22:55:35.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:35 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2711425484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:36.300 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 74 16 108410 108394 91.5133 53.3125 0.0151402 0.0109228 2023-09-23T22:55:37.301 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 75 16 109254 109238 90.9965 52.75 0.00788949 0.0109846 2023-09-23T22:55:37.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:37 smithi138 ceph-mon[104418]: pgmap v117: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2023-09-23T22:55:37.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:37 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2254144674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:37.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:37 smithi110 ceph-mon[118821]: pgmap v117: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2023-09-23T22:55:37.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:37 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2254144674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:37.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:37 smithi110 ceph-mon[100243]: pgmap v117: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2023-09-23T22:55:37.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:37 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2254144674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:38.301 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 76 16 110111 110095 90.5041 53.5625 0.01963 0.0110446 2023-09-23T22:55:39.301 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 77 16 110995 110979 90.0463 55.25 0.0120575 0.0111009 2023-09-23T22:55:39.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:39 smithi138 ceph-mon[104418]: pgmap v118: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2023-09-23T22:55:39.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:39 smithi110 ceph-mon[118821]: pgmap v118: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2023-09-23T22:55:39.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:39 smithi110 ceph-mon[100243]: pgmap v118: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2023-09-23T22:55:40.301 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 78 16 111930 111914 89.6411 58.4375 0.00787167 0.0111507 2023-09-23T22:55:40.359 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:55:40 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:55:40] "GET /metrics HTTP/1.1" 200 242413 "" "Prometheus/2.43.0" 2023-09-23T22:55:40.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:40 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3379621854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:40.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:40 smithi138 ceph-mon[104418]: pgmap v119: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T22:55:40.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:40 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3379621854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:40.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:40 smithi110 ceph-mon[100243]: pgmap v119: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T22:55:40.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:40 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3379621854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:40.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:40 smithi110 ceph-mon[118821]: pgmap v119: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T22:55:41.301 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 79 16 112905 112889 89.2778 60.9375 0.0297195 0.0111962 2023-09-23T22:55:42.194 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:41 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:55:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:55:42.301 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T22:55:42.301491+0000 min lat: 0.0011204 max lat: 0.355939 avg lat: 0.0112438 2023-09-23T22:55:42.301 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:55:42.301 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 80 16 113854 113838 88.9033 59.3125 0.0152074 0.0112438 2023-09-23T22:55:43.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:42 smithi138 ceph-mon[104418]: pgmap v120: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s 2023-09-23T22:55:43.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:42 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2680029885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:43.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:42 smithi110 ceph-mon[118821]: pgmap v120: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s 2023-09-23T22:55:43.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:42 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2680029885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:43.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:42 smithi110 ceph-mon[100243]: pgmap v120: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s 2023-09-23T22:55:43.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:42 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2680029885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:43.302 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 81 16 114833 114817 88.5612 61.1875 0.0211231 0.0112873 2023-09-23T22:55:44.301 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 82 16 115801 115785 88.219 60.5 0.0180322 0.0113314 2023-09-23T22:55:45.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:44 smithi138 ceph-mon[104418]: pgmap v121: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s 2023-09-23T22:55:45.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:44 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1415976932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:45.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:44 smithi110 ceph-mon[118821]: pgmap v121: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s 2023-09-23T22:55:45.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:44 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1415976932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:45.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:44 smithi110 ceph-mon[100243]: pgmap v121: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s 2023-09-23T22:55:45.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:44 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1415976932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:45.301 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 83 16 116741 116725 87.8639 58.75 0.0123393 0.0113766 2023-09-23T22:55:46.301 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 84 16 117688 117672 87.5225 59.1875 0.00902448 0.0114209 2023-09-23T22:55:47.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:46 smithi138 ceph-mon[104418]: pgmap v122: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T22:55:47.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:46 smithi110 ceph-mon[100243]: pgmap v122: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T22:55:47.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:46 smithi110 ceph-mon[118821]: pgmap v122: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T22:55:47.301 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 85 16 118628 118612 87.1841 58.75 0.0145126 0.0114647 2023-09-23T22:55:48.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:47 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1122523701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:48.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:47 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:48.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:47 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1122523701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:48.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:47 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:48.301 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 86 16 119580 119564 86.8622 59.5 0.0188456 0.0115081 2023-09-23T22:55:48.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:47 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1122523701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:48.496 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:47 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:49.302 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 87 16 120515 120499 86.5355 58.4375 0.0209417 0.0115514 2023-09-23T22:55:49.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:48 smithi138 ceph-mon[104418]: pgmap v123: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 752 op/s 2023-09-23T22:55:49.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:48 smithi110 ceph-mon[100243]: pgmap v123: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 752 op/s 2023-09-23T22:55:49.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:48 smithi110 ceph-mon[118821]: pgmap v123: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 752 op/s 2023-09-23T22:55:50.302 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 88 16 121421 121405 86.1956 56.625 0.0103578 0.0115967 2023-09-23T22:55:50.381 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:55:50 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:55:50] "GET /metrics HTTP/1.1" 200 242418 "" "Prometheus/2.43.0" 2023-09-23T22:55:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:50 smithi138 ceph-mon[104418]: pgmap v124: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T22:55:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:50 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3530767137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:50.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:50 smithi110 ceph-mon[118821]: pgmap v124: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T22:55:50.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:50 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3530767137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:50.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:50 smithi110 ceph-mon[100243]: pgmap v124: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T22:55:50.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:50 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3530767137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:51.302 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 89 16 122364 122348 85.8894 58.9375 0.0161659 0.0116382 2023-09-23T22:55:52.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:51 smithi138 conmon[107593]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:55:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:55:52.302 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total time run: 90.0075 2023-09-23T22:55:52.302 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total writes made: 123246 2023-09-23T22:55:52.302 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Write size: 65536 2023-09-23T22:55:52.302 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object size: 65536 2023-09-23T22:55:52.302 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Bandwidth (MB/sec): 85.5804 2023-09-23T22:55:52.302 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Bandwidth: 37.3823 2023-09-23T22:55:52.302 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max bandwidth (MB/sec): 228.688 2023-09-23T22:55:52.303 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min bandwidth (MB/sec): 50.5 2023-09-23T22:55:52.303 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average IOPS: 1369 2023-09-23T22:55:52.303 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev IOPS: 598.116 2023-09-23T22:55:52.303 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max IOPS: 3659 2023-09-23T22:55:52.303 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min IOPS: 808 2023-09-23T22:55:52.303 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average Latency(s): 0.0116814 2023-09-23T22:55:52.303 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Latency(s): 0.00894376 2023-09-23T22:55:52.303 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max latency(s): 0.355939 2023-09-23T22:55:52.303 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min latency(s): 0.0011204 2023-09-23T22:55:52.309 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T22:55:53.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:52 smithi138 ceph-mon[104418]: pgmap v125: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s 2023-09-23T22:55:53.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:52 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2120353460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:53.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:52 smithi110 ceph-mon[100243]: pgmap v125: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s 2023-09-23T22:55:53.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:52 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2120353460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:53.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:52 smithi110 ceph-mon[118821]: pgmap v125: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s 2023-09-23T22:55:53.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:52 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2120353460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:54.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:54 smithi138 ceph-mon[104418]: Upgrade: Updating mgr.x 2023-09-23T22:55:54.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:54 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:54.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:54 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:55:54.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:54 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:55:54.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:54 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:55:54.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:54 smithi110 ceph-mon[118821]: Upgrade: Updating mgr.x 2023-09-23T22:55:54.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:54 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:54.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:54 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:55:54.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:54 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:55:54.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:54 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:55:54.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:54 smithi110 ceph-mon[100243]: Upgrade: Updating mgr.x 2023-09-23T22:55:54.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:54 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:54.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:54 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:55:54.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:54 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:55:54.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:54 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:55:55.635 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:55 smithi138 ceph-mon[104418]: Deploying daemon mgr.x on smithi138 2023-09-23T22:55:55.635 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:55 smithi138 ceph-mon[104418]: pgmap v126: 89 pgs: 89 active+clean; 7.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 884 op/s 2023-09-23T22:55:55.635 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:55 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/259590210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:55.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:55 smithi110 ceph-mon[100243]: Deploying daemon mgr.x on smithi138 2023-09-23T22:55:55.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:55 smithi110 ceph-mon[100243]: pgmap v126: 89 pgs: 89 active+clean; 7.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 884 op/s 2023-09-23T22:55:55.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:55 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/259590210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:55.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:55 smithi110 ceph-mon[118821]: Deploying daemon mgr.x on smithi138 2023-09-23T22:55:55.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:55 smithi110 ceph-mon[118821]: pgmap v126: 89 pgs: 89 active+clean; 7.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 884 op/s 2023-09-23T22:55:55.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:55 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/259590210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:55.945 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:55 smithi138 systemd[1]: Stopping Ceph mgr.x for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:55:56.617 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:56 smithi138 bash[158794]: 5ce725349f64fe7a5c9c896f585d50b761caeda9fd631a56bc5aa749f3936f12 2023-09-23T22:55:56.618 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:56 smithi138 ceph-mon[104418]: pgmap v127: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 58 MiB/s wr, 1.05k op/s 2023-09-23T22:55:56.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:56 smithi110 ceph-mon[118821]: pgmap v127: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 58 MiB/s wr, 1.05k op/s 2023-09-23T22:55:56.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:56 smithi110 ceph-mon[100243]: pgmap v127: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 58 MiB/s wr, 1.05k op/s 2023-09-23T22:55:57.693 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:57 smithi138 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:55:57.693 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:57 smithi138 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T22:55:57.693 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:57 smithi138 systemd[1]: Stopped Ceph mgr.x for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:55:57.693 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:57 smithi138 systemd[1]: Starting Ceph mgr.x for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:55:58.198 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:57 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3928581147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:58.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:57 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3928581147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:58.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:57 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3928581147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:55:58.844 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:58 smithi138 ceph-mon[104418]: pgmap v128: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 777 op/s 2023-09-23T22:55:58.844 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:58 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:58.844 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:58 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:58.844 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:55:58 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:55:58.844 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:58 smithi138 bash[159135]: d6b943d65fcedd63f9a704ad1dad81f173f193188a297793892cdfb064ce87ee 2023-09-23T22:55:58.844 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:58 smithi138 systemd[1]: Started Ceph mgr.x for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:55:58.844 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:58 smithi138 conmon[159173]: 2023-09-23T22:55:58.768+0000 7f2d9e92b200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:55:59.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:58 smithi138 conmon[159173]: 2023-09-23T22:55:58.896+0000 7f2d9e92b200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:55:59.193 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:59 smithi138 conmon[159173]: 2023-09-23T22:55:59.045+0000 7f2d9e92b200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:55:59.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:58 smithi110 ceph-mon[118821]: pgmap v128: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 777 op/s 2023-09-23T22:55:59.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:58 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:59.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:58 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:59.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:55:58 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:55:59.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:58 smithi110 ceph-mon[100243]: pgmap v128: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 777 op/s 2023-09-23T22:55:59.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:58 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:59.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:58 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:55:59.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:55:58 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:55:59.527 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:59 smithi138 conmon[159173]: 2023-09-23T22:55:59.214+0000 7f2d9e92b200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:55:59.942 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:55:59 smithi138 conmon[159173]: 2023-09-23T22:55:59.516+0000 7f2d9e92b200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:56:00.393 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:00 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:56:00] "GET /metrics HTTP/1.1" 200 242418 "" "Prometheus/2.43.0" 2023-09-23T22:56:00.394 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:00 smithi138 conmon[159173]: 2023-09-23T22:56:00.145+0000 7f2d9e92b200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:56:00.394 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:00 smithi138 conmon[159173]: 2023-09-23T22:56:00.341+0000 7f2d9e92b200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:56:00.394 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:00 smithi110 ceph-mon[100243]: pgmap v129: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 1.13k op/s 2023-09-23T22:56:00.646 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:00 smithi138 conmon[159173]: 2023-09-23T22:56:00.475+0000 7f2d9e92b200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:56:00.646 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:00 smithi138 conmon[159173]: 2023-09-23T22:56:00.635+0000 7f2d9e92b200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:56:00.648 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:00 smithi138 ceph-mon[104418]: pgmap v129: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 1.13k op/s 2023-09-23T22:56:00.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:00 smithi110 ceph-mon[118821]: pgmap v129: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 1.13k op/s 2023-09-23T22:56:01.692 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:01 smithi138 conmon[159173]: 2023-09-23T22:56:01.339+0000 7f2d9e92b200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:56:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:01 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2623869739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:01.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:01 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2623869739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:01.703 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:01 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2623869739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:02.695 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:02 smithi138 conmon[159173]: 2023-09-23T22:56:02.397+0000 7f2d9e92b200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:56:02.696 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:02 smithi138 conmon[159173]: 2023-09-23T22:56:02.543+0000 7f2d9e92b200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:56:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:03 smithi138 ceph-mon[104418]: pgmap v130: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 733 op/s 2023-09-23T22:56:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:03 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:03 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:03 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:03 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/791135714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:03.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:03 smithi110 ceph-mon[100243]: pgmap v130: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 733 op/s 2023-09-23T22:56:03.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:03 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:03.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:03 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:03.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:03 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:03.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:03 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/791135714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:03.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:03 smithi110 ceph-mon[118821]: pgmap v130: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 733 op/s 2023-09-23T22:56:03.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:03 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:03.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:03 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:03.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:03 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:03.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:03 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/791135714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:04.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:03 smithi138 conmon[159173]: 2023-09-23T22:56:03.839+0000 7f2d9e92b200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:56:04.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:03 smithi138 conmon[159173]: 2023-09-23T22:56:03.986+0000 7f2d9e92b200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:56:04.693 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:04 smithi138 conmon[159173]: 2023-09-23T22:56:04.274+0000 7f2d9e92b200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:56:05.084 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:04 smithi138 conmon[159173]: 2023-09-23T22:56:04.743+0000 7f2d9e92b200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:56:05.441 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:05 smithi138 conmon[159173]: 2023-09-23T22:56:05.083+0000 7f2d9e92b200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:56:05.441 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:05 smithi138 conmon[159173]: 2023-09-23T22:56:05.321+0000 7f2d9e92b200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:56:05.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:05 smithi138 ceph-mon[104418]: pgmap v131: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 768 op/s 2023-09-23T22:56:05.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:05 smithi110 ceph-mon[100243]: pgmap v131: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 768 op/s 2023-09-23T22:56:05.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:05 smithi110 ceph-mon[118821]: pgmap v131: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 768 op/s 2023-09-23T22:56:06.196 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:05 smithi138 conmon[159173]: 2023-09-23T22:56:05.773+0000 7f2d9e92b200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:56:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:06 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:06 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:06 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/368283188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:06.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:06 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:06.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:06 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:06.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:06 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/368283188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:06.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:06 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:06.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:06 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:06.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:06 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/368283188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:07.193 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:06 smithi138 conmon[159173]: 2023-09-23T22:56:06.914+0000 7f2d9e92b200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:56:07.194 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:07 smithi138 conmon[159173]: 2023-09-23T22:56:07.132+0000 7f2d9e92b200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:56:07.480 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:07 smithi138 conmon[159173]: 2023-09-23T22:56:07.469+0000 7f2d9e92b200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:56:07.480 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:07 smithi138 ceph-mon[104418]: pgmap v132: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 949 op/s 2023-09-23T22:56:07.480 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:07 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:07.480 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:07 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:07.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:07 smithi110 ceph-mon[118821]: pgmap v132: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 949 op/s 2023-09-23T22:56:07.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:07 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:07.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:07 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:07.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:07 smithi110 ceph-mon[100243]: pgmap v132: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 949 op/s 2023-09-23T22:56:07.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:07 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:07.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:07 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:07.941 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:07 smithi138 conmon[159173]: 2023-09-23T22:56:07.664+0000 7f2d9e92b200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:56:08.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:08 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3996346440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:08.375 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:08 smithi138 conmon[159173]: 2023-09-23T22:56:08.052+0000 7f2d9e92b200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:56:08.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:08 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3996346440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:08.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:08 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3996346440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:09.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:09 smithi138 ceph-mon[104418]: pgmap v133: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-09-23T22:56:09.563 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:09 smithi138 conmon[159173]: 2023-09-23T22:56:09.442+0000 7f2d9e92b200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:56:09.563 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:09 smithi138 conmon[159173]: [23/Sep/2023:22:56:09] ENGINE Bus STARTING 2023-09-23T22:56:09.563 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:09 smithi138 conmon[159173]: CherryPy Checker: 2023-09-23T22:56:09.563 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:09 smithi138 conmon[159173]: The Application mounted at '' has an empty config. 2023-09-23T22:56:09.564 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:09 smithi138 conmon[159173]: 2023-09-23T22:56:09.564 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:09 smithi138 conmon[159173]: [23/Sep/2023:22:56:09] ENGINE Serving on http://:::9283 2023-09-23T22:56:09.564 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:09 smithi138 conmon[159173]: [23/Sep/2023:22:56:09] ENGINE Bus STARTED 2023-09-23T22:56:09.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:09 smithi110 ceph-mon[100243]: pgmap v133: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-09-23T22:56:09.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:09 smithi110 ceph-mon[118821]: pgmap v133: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-09-23T22:56:10.393 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:10 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:56:10] "GET /metrics HTTP/1.1" 200 242499 "" "Prometheus/2.43.0" 2023-09-23T22:56:10.397 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:10 smithi110 ceph-mon[100243]: Standby manager daemon x restarted 2023-09-23T22:56:10.398 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:10 smithi110 ceph-mon[100243]: Standby manager daemon x started 2023-09-23T22:56:10.398 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:10 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:56:10.398 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:10 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:56:10.398 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:10 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:56:10.398 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:10 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:56:10.398 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:10 smithi110 ceph-mon[100243]: pgmap v134: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 970 op/s 2023-09-23T22:56:10.674 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:10 smithi138 ceph-mon[104418]: Standby manager daemon x restarted 2023-09-23T22:56:10.674 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:10 smithi138 ceph-mon[104418]: Standby manager daemon x started 2023-09-23T22:56:10.674 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:10 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:56:10.674 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:10 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:56:10.674 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:10 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:56:10.675 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:10 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:56:10.675 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:10 smithi138 ceph-mon[104418]: pgmap v134: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 970 op/s 2023-09-23T22:56:10.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:10 smithi110 ceph-mon[118821]: Standby manager daemon x restarted 2023-09-23T22:56:10.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:10 smithi110 ceph-mon[118821]: Standby manager daemon x started 2023-09-23T22:56:10.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:10 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:56:10.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:10 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:56:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:10 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:56:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:10 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:56:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:10 smithi110 ceph-mon[118821]: pgmap v134: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 970 op/s 2023-09-23T22:56:11.397 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:11 smithi138 ceph-mon[104418]: mgrmap e20: y(active, since 3m), standbys: x 2023-09-23T22:56:11.397 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:11 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/546647173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:11.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:11 smithi110 ceph-mon[118821]: mgrmap e20: y(active, since 3m), standbys: x 2023-09-23T22:56:11.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:11 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/546647173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:11.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:11 smithi110 ceph-mon[100243]: mgrmap e20: y(active, since 3m), standbys: x 2023-09-23T22:56:11.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:11 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/546647173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:12.199 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:11 smithi138 conmon[159173]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:56:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:56:12.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:12 smithi138 ceph-mon[104418]: pgmap v135: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T22:56:12.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:12 smithi110 ceph-mon[100243]: pgmap v135: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T22:56:12.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:12 smithi110 ceph-mon[118821]: pgmap v135: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T22:56:13.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:13 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1710463334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:13.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:13 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1710463334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:13.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:13 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1710463334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:14.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:14 smithi138 ceph-mon[104418]: pgmap v136: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-09-23T22:56:14.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:14 smithi110 ceph-mon[100243]: pgmap v136: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-09-23T22:56:14.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:14 smithi110 ceph-mon[118821]: pgmap v136: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-09-23T22:56:16.197 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:15 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1923773412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:16.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:15 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1923773412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:16.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:15 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1923773412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:16.621 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:16 smithi110 conmon[100666]: 2023-09-23T22:56:16.322+0000 7f05e4d86700 -1 mgr handle_mgr_map I was active but no longer am 2023-09-23T22:56:16.621 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:16 smithi110 conmon[100666]: ignoring --setuser ceph since I am not root 2023-09-23T22:56:16.621 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:16 smithi110 conmon[100666]: ignoring --setgroup ceph since I am not root 2023-09-23T22:56:16.621 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:16 smithi110 ceph-mgr[100691]: -- 172.21.15.110:0/1943233455 <== mon.2 v2:172.21.15.110:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fc362a29c0 con 0x55fc3627a800 2023-09-23T22:56:16.621 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:16 smithi110 conmon[100666]: 2023-09-23T22:56:16.557+0000 7f9b14a7f1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:56:16.624 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:16 smithi138 conmon[159173]: [23/Sep/2023:22:56:16] ENGINE Bus STOPPING 2023-09-23T22:56:16.624 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:16 smithi138 conmon[159173]: [23/Sep/2023:22:56:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:56:16.624 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:16 smithi138 conmon[159173]: [23/Sep/2023:22:56:16] ENGINE Bus STOPPED 2023-09-23T22:56:16.624 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:16 smithi138 conmon[159173]: [23/Sep/2023:22:56:16] ENGINE Bus STARTING 2023-09-23T22:56:16.941 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:16 smithi110 conmon[100666]: 2023-09-23T22:56:16.670+0000 7f9b14a7f1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:56:16.942 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:16 smithi110 conmon[100666]: 2023-09-23T22:56:16.795+0000 7f9b14a7f1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:56:16.942 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:16 smithi110 conmon[100666]: 2023-09-23T22:56:16.932+0000 7f9b14a7f1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:56:16.943 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:16 smithi138 conmon[159173]: [23/Sep/2023:22:56:16] ENGINE Serving on http://:::9283 2023-09-23T22:56:16.943 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:16 smithi138 conmon[159173]: [23/Sep/2023:22:56:16] ENGINE Bus STARTED 2023-09-23T22:56:16.944 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:16 smithi138 conmon[159173]: 2023-09-23T22:56:16.745+0000 7f2cfbd87700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:56:16.944 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:16 smithi138 conmon[159173]: 2023-09-23T22:56:16.745+0000 7f2cfbd87700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:56:16.944 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:16 smithi138 conmon[159173]: 2023-09-23T22:56:16.745+0000 7f2cfbd87700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:56:16.944 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:16 smithi138 conmon[159173]: 2023-09-23T22:56:16.745+0000 7f2cfbd87700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:56:16.944 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:16 smithi138 conmon[159173]: 2023-09-23T22:56:16.745+0000 7f2cfbd87700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:56:16.944 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:16 smithi138 conmon[159173]: 2023-09-23T22:56:16.761+0000 7f2d01592700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:56:16.944 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:16 smithi138 conmon[159173]: 2023-09-23T22:56:16.761+0000 7f2d01592700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:56:16.944 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:16 smithi138 conmon[159173]: 2023-09-23T22:56:16.761+0000 7f2d01592700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:56:16.944 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:16 smithi138 conmon[159173]: 2023-09-23T22:56:16.761+0000 7f2d01592700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:56:16.944 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:16 smithi138 conmon[159173]: 2023-09-23T22:56:16.761+0000 7f2d01592700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:56:17.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: pgmap v137: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 910 op/s 2023-09-23T22:56:17.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:17.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:17.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:56:17.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:56:17.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:17.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:56:17.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T22:56:17.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:56:17.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T22:56:17.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: mgrmap e21: x(active, starting, since 0.0127521s) 2023-09-23T22:56:17.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:56:17.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:56:17.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:56:17.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.xrewco"}]: dispatch 2023-09-23T22:56:17.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.fuglpi"}]: dispatch 2023-09-23T22:56:17.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:56:17.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:56:17.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:56:17.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:56:17.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:56:17.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:56:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:56:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:56:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:56:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:56:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:56:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:56:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: Manager daemon x is now available 2023-09-23T22:56:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:56:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:56:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:56:17.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:56:17.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:56:17.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:56:17.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: pgmap v137: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 910 op/s 2023-09-23T22:56:17.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:17.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:17.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:56:17.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:56:17.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:17.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:56:17.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T22:56:17.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:56:17.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T22:56:17.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: mgrmap e21: x(active, starting, since 0.0127521s) 2023-09-23T22:56:17.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:56:17.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:56:17.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:56:17.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.xrewco"}]: dispatch 2023-09-23T22:56:17.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.fuglpi"}]: dispatch 2023-09-23T22:56:17.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:56:17.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:56:17.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:56:17.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:56:17.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:56:17.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:56:17.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:56:17.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:56:17.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:56:17.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:56:17.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:56:17.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:56:17.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: Manager daemon x is now available 2023-09-23T22:56:17.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:17.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:56:17.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:56:17.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:17.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:56:17.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:56:17.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:56:17.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:17 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:56:17.206 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:17 smithi110 conmon[100666]: 2023-09-23T22:56:17.197+0000 7f9b14a7f1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:56:17.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: pgmap v137: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 910 op/s 2023-09-23T22:56:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:56:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:56:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' 2023-09-23T22:56:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:56:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T22:56:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:56:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24347 172.21.15.110:0/1127954776' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T22:56:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: mgrmap e21: x(active, starting, since 0.0127521s) 2023-09-23T22:56:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:56:17.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:56:17.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:56:17.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.xrewco"}]: dispatch 2023-09-23T22:56:17.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.fuglpi"}]: dispatch 2023-09-23T22:56:17.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:56:17.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:56:17.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:56:17.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:56:17.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:56:17.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:56:17.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:56:17.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:56:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:56:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:56:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:56:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:56:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: Manager daemon x is now available 2023-09-23T22:56:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:56:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:56:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:56:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:56:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:56:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:17 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:56:17.940 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:17 smithi110 conmon[100666]: 2023-09-23T22:56:17.630+0000 7f9b14a7f1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:56:17.940 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:17 smithi110 conmon[100666]: 2023-09-23T22:56:17.753+0000 7f9b14a7f1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:56:17.940 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:17 smithi110 conmon[100666]: 2023-09-23T22:56:17.937+0000 7f9b14a7f1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:56:18.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:18 smithi110 conmon[100666]: 2023-09-23T22:56:18.088+0000 7f9b14a7f1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:56:18.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:18 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:18.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:18 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:18.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:18 smithi110 ceph-mon[118821]: Deploying cephadm binary to smithi110 2023-09-23T22:56:18.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:18 smithi110 ceph-mon[118821]: Deploying cephadm binary to smithi138 2023-09-23T22:56:18.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:18 smithi110 ceph-mon[118821]: mgrmap e22: x(active, since 1.03363s) 2023-09-23T22:56:18.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:18 smithi110 ceph-mon[118821]: pgmap v3: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T22:56:18.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:18 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:18.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:18 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:18.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:18 smithi110 ceph-mon[100243]: Deploying cephadm binary to smithi110 2023-09-23T22:56:18.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:18 smithi110 ceph-mon[100243]: Deploying cephadm binary to smithi138 2023-09-23T22:56:18.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:18 smithi110 ceph-mon[100243]: mgrmap e22: x(active, since 1.03363s) 2023-09-23T22:56:18.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:18 smithi110 ceph-mon[100243]: pgmap v3: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T22:56:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:18 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:18 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:18 smithi138 ceph-mon[104418]: Deploying cephadm binary to smithi110 2023-09-23T22:56:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:18 smithi138 ceph-mon[104418]: Deploying cephadm binary to smithi138 2023-09-23T22:56:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:18 smithi138 ceph-mon[104418]: mgrmap e22: x(active, since 1.03363s) 2023-09-23T22:56:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:18 smithi138 ceph-mon[104418]: pgmap v3: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T22:56:19.203 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:18 smithi110 conmon[100666]: 2023-09-23T22:56:18.722+0000 7f9b14a7f1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:56:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:19 smithi138 ceph-mon[104418]: [23/Sep/2023:22:56:18] ENGINE Bus STARTING 2023-09-23T22:56:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:19 smithi138 ceph-mon[104418]: pgmap v4: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T22:56:19.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:19 smithi138 ceph-mon[104418]: mgrmap e23: x(active, since 2s) 2023-09-23T22:56:19.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:19 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1544417698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:19.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:19 smithi138 ceph-mon[104418]: [23/Sep/2023:22:56:18] ENGINE Serving on http://172.21.15.138:8765 2023-09-23T22:56:19.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:19 smithi110 ceph-mon[118821]: [23/Sep/2023:22:56:18] ENGINE Bus STARTING 2023-09-23T22:56:19.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:19 smithi110 ceph-mon[118821]: pgmap v4: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T22:56:19.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:19 smithi110 ceph-mon[118821]: mgrmap e23: x(active, since 2s) 2023-09-23T22:56:19.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:19 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1544417698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:19.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:19 smithi110 ceph-mon[118821]: [23/Sep/2023:22:56:18] ENGINE Serving on http://172.21.15.138:8765 2023-09-23T22:56:19.702 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:19 smithi110 conmon[100666]: 2023-09-23T22:56:19.530+0000 7f9b14a7f1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:56:19.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:19 smithi110 ceph-mon[100243]: [23/Sep/2023:22:56:18] ENGINE Bus STARTING 2023-09-23T22:56:19.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:19 smithi110 ceph-mon[100243]: pgmap v4: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T22:56:19.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:19 smithi110 ceph-mon[100243]: mgrmap e23: x(active, since 2s) 2023-09-23T22:56:19.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:19 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1544417698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:19.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:19 smithi110 ceph-mon[100243]: [23/Sep/2023:22:56:18] ENGINE Serving on http://172.21.15.138:8765 2023-09-23T22:56:20.203 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:19 smithi110 conmon[100666]: 2023-09-23T22:56:19.770+0000 7f9b14a7f1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:56:20.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:20 smithi138 ceph-mon[104418]: [23/Sep/2023:22:56:18] ENGINE Serving on https://172.21.15.138:7150 2023-09-23T22:56:20.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:20 smithi138 ceph-mon[104418]: [23/Sep/2023:22:56:18] ENGINE Bus STARTED 2023-09-23T22:56:20.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:20 smithi110 ceph-mon[100243]: [23/Sep/2023:22:56:18] ENGINE Serving on https://172.21.15.138:7150 2023-09-23T22:56:20.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:20 smithi110 ceph-mon[100243]: [23/Sep/2023:22:56:18] ENGINE Bus STARTED 2023-09-23T22:56:20.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:20 smithi110 ceph-mon[118821]: [23/Sep/2023:22:56:18] ENGINE Serving on https://172.21.15.138:7150 2023-09-23T22:56:20.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:20 smithi110 ceph-mon[118821]: [23/Sep/2023:22:56:18] ENGINE Bus STARTED 2023-09-23T22:56:21.404 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:20 smithi110 conmon[100666]: 2023-09-23T22:56:20.973+0000 7f9b14a7f1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:56:21.404 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:21 smithi110 conmon[100666]: 2023-09-23T22:56:21.121+0000 7f9b14a7f1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:56:21.404 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:21 smithi110 conmon[100666]: 2023-09-23T22:56:21.399+0000 7f9b14a7f1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:56:21.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:21 smithi138 ceph-mon[104418]: pgmap v5: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T22:56:21.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:21 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3938461734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:21.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:21 smithi110 ceph-mon[100243]: pgmap v5: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T22:56:21.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:21 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3938461734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:21.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:21 smithi110 ceph-mon[118821]: pgmap v5: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T22:56:21.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:21 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3938461734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:22.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:21 smithi138 conmon[159173]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:56:21] "GET /metrics HTTP/1.1" 200 33296 "" "Prometheus/2.43.0" 2023-09-23T22:56:22.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:21 smithi110 conmon[100666]: 2023-09-23T22:56:21.852+0000 7f9b14a7f1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:56:22.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:22 smithi110 conmon[100666]: 2023-09-23T22:56:22.003+0000 7f9b14a7f1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:56:22.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:22 smithi110 conmon[100666]: 2023-09-23T22:56:22.448+0000 7f9b14a7f1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:56:22.702 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:22 smithi110 conmon[175324]: ts=2023-09-23T22:56:22.328Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi110:8443/api/prometheus_receiver\": dial tcp 172.21.15.110:8443: connect: connection refused" 2023-09-23T22:56:23.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:22 smithi110 conmon[100666]: 2023-09-23T22:56:22.882+0000 7f9b14a7f1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:56:23.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:23 smithi138 ceph-mon[104418]: pgmap v6: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T22:56:23.718 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:23 smithi110 ceph-mon[100243]: pgmap v6: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T22:56:23.719 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:23 smithi110 ceph-mon[118821]: pgmap v6: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T22:56:24.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:23 smithi110 conmon[100666]: 2023-09-23T22:56:23.961+0000 7f9b14a7f1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:56:24.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:24 smithi110 conmon[100666]: 2023-09-23T22:56:24.168+0000 7f9b14a7f1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:56:24.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:24 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2776323847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:24.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:24 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:24.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:24 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:24 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2776323847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:24 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:24 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:24.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:24 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2776323847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:24.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:24 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:24.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:24 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:24.957 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:24 smithi110 conmon[100666]: 2023-09-23T22:56:24.460+0000 7f9b14a7f1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:56:24.957 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:24 smithi110 conmon[100666]: 2023-09-23T22:56:24.642+0000 7f9b14a7f1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:56:25.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:24 smithi110 conmon[100666]: 2023-09-23T22:56:24.992+0000 7f9b14a7f1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:56:25.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:25 smithi138 ceph-mon[104418]: pgmap v7: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 513 op/s 2023-09-23T22:56:25.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:25 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:25.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:25 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:25 smithi110 ceph-mon[118821]: pgmap v7: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 513 op/s 2023-09-23T22:56:25.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:25 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:25.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:25 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:25.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:25 smithi110 ceph-mon[100243]: pgmap v7: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 513 op/s 2023-09-23T22:56:25.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:25 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:25.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:25 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:26.703 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:26 smithi110 conmon[100666]: 2023-09-23T22:56:26.301+0000 7f9b14a7f1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:56:26.703 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:26 smithi110 conmon[100666]: [23/Sep/2023:22:56:26] ENGINE Bus STARTING 2023-09-23T22:56:26.703 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:26 smithi110 conmon[100666]: CherryPy Checker: 2023-09-23T22:56:26.703 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:26 smithi110 conmon[100666]: The Application mounted at '' has an empty config. 2023-09-23T22:56:26.703 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:26 smithi110 conmon[100666]: 2023-09-23T22:56:26.703 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:26 smithi110 conmon[100666]: [23/Sep/2023:22:56:26] ENGINE Serving on http://:::9283 2023-09-23T22:56:26.703 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:26 smithi110 conmon[100666]: [23/Sep/2023:22:56:26] ENGINE Bus STARTED 2023-09-23T22:56:27.069 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:27.069 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:27.069 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.069 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.069 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.069 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.069 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.069 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.069 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.069 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.069 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: Adjusting osd_memory_target on smithi110 to 2500M 2023-09-23T22:56:27.070 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:27.070 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: Standby manager daemon y started 2023-09-23T22:56:27.070 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.110:0/581219531' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:56:27.070 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.110:0/581219531' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:56:27.070 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.110:0/581219531' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:56:27.070 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.110:0/581219531' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:56:27.070 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: pgmap v8: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 398 op/s 2023-09-23T22:56:27.070 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2324184234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:27.070 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:27.070 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:27.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:27.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:27.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.072 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: Adjusting osd_memory_target on smithi110 to 2500M 2023-09-23T22:56:27.072 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:27.072 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: Standby manager daemon y started 2023-09-23T22:56:27.072 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.110:0/581219531' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:56:27.072 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.110:0/581219531' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:56:27.072 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.110:0/581219531' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:56:27.072 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.110:0/581219531' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:56:27.072 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: pgmap v8: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 398 op/s 2023-09-23T22:56:27.072 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2324184234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:27.072 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:27.072 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:26 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:27.073 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:27 smithi110 conmon[175324]: ts=2023-09-23T22:56:27.065Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=5 2023-09-23T22:56:27.197 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:27.198 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:27.198 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.198 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.198 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.198 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.198 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.198 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.198 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.198 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:27.198 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: Adjusting osd_memory_target on smithi110 to 2500M 2023-09-23T22:56:27.199 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:27.199 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: Standby manager daemon y started 2023-09-23T22:56:27.199 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.110:0/581219531' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:56:27.199 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.110:0/581219531' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:56:27.199 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.110:0/581219531' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:56:27.199 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.110:0/581219531' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:56:27.199 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: pgmap v8: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 398 op/s 2023-09-23T22:56:27.199 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2324184234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:27.199 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:27.199 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:26 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:27.962 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:27 smithi138 ceph-mon[104418]: mgrmap e24: x(active, since 10s), standbys: y 2023-09-23T22:56:27.962 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:27 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:56:28.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:27 smithi110 ceph-mon[100243]: mgrmap e24: x(active, since 10s), standbys: y 2023-09-23T22:56:28.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:27 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:56:28.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:27 smithi110 ceph-mon[118821]: mgrmap e24: x(active, since 10s), standbys: y 2023-09-23T22:56:28.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:27 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:56:29.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:28 smithi138 ceph-mon[104418]: pgmap v9: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T22:56:29.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:28 smithi110 ceph-mon[100243]: pgmap v9: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T22:56:29.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:28 smithi110 ceph-mon[118821]: pgmap v9: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T22:56:30.197 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:29 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3530307232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:30.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:30 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:56:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:56:30.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:29 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3530307232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:30.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:29 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3530307232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:30.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:30 smithi138 ceph-mon[104418]: pgmap v10: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-09-23T22:56:31.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:30 smithi110 ceph-mon[100243]: pgmap v10: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-09-23T22:56:31.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:30 smithi110 ceph-mon[118821]: pgmap v10: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-09-23T22:56:32.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:31 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:32.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:31 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/4013194037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:32.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:31 smithi138 conmon[159173]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:56:31] "GET /metrics HTTP/1.1" 200 36232 "" "Prometheus/2.43.0" 2023-09-23T22:56:32.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:31 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:32.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:31 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/4013194037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:32.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:31 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:32.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:31 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/4013194037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:33.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:32 smithi138 ceph-mon[104418]: pgmap v11: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-09-23T22:56:33.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:32 smithi110 ceph-mon[100243]: pgmap v11: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-09-23T22:56:33.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:32 smithi110 ceph-mon[118821]: pgmap v11: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-09-23T22:56:34.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:34 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2998819102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:34.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:34 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2998819102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:34.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:34 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2998819102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:35.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:35 smithi138 ceph-mon[104418]: pgmap v12: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 924 op/s 2023-09-23T22:56:35.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:35 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:35.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:35 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:35.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:35 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:35 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:35 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:35 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:35 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:35 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:35 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[100243]: pgmap v12: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 924 op/s 2023-09-23T22:56:35.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:35.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:35.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[118821]: pgmap v12: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 924 op/s 2023-09-23T22:56:35.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:35.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:35.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:35.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:35 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:36.699 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:36 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:36.699 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:36 smithi138 ceph-mon[104418]: Adjusting osd_memory_target on smithi138 to 2500M 2023-09-23T22:56:36.699 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:36 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:36.699 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:36 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:56:36.699 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:36 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:56:36.699 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:36 smithi138 ceph-mon[104418]: Updating smithi110:/etc/ceph/ceph.conf 2023-09-23T22:56:36.700 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:36 smithi138 ceph-mon[104418]: Updating smithi138:/etc/ceph/ceph.conf 2023-09-23T22:56:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:36 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:36 smithi110 ceph-mon[100243]: Adjusting osd_memory_target on smithi138 to 2500M 2023-09-23T22:56:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:36 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:36 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:56:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:36 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:56:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:36 smithi110 ceph-mon[100243]: Updating smithi110:/etc/ceph/ceph.conf 2023-09-23T22:56:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:36 smithi110 ceph-mon[100243]: Updating smithi138:/etc/ceph/ceph.conf 2023-09-23T22:56:36.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:36 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:56:36.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:36 smithi110 ceph-mon[118821]: Adjusting osd_memory_target on smithi138 to 2500M 2023-09-23T22:56:36.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:36 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:36.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:36 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:56:36.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:36 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:56:36.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:36 smithi110 ceph-mon[118821]: Updating smithi110:/etc/ceph/ceph.conf 2023-09-23T22:56:36.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:36 smithi110 ceph-mon[118821]: Updating smithi138:/etc/ceph/ceph.conf 2023-09-23T22:56:37.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:37 smithi138 ceph-mon[104418]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:56:37.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:37 smithi138 ceph-mon[104418]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:56:37.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:37 smithi138 ceph-mon[104418]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:56:37.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:37 smithi138 ceph-mon[104418]: pgmap v13: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T22:56:37.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:37 smithi138 ceph-mon[104418]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:56:37.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:37 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3061487788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:37.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:37 smithi110 ceph-mon[118821]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:56:37.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:37 smithi110 ceph-mon[118821]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:56:37.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:37 smithi110 ceph-mon[118821]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:56:37.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:37 smithi110 ceph-mon[118821]: pgmap v13: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T22:56:37.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:37 smithi110 ceph-mon[118821]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:56:37.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:37 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3061487788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:37.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:37 smithi110 ceph-mon[100243]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:56:37.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:37 smithi110 ceph-mon[100243]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:56:37.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:37 smithi110 ceph-mon[100243]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:56:37.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:37 smithi110 ceph-mon[100243]: pgmap v13: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T22:56:37.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:37 smithi110 ceph-mon[100243]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:56:37.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:37 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3061487788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:38.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:38 smithi138 ceph-mon[104418]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:56:38.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:38 smithi138 ceph-mon[104418]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:56:38.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:38 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:38 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:38 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:38 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:38 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:38 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:38 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:38 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:38 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[100243]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:56:38.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[100243]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:56:38.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[118821]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:56:38.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[118821]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:56:38.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:38 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:39.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:39 smithi138 ceph-mon[104418]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:56:39.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:39 smithi138 ceph-mon[104418]: Reconfiguring daemon alertmanager.a on smithi110 2023-09-23T22:56:39.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:39 smithi138 ceph-mon[104418]: pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 943 op/s 2023-09-23T22:56:39.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:39 smithi110 ceph-mon[100243]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:56:39.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:39 smithi110 ceph-mon[100243]: Reconfiguring daemon alertmanager.a on smithi110 2023-09-23T22:56:39.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:39 smithi110 ceph-mon[100243]: pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 943 op/s 2023-09-23T22:56:39.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:39 smithi110 ceph-mon[118821]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:56:39.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:39 smithi110 ceph-mon[118821]: Reconfiguring daemon alertmanager.a on smithi110 2023-09-23T22:56:39.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:39 smithi110 ceph-mon[118821]: pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 943 op/s 2023-09-23T22:56:40.459 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:40 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:56:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:56:40.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:40 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2035370787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:40.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:40 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2035370787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:40.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:40 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2035370787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:41.507 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:41 smithi110 ceph-mon[100243]: pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T22:56:41.917 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:41 smithi138 ceph-mon[104418]: pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T22:56:41.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:41 smithi110 ceph-mon[118821]: pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T22:56:42.193 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:41 smithi138 conmon[159173]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:56:41] "GET /metrics HTTP/1.1" 200 36232 "" "Prometheus/2.43.0" 2023-09-23T22:56:42.922 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:42 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1703286374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:42.923 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:42 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1703286374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:42.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:42 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1703286374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:43.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:43 smithi110 ceph-mon[100243]: pgmap v16: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T22:56:43.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:43 smithi110 ceph-mon[118821]: pgmap v16: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T22:56:43.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:43 smithi138 ceph-mon[104418]: pgmap v16: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T22:56:44.454 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:44 smithi110 systemd[1]: Stopping Ceph alertmanager.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:56:44.951 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:44 smithi110 conmon[175324]: ts=2023-09-23T22:56:44.603Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T22:56:44.951 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:44 smithi110 bash[196597]: b148630013bf1353fac8665603dc6f07b49c8c1f7abca5c6e9340e89e4c3d8a5 2023-09-23T22:56:45.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:45 smithi138 ceph-mon[104418]: pgmap v17: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 960 op/s 2023-09-23T22:56:45.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:45 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3110082300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:45.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:45 smithi110 ceph-mon[100243]: pgmap v17: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 960 op/s 2023-09-23T22:56:45.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:45 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3110082300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:45.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:45 smithi110 ceph-mon[118821]: pgmap v17: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 960 op/s 2023-09-23T22:56:45.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:45 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3110082300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:45.953 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:45 smithi110 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T22:56:45.953 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:45 smithi110 systemd[1]: Stopped Ceph alertmanager.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:56:45.954 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:45 smithi110 systemd[1]: Starting Ceph alertmanager.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:56:46.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:46 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:46.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:46 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:46.951 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:46 smithi110 bash[196917]: d9641d6ed9caa90f8e0c45a56c037903f7eb3097b342e7adafb892187052668e 2023-09-23T22:56:46.952 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:46 smithi110 conmon[196953]: ts=2023-09-23T22:56:46.650Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T22:56:46.952 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:46 smithi110 conmon[196953]: ts=2023-09-23T22:56:46.650Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T22:56:46.952 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:46 smithi110 conmon[196953]: ts=2023-09-23T22:56:46.651Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.110 port=9094 2023-09-23T22:56:46.952 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:46 smithi110 conmon[196953]: ts=2023-09-23T22:56:46.655Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T22:56:46.952 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:46 smithi110 systemd[1]: Started Ceph alertmanager.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:56:46.952 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:46 smithi110 conmon[196953]: ts=2023-09-23T22:56:46.709Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T22:56:46.952 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:46 smithi110 conmon[196953]: ts=2023-09-23T22:56:46.710Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T22:56:46.952 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:46 smithi110 conmon[196953]: ts=2023-09-23T22:56:46.728Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T22:56:46.952 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:46 smithi110 conmon[196953]: ts=2023-09-23T22:56:46.728Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T22:56:46.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:46 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:47.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:47 smithi110 ceph-mon[118821]: pgmap v18: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-09-23T22:56:47.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:47 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:47.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:47 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:47.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:47 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1713712676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:47.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:47 smithi110 ceph-mon[100243]: pgmap v18: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-09-23T22:56:47.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:47 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:47.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:47 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:47.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:47 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1713712676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:47.704 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:47 smithi110 systemd[1]: Stopping Ceph node-exporter.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:56:47.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:47 smithi138 ceph-mon[104418]: pgmap v18: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-09-23T22:56:47.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:47 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:47.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:47 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:47.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:47 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1713712676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:48.839 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:48 smithi110 ceph-mon[118821]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T22:56:48.839 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:48 smithi110 ceph-mon[118821]: Reconfiguring daemon node-exporter.a on smithi110 2023-09-23T22:56:48.841 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:48 smithi110 ceph-mon[100243]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T22:56:48.841 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:48 smithi110 ceph-mon[100243]: Reconfiguring daemon node-exporter.a on smithi110 2023-09-23T22:56:48.841 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:48 smithi110 bash[197199]: 9d830b481d84cb61e561254f5a2226b49f7826684400f028a1603d16ffee0673 2023-09-23T22:56:48.841 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:48 smithi110 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:56:48.841 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:48 smithi110 conmon[196953]: ts=2023-09-23T22:56:48.655Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000727096s 2023-09-23T22:56:48.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:48 smithi138 ceph-mon[104418]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T22:56:48.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:48 smithi138 ceph-mon[104418]: Reconfiguring daemon node-exporter.a on smithi110 2023-09-23T22:56:49.453 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:49 smithi110 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T22:56:49.453 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:49 smithi110 systemd[1]: Stopped Ceph node-exporter.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:56:49.453 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:49 smithi110 systemd[1]: Starting Ceph node-exporter.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:56:49.914 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:49 smithi110 ceph-mon[100243]: pgmap v19: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 943 op/s 2023-09-23T22:56:49.914 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:49 smithi110 ceph-mon[118821]: pgmap v19: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 943 op/s 2023-09-23T22:56:49.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:49 smithi138 ceph-mon[104418]: pgmap v19: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 943 op/s 2023-09-23T22:56:50.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[100666]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:56:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:56:50.452 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.053Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T22:56:50.452 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.053Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T22:56:50.452 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.053Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T22:56:50.452 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.053Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T22:56:50.453 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T22:56:50.453 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T22:56:50.453 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T22:56:50.453 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T22:56:50.453 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T22:56:50.453 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T22:56:50.453 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T22:56:50.453 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T22:56:50.453 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T22:56:50.453 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T22:56:50.453 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T22:56:50.453 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T22:56:50.454 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T22:56:50.454 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T22:56:50.454 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T22:56:50.454 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T22:56:50.454 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T22:56:50.454 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T22:56:50.454 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T22:56:50.454 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T22:56:50.454 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T22:56:50.455 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T22:56:50.455 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T22:56:50.455 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T22:56:50.455 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T22:56:50.455 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T22:56:50.455 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T22:56:50.455 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T22:56:50.455 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T22:56:50.455 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T22:56:50.455 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T22:56:50.455 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.054Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T22:56:50.455 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.055Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T22:56:50.456 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.055Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T22:56:50.456 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.055Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T22:56:50.456 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.055Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T22:56:50.456 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.055Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T22:56:50.456 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.055Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T22:56:50.456 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.055Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T22:56:50.456 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.055Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T22:56:50.456 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.055Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T22:56:50.456 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.055Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T22:56:50.456 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.055Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T22:56:50.456 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.055Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T22:56:50.456 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.055Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T22:56:50.457 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.055Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T22:56:50.457 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.055Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T22:56:50.457 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.055Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T22:56:50.457 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 conmon[197563]: ts=2023-09-23T22:56:50.055Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T22:56:50.457 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 bash[197517]: 39d48d803cf2a37b843b8d37945c5e3d2adf26bef9fe2a8a71fcf24c02defe8d 2023-09-23T22:56:50.457 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 22:56:50 smithi110 systemd[1]: Started Ceph node-exporter.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:56:50.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:50 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3669287543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:50.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:50 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:50.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:50 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:50.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:50 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3669287543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:50.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:50 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:50.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:50 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:50.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:50 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3669287543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:50.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:50 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:50.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:50 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:51.601 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:51 smithi138 ceph-mon[104418]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:56:51.601 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:51 smithi138 ceph-mon[104418]: pgmap v20: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T22:56:51.602 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:51 smithi138 ceph-mon[104418]: Reconfiguring daemon prometheus.a on smithi138 2023-09-23T22:56:51.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:51 smithi110 ceph-mon[118821]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:56:51.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:51 smithi110 ceph-mon[118821]: pgmap v20: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T22:56:51.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:51 smithi110 ceph-mon[118821]: Reconfiguring daemon prometheus.a on smithi138 2023-09-23T22:56:51.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:51 smithi110 ceph-mon[100243]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:56:51.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:51 smithi110 ceph-mon[100243]: pgmap v20: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T22:56:51.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:51 smithi110 ceph-mon[100243]: Reconfiguring daemon prometheus.a on smithi138 2023-09-23T22:56:52.193 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:56:51 smithi138 conmon[159173]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:56:51] "GET /metrics HTTP/1.1" 200 36232 "" "Prometheus/2.43.0" 2023-09-23T22:56:53.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:53 smithi138 ceph-mon[104418]: pgmap v21: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T22:56:53.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:53 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2709669552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:53.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:53 smithi110 ceph-mon[100243]: pgmap v21: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T22:56:53.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:53 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2709669552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:53.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:53 smithi110 ceph-mon[118821]: pgmap v21: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T22:56:53.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:53 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2709669552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:55.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:55 smithi138 ceph-mon[104418]: pgmap v22: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 945 op/s 2023-09-23T22:56:55.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:55 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3086897247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:55.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:55 smithi110 ceph-mon[100243]: pgmap v22: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 945 op/s 2023-09-23T22:56:55.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:55 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3086897247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:55.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:55 smithi110 ceph-mon[118821]: pgmap v22: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 945 op/s 2023-09-23T22:56:55.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:55 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3086897247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:56.191 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:55 smithi138 systemd[1]: Stopping Ceph prometheus.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:56:56.810 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:56 smithi138 conmon[153898]: ts=2023-09-23T22:56:56.449Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T22:56:56.810 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:56 smithi138 conmon[153898]: ts=2023-09-23T22:56:56.449Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T22:56:56.810 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:56 smithi138 conmon[153898]: ts=2023-09-23T22:56:56.449Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T22:56:56.810 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:56 smithi138 conmon[153898]: ts=2023-09-23T22:56:56.449Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T22:56:56.810 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:56 smithi138 conmon[153898]: ts=2023-09-23T22:56:56.449Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T22:56:56.811 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:56 smithi138 conmon[153898]: ts=2023-09-23T22:56:56.449Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T22:56:56.811 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:56 smithi138 conmon[153898]: ts=2023-09-23T22:56:56.449Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T22:56:56.811 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:56 smithi138 conmon[153898]: ts=2023-09-23T22:56:56.449Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T22:56:56.811 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:56 smithi138 conmon[153898]: ts=2023-09-23T22:56:56.450Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T22:56:56.811 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:56 smithi138 conmon[153898]: ts=2023-09-23T22:56:56.450Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T22:56:56.811 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:56 smithi138 conmon[153898]: ts=2023-09-23T22:56:56.450Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T22:56:56.811 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:56 smithi138 conmon[153898]: ts=2023-09-23T22:56:56.450Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T22:56:56.811 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:56 smithi138 bash[171478]: 685848669b55d0cac80f930f30221ba0f5d2c60b26ff81b83b006d5ec64a07b0 2023-09-23T22:56:56.951 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 22:56:56 smithi110 conmon[196953]: ts=2023-09-23T22:56:56.658Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003027884s 2023-09-23T22:56:57.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:57 smithi138 ceph-mon[104418]: pgmap v23: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-09-23T22:56:57.944 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:57 smithi138 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T22:56:57.944 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:57 smithi138 systemd[1]: Stopped Ceph prometheus.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:56:57.944 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:57 smithi138 systemd[1]: Starting Ceph prometheus.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:56:57.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:57 smithi110 ceph-mon[100243]: pgmap v23: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-09-23T22:56:57.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:57 smithi110 ceph-mon[118821]: pgmap v23: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-09-23T22:56:58.691 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 bash[171789]: 41d979ed4d4fd4829e3650a54c6cab30dd0abd83f9db53ab98e639a22f8dd6b6 2023-09-23T22:56:58.691 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 systemd[1]: Started Ceph prometheus.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:56:58.691 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.439Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T22:56:58.692 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.439Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T22:56:58.692 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.440Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi138 (none))" 2023-09-23T22:56:58.692 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.440Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T22:56:58.692 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.440Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T22:56:58.692 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.442Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T22:56:58.692 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.443Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T22:56:58.692 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.444Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T22:56:58.692 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.444Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T22:56:58.692 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.460Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T22:56:58.692 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.460Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.806µs 2023-09-23T22:56:58.692 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.460Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T22:56:58.692 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.460Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-09-23T22:56:58.693 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.495Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-09-23T22:56:58.693 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.495Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-09-23T22:56:58.693 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.495Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.963µs wal_replay_duration=34.868622ms wbl_replay_duration=138ns total_replay_duration=34.915889ms 2023-09-23T22:56:58.693 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.498Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T22:56:58.693 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.498Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T22:56:58.693 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.498Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T22:56:58.693 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.531Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=33.194444ms db_storage=1.541µs remote_storage=1.561µs web_handler=497ns query_engine=881ns scrape=505.397µs scrape_sd=104.186µs notify=18.612µs notify_sd=12.857µs rules=32.244162ms tracing=6.591µs 2023-09-23T22:56:58.693 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.531Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T22:56:58.693 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:56:58 smithi138 conmon[171826]: ts=2023-09-23T22:56:58.531Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T22:56:58.698 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:58 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3102641273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:58.699 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:58 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:58.699 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:58 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:58.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:58 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3102641273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:58.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:58 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:58.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:58 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:58.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:58 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3102641273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:56:58.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:58 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:58.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:58 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:56:59.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:59 smithi138 ceph-mon[104418]: pgmap v24: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 924 op/s 2023-09-23T22:56:59.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:59 smithi138 ceph-mon[104418]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T22:56:59.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:56:59 smithi138 ceph-mon[104418]: Reconfiguring daemon grafana.a on smithi138 2023-09-23T22:56:59.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:59 smithi110 ceph-mon[100243]: pgmap v24: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 924 op/s 2023-09-23T22:56:59.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:59 smithi110 ceph-mon[100243]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T22:56:59.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:56:59 smithi110 ceph-mon[100243]: Reconfiguring daemon grafana.a on smithi138 2023-09-23T22:56:59.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:59 smithi110 ceph-mon[118821]: pgmap v24: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 924 op/s 2023-09-23T22:56:59.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:59 smithi110 ceph-mon[118821]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T22:56:59.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:56:59 smithi110 ceph-mon[118821]: Reconfiguring daemon grafana.a on smithi138 2023-09-23T22:57:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:00 smithi138 ceph-mon[104418]: pgmap v25: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T22:57:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:00 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1118068136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:00.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:00 smithi110 ceph-mon[118821]: pgmap v25: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T22:57:00.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:00 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1118068136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:00.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:00 smithi110 ceph-mon[100243]: pgmap v25: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T22:57:00.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:00 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1118068136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:01.654 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:01 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:01.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:01 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:01.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:01 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:02.671 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:02 smithi138 ceph-mon[104418]: pgmap v26: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T22:57:02.672 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:02 smithi138 systemd[1]: Stopping Ceph grafana.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:57:02.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:02 smithi110 ceph-mon[100243]: pgmap v26: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T22:57:02.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:02 smithi110 ceph-mon[118821]: pgmap v26: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T22:57:03.444 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:03 smithi138 bash[172302]: e7b0f17c56818bb7d07e9d5f51916ba0c0ae4489236e9e042590e19cacf8dc9f 2023-09-23T22:57:03.938 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:03 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2386849795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:03.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:03 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2386849795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:03.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:03 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2386849795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:04.447 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:03 smithi138 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:57:04.447 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:03 smithi138 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-23T22:57:04.447 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:03 smithi138 systemd[1]: Stopped Ceph grafana.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:57:04.447 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:03 smithi138 systemd[1]: Starting Ceph grafana.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:57:05.324 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 bash[172610]: 56929ec304ecde569332f0740e5cc8b5e9d3da0e3da108c31c5352f11dfd8dfe 2023-09-23T22:57:05.325 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 systemd[1]: Started Ceph grafana.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:57:05.325 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=settings t=2023-09-23T22:57:05.323388301Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-23T22:57:05.325 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=settings t=2023-09-23T22:57:05.323687388Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-23T22:57:05.325 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=settings t=2023-09-23T22:57:05.323717305Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-23T22:57:05.325 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=settings t=2023-09-23T22:57:05.323732357Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-23T22:57:05.325 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=settings t=2023-09-23T22:57:05.323744798Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T22:57:05.325 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=settings t=2023-09-23T22:57:05.323756492Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T22:57:05.325 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=settings t=2023-09-23T22:57:05.323768836Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T22:57:05.325 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=settings t=2023-09-23T22:57:05.323787788Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T22:57:05.325 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=settings t=2023-09-23T22:57:05.323799373Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-23T22:57:05.325 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=settings t=2023-09-23T22:57:05.323811268Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-23T22:57:05.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:05 smithi138 ceph-mon[104418]: pgmap v27: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2023-09-23T22:57:05.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:05 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:05.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:05 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:05.692 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=settings t=2023-09-23T22:57:05.323822818Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-23T22:57:05.693 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=settings t=2023-09-23T22:57:05.323834761Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-23T22:57:05.693 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=settings t=2023-09-23T22:57:05.323845841Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-23T22:57:05.693 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=settings t=2023-09-23T22:57:05.323856595Z level=info msg="App mode production" 2023-09-23T22:57:05.693 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=sqlstore t=2023-09-23T22:57:05.32393384Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-23T22:57:05.693 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=sqlstore t=2023-09-23T22:57:05.323960082Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T22:57:05.693 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=migrator t=2023-09-23T22:57:05.346569684Z level=info msg="Starting DB migrations" 2023-09-23T22:57:05.693 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=migrator t=2023-09-23T22:57:05.352645116Z level=info msg="migrations completed" performed=0 skipped=481 duration=738.167µs 2023-09-23T22:57:05.693 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=sqlstore t=2023-09-23T22:57:05.353260303Z level=info msg="Created default organization" 2023-09-23T22:57:05.693 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=plugin.loader t=2023-09-23T22:57:05.459666856Z level=info msg="Plugin registered" pluginID=input 2023-09-23T22:57:05.693 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=plugin.loader t=2023-09-23T22:57:05.504629211Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-23T22:57:05.693 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=plugin.loader t=2023-09-23T22:57:05.504684859Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-23T22:57:05.694 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=secrets t=2023-09-23T22:57:05.50474563Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-23T22:57:05.694 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=query_data t=2023-09-23T22:57:05.507532636Z level=info msg="Query Service initialization" 2023-09-23T22:57:05.694 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[172649]: logger=live.push_http t=2023-09-23T22:57:05.511309413Z level=info msg="Live Push Gateway initialization" 2023-09-23T22:57:05.694 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:05 smithi138 conmon[159173]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:57:05] "GET /metrics HTTP/1.1" 200 36227 "" "Prometheus/2.43.0" 2023-09-23T22:57:05.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:05 smithi110 ceph-mon[118821]: pgmap v27: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2023-09-23T22:57:05.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:05 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:05.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:05 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:05.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:05 smithi110 ceph-mon[100243]: pgmap v27: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2023-09-23T22:57:05.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:05 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:05.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:05 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:06.193 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:05 smithi138 systemd[1]: Stopping Ceph node-exporter.b for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:57:06.691 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:06 smithi138 bash[172867]: 68cfd3f377d7fbf19f8732cd5fc5cf1f6d25743ca7dee624bd5c2494cfb0d89e 2023-09-23T22:57:06.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:06 smithi138 ceph-mon[104418]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T22:57:06.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:06 smithi138 ceph-mon[104418]: Reconfiguring daemon node-exporter.b on smithi138 2023-09-23T22:57:06.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:06 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3057248164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:06.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:06 smithi110 ceph-mon[100243]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T22:57:06.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:06 smithi110 ceph-mon[100243]: Reconfiguring daemon node-exporter.b on smithi138 2023-09-23T22:57:06.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:06 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3057248164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:06.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:06 smithi110 ceph-mon[118821]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T22:57:06.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:06 smithi110 ceph-mon[118821]: Reconfiguring daemon node-exporter.b on smithi138 2023-09-23T22:57:06.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:06 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3057248164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:07.193 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:06 smithi138 conmon[172649]: logger=infra.usagestats.collector t=2023-09-23T22:57:06.773574081Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-23T22:57:07.693 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:07 smithi138 conmon[172649]: logger=provisioning.datasources t=2023-09-23T22:57:07.437977682Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2023-09-23T22:57:07.693 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:07 smithi138 conmon[172649]: logger=provisioning.datasources t=2023-09-23T22:57:07.438031566Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2023-09-23T22:57:07.693 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:07 smithi138 conmon[172649]: logger=provisioning.datasources t=2023-09-23T22:57:07.449232081Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T22:57:07.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:07 smithi138 ceph-mon[104418]: pgmap v28: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T22:57:07.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:07 smithi110 ceph-mon[100243]: pgmap v28: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T22:57:07.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:07 smithi110 ceph-mon[118821]: pgmap v28: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T22:57:08.197 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:07 smithi138 conmon[172649]: logger=provisioning.alerting t=2023-09-23T22:57:07.932960219Z level=info msg="starting to provision alerting" 2023-09-23T22:57:08.197 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:07 smithi138 conmon[172649]: logger=provisioning.alerting t=2023-09-23T22:57:07.933004545Z level=info msg="finished to provision alerting" 2023-09-23T22:57:08.197 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:07 smithi138 conmon[172649]: logger=grafanaStorageLogger t=2023-09-23T22:57:07.934346871Z level=info msg="storage starting" 2023-09-23T22:57:08.197 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:07 smithi138 conmon[172649]: logger=ngalert.state.manager t=2023-09-23T22:57:07.934844403Z level=info msg="Warming state cache for startup" 2023-09-23T22:57:08.197 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:07 smithi138 conmon[172649]: logger=http.server t=2023-09-23T22:57:07.935742029Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T22:57:08.197 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:07 smithi138 conmon[172649]: logger=ngalert.state.manager t=2023-09-23T22:57:07.994232475Z level=info msg="State cache has been initialized" states=0 duration=59.38431ms 2023-09-23T22:57:08.197 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:07 smithi138 conmon[172649]: logger=ticker t=2023-09-23T22:57:07.994332921Z level=info msg=starting first_tick=2023-09-23T22:57:10Z 2023-09-23T22:57:08.197 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 22:57:07 smithi138 conmon[172649]: logger=ngalert.multiorg.alertmanager t=2023-09-23T22:57:07.994365517Z level=info msg="starting MultiOrg Alertmanager" 2023-09-23T22:57:08.198 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:07 smithi138 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:57:08.198 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:07 smithi138 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T22:57:08.198 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:07 smithi138 systemd[1]: Stopped Ceph node-exporter.b for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:57:08.198 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:07 smithi138 systemd[1]: Starting Ceph node-exporter.b for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:57:08.691 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[171826]: ts=2023-09-23T22:57:08.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:57:08.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:08 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2119219169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:08.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:08 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2119219169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:08.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:08 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2119219169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:09.311 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE Bus STOPPING 2023-09-23T22:57:09.311 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:57:09.311 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE Bus STOPPED 2023-09-23T22:57:09.311 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE Bus STARTING 2023-09-23T22:57:09.311 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE Serving on http://:::9283 2023-09-23T22:57:09.311 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE Bus STARTED 2023-09-23T22:57:09.311 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE Bus STOPPING 2023-09-23T22:57:09.312 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.997Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T22:57:09.312 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.997Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T22:57:09.312 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.998Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T22:57:09.312 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.998Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T22:57:09.312 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.998Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T22:57:09.312 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.998Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T22:57:09.313 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.998Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T22:57:09.313 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.998Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T22:57:09.313 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.998Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T22:57:09.313 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.998Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T22:57:09.313 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.998Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T22:57:09.313 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.998Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T22:57:09.313 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.998Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T22:57:09.313 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.998Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T22:57:09.313 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.998Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T22:57:09.313 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T22:57:09.314 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T22:57:09.314 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T22:57:09.314 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T22:57:09.314 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T22:57:09.314 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T22:57:09.314 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T22:57:09.314 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T22:57:09.314 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T22:57:09.314 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T22:57:09.314 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T22:57:09.315 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T22:57:09.315 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T22:57:09.315 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T22:57:09.315 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T22:57:09.315 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T22:57:09.315 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T22:57:09.315 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T22:57:09.315 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T22:57:09.315 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T22:57:09.315 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T22:57:09.315 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T22:57:09.316 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T22:57:09.316 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T22:57:09.316 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T22:57:09.316 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T22:57:09.316 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T22:57:09.316 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T22:57:09.316 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T22:57:09.316 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T22:57:09.316 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T22:57:09.316 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T22:57:09.316 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T22:57:09.316 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T22:57:09.317 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T22:57:09.317 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T22:57:09.317 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T22:57:09.317 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:08 smithi138 conmon[173210]: ts=2023-09-23T22:57:08.999Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T22:57:09.317 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:09 smithi138 bash[173172]: a55c5d37ba4b2f0a4a0560771c481bd95e589f34b0092c51849a47564ce3f258 2023-09-23T22:57:09.317 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 22:57:09 smithi138 systemd[1]: Started Ceph node-exporter.b for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:57:09.570 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:57:09.570 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE Bus STOPPED 2023-09-23T22:57:09.570 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE Bus STARTING 2023-09-23T22:57:09.570 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE Serving on http://:::9283 2023-09-23T22:57:09.570 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE Bus STARTED 2023-09-23T22:57:09.570 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE Bus STOPPING 2023-09-23T22:57:09.570 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:57:09.571 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE Bus STOPPED 2023-09-23T22:57:09.571 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE Bus STARTING 2023-09-23T22:57:09.571 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE Serving on http://:::9283 2023-09-23T22:57:09.571 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:09 smithi138 conmon[159173]: [23/Sep/2023:22:57:09] ENGINE Bus STARTED 2023-09-23T22:57:09.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:09 smithi138 ceph-mon[104418]: pgmap v29: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-09-23T22:57:09.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:09 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:09.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:09 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:09.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:09 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:57:09.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:09 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:57:09.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:09 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:09.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:09 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:57:09.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:09 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138:9095"}]: dispatch 2023-09-23T22:57:09.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:09 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:09.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:09 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:57:09.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:09 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138:3000"}]: dispatch 2023-09-23T22:57:09.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:09 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:09.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:09 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:09.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[100243]: pgmap v29: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-09-23T22:57:09.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:09.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:09.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:57:09.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:57:09.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:09.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:57:09.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138:9095"}]: dispatch 2023-09-23T22:57:09.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:09.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:57:09.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138:3000"}]: dispatch 2023-09-23T22:57:09.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:09.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:09.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[118821]: pgmap v29: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-09-23T22:57:09.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:09.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:09.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:57:09.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:57:09.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:09.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:57:09.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138:9095"}]: dispatch 2023-09-23T22:57:09.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:09.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:57:09.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138:3000"}]: dispatch 2023-09-23T22:57:09.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:09.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:09 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:10.729 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:10 smithi110 ceph-mon[100243]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:57:10.729 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:10 smithi110 ceph-mon[100243]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:57:10.729 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:10 smithi110 ceph-mon[100243]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:57:10.729 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:10 smithi110 ceph-mon[100243]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138:9095"}]: dispatch 2023-09-23T22:57:10.729 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:10 smithi110 ceph-mon[100243]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:57:10.729 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:10 smithi110 ceph-mon[100243]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138:3000"}]: dispatch 2023-09-23T22:57:10.729 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:10 smithi110 ceph-mon[100243]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:57:10.729 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:10 smithi110 ceph-mon[100243]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:57:10.730 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:10 smithi110 ceph-mon[118821]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:57:10.730 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:10 smithi110 ceph-mon[118821]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:57:10.730 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:10 smithi110 ceph-mon[118821]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:57:10.730 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:10 smithi110 ceph-mon[118821]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138:9095"}]: dispatch 2023-09-23T22:57:10.730 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:10 smithi110 ceph-mon[118821]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:57:10.730 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:10 smithi110 ceph-mon[118821]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138:3000"}]: dispatch 2023-09-23T22:57:10.730 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:10 smithi110 ceph-mon[118821]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:57:10.730 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:10 smithi110 ceph-mon[118821]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:57:10.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:10 smithi138 ceph-mon[104418]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:57:10.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:10 smithi138 ceph-mon[104418]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi110.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:57:10.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:10 smithi138 ceph-mon[104418]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:57:10.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:10 smithi138 ceph-mon[104418]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138:9095"}]: dispatch 2023-09-23T22:57:10.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:10 smithi138 ceph-mon[104418]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:57:10.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:10 smithi138 ceph-mon[104418]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138:3000"}]: dispatch 2023-09-23T22:57:10.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:10 smithi138 ceph-mon[104418]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:57:10.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:10 smithi138 ceph-mon[104418]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:57:11.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:11 smithi138 ceph-mon[104418]: pgmap v30: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T22:57:11.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:11 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2930761608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:11.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:11 smithi110 ceph-mon[100243]: pgmap v30: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T22:57:11.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:11 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2930761608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:11.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:11 smithi110 ceph-mon[118821]: pgmap v30: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T22:57:11.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:11 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2930761608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:13.784 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:13 smithi110 ceph-mon[118821]: pgmap v31: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T22:57:13.784 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:13 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3432610341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:13.789 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:13 smithi110 ceph-mon[100243]: pgmap v31: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T22:57:13.789 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:13 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3432610341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:13.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:13 smithi138 ceph-mon[104418]: pgmap v31: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T22:57:13.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:13 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3432610341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:14.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:14 smithi110 ceph-mon[100243]: Upgrade: Updating mgr.y 2023-09-23T22:57:14.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:14 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:14.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:14 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:57:14.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:14 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:57:14.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:14 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:57:14.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:14 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:57:14.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:14 smithi110 ceph-mon[100243]: Deploying daemon mgr.y on smithi110 2023-09-23T22:57:14.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:14 smithi110 ceph-mon[100243]: pgmap v32: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2023-09-23T22:57:14.871 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:14 smithi110 ceph-mon[118821]: Upgrade: Updating mgr.y 2023-09-23T22:57:14.872 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:14 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:14.872 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:14 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:57:14.872 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:14 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:57:14.872 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:14 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:57:14.872 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:14 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:57:14.872 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:14 smithi110 ceph-mon[118821]: Deploying daemon mgr.y on smithi110 2023-09-23T22:57:14.872 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:14 smithi110 ceph-mon[118821]: pgmap v32: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2023-09-23T22:57:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:14 smithi138 ceph-mon[104418]: Upgrade: Updating mgr.y 2023-09-23T22:57:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:14 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:14 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:57:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:14 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:57:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:14 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:57:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:14 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:57:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:14 smithi138 ceph-mon[104418]: Deploying daemon mgr.y on smithi110 2023-09-23T22:57:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:14 smithi138 ceph-mon[104418]: pgmap v32: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2023-09-23T22:57:15.691 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:15 smithi138 conmon[159173]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:57:15] "GET /metrics HTTP/1.1" 200 36227 "" "Prometheus/2.43.0" 2023-09-23T22:57:16.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:15 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3209591477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:16.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:15 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3209591477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:16.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:15 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3209591477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:16.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:16 smithi110 systemd[1]: Stopping Ceph mgr.y for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:57:17.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:17 smithi110 ceph-mon[100243]: pgmap v33: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T22:57:17.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:17 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:17.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:17 smithi110 ceph-mon[118821]: pgmap v33: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T22:57:17.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:17 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:17.204 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:16 smithi110 bash[198413]: 906aab6b0363265390686c1b9fd3a07c494d97774e487c71bf99f1449c54ef9f 2023-09-23T22:57:17.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:17 smithi138 ceph-mon[104418]: pgmap v33: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T22:57:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:17 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:18.204 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:17 smithi110 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:57:18.204 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:17 smithi110 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-09-23T22:57:18.204 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:17 smithi110 systemd[1]: Stopped Ceph mgr.y for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:57:18.204 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:18 smithi110 systemd[1]: Starting Ceph mgr.y for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:57:18.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:18 smithi138 conmon[171826]: ts=2023-09-23T22:57:18.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:57:19.335 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:19 smithi110 bash[198773]: 09d917770c6ea1e5a466809db90aba1012e45fff8755754420a3c040194b9d2d 2023-09-23T22:57:19.336 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:19 smithi110 systemd[1]: Started Ceph mgr.y for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:57:19.336 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:19 smithi110 conmon[198813]: 2023-09-23T22:57:19.211+0000 7f2d328a0200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:57:19.628 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:19 smithi110 conmon[198813]: 2023-09-23T22:57:19.334+0000 7f2d328a0200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:57:19.628 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:19 smithi110 conmon[198813]: 2023-09-23T22:57:19.476+0000 7f2d328a0200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:57:19.628 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:19 smithi110 conmon[198813]: 2023-09-23T22:57:19.625+0000 7f2d328a0200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:57:19.628 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:19 smithi110 ceph-mon[118821]: pgmap v34: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2023-09-23T22:57:19.628 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:19 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/887163132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:19.628 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:19 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:19.628 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:19 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:19.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:19 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:19.629 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:19 smithi110 ceph-mon[100243]: pgmap v34: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2023-09-23T22:57:19.629 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:19 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/887163132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:19.629 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:19 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:19.629 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:19 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:19.629 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:19 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:19.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:19 smithi138 ceph-mon[104418]: pgmap v34: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2023-09-23T22:57:19.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:19 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/887163132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:19.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:19 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:19.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:19 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:19.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:19 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:20.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:19 smithi110 conmon[198813]: 2023-09-23T22:57:19.914+0000 7f2d328a0200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:57:20.878 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:20 smithi110 conmon[198813]: 2023-09-23T22:57:20.512+0000 7f2d328a0200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:57:20.878 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:20 smithi110 conmon[198813]: 2023-09-23T22:57:20.712+0000 7f2d328a0200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:57:20.878 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:20 smithi110 conmon[198813]: 2023-09-23T22:57:20.862+0000 7f2d328a0200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:57:21.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:21 smithi110 conmon[198813]: 2023-09-23T22:57:21.036+0000 7f2d328a0200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:57:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:21 smithi110 ceph-mon[100243]: pgmap v35: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T22:57:22.152 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:21 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2040781854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:22.153 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:21 smithi138 ceph-mon[104418]: pgmap v35: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T22:57:22.153 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:21 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2040781854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:22.154 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:21 smithi110 ceph-mon[118821]: pgmap v35: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T22:57:22.154 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:21 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2040781854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:22.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:21 smithi110 conmon[198813]: 2023-09-23T22:57:21.712+0000 7f2d328a0200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:57:23.204 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:22 smithi110 conmon[198813]: 2023-09-23T22:57:22.731+0000 7f2d328a0200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:57:23.204 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:22 smithi110 conmon[198813]: 2023-09-23T22:57:22.853+0000 7f2d328a0200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:57:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:23 smithi138 ceph-mon[104418]: pgmap v36: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T22:57:23.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:23 smithi110 ceph-mon[100243]: pgmap v36: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T22:57:23.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:23 smithi110 ceph-mon[118821]: pgmap v36: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-09-23T22:57:24.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:24 smithi110 conmon[198813]: 2023-09-23T22:57:24.101+0000 7f2d328a0200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:57:24.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:24 smithi110 conmon[198813]: 2023-09-23T22:57:24.249+0000 7f2d328a0200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:57:24.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:24 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2770062488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:24.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:24 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:24.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:24 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:24.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:24 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:24.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:24 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:24.957 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:24 smithi110 conmon[198813]: 2023-09-23T22:57:24.513+0000 7f2d328a0200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:57:24.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:24 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2770062488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:24.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:24 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:24.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:24 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:24.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:24 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:24.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:24 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:24.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:24 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2770062488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:24.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:24 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:24.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:24 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:24.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:24 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:24.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:24 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:25.353 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:25 smithi110 conmon[198813]: 2023-09-23T22:57:24.998+0000 7f2d328a0200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:57:25.353 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:25 smithi110 conmon[198813]: 2023-09-23T22:57:25.350+0000 7f2d328a0200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:57:25.603 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:25 smithi110 ceph-mon[100243]: pgmap v37: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2023-09-23T22:57:25.604 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:25 smithi110 ceph-mon[118821]: pgmap v37: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2023-09-23T22:57:25.691 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:25 smithi138 conmon[159173]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:57:25] "GET /metrics HTTP/1.1" 200 36231 "" "Prometheus/2.43.0" 2023-09-23T22:57:25.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:25 smithi138 ceph-mon[104418]: pgmap v37: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2023-09-23T22:57:25.955 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:25 smithi110 conmon[198813]: 2023-09-23T22:57:25.601+0000 7f2d328a0200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:57:26.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:26 smithi110 conmon[198813]: 2023-09-23T22:57:26.030+0000 7f2d328a0200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:57:26.900 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:26 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:26.900 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:26 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:26.901 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:26 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:26.901 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:26 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:26.940 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:26 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:26.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:26 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:27.639 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:27 smithi110 conmon[198813]: 2023-09-23T22:57:27.202+0000 7f2d328a0200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:57:27.640 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:27 smithi110 conmon[198813]: 2023-09-23T22:57:27.415+0000 7f2d328a0200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:57:27.640 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:27 smithi110 ceph-mon[100243]: pgmap v38: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-09-23T22:57:27.640 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:27 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/131789537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:27.911 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:27 smithi110 conmon[198813]: 2023-09-23T22:57:27.730+0000 7f2d328a0200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:57:27.911 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:27 smithi110 ceph-mon[118821]: pgmap v38: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-09-23T22:57:27.911 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:27 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/131789537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:27.949 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:27 smithi138 ceph-mon[104418]: pgmap v38: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-09-23T22:57:27.949 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:27 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/131789537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:28.702 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:27 smithi110 conmon[198813]: 2023-09-23T22:57:27.910+0000 7f2d328a0200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:57:28.702 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:28 smithi110 conmon[198813]: 2023-09-23T22:57:28.268+0000 7f2d328a0200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:57:28.947 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:28 smithi138 conmon[171826]: ts=2023-09-23T22:57:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:57:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:29 smithi138 ceph-mon[104418]: pgmap v39: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2023-09-23T22:57:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:29 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3162436051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:29 smithi138 ceph-mon[104418]: Standby manager daemon y restarted 2023-09-23T22:57:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:29 smithi138 ceph-mon[104418]: Standby manager daemon y started 2023-09-23T22:57:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:29 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:57:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:29 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:57:29.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:29 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:57:29.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:29 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:57:29.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:29 smithi110 ceph-mon[100243]: pgmap v39: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2023-09-23T22:57:29.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:29 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3162436051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:29.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:29 smithi110 ceph-mon[100243]: Standby manager daemon y restarted 2023-09-23T22:57:29.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:29 smithi110 ceph-mon[100243]: Standby manager daemon y started 2023-09-23T22:57:29.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:29 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:57:29.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:29 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:57:29.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:29 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:57:29.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:29 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:57:29.958 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:29 smithi110 conmon[198813]: 2023-09-23T22:57:29.563+0000 7f2d328a0200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:57:29.958 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:29 smithi110 conmon[198813]: [23/Sep/2023:22:57:29] ENGINE Bus STARTING 2023-09-23T22:57:29.958 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:29 smithi110 conmon[198813]: CherryPy Checker: 2023-09-23T22:57:29.958 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:29 smithi110 conmon[198813]: The Application mounted at '' has an empty config. 2023-09-23T22:57:29.958 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:29 smithi110 conmon[198813]: 2023-09-23T22:57:29.958 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:29 smithi110 conmon[198813]: [23/Sep/2023:22:57:29] ENGINE Serving on http://:::9283 2023-09-23T22:57:29.958 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:29 smithi110 conmon[198813]: [23/Sep/2023:22:57:29] ENGINE Bus STARTED 2023-09-23T22:57:29.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:29 smithi110 ceph-mon[118821]: pgmap v39: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2023-09-23T22:57:29.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:29 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3162436051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:29.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:29 smithi110 ceph-mon[118821]: Standby manager daemon y restarted 2023-09-23T22:57:29.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:29 smithi110 ceph-mon[118821]: Standby manager daemon y started 2023-09-23T22:57:29.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:29 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:57:29.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:29 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:57:29.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:29 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:57:29.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:29 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:57:31.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:31 smithi138 ceph-mon[104418]: pgmap v40: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-09-23T22:57:31.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:31 smithi138 ceph-mon[104418]: mgrmap e25: x(active, since 74s), standbys: y 2023-09-23T22:57:31.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:31 smithi110 ceph-mon[100243]: pgmap v40: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-09-23T22:57:31.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:31 smithi110 ceph-mon[100243]: mgrmap e25: x(active, since 74s), standbys: y 2023-09-23T22:57:31.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:31 smithi110 ceph-mon[118821]: pgmap v40: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-09-23T22:57:31.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:31 smithi110 ceph-mon[118821]: mgrmap e25: x(active, since 74s), standbys: y 2023-09-23T22:57:32.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:32 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:32.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:32 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/885588991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:32.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:32 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:32.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:32 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/885588991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:32.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:32 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:32.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:32 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/885588991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:33.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:33 smithi138 ceph-mon[104418]: pgmap v41: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-09-23T22:57:33.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:33 smithi110 ceph-mon[100243]: pgmap v41: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-09-23T22:57:33.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:33 smithi110 ceph-mon[118821]: pgmap v41: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-09-23T22:57:34.278 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:33 smithi110 conmon[198813]: [23/Sep/2023:22:57:33] ENGINE Bus STOPPING 2023-09-23T22:57:34.278 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:33 smithi110 conmon[198813]: [23/Sep/2023:22:57:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:57:34.279 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:33 smithi110 conmon[198813]: [23/Sep/2023:22:57:33] ENGINE Bus STOPPED 2023-09-23T22:57:34.279 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:34 smithi110 conmon[198813]: [23/Sep/2023:22:57:34] ENGINE Bus STARTING 2023-09-23T22:57:34.279 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:34 smithi110 conmon[198813]: [23/Sep/2023:22:57:34] ENGINE Serving on http://:::9283 2023-09-23T22:57:34.279 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:34 smithi110 conmon[198813]: [23/Sep/2023:22:57:34] ENGINE Bus STARTED 2023-09-23T22:57:34.292 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:34 smithi138 conmon[159173]: ignoring --setuser ceph since I am not root 2023-09-23T22:57:34.292 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:34 smithi138 conmon[159173]: ignoring --setgroup ceph since I am not root 2023-09-23T22:57:34.292 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mgr[159197]: -- 172.21.15.138:0/1505010791 <== mon.1 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56395a5871e0 con 0x56395a55f000 2023-09-23T22:57:34.292 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:34 smithi138 conmon[159173]: 2023-09-23T22:57:34.226+0000 7f5525744200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:57:34.606 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:34.607 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:34.607 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:57:34.607 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:57:34.607 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:34.607 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:34.607 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:57:34.608 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:57:34.608 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T22:57:34.608 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.24718 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T22:57:34.608 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: mgrmap e26: y(active, starting, since 0.0184985s) 2023-09-23T22:57:34.608 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:57:34.608 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:57:34.609 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:34.609 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:34.609 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:57:34.609 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:57:34.609 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:34.609 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:34.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:57:34.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:57:34.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T22:57:34.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.24718 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T22:57:34.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: mgrmap e26: y(active, starting, since 0.0184985s) 2023-09-23T22:57:34.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:57:34.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:57:34.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:57:34.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.xrewco"}]: dispatch 2023-09-23T22:57:34.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.fuglpi"}]: dispatch 2023-09-23T22:57:34.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:57:34.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:57:34.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:57:34.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:57:34.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:57:34.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:57:34.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:57:34.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:57:34.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:57:34.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:57:34.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:57:34.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:57:34.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: Manager daemon y is now available 2023-09-23T22:57:34.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:34.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:34.612 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:34.612 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:57:34.612 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:57:34.612 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:57:34.612 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:57:34.612 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:34 smithi110 conmon[198813]: 2023-09-23T22:57:34.336+0000 7f2c8fd12700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:57:34.612 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:34 smithi110 conmon[198813]: 2023-09-23T22:57:34.336+0000 7f2c8fd12700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:57:34.612 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:34 smithi110 conmon[198813]: 2023-09-23T22:57:34.336+0000 7f2c8fd12700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:57:34.612 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:34 smithi110 conmon[198813]: 2023-09-23T22:57:34.336+0000 7f2c8fd12700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:57:34.612 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:34 smithi110 conmon[198813]: 2023-09-23T22:57:34.336+0000 7f2c8fd12700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:57:34.612 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:34 smithi110 conmon[198813]: 2023-09-23T22:57:34.348+0000 7f2c93519700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:57:34.613 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:34 smithi110 conmon[198813]: 2023-09-23T22:57:34.348+0000 7f2c93519700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:57:34.613 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:34 smithi110 conmon[198813]: 2023-09-23T22:57:34.348+0000 7f2c93519700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:57:34.613 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:34 smithi110 conmon[198813]: 2023-09-23T22:57:34.348+0000 7f2c93519700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:57:34.613 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:34 smithi110 conmon[198813]: 2023-09-23T22:57:34.348+0000 7f2c93519700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:57:34.613 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:34 smithi138 conmon[159173]: 2023-09-23T22:57:34.340+0000 7f5525744200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:57:34.613 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:34 smithi138 conmon[159173]: 2023-09-23T22:57:34.479+0000 7f5525744200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:57:34.614 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:34.614 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:34.614 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:57:34.614 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:57:34.614 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' 2023-09-23T22:57:34.614 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:34.614 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.24718 172.21.15.138:0/3964525670' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:57:34.614 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:57:34.614 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T22:57:34.615 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.24718 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T22:57:34.615 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: mgrmap e26: y(active, starting, since 0.0184985s) 2023-09-23T22:57:34.615 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:57:34.615 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:57:34.615 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:57:34.615 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.xrewco"}]: dispatch 2023-09-23T22:57:34.615 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.fuglpi"}]: dispatch 2023-09-23T22:57:34.615 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:57:34.615 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:57:34.615 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:57:34.615 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:57:34.616 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:57:34.616 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:57:34.616 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:57:34.616 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:57:34.616 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:57:34.616 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:57:34.616 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:57:34.616 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:57:34.616 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: Manager daemon y is now available 2023-09-23T22:57:34.616 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:34.616 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:34.617 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:34.617 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:57:34.617 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:57:34.617 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:57:34.617 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:34 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:57:34.898 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:34 smithi138 conmon[159173]: 2023-09-23T22:57:34.620+0000 7f5525744200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:57:34.898 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:34 smithi138 conmon[159173]: 2023-09-23T22:57:34.892+0000 7f5525744200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:57:34.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:57:34.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.xrewco"}]: dispatch 2023-09-23T22:57:34.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.fuglpi"}]: dispatch 2023-09-23T22:57:34.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:57:34.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:57:34.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:57:34.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:57:34.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:57:34.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:57:34.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:57:34.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:57:34.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:57:34.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:57:34.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:57:34.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:57:34.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: Manager daemon y is now available 2023-09-23T22:57:34.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:34.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:34.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:34.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:57:34.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:57:34.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:57:34.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:34 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:57:35.876 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:35 smithi138 conmon[159173]: 2023-09-23T22:57:35.499+0000 7f5525744200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:57:35.876 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:35 smithi138 conmon[159173]: 2023-09-23T22:57:35.718+0000 7f5525744200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:57:35.876 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:35 smithi138 conmon[159173]: 2023-09-23T22:57:35.871+0000 7f5525744200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:57:35.878 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:35 smithi138 ceph-mon[104418]: [23/Sep/2023:22:57:34] ENGINE Bus STARTING 2023-09-23T22:57:35.879 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:35 smithi138 ceph-mon[104418]: [23/Sep/2023:22:57:34] ENGINE Serving on http://172.21.15.110:8765 2023-09-23T22:57:35.879 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:35 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/56238166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:35.879 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:35 smithi138 ceph-mon[104418]: mgrmap e27: y(active, since 1.01958s) 2023-09-23T22:57:35.879 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:35 smithi138 ceph-mon[104418]: pgmap v3: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T22:57:35.879 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:35 smithi138 ceph-mon[104418]: [23/Sep/2023:22:57:34] ENGINE Serving on https://172.21.15.110:7150 2023-09-23T22:57:35.879 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:35 smithi138 ceph-mon[104418]: [23/Sep/2023:22:57:34] ENGINE Bus STARTED 2023-09-23T22:57:35.879 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:35 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:35.879 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:35 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:35.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[100243]: [23/Sep/2023:22:57:34] ENGINE Bus STARTING 2023-09-23T22:57:35.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[100243]: [23/Sep/2023:22:57:34] ENGINE Serving on http://172.21.15.110:8765 2023-09-23T22:57:35.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/56238166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:35.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[100243]: mgrmap e27: y(active, since 1.01958s) 2023-09-23T22:57:35.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[100243]: pgmap v3: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T22:57:35.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[100243]: [23/Sep/2023:22:57:34] ENGINE Serving on https://172.21.15.110:7150 2023-09-23T22:57:35.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[100243]: [23/Sep/2023:22:57:34] ENGINE Bus STARTED 2023-09-23T22:57:35.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:35.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:35.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[118821]: [23/Sep/2023:22:57:34] ENGINE Bus STARTING 2023-09-23T22:57:35.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[118821]: [23/Sep/2023:22:57:34] ENGINE Serving on http://172.21.15.110:8765 2023-09-23T22:57:35.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/56238166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:35.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[118821]: mgrmap e27: y(active, since 1.01958s) 2023-09-23T22:57:35.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[118821]: pgmap v3: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T22:57:35.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[118821]: [23/Sep/2023:22:57:34] ENGINE Serving on https://172.21.15.110:7150 2023-09-23T22:57:35.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[118821]: [23/Sep/2023:22:57:34] ENGINE Bus STARTED 2023-09-23T22:57:35.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:35.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:35 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:36.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:36 smithi138 conmon[159173]: 2023-09-23T22:57:36.037+0000 7f5525744200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:57:37.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:36 smithi138 ceph-mon[104418]: pgmap v4: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T22:57:37.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:36 smithi138 ceph-mon[104418]: mgrmap e28: y(active, since 2s) 2023-09-23T22:57:37.196 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:36 smithi138 conmon[159173]: 2023-09-23T22:57:36.729+0000 7f5525744200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:57:37.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:36 smithi110 ceph-mon[100243]: pgmap v4: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T22:57:37.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:36 smithi110 ceph-mon[100243]: mgrmap e28: y(active, since 2s) 2023-09-23T22:57:37.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:36 smithi110 ceph-mon[118821]: pgmap v4: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T22:57:37.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:36 smithi110 ceph-mon[118821]: mgrmap e28: y(active, since 2s) 2023-09-23T22:57:38.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:37 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/506183410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:38.195 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:37 smithi138 conmon[159173]: 2023-09-23T22:57:37.782+0000 7f5525744200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:57:38.195 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:37 smithi138 conmon[159173]: 2023-09-23T22:57:37.917+0000 7f5525744200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:57:38.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:37 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/506183410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:38.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:37 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/506183410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:39.240 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:39 smithi138 conmon[159173]: 2023-09-23T22:57:39.180+0000 7f5525744200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:57:39.240 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:38 smithi138 ceph-mon[104418]: pgmap v5: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T22:57:39.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:38 smithi110 ceph-mon[100243]: pgmap v5: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T22:57:39.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:38 smithi110 ceph-mon[118821]: pgmap v5: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T22:57:39.643 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:39 smithi138 conmon[159173]: 2023-09-23T22:57:39.333+0000 7f5525744200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:57:39.945 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:39 smithi138 conmon[159173]: 2023-09-23T22:57:39.642+0000 7f5525744200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:57:40.441 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:40 smithi138 conmon[159173]: 2023-09-23T22:57:40.112+0000 7f5525744200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:57:40.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:40 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:40.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:40 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:40.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:40 smithi138 ceph-mon[104418]: pgmap v6: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T22:57:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:40 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:40 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:40 smithi110 ceph-mon[100243]: pgmap v6: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T22:57:40.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:40 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:40.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:40 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:40.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:40 smithi110 ceph-mon[118821]: pgmap v6: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T22:57:40.715 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:40 smithi138 conmon[159173]: 2023-09-23T22:57:40.459+0000 7f5525744200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:57:40.715 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:40 smithi138 conmon[159173]: 2023-09-23T22:57:40.712+0000 7f5525744200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:57:41.442 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:41 smithi138 conmon[159173]: 2023-09-23T22:57:41.176+0000 7f5525744200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:57:41.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:41 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2271842906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:41.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:41 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:41.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:41 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:41.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:41 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2271842906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:41.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:41 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:41.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:41 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:41.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:41 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2271842906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:41.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:41 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:41.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:41 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:42.635 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:42 smithi138 conmon[159173]: 2023-09-23T22:57:42.329+0000 7f5525744200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:57:42.635 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:42 smithi138 conmon[159173]: 2023-09-23T22:57:42.540+0000 7f5525744200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:57:42.889 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:42 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:42.889 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:42 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:42.889 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:42 smithi138 ceph-mon[104418]: pgmap v7: 89 pgs: 89 active+clean; 2.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 470 op/s 2023-09-23T22:57:42.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:42 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:42.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:42 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:42.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:42 smithi110 ceph-mon[118821]: pgmap v7: 89 pgs: 89 active+clean; 2.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 470 op/s 2023-09-23T22:57:42.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:42 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:42.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:42 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:42.947 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:42 smithi110 ceph-mon[100243]: pgmap v7: 89 pgs: 89 active+clean; 2.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 470 op/s 2023-09-23T22:57:43.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:42 smithi138 conmon[159173]: 2023-09-23T22:57:42.887+0000 7f5525744200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:57:43.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:43 smithi138 conmon[159173]: 2023-09-23T22:57:43.079+0000 7f5525744200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:57:43.678 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:43 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2571023970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:43.678 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:43 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:43.678 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:43 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:43.682 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:43 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2571023970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:43.682 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:43 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:43.682 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:43 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:43.691 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:43 smithi138 conmon[159173]: 2023-09-23T22:57:43.438+0000 7f5525744200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:57:43.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:43 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2571023970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:43.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:43 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:43.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:43 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:44.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:44 smithi110 ceph-mon[100243]: pgmap v8: 89 pgs: 89 active+clean; 2.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 365 op/s 2023-09-23T22:57:44.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:44 smithi110 ceph-mon[118821]: pgmap v8: 89 pgs: 89 active+clean; 2.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 365 op/s 2023-09-23T22:57:45.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:44 smithi138 conmon[159173]: 2023-09-23T22:57:44.782+0000 7f5525744200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:57:45.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:44 smithi138 conmon[159173]: [23/Sep/2023:22:57:44] ENGINE Bus STARTING 2023-09-23T22:57:45.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:44 smithi138 conmon[159173]: CherryPy Checker: 2023-09-23T22:57:45.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:44 smithi138 conmon[159173]: The Application mounted at '' has an empty config. 2023-09-23T22:57:45.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:44 smithi138 conmon[159173]: 2023-09-23T22:57:45.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:44 smithi138 conmon[159173]: [23/Sep/2023:22:57:44] ENGINE Serving on http://:::9283 2023-09-23T22:57:45.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:44 smithi138 conmon[159173]: [23/Sep/2023:22:57:44] ENGINE Bus STARTED 2023-09-23T22:57:45.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:44 smithi138 ceph-mon[104418]: pgmap v8: 89 pgs: 89 active+clean; 2.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 365 op/s 2023-09-23T22:57:45.691 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:57:45 smithi138 conmon[159173]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:57:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:57:45.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:45 smithi110 ceph-mon[100243]: Standby manager daemon x started 2023-09-23T22:57:45.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:45 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/4047902511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:57:45.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:45 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/4047902511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:57:45.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:45 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/4047902511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:57:45.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:45 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/4047902511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:57:45.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:45 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/512308630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:45.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:45 smithi110 ceph-mon[118821]: Standby manager daemon x started 2023-09-23T22:57:45.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:45 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/4047902511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:57:45.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:45 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/4047902511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:57:45.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:45 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/4047902511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:57:45.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:45 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/4047902511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:57:45.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:45 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/512308630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:46.199 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:45 smithi138 ceph-mon[104418]: Standby manager daemon x started 2023-09-23T22:57:46.199 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:45 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/4047902511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:57:46.199 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:45 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/4047902511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:57:46.199 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:45 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/4047902511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:57:46.199 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:45 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/4047902511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:57:46.199 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:45 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/512308630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:47.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:46 smithi138 ceph-mon[104418]: mgrmap e29: y(active, since 11s), standbys: x 2023-09-23T22:57:47.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:46 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:57:47.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:46 smithi138 ceph-mon[104418]: pgmap v9: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T22:57:47.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:46 smithi110 ceph-mon[118821]: mgrmap e29: y(active, since 11s), standbys: x 2023-09-23T22:57:47.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:46 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:57:47.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:46 smithi110 ceph-mon[118821]: pgmap v9: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T22:57:47.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:46 smithi110 ceph-mon[100243]: mgrmap e29: y(active, since 11s), standbys: x 2023-09-23T22:57:47.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:46 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:57:47.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:46 smithi110 ceph-mon[100243]: pgmap v9: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T22:57:48.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:47 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:48.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:47 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:48.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:47 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:57:48.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:47 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:57:48.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:47 smithi138 ceph-mon[104418]: Updating smithi110:/etc/ceph/ceph.conf 2023-09-23T22:57:48.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:47 smithi138 ceph-mon[104418]: Updating smithi138:/etc/ceph/ceph.conf 2023-09-23T22:57:48.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:47 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:48.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:47 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:48.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:47 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:57:48.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:47 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:57:48.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:47 smithi110 ceph-mon[100243]: Updating smithi110:/etc/ceph/ceph.conf 2023-09-23T22:57:48.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:47 smithi110 ceph-mon[100243]: Updating smithi138:/etc/ceph/ceph.conf 2023-09-23T22:57:48.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:47 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:48.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:47 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:48.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:47 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:57:48.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:47 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:57:48.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:47 smithi110 ceph-mon[118821]: Updating smithi110:/etc/ceph/ceph.conf 2023-09-23T22:57:48.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:47 smithi110 ceph-mon[118821]: Updating smithi138:/etc/ceph/ceph.conf 2023-09-23T22:57:49.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:48 smithi138 ceph-mon[104418]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:57:49.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:48 smithi138 ceph-mon[104418]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:57:49.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:48 smithi138 ceph-mon[104418]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:57:49.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:48 smithi138 ceph-mon[104418]: pgmap v10: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 561 op/s 2023-09-23T22:57:49.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:48 smithi138 ceph-mon[104418]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:57:49.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:48 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2510831737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:49.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:48 smithi110 ceph-mon[118821]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:57:49.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:48 smithi110 ceph-mon[118821]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:57:49.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:48 smithi110 ceph-mon[118821]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:57:49.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:48 smithi110 ceph-mon[118821]: pgmap v10: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 561 op/s 2023-09-23T22:57:49.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:48 smithi110 ceph-mon[118821]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:57:49.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:48 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2510831737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:49.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:48 smithi110 ceph-mon[100243]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:57:49.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:48 smithi110 ceph-mon[100243]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:57:49.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:48 smithi110 ceph-mon[100243]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:57:49.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:48 smithi110 ceph-mon[100243]: pgmap v10: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 561 op/s 2023-09-23T22:57:49.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:48 smithi110 ceph-mon[100243]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:57:49.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:48 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2510831737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:50.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:49 smithi138 ceph-mon[104418]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:57:50.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:49 smithi138 ceph-mon[104418]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:57:50.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:49 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:50.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:49 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:50.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:49 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:50.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:49 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:50.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:49 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:50.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:49 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:50.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:49 smithi110 ceph-mon[100243]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:57:50.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:49 smithi110 ceph-mon[100243]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:57:50.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:49 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:50.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:49 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:50.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:49 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:50.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:49 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:50.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:49 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:50.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:49 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:50.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:49 smithi110 ceph-mon[118821]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:57:50.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:49 smithi110 ceph-mon[118821]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:57:50.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:49 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:50.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:49 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:50.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:49 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:50.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:49 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:50.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:49 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:50.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:49 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:51.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:50 smithi138 ceph-mon[104418]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:57:51.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:50 smithi138 ceph-mon[104418]: Reconfiguring daemon prometheus.a on smithi138 2023-09-23T22:57:51.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:50 smithi138 ceph-mon[104418]: pgmap v11: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 561 op/s 2023-09-23T22:57:51.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:50 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1849393123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:51.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:50 smithi110 ceph-mon[118821]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:57:51.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:50 smithi110 ceph-mon[118821]: Reconfiguring daemon prometheus.a on smithi138 2023-09-23T22:57:51.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:50 smithi110 ceph-mon[118821]: pgmap v11: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 561 op/s 2023-09-23T22:57:51.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:50 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1849393123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:50 smithi110 ceph-mon[100243]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:57:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:50 smithi110 ceph-mon[100243]: Reconfiguring daemon prometheus.a on smithi138 2023-09-23T22:57:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:50 smithi110 ceph-mon[100243]: pgmap v11: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 561 op/s 2023-09-23T22:57:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:50 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1849393123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:53 smithi138 ceph-mon[104418]: pgmap v12: 89 pgs: 89 active+clean; 2.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-09-23T22:57:53.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:53 smithi110 ceph-mon[100243]: pgmap v12: 89 pgs: 89 active+clean; 2.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-09-23T22:57:53.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:53 smithi110 ceph-mon[118821]: pgmap v12: 89 pgs: 89 active+clean; 2.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-09-23T22:57:54.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:54 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3750195203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:54.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:54 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3750195203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:54.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:54 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3750195203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:54.949 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:54 smithi138 systemd[1]: Stopping Ceph prometheus.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:57:55.436 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:55 smithi138 ceph-mon[104418]: pgmap v13: 89 pgs: 89 active+clean; 2.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-09-23T22:57:55.436 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:55 smithi138 conmon[171826]: ts=2023-09-23T22:57:55.224Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T22:57:55.436 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:55 smithi138 conmon[171826]: ts=2023-09-23T22:57:55.224Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T22:57:55.436 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:55 smithi138 conmon[171826]: ts=2023-09-23T22:57:55.224Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T22:57:55.436 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:55 smithi138 conmon[171826]: ts=2023-09-23T22:57:55.224Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T22:57:55.436 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:55 smithi138 conmon[171826]: ts=2023-09-23T22:57:55.224Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T22:57:55.436 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:55 smithi138 conmon[171826]: ts=2023-09-23T22:57:55.224Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T22:57:55.436 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:55 smithi138 conmon[171826]: ts=2023-09-23T22:57:55.224Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T22:57:55.436 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:55 smithi138 conmon[171826]: ts=2023-09-23T22:57:55.224Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T22:57:55.436 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:55 smithi138 conmon[171826]: ts=2023-09-23T22:57:55.225Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T22:57:55.437 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:55 smithi138 conmon[171826]: ts=2023-09-23T22:57:55.225Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T22:57:55.437 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:55 smithi138 conmon[171826]: ts=2023-09-23T22:57:55.225Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T22:57:55.437 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:55 smithi138 conmon[171826]: ts=2023-09-23T22:57:55.225Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T22:57:55.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:55 smithi110 ceph-mon[100243]: pgmap v13: 89 pgs: 89 active+clean; 2.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-09-23T22:57:55.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:55 smithi110 ceph-mon[118821]: pgmap v13: 89 pgs: 89 active+clean; 2.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-09-23T22:57:55.694 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:55 smithi138 bash[177013]: 41d979ed4d4fd4829e3650a54c6cab30dd0abd83f9db53ab98e639a22f8dd6b6 2023-09-23T22:57:56.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:56 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1642246535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:56.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:56 smithi138 ceph-mon[104418]: pgmap v14: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-09-23T22:57:56.445 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:56 smithi138 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T22:57:56.445 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:56 smithi138 systemd[1]: Stopped Ceph prometheus.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:57:56.446 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:56 smithi138 systemd[1]: Starting Ceph prometheus.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:57:56.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:56 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1642246535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:56.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:56 smithi110 ceph-mon[100243]: pgmap v14: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-09-23T22:57:56.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:56 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1642246535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:56.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:56 smithi110 ceph-mon[118821]: pgmap v14: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-09-23T22:57:57.694 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 bash[177323]: 8c2bb4977f43cfbfdf2e1844dedd79ec52a7ae185ab5820aba1523fbc571f9d9 2023-09-23T22:57:57.694 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 systemd[1]: Started Ceph prometheus.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:57:57.694 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.336Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T22:57:57.694 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.336Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T22:57:57.694 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.336Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi138 (none))" 2023-09-23T22:57:57.694 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.336Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T22:57:57.694 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.336Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T22:57:57.694 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.339Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T22:57:57.695 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.340Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T22:57:57.695 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.343Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T22:57:57.695 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.343Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.629µs 2023-09-23T22:57:57.695 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.343Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T22:57:57.695 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.344Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-09-23T22:57:57.695 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.360Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T22:57:57.695 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.360Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T22:57:57.695 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.401Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-09-23T22:57:57.695 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.404Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-09-23T22:57:57.695 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.404Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-09-23T22:57:57.695 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.404Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=40.714µs wal_replay_duration=60.850398ms wbl_replay_duration=468ns total_replay_duration=60.920174ms 2023-09-23T22:57:57.695 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.413Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T22:57:57.696 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.413Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T22:57:57.696 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.413Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T22:57:57.696 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.436Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.991021ms db_storage=1.163µs remote_storage=1.481µs web_handler=486ns query_engine=883ns scrape=580.323µs scrape_sd=157.205µs notify=18.164µs notify_sd=13.485µs rules=21.903202ms tracing=7.807µs 2023-09-23T22:57:57.696 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.436Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T22:57:57.696 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:57:57 smithi138 conmon[177361]: ts=2023-09-23T22:57:57.436Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T22:57:57.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:57 smithi110 conmon[198813]: [23/Sep/2023:22:57:57] ENGINE Bus STOPPING 2023-09-23T22:57:57.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:57 smithi110 conmon[198813]: [23/Sep/2023:22:57:57] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:57:57.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:57 smithi110 conmon[198813]: [23/Sep/2023:22:57:57] ENGINE Bus STOPPED 2023-09-23T22:57:57.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:57 smithi110 conmon[198813]: [23/Sep/2023:22:57:57] ENGINE Bus STARTING 2023-09-23T22:57:57.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:57 smithi110 conmon[198813]: [23/Sep/2023:22:57:57] ENGINE Serving on http://:::9283 2023-09-23T22:57:57.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:57:57 smithi110 conmon[198813]: [23/Sep/2023:22:57:57] ENGINE Bus STARTED 2023-09-23T22:57:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:58 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:58 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:58 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:57:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:58 smithi138 ceph-mon[104418]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:57:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:58 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:57:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:58 smithi138 ceph-mon[104418]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:57:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:58 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:58 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:58 smithi138 ceph-mon[104418]: pgmap v15: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T22:57:58.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:57:58 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/4010282784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:58.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:58.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:58.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:57:58.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[100243]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:57:58.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:57:58.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[100243]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:57:58.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:58.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:58.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[100243]: pgmap v15: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T22:57:58.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/4010282784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:57:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:57:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[118821]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:57:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:57:58.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[118821]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:57:58.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:57:58.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:58.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[118821]: pgmap v15: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T22:57:58.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:57:58 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/4010282784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:00.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:00 smithi138 ceph-mon[104418]: pgmap v16: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T22:58:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:00 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:58:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:00 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:58:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:00 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:58:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:00 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:00.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:00 smithi110 ceph-mon[100243]: pgmap v16: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T22:58:00.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:00 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:58:00.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:00 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:58:00.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:00 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:58:00.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:00 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:00.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:00 smithi110 ceph-mon[118821]: pgmap v16: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T22:58:00.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:00 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:58:00.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:00 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:58:00.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:00 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:58:00.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:00 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:01.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:01 smithi138 ceph-mon[104418]: Upgrade: Updating mgr.x 2023-09-23T22:58:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:01 smithi138 ceph-mon[104418]: Deploying daemon mgr.x on smithi138 2023-09-23T22:58:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:01 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2538064615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:01.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:01 smithi110 ceph-mon[100243]: Upgrade: Updating mgr.x 2023-09-23T22:58:01.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:01 smithi110 ceph-mon[100243]: Deploying daemon mgr.x on smithi138 2023-09-23T22:58:01.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:01 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2538064615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:01.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:01 smithi110 ceph-mon[118821]: Upgrade: Updating mgr.x 2023-09-23T22:58:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:01 smithi110 ceph-mon[118821]: Deploying daemon mgr.x on smithi138 2023-09-23T22:58:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:01 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2538064615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:02.450 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:02 smithi138 systemd[1]: Stopping Ceph mgr.x for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:58:02.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:02 smithi110 ceph-mon[100243]: pgmap v17: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-09-23T22:58:02.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:02 smithi110 ceph-mon[118821]: pgmap v17: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-09-23T22:58:02.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:02 smithi138 ceph-mon[104418]: pgmap v17: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-09-23T22:58:03.441 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:03 smithi138 bash[177966]: d6b943d65fcedd63f9a704ad1dad81f173f193188a297793892cdfb064ce87ee 2023-09-23T22:58:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:03 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3678014183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:03.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:03 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3678014183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:03.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:03 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3678014183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:04.441 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:03 smithi138 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:58:04.442 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:03 smithi138 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T22:58:04.442 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:03 smithi138 systemd[1]: Stopped Ceph mgr.x for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:58:04.442 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:04 smithi138 systemd[1]: Starting Ceph mgr.x for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:58:04.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:04 smithi138 ceph-mon[104418]: pgmap v18: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T22:58:04.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:04 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:04.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:04 smithi110 ceph-mon[100243]: pgmap v18: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T22:58:04.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:04 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:04.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:04 smithi110 ceph-mon[118821]: pgmap v18: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T22:58:04.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:04 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:05.569 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:05 smithi138 bash[178301]: fef2a6a3bd6d5ca259eb2297c27591d7e590a0b3d4b54da25f74cad3ed024f0a 2023-09-23T22:58:05.569 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:05 smithi138 systemd[1]: Started Ceph mgr.x for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:58:05.569 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:05 smithi138 conmon[178339]: 2023-09-23T22:58:05.442+0000 7f5b70d4e200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:58:05.569 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:05 smithi138 conmon[178339]: 2023-09-23T22:58:05.566+0000 7f5b70d4e200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:58:05.863 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:05 smithi138 conmon[178339]: 2023-09-23T22:58:05.698+0000 7f5b70d4e200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:58:05.863 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:05 smithi138 conmon[178339]: 2023-09-23T22:58:05.853+0000 7f5b70d4e200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:58:06.444 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:06 smithi138 conmon[178339]: 2023-09-23T22:58:06.141+0000 7f5b70d4e200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:58:06.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:06 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:06.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:06 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:06.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:06 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:06.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:06 smithi138 ceph-mon[104418]: pgmap v19: 89 pgs: 89 active+clean; 1.5 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2023-09-23T22:58:06.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:06 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:06.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:06 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:06.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:06 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2936776138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:06.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:06 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:06.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:06 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:06.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:06 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:06.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:06 smithi110 ceph-mon[118821]: pgmap v19: 89 pgs: 89 active+clean; 1.5 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2023-09-23T22:58:06.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:06 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:06.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:06 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:06.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:06 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/2936776138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:06.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:06 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:06.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:06 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:06.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:06 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:06.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:06 smithi110 ceph-mon[100243]: pgmap v19: 89 pgs: 89 active+clean; 1.5 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2023-09-23T22:58:06.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:06 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:06.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:06 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:06.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:06 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/2936776138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:07.189 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:06 smithi138 conmon[178339]: 2023-09-23T22:58:06.808+0000 7f5b70d4e200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:58:07.189 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:07 smithi138 conmon[178339]: 2023-09-23T22:58:07.025+0000 7f5b70d4e200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:58:07.189 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:07 smithi138 conmon[178339]: 2023-09-23T22:58:07.176+0000 7f5b70d4e200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:58:07.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:58:06] "GET /metrics HTTP/1.1" 200 36224 "" "Prometheus/2.43.0" 2023-09-23T22:58:07.441 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:07 smithi138 conmon[178339]: 2023-09-23T22:58:07.344+0000 7f5b70d4e200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:58:08.442 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:08 smithi138 conmon[178339]: 2023-09-23T22:58:08.080+0000 7f5b70d4e200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:58:08.944 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:58:08 smithi138 conmon[177361]: ts=2023-09-23T22:58:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:58:09.340 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:09 smithi138 conmon[178339]: 2023-09-23T22:58:09.182+0000 7f5b70d4e200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:58:09.343 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:09 smithi138 ceph-mon[104418]: pgmap v20: 89 pgs: 89 active+clean; 1.5 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T22:58:09.344 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:09 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/999646429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:09.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:09 smithi110 ceph-mon[100243]: pgmap v20: 89 pgs: 89 active+clean; 1.5 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T22:58:09.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:09 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/999646429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:09.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:09 smithi110 ceph-mon[118821]: pgmap v20: 89 pgs: 89 active+clean; 1.5 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T22:58:09.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:09 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/999646429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:09.693 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:09 smithi138 conmon[178339]: 2023-09-23T22:58:09.338+0000 7f5b70d4e200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:58:10.698 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:10 smithi138 ceph-mon[104418]: pgmap v21: 89 pgs: 89 active+clean; 1.5 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T22:58:10.698 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:10 smithi138 conmon[178339]: 2023-09-23T22:58:10.650+0000 7f5b70d4e200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:58:10.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:10 smithi110 ceph-mon[100243]: pgmap v21: 89 pgs: 89 active+clean; 1.5 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T22:58:10.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:10 smithi110 ceph-mon[118821]: pgmap v21: 89 pgs: 89 active+clean; 1.5 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T22:58:11.089 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:10 smithi138 conmon[178339]: 2023-09-23T22:58:10.804+0000 7f5b70d4e200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:58:11.438 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:11 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3915109807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:11.439 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:11 smithi138 conmon[178339]: 2023-09-23T22:58:11.087+0000 7f5b70d4e200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:58:11.692 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:11 smithi138 conmon[178339]: 2023-09-23T22:58:11.581+0000 7f5b70d4e200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:58:11.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:11 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3915109807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:11.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:11 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3915109807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:12.447 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:11 smithi138 conmon[178339]: 2023-09-23T22:58:11.957+0000 7f5b70d4e200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:58:12.447 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:12 smithi138 conmon[178339]: 2023-09-23T22:58:12.207+0000 7f5b70d4e200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:58:12.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:12 smithi138 ceph-mon[104418]: pgmap v22: 89 pgs: 89 active+clean; 1.3 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T22:58:12.948 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:12 smithi138 conmon[178339]: 2023-09-23T22:58:12.656+0000 7f5b70d4e200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:58:12.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:12 smithi110 ceph-mon[118821]: pgmap v22: 89 pgs: 89 active+clean; 1.3 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T22:58:12.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:12 smithi110 ceph-mon[100243]: pgmap v22: 89 pgs: 89 active+clean; 1.3 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T22:58:14.196 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:14 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:14.196 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:14 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:14.196 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:14 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/873306438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:14.196 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:14 smithi138 ceph-mon[104418]: pgmap v23: 89 pgs: 89 active+clean; 1.3 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T22:58:14.196 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:13 smithi138 conmon[178339]: 2023-09-23T22:58:13.832+0000 7f5b70d4e200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:58:14.196 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:14 smithi138 conmon[178339]: 2023-09-23T22:58:14.055+0000 7f5b70d4e200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:58:14.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:14 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:14.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:14 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:14.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:14 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/873306438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:14.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:14 smithi110 ceph-mon[100243]: pgmap v23: 89 pgs: 89 active+clean; 1.3 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T22:58:14.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:14 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:14.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:14 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:14.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:14 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/873306438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:14.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:14 smithi110 ceph-mon[118821]: pgmap v23: 89 pgs: 89 active+clean; 1.3 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T22:58:14.692 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:14 smithi138 conmon[178339]: 2023-09-23T22:58:14.407+0000 7f5b70d4e200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:58:14.692 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:14 smithi138 conmon[178339]: 2023-09-23T22:58:14.600+0000 7f5b70d4e200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:58:15.441 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:14 smithi138 conmon[178339]: 2023-09-23T22:58:14.971+0000 7f5b70d4e200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:58:16.061 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:15 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:16.062 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:15 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:16.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:15 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:16.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:15 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:16.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:15 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:16.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:15 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:16.692 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:16 smithi138 conmon[178339]: 2023-09-23T22:58:16.427+0000 7f5b70d4e200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:58:16.692 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:16 smithi138 conmon[178339]: [23/Sep/2023:22:58:16] ENGINE Bus STARTING 2023-09-23T22:58:16.692 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:16 smithi138 conmon[178339]: CherryPy Checker: 2023-09-23T22:58:16.692 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:16 smithi138 conmon[178339]: The Application mounted at '' has an empty config. 2023-09-23T22:58:16.692 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:16 smithi138 conmon[178339]: 2023-09-23T22:58:16.692 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:16 smithi138 conmon[178339]: [23/Sep/2023:22:58:16] ENGINE Serving on http://:::9283 2023-09-23T22:58:16.692 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:16 smithi138 conmon[178339]: [23/Sep/2023:22:58:16] ENGINE Bus STARTED 2023-09-23T22:58:17.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:16 smithi138 ceph-mon[104418]: pgmap v24: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-09-23T22:58:17.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:16 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/571792630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:17.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:16 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/3963140683' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:58:17.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:16 smithi138 ceph-mon[104418]: Standby manager daemon x restarted 2023-09-23T22:58:17.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:16 smithi138 ceph-mon[104418]: Standby manager daemon x started 2023-09-23T22:58:17.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:16 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/3963140683' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:58:17.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:16 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/3963140683' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:58:17.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:16 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/3963140683' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:58:17.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:58:16] "GET /metrics HTTP/1.1" 200 36224 "" "Prometheus/2.43.0" 2023-09-23T22:58:17.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:16 smithi110 ceph-mon[100243]: pgmap v24: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-09-23T22:58:17.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:16 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/571792630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:17.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:16 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/3963140683' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:58:17.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:16 smithi110 ceph-mon[100243]: Standby manager daemon x restarted 2023-09-23T22:58:17.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:16 smithi110 ceph-mon[100243]: Standby manager daemon x started 2023-09-23T22:58:17.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:16 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/3963140683' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:58:17.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:16 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/3963140683' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:58:17.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:16 smithi110 ceph-mon[100243]: from='mgr.? 172.21.15.138:0/3963140683' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:58:17.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:16 smithi110 ceph-mon[118821]: pgmap v24: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-09-23T22:58:17.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:16 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/571792630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:17.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:16 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/3963140683' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:58:17.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:16 smithi110 ceph-mon[118821]: Standby manager daemon x restarted 2023-09-23T22:58:17.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:16 smithi110 ceph-mon[118821]: Standby manager daemon x started 2023-09-23T22:58:17.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:16 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/3963140683' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:58:17.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:16 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/3963140683' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:58:17.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:16 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/3963140683' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:58:18.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:17 smithi138 ceph-mon[104418]: mgrmap e30: y(active, since 42s), standbys: x 2023-09-23T22:58:18.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:17 smithi110 ceph-mon[100243]: mgrmap e30: y(active, since 42s), standbys: x 2023-09-23T22:58:18.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:17 smithi110 ceph-mon[118821]: mgrmap e30: y(active, since 42s), standbys: x 2023-09-23T22:58:18.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:18 smithi138 ceph-mon[104418]: pgmap v25: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 562 op/s 2023-09-23T22:58:18.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:58:18 smithi138 conmon[177361]: ts=2023-09-23T22:58:18.663Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:58:19.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:18 smithi110 ceph-mon[100243]: pgmap v25: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 562 op/s 2023-09-23T22:58:19.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:18 smithi110 ceph-mon[118821]: pgmap v25: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 562 op/s 2023-09-23T22:58:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:19 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1123885796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:19 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:20.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:19 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1123885796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:20.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:19 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:20.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:19 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1123885796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:20.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:19 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:21.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:20 smithi138 ceph-mon[104418]: pgmap v26: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 562 op/s 2023-09-23T22:58:21.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:20 smithi110 ceph-mon[100243]: pgmap v26: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 562 op/s 2023-09-23T22:58:21.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:20 smithi110 ceph-mon[118821]: pgmap v26: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 562 op/s 2023-09-23T22:58:22.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:21 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1651984066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:22.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:21 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/1651984066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:22.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:21 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1651984066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:23.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:22 smithi138 ceph-mon[104418]: pgmap v27: 89 pgs: 89 active+clean; 908 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-09-23T22:58:23.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:22 smithi110 ceph-mon[100243]: pgmap v27: 89 pgs: 89 active+clean; 908 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-09-23T22:58:23.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:22 smithi110 ceph-mon[118821]: pgmap v27: 89 pgs: 89 active+clean; 908 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-09-23T22:58:25.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: pgmap v28: 89 pgs: 89 active+clean; 908 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-09-23T22:58:25.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3346402074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:25.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:25.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:25.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:25.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:58:25.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:25.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:25.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:58:25.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:25.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:58:25.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:58:25.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T22:58:25.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:58:25.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:58:25.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T22:58:25.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:58:25.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T22:58:25.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: pgmap v28: 89 pgs: 89 active+clean; 908 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-09-23T22:58:25.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3346402074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:25.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:25.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:25.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:25.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:58:25.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:25.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:25.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:58:25.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:25.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:58:25.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:58:25.392 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T22:58:25.392 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:58:25.392 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:58:25.392 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T22:58:25.392 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:58:25.392 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:25 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T22:58:25.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: pgmap v28: 89 pgs: 89 active+clean; 908 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-09-23T22:58:25.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3346402074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:25.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:25.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:25.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:25.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:58:25.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:25.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:25.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:58:25.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:25.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:58:25.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:58:25.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T22:58:25.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:58:25.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:58:25.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T22:58:25.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:58:25.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:25 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T22:58:26.028 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:26 smithi110 ceph-mon[100243]: Upgrade: Setting container_image for all mgr 2023-09-23T22:58:26.028 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:26 smithi110 ceph-mon[100243]: Upgrade: It appears safe to stop mon.a 2023-09-23T22:58:26.029 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:26 smithi110 ceph-mon[118821]: Upgrade: Setting container_image for all mgr 2023-09-23T22:58:26.029 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:26 smithi110 ceph-mon[118821]: Upgrade: It appears safe to stop mon.a 2023-09-23T22:58:26.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:26 smithi138 ceph-mon[104418]: Upgrade: Setting container_image for all mgr 2023-09-23T22:58:26.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:26 smithi138 ceph-mon[104418]: Upgrade: It appears safe to stop mon.a 2023-09-23T22:58:27.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:58:26] "GET /metrics HTTP/1.1" 200 36225 "" "Prometheus/2.43.0" 2023-09-23T22:58:27.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:27 smithi110 ceph-mon[100243]: pgmap v29: 89 pgs: 89 active+clean; 671 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2023-09-23T22:58:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:27 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/206489106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:27.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:27 smithi110 ceph-mon[118821]: pgmap v29: 89 pgs: 89 active+clean; 671 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2023-09-23T22:58:27.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:27 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/206489106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:27.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:27 smithi138 ceph-mon[104418]: pgmap v29: 89 pgs: 89 active+clean; 671 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2023-09-23T22:58:27.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:27 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/206489106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:28.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:28 smithi110 ceph-mon[100243]: Upgrade: Updating mon.a 2023-09-23T22:58:28.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:28 smithi110 ceph-mon[100243]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:28.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:28 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:58:28.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:28 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:58:28.908 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:28 smithi110 ceph-mon[100243]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:28.908 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:28 smithi110 ceph-mon[100243]: Deploying daemon mon.a on smithi110 2023-09-23T22:58:28.908 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:28 smithi110 ceph-mon[100243]: pgmap v30: 89 pgs: 89 active+clean; 671 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T22:58:28.908 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:28 smithi110 ceph-mon[118821]: Upgrade: Updating mon.a 2023-09-23T22:58:28.908 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:28 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:28.908 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:28 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:58:28.908 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:28 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:58:28.908 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:28 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:28.909 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:28 smithi110 ceph-mon[118821]: Deploying daemon mon.a on smithi110 2023-09-23T22:58:28.909 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:28 smithi110 ceph-mon[118821]: pgmap v30: 89 pgs: 89 active+clean; 671 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T22:58:28.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:28 smithi138 ceph-mon[104418]: Upgrade: Updating mon.a 2023-09-23T22:58:28.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:28 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='mgr.y' 2023-09-23T22:58:28.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:28 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:58:28.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:28 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:58:28.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:28 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:28.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:28 smithi138 ceph-mon[104418]: Deploying daemon mon.a on smithi110 2023-09-23T22:58:28.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:28 smithi138 ceph-mon[104418]: pgmap v30: 89 pgs: 89 active+clean; 671 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T22:58:28.945 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:58:28 smithi138 conmon[177361]: ts=2023-09-23T22:58:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:58:30.009 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:29 smithi110 ceph-mon[100243]: from='client.? 172.21.15.110:0/3611703039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:30.010 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:30 smithi110 systemd[1]: Stopping Ceph mon.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:58:30.010 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:29 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/3611703039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:30.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:29 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3611703039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:30.818 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:30 smithi110 conmon[100219]: 2023-09-23T22:58:30.458+0000 7fbe101a4700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:58:30.818 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:30 smithi110 conmon[100219]: 2023-09-23T22:58:30.458+0000 7fbe101a4700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-09-23T22:58:30.818 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:30 smithi110 bash[212509]: c610848cb8f9c5edd35dd43d44bf10952759704924b6a402d58220de14c415d2 2023-09-23T22:58:31.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:31 smithi110 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T22:58:31.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:31 smithi110 systemd[1]: Stopped Ceph mon.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:58:31.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:31 smithi110 systemd[1]: Starting Ceph mon.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T22:58:32.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T22:58:32.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 8 2023-09-23T22:58:32.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: pidfile_write: ignore empty --pid-file 2023-09-23T22:58:32.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 bash[212875]: d6c089d81bde0ea9dbc676c3c112ab790973c04dd0cd29f29cd2c7fa95d19d08 2023-09-23T22:58:32.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: load: jerasure load: lrc 2023-09-23T22:58:32.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: RocksDB version: 7.9.2 2023-09-23T22:58:32.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Git sha 0 2023-09-23T22:58:32.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T22:58:32.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: DB SUMMARY 2023-09-23T22:58:32.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: DB Session ID: XAXXOQ2670DA0CPPQPN4 2023-09-23T22:58:32.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: CURRENT file: CURRENT 2023-09-23T22:58:32.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T22:58:32.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1169 Bytes 2023-09-23T22:58:32.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2023-09-23T22:58:32.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 2694057 ; 2023-09-23T22:58:32.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.error_if_exists: 0 2023-09-23T22:58:32.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.create_if_missing: 0 2023-09-23T22:58:32.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.paranoid_checks: 1 2023-09-23T22:58:32.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T22:58:32.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T22:58:32.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T22:58:32.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.env: 0x55e5e82a6720 2023-09-23T22:58:32.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T22:58:32.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.info_log: 0x55e5eb097420 2023-09-23T22:58:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T22:58:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.statistics: (nil) 2023-09-23T22:58:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.use_fsync: 0 2023-09-23T22:58:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_log_file_size: 0 2023-09-23T22:58:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T22:58:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T22:58:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T22:58:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T22:58:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.allow_fallocate: 1 2023-09-23T22:58:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T22:58:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T22:58:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.use_direct_reads: 0 2023-09-23T22:58:32.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T22:58:32.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T22:58:32.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.db_log_dir: 2023-09-23T22:58:32.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.wal_dir: 2023-09-23T22:58:32.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T22:58:32.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T22:58:32.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T22:58:32.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T22:58:32.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T22:58:32.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T22:58:32.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T22:58:32.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T22:58:32.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.write_buffer_manager: 0x55e5ea326c80 2023-09-23T22:58:32.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T22:58:32.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T22:58:32.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T22:58:32.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T22:58:32.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T22:58:32.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T22:58:32.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T22:58:32.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T22:58:32.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.unordered_write: 0 2023-09-23T22:58:32.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T22:58:32.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T22:58:32.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T22:58:32.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T22:58:32.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.row_cache: None 2023-09-23T22:58:32.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.wal_filter: None 2023-09-23T22:58:32.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T22:58:32.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T22:58:32.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.two_write_queues: 0 2023-09-23T22:58:32.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T22:58:32.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.wal_compression: 0 2023-09-23T22:58:32.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.atomic_flush: 0 2023-09-23T22:58:32.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T22:58:32.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T22:58:32.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T22:58:32.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.log_readahead_size: 0 2023-09-23T22:58:32.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T22:58:32.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T22:58:32.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T22:58:32.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T22:58:32.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T22:58:32.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T22:58:32.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T22:58:32.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_background_jobs: 2 2023-09-23T22:58:32.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_background_compactions: -1 2023-09-23T22:58:32.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_subcompactions: 1 2023-09-23T22:58:32.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T22:58:32.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T22:58:32.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T22:58:32.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T22:58:32.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T22:58:32.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T22:58:32.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T22:58:32.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T22:58:32.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_open_files: -1 2023-09-23T22:58:32.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T22:58:32.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T22:58:32.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T22:58:32.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T22:58:32.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_background_flushes: -1 2023-09-23T22:58:32.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Compression algorithms supported: 2023-09-23T22:58:32.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T22:58:32.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: kZSTD supported: 0 2023-09-23T22:58:32.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: kXpressCompression supported: 0 2023-09-23T22:58:32.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T22:58:32.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: kZlibCompression supported: 1 2023-09-23T22:58:32.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: kSnappyCompression supported: 1 2023-09-23T22:58:32.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: kLZ4Compression supported: 1 2023-09-23T22:58:32.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: kBZip2Compression supported: 0 2023-09-23T22:58:32.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T22:58:32.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T22:58:32.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2023-09-23T22:58:32.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T22:58:32.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T22:58:32.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.merge_operator: 2023-09-23T22:58:32.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compaction_filter: None 2023-09-23T22:58:32.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T22:58:32.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T22:58:32.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T22:58:32.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T22:58:32.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e5eb097520) 2023-09-23T22:58:32.962 INFO:journalctl@ceph.mon.a.smithi110.stdout: cache_index_and_filter_blocks: 1 2023-09-23T22:58:32.962 INFO:journalctl@ceph.mon.a.smithi110.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T22:58:32.962 INFO:journalctl@ceph.mon.a.smithi110.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T22:58:32.962 INFO:journalctl@ceph.mon.a.smithi110.stdout: pin_top_level_index_and_filter: 1 2023-09-23T22:58:32.962 INFO:journalctl@ceph.mon.a.smithi110.stdout: index_type: 0 2023-09-23T22:58:32.963 INFO:journalctl@ceph.mon.a.smithi110.stdout: data_block_index_type: 0 2023-09-23T22:58:32.963 INFO:journalctl@ceph.mon.a.smithi110.stdout: index_shortening: 1 2023-09-23T22:58:32.963 INFO:journalctl@ceph.mon.a.smithi110.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T22:58:32.963 INFO:journalctl@ceph.mon.a.smithi110.stdout: checksum: 4 2023-09-23T22:58:32.963 INFO:journalctl@ceph.mon.a.smithi110.stdout: no_block_cache: 0 2023-09-23T22:58:32.963 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_cache: 0x55e5ea3a9350 2023-09-23T22:58:32.963 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_cache_name: BinnedLRUCache 2023-09-23T22:58:32.963 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_cache_options: 2023-09-23T22:58:32.963 INFO:journalctl@ceph.mon.a.smithi110.stdout: capacity : 536870912 2023-09-23T22:58:32.963 INFO:journalctl@ceph.mon.a.smithi110.stdout: num_shard_bits : 4 2023-09-23T22:58:32.963 INFO:journalctl@ceph.mon.a.smithi110.stdout: strict_capacity_limit : 0 2023-09-23T22:58:32.963 INFO:journalctl@ceph.mon.a.smithi110.stdout: high_pri_pool_ratio: 0.000 2023-09-23T22:58:32.964 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_cache_compressed: (nil) 2023-09-23T22:58:32.964 INFO:journalctl@ceph.mon.a.smithi110.stdout: persistent_cache: (nil) 2023-09-23T22:58:32.964 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_size: 4096 2023-09-23T22:58:32.964 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_size_deviation: 10 2023-09-23T22:58:32.964 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_restart_interval: 16 2023-09-23T22:58:32.964 INFO:journalctl@ceph.mon.a.smithi110.stdout: index_block_restart_interval: 1 2023-09-23T22:58:32.964 INFO:journalctl@ceph.mon.a.smithi110.stdout: metadata_block_size: 4096 2023-09-23T22:58:32.964 INFO:journalctl@ceph.mon.a.smithi110.stdout: partition_filters: 0 2023-09-23T22:58:32.964 INFO:journalctl@ceph.mon.a.smithi110.stdout: use_delta_encoding: 1 2023-09-23T22:58:32.964 INFO:journalctl@ceph.mon.a.smithi110.stdout: filter_policy: bloomfilter 2023-09-23T22:58:32.964 INFO:journalctl@ceph.mon.a.smithi110.stdout: whole_key_filtering: 1 2023-09-23T22:58:32.964 INFO:journalctl@ceph.mon.a.smithi110.stdout: verify_compression: 0 2023-09-23T22:58:32.965 INFO:journalctl@ceph.mon.a.smithi110.stdout: read_amp_bytes_per_bit: 0 2023-09-23T22:58:32.965 INFO:journalctl@ceph.mon.a.smithi110.stdout: format_version: 5 2023-09-23T22:58:32.965 INFO:journalctl@ceph.mon.a.smithi110.stdout: enable_index_compression: 1 2023-09-23T22:58:32.965 INFO:journalctl@ceph.mon.a.smithi110.stdout: block_align: 0 2023-09-23T22:58:32.965 INFO:journalctl@ceph.mon.a.smithi110.stdout: max_auto_readahead_size: 262144 2023-09-23T22:58:32.965 INFO:journalctl@ceph.mon.a.smithi110.stdout: prepopulate_block_cache: 0 2023-09-23T22:58:32.965 INFO:journalctl@ceph.mon.a.smithi110.stdout: initial_auto_readahead_size: 8192 2023-09-23T22:58:32.965 INFO:journalctl@ceph.mon.a.smithi110.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T22:58:32.965 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T22:58:32.965 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T22:58:32.965 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compression: NoCompression 2023-09-23T22:58:32.965 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T22:58:32.965 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T22:58:32.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T22:58:32.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.num_levels: 7 2023-09-23T22:58:32.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T22:58:32.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T22:58:32.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T22:58:32.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T22:58:32.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T22:58:32.966 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T22:58:32.967 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T22:58:32.967 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:58:32.967 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T22:58:32.967 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T22:58:32.967 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T22:58:32.967 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T22:58:32.967 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T22:58:32.968 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T22:58:32.968 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T22:58:32.968 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T22:58:32.968 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:58:32.968 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T22:58:32.968 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T22:58:32.968 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T22:58:32.968 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T22:58:32.968 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T22:58:32.968 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T22:58:32.968 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T22:58:32.968 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T22:58:32.968 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T22:58:32.969 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T22:58:32.969 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T22:58:32.969 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T22:58:32.969 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T22:58:32.969 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T22:58:32.969 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T22:58:32.969 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T22:58:32.969 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T22:58:32.969 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T22:58:32.969 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T22:58:32.969 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T22:58:32.969 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T22:58:32.969 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T22:58:32.970 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T22:58:32.970 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T22:58:32.970 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T22:58:32.970 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T22:58:32.970 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T22:58:32.970 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T22:58:32.970 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T22:58:32.970 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T22:58:32.970 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T22:58:32.970 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T22:58:32.970 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T22:58:32.970 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T22:58:32.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T22:58:32.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T22:58:32.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.table_properties_collectors: 2023-09-23T22:58:32.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.inplace_update_support: 0 2023-09-23T22:58:32.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T22:58:32.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T22:58:32.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T22:58:32.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T22:58:32.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.bloom_locality: 0 2023-09-23T22:58:32.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.max_successive_merges: 0 2023-09-23T22:58:32.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T22:58:32.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T22:58:32.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T22:58:32.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T22:58:32.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.ttl: 2592000 2023-09-23T22:58:32.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T22:58:32.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T22:58:32.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T22:58:32.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.enable_blob_files: false 2023-09-23T22:58:32.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.min_blob_size: 0 2023-09-23T22:58:32.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T22:58:32.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T22:58:32.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T22:58:32.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T22:58:32.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T22:58:32.973 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T22:58:32.973 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T22:58:32.973 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T22:58:32.973 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2023-09-23T22:58:32.973 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 systemd[1]: Started Ceph mon.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T22:58:32.973 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T22:58:32.973 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12247, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T22:58:32.973 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2023-09-23T22:58:32.973 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3dfe1224-496e-4c8d-a221-ef6994288fd5 2023-09-23T22:58:32.973 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695509912681083, "job": 1, "event": "recovery_started", "wal_files": [31]} 2023-09-23T22:58:32.973 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2023-09-23T22:58:32.973 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695509912703774, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 2474209, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12248, "largest_seqno": 13674, "table_properties": {"data_size": 2467637, "index_size": 4017, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1541, "raw_key_size": 13725, "raw_average_key_size": 22, "raw_value_size": 2454759, "raw_average_value_size": 4098, "num_data_blocks": 185, "num_entries": 599, "num_filter_entries": 599, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695509912, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3dfe1224-496e-4c8d-a221-ef6994288fd5", "db_session_id": "XAXXOQ2670DA0CPPQPN4", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2023-09-23T22:58:32.973 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695509912703899, "job": 1, "event": "recovery_finished"} 2023-09-23T22:58:32.974 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2023-09-23T22:58:32.974 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T22:58:32.974 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T22:58:32.974 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e5ea444000 2023-09-23T22:58:32.974 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: DB pointer 0x55e5ea432000 2023-09-23T22:58:32.974 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T22:58:32.974 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T22:58:32.974 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** DB Stats ** 2023-09-23T22:58:32.974 INFO:journalctl@ceph.mon.a.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:58:32.974 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T22:58:32.974 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:58:32.975 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:58:32.975 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T22:58:32.975 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:58:32.975 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:58:32.975 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2023-09-23T22:58:32.975 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** Compaction Stats [default] ** 2023-09-23T22:58:32.975 INFO:journalctl@ceph.mon.a.smithi110.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T22:58:32.975 INFO:journalctl@ceph.mon.a.smithi110.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T22:58:32.975 INFO:journalctl@ceph.mon.a.smithi110.stdout: L0 1/0 2.36 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.6 0.02 0.00 1 0.017 0 0 0.0 0.0 2023-09-23T22:58:32.975 INFO:journalctl@ceph.mon.a.smithi110.stdout: L6 1/0 5.27 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T22:58:32.975 INFO:journalctl@ceph.mon.a.smithi110.stdout: Sum 2/0 7.63 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.6 0.02 0.00 1 0.017 0 0 0.0 0.0 2023-09-23T22:58:32.975 INFO:journalctl@ceph.mon.a.smithi110.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.6 0.02 0.00 1 0.017 0 0 0.0 0.0 2023-09-23T22:58:32.976 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2023-09-23T22:58:32.976 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** Compaction Stats [default] ** 2023-09-23T22:58:32.976 INFO:journalctl@ceph.mon.a.smithi110.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T22:58:32.976 INFO:journalctl@ceph.mon.a.smithi110.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:58:32.976 INFO:journalctl@ceph.mon.a.smithi110.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 142.6 0.02 0.00 1 0.017 0 0 0.0 0.0 2023-09-23T22:58:32.976 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2023-09-23T22:58:32.976 INFO:journalctl@ceph.mon.a.smithi110.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T22:58:32.976 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2023-09-23T22:58:32.976 INFO:journalctl@ceph.mon.a.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:58:32.976 INFO:journalctl@ceph.mon.a.smithi110.stdout: Flush(GB): cumulative 0.002, interval 0.002 2023-09-23T22:58:32.976 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:58:32.976 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:58:32.977 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:58:32.977 INFO:journalctl@ceph.mon.a.smithi110.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:58:32.977 INFO:journalctl@ceph.mon.a.smithi110.stdout: Cumulative compaction: 0.00 GB write, 68.04 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:58:32.977 INFO:journalctl@ceph.mon.a.smithi110.stdout: Interval compaction: 0.00 GB write, 68.04 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:58:32.977 INFO:journalctl@ceph.mon.a.smithi110.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:58:32.977 INFO:journalctl@ceph.mon.a.smithi110.stdout: Block cache BinnedLRUCache@0x55e5ea3a9350#8 capacity: 512.00 MB usage: 5.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-09-23T22:58:32.977 INFO:journalctl@ceph.mon.a.smithi110.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2023-09-23T22:58:32.977 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2023-09-23T22:58:32.977 INFO:journalctl@ceph.mon.a.smithi110.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:58:32.977 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: starting mon.a rank 0 at public addrs [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] at bind addrs [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:58:32.978 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: mon.a@-1(???) e3 preinit fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T22:58:32.978 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: mon.a@-1(???).mds e9 new map 2023-09-23T22:58:32.978 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: mon.a@-1(???).mds e9 print_map 2023-09-23T22:58:32.978 INFO:journalctl@ceph.mon.a.smithi110.stdout: e9 2023-09-23T22:58:32.978 INFO:journalctl@ceph.mon.a.smithi110.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T22:58:32.978 INFO:journalctl@ceph.mon.a.smithi110.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:58:32.978 INFO:journalctl@ceph.mon.a.smithi110.stdout: legacy client fscid: 1 2023-09-23T22:58:32.978 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2023-09-23T22:58:32.978 INFO:journalctl@ceph.mon.a.smithi110.stdout: Filesystem 'foo' (1) 2023-09-23T22:58:32.978 INFO:journalctl@ceph.mon.a.smithi110.stdout: fs_name foo 2023-09-23T22:58:32.978 INFO:journalctl@ceph.mon.a.smithi110.stdout: epoch 8 2023-09-23T22:58:32.979 INFO:journalctl@ceph.mon.a.smithi110.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T22:58:32.979 INFO:journalctl@ceph.mon.a.smithi110.stdout: created 2023-09-23T22:53:16.882348+0000 2023-09-23T22:58:32.979 INFO:journalctl@ceph.mon.a.smithi110.stdout: modified 2023-09-23T22:53:41.314442+0000 2023-09-23T22:58:32.979 INFO:journalctl@ceph.mon.a.smithi110.stdout: tableserver 0 2023-09-23T22:58:32.979 INFO:journalctl@ceph.mon.a.smithi110.stdout: root 0 2023-09-23T22:58:32.979 INFO:journalctl@ceph.mon.a.smithi110.stdout: session_timeout 60 2023-09-23T22:58:32.979 INFO:journalctl@ceph.mon.a.smithi110.stdout: session_autoclose 300 2023-09-23T22:58:32.979 INFO:journalctl@ceph.mon.a.smithi110.stdout: max_file_size 1099511627776 2023-09-23T22:58:32.979 INFO:journalctl@ceph.mon.a.smithi110.stdout: required_client_features {} 2023-09-23T22:58:32.979 INFO:journalctl@ceph.mon.a.smithi110.stdout: last_failure 0 2023-09-23T22:58:32.979 INFO:journalctl@ceph.mon.a.smithi110.stdout: last_failure_osd_epoch 0 2023-09-23T22:58:32.979 INFO:journalctl@ceph.mon.a.smithi110.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:58:32.979 INFO:journalctl@ceph.mon.a.smithi110.stdout: max_mds 1 2023-09-23T22:58:32.980 INFO:journalctl@ceph.mon.a.smithi110.stdout: in 0 2023-09-23T22:58:32.980 INFO:journalctl@ceph.mon.a.smithi110.stdout: up {0=24572} 2023-09-23T22:58:32.980 INFO:journalctl@ceph.mon.a.smithi110.stdout: failed 2023-09-23T22:58:32.980 INFO:journalctl@ceph.mon.a.smithi110.stdout: damaged 2023-09-23T22:58:32.980 INFO:journalctl@ceph.mon.a.smithi110.stdout: stopped 2023-09-23T22:58:32.980 INFO:journalctl@ceph.mon.a.smithi110.stdout: data_pools [4] 2023-09-23T22:58:32.980 INFO:journalctl@ceph.mon.a.smithi110.stdout: metadata_pool 3 2023-09-23T22:58:32.980 INFO:journalctl@ceph.mon.a.smithi110.stdout: inline_data disabled 2023-09-23T22:58:32.980 INFO:journalctl@ceph.mon.a.smithi110.stdout: balancer 2023-09-23T22:58:32.980 INFO:journalctl@ceph.mon.a.smithi110.stdout: bal_rank_mask -1 2023-09-23T22:58:32.980 INFO:journalctl@ceph.mon.a.smithi110.stdout: standby_count_wanted 1 2023-09-23T22:58:32.980 INFO:journalctl@ceph.mon.a.smithi110.stdout: [mds.foo.smithi110.fuglpi{0:24572} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.110:6834/1876054795,v1:172.21.15.110:6835/1876054795] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T22:58:32.980 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2023-09-23T22:58:32.981 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2023-09-23T22:58:32.981 INFO:journalctl@ceph.mon.a.smithi110.stdout: Standby daemons: 2023-09-23T22:58:32.981 INFO:journalctl@ceph.mon.a.smithi110.stdout: 2023-09-23T22:58:32.981 INFO:journalctl@ceph.mon.a.smithi110.stdout: [mds.foo.smithi138.xrewco{-1:24547} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.138:6832/276794957,v1:172.21.15.138:6833/276794957] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T22:58:32.981 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: mon.a@-1(???).osd e76 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T22:58:32.981 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: mon.a@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:58:32.981 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: mon.a@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:58:32.981 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: mon.a@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:58:32.981 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:32 smithi110 ceph-mon[212939]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2023-09-23T22:58:34.504 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:34 smithi110 conmon[198813]: ignoring --setuser ceph since I am not root 2023-09-23T22:58:34.504 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:34 smithi110 conmon[198813]: ignoring --setgroup ceph since I am not root 2023-09-23T22:58:34.504 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:34 smithi110 conmon[198813]: 2023-09-23T22:58:34.503+0000 7ff05b956200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:58:34.505 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[212939]: pgmap v32: 89 pgs: 89 active+clean; 441 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2023-09-23T22:58:34.505 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/457768789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:34.505 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[212939]: pgmap v33: 89 pgs: 89 active+clean; 441 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T22:58:34.505 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[212939]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:34.505 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[212939]: mon.a calling monitor election 2023-09-23T22:58:34.505 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[212939]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:58:34.505 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[212939]: monmap e3: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} 2023-09-23T22:58:34.505 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[212939]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T22:58:34.505 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[212939]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T22:58:34.505 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[212939]: mgrmap e30: y(active, since 60s), standbys: x 2023-09-23T22:58:34.506 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[212939]: mgrmap e31: y(active, since 60s), standbys: x 2023-09-23T22:58:34.506 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[212939]: overall HEALTH_OK 2023-09-23T22:58:34.506 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[212939]: from='mgr.15105 ' entity='' 2023-09-23T22:58:34.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[118821]: pgmap v32: 89 pgs: 89 active+clean; 441 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2023-09-23T22:58:34.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/457768789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:34.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[118821]: pgmap v33: 89 pgs: 89 active+clean; 441 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T22:58:34.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[118821]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:34.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[118821]: mon.a calling monitor election 2023-09-23T22:58:34.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[118821]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:58:34.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[118821]: monmap e3: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} 2023-09-23T22:58:34.507 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[118821]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T22:58:34.507 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[118821]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T22:58:34.507 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[118821]: mgrmap e30: y(active, since 60s), standbys: x 2023-09-23T22:58:34.507 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[118821]: mgrmap e31: y(active, since 60s), standbys: x 2023-09-23T22:58:34.507 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[118821]: overall HEALTH_OK 2023-09-23T22:58:34.507 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:34 smithi110 ceph-mon[118821]: from='mgr.15105 ' entity='' 2023-09-23T22:58:34.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:34 smithi138 ceph-mon[104418]: pgmap v32: 89 pgs: 89 active+clean; 441 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2023-09-23T22:58:34.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:34 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/457768789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:34.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:34 smithi138 ceph-mon[104418]: pgmap v33: 89 pgs: 89 active+clean; 441 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T22:58:34.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:34 smithi138 ceph-mon[104418]: from='mgr.15105 172.21.15.110:0/733114978' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:34.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:34 smithi138 ceph-mon[104418]: mon.a calling monitor election 2023-09-23T22:58:34.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:34 smithi138 ceph-mon[104418]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:58:34.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:34 smithi138 ceph-mon[104418]: monmap e3: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} 2023-09-23T22:58:34.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:34 smithi138 ceph-mon[104418]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T22:58:34.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:34 smithi138 ceph-mon[104418]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T22:58:34.577 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:34 smithi138 ceph-mon[104418]: mgrmap e30: y(active, since 60s), standbys: x 2023-09-23T22:58:34.577 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:34 smithi138 ceph-mon[104418]: mgrmap e31: y(active, since 60s), standbys: x 2023-09-23T22:58:34.577 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:34 smithi138 ceph-mon[104418]: overall HEALTH_OK 2023-09-23T22:58:34.577 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:34 smithi138 ceph-mon[104418]: from='mgr.15105 ' entity='' 2023-09-23T22:58:34.577 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:34 smithi138 conmon[178339]: ignoring --setuser ceph since I am not root 2023-09-23T22:58:34.577 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:34 smithi138 conmon[178339]: ignoring --setgroup ceph since I am not root 2023-09-23T22:58:34.577 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:34 smithi138 conmon[178339]: 2023-09-23T22:58:34.444+0000 7f200f12e200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:58:34.578 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:34 smithi138 conmon[178339]: 2023-09-23T22:58:34.574+0000 7f200f12e200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:58:34.779 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:34 smithi110 conmon[198813]: 2023-09-23T22:58:34.643+0000 7ff05b956200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:58:34.780 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:34 smithi110 conmon[198813]: 2023-09-23T22:58:34.777+0000 7ff05b956200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:58:34.823 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 1 2023-09-23T22:58:34.846 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:34 smithi138 conmon[178339]: 2023-09-23T22:58:34.711+0000 7f200f12e200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:58:35.139 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:34 smithi138 conmon[178339]: 2023-09-23T22:58:34.846+0000 7f200f12e200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:58:35.185 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:34 smithi110 conmon[198813]: 2023-09-23T22:58:34.932+0000 7ff05b956200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:58:35.185 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:35 smithi110 conmon[198813]: 2023-09-23T22:58:35.183+0000 7ff05b956200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:58:35.441 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:35 smithi138 conmon[178339]: 2023-09-23T22:58:35.139+0000 7f200f12e200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:58:35.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:35 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/247320560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:35.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:35 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/247320560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:35.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:35 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/247320560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:58:36.041 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:35 smithi138 conmon[178339]: 2023-09-23T22:58:35.709+0000 7f200f12e200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:58:36.041 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:35 smithi138 conmon[178339]: 2023-09-23T22:58:35.901+0000 7f200f12e200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:58:36.053 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:35 smithi110 conmon[198813]: 2023-09-23T22:58:35.724+0000 7ff05b956200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:58:36.054 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:35 smithi110 conmon[198813]: 2023-09-23T22:58:35.922+0000 7ff05b956200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:58:36.054 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:36 smithi110 conmon[198813]: 2023-09-23T22:58:36.052+0000 7ff05b956200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:58:36.145 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T22:58:36.441 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:36 smithi138 conmon[178339]: 2023-09-23T22:58:36.040+0000 7f200f12e200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:58:36.441 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:36 smithi138 conmon[178339]: 2023-09-23T22:58:36.186+0000 7f200f12e200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:58:36.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:36 smithi110 conmon[198813]: 2023-09-23T22:58:36.210+0000 7ff05b956200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:58:36.869 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:37.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:36 smithi138 conmon[178339]: 2023-09-23T22:58:36.841+0000 7f200f12e200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:58:37.198 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:36 smithi110 conmon[198813]: 2023-09-23T22:58:36.883+0000 7ff05b956200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:58:37.922 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:37 smithi110 conmon[198813]: 2023-09-23T22:58:37.821+0000 7ff05b956200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:58:38.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:37 smithi138 conmon[178339]: 2023-09-23T22:58:37.810+0000 7f200f12e200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:58:38.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:37 smithi138 conmon[178339]: 2023-09-23T22:58:37.920+0000 7f200f12e200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:58:38.210 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:37 smithi110 conmon[198813]: 2023-09-23T22:58:37.921+0000 7ff05b956200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:58:39.184 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:39 smithi110 conmon[198813]: 2023-09-23T22:58:39.029+0000 7ff05b956200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:58:39.184 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:39 smithi110 conmon[198813]: 2023-09-23T22:58:39.182+0000 7ff05b956200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:58:39.394 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:38 smithi138 conmon[178339]: 2023-09-23T22:58:38.988+0000 7f200f12e200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:58:39.394 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:39 smithi138 conmon[178339]: 2023-09-23T22:58:39.121+0000 7f200f12e200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:58:39.694 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:39 smithi138 conmon[178339]: 2023-09-23T22:58:39.390+0000 7f200f12e200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:58:39.891 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:39 smithi110 conmon[198813]: 2023-09-23T22:58:39.467+0000 7ff05b956200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:58:40.110 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Removed 123246 objects 2023-09-23T22:58:40.110 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Clean up completed and total clean up time :167.803 2023-09-23T22:58:40.151 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:39 smithi138 conmon[178339]: 2023-09-23T22:58:39.834+0000 7f200f12e200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:58:40.172 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:39 smithi110 conmon[198813]: 2023-09-23T22:58:39.891+0000 7ff05b956200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:58:40.441 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:40 smithi138 conmon[178339]: 2023-09-23T22:58:40.150+0000 7f200f12e200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:58:40.441 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:40 smithi138 conmon[178339]: 2023-09-23T22:58:40.298+0000 7f200f12e200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:58:40.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:40 smithi110 conmon[198813]: 2023-09-23T22:58:40.172+0000 7ff05b956200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:58:40.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:40 smithi110 conmon[198813]: 2023-09-23T22:58:40.320+0000 7ff05b956200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:58:40.941 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:40 smithi138 conmon[178339]: 2023-09-23T22:58:40.568+0000 7f200f12e200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:58:40.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:40 smithi110 conmon[198813]: 2023-09-23T22:58:40.592+0000 7ff05b956200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:58:41.604 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:41 smithi138 conmon[178339]: 2023-09-23T22:58:41.275+0000 7f200f12e200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:58:41.604 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:41 smithi138 conmon[178339]: 2023-09-23T22:58:41.407+0000 7f200f12e200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:58:41.628 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:41 smithi110 conmon[198813]: 2023-09-23T22:58:41.298+0000 7ff05b956200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:58:41.628 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:41 smithi110 conmon[198813]: 2023-09-23T22:58:41.430+0000 7ff05b956200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:58:41.940 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:41 smithi138 conmon[178339]: 2023-09-23T22:58:41.603+0000 7f200f12e200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:58:41.940 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:41 smithi138 conmon[178339]: 2023-09-23T22:58:41.718+0000 7f200f12e200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:58:41.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:41 smithi110 conmon[198813]: 2023-09-23T22:58:41.627+0000 7ff05b956200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:58:41.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:41 smithi110 conmon[198813]: 2023-09-23T22:58:41.743+0000 7ff05b956200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:58:42.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:41 smithi138 conmon[178339]: 2023-09-23T22:58:41.939+0000 7f200f12e200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:58:42.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:41 smithi110 conmon[198813]: 2023-09-23T22:58:41.965+0000 7ff05b956200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:58:43.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:42 smithi138 conmon[104364]: 2023-09-23T22:58:42.844+0000 7fa7a701c700 -1 mon.b@1(peon).osd e77 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T22:58:43.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:42 smithi138 conmon[178339]: 2023-09-23T22:58:42.805+0000 7f200f12e200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:58:43.191 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:42 smithi138 conmon[178339]: [23/Sep/2023:22:58:42] ENGINE Bus STARTING 2023-09-23T22:58:43.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:42 smithi138 conmon[178339]: CherryPy Checker: 2023-09-23T22:58:43.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:42 smithi138 conmon[178339]: The Application mounted at '' has an empty config. 2023-09-23T22:58:43.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:42 smithi138 conmon[178339]: 2023-09-23T22:58:43.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:42 smithi138 conmon[178339]: [23/Sep/2023:22:58:42] ENGINE Serving on http://:::9283 2023-09-23T22:58:43.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 22:58:42 smithi138 conmon[178339]: [23/Sep/2023:22:58:42] ENGINE Bus STARTED 2023-09-23T22:58:43.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:42 smithi110 conmon[118796]: 2023-09-23T22:58:42.844+0000 7f7b192f9700 -1 mon.c@2(peon).osd e77 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T22:58:43.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:42 smithi110 conmon[198813]: 2023-09-23T22:58:42.839+0000 7ff05b956200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:58:43.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:42 smithi110 conmon[198813]: [23/Sep/2023:22:58:42] ENGINE Bus STARTING 2023-09-23T22:58:43.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:42 smithi110 conmon[198813]: 2023-09-23T22:58:42.980+0000 7fefb7485700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:58:43.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:42 smithi110 conmon[198813]: 2023-09-23T22:58:42.980+0000 7fefb7485700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:58:43.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:42 smithi110 conmon[198813]: 2023-09-23T22:58:42.980+0000 7fefb7485700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:58:43.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:42 smithi110 conmon[198813]: 2023-09-23T22:58:42.980+0000 7fefb7485700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:58:43.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:42 smithi110 conmon[198813]: 2023-09-23T22:58:42.980+0000 7fefb7485700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:58:43.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:42 smithi110 conmon[198813]: 2023-09-23T22:58:42.980+0000 7fefbccd0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:58:43.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:42 smithi110 conmon[198813]: 2023-09-23T22:58:42.980+0000 7fefbccd0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:58:43.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:42 smithi110 conmon[198813]: 2023-09-23T22:58:42.980+0000 7fefbccd0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:58:43.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:42 smithi110 conmon[198813]: 2023-09-23T22:58:42.980+0000 7fefbccd0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:58:43.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:42 smithi110 conmon[198813]: 2023-09-23T22:58:42.980+0000 7fefbccd0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:58:43.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:43 smithi110 conmon[198813]: CherryPy Checker: 2023-09-23T22:58:43.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:43 smithi110 conmon[198813]: The Application mounted at '' has an empty config. 2023-09-23T22:58:43.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:43 smithi110 conmon[198813]: 2023-09-23T22:58:43.203 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:43 smithi110 conmon[198813]: [23/Sep/2023:22:58:43] ENGINE Serving on http://:::9283 2023-09-23T22:58:43.203 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:43 smithi110 conmon[198813]: [23/Sep/2023:22:58:43] ENGINE Bus STARTED 2023-09-23T22:58:43.864 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T22:58:43.865 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (117s) - 6m 17.0M - 0.25.0 c8568f914cd2 d9641d6ed9ca 2023-09-23T22:58:43.865 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi138 *:3000 running (99s) - 5m 82.3M - 9.4.7 2c41d148cca3 56929ec304ec 2023-09-23T22:58:43.865 INFO:teuthology.orchestra.run.smithi110.stdout:mds.foo.smithi110.fuglpi smithi110 running (5m) - 5m 18.4M - 17.2.6-1125-gc9bb1bcf 444539aec5bf f59eb5601951 2023-09-23T22:58:43.865 INFO:teuthology.orchestra.run.smithi110.stdout:mds.foo.smithi138.xrewco smithi138 running (5m) - 5m 14.7M - 17.2.6-1125-gc9bb1bcf 444539aec5bf cf02b45fd734 2023-09-23T22:58:43.865 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi138 *:8443,9283,8765 running (39s) - 10m 83.0M - 18.2.0-460-gbb0daeff e32c2a64c126 fef2a6a3bd6d 2023-09-23T22:58:43.865 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (85s) - 11m 483M - 18.2.0-460-gbb0daeff e32c2a64c126 09d917770c6e 2023-09-23T22:58:43.865 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (11m) - 11m 101M 2048M 17.2.6-1125-gc9bb1bcf 444539aec5bf c610848cb8f9 2023-09-23T22:58:43.865 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi138 running (10m) - 10m 72.7M 2048M 17.2.6-1125-gc9bb1bcf 444539aec5bf 522bb9710d62 2023-09-23T22:58:43.865 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (10m) - 10m 68.0M 2048M 17.2.6-1125-gc9bb1bcf 444539aec5bf aaa9613ef02d 2023-09-23T22:58:43.865 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (114s) - 5m 9298k - 1.5.0 0da6a335fe13 39d48d803cf2 2023-09-23T22:58:43.866 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi138 *:9100 running (95s) - 5m 9835k - 1.5.0 0da6a335fe13 a55c5d37ba4b 2023-09-23T22:58:43.866 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (9m) - 9m 1853M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf d7807df7af7a 2023-09-23T22:58:43.866 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (9m) - 9m 1120M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 9cdbba4d4209 2023-09-23T22:58:43.866 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (9m) - 9m 2398M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 8350abe92da2 2023-09-23T22:58:43.866 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (8m) - 8m 798M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf d34afe3cf9ea 2023-09-23T22:58:43.866 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi138 running (8m) - 8m 2012M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 47f7252bf067 2023-09-23T22:58:43.866 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi138 running (7m) - 7m 2156M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf fb3decabc2c4 2023-09-23T22:58:43.866 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi138 running (7m) - 7m 1034M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf c3f9d2352b7c 2023-09-23T22:58:43.866 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi138 running (7m) - 7m 1516M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 161d1d2d0445 2023-09-23T22:58:43.866 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi138 *:9095 running (46s) - 6m 47.7M - 2.43.0 a07b618ecd1d 8c2bb4977f43 2023-09-23T22:58:44.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: Standby manager daemon x restarted 2023-09-23T22:58:44.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: Standby manager daemon x started 2023-09-23T22:58:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/4141177387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:58:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/4141177387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:58:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/4141177387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:58:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.? 172.21.15.138:0/4141177387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:58:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: Active manager daemon y restarted 2023-09-23T22:58:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: Activating manager daemon y 2023-09-23T22:58:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T22:58:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: mgrmap e32: y(active, starting, since 0.0065465s), standbys: x 2023-09-23T22:58:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: failed to encode map e77 with expected crc 2023-09-23T22:58:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:58:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:58:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:58:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.xrewco"}]: dispatch 2023-09-23T22:58:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.fuglpi"}]: dispatch 2023-09-23T22:58:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:58:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:58:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:58:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:58:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:58:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:58:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:58:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:58:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:58:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:58:44.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:58:44.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:58:44.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:58:44.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: Manager daemon y is now available 2023-09-23T22:58:44.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:44.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:44.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:44.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:58:44.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:58:44.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: failed to encode map e77 with expected crc 2023-09-23T22:58:44.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:58:44.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:58:44.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:43 smithi138 ceph-mon[104418]: [23/Sep/2023:22:58:43] ENGINE Bus STARTING 2023-09-23T22:58:44.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:43 smithi110 conmon[198813]: 2023-09-23T22:58:43.859+0000 7ff0150b6700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-09-23T22:58:44.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: Standby manager daemon x restarted 2023-09-23T22:58:44.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: Standby manager daemon x started 2023-09-23T22:58:44.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.? 172.21.15.138:0/4141177387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:58:44.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.? 172.21.15.138:0/4141177387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:58:44.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.? 172.21.15.138:0/4141177387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:58:44.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.? 172.21.15.138:0/4141177387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:58:44.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: Active manager daemon y restarted 2023-09-23T22:58:44.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: Activating manager daemon y 2023-09-23T22:58:44.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T22:58:44.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: mgrmap e32: y(active, starting, since 0.0065465s), standbys: x 2023-09-23T22:58:44.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: failed to encode map e77 with expected crc 2023-09-23T22:58:44.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:58:44.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:58:44.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:58:44.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.xrewco"}]: dispatch 2023-09-23T22:58:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.fuglpi"}]: dispatch 2023-09-23T22:58:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:58:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:58:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:58:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:58:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:58:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:58:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:58:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:58:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:58:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:58:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:58:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:58:44.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:58:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: Standby manager daemon x restarted 2023-09-23T22:58:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: Standby manager daemon x started 2023-09-23T22:58:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/4141177387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:58:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/4141177387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:58:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/4141177387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:58:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.? 172.21.15.138:0/4141177387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:58:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: Active manager daemon y restarted 2023-09-23T22:58:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: Activating manager daemon y 2023-09-23T22:58:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T22:58:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: mgrmap e32: y(active, starting, since 0.0065465s), standbys: x 2023-09-23T22:58:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: failed to encode map e77 with expected crc 2023-09-23T22:58:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:58:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:58:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:58:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.xrewco"}]: dispatch 2023-09-23T22:58:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.fuglpi"}]: dispatch 2023-09-23T22:58:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:58:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:58:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:58:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:58:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:58:44.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:58:44.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:58:44.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:58:44.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:58:44.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:58:44.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:58:44.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:58:44.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:58:44.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: Manager daemon y is now available 2023-09-23T22:58:44.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:44.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:44.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:44.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:58:44.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:58:44.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: failed to encode map e77 with expected crc 2023-09-23T22:58:44.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:58:44.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:58:44.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[212939]: [23/Sep/2023:22:58:43] ENGINE Bus STARTING 2023-09-23T22:58:44.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: Manager daemon y is now available 2023-09-23T22:58:44.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:44.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:44.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:44.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:58:44.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:58:44.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: failed to encode map e77 with expected crc 2023-09-23T22:58:44.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:58:44.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:58:44.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:43 smithi110 ceph-mon[118821]: [23/Sep/2023:22:58:43] ENGINE Bus STARTING 2023-09-23T22:58:44.577 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-09-23T22:58:44.577 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-09-23T22:58:44.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[212939]: failed to encode map e77 with expected crc 2023-09-23T22:58:44.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[212939]: failed to encode map e77 with expected crc 2023-09-23T22:58:44.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[212939]: failed to encode map e77 with expected crc 2023-09-23T22:58:44.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[212939]: [23/Sep/2023:22:58:43] ENGINE Serving on https://172.21.15.110:7150 2023-09-23T22:58:44.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[212939]: [23/Sep/2023:22:58:43] ENGINE Serving on http://172.21.15.110:8765 2023-09-23T22:58:44.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[212939]: [23/Sep/2023:22:58:43] ENGINE Bus STARTED 2023-09-23T22:58:44.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[212939]: mgrmap e33: y(active, since 1.00976s), standbys: x 2023-09-23T22:58:44.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[212939]: from='client.34116 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:58:44.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[212939]: pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:44.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[118821]: failed to encode map e77 with expected crc 2023-09-23T22:58:44.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[118821]: failed to encode map e77 with expected crc 2023-09-23T22:58:44.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[118821]: failed to encode map e77 with expected crc 2023-09-23T22:58:44.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[118821]: [23/Sep/2023:22:58:43] ENGINE Serving on https://172.21.15.110:7150 2023-09-23T22:58:44.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[118821]: [23/Sep/2023:22:58:43] ENGINE Serving on http://172.21.15.110:8765 2023-09-23T22:58:44.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[118821]: [23/Sep/2023:22:58:43] ENGINE Bus STARTED 2023-09-23T22:58:44.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[118821]: mgrmap e33: y(active, since 1.00976s), standbys: x 2023-09-23T22:58:44.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[118821]: from='client.34116 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:58:44.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:44 smithi110 ceph-mon[118821]: pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:45.105 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T22:58:45.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:44 smithi138 ceph-mon[104418]: failed to encode map e77 with expected crc 2023-09-23T22:58:45.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:44 smithi138 ceph-mon[104418]: failed to encode map e77 with expected crc 2023-09-23T22:58:45.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:44 smithi138 ceph-mon[104418]: failed to encode map e77 with expected crc 2023-09-23T22:58:45.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:44 smithi138 ceph-mon[104418]: [23/Sep/2023:22:58:43] ENGINE Serving on https://172.21.15.110:7150 2023-09-23T22:58:45.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:44 smithi138 ceph-mon[104418]: [23/Sep/2023:22:58:43] ENGINE Serving on http://172.21.15.110:8765 2023-09-23T22:58:45.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:44 smithi138 ceph-mon[104418]: [23/Sep/2023:22:58:43] ENGINE Bus STARTED 2023-09-23T22:58:45.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:44 smithi138 ceph-mon[104418]: mgrmap e33: y(active, since 1.00976s), standbys: x 2023-09-23T22:58:45.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:44 smithi138 ceph-mon[104418]: from='client.34116 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:58:45.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:44 smithi138 ceph-mon[104418]: pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:45.391 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:45.536 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:45.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:45 smithi110 ceph-mon[212939]: pgmap v4: 89 pgs: 89 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:45.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:45 smithi110 ceph-mon[118821]: pgmap v4: 89 pgs: 89 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:46.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:45 smithi138 ceph-mon[104418]: pgmap v4: 89 pgs: 89 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:47.142 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:58:46] "GET /metrics HTTP/1.1" 200 33505 "" "Prometheus/2.43.0" 2023-09-23T22:58:47.142 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:46 smithi110 ceph-mon[212939]: mgrmap e34: y(active, since 3s), standbys: x 2023-09-23T22:58:47.142 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:46 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:47.143 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:46 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:47.143 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:46 smithi110 ceph-mon[118821]: mgrmap e34: y(active, since 3s), standbys: x 2023-09-23T22:58:47.143 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:46 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:47.143 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:46 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:47.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:46 smithi138 ceph-mon[104418]: mgrmap e34: y(active, since 3s), standbys: x 2023-09-23T22:58:47.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:46 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:47.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:46 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:48.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:47 smithi138 ceph-mon[104418]: pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:48.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:47 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:48.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:47 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:48.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:47 smithi110 ceph-mon[212939]: pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:48.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:48.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:48.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:47 smithi110 ceph-mon[118821]: pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:48.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:47 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:48.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:47 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:48.698 INFO:teuthology.orchestra.run.smithi110.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:58:48.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:58:48 smithi138 conmon[177361]: ts=2023-09-23T22:58:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:58:48.981 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:48 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1369417804' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:58:48.981 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:48 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:58:48.981 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:48.982 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:48.982 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:48.998 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_0' removed 2023-09-23T22:58:49.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:48 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/1369417804' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:58:49.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:48 smithi138 ceph-mon[104418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:58:49.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:48 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:49.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:48 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:49.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:48 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:48 smithi138 conmon[104364]: 2023-09-23T22:58:48.987+0000 7fa7a701c700 -1 mon.b@1(peon).osd e78 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T22:58:49.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:48 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/1369417804' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:58:49.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:48 smithi110 ceph-mon[118821]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:58:49.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:48 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:49.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:48 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:49.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:48 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:49.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:48 smithi110 conmon[118796]: 2023-09-23T22:58:48.986+0000 7f7b192f9700 -1 mon.c@2(peon).osd e78 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T22:58:49.963 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T22:58:50.013 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T22:58:50.013 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T22:58:50.013 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-09-23T22:58:50.014 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-09-23T22:58:50.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[212939]: from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:58:50.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[212939]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:58:50.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[212939]: pgmap v6: 89 pgs: 89 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:50.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:58:50.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[212939]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T22:58:50.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[212939]: failed to encode map e78 with expected crc 2023-09-23T22:58:50.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[212939]: failed to encode map e78 with expected crc 2023-09-23T22:58:50.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:50.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:50.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[118821]: from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:58:50.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[118821]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:58:50.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[118821]: pgmap v6: 89 pgs: 89 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:50.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[118821]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:58:50.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[118821]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T22:58:50.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[118821]: failed to encode map e78 with expected crc 2023-09-23T22:58:50.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[118821]: failed to encode map e78 with expected crc 2023-09-23T22:58:50.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:50.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:49 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:50.415 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:50.417 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:50.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:49 smithi138 ceph-mon[104418]: from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:58:50.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:49 smithi138 ceph-mon[104418]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:58:50.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:49 smithi138 ceph-mon[104418]: pgmap v6: 89 pgs: 89 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:50.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:49 smithi138 ceph-mon[104418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:58:50.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:49 smithi138 ceph-mon[104418]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T22:58:50.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:49 smithi138 ceph-mon[104418]: failed to encode map e78 with expected crc 2023-09-23T22:58:50.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:49 smithi138 ceph-mon[104418]: failed to encode map e78 with expected crc 2023-09-23T22:58:50.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:49 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:50.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:49 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:52.053 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:51 smithi110 ceph-mon[118821]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:52.053 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:51 smithi110 ceph-mon[212939]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:52.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:51 smithi138 ceph-mon[104418]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:52.995 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:52 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:52.996 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:52 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:52.996 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:52 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:52.996 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:52 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:58:52.996 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:52 smithi110 ceph-mon[118821]: Updating smithi110:/etc/ceph/ceph.conf 2023-09-23T22:58:52.996 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:52 smithi110 ceph-mon[118821]: Updating smithi138:/etc/ceph/ceph.conf 2023-09-23T22:58:52.996 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:52 smithi110 ceph-mon[118821]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:58:52.996 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:52 smithi110 ceph-mon[118821]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:58:52.996 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:52 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:52.996 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:52 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:52.996 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:52 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:52.997 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:52 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:58:52.997 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:52 smithi110 ceph-mon[212939]: Updating smithi110:/etc/ceph/ceph.conf 2023-09-23T22:58:52.997 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:52 smithi110 ceph-mon[212939]: Updating smithi138:/etc/ceph/ceph.conf 2023-09-23T22:58:52.997 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:52 smithi110 ceph-mon[212939]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:58:52.997 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:52 smithi110 ceph-mon[212939]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:58:53.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:52 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:53.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:52 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:53.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:52 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:53.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:52 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:58:53.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:52 smithi138 ceph-mon[104418]: Updating smithi110:/etc/ceph/ceph.conf 2023-09-23T22:58:53.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:52 smithi138 ceph-mon[104418]: Updating smithi138:/etc/ceph/ceph.conf 2023-09-23T22:58:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:52 smithi138 ceph-mon[104418]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:58:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:52 smithi138 ceph-mon[104418]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:58:54.177 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_1' created 2023-09-23T22:58:54.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[212939]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:58:54.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[212939]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:58:54.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[212939]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:58:54.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[212939]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:58:54.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[212939]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:54.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:54.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:54.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:58:54.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.390 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T22:58:54.390 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[118821]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:58:54.390 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[118821]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:58:54.390 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[118821]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:58:54.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[118821]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:58:54.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[118821]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:54.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:54.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:54.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:58:54.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:54 smithi110 ceph-mon[118821]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T22:58:54.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:54 smithi110 conmon[118796]: 2023-09-23T22:58:54.162+0000 7f7b192f9700 -1 mon.c@2(peon).osd e79 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T22:58:54.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:54 smithi138 ceph-mon[104418]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:58:54.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:54 smithi138 ceph-mon[104418]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:58:54.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:54 smithi138 ceph-mon[104418]: Updating smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:58:54.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:54 smithi138 ceph-mon[104418]: Updating smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:58:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:54 smithi138 ceph-mon[104418]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:54 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:54 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:54 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:54 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:54 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:54 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:54 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:54 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:58:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:54 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:58:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:54 smithi138 ceph-mon[104418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T22:58:54.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:54 smithi138 conmon[104364]: 2023-09-23T22:58:54.163+0000 7fa7a701c700 -1 mon.b@1(peon).osd e79 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T22:58:54.937 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-09-23T22:58:55.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:55 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T22:58:55.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:55 smithi110 ceph-mon[212939]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T22:58:55.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:55 smithi110 ceph-mon[212939]: failed to encode map e79 with expected crc 2023-09-23T22:58:55.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:55 smithi110 ceph-mon[212939]: failed to encode map e79 with expected crc 2023-09-23T22:58:55.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:55 smithi110 ceph-mon[212939]: failed to encode map e79 with expected crc 2023-09-23T22:58:55.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:55 smithi110 ceph-mon[118821]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T22:58:55.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:55 smithi110 ceph-mon[118821]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T22:58:55.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:55 smithi110 ceph-mon[118821]: failed to encode map e79 with expected crc 2023-09-23T22:58:55.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:55 smithi110 ceph-mon[118821]: failed to encode map e79 with expected crc 2023-09-23T22:58:55.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:55 smithi110 ceph-mon[118821]: failed to encode map e79 with expected crc 2023-09-23T22:58:55.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:55 smithi110 conmon[118796]: 2023-09-23T22:58:55.167+0000 7f7b192f9700 -1 mon.c@2(peon).osd e80 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T22:58:55.288 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:55.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:55 smithi138 ceph-mon[104418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T22:58:55.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:55 smithi138 ceph-mon[104418]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T22:58:55.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:55 smithi138 ceph-mon[104418]: failed to encode map e79 with expected crc 2023-09-23T22:58:55.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:55 smithi138 ceph-mon[104418]: failed to encode map e79 with expected crc 2023-09-23T22:58:55.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:55 smithi138 ceph-mon[104418]: failed to encode map e79 with expected crc 2023-09-23T22:58:55.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:55 smithi138 conmon[104364]: 2023-09-23T22:58:55.167+0000 7fa7a701c700 -1 mon.b@1(peon).osd e80 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T22:58:56.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:56 smithi138 ceph-mon[104418]: pgmap v11: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:56.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:56 smithi138 ceph-mon[104418]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T22:58:56.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:56 smithi138 ceph-mon[104418]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:56 smithi138 ceph-mon[104418]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:56 smithi138 ceph-mon[104418]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:56 smithi138 ceph-mon[104418]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:56 smithi138 ceph-mon[104418]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:56 smithi138 ceph-mon[104418]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:56 smithi138 ceph-mon[104418]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:56 smithi138 ceph-mon[104418]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:56 smithi138 ceph-mon[104418]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:56 smithi138 conmon[104364]: 2023-09-23T22:58:56.172+0000 7fa7a701c700 -1 mon.b@1(peon).osd e81 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T22:58:56.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[212939]: pgmap v11: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:56.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[212939]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T22:58:56.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[212939]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[212939]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[212939]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[212939]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[212939]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[212939]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[212939]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[212939]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[212939]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[118821]: pgmap v11: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:56.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[118821]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T22:58:56.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[118821]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[118821]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[118821]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[118821]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[118821]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[118821]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[118821]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[118821]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:56 smithi110 ceph-mon[118821]: failed to encode map e80 with expected crc 2023-09-23T22:58:56.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:56 smithi110 conmon[118796]: 2023-09-23T22:58:56.173+0000 7f7b192f9700 -1 mon.c@2(peon).osd e81 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T22:58:56.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:58:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:58:56] "GET /metrics HTTP/1.1" 200 33505 "" "Prometheus/2.43.0" 2023-09-23T22:58:57.188 INFO:teuthology.orchestra.run.smithi110.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-09-23T22:58:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:57 smithi138 conmon[104364]: 2023-09-23T22:58:57.177+0000 7fa7a701c700 -1 mon.b@1(peon).osd e82 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T22:58:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:57 smithi138 ceph-mon[104418]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T22:58:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:57 smithi138 ceph-mon[104418]: failed to encode map e81 with expected crc 2023-09-23T22:58:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:57 smithi138 ceph-mon[104418]: failed to encode map e81 with expected crc 2023-09-23T22:58:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:57 smithi138 ceph-mon[104418]: failed to encode map e81 with expected crc 2023-09-23T22:58:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:57 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/80007416' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:58:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:57 smithi138 ceph-mon[104418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:58:57.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:57 smithi110 ceph-mon[212939]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T22:58:57.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:57 smithi110 ceph-mon[212939]: failed to encode map e81 with expected crc 2023-09-23T22:58:57.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:57 smithi110 ceph-mon[212939]: failed to encode map e81 with expected crc 2023-09-23T22:58:57.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:57 smithi110 ceph-mon[212939]: failed to encode map e81 with expected crc 2023-09-23T22:58:57.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:57 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/80007416' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:58:57.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:57 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:58:57.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:57 smithi110 conmon[118796]: 2023-09-23T22:58:57.177+0000 7f7b192f9700 -1 mon.c@2(peon).osd e82 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T22:58:57.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:57 smithi110 ceph-mon[118821]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T22:58:57.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:57 smithi110 ceph-mon[118821]: failed to encode map e81 with expected crc 2023-09-23T22:58:57.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:57 smithi110 ceph-mon[118821]: failed to encode map e81 with expected crc 2023-09-23T22:58:57.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:57 smithi110 ceph-mon[118821]: failed to encode map e81 with expected crc 2023-09-23T22:58:57.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:57 smithi110 ceph-mon[118821]: from='client.? 172.21.15.110:0/80007416' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:58:57.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:57 smithi110 ceph-mon[118821]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:58:58.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:58 smithi138 ceph-mon[104418]: pgmap v14: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:58.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:58 smithi138 ceph-mon[104418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:58:58.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:58 smithi138 ceph-mon[104418]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T22:58:58.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:58 smithi138 ceph-mon[104418]: failed to encode map e82 with expected crc 2023-09-23T22:58:58.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:58 smithi138 ceph-mon[104418]: failed to encode map e82 with expected crc 2023-09-23T22:58:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:58:58 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:58.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:58 smithi110 ceph-mon[212939]: pgmap v14: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:58.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:58 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:58:58.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:58 smithi110 ceph-mon[212939]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T22:58:58.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:58 smithi110 ceph-mon[212939]: failed to encode map e82 with expected crc 2023-09-23T22:58:58.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:58 smithi110 ceph-mon[212939]: failed to encode map e82 with expected crc 2023-09-23T22:58:58.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:58:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:58 smithi110 ceph-mon[118821]: pgmap v14: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:58:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:58 smithi110 ceph-mon[118821]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:58:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:58 smithi110 ceph-mon[118821]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T22:58:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:58 smithi110 ceph-mon[118821]: failed to encode map e82 with expected crc 2023-09-23T22:58:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:58 smithi110 ceph-mon[118821]: failed to encode map e82 with expected crc 2023-09-23T22:58:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:58:58 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:58.929 DEBUG:teuthology.orchestra.run.smithi110:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-09-23T22:58:58.933 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T22:58:58.933 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T22:58:58.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:58:58 smithi138 conmon[177361]: ts=2023-09-23T22:58:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:58:58.986 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:hints = 1 2023-09-23T22:58:58.986 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T22:58:58.986 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object prefix: benchmark_data_smithi110_217814 2023-09-23T22:58:58.987 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:58:58.987 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 0 0 0 0 0 0 - 0 2023-09-23T22:58:59.987 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 1 16 4199 4183 261.423 261.438 0.0055155 0.00380875 2023-09-23T22:59:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:00 smithi138 ceph-mon[104418]: pgmap v16: 89 pgs: 89 active+clean; 582 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:59:00.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:00 smithi110 ceph-mon[118821]: pgmap v16: 89 pgs: 89 active+clean; 582 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:59:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:00 smithi110 ceph-mon[212939]: pgmap v16: 89 pgs: 89 active+clean; 582 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:59:00.987 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 2 16 8028 8012 250.325 239.312 0.00369772 0.00398638 2023-09-23T22:59:01.987 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 3 16 11508 11492 239.378 217.5 0.00400515 0.00417241 2023-09-23T22:59:02.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:02 smithi110 ceph-mon[118821]: pgmap v17: 89 pgs: 89 active+clean; 582 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:59:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:02 smithi110 ceph-mon[212939]: pgmap v17: 89 pgs: 89 active+clean; 582 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:59:02.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:02 smithi138 ceph-mon[104418]: pgmap v17: 89 pgs: 89 active+clean; 582 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:59:02.988 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 4 16 14699 14683 229.388 199.438 0.0049358 0.00435206 2023-09-23T22:59:03.988 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 5 16 17705 17689 221.082 187.875 0.00385721 0.00451743 2023-09-23T22:59:04.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:03 smithi138 ceph-mon[104418]: pgmap v18: 89 pgs: 89 active+clean; 582 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:59:04.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:03 smithi110 ceph-mon[212939]: pgmap v18: 89 pgs: 89 active+clean; 582 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:59:04.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:03 smithi110 ceph-mon[118821]: pgmap v18: 89 pgs: 89 active+clean; 582 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:59:04.988 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 6 16 20530 20514 213.66 176.562 0.00623676 0.00467391 2023-09-23T22:59:05.988 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 7 16 23223 23207 207.179 168.312 0.00548148 0.00482136 2023-09-23T22:59:06.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:05 smithi138 ceph-mon[104418]: pgmap v19: 89 pgs: 89 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s 2023-09-23T22:59:06.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:05 smithi110 ceph-mon[118821]: pgmap v19: 89 pgs: 89 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s 2023-09-23T22:59:06.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:05 smithi110 ceph-mon[212939]: pgmap v19: 89 pgs: 89 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s 2023-09-23T22:59:06.989 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 8 16 25769 25753 201.14 159.125 0.00677646 0.00496753 2023-09-23T22:59:07.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:59:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:59:06] "GET /metrics HTTP/1.1" 200 36170 "" "Prometheus/2.43.0" 2023-09-23T22:59:07.989 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 9 16 28194 28178 195.631 151.562 0.00474774 0.00510656 2023-09-23T22:59:08.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:07 smithi138 ceph-mon[104418]: pgmap v20: 89 pgs: 89 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-09-23T22:59:08.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:07 smithi110 ceph-mon[212939]: pgmap v20: 89 pgs: 89 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-09-23T22:59:08.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:07 smithi110 ceph-mon[118821]: pgmap v20: 89 pgs: 89 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-09-23T22:59:08.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:59:08 smithi138 conmon[177361]: ts=2023-09-23T22:59:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:59:08.989 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 10 16 30506 30490 190.517 144.5 0.00567607 0.00524427 2023-09-23T22:59:09.989 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 11 16 32733 32717 185.85 139.188 0.00800838 0.00537673 2023-09-23T22:59:10.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:09 smithi138 ceph-mon[104418]: pgmap v21: 89 pgs: 89 active+clean; 1.7 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2023-09-23T22:59:10.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:09 smithi110 ceph-mon[212939]: pgmap v21: 89 pgs: 89 active+clean; 1.7 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2023-09-23T22:59:10.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:09 smithi110 ceph-mon[118821]: pgmap v21: 89 pgs: 89 active+clean; 1.7 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2023-09-23T22:59:10.990 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 12 16 34877 34861 181.525 134 0.00663437 0.00550402 2023-09-23T22:59:11.990 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 13 16 36965 36949 177.6 130.5 0.00661558 0.00562544 2023-09-23T22:59:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:11 smithi138 ceph-mon[104418]: pgmap v22: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 157 MiB/s wr, 2.52k op/s 2023-09-23T22:59:12.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:11 smithi110 ceph-mon[212939]: pgmap v22: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 157 MiB/s wr, 2.52k op/s 2023-09-23T22:59:12.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:11 smithi110 ceph-mon[118821]: pgmap v22: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 157 MiB/s wr, 2.52k op/s 2023-09-23T22:59:12.990 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 14 16 38591 38575 172.173 101.625 0.00804762 0.00580385 2023-09-23T22:59:13.990 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 15 16 40541 40525 168.819 121.875 0.00648003 0.0059193 2023-09-23T22:59:14.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:13 smithi138 ceph-mon[104418]: pgmap v23: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 157 MiB/s wr, 2.52k op/s 2023-09-23T22:59:14.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:13 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:14.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:13 smithi110 ceph-mon[212939]: pgmap v23: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 157 MiB/s wr, 2.52k op/s 2023-09-23T22:59:14.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:14.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:13 smithi110 ceph-mon[118821]: pgmap v23: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 157 MiB/s wr, 2.52k op/s 2023-09-23T22:59:14.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:13 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:14.990 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 16 16 42331 42315 165.26 111.875 0.00839053 0.00604715 2023-09-23T22:59:15.990 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 17 16 44023 44007 161.759 105.75 0.0064709 0.00617772 2023-09-23T22:59:16.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:15 smithi138 ceph-mon[104418]: pgmap v24: 89 pgs: 89 active+clean; 2.5 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 210 MiB/s wr, 3.35k op/s 2023-09-23T22:59:16.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:15 smithi110 ceph-mon[212939]: pgmap v24: 89 pgs: 89 active+clean; 2.5 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 210 MiB/s wr, 3.35k op/s 2023-09-23T22:59:16.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:15 smithi110 ceph-mon[118821]: pgmap v24: 89 pgs: 89 active+clean; 2.5 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 210 MiB/s wr, 3.35k op/s 2023-09-23T22:59:16.990 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 18 16 45838 45822 159.075 113.438 0.00598897 0.00628132 2023-09-23T22:59:17.203 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:59:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:59:16] "GET /metrics HTTP/1.1" 200 36224 "" "Prometheus/2.43.0" 2023-09-23T22:59:17.990 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 19 16 47578 47562 156.426 108.75 0.00614007 0.00638843 2023-09-23T22:59:18.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:17 smithi138 ceph-mon[104418]: pgmap v25: 89 pgs: 89 active+clean; 2.5 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s 2023-09-23T22:59:18.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:17 smithi110 ceph-mon[212939]: pgmap v25: 89 pgs: 89 active+clean; 2.5 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s 2023-09-23T22:59:18.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:17 smithi110 ceph-mon[118821]: pgmap v25: 89 pgs: 89 active+clean; 2.5 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s 2023-09-23T22:59:18.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:59:18 smithi138 conmon[177361]: ts=2023-09-23T22:59:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:59:18.990 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T22:59:18.990873+0000 min lat: 0.00133192 max lat: 0.137727 avg lat: 0.00649436 2023-09-23T22:59:18.991 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:59:18.991 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 20 16 49263 49247 153.87 105.312 0.00991848 0.00649436 2023-09-23T22:59:19.991 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 21 16 50948 50932 151.555 105.312 0.0138475 0.00659263 2023-09-23T22:59:20.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:19 smithi110 ceph-mon[212939]: pgmap v26: 89 pgs: 89 active+clean; 2.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-09-23T22:59:20.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:19 smithi110 ceph-mon[118821]: pgmap v26: 89 pgs: 89 active+clean; 2.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-09-23T22:59:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:19 smithi138 ceph-mon[104418]: pgmap v26: 89 pgs: 89 active+clean; 2.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-09-23T22:59:20.991 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 22 16 52622 52606 149.422 104.625 0.00910991 0.00668815 2023-09-23T22:59:21.991 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 23 16 54251 54235 147.352 101.812 0.00696197 0.00678174 2023-09-23T22:59:22.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:21 smithi138 ceph-mon[104418]: pgmap v27: 89 pgs: 89 active+clean; 3.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T22:59:22.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:21 smithi110 ceph-mon[118821]: pgmap v27: 89 pgs: 89 active+clean; 3.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T22:59:22.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:21 smithi110 ceph-mon[212939]: pgmap v27: 89 pgs: 89 active+clean; 3.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T22:59:22.991 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 24 16 55851 55835 145.379 100 0.00643668 0.00687359 2023-09-23T22:59:23.991 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 25 16 57414 57398 143.471 97.6875 0.00776598 0.00696583 2023-09-23T22:59:24.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:23 smithi138 ceph-mon[104418]: pgmap v28: 89 pgs: 89 active+clean; 3.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T22:59:24.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:23 smithi110 ceph-mon[212939]: pgmap v28: 89 pgs: 89 active+clean; 3.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T22:59:24.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:23 smithi110 ceph-mon[118821]: pgmap v28: 89 pgs: 89 active+clean; 3.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T22:59:24.994 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 26 16 58951 58935 141.643 96.0625 0.00987433 0.00705553 2023-09-23T22:59:25.992 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 27 16 60465 60449 139.902 94.625 0.010813 0.00714296 2023-09-23T22:59:26.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:25 smithi138 ceph-mon[104418]: pgmap v29: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-09-23T22:59:26.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:25 smithi110 ceph-mon[118821]: pgmap v29: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-09-23T22:59:26.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:25 smithi110 ceph-mon[212939]: pgmap v29: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-09-23T22:59:26.992 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 28 16 61946 61930 138.211 92.5625 0.00637105 0.00723049 2023-09-23T22:59:27.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:59:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:59:26] "GET /metrics HTTP/1.1" 200 36224 "" "Prometheus/2.43.0" 2023-09-23T22:59:27.992 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 29 16 63408 63392 136.596 91.375 0.00754718 0.00731624 2023-09-23T22:59:28.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:27 smithi110 ceph-mon[118821]: pgmap v30: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T22:59:28.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:27 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:28.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:27 smithi110 ceph-mon[212939]: pgmap v30: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T22:59:28.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:27 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:27 smithi138 ceph-mon[104418]: pgmap v30: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T22:59:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:27 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:28.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:59:28 smithi138 conmon[177361]: ts=2023-09-23T22:59:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:59:28.994 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 30 16 64862 64846 135.071 90.875 0.00776233 0.00739877 2023-09-23T22:59:29.993 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 31 16 66296 66280 133.605 89.625 0.0037565 0.0074806 2023-09-23T22:59:30.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:29 smithi138 ceph-mon[104418]: pgmap v31: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T22:59:30.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:29 smithi110 ceph-mon[212939]: pgmap v31: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T22:59:30.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:29 smithi110 ceph-mon[118821]: pgmap v31: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T22:59:30.993 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 32 16 67698 67682 132.168 87.625 0.00930643 0.00756169 2023-09-23T22:59:31.993 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 33 16 69080 69064 130.78 86.375 0.00895952 0.00764192 2023-09-23T22:59:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:31 smithi138 ceph-mon[104418]: pgmap v32: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-09-23T22:59:32.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:31 smithi110 ceph-mon[212939]: pgmap v32: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-09-23T22:59:32.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:31 smithi110 ceph-mon[118821]: pgmap v32: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-09-23T22:59:32.993 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 34 16 70440 70424 129.434 85 0.0109252 0.00772147 2023-09-23T22:59:33.993 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 35 16 71768 71752 128.107 83 0.0146291 0.0078015 2023-09-23T22:59:34.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:33 smithi138 ceph-mon[104418]: pgmap v33: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T22:59:34.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:33 smithi110 ceph-mon[118821]: pgmap v33: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T22:59:34.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:33 smithi110 ceph-mon[212939]: pgmap v33: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T22:59:34.993 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 36 16 73085 73069 126.835 82.3125 0.00976408 0.00787969 2023-09-23T22:59:35.993 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 37 16 74395 74379 125.619 81.875 0.00608717 0.00795577 2023-09-23T22:59:36.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:35 smithi110 ceph-mon[212939]: pgmap v34: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-09-23T22:59:36.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:35 smithi110 ceph-mon[118821]: pgmap v34: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-09-23T22:59:36.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:35 smithi138 ceph-mon[104418]: pgmap v34: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-09-23T22:59:36.993 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 38 16 75677 75661 124.422 80.125 0.0134011 0.00803316 2023-09-23T22:59:37.210 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:59:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:59:36] "GET /metrics HTTP/1.1" 200 36224 "" "Prometheus/2.43.0" 2023-09-23T22:59:37.993 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 39 16 76939 76923 123.254 78.875 0.00806482 0.00810836 2023-09-23T22:59:38.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:37 smithi138 ceph-mon[104418]: pgmap v35: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T22:59:38.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:37 smithi110 ceph-mon[212939]: pgmap v35: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T22:59:38.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:37 smithi110 ceph-mon[118821]: pgmap v35: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T22:59:38.943 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:59:38 smithi138 conmon[177361]: ts=2023-09-23T22:59:38.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:59:38.993 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T22:59:38.993741+0000 min lat: 0.00133192 max lat: 0.137727 avg lat: 0.00818217 2023-09-23T22:59:38.994 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:59:38.994 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 40 16 78206 78190 122.152 79.1875 0.00632345 0.00818217 2023-09-23T22:59:39.993 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 41 16 79451 79435 121.071 77.8125 0.0180734 0.00825489 2023-09-23T22:59:40.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:39 smithi138 ceph-mon[104418]: pgmap v36: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T22:59:40.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:39 smithi110 ceph-mon[212939]: pgmap v36: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T22:59:40.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:39 smithi110 ceph-mon[118821]: pgmap v36: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T22:59:40.993 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 42 16 80680 80664 120.017 76.8125 0.0199797 0.00832804 2023-09-23T22:59:41.993 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 43 16 81878 81862 118.967 74.875 0.011234 0.00840137 2023-09-23T22:59:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:41 smithi138 ceph-mon[104418]: pgmap v37: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T22:59:42.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:41 smithi110 ceph-mon[118821]: pgmap v37: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T22:59:42.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:41 smithi110 ceph-mon[212939]: pgmap v37: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T22:59:42.996 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 44 16 83061 83045 117.941 73.9375 0.0163384 0.00847369 2023-09-23T22:59:43.995 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 45 16 84239 84223 116.956 73.625 0.013884 0.00854512 2023-09-23T22:59:44.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:43 smithi138 ceph-mon[104418]: pgmap v38: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T22:59:44.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:44.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:43 smithi110 ceph-mon[212939]: pgmap v38: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T22:59:44.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:44.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:43 smithi110 ceph-mon[118821]: pgmap v38: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T22:59:44.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:43 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:44.996 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 46 16 85414 85398 116.007 73.4375 0.0173029 0.00861551 2023-09-23T22:59:45.996 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 47 16 86497 86481 114.979 67.6875 0.0161985 0.00869258 2023-09-23T22:59:46.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:45 smithi138 ceph-mon[104418]: pgmap v39: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-09-23T22:59:46.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:45 smithi110 ceph-mon[212939]: pgmap v39: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-09-23T22:59:46.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:45 smithi110 ceph-mon[118821]: pgmap v39: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-09-23T22:59:46.996 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 48 16 87595 87579 114.013 68.625 0.0117622 0.00876637 2023-09-23T22:59:47.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:59:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:59:46] "GET /metrics HTTP/1.1" 200 36224 "" "Prometheus/2.43.0" 2023-09-23T22:59:47.996 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 49 16 88751 88735 113.161 72.25 0.012686 0.00883274 2023-09-23T22:59:48.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:47 smithi110 ceph-mon[118821]: pgmap v40: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T22:59:48.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:47 smithi110 ceph-mon[212939]: pgmap v40: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T22:59:48.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:47 smithi138 ceph-mon[104418]: pgmap v40: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T22:59:48.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:59:48 smithi138 conmon[177361]: ts=2023-09-23T22:59:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:59:48.996 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 50 16 89848 89832 112.269 68.5625 0.0142871 0.00890224 2023-09-23T22:59:49.997 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 51 16 90968 90952 111.44 70 0.00926593 0.00896893 2023-09-23T22:59:50.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:49 smithi110 ceph-mon[212939]: pgmap v41: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T22:59:50.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:49 smithi110 ceph-mon[118821]: pgmap v41: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T22:59:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:49 smithi138 ceph-mon[104418]: pgmap v41: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T22:59:50.997 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 52 16 92089 92073 110.644 70.0625 0.0121984 0.00903319 2023-09-23T22:59:51.999 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 53 16 93195 93179 109.858 69.125 0.0128729 0.00909775 2023-09-23T22:59:52.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:51 smithi110 ceph-mon[212939]: pgmap v42: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T22:59:52.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:51 smithi110 ceph-mon[118821]: pgmap v42: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T22:59:52.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:51 smithi138 ceph-mon[104418]: pgmap v42: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T22:59:52.998 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 54 16 94273 94257 109.071 67.375 0.019338 0.00916313 2023-09-23T22:59:53.998 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 55 16 95229 95213 108.174 59.75 0.0450216 0.009239 2023-09-23T22:59:54.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:53 smithi138 ceph-mon[104418]: pgmap v43: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-09-23T22:59:54.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:53 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:59:54.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:53 smithi110 ceph-mon[212939]: pgmap v43: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-09-23T22:59:54.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:53 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:59:54.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:53 smithi110 ceph-mon[118821]: pgmap v43: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-09-23T22:59:54.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:53 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:59:54.646 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T22:59:54.999 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 56 16 95889 95873 106.979 41.25 0.0154835 0.00934212 2023-09-23T22:59:55.493 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:56.001 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 57 16 96797 96781 106.093 56.75 0.016235 0.00942088 2023-09-23T22:59:56.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:55 smithi110 ceph-mon[212939]: pgmap v44: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T22:59:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:55 smithi110 ceph-mon[118821]: pgmap v44: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T22:59:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:55 smithi138 ceph-mon[104418]: pgmap v44: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T22:59:57.001 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 58 16 97784 97768 105.327 61.6875 0.017939 0.00949022 2023-09-23T22:59:57.191 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 22:59:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:22:59:56] "GET /metrics HTTP/1.1" 200 36224 "" "Prometheus/2.43.0" 2023-09-23T22:59:58.004 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 59 16 98701 98685 104.509 57.3125 0.00796766 0.00956398 2023-09-23T22:59:58.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:57 smithi110 ceph-mon[212939]: pgmap v45: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s 2023-09-23T22:59:58.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:57 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:59:58.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:57 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:59:58.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:57 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:58.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:57 smithi110 ceph-mon[118821]: pgmap v45: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s 2023-09-23T22:59:58.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:57 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:59:58.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:57 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:59:58.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:57 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:58.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:57 smithi138 ceph-mon[104418]: pgmap v45: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s 2023-09-23T22:59:58.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:57 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:59:58.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:57 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:59:58.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:57 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:58.650 INFO:teuthology.orchestra.run.smithi110.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:59:58.944 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 22:59:58 smithi138 conmon[177361]: ts=2023-09-23T22:59:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:59:59.007 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T22:59:59.007554+0000 min lat: 0.00133192 max lat: 0.317087 avg lat: 0.00964309 2023-09-23T22:59:59.007 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:59:59.007 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 60 16 99553 99537 103.649 53.25 0.00493149 0.00964309 2023-09-23T22:59:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:59 smithi138 ceph-mon[104418]: from='client.25084 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:59:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:59 smithi138 ceph-mon[104418]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:59:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:59 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:59:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 22:59:59 smithi138 ceph-mon[104418]: pgmap v46: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T22:59:59.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:59 smithi110 ceph-mon[212939]: from='client.25084 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:59:59.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:59 smithi110 ceph-mon[212939]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:59:59.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:59 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:59:59.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 22:59:59 smithi110 ceph-mon[212939]: pgmap v46: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T22:59:59.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:59 smithi110 ceph-mon[118821]: from='client.25084 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:59:59.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:59 smithi110 ceph-mon[118821]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:59:59.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:59 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T22:59:59.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 22:59:59 smithi110 ceph-mon[118821]: pgmap v46: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T23:00:00.016 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 61 16 100410 100394 102.813 53.5625 0.012921 0.00972092 2023-09-23T23:00:00.038 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for majority of mons to upgrade' 2023-09-23T23:00:00.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:00 smithi138 ceph-mon[104418]: overall HEALTH_OK 2023-09-23T23:00:00.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:00 smithi110 ceph-mon[212939]: overall HEALTH_OK 2023-09-23T23:00:00.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:00 smithi110 ceph-mon[118821]: overall HEALTH_OK 2023-09-23T23:00:01.016 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 62 16 101294 101278 102.047 55.25 0.00958223 0.00979558 2023-09-23T23:00:01.056 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:02.016 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 63 16 102159 102143 101.285 54.0625 0.00703887 0.00986866 2023-09-23T23:00:02.180 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:02 smithi110 ceph-mon[118821]: pgmap v47: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-09-23T23:00:02.180 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:02 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:02.180 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:02 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:02.180 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:02 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:00:02.180 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:02 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:00:02.180 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:02 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:02.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:02 smithi110 ceph-mon[212939]: pgmap v47: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-09-23T23:00:02.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:02 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:02.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:02 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:02.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:02 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:00:02.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:02 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:00:02.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:02 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:02.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:02 smithi138 ceph-mon[104418]: pgmap v47: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-09-23T23:00:02.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:02 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:02.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:02 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:02.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:02 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:00:02.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:02 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:00:02.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:02 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:03.021 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 64 16 103057 103041 100.58 56.125 0.0134692 0.00993801 2023-09-23T23:00:03.182 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:03 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:00:03.182 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:03 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:00:03.182 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:03 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T23:00:03.182 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:03 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T23:00:03.182 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:03 smithi110 ceph-mon[212939]: Upgrade: It appears safe to stop mon.c 2023-09-23T23:00:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:03 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:00:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:03 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:00:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:03 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T23:00:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:03 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T23:00:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:03 smithi138 ceph-mon[104418]: Upgrade: It appears safe to stop mon.c 2023-09-23T23:00:03.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:03 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:00:03.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:03 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:00:03.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:03 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T23:00:03.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:03 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T23:00:03.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:03 smithi110 ceph-mon[118821]: Upgrade: It appears safe to stop mon.c 2023-09-23T23:00:03.579 INFO:teuthology.orchestra.run.smithi110.stdout:wait for majority of mons to upgrade 2023-09-23T23:00:04.018 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 65 16 103974 103958 99.9114 57.3125 0.00907189 0.0100039 2023-09-23T23:00:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:04 smithi138 ceph-mon[104418]: pgmap v48: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s 2023-09-23T23:00:04.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:04 smithi110 ceph-mon[212939]: pgmap v48: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s 2023-09-23T23:00:04.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:04 smithi110 ceph-mon[118821]: pgmap v48: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s 2023-09-23T23:00:05.018 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 66 16 104665 104649 99.0522 43.1875 0.00344876 0.0100772 2023-09-23T23:00:05.250 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-09-23T23:00:06.018 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 67 16 105502 105486 98.3548 52.3125 0.027028 0.0101625 2023-09-23T23:00:06.409 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:06.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:06 smithi110 ceph-mon[118821]: pgmap v49: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T23:00:06.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:06 smithi110 ceph-mon[118821]: Upgrade: Updating mon.c 2023-09-23T23:00:06.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:06 smithi110 ceph-mon[118821]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:06.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:06 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T23:00:06.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:06 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T23:00:06.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:06 smithi110 ceph-mon[118821]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:00:06.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:06 smithi110 ceph-mon[118821]: Deploying daemon mon.c on smithi110 2023-09-23T23:00:06.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:06 smithi110 ceph-mon[212939]: pgmap v49: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T23:00:06.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:06 smithi110 ceph-mon[212939]: Upgrade: Updating mon.c 2023-09-23T23:00:06.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:06 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:06.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:06 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T23:00:06.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:06 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T23:00:06.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:06 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:00:06.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:06 smithi110 ceph-mon[212939]: Deploying daemon mon.c on smithi110 2023-09-23T23:00:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:06 smithi138 ceph-mon[104418]: pgmap v49: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T23:00:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:06 smithi138 ceph-mon[104418]: Upgrade: Updating mon.c 2023-09-23T23:00:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:06 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:06 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T23:00:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:06 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T23:00:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:06 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:00:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:06 smithi138 ceph-mon[104418]: Deploying daemon mon.c on smithi110 2023-09-23T23:00:06.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:00:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:00:06] "GET /metrics HTTP/1.1" 200 36228 "" "Prometheus/2.43.0" 2023-09-23T23:00:07.019 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 68 16 106323 106307 97.6631 51.3125 0.00412913 0.0102177 2023-09-23T23:00:08.020 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 69 16 107150 107134 96.9964 51.6875 0.0116651 0.010305 2023-09-23T23:00:08.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:07 smithi110 systemd[1]: Stopping Ceph mon.c for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:00:08.668 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:08 smithi138 ceph-mon[104418]: pgmap v50: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 725 op/s 2023-09-23T23:00:08.668 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:00:08 smithi138 conmon[177361]: ts=2023-09-23T23:00:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T23:00:08.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:08 smithi110 ceph-mon[212939]: pgmap v50: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 725 op/s 2023-09-23T23:00:08.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:08 smithi110 ceph-mon[118821]: pgmap v50: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 725 op/s 2023-09-23T23:00:09.019 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 70 16 108086 108070 96.4466 58.5 0.0165274 0.0103647 2023-09-23T23:00:09.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:09 smithi110 conmon[118796]: 2023-09-23T23:00:09.045+0000 7f7b1eb04700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:00:09.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:09 smithi110 conmon[118796]: 2023-09-23T23:00:09.045+0000 7f7b1eb04700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-09-23T23:00:09.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:09 smithi110 bash[220607]: aaa9613ef02dcdbf270885a3af9c77bd8d0ecf9c99c4176b4a9523d8189835a7 2023-09-23T23:00:10.019 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 71 16 108911 108895 95.8144 51.5625 0.0198405 0.0104317 2023-09-23T23:00:10.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:10 smithi110 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mon.c.service: Succeeded. 2023-09-23T23:00:10.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:10 smithi110 systemd[1]: Stopped Ceph mon.c for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:00:10.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:10 smithi110 systemd[1]: Starting Ceph mon.c for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:00:11.023 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 72 16 109769 109753 95.2279 53.625 0.0123607 0.010497 2023-09-23T23:00:12.027 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 73 16 110662 110646 94.6796 55.8125 0.00918171 0.0105578 2023-09-23T23:00:12.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T23:00:12.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 8 2023-09-23T23:00:12.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: pidfile_write: ignore empty --pid-file 2023-09-23T23:00:12.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: load: jerasure load: lrc 2023-09-23T23:00:12.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: RocksDB version: 7.9.2 2023-09-23T23:00:12.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Git sha 0 2023-09-23T23:00:12.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T23:00:12.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: DB SUMMARY 2023-09-23T23:00:12.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: DB Session ID: U66FTIOWVQWE9QDQ0LQR 2023-09-23T23:00:12.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: CURRENT file: CURRENT 2023-09-23T23:00:12.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T23:00:12.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1094 Bytes 2023-09-23T23:00:12.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T23:00:12.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 7481196 ; 2023-09-23T23:00:12.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.error_if_exists: 0 2023-09-23T23:00:12.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.create_if_missing: 0 2023-09-23T23:00:12.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.paranoid_checks: 1 2023-09-23T23:00:12.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T23:00:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T23:00:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T23:00:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.env: 0x55c9b6c27720 2023-09-23T23:00:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T23:00:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.info_log: 0x55c9b8b93420 2023-09-23T23:00:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T23:00:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.statistics: (nil) 2023-09-23T23:00:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.use_fsync: 0 2023-09-23T23:00:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_log_file_size: 0 2023-09-23T23:00:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T23:00:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T23:00:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T23:00:12.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T23:00:12.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.allow_fallocate: 1 2023-09-23T23:00:12.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T23:00:12.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T23:00:12.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.use_direct_reads: 0 2023-09-23T23:00:12.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T23:00:12.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T23:00:12.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.db_log_dir: 2023-09-23T23:00:12.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.wal_dir: 2023-09-23T23:00:12.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T23:00:12.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T23:00:12.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T23:00:12.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T23:00:12.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T23:00:12.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T23:00:12.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T23:00:12.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T23:00:12.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.write_buffer_manager: 0x55c9b7e22c80 2023-09-23T23:00:12.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T23:00:12.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T23:00:12.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T23:00:12.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T23:00:12.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T23:00:12.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 bash[221039]: 4c74500e80a3895a0a24bc380f53d76c840813dd4b2bb5ff72311ba693312930 2023-09-23T23:00:12.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T23:00:12.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T23:00:12.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T23:00:12.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.unordered_write: 0 2023-09-23T23:00:12.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T23:00:12.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T23:00:12.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T23:00:12.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T23:00:12.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.row_cache: None 2023-09-23T23:00:12.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.wal_filter: None 2023-09-23T23:00:12.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T23:00:12.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T23:00:12.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.two_write_queues: 0 2023-09-23T23:00:12.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T23:00:12.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.wal_compression: 0 2023-09-23T23:00:12.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.atomic_flush: 0 2023-09-23T23:00:12.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T23:00:12.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T23:00:12.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T23:00:12.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.log_readahead_size: 0 2023-09-23T23:00:12.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T23:00:12.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T23:00:12.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T23:00:12.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T23:00:12.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T23:00:12.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T23:00:12.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T23:00:12.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_background_jobs: 2 2023-09-23T23:00:12.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_background_compactions: -1 2023-09-23T23:00:12.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_subcompactions: 1 2023-09-23T23:00:12.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T23:00:12.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T23:00:12.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T23:00:12.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T23:00:12.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T23:00:12.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T23:00:12.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T23:00:12.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T23:00:12.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_open_files: -1 2023-09-23T23:00:12.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T23:00:12.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T23:00:12.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T23:00:12.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T23:00:12.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_background_flushes: -1 2023-09-23T23:00:12.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Compression algorithms supported: 2023-09-23T23:00:12.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T23:00:12.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: kZSTD supported: 0 2023-09-23T23:00:12.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: kXpressCompression supported: 0 2023-09-23T23:00:12.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T23:00:12.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: kZlibCompression supported: 1 2023-09-23T23:00:12.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: kSnappyCompression supported: 1 2023-09-23T23:00:12.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: kLZ4Compression supported: 1 2023-09-23T23:00:12.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: kBZip2Compression supported: 0 2023-09-23T23:00:12.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T23:00:12.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T23:00:12.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2023-09-23T23:00:12.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T23:00:12.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T23:00:12.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.merge_operator: 2023-09-23T23:00:12.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compaction_filter: None 2023-09-23T23:00:12.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T23:00:12.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T23:00:12.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T23:00:12.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T23:00:12.212 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c9b8b93520) 2023-09-23T23:00:12.212 INFO:journalctl@ceph.mon.c.smithi110.stdout: cache_index_and_filter_blocks: 1 2023-09-23T23:00:12.212 INFO:journalctl@ceph.mon.c.smithi110.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T23:00:12.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T23:00:12.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: pin_top_level_index_and_filter: 1 2023-09-23T23:00:12.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: index_type: 0 2023-09-23T23:00:12.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: data_block_index_type: 0 2023-09-23T23:00:12.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: index_shortening: 1 2023-09-23T23:00:12.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T23:00:12.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: checksum: 4 2023-09-23T23:00:12.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: no_block_cache: 0 2023-09-23T23:00:12.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_cache: 0x55c9b7ea5350 2023-09-23T23:00:12.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_cache_name: BinnedLRUCache 2023-09-23T23:00:12.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_cache_options: 2023-09-23T23:00:12.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: capacity : 536870912 2023-09-23T23:00:12.213 INFO:journalctl@ceph.mon.c.smithi110.stdout: num_shard_bits : 4 2023-09-23T23:00:12.214 INFO:journalctl@ceph.mon.c.smithi110.stdout: strict_capacity_limit : 0 2023-09-23T23:00:12.214 INFO:journalctl@ceph.mon.c.smithi110.stdout: high_pri_pool_ratio: 0.000 2023-09-23T23:00:12.214 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_cache_compressed: (nil) 2023-09-23T23:00:12.214 INFO:journalctl@ceph.mon.c.smithi110.stdout: persistent_cache: (nil) 2023-09-23T23:00:12.214 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_size: 4096 2023-09-23T23:00:12.214 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_size_deviation: 10 2023-09-23T23:00:12.214 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_restart_interval: 16 2023-09-23T23:00:12.214 INFO:journalctl@ceph.mon.c.smithi110.stdout: index_block_restart_interval: 1 2023-09-23T23:00:12.214 INFO:journalctl@ceph.mon.c.smithi110.stdout: metadata_block_size: 4096 2023-09-23T23:00:12.214 INFO:journalctl@ceph.mon.c.smithi110.stdout: partition_filters: 0 2023-09-23T23:00:12.214 INFO:journalctl@ceph.mon.c.smithi110.stdout: use_delta_encoding: 1 2023-09-23T23:00:12.214 INFO:journalctl@ceph.mon.c.smithi110.stdout: filter_policy: bloomfilter 2023-09-23T23:00:12.214 INFO:journalctl@ceph.mon.c.smithi110.stdout: whole_key_filtering: 1 2023-09-23T23:00:12.215 INFO:journalctl@ceph.mon.c.smithi110.stdout: verify_compression: 0 2023-09-23T23:00:12.215 INFO:journalctl@ceph.mon.c.smithi110.stdout: read_amp_bytes_per_bit: 0 2023-09-23T23:00:12.215 INFO:journalctl@ceph.mon.c.smithi110.stdout: format_version: 5 2023-09-23T23:00:12.215 INFO:journalctl@ceph.mon.c.smithi110.stdout: enable_index_compression: 1 2023-09-23T23:00:12.215 INFO:journalctl@ceph.mon.c.smithi110.stdout: block_align: 0 2023-09-23T23:00:12.215 INFO:journalctl@ceph.mon.c.smithi110.stdout: max_auto_readahead_size: 262144 2023-09-23T23:00:12.215 INFO:journalctl@ceph.mon.c.smithi110.stdout: prepopulate_block_cache: 0 2023-09-23T23:00:12.215 INFO:journalctl@ceph.mon.c.smithi110.stdout: initial_auto_readahead_size: 8192 2023-09-23T23:00:12.215 INFO:journalctl@ceph.mon.c.smithi110.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T23:00:12.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T23:00:12.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T23:00:12.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compression: NoCompression 2023-09-23T23:00:12.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T23:00:12.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T23:00:12.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T23:00:12.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.num_levels: 7 2023-09-23T23:00:12.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T23:00:12.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T23:00:12.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T23:00:12.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T23:00:12.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T23:00:12.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T23:00:12.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T23:00:12.216 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T23:00:12.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T23:00:12.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T23:00:12.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T23:00:12.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T23:00:12.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T23:00:12.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T23:00:12.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T23:00:12.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T23:00:12.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T23:00:12.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T23:00:12.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T23:00:12.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T23:00:12.217 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T23:00:12.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T23:00:12.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T23:00:12.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T23:00:12.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T23:00:12.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T23:00:12.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T23:00:12.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T23:00:12.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T23:00:12.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T23:00:12.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T23:00:12.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T23:00:12.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T23:00:12.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T23:00:12.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T23:00:12.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T23:00:12.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T23:00:12.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T23:00:12.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T23:00:12.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T23:00:12.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T23:00:12.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T23:00:12.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T23:00:12.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T23:00:12.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T23:00:12.219 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T23:00:12.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T23:00:12.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T23:00:12.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T23:00:12.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T23:00:12.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T23:00:12.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T23:00:12.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T23:00:12.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.table_properties_collectors: 2023-09-23T23:00:12.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.inplace_update_support: 0 2023-09-23T23:00:12.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T23:00:12.220 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T23:00:12.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T23:00:12.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T23:00:12.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.bloom_locality: 0 2023-09-23T23:00:12.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.max_successive_merges: 0 2023-09-23T23:00:12.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T23:00:12.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T23:00:12.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T23:00:12.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T23:00:12.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.ttl: 2592000 2023-09-23T23:00:12.221 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T23:00:12.222 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T23:00:12.222 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T23:00:12.222 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.enable_blob_files: false 2023-09-23T23:00:12.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.min_blob_size: 0 2023-09-23T23:00:12.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T23:00:12.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T23:00:12.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T23:00:12.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T23:00:12.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T23:00:12.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T23:00:12.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T23:00:12.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T23:00:12.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T23:00:12.223 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T23:00:12.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12803, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T23:00:12.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T23:00:12.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 4ff42085-b787-466c-b926-baa3cbd57f9e 2023-09-23T23:00:12.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695510011717426, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T23:00:12.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T23:00:12.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 systemd[1]: Started Ceph mon.c for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:00:12.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695510011752001, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 4535566, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12808, "largest_seqno": 16079, "table_properties": {"data_size": 4523381, "index_size": 7772, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3397, "raw_key_size": 28986, "raw_average_key_size": 21, "raw_value_size": 4496613, "raw_average_value_size": 3335, "num_data_blocks": 352, "num_entries": 1348, "num_filter_entries": 1348, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695510011, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "4ff42085-b787-466c-b926-baa3cbd57f9e", "db_session_id": "U66FTIOWVQWE9QDQ0LQR", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T23:00:12.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695510011752129, "job": 1, "event": "recovery_finished"} 2023-09-23T23:00:12.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T23:00:12.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T23:00:12.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T23:00:12.224 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55c9b7f40000 2023-09-23T23:00:12.225 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: DB pointer 0x55c9b7f2e000 2023-09-23T23:00:12.225 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T23:00:12.225 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T23:00:12.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** DB Stats ** 2023-09-23T23:00:12.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: Uptime(secs): 0.1 total, 0.1 interval 2023-09-23T23:00:12.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T23:00:12.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T23:00:12.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T23:00:12.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T23:00:12.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T23:00:12.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T23:00:12.225 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2023-09-23T23:00:12.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** Compaction Stats [default] ** 2023-09-23T23:00:12.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T23:00:12.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T23:00:12.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: L0 1/0 4.33 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 280.8 0.02 0.00 1 0.015 0 0 0.0 0.0 2023-09-23T23:00:12.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: L6 1/0 5.27 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T23:00:12.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: Sum 2/0 9.59 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 280.8 0.02 0.00 1 0.015 0 0 0.0 0.0 2023-09-23T23:00:12.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 280.8 0.02 0.00 1 0.015 0 0 0.0 0.0 2023-09-23T23:00:12.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2023-09-23T23:00:12.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** Compaction Stats [default] ** 2023-09-23T23:00:12.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T23:00:12.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T23:00:12.226 INFO:journalctl@ceph.mon.c.smithi110.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 280.8 0.02 0.00 1 0.015 0 0 0.0 0.0 2023-09-23T23:00:12.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2023-09-23T23:00:12.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T23:00:12.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2023-09-23T23:00:12.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: Uptime(secs): 0.1 total, 0.1 interval 2023-09-23T23:00:12.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: Flush(GB): cumulative 0.004, interval 0.004 2023-09-23T23:00:12.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T23:00:12.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T23:00:12.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T23:00:12.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T23:00:12.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: Cumulative compaction: 0.00 GB write, 73.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T23:00:12.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: Interval compaction: 0.00 GB write, 73.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T23:00:12.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T23:00:12.227 INFO:journalctl@ceph.mon.c.smithi110.stdout: Block cache BinnedLRUCache@0x55c9b7ea5350#8 capacity: 512.00 MB usage: 56.53 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2.1e-05 secs_since: 0 2023-09-23T23:00:12.228 INFO:journalctl@ceph.mon.c.smithi110.stdout: Block cache entry stats(count,size,portion): DataBlock(4,23.31 KB,0.00444651%) FilterBlock(2,12.06 KB,0.00230074%) IndexBlock(2,21.16 KB,0.00403523%) Misc(1,0.00 KB,0%) 2023-09-23T23:00:12.228 INFO:journalctl@ceph.mon.c.smithi110.stdout: 2023-09-23T23:00:12.228 INFO:journalctl@ceph.mon.c.smithi110.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T23:00:12.228 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: starting mon.c rank 2 at public addrs [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] at bind addrs [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T23:00:12.228 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: mon.c@-1(???) e3 preinit fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T23:00:12.228 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:11 smithi110 ceph-mon[221102]: mon.c@-1(???).mds e9 new map 2023-09-23T23:00:13.027 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 74 16 111598 111582 94.1909 58.5 0.018676 0.0106117 2023-09-23T23:00:14.027 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 75 15 112546 112531 93.726 59.3125 0.0146323 0.0106652 2023-09-23T23:00:15.029 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 76 16 113493 113477 93.2682 59.125 0.0169001 0.010716 2023-09-23T23:00:16.029 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 77 16 114442 114426 92.8274 59.3125 0.00871538 0.0107683 2023-09-23T23:00:17.029 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 78 16 115370 115354 92.381 58 0.020629 0.0108199 2023-09-23T23:00:17.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:00:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:00:16] "GET /metrics HTTP/1.1" 200 36228 "" "Prometheus/2.43.0" 2023-09-23T23:00:18.029 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 79 16 116216 116200 91.8811 52.875 0.0242432 0.0108793 2023-09-23T23:00:18.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:00:18 smithi138 conmon[177361]: ts=2023-09-23T23:00:18.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T23:00:18.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:18 smithi138 ceph-mon[104418]: mon.c calling monitor election 2023-09-23T23:00:18.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:18 smithi138 ceph-mon[104418]: mon.b calling monitor election 2023-09-23T23:00:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:18 smithi138 ceph-mon[104418]: mon.a calling monitor election 2023-09-23T23:00:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:18 smithi138 ceph-mon[104418]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T23:00:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:18 smithi138 ceph-mon[104418]: monmap e3: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} 2023-09-23T23:00:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:18 smithi138 ceph-mon[104418]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T23:00:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:18 smithi138 ceph-mon[104418]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T23:00:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:18 smithi138 ceph-mon[104418]: mgrmap e34: y(active, since 95s), standbys: x 2023-09-23T23:00:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:18 smithi138 ceph-mon[104418]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T23:00:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:18 smithi138 ceph-mon[104418]: Cluster is now healthy 2023-09-23T23:00:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:18 smithi138 ceph-mon[104418]: overall HEALTH_OK 2023-09-23T23:00:19.000 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T23:00:19.033 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:00:19.033661+0000 min lat: 0.00133192 max lat: 0.317087 avg lat: 0.0109445 2023-09-23T23:00:19.033 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:00:19.034 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 80 16 116990 116974 91.3331 48.375 0.0205601 0.0109445 2023-09-23T23:00:19.194 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[212939]: mon.c calling monitor election 2023-09-23T23:00:19.195 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[212939]: mon.b calling monitor election 2023-09-23T23:00:19.195 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[212939]: mon.a calling monitor election 2023-09-23T23:00:19.195 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[212939]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T23:00:19.195 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[212939]: monmap e3: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} 2023-09-23T23:00:19.195 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[212939]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T23:00:19.195 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[212939]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T23:00:19.195 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[212939]: mgrmap e34: y(active, since 95s), standbys: x 2023-09-23T23:00:19.195 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[212939]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T23:00:19.195 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:00:19.195 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[212939]: overall HEALTH_OK 2023-09-23T23:00:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[221102]: mon.c calling monitor election 2023-09-23T23:00:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[221102]: mon.b calling monitor election 2023-09-23T23:00:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[221102]: mon.a calling monitor election 2023-09-23T23:00:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[221102]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T23:00:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[221102]: monmap e3: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} 2023-09-23T23:00:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[221102]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T23:00:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[221102]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T23:00:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[221102]: mgrmap e34: y(active, since 95s), standbys: x 2023-09-23T23:00:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[221102]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T23:00:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:00:19.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:18 smithi110 ceph-mon[221102]: overall HEALTH_OK 2023-09-23T23:00:20.033 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 81 16 117877 117861 90.89 55.4375 0.0110242 0.0109978 2023-09-23T23:00:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:19 smithi138 ceph-mon[104418]: pgmap v56: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s 2023-09-23T23:00:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:19 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3440179231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T23:00:20.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:19 smithi110 ceph-mon[221102]: pgmap v56: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s 2023-09-23T23:00:20.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:19 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3440179231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T23:00:20.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:19 smithi110 ceph-mon[212939]: pgmap v56: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s 2023-09-23T23:00:20.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:19 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3440179231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T23:00:21.033 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 82 16 118537 118521 90.2849 41.25 0.0139853 0.0110716 2023-09-23T23:00:21.172 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:20 smithi110 ceph-mon[221102]: mgrmap e35: y(active, since 96s), standbys: x 2023-09-23T23:00:21.178 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:20 smithi110 ceph-mon[212939]: mgrmap e35: y(active, since 96s), standbys: x 2023-09-23T23:00:21.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:20 smithi138 ceph-mon[104418]: mgrmap e35: y(active, since 96s), standbys: x 2023-09-23T23:00:21.227 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T23:00:22.034 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 83 16 119321 119305 89.7877 49 0.0210516 0.0111323 2023-09-23T23:00:22.130 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:22.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:21 smithi138 ceph-mon[104418]: pgmap v57: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-09-23T23:00:22.208 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:21 smithi110 ceph-mon[212939]: pgmap v57: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-09-23T23:00:22.208 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:00:21 smithi110 conmon[198813]: 2023-09-23T23:00:21.787+0000 7ff0150b6700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-09-23T23:00:22.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:21 smithi110 ceph-mon[221102]: pgmap v57: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-09-23T23:00:23.034 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 84 16 120102 120086 89.2996 48.8125 0.010768 0.0111935 2023-09-23T23:00:24.037 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 85 16 120968 120952 88.8859 54.125 0.0305635 0.0112465 2023-09-23T23:00:24.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:23 smithi138 ceph-mon[104418]: pgmap v58: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 46 MiB/s wr, 741 op/s 2023-09-23T23:00:24.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:23 smithi110 ceph-mon[221102]: pgmap v58: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 46 MiB/s wr, 741 op/s 2023-09-23T23:00:24.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:23 smithi110 ceph-mon[212939]: pgmap v58: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 46 MiB/s wr, 741 op/s 2023-09-23T23:00:25.036 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 86 16 121737 121721 88.4106 48.0625 0.0189416 0.0113056 2023-09-23T23:00:25.156 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T23:00:25.157 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (3m) - 7m 26.7M - 0.25.0 c8568f914cd2 d9641d6ed9ca 2023-09-23T23:00:25.157 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi138 *:3000 running (3m) - 7m 90.0M - 9.4.7 2c41d148cca3 56929ec304ec 2023-09-23T23:00:25.157 INFO:teuthology.orchestra.run.smithi110.stdout:mds.foo.smithi110.fuglpi smithi110 running (6m) - 6m 19.2M - 17.2.6-1125-gc9bb1bcf 444539aec5bf f59eb5601951 2023-09-23T23:00:25.157 INFO:teuthology.orchestra.run.smithi110.stdout:mds.foo.smithi138.xrewco smithi138 running (6m) - 6m 15.1M - 17.2.6-1125-gc9bb1bcf 444539aec5bf cf02b45fd734 2023-09-23T23:00:25.157 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi138 *:8443,9283,8765 running (2m) - 11m 447M - 18.2.0-460-gbb0daeff e32c2a64c126 fef2a6a3bd6d 2023-09-23T23:00:25.158 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (3m) - 13m 502M - 18.2.0-460-gbb0daeff e32c2a64c126 09d917770c6e 2023-09-23T23:00:25.158 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (112s) - 13m 48.2M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 d6c089d81bde 2023-09-23T23:00:25.158 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi138 running (12m) - 12m 89.2M 2048M 17.2.6-1125-gc9bb1bcf 444539aec5bf 522bb9710d62 2023-09-23T23:00:25.158 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 starting - - - 2048M 2023-09-23T23:00:25.161 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (3m) - 7m 17.0M - 1.5.0 0da6a335fe13 39d48d803cf2 2023-09-23T23:00:25.161 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi138 *:9100 running (3m) - 7m 18.8M - 1.5.0 0da6a335fe13 a55c5d37ba4b 2023-09-23T23:00:25.161 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (11m) - 11m 2656M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf d7807df7af7a 2023-09-23T23:00:25.161 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (11m) - 11m 1897M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 9cdbba4d4209 2023-09-23T23:00:25.161 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (10m) - 10m 3521M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 8350abe92da2 2023-09-23T23:00:25.161 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (10m) - 10m 2124M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf d34afe3cf9ea 2023-09-23T23:00:25.161 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi138 running (9m) - 9m 2734M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 47f7252bf067 2023-09-23T23:00:25.162 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi138 running (9m) - 9m 3007M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf fb3decabc2c4 2023-09-23T23:00:25.162 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi138 running (9m) - 9m 1437M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf c3f9d2352b7c 2023-09-23T23:00:25.162 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi138 running (8m) - 8m 2455M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 161d1d2d0445 2023-09-23T23:00:25.162 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi138 *:9095 running (2m) - 8m 55.2M - 2.43.0 a07b618ecd1d 8c2bb4977f43 2023-09-23T23:00:26.036 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 87 16 122524 122508 87.9595 49.1875 0.0179074 0.0113642 2023-09-23T23:00:26.349 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:25 smithi110 ceph-mon[212939]: pgmap v59: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T23:00:26.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:25 smithi110 ceph-mon[212939]: from='client.25096 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:00:26.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:25 smithi110 ceph-mon[221102]: pgmap v59: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T23:00:26.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:25 smithi110 ceph-mon[221102]: from='client.25096 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:00:26.389 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T23:00:26.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:25 smithi138 ceph-mon[104418]: pgmap v59: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T23:00:26.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:25 smithi138 ceph-mon[104418]: from='client.25096 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:00:27.043 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 88 16 123291 123275 87.5049 47.9375 0.0200248 0.0114235 2023-09-23T23:00:27.204 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:00:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:00:26] "GET /metrics HTTP/1.1" 200 36220 "" "Prometheus/2.43.0" 2023-09-23T23:00:27.257 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:27.664 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:27 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:27.665 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:27 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:27.665 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:27 smithi110 ceph-mon[212939]: pgmap v60: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s 2023-09-23T23:00:27.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:27 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:27.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:27 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:27.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:27 smithi110 ceph-mon[221102]: pgmap v60: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s 2023-09-23T23:00:27.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:27 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:27.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:27 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:27.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:27 smithi138 ceph-mon[104418]: pgmap v60: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s 2023-09-23T23:00:28.036 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 89 16 124023 124007 87.0359 45.75 0.0165278 0.0114843 2023-09-23T23:00:28.943 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:00:28 smithi138 conmon[177361]: ts=2023-09-23T23:00:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T23:00:28.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:28 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:28.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:28 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:28.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:28 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:29.036 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total time run: 90.01 2023-09-23T23:00:29.036 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total writes made: 124726 2023-09-23T23:00:29.036 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Write size: 65536 2023-09-23T23:00:29.036 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object size: 65536 2023-09-23T23:00:29.036 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Bandwidth (MB/sec): 86.6056 2023-09-23T23:00:29.036 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Bandwidth: 44.9318 2023-09-23T23:00:29.037 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max bandwidth (MB/sec): 261.438 2023-09-23T23:00:29.037 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min bandwidth (MB/sec): 41.25 2023-09-23T23:00:29.037 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average IOPS: 1385 2023-09-23T23:00:29.037 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev IOPS: 718.908 2023-09-23T23:00:29.037 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max IOPS: 4183 2023-09-23T23:00:29.037 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min IOPS: 660 2023-09-23T23:00:29.037 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average Latency(s): 0.011543 2023-09-23T23:00:29.037 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Latency(s): 0.00918169 2023-09-23T23:00:29.037 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max latency(s): 0.321701 2023-09-23T23:00:29.037 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min latency(s): 0.00133192 2023-09-23T23:00:29.044 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T23:00:29.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:29.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:29.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:29.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:29.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:29.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:29.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:29 smithi110 ceph-mon[221102]: pgmap v61: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s 2023-09-23T23:00:29.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:29 smithi110 ceph-mon[212939]: pgmap v61: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s 2023-09-23T23:00:30.014 INFO:teuthology.orchestra.run.smithi110.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:00:30.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:29 smithi138 ceph-mon[104418]: pgmap v61: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s 2023-09-23T23:00:31.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:31 smithi110 ceph-mon[212939]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:00:31.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:31 smithi110 ceph-mon[212939]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:00:31.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:31 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:31.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:31 smithi110 ceph-mon[221102]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:00:31.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:31 smithi110 ceph-mon[221102]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:00:31.366 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:31 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:31.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:31 smithi138 ceph-mon[104418]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:00:31.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:31 smithi138 ceph-mon[104418]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:00:31.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:31 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:31.611 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T23:00:32.390 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:32.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:32 smithi138 ceph-mon[104418]: pgmap v62: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-09-23T23:00:32.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:32 smithi110 ceph-mon[212939]: pgmap v62: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-09-23T23:00:32.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:32 smithi110 ceph-mon[221102]: pgmap v62: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-09-23T23:00:34.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:33 smithi138 ceph-mon[104418]: pgmap v63: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 40 MiB/s wr, 647 op/s 2023-09-23T23:00:34.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:33 smithi110 ceph-mon[212939]: pgmap v63: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 40 MiB/s wr, 647 op/s 2023-09-23T23:00:34.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:33 smithi110 ceph-mon[221102]: pgmap v63: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 40 MiB/s wr, 647 op/s 2023-09-23T23:00:36.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:35 smithi138 ceph-mon[104418]: pgmap v64: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 956 op/s 2023-09-23T23:00:36.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:35 smithi110 ceph-mon[212939]: pgmap v64: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 956 op/s 2023-09-23T23:00:36.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:35 smithi110 ceph-mon[221102]: pgmap v64: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 956 op/s 2023-09-23T23:00:37.158 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:00:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:00:36] "GET /metrics HTTP/1.1" 200 36217 "" "Prometheus/2.43.0" 2023-09-23T23:00:38.660 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:38 smithi138 ceph-mon[104418]: pgmap v65: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 625 op/s 2023-09-23T23:00:38.661 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:38 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:38.661 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:38 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:38.661 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:38 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:00:38.661 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:38 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:00:38.661 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:38 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:38.661 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:38 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:00:38.661 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:38 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:00:38.661 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:38 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:00:38.662 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:38 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:38.662 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:00:38 smithi138 conmon[177361]: ts=2023-09-23T23:00:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T23:00:38.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[221102]: pgmap v65: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 625 op/s 2023-09-23T23:00:38.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:00:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:00:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:00:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:00:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:00:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:38.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[212939]: pgmap v65: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 625 op/s 2023-09-23T23:00:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:00:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:00:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:00:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:00:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:00:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:38 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:00:40.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:40 smithi138 ceph-mon[104418]: pgmap v66: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 840 op/s 2023-09-23T23:00:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:40 smithi110 ceph-mon[212939]: pgmap v66: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 840 op/s 2023-09-23T23:00:40.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:40 smithi110 ceph-mon[221102]: pgmap v66: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 840 op/s 2023-09-23T23:00:42.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:42 smithi138 ceph-mon[104418]: pgmap v67: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 831 op/s 2023-09-23T23:00:42.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:42 smithi110 ceph-mon[221102]: pgmap v67: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 831 op/s 2023-09-23T23:00:42.712 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:42 smithi110 ceph-mon[212939]: pgmap v67: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 831 op/s 2023-09-23T23:00:44.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:43 smithi138 ceph-mon[104418]: pgmap v68: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 426 KiB/s wr, 654 op/s 2023-09-23T23:00:44.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:44.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:43 smithi110 ceph-mon[221102]: pgmap v68: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 426 KiB/s wr, 654 op/s 2023-09-23T23:00:44.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:44.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:43 smithi110 ceph-mon[212939]: pgmap v68: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 426 KiB/s wr, 654 op/s 2023-09-23T23:00:44.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:46.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:45 smithi138 ceph-mon[104418]: pgmap v69: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 426 KiB/s wr, 1.02k op/s 2023-09-23T23:00:46.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:45 smithi110 ceph-mon[212939]: pgmap v69: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 426 KiB/s wr, 1.02k op/s 2023-09-23T23:00:46.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:45 smithi110 ceph-mon[221102]: pgmap v69: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 426 KiB/s wr, 1.02k op/s 2023-09-23T23:00:47.204 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:00:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:00:46] "GET /metrics HTTP/1.1" 200 36219 "" "Prometheus/2.43.0" 2023-09-23T23:00:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:47 smithi138 ceph-mon[104418]: pgmap v70: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-09-23T23:00:48.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:47 smithi110 ceph-mon[212939]: pgmap v70: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-09-23T23:00:48.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:47 smithi110 ceph-mon[221102]: pgmap v70: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-09-23T23:00:48.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:00:48 smithi138 conmon[177361]: ts=2023-09-23T23:00:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T23:00:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:50 smithi138 ceph-mon[104418]: pgmap v71: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 929 op/s 2023-09-23T23:00:50.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:49 smithi110 ceph-mon[212939]: pgmap v71: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 929 op/s 2023-09-23T23:00:50.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:49 smithi110 ceph-mon[221102]: pgmap v71: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 929 op/s 2023-09-23T23:00:52.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:52 smithi138 ceph-mon[104418]: pgmap v72: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2023-09-23T23:00:52.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:52 smithi110 ceph-mon[212939]: pgmap v72: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2023-09-23T23:00:52.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:52 smithi110 ceph-mon[221102]: pgmap v72: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2023-09-23T23:00:54.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:53 smithi138 ceph-mon[104418]: pgmap v73: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-09-23T23:00:54.247 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:53 smithi110 ceph-mon[212939]: pgmap v73: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-09-23T23:00:54.247 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:53 smithi110 ceph-mon[221102]: pgmap v73: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-09-23T23:00:56.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:55 smithi138 ceph-mon[104418]: pgmap v74: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2023-09-23T23:00:56.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:55 smithi110 ceph-mon[212939]: pgmap v74: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2023-09-23T23:00:56.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:55 smithi110 ceph-mon[221102]: pgmap v74: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2023-09-23T23:00:57.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:00:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:00:56] "GET /metrics HTTP/1.1" 200 36219 "" "Prometheus/2.43.0" 2023-09-23T23:00:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:57 smithi138 ceph-mon[104418]: pgmap v75: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T23:00:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:00:57 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:58.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:57 smithi110 ceph-mon[212939]: pgmap v75: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T23:00:58.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:00:57 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:58.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:57 smithi110 ceph-mon[221102]: pgmap v75: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T23:00:58.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:00:57 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:58.942 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:00:58 smithi138 conmon[177361]: ts=2023-09-23T23:00:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T23:01:00.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:00 smithi138 ceph-mon[104418]: pgmap v76: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-09-23T23:01:00.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:00 smithi110 ceph-mon[221102]: pgmap v76: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-09-23T23:01:00.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:00 smithi110 ceph-mon[212939]: pgmap v76: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-09-23T23:01:02.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:02 smithi138 ceph-mon[104418]: pgmap v77: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2023-09-23T23:01:02.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:02 smithi110 ceph-mon[221102]: pgmap v77: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2023-09-23T23:01:02.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:02 smithi110 ceph-mon[212939]: pgmap v77: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2023-09-23T23:01:04.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:03 smithi138 ceph-mon[104418]: pgmap v78: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-09-23T23:01:04.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:03 smithi110 ceph-mon[212939]: pgmap v78: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-09-23T23:01:04.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:03 smithi110 ceph-mon[221102]: pgmap v78: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-09-23T23:01:06.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:05 smithi138 ceph-mon[104418]: pgmap v79: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2023-09-23T23:01:06.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:05 smithi110 ceph-mon[221102]: pgmap v79: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2023-09-23T23:01:06.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:05 smithi110 ceph-mon[212939]: pgmap v79: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2023-09-23T23:01:07.205 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:01:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:01:06] "GET /metrics HTTP/1.1" 200 36233 "" "Prometheus/2.43.0" 2023-09-23T23:01:08.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:07 smithi138 ceph-mon[104418]: pgmap v80: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-09-23T23:01:08.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:07 smithi110 ceph-mon[221102]: pgmap v80: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-09-23T23:01:08.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:07 smithi110 ceph-mon[212939]: pgmap v80: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-09-23T23:01:08.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:01:08 smithi138 conmon[177361]: ts=2023-09-23T23:01:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T23:01:10.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:10 smithi138 ceph-mon[104418]: pgmap v81: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2023-09-23T23:01:10.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:10 smithi110 ceph-mon[221102]: pgmap v81: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2023-09-23T23:01:10.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:10 smithi110 ceph-mon[212939]: pgmap v81: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2023-09-23T23:01:12.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:12 smithi138 ceph-mon[104418]: pgmap v82: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2023-09-23T23:01:12.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:12 smithi110 ceph-mon[221102]: pgmap v82: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2023-09-23T23:01:12.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:12 smithi110 ceph-mon[212939]: pgmap v82: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2023-09-23T23:01:14.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:13 smithi138 ceph-mon[104418]: pgmap v83: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-09-23T23:01:14.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:13 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:14.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:13 smithi110 ceph-mon[212939]: pgmap v83: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-09-23T23:01:14.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:14.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:13 smithi110 ceph-mon[221102]: pgmap v83: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-09-23T23:01:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:16.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:15 smithi138 ceph-mon[104418]: pgmap v84: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2023-09-23T23:01:16.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:15 smithi110 ceph-mon[212939]: pgmap v84: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2023-09-23T23:01:16.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:15 smithi110 ceph-mon[221102]: pgmap v84: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2023-09-23T23:01:17.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:01:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:01:16] "GET /metrics HTTP/1.1" 200 36231 "" "Prometheus/2.43.0" 2023-09-23T23:01:18.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:18 smithi138 ceph-mon[104418]: pgmap v85: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T23:01:18.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:18 smithi110 ceph-mon[212939]: pgmap v85: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T23:01:18.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:18 smithi110 ceph-mon[221102]: pgmap v85: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T23:01:18.942 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:01:18 smithi138 conmon[177361]: ts=2023-09-23T23:01:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T23:01:20.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:20 smithi138 ceph-mon[104418]: pgmap v86: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-09-23T23:01:20.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:20 smithi110 ceph-mon[221102]: pgmap v86: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-09-23T23:01:20.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:20 smithi110 ceph-mon[212939]: pgmap v86: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-09-23T23:01:22.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:22 smithi138 ceph-mon[104418]: pgmap v87: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-09-23T23:01:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:22 smithi110 ceph-mon[221102]: pgmap v87: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-09-23T23:01:22.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:22 smithi110 ceph-mon[212939]: pgmap v87: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-09-23T23:01:24.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:23 smithi138 ceph-mon[104418]: pgmap v88: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T23:01:24.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:23 smithi110 ceph-mon[221102]: pgmap v88: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T23:01:24.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:23 smithi110 ceph-mon[212939]: pgmap v88: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T23:01:26.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:26 smithi138 ceph-mon[104418]: pgmap v89: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-09-23T23:01:26.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:26 smithi110 ceph-mon[221102]: pgmap v89: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-09-23T23:01:26.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:26 smithi110 ceph-mon[212939]: pgmap v89: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-09-23T23:01:27.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:01:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:01:26] "GET /metrics HTTP/1.1" 200 36231 "" "Prometheus/2.43.0" 2023-09-23T23:01:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:28 smithi138 ceph-mon[104418]: pgmap v90: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T23:01:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:28 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:28.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:28 smithi110 ceph-mon[212939]: pgmap v90: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T23:01:28.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:28.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:28 smithi110 ceph-mon[221102]: pgmap v90: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T23:01:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:28.943 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:01:28 smithi138 conmon[177361]: ts=2023-09-23T23:01:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T23:01:30.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:30 smithi138 ceph-mon[104418]: pgmap v91: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-09-23T23:01:30.460 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:30 smithi110 ceph-mon[212939]: pgmap v91: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-09-23T23:01:30.460 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:30 smithi110 ceph-mon[221102]: pgmap v91: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-09-23T23:01:32.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:32 smithi138 ceph-mon[104418]: pgmap v92: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-09-23T23:01:32.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:32 smithi110 ceph-mon[212939]: pgmap v92: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-09-23T23:01:32.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:32 smithi110 ceph-mon[221102]: pgmap v92: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-09-23T23:01:34.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:33 smithi138 ceph-mon[104418]: pgmap v93: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T23:01:34.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:33 smithi110 ceph-mon[212939]: pgmap v93: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T23:01:34.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:33 smithi110 ceph-mon[221102]: pgmap v93: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T23:01:36.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:35 smithi110 ceph-mon[212939]: pgmap v94: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2023-09-23T23:01:36.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:35 smithi110 ceph-mon[221102]: pgmap v94: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2023-09-23T23:01:36.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:35 smithi138 ceph-mon[104418]: pgmap v94: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2023-09-23T23:01:37.118 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:01:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:01:36] "GET /metrics HTTP/1.1" 200 36226 "" "Prometheus/2.43.0" 2023-09-23T23:01:37.143 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T23:01:37.889 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:38.277 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:38 smithi110 ceph-mon[212939]: pgmap v95: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T23:01:38.278 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:38 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:01:38.278 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:38 smithi110 ceph-mon[221102]: pgmap v95: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T23:01:38.278 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:38 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:01:38.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:38 smithi138 ceph-mon[104418]: pgmap v95: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T23:01:38.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:38 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:01:38.941 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:01:38 smithi138 conmon[177361]: ts=2023-09-23T23:01:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T23:01:40.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:40 smithi138 ceph-mon[104418]: pgmap v96: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-09-23T23:01:40.461 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:40 smithi110 ceph-mon[212939]: pgmap v96: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-09-23T23:01:40.461 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:40 smithi110 ceph-mon[221102]: pgmap v96: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-09-23T23:01:40.862 INFO:teuthology.orchestra.run.smithi110.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:01:41.865 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for all mons to upgrade' 2023-09-23T23:01:42.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:41 smithi138 ceph-mon[104418]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:01:42.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:41 smithi138 ceph-mon[104418]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:01:42.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:41 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:42.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:41 smithi138 ceph-mon[104418]: pgmap v97: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2023-09-23T23:01:42.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:41 smithi110 ceph-mon[221102]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:01:42.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:41 smithi110 ceph-mon[221102]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:01:42.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:41 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:42.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:41 smithi110 ceph-mon[221102]: pgmap v97: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2023-09-23T23:01:42.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:41 smithi110 ceph-mon[212939]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:01:42.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:41 smithi110 ceph-mon[212939]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:01:42.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:41 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:42.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:41 smithi110 ceph-mon[212939]: pgmap v97: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2023-09-23T23:01:42.843 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:43.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:43 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:43.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:43 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:43.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:43 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:43.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:43.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:43.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:44.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:44 smithi138 ceph-mon[104418]: pgmap v98: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T23:01:44.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:44 smithi110 ceph-mon[221102]: pgmap v98: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T23:01:44.459 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:44 smithi110 ceph-mon[212939]: pgmap v98: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T23:01:45.518 INFO:teuthology.orchestra.run.smithi110.stdout:wait for all mons to upgrade 2023-09-23T23:01:46.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:46 smithi138 ceph-mon[104418]: pgmap v99: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2023-09-23T23:01:46.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:46 smithi110 ceph-mon[221102]: pgmap v99: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2023-09-23T23:01:46.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:46 smithi110 ceph-mon[212939]: pgmap v99: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2023-09-23T23:01:46.640 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-09-23T23:01:46.958 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:01:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:01:46] "GET /metrics HTTP/1.1" 200 36228 "" "Prometheus/2.43.0" 2023-09-23T23:01:47.481 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:48.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:48 smithi110 ceph-mon[212939]: pgmap v100: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-09-23T23:01:48.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:48.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:48.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:48 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:01:48.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:48 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:01:48.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:48.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:48 smithi110 ceph-mon[221102]: pgmap v100: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-09-23T23:01:48.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:48 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:48.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:48 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:48.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:48 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:01:48.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:48 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:01:48.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:48 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:48.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:48 smithi138 ceph-mon[104418]: pgmap v100: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-09-23T23:01:48.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:48 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:48.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:48 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:48.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:48 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:01:48.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:48 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:01:48.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:48 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:48.913 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:01:48 smithi138 conmon[177361]: ts=2023-09-23T23:01:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi138.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.110\", device_class=\"ssd\", hostname=\"smithi110\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.110\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T23:01:49.375 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:49 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:01:49.375 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:49 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:01:49.375 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:49 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T23:01:49.375 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:49 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T23:01:49.376 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:49 smithi138 ceph-mon[104418]: Upgrade: It appears safe to stop mon.b 2023-09-23T23:01:49.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:49 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:01:49.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:49 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:01:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:49 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T23:01:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:49 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T23:01:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:49 smithi110 ceph-mon[221102]: Upgrade: It appears safe to stop mon.b 2023-09-23T23:01:49.463 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:49 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:01:49.463 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:49 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:01:49.463 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:49 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T23:01:49.463 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:49 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T23:01:49.463 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:49 smithi110 ceph-mon[212939]: Upgrade: It appears safe to stop mon.b 2023-09-23T23:01:50.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:50 smithi138 ceph-mon[104418]: pgmap v101: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T23:01:50.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:50 smithi110 ceph-mon[212939]: pgmap v101: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T23:01:50.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:50 smithi110 ceph-mon[221102]: pgmap v101: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T23:01:51.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:51 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/3657970009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T23:01:51.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:51 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3657970009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T23:01:51.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:51 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3657970009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T23:01:52.393 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:52 smithi138 ceph-mon[104418]: pgmap v102: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2023-09-23T23:01:52.393 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:52 smithi138 ceph-mon[104418]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:52.393 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:52 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T23:01:52.393 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:52 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T23:01:52.393 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:52 smithi138 ceph-mon[104418]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:01:52.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:52 smithi110 ceph-mon[212939]: pgmap v102: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2023-09-23T23:01:52.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:52 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:52.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:52 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T23:01:52.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:52 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T23:01:52.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:52 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:01:52.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:52 smithi110 ceph-mon[221102]: pgmap v102: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2023-09-23T23:01:52.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:52 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:01:52.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:52 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T23:01:52.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:52 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T23:01:52.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:52 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:01:53.449 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:53 smithi138 ceph-mon[104418]: Upgrade: Updating mon.b 2023-09-23T23:01:53.449 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:53 smithi138 ceph-mon[104418]: Deploying daemon mon.b on smithi138 2023-09-23T23:01:53.449 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:53 smithi138 ceph-mon[104418]: from='client.? 172.21.15.110:0/2489947930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T23:01:53.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:53 smithi110 ceph-mon[212939]: Upgrade: Updating mon.b 2023-09-23T23:01:53.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:53 smithi110 ceph-mon[212939]: Deploying daemon mon.b on smithi138 2023-09-23T23:01:53.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:53 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2489947930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T23:01:53.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:53 smithi110 ceph-mon[221102]: Upgrade: Updating mon.b 2023-09-23T23:01:53.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:53 smithi110 ceph-mon[221102]: Deploying daemon mon.b on smithi138 2023-09-23T23:01:53.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:53 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2489947930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T23:01:53.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:53 smithi138 systemd[1]: Stopping Ceph mon.b for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:01:54.414 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:53 smithi138 conmon[104364]: 2023-09-23T23:01:53.952+0000 7fa7ac827700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:01:54.414 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:53 smithi138 conmon[104364]: 2023-09-23T23:01:53.952+0000 7fa7ac827700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-09-23T23:01:54.415 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:54 smithi138 bash[193204]: 522bb9710d62f912e2235cecae3d663239cb620a3ec9cb74177a6270ab70d783 2023-09-23T23:01:54.453 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:01:54 smithi110 conmon[198813]: [23/Sep/2023:23:01:54] ENGINE Bus STOPPING 2023-09-23T23:01:54.453 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:01:54 smithi110 conmon[198813]: [23/Sep/2023:23:01:54] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T23:01:54.454 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:01:54 smithi110 conmon[198813]: [23/Sep/2023:23:01:54] ENGINE Bus STOPPED 2023-09-23T23:01:54.454 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:01:54 smithi110 conmon[198813]: [23/Sep/2023:23:01:54] ENGINE Bus STARTING 2023-09-23T23:01:54.454 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:01:54 smithi110 conmon[198813]: [23/Sep/2023:23:01:54] ENGINE Serving on http://:::9283 2023-09-23T23:01:54.454 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:01:54 smithi110 conmon[198813]: [23/Sep/2023:23:01:54] ENGINE Bus STARTED 2023-09-23T23:01:55.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:55 smithi138 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mon.b.service: Succeeded. 2023-09-23T23:01:55.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:55 smithi138 systemd[1]: Stopped Ceph mon.b for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:01:55.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:55 smithi138 systemd[1]: Starting Ceph mon.b for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:01:56.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T23:01:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T23:01:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: pidfile_write: ignore empty --pid-file 2023-09-23T23:01:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 bash[193540]: 42bfa5f10303324b8b746c6e95121336f0c60a98961946d816425d3d74f7ac86 2023-09-23T23:01:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: load: jerasure load: lrc 2023-09-23T23:01:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: RocksDB version: 7.9.2 2023-09-23T23:01:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Git sha 0 2023-09-23T23:01:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T23:01:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: DB SUMMARY 2023-09-23T23:01:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: DB Session ID: VNN3Z12073KO4GF5TPWW 2023-09-23T23:01:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: CURRENT file: CURRENT 2023-09-23T23:01:56.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T23:01:56.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1094 Bytes 2023-09-23T23:01:56.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T23:01:56.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 9425574 ; 2023-09-23T23:01:56.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.error_if_exists: 0 2023-09-23T23:01:56.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.create_if_missing: 0 2023-09-23T23:01:56.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.paranoid_checks: 1 2023-09-23T23:01:56.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T23:01:56.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T23:01:56.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T23:01:56.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.env: 0x55e422796720 2023-09-23T23:01:56.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T23:01:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.info_log: 0x55e424ba1420 2023-09-23T23:01:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T23:01:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.statistics: (nil) 2023-09-23T23:01:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.use_fsync: 0 2023-09-23T23:01:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_log_file_size: 0 2023-09-23T23:01:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T23:01:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T23:01:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T23:01:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T23:01:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.allow_fallocate: 1 2023-09-23T23:01:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T23:01:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T23:01:56.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.use_direct_reads: 0 2023-09-23T23:01:56.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T23:01:56.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T23:01:56.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.db_log_dir: 2023-09-23T23:01:56.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.wal_dir: 2023-09-23T23:01:56.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T23:01:56.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T23:01:56.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T23:01:56.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T23:01:56.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T23:01:56.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T23:01:56.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T23:01:56.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T23:01:56.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.write_buffer_manager: 0x55e423e30c80 2023-09-23T23:01:56.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T23:01:56.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T23:01:56.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T23:01:56.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T23:01:56.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T23:01:56.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T23:01:56.449 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T23:01:56.449 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T23:01:56.449 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.unordered_write: 0 2023-09-23T23:01:56.449 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T23:01:56.449 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T23:01:56.449 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T23:01:56.449 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T23:01:56.449 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.row_cache: None 2023-09-23T23:01:56.449 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.wal_filter: None 2023-09-23T23:01:56.450 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T23:01:56.450 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T23:01:56.450 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.two_write_queues: 0 2023-09-23T23:01:56.450 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T23:01:56.450 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.wal_compression: 0 2023-09-23T23:01:56.450 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.atomic_flush: 0 2023-09-23T23:01:56.450 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T23:01:56.450 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T23:01:56.450 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T23:01:56.450 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.log_readahead_size: 0 2023-09-23T23:01:56.450 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T23:01:56.451 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T23:01:56.451 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T23:01:56.451 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T23:01:56.451 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T23:01:56.451 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T23:01:56.451 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T23:01:56.451 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_background_jobs: 2 2023-09-23T23:01:56.451 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_background_compactions: -1 2023-09-23T23:01:56.451 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_subcompactions: 1 2023-09-23T23:01:56.451 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T23:01:56.451 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T23:01:56.451 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T23:01:56.451 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T23:01:56.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T23:01:56.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T23:01:56.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T23:01:56.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T23:01:56.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_open_files: -1 2023-09-23T23:01:56.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T23:01:56.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T23:01:56.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T23:01:56.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T23:01:56.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_background_flushes: -1 2023-09-23T23:01:56.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Compression algorithms supported: 2023-09-23T23:01:56.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T23:01:56.452 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: kZSTD supported: 0 2023-09-23T23:01:56.453 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: kXpressCompression supported: 0 2023-09-23T23:01:56.453 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T23:01:56.453 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: kZlibCompression supported: 1 2023-09-23T23:01:56.453 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: kSnappyCompression supported: 1 2023-09-23T23:01:56.453 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: kLZ4Compression supported: 1 2023-09-23T23:01:56.453 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: kBZip2Compression supported: 0 2023-09-23T23:01:56.453 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T23:01:56.453 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T23:01:56.453 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2023-09-23T23:01:56.453 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T23:01:56.453 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T23:01:56.453 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.merge_operator: 2023-09-23T23:01:56.454 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compaction_filter: None 2023-09-23T23:01:56.454 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T23:01:56.454 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T23:01:56.454 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T23:01:56.454 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T23:01:56.454 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e424ba1520) 2023-09-23T23:01:56.454 INFO:journalctl@ceph.mon.b.smithi138.stdout: cache_index_and_filter_blocks: 1 2023-09-23T23:01:56.454 INFO:journalctl@ceph.mon.b.smithi138.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T23:01:56.454 INFO:journalctl@ceph.mon.b.smithi138.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T23:01:56.454 INFO:journalctl@ceph.mon.b.smithi138.stdout: pin_top_level_index_and_filter: 1 2023-09-23T23:01:56.454 INFO:journalctl@ceph.mon.b.smithi138.stdout: index_type: 0 2023-09-23T23:01:56.454 INFO:journalctl@ceph.mon.b.smithi138.stdout: data_block_index_type: 0 2023-09-23T23:01:56.455 INFO:journalctl@ceph.mon.b.smithi138.stdout: index_shortening: 1 2023-09-23T23:01:56.455 INFO:journalctl@ceph.mon.b.smithi138.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T23:01:56.455 INFO:journalctl@ceph.mon.b.smithi138.stdout: checksum: 4 2023-09-23T23:01:56.455 INFO:journalctl@ceph.mon.b.smithi138.stdout: no_block_cache: 0 2023-09-23T23:01:56.455 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_cache: 0x55e423eb3350 2023-09-23T23:01:56.455 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_cache_name: BinnedLRUCache 2023-09-23T23:01:56.455 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_cache_options: 2023-09-23T23:01:56.455 INFO:journalctl@ceph.mon.b.smithi138.stdout: capacity : 536870912 2023-09-23T23:01:56.455 INFO:journalctl@ceph.mon.b.smithi138.stdout: num_shard_bits : 4 2023-09-23T23:01:56.455 INFO:journalctl@ceph.mon.b.smithi138.stdout: strict_capacity_limit : 0 2023-09-23T23:01:56.455 INFO:journalctl@ceph.mon.b.smithi138.stdout: high_pri_pool_ratio: 0.000 2023-09-23T23:01:56.455 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_cache_compressed: (nil) 2023-09-23T23:01:56.455 INFO:journalctl@ceph.mon.b.smithi138.stdout: persistent_cache: (nil) 2023-09-23T23:01:56.456 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_size: 4096 2023-09-23T23:01:56.456 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_size_deviation: 10 2023-09-23T23:01:56.456 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_restart_interval: 16 2023-09-23T23:01:56.456 INFO:journalctl@ceph.mon.b.smithi138.stdout: index_block_restart_interval: 1 2023-09-23T23:01:56.456 INFO:journalctl@ceph.mon.b.smithi138.stdout: metadata_block_size: 4096 2023-09-23T23:01:56.456 INFO:journalctl@ceph.mon.b.smithi138.stdout: partition_filters: 0 2023-09-23T23:01:56.456 INFO:journalctl@ceph.mon.b.smithi138.stdout: use_delta_encoding: 1 2023-09-23T23:01:56.456 INFO:journalctl@ceph.mon.b.smithi138.stdout: filter_policy: bloomfilter 2023-09-23T23:01:56.456 INFO:journalctl@ceph.mon.b.smithi138.stdout: whole_key_filtering: 1 2023-09-23T23:01:56.456 INFO:journalctl@ceph.mon.b.smithi138.stdout: verify_compression: 0 2023-09-23T23:01:56.456 INFO:journalctl@ceph.mon.b.smithi138.stdout: read_amp_bytes_per_bit: 0 2023-09-23T23:01:56.456 INFO:journalctl@ceph.mon.b.smithi138.stdout: format_version: 5 2023-09-23T23:01:56.457 INFO:journalctl@ceph.mon.b.smithi138.stdout: enable_index_compression: 1 2023-09-23T23:01:56.457 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_align: 0 2023-09-23T23:01:56.457 INFO:journalctl@ceph.mon.b.smithi138.stdout: max_auto_readahead_size: 262144 2023-09-23T23:01:56.457 INFO:journalctl@ceph.mon.b.smithi138.stdout: prepopulate_block_cache: 0 2023-09-23T23:01:56.457 INFO:journalctl@ceph.mon.b.smithi138.stdout: initial_auto_readahead_size: 8192 2023-09-23T23:01:56.457 INFO:journalctl@ceph.mon.b.smithi138.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T23:01:56.457 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T23:01:56.457 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T23:01:56.457 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compression: NoCompression 2023-09-23T23:01:56.457 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T23:01:56.457 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T23:01:56.457 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T23:01:56.457 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.num_levels: 7 2023-09-23T23:01:56.458 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T23:01:56.458 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T23:01:56.458 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T23:01:56.458 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T23:01:56.458 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T23:01:56.458 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T23:01:56.458 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T23:01:56.458 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T23:01:56.458 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T23:01:56.458 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T23:01:56.458 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T23:01:56.458 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T23:01:56.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T23:01:56.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T23:01:56.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T23:01:56.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T23:01:56.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T23:01:56.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T23:01:56.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T23:01:56.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T23:01:56.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T23:01:56.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T23:01:56.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T23:01:56.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T23:01:56.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T23:01:56.460 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T23:01:56.460 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T23:01:56.460 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T23:01:56.460 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T23:01:56.460 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T23:01:56.460 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T23:01:56.460 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T23:01:56.460 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T23:01:56.460 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T23:01:56.460 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T23:01:56.460 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T23:01:56.460 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T23:01:56.461 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T23:01:56.461 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T23:01:56.461 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T23:01:56.461 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T23:01:56.461 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T23:01:56.461 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T23:01:56.461 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T23:01:56.461 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T23:01:56.461 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T23:01:56.461 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T23:01:56.461 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T23:01:56.461 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T23:01:56.461 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T23:01:56.462 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T23:01:56.462 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T23:01:56.462 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T23:01:56.462 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.table_properties_collectors: 2023-09-23T23:01:56.462 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.inplace_update_support: 0 2023-09-23T23:01:56.462 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T23:01:56.462 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T23:01:56.462 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T23:01:56.462 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T23:01:56.462 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.bloom_locality: 0 2023-09-23T23:01:56.462 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.max_successive_merges: 0 2023-09-23T23:01:56.462 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T23:01:56.463 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T23:01:56.463 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T23:01:56.463 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T23:01:56.463 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.ttl: 2592000 2023-09-23T23:01:56.463 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T23:01:56.463 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T23:01:56.463 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T23:01:56.463 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.enable_blob_files: false 2023-09-23T23:01:56.463 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.min_blob_size: 0 2023-09-23T23:01:56.463 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T23:01:56.463 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T23:01:56.463 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T23:01:56.464 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T23:01:56.464 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T23:01:56.464 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T23:01:56.464 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T23:01:56.464 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T23:01:56.464 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T23:01:56.464 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T23:01:56.464 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12884, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T23:01:56.464 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T23:01:56.464 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3f235c90-35dc-4a63-b01f-cd2acfe6f82c 2023-09-23T23:01:56.464 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695510116161670, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T23:01:56.464 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T23:01:56.464 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 systemd[1]: Started Ceph mon.b for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:01:56.465 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695510116198317, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 5722498, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12890, "largest_seqno": 17477, "table_properties": {"data_size": 5706388, "index_size": 10737, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4357, "raw_key_size": 36881, "raw_average_key_size": 21, "raw_value_size": 5671589, "raw_average_value_size": 3282, "num_data_blocks": 484, "num_entries": 1728, "num_filter_entries": 1728, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695510116, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3f235c90-35dc-4a63-b01f-cd2acfe6f82c", "db_session_id": "VNN3Z12073KO4GF5TPWW", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T23:01:56.465 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695510116198443, "job": 1, "event": "recovery_finished"} 2023-09-23T23:01:56.465 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T23:01:56.465 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T23:01:56.465 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T23:01:56.465 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e423f4e000 2023-09-23T23:01:56.465 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: DB pointer 0x55e423f3c000 2023-09-23T23:01:56.465 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T23:01:56.465 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T23:01:56.465 INFO:journalctl@ceph.mon.b.smithi138.stdout: ** DB Stats ** 2023-09-23T23:01:56.465 INFO:journalctl@ceph.mon.b.smithi138.stdout: Uptime(secs): 0.1 total, 0.1 interval 2023-09-23T23:01:56.465 INFO:journalctl@ceph.mon.b.smithi138.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T23:01:56.466 INFO:journalctl@ceph.mon.b.smithi138.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T23:01:56.466 INFO:journalctl@ceph.mon.b.smithi138.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T23:01:56.466 INFO:journalctl@ceph.mon.b.smithi138.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T23:01:56.466 INFO:journalctl@ceph.mon.b.smithi138.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T23:01:56.466 INFO:journalctl@ceph.mon.b.smithi138.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T23:01:56.466 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2023-09-23T23:01:56.466 INFO:journalctl@ceph.mon.b.smithi138.stdout: ** Compaction Stats [default] ** 2023-09-23T23:01:56.466 INFO:journalctl@ceph.mon.b.smithi138.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T23:01:56.466 INFO:journalctl@ceph.mon.b.smithi138.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T23:01:56.466 INFO:journalctl@ceph.mon.b.smithi138.stdout: L0 1/0 5.46 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 350.0 0.02 0.00 1 0.016 0 0 0.0 0.0 2023-09-23T23:01:56.466 INFO:journalctl@ceph.mon.b.smithi138.stdout: L6 1/0 5.27 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T23:01:56.467 INFO:journalctl@ceph.mon.b.smithi138.stdout: Sum 2/0 10.73 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 350.0 0.02 0.00 1 0.016 0 0 0.0 0.0 2023-09-23T23:01:56.467 INFO:journalctl@ceph.mon.b.smithi138.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 350.0 0.02 0.00 1 0.016 0 0 0.0 0.0 2023-09-23T23:01:56.467 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2023-09-23T23:01:56.467 INFO:journalctl@ceph.mon.b.smithi138.stdout: ** Compaction Stats [default] ** 2023-09-23T23:01:56.467 INFO:journalctl@ceph.mon.b.smithi138.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T23:01:56.467 INFO:journalctl@ceph.mon.b.smithi138.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T23:01:56.467 INFO:journalctl@ceph.mon.b.smithi138.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 350.0 0.02 0.00 1 0.016 0 0 0.0 0.0 2023-09-23T23:01:56.467 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2023-09-23T23:01:56.467 INFO:journalctl@ceph.mon.b.smithi138.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T23:01:56.467 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2023-09-23T23:01:56.467 INFO:journalctl@ceph.mon.b.smithi138.stdout: Uptime(secs): 0.1 total, 0.1 interval 2023-09-23T23:01:56.467 INFO:journalctl@ceph.mon.b.smithi138.stdout: Flush(GB): cumulative 0.005, interval 0.005 2023-09-23T23:01:56.467 INFO:journalctl@ceph.mon.b.smithi138.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T23:01:56.468 INFO:journalctl@ceph.mon.b.smithi138.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T23:01:56.468 INFO:journalctl@ceph.mon.b.smithi138.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T23:01:56.468 INFO:journalctl@ceph.mon.b.smithi138.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T23:01:56.468 INFO:journalctl@ceph.mon.b.smithi138.stdout: Cumulative compaction: 0.01 GB write, 101.79 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T23:01:56.468 INFO:journalctl@ceph.mon.b.smithi138.stdout: Interval compaction: 0.01 GB write, 101.79 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T23:01:56.468 INFO:journalctl@ceph.mon.b.smithi138.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T23:01:56.468 INFO:journalctl@ceph.mon.b.smithi138.stdout: Block cache BinnedLRUCache@0x55e423eb3350#7 capacity: 512.00 MB usage: 16.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2e-05 secs_since: 0 2023-09-23T23:01:56.468 INFO:journalctl@ceph.mon.b.smithi138.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.53 KB,0.000864267%) IndexBlock(1,12.08 KB,0.00230372%) Misc(1,0.00 KB,0%) 2023-09-23T23:01:56.468 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2023-09-23T23:01:56.468 INFO:journalctl@ceph.mon.b.smithi138.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T23:01:56.468 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: starting mon.b rank 1 at public addrs [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] at bind addrs [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T23:01:56.468 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: mon.b@-1(???) e3 preinit fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T23:01:56.469 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: mon.b@-1(???).mds e9 new map 2023-09-23T23:01:56.469 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: mon.b@-1(???).mds e9 print_map 2023-09-23T23:01:56.469 INFO:journalctl@ceph.mon.b.smithi138.stdout: e9 2023-09-23T23:01:56.469 INFO:journalctl@ceph.mon.b.smithi138.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T23:01:56.469 INFO:journalctl@ceph.mon.b.smithi138.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T23:01:56.469 INFO:journalctl@ceph.mon.b.smithi138.stdout: legacy client fscid: 1 2023-09-23T23:01:56.469 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2023-09-23T23:01:56.469 INFO:journalctl@ceph.mon.b.smithi138.stdout: Filesystem 'foo' (1) 2023-09-23T23:01:56.469 INFO:journalctl@ceph.mon.b.smithi138.stdout: fs_name foo 2023-09-23T23:01:56.469 INFO:journalctl@ceph.mon.b.smithi138.stdout: epoch 8 2023-09-23T23:01:56.469 INFO:journalctl@ceph.mon.b.smithi138.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T23:01:56.469 INFO:journalctl@ceph.mon.b.smithi138.stdout: created 2023-09-23T22:53:16.882348+0000 2023-09-23T23:01:56.470 INFO:journalctl@ceph.mon.b.smithi138.stdout: modified 2023-09-23T22:53:41.314442+0000 2023-09-23T23:01:56.470 INFO:journalctl@ceph.mon.b.smithi138.stdout: tableserver 0 2023-09-23T23:01:56.470 INFO:journalctl@ceph.mon.b.smithi138.stdout: root 0 2023-09-23T23:01:56.470 INFO:journalctl@ceph.mon.b.smithi138.stdout: session_timeout 60 2023-09-23T23:01:56.470 INFO:journalctl@ceph.mon.b.smithi138.stdout: session_autoclose 300 2023-09-23T23:01:56.470 INFO:journalctl@ceph.mon.b.smithi138.stdout: max_file_size 1099511627776 2023-09-23T23:01:56.470 INFO:journalctl@ceph.mon.b.smithi138.stdout: required_client_features {} 2023-09-23T23:01:56.470 INFO:journalctl@ceph.mon.b.smithi138.stdout: last_failure 0 2023-09-23T23:01:56.470 INFO:journalctl@ceph.mon.b.smithi138.stdout: last_failure_osd_epoch 0 2023-09-23T23:01:56.470 INFO:journalctl@ceph.mon.b.smithi138.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T23:01:56.470 INFO:journalctl@ceph.mon.b.smithi138.stdout: max_mds 1 2023-09-23T23:01:56.470 INFO:journalctl@ceph.mon.b.smithi138.stdout: in 0 2023-09-23T23:01:56.470 INFO:journalctl@ceph.mon.b.smithi138.stdout: up {0=24572} 2023-09-23T23:01:56.471 INFO:journalctl@ceph.mon.b.smithi138.stdout: failed 2023-09-23T23:01:56.471 INFO:journalctl@ceph.mon.b.smithi138.stdout: damaged 2023-09-23T23:01:56.471 INFO:journalctl@ceph.mon.b.smithi138.stdout: stopped 2023-09-23T23:01:56.471 INFO:journalctl@ceph.mon.b.smithi138.stdout: data_pools [4] 2023-09-23T23:01:56.471 INFO:journalctl@ceph.mon.b.smithi138.stdout: metadata_pool 3 2023-09-23T23:01:56.471 INFO:journalctl@ceph.mon.b.smithi138.stdout: inline_data disabled 2023-09-23T23:01:56.471 INFO:journalctl@ceph.mon.b.smithi138.stdout: balancer 2023-09-23T23:01:56.471 INFO:journalctl@ceph.mon.b.smithi138.stdout: bal_rank_mask -1 2023-09-23T23:01:56.471 INFO:journalctl@ceph.mon.b.smithi138.stdout: standby_count_wanted 1 2023-09-23T23:01:56.471 INFO:journalctl@ceph.mon.b.smithi138.stdout: [mds.foo.smithi110.fuglpi{0:24572} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.110:6834/1876054795,v1:172.21.15.110:6835/1876054795] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T23:01:56.471 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2023-09-23T23:01:56.471 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2023-09-23T23:01:56.472 INFO:journalctl@ceph.mon.b.smithi138.stdout: Standby daemons: 2023-09-23T23:01:56.472 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2023-09-23T23:01:56.472 INFO:journalctl@ceph.mon.b.smithi138.stdout: [mds.foo.smithi138.xrewco{-1:24547} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.138:6832/276794957,v1:172.21.15.138:6833/276794957] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T23:01:56.472 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: mon.b@-1(???).osd e82 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T23:01:56.472 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T23:01:56.472 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T23:01:56.472 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T23:01:56.472 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:56 smithi138 ceph-mon[193606]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-09-23T23:01:57.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:57 smithi110 ceph-mon[212939]: pgmap v103: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 556 op/s 2023-09-23T23:01:57.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:57 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T23:01:57.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:57 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T23:01:57.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:01:57 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T23:01:57.187 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:57 smithi110 ceph-mon[221102]: pgmap v103: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 556 op/s 2023-09-23T23:01:57.187 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:57 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T23:01:57.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:57 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T23:01:57.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:01:57 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T23:01:57.189 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:01:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:01:56] "GET /metrics HTTP/1.1" 200 36228 "" "Prometheus/2.43.0" 2023-09-23T23:01:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:57 smithi138 ceph-mon[193606]: pgmap v103: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 556 op/s 2023-09-23T23:01:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:57 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T23:01:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:57 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T23:01:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:01:57 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T23:02:01.711 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:02:01 smithi110 conmon[198813]: 2023-09-23T23:02:01.231+0000 7ff0150b6700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-09-23T23:02:02.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: pgmap v105: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2023-09-23T23:02:02.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: overall HEALTH_OK 2023-09-23T23:02:02.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T23:02:02.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T23:02:02.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T23:02:02.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: mon.a calling monitor election 2023-09-23T23:02:02.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: mon.c calling monitor election 2023-09-23T23:02:02.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: mon.b calling monitor election 2023-09-23T23:02:02.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:02.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: pgmap v106: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T23:02:02.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: pgmap v107: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2023-09-23T23:02:02.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2023-09-23T23:02:02.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: overall HEALTH_OK 2023-09-23T23:02:02.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: mon.a calling monitor election 2023-09-23T23:02:02.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T23:02:02.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: monmap e4: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} 2023-09-23T23:02:02.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T23:02:02.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T23:02:02.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: mgrmap e35: y(active, since 3m), standbys: x 2023-09-23T23:02:02.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: overall HEALTH_OK 2023-09-23T23:02:02.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='' 2023-09-23T23:02:02.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:02 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:02.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: pgmap v105: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2023-09-23T23:02:02.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: overall HEALTH_OK 2023-09-23T23:02:02.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T23:02:02.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T23:02:02.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T23:02:02.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: mon.a calling monitor election 2023-09-23T23:02:02.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: mon.c calling monitor election 2023-09-23T23:02:02.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: mon.b calling monitor election 2023-09-23T23:02:02.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:02.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: pgmap v106: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T23:02:02.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: pgmap v107: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2023-09-23T23:02:02.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2023-09-23T23:02:02.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: overall HEALTH_OK 2023-09-23T23:02:02.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: mon.a calling monitor election 2023-09-23T23:02:02.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T23:02:02.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: monmap e4: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} 2023-09-23T23:02:02.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T23:02:02.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T23:02:02.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: mgrmap e35: y(active, since 3m), standbys: x 2023-09-23T23:02:02.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: overall HEALTH_OK 2023-09-23T23:02:02.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='' 2023-09-23T23:02:02.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:02.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: pgmap v105: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2023-09-23T23:02:02.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: overall HEALTH_OK 2023-09-23T23:02:02.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T23:02:02.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T23:02:02.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T23:02:02.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: mon.a calling monitor election 2023-09-23T23:02:02.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: mon.c calling monitor election 2023-09-23T23:02:02.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: mon.b calling monitor election 2023-09-23T23:02:02.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:02.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: pgmap v106: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T23:02:02.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: pgmap v107: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2023-09-23T23:02:02.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2023-09-23T23:02:02.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: overall HEALTH_OK 2023-09-23T23:02:02.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: mon.a calling monitor election 2023-09-23T23:02:02.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T23:02:02.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: monmap e4: 3 mons at {a=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]} removed_ranks: {} 2023-09-23T23:02:02.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: fsmap foo:1 {0=foo.smithi110.fuglpi=up:active} 1 up:standby 2023-09-23T23:02:02.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T23:02:02.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: mgrmap e35: y(active, since 3m), standbys: x 2023-09-23T23:02:02.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: overall HEALTH_OK 2023-09-23T23:02:02.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='' 2023-09-23T23:02:02.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:02 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:02.858 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T23:02:03.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:03 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:02:03.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:03 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1695524180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T23:02:03.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:03 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:02:03.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:03 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1695524180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T23:02:03.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:03 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:02:03.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:03 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1695524180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T23:02:04.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:04 smithi138 ceph-mon[193606]: pgmap v108: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-09-23T23:02:04.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:04 smithi110 ceph-mon[221102]: pgmap v108: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-09-23T23:02:04.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:04 smithi110 ceph-mon[212939]: pgmap v108: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-09-23T23:02:05.510 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T23:02:06.241 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:06.591 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:06 smithi110 ceph-mon[212939]: pgmap v109: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2023-09-23T23:02:06.591 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:06 smithi110 ceph-mon[221102]: pgmap v109: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2023-09-23T23:02:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:06 smithi138 ceph-mon[193606]: pgmap v109: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2023-09-23T23:02:06.953 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:02:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:02:06] "GET /metrics HTTP/1.1" 200 36225 "" "Prometheus/2.43.0" 2023-09-23T23:02:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:07 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:07 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:07 smithi138 ceph-mon[193606]: pgmap v110: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-09-23T23:02:08.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:07 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:08.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:07 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:08.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:07 smithi110 ceph-mon[212939]: pgmap v110: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-09-23T23:02:08.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:07 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:08.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:07 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:08.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:07 smithi110 ceph-mon[221102]: pgmap v110: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-09-23T23:02:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:08 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:08 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:08.994 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T23:02:08.994 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (5m) - 9m 26.7M - 0.25.0 c8568f914cd2 d9641d6ed9ca 2023-09-23T23:02:08.994 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi138 *:3000 running (5m) - 9m 92.0M - 9.4.7 2c41d148cca3 56929ec304ec 2023-09-23T23:02:08.994 INFO:teuthology.orchestra.run.smithi110.stdout:mds.foo.smithi110.fuglpi smithi110 running (8m) - 8m 19.7M - 17.2.6-1125-gc9bb1bcf 444539aec5bf f59eb5601951 2023-09-23T23:02:08.994 INFO:teuthology.orchestra.run.smithi110.stdout:mds.foo.smithi138.xrewco smithi138 running (8m) - 8m 15.8M - 17.2.6-1125-gc9bb1bcf 444539aec5bf cf02b45fd734 2023-09-23T23:02:08.994 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi138 *:8443,9283,8765 running (4m) - 13m 448M - 18.2.0-460-gbb0daeff e32c2a64c126 fef2a6a3bd6d 2023-09-23T23:02:08.995 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (4m) - 15m 506M - 18.2.0-460-gbb0daeff e32c2a64c126 09d917770c6e 2023-09-23T23:02:08.995 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (3m) - 15m 61.1M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 d6c089d81bde 2023-09-23T23:02:08.995 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi138 running (13s) - 13m 37.6M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 42bfa5f10303 2023-09-23T23:02:08.995 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (117s) - 13m 46.5M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 4c74500e80a3 2023-09-23T23:02:08.995 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (5m) - 9m 17.2M - 1.5.0 0da6a335fe13 39d48d803cf2 2023-09-23T23:02:08.995 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi138 *:9100 running (5m) - 8m 19.2M - 1.5.0 0da6a335fe13 a55c5d37ba4b 2023-09-23T23:02:08.995 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (13m) - 13m 3123M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf d7807df7af7a 2023-09-23T23:02:08.995 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (12m) - 12m 2439M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 9cdbba4d4209 2023-09-23T23:02:08.995 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (12m) - 12m 4110M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 8350abe92da2 2023-09-23T23:02:08.995 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (12m) - 12m 3078M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf d34afe3cf9ea 2023-09-23T23:02:08.995 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi138 running (11m) - 11m 3377M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 47f7252bf067 2023-09-23T23:02:08.995 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi138 running (11m) - 11m 3788M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf fb3decabc2c4 2023-09-23T23:02:08.995 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi138 running (10m) - 10m 1789M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf c3f9d2352b7c 2023-09-23T23:02:08.996 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi138 running (10m) - 10m 3322M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 161d1d2d0445 2023-09-23T23:02:08.996 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi138 *:9095 running (4m) - 9m 61.8M - 2.43.0 a07b618ecd1d 8c2bb4977f43 2023-09-23T23:02:09.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:08 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:09.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:08 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:09.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:08 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:09.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:08 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:10.098 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:09 smithi110 ceph-mon[221102]: pgmap v111: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T23:02:10.098 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:09 smithi110 ceph-mon[221102]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:02:10.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:09 smithi110 ceph-mon[212939]: pgmap v111: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T23:02:10.108 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:09 smithi110 ceph-mon[212939]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:02:10.134 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T23:02:10.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:09 smithi138 ceph-mon[193606]: pgmap v111: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T23:02:10.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:09 smithi138 ceph-mon[193606]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:02:10.889 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:12.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:12 smithi138 ceph-mon[193606]: pgmap v112: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-09-23T23:02:12.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:12 smithi110 ceph-mon[212939]: pgmap v112: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-09-23T23:02:12.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:12 smithi110 ceph-mon[221102]: pgmap v112: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-09-23T23:02:13.611 INFO:teuthology.orchestra.run.smithi110.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:02:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:13 smithi110 ceph-mon[212939]: pgmap v113: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T23:02:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:13 smithi110 ceph-mon[221102]: pgmap v113: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T23:02:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:14.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:14.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:13 smithi138 ceph-mon[193606]: pgmap v113: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T23:02:14.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:14.709 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T23:02:15.494 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:15.673 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[212939]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:02:15.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[212939]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:02:15.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:15.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:15.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:15.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:02:15.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:15.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T23:02:15.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T23:02:15.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:15.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[221102]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:02:15.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[221102]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:02:15.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:15.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:15.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:15.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:02:15.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:15.676 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T23:02:15.676 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T23:02:15.676 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:15 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:15.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:15 smithi138 ceph-mon[193606]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:02:15.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:15 smithi138 ceph-mon[193606]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:02:15.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:15 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:15.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:15 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:15.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:15 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:15.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:15 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:02:15.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:15 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:15.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:15 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T23:02:15.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:15 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T23:02:15.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:15 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:16.431 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:16 smithi110 ceph-mon[212939]: Reconfiguring mon.a (monmap changed)... 2023-09-23T23:02:16.431 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:16 smithi110 ceph-mon[212939]: Reconfiguring daemon mon.a on smithi110 2023-09-23T23:02:16.431 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:16 smithi110 ceph-mon[212939]: pgmap v114: 89 pgs: 89 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T23:02:16.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:16 smithi138 ceph-mon[193606]: Reconfiguring mon.a (monmap changed)... 2023-09-23T23:02:16.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:16 smithi138 ceph-mon[193606]: Reconfiguring daemon mon.a on smithi110 2023-09-23T23:02:16.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:16 smithi138 ceph-mon[193606]: pgmap v114: 89 pgs: 89 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T23:02:16.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:16 smithi110 ceph-mon[221102]: Reconfiguring mon.a (monmap changed)... 2023-09-23T23:02:16.749 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:16 smithi110 ceph-mon[221102]: Reconfiguring daemon mon.a on smithi110 2023-09-23T23:02:16.749 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:16 smithi110 ceph-mon[221102]: pgmap v114: 89 pgs: 89 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T23:02:17.189 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:02:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:02:16] "GET /metrics HTTP/1.1" 200 36222 "" "Prometheus/2.43.0" 2023-09-23T23:02:18.462 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:18 smithi110 ceph-mon[212939]: pgmap v115: 89 pgs: 89 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-09-23T23:02:18.462 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:18 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:18.462 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:18 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:18.462 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:18 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T23:02:18.462 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:18 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T23:02:18.463 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:18 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T23:02:18.463 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:18 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:18.821 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:18 smithi110 ceph-mon[221102]: pgmap v115: 89 pgs: 89 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-09-23T23:02:18.822 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:18 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:18.822 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:18 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:18.822 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:18 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T23:02:18.822 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:18 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T23:02:18.822 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:18 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T23:02:18.822 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:18 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:18.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:18 smithi138 ceph-mon[193606]: pgmap v115: 89 pgs: 89 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-09-23T23:02:18.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:18 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:18.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:18 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:18.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:18 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T23:02:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:18 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T23:02:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:18 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T23:02:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:18 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:19.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:19 smithi138 ceph-mon[193606]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T23:02:19.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:19 smithi138 ceph-mon[193606]: Reconfiguring daemon mgr.y on smithi110 2023-09-23T23:02:19.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:19 smithi138 ceph-mon[193606]: pgmap v116: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 562 op/s 2023-09-23T23:02:19.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:19 smithi110 ceph-mon[212939]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T23:02:19.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:19 smithi110 ceph-mon[212939]: Reconfiguring daemon mgr.y on smithi110 2023-09-23T23:02:19.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:19 smithi110 ceph-mon[212939]: pgmap v116: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 562 op/s 2023-09-23T23:02:19.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:19 smithi110 ceph-mon[221102]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T23:02:19.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:19 smithi110 ceph-mon[221102]: Reconfiguring daemon mgr.y on smithi110 2023-09-23T23:02:19.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:19 smithi110 ceph-mon[221102]: pgmap v116: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 562 op/s 2023-09-23T23:02:21.505 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:21 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:21.505 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:21 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:21.505 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:21 smithi110 ceph-mon[212939]: Reconfiguring mon.c (monmap changed)... 2023-09-23T23:02:21.505 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:21 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T23:02:21.505 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:21 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T23:02:21.506 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:21 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:21.506 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:21 smithi110 ceph-mon[212939]: Reconfiguring daemon mon.c on smithi110 2023-09-23T23:02:21.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:21 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:21.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:21 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:21.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:21 smithi110 ceph-mon[221102]: Reconfiguring mon.c (monmap changed)... 2023-09-23T23:02:21.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:21 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T23:02:21.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:21 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T23:02:21.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:21 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:21.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:21 smithi110 ceph-mon[221102]: Reconfiguring daemon mon.c on smithi110 2023-09-23T23:02:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:21 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:21 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:21 smithi138 ceph-mon[193606]: Reconfiguring mon.c (monmap changed)... 2023-09-23T23:02:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:21 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T23:02:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:21 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T23:02:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:21 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:21 smithi138 ceph-mon[193606]: Reconfiguring daemon mon.c on smithi110 2023-09-23T23:02:22.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:22 smithi138 ceph-mon[193606]: pgmap v117: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-09-23T23:02:22.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:22 smithi110 ceph-mon[212939]: pgmap v117: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-09-23T23:02:22.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:22 smithi110 ceph-mon[221102]: pgmap v117: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-09-23T23:02:23.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:23 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:23.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:23 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:23.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:23 smithi110 ceph-mon[212939]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T23:02:23.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:23 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T23:02:23.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:23 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:23.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:23 smithi110 ceph-mon[212939]: Reconfiguring daemon osd.0 on smithi110 2023-09-23T23:02:23.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:23 smithi110 ceph-mon[212939]: pgmap v118: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T23:02:23.778 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:23 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:23.778 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:23 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:23.778 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:23 smithi110 ceph-mon[221102]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T23:02:23.779 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:23 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T23:02:23.779 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:23 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:23.779 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:23 smithi110 ceph-mon[221102]: Reconfiguring daemon osd.0 on smithi110 2023-09-23T23:02:23.779 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:23 smithi110 ceph-mon[221102]: pgmap v118: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T23:02:23.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:23 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:23.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:23 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:23.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:23 smithi138 ceph-mon[193606]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T23:02:23.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:23 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T23:02:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:23 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:23 smithi138 ceph-mon[193606]: Reconfiguring daemon osd.0 on smithi110 2023-09-23T23:02:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:23 smithi138 ceph-mon[193606]: pgmap v118: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T23:02:26.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:25 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:26.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:25 smithi110 ceph-mon[212939]: pgmap v119: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T23:02:26.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:25 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:26.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:25 smithi110 ceph-mon[212939]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T23:02:26.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:25 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T23:02:26.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:25 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:26.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:25 smithi110 ceph-mon[212939]: Reconfiguring daemon osd.1 on smithi110 2023-09-23T23:02:26.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:25 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:26.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:25 smithi110 ceph-mon[221102]: pgmap v119: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T23:02:26.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:25 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:26.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:25 smithi110 ceph-mon[221102]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T23:02:26.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:25 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T23:02:26.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:25 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:26.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:25 smithi110 ceph-mon[221102]: Reconfiguring daemon osd.1 on smithi110 2023-09-23T23:02:26.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:25 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:26.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:25 smithi138 ceph-mon[193606]: pgmap v119: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T23:02:26.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:25 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:26.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:25 smithi138 ceph-mon[193606]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T23:02:26.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:25 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T23:02:26.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:25 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:26.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:25 smithi138 ceph-mon[193606]: Reconfiguring daemon osd.1 on smithi110 2023-09-23T23:02:27.204 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:02:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:02:26] "GET /metrics HTTP/1.1" 200 36222 "" "Prometheus/2.43.0" 2023-09-23T23:02:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:28 smithi110 ceph-mon[212939]: pgmap v120: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T23:02:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:28 smithi110 ceph-mon[212939]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T23:02:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T23:02:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:28 smithi110 ceph-mon[212939]: Reconfiguring daemon osd.2 on smithi110 2023-09-23T23:02:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:28.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:28 smithi110 ceph-mon[221102]: pgmap v120: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T23:02:28.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:28.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:28.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:28 smithi110 ceph-mon[221102]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T23:02:28.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T23:02:28.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:28.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:28 smithi110 ceph-mon[221102]: Reconfiguring daemon osd.2 on smithi110 2023-09-23T23:02:28.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:28.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:28 smithi138 ceph-mon[193606]: pgmap v120: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T23:02:28.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:28 smithi138 ceph-mon[193606]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T23:02:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T23:02:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:28 smithi138 ceph-mon[193606]: Reconfiguring daemon osd.2 on smithi110 2023-09-23T23:02:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:30.610 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:30 smithi110 ceph-mon[221102]: pgmap v121: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T23:02:30.610 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:30 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:30.610 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:30 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:30.610 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:30 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T23:02:30.610 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:30 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:30.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:30 smithi110 ceph-mon[212939]: pgmap v121: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T23:02:30.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:30 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:30.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:30 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:30.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:30 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T23:02:30.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:30 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:30.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:30 smithi138 ceph-mon[193606]: pgmap v121: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-09-23T23:02:30.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:30 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:30.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:30 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:30.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:30 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T23:02:30.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:30 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:31.628 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:31 smithi110 ceph-mon[212939]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T23:02:31.628 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:31 smithi110 ceph-mon[212939]: Reconfiguring daemon osd.3 on smithi110 2023-09-23T23:02:31.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:31 smithi110 ceph-mon[221102]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T23:02:31.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:31 smithi110 ceph-mon[221102]: Reconfiguring daemon osd.3 on smithi110 2023-09-23T23:02:31.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:31 smithi138 ceph-mon[193606]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T23:02:31.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:31 smithi138 ceph-mon[193606]: Reconfiguring daemon osd.3 on smithi110 2023-09-23T23:02:32.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:32 smithi110 ceph-mon[212939]: pgmap v122: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2023-09-23T23:02:32.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:32 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:32.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:32 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:32.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:32 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.fuglpi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T23:02:32.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:32 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.fuglpi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T23:02:32.675 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:32 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:32.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:32 smithi110 ceph-mon[221102]: pgmap v122: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2023-09-23T23:02:32.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:32 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:32.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:32 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:32.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:32 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.fuglpi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T23:02:32.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:32 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.fuglpi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T23:02:32.676 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:32 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:32.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:32 smithi138 ceph-mon[193606]: pgmap v122: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2023-09-23T23:02:32.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:32 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:32.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:32 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:32.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:32 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.fuglpi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T23:02:32.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:32 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.fuglpi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T23:02:32.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:32 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:33.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:33 smithi138 ceph-mon[193606]: Reconfiguring mds.foo.smithi110.fuglpi (monmap changed)... 2023-09-23T23:02:33.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:33 smithi138 ceph-mon[193606]: Reconfiguring daemon mds.foo.smithi110.fuglpi on smithi110 2023-09-23T23:02:33.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:33 smithi110 ceph-mon[212939]: Reconfiguring mds.foo.smithi110.fuglpi (monmap changed)... 2023-09-23T23:02:33.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:33 smithi110 ceph-mon[212939]: Reconfiguring daemon mds.foo.smithi110.fuglpi on smithi110 2023-09-23T23:02:33.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:33 smithi110 ceph-mon[221102]: Reconfiguring mds.foo.smithi110.fuglpi (monmap changed)... 2023-09-23T23:02:33.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:33 smithi110 ceph-mon[221102]: Reconfiguring daemon mds.foo.smithi110.fuglpi on smithi110 2023-09-23T23:02:34.258 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:34 smithi110 ceph-mon[212939]: pgmap v123: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-09-23T23:02:34.258 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:34 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:34.258 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:34 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:34.258 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:34 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T23:02:34.259 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:34 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T23:02:34.259 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:34 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:34.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:34 smithi138 ceph-mon[193606]: pgmap v123: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-09-23T23:02:34.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:34 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:34 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:34 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T23:02:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:34 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T23:02:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:34 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:34.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:34 smithi110 ceph-mon[221102]: pgmap v123: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-09-23T23:02:34.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:34 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:34.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:34 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:34.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:34 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T23:02:34.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:34 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T23:02:34.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:34 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:35.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:35 smithi138 ceph-mon[193606]: Reconfiguring mon.b (monmap changed)... 2023-09-23T23:02:35.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:35 smithi138 ceph-mon[193606]: Reconfiguring daemon mon.b on smithi138 2023-09-23T23:02:35.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:35 smithi110 ceph-mon[212939]: Reconfiguring mon.b (monmap changed)... 2023-09-23T23:02:35.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:35 smithi110 ceph-mon[212939]: Reconfiguring daemon mon.b on smithi138 2023-09-23T23:02:35.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:35 smithi110 ceph-mon[221102]: Reconfiguring mon.b (monmap changed)... 2023-09-23T23:02:35.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:35 smithi110 ceph-mon[221102]: Reconfiguring daemon mon.b on smithi138 2023-09-23T23:02:36.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:36 smithi138 ceph-mon[193606]: pgmap v124: 89 pgs: 89 active+clean; 1.7 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2023-09-23T23:02:36.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:36 smithi110 ceph-mon[221102]: pgmap v124: 89 pgs: 89 active+clean; 1.7 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2023-09-23T23:02:36.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:36 smithi110 ceph-mon[212939]: pgmap v124: 89 pgs: 89 active+clean; 1.7 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2023-09-23T23:02:37.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:02:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:02:36] "GET /metrics HTTP/1.1" 200 36220 "" "Prometheus/2.43.0" 2023-09-23T23:02:38.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:38 smithi138 ceph-mon[193606]: pgmap v125: 89 pgs: 89 active+clean; 1.7 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2023-09-23T23:02:38.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:38 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:38.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:38 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:38.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:38 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T23:02:38.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:38 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T23:02:38.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:38 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T23:02:38.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:38 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:38.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:38 smithi110 ceph-mon[212939]: pgmap v125: 89 pgs: 89 active+clean; 1.7 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2023-09-23T23:02:38.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:38 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:38 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:38 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T23:02:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:38 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T23:02:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:38 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T23:02:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:38 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:38.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:38 smithi110 ceph-mon[221102]: pgmap v125: 89 pgs: 89 active+clean; 1.7 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2023-09-23T23:02:38.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:38 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:38.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:38 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:38.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:38 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T23:02:38.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:38 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T23:02:38.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:38 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T23:02:38.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:38 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:39.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:39 smithi138 ceph-mon[193606]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T23:02:39.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:39 smithi138 ceph-mon[193606]: Reconfiguring daemon mgr.x on smithi138 2023-09-23T23:02:39.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:39 smithi110 ceph-mon[221102]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T23:02:39.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:39 smithi110 ceph-mon[221102]: Reconfiguring daemon mgr.x on smithi138 2023-09-23T23:02:39.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:39 smithi110 ceph-mon[212939]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T23:02:39.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:39 smithi110 ceph-mon[212939]: Reconfiguring daemon mgr.x on smithi138 2023-09-23T23:02:40.320 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:40 smithi138 ceph-mon[193606]: pgmap v126: 89 pgs: 89 active+clean; 1.7 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-09-23T23:02:40.320 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:40 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:40.320 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:40 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:40.321 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:40 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T23:02:40.321 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:40 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:40.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:40 smithi110 ceph-mon[212939]: pgmap v126: 89 pgs: 89 active+clean; 1.7 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-09-23T23:02:40.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:40 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:40.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:40 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:40.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:40 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T23:02:40.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:40 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:40.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:40 smithi110 ceph-mon[221102]: pgmap v126: 89 pgs: 89 active+clean; 1.7 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-09-23T23:02:40.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:40 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:40.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:40 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:40.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:40 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T23:02:40.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:40 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:41.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:41 smithi138 ceph-mon[193606]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T23:02:41.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:41 smithi138 ceph-mon[193606]: Reconfiguring daemon osd.4 on smithi138 2023-09-23T23:02:41.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:41 smithi110 ceph-mon[221102]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T23:02:41.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:41 smithi110 ceph-mon[221102]: Reconfiguring daemon osd.4 on smithi138 2023-09-23T23:02:41.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:41 smithi110 ceph-mon[212939]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T23:02:41.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:41 smithi110 ceph-mon[212939]: Reconfiguring daemon osd.4 on smithi138 2023-09-23T23:02:42.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:42 smithi138 ceph-mon[193606]: pgmap v127: 89 pgs: 89 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2023-09-23T23:02:42.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:42 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:42.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:42 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:42.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:42 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T23:02:42.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:42 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:42.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:42 smithi110 ceph-mon[221102]: pgmap v127: 89 pgs: 89 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2023-09-23T23:02:42.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:42 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:42.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:42 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:42.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:42 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T23:02:42.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:42 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:42.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:42 smithi110 ceph-mon[212939]: pgmap v127: 89 pgs: 89 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2023-09-23T23:02:42.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:42 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:42.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:42 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:42.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:42 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T23:02:42.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:42 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:43.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:43 smithi138 ceph-mon[193606]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T23:02:43.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:43 smithi138 ceph-mon[193606]: Reconfiguring daemon osd.5 on smithi138 2023-09-23T23:02:43.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:43.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:43 smithi110 ceph-mon[212939]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T23:02:43.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:43 smithi110 ceph-mon[212939]: Reconfiguring daemon osd.5 on smithi138 2023-09-23T23:02:43.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:43.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:43 smithi110 ceph-mon[221102]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T23:02:43.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:43 smithi110 ceph-mon[221102]: Reconfiguring daemon osd.5 on smithi138 2023-09-23T23:02:43.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:44.640 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:44 smithi138 ceph-mon[193606]: pgmap v128: 89 pgs: 89 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T23:02:44.640 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:44 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:44.640 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:44 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:44.640 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:44 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T23:02:44.640 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:44 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:44.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:44 smithi110 ceph-mon[212939]: pgmap v128: 89 pgs: 89 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T23:02:44.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:44 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:44.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:44 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:44.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:44 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T23:02:44.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:44 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:44.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:44 smithi110 ceph-mon[221102]: pgmap v128: 89 pgs: 89 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T23:02:44.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:44 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:44.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:44 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:44.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:44 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T23:02:44.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:44 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:45.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:45 smithi138 ceph-mon[193606]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T23:02:45.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:45 smithi138 ceph-mon[193606]: Reconfiguring daemon osd.6 on smithi138 2023-09-23T23:02:45.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:45 smithi110 ceph-mon[221102]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T23:02:45.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:45 smithi110 ceph-mon[221102]: Reconfiguring daemon osd.6 on smithi138 2023-09-23T23:02:45.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:45 smithi110 ceph-mon[212939]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T23:02:45.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:45 smithi110 ceph-mon[212939]: Reconfiguring daemon osd.6 on smithi138 2023-09-23T23:02:46.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:46 smithi138 ceph-mon[193606]: pgmap v129: 89 pgs: 89 active+clean; 1.2 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 929 op/s 2023-09-23T23:02:46.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:46 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:46.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:46 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:46.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:46 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T23:02:46.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:46 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:46.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:46 smithi110 ceph-mon[212939]: pgmap v129: 89 pgs: 89 active+clean; 1.2 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 929 op/s 2023-09-23T23:02:46.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:46 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:46.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:46 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:46.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:46 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T23:02:46.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:46 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:46.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:46 smithi110 ceph-mon[221102]: pgmap v129: 89 pgs: 89 active+clean; 1.2 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 929 op/s 2023-09-23T23:02:46.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:46 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:46.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:46 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:46.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:46 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T23:02:46.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:46 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:47.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:02:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:02:46] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.43.0" 2023-09-23T23:02:47.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:47 smithi138 ceph-mon[193606]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T23:02:47.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:47 smithi138 ceph-mon[193606]: Reconfiguring daemon osd.7 on smithi138 2023-09-23T23:02:47.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:47 smithi110 ceph-mon[221102]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T23:02:47.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:47 smithi110 ceph-mon[221102]: Reconfiguring daemon osd.7 on smithi138 2023-09-23T23:02:47.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:47 smithi110 ceph-mon[212939]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T23:02:47.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:47 smithi110 ceph-mon[212939]: Reconfiguring daemon osd.7 on smithi138 2023-09-23T23:02:48.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:48 smithi138 ceph-mon[193606]: pgmap v130: 89 pgs: 89 active+clean; 1.2 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-09-23T23:02:48.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:48 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:48.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:48 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:48.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:48 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.xrewco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T23:02:48.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:48 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.xrewco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T23:02:48.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:48 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:48.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:48 smithi110 ceph-mon[212939]: pgmap v130: 89 pgs: 89 active+clean; 1.2 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-09-23T23:02:48.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:48.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:48.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:48 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.xrewco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T23:02:48.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.xrewco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T23:02:48.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:48 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:48.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:48 smithi110 ceph-mon[221102]: pgmap v130: 89 pgs: 89 active+clean; 1.2 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-09-23T23:02:48.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:48 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:48.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:48 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:48.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:48 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.xrewco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T23:02:48.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:48 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.xrewco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T23:02:48.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:48 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:49.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:49 smithi138 ceph-mon[193606]: Reconfiguring mds.foo.smithi138.xrewco (monmap changed)... 2023-09-23T23:02:49.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:49 smithi138 ceph-mon[193606]: Reconfiguring daemon mds.foo.smithi138.xrewco on smithi138 2023-09-23T23:02:49.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:49 smithi110 ceph-mon[212939]: Reconfiguring mds.foo.smithi138.xrewco (monmap changed)... 2023-09-23T23:02:49.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:49 smithi110 ceph-mon[212939]: Reconfiguring daemon mds.foo.smithi138.xrewco on smithi138 2023-09-23T23:02:49.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:49 smithi110 ceph-mon[221102]: Reconfiguring mds.foo.smithi138.xrewco (monmap changed)... 2023-09-23T23:02:49.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:49 smithi110 ceph-mon[221102]: Reconfiguring daemon mds.foo.smithi138.xrewco on smithi138 2023-09-23T23:02:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:50 smithi138 ceph-mon[193606]: pgmap v131: 89 pgs: 89 active+clean; 1.2 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-09-23T23:02:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:50 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:50 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:50 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:02:50.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:50 smithi110 ceph-mon[221102]: pgmap v131: 89 pgs: 89 active+clean; 1.2 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-09-23T23:02:50.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:50 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:50.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:50 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:50.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:50 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:02:50.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:50 smithi110 ceph-mon[212939]: pgmap v131: 89 pgs: 89 active+clean; 1.2 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-09-23T23:02:50.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:50 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:50.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:50 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:50.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:50 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:02:52.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:52 smithi138 ceph-mon[193606]: pgmap v132: 89 pgs: 89 active+clean; 1005 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 921 op/s 2023-09-23T23:02:52.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:52 smithi110 ceph-mon[221102]: pgmap v132: 89 pgs: 89 active+clean; 1005 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 921 op/s 2023-09-23T23:02:52.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:52 smithi110 ceph-mon[212939]: pgmap v132: 89 pgs: 89 active+clean; 1005 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 921 op/s 2023-09-23T23:02:54.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:53 smithi138 ceph-mon[193606]: pgmap v133: 89 pgs: 89 active+clean; 1005 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-09-23T23:02:54.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:53 smithi110 ceph-mon[221102]: pgmap v133: 89 pgs: 89 active+clean; 1005 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-09-23T23:02:54.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:53 smithi110 ceph-mon[212939]: pgmap v133: 89 pgs: 89 active+clean; 1005 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-09-23T23:02:56.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:56 smithi138 ceph-mon[193606]: pgmap v134: 89 pgs: 89 active+clean; 783 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 932 op/s 2023-09-23T23:02:56.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:56 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:56.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:56 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:56.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:56 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:56 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:02:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:56 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:56.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:56 smithi110 ceph-mon[212939]: pgmap v134: 89 pgs: 89 active+clean; 783 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 932 op/s 2023-09-23T23:02:56.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:56 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:56.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:56 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:56.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:56 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:56.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:56 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:02:56.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:56 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:56.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:56 smithi110 ceph-mon[221102]: pgmap v134: 89 pgs: 89 active+clean; 783 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 932 op/s 2023-09-23T23:02:56.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:56 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:56.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:56 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:56.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:56 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:56.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:56 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:02:56.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:56 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:02:57.203 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:02:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:02:56] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.43.0" 2023-09-23T23:02:58.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:58 smithi138 ceph-mon[193606]: pgmap v135: 89 pgs: 89 active+clean; 783 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T23:02:58.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:02:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:58.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:58 smithi110 ceph-mon[212939]: pgmap v135: 89 pgs: 89 active+clean; 783 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T23:02:58.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:02:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:58 smithi110 ceph-mon[221102]: pgmap v135: 89 pgs: 89 active+clean; 783 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T23:02:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:02:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:00 smithi138 ceph-mon[193606]: pgmap v136: 89 pgs: 89 active+clean; 783 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T23:03:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:00 smithi110 ceph-mon[212939]: pgmap v136: 89 pgs: 89 active+clean; 783 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T23:03:00.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:00 smithi110 ceph-mon[221102]: pgmap v136: 89 pgs: 89 active+clean; 783 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T23:03:02.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:02 smithi138 ceph-mon[193606]: pgmap v137: 89 pgs: 89 active+clean; 540 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 943 op/s 2023-09-23T23:03:02.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:02 smithi110 ceph-mon[212939]: pgmap v137: 89 pgs: 89 active+clean; 540 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 943 op/s 2023-09-23T23:03:02.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:02 smithi110 ceph-mon[221102]: pgmap v137: 89 pgs: 89 active+clean; 540 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 943 op/s 2023-09-23T23:03:04.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:03 smithi138 ceph-mon[193606]: pgmap v138: 89 pgs: 89 active+clean; 540 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T23:03:04.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:03 smithi110 ceph-mon[212939]: pgmap v138: 89 pgs: 89 active+clean; 540 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T23:03:04.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:03 smithi110 ceph-mon[221102]: pgmap v138: 89 pgs: 89 active+clean; 540 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T23:03:06.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:06 smithi138 ceph-mon[193606]: pgmap v139: 89 pgs: 89 active+clean; 292 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2023-09-23T23:03:06.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:06 smithi110 ceph-mon[212939]: pgmap v139: 89 pgs: 89 active+clean; 292 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2023-09-23T23:03:06.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:06 smithi110 ceph-mon[221102]: pgmap v139: 89 pgs: 89 active+clean; 292 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2023-09-23T23:03:07.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:03:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:03:06] "GET /metrics HTTP/1.1" 200 36217 "" "Prometheus/2.43.0" 2023-09-23T23:03:08.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:08 smithi138 ceph-mon[193606]: pgmap v140: 89 pgs: 89 active+clean; 292 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T23:03:08.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:08 smithi110 ceph-mon[221102]: pgmap v140: 89 pgs: 89 active+clean; 292 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T23:03:08.460 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:08 smithi110 ceph-mon[212939]: pgmap v140: 89 pgs: 89 active+clean; 292 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T23:03:09.907 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Removed 124726 objects 2023-09-23T23:03:09.908 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Clean up completed and total clean up time :160.863 2023-09-23T23:03:10.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:10 smithi138 ceph-mon[193606]: pgmap v141: 89 pgs: 89 active+clean; 292 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T23:03:10.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:10 smithi110 ceph-mon[212939]: pgmap v141: 89 pgs: 89 active+clean; 292 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T23:03:10.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:10 smithi110 ceph-mon[221102]: pgmap v141: 89 pgs: 89 active+clean; 292 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T23:03:12.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:12 smithi138 ceph-mon[193606]: pgmap v142: 89 pgs: 89 active+clean; 40 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2023-09-23T23:03:12.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:12 smithi110 ceph-mon[212939]: pgmap v142: 89 pgs: 89 active+clean; 40 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2023-09-23T23:03:12.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:12 smithi110 ceph-mon[221102]: pgmap v142: 89 pgs: 89 active+clean; 40 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2023-09-23T23:03:14.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:14.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:13 smithi138 ceph-mon[193606]: pgmap v143: 89 pgs: 89 active+clean; 40 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-09-23T23:03:14.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:14.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:13 smithi110 ceph-mon[212939]: pgmap v143: 89 pgs: 89 active+clean; 40 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-09-23T23:03:14.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:14.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:13 smithi110 ceph-mon[221102]: pgmap v143: 89 pgs: 89 active+clean; 40 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-09-23T23:03:15.187 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-09-23T23:03:15.187 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-09-23T23:03:15.536 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:16.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:16 smithi138 ceph-mon[193606]: pgmap v144: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-09-23T23:03:16.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:16 smithi110 ceph-mon[212939]: pgmap v144: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-09-23T23:03:16.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:16 smithi110 ceph-mon[221102]: pgmap v144: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-09-23T23:03:17.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:03:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:03:16] "GET /metrics HTTP/1.1" 200 36209 "" "Prometheus/2.43.0" 2023-09-23T23:03:18.349 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:18 smithi110 ceph-mon[212939]: pgmap v145: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 388 op/s 2023-09-23T23:03:18.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:18 smithi110 ceph-mon[221102]: pgmap v145: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 388 op/s 2023-09-23T23:03:18.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:18 smithi138 ceph-mon[193606]: pgmap v145: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 388 op/s 2023-09-23T23:03:18.978 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T23:03:19.070 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_1' removed 2023-09-23T23:03:19.263 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:19 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1412051514' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:03:19.263 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:19 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:03:19.263 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:19 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1412051514' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:03:19.263 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:19 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:03:19.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:19 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1412051514' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:03:19.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:19 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:03:19.707 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T23:03:19.707 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T23:03:19.707 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-09-23T23:03:19.708 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-09-23T23:03:19.748 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:20.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:20 smithi138 ceph-mon[193606]: pgmap v146: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 388 op/s 2023-09-23T23:03:20.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:20 smithi138 ceph-mon[193606]: failed to encode map e83 with expected crc 2023-09-23T23:03:20.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:20 smithi138 ceph-mon[193606]: failed to encode map e83 with expected crc 2023-09-23T23:03:20.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:20 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:03:20.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:20 smithi138 ceph-mon[193606]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T23:03:20.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:20 smithi110 ceph-mon[212939]: pgmap v146: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 388 op/s 2023-09-23T23:03:20.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:20 smithi110 ceph-mon[212939]: failed to encode map e83 with expected crc 2023-09-23T23:03:20.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:20 smithi110 ceph-mon[212939]: failed to encode map e83 with expected crc 2023-09-23T23:03:20.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:20 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:03:20.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:20 smithi110 ceph-mon[212939]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T23:03:20.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:20 smithi110 ceph-mon[221102]: pgmap v146: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 388 op/s 2023-09-23T23:03:20.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:20 smithi110 ceph-mon[221102]: failed to encode map e83 with expected crc 2023-09-23T23:03:20.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:20 smithi110 ceph-mon[221102]: failed to encode map e83 with expected crc 2023-09-23T23:03:20.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:20 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:03:20.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:20 smithi110 ceph-mon[221102]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T23:03:20.459 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:22.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:22 smithi138 ceph-mon[193606]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:22.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:22 smithi110 ceph-mon[212939]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:22.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:22 smithi110 ceph-mon[221102]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:22.642 INFO:teuthology.orchestra.run.smithi110.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:03:23.435 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for half of osds to upgrade' 2023-09-23T23:03:23.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:23 smithi110 ceph-mon[221102]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:03:23.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:23 smithi110 ceph-mon[221102]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:03:23.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:23 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:23.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:23 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:03:23.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:23 smithi110 ceph-mon[221102]: pgmap v149: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:23.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:23 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3767525680' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T23:03:23.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:23 smithi110 ceph-mon[212939]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:03:23.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:23 smithi110 ceph-mon[212939]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:03:23.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:23 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:23.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:23 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:03:23.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:23 smithi110 ceph-mon[212939]: pgmap v149: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:23.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:23 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3767525680' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T23:03:23.818 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:23.862 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_2' created 2023-09-23T23:03:23.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:23 smithi138 ceph-mon[193606]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:03:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:23 smithi138 ceph-mon[193606]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:03:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:23 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:23 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:03:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:23 smithi138 ceph-mon[193606]: pgmap v149: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:23 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3767525680' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T23:03:24.606 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-09-23T23:03:25.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:24 smithi138 ceph-mon[193606]: failed to encode map e84 with expected crc 2023-09-23T23:03:25.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:24 smithi138 ceph-mon[193606]: failed to encode map e84 with expected crc 2023-09-23T23:03:25.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:24 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3767525680' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T23:03:25.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:24 smithi138 ceph-mon[193606]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T23:03:25.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:24 smithi110 ceph-mon[212939]: failed to encode map e84 with expected crc 2023-09-23T23:03:25.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:24 smithi110 ceph-mon[212939]: failed to encode map e84 with expected crc 2023-09-23T23:03:25.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:24 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3767525680' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T23:03:25.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:24 smithi110 ceph-mon[212939]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T23:03:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:24 smithi110 ceph-mon[221102]: failed to encode map e84 with expected crc 2023-09-23T23:03:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:24 smithi110 ceph-mon[221102]: failed to encode map e84 with expected crc 2023-09-23T23:03:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:24 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3767525680' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T23:03:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:24 smithi110 ceph-mon[221102]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T23:03:25.445 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:25.794 INFO:teuthology.orchestra.run.smithi110.stdout:wait for half of osds to upgrade 2023-09-23T23:03:25.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[212939]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[212939]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[212939]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T23:03:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[212939]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[212939]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[212939]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[212939]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[212939]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[212939]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[212939]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[212939]: pgmap v152: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[221102]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[221102]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[221102]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T23:03:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[221102]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[221102]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[221102]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[221102]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[221102]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[221102]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[221102]: failed to encode map e85 with expected crc 2023-09-23T23:03:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[221102]: pgmap v152: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:25.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:25 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:26.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:25 smithi138 ceph-mon[193606]: failed to encode map e85 with expected crc 2023-09-23T23:03:26.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:25 smithi138 ceph-mon[193606]: failed to encode map e85 with expected crc 2023-09-23T23:03:26.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:25 smithi138 ceph-mon[193606]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T23:03:26.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:25 smithi138 ceph-mon[193606]: failed to encode map e85 with expected crc 2023-09-23T23:03:26.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:25 smithi138 ceph-mon[193606]: failed to encode map e85 with expected crc 2023-09-23T23:03:26.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:25 smithi138 ceph-mon[193606]: failed to encode map e85 with expected crc 2023-09-23T23:03:26.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:25 smithi138 ceph-mon[193606]: failed to encode map e85 with expected crc 2023-09-23T23:03:26.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:25 smithi138 ceph-mon[193606]: failed to encode map e85 with expected crc 2023-09-23T23:03:26.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:25 smithi138 ceph-mon[193606]: failed to encode map e85 with expected crc 2023-09-23T23:03:26.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:25 smithi138 ceph-mon[193606]: failed to encode map e85 with expected crc 2023-09-23T23:03:26.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:25 smithi138 ceph-mon[193606]: pgmap v152: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:26.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:25 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:26.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:25 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:26.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:03:26 smithi110 conmon[198813]: 2023-09-23T23:03:26.313+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-09-23T23:03:26.750 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-09-23T23:03:27.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: failed to encode map e86 with expected crc 2023-09-23T23:03:27.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: failed to encode map e86 with expected crc 2023-09-23T23:03:27.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T23:03:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:03:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:03:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:03:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:03:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:03:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: Upgrade: Setting container_image for all mon 2023-09-23T23:03:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T23:03:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T23:03:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T23:03:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T23:03:27.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T23:03:27.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T23:03:27.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T23:03:27.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T23:03:27.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T23:03:27.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:03:27.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: Upgrade: Setting container_image for all crash 2023-09-23T23:03:27.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:27.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T23:03:27.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T23:03:27.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:26 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-09-23T23:03:27.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:03:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:03:26] "GET /metrics HTTP/1.1" 200 36209 "" "Prometheus/2.43.0" 2023-09-23T23:03:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: failed to encode map e86 with expected crc 2023-09-23T23:03:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: failed to encode map e86 with expected crc 2023-09-23T23:03:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T23:03:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:03:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:03:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:03:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:03:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:03:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: Upgrade: Setting container_image for all mon 2023-09-23T23:03:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:27.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T23:03:27.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T23:03:27.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T23:03:27.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T23:03:27.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T23:03:27.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T23:03:27.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T23:03:27.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T23:03:27.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T23:03:27.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:03:27.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: Upgrade: Setting container_image for all crash 2023-09-23T23:03:27.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:27.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T23:03:27.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T23:03:27.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-09-23T23:03:27.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: failed to encode map e86 with expected crc 2023-09-23T23:03:27.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: failed to encode map e86 with expected crc 2023-09-23T23:03:27.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T23:03:27.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:03:27.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:03:27.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:27.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:03:27.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:03:27.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:03:27.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: Upgrade: Setting container_image for all mon 2023-09-23T23:03:27.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:27.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T23:03:27.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T23:03:27.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T23:03:27.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T23:03:27.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T23:03:27.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T23:03:27.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T23:03:27.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T23:03:27.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T23:03:27.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:03:27.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: Upgrade: Setting container_image for all crash 2023-09-23T23:03:27.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:27.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T23:03:27.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T23:03:27.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:26 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-09-23T23:03:27.398 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:28.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:27 smithi138 ceph-mon[193606]: pgmap v154: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:28.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:27 smithi110 ceph-mon[221102]: pgmap v154: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:28.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:27 smithi110 ceph-mon[212939]: pgmap v154: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:28.877 INFO:teuthology.orchestra.run.smithi110.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-09-23T23:03:29.128 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:28 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3140310860' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:03:29.128 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:28 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:03:29.128 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:29.129 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:28 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3140310860' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:03:29.129 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:28 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:03:29.129 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:29.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:28 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3140310860' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:03:29.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:28 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:03:29.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:29.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:29 smithi110 ceph-mon[221102]: failed to encode map e87 with expected crc 2023-09-23T23:03:29.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:29 smithi110 ceph-mon[221102]: failed to encode map e87 with expected crc 2023-09-23T23:03:29.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:29 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:03:29.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:29 smithi110 ceph-mon[221102]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T23:03:29.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:29 smithi110 ceph-mon[221102]: pgmap v156: 89 pgs: 13 creating+peering, 76 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:29.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:29 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/909584313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:29.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:29 smithi110 ceph-mon[212939]: failed to encode map e87 with expected crc 2023-09-23T23:03:29.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:29 smithi110 ceph-mon[212939]: failed to encode map e87 with expected crc 2023-09-23T23:03:29.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:29 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:03:29.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:29 smithi110 ceph-mon[212939]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T23:03:29.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:29 smithi110 ceph-mon[212939]: pgmap v156: 89 pgs: 13 creating+peering, 76 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:29.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:29 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/909584313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:30.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:29 smithi138 ceph-mon[193606]: failed to encode map e87 with expected crc 2023-09-23T23:03:30.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:29 smithi138 ceph-mon[193606]: failed to encode map e87 with expected crc 2023-09-23T23:03:30.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:29 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:03:30.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:29 smithi138 ceph-mon[193606]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T23:03:30.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:29 smithi138 ceph-mon[193606]: pgmap v156: 89 pgs: 13 creating+peering, 76 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:30.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:29 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/909584313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:30.651 DEBUG:teuthology.orchestra.run.smithi110:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-09-23T23:03:30.653 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T23:03:30.653 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T23:03:30.708 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:hints = 1 2023-09-23T23:03:30.708 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T23:03:30.708 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object prefix: benchmark_data_smithi110_236826 2023-09-23T23:03:30.709 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:03:30.709 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 0 0 0 0 0 0 - 0 2023-09-23T23:03:31.708 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 1 16 3654 3638 227.205 227.375 0.00348182 0.00439335 2023-09-23T23:03:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:32 smithi138 ceph-mon[193606]: failed to encode map e87 with expected crc 2023-09-23T23:03:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:32 smithi138 ceph-mon[193606]: failed to encode map e87 with expected crc 2023-09-23T23:03:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:32 smithi138 ceph-mon[193606]: pgmap v157: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:32 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/589038232' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:32.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:32 smithi110 ceph-mon[221102]: failed to encode map e87 with expected crc 2023-09-23T23:03:32.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:32 smithi110 ceph-mon[221102]: failed to encode map e87 with expected crc 2023-09-23T23:03:32.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:32 smithi110 ceph-mon[221102]: pgmap v157: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:32.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:32 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/589038232' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:32.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:32 smithi110 ceph-mon[212939]: failed to encode map e87 with expected crc 2023-09-23T23:03:32.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:32 smithi110 ceph-mon[212939]: failed to encode map e87 with expected crc 2023-09-23T23:03:32.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:32 smithi110 ceph-mon[212939]: pgmap v157: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:32.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:32 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/589038232' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:32.708 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 2 16 7174 7158 223.592 220 0.00303941 0.00446427 2023-09-23T23:03:33.709 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 3 16 10473 10457 217.769 206.188 0.00518112 0.00458418 2023-09-23T23:03:34.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:33 smithi138 ceph-mon[193606]: pgmap v158: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:34.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:33 smithi110 ceph-mon[212939]: pgmap v158: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:34.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:33 smithi110 ceph-mon[221102]: pgmap v158: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T23:03:34.710 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 4 16 13300 13284 207.442 176.688 0.00386437 0.00481549 2023-09-23T23:03:35.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:34 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2163121660' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:35.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:34 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2163121660' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:35.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:34 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2163121660' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:35.710 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 5 16 16216 16200 202.402 182.25 0.00432964 0.00493733 2023-09-23T23:03:36.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:35 smithi138 ceph-mon[193606]: pgmap v159: 89 pgs: 89 active+clean; 723 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T23:03:36.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:35 smithi110 ceph-mon[221102]: pgmap v159: 89 pgs: 89 active+clean; 723 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T23:03:36.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:35 smithi110 ceph-mon[212939]: pgmap v159: 89 pgs: 89 active+clean; 723 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T23:03:36.710 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 6 16 18829 18813 195.887 163.312 0.00583716 0.00509833 2023-09-23T23:03:37.072 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:03:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:03:36] "GET /metrics HTTP/1.1" 200 36157 "" "Prometheus/2.43.0" 2023-09-23T23:03:37.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:37 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2792795592' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:37.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:37 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2792795592' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:37.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:37 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2792795592' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:37.710 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 7 16 21276 21260 189.75 152.938 0.00540508 0.00526391 2023-09-23T23:03:38.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:38 smithi138 ceph-mon[193606]: pgmap v160: 89 pgs: 89 active+clean; 723 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T23:03:38.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:38 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:38.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:38 smithi110 ceph-mon[221102]: pgmap v160: 89 pgs: 89 active+clean; 723 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T23:03:38.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:38 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:38.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:38 smithi110 ceph-mon[212939]: pgmap v160: 89 pgs: 89 active+clean; 723 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T23:03:38.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:38 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:38.711 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 8 16 23748 23732 185.325 154.5 0.00641247 0.00539032 2023-09-23T23:03:39.711 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 9 16 25907 25891 179.719 134.938 0.00636272 0.00555846 2023-09-23T23:03:40.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:40 smithi138 ceph-mon[193606]: pgmap v161: 89 pgs: 89 active+clean; 880 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T23:03:40.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:40 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2214708238' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:40 smithi110 ceph-mon[212939]: pgmap v161: 89 pgs: 89 active+clean; 880 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T23:03:40.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:40 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2214708238' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:40.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:40 smithi110 ceph-mon[221102]: pgmap v161: 89 pgs: 89 active+clean; 880 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T23:03:40.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:40 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2214708238' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:40.713 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 10 16 28150 28134 175.742 140.188 0.0141736 0.00568479 2023-09-23T23:03:41.713 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 11 16 30170 30154 171.243 126.25 0.0109109 0.00583428 2023-09-23T23:03:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:42 smithi138 ceph-mon[193606]: pgmap v162: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T23:03:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:42 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T23:03:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:42 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T23:03:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:42 smithi138 ceph-mon[193606]: Upgrade: osd.0 is safe to restart 2023-09-23T23:03:42.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:42 smithi110 ceph-mon[212939]: pgmap v162: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T23:03:42.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:42 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T23:03:42.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:42 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T23:03:42.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:42 smithi110 ceph-mon[212939]: Upgrade: osd.0 is safe to restart 2023-09-23T23:03:42.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:42 smithi110 ceph-mon[221102]: pgmap v162: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T23:03:42.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:42 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T23:03:42.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:42 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T23:03:42.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:42 smithi110 ceph-mon[221102]: Upgrade: osd.0 is safe to restart 2023-09-23T23:03:42.713 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 12 16 31816 31800 165.547 102.875 0.00438515 0.00603555 2023-09-23T23:03:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:43 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2514358412' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:43.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:43 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2514358412' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:43.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:43 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2514358412' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:43.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:43.713 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 13 16 33779 33763 162.251 122.688 0.00523582 0.00615753 2023-09-23T23:03:44.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:44 smithi110 ceph-mon[221102]: pgmap v163: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T23:03:44.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:44 smithi110 ceph-mon[212939]: pgmap v163: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T23:03:44.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:44 smithi138 ceph-mon[193606]: pgmap v163: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T23:03:44.713 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 14 16 35495 35479 158.323 107.25 0.0068051 0.00631045 2023-09-23T23:03:45.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:45 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4124939477' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:45.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:45 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4124939477' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:45.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:45 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4124939477' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:45.713 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 15 16 37376 37360 155.605 117.562 0.00675214 0.00642168 2023-09-23T23:03:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:46 smithi138 ceph-mon[193606]: pgmap v164: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 179 MiB/s wr, 2.87k op/s 2023-09-23T23:03:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:46 smithi110 ceph-mon[212939]: pgmap v164: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 179 MiB/s wr, 2.87k op/s 2023-09-23T23:03:46.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:46 smithi110 ceph-mon[221102]: pgmap v164: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 179 MiB/s wr, 2.87k op/s 2023-09-23T23:03:46.713 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 16 16 39090 39074 152.575 107.125 0.00620442 0.00654958 2023-09-23T23:03:47.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:03:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:03:46] "GET /metrics HTTP/1.1" 200 36210 "" "Prometheus/2.43.0" 2023-09-23T23:03:47.714 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 17 16 40544 40528 148.946 90.875 0.00904419 0.00670754 2023-09-23T23:03:48.415 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:48 smithi110 ceph-mon[212939]: pgmap v165: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-09-23T23:03:48.415 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:48 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/374271719' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:48.415 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:48.415 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:48 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T23:03:48.416 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:48 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:03:48.416 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:48 smithi110 ceph-mon[221102]: pgmap v165: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-09-23T23:03:48.416 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:48 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/374271719' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:48.416 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:48 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:48.416 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:48 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T23:03:48.416 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:48 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:03:48.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:48 smithi138 ceph-mon[193606]: pgmap v165: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-09-23T23:03:48.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:48 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/374271719' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:48.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:48 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:48.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:48 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T23:03:48.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:48 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:03:48.714 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 18 16 42154 42138 146.261 100.625 0.011647 0.00683251 2023-09-23T23:03:49.144 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:49 smithi110 ceph-mon[212939]: Upgrade: Updating osd.0 2023-09-23T23:03:49.144 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:49 smithi110 ceph-mon[212939]: Deploying daemon osd.0 on smithi110 2023-09-23T23:03:49.144 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:49 smithi110 ceph-mon[221102]: Upgrade: Updating osd.0 2023-09-23T23:03:49.144 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:49 smithi110 ceph-mon[221102]: Deploying daemon osd.0 on smithi110 2023-09-23T23:03:49.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:49 smithi138 ceph-mon[193606]: Upgrade: Updating osd.0 2023-09-23T23:03:49.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:49 smithi138 ceph-mon[193606]: Deploying daemon osd.0 on smithi110 2023-09-23T23:03:49.714 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 19 16 43427 43411 142.751 79.5625 0.00210981 0.00692123 2023-09-23T23:03:50.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:50 smithi138 ceph-mon[193606]: pgmap v166: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T23:03:50.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:50 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4162598385' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:50.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:50 smithi110 ceph-mon[212939]: pgmap v166: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T23:03:50.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:50 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4162598385' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:50.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:50 smithi110 ceph-mon[221102]: pgmap v166: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T23:03:50.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:50 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4162598385' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:50.714 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:03:50.714277+0000 min lat: 0.00103361 max lat: 0.249764 avg lat: 0.007114 2023-09-23T23:03:50.714 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:03:50.714 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 20 16 44971 44955 140.438 96.5 0.0157953 0.007114 2023-09-23T23:03:51.716 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 21 16 46513 46497 138.328 96.375 0.00738944 0.00722309 2023-09-23T23:03:52.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:52 smithi138 ceph-mon[193606]: pgmap v167: 89 pgs: 89 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 150 MiB/s wr, 2.39k op/s 2023-09-23T23:03:52.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:52 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1063170256' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:52.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:52 smithi110 ceph-mon[221102]: pgmap v167: 89 pgs: 89 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 150 MiB/s wr, 2.39k op/s 2023-09-23T23:03:52.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:52 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1063170256' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:52.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:52 smithi110 ceph-mon[212939]: pgmap v167: 89 pgs: 89 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 150 MiB/s wr, 2.39k op/s 2023-09-23T23:03:52.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:52 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1063170256' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:52.717 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 22 16 47627 47611 135.199 69.625 0.0125906 0.00739185 2023-09-23T23:03:53.202 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:03:53 smithi110 systemd[1]: Stopping Ceph osd.0 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:03:53.718 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 23 16 49090 49074 133.296 91.4375 0.00794027 0.00749673 2023-09-23T23:03:53.881 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:03:53 smithi110 conmon[125066]: 2023-09-23T23:03:53.484+0000 7f587e941700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:03:53.881 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:03:53 smithi110 conmon[125066]: 2023-09-23T23:03:53.484+0000 7f587e941700 -1 osd.0 87 *** Got signal Terminated *** 2023-09-23T23:03:53.881 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:03:53 smithi110 conmon[125066]: 2023-09-23T23:03:53.484+0000 7f587e941700 -1 osd.0 87 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:03:53.882 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:53 smithi110 ceph-mon[221102]: pgmap v168: 89 pgs: 89 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-09-23T23:03:53.883 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:53 smithi110 ceph-mon[221102]: osd.0 marked itself down and dead 2023-09-23T23:03:54.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:53 smithi138 ceph-mon[193606]: pgmap v168: 89 pgs: 89 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-09-23T23:03:54.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:53 smithi138 ceph-mon[193606]: osd.0 marked itself down and dead 2023-09-23T23:03:54.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:53 smithi110 ceph-mon[212939]: pgmap v168: 89 pgs: 89 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-09-23T23:03:54.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:53 smithi110 ceph-mon[212939]: osd.0 marked itself down and dead 2023-09-23T23:03:54.717 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 24 16 49369 49353 128.471 17.4375 0.00250189 0.00751253 2023-09-23T23:03:54.880 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:03:54 smithi110 bash[237810]: d7807df7af7a63230e2d592785628586f19cae5a7cfa4ab879f79b7328548b72 2023-09-23T23:03:55.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:54 smithi138 ceph-mon[193606]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:03:55.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:54 smithi138 ceph-mon[193606]: failed to encode map e88 with expected crc 2023-09-23T23:03:55.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:54 smithi138 ceph-mon[193606]: failed to encode map e88 with expected crc 2023-09-23T23:03:55.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:54 smithi138 ceph-mon[193606]: osdmap e88: 8 total, 7 up, 8 in 2023-09-23T23:03:55.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:54 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2973277799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:55.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:54 smithi110 ceph-mon[212939]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:03:55.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:54 smithi110 ceph-mon[212939]: failed to encode map e88 with expected crc 2023-09-23T23:03:55.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:54 smithi110 ceph-mon[212939]: failed to encode map e88 with expected crc 2023-09-23T23:03:55.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:54 smithi110 ceph-mon[212939]: osdmap e88: 8 total, 7 up, 8 in 2023-09-23T23:03:55.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:54 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2973277799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:55.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:54 smithi110 ceph-mon[221102]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:03:55.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:54 smithi110 ceph-mon[221102]: failed to encode map e88 with expected crc 2023-09-23T23:03:55.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:54 smithi110 ceph-mon[221102]: failed to encode map e88 with expected crc 2023-09-23T23:03:55.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:54 smithi110 ceph-mon[221102]: osdmap e88: 8 total, 7 up, 8 in 2023-09-23T23:03:55.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:54 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2973277799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:55.718 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 25 16 50462 50446 126.065 68.3125 0.00743577 0.00792812 2023-09-23T23:03:56.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:55 smithi138 ceph-mon[193606]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:55 smithi138 ceph-mon[193606]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:55 smithi138 ceph-mon[193606]: osdmap e89: 8 total, 7 up, 8 in 2023-09-23T23:03:56.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:55 smithi138 ceph-mon[193606]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:55 smithi138 ceph-mon[193606]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:55 smithi138 ceph-mon[193606]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:55 smithi138 ceph-mon[193606]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:55 smithi138 ceph-mon[193606]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:55 smithi138 ceph-mon[193606]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:55 smithi138 ceph-mon[193606]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:55 smithi138 ceph-mon[193606]: pgmap v171: 89 pgs: 34 peering, 55 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-09-23T23:03:56.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[212939]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[212939]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[212939]: osdmap e89: 8 total, 7 up, 8 in 2023-09-23T23:03:56.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[212939]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[212939]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[212939]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[212939]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[212939]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[212939]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[212939]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[212939]: pgmap v171: 89 pgs: 34 peering, 55 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-09-23T23:03:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[221102]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[221102]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[221102]: osdmap e89: 8 total, 7 up, 8 in 2023-09-23T23:03:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[221102]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[221102]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[221102]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[221102]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[221102]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[221102]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[221102]: failed to encode map e89 with expected crc 2023-09-23T23:03:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:55 smithi110 ceph-mon[221102]: pgmap v171: 89 pgs: 34 peering, 55 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-09-23T23:03:56.718 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 26 16 52172 52156 125.327 106.875 0.00631228 0.00797429 2023-09-23T23:03:57.124 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:56 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 9 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2023-09-23T23:03:57.125 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:56 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 9 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2023-09-23T23:03:57.125 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:03:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:03:56] "GET /metrics HTTP/1.1" 200 36210 "" "Prometheus/2.43.0" 2023-09-23T23:03:57.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:56 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 9 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2023-09-23T23:03:57.717 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 27 16 53969 53953 124.844 112.312 0.00668671 0.00800545 2023-09-23T23:03:58.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:57 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3507586229' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:58.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:57 smithi138 ceph-mon[193606]: pgmap v172: 89 pgs: 34 peering, 55 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-09-23T23:03:58.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:57 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3507586229' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:58.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:57 smithi110 ceph-mon[212939]: pgmap v172: 89 pgs: 34 peering, 55 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-09-23T23:03:58.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:57 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3507586229' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:03:58.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:57 smithi110 ceph-mon[221102]: pgmap v172: 89 pgs: 34 peering, 55 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-09-23T23:03:58.701 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:03:58 smithi110 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.0.service: Succeeded. 2023-09-23T23:03:58.701 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:03:58 smithi110 systemd[1]: Stopped Ceph osd.0 for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:03:58.702 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:03:58 smithi110 systemd[1]: Starting Ceph osd.0 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:03:58.718 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 28 16 55748 55732 124.356 111.188 0.0118344 0.0080371 2023-09-23T23:03:59.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:59.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:59.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:59.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:03:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:59.718 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 29 16 57312 57296 123.439 97.75 0.00391098 0.00809679 2023-09-23T23:04:00.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:59 smithi110 ceph-mon[212939]: pgmap v173: 89 pgs: 3 active+undersized+degraded, 31 peering, 55 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s; 7065/152001 objects degraded (4.648%) 2023-09-23T23:04:00.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:03:59 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/26126426' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:00.202 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:04:00 smithi110 bash[238473]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T23:04:00.202 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:04:00 smithi110 bash[238473]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T23:04:00.203 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:04:00 smithi110 bash[238473]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-09-23T23:04:00.203 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:04:00 smithi110 bash[238473]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-09-23T23:04:00.203 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:04:00 smithi110 bash[238473]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-09-23T23:04:00.203 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:04:00 smithi110 bash[238473]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T23:04:00.203 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:04:00 smithi110 bash[238473]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T23:04:00.203 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:04:00 smithi110 bash[238473]: --> ceph-volume lvm activate successful for osd ID: 0 2023-09-23T23:04:00.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:59 smithi110 ceph-mon[221102]: pgmap v173: 89 pgs: 3 active+undersized+degraded, 31 peering, 55 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s; 7065/152001 objects degraded (4.648%) 2023-09-23T23:04:00.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:03:59 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/26126426' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:59 smithi138 ceph-mon[193606]: pgmap v173: 89 pgs: 3 active+undersized+degraded, 31 peering, 55 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s; 7065/152001 objects degraded (4.648%) 2023-09-23T23:04:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:03:59 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/26126426' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:00.721 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 30 16 59039 59023 122.908 107.938 0.0117654 0.00813064 2023-09-23T23:04:01.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:00 smithi110 ceph-mon[212939]: Health check failed: Degraded data redundancy: 7065/152001 objects degraded (4.648%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:01.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:00 smithi110 ceph-mon[221102]: Health check failed: Degraded data redundancy: 7065/152001 objects degraded (4.648%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:01.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:00 smithi138 ceph-mon[193606]: Health check failed: Degraded data redundancy: 7065/152001 objects degraded (4.648%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:01.726 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 31 16 60295 60279 121.455 78.5 0.0103371 0.00822865 2023-09-23T23:04:02.234 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:01 smithi110 ceph-mon[221102]: pgmap v174: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 28194/169278 objects degraded (16.655%) 2023-09-23T23:04:02.234 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:01 smithi110 ceph-mon[212939]: pgmap v174: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 28194/169278 objects degraded (16.655%) 2023-09-23T23:04:02.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:01 smithi138 ceph-mon[193606]: pgmap v174: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 28194/169278 objects degraded (16.655%) 2023-09-23T23:04:02.597 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:04:02 smithi110 bash[238931]: 0c340b7feaa9953c955694b859f2ec198c46966573902eb017dc1c54aa851f5c 2023-09-23T23:04:02.597 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:04:02 smithi110 systemd[1]: Started Ceph osd.0 for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:04:02.727 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 32 16 61821 61805 120.64 95.375 0.0281443 0.00828425 2023-09-23T23:04:02.954 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:04:02 smithi110 conmon[238972]: 2023-09-23T23:04:02.589+0000 7fb1b1f2a640 -1 Falling back to public interface 2023-09-23T23:04:03.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:02 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 11 pgs peering) 2023-09-23T23:04:03.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:02 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3133757830' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:02 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:02 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:02 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:04:03.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:02 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 11 pgs peering) 2023-09-23T23:04:03.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:02 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3133757830' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:03.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:02 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:03.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:02 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:03.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:02 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:04:03.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:02 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 11 pgs peering) 2023-09-23T23:04:03.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:02 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3133757830' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:03.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:02 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:03.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:02 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:03.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:02 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:04:03.729 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 33 16 63369 63353 119.907 96.75 0.0108175 0.00833491 2023-09-23T23:04:04.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:03 smithi138 ceph-mon[193606]: pgmap v175: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 28194/169278 objects degraded (16.655%) 2023-09-23T23:04:04.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:03 smithi110 ceph-mon[221102]: pgmap v175: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 28194/169278 objects degraded (16.655%) 2023-09-23T23:04:04.458 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:03 smithi110 ceph-mon[212939]: pgmap v175: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 28194/169278 objects degraded (16.655%) 2023-09-23T23:04:04.731 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 34 16 64749 64733 118.91 86.25 0.0186979 0.00840529 2023-09-23T23:04:05.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:04 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3679854255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:05.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:04 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3679854255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:05.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:04 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3679854255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:05.732 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 35 16 66110 66094 117.94 85.0625 0.0157382 0.00847351 2023-09-23T23:04:06.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:06 smithi138 ceph-mon[193606]: pgmap v176: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 31978/191955 objects degraded (16.659%) 2023-09-23T23:04:06.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:06 smithi110 ceph-mon[212939]: pgmap v176: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 31978/191955 objects degraded (16.659%) 2023-09-23T23:04:06.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:06 smithi110 ceph-mon[221102]: pgmap v176: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 31978/191955 objects degraded (16.659%) 2023-09-23T23:04:06.734 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 36 16 67476 67460 117.03 85.375 0.00746587 0.00853953 2023-09-23T23:04:07.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:04:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:04:06] "GET /metrics HTTP/1.1" 200 36349 "" "Prometheus/2.43.0" 2023-09-23T23:04:07.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:07 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 31978/191955 objects degraded (16.659%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:07.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:07 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 31978/191955 objects degraded (16.659%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:07.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:07 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 31978/191955 objects degraded (16.659%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:07.735 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 37 16 68692 68676 115.921 76 0.0130136 0.00862144 2023-09-23T23:04:08.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:08 smithi138 ceph-mon[193606]: pgmap v177: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 31978/191955 objects degraded (16.659%) 2023-09-23T23:04:08.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:08 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:08.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:08 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:08.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:08 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/327131991' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:08.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:08 smithi110 ceph-mon[221102]: pgmap v177: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 31978/191955 objects degraded (16.659%) 2023-09-23T23:04:08.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:08 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:08.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:08 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:08.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:08 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/327131991' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:08.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:08 smithi110 ceph-mon[212939]: pgmap v177: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 31978/191955 objects degraded (16.659%) 2023-09-23T23:04:08.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:08 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:08.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:08 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:08.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:08 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/327131991' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:08.735 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 38 16 69891 69875 114.843 74.9375 0.0125823 0.00870223 2023-09-23T23:04:09.202 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:04:08 smithi110 conmon[238972]: 2023-09-23T23:04:08.795+0000 7fb1b1f2a640 -1 osd.0 87 log_to_monitors true 2023-09-23T23:04:09.739 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 39 16 71230 71214 114.032 83.6875 0.0102984 0.00876482 2023-09-23T23:04:09.908 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:09 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:09.909 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:09 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:09.909 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:09 smithi110 ceph-mon[221102]: from='osd.0 [v2:172.21.15.110:6802/983471373,v1:172.21.15.110:6803/983471373]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T23:04:09.909 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:09 smithi110 ceph-mon[221102]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T23:04:09.909 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:09 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:09.909 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:09 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:09.909 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:09 smithi110 ceph-mon[212939]: from='osd.0 [v2:172.21.15.110:6802/983471373,v1:172.21.15.110:6803/983471373]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T23:04:09.910 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:09 smithi110 ceph-mon[212939]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T23:04:09.910 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:04:09 smithi110 conmon[238972]: 2023-09-23T23:04:09.524+0000 7fb1a0b92700 -1 osd.0 87 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T23:04:09.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:09 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:09.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:09 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:09.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:09 smithi138 ceph-mon[193606]: from='osd.0 [v2:172.21.15.110:6802/983471373,v1:172.21.15.110:6803/983471373]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T23:04:09.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:09 smithi138 ceph-mon[193606]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T23:04:10.739 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:04:10.739655+0000 min lat: 0.00103361 max lat: 1.03394 avg lat: 0.00879457 2023-09-23T23:04:10.739 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:04:10.740 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 40 16 72396 72380 113.003 72.875 0.00264297 0.00879457 2023-09-23T23:04:10.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:10 smithi138 ceph-mon[193606]: pgmap v178: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 32784/195606 objects degraded (16.760%) 2023-09-23T23:04:10.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:10 smithi138 ceph-mon[193606]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T23:04:10.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:10 smithi138 ceph-mon[193606]: failed to encode map e90 with expected crc 2023-09-23T23:04:10.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:10 smithi138 ceph-mon[193606]: osdmap e90: 8 total, 7 up, 8 in 2023-09-23T23:04:10.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:10 smithi138 ceph-mon[193606]: from='osd.0 [v2:172.21.15.110:6802/983471373,v1:172.21.15.110:6803/983471373]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:04:10.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:10 smithi138 ceph-mon[193606]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:04:10.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:10 smithi138 ceph-mon[193606]: failed to encode map e90 with expected crc 2023-09-23T23:04:10.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:10 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/384288066' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:10.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:10 smithi110 ceph-mon[212939]: pgmap v178: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 32784/195606 objects degraded (16.760%) 2023-09-23T23:04:10.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:10 smithi110 ceph-mon[212939]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T23:04:10.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:10 smithi110 ceph-mon[212939]: failed to encode map e90 with expected crc 2023-09-23T23:04:10.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:10 smithi110 ceph-mon[212939]: osdmap e90: 8 total, 7 up, 8 in 2023-09-23T23:04:10.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:10 smithi110 ceph-mon[212939]: from='osd.0 [v2:172.21.15.110:6802/983471373,v1:172.21.15.110:6803/983471373]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:04:10.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:10 smithi110 ceph-mon[212939]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:04:10.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:10 smithi110 ceph-mon[212939]: failed to encode map e90 with expected crc 2023-09-23T23:04:10.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:10 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/384288066' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:10.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:10 smithi110 ceph-mon[221102]: pgmap v178: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 32784/195606 objects degraded (16.760%) 2023-09-23T23:04:10.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:10 smithi110 ceph-mon[221102]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T23:04:10.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:10 smithi110 ceph-mon[221102]: failed to encode map e90 with expected crc 2023-09-23T23:04:10.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:10 smithi110 ceph-mon[221102]: osdmap e90: 8 total, 7 up, 8 in 2023-09-23T23:04:10.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:10 smithi110 ceph-mon[221102]: from='osd.0 [v2:172.21.15.110:6802/983471373,v1:172.21.15.110:6803/983471373]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:04:10.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:10 smithi110 ceph-mon[221102]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:04:10.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:10 smithi110 ceph-mon[221102]: failed to encode map e90 with expected crc 2023-09-23T23:04:10.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:10 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/384288066' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:11.739 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 41 16 72396 72380 110.249 0 - 0.00879457 2023-09-23T23:04:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:11 smithi138 ceph-mon[193606]: failed to encode map e90 with expected crc 2023-09-23T23:04:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:11 smithi138 ceph-mon[193606]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:04:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:11 smithi138 ceph-mon[193606]: osd.0 [v2:172.21.15.110:6802/983471373,v1:172.21.15.110:6803/983471373] boot 2023-09-23T23:04:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:11 smithi138 ceph-mon[193606]: osdmap e91: 8 total, 8 up, 8 in 2023-09-23T23:04:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:11 smithi138 ceph-mon[193606]: failed to encode map e91 with expected crc 2023-09-23T23:04:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:11 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T23:04:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:11 smithi138 ceph-mon[193606]: failed to encode map e91 with expected crc 2023-09-23T23:04:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:11 smithi138 ceph-mon[193606]: pgmap v181: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s; 35235/211338 objects degraded (16.672%) 2023-09-23T23:04:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:11 smithi138 ceph-mon[193606]: osdmap e92: 8 total, 8 up, 8 in 2023-09-23T23:04:11.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:11 smithi138 ceph-mon[193606]: failed to encode map e92 with expected crc 2023-09-23T23:04:11.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[212939]: failed to encode map e90 with expected crc 2023-09-23T23:04:11.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[212939]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:04:11.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[212939]: osd.0 [v2:172.21.15.110:6802/983471373,v1:172.21.15.110:6803/983471373] boot 2023-09-23T23:04:11.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[212939]: osdmap e91: 8 total, 8 up, 8 in 2023-09-23T23:04:11.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[212939]: failed to encode map e91 with expected crc 2023-09-23T23:04:11.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T23:04:11.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[212939]: failed to encode map e91 with expected crc 2023-09-23T23:04:11.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[212939]: pgmap v181: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s; 35235/211338 objects degraded (16.672%) 2023-09-23T23:04:11.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[212939]: osdmap e92: 8 total, 8 up, 8 in 2023-09-23T23:04:11.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[212939]: failed to encode map e92 with expected crc 2023-09-23T23:04:11.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[221102]: failed to encode map e90 with expected crc 2023-09-23T23:04:11.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[221102]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:04:11.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[221102]: osd.0 [v2:172.21.15.110:6802/983471373,v1:172.21.15.110:6803/983471373] boot 2023-09-23T23:04:11.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[221102]: osdmap e91: 8 total, 8 up, 8 in 2023-09-23T23:04:11.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[221102]: failed to encode map e91 with expected crc 2023-09-23T23:04:11.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T23:04:11.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[221102]: failed to encode map e91 with expected crc 2023-09-23T23:04:11.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[221102]: pgmap v181: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s; 35235/211338 objects degraded (16.672%) 2023-09-23T23:04:11.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[221102]: osdmap e92: 8 total, 8 up, 8 in 2023-09-23T23:04:11.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:11 smithi110 ceph-mon[221102]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.609 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[212939]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[212939]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[212939]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[212939]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[212939]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[212939]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 35235/211338 objects degraded (16.672%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:12.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2477171977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:12.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[212939]: osdmap e93: 8 total, 8 up, 8 in 2023-09-23T23:04:12.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[212939]: failed to encode map e93 with expected crc 2023-09-23T23:04:12.610 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[221102]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.611 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[221102]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.611 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[221102]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.611 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[221102]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.611 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[221102]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.611 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[221102]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.611 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 35235/211338 objects degraded (16.672%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:12.611 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2477171977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:12.611 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[221102]: osdmap e93: 8 total, 8 up, 8 in 2023-09-23T23:04:12.611 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:12 smithi110 ceph-mon[221102]: failed to encode map e93 with expected crc 2023-09-23T23:04:12.740 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 42 16 72396 72380 107.626 0 - 0.00879457 2023-09-23T23:04:12.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:12 smithi138 ceph-mon[193606]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:12 smithi138 ceph-mon[193606]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:12 smithi138 ceph-mon[193606]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:12 smithi138 ceph-mon[193606]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:12 smithi138 ceph-mon[193606]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:12 smithi138 ceph-mon[193606]: failed to encode map e92 with expected crc 2023-09-23T23:04:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:12 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 35235/211338 objects degraded (16.672%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:12 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2477171977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:12.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:12 smithi138 ceph-mon[193606]: osdmap e93: 8 total, 8 up, 8 in 2023-09-23T23:04:12.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:12 smithi138 ceph-mon[193606]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.740 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 43 16 73169 73153 106.247 16.1042 0.013812 0.00940689 2023-09-23T23:04:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:13 smithi138 ceph-mon[193606]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:13 smithi138 ceph-mon[193606]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:13 smithi138 ceph-mon[193606]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:13 smithi138 ceph-mon[193606]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:13 smithi138 ceph-mon[193606]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:13 smithi138 ceph-mon[193606]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:13.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:13 smithi138 ceph-mon[193606]: pgmap v184: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 35235/211338 objects degraded (16.672%) 2023-09-23T23:04:13.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[221102]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[221102]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[221102]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[221102]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[221102]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[221102]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[221102]: pgmap v184: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 35235/211338 objects degraded (16.672%) 2023-09-23T23:04:13.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[212939]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[212939]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[212939]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[212939]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[212939]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[212939]: failed to encode map e93 with expected crc 2023-09-23T23:04:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:13 smithi110 ceph-mon[212939]: pgmap v184: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 35235/211338 objects degraded (16.672%) 2023-09-23T23:04:14.745 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 44 16 74067 74051 105.096 56.125 0.0160124 0.00950999 2023-09-23T23:04:15.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:14 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2278519699' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:15.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:14 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2278519699' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:15.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:14 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2278519699' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:15.747 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 45 16 75044 75028 104.114 61.0625 0.0220829 0.00960039 2023-09-23T23:04:16.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:15 smithi138 ceph-mon[193606]: pgmap v185: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 35 MiB/s wr, 564 op/s; 11464/220701 objects degraded (5.194%); 786 KiB/s, 12 objects/s recovering 2023-09-23T23:04:16.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:15 smithi110 ceph-mon[212939]: pgmap v185: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 35 MiB/s wr, 564 op/s; 11464/220701 objects degraded (5.194%); 786 KiB/s, 12 objects/s recovering 2023-09-23T23:04:16.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:15 smithi110 ceph-mon[221102]: pgmap v185: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 35 MiB/s wr, 564 op/s; 11464/220701 objects degraded (5.194%); 786 KiB/s, 12 objects/s recovering 2023-09-23T23:04:16.747 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 46 16 75959 75943 103.095 57.1875 0.0176897 0.00969485 2023-09-23T23:04:17.204 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:04:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:04:16] "GET /metrics HTTP/1.1" 200 36271 "" "Prometheus/2.43.0" 2023-09-23T23:04:17.747 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 47 16 76878 76862 102.124 57.4375 0.01022 0.00978821 2023-09-23T23:04:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:18 smithi138 ceph-mon[193606]: pgmap v186: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 30 MiB/s wr, 478 op/s; 11464/220701 objects degraded (5.194%); 667 KiB/s, 10 objects/s recovering 2023-09-23T23:04:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/693000244' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:18 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 11464/220701 objects degraded (5.194%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:18.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:18 smithi110 ceph-mon[221102]: pgmap v186: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 30 MiB/s wr, 478 op/s; 11464/220701 objects degraded (5.194%); 667 KiB/s, 10 objects/s recovering 2023-09-23T23:04:18.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/693000244' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:18.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:18 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 11464/220701 objects degraded (5.194%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:18.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:18 smithi110 ceph-mon[212939]: pgmap v186: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 30 MiB/s wr, 478 op/s; 11464/220701 objects degraded (5.194%); 667 KiB/s, 10 objects/s recovering 2023-09-23T23:04:18.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/693000244' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:18.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:18 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 11464/220701 objects degraded (5.194%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:18.747 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 48 16 77841 77825 101.251 60.1875 0.0214958 0.00987104 2023-09-23T23:04:19.454 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:04:19 smithi110 conmon[198813]: 2023-09-23T23:04:19.165+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T23:04:19.747 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 49 16 78735 78719 100.325 55.875 0.0240853 0.00996095 2023-09-23T23:04:20.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:19 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:20.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:19 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:20.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:19 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:04:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:19 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:04:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:19 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:19 smithi138 ceph-mon[193606]: pgmap v187: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 31 MiB/s wr, 492 op/s; 11464/223170 objects degraded (5.137%); 544 KiB/s, 8 objects/s recovering 2023-09-23T23:04:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:19 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:04:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:19 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:04:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:19 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:04:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:19 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:04:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:19 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:19 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:20.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:19 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T23:04:20.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:19 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1381845740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:20.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:20.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:20.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:04:20.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:04:20.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:20.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[212939]: pgmap v187: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 31 MiB/s wr, 492 op/s; 11464/223170 objects degraded (5.137%); 544 KiB/s, 8 objects/s recovering 2023-09-23T23:04:20.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:04:20.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:04:20.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:04:20.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:04:20.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:20.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:20.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T23:04:20.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1381845740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:20.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:20.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:20.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:04:20.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:04:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:04:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[221102]: pgmap v187: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 31 MiB/s wr, 492 op/s; 11464/223170 objects degraded (5.137%); 544 KiB/s, 8 objects/s recovering 2023-09-23T23:04:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:04:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:04:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:04:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:04:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T23:04:20.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:19 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1381845740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:20.750 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 50 16 79716 79700 99.5417 61.3125 0.0174768 0.0100409 2023-09-23T23:04:21.750 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 51 16 80697 80681 98.7925 61.3125 0.00882452 0.0101179 2023-09-23T23:04:22.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:22 smithi138 ceph-mon[193606]: pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s; 11133/234696 objects degraded (4.744%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:04:22.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:22 smithi110 ceph-mon[221102]: pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s; 11133/234696 objects degraded (4.744%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:04:22.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:22 smithi110 ceph-mon[212939]: pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s; 11133/234696 objects degraded (4.744%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:04:22.750 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 52 16 81586 81570 97.9617 55.5625 0.0163939 0.0102027 2023-09-23T23:04:23.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:23 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4121175182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:23.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:23 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 11133/234696 objects degraded (4.744%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:23.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:23 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4121175182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:23.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:23 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 11133/234696 objects degraded (4.744%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:23.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:23 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4121175182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:23.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:23 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 11133/234696 objects degraded (4.744%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:23.749 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 53 16 82543 82527 97.2423 59.8125 0.0173183 0.0102786 2023-09-23T23:04:24.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:24 smithi138 ceph-mon[193606]: pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 11133/234696 objects degraded (4.744%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:04:24.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:24 smithi110 ceph-mon[221102]: pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 11133/234696 objects degraded (4.744%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:04:24.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:24 smithi110 ceph-mon[212939]: pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 11133/234696 objects degraded (4.744%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:04:24.750 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 54 16 83473 83457 96.5183 58.125 0.00714142 0.0103555 2023-09-23T23:04:25.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:25 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/938185629' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:25.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:25 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/938185629' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:25.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:25 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/938185629' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:25.753 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 55 16 84412 84396 95.8259 58.6875 0.0154889 0.0104312 2023-09-23T23:04:26.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:26 smithi138 ceph-mon[193606]: pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 10775/248838 objects degraded (4.330%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T23:04:26.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:26 smithi110 ceph-mon[212939]: pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 10775/248838 objects degraded (4.330%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T23:04:26.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:26 smithi110 ceph-mon[221102]: pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 10775/248838 objects degraded (4.330%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T23:04:26.753 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 56 16 85358 85342 95.1709 59.125 0.0152999 0.0105028 2023-09-23T23:04:27.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:04:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:04:26] "GET /metrics HTTP/1.1" 200 36271 "" "Prometheus/2.43.0" 2023-09-23T23:04:27.753 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 57 16 86228 86212 94.4556 54.375 0.0113801 0.010582 2023-09-23T23:04:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:28 smithi138 ceph-mon[193606]: pgmap v191: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s; 10775/248838 objects degraded (4.330%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:04:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:28 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1931567997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:28 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 10775/248838 objects degraded (4.330%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:28.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:28 smithi110 ceph-mon[212939]: pgmap v191: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s; 10775/248838 objects degraded (4.330%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:04:28.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:28 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1931567997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:28.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:28 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 10775/248838 objects degraded (4.330%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:28.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:28 smithi110 ceph-mon[221102]: pgmap v191: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s; 10775/248838 objects degraded (4.330%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:04:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:28 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1931567997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:28.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:28 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 10775/248838 objects degraded (4.330%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:28.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:28.753 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 58 16 87163 87147 93.8348 58.4375 0.019335 0.0106518 2023-09-23T23:04:29.753 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 59 16 88086 88070 93.2225 57.6875 0.0143611 0.010723 2023-09-23T23:04:30.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:30 smithi138 ceph-mon[193606]: pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s; 10775/252294 objects degraded (4.271%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:04:30.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:30 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1253553934' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:30 smithi110 ceph-mon[212939]: pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s; 10775/252294 objects degraded (4.271%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:04:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:30 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1253553934' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:30.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:30 smithi110 ceph-mon[221102]: pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s; 10775/252294 objects degraded (4.271%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:04:30.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:30 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1253553934' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:30.753 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:04:30.753369+0000 min lat: 0.00103361 max lat: 2.45499 avg lat: 0.0107945 2023-09-23T23:04:30.753 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:04:30.753 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 60 16 88979 88963 92.5993 55.8125 0.0112815 0.0107945 2023-09-23T23:04:31.753 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 61 16 89891 89875 92.016 57 0.0120987 0.0108618 2023-09-23T23:04:32.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:32 smithi138 ceph-mon[193606]: pgmap v193: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 10387/262638 objects degraded (3.955%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T23:04:32.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:32 smithi110 ceph-mon[212939]: pgmap v193: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 10387/262638 objects degraded (3.955%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T23:04:32.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:32 smithi110 ceph-mon[221102]: pgmap v193: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 10387/262638 objects degraded (3.955%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T23:04:32.753 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 62 16 90656 90640 91.3034 47.8125 0.00403349 0.0108992 2023-09-23T23:04:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:33 smithi138 ceph-mon[193606]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T23:04:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:33 smithi138 ceph-mon[193606]: failed to encode map e94 with expected crc 2023-09-23T23:04:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:33 smithi138 ceph-mon[193606]: failed to encode map e94 with expected crc 2023-09-23T23:04:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:33 smithi138 ceph-mon[193606]: failed to encode map e94 with expected crc 2023-09-23T23:04:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:33 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4007359221' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:33 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 10387/262638 objects degraded (3.955%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:33.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:33 smithi110 ceph-mon[212939]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T23:04:33.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:33 smithi110 ceph-mon[212939]: failed to encode map e94 with expected crc 2023-09-23T23:04:33.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:33 smithi110 ceph-mon[212939]: failed to encode map e94 with expected crc 2023-09-23T23:04:33.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:33 smithi110 ceph-mon[212939]: failed to encode map e94 with expected crc 2023-09-23T23:04:33.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:33 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4007359221' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:33.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:33 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 10387/262638 objects degraded (3.955%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:33.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:33 smithi110 ceph-mon[221102]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T23:04:33.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:33 smithi110 ceph-mon[221102]: failed to encode map e94 with expected crc 2023-09-23T23:04:33.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:33 smithi110 ceph-mon[221102]: failed to encode map e94 with expected crc 2023-09-23T23:04:33.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:33 smithi110 ceph-mon[221102]: failed to encode map e94 with expected crc 2023-09-23T23:04:33.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:33 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4007359221' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:33.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:33 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 10387/262638 objects degraded (3.955%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:33.757 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 63 15 91107 91092 90.299 28.25 0.0191124 0.0110687 2023-09-23T23:04:34.451 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:04:34 smithi110 conmon[198813]: 2023-09-23T23:04:34.188+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T23:04:34.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:34 smithi110 ceph-mon[221102]: pgmap v195: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s; 10387/262638 objects degraded (3.955%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:04:34.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:34 smithi110 ceph-mon[221102]: failed to encode map e95 with expected crc 2023-09-23T23:04:34.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:34 smithi110 ceph-mon[221102]: failed to encode map e95 with expected crc 2023-09-23T23:04:34.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:34 smithi110 ceph-mon[221102]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T23:04:34.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:34 smithi110 ceph-mon[221102]: failed to encode map e95 with expected crc 2023-09-23T23:04:34.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:34 smithi110 ceph-mon[221102]: failed to encode map e95 with expected crc 2023-09-23T23:04:34.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:34 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:34.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:34 smithi110 ceph-mon[212939]: pgmap v195: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s; 10387/262638 objects degraded (3.955%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:04:34.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:34 smithi110 ceph-mon[212939]: failed to encode map e95 with expected crc 2023-09-23T23:04:34.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:34 smithi110 ceph-mon[212939]: failed to encode map e95 with expected crc 2023-09-23T23:04:34.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:34 smithi110 ceph-mon[212939]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T23:04:34.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:34 smithi110 ceph-mon[212939]: failed to encode map e95 with expected crc 2023-09-23T23:04:34.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:34 smithi110 ceph-mon[212939]: failed to encode map e95 with expected crc 2023-09-23T23:04:34.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:34 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:34.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:34 smithi138 ceph-mon[193606]: pgmap v195: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s; 10387/262638 objects degraded (3.955%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:04:34.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:34 smithi138 ceph-mon[193606]: failed to encode map e95 with expected crc 2023-09-23T23:04:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:34 smithi138 ceph-mon[193606]: failed to encode map e95 with expected crc 2023-09-23T23:04:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:34 smithi138 ceph-mon[193606]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T23:04:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:34 smithi138 ceph-mon[193606]: failed to encode map e95 with expected crc 2023-09-23T23:04:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:34 smithi138 ceph-mon[193606]: failed to encode map e95 with expected crc 2023-09-23T23:04:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:34 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:34.758 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 64 16 92016 92000 89.7725 56.75 0.0207499 0.011134 2023-09-23T23:04:35.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:35 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:35.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:35 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T23:04:35.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:35 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3488752125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:35.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:35 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:35.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:35 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T23:04:35.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:35 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3488752125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:35.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:35 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:35.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:35 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T23:04:35.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:35 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3488752125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:35.758 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 65 16 92866 92850 89.209 53.125 0.0132285 0.0112044 2023-09-23T23:04:36.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:36 smithi138 ceph-mon[193606]: pgmap v197: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 9899/274602 objects degraded (3.605%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T23:04:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:36 smithi110 ceph-mon[212939]: pgmap v197: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 9899/274602 objects degraded (3.605%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T23:04:36.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:36 smithi110 ceph-mon[221102]: pgmap v197: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 9899/274602 objects degraded (3.605%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T23:04:36.760 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 66 16 93790 93774 88.7298 57.75 0.0134846 0.0112656 2023-09-23T23:04:37.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:04:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:04:36] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.43.0" 2023-09-23T23:04:37.763 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 67 16 94643 94627 88.1979 53.3125 0.00813417 0.0113333 2023-09-23T23:04:38.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:38 smithi138 ceph-mon[193606]: pgmap v198: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s; 9899/274602 objects degraded (3.605%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T23:04:38.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:38 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1794195146' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:38.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:38 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 9899/274602 objects degraded (3.605%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:38.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:38 smithi110 ceph-mon[212939]: pgmap v198: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s; 9899/274602 objects degraded (3.605%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T23:04:38.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:38 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1794195146' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:38.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:38 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 9899/274602 objects degraded (3.605%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:38.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:38 smithi110 ceph-mon[221102]: pgmap v198: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s; 9899/274602 objects degraded (3.605%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T23:04:38.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:38 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1794195146' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:38.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:38 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 9899/274602 objects degraded (3.605%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:38.766 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 68 16 95532 95516 87.7142 55.5625 0.0214432 0.0113965 2023-09-23T23:04:39.770 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 69 16 96426 96410 87.2491 55.875 0.00777023 0.0114566 2023-09-23T23:04:40.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:40 smithi138 ceph-mon[193606]: pgmap v199: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 632 op/s; 9536/277839 objects degraded (3.432%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T23:04:40.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:40 smithi110 ceph-mon[221102]: pgmap v199: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 632 op/s; 9536/277839 objects degraded (3.432%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T23:04:40.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:40 smithi110 ceph-mon[212939]: pgmap v199: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 632 op/s; 9536/277839 objects degraded (3.432%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T23:04:40.770 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 70 16 97254 97238 86.7423 51.75 0.00785006 0.0115229 2023-09-23T23:04:41.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:41 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2730069547' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:41.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:41 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2730069547' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:41.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:41 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2730069547' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:41.772 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 71 16 98133 98117 86.2917 54.9375 0.0126295 0.0115831 2023-09-23T23:04:42.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:42 smithi138 ceph-mon[193606]: pgmap v200: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s; 9536/287802 objects degraded (3.313%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T23:04:42.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:42 smithi110 ceph-mon[221102]: pgmap v200: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s; 9536/287802 objects degraded (3.313%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T23:04:42.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:42 smithi110 ceph-mon[212939]: pgmap v200: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s; 9536/287802 objects degraded (3.313%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T23:04:42.772 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 72 16 98966 98950 85.8166 52.0625 0.00343171 0.0116448 2023-09-23T23:04:43.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:43 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 9536/287802 objects degraded (3.313%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:43.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:43 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3509969790' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:43.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:43.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:43 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 9536/287802 objects degraded (3.313%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:43.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:43 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3509969790' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:43.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:43.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:43 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 9536/287802 objects degraded (3.313%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:43.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:43 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3509969790' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:43.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:43.776 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 73 16 99796 99780 85.348 51.875 0.0260263 0.0117115 2023-09-23T23:04:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:44 smithi138 ceph-mon[193606]: pgmap v201: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s; 9536/287802 objects degraded (3.313%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:04:44.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:44 smithi110 ceph-mon[221102]: pgmap v201: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s; 9536/287802 objects degraded (3.313%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:04:44.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:44 smithi110 ceph-mon[212939]: pgmap v201: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s; 9536/287802 objects degraded (3.313%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:04:44.778 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 74 16 100625 100609 84.8926 51.8125 0.0167984 0.0117748 2023-09-23T23:04:45.778 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 75 16 101429 101413 84.4311 50.25 0.0261465 0.0118391 2023-09-23T23:04:46.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:46 smithi138 ceph-mon[193606]: pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 9142/300480 objects degraded (3.042%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T23:04:46.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:46 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2513285642' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:46.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:46 smithi110 ceph-mon[212939]: pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 9142/300480 objects degraded (3.042%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T23:04:46.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:46 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2513285642' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:46.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:46 smithi110 ceph-mon[221102]: pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 9142/300480 objects degraded (3.042%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T23:04:46.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:46 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2513285642' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:46.778 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 76 16 102280 102264 84.0201 53.1875 0.0103168 0.0118966 2023-09-23T23:04:47.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:04:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:04:46] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.43.0" 2023-09-23T23:04:47.781 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 77 16 103105 103089 83.5961 51.5625 0.0250249 0.0119567 2023-09-23T23:04:48.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:48 smithi138 ceph-mon[193606]: pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s; 9142/300480 objects degraded (3.042%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T23:04:48.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:48 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 9142/300480 objects degraded (3.042%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:48.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:48 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1239763606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:48.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:48 smithi110 ceph-mon[212939]: pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s; 9142/300480 objects degraded (3.042%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T23:04:48.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:48 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 9142/300480 objects degraded (3.042%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:48.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:48 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1239763606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:48.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:48 smithi110 ceph-mon[221102]: pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s; 9142/300480 objects degraded (3.042%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T23:04:48.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:48 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 9142/300480 objects degraded (3.042%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:48.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:48 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1239763606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:48.781 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 78 16 103933 103917 83.1881 51.75 0.0315709 0.0120166 2023-09-23T23:04:49.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:49 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:49.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:49 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:49.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:04:49 smithi110 conmon[198813]: 2023-09-23T23:04:49.226+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:04:49.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:49 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:49.781 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 79 16 104570 104554 82.6394 39.8125 0.00383575 0.0120531 2023-09-23T23:04:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:50 smithi138 ceph-mon[193606]: pgmap v204: 89 pgs: 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s; 8752/303561 objects degraded (2.883%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T23:04:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:50 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:50 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:04:50.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:50 smithi138 ceph-mon[193606]: failed to encode map e96 with expected crc 2023-09-23T23:04:50.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:50 smithi138 ceph-mon[193606]: osdmap e96: 8 total, 8 up, 8 in 2023-09-23T23:04:50.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:50 smithi138 ceph-mon[193606]: failed to encode map e96 with expected crc 2023-09-23T23:04:50.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:50 smithi138 ceph-mon[193606]: failed to encode map e96 with expected crc 2023-09-23T23:04:50.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:50 smithi110 ceph-mon[221102]: pgmap v204: 89 pgs: 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s; 8752/303561 objects degraded (2.883%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T23:04:50.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:50 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:50.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:50 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:04:50.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:50 smithi110 ceph-mon[221102]: failed to encode map e96 with expected crc 2023-09-23T23:04:50.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:50 smithi110 ceph-mon[221102]: osdmap e96: 8 total, 8 up, 8 in 2023-09-23T23:04:50.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:50 smithi110 ceph-mon[221102]: failed to encode map e96 with expected crc 2023-09-23T23:04:50.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:50 smithi110 ceph-mon[221102]: failed to encode map e96 with expected crc 2023-09-23T23:04:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:50 smithi110 ceph-mon[212939]: pgmap v204: 89 pgs: 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s; 8752/303561 objects degraded (2.883%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T23:04:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:50 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:04:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:50 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:04:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:50 smithi110 ceph-mon[212939]: failed to encode map e96 with expected crc 2023-09-23T23:04:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:50 smithi110 ceph-mon[212939]: osdmap e96: 8 total, 8 up, 8 in 2023-09-23T23:04:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:50 smithi110 ceph-mon[212939]: failed to encode map e96 with expected crc 2023-09-23T23:04:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:50 smithi110 ceph-mon[212939]: failed to encode map e96 with expected crc 2023-09-23T23:04:50.781 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:04:50.781726+0000 min lat: 0.00103361 max lat: 2.45499 avg lat: 0.0122143 2023-09-23T23:04:50.782 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:04:50.782 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 80 16 104860 104844 81.8336 18.125 0.00883482 0.0122143 2023-09-23T23:04:51.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:51 smithi138 ceph-mon[193606]: failed to encode map e97 with expected crc 2023-09-23T23:04:51.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:51 smithi138 ceph-mon[193606]: failed to encode map e97 with expected crc 2023-09-23T23:04:51.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:51 smithi138 ceph-mon[193606]: osdmap e97: 8 total, 8 up, 8 in 2023-09-23T23:04:51.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:51 smithi138 ceph-mon[193606]: failed to encode map e97 with expected crc 2023-09-23T23:04:51.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:51 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1478828303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:51.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:51 smithi138 ceph-mon[193606]: failed to encode map e97 with expected crc 2023-09-23T23:04:51.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:51 smithi110 ceph-mon[212939]: failed to encode map e97 with expected crc 2023-09-23T23:04:51.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:51 smithi110 ceph-mon[212939]: failed to encode map e97 with expected crc 2023-09-23T23:04:51.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:51 smithi110 ceph-mon[212939]: osdmap e97: 8 total, 8 up, 8 in 2023-09-23T23:04:51.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:51 smithi110 ceph-mon[212939]: failed to encode map e97 with expected crc 2023-09-23T23:04:51.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:51 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1478828303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:51.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:51 smithi110 ceph-mon[212939]: failed to encode map e97 with expected crc 2023-09-23T23:04:51.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:51 smithi110 ceph-mon[221102]: failed to encode map e97 with expected crc 2023-09-23T23:04:51.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:51 smithi110 ceph-mon[221102]: failed to encode map e97 with expected crc 2023-09-23T23:04:51.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:51 smithi110 ceph-mon[221102]: osdmap e97: 8 total, 8 up, 8 in 2023-09-23T23:04:51.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:51 smithi110 ceph-mon[221102]: failed to encode map e97 with expected crc 2023-09-23T23:04:51.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:51 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1478828303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:51.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:51 smithi110 ceph-mon[221102]: failed to encode map e97 with expected crc 2023-09-23T23:04:51.788 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 81 16 105676 105660 81.4532 51 0.0152573 0.0122723 2023-09-23T23:04:52.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:52 smithi138 ceph-mon[193606]: pgmap v207: 89 pgs: 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 8720/312897 objects degraded (2.787%); 6.4 MiB/s, 101 objects/s recovering 2023-09-23T23:04:52.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:52 smithi110 ceph-mon[212939]: pgmap v207: 89 pgs: 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 8720/312897 objects degraded (2.787%); 6.4 MiB/s, 101 objects/s recovering 2023-09-23T23:04:52.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:52 smithi110 ceph-mon[221102]: pgmap v207: 89 pgs: 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 8720/312897 objects degraded (2.787%); 6.4 MiB/s, 101 objects/s recovering 2023-09-23T23:04:52.785 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 82 16 106463 106447 81.0569 49.1875 0.013415 0.0123316 2023-09-23T23:04:53.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:53 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 8720/312897 objects degraded (2.787%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:53.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:53 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2565525404' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:53.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:53 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 8720/312897 objects degraded (2.787%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:53.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:53 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2565525404' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:53.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:53 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 8720/312897 objects degraded (2.787%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:53.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:53 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2565525404' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:53.785 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 83 16 107265 107249 80.6845 50.125 0.0158676 0.0123889 2023-09-23T23:04:54.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:54 smithi138 ceph-mon[193606]: pgmap v208: 89 pgs: 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32 MiB/s wr, 516 op/s; 8720/312897 objects degraded (2.787%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T23:04:54.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:54 smithi110 ceph-mon[221102]: pgmap v208: 89 pgs: 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32 MiB/s wr, 516 op/s; 8720/312897 objects degraded (2.787%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T23:04:54.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:54 smithi110 ceph-mon[212939]: pgmap v208: 89 pgs: 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32 MiB/s wr, 516 op/s; 8720/312897 objects degraded (2.787%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T23:04:54.785 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 84 16 108087 108071 80.3358 51.375 0.00892098 0.0124436 2023-09-23T23:04:55.785 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 85 16 108847 108831 79.9496 47.5 0.0180427 0.012503 2023-09-23T23:04:56.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:56 smithi110 ceph-mon[221102]: pgmap v209: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s; 8304/322896 objects degraded (2.572%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T23:04:56.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:56 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2189161565' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:56.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:56 smithi110 ceph-mon[212939]: pgmap v209: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s; 8304/322896 objects degraded (2.572%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T23:04:56.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:56 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2189161565' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:56.785 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 86 16 109670 109654 79.6182 51.4375 0.0281943 0.0125551 2023-09-23T23:04:56.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:56 smithi138 ceph-mon[193606]: pgmap v209: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s; 8304/322896 objects degraded (2.572%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T23:04:56.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:56 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2189161565' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:57.204 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:04:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:04:56] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.43.0" 2023-09-23T23:04:57.788 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 87 16 110474 110458 79.2781 50.25 0.00863575 0.0126089 2023-09-23T23:04:58.789 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 88 16 111224 111208 78.91 46.875 0.0283718 0.012668 2023-09-23T23:04:58.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:58 smithi138 ceph-mon[193606]: pgmap v210: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s; 8304/322896 objects degraded (2.572%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T23:04:58.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:58 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 8304/322896 objects degraded (2.572%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:58.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:58.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:04:58 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/654706813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:58.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:58 smithi110 ceph-mon[212939]: pgmap v210: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s; 8304/322896 objects degraded (2.572%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T23:04:58.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:58 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 8304/322896 objects degraded (2.572%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:58.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:58.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:04:58 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/654706813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:58.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:58 smithi110 ceph-mon[221102]: pgmap v210: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s; 8304/322896 objects degraded (2.572%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T23:04:58.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:58 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 8304/322896 objects degraded (2.572%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:04:58.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:58.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:04:58 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/654706813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:04:59.789 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 89 16 112023 112007 78.5846 49.9375 0.0155266 0.0127201 2023-09-23T23:05:00.789 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total time run: 90.0201 2023-09-23T23:05:00.789 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total writes made: 112705 2023-09-23T23:05:00.790 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Write size: 65536 2023-09-23T23:05:00.790 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object size: 65536 2023-09-23T23:05:00.790 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Bandwidth (MB/sec): 78.2499 2023-09-23T23:05:00.790 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Bandwidth: 44.2399 2023-09-23T23:05:00.790 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max bandwidth (MB/sec): 227.375 2023-09-23T23:05:00.790 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min bandwidth (MB/sec): 0 2023-09-23T23:05:00.790 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average IOPS: 1251 2023-09-23T23:05:00.790 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev IOPS: 707.849 2023-09-23T23:05:00.790 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max IOPS: 3638 2023-09-23T23:05:00.790 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min IOPS: 0 2023-09-23T23:05:00.790 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average Latency(s): 0.0127751 2023-09-23T23:05:00.791 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Latency(s): 0.0356639 2023-09-23T23:05:00.791 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max latency(s): 2.45499 2023-09-23T23:05:00.791 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min latency(s): 0.00103361 2023-09-23T23:05:00.796 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T23:05:00.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:00 smithi138 ceph-mon[193606]: pgmap v211: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s; 8304/327186 objects degraded (2.538%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:05:00.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:00 smithi110 ceph-mon[212939]: pgmap v211: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s; 8304/327186 objects degraded (2.538%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:05:00.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:00 smithi110 ceph-mon[221102]: pgmap v211: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s; 8304/327186 objects degraded (2.538%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:05:01.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:01 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2235829052' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:01.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:01 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2235829052' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:01.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:01 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2235829052' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:02.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:02 smithi138 ceph-mon[193606]: pgmap v212: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 8013/334704 objects degraded (2.394%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T23:05:02.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:02 smithi110 ceph-mon[212939]: pgmap v212: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 8013/334704 objects degraded (2.394%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T23:05:02.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:02 smithi110 ceph-mon[221102]: pgmap v212: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 8013/334704 objects degraded (2.394%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T23:05:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:03 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 8013/334704 objects degraded (2.394%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:03 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1558282556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:03.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:03 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 8013/334704 objects degraded (2.394%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:03.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:03 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1558282556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:03.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:03 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 8013/334704 objects degraded (2.394%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:03.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:03 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1558282556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:04.510 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:05:04 smithi110 conmon[198813]: 2023-09-23T23:05:04.255+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:05:04.510 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:04 smithi110 ceph-mon[221102]: pgmap v213: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 38 MiB/s wr, 605 op/s; 8013/334704 objects degraded (2.394%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T23:05:04.510 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:04 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:04.515 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:04 smithi110 ceph-mon[212939]: pgmap v213: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 38 MiB/s wr, 605 op/s; 8013/334704 objects degraded (2.394%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T23:05:04.516 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:04 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:04.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:04 smithi138 ceph-mon[193606]: pgmap v213: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 38 MiB/s wr, 605 op/s; 8013/334704 objects degraded (2.394%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T23:05:04.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:04 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:05.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:05 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:05.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:05 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:05:05.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:05 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:05.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:05 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:05:05.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:05 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:05.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:05 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:05:06.802 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:06 smithi110 ceph-mon[212939]: pgmap v214: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 930 op/s; 7687/329961 objects degraded (2.330%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T23:05:06.802 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:06 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3694965025' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:06.803 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:05:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:05:06] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.43.0" 2023-09-23T23:05:06.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:06 smithi110 ceph-mon[221102]: pgmap v214: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 930 op/s; 7687/329961 objects degraded (2.330%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T23:05:06.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:06 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3694965025' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:06.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:06 smithi138 ceph-mon[193606]: pgmap v214: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 930 op/s; 7687/329961 objects degraded (2.330%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T23:05:06.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:06 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3694965025' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:08 smithi138 ceph-mon[193606]: pgmap v215: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 653 op/s; 7687/329961 objects degraded (2.330%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T23:05:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:08 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 7687/329961 objects degraded (2.330%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:08 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2242508090' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:08.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:08 smithi110 ceph-mon[212939]: pgmap v215: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 653 op/s; 7687/329961 objects degraded (2.330%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T23:05:08.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:08 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 7687/329961 objects degraded (2.330%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:08.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:08 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2242508090' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:08.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:08 smithi110 ceph-mon[221102]: pgmap v215: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 653 op/s; 7687/329961 objects degraded (2.330%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T23:05:08.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:08 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 7687/329961 objects degraded (2.330%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:08.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:08 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2242508090' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:09.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:09 smithi138 ceph-mon[193606]: pgmap v216: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 775 op/s; 7687/325551 objects degraded (2.361%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T23:05:09.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:09 smithi110 ceph-mon[212939]: pgmap v216: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 775 op/s; 7687/325551 objects degraded (2.361%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T23:05:09.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:09 smithi110 ceph-mon[221102]: pgmap v216: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 775 op/s; 7687/325551 objects degraded (2.361%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T23:05:12.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:11 smithi138 ceph-mon[193606]: failed to encode map e98 with expected crc 2023-09-23T23:05:12.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:11 smithi138 ceph-mon[193606]: osdmap e98: 8 total, 8 up, 8 in 2023-09-23T23:05:12.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:11 smithi138 ceph-mon[193606]: failed to encode map e98 with expected crc 2023-09-23T23:05:12.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:11 smithi138 ceph-mon[193606]: failed to encode map e98 with expected crc 2023-09-23T23:05:12.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:11 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4044855588' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:12.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:11 smithi138 ceph-mon[193606]: pgmap v218: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 7.3 MiB/s wr, 784 op/s; 7347/318129 objects degraded (2.309%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T23:05:12.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:11 smithi110 ceph-mon[212939]: failed to encode map e98 with expected crc 2023-09-23T23:05:12.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:11 smithi110 ceph-mon[212939]: osdmap e98: 8 total, 8 up, 8 in 2023-09-23T23:05:12.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:11 smithi110 ceph-mon[212939]: failed to encode map e98 with expected crc 2023-09-23T23:05:12.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:11 smithi110 ceph-mon[212939]: failed to encode map e98 with expected crc 2023-09-23T23:05:12.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:11 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4044855588' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:12.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:11 smithi110 ceph-mon[212939]: pgmap v218: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 7.3 MiB/s wr, 784 op/s; 7347/318129 objects degraded (2.309%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T23:05:12.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:11 smithi110 ceph-mon[221102]: failed to encode map e98 with expected crc 2023-09-23T23:05:12.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:11 smithi110 ceph-mon[221102]: osdmap e98: 8 total, 8 up, 8 in 2023-09-23T23:05:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:11 smithi110 ceph-mon[221102]: failed to encode map e98 with expected crc 2023-09-23T23:05:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:11 smithi110 ceph-mon[221102]: failed to encode map e98 with expected crc 2023-09-23T23:05:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:11 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4044855588' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:11 smithi110 ceph-mon[221102]: pgmap v218: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 7.3 MiB/s wr, 784 op/s; 7347/318129 objects degraded (2.309%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T23:05:13.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:12 smithi138 ceph-mon[193606]: failed to encode map e99 with expected crc 2023-09-23T23:05:13.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:12 smithi138 ceph-mon[193606]: failed to encode map e99 with expected crc 2023-09-23T23:05:13.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:12 smithi138 ceph-mon[193606]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T23:05:13.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:12 smithi138 ceph-mon[193606]: failed to encode map e99 with expected crc 2023-09-23T23:05:13.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:12 smithi110 ceph-mon[221102]: failed to encode map e99 with expected crc 2023-09-23T23:05:13.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:12 smithi110 ceph-mon[221102]: failed to encode map e99 with expected crc 2023-09-23T23:05:13.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:12 smithi110 ceph-mon[221102]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T23:05:13.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:12 smithi110 ceph-mon[221102]: failed to encode map e99 with expected crc 2023-09-23T23:05:13.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:12 smithi110 ceph-mon[212939]: failed to encode map e99 with expected crc 2023-09-23T23:05:13.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:12 smithi110 ceph-mon[212939]: failed to encode map e99 with expected crc 2023-09-23T23:05:13.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:12 smithi110 ceph-mon[212939]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T23:05:13.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:12 smithi110 ceph-mon[212939]: failed to encode map e99 with expected crc 2023-09-23T23:05:14.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:13 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 7347/318129 objects degraded (2.309%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:14.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:14.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:13 smithi138 ceph-mon[193606]: pgmap v220: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 7347/318129 objects degraded (2.309%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:05:14.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:13 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1673672068' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:14.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:13 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 7347/318129 objects degraded (2.309%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:14.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:14.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:13 smithi110 ceph-mon[221102]: pgmap v220: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 7347/318129 objects degraded (2.309%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:05:14.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:13 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1673672068' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:14.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:13 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 7347/318129 objects degraded (2.309%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:14.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:14.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:13 smithi110 ceph-mon[212939]: pgmap v220: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 7347/318129 objects degraded (2.309%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:05:14.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:13 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1673672068' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:16.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:16 smithi138 ceph-mon[193606]: pgmap v221: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 6941/308589 objects degraded (2.249%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T23:05:16.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:16 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3062679369' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:16.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:16 smithi110 ceph-mon[221102]: pgmap v221: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 6941/308589 objects degraded (2.249%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T23:05:16.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:16 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3062679369' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:16.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:16 smithi110 ceph-mon[212939]: pgmap v221: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 6941/308589 objects degraded (2.249%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T23:05:16.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:16 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3062679369' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:17.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:05:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:05:16] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.43.0" 2023-09-23T23:05:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:18 smithi138 ceph-mon[193606]: pgmap v222: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 6941/308589 objects degraded (2.249%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T23:05:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:18 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 6941/308589 objects degraded (2.249%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:18.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:18 smithi110 ceph-mon[212939]: pgmap v222: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 6941/308589 objects degraded (2.249%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T23:05:18.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:18 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 6941/308589 objects degraded (2.249%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:18 smithi110 ceph-mon[221102]: pgmap v222: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 6941/308589 objects degraded (2.249%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T23:05:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:18 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 6941/308589 objects degraded (2.249%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:19.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:19 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2157017256' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:19.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:19 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2157017256' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:19.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:19 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2157017256' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:20.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:20 smithi138 ceph-mon[193606]: pgmap v223: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 6941/304272 objects degraded (2.281%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T23:05:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:20 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:05:20.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:20 smithi110 ceph-mon[212939]: pgmap v223: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 6941/304272 objects degraded (2.281%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T23:05:20.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:20 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:05:20.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:20 smithi110 ceph-mon[221102]: pgmap v223: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 6941/304272 objects degraded (2.281%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T23:05:20.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:20 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:05:22.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:22 smithi138 ceph-mon[193606]: pgmap v224: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 6705/297210 objects degraded (2.256%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T23:05:22.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:22 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2184198876' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:22.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:22 smithi110 ceph-mon[221102]: pgmap v224: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 6705/297210 objects degraded (2.256%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T23:05:22.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:22 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2184198876' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:22.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:22 smithi110 ceph-mon[212939]: pgmap v224: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 6705/297210 objects degraded (2.256%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T23:05:22.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:22 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2184198876' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:23.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:23 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 6705/297210 objects degraded (2.256%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:23.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:23 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 6705/297210 objects degraded (2.256%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:23.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:23 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 6705/297210 objects degraded (2.256%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:24.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:24 smithi138 ceph-mon[193606]: pgmap v225: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 6705/297210 objects degraded (2.256%); 3.6 MiB/s, 56 objects/s recovering 2023-09-23T23:05:24.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:24 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3792607202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:24.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:24 smithi110 ceph-mon[212939]: pgmap v225: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 6705/297210 objects degraded (2.256%); 3.6 MiB/s, 56 objects/s recovering 2023-09-23T23:05:24.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:24 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3792607202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:24.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:24 smithi110 ceph-mon[221102]: pgmap v225: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 6705/297210 objects degraded (2.256%); 3.6 MiB/s, 56 objects/s recovering 2023-09-23T23:05:24.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:24 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3792607202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:25.814 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:05:25 smithi110 conmon[198813]: 2023-09-23T23:05:25.809+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:05:25.815 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:25 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:05:25.815 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:25 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:05:25.815 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:25 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:05:25.815 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:25 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:05:25.818 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:25 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:05:25.818 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:25 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:05:25.818 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:25 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:05:25.819 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:25 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:05:25.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:25 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:05:25.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:25 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:05:25.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:25 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:05:25.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:25 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:05:26.800 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[212939]: pgmap v226: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 6455/286932 objects degraded (2.250%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T23:05:26.800 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:05:26.800 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:05:26.800 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:05:26.801 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:05:26.801 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:05:26.801 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:05:26.801 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:05:26.801 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:26.801 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1035312774' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:26.801 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[221102]: pgmap v226: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 6455/286932 objects degraded (2.250%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T23:05:26.801 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:05:26.801 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:05:26.801 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:05:26.801 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:05:26.802 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:05:26.802 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:05:26.802 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:05:26.802 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:26.802 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:26 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1035312774' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:26.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:26 smithi138 ceph-mon[193606]: pgmap v226: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 6455/286932 objects degraded (2.250%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T23:05:26.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:05:26.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:05:26.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:26 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:05:26.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:05:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:05:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:05:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:05:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:26 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1035312774' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:27.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:05:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:05:26] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.43.0" 2023-09-23T23:05:27.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:27 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:27.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:27 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:05:27.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:27 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:27.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:27 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:05:27.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:27 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:27.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:27 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:05:28.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:28 smithi138 ceph-mon[193606]: pgmap v227: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 6455/286932 objects degraded (2.250%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:05:28.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:28 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 6455/286932 objects degraded (2.250%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:28.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:28.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:28 smithi110 ceph-mon[221102]: pgmap v227: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 6455/286932 objects degraded (2.250%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:05:28.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:28 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 6455/286932 objects degraded (2.250%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:28.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:28.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:28 smithi110 ceph-mon[212939]: pgmap v227: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 6455/286932 objects degraded (2.250%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:05:28.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:28 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 6455/286932 objects degraded (2.250%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:28.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:29.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:29 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2548776264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:29.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:29 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2548776264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:29.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:29 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2548776264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:30.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:30 smithi138 ceph-mon[193606]: pgmap v228: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 6455/282819 objects degraded (2.282%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:05:30.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:30 smithi110 ceph-mon[212939]: pgmap v228: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 6455/282819 objects degraded (2.282%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:05:30.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:30 smithi110 ceph-mon[221102]: pgmap v228: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 6455/282819 objects degraded (2.282%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:05:31.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:31 smithi138 ceph-mon[193606]: pgmap v229: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 6185/275943 objects degraded (2.241%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T23:05:31.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:31 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/451149009' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:31.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:31 smithi110 ceph-mon[212939]: pgmap v229: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 6185/275943 objects degraded (2.241%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T23:05:31.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:31 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/451149009' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:31.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:31 smithi110 ceph-mon[221102]: pgmap v229: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 6185/275943 objects degraded (2.241%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T23:05:31.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:31 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/451149009' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:33.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:32 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 6185/275943 objects degraded (2.241%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:33.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:32 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 6185/275943 objects degraded (2.241%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:33.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:32 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 6185/275943 objects degraded (2.241%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:34.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:33 smithi138 ceph-mon[193606]: pgmap v230: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 6185/275943 objects degraded (2.241%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:05:34.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:33 smithi110 ceph-mon[221102]: pgmap v230: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 6185/275943 objects degraded (2.241%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:05:34.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:33 smithi110 ceph-mon[212939]: pgmap v230: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 6185/275943 objects degraded (2.241%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:05:35.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:35 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/946994807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:35 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/946994807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:35.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:34 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/946994807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:36.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:36 smithi138 ceph-mon[193606]: pgmap v231: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 5881/264780 objects degraded (2.221%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:05:36.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:36 smithi110 ceph-mon[212939]: pgmap v231: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 5881/264780 objects degraded (2.221%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:05:36.459 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:36 smithi110 ceph-mon[221102]: pgmap v231: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 5881/264780 objects degraded (2.221%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:05:37.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:05:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:05:36] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.43.0" 2023-09-23T23:05:37.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:37 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2047551458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:37.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:37 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2047551458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:37.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:37 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2047551458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:38.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:38 smithi138 ceph-mon[193606]: osdmap e100: 8 total, 8 up, 8 in 2023-09-23T23:05:38.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:38 smithi138 ceph-mon[193606]: failed to encode map e100 with expected crc 2023-09-23T23:05:38.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:38 smithi138 ceph-mon[193606]: failed to encode map e100 with expected crc 2023-09-23T23:05:38.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:38 smithi138 ceph-mon[193606]: failed to encode map e100 with expected crc 2023-09-23T23:05:38.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:38 smithi138 ceph-mon[193606]: failed to encode map e100 with expected crc 2023-09-23T23:05:38.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:38 smithi138 ceph-mon[193606]: failed to encode map e100 with expected crc 2023-09-23T23:05:38.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:38 smithi138 ceph-mon[193606]: pgmap v233: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 5881/264780 objects degraded (2.221%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:05:38.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:38 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 5881/264780 objects degraded (2.221%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:38.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:38 smithi110 ceph-mon[212939]: osdmap e100: 8 total, 8 up, 8 in 2023-09-23T23:05:38.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:38 smithi110 ceph-mon[212939]: failed to encode map e100 with expected crc 2023-09-23T23:05:38.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:38 smithi110 ceph-mon[212939]: failed to encode map e100 with expected crc 2023-09-23T23:05:38.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:38 smithi110 ceph-mon[212939]: failed to encode map e100 with expected crc 2023-09-23T23:05:38.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:38 smithi110 ceph-mon[212939]: failed to encode map e100 with expected crc 2023-09-23T23:05:38.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:38 smithi110 ceph-mon[212939]: failed to encode map e100 with expected crc 2023-09-23T23:05:38.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:38 smithi110 ceph-mon[212939]: pgmap v233: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 5881/264780 objects degraded (2.221%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:05:38.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:38 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 5881/264780 objects degraded (2.221%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:38.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:38 smithi110 ceph-mon[221102]: osdmap e100: 8 total, 8 up, 8 in 2023-09-23T23:05:38.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:38 smithi110 ceph-mon[221102]: failed to encode map e100 with expected crc 2023-09-23T23:05:38.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:38 smithi110 ceph-mon[221102]: failed to encode map e100 with expected crc 2023-09-23T23:05:38.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:38 smithi110 ceph-mon[221102]: failed to encode map e100 with expected crc 2023-09-23T23:05:38.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:38 smithi110 ceph-mon[221102]: failed to encode map e100 with expected crc 2023-09-23T23:05:38.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:38 smithi110 ceph-mon[221102]: failed to encode map e100 with expected crc 2023-09-23T23:05:38.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:38 smithi110 ceph-mon[221102]: pgmap v233: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 5881/264780 objects degraded (2.221%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:05:38.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:38 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 5881/264780 objects degraded (2.221%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:39.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:39 smithi138 ceph-mon[193606]: failed to encode map e101 with expected crc 2023-09-23T23:05:39.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:39 smithi138 ceph-mon[193606]: failed to encode map e101 with expected crc 2023-09-23T23:05:39.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:39 smithi138 ceph-mon[193606]: osdmap e101: 8 total, 8 up, 8 in 2023-09-23T23:05:39.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:39 smithi138 ceph-mon[193606]: failed to encode map e101 with expected crc 2023-09-23T23:05:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:39 smithi138 ceph-mon[193606]: failed to encode map e101 with expected crc 2023-09-23T23:05:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:39 smithi138 ceph-mon[193606]: failed to encode map e101 with expected crc 2023-09-23T23:05:39.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:39 smithi110 ceph-mon[212939]: failed to encode map e101 with expected crc 2023-09-23T23:05:39.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:39 smithi110 ceph-mon[212939]: failed to encode map e101 with expected crc 2023-09-23T23:05:39.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:39 smithi110 ceph-mon[212939]: osdmap e101: 8 total, 8 up, 8 in 2023-09-23T23:05:39.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:39 smithi110 ceph-mon[212939]: failed to encode map e101 with expected crc 2023-09-23T23:05:39.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:39 smithi110 ceph-mon[212939]: failed to encode map e101 with expected crc 2023-09-23T23:05:39.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:39 smithi110 ceph-mon[212939]: failed to encode map e101 with expected crc 2023-09-23T23:05:39.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:39 smithi110 ceph-mon[221102]: failed to encode map e101 with expected crc 2023-09-23T23:05:39.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:39 smithi110 ceph-mon[221102]: failed to encode map e101 with expected crc 2023-09-23T23:05:39.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:39 smithi110 ceph-mon[221102]: osdmap e101: 8 total, 8 up, 8 in 2023-09-23T23:05:39.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:39 smithi110 ceph-mon[221102]: failed to encode map e101 with expected crc 2023-09-23T23:05:39.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:39 smithi110 ceph-mon[221102]: failed to encode map e101 with expected crc 2023-09-23T23:05:39.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:39 smithi110 ceph-mon[221102]: failed to encode map e101 with expected crc 2023-09-23T23:05:40.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:40 smithi138 ceph-mon[193606]: pgmap v235: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 5609/261168 objects degraded (2.148%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:05:40.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:40 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1137222968' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:40.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:40 smithi110 ceph-mon[221102]: pgmap v235: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 5609/261168 objects degraded (2.148%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:05:40.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:40 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1137222968' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:40.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:40 smithi110 ceph-mon[212939]: pgmap v235: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 5609/261168 objects degraded (2.148%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:05:40.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:40 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1137222968' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:41.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:41 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:41.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:05:40 smithi110 conmon[198813]: 2023-09-23T23:05:40.845+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:05:41.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:41 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:41 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:42 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:42 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:05:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:42 smithi138 ceph-mon[193606]: pgmap v236: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 5506/255162 objects degraded (2.158%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T23:05:42.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:42 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2218845592' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:42.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:42 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:42.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:42 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:05:42.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:42 smithi110 ceph-mon[212939]: pgmap v236: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 5506/255162 objects degraded (2.158%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T23:05:42.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:42 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2218845592' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:42.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:42 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:42.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:42 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:05:42.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:42 smithi110 ceph-mon[221102]: pgmap v236: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 5506/255162 objects degraded (2.158%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T23:05:42.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:42 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2218845592' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:43.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:43 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 5506/255162 objects degraded (2.158%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:43.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:43.460 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:43 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 5506/255162 objects degraded (2.158%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:43.460 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:43.460 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:43 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 5506/255162 objects degraded (2.158%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:43.460 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:44.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:44 smithi138 ceph-mon[193606]: pgmap v237: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 5506/255162 objects degraded (2.158%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T23:05:44.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:44 smithi110 ceph-mon[221102]: pgmap v237: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 5506/255162 objects degraded (2.158%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T23:05:44.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:44 smithi110 ceph-mon[212939]: pgmap v237: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 5506/255162 objects degraded (2.158%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T23:05:45.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:45 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1471978665' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:45.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:45 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1471978665' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:45.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:45 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1471978665' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:46.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:46 smithi138 ceph-mon[193606]: pgmap v238: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 5225/244029 objects degraded (2.141%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T23:05:46.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:46 smithi110 ceph-mon[221102]: pgmap v238: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 5225/244029 objects degraded (2.141%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T23:05:46.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:46 smithi110 ceph-mon[212939]: pgmap v238: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 5225/244029 objects degraded (2.141%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T23:05:47.163 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:05:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:05:46] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.43.0" 2023-09-23T23:05:47.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:47 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/35754965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:47.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:47 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/35754965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:47.460 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:47 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/35754965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:48 smithi138 ceph-mon[193606]: pgmap v239: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 5225/244029 objects degraded (2.141%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:05:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:48 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 5225/244029 objects degraded (2.141%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:48.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:48 smithi110 ceph-mon[221102]: pgmap v239: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 5225/244029 objects degraded (2.141%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:05:48.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:48 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 5225/244029 objects degraded (2.141%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:48.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:48 smithi110 ceph-mon[212939]: pgmap v239: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 5225/244029 objects degraded (2.141%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:05:48.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:48 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 5225/244029 objects degraded (2.141%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:50.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:50 smithi138 ceph-mon[193606]: pgmap v240: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 4927/240072 objects degraded (2.052%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T23:05:50.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:50 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1933680426' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:50.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:50 smithi110 ceph-mon[221102]: pgmap v240: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 4927/240072 objects degraded (2.052%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T23:05:50.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:50 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1933680426' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:50.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:50 smithi110 ceph-mon[212939]: pgmap v240: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 4927/240072 objects degraded (2.052%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T23:05:50.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:50 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1933680426' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:52.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:52 smithi138 ceph-mon[193606]: pgmap v241: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 4927/233121 objects degraded (2.113%); 3.6 MiB/s, 56 objects/s recovering 2023-09-23T23:05:52.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:52 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3193306692' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:52.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:52 smithi110 ceph-mon[221102]: pgmap v241: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 4927/233121 objects degraded (2.113%); 3.6 MiB/s, 56 objects/s recovering 2023-09-23T23:05:52.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:52 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3193306692' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:52.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:52 smithi110 ceph-mon[212939]: pgmap v241: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 4927/233121 objects degraded (2.113%); 3.6 MiB/s, 56 objects/s recovering 2023-09-23T23:05:52.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:52 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3193306692' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:53.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:53 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 4927/233121 objects degraded (2.113%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:53.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:53 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 4927/233121 objects degraded (2.113%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:53.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:53 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 4927/233121 objects degraded (2.113%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:54.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:54 smithi138 ceph-mon[193606]: pgmap v242: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 4927/233121 objects degraded (2.113%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T23:05:54.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:54 smithi110 ceph-mon[221102]: pgmap v242: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 4927/233121 objects degraded (2.113%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T23:05:54.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:54 smithi110 ceph-mon[212939]: pgmap v242: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 4927/233121 objects degraded (2.113%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T23:05:55.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:55 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2867170565' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:55.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:55 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2867170565' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:55.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:55 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2867170565' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:56.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:05:55 smithi110 conmon[198813]: 2023-09-23T23:05:55.871+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:05:56.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:56 smithi138 ceph-mon[193606]: pgmap v243: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 4618/222225 objects degraded (2.078%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:05:56.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:56 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:56.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:56 smithi110 ceph-mon[221102]: pgmap v243: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 4618/222225 objects degraded (2.078%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:05:56.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:56 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:56.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:56 smithi110 ceph-mon[212939]: pgmap v243: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 4618/222225 objects degraded (2.078%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:05:56.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:56 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:57.206 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:05:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:05:56] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.43.0" 2023-09-23T23:05:57.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:57 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:57.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:57 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:05:57.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:57 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1108557525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:57.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:57 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:57.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:57 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:05:57.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:57 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1108557525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:57.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:57 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:05:57.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:57 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:05:57.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:57 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1108557525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:05:58.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:58 smithi138 ceph-mon[193606]: pgmap v244: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 4618/222225 objects degraded (2.078%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T23:05:58.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:58.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:58 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 4618/222225 objects degraded (2.078%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:58 smithi138 ceph-mon[193606]: osdmap e102: 8 total, 8 up, 8 in 2023-09-23T23:05:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:58 smithi138 ceph-mon[193606]: failed to encode map e102 with expected crc 2023-09-23T23:05:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:58 smithi138 ceph-mon[193606]: failed to encode map e102 with expected crc 2023-09-23T23:05:58.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:58 smithi110 ceph-mon[212939]: pgmap v244: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 4618/222225 objects degraded (2.078%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T23:05:58.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:58.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:58 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 4618/222225 objects degraded (2.078%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:58.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:58 smithi110 ceph-mon[212939]: osdmap e102: 8 total, 8 up, 8 in 2023-09-23T23:05:58.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:58 smithi110 ceph-mon[212939]: failed to encode map e102 with expected crc 2023-09-23T23:05:58.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:58 smithi110 ceph-mon[212939]: failed to encode map e102 with expected crc 2023-09-23T23:05:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:58 smithi110 ceph-mon[221102]: pgmap v244: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 4618/222225 objects degraded (2.078%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T23:05:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:58 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 4618/222225 objects degraded (2.078%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:05:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:58 smithi110 ceph-mon[221102]: osdmap e102: 8 total, 8 up, 8 in 2023-09-23T23:05:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:58 smithi110 ceph-mon[221102]: failed to encode map e102 with expected crc 2023-09-23T23:05:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:58 smithi110 ceph-mon[221102]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:59 smithi138 ceph-mon[193606]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:59 smithi138 ceph-mon[193606]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:59 smithi138 ceph-mon[193606]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:59 smithi138 ceph-mon[193606]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:59 smithi138 ceph-mon[193606]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:59 smithi138 ceph-mon[193606]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:05:59 smithi138 ceph-mon[193606]: osdmap e103: 8 total, 8 up, 8 in 2023-09-23T23:05:59.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:59 smithi110 ceph-mon[212939]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:59 smithi110 ceph-mon[212939]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:59 smithi110 ceph-mon[212939]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:59 smithi110 ceph-mon[212939]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:59 smithi110 ceph-mon[212939]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:59 smithi110 ceph-mon[212939]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:05:59 smithi110 ceph-mon[212939]: osdmap e103: 8 total, 8 up, 8 in 2023-09-23T23:05:59.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:59 smithi110 ceph-mon[221102]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:59 smithi110 ceph-mon[221102]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:59 smithi110 ceph-mon[221102]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:59 smithi110 ceph-mon[221102]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:59 smithi110 ceph-mon[221102]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:59 smithi110 ceph-mon[221102]: failed to encode map e102 with expected crc 2023-09-23T23:05:59.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:05:59 smithi110 ceph-mon[221102]: osdmap e103: 8 total, 8 up, 8 in 2023-09-23T23:06:00.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:00 smithi138 ceph-mon[193606]: failed to encode map e103 with expected crc 2023-09-23T23:06:00.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:00 smithi138 ceph-mon[193606]: failed to encode map e103 with expected crc 2023-09-23T23:06:00.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:00 smithi138 ceph-mon[193606]: failed to encode map e103 with expected crc 2023-09-23T23:06:00.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:00 smithi138 ceph-mon[193606]: failed to encode map e103 with expected crc 2023-09-23T23:06:00.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:00 smithi138 ceph-mon[193606]: pgmap v247: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 4390/218826 objects degraded (2.006%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T23:06:00.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:00 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/224965407' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:00.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:00 smithi110 ceph-mon[212939]: failed to encode map e103 with expected crc 2023-09-23T23:06:00.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:00 smithi110 ceph-mon[212939]: failed to encode map e103 with expected crc 2023-09-23T23:06:00.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:00 smithi110 ceph-mon[212939]: failed to encode map e103 with expected crc 2023-09-23T23:06:00.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:00 smithi110 ceph-mon[212939]: failed to encode map e103 with expected crc 2023-09-23T23:06:00.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:00 smithi110 ceph-mon[212939]: pgmap v247: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 4390/218826 objects degraded (2.006%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T23:06:00.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:00 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/224965407' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:00.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:00 smithi110 ceph-mon[221102]: failed to encode map e103 with expected crc 2023-09-23T23:06:00.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:00 smithi110 ceph-mon[221102]: failed to encode map e103 with expected crc 2023-09-23T23:06:00.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:00 smithi110 ceph-mon[221102]: failed to encode map e103 with expected crc 2023-09-23T23:06:00.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:00 smithi110 ceph-mon[221102]: failed to encode map e103 with expected crc 2023-09-23T23:06:00.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:00 smithi110 ceph-mon[221102]: pgmap v247: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 4390/218826 objects degraded (2.006%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T23:06:00.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:00 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/224965407' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:02.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:02 smithi138 ceph-mon[193606]: pgmap v248: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 4215/212970 objects degraded (1.979%); 5.6 MiB/s, 88 objects/s recovering 2023-09-23T23:06:02.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:02 smithi110 ceph-mon[212939]: pgmap v248: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 4215/212970 objects degraded (1.979%); 5.6 MiB/s, 88 objects/s recovering 2023-09-23T23:06:02.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:02 smithi110 ceph-mon[221102]: pgmap v248: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 4215/212970 objects degraded (1.979%); 5.6 MiB/s, 88 objects/s recovering 2023-09-23T23:06:03.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:03 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3144488412' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:03.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:03 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 4215/212970 objects degraded (1.979%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:03.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:03 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3144488412' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:03.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:03 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 4215/212970 objects degraded (1.979%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:03.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:03 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3144488412' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:03.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:03 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 4215/212970 objects degraded (1.979%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:04.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:04 smithi138 ceph-mon[193606]: pgmap v249: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 385 op/s; 4215/212970 objects degraded (1.979%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T23:06:04.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:04 smithi110 ceph-mon[212939]: pgmap v249: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 385 op/s; 4215/212970 objects degraded (1.979%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T23:06:04.711 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:04 smithi110 ceph-mon[221102]: pgmap v249: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 385 op/s; 4215/212970 objects degraded (1.979%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T23:06:05.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:05 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/957642736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:05.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:05 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/957642736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:05.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:05 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/957642736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:06 smithi138 ceph-mon[193606]: pgmap v250: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 3974/202323 objects degraded (1.964%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T23:06:06.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:06 smithi110 ceph-mon[221102]: pgmap v250: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 3974/202323 objects degraded (1.964%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T23:06:06.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:06 smithi110 ceph-mon[212939]: pgmap v250: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 3974/202323 objects degraded (1.964%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T23:06:07.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:06:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:06:06] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.43.0" 2023-09-23T23:06:08.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:08 smithi138 ceph-mon[193606]: pgmap v251: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3974/202323 objects degraded (1.964%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T23:06:08.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:08 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2282799939' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:08.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:08 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 3974/202323 objects degraded (1.964%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:08.709 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:08 smithi110 ceph-mon[212939]: pgmap v251: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3974/202323 objects degraded (1.964%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T23:06:08.709 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:08 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2282799939' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:08.710 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:08 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 3974/202323 objects degraded (1.964%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:08.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:08 smithi110 ceph-mon[221102]: pgmap v251: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3974/202323 objects degraded (1.964%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T23:06:08.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:08 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2282799939' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:08.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:08 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 3974/202323 objects degraded (1.964%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:10.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:10 smithi138 ceph-mon[193606]: pgmap v252: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 3974/198219 objects degraded (2.005%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T23:06:10.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:10 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/330305699' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:10.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:10 smithi110 ceph-mon[212939]: pgmap v252: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 3974/198219 objects degraded (2.005%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T23:06:10.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:10 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/330305699' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:10.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:10 smithi110 ceph-mon[221102]: pgmap v252: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 3974/198219 objects degraded (2.005%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T23:06:10.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:10 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/330305699' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:11.204 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:06:10 smithi110 conmon[198813]: 2023-09-23T23:06:10.898+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:06:11.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:11 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:11.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:11 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:11.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:11 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:12.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:12 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:12.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:12 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:06:12.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:12 smithi138 ceph-mon[193606]: pgmap v253: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 3705/191697 objects degraded (1.933%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:06:12.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:12 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:12.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:12 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:06:12.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:12 smithi110 ceph-mon[212939]: pgmap v253: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 3705/191697 objects degraded (1.933%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:06:12.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:12 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:12.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:12 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T23:06:12.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:12 smithi110 ceph-mon[221102]: pgmap v253: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 3705/191697 objects degraded (1.933%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:06:13.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:13 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3938833276' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:13.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:13.708 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:13 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 3705/191697 objects degraded (1.933%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:13.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:13 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3938833276' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:13.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:13.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:13 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 3705/191697 objects degraded (1.933%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:13 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3938833276' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:13 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 3705/191697 objects degraded (1.933%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:14.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:14 smithi138 ceph-mon[193606]: pgmap v254: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3705/191697 objects degraded (1.933%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:06:14.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:14 smithi110 ceph-mon[212939]: pgmap v254: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3705/191697 objects degraded (1.933%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:06:14.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:14 smithi110 ceph-mon[221102]: pgmap v254: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3705/191697 objects degraded (1.933%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:06:15.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:15 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/299841716' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:15.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:15 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/299841716' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:15.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:15 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/299841716' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:16.800 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:16 smithi110 ceph-mon[212939]: pgmap v255: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 3437/181047 objects degraded (1.898%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T23:06:16.801 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:16 smithi110 ceph-mon[221102]: pgmap v255: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 3437/181047 objects degraded (1.898%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T23:06:16.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:16 smithi138 ceph-mon[193606]: pgmap v255: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 3437/181047 objects degraded (1.898%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T23:06:17.209 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:06:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:06:16] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.43.0" 2023-09-23T23:06:18.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:18 smithi138 ceph-mon[193606]: pgmap v256: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3437/181047 objects degraded (1.898%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:06:18.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3409017997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:18.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:18 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 3437/181047 objects degraded (1.898%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:18.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:18 smithi110 ceph-mon[221102]: pgmap v256: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3437/181047 objects degraded (1.898%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:06:18.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3409017997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:18.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:18 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 3437/181047 objects degraded (1.898%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:18.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:18 smithi110 ceph-mon[212939]: pgmap v256: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3437/181047 objects degraded (1.898%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:06:18.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3409017997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:18.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:18 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 3437/181047 objects degraded (1.898%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:20.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:20 smithi138 ceph-mon[193606]: pgmap v257: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 3437/177033 objects degraded (1.941%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:06:20.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:20 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2195065937' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:20.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:20 smithi110 ceph-mon[221102]: pgmap v257: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 3437/177033 objects degraded (1.941%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:06:20.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:20 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2195065937' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:20.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:20 smithi110 ceph-mon[212939]: pgmap v257: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 3437/177033 objects degraded (1.941%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:06:20.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:20 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2195065937' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:22.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:22 smithi138 ceph-mon[193606]: pgmap v258: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 3147/170535 objects degraded (1.845%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:06:22.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:22 smithi110 ceph-mon[212939]: pgmap v258: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 3147/170535 objects degraded (1.845%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:06:22.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:22 smithi110 ceph-mon[221102]: pgmap v258: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 3147/170535 objects degraded (1.845%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:06:23.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:23 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/139733289' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:23.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:23 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 3147/170535 objects degraded (1.845%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:23.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:23 smithi138 ceph-mon[193606]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T23:06:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:23 smithi138 ceph-mon[193606]: failed to encode map e104 with expected crc 2023-09-23T23:06:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:23 smithi138 ceph-mon[193606]: failed to encode map e104 with expected crc 2023-09-23T23:06:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:23 smithi138 ceph-mon[193606]: failed to encode map e104 with expected crc 2023-09-23T23:06:23.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:23 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/139733289' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:23.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:23 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 3147/170535 objects degraded (1.845%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:23.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:23 smithi110 ceph-mon[221102]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T23:06:23.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:23 smithi110 ceph-mon[221102]: failed to encode map e104 with expected crc 2023-09-23T23:06:23.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:23 smithi110 ceph-mon[221102]: failed to encode map e104 with expected crc 2023-09-23T23:06:23.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:23 smithi110 ceph-mon[221102]: failed to encode map e104 with expected crc 2023-09-23T23:06:23.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:23 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/139733289' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:23.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:23 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 3147/170535 objects degraded (1.845%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:23.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:23 smithi110 ceph-mon[212939]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T23:06:23.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:23 smithi110 ceph-mon[212939]: failed to encode map e104 with expected crc 2023-09-23T23:06:23.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:23 smithi110 ceph-mon[212939]: failed to encode map e104 with expected crc 2023-09-23T23:06:23.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:23 smithi110 ceph-mon[212939]: failed to encode map e104 with expected crc 2023-09-23T23:06:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:24 smithi138 ceph-mon[193606]: failed to encode map e104 with expected crc 2023-09-23T23:06:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:24 smithi138 ceph-mon[193606]: failed to encode map e104 with expected crc 2023-09-23T23:06:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:24 smithi138 ceph-mon[193606]: failed to encode map e104 with expected crc 2023-09-23T23:06:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:24 smithi138 ceph-mon[193606]: failed to encode map e104 with expected crc 2023-09-23T23:06:24.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:24 smithi138 ceph-mon[193606]: pgmap v260: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 3147/170535 objects degraded (1.845%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T23:06:24.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:24 smithi138 ceph-mon[193606]: failed to encode map e105 with expected crc 2023-09-23T23:06:24.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:24 smithi138 ceph-mon[193606]: osdmap e105: 8 total, 8 up, 8 in 2023-09-23T23:06:24.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:24 smithi138 ceph-mon[193606]: failed to encode map e105 with expected crc 2023-09-23T23:06:24.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:24 smithi138 ceph-mon[193606]: failed to encode map e105 with expected crc 2023-09-23T23:06:24.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:24 smithi138 ceph-mon[193606]: failed to encode map e105 with expected crc 2023-09-23T23:06:24.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[221102]: failed to encode map e104 with expected crc 2023-09-23T23:06:24.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[221102]: failed to encode map e104 with expected crc 2023-09-23T23:06:24.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[221102]: failed to encode map e104 with expected crc 2023-09-23T23:06:24.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[221102]: failed to encode map e104 with expected crc 2023-09-23T23:06:24.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[221102]: pgmap v260: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 3147/170535 objects degraded (1.845%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T23:06:24.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[221102]: failed to encode map e105 with expected crc 2023-09-23T23:06:24.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[221102]: osdmap e105: 8 total, 8 up, 8 in 2023-09-23T23:06:24.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[221102]: failed to encode map e105 with expected crc 2023-09-23T23:06:24.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[221102]: failed to encode map e105 with expected crc 2023-09-23T23:06:24.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[221102]: failed to encode map e105 with expected crc 2023-09-23T23:06:24.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[212939]: failed to encode map e104 with expected crc 2023-09-23T23:06:24.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[212939]: failed to encode map e104 with expected crc 2023-09-23T23:06:24.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[212939]: failed to encode map e104 with expected crc 2023-09-23T23:06:24.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[212939]: failed to encode map e104 with expected crc 2023-09-23T23:06:24.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[212939]: pgmap v260: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 3147/170535 objects degraded (1.845%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T23:06:24.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[212939]: failed to encode map e105 with expected crc 2023-09-23T23:06:24.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[212939]: osdmap e105: 8 total, 8 up, 8 in 2023-09-23T23:06:24.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[212939]: failed to encode map e105 with expected crc 2023-09-23T23:06:24.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[212939]: failed to encode map e105 with expected crc 2023-09-23T23:06:24.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:24 smithi110 ceph-mon[212939]: failed to encode map e105 with expected crc 2023-09-23T23:06:25.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:25 smithi138 ceph-mon[193606]: failed to encode map e105 with expected crc 2023-09-23T23:06:25.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:25 smithi138 ceph-mon[193606]: failed to encode map e105 with expected crc 2023-09-23T23:06:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:25 smithi138 ceph-mon[193606]: failed to encode map e105 with expected crc 2023-09-23T23:06:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:25 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3855997408' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:25.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:25 smithi110 ceph-mon[221102]: failed to encode map e105 with expected crc 2023-09-23T23:06:25.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:25 smithi110 ceph-mon[221102]: failed to encode map e105 with expected crc 2023-09-23T23:06:25.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:25 smithi110 ceph-mon[221102]: failed to encode map e105 with expected crc 2023-09-23T23:06:25.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:25 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3855997408' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:25.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:25 smithi110 ceph-mon[212939]: failed to encode map e105 with expected crc 2023-09-23T23:06:25.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:25 smithi110 ceph-mon[212939]: failed to encode map e105 with expected crc 2023-09-23T23:06:25.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:25 smithi110 ceph-mon[212939]: failed to encode map e105 with expected crc 2023-09-23T23:06:25.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:25 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3855997408' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:26.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:26 smithi138 ceph-mon[193606]: pgmap v262: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 2755/161454 objects degraded (1.706%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:06:26.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:06:26.953 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:06:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:06:26] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.43.0" 2023-09-23T23:06:26.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:26 smithi110 ceph-mon[212939]: pgmap v262: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 2755/161454 objects degraded (1.706%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:06:26.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:06:26.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:26 smithi110 ceph-mon[221102]: pgmap v262: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 2755/161454 objects degraded (1.706%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:06:26.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:06:28.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:28 smithi138 ceph-mon[193606]: pgmap v263: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 2755/161454 objects degraded (1.706%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:06:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:28 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2755/161454 objects degraded (1.706%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:28 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4242018601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:28.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:28 smithi110 ceph-mon[212939]: pgmap v263: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 2755/161454 objects degraded (1.706%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:06:28.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:28.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:28 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2755/161454 objects degraded (1.706%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:28.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:28 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4242018601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:28.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:28 smithi110 ceph-mon[221102]: pgmap v263: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 2755/161454 objects degraded (1.706%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:06:28.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:28.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:28 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2755/161454 objects degraded (1.706%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:28.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:28 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4242018601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:30.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:30 smithi138 ceph-mon[193606]: pgmap v264: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 2509/157509 objects degraded (1.593%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T23:06:30.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:30 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:06:30.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:30 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:06:30.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:30 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4240419139' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:30.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:30 smithi110 ceph-mon[212939]: pgmap v264: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 2509/157509 objects degraded (1.593%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T23:06:30.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:30 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:06:30.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:30 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:06:30.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:30 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4240419139' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:30.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:30 smithi110 ceph-mon[221102]: pgmap v264: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 2509/157509 objects degraded (1.593%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T23:06:30.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:30 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:06:30.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:30 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:06:30.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:30 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4240419139' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:32.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:32 smithi138 ceph-mon[193606]: pgmap v265: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 2509/151365 objects degraded (1.658%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T23:06:32.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:32 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:06:32.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:32 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:06:32.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:32 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:06:32.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:32 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:06:32.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:32 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:06:32.703 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:06:32 smithi110 conmon[198813]: 2023-09-23T23:06:32.307+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:06:32.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:32 smithi110 ceph-mon[221102]: pgmap v265: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 2509/151365 objects degraded (1.658%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T23:06:32.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:32 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:06:32.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:32 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:06:32.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:32 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:06:32.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:32 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:06:32.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:32 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:06:32.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:32 smithi110 ceph-mon[212939]: pgmap v265: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 2509/151365 objects degraded (1.658%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T23:06:32.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:32 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:06:32.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:32 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:06:32.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:32 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:06:32.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:32 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:06:32.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:32 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:06:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:33 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:06:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:33 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:06:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:33 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:06:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:33 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:06:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:33 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:33 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:33 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:06:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:33 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2509/151365 objects degraded (1.658%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:33 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2914838772' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:33.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:06:33.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:06:33.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:06:33.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:06:33.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:33.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:33.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:06:33.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2509/151365 objects degraded (1.658%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:33.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2914838772' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:33.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:06:33.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:06:33.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:06:33.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:06:33.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:33.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:33.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:06:33.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2509/151365 objects degraded (1.658%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:33.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:33 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2914838772' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:34.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:34 smithi138 ceph-mon[193606]: pgmap v266: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 2509/151365 objects degraded (1.658%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T23:06:34.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:34 smithi110 ceph-mon[212939]: pgmap v266: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 2509/151365 objects degraded (1.658%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T23:06:34.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:34 smithi110 ceph-mon[221102]: pgmap v266: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 2509/151365 objects degraded (1.658%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T23:06:36.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:36 smithi138 ceph-mon[193606]: pgmap v267: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 2237/141087 objects degraded (1.586%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T23:06:36.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:36 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3428531227' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:36.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:36 smithi110 ceph-mon[212939]: pgmap v267: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 2237/141087 objects degraded (1.586%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T23:06:36.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:36 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3428531227' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:36.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:36 smithi110 ceph-mon[221102]: pgmap v267: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 2237/141087 objects degraded (1.586%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T23:06:36.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:36 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3428531227' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:37.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:06:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:06:36] "GET /metrics HTTP/1.1" 200 36434 "" "Prometheus/2.43.0" 2023-09-23T23:06:38.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:38 smithi138 ceph-mon[193606]: pgmap v268: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 2237/141087 objects degraded (1.586%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:06:38.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:38 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2237/141087 objects degraded (1.586%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:38.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:38 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/583048874' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:38 smithi110 ceph-mon[212939]: pgmap v268: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 2237/141087 objects degraded (1.586%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:06:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:38 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2237/141087 objects degraded (1.586%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:38 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/583048874' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:38.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:38 smithi110 ceph-mon[221102]: pgmap v268: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 2237/141087 objects degraded (1.586%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:06:38.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:38 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2237/141087 objects degraded (1.586%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:38.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:38 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/583048874' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:40.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:40 smithi138 ceph-mon[193606]: pgmap v269: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 1962/137136 objects degraded (1.431%); 3.8 MiB/s, 66 objects/s recovering 2023-09-23T23:06:40.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:40 smithi110 ceph-mon[212939]: pgmap v269: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 1962/137136 objects degraded (1.431%); 3.8 MiB/s, 66 objects/s recovering 2023-09-23T23:06:40.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:40 smithi110 ceph-mon[221102]: pgmap v269: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 1962/137136 objects degraded (1.431%); 3.8 MiB/s, 66 objects/s recovering 2023-09-23T23:06:41.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:41 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1050908941' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:41.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:41 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1050908941' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:41.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:41 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1050908941' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:42.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:42 smithi138 ceph-mon[193606]: pgmap v270: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 1962/130338 objects degraded (1.505%); 2.5 MiB/s, 45 objects/s recovering 2023-09-23T23:06:42.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:42 smithi110 ceph-mon[212939]: pgmap v270: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 1962/130338 objects degraded (1.505%); 2.5 MiB/s, 45 objects/s recovering 2023-09-23T23:06:42.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:42 smithi110 ceph-mon[221102]: pgmap v270: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 1962/130338 objects degraded (1.505%); 2.5 MiB/s, 45 objects/s recovering 2023-09-23T23:06:43.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:43.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:43 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1962/130338 objects degraded (1.505%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:43.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:43.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:43 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1962/130338 objects degraded (1.505%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:43.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:43.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:43 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1962/130338 objects degraded (1.505%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:44 smithi138 ceph-mon[193606]: pgmap v271: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 1962/130338 objects degraded (1.505%); 2.5 MiB/s, 45 objects/s recovering 2023-09-23T23:06:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:44 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/216578800' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:44.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:44 smithi110 ceph-mon[212939]: pgmap v271: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 1962/130338 objects degraded (1.505%); 2.5 MiB/s, 45 objects/s recovering 2023-09-23T23:06:44.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:44 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/216578800' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:44.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:44 smithi110 ceph-mon[221102]: pgmap v271: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 1962/130338 objects degraded (1.505%); 2.5 MiB/s, 45 objects/s recovering 2023-09-23T23:06:44.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:44 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/216578800' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:46.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:46 smithi138 ceph-mon[193606]: pgmap v272: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 1694/119643 objects degraded (1.416%); 2.5 MiB/s, 67 objects/s recovering 2023-09-23T23:06:46.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:46 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/16592438' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:46.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:46 smithi110 ceph-mon[212939]: pgmap v272: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 1694/119643 objects degraded (1.416%); 2.5 MiB/s, 67 objects/s recovering 2023-09-23T23:06:46.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:46 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/16592438' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:46.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:46 smithi110 ceph-mon[221102]: pgmap v272: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 1694/119643 objects degraded (1.416%); 2.5 MiB/s, 67 objects/s recovering 2023-09-23T23:06:46.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:46 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/16592438' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:47.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:06:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:06:46] "GET /metrics HTTP/1.1" 200 36434 "" "Prometheus/2.43.0" 2023-09-23T23:06:47.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:47 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:47.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:47 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:47.704 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:06:47 smithi110 conmon[198813]: 2023-09-23T23:06:47.336+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:06:47.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:47 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:48.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:48 smithi138 ceph-mon[193606]: pgmap v273: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 1694/119643 objects degraded (1.416%); 1.1 MiB/s, 45 objects/s recovering 2023-09-23T23:06:48.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:48 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:48.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:48 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:06:48.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:48 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1694/119643 objects degraded (1.416%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:48.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:48 smithi110 ceph-mon[212939]: pgmap v273: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 1694/119643 objects degraded (1.416%); 1.1 MiB/s, 45 objects/s recovering 2023-09-23T23:06:48.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:48 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:48.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:48 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:06:48.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:48 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1694/119643 objects degraded (1.416%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:48.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:48 smithi110 ceph-mon[221102]: pgmap v273: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 1694/119643 objects degraded (1.416%); 1.1 MiB/s, 45 objects/s recovering 2023-09-23T23:06:48.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:48 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:06:48.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:48 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:06:48.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:48 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1694/119643 objects degraded (1.416%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:49.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:49 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2102457592' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:49.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:49 smithi138 ceph-mon[193606]: osdmap e106: 8 total, 8 up, 8 in 2023-09-23T23:06:49.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:49 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2102457592' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:49.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:49 smithi110 ceph-mon[212939]: osdmap e106: 8 total, 8 up, 8 in 2023-09-23T23:06:49.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:49 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2102457592' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:49.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:49 smithi110 ceph-mon[221102]: osdmap e106: 8 total, 8 up, 8 in 2023-09-23T23:06:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:50 smithi138 ceph-mon[193606]: failed to encode map e106 with expected crc 2023-09-23T23:06:50.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:50 smithi138 ceph-mon[193606]: failed to encode map e106 with expected crc 2023-09-23T23:06:50.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:50 smithi138 ceph-mon[193606]: failed to encode map e106 with expected crc 2023-09-23T23:06:50.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:50 smithi138 ceph-mon[193606]: pgmap v275: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 1444/115668 objects degraded (1.248%); 0 B/s, 51 objects/s recovering 2023-09-23T23:06:50.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:50 smithi138 ceph-mon[193606]: osdmap e107: 8 total, 8 up, 8 in 2023-09-23T23:06:50.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:50 smithi110 ceph-mon[221102]: failed to encode map e106 with expected crc 2023-09-23T23:06:50.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:50 smithi110 ceph-mon[221102]: failed to encode map e106 with expected crc 2023-09-23T23:06:50.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:50 smithi110 ceph-mon[221102]: failed to encode map e106 with expected crc 2023-09-23T23:06:50.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:50 smithi110 ceph-mon[221102]: pgmap v275: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 1444/115668 objects degraded (1.248%); 0 B/s, 51 objects/s recovering 2023-09-23T23:06:50.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:50 smithi110 ceph-mon[221102]: osdmap e107: 8 total, 8 up, 8 in 2023-09-23T23:06:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:50 smithi110 ceph-mon[212939]: failed to encode map e106 with expected crc 2023-09-23T23:06:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:50 smithi110 ceph-mon[212939]: failed to encode map e106 with expected crc 2023-09-23T23:06:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:50 smithi110 ceph-mon[212939]: failed to encode map e106 with expected crc 2023-09-23T23:06:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:50 smithi110 ceph-mon[212939]: pgmap v275: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 1444/115668 objects degraded (1.248%); 0 B/s, 51 objects/s recovering 2023-09-23T23:06:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:50 smithi110 ceph-mon[212939]: osdmap e107: 8 total, 8 up, 8 in 2023-09-23T23:06:51.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:51 smithi138 ceph-mon[193606]: failed to encode map e107 with expected crc 2023-09-23T23:06:51.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:51 smithi138 ceph-mon[193606]: failed to encode map e107 with expected crc 2023-09-23T23:06:51.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:51 smithi138 ceph-mon[193606]: failed to encode map e107 with expected crc 2023-09-23T23:06:51.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:51 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1667996052' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:51.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:51 smithi110 ceph-mon[221102]: failed to encode map e107 with expected crc 2023-09-23T23:06:51.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:51 smithi110 ceph-mon[221102]: failed to encode map e107 with expected crc 2023-09-23T23:06:51.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:51 smithi110 ceph-mon[221102]: failed to encode map e107 with expected crc 2023-09-23T23:06:51.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:51 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1667996052' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:51.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:51 smithi110 ceph-mon[212939]: failed to encode map e107 with expected crc 2023-09-23T23:06:51.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:51 smithi110 ceph-mon[212939]: failed to encode map e107 with expected crc 2023-09-23T23:06:51.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:51 smithi110 ceph-mon[212939]: failed to encode map e107 with expected crc 2023-09-23T23:06:51.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:51 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1667996052' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:52.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:52 smithi138 ceph-mon[193606]: pgmap v277: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 1343/109152 objects degraded (1.230%); 0 B/s, 77 objects/s recovering 2023-09-23T23:06:52.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:52 smithi110 ceph-mon[212939]: pgmap v277: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 1343/109152 objects degraded (1.230%); 0 B/s, 77 objects/s recovering 2023-09-23T23:06:52.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:52 smithi110 ceph-mon[221102]: pgmap v277: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 1343/109152 objects degraded (1.230%); 0 B/s, 77 objects/s recovering 2023-09-23T23:06:53.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:53 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1343/109152 objects degraded (1.230%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:53.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:53 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1343/109152 objects degraded (1.230%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:53.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:53 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1343/109152 objects degraded (1.230%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:54.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:54 smithi138 ceph-mon[193606]: pgmap v278: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 437 op/s; 1343/109152 objects degraded (1.230%); 0 B/s, 43 objects/s recovering 2023-09-23T23:06:54.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:54 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1300137374' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:54.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:54 smithi110 ceph-mon[221102]: pgmap v278: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 437 op/s; 1343/109152 objects degraded (1.230%); 0 B/s, 43 objects/s recovering 2023-09-23T23:06:54.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:54 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1300137374' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:54.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:54 smithi110 ceph-mon[212939]: pgmap v278: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 437 op/s; 1343/109152 objects degraded (1.230%); 0 B/s, 43 objects/s recovering 2023-09-23T23:06:54.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:54 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1300137374' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:56.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:56 smithi110 ceph-mon[212939]: pgmap v279: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 1013/99756 objects degraded (1.015%); 0 B/s, 85 objects/s recovering 2023-09-23T23:06:56.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:56 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4083413610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:56.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:56 smithi110 ceph-mon[221102]: pgmap v279: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 1013/99756 objects degraded (1.015%); 0 B/s, 85 objects/s recovering 2023-09-23T23:06:56.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:56 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4083413610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:56.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:56 smithi138 ceph-mon[193606]: pgmap v279: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 1013/99756 objects degraded (1.015%); 0 B/s, 85 objects/s recovering 2023-09-23T23:06:56.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:56 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4083413610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:57.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:06:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:06:56] "GET /metrics HTTP/1.1" 200 36434 "" "Prometheus/2.43.0" 2023-09-23T23:06:58.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:58 smithi110 ceph-mon[212939]: pgmap v280: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 1013/99756 objects degraded (1.015%); 0 B/s, 83 objects/s recovering 2023-09-23T23:06:58.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:58.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:58 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1013/99756 objects degraded (1.015%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:58 smithi110 ceph-mon[221102]: pgmap v280: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 1013/99756 objects degraded (1.015%); 0 B/s, 83 objects/s recovering 2023-09-23T23:06:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:58 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1013/99756 objects degraded (1.015%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:58.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:58 smithi138 ceph-mon[193606]: pgmap v280: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 1013/99756 objects degraded (1.015%); 0 B/s, 83 objects/s recovering 2023-09-23T23:06:58.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:58.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:58 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1013/99756 objects degraded (1.015%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:06:59 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3299569669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:59.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:06:59 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3299569669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:06:59.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:06:59 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3299569669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:00.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:00 smithi138 ceph-mon[193606]: pgmap v281: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 1013/95850 objects degraded (1.057%); 0 B/s, 43 objects/s recovering 2023-09-23T23:07:00.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:00 smithi110 ceph-mon[212939]: pgmap v281: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 1013/95850 objects degraded (1.057%); 0 B/s, 43 objects/s recovering 2023-09-23T23:07:00.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:00 smithi110 ceph-mon[221102]: pgmap v281: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 1013/95850 objects degraded (1.057%); 0 B/s, 43 objects/s recovering 2023-09-23T23:07:01.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:01 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3807649215' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:01.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:01 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3807649215' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:01.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:01 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3807649215' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:02.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:02 smithi138 ceph-mon[193606]: pgmap v282: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 785/89136 objects degraded (0.881%); 0 B/s, 50 objects/s recovering 2023-09-23T23:07:02.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:02 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:07:02.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:02 smithi110 ceph-mon[212939]: pgmap v282: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 785/89136 objects degraded (0.881%); 0 B/s, 50 objects/s recovering 2023-09-23T23:07:02.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:02 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:07:02.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:02 smithi110 ceph-mon[221102]: pgmap v282: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 785/89136 objects degraded (0.881%); 0 B/s, 50 objects/s recovering 2023-09-23T23:07:02.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:02 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:07:03.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:03 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:07:03.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:03 smithi110 ceph-mon[221102]: Upgrade: osd.1 is safe to restart 2023-09-23T23:07:03.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:03 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 785/89136 objects degraded (0.881%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:07:03.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:03 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:07:03.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:03 smithi110 ceph-mon[212939]: Upgrade: osd.1 is safe to restart 2023-09-23T23:07:03.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:03 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 785/89136 objects degraded (0.881%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:07:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:03 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T23:07:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:03 smithi138 ceph-mon[193606]: Upgrade: osd.1 is safe to restart 2023-09-23T23:07:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:03 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 785/89136 objects degraded (0.881%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:07:04.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:04 smithi138 ceph-mon[193606]: pgmap v283: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 785/89136 objects degraded (0.881%); 0 B/s, 46 objects/s recovering 2023-09-23T23:07:04.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:04 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3574324726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:04.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:04 smithi110 ceph-mon[212939]: pgmap v283: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 785/89136 objects degraded (0.881%); 0 B/s, 46 objects/s recovering 2023-09-23T23:07:04.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:04 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3574324726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:04.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:04 smithi110 ceph-mon[221102]: pgmap v283: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 785/89136 objects degraded (0.881%); 0 B/s, 46 objects/s recovering 2023-09-23T23:07:04.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:04 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3574324726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:06.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:06 smithi110 ceph-mon[212939]: pgmap v284: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 553/78774 objects degraded (0.702%); 0 B/s, 65 objects/s recovering 2023-09-23T23:07:06.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:06 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3071333959' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:06.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:06 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:06.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:06 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T23:07:06.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:06 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:07:06.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:06 smithi110 ceph-mon[221102]: pgmap v284: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 553/78774 objects degraded (0.702%); 0 B/s, 65 objects/s recovering 2023-09-23T23:07:06.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:06 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3071333959' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:06.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:06 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:06.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:06 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T23:07:06.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:06 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:07:06.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:06 smithi138 ceph-mon[193606]: pgmap v284: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 553/78774 objects degraded (0.702%); 0 B/s, 65 objects/s recovering 2023-09-23T23:07:06.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:06 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3071333959' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:06.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:06 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:06.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:06 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T23:07:06.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:06 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:07:07.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:07:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:07:06] "GET /metrics HTTP/1.1" 200 36416 "" "Prometheus/2.43.0" 2023-09-23T23:07:07.636 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:07 smithi110 ceph-mon[212939]: Upgrade: Updating osd.1 2023-09-23T23:07:07.636 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:07 smithi110 ceph-mon[212939]: Deploying daemon osd.1 on smithi110 2023-09-23T23:07:07.637 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:07 smithi110 ceph-mon[221102]: Upgrade: Updating osd.1 2023-09-23T23:07:07.637 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:07 smithi110 ceph-mon[221102]: Deploying daemon osd.1 on smithi110 2023-09-23T23:07:07.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:07 smithi138 ceph-mon[193606]: Upgrade: Updating osd.1 2023-09-23T23:07:07.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:07 smithi138 ceph-mon[193606]: Deploying daemon osd.1 on smithi110 2023-09-23T23:07:08.441 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 23:07:08 smithi138 conmon[172649]: logger=cleanup t=2023-09-23T23:07:08.07937134Z level=info msg="Completed cleanup jobs" duration=142.081579ms 2023-09-23T23:07:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:08 smithi138 ceph-mon[193606]: pgmap v285: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 553/78774 objects degraded (0.702%); 0 B/s, 38 objects/s recovering 2023-09-23T23:07:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:08 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 553/78774 objects degraded (0.702%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:07:08.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:08 smithi110 ceph-mon[221102]: pgmap v285: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 553/78774 objects degraded (0.702%); 0 B/s, 38 objects/s recovering 2023-09-23T23:07:08.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:08 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 553/78774 objects degraded (0.702%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:07:08.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:08 smithi110 ceph-mon[212939]: pgmap v285: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 553/78774 objects degraded (0.702%); 0 B/s, 38 objects/s recovering 2023-09-23T23:07:08.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:08 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 553/78774 objects degraded (0.702%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:07:09.709 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:09 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3794555316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:09.710 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:09 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3794555316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:09.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:09 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3794555316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:10.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:10 smithi138 ceph-mon[193606]: pgmap v286: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 553/75024 objects degraded (0.737%); 0 B/s, 38 objects/s recovering 2023-09-23T23:07:10.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:10 smithi110 ceph-mon[212939]: pgmap v286: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 553/75024 objects degraded (0.737%); 0 B/s, 38 objects/s recovering 2023-09-23T23:07:10.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:10 smithi110 ceph-mon[221102]: pgmap v286: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 553/75024 objects degraded (0.737%); 0 B/s, 38 objects/s recovering 2023-09-23T23:07:11.702 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:11 smithi110 systemd[1]: Stopping Ceph osd.1 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:07:12.451 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:12 smithi110 conmon[134020]: 2023-09-23T23:07:12.121+0000 7f941cca5700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:07:12.451 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:12 smithi110 conmon[134020]: 2023-09-23T23:07:12.121+0000 7f941cca5700 -1 osd.1 107 *** Got signal Terminated *** 2023-09-23T23:07:12.451 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:12 smithi110 conmon[134020]: 2023-09-23T23:07:12.121+0000 7f941cca5700 -1 osd.1 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:07:12.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:12 smithi138 ceph-mon[193606]: pgmap v287: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 322/68886 objects degraded (0.467%); 0 B/s, 57 objects/s recovering 2023-09-23T23:07:12.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:12 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3934394910' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:12.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:12 smithi138 ceph-mon[193606]: osd.1 marked itself down and dead 2023-09-23T23:07:12.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:12 smithi110 ceph-mon[212939]: pgmap v287: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 322/68886 objects degraded (0.467%); 0 B/s, 57 objects/s recovering 2023-09-23T23:07:12.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:12 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3934394910' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:12.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:12 smithi110 ceph-mon[212939]: osd.1 marked itself down and dead 2023-09-23T23:07:12.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:12 smithi110 ceph-mon[221102]: pgmap v287: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 322/68886 objects degraded (0.467%); 0 B/s, 57 objects/s recovering 2023-09-23T23:07:12.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:12 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3934394910' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:12.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:12 smithi110 ceph-mon[221102]: osd.1 marked itself down and dead 2023-09-23T23:07:13.587 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:13 smithi110 bash[249297]: 9cdbba4d4209abbd23675037d4aef237cafac2dd883763f39c952dd64f4fcaa8 2023-09-23T23:07:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:13 smithi138 ceph-mon[193606]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:07:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:13 smithi138 ceph-mon[193606]: osdmap e108: 8 total, 7 up, 8 in 2023-09-23T23:07:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:13 smithi138 ceph-mon[193606]: failed to encode map e108 with expected crc 2023-09-23T23:07:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:13 smithi138 ceph-mon[193606]: failed to encode map e108 with expected crc 2023-09-23T23:07:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:13 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 322/68886 objects degraded (0.467%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:07:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:13 smithi138 ceph-mon[193606]: failed to encode map e109 with expected crc 2023-09-23T23:07:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:13 smithi138 ceph-mon[193606]: osdmap e109: 8 total, 7 up, 8 in 2023-09-23T23:07:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:13 smithi138 ceph-mon[193606]: failed to encode map e109 with expected crc 2023-09-23T23:07:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:13 smithi138 ceph-mon[193606]: failed to encode map e109 with expected crc 2023-09-23T23:07:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:13 smithi138 ceph-mon[193606]: failed to encode map e109 with expected crc 2023-09-23T23:07:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:13 smithi138 ceph-mon[193606]: failed to encode map e109 with expected crc 2023-09-23T23:07:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:13 smithi138 ceph-mon[193606]: failed to encode map e109 with expected crc 2023-09-23T23:07:13.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[212939]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:07:13.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[212939]: osdmap e108: 8 total, 7 up, 8 in 2023-09-23T23:07:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[212939]: failed to encode map e108 with expected crc 2023-09-23T23:07:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[212939]: failed to encode map e108 with expected crc 2023-09-23T23:07:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 322/68886 objects degraded (0.467%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:07:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[212939]: failed to encode map e109 with expected crc 2023-09-23T23:07:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[212939]: osdmap e109: 8 total, 7 up, 8 in 2023-09-23T23:07:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[212939]: failed to encode map e109 with expected crc 2023-09-23T23:07:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[212939]: failed to encode map e109 with expected crc 2023-09-23T23:07:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[212939]: failed to encode map e109 with expected crc 2023-09-23T23:07:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[212939]: failed to encode map e109 with expected crc 2023-09-23T23:07:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[212939]: failed to encode map e109 with expected crc 2023-09-23T23:07:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[221102]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:07:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[221102]: osdmap e108: 8 total, 7 up, 8 in 2023-09-23T23:07:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[221102]: failed to encode map e108 with expected crc 2023-09-23T23:07:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[221102]: failed to encode map e108 with expected crc 2023-09-23T23:07:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 322/68886 objects degraded (0.467%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:07:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[221102]: failed to encode map e109 with expected crc 2023-09-23T23:07:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[221102]: osdmap e109: 8 total, 7 up, 8 in 2023-09-23T23:07:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[221102]: failed to encode map e109 with expected crc 2023-09-23T23:07:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[221102]: failed to encode map e109 with expected crc 2023-09-23T23:07:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[221102]: failed to encode map e109 with expected crc 2023-09-23T23:07:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[221102]: failed to encode map e109 with expected crc 2023-09-23T23:07:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:13 smithi110 ceph-mon[221102]: failed to encode map e109 with expected crc 2023-09-23T23:07:14.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:14 smithi138 ceph-mon[193606]: failed to encode map e109 with expected crc 2023-09-23T23:07:14.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:14 smithi138 ceph-mon[193606]: pgmap v290: 89 pgs: 11 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 77 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 411 op/s; 322/68886 objects degraded (0.467%); 0 B/s, 28 objects/s recovering 2023-09-23T23:07:14.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:14 smithi138 ceph-mon[193606]: failed to encode map e110 with expected crc 2023-09-23T23:07:14.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:14 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/205289281' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:14.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:14 smithi138 ceph-mon[193606]: osdmap e110: 8 total, 7 up, 8 in 2023-09-23T23:07:14.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:14 smithi110 ceph-mon[212939]: failed to encode map e109 with expected crc 2023-09-23T23:07:14.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:14 smithi110 ceph-mon[212939]: pgmap v290: 89 pgs: 11 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 77 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 411 op/s; 322/68886 objects degraded (0.467%); 0 B/s, 28 objects/s recovering 2023-09-23T23:07:14.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:14 smithi110 ceph-mon[212939]: failed to encode map e110 with expected crc 2023-09-23T23:07:14.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:14 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/205289281' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:14.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:14 smithi110 ceph-mon[212939]: osdmap e110: 8 total, 7 up, 8 in 2023-09-23T23:07:14.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:14 smithi110 ceph-mon[221102]: failed to encode map e109 with expected crc 2023-09-23T23:07:14.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:14 smithi110 ceph-mon[221102]: pgmap v290: 89 pgs: 11 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 77 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 411 op/s; 322/68886 objects degraded (0.467%); 0 B/s, 28 objects/s recovering 2023-09-23T23:07:14.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:14 smithi110 ceph-mon[221102]: failed to encode map e110 with expected crc 2023-09-23T23:07:14.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:14 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/205289281' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:14.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:14 smithi110 ceph-mon[221102]: osdmap e110: 8 total, 7 up, 8 in 2023-09-23T23:07:15.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:15 smithi110 ceph-mon[212939]: failed to encode map e110 with expected crc 2023-09-23T23:07:15.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:15 smithi110 ceph-mon[212939]: failed to encode map e110 with expected crc 2023-09-23T23:07:15.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:15 smithi110 ceph-mon[221102]: failed to encode map e110 with expected crc 2023-09-23T23:07:15.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:15 smithi110 ceph-mon[221102]: failed to encode map e110 with expected crc 2023-09-23T23:07:15.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:15 smithi138 ceph-mon[193606]: failed to encode map e110 with expected crc 2023-09-23T23:07:15.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:15 smithi138 ceph-mon[193606]: failed to encode map e110 with expected crc 2023-09-23T23:07:16.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:16 smithi138 ceph-mon[193606]: pgmap v292: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 7560/61191 objects degraded (12.355%); 0 B/s, 92 objects/s recovering 2023-09-23T23:07:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:16 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3503854581' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:16.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:07:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:07:16] "GET /metrics HTTP/1.1" 200 36416 "" "Prometheus/2.43.0" 2023-09-23T23:07:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:16 smithi110 ceph-mon[212939]: pgmap v292: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 7560/61191 objects degraded (12.355%); 0 B/s, 92 objects/s recovering 2023-09-23T23:07:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:16 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3503854581' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:16.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:16 smithi110 ceph-mon[221102]: pgmap v292: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 7560/61191 objects degraded (12.355%); 0 B/s, 92 objects/s recovering 2023-09-23T23:07:16.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:16 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3503854581' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:17.451 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:17 smithi110 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.1.service: Succeeded. 2023-09-23T23:07:17.452 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:17 smithi110 systemd[1]: Stopped Ceph osd.1 for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:07:17.452 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:17 smithi110 systemd[1]: Starting Ceph osd.1 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:07:18.658 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:18 smithi110 ceph-mon[212939]: pgmap v293: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 7560/61191 objects degraded (12.355%); 0 B/s, 53 objects/s recovering 2023-09-23T23:07:18.658 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:18 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 7560/61191 objects degraded (12.355%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:18.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:18 smithi138 ceph-mon[193606]: pgmap v293: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 7560/61191 objects degraded (12.355%); 0 B/s, 53 objects/s recovering 2023-09-23T23:07:18.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:18 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 7560/61191 objects degraded (12.355%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:18.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:18 smithi110 ceph-mon[221102]: pgmap v293: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 7560/61191 objects degraded (12.355%); 0 B/s, 53 objects/s recovering 2023-09-23T23:07:18.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:18 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 7560/61191 objects degraded (12.355%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:19.454 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:19 smithi110 bash[249958]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T23:07:19.454 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:19 smithi110 bash[249958]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T23:07:19.454 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:19 smithi110 bash[249958]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T23:07:19.454 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:19 smithi110 bash[249958]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T23:07:19.455 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:19 smithi110 bash[249958]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T23:07:19.455 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:19 smithi110 bash[249958]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T23:07:19.455 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:19 smithi110 bash[249958]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T23:07:19.455 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:19 smithi110 bash[249958]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T23:07:19.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:19 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4268168386' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:19.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:19 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4268168386' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:19.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:19 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4268168386' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:20 smithi138 ceph-mon[193606]: pgmap v294: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.1 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 6851/56136 objects degraded (12.204%); 0 B/s, 49 objects/s recovering 2023-09-23T23:07:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:20 smithi110 ceph-mon[212939]: pgmap v294: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.1 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 6851/56136 objects degraded (12.204%); 0 B/s, 49 objects/s recovering 2023-09-23T23:07:20.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:20 smithi110 ceph-mon[221102]: pgmap v294: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1.1 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 6851/56136 objects degraded (12.204%); 0 B/s, 49 objects/s recovering 2023-09-23T23:07:21.457 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:21 smithi110 ceph-osd[250433]: -- 172.21.15.110:0/2884127710 <== mon.2 v2:172.21.15.110:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5574fc6804e0 con 0x5574fc652c00 2023-09-23T23:07:21.457 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:21 smithi110 bash[250363]: 038c0580de94c2fe3da0388aa26ae41baf6833e17db7460b1d2cebe6c8bc1315 2023-09-23T23:07:21.457 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:21 smithi110 systemd[1]: Started Ceph osd.1 for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:07:21.955 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:21 smithi110 conmon[250401]: 2023-09-23T23:07:21.590+0000 7fdf950c5640 -1 Falling back to public interface 2023-09-23T23:07:22.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:22 smithi138 ceph-mon[193606]: pgmap v295: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1019 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 6047/48966 objects degraded (12.349%); 0 B/s, 39 objects/s recovering 2023-09-23T23:07:22.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:22 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:22.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:22 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:22.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:22 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:07:22.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:22 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2554944790' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:22.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:22 smithi110 ceph-mon[212939]: pgmap v295: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1019 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 6047/48966 objects degraded (12.349%); 0 B/s, 39 objects/s recovering 2023-09-23T23:07:22.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:22 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:22.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:22 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:22.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:22 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:07:22.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:22 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2554944790' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:22.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:22 smithi110 ceph-mon[221102]: pgmap v295: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1019 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 6047/48966 objects degraded (12.349%); 0 B/s, 39 objects/s recovering 2023-09-23T23:07:22.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:22 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:22.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:22 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:22.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:22 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:07:22.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:22 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2554944790' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:23.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:23 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 6047/48966 objects degraded (12.349%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:23.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:23 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 6047/48966 objects degraded (12.349%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:23.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:23 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 6047/48966 objects degraded (12.349%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:24.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:24 smithi138 ceph-mon[193606]: pgmap v296: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1019 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 6047/48966 objects degraded (12.349%); 0 B/s, 32 objects/s recovering 2023-09-23T23:07:24.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:24 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2683686053' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:24.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:24 smithi110 ceph-mon[212939]: pgmap v296: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1019 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 6047/48966 objects degraded (12.349%); 0 B/s, 32 objects/s recovering 2023-09-23T23:07:24.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:24 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2683686053' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:24.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:24 smithi110 ceph-mon[221102]: pgmap v296: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 1019 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 6047/48966 objects degraded (12.349%); 0 B/s, 32 objects/s recovering 2023-09-23T23:07:24.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:24 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2683686053' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:26.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:26 smithi138 ceph-mon[193606]: pgmap v297: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 776 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 4610/37317 objects degraded (12.354%); 0 B/s, 29 objects/s recovering 2023-09-23T23:07:26.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:26 smithi110 ceph-mon[212939]: pgmap v297: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 776 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 4610/37317 objects degraded (12.354%); 0 B/s, 29 objects/s recovering 2023-09-23T23:07:26.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:26 smithi110 ceph-mon[221102]: pgmap v297: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 776 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 4610/37317 objects degraded (12.354%); 0 B/s, 29 objects/s recovering 2023-09-23T23:07:27.171 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:07:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:07:26] "GET /metrics HTTP/1.1" 200 36416 "" "Prometheus/2.43.0" 2023-09-23T23:07:27.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:27 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2388545220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:27.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:27 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:27.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:27 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:27.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:27 smithi110 ceph-mon[212939]: from='osd.1 [v2:172.21.15.110:6810/594715751,v1:172.21.15.110:6811/594715751]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T23:07:27.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:27 smithi110 ceph-mon[212939]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T23:07:27.449 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:27 smithi110 conmon[250401]: 2023-09-23T23:07:27.166+0000 7fdf950c5640 -1 osd.1 107 log_to_monitors true 2023-09-23T23:07:27.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:27 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2388545220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:27.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:27 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:27.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:27 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:27.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:27 smithi110 ceph-mon[221102]: from='osd.1 [v2:172.21.15.110:6810/594715751,v1:172.21.15.110:6811/594715751]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T23:07:27.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:27 smithi110 ceph-mon[221102]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T23:07:27.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:27 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2388545220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:27.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:27 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:27.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:27 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:27.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:27 smithi138 ceph-mon[193606]: from='osd.1 [v2:172.21.15.110:6810/594715751,v1:172.21.15.110:6811/594715751]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T23:07:27.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:27 smithi138 ceph-mon[193606]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T23:07:28.446 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:07:28 smithi110 conmon[250401]: 2023-09-23T23:07:28.088+0000 7fdf86532700 -1 osd.1 107 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T23:07:28.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[212939]: pgmap v298: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 776 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 4610/37317 objects degraded (12.354%) 2023-09-23T23:07:28.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:28.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 4610/37317 objects degraded (12.354%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:28.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[212939]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T23:07:28.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[212939]: osdmap e111: 8 total, 7 up, 8 in 2023-09-23T23:07:28.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[212939]: from='osd.1 [v2:172.21.15.110:6810/594715751,v1:172.21.15.110:6811/594715751]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:07:28.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[212939]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:07:28.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:28.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:28.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[221102]: pgmap v298: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 776 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 4610/37317 objects degraded (12.354%) 2023-09-23T23:07:28.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:28.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 4610/37317 objects degraded (12.354%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:28.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[221102]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T23:07:28.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[221102]: osdmap e111: 8 total, 7 up, 8 in 2023-09-23T23:07:28.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[221102]: from='osd.1 [v2:172.21.15.110:6810/594715751,v1:172.21.15.110:6811/594715751]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:07:28.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[221102]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:07:28.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:28.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:28.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:28 smithi138 ceph-mon[193606]: pgmap v298: 89 pgs: 9 active+undersized+degraded, 29 active+undersized, 51 active+clean; 776 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 4610/37317 objects degraded (12.354%) 2023-09-23T23:07:28.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:28.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:28 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 4610/37317 objects degraded (12.354%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:28.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:28 smithi138 ceph-mon[193606]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T23:07:28.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:28 smithi138 ceph-mon[193606]: osdmap e111: 8 total, 7 up, 8 in 2023-09-23T23:07:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:28 smithi138 ceph-mon[193606]: from='osd.1 [v2:172.21.15.110:6810/594715751,v1:172.21.15.110:6811/594715751]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:07:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:28 smithi138 ceph-mon[193606]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:07:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:29.679 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:29 smithi110 ceph-mon[212939]: failed to encode map e111 with expected crc 2023-09-23T23:07:29.679 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:29 smithi110 ceph-mon[212939]: failed to encode map e111 with expected crc 2023-09-23T23:07:29.679 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:29 smithi110 ceph-mon[212939]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:07:29.680 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:29 smithi110 ceph-mon[212939]: osd.1 [v2:172.21.15.110:6810/594715751,v1:172.21.15.110:6811/594715751] boot 2023-09-23T23:07:29.680 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:29 smithi110 ceph-mon[212939]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T23:07:29.680 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:29 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T23:07:29.680 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:29 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/879157288' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:29.680 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:29 smithi110 ceph-mon[221102]: failed to encode map e111 with expected crc 2023-09-23T23:07:29.680 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:29 smithi110 ceph-mon[221102]: failed to encode map e111 with expected crc 2023-09-23T23:07:29.680 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:29 smithi110 ceph-mon[221102]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:07:29.680 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:29 smithi110 ceph-mon[221102]: osd.1 [v2:172.21.15.110:6810/594715751,v1:172.21.15.110:6811/594715751] boot 2023-09-23T23:07:29.680 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:29 smithi110 ceph-mon[221102]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T23:07:29.680 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:29 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T23:07:29.681 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:29 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/879157288' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:29.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:29 smithi138 ceph-mon[193606]: failed to encode map e111 with expected crc 2023-09-23T23:07:29.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:29 smithi138 ceph-mon[193606]: failed to encode map e111 with expected crc 2023-09-23T23:07:29.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:29 smithi138 ceph-mon[193606]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:07:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:29 smithi138 ceph-mon[193606]: osd.1 [v2:172.21.15.110:6810/594715751,v1:172.21.15.110:6811/594715751] boot 2023-09-23T23:07:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:29 smithi138 ceph-mon[193606]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T23:07:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:29 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T23:07:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:29 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/879157288' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:30.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:30 smithi110 ceph-mon[212939]: failed to encode map e112 with expected crc 2023-09-23T23:07:30.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:30 smithi110 ceph-mon[212939]: failed to encode map e112 with expected crc 2023-09-23T23:07:30.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:30 smithi110 ceph-mon[212939]: pgmap v301: 89 pgs: 12 peering, 6 active+undersized+degraded, 20 active+undersized, 51 active+clean; 690 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 3634/33174 objects degraded (10.954%) 2023-09-23T23:07:30.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:30 smithi110 ceph-mon[212939]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T23:07:30.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:30 smithi110 ceph-mon[212939]: failed to encode map e113 with expected crc 2023-09-23T23:07:30.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:30 smithi110 ceph-mon[212939]: failed to encode map e113 with expected crc 2023-09-23T23:07:30.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:30 smithi110 ceph-mon[212939]: failed to encode map e113 with expected crc 2023-09-23T23:07:30.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:30 smithi110 ceph-mon[221102]: failed to encode map e112 with expected crc 2023-09-23T23:07:30.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:30 smithi110 ceph-mon[221102]: failed to encode map e112 with expected crc 2023-09-23T23:07:30.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:30 smithi110 ceph-mon[221102]: pgmap v301: 89 pgs: 12 peering, 6 active+undersized+degraded, 20 active+undersized, 51 active+clean; 690 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 3634/33174 objects degraded (10.954%) 2023-09-23T23:07:30.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:30 smithi110 ceph-mon[221102]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T23:07:30.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:30 smithi110 ceph-mon[221102]: failed to encode map e113 with expected crc 2023-09-23T23:07:30.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:30 smithi110 ceph-mon[221102]: failed to encode map e113 with expected crc 2023-09-23T23:07:30.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:30 smithi110 ceph-mon[221102]: failed to encode map e113 with expected crc 2023-09-23T23:07:30.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:30 smithi138 ceph-mon[193606]: failed to encode map e112 with expected crc 2023-09-23T23:07:30.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:30 smithi138 ceph-mon[193606]: failed to encode map e112 with expected crc 2023-09-23T23:07:30.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:30 smithi138 ceph-mon[193606]: pgmap v301: 89 pgs: 12 peering, 6 active+undersized+degraded, 20 active+undersized, 51 active+clean; 690 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 3634/33174 objects degraded (10.954%) 2023-09-23T23:07:30.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:30 smithi138 ceph-mon[193606]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T23:07:30.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:30 smithi138 ceph-mon[193606]: failed to encode map e113 with expected crc 2023-09-23T23:07:30.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:30 smithi138 ceph-mon[193606]: failed to encode map e113 with expected crc 2023-09-23T23:07:30.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:30 smithi138 ceph-mon[193606]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[212939]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[212939]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[212939]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[212939]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[212939]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[212939]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[212939]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[212939]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[212939]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[212939]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[212939]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[212939]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[212939]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[212939]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[212939]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T23:07:31.709 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[221102]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.709 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[221102]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[221102]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[221102]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[221102]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[221102]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[221102]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[221102]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[221102]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[221102]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[221102]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[221102]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[221102]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[221102]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.711 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:31 smithi110 ceph-mon[221102]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T23:07:31.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:31 smithi138 ceph-mon[193606]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:31 smithi138 ceph-mon[193606]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:31 smithi138 ceph-mon[193606]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:31 smithi138 ceph-mon[193606]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:31 smithi138 ceph-mon[193606]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:31 smithi138 ceph-mon[193606]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:31 smithi138 ceph-mon[193606]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:31 smithi138 ceph-mon[193606]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:31 smithi138 ceph-mon[193606]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:31 smithi138 ceph-mon[193606]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:31 smithi138 ceph-mon[193606]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:31 smithi138 ceph-mon[193606]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:31 smithi138 ceph-mon[193606]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:31 smithi138 ceph-mon[193606]: failed to encode map e113 with expected crc 2023-09-23T23:07:31.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:31 smithi138 ceph-mon[193606]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T23:07:32.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:32 smithi110 ceph-mon[212939]: failed to encode map e114 with expected crc 2023-09-23T23:07:32.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:32 smithi110 ceph-mon[212939]: failed to encode map e114 with expected crc 2023-09-23T23:07:32.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:32 smithi110 ceph-mon[212939]: failed to encode map e114 with expected crc 2023-09-23T23:07:32.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:32 smithi110 ceph-mon[212939]: failed to encode map e114 with expected crc 2023-09-23T23:07:32.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:32 smithi110 ceph-mon[212939]: failed to encode map e114 with expected crc 2023-09-23T23:07:32.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:32 smithi110 ceph-mon[212939]: pgmap v304: 89 pgs: 24 peering, 2 active+undersized+degraded, 12 active+undersized, 51 active+clean; 577 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 1356/27726 objects degraded (4.891%) 2023-09-23T23:07:32.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:32 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/201165184' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:32.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:32 smithi110 ceph-mon[221102]: failed to encode map e114 with expected crc 2023-09-23T23:07:32.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:32 smithi110 ceph-mon[221102]: failed to encode map e114 with expected crc 2023-09-23T23:07:32.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:32 smithi110 ceph-mon[221102]: failed to encode map e114 with expected crc 2023-09-23T23:07:32.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:32 smithi110 ceph-mon[221102]: failed to encode map e114 with expected crc 2023-09-23T23:07:32.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:32 smithi110 ceph-mon[221102]: failed to encode map e114 with expected crc 2023-09-23T23:07:32.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:32 smithi110 ceph-mon[221102]: pgmap v304: 89 pgs: 24 peering, 2 active+undersized+degraded, 12 active+undersized, 51 active+clean; 577 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 1356/27726 objects degraded (4.891%) 2023-09-23T23:07:32.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:32 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/201165184' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:32.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:32 smithi138 ceph-mon[193606]: failed to encode map e114 with expected crc 2023-09-23T23:07:32.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:32 smithi138 ceph-mon[193606]: failed to encode map e114 with expected crc 2023-09-23T23:07:32.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:32 smithi138 ceph-mon[193606]: failed to encode map e114 with expected crc 2023-09-23T23:07:32.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:32 smithi138 ceph-mon[193606]: failed to encode map e114 with expected crc 2023-09-23T23:07:32.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:32 smithi138 ceph-mon[193606]: failed to encode map e114 with expected crc 2023-09-23T23:07:32.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:32 smithi138 ceph-mon[193606]: pgmap v304: 89 pgs: 24 peering, 2 active+undersized+degraded, 12 active+undersized, 51 active+clean; 577 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 1356/27726 objects degraded (4.891%) 2023-09-23T23:07:32.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:32 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/201165184' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:33.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:33 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1356/27726 objects degraded (4.891%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:33.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:33 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1356/27726 objects degraded (4.891%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:33.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:33 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1356/27726 objects degraded (4.891%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:34.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:34 smithi138 ceph-mon[193606]: pgmap v305: 89 pgs: 24 peering, 2 active+undersized+degraded, 2 active+undersized, 61 active+clean; 577 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 1356/27726 objects degraded (4.891%) 2023-09-23T23:07:34.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:34 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2360867815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:34.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:34 smithi110 ceph-mon[212939]: pgmap v305: 89 pgs: 24 peering, 2 active+undersized+degraded, 2 active+undersized, 61 active+clean; 577 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 1356/27726 objects degraded (4.891%) 2023-09-23T23:07:34.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:34 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2360867815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:34.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:34 smithi110 ceph-mon[221102]: pgmap v305: 89 pgs: 24 peering, 2 active+undersized+degraded, 2 active+undersized, 61 active+clean; 577 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 1356/27726 objects degraded (4.891%) 2023-09-23T23:07:34.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:34 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2360867815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:36.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:36 smithi110 ceph-mon[221102]: pgmap v306: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 440 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 3708/21183 objects degraded (17.505%); 0 B/s, 33 objects/s recovering 2023-09-23T23:07:36.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:36 smithi110 ceph-mon[212939]: pgmap v306: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 440 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 3708/21183 objects degraded (17.505%); 0 B/s, 33 objects/s recovering 2023-09-23T23:07:36.805 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:07:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:07:36] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.43.0" 2023-09-23T23:07:36.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:36 smithi138 ceph-mon[193606]: pgmap v306: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 440 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 3708/21183 objects degraded (17.505%); 0 B/s, 33 objects/s recovering 2023-09-23T23:07:37.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:37 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1328224228' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:37.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:37 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1328224228' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:37.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:37 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1328224228' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:38.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:38 smithi138 ceph-mon[193606]: pgmap v307: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 440 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 3708/21183 objects degraded (17.505%); 0 B/s, 25 objects/s recovering 2023-09-23T23:07:38.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:38 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 3708/21183 objects degraded (17.505%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:38.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:38 smithi110 ceph-mon[221102]: pgmap v307: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 440 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 3708/21183 objects degraded (17.505%); 0 B/s, 25 objects/s recovering 2023-09-23T23:07:38.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:38 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 3708/21183 objects degraded (17.505%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:38.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:38 smithi110 ceph-mon[212939]: pgmap v307: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 440 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 3708/21183 objects degraded (17.505%); 0 B/s, 25 objects/s recovering 2023-09-23T23:07:38.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:38 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 3708/21183 objects degraded (17.505%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:39.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:39 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1832086740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:39.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:39 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1832086740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:39.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:39 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1832086740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:40.355 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:07:40 smithi110 conmon[198813]: 2023-09-23T23:07:40.351+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:07:40.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[212939]: pgmap v308: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 3708/16557 objects degraded (22.395%); 0 B/s, 22 objects/s recovering 2023-09-23T23:07:40.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:07:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:07:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:07:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:07:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:07:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:07:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:07:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[221102]: pgmap v308: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 3708/16557 objects degraded (22.395%); 0 B/s, 22 objects/s recovering 2023-09-23T23:07:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:07:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:07:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:07:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:07:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:07:40.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:07:40.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:40 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:07:40.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:40 smithi138 ceph-mon[193606]: pgmap v308: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 3708/16557 objects degraded (22.395%); 0 B/s, 22 objects/s recovering 2023-09-23T23:07:40.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:40 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:40.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:40 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:40.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:40 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:07:40.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:40 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:07:40.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:40 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:40.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:40 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:07:40.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:40 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:07:40.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:40 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:07:40.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:40 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:07:40.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:40 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:07:41.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:41 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:07:41.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:41 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:07:41.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:41 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:07:41.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:41 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:07:41.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:41 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:07:41.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:41 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:07:42.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:42 smithi138 ceph-mon[193606]: pgmap v309: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 225 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 3708/10860 objects degraded (34.144%); 0 B/s, 61 objects/s recovering 2023-09-23T23:07:42.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:42 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/772037711' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:42.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:42 smithi110 ceph-mon[221102]: pgmap v309: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 225 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 3708/10860 objects degraded (34.144%); 0 B/s, 61 objects/s recovering 2023-09-23T23:07:42.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:42 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/772037711' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:42.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:42 smithi110 ceph-mon[212939]: pgmap v309: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 225 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 3708/10860 objects degraded (34.144%); 0 B/s, 61 objects/s recovering 2023-09-23T23:07:42.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:42 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/772037711' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:43 smithi138 ceph-mon[193606]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T23:07:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:43 smithi138 ceph-mon[193606]: failed to encode map e115 with expected crc 2023-09-23T23:07:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:43 smithi138 ceph-mon[193606]: failed to encode map e115 with expected crc 2023-09-23T23:07:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:43 smithi138 ceph-mon[193606]: failed to encode map e115 with expected crc 2023-09-23T23:07:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:43 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 3708/10860 objects degraded (34.144%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:43.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:43 smithi110 ceph-mon[221102]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T23:07:43.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:43 smithi110 ceph-mon[221102]: failed to encode map e115 with expected crc 2023-09-23T23:07:43.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:43 smithi110 ceph-mon[221102]: failed to encode map e115 with expected crc 2023-09-23T23:07:43.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:43 smithi110 ceph-mon[221102]: failed to encode map e115 with expected crc 2023-09-23T23:07:43.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:43.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:43.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:43 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 3708/10860 objects degraded (34.144%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:43.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:43 smithi110 ceph-mon[212939]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T23:07:43.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:43 smithi110 ceph-mon[212939]: failed to encode map e115 with expected crc 2023-09-23T23:07:43.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:43 smithi110 ceph-mon[212939]: failed to encode map e115 with expected crc 2023-09-23T23:07:43.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:43 smithi110 ceph-mon[212939]: failed to encode map e115 with expected crc 2023-09-23T23:07:43.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:43.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:43.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:43 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 3708/10860 objects degraded (34.144%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:44.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:44 smithi138 ceph-mon[193606]: pgmap v311: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 225 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 3708/10860 objects degraded (34.144%); 0 B/s, 62 objects/s recovering 2023-09-23T23:07:44.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:44 smithi138 ceph-mon[193606]: failed to encode map e116 with expected crc 2023-09-23T23:07:44.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:44 smithi138 ceph-mon[193606]: failed to encode map e116 with expected crc 2023-09-23T23:07:44.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:44 smithi138 ceph-mon[193606]: osdmap e116: 8 total, 8 up, 8 in 2023-09-23T23:07:44.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:44 smithi138 ceph-mon[193606]: failed to encode map e116 with expected crc 2023-09-23T23:07:44.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:44 smithi138 ceph-mon[193606]: failed to encode map e116 with expected crc 2023-09-23T23:07:44.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:44 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1790008431' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:44.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:44 smithi110 ceph-mon[212939]: pgmap v311: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 225 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 3708/10860 objects degraded (34.144%); 0 B/s, 62 objects/s recovering 2023-09-23T23:07:44.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:44 smithi110 ceph-mon[212939]: failed to encode map e116 with expected crc 2023-09-23T23:07:44.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:44 smithi110 ceph-mon[212939]: failed to encode map e116 with expected crc 2023-09-23T23:07:44.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:44 smithi110 ceph-mon[212939]: osdmap e116: 8 total, 8 up, 8 in 2023-09-23T23:07:44.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:44 smithi110 ceph-mon[212939]: failed to encode map e116 with expected crc 2023-09-23T23:07:44.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:44 smithi110 ceph-mon[212939]: failed to encode map e116 with expected crc 2023-09-23T23:07:44.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:44 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1790008431' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:44.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:44 smithi110 ceph-mon[221102]: pgmap v311: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 225 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 3708/10860 objects degraded (34.144%); 0 B/s, 62 objects/s recovering 2023-09-23T23:07:44.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:44 smithi110 ceph-mon[221102]: failed to encode map e116 with expected crc 2023-09-23T23:07:44.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:44 smithi110 ceph-mon[221102]: failed to encode map e116 with expected crc 2023-09-23T23:07:44.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:44 smithi110 ceph-mon[221102]: osdmap e116: 8 total, 8 up, 8 in 2023-09-23T23:07:44.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:44 smithi110 ceph-mon[221102]: failed to encode map e116 with expected crc 2023-09-23T23:07:44.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:44 smithi110 ceph-mon[221102]: failed to encode map e116 with expected crc 2023-09-23T23:07:44.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:44 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1790008431' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:45.298 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Removed 112705 objects 2023-09-23T23:07:45.299 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Clean up completed and total clean up time :164.502 2023-09-23T23:07:46.911 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-09-23T23:07:46.911 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-09-23T23:07:46.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:46 smithi138 ceph-mon[193606]: pgmap v313: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 36 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 2947/1785 objects degraded (165.098%); 0 B/s, 119 objects/s recovering 2023-09-23T23:07:46.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:46 smithi110 ceph-mon[212939]: pgmap v313: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 36 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 2947/1785 objects degraded (165.098%); 0 B/s, 119 objects/s recovering 2023-09-23T23:07:46.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:07:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:07:46] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.43.0" 2023-09-23T23:07:46.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:46 smithi110 ceph-mon[221102]: pgmap v313: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 36 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 2947/1785 objects degraded (165.098%); 0 B/s, 119 objects/s recovering 2023-09-23T23:07:47.288 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:47.894 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:47 smithi110 ceph-mon[212939]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.894 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:47 smithi110 ceph-mon[212939]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.894 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:47 smithi110 ceph-mon[212939]: osdmap e117: 8 total, 8 up, 8 in 2023-09-23T23:07:47.894 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:47 smithi110 ceph-mon[212939]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.895 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:47 smithi110 ceph-mon[212939]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.895 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:47 smithi110 ceph-mon[212939]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.895 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:47 smithi110 ceph-mon[212939]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.895 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:47 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3714948609' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:47.895 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:47 smithi110 ceph-mon[221102]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.895 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:47 smithi110 ceph-mon[221102]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.895 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:47 smithi110 ceph-mon[221102]: osdmap e117: 8 total, 8 up, 8 in 2023-09-23T23:07:47.895 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:47 smithi110 ceph-mon[221102]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.895 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:47 smithi110 ceph-mon[221102]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.896 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:47 smithi110 ceph-mon[221102]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.896 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:47 smithi110 ceph-mon[221102]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.896 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:47 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3714948609' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:47.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:47 smithi138 ceph-mon[193606]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:47 smithi138 ceph-mon[193606]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:47 smithi138 ceph-mon[193606]: osdmap e117: 8 total, 8 up, 8 in 2023-09-23T23:07:47.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:47 smithi138 ceph-mon[193606]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:47 smithi138 ceph-mon[193606]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:47 smithi138 ceph-mon[193606]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:47 smithi138 ceph-mon[193606]: failed to encode map e117 with expected crc 2023-09-23T23:07:47.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:47 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3714948609' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:48.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:48 smithi138 ceph-mon[193606]: pgmap v315: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 36 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 2947/1785 objects degraded (165.098%); 0 B/s, 89 objects/s recovering 2023-09-23T23:07:48.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:48 smithi138 ceph-mon[193606]: failed to encode map e118 with expected crc 2023-09-23T23:07:48.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:48 smithi138 ceph-mon[193606]: failed to encode map e118 with expected crc 2023-09-23T23:07:48.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:48 smithi138 ceph-mon[193606]: osdmap e118: 8 total, 8 up, 8 in 2023-09-23T23:07:48.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:48 smithi138 ceph-mon[193606]: failed to encode map e118 with expected crc 2023-09-23T23:07:48.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:48 smithi138 ceph-mon[193606]: failed to encode map e118 with expected crc 2023-09-23T23:07:48.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:48 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2947/1785 objects degraded (165.098%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:48.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:48 smithi138 ceph-mon[193606]: failed to encode map e119 with expected crc 2023-09-23T23:07:48.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:48 smithi138 ceph-mon[193606]: osdmap e119: 8 total, 8 up, 8 in 2023-09-23T23:07:48.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:48 smithi138 ceph-mon[193606]: failed to encode map e119 with expected crc 2023-09-23T23:07:48.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:48 smithi138 ceph-mon[193606]: failed to encode map e119 with expected crc 2023-09-23T23:07:48.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:48 smithi138 ceph-mon[193606]: failed to encode map e119 with expected crc 2023-09-23T23:07:48.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[212939]: pgmap v315: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 36 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 2947/1785 objects degraded (165.098%); 0 B/s, 89 objects/s recovering 2023-09-23T23:07:48.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[212939]: failed to encode map e118 with expected crc 2023-09-23T23:07:48.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[212939]: failed to encode map e118 with expected crc 2023-09-23T23:07:48.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[212939]: osdmap e118: 8 total, 8 up, 8 in 2023-09-23T23:07:48.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[212939]: failed to encode map e118 with expected crc 2023-09-23T23:07:48.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[212939]: failed to encode map e118 with expected crc 2023-09-23T23:07:48.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2947/1785 objects degraded (165.098%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:48.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[212939]: failed to encode map e119 with expected crc 2023-09-23T23:07:48.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[212939]: osdmap e119: 8 total, 8 up, 8 in 2023-09-23T23:07:48.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[212939]: failed to encode map e119 with expected crc 2023-09-23T23:07:48.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[212939]: failed to encode map e119 with expected crc 2023-09-23T23:07:48.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[212939]: failed to encode map e119 with expected crc 2023-09-23T23:07:48.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[221102]: pgmap v315: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 36 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 2947/1785 objects degraded (165.098%); 0 B/s, 89 objects/s recovering 2023-09-23T23:07:48.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[221102]: failed to encode map e118 with expected crc 2023-09-23T23:07:48.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[221102]: failed to encode map e118 with expected crc 2023-09-23T23:07:48.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[221102]: osdmap e118: 8 total, 8 up, 8 in 2023-09-23T23:07:48.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[221102]: failed to encode map e118 with expected crc 2023-09-23T23:07:48.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[221102]: failed to encode map e118 with expected crc 2023-09-23T23:07:48.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2947/1785 objects degraded (165.098%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:07:48.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[221102]: failed to encode map e119 with expected crc 2023-09-23T23:07:48.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[221102]: osdmap e119: 8 total, 8 up, 8 in 2023-09-23T23:07:48.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[221102]: failed to encode map e119 with expected crc 2023-09-23T23:07:48.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[221102]: failed to encode map e119 with expected crc 2023-09-23T23:07:48.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:48 smithi110 ceph-mon[221102]: failed to encode map e119 with expected crc 2023-09-23T23:07:50.019 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:50 smithi110 ceph-mon[212939]: failed to encode map e120 with expected crc 2023-09-23T23:07:50.019 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:50 smithi110 ceph-mon[212939]: osdmap e120: 8 total, 8 up, 8 in 2023-09-23T23:07:50.019 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:50 smithi110 ceph-mon[212939]: failed to encode map e120 with expected crc 2023-09-23T23:07:50.019 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:50 smithi110 ceph-mon[212939]: failed to encode map e120 with expected crc 2023-09-23T23:07:50.019 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:50 smithi110 ceph-mon[212939]: pgmap v319: 89 pgs: 1 active, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 14 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 90 op/s; 1497/699 objects degraded (214.163%); 0 B/s, 347 objects/s recovering 2023-09-23T23:07:50.020 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:50 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1574736254' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:50.020 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:50 smithi110 ceph-mon[221102]: failed to encode map e120 with expected crc 2023-09-23T23:07:50.020 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:50 smithi110 ceph-mon[221102]: osdmap e120: 8 total, 8 up, 8 in 2023-09-23T23:07:50.020 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:50 smithi110 ceph-mon[221102]: failed to encode map e120 with expected crc 2023-09-23T23:07:50.020 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:50 smithi110 ceph-mon[221102]: failed to encode map e120 with expected crc 2023-09-23T23:07:50.020 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:50 smithi110 ceph-mon[221102]: pgmap v319: 89 pgs: 1 active, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 14 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 90 op/s; 1497/699 objects degraded (214.163%); 0 B/s, 347 objects/s recovering 2023-09-23T23:07:50.020 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:50 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1574736254' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:50 smithi138 ceph-mon[193606]: failed to encode map e120 with expected crc 2023-09-23T23:07:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:50 smithi138 ceph-mon[193606]: osdmap e120: 8 total, 8 up, 8 in 2023-09-23T23:07:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:50 smithi138 ceph-mon[193606]: failed to encode map e120 with expected crc 2023-09-23T23:07:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:50 smithi138 ceph-mon[193606]: failed to encode map e120 with expected crc 2023-09-23T23:07:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:50 smithi138 ceph-mon[193606]: pgmap v319: 89 pgs: 1 active, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 14 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 90 op/s; 1497/699 objects degraded (214.163%); 0 B/s, 347 objects/s recovering 2023-09-23T23:07:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:50 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1574736254' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:51.070 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_2' removed 2023-09-23T23:07:51.326 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:51 smithi110 ceph-mon[212939]: failed to encode map e120 with expected crc 2023-09-23T23:07:51.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:51 smithi110 ceph-mon[212939]: failed to encode map e120 with expected crc 2023-09-23T23:07:51.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:51 smithi110 ceph-mon[212939]: failed to encode map e121 with expected crc 2023-09-23T23:07:51.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:51 smithi110 ceph-mon[212939]: failed to encode map e121 with expected crc 2023-09-23T23:07:51.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:51 smithi110 ceph-mon[212939]: osdmap e121: 8 total, 8 up, 8 in 2023-09-23T23:07:51.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:51 smithi110 ceph-mon[212939]: failed to encode map e121 with expected crc 2023-09-23T23:07:51.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:51 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2165856555' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:07:51.327 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:51 smithi110 ceph-mon[221102]: failed to encode map e120 with expected crc 2023-09-23T23:07:51.327 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:51 smithi110 ceph-mon[221102]: failed to encode map e120 with expected crc 2023-09-23T23:07:51.327 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:51 smithi110 ceph-mon[221102]: failed to encode map e121 with expected crc 2023-09-23T23:07:51.328 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:51 smithi110 ceph-mon[221102]: failed to encode map e121 with expected crc 2023-09-23T23:07:51.328 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:51 smithi110 ceph-mon[221102]: osdmap e121: 8 total, 8 up, 8 in 2023-09-23T23:07:51.328 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:51 smithi110 ceph-mon[221102]: failed to encode map e121 with expected crc 2023-09-23T23:07:51.328 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:51 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2165856555' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:07:51.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:51 smithi138 ceph-mon[193606]: failed to encode map e120 with expected crc 2023-09-23T23:07:51.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:51 smithi138 ceph-mon[193606]: failed to encode map e120 with expected crc 2023-09-23T23:07:51.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:51 smithi138 ceph-mon[193606]: failed to encode map e121 with expected crc 2023-09-23T23:07:51.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:51 smithi138 ceph-mon[193606]: failed to encode map e121 with expected crc 2023-09-23T23:07:51.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:51 smithi138 ceph-mon[193606]: osdmap e121: 8 total, 8 up, 8 in 2023-09-23T23:07:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:51 smithi138 ceph-mon[193606]: failed to encode map e121 with expected crc 2023-09-23T23:07:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:51 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2165856555' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:07:52.332 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:52 smithi110 ceph-mon[212939]: failed to encode map e121 with expected crc 2023-09-23T23:07:52.332 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:52 smithi110 ceph-mon[212939]: failed to encode map e121 with expected crc 2023-09-23T23:07:52.332 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:52 smithi110 ceph-mon[212939]: failed to encode map e122 with expected crc 2023-09-23T23:07:52.332 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:52 smithi110 ceph-mon[212939]: failed to encode map e122 with expected crc 2023-09-23T23:07:52.332 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:52 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2165856555' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:07:52.333 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:52 smithi110 ceph-mon[212939]: osdmap e122: 8 total, 8 up, 8 in 2023-09-23T23:07:52.333 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:52 smithi110 ceph-mon[212939]: pgmap v322: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:07:52.333 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:52 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2487728600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:52.333 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:52 smithi110 ceph-mon[221102]: failed to encode map e121 with expected crc 2023-09-23T23:07:52.333 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:52 smithi110 ceph-mon[221102]: failed to encode map e121 with expected crc 2023-09-23T23:07:52.333 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:52 smithi110 ceph-mon[221102]: failed to encode map e122 with expected crc 2023-09-23T23:07:52.333 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:52 smithi110 ceph-mon[221102]: failed to encode map e122 with expected crc 2023-09-23T23:07:52.333 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:52 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2165856555' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:07:52.333 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:52 smithi110 ceph-mon[221102]: osdmap e122: 8 total, 8 up, 8 in 2023-09-23T23:07:52.334 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:52 smithi110 ceph-mon[221102]: pgmap v322: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:07:52.334 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:52 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2487728600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:52.352 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T23:07:52.353 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T23:07:52.353 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-09-23T23:07:52.353 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-09-23T23:07:52.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:52 smithi138 ceph-mon[193606]: failed to encode map e121 with expected crc 2023-09-23T23:07:52.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:52 smithi138 ceph-mon[193606]: failed to encode map e121 with expected crc 2023-09-23T23:07:52.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:52 smithi138 ceph-mon[193606]: failed to encode map e122 with expected crc 2023-09-23T23:07:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:52 smithi138 ceph-mon[193606]: failed to encode map e122 with expected crc 2023-09-23T23:07:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:52 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2165856555' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:07:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:52 smithi138 ceph-mon[193606]: osdmap e122: 8 total, 8 up, 8 in 2023-09-23T23:07:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:52 smithi138 ceph-mon[193606]: pgmap v322: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:07:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:52 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2487728600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:52.744 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:53.360 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:53 smithi110 ceph-mon[212939]: failed to encode map e122 with expected crc 2023-09-23T23:07:53.360 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:53 smithi110 ceph-mon[212939]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1497/699 objects degraded (214.163%), 2 pgs degraded) 2023-09-23T23:07:53.360 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:53 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:07:53.360 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:53 smithi110 ceph-mon[221102]: failed to encode map e122 with expected crc 2023-09-23T23:07:53.360 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:53 smithi110 ceph-mon[221102]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1497/699 objects degraded (214.163%), 2 pgs degraded) 2023-09-23T23:07:53.361 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:53 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:07:53.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:53 smithi138 ceph-mon[193606]: failed to encode map e122 with expected crc 2023-09-23T23:07:53.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:53 smithi138 ceph-mon[193606]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1497/699 objects degraded (214.163%), 2 pgs degraded) 2023-09-23T23:07:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:53 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:07:54.144 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:54 smithi110 ceph-mon[212939]: pgmap v323: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:07:54.144 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:54 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/804542232' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:54.145 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:54 smithi110 ceph-mon[221102]: pgmap v323: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:07:54.145 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:54 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/804542232' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:54.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:54 smithi138 ceph-mon[193606]: pgmap v323: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:07:54.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:54 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/804542232' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:55.044 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_3' created 2023-09-23T23:07:55.379 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:55 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3324260444' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T23:07:55.379 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:55 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3324260444' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T23:07:55.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:55 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3324260444' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T23:07:55.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:07:55 smithi110 conmon[198813]: 2023-09-23T23:07:55.377+0000 7ff0150b6700 -1 mgr.server reply reply (11) Resource temporarily unavailable 16 pgs have unknown state; cannot draw any conclusions 2023-09-23T23:07:55.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:07:55 smithi110 conmon[198813]: 2023-09-23T23:07:55.377+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy 16 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (0 PGs are or would become offline) 2023-09-23T23:07:55.765 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-09-23T23:07:56.120 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:56.326 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[212939]: failed to encode map e123 with expected crc 2023-09-23T23:07:56.326 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[212939]: failed to encode map e123 with expected crc 2023-09-23T23:07:56.326 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3324260444' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T23:07:56.326 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[212939]: osdmap e123: 8 total, 8 up, 8 in 2023-09-23T23:07:56.326 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[212939]: failed to encode map e123 with expected crc 2023-09-23T23:07:56.326 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[212939]: failed to encode map e123 with expected crc 2023-09-23T23:07:56.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[212939]: failed to encode map e123 with expected crc 2023-09-23T23:07:56.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[212939]: pgmap v325: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:07:56.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:07:56.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:07:56.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[212939]: Upgrade: 16 pgs have unknown state; cannot draw any conclusions 2023-09-23T23:07:56.327 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[221102]: failed to encode map e123 with expected crc 2023-09-23T23:07:56.327 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[221102]: failed to encode map e123 with expected crc 2023-09-23T23:07:56.327 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3324260444' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T23:07:56.327 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[221102]: osdmap e123: 8 total, 8 up, 8 in 2023-09-23T23:07:56.327 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[221102]: failed to encode map e123 with expected crc 2023-09-23T23:07:56.328 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[221102]: failed to encode map e123 with expected crc 2023-09-23T23:07:56.328 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[221102]: failed to encode map e123 with expected crc 2023-09-23T23:07:56.328 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[221102]: pgmap v325: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:07:56.328 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:07:56.328 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:07:56.328 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:56 smithi110 ceph-mon[221102]: Upgrade: 16 pgs have unknown state; cannot draw any conclusions 2023-09-23T23:07:56.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:56 smithi138 ceph-mon[193606]: failed to encode map e123 with expected crc 2023-09-23T23:07:56.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:56 smithi138 ceph-mon[193606]: failed to encode map e123 with expected crc 2023-09-23T23:07:56.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:56 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3324260444' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T23:07:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:56 smithi138 ceph-mon[193606]: osdmap e123: 8 total, 8 up, 8 in 2023-09-23T23:07:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:56 smithi138 ceph-mon[193606]: failed to encode map e123 with expected crc 2023-09-23T23:07:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:56 smithi138 ceph-mon[193606]: failed to encode map e123 with expected crc 2023-09-23T23:07:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:56 smithi138 ceph-mon[193606]: failed to encode map e123 with expected crc 2023-09-23T23:07:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:56 smithi138 ceph-mon[193606]: pgmap v325: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:07:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:56 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:07:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:56 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:07:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:56 smithi138 ceph-mon[193606]: Upgrade: 16 pgs have unknown state; cannot draw any conclusions 2023-09-23T23:07:57.045 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:07:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:07:56] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.43.0" 2023-09-23T23:07:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:57 smithi138 ceph-mon[193606]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:57 smithi138 ceph-mon[193606]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:57 smithi138 ceph-mon[193606]: osdmap e124: 8 total, 8 up, 8 in 2023-09-23T23:07:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:57 smithi138 ceph-mon[193606]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:57 smithi138 ceph-mon[193606]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:57 smithi138 ceph-mon[193606]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:57 smithi138 ceph-mon[193606]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:57 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3936971478' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:57.444 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:57 smithi110 ceph-mon[221102]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.444 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:57 smithi110 ceph-mon[221102]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.444 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:57 smithi110 ceph-mon[221102]: osdmap e124: 8 total, 8 up, 8 in 2023-09-23T23:07:57.444 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:57 smithi110 ceph-mon[221102]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.444 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:57 smithi110 ceph-mon[221102]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.444 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:57 smithi110 ceph-mon[221102]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.444 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:57 smithi110 ceph-mon[221102]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.445 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:57 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3936971478' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:57.445 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:57 smithi110 ceph-mon[212939]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.445 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:57 smithi110 ceph-mon[212939]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.445 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:57 smithi110 ceph-mon[212939]: osdmap e124: 8 total, 8 up, 8 in 2023-09-23T23:07:57.445 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:57 smithi110 ceph-mon[212939]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.445 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:57 smithi110 ceph-mon[212939]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.445 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:57 smithi110 ceph-mon[212939]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.445 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:57 smithi110 ceph-mon[212939]: failed to encode map e124 with expected crc 2023-09-23T23:07:57.445 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:57 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3936971478' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:58.059 INFO:teuthology.orchestra.run.smithi110.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-09-23T23:07:58.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:58 smithi138 ceph-mon[193606]: failed to encode map e125 with expected crc 2023-09-23T23:07:58.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:58 smithi138 ceph-mon[193606]: failed to encode map e125 with expected crc 2023-09-23T23:07:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:58 smithi138 ceph-mon[193606]: osdmap e125: 8 total, 8 up, 8 in 2023-09-23T23:07:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:58 smithi138 ceph-mon[193606]: pgmap v328: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:07:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:58 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1054655061' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:07:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:58 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:07:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:58.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:58 smithi110 ceph-mon[212939]: failed to encode map e125 with expected crc 2023-09-23T23:07:58.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:58 smithi110 ceph-mon[212939]: failed to encode map e125 with expected crc 2023-09-23T23:07:58.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:58 smithi110 ceph-mon[212939]: osdmap e125: 8 total, 8 up, 8 in 2023-09-23T23:07:58.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:58 smithi110 ceph-mon[212939]: pgmap v328: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:07:58.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:58 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1054655061' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:07:58.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:58 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:07:58.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:58.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:58 smithi110 ceph-mon[221102]: failed to encode map e125 with expected crc 2023-09-23T23:07:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:58 smithi110 ceph-mon[221102]: failed to encode map e125 with expected crc 2023-09-23T23:07:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:58 smithi110 ceph-mon[221102]: osdmap e125: 8 total, 8 up, 8 in 2023-09-23T23:07:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:58 smithi110 ceph-mon[221102]: pgmap v328: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:07:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:58 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1054655061' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:07:58.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:58 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:07:58.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:58.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:59.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:59 smithi138 ceph-mon[193606]: failed to encode map e126 with expected crc 2023-09-23T23:07:59.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:59 smithi138 ceph-mon[193606]: failed to encode map e126 with expected crc 2023-09-23T23:07:59.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:59 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:07:59.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:59 smithi138 ceph-mon[193606]: osdmap e126: 8 total, 8 up, 8 in 2023-09-23T23:07:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:59 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:59 smithi138 ceph-mon[193606]: failed to encode map e126 with expected crc 2023-09-23T23:07:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:07:59 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2805168965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:59.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:59 smithi110 ceph-mon[221102]: failed to encode map e126 with expected crc 2023-09-23T23:07:59.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:59 smithi110 ceph-mon[221102]: failed to encode map e126 with expected crc 2023-09-23T23:07:59.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:59 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:07:59.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:59 smithi110 ceph-mon[221102]: osdmap e126: 8 total, 8 up, 8 in 2023-09-23T23:07:59.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:59 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:59.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:59 smithi110 ceph-mon[221102]: failed to encode map e126 with expected crc 2023-09-23T23:07:59.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:07:59 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2805168965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:59.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:59 smithi110 ceph-mon[212939]: failed to encode map e126 with expected crc 2023-09-23T23:07:59.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:59 smithi110 ceph-mon[212939]: failed to encode map e126 with expected crc 2023-09-23T23:07:59.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:59 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:07:59.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:59 smithi110 ceph-mon[212939]: osdmap e126: 8 total, 8 up, 8 in 2023-09-23T23:07:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:59 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:07:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:59 smithi110 ceph-mon[212939]: failed to encode map e126 with expected crc 2023-09-23T23:07:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:07:59 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2805168965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:07:59.707 DEBUG:teuthology.orchestra.run.smithi110:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-09-23T23:07:59.710 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T23:07:59.711 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T23:07:59.765 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:hints = 1 2023-09-23T23:07:59.765 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T23:07:59.765 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object prefix: benchmark_data_smithi110_258128 2023-09-23T23:07:59.766 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:07:59.766 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 0 0 0 0 0 0 - 0 2023-09-23T23:08:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:00 smithi138 ceph-mon[193606]: pgmap v330: 89 pgs: 6 unknown, 83 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:08:00.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:00 smithi110 ceph-mon[221102]: pgmap v330: 89 pgs: 6 unknown, 83 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:08:00.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:00 smithi110 ceph-mon[212939]: pgmap v330: 89 pgs: 6 unknown, 83 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:08:00.765 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 1 15 4017 4002 250.111 250.125 0.00505216 0.0039878 2023-09-23T23:08:01.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:01 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3735457644' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:01.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:01 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3735457644' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:01.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:01 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3735457644' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:01.766 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 2 16 7576 7560 236.195 222.375 0.00376325 0.00422597 2023-09-23T23:08:02.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:02 smithi138 ceph-mon[193606]: pgmap v331: 89 pgs: 89 active+clean; 13 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s wr, 31 op/s 2023-09-23T23:08:02.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:02 smithi110 ceph-mon[221102]: pgmap v331: 89 pgs: 89 active+clean; 13 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s wr, 31 op/s 2023-09-23T23:08:02.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:02 smithi110 ceph-mon[212939]: pgmap v331: 89 pgs: 89 active+clean; 13 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s wr, 31 op/s 2023-09-23T23:08:02.766 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 3 16 11018 11002 229.146 215.125 0.0036246 0.0043578 2023-09-23T23:08:03.766 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 4 16 14001 13985 218.466 186.438 0.00580739 0.00457191 2023-09-23T23:08:04.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:04 smithi138 ceph-mon[193606]: pgmap v332: 89 pgs: 89 active+clean; 204 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s wr, 457 op/s 2023-09-23T23:08:04.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:04 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1689755267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:04.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:04 smithi110 ceph-mon[221102]: pgmap v332: 89 pgs: 89 active+clean; 204 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s wr, 457 op/s 2023-09-23T23:08:04.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:04 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1689755267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:04.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:04 smithi110 ceph-mon[212939]: pgmap v332: 89 pgs: 89 active+clean; 204 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s wr, 457 op/s 2023-09-23T23:08:04.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:04 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1689755267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:04.766 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 5 16 16997 16981 212.22 187.25 0.00476964 0.00470635 2023-09-23T23:08:05.767 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 6 16 19791 19775 205.936 174.625 0.00541659 0.00484917 2023-09-23T23:08:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:06 smithi138 ceph-mon[193606]: pgmap v333: 89 pgs: 89 active+clean; 947 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T23:08:06.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:06 smithi110 ceph-mon[212939]: pgmap v333: 89 pgs: 89 active+clean; 947 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T23:08:06.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:06 smithi110 ceph-mon[221102]: pgmap v333: 89 pgs: 89 active+clean; 947 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T23:08:06.767 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 7 16 22304 22288 198.953 157.062 0.00399134 0.00502044 2023-09-23T23:08:07.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:08:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:08:06] "GET /metrics HTTP/1.1" 200 36350 "" "Prometheus/2.43.0" 2023-09-23T23:08:07.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:07 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1970379344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:07.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:07 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1970379344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:07.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:07 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1970379344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:07.769 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 8 15 24849 24834 193.914 159.125 0.00376688 0.00515172 2023-09-23T23:08:08.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:08 smithi138 ceph-mon[193606]: pgmap v334: 89 pgs: 89 active+clean; 947 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-09-23T23:08:08.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:08 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:08.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:08 smithi110 ceph-mon[221102]: pgmap v334: 89 pgs: 89 active+clean; 947 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-09-23T23:08:08.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:08 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:08.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:08 smithi110 ceph-mon[212939]: pgmap v334: 89 pgs: 89 active+clean; 947 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-09-23T23:08:08.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:08 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:08.770 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 9 16 27035 27019 187.543 136.562 0.00938312 0.00532722 2023-09-23T23:08:09.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:09 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1398781029' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:09.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:09 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1398781029' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:09.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:09 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1398781029' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:09.770 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 10 16 29125 29109 181.852 130.625 0.00191019 0.00547593 2023-09-23T23:08:10.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:10 smithi138 ceph-mon[193606]: pgmap v335: 89 pgs: 89 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-09-23T23:08:10.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:10 smithi110 ceph-mon[212939]: pgmap v335: 89 pgs: 89 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-09-23T23:08:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:10 smithi110 ceph-mon[221102]: pgmap v335: 89 pgs: 89 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-09-23T23:08:10.770 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 11 16 31259 31243 177.445 133.375 0.00591017 0.0056301 2023-09-23T23:08:11.675 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:11 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:08:11.675 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:11 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:08:11.676 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:11 smithi110 ceph-mon[212939]: Upgrade: osd.2 is safe to restart 2023-09-23T23:08:11.676 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:11 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:08:11.676 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:11 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:08:11.676 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:11 smithi110 ceph-mon[221102]: Upgrade: osd.2 is safe to restart 2023-09-23T23:08:11.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:11 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:08:11.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:11 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T23:08:11.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:11 smithi138 ceph-mon[193606]: Upgrade: osd.2 is safe to restart 2023-09-23T23:08:11.770 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 12 16 33097 33081 172.232 114.875 0.00967393 0.00580152 2023-09-23T23:08:12.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:12 smithi138 ceph-mon[193606]: pgmap v336: 89 pgs: 89 active+clean; 1.7 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2023-09-23T23:08:12.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:12 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2876341927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:12.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:12 smithi110 ceph-mon[212939]: pgmap v336: 89 pgs: 89 active+clean; 1.7 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2023-09-23T23:08:12.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:12 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2876341927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:12.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:12 smithi110 ceph-mon[221102]: pgmap v336: 89 pgs: 89 active+clean; 1.7 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2023-09-23T23:08:12.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:12 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2876341927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:12.770 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 13 16 35091 35075 168.57 124.625 0.00912298 0.00592672 2023-09-23T23:08:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:13.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:13.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:13.770 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 14 16 37054 37038 165.293 122.688 0.00804249 0.00604526 2023-09-23T23:08:14.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:14 smithi138 ceph-mon[193606]: pgmap v337: 89 pgs: 89 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 157 MiB/s wr, 2.51k op/s 2023-09-23T23:08:14.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:14 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2566667709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:14.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:14 smithi110 ceph-mon[212939]: pgmap v337: 89 pgs: 89 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 157 MiB/s wr, 2.51k op/s 2023-09-23T23:08:14.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:14 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2566667709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:14.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:14 smithi110 ceph-mon[221102]: pgmap v337: 89 pgs: 89 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 157 MiB/s wr, 2.51k op/s 2023-09-23T23:08:14.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:14 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2566667709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:14.770 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 15 16 38973 38957 162.269 119.938 0.00955403 0.00615843 2023-09-23T23:08:15.772 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 16 16 40901 40885 159.658 120.5 0.00797705 0.0062585 2023-09-23T23:08:16.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:16 smithi138 ceph-mon[193606]: pgmap v338: 89 pgs: 89 active+clean; 2.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 179 MiB/s wr, 2.87k op/s 2023-09-23T23:08:16.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:16 smithi110 ceph-mon[212939]: pgmap v338: 89 pgs: 89 active+clean; 2.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 179 MiB/s wr, 2.87k op/s 2023-09-23T23:08:16.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:16 smithi110 ceph-mon[221102]: pgmap v338: 89 pgs: 89 active+clean; 2.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 179 MiB/s wr, 2.87k op/s 2023-09-23T23:08:16.771 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 17 16 42652 42636 156.695 109.438 0.0177306 0.00637609 2023-09-23T23:08:17.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:08:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:08:16] "GET /metrics HTTP/1.1" 200 36404 "" "Prometheus/2.43.0" 2023-09-23T23:08:17.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:17 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4052549362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:17.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:17 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4052549362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:17.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:17 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4052549362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:17.771 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 18 16 44478 44462 154.33 114.125 0.00469255 0.00647427 2023-09-23T23:08:18.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:18 smithi138 ceph-mon[193606]: pgmap v339: 89 pgs: 89 active+clean; 2.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-09-23T23:08:18.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:18 smithi110 ceph-mon[221102]: pgmap v339: 89 pgs: 89 active+clean; 2.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-09-23T23:08:18.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:18 smithi110 ceph-mon[212939]: pgmap v339: 89 pgs: 89 active+clean; 2.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-09-23T23:08:18.771 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 19 16 46241 46225 152.007 110.188 0.0120132 0.00657356 2023-09-23T23:08:19.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:19 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3731002252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:19.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:19 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3731002252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:19.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:19 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3731002252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:19.778 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:08:19.775317+0000 min lat: 0.00124287 max lat: 0.10242 avg lat: 0.00668832 2023-09-23T23:08:19.778 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:08:19.778 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 20 16 47847 47831 149.399 100.375 0.00848866 0.00668832 2023-09-23T23:08:20.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:20 smithi110 ceph-mon[212939]: pgmap v340: 89 pgs: 89 active+clean; 2.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 148 MiB/s wr, 2.36k op/s 2023-09-23T23:08:20.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:20 smithi110 ceph-mon[212939]: Upgrade: Updating osd.2 2023-09-23T23:08:20.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:20 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:20.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:20 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T23:08:20.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:20 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:08:20.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:20 smithi110 ceph-mon[212939]: Deploying daemon osd.2 on smithi110 2023-09-23T23:08:20.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:20 smithi110 ceph-mon[221102]: pgmap v340: 89 pgs: 89 active+clean; 2.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 148 MiB/s wr, 2.36k op/s 2023-09-23T23:08:20.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:20 smithi110 ceph-mon[221102]: Upgrade: Updating osd.2 2023-09-23T23:08:20.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:20 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:20.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:20 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T23:08:20.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:20 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:08:20.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:20 smithi110 ceph-mon[221102]: Deploying daemon osd.2 on smithi110 2023-09-23T23:08:20.777 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 21 16 49423 49407 146.962 98.5 0.0184025 0.00679938 2023-09-23T23:08:20.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:20 smithi138 ceph-mon[193606]: pgmap v340: 89 pgs: 89 active+clean; 2.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 148 MiB/s wr, 2.36k op/s 2023-09-23T23:08:20.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:20 smithi138 ceph-mon[193606]: Upgrade: Updating osd.2 2023-09-23T23:08:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:20 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:20 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T23:08:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:20 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:08:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:20 smithi138 ceph-mon[193606]: Deploying daemon osd.2 on smithi110 2023-09-23T23:08:21.777 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 22 16 50974 50958 144.689 96.9375 0.00743457 0.00690643 2023-09-23T23:08:22.778 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 23 16 52513 52497 142.58 96.1875 0.0071471 0.00700934 2023-09-23T23:08:22.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:22 smithi138 ceph-mon[193606]: pgmap v341: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2023-09-23T23:08:22.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:22 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/881496127' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:22.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:22 smithi110 ceph-mon[221102]: pgmap v341: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2023-09-23T23:08:22.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:22 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/881496127' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:22.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:22 smithi110 ceph-mon[212939]: pgmap v341: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2023-09-23T23:08:22.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:22 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/881496127' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:23.778 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 24 16 53923 53907 140.312 88.125 0.0133621 0.00712256 2023-09-23T23:08:24.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:24 smithi138 ceph-mon[193606]: pgmap v342: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-23T23:08:24.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:24 smithi110 ceph-mon[212939]: pgmap v342: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-23T23:08:24.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:24 smithi110 ceph-mon[221102]: pgmap v342: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-23T23:08:24.778 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 25 16 55142 55126 137.747 76.1875 0.0101536 0.00725441 2023-09-23T23:08:25.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:25 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/654789233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:25.202 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:25 smithi110 systemd[1]: Stopping Ceph osd.2 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:08:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:25 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/654789233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:25.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:25 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/654789233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:25.778 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 26 16 56591 56575 135.933 90.5625 0.0103212 0.00735172 2023-09-23T23:08:25.957 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:25 smithi110 conmon[142903]: 2023-09-23T23:08:25.489+0000 7f9aeb1cc700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:08:25.957 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:25 smithi110 conmon[142903]: 2023-09-23T23:08:25.489+0000 7f9aeb1cc700 -1 osd.2 126 *** Got signal Terminated *** 2023-09-23T23:08:25.957 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:25 smithi110 conmon[142903]: 2023-09-23T23:08:25.489+0000 7f9aeb1cc700 -1 osd.2 126 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:08:26.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:26 smithi138 ceph-mon[193606]: pgmap v343: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-09-23T23:08:26.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:26 smithi138 ceph-mon[193606]: osd.2 marked itself down and dead 2023-09-23T23:08:26.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:26 smithi110 ceph-mon[212939]: pgmap v343: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-09-23T23:08:26.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:26 smithi110 ceph-mon[212939]: osd.2 marked itself down and dead 2023-09-23T23:08:26.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:26 smithi110 ceph-mon[221102]: pgmap v343: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-09-23T23:08:26.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:26 smithi110 ceph-mon[221102]: osd.2 marked itself down and dead 2023-09-23T23:08:26.778 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 27 16 57042 57026 131.944 28.1875 0.00256176 0.00737651 2023-09-23T23:08:27.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:27 smithi110 ceph-mon[212939]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:08:27.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:27 smithi110 ceph-mon[212939]: osdmap e127: 8 total, 7 up, 8 in 2023-09-23T23:08:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:27 smithi110 ceph-mon[212939]: failed to encode map e127 with expected crc 2023-09-23T23:08:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:27 smithi110 ceph-mon[212939]: failed to encode map e127 with expected crc 2023-09-23T23:08:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:27 smithi110 ceph-mon[212939]: failed to encode map e127 with expected crc 2023-09-23T23:08:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:27 smithi110 ceph-mon[212939]: failed to encode map e127 with expected crc 2023-09-23T23:08:27.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:27 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3158940857' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:27.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:08:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:08:26] "GET /metrics HTTP/1.1" 200 36404 "" "Prometheus/2.43.0" 2023-09-23T23:08:27.203 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:26 smithi110 bash[259146]: 8350abe92da20bb874a273b30ba7d4a992abbde145322f246c4cefa6b6afbe38 2023-09-23T23:08:27.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:27 smithi110 ceph-mon[221102]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:08:27.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:27 smithi110 ceph-mon[221102]: osdmap e127: 8 total, 7 up, 8 in 2023-09-23T23:08:27.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:27 smithi110 ceph-mon[221102]: failed to encode map e127 with expected crc 2023-09-23T23:08:27.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:27 smithi110 ceph-mon[221102]: failed to encode map e127 with expected crc 2023-09-23T23:08:27.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:27 smithi110 ceph-mon[221102]: failed to encode map e127 with expected crc 2023-09-23T23:08:27.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:27 smithi110 ceph-mon[221102]: failed to encode map e127 with expected crc 2023-09-23T23:08:27.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:27 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3158940857' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:27 smithi138 ceph-mon[193606]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:08:27.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:27 smithi138 ceph-mon[193606]: osdmap e127: 8 total, 7 up, 8 in 2023-09-23T23:08:27.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:27 smithi138 ceph-mon[193606]: failed to encode map e127 with expected crc 2023-09-23T23:08:27.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:27 smithi138 ceph-mon[193606]: failed to encode map e127 with expected crc 2023-09-23T23:08:27.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:27 smithi138 ceph-mon[193606]: failed to encode map e127 with expected crc 2023-09-23T23:08:27.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:27 smithi138 ceph-mon[193606]: failed to encode map e127 with expected crc 2023-09-23T23:08:27.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:27 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3158940857' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:27.778 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 28 16 58159 58143 129.725 69.8125 0.00871596 0.00770452 2023-09-23T23:08:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:28 smithi138 ceph-mon[193606]: failed to encode map e127 with expected crc 2023-09-23T23:08:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:28 smithi138 ceph-mon[193606]: osdmap e128: 8 total, 7 up, 8 in 2023-09-23T23:08:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:28 smithi138 ceph-mon[193606]: failed to encode map e128 with expected crc 2023-09-23T23:08:28.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:28 smithi138 ceph-mon[193606]: failed to encode map e128 with expected crc 2023-09-23T23:08:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:28 smithi138 ceph-mon[193606]: failed to encode map e128 with expected crc 2023-09-23T23:08:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:28 smithi138 ceph-mon[193606]: failed to encode map e128 with expected crc 2023-09-23T23:08:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:28 smithi138 ceph-mon[193606]: failed to encode map e128 with expected crc 2023-09-23T23:08:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:28 smithi138 ceph-mon[193606]: pgmap v346: 89 pgs: 2 stale+active+clean, 87 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T23:08:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:28.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[221102]: failed to encode map e127 with expected crc 2023-09-23T23:08:28.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[221102]: osdmap e128: 8 total, 7 up, 8 in 2023-09-23T23:08:28.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[221102]: failed to encode map e128 with expected crc 2023-09-23T23:08:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[221102]: failed to encode map e128 with expected crc 2023-09-23T23:08:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[221102]: failed to encode map e128 with expected crc 2023-09-23T23:08:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[221102]: failed to encode map e128 with expected crc 2023-09-23T23:08:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[221102]: failed to encode map e128 with expected crc 2023-09-23T23:08:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[221102]: pgmap v346: 89 pgs: 2 stale+active+clean, 87 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T23:08:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:28.462 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[212939]: failed to encode map e127 with expected crc 2023-09-23T23:08:28.462 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[212939]: osdmap e128: 8 total, 7 up, 8 in 2023-09-23T23:08:28.463 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[212939]: failed to encode map e128 with expected crc 2023-09-23T23:08:28.463 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[212939]: failed to encode map e128 with expected crc 2023-09-23T23:08:28.463 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[212939]: failed to encode map e128 with expected crc 2023-09-23T23:08:28.463 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[212939]: failed to encode map e128 with expected crc 2023-09-23T23:08:28.463 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[212939]: failed to encode map e128 with expected crc 2023-09-23T23:08:28.463 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[212939]: pgmap v346: 89 pgs: 2 stale+active+clean, 87 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T23:08:28.463 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:28.463 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:28.778 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 29 16 59707 59691 128.588 96.75 0.0136736 0.007772 2023-09-23T23:08:29.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:29 smithi110 ceph-mon[212939]: failed to encode map e128 with expected crc 2023-09-23T23:08:29.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:29 smithi110 ceph-mon[212939]: failed to encode map e128 with expected crc 2023-09-23T23:08:29.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:29 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1081140717' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:29.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:29 smithi110 ceph-mon[221102]: failed to encode map e128 with expected crc 2023-09-23T23:08:29.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:29 smithi110 ceph-mon[221102]: failed to encode map e128 with expected crc 2023-09-23T23:08:29.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:29 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1081140717' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:29 smithi138 ceph-mon[193606]: failed to encode map e128 with expected crc 2023-09-23T23:08:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:29 smithi138 ceph-mon[193606]: failed to encode map e128 with expected crc 2023-09-23T23:08:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:29 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1081140717' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:29.778 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 30 16 61306 61290 127.633 99.9375 0.00558166 0.00783122 2023-09-23T23:08:30.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:30 smithi138 ceph-mon[193606]: pgmap v347: 89 pgs: 5 active+undersized+degraded, 1 stale+active+clean, 13 active+undersized, 70 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 14854/173790 objects degraded (8.547%) 2023-09-23T23:08:30.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:30 smithi110 ceph-mon[212939]: pgmap v347: 89 pgs: 5 active+undersized+degraded, 1 stale+active+clean, 13 active+undersized, 70 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 14854/173790 objects degraded (8.547%) 2023-09-23T23:08:30.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:30 smithi110 ceph-mon[221102]: pgmap v347: 89 pgs: 5 active+undersized+degraded, 1 stale+active+clean, 13 active+undersized, 70 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 14854/173790 objects degraded (8.547%) 2023-09-23T23:08:30.778 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 31 16 62866 62850 126.661 97.5 0.00989104 0.00789063 2023-09-23T23:08:30.951 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:30 smithi110 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.2.service: Succeeded. 2023-09-23T23:08:30.951 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:30 smithi110 systemd[1]: Stopped Ceph osd.2 for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:08:30.951 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:30 smithi110 systemd[1]: Starting Ceph osd.2 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:08:31.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:31 smithi138 ceph-mon[193606]: Health check failed: Degraded data redundancy: 14854/173790 objects degraded (8.547%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:31.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:31 smithi110 ceph-mon[212939]: Health check failed: Degraded data redundancy: 14854/173790 objects degraded (8.547%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:31.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:31 smithi110 ceph-mon[221102]: Health check failed: Degraded data redundancy: 14854/173790 objects degraded (8.547%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:31.779 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 32 16 64277 64261 125.459 88.1875 0.0134488 0.00796698 2023-09-23T23:08:32.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:32 smithi138 ceph-mon[193606]: pgmap v348: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 22368/180804 objects degraded (12.371%) 2023-09-23T23:08:32.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:32 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4175279176' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:32.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:32 smithi110 ceph-mon[212939]: pgmap v348: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 22368/180804 objects degraded (12.371%) 2023-09-23T23:08:32.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:32 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4175279176' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:32.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:32 smithi110 ceph-mon[221102]: pgmap v348: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 22368/180804 objects degraded (12.371%) 2023-09-23T23:08:32.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:32 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4175279176' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:32.453 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:32 smithi110 bash[259825]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T23:08:32.453 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:32 smithi110 bash[259825]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T23:08:32.453 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:32 smithi110 bash[259825]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T23:08:32.453 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:32 smithi110 bash[259825]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T23:08:32.453 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:32 smithi110 bash[259825]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T23:08:32.453 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:32 smithi110 bash[259825]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T23:08:32.453 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:32 smithi110 bash[259825]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T23:08:32.453 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:32 smithi110 bash[259825]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T23:08:32.779 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 33 16 65789 65773 124.517 94.5 0.0141724 0.0080262 2023-09-23T23:08:33.781 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 34 16 66935 66919 122.955 71.625 0.0113919 0.00812898 2023-09-23T23:08:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:34 smithi138 ceph-mon[193606]: pgmap v349: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 23352/186447 objects degraded (12.525%) 2023-09-23T23:08:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:34 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4098016365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:34 smithi110 ceph-mon[212939]: pgmap v349: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 23352/186447 objects degraded (12.525%) 2023-09-23T23:08:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:34 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4098016365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:34.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:34 smithi110 ceph-mon[221102]: pgmap v349: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 23352/186447 objects degraded (12.525%) 2023-09-23T23:08:34.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:34 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4098016365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:34.783 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 35 16 68356 68340 121.975 88.8125 0.0153287 0.00819356 2023-09-23T23:08:34.951 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:34 smithi110 bash[260261]: 94b7643f91e8dfd098dfd2109d459edc6b49adfe015546cb45abae514b42d926 2023-09-23T23:08:34.951 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:34 smithi110 ceph-osd[260322]: -- 172.21.15.110:0/1405984305 <== mon.2 v2:172.21.15.110:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5618875344e0 con 0x561887507000 2023-09-23T23:08:34.952 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:34 smithi110 systemd[1]: Started Ceph osd.2 for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:08:35.453 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:35 smithi110 conmon[260298]: 2023-09-23T23:08:35.122+0000 7ff8d7089640 -1 Falling back to public interface 2023-09-23T23:08:35.783 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 36 16 69677 69661 120.881 82.5625 0.0130762 0.00826733 2023-09-23T23:08:35.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:35 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:35.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:35 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:35.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:35 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:08:35.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:35 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:35.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:35 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:35.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:35 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:35.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:35 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:35.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:35 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:35.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:35 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:35.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:35 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:08:35.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:35 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:35.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:35 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:35.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:35 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:35.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:35 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:35.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:35 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:35.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:35 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:35.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:35 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:08:35.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:35 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:35.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:35 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:35.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:35 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:35.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:35 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:36.690 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:36 smithi110 ceph-mon[212939]: pgmap v350: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s; 25133/202536 objects degraded (12.409%) 2023-09-23T23:08:36.690 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:36 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 25133/202536 objects degraded (12.409%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:36.690 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:36 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/713146349' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:36.787 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 37 16 70955 70939 119.764 79.875 0.0102441 0.00834477 2023-09-23T23:08:36.953 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:08:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:08:36] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.43.0" 2023-09-23T23:08:36.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:36 smithi110 ceph-mon[221102]: pgmap v350: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s; 25133/202536 objects degraded (12.409%) 2023-09-23T23:08:36.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:36 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 25133/202536 objects degraded (12.409%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:36.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:36 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/713146349' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:37.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:36 smithi138 ceph-mon[193606]: pgmap v350: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s; 25133/202536 objects degraded (12.409%) 2023-09-23T23:08:37.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:36 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 25133/202536 objects degraded (12.409%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:37.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:36 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/713146349' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:37.788 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 38 16 72179 72163 118.62 76.5 0.0168655 0.008426 2023-09-23T23:08:38.788 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 39 16 73395 73379 117.527 76 0.0221314 0.00850381 2023-09-23T23:08:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:38 smithi138 ceph-mon[193606]: pgmap v351: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 25133/202536 objects degraded (12.409%) 2023-09-23T23:08:39.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:38 smithi110 ceph-mon[212939]: pgmap v351: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 25133/202536 objects degraded (12.409%) 2023-09-23T23:08:39.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:38 smithi110 ceph-mon[221102]: pgmap v351: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 25133/202536 objects degraded (12.409%) 2023-09-23T23:08:39.791 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:08:39.790684+0000 min lat: 0.00124287 max lat: 1.01785 avg lat: 0.00858355 2023-09-23T23:08:39.791 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:08:39.791 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 40 16 74592 74576 116.452 74.8125 0.0100893 0.00858355 2023-09-23T23:08:39.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:39 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2921259031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:39.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:39 smithi138 ceph-mon[193606]: pgmap v352: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 26724/214527 objects degraded (12.457%) 2023-09-23T23:08:40.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:39 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2921259031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:40.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:39 smithi110 ceph-mon[212939]: pgmap v352: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 26724/214527 objects degraded (12.457%) 2023-09-23T23:08:40.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:39 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2921259031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:40.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:39 smithi110 ceph-mon[221102]: pgmap v352: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 26724/214527 objects degraded (12.457%) 2023-09-23T23:08:40.791 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 41 16 75752 75736 115.38 72.5 0.0203961 0.00866199 2023-09-23T23:08:41.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:40 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:41.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:40 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:41.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:40 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:41.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:40 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:41.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:40 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:41.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:40 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:41.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:40 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:41.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:40 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:41.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:40 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:41.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:40 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:41.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:40 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:41.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:40 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:41.793 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 42 16 76811 76795 114.204 66.1875 0.0102507 0.008752 2023-09-23T23:08:42.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:42 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:42.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:42 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:42.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:42 smithi110 ceph-mon[212939]: pgmap v353: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 27482/221466 objects degraded (12.409%) 2023-09-23T23:08:42.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:42 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2879606205' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:42.327 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:42 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:42.328 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:42 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:42.328 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:42 smithi110 ceph-mon[221102]: pgmap v353: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 27482/221466 objects degraded (12.409%) 2023-09-23T23:08:42.328 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:42 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2879606205' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:42 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:42 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:42 smithi138 ceph-mon[193606]: pgmap v353: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 27482/221466 objects degraded (12.409%) 2023-09-23T23:08:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:42 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2879606205' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:42.795 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 43 16 77989 77973 113.254 73.625 0.00919877 0.00882471 2023-09-23T23:08:43.202 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:42 smithi110 conmon[260298]: 2023-09-23T23:08:42.861+0000 7ff8d7089640 -1 osd.2 126 log_to_monitors true 2023-09-23T23:08:43.202 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:08:43 smithi110 conmon[260298]: 2023-09-23T23:08:43.111+0000 7ff8c5cf1700 -1 osd.2 126 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T23:08:43.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:43 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 27482/221466 objects degraded (12.409%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:43.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:43 smithi110 ceph-mon[212939]: from='osd.2 [v2:172.21.15.110:6818/3154710012,v1:172.21.15.110:6819/3154710012]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T23:08:43.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:43 smithi110 ceph-mon[212939]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T23:08:43.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:43.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:43.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:43 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 27482/221466 objects degraded (12.409%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:43.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:43 smithi110 ceph-mon[221102]: from='osd.2 [v2:172.21.15.110:6818/3154710012,v1:172.21.15.110:6819/3154710012]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T23:08:43.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:43 smithi110 ceph-mon[221102]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T23:08:43.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:43.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:43 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 27482/221466 objects degraded (12.409%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:43 smithi138 ceph-mon[193606]: from='osd.2 [v2:172.21.15.110:6818/3154710012,v1:172.21.15.110:6819/3154710012]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T23:08:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:43 smithi138 ceph-mon[193606]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T23:08:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:43.795 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 44 16 79358 79342 112.625 85.5625 0.00664989 0.00887479 2023-09-23T23:08:44.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:44 smithi138 ceph-mon[193606]: failed to encode map e129 with expected crc 2023-09-23T23:08:44.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:44 smithi138 ceph-mon[193606]: failed to encode map e129 with expected crc 2023-09-23T23:08:44.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:44 smithi138 ceph-mon[193606]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T23:08:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:44 smithi138 ceph-mon[193606]: osdmap e129: 8 total, 7 up, 8 in 2023-09-23T23:08:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:44 smithi138 ceph-mon[193606]: from='osd.2 [v2:172.21.15.110:6818/3154710012,v1:172.21.15.110:6819/3154710012]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:08:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:44 smithi138 ceph-mon[193606]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:08:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:44 smithi138 ceph-mon[193606]: pgmap v355: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 28184/225714 objects degraded (12.487%) 2023-09-23T23:08:44.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:44 smithi110 ceph-mon[212939]: failed to encode map e129 with expected crc 2023-09-23T23:08:44.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:44 smithi110 ceph-mon[212939]: failed to encode map e129 with expected crc 2023-09-23T23:08:44.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:44 smithi110 ceph-mon[212939]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T23:08:44.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:44 smithi110 ceph-mon[212939]: osdmap e129: 8 total, 7 up, 8 in 2023-09-23T23:08:44.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:44 smithi110 ceph-mon[212939]: from='osd.2 [v2:172.21.15.110:6818/3154710012,v1:172.21.15.110:6819/3154710012]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:08:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:44 smithi110 ceph-mon[212939]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:08:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:44 smithi110 ceph-mon[212939]: pgmap v355: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 28184/225714 objects degraded (12.487%) 2023-09-23T23:08:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:44 smithi110 ceph-mon[221102]: failed to encode map e129 with expected crc 2023-09-23T23:08:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:44 smithi110 ceph-mon[221102]: failed to encode map e129 with expected crc 2023-09-23T23:08:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:44 smithi110 ceph-mon[221102]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T23:08:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:44 smithi110 ceph-mon[221102]: osdmap e129: 8 total, 7 up, 8 in 2023-09-23T23:08:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:44 smithi110 ceph-mon[221102]: from='osd.2 [v2:172.21.15.110:6818/3154710012,v1:172.21.15.110:6819/3154710012]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:08:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:44 smithi110 ceph-mon[221102]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:08:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:44 smithi110 ceph-mon[221102]: pgmap v355: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 28184/225714 objects degraded (12.487%) 2023-09-23T23:08:44.796 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 45 16 79786 79770 110.717 26.75 0.00274382 0.0088947 2023-09-23T23:08:45.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:45 smithi138 ceph-mon[193606]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:08:45.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:45 smithi138 ceph-mon[193606]: failed to encode map e130 with expected crc 2023-09-23T23:08:45.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:45 smithi138 ceph-mon[193606]: osd.2 [v2:172.21.15.110:6818/3154710012,v1:172.21.15.110:6819/3154710012] boot 2023-09-23T23:08:45.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:45 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T23:08:45.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:45 smithi138 ceph-mon[193606]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T23:08:45.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:45 smithi138 ceph-mon[193606]: failed to encode map e130 with expected crc 2023-09-23T23:08:45.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:45 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3124344002' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:45.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:45 smithi110 ceph-mon[212939]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:08:45.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:45 smithi110 ceph-mon[212939]: failed to encode map e130 with expected crc 2023-09-23T23:08:45.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:45 smithi110 ceph-mon[212939]: osd.2 [v2:172.21.15.110:6818/3154710012,v1:172.21.15.110:6819/3154710012] boot 2023-09-23T23:08:45.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:45 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T23:08:45.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:45 smithi110 ceph-mon[212939]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T23:08:45.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:45 smithi110 ceph-mon[212939]: failed to encode map e130 with expected crc 2023-09-23T23:08:45.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:45 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3124344002' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:45.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:45 smithi110 ceph-mon[221102]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:08:45.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:45 smithi110 ceph-mon[221102]: failed to encode map e130 with expected crc 2023-09-23T23:08:45.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:45 smithi110 ceph-mon[221102]: osd.2 [v2:172.21.15.110:6818/3154710012,v1:172.21.15.110:6819/3154710012] boot 2023-09-23T23:08:45.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:45 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T23:08:45.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:45 smithi110 ceph-mon[221102]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T23:08:45.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:45 smithi110 ceph-mon[221102]: failed to encode map e130 with expected crc 2023-09-23T23:08:45.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:45 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3124344002' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:45.796 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 46 16 79786 79770 108.312 0 - 0.0088947 2023-09-23T23:08:46.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:46 smithi138 ceph-mon[193606]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:46 smithi138 ceph-mon[193606]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:46 smithi138 ceph-mon[193606]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T23:08:46.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:46 smithi138 ceph-mon[193606]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:46 smithi138 ceph-mon[193606]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:46 smithi138 ceph-mon[193606]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:46 smithi138 ceph-mon[193606]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:46 smithi138 ceph-mon[193606]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:46 smithi138 ceph-mon[193606]: pgmap v358: 89 pgs: 15 peering, 2 active+undersized+degraded, 8 active+undersized, 64 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 9804/238413 objects degraded (4.112%) 2023-09-23T23:08:46.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[212939]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[212939]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[212939]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T23:08:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[212939]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[212939]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[212939]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[212939]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[212939]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[212939]: pgmap v358: 89 pgs: 15 peering, 2 active+undersized+degraded, 8 active+undersized, 64 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 9804/238413 objects degraded (4.112%) 2023-09-23T23:08:46.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[221102]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[221102]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[221102]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T23:08:46.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[221102]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[221102]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[221102]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[221102]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[221102]: failed to encode map e131 with expected crc 2023-09-23T23:08:46.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:46 smithi110 ceph-mon[221102]: pgmap v358: 89 pgs: 15 peering, 2 active+undersized+degraded, 8 active+undersized, 64 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 9804/238413 objects degraded (4.112%) 2023-09-23T23:08:46.796 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 47 16 80352 80336 106.761 17.6875 0.0119284 0.0093623 2023-09-23T23:08:47.118 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[212939]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.118 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[212939]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.118 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[212939]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.118 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[212939]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T23:08:47.118 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[212939]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.118 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[212939]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.118 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[212939]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.118 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[212939]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.118 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[212939]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.119 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/257550343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:47.119 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:08:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:08:46] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.43.0" 2023-09-23T23:08:47.119 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[221102]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.119 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[221102]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.119 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[221102]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.119 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[221102]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T23:08:47.119 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[221102]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.119 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[221102]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.120 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[221102]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.120 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[221102]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.120 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[221102]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.120 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:47 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/257550343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:47.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:47 smithi138 ceph-mon[193606]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:47 smithi138 ceph-mon[193606]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:47 smithi138 ceph-mon[193606]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:47 smithi138 ceph-mon[193606]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T23:08:47.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:47 smithi138 ceph-mon[193606]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:47 smithi138 ceph-mon[193606]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:47 smithi138 ceph-mon[193606]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:47 smithi138 ceph-mon[193606]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:47 smithi138 ceph-mon[193606]: failed to encode map e132 with expected crc 2023-09-23T23:08:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:47 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/257550343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:47.800 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 48 16 81411 81395 105.906 66.1875 0.0158548 0.00943804 2023-09-23T23:08:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:48 smithi138 ceph-mon[193606]: pgmap v360: 89 pgs: 15 peering, 2 active+undersized+degraded, 8 active+undersized, 64 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 9804/238413 objects degraded (4.112%) 2023-09-23T23:08:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:48 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 9804/238413 objects degraded (4.112%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:48.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:48 smithi110 ceph-mon[212939]: pgmap v360: 89 pgs: 15 peering, 2 active+undersized+degraded, 8 active+undersized, 64 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 9804/238413 objects degraded (4.112%) 2023-09-23T23:08:48.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:48 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 9804/238413 objects degraded (4.112%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:48.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:48 smithi110 ceph-mon[221102]: pgmap v360: 89 pgs: 15 peering, 2 active+undersized+degraded, 8 active+undersized, 64 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 9804/238413 objects degraded (4.112%) 2023-09-23T23:08:48.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:48 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 9804/238413 objects degraded (4.112%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:48.801 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 49 16 82445 82429 105.064 64.625 0.0257932 0.0095117 2023-09-23T23:08:49.800 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 50 16 83427 83411 104.191 61.375 0.0130942 0.00959324 2023-09-23T23:08:50.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:50 smithi138 ceph-mon[193606]: pgmap v361: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 6 peering, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 5449/244308 objects degraded (2.230%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T23:08:50.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:50 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4215452321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:50 smithi110 ceph-mon[212939]: pgmap v361: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 6 peering, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 5449/244308 objects degraded (2.230%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T23:08:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:50 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4215452321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:50.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:50 smithi110 ceph-mon[221102]: pgmap v361: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 6 peering, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 5449/244308 objects degraded (2.230%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T23:08:50.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:50 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4215452321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:50.800 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 51 16 84348 84332 103.277 57.5625 0.0142128 0.009678 2023-09-23T23:08:51.454 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:08:51 smithi110 conmon[198813]: 2023-09-23T23:08:51.451+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:08:51.801 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 52 16 85328 85312 102.469 61.25 0.00683559 0.00975476 2023-09-23T23:08:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:52 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:52.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:52 smithi138 ceph-mon[193606]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 8259/248433 objects degraded (3.324%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T23:08:52.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:52 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:52.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:52 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:08:52.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:52 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:08:52.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:52 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:52.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:52 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:08:52.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:52 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:08:52.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:52 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:08:52.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:52 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:08:52.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:52 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:08:52.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:52 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:08:52.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:52 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:08:52.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:52 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2415103715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:52.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:52.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[212939]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 8259/248433 objects degraded (3.324%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T23:08:52.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:52.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:08:52.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:08:52.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:52.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:08:52.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:08:52.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:08:52.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:08:52.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:08:52.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:08:52.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:08:52.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2415103715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:52.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:52.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[221102]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 8259/248433 objects degraded (3.324%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T23:08:52.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:52.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:08:52.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:08:52.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:52.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:08:52.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:08:52.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:08:52.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:08:52.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:08:52.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:08:52.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:08:52.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:52 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2415103715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:52.801 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 53 16 86376 86360 101.771 65.5 0.0192915 0.0098209 2023-09-23T23:08:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:53 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 8259/248433 objects degraded (3.324%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:53.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:53 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 8259/248433 objects degraded (3.324%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:53.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:53 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 8259/248433 objects degraded (3.324%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:53.801 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 54 16 87410 87394 101.083 64.625 0.0112043 0.00988617 2023-09-23T23:08:54.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:54 smithi110 ceph-mon[212939]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 35 MiB/s wr, 563 op/s; 8259/252069 objects degraded (3.276%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T23:08:54.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:54 smithi110 ceph-mon[221102]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 35 MiB/s wr, 563 op/s; 8259/252069 objects degraded (3.276%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T23:08:54.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:54 smithi138 ceph-mon[193606]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 35 MiB/s wr, 563 op/s; 8259/252069 objects degraded (3.276%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T23:08:54.802 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 55 16 88392 88376 100.361 61.375 0.00732251 0.00995958 2023-09-23T23:08:55.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:55 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/256772571' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:55.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:55 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/256772571' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:55.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:55 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/256772571' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:55.801 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 56 16 89438 89422 99.7366 65.375 0.0158401 0.0100206 2023-09-23T23:08:56.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:56 smithi138 ceph-mon[193606]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s; 7841/263301 objects degraded (2.978%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T23:08:56.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:56 smithi110 ceph-mon[212939]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s; 7841/263301 objects degraded (2.978%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T23:08:56.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:56 smithi110 ceph-mon[221102]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s; 7841/263301 objects degraded (2.978%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T23:08:56.802 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 57 16 90420 90404 99.0639 61.375 0.00996241 0.0100893 2023-09-23T23:08:57.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:08:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:08:56] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.43.0" 2023-09-23T23:08:57.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:57 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1867114267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:57.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:57 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1867114267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:57.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:57 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1867114267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:08:57.802 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 58 16 91444 91428 98.4596 64 0.00436141 0.0101526 2023-09-23T23:08:58.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:58 smithi138 ceph-mon[193606]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 7841/263301 objects degraded (2.978%); 3.6 MiB/s, 56 objects/s recovering 2023-09-23T23:08:58.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:58.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:58.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:08:58 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 7841/263301 objects degraded (2.978%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:58.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:58 smithi110 ceph-mon[212939]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 7841/263301 objects degraded (2.978%); 3.6 MiB/s, 56 objects/s recovering 2023-09-23T23:08:58.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:58.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:58.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:08:58 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 7841/263301 objects degraded (2.978%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:58 smithi110 ceph-mon[221102]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 7841/263301 objects degraded (2.978%); 3.6 MiB/s, 56 objects/s recovering 2023-09-23T23:08:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:08:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:08:58 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 7841/263301 objects degraded (2.978%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:08:58.802 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 59 16 92449 92433 97.8556 62.8125 0.0159696 0.0102139 2023-09-23T23:08:59.802 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:08:59.802295+0000 min lat: 0.00124287 max lat: 2.12462 avg lat: 0.0102835 2023-09-23T23:08:59.802 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:08:59.802 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 60 16 93378 93362 97.1927 58.0625 0.0123785 0.0102835 2023-09-23T23:09:00.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:00 smithi138 ceph-mon[193606]: pgmap v366: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 7399/272826 objects degraded (2.712%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T23:09:00.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:00 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/711566705' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:00.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:00 smithi110 ceph-mon[212939]: pgmap v366: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 7399/272826 objects degraded (2.712%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T23:09:00.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:00 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/711566705' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:00.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:00 smithi110 ceph-mon[221102]: pgmap v366: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 7399/272826 objects degraded (2.712%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T23:09:00.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:00 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/711566705' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:00.802 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 61 16 94389 94373 96.6354 63.1875 0.00747302 0.0103438 2023-09-23T23:09:01.802 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 62 16 95350 95334 96.0457 60.0625 0.0173823 0.0104063 2023-09-23T23:09:02.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:02 smithi138 ceph-mon[193606]: pgmap v367: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s; 7399/278325 objects degraded (2.658%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:09:02.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:02 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1943218075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:02.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:02 smithi110 ceph-mon[212939]: pgmap v367: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s; 7399/278325 objects degraded (2.658%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:09:02.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:02 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1943218075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:02.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:02 smithi110 ceph-mon[221102]: pgmap v367: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s; 7399/278325 objects degraded (2.658%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:09:02.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:02 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1943218075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:02.806 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 63 16 96286 96270 95.4484 58.5 0.0148957 0.0104718 2023-09-23T23:09:03.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:03 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 7399/278325 objects degraded (2.658%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:03.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:03 smithi138 ceph-mon[193606]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T23:09:03.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:03 smithi138 ceph-mon[193606]: failed to encode map e133 with expected crc 2023-09-23T23:09:03.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:03 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 7399/278325 objects degraded (2.658%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:03.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:03 smithi110 ceph-mon[212939]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T23:09:03.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:03 smithi110 ceph-mon[212939]: failed to encode map e133 with expected crc 2023-09-23T23:09:03.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:03 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 7399/278325 objects degraded (2.658%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:03.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:03 smithi110 ceph-mon[221102]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T23:09:03.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:03 smithi110 ceph-mon[221102]: failed to encode map e133 with expected crc 2023-09-23T23:09:03.803 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 64 16 96698 96682 94.3599 25.75 0.00346764 0.0104873 2023-09-23T23:09:04.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:04 smithi138 ceph-mon[193606]: failed to encode map e133 with expected crc 2023-09-23T23:09:04.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:04 smithi138 ceph-mon[193606]: failed to encode map e133 with expected crc 2023-09-23T23:09:04.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:04 smithi138 ceph-mon[193606]: pgmap v369: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 7371/282063 objects degraded (2.613%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T23:09:04.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:04 smithi138 ceph-mon[193606]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T23:09:04.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:04 smithi110 ceph-mon[212939]: failed to encode map e133 with expected crc 2023-09-23T23:09:04.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:04 smithi110 ceph-mon[212939]: failed to encode map e133 with expected crc 2023-09-23T23:09:04.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:04 smithi110 ceph-mon[212939]: pgmap v369: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 7371/282063 objects degraded (2.613%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T23:09:04.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:04 smithi110 ceph-mon[212939]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T23:09:04.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:04 smithi110 ceph-mon[221102]: failed to encode map e133 with expected crc 2023-09-23T23:09:04.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:04 smithi110 ceph-mon[221102]: failed to encode map e133 with expected crc 2023-09-23T23:09:04.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:04 smithi110 ceph-mon[221102]: pgmap v369: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 7371/282063 objects degraded (2.613%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T23:09:04.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:04 smithi110 ceph-mon[221102]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T23:09:04.803 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 65 16 97319 97303 93.5057 38.8125 0.0165078 0.0106893 2023-09-23T23:09:05.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:05 smithi138 ceph-mon[193606]: failed to encode map e134 with expected crc 2023-09-23T23:09:05.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:05 smithi138 ceph-mon[193606]: failed to encode map e134 with expected crc 2023-09-23T23:09:05.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:05 smithi138 ceph-mon[193606]: failed to encode map e134 with expected crc 2023-09-23T23:09:05.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:05 smithi138 ceph-mon[193606]: failed to encode map e134 with expected crc 2023-09-23T23:09:05.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:05 smithi138 ceph-mon[193606]: failed to encode map e134 with expected crc 2023-09-23T23:09:05.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:05 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/977198467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:05.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:05 smithi110 ceph-mon[212939]: failed to encode map e134 with expected crc 2023-09-23T23:09:05.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:05 smithi110 ceph-mon[212939]: failed to encode map e134 with expected crc 2023-09-23T23:09:05.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:05 smithi110 ceph-mon[212939]: failed to encode map e134 with expected crc 2023-09-23T23:09:05.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:05 smithi110 ceph-mon[212939]: failed to encode map e134 with expected crc 2023-09-23T23:09:05.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:05 smithi110 ceph-mon[212939]: failed to encode map e134 with expected crc 2023-09-23T23:09:05.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:05 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/977198467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:05.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:05 smithi110 ceph-mon[221102]: failed to encode map e134 with expected crc 2023-09-23T23:09:05.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:05 smithi110 ceph-mon[221102]: failed to encode map e134 with expected crc 2023-09-23T23:09:05.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:05 smithi110 ceph-mon[221102]: failed to encode map e134 with expected crc 2023-09-23T23:09:05.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:05 smithi110 ceph-mon[221102]: failed to encode map e134 with expected crc 2023-09-23T23:09:05.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:05 smithi110 ceph-mon[221102]: failed to encode map e134 with expected crc 2023-09-23T23:09:05.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:05 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/977198467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:05.803 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 66 16 98261 98245 92.9812 58.875 0.0079364 0.0107502 2023-09-23T23:09:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:06 smithi138 ceph-mon[193606]: pgmap v371: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 7029/290832 objects degraded (2.417%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T23:09:06.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:06 smithi110 ceph-mon[212939]: pgmap v371: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 7029/290832 objects degraded (2.417%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T23:09:06.702 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:09:06 smithi110 conmon[198813]: 2023-09-23T23:09:06.468+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:09:06.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:06 smithi110 ceph-mon[221102]: pgmap v371: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 7029/290832 objects degraded (2.417%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T23:09:06.804 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 67 16 99175 99159 92.4462 57.125 0.0265514 0.0108121 2023-09-23T23:09:07.204 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:09:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:09:06] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.43.0" 2023-09-23T23:09:07.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:07 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:07.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:07 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:07.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:07 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:09:07.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:07 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/851249839' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:07.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:07 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:07.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:07 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:07.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:07 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:09:07.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:07 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/851249839' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:07.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:07 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:07.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:07 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:07.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:07 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:09:07.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:07 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/851249839' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:07.804 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 68 15 100078 100063 91.9178 56.5 0.0141611 0.0108747 2023-09-23T23:09:08.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:08 smithi138 ceph-mon[193606]: pgmap v372: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 7029/290832 objects degraded (2.417%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T23:09:08.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:08 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 7029/290832 objects degraded (2.417%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:08.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:08 smithi110 ceph-mon[212939]: pgmap v372: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 7029/290832 objects degraded (2.417%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T23:09:08.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:08 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 7029/290832 objects degraded (2.417%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:08.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:08 smithi110 ceph-mon[221102]: pgmap v372: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 7029/290832 objects degraded (2.417%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T23:09:08.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:08 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 7029/290832 objects degraded (2.417%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:08.807 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 69 16 101011 100995 91.4261 58.25 0.0140495 0.0109328 2023-09-23T23:09:09.807 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 70 16 101878 101862 90.8943 54.1875 0.0135051 0.0109975 2023-09-23T23:09:10.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:10 smithi138 ceph-mon[193606]: pgmap v373: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 6484/298953 objects degraded (2.169%); 7.1 MiB/s, 114 objects/s recovering 2023-09-23T23:09:10.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:10 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3395392851' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:10.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:10 smithi110 ceph-mon[221102]: pgmap v373: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 6484/298953 objects degraded (2.169%); 7.1 MiB/s, 114 objects/s recovering 2023-09-23T23:09:10.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:10 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3395392851' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:10.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:10 smithi110 ceph-mon[212939]: pgmap v373: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 6484/298953 objects degraded (2.169%); 7.1 MiB/s, 114 objects/s recovering 2023-09-23T23:09:10.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:10 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3395392851' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:10.807 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 71 16 102815 102799 90.4391 58.5625 0.0202182 0.0110523 2023-09-23T23:09:11.807 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 72 16 103718 103702 89.967 56.4375 0.019486 0.0111089 2023-09-23T23:09:12.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:12 smithi138 ceph-mon[193606]: pgmap v374: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 6484/303984 objects degraded (2.133%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T23:09:12.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:12 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1087758507' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:12.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:12 smithi110 ceph-mon[212939]: pgmap v374: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 6484/303984 objects degraded (2.133%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T23:09:12.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:12 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1087758507' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:12.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:12 smithi110 ceph-mon[221102]: pgmap v374: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 6484/303984 objects degraded (2.133%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T23:09:12.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:12 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1087758507' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:12.807 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 73 16 104582 104566 89.4745 54 0.0158959 0.0111716 2023-09-23T23:09:13.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:13.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:13 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 6484/303984 objects degraded (2.133%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:13.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:13.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:13 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 6484/303984 objects degraded (2.133%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:13.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:13.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:13 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 6484/303984 objects degraded (2.133%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:13.807 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 74 16 105486 105470 89.0288 56.5 0.016248 0.0112276 2023-09-23T23:09:14.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:14 smithi138 ceph-mon[193606]: pgmap v375: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 6073/307374 objects degraded (1.976%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:09:14.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:14 smithi110 ceph-mon[221102]: pgmap v375: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 6073/307374 objects degraded (1.976%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:09:14.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:14 smithi110 ceph-mon[212939]: pgmap v375: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 6073/307374 objects degraded (1.976%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:09:14.807 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 75 16 106295 106279 88.5161 50.5625 0.0128192 0.011292 2023-09-23T23:09:15.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:15 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2473779898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:15.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:15 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2473779898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:15.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:15 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2473779898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:15.810 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 76 16 107169 107153 88.0672 54.625 0.0155639 0.0113507 2023-09-23T23:09:16.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:16 smithi138 ceph-mon[193606]: pgmap v376: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 6073/317430 objects degraded (1.913%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T23:09:16.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:16 smithi110 ceph-mon[212939]: pgmap v376: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 6073/317430 objects degraded (1.913%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T23:09:16.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:16 smithi110 ceph-mon[221102]: pgmap v376: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 6073/317430 objects degraded (1.913%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T23:09:16.810 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 77 16 108061 108045 87.6476 55.75 0.0240066 0.0114046 2023-09-23T23:09:17.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:09:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:09:16] "GET /metrics HTTP/1.1" 200 36427 "" "Prometheus/2.43.0" 2023-09-23T23:09:17.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:17 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1638228072' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:17.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:17 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1638228072' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:17.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:17 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1638228072' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:17.815 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 78 16 108922 108906 87.2093 53.8125 0.0194951 0.0114618 2023-09-23T23:09:18.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:18 smithi138 ceph-mon[193606]: pgmap v377: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 738 op/s; 6073/317430 objects degraded (1.913%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T23:09:18.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:18 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 6073/317430 objects degraded (1.913%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:18.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:18 smithi138 ceph-mon[193606]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T23:09:18.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:18 smithi138 ceph-mon[193606]: failed to encode map e135 with expected crc 2023-09-23T23:09:18.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:18 smithi110 ceph-mon[212939]: pgmap v377: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 738 op/s; 6073/317430 objects degraded (1.913%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T23:09:18.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:18 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 6073/317430 objects degraded (1.913%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:18.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:18 smithi110 ceph-mon[212939]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T23:09:18.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:18 smithi110 ceph-mon[212939]: failed to encode map e135 with expected crc 2023-09-23T23:09:18.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:18 smithi110 ceph-mon[221102]: pgmap v377: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 738 op/s; 6073/317430 objects degraded (1.913%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T23:09:18.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:18 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 6073/317430 objects degraded (1.913%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:18.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:18 smithi110 ceph-mon[221102]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T23:09:18.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:18 smithi110 ceph-mon[221102]: failed to encode map e135 with expected crc 2023-09-23T23:09:18.815 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 79 16 109608 109592 86.6483 42.875 0.00466646 0.0114712 2023-09-23T23:09:19.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:19 smithi138 ceph-mon[193606]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:19 smithi138 ceph-mon[193606]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:19 smithi138 ceph-mon[193606]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:19 smithi138 ceph-mon[193606]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:19 smithi138 ceph-mon[193606]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:19 smithi138 ceph-mon[193606]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:19 smithi138 ceph-mon[193606]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T23:09:19.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:19 smithi138 ceph-mon[193606]: failed to encode map e136 with expected crc 2023-09-23T23:09:19.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:19 smithi110 ceph-mon[212939]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:19 smithi110 ceph-mon[212939]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:19 smithi110 ceph-mon[212939]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:19 smithi110 ceph-mon[212939]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:19 smithi110 ceph-mon[212939]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:19 smithi110 ceph-mon[212939]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:19 smithi110 ceph-mon[212939]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T23:09:19.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:19 smithi110 ceph-mon[212939]: failed to encode map e136 with expected crc 2023-09-23T23:09:19.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:19 smithi110 ceph-mon[221102]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:19 smithi110 ceph-mon[221102]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:19 smithi110 ceph-mon[221102]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:19 smithi110 ceph-mon[221102]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:19 smithi110 ceph-mon[221102]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:19 smithi110 ceph-mon[221102]: failed to encode map e135 with expected crc 2023-09-23T23:09:19.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:19 smithi110 ceph-mon[221102]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T23:09:19.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:19 smithi110 ceph-mon[221102]: failed to encode map e136 with expected crc 2023-09-23T23:09:19.815 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:09:19.815137+0000 min lat: 0.00124287 max lat: 2.12462 avg lat: 0.0116189 2023-09-23T23:09:19.815 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:09:19.815 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 80 16 110199 110183 86.0272 36.9375 0.0175212 0.0116189 2023-09-23T23:09:20.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:20 smithi138 ceph-mon[193606]: failed to encode map e136 with expected crc 2023-09-23T23:09:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:20 smithi138 ceph-mon[193606]: failed to encode map e136 with expected crc 2023-09-23T23:09:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:20 smithi138 ceph-mon[193606]: failed to encode map e136 with expected crc 2023-09-23T23:09:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:20 smithi138 ceph-mon[193606]: failed to encode map e136 with expected crc 2023-09-23T23:09:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:20 smithi138 ceph-mon[193606]: pgmap v380: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 5485/324945 objects degraded (1.688%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T23:09:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:20 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3290126884' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:20.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:20 smithi110 ceph-mon[212939]: failed to encode map e136 with expected crc 2023-09-23T23:09:20.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:20 smithi110 ceph-mon[212939]: failed to encode map e136 with expected crc 2023-09-23T23:09:20.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:20 smithi110 ceph-mon[212939]: failed to encode map e136 with expected crc 2023-09-23T23:09:20.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:20 smithi110 ceph-mon[212939]: failed to encode map e136 with expected crc 2023-09-23T23:09:20.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:20 smithi110 ceph-mon[212939]: pgmap v380: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 5485/324945 objects degraded (1.688%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T23:09:20.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:20 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3290126884' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:20.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:20 smithi110 ceph-mon[221102]: failed to encode map e136 with expected crc 2023-09-23T23:09:20.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:20 smithi110 ceph-mon[221102]: failed to encode map e136 with expected crc 2023-09-23T23:09:20.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:20 smithi110 ceph-mon[221102]: failed to encode map e136 with expected crc 2023-09-23T23:09:20.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:20 smithi110 ceph-mon[221102]: failed to encode map e136 with expected crc 2023-09-23T23:09:20.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:20 smithi110 ceph-mon[221102]: pgmap v380: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 5485/324945 objects degraded (1.688%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T23:09:20.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:20 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3290126884' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:20.815 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 81 16 111085 111069 85.6489 55.375 0.0177697 0.0116709 2023-09-23T23:09:21.815 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 82 16 111968 111952 85.2776 55.1875 0.016705 0.0117211 2023-09-23T23:09:21.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:09:21 smithi110 conmon[198813]: 2023-09-23T23:09:21.484+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:09:22.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:22 smithi138 ceph-mon[193606]: pgmap v381: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s; 5485/329433 objects degraded (1.665%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:09:22.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:22 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:22.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:22 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:22.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:22 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:09:22.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:22 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1303999831' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:22.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:22 smithi110 ceph-mon[212939]: pgmap v381: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s; 5485/329433 objects degraded (1.665%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:09:22.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:22 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:22.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:22 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:22.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:22 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:09:22.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:22 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1303999831' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:22.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:22 smithi110 ceph-mon[221102]: pgmap v381: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s; 5485/329433 objects degraded (1.665%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:09:22.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:22 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:22.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:22 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:22.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:22 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:09:22.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:22 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1303999831' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:22.817 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 83 16 112822 112806 84.8919 53.375 0.0232994 0.0117753 2023-09-23T23:09:23.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:23 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 5485/329433 objects degraded (1.665%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:23.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:23 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 5485/329433 objects degraded (1.665%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:23.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:23 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 5485/329433 objects degraded (1.665%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:23.820 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 84 16 113721 113705 84.5468 56.1875 0.0183736 0.0118231 2023-09-23T23:09:24.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:24 smithi110 ceph-mon[221102]: pgmap v382: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 611 op/s; 5485/332118 objects degraded (1.652%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:09:24.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:24 smithi110 ceph-mon[212939]: pgmap v382: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 611 op/s; 5485/332118 objects degraded (1.652%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:09:24.820 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 85 16 114563 114547 84.1714 52.625 0.0188474 0.011876 2023-09-23T23:09:24.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:24 smithi138 ceph-mon[193606]: pgmap v382: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 611 op/s; 5485/332118 objects degraded (1.652%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:09:25.820 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 86 16 115421 115405 83.8164 53.625 0.0118471 0.0119262 2023-09-23T23:09:25.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:25 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3938208798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:25.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:25 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3938208798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:25 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3938208798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:26.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:26 smithi110 ceph-mon[212939]: pgmap v383: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 5138/342087 objects degraded (1.502%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T23:09:26.804 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:09:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:09:26] "GET /metrics HTTP/1.1" 200 36427 "" "Prometheus/2.43.0" 2023-09-23T23:09:26.805 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:26 smithi110 ceph-mon[221102]: pgmap v383: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 5138/342087 objects degraded (1.502%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T23:09:26.821 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 87 16 116304 116288 83.4874 55.1875 0.00939304 0.0119727 2023-09-23T23:09:26.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:26 smithi138 ceph-mon[193606]: pgmap v383: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 5138/342087 objects degraded (1.502%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T23:09:27.825 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 88 16 117147 117131 83.1351 52.6875 0.0149905 0.0120234 2023-09-23T23:09:28.825 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 89 16 118021 118005 82.8128 54.625 0.0140226 0.0120708 2023-09-23T23:09:28.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:28 smithi138 ceph-mon[193606]: pgmap v384: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 5138/342087 objects degraded (1.502%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T23:09:28.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:28 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/473316930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:28.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:28.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:28 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 5138/342087 objects degraded (1.502%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:28.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:28 smithi110 ceph-mon[212939]: pgmap v384: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 5138/342087 objects degraded (1.502%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T23:09:28.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:28 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/473316930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:28.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:28.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:28 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 5138/342087 objects degraded (1.502%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:28.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:28 smithi110 ceph-mon[221102]: pgmap v384: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 5138/342087 objects degraded (1.502%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T23:09:28.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:28 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/473316930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:28.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:28.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:28 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 5138/342087 objects degraded (1.502%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:29.826 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total time run: 90.0175 2023-09-23T23:09:29.826 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total writes made: 118815 2023-09-23T23:09:29.826 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Write size: 65536 2023-09-23T23:09:29.826 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object size: 65536 2023-09-23T23:09:29.826 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Bandwidth (MB/sec): 82.4944 2023-09-23T23:09:29.826 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Bandwidth: 44.7853 2023-09-23T23:09:29.826 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max bandwidth (MB/sec): 250.125 2023-09-23T23:09:29.826 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min bandwidth (MB/sec): 0 2023-09-23T23:09:29.826 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average IOPS: 1319 2023-09-23T23:09:29.826 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev IOPS: 716.564 2023-09-23T23:09:29.827 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max IOPS: 4002 2023-09-23T23:09:29.827 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min IOPS: 0 2023-09-23T23:09:29.827 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average Latency(s): 0.012118 2023-09-23T23:09:29.827 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Latency(s): 0.0313514 2023-09-23T23:09:29.827 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max latency(s): 2.12462 2023-09-23T23:09:29.827 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min latency(s): 0.00124287 2023-09-23T23:09:29.832 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T23:09:30.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:30 smithi138 ceph-mon[193606]: pgmap v385: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s; 4770/350283 objects degraded (1.362%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T23:09:30.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:30 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/134463271' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:30.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:30 smithi110 ceph-mon[212939]: pgmap v385: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s; 4770/350283 objects degraded (1.362%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T23:09:30.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:30 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/134463271' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:30.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:30 smithi110 ceph-mon[221102]: pgmap v385: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s; 4770/350283 objects degraded (1.362%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T23:09:30.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:30 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/134463271' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:32.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:32 smithi138 ceph-mon[193606]: pgmap v386: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 4770/354630 objects degraded (1.345%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T23:09:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:32 smithi110 ceph-mon[212939]: pgmap v386: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 4770/354630 objects degraded (1.345%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T23:09:32.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:32 smithi110 ceph-mon[221102]: pgmap v386: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 4770/354630 objects degraded (1.345%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T23:09:33.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:33 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2617312431' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:33.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:33 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 4770/354630 objects degraded (1.345%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:33.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:33 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2617312431' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:33.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:33 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 4770/354630 objects degraded (1.345%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:33.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:33 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2617312431' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:33.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:33 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 4770/354630 objects degraded (1.345%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:34.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:34 smithi138 ceph-mon[193606]: pgmap v387: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 46 MiB/s wr, 798 op/s; 4770/353550 objects degraded (1.349%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T23:09:34.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:34 smithi110 ceph-mon[212939]: pgmap v387: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 46 MiB/s wr, 798 op/s; 4770/353550 objects degraded (1.349%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T23:09:34.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:34 smithi110 ceph-mon[221102]: pgmap v387: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 46 MiB/s wr, 798 op/s; 4770/353550 objects degraded (1.349%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T23:09:35.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:35 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2201006343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:35.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:35 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2201006343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:35.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:35 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2201006343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:36.800 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:36 smithi110 ceph-mon[212939]: pgmap v388: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 993 op/s; 4386/345138 objects degraded (1.271%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T23:09:36.801 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:36 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:36.801 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:36 smithi110 ceph-mon[221102]: pgmap v388: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 993 op/s; 4386/345138 objects degraded (1.271%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T23:09:36.801 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:36 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:36.801 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:09:36 smithi110 conmon[198813]: 2023-09-23T23:09:36.501+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:09:36.801 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:09:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:09:36] "GET /metrics HTTP/1.1" 200 36427 "" "Prometheus/2.43.0" 2023-09-23T23:09:36.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:36 smithi138 ceph-mon[193606]: pgmap v388: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 993 op/s; 4386/345138 objects degraded (1.271%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T23:09:36.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:36 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:37.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:37 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:37.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:37 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:09:37.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:37 smithi138 ceph-mon[193606]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T23:09:37.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:37 smithi138 ceph-mon[193606]: failed to encode map e137 with expected crc 2023-09-23T23:09:37.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:37 smithi138 ceph-mon[193606]: failed to encode map e137 with expected crc 2023-09-23T23:09:37.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:37 smithi138 ceph-mon[193606]: failed to encode map e137 with expected crc 2023-09-23T23:09:37.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:37 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3189079380' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:37.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:37 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:37.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:37 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:09:37.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:37 smithi110 ceph-mon[212939]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T23:09:37.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:37 smithi110 ceph-mon[212939]: failed to encode map e137 with expected crc 2023-09-23T23:09:37.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:37 smithi110 ceph-mon[212939]: failed to encode map e137 with expected crc 2023-09-23T23:09:37.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:37 smithi110 ceph-mon[212939]: failed to encode map e137 with expected crc 2023-09-23T23:09:37.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:37 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3189079380' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:37.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:37 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:09:37.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:37 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:09:37.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:37 smithi110 ceph-mon[221102]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T23:09:37.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:37 smithi110 ceph-mon[221102]: failed to encode map e137 with expected crc 2023-09-23T23:09:37.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:37 smithi110 ceph-mon[221102]: failed to encode map e137 with expected crc 2023-09-23T23:09:37.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:37 smithi110 ceph-mon[221102]: failed to encode map e137 with expected crc 2023-09-23T23:09:37.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:37 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3189079380' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:38.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:38 smithi138 ceph-mon[193606]: pgmap v390: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 860 op/s; 4386/345138 objects degraded (1.271%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T23:09:38.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:38 smithi138 ceph-mon[193606]: failed to encode map e138 with expected crc 2023-09-23T23:09:38.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:38 smithi138 ceph-mon[193606]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T23:09:38.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:38 smithi138 ceph-mon[193606]: failed to encode map e138 with expected crc 2023-09-23T23:09:38.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:38 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 4386/345138 objects degraded (1.271%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:38.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:38 smithi110 ceph-mon[221102]: pgmap v390: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 860 op/s; 4386/345138 objects degraded (1.271%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T23:09:38.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:38 smithi110 ceph-mon[221102]: failed to encode map e138 with expected crc 2023-09-23T23:09:38.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:38 smithi110 ceph-mon[221102]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T23:09:38.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:38 smithi110 ceph-mon[221102]: failed to encode map e138 with expected crc 2023-09-23T23:09:38.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:38 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 4386/345138 objects degraded (1.271%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:38 smithi110 ceph-mon[212939]: pgmap v390: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 860 op/s; 4386/345138 objects degraded (1.271%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T23:09:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:38 smithi110 ceph-mon[212939]: failed to encode map e138 with expected crc 2023-09-23T23:09:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:38 smithi110 ceph-mon[212939]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T23:09:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:38 smithi110 ceph-mon[212939]: failed to encode map e138 with expected crc 2023-09-23T23:09:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:38 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 4386/345138 objects degraded (1.271%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:39.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:39 smithi138 ceph-mon[193606]: failed to encode map e138 with expected crc 2023-09-23T23:09:39.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:39 smithi110 ceph-mon[212939]: failed to encode map e138 with expected crc 2023-09-23T23:09:39.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:39 smithi110 ceph-mon[221102]: failed to encode map e138 with expected crc 2023-09-23T23:09:40.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:40 smithi138 ceph-mon[193606]: pgmap v392: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 4.5 MiB/s wr, 814 op/s; 4246/338562 objects degraded (1.254%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:09:40.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:40 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3229107691' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:40.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:40 smithi110 ceph-mon[221102]: pgmap v392: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 4.5 MiB/s wr, 814 op/s; 4246/338562 objects degraded (1.254%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:09:40.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:40 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3229107691' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:40.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:40 smithi110 ceph-mon[212939]: pgmap v392: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 4.5 MiB/s wr, 814 op/s; 4246/338562 objects degraded (1.254%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:09:40.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:40 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3229107691' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:42.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:42 smithi138 ceph-mon[193606]: pgmap v393: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 1.7 MiB/s wr, 843 op/s; 3869/334596 objects degraded (1.156%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T23:09:42.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:42 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3022119243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:42.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:42 smithi110 ceph-mon[212939]: pgmap v393: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 1.7 MiB/s wr, 843 op/s; 3869/334596 objects degraded (1.156%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T23:09:42.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:42 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3022119243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:42.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:42 smithi110 ceph-mon[221102]: pgmap v393: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 1.7 MiB/s wr, 843 op/s; 3869/334596 objects degraded (1.156%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T23:09:42.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:42 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3022119243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:43.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:43.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:43 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 3869/334596 objects degraded (1.156%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:43.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:43.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:43 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 3869/334596 objects degraded (1.156%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:43.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:43.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:43 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 3869/334596 objects degraded (1.156%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:09:44.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:44 smithi138 ceph-mon[193606]: pgmap v394: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 3869/331362 objects degraded (1.168%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T23:09:44.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:44 smithi110 ceph-mon[212939]: pgmap v394: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 3869/331362 objects degraded (1.168%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T23:09:44.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:44 smithi110 ceph-mon[221102]: pgmap v394: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 3869/331362 objects degraded (1.168%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T23:09:45.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:45 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2797782565' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:45.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:45 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2797782565' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:45.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:45 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2797782565' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:46.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:46 smithi138 ceph-mon[193606]: pgmap v395: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 3554/322191 objects degraded (1.103%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T23:09:46.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:09:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:09:46] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.43.0" 2023-09-23T23:09:46.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:46 smithi110 ceph-mon[212939]: pgmap v395: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 3554/322191 objects degraded (1.103%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T23:09:46.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:46 smithi110 ceph-mon[221102]: pgmap v395: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 3554/322191 objects degraded (1.103%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T23:09:48.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:48 smithi138 ceph-mon[193606]: pgmap v396: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 3554/322191 objects degraded (1.103%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T23:09:48.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:48 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3453202878' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:48.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:48 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 3554/322191 objects degraded (1.103%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:48.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:48 smithi110 ceph-mon[212939]: pgmap v396: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 3554/322191 objects degraded (1.103%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T23:09:48.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:48 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3453202878' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:48.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:48 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 3554/322191 objects degraded (1.103%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:48.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:48 smithi110 ceph-mon[221102]: pgmap v396: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 3554/322191 objects degraded (1.103%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T23:09:48.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:48 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3453202878' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:48.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:48 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 3554/322191 objects degraded (1.103%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:50.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:50 smithi110 ceph-mon[212939]: pgmap v397: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 3554/314610 objects degraded (1.130%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:09:50.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:50 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2906416136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:50.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:50 smithi110 ceph-mon[221102]: pgmap v397: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 3554/314610 objects degraded (1.130%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:09:50.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:50 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2906416136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:51.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:50 smithi138 ceph-mon[193606]: pgmap v397: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 3554/314610 objects degraded (1.130%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:09:51.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:50 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2906416136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:52.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:51 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:09:52.225 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:51 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:09:52.226 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:51 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:09:53.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:52 smithi138 ceph-mon[193606]: pgmap v398: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 3213/310023 objects degraded (1.036%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T23:09:53.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:52 smithi110 ceph-mon[212939]: pgmap v398: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 3213/310023 objects degraded (1.036%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T23:09:53.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:52 smithi110 ceph-mon[221102]: pgmap v398: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 3213/310023 objects degraded (1.036%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T23:09:54.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:53 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3701193103' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:54.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:53 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 3213/310023 objects degraded (1.036%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:54.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:53 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3701193103' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:54.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:53 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 3213/310023 objects degraded (1.036%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:54.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:53 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3701193103' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:54.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:53 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 3213/310023 objects degraded (1.036%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:55.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:54 smithi138 ceph-mon[193606]: pgmap v399: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 3213/307125 objects degraded (1.046%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T23:09:55.225 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:54 smithi110 ceph-mon[212939]: pgmap v399: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 3213/307125 objects degraded (1.046%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T23:09:55.225 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:54 smithi110 ceph-mon[221102]: pgmap v399: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 3213/307125 objects degraded (1.046%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T23:09:56.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:55 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3346224448' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:56.603 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:55 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3346224448' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:56.603 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:55 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3346224448' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:57.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:56 smithi138 ceph-mon[193606]: pgmap v400: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 2856/298473 objects degraded (0.957%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T23:09:57.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:56 smithi138 ceph-mon[193606]: failed to encode map e139 with expected crc 2023-09-23T23:09:57.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:56 smithi138 ceph-mon[193606]: failed to encode map e139 with expected crc 2023-09-23T23:09:57.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:56 smithi138 ceph-mon[193606]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T23:09:57.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:56 smithi138 ceph-mon[193606]: failed to encode map e139 with expected crc 2023-09-23T23:09:57.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:56 smithi138 ceph-mon[193606]: failed to encode map e139 with expected crc 2023-09-23T23:09:57.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:56 smithi138 ceph-mon[193606]: failed to encode map e139 with expected crc 2023-09-23T23:09:57.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:56 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:09:57.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:56 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:09:57.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[212939]: pgmap v400: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 2856/298473 objects degraded (0.957%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T23:09:57.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[212939]: failed to encode map e139 with expected crc 2023-09-23T23:09:57.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[212939]: failed to encode map e139 with expected crc 2023-09-23T23:09:57.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[212939]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T23:09:57.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[212939]: failed to encode map e139 with expected crc 2023-09-23T23:09:57.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[212939]: failed to encode map e139 with expected crc 2023-09-23T23:09:57.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[212939]: failed to encode map e139 with expected crc 2023-09-23T23:09:57.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:09:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:09:57.204 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:09:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:09:56] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.43.0" 2023-09-23T23:09:57.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[221102]: pgmap v400: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 2856/298473 objects degraded (0.957%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T23:09:57.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[221102]: failed to encode map e139 with expected crc 2023-09-23T23:09:57.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[221102]: failed to encode map e139 with expected crc 2023-09-23T23:09:57.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[221102]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T23:09:57.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[221102]: failed to encode map e139 with expected crc 2023-09-23T23:09:57.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[221102]: failed to encode map e139 with expected crc 2023-09-23T23:09:57.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[221102]: failed to encode map e139 with expected crc 2023-09-23T23:09:57.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:09:57.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:56 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:09:58.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:57 smithi138 ceph-mon[193606]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:57 smithi138 ceph-mon[193606]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T23:09:58.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:57 smithi138 ceph-mon[193606]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:57 smithi138 ceph-mon[193606]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:57 smithi138 ceph-mon[193606]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:57 smithi138 ceph-mon[193606]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:57 smithi138 ceph-mon[193606]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:57 smithi138 ceph-mon[193606]: pgmap v403: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2856/298473 objects degraded (0.957%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T23:09:58.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:57 smithi110 ceph-mon[212939]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:57 smithi110 ceph-mon[212939]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T23:09:58.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:57 smithi110 ceph-mon[212939]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:57 smithi110 ceph-mon[212939]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:57 smithi110 ceph-mon[212939]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:57 smithi110 ceph-mon[212939]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:57 smithi110 ceph-mon[212939]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:57 smithi110 ceph-mon[212939]: pgmap v403: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2856/298473 objects degraded (0.957%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T23:09:58.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:57 smithi110 ceph-mon[221102]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:57 smithi110 ceph-mon[221102]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T23:09:58.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:57 smithi110 ceph-mon[221102]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:57 smithi110 ceph-mon[221102]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:57 smithi110 ceph-mon[221102]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:57 smithi110 ceph-mon[221102]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:57 smithi110 ceph-mon[221102]: failed to encode map e140 with expected crc 2023-09-23T23:09:58.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:57 smithi110 ceph-mon[221102]: pgmap v403: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2856/298473 objects degraded (0.957%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T23:09:59.196 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:59.196 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:58 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2977227631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:59.196 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:09:58 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2856/298473 objects degraded (0.957%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:59.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:59.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:58 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2977227631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:59.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:09:58 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2856/298473 objects degraded (0.957%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:59.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:59.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:58 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2977227631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:09:59.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:09:58 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2856/298473 objects degraded (0.957%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:10:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:00 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:00 smithi110 ceph-mon[212939]: pgmap v404: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 2856/292476 objects degraded (0.976%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:10:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:00 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:00 smithi110 ceph-mon[212939]: Health detail: HEALTH_WARN Degraded data redundancy: 2856/298473 objects degraded (0.957%), 2 pgs degraded, 3 pgs undersized 2023-09-23T23:10:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:00 smithi110 ceph-mon[212939]: [WRN] PG_DEGRADED: Degraded data redundancy: 2856/298473 objects degraded (0.957%), 2 pgs degraded, 3 pgs undersized 2023-09-23T23:10:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:00 smithi110 ceph-mon[212939]: pg 8.1 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,0] 2023-09-23T23:10:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:00 smithi110 ceph-mon[212939]: pg 8.e is stuck undersized for 71s, current state active+undersized+remapped, last acting [0,4] 2023-09-23T23:10:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:00 smithi110 ceph-mon[212939]: pg 8.f is stuck undersized for 71s, current state active+recovering+undersized+degraded+remapped, last acting [3,1] 2023-09-23T23:10:00.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:00 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:00.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:00 smithi110 ceph-mon[221102]: pgmap v404: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 2856/292476 objects degraded (0.976%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:10:00.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:00 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:00.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:00 smithi110 ceph-mon[221102]: Health detail: HEALTH_WARN Degraded data redundancy: 2856/298473 objects degraded (0.957%), 2 pgs degraded, 3 pgs undersized 2023-09-23T23:10:00.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:00 smithi110 ceph-mon[221102]: [WRN] PG_DEGRADED: Degraded data redundancy: 2856/298473 objects degraded (0.957%), 2 pgs degraded, 3 pgs undersized 2023-09-23T23:10:00.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:00 smithi110 ceph-mon[221102]: pg 8.1 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,0] 2023-09-23T23:10:00.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:00 smithi110 ceph-mon[221102]: pg 8.e is stuck undersized for 71s, current state active+undersized+remapped, last acting [0,4] 2023-09-23T23:10:00.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:00 smithi110 ceph-mon[221102]: pg 8.f is stuck undersized for 71s, current state active+recovering+undersized+degraded+remapped, last acting [3,1] 2023-09-23T23:10:00.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:00 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:00.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:00 smithi138 ceph-mon[193606]: pgmap v404: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 2856/292476 objects degraded (0.976%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:10:00.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:00 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:00.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:00 smithi138 ceph-mon[193606]: Health detail: HEALTH_WARN Degraded data redundancy: 2856/298473 objects degraded (0.957%), 2 pgs degraded, 3 pgs undersized 2023-09-23T23:10:00.697 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:00 smithi138 ceph-mon[193606]: [WRN] PG_DEGRADED: Degraded data redundancy: 2856/298473 objects degraded (0.957%), 2 pgs degraded, 3 pgs undersized 2023-09-23T23:10:00.697 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:00 smithi138 ceph-mon[193606]: pg 8.1 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,0] 2023-09-23T23:10:00.697 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:00 smithi138 ceph-mon[193606]: pg 8.e is stuck undersized for 71s, current state active+undersized+remapped, last acting [0,4] 2023-09-23T23:10:00.697 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:00 smithi138 ceph-mon[193606]: pg 8.f is stuck undersized for 71s, current state active+recovering+undersized+degraded+remapped, last acting [3,1] 2023-09-23T23:10:00.953 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:10:00 smithi110 conmon[198813]: 2023-09-23T23:10:00.526+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:10:01.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:01 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:10:01.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:01 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:10:01.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:01 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:01.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:01 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:10:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:01 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:10:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:01 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:10:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:01 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/742665195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:01 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:10:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:01 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:01 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:01 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:10:01.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:10:01.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:10:01.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:01.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:10:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:10:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:10:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/742665195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:10:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:10:01.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:10:01.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:10:01.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:01.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:10:01.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:10:01.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:10:01.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/742665195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:01.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:10:01.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:01.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:01.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:01 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:10:02.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:02 smithi138 ceph-mon[193606]: pgmap v405: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 2460/288894 objects degraded (0.852%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T23:10:02.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:02 smithi110 ceph-mon[221102]: pgmap v405: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 2460/288894 objects degraded (0.852%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T23:10:02.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:02 smithi110 ceph-mon[212939]: pgmap v405: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 2460/288894 objects degraded (0.852%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T23:10:03.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:03 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2460/288894 objects degraded (0.852%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:10:03.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:03 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2654239767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:03.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:03 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2460/288894 objects degraded (0.852%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:10:03.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:03 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2654239767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:03.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:03 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2460/288894 objects degraded (0.852%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:10:03.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:03 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2654239767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:04.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:04 smithi138 ceph-mon[193606]: pgmap v406: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 2460/285987 objects degraded (0.860%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:10:04.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:04 smithi110 ceph-mon[212939]: pgmap v406: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 2460/285987 objects degraded (0.860%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:10:04.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:04 smithi110 ceph-mon[221102]: pgmap v406: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 2460/285987 objects degraded (0.860%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:10:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:06 smithi138 ceph-mon[193606]: pgmap v407: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 2150/277317 objects degraded (0.775%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:10:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:06 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2829765469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:06.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:06 smithi110 ceph-mon[212939]: pgmap v407: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 2150/277317 objects degraded (0.775%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:10:06.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:06 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2829765469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:06.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:06 smithi110 ceph-mon[221102]: pgmap v407: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 2150/277317 objects degraded (0.775%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:10:06.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:06 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2829765469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:07.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:10:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:10:06] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.43.0" 2023-09-23T23:10:08.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:08 smithi138 ceph-mon[193606]: pgmap v408: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 2150/277317 objects degraded (0.775%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T23:10:08.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:08 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2150/277317 objects degraded (0.775%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:10:08.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:08 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1943053147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:08.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:08 smithi110 ceph-mon[212939]: pgmap v408: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 2150/277317 objects degraded (0.775%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T23:10:08.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:08 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2150/277317 objects degraded (0.775%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:10:08.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:08 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1943053147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:08.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:08 smithi110 ceph-mon[221102]: pgmap v408: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 2150/277317 objects degraded (0.775%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T23:10:08.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:08 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2150/277317 objects degraded (0.775%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:10:08.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:08 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1943053147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:10.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:10 smithi138 ceph-mon[193606]: pgmap v409: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 2150/272097 objects degraded (0.790%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T23:10:10.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:10 smithi110 ceph-mon[212939]: pgmap v409: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 2150/272097 objects degraded (0.790%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T23:10:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:10 smithi110 ceph-mon[221102]: pgmap v409: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 2150/272097 objects degraded (0.790%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T23:10:11.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:11 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/669840155' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:11.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:11 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/669840155' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:11.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:11 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/669840155' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:12.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:12 smithi138 ceph-mon[193606]: pgmap v410: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 1809/265323 objects degraded (0.682%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T23:10:12.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:12 smithi110 ceph-mon[221102]: pgmap v410: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 1809/265323 objects degraded (0.682%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T23:10:12.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:12 smithi110 ceph-mon[212939]: pgmap v410: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 1809/265323 objects degraded (0.682%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T23:10:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:13 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1809/265323 objects degraded (0.682%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:10:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:13 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/462687258' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:13.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:13.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:13 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1809/265323 objects degraded (0.682%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:10:13.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:13 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/462687258' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:13.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:13.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:13 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1809/265323 objects degraded (0.682%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:10:13.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:13 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/462687258' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:14.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:14 smithi138 ceph-mon[193606]: pgmap v411: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 1809/262434 objects degraded (0.689%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T23:10:14.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:14 smithi110 ceph-mon[221102]: pgmap v411: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 1809/262434 objects degraded (0.689%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T23:10:14.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:14 smithi110 ceph-mon[212939]: pgmap v411: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 1809/262434 objects degraded (0.689%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T23:10:15.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:10:15 smithi110 conmon[198813]: 2023-09-23T23:10:15.562+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:10:16.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:16 smithi138 ceph-mon[193606]: pgmap v412: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 1442/253359 objects degraded (0.569%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T23:10:16.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:16 smithi138 ceph-mon[193606]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T23:10:16.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:16 smithi138 ceph-mon[193606]: failed to encode map e141 with expected crc 2023-09-23T23:10:16.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:16 smithi138 ceph-mon[193606]: failed to encode map e141 with expected crc 2023-09-23T23:10:16.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:16 smithi138 ceph-mon[193606]: failed to encode map e141 with expected crc 2023-09-23T23:10:16.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:16 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:16.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:16 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:16.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:16 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:10:16.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:16 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2707267417' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:16.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[212939]: pgmap v412: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 1442/253359 objects degraded (0.569%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T23:10:16.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[212939]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T23:10:16.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[212939]: failed to encode map e141 with expected crc 2023-09-23T23:10:16.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[212939]: failed to encode map e141 with expected crc 2023-09-23T23:10:16.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[212939]: failed to encode map e141 with expected crc 2023-09-23T23:10:16.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:16.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:16.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:10:16.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2707267417' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[221102]: pgmap v412: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 1442/253359 objects degraded (0.569%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T23:10:16.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[221102]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T23:10:16.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[221102]: failed to encode map e141 with expected crc 2023-09-23T23:10:16.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[221102]: failed to encode map e141 with expected crc 2023-09-23T23:10:16.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[221102]: failed to encode map e141 with expected crc 2023-09-23T23:10:16.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:16.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:16.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T23:10:16.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:16 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2707267417' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:17.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:10:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:10:16] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.43.0" 2023-09-23T23:10:17.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:17 smithi138 ceph-mon[193606]: failed to encode map e142 with expected crc 2023-09-23T23:10:17.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:17 smithi138 ceph-mon[193606]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T23:10:17.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:17 smithi138 ceph-mon[193606]: failed to encode map e142 with expected crc 2023-09-23T23:10:17.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:17 smithi138 ceph-mon[193606]: failed to encode map e142 with expected crc 2023-09-23T23:10:17.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:17 smithi138 ceph-mon[193606]: failed to encode map e142 with expected crc 2023-09-23T23:10:17.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:17 smithi138 ceph-mon[193606]: failed to encode map e142 with expected crc 2023-09-23T23:10:17.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:17 smithi110 ceph-mon[212939]: failed to encode map e142 with expected crc 2023-09-23T23:10:17.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:17 smithi110 ceph-mon[212939]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T23:10:17.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:17 smithi110 ceph-mon[212939]: failed to encode map e142 with expected crc 2023-09-23T23:10:17.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:17 smithi110 ceph-mon[212939]: failed to encode map e142 with expected crc 2023-09-23T23:10:17.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:17 smithi110 ceph-mon[212939]: failed to encode map e142 with expected crc 2023-09-23T23:10:17.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:17 smithi110 ceph-mon[212939]: failed to encode map e142 with expected crc 2023-09-23T23:10:17.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:17 smithi110 ceph-mon[221102]: failed to encode map e142 with expected crc 2023-09-23T23:10:17.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:17 smithi110 ceph-mon[221102]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T23:10:17.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:17 smithi110 ceph-mon[221102]: failed to encode map e142 with expected crc 2023-09-23T23:10:17.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:17 smithi110 ceph-mon[221102]: failed to encode map e142 with expected crc 2023-09-23T23:10:17.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:17 smithi110 ceph-mon[221102]: failed to encode map e142 with expected crc 2023-09-23T23:10:17.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:17 smithi110 ceph-mon[221102]: failed to encode map e142 with expected crc 2023-09-23T23:10:18.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:18 smithi138 ceph-mon[193606]: pgmap v415: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 1442/253359 objects degraded (0.569%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T23:10:18.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:18 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1442/253359 objects degraded (0.569%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:10:18.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/513096392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:18.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:18 smithi110 ceph-mon[221102]: pgmap v415: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 1442/253359 objects degraded (0.569%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T23:10:18.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:18 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1442/253359 objects degraded (0.569%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:10:18.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/513096392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:18.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:18 smithi110 ceph-mon[212939]: pgmap v415: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 1442/253359 objects degraded (0.569%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T23:10:18.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:18 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1442/253359 objects degraded (0.569%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:10:18.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/513096392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:20 smithi138 ceph-mon[193606]: pgmap v416: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 1123/248976 objects degraded (0.451%); 5.4 MiB/s, 85 objects/s recovering 2023-09-23T23:10:20.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:20 smithi110 ceph-mon[212939]: pgmap v416: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 1123/248976 objects degraded (0.451%); 5.4 MiB/s, 85 objects/s recovering 2023-09-23T23:10:20.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:20 smithi110 ceph-mon[221102]: pgmap v416: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 1123/248976 objects degraded (0.451%); 5.4 MiB/s, 85 objects/s recovering 2023-09-23T23:10:21.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:21 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/14690106' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:21.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:21 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/14690106' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:21.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:21 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/14690106' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:22.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:22 smithi138 ceph-mon[193606]: pgmap v417: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 1117/243468 objects degraded (0.459%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T23:10:22.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:22 smithi110 ceph-mon[221102]: pgmap v417: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 1117/243468 objects degraded (0.459%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T23:10:22.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:22 smithi110 ceph-mon[212939]: pgmap v417: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 1117/243468 objects degraded (0.459%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T23:10:23.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:23 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1117/243468 objects degraded (0.459%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:10:23.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:23 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2405811636' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:23.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:23 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1117/243468 objects degraded (0.459%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:10:23.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:23 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2405811636' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:23.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:23 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1117/243468 objects degraded (0.459%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:10:23.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:23 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2405811636' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:24.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:24 smithi138 ceph-mon[193606]: pgmap v418: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 783/240468 objects degraded (0.326%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T23:10:24.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:24 smithi110 ceph-mon[221102]: pgmap v418: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 783/240468 objects degraded (0.326%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T23:10:24.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:24 smithi110 ceph-mon[212939]: pgmap v418: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 783/240468 objects degraded (0.326%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T23:10:26.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:26 smithi110 ceph-mon[212939]: pgmap v419: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 783/231522 objects degraded (0.338%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T23:10:26.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:26 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2553017921' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:26.804 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:26 smithi110 ceph-mon[221102]: pgmap v419: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 783/231522 objects degraded (0.338%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T23:10:26.805 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:26 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2553017921' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:26.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:26 smithi138 ceph-mon[193606]: pgmap v419: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 783/231522 objects degraded (0.338%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T23:10:26.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:26 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2553017921' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:27.205 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:10:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:10:26] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.43.0" 2023-09-23T23:10:28.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:28 smithi138 ceph-mon[193606]: pgmap v420: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 783/231522 objects degraded (0.338%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T23:10:28.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:28.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:28 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 783/231522 objects degraded (0.338%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:10:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:28 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3471656116' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:28.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:28 smithi110 ceph-mon[212939]: pgmap v420: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 783/231522 objects degraded (0.338%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T23:10:28.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:28.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:28 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 783/231522 objects degraded (0.338%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:10:28.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:28 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3471656116' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:28.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:28 smithi110 ceph-mon[221102]: pgmap v420: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 783/231522 objects degraded (0.338%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T23:10:28.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:28.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:28 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 783/231522 objects degraded (0.338%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:10:28.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:28 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3471656116' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:30.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:30 smithi138 ceph-mon[193606]: pgmap v421: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 393/226455 objects degraded (0.174%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T23:10:30.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:30 smithi110 ceph-mon[212939]: pgmap v421: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 393/226455 objects degraded (0.174%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T23:10:30.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:30 smithi110 ceph-mon[221102]: pgmap v421: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 393/226455 objects degraded (0.174%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T23:10:31.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:31 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:31 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:31 smithi138 ceph-mon[193606]: Upgrade: osd.3 is safe to restart 2023-09-23T23:10:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:31 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1315731362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:31.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:31 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:31.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:31 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:31.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:31 smithi110 ceph-mon[212939]: Upgrade: osd.3 is safe to restart 2023-09-23T23:10:31.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:31 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1315731362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:31.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:31 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:31.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:31 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T23:10:31.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:31 smithi110 ceph-mon[221102]: Upgrade: osd.3 is safe to restart 2023-09-23T23:10:31.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:31 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1315731362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:32.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:32 smithi138 ceph-mon[193606]: pgmap v422: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 393/219786 objects degraded (0.179%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T23:10:32.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:32 smithi110 ceph-mon[212939]: pgmap v422: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 393/219786 objects degraded (0.179%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T23:10:32.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:32 smithi110 ceph-mon[221102]: pgmap v422: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 393/219786 objects degraded (0.179%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T23:10:33.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:33 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 393/219786 objects degraded (0.179%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:10:33.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:33 smithi138 ceph-mon[193606]: failed to encode map e143 with expected crc 2023-09-23T23:10:33.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:33 smithi138 ceph-mon[193606]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T23:10:33.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:33 smithi138 ceph-mon[193606]: failed to encode map e143 with expected crc 2023-09-23T23:10:33.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:33 smithi138 ceph-mon[193606]: failed to encode map e143 with expected crc 2023-09-23T23:10:33.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:33 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3959339937' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:33.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:33 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 393/219786 objects degraded (0.179%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:10:33.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:33 smithi110 ceph-mon[212939]: failed to encode map e143 with expected crc 2023-09-23T23:10:33.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:33 smithi110 ceph-mon[212939]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T23:10:33.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:33 smithi110 ceph-mon[212939]: failed to encode map e143 with expected crc 2023-09-23T23:10:33.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:33 smithi110 ceph-mon[212939]: failed to encode map e143 with expected crc 2023-09-23T23:10:33.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:33 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3959339937' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:33.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:33 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 393/219786 objects degraded (0.179%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:10:33.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:33 smithi110 ceph-mon[221102]: failed to encode map e143 with expected crc 2023-09-23T23:10:33.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:33 smithi110 ceph-mon[221102]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T23:10:33.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:33 smithi110 ceph-mon[221102]: failed to encode map e143 with expected crc 2023-09-23T23:10:33.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:33 smithi110 ceph-mon[221102]: failed to encode map e143 with expected crc 2023-09-23T23:10:33.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:33 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3959339937' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:34.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:34 smithi138 ceph-mon[193606]: failed to encode map e143 with expected crc 2023-09-23T23:10:34.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:34 smithi138 ceph-mon[193606]: failed to encode map e143 with expected crc 2023-09-23T23:10:34.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:34 smithi138 ceph-mon[193606]: pgmap v424: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 393/219786 objects degraded (0.179%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T23:10:34.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:34 smithi138 ceph-mon[193606]: Upgrade: Updating osd.3 2023-09-23T23:10:34.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:34 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:34.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:34 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T23:10:34.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:34 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:10:34.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:34 smithi138 ceph-mon[193606]: Deploying daemon osd.3 on smithi110 2023-09-23T23:10:34.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:34 smithi138 ceph-mon[193606]: osdmap e144: 8 total, 8 up, 8 in 2023-09-23T23:10:34.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:34 smithi138 ceph-mon[193606]: failed to encode map e144 with expected crc 2023-09-23T23:10:34.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[212939]: failed to encode map e143 with expected crc 2023-09-23T23:10:34.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[212939]: failed to encode map e143 with expected crc 2023-09-23T23:10:34.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[212939]: pgmap v424: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 393/219786 objects degraded (0.179%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T23:10:34.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[212939]: Upgrade: Updating osd.3 2023-09-23T23:10:34.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:34.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T23:10:34.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:10:34.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[212939]: Deploying daemon osd.3 on smithi110 2023-09-23T23:10:34.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[212939]: osdmap e144: 8 total, 8 up, 8 in 2023-09-23T23:10:34.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[212939]: failed to encode map e144 with expected crc 2023-09-23T23:10:34.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[221102]: failed to encode map e143 with expected crc 2023-09-23T23:10:34.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[221102]: failed to encode map e143 with expected crc 2023-09-23T23:10:34.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[221102]: pgmap v424: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 393/219786 objects degraded (0.179%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T23:10:34.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[221102]: Upgrade: Updating osd.3 2023-09-23T23:10:34.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:34.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T23:10:34.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:10:34.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[221102]: Deploying daemon osd.3 on smithi110 2023-09-23T23:10:34.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[221102]: osdmap e144: 8 total, 8 up, 8 in 2023-09-23T23:10:34.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:34 smithi110 ceph-mon[221102]: failed to encode map e144 with expected crc 2023-09-23T23:10:35.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:35 smithi138 ceph-mon[193606]: failed to encode map e144 with expected crc 2023-09-23T23:10:35.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:35 smithi138 ceph-mon[193606]: failed to encode map e144 with expected crc 2023-09-23T23:10:35.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:35 smithi138 ceph-mon[193606]: failed to encode map e144 with expected crc 2023-09-23T23:10:35.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:35 smithi138 ceph-mon[193606]: failed to encode map e144 with expected crc 2023-09-23T23:10:35.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:35 smithi110 ceph-mon[221102]: failed to encode map e144 with expected crc 2023-09-23T23:10:35.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:35 smithi110 ceph-mon[221102]: failed to encode map e144 with expected crc 2023-09-23T23:10:35.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:35 smithi110 ceph-mon[221102]: failed to encode map e144 with expected crc 2023-09-23T23:10:35.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:35 smithi110 ceph-mon[221102]: failed to encode map e144 with expected crc 2023-09-23T23:10:35.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:35 smithi110 ceph-mon[212939]: failed to encode map e144 with expected crc 2023-09-23T23:10:35.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:35 smithi110 ceph-mon[212939]: failed to encode map e144 with expected crc 2023-09-23T23:10:35.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:35 smithi110 ceph-mon[212939]: failed to encode map e144 with expected crc 2023-09-23T23:10:35.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:35 smithi110 ceph-mon[212939]: failed to encode map e144 with expected crc 2023-09-23T23:10:36.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:36 smithi138 ceph-mon[193606]: pgmap v426: 89 pgs: 1 peering, 88 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 6.1 MiB/s, 97 objects/s recovering 2023-09-23T23:10:36.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:36 smithi138 ceph-mon[193606]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 393/219786 objects degraded (0.179%), 1 pg degraded, 1 pg undersized) 2023-09-23T23:10:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:36 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:10:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:36 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3749473217' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:36.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:36 smithi110 ceph-mon[212939]: pgmap v426: 89 pgs: 1 peering, 88 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 6.1 MiB/s, 97 objects/s recovering 2023-09-23T23:10:36.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:36 smithi110 ceph-mon[212939]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 393/219786 objects degraded (0.179%), 1 pg degraded, 1 pg undersized) 2023-09-23T23:10:36.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:36 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:10:36.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:36 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3749473217' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:36.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:10:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:10:36] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.43.0" 2023-09-23T23:10:36.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:36 smithi110 ceph-mon[221102]: pgmap v426: 89 pgs: 1 peering, 88 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 6.1 MiB/s, 97 objects/s recovering 2023-09-23T23:10:36.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:36 smithi110 ceph-mon[221102]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 393/219786 objects degraded (0.179%), 1 pg degraded, 1 pg undersized) 2023-09-23T23:10:36.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:36 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:10:36.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:36 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3749473217' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:38.453 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:38 smithi110 systemd[1]: Stopping Ceph osd.3 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:10:38.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:38 smithi138 ceph-mon[193606]: pgmap v427: 89 pgs: 1 peering, 88 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:10:38.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:38 smithi138 ceph-mon[193606]: osd.3 marked itself down and dead 2023-09-23T23:10:38.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:38 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1486440310' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:38.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:38 smithi110 ceph-mon[212939]: pgmap v427: 89 pgs: 1 peering, 88 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:10:38.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:38 smithi110 ceph-mon[212939]: osd.3 marked itself down and dead 2023-09-23T23:10:38.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:38 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1486440310' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:38.953 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:38 smithi110 conmon[151896]: 2023-09-23T23:10:38.554+0000 7fb85b930700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:10:38.953 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:38 smithi110 conmon[151896]: 2023-09-23T23:10:38.554+0000 7fb85b930700 -1 osd.3 144 *** Got signal Terminated *** 2023-09-23T23:10:38.953 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:38 smithi110 conmon[151896]: 2023-09-23T23:10:38.554+0000 7fb85b930700 -1 osd.3 144 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:10:38.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:38 smithi110 ceph-mon[221102]: pgmap v427: 89 pgs: 1 peering, 88 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:10:38.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:38 smithi110 ceph-mon[221102]: osd.3 marked itself down and dead 2023-09-23T23:10:38.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:38 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1486440310' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:39.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:39 smithi138 ceph-mon[193606]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:10:39.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:39 smithi138 ceph-mon[193606]: osdmap e145: 8 total, 7 up, 8 in 2023-09-23T23:10:39.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:39 smithi138 ceph-mon[193606]: failed to encode map e145 with expected crc 2023-09-23T23:10:39.951 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:39 smithi110 bash[269611]: d34afe3cf9ea10547f094266d6df92f6371cbda0759a93e3eefa1fae944b3a6d 2023-09-23T23:10:39.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:39 smithi110 ceph-mon[212939]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:10:39.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:39 smithi110 ceph-mon[212939]: osdmap e145: 8 total, 7 up, 8 in 2023-09-23T23:10:39.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:39 smithi110 ceph-mon[212939]: failed to encode map e145 with expected crc 2023-09-23T23:10:39.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:39 smithi110 ceph-mon[221102]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:10:39.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:39 smithi110 ceph-mon[221102]: osdmap e145: 8 total, 7 up, 8 in 2023-09-23T23:10:39.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:39 smithi110 ceph-mon[221102]: failed to encode map e145 with expected crc 2023-09-23T23:10:40.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:40 smithi138 ceph-mon[193606]: failed to encode map e145 with expected crc 2023-09-23T23:10:40.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:40 smithi138 ceph-mon[193606]: pgmap v429: 89 pgs: 10 peering, 11 stale+active+clean, 68 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 4.0 MiB/s, 64 objects/s recovering 2023-09-23T23:10:40.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:40 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T23:10:40.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:40 smithi138 ceph-mon[193606]: osdmap e146: 8 total, 7 up, 8 in 2023-09-23T23:10:40.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:40 smithi138 ceph-mon[193606]: failed to encode map e146 with expected crc 2023-09-23T23:10:40.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:40 smithi138 ceph-mon[193606]: failed to encode map e146 with expected crc 2023-09-23T23:10:40.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:40 smithi110 ceph-mon[221102]: failed to encode map e145 with expected crc 2023-09-23T23:10:40.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:40 smithi110 ceph-mon[221102]: pgmap v429: 89 pgs: 10 peering, 11 stale+active+clean, 68 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 4.0 MiB/s, 64 objects/s recovering 2023-09-23T23:10:40.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:40 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T23:10:40.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:40 smithi110 ceph-mon[221102]: osdmap e146: 8 total, 7 up, 8 in 2023-09-23T23:10:40.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:40 smithi110 ceph-mon[221102]: failed to encode map e146 with expected crc 2023-09-23T23:10:40.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:40 smithi110 ceph-mon[221102]: failed to encode map e146 with expected crc 2023-09-23T23:10:40.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:40 smithi110 ceph-mon[212939]: failed to encode map e145 with expected crc 2023-09-23T23:10:40.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:40 smithi110 ceph-mon[212939]: pgmap v429: 89 pgs: 10 peering, 11 stale+active+clean, 68 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 4.0 MiB/s, 64 objects/s recovering 2023-09-23T23:10:40.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:40 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T23:10:40.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:40 smithi110 ceph-mon[212939]: osdmap e146: 8 total, 7 up, 8 in 2023-09-23T23:10:40.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:40 smithi110 ceph-mon[212939]: failed to encode map e146 with expected crc 2023-09-23T23:10:40.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:40 smithi110 ceph-mon[212939]: failed to encode map e146 with expected crc 2023-09-23T23:10:41.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:41 smithi138 ceph-mon[193606]: failed to encode map e146 with expected crc 2023-09-23T23:10:41.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:41 smithi138 ceph-mon[193606]: failed to encode map e146 with expected crc 2023-09-23T23:10:41.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:41 smithi138 ceph-mon[193606]: failed to encode map e146 with expected crc 2023-09-23T23:10:41.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:41 smithi138 ceph-mon[193606]: failed to encode map e146 with expected crc 2023-09-23T23:10:41.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:41 smithi138 ceph-mon[193606]: failed to encode map e146 with expected crc 2023-09-23T23:10:41.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:41 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/238864477' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:41.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:41 smithi110 ceph-mon[221102]: failed to encode map e146 with expected crc 2023-09-23T23:10:41.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:41 smithi110 ceph-mon[221102]: failed to encode map e146 with expected crc 2023-09-23T23:10:41.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:41 smithi110 ceph-mon[221102]: failed to encode map e146 with expected crc 2023-09-23T23:10:41.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:41 smithi110 ceph-mon[221102]: failed to encode map e146 with expected crc 2023-09-23T23:10:41.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:41 smithi110 ceph-mon[221102]: failed to encode map e146 with expected crc 2023-09-23T23:10:41.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:41 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/238864477' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:41.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:41 smithi110 ceph-mon[212939]: failed to encode map e146 with expected crc 2023-09-23T23:10:41.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:41 smithi110 ceph-mon[212939]: failed to encode map e146 with expected crc 2023-09-23T23:10:41.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:41 smithi110 ceph-mon[212939]: failed to encode map e146 with expected crc 2023-09-23T23:10:41.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:41 smithi110 ceph-mon[212939]: failed to encode map e146 with expected crc 2023-09-23T23:10:41.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:41 smithi110 ceph-mon[212939]: failed to encode map e146 with expected crc 2023-09-23T23:10:41.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:41 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/238864477' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:42.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:42 smithi138 ceph-mon[193606]: pgmap v431: 89 pgs: 1 activating+undersized, 1 active+undersized+degraded, 16 peering, 7 stale+active+clean, 6 active+undersized, 58 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 4197/200139 objects degraded (2.097%) 2023-09-23T23:10:42.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:42 smithi138 ceph-mon[193606]: Health check failed: Degraded data redundancy: 4197/200139 objects degraded (2.097%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:10:42.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:42 smithi110 ceph-mon[221102]: pgmap v431: 89 pgs: 1 activating+undersized, 1 active+undersized+degraded, 16 peering, 7 stale+active+clean, 6 active+undersized, 58 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 4197/200139 objects degraded (2.097%) 2023-09-23T23:10:42.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:42 smithi110 ceph-mon[221102]: Health check failed: Degraded data redundancy: 4197/200139 objects degraded (2.097%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:10:42.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:42 smithi110 ceph-mon[212939]: pgmap v431: 89 pgs: 1 activating+undersized, 1 active+undersized+degraded, 16 peering, 7 stale+active+clean, 6 active+undersized, 58 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 4197/200139 objects degraded (2.097%) 2023-09-23T23:10:42.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:42 smithi110 ceph-mon[212939]: Health check failed: Degraded data redundancy: 4197/200139 objects degraded (2.097%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:10:43.453 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:43 smithi110 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.3.service: Succeeded. 2023-09-23T23:10:43.453 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:43 smithi110 systemd[1]: Stopped Ceph osd.3 for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:10:43.453 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:43 smithi110 systemd[1]: Starting Ceph osd.3 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:10:44.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:44.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:44.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:43 smithi138 ceph-mon[193606]: pgmap v432: 89 pgs: 1 activating+undersized, 2 active+undersized+degraded, 16 peering, 5 stale+active+clean, 9 active+undersized, 56 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 8212/199158 objects degraded (4.123%) 2023-09-23T23:10:44.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:43 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2118426038' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:43 smithi110 ceph-mon[212939]: pgmap v432: 89 pgs: 1 activating+undersized, 2 active+undersized+degraded, 16 peering, 5 stale+active+clean, 9 active+undersized, 56 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 8212/199158 objects degraded (4.123%) 2023-09-23T23:10:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:43 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2118426038' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:44.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:44.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:44.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:43 smithi110 ceph-mon[221102]: pgmap v432: 89 pgs: 1 activating+undersized, 2 active+undersized+degraded, 16 peering, 5 stale+active+clean, 9 active+undersized, 56 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 8212/199158 objects degraded (4.123%) 2023-09-23T23:10:44.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:43 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2118426038' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:44.952 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:44 smithi110 bash[270274]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T23:10:44.953 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:44 smithi110 bash[270274]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T23:10:44.953 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:44 smithi110 bash[270274]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-09-23T23:10:44.953 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:44 smithi110 bash[270274]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-09-23T23:10:44.953 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:44 smithi110 bash[270274]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-09-23T23:10:44.953 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:44 smithi110 bash[270274]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-23T23:10:44.953 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:44 smithi110 bash[270274]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T23:10:44.953 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:44 smithi110 bash[270274]: --> ceph-volume lvm activate successful for osd ID: 3 2023-09-23T23:10:45.701 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:45 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-09-23T23:10:45.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:45 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-09-23T23:10:45.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:45 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-09-23T23:10:46.587 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:46 smithi110 ceph-mon[221102]: pgmap v433: 89 pgs: 1 activating+undersized, 8 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 24062/192201 objects degraded (12.519%) 2023-09-23T23:10:46.587 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:46 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1587231509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:46.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:46 smithi110 ceph-mon[212939]: pgmap v433: 89 pgs: 1 activating+undersized, 8 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 24062/192201 objects degraded (12.519%) 2023-09-23T23:10:46.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:46 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1587231509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:46.587 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:46 smithi110 bash[270708]: ff8e0fe10bdc86c51c90e979e94a6df14944096d4a2105d6081f57165af9d11c 2023-09-23T23:10:46.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:46 smithi138 ceph-mon[193606]: pgmap v433: 89 pgs: 1 activating+undersized, 8 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 24062/192201 objects degraded (12.519%) 2023-09-23T23:10:46.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:46 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1587231509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:46.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:10:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:10:46] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.43.0" 2023-09-23T23:10:46.952 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:46 smithi110 systemd[1]: Started Ceph osd.3 for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:10:46.953 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:46 smithi110 ceph-osd[270774]: -- 172.21.15.110:0/1405053197 <== mon.0 v2:172.21.15.110:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56337679a4e0 con 0x56337676cc00 2023-09-23T23:10:47.453 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:47 smithi110 conmon[270750]: 2023-09-23T23:10:47.116+0000 7f5fe4a78640 -1 Falling back to public interface 2023-09-23T23:10:47.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:47 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:47.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:47 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:47.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:47 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:10:47.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:47.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:47.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:47 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:10:47.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:47 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:47.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:47 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:47.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:47 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:10:48.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:48 smithi138 ceph-mon[193606]: pgmap v434: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 23800/189951 objects degraded (12.530%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:48.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:48 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 23800/189951 objects degraded (12.530%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:10:48.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:48 smithi110 ceph-mon[212939]: pgmap v434: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 23800/189951 objects degraded (12.530%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:48.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:48 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 23800/189951 objects degraded (12.530%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:10:48.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:48 smithi110 ceph-mon[221102]: pgmap v434: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 23800/189951 objects degraded (12.530%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:48.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:48 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 23800/189951 objects degraded (12.530%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:10:49.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:49 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1220840623' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:49.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:49 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1220840623' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:50.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:49 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1220840623' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:50.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:50 smithi110 ceph-mon[221102]: pgmap v435: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 23016/183051 objects degraded (12.574%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:50.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:50 smithi110 ceph-mon[212939]: pgmap v435: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 23016/183051 objects degraded (12.574%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:51.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:50 smithi138 ceph-mon[193606]: pgmap v435: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 23016/183051 objects degraded (12.574%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:52.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:52 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:52.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:52 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:52.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:52 smithi138 ceph-mon[193606]: pgmap v436: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 22210/177600 objects degraded (12.506%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:52 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2174105776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:52.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:52 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:52.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:52 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:52.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:52 smithi110 ceph-mon[212939]: pgmap v436: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 22210/177600 objects degraded (12.506%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:52.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:52 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2174105776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:52.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:52 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:52.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:52 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:52.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:52 smithi110 ceph-mon[221102]: pgmap v436: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 22210/177600 objects degraded (12.506%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:52.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:52 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2174105776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:53.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:53 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:53.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:53 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:53.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:53 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 22210/177600 objects degraded (12.506%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:10:53.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:53 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:53.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:53 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:53.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:53 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 22210/177600 objects degraded (12.506%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:10:53.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:53 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:53.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:53 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:53.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:53 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 22210/177600 objects degraded (12.506%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:10:54.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:54 smithi110 ceph-mon[212939]: pgmap v437: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 21958/176844 objects degraded (12.417%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:54.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:54 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2640606986' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:54.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:54 smithi110 ceph-mon[221102]: pgmap v437: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 21958/176844 objects degraded (12.417%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:54.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:54 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2640606986' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:54.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:54 smithi138 ceph-mon[193606]: pgmap v437: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 21958/176844 objects degraded (12.417%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:54.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:54 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2640606986' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:56.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:56 smithi110 ceph-mon[212939]: pgmap v438: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 20965/167598 objects degraded (12.509%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:56.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:56 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3623694911' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:56.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:56 smithi110 ceph-mon[221102]: pgmap v438: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 20965/167598 objects degraded (12.509%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:56.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:56 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3623694911' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:56.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:56 smithi138 ceph-mon[193606]: pgmap v438: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 20965/167598 objects degraded (12.509%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:56.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:56 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3623694911' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:57.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:10:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:10:56] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.43.0" 2023-09-23T23:10:57.202 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:57 smithi110 conmon[270750]: 2023-09-23T23:10:56.997+0000 7f5fe4a78640 -1 osd.3 144 log_to_monitors true 2023-09-23T23:10:57.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:57 smithi138 ceph-mon[193606]: from='osd.3 [v2:172.21.15.110:6826/2333701749,v1:172.21.15.110:6827/2333701749]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T23:10:57.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:57 smithi110 ceph-mon[221102]: from='osd.3 [v2:172.21.15.110:6826/2333701749,v1:172.21.15.110:6827/2333701749]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T23:10:57.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:57 smithi110 ceph-mon[212939]: from='osd.3 [v2:172.21.15.110:6826/2333701749,v1:172.21.15.110:6827/2333701749]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T23:10:57.961 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:10:57 smithi110 conmon[270750]: 2023-09-23T23:10:57.506+0000 7f5fd46e2700 -1 osd.3 144 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T23:10:58.800 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T23:10:58.816 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[212939]: pgmap v439: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 20686/165327 objects degraded (12.512%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:58.816 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[212939]: failed to encode map e147 with expected crc 2023-09-23T23:10:58.816 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[212939]: from='osd.3 [v2:172.21.15.110:6826/2333701749,v1:172.21.15.110:6827/2333701749]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T23:10:58.817 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[212939]: osdmap e147: 8 total, 7 up, 8 in 2023-09-23T23:10:58.817 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[212939]: from='osd.3 [v2:172.21.15.110:6826/2333701749,v1:172.21.15.110:6827/2333701749]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:10:58.817 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[212939]: failed to encode map e147 with expected crc 2023-09-23T23:10:58.817 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[212939]: failed to encode map e147 with expected crc 2023-09-23T23:10:58.817 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[212939]: failed to encode map e147 with expected crc 2023-09-23T23:10:58.817 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:58.817 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:58.817 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 20686/165327 objects degraded (12.512%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:10:58.817 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[221102]: pgmap v439: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 20686/165327 objects degraded (12.512%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:58.818 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[221102]: failed to encode map e147 with expected crc 2023-09-23T23:10:58.818 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[221102]: from='osd.3 [v2:172.21.15.110:6826/2333701749,v1:172.21.15.110:6827/2333701749]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T23:10:58.818 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[221102]: osdmap e147: 8 total, 7 up, 8 in 2023-09-23T23:10:58.818 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[221102]: from='osd.3 [v2:172.21.15.110:6826/2333701749,v1:172.21.15.110:6827/2333701749]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:10:58.818 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[221102]: failed to encode map e147 with expected crc 2023-09-23T23:10:58.818 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[221102]: failed to encode map e147 with expected crc 2023-09-23T23:10:58.818 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[221102]: failed to encode map e147 with expected crc 2023-09-23T23:10:58.818 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:58.818 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:58.818 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:58 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 20686/165327 objects degraded (12.512%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:10:58.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:58 smithi138 ceph-mon[193606]: pgmap v439: 89 pgs: 9 active+undersized+degraded, 28 active+undersized, 52 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 20686/165327 objects degraded (12.512%); 0 B/s, 0 objects/s recovering 2023-09-23T23:10:58.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:58 smithi138 ceph-mon[193606]: failed to encode map e147 with expected crc 2023-09-23T23:10:58.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:58 smithi138 ceph-mon[193606]: from='osd.3 [v2:172.21.15.110:6826/2333701749,v1:172.21.15.110:6827/2333701749]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T23:10:58.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:58 smithi138 ceph-mon[193606]: osdmap e147: 8 total, 7 up, 8 in 2023-09-23T23:10:58.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:58 smithi138 ceph-mon[193606]: from='osd.3 [v2:172.21.15.110:6826/2333701749,v1:172.21.15.110:6827/2333701749]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:10:58.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:58 smithi138 ceph-mon[193606]: failed to encode map e147 with expected crc 2023-09-23T23:10:58.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:58 smithi138 ceph-mon[193606]: failed to encode map e147 with expected crc 2023-09-23T23:10:58.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:58 smithi138 ceph-mon[193606]: failed to encode map e147 with expected crc 2023-09-23T23:10:58.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:10:58.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:58.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:58 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 20686/165327 objects degraded (12.512%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:10:59.906 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:59 smithi110 ceph-mon[221102]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:10:59.906 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:59 smithi110 ceph-mon[221102]: failed to encode map e148 with expected crc 2023-09-23T23:10:59.906 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:59 smithi110 ceph-mon[221102]: osd.3 [v2:172.21.15.110:6826/2333701749,v1:172.21.15.110:6827/2333701749] boot 2023-09-23T23:10:59.906 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:59 smithi110 ceph-mon[221102]: osdmap e148: 8 total, 8 up, 8 in 2023-09-23T23:10:59.906 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:59 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T23:10:59.906 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:59 smithi110 ceph-mon[221102]: failed to encode map e148 with expected crc 2023-09-23T23:10:59.906 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:59 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3199398823' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:59.906 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:10:59 smithi110 ceph-mon[221102]: osdmap e149: 8 total, 8 up, 8 in 2023-09-23T23:10:59.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:59 smithi110 ceph-mon[212939]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:10:59.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:59 smithi110 ceph-mon[212939]: failed to encode map e148 with expected crc 2023-09-23T23:10:59.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:59 smithi110 ceph-mon[212939]: osd.3 [v2:172.21.15.110:6826/2333701749,v1:172.21.15.110:6827/2333701749] boot 2023-09-23T23:10:59.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:59 smithi110 ceph-mon[212939]: osdmap e148: 8 total, 8 up, 8 in 2023-09-23T23:10:59.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:59 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T23:10:59.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:59 smithi110 ceph-mon[212939]: failed to encode map e148 with expected crc 2023-09-23T23:10:59.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:59 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3199398823' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:59.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:10:59 smithi110 ceph-mon[212939]: osdmap e149: 8 total, 8 up, 8 in 2023-09-23T23:10:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:59 smithi138 ceph-mon[193606]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:10:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:59 smithi138 ceph-mon[193606]: failed to encode map e148 with expected crc 2023-09-23T23:10:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:59 smithi138 ceph-mon[193606]: osd.3 [v2:172.21.15.110:6826/2333701749,v1:172.21.15.110:6827/2333701749] boot 2023-09-23T23:10:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:59 smithi138 ceph-mon[193606]: osdmap e148: 8 total, 8 up, 8 in 2023-09-23T23:10:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:59 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T23:10:59.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:59 smithi138 ceph-mon[193606]: failed to encode map e148 with expected crc 2023-09-23T23:10:59.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:59 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3199398823' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T23:10:59.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:10:59 smithi138 ceph-mon[193606]: osdmap e149: 8 total, 8 up, 8 in 2023-09-23T23:11:00.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:11:00 smithi110 conmon[198813]: 2023-09-23T23:11:00.083+0000 7ff0150b6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2023-09-23T23:11:00.265 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T23:11:00.510 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: pgmap v442: 89 pgs: 5 peering, 8 active+undersized+degraded, 24 active+undersized, 52 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 20161/159435 objects degraded (12.645%) 2023-09-23T23:11:00.510 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.510 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.510 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.510 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.511 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.511 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.511 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.511 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.511 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:00.511 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:00.511 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:11:00.511 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:11:00.511 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:00.511 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:11:00.511 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:11:00.511 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:11:00.511 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:11:00.512 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T23:11:00.512 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[212939]: osdmap e150: 8 total, 8 up, 8 in 2023-09-23T23:11:00.512 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: pgmap v442: 89 pgs: 5 peering, 8 active+undersized+degraded, 24 active+undersized, 52 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 20161/159435 objects degraded (12.645%) 2023-09-23T23:11:00.512 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.512 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.512 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.512 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.512 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.512 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.512 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.512 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.513 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:00.513 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:00.513 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:11:00.513 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:11:00.513 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:00.513 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:11:00.513 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:11:00.513 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:11:00.513 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:11:00.513 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T23:11:00.513 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:00 smithi110 ceph-mon[221102]: osdmap e150: 8 total, 8 up, 8 in 2023-09-23T23:11:00.749 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:00.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: pgmap v442: 89 pgs: 5 peering, 8 active+undersized+degraded, 24 active+undersized, 52 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 20161/159435 objects degraded (12.645%) 2023-09-23T23:11:00.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: failed to encode map e149 with expected crc 2023-09-23T23:11:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:11:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:11:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:11:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:11:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:11:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:11:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T23:11:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:00 smithi138 ceph-mon[193606]: osdmap e150: 8 total, 8 up, 8 in 2023-09-23T23:11:01.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:01 smithi110 ceph-mon[212939]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T23:11:01.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:01 smithi110 ceph-mon[212939]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2023-09-23T23:11:01.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:01 smithi110 ceph-mon[212939]: failed to encode map e150 with expected crc 2023-09-23T23:11:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:01 smithi110 ceph-mon[212939]: failed to encode map e150 with expected crc 2023-09-23T23:11:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:01 smithi110 ceph-mon[212939]: failed to encode map e150 with expected crc 2023-09-23T23:11:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:01 smithi110 ceph-mon[212939]: failed to encode map e150 with expected crc 2023-09-23T23:11:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:01 smithi110 ceph-mon[221102]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T23:11:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:01 smithi110 ceph-mon[221102]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2023-09-23T23:11:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:01 smithi110 ceph-mon[221102]: failed to encode map e150 with expected crc 2023-09-23T23:11:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:01 smithi110 ceph-mon[221102]: failed to encode map e150 with expected crc 2023-09-23T23:11:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:01 smithi110 ceph-mon[221102]: failed to encode map e150 with expected crc 2023-09-23T23:11:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:01 smithi110 ceph-mon[221102]: failed to encode map e150 with expected crc 2023-09-23T23:11:01.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:01 smithi138 ceph-mon[193606]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T23:11:01.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:01 smithi138 ceph-mon[193606]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2023-09-23T23:11:01.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:01 smithi138 ceph-mon[193606]: failed to encode map e150 with expected crc 2023-09-23T23:11:01.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:01 smithi138 ceph-mon[193606]: failed to encode map e150 with expected crc 2023-09-23T23:11:01.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:01 smithi138 ceph-mon[193606]: failed to encode map e150 with expected crc 2023-09-23T23:11:01.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:01 smithi138 ceph-mon[193606]: failed to encode map e150 with expected crc 2023-09-23T23:11:02.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:02 smithi110 ceph-mon[212939]: pgmap v445: 89 pgs: 1 remapped+peering, 9 peering, 6 active+undersized+degraded, 16 active+undersized, 57 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 13304/155862 objects degraded (8.536%) 2023-09-23T23:11:02.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:02 smithi110 ceph-mon[221102]: pgmap v445: 89 pgs: 1 remapped+peering, 9 peering, 6 active+undersized+degraded, 16 active+undersized, 57 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 13304/155862 objects degraded (8.536%) 2023-09-23T23:11:02.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:02 smithi138 ceph-mon[193606]: pgmap v445: 89 pgs: 1 remapped+peering, 9 peering, 6 active+undersized+degraded, 16 active+undersized, 57 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 13304/155862 objects degraded (8.536%) 2023-09-23T23:11:03.505 INFO:teuthology.orchestra.run.smithi110.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:11:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:03 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 13304/155862 objects degraded (8.536%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:03 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:03.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:03 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 13304/155862 objects degraded (8.536%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:03.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:03 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:03.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:03 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 13304/155862 objects degraded (8.536%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:03.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:03 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:04.898 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:04 smithi110 ceph-mon[212939]: pgmap v446: 89 pgs: 1 remapped+peering, 9 peering, 4 active+undersized+degraded, 3 active+undersized, 72 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 13302/155862 objects degraded (8.534%) 2023-09-23T23:11:04.899 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:04 smithi110 ceph-mon[212939]: from='client.44948 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:11:04.899 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:04 smithi110 ceph-mon[212939]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:11:04.899 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:04 smithi110 ceph-mon[221102]: pgmap v446: 89 pgs: 1 remapped+peering, 9 peering, 4 active+undersized+degraded, 3 active+undersized, 72 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 13302/155862 objects degraded (8.534%) 2023-09-23T23:11:04.899 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:04 smithi110 ceph-mon[221102]: from='client.44948 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:11:04.899 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:04 smithi110 ceph-mon[221102]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:11:04.937 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T23:11:04.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:04 smithi138 ceph-mon[193606]: pgmap v446: 89 pgs: 1 remapped+peering, 9 peering, 4 active+undersized+degraded, 3 active+undersized, 72 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 13302/155862 objects degraded (8.534%) 2023-09-23T23:11:04.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:04 smithi138 ceph-mon[193606]: from='client.44948 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:11:04.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:04 smithi138 ceph-mon[193606]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T23:11:05.650 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:06.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:06 smithi110 ceph-mon[212939]: pgmap v447: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 83 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 3698/149445 objects degraded (2.474%); 0 B/s, 20 objects/s recovering 2023-09-23T23:11:06.805 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:06 smithi110 ceph-mon[221102]: pgmap v447: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 83 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 3698/149445 objects degraded (2.474%); 0 B/s, 20 objects/s recovering 2023-09-23T23:11:06.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:06 smithi138 ceph-mon[193606]: pgmap v447: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 83 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 3698/149445 objects degraded (2.474%); 0 B/s, 20 objects/s recovering 2023-09-23T23:11:07.205 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:11:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:11:06] "GET /metrics HTTP/1.1" 200 36268 "" "Prometheus/2.43.0" 2023-09-23T23:11:08.581 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T23:11:08.581 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (14m) - 18m 27.2M - 0.25.0 c8568f914cd2 d9641d6ed9ca 2023-09-23T23:11:08.581 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi138 *:3000 running (14m) - 18m 94.2M - 9.4.7 2c41d148cca3 56929ec304ec 2023-09-23T23:11:08.581 INFO:teuthology.orchestra.run.smithi110.stdout:mds.foo.smithi110.fuglpi smithi110 running (17m) - 17m 22.3M - 17.2.6-1125-gc9bb1bcf 444539aec5bf f59eb5601951 2023-09-23T23:11:08.581 INFO:teuthology.orchestra.run.smithi110.stdout:mds.foo.smithi138.xrewco smithi138 running (17m) - 17m 17.2M - 17.2.6-1125-gc9bb1bcf 444539aec5bf cf02b45fd734 2023-09-23T23:11:08.581 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi138 *:8443,9283,8765 running (13m) - 22m 452M - 18.2.0-460-gbb0daeff e32c2a64c126 fef2a6a3bd6d 2023-09-23T23:11:08.582 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (13m) - 24m 523M - 18.2.0-460-gbb0daeff e32c2a64c126 09d917770c6e 2023-09-23T23:11:08.582 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (12m) - 24m 118M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 d6c089d81bde 2023-09-23T23:11:08.582 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi138 running (9m) - 22m 72.2M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 42bfa5f10303 2023-09-23T23:11:08.582 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (10m) - 22m 86.7M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 4c74500e80a3 2023-09-23T23:11:08.582 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (14m) - 18m 20.3M - 1.5.0 0da6a335fe13 39d48d803cf2 2023-09-23T23:11:08.582 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi138 *:9100 running (13m) - 17m 21.0M - 1.5.0 0da6a335fe13 a55c5d37ba4b 2023-09-23T23:11:08.582 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (7m) - 22m 3746M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 0c340b7feaa9 2023-09-23T23:11:08.582 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (3m) - 21m 2999M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 038c0580de94 2023-09-23T23:11:08.582 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (2m) - 21m 1193M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 94b7643f91e8 2023-09-23T23:11:08.582 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (22s) - 21m 13.9M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 ff8e0fe10bdc 2023-09-23T23:11:08.582 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi138 running (20m) - 20m 5462M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 47f7252bf067 2023-09-23T23:11:08.582 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi138 running (20m) - 20m 7301M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf fb3decabc2c4 2023-09-23T23:11:08.583 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi138 running (19m) - 19m 5377M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf c3f9d2352b7c 2023-09-23T23:11:08.583 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi138 running (19m) - 19m 5615M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 161d1d2d0445 2023-09-23T23:11:08.583 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi138 *:9095 running (13m) - 18m 69.1M - 2.43.0 a07b618ecd1d 8c2bb4977f43 2023-09-23T23:11:08.868 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:08 smithi110 ceph-mon[212939]: pgmap v448: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 4687/147732 objects degraded (3.173%); 0 B/s, 17 objects/s recovering 2023-09-23T23:11:08.869 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:08 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 4687/147732 objects degraded (3.173%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:08.869 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:08 smithi110 ceph-mon[221102]: pgmap v448: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 4687/147732 objects degraded (3.173%); 0 B/s, 17 objects/s recovering 2023-09-23T23:11:08.869 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:08 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 4687/147732 objects degraded (3.173%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:08.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:08 smithi138 ceph-mon[193606]: pgmap v448: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 4687/147732 objects degraded (3.173%); 0 B/s, 17 objects/s recovering 2023-09-23T23:11:08.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:08 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 4687/147732 objects degraded (3.173%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:09.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:09 smithi138 ceph-mon[193606]: from='client.54961 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:11:09.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:09 smithi110 ceph-mon[212939]: from='client.54961 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:11:09.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:09 smithi110 ceph-mon[221102]: from='client.54961 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:11:09.996 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T23:11:10.775 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:10.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:10 smithi138 ceph-mon[193606]: pgmap v449: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 4687/140799 objects degraded (3.329%); 0 B/s, 46 objects/s recovering 2023-09-23T23:11:10.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:10 smithi110 ceph-mon[212939]: pgmap v449: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 4687/140799 objects degraded (3.329%); 0 B/s, 46 objects/s recovering 2023-09-23T23:11:10.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:10 smithi110 ceph-mon[221102]: pgmap v449: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 4687/140799 objects degraded (3.329%); 0 B/s, 46 objects/s recovering 2023-09-23T23:11:12.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:12 smithi110 ceph-mon[221102]: pgmap v450: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 4687/136482 objects degraded (3.434%); 0 B/s, 41 objects/s recovering 2023-09-23T23:11:12.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:12 smithi110 ceph-mon[212939]: pgmap v450: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 4687/136482 objects degraded (3.434%); 0 B/s, 41 objects/s recovering 2023-09-23T23:11:12.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:12 smithi138 ceph-mon[193606]: pgmap v450: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 4687/136482 objects degraded (3.434%); 0 B/s, 41 objects/s recovering 2023-09-23T23:11:13.544 INFO:teuthology.orchestra.run.smithi110.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T23:11:13.544 INFO:teuthology.orchestra.run.smithi110.stdout:alertmanager.a smithi110 *:9093,9094 running (14m) - 18m 27.2M - 0.25.0 c8568f914cd2 d9641d6ed9ca 2023-09-23T23:11:13.545 INFO:teuthology.orchestra.run.smithi110.stdout:grafana.a smithi138 *:3000 running (14m) - 18m 94.2M - 9.4.7 2c41d148cca3 56929ec304ec 2023-09-23T23:11:13.545 INFO:teuthology.orchestra.run.smithi110.stdout:mds.foo.smithi110.fuglpi smithi110 running (17m) - 17m 22.3M - 17.2.6-1125-gc9bb1bcf 444539aec5bf f59eb5601951 2023-09-23T23:11:13.545 INFO:teuthology.orchestra.run.smithi110.stdout:mds.foo.smithi138.xrewco smithi138 running (17m) - 17m 17.2M - 17.2.6-1125-gc9bb1bcf 444539aec5bf cf02b45fd734 2023-09-23T23:11:13.545 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.x smithi138 *:8443,9283,8765 running (13m) - 22m 452M - 18.2.0-460-gbb0daeff e32c2a64c126 fef2a6a3bd6d 2023-09-23T23:11:13.545 INFO:teuthology.orchestra.run.smithi110.stdout:mgr.y smithi110 *:8443,9283,8765 running (13m) - 24m 523M - 18.2.0-460-gbb0daeff e32c2a64c126 09d917770c6e 2023-09-23T23:11:13.545 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a smithi110 running (12m) - 24m 118M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 d6c089d81bde 2023-09-23T23:11:13.545 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b smithi138 running (9m) - 23m 72.2M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 42bfa5f10303 2023-09-23T23:11:13.545 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c smithi110 running (11m) - 22m 86.7M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 4c74500e80a3 2023-09-23T23:11:13.545 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.a smithi110 *:9100 running (14m) - 18m 20.3M - 1.5.0 0da6a335fe13 39d48d803cf2 2023-09-23T23:11:13.545 INFO:teuthology.orchestra.run.smithi110.stdout:node-exporter.b smithi138 *:9100 running (14m) - 18m 21.0M - 1.5.0 0da6a335fe13 a55c5d37ba4b 2023-09-23T23:11:13.545 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 smithi110 running (7m) - 22m 3746M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 0c340b7feaa9 2023-09-23T23:11:13.545 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 smithi110 running (3m) - 21m 2999M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 038c0580de94 2023-09-23T23:11:13.546 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 smithi110 running (2m) - 21m 1193M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 94b7643f91e8 2023-09-23T23:11:13.546 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 smithi110 running (27s) - 21m 13.9M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 ff8e0fe10bdc 2023-09-23T23:11:13.546 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 smithi138 running (20m) - 20m 5462M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 47f7252bf067 2023-09-23T23:11:13.546 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 smithi138 running (20m) - 20m 7301M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf fb3decabc2c4 2023-09-23T23:11:13.546 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 smithi138 running (19m) - 19m 5377M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf c3f9d2352b7c 2023-09-23T23:11:13.546 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 smithi138 running (19m) - 19m 5615M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 161d1d2d0445 2023-09-23T23:11:13.546 INFO:teuthology.orchestra.run.smithi110.stdout:prometheus.a smithi138 *:9095 running (13m) - 19m 69.1M - 2.43.0 a07b618ecd1d 8c2bb4977f43 2023-09-23T23:11:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:13 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 4687/136482 objects degraded (3.434%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:13 smithi138 ceph-mon[193606]: pgmap v451: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 4687/136482 objects degraded (3.434%); 0 B/s, 37 objects/s recovering 2023-09-23T23:11:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:13 smithi138 ceph-mon[193606]: from='client.54967 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:11:14.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:14.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:14.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:13 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 4687/136482 objects degraded (3.434%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:14.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:13 smithi110 ceph-mon[212939]: pgmap v451: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 4687/136482 objects degraded (3.434%); 0 B/s, 37 objects/s recovering 2023-09-23T23:11:14.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:13 smithi110 ceph-mon[212939]: from='client.54967 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:11:14.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:14.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:14.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:13 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 4687/136482 objects degraded (3.434%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:14.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:13 smithi110 ceph-mon[221102]: pgmap v451: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 4687/136482 objects degraded (3.434%); 0 B/s, 37 objects/s recovering 2023-09-23T23:11:14.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:13 smithi110 ceph-mon[221102]: from='client.54967 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:11:14.824 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph versions' 2023-09-23T23:11:15.135 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:15 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:11:15.400 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:15.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:15 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:11:15.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:15 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:11:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:16 smithi138 ceph-mon[193606]: osdmap e151: 8 total, 8 up, 8 in 2023-09-23T23:11:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:16 smithi138 ceph-mon[193606]: failed to encode map e151 with expected crc 2023-09-23T23:11:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:16 smithi138 ceph-mon[193606]: failed to encode map e151 with expected crc 2023-09-23T23:11:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:16 smithi138 ceph-mon[193606]: failed to encode map e151 with expected crc 2023-09-23T23:11:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:16 smithi138 ceph-mon[193606]: failed to encode map e151 with expected crc 2023-09-23T23:11:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:16 smithi138 ceph-mon[193606]: pgmap v453: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 4687/127422 objects degraded (3.678%); 0 B/s, 67 objects/s recovering 2023-09-23T23:11:16.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:16 smithi110 ceph-mon[212939]: osdmap e151: 8 total, 8 up, 8 in 2023-09-23T23:11:16.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:16 smithi110 ceph-mon[212939]: failed to encode map e151 with expected crc 2023-09-23T23:11:16.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:16 smithi110 ceph-mon[212939]: failed to encode map e151 with expected crc 2023-09-23T23:11:16.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:16 smithi110 ceph-mon[212939]: failed to encode map e151 with expected crc 2023-09-23T23:11:16.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:16 smithi110 ceph-mon[212939]: failed to encode map e151 with expected crc 2023-09-23T23:11:16.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:16 smithi110 ceph-mon[212939]: pgmap v453: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 4687/127422 objects degraded (3.678%); 0 B/s, 67 objects/s recovering 2023-09-23T23:11:16.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:16 smithi110 ceph-mon[221102]: osdmap e151: 8 total, 8 up, 8 in 2023-09-23T23:11:16.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:16 smithi110 ceph-mon[221102]: failed to encode map e151 with expected crc 2023-09-23T23:11:16.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:16 smithi110 ceph-mon[221102]: failed to encode map e151 with expected crc 2023-09-23T23:11:16.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:16 smithi110 ceph-mon[221102]: failed to encode map e151 with expected crc 2023-09-23T23:11:16.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:16 smithi110 ceph-mon[221102]: failed to encode map e151 with expected crc 2023-09-23T23:11:16.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:16 smithi110 ceph-mon[221102]: pgmap v453: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 4687/127422 objects degraded (3.678%); 0 B/s, 67 objects/s recovering 2023-09-23T23:11:17.096 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:11:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:11:16] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-09-23T23:11:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:17 smithi138 ceph-mon[193606]: failed to encode map e152 with expected crc 2023-09-23T23:11:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:17 smithi138 ceph-mon[193606]: osdmap e152: 8 total, 8 up, 8 in 2023-09-23T23:11:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:17 smithi138 ceph-mon[193606]: failed to encode map e152 with expected crc 2023-09-23T23:11:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:17 smithi138 ceph-mon[193606]: failed to encode map e152 with expected crc 2023-09-23T23:11:17.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:17 smithi110 ceph-mon[212939]: failed to encode map e152 with expected crc 2023-09-23T23:11:17.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:17 smithi110 ceph-mon[212939]: osdmap e152: 8 total, 8 up, 8 in 2023-09-23T23:11:17.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:17 smithi110 ceph-mon[212939]: failed to encode map e152 with expected crc 2023-09-23T23:11:17.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:17 smithi110 ceph-mon[212939]: failed to encode map e152 with expected crc 2023-09-23T23:11:17.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:17 smithi110 ceph-mon[221102]: failed to encode map e152 with expected crc 2023-09-23T23:11:17.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:17 smithi110 ceph-mon[221102]: osdmap e152: 8 total, 8 up, 8 in 2023-09-23T23:11:17.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:17 smithi110 ceph-mon[221102]: failed to encode map e152 with expected crc 2023-09-23T23:11:17.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:17 smithi110 ceph-mon[221102]: failed to encode map e152 with expected crc 2023-09-23T23:11:18.302 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:11:18.303 INFO:teuthology.orchestra.run.smithi110.stdout: "mon": { 2023-09-23T23:11:18.303 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T23:11:18.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:11:18.303 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr": { 2023-09-23T23:11:18.303 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T23:11:18.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:11:18.303 INFO:teuthology.orchestra.run.smithi110.stdout: "osd": { 2023-09-23T23:11:18.303 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)": 4, 2023-09-23T23:11:18.303 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T23:11:18.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:11:18.303 INFO:teuthology.orchestra.run.smithi110.stdout: "mds": { 2023-09-23T23:11:18.304 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)": 2 2023-09-23T23:11:18.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:11:18.304 INFO:teuthology.orchestra.run.smithi110.stdout: "overall": { 2023-09-23T23:11:18.304 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)": 6, 2023-09-23T23:11:18.304 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 9 2023-09-23T23:11:18.304 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:11:18.304 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:11:18.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:18 smithi138 ceph-mon[193606]: pgmap v455: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 3698/125379 objects degraded (2.949%); 0 B/s, 52 objects/s recovering 2023-09-23T23:11:18.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:18 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 4687/127422 objects degraded (3.678%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:18.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:18 smithi110 ceph-mon[212939]: pgmap v455: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 3698/125379 objects degraded (2.949%); 0 B/s, 52 objects/s recovering 2023-09-23T23:11:18.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:18 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 4687/127422 objects degraded (3.678%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:18.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:18 smithi110 ceph-mon[221102]: pgmap v455: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 3698/125379 objects degraded (2.949%); 0 B/s, 52 objects/s recovering 2023-09-23T23:11:18.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:18 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 4687/127422 objects degraded (3.678%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:19.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:19 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3118712387' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:11:19.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:19 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3118712387' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:11:19.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:19 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3118712387' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T23:11:19.621 DEBUG:teuthology.parallel:result is None 2023-09-23T23:11:20.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:20 smithi138 ceph-mon[193606]: pgmap v456: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 3698/120354 objects degraded (3.073%); 0 B/s, 67 objects/s recovering 2023-09-23T23:11:20.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:20 smithi110 ceph-mon[221102]: pgmap v456: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 3698/120354 objects degraded (3.073%); 0 B/s, 67 objects/s recovering 2023-09-23T23:11:20.460 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:20 smithi110 ceph-mon[212939]: pgmap v456: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 3698/120354 objects degraded (3.073%); 0 B/s, 67 objects/s recovering 2023-09-23T23:11:21.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:21 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:21 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:21 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:11:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:21 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:11:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:21 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:21 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:11:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:21 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:11:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:21 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:11:21.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:21 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:21.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:21 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:21.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:21 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:11:21.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:21 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:11:21.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:21 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:21.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:21 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:11:21.708 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:21 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:11:21.709 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:21 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:11:21.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:21 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:21.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:21 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:21.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:21 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:11:21.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:21 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:11:21.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:21 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:21.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:21 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:11:21.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:21 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:11:21.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:21 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:11:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:22 smithi138 ceph-mon[193606]: pgmap v457: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 3698/117291 objects degraded (3.153%); 0 B/s, 133 objects/s recovering 2023-09-23T23:11:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:22 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:22.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:22 smithi110 ceph-mon[221102]: pgmap v457: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 3698/117291 objects degraded (3.153%); 0 B/s, 133 objects/s recovering 2023-09-23T23:11:22.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:22 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:22.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:22 smithi110 ceph-mon[212939]: pgmap v457: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 3698/117291 objects degraded (3.153%); 0 B/s, 133 objects/s recovering 2023-09-23T23:11:22.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:22 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:23.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:23 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 3698/117291 objects degraded (3.153%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:23.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:23 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 3698/117291 objects degraded (3.153%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:23.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:23 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 3698/117291 objects degraded (3.153%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:24.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:24 smithi138 ceph-mon[193606]: pgmap v458: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 3698/116634 objects degraded (3.171%); 0 B/s, 87 objects/s recovering 2023-09-23T23:11:24.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:24 smithi110 ceph-mon[212939]: pgmap v458: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 3698/116634 objects degraded (3.171%); 0 B/s, 87 objects/s recovering 2023-09-23T23:11:24.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:24 smithi110 ceph-mon[221102]: pgmap v458: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 3698/116634 objects degraded (3.171%); 0 B/s, 87 objects/s recovering 2023-09-23T23:11:26.803 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:26 smithi110 ceph-mon[212939]: pgmap v459: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 3698/107793 objects degraded (3.431%); 0 B/s, 70 objects/s recovering 2023-09-23T23:11:26.803 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:26 smithi110 ceph-mon[212939]: failed to encode map e153 with expected crc 2023-09-23T23:11:26.803 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:26 smithi110 ceph-mon[212939]: osdmap e153: 8 total, 8 up, 8 in 2023-09-23T23:11:26.803 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:26 smithi110 ceph-mon[212939]: failed to encode map e153 with expected crc 2023-09-23T23:11:26.803 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:26 smithi110 ceph-mon[212939]: failed to encode map e153 with expected crc 2023-09-23T23:11:26.804 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:26 smithi110 ceph-mon[221102]: pgmap v459: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 3698/107793 objects degraded (3.431%); 0 B/s, 70 objects/s recovering 2023-09-23T23:11:26.804 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:26 smithi110 ceph-mon[221102]: failed to encode map e153 with expected crc 2023-09-23T23:11:26.804 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:26 smithi110 ceph-mon[221102]: osdmap e153: 8 total, 8 up, 8 in 2023-09-23T23:11:26.804 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:26 smithi110 ceph-mon[221102]: failed to encode map e153 with expected crc 2023-09-23T23:11:26.804 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:26 smithi110 ceph-mon[221102]: failed to encode map e153 with expected crc 2023-09-23T23:11:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:26 smithi138 ceph-mon[193606]: pgmap v459: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 3698/107793 objects degraded (3.431%); 0 B/s, 70 objects/s recovering 2023-09-23T23:11:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:26 smithi138 ceph-mon[193606]: failed to encode map e153 with expected crc 2023-09-23T23:11:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:26 smithi138 ceph-mon[193606]: osdmap e153: 8 total, 8 up, 8 in 2023-09-23T23:11:26.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:26 smithi138 ceph-mon[193606]: failed to encode map e153 with expected crc 2023-09-23T23:11:26.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:26 smithi138 ceph-mon[193606]: failed to encode map e153 with expected crc 2023-09-23T23:11:27.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:11:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:11:26] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-09-23T23:11:27.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:27 smithi138 ceph-mon[193606]: failed to encode map e154 with expected crc 2023-09-23T23:11:27.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:27 smithi138 ceph-mon[193606]: osdmap e154: 8 total, 8 up, 8 in 2023-09-23T23:11:27.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:27 smithi138 ceph-mon[193606]: failed to encode map e154 with expected crc 2023-09-23T23:11:27.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:27 smithi138 ceph-mon[193606]: failed to encode map e154 with expected crc 2023-09-23T23:11:27.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:27 smithi138 ceph-mon[193606]: failed to encode map e154 with expected crc 2023-09-23T23:11:27.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:27 smithi110 ceph-mon[212939]: failed to encode map e154 with expected crc 2023-09-23T23:11:27.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:27 smithi110 ceph-mon[212939]: osdmap e154: 8 total, 8 up, 8 in 2023-09-23T23:11:27.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:27 smithi110 ceph-mon[212939]: failed to encode map e154 with expected crc 2023-09-23T23:11:27.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:27 smithi110 ceph-mon[212939]: failed to encode map e154 with expected crc 2023-09-23T23:11:27.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:27 smithi110 ceph-mon[212939]: failed to encode map e154 with expected crc 2023-09-23T23:11:27.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:27 smithi110 ceph-mon[221102]: failed to encode map e154 with expected crc 2023-09-23T23:11:27.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:27 smithi110 ceph-mon[221102]: osdmap e154: 8 total, 8 up, 8 in 2023-09-23T23:11:27.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:27 smithi110 ceph-mon[221102]: failed to encode map e154 with expected crc 2023-09-23T23:11:27.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:27 smithi110 ceph-mon[221102]: failed to encode map e154 with expected crc 2023-09-23T23:11:27.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:27 smithi110 ceph-mon[221102]: failed to encode map e154 with expected crc 2023-09-23T23:11:28.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:28 smithi138 ceph-mon[193606]: pgmap v462: 89 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 3698/105462 objects degraded (3.506%); 0 B/s, 115 objects/s recovering 2023-09-23T23:11:28.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:28.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:28 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 3698/105462 objects degraded (3.506%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:28.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:28 smithi110 ceph-mon[212939]: pgmap v462: 89 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 3698/105462 objects degraded (3.506%); 0 B/s, 115 objects/s recovering 2023-09-23T23:11:28.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:28.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:28 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 3698/105462 objects degraded (3.506%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:28.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:28 smithi110 ceph-mon[221102]: pgmap v462: 89 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 3698/105462 objects degraded (3.506%); 0 B/s, 115 objects/s recovering 2023-09-23T23:11:28.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:28.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:28 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 3698/105462 objects degraded (3.506%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:30.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:30 smithi138 ceph-mon[193606]: pgmap v463: 89 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 3698/98937 objects degraded (3.738%); 0 B/s, 49 objects/s recovering 2023-09-23T23:11:30.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:30 smithi110 ceph-mon[212939]: pgmap v463: 89 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 3698/98937 objects degraded (3.738%); 0 B/s, 49 objects/s recovering 2023-09-23T23:11:30.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:30 smithi110 ceph-mon[221102]: pgmap v463: 89 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 3698/98937 objects degraded (3.738%); 0 B/s, 49 objects/s recovering 2023-09-23T23:11:32.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:32 smithi138 ceph-mon[193606]: pgmap v464: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 2771/95070 objects degraded (2.915%); 0 B/s, 105 objects/s recovering 2023-09-23T23:11:32.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:32 smithi110 ceph-mon[221102]: pgmap v464: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 2771/95070 objects degraded (2.915%); 0 B/s, 105 objects/s recovering 2023-09-23T23:11:32.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:32 smithi110 ceph-mon[212939]: pgmap v464: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 2771/95070 objects degraded (2.915%); 0 B/s, 105 objects/s recovering 2023-09-23T23:11:33.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:33 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2771/95070 objects degraded (2.915%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:33.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:33 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2771/95070 objects degraded (2.915%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:33.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:33 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2771/95070 objects degraded (2.915%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:34.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:34 smithi138 ceph-mon[193606]: pgmap v465: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 2771/94329 objects degraded (2.938%); 0 B/s, 105 objects/s recovering 2023-09-23T23:11:34.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:34 smithi110 ceph-mon[212939]: pgmap v465: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 2771/94329 objects degraded (2.938%); 0 B/s, 105 objects/s recovering 2023-09-23T23:11:34.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:34 smithi110 ceph-mon[221102]: pgmap v465: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 2771/94329 objects degraded (2.938%); 0 B/s, 105 objects/s recovering 2023-09-23T23:11:36.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:36 smithi138 ceph-mon[193606]: pgmap v466: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 2771/84945 objects degraded (3.262%); 0 B/s, 80 objects/s recovering 2023-09-23T23:11:36.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:36 smithi110 ceph-mon[212939]: pgmap v466: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 2771/84945 objects degraded (3.262%); 0 B/s, 80 objects/s recovering 2023-09-23T23:11:36.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:11:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:11:36] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-09-23T23:11:36.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:36 smithi110 ceph-mon[221102]: pgmap v466: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 2771/84945 objects degraded (3.262%); 0 B/s, 80 objects/s recovering 2023-09-23T23:11:37.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:37 smithi138 ceph-mon[193606]: failed to encode map e155 with expected crc 2023-09-23T23:11:37.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:37 smithi138 ceph-mon[193606]: osdmap e155: 8 total, 8 up, 8 in 2023-09-23T23:11:37.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:37 smithi138 ceph-mon[193606]: failed to encode map e155 with expected crc 2023-09-23T23:11:37.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:37 smithi138 ceph-mon[193606]: failed to encode map e155 with expected crc 2023-09-23T23:11:37.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:37 smithi110 ceph-mon[212939]: failed to encode map e155 with expected crc 2023-09-23T23:11:37.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:37 smithi110 ceph-mon[212939]: osdmap e155: 8 total, 8 up, 8 in 2023-09-23T23:11:37.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:37 smithi110 ceph-mon[212939]: failed to encode map e155 with expected crc 2023-09-23T23:11:37.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:37 smithi110 ceph-mon[212939]: failed to encode map e155 with expected crc 2023-09-23T23:11:37.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:37 smithi110 ceph-mon[221102]: failed to encode map e155 with expected crc 2023-09-23T23:11:37.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:37 smithi110 ceph-mon[221102]: osdmap e155: 8 total, 8 up, 8 in 2023-09-23T23:11:37.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:37 smithi110 ceph-mon[221102]: failed to encode map e155 with expected crc 2023-09-23T23:11:37.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:37 smithi110 ceph-mon[221102]: failed to encode map e155 with expected crc 2023-09-23T23:11:38.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:38 smithi138 ceph-mon[193606]: pgmap v468: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 2771/82653 objects degraded (3.353%); 0 B/s, 78 objects/s recovering 2023-09-23T23:11:38.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:38 smithi138 ceph-mon[193606]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:38 smithi138 ceph-mon[193606]: osdmap e156: 8 total, 8 up, 8 in 2023-09-23T23:11:38.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:38 smithi138 ceph-mon[193606]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:38 smithi138 ceph-mon[193606]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:38 smithi138 ceph-mon[193606]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:38 smithi138 ceph-mon[193606]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:38 smithi138 ceph-mon[193606]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:38 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2771/82653 objects degraded (3.353%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:38.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[212939]: pgmap v468: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 2771/82653 objects degraded (3.353%); 0 B/s, 78 objects/s recovering 2023-09-23T23:11:38.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[212939]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[212939]: osdmap e156: 8 total, 8 up, 8 in 2023-09-23T23:11:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[212939]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[212939]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[212939]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[212939]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[212939]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2771/82653 objects degraded (3.353%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:38.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[221102]: pgmap v468: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 2771/82653 objects degraded (3.353%); 0 B/s, 78 objects/s recovering 2023-09-23T23:11:38.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[221102]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[221102]: osdmap e156: 8 total, 8 up, 8 in 2023-09-23T23:11:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[221102]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[221102]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[221102]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[221102]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[221102]: failed to encode map e156 with expected crc 2023-09-23T23:11:38.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:38 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2771/82653 objects degraded (3.353%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:40.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:40 smithi138 ceph-mon[193606]: pgmap v470: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 1840/76341 objects degraded (2.410%); 0 B/s, 92 objects/s recovering 2023-09-23T23:11:40.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:40 smithi110 ceph-mon[212939]: pgmap v470: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 1840/76341 objects degraded (2.410%); 0 B/s, 92 objects/s recovering 2023-09-23T23:11:40.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:40 smithi110 ceph-mon[221102]: pgmap v470: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 1840/76341 objects degraded (2.410%); 0 B/s, 92 objects/s recovering 2023-09-23T23:11:42.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:42 smithi138 ceph-mon[193606]: pgmap v471: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 1840/72408 objects degraded (2.541%); 0 B/s, 110 objects/s recovering 2023-09-23T23:11:42.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:42 smithi110 ceph-mon[212939]: pgmap v471: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 1840/72408 objects degraded (2.541%); 0 B/s, 110 objects/s recovering 2023-09-23T23:11:42.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:42 smithi110 ceph-mon[221102]: pgmap v471: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 1840/72408 objects degraded (2.541%); 0 B/s, 110 objects/s recovering 2023-09-23T23:11:43.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:43.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:43 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1840/72408 objects degraded (2.541%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:43.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:43.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:43 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1840/72408 objects degraded (2.541%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:43.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:43.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:43 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1840/72408 objects degraded (2.541%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:44.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:44 smithi138 ceph-mon[193606]: pgmap v472: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 1840/71697 objects degraded (2.566%); 0 B/s, 68 objects/s recovering 2023-09-23T23:11:44.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:44 smithi110 ceph-mon[212939]: pgmap v472: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 1840/71697 objects degraded (2.566%); 0 B/s, 68 objects/s recovering 2023-09-23T23:11:44.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:44 smithi110 ceph-mon[221102]: pgmap v472: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 1840/71697 objects degraded (2.566%); 0 B/s, 68 objects/s recovering 2023-09-23T23:11:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:46 smithi138 ceph-mon[193606]: osdmap e157: 8 total, 8 up, 8 in 2023-09-23T23:11:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:46 smithi138 ceph-mon[193606]: failed to encode map e157 with expected crc 2023-09-23T23:11:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:46 smithi138 ceph-mon[193606]: failed to encode map e157 with expected crc 2023-09-23T23:11:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:46 smithi138 ceph-mon[193606]: pgmap v474: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 1840/62517 objects degraded (2.943%); 0 B/s, 120 objects/s recovering 2023-09-23T23:11:46.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:46 smithi110 ceph-mon[221102]: osdmap e157: 8 total, 8 up, 8 in 2023-09-23T23:11:46.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:46 smithi110 ceph-mon[221102]: failed to encode map e157 with expected crc 2023-09-23T23:11:46.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:46 smithi110 ceph-mon[221102]: failed to encode map e157 with expected crc 2023-09-23T23:11:46.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:46 smithi110 ceph-mon[221102]: pgmap v474: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 1840/62517 objects degraded (2.943%); 0 B/s, 120 objects/s recovering 2023-09-23T23:11:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:46 smithi110 ceph-mon[212939]: osdmap e157: 8 total, 8 up, 8 in 2023-09-23T23:11:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:46 smithi110 ceph-mon[212939]: failed to encode map e157 with expected crc 2023-09-23T23:11:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:46 smithi110 ceph-mon[212939]: failed to encode map e157 with expected crc 2023-09-23T23:11:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:46 smithi110 ceph-mon[212939]: pgmap v474: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 1840/62517 objects degraded (2.943%); 0 B/s, 120 objects/s recovering 2023-09-23T23:11:47.159 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:11:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:11:46] "GET /metrics HTTP/1.1" 200 36410 "" "Prometheus/2.43.0" 2023-09-23T23:11:47.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:47 smithi138 ceph-mon[193606]: failed to encode map e158 with expected crc 2023-09-23T23:11:47.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:47 smithi138 ceph-mon[193606]: failed to encode map e158 with expected crc 2023-09-23T23:11:47.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:47 smithi138 ceph-mon[193606]: osdmap e158: 8 total, 8 up, 8 in 2023-09-23T23:11:47.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:47 smithi110 ceph-mon[212939]: failed to encode map e158 with expected crc 2023-09-23T23:11:47.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:47 smithi110 ceph-mon[212939]: failed to encode map e158 with expected crc 2023-09-23T23:11:47.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:47 smithi110 ceph-mon[212939]: osdmap e158: 8 total, 8 up, 8 in 2023-09-23T23:11:47.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:47 smithi110 ceph-mon[221102]: failed to encode map e158 with expected crc 2023-09-23T23:11:47.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:47 smithi110 ceph-mon[221102]: failed to encode map e158 with expected crc 2023-09-23T23:11:47.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:47 smithi110 ceph-mon[221102]: osdmap e158: 8 total, 8 up, 8 in 2023-09-23T23:11:48.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:48 smithi138 ceph-mon[193606]: pgmap v476: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 1840/60414 objects degraded (3.046%); 0 B/s, 69 objects/s recovering 2023-09-23T23:11:48.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:48 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1840/62517 objects degraded (2.943%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:48.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:48 smithi110 ceph-mon[221102]: pgmap v476: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 1840/60414 objects degraded (3.046%); 0 B/s, 69 objects/s recovering 2023-09-23T23:11:48.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:48 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1840/62517 objects degraded (2.943%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:48.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:48 smithi110 ceph-mon[212939]: pgmap v476: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 1840/60414 objects degraded (3.046%); 0 B/s, 69 objects/s recovering 2023-09-23T23:11:48.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:48 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1840/62517 objects degraded (2.943%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:50.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:50 smithi138 ceph-mon[193606]: pgmap v477: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 936/54201 objects degraded (1.727%); 0 B/s, 132 objects/s recovering 2023-09-23T23:11:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:50 smithi110 ceph-mon[212939]: pgmap v477: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 936/54201 objects degraded (1.727%); 0 B/s, 132 objects/s recovering 2023-09-23T23:11:50.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:50 smithi110 ceph-mon[221102]: pgmap v477: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 936/54201 objects degraded (1.727%); 0 B/s, 132 objects/s recovering 2023-09-23T23:11:52.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:52 smithi138 ceph-mon[193606]: pgmap v478: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 896 op/s; 936/50163 objects degraded (1.866%); 0 B/s, 132 objects/s recovering 2023-09-23T23:11:52.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:52 smithi110 ceph-mon[221102]: pgmap v478: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 896 op/s; 936/50163 objects degraded (1.866%); 0 B/s, 132 objects/s recovering 2023-09-23T23:11:52.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:52 smithi110 ceph-mon[212939]: pgmap v478: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 896 op/s; 936/50163 objects degraded (1.866%); 0 B/s, 132 objects/s recovering 2023-09-23T23:11:53.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:53 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 936/50163 objects degraded (1.866%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:11:53.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:53 smithi138 ceph-mon[193606]: osdmap e159: 8 total, 8 up, 8 in 2023-09-23T23:11:53.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:53 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 936/50163 objects degraded (1.866%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:11:53.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:53 smithi110 ceph-mon[212939]: osdmap e159: 8 total, 8 up, 8 in 2023-09-23T23:11:53.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:53 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 936/50163 objects degraded (1.866%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:11:53.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:53 smithi110 ceph-mon[221102]: osdmap e159: 8 total, 8 up, 8 in 2023-09-23T23:11:54.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:54 smithi138 ceph-mon[193606]: failed to encode map e159 with expected crc 2023-09-23T23:11:54.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:54 smithi138 ceph-mon[193606]: failed to encode map e159 with expected crc 2023-09-23T23:11:54.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:54 smithi138 ceph-mon[193606]: pgmap v480: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1013 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 936/48651 objects degraded (1.924%); 0 B/s, 80 objects/s recovering 2023-09-23T23:11:54.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:54 smithi138 ceph-mon[193606]: osdmap e160: 8 total, 8 up, 8 in 2023-09-23T23:11:54.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:54 smithi110 ceph-mon[212939]: failed to encode map e159 with expected crc 2023-09-23T23:11:54.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:54 smithi110 ceph-mon[212939]: failed to encode map e159 with expected crc 2023-09-23T23:11:54.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:54 smithi110 ceph-mon[212939]: pgmap v480: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1013 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 936/48651 objects degraded (1.924%); 0 B/s, 80 objects/s recovering 2023-09-23T23:11:54.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:54 smithi110 ceph-mon[212939]: osdmap e160: 8 total, 8 up, 8 in 2023-09-23T23:11:54.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:54 smithi110 ceph-mon[221102]: failed to encode map e159 with expected crc 2023-09-23T23:11:54.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:54 smithi110 ceph-mon[221102]: failed to encode map e159 with expected crc 2023-09-23T23:11:54.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:54 smithi110 ceph-mon[221102]: pgmap v480: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1013 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 936/48651 objects degraded (1.924%); 0 B/s, 80 objects/s recovering 2023-09-23T23:11:54.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:54 smithi110 ceph-mon[221102]: osdmap e160: 8 total, 8 up, 8 in 2023-09-23T23:11:55.698 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:55 smithi138 ceph-mon[193606]: failed to encode map e160 with expected crc 2023-09-23T23:11:55.699 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:55 smithi138 ceph-mon[193606]: failed to encode map e160 with expected crc 2023-09-23T23:11:55.699 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:55 smithi138 ceph-mon[193606]: failed to encode map e160 with expected crc 2023-09-23T23:11:55.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:55 smithi110 ceph-mon[221102]: failed to encode map e160 with expected crc 2023-09-23T23:11:55.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:55 smithi110 ceph-mon[221102]: failed to encode map e160 with expected crc 2023-09-23T23:11:55.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:55 smithi110 ceph-mon[221102]: failed to encode map e160 with expected crc 2023-09-23T23:11:55.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:55 smithi110 ceph-mon[212939]: failed to encode map e160 with expected crc 2023-09-23T23:11:55.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:55 smithi110 ceph-mon[212939]: failed to encode map e160 with expected crc 2023-09-23T23:11:55.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:55 smithi110 ceph-mon[212939]: failed to encode map e160 with expected crc 2023-09-23T23:11:56.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:56 smithi138 ceph-mon[193606]: pgmap v482: 89 pgs: 2 active+recovering+undersized+remapped, 87 active+clean; 866 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 0 B/s, 84 objects/s recovering 2023-09-23T23:11:56.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:56 smithi138 ceph-mon[193606]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 936/48651 objects degraded (1.924%), 1 pg degraded) 2023-09-23T23:11:56.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:56 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:11:56.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:56 smithi110 ceph-mon[212939]: pgmap v482: 89 pgs: 2 active+recovering+undersized+remapped, 87 active+clean; 866 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 0 B/s, 84 objects/s recovering 2023-09-23T23:11:56.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:56 smithi110 ceph-mon[212939]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 936/48651 objects degraded (1.924%), 1 pg degraded) 2023-09-23T23:11:56.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:56 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:11:56.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:56 smithi110 ceph-mon[221102]: pgmap v482: 89 pgs: 2 active+recovering+undersized+remapped, 87 active+clean; 866 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 0 B/s, 84 objects/s recovering 2023-09-23T23:11:56.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:56 smithi110 ceph-mon[221102]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 936/48651 objects degraded (1.924%), 1 pg degraded) 2023-09-23T23:11:56.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:56 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:11:57.207 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:11:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:11:56] "GET /metrics HTTP/1.1" 200 36410 "" "Prometheus/2.43.0" 2023-09-23T23:11:58.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:58 smithi138 ceph-mon[193606]: pgmap v483: 89 pgs: 2 active+recovering+undersized+remapped, 87 active+clean; 826 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 0 B/s, 4 objects/s recovering 2023-09-23T23:11:58.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:58.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:11:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:58 smithi110 ceph-mon[221102]: pgmap v483: 89 pgs: 2 active+recovering+undersized+remapped, 87 active+clean; 826 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 0 B/s, 4 objects/s recovering 2023-09-23T23:11:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:11:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:58.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:58 smithi110 ceph-mon[212939]: pgmap v483: 89 pgs: 2 active+recovering+undersized+remapped, 87 active+clean; 826 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 0 B/s, 4 objects/s recovering 2023-09-23T23:11:58.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:11:58.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:11:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:00.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:00 smithi138 ceph-mon[193606]: pgmap v484: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 670 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 0 B/s, 121 objects/s recovering 2023-09-23T23:12:00.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:00 smithi110 ceph-mon[212939]: pgmap v484: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 670 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 0 B/s, 121 objects/s recovering 2023-09-23T23:12:00.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:00 smithi110 ceph-mon[221102]: pgmap v484: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 670 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 0 B/s, 121 objects/s recovering 2023-09-23T23:12:01.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:01 smithi138 ceph-mon[193606]: Health check failed: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-09-23T23:12:01.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:01 smithi110 ceph-mon[212939]: Health check failed: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-09-23T23:12:01.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:01 smithi110 ceph-mon[221102]: Health check failed: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-09-23T23:12:02.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:02 smithi138 ceph-mon[193606]: pgmap v485: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 571 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 0 B/s, 119 objects/s recovering 2023-09-23T23:12:02.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:02 smithi110 ceph-mon[212939]: pgmap v485: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 571 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 0 B/s, 119 objects/s recovering 2023-09-23T23:12:02.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:02 smithi110 ceph-mon[221102]: pgmap v485: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 571 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 0 B/s, 119 objects/s recovering 2023-09-23T23:12:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:03 smithi138 ceph-mon[193606]: osdmap e161: 8 total, 8 up, 8 in 2023-09-23T23:12:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:03 smithi138 ceph-mon[193606]: failed to encode map e161 with expected crc 2023-09-23T23:12:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:03 smithi138 ceph-mon[193606]: failed to encode map e161 with expected crc 2023-09-23T23:12:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:03 smithi138 ceph-mon[193606]: failed to encode map e161 with expected crc 2023-09-23T23:12:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:03 smithi138 ceph-mon[193606]: failed to encode map e161 with expected crc 2023-09-23T23:12:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:03 smithi138 ceph-mon[193606]: failed to encode map e161 with expected crc 2023-09-23T23:12:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:03 smithi138 ceph-mon[193606]: failed to encode map e161 with expected crc 2023-09-23T23:12:03.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:03 smithi110 ceph-mon[212939]: osdmap e161: 8 total, 8 up, 8 in 2023-09-23T23:12:03.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:03 smithi110 ceph-mon[212939]: failed to encode map e161 with expected crc 2023-09-23T23:12:03.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:03 smithi110 ceph-mon[212939]: failed to encode map e161 with expected crc 2023-09-23T23:12:03.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:03 smithi110 ceph-mon[212939]: failed to encode map e161 with expected crc 2023-09-23T23:12:03.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:03 smithi110 ceph-mon[212939]: failed to encode map e161 with expected crc 2023-09-23T23:12:03.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:03 smithi110 ceph-mon[212939]: failed to encode map e161 with expected crc 2023-09-23T23:12:03.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:03 smithi110 ceph-mon[212939]: failed to encode map e161 with expected crc 2023-09-23T23:12:03.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:03 smithi110 ceph-mon[221102]: osdmap e161: 8 total, 8 up, 8 in 2023-09-23T23:12:03.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:03 smithi110 ceph-mon[221102]: failed to encode map e161 with expected crc 2023-09-23T23:12:03.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:03 smithi110 ceph-mon[221102]: failed to encode map e161 with expected crc 2023-09-23T23:12:03.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:03 smithi110 ceph-mon[221102]: failed to encode map e161 with expected crc 2023-09-23T23:12:03.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:03 smithi110 ceph-mon[221102]: failed to encode map e161 with expected crc 2023-09-23T23:12:03.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:03 smithi110 ceph-mon[221102]: failed to encode map e161 with expected crc 2023-09-23T23:12:03.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:03 smithi110 ceph-mon[221102]: failed to encode map e161 with expected crc 2023-09-23T23:12:04.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:04 smithi138 ceph-mon[193606]: pgmap v487: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 519 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 0 B/s, 102 objects/s recovering 2023-09-23T23:12:04.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:04 smithi138 ceph-mon[193606]: failed to encode map e162 with expected crc 2023-09-23T23:12:04.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:04 smithi138 ceph-mon[193606]: osdmap e162: 8 total, 8 up, 8 in 2023-09-23T23:12:04.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:04 smithi138 ceph-mon[193606]: failed to encode map e162 with expected crc 2023-09-23T23:12:04.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:04 smithi138 ceph-mon[193606]: failed to encode map e162 with expected crc 2023-09-23T23:12:04.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:04 smithi138 ceph-mon[193606]: failed to encode map e162 with expected crc 2023-09-23T23:12:04.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:04 smithi110 ceph-mon[221102]: pgmap v487: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 519 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 0 B/s, 102 objects/s recovering 2023-09-23T23:12:04.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:04 smithi110 ceph-mon[221102]: failed to encode map e162 with expected crc 2023-09-23T23:12:04.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:04 smithi110 ceph-mon[221102]: osdmap e162: 8 total, 8 up, 8 in 2023-09-23T23:12:04.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:04 smithi110 ceph-mon[221102]: failed to encode map e162 with expected crc 2023-09-23T23:12:04.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:04 smithi110 ceph-mon[221102]: failed to encode map e162 with expected crc 2023-09-23T23:12:04.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:04 smithi110 ceph-mon[221102]: failed to encode map e162 with expected crc 2023-09-23T23:12:04.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:04 smithi110 ceph-mon[212939]: pgmap v487: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 519 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 0 B/s, 102 objects/s recovering 2023-09-23T23:12:04.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:04 smithi110 ceph-mon[212939]: failed to encode map e162 with expected crc 2023-09-23T23:12:04.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:04 smithi110 ceph-mon[212939]: osdmap e162: 8 total, 8 up, 8 in 2023-09-23T23:12:04.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:04 smithi110 ceph-mon[212939]: failed to encode map e162 with expected crc 2023-09-23T23:12:04.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:04 smithi110 ceph-mon[212939]: failed to encode map e162 with expected crc 2023-09-23T23:12:04.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:04 smithi110 ceph-mon[212939]: failed to encode map e162 with expected crc 2023-09-23T23:12:05.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:05 smithi138 ceph-mon[193606]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-09-23T23:12:05.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:05 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:12:05.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:05 smithi110 ceph-mon[221102]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-09-23T23:12:05.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:05 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:12:05.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:05 smithi110 ceph-mon[212939]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-09-23T23:12:05.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:05 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:12:06.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:06 smithi138 ceph-mon[193606]: pgmap v489: 89 pgs: 1 peering, 88 active+clean; 384 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 0 B/s, 159 objects/s recovering 2023-09-23T23:12:06.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:06 smithi110 ceph-mon[221102]: pgmap v489: 89 pgs: 1 peering, 88 active+clean; 384 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 0 B/s, 159 objects/s recovering 2023-09-23T23:12:06.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:06 smithi110 ceph-mon[212939]: pgmap v489: 89 pgs: 1 peering, 88 active+clean; 384 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 0 B/s, 159 objects/s recovering 2023-09-23T23:12:06.954 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:12:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:12:06] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.43.0" 2023-09-23T23:12:08.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:08 smithi138 ceph-mon[193606]: pgmap v490: 89 pgs: 1 peering, 88 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 0 B/s, 42 objects/s recovering 2023-09-23T23:12:08.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:08 smithi110 ceph-mon[221102]: pgmap v490: 89 pgs: 1 peering, 88 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 0 B/s, 42 objects/s recovering 2023-09-23T23:12:08.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:08 smithi110 ceph-mon[212939]: pgmap v490: 89 pgs: 1 peering, 88 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 0 B/s, 42 objects/s recovering 2023-09-23T23:12:10.656 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Removed 118815 objects 2023-09-23T23:12:10.657 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Clean up completed and total clean up time :160.824 2023-09-23T23:12:10.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:10 smithi138 ceph-mon[193606]: pgmap v491: 89 pgs: 89 active+clean; 190 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 0 B/s, 42 objects/s recovering 2023-09-23T23:12:10.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:10 smithi110 ceph-mon[212939]: pgmap v491: 89 pgs: 89 active+clean; 190 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 0 B/s, 42 objects/s recovering 2023-09-23T23:12:10.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:10 smithi110 ceph-mon[221102]: pgmap v491: 89 pgs: 89 active+clean; 190 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 0 B/s, 42 objects/s recovering 2023-09-23T23:12:12.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:12 smithi138 ceph-mon[193606]: pgmap v492: 89 pgs: 89 active+clean; 90 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 0 B/s, 38 objects/s recovering 2023-09-23T23:12:12.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:12 smithi110 ceph-mon[212939]: pgmap v492: 89 pgs: 89 active+clean; 90 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 0 B/s, 38 objects/s recovering 2023-09-23T23:12:12.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:12 smithi110 ceph-mon[221102]: pgmap v492: 89 pgs: 89 active+clean; 90 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 0 B/s, 38 objects/s recovering 2023-09-23T23:12:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:13.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:14.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:14 smithi138 ceph-mon[193606]: pgmap v493: 89 pgs: 89 active+clean; 62 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 0 B/s, 33 objects/s recovering 2023-09-23T23:12:14.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:14 smithi110 ceph-mon[212939]: pgmap v493: 89 pgs: 89 active+clean; 62 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 0 B/s, 33 objects/s recovering 2023-09-23T23:12:14.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:14 smithi110 ceph-mon[221102]: pgmap v493: 89 pgs: 89 active+clean; 62 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 0 B/s, 33 objects/s recovering 2023-09-23T23:12:15.962 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-09-23T23:12:15.963 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-09-23T23:12:16.363 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:16.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:16 smithi138 ceph-mon[193606]: pgmap v494: 89 pgs: 89 active+clean; 2.3 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T23:12:16.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:16 smithi110 ceph-mon[212939]: pgmap v494: 89 pgs: 89 active+clean; 2.3 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T23:12:16.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:12:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:12:16] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.43.0" 2023-09-23T23:12:16.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:16 smithi110 ceph-mon[221102]: pgmap v494: 89 pgs: 89 active+clean; 2.3 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T23:12:18.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:18 smithi138 ceph-mon[193606]: pgmap v495: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 511 op/s 2023-09-23T23:12:18.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2903142549' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:12:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:18 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:18.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:18 smithi110 ceph-mon[221102]: pgmap v495: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 511 op/s 2023-09-23T23:12:18.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2903142549' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:12:18.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:18 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:18.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:18 smithi110 ceph-mon[212939]: pgmap v495: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 511 op/s 2023-09-23T23:12:18.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2903142549' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:12:18.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:18 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:19.143 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_3' removed 2023-09-23T23:12:19.890 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T23:12:19.890 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-09-23T23:12:19.892 DEBUG:teuthology.parallel:result is None 2023-09-23T23:12:19.892 INFO:teuthology.run_tasks:Running task print... 2023-09-23T23:12:19.901 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-09-23T23:12:19.901 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T23:12:19.909 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T23:12:19.909 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T23:12:19.909 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T23:12:20.136 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:20 smithi110 ceph-mon[212939]: failed to encode map e163 with expected crc 2023-09-23T23:12:20.137 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:20 smithi110 ceph-mon[221102]: failed to encode map e163 with expected crc 2023-09-23T23:12:20.138 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:20 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2903142549' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:12:20.138 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:20 smithi110 ceph-mon[221102]: osdmap e163: 8 total, 8 up, 8 in 2023-09-23T23:12:20.138 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:20 smithi110 ceph-mon[221102]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:20.138 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:20 smithi110 ceph-mon[221102]: failed to encode map e163 with expected crc 2023-09-23T23:12:20.283 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:20.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:20 smithi138 ceph-mon[193606]: failed to encode map e163 with expected crc 2023-09-23T23:12:20.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:20 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2903142549' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:12:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:20 smithi138 ceph-mon[193606]: osdmap e163: 8 total, 8 up, 8 in 2023-09-23T23:12:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:20 smithi138 ceph-mon[193606]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:20 smithi138 ceph-mon[193606]: failed to encode map e163 with expected crc 2023-09-23T23:12:20.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:20 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2903142549' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:12:20.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:20 smithi110 ceph-mon[212939]: osdmap e163: 8 total, 8 up, 8 in 2023-09-23T23:12:20.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:20 smithi110 ceph-mon[212939]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:20.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:20 smithi110 ceph-mon[212939]: failed to encode map e163 with expected crc 2023-09-23T23:12:21.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:21 smithi110 ceph-mon[212939]: failed to encode map e163 with expected crc 2023-09-23T23:12:21.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:21 smithi110 ceph-mon[221102]: failed to encode map e163 with expected crc 2023-09-23T23:12:21.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:21 smithi138 ceph-mon[193606]: failed to encode map e163 with expected crc 2023-09-23T23:12:22.139 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:22 smithi110 ceph-mon[212939]: failed to encode map e163 with expected crc 2023-09-23T23:12:22.139 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:22 smithi110 ceph-mon[212939]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:22.140 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:22 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:12:22.140 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:22 smithi110 ceph-mon[221102]: failed to encode map e163 with expected crc 2023-09-23T23:12:22.140 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:22 smithi110 ceph-mon[221102]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:22.140 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:22 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:12:22.255 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:12:22.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:22 smithi138 ceph-mon[193606]: failed to encode map e163 with expected crc 2023-09-23T23:12:22.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:22 smithi138 ceph-mon[193606]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:22.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:22 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:12:23.299 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":35,"active_gid":25208,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6800","nonce":1116120623},{"type":"v1","addr":"172.21.15.110:6801","nonce":1116120623}]},"active_addr":"172.21.15.110:6801/1116120623","active_change":"2023-09-23T22:58:42.842462+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":34098,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.110:8443/","prometheus":"http://172.21.15.110:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":77,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":1237510003}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":4234265972}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":2788143216}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.110:0","nonce":2821868301}]}]} 2023-09-23T23:12:23.303 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T23:12:23.304 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T23:12:23.304 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T23:12:23.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:23 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1488717290' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T23:12:23.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:23 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1488717290' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T23:12:23.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:23 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1488717290' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T23:12:23.760 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:24.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:24 smithi138 ceph-mon[193606]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:24.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:24 smithi110 ceph-mon[212939]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:24.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:24 smithi110 ceph-mon[221102]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:26.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:25 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:26.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:25 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:26.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:25 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:26.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:25 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:26.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:25 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:26.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:25 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:26.371 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:12:26.371 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":163,"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:46:49.032269+0000","modified":"2023-09-23T23:12:19.130604+0000","last_up_change":"2023-09-23T23:10:58.470482+0000","last_in_change":"2023-09-23T22:51:26.130429+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:49:49.263399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:52:19.908332+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:53:15.155816+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:53:16.112056+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"78791acb-c93c-4e4f-bf04-ea3eee46408d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":153,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":983471373},{"type":"v1","addr":"172.21.15.110:6803","nonce":983471373}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":983471373},{"type":"v1","addr":"172.21.15.110:6805","nonce":983471373}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":983471373},{"type":"v1","addr":"172.21.15.110:6809","nonce":983471373}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":983471373},{"type":"v1","addr":"172.21.15.110:6807","nonce":983471373}]},"public_addr":"172.21.15.110:6803/983471373","cluster_addr":"172.21.15.110:6805/983471373","heartbeat_back_addr":"172.21.15.110:6809/983471373","heartbeat_front_addr":"172.21.15.110:6807/983471373","state":["exists","up"]},{"osd":1,"uuid":"6793178f-2ef1-4c48-927c-831a083a3356","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":107,"up_from":112,"up_thru":161,"down_at":108,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":594715751},{"type":"v1","addr":"172.21.15.110:6811","nonce":594715751}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":594715751},{"type":"v1","addr":"172.21.15.110:6813","nonce":594715751}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":594715751},{"type":"v1","addr":"172.21.15.110:6817","nonce":594715751}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":594715751},{"type":"v1","addr":"172.21.15.110:6815","nonce":594715751}]},"public_addr":"172.21.15.110:6811/594715751","cluster_addr":"172.21.15.110:6813/594715751","heartbeat_back_addr":"172.21.15.110:6817/594715751","heartbeat_front_addr":"172.21.15.110:6815/594715751","state":["exists","up"]},{"osd":2,"uuid":"146fbd28-a365-4545-8e97-d629c6ddba21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":126,"up_from":130,"up_thru":148,"down_at":127,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3154710012},{"type":"v1","addr":"172.21.15.110:6819","nonce":3154710012}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3154710012},{"type":"v1","addr":"172.21.15.110:6821","nonce":3154710012}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":3154710012},{"type":"v1","addr":"172.21.15.110:6825","nonce":3154710012}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3154710012},{"type":"v1","addr":"172.21.15.110:6823","nonce":3154710012}]},"public_addr":"172.21.15.110:6819/3154710012","cluster_addr":"172.21.15.110:6821/3154710012","heartbeat_back_addr":"172.21.15.110:6825/3154710012","heartbeat_front_addr":"172.21.15.110:6823/3154710012","state":["exists","up"]},{"osd":3,"uuid":"6fcf80d5-6977-4929-b7ba-0e9ab65df0a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":144,"up_from":148,"up_thru":159,"down_at":145,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":2333701749},{"type":"v1","addr":"172.21.15.110:6827","nonce":2333701749}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":2333701749},{"type":"v1","addr":"172.21.15.110:6829","nonce":2333701749}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":2333701749},{"type":"v1","addr":"172.21.15.110:6833","nonce":2333701749}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":2333701749},{"type":"v1","addr":"172.21.15.110:6831","nonce":2333701749}]},"public_addr":"172.21.15.110:6827/2333701749","cluster_addr":"172.21.15.110:6829/2333701749","heartbeat_back_addr":"172.21.15.110:6833/2333701749","heartbeat_front_addr":"172.21.15.110:6831/2333701749","state":["exists","up"]},{"osd":4,"uuid":"131d0749-c46c-44f0-ab7f-03f8ad6ed502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":148,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6801","nonce":941787557}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6803","nonce":941787557}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6807","nonce":941787557}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6805","nonce":941787557}]},"public_addr":"172.21.15.138:6801/941787557","cluster_addr":"172.21.15.138:6803/941787557","heartbeat_back_addr":"172.21.15.138:6807/941787557","heartbeat_front_addr":"172.21.15.138:6805/941787557","state":["exists","up"]},{"osd":5,"uuid":"8cb496e5-d242-4e41-b279-e8d5ecea2996","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":155,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6809","nonce":3098195887}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6811","nonce":3098195887}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6815","nonce":3098195887}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6813","nonce":3098195887}]},"public_addr":"172.21.15.138:6809/3098195887","cluster_addr":"172.21.15.138:6811/3098195887","heartbeat_back_addr":"172.21.15.138:6815/3098195887","heartbeat_front_addr":"172.21.15.138:6813/3098195887","state":["exists","up"]},{"osd":6,"uuid":"c709e629-85ab-4273-9ca8-f2c4e8fc9074","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":145,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6817","nonce":187413403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6819","nonce":187413403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6823","nonce":187413403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6821","nonce":187413403}]},"public_addr":"172.21.15.138:6817/187413403","cluster_addr":"172.21.15.138:6819/187413403","heartbeat_back_addr":"172.21.15.138:6823/187413403","heartbeat_front_addr":"172.21.15.138:6821/187413403","state":["exists","up"]},{"osd":7,"uuid":"ed15f3b0-cd2e-43b7-b803-e45252b62a45","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":148,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6825","nonce":676244673}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6827","nonce":676244673}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6831","nonce":676244673}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6829","nonce":676244673}]},"public_addr":"172.21.15.138:6825/676244673","cluster_addr":"172.21.15.138:6827/676244673","heartbeat_back_addr":"172.21.15.138:6831/676244673","heartbeat_front_addr":"172.21.15.138:6829/676244673","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:48:58.523930+0000","dead_epoch":87},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:22.993201+0000","dead_epoch":107},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:46.747264+0000","dead_epoch":126},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:11.206564+0000","dead_epoch":144},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:33.511236+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:56.364297+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:18.141750+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:41.242882+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.b","mappings":[{"from":1,"to":5}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:0/3250066431":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/3705879030":"2023-09-24T22:58:42.842267+0000","172.21.15.110:0/1399491671":"2023-09-24T22:56:16.309980+0000","172.21.15.110:0/1649304640":"2023-09-24T22:47:38.935467+0000","172.21.15.138:0/2647044257":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/1326657364":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1731974664":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/1506915348":"2023-09-24T22:47:15.528545+0000","172.21.15.138:6834/867141825":"2023-09-24T22:57:33.926232+0000","172.21.15.110:6800/1771333922":"2023-09-24T22:58:42.842267+0000","172.21.15.110:6801/4228126008":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/478775577":"2023-09-24T22:58:42.842267+0000","172.21.15.110:0/380987145":"2023-09-24T22:56:16.309980+0000","172.21.15.110:6800/4228126008":"2023-09-24T22:47:38.935467+0000","172.21.15.110:6800/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.138:0/1613829186":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/3889938710":"2023-09-24T22:58:42.842267+0000","172.21.15.110:6801/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/657635177":"2023-09-24T22:52:17.775108+0000","172.21.15.138:0/331679217":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/410644117":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6801/2914393040":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/3674528218":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6801/576619896":"2023-09-24T22:56:16.309980+0000","172.21.15.110:0/2850586742":"2023-09-24T22:56:16.309980+0000","172.21.15.110:0/2523943302":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6801/1771333922":"2023-09-24T22:58:42.842267+0000","172.21.15.110:0/653682064":"2023-09-24T22:58:42.842267+0000","172.21.15.138:0/3577646480":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/3669919402":"2023-09-24T22:58:42.842267+0000","172.21.15.110:6800/576619896":"2023-09-24T22:56:16.309980+0000","172.21.15.138:0/3320282773":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/3464832160":"2023-09-24T22:56:16.309980+0000","172.21.15.138:6835/867141825":"2023-09-24T22:57:33.926232+0000","172.21.15.110:6800/2914393040":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/3412430555":"2023-09-24T22:47:38.935467+0000","172.21.15.138:0/3258575069":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/2804253682":"2023-09-24T22:58:42.842267+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T23:12:27.099 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:26 smithi110 ceph-mon[212939]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:27.099 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:26 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:27.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:26 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:27.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:12:27.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:26 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:12:27.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:26 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:27.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:26 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2949578766' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:12:27.100 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:12:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:12:26] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.43.0" 2023-09-23T23:12:27.100 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:26 smithi110 ceph-mon[221102]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:27.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:26 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:27.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:26 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:27.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:12:27.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:26 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:12:27.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:26 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:27.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:26 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2949578766' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:12:27.114 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T23:12:27.114 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T23:12:27.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:26 smithi138 ceph-mon[193606]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:27.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:26 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:27.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:26 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:27.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:12:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:26 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:12:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:26 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:12:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:26 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2949578766' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:12:27.458 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:29.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:28 smithi138 ceph-mon[193606]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:29.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:29.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:28 smithi110 ceph-mon[212939]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:29.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:29.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:28 smithi110 ceph-mon[221102]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:29.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:29.248 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:12:29.248 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":163,"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:46:49.032269+0000","modified":"2023-09-23T23:12:19.130604+0000","last_up_change":"2023-09-23T23:10:58.470482+0000","last_in_change":"2023-09-23T22:51:26.130429+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:49:49.263399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:52:19.908332+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:53:15.155816+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:53:16.112056+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"78791acb-c93c-4e4f-bf04-ea3eee46408d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":153,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":983471373},{"type":"v1","addr":"172.21.15.110:6803","nonce":983471373}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":983471373},{"type":"v1","addr":"172.21.15.110:6805","nonce":983471373}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":983471373},{"type":"v1","addr":"172.21.15.110:6809","nonce":983471373}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":983471373},{"type":"v1","addr":"172.21.15.110:6807","nonce":983471373}]},"public_addr":"172.21.15.110:6803/983471373","cluster_addr":"172.21.15.110:6805/983471373","heartbeat_back_addr":"172.21.15.110:6809/983471373","heartbeat_front_addr":"172.21.15.110:6807/983471373","state":["exists","up"]},{"osd":1,"uuid":"6793178f-2ef1-4c48-927c-831a083a3356","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":107,"up_from":112,"up_thru":161,"down_at":108,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":594715751},{"type":"v1","addr":"172.21.15.110:6811","nonce":594715751}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":594715751},{"type":"v1","addr":"172.21.15.110:6813","nonce":594715751}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":594715751},{"type":"v1","addr":"172.21.15.110:6817","nonce":594715751}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":594715751},{"type":"v1","addr":"172.21.15.110:6815","nonce":594715751}]},"public_addr":"172.21.15.110:6811/594715751","cluster_addr":"172.21.15.110:6813/594715751","heartbeat_back_addr":"172.21.15.110:6817/594715751","heartbeat_front_addr":"172.21.15.110:6815/594715751","state":["exists","up"]},{"osd":2,"uuid":"146fbd28-a365-4545-8e97-d629c6ddba21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":126,"up_from":130,"up_thru":148,"down_at":127,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3154710012},{"type":"v1","addr":"172.21.15.110:6819","nonce":3154710012}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3154710012},{"type":"v1","addr":"172.21.15.110:6821","nonce":3154710012}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":3154710012},{"type":"v1","addr":"172.21.15.110:6825","nonce":3154710012}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3154710012},{"type":"v1","addr":"172.21.15.110:6823","nonce":3154710012}]},"public_addr":"172.21.15.110:6819/3154710012","cluster_addr":"172.21.15.110:6821/3154710012","heartbeat_back_addr":"172.21.15.110:6825/3154710012","heartbeat_front_addr":"172.21.15.110:6823/3154710012","state":["exists","up"]},{"osd":3,"uuid":"6fcf80d5-6977-4929-b7ba-0e9ab65df0a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":144,"up_from":148,"up_thru":159,"down_at":145,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":2333701749},{"type":"v1","addr":"172.21.15.110:6827","nonce":2333701749}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":2333701749},{"type":"v1","addr":"172.21.15.110:6829","nonce":2333701749}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":2333701749},{"type":"v1","addr":"172.21.15.110:6833","nonce":2333701749}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":2333701749},{"type":"v1","addr":"172.21.15.110:6831","nonce":2333701749}]},"public_addr":"172.21.15.110:6827/2333701749","cluster_addr":"172.21.15.110:6829/2333701749","heartbeat_back_addr":"172.21.15.110:6833/2333701749","heartbeat_front_addr":"172.21.15.110:6831/2333701749","state":["exists","up"]},{"osd":4,"uuid":"131d0749-c46c-44f0-ab7f-03f8ad6ed502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":148,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6801","nonce":941787557}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6803","nonce":941787557}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6807","nonce":941787557}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":941787557},{"type":"v1","addr":"172.21.15.138:6805","nonce":941787557}]},"public_addr":"172.21.15.138:6801/941787557","cluster_addr":"172.21.15.138:6803/941787557","heartbeat_back_addr":"172.21.15.138:6807/941787557","heartbeat_front_addr":"172.21.15.138:6805/941787557","state":["exists","up"]},{"osd":5,"uuid":"8cb496e5-d242-4e41-b279-e8d5ecea2996","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":155,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6809","nonce":3098195887}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6811","nonce":3098195887}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6815","nonce":3098195887}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6813","nonce":3098195887}]},"public_addr":"172.21.15.138:6809/3098195887","cluster_addr":"172.21.15.138:6811/3098195887","heartbeat_back_addr":"172.21.15.138:6815/3098195887","heartbeat_front_addr":"172.21.15.138:6813/3098195887","state":["exists","up"]},{"osd":6,"uuid":"c709e629-85ab-4273-9ca8-f2c4e8fc9074","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":145,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6817","nonce":187413403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6819","nonce":187413403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6823","nonce":187413403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6821","nonce":187413403}]},"public_addr":"172.21.15.138:6817/187413403","cluster_addr":"172.21.15.138:6819/187413403","heartbeat_back_addr":"172.21.15.138:6823/187413403","heartbeat_front_addr":"172.21.15.138:6821/187413403","state":["exists","up"]},{"osd":7,"uuid":"ed15f3b0-cd2e-43b7-b803-e45252b62a45","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":148,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6825","nonce":676244673}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6827","nonce":676244673}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6831","nonce":676244673}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":676244673},{"type":"v1","addr":"172.21.15.138:6829","nonce":676244673}]},"public_addr":"172.21.15.138:6825/676244673","cluster_addr":"172.21.15.138:6827/676244673","heartbeat_back_addr":"172.21.15.138:6831/676244673","heartbeat_front_addr":"172.21.15.138:6829/676244673","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:48:58.523930+0000","dead_epoch":87},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:22.993201+0000","dead_epoch":107},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:46.747264+0000","dead_epoch":126},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:11.206564+0000","dead_epoch":144},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:33.511236+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:56.364297+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:18.141750+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:41.242882+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.b","mappings":[{"from":1,"to":5}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:0/3250066431":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/3705879030":"2023-09-24T22:58:42.842267+0000","172.21.15.110:0/1399491671":"2023-09-24T22:56:16.309980+0000","172.21.15.110:0/1649304640":"2023-09-24T22:47:38.935467+0000","172.21.15.138:0/2647044257":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/1326657364":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1731974664":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/1506915348":"2023-09-24T22:47:15.528545+0000","172.21.15.138:6834/867141825":"2023-09-24T22:57:33.926232+0000","172.21.15.110:6800/1771333922":"2023-09-24T22:58:42.842267+0000","172.21.15.110:6801/4228126008":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/478775577":"2023-09-24T22:58:42.842267+0000","172.21.15.110:0/380987145":"2023-09-24T22:56:16.309980+0000","172.21.15.110:6800/4228126008":"2023-09-24T22:47:38.935467+0000","172.21.15.110:6800/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.138:0/1613829186":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/3889938710":"2023-09-24T22:58:42.842267+0000","172.21.15.110:6801/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/657635177":"2023-09-24T22:52:17.775108+0000","172.21.15.138:0/331679217":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/410644117":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6801/2914393040":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/3674528218":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6801/576619896":"2023-09-24T22:56:16.309980+0000","172.21.15.110:0/2850586742":"2023-09-24T22:56:16.309980+0000","172.21.15.110:0/2523943302":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6801/1771333922":"2023-09-24T22:58:42.842267+0000","172.21.15.110:0/653682064":"2023-09-24T22:58:42.842267+0000","172.21.15.138:0/3577646480":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/3669919402":"2023-09-24T22:58:42.842267+0000","172.21.15.110:6800/576619896":"2023-09-24T22:56:16.309980+0000","172.21.15.138:0/3320282773":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/3464832160":"2023-09-24T22:56:16.309980+0000","172.21.15.138:6835/867141825":"2023-09-24T22:57:33.926232+0000","172.21.15.110:6800/2914393040":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/3412430555":"2023-09-24T22:47:38.935467+0000","172.21.15.138:0/3258575069":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/2804253682":"2023-09-24T22:58:42.842267+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T23:12:29.957 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T23:12:29.957 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T23:12:29.957 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T23:12:29.958 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T23:12:29.958 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T23:12:29.959 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T23:12:29.959 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T23:12:29.960 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T23:12:30.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:29 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1430632197' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:12:30.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:29 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1430632197' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:12:30.214 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:29 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1430632197' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:12:31.068 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:31.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:30 smithi138 ceph-mon[193606]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:31.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:30 smithi110 ceph-mon[212939]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:31.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:30 smithi110 ceph-mon[221102]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:31.225 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:31.227 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:31.230 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:31.399 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:31.430 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:31.626 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:31.676 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:33.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:32 smithi138 ceph-mon[193606]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:33.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:32 smithi110 ceph-mon[212939]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:33.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:32 smithi110 ceph-mon[221102]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:34.176 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:34 smithi110 ceph-mon[212939]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:34.176 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:34 smithi110 ceph-mon[221102]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:34.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:34 smithi138 ceph-mon[193606]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:36.669 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:36 smithi110 ceph-mon[212939]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:36.670 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:36 smithi110 ceph-mon[221102]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:36.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:36 smithi138 ceph-mon[193606]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:36.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:12:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:12:36] "GET /metrics HTTP/1.1" 200 34056 "" "Prometheus/2.43.0" 2023-09-23T23:12:38.579 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:38 smithi110 ceph-mon[212939]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:38.579 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:38 smithi110 ceph-mon[221102]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:38.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:38 smithi138 ceph-mon[193606]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:39.183 INFO:teuthology.orchestra.run.smithi110.stdout:128849019147 2023-09-23T23:12:39.183 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T23:12:40.226 INFO:teuthology.orchestra.run.smithi110.stdout:635655159830 2023-09-23T23:12:40.226 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T23:12:40.614 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:40 smithi110 ceph-mon[221102]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:40.615 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:40 smithi110 ceph-mon[212939]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:40.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:40 smithi138 ceph-mon[193606]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:42.170 INFO:teuthology.orchestra.run.smithi110.stdout:558345748529 2023-09-23T23:12:42.171 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T23:12:42.418 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:42 smithi110 ceph-mon[212939]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:42.418 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:42 smithi110 ceph-mon[221102]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:42.419 INFO:teuthology.orchestra.run.smithi110.stdout:390842024040 2023-09-23T23:12:42.419 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T23:12:42.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:42 smithi138 ceph-mon[193606]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:43.027 INFO:teuthology.orchestra.run.smithi110.stdout:163208757511 2023-09-23T23:12:43.027 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T23:12:43.565 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:43.566 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:43.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:43.811 INFO:teuthology.orchestra.run.smithi110.stdout:197568495875 2023-09-23T23:12:43.811 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T23:12:44.370 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:44 smithi110 ceph-mon[212939]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:44.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:44 smithi110 ceph-mon[221102]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:44.371 INFO:teuthology.orchestra.run.smithi110.stdout:481036337216 2023-09-23T23:12:44.372 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T23:12:44.554 INFO:teuthology.orchestra.run.smithi110.stdout:231928234239 2023-09-23T23:12:44.554 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T23:12:44.586 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:44 smithi138 ceph-mon[193606]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:44.717 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:44.826 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:44.872 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:45.112 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:45.113 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:46.619 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:46 smithi110 ceph-mon[212939]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:46.619 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:46 smithi110 ceph-mon[221102]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:46.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:46 smithi138 ceph-mon[193606]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:46.902 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:12:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:12:46] "GET /metrics HTTP/1.1" 200 34042 "" "Prometheus/2.43.0" 2023-09-23T23:12:47.388 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:48.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:48 smithi110 ceph-mon[212939]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:48.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:48 smithi110 ceph-mon[221102]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:48.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:48 smithi138 ceph-mon[193606]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:49.069 INFO:teuthology.orchestra.run.smithi110.stdout:128849019149 2023-09-23T23:12:49.186 INFO:teuthology.orchestra.run.smithi110.stdout:635655159831 2023-09-23T23:12:49.430 INFO:teuthology.orchestra.run.smithi110.stdout:390842024041 2023-09-23T23:12:49.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:49 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2248970851' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T23:12:49.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:49 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4147311072' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T23:12:49.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:49 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2248970851' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T23:12:49.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:49 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4147311072' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T23:12:49.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:49 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2248970851' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T23:12:49.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:49 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4147311072' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T23:12:50.680 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:50 smithi110 ceph-mon[212939]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:50.680 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:50 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1795736578' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T23:12:50.681 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:50 smithi110 ceph-mon[221102]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:50.681 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:50 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1795736578' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T23:12:50.682 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019147 got 128849019149 for osd.4 2023-09-23T23:12:50.682 DEBUG:teuthology.parallel:result is None 2023-09-23T23:12:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:50 smithi138 ceph-mon[193606]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:50 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1795736578' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T23:12:51.024 INFO:tasks.cephadm.ceph_manager.ceph:need seq 390842024040 got 390842024041 for osd.0 2023-09-23T23:12:51.025 DEBUG:teuthology.parallel:result is None 2023-09-23T23:12:51.118 INFO:tasks.cephadm.ceph_manager.ceph:need seq 635655159830 got 635655159831 for osd.3 2023-09-23T23:12:51.118 DEBUG:teuthology.parallel:result is None 2023-09-23T23:12:51.857 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:52.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:52 smithi138 ceph-mon[193606]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:52.692 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:52 smithi110 ceph-mon[212939]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:52.692 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:52 smithi110 ceph-mon[221102]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:52.954 INFO:teuthology.orchestra.run.smithi110.stdout:481036337218 2023-09-23T23:12:53.247 INFO:teuthology.orchestra.run.smithi110.stdout:163208757513 2023-09-23T23:12:53.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:53 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1177116477' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T23:12:53.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:53 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3486020424' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T23:12:53.695 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:53 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1177116477' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T23:12:53.695 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:53 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3486020424' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T23:12:53.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:53 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1177116477' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T23:12:53.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:53 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3486020424' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T23:12:54.309 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757511 got 163208757513 for osd.5 2023-09-23T23:12:54.310 DEBUG:teuthology.parallel:result is None 2023-09-23T23:12:54.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:54 smithi110 ceph-mon[212939]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:54.556 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:54 smithi110 ceph-mon[221102]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:54.583 INFO:tasks.cephadm.ceph_manager.ceph:need seq 481036337216 got 481036337218 for osd.1 2023-09-23T23:12:54.584 DEBUG:teuthology.parallel:result is None 2023-09-23T23:12:54.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:54 smithi138 ceph-mon[193606]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:54.963 INFO:teuthology.orchestra.run.smithi110.stdout:558345748532 2023-09-23T23:12:55.187 INFO:teuthology.orchestra.run.smithi110.stdout:197568495877 2023-09-23T23:12:55.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:55 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3444302162' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T23:12:55.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:55 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3030764351' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T23:12:55.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:55 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3444302162' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T23:12:55.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:55 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3030764351' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T23:12:55.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:55 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3444302162' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T23:12:55.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:55 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3030764351' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T23:12:55.968 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495875 got 197568495877 for osd.6 2023-09-23T23:12:55.968 DEBUG:teuthology.parallel:result is None 2023-09-23T23:12:56.354 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:56 smithi110 ceph-mon[212939]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:56.632 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:56 smithi110 ceph-mon[221102]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:56.651 INFO:tasks.cephadm.ceph_manager.ceph:need seq 558345748529 got 558345748532 for osd.2 2023-09-23T23:12:56.651 DEBUG:teuthology.parallel:result is None 2023-09-23T23:12:56.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:56 smithi138 ceph-mon[193606]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:56.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:12:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:12:56] "GET /metrics HTTP/1.1" 200 34042 "" "Prometheus/2.43.0" 2023-09-23T23:12:57.000 INFO:teuthology.orchestra.run.smithi110.stdout:231928234242 2023-09-23T23:12:57.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:57 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2659687112' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T23:12:57.678 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:57 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2659687112' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T23:12:57.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:57 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2659687112' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T23:12:57.692 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234239 got 231928234242 for osd.7 2023-09-23T23:12:57.692 DEBUG:teuthology.parallel:result is None 2023-09-23T23:12:57.693 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T23:12:57.693 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:12:58.055 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:58.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:58 smithi138 ceph-mon[193606]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:12:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:58.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:58 smithi110 ceph-mon[212939]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:58.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:12:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:58 smithi110 ceph-mon[221102]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:12:58.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:12:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:59.946 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:12:59.948 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:13:00.676 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:00 smithi110 ceph-mon[212939]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:00.676 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:00 smithi110 ceph-mon[221102]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:00.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:00 smithi138 ceph-mon[193606]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:00.695 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":517,"stamp":"2023-09-23T23:12:59.276334+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2298112,"kb_used_data":14848,"kb_used_omap":20,"kb_used_meta":2283179,"kb_avail":747663104,"statfs":{"total":767960285184,"available":765607018496,"internally_reserved":0,"allocated":15204352,"data_stored":10398334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2337975950},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003202"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":142,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548788+0000","last_change":"2023-09-23T23:08:45.116934+0000","last_active":"2023-09-23T23:12:03.548788+0000","last_peered":"2023-09-23T23:12:03.548788+0000","last_clean":"2023-09-23T23:12:03.548788+0000","last_became_active":"2023-09-23T23:08:45.116576+0000","last_became_peered":"2023-09-23T23:08:45.116576+0000","last_unstale":"2023-09-23T23:12:03.548788+0000","last_undegraded":"2023-09-23T23:12:03.548788+0000","last_fullsized":"2023-09-23T23:12:03.548788+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:06.398968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484292+0000","last_change":"2023-09-23T23:10:59.514211+0000","last_active":"2023-09-23T23:12:03.484292+0000","last_peered":"2023-09-23T23:12:03.484292+0000","last_clean":"2023-09-23T23:12:03.484292+0000","last_became_active":"2023-09-23T23:10:59.513917+0000","last_became_peered":"2023-09-23T23:10:59.513917+0000","last_unstale":"2023-09-23T23:12:03.484292+0000","last_undegraded":"2023-09-23T23:12:03.484292+0000","last_fullsized":"2023-09-23T23:12:03.484292+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:55.533586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:18.348019+0000","last_change":"2023-09-23T23:10:59.505490+0000","last_active":"2023-09-23T23:12:18.348019+0000","last_peered":"2023-09-23T23:12:18.348019+0000","last_clean":"2023-09-23T23:12:18.348019+0000","last_became_active":"2023-09-23T23:10:59.504874+0000","last_became_peered":"2023-09-23T23:10:59.504874+0000","last_unstale":"2023-09-23T23:12:18.348019+0000","last_undegraded":"2023-09-23T23:12:18.348019+0000","last_fullsized":"2023-09-23T23:12:18.348019+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:15.279668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":112,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:10.432268+0000","last_change":"2023-09-23T23:10:59.506092+0000","last_active":"2023-09-23T23:12:10.432268+0000","last_peered":"2023-09-23T23:12:10.432268+0000","last_clean":"2023-09-23T23:12:10.432268+0000","last_became_active":"2023-09-23T23:10:59.505550+0000","last_became_peered":"2023-09-23T23:10:59.505550+0000","last_unstale":"2023-09-23T23:12:10.432268+0000","last_undegraded":"2023-09-23T23:12:10.432268+0000","last_fullsized":"2023-09-23T23:12:10.432268+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:10.576890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":118,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.483861+0000","last_change":"2023-09-23T23:10:59.515675+0000","last_active":"2023-09-23T23:12:03.483861+0000","last_peered":"2023-09-23T23:12:03.483861+0000","last_clean":"2023-09-23T23:12:03.483861+0000","last_became_active":"2023-09-23T23:10:59.514600+0000","last_became_peered":"2023-09-23T23:10:59.514600+0000","last_unstale":"2023-09-23T23:12:03.483861+0000","last_undegraded":"2023-09-23T23:12:03.483861+0000","last_fullsized":"2023-09-23T23:12:03.483861+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.273467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489315+0000","last_change":"2023-09-23T23:07:30.078179+0000","last_active":"2023-09-23T23:12:03.489315+0000","last_peered":"2023-09-23T23:12:03.489315+0000","last_clean":"2023-09-23T23:12:03.489315+0000","last_became_active":"2023-09-23T23:07:30.077810+0000","last_became_peered":"2023-09-23T23:07:30.077810+0000","last_unstale":"2023-09-23T23:12:03.489315+0000","last_undegraded":"2023-09-23T23:12:03.489315+0000","last_fullsized":"2023-09-23T23:12:03.489315+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:19.218333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.538903+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:12:03.538903+0000","last_peered":"2023-09-23T23:12:03.538903+0000","last_clean":"2023-09-23T23:12:03.538903+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:12:03.538903+0000","last_undegraded":"2023-09-23T23:12:03.538903+0000","last_fullsized":"2023-09-23T23:12:03.538903+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.483884+0000","last_change":"2023-09-23T23:10:59.515840+0000","last_active":"2023-09-23T23:12:03.483884+0000","last_peered":"2023-09-23T23:12:03.483884+0000","last_clean":"2023-09-23T23:12:03.483884+0000","last_became_active":"2023-09-23T23:10:59.515000+0000","last_became_peered":"2023-09-23T23:10:59.515000+0000","last_unstale":"2023-09-23T23:12:03.483884+0000","last_undegraded":"2023-09-23T23:12:03.483884+0000","last_fullsized":"2023-09-23T23:12:03.483884+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:47.355624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540206+0000","last_change":"2023-09-23T23:07:30.076036+0000","last_active":"2023-09-23T23:12:03.540206+0000","last_peered":"2023-09-23T23:12:03.540206+0000","last_clean":"2023-09-23T23:12:03.540206+0000","last_became_active":"2023-09-23T23:07:30.075733+0000","last_became_peered":"2023-09-23T23:07:30.075733+0000","last_unstale":"2023-09-23T23:12:03.540206+0000","last_undegraded":"2023-09-23T23:12:03.540206+0000","last_fullsized":"2023-09-23T23:12:03.540206+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:56.342424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484007+0000","last_change":"2023-09-23T23:10:59.517290+0000","last_active":"2023-09-23T23:12:03.484007+0000","last_peered":"2023-09-23T23:12:03.484007+0000","last_clean":"2023-09-23T23:12:03.484007+0000","last_became_active":"2023-09-23T23:10:59.513367+0000","last_became_peered":"2023-09-23T23:10:59.513367+0000","last_unstale":"2023-09-23T23:12:03.484007+0000","last_undegraded":"2023-09-23T23:12:03.484007+0000","last_fullsized":"2023-09-23T23:12:03.484007+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:04.328676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488519+0000","last_change":"2023-09-23T23:10:59.500837+0000","last_active":"2023-09-23T23:12:03.488519+0000","last_peered":"2023-09-23T23:12:03.488519+0000","last_clean":"2023-09-23T23:12:03.488519+0000","last_became_active":"2023-09-23T23:10:59.500455+0000","last_became_peered":"2023-09-23T23:10:59.500455+0000","last_unstale":"2023-09-23T23:12:03.488519+0000","last_undegraded":"2023-09-23T23:12:03.488519+0000","last_fullsized":"2023-09-23T23:12:03.488519+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:34.801149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":130,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:08.347005+0000","last_change":"2023-09-23T23:08:45.122701+0000","last_active":"2023-09-23T23:12:08.347005+0000","last_peered":"2023-09-23T23:12:08.347005+0000","last_clean":"2023-09-23T23:12:08.347005+0000","last_became_active":"2023-09-23T23:08:45.122332+0000","last_became_peered":"2023-09-23T23:08:45.122332+0000","last_unstale":"2023-09-23T23:12:08.347005+0000","last_undegraded":"2023-09-23T23:12:08.347005+0000","last_fullsized":"2023-09-23T23:12:08.347005+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:05.771906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":129,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.550194+0000","last_change":"2023-09-23T22:53:16.891930+0000","last_active":"2023-09-23T23:12:03.550194+0000","last_peered":"2023-09-23T23:12:03.550194+0000","last_clean":"2023-09-23T23:12:03.550194+0000","last_became_active":"2023-09-23T22:53:16.891658+0000","last_became_peered":"2023-09-23T22:53:16.891658+0000","last_unstale":"2023-09-23T23:12:03.550194+0000","last_undegraded":"2023-09-23T23:12:03.550194+0000","last_fullsized":"2023-09-23T23:12:03.550194+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:19.762775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.549002+0000","last_change":"2023-09-23T23:07:30.082476+0000","last_active":"2023-09-23T23:12:03.549002+0000","last_peered":"2023-09-23T23:12:03.549002+0000","last_clean":"2023-09-23T23:12:03.549002+0000","last_became_active":"2023-09-23T23:07:30.082201+0000","last_became_peered":"2023-09-23T23:07:30.082201+0000","last_unstale":"2023-09-23T23:12:03.549002+0000","last_undegraded":"2023-09-23T23:12:03.549002+0000","last_fullsized":"2023-09-23T23:12:03.549002+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:08.809163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488205+0000","last_change":"2023-09-23T23:08:45.115767+0000","last_active":"2023-09-23T23:12:03.488205+0000","last_peered":"2023-09-23T23:12:03.488205+0000","last_clean":"2023-09-23T23:12:03.488205+0000","last_became_active":"2023-09-23T23:08:45.115422+0000","last_became_peered":"2023-09-23T23:08:45.115422+0000","last_unstale":"2023-09-23T23:12:03.488205+0000","last_undegraded":"2023-09-23T23:12:03.488205+0000","last_fullsized":"2023-09-23T23:12:03.488205+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:53.216816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484131+0000","last_change":"2023-09-23T23:10:59.518106+0000","last_active":"2023-09-23T23:12:03.484131+0000","last_peered":"2023-09-23T23:12:03.484131+0000","last_clean":"2023-09-23T23:12:03.484131+0000","last_became_active":"2023-09-23T23:10:59.517633+0000","last_became_peered":"2023-09-23T23:10:59.517633+0000","last_unstale":"2023-09-23T23:12:03.484131+0000","last_undegraded":"2023-09-23T23:12:03.484131+0000","last_fullsized":"2023-09-23T23:12:03.484131+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:27.245420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.538685+0000","last_change":"2023-09-23T23:10:59.516989+0000","last_active":"2023-09-23T23:12:03.538685+0000","last_peered":"2023-09-23T23:12:03.538685+0000","last_clean":"2023-09-23T23:12:03.538685+0000","last_became_active":"2023-09-23T23:10:59.516507+0000","last_became_peered":"2023-09-23T23:10:59.516507+0000","last_unstale":"2023-09-23T23:12:03.538685+0000","last_undegraded":"2023-09-23T23:12:03.538685+0000","last_fullsized":"2023-09-23T23:12:03.538685+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:40.834591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540360+0000","last_change":"2023-09-23T23:10:59.498539+0000","last_active":"2023-09-23T23:12:03.540360+0000","last_peered":"2023-09-23T23:12:03.540360+0000","last_clean":"2023-09-23T23:12:03.540360+0000","last_became_active":"2023-09-23T23:10:59.498205+0000","last_became_peered":"2023-09-23T23:10:59.498205+0000","last_unstale":"2023-09-23T23:12:03.540360+0000","last_undegraded":"2023-09-23T23:12:03.540360+0000","last_fullsized":"2023-09-23T23:12:03.540360+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:16.307572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":107,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488255+0000","last_change":"2023-09-23T22:53:16.889362+0000","last_active":"2023-09-23T23:12:03.488255+0000","last_peered":"2023-09-23T23:12:03.488255+0000","last_clean":"2023-09-23T23:12:03.488255+0000","last_became_active":"2023-09-23T22:53:16.889081+0000","last_became_peered":"2023-09-23T22:53:16.889081+0000","last_unstale":"2023-09-23T23:12:03.488255+0000","last_undegraded":"2023-09-23T23:12:03.488255+0000","last_fullsized":"2023-09-23T23:12:03.488255+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:11.321151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.538765+0000","last_change":"2023-09-23T23:10:59.506198+0000","last_active":"2023-09-23T23:12:03.538765+0000","last_peered":"2023-09-23T23:12:03.538765+0000","last_clean":"2023-09-23T23:12:03.538765+0000","last_became_active":"2023-09-23T23:10:59.505912+0000","last_became_peered":"2023-09-23T23:10:59.505912+0000","last_unstale":"2023-09-23T23:12:03.538765+0000","last_undegraded":"2023-09-23T23:12:03.538765+0000","last_fullsized":"2023-09-23T23:12:03.538765+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:30.343257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":132,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488922+0000","last_change":"2023-09-23T23:04:11.518188+0000","last_active":"2023-09-23T23:12:03.488922+0000","last_peered":"2023-09-23T23:12:03.488922+0000","last_clean":"2023-09-23T23:12:03.488922+0000","last_became_active":"2023-09-23T23:04:11.517785+0000","last_became_peered":"2023-09-23T23:04:11.517785+0000","last_unstale":"2023-09-23T23:12:03.488922+0000","last_undegraded":"2023-09-23T23:12:03.488922+0000","last_fullsized":"2023-09-23T23:12:03.488922+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:26.382580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484760+0000","last_change":"2023-09-23T23:10:59.525228+0000","last_active":"2023-09-23T23:12:03.484760+0000","last_peered":"2023-09-23T23:12:03.484760+0000","last_clean":"2023-09-23T23:12:03.484760+0000","last_became_active":"2023-09-23T23:10:59.524665+0000","last_became_peered":"2023-09-23T23:10:59.524665+0000","last_unstale":"2023-09-23T23:12:03.484760+0000","last_undegraded":"2023-09-23T23:12:03.484760+0000","last_fullsized":"2023-09-23T23:12:03.484760+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:42.632316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488678+0000","last_change":"2023-09-23T23:04:11.907427+0000","last_active":"2023-09-23T23:12:03.488678+0000","last_peered":"2023-09-23T23:12:03.488678+0000","last_clean":"2023-09-23T23:12:03.488678+0000","last_became_active":"2023-09-23T23:04:11.907016+0000","last_became_peered":"2023-09-23T23:04:11.907016+0000","last_unstale":"2023-09-23T23:12:03.488678+0000","last_undegraded":"2023-09-23T23:12:03.488678+0000","last_fullsized":"2023-09-23T23:12:03.488678+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:20.715977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":106,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.541229+0000","last_change":"2023-09-23T22:53:18.075506+0000","last_active":"2023-09-23T23:12:03.541229+0000","last_peered":"2023-09-23T23:12:03.541229+0000","last_clean":"2023-09-23T23:12:03.541229+0000","last_became_active":"2023-09-23T22:53:18.075206+0000","last_became_peered":"2023-09-23T22:53:18.075206+0000","last_unstale":"2023-09-23T23:12:03.541229+0000","last_undegraded":"2023-09-23T23:12:03.541229+0000","last_fullsized":"2023-09-23T23:12:03.541229+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:32:16.168188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488500+0000","last_change":"2023-09-23T23:07:30.072322+0000","last_active":"2023-09-23T23:12:03.488500+0000","last_peered":"2023-09-23T23:12:03.488500+0000","last_clean":"2023-09-23T23:12:03.488500+0000","last_became_active":"2023-09-23T23:07:30.071941+0000","last_became_peered":"2023-09-23T23:07:30.071941+0000","last_unstale":"2023-09-23T23:12:03.488500+0000","last_undegraded":"2023-09-23T23:12:03.488500+0000","last_fullsized":"2023-09-23T23:12:03.488500+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:38.994624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548304+0000","last_change":"2023-09-23T23:10:59.512748+0000","last_active":"2023-09-23T23:12:03.548304+0000","last_peered":"2023-09-23T23:12:03.548304+0000","last_clean":"2023-09-23T23:12:03.548304+0000","last_became_active":"2023-09-23T23:10:59.512420+0000","last_became_peered":"2023-09-23T23:10:59.512420+0000","last_unstale":"2023-09-23T23:12:03.548304+0000","last_undegraded":"2023-09-23T23:12:03.548304+0000","last_fullsized":"2023-09-23T23:12:03.548304+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:09.269366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":108,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488293+0000","last_change":"2023-09-23T22:53:16.887968+0000","last_active":"2023-09-23T23:12:03.488293+0000","last_peered":"2023-09-23T23:12:03.488293+0000","last_clean":"2023-09-23T23:12:03.488293+0000","last_became_active":"2023-09-23T22:53:16.887692+0000","last_became_peered":"2023-09-23T22:53:16.887692+0000","last_unstale":"2023-09-23T23:12:03.488293+0000","last_undegraded":"2023-09-23T23:12:03.488293+0000","last_fullsized":"2023-09-23T23:12:03.488293+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:24.662748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":133,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489141+0000","last_change":"2023-09-23T23:08:45.116813+0000","last_active":"2023-09-23T23:12:03.489141+0000","last_peered":"2023-09-23T23:12:03.489141+0000","last_clean":"2023-09-23T23:12:03.489141+0000","last_became_active":"2023-09-23T23:08:45.116435+0000","last_became_peered":"2023-09-23T23:08:45.116435+0000","last_unstale":"2023-09-23T23:12:03.489141+0000","last_undegraded":"2023-09-23T23:12:03.489141+0000","last_fullsized":"2023-09-23T23:12:03.489141+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:40.616551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.539969+0000","last_change":"2023-09-23T23:10:59.502426+0000","last_active":"2023-09-23T23:12:03.539969+0000","last_peered":"2023-09-23T23:12:03.539969+0000","last_clean":"2023-09-23T23:12:03.539969+0000","last_became_active":"2023-09-23T23:10:59.502074+0000","last_became_peered":"2023-09-23T23:10:59.502074+0000","last_unstale":"2023-09-23T23:12:03.539969+0000","last_undegraded":"2023-09-23T23:12:03.539969+0000","last_fullsized":"2023-09-23T23:12:03.539969+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:46.029347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":114,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.539432+0000","last_change":"2023-09-23T23:04:11.912493+0000","last_active":"2023-09-23T23:12:03.539432+0000","last_peered":"2023-09-23T23:12:03.539432+0000","last_clean":"2023-09-23T23:12:03.539432+0000","last_became_active":"2023-09-23T23:04:11.906712+0000","last_became_peered":"2023-09-23T23:04:11.906712+0000","last_unstale":"2023-09-23T23:12:03.539432+0000","last_undegraded":"2023-09-23T23:12:03.539432+0000","last_fullsized":"2023-09-23T23:12:03.539432+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:58.932610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484285+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:12:03.484285+0000","last_peered":"2023-09-23T23:12:03.484285+0000","last_clean":"2023-09-23T23:12:03.484285+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:12:03.484285+0000","last_undegraded":"2023-09-23T23:12:03.484285+0000","last_fullsized":"2023-09-23T23:12:03.484285+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548351+0000","last_change":"2023-09-23T23:10:59.512363+0000","last_active":"2023-09-23T23:12:03.548351+0000","last_peered":"2023-09-23T23:12:03.548351+0000","last_clean":"2023-09-23T23:12:03.548351+0000","last_became_active":"2023-09-23T23:10:59.512022+0000","last_became_peered":"2023-09-23T23:10:59.512022+0000","last_unstale":"2023-09-23T23:12:03.548351+0000","last_undegraded":"2023-09-23T23:12:03.548351+0000","last_fullsized":"2023-09-23T23:12:03.548351+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:45.523432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489017+0000","last_change":"2023-09-23T23:07:30.077349+0000","last_active":"2023-09-23T23:12:03.489017+0000","last_peered":"2023-09-23T23:12:03.489017+0000","last_clean":"2023-09-23T23:12:03.489017+0000","last_became_active":"2023-09-23T23:07:30.077031+0000","last_became_peered":"2023-09-23T23:07:30.077031+0000","last_unstale":"2023-09-23T23:12:03.489017+0000","last_undegraded":"2023-09-23T23:12:03.489017+0000","last_fullsized":"2023-09-23T23:12:03.489017+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:27.708541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548925+0000","last_change":"2023-09-23T23:08:45.117579+0000","last_active":"2023-09-23T23:12:03.548925+0000","last_peered":"2023-09-23T23:12:03.548925+0000","last_clean":"2023-09-23T23:12:03.548925+0000","last_became_active":"2023-09-23T23:08:45.117216+0000","last_became_peered":"2023-09-23T23:08:45.117216+0000","last_unstale":"2023-09-23T23:12:03.548925+0000","last_undegraded":"2023-09-23T23:12:03.548925+0000","last_fullsized":"2023-09-23T23:12:03.548925+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:27.731522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548772+0000","last_change":"2023-09-23T23:08:45.120505+0000","last_active":"2023-09-23T23:12:03.548772+0000","last_peered":"2023-09-23T23:12:03.548772+0000","last_clean":"2023-09-23T23:12:03.548772+0000","last_became_active":"2023-09-23T23:08:45.120136+0000","last_became_peered":"2023-09-23T23:08:45.120136+0000","last_unstale":"2023-09-23T23:12:03.548772+0000","last_undegraded":"2023-09-23T23:12:03.548772+0000","last_fullsized":"2023-09-23T23:12:03.548772+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:56.726515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.549135+0000","last_change":"2023-09-23T23:07:30.081779+0000","last_active":"2023-09-23T23:12:03.549135+0000","last_peered":"2023-09-23T23:12:03.549135+0000","last_clean":"2023-09-23T23:12:03.549135+0000","last_became_active":"2023-09-23T23:07:30.081425+0000","last_became_peered":"2023-09-23T23:07:30.081425+0000","last_unstale":"2023-09-23T23:12:03.549135+0000","last_undegraded":"2023-09-23T23:12:03.549135+0000","last_fullsized":"2023-09-23T23:12:03.549135+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:01.231206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.483964+0000","last_change":"2023-09-23T23:10:59.516007+0000","last_active":"2023-09-23T23:12:03.483964+0000","last_peered":"2023-09-23T23:12:03.483964+0000","last_clean":"2023-09-23T23:12:03.483964+0000","last_became_active":"2023-09-23T23:10:59.515388+0000","last_became_peered":"2023-09-23T23:10:59.515388+0000","last_unstale":"2023-09-23T23:12:03.483964+0000","last_undegraded":"2023-09-23T23:12:03.483964+0000","last_fullsized":"2023-09-23T23:12:03.483964+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:13.171258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":130,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486491+0000","last_change":"2023-09-23T23:10:59.503483+0000","last_active":"2023-09-23T23:12:03.486491+0000","last_peered":"2023-09-23T23:12:03.486491+0000","last_clean":"2023-09-23T23:12:03.486491+0000","last_became_active":"2023-09-23T23:10:59.502851+0000","last_became_peered":"2023-09-23T23:10:59.502851+0000","last_unstale":"2023-09-23T23:12:03.486491+0000","last_undegraded":"2023-09-23T23:12:03.486491+0000","last_fullsized":"2023-09-23T23:12:03.486491+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:20.388467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486449+0000","last_change":"2023-09-23T23:07:30.069993+0000","last_active":"2023-09-23T23:12:03.486449+0000","last_peered":"2023-09-23T23:12:03.486449+0000","last_clean":"2023-09-23T23:12:03.486449+0000","last_became_active":"2023-09-23T23:07:30.069634+0000","last_became_peered":"2023-09-23T23:07:30.069634+0000","last_unstale":"2023-09-23T23:12:03.486449+0000","last_undegraded":"2023-09-23T23:12:03.486449+0000","last_fullsized":"2023-09-23T23:12:03.486449+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:24.608905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":141,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540255+0000","last_change":"2023-09-23T23:10:59.498546+0000","last_active":"2023-09-23T23:12:03.540255+0000","last_peered":"2023-09-23T23:12:03.540255+0000","last_clean":"2023-09-23T23:12:03.540255+0000","last_became_active":"2023-09-23T23:10:59.498231+0000","last_became_peered":"2023-09-23T23:10:59.498231+0000","last_unstale":"2023-09-23T23:12:03.540255+0000","last_undegraded":"2023-09-23T23:12:03.540255+0000","last_fullsized":"2023-09-23T23:12:03.540255+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:37.404273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":115,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486594+0000","last_change":"2023-09-23T23:07:30.082833+0000","last_active":"2023-09-23T23:12:03.486594+0000","last_peered":"2023-09-23T23:12:03.486594+0000","last_clean":"2023-09-23T23:12:03.486594+0000","last_became_active":"2023-09-23T23:07:30.082454+0000","last_became_peered":"2023-09-23T23:07:30.082454+0000","last_unstale":"2023-09-23T23:12:03.486594+0000","last_undegraded":"2023-09-23T23:12:03.486594+0000","last_fullsized":"2023-09-23T23:12:03.486594+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:50.336760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540250+0000","last_change":"2023-09-23T23:07:30.075093+0000","last_active":"2023-09-23T23:12:03.540250+0000","last_peered":"2023-09-23T23:12:03.540250+0000","last_clean":"2023-09-23T23:12:03.540250+0000","last_became_active":"2023-09-23T23:07:30.074729+0000","last_became_peered":"2023-09-23T23:07:30.074729+0000","last_unstale":"2023-09-23T23:12:03.540250+0000","last_undegraded":"2023-09-23T23:12:03.540250+0000","last_fullsized":"2023-09-23T23:12:03.540250+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:34.502057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548916+0000","last_change":"2023-09-23T23:04:11.514703+0000","last_active":"2023-09-23T23:12:03.548916+0000","last_peered":"2023-09-23T23:12:03.548916+0000","last_clean":"2023-09-23T23:12:03.548916+0000","last_became_active":"2023-09-23T23:04:11.514403+0000","last_became_peered":"2023-09-23T23:04:11.514403+0000","last_unstale":"2023-09-23T23:12:03.548916+0000","last_undegraded":"2023-09-23T23:12:03.548916+0000","last_fullsized":"2023-09-23T23:12:03.548916+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:26.374346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":145,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486345+0000","last_change":"2023-09-23T23:10:59.502030+0000","last_active":"2023-09-23T23:12:03.486345+0000","last_peered":"2023-09-23T23:12:03.486345+0000","last_clean":"2023-09-23T23:12:03.486345+0000","last_became_active":"2023-09-23T23:10:59.501714+0000","last_became_peered":"2023-09-23T23:10:59.501714+0000","last_unstale":"2023-09-23T23:12:03.486345+0000","last_undegraded":"2023-09-23T23:12:03.486345+0000","last_fullsized":"2023-09-23T23:12:03.486345+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:52.898368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":119,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.353626+0000","last_change":"2023-09-23T23:10:59.501544+0000","last_active":"2023-09-23T23:12:20.353626+0000","last_peered":"2023-09-23T23:12:20.353626+0000","last_clean":"2023-09-23T23:12:20.353626+0000","last_became_active":"2023-09-23T23:10:59.501247+0000","last_became_peered":"2023-09-23T23:10:59.501247+0000","last_unstale":"2023-09-23T23:12:20.353626+0000","last_undegraded":"2023-09-23T23:12:20.353626+0000","last_fullsized":"2023-09-23T23:12:20.353626+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:54.254755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486076+0000","last_change":"2023-09-23T23:08:45.122712+0000","last_active":"2023-09-23T23:12:03.486076+0000","last_peered":"2023-09-23T23:12:03.486076+0000","last_clean":"2023-09-23T23:12:03.486076+0000","last_became_active":"2023-09-23T23:08:45.122391+0000","last_became_peered":"2023-09-23T23:08:45.122391+0000","last_unstale":"2023-09-23T23:12:03.486076+0000","last_undegraded":"2023-09-23T23:12:03.486076+0000","last_fullsized":"2023-09-23T23:12:03.486076+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:44.916349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":117,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486558+0000","last_change":"2023-09-23T23:07:30.074957+0000","last_active":"2023-09-23T23:12:03.486558+0000","last_peered":"2023-09-23T23:12:03.486558+0000","last_clean":"2023-09-23T23:12:03.486558+0000","last_became_active":"2023-09-23T23:07:30.074475+0000","last_became_peered":"2023-09-23T23:07:30.074475+0000","last_unstale":"2023-09-23T23:12:03.486558+0000","last_undegraded":"2023-09-23T23:12:03.486558+0000","last_fullsized":"2023-09-23T23:12:03.486558+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:25.337999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540551+0000","last_change":"2023-09-23T23:08:45.111935+0000","last_active":"2023-09-23T23:12:03.540551+0000","last_peered":"2023-09-23T23:12:03.540551+0000","last_clean":"2023-09-23T23:12:03.540551+0000","last_became_active":"2023-09-23T23:08:45.111549+0000","last_became_peered":"2023-09-23T23:08:45.111549+0000","last_unstale":"2023-09-23T23:12:03.540551+0000","last_undegraded":"2023-09-23T23:12:03.540551+0000","last_fullsized":"2023-09-23T23:12:03.540551+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:49.879205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":504,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488060+0000","last_change":"2023-09-23T23:04:11.517317+0000","last_active":"2023-09-23T23:12:03.488060+0000","last_peered":"2023-09-23T23:12:03.488060+0000","last_clean":"2023-09-23T23:12:03.488060+0000","last_became_active":"2023-09-23T23:04:11.516929+0000","last_became_peered":"2023-09-23T23:04:11.516929+0000","last_unstale":"2023-09-23T23:12:03.488060+0000","last_undegraded":"2023-09-23T23:12:03.488060+0000","last_fullsized":"2023-09-23T23:12:03.488060+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T22:49:49.548021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:19.871312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":78,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484076+0000","last_change":"2023-09-23T23:10:59.512923+0000","last_active":"2023-09-23T23:12:03.484076+0000","last_peered":"2023-09-23T23:12:03.484076+0000","last_clean":"2023-09-23T23:12:03.484076+0000","last_became_active":"2023-09-23T23:10:59.512637+0000","last_became_peered":"2023-09-23T23:10:59.512637+0000","last_unstale":"2023-09-23T23:12:03.484076+0000","last_undegraded":"2023-09-23T23:12:03.484076+0000","last_fullsized":"2023-09-23T23:12:03.484076+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:43:06.983322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":106,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489400+0000","last_change":"2023-09-23T22:53:18.075922+0000","last_active":"2023-09-23T23:12:03.489400+0000","last_peered":"2023-09-23T23:12:03.489400+0000","last_clean":"2023-09-23T23:12:03.489400+0000","last_became_active":"2023-09-23T22:53:18.075621+0000","last_became_peered":"2023-09-23T22:53:18.075621+0000","last_unstale":"2023-09-23T23:12:03.489400+0000","last_undegraded":"2023-09-23T23:12:03.489400+0000","last_fullsized":"2023-09-23T23:12:03.489400+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:01.490333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":145,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488321+0000","last_change":"2023-09-23T23:07:30.068847+0000","last_active":"2023-09-23T23:12:03.488321+0000","last_peered":"2023-09-23T23:12:03.488321+0000","last_clean":"2023-09-23T23:12:03.488321+0000","last_became_active":"2023-09-23T23:07:30.068533+0000","last_became_peered":"2023-09-23T23:07:30.068533+0000","last_unstale":"2023-09-23T23:12:03.488321+0000","last_undegraded":"2023-09-23T23:12:03.488321+0000","last_fullsized":"2023-09-23T23:12:03.488321+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:59.598863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":112,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.255403+0000","last_change":"2023-09-23T23:10:59.503085+0000","last_active":"2023-09-23T23:12:20.255403+0000","last_peered":"2023-09-23T23:12:20.255403+0000","last_clean":"2023-09-23T23:12:20.255403+0000","last_became_active":"2023-09-23T23:10:59.502737+0000","last_became_peered":"2023-09-23T23:10:59.502737+0000","last_unstale":"2023-09-23T23:12:20.255403+0000","last_undegraded":"2023-09-23T23:12:20.255403+0000","last_fullsized":"2023-09-23T23:12:20.255403+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:22.783324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":127,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.259269+0000","last_change":"2023-09-23T23:08:45.116856+0000","last_active":"2023-09-23T23:12:20.259269+0000","last_peered":"2023-09-23T23:12:20.259269+0000","last_clean":"2023-09-23T23:12:20.259269+0000","last_became_active":"2023-09-23T23:08:45.116519+0000","last_became_peered":"2023-09-23T23:08:45.116519+0000","last_unstale":"2023-09-23T23:12:20.259269+0000","last_undegraded":"2023-09-23T23:12:20.259269+0000","last_fullsized":"2023-09-23T23:12:20.259269+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:25.917776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488391+0000","last_change":"2023-09-23T23:04:11.900743+0000","last_active":"2023-09-23T23:12:03.488391+0000","last_peered":"2023-09-23T23:12:03.488391+0000","last_clean":"2023-09-23T23:12:03.488391+0000","last_became_active":"2023-09-23T23:04:11.900418+0000","last_became_peered":"2023-09-23T23:04:11.900418+0000","last_unstale":"2023-09-23T23:12:03.488391+0000","last_undegraded":"2023-09-23T23:12:03.488391+0000","last_fullsized":"2023-09-23T23:12:03.488391+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:45:31.893513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486279+0000","last_change":"2023-09-23T23:08:45.124284+0000","last_active":"2023-09-23T23:12:03.486279+0000","last_peered":"2023-09-23T23:12:03.486279+0000","last_clean":"2023-09-23T23:12:03.486279+0000","last_became_active":"2023-09-23T23:08:45.123988+0000","last_became_peered":"2023-09-23T23:08:45.123988+0000","last_unstale":"2023-09-23T23:12:03.486279+0000","last_undegraded":"2023-09-23T23:12:03.486279+0000","last_fullsized":"2023-09-23T23:12:03.486279+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:26.726970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":97,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.255678+0000","last_change":"2023-09-23T23:04:11.911507+0000","last_active":"2023-09-23T23:12:20.255678+0000","last_peered":"2023-09-23T23:12:20.255678+0000","last_clean":"2023-09-23T23:12:20.255678+0000","last_became_active":"2023-09-23T23:04:11.908281+0000","last_became_peered":"2023-09-23T23:04:11.908281+0000","last_unstale":"2023-09-23T23:12:20.255678+0000","last_undegraded":"2023-09-23T23:12:20.255678+0000","last_fullsized":"2023-09-23T23:12:20.255678+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:30.414860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484694+0000","last_change":"2023-09-23T23:10:59.511903+0000","last_active":"2023-09-23T23:12:03.484694+0000","last_peered":"2023-09-23T23:12:03.484694+0000","last_clean":"2023-09-23T23:12:03.484694+0000","last_became_active":"2023-09-23T23:10:59.511022+0000","last_became_peered":"2023-09-23T23:10:59.511022+0000","last_unstale":"2023-09-23T23:12:03.484694+0000","last_undegraded":"2023-09-23T23:12:03.484694+0000","last_fullsized":"2023-09-23T23:12:03.484694+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:03.498373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":116,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486326+0000","last_change":"2023-09-23T23:07:30.083011+0000","last_active":"2023-09-23T23:12:03.486326+0000","last_peered":"2023-09-23T23:12:03.486326+0000","last_clean":"2023-09-23T23:12:03.486326+0000","last_became_active":"2023-09-23T23:07:30.082544+0000","last_became_peered":"2023-09-23T23:07:30.082544+0000","last_unstale":"2023-09-23T23:12:03.486326+0000","last_undegraded":"2023-09-23T23:12:03.486326+0000","last_fullsized":"2023-09-23T23:12:03.486326+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:20.624316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":122,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:19.134518+0000","last_change":"2023-09-23T23:10:59.512305+0000","last_active":"2023-09-23T23:12:19.134518+0000","last_peered":"2023-09-23T23:12:19.134518+0000","last_clean":"2023-09-23T23:12:19.134518+0000","last_became_active":"2023-09-23T23:10:59.511941+0000","last_became_peered":"2023-09-23T23:10:59.511941+0000","last_unstale":"2023-09-23T23:12:19.134518+0000","last_undegraded":"2023-09-23T23:12:19.134518+0000","last_fullsized":"2023-09-23T23:12:19.134518+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:06.293763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489269+0000","last_change":"2023-09-23T23:08:45.116806+0000","last_active":"2023-09-23T23:12:03.489269+0000","last_peered":"2023-09-23T23:12:03.489269+0000","last_clean":"2023-09-23T23:12:03.489269+0000","last_became_active":"2023-09-23T23:08:45.116462+0000","last_became_peered":"2023-09-23T23:08:45.116462+0000","last_unstale":"2023-09-23T23:12:03.489269+0000","last_undegraded":"2023-09-23T23:12:03.489269+0000","last_fullsized":"2023-09-23T23:12:03.489269+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:24.502987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":116,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.259196+0000","last_change":"2023-09-23T23:07:30.072388+0000","last_active":"2023-09-23T23:12:20.259196+0000","last_peered":"2023-09-23T23:12:20.259196+0000","last_clean":"2023-09-23T23:12:20.259196+0000","last_became_active":"2023-09-23T23:07:30.072099+0000","last_became_peered":"2023-09-23T23:07:30.072099+0000","last_unstale":"2023-09-23T23:12:20.259196+0000","last_undegraded":"2023-09-23T23:12:20.259196+0000","last_fullsized":"2023-09-23T23:12:20.259196+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:19.758734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":106,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548614+0000","last_change":"2023-09-23T22:53:17.905706+0000","last_active":"2023-09-23T23:12:03.548614+0000","last_peered":"2023-09-23T23:12:03.548614+0000","last_clean":"2023-09-23T23:12:03.548614+0000","last_became_active":"2023-09-23T22:53:17.905309+0000","last_became_peered":"2023-09-23T22:53:17.905309+0000","last_unstale":"2023-09-23T23:12:03.548614+0000","last_undegraded":"2023-09-23T23:12:03.548614+0000","last_fullsized":"2023-09-23T23:12:03.548614+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:06.105426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.539475+0000","last_change":"2023-09-23T23:04:11.900795+0000","last_active":"2023-09-23T23:12:03.539475+0000","last_peered":"2023-09-23T23:12:03.539475+0000","last_clean":"2023-09-23T23:12:03.539475+0000","last_became_active":"2023-09-23T23:04:11.899751+0000","last_became_peered":"2023-09-23T23:04:11.899751+0000","last_unstale":"2023-09-23T23:12:03.539475+0000","last_undegraded":"2023-09-23T23:12:03.539475+0000","last_fullsized":"2023-09-23T23:12:03.539475+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:41.206493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":134,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.525133+0000","last_change":"2023-09-23T23:10:59.511474+0000","last_active":"2023-09-23T23:12:03.525133+0000","last_peered":"2023-09-23T23:12:03.525133+0000","last_clean":"2023-09-23T23:12:03.525133+0000","last_became_active":"2023-09-23T23:10:59.511175+0000","last_became_peered":"2023-09-23T23:10:59.511175+0000","last_unstale":"2023-09-23T23:12:03.525133+0000","last_undegraded":"2023-09-23T23:12:03.525133+0000","last_fullsized":"2023-09-23T23:12:03.525133+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:41.647976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540448+0000","last_change":"2023-09-23T23:07:30.075311+0000","last_active":"2023-09-23T23:12:03.540448+0000","last_peered":"2023-09-23T23:12:03.540448+0000","last_clean":"2023-09-23T23:12:03.540448+0000","last_became_active":"2023-09-23T23:07:30.074952+0000","last_became_peered":"2023-09-23T23:07:30.074952+0000","last_unstale":"2023-09-23T23:12:03.540448+0000","last_undegraded":"2023-09-23T23:12:03.540448+0000","last_fullsized":"2023-09-23T23:12:03.540448+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:16.724083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548332+0000","last_change":"2023-09-23T23:10:59.512738+0000","last_active":"2023-09-23T23:12:03.548332+0000","last_peered":"2023-09-23T23:12:03.548332+0000","last_clean":"2023-09-23T23:12:03.548332+0000","last_became_active":"2023-09-23T23:10:59.512011+0000","last_became_peered":"2023-09-23T23:10:59.512011+0000","last_unstale":"2023-09-23T23:12:03.548332+0000","last_undegraded":"2023-09-23T23:12:03.548332+0000","last_fullsized":"2023-09-23T23:12:03.548332+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:18.088964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":109,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540827+0000","last_change":"2023-09-23T22:53:16.891297+0000","last_active":"2023-09-23T23:12:03.540827+0000","last_peered":"2023-09-23T23:12:03.540827+0000","last_clean":"2023-09-23T23:12:03.540827+0000","last_became_active":"2023-09-23T22:53:16.891025+0000","last_became_peered":"2023-09-23T22:53:16.891025+0000","last_unstale":"2023-09-23T23:12:03.540827+0000","last_undegraded":"2023-09-23T23:12:03.540827+0000","last_fullsized":"2023-09-23T23:12:03.540827+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:40:50.608339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548305+0000","last_change":"2023-09-23T23:10:59.512247+0000","last_active":"2023-09-23T23:12:03.548305+0000","last_peered":"2023-09-23T23:12:03.548305+0000","last_clean":"2023-09-23T23:12:03.548305+0000","last_became_active":"2023-09-23T23:10:59.511865+0000","last_became_peered":"2023-09-23T23:10:59.511865+0000","last_unstale":"2023-09-23T23:12:03.548305+0000","last_undegraded":"2023-09-23T23:12:03.548305+0000","last_fullsized":"2023-09-23T23:12:03.548305+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:20.981239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":121,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:19.134146+0000","last_change":"2023-09-23T23:10:59.512958+0000","last_active":"2023-09-23T23:12:19.134146+0000","last_peered":"2023-09-23T23:12:19.134146+0000","last_clean":"2023-09-23T23:12:19.134146+0000","last_became_active":"2023-09-23T23:10:59.512628+0000","last_became_peered":"2023-09-23T23:10:59.512628+0000","last_unstale":"2023-09-23T23:12:19.134146+0000","last_undegraded":"2023-09-23T23:12:19.134146+0000","last_fullsized":"2023-09-23T23:12:19.134146+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:16:26.998696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":135,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.483842+0000","last_change":"2023-09-23T23:10:59.511365+0000","last_active":"2023-09-23T23:12:03.483842+0000","last_peered":"2023-09-23T23:12:03.483842+0000","last_clean":"2023-09-23T23:12:03.483842+0000","last_became_active":"2023-09-23T23:10:59.511073+0000","last_became_peered":"2023-09-23T23:10:59.511073+0000","last_unstale":"2023-09-23T23:12:03.483842+0000","last_undegraded":"2023-09-23T23:12:03.483842+0000","last_fullsized":"2023-09-23T23:12:03.483842+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:35.196455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":116,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.256275+0000","last_change":"2023-09-23T23:08:45.113546+0000","last_active":"2023-09-23T23:12:20.256275+0000","last_peered":"2023-09-23T23:12:20.256275+0000","last_clean":"2023-09-23T23:12:20.256275+0000","last_became_active":"2023-09-23T23:08:45.113191+0000","last_became_peered":"2023-09-23T23:08:45.113191+0000","last_unstale":"2023-09-23T23:12:20.256275+0000","last_undegraded":"2023-09-23T23:12:20.256275+0000","last_fullsized":"2023-09-23T23:12:20.256275+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:06.372282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":133,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548440+0000","last_change":"2023-09-23T23:08:45.116882+0000","last_active":"2023-09-23T23:12:03.548440+0000","last_peered":"2023-09-23T23:12:03.548440+0000","last_clean":"2023-09-23T23:12:03.548440+0000","last_became_active":"2023-09-23T23:08:45.116543+0000","last_became_peered":"2023-09-23T23:08:45.116543+0000","last_unstale":"2023-09-23T23:12:03.548440+0000","last_undegraded":"2023-09-23T23:12:03.548440+0000","last_fullsized":"2023-09-23T23:12:03.548440+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:08.722088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234242,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":287872,"kb_used_data":2168,"kb_used_omap":0,"kb_used_meta":285696,"kb_avail":93457280,"statfs":{"total":95995035648,"available":95700254720,"internally_reserved":0,"allocated":2220032,"data_stored":1663377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":292552704},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5429999999999999,"5min":2.911,"15min":2.7999999999999998},"min":{"1min":0.23799999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":4.9340000000000002,"5min":92.650999999999996,"15min":230.249},"last":0.38100000000000001},{"interface":"front","average":{"1min":2.863,"5min":3.133,"15min":2.8500000000000001},"min":{"1min":0.28799999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":12.004,"5min":93.171000000000006,"15min":230.261},"last":0.50700000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:12:39 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":4.069,"15min":5.1349999999999998},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.2649999999999997,"5min":93.245999999999995,"15min":93.245999999999995},"last":0.55500000000000005},{"interface":"front","average":{"1min":1.889,"5min":3.7010000000000001,"15min":4.9349999999999996},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.9349999999999996,"5min":92.748000000000005,"15min":92.748000000000005},"last":0.46600000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:12:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":2.6680000000000001,"15min":3.1509999999999998},"min":{"1min":0.435,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":12.042,"5min":12.183,"15min":12.183},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.3360000000000001,"5min":2.0720000000000001,"15min":2.25},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.327,"5min":11.781000000000001,"15min":11.781000000000001},"last":0.44}]},{"osd":3,"last update":"Sat Sep 23 23:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4649999999999999,"5min":2.4649999999999999,"15min":2.4649999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.507999999999999,"5min":11.507999999999999,"15min":11.507999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":2.569,"5min":2.569,"15min":2.569},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":8.9670000000000005,"5min":8.9670000000000005,"15min":8.9670000000000005},"last":0.59599999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:12:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1269999999999998,"5min":2.3140000000000001,"15min":2.2250000000000001},"min":{"1min":0.28199999999999997,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":7.8529999999999998,"5min":92.781000000000006,"15min":194.797},"last":0.64400000000000002},{"interface":"front","average":{"1min":2.2770000000000001,"5min":2.2909999999999999,"15min":2.1859999999999999},"min":{"1min":0.33500000000000002,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":7.3659999999999997,"5min":92.626999999999995,"15min":190.82599999999999},"last":0.54300000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:12:16 2023","interfaces":[{"interface":"back","average":{"1min":2.9049999999999998,"5min":2.8050000000000002,"15min":1.9490000000000001},"min":{"1min":0.29799999999999999,"5min":0.29299999999999998,"15min":0.26500000000000001},"max":{"1min":12.067,"5min":93.135000000000005,"15min":93.135000000000005},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.111,"5min":2.2450000000000001,"15min":1.742},"min":{"1min":0.35799999999999998,"5min":0.29799999999999999,"15min":0.29399999999999998},"max":{"1min":5.1890000000000001,"5min":92.885000000000005,"15min":92.885000000000005},"last":0.69599999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:12:16 2023","interfaces":[{"interface":"back","average":{"1min":2.282,"5min":2.274,"15min":1.784},"min":{"1min":0.315,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":13.590999999999999,"5min":93.105999999999995,"15min":93.105999999999995},"last":0.74299999999999999},{"interface":"front","average":{"1min":2.6779999999999999,"5min":2.512,"15min":1.585},"min":{"1min":0.33800000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.6240000000000006,"5min":93.198999999999998,"15min":93.198999999999998},"last":0.68100000000000005}]}]},{"osd":6,"up_from":46,"seq":197568495879,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":267184,"kb_used_data":2152,"kb_used_omap":0,"kb_used_meta":265024,"kb_avail":93477968,"statfs":{"total":95995035648,"available":95721439232,"internally_reserved":0,"allocated":2203648,"data_stored":1662966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":271384576},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6439999999999999,"5min":1.706,"15min":4.2039999999999997},"min":{"1min":0.246,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":5.6310000000000002,"5min":11.004,"15min":97.858000000000004},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.208,"5min":1.6830000000000001,"15min":4.8630000000000004},"min":{"1min":0.27100000000000002,"5min":0.24399999999999999,"15min":0.22700000000000001},"max":{"1min":4.367,"5min":8.3059999999999992,"15min":99.793000000000006},"last":0.625}]},{"osd":1,"last update":"Sat Sep 23 23:12:34 2023","interfaces":[{"interface":"back","average":{"1min":1.601,"5min":4.2450000000000001,"15min":2.02},"min":{"1min":0.28599999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.6779999999999999,"5min":209.00899999999999,"15min":209.00899999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":1.417,"5min":4.4219999999999997,"15min":2.367},"min":{"1min":0.30599999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.173,"5min":209.03,"15min":209.03},"last":0.58199999999999996}]},{"osd":2,"last update":"Sat Sep 23 23:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3420000000000001,"5min":1.827,"15min":1.6180000000000001},"min":{"1min":0.38600000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.657,"5min":10.871,"15min":10.871},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.5609999999999999,"5min":2.242,"15min":1.9670000000000001},"min":{"1min":0.40999999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.3339999999999996,"5min":12.148,"15min":12.148},"last":0.47199999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.6850000000000001,"5min":2.6850000000000001,"15min":2.6850000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":7.4059999999999997,"5min":7.4059999999999997,"15min":7.4059999999999997},"last":0.53800000000000003},{"interface":"front","average":{"1min":3.1320000000000001,"5min":3.1320000000000001,"15min":3.1320000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":9.4550000000000001,"5min":9.4550000000000001,"15min":9.4550000000000001},"last":0.71599999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":1.4530000000000001,"15min":1.262},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.8879999999999999,"5min":11.890000000000001,"15min":12.919},"last":0.436},{"interface":"front","average":{"1min":0.997,"5min":1.6919999999999999,"15min":1.4039999999999999},"min":{"1min":0.312,"5min":0.308,"15min":0.24299999999999999},"max":{"1min":6.3079999999999998,"5min":11.728,"15min":11.728},"last":0.59999999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.804,"15min":1.5289999999999999},"min":{"1min":0.41799999999999998,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":3.9289999999999998,"5min":26.158999999999999,"15min":26.158999999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.623,"15min":1.4359999999999999},"min":{"1min":0.45700000000000002,"5min":0.30599999999999999,"15min":0.30199999999999999},"max":{"1min":3.5179999999999998,"5min":25.786000000000001,"15min":25.786000000000001},"last":0.66100000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:12:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1099999999999999,"5min":1.589,"15min":1.716},"min":{"1min":0.36399999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.7149999999999999,"5min":10.968999999999999,"15min":12.676},"last":0.70199999999999996},{"interface":"front","average":{"1min":1.9079999999999999,"5min":1.3799999999999999,"15min":1.4350000000000001},"min":{"1min":0.33700000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.5919999999999996,"5min":7.9669999999999996,"15min":9.7889999999999997},"last":0.74099999999999999}]}]},{"osd":1,"up_from":112,"seq":481036337220,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":239752,"kb_used_data":1728,"kb_used_omap":4,"kb_used_meta":238011,"kb_avail":93505400,"statfs":{"total":95995035648,"available":95749529600,"internally_reserved":0,"allocated":1769472,"data_stored":1089061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":243724174},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":2.012,"15min":1.6759999999999999},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":5.8730000000000002,"5min":10.221,"15min":10.221},"last":0.24299999999999999},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.718,"15min":1.486},"min":{"1min":0.255,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":4.2549999999999999,"5min":12.212,"15min":12.212},"last":0.35699999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:12:47 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":2.3410000000000002,"15min":2.4079999999999999},"min":{"1min":0.22,"5min":0.192,"15min":0.192},"max":{"1min":7.4939999999999998,"5min":16.696999999999999,"15min":16.696999999999999},"last":0.317},{"interface":"front","average":{"1min":0.58299999999999996,"5min":2.1960000000000002,"15min":2.3069999999999999},"min":{"1min":0.23300000000000001,"5min":0.215,"15min":0.215},"max":{"1min":2.448,"5min":9.0879999999999992,"15min":9.0879999999999992},"last":0.39700000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:12:03 2023","interfaces":[{"interface":"back","average":{"1min":2.7029999999999998,"5min":2.7029999999999998,"15min":2.7029999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":12.234999999999999,"5min":12.234999999999999,"15min":12.234999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":2.2909999999999999,"5min":2.2909999999999999,"15min":2.2909999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":12.256,"5min":12.256,"15min":12.256},"last":0.38800000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":1.841,"15min":1.552},"min":{"1min":0.372,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.0720000000000001,"5min":12.676,"15min":12.676},"last":0.68999999999999995},{"interface":"front","average":{"1min":1.403,"5min":1.873,"15min":1.4810000000000001},"min":{"1min":0.45200000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.8710000000000004,"5min":11.125,"15min":11.125},"last":0.56100000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":2.5619999999999998,"15min":3.9620000000000002},"min":{"1min":0.34999999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.6749999999999998,"5min":67.497,"15min":67.497},"last":0.71599999999999997},{"interface":"front","average":{"1min":1.827,"5min":2.7170000000000001,"15min":4.016},"min":{"1min":0.34599999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.4660000000000002,"5min":67.540999999999997,"15min":67.540999999999997},"last":0.57299999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.6739999999999999,"15min":1.5780000000000001},"min":{"1min":0.40799999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.6429999999999998,"5min":12.186,"15min":12.186},"last":0.67900000000000005},{"interface":"front","average":{"1min":1.9239999999999999,"5min":2.2730000000000001,"15min":1.679},"min":{"1min":0.39700000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.4029999999999996,"5min":12.785,"15min":12.785},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":2.0830000000000002,"5min":2.2549999999999999,"15min":1.7150000000000001},"min":{"1min":0.46300000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.4359999999999999,"5min":12.727,"15min":12.727},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.3240000000000001,"5min":1.9099999999999999,"15min":1.5960000000000001},"min":{"1min":0.40300000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":5.593,"5min":9.8559999999999999,"15min":9.8559999999999999},"last":0.70199999999999996}]}]},{"osd":0,"up_from":91,"seq":390842024043,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":349732,"kb_used_data":2332,"kb_used_omap":5,"kb_used_meta":347386,"kb_avail":93395420,"statfs":{"total":95995035648,"available":95636910080,"internally_reserved":0,"allocated":2387968,"data_stored":1680718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":355723898},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:12:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":2.1930000000000001,"15min":2.452},"min":{"1min":0.19700000000000001,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":11.029,"5min":21.629999999999999,"15min":21.629999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":1.4510000000000001,"5min":2.25,"15min":2.2759999999999998},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":9.702,"5min":13.382,"15min":13.382},"last":0.30099999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:12:54 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":2.1560000000000001,"15min":2.1179999999999999},"min":{"1min":0.22700000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":6.0010000000000003,"5min":10.199,"15min":10.199},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":2.0129999999999999,"15min":2.0030000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.7679999999999998,"5min":9.0239999999999991,"15min":9.0239999999999991},"last":0.34200000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:11:58 2023","interfaces":[{"interface":"back","average":{"1min":3.4590000000000001,"5min":3.4590000000000001,"15min":3.4590000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":11.866,"5min":11.866,"15min":11.866},"last":0.32100000000000001},{"interface":"front","average":{"1min":2.504,"5min":2.504,"15min":2.504},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":9.0980000000000008,"5min":9.0980000000000008,"15min":9.0980000000000008},"last":0.25}]},{"osd":4,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":2.4289999999999998,"15min":2.2389999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.245},"max":{"1min":8.8469999999999995,"5min":28.376999999999999,"15min":28.376999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.6899999999999999,"5min":2.2480000000000002,"15min":2.5910000000000002},"min":{"1min":0.32700000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.1029999999999998,"5min":30.706,"15min":30.706},"last":0.42699999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":2.2519999999999998,"15min":2.6110000000000002},"min":{"1min":0.38700000000000001,"5min":0.33700000000000002,"15min":0.30499999999999999},"max":{"1min":4.3419999999999996,"5min":15.819000000000001,"15min":30.873999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.9330000000000001,"5min":2.0699999999999998,"15min":1.956},"min":{"1min":0.47199999999999998,"5min":0.30299999999999999,"15min":0.29499999999999998},"max":{"1min":6.0709999999999997,"5min":10.117000000000001,"15min":10.117000000000001},"last":0.437}]},{"osd":6,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.0270000000000001,"5min":2.3090000000000002,"15min":2.6360000000000001},"min":{"1min":0.35899999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":10.430999999999999,"5min":15.859,"15min":21.047999999999998},"last":0.46700000000000003},{"interface":"front","average":{"1min":2.3130000000000002,"5min":2.3140000000000001,"15min":2.036},"min":{"1min":0.39700000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":10.414,"5min":10.414,"15min":10.414},"last":0.49399999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.339,"5min":2.3809999999999998,"15min":2.0600000000000001},"min":{"1min":0.45900000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.1899999999999995,"5min":10.145,"15min":10.145},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.9650000000000001,"5min":2.2469999999999999,"15min":2.2069999999999999},"min":{"1min":0.34999999999999998,"5min":0.313,"15min":0.313},"max":{"1min":9.3059999999999992,"5min":10.513999999999999,"15min":10.513999999999999},"last":0.54100000000000004}]}]},{"osd":2,"up_from":130,"seq":558345748533,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":277716,"kb_used_data":1676,"kb_used_omap":7,"kb_used_meta":276024,"kb_avail":93467436,"statfs":{"total":95995035648,"available":95710654464,"internally_reserved":0,"allocated":1716224,"data_stored":1082695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":282648657},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":2.0609999999999999,"15min":2.036},"min":{"1min":0.185,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":8.9209999999999994,"5min":16.021000000000001,"15min":16.021000000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.96999999999999997,"5min":2.5609999999999999,"15min":2.9990000000000001},"min":{"1min":0.23799999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.9649999999999999,"5min":16.152999999999999,"15min":16.152999999999999},"last":0.443}]},{"osd":1,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":2.3210000000000002,"15min":2.6389999999999998},"min":{"1min":0.23200000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":5.0810000000000004,"5min":9.3719999999999999,"15min":9.3719999999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.84699999999999998,"5min":2.141,"15min":2.649},"min":{"1min":0.25800000000000001,"5min":0.218,"15min":0.218},"max":{"1min":5.7359999999999998,"5min":11.625,"15min":11.625},"last":0.41199999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:12:02 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":1.27,"15min":1.27},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.8230000000000004,"5min":6.8230000000000004,"15min":6.8230000000000004},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.9470000000000001,"5min":1.9470000000000001,"15min":1.9470000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.79,"5min":6.79,"15min":6.79},"last":0.46400000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":1.9990000000000001,"15min":2.0840000000000001},"min":{"1min":0.41099999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.1139999999999999,"5min":10.666,"15min":10.666},"last":0.72799999999999998},{"interface":"front","average":{"1min":1.337,"5min":1.5940000000000001,"15min":1.508},"min":{"1min":0.30399999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.8149999999999995,"5min":14.829000000000001,"15min":14.829000000000001},"last":0.76900000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":2.427,"15min":2.5390000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.9359999999999999,"5min":12.173,"15min":12.173},"last":0.70299999999999996},{"interface":"front","average":{"1min":1.004,"5min":1.619,"15min":1.6519999999999999},"min":{"1min":0.38200000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.7149999999999999,"5min":11.548999999999999,"15min":11.548999999999999},"last":0.61199999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.919,"15min":1.891},"min":{"1min":0.33800000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":8.8620000000000001,"5min":15.205,"15min":15.205},"last":0.71299999999999997},{"interface":"front","average":{"1min":1.284,"5min":2.411,"15min":2.5800000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.9059999999999997,"5min":11.923999999999999,"15min":11.923999999999999},"last":0.68300000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":2.2890000000000001,"15min":2.3250000000000002},"min":{"1min":0.36299999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.8940000000000001,"5min":14.792999999999999,"15min":14.792999999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.446,"5min":2.7799999999999998,"15min":2.9460000000000002},"min":{"1min":0.42799999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.9880000000000004,"5min":15.045,"15min":15.045},"last":0.622}]}]},{"osd":3,"up_from":148,"seq":635655159834,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":206060,"kb_used_data":1636,"kb_used_omap":2,"kb_used_meta":204413,"kb_avail":93539092,"statfs":{"total":95995035648,"available":95784030208,"internally_reserved":0,"allocated":1675264,"data_stored":1077514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":209318965},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":1.391,"15min":1.391},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":5.9660000000000002,"5min":5.9660000000000002,"15min":5.9660000000000002},"last":0.26000000000000001},{"interface":"front","average":{"1min":2.1859999999999999,"5min":2.1859999999999999,"15min":2.1859999999999999},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":9.5839999999999996,"5min":9.5839999999999996,"15min":9.5839999999999996},"last":0.35799999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1960000000000002,"5min":2.1960000000000002,"15min":2.1960000000000002},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":7.1219999999999999,"5min":7.1219999999999999,"15min":7.1219999999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":1.9510000000000001,"5min":1.9510000000000001,"15min":1.9510000000000001},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":6.4160000000000004,"5min":6.4160000000000004,"15min":6.4160000000000004},"last":0.308}]},{"osd":2,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":2.46,"5min":2.46,"15min":2.46},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.9930000000000003,"5min":7.9930000000000003,"15min":7.9930000000000003},"last":0.373},{"interface":"front","average":{"1min":2.754,"5min":2.754,"15min":2.754},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.721,"5min":12.721,"15min":12.721},"last":0.34599999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7050000000000001,"5min":1.7050000000000001,"15min":1.7050000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.9139999999999997,"5min":5.9139999999999997,"15min":5.9139999999999997},"last":0.56000000000000005},{"interface":"front","average":{"1min":3.286,"5min":3.286,"15min":3.286},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":13.532999999999999,"5min":13.532999999999999,"15min":13.532999999999999},"last":0.68200000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":2.8130000000000002,"5min":2.8130000000000002,"15min":2.8130000000000002},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":10.117000000000001,"5min":10.117000000000001,"15min":10.117000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":2.1859999999999999,"5min":2.1859999999999999,"15min":2.1859999999999999},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":7.3250000000000002,"5min":7.3250000000000002,"15min":7.3250000000000002},"last":0.68999999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":2.9510000000000001,"5min":2.9510000000000001,"15min":2.9510000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":14.492000000000001,"5min":14.492000000000001,"15min":14.492000000000001},"last":0.75},{"interface":"front","average":{"1min":2.3399999999999999,"5min":2.3399999999999999,"15min":2.3399999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":7.9640000000000004,"5min":7.9640000000000004,"15min":7.9640000000000004},"last":0.60299999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":2.5529999999999999,"5min":2.5529999999999999,"15min":2.5529999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":9.7059999999999995,"5min":9.7059999999999995,"15min":9.7059999999999995},"last":0.58799999999999997},{"interface":"front","average":{"1min":2.7050000000000001,"5min":2.7050000000000001,"15min":2.7050000000000001},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":9.8010000000000002,"5min":9.8010000000000002,"15min":9.8010000000000002},"last":0.77400000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019152,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312124,"kb_used_data":1588,"kb_used_omap":0,"kb_used_meta":310528,"kb_avail":93433028,"statfs":{"total":95995035648,"available":95675420672,"internally_reserved":0,"allocated":1626112,"data_stored":1073262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":317980672},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6639999999999999,"5min":3.8860000000000001,"15min":2.524},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":10.090999999999999,"5min":238.62200000000001,"15min":238.62200000000001},"last":0.59999999999999998},{"interface":"front","average":{"1min":3.04,"5min":4.8819999999999997,"15min":3.0569999999999999},"min":{"1min":0.42699999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":11.519,"5min":238.58199999999999,"15min":238.58199999999999},"last":0.50900000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:12:37 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":4.665,"15min":2.3730000000000002},"min":{"1min":0.30499999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":10.132999999999999,"5min":175.00899999999999,"15min":175.00899999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":1.5760000000000001,"5min":4.9370000000000003,"15min":2.3450000000000002},"min":{"1min":0.35899999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.452,"5min":174.821,"15min":174.821},"last":0.45500000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.762,"15min":1.9119999999999999},"min":{"1min":0.29999999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.664999999999999,"5min":11.664999999999999,"15min":11.664999999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.4239999999999999,"5min":2.3740000000000001,"15min":2.7280000000000002},"min":{"1min":0.443,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":8.3879999999999999,"5min":10.576000000000001,"15min":10.576000000000001},"last":0.65900000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":1.827,"5min":1.827,"15min":1.827},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":8.1609999999999996,"5min":8.1609999999999996,"15min":8.1609999999999996},"last":0.497},{"interface":"front","average":{"1min":2.3610000000000002,"5min":2.3610000000000002,"15min":2.3610000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":11.455,"5min":11.455,"15min":11.455},"last":0.61599999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:12:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":1.1259999999999999,"15min":1.1919999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.23999999999999999},"max":{"1min":10.16,"5min":10.16,"15min":27.391999999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.512,"15min":1.6140000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":8.2680000000000007,"5min":8.2680000000000007,"15min":27.222999999999999},"last":0.56200000000000006}]},{"osd":6,"last update":"Sat Sep 23 23:12:03 2023","interfaces":[{"interface":"back","average":{"1min":1.956,"5min":1.232,"15min":1.23},"min":{"1min":0.32200000000000001,"5min":0.27200000000000002,"15min":0.223},"max":{"1min":9.3350000000000009,"5min":9.3350000000000009,"15min":14.4},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.5800000000000001,"5min":1.3620000000000001,"15min":1.377},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"max":{"1min":13.888999999999999,"5min":13.888999999999999,"15min":15.807},"last":0.64700000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:12:25 2023","interfaces":[{"interface":"back","average":{"1min":2.1589999999999998,"5min":1.5489999999999999,"15min":1.849},"min":{"1min":0.318,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":13.792,"5min":13.792,"15min":119.188},"last":0.67900000000000005},{"interface":"front","average":{"1min":1.6919999999999999,"5min":1.532,"15min":1.847},"min":{"1min":0.35499999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.5960000000000001,"5min":13.055,"15min":119.248},"last":0.63400000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757514,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":357672,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":356096,"kb_avail":93387480,"statfs":{"total":95995035648,"available":95628779520,"internally_reserved":0,"allocated":1605632,"data_stored":1068741,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":364642304},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:27 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":1.431,"15min":6.6580000000000004},"min":{"1min":0.32100000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.5090000000000003,"5min":10.227,"15min":190.483},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.9870000000000001,"5min":2.0670000000000002,"15min":7.4690000000000003},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.9359999999999999,"5min":12.762,"15min":194.11099999999999},"last":0.433}]},{"osd":1,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":2.8759999999999999,"15min":1.669},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":7.6870000000000003,"5min":91.295000000000002,"15min":91.295000000000002},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.28,"5min":3.1909999999999998,"15min":1.8009999999999999},"min":{"1min":0.26600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.7110000000000003,"5min":91.242000000000004,"15min":91.242000000000004},"last":0.59599999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:12:51 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":2.5049999999999999,"15min":2.798},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.734,"5min":9.8279999999999994,"15min":9.8279999999999994},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.2150000000000001,"5min":1.702,"15min":1.958},"min":{"1min":0.40300000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.2119999999999997,"5min":8.093,"15min":8.093},"last":0.47599999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:11:58 2023","interfaces":[{"interface":"back","average":{"1min":1.8600000000000001,"5min":1.8600000000000001,"15min":1.8600000000000001},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":6.2640000000000002,"5min":6.2640000000000002,"15min":6.2640000000000002},"last":0.51700000000000002},{"interface":"front","average":{"1min":3.1099999999999999,"5min":3.1099999999999999,"15min":3.1099999999999999},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":12.802,"5min":12.802,"15min":12.802},"last":0.49399999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:12:51 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.726,"15min":1.3560000000000001},"min":{"1min":0.38300000000000001,"5min":0.27500000000000002,"15min":0.221},"max":{"1min":7.6459999999999999,"5min":14.538,"15min":14.538},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.92100000000000004,"5min":1.089,"15min":1.153},"min":{"1min":0.315,"5min":0.28100000000000003,"15min":0.25},"max":{"1min":3.8919999999999999,"5min":11.779,"15min":18.827999999999999},"last":0.58199999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:12:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.5289999999999999,"15min":1.3959999999999999},"min":{"1min":0.48699999999999999,"5min":0.315,"15min":0.255},"max":{"1min":3.992,"5min":13.226000000000001,"15min":13.226000000000001},"last":0.625},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.833,"15min":1.383},"min":{"1min":0.435,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.1760000000000002,"5min":13.098000000000001,"15min":13.098000000000001},"last":0.56299999999999994}]},{"osd":7,"last update":"Sat Sep 23 23:12:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1829999999999998,"5min":1.8380000000000001,"15min":1.7689999999999999},"min":{"1min":0.38800000000000001,"5min":0.33900000000000002,"15min":0.29199999999999998},"max":{"1min":7.9390000000000001,"5min":14.929,"15min":14.929},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.7430000000000001,"5min":1.722,"15min":1.6180000000000001},"min":{"1min":0.40500000000000003,"5min":0.33400000000000002,"15min":0.29599999999999999},"max":{"1min":10.227,"5min":15.090999999999999,"15min":18.218},"last":0.71499999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:13:00.700 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:13:01.052 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:01.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:01 smithi138 ceph-mon[193606]: from='client.45083 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:13:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:01 smithi110 ceph-mon[212939]: from='client.45083 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:13:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:01 smithi110 ceph-mon[221102]: from='client.45083 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:13:02.494 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:02 smithi110 ceph-mon[212939]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:02.495 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:02 smithi110 ceph-mon[221102]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:02.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:02 smithi138 ceph-mon[193606]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:02.933 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:13:02.934 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:13:03.677 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":518,"stamp":"2023-09-23T23:13:01.276804+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2298112,"kb_used_data":14848,"kb_used_omap":20,"kb_used_meta":2283179,"kb_avail":747663104,"statfs":{"total":767960285184,"available":765607018496,"internally_reserved":0,"allocated":15204352,"data_stored":10398334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2337975950},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003322"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":142,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548788+0000","last_change":"2023-09-23T23:08:45.116934+0000","last_active":"2023-09-23T23:12:03.548788+0000","last_peered":"2023-09-23T23:12:03.548788+0000","last_clean":"2023-09-23T23:12:03.548788+0000","last_became_active":"2023-09-23T23:08:45.116576+0000","last_became_peered":"2023-09-23T23:08:45.116576+0000","last_unstale":"2023-09-23T23:12:03.548788+0000","last_undegraded":"2023-09-23T23:12:03.548788+0000","last_fullsized":"2023-09-23T23:12:03.548788+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:06.398968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484292+0000","last_change":"2023-09-23T23:10:59.514211+0000","last_active":"2023-09-23T23:12:03.484292+0000","last_peered":"2023-09-23T23:12:03.484292+0000","last_clean":"2023-09-23T23:12:03.484292+0000","last_became_active":"2023-09-23T23:10:59.513917+0000","last_became_peered":"2023-09-23T23:10:59.513917+0000","last_unstale":"2023-09-23T23:12:03.484292+0000","last_undegraded":"2023-09-23T23:12:03.484292+0000","last_fullsized":"2023-09-23T23:12:03.484292+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:55.533586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:18.348019+0000","last_change":"2023-09-23T23:10:59.505490+0000","last_active":"2023-09-23T23:12:18.348019+0000","last_peered":"2023-09-23T23:12:18.348019+0000","last_clean":"2023-09-23T23:12:18.348019+0000","last_became_active":"2023-09-23T23:10:59.504874+0000","last_became_peered":"2023-09-23T23:10:59.504874+0000","last_unstale":"2023-09-23T23:12:18.348019+0000","last_undegraded":"2023-09-23T23:12:18.348019+0000","last_fullsized":"2023-09-23T23:12:18.348019+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:15.279668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":112,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:10.432268+0000","last_change":"2023-09-23T23:10:59.506092+0000","last_active":"2023-09-23T23:12:10.432268+0000","last_peered":"2023-09-23T23:12:10.432268+0000","last_clean":"2023-09-23T23:12:10.432268+0000","last_became_active":"2023-09-23T23:10:59.505550+0000","last_became_peered":"2023-09-23T23:10:59.505550+0000","last_unstale":"2023-09-23T23:12:10.432268+0000","last_undegraded":"2023-09-23T23:12:10.432268+0000","last_fullsized":"2023-09-23T23:12:10.432268+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:10.576890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":118,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.483861+0000","last_change":"2023-09-23T23:10:59.515675+0000","last_active":"2023-09-23T23:12:03.483861+0000","last_peered":"2023-09-23T23:12:03.483861+0000","last_clean":"2023-09-23T23:12:03.483861+0000","last_became_active":"2023-09-23T23:10:59.514600+0000","last_became_peered":"2023-09-23T23:10:59.514600+0000","last_unstale":"2023-09-23T23:12:03.483861+0000","last_undegraded":"2023-09-23T23:12:03.483861+0000","last_fullsized":"2023-09-23T23:12:03.483861+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.273467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489315+0000","last_change":"2023-09-23T23:07:30.078179+0000","last_active":"2023-09-23T23:12:03.489315+0000","last_peered":"2023-09-23T23:12:03.489315+0000","last_clean":"2023-09-23T23:12:03.489315+0000","last_became_active":"2023-09-23T23:07:30.077810+0000","last_became_peered":"2023-09-23T23:07:30.077810+0000","last_unstale":"2023-09-23T23:12:03.489315+0000","last_undegraded":"2023-09-23T23:12:03.489315+0000","last_fullsized":"2023-09-23T23:12:03.489315+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:19.218333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.538903+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:12:03.538903+0000","last_peered":"2023-09-23T23:12:03.538903+0000","last_clean":"2023-09-23T23:12:03.538903+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:12:03.538903+0000","last_undegraded":"2023-09-23T23:12:03.538903+0000","last_fullsized":"2023-09-23T23:12:03.538903+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.483884+0000","last_change":"2023-09-23T23:10:59.515840+0000","last_active":"2023-09-23T23:12:03.483884+0000","last_peered":"2023-09-23T23:12:03.483884+0000","last_clean":"2023-09-23T23:12:03.483884+0000","last_became_active":"2023-09-23T23:10:59.515000+0000","last_became_peered":"2023-09-23T23:10:59.515000+0000","last_unstale":"2023-09-23T23:12:03.483884+0000","last_undegraded":"2023-09-23T23:12:03.483884+0000","last_fullsized":"2023-09-23T23:12:03.483884+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:47.355624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540206+0000","last_change":"2023-09-23T23:07:30.076036+0000","last_active":"2023-09-23T23:12:03.540206+0000","last_peered":"2023-09-23T23:12:03.540206+0000","last_clean":"2023-09-23T23:12:03.540206+0000","last_became_active":"2023-09-23T23:07:30.075733+0000","last_became_peered":"2023-09-23T23:07:30.075733+0000","last_unstale":"2023-09-23T23:12:03.540206+0000","last_undegraded":"2023-09-23T23:12:03.540206+0000","last_fullsized":"2023-09-23T23:12:03.540206+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:56.342424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484007+0000","last_change":"2023-09-23T23:10:59.517290+0000","last_active":"2023-09-23T23:12:03.484007+0000","last_peered":"2023-09-23T23:12:03.484007+0000","last_clean":"2023-09-23T23:12:03.484007+0000","last_became_active":"2023-09-23T23:10:59.513367+0000","last_became_peered":"2023-09-23T23:10:59.513367+0000","last_unstale":"2023-09-23T23:12:03.484007+0000","last_undegraded":"2023-09-23T23:12:03.484007+0000","last_fullsized":"2023-09-23T23:12:03.484007+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:04.328676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488519+0000","last_change":"2023-09-23T23:10:59.500837+0000","last_active":"2023-09-23T23:12:03.488519+0000","last_peered":"2023-09-23T23:12:03.488519+0000","last_clean":"2023-09-23T23:12:03.488519+0000","last_became_active":"2023-09-23T23:10:59.500455+0000","last_became_peered":"2023-09-23T23:10:59.500455+0000","last_unstale":"2023-09-23T23:12:03.488519+0000","last_undegraded":"2023-09-23T23:12:03.488519+0000","last_fullsized":"2023-09-23T23:12:03.488519+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:34.801149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":130,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:08.347005+0000","last_change":"2023-09-23T23:08:45.122701+0000","last_active":"2023-09-23T23:12:08.347005+0000","last_peered":"2023-09-23T23:12:08.347005+0000","last_clean":"2023-09-23T23:12:08.347005+0000","last_became_active":"2023-09-23T23:08:45.122332+0000","last_became_peered":"2023-09-23T23:08:45.122332+0000","last_unstale":"2023-09-23T23:12:08.347005+0000","last_undegraded":"2023-09-23T23:12:08.347005+0000","last_fullsized":"2023-09-23T23:12:08.347005+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:05.771906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":129,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.550194+0000","last_change":"2023-09-23T22:53:16.891930+0000","last_active":"2023-09-23T23:12:03.550194+0000","last_peered":"2023-09-23T23:12:03.550194+0000","last_clean":"2023-09-23T23:12:03.550194+0000","last_became_active":"2023-09-23T22:53:16.891658+0000","last_became_peered":"2023-09-23T22:53:16.891658+0000","last_unstale":"2023-09-23T23:12:03.550194+0000","last_undegraded":"2023-09-23T23:12:03.550194+0000","last_fullsized":"2023-09-23T23:12:03.550194+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:19.762775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.549002+0000","last_change":"2023-09-23T23:07:30.082476+0000","last_active":"2023-09-23T23:12:03.549002+0000","last_peered":"2023-09-23T23:12:03.549002+0000","last_clean":"2023-09-23T23:12:03.549002+0000","last_became_active":"2023-09-23T23:07:30.082201+0000","last_became_peered":"2023-09-23T23:07:30.082201+0000","last_unstale":"2023-09-23T23:12:03.549002+0000","last_undegraded":"2023-09-23T23:12:03.549002+0000","last_fullsized":"2023-09-23T23:12:03.549002+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:08.809163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488205+0000","last_change":"2023-09-23T23:08:45.115767+0000","last_active":"2023-09-23T23:12:03.488205+0000","last_peered":"2023-09-23T23:12:03.488205+0000","last_clean":"2023-09-23T23:12:03.488205+0000","last_became_active":"2023-09-23T23:08:45.115422+0000","last_became_peered":"2023-09-23T23:08:45.115422+0000","last_unstale":"2023-09-23T23:12:03.488205+0000","last_undegraded":"2023-09-23T23:12:03.488205+0000","last_fullsized":"2023-09-23T23:12:03.488205+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:53.216816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484131+0000","last_change":"2023-09-23T23:10:59.518106+0000","last_active":"2023-09-23T23:12:03.484131+0000","last_peered":"2023-09-23T23:12:03.484131+0000","last_clean":"2023-09-23T23:12:03.484131+0000","last_became_active":"2023-09-23T23:10:59.517633+0000","last_became_peered":"2023-09-23T23:10:59.517633+0000","last_unstale":"2023-09-23T23:12:03.484131+0000","last_undegraded":"2023-09-23T23:12:03.484131+0000","last_fullsized":"2023-09-23T23:12:03.484131+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:27.245420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.538685+0000","last_change":"2023-09-23T23:10:59.516989+0000","last_active":"2023-09-23T23:12:03.538685+0000","last_peered":"2023-09-23T23:12:03.538685+0000","last_clean":"2023-09-23T23:12:03.538685+0000","last_became_active":"2023-09-23T23:10:59.516507+0000","last_became_peered":"2023-09-23T23:10:59.516507+0000","last_unstale":"2023-09-23T23:12:03.538685+0000","last_undegraded":"2023-09-23T23:12:03.538685+0000","last_fullsized":"2023-09-23T23:12:03.538685+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:40.834591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540360+0000","last_change":"2023-09-23T23:10:59.498539+0000","last_active":"2023-09-23T23:12:03.540360+0000","last_peered":"2023-09-23T23:12:03.540360+0000","last_clean":"2023-09-23T23:12:03.540360+0000","last_became_active":"2023-09-23T23:10:59.498205+0000","last_became_peered":"2023-09-23T23:10:59.498205+0000","last_unstale":"2023-09-23T23:12:03.540360+0000","last_undegraded":"2023-09-23T23:12:03.540360+0000","last_fullsized":"2023-09-23T23:12:03.540360+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:16.307572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":107,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488255+0000","last_change":"2023-09-23T22:53:16.889362+0000","last_active":"2023-09-23T23:12:03.488255+0000","last_peered":"2023-09-23T23:12:03.488255+0000","last_clean":"2023-09-23T23:12:03.488255+0000","last_became_active":"2023-09-23T22:53:16.889081+0000","last_became_peered":"2023-09-23T22:53:16.889081+0000","last_unstale":"2023-09-23T23:12:03.488255+0000","last_undegraded":"2023-09-23T23:12:03.488255+0000","last_fullsized":"2023-09-23T23:12:03.488255+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:11.321151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.538765+0000","last_change":"2023-09-23T23:10:59.506198+0000","last_active":"2023-09-23T23:12:03.538765+0000","last_peered":"2023-09-23T23:12:03.538765+0000","last_clean":"2023-09-23T23:12:03.538765+0000","last_became_active":"2023-09-23T23:10:59.505912+0000","last_became_peered":"2023-09-23T23:10:59.505912+0000","last_unstale":"2023-09-23T23:12:03.538765+0000","last_undegraded":"2023-09-23T23:12:03.538765+0000","last_fullsized":"2023-09-23T23:12:03.538765+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:30.343257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":132,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488922+0000","last_change":"2023-09-23T23:04:11.518188+0000","last_active":"2023-09-23T23:12:03.488922+0000","last_peered":"2023-09-23T23:12:03.488922+0000","last_clean":"2023-09-23T23:12:03.488922+0000","last_became_active":"2023-09-23T23:04:11.517785+0000","last_became_peered":"2023-09-23T23:04:11.517785+0000","last_unstale":"2023-09-23T23:12:03.488922+0000","last_undegraded":"2023-09-23T23:12:03.488922+0000","last_fullsized":"2023-09-23T23:12:03.488922+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:26.382580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484760+0000","last_change":"2023-09-23T23:10:59.525228+0000","last_active":"2023-09-23T23:12:03.484760+0000","last_peered":"2023-09-23T23:12:03.484760+0000","last_clean":"2023-09-23T23:12:03.484760+0000","last_became_active":"2023-09-23T23:10:59.524665+0000","last_became_peered":"2023-09-23T23:10:59.524665+0000","last_unstale":"2023-09-23T23:12:03.484760+0000","last_undegraded":"2023-09-23T23:12:03.484760+0000","last_fullsized":"2023-09-23T23:12:03.484760+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:42.632316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488678+0000","last_change":"2023-09-23T23:04:11.907427+0000","last_active":"2023-09-23T23:12:03.488678+0000","last_peered":"2023-09-23T23:12:03.488678+0000","last_clean":"2023-09-23T23:12:03.488678+0000","last_became_active":"2023-09-23T23:04:11.907016+0000","last_became_peered":"2023-09-23T23:04:11.907016+0000","last_unstale":"2023-09-23T23:12:03.488678+0000","last_undegraded":"2023-09-23T23:12:03.488678+0000","last_fullsized":"2023-09-23T23:12:03.488678+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:20.715977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":106,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.541229+0000","last_change":"2023-09-23T22:53:18.075506+0000","last_active":"2023-09-23T23:12:03.541229+0000","last_peered":"2023-09-23T23:12:03.541229+0000","last_clean":"2023-09-23T23:12:03.541229+0000","last_became_active":"2023-09-23T22:53:18.075206+0000","last_became_peered":"2023-09-23T22:53:18.075206+0000","last_unstale":"2023-09-23T23:12:03.541229+0000","last_undegraded":"2023-09-23T23:12:03.541229+0000","last_fullsized":"2023-09-23T23:12:03.541229+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:32:16.168188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488500+0000","last_change":"2023-09-23T23:07:30.072322+0000","last_active":"2023-09-23T23:12:03.488500+0000","last_peered":"2023-09-23T23:12:03.488500+0000","last_clean":"2023-09-23T23:12:03.488500+0000","last_became_active":"2023-09-23T23:07:30.071941+0000","last_became_peered":"2023-09-23T23:07:30.071941+0000","last_unstale":"2023-09-23T23:12:03.488500+0000","last_undegraded":"2023-09-23T23:12:03.488500+0000","last_fullsized":"2023-09-23T23:12:03.488500+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:38.994624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548304+0000","last_change":"2023-09-23T23:10:59.512748+0000","last_active":"2023-09-23T23:12:03.548304+0000","last_peered":"2023-09-23T23:12:03.548304+0000","last_clean":"2023-09-23T23:12:03.548304+0000","last_became_active":"2023-09-23T23:10:59.512420+0000","last_became_peered":"2023-09-23T23:10:59.512420+0000","last_unstale":"2023-09-23T23:12:03.548304+0000","last_undegraded":"2023-09-23T23:12:03.548304+0000","last_fullsized":"2023-09-23T23:12:03.548304+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:09.269366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":108,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488293+0000","last_change":"2023-09-23T22:53:16.887968+0000","last_active":"2023-09-23T23:12:03.488293+0000","last_peered":"2023-09-23T23:12:03.488293+0000","last_clean":"2023-09-23T23:12:03.488293+0000","last_became_active":"2023-09-23T22:53:16.887692+0000","last_became_peered":"2023-09-23T22:53:16.887692+0000","last_unstale":"2023-09-23T23:12:03.488293+0000","last_undegraded":"2023-09-23T23:12:03.488293+0000","last_fullsized":"2023-09-23T23:12:03.488293+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:24.662748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":133,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489141+0000","last_change":"2023-09-23T23:08:45.116813+0000","last_active":"2023-09-23T23:12:03.489141+0000","last_peered":"2023-09-23T23:12:03.489141+0000","last_clean":"2023-09-23T23:12:03.489141+0000","last_became_active":"2023-09-23T23:08:45.116435+0000","last_became_peered":"2023-09-23T23:08:45.116435+0000","last_unstale":"2023-09-23T23:12:03.489141+0000","last_undegraded":"2023-09-23T23:12:03.489141+0000","last_fullsized":"2023-09-23T23:12:03.489141+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:40.616551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.539969+0000","last_change":"2023-09-23T23:10:59.502426+0000","last_active":"2023-09-23T23:12:03.539969+0000","last_peered":"2023-09-23T23:12:03.539969+0000","last_clean":"2023-09-23T23:12:03.539969+0000","last_became_active":"2023-09-23T23:10:59.502074+0000","last_became_peered":"2023-09-23T23:10:59.502074+0000","last_unstale":"2023-09-23T23:12:03.539969+0000","last_undegraded":"2023-09-23T23:12:03.539969+0000","last_fullsized":"2023-09-23T23:12:03.539969+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:46.029347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":114,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.539432+0000","last_change":"2023-09-23T23:04:11.912493+0000","last_active":"2023-09-23T23:12:03.539432+0000","last_peered":"2023-09-23T23:12:03.539432+0000","last_clean":"2023-09-23T23:12:03.539432+0000","last_became_active":"2023-09-23T23:04:11.906712+0000","last_became_peered":"2023-09-23T23:04:11.906712+0000","last_unstale":"2023-09-23T23:12:03.539432+0000","last_undegraded":"2023-09-23T23:12:03.539432+0000","last_fullsized":"2023-09-23T23:12:03.539432+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:58.932610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484285+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:12:03.484285+0000","last_peered":"2023-09-23T23:12:03.484285+0000","last_clean":"2023-09-23T23:12:03.484285+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:12:03.484285+0000","last_undegraded":"2023-09-23T23:12:03.484285+0000","last_fullsized":"2023-09-23T23:12:03.484285+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548351+0000","last_change":"2023-09-23T23:10:59.512363+0000","last_active":"2023-09-23T23:12:03.548351+0000","last_peered":"2023-09-23T23:12:03.548351+0000","last_clean":"2023-09-23T23:12:03.548351+0000","last_became_active":"2023-09-23T23:10:59.512022+0000","last_became_peered":"2023-09-23T23:10:59.512022+0000","last_unstale":"2023-09-23T23:12:03.548351+0000","last_undegraded":"2023-09-23T23:12:03.548351+0000","last_fullsized":"2023-09-23T23:12:03.548351+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:45.523432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489017+0000","last_change":"2023-09-23T23:07:30.077349+0000","last_active":"2023-09-23T23:12:03.489017+0000","last_peered":"2023-09-23T23:12:03.489017+0000","last_clean":"2023-09-23T23:12:03.489017+0000","last_became_active":"2023-09-23T23:07:30.077031+0000","last_became_peered":"2023-09-23T23:07:30.077031+0000","last_unstale":"2023-09-23T23:12:03.489017+0000","last_undegraded":"2023-09-23T23:12:03.489017+0000","last_fullsized":"2023-09-23T23:12:03.489017+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:27.708541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548925+0000","last_change":"2023-09-23T23:08:45.117579+0000","last_active":"2023-09-23T23:12:03.548925+0000","last_peered":"2023-09-23T23:12:03.548925+0000","last_clean":"2023-09-23T23:12:03.548925+0000","last_became_active":"2023-09-23T23:08:45.117216+0000","last_became_peered":"2023-09-23T23:08:45.117216+0000","last_unstale":"2023-09-23T23:12:03.548925+0000","last_undegraded":"2023-09-23T23:12:03.548925+0000","last_fullsized":"2023-09-23T23:12:03.548925+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:27.731522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548772+0000","last_change":"2023-09-23T23:08:45.120505+0000","last_active":"2023-09-23T23:12:03.548772+0000","last_peered":"2023-09-23T23:12:03.548772+0000","last_clean":"2023-09-23T23:12:03.548772+0000","last_became_active":"2023-09-23T23:08:45.120136+0000","last_became_peered":"2023-09-23T23:08:45.120136+0000","last_unstale":"2023-09-23T23:12:03.548772+0000","last_undegraded":"2023-09-23T23:12:03.548772+0000","last_fullsized":"2023-09-23T23:12:03.548772+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:56.726515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.549135+0000","last_change":"2023-09-23T23:07:30.081779+0000","last_active":"2023-09-23T23:12:03.549135+0000","last_peered":"2023-09-23T23:12:03.549135+0000","last_clean":"2023-09-23T23:12:03.549135+0000","last_became_active":"2023-09-23T23:07:30.081425+0000","last_became_peered":"2023-09-23T23:07:30.081425+0000","last_unstale":"2023-09-23T23:12:03.549135+0000","last_undegraded":"2023-09-23T23:12:03.549135+0000","last_fullsized":"2023-09-23T23:12:03.549135+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:01.231206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.483964+0000","last_change":"2023-09-23T23:10:59.516007+0000","last_active":"2023-09-23T23:12:03.483964+0000","last_peered":"2023-09-23T23:12:03.483964+0000","last_clean":"2023-09-23T23:12:03.483964+0000","last_became_active":"2023-09-23T23:10:59.515388+0000","last_became_peered":"2023-09-23T23:10:59.515388+0000","last_unstale":"2023-09-23T23:12:03.483964+0000","last_undegraded":"2023-09-23T23:12:03.483964+0000","last_fullsized":"2023-09-23T23:12:03.483964+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:13.171258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":130,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486491+0000","last_change":"2023-09-23T23:10:59.503483+0000","last_active":"2023-09-23T23:12:03.486491+0000","last_peered":"2023-09-23T23:12:03.486491+0000","last_clean":"2023-09-23T23:12:03.486491+0000","last_became_active":"2023-09-23T23:10:59.502851+0000","last_became_peered":"2023-09-23T23:10:59.502851+0000","last_unstale":"2023-09-23T23:12:03.486491+0000","last_undegraded":"2023-09-23T23:12:03.486491+0000","last_fullsized":"2023-09-23T23:12:03.486491+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:20.388467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486449+0000","last_change":"2023-09-23T23:07:30.069993+0000","last_active":"2023-09-23T23:12:03.486449+0000","last_peered":"2023-09-23T23:12:03.486449+0000","last_clean":"2023-09-23T23:12:03.486449+0000","last_became_active":"2023-09-23T23:07:30.069634+0000","last_became_peered":"2023-09-23T23:07:30.069634+0000","last_unstale":"2023-09-23T23:12:03.486449+0000","last_undegraded":"2023-09-23T23:12:03.486449+0000","last_fullsized":"2023-09-23T23:12:03.486449+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:24.608905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":141,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540255+0000","last_change":"2023-09-23T23:10:59.498546+0000","last_active":"2023-09-23T23:12:03.540255+0000","last_peered":"2023-09-23T23:12:03.540255+0000","last_clean":"2023-09-23T23:12:03.540255+0000","last_became_active":"2023-09-23T23:10:59.498231+0000","last_became_peered":"2023-09-23T23:10:59.498231+0000","last_unstale":"2023-09-23T23:12:03.540255+0000","last_undegraded":"2023-09-23T23:12:03.540255+0000","last_fullsized":"2023-09-23T23:12:03.540255+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:37.404273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":115,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486594+0000","last_change":"2023-09-23T23:07:30.082833+0000","last_active":"2023-09-23T23:12:03.486594+0000","last_peered":"2023-09-23T23:12:03.486594+0000","last_clean":"2023-09-23T23:12:03.486594+0000","last_became_active":"2023-09-23T23:07:30.082454+0000","last_became_peered":"2023-09-23T23:07:30.082454+0000","last_unstale":"2023-09-23T23:12:03.486594+0000","last_undegraded":"2023-09-23T23:12:03.486594+0000","last_fullsized":"2023-09-23T23:12:03.486594+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:50.336760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540250+0000","last_change":"2023-09-23T23:07:30.075093+0000","last_active":"2023-09-23T23:12:03.540250+0000","last_peered":"2023-09-23T23:12:03.540250+0000","last_clean":"2023-09-23T23:12:03.540250+0000","last_became_active":"2023-09-23T23:07:30.074729+0000","last_became_peered":"2023-09-23T23:07:30.074729+0000","last_unstale":"2023-09-23T23:12:03.540250+0000","last_undegraded":"2023-09-23T23:12:03.540250+0000","last_fullsized":"2023-09-23T23:12:03.540250+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:34.502057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548916+0000","last_change":"2023-09-23T23:04:11.514703+0000","last_active":"2023-09-23T23:12:03.548916+0000","last_peered":"2023-09-23T23:12:03.548916+0000","last_clean":"2023-09-23T23:12:03.548916+0000","last_became_active":"2023-09-23T23:04:11.514403+0000","last_became_peered":"2023-09-23T23:04:11.514403+0000","last_unstale":"2023-09-23T23:12:03.548916+0000","last_undegraded":"2023-09-23T23:12:03.548916+0000","last_fullsized":"2023-09-23T23:12:03.548916+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:26.374346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":145,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486345+0000","last_change":"2023-09-23T23:10:59.502030+0000","last_active":"2023-09-23T23:12:03.486345+0000","last_peered":"2023-09-23T23:12:03.486345+0000","last_clean":"2023-09-23T23:12:03.486345+0000","last_became_active":"2023-09-23T23:10:59.501714+0000","last_became_peered":"2023-09-23T23:10:59.501714+0000","last_unstale":"2023-09-23T23:12:03.486345+0000","last_undegraded":"2023-09-23T23:12:03.486345+0000","last_fullsized":"2023-09-23T23:12:03.486345+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:52.898368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":119,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.353626+0000","last_change":"2023-09-23T23:10:59.501544+0000","last_active":"2023-09-23T23:12:20.353626+0000","last_peered":"2023-09-23T23:12:20.353626+0000","last_clean":"2023-09-23T23:12:20.353626+0000","last_became_active":"2023-09-23T23:10:59.501247+0000","last_became_peered":"2023-09-23T23:10:59.501247+0000","last_unstale":"2023-09-23T23:12:20.353626+0000","last_undegraded":"2023-09-23T23:12:20.353626+0000","last_fullsized":"2023-09-23T23:12:20.353626+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:54.254755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486076+0000","last_change":"2023-09-23T23:08:45.122712+0000","last_active":"2023-09-23T23:12:03.486076+0000","last_peered":"2023-09-23T23:12:03.486076+0000","last_clean":"2023-09-23T23:12:03.486076+0000","last_became_active":"2023-09-23T23:08:45.122391+0000","last_became_peered":"2023-09-23T23:08:45.122391+0000","last_unstale":"2023-09-23T23:12:03.486076+0000","last_undegraded":"2023-09-23T23:12:03.486076+0000","last_fullsized":"2023-09-23T23:12:03.486076+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:44.916349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":117,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486558+0000","last_change":"2023-09-23T23:07:30.074957+0000","last_active":"2023-09-23T23:12:03.486558+0000","last_peered":"2023-09-23T23:12:03.486558+0000","last_clean":"2023-09-23T23:12:03.486558+0000","last_became_active":"2023-09-23T23:07:30.074475+0000","last_became_peered":"2023-09-23T23:07:30.074475+0000","last_unstale":"2023-09-23T23:12:03.486558+0000","last_undegraded":"2023-09-23T23:12:03.486558+0000","last_fullsized":"2023-09-23T23:12:03.486558+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:25.337999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540551+0000","last_change":"2023-09-23T23:08:45.111935+0000","last_active":"2023-09-23T23:12:03.540551+0000","last_peered":"2023-09-23T23:12:03.540551+0000","last_clean":"2023-09-23T23:12:03.540551+0000","last_became_active":"2023-09-23T23:08:45.111549+0000","last_became_peered":"2023-09-23T23:08:45.111549+0000","last_unstale":"2023-09-23T23:12:03.540551+0000","last_undegraded":"2023-09-23T23:12:03.540551+0000","last_fullsized":"2023-09-23T23:12:03.540551+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:49.879205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":504,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488060+0000","last_change":"2023-09-23T23:04:11.517317+0000","last_active":"2023-09-23T23:12:03.488060+0000","last_peered":"2023-09-23T23:12:03.488060+0000","last_clean":"2023-09-23T23:12:03.488060+0000","last_became_active":"2023-09-23T23:04:11.516929+0000","last_became_peered":"2023-09-23T23:04:11.516929+0000","last_unstale":"2023-09-23T23:12:03.488060+0000","last_undegraded":"2023-09-23T23:12:03.488060+0000","last_fullsized":"2023-09-23T23:12:03.488060+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T22:49:49.548021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:19.871312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":78,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484076+0000","last_change":"2023-09-23T23:10:59.512923+0000","last_active":"2023-09-23T23:12:03.484076+0000","last_peered":"2023-09-23T23:12:03.484076+0000","last_clean":"2023-09-23T23:12:03.484076+0000","last_became_active":"2023-09-23T23:10:59.512637+0000","last_became_peered":"2023-09-23T23:10:59.512637+0000","last_unstale":"2023-09-23T23:12:03.484076+0000","last_undegraded":"2023-09-23T23:12:03.484076+0000","last_fullsized":"2023-09-23T23:12:03.484076+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:43:06.983322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":106,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489400+0000","last_change":"2023-09-23T22:53:18.075922+0000","last_active":"2023-09-23T23:12:03.489400+0000","last_peered":"2023-09-23T23:12:03.489400+0000","last_clean":"2023-09-23T23:12:03.489400+0000","last_became_active":"2023-09-23T22:53:18.075621+0000","last_became_peered":"2023-09-23T22:53:18.075621+0000","last_unstale":"2023-09-23T23:12:03.489400+0000","last_undegraded":"2023-09-23T23:12:03.489400+0000","last_fullsized":"2023-09-23T23:12:03.489400+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:01.490333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":145,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488321+0000","last_change":"2023-09-23T23:07:30.068847+0000","last_active":"2023-09-23T23:12:03.488321+0000","last_peered":"2023-09-23T23:12:03.488321+0000","last_clean":"2023-09-23T23:12:03.488321+0000","last_became_active":"2023-09-23T23:07:30.068533+0000","last_became_peered":"2023-09-23T23:07:30.068533+0000","last_unstale":"2023-09-23T23:12:03.488321+0000","last_undegraded":"2023-09-23T23:12:03.488321+0000","last_fullsized":"2023-09-23T23:12:03.488321+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:59.598863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":112,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.255403+0000","last_change":"2023-09-23T23:10:59.503085+0000","last_active":"2023-09-23T23:12:20.255403+0000","last_peered":"2023-09-23T23:12:20.255403+0000","last_clean":"2023-09-23T23:12:20.255403+0000","last_became_active":"2023-09-23T23:10:59.502737+0000","last_became_peered":"2023-09-23T23:10:59.502737+0000","last_unstale":"2023-09-23T23:12:20.255403+0000","last_undegraded":"2023-09-23T23:12:20.255403+0000","last_fullsized":"2023-09-23T23:12:20.255403+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:22.783324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":127,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.259269+0000","last_change":"2023-09-23T23:08:45.116856+0000","last_active":"2023-09-23T23:12:20.259269+0000","last_peered":"2023-09-23T23:12:20.259269+0000","last_clean":"2023-09-23T23:12:20.259269+0000","last_became_active":"2023-09-23T23:08:45.116519+0000","last_became_peered":"2023-09-23T23:08:45.116519+0000","last_unstale":"2023-09-23T23:12:20.259269+0000","last_undegraded":"2023-09-23T23:12:20.259269+0000","last_fullsized":"2023-09-23T23:12:20.259269+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:25.917776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488391+0000","last_change":"2023-09-23T23:04:11.900743+0000","last_active":"2023-09-23T23:12:03.488391+0000","last_peered":"2023-09-23T23:12:03.488391+0000","last_clean":"2023-09-23T23:12:03.488391+0000","last_became_active":"2023-09-23T23:04:11.900418+0000","last_became_peered":"2023-09-23T23:04:11.900418+0000","last_unstale":"2023-09-23T23:12:03.488391+0000","last_undegraded":"2023-09-23T23:12:03.488391+0000","last_fullsized":"2023-09-23T23:12:03.488391+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:45:31.893513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486279+0000","last_change":"2023-09-23T23:08:45.124284+0000","last_active":"2023-09-23T23:12:03.486279+0000","last_peered":"2023-09-23T23:12:03.486279+0000","last_clean":"2023-09-23T23:12:03.486279+0000","last_became_active":"2023-09-23T23:08:45.123988+0000","last_became_peered":"2023-09-23T23:08:45.123988+0000","last_unstale":"2023-09-23T23:12:03.486279+0000","last_undegraded":"2023-09-23T23:12:03.486279+0000","last_fullsized":"2023-09-23T23:12:03.486279+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:26.726970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":97,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.255678+0000","last_change":"2023-09-23T23:04:11.911507+0000","last_active":"2023-09-23T23:12:20.255678+0000","last_peered":"2023-09-23T23:12:20.255678+0000","last_clean":"2023-09-23T23:12:20.255678+0000","last_became_active":"2023-09-23T23:04:11.908281+0000","last_became_peered":"2023-09-23T23:04:11.908281+0000","last_unstale":"2023-09-23T23:12:20.255678+0000","last_undegraded":"2023-09-23T23:12:20.255678+0000","last_fullsized":"2023-09-23T23:12:20.255678+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:30.414860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484694+0000","last_change":"2023-09-23T23:10:59.511903+0000","last_active":"2023-09-23T23:12:03.484694+0000","last_peered":"2023-09-23T23:12:03.484694+0000","last_clean":"2023-09-23T23:12:03.484694+0000","last_became_active":"2023-09-23T23:10:59.511022+0000","last_became_peered":"2023-09-23T23:10:59.511022+0000","last_unstale":"2023-09-23T23:12:03.484694+0000","last_undegraded":"2023-09-23T23:12:03.484694+0000","last_fullsized":"2023-09-23T23:12:03.484694+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:03.498373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":116,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486326+0000","last_change":"2023-09-23T23:07:30.083011+0000","last_active":"2023-09-23T23:12:03.486326+0000","last_peered":"2023-09-23T23:12:03.486326+0000","last_clean":"2023-09-23T23:12:03.486326+0000","last_became_active":"2023-09-23T23:07:30.082544+0000","last_became_peered":"2023-09-23T23:07:30.082544+0000","last_unstale":"2023-09-23T23:12:03.486326+0000","last_undegraded":"2023-09-23T23:12:03.486326+0000","last_fullsized":"2023-09-23T23:12:03.486326+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:20.624316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":122,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:19.134518+0000","last_change":"2023-09-23T23:10:59.512305+0000","last_active":"2023-09-23T23:12:19.134518+0000","last_peered":"2023-09-23T23:12:19.134518+0000","last_clean":"2023-09-23T23:12:19.134518+0000","last_became_active":"2023-09-23T23:10:59.511941+0000","last_became_peered":"2023-09-23T23:10:59.511941+0000","last_unstale":"2023-09-23T23:12:19.134518+0000","last_undegraded":"2023-09-23T23:12:19.134518+0000","last_fullsized":"2023-09-23T23:12:19.134518+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:06.293763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489269+0000","last_change":"2023-09-23T23:08:45.116806+0000","last_active":"2023-09-23T23:12:03.489269+0000","last_peered":"2023-09-23T23:12:03.489269+0000","last_clean":"2023-09-23T23:12:03.489269+0000","last_became_active":"2023-09-23T23:08:45.116462+0000","last_became_peered":"2023-09-23T23:08:45.116462+0000","last_unstale":"2023-09-23T23:12:03.489269+0000","last_undegraded":"2023-09-23T23:12:03.489269+0000","last_fullsized":"2023-09-23T23:12:03.489269+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:24.502987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":116,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.259196+0000","last_change":"2023-09-23T23:07:30.072388+0000","last_active":"2023-09-23T23:12:20.259196+0000","last_peered":"2023-09-23T23:12:20.259196+0000","last_clean":"2023-09-23T23:12:20.259196+0000","last_became_active":"2023-09-23T23:07:30.072099+0000","last_became_peered":"2023-09-23T23:07:30.072099+0000","last_unstale":"2023-09-23T23:12:20.259196+0000","last_undegraded":"2023-09-23T23:12:20.259196+0000","last_fullsized":"2023-09-23T23:12:20.259196+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:19.758734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":106,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548614+0000","last_change":"2023-09-23T22:53:17.905706+0000","last_active":"2023-09-23T23:12:03.548614+0000","last_peered":"2023-09-23T23:12:03.548614+0000","last_clean":"2023-09-23T23:12:03.548614+0000","last_became_active":"2023-09-23T22:53:17.905309+0000","last_became_peered":"2023-09-23T22:53:17.905309+0000","last_unstale":"2023-09-23T23:12:03.548614+0000","last_undegraded":"2023-09-23T23:12:03.548614+0000","last_fullsized":"2023-09-23T23:12:03.548614+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:06.105426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.539475+0000","last_change":"2023-09-23T23:04:11.900795+0000","last_active":"2023-09-23T23:12:03.539475+0000","last_peered":"2023-09-23T23:12:03.539475+0000","last_clean":"2023-09-23T23:12:03.539475+0000","last_became_active":"2023-09-23T23:04:11.899751+0000","last_became_peered":"2023-09-23T23:04:11.899751+0000","last_unstale":"2023-09-23T23:12:03.539475+0000","last_undegraded":"2023-09-23T23:12:03.539475+0000","last_fullsized":"2023-09-23T23:12:03.539475+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:41.206493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":134,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.525133+0000","last_change":"2023-09-23T23:10:59.511474+0000","last_active":"2023-09-23T23:12:03.525133+0000","last_peered":"2023-09-23T23:12:03.525133+0000","last_clean":"2023-09-23T23:12:03.525133+0000","last_became_active":"2023-09-23T23:10:59.511175+0000","last_became_peered":"2023-09-23T23:10:59.511175+0000","last_unstale":"2023-09-23T23:12:03.525133+0000","last_undegraded":"2023-09-23T23:12:03.525133+0000","last_fullsized":"2023-09-23T23:12:03.525133+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:41.647976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540448+0000","last_change":"2023-09-23T23:07:30.075311+0000","last_active":"2023-09-23T23:12:03.540448+0000","last_peered":"2023-09-23T23:12:03.540448+0000","last_clean":"2023-09-23T23:12:03.540448+0000","last_became_active":"2023-09-23T23:07:30.074952+0000","last_became_peered":"2023-09-23T23:07:30.074952+0000","last_unstale":"2023-09-23T23:12:03.540448+0000","last_undegraded":"2023-09-23T23:12:03.540448+0000","last_fullsized":"2023-09-23T23:12:03.540448+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:16.724083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548332+0000","last_change":"2023-09-23T23:10:59.512738+0000","last_active":"2023-09-23T23:12:03.548332+0000","last_peered":"2023-09-23T23:12:03.548332+0000","last_clean":"2023-09-23T23:12:03.548332+0000","last_became_active":"2023-09-23T23:10:59.512011+0000","last_became_peered":"2023-09-23T23:10:59.512011+0000","last_unstale":"2023-09-23T23:12:03.548332+0000","last_undegraded":"2023-09-23T23:12:03.548332+0000","last_fullsized":"2023-09-23T23:12:03.548332+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:18.088964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":109,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540827+0000","last_change":"2023-09-23T22:53:16.891297+0000","last_active":"2023-09-23T23:12:03.540827+0000","last_peered":"2023-09-23T23:12:03.540827+0000","last_clean":"2023-09-23T23:12:03.540827+0000","last_became_active":"2023-09-23T22:53:16.891025+0000","last_became_peered":"2023-09-23T22:53:16.891025+0000","last_unstale":"2023-09-23T23:12:03.540827+0000","last_undegraded":"2023-09-23T23:12:03.540827+0000","last_fullsized":"2023-09-23T23:12:03.540827+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:40:50.608339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548305+0000","last_change":"2023-09-23T23:10:59.512247+0000","last_active":"2023-09-23T23:12:03.548305+0000","last_peered":"2023-09-23T23:12:03.548305+0000","last_clean":"2023-09-23T23:12:03.548305+0000","last_became_active":"2023-09-23T23:10:59.511865+0000","last_became_peered":"2023-09-23T23:10:59.511865+0000","last_unstale":"2023-09-23T23:12:03.548305+0000","last_undegraded":"2023-09-23T23:12:03.548305+0000","last_fullsized":"2023-09-23T23:12:03.548305+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:20.981239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":121,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:19.134146+0000","last_change":"2023-09-23T23:10:59.512958+0000","last_active":"2023-09-23T23:12:19.134146+0000","last_peered":"2023-09-23T23:12:19.134146+0000","last_clean":"2023-09-23T23:12:19.134146+0000","last_became_active":"2023-09-23T23:10:59.512628+0000","last_became_peered":"2023-09-23T23:10:59.512628+0000","last_unstale":"2023-09-23T23:12:19.134146+0000","last_undegraded":"2023-09-23T23:12:19.134146+0000","last_fullsized":"2023-09-23T23:12:19.134146+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:16:26.998696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":135,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.483842+0000","last_change":"2023-09-23T23:10:59.511365+0000","last_active":"2023-09-23T23:12:03.483842+0000","last_peered":"2023-09-23T23:12:03.483842+0000","last_clean":"2023-09-23T23:12:03.483842+0000","last_became_active":"2023-09-23T23:10:59.511073+0000","last_became_peered":"2023-09-23T23:10:59.511073+0000","last_unstale":"2023-09-23T23:12:03.483842+0000","last_undegraded":"2023-09-23T23:12:03.483842+0000","last_fullsized":"2023-09-23T23:12:03.483842+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:35.196455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":116,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.256275+0000","last_change":"2023-09-23T23:08:45.113546+0000","last_active":"2023-09-23T23:12:20.256275+0000","last_peered":"2023-09-23T23:12:20.256275+0000","last_clean":"2023-09-23T23:12:20.256275+0000","last_became_active":"2023-09-23T23:08:45.113191+0000","last_became_peered":"2023-09-23T23:08:45.113191+0000","last_unstale":"2023-09-23T23:12:20.256275+0000","last_undegraded":"2023-09-23T23:12:20.256275+0000","last_fullsized":"2023-09-23T23:12:20.256275+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:06.372282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":133,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548440+0000","last_change":"2023-09-23T23:08:45.116882+0000","last_active":"2023-09-23T23:12:03.548440+0000","last_peered":"2023-09-23T23:12:03.548440+0000","last_clean":"2023-09-23T23:12:03.548440+0000","last_became_active":"2023-09-23T23:08:45.116543+0000","last_became_peered":"2023-09-23T23:08:45.116543+0000","last_unstale":"2023-09-23T23:12:03.548440+0000","last_undegraded":"2023-09-23T23:12:03.548440+0000","last_fullsized":"2023-09-23T23:12:03.548440+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:08.722088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234243,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":287872,"kb_used_data":2168,"kb_used_omap":0,"kb_used_meta":285696,"kb_avail":93457280,"statfs":{"total":95995035648,"available":95700254720,"internally_reserved":0,"allocated":2220032,"data_stored":1663377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":292552704},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5429999999999999,"5min":2.911,"15min":2.7999999999999998},"min":{"1min":0.23799999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":4.9340000000000002,"5min":92.650999999999996,"15min":230.249},"last":0.72799999999999998},{"interface":"front","average":{"1min":2.863,"5min":3.133,"15min":2.8500000000000001},"min":{"1min":0.28799999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":12.004,"5min":93.171000000000006,"15min":230.261},"last":0.66400000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:12:39 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":4.069,"15min":5.1349999999999998},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.2649999999999997,"5min":93.245999999999995,"15min":93.245999999999995},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.889,"5min":3.7010000000000001,"15min":4.9349999999999996},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.9349999999999996,"5min":92.748000000000005,"15min":92.748000000000005},"last":0.56499999999999995}]},{"osd":2,"last update":"Sat Sep 23 23:12:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":2.6680000000000001,"15min":3.1509999999999998},"min":{"1min":0.435,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":12.042,"5min":12.183,"15min":12.183},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.3360000000000001,"5min":2.0720000000000001,"15min":2.25},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.327,"5min":11.781000000000001,"15min":11.781000000000001},"last":0.70899999999999996}]},{"osd":3,"last update":"Sat Sep 23 23:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4649999999999999,"5min":2.4649999999999999,"15min":2.4649999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.507999999999999,"5min":11.507999999999999,"15min":11.507999999999999},"last":0.69299999999999995},{"interface":"front","average":{"1min":2.569,"5min":2.569,"15min":2.569},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":8.9670000000000005,"5min":8.9670000000000005,"15min":8.9670000000000005},"last":0.68000000000000005}]},{"osd":4,"last update":"Sat Sep 23 23:12:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1269999999999998,"5min":2.3140000000000001,"15min":2.2250000000000001},"min":{"1min":0.28199999999999997,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":7.8529999999999998,"5min":92.781000000000006,"15min":194.797},"last":0.77200000000000002},{"interface":"front","average":{"1min":2.2770000000000001,"5min":2.2909999999999999,"15min":2.1859999999999999},"min":{"1min":0.33500000000000002,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":7.3659999999999997,"5min":92.626999999999995,"15min":190.82599999999999},"last":0.64000000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:12:16 2023","interfaces":[{"interface":"back","average":{"1min":2.9049999999999998,"5min":2.8050000000000002,"15min":1.9490000000000001},"min":{"1min":0.29799999999999999,"5min":0.29299999999999998,"15min":0.26500000000000001},"max":{"1min":12.067,"5min":93.135000000000005,"15min":93.135000000000005},"last":0.745},{"interface":"front","average":{"1min":1.111,"5min":2.2450000000000001,"15min":1.742},"min":{"1min":0.35799999999999998,"5min":0.29799999999999999,"15min":0.29399999999999998},"max":{"1min":5.1890000000000001,"5min":92.885000000000005,"15min":92.885000000000005},"last":0.80500000000000005}]},{"osd":6,"last update":"Sat Sep 23 23:12:16 2023","interfaces":[{"interface":"back","average":{"1min":2.282,"5min":2.274,"15min":1.784},"min":{"1min":0.315,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":13.590999999999999,"5min":93.105999999999995,"15min":93.105999999999995},"last":0.82899999999999996},{"interface":"front","average":{"1min":2.6779999999999999,"5min":2.512,"15min":1.585},"min":{"1min":0.33800000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.6240000000000006,"5min":93.198999999999998,"15min":93.198999999999998},"last":0.76100000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495879,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":267184,"kb_used_data":2152,"kb_used_omap":0,"kb_used_meta":265024,"kb_avail":93477968,"statfs":{"total":95995035648,"available":95721439232,"internally_reserved":0,"allocated":2203648,"data_stored":1662966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":271384576},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6439999999999999,"5min":1.706,"15min":4.2039999999999997},"min":{"1min":0.246,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":5.6310000000000002,"5min":11.004,"15min":97.858000000000004},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.208,"5min":1.6830000000000001,"15min":4.8630000000000004},"min":{"1min":0.27100000000000002,"5min":0.24399999999999999,"15min":0.22700000000000001},"max":{"1min":4.367,"5min":8.3059999999999992,"15min":99.793000000000006},"last":0.625}]},{"osd":1,"last update":"Sat Sep 23 23:12:34 2023","interfaces":[{"interface":"back","average":{"1min":1.601,"5min":4.2450000000000001,"15min":2.02},"min":{"1min":0.28599999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.6779999999999999,"5min":209.00899999999999,"15min":209.00899999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":1.417,"5min":4.4219999999999997,"15min":2.367},"min":{"1min":0.30599999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.173,"5min":209.03,"15min":209.03},"last":0.58199999999999996}]},{"osd":2,"last update":"Sat Sep 23 23:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3420000000000001,"5min":1.827,"15min":1.6180000000000001},"min":{"1min":0.38600000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.657,"5min":10.871,"15min":10.871},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.5609999999999999,"5min":2.242,"15min":1.9670000000000001},"min":{"1min":0.40999999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.3339999999999996,"5min":12.148,"15min":12.148},"last":0.47199999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.6850000000000001,"5min":2.6850000000000001,"15min":2.6850000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":7.4059999999999997,"5min":7.4059999999999997,"15min":7.4059999999999997},"last":0.53800000000000003},{"interface":"front","average":{"1min":3.1320000000000001,"5min":3.1320000000000001,"15min":3.1320000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":9.4550000000000001,"5min":9.4550000000000001,"15min":9.4550000000000001},"last":0.71599999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":1.4530000000000001,"15min":1.262},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.8879999999999999,"5min":11.890000000000001,"15min":12.919},"last":0.436},{"interface":"front","average":{"1min":0.997,"5min":1.6919999999999999,"15min":1.4039999999999999},"min":{"1min":0.312,"5min":0.308,"15min":0.24299999999999999},"max":{"1min":6.3079999999999998,"5min":11.728,"15min":11.728},"last":0.59999999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.804,"15min":1.5289999999999999},"min":{"1min":0.41799999999999998,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":3.9289999999999998,"5min":26.158999999999999,"15min":26.158999999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.623,"15min":1.4359999999999999},"min":{"1min":0.45700000000000002,"5min":0.30599999999999999,"15min":0.30199999999999999},"max":{"1min":3.5179999999999998,"5min":25.786000000000001,"15min":25.786000000000001},"last":0.66100000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:12:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1099999999999999,"5min":1.589,"15min":1.716},"min":{"1min":0.36399999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.7149999999999999,"5min":10.968999999999999,"15min":12.676},"last":0.70199999999999996},{"interface":"front","average":{"1min":1.9079999999999999,"5min":1.3799999999999999,"15min":1.4350000000000001},"min":{"1min":0.33700000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.5919999999999996,"5min":7.9669999999999996,"15min":9.7889999999999997},"last":0.74099999999999999}]}]},{"osd":1,"up_from":112,"seq":481036337220,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":239752,"kb_used_data":1728,"kb_used_omap":4,"kb_used_meta":238011,"kb_avail":93505400,"statfs":{"total":95995035648,"available":95749529600,"internally_reserved":0,"allocated":1769472,"data_stored":1089061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":243724174},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":2.012,"15min":1.6759999999999999},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":5.8730000000000002,"5min":10.221,"15min":10.221},"last":0.24299999999999999},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.718,"15min":1.486},"min":{"1min":0.255,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":4.2549999999999999,"5min":12.212,"15min":12.212},"last":0.35699999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:12:47 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":2.3410000000000002,"15min":2.4079999999999999},"min":{"1min":0.22,"5min":0.192,"15min":0.192},"max":{"1min":7.4939999999999998,"5min":16.696999999999999,"15min":16.696999999999999},"last":0.317},{"interface":"front","average":{"1min":0.58299999999999996,"5min":2.1960000000000002,"15min":2.3069999999999999},"min":{"1min":0.23300000000000001,"5min":0.215,"15min":0.215},"max":{"1min":2.448,"5min":9.0879999999999992,"15min":9.0879999999999992},"last":0.39700000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:12:03 2023","interfaces":[{"interface":"back","average":{"1min":2.7029999999999998,"5min":2.7029999999999998,"15min":2.7029999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":12.234999999999999,"5min":12.234999999999999,"15min":12.234999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":2.2909999999999999,"5min":2.2909999999999999,"15min":2.2909999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":12.256,"5min":12.256,"15min":12.256},"last":0.38800000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":1.841,"15min":1.552},"min":{"1min":0.372,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.0720000000000001,"5min":12.676,"15min":12.676},"last":0.68999999999999995},{"interface":"front","average":{"1min":1.403,"5min":1.873,"15min":1.4810000000000001},"min":{"1min":0.45200000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.8710000000000004,"5min":11.125,"15min":11.125},"last":0.56100000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":2.5619999999999998,"15min":3.9620000000000002},"min":{"1min":0.34999999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.6749999999999998,"5min":67.497,"15min":67.497},"last":0.71599999999999997},{"interface":"front","average":{"1min":1.827,"5min":2.7170000000000001,"15min":4.016},"min":{"1min":0.34599999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.4660000000000002,"5min":67.540999999999997,"15min":67.540999999999997},"last":0.57299999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.6739999999999999,"15min":1.5780000000000001},"min":{"1min":0.40799999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.6429999999999998,"5min":12.186,"15min":12.186},"last":0.67900000000000005},{"interface":"front","average":{"1min":1.9239999999999999,"5min":2.2730000000000001,"15min":1.679},"min":{"1min":0.39700000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.4029999999999996,"5min":12.785,"15min":12.785},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":2.0830000000000002,"5min":2.2549999999999999,"15min":1.7150000000000001},"min":{"1min":0.46300000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.4359999999999999,"5min":12.727,"15min":12.727},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.3240000000000001,"5min":1.9099999999999999,"15min":1.5960000000000001},"min":{"1min":0.40300000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":5.593,"5min":9.8559999999999999,"15min":9.8559999999999999},"last":0.70199999999999996}]}]},{"osd":0,"up_from":91,"seq":390842024044,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":349732,"kb_used_data":2332,"kb_used_omap":5,"kb_used_meta":347386,"kb_avail":93395420,"statfs":{"total":95995035648,"available":95636910080,"internally_reserved":0,"allocated":2387968,"data_stored":1680718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":355723898},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:12:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":2.1930000000000001,"15min":2.452},"min":{"1min":0.19700000000000001,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":11.029,"5min":21.629999999999999,"15min":21.629999999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.4510000000000001,"5min":2.25,"15min":2.2759999999999998},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":9.702,"5min":13.382,"15min":13.382},"last":0.27000000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:12:54 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":2.1560000000000001,"15min":2.1179999999999999},"min":{"1min":0.22700000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":6.0010000000000003,"5min":10.199,"15min":10.199},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.55200000000000005,"5min":2.0129999999999999,"15min":2.0030000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.7679999999999998,"5min":9.0239999999999991,"15min":9.0239999999999991},"last":0.34599999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:12:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":2.895,"15min":3.2709999999999999},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.0460000000000003,"5min":11.866,"15min":11.866},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.47999999999999998,"5min":2.0990000000000002,"15min":2.3690000000000002},"min":{"1min":0.25,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.351,"5min":9.0980000000000008,"15min":9.0980000000000008},"last":0.313}]},{"osd":4,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":2.4289999999999998,"15min":2.2389999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.245},"max":{"1min":8.8469999999999995,"5min":28.376999999999999,"15min":28.376999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.6899999999999999,"5min":2.2480000000000002,"15min":2.5910000000000002},"min":{"1min":0.32700000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.1029999999999998,"5min":30.706,"15min":30.706},"last":0.49399999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":2.2519999999999998,"15min":2.6110000000000002},"min":{"1min":0.38700000000000001,"5min":0.33700000000000002,"15min":0.30499999999999999},"max":{"1min":4.3419999999999996,"5min":15.819000000000001,"15min":30.873999999999999},"last":0.44},{"interface":"front","average":{"1min":1.9330000000000001,"5min":2.0699999999999998,"15min":1.956},"min":{"1min":0.47199999999999998,"5min":0.30299999999999999,"15min":0.29499999999999998},"max":{"1min":6.0709999999999997,"5min":10.117000000000001,"15min":10.117000000000001},"last":0.51300000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.0270000000000001,"5min":2.3090000000000002,"15min":2.6360000000000001},"min":{"1min":0.35899999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":10.430999999999999,"5min":15.859,"15min":21.047999999999998},"last":0.60899999999999999},{"interface":"front","average":{"1min":2.3130000000000002,"5min":2.3140000000000001,"15min":2.036},"min":{"1min":0.39700000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":10.414,"5min":10.414,"15min":10.414},"last":0.63600000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.339,"5min":2.3809999999999998,"15min":2.0600000000000001},"min":{"1min":0.45900000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.1899999999999995,"5min":10.145,"15min":10.145},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.9650000000000001,"5min":2.2469999999999999,"15min":2.2069999999999999},"min":{"1min":0.34999999999999998,"5min":0.313,"15min":0.313},"max":{"1min":9.3059999999999992,"5min":10.513999999999999,"15min":10.513999999999999},"last":0.626}]}]},{"osd":2,"up_from":130,"seq":558345748533,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":277716,"kb_used_data":1676,"kb_used_omap":7,"kb_used_meta":276024,"kb_avail":93467436,"statfs":{"total":95995035648,"available":95710654464,"internally_reserved":0,"allocated":1716224,"data_stored":1082695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":282648657},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":2.0609999999999999,"15min":2.036},"min":{"1min":0.185,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":8.9209999999999994,"5min":16.021000000000001,"15min":16.021000000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.96999999999999997,"5min":2.5609999999999999,"15min":2.9990000000000001},"min":{"1min":0.23799999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.9649999999999999,"5min":16.152999999999999,"15min":16.152999999999999},"last":0.443}]},{"osd":1,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":2.3210000000000002,"15min":2.6389999999999998},"min":{"1min":0.23200000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":5.0810000000000004,"5min":9.3719999999999999,"15min":9.3719999999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.84699999999999998,"5min":2.141,"15min":2.649},"min":{"1min":0.25800000000000001,"5min":0.218,"15min":0.218},"max":{"1min":5.7359999999999998,"5min":11.625,"15min":11.625},"last":0.41199999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:12:02 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":1.27,"15min":1.27},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.8230000000000004,"5min":6.8230000000000004,"15min":6.8230000000000004},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.9470000000000001,"5min":1.9470000000000001,"15min":1.9470000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.79,"5min":6.79,"15min":6.79},"last":0.46400000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":1.9990000000000001,"15min":2.0840000000000001},"min":{"1min":0.41099999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.1139999999999999,"5min":10.666,"15min":10.666},"last":0.72799999999999998},{"interface":"front","average":{"1min":1.337,"5min":1.5940000000000001,"15min":1.508},"min":{"1min":0.30399999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.8149999999999995,"5min":14.829000000000001,"15min":14.829000000000001},"last":0.76900000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":2.427,"15min":2.5390000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.9359999999999999,"5min":12.173,"15min":12.173},"last":0.70299999999999996},{"interface":"front","average":{"1min":1.004,"5min":1.619,"15min":1.6519999999999999},"min":{"1min":0.38200000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.7149999999999999,"5min":11.548999999999999,"15min":11.548999999999999},"last":0.61199999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.919,"15min":1.891},"min":{"1min":0.33800000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":8.8620000000000001,"5min":15.205,"15min":15.205},"last":0.71299999999999997},{"interface":"front","average":{"1min":1.284,"5min":2.411,"15min":2.5800000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.9059999999999997,"5min":11.923999999999999,"15min":11.923999999999999},"last":0.68300000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":2.2890000000000001,"15min":2.3250000000000002},"min":{"1min":0.36299999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.8940000000000001,"5min":14.792999999999999,"15min":14.792999999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.446,"5min":2.7799999999999998,"15min":2.9460000000000002},"min":{"1min":0.42799999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.9880000000000004,"5min":15.045,"15min":15.045},"last":0.622}]}]},{"osd":3,"up_from":148,"seq":635655159834,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":206060,"kb_used_data":1636,"kb_used_omap":2,"kb_used_meta":204413,"kb_avail":93539092,"statfs":{"total":95995035648,"available":95784030208,"internally_reserved":0,"allocated":1675264,"data_stored":1077514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":209318965},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":1.391,"15min":1.391},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":5.9660000000000002,"5min":5.9660000000000002,"15min":5.9660000000000002},"last":0.26000000000000001},{"interface":"front","average":{"1min":2.1859999999999999,"5min":2.1859999999999999,"15min":2.1859999999999999},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":9.5839999999999996,"5min":9.5839999999999996,"15min":9.5839999999999996},"last":0.35799999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1960000000000002,"5min":2.1960000000000002,"15min":2.1960000000000002},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":7.1219999999999999,"5min":7.1219999999999999,"15min":7.1219999999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":1.9510000000000001,"5min":1.9510000000000001,"15min":1.9510000000000001},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":6.4160000000000004,"5min":6.4160000000000004,"15min":6.4160000000000004},"last":0.308}]},{"osd":2,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":2.46,"5min":2.46,"15min":2.46},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.9930000000000003,"5min":7.9930000000000003,"15min":7.9930000000000003},"last":0.373},{"interface":"front","average":{"1min":2.754,"5min":2.754,"15min":2.754},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.721,"5min":12.721,"15min":12.721},"last":0.34599999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7050000000000001,"5min":1.7050000000000001,"15min":1.7050000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.9139999999999997,"5min":5.9139999999999997,"15min":5.9139999999999997},"last":0.56000000000000005},{"interface":"front","average":{"1min":3.286,"5min":3.286,"15min":3.286},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":13.532999999999999,"5min":13.532999999999999,"15min":13.532999999999999},"last":0.68200000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":2.8130000000000002,"5min":2.8130000000000002,"15min":2.8130000000000002},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":10.117000000000001,"5min":10.117000000000001,"15min":10.117000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":2.1859999999999999,"5min":2.1859999999999999,"15min":2.1859999999999999},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":7.3250000000000002,"5min":7.3250000000000002,"15min":7.3250000000000002},"last":0.68999999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":2.9510000000000001,"5min":2.9510000000000001,"15min":2.9510000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":14.492000000000001,"5min":14.492000000000001,"15min":14.492000000000001},"last":0.75},{"interface":"front","average":{"1min":2.3399999999999999,"5min":2.3399999999999999,"15min":2.3399999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":7.9640000000000004,"5min":7.9640000000000004,"15min":7.9640000000000004},"last":0.60299999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":2.5529999999999999,"5min":2.5529999999999999,"15min":2.5529999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":9.7059999999999995,"5min":9.7059999999999995,"15min":9.7059999999999995},"last":0.58799999999999997},{"interface":"front","average":{"1min":2.7050000000000001,"5min":2.7050000000000001,"15min":2.7050000000000001},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":9.8010000000000002,"5min":9.8010000000000002,"15min":9.8010000000000002},"last":0.77400000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019152,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312124,"kb_used_data":1588,"kb_used_omap":0,"kb_used_meta":310528,"kb_avail":93433028,"statfs":{"total":95995035648,"available":95675420672,"internally_reserved":0,"allocated":1626112,"data_stored":1073262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":317980672},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6639999999999999,"5min":3.8860000000000001,"15min":2.524},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":10.090999999999999,"5min":238.62200000000001,"15min":238.62200000000001},"last":0.59999999999999998},{"interface":"front","average":{"1min":3.04,"5min":4.8819999999999997,"15min":3.0569999999999999},"min":{"1min":0.42699999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":11.519,"5min":238.58199999999999,"15min":238.58199999999999},"last":0.50900000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:12:37 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":4.665,"15min":2.3730000000000002},"min":{"1min":0.30499999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":10.132999999999999,"5min":175.00899999999999,"15min":175.00899999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":1.5760000000000001,"5min":4.9370000000000003,"15min":2.3450000000000002},"min":{"1min":0.35899999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.452,"5min":174.821,"15min":174.821},"last":0.45500000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.762,"15min":1.9119999999999999},"min":{"1min":0.29999999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.664999999999999,"5min":11.664999999999999,"15min":11.664999999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.4239999999999999,"5min":2.3740000000000001,"15min":2.7280000000000002},"min":{"1min":0.443,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":8.3879999999999999,"5min":10.576000000000001,"15min":10.576000000000001},"last":0.65900000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":1.827,"5min":1.827,"15min":1.827},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":8.1609999999999996,"5min":8.1609999999999996,"15min":8.1609999999999996},"last":0.497},{"interface":"front","average":{"1min":2.3610000000000002,"5min":2.3610000000000002,"15min":2.3610000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":11.455,"5min":11.455,"15min":11.455},"last":0.61599999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:12:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":1.1259999999999999,"15min":1.1919999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.23999999999999999},"max":{"1min":10.16,"5min":10.16,"15min":27.391999999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.512,"15min":1.6140000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":8.2680000000000007,"5min":8.2680000000000007,"15min":27.222999999999999},"last":0.56200000000000006}]},{"osd":6,"last update":"Sat Sep 23 23:12:03 2023","interfaces":[{"interface":"back","average":{"1min":1.956,"5min":1.232,"15min":1.23},"min":{"1min":0.32200000000000001,"5min":0.27200000000000002,"15min":0.223},"max":{"1min":9.3350000000000009,"5min":9.3350000000000009,"15min":14.4},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.5800000000000001,"5min":1.3620000000000001,"15min":1.377},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"max":{"1min":13.888999999999999,"5min":13.888999999999999,"15min":15.807},"last":0.64700000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:12:25 2023","interfaces":[{"interface":"back","average":{"1min":2.1589999999999998,"5min":1.5489999999999999,"15min":1.849},"min":{"1min":0.318,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":13.792,"5min":13.792,"15min":119.188},"last":0.67900000000000005},{"interface":"front","average":{"1min":1.6919999999999999,"5min":1.532,"15min":1.847},"min":{"1min":0.35499999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.5960000000000001,"5min":13.055,"15min":119.248},"last":0.63400000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757515,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":357672,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":356096,"kb_avail":93387480,"statfs":{"total":95995035648,"available":95628779520,"internally_reserved":0,"allocated":1605632,"data_stored":1068741,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":364642304},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:27 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":1.431,"15min":6.6580000000000004},"min":{"1min":0.32100000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.5090000000000003,"5min":10.227,"15min":190.483},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.9870000000000001,"5min":2.0670000000000002,"15min":7.4690000000000003},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.9359999999999999,"5min":12.762,"15min":194.11099999999999},"last":0.501}]},{"osd":1,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":2.8759999999999999,"15min":1.669},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":7.6870000000000003,"5min":91.295000000000002,"15min":91.295000000000002},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.28,"5min":3.1909999999999998,"15min":1.8009999999999999},"min":{"1min":0.26600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.7110000000000003,"5min":91.242000000000004,"15min":91.242000000000004},"last":0.45800000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:12:51 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":2.5049999999999999,"15min":2.798},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.734,"5min":9.8279999999999994,"15min":9.8279999999999994},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.2150000000000001,"5min":1.702,"15min":1.958},"min":{"1min":0.40300000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.2119999999999997,"5min":8.093,"15min":8.093},"last":0.48499999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:12:58 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":1.6499999999999999,"15min":1.79},"min":{"1min":0.44700000000000001,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":2.633,"5min":6.2640000000000002,"15min":6.2640000000000002},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.002,"5min":2.6880000000000002,"15min":2.9689999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.9420000000000002,"5min":12.802,"15min":12.802},"last":0.54300000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:12:51 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.726,"15min":1.3560000000000001},"min":{"1min":0.38300000000000001,"5min":0.27500000000000002,"15min":0.221},"max":{"1min":7.6459999999999999,"5min":14.538,"15min":14.538},"last":0.63},{"interface":"front","average":{"1min":0.92100000000000004,"5min":1.089,"15min":1.153},"min":{"1min":0.315,"5min":0.28100000000000003,"15min":0.25},"max":{"1min":3.8919999999999999,"5min":11.779,"15min":18.827999999999999},"last":0.442}]},{"osd":6,"last update":"Sat Sep 23 23:12:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.5289999999999999,"15min":1.3959999999999999},"min":{"1min":0.48699999999999999,"5min":0.315,"15min":0.255},"max":{"1min":3.992,"5min":13.226000000000001,"15min":13.226000000000001},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.833,"15min":1.383},"min":{"1min":0.435,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.1760000000000002,"5min":13.098000000000001,"15min":13.098000000000001},"last":0.41699999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:12:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1829999999999998,"5min":1.8380000000000001,"15min":1.7689999999999999},"min":{"1min":0.38800000000000001,"5min":0.33900000000000002,"15min":0.29199999999999998},"max":{"1min":7.9390000000000001,"5min":14.929,"15min":14.929},"last":0.59699999999999998},{"interface":"front","average":{"1min":1.7430000000000001,"5min":1.722,"15min":1.6180000000000001},"min":{"1min":0.40500000000000003,"5min":0.33400000000000002,"15min":0.29599999999999999},"max":{"1min":10.227,"5min":15.090999999999999,"15min":18.218},"last":0.65800000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:13:03.682 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T23:13:03.682 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T23:13:03.682 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T23:13:03.682 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T23:13:04.032 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:04.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:04 smithi138 ceph-mon[193606]: from='client.45089 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:13:04.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:04 smithi138 ceph-mon[193606]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:04.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:04 smithi110 ceph-mon[212939]: from='client.45089 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:13:04.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:04 smithi110 ceph-mon[212939]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:04.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:04 smithi110 ceph-mon[221102]: from='client.45089 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:13:04.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:04 smithi110 ceph-mon[221102]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:05.941 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:13:05.941 INFO:teuthology.orchestra.run.smithi110.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T23:13:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:06 smithi138 ceph-mon[193606]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:06 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2900596240' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T23:13:06.692 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:06 smithi110 ceph-mon[212939]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:06.692 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:06 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2900596240' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T23:13:06.693 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:06 smithi110 ceph-mon[221102]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:06.693 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:06 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2900596240' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T23:13:06.709 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T23:13:06.710 INFO:teuthology.run_tasks:Running task print... 2023-09-23T23:13:06.720 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-09-23T23:13:06.720 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T23:13:06.794 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T23:13:06.795 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T23:13:06.795 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-09-23T23:13:06.796 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T23:13:06.796 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T23:13:06.797 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T23:13:06.797 INFO:tasks.thrashosds:Beginning thrashosds... 2023-09-23T23:13:06.797 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T23:13:06.797 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:13:06.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:13:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:13:06] "GET /metrics HTTP/1.1" 200 34045 "" "Prometheus/2.43.0" 2023-09-23T23:13:07.146 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:08.511 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:08 smithi110 ceph-mon[212939]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:08.511 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:08 smithi110 ceph-mon[221102]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:08.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:08 smithi138 ceph-mon[193606]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:08.973 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:13:08.975 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:13:09.768 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":521,"stamp":"2023-09-23T23:13:07.278262+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2298112,"kb_used_data":14848,"kb_used_omap":20,"kb_used_meta":2283179,"kb_avail":747663104,"statfs":{"total":767960285184,"available":765607018496,"internally_reserved":0,"allocated":15204352,"data_stored":10398334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2337975950},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003134"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":142,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548788+0000","last_change":"2023-09-23T23:08:45.116934+0000","last_active":"2023-09-23T23:12:03.548788+0000","last_peered":"2023-09-23T23:12:03.548788+0000","last_clean":"2023-09-23T23:12:03.548788+0000","last_became_active":"2023-09-23T23:08:45.116576+0000","last_became_peered":"2023-09-23T23:08:45.116576+0000","last_unstale":"2023-09-23T23:12:03.548788+0000","last_undegraded":"2023-09-23T23:12:03.548788+0000","last_fullsized":"2023-09-23T23:12:03.548788+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:06.398968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484292+0000","last_change":"2023-09-23T23:10:59.514211+0000","last_active":"2023-09-23T23:12:03.484292+0000","last_peered":"2023-09-23T23:12:03.484292+0000","last_clean":"2023-09-23T23:12:03.484292+0000","last_became_active":"2023-09-23T23:10:59.513917+0000","last_became_peered":"2023-09-23T23:10:59.513917+0000","last_unstale":"2023-09-23T23:12:03.484292+0000","last_undegraded":"2023-09-23T23:12:03.484292+0000","last_fullsized":"2023-09-23T23:12:03.484292+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:55.533586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:18.348019+0000","last_change":"2023-09-23T23:10:59.505490+0000","last_active":"2023-09-23T23:12:18.348019+0000","last_peered":"2023-09-23T23:12:18.348019+0000","last_clean":"2023-09-23T23:12:18.348019+0000","last_became_active":"2023-09-23T23:10:59.504874+0000","last_became_peered":"2023-09-23T23:10:59.504874+0000","last_unstale":"2023-09-23T23:12:18.348019+0000","last_undegraded":"2023-09-23T23:12:18.348019+0000","last_fullsized":"2023-09-23T23:12:18.348019+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:15.279668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":112,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:10.432268+0000","last_change":"2023-09-23T23:10:59.506092+0000","last_active":"2023-09-23T23:12:10.432268+0000","last_peered":"2023-09-23T23:12:10.432268+0000","last_clean":"2023-09-23T23:12:10.432268+0000","last_became_active":"2023-09-23T23:10:59.505550+0000","last_became_peered":"2023-09-23T23:10:59.505550+0000","last_unstale":"2023-09-23T23:12:10.432268+0000","last_undegraded":"2023-09-23T23:12:10.432268+0000","last_fullsized":"2023-09-23T23:12:10.432268+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:10.576890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":118,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.483861+0000","last_change":"2023-09-23T23:10:59.515675+0000","last_active":"2023-09-23T23:12:03.483861+0000","last_peered":"2023-09-23T23:12:03.483861+0000","last_clean":"2023-09-23T23:12:03.483861+0000","last_became_active":"2023-09-23T23:10:59.514600+0000","last_became_peered":"2023-09-23T23:10:59.514600+0000","last_unstale":"2023-09-23T23:12:03.483861+0000","last_undegraded":"2023-09-23T23:12:03.483861+0000","last_fullsized":"2023-09-23T23:12:03.483861+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.273467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489315+0000","last_change":"2023-09-23T23:07:30.078179+0000","last_active":"2023-09-23T23:12:03.489315+0000","last_peered":"2023-09-23T23:12:03.489315+0000","last_clean":"2023-09-23T23:12:03.489315+0000","last_became_active":"2023-09-23T23:07:30.077810+0000","last_became_peered":"2023-09-23T23:07:30.077810+0000","last_unstale":"2023-09-23T23:12:03.489315+0000","last_undegraded":"2023-09-23T23:12:03.489315+0000","last_fullsized":"2023-09-23T23:12:03.489315+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:19.218333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.538903+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:12:03.538903+0000","last_peered":"2023-09-23T23:12:03.538903+0000","last_clean":"2023-09-23T23:12:03.538903+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:12:03.538903+0000","last_undegraded":"2023-09-23T23:12:03.538903+0000","last_fullsized":"2023-09-23T23:12:03.538903+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.483884+0000","last_change":"2023-09-23T23:10:59.515840+0000","last_active":"2023-09-23T23:12:03.483884+0000","last_peered":"2023-09-23T23:12:03.483884+0000","last_clean":"2023-09-23T23:12:03.483884+0000","last_became_active":"2023-09-23T23:10:59.515000+0000","last_became_peered":"2023-09-23T23:10:59.515000+0000","last_unstale":"2023-09-23T23:12:03.483884+0000","last_undegraded":"2023-09-23T23:12:03.483884+0000","last_fullsized":"2023-09-23T23:12:03.483884+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:47.355624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540206+0000","last_change":"2023-09-23T23:07:30.076036+0000","last_active":"2023-09-23T23:12:03.540206+0000","last_peered":"2023-09-23T23:12:03.540206+0000","last_clean":"2023-09-23T23:12:03.540206+0000","last_became_active":"2023-09-23T23:07:30.075733+0000","last_became_peered":"2023-09-23T23:07:30.075733+0000","last_unstale":"2023-09-23T23:12:03.540206+0000","last_undegraded":"2023-09-23T23:12:03.540206+0000","last_fullsized":"2023-09-23T23:12:03.540206+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:56.342424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484007+0000","last_change":"2023-09-23T23:10:59.517290+0000","last_active":"2023-09-23T23:12:03.484007+0000","last_peered":"2023-09-23T23:12:03.484007+0000","last_clean":"2023-09-23T23:12:03.484007+0000","last_became_active":"2023-09-23T23:10:59.513367+0000","last_became_peered":"2023-09-23T23:10:59.513367+0000","last_unstale":"2023-09-23T23:12:03.484007+0000","last_undegraded":"2023-09-23T23:12:03.484007+0000","last_fullsized":"2023-09-23T23:12:03.484007+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:04.328676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488519+0000","last_change":"2023-09-23T23:10:59.500837+0000","last_active":"2023-09-23T23:12:03.488519+0000","last_peered":"2023-09-23T23:12:03.488519+0000","last_clean":"2023-09-23T23:12:03.488519+0000","last_became_active":"2023-09-23T23:10:59.500455+0000","last_became_peered":"2023-09-23T23:10:59.500455+0000","last_unstale":"2023-09-23T23:12:03.488519+0000","last_undegraded":"2023-09-23T23:12:03.488519+0000","last_fullsized":"2023-09-23T23:12:03.488519+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:34.801149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":130,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:08.347005+0000","last_change":"2023-09-23T23:08:45.122701+0000","last_active":"2023-09-23T23:12:08.347005+0000","last_peered":"2023-09-23T23:12:08.347005+0000","last_clean":"2023-09-23T23:12:08.347005+0000","last_became_active":"2023-09-23T23:08:45.122332+0000","last_became_peered":"2023-09-23T23:08:45.122332+0000","last_unstale":"2023-09-23T23:12:08.347005+0000","last_undegraded":"2023-09-23T23:12:08.347005+0000","last_fullsized":"2023-09-23T23:12:08.347005+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:05.771906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":129,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.550194+0000","last_change":"2023-09-23T22:53:16.891930+0000","last_active":"2023-09-23T23:12:03.550194+0000","last_peered":"2023-09-23T23:12:03.550194+0000","last_clean":"2023-09-23T23:12:03.550194+0000","last_became_active":"2023-09-23T22:53:16.891658+0000","last_became_peered":"2023-09-23T22:53:16.891658+0000","last_unstale":"2023-09-23T23:12:03.550194+0000","last_undegraded":"2023-09-23T23:12:03.550194+0000","last_fullsized":"2023-09-23T23:12:03.550194+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:19.762775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.549002+0000","last_change":"2023-09-23T23:07:30.082476+0000","last_active":"2023-09-23T23:12:03.549002+0000","last_peered":"2023-09-23T23:12:03.549002+0000","last_clean":"2023-09-23T23:12:03.549002+0000","last_became_active":"2023-09-23T23:07:30.082201+0000","last_became_peered":"2023-09-23T23:07:30.082201+0000","last_unstale":"2023-09-23T23:12:03.549002+0000","last_undegraded":"2023-09-23T23:12:03.549002+0000","last_fullsized":"2023-09-23T23:12:03.549002+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:08.809163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488205+0000","last_change":"2023-09-23T23:08:45.115767+0000","last_active":"2023-09-23T23:12:03.488205+0000","last_peered":"2023-09-23T23:12:03.488205+0000","last_clean":"2023-09-23T23:12:03.488205+0000","last_became_active":"2023-09-23T23:08:45.115422+0000","last_became_peered":"2023-09-23T23:08:45.115422+0000","last_unstale":"2023-09-23T23:12:03.488205+0000","last_undegraded":"2023-09-23T23:12:03.488205+0000","last_fullsized":"2023-09-23T23:12:03.488205+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:53.216816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484131+0000","last_change":"2023-09-23T23:10:59.518106+0000","last_active":"2023-09-23T23:12:03.484131+0000","last_peered":"2023-09-23T23:12:03.484131+0000","last_clean":"2023-09-23T23:12:03.484131+0000","last_became_active":"2023-09-23T23:10:59.517633+0000","last_became_peered":"2023-09-23T23:10:59.517633+0000","last_unstale":"2023-09-23T23:12:03.484131+0000","last_undegraded":"2023-09-23T23:12:03.484131+0000","last_fullsized":"2023-09-23T23:12:03.484131+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:27.245420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.538685+0000","last_change":"2023-09-23T23:10:59.516989+0000","last_active":"2023-09-23T23:12:03.538685+0000","last_peered":"2023-09-23T23:12:03.538685+0000","last_clean":"2023-09-23T23:12:03.538685+0000","last_became_active":"2023-09-23T23:10:59.516507+0000","last_became_peered":"2023-09-23T23:10:59.516507+0000","last_unstale":"2023-09-23T23:12:03.538685+0000","last_undegraded":"2023-09-23T23:12:03.538685+0000","last_fullsized":"2023-09-23T23:12:03.538685+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:40.834591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540360+0000","last_change":"2023-09-23T23:10:59.498539+0000","last_active":"2023-09-23T23:12:03.540360+0000","last_peered":"2023-09-23T23:12:03.540360+0000","last_clean":"2023-09-23T23:12:03.540360+0000","last_became_active":"2023-09-23T23:10:59.498205+0000","last_became_peered":"2023-09-23T23:10:59.498205+0000","last_unstale":"2023-09-23T23:12:03.540360+0000","last_undegraded":"2023-09-23T23:12:03.540360+0000","last_fullsized":"2023-09-23T23:12:03.540360+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:16.307572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":107,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488255+0000","last_change":"2023-09-23T22:53:16.889362+0000","last_active":"2023-09-23T23:12:03.488255+0000","last_peered":"2023-09-23T23:12:03.488255+0000","last_clean":"2023-09-23T23:12:03.488255+0000","last_became_active":"2023-09-23T22:53:16.889081+0000","last_became_peered":"2023-09-23T22:53:16.889081+0000","last_unstale":"2023-09-23T23:12:03.488255+0000","last_undegraded":"2023-09-23T23:12:03.488255+0000","last_fullsized":"2023-09-23T23:12:03.488255+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:11.321151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.538765+0000","last_change":"2023-09-23T23:10:59.506198+0000","last_active":"2023-09-23T23:12:03.538765+0000","last_peered":"2023-09-23T23:12:03.538765+0000","last_clean":"2023-09-23T23:12:03.538765+0000","last_became_active":"2023-09-23T23:10:59.505912+0000","last_became_peered":"2023-09-23T23:10:59.505912+0000","last_unstale":"2023-09-23T23:12:03.538765+0000","last_undegraded":"2023-09-23T23:12:03.538765+0000","last_fullsized":"2023-09-23T23:12:03.538765+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:30.343257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":132,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488922+0000","last_change":"2023-09-23T23:04:11.518188+0000","last_active":"2023-09-23T23:12:03.488922+0000","last_peered":"2023-09-23T23:12:03.488922+0000","last_clean":"2023-09-23T23:12:03.488922+0000","last_became_active":"2023-09-23T23:04:11.517785+0000","last_became_peered":"2023-09-23T23:04:11.517785+0000","last_unstale":"2023-09-23T23:12:03.488922+0000","last_undegraded":"2023-09-23T23:12:03.488922+0000","last_fullsized":"2023-09-23T23:12:03.488922+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:26.382580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484760+0000","last_change":"2023-09-23T23:10:59.525228+0000","last_active":"2023-09-23T23:12:03.484760+0000","last_peered":"2023-09-23T23:12:03.484760+0000","last_clean":"2023-09-23T23:12:03.484760+0000","last_became_active":"2023-09-23T23:10:59.524665+0000","last_became_peered":"2023-09-23T23:10:59.524665+0000","last_unstale":"2023-09-23T23:12:03.484760+0000","last_undegraded":"2023-09-23T23:12:03.484760+0000","last_fullsized":"2023-09-23T23:12:03.484760+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:42.632316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488678+0000","last_change":"2023-09-23T23:04:11.907427+0000","last_active":"2023-09-23T23:12:03.488678+0000","last_peered":"2023-09-23T23:12:03.488678+0000","last_clean":"2023-09-23T23:12:03.488678+0000","last_became_active":"2023-09-23T23:04:11.907016+0000","last_became_peered":"2023-09-23T23:04:11.907016+0000","last_unstale":"2023-09-23T23:12:03.488678+0000","last_undegraded":"2023-09-23T23:12:03.488678+0000","last_fullsized":"2023-09-23T23:12:03.488678+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:20.715977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":106,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.541229+0000","last_change":"2023-09-23T22:53:18.075506+0000","last_active":"2023-09-23T23:12:03.541229+0000","last_peered":"2023-09-23T23:12:03.541229+0000","last_clean":"2023-09-23T23:12:03.541229+0000","last_became_active":"2023-09-23T22:53:18.075206+0000","last_became_peered":"2023-09-23T22:53:18.075206+0000","last_unstale":"2023-09-23T23:12:03.541229+0000","last_undegraded":"2023-09-23T23:12:03.541229+0000","last_fullsized":"2023-09-23T23:12:03.541229+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:32:16.168188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488500+0000","last_change":"2023-09-23T23:07:30.072322+0000","last_active":"2023-09-23T23:12:03.488500+0000","last_peered":"2023-09-23T23:12:03.488500+0000","last_clean":"2023-09-23T23:12:03.488500+0000","last_became_active":"2023-09-23T23:07:30.071941+0000","last_became_peered":"2023-09-23T23:07:30.071941+0000","last_unstale":"2023-09-23T23:12:03.488500+0000","last_undegraded":"2023-09-23T23:12:03.488500+0000","last_fullsized":"2023-09-23T23:12:03.488500+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:38.994624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548304+0000","last_change":"2023-09-23T23:10:59.512748+0000","last_active":"2023-09-23T23:12:03.548304+0000","last_peered":"2023-09-23T23:12:03.548304+0000","last_clean":"2023-09-23T23:12:03.548304+0000","last_became_active":"2023-09-23T23:10:59.512420+0000","last_became_peered":"2023-09-23T23:10:59.512420+0000","last_unstale":"2023-09-23T23:12:03.548304+0000","last_undegraded":"2023-09-23T23:12:03.548304+0000","last_fullsized":"2023-09-23T23:12:03.548304+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:09.269366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":108,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488293+0000","last_change":"2023-09-23T22:53:16.887968+0000","last_active":"2023-09-23T23:12:03.488293+0000","last_peered":"2023-09-23T23:12:03.488293+0000","last_clean":"2023-09-23T23:12:03.488293+0000","last_became_active":"2023-09-23T22:53:16.887692+0000","last_became_peered":"2023-09-23T22:53:16.887692+0000","last_unstale":"2023-09-23T23:12:03.488293+0000","last_undegraded":"2023-09-23T23:12:03.488293+0000","last_fullsized":"2023-09-23T23:12:03.488293+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:24.662748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":133,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489141+0000","last_change":"2023-09-23T23:08:45.116813+0000","last_active":"2023-09-23T23:12:03.489141+0000","last_peered":"2023-09-23T23:12:03.489141+0000","last_clean":"2023-09-23T23:12:03.489141+0000","last_became_active":"2023-09-23T23:08:45.116435+0000","last_became_peered":"2023-09-23T23:08:45.116435+0000","last_unstale":"2023-09-23T23:12:03.489141+0000","last_undegraded":"2023-09-23T23:12:03.489141+0000","last_fullsized":"2023-09-23T23:12:03.489141+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:40.616551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.539969+0000","last_change":"2023-09-23T23:10:59.502426+0000","last_active":"2023-09-23T23:12:03.539969+0000","last_peered":"2023-09-23T23:12:03.539969+0000","last_clean":"2023-09-23T23:12:03.539969+0000","last_became_active":"2023-09-23T23:10:59.502074+0000","last_became_peered":"2023-09-23T23:10:59.502074+0000","last_unstale":"2023-09-23T23:12:03.539969+0000","last_undegraded":"2023-09-23T23:12:03.539969+0000","last_fullsized":"2023-09-23T23:12:03.539969+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:46.029347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":114,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.539432+0000","last_change":"2023-09-23T23:04:11.912493+0000","last_active":"2023-09-23T23:12:03.539432+0000","last_peered":"2023-09-23T23:12:03.539432+0000","last_clean":"2023-09-23T23:12:03.539432+0000","last_became_active":"2023-09-23T23:04:11.906712+0000","last_became_peered":"2023-09-23T23:04:11.906712+0000","last_unstale":"2023-09-23T23:12:03.539432+0000","last_undegraded":"2023-09-23T23:12:03.539432+0000","last_fullsized":"2023-09-23T23:12:03.539432+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:58.932610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484285+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:12:03.484285+0000","last_peered":"2023-09-23T23:12:03.484285+0000","last_clean":"2023-09-23T23:12:03.484285+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:12:03.484285+0000","last_undegraded":"2023-09-23T23:12:03.484285+0000","last_fullsized":"2023-09-23T23:12:03.484285+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548351+0000","last_change":"2023-09-23T23:10:59.512363+0000","last_active":"2023-09-23T23:12:03.548351+0000","last_peered":"2023-09-23T23:12:03.548351+0000","last_clean":"2023-09-23T23:12:03.548351+0000","last_became_active":"2023-09-23T23:10:59.512022+0000","last_became_peered":"2023-09-23T23:10:59.512022+0000","last_unstale":"2023-09-23T23:12:03.548351+0000","last_undegraded":"2023-09-23T23:12:03.548351+0000","last_fullsized":"2023-09-23T23:12:03.548351+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:45.523432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489017+0000","last_change":"2023-09-23T23:07:30.077349+0000","last_active":"2023-09-23T23:12:03.489017+0000","last_peered":"2023-09-23T23:12:03.489017+0000","last_clean":"2023-09-23T23:12:03.489017+0000","last_became_active":"2023-09-23T23:07:30.077031+0000","last_became_peered":"2023-09-23T23:07:30.077031+0000","last_unstale":"2023-09-23T23:12:03.489017+0000","last_undegraded":"2023-09-23T23:12:03.489017+0000","last_fullsized":"2023-09-23T23:12:03.489017+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:27.708541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548925+0000","last_change":"2023-09-23T23:08:45.117579+0000","last_active":"2023-09-23T23:12:03.548925+0000","last_peered":"2023-09-23T23:12:03.548925+0000","last_clean":"2023-09-23T23:12:03.548925+0000","last_became_active":"2023-09-23T23:08:45.117216+0000","last_became_peered":"2023-09-23T23:08:45.117216+0000","last_unstale":"2023-09-23T23:12:03.548925+0000","last_undegraded":"2023-09-23T23:12:03.548925+0000","last_fullsized":"2023-09-23T23:12:03.548925+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:27.731522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548772+0000","last_change":"2023-09-23T23:08:45.120505+0000","last_active":"2023-09-23T23:12:03.548772+0000","last_peered":"2023-09-23T23:12:03.548772+0000","last_clean":"2023-09-23T23:12:03.548772+0000","last_became_active":"2023-09-23T23:08:45.120136+0000","last_became_peered":"2023-09-23T23:08:45.120136+0000","last_unstale":"2023-09-23T23:12:03.548772+0000","last_undegraded":"2023-09-23T23:12:03.548772+0000","last_fullsized":"2023-09-23T23:12:03.548772+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:56.726515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.549135+0000","last_change":"2023-09-23T23:07:30.081779+0000","last_active":"2023-09-23T23:12:03.549135+0000","last_peered":"2023-09-23T23:12:03.549135+0000","last_clean":"2023-09-23T23:12:03.549135+0000","last_became_active":"2023-09-23T23:07:30.081425+0000","last_became_peered":"2023-09-23T23:07:30.081425+0000","last_unstale":"2023-09-23T23:12:03.549135+0000","last_undegraded":"2023-09-23T23:12:03.549135+0000","last_fullsized":"2023-09-23T23:12:03.549135+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:01.231206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.483964+0000","last_change":"2023-09-23T23:10:59.516007+0000","last_active":"2023-09-23T23:12:03.483964+0000","last_peered":"2023-09-23T23:12:03.483964+0000","last_clean":"2023-09-23T23:12:03.483964+0000","last_became_active":"2023-09-23T23:10:59.515388+0000","last_became_peered":"2023-09-23T23:10:59.515388+0000","last_unstale":"2023-09-23T23:12:03.483964+0000","last_undegraded":"2023-09-23T23:12:03.483964+0000","last_fullsized":"2023-09-23T23:12:03.483964+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:13.171258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":130,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486491+0000","last_change":"2023-09-23T23:10:59.503483+0000","last_active":"2023-09-23T23:12:03.486491+0000","last_peered":"2023-09-23T23:12:03.486491+0000","last_clean":"2023-09-23T23:12:03.486491+0000","last_became_active":"2023-09-23T23:10:59.502851+0000","last_became_peered":"2023-09-23T23:10:59.502851+0000","last_unstale":"2023-09-23T23:12:03.486491+0000","last_undegraded":"2023-09-23T23:12:03.486491+0000","last_fullsized":"2023-09-23T23:12:03.486491+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:20.388467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486449+0000","last_change":"2023-09-23T23:07:30.069993+0000","last_active":"2023-09-23T23:12:03.486449+0000","last_peered":"2023-09-23T23:12:03.486449+0000","last_clean":"2023-09-23T23:12:03.486449+0000","last_became_active":"2023-09-23T23:07:30.069634+0000","last_became_peered":"2023-09-23T23:07:30.069634+0000","last_unstale":"2023-09-23T23:12:03.486449+0000","last_undegraded":"2023-09-23T23:12:03.486449+0000","last_fullsized":"2023-09-23T23:12:03.486449+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:24.608905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":141,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540255+0000","last_change":"2023-09-23T23:10:59.498546+0000","last_active":"2023-09-23T23:12:03.540255+0000","last_peered":"2023-09-23T23:12:03.540255+0000","last_clean":"2023-09-23T23:12:03.540255+0000","last_became_active":"2023-09-23T23:10:59.498231+0000","last_became_peered":"2023-09-23T23:10:59.498231+0000","last_unstale":"2023-09-23T23:12:03.540255+0000","last_undegraded":"2023-09-23T23:12:03.540255+0000","last_fullsized":"2023-09-23T23:12:03.540255+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:37.404273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":115,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486594+0000","last_change":"2023-09-23T23:07:30.082833+0000","last_active":"2023-09-23T23:12:03.486594+0000","last_peered":"2023-09-23T23:12:03.486594+0000","last_clean":"2023-09-23T23:12:03.486594+0000","last_became_active":"2023-09-23T23:07:30.082454+0000","last_became_peered":"2023-09-23T23:07:30.082454+0000","last_unstale":"2023-09-23T23:12:03.486594+0000","last_undegraded":"2023-09-23T23:12:03.486594+0000","last_fullsized":"2023-09-23T23:12:03.486594+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:50.336760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540250+0000","last_change":"2023-09-23T23:07:30.075093+0000","last_active":"2023-09-23T23:12:03.540250+0000","last_peered":"2023-09-23T23:12:03.540250+0000","last_clean":"2023-09-23T23:12:03.540250+0000","last_became_active":"2023-09-23T23:07:30.074729+0000","last_became_peered":"2023-09-23T23:07:30.074729+0000","last_unstale":"2023-09-23T23:12:03.540250+0000","last_undegraded":"2023-09-23T23:12:03.540250+0000","last_fullsized":"2023-09-23T23:12:03.540250+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:34.502057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548916+0000","last_change":"2023-09-23T23:04:11.514703+0000","last_active":"2023-09-23T23:12:03.548916+0000","last_peered":"2023-09-23T23:12:03.548916+0000","last_clean":"2023-09-23T23:12:03.548916+0000","last_became_active":"2023-09-23T23:04:11.514403+0000","last_became_peered":"2023-09-23T23:04:11.514403+0000","last_unstale":"2023-09-23T23:12:03.548916+0000","last_undegraded":"2023-09-23T23:12:03.548916+0000","last_fullsized":"2023-09-23T23:12:03.548916+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:26.374346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":145,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486345+0000","last_change":"2023-09-23T23:10:59.502030+0000","last_active":"2023-09-23T23:12:03.486345+0000","last_peered":"2023-09-23T23:12:03.486345+0000","last_clean":"2023-09-23T23:12:03.486345+0000","last_became_active":"2023-09-23T23:10:59.501714+0000","last_became_peered":"2023-09-23T23:10:59.501714+0000","last_unstale":"2023-09-23T23:12:03.486345+0000","last_undegraded":"2023-09-23T23:12:03.486345+0000","last_fullsized":"2023-09-23T23:12:03.486345+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:52.898368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":119,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.353626+0000","last_change":"2023-09-23T23:10:59.501544+0000","last_active":"2023-09-23T23:12:20.353626+0000","last_peered":"2023-09-23T23:12:20.353626+0000","last_clean":"2023-09-23T23:12:20.353626+0000","last_became_active":"2023-09-23T23:10:59.501247+0000","last_became_peered":"2023-09-23T23:10:59.501247+0000","last_unstale":"2023-09-23T23:12:20.353626+0000","last_undegraded":"2023-09-23T23:12:20.353626+0000","last_fullsized":"2023-09-23T23:12:20.353626+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:54.254755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486076+0000","last_change":"2023-09-23T23:08:45.122712+0000","last_active":"2023-09-23T23:12:03.486076+0000","last_peered":"2023-09-23T23:12:03.486076+0000","last_clean":"2023-09-23T23:12:03.486076+0000","last_became_active":"2023-09-23T23:08:45.122391+0000","last_became_peered":"2023-09-23T23:08:45.122391+0000","last_unstale":"2023-09-23T23:12:03.486076+0000","last_undegraded":"2023-09-23T23:12:03.486076+0000","last_fullsized":"2023-09-23T23:12:03.486076+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:44.916349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":117,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486558+0000","last_change":"2023-09-23T23:07:30.074957+0000","last_active":"2023-09-23T23:12:03.486558+0000","last_peered":"2023-09-23T23:12:03.486558+0000","last_clean":"2023-09-23T23:12:03.486558+0000","last_became_active":"2023-09-23T23:07:30.074475+0000","last_became_peered":"2023-09-23T23:07:30.074475+0000","last_unstale":"2023-09-23T23:12:03.486558+0000","last_undegraded":"2023-09-23T23:12:03.486558+0000","last_fullsized":"2023-09-23T23:12:03.486558+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:25.337999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540551+0000","last_change":"2023-09-23T23:08:45.111935+0000","last_active":"2023-09-23T23:12:03.540551+0000","last_peered":"2023-09-23T23:12:03.540551+0000","last_clean":"2023-09-23T23:12:03.540551+0000","last_became_active":"2023-09-23T23:08:45.111549+0000","last_became_peered":"2023-09-23T23:08:45.111549+0000","last_unstale":"2023-09-23T23:12:03.540551+0000","last_undegraded":"2023-09-23T23:12:03.540551+0000","last_fullsized":"2023-09-23T23:12:03.540551+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:49.879205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":504,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488060+0000","last_change":"2023-09-23T23:04:11.517317+0000","last_active":"2023-09-23T23:12:03.488060+0000","last_peered":"2023-09-23T23:12:03.488060+0000","last_clean":"2023-09-23T23:12:03.488060+0000","last_became_active":"2023-09-23T23:04:11.516929+0000","last_became_peered":"2023-09-23T23:04:11.516929+0000","last_unstale":"2023-09-23T23:12:03.488060+0000","last_undegraded":"2023-09-23T23:12:03.488060+0000","last_fullsized":"2023-09-23T23:12:03.488060+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T22:49:49.548021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:19.871312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":79,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:19.134029+0000","last_change":"2023-09-23T23:10:59.512923+0000","last_active":"2023-09-23T23:12:19.134029+0000","last_peered":"2023-09-23T23:12:19.134029+0000","last_clean":"2023-09-23T23:12:19.134029+0000","last_became_active":"2023-09-23T23:10:59.512637+0000","last_became_peered":"2023-09-23T23:10:59.512637+0000","last_unstale":"2023-09-23T23:12:19.134029+0000","last_undegraded":"2023-09-23T23:12:19.134029+0000","last_fullsized":"2023-09-23T23:12:19.134029+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:43:06.983322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":106,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489400+0000","last_change":"2023-09-23T22:53:18.075922+0000","last_active":"2023-09-23T23:12:03.489400+0000","last_peered":"2023-09-23T23:12:03.489400+0000","last_clean":"2023-09-23T23:12:03.489400+0000","last_became_active":"2023-09-23T22:53:18.075621+0000","last_became_peered":"2023-09-23T22:53:18.075621+0000","last_unstale":"2023-09-23T23:12:03.489400+0000","last_undegraded":"2023-09-23T23:12:03.489400+0000","last_fullsized":"2023-09-23T23:12:03.489400+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:01.490333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":145,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488321+0000","last_change":"2023-09-23T23:07:30.068847+0000","last_active":"2023-09-23T23:12:03.488321+0000","last_peered":"2023-09-23T23:12:03.488321+0000","last_clean":"2023-09-23T23:12:03.488321+0000","last_became_active":"2023-09-23T23:07:30.068533+0000","last_became_peered":"2023-09-23T23:07:30.068533+0000","last_unstale":"2023-09-23T23:12:03.488321+0000","last_undegraded":"2023-09-23T23:12:03.488321+0000","last_fullsized":"2023-09-23T23:12:03.488321+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:59.598863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":112,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.255403+0000","last_change":"2023-09-23T23:10:59.503085+0000","last_active":"2023-09-23T23:12:20.255403+0000","last_peered":"2023-09-23T23:12:20.255403+0000","last_clean":"2023-09-23T23:12:20.255403+0000","last_became_active":"2023-09-23T23:10:59.502737+0000","last_became_peered":"2023-09-23T23:10:59.502737+0000","last_unstale":"2023-09-23T23:12:20.255403+0000","last_undegraded":"2023-09-23T23:12:20.255403+0000","last_fullsized":"2023-09-23T23:12:20.255403+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:22.783324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":127,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.259269+0000","last_change":"2023-09-23T23:08:45.116856+0000","last_active":"2023-09-23T23:12:20.259269+0000","last_peered":"2023-09-23T23:12:20.259269+0000","last_clean":"2023-09-23T23:12:20.259269+0000","last_became_active":"2023-09-23T23:08:45.116519+0000","last_became_peered":"2023-09-23T23:08:45.116519+0000","last_unstale":"2023-09-23T23:12:20.259269+0000","last_undegraded":"2023-09-23T23:12:20.259269+0000","last_fullsized":"2023-09-23T23:12:20.259269+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:25.917776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488391+0000","last_change":"2023-09-23T23:04:11.900743+0000","last_active":"2023-09-23T23:12:03.488391+0000","last_peered":"2023-09-23T23:12:03.488391+0000","last_clean":"2023-09-23T23:12:03.488391+0000","last_became_active":"2023-09-23T23:04:11.900418+0000","last_became_peered":"2023-09-23T23:04:11.900418+0000","last_unstale":"2023-09-23T23:12:03.488391+0000","last_undegraded":"2023-09-23T23:12:03.488391+0000","last_fullsized":"2023-09-23T23:12:03.488391+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:45:31.893513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486279+0000","last_change":"2023-09-23T23:08:45.124284+0000","last_active":"2023-09-23T23:12:03.486279+0000","last_peered":"2023-09-23T23:12:03.486279+0000","last_clean":"2023-09-23T23:12:03.486279+0000","last_became_active":"2023-09-23T23:08:45.123988+0000","last_became_peered":"2023-09-23T23:08:45.123988+0000","last_unstale":"2023-09-23T23:12:03.486279+0000","last_undegraded":"2023-09-23T23:12:03.486279+0000","last_fullsized":"2023-09-23T23:12:03.486279+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:26.726970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":97,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.255678+0000","last_change":"2023-09-23T23:04:11.911507+0000","last_active":"2023-09-23T23:12:20.255678+0000","last_peered":"2023-09-23T23:12:20.255678+0000","last_clean":"2023-09-23T23:12:20.255678+0000","last_became_active":"2023-09-23T23:04:11.908281+0000","last_became_peered":"2023-09-23T23:04:11.908281+0000","last_unstale":"2023-09-23T23:12:20.255678+0000","last_undegraded":"2023-09-23T23:12:20.255678+0000","last_fullsized":"2023-09-23T23:12:20.255678+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:30.414860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484694+0000","last_change":"2023-09-23T23:10:59.511903+0000","last_active":"2023-09-23T23:12:03.484694+0000","last_peered":"2023-09-23T23:12:03.484694+0000","last_clean":"2023-09-23T23:12:03.484694+0000","last_became_active":"2023-09-23T23:10:59.511022+0000","last_became_peered":"2023-09-23T23:10:59.511022+0000","last_unstale":"2023-09-23T23:12:03.484694+0000","last_undegraded":"2023-09-23T23:12:03.484694+0000","last_fullsized":"2023-09-23T23:12:03.484694+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:03.498373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":116,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486326+0000","last_change":"2023-09-23T23:07:30.083011+0000","last_active":"2023-09-23T23:12:03.486326+0000","last_peered":"2023-09-23T23:12:03.486326+0000","last_clean":"2023-09-23T23:12:03.486326+0000","last_became_active":"2023-09-23T23:07:30.082544+0000","last_became_peered":"2023-09-23T23:07:30.082544+0000","last_unstale":"2023-09-23T23:12:03.486326+0000","last_undegraded":"2023-09-23T23:12:03.486326+0000","last_fullsized":"2023-09-23T23:12:03.486326+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:20.624316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":122,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:19.134518+0000","last_change":"2023-09-23T23:10:59.512305+0000","last_active":"2023-09-23T23:12:19.134518+0000","last_peered":"2023-09-23T23:12:19.134518+0000","last_clean":"2023-09-23T23:12:19.134518+0000","last_became_active":"2023-09-23T23:10:59.511941+0000","last_became_peered":"2023-09-23T23:10:59.511941+0000","last_unstale":"2023-09-23T23:12:19.134518+0000","last_undegraded":"2023-09-23T23:12:19.134518+0000","last_fullsized":"2023-09-23T23:12:19.134518+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:06.293763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489269+0000","last_change":"2023-09-23T23:08:45.116806+0000","last_active":"2023-09-23T23:12:03.489269+0000","last_peered":"2023-09-23T23:12:03.489269+0000","last_clean":"2023-09-23T23:12:03.489269+0000","last_became_active":"2023-09-23T23:08:45.116462+0000","last_became_peered":"2023-09-23T23:08:45.116462+0000","last_unstale":"2023-09-23T23:12:03.489269+0000","last_undegraded":"2023-09-23T23:12:03.489269+0000","last_fullsized":"2023-09-23T23:12:03.489269+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:24.502987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":116,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.259196+0000","last_change":"2023-09-23T23:07:30.072388+0000","last_active":"2023-09-23T23:12:20.259196+0000","last_peered":"2023-09-23T23:12:20.259196+0000","last_clean":"2023-09-23T23:12:20.259196+0000","last_became_active":"2023-09-23T23:07:30.072099+0000","last_became_peered":"2023-09-23T23:07:30.072099+0000","last_unstale":"2023-09-23T23:12:20.259196+0000","last_undegraded":"2023-09-23T23:12:20.259196+0000","last_fullsized":"2023-09-23T23:12:20.259196+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:19.758734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":106,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548614+0000","last_change":"2023-09-23T22:53:17.905706+0000","last_active":"2023-09-23T23:12:03.548614+0000","last_peered":"2023-09-23T23:12:03.548614+0000","last_clean":"2023-09-23T23:12:03.548614+0000","last_became_active":"2023-09-23T22:53:17.905309+0000","last_became_peered":"2023-09-23T22:53:17.905309+0000","last_unstale":"2023-09-23T23:12:03.548614+0000","last_undegraded":"2023-09-23T23:12:03.548614+0000","last_fullsized":"2023-09-23T23:12:03.548614+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:06.105426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.539475+0000","last_change":"2023-09-23T23:04:11.900795+0000","last_active":"2023-09-23T23:12:03.539475+0000","last_peered":"2023-09-23T23:12:03.539475+0000","last_clean":"2023-09-23T23:12:03.539475+0000","last_became_active":"2023-09-23T23:04:11.899751+0000","last_became_peered":"2023-09-23T23:04:11.899751+0000","last_unstale":"2023-09-23T23:12:03.539475+0000","last_undegraded":"2023-09-23T23:12:03.539475+0000","last_fullsized":"2023-09-23T23:12:03.539475+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:41.206493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":134,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.525133+0000","last_change":"2023-09-23T23:10:59.511474+0000","last_active":"2023-09-23T23:12:03.525133+0000","last_peered":"2023-09-23T23:12:03.525133+0000","last_clean":"2023-09-23T23:12:03.525133+0000","last_became_active":"2023-09-23T23:10:59.511175+0000","last_became_peered":"2023-09-23T23:10:59.511175+0000","last_unstale":"2023-09-23T23:12:03.525133+0000","last_undegraded":"2023-09-23T23:12:03.525133+0000","last_fullsized":"2023-09-23T23:12:03.525133+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:41.647976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540448+0000","last_change":"2023-09-23T23:07:30.075311+0000","last_active":"2023-09-23T23:12:03.540448+0000","last_peered":"2023-09-23T23:12:03.540448+0000","last_clean":"2023-09-23T23:12:03.540448+0000","last_became_active":"2023-09-23T23:07:30.074952+0000","last_became_peered":"2023-09-23T23:07:30.074952+0000","last_unstale":"2023-09-23T23:12:03.540448+0000","last_undegraded":"2023-09-23T23:12:03.540448+0000","last_fullsized":"2023-09-23T23:12:03.540448+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:16.724083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548332+0000","last_change":"2023-09-23T23:10:59.512738+0000","last_active":"2023-09-23T23:12:03.548332+0000","last_peered":"2023-09-23T23:12:03.548332+0000","last_clean":"2023-09-23T23:12:03.548332+0000","last_became_active":"2023-09-23T23:10:59.512011+0000","last_became_peered":"2023-09-23T23:10:59.512011+0000","last_unstale":"2023-09-23T23:12:03.548332+0000","last_undegraded":"2023-09-23T23:12:03.548332+0000","last_fullsized":"2023-09-23T23:12:03.548332+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:18.088964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":109,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540827+0000","last_change":"2023-09-23T22:53:16.891297+0000","last_active":"2023-09-23T23:12:03.540827+0000","last_peered":"2023-09-23T23:12:03.540827+0000","last_clean":"2023-09-23T23:12:03.540827+0000","last_became_active":"2023-09-23T22:53:16.891025+0000","last_became_peered":"2023-09-23T22:53:16.891025+0000","last_unstale":"2023-09-23T23:12:03.540827+0000","last_undegraded":"2023-09-23T23:12:03.540827+0000","last_fullsized":"2023-09-23T23:12:03.540827+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:40:50.608339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548305+0000","last_change":"2023-09-23T23:10:59.512247+0000","last_active":"2023-09-23T23:12:03.548305+0000","last_peered":"2023-09-23T23:12:03.548305+0000","last_clean":"2023-09-23T23:12:03.548305+0000","last_became_active":"2023-09-23T23:10:59.511865+0000","last_became_peered":"2023-09-23T23:10:59.511865+0000","last_unstale":"2023-09-23T23:12:03.548305+0000","last_undegraded":"2023-09-23T23:12:03.548305+0000","last_fullsized":"2023-09-23T23:12:03.548305+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:20.981239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":121,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:19.134146+0000","last_change":"2023-09-23T23:10:59.512958+0000","last_active":"2023-09-23T23:12:19.134146+0000","last_peered":"2023-09-23T23:12:19.134146+0000","last_clean":"2023-09-23T23:12:19.134146+0000","last_became_active":"2023-09-23T23:10:59.512628+0000","last_became_peered":"2023-09-23T23:10:59.512628+0000","last_unstale":"2023-09-23T23:12:19.134146+0000","last_undegraded":"2023-09-23T23:12:19.134146+0000","last_fullsized":"2023-09-23T23:12:19.134146+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:16:26.998696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":135,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.483842+0000","last_change":"2023-09-23T23:10:59.511365+0000","last_active":"2023-09-23T23:12:03.483842+0000","last_peered":"2023-09-23T23:12:03.483842+0000","last_clean":"2023-09-23T23:12:03.483842+0000","last_became_active":"2023-09-23T23:10:59.511073+0000","last_became_peered":"2023-09-23T23:10:59.511073+0000","last_unstale":"2023-09-23T23:12:03.483842+0000","last_undegraded":"2023-09-23T23:12:03.483842+0000","last_fullsized":"2023-09-23T23:12:03.483842+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:35.196455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":116,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.256275+0000","last_change":"2023-09-23T23:08:45.113546+0000","last_active":"2023-09-23T23:12:20.256275+0000","last_peered":"2023-09-23T23:12:20.256275+0000","last_clean":"2023-09-23T23:12:20.256275+0000","last_became_active":"2023-09-23T23:08:45.113191+0000","last_became_peered":"2023-09-23T23:08:45.113191+0000","last_unstale":"2023-09-23T23:12:20.256275+0000","last_undegraded":"2023-09-23T23:12:20.256275+0000","last_fullsized":"2023-09-23T23:12:20.256275+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:06.372282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":133,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548440+0000","last_change":"2023-09-23T23:08:45.116882+0000","last_active":"2023-09-23T23:12:03.548440+0000","last_peered":"2023-09-23T23:12:03.548440+0000","last_clean":"2023-09-23T23:12:03.548440+0000","last_became_active":"2023-09-23T23:08:45.116543+0000","last_became_peered":"2023-09-23T23:08:45.116543+0000","last_unstale":"2023-09-23T23:12:03.548440+0000","last_undegraded":"2023-09-23T23:12:03.548440+0000","last_fullsized":"2023-09-23T23:12:03.548440+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:08.722088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234244,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":287872,"kb_used_data":2168,"kb_used_omap":0,"kb_used_meta":285696,"kb_avail":93457280,"statfs":{"total":95995035648,"available":95700254720,"internally_reserved":0,"allocated":2220032,"data_stored":1663377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":292552704},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5429999999999999,"5min":2.911,"15min":2.7999999999999998},"min":{"1min":0.23799999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":4.9340000000000002,"5min":92.650999999999996,"15min":230.249},"last":0.48799999999999999},{"interface":"front","average":{"1min":2.863,"5min":3.133,"15min":2.8500000000000001},"min":{"1min":0.28799999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":12.004,"5min":93.171000000000006,"15min":230.261},"last":0.54300000000000004}]},{"osd":1,"last update":"Sat Sep 23 23:12:39 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":4.069,"15min":5.1349999999999998},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.2649999999999997,"5min":93.245999999999995,"15min":93.245999999999995},"last":0.69599999999999995},{"interface":"front","average":{"1min":1.889,"5min":3.7010000000000001,"15min":4.9349999999999996},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.9349999999999996,"5min":92.748000000000005,"15min":92.748000000000005},"last":0.61199999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:12:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":2.6680000000000001,"15min":3.1509999999999998},"min":{"1min":0.435,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":12.042,"5min":12.183,"15min":12.183},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.3360000000000001,"5min":2.0720000000000001,"15min":2.25},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.327,"5min":11.781000000000001,"15min":11.781000000000001},"last":0.56899999999999995}]},{"osd":3,"last update":"Sat Sep 23 23:13:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":2.2090000000000001,"15min":2.379},"min":{"1min":0.40600000000000003,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.2610000000000001,"5min":11.507999999999999,"15min":11.507999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.90700000000000003,"5min":2.2360000000000002,"15min":2.4580000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":3.569,"5min":8.9670000000000005,"15min":8.9670000000000005},"last":0.52500000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:12:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1269999999999998,"5min":2.3140000000000001,"15min":2.2250000000000001},"min":{"1min":0.28199999999999997,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":7.8529999999999998,"5min":92.781000000000006,"15min":194.797},"last":0.43099999999999999},{"interface":"front","average":{"1min":2.2770000000000001,"5min":2.2909999999999999,"15min":2.1859999999999999},"min":{"1min":0.33500000000000002,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":7.3659999999999997,"5min":92.626999999999995,"15min":190.82599999999999},"last":0.50600000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:12:16 2023","interfaces":[{"interface":"back","average":{"1min":2.9049999999999998,"5min":2.8050000000000002,"15min":1.9490000000000001},"min":{"1min":0.29799999999999999,"5min":0.29299999999999998,"15min":0.26500000000000001},"max":{"1min":12.067,"5min":93.135000000000005,"15min":93.135000000000005},"last":0.59599999999999997},{"interface":"front","average":{"1min":1.111,"5min":2.2450000000000001,"15min":1.742},"min":{"1min":0.35799999999999998,"5min":0.29799999999999999,"15min":0.29399999999999998},"max":{"1min":5.1890000000000001,"5min":92.885000000000005,"15min":92.885000000000005},"last":0.66000000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:12:16 2023","interfaces":[{"interface":"back","average":{"1min":2.282,"5min":2.274,"15min":1.784},"min":{"1min":0.315,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":13.590999999999999,"5min":93.105999999999995,"15min":93.105999999999995},"last":0.71399999999999997},{"interface":"front","average":{"1min":2.6779999999999999,"5min":2.512,"15min":1.585},"min":{"1min":0.33800000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.6240000000000006,"5min":93.198999999999998,"15min":93.198999999999998},"last":0.63100000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495880,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":267184,"kb_used_data":2152,"kb_used_omap":0,"kb_used_meta":265024,"kb_avail":93477968,"statfs":{"total":95995035648,"available":95721439232,"internally_reserved":0,"allocated":2203648,"data_stored":1662966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":271384576},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6439999999999999,"5min":1.706,"15min":4.2039999999999997},"min":{"1min":0.246,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":5.6310000000000002,"5min":11.004,"15min":97.858000000000004},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.208,"5min":1.6830000000000001,"15min":4.8630000000000004},"min":{"1min":0.27100000000000002,"5min":0.24399999999999999,"15min":0.22700000000000001},"max":{"1min":4.367,"5min":8.3059999999999992,"15min":99.793000000000006},"last":0.38900000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:12:34 2023","interfaces":[{"interface":"back","average":{"1min":1.601,"5min":4.2450000000000001,"15min":2.02},"min":{"1min":0.28599999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.6779999999999999,"5min":209.00899999999999,"15min":209.00899999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.417,"5min":4.4219999999999997,"15min":2.367},"min":{"1min":0.30599999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.173,"5min":209.03,"15min":209.03},"last":0.34300000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3420000000000001,"5min":1.827,"15min":1.6180000000000001},"min":{"1min":0.38600000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.657,"5min":10.871,"15min":10.871},"last":0.60199999999999998},{"interface":"front","average":{"1min":1.5609999999999999,"5min":2.242,"15min":1.9670000000000001},"min":{"1min":0.40999999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.3339999999999996,"5min":12.148,"15min":12.148},"last":0.67300000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:13:02 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":2.2909999999999999,"15min":2.5529999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.3529999999999998,"5min":7.4059999999999997,"15min":7.4059999999999997},"last":0.5},{"interface":"front","average":{"1min":1.0649999999999999,"5min":2.718,"15min":2.9940000000000002},"min":{"1min":0.432,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":5.3250000000000002,"5min":9.4550000000000001,"15min":9.4550000000000001},"last":0.69599999999999995}]},{"osd":4,"last update":"Sat Sep 23 23:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":1.4530000000000001,"15min":1.262},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.8879999999999999,"5min":11.890000000000001,"15min":12.919},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.997,"5min":1.6919999999999999,"15min":1.4039999999999999},"min":{"1min":0.312,"5min":0.308,"15min":0.24299999999999999},"max":{"1min":6.3079999999999998,"5min":11.728,"15min":11.728},"last":0.54500000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.804,"15min":1.5289999999999999},"min":{"1min":0.41799999999999998,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":3.9289999999999998,"5min":26.158999999999999,"15min":26.158999999999999},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.623,"15min":1.4359999999999999},"min":{"1min":0.45700000000000002,"5min":0.30599999999999999,"15min":0.30199999999999999},"max":{"1min":3.5179999999999998,"5min":25.786000000000001,"15min":25.786000000000001},"last":0.61399999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:12:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1099999999999999,"5min":1.589,"15min":1.716},"min":{"1min":0.36399999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.7149999999999999,"5min":10.968999999999999,"15min":12.676},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.9079999999999999,"5min":1.3799999999999999,"15min":1.4350000000000001},"min":{"1min":0.33700000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.5919999999999996,"5min":7.9669999999999996,"15min":9.7889999999999997},"last":0.72299999999999998}]}]},{"osd":1,"up_from":112,"seq":481036337221,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":239752,"kb_used_data":1728,"kb_used_omap":4,"kb_used_meta":238011,"kb_avail":93505400,"statfs":{"total":95995035648,"available":95749529600,"internally_reserved":0,"allocated":1769472,"data_stored":1089061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":243724174},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":2.012,"15min":1.6759999999999999},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":5.8730000000000002,"5min":10.221,"15min":10.221},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.718,"15min":1.486},"min":{"1min":0.255,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":4.2549999999999999,"5min":12.212,"15min":12.212},"last":0.436}]},{"osd":2,"last update":"Sat Sep 23 23:12:47 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":2.3410000000000002,"15min":2.4079999999999999},"min":{"1min":0.22,"5min":0.192,"15min":0.192},"max":{"1min":7.4939999999999998,"5min":16.696999999999999,"15min":16.696999999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":2.1960000000000002,"15min":2.3069999999999999},"min":{"1min":0.23300000000000001,"5min":0.215,"15min":0.215},"max":{"1min":2.448,"5min":9.0879999999999992,"15min":9.0879999999999992},"last":0.36299999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:12:03 2023","interfaces":[{"interface":"back","average":{"1min":2.7029999999999998,"5min":2.7029999999999998,"15min":2.7029999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":12.234999999999999,"5min":12.234999999999999,"15min":12.234999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":2.2909999999999999,"5min":2.2909999999999999,"15min":2.2909999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":12.256,"5min":12.256,"15min":12.256},"last":0.41299999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":1.841,"15min":1.552},"min":{"1min":0.372,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.0720000000000001,"5min":12.676,"15min":12.676},"last":0.624},{"interface":"front","average":{"1min":1.403,"5min":1.873,"15min":1.4810000000000001},"min":{"1min":0.45200000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.8710000000000004,"5min":11.125,"15min":11.125},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":2.5619999999999998,"15min":3.9620000000000002},"min":{"1min":0.34999999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.6749999999999998,"5min":67.497,"15min":67.497},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.827,"5min":2.7170000000000001,"15min":4.016},"min":{"1min":0.34599999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.4660000000000002,"5min":67.540999999999997,"15min":67.540999999999997},"last":0.51800000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.6739999999999999,"15min":1.5780000000000001},"min":{"1min":0.40799999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.6429999999999998,"5min":12.186,"15min":12.186},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.9239999999999999,"5min":2.2730000000000001,"15min":1.679},"min":{"1min":0.39700000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.4029999999999996,"5min":12.785,"15min":12.785},"last":0.60299999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":2.0830000000000002,"5min":2.2549999999999999,"15min":1.7150000000000001},"min":{"1min":0.46300000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.4359999999999999,"5min":12.727,"15min":12.727},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.3240000000000001,"5min":1.9099999999999999,"15min":1.5960000000000001},"min":{"1min":0.40300000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":5.593,"5min":9.8559999999999999,"15min":9.8559999999999999},"last":0.53900000000000003}]}]},{"osd":0,"up_from":91,"seq":390842024045,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":349732,"kb_used_data":2332,"kb_used_omap":5,"kb_used_meta":347386,"kb_avail":93395420,"statfs":{"total":95995035648,"available":95636910080,"internally_reserved":0,"allocated":2387968,"data_stored":1680718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":355723898},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:12:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":2.1930000000000001,"15min":2.452},"min":{"1min":0.19700000000000001,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":11.029,"5min":21.629999999999999,"15min":21.629999999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.4510000000000001,"5min":2.25,"15min":2.2759999999999998},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":9.702,"5min":13.382,"15min":13.382},"last":0.31900000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:12:54 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":2.1560000000000001,"15min":2.1179999999999999},"min":{"1min":0.22700000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":6.0010000000000003,"5min":10.199,"15min":10.199},"last":0.307},{"interface":"front","average":{"1min":0.55200000000000005,"5min":2.0129999999999999,"15min":2.0030000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.7679999999999998,"5min":9.0239999999999991,"15min":9.0239999999999991},"last":0.35699999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:12:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":2.895,"15min":3.2709999999999999},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.0460000000000003,"5min":11.866,"15min":11.866},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.47999999999999998,"5min":2.0990000000000002,"15min":2.3690000000000002},"min":{"1min":0.25,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.351,"5min":9.0980000000000008,"15min":9.0980000000000008},"last":0.27900000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":2.4289999999999998,"15min":2.2389999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.245},"max":{"1min":8.8469999999999995,"5min":28.376999999999999,"15min":28.376999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.6899999999999999,"5min":2.2480000000000002,"15min":2.5910000000000002},"min":{"1min":0.32700000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.1029999999999998,"5min":30.706,"15min":30.706},"last":0.56599999999999995}]},{"osd":5,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":2.2519999999999998,"15min":2.6110000000000002},"min":{"1min":0.38700000000000001,"5min":0.33700000000000002,"15min":0.30499999999999999},"max":{"1min":4.3419999999999996,"5min":15.819000000000001,"15min":30.873999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.9330000000000001,"5min":2.0699999999999998,"15min":1.956},"min":{"1min":0.47199999999999998,"5min":0.30299999999999999,"15min":0.29499999999999998},"max":{"1min":6.0709999999999997,"5min":10.117000000000001,"15min":10.117000000000001},"last":0.52100000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.0270000000000001,"5min":2.3090000000000002,"15min":2.6360000000000001},"min":{"1min":0.35899999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":10.430999999999999,"5min":15.859,"15min":21.047999999999998},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.3130000000000002,"5min":2.3140000000000001,"15min":2.036},"min":{"1min":0.39700000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":10.414,"5min":10.414,"15min":10.414},"last":0.55300000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.339,"5min":2.3809999999999998,"15min":2.0600000000000001},"min":{"1min":0.45900000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.1899999999999995,"5min":10.145,"15min":10.145},"last":0.435},{"interface":"front","average":{"1min":1.9650000000000001,"5min":2.2469999999999999,"15min":2.2069999999999999},"min":{"1min":0.34999999999999998,"5min":0.313,"15min":0.313},"max":{"1min":9.3059999999999992,"5min":10.513999999999999,"15min":10.513999999999999},"last":0.54300000000000004}]}]},{"osd":2,"up_from":130,"seq":558345748534,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":277716,"kb_used_data":1676,"kb_used_omap":7,"kb_used_meta":276024,"kb_avail":93467436,"statfs":{"total":95995035648,"available":95710654464,"internally_reserved":0,"allocated":1716224,"data_stored":1082695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":282648657},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":2.0609999999999999,"15min":2.036},"min":{"1min":0.185,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":8.9209999999999994,"5min":16.021000000000001,"15min":16.021000000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.96999999999999997,"5min":2.5609999999999999,"15min":2.9990000000000001},"min":{"1min":0.23799999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.9649999999999999,"5min":16.152999999999999,"15min":16.152999999999999},"last":0.40899999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":2.3210000000000002,"15min":2.6389999999999998},"min":{"1min":0.23200000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":5.0810000000000004,"5min":9.3719999999999999,"15min":9.3719999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.84699999999999998,"5min":2.141,"15min":2.649},"min":{"1min":0.25800000000000001,"5min":0.218,"15min":0.218},"max":{"1min":5.7359999999999998,"5min":11.625,"15min":11.625},"last":0.31}]},{"osd":3,"last update":"Sat Sep 23 23:12:02 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":1.27,"15min":1.27},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.8230000000000004,"5min":6.8230000000000004,"15min":6.8230000000000004},"last":0.255},{"interface":"front","average":{"1min":1.9470000000000001,"5min":1.9470000000000001,"15min":1.9470000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.79,"5min":6.79,"15min":6.79},"last":0.33900000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":1.9990000000000001,"15min":2.0840000000000001},"min":{"1min":0.41099999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.1139999999999999,"5min":10.666,"15min":10.666},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.337,"5min":1.5940000000000001,"15min":1.508},"min":{"1min":0.30399999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.8149999999999995,"5min":14.829000000000001,"15min":14.829000000000001},"last":0.65400000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":2.427,"15min":2.5390000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.9359999999999999,"5min":12.173,"15min":12.173},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.004,"5min":1.619,"15min":1.6519999999999999},"min":{"1min":0.38200000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.7149999999999999,"5min":11.548999999999999,"15min":11.548999999999999},"last":0.55000000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.919,"15min":1.891},"min":{"1min":0.33800000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":8.8620000000000001,"5min":15.205,"15min":15.205},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.284,"5min":2.411,"15min":2.5800000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.9059999999999997,"5min":11.923999999999999,"15min":11.923999999999999},"last":0.51400000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":2.2890000000000001,"15min":2.3250000000000002},"min":{"1min":0.36299999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.8940000000000001,"5min":14.792999999999999,"15min":14.792999999999999},"last":0.63100000000000001},{"interface":"front","average":{"1min":1.446,"5min":2.7799999999999998,"15min":2.9460000000000002},"min":{"1min":0.42799999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.9880000000000004,"5min":15.045,"15min":15.045},"last":0.66800000000000004}]}]},{"osd":3,"up_from":148,"seq":635655159835,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":206060,"kb_used_data":1636,"kb_used_omap":2,"kb_used_meta":204413,"kb_avail":93539092,"statfs":{"total":95995035648,"available":95784030208,"internally_reserved":0,"allocated":1675264,"data_stored":1077514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":209318965},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.1970000000000001,"15min":1.3260000000000001},"min":{"1min":0.221,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":2.2949999999999999,"5min":5.9660000000000002,"15min":5.9660000000000002},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.32100000000000001,"5min":1.8129999999999999,"15min":2.0609999999999999},"min":{"1min":0.221,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.58499999999999996,"5min":9.5839999999999996,"15min":9.5839999999999996},"last":0.33600000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.8700000000000001,"15min":2.0870000000000002},"min":{"1min":0.245,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":2.9740000000000002,"5min":7.1219999999999999,"15min":7.1219999999999999},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.371,"5min":1.635,"15min":1.845},"min":{"1min":0.246,"5min":0.185,"15min":0.185},"max":{"1min":0.71199999999999997,"5min":6.4160000000000004,"15min":6.4160000000000004},"last":0.40600000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":2.0859999999999999,"15min":2.335},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.8399999999999999,"5min":7.9930000000000003,"15min":7.9930000000000003},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.39200000000000002,"5min":2.2810000000000001,"15min":2.5960000000000001},"min":{"1min":0.28799999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.72999999999999998,"5min":12.721,"15min":12.721},"last":0.36699999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":1.544,"15min":1.651},"min":{"1min":0.38600000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.1040000000000001,"5min":7.1040000000000001,"15min":7.1040000000000001},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.90300000000000002,"5min":2.8090000000000002,"15min":3.1269999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.7850000000000001,"5min":13.532999999999999,"15min":13.532999999999999},"last":0.90700000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":2.3700000000000001,"15min":2.665},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":1.1279999999999999,"5min":10.117000000000001,"15min":10.117000000000001},"last":0.97499999999999998},{"interface":"front","average":{"1min":0.81799999999999995,"5min":1.9119999999999999,"15min":2.0939999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":2.8690000000000002,"5min":7.3250000000000002,"15min":7.3250000000000002},"last":0.85999999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":2.4870000000000001,"15min":2.7959999999999998},"min":{"1min":0.45300000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.208,"5min":14.492000000000001,"15min":14.492000000000001},"last":0.76500000000000001},{"interface":"front","average":{"1min":1.173,"5min":2.1059999999999999,"15min":2.262},"min":{"1min":0.42399999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":7.6139999999999999,"5min":7.9640000000000004,"15min":7.9640000000000004},"last":0.93500000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":2.1709999999999998,"15min":2.4249999999999998},"min":{"1min":0.44900000000000001,"5min":0.377,"15min":0.377},"max":{"1min":1.288,"5min":9.7059999999999995,"15min":9.7059999999999995},"last":0.879},{"interface":"front","average":{"1min":0.67100000000000004,"5min":2.298,"15min":2.569},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.379,"5min":9.8010000000000002,"15min":9.8010000000000002},"last":1.0449999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019153,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312124,"kb_used_data":1588,"kb_used_omap":0,"kb_used_meta":310528,"kb_avail":93433028,"statfs":{"total":95995035648,"available":95675420672,"internally_reserved":0,"allocated":1626112,"data_stored":1073262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":317980672},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6639999999999999,"5min":3.8860000000000001,"15min":2.524},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":10.090999999999999,"5min":238.62200000000001,"15min":238.62200000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":3.04,"5min":4.8819999999999997,"15min":3.0569999999999999},"min":{"1min":0.42699999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":11.519,"5min":238.58199999999999,"15min":238.58199999999999},"last":0.45800000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:12:37 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":4.665,"15min":2.3730000000000002},"min":{"1min":0.30499999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":10.132999999999999,"5min":175.00899999999999,"15min":175.00899999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.5760000000000001,"5min":4.9370000000000003,"15min":2.3450000000000002},"min":{"1min":0.35899999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.452,"5min":174.821,"15min":174.821},"last":0.51800000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.762,"15min":1.9119999999999999},"min":{"1min":0.29999999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.664999999999999,"5min":11.664999999999999,"15min":11.664999999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.4239999999999999,"5min":2.3740000000000001,"15min":2.7280000000000002},"min":{"1min":0.443,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":8.3879999999999999,"5min":10.576000000000001,"15min":10.576000000000001},"last":0.54800000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:13:02 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":1.5760000000000001,"15min":1.7430000000000001},"min":{"1min":0.36899999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.498,"5min":8.1609999999999996,"15min":8.1609999999999996},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.90900000000000003,"5min":2.0699999999999998,"15min":2.2639999999999998},"min":{"1min":0.45500000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.0309999999999997,"5min":11.455,"15min":11.455},"last":0.58399999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:12:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":1.1259999999999999,"15min":1.1919999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.23999999999999999},"max":{"1min":10.16,"5min":10.16,"15min":27.391999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.512,"15min":1.6140000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":8.2680000000000007,"5min":8.2680000000000007,"15min":27.222999999999999},"last":0.623}]},{"osd":6,"last update":"Sat Sep 23 23:13:04 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.2130000000000001,"15min":1.1419999999999999},"min":{"1min":0.372,"5min":0.27200000000000002,"15min":0.223},"max":{"1min":0.92000000000000004,"5min":9.3350000000000009,"15min":14.4},"last":0.433},{"interface":"front","average":{"1min":0.70499999999999996,"5min":1.2949999999999999,"15min":1.2689999999999999},"min":{"1min":0.33700000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"max":{"1min":3.3170000000000002,"5min":13.888999999999999,"15min":15.641999999999999},"last":0.59999999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:12:25 2023","interfaces":[{"interface":"back","average":{"1min":2.1589999999999998,"5min":1.5489999999999999,"15min":1.849},"min":{"1min":0.318,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":13.792,"5min":13.792,"15min":119.188},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.6919999999999999,"5min":1.532,"15min":1.847},"min":{"1min":0.35499999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.5960000000000001,"5min":13.055,"15min":119.248},"last":0.502}]}]},{"osd":5,"up_from":38,"seq":163208757516,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":357672,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":356096,"kb_avail":93387480,"statfs":{"total":95995035648,"available":95628779520,"internally_reserved":0,"allocated":1605632,"data_stored":1068741,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":364642304},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:27 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":1.431,"15min":6.6580000000000004},"min":{"1min":0.32100000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.5090000000000003,"5min":10.227,"15min":190.483},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.9870000000000001,"5min":2.0670000000000002,"15min":7.4690000000000003},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.9359999999999999,"5min":12.762,"15min":194.11099999999999},"last":0.496}]},{"osd":1,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":2.8759999999999999,"15min":1.669},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":7.6870000000000003,"5min":91.295000000000002,"15min":91.295000000000002},"last":0.54700000000000004},{"interface":"front","average":{"1min":1.28,"5min":3.1909999999999998,"15min":1.8009999999999999},"min":{"1min":0.26600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.7110000000000003,"5min":91.242000000000004,"15min":91.242000000000004},"last":0.65800000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:12:51 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":2.5049999999999999,"15min":2.798},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.734,"5min":9.8279999999999994,"15min":9.8279999999999994},"last":0.80200000000000005},{"interface":"front","average":{"1min":1.2150000000000001,"5min":1.702,"15min":1.958},"min":{"1min":0.40300000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.2119999999999997,"5min":8.093,"15min":8.093},"last":0.61199999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:12:58 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":1.6499999999999999,"15min":1.79},"min":{"1min":0.44700000000000001,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":2.633,"5min":6.2640000000000002,"15min":6.2640000000000002},"last":0.75},{"interface":"front","average":{"1min":1.002,"5min":2.6880000000000002,"15min":2.9689999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.9420000000000002,"5min":12.802,"15min":12.802},"last":0.64400000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:12:51 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.726,"15min":1.3560000000000001},"min":{"1min":0.38300000000000001,"5min":0.27500000000000002,"15min":0.221},"max":{"1min":7.6459999999999999,"5min":14.538,"15min":14.538},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.92100000000000004,"5min":1.089,"15min":1.153},"min":{"1min":0.315,"5min":0.28100000000000003,"15min":0.25},"max":{"1min":3.8919999999999999,"5min":11.779,"15min":18.827999999999999},"last":0.78800000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:12:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.5289999999999999,"15min":1.3959999999999999},"min":{"1min":0.48699999999999999,"5min":0.315,"15min":0.255},"max":{"1min":3.992,"5min":13.226000000000001,"15min":13.226000000000001},"last":0.68600000000000005},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.833,"15min":1.383},"min":{"1min":0.435,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.1760000000000002,"5min":13.098000000000001,"15min":13.098000000000001},"last":0.70199999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:12:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1829999999999998,"5min":1.8380000000000001,"15min":1.7689999999999999},"min":{"1min":0.38800000000000001,"5min":0.33900000000000002,"15min":0.29199999999999998},"max":{"1min":7.9390000000000001,"5min":14.929,"15min":14.929},"last":0.76700000000000002},{"interface":"front","average":{"1min":1.7430000000000001,"5min":1.722,"15min":1.6180000000000001},"min":{"1min":0.40500000000000003,"5min":0.33400000000000002,"15min":0.29599999999999999},"max":{"1min":10.227,"5min":15.090999999999999,"15min":18.218},"last":0.85399999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:13:09.774 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:13:10.125 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:10.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:10 smithi138 ceph-mon[193606]: from='client.35124 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:13:10.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:10 smithi138 ceph-mon[193606]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:10.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:10 smithi110 ceph-mon[212939]: from='client.35124 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:13:10.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:10 smithi110 ceph-mon[212939]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:10 smithi110 ceph-mon[221102]: from='client.35124 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:13:10.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:10 smithi110 ceph-mon[221102]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:12.022 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:13:12.023 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:13:12.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:12 smithi138 ceph-mon[193606]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:12.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:12 smithi110 ceph-mon[212939]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:12.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:12 smithi110 ceph-mon[221102]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:12.758 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":523,"stamp":"2023-09-23T23:13:11.279153+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2298112,"kb_used_data":14848,"kb_used_omap":20,"kb_used_meta":2283179,"kb_avail":747663104,"statfs":{"total":767960285184,"available":765607018496,"internally_reserved":0,"allocated":15204352,"data_stored":10398334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2337975950},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002819"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":142,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548788+0000","last_change":"2023-09-23T23:08:45.116934+0000","last_active":"2023-09-23T23:12:03.548788+0000","last_peered":"2023-09-23T23:12:03.548788+0000","last_clean":"2023-09-23T23:12:03.548788+0000","last_became_active":"2023-09-23T23:08:45.116576+0000","last_became_peered":"2023-09-23T23:08:45.116576+0000","last_unstale":"2023-09-23T23:12:03.548788+0000","last_undegraded":"2023-09-23T23:12:03.548788+0000","last_fullsized":"2023-09-23T23:12:03.548788+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:06.398968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484292+0000","last_change":"2023-09-23T23:10:59.514211+0000","last_active":"2023-09-23T23:12:03.484292+0000","last_peered":"2023-09-23T23:12:03.484292+0000","last_clean":"2023-09-23T23:12:03.484292+0000","last_became_active":"2023-09-23T23:10:59.513917+0000","last_became_peered":"2023-09-23T23:10:59.513917+0000","last_unstale":"2023-09-23T23:12:03.484292+0000","last_undegraded":"2023-09-23T23:12:03.484292+0000","last_fullsized":"2023-09-23T23:12:03.484292+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:55.533586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:18.348019+0000","last_change":"2023-09-23T23:10:59.505490+0000","last_active":"2023-09-23T23:12:18.348019+0000","last_peered":"2023-09-23T23:12:18.348019+0000","last_clean":"2023-09-23T23:12:18.348019+0000","last_became_active":"2023-09-23T23:10:59.504874+0000","last_became_peered":"2023-09-23T23:10:59.504874+0000","last_unstale":"2023-09-23T23:12:18.348019+0000","last_undegraded":"2023-09-23T23:12:18.348019+0000","last_fullsized":"2023-09-23T23:12:18.348019+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:15.279668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":112,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:10.432268+0000","last_change":"2023-09-23T23:10:59.506092+0000","last_active":"2023-09-23T23:12:10.432268+0000","last_peered":"2023-09-23T23:12:10.432268+0000","last_clean":"2023-09-23T23:12:10.432268+0000","last_became_active":"2023-09-23T23:10:59.505550+0000","last_became_peered":"2023-09-23T23:10:59.505550+0000","last_unstale":"2023-09-23T23:12:10.432268+0000","last_undegraded":"2023-09-23T23:12:10.432268+0000","last_fullsized":"2023-09-23T23:12:10.432268+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:10.576890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":118,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.483861+0000","last_change":"2023-09-23T23:10:59.515675+0000","last_active":"2023-09-23T23:12:03.483861+0000","last_peered":"2023-09-23T23:12:03.483861+0000","last_clean":"2023-09-23T23:12:03.483861+0000","last_became_active":"2023-09-23T23:10:59.514600+0000","last_became_peered":"2023-09-23T23:10:59.514600+0000","last_unstale":"2023-09-23T23:12:03.483861+0000","last_undegraded":"2023-09-23T23:12:03.483861+0000","last_fullsized":"2023-09-23T23:12:03.483861+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.273467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489315+0000","last_change":"2023-09-23T23:07:30.078179+0000","last_active":"2023-09-23T23:12:03.489315+0000","last_peered":"2023-09-23T23:12:03.489315+0000","last_clean":"2023-09-23T23:12:03.489315+0000","last_became_active":"2023-09-23T23:07:30.077810+0000","last_became_peered":"2023-09-23T23:07:30.077810+0000","last_unstale":"2023-09-23T23:12:03.489315+0000","last_undegraded":"2023-09-23T23:12:03.489315+0000","last_fullsized":"2023-09-23T23:12:03.489315+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:19.218333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.538903+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:12:03.538903+0000","last_peered":"2023-09-23T23:12:03.538903+0000","last_clean":"2023-09-23T23:12:03.538903+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:12:03.538903+0000","last_undegraded":"2023-09-23T23:12:03.538903+0000","last_fullsized":"2023-09-23T23:12:03.538903+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.483884+0000","last_change":"2023-09-23T23:10:59.515840+0000","last_active":"2023-09-23T23:12:03.483884+0000","last_peered":"2023-09-23T23:12:03.483884+0000","last_clean":"2023-09-23T23:12:03.483884+0000","last_became_active":"2023-09-23T23:10:59.515000+0000","last_became_peered":"2023-09-23T23:10:59.515000+0000","last_unstale":"2023-09-23T23:12:03.483884+0000","last_undegraded":"2023-09-23T23:12:03.483884+0000","last_fullsized":"2023-09-23T23:12:03.483884+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:47.355624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540206+0000","last_change":"2023-09-23T23:07:30.076036+0000","last_active":"2023-09-23T23:12:03.540206+0000","last_peered":"2023-09-23T23:12:03.540206+0000","last_clean":"2023-09-23T23:12:03.540206+0000","last_became_active":"2023-09-23T23:07:30.075733+0000","last_became_peered":"2023-09-23T23:07:30.075733+0000","last_unstale":"2023-09-23T23:12:03.540206+0000","last_undegraded":"2023-09-23T23:12:03.540206+0000","last_fullsized":"2023-09-23T23:12:03.540206+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:56.342424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484007+0000","last_change":"2023-09-23T23:10:59.517290+0000","last_active":"2023-09-23T23:12:03.484007+0000","last_peered":"2023-09-23T23:12:03.484007+0000","last_clean":"2023-09-23T23:12:03.484007+0000","last_became_active":"2023-09-23T23:10:59.513367+0000","last_became_peered":"2023-09-23T23:10:59.513367+0000","last_unstale":"2023-09-23T23:12:03.484007+0000","last_undegraded":"2023-09-23T23:12:03.484007+0000","last_fullsized":"2023-09-23T23:12:03.484007+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:04.328676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488519+0000","last_change":"2023-09-23T23:10:59.500837+0000","last_active":"2023-09-23T23:12:03.488519+0000","last_peered":"2023-09-23T23:12:03.488519+0000","last_clean":"2023-09-23T23:12:03.488519+0000","last_became_active":"2023-09-23T23:10:59.500455+0000","last_became_peered":"2023-09-23T23:10:59.500455+0000","last_unstale":"2023-09-23T23:12:03.488519+0000","last_undegraded":"2023-09-23T23:12:03.488519+0000","last_fullsized":"2023-09-23T23:12:03.488519+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:34.801149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":130,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:08.347005+0000","last_change":"2023-09-23T23:08:45.122701+0000","last_active":"2023-09-23T23:12:08.347005+0000","last_peered":"2023-09-23T23:12:08.347005+0000","last_clean":"2023-09-23T23:12:08.347005+0000","last_became_active":"2023-09-23T23:08:45.122332+0000","last_became_peered":"2023-09-23T23:08:45.122332+0000","last_unstale":"2023-09-23T23:12:08.347005+0000","last_undegraded":"2023-09-23T23:12:08.347005+0000","last_fullsized":"2023-09-23T23:12:08.347005+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:05.771906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":129,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.550194+0000","last_change":"2023-09-23T22:53:16.891930+0000","last_active":"2023-09-23T23:12:03.550194+0000","last_peered":"2023-09-23T23:12:03.550194+0000","last_clean":"2023-09-23T23:12:03.550194+0000","last_became_active":"2023-09-23T22:53:16.891658+0000","last_became_peered":"2023-09-23T22:53:16.891658+0000","last_unstale":"2023-09-23T23:12:03.550194+0000","last_undegraded":"2023-09-23T23:12:03.550194+0000","last_fullsized":"2023-09-23T23:12:03.550194+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:19.762775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.549002+0000","last_change":"2023-09-23T23:07:30.082476+0000","last_active":"2023-09-23T23:12:03.549002+0000","last_peered":"2023-09-23T23:12:03.549002+0000","last_clean":"2023-09-23T23:12:03.549002+0000","last_became_active":"2023-09-23T23:07:30.082201+0000","last_became_peered":"2023-09-23T23:07:30.082201+0000","last_unstale":"2023-09-23T23:12:03.549002+0000","last_undegraded":"2023-09-23T23:12:03.549002+0000","last_fullsized":"2023-09-23T23:12:03.549002+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:08.809163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488205+0000","last_change":"2023-09-23T23:08:45.115767+0000","last_active":"2023-09-23T23:12:03.488205+0000","last_peered":"2023-09-23T23:12:03.488205+0000","last_clean":"2023-09-23T23:12:03.488205+0000","last_became_active":"2023-09-23T23:08:45.115422+0000","last_became_peered":"2023-09-23T23:08:45.115422+0000","last_unstale":"2023-09-23T23:12:03.488205+0000","last_undegraded":"2023-09-23T23:12:03.488205+0000","last_fullsized":"2023-09-23T23:12:03.488205+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:53.216816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484131+0000","last_change":"2023-09-23T23:10:59.518106+0000","last_active":"2023-09-23T23:12:03.484131+0000","last_peered":"2023-09-23T23:12:03.484131+0000","last_clean":"2023-09-23T23:12:03.484131+0000","last_became_active":"2023-09-23T23:10:59.517633+0000","last_became_peered":"2023-09-23T23:10:59.517633+0000","last_unstale":"2023-09-23T23:12:03.484131+0000","last_undegraded":"2023-09-23T23:12:03.484131+0000","last_fullsized":"2023-09-23T23:12:03.484131+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:27.245420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.538685+0000","last_change":"2023-09-23T23:10:59.516989+0000","last_active":"2023-09-23T23:12:03.538685+0000","last_peered":"2023-09-23T23:12:03.538685+0000","last_clean":"2023-09-23T23:12:03.538685+0000","last_became_active":"2023-09-23T23:10:59.516507+0000","last_became_peered":"2023-09-23T23:10:59.516507+0000","last_unstale":"2023-09-23T23:12:03.538685+0000","last_undegraded":"2023-09-23T23:12:03.538685+0000","last_fullsized":"2023-09-23T23:12:03.538685+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:40.834591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540360+0000","last_change":"2023-09-23T23:10:59.498539+0000","last_active":"2023-09-23T23:12:03.540360+0000","last_peered":"2023-09-23T23:12:03.540360+0000","last_clean":"2023-09-23T23:12:03.540360+0000","last_became_active":"2023-09-23T23:10:59.498205+0000","last_became_peered":"2023-09-23T23:10:59.498205+0000","last_unstale":"2023-09-23T23:12:03.540360+0000","last_undegraded":"2023-09-23T23:12:03.540360+0000","last_fullsized":"2023-09-23T23:12:03.540360+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:16.307572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":107,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488255+0000","last_change":"2023-09-23T22:53:16.889362+0000","last_active":"2023-09-23T23:12:03.488255+0000","last_peered":"2023-09-23T23:12:03.488255+0000","last_clean":"2023-09-23T23:12:03.488255+0000","last_became_active":"2023-09-23T22:53:16.889081+0000","last_became_peered":"2023-09-23T22:53:16.889081+0000","last_unstale":"2023-09-23T23:12:03.488255+0000","last_undegraded":"2023-09-23T23:12:03.488255+0000","last_fullsized":"2023-09-23T23:12:03.488255+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:11.321151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.538765+0000","last_change":"2023-09-23T23:10:59.506198+0000","last_active":"2023-09-23T23:12:03.538765+0000","last_peered":"2023-09-23T23:12:03.538765+0000","last_clean":"2023-09-23T23:12:03.538765+0000","last_became_active":"2023-09-23T23:10:59.505912+0000","last_became_peered":"2023-09-23T23:10:59.505912+0000","last_unstale":"2023-09-23T23:12:03.538765+0000","last_undegraded":"2023-09-23T23:12:03.538765+0000","last_fullsized":"2023-09-23T23:12:03.538765+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:30.343257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":132,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488922+0000","last_change":"2023-09-23T23:04:11.518188+0000","last_active":"2023-09-23T23:12:03.488922+0000","last_peered":"2023-09-23T23:12:03.488922+0000","last_clean":"2023-09-23T23:12:03.488922+0000","last_became_active":"2023-09-23T23:04:11.517785+0000","last_became_peered":"2023-09-23T23:04:11.517785+0000","last_unstale":"2023-09-23T23:12:03.488922+0000","last_undegraded":"2023-09-23T23:12:03.488922+0000","last_fullsized":"2023-09-23T23:12:03.488922+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:26.382580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484760+0000","last_change":"2023-09-23T23:10:59.525228+0000","last_active":"2023-09-23T23:12:03.484760+0000","last_peered":"2023-09-23T23:12:03.484760+0000","last_clean":"2023-09-23T23:12:03.484760+0000","last_became_active":"2023-09-23T23:10:59.524665+0000","last_became_peered":"2023-09-23T23:10:59.524665+0000","last_unstale":"2023-09-23T23:12:03.484760+0000","last_undegraded":"2023-09-23T23:12:03.484760+0000","last_fullsized":"2023-09-23T23:12:03.484760+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:42.632316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488678+0000","last_change":"2023-09-23T23:04:11.907427+0000","last_active":"2023-09-23T23:12:03.488678+0000","last_peered":"2023-09-23T23:12:03.488678+0000","last_clean":"2023-09-23T23:12:03.488678+0000","last_became_active":"2023-09-23T23:04:11.907016+0000","last_became_peered":"2023-09-23T23:04:11.907016+0000","last_unstale":"2023-09-23T23:12:03.488678+0000","last_undegraded":"2023-09-23T23:12:03.488678+0000","last_fullsized":"2023-09-23T23:12:03.488678+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:20.715977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":106,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.541229+0000","last_change":"2023-09-23T22:53:18.075506+0000","last_active":"2023-09-23T23:12:03.541229+0000","last_peered":"2023-09-23T23:12:03.541229+0000","last_clean":"2023-09-23T23:12:03.541229+0000","last_became_active":"2023-09-23T22:53:18.075206+0000","last_became_peered":"2023-09-23T22:53:18.075206+0000","last_unstale":"2023-09-23T23:12:03.541229+0000","last_undegraded":"2023-09-23T23:12:03.541229+0000","last_fullsized":"2023-09-23T23:12:03.541229+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:32:16.168188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488500+0000","last_change":"2023-09-23T23:07:30.072322+0000","last_active":"2023-09-23T23:12:03.488500+0000","last_peered":"2023-09-23T23:12:03.488500+0000","last_clean":"2023-09-23T23:12:03.488500+0000","last_became_active":"2023-09-23T23:07:30.071941+0000","last_became_peered":"2023-09-23T23:07:30.071941+0000","last_unstale":"2023-09-23T23:12:03.488500+0000","last_undegraded":"2023-09-23T23:12:03.488500+0000","last_fullsized":"2023-09-23T23:12:03.488500+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:38.994624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548304+0000","last_change":"2023-09-23T23:10:59.512748+0000","last_active":"2023-09-23T23:12:03.548304+0000","last_peered":"2023-09-23T23:12:03.548304+0000","last_clean":"2023-09-23T23:12:03.548304+0000","last_became_active":"2023-09-23T23:10:59.512420+0000","last_became_peered":"2023-09-23T23:10:59.512420+0000","last_unstale":"2023-09-23T23:12:03.548304+0000","last_undegraded":"2023-09-23T23:12:03.548304+0000","last_fullsized":"2023-09-23T23:12:03.548304+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:09.269366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":108,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488293+0000","last_change":"2023-09-23T22:53:16.887968+0000","last_active":"2023-09-23T23:12:03.488293+0000","last_peered":"2023-09-23T23:12:03.488293+0000","last_clean":"2023-09-23T23:12:03.488293+0000","last_became_active":"2023-09-23T22:53:16.887692+0000","last_became_peered":"2023-09-23T22:53:16.887692+0000","last_unstale":"2023-09-23T23:12:03.488293+0000","last_undegraded":"2023-09-23T23:12:03.488293+0000","last_fullsized":"2023-09-23T23:12:03.488293+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:24.662748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":133,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489141+0000","last_change":"2023-09-23T23:08:45.116813+0000","last_active":"2023-09-23T23:12:03.489141+0000","last_peered":"2023-09-23T23:12:03.489141+0000","last_clean":"2023-09-23T23:12:03.489141+0000","last_became_active":"2023-09-23T23:08:45.116435+0000","last_became_peered":"2023-09-23T23:08:45.116435+0000","last_unstale":"2023-09-23T23:12:03.489141+0000","last_undegraded":"2023-09-23T23:12:03.489141+0000","last_fullsized":"2023-09-23T23:12:03.489141+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:40.616551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.539969+0000","last_change":"2023-09-23T23:10:59.502426+0000","last_active":"2023-09-23T23:12:03.539969+0000","last_peered":"2023-09-23T23:12:03.539969+0000","last_clean":"2023-09-23T23:12:03.539969+0000","last_became_active":"2023-09-23T23:10:59.502074+0000","last_became_peered":"2023-09-23T23:10:59.502074+0000","last_unstale":"2023-09-23T23:12:03.539969+0000","last_undegraded":"2023-09-23T23:12:03.539969+0000","last_fullsized":"2023-09-23T23:12:03.539969+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:46.029347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":115,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.255458+0000","last_change":"2023-09-23T23:04:11.912493+0000","last_active":"2023-09-23T23:12:20.255458+0000","last_peered":"2023-09-23T23:12:20.255458+0000","last_clean":"2023-09-23T23:12:20.255458+0000","last_became_active":"2023-09-23T23:04:11.906712+0000","last_became_peered":"2023-09-23T23:04:11.906712+0000","last_unstale":"2023-09-23T23:12:20.255458+0000","last_undegraded":"2023-09-23T23:12:20.255458+0000","last_fullsized":"2023-09-23T23:12:20.255458+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:58.932610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.484285+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:12:03.484285+0000","last_peered":"2023-09-23T23:12:03.484285+0000","last_clean":"2023-09-23T23:12:03.484285+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:12:03.484285+0000","last_undegraded":"2023-09-23T23:12:03.484285+0000","last_fullsized":"2023-09-23T23:12:03.484285+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548351+0000","last_change":"2023-09-23T23:10:59.512363+0000","last_active":"2023-09-23T23:12:03.548351+0000","last_peered":"2023-09-23T23:12:03.548351+0000","last_clean":"2023-09-23T23:12:03.548351+0000","last_became_active":"2023-09-23T23:10:59.512022+0000","last_became_peered":"2023-09-23T23:10:59.512022+0000","last_unstale":"2023-09-23T23:12:03.548351+0000","last_undegraded":"2023-09-23T23:12:03.548351+0000","last_fullsized":"2023-09-23T23:12:03.548351+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:45.523432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489017+0000","last_change":"2023-09-23T23:07:30.077349+0000","last_active":"2023-09-23T23:12:03.489017+0000","last_peered":"2023-09-23T23:12:03.489017+0000","last_clean":"2023-09-23T23:12:03.489017+0000","last_became_active":"2023-09-23T23:07:30.077031+0000","last_became_peered":"2023-09-23T23:07:30.077031+0000","last_unstale":"2023-09-23T23:12:03.489017+0000","last_undegraded":"2023-09-23T23:12:03.489017+0000","last_fullsized":"2023-09-23T23:12:03.489017+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:27.708541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548925+0000","last_change":"2023-09-23T23:08:45.117579+0000","last_active":"2023-09-23T23:12:03.548925+0000","last_peered":"2023-09-23T23:12:03.548925+0000","last_clean":"2023-09-23T23:12:03.548925+0000","last_became_active":"2023-09-23T23:08:45.117216+0000","last_became_peered":"2023-09-23T23:08:45.117216+0000","last_unstale":"2023-09-23T23:12:03.548925+0000","last_undegraded":"2023-09-23T23:12:03.548925+0000","last_fullsized":"2023-09-23T23:12:03.548925+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:27.731522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548772+0000","last_change":"2023-09-23T23:08:45.120505+0000","last_active":"2023-09-23T23:12:03.548772+0000","last_peered":"2023-09-23T23:12:03.548772+0000","last_clean":"2023-09-23T23:12:03.548772+0000","last_became_active":"2023-09-23T23:08:45.120136+0000","last_became_peered":"2023-09-23T23:08:45.120136+0000","last_unstale":"2023-09-23T23:12:03.548772+0000","last_undegraded":"2023-09-23T23:12:03.548772+0000","last_fullsized":"2023-09-23T23:12:03.548772+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:56.726515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.549135+0000","last_change":"2023-09-23T23:07:30.081779+0000","last_active":"2023-09-23T23:12:03.549135+0000","last_peered":"2023-09-23T23:12:03.549135+0000","last_clean":"2023-09-23T23:12:03.549135+0000","last_became_active":"2023-09-23T23:07:30.081425+0000","last_became_peered":"2023-09-23T23:07:30.081425+0000","last_unstale":"2023-09-23T23:12:03.549135+0000","last_undegraded":"2023-09-23T23:12:03.549135+0000","last_fullsized":"2023-09-23T23:12:03.549135+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:01.231206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":102,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:19.134430+0000","last_change":"2023-09-23T23:10:59.516007+0000","last_active":"2023-09-23T23:12:19.134430+0000","last_peered":"2023-09-23T23:12:19.134430+0000","last_clean":"2023-09-23T23:12:19.134430+0000","last_became_active":"2023-09-23T23:10:59.515388+0000","last_became_peered":"2023-09-23T23:10:59.515388+0000","last_unstale":"2023-09-23T23:12:19.134430+0000","last_undegraded":"2023-09-23T23:12:19.134430+0000","last_fullsized":"2023-09-23T23:12:19.134430+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:13.171258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":130,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486491+0000","last_change":"2023-09-23T23:10:59.503483+0000","last_active":"2023-09-23T23:12:03.486491+0000","last_peered":"2023-09-23T23:12:03.486491+0000","last_clean":"2023-09-23T23:12:03.486491+0000","last_became_active":"2023-09-23T23:10:59.502851+0000","last_became_peered":"2023-09-23T23:10:59.502851+0000","last_unstale":"2023-09-23T23:12:03.486491+0000","last_undegraded":"2023-09-23T23:12:03.486491+0000","last_fullsized":"2023-09-23T23:12:03.486491+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:20.388467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486449+0000","last_change":"2023-09-23T23:07:30.069993+0000","last_active":"2023-09-23T23:12:03.486449+0000","last_peered":"2023-09-23T23:12:03.486449+0000","last_clean":"2023-09-23T23:12:03.486449+0000","last_became_active":"2023-09-23T23:07:30.069634+0000","last_became_peered":"2023-09-23T23:07:30.069634+0000","last_unstale":"2023-09-23T23:12:03.486449+0000","last_undegraded":"2023-09-23T23:12:03.486449+0000","last_fullsized":"2023-09-23T23:12:03.486449+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:24.608905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":141,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540255+0000","last_change":"2023-09-23T23:10:59.498546+0000","last_active":"2023-09-23T23:12:03.540255+0000","last_peered":"2023-09-23T23:12:03.540255+0000","last_clean":"2023-09-23T23:12:03.540255+0000","last_became_active":"2023-09-23T23:10:59.498231+0000","last_became_peered":"2023-09-23T23:10:59.498231+0000","last_unstale":"2023-09-23T23:12:03.540255+0000","last_undegraded":"2023-09-23T23:12:03.540255+0000","last_fullsized":"2023-09-23T23:12:03.540255+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:37.404273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":115,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486594+0000","last_change":"2023-09-23T23:07:30.082833+0000","last_active":"2023-09-23T23:12:03.486594+0000","last_peered":"2023-09-23T23:12:03.486594+0000","last_clean":"2023-09-23T23:12:03.486594+0000","last_became_active":"2023-09-23T23:07:30.082454+0000","last_became_peered":"2023-09-23T23:07:30.082454+0000","last_unstale":"2023-09-23T23:12:03.486594+0000","last_undegraded":"2023-09-23T23:12:03.486594+0000","last_fullsized":"2023-09-23T23:12:03.486594+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:50.336760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540250+0000","last_change":"2023-09-23T23:07:30.075093+0000","last_active":"2023-09-23T23:12:03.540250+0000","last_peered":"2023-09-23T23:12:03.540250+0000","last_clean":"2023-09-23T23:12:03.540250+0000","last_became_active":"2023-09-23T23:07:30.074729+0000","last_became_peered":"2023-09-23T23:07:30.074729+0000","last_unstale":"2023-09-23T23:12:03.540250+0000","last_undegraded":"2023-09-23T23:12:03.540250+0000","last_fullsized":"2023-09-23T23:12:03.540250+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:34.502057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548916+0000","last_change":"2023-09-23T23:04:11.514703+0000","last_active":"2023-09-23T23:12:03.548916+0000","last_peered":"2023-09-23T23:12:03.548916+0000","last_clean":"2023-09-23T23:12:03.548916+0000","last_became_active":"2023-09-23T23:04:11.514403+0000","last_became_peered":"2023-09-23T23:04:11.514403+0000","last_unstale":"2023-09-23T23:12:03.548916+0000","last_undegraded":"2023-09-23T23:12:03.548916+0000","last_fullsized":"2023-09-23T23:12:03.548916+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:26.374346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":145,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486345+0000","last_change":"2023-09-23T23:10:59.502030+0000","last_active":"2023-09-23T23:12:03.486345+0000","last_peered":"2023-09-23T23:12:03.486345+0000","last_clean":"2023-09-23T23:12:03.486345+0000","last_became_active":"2023-09-23T23:10:59.501714+0000","last_became_peered":"2023-09-23T23:10:59.501714+0000","last_unstale":"2023-09-23T23:12:03.486345+0000","last_undegraded":"2023-09-23T23:12:03.486345+0000","last_fullsized":"2023-09-23T23:12:03.486345+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:52.898368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":119,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.353626+0000","last_change":"2023-09-23T23:10:59.501544+0000","last_active":"2023-09-23T23:12:20.353626+0000","last_peered":"2023-09-23T23:12:20.353626+0000","last_clean":"2023-09-23T23:12:20.353626+0000","last_became_active":"2023-09-23T23:10:59.501247+0000","last_became_peered":"2023-09-23T23:10:59.501247+0000","last_unstale":"2023-09-23T23:12:20.353626+0000","last_undegraded":"2023-09-23T23:12:20.353626+0000","last_fullsized":"2023-09-23T23:12:20.353626+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:54.254755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486076+0000","last_change":"2023-09-23T23:08:45.122712+0000","last_active":"2023-09-23T23:12:03.486076+0000","last_peered":"2023-09-23T23:12:03.486076+0000","last_clean":"2023-09-23T23:12:03.486076+0000","last_became_active":"2023-09-23T23:08:45.122391+0000","last_became_peered":"2023-09-23T23:08:45.122391+0000","last_unstale":"2023-09-23T23:12:03.486076+0000","last_undegraded":"2023-09-23T23:12:03.486076+0000","last_fullsized":"2023-09-23T23:12:03.486076+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:44.916349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":117,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486558+0000","last_change":"2023-09-23T23:07:30.074957+0000","last_active":"2023-09-23T23:12:03.486558+0000","last_peered":"2023-09-23T23:12:03.486558+0000","last_clean":"2023-09-23T23:12:03.486558+0000","last_became_active":"2023-09-23T23:07:30.074475+0000","last_became_peered":"2023-09-23T23:07:30.074475+0000","last_unstale":"2023-09-23T23:12:03.486558+0000","last_undegraded":"2023-09-23T23:12:03.486558+0000","last_fullsized":"2023-09-23T23:12:03.486558+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:25.337999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540551+0000","last_change":"2023-09-23T23:08:45.111935+0000","last_active":"2023-09-23T23:12:03.540551+0000","last_peered":"2023-09-23T23:12:03.540551+0000","last_clean":"2023-09-23T23:12:03.540551+0000","last_became_active":"2023-09-23T23:08:45.111549+0000","last_became_peered":"2023-09-23T23:08:45.111549+0000","last_unstale":"2023-09-23T23:12:03.540551+0000","last_undegraded":"2023-09-23T23:12:03.540551+0000","last_fullsized":"2023-09-23T23:12:03.540551+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:49.879205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":504,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488060+0000","last_change":"2023-09-23T23:04:11.517317+0000","last_active":"2023-09-23T23:12:03.488060+0000","last_peered":"2023-09-23T23:12:03.488060+0000","last_clean":"2023-09-23T23:12:03.488060+0000","last_became_active":"2023-09-23T23:04:11.516929+0000","last_became_peered":"2023-09-23T23:04:11.516929+0000","last_unstale":"2023-09-23T23:12:03.488060+0000","last_undegraded":"2023-09-23T23:12:03.488060+0000","last_fullsized":"2023-09-23T23:12:03.488060+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T22:49:49.548021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:19.871312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":79,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:19.134029+0000","last_change":"2023-09-23T23:10:59.512923+0000","last_active":"2023-09-23T23:12:19.134029+0000","last_peered":"2023-09-23T23:12:19.134029+0000","last_clean":"2023-09-23T23:12:19.134029+0000","last_became_active":"2023-09-23T23:10:59.512637+0000","last_became_peered":"2023-09-23T23:10:59.512637+0000","last_unstale":"2023-09-23T23:12:19.134029+0000","last_undegraded":"2023-09-23T23:12:19.134029+0000","last_fullsized":"2023-09-23T23:12:19.134029+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:43:06.983322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":106,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489400+0000","last_change":"2023-09-23T22:53:18.075922+0000","last_active":"2023-09-23T23:12:03.489400+0000","last_peered":"2023-09-23T23:12:03.489400+0000","last_clean":"2023-09-23T23:12:03.489400+0000","last_became_active":"2023-09-23T22:53:18.075621+0000","last_became_peered":"2023-09-23T22:53:18.075621+0000","last_unstale":"2023-09-23T23:12:03.489400+0000","last_undegraded":"2023-09-23T23:12:03.489400+0000","last_fullsized":"2023-09-23T23:12:03.489400+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:01.490333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":145,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488321+0000","last_change":"2023-09-23T23:07:30.068847+0000","last_active":"2023-09-23T23:12:03.488321+0000","last_peered":"2023-09-23T23:12:03.488321+0000","last_clean":"2023-09-23T23:12:03.488321+0000","last_became_active":"2023-09-23T23:07:30.068533+0000","last_became_peered":"2023-09-23T23:07:30.068533+0000","last_unstale":"2023-09-23T23:12:03.488321+0000","last_undegraded":"2023-09-23T23:12:03.488321+0000","last_fullsized":"2023-09-23T23:12:03.488321+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:59.598863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":112,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.255403+0000","last_change":"2023-09-23T23:10:59.503085+0000","last_active":"2023-09-23T23:12:20.255403+0000","last_peered":"2023-09-23T23:12:20.255403+0000","last_clean":"2023-09-23T23:12:20.255403+0000","last_became_active":"2023-09-23T23:10:59.502737+0000","last_became_peered":"2023-09-23T23:10:59.502737+0000","last_unstale":"2023-09-23T23:12:20.255403+0000","last_undegraded":"2023-09-23T23:12:20.255403+0000","last_fullsized":"2023-09-23T23:12:20.255403+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:22.783324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":127,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.259269+0000","last_change":"2023-09-23T23:08:45.116856+0000","last_active":"2023-09-23T23:12:20.259269+0000","last_peered":"2023-09-23T23:12:20.259269+0000","last_clean":"2023-09-23T23:12:20.259269+0000","last_became_active":"2023-09-23T23:08:45.116519+0000","last_became_peered":"2023-09-23T23:08:45.116519+0000","last_unstale":"2023-09-23T23:12:20.259269+0000","last_undegraded":"2023-09-23T23:12:20.259269+0000","last_fullsized":"2023-09-23T23:12:20.259269+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:25.917776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.488391+0000","last_change":"2023-09-23T23:04:11.900743+0000","last_active":"2023-09-23T23:12:03.488391+0000","last_peered":"2023-09-23T23:12:03.488391+0000","last_clean":"2023-09-23T23:12:03.488391+0000","last_became_active":"2023-09-23T23:04:11.900418+0000","last_became_peered":"2023-09-23T23:04:11.900418+0000","last_unstale":"2023-09-23T23:12:03.488391+0000","last_undegraded":"2023-09-23T23:12:03.488391+0000","last_fullsized":"2023-09-23T23:12:03.488391+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:45:31.893513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486279+0000","last_change":"2023-09-23T23:08:45.124284+0000","last_active":"2023-09-23T23:12:03.486279+0000","last_peered":"2023-09-23T23:12:03.486279+0000","last_clean":"2023-09-23T23:12:03.486279+0000","last_became_active":"2023-09-23T23:08:45.123988+0000","last_became_peered":"2023-09-23T23:08:45.123988+0000","last_unstale":"2023-09-23T23:12:03.486279+0000","last_undegraded":"2023-09-23T23:12:03.486279+0000","last_fullsized":"2023-09-23T23:12:03.486279+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:26.726970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":97,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.255678+0000","last_change":"2023-09-23T23:04:11.911507+0000","last_active":"2023-09-23T23:12:20.255678+0000","last_peered":"2023-09-23T23:12:20.255678+0000","last_clean":"2023-09-23T23:12:20.255678+0000","last_became_active":"2023-09-23T23:04:11.908281+0000","last_became_peered":"2023-09-23T23:04:11.908281+0000","last_unstale":"2023-09-23T23:12:20.255678+0000","last_undegraded":"2023-09-23T23:12:20.255678+0000","last_fullsized":"2023-09-23T23:12:20.255678+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:30.414860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":125,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:19.134893+0000","last_change":"2023-09-23T23:10:59.511903+0000","last_active":"2023-09-23T23:12:19.134893+0000","last_peered":"2023-09-23T23:12:19.134893+0000","last_clean":"2023-09-23T23:12:19.134893+0000","last_became_active":"2023-09-23T23:10:59.511022+0000","last_became_peered":"2023-09-23T23:10:59.511022+0000","last_unstale":"2023-09-23T23:12:19.134893+0000","last_undegraded":"2023-09-23T23:12:19.134893+0000","last_fullsized":"2023-09-23T23:12:19.134893+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:03.498373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":116,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.486326+0000","last_change":"2023-09-23T23:07:30.083011+0000","last_active":"2023-09-23T23:12:03.486326+0000","last_peered":"2023-09-23T23:12:03.486326+0000","last_clean":"2023-09-23T23:12:03.486326+0000","last_became_active":"2023-09-23T23:07:30.082544+0000","last_became_peered":"2023-09-23T23:07:30.082544+0000","last_unstale":"2023-09-23T23:12:03.486326+0000","last_undegraded":"2023-09-23T23:12:03.486326+0000","last_fullsized":"2023-09-23T23:12:03.486326+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:20.624316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":122,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:19.134518+0000","last_change":"2023-09-23T23:10:59.512305+0000","last_active":"2023-09-23T23:12:19.134518+0000","last_peered":"2023-09-23T23:12:19.134518+0000","last_clean":"2023-09-23T23:12:19.134518+0000","last_became_active":"2023-09-23T23:10:59.511941+0000","last_became_peered":"2023-09-23T23:10:59.511941+0000","last_unstale":"2023-09-23T23:12:19.134518+0000","last_undegraded":"2023-09-23T23:12:19.134518+0000","last_fullsized":"2023-09-23T23:12:19.134518+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:06.293763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.489269+0000","last_change":"2023-09-23T23:08:45.116806+0000","last_active":"2023-09-23T23:12:03.489269+0000","last_peered":"2023-09-23T23:12:03.489269+0000","last_clean":"2023-09-23T23:12:03.489269+0000","last_became_active":"2023-09-23T23:08:45.116462+0000","last_became_peered":"2023-09-23T23:08:45.116462+0000","last_unstale":"2023-09-23T23:12:03.489269+0000","last_undegraded":"2023-09-23T23:12:03.489269+0000","last_fullsized":"2023-09-23T23:12:03.489269+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:24.502987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":116,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.259196+0000","last_change":"2023-09-23T23:07:30.072388+0000","last_active":"2023-09-23T23:12:20.259196+0000","last_peered":"2023-09-23T23:12:20.259196+0000","last_clean":"2023-09-23T23:12:20.259196+0000","last_became_active":"2023-09-23T23:07:30.072099+0000","last_became_peered":"2023-09-23T23:07:30.072099+0000","last_unstale":"2023-09-23T23:12:20.259196+0000","last_undegraded":"2023-09-23T23:12:20.259196+0000","last_fullsized":"2023-09-23T23:12:20.259196+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:19.758734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":106,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548614+0000","last_change":"2023-09-23T22:53:17.905706+0000","last_active":"2023-09-23T23:12:03.548614+0000","last_peered":"2023-09-23T23:12:03.548614+0000","last_clean":"2023-09-23T23:12:03.548614+0000","last_became_active":"2023-09-23T22:53:17.905309+0000","last_became_peered":"2023-09-23T22:53:17.905309+0000","last_unstale":"2023-09-23T23:12:03.548614+0000","last_undegraded":"2023-09-23T23:12:03.548614+0000","last_fullsized":"2023-09-23T23:12:03.548614+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:06.105426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.539475+0000","last_change":"2023-09-23T23:04:11.900795+0000","last_active":"2023-09-23T23:12:03.539475+0000","last_peered":"2023-09-23T23:12:03.539475+0000","last_clean":"2023-09-23T23:12:03.539475+0000","last_became_active":"2023-09-23T23:04:11.899751+0000","last_became_peered":"2023-09-23T23:04:11.899751+0000","last_unstale":"2023-09-23T23:12:03.539475+0000","last_undegraded":"2023-09-23T23:12:03.539475+0000","last_fullsized":"2023-09-23T23:12:03.539475+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:41.206493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":134,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.525133+0000","last_change":"2023-09-23T23:10:59.511474+0000","last_active":"2023-09-23T23:12:03.525133+0000","last_peered":"2023-09-23T23:12:03.525133+0000","last_clean":"2023-09-23T23:12:03.525133+0000","last_became_active":"2023-09-23T23:10:59.511175+0000","last_became_peered":"2023-09-23T23:10:59.511175+0000","last_unstale":"2023-09-23T23:12:03.525133+0000","last_undegraded":"2023-09-23T23:12:03.525133+0000","last_fullsized":"2023-09-23T23:12:03.525133+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:41.647976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540448+0000","last_change":"2023-09-23T23:07:30.075311+0000","last_active":"2023-09-23T23:12:03.540448+0000","last_peered":"2023-09-23T23:12:03.540448+0000","last_clean":"2023-09-23T23:12:03.540448+0000","last_became_active":"2023-09-23T23:07:30.074952+0000","last_became_peered":"2023-09-23T23:07:30.074952+0000","last_unstale":"2023-09-23T23:12:03.540448+0000","last_undegraded":"2023-09-23T23:12:03.540448+0000","last_fullsized":"2023-09-23T23:12:03.540448+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:16.724083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548332+0000","last_change":"2023-09-23T23:10:59.512738+0000","last_active":"2023-09-23T23:12:03.548332+0000","last_peered":"2023-09-23T23:12:03.548332+0000","last_clean":"2023-09-23T23:12:03.548332+0000","last_became_active":"2023-09-23T23:10:59.512011+0000","last_became_peered":"2023-09-23T23:10:59.512011+0000","last_unstale":"2023-09-23T23:12:03.548332+0000","last_undegraded":"2023-09-23T23:12:03.548332+0000","last_fullsized":"2023-09-23T23:12:03.548332+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:18.088964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":109,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.540827+0000","last_change":"2023-09-23T22:53:16.891297+0000","last_active":"2023-09-23T23:12:03.540827+0000","last_peered":"2023-09-23T23:12:03.540827+0000","last_clean":"2023-09-23T23:12:03.540827+0000","last_became_active":"2023-09-23T22:53:16.891025+0000","last_became_peered":"2023-09-23T22:53:16.891025+0000","last_unstale":"2023-09-23T23:12:03.540827+0000","last_undegraded":"2023-09-23T23:12:03.540827+0000","last_fullsized":"2023-09-23T23:12:03.540827+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:40:50.608339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548305+0000","last_change":"2023-09-23T23:10:59.512247+0000","last_active":"2023-09-23T23:12:03.548305+0000","last_peered":"2023-09-23T23:12:03.548305+0000","last_clean":"2023-09-23T23:12:03.548305+0000","last_became_active":"2023-09-23T23:10:59.511865+0000","last_became_peered":"2023-09-23T23:10:59.511865+0000","last_unstale":"2023-09-23T23:12:03.548305+0000","last_undegraded":"2023-09-23T23:12:03.548305+0000","last_fullsized":"2023-09-23T23:12:03.548305+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:20.981239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":121,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:19.134146+0000","last_change":"2023-09-23T23:10:59.512958+0000","last_active":"2023-09-23T23:12:19.134146+0000","last_peered":"2023-09-23T23:12:19.134146+0000","last_clean":"2023-09-23T23:12:19.134146+0000","last_became_active":"2023-09-23T23:10:59.512628+0000","last_became_peered":"2023-09-23T23:10:59.512628+0000","last_unstale":"2023-09-23T23:12:19.134146+0000","last_undegraded":"2023-09-23T23:12:19.134146+0000","last_fullsized":"2023-09-23T23:12:19.134146+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:16:26.998696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":135,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.483842+0000","last_change":"2023-09-23T23:10:59.511365+0000","last_active":"2023-09-23T23:12:03.483842+0000","last_peered":"2023-09-23T23:12:03.483842+0000","last_clean":"2023-09-23T23:12:03.483842+0000","last_became_active":"2023-09-23T23:10:59.511073+0000","last_became_peered":"2023-09-23T23:10:59.511073+0000","last_unstale":"2023-09-23T23:12:03.483842+0000","last_undegraded":"2023-09-23T23:12:03.483842+0000","last_fullsized":"2023-09-23T23:12:03.483842+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:35.196455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":116,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T23:12:20.256275+0000","last_change":"2023-09-23T23:08:45.113546+0000","last_active":"2023-09-23T23:12:20.256275+0000","last_peered":"2023-09-23T23:12:20.256275+0000","last_clean":"2023-09-23T23:12:20.256275+0000","last_became_active":"2023-09-23T23:08:45.113191+0000","last_became_peered":"2023-09-23T23:08:45.113191+0000","last_unstale":"2023-09-23T23:12:20.256275+0000","last_undegraded":"2023-09-23T23:12:20.256275+0000","last_fullsized":"2023-09-23T23:12:20.256275+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:06.372282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":133,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T23:12:03.548440+0000","last_change":"2023-09-23T23:08:45.116882+0000","last_active":"2023-09-23T23:12:03.548440+0000","last_peered":"2023-09-23T23:12:03.548440+0000","last_clean":"2023-09-23T23:12:03.548440+0000","last_became_active":"2023-09-23T23:08:45.116543+0000","last_became_peered":"2023-09-23T23:08:45.116543+0000","last_unstale":"2023-09-23T23:12:03.548440+0000","last_undegraded":"2023-09-23T23:12:03.548440+0000","last_fullsized":"2023-09-23T23:12:03.548440+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:08.722088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234245,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":287872,"kb_used_data":2168,"kb_used_omap":0,"kb_used_meta":285696,"kb_avail":93457280,"statfs":{"total":95995035648,"available":95700254720,"internally_reserved":0,"allocated":2220032,"data_stored":1663377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":292552704},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5429999999999999,"5min":2.911,"15min":2.7999999999999998},"min":{"1min":0.23799999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":4.9340000000000002,"5min":92.650999999999996,"15min":230.249},"last":0.69599999999999995},{"interface":"front","average":{"1min":2.863,"5min":3.133,"15min":2.8500000000000001},"min":{"1min":0.28799999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":12.004,"5min":93.171000000000006,"15min":230.261},"last":0.56899999999999995}]},{"osd":1,"last update":"Sat Sep 23 23:12:39 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":4.069,"15min":5.1349999999999998},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.2649999999999997,"5min":93.245999999999995,"15min":93.245999999999995},"last":0.68400000000000005},{"interface":"front","average":{"1min":1.889,"5min":3.7010000000000001,"15min":4.9349999999999996},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.9349999999999996,"5min":92.748000000000005,"15min":92.748000000000005},"last":0.80600000000000005}]},{"osd":2,"last update":"Sat Sep 23 23:12:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":2.6680000000000001,"15min":3.1509999999999998},"min":{"1min":0.435,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":12.042,"5min":12.183,"15min":12.183},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.3360000000000001,"5min":2.0720000000000001,"15min":2.25},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.327,"5min":11.781000000000001,"15min":11.781000000000001},"last":0.66900000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:13:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":2.2090000000000001,"15min":2.379},"min":{"1min":0.40600000000000003,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.2610000000000001,"5min":11.507999999999999,"15min":11.507999999999999},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.90700000000000003,"5min":2.2360000000000002,"15min":2.4580000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":3.569,"5min":8.9670000000000005,"15min":8.9670000000000005},"last":0.83299999999999996}]},{"osd":4,"last update":"Sat Sep 23 23:12:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1269999999999998,"5min":2.3140000000000001,"15min":2.2250000000000001},"min":{"1min":0.28199999999999997,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":7.8529999999999998,"5min":92.781000000000006,"15min":194.797},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.2770000000000001,"5min":2.2909999999999999,"15min":2.1859999999999999},"min":{"1min":0.33500000000000002,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":7.3659999999999997,"5min":92.626999999999995,"15min":190.82599999999999},"last":0.60899999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:12:16 2023","interfaces":[{"interface":"back","average":{"1min":2.9049999999999998,"5min":2.8050000000000002,"15min":1.9490000000000001},"min":{"1min":0.29799999999999999,"5min":0.29299999999999998,"15min":0.26500000000000001},"max":{"1min":12.067,"5min":93.135000000000005,"15min":93.135000000000005},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.111,"5min":2.2450000000000001,"15min":1.742},"min":{"1min":0.35799999999999998,"5min":0.29799999999999999,"15min":0.29399999999999998},"max":{"1min":5.1890000000000001,"5min":92.885000000000005,"15min":92.885000000000005},"last":0.65200000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:12:16 2023","interfaces":[{"interface":"back","average":{"1min":2.282,"5min":2.274,"15min":1.784},"min":{"1min":0.315,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":13.590999999999999,"5min":93.105999999999995,"15min":93.105999999999995},"last":0.76600000000000001},{"interface":"front","average":{"1min":2.6779999999999999,"5min":2.512,"15min":1.585},"min":{"1min":0.33800000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.6240000000000006,"5min":93.198999999999998,"15min":93.198999999999998},"last":0.71599999999999997}]}]},{"osd":6,"up_from":46,"seq":197568495881,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":267184,"kb_used_data":2152,"kb_used_omap":0,"kb_used_meta":265024,"kb_avail":93477968,"statfs":{"total":95995035648,"available":95721439232,"internally_reserved":0,"allocated":2203648,"data_stored":1662966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":271384576},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6439999999999999,"5min":1.706,"15min":4.2039999999999997},"min":{"1min":0.246,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":5.6310000000000002,"5min":11.004,"15min":97.858000000000004},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.208,"5min":1.6830000000000001,"15min":4.8630000000000004},"min":{"1min":0.27100000000000002,"5min":0.24399999999999999,"15min":0.22700000000000001},"max":{"1min":4.367,"5min":8.3059999999999992,"15min":99.793000000000006},"last":0.53700000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:12:34 2023","interfaces":[{"interface":"back","average":{"1min":1.601,"5min":4.2450000000000001,"15min":2.02},"min":{"1min":0.28599999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.6779999999999999,"5min":209.00899999999999,"15min":209.00899999999999},"last":0.72799999999999998},{"interface":"front","average":{"1min":1.417,"5min":4.4219999999999997,"15min":2.367},"min":{"1min":0.30599999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.173,"5min":209.03,"15min":209.03},"last":0.434}]},{"osd":2,"last update":"Sat Sep 23 23:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3420000000000001,"5min":1.827,"15min":1.6180000000000001},"min":{"1min":0.38600000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.657,"5min":10.871,"15min":10.871},"last":0.77600000000000002},{"interface":"front","average":{"1min":1.5609999999999999,"5min":2.242,"15min":1.9670000000000001},"min":{"1min":0.40999999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.3339999999999996,"5min":12.148,"15min":12.148},"last":0.69599999999999995}]},{"osd":3,"last update":"Sat Sep 23 23:13:02 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":2.2909999999999999,"15min":2.5529999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.3529999999999998,"5min":7.4059999999999997,"15min":7.4059999999999997},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.0649999999999999,"5min":2.718,"15min":2.9940000000000002},"min":{"1min":0.432,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":5.3250000000000002,"5min":9.4550000000000001,"15min":9.4550000000000001},"last":0.751}]},{"osd":4,"last update":"Sat Sep 23 23:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":1.4530000000000001,"15min":1.262},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.8879999999999999,"5min":11.890000000000001,"15min":12.919},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.997,"5min":1.6919999999999999,"15min":1.4039999999999999},"min":{"1min":0.312,"5min":0.308,"15min":0.24299999999999999},"max":{"1min":6.3079999999999998,"5min":11.728,"15min":11.728},"last":0.61099999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.804,"15min":1.5289999999999999},"min":{"1min":0.41799999999999998,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":3.9289999999999998,"5min":26.158999999999999,"15min":26.158999999999999},"last":0.63100000000000001},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.623,"15min":1.4359999999999999},"min":{"1min":0.45700000000000002,"5min":0.30599999999999999,"15min":0.30199999999999999},"max":{"1min":3.5179999999999998,"5min":25.786000000000001,"15min":25.786000000000001},"last":0.67900000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:12:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1099999999999999,"5min":1.589,"15min":1.716},"min":{"1min":0.36399999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.7149999999999999,"5min":10.968999999999999,"15min":12.676},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.9079999999999999,"5min":1.3799999999999999,"15min":1.4350000000000001},"min":{"1min":0.33700000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.5919999999999996,"5min":7.9669999999999996,"15min":9.7889999999999997},"last":0.80200000000000005}]}]},{"osd":1,"up_from":112,"seq":481036337222,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":239752,"kb_used_data":1728,"kb_used_omap":4,"kb_used_meta":238011,"kb_avail":93505400,"statfs":{"total":95995035648,"available":95749529600,"internally_reserved":0,"allocated":1769472,"data_stored":1089061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":243724174},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":2.012,"15min":1.6759999999999999},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":5.8730000000000002,"5min":10.221,"15min":10.221},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.718,"15min":1.486},"min":{"1min":0.255,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":4.2549999999999999,"5min":12.212,"15min":12.212},"last":0.29699999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:12:47 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":2.3410000000000002,"15min":2.4079999999999999},"min":{"1min":0.22,"5min":0.192,"15min":0.192},"max":{"1min":7.4939999999999998,"5min":16.696999999999999,"15min":16.696999999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":2.1960000000000002,"15min":2.3069999999999999},"min":{"1min":0.23300000000000001,"5min":0.215,"15min":0.215},"max":{"1min":2.448,"5min":9.0879999999999992,"15min":9.0879999999999992},"last":0.38600000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:13:06 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":2.2850000000000001,"15min":2.5630000000000002},"min":{"1min":0.27600000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.8769999999999998,"5min":12.234999999999999,"15min":12.234999999999999},"last":0.37},{"interface":"front","average":{"1min":0.58299999999999996,"5min":1.9490000000000001,"15min":2.177},"min":{"1min":0.29399999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.9359999999999999,"5min":12.256,"15min":12.256},"last":0.40799999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":1.841,"15min":1.552},"min":{"1min":0.372,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.0720000000000001,"5min":12.676,"15min":12.676},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.403,"5min":1.873,"15min":1.4810000000000001},"min":{"1min":0.45200000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.8710000000000004,"5min":11.125,"15min":11.125},"last":0.622}]},{"osd":5,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":2.5619999999999998,"15min":3.9620000000000002},"min":{"1min":0.34999999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.6749999999999998,"5min":67.497,"15min":67.497},"last":0.70699999999999996},{"interface":"front","average":{"1min":1.827,"5min":2.7170000000000001,"15min":4.016},"min":{"1min":0.34599999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.4660000000000002,"5min":67.540999999999997,"15min":67.540999999999997},"last":0.63500000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.6739999999999999,"15min":1.5780000000000001},"min":{"1min":0.40799999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.6429999999999998,"5min":12.186,"15min":12.186},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.9239999999999999,"5min":2.2730000000000001,"15min":1.679},"min":{"1min":0.39700000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.4029999999999996,"5min":12.785,"15min":12.785},"last":0.66200000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":2.0830000000000002,"5min":2.2549999999999999,"15min":1.7150000000000001},"min":{"1min":0.46300000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.4359999999999999,"5min":12.727,"15min":12.727},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.3240000000000001,"5min":1.9099999999999999,"15min":1.5960000000000001},"min":{"1min":0.40300000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":5.593,"5min":9.8559999999999999,"15min":9.8559999999999999},"last":0.65400000000000003}]}]},{"osd":0,"up_from":91,"seq":390842024046,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":349732,"kb_used_data":2332,"kb_used_omap":5,"kb_used_meta":347386,"kb_avail":93395420,"statfs":{"total":95995035648,"available":95636910080,"internally_reserved":0,"allocated":2387968,"data_stored":1680718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":355723898},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:12:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":2.1930000000000001,"15min":2.452},"min":{"1min":0.19700000000000001,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":11.029,"5min":21.629999999999999,"15min":21.629999999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.4510000000000001,"5min":2.25,"15min":2.2759999999999998},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":9.702,"5min":13.382,"15min":13.382},"last":0.29199999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:12:54 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":2.1560000000000001,"15min":2.1179999999999999},"min":{"1min":0.22700000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":6.0010000000000003,"5min":10.199,"15min":10.199},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":2.0129999999999999,"15min":2.0030000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.7679999999999998,"5min":9.0239999999999991,"15min":9.0239999999999991},"last":0.35699999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:12:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":2.895,"15min":3.2709999999999999},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.0460000000000003,"5min":11.866,"15min":11.866},"last":0.371},{"interface":"front","average":{"1min":0.47999999999999998,"5min":2.0990000000000002,"15min":2.3690000000000002},"min":{"1min":0.25,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.351,"5min":9.0980000000000008,"15min":9.0980000000000008},"last":0.32800000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":2.4289999999999998,"15min":2.2389999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.245},"max":{"1min":8.8469999999999995,"5min":28.376999999999999,"15min":28.376999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.6899999999999999,"5min":2.2480000000000002,"15min":2.5910000000000002},"min":{"1min":0.32700000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.1029999999999998,"5min":30.706,"15min":30.706},"last":0.439}]},{"osd":5,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":2.2519999999999998,"15min":2.6110000000000002},"min":{"1min":0.38700000000000001,"5min":0.33700000000000002,"15min":0.30499999999999999},"max":{"1min":4.3419999999999996,"5min":15.819000000000001,"15min":30.873999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.9330000000000001,"5min":2.0699999999999998,"15min":1.956},"min":{"1min":0.47199999999999998,"5min":0.30299999999999999,"15min":0.29499999999999998},"max":{"1min":6.0709999999999997,"5min":10.117000000000001,"15min":10.117000000000001},"last":0.43099999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.0270000000000001,"5min":2.3090000000000002,"15min":2.6360000000000001},"min":{"1min":0.35899999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":10.430999999999999,"5min":15.859,"15min":21.047999999999998},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.3130000000000002,"5min":2.3140000000000001,"15min":2.036},"min":{"1min":0.39700000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":10.414,"5min":10.414,"15min":10.414},"last":0.48199999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.339,"5min":2.3809999999999998,"15min":2.0600000000000001},"min":{"1min":0.45900000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.1899999999999995,"5min":10.145,"15min":10.145},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.9650000000000001,"5min":2.2469999999999999,"15min":2.2069999999999999},"min":{"1min":0.34999999999999998,"5min":0.313,"15min":0.313},"max":{"1min":9.3059999999999992,"5min":10.513999999999999,"15min":10.513999999999999},"last":0.52700000000000002}]}]},{"osd":2,"up_from":130,"seq":558345748535,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":277716,"kb_used_data":1676,"kb_used_omap":7,"kb_used_meta":276024,"kb_avail":93467436,"statfs":{"total":95995035648,"available":95710654464,"internally_reserved":0,"allocated":1716224,"data_stored":1082695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":282648657},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":2.0609999999999999,"15min":2.036},"min":{"1min":0.185,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":8.9209999999999994,"5min":16.021000000000001,"15min":16.021000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.96999999999999997,"5min":2.5609999999999999,"15min":2.9990000000000001},"min":{"1min":0.23799999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.9649999999999999,"5min":16.152999999999999,"15min":16.152999999999999},"last":0.38200000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":2.3210000000000002,"15min":2.6389999999999998},"min":{"1min":0.23200000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":5.0810000000000004,"5min":9.3719999999999999,"15min":9.3719999999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.84699999999999998,"5min":2.141,"15min":2.649},"min":{"1min":0.25800000000000001,"5min":0.218,"15min":0.218},"max":{"1min":5.7359999999999998,"5min":11.625,"15min":11.625},"last":0.372}]},{"osd":3,"last update":"Sat Sep 23 23:13:03 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":1.149,"15min":1.2290000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":5.6639999999999997,"5min":6.8230000000000004,"15min":6.8230000000000004},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.7769999999999999,"15min":1.8899999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.7110000000000003,"5min":7.7110000000000003,"15min":7.7110000000000003},"last":0.33300000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":1.9990000000000001,"15min":2.0840000000000001},"min":{"1min":0.41099999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.1139999999999999,"5min":10.666,"15min":10.666},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.337,"5min":1.5940000000000001,"15min":1.508},"min":{"1min":0.30399999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.8149999999999995,"5min":14.829000000000001,"15min":14.829000000000001},"last":0.56799999999999995}]},{"osd":5,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":2.427,"15min":2.5390000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.9359999999999999,"5min":12.173,"15min":12.173},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.004,"5min":1.619,"15min":1.6519999999999999},"min":{"1min":0.38200000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.7149999999999999,"5min":11.548999999999999,"15min":11.548999999999999},"last":0.49199999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.919,"15min":1.891},"min":{"1min":0.33800000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":8.8620000000000001,"5min":15.205,"15min":15.205},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.284,"5min":2.411,"15min":2.5800000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.9059999999999997,"5min":11.923999999999999,"15min":11.923999999999999},"last":0.621}]},{"osd":7,"last update":"Sat Sep 23 23:12:53 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":2.2890000000000001,"15min":2.3250000000000002},"min":{"1min":0.36299999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.8940000000000001,"5min":14.792999999999999,"15min":14.792999999999999},"last":0.504},{"interface":"front","average":{"1min":1.446,"5min":2.7799999999999998,"15min":2.9460000000000002},"min":{"1min":0.42799999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.9880000000000004,"5min":15.045,"15min":15.045},"last":0.59699999999999998}]}]},{"osd":3,"up_from":148,"seq":635655159836,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":206060,"kb_used_data":1636,"kb_used_omap":2,"kb_used_meta":204413,"kb_avail":93539092,"statfs":{"total":95995035648,"available":95784030208,"internally_reserved":0,"allocated":1675264,"data_stored":1077514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":209318965},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.1970000000000001,"15min":1.3260000000000001},"min":{"1min":0.221,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":2.2949999999999999,"5min":5.9660000000000002,"15min":5.9660000000000002},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.32100000000000001,"5min":1.8129999999999999,"15min":2.0609999999999999},"min":{"1min":0.221,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.58499999999999996,"5min":9.5839999999999996,"15min":9.5839999999999996},"last":0.51700000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.8700000000000001,"15min":2.0870000000000002},"min":{"1min":0.245,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":2.9740000000000002,"5min":7.1219999999999999,"15min":7.1219999999999999},"last":0.505},{"interface":"front","average":{"1min":0.371,"5min":1.635,"15min":1.845},"min":{"1min":0.246,"5min":0.185,"15min":0.185},"max":{"1min":0.71199999999999997,"5min":6.4160000000000004,"15min":6.4160000000000004},"last":0.54300000000000004}]},{"osd":2,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":2.0859999999999999,"15min":2.335},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.8399999999999999,"5min":7.9930000000000003,"15min":7.9930000000000003},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.39200000000000002,"5min":2.2810000000000001,"15min":2.5960000000000001},"min":{"1min":0.28799999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.72999999999999998,"5min":12.721,"15min":12.721},"last":0.57499999999999996}]},{"osd":4,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":1.544,"15min":1.651},"min":{"1min":0.38600000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.1040000000000001,"5min":7.1040000000000001,"15min":7.1040000000000001},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.90300000000000002,"5min":2.8090000000000002,"15min":3.1269999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.7850000000000001,"5min":13.532999999999999,"15min":13.532999999999999},"last":0.95399999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":2.3700000000000001,"15min":2.665},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":1.1279999999999999,"5min":10.117000000000001,"15min":10.117000000000001},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.81799999999999995,"5min":1.9119999999999999,"15min":2.0939999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":2.8690000000000002,"5min":7.3250000000000002,"15min":7.3250000000000002},"last":0.94099999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":2.4870000000000001,"15min":2.7959999999999998},"min":{"1min":0.45300000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.208,"5min":14.492000000000001,"15min":14.492000000000001},"last":0.91400000000000003},{"interface":"front","average":{"1min":1.173,"5min":2.1059999999999999,"15min":2.262},"min":{"1min":0.42399999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":7.6139999999999999,"5min":7.9640000000000004,"15min":7.9640000000000004},"last":0.753}]},{"osd":7,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":2.1709999999999998,"15min":2.4249999999999998},"min":{"1min":0.44900000000000001,"5min":0.377,"15min":0.377},"max":{"1min":1.288,"5min":9.7059999999999995,"15min":9.7059999999999995},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.67100000000000004,"5min":2.298,"15min":2.569},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.379,"5min":9.8010000000000002,"15min":9.8010000000000002},"last":0.97399999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019154,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312124,"kb_used_data":1588,"kb_used_omap":0,"kb_used_meta":310528,"kb_avail":93433028,"statfs":{"total":95995035648,"available":95675420672,"internally_reserved":0,"allocated":1626112,"data_stored":1073262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":317980672},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6639999999999999,"5min":3.8860000000000001,"15min":2.524},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":10.090999999999999,"5min":238.62200000000001,"15min":238.62200000000001},"last":0.65000000000000002},{"interface":"front","average":{"1min":3.04,"5min":4.8819999999999997,"15min":3.0569999999999999},"min":{"1min":0.42699999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":11.519,"5min":238.58199999999999,"15min":238.58199999999999},"last":0.72999999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:12:37 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":4.665,"15min":2.3730000000000002},"min":{"1min":0.30499999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":10.132999999999999,"5min":175.00899999999999,"15min":175.00899999999999},"last":0.67800000000000005},{"interface":"front","average":{"1min":1.5760000000000001,"5min":4.9370000000000003,"15min":2.3450000000000002},"min":{"1min":0.35899999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.452,"5min":174.821,"15min":174.821},"last":0.54700000000000004}]},{"osd":2,"last update":"Sat Sep 23 23:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.762,"15min":1.9119999999999999},"min":{"1min":0.29999999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.664999999999999,"5min":11.664999999999999,"15min":11.664999999999999},"last":0.71799999999999997},{"interface":"front","average":{"1min":1.4239999999999999,"5min":2.3740000000000001,"15min":2.7280000000000002},"min":{"1min":0.443,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":8.3879999999999999,"5min":10.576000000000001,"15min":10.576000000000001},"last":0.63100000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:13:02 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":1.5760000000000001,"15min":1.7430000000000001},"min":{"1min":0.36899999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.498,"5min":8.1609999999999996,"15min":8.1609999999999996},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.90900000000000003,"5min":2.0699999999999998,"15min":2.2639999999999998},"min":{"1min":0.45500000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.0309999999999997,"5min":11.455,"15min":11.455},"last":0.69199999999999995}]},{"osd":5,"last update":"Sat Sep 23 23:12:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":1.1259999999999999,"15min":1.1919999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.23999999999999999},"max":{"1min":10.16,"5min":10.16,"15min":27.391999999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.512,"15min":1.6140000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":8.2680000000000007,"5min":8.2680000000000007,"15min":27.222999999999999},"last":0.46300000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:13:04 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.2130000000000001,"15min":1.1419999999999999},"min":{"1min":0.372,"5min":0.27200000000000002,"15min":0.223},"max":{"1min":0.92000000000000004,"5min":9.3350000000000009,"15min":14.4},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.70499999999999996,"5min":1.2949999999999999,"15min":1.2689999999999999},"min":{"1min":0.33700000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"max":{"1min":3.3170000000000002,"5min":13.888999999999999,"15min":15.641999999999999},"last":0.43099999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:12:25 2023","interfaces":[{"interface":"back","average":{"1min":2.1589999999999998,"5min":1.5489999999999999,"15min":1.849},"min":{"1min":0.318,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":13.792,"5min":13.792,"15min":119.188},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.6919999999999999,"5min":1.532,"15min":1.847},"min":{"1min":0.35499999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.5960000000000001,"5min":13.055,"15min":119.248},"last":0.61199999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757517,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":357672,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":356096,"kb_avail":93387480,"statfs":{"total":95995035648,"available":95628779520,"internally_reserved":0,"allocated":1605632,"data_stored":1068741,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":364642304},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:27 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":1.431,"15min":6.6580000000000004},"min":{"1min":0.32100000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.5090000000000003,"5min":10.227,"15min":190.483},"last":0.72199999999999998},{"interface":"front","average":{"1min":1.9870000000000001,"5min":2.0670000000000002,"15min":7.4690000000000003},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.9359999999999999,"5min":12.762,"15min":194.11099999999999},"last":0.69299999999999995}]},{"osd":1,"last update":"Sat Sep 23 23:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":2.8759999999999999,"15min":1.669},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":7.6870000000000003,"5min":91.295000000000002,"15min":91.295000000000002},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.28,"5min":3.1909999999999998,"15min":1.8009999999999999},"min":{"1min":0.26600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.7110000000000003,"5min":91.242000000000004,"15min":91.242000000000004},"last":0.70799999999999996}]},{"osd":2,"last update":"Sat Sep 23 23:12:51 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":2.5049999999999999,"15min":2.798},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.734,"5min":9.8279999999999994,"15min":9.8279999999999994},"last":0.68200000000000005},{"interface":"front","average":{"1min":1.2150000000000001,"5min":1.702,"15min":1.958},"min":{"1min":0.40300000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.2119999999999997,"5min":8.093,"15min":8.093},"last":0.75600000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:12:58 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":1.6499999999999999,"15min":1.79},"min":{"1min":0.44700000000000001,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":2.633,"5min":6.2640000000000002,"15min":6.2640000000000002},"last":0.78600000000000003},{"interface":"front","average":{"1min":1.002,"5min":2.6880000000000002,"15min":2.9689999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.9420000000000002,"5min":12.802,"15min":12.802},"last":0.74099999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:12:51 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.726,"15min":1.3560000000000001},"min":{"1min":0.38300000000000001,"5min":0.27500000000000002,"15min":0.221},"max":{"1min":7.6459999999999999,"5min":14.538,"15min":14.538},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.92100000000000004,"5min":1.089,"15min":1.153},"min":{"1min":0.315,"5min":0.28100000000000003,"15min":0.25},"max":{"1min":3.8919999999999999,"5min":11.779,"15min":18.827999999999999},"last":0.58099999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:12:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.5289999999999999,"15min":1.3959999999999999},"min":{"1min":0.48699999999999999,"5min":0.315,"15min":0.255},"max":{"1min":3.992,"5min":13.226000000000001,"15min":13.226000000000001},"last":0.67000000000000004},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.833,"15min":1.383},"min":{"1min":0.435,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.1760000000000002,"5min":13.098000000000001,"15min":13.098000000000001},"last":0.63800000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:12:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1829999999999998,"5min":1.8380000000000001,"15min":1.7689999999999999},"min":{"1min":0.38800000000000001,"5min":0.33900000000000002,"15min":0.29199999999999998},"max":{"1min":7.9390000000000001,"5min":14.929,"15min":14.929},"last":0.76900000000000002},{"interface":"front","average":{"1min":1.7430000000000001,"5min":1.722,"15min":1.6180000000000001},"min":{"1min":0.40500000000000003,"5min":0.33400000000000002,"15min":0.29599999999999999},"max":{"1min":10.227,"5min":15.090999999999999,"15min":18.218},"last":0.65700000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:13:12.764 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T23:13:12.764 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd dump 2023-09-23T23:13:13.113 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:13.667 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:13 smithi110 ceph-mon[212939]: from='client.45107 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:13:13.668 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:13:13.668 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:13 smithi110 ceph-mon[221102]: from='client.45107 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:13:13.668 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:13:13.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:13 smithi138 ceph-mon[193606]: from='client.45107 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:13:13.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:13:14.435 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:14 smithi110 ceph-mon[212939]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:14.435 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:14 smithi110 ceph-mon[221102]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:14.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:14 smithi138 ceph-mon[193606]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:14.828 INFO:teuthology.orchestra.run.smithi110.stdout:epoch 163 2023-09-23T23:13:14.828 INFO:teuthology.orchestra.run.smithi110.stdout:fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 2023-09-23T23:13:14.828 INFO:teuthology.orchestra.run.smithi110.stdout:created 2023-09-23T22:46:49.032269+0000 2023-09-23T23:13:14.828 INFO:teuthology.orchestra.run.smithi110.stdout:modified 2023-09-23T23:12:19.130604+0000 2023-09-23T23:13:14.828 INFO:teuthology.orchestra.run.smithi110.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-09-23T23:13:14.828 INFO:teuthology.orchestra.run.smithi110.stdout:crush_version 22 2023-09-23T23:13:14.828 INFO:teuthology.orchestra.run.smithi110.stdout:full_ratio 0.95 2023-09-23T23:13:14.828 INFO:teuthology.orchestra.run.smithi110.stdout:backfillfull_ratio 0.9 2023-09-23T23:13:14.829 INFO:teuthology.orchestra.run.smithi110.stdout:nearfull_ratio 0.85 2023-09-23T23:13:14.829 INFO:teuthology.orchestra.run.smithi110.stdout:require_min_compat_client luminous 2023-09-23T23:13:14.829 INFO:teuthology.orchestra.run.smithi110.stdout:min_compat_client luminous 2023-09-23T23:13:14.829 INFO:teuthology.orchestra.run.smithi110.stdout:require_osd_release quincy 2023-09-23T23:13:14.829 INFO:teuthology.orchestra.run.smithi110.stdout:stretch_mode_enabled false 2023-09-23T23:13:14.829 INFO:teuthology.orchestra.run.smithi110.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2023-09-23T23:13:14.829 INFO:teuthology.orchestra.run.smithi110.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 63 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2023-09-23T23:13:14.829 INFO:teuthology.orchestra.run.smithi110.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 67 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-09-23T23:13:14.829 INFO:teuthology.orchestra.run.smithi110.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 68 flags hashpspool stripe_width 0 application cephfs read_balance_score 2.25 2023-09-23T23:13:14.829 INFO:teuthology.orchestra.run.smithi110.stdout:max_osd 8 2023-09-23T23:13:14.830 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 up in weight 1 up_from 91 up_thru 153 down_at 88 last_clean_interval [8,87) [v2:172.21.15.110:6802/983471373,v1:172.21.15.110:6803/983471373] [v2:172.21.15.110:6804/983471373,v1:172.21.15.110:6805/983471373] exists,up 78791acb-c93c-4e4f-bf04-ea3eee46408d 2023-09-23T23:13:14.830 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 up in weight 1 up_from 112 up_thru 161 down_at 108 last_clean_interval [13,107) [v2:172.21.15.110:6810/594715751,v1:172.21.15.110:6811/594715751] [v2:172.21.15.110:6812/594715751,v1:172.21.15.110:6813/594715751] exists,up 6793178f-2ef1-4c48-927c-831a083a3356 2023-09-23T23:13:14.830 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2 up in weight 1 up_from 130 up_thru 148 down_at 127 last_clean_interval [18,126) [v2:172.21.15.110:6818/3154710012,v1:172.21.15.110:6819/3154710012] [v2:172.21.15.110:6820/3154710012,v1:172.21.15.110:6821/3154710012] exists,up 146fbd28-a365-4545-8e97-d629c6ddba21 2023-09-23T23:13:14.830 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 up in weight 1 up_from 148 up_thru 159 down_at 145 last_clean_interval [25,144) [v2:172.21.15.110:6826/2333701749,v1:172.21.15.110:6827/2333701749] [v2:172.21.15.110:6828/2333701749,v1:172.21.15.110:6829/2333701749] exists,up 6fcf80d5-6977-4929-b7ba-0e9ab65df0a8 2023-09-23T23:13:14.830 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 up in weight 1 up_from 30 up_thru 148 down_at 0 last_clean_interval [0,0) [v2:172.21.15.138:6800/941787557,v1:172.21.15.138:6801/941787557] [v2:172.21.15.138:6802/941787557,v1:172.21.15.138:6803/941787557] exists,up 131d0749-c46c-44f0-ab7f-03f8ad6ed502 2023-09-23T23:13:14.830 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 up in weight 1 up_from 38 up_thru 155 down_at 0 last_clean_interval [0,0) [v2:172.21.15.138:6808/3098195887,v1:172.21.15.138:6809/3098195887] [v2:172.21.15.138:6810/3098195887,v1:172.21.15.138:6811/3098195887] exists,up 8cb496e5-d242-4e41-b279-e8d5ecea2996 2023-09-23T23:13:14.830 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 up in weight 1 up_from 46 up_thru 145 down_at 0 last_clean_interval [0,0) [v2:172.21.15.138:6816/187413403,v1:172.21.15.138:6817/187413403] [v2:172.21.15.138:6818/187413403,v1:172.21.15.138:6819/187413403] exists,up c709e629-85ab-4273-9ca8-f2c4e8fc9074 2023-09-23T23:13:14.830 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 up in weight 1 up_from 54 up_thru 148 down_at 0 last_clean_interval [0,0) [v2:172.21.15.138:6824/676244673,v1:172.21.15.138:6825/676244673] [v2:172.21.15.138:6826/676244673,v1:172.21.15.138:6827/676244673] exists,up ed15f3b0-cd2e-43b7-b803-e45252b62a45 2023-09-23T23:13:14.830 INFO:teuthology.orchestra.run.smithi110.stdout:pg_upmap_items 4.a [1,2] 2023-09-23T23:13:14.831 INFO:teuthology.orchestra.run.smithi110.stdout:pg_upmap_items 4.b [1,5] 2023-09-23T23:13:14.831 INFO:teuthology.orchestra.run.smithi110.stdout:pg_upmap_items 4.18 [1,2] 2023-09-23T23:13:14.831 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/3250066431 expires 2023-09-24T22:47:15.528545+0000 2023-09-23T23:13:14.831 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/3705879030 expires 2023-09-24T22:58:42.842267+0000 2023-09-23T23:13:14.831 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/1399491671 expires 2023-09-24T22:56:16.309980+0000 2023-09-23T23:13:14.831 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/1649304640 expires 2023-09-24T22:47:38.935467+0000 2023-09-23T23:13:14.831 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.138:0/2647044257 expires 2023-09-24T22:57:33.926232+0000 2023-09-23T23:13:14.831 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/1326657364 expires 2023-09-24T22:47:38.935467+0000 2023-09-23T23:13:14.831 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/1731974664 expires 2023-09-24T22:47:15.528545+0000 2023-09-23T23:13:14.832 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/1506915348 expires 2023-09-24T22:47:15.528545+0000 2023-09-23T23:13:14.832 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.138:6834/867141825 expires 2023-09-24T22:57:33.926232+0000 2023-09-23T23:13:14.832 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:6800/1771333922 expires 2023-09-24T22:58:42.842267+0000 2023-09-23T23:13:14.832 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:6801/4228126008 expires 2023-09-24T22:47:38.935467+0000 2023-09-23T23:13:14.832 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/478775577 expires 2023-09-24T22:58:42.842267+0000 2023-09-23T23:13:14.832 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/380987145 expires 2023-09-24T22:56:16.309980+0000 2023-09-23T23:13:14.832 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:6800/4228126008 expires 2023-09-24T22:47:38.935467+0000 2023-09-23T23:13:14.832 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:6800/4112551188 expires 2023-09-24T22:47:15.528545+0000 2023-09-23T23:13:14.832 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.138:0/1613829186 expires 2023-09-24T22:57:33.926232+0000 2023-09-23T23:13:14.832 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/3889938710 expires 2023-09-24T22:58:42.842267+0000 2023-09-23T23:13:14.832 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:6801/4112551188 expires 2023-09-24T22:47:15.528545+0000 2023-09-23T23:13:14.833 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/657635177 expires 2023-09-24T22:52:17.775108+0000 2023-09-23T23:13:14.833 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.138:0/331679217 expires 2023-09-24T22:57:33.926232+0000 2023-09-23T23:13:14.833 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/410644117 expires 2023-09-24T22:52:17.775108+0000 2023-09-23T23:13:14.833 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:6801/2914393040 expires 2023-09-24T22:52:17.775108+0000 2023-09-23T23:13:14.833 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/3674528218 expires 2023-09-24T22:52:17.775108+0000 2023-09-23T23:13:14.833 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:6801/576619896 expires 2023-09-24T22:56:16.309980+0000 2023-09-23T23:13:14.833 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/2850586742 expires 2023-09-24T22:56:16.309980+0000 2023-09-23T23:13:14.833 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/2523943302 expires 2023-09-24T22:52:17.775108+0000 2023-09-23T23:13:14.833 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:6801/1771333922 expires 2023-09-24T22:58:42.842267+0000 2023-09-23T23:13:14.833 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/653682064 expires 2023-09-24T22:58:42.842267+0000 2023-09-23T23:13:14.834 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.138:0/3577646480 expires 2023-09-24T22:57:33.926232+0000 2023-09-23T23:13:14.834 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/3669919402 expires 2023-09-24T22:58:42.842267+0000 2023-09-23T23:13:14.834 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:6800/576619896 expires 2023-09-24T22:56:16.309980+0000 2023-09-23T23:13:14.834 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.138:0/3320282773 expires 2023-09-24T22:57:33.926232+0000 2023-09-23T23:13:14.834 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/3464832160 expires 2023-09-24T22:56:16.309980+0000 2023-09-23T23:13:14.834 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.138:6835/867141825 expires 2023-09-24T22:57:33.926232+0000 2023-09-23T23:13:14.834 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:6800/2914393040 expires 2023-09-24T22:52:17.775108+0000 2023-09-23T23:13:14.834 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/3412430555 expires 2023-09-24T22:47:38.935467+0000 2023-09-23T23:13:14.834 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.138:0/3258575069 expires 2023-09-24T22:57:33.926232+0000 2023-09-23T23:13:14.834 INFO:teuthology.orchestra.run.smithi110.stdout:blocklist 172.21.15.110:0/2804253682 expires 2023-09-24T22:58:42.842267+0000 2023-09-23T23:13:15.615 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:15 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2796488273' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T23:13:15.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:15 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2796488273' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T23:13:15.633 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 91 up_thru 153 down_at 88 last_clean_interval [8,87) [v2:172.21.15.110:6802/983471373,v1:172.21.15.110:6803/983471373] [v2:172.21.15.110:6804/983471373,v1:172.21.15.110:6805/983471373] exists,up 78791acb-c93c-4e4f-bf04-ea3eee46408d', 'osd.1 up in weight 1 up_from 112 up_thru 161 down_at 108 last_clean_interval [13,107) [v2:172.21.15.110:6810/594715751,v1:172.21.15.110:6811/594715751] [v2:172.21.15.110:6812/594715751,v1:172.21.15.110:6813/594715751] exists,up 6793178f-2ef1-4c48-927c-831a083a3356', 'osd.2 up in weight 1 up_from 130 up_thru 148 down_at 127 last_clean_interval [18,126) [v2:172.21.15.110:6818/3154710012,v1:172.21.15.110:6819/3154710012] [v2:172.21.15.110:6820/3154710012,v1:172.21.15.110:6821/3154710012] exists,up 146fbd28-a365-4545-8e97-d629c6ddba21', 'osd.3 up in weight 1 up_from 148 up_thru 159 down_at 145 last_clean_interval [25,144) [v2:172.21.15.110:6826/2333701749,v1:172.21.15.110:6827/2333701749] [v2:172.21.15.110:6828/2333701749,v1:172.21.15.110:6829/2333701749] exists,up 6fcf80d5-6977-4929-b7ba-0e9ab65df0a8', 'osd.4 up in weight 1 up_from 30 up_thru 148 down_at 0 last_clean_interval [0,0) [v2:172.21.15.138:6800/941787557,v1:172.21.15.138:6801/941787557] [v2:172.21.15.138:6802/941787557,v1:172.21.15.138:6803/941787557] exists,up 131d0749-c46c-44f0-ab7f-03f8ad6ed502', 'osd.5 up in weight 1 up_from 38 up_thru 155 down_at 0 last_clean_interval [0,0) [v2:172.21.15.138:6808/3098195887,v1:172.21.15.138:6809/3098195887] [v2:172.21.15.138:6810/3098195887,v1:172.21.15.138:6811/3098195887] exists,up 8cb496e5-d242-4e41-b279-e8d5ecea2996', 'osd.6 up in weight 1 up_from 46 up_thru 145 down_at 0 last_clean_interval [0,0) [v2:172.21.15.138:6816/187413403,v1:172.21.15.138:6817/187413403] [v2:172.21.15.138:6818/187413403,v1:172.21.15.138:6819/187413403] exists,up c709e629-85ab-4273-9ca8-f2c4e8fc9074', 'osd.7 up in weight 1 up_from 54 up_thru 148 down_at 0 last_clean_interval [0,0) [v2:172.21.15.138:6824/676244673,v1:172.21.15.138:6825/676244673] [v2:172.21.15.138:6826/676244673,v1:172.21.15.138:6827/676244673] exists,up ed15f3b0-cd2e-43b7-b803-e45252b62a45'] 2023-09-23T23:13:15.633 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon mon.a config show 2023-09-23T23:13:15.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:15 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2796488273' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T23:13:15.979 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:16.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:16 smithi110 ceph-mon[212939]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:16.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:16 smithi110 ceph-mon[221102]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:16.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:16 smithi138 ceph-mon[193606]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:17.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:13:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:13:16] "GET /metrics HTTP/1.1" 200 34047 "" "Prometheus/2.43.0" 2023-09-23T23:13:17.544 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:13:17.544 INFO:teuthology.orchestra.run.smithi110.stdout: "name": "mon.a", 2023-09-23T23:13:17.544 INFO:teuthology.orchestra.run.smithi110.stdout: "cluster": "ceph", 2023-09-23T23:13:17.544 INFO:teuthology.orchestra.run.smithi110.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-09-23T23:13:17.544 INFO:teuthology.orchestra.run.smithi110.stdout: "admin_socket_mode": "", 2023-09-23T23:13:17.544 INFO:teuthology.orchestra.run.smithi110.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-09-23T23:13:17.544 INFO:teuthology.orchestra.run.smithi110.stdout: "auth_client_required": "cephx, none", 2023-09-23T23:13:17.544 INFO:teuthology.orchestra.run.smithi110.stdout: "auth_cluster_required": "cephx", 2023-09-23T23:13:17.545 INFO:teuthology.orchestra.run.smithi110.stdout: "auth_debug": "true", 2023-09-23T23:13:17.545 INFO:teuthology.orchestra.run.smithi110.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-09-23T23:13:17.545 INFO:teuthology.orchestra.run.smithi110.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-09-23T23:13:17.545 INFO:teuthology.orchestra.run.smithi110.stdout: "auth_service_required": "cephx", 2023-09-23T23:13:17.545 INFO:teuthology.orchestra.run.smithi110.stdout: "auth_service_ticket_ttl": "240.000000", 2023-09-23T23:13:17.545 INFO:teuthology.orchestra.run.smithi110.stdout: "auth_supported": "", 2023-09-23T23:13:17.545 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_aio": "true", 2023-09-23T23:13:17.545 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_aio_max_queue_depth": "1024", 2023-09-23T23:13:17.545 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_aio_poll_ms": "250", 2023-09-23T23:13:17.545 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_aio_reap_max": "16", 2023-09-23T23:13:17.545 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_async_discard": "false", 2023-09-23T23:13:17.545 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_block_size": "4096", 2023-09-23T23:13:17.545 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_debug_aio": "false", 2023-09-23T23:13:17.546 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-09-23T23:13:17.546 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-09-23T23:13:17.546 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_debug_inflight_ios": "false", 2023-09-23T23:13:17.546 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_enable_discard": "false", 2023-09-23T23:13:17.546 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_flock_retry": "3", 2023-09-23T23:13:17.546 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_flock_retry_interval": "0.100000", 2023-09-23T23:13:17.546 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_inject_crash": "0", 2023-09-23T23:13:17.546 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_inject_crash_flush_delay": "2", 2023-09-23T23:13:17.546 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_ioring": "false", 2023-09-23T23:13:17.546 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_ioring_hipri": "false", 2023-09-23T23:13:17.546 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_ioring_sqthread_poll": "false", 2023-09-23T23:13:17.546 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-09-23T23:13:17.547 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_read_buffer_alignment": "4096", 2023-09-23T23:13:17.547 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-09-23T23:13:17.547 INFO:teuthology.orchestra.run.smithi110.stdout: "bdev_type": "", 2023-09-23T23:13:17.547 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_alloc_size": "1048576", 2023-09-23T23:13:17.547 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_allocator": "hybrid", 2023-09-23T23:13:17.547 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_buffered_io": "true", 2023-09-23T23:13:17.547 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_check_for_zeros": "false", 2023-09-23T23:13:17.547 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_check_volume_selector_often": "false", 2023-09-23T23:13:17.547 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-09-23T23:13:17.547 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_compact_log_sync": "false", 2023-09-23T23:13:17.547 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-09-23T23:13:17.547 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-09-23T23:13:17.547 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_log_compact_min_size": "16777216", 2023-09-23T23:13:17.548 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_log_replay_check_allocations": "true", 2023-09-23T23:13:17.548 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_max_log_runway": "4194304", 2023-09-23T23:13:17.548 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_max_prefetch": "1048576", 2023-09-23T23:13:17.548 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_min_flush_size": "524288", 2023-09-23T23:13:17.548 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_min_log_runway": "1048576", 2023-09-23T23:13:17.548 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_replay_recovery": "false", 2023-09-23T23:13:17.548 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-09-23T23:13:17.548 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_shared_alloc_size": "65536", 2023-09-23T23:13:17.548 INFO:teuthology.orchestra.run.smithi110.stdout: "bluefs_sync_write": "false", 2023-09-23T23:13:17.548 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-09-23T23:13:17.548 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-09-23T23:13:17.548 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-09-23T23:13:17.548 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_allocation_from_file": "true", 2023-09-23T23:13:17.549 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_allocator": "hybrid", 2023-09-23T23:13:17.549 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-09-23T23:13:17.549 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-09-23T23:13:17.549 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-09-23T23:13:17.549 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-09-23T23:13:17.549 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-09-23T23:13:17.549 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-09-23T23:13:17.549 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_blobid_prealloc": "10240", 2023-09-23T23:13:17.549 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_block_create": "true", 2023-09-23T23:13:17.549 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_block_db_create": "false", 2023-09-23T23:13:17.549 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_block_db_path": "", 2023-09-23T23:13:17.549 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_block_db_size": "0", 2023-09-23T23:13:17.549 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_block_path": "", 2023-09-23T23:13:17.550 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_block_preallocate_file": "false", 2023-09-23T23:13:17.550 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_block_size": "107374182400", 2023-09-23T23:13:17.550 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_block_wal_create": "false", 2023-09-23T23:13:17.550 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_block_wal_path": "", 2023-09-23T23:13:17.550 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_block_wal_size": "100663296", 2023-09-23T23:13:17.550 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_bluefs": "true", 2023-09-23T23:13:17.550 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-09-23T23:13:17.550 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_bluefs_env_mirror": "false", 2023-09-23T23:13:17.550 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-09-23T23:13:17.550 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-09-23T23:13:17.550 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T23:13:17.550 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T23:13:17.551 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-09-23T23:13:17.551 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T23:13:17.551 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_cache_autotune": "true", 2023-09-23T23:13:17.551 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-09-23T23:13:17.551 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-09-23T23:13:17.551 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-09-23T23:13:17.551 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-09-23T23:13:17.551 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_cache_size": "0", 2023-09-23T23:13:17.551 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-09-23T23:13:17.551 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-09-23T23:13:17.551 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-09-23T23:13:17.551 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-09-23T23:13:17.551 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_cache_type": "2q", 2023-09-23T23:13:17.552 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-09-23T23:13:17.552 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_clone_cow": "true", 2023-09-23T23:13:17.552 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_compression_algorithm": "snappy", 2023-09-23T23:13:17.552 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_compression_max_blob_size": "0", 2023-09-23T23:13:17.552 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-09-23T23:13:17.552 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-09-23T23:13:17.552 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_compression_min_blob_size": "0", 2023-09-23T23:13:17.552 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-09-23T23:13:17.552 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-09-23T23:13:17.552 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_compression_mode": "none", 2023-09-23T23:13:17.552 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-09-23T23:13:17.552 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_csum_type": "crc32c", 2023-09-23T23:13:17.552 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_enforce_settings": "default", 2023-09-23T23:13:17.553 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_freelist": "false", 2023-09-23T23:13:17.553 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_fsck_abort": "false", 2023-09-23T23:13:17.553 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-09-23T23:13:17.553 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_inject_bug21040": "false", 2023-09-23T23:13:17.553 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-09-23T23:13:17.553 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_inject_read_err": "false", 2023-09-23T23:13:17.553 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_legacy_omap": "false", 2023-09-23T23:13:17.553 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-09-23T23:13:17.553 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-09-23T23:13:17.553 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-09-23T23:13:17.553 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-09-23T23:13:17.553 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_prefill": "0.000000", 2023-09-23T23:13:17.554 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-09-23T23:13:17.554 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-09-23T23:13:17.554 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-09-23T23:13:17.554 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_small_allocations": "0", 2023-09-23T23:13:17.554 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-09-23T23:13:17.554 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_default_buffered_read": "true", 2023-09-23T23:13:17.554 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_default_buffered_write": "false", 2023-09-23T23:13:17.554 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_deferred_batch_ops": "0", 2023-09-23T23:13:17.554 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-09-23T23:13:17.554 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-09-23T23:13:17.554 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-09-23T23:13:17.554 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-09-23T23:13:17.555 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-09-23T23:13:17.555 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-09-23T23:13:17.555 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-09-23T23:13:17.555 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_fail_eio": "false", 2023-09-23T23:13:17.555 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-09-23T23:13:17.555 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-09-23T23:13:17.555 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-09-23T23:13:17.555 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-09-23T23:13:17.555 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_fsck_on_mkfs": "true", 2023-09-23T23:13:17.555 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-09-23T23:13:17.555 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_fsck_on_mount": "false", 2023-09-23T23:13:17.555 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-09-23T23:13:17.555 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_fsck_on_umount": "false", 2023-09-23T23:13:17.556 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-09-23T23:13:17.556 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-09-23T23:13:17.556 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-09-23T23:13:17.556 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-09-23T23:13:17.556 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-09-23T23:13:17.556 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-09-23T23:13:17.556 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-09-23T23:13:17.556 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-09-23T23:13:17.556 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_ignore_data_csum": "false", 2023-09-23T23:13:17.556 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-09-23T23:13:17.556 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_kvbackend": "rocksdb", 2023-09-23T23:13:17.556 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-09-23T23:13:17.557 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-09-23T23:13:17.557 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_log_op_age": "5.000000", 2023-09-23T23:13:17.557 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_max_alloc_size": "0", 2023-09-23T23:13:17.557 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_max_blob_size": "0", 2023-09-23T23:13:17.557 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-09-23T23:13:17.557 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-09-23T23:13:17.557 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_max_defer_interval": "3.000000", 2023-09-23T23:13:17.557 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_max_deferred_txc": "32", 2023-09-23T23:13:17.557 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_min_alloc_size": "0", 2023-09-23T23:13:17.557 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-09-23T23:13:17.557 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-09-23T23:13:17.557 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_nid_prealloc": "1024", 2023-09-23T23:13:17.557 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_prefer_deferred_size": "0", 2023-09-23T23:13:17.558 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-09-23T23:13:17.558 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-09-23T23:13:17.558 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_qfsck_on_mount": "true", 2023-09-23T23:13:17.558 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_retry_disk_reads": "3", 2023-09-23T23:13:17.558 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_rocksdb_cf": "true", 2023-09-23T23:13:17.558 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-09-23T23:13:17.558 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-09-23T23:13:17.558 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_rocksdb_options_annex": "", 2023-09-23T23:13:17.558 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_spdk_coremask": "0x1", 2023-09-23T23:13:17.558 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_spdk_io_sleep": "5", 2023-09-23T23:13:17.558 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_spdk_max_io_completion": "0", 2023-09-23T23:13:17.559 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_spdk_mem": "512", 2023-09-23T23:13:17.559 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_sync_submit_transaction": "false", 2023-09-23T23:13:17.559 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_throttle_bytes": "67108864", 2023-09-23T23:13:17.559 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_throttle_cost_per_io": "0", 2023-09-23T23:13:17.559 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-09-23T23:13:17.559 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-09-23T23:13:17.559 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-09-23T23:13:17.559 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-09-23T23:13:17.559 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_tracing": "false", 2023-09-23T23:13:17.559 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-09-23T23:13:17.559 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-09-23T23:13:17.559 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_volume_selection_reserved": "0", 2023-09-23T23:13:17.560 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-09-23T23:13:17.560 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-09-23T23:13:17.560 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-09-23T23:13:17.560 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-09-23T23:13:17.560 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-09-23T23:13:17.560 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-09-23T23:13:17.560 INFO:teuthology.orchestra.run.smithi110.stdout: "bluestore_zero_block_detection": "false", 2023-09-23T23:13:17.560 INFO:teuthology.orchestra.run.smithi110.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-09-23T23:13:17.560 INFO:teuthology.orchestra.run.smithi110.stdout: "cephfs_mirror_action_update_interval": "2", 2023-09-23T23:13:17.560 INFO:teuthology.orchestra.run.smithi110.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-09-23T23:13:17.560 INFO:teuthology.orchestra.run.smithi110.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-09-23T23:13:17.560 INFO:teuthology.orchestra.run.smithi110.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-09-23T23:13:17.560 INFO:teuthology.orchestra.run.smithi110.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-09-23T23:13:17.561 INFO:teuthology.orchestra.run.smithi110.stdout: "cephfs_mirror_mount_timeout": "10", 2023-09-23T23:13:17.561 INFO:teuthology.orchestra.run.smithi110.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-09-23T23:13:17.561 INFO:teuthology.orchestra.run.smithi110.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-09-23T23:13:17.561 INFO:teuthology.orchestra.run.smithi110.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-09-23T23:13:17.561 INFO:teuthology.orchestra.run.smithi110.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-09-23T23:13:17.561 INFO:teuthology.orchestra.run.smithi110.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-09-23T23:13:17.561 INFO:teuthology.orchestra.run.smithi110.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-09-23T23:13:17.561 INFO:teuthology.orchestra.run.smithi110.stdout: "cephx_cluster_require_signatures": "false", 2023-09-23T23:13:17.561 INFO:teuthology.orchestra.run.smithi110.stdout: "cephx_cluster_require_version": "2", 2023-09-23T23:13:17.561 INFO:teuthology.orchestra.run.smithi110.stdout: "cephx_require_signatures": "false", 2023-09-23T23:13:17.561 INFO:teuthology.orchestra.run.smithi110.stdout: "cephx_require_version": "2", 2023-09-23T23:13:17.561 INFO:teuthology.orchestra.run.smithi110.stdout: "cephx_service_require_signatures": "false", 2023-09-23T23:13:17.562 INFO:teuthology.orchestra.run.smithi110.stdout: "cephx_service_require_version": "2", 2023-09-23T23:13:17.562 INFO:teuthology.orchestra.run.smithi110.stdout: "cephx_sign_messages": "true", 2023-09-23T23:13:17.562 INFO:teuthology.orchestra.run.smithi110.stdout: "chdir": "", 2023-09-23T23:13:17.562 INFO:teuthology.orchestra.run.smithi110.stdout: "client_acl_type": "", 2023-09-23T23:13:17.562 INFO:teuthology.orchestra.run.smithi110.stdout: "client_asio_thread_count": "2", 2023-09-23T23:13:17.562 INFO:teuthology.orchestra.run.smithi110.stdout: "client_cache_mid": "0.750000", 2023-09-23T23:13:17.562 INFO:teuthology.orchestra.run.smithi110.stdout: "client_cache_size": "16384", 2023-09-23T23:13:17.562 INFO:teuthology.orchestra.run.smithi110.stdout: "client_caps_release_delay": "5", 2023-09-23T23:13:17.562 INFO:teuthology.orchestra.run.smithi110.stdout: "client_check_pool_perm": "true", 2023-09-23T23:13:17.562 INFO:teuthology.orchestra.run.smithi110.stdout: "client_collect_and_send_global_metrics": "false", 2023-09-23T23:13:17.562 INFO:teuthology.orchestra.run.smithi110.stdout: "client_debug_force_sync_read": "false", 2023-09-23T23:13:17.562 INFO:teuthology.orchestra.run.smithi110.stdout: "client_debug_getattr_caps": "false", 2023-09-23T23:13:17.563 INFO:teuthology.orchestra.run.smithi110.stdout: "client_debug_inject_tick_delay": "0", 2023-09-23T23:13:17.563 INFO:teuthology.orchestra.run.smithi110.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-09-23T23:13:17.563 INFO:teuthology.orchestra.run.smithi110.stdout: "client_die_on_failed_remount": "false", 2023-09-23T23:13:17.563 INFO:teuthology.orchestra.run.smithi110.stdout: "client_dirsize_rbytes": "true", 2023-09-23T23:13:17.563 INFO:teuthology.orchestra.run.smithi110.stdout: "client_force_lazyio": "false", 2023-09-23T23:13:17.563 INFO:teuthology.orchestra.run.smithi110.stdout: "client_fs": "", 2023-09-23T23:13:17.563 INFO:teuthology.orchestra.run.smithi110.stdout: "client_inject_fixed_oldest_tid": "false", 2023-09-23T23:13:17.563 INFO:teuthology.orchestra.run.smithi110.stdout: "client_inject_release_failure": "false", 2023-09-23T23:13:17.563 INFO:teuthology.orchestra.run.smithi110.stdout: "client_max_inline_size": "4096", 2023-09-23T23:13:17.563 INFO:teuthology.orchestra.run.smithi110.stdout: "client_max_retries_on_remount_failure": "5", 2023-09-23T23:13:17.563 INFO:teuthology.orchestra.run.smithi110.stdout: "client_mds_namespace": "", 2023-09-23T23:13:17.563 INFO:teuthology.orchestra.run.smithi110.stdout: "client_metadata": "", 2023-09-23T23:13:17.564 INFO:teuthology.orchestra.run.smithi110.stdout: "client_mount_gid": "-1", 2023-09-23T23:13:17.564 INFO:teuthology.orchestra.run.smithi110.stdout: "client_mount_timeout": "300", 2023-09-23T23:13:17.564 INFO:teuthology.orchestra.run.smithi110.stdout: "client_mount_uid": "-1", 2023-09-23T23:13:17.564 INFO:teuthology.orchestra.run.smithi110.stdout: "client_mountpoint": "/", 2023-09-23T23:13:17.564 INFO:teuthology.orchestra.run.smithi110.stdout: "client_notify_timeout": "10", 2023-09-23T23:13:17.564 INFO:teuthology.orchestra.run.smithi110.stdout: "client_oc": "true", 2023-09-23T23:13:17.564 INFO:teuthology.orchestra.run.smithi110.stdout: "client_oc_max_dirty": "104857600", 2023-09-23T23:13:17.564 INFO:teuthology.orchestra.run.smithi110.stdout: "client_oc_max_dirty_age": "5.000000", 2023-09-23T23:13:17.564 INFO:teuthology.orchestra.run.smithi110.stdout: "client_oc_max_objects": "1000", 2023-09-23T23:13:17.564 INFO:teuthology.orchestra.run.smithi110.stdout: "client_oc_size": "209715200", 2023-09-23T23:13:17.564 INFO:teuthology.orchestra.run.smithi110.stdout: "client_oc_target_dirty": "8388608", 2023-09-23T23:13:17.564 INFO:teuthology.orchestra.run.smithi110.stdout: "client_permissions": "true", 2023-09-23T23:13:17.565 INFO:teuthology.orchestra.run.smithi110.stdout: "client_quota": "true", 2023-09-23T23:13:17.565 INFO:teuthology.orchestra.run.smithi110.stdout: "client_quota_df": "true", 2023-09-23T23:13:17.565 INFO:teuthology.orchestra.run.smithi110.stdout: "client_readahead_max_bytes": "0", 2023-09-23T23:13:17.565 INFO:teuthology.orchestra.run.smithi110.stdout: "client_readahead_max_periods": "4", 2023-09-23T23:13:17.565 INFO:teuthology.orchestra.run.smithi110.stdout: "client_readahead_min": "131072", 2023-09-23T23:13:17.565 INFO:teuthology.orchestra.run.smithi110.stdout: "client_reconnect_stale": "false", 2023-09-23T23:13:17.565 INFO:teuthology.orchestra.run.smithi110.stdout: "client_shutdown_timeout": "30", 2023-09-23T23:13:17.565 INFO:teuthology.orchestra.run.smithi110.stdout: "client_snapdir": ".snap", 2023-09-23T23:13:17.565 INFO:teuthology.orchestra.run.smithi110.stdout: "client_tick_interval": "1", 2023-09-23T23:13:17.565 INFO:teuthology.orchestra.run.smithi110.stdout: "client_trace": "", 2023-09-23T23:13:17.565 INFO:teuthology.orchestra.run.smithi110.stdout: "client_try_dentry_invalidate": "false", 2023-09-23T23:13:17.565 INFO:teuthology.orchestra.run.smithi110.stdout: "client_use_faked_inos": "false", 2023-09-23T23:13:17.565 INFO:teuthology.orchestra.run.smithi110.stdout: "client_use_random_mds": "false", 2023-09-23T23:13:17.566 INFO:teuthology.orchestra.run.smithi110.stdout: "clog_to_graylog": "false", 2023-09-23T23:13:17.566 INFO:teuthology.orchestra.run.smithi110.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-09-23T23:13:17.566 INFO:teuthology.orchestra.run.smithi110.stdout: "clog_to_graylog_port": "12201", 2023-09-23T23:13:17.566 INFO:teuthology.orchestra.run.smithi110.stdout: "clog_to_monitors": "default=true", 2023-09-23T23:13:17.566 INFO:teuthology.orchestra.run.smithi110.stdout: "clog_to_syslog": "false", 2023-09-23T23:13:17.566 INFO:teuthology.orchestra.run.smithi110.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-09-23T23:13:17.566 INFO:teuthology.orchestra.run.smithi110.stdout: "clog_to_syslog_level": "info", 2023-09-23T23:13:17.566 INFO:teuthology.orchestra.run.smithi110.stdout: "cluster_addr": "-", 2023-09-23T23:13:17.566 INFO:teuthology.orchestra.run.smithi110.stdout: "cluster_network": "", 2023-09-23T23:13:17.566 INFO:teuthology.orchestra.run.smithi110.stdout: "cluster_network_interface": "", 2023-09-23T23:13:17.566 INFO:teuthology.orchestra.run.smithi110.stdout: "compressor_zlib_isal": "false", 2023-09-23T23:13:17.566 INFO:teuthology.orchestra.run.smithi110.stdout: "compressor_zlib_level": "5", 2023-09-23T23:13:17.567 INFO:teuthology.orchestra.run.smithi110.stdout: "compressor_zlib_winsize": "-15", 2023-09-23T23:13:17.567 INFO:teuthology.orchestra.run.smithi110.stdout: "compressor_zstd_level": "1", 2023-09-23T23:13:17.567 INFO:teuthology.orchestra.run.smithi110.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d", 2023-09-23T23:13:17.567 INFO:teuthology.orchestra.run.smithi110.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-09-23T23:13:17.567 INFO:teuthology.orchestra.run.smithi110.stdout: "crimson_alien_op_num_threads": "6", 2023-09-23T23:13:17.567 INFO:teuthology.orchestra.run.smithi110.stdout: "crimson_alien_thread_cpu_cores": "", 2023-09-23T23:13:17.567 INFO:teuthology.orchestra.run.smithi110.stdout: "crimson_osd_obc_lru_size": "10", 2023-09-23T23:13:17.567 INFO:teuthology.orchestra.run.smithi110.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-09-23T23:13:17.567 INFO:teuthology.orchestra.run.smithi110.stdout: "crimson_seastar_smp": "1", 2023-09-23T23:13:17.567 INFO:teuthology.orchestra.run.smithi110.stdout: "crush_location": "", 2023-09-23T23:13:17.567 INFO:teuthology.orchestra.run.smithi110.stdout: "crush_location_hook": "", 2023-09-23T23:13:17.567 INFO:teuthology.orchestra.run.smithi110.stdout: "crush_location_hook_timeout": "10", 2023-09-23T23:13:17.568 INFO:teuthology.orchestra.run.smithi110.stdout: "daemonize": "false", 2023-09-23T23:13:17.568 INFO:teuthology.orchestra.run.smithi110.stdout: "daos_pool": "tank", 2023-09-23T23:13:17.568 INFO:teuthology.orchestra.run.smithi110.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-09-23T23:13:17.568 INFO:teuthology.orchestra.run.smithi110.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-09-23T23:13:17.568 INFO:teuthology.orchestra.run.smithi110.stdout: "dbstore_db_name_prefix": "dbstore", 2023-09-23T23:13:17.568 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_alienstore": "0/5", 2023-09-23T23:13:17.568 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_allow_any_pool_priority": "false", 2023-09-23T23:13:17.568 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_asok": "1/5", 2023-09-23T23:13:17.568 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_asok_assert_abort": "false", 2023-09-23T23:13:17.568 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_asserts_on_shutdown": "true", 2023-09-23T23:13:17.568 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_auth": "1/5", 2023-09-23T23:13:17.568 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_bdev": "1/3", 2023-09-23T23:13:17.568 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_bluefs": "1/5", 2023-09-23T23:13:17.569 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_bluestore": "1/5", 2023-09-23T23:13:17.569 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_buffer": "0/1", 2023-09-23T23:13:17.569 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_ceph_exporter": "1/5", 2023-09-23T23:13:17.569 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_cephfs_mirror": "0/5", 2023-09-23T23:13:17.569 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_cephsqlite": "0/5", 2023-09-23T23:13:17.569 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_client": "0/5", 2023-09-23T23:13:17.569 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_compressor": "1/5", 2023-09-23T23:13:17.569 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_context": "0/1", 2023-09-23T23:13:17.569 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_crush": "1/1", 2023-09-23T23:13:17.569 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_crypto": "1/5", 2023-09-23T23:13:17.569 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_cyanstore": "0/5", 2023-09-23T23:13:17.569 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_deliberately_leak_memory": "false", 2023-09-23T23:13:17.569 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_disable_randomized_ping": "false", 2023-09-23T23:13:17.570 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_dpdk": "1/5", 2023-09-23T23:13:17.570 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_eventtrace": "1/5", 2023-09-23T23:13:17.570 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_filer": "0/1", 2023-09-23T23:13:17.570 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_filestore": "1/3", 2023-09-23T23:13:17.570 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_finisher": "1/1", 2023-09-23T23:13:17.570 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_fuse": "1/5", 2023-09-23T23:13:17.570 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_heartbeat_testing_span": "0", 2023-09-23T23:13:17.570 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_heartbeatmap": "1/5", 2023-09-23T23:13:17.570 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_immutable_obj_cache": "0/5", 2023-09-23T23:13:17.570 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_javaclient": "1/5", 2023-09-23T23:13:17.570 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_journal": "1/3", 2023-09-23T23:13:17.571 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_journaler": "0/5", 2023-09-23T23:13:17.571 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_kstore": "1/5", 2023-09-23T23:13:17.571 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_leveldb": "4/5", 2023-09-23T23:13:17.571 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_lockdep": "0/1", 2023-09-23T23:13:17.571 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_mclock": "1/5", 2023-09-23T23:13:17.571 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_mds": "1/5", 2023-09-23T23:13:17.571 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_mds_balancer": "1/5", 2023-09-23T23:13:17.571 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_mds_locker": "1/5", 2023-09-23T23:13:17.571 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_mds_log": "1/5", 2023-09-23T23:13:17.571 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_mds_log_expire": "1/5", 2023-09-23T23:13:17.571 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_mds_migrator": "1/5", 2023-09-23T23:13:17.571 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_memstore": "1/5", 2023-09-23T23:13:17.571 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_mgr": "2/5", 2023-09-23T23:13:17.572 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_mgrc": "1/5", 2023-09-23T23:13:17.572 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_mon": "20/20", 2023-09-23T23:13:17.572 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_monc": "0/10", 2023-09-23T23:13:17.572 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_ms": "1/1", 2023-09-23T23:13:17.572 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_none": "0/5", 2023-09-23T23:13:17.572 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_objclass": "0/5", 2023-09-23T23:13:17.572 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_objectcacher": "0/5", 2023-09-23T23:13:17.572 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_objecter": "0/1", 2023-09-23T23:13:17.572 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_optracker": "0/5", 2023-09-23T23:13:17.572 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_osd": "1/5", 2023-09-23T23:13:17.572 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_paxos": "20/20", 2023-09-23T23:13:17.572 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_perfcounter": "1/5", 2023-09-23T23:13:17.572 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_prioritycache": "1/5", 2023-09-23T23:13:17.573 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_rados": "0/5", 2023-09-23T23:13:17.573 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_rbd": "0/5", 2023-09-23T23:13:17.573 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_rbd_mirror": "0/5", 2023-09-23T23:13:17.573 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_rbd_pwl": "0/5", 2023-09-23T23:13:17.573 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_rbd_replay": "0/5", 2023-09-23T23:13:17.573 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_refs": "0/0", 2023-09-23T23:13:17.573 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_reserver": "1/1", 2023-09-23T23:13:17.573 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_rgw": "1/5", 2023-09-23T23:13:17.573 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_rgw_access": "1/5", 2023-09-23T23:13:17.573 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_rgw_datacache": "1/5", 2023-09-23T23:13:17.574 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_rgw_dbstore": "1/5", 2023-09-23T23:13:17.574 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_rgw_flight": "1/5", 2023-09-23T23:13:17.574 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_rgw_sync": "1/5", 2023-09-23T23:13:17.574 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_rocksdb": "4/5", 2023-09-23T23:13:17.574 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_seastore": "0/5", 2023-09-23T23:13:17.574 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_seastore_backref": "0/5", 2023-09-23T23:13:17.574 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_seastore_cache": "0/5", 2023-09-23T23:13:17.574 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_seastore_cleaner": "0/5", 2023-09-23T23:13:17.574 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_seastore_device": "0/5", 2023-09-23T23:13:17.574 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_seastore_epm": "0/5", 2023-09-23T23:13:17.574 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-09-23T23:13:17.575 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_seastore_journal": "0/5", 2023-09-23T23:13:17.575 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_seastore_lba": "0/5", 2023-09-23T23:13:17.575 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_seastore_odata": "0/5", 2023-09-23T23:13:17.575 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_seastore_omap": "0/5", 2023-09-23T23:13:17.575 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_seastore_onode": "0/5", 2023-09-23T23:13:17.575 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_seastore_t": "0/5", 2023-09-23T23:13:17.575 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_seastore_tm": "0/5", 2023-09-23T23:13:17.575 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_striper": "0/1", 2023-09-23T23:13:17.575 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_test": "0/5", 2023-09-23T23:13:17.575 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_throttle": "1/1", 2023-09-23T23:13:17.575 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_timer": "0/1", 2023-09-23T23:13:17.575 INFO:teuthology.orchestra.run.smithi110.stdout: "debug_tp": "0/5", 2023-09-23T23:13:17.575 INFO:teuthology.orchestra.run.smithi110.stdout: "device_failure_prediction_mode": "none", 2023-09-23T23:13:17.576 INFO:teuthology.orchestra.run.smithi110.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-09-23T23:13:17.576 INFO:teuthology.orchestra.run.smithi110.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-09-23T23:13:17.576 INFO:teuthology.orchestra.run.smithi110.stdout: "err_to_graylog": "false", 2023-09-23T23:13:17.576 INFO:teuthology.orchestra.run.smithi110.stdout: "err_to_journald": "false", 2023-09-23T23:13:17.576 INFO:teuthology.orchestra.run.smithi110.stdout: "err_to_stderr": "true", 2023-09-23T23:13:17.576 INFO:teuthology.orchestra.run.smithi110.stdout: "err_to_syslog": "false", 2023-09-23T23:13:17.576 INFO:teuthology.orchestra.run.smithi110.stdout: "event_tracing": "false", 2023-09-23T23:13:17.576 INFO:teuthology.orchestra.run.smithi110.stdout: "exporter_addr": "0.0.0.0", 2023-09-23T23:13:17.576 INFO:teuthology.orchestra.run.smithi110.stdout: "exporter_http_port": "9926", 2023-09-23T23:13:17.576 INFO:teuthology.orchestra.run.smithi110.stdout: "exporter_prio_limit": "5", 2023-09-23T23:13:17.576 INFO:teuthology.orchestra.run.smithi110.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-09-23T23:13:17.576 INFO:teuthology.orchestra.run.smithi110.stdout: "exporter_sort_metrics": "true", 2023-09-23T23:13:17.577 INFO:teuthology.orchestra.run.smithi110.stdout: "exporter_stats_period": "5", 2023-09-23T23:13:17.577 INFO:teuthology.orchestra.run.smithi110.stdout: "fake_statfs_for_testing": "0", 2023-09-23T23:13:17.577 INFO:teuthology.orchestra.run.smithi110.stdout: "fatal_signal_handlers": "true", 2023-09-23T23:13:17.577 INFO:teuthology.orchestra.run.smithi110.stdout: "filer_max_purge_ops": "10", 2023-09-23T23:13:17.577 INFO:teuthology.orchestra.run.smithi110.stdout: "filer_max_truncate_ops": "128", 2023-09-23T23:13:17.577 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_apply_finisher_threads": "1", 2023-09-23T23:13:17.577 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_blackhole": "false", 2023-09-23T23:13:17.577 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_btrfs_clone_range": "true", 2023-09-23T23:13:17.577 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_btrfs_snap": "true", 2023-09-23T23:13:17.577 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_caller_concurrency": "10", 2023-09-23T23:13:17.577 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_collect_device_partition_information": "true", 2023-09-23T23:13:17.577 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_commit_timeout": "600.000000", 2023-09-23T23:13:17.578 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_debug_inject_read_err": "false", 2023-09-23T23:13:17.578 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_debug_omap_check": "false", 2023-09-23T23:13:17.578 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_debug_random_read_err": "0.000000", 2023-09-23T23:13:17.578 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_debug_verify_split": "false", 2023-09-23T23:13:17.578 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_dump_file": "", 2023-09-23T23:13:17.578 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-09-23T23:13:17.578 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-09-23T23:13:17.578 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_fadvise": "true", 2023-09-23T23:13:17.578 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_fail_eio": "true", 2023-09-23T23:13:17.578 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_fd_cache_shards": "16", 2023-09-23T23:13:17.578 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_fd_cache_size": "128", 2023-09-23T23:13:17.578 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_fiemap": "false", 2023-09-23T23:13:17.579 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_fiemap_threshold": "4096", 2023-09-23T23:13:17.579 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-09-23T23:13:17.579 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_index_retry_probability": "0.000000", 2023-09-23T23:13:17.579 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_inject_stall": "0", 2023-09-23T23:13:17.579 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_journal_parallel": "false", 2023-09-23T23:13:17.579 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_journal_trailing": "false", 2023-09-23T23:13:17.579 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_journal_writeahead": "false", 2023-09-23T23:13:17.579 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_kill_at": "0", 2023-09-23T23:13:17.579 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-09-23T23:13:17.579 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_max_inline_xattr_size": "0", 2023-09-23T23:13:17.579 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-09-23T23:13:17.579 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-09-23T23:13:17.580 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-09-23T23:13:17.580 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_max_inline_xattrs": "0", 2023-09-23T23:13:17.580 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-09-23T23:13:17.580 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_max_inline_xattrs_other": "2", 2023-09-23T23:13:17.580 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-09-23T23:13:17.580 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_max_sync_interval": "5.000000", 2023-09-23T23:13:17.580 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_max_xattr_value_size": "0", 2023-09-23T23:13:17.580 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-09-23T23:13:17.580 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-09-23T23:13:17.580 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-09-23T23:13:17.580 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_merge_threshold": "-10", 2023-09-23T23:13:17.580 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_min_sync_interval": "0.010000", 2023-09-23T23:13:17.581 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_odsync_write": "false", 2023-09-23T23:13:17.581 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_omap_backend": "rocksdb", 2023-09-23T23:13:17.581 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_omap_backend_path": "", 2023-09-23T23:13:17.581 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_omap_header_cache_size": "1024", 2023-09-23T23:13:17.581 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_ondisk_finisher_threads": "1", 2023-09-23T23:13:17.581 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-09-23T23:13:17.581 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_op_thread_timeout": "60", 2023-09-23T23:13:17.581 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_op_threads": "2", 2023-09-23T23:13:17.581 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_punch_hole": "false", 2023-09-23T23:13:17.581 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-09-23T23:13:17.581 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-09-23T23:13:17.581 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-09-23T23:13:17.582 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-09-23T23:13:17.582 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-09-23T23:13:17.582 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_queue_max_bytes": "104857600", 2023-09-23T23:13:17.582 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-09-23T23:13:17.582 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-09-23T23:13:17.582 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-09-23T23:13:17.582 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_queue_max_ops": "50", 2023-09-23T23:13:17.582 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-09-23T23:13:17.582 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_seek_data_hole": "false", 2023-09-23T23:13:17.582 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_sloppy_crc": "false", 2023-09-23T23:13:17.582 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-09-23T23:13:17.583 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_splice": "false", 2023-09-23T23:13:17.583 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_split_multiple": "2", 2023-09-23T23:13:17.583 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_split_rand_factor": "20", 2023-09-23T23:13:17.583 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_update_to": "1000", 2023-09-23T23:13:17.583 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-09-23T23:13:17.583 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-09-23T23:13:17.583 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-09-23T23:13:17.583 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-09-23T23:13:17.583 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-09-23T23:13:17.583 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-09-23T23:13:17.583 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_wbthrottle_enable": "true", 2023-09-23T23:13:17.583 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-09-23T23:13:17.583 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-09-23T23:13:17.584 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-09-23T23:13:17.584 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-09-23T23:13:17.584 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-09-23T23:13:17.584 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-09-23T23:13:17.584 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_xfs_extsize": "false", 2023-09-23T23:13:17.584 INFO:teuthology.orchestra.run.smithi110.stdout: "filestore_zfs_snap": "false", 2023-09-23T23:13:17.584 INFO:teuthology.orchestra.run.smithi110.stdout: "fio_dir": "/tmp/fio", 2023-09-23T23:13:17.584 INFO:teuthology.orchestra.run.smithi110.stdout: "fsid": "f5f05148-5a62-11ee-8db4-212e2dc638e7", 2023-09-23T23:13:17.584 INFO:teuthology.orchestra.run.smithi110.stdout: "fuse_allow_other": "true", 2023-09-23T23:13:17.584 INFO:teuthology.orchestra.run.smithi110.stdout: "fuse_atomic_o_trunc": "true", 2023-09-23T23:13:17.584 INFO:teuthology.orchestra.run.smithi110.stdout: "fuse_big_writes": "true", 2023-09-23T23:13:17.584 INFO:teuthology.orchestra.run.smithi110.stdout: "fuse_debug": "false", 2023-09-23T23:13:17.584 INFO:teuthology.orchestra.run.smithi110.stdout: "fuse_default_permissions": "false", 2023-09-23T23:13:17.585 INFO:teuthology.orchestra.run.smithi110.stdout: "fuse_disable_pagecache": "false", 2023-09-23T23:13:17.585 INFO:teuthology.orchestra.run.smithi110.stdout: "fuse_max_write": "0", 2023-09-23T23:13:17.585 INFO:teuthology.orchestra.run.smithi110.stdout: "fuse_multithreaded": "true", 2023-09-23T23:13:17.585 INFO:teuthology.orchestra.run.smithi110.stdout: "fuse_require_active_mds": "true", 2023-09-23T23:13:17.585 INFO:teuthology.orchestra.run.smithi110.stdout: "fuse_set_user_groups": "true", 2023-09-23T23:13:17.585 INFO:teuthology.orchestra.run.smithi110.stdout: "fuse_splice_move": "true", 2023-09-23T23:13:17.585 INFO:teuthology.orchestra.run.smithi110.stdout: "fuse_splice_read": "true", 2023-09-23T23:13:17.585 INFO:teuthology.orchestra.run.smithi110.stdout: "fuse_splice_write": "true", 2023-09-23T23:13:17.585 INFO:teuthology.orchestra.run.smithi110.stdout: "fuse_syncfs_on_mksnap": "true", 2023-09-23T23:13:17.585 INFO:teuthology.orchestra.run.smithi110.stdout: "fuse_use_invalidate_cb": "true", 2023-09-23T23:13:17.585 INFO:teuthology.orchestra.run.smithi110.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-09-23T23:13:17.585 INFO:teuthology.orchestra.run.smithi110.stdout: "gss_target_name": "ceph", 2023-09-23T23:13:17.586 INFO:teuthology.orchestra.run.smithi110.stdout: "heartbeat_file": "", 2023-09-23T23:13:17.586 INFO:teuthology.orchestra.run.smithi110.stdout: "heartbeat_inject_failure": "0", 2023-09-23T23:13:17.586 INFO:teuthology.orchestra.run.smithi110.stdout: "heartbeat_interval": "5", 2023-09-23T23:13:17.586 INFO:teuthology.orchestra.run.smithi110.stdout: "host": "smithi110", 2023-09-23T23:13:17.586 INFO:teuthology.orchestra.run.smithi110.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-09-23T23:13:17.586 INFO:teuthology.orchestra.run.smithi110.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-09-23T23:13:17.586 INFO:teuthology.orchestra.run.smithi110.stdout: "immutable_object_cache_max_size": "1073741824", 2023-09-23T23:13:17.586 INFO:teuthology.orchestra.run.smithi110.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-09-23T23:13:17.586 INFO:teuthology.orchestra.run.smithi110.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-09-23T23:13:17.586 INFO:teuthology.orchestra.run.smithi110.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-09-23T23:13:17.586 INFO:teuthology.orchestra.run.smithi110.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-09-23T23:13:17.587 INFO:teuthology.orchestra.run.smithi110.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-09-23T23:13:17.587 INFO:teuthology.orchestra.run.smithi110.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-09-23T23:13:17.587 INFO:teuthology.orchestra.run.smithi110.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-09-23T23:13:17.587 INFO:teuthology.orchestra.run.smithi110.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-09-23T23:13:17.587 INFO:teuthology.orchestra.run.smithi110.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-09-23T23:13:17.587 INFO:teuthology.orchestra.run.smithi110.stdout: "immutable_object_cache_watermark": "0.900000", 2023-09-23T23:13:17.587 INFO:teuthology.orchestra.run.smithi110.stdout: "inject_early_sigterm": "false", 2023-09-23T23:13:17.587 INFO:teuthology.orchestra.run.smithi110.stdout: "jaeger_agent_port": "6799", 2023-09-23T23:13:17.587 INFO:teuthology.orchestra.run.smithi110.stdout: "jaeger_tracing_enable": "false", 2023-09-23T23:13:17.587 INFO:teuthology.orchestra.run.smithi110.stdout: "journal_aio": "true", 2023-09-23T23:13:17.587 INFO:teuthology.orchestra.run.smithi110.stdout: "journal_align_min_size": "65536", 2023-09-23T23:13:17.587 INFO:teuthology.orchestra.run.smithi110.stdout: "journal_block_align": "true", 2023-09-23T23:13:17.588 INFO:teuthology.orchestra.run.smithi110.stdout: "journal_block_size": "4096", 2023-09-23T23:13:17.588 INFO:teuthology.orchestra.run.smithi110.stdout: "journal_dio": "true", 2023-09-23T23:13:17.588 INFO:teuthology.orchestra.run.smithi110.stdout: "journal_discard": "false", 2023-09-23T23:13:17.588 INFO:teuthology.orchestra.run.smithi110.stdout: "journal_force_aio": "false", 2023-09-23T23:13:17.588 INFO:teuthology.orchestra.run.smithi110.stdout: "journal_ignore_corruption": "false", 2023-09-23T23:13:17.588 INFO:teuthology.orchestra.run.smithi110.stdout: "journal_max_write_bytes": "10485760", 2023-09-23T23:13:17.588 INFO:teuthology.orchestra.run.smithi110.stdout: "journal_max_write_entries": "100", 2023-09-23T23:13:17.588 INFO:teuthology.orchestra.run.smithi110.stdout: "journal_replay_from": "0", 2023-09-23T23:13:17.588 INFO:teuthology.orchestra.run.smithi110.stdout: "journal_throttle_high_multiple": "0.000000", 2023-09-23T23:13:17.588 INFO:teuthology.orchestra.run.smithi110.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-09-23T23:13:17.588 INFO:teuthology.orchestra.run.smithi110.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-09-23T23:13:17.588 INFO:teuthology.orchestra.run.smithi110.stdout: "journal_throttle_max_multiple": "0.000000", 2023-09-23T23:13:17.589 INFO:teuthology.orchestra.run.smithi110.stdout: "journal_write_header_frequency": "0", 2023-09-23T23:13:17.589 INFO:teuthology.orchestra.run.smithi110.stdout: "journal_zero_on_create": "false", 2023-09-23T23:13:17.589 INFO:teuthology.orchestra.run.smithi110.stdout: "journaler_prefetch_periods": "10", 2023-09-23T23:13:17.589 INFO:teuthology.orchestra.run.smithi110.stdout: "journaler_prezero_periods": "5", 2023-09-23T23:13:17.589 INFO:teuthology.orchestra.run.smithi110.stdout: "journaler_write_head_interval": "15", 2023-09-23T23:13:17.589 INFO:teuthology.orchestra.run.smithi110.stdout: "key": "", 2023-09-23T23:13:17.589 INFO:teuthology.orchestra.run.smithi110.stdout: "keyfile": "", 2023-09-23T23:13:17.589 INFO:teuthology.orchestra.run.smithi110.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-09-23T23:13:17.589 INFO:teuthology.orchestra.run.smithi110.stdout: "kstore_backend": "rocksdb", 2023-09-23T23:13:17.589 INFO:teuthology.orchestra.run.smithi110.stdout: "kstore_default_stripe_size": "65536", 2023-09-23T23:13:17.589 INFO:teuthology.orchestra.run.smithi110.stdout: "kstore_fsck_on_mount": "false", 2023-09-23T23:13:17.589 INFO:teuthology.orchestra.run.smithi110.stdout: "kstore_fsck_on_mount_deep": "true", 2023-09-23T23:13:17.590 INFO:teuthology.orchestra.run.smithi110.stdout: "kstore_max_bytes": "67108864", 2023-09-23T23:13:17.590 INFO:teuthology.orchestra.run.smithi110.stdout: "kstore_max_ops": "512", 2023-09-23T23:13:17.590 INFO:teuthology.orchestra.run.smithi110.stdout: "kstore_nid_prealloc": "1024", 2023-09-23T23:13:17.590 INFO:teuthology.orchestra.run.smithi110.stdout: "kstore_onode_map_size": "1024", 2023-09-23T23:13:17.590 INFO:teuthology.orchestra.run.smithi110.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-09-23T23:13:17.590 INFO:teuthology.orchestra.run.smithi110.stdout: "kstore_sync_submit_transaction": "false", 2023-09-23T23:13:17.590 INFO:teuthology.orchestra.run.smithi110.stdout: "kstore_sync_transaction": "false", 2023-09-23T23:13:17.590 INFO:teuthology.orchestra.run.smithi110.stdout: "leveldb_block_size": "65536", 2023-09-23T23:13:17.590 INFO:teuthology.orchestra.run.smithi110.stdout: "leveldb_bloom_size": "0", 2023-09-23T23:13:17.590 INFO:teuthology.orchestra.run.smithi110.stdout: "leveldb_cache_size": "536870912", 2023-09-23T23:13:17.590 INFO:teuthology.orchestra.run.smithi110.stdout: "leveldb_compact_on_mount": "false", 2023-09-23T23:13:17.590 INFO:teuthology.orchestra.run.smithi110.stdout: "leveldb_compression": "false", 2023-09-23T23:13:17.591 INFO:teuthology.orchestra.run.smithi110.stdout: "leveldb_log": "", 2023-09-23T23:13:17.591 INFO:teuthology.orchestra.run.smithi110.stdout: "leveldb_log_to_ceph_log": "true", 2023-09-23T23:13:17.591 INFO:teuthology.orchestra.run.smithi110.stdout: "leveldb_max_open_files": "0", 2023-09-23T23:13:17.591 INFO:teuthology.orchestra.run.smithi110.stdout: "leveldb_paranoid": "false", 2023-09-23T23:13:17.591 INFO:teuthology.orchestra.run.smithi110.stdout: "leveldb_write_buffer_size": "33554432", 2023-09-23T23:13:17.591 INFO:teuthology.orchestra.run.smithi110.stdout: "librados_thread_count": "2", 2023-09-23T23:13:17.591 INFO:teuthology.orchestra.run.smithi110.stdout: "lockdep": "false", 2023-09-23T23:13:17.591 INFO:teuthology.orchestra.run.smithi110.stdout: "lockdep_force_backtrace": "false", 2023-09-23T23:13:17.591 INFO:teuthology.orchestra.run.smithi110.stdout: "log_coarse_timestamps": "true", 2023-09-23T23:13:17.591 INFO:teuthology.orchestra.run.smithi110.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-09-23T23:13:17.591 INFO:teuthology.orchestra.run.smithi110.stdout: "log_flush_on_exit": "false", 2023-09-23T23:13:17.591 INFO:teuthology.orchestra.run.smithi110.stdout: "log_graylog_host": "127.0.0.1", 2023-09-23T23:13:17.591 INFO:teuthology.orchestra.run.smithi110.stdout: "log_graylog_port": "12201", 2023-09-23T23:13:17.592 INFO:teuthology.orchestra.run.smithi110.stdout: "log_max_new": "1000", 2023-09-23T23:13:17.592 INFO:teuthology.orchestra.run.smithi110.stdout: "log_max_recent": "10000", 2023-09-23T23:13:17.592 INFO:teuthology.orchestra.run.smithi110.stdout: "log_stderr_prefix": "", 2023-09-23T23:13:17.592 INFO:teuthology.orchestra.run.smithi110.stdout: "log_stop_at_utilization": "0.970000", 2023-09-23T23:13:17.592 INFO:teuthology.orchestra.run.smithi110.stdout: "log_to_file": "true", 2023-09-23T23:13:17.592 INFO:teuthology.orchestra.run.smithi110.stdout: "log_to_graylog": "false", 2023-09-23T23:13:17.592 INFO:teuthology.orchestra.run.smithi110.stdout: "log_to_journald": "false", 2023-09-23T23:13:17.592 INFO:teuthology.orchestra.run.smithi110.stdout: "log_to_stderr": "false", 2023-09-23T23:13:17.592 INFO:teuthology.orchestra.run.smithi110.stdout: "log_to_syslog": "false", 2023-09-23T23:13:17.592 INFO:teuthology.orchestra.run.smithi110.stdout: "max_rotating_auth_attempts": "10", 2023-09-23T23:13:17.592 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-09-23T23:13:17.592 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_action_on_write_error": "1", 2023-09-23T23:13:17.593 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_alternate_name_max": "8192", 2023-09-23T23:13:17.593 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_asio_thread_count": "2", 2023-09-23T23:13:17.593 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_export_pin": "true", 2023-09-23T23:13:17.593 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_fragment_dirs": "true", 2023-09-23T23:13:17.593 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-09-23T23:13:17.593 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_fragment_interval": "5", 2023-09-23T23:13:17.593 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_fragment_size_max": "100000", 2023-09-23T23:13:17.593 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_idle_threshold": "0.000000", 2023-09-23T23:13:17.593 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_interval": "10", 2023-09-23T23:13:17.593 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_max": "-1", 2023-09-23T23:13:17.593 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_max_until": "-1", 2023-09-23T23:13:17.593 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_merge_size": "50", 2023-09-23T23:13:17.594 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_midchunk": "0.300000", 2023-09-23T23:13:17.594 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_min_rebalance": "0.100000", 2023-09-23T23:13:17.594 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_min_start": "0.200000", 2023-09-23T23:13:17.594 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_minchunk": "0.001000", 2023-09-23T23:13:17.594 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_mode": "0", 2023-09-23T23:13:17.594 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_need_max": "1.200000", 2023-09-23T23:13:17.594 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_need_min": "0.800000", 2023-09-23T23:13:17.594 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-09-23T23:13:17.594 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_sample_interval": "3.000000", 2023-09-23T23:13:17.594 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_split_bits": "3", 2023-09-23T23:13:17.595 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_split_rd": "25000.000000", 2023-09-23T23:13:17.595 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_split_size": "10000", 2023-09-23T23:13:17.595 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_split_wr": "10000.000000", 2023-09-23T23:13:17.595 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_target_decay": "10.000000", 2023-09-23T23:13:17.595 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-09-23T23:13:17.595 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_beacon_grace": "15.000000", 2023-09-23T23:13:17.595 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_beacon_interval": "4.000000", 2023-09-23T23:13:17.595 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_beacon_mon_down_grace": "60", 2023-09-23T23:13:17.595 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_cache_memory_limit": "4294967296", 2023-09-23T23:13:17.595 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_cache_mid": "0.700000", 2023-09-23T23:13:17.595 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_cache_release_free_interval": "10", 2023-09-23T23:13:17.596 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_cache_reservation": "0.050000", 2023-09-23T23:13:17.596 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-09-23T23:13:17.596 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_cache_trim_interval": "1", 2023-09-23T23:13:17.596 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_cache_trim_threshold": "262144", 2023-09-23T23:13:17.596 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-09-23T23:13:17.596 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-09-23T23:13:17.596 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_client_delegate_inos_pct": "50", 2023-09-23T23:13:17.596 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_client_prealloc_inos": "1000", 2023-09-23T23:13:17.596 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-09-23T23:13:17.596 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_connect_bootstrapping": "false", 2023-09-23T23:13:17.596 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_damage_table_max_entries": "10000", 2023-09-23T23:13:17.596 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-09-23T23:13:17.596 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_debug_auth_pins": "false", 2023-09-23T23:13:17.597 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_debug_frag": "false", 2023-09-23T23:13:17.597 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_debug_scatterstat": "false", 2023-09-23T23:13:17.597 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_debug_subtrees": "false", 2023-09-23T23:13:17.597 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_decay_halflife": "5.000000", 2023-09-23T23:13:17.597 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_default_dir_hash": "2", 2023-09-23T23:13:17.597 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_defer_session_stale": "true", 2023-09-23T23:13:17.597 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_deny_all_reconnect": "false", 2023-09-23T23:13:17.597 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_dir_keys_per_op": "16384", 2023-09-23T23:13:17.597 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_dir_max_commit_size": "10", 2023-09-23T23:13:17.597 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_dir_max_entries": "0", 2023-09-23T23:13:17.597 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_dir_prefetch": "true", 2023-09-23T23:13:17.597 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_dirstat_min_interval": "1.000000", 2023-09-23T23:13:17.598 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_dump_cache_after_rejoin": "false", 2023-09-23T23:13:17.598 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_dump_cache_on_map": "false", 2023-09-23T23:13:17.598 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_dump_cache_threshold_file": "0", 2023-09-23T23:13:17.598 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-09-23T23:13:17.598 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_early_reply": "true", 2023-09-23T23:13:17.598 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_enable_op_tracker": "true", 2023-09-23T23:13:17.598 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_enforce_unique_name": "true", 2023-09-23T23:13:17.598 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_export_ephemeral_distributed": "true", 2023-09-23T23:13:17.598 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-09-23T23:13:17.598 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_export_ephemeral_random": "true", 2023-09-23T23:13:17.598 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-09-23T23:13:17.598 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-09-23T23:13:17.599 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_forward_all_requests_to_auth": "false", 2023-09-23T23:13:17.599 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-09-23T23:13:17.599 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-09-23T23:13:17.599 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-09-23T23:13:17.599 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-09-23T23:13:17.599 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_health_cache_threshold": "1.500000", 2023-09-23T23:13:17.599 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_health_summarize_threshold": "10", 2023-09-23T23:13:17.599 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_heartbeat_grace": "15.000000", 2023-09-23T23:13:17.599 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_heartbeat_reset_grace": "1000", 2023-09-23T23:13:17.599 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_inject_health_dummy": "false", 2023-09-23T23:13:17.599 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-09-23T23:13:17.599 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_inject_migrator_session_race": "false", 2023-09-23T23:13:17.599 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-09-23T23:13:17.600 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-09-23T23:13:17.600 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-09-23T23:13:17.600 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_join_fs": "", 2023-09-23T23:13:17.600 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_journal_format": "1", 2023-09-23T23:13:17.600 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_kill_create_at": "0", 2023-09-23T23:13:17.600 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_kill_export_at": "0", 2023-09-23T23:13:17.600 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_kill_import_at": "0", 2023-09-23T23:13:17.600 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_kill_journal_at": "0", 2023-09-23T23:13:17.600 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_kill_journal_expire_at": "0", 2023-09-23T23:13:17.600 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_kill_journal_replay_at": "0", 2023-09-23T23:13:17.600 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_kill_link_at": "0", 2023-09-23T23:13:17.600 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_kill_mdstable_at": "0", 2023-09-23T23:13:17.601 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_kill_openc_at": "0", 2023-09-23T23:13:17.601 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_kill_rename_at": "0", 2023-09-23T23:13:17.601 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-09-23T23:13:17.601 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_log_events_per_segment": "1024", 2023-09-23T23:13:17.601 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_log_max_events": "-1", 2023-09-23T23:13:17.601 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_log_max_segments": "128", 2023-09-23T23:13:17.601 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_log_pause": "false", 2023-09-23T23:13:17.601 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_log_segment_size": "0", 2023-09-23T23:13:17.601 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_log_skip_corrupt_events": "false", 2023-09-23T23:13:17.601 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_log_warn_factor": "2.000000", 2023-09-23T23:13:17.601 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_max_caps_per_client": "1048576", 2023-09-23T23:13:17.601 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_max_completed_flushes": "100000", 2023-09-23T23:13:17.602 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_max_completed_requests": "100000", 2023-09-23T23:13:17.602 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_max_export_size": "20971520", 2023-09-23T23:13:17.602 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_max_file_recover": "32", 2023-09-23T23:13:17.602 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_max_purge_files": "64", 2023-09-23T23:13:17.602 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_max_purge_ops": "8192", 2023-09-23T23:13:17.602 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-09-23T23:13:17.602 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-09-23T23:13:17.602 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_max_snaps_per_dir": "100", 2023-09-23T23:13:17.602 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_max_xattr_pairs_size": "65536", 2023-09-23T23:13:17.602 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_metrics_update_interval": "2", 2023-09-23T23:13:17.602 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_min_caps_per_client": "100", 2023-09-23T23:13:17.602 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_min_caps_working_set": "10000", 2023-09-23T23:13:17.602 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-09-23T23:13:17.603 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_numa_node": "-1", 2023-09-23T23:13:17.603 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-09-23T23:13:17.603 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_op_complaint_time": "30.000000", 2023-09-23T23:13:17.603 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_op_history_duration": "600", 2023-09-23T23:13:17.603 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_op_history_size": "20", 2023-09-23T23:13:17.603 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_op_log_threshold": "5", 2023-09-23T23:13:17.603 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_ping_grace": "15", 2023-09-23T23:13:17.603 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_ping_interval": "5", 2023-09-23T23:13:17.603 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-09-23T23:13:17.603 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-09-23T23:13:17.603 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_recall_max_caps": "30000", 2023-09-23T23:13:17.603 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-09-23T23:13:17.604 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_recall_max_decay_threshold": "131072", 2023-09-23T23:13:17.604 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-09-23T23:13:17.604 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_recall_warning_threshold": "262144", 2023-09-23T23:13:17.604 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_reconnect_timeout": "45.000000", 2023-09-23T23:13:17.604 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_replay_interval": "1.000000", 2023-09-23T23:13:17.604 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-09-23T23:13:17.604 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-09-23T23:13:17.604 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_root_ino_gid": "0", 2023-09-23T23:13:17.604 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_root_ino_uid": "0", 2023-09-23T23:13:17.604 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-09-23T23:13:17.604 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_session_blocklist_on_evict": "true", 2023-09-23T23:13:17.604 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_session_blocklist_on_timeout": "true", 2023-09-23T23:13:17.604 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-09-23T23:13:17.605 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-09-23T23:13:17.605 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-09-23T23:13:17.605 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-09-23T23:13:17.605 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-09-23T23:13:17.605 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-09-23T23:13:17.605 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_shutdown_check": "0", 2023-09-23T23:13:17.605 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_skip_ino": "0", 2023-09-23T23:13:17.605 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_sleep_rank_change": "0.000000", 2023-09-23T23:13:17.605 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_snap_max_uid": "4294967294", 2023-09-23T23:13:17.605 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_snap_min_uid": "0", 2023-09-23T23:13:17.605 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_snap_rstat": "false", 2023-09-23T23:13:17.605 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_standby_replay_damaged": "false", 2023-09-23T23:13:17.606 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_symlink_recovery": "true", 2023-09-23T23:13:17.606 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_task_status_update_interval": "2.000000", 2023-09-23T23:13:17.606 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_thrash_exports": "0", 2023-09-23T23:13:17.606 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_thrash_fragments": "0", 2023-09-23T23:13:17.606 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_tick_interval": "5.000000", 2023-09-23T23:13:17.606 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_valgrind_exit": "false", 2023-09-23T23:13:17.606 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_verify_backtrace": "1", 2023-09-23T23:13:17.606 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_verify_scatter": "false", 2023-09-23T23:13:17.606 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_wipe_ino_prealloc": "false", 2023-09-23T23:13:17.606 INFO:teuthology.orchestra.run.smithi110.stdout: "mds_wipe_sessions": "false", 2023-09-23T23:13:17.606 INFO:teuthology.orchestra.run.smithi110.stdout: "mempool_debug": "false", 2023-09-23T23:13:17.606 INFO:teuthology.orchestra.run.smithi110.stdout: "memstore_debug_omit_block_device_write": "false", 2023-09-23T23:13:17.607 INFO:teuthology.orchestra.run.smithi110.stdout: "memstore_device_bytes": "1073741824", 2023-09-23T23:13:17.607 INFO:teuthology.orchestra.run.smithi110.stdout: "memstore_page_set": "false", 2023-09-23T23:13:17.607 INFO:teuthology.orchestra.run.smithi110.stdout: "memstore_page_size": "65536", 2023-09-23T23:13:17.607 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_client_bytes": "134217728", 2023-09-23T23:13:17.607 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_client_messages": "512", 2023-09-23T23:13:17.609 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-09-23T23:13:17.609 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_connect_retry_interval": "1.000000", 2023-09-23T23:13:17.609 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-09-23T23:13:17.609 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-09-23T23:13:17.609 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_disabled_modules": "", 2023-09-23T23:13:17.609 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-09-23T23:13:17.609 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_max_pg_num_change": "128", 2023-09-23T23:13:17.609 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_mds_bytes": "134217728", 2023-09-23T23:13:17.610 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_mds_messages": "128", 2023-09-23T23:13:17.610 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-09-23T23:13:17.610 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_mon_bytes": "134217728", 2023-09-23T23:13:17.610 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_mon_messages": "128", 2023-09-23T23:13:17.610 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_osd_bytes": "536870912", 2023-09-23T23:13:17.610 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_osd_messages": "8192", 2023-09-23T23:13:17.610 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_pool": "true", 2023-09-23T23:13:17.610 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_service_beacon_grace": "60.000000", 2023-09-23T23:13:17.610 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_standby_modules": "true", 2023-09-23T23:13:17.610 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_stats_period": "5", 2023-09-23T23:13:17.610 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_stats_threshold": "5", 2023-09-23T23:13:17.610 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_tick_period": "2", 2023-09-23T23:13:17.611 INFO:teuthology.orchestra.run.smithi110.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-09-23T23:13:17.611 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_accept_timeout_factor": "2.000000", 2023-09-23T23:13:17.611 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_allow_pool_delete": "true", 2023-09-23T23:13:17.611 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_allow_pool_size_one": "false", 2023-09-23T23:13:17.611 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_auth_validate_all_caps": "true", 2023-09-23T23:13:17.611 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-09-23T23:13:17.611 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-09-23T23:13:17.611 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_client_bytes": "104857600", 2023-09-23T23:13:17.611 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_client_directed_command_retry": "2", 2023-09-23T23:13:17.611 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_client_hunt_interval": "3.000000", 2023-09-23T23:13:17.611 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-09-23T23:13:17.612 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-09-23T23:13:17.612 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-09-23T23:13:17.612 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_client_hunt_parallel": "3", 2023-09-23T23:13:17.612 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_client_log_interval": "1.000000", 2023-09-23T23:13:17.612 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-09-23T23:13:17.612 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_client_ping_interval": "10.000000", 2023-09-23T23:13:17.612 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_client_ping_timeout": "30.000000", 2023-09-23T23:13:17.612 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_clock_drift_allowed": "1.000000", 2023-09-23T23:13:17.612 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-09-23T23:13:17.612 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-09-23T23:13:17.612 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_cluster_log_file_level": "debug", 2023-09-23T23:13:17.612 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_cluster_log_to_file": "false", 2023-09-23T23:13:17.612 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_cluster_log_to_graylog": "false", 2023-09-23T23:13:17.612 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-09-23T23:13:17.613 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-09-23T23:13:17.613 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_cluster_log_to_journald": "true", 2023-09-23T23:13:17.613 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_cluster_log_to_stderr": "false", 2023-09-23T23:13:17.613 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-09-23T23:13:17.613 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-09-23T23:13:17.613 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-09-23T23:13:17.613 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_compact_on_bootstrap": "false", 2023-09-23T23:13:17.613 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_compact_on_start": "false", 2023-09-23T23:13:17.613 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_compact_on_trim": "true", 2023-09-23T23:13:17.613 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_con_tracker_persist_interval": "10", 2023-09-23T23:13:17.613 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_con_tracker_score_halflife": "43200", 2023-09-23T23:13:17.613 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_config_key_max_entry_size": "65536", 2023-09-23T23:13:17.614 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_cpu_threads": "4", 2023-09-23T23:13:17.614 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_crush_min_required_version": "hammer", 2023-09-23T23:13:17.614 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_daemon_bytes": "419430400", 2023-09-23T23:13:17.614 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-09-23T23:13:17.614 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_data_avail_crit": "5", 2023-09-23T23:13:17.614 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_data_avail_warn": "5", 2023-09-23T23:13:17.614 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_data_size_warn": "16106127360", 2023-09-23T23:13:17.614 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_debug_block_osdmap_trim": "false", 2023-09-23T23:13:17.614 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-09-23T23:13:17.614 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_debug_dump_json": "false", 2023-09-23T23:13:17.614 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-09-23T23:13:17.615 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_debug_dump_transactions": "false", 2023-09-23T23:13:17.615 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_debug_extra_checks": "false", 2023-09-23T23:13:17.615 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-09-23T23:13:17.615 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-09-23T23:13:17.615 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_debug_no_require_quincy": "false", 2023-09-23T23:13:17.615 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_debug_no_require_reef": "false", 2023-09-23T23:13:17.615 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-09-23T23:13:17.615 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_delta_reset_interval": "10.000000", 2023-09-23T23:13:17.615 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_dns_srv_name": "ceph-mon", 2023-09-23T23:13:17.615 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_down_mkfs_grace": "60", 2023-09-23T23:13:17.615 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_election_default_strategy": "3", 2023-09-23T23:13:17.615 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_election_timeout": "5.000000", 2023-09-23T23:13:17.616 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-09-23T23:13:17.616 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_elector_ping_divisor": "2", 2023-09-23T23:13:17.616 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_elector_ping_timeout": "2.000000", 2023-09-23T23:13:17.616 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_enable_op_tracker": "true", 2023-09-23T23:13:17.616 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_fake_pool_delete": "false", 2023-09-23T23:13:17.616 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_force_quorum_join": "false", 2023-09-23T23:13:17.616 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_globalid_prealloc": "10000", 2023-09-23T23:13:17.616 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_health_detail_to_clog": "true", 2023-09-23T23:13:17.616 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_health_log_update_period": "5", 2023-09-23T23:13:17.616 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_health_max_detail": "50", 2023-09-23T23:13:17.616 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_health_to_clog": "true", 2023-09-23T23:13:17.616 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_health_to_clog_interval": "600", 2023-09-23T23:13:17.617 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-09-23T23:13:17.617 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_host": "[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] [v2:172.21.15.110:3301/0,v1:172.21.15.110:6790/0]", 2023-09-23T23:13:17.617 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_host_override": "", 2023-09-23T23:13:17.617 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_initial_members": "", 2023-09-23T23:13:17.617 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-09-23T23:13:17.617 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-09-23T23:13:17.617 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-09-23T23:13:17.617 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-09-23T23:13:17.617 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_keyvaluedb": "rocksdb", 2023-09-23T23:13:17.617 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_lease": "5.000000", 2023-09-23T23:13:17.617 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-09-23T23:13:17.617 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-09-23T23:13:17.617 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_log_full_interval": "50", 2023-09-23T23:13:17.617 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_log_max": "10000", 2023-09-23T23:13:17.618 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_log_max_summary": "50", 2023-09-23T23:13:17.618 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_max_log_entries_per_event": "4096", 2023-09-23T23:13:17.618 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_max_log_epochs": "500", 2023-09-23T23:13:17.618 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_max_mdsmap_epochs": "500", 2023-09-23T23:13:17.618 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_max_mgrmap_epochs": "500", 2023-09-23T23:13:17.618 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_max_osd": "10000", 2023-09-23T23:13:17.618 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_max_pg_per_osd": "10000", 2023-09-23T23:13:17.618 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_max_pool_pg_num": "65536", 2023-09-23T23:13:17.618 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-09-23T23:13:17.618 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-09-23T23:13:17.618 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_mds_force_trim_to": "0", 2023-09-23T23:13:17.618 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_mds_skip_sanity": "true", 2023-09-23T23:13:17.619 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_memory_autotune": "true", 2023-09-23T23:13:17.619 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_memory_target": "2147483648", 2023-09-23T23:13:17.619 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_mgr_beacon_grace": "30", 2023-09-23T23:13:17.619 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-09-23T23:13:17.619 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_mgr_digest_period": "5", 2023-09-23T23:13:17.619 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_mgr_inactive_grace": "60", 2023-09-23T23:13:17.619 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_mgr_mkfs_grace": "240", 2023-09-23T23:13:17.619 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-09-23T23:13:17.619 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_min_osdmap_epochs": "500", 2023-09-23T23:13:17.619 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_op_complaint_time": "30", 2023-09-23T23:13:17.619 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_op_history_duration": "600", 2023-09-23T23:13:17.619 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_op_history_size": "20", 2023-09-23T23:13:17.619 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_op_history_slow_op_size": "20", 2023-09-23T23:13:17.620 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_op_history_slow_op_threshold": "10", 2023-09-23T23:13:17.620 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_op_log_threshold": "5", 2023-09-23T23:13:17.620 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-09-23T23:13:17.620 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-09-23T23:13:17.620 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-09-23T23:13:17.620 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_auto_mark_in": "false", 2023-09-23T23:13:17.620 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_auto_mark_new_in": "true", 2023-09-23T23:13:17.620 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-09-23T23:13:17.620 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-09-23T23:13:17.620 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_cache_size": "500", 2023-09-23T23:13:17.620 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_cache_size_min": "134217728", 2023-09-23T23:13:17.620 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_crush_smoke_test": "true", 2023-09-23T23:13:17.620 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_destroyed_out_interval": "600", 2023-09-23T23:13:17.621 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_down_out_interval": "600", 2023-09-23T23:13:17.621 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-09-23T23:13:17.621 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-09-23T23:13:17.621 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_force_trim_to": "0", 2023-09-23T23:13:17.621 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_full_ratio": "0.950000", 2023-09-23T23:13:17.621 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-09-23T23:13:17.621 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_laggy_halflife": "3600", 2023-09-23T23:13:17.621 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_laggy_max_interval": "300", 2023-09-23T23:13:17.621 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_laggy_weight": "0.300000", 2023-09-23T23:13:17.621 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-09-23T23:13:17.621 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_max_creating_pgs": "1024", 2023-09-23T23:13:17.621 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_max_initial_pgs": "1024", 2023-09-23T23:13:17.622 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_min_down_reporters": "2", 2023-09-23T23:13:17.622 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-09-23T23:13:17.622 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-09-23T23:13:17.622 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-09-23T23:13:17.622 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_prime_pg_temp": "true", 2023-09-23T23:13:17.622 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-09-23T23:13:17.622 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-09-23T23:13:17.622 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_report_timeout": "900", 2023-09-23T23:13:17.622 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-09-23T23:13:17.622 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-09-23T23:13:17.622 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_warn_num_repaired": "10", 2023-09-23T23:13:17.622 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osd_warn_op_age": "32.000000", 2023-09-23T23:13:17.623 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-09-23T23:13:17.623 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osdmap_full_prune_interval": "10", 2023-09-23T23:13:17.623 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osdmap_full_prune_min": "10000", 2023-09-23T23:13:17.623 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-09-23T23:13:17.623 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-09-23T23:13:17.623 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_pg_stuck_threshold": "60", 2023-09-23T23:13:17.623 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-09-23T23:13:17.623 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_pg_warn_min_objects": "10000", 2023-09-23T23:13:17.623 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_pg_warn_min_per_osd": "0", 2023-09-23T23:13:17.623 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-09-23T23:13:17.623 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_pool_quota_crit_threshold": "0", 2023-09-23T23:13:17.623 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_pool_quota_warn_threshold": "0", 2023-09-23T23:13:17.624 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_probe_timeout": "2.000000", 2023-09-23T23:13:17.624 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_reweight_max_change": "0.050000", 2023-09-23T23:13:17.624 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_reweight_max_osds": "4", 2023-09-23T23:13:17.624 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-09-23T23:13:17.624 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-09-23T23:13:17.624 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-09-23T23:13:17.624 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-09-23T23:13:17.624 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-09-23T23:13:17.624 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_scrub_interval": "86400", 2023-09-23T23:13:17.624 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_scrub_max_keys": "100", 2023-09-23T23:13:17.624 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_scrub_timeout": "300", 2023-09-23T23:13:17.624 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_session_timeout": "300", 2023-09-23T23:13:17.625 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_smart_report_timeout": "5", 2023-09-23T23:13:17.625 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_stat_smooth_intervals": "6", 2023-09-23T23:13:17.625 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-09-23T23:13:17.625 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_stretch_pool_min_size": "2", 2023-09-23T23:13:17.625 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_stretch_pool_size": "4", 2023-09-23T23:13:17.625 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-09-23T23:13:17.625 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_subscribe_interval": "86400.000000", 2023-09-23T23:13:17.625 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_sync_debug": "false", 2023-09-23T23:13:17.625 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_sync_max_payload_keys": "2000", 2023-09-23T23:13:17.625 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_sync_max_payload_size": "1048576", 2023-09-23T23:13:17.625 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_sync_provider_kill_at": "0", 2023-09-23T23:13:17.625 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_sync_requester_kill_at": "0", 2023-09-23T23:13:17.626 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_sync_timeout": "60.000000", 2023-09-23T23:13:17.626 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_target_pg_per_osd": "100", 2023-09-23T23:13:17.626 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_tick_interval": "5", 2023-09-23T23:13:17.626 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_timecheck_interval": "300.000000", 2023-09-23T23:13:17.626 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-09-23T23:13:17.626 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_use_min_delay_socket": "false", 2023-09-23T23:13:17.626 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_older_version_delay": "604800", 2023-09-23T23:13:17.626 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-09-23T23:13:17.626 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-09-23T23:13:17.626 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-09-23T23:13:17.626 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_on_filestore_osds": "true", 2023-09-23T23:13:17.626 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-09-23T23:13:17.627 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-09-23T23:13:17.627 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-09-23T23:13:17.627 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_on_misplaced": "false", 2023-09-23T23:13:17.627 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-09-23T23:13:17.627 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_on_older_version": "true", 2023-09-23T23:13:17.627 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-09-23T23:13:17.627 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_on_pool_no_app": "true", 2023-09-23T23:13:17.627 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-09-23T23:13:17.627 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-09-23T23:13:17.627 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-09-23T23:13:17.627 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-09-23T23:13:17.627 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_on_too_few_osds": "false", 2023-09-23T23:13:17.628 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-09-23T23:13:17.628 INFO:teuthology.orchestra.run.smithi110.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-09-23T23:13:17.628 INFO:teuthology.orchestra.run.smithi110.stdout: "monmap": "", 2023-09-23T23:13:17.628 INFO:teuthology.orchestra.run.smithi110.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T23:13:17.628 INFO:teuthology.orchestra.run.smithi110.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-09-23T23:13:17.628 INFO:teuthology.orchestra.run.smithi110.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-09-23T23:13:17.628 INFO:teuthology.orchestra.run.smithi110.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T23:13:17.628 INFO:teuthology.orchestra.run.smithi110.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-09-23T23:13:17.628 INFO:teuthology.orchestra.run.smithi110.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-09-23T23:13:17.628 INFO:teuthology.orchestra.run.smithi110.stdout: "motr_tracing_enabled": "false", 2023-09-23T23:13:17.628 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_op_threads": "3", 2023-09-23T23:13:17.629 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_rdma_buffer_size": "131072", 2023-09-23T23:13:17.629 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_rdma_cm": "false", 2023-09-23T23:13:17.629 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_rdma_device_name": "", 2023-09-23T23:13:17.629 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_rdma_dscp": "96", 2023-09-23T23:13:17.629 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-09-23T23:13:17.629 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_rdma_gid_idx": "0", 2023-09-23T23:13:17.629 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_rdma_local_gid": "", 2023-09-23T23:13:17.629 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_rdma_polling_us": "1000", 2023-09-23T23:13:17.629 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_rdma_port_num": "1", 2023-09-23T23:13:17.629 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-09-23T23:13:17.629 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-09-23T23:13:17.629 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_rdma_roce_ver": "1", 2023-09-23T23:13:17.630 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_rdma_send_buffers": "1024", 2023-09-23T23:13:17.630 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_rdma_sl": "3", 2023-09-23T23:13:17.630 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_rdma_support_srq": "true", 2023-09-23T23:13:17.630 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_rdma_type": "ib", 2023-09-23T23:13:17.630 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_async_reap_threshold": "5", 2023-09-23T23:13:17.630 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_bind_before_connect": "false", 2023-09-23T23:13:17.630 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_bind_ipv4": "true", 2023-09-23T23:13:17.630 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_bind_ipv6": "false", 2023-09-23T23:13:17.630 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_bind_msgr1": "true", 2023-09-23T23:13:17.630 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_bind_msgr2": "true", 2023-09-23T23:13:17.630 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_bind_port_max": "7568", 2023-09-23T23:13:17.630 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_bind_port_min": "6800", 2023-09-23T23:13:17.631 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_bind_prefer_ipv4": "false", 2023-09-23T23:13:17.631 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_bind_retry_count": "3", 2023-09-23T23:13:17.631 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_bind_retry_delay": "5", 2023-09-23T23:13:17.631 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_blackhole_client": "false", 2023-09-23T23:13:17.631 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_blackhole_mds": "false", 2023-09-23T23:13:17.631 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_blackhole_mgr": "false", 2023-09-23T23:13:17.631 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_blackhole_mon": "false", 2023-09-23T23:13:17.631 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_blackhole_osd": "false", 2023-09-23T23:13:17.631 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_client_mode": "crc secure", 2023-09-23T23:13:17.631 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_cluster_mode": "crc secure", 2023-09-23T23:13:17.631 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_cluster_type": "", 2023-09-23T23:13:17.631 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_compress_secure": "false", 2023-09-23T23:13:17.632 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_connection_idle_timeout": "900", 2023-09-23T23:13:17.632 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_connection_ready_timeout": "10", 2023-09-23T23:13:17.632 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_crc_data": "true", 2023-09-23T23:13:17.632 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_crc_header": "true", 2023-09-23T23:13:17.632 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_die_on_bad_msg": "false", 2023-09-23T23:13:17.632 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_die_on_bug": "true", 2023-09-23T23:13:17.632 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_die_on_old_message": "true", 2023-09-23T23:13:17.632 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_die_on_skipped_message": "false", 2023-09-23T23:13:17.632 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_die_on_unhandled_msg": "false", 2023-09-23T23:13:17.632 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-09-23T23:13:17.632 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dpdk_coremask": "0xF", 2023-09-23T23:13:17.632 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-09-23T23:13:17.632 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dpdk_devs_allowlist": "", 2023-09-23T23:13:17.633 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dpdk_enable_tso": "true", 2023-09-23T23:13:17.633 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-09-23T23:13:17.633 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-09-23T23:13:17.633 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dpdk_hugepages": "", 2023-09-23T23:13:17.633 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dpdk_hw_flow_control": "true", 2023-09-23T23:13:17.633 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-09-23T23:13:17.633 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dpdk_lro": "true", 2023-09-23T23:13:17.633 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dpdk_memory_channel": "4", 2023-09-23T23:13:17.633 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-09-23T23:13:17.633 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dpdk_pmd": "", 2023-09-23T23:13:17.633 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dpdk_port_id": "0", 2023-09-23T23:13:17.633 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-09-23T23:13:17.634 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dump_corrupt_message_level": "1", 2023-09-23T23:13:17.634 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_dump_on_send": "false", 2023-09-23T23:13:17.634 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_initial_backoff": "0.200000", 2023-09-23T23:13:17.634 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_inject_delay_max": "1.000000", 2023-09-23T23:13:17.634 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_inject_delay_probability": "0.000000", 2023-09-23T23:13:17.634 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_inject_delay_type": "", 2023-09-23T23:13:17.634 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_inject_internal_delays": "0.000000", 2023-09-23T23:13:17.634 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_inject_network_congestion": "0", 2023-09-23T23:13:17.634 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_inject_socket_failures": "0", 2023-09-23T23:13:17.634 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_learn_addr_from_peer": "true", 2023-09-23T23:13:17.634 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_max_accept_failures": "4", 2023-09-23T23:13:17.634 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_max_backoff": "15.000000", 2023-09-23T23:13:17.635 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_mon_client_mode": "secure crc", 2023-09-23T23:13:17.635 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_mon_cluster_mode": "secure crc", 2023-09-23T23:13:17.635 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_mon_service_mode": "secure crc", 2023-09-23T23:13:17.635 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_osd_compress_min_size": "1024", 2023-09-23T23:13:17.635 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_osd_compress_mode": "none", 2023-09-23T23:13:17.635 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_osd_compression_algorithm": "snappy", 2023-09-23T23:13:17.635 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-09-23T23:13:17.635 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_pq_min_cost": "65536", 2023-09-23T23:13:17.635 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_public_type": "", 2023-09-23T23:13:17.635 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_service_mode": "crc secure", 2023-09-23T23:13:17.635 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_tcp_listen_backlog": "512", 2023-09-23T23:13:17.635 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_tcp_nodelay": "true", 2023-09-23T23:13:17.636 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-09-23T23:13:17.636 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_tcp_rcvbuf": "0", 2023-09-23T23:13:17.636 INFO:teuthology.orchestra.run.smithi110.stdout: "ms_type": "async+posix", 2023-09-23T23:13:17.636 INFO:teuthology.orchestra.run.smithi110.stdout: "no_config_file": "false", 2023-09-23T23:13:17.636 INFO:teuthology.orchestra.run.smithi110.stdout: "objecter_completion_locks_per_session": "32", 2023-09-23T23:13:17.636 INFO:teuthology.orchestra.run.smithi110.stdout: "objecter_debug_inject_relock_delay": "false", 2023-09-23T23:13:17.636 INFO:teuthology.orchestra.run.smithi110.stdout: "objecter_inflight_op_bytes": "104857600", 2023-09-23T23:13:17.636 INFO:teuthology.orchestra.run.smithi110.stdout: "objecter_inflight_ops": "1024", 2023-09-23T23:13:17.636 INFO:teuthology.orchestra.run.smithi110.stdout: "objecter_inject_no_watch_ping": "false", 2023-09-23T23:13:17.636 INFO:teuthology.orchestra.run.smithi110.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-09-23T23:13:17.636 INFO:teuthology.orchestra.run.smithi110.stdout: "objecter_tick_interval": "5.000000", 2023-09-23T23:13:17.636 INFO:teuthology.orchestra.run.smithi110.stdout: "objecter_timeout": "10.000000", 2023-09-23T23:13:17.636 INFO:teuthology.orchestra.run.smithi110.stdout: "objectstore_blackhole": "false", 2023-09-23T23:13:17.637 INFO:teuthology.orchestra.run.smithi110.stdout: "openssl_engine_opts": "", 2023-09-23T23:13:17.637 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_agent_delay_time": "5.000000", 2023-09-23T23:13:17.637 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_agent_hist_halflife": "1000", 2023-09-23T23:13:17.637 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_agent_max_low_ops": "2", 2023-09-23T23:13:17.637 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_agent_max_ops": "4", 2023-09-23T23:13:17.637 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-09-23T23:13:17.637 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_agent_quantize_effort": "0.100000", 2023-09-23T23:13:17.637 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_agent_slop": "0.020000", 2023-09-23T23:13:17.637 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-09-23T23:13:17.637 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_allow_recovery_below_min_size": "true", 2023-09-23T23:13:17.637 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_asio_thread_count": "2", 2023-09-23T23:13:17.637 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_async_recovery_min_cost": "100", 2023-09-23T23:13:17.638 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_auto_mark_unfound_lost": "false", 2023-09-23T23:13:17.638 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_backfill_retry_interval": "30.000000", 2023-09-23T23:13:17.638 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_backfill_scan_max": "512", 2023-09-23T23:13:17.638 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_backfill_scan_min": "64", 2023-09-23T23:13:17.638 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_backoff_on_degraded": "false", 2023-09-23T23:13:17.638 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_backoff_on_peering": "false", 2023-09-23T23:13:17.638 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_backoff_on_unfound": "true", 2023-09-23T23:13:17.638 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_beacon_report_interval": "300", 2023-09-23T23:13:17.638 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_bench_duration": "30", 2023-09-23T23:13:17.638 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-09-23T23:13:17.638 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_bench_max_block_size": "67108864", 2023-09-23T23:13:17.638 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_bench_small_size_max_iops": "100", 2023-09-23T23:13:17.639 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_blkin_trace_all": "false", 2023-09-23T23:13:17.639 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_blocked_scrub_grace_period": "120", 2023-09-23T23:13:17.639 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-09-23T23:13:17.639 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-09-23T23:13:17.639 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-09-23T23:13:17.639 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_check_for_log_corruption": "false", 2023-09-23T23:13:17.639 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-09-23T23:13:17.639 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T23:13:17.639 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-09-23T23:13:17.639 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T23:13:17.639 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_class_update_on_start": "true", 2023-09-23T23:13:17.639 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_client_message_cap": "256", 2023-09-23T23:13:17.640 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_client_message_size_cap": "524288000", 2023-09-23T23:13:17.640 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_client_op_priority": "63", 2023-09-23T23:13:17.640 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_client_watch_timeout": "30", 2023-09-23T23:13:17.640 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_command_max_records": "256", 2023-09-23T23:13:17.640 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_compact_on_start": "false", 2023-09-23T23:13:17.640 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-09-23T23:13:17.640 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_crush_chooseleaf_type": "1", 2023-09-23T23:13:17.640 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_crush_initial_weight": "-1.000000", 2023-09-23T23:13:17.640 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_crush_update_on_start": "true", 2023-09-23T23:13:17.640 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_crush_update_weight_set": "true", 2023-09-23T23:13:17.640 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-09-23T23:13:17.640 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-09-23T23:13:17.640 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-09-23T23:13:17.641 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_drop_ping_duration": "0", 2023-09-23T23:13:17.641 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-09-23T23:13:17.641 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_feed_pullee": "-1", 2023-09-23T23:13:17.641 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-09-23T23:13:17.641 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-09-23T23:13:17.641 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-09-23T23:13:17.641 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_misdirected_ops": "false", 2023-09-23T23:13:17.641 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_no_acting_change": "false", 2023-09-23T23:13:17.641 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_no_purge_strays": "false", 2023-09-23T23:13:17.641 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_op_order": "false", 2023-09-23T23:13:17.641 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_pg_log_writeout": "false", 2023-09-23T23:13:17.641 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_pretend_recovery_active": "false", 2023-09-23T23:13:17.641 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-09-23T23:13:17.642 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-09-23T23:13:17.642 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_shutdown": "false", 2023-09-23T23:13:17.642 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-09-23T23:13:17.642 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-09-23T23:13:17.642 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_verify_cached_snaps": "false", 2023-09-23T23:13:17.642 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_verify_missing_on_start": "false", 2023-09-23T23:13:17.642 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_verify_snaps": "false", 2023-09-23T23:13:17.642 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-09-23T23:13:17.642 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-09-23T23:13:17.642 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_deep_scrub_keys": "1024", 2023-09-23T23:13:17.642 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-09-23T23:13:17.643 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-09-23T23:13:17.643 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-09-23T23:13:17.643 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_deep_scrub_stride": "524288", 2023-09-23T23:13:17.643 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-09-23T23:13:17.643 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_default_data_pool_replay_window": "45", 2023-09-23T23:13:17.643 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_default_notify_timeout": "30", 2023-09-23T23:13:17.643 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_delete_sleep": "0.000000", 2023-09-23T23:13:17.643 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-09-23T23:13:17.643 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-09-23T23:13:17.643 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-09-23T23:13:17.643 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_discard_disconnected_ops": "true", 2023-09-23T23:13:17.643 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_enable_op_tracker": "true", 2023-09-23T23:13:17.643 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-09-23T23:13:17.644 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_extblkdev_plugins": "vdo", 2023-09-23T23:13:17.644 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-09-23T23:13:17.644 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-09-23T23:13:17.644 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_fast_info": "true", 2023-09-23T23:13:17.644 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_fast_shutdown": "true", 2023-09-23T23:13:17.644 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-09-23T23:13:17.644 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_fast_shutdown_timeout": "15", 2023-09-23T23:13:17.644 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-09-23T23:13:17.644 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-09-23T23:13:17.644 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-09-23T23:13:17.644 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_function_tracing": "false", 2023-09-23T23:13:17.644 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_heartbeat_grace": "20", 2023-09-23T23:13:17.645 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_heartbeat_interval": "6", 2023-09-23T23:13:17.645 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-09-23T23:13:17.645 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_heartbeat_min_peers": "10", 2023-09-23T23:13:17.645 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_heartbeat_min_size": "2000", 2023-09-23T23:13:17.645 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_heartbeat_stale": "600", 2023-09-23T23:13:17.645 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-09-23T23:13:17.645 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_hit_set_max_size": "100000", 2023-09-23T23:13:17.645 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_hit_set_min_size": "1000", 2023-09-23T23:13:17.645 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-09-23T23:13:17.645 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-09-23T23:13:17.645 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-09-23T23:13:17.645 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-09-23T23:13:17.645 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-09-23T23:13:17.646 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_journal_flush_on_shutdown": "true", 2023-09-23T23:13:17.646 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_journal_size": "5120", 2023-09-23T23:13:17.646 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_kill_backfill_at": "0", 2023-09-23T23:13:17.646 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_loop_before_reset_tphandle": "64", 2023-09-23T23:13:17.646 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_map_cache_size": "50", 2023-09-23T23:13:17.646 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_map_dedup": "true", 2023-09-23T23:13:17.646 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_map_message_max": "40", 2023-09-23T23:13:17.646 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_map_message_max_bytes": "10485760", 2023-09-23T23:13:17.646 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_map_share_max_epochs": "40", 2023-09-23T23:13:17.646 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_attr_name_len": "100", 2023-09-23T23:13:17.646 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_attr_size": "0", 2023-09-23T23:13:17.646 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_backfills": "1", 2023-09-23T23:13:17.647 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_markdown_count": "5", 2023-09-23T23:13:17.647 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_markdown_period": "600", 2023-09-23T23:13:17.647 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_object_name_len": "2048", 2023-09-23T23:13:17.647 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_object_namespace_len": "256", 2023-09-23T23:13:17.647 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_object_size": "134217728", 2023-09-23T23:13:17.647 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-09-23T23:13:17.647 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_omap_entries_per_request": "1024", 2023-09-23T23:13:17.647 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_pg_blocked_by": "16", 2023-09-23T23:13:17.647 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_pg_log_entries": "10000", 2023-09-23T23:13:17.647 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-09-23T23:13:17.647 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_pgls": "1024", 2023-09-23T23:13:17.647 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_push_cost": "8388608", 2023-09-23T23:13:17.647 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_push_objects": "10", 2023-09-23T23:13:17.648 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_scrubs": "1", 2023-09-23T23:13:17.648 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-09-23T23:13:17.648 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_trimming_pgs": "2", 2023-09-23T23:13:17.648 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_write_op_reply_len": "64", 2023-09-23T23:13:17.648 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_max_write_size": "90", 2023-09-23T23:13:17.648 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-09-23T23:13:17.648 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-09-23T23:13:17.648 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-09-23T23:13:17.648 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-09-23T23:13:17.648 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-09-23T23:13:17.648 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-09-23T23:13:17.648 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-09-23T23:13:17.648 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_override_recovery_settings": "false", 2023-09-23T23:13:17.649 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_profile": "balanced", 2023-09-23T23:13:17.649 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-09-23T23:13:17.649 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-09-23T23:13:17.649 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-09-23T23:13:17.649 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-09-23T23:13:17.649 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-09-23T23:13:17.649 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-09-23T23:13:17.649 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-09-23T23:13:17.649 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-09-23T23:13:17.649 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-09-23T23:13:17.649 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-09-23T23:13:17.649 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mclock_skip_benchmark": "false", 2023-09-23T23:13:17.650 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_memory_base": "805306368", 2023-09-23T23:13:17.650 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_memory_cache_min": "134217728", 2023-09-23T23:13:17.650 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-09-23T23:13:17.650 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-09-23T23:13:17.650 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_memory_target": "4294967296", 2023-09-23T23:13:17.650 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_memory_target_autotune": "false", 2023-09-23T23:13:17.650 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-09-23T23:13:17.650 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_min_pg_log_entries": "250", 2023-09-23T23:13:17.650 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_min_recovery_priority": "0", 2023-09-23T23:13:17.650 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mon_heartbeat_interval": "30", 2023-09-23T23:13:17.650 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-09-23T23:13:17.650 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mon_report_interval": "5", 2023-09-23T23:13:17.651 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mon_report_max_in_flight": "2", 2023-09-23T23:13:17.651 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-09-23T23:13:17.651 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_num_cache_shards": "32", 2023-09-23T23:13:17.651 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_num_op_tracker_shard": "32", 2023-09-23T23:13:17.651 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_numa_auto_affinity": "true", 2023-09-23T23:13:17.651 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_numa_node": "-1", 2023-09-23T23:13:17.651 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_numa_prefer_iface": "true", 2023-09-23T23:13:17.651 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-09-23T23:13:17.651 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_objecter_finishers": "1", 2023-09-23T23:13:17.651 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_objectstore": "bluestore", 2023-09-23T23:13:17.651 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_objectstore_fuse": "false", 2023-09-23T23:13:17.651 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_objectstore_tracing": "false", 2023-09-23T23:13:17.652 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_complaint_time": "30.000000", 2023-09-23T23:13:17.652 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_history_duration": "600", 2023-09-23T23:13:17.652 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_history_size": "20", 2023-09-23T23:13:17.652 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_history_slow_op_size": "20", 2023-09-23T23:13:17.652 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-09-23T23:13:17.652 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_log_threshold": "5", 2023-09-23T23:13:17.652 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_num_shards": "0", 2023-09-23T23:13:17.652 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_num_shards_hdd": "5", 2023-09-23T23:13:17.652 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_num_shards_ssd": "8", 2023-09-23T23:13:17.652 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_num_threads_per_shard": "0", 2023-09-23T23:13:17.652 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-09-23T23:13:17.652 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-09-23T23:13:17.653 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-09-23T23:13:17.653 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_pq_min_cost": "65536", 2023-09-23T23:13:17.653 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_queue": "mclock_scheduler", 2023-09-23T23:13:17.653 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_queue_cut_off": "high", 2023-09-23T23:13:17.653 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_thread_suicide_timeout": "150", 2023-09-23T23:13:17.653 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_op_thread_timeout": "15", 2023-09-23T23:13:17.653 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_open_classes_on_start": "true", 2023-09-23T23:13:17.653 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_os_flags": "0", 2023-09-23T23:13:17.653 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_peering_op_priority": "255", 2023-09-23T23:13:17.653 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pg_delete_cost": "1048576", 2023-09-23T23:13:17.653 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pg_delete_priority": "5", 2023-09-23T23:13:17.653 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-09-23T23:13:17.654 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-09-23T23:13:17.654 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pg_log_dups_tracked": "3000", 2023-09-23T23:13:17.654 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pg_log_trim_max": "10000", 2023-09-23T23:13:17.654 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pg_log_trim_min": "100", 2023-09-23T23:13:17.654 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-09-23T23:13:17.654 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pg_object_context_cache_count": "64", 2023-09-23T23:13:17.654 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pg_stat_report_interval_max": "500", 2023-09-23T23:13:17.654 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-09-23T23:13:17.654 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-09-23T23:13:17.654 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-09-23T23:13:17.654 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-09-23T23:13:17.654 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-09-23T23:13:17.655 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-09-23T23:13:17.655 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_crimson": "false", 2023-09-23T23:13:17.655 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_crush_rule": "-1", 2023-09-23T23:13:17.655 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_ec_fast_read": "false", 2023-09-23T23:13:17.655 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-09-23T23:13:17.655 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_flag_bulk": "false", 2023-09-23T23:13:17.655 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-09-23T23:13:17.655 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_flag_nodelete": "false", 2023-09-23T23:13:17.655 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-09-23T23:13:17.655 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-09-23T23:13:17.655 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_flags": "0", 2023-09-23T23:13:17.655 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-09-23T23:13:17.656 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_min_size": "0", 2023-09-23T23:13:17.656 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-09-23T23:13:17.656 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_pg_num": "32", 2023-09-23T23:13:17.656 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_pgp_num": "0", 2023-09-23T23:13:17.656 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-09-23T23:13:17.656 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_size": "3", 2023-09-23T23:13:17.656 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_default_type": "replicated", 2023-09-23T23:13:17.656 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-09-23T23:13:17.656 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_pool_use_gmt_hitset": "true", 2023-09-23T23:13:17.656 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_push_per_object_cost": "1000", 2023-09-23T23:13:17.656 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_read_ec_check_for_errors": "false", 2023-09-23T23:13:17.656 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recover_clone_overlap": "true", 2023-09-23T23:13:17.656 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recover_clone_overlap_limit": "10", 2023-09-23T23:13:17.657 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recovery_cost": "20971520", 2023-09-23T23:13:17.657 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recovery_delay_start": "0.000000", 2023-09-23T23:13:17.657 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recovery_max_active": "0", 2023-09-23T23:13:17.657 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recovery_max_active_hdd": "3", 2023-09-23T23:13:17.657 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recovery_max_active_ssd": "10", 2023-09-23T23:13:17.657 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recovery_max_chunk": "8388608", 2023-09-23T23:13:17.657 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-09-23T23:13:17.657 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recovery_max_single_start": "1", 2023-09-23T23:13:17.657 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recovery_op_priority": "3", 2023-09-23T23:13:17.657 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recovery_op_warn_multiple": "16", 2023-09-23T23:13:17.657 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recovery_priority": "5", 2023-09-23T23:13:17.657 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recovery_retry_interval": "30.000000", 2023-09-23T23:13:17.658 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recovery_sleep": "0.000000", 2023-09-23T23:13:17.658 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-09-23T23:13:17.658 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-09-23T23:13:17.658 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-09-23T23:13:17.658 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_repair_during_recovery": "false", 2023-09-23T23:13:17.658 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_requested_scrub_priority": "120", 2023-09-23T23:13:17.658 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-09-23T23:13:17.658 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_rollback_to_cluster_snap": "", 2023-09-23T23:13:17.658 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_auto_repair": "false", 2023-09-23T23:13:17.658 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-09-23T23:13:17.658 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-09-23T23:13:17.658 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_begin_hour": "0", 2023-09-23T23:13:17.659 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_begin_week_day": "0", 2023-09-23T23:13:17.659 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_chunk_max": "25", 2023-09-23T23:13:17.659 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_chunk_min": "5", 2023-09-23T23:13:17.659 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_cost": "52428800", 2023-09-23T23:13:17.659 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_during_recovery": "false", 2023-09-23T23:13:17.659 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_end_hour": "0", 2023-09-23T23:13:17.659 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_end_week_day": "0", 2023-09-23T23:13:17.659 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_event_cost": "4096", 2023-09-23T23:13:17.659 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-09-23T23:13:17.659 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-09-23T23:13:17.659 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_invalid_stats": "true", 2023-09-23T23:13:17.659 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_load_threshold": "0.500000", 2023-09-23T23:13:17.660 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_max_interval": "604800.000000", 2023-09-23T23:13:17.660 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_max_preemptions": "5", 2023-09-23T23:13:17.660 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_min_interval": "86400.000000", 2023-09-23T23:13:17.660 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_priority": "5", 2023-09-23T23:13:17.660 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_reservation_timeout": "5000", 2023-09-23T23:13:17.660 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_sleep": "0.000000", 2023-09-23T23:13:17.660 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-09-23T23:13:17.660 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-09-23T23:13:17.660 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-09-23T23:13:17.660 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_shutdown_pgref_assert": "false", 2023-09-23T23:13:17.660 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_skip_data_digest": "false", 2023-09-23T23:13:17.660 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_smart_report_timeout": "5", 2023-09-23T23:13:17.660 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_snap_trim_cost": "1048576", 2023-09-23T23:13:17.661 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_snap_trim_priority": "5", 2023-09-23T23:13:17.661 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_snap_trim_sleep": "0.000000", 2023-09-23T23:13:17.661 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-09-23T23:13:17.661 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-09-23T23:13:17.661 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-09-23T23:13:17.661 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-09-23T23:13:17.661 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_stats_update_period_scrubbing": "15", 2023-09-23T23:13:17.661 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-09-23T23:13:17.661 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_target_transaction_size": "30", 2023-09-23T23:13:17.661 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-09-23T23:13:17.661 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-09-23T23:13:17.661 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-09-23T23:13:17.662 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-09-23T23:13:17.662 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-09-23T23:13:17.662 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-09-23T23:13:17.662 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-09-23T23:13:17.662 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_tier_default_cache_mode": "writeback", 2023-09-23T23:13:17.662 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-09-23T23:13:17.662 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-09-23T23:13:17.662 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_tracing": "false", 2023-09-23T23:13:17.662 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_use_stale_snap": "false", 2023-09-23T23:13:17.662 INFO:teuthology.orchestra.run.smithi110.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-09-23T23:13:17.662 INFO:teuthology.orchestra.run.smithi110.stdout: "osdc_blkin_trace_all": "false", 2023-09-23T23:13:17.662 INFO:teuthology.orchestra.run.smithi110.stdout: "paxos_kill_at": "0", 2023-09-23T23:13:17.663 INFO:teuthology.orchestra.run.smithi110.stdout: "paxos_max_join_drift": "10", 2023-09-23T23:13:17.663 INFO:teuthology.orchestra.run.smithi110.stdout: "paxos_min": "500", 2023-09-23T23:13:17.663 INFO:teuthology.orchestra.run.smithi110.stdout: "paxos_min_wait": "0.050000", 2023-09-23T23:13:17.663 INFO:teuthology.orchestra.run.smithi110.stdout: "paxos_propose_interval": "1.000000", 2023-09-23T23:13:17.663 INFO:teuthology.orchestra.run.smithi110.stdout: "paxos_service_trim_max": "500", 2023-09-23T23:13:17.663 INFO:teuthology.orchestra.run.smithi110.stdout: "paxos_service_trim_max_multiplier": "20", 2023-09-23T23:13:17.663 INFO:teuthology.orchestra.run.smithi110.stdout: "paxos_service_trim_min": "250", 2023-09-23T23:13:17.663 INFO:teuthology.orchestra.run.smithi110.stdout: "paxos_stash_full_interval": "25", 2023-09-23T23:13:17.663 INFO:teuthology.orchestra.run.smithi110.stdout: "paxos_trim_max": "500", 2023-09-23T23:13:17.663 INFO:teuthology.orchestra.run.smithi110.stdout: "paxos_trim_min": "250", 2023-09-23T23:13:17.663 INFO:teuthology.orchestra.run.smithi110.stdout: "perf": "true", 2023-09-23T23:13:17.663 INFO:teuthology.orchestra.run.smithi110.stdout: "pid_file": "", 2023-09-23T23:13:17.663 INFO:teuthology.orchestra.run.smithi110.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-09-23T23:13:17.664 INFO:teuthology.orchestra.run.smithi110.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-09-23T23:13:17.664 INFO:teuthology.orchestra.run.smithi110.stdout: "public_addr": "-", 2023-09-23T23:13:17.664 INFO:teuthology.orchestra.run.smithi110.stdout: "public_addrv": "", 2023-09-23T23:13:17.664 INFO:teuthology.orchestra.run.smithi110.stdout: "public_bind_addr": "-", 2023-09-23T23:13:17.664 INFO:teuthology.orchestra.run.smithi110.stdout: "public_network": "172.21.0.0/20", 2023-09-23T23:13:17.664 INFO:teuthology.orchestra.run.smithi110.stdout: "public_network_interface": "", 2023-09-23T23:13:17.664 INFO:teuthology.orchestra.run.smithi110.stdout: "qat_compressor_enabled": "false", 2023-09-23T23:13:17.664 INFO:teuthology.orchestra.run.smithi110.stdout: "qat_compressor_session_max_number": "256", 2023-09-23T23:13:17.664 INFO:teuthology.orchestra.run.smithi110.stdout: "rados_mon_op_timeout": "0", 2023-09-23T23:13:17.664 INFO:teuthology.orchestra.run.smithi110.stdout: "rados_osd_op_timeout": "0", 2023-09-23T23:13:17.664 INFO:teuthology.orchestra.run.smithi110.stdout: "rados_tracing": "false", 2023-09-23T23:13:17.664 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_atime_update_interval": "60", 2023-09-23T23:13:17.665 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-09-23T23:13:17.665 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_balance_parent_reads": "false", 2023-09-23T23:13:17.665 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_balance_snap_reads": "false", 2023-09-23T23:13:17.665 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_blkin_trace_all": "false", 2023-09-23T23:13:17.665 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_blocklist_expire_seconds": "0", 2023-09-23T23:13:17.665 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_blocklist_on_break_lock": "true", 2023-09-23T23:13:17.665 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_cache": "true", 2023-09-23T23:13:17.665 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_cache_block_writes_upfront": "false", 2023-09-23T23:13:17.665 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_cache_max_dirty": "25165824", 2023-09-23T23:13:17.665 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-09-23T23:13:17.665 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_cache_max_dirty_object": "0", 2023-09-23T23:13:17.665 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_cache_policy": "writearound", 2023-09-23T23:13:17.666 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_cache_size": "33554432", 2023-09-23T23:13:17.666 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_cache_target_dirty": "16777216", 2023-09-23T23:13:17.666 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-09-23T23:13:17.666 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_clone_copy_on_read": "false", 2023-09-23T23:13:17.666 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_compression_hint": "none", 2023-09-23T23:13:17.666 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_concurrent_management_ops": "10", 2023-09-23T23:13:17.666 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-09-23T23:13:17.666 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_default_clone_format": "auto", 2023-09-23T23:13:17.666 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_default_data_pool": "", 2023-09-23T23:13:17.666 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_default_features": "61", 2023-09-23T23:13:17.666 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_default_format": "2", 2023-09-23T23:13:17.666 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_default_map_options": "", 2023-09-23T23:13:17.667 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_default_order": "22", 2023-09-23T23:13:17.667 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_default_pool": "rbd", 2023-09-23T23:13:17.667 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-09-23T23:13:17.667 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_default_stripe_count": "0", 2023-09-23T23:13:17.667 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_default_stripe_unit": "0", 2023-09-23T23:13:17.667 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_disable_zero_copy_writes": "true", 2023-09-23T23:13:17.667 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_discard_granularity_bytes": "65536", 2023-09-23T23:13:17.667 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-09-23T23:13:17.667 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_enable_alloc_hint": "true", 2023-09-23T23:13:17.667 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-09-23T23:13:17.667 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_io_scheduler": "simple", 2023-09-23T23:13:17.667 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-09-23T23:13:17.667 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_journal_commit_age": "5.000000", 2023-09-23T23:13:17.668 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-09-23T23:13:17.668 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-09-23T23:13:17.668 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-09-23T23:13:17.668 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-09-23T23:13:17.668 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_journal_object_flush_interval": "0", 2023-09-23T23:13:17.668 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-09-23T23:13:17.668 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-09-23T23:13:17.668 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_journal_order": "24", 2023-09-23T23:13:17.668 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_journal_pool": "", 2023-09-23T23:13:17.668 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_journal_splay_width": "4", 2023-09-23T23:13:17.668 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_localize_parent_reads": "false", 2023-09-23T23:13:17.668 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_localize_snap_reads": "false", 2023-09-23T23:13:17.669 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-09-23T23:13:17.669 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-09-23T23:13:17.669 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-09-23T23:13:17.669 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-09-23T23:13:17.669 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-09-23T23:13:17.669 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-09-23T23:13:17.669 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_image_policy_type": "simple", 2023-09-23T23:13:17.669 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-09-23T23:13:17.669 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-09-23T23:13:17.669 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-09-23T23:13:17.669 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-09-23T23:13:17.669 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-09-23T23:13:17.669 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-09-23T23:13:17.669 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-09-23T23:13:17.670 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_memory_autotune": "true", 2023-09-23T23:13:17.670 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_memory_base": "805306368", 2023-09-23T23:13:17.670 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-09-23T23:13:17.670 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-09-23T23:13:17.670 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-09-23T23:13:17.670 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-09-23T23:13:17.670 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_memory_target": "4294967296", 2023-09-23T23:13:17.670 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-09-23T23:13:17.670 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-09-23T23:13:17.670 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-09-23T23:13:17.670 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirroring_delete_delay": "0", 2023-09-23T23:13:17.670 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-09-23T23:13:17.671 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirroring_replay_delay": "0", 2023-09-23T23:13:17.671 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-09-23T23:13:17.671 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-09-23T23:13:17.671 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_move_to_trash_on_remove": "false", 2023-09-23T23:13:17.671 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-09-23T23:13:17.671 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_mtime_update_interval": "60", 2023-09-23T23:13:17.671 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_non_blocking_aio": "true", 2023-09-23T23:13:17.671 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_op_thread_timeout": "60", 2023-09-23T23:13:17.671 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_op_threads": "1", 2023-09-23T23:13:17.671 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_parent_cache_enabled": "false", 2023-09-23T23:13:17.671 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_persistent_cache_mode": "disabled", 2023-09-23T23:13:17.671 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_persistent_cache_path": "/tmp", 2023-09-23T23:13:17.671 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_persistent_cache_size": "1073741824", 2023-09-23T23:13:17.672 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_plugins": "", 2023-09-23T23:13:17.672 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_bps_burst": "0", 2023-09-23T23:13:17.672 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-09-23T23:13:17.672 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_bps_limit": "0", 2023-09-23T23:13:17.672 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_exclude_ops": "0", 2023-09-23T23:13:17.672 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_iops_burst": "0", 2023-09-23T23:13:17.672 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-09-23T23:13:17.672 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_iops_limit": "0", 2023-09-23T23:13:17.672 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_read_bps_burst": "0", 2023-09-23T23:13:17.672 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-09-23T23:13:17.672 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_read_bps_limit": "0", 2023-09-23T23:13:17.672 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_read_iops_burst": "0", 2023-09-23T23:13:17.673 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-09-23T23:13:17.673 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_read_iops_limit": "0", 2023-09-23T23:13:17.673 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_schedule_tick_min": "50", 2023-09-23T23:13:17.673 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_write_bps_burst": "0", 2023-09-23T23:13:17.673 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-09-23T23:13:17.673 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_write_bps_limit": "0", 2023-09-23T23:13:17.673 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_write_iops_burst": "0", 2023-09-23T23:13:17.673 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-09-23T23:13:17.673 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_qos_write_iops_limit": "0", 2023-09-23T23:13:17.673 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_quiesce_notification_attempts": "10", 2023-09-23T23:13:17.673 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_read_from_replica_policy": "default", 2023-09-23T23:13:17.673 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-09-23T23:13:17.674 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_readahead_max_bytes": "524288", 2023-09-23T23:13:17.674 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_readahead_trigger_requests": "10", 2023-09-23T23:13:17.674 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_request_timed_out_seconds": "30", 2023-09-23T23:13:17.674 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_skip_partial_discard": "true", 2023-09-23T23:13:17.674 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-09-23T23:13:17.674 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_tracing": "false", 2023-09-23T23:13:17.674 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_validate_names": "true", 2023-09-23T23:13:17.674 INFO:teuthology.orchestra.run.smithi110.stdout: "rbd_validate_pool": "true", 2023-09-23T23:13:17.674 INFO:teuthology.orchestra.run.smithi110.stdout: "restapi_base_url": "", 2023-09-23T23:13:17.674 INFO:teuthology.orchestra.run.smithi110.stdout: "restapi_log_level": "", 2023-09-23T23:13:17.674 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_acl_grants_max_num": "100", 2023-09-23T23:13:17.674 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_admin_entry": "admin", 2023-09-23T23:13:17.675 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-09-23T23:13:17.675 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_backend_store": "rados", 2023-09-23T23:13:17.675 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_barbican_url": "", 2023-09-23T23:13:17.675 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_beast_enable_async": "true", 2023-09-23T23:13:17.675 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-09-23T23:13:17.675 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-09-23T23:13:17.675 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_bucket_index_max_aio": "128", 2023-09-23T23:13:17.675 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-09-23T23:13:17.675 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-09-23T23:13:17.675 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_bucket_quota_ttl": "600", 2023-09-23T23:13:17.675 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-09-23T23:13:17.675 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_cache_enabled": "true", 2023-09-23T23:13:17.676 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_cache_expiry_interval": "900", 2023-09-23T23:13:17.676 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_cache_lru_size": "10000", 2023-09-23T23:13:17.676 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_config_store": "rados", 2023-09-23T23:13:17.676 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_content_length_compat": "false", 2023-09-23T23:13:17.676 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_copy_obj_progress": "true", 2023-09-23T23:13:17.676 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-09-23T23:13:17.676 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_cors_rules_max_num": "100", 2023-09-23T23:13:17.676 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_cross_domain_policy": "", 2023-09-23T23:13:17.676 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_default_encryption_key": "", 2023-09-23T23:13:17.676 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_kmip_addr": "", 2023-09-23T23:13:17.676 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_kmip_ca_path": "", 2023-09-23T23:13:17.676 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_kmip_client_cert": "", 2023-09-23T23:13:17.677 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_kmip_client_key": "", 2023-09-23T23:13:17.677 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-09-23T23:13:17.677 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_kmip_password": "", 2023-09-23T23:13:17.677 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-09-23T23:13:17.677 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_kmip_username": "", 2023-09-23T23:13:17.677 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_require_ssl": "true", 2023-09-23T23:13:17.677 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-09-23T23:13:17.677 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-09-23T23:13:17.677 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-09-23T23:13:17.677 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-09-23T23:13:17.677 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-09-23T23:13:17.677 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-09-23T23:13:17.678 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-09-23T23:13:17.678 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-09-23T23:13:17.678 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-09-23T23:13:17.678 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-09-23T23:13:17.678 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-09-23T23:13:17.678 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-09-23T23:13:17.678 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-09-23T23:13:17.678 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-09-23T23:13:17.678 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_suppress_logs": "true", 2023-09-23T23:13:17.678 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_vault_addr": "", 2023-09-23T23:13:17.678 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_vault_auth": "token", 2023-09-23T23:13:17.678 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_vault_namespace": "", 2023-09-23T23:13:17.678 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_vault_prefix": "", 2023-09-23T23:13:17.679 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-09-23T23:13:17.679 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-09-23T23:13:17.679 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-09-23T23:13:17.679 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-09-23T23:13:17.679 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_vault_token_file": "", 2023-09-23T23:13:17.679 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-09-23T23:13:17.679 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_curl_buffersize": "524288", 2023-09-23T23:13:17.679 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_curl_low_speed_limit": "1024", 2023-09-23T23:13:17.679 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_curl_low_speed_time": "300", 2023-09-23T23:13:17.679 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_curl_tcp_keepalive": "0", 2023-09-23T23:13:17.679 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-09-23T23:13:17.679 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-09-23T23:13:17.680 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-09-23T23:13:17.680 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-09-23T23:13:17.680 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-09-23T23:13:17.680 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_d3n_l1_fadvise": "4", 2023-09-23T23:13:17.680 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-09-23T23:13:17.680 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-09-23T23:13:17.680 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-09-23T23:13:17.680 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-09-23T23:13:17.680 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_data_log_changes_size": "1000", 2023-09-23T23:13:17.680 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_data_log_num_shards": "128", 2023-09-23T23:13:17.680 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-09-23T23:13:17.680 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_data_log_window": "30", 2023-09-23T23:13:17.681 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_data_notify_interval_msec": "0", 2023-09-23T23:13:17.681 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_data_sync_poll_interval": "20", 2023-09-23T23:13:17.681 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_data_sync_spawn_window": "20", 2023-09-23T23:13:17.681 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-09-23T23:13:17.681 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-09-23T23:13:17.681 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_default_data_log_backing": "fifo", 2023-09-23T23:13:17.681 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-09-23T23:13:17.681 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_default_region_info_oid": "default.region", 2023-09-23T23:13:17.681 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-09-23T23:13:17.681 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-09-23T23:13:17.681 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_defer_to_bucket_acls": "", 2023-09-23T23:13:17.681 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-09-23T23:13:17.682 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-09-23T23:13:17.682 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-09-23T23:13:17.682 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-09-23T23:13:17.682 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-09-23T23:13:17.682 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-09-23T23:13:17.682 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-09-23T23:13:17.682 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-09-23T23:13:17.682 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_dmclock_data_res": "500.000000", 2023-09-23T23:13:17.682 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-09-23T23:13:17.682 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-09-23T23:13:17.682 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-09-23T23:13:17.682 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-09-23T23:13:17.683 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_dns_name": "", 2023-09-23T23:13:17.683 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_dns_s3website_name": "", 2023-09-23T23:13:17.683 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_dynamic_resharding": "true", 2023-09-23T23:13:17.683 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-09-23T23:13:17.683 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_enable_gc_threads": "true", 2023-09-23T23:13:17.683 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_enable_lc_threads": "true", 2023-09-23T23:13:17.683 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_enable_ops_log": "false", 2023-09-23T23:13:17.683 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_enable_quota_threads": "true", 2023-09-23T23:13:17.683 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_enable_static_website": "false", 2023-09-23T23:13:17.683 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_enable_usage_log": "false", 2023-09-23T23:13:17.683 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_enforce_swift_acls": "true", 2023-09-23T23:13:17.683 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_exit_timeout_secs": "120", 2023-09-23T23:13:17.683 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_expose_bucket": "false", 2023-09-23T23:13:17.684 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_extended_http_attrs": "", 2023-09-23T23:13:17.684 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_filter": "none", 2023-09-23T23:13:17.684 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-09-23T23:13:17.684 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_frontends": "beast port=7480", 2023-09-23T23:13:17.684 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_gc_max_concurrent_io": "10", 2023-09-23T23:13:17.684 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_gc_max_deferred": "50", 2023-09-23T23:13:17.684 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-09-23T23:13:17.684 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_gc_max_objs": "32", 2023-09-23T23:13:17.684 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_gc_max_queue_size": "134213632", 2023-09-23T23:13:17.684 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_gc_max_trim_chunk": "16", 2023-09-23T23:13:17.684 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_gc_obj_min_wait": "7200", 2023-09-23T23:13:17.684 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_gc_processor_max_time": "3600", 2023-09-23T23:13:17.685 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_gc_processor_period": "3600", 2023-09-23T23:13:17.685 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-09-23T23:13:17.685 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_get_obj_window_size": "16777216", 2023-09-23T23:13:17.685 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_healthcheck_disabling_path": "", 2023-09-23T23:13:17.685 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_ignore_get_invalid_range": "false", 2023-09-23T23:13:17.685 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_init_timeout": "300", 2023-09-23T23:13:17.685 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-09-23T23:13:17.685 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-09-23T23:13:17.685 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-09-23T23:13:17.685 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-09-23T23:13:17.685 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_admin_domain": "", 2023-09-23T23:13:17.685 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_admin_password": "", 2023-09-23T23:13:17.685 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_admin_password_path": "", 2023-09-23T23:13:17.686 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_admin_project": "", 2023-09-23T23:13:17.686 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_admin_tenant": "", 2023-09-23T23:13:17.686 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_admin_token": "", 2023-09-23T23:13:17.686 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_admin_token_path": "", 2023-09-23T23:13:17.686 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_admin_user": "", 2023-09-23T23:13:17.686 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_api_version": "2", 2023-09-23T23:13:17.686 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_barbican_domain": "", 2023-09-23T23:13:17.686 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_barbican_password": "", 2023-09-23T23:13:17.686 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_barbican_project": "", 2023-09-23T23:13:17.686 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_barbican_tenant": "", 2023-09-23T23:13:17.686 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_barbican_user": "", 2023-09-23T23:13:17.686 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-09-23T23:13:17.687 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_implicit_tenants": "false", 2023-09-23T23:13:17.687 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-09-23T23:13:17.687 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_service_token_enabled": "false", 2023-09-23T23:13:17.687 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_token_cache_size": "10000", 2023-09-23T23:13:17.687 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_url": "", 2023-09-23T23:13:17.687 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_keystone_verify_ssl": "true", 2023-09-23T23:13:17.687 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_lc_debug_interval": "-1", 2023-09-23T23:13:17.687 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_lc_lock_max_time": "90", 2023-09-23T23:13:17.687 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_lc_max_objs": "32", 2023-09-23T23:13:17.687 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_lc_max_rules": "1000", 2023-09-23T23:13:17.687 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_lc_max_worker": "3", 2023-09-23T23:13:17.687 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_lc_max_wp_worker": "3", 2023-09-23T23:13:17.688 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_lc_thread_delay": "0", 2023-09-23T23:13:17.688 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-09-23T23:13:17.688 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_ldap_dnattr": "uid", 2023-09-23T23:13:17.688 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-09-23T23:13:17.688 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_ldap_searchfilter": "", 2023-09-23T23:13:17.688 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-09-23T23:13:17.688 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_ldap_uri": "ldaps://", 2023-09-23T23:13:17.688 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-09-23T23:13:17.688 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-09-23T23:13:17.688 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-09-23T23:13:17.688 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_log_http_headers": "", 2023-09-23T23:13:17.688 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_log_nonexistent_bucket": "false", 2023-09-23T23:13:17.689 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-09-23T23:13:17.689 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_log_object_name_utc": "false", 2023-09-23T23:13:17.689 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-09-23T23:13:17.689 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_max_attr_name_len": "0", 2023-09-23T23:13:17.689 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_max_attr_size": "0", 2023-09-23T23:13:17.689 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_max_attrs_num_in_req": "0", 2023-09-23T23:13:17.689 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_max_chunk_size": "4194304", 2023-09-23T23:13:17.689 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_max_concurrent_requests": "1024", 2023-09-23T23:13:17.689 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-09-23T23:13:17.689 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_max_dynamic_shards": "1999", 2023-09-23T23:13:17.689 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_max_listing_results": "1000", 2023-09-23T23:13:17.690 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_max_notify_retries": "10", 2023-09-23T23:13:17.690 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_max_objs_per_shard": "100000", 2023-09-23T23:13:17.690 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_max_put_param_size": "1048576", 2023-09-23T23:13:17.690 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_max_put_size": "5368709120", 2023-09-23T23:13:17.690 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_max_slo_entries": "1000", 2023-09-23T23:13:17.690 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_md_log_max_shards": "64", 2023-09-23T23:13:17.690 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_md_notify_interval_msec": "200", 2023-09-23T23:13:17.690 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_meta_sync_poll_interval": "20", 2023-09-23T23:13:17.690 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_meta_sync_spawn_window": "20", 2023-09-23T23:13:17.690 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-09-23T23:13:17.690 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_mp_lock_max_time": "600", 2023-09-23T23:13:17.690 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-09-23T23:13:17.691 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_multipart_min_part_size": "5242880", 2023-09-23T23:13:17.691 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-09-23T23:13:17.691 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-09-23T23:13:17.691 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_nfs_fhcache_size": "2017", 2023-09-23T23:13:17.691 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-09-23T23:13:17.691 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-09-23T23:13:17.691 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_nfs_lru_lanes": "5", 2023-09-23T23:13:17.691 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_nfs_max_gc": "300", 2023-09-23T23:13:17.691 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-09-23T23:13:17.691 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_nfs_run_gc_threads": "false", 2023-09-23T23:13:17.691 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_nfs_run_lc_threads": "false", 2023-09-23T23:13:17.691 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_nfs_run_quota_threads": "false", 2023-09-23T23:13:17.692 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_nfs_run_sync_thread": "false", 2023-09-23T23:13:17.692 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-09-23T23:13:17.692 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-09-23T23:13:17.692 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_num_async_rados_threads": "32", 2023-09-23T23:13:17.692 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_num_control_oids": "8", 2023-09-23T23:13:17.692 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_numa_node": "-1", 2023-09-23T23:13:17.692 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_obj_stripe_size": "4194304", 2023-09-23T23:13:17.692 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-09-23T23:13:17.692 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_objexp_chunk_size": "100", 2023-09-23T23:13:17.692 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_objexp_gc_interval": "600", 2023-09-23T23:13:17.692 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_objexp_hints_num_shards": "127", 2023-09-23T23:13:17.692 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-09-23T23:13:17.693 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-09-23T23:13:17.693 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_op_thread_timeout": "600", 2023-09-23T23:13:17.693 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_op_tracing": "false", 2023-09-23T23:13:17.693 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_opa_token": "", 2023-09-23T23:13:17.693 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_opa_url": "", 2023-09-23T23:13:17.693 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_opa_verify_ssl": "true", 2023-09-23T23:13:17.693 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-09-23T23:13:17.693 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-09-23T23:13:17.693 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_ops_log_rados": "false", 2023-09-23T23:13:17.693 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_ops_log_socket_path": "", 2023-09-23T23:13:17.693 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-09-23T23:13:17.693 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-09-23T23:13:17.694 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-09-23T23:13:17.694 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_period_push_interval": "2.000000", 2023-09-23T23:13:17.694 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_period_push_interval_max": "30.000000", 2023-09-23T23:13:17.694 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_period_root_pool": ".rgw.root", 2023-09-23T23:13:17.694 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-09-23T23:13:17.694 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_print_continue": "true", 2023-09-23T23:13:17.694 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_print_prohibited_content_length": "false", 2023-09-23T23:13:17.694 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-09-23T23:13:17.694 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-09-23T23:13:17.694 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-09-23T23:13:17.694 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-09-23T23:13:17.695 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_rados_tracing": "false", 2023-09-23T23:13:17.695 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_realm": "", 2023-09-23T23:13:17.695 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_realm_id": "", 2023-09-23T23:13:17.695 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-09-23T23:13:17.695 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_region": "", 2023-09-23T23:13:17.695 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_region_root_pool": ".rgw.root", 2023-09-23T23:13:17.695 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_relaxed_region_enforcement": "false", 2023-09-23T23:13:17.695 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-09-23T23:13:17.695 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-09-23T23:13:17.695 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_request_uri": "", 2023-09-23T23:13:17.695 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_reshard_batch_size": "64", 2023-09-23T23:13:17.695 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-09-23T23:13:17.696 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_reshard_max_aio": "128", 2023-09-23T23:13:17.696 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_reshard_num_logs": "16", 2023-09-23T23:13:17.696 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_reshard_thread_interval": "600", 2023-09-23T23:13:17.696 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_resolve_cname": "false", 2023-09-23T23:13:17.696 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_rest_getusage_op_compat": "false", 2023-09-23T23:13:17.696 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_run_sync_thread": "true", 2023-09-23T23:13:17.696 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-09-23T23:13:17.696 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_s3_auth_use_keystone": "false", 2023-09-23T23:13:17.696 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_s3_auth_use_ldap": "false", 2023-09-23T23:13:17.696 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_s3_auth_use_rados": "true", 2023-09-23T23:13:17.696 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_s3_auth_use_sts": "false", 2023-09-23T23:13:17.696 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-09-23T23:13:17.697 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_s3_success_create_obj_status": "0", 2023-09-23T23:13:17.697 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-09-23T23:13:17.697 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_scheduler_type": "throttler", 2023-09-23T23:13:17.697 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_script_uri": "", 2023-09-23T23:13:17.697 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_service_provider_name": "", 2023-09-23T23:13:17.697 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-09-23T23:13:17.697 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sts_client_id": "", 2023-09-23T23:13:17.697 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sts_client_secret": "", 2023-09-23T23:13:17.697 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sts_entry": "sts", 2023-09-23T23:13:17.697 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sts_key": "sts", 2023-09-23T23:13:17.697 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sts_max_session_duration": "43200", 2023-09-23T23:13:17.697 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sts_min_session_duration": "900", 2023-09-23T23:13:17.698 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sts_token_introspection_url": "", 2023-09-23T23:13:17.698 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_swift_account_in_url": "false", 2023-09-23T23:13:17.698 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_swift_auth_entry": "auth", 2023-09-23T23:13:17.698 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_swift_auth_url": "", 2023-09-23T23:13:17.698 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_swift_custom_header": "", 2023-09-23T23:13:17.698 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_swift_enforce_content_length": "false", 2023-09-23T23:13:17.698 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_swift_need_stats": "true", 2023-09-23T23:13:17.698 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_swift_tenant_name": "", 2023-09-23T23:13:17.698 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_swift_token_expiration": "86400", 2023-09-23T23:13:17.698 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_swift_url": "", 2023-09-23T23:13:17.698 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_swift_url_prefix": "swift", 2023-09-23T23:13:17.698 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_swift_versioning_enabled": "false", 2023-09-23T23:13:17.699 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-09-23T23:13:17.699 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-09-23T23:13:17.699 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sync_lease_period": "120", 2023-09-23T23:13:17.699 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-09-23T23:13:17.699 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sync_log_trim_interval": "1200", 2023-09-23T23:13:17.699 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-09-23T23:13:17.699 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-09-23T23:13:17.699 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-09-23T23:13:17.699 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sync_obj_etag_verify": "false", 2023-09-23T23:13:17.699 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sync_trace_history_size": "4096", 2023-09-23T23:13:17.699 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-09-23T23:13:17.699 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-09-23T23:13:17.700 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_thread_pool_size": "512", 2023-09-23T23:13:17.700 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_torrent_comment": "", 2023-09-23T23:13:17.700 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_torrent_createby": "", 2023-09-23T23:13:17.700 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_torrent_encoding": "", 2023-09-23T23:13:17.700 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_torrent_flag": "false", 2023-09-23T23:13:17.700 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_torrent_origin": "", 2023-09-23T23:13:17.700 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_torrent_sha_unit": "524288", 2023-09-23T23:13:17.700 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_torrent_tracker": "", 2023-09-23T23:13:17.700 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_trust_forwarded_https": "false", 2023-09-23T23:13:17.700 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-09-23T23:13:17.700 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_usage_log_tick_interval": "30", 2023-09-23T23:13:17.700 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_usage_max_shards": "32", 2023-09-23T23:13:17.700 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_usage_max_user_shards": "1", 2023-09-23T23:13:17.701 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_use_opa_authz": "false", 2023-09-23T23:13:17.701 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-09-23T23:13:17.701 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_user_default_quota_max_size": "-1", 2023-09-23T23:13:17.701 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_user_max_buckets": "1000", 2023-09-23T23:13:17.701 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_user_policies_max_num": "100", 2023-09-23T23:13:17.701 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-09-23T23:13:17.701 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-09-23T23:13:17.701 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_user_quota_sync_interval": "86400", 2023-09-23T23:13:17.701 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-09-23T23:13:17.701 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_user_unique_email": "true", 2023-09-23T23:13:17.701 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_verify_ssl": "true", 2023-09-23T23:13:17.701 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_website_routing_rules_max_num": "50", 2023-09-23T23:13:17.701 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_zone": "", 2023-09-23T23:13:17.702 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_zone_id": "", 2023-09-23T23:13:17.702 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-09-23T23:13:17.702 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_zonegroup": "", 2023-09-23T23:13:17.702 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_zonegroup_id": "", 2023-09-23T23:13:17.702 INFO:teuthology.orchestra.run.smithi110.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-09-23T23:13:17.702 INFO:teuthology.orchestra.run.smithi110.stdout: "rgwlc_auto_session_clear": "true", 2023-09-23T23:13:17.702 INFO:teuthology.orchestra.run.smithi110.stdout: "rgwlc_skip_bucket_step": "false", 2023-09-23T23:13:17.702 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_block_size": "4096", 2023-09-23T23:13:17.702 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-09-23T23:13:17.702 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-09-23T23:13:17.702 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-09-23T23:13:17.702 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-09-23T23:13:17.703 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_cache_shard_bits": "4", 2023-09-23T23:13:17.703 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_cache_size": "536870912", 2023-09-23T23:13:17.703 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_cache_type": "binned_lru", 2023-09-23T23:13:17.703 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-09-23T23:13:17.703 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-09-23T23:13:17.703 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-09-23T23:13:17.703 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_collect_compaction_stats": "false", 2023-09-23T23:13:17.703 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_collect_extended_stats": "false", 2023-09-23T23:13:17.703 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_collect_memory_stats": "false", 2023-09-23T23:13:17.703 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-09-23T23:13:17.703 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_index_type": "binary_search", 2023-09-23T23:13:17.703 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_log_to_ceph_log": "true", 2023-09-23T23:13:17.703 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_metadata_block_size": "4096", 2023-09-23T23:13:17.704 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_partition_filters": "false", 2023-09-23T23:13:17.704 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_perf": "false", 2023-09-23T23:13:17.704 INFO:teuthology.orchestra.run.smithi110.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-09-23T23:13:17.704 INFO:teuthology.orchestra.run.smithi110.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-09-23T23:13:17.704 INFO:teuthology.orchestra.run.smithi110.stdout: "rotating_keys_renewal_timeout": "10", 2023-09-23T23:13:17.704 INFO:teuthology.orchestra.run.smithi110.stdout: "run_dir": "/var/run/ceph", 2023-09-23T23:13:17.704 INFO:teuthology.orchestra.run.smithi110.stdout: "seastore_block_create": "true", 2023-09-23T23:13:17.704 INFO:teuthology.orchestra.run.smithi110.stdout: "seastore_cache_lru_size": "67108864", 2023-09-23T23:13:17.704 INFO:teuthology.orchestra.run.smithi110.stdout: "seastore_cbjournal_size": "5368709120", 2023-09-23T23:13:17.704 INFO:teuthology.orchestra.run.smithi110.stdout: "seastore_default_max_object_size": "16777216", 2023-09-23T23:13:17.704 INFO:teuthology.orchestra.run.smithi110.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-09-23T23:13:17.704 INFO:teuthology.orchestra.run.smithi110.stdout: "seastore_device_size": "53687091200", 2023-09-23T23:13:17.705 INFO:teuthology.orchestra.run.smithi110.stdout: "seastore_journal_batch_capacity": "16", 2023-09-23T23:13:17.705 INFO:teuthology.orchestra.run.smithi110.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-09-23T23:13:17.705 INFO:teuthology.orchestra.run.smithi110.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-09-23T23:13:17.705 INFO:teuthology.orchestra.run.smithi110.stdout: "seastore_journal_iodepth_limit": "5", 2023-09-23T23:13:17.705 INFO:teuthology.orchestra.run.smithi110.stdout: "seastore_main_device_type": "SSD", 2023-09-23T23:13:17.705 INFO:teuthology.orchestra.run.smithi110.stdout: "seastore_max_concurrent_transactions": "8", 2023-09-23T23:13:17.705 INFO:teuthology.orchestra.run.smithi110.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-09-23T23:13:17.705 INFO:teuthology.orchestra.run.smithi110.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-09-23T23:13:17.705 INFO:teuthology.orchestra.run.smithi110.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-09-23T23:13:17.705 INFO:teuthology.orchestra.run.smithi110.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-09-23T23:13:17.705 INFO:teuthology.orchestra.run.smithi110.stdout: "seastore_segment_size": "67108864", 2023-09-23T23:13:17.705 INFO:teuthology.orchestra.run.smithi110.stdout: "set_keepcaps": "false", 2023-09-23T23:13:17.706 INFO:teuthology.orchestra.run.smithi110.stdout: "setgroup": "ceph", 2023-09-23T23:13:17.706 INFO:teuthology.orchestra.run.smithi110.stdout: "setuser": "ceph", 2023-09-23T23:13:17.706 INFO:teuthology.orchestra.run.smithi110.stdout: "setuser_match_path": "", 2023-09-23T23:13:17.706 INFO:teuthology.orchestra.run.smithi110.stdout: "target_max_misplaced_ratio": "0.050000", 2023-09-23T23:13:17.706 INFO:teuthology.orchestra.run.smithi110.stdout: "thp": "false", 2023-09-23T23:13:17.706 INFO:teuthology.orchestra.run.smithi110.stdout: "threadpool_default_timeout": "60", 2023-09-23T23:13:17.706 INFO:teuthology.orchestra.run.smithi110.stdout: "threadpool_empty_queue_max_wait": "2", 2023-09-23T23:13:17.706 INFO:teuthology.orchestra.run.smithi110.stdout: "throttler_perf_counter": "true" 2023-09-23T23:13:17.706 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:13:18.300 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-09-23T23:13:18.644 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:18.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:18 smithi138 ceph-mon[193606]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:18.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:18 smithi110 ceph-mon[212939]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:18.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:18 smithi110 ceph-mon[221102]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:20.287 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a: {} 2023-09-23T23:13:20.296 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b: {} 2023-09-23T23:13:20.306 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c: {} 2023-09-23T23:13:20.307 INFO:teuthology.orchestra.run.smithi110.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T23:13:20.307 INFO:teuthology.orchestra.run.smithi110.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T23:13:20.307 INFO:teuthology.orchestra.run.smithi110.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T23:13:20.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:20 smithi138 ceph-mon[193606]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:20.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:20 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/682737033' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T23:13:20.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:20 smithi110 ceph-mon[212939]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:20.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:20 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/682737033' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T23:13:20.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:20 smithi110 ceph-mon[221102]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:20.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:20 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/682737033' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T23:13:20.990 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-09-23T23:13:20.990 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-09-23T23:13:20.991 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T23:13:20.991 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T23:13:20.991 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-09-23T23:13:20.991 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-09-23T23:13:20.992 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-09-23T23:13:20.992 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T23:13:20.992 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:13:20.992 INFO:tasks.thrashosds.thrasher:Reweighting osd 5 to 0.7526875838843302 2023-09-23T23:13:20.992 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 0.7526875838843302 2023-09-23T23:13:20.993 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-09-23T23:13:20.993 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:13:20.993 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-09-23T23:13:20.993 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:13:20.994 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-09-23T23:13:20.994 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:13:20.994 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-09-23T23:13:20.995 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:13:21.051 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:21.134 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:13:21.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:21.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:13:21.241 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:13:21.287 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:21.411 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:13:21.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:21.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:13:21.439 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:21.516 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:13:21.519 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:21.530 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:21.550 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:21.657 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:13:21.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:21.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:13:21.672 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:21.701 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:21.761 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:13:21.811 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:21.862 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:13:21.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:21.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:13:21.969 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:13:21.999 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:22.113 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:13:22.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:22.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:13:22.216 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:13:22.243 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:22.367 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:22 smithi110 ceph-mon[212939]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:22.367 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:22 smithi110 ceph-mon[221102]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:22.591 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:13:22.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:22.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:13:22.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:22 smithi138 ceph-mon[193606]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:22.696 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:13:22.723 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:22.886 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:13:22.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:22.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:13:22.990 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:13:23.018 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:23.371 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:13:23.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:23.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:13:23.476 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:13:23.503 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:23.966 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:13:23.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:23.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:13:24.071 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:13:24.104 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:24.153 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:13:24.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:24.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:13:24.258 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:13:24.286 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:24.296 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:24 smithi110 ceph-mon[212939]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:24.296 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:24 smithi110 ceph-mon[221102]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:24.329 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:13:24.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:24.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:13:24.436 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:13:24.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:24 smithi138 ceph-mon[193606]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:24.464 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:25.110 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:13:25.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:25.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:13:25.216 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:13:25.248 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:25.296 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:13:25.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:25.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:13:25.401 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:13:25.430 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:25.878 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:13:25.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:25.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:13:25.982 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:13:26.014 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:26.063 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:13:26.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:26.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:13:26.150 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:13:26.171 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:13:26.182 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:13:26.188 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:13:26.198 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:26.211 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:13:26.231 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:13:26.242 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:13:26.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:26.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:13:26.255 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:13:26.278 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:13:26.300 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:13:26.301 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:13:26.301 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:13:26.301 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:13:26.301 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:13:26.302 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:13:26.302 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:13:26.302 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:13:26.302 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:13:26.347 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:13:26.381 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:26.653 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:26 smithi110 ceph-mon[212939]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:26.653 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:26 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3303804090' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:13:26.654 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:26 smithi110 ceph-mon[221102]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:26.654 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:26 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3303804090' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:13:26.665 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:13:26.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:26.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:13:26.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:26 smithi138 ceph-mon[193606]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:26.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:26 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3303804090' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:13:26.771 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:13:26.799 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:26.804 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:13:26.804 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:13:26.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:13:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:13:26] "GET /metrics HTTP/1.1" 200 34047 "" "Prometheus/2.43.0" 2023-09-23T23:13:27.245 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:13:27.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:27.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:13:27.349 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:13:27.355 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_4' created 2023-09-23T23:13:27.355 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:13:27.382 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:27.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:27 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:13:27.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:27 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3386593645' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T23:13:27.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:27 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T23:13:27.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:27 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1966220217' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:13:27.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:27 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:13:27.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:27 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:13:27.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:27 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3386593645' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T23:13:27.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:27 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T23:13:27.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:27 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1966220217' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:13:27.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:27 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:13:27.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:27 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:13:27.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:27 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3386593645' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T23:13:27.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:27 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T23:13:27.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:27 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1966220217' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:13:27.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:27 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:13:27.882 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:13:27.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:27.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:13:27.987 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:13:28.015 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:28.063 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:13:28.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:28.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:13:28.170 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:13:28.196 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:28.301 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:13:28.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:28.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:13:28.406 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:13:28.432 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:28.665 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.5 to 0.752688 (c0b0) 2023-09-23T23:13:28.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:28 smithi138 ceph-mon[193606]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:28.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:28 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:13:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:28 smithi138 ceph-mon[193606]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:28 smithi138 ceph-mon[193606]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:28 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T23:13:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:28 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:13:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:28 smithi138 ceph-mon[193606]: osdmap e164: 8 total, 8 up, 8 in 2023-09-23T23:13:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:28 smithi138 ceph-mon[193606]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:28 smithi138 ceph-mon[193606]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:28 smithi138 ceph-mon[193606]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:28 smithi138 ceph-mon[193606]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:28 smithi138 ceph-mon[193606]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:28 smithi138 ceph-mon[193606]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:28 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2015481685' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.7526875838843302}]: dispatch 2023-09-23T23:13:28.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:28 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.7526875838843302}]: dispatch 2023-09-23T23:13:28.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:28.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:13:28.693 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[212939]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:28.693 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:13:28.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[212939]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[212939]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T23:13:28.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:13:28.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[212939]: osdmap e164: 8 total, 8 up, 8 in 2023-09-23T23:13:28.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[212939]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[212939]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[212939]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[212939]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[212939]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[212939]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.695 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2015481685' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.7526875838843302}]: dispatch 2023-09-23T23:13:28.695 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.7526875838843302}]: dispatch 2023-09-23T23:13:28.695 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:28.695 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:13:28.695 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[221102]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:28.695 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:13:28.695 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[221102]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.695 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[221102]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.695 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T23:13:28.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:13:28.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[221102]: osdmap e164: 8 total, 8 up, 8 in 2023-09-23T23:13:28.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[221102]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[221102]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[221102]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[221102]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[221102]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[221102]: failed to encode map e164 with expected crc 2023-09-23T23:13:28.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2015481685' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.7526875838843302}]: dispatch 2023-09-23T23:13:28.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.7526875838843302}]: dispatch 2023-09-23T23:13:28.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:28.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:13:28.697 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-09-23T23:13:28.957 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:13:29.282 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:13:29.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:29.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:13:29.386 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:13:29.411 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:29.529 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:13:29.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:29.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:13:29.635 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:13:29.639 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:29 smithi110 ceph-mon[212939]: failed to encode map e165 with expected crc 2023-09-23T23:13:29.639 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:29 smithi110 ceph-mon[212939]: failed to encode map e165 with expected crc 2023-09-23T23:13:29.639 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:29 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.7526875838843302}]': finished 2023-09-23T23:13:29.639 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:29 smithi110 ceph-mon[212939]: osdmap e165: 8 total, 8 up, 8 in 2023-09-23T23:13:29.639 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:29 smithi110 ceph-mon[212939]: failed to encode map e165 with expected crc 2023-09-23T23:13:29.640 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:29 smithi110 ceph-mon[221102]: failed to encode map e165 with expected crc 2023-09-23T23:13:29.640 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:29 smithi110 ceph-mon[221102]: failed to encode map e165 with expected crc 2023-09-23T23:13:29.640 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:29 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.7526875838843302}]': finished 2023-09-23T23:13:29.640 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:29 smithi110 ceph-mon[221102]: osdmap e165: 8 total, 8 up, 8 in 2023-09-23T23:13:29.640 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:29 smithi110 ceph-mon[221102]: failed to encode map e165 with expected crc 2023-09-23T23:13:29.660 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:29 smithi138 ceph-mon[193606]: failed to encode map e165 with expected crc 2023-09-23T23:13:29.660 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:29 smithi138 ceph-mon[193606]: failed to encode map e165 with expected crc 2023-09-23T23:13:29.660 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:29 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.7526875838843302}]': finished 2023-09-23T23:13:29.660 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:29 smithi138 ceph-mon[193606]: osdmap e165: 8 total, 8 up, 8 in 2023-09-23T23:13:29.660 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:29 smithi138 ceph-mon[193606]: failed to encode map e165 with expected crc 2023-09-23T23:13:29.662 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:29.705 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:13:29.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:29.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:13:29.755 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:29.807 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:29.810 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:13:29.837 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:29.879 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:13:29.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:29.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:13:29.985 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:13:30.016 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:30.208 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:13:30.564 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:13:30.566 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:30 smithi110 ceph-mon[212939]: pgmap v534: 89 pgs: 2 peering, 2 creating+activating, 5 creating+peering, 9 unknown, 71 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:30.566 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:30 smithi110 ceph-mon[212939]: failed to encode map e166 with expected crc 2023-09-23T23:13:30.566 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:30 smithi110 ceph-mon[212939]: osdmap e166: 8 total, 8 up, 8 in 2023-09-23T23:13:30.566 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:30 smithi110 ceph-mon[212939]: failed to encode map e166 with expected crc 2023-09-23T23:13:30.566 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:30 smithi110 ceph-mon[212939]: failed to encode map e166 with expected crc 2023-09-23T23:13:30.566 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:30 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:30.566 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:30 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:30.566 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:30 smithi110 ceph-mon[221102]: pgmap v534: 89 pgs: 2 peering, 2 creating+activating, 5 creating+peering, 9 unknown, 71 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:30.567 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:30 smithi110 ceph-mon[221102]: failed to encode map e166 with expected crc 2023-09-23T23:13:30.567 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:30 smithi110 ceph-mon[221102]: osdmap e166: 8 total, 8 up, 8 in 2023-09-23T23:13:30.567 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:30 smithi110 ceph-mon[221102]: failed to encode map e166 with expected crc 2023-09-23T23:13:30.567 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:30 smithi110 ceph-mon[221102]: failed to encode map e166 with expected crc 2023-09-23T23:13:30.567 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:30 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:30.567 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:30 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:30.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:30.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:13:30.669 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:13:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:30 smithi138 ceph-mon[193606]: pgmap v534: 89 pgs: 2 peering, 2 creating+activating, 5 creating+peering, 9 unknown, 71 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:30 smithi138 ceph-mon[193606]: failed to encode map e166 with expected crc 2023-09-23T23:13:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:30 smithi138 ceph-mon[193606]: osdmap e166: 8 total, 8 up, 8 in 2023-09-23T23:13:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:30 smithi138 ceph-mon[193606]: failed to encode map e166 with expected crc 2023-09-23T23:13:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:30 smithi138 ceph-mon[193606]: failed to encode map e166 with expected crc 2023-09-23T23:13:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:30 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:30.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:30 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:30.695 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:31.242 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:13:31.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:31.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:13:31.343 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:13:31.347 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:13:31.378 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:31.428 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:13:31.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:31.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:13:31.516 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:31 smithi110 ceph-mon[212939]: failed to encode map e166 with expected crc 2023-09-23T23:13:31.516 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:31 smithi110 ceph-mon[212939]: failed to encode map e167 with expected crc 2023-09-23T23:13:31.516 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:31 smithi110 ceph-mon[212939]: failed to encode map e167 with expected crc 2023-09-23T23:13:31.516 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:31 smithi110 ceph-mon[212939]: osdmap e167: 8 total, 8 up, 8 in 2023-09-23T23:13:31.517 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:31 smithi110 ceph-mon[221102]: failed to encode map e166 with expected crc 2023-09-23T23:13:31.517 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:31 smithi110 ceph-mon[221102]: failed to encode map e167 with expected crc 2023-09-23T23:13:31.517 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:31 smithi110 ceph-mon[221102]: failed to encode map e167 with expected crc 2023-09-23T23:13:31.517 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:31 smithi110 ceph-mon[221102]: osdmap e167: 8 total, 8 up, 8 in 2023-09-23T23:13:31.534 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:13:31.562 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:31.605 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:13:31.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:31.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:13:31.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:31 smithi138 ceph-mon[193606]: failed to encode map e166 with expected crc 2023-09-23T23:13:31.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:31 smithi138 ceph-mon[193606]: failed to encode map e167 with expected crc 2023-09-23T23:13:31.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:31 smithi138 ceph-mon[193606]: failed to encode map e167 with expected crc 2023-09-23T23:13:31.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:31 smithi138 ceph-mon[193606]: osdmap e167: 8 total, 8 up, 8 in 2023-09-23T23:13:31.713 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:13:31.739 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:31.781 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:13:31.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:31.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:13:31.887 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:13:31.912 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:31.954 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:13:31.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:31.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:13:32.059 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:13:32.085 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:32.129 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:13:32.133 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:32.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:32.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:13:32.236 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:13:32.262 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:32.303 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:13:32.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:32.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:13:32.408 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:13:32.434 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:32.476 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:13:32.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:32.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:13:32.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:13:32.617 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:32 smithi110 ceph-mon[212939]: pgmap v537: 89 pgs: 5 peering, 2 creating+activating, 5 creating+peering, 77 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:32.617 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:32 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:13:32.617 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:32 smithi110 ceph-mon[221102]: pgmap v537: 89 pgs: 5 peering, 2 creating+activating, 5 creating+peering, 77 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:32.617 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:32 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:13:32.620 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:32.672 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:13:32.672 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:13:32.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:32 smithi138 ceph-mon[193606]: pgmap v537: 89 pgs: 5 peering, 2 creating+activating, 5 creating+peering, 77 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:32.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:32 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:13:32.910 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:13:32.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:32.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:13:33.015 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:13:33.044 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:33.090 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:13:33.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:33.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:13:33.196 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:13:33.224 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:33.376 INFO:teuthology.orchestra.run.smithi110.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-09-23T23:13:33.497 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:33 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1085239276' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:13:33.497 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:33 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:13:33.497 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:33 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1085239276' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:13:33.497 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:33 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:13:33.503 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:33.520 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:13:33.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:33 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1085239276' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:13:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:33 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:13:34.219 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:13:34.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:34.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:13:34.323 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:13:34.351 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:34.448 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:13:34.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:34.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:13:34.478 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:34 smithi110 ceph-mon[212939]: pgmap v538: 89 pgs: 5 peering, 2 creating+activating, 1 creating+peering, 81 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:34.478 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:34 smithi110 ceph-mon[212939]: failed to encode map e168 with expected crc 2023-09-23T23:13:34.478 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:34 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:13:34.479 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:34 smithi110 ceph-mon[212939]: osdmap e168: 8 total, 8 up, 8 in 2023-09-23T23:13:34.479 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:34 smithi110 ceph-mon[212939]: failed to encode map e168 with expected crc 2023-09-23T23:13:34.479 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:34 smithi110 ceph-mon[221102]: pgmap v538: 89 pgs: 5 peering, 2 creating+activating, 1 creating+peering, 81 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:34.479 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:34 smithi110 ceph-mon[221102]: failed to encode map e168 with expected crc 2023-09-23T23:13:34.479 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:34 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:13:34.479 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:34 smithi110 ceph-mon[221102]: osdmap e168: 8 total, 8 up, 8 in 2023-09-23T23:13:34.479 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:34 smithi110 ceph-mon[221102]: failed to encode map e168 with expected crc 2023-09-23T23:13:34.552 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:13:34.584 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:34.634 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:13:34.638 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:34 smithi138 ceph-mon[193606]: pgmap v538: 89 pgs: 5 peering, 2 creating+activating, 1 creating+peering, 81 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T23:13:34.638 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:34 smithi138 ceph-mon[193606]: failed to encode map e168 with expected crc 2023-09-23T23:13:34.639 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:34 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:13:34.639 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:34 smithi138 ceph-mon[193606]: osdmap e168: 8 total, 8 up, 8 in 2023-09-23T23:13:34.639 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:34 smithi138 ceph-mon[193606]: failed to encode map e168 with expected crc 2023-09-23T23:13:34.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:34.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:13:34.662 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T23:13:34.662 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:13:34.663 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T23:13:34.663 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T23:13:34.663 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T23:13:34.663 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T23:13:34.741 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:13:34.767 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:35.119 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:13:35.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:35.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:13:35.212 DEBUG:teuthology.orchestra.run.smithi110:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-09-23T23:13:35.214 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T23:13:35.214 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T23:13:35.223 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:13:35.254 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:35.512 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:hints = 1 2023-09-23T23:13:35.512 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T23:13:35.512 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object prefix: benchmark_data_smithi110_295904 2023-09-23T23:13:35.512 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:13:35.513 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 0 16 16 0 0 0 - 0 2023-09-23T23:13:35.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:35 smithi138 ceph-mon[193606]: failed to encode map e168 with expected crc 2023-09-23T23:13:35.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:35 smithi110 ceph-mon[221102]: failed to encode map e168 with expected crc 2023-09-23T23:13:35.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:35 smithi110 ceph-mon[212939]: failed to encode map e168 with expected crc 2023-09-23T23:13:35.754 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:13:35.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:35.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:13:35.861 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:13:35.921 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:35.923 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:36.353 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:13:36.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:36.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:13:36.383 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:13:36.432 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:13:36.456 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:13:36.471 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:13:36.511 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:13:36.513 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 1 16 3212 3196 199.615 199.75 0.00685214 0.00498838 2023-09-23T23:13:36.513 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:36.517 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:36.549 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:13:36.596 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:13:36.633 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:13:36.680 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:13:36.683 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:13:36.683 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:13:36.683 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:13:36.683 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:13:36.683 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:13:36.684 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:13:36.684 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:13:36.684 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:13:36.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:36 smithi138 ceph-mon[193606]: pgmap v540: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 B/s, 0 objects/s recovering 2023-09-23T23:13:36.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:36 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T23:13:36.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:36 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/111448607' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:13:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:36 smithi110 ceph-mon[212939]: pgmap v540: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 B/s, 0 objects/s recovering 2023-09-23T23:13:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:36 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T23:13:36.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:36 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/111448607' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:13:36.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:36 smithi110 ceph-mon[221102]: pgmap v540: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 B/s, 0 objects/s recovering 2023-09-23T23:13:36.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:36 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T23:13:36.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:36 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/111448607' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:13:36.866 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:13:36.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:36.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:13:36.972 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:13:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:13:36] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.43.0" 2023-09-23T23:13:36.973 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:13:37.021 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:37.087 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:13:37.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:37.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:13:37.195 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:13:37.236 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:37.317 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:13:37.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:37.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:13:37.424 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:13:37.485 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:37.513 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 2 16 6368 6352 198.421 197.25 0.0058703 0.00502813 2023-09-23T23:13:38.514 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 3 16 9452 9436 196.459 192.75 0.00370901 0.00508042 2023-09-23T23:13:38.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:38 smithi138 ceph-mon[193606]: pgmap v541: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2023-09-23T23:13:38.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:38 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:38 smithi110 ceph-mon[212939]: pgmap v541: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2023-09-23T23:13:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:38 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:38.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:38 smithi110 ceph-mon[221102]: pgmap v541: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2023-09-23T23:13:38.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:38 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:39.418 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:13:39.514 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 4 16 12612 12596 196.715 197.5 0.00429518 0.00507538 2023-09-23T23:13:39.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:39 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3390091335' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:13:39.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:39 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3390091335' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:13:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:39 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3390091335' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:13:40.514 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 5 16 15547 15531 194.057 183.438 0.0015758 0.00510507 2023-09-23T23:13:40.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:40 smithi138 ceph-mon[193606]: pgmap v542: 89 pgs: 89 active+clean; 211 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s wr, 376 op/s; 7 B/s, 0 objects/s recovering 2023-09-23T23:13:40.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:40 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:13:40.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:40 smithi138 ceph-mon[193606]: failed to encode map e169 with expected crc 2023-09-23T23:13:40.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:40 smithi138 ceph-mon[193606]: failed to encode map e169 with expected crc 2023-09-23T23:13:40.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:40 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3390091335' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:13:40.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:40 smithi138 ceph-mon[193606]: failed to encode map e169 with expected crc 2023-09-23T23:13:40.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:40 smithi138 ceph-mon[193606]: osdmap e169: 8 total, 8 up, 8 in 2023-09-23T23:13:40.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:40 smithi138 ceph-mon[193606]: failed to encode map e169 with expected crc 2023-09-23T23:13:40.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:40 smithi138 ceph-mon[193606]: failed to encode map e169 with expected crc 2023-09-23T23:13:40.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:40 smithi138 ceph-mon[193606]: failed to encode map e169 with expected crc 2023-09-23T23:13:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[212939]: pgmap v542: 89 pgs: 89 active+clean; 211 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s wr, 376 op/s; 7 B/s, 0 objects/s recovering 2023-09-23T23:13:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:13:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[212939]: failed to encode map e169 with expected crc 2023-09-23T23:13:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[212939]: failed to encode map e169 with expected crc 2023-09-23T23:13:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3390091335' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:13:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[212939]: failed to encode map e169 with expected crc 2023-09-23T23:13:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[212939]: osdmap e169: 8 total, 8 up, 8 in 2023-09-23T23:13:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[212939]: failed to encode map e169 with expected crc 2023-09-23T23:13:40.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[212939]: failed to encode map e169 with expected crc 2023-09-23T23:13:40.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[212939]: failed to encode map e169 with expected crc 2023-09-23T23:13:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[221102]: pgmap v542: 89 pgs: 89 active+clean; 211 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s wr, 376 op/s; 7 B/s, 0 objects/s recovering 2023-09-23T23:13:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:13:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[221102]: failed to encode map e169 with expected crc 2023-09-23T23:13:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[221102]: failed to encode map e169 with expected crc 2023-09-23T23:13:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3390091335' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:13:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[221102]: failed to encode map e169 with expected crc 2023-09-23T23:13:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[221102]: osdmap e169: 8 total, 8 up, 8 in 2023-09-23T23:13:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[221102]: failed to encode map e169 with expected crc 2023-09-23T23:13:40.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[221102]: failed to encode map e169 with expected crc 2023-09-23T23:13:40.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:40 smithi110 ceph-mon[221102]: failed to encode map e169 with expected crc 2023-09-23T23:13:41.514 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 6 16 18187 18171 189.213 165 0.00416343 0.00527959 2023-09-23T23:13:41.797 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:13:41.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:41.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:13:41.901 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:13:41.973 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:42.515 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 7 16 19883 19867 177.321 106 0.0066632 0.00563303 2023-09-23T23:13:42.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:42 smithi138 ceph-mon[193606]: pgmap v544: 89 pgs: 89 active+clean; 669 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 8 B/s, 0 objects/s recovering 2023-09-23T23:13:42.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:42 smithi110 ceph-mon[212939]: pgmap v544: 89 pgs: 89 active+clean; 669 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 8 B/s, 0 objects/s recovering 2023-09-23T23:13:42.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:42 smithi110 ceph-mon[221102]: pgmap v544: 89 pgs: 89 active+clean; 669 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 8 B/s, 0 objects/s recovering 2023-09-23T23:13:42.739 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:13:42.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:42.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:13:42.844 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:13:43.104 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:43.411 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:13:43.465 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:13:43.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:43.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:13:43.515 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 8 16 22152 22136 172.878 141.812 0.00744703 0.00578099 2023-09-23T23:13:43.573 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:13:43.636 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:13:43.642 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:13:43.643 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:43.892 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:13:43.892 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:13:43.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:13:43.892 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:13:43.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.892 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34269.0:216422 7.9 7:925bb30d:::benchmark_data_smithi110_236826_object103714:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-09-23T23:13:43.893 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:07:29.062199+0000", 2023-09-23T23:13:43.893 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 374.82386082099998, 2023-09-23T23:13:43.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4987927110000001, 2023-09-23T23:13:43.893 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:43.893 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:13:43.893 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:43.893 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34269", 2023-09-23T23:13:43.893 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:13:43.893 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 216422 2023-09-23T23:13:43.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.893 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:43.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:43.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.062199+0000", 2023-09-23T23:13:43.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:43.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.062199+0000", 2023-09-23T23:13:43.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:43.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.062198+0000", 2023-09-23T23:13:43.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:13:43.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:43.895 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.062205+0000", 2023-09-23T23:13:43.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7390000000000002e-06 2023-09-23T23:13:43.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:43.895 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.062221+0000", 2023-09-23T23:13:43.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6388999999999998e-05 2023-09-23T23:13:43.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:43.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.062232+0000", 2023-09-23T23:13:43.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0049e-05 2023-09-23T23:13:43.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.063667+0000", 2023-09-23T23:13:43.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014357389999999999 2023-09-23T23:13:43.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:43.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.063819+0000", 2023-09-23T23:13:43.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015152399999999999 2023-09-23T23:13:43.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4", 2023-09-23T23:13:43.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.063942+0000", 2023-09-23T23:13:43.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012279500000000001 2023-09-23T23:13:43.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:13:43.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.068244+0000", 2023-09-23T23:13:43.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043022759999999998 2023-09-23T23:13:43.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.090887+0000", 2023-09-23T23:13:43.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022643020999999999 2023-09-23T23:13:43.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.898 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.090897+0000", 2023-09-23T23:13:43.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.031e-05 2023-09-23T23:13:43.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.898 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.062358+0000", 2023-09-23T23:13:43.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97146087800000003 2023-09-23T23:13:43.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.062365+0000", 2023-09-23T23:13:43.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3010000000000001e-06 2023-09-23T23:13:43.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.538200+0000", 2023-09-23T23:13:43.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4758343329999999 2023-09-23T23:13:43.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:13:43.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.538358+0000", 2023-09-23T23:13:43.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015803599999999999 2023-09-23T23:13:43.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.560933+0000", 2023-09-23T23:13:43.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022575237000000001 2023-09-23T23:13:43.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:43.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.560992+0000", 2023-09-23T23:13:43.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9138000000000002e-05 2023-09-23T23:13:43.900 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.900 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:43.901 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.901 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34269.0:216423 7.9 7:97bff28d:::benchmark_data_smithi110_236826_object103715:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-09-23T23:13:43.901 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:07:29.063468+0000", 2023-09-23T23:13:43.901 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 374.82259190799999, 2023-09-23T23:13:43.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5101274180000002, 2023-09-23T23:13:43.901 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:43.901 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:13:43.901 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:43.901 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34269", 2023-09-23T23:13:43.901 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:13:43.901 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 216423 2023-09-23T23:13:43.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.902 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:43.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.902 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:43.902 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.063468+0000", 2023-09-23T23:13:43.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.902 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:43.902 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.063468+0000", 2023-09-23T23:13:43.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:43.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.063467+0000", 2023-09-23T23:13:43.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:13:43.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:43.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.063474+0000", 2023-09-23T23:13:43.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5749999999999997e-06 2023-09-23T23:13:43.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:43.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.063489+0000", 2023-09-23T23:13:43.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5665999999999999e-05 2023-09-23T23:13:43.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:43.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.063497+0000", 2023-09-23T23:13:43.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0770000000000008e-06 2023-09-23T23:13:43.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.074211+0000", 2023-09-23T23:13:43.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010713660999999999 2023-09-23T23:13:43.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:43.905 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.074369+0000", 2023-09-23T23:13:43.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015824400000000001 2023-09-23T23:13:43.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.905 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4", 2023-09-23T23:13:43.905 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.074530+0000", 2023-09-23T23:13:43.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016089000000000001 2023-09-23T23:13:43.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.905 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.905 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.090914+0000", 2023-09-23T23:13:43.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016383844000000002 2023-09-23T23:13:43.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.090921+0000", 2023-09-23T23:13:43.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0480000000000001e-06 2023-09-23T23:13:43.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.062403+0000", 2023-09-23T23:13:43.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.971481594 2023-09-23T23:13:43.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.062412+0000", 2023-09-23T23:13:43.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7730000000000005e-06 2023-09-23T23:13:43.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.538668+0000", 2023-09-23T23:13:43.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.476256416 2023-09-23T23:13:43.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:13:43.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.538836+0000", 2023-09-23T23:13:43.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016759000000000001 2023-09-23T23:13:43.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.908 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.573498+0000", 2023-09-23T23:13:43.908 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.034662279999999997 2023-09-23T23:13:43.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:43.908 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.573596+0000", 2023-09-23T23:13:43.908 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7950999999999998e-05 2023-09-23T23:13:43.908 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.908 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:43.908 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.909 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34269.0:216430 7.4 7:2001ce57:::benchmark_data_smithi110_236826_object103722:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-09-23T23:13:43.909 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:07:29.082964+0000", 2023-09-23T23:13:43.909 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 374.803095869, 2023-09-23T23:13:43.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.451713856, 2023-09-23T23:13:43.909 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:43.909 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:13:43.909 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:43.909 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34269", 2023-09-23T23:13:43.909 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:13:43.909 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 216430 2023-09-23T23:13:43.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.909 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:43.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:43.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.082964+0000", 2023-09-23T23:13:43.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.910 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:43.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.082964+0000", 2023-09-23T23:13:43.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.910 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:43.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.082963+0000", 2023-09-23T23:13:43.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:13:43.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.911 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.911 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:43.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.082968+0000", 2023-09-23T23:13:43.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3709999999999999e-06 2023-09-23T23:13:43.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.911 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.911 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:43.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.082978+0000", 2023-09-23T23:13:43.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9850000000000001e-06 2023-09-23T23:13:43.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:43.912 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.082986+0000", 2023-09-23T23:13:43.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.875e-06 2023-09-23T23:13:43.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.912 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.083287+0000", 2023-09-23T23:13:43.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030073200000000001 2023-09-23T23:13:43.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:43.912 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.083419+0000", 2023-09-23T23:13:43.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000132146 2023-09-23T23:13:43.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.913 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4", 2023-09-23T23:13:43.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.083557+0000", 2023-09-23T23:13:43.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000137904 2023-09-23T23:13:43.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.913 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.095401+0000", 2023-09-23T23:13:43.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011844106 2023-09-23T23:13:43.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.095413+0000", 2023-09-23T23:13:43.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1654999999999999e-05 2023-09-23T23:13:43.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.060967+0000", 2023-09-23T23:13:43.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96555394500000002 2023-09-23T23:13:43.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.060981+0000", 2023-09-23T23:13:43.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4296e-05 2023-09-23T23:13:43.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.526043+0000", 2023-09-23T23:13:43.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4650625719999999 2023-09-23T23:13:43.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:13:43.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.526212+0000", 2023-09-23T23:13:43.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016892 2023-09-23T23:13:43.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.534564+0000", 2023-09-23T23:13:43.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083514750000000006 2023-09-23T23:13:43.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:43.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.534678+0000", 2023-09-23T23:13:43.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011441 2023-09-23T23:13:43.916 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.916 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:43.916 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.917 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34269.0:216436 7.4 7:2da04721:::benchmark_data_smithi110_236826_object103728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-09-23T23:13:43.917 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:07:29.100647+0000", 2023-09-23T23:13:43.917 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 374.78541333800001, 2023-09-23T23:13:43.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4511499689999998, 2023-09-23T23:13:43.917 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:43.917 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:13:43.917 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:43.917 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34269", 2023-09-23T23:13:43.917 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:13:43.917 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 216436 2023-09-23T23:13:43.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.917 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:43.918 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:43.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.100647+0000", 2023-09-23T23:13:43.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.918 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:43.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.100647+0000", 2023-09-23T23:13:43.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.918 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:43.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.100645+0000", 2023-09-23T23:13:43.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:13:43.919 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.919 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:43.919 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.100652+0000", 2023-09-23T23:13:43.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9630000000000001e-06 2023-09-23T23:13:43.919 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.919 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:43.919 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.100668+0000", 2023-09-23T23:13:43.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6118000000000001e-05 2023-09-23T23:13:43.919 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.920 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:43.920 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.100678+0000", 2023-09-23T23:13:43.920 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.011e-05 2023-09-23T23:13:43.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.920 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.920 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.101043+0000", 2023-09-23T23:13:43.920 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036507199999999998 2023-09-23T23:13:43.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.920 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.920 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.101057+0000", 2023-09-23T23:13:43.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3421000000000001e-05 2023-09-23T23:13:43.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.061036+0000", 2023-09-23T23:13:43.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95997920400000003 2023-09-23T23:13:43.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.061044+0000", 2023-09-23T23:13:43.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2090000000000005e-06 2023-09-23T23:13:43.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.526456+0000", 2023-09-23T23:13:43.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4654118979999999 2023-09-23T23:13:43.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:43.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.526618+0000", 2023-09-23T23:13:43.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000162459 2023-09-23T23:13:43.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:13:43.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.526771+0000", 2023-09-23T23:13:43.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015212099999999999 2023-09-23T23:13:43.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:13:43.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.534277+0000", 2023-09-23T23:13:43.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007506586 2023-09-23T23:13:43.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.534331+0000", 2023-09-23T23:13:43.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4265000000000003e-05 2023-09-23T23:13:43.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.551593+0000", 2023-09-23T23:13:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017261520999999998 2023-09-23T23:13:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:13:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.551635+0000", 2023-09-23T23:13:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2222000000000003e-05 2023-09-23T23:13:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:43.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.551797+0000", 2023-09-23T23:13:43.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000161664 2023-09-23T23:13:43.925 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.925 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:43.925 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.925 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34269.0:216442 7.4 7:23d03117:::benchmark_data_smithi110_236826_object103734:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-09-23T23:13:43.925 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:07:29.109465+0000", 2023-09-23T23:13:43.925 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 374.77659506399999, 2023-09-23T23:13:43.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4425073639999999, 2023-09-23T23:13:43.925 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:43.925 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:13:43.926 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:43.926 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34269", 2023-09-23T23:13:43.926 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:13:43.926 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 216442 2023-09-23T23:13:43.926 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.926 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:43.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:43.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.109465+0000", 2023-09-23T23:13:43.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.926 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.927 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:43.927 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.109465+0000", 2023-09-23T23:13:43.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.927 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.927 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.927 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:43.927 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.109463+0000", 2023-09-23T23:13:43.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:13:43.927 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.927 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.927 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:43.927 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.109475+0000", 2023-09-23T23:13:43.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1318000000000001e-05 2023-09-23T23:13:43.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.928 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:43.928 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.109499+0000", 2023-09-23T23:13:43.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4644e-05 2023-09-23T23:13:43.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.928 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:43.928 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.109511+0000", 2023-09-23T23:13:43.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1586999999999999e-05 2023-09-23T23:13:43.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.109604+0000", 2023-09-23T23:13:43.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3012999999999999e-05 2023-09-23T23:13:43.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.109616+0000", 2023-09-23T23:13:43.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2668e-05 2023-09-23T23:13:43.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.061077+0000", 2023-09-23T23:13:43.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95146007499999996 2023-09-23T23:13:43.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.061087+0000", 2023-09-23T23:13:43.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0929e-05 2023-09-23T23:13:43.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.530262+0000", 2023-09-23T23:13:43.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4691740760000001 2023-09-23T23:13:43.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:43.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.530406+0000", 2023-09-23T23:13:43.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000144695 2023-09-23T23:13:43.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:13:43.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.530501+0000", 2023-09-23T23:13:43.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4968000000000003e-05 2023-09-23T23:13:43.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:13:43.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.534513+0000", 2023-09-23T23:13:43.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040116140000000002 2023-09-23T23:13:43.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.932 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.932 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.539386+0000", 2023-09-23T23:13:43.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048732000000000003 2023-09-23T23:13:43.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.932 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.932 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.551833+0000", 2023-09-23T23:13:43.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012447299 2023-09-23T23:13:43.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.932 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:13:43.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.551872+0000", 2023-09-23T23:13:43.933 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8328999999999998e-05 2023-09-23T23:13:43.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:43.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.551973+0000", 2023-09-23T23:13:43.933 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010090300000000001 2023-09-23T23:13:43.933 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.933 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:43.933 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.934 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34269.0:216443 7.4 7:2f339ff9:::benchmark_data_smithi110_236826_object103735:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-09-23T23:13:43.934 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:07:29.111973+0000", 2023-09-23T23:13:43.934 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 374.77408770099998, 2023-09-23T23:13:43.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.440169918, 2023-09-23T23:13:43.934 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:43.934 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:13:43.934 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:43.934 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34269", 2023-09-23T23:13:43.934 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:13:43.934 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 216443 2023-09-23T23:13:43.934 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.934 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:43.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:43.935 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.111973+0000", 2023-09-23T23:13:43.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:43.935 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.111973+0000", 2023-09-23T23:13:43.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:43.935 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.111971+0000", 2023-09-23T23:13:43.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:13:43.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.936 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:43.936 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.111978+0000", 2023-09-23T23:13:43.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3769999999999998e-06 2023-09-23T23:13:43.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.936 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:43.936 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.111991+0000", 2023-09-23T23:13:43.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2425e-05 2023-09-23T23:13:43.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:43.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.111999+0000", 2023-09-23T23:13:43.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4449999999999996e-06 2023-09-23T23:13:43.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.112050+0000", 2023-09-23T23:13:43.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0664000000000001e-05 2023-09-23T23:13:43.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.112057+0000", 2023-09-23T23:13:43.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7659999999999998e-06 2023-09-23T23:13:43.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.061110+0000", 2023-09-23T23:13:43.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94905298599999999 2023-09-23T23:13:43.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.061115+0000", 2023-09-23T23:13:43.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6120000000000001e-06 2023-09-23T23:13:43.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.534761+0000", 2023-09-23T23:13:43.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4736456549999999 2023-09-23T23:13:43.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:43.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.534861+0000", 2023-09-23T23:13:43.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9973e-05 2023-09-23T23:13:43.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:13:43.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.534992+0000", 2023-09-23T23:13:43.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013150999999999999 2023-09-23T23:13:43.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:13:43.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.547483+0000", 2023-09-23T23:13:43.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012490925 2023-09-23T23:13:43.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.551560+0000", 2023-09-23T23:13:43.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040768250000000001 2023-09-23T23:13:43.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.941 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.552008+0000", 2023-09-23T23:13:43.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00044822999999999998 2023-09-23T23:13:43.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.941 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:13:43.941 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.552045+0000", 2023-09-23T23:13:43.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6656000000000003e-05 2023-09-23T23:13:43.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.941 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:43.941 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.552142+0000", 2023-09-23T23:13:43.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7689000000000006e-05 2023-09-23T23:13:43.941 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.942 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:43.942 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.942 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.942 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34269.0:216445 7.9 7:9078889e:::benchmark_data_smithi110_236826_object103737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-09-23T23:13:43.942 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:07:29.118391+0000", 2023-09-23T23:13:43.942 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 374.76766948800002, 2023-09-23T23:13:43.942 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.442141237, 2023-09-23T23:13:43.942 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:43.942 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:13:43.942 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:43.942 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34269", 2023-09-23T23:13:43.942 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:13:43.943 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 216445 2023-09-23T23:13:43.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.943 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:43.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.943 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:43.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.118391+0000", 2023-09-23T23:13:43.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.943 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:43.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.118391+0000", 2023-09-23T23:13:43.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:43.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.118389+0000", 2023-09-23T23:13:43.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:13:43.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:43.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.118398+0000", 2023-09-23T23:13:43.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5889999999999992e-06 2023-09-23T23:13:43.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:43.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.118417+0000", 2023-09-23T23:13:43.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.872e-05 2023-09-23T23:13:43.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:43.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.118433+0000", 2023-09-23T23:13:43.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5736999999999999e-05 2023-09-23T23:13:43.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.118503+0000", 2023-09-23T23:13:43.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0469999999999994e-05 2023-09-23T23:13:43.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.946 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.946 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.118515+0000", 2023-09-23T23:13:43.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2199e-05 2023-09-23T23:13:43.946 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.946 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.946 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.062447+0000", 2023-09-23T23:13:43.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94393131200000002 2023-09-23T23:13:43.946 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.946 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.946 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.062455+0000", 2023-09-23T23:13:43.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8110000000000003e-06 2023-09-23T23:13:43.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.538871+0000", 2023-09-23T23:13:43.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.47641617 2023-09-23T23:13:43.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:43.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.538957+0000", 2023-09-23T23:13:43.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6049000000000002e-05 2023-09-23T23:13:43.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:13:43.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.539070+0000", 2023-09-23T23:13:43.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000113433 2023-09-23T23:13:43.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.559903+0000", 2023-09-23T23:13:43.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020832719 2023-09-23T23:13:43.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:13:43.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.560114+0000", 2023-09-23T23:13:43.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021144600000000001 2023-09-23T23:13:43.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.560362+0000", 2023-09-23T23:13:43.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024759000000000001 2023-09-23T23:13:43.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:13:43.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.560414+0000", 2023-09-23T23:13:43.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1776e-05 2023-09-23T23:13:43.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:43.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.560532+0000", 2023-09-23T23:13:43.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118211 2023-09-23T23:13:43.950 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.950 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:43.950 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.950 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34269.0:216451 7.4 7:21539bc2:::benchmark_data_smithi110_236826_object103743:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-09-23T23:13:43.950 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:07:29.127168+0000", 2023-09-23T23:13:43.950 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 374.75889261200001, 2023-09-23T23:13:43.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4403922050000002, 2023-09-23T23:13:43.950 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:43.950 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:13:43.950 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:43.951 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34269", 2023-09-23T23:13:43.951 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:13:43.951 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 216451 2023-09-23T23:13:43.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.951 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:43.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:43.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.127168+0000", 2023-09-23T23:13:43.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:43.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.127168+0000", 2023-09-23T23:13:43.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:43.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.127165+0000", 2023-09-23T23:13:43.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-09-23T23:13:43.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:43.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.127178+0000", 2023-09-23T23:13:43.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3078000000000001e-05 2023-09-23T23:13:43.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:43.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.127202+0000", 2023-09-23T23:13:43.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4076000000000001e-05 2023-09-23T23:13:43.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:43.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.127215+0000", 2023-09-23T23:13:43.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2894000000000001e-05 2023-09-23T23:13:43.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.127303+0000", 2023-09-23T23:13:43.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8103999999999995e-05 2023-09-23T23:13:43.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.954 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.127319+0000", 2023-09-23T23:13:43.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5981000000000002e-05 2023-09-23T23:13:43.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.954 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.061128+0000", 2023-09-23T23:13:43.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93380840899999995 2023-09-23T23:13:43.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.061132+0000", 2023-09-23T23:13:43.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1370000000000004e-06 2023-09-23T23:13:43.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.539410+0000", 2023-09-23T23:13:43.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.478277587 2023-09-23T23:13:43.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:43.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.539518+0000", 2023-09-23T23:13:43.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107948 2023-09-23T23:13:43.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:13:43.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.539650+0000", 2023-09-23T23:13:43.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013290599999999999 2023-09-23T23:13:43.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.552177+0000", 2023-09-23T23:13:43.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012526885 2023-09-23T23:13:43.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.567312+0000", 2023-09-23T23:13:43.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0151343 2023-09-23T23:13:43.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:13:43.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.567323+0000", 2023-09-23T23:13:43.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0957e-05 2023-09-23T23:13:43.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:13:43.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.567373+0000", 2023-09-23T23:13:43.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0615000000000002e-05 2023-09-23T23:13:43.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:43.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.567560+0000", 2023-09-23T23:13:43.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000186632 2023-09-23T23:13:43.958 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.958 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:43.958 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.958 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34269.0:216452 7.4 7:25e75ec5:::benchmark_data_smithi110_236826_object103744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-09-23T23:13:43.958 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:07:29.127309+0000", 2023-09-23T23:13:43.959 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 374.75875157899998, 2023-09-23T23:13:43.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4463695639999998, 2023-09-23T23:13:43.959 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:43.959 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:13:43.959 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:43.959 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34269", 2023-09-23T23:13:43.959 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:13:43.959 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 216452 2023-09-23T23:13:43.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.959 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:43.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:43.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.127309+0000", 2023-09-23T23:13:43.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:43.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.127309+0000", 2023-09-23T23:13:43.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:43.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.127308+0000", 2023-09-23T23:13:43.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:13:43.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:43.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.127313+0000", 2023-09-23T23:13:43.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7679999999999999e-06 2023-09-23T23:13:43.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:43.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.127321+0000", 2023-09-23T23:13:43.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7290000000000006e-06 2023-09-23T23:13:43.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:43.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.127327+0000", 2023-09-23T23:13:43.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1020000000000002e-06 2023-09-23T23:13:43.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.127379+0000", 2023-09-23T23:13:43.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1418999999999997e-05 2023-09-23T23:13:43.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.127388+0000", 2023-09-23T23:13:43.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8789999999999992e-06 2023-09-23T23:13:43.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.061144+0000", 2023-09-23T23:13:43.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93375624800000001 2023-09-23T23:13:43.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.061148+0000", 2023-09-23T23:13:43.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1359999999999999e-06 2023-09-23T23:13:43.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.543814+0000", 2023-09-23T23:13:43.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4826660229999999 2023-09-23T23:13:43.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:43.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.543927+0000", 2023-09-23T23:13:43.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000112661 2023-09-23T23:13:43.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:13:43.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.544007+0000", 2023-09-23T23:13:43.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0655999999999995e-05 2023-09-23T23:13:43.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.552209+0000", 2023-09-23T23:13:43.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0082011830000000008 2023-09-23T23:13:43.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:13:43.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.567572+0000", 2023-09-23T23:13:43.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015363389999999999 2023-09-23T23:13:43.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.573456+0000", 2023-09-23T23:13:43.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058836089999999997 2023-09-23T23:13:43.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:13:43.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.573507+0000", 2023-09-23T23:13:43.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1876999999999998e-05 2023-09-23T23:13:43.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:43.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.573678+0000", 2023-09-23T23:13:43.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000170765 2023-09-23T23:13:43.966 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.966 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:43.966 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.967 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34269.0:216453 7.9 7:9d4f91f4:::benchmark_data_smithi110_236826_object103745:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-09-23T23:13:43.967 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:07:29.128326+0000", 2023-09-23T23:13:43.967 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 374.75773423599998, 2023-09-23T23:13:43.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4454867629999999, 2023-09-23T23:13:43.967 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:43.967 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:13:43.967 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:43.967 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34269", 2023-09-23T23:13:43.967 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:13:43.967 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 216453 2023-09-23T23:13:43.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.968 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:43.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:43.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.128326+0000", 2023-09-23T23:13:43.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:43.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.128326+0000", 2023-09-23T23:13:43.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:43.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.128325+0000", 2023-09-23T23:13:43.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:13:43.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:43.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.128331+0000", 2023-09-23T23:13:43.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8780000000000003e-06 2023-09-23T23:13:43.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:43.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.128348+0000", 2023-09-23T23:13:43.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6629999999999998e-05 2023-09-23T23:13:43.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:43.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.128356+0000", 2023-09-23T23:13:43.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7309999999999999e-06 2023-09-23T23:13:43.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.128403+0000", 2023-09-23T23:13:43.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6523999999999997e-05 2023-09-23T23:13:43.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:29.128412+0000", 2023-09-23T23:13:43.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8249999999999994e-06 2023-09-23T23:13:43.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.062499+0000", 2023-09-23T23:13:43.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.934087047 2023-09-23T23:13:43.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:30.062511+0000", 2023-09-23T23:13:43.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2629999999999999e-05 2023-09-23T23:13:43.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.542841+0000", 2023-09-23T23:13:43.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.480329955 2023-09-23T23:13:43.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:43.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.542936+0000", 2023-09-23T23:13:43.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4302000000000006e-05 2023-09-23T23:13:43.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:13:43.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.543042+0000", 2023-09-23T23:13:43.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000106554 2023-09-23T23:13:43.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.559938+0000", 2023-09-23T23:13:43.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016896096999999999 2023-09-23T23:13:43.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:13:43.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.560242+0000", 2023-09-23T23:13:43.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030343500000000001 2023-09-23T23:13:43.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.573670+0000", 2023-09-23T23:13:43.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013428404 2023-09-23T23:13:43.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:13:43.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.573713+0000", 2023-09-23T23:13:43.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3186000000000003e-05 2023-09-23T23:13:43.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:43.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:07:31.573813+0000", 2023-09-23T23:13:43.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9525999999999995e-05 2023-09-23T23:13:43.975 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.975 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:43.975 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.975 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:79750 8.e 8:7fa2ab83:::benchmark_data_smithi110_258128_object79749:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T23:13:43.975 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:08:44.102150+0000", 2023-09-23T23:13:43.975 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 299.78391049099997, 2023-09-23T23:13:43.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0795274450000001, 2023-09-23T23:13:43.976 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:43.976 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:13:43.976 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:43.976 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:13:43.976 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:13:43.976 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 79750 2023-09-23T23:13:43.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.976 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:43.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:43.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.102150+0000", 2023-09-23T23:13:43.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:43.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.102150+0000", 2023-09-23T23:13:43.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:43.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.102148+0000", 2023-09-23T23:13:43.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:13:43.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:43.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.102180+0000", 2023-09-23T23:13:43.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1715999999999999e-05 2023-09-23T23:13:43.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:43.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.102194+0000", 2023-09-23T23:13:43.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4167e-05 2023-09-23T23:13:43.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:43.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.102203+0000", 2023-09-23T23:13:43.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.833e-06 2023-09-23T23:13:43.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.102248+0000", 2023-09-23T23:13:43.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4836999999999998e-05 2023-09-23T23:13:43.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:43.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.105394+0000", 2023-09-23T23:13:43.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031463950000000002 2023-09-23T23:13:43.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4", 2023-09-23T23:13:43.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.105578+0000", 2023-09-23T23:13:43.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018416600000000001 2023-09-23T23:13:43.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:13:43.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.113263+0000", 2023-09-23T23:13:43.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076847219999999997 2023-09-23T23:13:43.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.115154+0000", 2023-09-23T23:13:43.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001891127 2023-09-23T23:13:43.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.115171+0000", 2023-09-23T23:13:43.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7071999999999998e-05 2023-09-23T23:13:43.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:45.105860+0000", 2023-09-23T23:13:43.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99068911699999995 2023-09-23T23:13:43.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:45.105881+0000", 2023-09-23T23:13:43.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0290000000000001e-05 2023-09-23T23:13:43.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.181583+0000", 2023-09-23T23:13:43.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0757027450000001 2023-09-23T23:13:43.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:43.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.181677+0000", 2023-09-23T23:13:43.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3691999999999997e-05 2023-09-23T23:13:43.982 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.983 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:43.983 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.983 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:79781 8.e 8:71d6d9f2:::benchmark_data_smithi110_258128_object79780:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T23:13:43.983 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:08:44.147184+0000", 2023-09-23T23:13:43.983 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 299.73887605300001, 2023-09-23T23:13:43.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0545239180000001, 2023-09-23T23:13:43.983 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:43.983 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:13:43.983 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:43.983 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:13:43.984 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:13:43.984 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 79781 2023-09-23T23:13:43.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.984 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:43.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:43.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.147184+0000", 2023-09-23T23:13:43.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:43.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.147184+0000", 2023-09-23T23:13:43.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:43.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.147183+0000", 2023-09-23T23:13:43.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:13:43.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:43.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.147206+0000", 2023-09-23T23:13:43.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3251000000000002e-05 2023-09-23T23:13:43.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:43.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.147223+0000", 2023-09-23T23:13:43.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6699999999999999e-05 2023-09-23T23:13:43.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:43.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.147232+0000", 2023-09-23T23:13:43.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8980000000000002e-06 2023-09-23T23:13:43.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.147284+0000", 2023-09-23T23:13:43.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1669000000000003e-05 2023-09-23T23:13:43.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.147295+0000", 2023-09-23T23:13:43.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0858e-05 2023-09-23T23:13:43.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:45.105908+0000", 2023-09-23T23:13:43.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95861380600000001 2023-09-23T23:13:43.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:45.105916+0000", 2023-09-23T23:13:43.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0420000000000003e-06 2023-09-23T23:13:43.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.182800+0000", 2023-09-23T23:13:43.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0768834199999999 2023-09-23T23:13:43.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:43.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.182919+0000", 2023-09-23T23:13:43.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011866000000000001 2023-09-23T23:13:43.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:13:43.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.183038+0000", 2023-09-23T23:13:43.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000119701 2023-09-23T23:13:43.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.189667+0000", 2023-09-23T23:13:43.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0066287530000000002 2023-09-23T23:13:43.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:13:43.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.189760+0000", 2023-09-23T23:13:43.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3104999999999997e-05 2023-09-23T23:13:43.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.200564+0000", 2023-09-23T23:13:43.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010803989999999999 2023-09-23T23:13:43.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:13:43.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.200630+0000", 2023-09-23T23:13:43.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5883999999999995e-05 2023-09-23T23:13:43.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:43.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.201708+0000", 2023-09-23T23:13:43.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010781269999999999 2023-09-23T23:13:43.991 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.991 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:43.991 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:43.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.991 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:79786 8.e 8:7c07a634:::benchmark_data_smithi110_258128_object79785:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T23:13:43.991 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:08:44.158858+0000", 2023-09-23T23:13:43.992 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 299.72720208999999, 2023-09-23T23:13:43.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0435438530000001, 2023-09-23T23:13:43.992 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:43.992 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:13:43.992 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:43.992 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:13:43.992 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:13:43.992 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 79786 2023-09-23T23:13:43.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.992 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:43.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:43.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.158858+0000", 2023-09-23T23:13:43.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:43.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.158858+0000", 2023-09-23T23:13:43.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:43.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:43.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.158857+0000", 2023-09-23T23:13:43.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:13:43.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:43.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.158883+0000", 2023-09-23T23:13:43.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6242e-05 2023-09-23T23:13:43.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:43.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.158901+0000", 2023-09-23T23:13:43.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7867999999999999e-05 2023-09-23T23:13:43.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:43.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.158910+0000", 2023-09-23T23:13:43.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2359999999999993e-06 2023-09-23T23:13:43.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.158969+0000", 2023-09-23T23:13:43.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9375e-05 2023-09-23T23:13:43.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.158982+0000", 2023-09-23T23:13:43.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.314e-05 2023-09-23T23:13:43.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:45.105940+0000", 2023-09-23T23:13:43.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94695715899999999 2023-09-23T23:13:43.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:43.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:45.105947+0000", 2023-09-23T23:13:43.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8269999999999998e-06 2023-09-23T23:13:43.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:43.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.186123+0000", 2023-09-23T23:13:43.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0801752470000001 2023-09-23T23:13:43.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:43.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.186280+0000", 2023-09-23T23:13:43.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000156885 2023-09-23T23:13:43.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:13:43.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.186461+0000", 2023-09-23T23:13:43.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000181566 2023-09-23T23:13:43.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:13:43.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.189767+0000", 2023-09-23T23:13:43.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033055250000000001 2023-09-23T23:13:43.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.195804+0000", 2023-09-23T23:13:43.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060371870000000003 2023-09-23T23:13:43.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:43.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.201760+0000", 2023-09-23T23:13:43.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059564450000000003 2023-09-23T23:13:43.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:13:43.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.201814+0000", 2023-09-23T23:13:43.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3264999999999999e-05 2023-09-23T23:13:43.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:43.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:43.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:43.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.202402+0000", 2023-09-23T23:13:43.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00058842699999999998 2023-09-23T23:13:43.999 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:44.000 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:44.000 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:44.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.000 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:171285 8.c 8:3d91b401:::benchmark_data_smithi110_258128_object52467:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T23:13:44.000 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:39.097455+0000", 2023-09-23T23:13:44.000 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 184.78860540599999, 2023-09-23T23:13:44.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2602485080000001, 2023-09-23T23:13:44.000 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:44.000 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:13:44.000 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:44.000 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:13:44.000 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:13:44.001 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 171285 2023-09-23T23:13:44.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.001 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:44.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:44.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.097455+0000", 2023-09-23T23:13:44.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:44.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:44.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.097455+0000", 2023-09-23T23:13:44.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:44.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:44.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.097453+0000", 2023-09-23T23:13:44.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:13:44.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:44.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.097460+0000", 2023-09-23T23:13:44.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7789999999999998e-06 2023-09-23T23:13:44.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:44.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.097472+0000", 2023-09-23T23:13:44.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2517999999999999e-05 2023-09-23T23:13:44.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:44.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.097480+0000", 2023-09-23T23:13:44.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0590000000000003e-06 2023-09-23T23:13:44.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.097549+0000", 2023-09-23T23:13:44.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8516000000000006e-05 2023-09-23T23:13:44.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:44.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.097656+0000", 2023-09-23T23:13:44.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010733100000000001 2023-09-23T23:13:44.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:13:44.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.097787+0000", 2023-09-23T23:13:44.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000130297 2023-09-23T23:13:44.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:13:44.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.107367+0000", 2023-09-23T23:13:44.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0095800039999999996 2023-09-23T23:13:44.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:44.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.110105+0000", 2023-09-23T23:13:44.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027379959999999999 2023-09-23T23:13:44.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.134020+0000", 2023-09-23T23:13:44.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023915948999999999 2023-09-23T23:13:44.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:44.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.134030+0000", 2023-09-23T23:13:44.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2299999999999997e-06 2023-09-23T23:13:44.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.357621+0000", 2023-09-23T23:13:44.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2235911129999999 2023-09-23T23:13:44.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:44.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.357703+0000", 2023-09-23T23:13:44.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2496000000000004e-05 2023-09-23T23:13:44.007 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:44.007 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:44.007 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:44.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.007 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:171288 8.c 8:3d702d4c:::benchmark_data_smithi110_258128_object52470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T23:13:44.007 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:39.100656+0000", 2023-09-23T23:13:44.007 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 184.785404105, 2023-09-23T23:13:44.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.257311412, 2023-09-23T23:13:44.007 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:44.007 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:13:44.008 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:44.008 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:13:44.008 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:13:44.008 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 171288 2023-09-23T23:13:44.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.008 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:44.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:44.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.100656+0000", 2023-09-23T23:13:44.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:44.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:44.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.100656+0000", 2023-09-23T23:13:44.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:44.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:44.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.100655+0000", 2023-09-23T23:13:44.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:13:44.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:44.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.100662+0000", 2023-09-23T23:13:44.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7149999999999998e-06 2023-09-23T23:13:44.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:44.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.100680+0000", 2023-09-23T23:13:44.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7439e-05 2023-09-23T23:13:44.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:44.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.100689+0000", 2023-09-23T23:13:44.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7669999999999992e-06 2023-09-23T23:13:44.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.102316+0000", 2023-09-23T23:13:44.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001627781 2023-09-23T23:13:44.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:44.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.102460+0000", 2023-09-23T23:13:44.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000143642 2023-09-23T23:13:44.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:13:44.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.102604+0000", 2023-09-23T23:13:44.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000143607 2023-09-23T23:13:44.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:13:44.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.107522+0000", 2023-09-23T23:13:44.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0049182399999999999 2023-09-23T23:13:44.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:44.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.110247+0000", 2023-09-23T23:13:44.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027246390000000001 2023-09-23T23:13:44.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.134053+0000", 2023-09-23T23:13:44.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023806898 2023-09-23T23:13:44.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:44.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.134060+0000", 2023-09-23T23:13:44.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9619999999999996e-06 2023-09-23T23:13:44.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.357904+0000", 2023-09-23T23:13:44.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.223843732 2023-09-23T23:13:44.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:44.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.357968+0000", 2023-09-23T23:13:44.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3409000000000002e-05 2023-09-23T23:13:44.014 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:44.014 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:44.014 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:44.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.014 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:171299 8.c 8:3ba7af02:::benchmark_data_smithi110_258128_object52481:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T23:13:44.014 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:39.116693+0000", 2023-09-23T23:13:44.014 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 184.76936685800001, 2023-09-23T23:13:44.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2604995240000001, 2023-09-23T23:13:44.015 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:44.015 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:13:44.015 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:44.015 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:13:44.015 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:13:44.015 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 171299 2023-09-23T23:13:44.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.015 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:44.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:44.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.116693+0000", 2023-09-23T23:13:44.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:44.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:44.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.116693+0000", 2023-09-23T23:13:44.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:44.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:44.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.116693+0000", 2023-09-23T23:13:44.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:13:44.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:44.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.116697+0000", 2023-09-23T23:13:44.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7280000000000003e-06 2023-09-23T23:13:44.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:44.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.116710+0000", 2023-09-23T23:13:44.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2889e-05 2023-09-23T23:13:44.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:44.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.116718+0000", 2023-09-23T23:13:44.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7500000000000003e-06 2023-09-23T23:13:44.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.116806+0000", 2023-09-23T23:13:44.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7744999999999994e-05 2023-09-23T23:13:44.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:44.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.116951+0000", 2023-09-23T23:13:44.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014493399999999999 2023-09-23T23:13:44.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:13:44.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.117086+0000", 2023-09-23T23:13:44.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000135807 2023-09-23T23:13:44.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:13:44.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.123596+0000", 2023-09-23T23:13:44.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0065094020000000001 2023-09-23T23:13:44.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.134091+0000", 2023-09-23T23:13:44.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010495066000000001 2023-09-23T23:13:44.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:44.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.134099+0000", 2023-09-23T23:13:44.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0660000000000004e-06 2023-09-23T23:13:44.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.358000+0000", 2023-09-23T23:13:44.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.223901358 2023-09-23T23:13:44.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:13:44.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.358111+0000", 2023-09-23T23:13:44.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110785 2023-09-23T23:13:44.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.377124+0000", 2023-09-23T23:13:44.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019012814999999999 2023-09-23T23:13:44.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:44.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.377193+0000", 2023-09-23T23:13:44.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8938999999999998e-05 2023-09-23T23:13:44.022 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:44.022 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:44.022 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:44.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.022 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:171328 8.c 8:3618bc2d:::benchmark_data_smithi110_258128_object52510:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T23:13:44.022 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:39.183634+0000", 2023-09-23T23:13:44.022 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 184.70242642100001, 2023-09-23T23:13:44.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.19223843, 2023-09-23T23:13:44.022 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:44.022 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:13:44.022 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:44.023 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:13:44.023 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:13:44.023 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 171328 2023-09-23T23:13:44.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.023 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:44.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:44.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.183634+0000", 2023-09-23T23:13:44.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:44.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:44.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.183634+0000", 2023-09-23T23:13:44.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:44.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:44.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.183632+0000", 2023-09-23T23:13:44.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:13:44.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:44.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.183639+0000", 2023-09-23T23:13:44.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5660000000000003e-06 2023-09-23T23:13:44.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:44.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.183656+0000", 2023-09-23T23:13:44.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6748e-05 2023-09-23T23:13:44.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:44.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.183665+0000", 2023-09-23T23:13:44.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8559999999999996e-06 2023-09-23T23:13:44.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.183710+0000", 2023-09-23T23:13:44.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5850999999999999e-05 2023-09-23T23:13:44.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:44.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.183718+0000", 2023-09-23T23:13:44.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7889999999999995e-06 2023-09-23T23:13:44.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.358143+0000", 2023-09-23T23:13:44.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.174424865 2023-09-23T23:13:44.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:44.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.358256+0000", 2023-09-23T23:13:44.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000112921 2023-09-23T23:13:44.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 7", 2023-09-23T23:13:44.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.358374+0000", 2023-09-23T23:13:44.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117713 2023-09-23T23:13:44.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:13:44.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.366339+0000", 2023-09-23T23:13:44.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079653629999999996 2023-09-23T23:13:44.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:44.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.375767+0000", 2023-09-23T23:13:44.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0094274479999999997 2023-09-23T23:13:44.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:13:44.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.375801+0000", 2023-09-23T23:13:44.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4107999999999999e-05 2023-09-23T23:13:44.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:44.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.375872+0000", 2023-09-23T23:13:44.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1612000000000005e-05 2023-09-23T23:13:44.031 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:44.031 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:44.031 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:44.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.031 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:186174 8.c 8:3664f36a:::benchmark_data_smithi110_258128_object67356:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T23:13:44.031 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:58.485934+0000", 2023-09-23T23:13:44.031 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 165.40012637300001, 2023-09-23T23:13:44.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3897138770000002, 2023-09-23T23:13:44.031 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:44.032 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:13:44.032 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:44.032 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:13:44.032 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:13:44.032 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 186174 2023-09-23T23:13:44.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.032 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:44.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:44.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.485934+0000", 2023-09-23T23:13:44.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:44.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:44.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.485934+0000", 2023-09-23T23:13:44.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:44.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:44.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.485932+0000", 2023-09-23T23:13:44.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:13:44.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:44.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.485942+0000", 2023-09-23T23:13:44.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0434e-05 2023-09-23T23:13:44.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:44.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.485961+0000", 2023-09-23T23:13:44.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8805999999999999e-05 2023-09-23T23:13:44.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:44.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.485973+0000", 2023-09-23T23:13:44.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1357e-05 2023-09-23T23:13:44.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.486149+0000", 2023-09-23T23:13:44.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017626100000000001 2023-09-23T23:13:44.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:44.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.486270+0000", 2023-09-23T23:13:44.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012125099999999999 2023-09-23T23:13:44.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 7", 2023-09-23T23:13:44.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.486394+0000", 2023-09-23T23:13:44.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012359799999999999 2023-09-23T23:13:44.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:13:44.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.503263+0000", 2023-09-23T23:13:44.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016869400999999999 2023-09-23T23:13:44.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.505278+0000", 2023-09-23T23:13:44.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020146769999999999 2023-09-23T23:13:44.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:44.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.505295+0000", 2023-09-23T23:13:44.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7181e-05 2023-09-23T23:13:44.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.488722+0000", 2023-09-23T23:13:44.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98342742599999999 2023-09-23T23:13:44.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:44.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.488741+0000", 2023-09-23T23:13:44.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8278e-05 2023-09-23T23:13:44.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.859554+0000", 2023-09-23T23:13:44.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.370813383 2023-09-23T23:13:44.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:13:44.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.859680+0000", 2023-09-23T23:13:44.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012620599999999999 2023-09-23T23:13:44.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.875576+0000", 2023-09-23T23:13:44.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015895796 2023-09-23T23:13:44.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:44.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.875648+0000", 2023-09-23T23:13:44.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1631999999999995e-05 2023-09-23T23:13:44.039 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:44.039 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:44.039 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:44.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.040 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:186177 8.c 8:3e0e7239:::benchmark_data_smithi110_258128_object67359:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T23:13:44.040 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:58.485993+0000", 2023-09-23T23:13:44.040 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 165.40006750000001, 2023-09-23T23:13:44.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4017967730000001, 2023-09-23T23:13:44.040 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:44.040 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:13:44.040 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:44.040 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:13:44.040 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:13:44.040 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 186177 2023-09-23T23:13:44.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.041 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:44.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:44.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.485993+0000", 2023-09-23T23:13:44.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:44.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:44.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.485993+0000", 2023-09-23T23:13:44.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:44.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:44.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.485992+0000", 2023-09-23T23:13:44.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:13:44.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:44.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.485996+0000", 2023-09-23T23:13:44.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5619999999999999e-06 2023-09-23T23:13:44.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:44.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.486001+0000", 2023-09-23T23:13:44.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0660000000000003e-06 2023-09-23T23:13:44.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:44.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.486005+0000", 2023-09-23T23:13:44.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.642e-06 2023-09-23T23:13:44.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.491196+0000", 2023-09-23T23:13:44.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051913870000000004 2023-09-23T23:13:44.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:44.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.491333+0000", 2023-09-23T23:13:44.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000136744 2023-09-23T23:13:44.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 7", 2023-09-23T23:13:44.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.491467+0000", 2023-09-23T23:13:44.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013422999999999999 2023-09-23T23:13:44.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.505336+0000", 2023-09-23T23:13:44.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013869126000000001 2023-09-23T23:13:44.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:44.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.505355+0000", 2023-09-23T23:13:44.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.914e-05 2023-09-23T23:13:44.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.488958+0000", 2023-09-23T23:13:44.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98360241999999998 2023-09-23T23:13:44.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:44.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.488969+0000", 2023-09-23T23:13:44.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.169e-05 2023-09-23T23:13:44.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.859928+0000", 2023-09-23T23:13:44.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.370958594 2023-09-23T23:13:44.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:13:44.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.860027+0000", 2023-09-23T23:13:44.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9011000000000005e-05 2023-09-23T23:13:44.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.887695+0000", 2023-09-23T23:13:44.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.027667818 2023-09-23T23:13:44.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:44.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.887790+0000", 2023-09-23T23:13:44.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4838000000000002e-05 2023-09-23T23:13:44.048 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:44.048 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:44.048 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:44.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.048 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:186222 8.c 8:30266fd3:::benchmark_data_smithi110_258128_object67404:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T23:13:44.048 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:58.555499+0000", 2023-09-23T23:13:44.048 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 165.33056084699999, 2023-09-23T23:13:44.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3200290209999999, 2023-09-23T23:13:44.048 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:13:44.048 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:13:44.049 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:13:44.049 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:13:44.049 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:13:44.049 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 186222 2023-09-23T23:13:44.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.049 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:13:44.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:13:44.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.555499+0000", 2023-09-23T23:13:44.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:44.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:13:44.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.555499+0000", 2023-09-23T23:13:44.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:13:44.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:13:44.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.555498+0000", 2023-09-23T23:13:44.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:13:44.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:13:44.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.555503+0000", 2023-09-23T23:13:44.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9910000000000002e-06 2023-09-23T23:13:44.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:13:44.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.555514+0000", 2023-09-23T23:13:44.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0966e-05 2023-09-23T23:13:44.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:13:44.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.555520+0000", 2023-09-23T23:13:44.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3890000000000004e-06 2023-09-23T23:13:44.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.555588+0000", 2023-09-23T23:13:44.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8279000000000001e-05 2023-09-23T23:13:44.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:44.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.555600+0000", 2023-09-23T23:13:44.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1955e-05 2023-09-23T23:13:44.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.488994+0000", 2023-09-23T23:13:44.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93339440699999998 2023-09-23T23:13:44.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:13:44.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.489003+0000", 2023-09-23T23:13:44.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8589999999999994e-06 2023-09-23T23:13:44.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:13:44.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.860118+0000", 2023-09-23T23:13:44.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3711150220000001 2023-09-23T23:13:44.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:13:44.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.860272+0000", 2023-09-23T23:13:44.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015393399999999999 2023-09-23T23:13:44.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:13:44.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.860444+0000", 2023-09-23T23:13:44.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017180799999999999 2023-09-23T23:13:44.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:44.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.865472+0000", 2023-09-23T23:13:44.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0050281429999999997 2023-09-23T23:13:44.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:13:44.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.865485+0000", 2023-09-23T23:13:44.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2935000000000001e-05 2023-09-23T23:13:44.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:13:44.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.875356+0000", 2023-09-23T23:13:44.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0098710840000000005 2023-09-23T23:13:44.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:13:44.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.875415+0000", 2023-09-23T23:13:44.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8959000000000003e-05 2023-09-23T23:13:44.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:13:44.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:13:44.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:13:44.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.875528+0000", 2023-09-23T23:13:44.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000113288 2023-09-23T23:13:44.056 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:44.056 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:44.056 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:44.056 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:13:44.057 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:13:44.057 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:13:44.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:44.058 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:13:44.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:13:44.159 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:13:44.210 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:44.383 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:13:44.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:44.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:13:44.493 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:13:44.516 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 9 16 24405 24389 169.314 140.812 0.00616771 0.00589998 2023-09-23T23:13:44.557 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:44.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:44 smithi110 ceph-mon[212939]: pgmap v545: 89 pgs: 89 active+clean; 883 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 7 B/s, 0 objects/s recovering 2023-09-23T23:13:44.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:44 smithi110 ceph-mon[221102]: pgmap v545: 89 pgs: 89 active+clean; 883 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 7 B/s, 0 objects/s recovering 2023-09-23T23:13:44.792 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:44 smithi138 ceph-mon[193606]: pgmap v545: 89 pgs: 89 active+clean; 883 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 7 B/s, 0 objects/s recovering 2023-09-23T23:13:44.972 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:13:45.012 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:13:45.515 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 10 16 26367 26351 164.645 122.625 0.00569615 0.00606965 2023-09-23T23:13:46.405 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:13:46.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:46.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:13:46.515 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:13:46.516 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 11 16 28618 28602 162.466 140.688 0.00437623 0.00614912 2023-09-23T23:13:46.580 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:46.625 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:13:46.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:46.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:13:46.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:46 smithi138 ceph-mon[193606]: pgmap v546: 89 pgs: 89 active+clean; 1.3 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T23:13:46.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:46 smithi110 ceph-mon[212939]: pgmap v546: 89 pgs: 89 active+clean; 1.3 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T23:13:46.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:46 smithi110 ceph-mon[221102]: pgmap v546: 89 pgs: 89 active+clean; 1.3 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T23:13:46.728 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:13:46.847 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:47.142 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:13:47.187 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:13:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:13:46] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2023-09-23T23:13:47.253 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:13:47.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:47.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:13:47.417 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:47.427 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:13:47.470 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:47.517 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 12 16 29991 29975 156.062 85.8125 0.0130541 0.00640304 2023-09-23T23:13:47.657 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:47.718 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:13:47.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:47.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:13:47.826 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:13:47.899 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:48.178 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:13:48.178 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T23:13:48.178 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:13:48.428 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:48 smithi110 ceph-mon[221102]: pgmap v547: 89 pgs: 89 active+clean; 1.4 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-09-23T23:13:48.428 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:48 smithi110 ceph-mon[212939]: pgmap v547: 89 pgs: 89 active+clean; 1.4 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-09-23T23:13:48.517 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 13 16 31674 31658 152.149 105.188 0.00762166 0.00656702 2023-09-23T23:13:48.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:48 smithi138 ceph-mon[193606]: pgmap v547: 89 pgs: 89 active+clean; 1.4 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-09-23T23:13:48.955 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:13:48.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:48.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:13:49.063 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:13:49.156 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:49.508 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:13:49.517 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 14 16 33650 33634 150.102 123.5 0.0120906 0.00665679 2023-09-23T23:13:49.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:49.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:13:49.619 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:13:49.711 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:49.789 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:13:49.795 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:49 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:49.795 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:49 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:49.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:49.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:13:49.897 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:13:49.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:49 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:49.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:49 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:49.944 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:49 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:49.945 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:49 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:49.950 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:50.517 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 15 16 35660 35644 148.47 125.625 0.00788435 0.00673094 2023-09-23T23:13:50.950 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:50 smithi138 ceph-mon[193606]: pgmap v548: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 169 MiB/s wr, 2.70k op/s 2023-09-23T23:13:50.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:50 smithi110 ceph-mon[212939]: pgmap v548: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 169 MiB/s wr, 2.70k op/s 2023-09-23T23:13:50.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:50 smithi110 ceph-mon[221102]: pgmap v548: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 169 MiB/s wr, 2.70k op/s 2023-09-23T23:13:51.400 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:13:51.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:51.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:13:51.506 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:13:51.517 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 16 16 37207 37191 145.234 96.6875 0.0063116 0.00688074 2023-09-23T23:13:51.556 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:51.661 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:13:51.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:51.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:13:51.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:51 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:13:51.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:51 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:13:51.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:51 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:51.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:51 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:13:51.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:51 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:13:51.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:51 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:51.775 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:13:51.813 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:51 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:13:51.814 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:51 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:13:51.814 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:51 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:13:51.816 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:51.858 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:13:51.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:51.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:13:51.963 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:13:51.990 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:52.380 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:13:52.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:52.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:13:52.495 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:13:52.517 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 17 16 38641 38625 141.963 89.625 0.0182086 0.00703855 2023-09-23T23:13:52.559 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:52.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:52 smithi110 ceph-mon[212939]: pgmap v549: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2023-09-23T23:13:52.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:52 smithi110 ceph-mon[221102]: pgmap v549: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2023-09-23T23:13:52.746 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:52.772 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:13:52.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:52.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:13:52.876 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:13:52.934 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:52.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:52 smithi138 ceph-mon[193606]: pgmap v549: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2023-09-23T23:13:53.085 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:13:53.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:53.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:13:53.192 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:13:53.257 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:53.431 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:13:53.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:53.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:13:53.480 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:13:53.517 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 18 16 40081 40065 139.076 90 0.00522242 0.00718485 2023-09-23T23:13:53.522 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:13:53.549 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:13:53.562 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:13:53.606 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:13:53.639 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:13:53.651 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:53.684 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:13:53.733 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:13:53.789 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:13:53.792 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:13:53.793 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:13:53.793 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:13:53.793 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:13:53.793 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:13:53.793 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:13:53.793 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:13:53.793 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:13:53.867 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:13:53.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:53.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:13:53.975 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:13:54.025 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:54.182 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:54 smithi110 ceph-mon[212939]: pgmap v550: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T23:13:54.183 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:54 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1662919188' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:13:54.183 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:54 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/895861302' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:13:54.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:54 smithi110 ceph-mon[221102]: pgmap v550: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T23:13:54.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:54 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1662919188' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:13:54.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:54 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/895861302' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:13:54.202 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:13:54.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:54 smithi138 ceph-mon[193606]: pgmap v550: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T23:13:54.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:54 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1662919188' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:13:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:54 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/895861302' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:13:54.517 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 19 16 41710 41694 137.115 101.812 0.0075306 0.00728788 2023-09-23T23:13:55.186 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:13:55.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:55.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:13:55.296 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:13:55.352 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:55.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:55 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:13:55.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:55 smithi138 ceph-mon[193606]: failed to encode map e170 with expected crc 2023-09-23T23:13:55.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:55 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/895861302' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:13:55.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:55 smithi138 ceph-mon[193606]: osdmap e170: 8 total, 8 up, 8 in 2023-09-23T23:13:55.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:55 smithi138 ceph-mon[193606]: failed to encode map e170 with expected crc 2023-09-23T23:13:55.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:55 smithi138 ceph-mon[193606]: failed to encode map e170 with expected crc 2023-09-23T23:13:55.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:55 smithi138 ceph-mon[193606]: failed to encode map e170 with expected crc 2023-09-23T23:13:55.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:55 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:13:55.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:55 smithi110 ceph-mon[212939]: failed to encode map e170 with expected crc 2023-09-23T23:13:55.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:55 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/895861302' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:13:55.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:55 smithi110 ceph-mon[212939]: osdmap e170: 8 total, 8 up, 8 in 2023-09-23T23:13:55.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:55 smithi110 ceph-mon[212939]: failed to encode map e170 with expected crc 2023-09-23T23:13:55.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:55 smithi110 ceph-mon[212939]: failed to encode map e170 with expected crc 2023-09-23T23:13:55.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:55 smithi110 ceph-mon[212939]: failed to encode map e170 with expected crc 2023-09-23T23:13:55.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:55 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:13:55.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:55 smithi110 ceph-mon[221102]: failed to encode map e170 with expected crc 2023-09-23T23:13:55.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:55 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/895861302' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:13:55.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:55 smithi110 ceph-mon[221102]: osdmap e170: 8 total, 8 up, 8 in 2023-09-23T23:13:55.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:55 smithi110 ceph-mon[221102]: failed to encode map e170 with expected crc 2023-09-23T23:13:55.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:55 smithi110 ceph-mon[221102]: failed to encode map e170 with expected crc 2023-09-23T23:13:55.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:55 smithi110 ceph-mon[221102]: failed to encode map e170 with expected crc 2023-09-23T23:13:55.517 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:13:55.517435+0000 min lat: 0.00117478 max lat: 0.22385 avg lat: 0.00742306 2023-09-23T23:13:55.517 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:13:55.517 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 20 16 43054 43038 134.459 84 0.050627 0.00742306 2023-09-23T23:13:55.676 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:13:55.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:55.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:13:55.779 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:13:55.835 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:56.088 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:13:56.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:56.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:13:56.192 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:13:56.236 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:56.438 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:56 smithi110 ceph-mon[212939]: pgmap v552: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T23:13:56.439 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:56 smithi110 ceph-mon[221102]: pgmap v552: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T23:13:56.517 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 21 16 44085 44069 131.125 64.4375 0.005246 0.00762242 2023-09-23T23:13:56.637 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:13:56.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:56.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:13:56.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:56 smithi138 ceph-mon[193606]: pgmap v552: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T23:13:56.744 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:13:56.804 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:56.930 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:13:56.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:56.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:13:57.041 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:13:57.101 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:13:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:13:56] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2023-09-23T23:13:57.107 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:57.210 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:13:57.211 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:13:57.216 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:13:57.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:57.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:13:57.329 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:13:57.368 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:57.460 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:13:57.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:57.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:13:57.512 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T23:13:57.512 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:13:57.512 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T23:13:57.513 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:13:57.517 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 22 16 45644 45628 129.593 97.4375 0.00433153 0.00771196 2023-09-23T23:13:57.569 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:13:57.637 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:57.779 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:13:57.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:57.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:13:57.891 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:13:57.935 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:58.024 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:13:58.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:58.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:13:58.134 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:13:58.183 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:58.255 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:13:58.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:58.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:13:58.364 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:13:58.420 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:58.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:58 smithi138 ceph-mon[193606]: pgmap v553: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T23:13:58.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:13:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:13:58.505 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:13:58.517 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 23 16 47218 47202 128.236 98.375 0.00980663 0.0077928 2023-09-23T23:13:58.563 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:13:58.575 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:13:58.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:58.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:13:58.682 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:13:58.690 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:13:58.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:58 smithi110 ceph-mon[212939]: pgmap v553: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T23:13:58.699 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:13:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:13:58.700 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:58 smithi110 ceph-mon[221102]: pgmap v553: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T23:13:58.700 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:13:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:13:58.710 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:13:58.887 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:59.136 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:13:59.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:59.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:13:59.242 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:13:59.289 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:59.364 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:13:59.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:59.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:13:59.472 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:13:59.515 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:59.518 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 24 16 48474 48458 126.164 78.5 0.0133727 0.0079203 2023-09-23T23:13:59.665 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:13:59.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:59.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:13:59.773 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:13:59.831 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:13:59.875 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:13:59.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:13:59.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:13:59.978 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:00.025 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:00.112 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:00.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:00.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:00.220 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:00.245 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:00.287 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:00.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:00.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:00.390 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:00.416 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:00.457 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:00.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:00.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:00.518 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 25 16 49509 49493 123.704 64.6875 0.00270886 0.00805554 2023-09-23T23:14:00.560 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:14:00.613 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:00.689 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:14:00.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:00 smithi138 ceph-mon[193606]: pgmap v554: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T23:14:00.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:00.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:14:00.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:00 smithi110 ceph-mon[212939]: pgmap v554: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T23:14:00.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:00 smithi110 ceph-mon[221102]: pgmap v554: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T23:14:00.798 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:14:00.853 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:00.923 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:00.954 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:14:00.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:00.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:14:00.986 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:00.994 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:01.059 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:14:01.090 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:01.167 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:14:01.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:01.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:14:01.273 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:01.302 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:01.403 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:01.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:01.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:01.510 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:14:01.518 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 26 16 50944 50928 122.396 89.6875 0.00750644 0.00816408 2023-09-23T23:14:01.593 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:02.518 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 27 16 52333 52317 121.078 86.8125 0.014414 0.00825373 2023-09-23T23:14:02.577 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:14:02.586 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:14:02.618 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:02 smithi110 ceph-mon[212939]: pgmap v555: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T23:14:02.618 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:02 smithi110 ceph-mon[221102]: pgmap v555: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T23:14:02.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:02 smithi138 ceph-mon[193606]: pgmap v555: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T23:14:03.518 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 28 16 53338 53322 118.997 62.8125 0.00915626 0.00839925 2023-09-23T23:14:03.572 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:14:03.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:03.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:14:03.682 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:14:03.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:03 smithi138 ceph-mon[193606]: from='client.45185 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:14:03.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:03 smithi110 ceph-mon[221102]: from='client.45185 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:14:03.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:03 smithi110 ceph-mon[212939]: from='client.45185 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:14:03.729 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:03.848 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:14:03.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:03.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:14:03.957 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:04.003 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:04.092 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:04.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:04.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:04.197 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:04.245 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:04.290 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:04.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:04.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:04.397 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:14:04.422 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:04.471 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:14:04.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:04.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:14:04.518 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 29 16 54440 54424 117.269 68.875 0.00704885 0.00849185 2023-09-23T23:14:04.577 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:04.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:04 smithi138 ceph-mon[193606]: pgmap v556: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-09-23T23:14:04.614 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:04.700 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:04.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:04 smithi110 ceph-mon[212939]: pgmap v556: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-09-23T23:14:04.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:04 smithi110 ceph-mon[221102]: pgmap v556: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-09-23T23:14:04.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:04.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:04.807 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:14:04.858 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:04.945 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:14:04.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:04.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:14:05.052 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:14:05.082 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:05.518 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 30 16 55530 55514 115.63 68.125 0.009603 0.00864305 2023-09-23T23:14:05.535 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:14:05.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:05.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:14:05.642 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:05.703 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:05.764 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:05.765 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":555,"stamp":"2023-09-23T23:14:01.300718+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3107723168,"num_objects":47435,"num_object_clones":0,"num_object_copies":142305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":47435,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":47711,"num_write_kb":3036749,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":1771174,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":47627,"ondisk_log_size":47627,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11370380,"kb_used_data":8930572,"kb_used_omap":20,"kb_used_meta":2439723,"kb_avail":738590836,"statfs":{"total":767960285184,"available":756317016064,"internally_reserved":0,"allocated":9144905728,"data_stored":9140024410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2498277006},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1002373120,"num_objects":15295,"num_object_clones":0,"num_object_copies":45885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15295,"num_write_kb":978880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002566"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":149,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.407568+0000","last_change":"2023-09-23T23:08:45.116934+0000","last_active":"2023-09-23T23:13:39.407568+0000","last_peered":"2023-09-23T23:13:39.407568+0000","last_clean":"2023-09-23T23:13:39.407568+0000","last_became_active":"2023-09-23T23:08:45.116576+0000","last_became_peered":"2023-09-23T23:08:45.116576+0000","last_unstale":"2023-09-23T23:13:39.407568+0000","last_undegraded":"2023-09-23T23:13:39.407568+0000","last_fullsized":"2023-09-23T23:13:39.407568+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:06.398968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":108,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.405301+0000","last_change":"2023-09-23T23:10:59.514211+0000","last_active":"2023-09-23T23:13:39.405301+0000","last_peered":"2023-09-23T23:13:39.405301+0000","last_clean":"2023-09-23T23:13:39.405301+0000","last_became_active":"2023-09-23T23:10:59.513917+0000","last_became_peered":"2023-09-23T23:10:59.513917+0000","last_unstale":"2023-09-23T23:13:39.405301+0000","last_undegraded":"2023-09-23T23:13:39.405301+0000","last_fullsized":"2023-09-23T23:13:39.405301+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:55.533586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":138,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.407762+0000","last_change":"2023-09-23T23:10:59.505490+0000","last_active":"2023-09-23T23:13:39.407762+0000","last_peered":"2023-09-23T23:13:39.407762+0000","last_clean":"2023-09-23T23:13:39.407762+0000","last_became_active":"2023-09-23T23:10:59.504874+0000","last_became_peered":"2023-09-23T23:10:59.504874+0000","last_unstale":"2023-09-23T23:13:39.407762+0000","last_undegraded":"2023-09-23T23:13:39.407762+0000","last_fullsized":"2023-09-23T23:13:39.407762+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:15.279668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":119,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.412116+0000","last_change":"2023-09-23T23:10:59.506092+0000","last_active":"2023-09-23T23:13:39.412116+0000","last_peered":"2023-09-23T23:13:39.412116+0000","last_clean":"2023-09-23T23:13:39.412116+0000","last_became_active":"2023-09-23T23:10:59.505550+0000","last_became_peered":"2023-09-23T23:10:59.505550+0000","last_unstale":"2023-09-23T23:13:39.412116+0000","last_undegraded":"2023-09-23T23:13:39.412116+0000","last_fullsized":"2023-09-23T23:13:39.412116+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:10.576890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":126,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.404603+0000","last_change":"2023-09-23T23:10:59.515675+0000","last_active":"2023-09-23T23:13:39.404603+0000","last_peered":"2023-09-23T23:13:39.404603+0000","last_clean":"2023-09-23T23:13:39.404603+0000","last_became_active":"2023-09-23T23:10:59.514600+0000","last_became_peered":"2023-09-23T23:10:59.514600+0000","last_unstale":"2023-09-23T23:13:39.404603+0000","last_undegraded":"2023-09-23T23:13:39.404603+0000","last_fullsized":"2023-09-23T23:13:39.404603+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.273467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":130,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.412060+0000","last_change":"2023-09-23T23:07:30.078179+0000","last_active":"2023-09-23T23:13:39.412060+0000","last_peered":"2023-09-23T23:13:39.412060+0000","last_clean":"2023-09-23T23:13:39.412060+0000","last_became_active":"2023-09-23T23:07:30.077810+0000","last_became_peered":"2023-09-23T23:07:30.077810+0000","last_unstale":"2023-09-23T23:13:39.412060+0000","last_undegraded":"2023-09-23T23:13:39.412060+0000","last_fullsized":"2023-09-23T23:13:39.412060+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:19.218333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":119,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:50.446339+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:13:50.446339+0000","last_peered":"2023-09-23T23:13:50.446339+0000","last_clean":"2023-09-23T23:13:50.446339+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:13:50.446339+0000","last_undegraded":"2023-09-23T23:13:50.446339+0000","last_fullsized":"2023-09-23T23:13:50.446339+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":108,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.404675+0000","last_change":"2023-09-23T23:10:59.515840+0000","last_active":"2023-09-23T23:13:39.404675+0000","last_peered":"2023-09-23T23:13:39.404675+0000","last_clean":"2023-09-23T23:13:39.404675+0000","last_became_active":"2023-09-23T23:10:59.515000+0000","last_became_peered":"2023-09-23T23:10:59.515000+0000","last_unstale":"2023-09-23T23:13:39.404675+0000","last_undegraded":"2023-09-23T23:13:39.404675+0000","last_fullsized":"2023-09-23T23:13:39.404675+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:47.355624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":131,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408515+0000","last_change":"2023-09-23T23:07:30.076036+0000","last_active":"2023-09-23T23:13:39.408515+0000","last_peered":"2023-09-23T23:13:39.408515+0000","last_clean":"2023-09-23T23:13:39.408515+0000","last_became_active":"2023-09-23T23:07:30.075733+0000","last_became_peered":"2023-09-23T23:07:30.075733+0000","last_unstale":"2023-09-23T23:13:39.408515+0000","last_undegraded":"2023-09-23T23:13:39.408515+0000","last_fullsized":"2023-09-23T23:13:39.408515+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:56.342424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":87,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.404890+0000","last_change":"2023-09-23T23:10:59.517290+0000","last_active":"2023-09-23T23:13:39.404890+0000","last_peered":"2023-09-23T23:13:39.404890+0000","last_clean":"2023-09-23T23:13:39.404890+0000","last_became_active":"2023-09-23T23:10:59.513367+0000","last_became_peered":"2023-09-23T23:10:59.513367+0000","last_unstale":"2023-09-23T23:13:39.404890+0000","last_undegraded":"2023-09-23T23:13:39.404890+0000","last_fullsized":"2023-09-23T23:13:39.404890+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:04.328676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":134,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408768+0000","last_change":"2023-09-23T23:10:59.500837+0000","last_active":"2023-09-23T23:13:39.408768+0000","last_peered":"2023-09-23T23:13:39.408768+0000","last_clean":"2023-09-23T23:13:39.408768+0000","last_became_active":"2023-09-23T23:10:59.500455+0000","last_became_peered":"2023-09-23T23:10:59.500455+0000","last_unstale":"2023-09-23T23:13:39.408768+0000","last_undegraded":"2023-09-23T23:13:39.408768+0000","last_fullsized":"2023-09-23T23:13:39.408768+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:34.801149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":137,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408919+0000","last_change":"2023-09-23T23:08:45.122701+0000","last_active":"2023-09-23T23:13:39.408919+0000","last_peered":"2023-09-23T23:13:39.408919+0000","last_clean":"2023-09-23T23:13:39.408919+0000","last_became_active":"2023-09-23T23:08:45.122332+0000","last_became_peered":"2023-09-23T23:08:45.122332+0000","last_unstale":"2023-09-23T23:13:39.408919+0000","last_undegraded":"2023-09-23T23:13:39.408919+0000","last_fullsized":"2023-09-23T23:13:39.408919+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:05.771906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":136,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408371+0000","last_change":"2023-09-23T22:53:16.891930+0000","last_active":"2023-09-23T23:13:39.408371+0000","last_peered":"2023-09-23T23:13:39.408371+0000","last_clean":"2023-09-23T23:13:39.408371+0000","last_became_active":"2023-09-23T22:53:16.891658+0000","last_became_peered":"2023-09-23T22:53:16.891658+0000","last_unstale":"2023-09-23T23:13:39.408371+0000","last_undegraded":"2023-09-23T23:13:39.408371+0000","last_fullsized":"2023-09-23T23:13:39.408371+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:19.762775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":130,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408236+0000","last_change":"2023-09-23T23:07:30.082476+0000","last_active":"2023-09-23T23:13:39.408236+0000","last_peered":"2023-09-23T23:13:39.408236+0000","last_clean":"2023-09-23T23:13:39.408236+0000","last_became_active":"2023-09-23T23:07:30.082201+0000","last_became_peered":"2023-09-23T23:07:30.082201+0000","last_unstale":"2023-09-23T23:13:39.408236+0000","last_undegraded":"2023-09-23T23:13:39.408236+0000","last_fullsized":"2023-09-23T23:13:39.408236+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:08.809163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":133,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408994+0000","last_change":"2023-09-23T23:08:45.115767+0000","last_active":"2023-09-23T23:13:39.408994+0000","last_peered":"2023-09-23T23:13:39.408994+0000","last_clean":"2023-09-23T23:13:39.408994+0000","last_became_active":"2023-09-23T23:08:45.115422+0000","last_became_peered":"2023-09-23T23:08:45.115422+0000","last_unstale":"2023-09-23T23:13:39.408994+0000","last_undegraded":"2023-09-23T23:13:39.408994+0000","last_fullsized":"2023-09-23T23:13:39.408994+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:53.216816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":108,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.406014+0000","last_change":"2023-09-23T23:10:59.518106+0000","last_active":"2023-09-23T23:13:39.406014+0000","last_peered":"2023-09-23T23:13:39.406014+0000","last_clean":"2023-09-23T23:13:39.406014+0000","last_became_active":"2023-09-23T23:10:59.517633+0000","last_became_peered":"2023-09-23T23:10:59.517633+0000","last_unstale":"2023-09-23T23:13:39.406014+0000","last_undegraded":"2023-09-23T23:13:39.406014+0000","last_fullsized":"2023-09-23T23:13:39.406014+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:27.245420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":119,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.411924+0000","last_change":"2023-09-23T23:10:59.516989+0000","last_active":"2023-09-23T23:13:39.411924+0000","last_peered":"2023-09-23T23:13:39.411924+0000","last_clean":"2023-09-23T23:13:39.411924+0000","last_became_active":"2023-09-23T23:10:59.516507+0000","last_became_peered":"2023-09-23T23:10:59.516507+0000","last_unstale":"2023-09-23T23:13:39.411924+0000","last_undegraded":"2023-09-23T23:13:39.411924+0000","last_fullsized":"2023-09-23T23:13:39.411924+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:40.834591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":117,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408867+0000","last_change":"2023-09-23T23:13:29.360403+0000","last_active":"2023-09-23T23:13:39.408867+0000","last_peered":"2023-09-23T23:13:39.408867+0000","last_clean":"2023-09-23T23:13:39.408867+0000","last_became_active":"2023-09-23T23:13:29.359519+0000","last_became_peered":"2023-09-23T23:13:29.359519+0000","last_unstale":"2023-09-23T23:13:39.408867+0000","last_undegraded":"2023-09-23T23:13:39.408867+0000","last_fullsized":"2023-09-23T23:13:39.408867+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:27.911252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":114,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408214+0000","last_change":"2023-09-23T22:53:16.889362+0000","last_active":"2023-09-23T23:13:39.408214+0000","last_peered":"2023-09-23T23:13:39.408214+0000","last_clean":"2023-09-23T23:13:39.408214+0000","last_became_active":"2023-09-23T22:53:16.889081+0000","last_became_peered":"2023-09-23T22:53:16.889081+0000","last_unstale":"2023-09-23T23:13:39.408214+0000","last_undegraded":"2023-09-23T23:13:39.408214+0000","last_fullsized":"2023-09-23T23:13:39.408214+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:11.321151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":119,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:50.446312+0000","last_change":"2023-09-23T23:10:59.506198+0000","last_active":"2023-09-23T23:13:50.446312+0000","last_peered":"2023-09-23T23:13:50.446312+0000","last_clean":"2023-09-23T23:13:50.446312+0000","last_became_active":"2023-09-23T23:10:59.505912+0000","last_became_peered":"2023-09-23T23:10:59.505912+0000","last_unstale":"2023-09-23T23:13:50.446312+0000","last_undegraded":"2023-09-23T23:13:50.446312+0000","last_fullsized":"2023-09-23T23:13:50.446312+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:30.343257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":159,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.410771+0000","last_change":"2023-09-23T23:13:29.368175+0000","last_active":"2023-09-23T23:13:39.410771+0000","last_peered":"2023-09-23T23:13:39.410771+0000","last_clean":"2023-09-23T23:13:39.410771+0000","last_became_active":"2023-09-23T23:13:29.363603+0000","last_became_peered":"2023-09-23T23:13:29.363603+0000","last_unstale":"2023-09-23T23:13:39.410771+0000","last_undegraded":"2023-09-23T23:13:39.410771+0000","last_fullsized":"2023-09-23T23:13:39.410771+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:25.277871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":108,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.405590+0000","last_change":"2023-09-23T23:10:59.525228+0000","last_active":"2023-09-23T23:13:39.405590+0000","last_peered":"2023-09-23T23:13:39.405590+0000","last_clean":"2023-09-23T23:13:39.405590+0000","last_became_active":"2023-09-23T23:10:59.524665+0000","last_became_peered":"2023-09-23T23:10:59.524665+0000","last_unstale":"2023-09-23T23:13:39.405590+0000","last_undegraded":"2023-09-23T23:13:39.405590+0000","last_fullsized":"2023-09-23T23:13:39.405590+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:42.632316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"170'3071","reported_seq":3086,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:14:00.212759+0000","last_change":"2023-09-23T23:13:28.360242+0000","last_active":"2023-09-23T23:14:00.212759+0000","last_peered":"2023-09-23T23:14:00.212759+0000","last_clean":"2023-09-23T23:14:00.212759+0000","last_became_active":"2023-09-23T23:13:28.359368+0000","last_became_peered":"2023-09-23T23:13:28.359368+0000","last_unstale":"2023-09-23T23:14:00.212759+0000","last_undegraded":"2023-09-23T23:14:00.212759+0000","last_fullsized":"2023-09-23T23:14:00.212759+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":3071,"log_dups_size":0,"ondisk_log_size":3071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:57.722768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201261056,"num_objects":3071,"num_object_clones":0,"num_object_copies":9213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3071,"num_write_kb":196544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":132,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.407938+0000","last_change":"2023-09-23T23:04:11.907427+0000","last_active":"2023-09-23T23:13:39.407938+0000","last_peered":"2023-09-23T23:13:39.407938+0000","last_clean":"2023-09-23T23:13:39.407938+0000","last_became_active":"2023-09-23T23:04:11.907016+0000","last_became_peered":"2023-09-23T23:04:11.907016+0000","last_unstale":"2023-09-23T23:13:39.407938+0000","last_undegraded":"2023-09-23T23:13:39.407938+0000","last_fullsized":"2023-09-23T23:13:39.407938+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:20.715977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":18,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.407287+0000","last_change":"2023-09-23T23:13:29.421286+0000","last_active":"2023-09-23T23:13:39.407287+0000","last_peered":"2023-09-23T23:13:39.407287+0000","last_clean":"2023-09-23T23:13:39.407287+0000","last_became_active":"2023-09-23T23:13:29.420967+0000","last_became_peered":"2023-09-23T23:13:29.420967+0000","last_unstale":"2023-09-23T23:13:39.407287+0000","last_undegraded":"2023-09-23T23:13:39.407287+0000","last_fullsized":"2023-09-23T23:13:39.407287+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:59.756339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"170'3049","reported_seq":3064,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:13:58.346485+0000","last_change":"2023-09-23T23:13:28.360159+0000","last_active":"2023-09-23T23:13:58.346485+0000","last_peered":"2023-09-23T23:13:58.346485+0000","last_clean":"2023-09-23T23:13:58.346485+0000","last_became_active":"2023-09-23T23:13:28.359915+0000","last_became_peered":"2023-09-23T23:13:28.359915+0000","last_unstale":"2023-09-23T23:13:58.346485+0000","last_undegraded":"2023-09-23T23:13:58.346485+0000","last_fullsized":"2023-09-23T23:13:58.346485+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":3049,"log_dups_size":0,"ondisk_log_size":3049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:48.297395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199819264,"num_objects":3049,"num_object_clones":0,"num_object_copies":9147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3049,"num_write_kb":195136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":133,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.407715+0000","last_change":"2023-09-23T23:07:30.072322+0000","last_active":"2023-09-23T23:13:39.407715+0000","last_peered":"2023-09-23T23:13:39.407715+0000","last_clean":"2023-09-23T23:13:39.407715+0000","last_became_active":"2023-09-23T23:07:30.071941+0000","last_became_peered":"2023-09-23T23:07:30.071941+0000","last_unstale":"2023-09-23T23:13:39.407715+0000","last_undegraded":"2023-09-23T23:13:39.407715+0000","last_fullsized":"2023-09-23T23:13:39.407715+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:38.994624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":147,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.407607+0000","last_change":"2023-09-23T23:10:59.512748+0000","last_active":"2023-09-23T23:13:39.407607+0000","last_peered":"2023-09-23T23:13:39.407607+0000","last_clean":"2023-09-23T23:13:39.407607+0000","last_became_active":"2023-09-23T23:10:59.512420+0000","last_became_peered":"2023-09-23T23:10:59.512420+0000","last_unstale":"2023-09-23T23:13:39.407607+0000","last_undegraded":"2023-09-23T23:13:39.407607+0000","last_fullsized":"2023-09-23T23:13:39.407607+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:09.269366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"170'2894","reported_seq":2908,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:13:58.825493+0000","last_change":"2023-09-23T23:13:28.354971+0000","last_active":"2023-09-23T23:13:58.825493+0000","last_peered":"2023-09-23T23:13:58.825493+0000","last_clean":"2023-09-23T23:13:58.825493+0000","last_became_active":"2023-09-23T23:13:28.354731+0000","last_became_peered":"2023-09-23T23:13:28.354731+0000","last_unstale":"2023-09-23T23:13:58.825493+0000","last_undegraded":"2023-09-23T23:13:58.825493+0000","last_fullsized":"2023-09-23T23:13:58.825493+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":2894,"log_dups_size":0,"ondisk_log_size":2894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:20.209676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189661184,"num_objects":2894,"num_object_clones":0,"num_object_copies":8682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2894,"num_write_kb":185216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":129,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408513+0000","last_change":"2023-09-23T23:13:29.362820+0000","last_active":"2023-09-23T23:13:39.408513+0000","last_peered":"2023-09-23T23:13:39.408513+0000","last_clean":"2023-09-23T23:13:39.408513+0000","last_became_active":"2023-09-23T23:13:29.360151+0000","last_became_peered":"2023-09-23T23:13:29.360151+0000","last_unstale":"2023-09-23T23:13:39.408513+0000","last_undegraded":"2023-09-23T23:13:39.408513+0000","last_fullsized":"2023-09-23T23:13:39.408513+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:36.242850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":140,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.411204+0000","last_change":"2023-09-23T23:08:45.116813+0000","last_active":"2023-09-23T23:13:39.411204+0000","last_peered":"2023-09-23T23:13:39.411204+0000","last_clean":"2023-09-23T23:13:39.411204+0000","last_became_active":"2023-09-23T23:08:45.116435+0000","last_became_peered":"2023-09-23T23:08:45.116435+0000","last_unstale":"2023-09-23T23:13:39.411204+0000","last_undegraded":"2023-09-23T23:13:39.411204+0000","last_fullsized":"2023-09-23T23:13:39.411204+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:40.616551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"170'3016","reported_seq":3030,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:13:58.835974+0000","last_change":"2023-09-23T23:13:28.505667+0000","last_active":"2023-09-23T23:13:58.835974+0000","last_peered":"2023-09-23T23:13:58.835974+0000","last_clean":"2023-09-23T23:13:58.835974+0000","last_became_active":"2023-09-23T23:13:28.505309+0000","last_became_peered":"2023-09-23T23:13:28.505309+0000","last_unstale":"2023-09-23T23:13:58.835974+0000","last_undegraded":"2023-09-23T23:13:58.835974+0000","last_fullsized":"2023-09-23T23:13:58.835974+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":3016,"log_dups_size":0,"ondisk_log_size":3016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:18.432866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197656576,"num_objects":3016,"num_object_clones":0,"num_object_copies":9048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3016,"num_write_kb":193024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":131,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408392+0000","last_change":"2023-09-23T23:10:59.502426+0000","last_active":"2023-09-23T23:13:39.408392+0000","last_peered":"2023-09-23T23:13:39.408392+0000","last_clean":"2023-09-23T23:13:39.408392+0000","last_became_active":"2023-09-23T23:10:59.502074+0000","last_became_peered":"2023-09-23T23:10:59.502074+0000","last_unstale":"2023-09-23T23:13:39.408392+0000","last_undegraded":"2023-09-23T23:13:39.408392+0000","last_fullsized":"2023-09-23T23:13:39.408392+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:46.029347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":122,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.412955+0000","last_change":"2023-09-23T23:04:11.912493+0000","last_active":"2023-09-23T23:13:39.412955+0000","last_peered":"2023-09-23T23:13:39.412955+0000","last_clean":"2023-09-23T23:13:39.412955+0000","last_became_active":"2023-09-23T23:04:11.906712+0000","last_became_peered":"2023-09-23T23:04:11.906712+0000","last_unstale":"2023-09-23T23:13:39.412955+0000","last_undegraded":"2023-09-23T23:13:39.412955+0000","last_fullsized":"2023-09-23T23:13:39.412955+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:58.932610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"170'2855","reported_seq":2870,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:13:57.557885+0000","last_change":"2023-09-23T23:13:28.358022+0000","last_active":"2023-09-23T23:13:57.557885+0000","last_peered":"2023-09-23T23:13:57.557885+0000","last_clean":"2023-09-23T23:13:57.557885+0000","last_became_active":"2023-09-23T23:13:28.357728+0000","last_became_peered":"2023-09-23T23:13:28.357728+0000","last_unstale":"2023-09-23T23:13:57.557885+0000","last_undegraded":"2023-09-23T23:13:57.557885+0000","last_fullsized":"2023-09-23T23:13:57.557885+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":2855,"log_dups_size":0,"ondisk_log_size":2855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:08.765537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187105280,"num_objects":2855,"num_object_clones":0,"num_object_copies":8565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2855,"num_write_kb":182720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":132,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.404643+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:13:39.404643+0000","last_peered":"2023-09-23T23:13:39.404643+0000","last_clean":"2023-09-23T23:13:39.404643+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:13:39.404643+0000","last_undegraded":"2023-09-23T23:13:39.404643+0000","last_fullsized":"2023-09-23T23:13:39.404643+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":130,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408336+0000","last_change":"2023-09-23T23:10:59.512363+0000","last_active":"2023-09-23T23:13:39.408336+0000","last_peered":"2023-09-23T23:13:39.408336+0000","last_clean":"2023-09-23T23:13:39.408336+0000","last_became_active":"2023-09-23T23:10:59.512022+0000","last_became_peered":"2023-09-23T23:10:59.512022+0000","last_unstale":"2023-09-23T23:13:39.408336+0000","last_undegraded":"2023-09-23T23:13:39.408336+0000","last_fullsized":"2023-09-23T23:13:39.408336+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:45.523432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"170'2946","reported_seq":2958,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:13:58.349410+0000","last_change":"2023-09-23T23:13:28.505728+0000","last_active":"2023-09-23T23:13:58.349410+0000","last_peered":"2023-09-23T23:13:58.349410+0000","last_clean":"2023-09-23T23:13:58.349410+0000","last_became_active":"2023-09-23T23:13:28.505371+0000","last_became_peered":"2023-09-23T23:13:28.505371+0000","last_unstale":"2023-09-23T23:13:58.349410+0000","last_undegraded":"2023-09-23T23:13:58.349410+0000","last_fullsized":"2023-09-23T23:13:58.349410+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":2946,"log_dups_size":0,"ondisk_log_size":2946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:32.082686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193069056,"num_objects":2946,"num_object_clones":0,"num_object_copies":8838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2946,"num_write_kb":188544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":131,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.411271+0000","last_change":"2023-09-23T23:07:30.077349+0000","last_active":"2023-09-23T23:13:39.411271+0000","last_peered":"2023-09-23T23:13:39.411271+0000","last_clean":"2023-09-23T23:13:39.411271+0000","last_became_active":"2023-09-23T23:07:30.077031+0000","last_became_peered":"2023-09-23T23:07:30.077031+0000","last_unstale":"2023-09-23T23:13:39.411271+0000","last_undegraded":"2023-09-23T23:13:39.411271+0000","last_fullsized":"2023-09-23T23:13:39.411271+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:27.708541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":147,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.407911+0000","last_change":"2023-09-23T23:08:45.117579+0000","last_active":"2023-09-23T23:13:39.407911+0000","last_peered":"2023-09-23T23:13:39.407911+0000","last_clean":"2023-09-23T23:13:39.407911+0000","last_became_active":"2023-09-23T23:08:45.117216+0000","last_became_peered":"2023-09-23T23:08:45.117216+0000","last_unstale":"2023-09-23T23:13:39.407911+0000","last_undegraded":"2023-09-23T23:13:39.407911+0000","last_fullsized":"2023-09-23T23:13:39.407911+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:27.731522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"170'2928","reported_seq":2942,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:13:58.851829+0000","last_change":"2023-09-23T23:13:28.356430+0000","last_active":"2023-09-23T23:13:58.851829+0000","last_peered":"2023-09-23T23:13:58.851829+0000","last_clean":"2023-09-23T23:13:58.851829+0000","last_became_active":"2023-09-23T23:13:28.356172+0000","last_became_peered":"2023-09-23T23:13:28.356172+0000","last_unstale":"2023-09-23T23:13:58.851829+0000","last_undegraded":"2023-09-23T23:13:58.851829+0000","last_fullsized":"2023-09-23T23:13:58.851829+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":2928,"log_dups_size":0,"ondisk_log_size":2928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:06.046311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191889408,"num_objects":2928,"num_object_clones":0,"num_object_copies":8784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2928,"num_write_kb":187392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":131,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.407470+0000","last_change":"2023-09-23T23:08:45.120505+0000","last_active":"2023-09-23T23:13:39.407470+0000","last_peered":"2023-09-23T23:13:39.407470+0000","last_clean":"2023-09-23T23:13:39.407470+0000","last_became_active":"2023-09-23T23:08:45.120136+0000","last_became_peered":"2023-09-23T23:08:45.120136+0000","last_unstale":"2023-09-23T23:13:39.407470+0000","last_undegraded":"2023-09-23T23:13:39.407470+0000","last_fullsized":"2023-09-23T23:13:39.407470+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:56.726515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":130,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408224+0000","last_change":"2023-09-23T23:07:30.081779+0000","last_active":"2023-09-23T23:13:39.408224+0000","last_peered":"2023-09-23T23:13:39.408224+0000","last_clean":"2023-09-23T23:13:39.408224+0000","last_became_active":"2023-09-23T23:07:30.081425+0000","last_became_peered":"2023-09-23T23:07:30.081425+0000","last_unstale":"2023-09-23T23:13:39.408224+0000","last_undegraded":"2023-09-23T23:13:39.408224+0000","last_fullsized":"2023-09-23T23:13:39.408224+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:01.231206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"170'2924","reported_seq":2937,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:13:57.567622+0000","last_change":"2023-09-23T23:13:28.506227+0000","last_active":"2023-09-23T23:13:57.567622+0000","last_peered":"2023-09-23T23:13:57.567622+0000","last_clean":"2023-09-23T23:13:57.567622+0000","last_became_active":"2023-09-23T23:13:28.505885+0000","last_became_peered":"2023-09-23T23:13:28.505885+0000","last_unstale":"2023-09-23T23:13:57.567622+0000","last_undegraded":"2023-09-23T23:13:57.567622+0000","last_fullsized":"2023-09-23T23:13:57.567622+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":2924,"log_dups_size":0,"ondisk_log_size":2924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:21.630703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191627264,"num_objects":2924,"num_object_clones":0,"num_object_copies":8772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2924,"num_write_kb":187136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":109,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.404739+0000","last_change":"2023-09-23T23:10:59.516007+0000","last_active":"2023-09-23T23:13:39.404739+0000","last_peered":"2023-09-23T23:13:39.404739+0000","last_clean":"2023-09-23T23:13:39.404739+0000","last_became_active":"2023-09-23T23:10:59.515388+0000","last_became_peered":"2023-09-23T23:10:59.515388+0000","last_unstale":"2023-09-23T23:13:39.404739+0000","last_undegraded":"2023-09-23T23:13:39.404739+0000","last_fullsized":"2023-09-23T23:13:39.404739+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:13.171258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":137,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.407932+0000","last_change":"2023-09-23T23:10:59.503483+0000","last_active":"2023-09-23T23:13:39.407932+0000","last_peered":"2023-09-23T23:13:39.407932+0000","last_clean":"2023-09-23T23:13:39.407932+0000","last_became_active":"2023-09-23T23:10:59.502851+0000","last_became_peered":"2023-09-23T23:10:59.502851+0000","last_unstale":"2023-09-23T23:13:39.407932+0000","last_undegraded":"2023-09-23T23:13:39.407932+0000","last_fullsized":"2023-09-23T23:13:39.407932+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:20.388467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"170'2942","reported_seq":2956,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:13:58.827560+0000","last_change":"2023-09-23T23:13:28.355012+0000","last_active":"2023-09-23T23:13:58.827560+0000","last_peered":"2023-09-23T23:13:58.827560+0000","last_clean":"2023-09-23T23:13:58.827560+0000","last_became_active":"2023-09-23T23:13:28.354788+0000","last_became_peered":"2023-09-23T23:13:28.354788+0000","last_unstale":"2023-09-23T23:13:58.827560+0000","last_undegraded":"2023-09-23T23:13:58.827560+0000","last_fullsized":"2023-09-23T23:13:58.827560+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":2942,"log_dups_size":0,"ondisk_log_size":2942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:04.013664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192806912,"num_objects":2942,"num_object_clones":0,"num_object_copies":8826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2942,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":130,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:43.358359+0000","last_change":"2023-09-23T23:07:30.069993+0000","last_active":"2023-09-23T23:13:43.358359+0000","last_peered":"2023-09-23T23:13:43.358359+0000","last_clean":"2023-09-23T23:13:43.358359+0000","last_became_active":"2023-09-23T23:07:30.069634+0000","last_became_peered":"2023-09-23T23:07:30.069634+0000","last_unstale":"2023-09-23T23:13:43.358359+0000","last_undegraded":"2023-09-23T23:13:43.358359+0000","last_fullsized":"2023-09-23T23:13:43.358359+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:24.608905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":148,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408899+0000","last_change":"2023-09-23T23:10:59.498546+0000","last_active":"2023-09-23T23:13:39.408899+0000","last_peered":"2023-09-23T23:13:39.408899+0000","last_clean":"2023-09-23T23:13:39.408899+0000","last_became_active":"2023-09-23T23:10:59.498231+0000","last_became_peered":"2023-09-23T23:10:59.498231+0000","last_unstale":"2023-09-23T23:13:39.408899+0000","last_undegraded":"2023-09-23T23:13:39.408899+0000","last_fullsized":"2023-09-23T23:13:39.408899+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:37.404273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":132,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:43.358291+0000","last_change":"2023-09-23T23:13:29.421393+0000","last_active":"2023-09-23T23:13:43.358291+0000","last_peered":"2023-09-23T23:13:43.358291+0000","last_clean":"2023-09-23T23:13:43.358291+0000","last_became_active":"2023-09-23T23:13:29.421042+0000","last_became_peered":"2023-09-23T23:13:29.421042+0000","last_unstale":"2023-09-23T23:13:43.358291+0000","last_undegraded":"2023-09-23T23:13:43.358291+0000","last_fullsized":"2023-09-23T23:13:43.358291+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:22:35.586669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"170'3061","reported_seq":3076,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:14:00.212215+0000","last_change":"2023-09-23T23:13:28.360018+0000","last_active":"2023-09-23T23:14:00.212215+0000","last_peered":"2023-09-23T23:14:00.212215+0000","last_clean":"2023-09-23T23:14:00.212215+0000","last_became_active":"2023-09-23T23:13:28.359701+0000","last_became_peered":"2023-09-23T23:13:28.359701+0000","last_unstale":"2023-09-23T23:14:00.212215+0000","last_undegraded":"2023-09-23T23:14:00.212215+0000","last_fullsized":"2023-09-23T23:14:00.212215+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":3061,"log_dups_size":0,"ondisk_log_size":3061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:40.620740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200605696,"num_objects":3061,"num_object_clones":0,"num_object_copies":9183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3061,"num_write_kb":195904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":135,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408148+0000","last_change":"2023-09-23T23:07:30.075093+0000","last_active":"2023-09-23T23:13:39.408148+0000","last_peered":"2023-09-23T23:13:39.408148+0000","last_clean":"2023-09-23T23:13:39.408148+0000","last_became_active":"2023-09-23T23:07:30.074729+0000","last_became_peered":"2023-09-23T23:07:30.074729+0000","last_unstale":"2023-09-23T23:13:39.408148+0000","last_undegraded":"2023-09-23T23:13:39.408148+0000","last_fullsized":"2023-09-23T23:13:39.408148+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:34.502057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":131,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408064+0000","last_change":"2023-09-23T23:04:11.514703+0000","last_active":"2023-09-23T23:13:39.408064+0000","last_peered":"2023-09-23T23:13:39.408064+0000","last_clean":"2023-09-23T23:13:39.408064+0000","last_became_active":"2023-09-23T23:04:11.514403+0000","last_became_peered":"2023-09-23T23:04:11.514403+0000","last_unstale":"2023-09-23T23:13:39.408064+0000","last_undegraded":"2023-09-23T23:13:39.408064+0000","last_fullsized":"2023-09-23T23:13:39.408064+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:26.374346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":153,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:43.358343+0000","last_change":"2023-09-23T23:10:59.502030+0000","last_active":"2023-09-23T23:13:43.358343+0000","last_peered":"2023-09-23T23:13:43.358343+0000","last_clean":"2023-09-23T23:13:43.358343+0000","last_became_active":"2023-09-23T23:10:59.501714+0000","last_became_peered":"2023-09-23T23:10:59.501714+0000","last_unstale":"2023-09-23T23:13:43.358343+0000","last_undegraded":"2023-09-23T23:13:43.358343+0000","last_fullsized":"2023-09-23T23:13:43.358343+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:52.898368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"170'2889","reported_seq":2903,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:13:58.349364+0000","last_change":"2023-09-23T23:13:29.358375+0000","last_active":"2023-09-23T23:13:58.349364+0000","last_peered":"2023-09-23T23:13:58.349364+0000","last_clean":"2023-09-23T23:13:58.349364+0000","last_became_active":"2023-09-23T23:13:29.358036+0000","last_became_peered":"2023-09-23T23:13:29.358036+0000","last_unstale":"2023-09-23T23:13:58.349364+0000","last_undegraded":"2023-09-23T23:13:58.349364+0000","last_fullsized":"2023-09-23T23:13:58.349364+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":2889,"log_dups_size":0,"ondisk_log_size":2889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:57.055547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189333504,"num_objects":2889,"num_object_clones":0,"num_object_copies":8667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2889,"num_write_kb":184896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":126,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.409936+0000","last_change":"2023-09-23T23:10:59.501544+0000","last_active":"2023-09-23T23:13:39.409936+0000","last_peered":"2023-09-23T23:13:39.409936+0000","last_clean":"2023-09-23T23:13:39.409936+0000","last_became_active":"2023-09-23T23:10:59.501247+0000","last_became_peered":"2023-09-23T23:10:59.501247+0000","last_unstale":"2023-09-23T23:13:39.409936+0000","last_undegraded":"2023-09-23T23:13:39.409936+0000","last_fullsized":"2023-09-23T23:13:39.409936+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:54.254755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":139,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408472+0000","last_change":"2023-09-23T23:08:45.122712+0000","last_active":"2023-09-23T23:13:39.408472+0000","last_peered":"2023-09-23T23:13:39.408472+0000","last_clean":"2023-09-23T23:13:39.408472+0000","last_became_active":"2023-09-23T23:08:45.122391+0000","last_became_peered":"2023-09-23T23:08:45.122391+0000","last_unstale":"2023-09-23T23:13:39.408472+0000","last_undegraded":"2023-09-23T23:13:39.408472+0000","last_fullsized":"2023-09-23T23:13:39.408472+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:44.916349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":125,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:13:54.194987+0000","last_change":"2023-09-23T23:07:30.074957+0000","last_active":"2023-09-23T23:13:54.194987+0000","last_peered":"2023-09-23T23:13:54.194987+0000","last_clean":"2023-09-23T23:13:54.194987+0000","last_became_active":"2023-09-23T23:07:30.074475+0000","last_became_peered":"2023-09-23T23:07:30.074475+0000","last_unstale":"2023-09-23T23:13:54.194987+0000","last_undegraded":"2023-09-23T23:13:54.194987+0000","last_fullsized":"2023-09-23T23:13:54.194987+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:25.337999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"170'2892","reported_seq":2908,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:13:57.543240+0000","last_change":"2023-09-23T23:13:29.364074+0000","last_active":"2023-09-23T23:13:57.543240+0000","last_peered":"2023-09-23T23:13:57.543240+0000","last_clean":"2023-09-23T23:13:57.543240+0000","last_became_active":"2023-09-23T23:13:29.363791+0000","last_became_peered":"2023-09-23T23:13:29.363791+0000","last_unstale":"2023-09-23T23:13:57.543240+0000","last_undegraded":"2023-09-23T23:13:57.543240+0000","last_fullsized":"2023-09-23T23:13:57.543240+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":2892,"log_dups_size":0,"ondisk_log_size":2892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:53:07.822571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189530112,"num_objects":2892,"num_object_clones":0,"num_object_copies":8676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2892,"num_write_kb":185088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":135,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408560+0000","last_change":"2023-09-23T23:08:45.111935+0000","last_active":"2023-09-23T23:13:39.408560+0000","last_peered":"2023-09-23T23:13:39.408560+0000","last_clean":"2023-09-23T23:13:39.408560+0000","last_became_active":"2023-09-23T23:08:45.111549+0000","last_became_peered":"2023-09-23T23:08:45.111549+0000","last_unstale":"2023-09-23T23:13:39.408560+0000","last_undegraded":"2023-09-23T23:13:39.408560+0000","last_fullsized":"2023-09-23T23:13:39.408560+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:49.879205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":511,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.407549+0000","last_change":"2023-09-23T23:04:11.517317+0000","last_active":"2023-09-23T23:13:39.407549+0000","last_peered":"2023-09-23T23:13:39.407549+0000","last_clean":"2023-09-23T23:13:39.407549+0000","last_became_active":"2023-09-23T23:04:11.516929+0000","last_became_peered":"2023-09-23T23:04:11.516929+0000","last_unstale":"2023-09-23T23:13:39.407549+0000","last_undegraded":"2023-09-23T23:13:39.407549+0000","last_fullsized":"2023-09-23T23:13:39.407549+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T22:49:49.548021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:19.871312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":86,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.405145+0000","last_change":"2023-09-23T23:10:59.512923+0000","last_active":"2023-09-23T23:13:39.405145+0000","last_peered":"2023-09-23T23:13:39.405145+0000","last_clean":"2023-09-23T23:13:39.405145+0000","last_became_active":"2023-09-23T23:10:59.512637+0000","last_became_peered":"2023-09-23T23:10:59.512637+0000","last_unstale":"2023-09-23T23:13:39.405145+0000","last_undegraded":"2023-09-23T23:13:39.405145+0000","last_fullsized":"2023-09-23T23:13:39.405145+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:43:06.983322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":113,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.411721+0000","last_change":"2023-09-23T22:53:18.075922+0000","last_active":"2023-09-23T23:13:39.411721+0000","last_peered":"2023-09-23T23:13:39.411721+0000","last_clean":"2023-09-23T23:13:39.411721+0000","last_became_active":"2023-09-23T22:53:18.075621+0000","last_became_peered":"2023-09-23T22:53:18.075621+0000","last_unstale":"2023-09-23T23:13:39.411721+0000","last_undegraded":"2023-09-23T23:13:39.411721+0000","last_fullsized":"2023-09-23T23:13:39.411721+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:01.490333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"170'3082","reported_seq":3097,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:14:00.212325+0000","last_change":"2023-09-23T23:13:28.357808+0000","last_active":"2023-09-23T23:14:00.212325+0000","last_peered":"2023-09-23T23:14:00.212325+0000","last_clean":"2023-09-23T23:14:00.212325+0000","last_became_active":"2023-09-23T23:13:28.357474+0000","last_became_peered":"2023-09-23T23:13:28.357474+0000","last_unstale":"2023-09-23T23:14:00.212325+0000","last_undegraded":"2023-09-23T23:14:00.212325+0000","last_fullsized":"2023-09-23T23:14:00.212325+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":3082,"log_dups_size":0,"ondisk_log_size":3082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:08.042264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201981952,"num_objects":3082,"num_object_clones":0,"num_object_copies":9246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3082,"num_write_kb":197248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":152,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.407627+0000","last_change":"2023-09-23T23:07:30.068847+0000","last_active":"2023-09-23T23:13:39.407627+0000","last_peered":"2023-09-23T23:13:39.407627+0000","last_clean":"2023-09-23T23:13:39.407627+0000","last_became_active":"2023-09-23T23:07:30.068533+0000","last_became_peered":"2023-09-23T23:07:30.068533+0000","last_unstale":"2023-09-23T23:13:39.407627+0000","last_undegraded":"2023-09-23T23:13:39.407627+0000","last_fullsized":"2023-09-23T23:13:39.407627+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:59.598863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":119,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.412695+0000","last_change":"2023-09-23T23:10:59.503085+0000","last_active":"2023-09-23T23:13:39.412695+0000","last_peered":"2023-09-23T23:13:39.412695+0000","last_clean":"2023-09-23T23:13:39.412695+0000","last_became_active":"2023-09-23T23:10:59.502737+0000","last_became_peered":"2023-09-23T23:10:59.502737+0000","last_unstale":"2023-09-23T23:13:39.412695+0000","last_undegraded":"2023-09-23T23:13:39.412695+0000","last_fullsized":"2023-09-23T23:13:39.412695+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:22.783324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":134,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.412189+0000","last_change":"2023-09-23T23:08:45.116856+0000","last_active":"2023-09-23T23:13:39.412189+0000","last_peered":"2023-09-23T23:13:39.412189+0000","last_clean":"2023-09-23T23:13:39.412189+0000","last_became_active":"2023-09-23T23:08:45.116519+0000","last_became_peered":"2023-09-23T23:08:45.116519+0000","last_unstale":"2023-09-23T23:13:39.412189+0000","last_undegraded":"2023-09-23T23:13:39.412189+0000","last_fullsized":"2023-09-23T23:13:39.412189+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:25.917776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"170'3018","reported_seq":3032,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:13:58.793893+0000","last_change":"2023-09-23T23:13:28.359433+0000","last_active":"2023-09-23T23:13:58.793893+0000","last_peered":"2023-09-23T23:13:58.793893+0000","last_clean":"2023-09-23T23:13:58.793893+0000","last_became_active":"2023-09-23T23:13:28.359147+0000","last_became_peered":"2023-09-23T23:13:28.359147+0000","last_unstale":"2023-09-23T23:13:58.793893+0000","last_undegraded":"2023-09-23T23:13:58.793893+0000","last_fullsized":"2023-09-23T23:13:58.793893+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":3018,"log_dups_size":0,"ondisk_log_size":3018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:27.415293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197787648,"num_objects":3018,"num_object_clones":0,"num_object_copies":9054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3018,"num_write_kb":193152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":135,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408587+0000","last_change":"2023-09-23T23:04:11.900743+0000","last_active":"2023-09-23T23:13:39.408587+0000","last_peered":"2023-09-23T23:13:39.408587+0000","last_clean":"2023-09-23T23:13:39.408587+0000","last_became_active":"2023-09-23T23:04:11.900418+0000","last_became_peered":"2023-09-23T23:04:11.900418+0000","last_unstale":"2023-09-23T23:13:39.408587+0000","last_undegraded":"2023-09-23T23:13:39.408587+0000","last_fullsized":"2023-09-23T23:13:39.408587+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:45:31.893513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":139,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:43.358329+0000","last_change":"2023-09-23T23:08:45.124284+0000","last_active":"2023-09-23T23:13:43.358329+0000","last_peered":"2023-09-23T23:13:43.358329+0000","last_clean":"2023-09-23T23:13:43.358329+0000","last_became_active":"2023-09-23T23:08:45.123988+0000","last_became_peered":"2023-09-23T23:08:45.123988+0000","last_unstale":"2023-09-23T23:13:43.358329+0000","last_undegraded":"2023-09-23T23:13:43.358329+0000","last_fullsized":"2023-09-23T23:13:43.358329+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:26.726970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":104,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.412832+0000","last_change":"2023-09-23T23:04:11.911507+0000","last_active":"2023-09-23T23:13:39.412832+0000","last_peered":"2023-09-23T23:13:39.412832+0000","last_clean":"2023-09-23T23:13:39.412832+0000","last_became_active":"2023-09-23T23:04:11.908281+0000","last_became_peered":"2023-09-23T23:04:11.908281+0000","last_unstale":"2023-09-23T23:13:39.412832+0000","last_undegraded":"2023-09-23T23:13:39.412832+0000","last_fullsized":"2023-09-23T23:13:39.412832+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:30.414860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"170'2950","reported_seq":2964,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:13:58.837153+0000","last_change":"2023-09-23T23:13:28.354568+0000","last_active":"2023-09-23T23:13:58.837153+0000","last_peered":"2023-09-23T23:13:58.837153+0000","last_clean":"2023-09-23T23:13:58.837153+0000","last_became_active":"2023-09-23T23:13:28.354292+0000","last_became_peered":"2023-09-23T23:13:28.354292+0000","last_unstale":"2023-09-23T23:13:58.837153+0000","last_undegraded":"2023-09-23T23:13:58.837153+0000","last_fullsized":"2023-09-23T23:13:58.837153+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":2950,"log_dups_size":0,"ondisk_log_size":2950,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:12.723223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193331200,"num_objects":2950,"num_object_clones":0,"num_object_copies":8850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2950,"num_write_kb":188800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":132,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.405396+0000","last_change":"2023-09-23T23:10:59.511903+0000","last_active":"2023-09-23T23:13:39.405396+0000","last_peered":"2023-09-23T23:13:39.405396+0000","last_clean":"2023-09-23T23:13:39.405396+0000","last_became_active":"2023-09-23T23:10:59.511022+0000","last_became_peered":"2023-09-23T23:10:59.511022+0000","last_unstale":"2023-09-23T23:13:39.405396+0000","last_undegraded":"2023-09-23T23:13:39.405396+0000","last_fullsized":"2023-09-23T23:13:39.405396+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:03.498373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":124,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:13:54.197787+0000","last_change":"2023-09-23T23:07:30.083011+0000","last_active":"2023-09-23T23:13:54.197787+0000","last_peered":"2023-09-23T23:13:54.197787+0000","last_clean":"2023-09-23T23:13:54.197787+0000","last_became_active":"2023-09-23T23:07:30.082544+0000","last_became_peered":"2023-09-23T23:07:30.082544+0000","last_unstale":"2023-09-23T23:13:54.197787+0000","last_undegraded":"2023-09-23T23:13:54.197787+0000","last_fullsized":"2023-09-23T23:13:54.197787+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:20.624316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":129,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.404807+0000","last_change":"2023-09-23T23:10:59.512305+0000","last_active":"2023-09-23T23:13:39.404807+0000","last_peered":"2023-09-23T23:13:39.404807+0000","last_clean":"2023-09-23T23:13:39.404807+0000","last_became_active":"2023-09-23T23:10:59.511941+0000","last_became_peered":"2023-09-23T23:10:59.511941+0000","last_unstale":"2023-09-23T23:13:39.404807+0000","last_undegraded":"2023-09-23T23:13:39.404807+0000","last_fullsized":"2023-09-23T23:13:39.404807+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:06.293763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"170'2894","reported_seq":2909,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T23:13:57.577819+0000","last_change":"2023-09-23T23:13:28.359953+0000","last_active":"2023-09-23T23:13:57.577819+0000","last_peered":"2023-09-23T23:13:57.577819+0000","last_clean":"2023-09-23T23:13:57.577819+0000","last_became_active":"2023-09-23T23:13:28.359690+0000","last_became_peered":"2023-09-23T23:13:28.359690+0000","last_unstale":"2023-09-23T23:13:57.577819+0000","last_undegraded":"2023-09-23T23:13:57.577819+0000","last_fullsized":"2023-09-23T23:13:57.577819+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":2894,"log_dups_size":0,"ondisk_log_size":2894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:56.963155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189661184,"num_objects":2894,"num_object_clones":0,"num_object_copies":8682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2894,"num_write_kb":185216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":135,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.411981+0000","last_change":"2023-09-23T23:08:45.116806+0000","last_active":"2023-09-23T23:13:39.411981+0000","last_peered":"2023-09-23T23:13:39.411981+0000","last_clean":"2023-09-23T23:13:39.411981+0000","last_became_active":"2023-09-23T23:08:45.116462+0000","last_became_peered":"2023-09-23T23:08:45.116462+0000","last_unstale":"2023-09-23T23:13:39.411981+0000","last_undegraded":"2023-09-23T23:13:39.411981+0000","last_fullsized":"2023-09-23T23:13:39.411981+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:24.502987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":123,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.412036+0000","last_change":"2023-09-23T23:07:30.072388+0000","last_active":"2023-09-23T23:13:39.412036+0000","last_peered":"2023-09-23T23:13:39.412036+0000","last_clean":"2023-09-23T23:13:39.412036+0000","last_became_active":"2023-09-23T23:07:30.072099+0000","last_became_peered":"2023-09-23T23:07:30.072099+0000","last_unstale":"2023-09-23T23:13:39.412036+0000","last_undegraded":"2023-09-23T23:13:39.412036+0000","last_fullsized":"2023-09-23T23:13:39.412036+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:19.758734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":113,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.407317+0000","last_change":"2023-09-23T22:53:17.905706+0000","last_active":"2023-09-23T23:13:39.407317+0000","last_peered":"2023-09-23T23:13:39.407317+0000","last_clean":"2023-09-23T23:13:39.407317+0000","last_became_active":"2023-09-23T22:53:17.905309+0000","last_became_peered":"2023-09-23T22:53:17.905309+0000","last_unstale":"2023-09-23T23:13:39.407317+0000","last_undegraded":"2023-09-23T23:13:39.407317+0000","last_fullsized":"2023-09-23T23:13:39.407317+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:06.105426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":107,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.413036+0000","last_change":"2023-09-23T23:04:11.900795+0000","last_active":"2023-09-23T23:13:39.413036+0000","last_peered":"2023-09-23T23:13:39.413036+0000","last_clean":"2023-09-23T23:13:39.413036+0000","last_became_active":"2023-09-23T23:04:11.899751+0000","last_became_peered":"2023-09-23T23:04:11.899751+0000","last_unstale":"2023-09-23T23:13:39.413036+0000","last_undegraded":"2023-09-23T23:13:39.413036+0000","last_fullsized":"2023-09-23T23:13:39.413036+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:41.206493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":142,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:43.237487+0000","last_change":"2023-09-23T23:10:59.511474+0000","last_active":"2023-09-23T23:13:43.237487+0000","last_peered":"2023-09-23T23:13:43.237487+0000","last_clean":"2023-09-23T23:13:43.237487+0000","last_became_active":"2023-09-23T23:10:59.511175+0000","last_became_peered":"2023-09-23T23:10:59.511175+0000","last_unstale":"2023-09-23T23:13:43.237487+0000","last_undegraded":"2023-09-23T23:13:43.237487+0000","last_fullsized":"2023-09-23T23:13:43.237487+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:41.647976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":133,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408251+0000","last_change":"2023-09-23T23:07:30.075311+0000","last_active":"2023-09-23T23:13:39.408251+0000","last_peered":"2023-09-23T23:13:39.408251+0000","last_clean":"2023-09-23T23:13:39.408251+0000","last_became_active":"2023-09-23T23:07:30.074952+0000","last_became_peered":"2023-09-23T23:07:30.074952+0000","last_unstale":"2023-09-23T23:13:39.408251+0000","last_undegraded":"2023-09-23T23:13:39.408251+0000","last_fullsized":"2023-09-23T23:13:39.408251+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:16.724083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":147,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.407911+0000","last_change":"2023-09-23T23:10:59.512738+0000","last_active":"2023-09-23T23:13:39.407911+0000","last_peered":"2023-09-23T23:13:39.407911+0000","last_clean":"2023-09-23T23:13:39.407911+0000","last_became_active":"2023-09-23T23:10:59.512011+0000","last_became_peered":"2023-09-23T23:10:59.512011+0000","last_unstale":"2023-09-23T23:13:39.407911+0000","last_undegraded":"2023-09-23T23:13:39.407911+0000","last_fullsized":"2023-09-23T23:13:39.407911+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:18.088964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":116,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.408913+0000","last_change":"2023-09-23T22:53:16.891297+0000","last_active":"2023-09-23T23:13:39.408913+0000","last_peered":"2023-09-23T23:13:39.408913+0000","last_clean":"2023-09-23T23:13:39.408913+0000","last_became_active":"2023-09-23T22:53:16.891025+0000","last_became_peered":"2023-09-23T22:53:16.891025+0000","last_unstale":"2023-09-23T23:13:39.408913+0000","last_undegraded":"2023-09-23T23:13:39.408913+0000","last_fullsized":"2023-09-23T23:13:39.408913+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:40:50.608339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":147,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.407707+0000","last_change":"2023-09-23T23:10:59.512247+0000","last_active":"2023-09-23T23:13:39.407707+0000","last_peered":"2023-09-23T23:13:39.407707+0000","last_clean":"2023-09-23T23:13:39.407707+0000","last_became_active":"2023-09-23T23:10:59.511865+0000","last_became_peered":"2023-09-23T23:10:59.511865+0000","last_unstale":"2023-09-23T23:13:39.407707+0000","last_undegraded":"2023-09-23T23:13:39.407707+0000","last_fullsized":"2023-09-23T23:13:39.407707+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:20.981239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":128,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.405340+0000","last_change":"2023-09-23T23:10:59.512958+0000","last_active":"2023-09-23T23:13:39.405340+0000","last_peered":"2023-09-23T23:13:39.405340+0000","last_clean":"2023-09-23T23:13:39.405340+0000","last_became_active":"2023-09-23T23:10:59.512628+0000","last_became_peered":"2023-09-23T23:10:59.512628+0000","last_unstale":"2023-09-23T23:13:39.405340+0000","last_undegraded":"2023-09-23T23:13:39.405340+0000","last_fullsized":"2023-09-23T23:13:39.405340+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:16:26.998696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":143,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.405035+0000","last_change":"2023-09-23T23:10:59.511365+0000","last_active":"2023-09-23T23:13:39.405035+0000","last_peered":"2023-09-23T23:13:39.405035+0000","last_clean":"2023-09-23T23:13:39.405035+0000","last_became_active":"2023-09-23T23:10:59.511073+0000","last_became_peered":"2023-09-23T23:10:59.511073+0000","last_unstale":"2023-09-23T23:13:39.405035+0000","last_undegraded":"2023-09-23T23:13:39.405035+0000","last_fullsized":"2023-09-23T23:13:39.405035+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:35.196455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":123,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.412444+0000","last_change":"2023-09-23T23:08:45.113546+0000","last_active":"2023-09-23T23:13:39.412444+0000","last_peered":"2023-09-23T23:13:39.412444+0000","last_clean":"2023-09-23T23:13:39.412444+0000","last_became_active":"2023-09-23T23:08:45.113191+0000","last_became_peered":"2023-09-23T23:08:45.113191+0000","last_unstale":"2023-09-23T23:13:39.412444+0000","last_undegraded":"2023-09-23T23:13:39.412444+0000","last_fullsized":"2023-09-23T23:13:39.412444+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:06.372282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":140,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T23:13:39.407826+0000","last_change":"2023-09-23T23:08:45.116882+0000","last_active":"2023-09-23T23:13:39.407826+0000","last_peered":"2023-09-23T23:13:39.407826+0000","last_clean":"2023-09-23T23:13:39.407826+0000","last_became_active":"2023-09-23T23:08:45.116543+0000","last_became_peered":"2023-09-23T23:08:45.116543+0000","last_unstale":"2023-09-23T23:13:39.407826+0000","last_undegraded":"2023-09-23T23:13:39.407826+0000","last_fullsized":"2023-09-23T23:13:39.407826+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:08.722088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3107127296,"num_objects":47411,"num_object_clones":0,"num_object_copies":142233,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":47411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":47411,"num_write_kb":3034304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9305063424,"data_stored":9305063424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":47411,"ondisk_log_size":47411,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234255,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2191480,"kb_used_data":1864816,"kb_used_omap":0,"kb_used_meta":326656,"kb_avail":91553672,"statfs":{"total":95995035648,"available":93750960128,"internally_reserved":0,"allocated":1909571584,"data_stored":1909014043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":334495744},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:13:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":1.9119999999999999,"15min":2.742},"min":{"1min":0.35599999999999998,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.81899999999999995,"5min":29.928999999999998,"15min":230.249},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":2.137,"15min":2.7919999999999998},"min":{"1min":0.30499999999999999,"5min":0.24299999999999999,"15min":0.24199999999999999},"max":{"1min":0.93200000000000005,"5min":29.977,"15min":230.261},"last":3.4950000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:13:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":3.0459999999999998,"15min":4.7939999999999996},"min":{"1min":0.371,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.748,"5min":91.509,"15min":93.245999999999995},"last":0.95999999999999996},{"interface":"front","average":{"1min":0.56100000000000005,"5min":2.702,"15min":4.6020000000000003},"min":{"1min":0.33100000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.0009999999999999,"5min":91.691000000000003,"15min":92.748000000000005},"last":1.117}]},{"osd":2,"last update":"Sat Sep 23 23:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":2.145,"15min":2.9769999999999999},"min":{"1min":0.40200000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.5430000000000001,"5min":12.183,"15min":12.183},"last":0.89200000000000002},{"interface":"front","average":{"1min":0.67400000000000004,"5min":1.7390000000000001,"15min":2.1389999999999998},"min":{"1min":0.30099999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.819,"5min":11.781000000000001,"15min":11.781000000000001},"last":1.819}]},{"osd":3,"last update":"Sat Sep 23 23:13:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":2.2090000000000001,"15min":2.379},"min":{"1min":0.40600000000000003,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.2610000000000001,"5min":11.507999999999999,"15min":11.507999999999999},"last":1.6619999999999999},{"interface":"front","average":{"1min":0.90700000000000003,"5min":2.2360000000000002,"15min":2.4580000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":3.569,"5min":8.9670000000000005,"15min":8.9670000000000005},"last":1.6759999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:13:17 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.3660000000000001,"15min":1.5900000000000001},"min":{"1min":0.38400000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.022,"5min":8.984,"15min":92.781000000000006},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.331,"15min":1.494},"min":{"1min":0.39400000000000002,"5min":0.28499999999999998,"15min":0.254},"max":{"1min":0.72499999999999998,"5min":7.7290000000000001,"15min":92.626999999999995},"last":0.38300000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:13:17 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":1.788,"15min":1.8109999999999999},"min":{"1min":0.34899999999999998,"5min":0.29299999999999998,"15min":0.26500000000000001},"max":{"1min":0.83499999999999996,"5min":12.067,"15min":93.135000000000005},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.66700000000000004,"5min":1.284,"15min":1.649},"min":{"1min":0.45700000000000002,"5min":0.29999999999999999,"15min":0.29399999999999998},"max":{"1min":1.069,"5min":8.9290000000000003,"15min":92.885000000000005},"last":0.53000000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:13:17 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":1.353,"15min":1.603},"min":{"1min":0.495,"5min":0.315,"15min":0.27300000000000002},"max":{"1min":1.0900000000000001,"5min":13.590999999999999,"15min":93.105999999999995},"last":0.505},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.53,"15min":1.4970000000000001},"min":{"1min":0.438,"5min":0.32300000000000001,"15min":0.28399999999999997},"max":{"1min":0.79300000000000004,"5min":9.6240000000000006,"15min":93.198999999999998},"last":0.33300000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495891,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":840936,"kb_used_data":567136,"kb_used_omap":0,"kb_used_meta":273792,"kb_avail":92904216,"statfs":{"total":95995035648,"available":95133917184,"internally_reserved":0,"allocated":580747264,"data_stored":580205696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":280363008},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.4570000000000001,"15min":3.8119999999999998},"min":{"1min":0.34899999999999998,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":1.5289999999999999,"5min":7.649,"15min":97.858000000000004},"last":1.4019999999999999},{"interface":"front","average":{"1min":0.52000000000000002,"5min":1.494,"15min":4.399},"min":{"1min":0.35499999999999998,"5min":0.24399999999999999,"15min":0.22700000000000001},"max":{"1min":0.91900000000000004,"5min":8.3059999999999992,"15min":99.793000000000006},"last":0.49199999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":4.1820000000000004,"15min":1.9990000000000001},"min":{"1min":0.38200000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.93799999999999994,"5min":209.00899999999999,"15min":209.00899999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.56399999999999995,"5min":4.2670000000000003,"15min":2.3149999999999999},"min":{"1min":0.34300000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.95399999999999996,"5min":209.03,"15min":209.03},"last":0.54600000000000004}]},{"osd":2,"last update":"Sat Sep 23 23:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":1.681,"15min":1.569},"min":{"1min":0.39400000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.0369999999999999,"5min":10.871,"15min":10.871},"last":0.83699999999999997},{"interface":"front","average":{"1min":1.03,"5min":2.0819999999999999,"15min":1.9139999999999999},"min":{"1min":0.41399999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.343,"5min":12.148,"15min":12.148},"last":0.67500000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:13:02 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":2.2909999999999999,"15min":2.5529999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.3529999999999998,"5min":7.4059999999999997,"15min":7.4059999999999997},"last":4.3289999999999997},{"interface":"front","average":{"1min":1.0649999999999999,"5min":2.718,"15min":2.9940000000000002},"min":{"1min":0.432,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":5.3250000000000002,"5min":9.4550000000000001,"15min":9.4550000000000001},"last":0.875}]},{"osd":4,"last update":"Sat Sep 23 23:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":1.405,"15min":1.093},"min":{"1min":0.30499999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.099,"5min":11.890000000000001,"15min":11.890000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.66700000000000004,"5min":1.6220000000000001,"15min":1.194},"min":{"1min":0.33000000000000002,"5min":0.308,"15min":0.24299999999999999},"max":{"1min":2.093,"5min":11.728,"15min":11.728},"last":4.2569999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":1.516,"15min":1.339},"min":{"1min":0.378,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":1.359,"5min":8.0850000000000009,"15min":26.158999999999999},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.63200000000000001,"5min":1.3149999999999999,"15min":1.256},"min":{"1min":0.374,"5min":0.30599999999999999,"15min":0.30199999999999999},"max":{"1min":1.2130000000000001,"5min":8.0169999999999995,"15min":25.786000000000001},"last":4.298}]},{"osd":7,"last update":"Sat Sep 23 23:13:15 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":1.569,"15min":1.504},"min":{"1min":0.40400000000000003,"5min":0.32300000000000001,"15min":0.32100000000000001},"max":{"1min":0.80600000000000005,"5min":10.968999999999999,"15min":12.676},"last":4.3520000000000003},{"interface":"front","average":{"1min":0.68700000000000006,"5min":1.3839999999999999,"15min":1.3100000000000001},"min":{"1min":0.53700000000000003,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.86799999999999999,"5min":7.9669999999999996,"15min":9.7889999999999997},"last":1.524}]}]},{"osd":1,"up_from":112,"seq":481036337232,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1869588,"kb_used_data":1640076,"kb_used_omap":4,"kb_used_meta":229499,"kb_avail":91875564,"statfs":{"total":95995035648,"available":94080577536,"internally_reserved":0,"allocated":1679437824,"data_stored":1678738579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":235007886},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.29599999999999999,"5min":1.77,"15min":1.5960000000000001},"min":{"1min":0.22700000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.50700000000000001,"5min":10.221,"15min":10.221},"last":0.311},{"interface":"front","average":{"1min":0.33000000000000002,"5min":1.51,"15min":1.4159999999999999},"min":{"1min":0.23000000000000001,"5min":0.21199999999999999,"15min":0.20100000000000001},"max":{"1min":0.436,"5min":12.212,"15min":12.212},"last":2.0230000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:13:51 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":1.9430000000000001,"15min":2.2749999999999999},"min":{"1min":0.21099999999999999,"5min":0.192,"15min":0.192},"max":{"1min":1.5600000000000001,"5min":16.696999999999999,"15min":16.696999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.44900000000000001,"5min":1.8129999999999999,"15min":2.1789999999999998},"min":{"1min":0.29799999999999999,"5min":0.215,"15min":0.215},"max":{"1min":1.028,"5min":9.0879999999999992,"15min":9.0879999999999992},"last":0.23200000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:13:06 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":2.2850000000000001,"15min":2.5630000000000002},"min":{"1min":0.27600000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.8769999999999998,"5min":12.234999999999999,"15min":12.234999999999999},"last":2.0950000000000002},{"interface":"front","average":{"1min":0.58299999999999996,"5min":1.9490000000000001,"15min":2.177},"min":{"1min":0.29399999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.9359999999999999,"5min":12.256,"15min":12.256},"last":2.2269999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":1.6799999999999999,"15min":1.498},"min":{"1min":0.39100000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.83799999999999997,"5min":12.676,"15min":12.676},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.56599999999999995,"5min":1.7290000000000001,"15min":1.4330000000000001},"min":{"1min":0.36399999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.67000000000000004,"5min":11.125,"15min":11.125},"last":0.371}]},{"osd":5,"last update":"Sat Sep 23 23:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.7450000000000001,"15min":3.6890000000000001},"min":{"1min":0.38500000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.78700000000000003,"5min":12.351000000000001,"15min":67.497},"last":2.0600000000000001},{"interface":"front","average":{"1min":0.623,"5min":1.9079999999999999,"15min":3.746},"min":{"1min":0.45900000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.94299999999999995,"5min":12.757999999999999,"15min":67.540999999999997},"last":4.7949999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.486,"15min":1.5149999999999999},"min":{"1min":0.41399999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.83599999999999997,"5min":12.186,"15min":12.186},"last":2.1970000000000001},{"interface":"front","average":{"1min":0.63200000000000001,"5min":2.1230000000000002,"15min":1.629},"min":{"1min":0.45300000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.85899999999999999,"5min":12.785,"15min":12.785},"last":4.7279999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":2.0910000000000002,"15min":1.661},"min":{"1min":0.496,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.90500000000000003,"5min":12.727,"15min":12.727},"last":4.7699999999999996},{"interface":"front","average":{"1min":0.65700000000000003,"5min":1.7529999999999999,"15min":1.544},"min":{"1min":0.45200000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.97399999999999998,"5min":9.8559999999999999,"15min":9.8559999999999999},"last":0.82699999999999996}]}]},{"osd":0,"up_from":91,"seq":390842024056,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1199336,"kb_used_data":785504,"kb_used_omap":5,"kb_used_meta":413818,"kb_avail":92545816,"statfs":{"total":95995035648,"available":94766915584,"internally_reserved":0,"allocated":804356096,"data_stored":803637510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":423750266},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:13:40 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":1.762,"15min":2.3079999999999998},"min":{"1min":0.19600000000000001,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":1.786,"5min":11.029,"15min":21.629999999999999},"last":7.524},{"interface":"front","average":{"1min":0.495,"5min":1.8919999999999999,"15min":2.1560000000000001},"min":{"1min":0.252,"5min":0.189,"15min":0.189},"max":{"1min":3.508,"5min":11.922000000000001,"15min":13.382},"last":0.71699999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:13:57 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.905,"15min":2.0350000000000001},"min":{"1min":0.23499999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":4.6929999999999996,"5min":10.199,"15min":10.199},"last":4.6929999999999996},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.7430000000000001,"15min":1.913},"min":{"1min":0.25800000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.5590000000000002,"5min":9.0239999999999991,"15min":9.0239999999999991},"last":1.29}]},{"osd":3,"last update":"Sat Sep 23 23:12:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":2.895,"15min":3.2709999999999999},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.0460000000000003,"5min":11.866,"15min":11.866},"last":7.2560000000000002},{"interface":"front","average":{"1min":0.47999999999999998,"5min":2.0990000000000002,"15min":2.3690000000000002},"min":{"1min":0.25,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.351,"5min":9.0980000000000008,"15min":9.0980000000000008},"last":0.55300000000000005}]},{"osd":4,"last update":"Sat Sep 23 23:13:21 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":2.266,"15min":2.1509999999999998},"min":{"1min":0.39600000000000002,"5min":0.247,"15min":0.245},"max":{"1min":7.0759999999999996,"5min":28.376999999999999,"15min":28.376999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":1.921,"15min":2.4649999999999999},"min":{"1min":0.38200000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.73399999999999999,"5min":30.706,"15min":30.706},"last":0.498}]},{"osd":5,"last update":"Sat Sep 23 23:13:21 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":1.847,"15min":2.4849999999999999},"min":{"1min":0.42399999999999999,"5min":0.33700000000000002,"15min":0.30499999999999999},"max":{"1min":0.65400000000000003,"5min":11.157,"15min":30.873999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":1.7749999999999999,"15min":1.875},"min":{"1min":0.40799999999999997,"5min":0.30299999999999999,"15min":0.29499999999999998},"max":{"1min":0.755,"5min":10.117000000000001,"15min":10.117000000000001},"last":4.7800000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:13:21 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.893,"15min":2.5110000000000001},"min":{"1min":0.437,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.79800000000000004,"5min":10.605,"15min":21.047999999999998},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.58099999999999996,"5min":2.0350000000000001,"15min":1.958},"min":{"1min":0.41799999999999998,"5min":0.36499999999999999,"15min":0.30499999999999999},"max":{"1min":0.80600000000000005,"5min":10.414,"15min":10.414},"last":4.7510000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:13:21 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":2.1070000000000002,"15min":1.9810000000000001},"min":{"1min":0.40200000000000002,"5min":0.36099999999999999,"15min":0.29299999999999998},"max":{"1min":0.73999999999999999,"5min":10.145,"15min":10.145},"last":4.8049999999999997},{"interface":"front","average":{"1min":0.89800000000000002,"5min":2.0990000000000002,"15min":2.121},"min":{"1min":0.442,"5min":0.34999999999999998,"15min":0.313},"max":{"1min":7.1120000000000001,"5min":10.513999999999999,"15min":10.513999999999999},"last":0.58199999999999996}]}]},{"osd":2,"up_from":130,"seq":558345748545,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":835680,"kb_used_data":559640,"kb_used_omap":7,"kb_used_meta":276024,"kb_avail":92909472,"statfs":{"total":95995035648,"available":95139299328,"internally_reserved":0,"allocated":573071360,"data_stored":572418844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":282648657},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":1.726,"15min":1.9239999999999999},"min":{"1min":0.17999999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.58999999999999997,"5min":16.021000000000001,"15min":16.021000000000001},"last":0.31},{"interface":"front","average":{"1min":0.40400000000000003,"5min":1.998,"15min":2.8119999999999998},"min":{"1min":0.25700000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.246,"5min":16.152999999999999,"15min":16.152999999999999},"last":0.32700000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.859,"15min":2.4849999999999999},"min":{"1min":0.22,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":1.9019999999999999,"5min":9.3719999999999999,"15min":9.3719999999999999},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.45100000000000001,"5min":1.6499999999999999,"15min":2.4860000000000002},"min":{"1min":0.26000000000000001,"5min":0.218,"15min":0.218},"max":{"1min":1.4239999999999999,"5min":11.625,"15min":11.625},"last":0.29399999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:13:03 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":1.149,"15min":1.2290000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":5.6639999999999997,"5min":6.8230000000000004,"15min":6.8230000000000004},"last":3.431},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.7769999999999999,"15min":1.8899999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.7110000000000003,"5min":7.7110000000000003,"15min":7.7110000000000003},"last":3.8119999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":1.714,"15min":1.9890000000000001},"min":{"1min":0.33400000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.944,"5min":10.666,"15min":10.666},"last":0.442},{"interface":"front","average":{"1min":0.626,"5min":1.4259999999999999,"15min":1.452},"min":{"1min":0.38,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.47,"5min":14.829000000000001,"15min":14.829000000000001},"last":0.42599999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3440000000000001,"5min":2.1760000000000002,"15min":2.456},"min":{"1min":0.38700000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":12.988,"5min":12.988,"15min":12.988},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.3109999999999999,"5min":1.548,"15min":1.6279999999999999},"min":{"1min":0.375,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":13.385999999999999,"5min":13.385999999999999,"15min":13.385999999999999},"last":0.46000000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.6950000000000001,"15min":1.8169999999999999},"min":{"1min":0.42799999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.5059999999999998,"5min":15.205,"15min":15.205},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.80700000000000005,"5min":2.04,"15min":2.456},"min":{"1min":0.41999999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.4849999999999999,"5min":11.923999999999999,"15min":11.923999999999999},"last":0.47599999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":1.944,"15min":2.21},"min":{"1min":0.376,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.4990000000000001,"5min":14.792999999999999,"15min":14.792999999999999},"last":0.376},{"interface":"front","average":{"1min":0.85899999999999999,"5min":2.3460000000000001,"15min":2.8010000000000002},"min":{"1min":0.39000000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":3.7789999999999999,"5min":15.045,"15min":15.045},"last":0.49399999999999999}]}]},{"osd":3,"up_from":148,"seq":635655159846,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1854784,"kb_used_data":1622456,"kb_used_omap":2,"kb_used_meta":232317,"kb_avail":91890368,"statfs":{"total":95995035648,"available":94095736832,"internally_reserved":0,"allocated":1661394944,"data_stored":1660770569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":237892661},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.1970000000000001,"15min":1.3260000000000001},"min":{"1min":0.221,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":2.2949999999999999,"5min":5.9660000000000002,"15min":5.9660000000000002},"last":1.0780000000000001},{"interface":"front","average":{"1min":0.32100000000000001,"5min":1.8129999999999999,"15min":2.0609999999999999},"min":{"1min":0.221,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.58499999999999996,"5min":9.5839999999999996,"15min":9.5839999999999996},"last":0.193}]},{"osd":1,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.8700000000000001,"15min":2.0870000000000002},"min":{"1min":0.245,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":2.9740000000000002,"5min":7.1219999999999999,"15min":7.1219999999999999},"last":1.0089999999999999},{"interface":"front","average":{"1min":0.371,"5min":1.635,"15min":1.845},"min":{"1min":0.246,"5min":0.185,"15min":0.185},"max":{"1min":0.71199999999999997,"5min":6.4160000000000004,"15min":6.4160000000000004},"last":4.0099999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":2.0859999999999999,"15min":2.335},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.8399999999999999,"5min":7.9930000000000003,"15min":7.9930000000000003},"last":1.046},{"interface":"front","average":{"1min":0.39200000000000002,"5min":2.2810000000000001,"15min":2.5960000000000001},"min":{"1min":0.28799999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.72999999999999998,"5min":12.721,"15min":12.721},"last":4.2569999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":1.544,"15min":1.651},"min":{"1min":0.38600000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.1040000000000001,"5min":7.1040000000000001,"15min":7.1040000000000001},"last":1.1100000000000001},{"interface":"front","average":{"1min":0.90300000000000002,"5min":2.8090000000000002,"15min":3.1269999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.7850000000000001,"5min":13.532999999999999,"15min":13.532999999999999},"last":0.312}]},{"osd":5,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":2.3700000000000001,"15min":2.665},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":1.1279999999999999,"5min":10.117000000000001,"15min":10.117000000000001},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.81799999999999995,"5min":1.9119999999999999,"15min":2.0939999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":2.8690000000000002,"5min":7.3250000000000002,"15min":7.3250000000000002},"last":1.988}]},{"osd":6,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":2.4870000000000001,"15min":2.7959999999999998},"min":{"1min":0.45300000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.208,"5min":14.492000000000001,"15min":14.492000000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":1.173,"5min":2.1059999999999999,"15min":2.262},"min":{"1min":0.42399999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":7.6139999999999999,"5min":7.9640000000000004,"15min":7.9640000000000004},"last":1.157}]},{"osd":7,"last update":"Sat Sep 23 23:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":2.1709999999999998,"15min":2.4249999999999998},"min":{"1min":0.44900000000000001,"5min":0.377,"15min":0.377},"max":{"1min":1.288,"5min":9.7059999999999995,"15min":9.7059999999999995},"last":1.131},{"interface":"front","average":{"1min":0.67100000000000004,"5min":2.298,"15min":2.569},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.379,"5min":9.8010000000000002,"15min":9.8010000000000002},"last":1.282}]}]},{"osd":4,"up_from":30,"seq":128849019164,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":902324,"kb_used_data":578732,"kb_used_omap":0,"kb_used_meta":323584,"kb_avail":92842828,"statfs":{"total":95995035648,"available":95071055872,"internally_reserved":0,"allocated":592621568,"data_stored":592067832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":331350016},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:13:24 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":3.8159999999999998,"15min":2.4649999999999999},"min":{"1min":0.315,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.83699999999999997,"5min":238.62200000000001,"15min":238.62200000000001},"last":1.863},{"interface":"front","average":{"1min":0.46200000000000002,"5min":4.75,"15min":2.9729999999999999},"min":{"1min":0.29299999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.72999999999999998,"5min":238.58199999999999,"15min":238.58199999999999},"last":1.0840000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:13:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":4.5289999999999999,"15min":2.3279999999999998},"min":{"1min":0.30099999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.6359999999999999,"5min":175.00899999999999,"15min":175.00899999999999},"last":1.8260000000000001},{"interface":"front","average":{"1min":0.56699999999999995,"5min":4.8410000000000002,"15min":2.3130000000000002},"min":{"1min":0.308,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.6799999999999999,"5min":174.821,"15min":174.821},"last":0.38500000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:13:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.502,"15min":1.8260000000000001},"min":{"1min":0.313,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.343,"5min":11.664999999999999,"15min":11.664999999999999},"last":2.1110000000000002},{"interface":"front","average":{"1min":0.72999999999999998,"5min":1.9390000000000001,"15min":2.5830000000000002},"min":{"1min":0.47699999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.3079999999999998,"5min":10.576000000000001,"15min":10.576000000000001},"last":2.1200000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:13:02 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":1.5760000000000001,"15min":1.7430000000000001},"min":{"1min":0.36899999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.498,"5min":8.1609999999999996,"15min":8.1609999999999996},"last":2.4769999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":2.0699999999999998,"15min":2.2639999999999998},"min":{"1min":0.45500000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.0309999999999997,"5min":11.455,"15min":11.455},"last":2.4199999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:13:41 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.1319999999999999,"15min":1.0009999999999999},"min":{"1min":0.375,"5min":0.25,"15min":0.23999999999999999},"max":{"1min":0.79800000000000004,"5min":10.16,"15min":27.391999999999999},"last":2.004},{"interface":"front","average":{"1min":0.71799999999999997,"5min":1.5249999999999999,"15min":1.3420000000000001},"min":{"1min":0.441,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":1.8440000000000001,"5min":8.2680000000000007,"15min":27.222999999999999},"last":0.33600000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:13:04 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.2130000000000001,"15min":1.1419999999999999},"min":{"1min":0.372,"5min":0.27200000000000002,"15min":0.223},"max":{"1min":0.92000000000000004,"5min":9.3350000000000009,"15min":14.4},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.70499999999999996,"5min":1.2949999999999999,"15min":1.2689999999999999},"min":{"1min":0.33700000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"max":{"1min":3.3170000000000002,"5min":13.888999999999999,"15min":15.641999999999999},"last":2.0350000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:13:29 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.5580000000000001,"15min":1.611},"min":{"1min":0.35999999999999999,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":0.86099999999999999,"5min":13.792,"15min":119.188},"last":2.0569999999999999},{"interface":"front","average":{"1min":0.61599999999999999,"5min":1.54,"15min":1.6930000000000001},"min":{"1min":0.442,"5min":0.35399999999999998,"15min":0.34200000000000003},"max":{"1min":1.0509999999999999,"5min":13.055,"15min":119.248},"last":1.1339999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757527,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1676252,"kb_used_data":1312212,"kb_used_omap":0,"kb_used_meta":364032,"kb_avail":92068900,"statfs":{"total":95995035648,"available":94278553600,"internally_reserved":0,"allocated":1343705088,"data_stored":1343171337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":372768768},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:13:28 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":1.3340000000000001,"15min":5.944},"min":{"1min":0.34799999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.72199999999999998,"5min":10.227,"15min":190.483},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.499,"5min":1.9570000000000001,"15min":6.702},"min":{"1min":0.36499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.69299999999999995,"5min":12.762,"15min":194.11099999999999},"last":0.46800000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":2.7770000000000001,"15min":1.6359999999999999},"min":{"1min":0.372,"5min":0.246,"15min":0.246},"max":{"1min":1.413,"5min":91.295000000000002,"15min":91.295000000000002},"last":0.373},{"interface":"front","average":{"1min":0.54000000000000004,"5min":3.0779999999999998,"15min":1.7629999999999999},"min":{"1min":0.38600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.82299999999999995,"5min":91.242000000000004,"15min":91.242000000000004},"last":0.32000000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":2.0720000000000001,"15min":2.6539999999999999},"min":{"1min":0.32900000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.1539999999999999,"5min":9.8279999999999994,"15min":9.8279999999999994},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.71099999999999997,"5min":1.427,"15min":1.8660000000000001},"min":{"1min":0.38100000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":2.0299999999999998,"5min":8.093,"15min":8.093},"last":0.53900000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:12:58 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":1.6499999999999999,"15min":1.79},"min":{"1min":0.44700000000000001,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":2.633,"5min":6.2640000000000002,"15min":6.2640000000000002},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.002,"5min":2.6880000000000002,"15min":2.9689999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.9420000000000002,"5min":12.802,"15min":12.802},"last":0.56000000000000005}]},{"osd":4,"last update":"Sat Sep 23 23:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.625,"15min":1.153},"min":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.221},"max":{"1min":1.343,"5min":14.538,"15min":14.538},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.54000000000000004,"5min":1.0609999999999999,"15min":0.99199999999999999},"min":{"1min":0.36399999999999999,"5min":0.28100000000000003,"15min":0.25},"max":{"1min":0.78800000000000003,"5min":11.779,"15min":11.779},"last":0.45500000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.468,"15min":1.2370000000000001},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.255},"max":{"1min":0.76900000000000002,"5min":13.226000000000001,"15min":13.226000000000001},"last":0.496},{"interface":"front","average":{"1min":0.63700000000000001,"5min":1.76,"15min":1.222},"min":{"1min":0.41699999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.514,"5min":13.098000000000001,"15min":13.098000000000001},"last":0.52300000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:13:34 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.829,"15min":1.512},"min":{"1min":0.47299999999999998,"5min":0.33900000000000002,"15min":0.29199999999999998},"max":{"1min":0.86599999999999999,"5min":14.929,"15min":14.929},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.677,"15min":1.4119999999999999},"min":{"1min":0.45800000000000002,"5min":0.33400000000000002,"15min":0.29599999999999999},"max":{"1min":0.85399999999999998,"5min":15.090999999999999,"15min":15.090999999999999},"last":0.42999999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":808386560,"data_stored":808386560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1739325440,"data_stored":1739325440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":582287360,"data_stored":582287360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1690042368,"data_stored":1690042368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":603389952,"data_stored":603389952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1353711616,"data_stored":1353711616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":589168640,"data_stored":589168640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1938751488,"data_stored":1938751488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:14:05.772 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 1 2023-09-23T23:14:05.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:05.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:05.876 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:05.974 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:06.185 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:14:06.221 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:14:06.262 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:14:06.294 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:14:06.323 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:06.350 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:14:06.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:06.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:06.382 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:14:06.416 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:14:06.447 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:14:06.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:06 smithi110 ceph-mon[221102]: pgmap v557: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T23:14:06.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:06 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/212681054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:06.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:06 smithi110 ceph-mon[212939]: pgmap v557: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T23:14:06.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:06 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/212681054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:06.460 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:14:06.465 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:14:06.465 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:14:06.465 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:14:06.465 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:14:06.465 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:14:06.465 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:14:06.465 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:14:06.465 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:14:06.518 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 31 15 56702 56687 114.265 73.3125 0.0220093 0.00874623 2023-09-23T23:14:06.537 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:06.670 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:14:06.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:06.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:14:06.680 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:14:06.680 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:14:06.680 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:14:06.680 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:14:06.680 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:14:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:06 smithi138 ceph-mon[193606]: pgmap v557: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T23:14:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:06 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/212681054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:06.774 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:06.806 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:14:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:14:06] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2023-09-23T23:14:06.854 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:06.995 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:06.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:06.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:07.100 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:07.145 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:07.238 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:07.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:07.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:07.346 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:14:07.371 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:07.411 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:07 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3198072396' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:07.411 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:07 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:07.412 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:07 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3198072396' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:07.412 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:07 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:07.418 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:14:07.425 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:14:07.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:07.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:14:07.518 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 32 16 57708 57692 112.657 62.8125 0.00245982 0.00882131 2023-09-23T23:14:07.533 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:14:07.559 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:07.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:07 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3198072396' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:07.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:07 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:08.320 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:14:08.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:08.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:14:08.347 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:14:08.361 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:08.430 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:14:08.525 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 33 16 58763 58747 111.218 65.9375 0.0139558 0.00898673 2023-09-23T23:14:08.553 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:08.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:08 smithi138 ceph-mon[193606]: pgmap v558: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T23:14:08.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:08 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:14:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:08 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:14:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:08 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:14:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:08 smithi138 ceph-mon[193606]: osdmap e171: 8 total, 8 up, 8 in 2023-09-23T23:14:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:08 smithi138 ceph-mon[193606]: failed to encode map e171 with expected crc 2023-09-23T23:14:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:08 smithi138 ceph-mon[193606]: failed to encode map e171 with expected crc 2023-09-23T23:14:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:08 smithi138 ceph-mon[193606]: failed to encode map e171 with expected crc 2023-09-23T23:14:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:08 smithi138 ceph-mon[193606]: failed to encode map e171 with expected crc 2023-09-23T23:14:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:08 smithi138 ceph-mon[193606]: failed to encode map e171 with expected crc 2023-09-23T23:14:08.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[212939]: pgmap v558: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T23:14:08.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:14:08.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:14:08.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:14:08.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[212939]: osdmap e171: 8 total, 8 up, 8 in 2023-09-23T23:14:08.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[212939]: failed to encode map e171 with expected crc 2023-09-23T23:14:08.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[212939]: failed to encode map e171 with expected crc 2023-09-23T23:14:08.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[212939]: failed to encode map e171 with expected crc 2023-09-23T23:14:08.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[212939]: failed to encode map e171 with expected crc 2023-09-23T23:14:08.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[212939]: failed to encode map e171 with expected crc 2023-09-23T23:14:08.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[221102]: pgmap v558: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T23:14:08.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:14:08.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:14:08.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:14:08.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[221102]: osdmap e171: 8 total, 8 up, 8 in 2023-09-23T23:14:08.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[221102]: failed to encode map e171 with expected crc 2023-09-23T23:14:08.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[221102]: failed to encode map e171 with expected crc 2023-09-23T23:14:08.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[221102]: failed to encode map e171 with expected crc 2023-09-23T23:14:08.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[221102]: failed to encode map e171 with expected crc 2023-09-23T23:14:08.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:08 smithi110 ceph-mon[221102]: failed to encode map e171 with expected crc 2023-09-23T23:14:09.141 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:14:09.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:09.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:14:09.248 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:14:09.297 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:09.360 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:14:09.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:09.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:14:09.471 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:09.525 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 34 16 59799 59783 109.852 64.75 0.00709914 0.00909868 2023-09-23T23:14:09.544 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:09.713 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:09.919 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:09.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:09.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:10.026 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:10.112 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:10.400 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:14:10.528 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 35 16 61191 61175 109.191 87 0.00998343 0.00915382 2023-09-23T23:14:10.675 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:10.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:10.682 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:10 smithi110 ceph-mon[212939]: pgmap v560: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T23:14:10.682 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:10 smithi110 ceph-mon[221102]: pgmap v560: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T23:14:10.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:10.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:10 smithi138 ceph-mon[193606]: pgmap v560: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T23:14:10.783 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:14:10.832 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:10.903 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:14:10.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:10.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:14:11.010 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:14:11.085 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:11.324 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:14:11.360 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:14:11.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:11.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:14:11.474 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:11.520 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:11.528 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 36 16 62377 62361 108.217 74.125 0.00862805 0.00923418 2023-09-23T23:14:11.581 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:11.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:11.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:11.593 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:11.691 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:14:11.751 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:12.450 INFO:teuthology.orchestra.run.smithi110.stderr:set pool 1 pgp_num to 1 2023-09-23T23:14:12.529 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 37 16 63226 63210 106.725 53.0625 0.00719843 0.00936441 2023-09-23T23:14:12.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:12 smithi110 ceph-mon[221102]: pgmap v561: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T23:14:12.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:12 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3453888070' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T23:14:12.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:12 smithi110 ceph-mon[212939]: pgmap v561: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T23:14:12.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:12 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3453888070' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T23:14:12.847 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:14:12.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:12.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:14:12.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:12 smithi138 ceph-mon[193606]: pgmap v561: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T23:14:12.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:12 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3453888070' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T23:14:12.958 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:14:13.023 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:13.510 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:14:13.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:13.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:14:13.530 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 38 15 64487 64472 105.99 78.875 0.0173266 0.00943082 2023-09-23T23:14:13.620 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:13.697 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:13.721 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3453888070' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T23:14:13.721 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[212939]: osdmap e172: 8 total, 8 up, 8 in 2023-09-23T23:14:13.721 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[212939]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.721 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[212939]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.721 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[212939]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.721 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[212939]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.721 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[212939]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.721 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[212939]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.722 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[212939]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.722 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:13.722 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:14:13.723 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3453888070' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T23:14:13.723 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[221102]: osdmap e172: 8 total, 8 up, 8 in 2023-09-23T23:14:13.723 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[221102]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.723 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[221102]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.724 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[221102]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.724 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[221102]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.724 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[221102]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.724 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[221102]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.724 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[221102]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.724 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:13.724 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:14:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:13 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3453888070' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T23:14:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:13 smithi138 ceph-mon[193606]: osdmap e172: 8 total, 8 up, 8 in 2023-09-23T23:14:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:13 smithi138 ceph-mon[193606]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:13 smithi138 ceph-mon[193606]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:13 smithi138 ceph-mon[193606]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:13 smithi138 ceph-mon[193606]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:13 smithi138 ceph-mon[193606]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:13 smithi138 ceph-mon[193606]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:13 smithi138 ceph-mon[193606]: failed to encode map e172 with expected crc 2023-09-23T23:14:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:14:13.986 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:13.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:13.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:14.094 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:14:14.105 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:14.163 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:14.370 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:14:14.371 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:14:14.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:14:14.371 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:14:14.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.371 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:79757 8.1 8:8223ab19:::benchmark_data_smithi110_258128_object79756:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T23:14:14.371 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:08:44.124280+0000", 2023-09-23T23:14:14.371 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 330.241592105, 2023-09-23T23:14:14.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0958221720000001, 2023-09-23T23:14:14.371 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.371 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.371 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.372 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.372 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.372 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 79757 2023-09-23T23:14:14.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.372 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.124280+0000", 2023-09-23T23:14:14.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.124280+0000", 2023-09-23T23:14:14.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.124278+0000", 2023-09-23T23:14:14.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:14:14.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.124302+0000", 2023-09-23T23:14:14.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3597000000000001e-05 2023-09-23T23:14:14.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.124313+0000", 2023-09-23T23:14:14.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1409e-05 2023-09-23T23:14:14.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.124323+0000", 2023-09-23T23:14:14.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1269999999999992e-06 2023-09-23T23:14:14.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.124377+0000", 2023-09-23T23:14:14.375 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4321999999999999e-05 2023-09-23T23:14:14.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.124389+0000", 2023-09-23T23:14:14.375 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2615e-05 2023-09-23T23:14:14.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:45.112628+0000", 2023-09-23T23:14:14.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98823848700000005 2023-09-23T23:14:14.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:45.112643+0000", 2023-09-23T23:14:14.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5336e-05 2023-09-23T23:14:14.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.201214+0000", 2023-09-23T23:14:14.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.088570214 2023-09-23T23:14:14.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.201379+0000", 2023-09-23T23:14:14.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016533100000000001 2023-09-23T23:14:14.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-09-23T23:14:14.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.201583+0000", 2023-09-23T23:14:14.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000203797 2023-09-23T23:14:14.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.209142+0000", 2023-09-23T23:14:14.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075589309999999996 2023-09-23T23:14:14.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.216450+0000", 2023-09-23T23:14:14.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0073086610000000002 2023-09-23T23:14:14.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.219035+0000", 2023-09-23T23:14:14.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025845550000000001 2023-09-23T23:14:14.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.379 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.379 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.219089+0000", 2023-09-23T23:14:14.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4561999999999997e-05 2023-09-23T23:14:14.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.379 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.379 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.220102+0000", 2023-09-23T23:14:14.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001012951 2023-09-23T23:14:14.379 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.380 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.380 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.380 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:79766 8.1 8:8a96788b:::benchmark_data_smithi110_258128_object79765:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T23:14:14.380 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:08:44.129227+0000", 2023-09-23T23:14:14.380 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 330.236644953, 2023-09-23T23:14:14.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0952115990000002, 2023-09-23T23:14:14.380 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.380 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.380 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.380 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.381 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.381 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 79766 2023-09-23T23:14:14.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.381 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.129227+0000", 2023-09-23T23:14:14.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.129227+0000", 2023-09-23T23:14:14.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.129226+0000", 2023-09-23T23:14:14.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:14:14.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.129250+0000", 2023-09-23T23:14:14.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4691000000000001e-05 2023-09-23T23:14:14.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.129266+0000", 2023-09-23T23:14:14.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5688e-05 2023-09-23T23:14:14.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.129274+0000", 2023-09-23T23:14:14.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0600000000000008e-06 2023-09-23T23:14:14.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.384 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.384 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.129329+0000", 2023-09-23T23:14:14.384 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4721e-05 2023-09-23T23:14:14.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.384 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.384 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.129340+0000", 2023-09-23T23:14:14.384 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0909e-05 2023-09-23T23:14:14.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.384 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.384 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:45.113197+0000", 2023-09-23T23:14:14.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98385710800000004 2023-09-23T23:14:14.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.385 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:45.113209+0000", 2023-09-23T23:14:14.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1694e-05 2023-09-23T23:14:14.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.385 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.205779+0000", 2023-09-23T23:14:14.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0925704430000001 2023-09-23T23:14:14.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.205957+0000", 2023-09-23T23:14:14.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000177858 2023-09-23T23:14:14.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-09-23T23:14:14.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.206167+0000", 2023-09-23T23:14:14.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000209936 2023-09-23T23:14:14.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.212899+0000", 2023-09-23T23:14:14.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067326469999999996 2023-09-23T23:14:14.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.216485+0000", 2023-09-23T23:14:14.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035853999999999999 2023-09-23T23:14:14.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.222846+0000", 2023-09-23T23:14:14.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063615360000000001 2023-09-23T23:14:14.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.222893+0000", 2023-09-23T23:14:14.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6851999999999998e-05 2023-09-23T23:14:14.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.224439+0000", 2023-09-23T23:14:14.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015457050000000001 2023-09-23T23:14:14.388 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.388 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.389 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.389 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.389 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:79769 8.1 8:8ba3fe28:::benchmark_data_smithi110_258128_object79768:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T23:14:14.389 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:08:44.129356+0000", 2023-09-23T23:14:14.389 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 330.236516499, 2023-09-23T23:14:14.389 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0992347320000002, 2023-09-23T23:14:14.389 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.389 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.389 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.389 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.389 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.390 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 79769 2023-09-23T23:14:14.390 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.390 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.390 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.390 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.390 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.129356+0000", 2023-09-23T23:14:14.390 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.390 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.390 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.390 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.390 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.129356+0000", 2023-09-23T23:14:14.390 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.391 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.129355+0000", 2023-09-23T23:14:14.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:14:14.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.391 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.129385+0000", 2023-09-23T23:14:14.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0224000000000001e-05 2023-09-23T23:14:14.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.129395+0000", 2023-09-23T23:14:14.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3570000000000004e-06 2023-09-23T23:14:14.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.129401+0000", 2023-09-23T23:14:14.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3790000000000003e-06 2023-09-23T23:14:14.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.129450+0000", 2023-09-23T23:14:14.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8869000000000003e-05 2023-09-23T23:14:14.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.393 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.129458+0000", 2023-09-23T23:14:14.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1990000000000006e-06 2023-09-23T23:14:14.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.393 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:45.113470+0000", 2023-09-23T23:14:14.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98401165700000004 2023-09-23T23:14:14.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:45.113481+0000", 2023-09-23T23:14:14.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0794e-05 2023-09-23T23:14:14.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.209918+0000", 2023-09-23T23:14:14.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0964377080000001 2023-09-23T23:14:14.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.210058+0000", 2023-09-23T23:14:14.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000139419 2023-09-23T23:14:14.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-09-23T23:14:14.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.210226+0000", 2023-09-23T23:14:14.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016849000000000001 2023-09-23T23:14:14.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.220120+0000", 2023-09-23T23:14:14.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0098940940000000008 2023-09-23T23:14:14.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.227445+0000", 2023-09-23T23:14:14.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007324367 2023-09-23T23:14:14.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.227479+0000", 2023-09-23T23:14:14.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4396000000000001e-05 2023-09-23T23:14:14.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.397 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.397 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.227541+0000", 2023-09-23T23:14:14.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1710000000000004e-05 2023-09-23T23:14:14.397 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.397 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.397 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.397 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.228590+0000", 2023-09-23T23:14:14.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010495840000000001 2023-09-23T23:14:14.397 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.397 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.397 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.397 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.398 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.398 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:79777 8.a 8:5bc8df09:::benchmark_data_smithi110_258128_object79776:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T23:14:14.398 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:08:44.138724+0000", 2023-09-23T23:14:14.398 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 330.22714867000002, 2023-09-23T23:14:14.398 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.068394472, 2023-09-23T23:14:14.398 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.398 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.398 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.398 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.398 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.398 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 79777 2023-09-23T23:14:14.398 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.399 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.399 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.399 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.138724+0000", 2023-09-23T23:14:14.399 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.399 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.399 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.138724+0000", 2023-09-23T23:14:14.399 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.400 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.138723+0000", 2023-09-23T23:14:14.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:14:14.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.400 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.400 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.138742+0000", 2023-09-23T23:14:14.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9585e-05 2023-09-23T23:14:14.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.400 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.400 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.138754+0000", 2023-09-23T23:14:14.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1878999999999999e-05 2023-09-23T23:14:14.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.138763+0000", 2023-09-23T23:14:14.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1749999999999994e-06 2023-09-23T23:14:14.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.138825+0000", 2023-09-23T23:14:14.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2224000000000006e-05 2023-09-23T23:14:14.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:44.138837+0000", 2023-09-23T23:14:14.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1333e-05 2023-09-23T23:14:14.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:45.113730+0000", 2023-09-23T23:14:14.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97489280499999997 2023-09-23T23:14:14.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:45.113742+0000", 2023-09-23T23:14:14.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2024e-05 2023-09-23T23:14:14.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.194394+0000", 2023-09-23T23:14:14.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0806528310000001 2023-09-23T23:14:14.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.194510+0000", 2023-09-23T23:14:14.404 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000115707 2023-09-23T23:14:14.404 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.404 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.404 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:14:14.404 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.194690+0000", 2023-09-23T23:14:14.404 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000179703 2023-09-23T23:14:14.404 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.404 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.404 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.404 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.198899+0000", 2023-09-23T23:14:14.404 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042095689999999998 2023-09-23T23:14:14.404 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.405 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.405 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.405 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.203819+0000", 2023-09-23T23:14:14.405 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0049195640000000004 2023-09-23T23:14:14.405 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.405 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.405 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.405 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.206116+0000", 2023-09-23T23:14:14.405 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002296766 2023-09-23T23:14:14.405 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.405 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.406 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.406 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.206188+0000", 2023-09-23T23:14:14.406 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1975000000000001e-05 2023-09-23T23:14:14.406 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.406 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.406 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.406 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:08:46.207118+0000", 2023-09-23T23:14:14.406 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00093038500000000004 2023-09-23T23:14:14.406 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.406 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.406 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.406 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.407 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.407 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:167360 8.1 8:811d435b:::benchmark_data_smithi110_258128_object48542:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T23:14:14.407 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:33.109922+0000", 2023-09-23T23:14:14.407 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 221.255949899, 2023-09-23T23:14:14.407 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0486706139999999, 2023-09-23T23:14:14.407 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.407 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.407 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.407 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.407 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.407 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 167360 2023-09-23T23:14:14.407 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.408 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.408 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.408 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:33.109922+0000", 2023-09-23T23:14:14.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.408 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.408 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.408 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:33.109922+0000", 2023-09-23T23:14:14.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.408 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.408 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.409 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:33.109921+0000", 2023-09-23T23:14:14.409 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:14:14.409 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.409 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.409 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.409 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:33.109926+0000", 2023-09-23T23:14:14.409 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2410000000000003e-06 2023-09-23T23:14:14.409 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.409 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.409 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.409 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:33.109939+0000", 2023-09-23T23:14:14.409 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2561000000000001e-05 2023-09-23T23:14:14.410 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.410 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.410 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.410 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:33.109946+0000", 2023-09-23T23:14:14.410 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6410000000000002e-06 2023-09-23T23:14:14.410 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.410 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.410 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.410 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:33.114215+0000", 2023-09-23T23:14:14.410 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042684819999999997 2023-09-23T23:14:14.410 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.410 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.411 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:33.114229+0000", 2023-09-23T23:14:14.411 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.433e-05 2023-09-23T23:14:14.411 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.411 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.411 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:34.147292+0000", 2023-09-23T23:14:14.411 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.033062374 2023-09-23T23:14:14.411 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.411 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.411 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:34.147385+0000", 2023-09-23T23:14:14.412 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2927e-05 2023-09-23T23:14:14.412 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.412 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.412 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-09-23T23:14:14.412 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:34.147467+0000", 2023-09-23T23:14:14.412 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2063000000000004e-05 2023-09-23T23:14:14.412 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.412 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.412 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.412 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:34.150606+0000", 2023-09-23T23:14:14.412 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031394080000000002 2023-09-23T23:14:14.412 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.413 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.413 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.413 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:34.158479+0000", 2023-09-23T23:14:14.413 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078725649999999998 2023-09-23T23:14:14.413 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.413 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.413 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.413 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:34.158524+0000", 2023-09-23T23:14:14.413 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5862000000000002e-05 2023-09-23T23:14:14.413 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.413 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.413 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.413 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:34.158569+0000", 2023-09-23T23:14:14.414 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4752000000000001e-05 2023-09-23T23:14:14.414 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.414 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.414 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.414 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:34.158593+0000", 2023-09-23T23:14:14.414 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3779000000000001e-05 2023-09-23T23:14:14.414 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.414 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.414 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.414 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.414 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.415 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:171312 8.7 8:e7f3ad60:::benchmark_data_smithi110_258128_object52494:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T23:14:14.415 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:39.138627+0000", 2023-09-23T23:14:14.415 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 215.22724544100001, 2023-09-23T23:14:14.415 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.217869876, 2023-09-23T23:14:14.415 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.415 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.415 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.415 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.415 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.415 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 171312 2023-09-23T23:14:14.415 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.415 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.416 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.416 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.416 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.138627+0000", 2023-09-23T23:14:14.416 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.416 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.416 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.416 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.416 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.138627+0000", 2023-09-23T23:14:14.416 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.416 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.416 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.416 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.417 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.138625+0000", 2023-09-23T23:14:14.417 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:14:14.417 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.417 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.417 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.417 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.138634+0000", 2023-09-23T23:14:14.417 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1910000000000006e-06 2023-09-23T23:14:14.417 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.417 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.417 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.417 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.138653+0000", 2023-09-23T23:14:14.417 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9346000000000001e-05 2023-09-23T23:14:14.417 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.418 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.418 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.418 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.138661+0000", 2023-09-23T23:14:14.418 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5719999999999997e-06 2023-09-23T23:14:14.418 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.418 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.418 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.418 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.138730+0000", 2023-09-23T23:14:14.418 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8798000000000001e-05 2023-09-23T23:14:14.418 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.418 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.419 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.419 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.138742+0000", 2023-09-23T23:14:14.419 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2143999999999999e-05 2023-09-23T23:14:14.419 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.419 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.419 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.419 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.347291+0000", 2023-09-23T23:14:14.419 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2085489270000001 2023-09-23T23:14:14.419 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.419 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.419 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.419 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.347408+0000", 2023-09-23T23:14:14.419 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000116766 2023-09-23T23:14:14.420 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.420 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.420 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 6", 2023-09-23T23:14:14.420 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.347525+0000", 2023-09-23T23:14:14.420 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011705 2023-09-23T23:14:14.420 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.420 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.420 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.420 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.356353+0000", 2023-09-23T23:14:14.420 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0088281999999999996 2023-09-23T23:14:14.420 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.421 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.421 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.421 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.356361+0000", 2023-09-23T23:14:14.421 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5229999999999995e-06 2023-09-23T23:14:14.421 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.421 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.421 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.421 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.356409+0000", 2023-09-23T23:14:14.421 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7580000000000002e-05 2023-09-23T23:14:14.421 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.421 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.421 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.422 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.356497+0000", 2023-09-23T23:14:14.422 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7669000000000004e-05 2023-09-23T23:14:14.422 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.422 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.422 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.422 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.422 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.422 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:171318 8.7 8:e314d875:::benchmark_data_smithi110_258128_object52500:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T23:14:14.422 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:39.150216+0000", 2023-09-23T23:14:14.422 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 215.21565618700001, 2023-09-23T23:14:14.422 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.220534429, 2023-09-23T23:14:14.423 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.423 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.423 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.423 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.423 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.423 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 171318 2023-09-23T23:14:14.423 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.423 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.423 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.423 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.423 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.150216+0000", 2023-09-23T23:14:14.423 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.424 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.424 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.424 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.424 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.150216+0000", 2023-09-23T23:14:14.424 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.424 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.424 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.424 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.424 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.150215+0000", 2023-09-23T23:14:14.424 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:14:14.424 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.424 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.425 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.425 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.150219+0000", 2023-09-23T23:14:14.425 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2910000000000001e-06 2023-09-23T23:14:14.425 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.425 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.425 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.425 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.150229+0000", 2023-09-23T23:14:14.425 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5349999999999993e-06 2023-09-23T23:14:14.425 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.425 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.425 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.426 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.150234+0000", 2023-09-23T23:14:14.426 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2100000000000001e-06 2023-09-23T23:14:14.426 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.426 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.426 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.426 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.150269+0000", 2023-09-23T23:14:14.426 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4434000000000003e-05 2023-09-23T23:14:14.426 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.426 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.426 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.426 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:39.150277+0000", 2023-09-23T23:14:14.426 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.96e-06 2023-09-23T23:14:14.427 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.427 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.427 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.427 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.352407+0000", 2023-09-23T23:14:14.427 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.202130817 2023-09-23T23:14:14.427 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.427 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.427 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.427 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.352550+0000", 2023-09-23T23:14:14.427 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000142464 2023-09-23T23:14:14.427 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.427 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.428 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 6", 2023-09-23T23:14:14.428 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.352676+0000", 2023-09-23T23:14:14.428 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012622800000000001 2023-09-23T23:14:14.428 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.428 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.428 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.428 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.363512+0000", 2023-09-23T23:14:14.428 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010836411000000001 2023-09-23T23:14:14.428 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.428 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.428 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.428 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.370619+0000", 2023-09-23T23:14:14.428 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071061450000000003 2023-09-23T23:14:14.428 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.429 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.429 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.429 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.370653+0000", 2023-09-23T23:14:14.429 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4601000000000003e-05 2023-09-23T23:14:14.429 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.429 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.429 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.429 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:40.370750+0000", 2023-09-23T23:14:14.429 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7288999999999996e-05 2023-09-23T23:14:14.429 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.429 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.429 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.429 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.430 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.430 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:186175 8.7 8:e9a8db8e:::benchmark_data_smithi110_258128_object67357:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T23:14:14.430 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:58.481215+0000", 2023-09-23T23:14:14.430 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 195.884657092, 2023-09-23T23:14:14.430 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4581557780000001, 2023-09-23T23:14:14.430 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.430 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:14:14.430 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.430 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.430 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.430 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 186175 2023-09-23T23:14:14.431 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.431 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.431 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.431 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.431 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.481215+0000", 2023-09-23T23:14:14.431 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.431 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.431 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.431 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.431 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.481215+0000", 2023-09-23T23:14:14.431 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.431 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.432 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.432 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.432 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.481214+0000", 2023-09-23T23:14:14.432 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:14:14.432 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.432 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.432 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.432 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.481222+0000", 2023-09-23T23:14:14.432 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7510000000000008e-06 2023-09-23T23:14:14.432 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.432 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.432 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.433 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.481235+0000", 2023-09-23T23:14:14.433 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2918000000000001e-05 2023-09-23T23:14:14.433 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.433 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.433 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.433 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.481243+0000", 2023-09-23T23:14:14.433 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9799999999999998e-06 2023-09-23T23:14:14.433 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.433 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.433 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.433 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.481294+0000", 2023-09-23T23:14:14.433 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1684999999999997e-05 2023-09-23T23:14:14.433 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.434 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.434 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.434 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.481428+0000", 2023-09-23T23:14:14.434 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000133528 2023-09-23T23:14:14.434 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.434 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.434 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 6", 2023-09-23T23:14:14.434 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.481511+0000", 2023-09-23T23:14:14.434 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3660999999999997e-05 2023-09-23T23:14:14.434 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.434 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.435 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.435 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.486774+0000", 2023-09-23T23:14:14.435 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052620870000000004 2023-09-23T23:14:14.435 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.435 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.435 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.435 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.495519+0000", 2023-09-23T23:14:14.435 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0087452769999999992 2023-09-23T23:14:14.435 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.435 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.435 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.435 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.495531+0000", 2023-09-23T23:14:14.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2680999999999999e-05 2023-09-23T23:14:14.436 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.436 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.436 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.492744+0000", 2023-09-23T23:14:14.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99721228799999995 2023-09-23T23:14:14.436 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.436 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.436 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.492756+0000", 2023-09-23T23:14:14.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1790999999999999e-05 2023-09-23T23:14:14.436 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.437 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.921862+0000", 2023-09-23T23:14:14.437 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.429106473 2023-09-23T23:14:14.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:14:14.437 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.921984+0000", 2023-09-23T23:14:14.437 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000122104 2023-09-23T23:14:14.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.939306+0000", 2023-09-23T23:14:14.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017322140999999999 2023-09-23T23:14:14.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.438 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.939371+0000", 2023-09-23T23:14:14.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4682000000000002e-05 2023-09-23T23:14:14.438 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.438 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.438 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.439 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.439 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:186193 8.7 8:e7c1e81f:::benchmark_data_smithi110_258128_object67375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T23:14:14.439 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:58.505403+0000", 2023-09-23T23:14:14.439 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 195.860469341, 2023-09-23T23:14:14.439 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4336448380000002, 2023-09-23T23:14:14.439 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.439 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.439 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.439 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.439 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.439 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 186193 2023-09-23T23:14:14.440 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.440 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.440 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.440 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.440 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.505403+0000", 2023-09-23T23:14:14.440 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.440 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.440 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.440 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.440 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.505403+0000", 2023-09-23T23:14:14.440 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.440 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.441 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.441 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.505402+0000", 2023-09-23T23:14:14.441 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:14:14.441 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.441 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.441 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.505406+0000", 2023-09-23T23:14:14.441 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2430000000000001e-06 2023-09-23T23:14:14.441 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.441 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.441 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.505413+0000", 2023-09-23T23:14:14.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0310000000000001e-06 2023-09-23T23:14:14.442 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.442 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.442 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.442 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.505418+0000", 2023-09-23T23:14:14.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8489999999999997e-06 2023-09-23T23:14:14.442 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.442 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.442 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.442 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.509792+0000", 2023-09-23T23:14:14.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043742190000000004 2023-09-23T23:14:14.442 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.443 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.509805+0000", 2023-09-23T23:14:14.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.359e-05 2023-09-23T23:14:14.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.443 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.492810+0000", 2023-09-23T23:14:14.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98300426500000004 2023-09-23T23:14:14.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.492817+0000", 2023-09-23T23:14:14.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6079999999999998e-06 2023-09-23T23:14:14.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.444 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.922472+0000", 2023-09-23T23:14:14.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.429655176 2023-09-23T23:14:14.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.444 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.922599+0000", 2023-09-23T23:14:14.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000126611 2023-09-23T23:14:14.445 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.445 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.445 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:14:14.445 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.922837+0000", 2023-09-23T23:14:14.445 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023818999999999999 2023-09-23T23:14:14.445 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.445 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.445 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.445 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.933892+0000", 2023-09-23T23:14:14.445 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011054401 2023-09-23T23:14:14.445 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.446 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.446 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.933956+0000", 2023-09-23T23:14:14.446 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4387000000000005e-05 2023-09-23T23:14:14.446 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.446 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.446 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.938843+0000", 2023-09-23T23:14:14.446 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048870140000000003 2023-09-23T23:14:14.446 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.446 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.446 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.938881+0000", 2023-09-23T23:14:14.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7978000000000001e-05 2023-09-23T23:14:14.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.447 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.447 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.447 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.939048+0000", 2023-09-23T23:14:14.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016673400000000001 2023-09-23T23:14:14.447 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.447 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.447 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.447 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.447 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:186199 8.7 8:e8c863d1:::benchmark_data_smithi110_258128_object67381:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T23:14:14.448 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:58.517522+0000", 2023-09-23T23:14:14.448 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 195.848350125, 2023-09-23T23:14:14.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4221203579999999, 2023-09-23T23:14:14.448 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.448 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.448 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.448 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.448 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.448 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 186199 2023-09-23T23:14:14.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.448 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.517522+0000", 2023-09-23T23:14:14.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.517522+0000", 2023-09-23T23:14:14.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.517521+0000", 2023-09-23T23:14:14.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:14:14.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.450 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.517525+0000", 2023-09-23T23:14:14.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5090000000000002e-06 2023-09-23T23:14:14.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.450 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.517533+0000", 2023-09-23T23:14:14.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2660000000000001e-06 2023-09-23T23:14:14.451 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.451 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.451 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.451 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.517554+0000", 2023-09-23T23:14:14.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0707999999999999e-05 2023-09-23T23:14:14.451 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.451 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.451 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.451 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.517601+0000", 2023-09-23T23:14:14.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7021999999999998e-05 2023-09-23T23:14:14.451 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.517612+0000", 2023-09-23T23:14:14.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.084e-05 2023-09-23T23:14:14.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.492842+0000", 2023-09-23T23:14:14.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97523037099999998 2023-09-23T23:14:14.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.492849+0000", 2023-09-23T23:14:14.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5689999999999999e-06 2023-09-23T23:14:14.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.926191+0000", 2023-09-23T23:14:14.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4333417960000001 2023-09-23T23:14:14.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.926318+0000", 2023-09-23T23:14:14.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000127038 2023-09-23T23:14:14.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:14:14.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.926408+0000", 2023-09-23T23:14:14.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9640000000000002e-05 2023-09-23T23:14:14.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.933970+0000", 2023-09-23T23:14:14.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075625149999999997 2023-09-23T23:14:14.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.938806+0000", 2023-09-23T23:14:14.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048353789999999999 2023-09-23T23:14:14.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.939429+0000", 2023-09-23T23:14:14.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00062304899999999998 2023-09-23T23:14:14.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.939488+0000", 2023-09-23T23:14:14.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9259000000000003e-05 2023-09-23T23:14:14.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.456 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.939642+0000", 2023-09-23T23:14:14.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000154365 2023-09-23T23:14:14.456 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.456 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.456 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.457 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:186201 8.7 8:e3b026a9:::benchmark_data_smithi110_258128_object67383:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T23:14:14.457 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:58.525568+0000", 2023-09-23T23:14:14.457 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 195.84030471700001, 2023-09-23T23:14:14.457 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4347360390000001, 2023-09-23T23:14:14.457 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.457 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.457 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.457 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.457 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.457 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 186201 2023-09-23T23:14:14.457 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.457 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.458 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.458 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.458 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.525568+0000", 2023-09-23T23:14:14.458 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.458 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.458 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.458 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.525568+0000", 2023-09-23T23:14:14.458 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.458 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.458 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.525566+0000", 2023-09-23T23:14:14.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:14:14.459 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.459 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.525574+0000", 2023-09-23T23:14:14.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2269999999999993e-06 2023-09-23T23:14:14.459 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.459 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.525588+0000", 2023-09-23T23:14:14.460 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.393e-05 2023-09-23T23:14:14.460 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.460 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.460 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.460 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.525597+0000", 2023-09-23T23:14:14.460 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8789999999999992e-06 2023-09-23T23:14:14.460 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.460 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.460 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.460 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.525652+0000", 2023-09-23T23:14:14.460 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5108999999999997e-05 2023-09-23T23:14:14.460 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.461 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.525660+0000", 2023-09-23T23:14:14.461 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3000000000000002e-06 2023-09-23T23:14:14.461 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.461 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.492873+0000", 2023-09-23T23:14:14.461 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.967213141 2023-09-23T23:14:14.461 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.461 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.492881+0000", 2023-09-23T23:14:14.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5970000000000003e-06 2023-09-23T23:14:14.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.929858+0000", 2023-09-23T23:14:14.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4369778090000001 2023-09-23T23:14:14.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.929978+0000", 2023-09-23T23:14:14.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0001193 2023-09-23T23:14:14.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.463 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:14:14.463 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.930058+0000", 2023-09-23T23:14:14.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0391999999999999e-05 2023-09-23T23:14:14.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.463 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.463 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.938815+0000", 2023-09-23T23:14:14.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0087569320000000003 2023-09-23T23:14:14.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.463 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.939088+0000", 2023-09-23T23:14:14.464 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027249300000000001 2023-09-23T23:14:14.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.960108+0000", 2023-09-23T23:14:14.464 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021020794999999998 2023-09-23T23:14:14.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.960147+0000", 2023-09-23T23:14:14.464 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8859000000000002e-05 2023-09-23T23:14:14.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.465 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.465 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.960304+0000", 2023-09-23T23:14:14.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015631099999999999 2023-09-23T23:14:14.465 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.465 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.465 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.465 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:186221 8.7 8:efe859c8:::benchmark_data_smithi110_258128_object67403:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T23:14:14.465 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:58.555350+0000", 2023-09-23T23:14:14.465 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 195.810521765, 2023-09-23T23:14:14.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4051608560000002, 2023-09-23T23:14:14.466 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.466 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.466 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.466 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.466 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.466 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 186221 2023-09-23T23:14:14.466 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.466 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.466 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.466 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.466 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.555350+0000", 2023-09-23T23:14:14.466 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.467 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.467 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.555350+0000", 2023-09-23T23:14:14.467 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.467 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.467 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.555350+0000", 2023-09-23T23:14:14.467 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:14:14.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.468 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.468 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.555353+0000", 2023-09-23T23:14:14.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3390000000000001e-06 2023-09-23T23:14:14.468 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.468 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.468 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.555360+0000", 2023-09-23T23:14:14.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0190000000000001e-06 2023-09-23T23:14:14.468 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.468 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.469 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.555367+0000", 2023-09-23T23:14:14.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1340000000000001e-06 2023-09-23T23:14:14.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.469 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.469 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.469 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.555407+0000", 2023-09-23T23:14:14.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0865000000000003e-05 2023-09-23T23:14:14.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.469 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.469 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.469 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:58.555415+0000", 2023-09-23T23:14:14.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3629999999999996e-06 2023-09-23T23:14:14.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.493593+0000", 2023-09-23T23:14:14.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93817841099999999 2023-09-23T23:14:14.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.493606+0000", 2023-09-23T23:14:14.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3078000000000001e-05 2023-09-23T23:14:14.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.934633+0000", 2023-09-23T23:14:14.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4410272179999999 2023-09-23T23:14:14.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.934749+0000", 2023-09-23T23:14:14.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011522 2023-09-23T23:14:14.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:14:14.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.934870+0000", 2023-09-23T23:14:14.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000121093 2023-09-23T23:14:14.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.943845+0000", 2023-09-23T23:14:14.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0089752530000000007 2023-09-23T23:14:14.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.947970+0000", 2023-09-23T23:14:14.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041249069999999997 2023-09-23T23:14:14.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.960370+0000", 2023-09-23T23:14:14.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012399988000000001 2023-09-23T23:14:14.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.960408+0000", 2023-09-23T23:14:14.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7784000000000003e-05 2023-09-23T23:14:14.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.960511+0000", 2023-09-23T23:14:14.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103663 2023-09-23T23:14:14.474 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.474 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.474 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.474 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:186186 8.0 8:097a68e4:::benchmark_data_smithi110_258128_object67368:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T23:14:14.474 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:59.490205+0000", 2023-09-23T23:14:14.474 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 194.87566677999999, 2023-09-23T23:14:14.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4007943649999999, 2023-09-23T23:14:14.474 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.475 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.475 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.475 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.475 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.475 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 186186 2023-09-23T23:14:14.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.475 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490205+0000", 2023-09-23T23:14:14.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490205+0000", 2023-09-23T23:14:14.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490204+0000", 2023-09-23T23:14:14.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:14:14.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490213+0000", 2023-09-23T23:14:14.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9320000000000002e-06 2023-09-23T23:14:14.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.477 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.477 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.477 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490228+0000", 2023-09-23T23:14:14.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5301000000000001e-05 2023-09-23T23:14:14.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.477 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.477 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.477 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490240+0000", 2023-09-23T23:14:14.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2e-05 2023-09-23T23:14:14.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.478 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.478 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490654+0000", 2023-09-23T23:14:14.478 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00041429299999999999 2023-09-23T23:14:14.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.478 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.478 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490666+0000", 2023-09-23T23:14:14.478 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1887e-05 2023-09-23T23:14:14.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.478 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.859551+0000", 2023-09-23T23:14:14.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3688844499999999 2023-09-23T23:14:14.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.862946+0000", 2023-09-23T23:14:14.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033956770000000002 2023-09-23T23:14:14.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:14:14.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.863380+0000", 2023-09-23T23:14:14.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00043338099999999999 2023-09-23T23:14:14.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.881825+0000", 2023-09-23T23:14:14.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018445158 2023-09-23T23:14:14.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.890204+0000", 2023-09-23T23:14:14.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083789740000000008 2023-09-23T23:14:14.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.890791+0000", 2023-09-23T23:14:14.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00058678199999999999 2023-09-23T23:14:14.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.890851+0000", 2023-09-23T23:14:14.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.067e-05 2023-09-23T23:14:14.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.891000+0000", 2023-09-23T23:14:14.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014861900000000001 2023-09-23T23:14:14.482 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.482 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.482 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.482 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:186187 8.f 8:f88181aa:::benchmark_data_smithi110_258128_object67369:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T23:14:14.482 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:59.490248+0000", 2023-09-23T23:14:14.482 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 194.87562452500001, 2023-09-23T23:14:14.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4320504089999999, 2023-09-23T23:14:14.482 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.482 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.482 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.483 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.483 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.483 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 186187 2023-09-23T23:14:14.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.483 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490248+0000", 2023-09-23T23:14:14.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.484 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.484 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490248+0000", 2023-09-23T23:14:14.484 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.484 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.484 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.484 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490247+0000", 2023-09-23T23:14:14.484 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:14:14.484 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.484 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.485 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490251+0000", 2023-09-23T23:14:14.485 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7610000000000001e-06 2023-09-23T23:14:14.485 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.485 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.485 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.485 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490258+0000", 2023-09-23T23:14:14.485 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4649999999999997e-06 2023-09-23T23:14:14.485 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.485 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.485 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.485 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490264+0000", 2023-09-23T23:14:14.485 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8109999999999999e-06 2023-09-23T23:14:14.485 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.486 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.486 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.492901+0000", 2023-09-23T23:14:14.486 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026372499999999998 2023-09-23T23:14:14.486 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.486 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.486 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.492910+0000", 2023-09-23T23:14:14.486 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0520000000000002e-06 2023-09-23T23:14:14.486 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.486 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.867829+0000", 2023-09-23T23:14:14.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.374919421 2023-09-23T23:14:14.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.871367+0000", 2023-09-23T23:14:14.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035383490000000001 2023-09-23T23:14:14.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:14:14.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.872135+0000", 2023-09-23T23:14:14.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00076768599999999995 2023-09-23T23:14:14.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.888305+0000", 2023-09-23T23:14:14.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016170205 2023-09-23T23:14:14.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.922065+0000", 2023-09-23T23:14:14.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.033760112000000002 2023-09-23T23:14:14.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.489 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.489 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.922098+0000", 2023-09-23T23:14:14.489 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2623999999999998e-05 2023-09-23T23:14:14.489 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.489 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.489 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.922147+0000", 2023-09-23T23:14:14.489 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9054000000000002e-05 2023-09-23T23:14:14.489 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.489 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.489 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.922298+0000", 2023-09-23T23:14:14.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015118 2023-09-23T23:14:14.490 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.490 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.490 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.490 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.490 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:186215 8.f 8:fff0bc03:::benchmark_data_smithi110_258128_object67397:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T23:14:14.490 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:59.490270+0000", 2023-09-23T23:14:14.490 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 194.87560240900001, 2023-09-23T23:14:14.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4697996550000001, 2023-09-23T23:14:14.490 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.490 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.491 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.491 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.491 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.491 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 186215 2023-09-23T23:14:14.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.491 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490270+0000", 2023-09-23T23:14:14.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.492 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490270+0000", 2023-09-23T23:14:14.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.492 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.492 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490269+0000", 2023-09-23T23:14:14.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:14:14.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.492 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.492 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490272+0000", 2023-09-23T23:14:14.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1089999999999998e-06 2023-09-23T23:14:14.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.493 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.493 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490279+0000", 2023-09-23T23:14:14.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9020000000000002e-06 2023-09-23T23:14:14.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.493 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.493 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490285+0000", 2023-09-23T23:14:14.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5090000000000001e-06 2023-09-23T23:14:14.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.493 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.494 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.492931+0000", 2023-09-23T23:14:14.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002646055 2023-09-23T23:14:14.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.494 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.494 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.492939+0000", 2023-09-23T23:14:14.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.841e-06 2023-09-23T23:14:14.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.494 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.494 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.933998+0000", 2023-09-23T23:14:14.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4410593229999999 2023-09-23T23:14:14.495 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.495 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.934112+0000", 2023-09-23T23:14:14.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000114076 2023-09-23T23:14:14.495 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.495 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:14:14.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.934232+0000", 2023-09-23T23:14:14.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011956299999999999 2023-09-23T23:14:14.495 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.947963+0000", 2023-09-23T23:14:14.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013731587 2023-09-23T23:14:14.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.959843+0000", 2023-09-23T23:14:14.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01187969 2023-09-23T23:14:14.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.959877+0000", 2023-09-23T23:14:14.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3889e-05 2023-09-23T23:14:14.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.959924+0000", 2023-09-23T23:14:14.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6609e-05 2023-09-23T23:14:14.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.960070+0000", 2023-09-23T23:14:14.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014599799999999999 2023-09-23T23:14:14.497 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.497 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.497 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.498 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.498 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:186219 8.0 8:0068d01c:::benchmark_data_smithi110_258128_object67401:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T23:14:14.498 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:59.490290+0000", 2023-09-23T23:14:14.498 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 194.87558261000001, 2023-09-23T23:14:14.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4073548769999999, 2023-09-23T23:14:14.498 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.498 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.498 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.498 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.498 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.498 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 186219 2023-09-23T23:14:14.498 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.499 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.499 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490290+0000", 2023-09-23T23:14:14.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.499 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490290+0000", 2023-09-23T23:14:14.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490289+0000", 2023-09-23T23:14:14.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:14:14.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490292+0000", 2023-09-23T23:14:14.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0539999999999999e-06 2023-09-23T23:14:14.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490301+0000", 2023-09-23T23:14:14.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0650000000000005e-06 2023-09-23T23:14:14.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490307+0000", 2023-09-23T23:14:14.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.852e-06 2023-09-23T23:14:14.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490689+0000", 2023-09-23T23:14:14.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00038154900000000002 2023-09-23T23:14:14.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490697+0000", 2023-09-23T23:14:14.502 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2579999999999995e-06 2023-09-23T23:14:14.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.502 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.502 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.877677+0000", 2023-09-23T23:14:14.502 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.386980136 2023-09-23T23:14:14.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.502 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.502 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.877813+0000", 2023-09-23T23:14:14.502 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000136243 2023-09-23T23:14:14.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.503 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:14:14.503 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.877946+0000", 2023-09-23T23:14:14.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013237500000000001 2023-09-23T23:14:14.503 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.503 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.503 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.886153+0000", 2023-09-23T23:14:14.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0082074149999999992 2023-09-23T23:14:14.503 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.503 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.504 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.891062+0000", 2023-09-23T23:14:14.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0049086199999999998 2023-09-23T23:14:14.504 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.504 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.504 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.504 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.897427+0000", 2023-09-23T23:14:14.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063654480000000001 2023-09-23T23:14:14.504 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.504 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.505 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.505 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.897483+0000", 2023-09-23T23:14:14.505 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5887999999999998e-05 2023-09-23T23:14:14.505 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.505 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.505 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.505 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.897645+0000", 2023-09-23T23:14:14.505 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000161435 2023-09-23T23:14:14.505 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.505 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.505 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.505 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.505 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.506 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:186229 8.0 8:0b265f76:::benchmark_data_smithi110_258128_object67411:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T23:14:14.506 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:10:59.490312+0000", 2023-09-23T23:14:14.506 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 194.87556074, 2023-09-23T23:14:14.506 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4075680909999999, 2023-09-23T23:14:14.506 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.506 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.506 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.506 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.506 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.506 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 186229 2023-09-23T23:14:14.506 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.506 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.507 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.507 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490312+0000", 2023-09-23T23:14:14.507 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.507 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.507 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490312+0000", 2023-09-23T23:14:14.507 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490311+0000", 2023-09-23T23:14:14.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:14:14.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490314+0000", 2023-09-23T23:14:14.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.931e-06 2023-09-23T23:14:14.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.509 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490321+0000", 2023-09-23T23:14:14.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6379999999999998e-06 2023-09-23T23:14:14.509 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.509 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.509 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490327+0000", 2023-09-23T23:14:14.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7070000000000003e-06 2023-09-23T23:14:14.509 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.509 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.509 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490720+0000", 2023-09-23T23:14:14.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039271800000000002 2023-09-23T23:14:14.509 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.510 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.510 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:10:59.490728+0000", 2023-09-23T23:14:14.510 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.259e-06 2023-09-23T23:14:14.510 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.510 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.510 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.510 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.881902+0000", 2023-09-23T23:14:14.510 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3911731569999999 2023-09-23T23:14:14.510 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.510 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.510 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.510 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.882061+0000", 2023-09-23T23:14:14.510 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000159548 2023-09-23T23:14:14.511 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.511 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.511 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:14:14.511 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.882234+0000", 2023-09-23T23:14:14.511 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017249799999999999 2023-09-23T23:14:14.511 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.511 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.511 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.511 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.886245+0000", 2023-09-23T23:14:14.511 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040113370000000002 2023-09-23T23:14:14.511 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.512 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.512 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.512 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.891102+0000", 2023-09-23T23:14:14.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048570649999999998 2023-09-23T23:14:14.512 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.512 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.512 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.512 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.897701+0000", 2023-09-23T23:14:14.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0065986769999999998 2023-09-23T23:14:14.512 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.512 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.512 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.512 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.897749+0000", 2023-09-23T23:14:14.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8769e-05 2023-09-23T23:14:14.513 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.513 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.513 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.513 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:00.897880+0000", 2023-09-23T23:14:14.513 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000130195 2023-09-23T23:14:14.513 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.513 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.513 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.513 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.513 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.513 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:231460 8.7 8:ec9457da:::benchmark_data_smithi110_258128_object112642:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160)", 2023-09-23T23:14:14.513 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:12:02.488704+0000", 2023-09-23T23:14:14.514 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 131.87716856599999, 2023-09-23T23:14:14.514 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0484992630000001, 2023-09-23T23:14:14.514 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.514 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.514 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.514 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.514 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.514 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 231460 2023-09-23T23:14:14.514 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.514 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.514 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.515 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.515 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.488704+0000", 2023-09-23T23:14:14.515 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.515 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.515 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.515 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.515 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.488704+0000", 2023-09-23T23:14:14.515 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.515 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.515 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.515 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.515 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.488702+0000", 2023-09-23T23:14:14.516 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:14:14.516 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.516 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.516 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.516 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.488709+0000", 2023-09-23T23:14:14.516 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9890000000000003e-06 2023-09-23T23:14:14.516 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.516 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.516 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.516 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.488722+0000", 2023-09-23T23:14:14.516 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2554e-05 2023-09-23T23:14:14.516 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.516 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.517 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.517 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.488729+0000", 2023-09-23T23:14:14.517 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5270000000000001e-06 2023-09-23T23:14:14.517 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.517 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.517 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.517 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.490805+0000", 2023-09-23T23:14:14.517 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020759279999999999 2023-09-23T23:14:14.517 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.517 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.517 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.517 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.490813+0000", 2023-09-23T23:14:14.518 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5009999999999998e-06 2023-09-23T23:14:14.518 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.518 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.518 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.518 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.527341+0000", 2023-09-23T23:14:14.518 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0365287249999999 2023-09-23T23:14:14.518 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.518 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.518 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.518 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.527468+0000", 2023-09-23T23:14:14.518 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012719399999999999 2023-09-23T23:14:14.518 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.519 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.519 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:14:14.519 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.527586+0000", 2023-09-23T23:14:14.519 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117082 2023-09-23T23:14:14.519 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.519 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.519 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.519 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.531041+0000", 2023-09-23T23:14:14.519 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003455329 2023-09-23T23:14:14.519 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.519 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.519 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.520 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.531050+0000", 2023-09-23T23:14:14.520 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7110000000000001e-06 2023-09-23T23:14:14.520 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.520 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.520 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.520 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.537149+0000", 2023-09-23T23:14:14.520 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060991860000000004 2023-09-23T23:14:14.520 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.520 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.520 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.520 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.537179+0000", 2023-09-23T23:14:14.521 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0236999999999999e-05 2023-09-23T23:14:14.521 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.521 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.521 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.521 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.537203+0000", 2023-09-23T23:14:14.521 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3960000000000001e-05 2023-09-23T23:14:14.521 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.521 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.521 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.521 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.521 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.521 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:231461 8.7 8:e91d627b:::benchmark_data_smithi110_258128_object112643:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160)", 2023-09-23T23:14:14.522 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:12:02.488735+0000", 2023-09-23T23:14:14.522 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 131.877136896, 2023-09-23T23:14:14.522 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.048625366, 2023-09-23T23:14:14.522 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.522 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.522 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.522 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.522 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.522 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 231461 2023-09-23T23:14:14.522 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.522 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.523 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.523 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.488735+0000", 2023-09-23T23:14:14.523 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.523 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.523 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.488735+0000", 2023-09-23T23:14:14.523 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.488735+0000", 2023-09-23T23:14:14.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:14:14.524 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.488737+0000", 2023-09-23T23:14:14.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3319999999999999e-06 2023-09-23T23:14:14.524 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.525 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.488743+0000", 2023-09-23T23:14:14.525 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4399999999999996e-06 2023-09-23T23:14:14.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.525 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.525 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.488747+0000", 2023-09-23T23:14:14.525 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6009999999999997e-06 2023-09-23T23:14:14.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.525 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.526 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.490859+0000", 2023-09-23T23:14:14.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021120689999999998 2023-09-23T23:14:14.526 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.526 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.526 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.526 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.490870+0000", 2023-09-23T23:14:14.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0139999999999999e-05 2023-09-23T23:14:14.526 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.526 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.526 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.527 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.531084+0000", 2023-09-23T23:14:14.527 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.040214422 2023-09-23T23:14:14.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.527 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.527 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.527 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.531170+0000", 2023-09-23T23:14:14.527 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5682999999999999e-05 2023-09-23T23:14:14.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.527 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.527 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:14:14.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.531241+0000", 2023-09-23T23:14:14.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1167000000000005e-05 2023-09-23T23:14:14.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.529 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.537102+0000", 2023-09-23T23:14:14.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058613299999999997 2023-09-23T23:14:14.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.529 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.530 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.537247+0000", 2023-09-23T23:14:14.530 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014445699999999999 2023-09-23T23:14:14.530 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.530 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.530 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.530 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.537317+0000", 2023-09-23T23:14:14.530 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.004e-05 2023-09-23T23:14:14.530 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.530 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.530 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.530 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.537341+0000", 2023-09-23T23:14:14.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3968000000000002e-05 2023-09-23T23:14:14.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.537361+0000", 2023-09-23T23:14:14.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0077000000000001e-05 2023-09-23T23:14:14.531 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.531 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.531 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.531 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:231473 8.7 8:ef25588c:::benchmark_data_smithi110_258128_object112655:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2023-09-23T23:14:14.531 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:12:02.501667+0000", 2023-09-23T23:14:14.531 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 131.86420511700001, 2023-09-23T23:14:14.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0472035390000001, 2023-09-23T23:14:14.532 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:14.532 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:14:14.532 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:14.532 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:14:14.532 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:14:14.532 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 231473 2023-09-23T23:14:14.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.532 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:14.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:14.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.501667+0000", 2023-09-23T23:14:14.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.533 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:14.533 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.501667+0000", 2023-09-23T23:14:14.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:14.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.533 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:14.533 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.501665+0000", 2023-09-23T23:14:14.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:14:14.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:14.534 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.501672+0000", 2023-09-23T23:14:14.534 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.533e-06 2023-09-23T23:14:14.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:14.534 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.501683+0000", 2023-09-23T23:14:14.534 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0988999999999999e-05 2023-09-23T23:14:14.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:14.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.501690+0000", 2023-09-23T23:14:14.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0500000000000003e-06 2023-09-23T23:14:14.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.501758+0000", 2023-09-23T23:14:14.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7787999999999995e-05 2023-09-23T23:14:14.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:14:14.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:02.501768+0000", 2023-09-23T23:14:14.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0298e-05 2023-09-23T23:14:14.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:14.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.537560+0000", 2023-09-23T23:14:14.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0357924919999999 2023-09-23T23:14:14.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:14.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.537684+0000", 2023-09-23T23:14:14.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012374599999999999 2023-09-23T23:14:14.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:14:14.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.537805+0000", 2023-09-23T23:14:14.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000120393 2023-09-23T23:14:14.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.545666+0000", 2023-09-23T23:14:14.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078613209999999992 2023-09-23T23:14:14.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:14:14.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.545708+0000", 2023-09-23T23:14:14.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1956000000000003e-05 2023-09-23T23:14:14.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:14:14.538 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.548810+0000", 2023-09-23T23:14:14.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031026000000000001 2023-09-23T23:14:14.538 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.538 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:14:14.538 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.548845+0000", 2023-09-23T23:14:14.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4153999999999999e-05 2023-09-23T23:14:14.538 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:14.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:14.538 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:14:14.538 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:12:03.548871+0000", 2023-09-23T23:14:14.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6089e-05 2023-09-23T23:14:14.539 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.539 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.539 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.539 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:14.539 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:14.539 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:14:14.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:14.540 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:14:14.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:14:14.541 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 39 16 65689 65673 105.197 75.0625 0.00671306 0.00950165 2023-09-23T23:14:14.641 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:14:14.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:14 smithi110 ceph-mon[212939]: pgmap v563: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s 2023-09-23T23:14:14.695 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:14 smithi110 ceph-mon[221102]: pgmap v563: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s 2023-09-23T23:14:14.700 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:14.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:14 smithi138 ceph-mon[193606]: pgmap v563: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s 2023-09-23T23:14:15.532 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:14:15.530222+0000 min lat: 0.00117478 max lat: 0.336316 avg lat: 0.00960689 2023-09-23T23:14:15.533 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:14:15.533 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 40 16 66633 66617 104.042 59 0.0120745 0.00960689 2023-09-23T23:14:16.040 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:14:16.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:16.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:14:16.158 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:14:16.221 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:16.328 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:14:16.387 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:14:16.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:16.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:14:16.497 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:14:16.530 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 41 16 67730 67714 103.177 68.5625 0.00701553 0.00968741 2023-09-23T23:14:16.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:16 smithi110 ceph-mon[212939]: pgmap v564: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T23:14:16.577 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:16 smithi110 ceph-mon[221102]: pgmap v564: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T23:14:16.580 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:16.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:16 smithi138 ceph-mon[193606]: pgmap v564: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T23:14:16.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:14:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:14:16] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-09-23T23:14:17.040 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:14:17.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:17.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:14:17.146 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:17.196 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:17.278 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:17.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:17.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:17.383 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:17.431 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:17.519 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:17.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:17.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:17.533 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 42 16 68747 68731 102.226 63.5625 0.00944191 0.00977774 2023-09-23T23:14:17.630 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:14:17.705 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:17.848 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:14:17.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:17.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:14:17.955 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:17.962 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:14:17.966 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:14:18.009 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:14:18.041 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:18.041 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:14:18.081 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:14:18.131 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:14:18.143 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:14:18.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:18.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:14:18.176 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:14:18.211 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:14:18.249 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:18.282 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:18.292 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:14:18.292 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:14:18.292 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:14:18.292 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:14:18.292 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:14:18.292 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:14:18.292 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:14:18.293 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:14:18.293 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:14:18.353 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:18.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:18.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:18.461 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:14:18.510 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:18.534 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 43 16 69772 69756 101.339 64.0625 0.00751713 0.00986287 2023-09-23T23:14:18.580 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:14:18.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:18.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:18 smithi138 ceph-mon[193606]: pgmap v565: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T23:14:18.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2672453927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:18.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:14:18.689 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:14:18.742 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T23:14:18.742 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:14:18.743 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T23:14:18.743 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:14:18.752 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:18 smithi110 ceph-mon[212939]: pgmap v565: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T23:14:18.753 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2672453927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:18.753 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:18 smithi110 ceph-mon[221102]: pgmap v565: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T23:14:18.753 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2672453927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:18.763 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:19.498 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:14:19.533 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 44 16 70731 70715 100.398 59.9375 0.020436 0.00995567 2023-09-23T23:14:19.640 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:14:19.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:19.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:14:19.764 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:14:19.788 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:19 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3072596896' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:14:19.788 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:19 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:14:19.789 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:19 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3072596896' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:14:19.789 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:19 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:14:19.832 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:19.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:19 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3072596896' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:14:19.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:19 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:14:20.534 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 45 16 71714 71698 99.5328 61.4375 0.00840511 0.0100429 2023-09-23T23:14:20.772 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:20 smithi110 ceph-mon[212939]: pgmap v566: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T23:14:20.772 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:20 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:14:20.772 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:20 smithi110 ceph-mon[212939]: failed to encode map e173 with expected crc 2023-09-23T23:14:20.772 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:20 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:14:20.772 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:20 smithi110 ceph-mon[212939]: osdmap e173: 8 total, 8 up, 8 in 2023-09-23T23:14:20.772 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:20 smithi110 ceph-mon[212939]: failed to encode map e173 with expected crc 2023-09-23T23:14:20.774 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:20 smithi110 ceph-mon[221102]: pgmap v566: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T23:14:20.774 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:20 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:14:20.774 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:20 smithi110 ceph-mon[221102]: failed to encode map e173 with expected crc 2023-09-23T23:14:20.774 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:20 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:14:20.774 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:20 smithi110 ceph-mon[221102]: osdmap e173: 8 total, 8 up, 8 in 2023-09-23T23:14:20.774 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:20 smithi110 ceph-mon[221102]: failed to encode map e173 with expected crc 2023-09-23T23:14:20.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:20 smithi138 ceph-mon[193606]: pgmap v566: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T23:14:20.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:20 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:14:20.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:20 smithi138 ceph-mon[193606]: failed to encode map e173 with expected crc 2023-09-23T23:14:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:20 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:14:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:20 smithi138 ceph-mon[193606]: osdmap e173: 8 total, 8 up, 8 in 2023-09-23T23:14:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:20 smithi138 ceph-mon[193606]: failed to encode map e173 with expected crc 2023-09-23T23:14:20.993 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:14:20.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:20.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:14:21.098 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:21.144 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:21.534 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 46 16 72694 72678 98.7007 61.25 0.00634433 0.0101265 2023-09-23T23:14:21.541 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:21.567 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:21.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:21.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:21.674 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:14:21.682 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:21.729 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:21.816 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:21.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:21.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:21.924 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:21.962 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:22.037 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:22.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:22.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:22.144 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:22.213 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:22.450 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:14:22.450 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:14:22.506 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:22.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:22.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:22.534 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 47 16 73758 73742 98.0157 66.5 0.0111605 0.0101977 2023-09-23T23:14:22.615 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:22.667 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:22.751 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:22.811 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:22 smithi110 ceph-mon[212939]: pgmap v568: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T23:14:22.811 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:22 smithi110 ceph-mon[221102]: pgmap v568: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T23:14:22.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:22.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:22.812 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:22 smithi138 ceph-mon[193606]: pgmap v568: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T23:14:22.913 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:14:22.939 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:23.221 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:14:23.534 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 48 16 74890 74874 97.4477 70.75 0.0173491 0.0102569 2023-09-23T23:14:23.765 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:14:23.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:23.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:14:23.876 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:23.903 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:24.194 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:24 smithi110 ceph-mon[212939]: pgmap v569: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T23:14:24.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:24 smithi110 ceph-mon[221102]: pgmap v569: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T23:14:24.249 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:24.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:24.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:24.334 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:24.357 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:14:24.377 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:14:24.464 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:24.534 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 49 15 75855 75840 96.6913 60.375 0.0285717 0.0103365 2023-09-23T23:14:24.592 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:14:24.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:24.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:14:24.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:24 smithi138 ceph-mon[193606]: pgmap v569: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T23:14:24.704 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:14:24.788 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:24.803 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:24.942 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:14:24.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:24.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:14:25.057 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:14:25.131 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:25.230 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:14:25.238 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:14:25.534 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 50 16 76669 76653 95.774 50.8125 0.0137151 0.0104365 2023-09-23T23:14:26.178 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:14:26.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:26.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:14:26.285 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:14:26.348 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:26.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:26 smithi110 ceph-mon[212939]: from='client.45203 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:14:26.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:26 smithi110 ceph-mon[212939]: pgmap v570: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T23:14:26.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:26 smithi110 ceph-mon[221102]: from='client.45203 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:14:26.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:26 smithi110 ceph-mon[221102]: pgmap v570: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T23:14:26.534 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 51 16 77635 77619 95.08 60.375 0.00896898 0.0105132 2023-09-23T23:14:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:26 smithi138 ceph-mon[193606]: from='client.45203 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:14:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:26 smithi138 ceph-mon[193606]: pgmap v570: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T23:14:26.786 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:14:26.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:26.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:14:26.816 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":569,"stamp":"2023-09-23T23:14:23.307756+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4668397472,"num_objects":71249,"num_object_clones":0,"num_object_copies":213747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71249,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":71525,"num_write_kb":4560845,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":1771174,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":71441,"ondisk_log_size":71441,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16170248,"kb_used_data":13590920,"kb_used_omap":20,"kb_used_meta":2579243,"kb_avail":733790968,"statfs":{"total":767960285184,"available":751401951232,"internally_reserved":0,"allocated":13917102080,"data_stored":13912186039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2641145486},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,2,2,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":711720960,"num_objects":10860,"num_object_clones":0,"num_object_copies":32580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10860,"num_write_kb":695040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004253"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":151,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.585547+0000","last_change":"2023-09-23T23:08:45.116934+0000","last_active":"2023-09-23T23:14:07.585547+0000","last_peered":"2023-09-23T23:14:07.585547+0000","last_clean":"2023-09-23T23:14:07.585547+0000","last_became_active":"2023-09-23T23:08:45.116576+0000","last_became_peered":"2023-09-23T23:08:45.116576+0000","last_unstale":"2023-09-23T23:14:07.585547+0000","last_undegraded":"2023-09-23T23:14:07.585547+0000","last_fullsized":"2023-09-23T23:14:07.585547+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:06.398968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":111,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.452955+0000","last_change":"2023-09-23T23:10:59.514211+0000","last_active":"2023-09-23T23:14:12.452955+0000","last_peered":"2023-09-23T23:14:12.452955+0000","last_clean":"2023-09-23T23:14:12.452955+0000","last_became_active":"2023-09-23T23:10:59.513917+0000","last_became_peered":"2023-09-23T23:10:59.513917+0000","last_unstale":"2023-09-23T23:14:12.452955+0000","last_undegraded":"2023-09-23T23:14:12.452955+0000","last_fullsized":"2023-09-23T23:14:12.452955+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:55.533586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":140,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.573258+0000","last_change":"2023-09-23T23:10:59.505490+0000","last_active":"2023-09-23T23:14:07.573258+0000","last_peered":"2023-09-23T23:14:07.573258+0000","last_clean":"2023-09-23T23:14:07.573258+0000","last_became_active":"2023-09-23T23:10:59.504874+0000","last_became_peered":"2023-09-23T23:10:59.504874+0000","last_unstale":"2023-09-23T23:14:07.573258+0000","last_undegraded":"2023-09-23T23:14:07.573258+0000","last_fullsized":"2023-09-23T23:14:07.573258+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:15.279668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":123,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.463598+0000","last_change":"2023-09-23T23:10:59.506092+0000","last_active":"2023-09-23T23:14:12.463598+0000","last_peered":"2023-09-23T23:14:12.463598+0000","last_clean":"2023-09-23T23:14:12.463598+0000","last_became_active":"2023-09-23T23:10:59.505550+0000","last_became_peered":"2023-09-23T23:10:59.505550+0000","last_unstale":"2023-09-23T23:14:12.463598+0000","last_undegraded":"2023-09-23T23:14:12.463598+0000","last_fullsized":"2023-09-23T23:14:12.463598+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:10.576890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":129,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.464816+0000","last_change":"2023-09-23T23:10:59.515675+0000","last_active":"2023-09-23T23:14:12.464816+0000","last_peered":"2023-09-23T23:14:12.464816+0000","last_clean":"2023-09-23T23:14:12.464816+0000","last_became_active":"2023-09-23T23:10:59.514600+0000","last_became_peered":"2023-09-23T23:10:59.514600+0000","last_unstale":"2023-09-23T23:14:12.464816+0000","last_undegraded":"2023-09-23T23:14:12.464816+0000","last_fullsized":"2023-09-23T23:14:12.464816+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.273467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":132,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.579890+0000","last_change":"2023-09-23T23:07:30.078179+0000","last_active":"2023-09-23T23:14:07.579890+0000","last_peered":"2023-09-23T23:14:07.579890+0000","last_clean":"2023-09-23T23:14:07.579890+0000","last_became_active":"2023-09-23T23:07:30.077810+0000","last_became_peered":"2023-09-23T23:07:30.077810+0000","last_unstale":"2023-09-23T23:14:07.579890+0000","last_undegraded":"2023-09-23T23:14:07.579890+0000","last_fullsized":"2023-09-23T23:14:07.579890+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:19.218333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":122,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.464128+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:14:12.464128+0000","last_peered":"2023-09-23T23:14:12.464128+0000","last_clean":"2023-09-23T23:14:12.464128+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:14:12.464128+0000","last_undegraded":"2023-09-23T23:14:12.464128+0000","last_fullsized":"2023-09-23T23:14:12.464128+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":111,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.464887+0000","last_change":"2023-09-23T23:10:59.515840+0000","last_active":"2023-09-23T23:14:12.464887+0000","last_peered":"2023-09-23T23:14:12.464887+0000","last_clean":"2023-09-23T23:14:12.464887+0000","last_became_active":"2023-09-23T23:10:59.515000+0000","last_became_peered":"2023-09-23T23:10:59.515000+0000","last_unstale":"2023-09-23T23:14:12.464887+0000","last_undegraded":"2023-09-23T23:14:12.464887+0000","last_fullsized":"2023-09-23T23:14:12.464887+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:47.355624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":133,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.576514+0000","last_change":"2023-09-23T23:07:30.076036+0000","last_active":"2023-09-23T23:14:07.576514+0000","last_peered":"2023-09-23T23:14:07.576514+0000","last_clean":"2023-09-23T23:14:07.576514+0000","last_became_active":"2023-09-23T23:07:30.075733+0000","last_became_peered":"2023-09-23T23:07:30.075733+0000","last_unstale":"2023-09-23T23:14:07.576514+0000","last_undegraded":"2023-09-23T23:14:07.576514+0000","last_fullsized":"2023-09-23T23:14:07.576514+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:56.342424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":90,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.453295+0000","last_change":"2023-09-23T23:10:59.517290+0000","last_active":"2023-09-23T23:14:12.453295+0000","last_peered":"2023-09-23T23:14:12.453295+0000","last_clean":"2023-09-23T23:14:12.453295+0000","last_became_active":"2023-09-23T23:10:59.513367+0000","last_became_peered":"2023-09-23T23:10:59.513367+0000","last_unstale":"2023-09-23T23:14:12.453295+0000","last_undegraded":"2023-09-23T23:14:12.453295+0000","last_fullsized":"2023-09-23T23:14:12.453295+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:04.328676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":136,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.582202+0000","last_change":"2023-09-23T23:10:59.500837+0000","last_active":"2023-09-23T23:14:07.582202+0000","last_peered":"2023-09-23T23:14:07.582202+0000","last_clean":"2023-09-23T23:14:07.582202+0000","last_became_active":"2023-09-23T23:10:59.500455+0000","last_became_peered":"2023-09-23T23:10:59.500455+0000","last_unstale":"2023-09-23T23:14:07.582202+0000","last_undegraded":"2023-09-23T23:14:07.582202+0000","last_fullsized":"2023-09-23T23:14:07.582202+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:34.801149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":139,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.574509+0000","last_change":"2023-09-23T23:08:45.122701+0000","last_active":"2023-09-23T23:14:07.574509+0000","last_peered":"2023-09-23T23:14:07.574509+0000","last_clean":"2023-09-23T23:14:07.574509+0000","last_became_active":"2023-09-23T23:08:45.122332+0000","last_became_peered":"2023-09-23T23:08:45.122332+0000","last_unstale":"2023-09-23T23:14:07.574509+0000","last_undegraded":"2023-09-23T23:14:07.574509+0000","last_fullsized":"2023-09-23T23:14:07.574509+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:05.771906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":138,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.586172+0000","last_change":"2023-09-23T22:53:16.891930+0000","last_active":"2023-09-23T23:14:07.586172+0000","last_peered":"2023-09-23T23:14:07.586172+0000","last_clean":"2023-09-23T23:14:07.586172+0000","last_became_active":"2023-09-23T22:53:16.891658+0000","last_became_peered":"2023-09-23T22:53:16.891658+0000","last_unstale":"2023-09-23T23:14:07.586172+0000","last_undegraded":"2023-09-23T23:14:07.586172+0000","last_fullsized":"2023-09-23T23:14:07.586172+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:19.762775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":132,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.586121+0000","last_change":"2023-09-23T23:07:30.082476+0000","last_active":"2023-09-23T23:14:07.586121+0000","last_peered":"2023-09-23T23:14:07.586121+0000","last_clean":"2023-09-23T23:14:07.586121+0000","last_became_active":"2023-09-23T23:07:30.082201+0000","last_became_peered":"2023-09-23T23:07:30.082201+0000","last_unstale":"2023-09-23T23:14:07.586121+0000","last_undegraded":"2023-09-23T23:14:07.586121+0000","last_fullsized":"2023-09-23T23:14:07.586121+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:08.809163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":135,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.580581+0000","last_change":"2023-09-23T23:08:45.115767+0000","last_active":"2023-09-23T23:14:07.580581+0000","last_peered":"2023-09-23T23:14:07.580581+0000","last_clean":"2023-09-23T23:14:07.580581+0000","last_became_active":"2023-09-23T23:08:45.115422+0000","last_became_peered":"2023-09-23T23:08:45.115422+0000","last_unstale":"2023-09-23T23:14:07.580581+0000","last_undegraded":"2023-09-23T23:14:07.580581+0000","last_fullsized":"2023-09-23T23:14:07.580581+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:53.216816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":111,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.456220+0000","last_change":"2023-09-23T23:10:59.518106+0000","last_active":"2023-09-23T23:14:12.456220+0000","last_peered":"2023-09-23T23:14:12.456220+0000","last_clean":"2023-09-23T23:14:12.456220+0000","last_became_active":"2023-09-23T23:10:59.517633+0000","last_became_peered":"2023-09-23T23:10:59.517633+0000","last_unstale":"2023-09-23T23:14:12.456220+0000","last_undegraded":"2023-09-23T23:14:12.456220+0000","last_fullsized":"2023-09-23T23:14:12.456220+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:27.245420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":122,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.464219+0000","last_change":"2023-09-23T23:10:59.516989+0000","last_active":"2023-09-23T23:14:12.464219+0000","last_peered":"2023-09-23T23:14:12.464219+0000","last_clean":"2023-09-23T23:14:12.464219+0000","last_became_active":"2023-09-23T23:10:59.516507+0000","last_became_peered":"2023-09-23T23:10:59.516507+0000","last_unstale":"2023-09-23T23:14:12.464219+0000","last_undegraded":"2023-09-23T23:14:12.464219+0000","last_fullsized":"2023-09-23T23:14:12.464219+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:40.834591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":119,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.582301+0000","last_change":"2023-09-23T23:13:29.360403+0000","last_active":"2023-09-23T23:14:07.582301+0000","last_peered":"2023-09-23T23:14:07.582301+0000","last_clean":"2023-09-23T23:14:07.582301+0000","last_became_active":"2023-09-23T23:13:29.359519+0000","last_became_peered":"2023-09-23T23:13:29.359519+0000","last_unstale":"2023-09-23T23:14:07.582301+0000","last_undegraded":"2023-09-23T23:14:07.582301+0000","last_fullsized":"2023-09-23T23:14:07.582301+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:27.911252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":116,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.580546+0000","last_change":"2023-09-23T22:53:16.889362+0000","last_active":"2023-09-23T23:14:07.580546+0000","last_peered":"2023-09-23T23:14:07.580546+0000","last_clean":"2023-09-23T23:14:07.580546+0000","last_became_active":"2023-09-23T22:53:16.889081+0000","last_became_peered":"2023-09-23T22:53:16.889081+0000","last_unstale":"2023-09-23T23:14:07.580546+0000","last_undegraded":"2023-09-23T23:14:07.580546+0000","last_fullsized":"2023-09-23T23:14:07.580546+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:11.321151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":122,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.463768+0000","last_change":"2023-09-23T23:10:59.506198+0000","last_active":"2023-09-23T23:14:12.463768+0000","last_peered":"2023-09-23T23:14:12.463768+0000","last_clean":"2023-09-23T23:14:12.463768+0000","last_became_active":"2023-09-23T23:10:59.505912+0000","last_became_peered":"2023-09-23T23:10:59.505912+0000","last_unstale":"2023-09-23T23:14:12.463768+0000","last_undegraded":"2023-09-23T23:14:12.463768+0000","last_fullsized":"2023-09-23T23:14:12.463768+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:30.343257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":161,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.579226+0000","last_change":"2023-09-23T23:13:29.368175+0000","last_active":"2023-09-23T23:14:07.579226+0000","last_peered":"2023-09-23T23:14:07.579226+0000","last_clean":"2023-09-23T23:14:07.579226+0000","last_became_active":"2023-09-23T23:13:29.363603+0000","last_became_peered":"2023-09-23T23:13:29.363603+0000","last_unstale":"2023-09-23T23:14:07.579226+0000","last_undegraded":"2023-09-23T23:14:07.579226+0000","last_fullsized":"2023-09-23T23:14:07.579226+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:25.277871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":111,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.455416+0000","last_change":"2023-09-23T23:10:59.525228+0000","last_active":"2023-09-23T23:14:12.455416+0000","last_peered":"2023-09-23T23:14:12.455416+0000","last_clean":"2023-09-23T23:14:12.455416+0000","last_became_active":"2023-09-23T23:10:59.524665+0000","last_became_peered":"2023-09-23T23:10:59.524665+0000","last_unstale":"2023-09-23T23:14:12.455416+0000","last_undegraded":"2023-09-23T23:14:12.455416+0000","last_fullsized":"2023-09-23T23:14:12.455416+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:42.632316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"173'4493","reported_seq":4511,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T23:14:20.428031+0000","last_change":"2023-09-23T23:13:28.360242+0000","last_active":"2023-09-23T23:14:20.428031+0000","last_peered":"2023-09-23T23:14:20.428031+0000","last_clean":"2023-09-23T23:14:20.428031+0000","last_became_active":"2023-09-23T23:13:28.359368+0000","last_became_peered":"2023-09-23T23:13:28.359368+0000","last_unstale":"2023-09-23T23:14:20.428031+0000","last_undegraded":"2023-09-23T23:14:20.428031+0000","last_fullsized":"2023-09-23T23:14:20.428031+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":4493,"log_dups_size":0,"ondisk_log_size":4493,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:57.722768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294453248,"num_objects":4493,"num_object_clones":0,"num_object_copies":13479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4493,"num_write_kb":287552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":134,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.580911+0000","last_change":"2023-09-23T23:04:11.907427+0000","last_active":"2023-09-23T23:14:07.580911+0000","last_peered":"2023-09-23T23:14:07.580911+0000","last_clean":"2023-09-23T23:14:07.580911+0000","last_became_active":"2023-09-23T23:04:11.907016+0000","last_became_peered":"2023-09-23T23:04:11.907016+0000","last_unstale":"2023-09-23T23:14:07.580911+0000","last_undegraded":"2023-09-23T23:14:07.580911+0000","last_fullsized":"2023-09-23T23:14:07.580911+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:20.715977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":20,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.579055+0000","last_change":"2023-09-23T23:13:29.421286+0000","last_active":"2023-09-23T23:14:07.579055+0000","last_peered":"2023-09-23T23:14:07.579055+0000","last_clean":"2023-09-23T23:14:07.579055+0000","last_became_active":"2023-09-23T23:13:29.420967+0000","last_became_peered":"2023-09-23T23:13:29.420967+0000","last_unstale":"2023-09-23T23:14:07.579055+0000","last_undegraded":"2023-09-23T23:14:07.579055+0000","last_fullsized":"2023-09-23T23:14:07.579055+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:59.756339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"172'4488","reported_seq":4503,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:18.358280+0000","last_change":"2023-09-23T23:13:28.360159+0000","last_active":"2023-09-23T23:14:18.358280+0000","last_peered":"2023-09-23T23:14:18.358280+0000","last_clean":"2023-09-23T23:14:18.358280+0000","last_became_active":"2023-09-23T23:13:28.359915+0000","last_became_peered":"2023-09-23T23:13:28.359915+0000","last_unstale":"2023-09-23T23:14:18.358280+0000","last_undegraded":"2023-09-23T23:14:18.358280+0000","last_fullsized":"2023-09-23T23:14:18.358280+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":4488,"log_dups_size":0,"ondisk_log_size":4488,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:48.297395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294125568,"num_objects":4488,"num_object_clones":0,"num_object_copies":13464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4488,"num_write_kb":287232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":135,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.580995+0000","last_change":"2023-09-23T23:07:30.072322+0000","last_active":"2023-09-23T23:14:07.580995+0000","last_peered":"2023-09-23T23:14:07.580995+0000","last_clean":"2023-09-23T23:14:07.580995+0000","last_became_active":"2023-09-23T23:07:30.071941+0000","last_became_peered":"2023-09-23T23:07:30.071941+0000","last_unstale":"2023-09-23T23:14:07.580995+0000","last_undegraded":"2023-09-23T23:14:07.580995+0000","last_fullsized":"2023-09-23T23:14:07.580995+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:38.994624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":149,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.585443+0000","last_change":"2023-09-23T23:10:59.512748+0000","last_active":"2023-09-23T23:14:07.585443+0000","last_peered":"2023-09-23T23:14:07.585443+0000","last_clean":"2023-09-23T23:14:07.585443+0000","last_became_active":"2023-09-23T23:10:59.512420+0000","last_became_peered":"2023-09-23T23:10:59.512420+0000","last_unstale":"2023-09-23T23:14:07.585443+0000","last_undegraded":"2023-09-23T23:14:07.585443+0000","last_fullsized":"2023-09-23T23:14:07.585443+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:09.269366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"172'4308","reported_seq":4324,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:18.812772+0000","last_change":"2023-09-23T23:13:28.354971+0000","last_active":"2023-09-23T23:14:18.812772+0000","last_peered":"2023-09-23T23:14:18.812772+0000","last_clean":"2023-09-23T23:14:18.812772+0000","last_became_active":"2023-09-23T23:13:28.354731+0000","last_became_peered":"2023-09-23T23:13:28.354731+0000","last_unstale":"2023-09-23T23:14:18.812772+0000","last_undegraded":"2023-09-23T23:14:18.812772+0000","last_fullsized":"2023-09-23T23:14:18.812772+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":4308,"log_dups_size":0,"ondisk_log_size":4308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:20.209676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282329088,"num_objects":4308,"num_object_clones":0,"num_object_copies":12924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4308,"num_write_kb":275712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":131,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.582071+0000","last_change":"2023-09-23T23:13:29.362820+0000","last_active":"2023-09-23T23:14:07.582071+0000","last_peered":"2023-09-23T23:14:07.582071+0000","last_clean":"2023-09-23T23:14:07.582071+0000","last_became_active":"2023-09-23T23:13:29.360151+0000","last_became_peered":"2023-09-23T23:13:29.360151+0000","last_unstale":"2023-09-23T23:14:07.582071+0000","last_undegraded":"2023-09-23T23:14:07.582071+0000","last_fullsized":"2023-09-23T23:14:07.582071+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:36.242850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":142,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.579458+0000","last_change":"2023-09-23T23:08:45.116813+0000","last_active":"2023-09-23T23:14:07.579458+0000","last_peered":"2023-09-23T23:14:07.579458+0000","last_clean":"2023-09-23T23:14:07.579458+0000","last_became_active":"2023-09-23T23:08:45.116435+0000","last_became_peered":"2023-09-23T23:08:45.116435+0000","last_unstale":"2023-09-23T23:14:07.579458+0000","last_undegraded":"2023-09-23T23:14:07.579458+0000","last_fullsized":"2023-09-23T23:14:07.579458+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:40.616551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"172'4431","reported_seq":4447,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:18.814718+0000","last_change":"2023-09-23T23:13:28.505667+0000","last_active":"2023-09-23T23:14:18.814718+0000","last_peered":"2023-09-23T23:14:18.814718+0000","last_clean":"2023-09-23T23:14:18.814718+0000","last_became_active":"2023-09-23T23:13:28.505309+0000","last_became_peered":"2023-09-23T23:13:28.505309+0000","last_unstale":"2023-09-23T23:14:18.814718+0000","last_undegraded":"2023-09-23T23:14:18.814718+0000","last_fullsized":"2023-09-23T23:14:18.814718+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":4431,"log_dups_size":0,"ondisk_log_size":4431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:18.432866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290390016,"num_objects":4431,"num_object_clones":0,"num_object_copies":13293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4431,"num_write_kb":283584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":133,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.576328+0000","last_change":"2023-09-23T23:10:59.502426+0000","last_active":"2023-09-23T23:14:07.576328+0000","last_peered":"2023-09-23T23:14:07.576328+0000","last_clean":"2023-09-23T23:14:07.576328+0000","last_became_active":"2023-09-23T23:10:59.502074+0000","last_became_peered":"2023-09-23T23:10:59.502074+0000","last_unstale":"2023-09-23T23:14:07.576328+0000","last_undegraded":"2023-09-23T23:14:07.576328+0000","last_fullsized":"2023-09-23T23:14:07.576328+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:46.029347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":125,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.464118+0000","last_change":"2023-09-23T23:04:11.912493+0000","last_active":"2023-09-23T23:14:12.464118+0000","last_peered":"2023-09-23T23:14:12.464118+0000","last_clean":"2023-09-23T23:14:12.464118+0000","last_became_active":"2023-09-23T23:04:11.906712+0000","last_became_peered":"2023-09-23T23:04:11.906712+0000","last_unstale":"2023-09-23T23:14:12.464118+0000","last_undegraded":"2023-09-23T23:14:12.464118+0000","last_fullsized":"2023-09-23T23:14:12.464118+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:58.932610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"173'4597","reported_seq":4615,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T23:14:22.571836+0000","last_change":"2023-09-23T23:13:28.358022+0000","last_active":"2023-09-23T23:14:22.571836+0000","last_peered":"2023-09-23T23:14:22.571836+0000","last_clean":"2023-09-23T23:14:22.571836+0000","last_became_active":"2023-09-23T23:13:28.357728+0000","last_became_peered":"2023-09-23T23:13:28.357728+0000","last_unstale":"2023-09-23T23:14:22.571836+0000","last_undegraded":"2023-09-23T23:14:22.571836+0000","last_fullsized":"2023-09-23T23:14:22.571836+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":4597,"log_dups_size":0,"ondisk_log_size":4597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:08.765537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301268992,"num_objects":4597,"num_object_clones":0,"num_object_copies":13791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4597,"num_write_kb":294208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":135,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.453315+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:14:12.453315+0000","last_peered":"2023-09-23T23:14:12.453315+0000","last_clean":"2023-09-23T23:14:12.453315+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:14:12.453315+0000","last_undegraded":"2023-09-23T23:14:12.453315+0000","last_fullsized":"2023-09-23T23:14:12.453315+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":132,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.586066+0000","last_change":"2023-09-23T23:10:59.512363+0000","last_active":"2023-09-23T23:14:07.586066+0000","last_peered":"2023-09-23T23:14:07.586066+0000","last_clean":"2023-09-23T23:14:07.586066+0000","last_became_active":"2023-09-23T23:10:59.512022+0000","last_became_peered":"2023-09-23T23:10:59.512022+0000","last_unstale":"2023-09-23T23:14:07.586066+0000","last_undegraded":"2023-09-23T23:14:07.586066+0000","last_fullsized":"2023-09-23T23:14:07.586066+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:45.523432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"172'4264","reported_seq":4281,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:18.310091+0000","last_change":"2023-09-23T23:13:28.505728+0000","last_active":"2023-09-23T23:14:18.310091+0000","last_peered":"2023-09-23T23:14:18.310091+0000","last_clean":"2023-09-23T23:14:18.310091+0000","last_became_active":"2023-09-23T23:13:28.505371+0000","last_became_peered":"2023-09-23T23:13:28.505371+0000","last_unstale":"2023-09-23T23:14:18.310091+0000","last_undegraded":"2023-09-23T23:14:18.310091+0000","last_fullsized":"2023-09-23T23:14:18.310091+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":4264,"log_dups_size":0,"ondisk_log_size":4264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:32.082686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279445504,"num_objects":4264,"num_object_clones":0,"num_object_copies":12792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4264,"num_write_kb":272896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":133,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.579477+0000","last_change":"2023-09-23T23:07:30.077349+0000","last_active":"2023-09-23T23:14:07.579477+0000","last_peered":"2023-09-23T23:14:07.579477+0000","last_clean":"2023-09-23T23:14:07.579477+0000","last_became_active":"2023-09-23T23:07:30.077031+0000","last_became_peered":"2023-09-23T23:07:30.077031+0000","last_unstale":"2023-09-23T23:14:07.579477+0000","last_undegraded":"2023-09-23T23:14:07.579477+0000","last_fullsized":"2023-09-23T23:14:07.579477+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:27.708541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":149,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.585562+0000","last_change":"2023-09-23T23:08:45.117579+0000","last_active":"2023-09-23T23:14:07.585562+0000","last_peered":"2023-09-23T23:14:07.585562+0000","last_clean":"2023-09-23T23:14:07.585562+0000","last_became_active":"2023-09-23T23:08:45.117216+0000","last_became_peered":"2023-09-23T23:08:45.117216+0000","last_unstale":"2023-09-23T23:14:07.585562+0000","last_undegraded":"2023-09-23T23:14:07.585562+0000","last_fullsized":"2023-09-23T23:14:07.585562+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:27.731522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"172'4353","reported_seq":4369,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:18.829709+0000","last_change":"2023-09-23T23:13:28.356430+0000","last_active":"2023-09-23T23:14:18.829709+0000","last_peered":"2023-09-23T23:14:18.829709+0000","last_clean":"2023-09-23T23:14:18.829709+0000","last_became_active":"2023-09-23T23:13:28.356172+0000","last_became_peered":"2023-09-23T23:13:28.356172+0000","last_unstale":"2023-09-23T23:14:18.829709+0000","last_undegraded":"2023-09-23T23:14:18.829709+0000","last_fullsized":"2023-09-23T23:14:18.829709+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":4353,"log_dups_size":0,"ondisk_log_size":4353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:06.046311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285278208,"num_objects":4353,"num_object_clones":0,"num_object_copies":13059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4353,"num_write_kb":278592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":133,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.585351+0000","last_change":"2023-09-23T23:08:45.120505+0000","last_active":"2023-09-23T23:14:07.585351+0000","last_peered":"2023-09-23T23:14:07.585351+0000","last_clean":"2023-09-23T23:14:07.585351+0000","last_became_active":"2023-09-23T23:08:45.120136+0000","last_became_peered":"2023-09-23T23:08:45.120136+0000","last_unstale":"2023-09-23T23:14:07.585351+0000","last_undegraded":"2023-09-23T23:14:07.585351+0000","last_fullsized":"2023-09-23T23:14:07.585351+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:56.726515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":132,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.585383+0000","last_change":"2023-09-23T23:07:30.081779+0000","last_active":"2023-09-23T23:14:07.585383+0000","last_peered":"2023-09-23T23:14:07.585383+0000","last_clean":"2023-09-23T23:14:07.585383+0000","last_became_active":"2023-09-23T23:07:30.081425+0000","last_became_peered":"2023-09-23T23:07:30.081425+0000","last_unstale":"2023-09-23T23:14:07.585383+0000","last_undegraded":"2023-09-23T23:14:07.585383+0000","last_fullsized":"2023-09-23T23:14:07.585383+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:01.231206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"173'4708","reported_seq":4726,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T23:14:22.610121+0000","last_change":"2023-09-23T23:13:28.506227+0000","last_active":"2023-09-23T23:14:22.610121+0000","last_peered":"2023-09-23T23:14:22.610121+0000","last_clean":"2023-09-23T23:14:22.610121+0000","last_became_active":"2023-09-23T23:13:28.505885+0000","last_became_peered":"2023-09-23T23:13:28.505885+0000","last_unstale":"2023-09-23T23:14:22.610121+0000","last_undegraded":"2023-09-23T23:14:22.610121+0000","last_fullsized":"2023-09-23T23:14:22.610121+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":4708,"log_dups_size":0,"ondisk_log_size":4708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:21.630703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308543488,"num_objects":4708,"num_object_clones":0,"num_object_copies":14124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4708,"num_write_kb":301312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":112,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.464964+0000","last_change":"2023-09-23T23:10:59.516007+0000","last_active":"2023-09-23T23:14:12.464964+0000","last_peered":"2023-09-23T23:14:12.464964+0000","last_clean":"2023-09-23T23:14:12.464964+0000","last_became_active":"2023-09-23T23:10:59.515388+0000","last_became_peered":"2023-09-23T23:10:59.515388+0000","last_unstale":"2023-09-23T23:14:12.464964+0000","last_undegraded":"2023-09-23T23:14:12.464964+0000","last_fullsized":"2023-09-23T23:14:12.464964+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:13.171258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":140,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.454115+0000","last_change":"2023-09-23T23:10:59.503483+0000","last_active":"2023-09-23T23:14:12.454115+0000","last_peered":"2023-09-23T23:14:12.454115+0000","last_clean":"2023-09-23T23:14:12.454115+0000","last_became_active":"2023-09-23T23:10:59.502851+0000","last_became_peered":"2023-09-23T23:10:59.502851+0000","last_unstale":"2023-09-23T23:14:12.454115+0000","last_undegraded":"2023-09-23T23:14:12.454115+0000","last_fullsized":"2023-09-23T23:14:12.454115+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:20.388467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"172'4367","reported_seq":4383,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:18.797667+0000","last_change":"2023-09-23T23:13:28.355012+0000","last_active":"2023-09-23T23:14:18.797667+0000","last_peered":"2023-09-23T23:14:18.797667+0000","last_clean":"2023-09-23T23:14:18.797667+0000","last_became_active":"2023-09-23T23:13:28.354788+0000","last_became_peered":"2023-09-23T23:13:28.354788+0000","last_unstale":"2023-09-23T23:14:18.797667+0000","last_undegraded":"2023-09-23T23:14:18.797667+0000","last_fullsized":"2023-09-23T23:14:18.797667+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":4367,"log_dups_size":0,"ondisk_log_size":4367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:04.013664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286195712,"num_objects":4367,"num_object_clones":0,"num_object_copies":13101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4367,"num_write_kb":279488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":132,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.574176+0000","last_change":"2023-09-23T23:07:30.069993+0000","last_active":"2023-09-23T23:14:07.574176+0000","last_peered":"2023-09-23T23:14:07.574176+0000","last_clean":"2023-09-23T23:14:07.574176+0000","last_became_active":"2023-09-23T23:07:30.069634+0000","last_became_peered":"2023-09-23T23:07:30.069634+0000","last_unstale":"2023-09-23T23:14:07.574176+0000","last_undegraded":"2023-09-23T23:14:07.574176+0000","last_fullsized":"2023-09-23T23:14:07.574176+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:24.608905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":150,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.576328+0000","last_change":"2023-09-23T23:10:59.498546+0000","last_active":"2023-09-23T23:14:07.576328+0000","last_peered":"2023-09-23T23:14:07.576328+0000","last_clean":"2023-09-23T23:14:07.576328+0000","last_became_active":"2023-09-23T23:10:59.498231+0000","last_became_peered":"2023-09-23T23:10:59.498231+0000","last_unstale":"2023-09-23T23:14:07.576328+0000","last_undegraded":"2023-09-23T23:14:07.576328+0000","last_fullsized":"2023-09-23T23:14:07.576328+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:37.404273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":134,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.574522+0000","last_change":"2023-09-23T23:13:29.421393+0000","last_active":"2023-09-23T23:14:07.574522+0000","last_peered":"2023-09-23T23:14:07.574522+0000","last_clean":"2023-09-23T23:14:07.574522+0000","last_became_active":"2023-09-23T23:13:29.421042+0000","last_became_peered":"2023-09-23T23:13:29.421042+0000","last_unstale":"2023-09-23T23:14:07.574522+0000","last_undegraded":"2023-09-23T23:14:07.574522+0000","last_fullsized":"2023-09-23T23:14:07.574522+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:22:35.586669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"173'4461","reported_seq":4479,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T23:14:20.428240+0000","last_change":"2023-09-23T23:13:28.360018+0000","last_active":"2023-09-23T23:14:20.428240+0000","last_peered":"2023-09-23T23:14:20.428240+0000","last_clean":"2023-09-23T23:14:20.428240+0000","last_became_active":"2023-09-23T23:13:28.359701+0000","last_became_peered":"2023-09-23T23:13:28.359701+0000","last_unstale":"2023-09-23T23:14:20.428240+0000","last_undegraded":"2023-09-23T23:14:20.428240+0000","last_fullsized":"2023-09-23T23:14:20.428240+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":4461,"log_dups_size":0,"ondisk_log_size":4461,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:40.620740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292356096,"num_objects":4461,"num_object_clones":0,"num_object_copies":13383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4461,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4461,"num_write_kb":285504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":137,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.576056+0000","last_change":"2023-09-23T23:07:30.075093+0000","last_active":"2023-09-23T23:14:07.576056+0000","last_peered":"2023-09-23T23:14:07.576056+0000","last_clean":"2023-09-23T23:14:07.576056+0000","last_became_active":"2023-09-23T23:07:30.074729+0000","last_became_peered":"2023-09-23T23:07:30.074729+0000","last_unstale":"2023-09-23T23:14:07.576056+0000","last_undegraded":"2023-09-23T23:14:07.576056+0000","last_fullsized":"2023-09-23T23:14:07.576056+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:34.502057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":133,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.586000+0000","last_change":"2023-09-23T23:04:11.514703+0000","last_active":"2023-09-23T23:14:07.586000+0000","last_peered":"2023-09-23T23:14:07.586000+0000","last_clean":"2023-09-23T23:14:07.586000+0000","last_became_active":"2023-09-23T23:04:11.514403+0000","last_became_peered":"2023-09-23T23:04:11.514403+0000","last_unstale":"2023-09-23T23:14:07.586000+0000","last_undegraded":"2023-09-23T23:14:07.586000+0000","last_fullsized":"2023-09-23T23:14:07.586000+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:26.374346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":155,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.574107+0000","last_change":"2023-09-23T23:10:59.502030+0000","last_active":"2023-09-23T23:14:07.574107+0000","last_peered":"2023-09-23T23:14:07.574107+0000","last_clean":"2023-09-23T23:14:07.574107+0000","last_became_active":"2023-09-23T23:10:59.501714+0000","last_became_peered":"2023-09-23T23:10:59.501714+0000","last_unstale":"2023-09-23T23:14:07.574107+0000","last_undegraded":"2023-09-23T23:14:07.574107+0000","last_fullsized":"2023-09-23T23:14:07.574107+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:52.898368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"172'4277","reported_seq":4293,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:18.297395+0000","last_change":"2023-09-23T23:13:29.358375+0000","last_active":"2023-09-23T23:14:18.297395+0000","last_peered":"2023-09-23T23:14:18.297395+0000","last_clean":"2023-09-23T23:14:18.297395+0000","last_became_active":"2023-09-23T23:13:29.358036+0000","last_became_peered":"2023-09-23T23:13:29.358036+0000","last_unstale":"2023-09-23T23:14:18.297395+0000","last_undegraded":"2023-09-23T23:14:18.297395+0000","last_fullsized":"2023-09-23T23:14:18.297395+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":4277,"log_dups_size":0,"ondisk_log_size":4277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:57.055547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280297472,"num_objects":4277,"num_object_clones":0,"num_object_copies":12831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4277,"num_write_kb":273728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":129,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.459352+0000","last_change":"2023-09-23T23:10:59.501544+0000","last_active":"2023-09-23T23:14:12.459352+0000","last_peered":"2023-09-23T23:14:12.459352+0000","last_clean":"2023-09-23T23:14:12.459352+0000","last_became_active":"2023-09-23T23:10:59.501247+0000","last_became_peered":"2023-09-23T23:10:59.501247+0000","last_unstale":"2023-09-23T23:14:12.459352+0000","last_undegraded":"2023-09-23T23:14:12.459352+0000","last_fullsized":"2023-09-23T23:14:12.459352+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:54.254755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":141,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.573289+0000","last_change":"2023-09-23T23:08:45.122712+0000","last_active":"2023-09-23T23:14:07.573289+0000","last_peered":"2023-09-23T23:14:07.573289+0000","last_clean":"2023-09-23T23:14:07.573289+0000","last_became_active":"2023-09-23T23:08:45.122391+0000","last_became_peered":"2023-09-23T23:08:45.122391+0000","last_unstale":"2023-09-23T23:14:07.573289+0000","last_undegraded":"2023-09-23T23:14:07.573289+0000","last_fullsized":"2023-09-23T23:14:07.573289+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:44.916349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":127,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.573492+0000","last_change":"2023-09-23T23:07:30.074957+0000","last_active":"2023-09-23T23:14:07.573492+0000","last_peered":"2023-09-23T23:14:07.573492+0000","last_clean":"2023-09-23T23:14:07.573492+0000","last_became_active":"2023-09-23T23:07:30.074475+0000","last_became_peered":"2023-09-23T23:07:30.074475+0000","last_unstale":"2023-09-23T23:14:07.573492+0000","last_undegraded":"2023-09-23T23:14:07.573492+0000","last_fullsized":"2023-09-23T23:14:07.573492+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:25.337999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"173'4664","reported_seq":4683,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T23:14:22.591812+0000","last_change":"2023-09-23T23:13:29.364074+0000","last_active":"2023-09-23T23:14:22.591812+0000","last_peered":"2023-09-23T23:14:22.591812+0000","last_clean":"2023-09-23T23:14:22.591812+0000","last_became_active":"2023-09-23T23:13:29.363791+0000","last_became_peered":"2023-09-23T23:13:29.363791+0000","last_unstale":"2023-09-23T23:14:22.591812+0000","last_undegraded":"2023-09-23T23:14:22.591812+0000","last_fullsized":"2023-09-23T23:14:22.591812+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":4664,"log_dups_size":0,"ondisk_log_size":4664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:53:07.822571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305659904,"num_objects":4664,"num_object_clones":0,"num_object_copies":13992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4664,"num_write_kb":298496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":137,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.576156+0000","last_change":"2023-09-23T23:08:45.111935+0000","last_active":"2023-09-23T23:14:07.576156+0000","last_peered":"2023-09-23T23:14:07.576156+0000","last_clean":"2023-09-23T23:14:07.576156+0000","last_became_active":"2023-09-23T23:08:45.111549+0000","last_became_peered":"2023-09-23T23:08:45.111549+0000","last_unstale":"2023-09-23T23:14:07.576156+0000","last_undegraded":"2023-09-23T23:14:07.576156+0000","last_fullsized":"2023-09-23T23:14:07.576156+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:49.879205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":513,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.580308+0000","last_change":"2023-09-23T23:04:11.517317+0000","last_active":"2023-09-23T23:14:07.580308+0000","last_peered":"2023-09-23T23:14:07.580308+0000","last_clean":"2023-09-23T23:14:07.580308+0000","last_became_active":"2023-09-23T23:04:11.516929+0000","last_became_peered":"2023-09-23T23:04:11.516929+0000","last_unstale":"2023-09-23T23:14:07.580308+0000","last_undegraded":"2023-09-23T23:14:07.580308+0000","last_fullsized":"2023-09-23T23:14:07.580308+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T22:49:49.548021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:19.871312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":89,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.453101+0000","last_change":"2023-09-23T23:10:59.512923+0000","last_active":"2023-09-23T23:14:12.453101+0000","last_peered":"2023-09-23T23:14:12.453101+0000","last_clean":"2023-09-23T23:14:12.453101+0000","last_became_active":"2023-09-23T23:10:59.512637+0000","last_became_peered":"2023-09-23T23:10:59.512637+0000","last_unstale":"2023-09-23T23:14:12.453101+0000","last_undegraded":"2023-09-23T23:14:12.453101+0000","last_fullsized":"2023-09-23T23:14:12.453101+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:43:06.983322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":115,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.579628+0000","last_change":"2023-09-23T22:53:18.075922+0000","last_active":"2023-09-23T23:14:07.579628+0000","last_peered":"2023-09-23T23:14:07.579628+0000","last_clean":"2023-09-23T23:14:07.579628+0000","last_became_active":"2023-09-23T22:53:18.075621+0000","last_became_peered":"2023-09-23T22:53:18.075621+0000","last_unstale":"2023-09-23T23:14:07.579628+0000","last_undegraded":"2023-09-23T23:14:07.579628+0000","last_fullsized":"2023-09-23T23:14:07.579628+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:01.490333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"173'4406","reported_seq":4424,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T23:14:20.403548+0000","last_change":"2023-09-23T23:13:28.357808+0000","last_active":"2023-09-23T23:14:20.403548+0000","last_peered":"2023-09-23T23:14:20.403548+0000","last_clean":"2023-09-23T23:14:20.403548+0000","last_became_active":"2023-09-23T23:13:28.357474+0000","last_became_peered":"2023-09-23T23:13:28.357474+0000","last_unstale":"2023-09-23T23:14:20.403548+0000","last_undegraded":"2023-09-23T23:14:20.403548+0000","last_fullsized":"2023-09-23T23:14:20.403548+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":4406,"log_dups_size":0,"ondisk_log_size":4406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:08.042264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288751616,"num_objects":4406,"num_object_clones":0,"num_object_copies":13218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4406,"num_write_kb":281984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":154,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.580322+0000","last_change":"2023-09-23T23:07:30.068847+0000","last_active":"2023-09-23T23:14:07.580322+0000","last_peered":"2023-09-23T23:14:07.580322+0000","last_clean":"2023-09-23T23:14:07.580322+0000","last_became_active":"2023-09-23T23:07:30.068533+0000","last_became_peered":"2023-09-23T23:07:30.068533+0000","last_unstale":"2023-09-23T23:14:07.580322+0000","last_undegraded":"2023-09-23T23:14:07.580322+0000","last_fullsized":"2023-09-23T23:14:07.580322+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:59.598863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":122,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.463426+0000","last_change":"2023-09-23T23:10:59.503085+0000","last_active":"2023-09-23T23:14:12.463426+0000","last_peered":"2023-09-23T23:14:12.463426+0000","last_clean":"2023-09-23T23:14:12.463426+0000","last_became_active":"2023-09-23T23:10:59.502737+0000","last_became_peered":"2023-09-23T23:10:59.502737+0000","last_unstale":"2023-09-23T23:14:12.463426+0000","last_undegraded":"2023-09-23T23:14:12.463426+0000","last_fullsized":"2023-09-23T23:14:12.463426+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:22.783324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":137,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.463663+0000","last_change":"2023-09-23T23:08:45.116856+0000","last_active":"2023-09-23T23:14:12.463663+0000","last_peered":"2023-09-23T23:14:12.463663+0000","last_clean":"2023-09-23T23:14:12.463663+0000","last_became_active":"2023-09-23T23:08:45.116519+0000","last_became_peered":"2023-09-23T23:08:45.116519+0000","last_unstale":"2023-09-23T23:14:12.463663+0000","last_undegraded":"2023-09-23T23:14:12.463663+0000","last_fullsized":"2023-09-23T23:14:12.463663+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:25.917776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"172'4418","reported_seq":4434,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:18.814765+0000","last_change":"2023-09-23T23:13:28.359433+0000","last_active":"2023-09-23T23:14:18.814765+0000","last_peered":"2023-09-23T23:14:18.814765+0000","last_clean":"2023-09-23T23:14:18.814765+0000","last_became_active":"2023-09-23T23:13:28.359147+0000","last_became_peered":"2023-09-23T23:13:28.359147+0000","last_unstale":"2023-09-23T23:14:18.814765+0000","last_undegraded":"2023-09-23T23:14:18.814765+0000","last_fullsized":"2023-09-23T23:14:18.814765+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":4418,"log_dups_size":0,"ondisk_log_size":4418,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:27.415293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289538048,"num_objects":4418,"num_object_clones":0,"num_object_copies":13254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4418,"num_write_kb":282752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":137,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.582185+0000","last_change":"2023-09-23T23:04:11.900743+0000","last_active":"2023-09-23T23:14:07.582185+0000","last_peered":"2023-09-23T23:14:07.582185+0000","last_clean":"2023-09-23T23:14:07.582185+0000","last_became_active":"2023-09-23T23:04:11.900418+0000","last_became_peered":"2023-09-23T23:04:11.900418+0000","last_unstale":"2023-09-23T23:14:07.582185+0000","last_undegraded":"2023-09-23T23:14:07.582185+0000","last_fullsized":"2023-09-23T23:14:07.582185+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:45:31.893513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":141,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.574061+0000","last_change":"2023-09-23T23:08:45.124284+0000","last_active":"2023-09-23T23:14:07.574061+0000","last_peered":"2023-09-23T23:14:07.574061+0000","last_clean":"2023-09-23T23:14:07.574061+0000","last_became_active":"2023-09-23T23:08:45.123988+0000","last_became_peered":"2023-09-23T23:08:45.123988+0000","last_unstale":"2023-09-23T23:14:07.574061+0000","last_undegraded":"2023-09-23T23:14:07.574061+0000","last_fullsized":"2023-09-23T23:14:07.574061+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:26.726970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":107,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.463978+0000","last_change":"2023-09-23T23:04:11.911507+0000","last_active":"2023-09-23T23:14:12.463978+0000","last_peered":"2023-09-23T23:14:12.463978+0000","last_clean":"2023-09-23T23:14:12.463978+0000","last_became_active":"2023-09-23T23:04:11.908281+0000","last_became_peered":"2023-09-23T23:04:11.908281+0000","last_unstale":"2023-09-23T23:14:12.463978+0000","last_undegraded":"2023-09-23T23:14:12.463978+0000","last_fullsized":"2023-09-23T23:14:12.463978+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:30.414860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"172'4371","reported_seq":4387,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:18.841091+0000","last_change":"2023-09-23T23:13:28.354568+0000","last_active":"2023-09-23T23:14:18.841091+0000","last_peered":"2023-09-23T23:14:18.841091+0000","last_clean":"2023-09-23T23:14:18.841091+0000","last_became_active":"2023-09-23T23:13:28.354292+0000","last_became_peered":"2023-09-23T23:13:28.354292+0000","last_unstale":"2023-09-23T23:14:18.841091+0000","last_undegraded":"2023-09-23T23:14:18.841091+0000","last_fullsized":"2023-09-23T23:14:18.841091+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":4371,"log_dups_size":0,"ondisk_log_size":4371,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:12.723223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286457856,"num_objects":4371,"num_object_clones":0,"num_object_copies":13113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4371,"num_write_kb":279744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":135,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.455347+0000","last_change":"2023-09-23T23:10:59.511903+0000","last_active":"2023-09-23T23:14:12.455347+0000","last_peered":"2023-09-23T23:14:12.455347+0000","last_clean":"2023-09-23T23:14:12.455347+0000","last_became_active":"2023-09-23T23:10:59.511022+0000","last_became_peered":"2023-09-23T23:10:59.511022+0000","last_unstale":"2023-09-23T23:14:12.455347+0000","last_undegraded":"2023-09-23T23:14:12.455347+0000","last_fullsized":"2023-09-23T23:14:12.455347+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:03.498373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":126,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.574229+0000","last_change":"2023-09-23T23:07:30.083011+0000","last_active":"2023-09-23T23:14:07.574229+0000","last_peered":"2023-09-23T23:14:07.574229+0000","last_clean":"2023-09-23T23:14:07.574229+0000","last_became_active":"2023-09-23T23:07:30.082544+0000","last_became_peered":"2023-09-23T23:07:30.082544+0000","last_unstale":"2023-09-23T23:14:07.574229+0000","last_undegraded":"2023-09-23T23:14:07.574229+0000","last_fullsized":"2023-09-23T23:14:07.574229+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:20.624316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":132,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.465036+0000","last_change":"2023-09-23T23:10:59.512305+0000","last_active":"2023-09-23T23:14:12.465036+0000","last_peered":"2023-09-23T23:14:12.465036+0000","last_clean":"2023-09-23T23:14:12.465036+0000","last_became_active":"2023-09-23T23:10:59.511941+0000","last_became_peered":"2023-09-23T23:10:59.511941+0000","last_unstale":"2023-09-23T23:14:12.465036+0000","last_undegraded":"2023-09-23T23:14:12.465036+0000","last_fullsized":"2023-09-23T23:14:12.465036+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:06.293763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"173'4619","reported_seq":4637,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-09-23T23:14:22.626596+0000","last_change":"2023-09-23T23:13:28.359953+0000","last_active":"2023-09-23T23:14:22.626596+0000","last_peered":"2023-09-23T23:14:22.626596+0000","last_clean":"2023-09-23T23:14:22.626596+0000","last_became_active":"2023-09-23T23:13:28.359690+0000","last_became_peered":"2023-09-23T23:13:28.359690+0000","last_unstale":"2023-09-23T23:14:22.626596+0000","last_undegraded":"2023-09-23T23:14:22.626596+0000","last_fullsized":"2023-09-23T23:14:22.626596+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":4619,"log_dups_size":0,"ondisk_log_size":4619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:56.963155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302710784,"num_objects":4619,"num_object_clones":0,"num_object_copies":13857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4619,"num_write_kb":295616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":137,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.579865+0000","last_change":"2023-09-23T23:08:45.116806+0000","last_active":"2023-09-23T23:14:07.579865+0000","last_peered":"2023-09-23T23:14:07.579865+0000","last_clean":"2023-09-23T23:14:07.579865+0000","last_became_active":"2023-09-23T23:08:45.116462+0000","last_became_peered":"2023-09-23T23:08:45.116462+0000","last_unstale":"2023-09-23T23:14:07.579865+0000","last_undegraded":"2023-09-23T23:14:07.579865+0000","last_fullsized":"2023-09-23T23:14:07.579865+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:24.502987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":126,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.463513+0000","last_change":"2023-09-23T23:07:30.072388+0000","last_active":"2023-09-23T23:14:12.463513+0000","last_peered":"2023-09-23T23:14:12.463513+0000","last_clean":"2023-09-23T23:14:12.463513+0000","last_became_active":"2023-09-23T23:07:30.072099+0000","last_became_peered":"2023-09-23T23:07:30.072099+0000","last_unstale":"2023-09-23T23:14:12.463513+0000","last_undegraded":"2023-09-23T23:14:12.463513+0000","last_fullsized":"2023-09-23T23:14:12.463513+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:19.758734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":115,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.585208+0000","last_change":"2023-09-23T22:53:17.905706+0000","last_active":"2023-09-23T23:14:07.585208+0000","last_peered":"2023-09-23T23:14:07.585208+0000","last_clean":"2023-09-23T23:14:07.585208+0000","last_became_active":"2023-09-23T22:53:17.905309+0000","last_became_peered":"2023-09-23T22:53:17.905309+0000","last_unstale":"2023-09-23T23:14:07.585208+0000","last_undegraded":"2023-09-23T23:14:07.585208+0000","last_fullsized":"2023-09-23T23:14:07.585208+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:06.105426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":111,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.464207+0000","last_change":"2023-09-23T23:04:11.900795+0000","last_active":"2023-09-23T23:14:12.464207+0000","last_peered":"2023-09-23T23:14:12.464207+0000","last_clean":"2023-09-23T23:14:12.464207+0000","last_became_active":"2023-09-23T23:04:11.899751+0000","last_became_peered":"2023-09-23T23:04:11.899751+0000","last_unstale":"2023-09-23T23:14:12.464207+0000","last_undegraded":"2023-09-23T23:14:12.464207+0000","last_fullsized":"2023-09-23T23:14:12.464207+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:41.206493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":145,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.460153+0000","last_change":"2023-09-23T23:10:59.511474+0000","last_active":"2023-09-23T23:14:12.460153+0000","last_peered":"2023-09-23T23:14:12.460153+0000","last_clean":"2023-09-23T23:14:12.460153+0000","last_became_active":"2023-09-23T23:10:59.511175+0000","last_became_peered":"2023-09-23T23:10:59.511175+0000","last_unstale":"2023-09-23T23:14:12.460153+0000","last_undegraded":"2023-09-23T23:14:12.460153+0000","last_fullsized":"2023-09-23T23:14:12.460153+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:41.647976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":135,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.576190+0000","last_change":"2023-09-23T23:07:30.075311+0000","last_active":"2023-09-23T23:14:07.576190+0000","last_peered":"2023-09-23T23:14:07.576190+0000","last_clean":"2023-09-23T23:14:07.576190+0000","last_became_active":"2023-09-23T23:07:30.074952+0000","last_became_peered":"2023-09-23T23:07:30.074952+0000","last_unstale":"2023-09-23T23:14:07.576190+0000","last_undegraded":"2023-09-23T23:14:07.576190+0000","last_fullsized":"2023-09-23T23:14:07.576190+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:16.724083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":149,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.585876+0000","last_change":"2023-09-23T23:10:59.512738+0000","last_active":"2023-09-23T23:14:07.585876+0000","last_peered":"2023-09-23T23:14:07.585876+0000","last_clean":"2023-09-23T23:14:07.585876+0000","last_became_active":"2023-09-23T23:10:59.512011+0000","last_became_peered":"2023-09-23T23:10:59.512011+0000","last_unstale":"2023-09-23T23:14:07.585876+0000","last_undegraded":"2023-09-23T23:14:07.585876+0000","last_fullsized":"2023-09-23T23:14:07.585876+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:18.088964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":118,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.576626+0000","last_change":"2023-09-23T22:53:16.891297+0000","last_active":"2023-09-23T23:14:07.576626+0000","last_peered":"2023-09-23T23:14:07.576626+0000","last_clean":"2023-09-23T23:14:07.576626+0000","last_became_active":"2023-09-23T22:53:16.891025+0000","last_became_peered":"2023-09-23T22:53:16.891025+0000","last_unstale":"2023-09-23T23:14:07.576626+0000","last_undegraded":"2023-09-23T23:14:07.576626+0000","last_fullsized":"2023-09-23T23:14:07.576626+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:40:50.608339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":149,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.585685+0000","last_change":"2023-09-23T23:10:59.512247+0000","last_active":"2023-09-23T23:14:07.585685+0000","last_peered":"2023-09-23T23:14:07.585685+0000","last_clean":"2023-09-23T23:14:07.585685+0000","last_became_active":"2023-09-23T23:10:59.511865+0000","last_became_peered":"2023-09-23T23:10:59.511865+0000","last_unstale":"2023-09-23T23:14:07.585685+0000","last_undegraded":"2023-09-23T23:14:07.585685+0000","last_fullsized":"2023-09-23T23:14:07.585685+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:20.981239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":131,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.453318+0000","last_change":"2023-09-23T23:10:59.512958+0000","last_active":"2023-09-23T23:14:12.453318+0000","last_peered":"2023-09-23T23:14:12.453318+0000","last_clean":"2023-09-23T23:14:12.453318+0000","last_became_active":"2023-09-23T23:10:59.512628+0000","last_became_peered":"2023-09-23T23:10:59.512628+0000","last_unstale":"2023-09-23T23:14:12.453318+0000","last_undegraded":"2023-09-23T23:14:12.453318+0000","last_fullsized":"2023-09-23T23:14:12.453318+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:16:26.998696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":146,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.453669+0000","last_change":"2023-09-23T23:10:59.511365+0000","last_active":"2023-09-23T23:14:12.453669+0000","last_peered":"2023-09-23T23:14:12.453669+0000","last_clean":"2023-09-23T23:14:12.453669+0000","last_became_active":"2023-09-23T23:10:59.511073+0000","last_became_peered":"2023-09-23T23:10:59.511073+0000","last_unstale":"2023-09-23T23:14:12.453669+0000","last_undegraded":"2023-09-23T23:14:12.453669+0000","last_fullsized":"2023-09-23T23:14:12.453669+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:35.196455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":126,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-09-23T23:14:12.463977+0000","last_change":"2023-09-23T23:08:45.113546+0000","last_active":"2023-09-23T23:14:12.463977+0000","last_peered":"2023-09-23T23:14:12.463977+0000","last_clean":"2023-09-23T23:14:12.463977+0000","last_became_active":"2023-09-23T23:08:45.113191+0000","last_became_peered":"2023-09-23T23:08:45.113191+0000","last_unstale":"2023-09-23T23:14:12.463977+0000","last_undegraded":"2023-09-23T23:14:12.463977+0000","last_fullsized":"2023-09-23T23:14:12.463977+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:06.372282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":142,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-09-23T23:14:07.585001+0000","last_change":"2023-09-23T23:08:45.116882+0000","last_active":"2023-09-23T23:14:07.585001+0000","last_peered":"2023-09-23T23:14:07.585001+0000","last_clean":"2023-09-23T23:14:07.585001+0000","last_became_active":"2023-09-23T23:08:45.116543+0000","last_became_peered":"2023-09-23T23:08:45.116543+0000","last_unstale":"2023-09-23T23:14:07.585001+0000","last_undegraded":"2023-09-23T23:14:07.585001+0000","last_fullsized":"2023-09-23T23:14:07.585001+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:08.722088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4667801600,"num_objects":71225,"num_object_clones":0,"num_object_copies":213675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":71225,"num_write_kb":4558400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13987217408,"data_stored":13987217408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":71225,"ondisk_log_size":71225,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234259,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3116808,"kb_used_data":2776064,"kb_used_omap":0,"kb_used_meta":340736,"kb_avail":90628344,"statfs":{"total":95995035648,"available":92803424256,"internally_reserved":0,"allocated":2842689536,"data_stored":2842131751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":348913664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,2,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:13:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":1.9119999999999999,"15min":2.742},"min":{"1min":0.35599999999999998,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.81899999999999995,"5min":29.928999999999998,"15min":230.249},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":2.137,"15min":2.7919999999999998},"min":{"1min":0.30499999999999999,"5min":0.24299999999999999,"15min":0.24199999999999999},"max":{"1min":0.93200000000000005,"5min":29.977,"15min":230.261},"last":1.206}]},{"osd":1,"last update":"Sat Sep 23 23:13:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":3.0459999999999998,"15min":4.7939999999999996},"min":{"1min":0.371,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.748,"5min":91.509,"15min":93.245999999999995},"last":0.307},{"interface":"front","average":{"1min":0.56100000000000005,"5min":2.702,"15min":4.6020000000000003},"min":{"1min":0.33100000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.0009999999999999,"5min":91.691000000000003,"15min":92.748000000000005},"last":0.98499999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":2.145,"15min":2.9769999999999999},"min":{"1min":0.40200000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.5430000000000001,"5min":12.183,"15min":12.183},"last":3.1400000000000001},{"interface":"front","average":{"1min":0.67400000000000004,"5min":1.7390000000000001,"15min":2.1389999999999998},"min":{"1min":0.30099999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.819,"5min":11.781000000000001,"15min":11.781000000000001},"last":4.3280000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:14:03 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":1.9039999999999999,"15min":2.278},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.3420000000000001,"5min":11.507999999999999,"15min":11.507999999999999},"last":1.341},{"interface":"front","average":{"1min":0.88800000000000001,"5min":1.8999999999999999,"15min":2.3460000000000001},"min":{"1min":0.314,"5min":0.307,"15min":0.307},"max":{"1min":2.0710000000000002,"5min":8.9670000000000005,"15min":8.9670000000000005},"last":1.3620000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:14:18 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":1.2410000000000001,"15min":1.375},"min":{"1min":0.29899999999999999,"5min":0.24299999999999999,"15min":0.23599999999999999},"max":{"1min":0.64300000000000002,"5min":8.984,"15min":92.781000000000006},"last":0.437},{"interface":"front","average":{"1min":0.82299999999999995,"5min":1.2669999999999999,"15min":1.3280000000000001},"min":{"1min":0.34300000000000003,"5min":0.28499999999999998,"15min":0.254},"max":{"1min":4.3630000000000004,"5min":7.7290000000000001,"15min":92.626999999999995},"last":4.3630000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:14:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":1.6759999999999999,"15min":1.5629999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26500000000000001},"max":{"1min":2.5590000000000002,"5min":12.067,"15min":93.135000000000005},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.1919999999999999,"15min":1.4299999999999999},"min":{"1min":0.378,"5min":0.29999999999999999,"15min":0.29399999999999998},"max":{"1min":1.3280000000000001,"5min":8.9290000000000003,"15min":92.885000000000005},"last":1.3280000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:14:18 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.282,"15min":1.4330000000000001},"min":{"1min":0.375,"5min":0.315,"15min":0.27300000000000002},"max":{"1min":1.5580000000000001,"5min":13.590999999999999,"15min":93.105999999999995},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.84399999999999997,"5min":1.4379999999999999,"15min":1.3700000000000001},"min":{"1min":0.33300000000000002,"5min":0.32300000000000001,"15min":0.28399999999999997},"max":{"1min":4.2119999999999997,"5min":9.6240000000000006,"15min":93.198999999999998},"last":4.2119999999999997}]}]},{"osd":6,"up_from":46,"seq":197568495895,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1107128,"kb_used_data":829232,"kb_used_omap":0,"kb_used_meta":277888,"kb_avail":92638024,"statfs":{"total":95995035648,"available":94861336576,"internally_reserved":0,"allocated":849133568,"data_stored":848591756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":284557312},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.4570000000000001,"15min":3.8119999999999998},"min":{"1min":0.34899999999999998,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":1.5289999999999999,"5min":7.649,"15min":97.858000000000004},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.52000000000000002,"5min":1.494,"15min":4.399},"min":{"1min":0.35499999999999998,"5min":0.24399999999999999,"15min":0.22700000000000001},"max":{"1min":0.91900000000000004,"5min":8.3059999999999992,"15min":99.793000000000006},"last":0.88500000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":4.1820000000000004,"15min":1.9990000000000001},"min":{"1min":0.38200000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.93799999999999994,"5min":209.00899999999999,"15min":209.00899999999999},"last":1.242},{"interface":"front","average":{"1min":0.56399999999999995,"5min":4.2670000000000003,"15min":2.3149999999999999},"min":{"1min":0.34300000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.95399999999999996,"5min":209.03,"15min":209.03},"last":1.7929999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":1.681,"15min":1.569},"min":{"1min":0.39400000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.0369999999999999,"5min":10.871,"15min":10.871},"last":0.29999999999999999},{"interface":"front","average":{"1min":1.03,"5min":2.0819999999999999,"15min":1.9139999999999999},"min":{"1min":0.41399999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.343,"5min":12.148,"15min":12.148},"last":1.127}]},{"osd":3,"last update":"Sat Sep 23 23:14:05 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.9870000000000001,"15min":2.452},"min":{"1min":0.436,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":6.3019999999999996,"5min":7.4059999999999997,"15min":7.4059999999999997},"last":3.073},{"interface":"front","average":{"1min":1.105,"5min":2.3130000000000002,"15min":2.859},"min":{"1min":0.45000000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":7.1520000000000001,"5min":9.4550000000000001,"15min":9.4550000000000001},"last":0.94899999999999995}]},{"osd":4,"last update":"Sat Sep 23 23:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":1.405,"15min":1.093},"min":{"1min":0.30499999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.099,"5min":11.890000000000001,"15min":11.890000000000001},"last":0.318},{"interface":"front","average":{"1min":0.66700000000000004,"5min":1.6220000000000001,"15min":1.194},"min":{"1min":0.33000000000000002,"5min":0.308,"15min":0.24299999999999999},"max":{"1min":2.093,"5min":11.728,"15min":11.728},"last":0.63600000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":1.516,"15min":1.339},"min":{"1min":0.378,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":1.359,"5min":8.0850000000000009,"15min":26.158999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.63200000000000001,"5min":1.3149999999999999,"15min":1.256},"min":{"1min":0.374,"5min":0.30599999999999999,"15min":0.30199999999999999},"max":{"1min":1.2130000000000001,"5min":8.0169999999999995,"15min":25.786000000000001},"last":0.42999999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:14:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.6419999999999999,"15min":1.423},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32100000000000001},"max":{"1min":4.3520000000000003,"5min":10.968999999999999,"15min":12.676},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.627,"5min":1.29,"15min":1.198},"min":{"1min":0.35099999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.5269999999999999,"5min":7.9669999999999996,"15min":8.1310000000000002},"last":0.52200000000000002}]}]},{"osd":1,"up_from":112,"seq":481036337236,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2794540,"kb_used_data":2488740,"kb_used_omap":4,"kb_used_meta":305787,"kb_avail":90950612,"statfs":{"total":95995035648,"available":93133426688,"internally_reserved":0,"allocated":2548469760,"data_stored":2547763111,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":313126798},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.29599999999999999,"5min":1.77,"15min":1.5960000000000001},"min":{"1min":0.22700000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.50700000000000001,"5min":10.221,"15min":10.221},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.33000000000000002,"5min":1.51,"15min":1.4159999999999999},"min":{"1min":0.23000000000000001,"5min":0.21199999999999999,"15min":0.20100000000000001},"max":{"1min":0.436,"5min":12.212,"15min":12.212},"last":0.41299999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:13:51 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":1.9430000000000001,"15min":2.2749999999999999},"min":{"1min":0.21099999999999999,"5min":0.192,"15min":0.192},"max":{"1min":1.5600000000000001,"5min":16.696999999999999,"15min":16.696999999999999},"last":0.248},{"interface":"front","average":{"1min":0.44900000000000001,"5min":1.8129999999999999,"15min":2.1789999999999998},"min":{"1min":0.29799999999999999,"5min":0.215,"15min":0.215},"max":{"1min":1.028,"5min":9.0879999999999992,"15min":9.0879999999999992},"last":0.312}]},{"osd":3,"last update":"Sat Sep 23 23:14:07 2023","interfaces":[{"interface":"back","average":{"1min":1.913,"5min":2.1269999999999998,"15min":2.5110000000000001},"min":{"1min":0.28100000000000003,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.242000000000001,"5min":12.234999999999999,"15min":12.234999999999999},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.448,"5min":1.78,"15min":2.1200000000000001},"min":{"1min":0.307,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.0739999999999998,"5min":12.256,"15min":12.256},"last":0.36899999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":1.6799999999999999,"15min":1.498},"min":{"1min":0.39100000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.83799999999999997,"5min":12.676,"15min":12.676},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":1.7290000000000001,"15min":1.4330000000000001},"min":{"1min":0.36399999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.67000000000000004,"5min":11.125,"15min":11.125},"last":0.38200000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.7450000000000001,"15min":3.6890000000000001},"min":{"1min":0.38500000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.78700000000000003,"5min":12.351000000000001,"15min":67.497},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.623,"5min":1.9079999999999999,"15min":3.746},"min":{"1min":0.45900000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.94299999999999995,"5min":12.757999999999999,"15min":67.540999999999997},"last":0.44}]},{"osd":6,"last update":"Sat Sep 23 23:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.486,"15min":1.5149999999999999},"min":{"1min":0.41399999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.83599999999999997,"5min":12.186,"15min":12.186},"last":0.622},{"interface":"front","average":{"1min":0.63200000000000001,"5min":2.1230000000000002,"15min":1.629},"min":{"1min":0.45300000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.85899999999999999,"5min":12.785,"15min":12.785},"last":0.35599999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":2.0910000000000002,"15min":1.661},"min":{"1min":0.496,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.90500000000000003,"5min":12.727,"15min":12.727},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.65700000000000003,"5min":1.7529999999999999,"15min":1.544},"min":{"1min":0.45200000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.97399999999999998,"5min":9.8559999999999999,"15min":9.8559999999999999},"last":0.47899999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024060,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1540940,"kb_used_data":1143492,"kb_used_omap":5,"kb_used_meta":397434,"kb_avail":92204212,"statfs":{"total":95995035648,"available":94417113088,"internally_reserved":0,"allocated":1170935808,"data_stored":1170205931,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":406973050},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:13:40 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":1.762,"15min":2.3079999999999998},"min":{"1min":0.19600000000000001,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":1.786,"5min":11.029,"15min":21.629999999999999},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.495,"5min":1.8919999999999999,"15min":2.1560000000000001},"min":{"1min":0.252,"5min":0.189,"15min":0.189},"max":{"1min":3.508,"5min":11.922000000000001,"15min":13.382},"last":0.19}]},{"osd":2,"last update":"Sat Sep 23 23:13:57 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.905,"15min":2.0350000000000001},"min":{"1min":0.23499999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":4.6929999999999996,"5min":10.199,"15min":10.199},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.7430000000000001,"15min":1.913},"min":{"1min":0.25800000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.5590000000000002,"5min":9.0239999999999991,"15min":9.0239999999999991},"last":0.246}]},{"osd":3,"last update":"Sat Sep 23 23:14:02 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":2.3860000000000001,"15min":3.101},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":7.2560000000000002,"5min":11.866,"15min":11.866},"last":82.691000000000003},{"interface":"front","average":{"1min":0.45500000000000002,"5min":1.6890000000000001,"15min":2.2320000000000002},"min":{"1min":0.27000000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.119,"5min":9.0980000000000008,"15min":9.0980000000000008},"last":82.429000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:13:21 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":2.266,"15min":2.1509999999999998},"min":{"1min":0.39600000000000002,"5min":0.247,"15min":0.245},"max":{"1min":7.0759999999999996,"5min":28.376999999999999,"15min":28.376999999999999},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.53700000000000003,"5min":1.921,"15min":2.4649999999999999},"min":{"1min":0.38200000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.73399999999999999,"5min":30.706,"15min":30.706},"last":0.32700000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:13:21 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":1.847,"15min":2.4849999999999999},"min":{"1min":0.42399999999999999,"5min":0.33700000000000002,"15min":0.30499999999999999},"max":{"1min":0.65400000000000003,"5min":11.157,"15min":30.873999999999999},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.55500000000000005,"5min":1.7749999999999999,"15min":1.875},"min":{"1min":0.40799999999999997,"5min":0.30299999999999999,"15min":0.29499999999999998},"max":{"1min":0.755,"5min":10.117000000000001,"15min":10.117000000000001},"last":0.35299999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:13:21 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.893,"15min":2.5110000000000001},"min":{"1min":0.437,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.79800000000000004,"5min":10.605,"15min":21.047999999999998},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":2.0350000000000001,"15min":1.958},"min":{"1min":0.41799999999999998,"5min":0.36499999999999999,"15min":0.30499999999999999},"max":{"1min":0.80600000000000005,"5min":10.414,"15min":10.414},"last":0.373}]},{"osd":7,"last update":"Sat Sep 23 23:13:21 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":2.1070000000000002,"15min":1.9810000000000001},"min":{"1min":0.40200000000000002,"5min":0.36099999999999999,"15min":0.29299999999999998},"max":{"1min":0.73999999999999999,"5min":10.145,"15min":10.145},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.89800000000000002,"5min":2.0990000000000002,"15min":2.121},"min":{"1min":0.442,"5min":0.34999999999999998,"15min":0.313},"max":{"1min":7.1120000000000001,"5min":10.513999999999999,"15min":10.513999999999999},"last":0.44900000000000001}]}]},{"osd":2,"up_from":130,"seq":558345748550,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1177472,"kb_used_data":897336,"kb_used_omap":7,"kb_used_meta":280120,"kb_avail":92567680,"statfs":{"total":95995035648,"available":94789304320,"internally_reserved":0,"allocated":918872064,"data_stored":918212022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":286842961},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":1.726,"15min":1.9239999999999999},"min":{"1min":0.17999999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.58999999999999997,"5min":16.021000000000001,"15min":16.021000000000001},"last":1.774},{"interface":"front","average":{"1min":0.40400000000000003,"5min":1.998,"15min":2.8119999999999998},"min":{"1min":0.25700000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.246,"5min":16.152999999999999,"15min":16.152999999999999},"last":2.532}]},{"osd":1,"last update":"Sat Sep 23 23:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.859,"15min":2.4849999999999999},"min":{"1min":0.22,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":1.9019999999999999,"5min":9.3719999999999999,"15min":9.3719999999999999},"last":2.496},{"interface":"front","average":{"1min":0.45100000000000001,"5min":1.6499999999999999,"15min":2.4860000000000002},"min":{"1min":0.26000000000000001,"5min":0.218,"15min":0.218},"max":{"1min":1.4239999999999999,"5min":11.625,"15min":11.625},"last":1.5129999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:14:08 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.0960000000000001,"15min":1.212},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.1980000000000004,"5min":6.8230000000000004,"15min":6.8230000000000004},"last":3.4249999999999998},{"interface":"front","average":{"1min":0.93400000000000005,"5min":1.575,"15min":1.823},"min":{"1min":0.30299999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.8119999999999998,"5min":7.7110000000000003,"15min":7.7110000000000003},"last":1.542}]},{"osd":4,"last update":"Sat Sep 23 23:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":1.714,"15min":1.9890000000000001},"min":{"1min":0.33400000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.944,"5min":10.666,"15min":10.666},"last":4.8040000000000003},{"interface":"front","average":{"1min":0.626,"5min":1.4259999999999999,"15min":1.452},"min":{"1min":0.38,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.47,"5min":14.829000000000001,"15min":14.829000000000001},"last":2.3450000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3440000000000001,"5min":2.1760000000000002,"15min":2.456},"min":{"1min":0.38700000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":12.988,"5min":12.988,"15min":12.988},"last":2.661},{"interface":"front","average":{"1min":1.3109999999999999,"5min":1.548,"15min":1.6279999999999999},"min":{"1min":0.375,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":13.385999999999999,"5min":13.385999999999999,"15min":13.385999999999999},"last":3.3900000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.6950000000000001,"15min":1.8169999999999999},"min":{"1min":0.42799999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.5059999999999998,"5min":15.205,"15min":15.205},"last":1.571},{"interface":"front","average":{"1min":0.80700000000000005,"5min":2.04,"15min":2.456},"min":{"1min":0.41999999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.4849999999999999,"5min":11.923999999999999,"15min":11.923999999999999},"last":2.5950000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":1.944,"15min":2.21},"min":{"1min":0.376,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.4990000000000001,"5min":14.792999999999999,"15min":14.792999999999999},"last":2.452},{"interface":"front","average":{"1min":0.85899999999999999,"5min":2.3460000000000001,"15min":2.8010000000000002},"min":{"1min":0.39000000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":3.7789999999999999,"5min":15.045,"15min":15.045},"last":2.992}]}]},{"osd":3,"up_from":148,"seq":635655159851,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2911840,"kb_used_data":2649176,"kb_used_omap":2,"kb_used_meta":262653,"kb_avail":90833312,"statfs":{"total":95995035648,"available":93013311488,"internally_reserved":0,"allocated":2712756224,"data_stored":2712124323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":268956725},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:14:01 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.0289999999999999,"15min":1.27},"min":{"1min":0.27000000000000002,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":2.3650000000000002,"5min":5.9660000000000002,"15min":5.9660000000000002},"last":6.2569999999999997},{"interface":"front","average":{"1min":0.69699999999999995,"5min":1.5149999999999999,"15min":1.962},"min":{"1min":0.189,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":4.2460000000000004,"5min":9.5839999999999996,"15min":9.5839999999999996},"last":0.35899999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:14:01 2023","interfaces":[{"interface":"back","average":{"1min":4.8789999999999996,"5min":2.4060000000000001,"15min":2.266},"min":{"1min":0.26600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":70.066000000000003,"5min":70.066000000000003,"15min":70.066000000000003},"last":1.5449999999999999},{"interface":"front","average":{"1min":4.9130000000000003,"5min":2.2269999999999999,"15min":2.0430000000000001},"min":{"1min":0.20799999999999999,"5min":0.185,"15min":0.185},"max":{"1min":69.971000000000004,"5min":69.971000000000004,"15min":69.971000000000004},"last":6.1619999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:14:01 2023","interfaces":[{"interface":"back","average":{"1min":2.274,"5min":2.048,"15min":2.3220000000000001},"min":{"1min":0.30399999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":19.989999999999998,"5min":19.989999999999998,"15min":19.989999999999998},"last":1.496},{"interface":"front","average":{"1min":2.3999999999999999,"5min":2.21,"15min":2.5720000000000001},"min":{"1min":0.27200000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":20.125,"5min":20.125,"15min":20.125},"last":6.96}]},{"osd":4,"last update":"Sat Sep 23 23:14:01 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":1.347,"15min":1.585},"min":{"1min":0.34000000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.4900000000000002,"5min":7.1040000000000001,"15min":7.1040000000000001},"last":6.2329999999999997},{"interface":"front","average":{"1min":0.70599999999999996,"5min":2.2930000000000001,"15min":2.9550000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":2.1539999999999999,"5min":13.532999999999999,"15min":13.532999999999999},"last":0.44}]},{"osd":5,"last update":"Sat Sep 23 23:14:01 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":1.944,"15min":2.5230000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.121,"5min":10.117000000000001,"15min":10.117000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.734,"15min":2.0350000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":5.9800000000000004,"5min":7.3250000000000002,"15min":7.3250000000000002},"last":1.427}]},{"osd":6,"last update":"Sat Sep 23 23:14:01 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":2.0539999999999998,"15min":2.6520000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.1949999999999998,"5min":14.492000000000001,"15min":14.492000000000001},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.2729999999999999,"5min":1.893,"15min":2.1909999999999998},"min":{"1min":0.44500000000000001,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":6.0149999999999997,"5min":7.9640000000000004,"15min":7.9640000000000004},"last":1.466}]},{"osd":7,"last update":"Sat Sep 23 23:14:01 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.8220000000000001,"15min":2.3090000000000002},"min":{"1min":0.42299999999999999,"5min":0.377,"15min":0.377},"max":{"1min":2.427,"5min":9.7059999999999995,"15min":9.7059999999999995},"last":6.2030000000000003},{"interface":"front","average":{"1min":0.77500000000000002,"5min":1.9119999999999999,"15min":2.4399999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":2.2210000000000001,"5min":9.8010000000000002,"15min":9.8010000000000002},"last":2.8279999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019168,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1178052,"kb_used_data":850236,"kb_used_omap":0,"kb_used_meta":327808,"kb_avail":92567100,"statfs":{"total":95995035648,"available":94788710400,"internally_reserved":0,"allocated":870641664,"data_stored":870087684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":335675392},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:13:24 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":3.8159999999999998,"15min":2.4649999999999999},"min":{"1min":0.315,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.83699999999999997,"5min":238.62200000000001,"15min":238.62200000000001},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":4.75,"15min":2.9729999999999999},"min":{"1min":0.29299999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.72999999999999998,"5min":238.58199999999999,"15min":238.58199999999999},"last":0.28699999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:13:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":4.5289999999999999,"15min":2.3279999999999998},"min":{"1min":0.30099999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.6359999999999999,"5min":175.00899999999999,"15min":175.00899999999999},"last":4.8730000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":4.8410000000000002,"15min":2.3130000000000002},"min":{"1min":0.308,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.6799999999999999,"5min":174.821,"15min":174.821},"last":4.8869999999999996}]},{"osd":2,"last update":"Sat Sep 23 23:13:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.502,"15min":1.8260000000000001},"min":{"1min":0.313,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.343,"5min":11.664999999999999,"15min":11.664999999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.72999999999999998,"5min":1.9390000000000001,"15min":2.5830000000000002},"min":{"1min":0.47699999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.3079999999999998,"5min":10.576000000000001,"15min":10.576000000000001},"last":0.39700000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:14:03 2023","interfaces":[{"interface":"back","average":{"1min":4.2789999999999999,"5min":2.0670000000000002,"15min":1.907},"min":{"1min":0.34599999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":64.119,"5min":64.119,"15min":64.119},"last":0.38200000000000001},{"interface":"front","average":{"1min":4.4870000000000001,"5min":2.4950000000000001,"15min":2.4049999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":63.896000000000001,"5min":63.896000000000001,"15min":63.896000000000001},"last":0.36499999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:13:41 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.1319999999999999,"15min":1.0009999999999999},"min":{"1min":0.375,"5min":0.25,"15min":0.23999999999999999},"max":{"1min":0.79800000000000004,"5min":10.16,"15min":27.391999999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.71799999999999997,"5min":1.5249999999999999,"15min":1.3420000000000001},"min":{"1min":0.441,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":1.8440000000000001,"5min":8.2680000000000007,"15min":27.222999999999999},"last":0.42899999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:14:07 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.248,"15min":1.089},"min":{"1min":0.31,"5min":0.27300000000000002,"15min":0.253},"max":{"1min":5.5750000000000002,"5min":9.3350000000000009,"15min":14.4},"last":0.498},{"interface":"front","average":{"1min":1.081,"5min":1.288,"15min":1.0580000000000001},"min":{"1min":0.39200000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"max":{"1min":6.056,"5min":13.888999999999999,"15min":13.888999999999999},"last":0.48999999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:13:29 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.5580000000000001,"15min":1.611},"min":{"1min":0.35999999999999999,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":0.86099999999999999,"5min":13.792,"15min":119.188},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.61599999999999999,"5min":1.54,"15min":1.6930000000000001},"min":{"1min":0.442,"5min":0.35399999999999998,"15min":0.34200000000000003},"max":{"1min":1.0509999999999999,"5min":13.055,"15min":119.248},"last":0.53800000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757531,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2343468,"kb_used_data":1956644,"kb_used_omap":0,"kb_used_meta":386816,"kb_avail":91401684,"statfs":{"total":95995035648,"available":93595324416,"internally_reserved":0,"allocated":2003603456,"data_stored":2003069461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":396099584},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:13:28 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":1.3340000000000001,"15min":5.944},"min":{"1min":0.34799999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.72199999999999998,"5min":10.227,"15min":190.483},"last":0.37},{"interface":"front","average":{"1min":0.499,"5min":1.9570000000000001,"15min":6.702},"min":{"1min":0.36499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.69299999999999995,"5min":12.762,"15min":194.11099999999999},"last":6.8070000000000004}]},{"osd":1,"last update":"Sat Sep 23 23:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":2.7770000000000001,"15min":1.6359999999999999},"min":{"1min":0.372,"5min":0.246,"15min":0.246},"max":{"1min":1.413,"5min":91.295000000000002,"15min":91.295000000000002},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.54000000000000004,"5min":3.0779999999999998,"15min":1.7629999999999999},"min":{"1min":0.38600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.82299999999999995,"5min":91.242000000000004,"15min":91.242000000000004},"last":0.34499999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":2.0720000000000001,"15min":2.6539999999999999},"min":{"1min":0.32900000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.1539999999999999,"5min":9.8279999999999994,"15min":9.8279999999999994},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.71099999999999997,"5min":1.427,"15min":1.8660000000000001},"min":{"1min":0.38100000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":2.0299999999999998,"5min":8.093,"15min":8.093},"last":0.44400000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:14:00 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.405,"15min":1.708},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.96299999999999997,"5min":6.2640000000000002,"15min":6.2640000000000002},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.747,"5min":2.2149999999999999,"15min":2.8109999999999999},"min":{"1min":0.40500000000000003,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.4279999999999999,"5min":12.802,"15min":12.802},"last":0.54400000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.625,"15min":1.153},"min":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.221},"max":{"1min":1.343,"5min":14.538,"15min":14.538},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.54000000000000004,"5min":1.0609999999999999,"15min":0.99199999999999999},"min":{"1min":0.36399999999999999,"5min":0.28100000000000003,"15min":0.25},"max":{"1min":0.78800000000000003,"5min":11.779,"15min":11.779},"last":0.39700000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.468,"15min":1.2370000000000001},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.255},"max":{"1min":0.76900000000000002,"5min":13.226000000000001,"15min":13.226000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.63700000000000001,"5min":1.76,"15min":1.222},"min":{"1min":0.41699999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.514,"5min":13.098000000000001,"15min":13.098000000000001},"last":0.38300000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:13:34 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.829,"15min":1.512},"min":{"1min":0.47299999999999998,"5min":0.33900000000000002,"15min":0.29199999999999998},"max":{"1min":0.86599999999999999,"5min":14.929,"15min":14.929},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.677,"15min":1.4119999999999999},"min":{"1min":0.45800000000000002,"5min":0.33400000000000002,"15min":0.29599999999999999},"max":{"1min":0.85399999999999998,"5min":15.090999999999999,"15min":15.090999999999999},"last":0.45300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1173291008,"data_stored":1173291008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2570911744,"data_stored":2570911744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":919994368,"data_stored":919994368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2724921344,"data_stored":2724921344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":877002752,"data_stored":877002752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2002059264,"data_stored":2002059264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":852426752,"data_stored":852426752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2866610176,"data_stored":2866610176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:14:26.823 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T23:14:26.823 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 11 2023-09-23T23:14:26.891 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:26.911 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:14:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:14:26] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-09-23T23:14:26.931 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:27.001 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:27.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:27.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:27.047 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:27.110 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:27.155 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:27.236 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:27.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:27.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:27.346 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:27.403 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:27.495 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:27.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:27.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:27.534 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 52 16 78707 78691 94.5401 67 0.0109861 0.0105723 2023-09-23T23:14:27.611 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:14:27.671 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:27.763 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:14:27.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:27.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:14:27.874 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:27.931 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:28.121 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:28.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:28.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:28.225 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:14:28.267 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:28.333 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:14:28.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:28.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:14:28.435 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:14:28.439 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:14:28.472 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:14:28.490 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:28.509 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:14:28.534 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 53 15 79768 79753 94.0087 66.375 0.0159423 0.0106326 2023-09-23T23:14:28.543 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:14:28.571 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:14:28.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:28.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:14:28.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:28 smithi138 ceph-mon[193606]: pgmap v571: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T23:14:28.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:28.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:14:28.578 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:14:28.609 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:28 smithi110 ceph-mon[212939]: pgmap v571: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T23:14:28.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:28.610 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:14:28.610 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:28 smithi110 ceph-mon[221102]: pgmap v571: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T23:14:28.610 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:28.610 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:14:28.622 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:14:28.668 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:14:28.677 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:14:28.712 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:14:28.713 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:14:28.713 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:14:28.713 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:14:28.713 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:14:28.713 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:14:28.714 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:14:28.714 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:14:28.714 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:14:28.759 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:29.079 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:14:29.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:29.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:14:29.186 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:14:29.230 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:29.311 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:14:29.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:29.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:14:29.420 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:14:29.426 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:14:29.492 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:29.535 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 54 16 80802 80786 93.4635 64.5625 0.0109619 0.0106955 2023-09-23T23:14:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:29 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2312098368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:29 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4165276902' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:29 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:29.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:29 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2312098368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:29.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:29 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4165276902' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:29.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:29 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:29.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:29 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2312098368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:29.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:29 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4165276902' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:29.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:29 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:30.402 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:30 smithi110 ceph-mon[212939]: pgmap v572: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T23:14:30.535 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 55 16 81907 81891 93.0199 69.0625 0.00754904 0.0107457 2023-09-23T23:14:30.611 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:14:30.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:30.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:14:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:30 smithi138 ceph-mon[193606]: pgmap v572: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T23:14:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:30 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:14:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:30 smithi138 ceph-mon[193606]: failed to encode map e174 with expected crc 2023-09-23T23:14:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:30 smithi138 ceph-mon[193606]: failed to encode map e174 with expected crc 2023-09-23T23:14:30.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:30 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:14:30.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:30 smithi138 ceph-mon[193606]: osdmap e174: 8 total, 8 up, 8 in 2023-09-23T23:14:30.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:30 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:14:30.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:30 smithi110 ceph-mon[212939]: failed to encode map e174 with expected crc 2023-09-23T23:14:30.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:30 smithi110 ceph-mon[212939]: failed to encode map e174 with expected crc 2023-09-23T23:14:30.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:30 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:14:30.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:30 smithi110 ceph-mon[212939]: osdmap e174: 8 total, 8 up, 8 in 2023-09-23T23:14:30.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:30 smithi110 ceph-mon[221102]: pgmap v572: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T23:14:30.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:30 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:14:30.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:30 smithi110 ceph-mon[221102]: failed to encode map e174 with expected crc 2023-09-23T23:14:30.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:30 smithi110 ceph-mon[221102]: failed to encode map e174 with expected crc 2023-09-23T23:14:30.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:30 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:14:30.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:30 smithi110 ceph-mon[221102]: osdmap e174: 8 total, 8 up, 8 in 2023-09-23T23:14:30.724 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:30.749 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:30.796 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:30.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:30.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:30.901 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:14:30.946 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:30.958 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:31.014 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:14:31.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:31.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:14:31.119 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:31.198 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:31.535 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 56 16 82859 82843 92.4214 59.5 0.00671205 0.0108162 2023-09-23T23:14:31.545 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:31.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:31.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:31.658 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:14:31.734 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:32.502 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:14:32.503 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:14:32.503 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:14:32.503 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:14:32.503 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:14:32.536 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 57 16 83826 83810 91.8583 60.4375 0.00951366 0.0108821 2023-09-23T23:14:32.546 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:14:32.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:32.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:14:32.650 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:14:32.673 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:32 smithi138 ceph-mon[193606]: pgmap v574: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2023-09-23T23:14:32.676 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:32.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:32 smithi110 ceph-mon[212939]: pgmap v574: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2023-09-23T23:14:32.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:32 smithi110 ceph-mon[221102]: pgmap v574: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2023-09-23T23:14:32.726 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:14:32.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:32.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:14:32.834 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:14:32.912 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:32.999 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:14:33.139 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:14:33.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:33.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:14:33.244 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:33.286 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:33.338 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:14:33.373 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:33.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:33.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:33.488 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:14:33.533 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:33.538 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 58 16 84706 84690 91.2192 55 0.00859867 0.0109581 2023-09-23T23:14:33.607 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:14:33.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:33.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:14:33.717 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:14:33.746 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:14:33.754 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:33.814 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:14:33.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:33.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:14:33.921 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:33.971 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:34.025 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:34.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:34.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:34.133 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:14:34.195 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:34.357 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:34.393 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:14:34.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:34.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:14:34.398 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:34 smithi110 ceph-mon[221102]: pgmap v575: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T23:14:34.402 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:34 smithi110 ceph-mon[212939]: pgmap v575: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T23:14:34.418 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:34.498 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:34.545 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 59 16 85559 85543 90.5768 53.3125 0.0114714 0.0110347 2023-09-23T23:14:34.575 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:34.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:34 smithi138 ceph-mon[193606]: pgmap v575: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T23:14:35.262 INFO:teuthology.orchestra.run.smithi110.stderr:set pool 1 pg_num to 11 2023-09-23T23:14:35.291 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:35 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2701184762' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T23:14:35.291 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:35 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T23:14:35.292 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:35 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2701184762' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T23:14:35.293 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:35 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T23:14:35.479 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:35.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:35.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:35.539 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:14:35.538888+0000 min lat: 0.00117478 max lat: 0.336316 avg lat: 0.0110933 2023-09-23T23:14:35.539 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:14:35.539 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 60 16 86551 86535 90.1006 62 0.013966 0.0110933 2023-09-23T23:14:35.584 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:14:35.635 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:35 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2701184762' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T23:14:35.635 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:35 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T23:14:35.637 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:35.720 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:14:35.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:35.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:14:35.826 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:35.856 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:35.897 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:35.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:35.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:36.002 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:14:36.077 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:36.246 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:36.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:36 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T23:14:36.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:36 smithi110 ceph-mon[212939]: osdmap e175: 8 total, 8 up, 8 in 2023-09-23T23:14:36.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:36 smithi110 ceph-mon[212939]: failed to encode map e175 with expected crc 2023-09-23T23:14:36.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:36 smithi110 ceph-mon[212939]: failed to encode map e175 with expected crc 2023-09-23T23:14:36.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:36 smithi110 ceph-mon[212939]: pgmap v577: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T23:14:36.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:36 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T23:14:36.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:36 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T23:14:36.427 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:36 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T23:14:36.428 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:36 smithi110 ceph-mon[221102]: osdmap e175: 8 total, 8 up, 8 in 2023-09-23T23:14:36.428 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:36 smithi110 ceph-mon[221102]: failed to encode map e175 with expected crc 2023-09-23T23:14:36.428 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:36 smithi110 ceph-mon[221102]: failed to encode map e175 with expected crc 2023-09-23T23:14:36.428 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:36 smithi110 ceph-mon[221102]: pgmap v577: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T23:14:36.428 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:36 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T23:14:36.428 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:36 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T23:14:36.539 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 61 16 87435 87419 89.5293 55.25 0.01873 0.0111578 2023-09-23T23:14:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:36 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T23:14:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:36 smithi138 ceph-mon[193606]: osdmap e175: 8 total, 8 up, 8 in 2023-09-23T23:14:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:36 smithi138 ceph-mon[193606]: failed to encode map e175 with expected crc 2023-09-23T23:14:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:36 smithi138 ceph-mon[193606]: failed to encode map e175 with expected crc 2023-09-23T23:14:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:36 smithi138 ceph-mon[193606]: pgmap v577: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T23:14:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:36 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T23:14:36.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:36 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T23:14:37.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:14:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:14:36] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.43.0" 2023-09-23T23:14:37.539 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 62 16 88336 88320 88.9936 56.3125 0.0103676 0.0112327 2023-09-23T23:14:37.622 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:14:37.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:37.628 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:37 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T23:14:37.628 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:37 smithi110 ceph-mon[212939]: osdmap e176: 8 total, 8 up, 8 in 2023-09-23T23:14:37.628 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:37 smithi110 ceph-mon[212939]: failed to encode map e176 with expected crc 2023-09-23T23:14:37.628 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:37 smithi110 ceph-mon[212939]: failed to encode map e176 with expected crc 2023-09-23T23:14:37.629 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:37 smithi110 ceph-mon[212939]: failed to encode map e176 with expected crc 2023-09-23T23:14:37.629 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:37 smithi110 ceph-mon[212939]: failed to encode map e176 with expected crc 2023-09-23T23:14:37.629 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:37 smithi110 ceph-mon[212939]: failed to encode map e176 with expected crc 2023-09-23T23:14:37.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:37 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T23:14:37.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:37 smithi110 ceph-mon[221102]: osdmap e176: 8 total, 8 up, 8 in 2023-09-23T23:14:37.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:37 smithi110 ceph-mon[221102]: failed to encode map e176 with expected crc 2023-09-23T23:14:37.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:37 smithi110 ceph-mon[221102]: failed to encode map e176 with expected crc 2023-09-23T23:14:37.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:37 smithi110 ceph-mon[221102]: failed to encode map e176 with expected crc 2023-09-23T23:14:37.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:37 smithi110 ceph-mon[221102]: failed to encode map e176 with expected crc 2023-09-23T23:14:37.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:37 smithi110 ceph-mon[221102]: failed to encode map e176 with expected crc 2023-09-23T23:14:37.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:14:37.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:37 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T23:14:37.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:37 smithi138 ceph-mon[193606]: osdmap e176: 8 total, 8 up, 8 in 2023-09-23T23:14:37.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:37 smithi138 ceph-mon[193606]: failed to encode map e176 with expected crc 2023-09-23T23:14:37.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:37 smithi138 ceph-mon[193606]: failed to encode map e176 with expected crc 2023-09-23T23:14:37.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:37 smithi138 ceph-mon[193606]: failed to encode map e176 with expected crc 2023-09-23T23:14:37.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:37 smithi138 ceph-mon[193606]: failed to encode map e176 with expected crc 2023-09-23T23:14:37.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:37 smithi138 ceph-mon[193606]: failed to encode map e176 with expected crc 2023-09-23T23:14:37.730 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:37.808 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:38.072 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:38.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:38.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:38.181 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:38.258 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:38.349 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:38.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:38.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:38.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:38 smithi110 ceph-mon[212939]: failed to encode map e177 with expected crc 2023-09-23T23:14:38.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:38 smithi110 ceph-mon[212939]: osdmap e177: 8 total, 8 up, 8 in 2023-09-23T23:14:38.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:38 smithi110 ceph-mon[212939]: failed to encode map e177 with expected crc 2023-09-23T23:14:38.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:38 smithi110 ceph-mon[212939]: failed to encode map e177 with expected crc 2023-09-23T23:14:38.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:38 smithi110 ceph-mon[212939]: failed to encode map e177 with expected crc 2023-09-23T23:14:38.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:38 smithi110 ceph-mon[212939]: failed to encode map e177 with expected crc 2023-09-23T23:14:38.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:38 smithi110 ceph-mon[212939]: pgmap v580: 99 pgs: 10 unknown, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s 2023-09-23T23:14:38.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:38 smithi110 ceph-mon[221102]: failed to encode map e177 with expected crc 2023-09-23T23:14:38.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:38 smithi110 ceph-mon[221102]: osdmap e177: 8 total, 8 up, 8 in 2023-09-23T23:14:38.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:38 smithi110 ceph-mon[221102]: failed to encode map e177 with expected crc 2023-09-23T23:14:38.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:38 smithi110 ceph-mon[221102]: failed to encode map e177 with expected crc 2023-09-23T23:14:38.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:38 smithi110 ceph-mon[221102]: failed to encode map e177 with expected crc 2023-09-23T23:14:38.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:38 smithi110 ceph-mon[221102]: failed to encode map e177 with expected crc 2023-09-23T23:14:38.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:38 smithi110 ceph-mon[221102]: pgmap v580: 99 pgs: 10 unknown, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s 2023-09-23T23:14:38.458 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:38.509 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:38 smithi138 ceph-mon[193606]: failed to encode map e177 with expected crc 2023-09-23T23:14:38.509 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:38 smithi138 ceph-mon[193606]: osdmap e177: 8 total, 8 up, 8 in 2023-09-23T23:14:38.509 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:38 smithi138 ceph-mon[193606]: failed to encode map e177 with expected crc 2023-09-23T23:14:38.510 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:38 smithi138 ceph-mon[193606]: failed to encode map e177 with expected crc 2023-09-23T23:14:38.510 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:38 smithi138 ceph-mon[193606]: failed to encode map e177 with expected crc 2023-09-23T23:14:38.510 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:38 smithi138 ceph-mon[193606]: failed to encode map e177 with expected crc 2023-09-23T23:14:38.510 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:38 smithi138 ceph-mon[193606]: pgmap v580: 99 pgs: 10 unknown, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s 2023-09-23T23:14:38.511 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:38.539 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 63 16 89327 89311 88.5642 61.9375 0.00801297 0.0112868 2023-09-23T23:14:38.599 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:38.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:38.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:38.709 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:14:38.778 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:14:38.780 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:38.815 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:14:38.853 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:14:38.876 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:14:38.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:38.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:14:38.883 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:14:38.908 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:14:38.931 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:14:38.975 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:14:38.981 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:14:39.025 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:39.025 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:14:39.026 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:14:39.026 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:14:39.026 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:14:39.026 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:14:39.026 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:14:39.026 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:14:39.026 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:14:39.026 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:14:39.094 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:14:39.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:39.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:14:39.206 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:14:39.257 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:39.345 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:14:39.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:39.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:14:39.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:39 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3527453333' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:39.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:39 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3527453333' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:39.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:39 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3527453333' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:39.456 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:39.499 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:39.539 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 64 16 90311 90295 88.1414 61.5 0.00400612 0.0113391 2023-09-23T23:14:39.563 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:39.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:39.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:39.671 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:14:39.696 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:39.737 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:14:39.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:39.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:14:39.844 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:39.909 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:40.400 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:40.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:40.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:40.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:40 smithi110 ceph-mon[221102]: pgmap v581: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-09-23T23:14:40.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:40 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T23:14:40.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:40 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T23:14:40.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:40 smithi110 ceph-mon[212939]: pgmap v581: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-09-23T23:14:40.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:40 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T23:14:40.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:40 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T23:14:40.508 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:40.543 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 65 16 91202 91186 87.6366 55.6875 0.0146182 0.011406 2023-09-23T23:14:40.549 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:40 smithi138 ceph-mon[193606]: pgmap v581: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-09-23T23:14:40.549 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:40 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T23:14:40.549 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:40 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T23:14:40.553 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:40.614 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:40.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:40.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:40.723 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:40.760 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:14:40.760 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:14:40.770 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:40.855 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:40.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:40.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:40.963 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:41.015 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:41.117 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:41.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:41.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:41.225 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:41.250 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:41.286 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:14:41.298 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:41.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:41.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:41.404 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:14:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:41 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T23:14:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:41 smithi138 ceph-mon[193606]: osdmap e178: 8 total, 8 up, 8 in 2023-09-23T23:14:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:41 smithi138 ceph-mon[193606]: failed to encode map e178 with expected crc 2023-09-23T23:14:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:41 smithi138 ceph-mon[193606]: failed to encode map e178 with expected crc 2023-09-23T23:14:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:41 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1264972425' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:14:41.444 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:41.534 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:14:41.539 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:41 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T23:14:41.539 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:41 smithi110 ceph-mon[212939]: osdmap e178: 8 total, 8 up, 8 in 2023-09-23T23:14:41.539 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:41 smithi110 ceph-mon[212939]: failed to encode map e178 with expected crc 2023-09-23T23:14:41.539 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:41 smithi110 ceph-mon[212939]: failed to encode map e178 with expected crc 2023-09-23T23:14:41.539 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:41 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1264972425' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:14:41.540 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:41 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T23:14:41.540 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:41 smithi110 ceph-mon[221102]: osdmap e178: 8 total, 8 up, 8 in 2023-09-23T23:14:41.540 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:41 smithi110 ceph-mon[221102]: failed to encode map e178 with expected crc 2023-09-23T23:14:41.540 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:41 smithi110 ceph-mon[221102]: failed to encode map e178 with expected crc 2023-09-23T23:14:41.540 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:41 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1264972425' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:14:41.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:41.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:14:41.544 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 66 16 91946 91930 87.0124 46.5 0.00510116 0.011488 2023-09-23T23:14:41.642 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:41.700 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:41.848 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:41.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:41.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:41.954 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:14:42.060 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:42.259 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:14:42.418 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:14:42.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:42.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:14:42.526 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:14:42.527 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:14:42.543 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:14:42.544 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[212939]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.544 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1264972425' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:14:42.544 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[212939]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.544 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[212939]: osdmap e179: 8 total, 8 up, 8 in 2023-09-23T23:14:42.544 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[212939]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.544 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[212939]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.544 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[212939]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.544 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[212939]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.544 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[212939]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.544 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[212939]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.544 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[212939]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.545 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[212939]: pgmap v584: 99 pgs: 99 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-09-23T23:14:42.545 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:14:42.545 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[221102]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.545 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1264972425' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:14:42.545 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[221102]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.545 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[221102]: osdmap e179: 8 total, 8 up, 8 in 2023-09-23T23:14:42.545 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[221102]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.545 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[221102]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.545 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[221102]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.545 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[221102]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.546 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[221102]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.546 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[221102]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.546 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[221102]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.546 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:42 smithi110 ceph-mon[221102]: pgmap v584: 99 pgs: 99 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-09-23T23:14:42.546 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 67 16 92798 92782 86.5086 53.25 0.0156681 0.0115543 2023-09-23T23:14:42.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:42 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:14:42.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:42 smithi138 ceph-mon[193606]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:42 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1264972425' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:14:42.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:42 smithi138 ceph-mon[193606]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:42 smithi138 ceph-mon[193606]: osdmap e179: 8 total, 8 up, 8 in 2023-09-23T23:14:42.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:42 smithi138 ceph-mon[193606]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:42 smithi138 ceph-mon[193606]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:42 smithi138 ceph-mon[193606]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:42 smithi138 ceph-mon[193606]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:42 smithi138 ceph-mon[193606]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:42 smithi138 ceph-mon[193606]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:42 smithi138 ceph-mon[193606]: failed to encode map e179 with expected crc 2023-09-23T23:14:42.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:42 smithi138 ceph-mon[193606]: pgmap v584: 99 pgs: 99 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-09-23T23:14:42.567 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:42.624 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:14:42.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:42.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:14:42.734 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:42.778 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:42.843 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T23:14:42.843 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:14:42.844 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T23:14:42.844 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T23:14:42.844 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T23:14:42.844 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T23:14:42.856 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:42.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:42.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:42.965 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:14:42.998 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:43.011 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:43.061 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:14:43.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:43.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:14:43.168 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:43.242 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:43.427 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:43.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:43.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:43.534 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:14:43.545 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 68 16 93642 93626 86.011 52.75 0.00939103 0.0116215 2023-09-23T23:14:43.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:43 smithi110 ceph-mon[212939]: failed to encode map e180 with expected crc 2023-09-23T23:14:43.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:43 smithi110 ceph-mon[212939]: osdmap e180: 8 total, 8 up, 8 in 2023-09-23T23:14:43.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:43 smithi110 ceph-mon[212939]: failed to encode map e180 with expected crc 2023-09-23T23:14:43.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:43 smithi110 ceph-mon[212939]: failed to encode map e180 with expected crc 2023-09-23T23:14:43.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:43 smithi110 ceph-mon[212939]: failed to encode map e180 with expected crc 2023-09-23T23:14:43.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:14:43.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:43 smithi110 ceph-mon[212939]: osdmap e181: 8 total, 8 up, 8 in 2023-09-23T23:14:43.588 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:43 smithi110 ceph-mon[221102]: failed to encode map e180 with expected crc 2023-09-23T23:14:43.588 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:43 smithi110 ceph-mon[221102]: osdmap e180: 8 total, 8 up, 8 in 2023-09-23T23:14:43.588 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:43 smithi110 ceph-mon[221102]: failed to encode map e180 with expected crc 2023-09-23T23:14:43.588 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:43 smithi110 ceph-mon[221102]: failed to encode map e180 with expected crc 2023-09-23T23:14:43.588 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:43 smithi110 ceph-mon[221102]: failed to encode map e180 with expected crc 2023-09-23T23:14:43.588 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:14:43.588 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:43 smithi110 ceph-mon[221102]: osdmap e181: 8 total, 8 up, 8 in 2023-09-23T23:14:43.632 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:43.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:43 smithi138 ceph-mon[193606]: failed to encode map e180 with expected crc 2023-09-23T23:14:43.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:43 smithi138 ceph-mon[193606]: osdmap e180: 8 total, 8 up, 8 in 2023-09-23T23:14:43.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:43 smithi138 ceph-mon[193606]: failed to encode map e180 with expected crc 2023-09-23T23:14:43.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:43 smithi138 ceph-mon[193606]: failed to encode map e180 with expected crc 2023-09-23T23:14:43.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:43 smithi138 ceph-mon[193606]: failed to encode map e180 with expected crc 2023-09-23T23:14:43.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:14:43.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:43 smithi138 ceph-mon[193606]: osdmap e181: 8 total, 8 up, 8 in 2023-09-23T23:14:43.751 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:14:43.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:43.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:14:43.861 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:43.929 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:44.041 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:44.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:44.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:44.145 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:44.189 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:44.256 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:44.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:44.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:44.361 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:14:44.387 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:44.429 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:14:44.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:44.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:14:44.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:44 smithi138 ceph-mon[193606]: 1.0 scrub starts 2023-09-23T23:14:44.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:44 smithi138 ceph-mon[193606]: 1.0 scrub ok 2023-09-23T23:14:44.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:44 smithi138 ceph-mon[193606]: failed to encode map e181 with expected crc 2023-09-23T23:14:44.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:44 smithi138 ceph-mon[193606]: failed to encode map e181 with expected crc 2023-09-23T23:14:44.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:44 smithi138 ceph-mon[193606]: failed to encode map e181 with expected crc 2023-09-23T23:14:44.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:44 smithi138 ceph-mon[193606]: failed to encode map e181 with expected crc 2023-09-23T23:14:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:44 smithi138 ceph-mon[193606]: pgmap v587: 99 pgs: 1 peering, 98 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31 MiB/s wr, 490 op/s; 64 KiB/s, 0 objects/s recovering 2023-09-23T23:14:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:44 smithi138 ceph-mon[193606]: osdmap e182: 8 total, 8 up, 8 in 2023-09-23T23:14:44.532 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:14:44.546 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 69 16 94408 94392 85.4582 47.875 0.0104606 0.0116979 2023-09-23T23:14:44.581 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:44.590 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:14:44.647 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:44 smithi110 ceph-mon[212939]: 1.0 scrub starts 2023-09-23T23:14:44.647 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:44 smithi110 ceph-mon[212939]: 1.0 scrub ok 2023-09-23T23:14:44.647 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:44 smithi110 ceph-mon[212939]: failed to encode map e181 with expected crc 2023-09-23T23:14:44.647 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:44 smithi110 ceph-mon[212939]: failed to encode map e181 with expected crc 2023-09-23T23:14:44.647 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:44 smithi110 ceph-mon[212939]: failed to encode map e181 with expected crc 2023-09-23T23:14:44.647 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:44 smithi110 ceph-mon[212939]: failed to encode map e181 with expected crc 2023-09-23T23:14:44.647 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:44 smithi110 ceph-mon[212939]: pgmap v587: 99 pgs: 1 peering, 98 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31 MiB/s wr, 490 op/s; 64 KiB/s, 0 objects/s recovering 2023-09-23T23:14:44.647 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:44 smithi110 ceph-mon[212939]: osdmap e182: 8 total, 8 up, 8 in 2023-09-23T23:14:44.648 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:44 smithi110 ceph-mon[221102]: 1.0 scrub starts 2023-09-23T23:14:44.648 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:44 smithi110 ceph-mon[221102]: 1.0 scrub ok 2023-09-23T23:14:44.648 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:44 smithi110 ceph-mon[221102]: failed to encode map e181 with expected crc 2023-09-23T23:14:44.648 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:44 smithi110 ceph-mon[221102]: failed to encode map e181 with expected crc 2023-09-23T23:14:44.648 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:44 smithi110 ceph-mon[221102]: failed to encode map e181 with expected crc 2023-09-23T23:14:44.648 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:44 smithi110 ceph-mon[221102]: failed to encode map e181 with expected crc 2023-09-23T23:14:44.648 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:44 smithi110 ceph-mon[221102]: pgmap v587: 99 pgs: 1 peering, 98 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31 MiB/s wr, 490 op/s; 64 KiB/s, 0 objects/s recovering 2023-09-23T23:14:44.648 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:44 smithi110 ceph-mon[221102]: osdmap e182: 8 total, 8 up, 8 in 2023-09-23T23:14:44.673 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:14:44.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:44.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:14:44.783 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:44.846 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:44.985 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:44.986 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:44.994 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:45.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:45.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:45.111 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:14:45.179 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:45.390 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:14:45.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:45.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:14:45.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[212939]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[212939]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[212939]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[212939]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[212939]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[212939]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[212939]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[212939]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[212939]: 1.3 scrub starts 2023-09-23T23:14:45.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[212939]: 1.3 scrub ok 2023-09-23T23:14:45.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[221102]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[221102]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[221102]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[221102]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[221102]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[221102]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[221102]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[221102]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[221102]: 1.3 scrub starts 2023-09-23T23:14:45.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:45 smithi110 ceph-mon[221102]: 1.3 scrub ok 2023-09-23T23:14:45.499 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:45.541 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:45.546 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 70 16 95301 95285 85.0348 55.8125 0.0132454 0.0117547 2023-09-23T23:14:45.625 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:45.628 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:45 smithi138 ceph-mon[193606]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.628 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:45 smithi138 ceph-mon[193606]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.629 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:45 smithi138 ceph-mon[193606]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.629 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:45 smithi138 ceph-mon[193606]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.629 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:45 smithi138 ceph-mon[193606]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.629 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:45 smithi138 ceph-mon[193606]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.629 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:45 smithi138 ceph-mon[193606]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.629 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:45 smithi138 ceph-mon[193606]: failed to encode map e182 with expected crc 2023-09-23T23:14:45.629 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:45 smithi138 ceph-mon[193606]: 1.3 scrub starts 2023-09-23T23:14:45.629 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:45 smithi138 ceph-mon[193606]: 1.3 scrub ok 2023-09-23T23:14:45.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:45.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:45.731 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:45.766 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:45.852 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:45.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:45.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:45.960 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:14:46.023 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:46.171 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:14:46.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:46.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:14:46.279 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:46.345 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:46.538 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:46.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:46.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:46.546 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 71 16 96228 96212 84.6532 57.9375 0.00295789 0.0118074 2023-09-23T23:14:46.643 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:46.689 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:46 smithi110 ceph-mon[212939]: 1.2 scrub starts 2023-09-23T23:14:46.689 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:46 smithi110 ceph-mon[212939]: 1.2 scrub ok 2023-09-23T23:14:46.689 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:46 smithi110 ceph-mon[212939]: pgmap v589: 99 pgs: 8 peering, 91 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 871 op/s; 25 KiB/s, 0 objects/s recovering 2023-09-23T23:14:46.689 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:46 smithi110 ceph-mon[212939]: 1.4 scrub starts 2023-09-23T23:14:46.689 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:46 smithi110 ceph-mon[212939]: 1.4 scrub ok 2023-09-23T23:14:46.690 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:46 smithi110 ceph-mon[221102]: 1.2 scrub starts 2023-09-23T23:14:46.690 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:46 smithi110 ceph-mon[221102]: 1.2 scrub ok 2023-09-23T23:14:46.690 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:46 smithi110 ceph-mon[221102]: pgmap v589: 99 pgs: 8 peering, 91 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 871 op/s; 25 KiB/s, 0 objects/s recovering 2023-09-23T23:14:46.690 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:46 smithi110 ceph-mon[221102]: 1.4 scrub starts 2023-09-23T23:14:46.690 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:46 smithi110 ceph-mon[221102]: 1.4 scrub ok 2023-09-23T23:14:46.693 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:46.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:46 smithi138 ceph-mon[193606]: 1.2 scrub starts 2023-09-23T23:14:46.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:46 smithi138 ceph-mon[193606]: 1.2 scrub ok 2023-09-23T23:14:46.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:46 smithi138 ceph-mon[193606]: pgmap v589: 99 pgs: 8 peering, 91 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 871 op/s; 25 KiB/s, 0 objects/s recovering 2023-09-23T23:14:46.697 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:46 smithi138 ceph-mon[193606]: 1.4 scrub starts 2023-09-23T23:14:46.697 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:46 smithi138 ceph-mon[193606]: 1.4 scrub ok 2023-09-23T23:14:46.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:14:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:14:46] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2023-09-23T23:14:47.300 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:47.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:47.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:47.387 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:47.408 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:14:47.484 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:47.547 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 72 16 96845 96829 84.0133 38.5625 0.0223032 0.0118981 2023-09-23T23:14:47.575 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:14:47.636 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:14:47.652 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:14:47.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:47.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:14:47.658 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:47 smithi110 ceph-mon[212939]: 1.6 scrub starts 2023-09-23T23:14:47.658 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:47 smithi110 ceph-mon[212939]: 1.6 scrub ok 2023-09-23T23:14:47.658 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:47 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:14:47.658 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:47 smithi110 ceph-mon[212939]: 1.9 scrub ok 2023-09-23T23:14:47.658 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:47 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:14:47.658 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:47 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:14:47.658 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:47 smithi110 ceph-mon[212939]: 1.8 scrub ok 2023-09-23T23:14:47.659 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:47 smithi110 ceph-mon[221102]: 1.6 scrub starts 2023-09-23T23:14:47.659 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:47 smithi110 ceph-mon[221102]: 1.6 scrub ok 2023-09-23T23:14:47.659 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:47 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:14:47.659 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:47 smithi110 ceph-mon[221102]: 1.9 scrub ok 2023-09-23T23:14:47.659 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:47 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:14:47.659 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:47 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:14:47.659 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:47 smithi110 ceph-mon[221102]: 1.8 scrub ok 2023-09-23T23:14:47.675 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:14:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:47 smithi138 ceph-mon[193606]: 1.6 scrub starts 2023-09-23T23:14:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:47 smithi138 ceph-mon[193606]: 1.6 scrub ok 2023-09-23T23:14:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:47 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:14:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:47 smithi138 ceph-mon[193606]: 1.9 scrub ok 2023-09-23T23:14:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:47 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:14:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:47 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:14:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:47 smithi138 ceph-mon[193606]: 1.8 scrub ok 2023-09-23T23:14:47.707 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:14:47.743 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:14:47.758 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:47.777 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:14:47.820 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:14:47.840 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:47.848 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:14:47.906 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:14:47.906 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:14:47.906 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:14:47.906 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:14:47.906 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:14:47.906 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:14:47.906 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:14:47.906 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:14:48.447 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:14:48.447 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:14:48.447 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.447 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45164.0:97572 9.4 e182/164 9:29e671f6:::benchmark_data_smithi110_295904_object97571:head v 182'6307, mlcod=182'6304)", 2023-09-23T23:14:48.447 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:14:48.433782+0000", 2023-09-23T23:14:48.447 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.008694093, 2023-09-23T23:14:48.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.008724697, 2023-09-23T23:14:48.447 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:48.448 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:14:48.448 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:48.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:48.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.433782+0000", 2023-09-23T23:14:48.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:48.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:48.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.433782+0000", 2023-09-23T23:14:48.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:48.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:48.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.433782+0000", 2023-09-23T23:14:48.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:14:48.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:48.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.433805+0000", 2023-09-23T23:14:48.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3195999999999999e-05 2023-09-23T23:14:48.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:48.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.433813+0000", 2023-09-23T23:14:48.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3459999999999994e-06 2023-09-23T23:14:48.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.450 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:48.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.433819+0000", 2023-09-23T23:14:48.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5570000000000003e-06 2023-09-23T23:14:48.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.450 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:48.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.433853+0000", 2023-09-23T23:14:48.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4473e-05 2023-09-23T23:14:48.451 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.451 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.451 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:48.451 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.433873+0000", 2023-09-23T23:14:48.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9386999999999999e-05 2023-09-23T23:14:48.451 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:48.451 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:48.451 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:48.451 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.451 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.451 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45164.0:97573 9.4 e182/164)", 2023-09-23T23:14:48.451 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:14:48.433832+0000", 2023-09-23T23:14:48.452 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0086441629999999998, 2023-09-23T23:14:48.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0087553370000000002, 2023-09-23T23:14:48.452 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:48.452 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:14:48.452 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:48.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:48.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.433832+0000", 2023-09-23T23:14:48.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:48.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:48.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.433832+0000", 2023-09-23T23:14:48.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:48.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:48.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.433832+0000", 2023-09-23T23:14:48.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:14:48.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:48.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.433853+0000", 2023-09-23T23:14:48.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1702999999999999e-05 2023-09-23T23:14:48.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:48.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.433861+0000", 2023-09-23T23:14:48.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2359999999999998e-06 2023-09-23T23:14:48.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:48.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.433865+0000", 2023-09-23T23:14:48.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7350000000000004e-06 2023-09-23T23:14:48.454 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:48.455 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:48.455 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:48.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.455 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:97579 9.c 9:3988033a:::benchmark_data_smithi110_295904_object97578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-23T23:14:48.455 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:14:48.434789+0000", 2023-09-23T23:14:48.455 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0076869540000000002, 2023-09-23T23:14:48.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007855038, 2023-09-23T23:14:48.455 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:48.455 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:14:48.455 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:48.456 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:14:48.456 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:14:48.456 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 97579 2023-09-23T23:14:48.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.456 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:48.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.456 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:48.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.434789+0000", 2023-09-23T23:14:48.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:48.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.456 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:48.457 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.434789+0000", 2023-09-23T23:14:48.457 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:48.457 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.457 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.457 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:48.457 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.434788+0000", 2023-09-23T23:14:48.457 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:14:48.457 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.457 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.457 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:48.457 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.434813+0000", 2023-09-23T23:14:48.457 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4689999999999999e-05 2023-09-23T23:14:48.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.458 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.458 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:48.458 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.434831+0000", 2023-09-23T23:14:48.458 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8797999999999999e-05 2023-09-23T23:14:48.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.458 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.458 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:48.458 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.434841+0000", 2023-09-23T23:14:48.458 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5999999999999996e-06 2023-09-23T23:14:48.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.459 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:48.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.436839+0000", 2023-09-23T23:14:48.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019976830000000001 2023-09-23T23:14:48.459 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.459 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:48.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.436956+0000", 2023-09-23T23:14:48.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117024 2023-09-23T23:14:48.459 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.459 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,7", 2023-09-23T23:14:48.460 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.437081+0000", 2023-09-23T23:14:48.460 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012544800000000001 2023-09-23T23:14:48.460 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:48.460 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:48.460 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:48.460 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.460 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.460 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:97581 9.c 9.8e2c277c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-23T23:14:48.460 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:14:48.435115+0000", 2023-09-23T23:14:48.460 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0073616200000000001, 2023-09-23T23:14:48.460 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076028830000000004, 2023-09-23T23:14:48.460 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:48.461 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:14:48.461 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:48.461 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:14:48.461 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:14:48.461 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 97581 2023-09-23T23:14:48.461 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.461 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:48.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:48.461 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435115+0000", 2023-09-23T23:14:48.461 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:48.461 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:48.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435115+0000", 2023-09-23T23:14:48.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:48.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:48.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435114+0000", 2023-09-23T23:14:48.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:14:48.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:48.463 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435136+0000", 2023-09-23T23:14:48.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2081000000000001e-05 2023-09-23T23:14:48.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.463 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:48.463 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435155+0000", 2023-09-23T23:14:48.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8910999999999999e-05 2023-09-23T23:14:48.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.463 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:48.463 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435162+0000", 2023-09-23T23:14:48.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.08e-06 2023-09-23T23:14:48.464 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:48.464 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:48.464 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:48.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.464 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:97587 9.2 9:4242015b:::benchmark_data_smithi110_295904_object97586:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-23T23:14:48.464 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:14:48.435168+0000", 2023-09-23T23:14:48.464 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0073085249999999997, 2023-09-23T23:14:48.464 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076181599999999997, 2023-09-23T23:14:48.464 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:48.464 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:14:48.464 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:48.465 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:14:48.465 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:14:48.465 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 97587 2023-09-23T23:14:48.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.465 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:48.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.465 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:48.465 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435168+0000", 2023-09-23T23:14:48.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:48.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.465 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:48.466 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435168+0000", 2023-09-23T23:14:48.466 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:48.466 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.466 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.466 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:48.466 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435167+0000", 2023-09-23T23:14:48.466 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:14:48.466 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.466 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.466 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:48.466 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435184+0000", 2023-09-23T23:14:48.466 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6878e-05 2023-09-23T23:14:48.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.467 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:48.467 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435190+0000", 2023-09-23T23:14:48.467 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0959999999999997e-06 2023-09-23T23:14:48.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.467 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:48.467 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435195+0000", 2023-09-23T23:14:48.467 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9219999999999996e-06 2023-09-23T23:14:48.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.468 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:48.468 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.436906+0000", 2023-09-23T23:14:48.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017113479999999999 2023-09-23T23:14:48.468 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.468 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:48.468 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.437027+0000", 2023-09-23T23:14:48.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000120955 2023-09-23T23:14:48.468 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.468 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:14:48.468 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.437157+0000", 2023-09-23T23:14:48.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012998799999999999 2023-09-23T23:14:48.469 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:48.469 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:48.469 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:48.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.469 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.469 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:97588 9.2 9:4d1d9f29:::benchmark_data_smithi110_295904_object97587:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-23T23:14:48.469 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:14:48.435208+0000", 2023-09-23T23:14:48.469 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0072682720000000001, 2023-09-23T23:14:48.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076526140000000003, 2023-09-23T23:14:48.469 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:48.469 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:14:48.470 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:14:48.470 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:14:48.470 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:14:48.470 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 97588 2023-09-23T23:14:48.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.470 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:48.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:48.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435208+0000", 2023-09-23T23:14:48.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:48.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:48.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435208+0000", 2023-09-23T23:14:48.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:48.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:48.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435207+0000", 2023-09-23T23:14:48.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:14:48.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:48.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435225+0000", 2023-09-23T23:14:48.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7938e-05 2023-09-23T23:14:48.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:48.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435232+0000", 2023-09-23T23:14:48.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1879999999999999e-06 2023-09-23T23:14:48.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:48.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435237+0000", 2023-09-23T23:14:48.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.967e-06 2023-09-23T23:14:48.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:48.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.440366+0000", 2023-09-23T23:14:48.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051297959999999998 2023-09-23T23:14:48.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:48.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.440447+0000", 2023-09-23T23:14:48.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0721000000000002e-05 2023-09-23T23:14:48.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:14:48.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.440529+0000", 2023-09-23T23:14:48.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1859000000000003e-05 2023-09-23T23:14:48.474 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:48.474 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:48.474 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:48.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.474 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45164.0:97586 9.3 e182/164 9:ca737582:::benchmark_data_smithi110_295904_object97585:head v 182'6106, mlcod=182'6104)", 2023-09-23T23:14:48.474 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:14:48.435702+0000", 2023-09-23T23:14:48.474 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0067739360000000004, 2023-09-23T23:14:48.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007252807, 2023-09-23T23:14:48.474 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:14:48.475 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:14:48.475 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:14:48.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:14:48.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435702+0000", 2023-09-23T23:14:48.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:48.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:14:48.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435702+0000", 2023-09-23T23:14:48.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:14:48.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:14:48.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435702+0000", 2023-09-23T23:14:48.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:14:48.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:14:48.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435758+0000", 2023-09-23T23:14:48.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6060000000000002e-05 2023-09-23T23:14:48.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:14:48.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435771+0000", 2023-09-23T23:14:48.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3444e-05 2023-09-23T23:14:48.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.477 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.477 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:14:48.477 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435777+0000", 2023-09-23T23:14:48.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1920000000000002e-06 2023-09-23T23:14:48.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.477 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.477 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:14:48.477 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435830+0000", 2023-09-23T23:14:48.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2663e-05 2023-09-23T23:14:48.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:14:48.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:14:48.478 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:14:48.478 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:14:48.435850+0000", 2023-09-23T23:14:48.478 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0523e-05 2023-09-23T23:14:48.478 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:48.478 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:14:48.478 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:48.478 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:14:48.478 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-09-23T23:14:48.478 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 7 2023-09-23T23:14:48.478 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:14:48.479 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:48 smithi110 ceph-mon[221102]: pgmap v590: 99 pgs: 8 peering, 91 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 709 op/s; 17 KiB/s, 0 objects/s recovering 2023-09-23T23:14:48.479 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:48 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3790076016' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:48.479 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:48 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:48.479 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:48 smithi110 ceph-mon[212939]: pgmap v590: 99 pgs: 8 peering, 91 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 709 op/s; 17 KiB/s, 0 objects/s recovering 2023-09-23T23:14:48.479 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:48 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3790076016' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:48.479 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:48.485 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:48.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:48.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:48.547 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 73 16 97658 97642 83.5578 50.8125 0.00677423 0.011959 2023-09-23T23:14:48.550 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:14:48.550 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "" 2023-09-23T23:14:48.550 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:14:48.597 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:48.642 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:48 smithi138 ceph-mon[193606]: pgmap v590: 99 pgs: 8 peering, 91 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 709 op/s; 17 KiB/s, 0 objects/s recovering 2023-09-23T23:14:48.642 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:48 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3790076016' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:48.642 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:48 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:48.652 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:48.722 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:48.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:48.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:48.829 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:14:48.880 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:48.925 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:14:48.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:48.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:14:49.030 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:14:49.108 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:49.361 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:49 smithi110 ceph-mon[221102]: 1.7 scrub starts 2023-09-23T23:14:49.361 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:49 smithi110 ceph-mon[221102]: 1.7 scrub ok 2023-09-23T23:14:49.362 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:49 smithi110 ceph-mon[212939]: 1.7 scrub starts 2023-09-23T23:14:49.362 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:49 smithi110 ceph-mon[212939]: 1.7 scrub ok 2023-09-23T23:14:49.548 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 74 16 98598 98582 83.2217 58.75 0.00981847 0.0120114 2023-09-23T23:14:49.577 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:14:49.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:49.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:14:49.684 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:14:49.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:49 smithi138 ceph-mon[193606]: 1.7 scrub starts 2023-09-23T23:14:49.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:49 smithi138 ceph-mon[193606]: 1.7 scrub ok 2023-09-23T23:14:49.711 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:50.548 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 75 16 99253 99237 82.6581 40.9375 0.00861879 0.0120941 2023-09-23T23:14:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:50 smithi138 ceph-mon[193606]: pgmap v591: 99 pgs: 99 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-09-23T23:14:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:50 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-09-23T23:14:50.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:50 smithi138 ceph-mon[193606]: 1.1 scrub starts 2023-09-23T23:14:50.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:50 smithi138 ceph-mon[193606]: 1.1 scrub ok 2023-09-23T23:14:50.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:50 smithi110 ceph-mon[212939]: pgmap v591: 99 pgs: 99 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-09-23T23:14:50.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:50 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-09-23T23:14:50.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:50 smithi110 ceph-mon[212939]: 1.1 scrub starts 2023-09-23T23:14:50.696 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:50 smithi110 ceph-mon[212939]: 1.1 scrub ok 2023-09-23T23:14:50.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:50 smithi110 ceph-mon[221102]: pgmap v591: 99 pgs: 99 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-09-23T23:14:50.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:50 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-09-23T23:14:50.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:50 smithi110 ceph-mon[221102]: 1.1 scrub starts 2023-09-23T23:14:50.697 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:50 smithi110 ceph-mon[221102]: 1.1 scrub ok 2023-09-23T23:14:50.838 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:14:50.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:50.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:14:50.955 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:14:51.014 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:51.108 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:14:51.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:51.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:14:51.219 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:14:51.266 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:51.425 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:14:51.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:51.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:14:51.438 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:14:51.493 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:14:51.535 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:14:51.552 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 76 16 100218 100202 82.3593 60.3125 0.0158835 0.0121365 2023-09-23T23:14:51.691 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:51.966 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:14:51.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:51.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:14:52.071 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:52.117 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:52.198 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:52.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:52.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:52.305 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:52.307 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:14:52.374 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:52 smithi110 ceph-mon[212939]: pgmap v592: 99 pgs: 99 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 867 op/s 2023-09-23T23:14:52.374 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:52 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:14:52.374 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:52 smithi110 ceph-mon[212939]: 1.a scrub starts 2023-09-23T23:14:52.374 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:52 smithi110 ceph-mon[212939]: 1.a scrub ok 2023-09-23T23:14:52.375 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:52 smithi110 ceph-mon[221102]: pgmap v592: 99 pgs: 99 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 867 op/s 2023-09-23T23:14:52.375 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:52 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:14:52.375 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:52 smithi110 ceph-mon[221102]: 1.a scrub starts 2023-09-23T23:14:52.375 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:52 smithi110 ceph-mon[221102]: 1.a scrub ok 2023-09-23T23:14:52.403 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:52.443 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:52.544 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:14:52.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:52.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:14:52.557 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 77 16 100890 100874 81.8303 42 0.0181613 0.0122155 2023-09-23T23:14:52.656 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:14:52.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:52 smithi138 ceph-mon[193606]: pgmap v592: 99 pgs: 99 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 867 op/s 2023-09-23T23:14:52.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:52 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:14:52.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:52 smithi138 ceph-mon[193606]: 1.a scrub starts 2023-09-23T23:14:52.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:52 smithi138 ceph-mon[193606]: 1.a scrub ok 2023-09-23T23:14:52.725 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:52.830 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:14:52.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:52.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:14:52.944 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:14:52.971 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:53.383 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:14:53.403 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:53 smithi110 ceph-mon[212939]: 1.5 scrub starts 2023-09-23T23:14:53.403 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:53 smithi110 ceph-mon[212939]: 1.5 scrub ok 2023-09-23T23:14:53.403 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:53 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2896632802' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:53.403 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:53 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:53.403 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:53 smithi110 ceph-mon[221102]: 1.5 scrub starts 2023-09-23T23:14:53.403 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:53 smithi110 ceph-mon[221102]: 1.5 scrub ok 2023-09-23T23:14:53.403 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:53 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2896632802' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:53.403 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:53 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:53.469 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:14:53.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:53.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:14:53.557 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 78 16 101548 101532 81.3086 41.125 0.0221265 0.0122939 2023-09-23T23:14:53.599 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:14:53.653 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:53 smithi138 ceph-mon[193606]: 1.5 scrub starts 2023-09-23T23:14:53.653 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:53 smithi138 ceph-mon[193606]: 1.5 scrub ok 2023-09-23T23:14:53.653 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:53 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2896632802' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:53.653 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:53 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:14:53.656 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:53.743 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:14:53.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:53.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:14:53.848 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:53.908 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:54.003 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:54.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:54.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:54.107 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:14:54.202 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:54.377 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:54 smithi110 ceph-mon[212939]: pgmap v593: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 805 op/s 2023-09-23T23:14:54.377 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:54 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:14:54.377 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:54 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:14:54.378 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:54 smithi110 ceph-mon[212939]: failed to encode map e183 with expected crc 2023-09-23T23:14:54.378 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:54 smithi110 ceph-mon[212939]: failed to encode map e183 with expected crc 2023-09-23T23:14:54.378 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:54 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:14:54.378 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:54 smithi138 ceph-mon[193606]: pgmap v593: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 805 op/s 2023-09-23T23:14:54.378 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:54 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:14:54.378 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:54 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:14:54.378 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:54 smithi138 ceph-mon[193606]: failed to encode map e183 with expected crc 2023-09-23T23:14:54.379 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:54 smithi138 ceph-mon[193606]: failed to encode map e183 with expected crc 2023-09-23T23:14:54.379 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:54 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:14:54.379 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:54 smithi138 ceph-mon[193606]: osdmap e183: 8 total, 8 up, 8 in 2023-09-23T23:14:54.381 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:54 smithi110 ceph-mon[221102]: pgmap v593: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 805 op/s 2023-09-23T23:14:54.381 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:54 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:14:54.382 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:54 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:14:54.382 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:54 smithi110 ceph-mon[221102]: failed to encode map e183 with expected crc 2023-09-23T23:14:54.382 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:54 smithi110 ceph-mon[221102]: failed to encode map e183 with expected crc 2023-09-23T23:14:54.382 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:54 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:14:54.382 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:54 smithi110 ceph-mon[221102]: osdmap e183: 8 total, 8 up, 8 in 2023-09-23T23:14:54.557 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 79 16 102359 102343 80.9212 50.6875 0.0198198 0.012353 2023-09-23T23:14:54.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:54 smithi110 ceph-mon[212939]: osdmap e183: 8 total, 8 up, 8 in 2023-09-23T23:14:55.325 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:14:55.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:55.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:14:55.435 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:14:55.500 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:55.557 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:14:55.557623+0000 min lat: 0.00117478 max lat: 0.336316 avg lat: 0.0123963 2023-09-23T23:14:55.557 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:14:55.557 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 80 16 103280 103264 80.6293 57.5625 0.00523923 0.0123963 2023-09-23T23:14:55.571 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:14:55.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:55.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:14:55.675 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:14:55.729 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:55.758 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T23:14:55.758 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:14:55.758 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T23:14:55.758 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-09-23T23:14:56.155 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:14:56.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:56.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:14:56.265 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:56.298 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:56.389 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:56.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:56.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:56.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:56 smithi110 ceph-mon[212939]: pgmap v595: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 776 op/s 2023-09-23T23:14:56.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:56 smithi110 ceph-mon[221102]: pgmap v595: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 776 op/s 2023-09-23T23:14:56.493 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:14:56.558 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 81 16 104021 104005 80.2058 46.3125 0.0160402 0.0124629 2023-09-23T23:14:56.569 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:56.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:56 smithi138 ceph-mon[193606]: pgmap v595: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 776 op/s 2023-09-23T23:14:56.707 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:14:56.708 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:14:56.708 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:14:56.708 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:14:56.708 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:14:56.735 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:14:56.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:56.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:14:56.828 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:14:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:14:56] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2023-09-23T23:14:56.845 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:14:56.950 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:57.067 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:14:57.117 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:14:57.153 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:14:57.192 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:14:57.200 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:14:57.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:57.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:14:57.228 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:14:57.269 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:14:57.297 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:14:57.307 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:57.333 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:14:57.356 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:57.357 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:14:57.357 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:14:57.358 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:14:57.358 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:14:57.358 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:14:57.358 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:14:57.358 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:14:57.358 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:14:57.446 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:57.460 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:57 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2921027607' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:57.460 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:57 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:57.460 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:57 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:57.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:57.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:57.559 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 82 16 104811 104795 79.8286 49.375 0.0145593 0.0125224 2023-09-23T23:14:57.561 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:14:57.634 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:57.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:57 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2921027607' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:57.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:57 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:57.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:57 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:57.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:57 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2921027607' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:14:57.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:57 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:57.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:57 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:57.835 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:14:58.548 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:14:58.560 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 83 16 105470 105454 79.3632 41.1875 0.0131749 0.0125955 2023-09-23T23:14:58.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:58 smithi138 ceph-mon[193606]: pgmap v596: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 775 op/s 2023-09-23T23:14:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:14:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:14:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:58.695 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:14:58.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:58 smithi110 ceph-mon[212939]: pgmap v596: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 775 op/s 2023-09-23T23:14:58.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:58.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:14:58.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:14:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:58.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:58 smithi110 ceph-mon[221102]: pgmap v596: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 775 op/s 2023-09-23T23:14:58.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:58.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:14:58.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:14:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:14:58.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:58.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:14:58.785 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:58.807 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:14:58.856 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:58.915 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:14:58.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:58.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:14:58.946 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:14:59.023 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:59.071 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:59.142 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:59.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:59.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:59.249 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:59.297 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:59.372 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:14:59.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:59.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:14:59.477 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:14:59.549 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:59.566 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 84 16 106273 106257 79.016 50.1875 0.0160478 0.0126512 2023-09-23T23:14:59.759 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:14:59.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:14:59.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:14:59.871 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:14:59.914 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:14:59.997 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:00.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:00.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:00.112 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:15:00.170 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:00.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:00 smithi110 ceph-mon[212939]: pgmap v597: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2023-09-23T23:15:00.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:00 smithi110 ceph-mon[221102]: pgmap v597: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2023-09-23T23:15:00.559 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 85 16 107177 107161 78.7512 56.5 0.0193315 0.0126935 2023-09-23T23:15:00.566 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:15:00.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:00.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:15:00.671 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:00.743 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:00.771 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:15:00.806 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:15:00.859 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:00.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:00.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:00.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:00 smithi138 ceph-mon[193606]: pgmap v597: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2023-09-23T23:15:00.965 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:01.015 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:01.090 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:01.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:01.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:01.198 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:15:01.299 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:01.559 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 86 15 108025 108010 78.4525 53.0625 0.0228353 0.0127426 2023-09-23T23:15:02.440 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:15:02.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:02.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:15:02.548 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:02.559 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 87 16 108831 108815 78.1292 50.3125 0.0139658 0.0127946 2023-09-23T23:15:02.634 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:02.634 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:02 smithi110 ceph-mon[212939]: pgmap v598: 99 pgs: 99 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2023-09-23T23:15:02.635 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:02 smithi110 ceph-mon[221102]: pgmap v598: 99 pgs: 99 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2023-09-23T23:15:02.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:02 smithi138 ceph-mon[193606]: pgmap v598: 99 pgs: 99 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2023-09-23T23:15:03.098 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:03.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:03.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:03.203 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:03.284 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:03.560 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 88 16 109701 109685 77.8586 54.375 0.0214732 0.0128388 2023-09-23T23:15:03.637 INFO:teuthology.orchestra.run.smithi110.stdout:moved 0 / 297 (0%) 2023-09-23T23:15:03.638 INFO:teuthology.orchestra.run.smithi110.stdout:avg 37.125 2023-09-23T23:15:03.638 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 7.38982 -> 7.38982 (expected baseline 5.69951) 2023-09-23T23:15:03.638 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.5 with 25 -> 25 pgs (0.673401 -> 0.673401 * mean) 2023-09-23T23:15:03.638 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.3 with 47 -> 47 pgs (1.26599 -> 1.26599 * mean) 2023-09-23T23:15:03.638 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:15:03.638 INFO:teuthology.orchestra.run.smithi110.stdout:oload 1000 2023-09-23T23:15:03.638 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 3 2023-09-23T23:15:03.638 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:15:03.638 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0308 2023-09-23T23:15:03.638 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.3082 2023-09-23T23:15:03.641 INFO:teuthology.orchestra.run.smithi110.stderr:no change 2023-09-23T23:15:04.055 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:04.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:04.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:04.172 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:15:04.211 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:04.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:04 smithi110 ceph-mon[212939]: pgmap v599: 99 pgs: 99 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s 2023-09-23T23:15:04.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:04 smithi110 ceph-mon[212939]: from='client.35265 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:15:04.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:04 smithi110 ceph-mon[221102]: pgmap v599: 99 pgs: 99 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s 2023-09-23T23:15:04.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:04 smithi110 ceph-mon[221102]: from='client.35265 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:15:04.561 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 89 16 110530 110514 77.566 51.8125 0.0217681 0.0128876 2023-09-23T23:15:04.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:04 smithi138 ceph-mon[193606]: pgmap v599: 99 pgs: 99 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s 2023-09-23T23:15:04.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:04 smithi138 ceph-mon[193606]: from='client.35265 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:15:04.993 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:15:05.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:05.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:15:05.033 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T23:15:05.033 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-09-23T23:15:05.102 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:15:05.236 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:05.312 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:15:05.546 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:15:05.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:05.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:15:05.561 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total time run: 90.0053 2023-09-23T23:15:05.562 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total writes made: 111178 2023-09-23T23:15:05.562 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Write size: 65536 2023-09-23T23:15:05.562 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object size: 65536 2023-09-23T23:15:05.562 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Bandwidth (MB/sec): 77.2024 2023-09-23T23:15:05.562 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Bandwidth: 37.9079 2023-09-23T23:15:05.562 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max bandwidth (MB/sec): 199.75 2023-09-23T23:15:05.562 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min bandwidth (MB/sec): 38.5625 2023-09-23T23:15:05.562 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average IOPS: 1235 2023-09-23T23:15:05.562 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev IOPS: 606.526 2023-09-23T23:15:05.562 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max IOPS: 3196 2023-09-23T23:15:05.562 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min IOPS: 617 2023-09-23T23:15:05.562 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average Latency(s): 0.0129497 2023-09-23T23:15:05.563 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Latency(s): 0.0154706 2023-09-23T23:15:05.563 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max latency(s): 0.336316 2023-09-23T23:15:05.563 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min latency(s): 0.00117478 2023-09-23T23:15:05.572 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T23:15:05.608 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:15:05.608 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:15:05.650 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:05.771 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:05.941 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:05.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:05.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:06.047 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:06.104 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:06.185 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:06.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:06.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:06.291 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:06.348 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:06.445 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:15:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:06 smithi138 ceph-mon[193606]: pgmap v600: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-09-23T23:15:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:06 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:06 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:06 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2108800041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:15:06.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:06 smithi110 ceph-mon[212939]: pgmap v600: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-09-23T23:15:06.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:06 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:06.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:06 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:06.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:06 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2108800041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:15:06.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:06 smithi110 ceph-mon[221102]: pgmap v600: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-09-23T23:15:06.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:06 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:06.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:06 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:06.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:06 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2108800041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:15:06.923 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:06.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:06.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:07.031 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:07.069 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:07.077 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:15:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:15:06] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.43.0" 2023-09-23T23:15:07.145 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:07.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:07.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:07.250 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:07.275 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:07.318 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:07.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:07.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:07.424 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:15:07.426 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:15:07.426 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2108800041' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:15:07.426 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[221102]: osdmap e184: 8 total, 8 up, 8 in 2023-09-23T23:15:07.426 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[221102]: failed to encode map e184 with expected crc 2023-09-23T23:15:07.426 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[221102]: failed to encode map e184 with expected crc 2023-09-23T23:15:07.426 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[221102]: failed to encode map e184 with expected crc 2023-09-23T23:15:07.427 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[221102]: failed to encode map e184 with expected crc 2023-09-23T23:15:07.427 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[221102]: failed to encode map e184 with expected crc 2023-09-23T23:15:07.427 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:15:07.427 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:15:07.427 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:07.474 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:07.552 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:15:07.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:07.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:15:07.569 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:15:07.624 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:07 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:15:07.624 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:07 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2108800041' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:15:07.624 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:07 smithi138 ceph-mon[193606]: osdmap e184: 8 total, 8 up, 8 in 2023-09-23T23:15:07.624 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:07 smithi138 ceph-mon[193606]: failed to encode map e184 with expected crc 2023-09-23T23:15:07.624 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:07 smithi138 ceph-mon[193606]: failed to encode map e184 with expected crc 2023-09-23T23:15:07.624 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:07 smithi138 ceph-mon[193606]: failed to encode map e184 with expected crc 2023-09-23T23:15:07.624 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:07 smithi138 ceph-mon[193606]: failed to encode map e184 with expected crc 2023-09-23T23:15:07.624 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:07 smithi138 ceph-mon[193606]: failed to encode map e184 with expected crc 2023-09-23T23:15:07.624 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:07 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:15:07.625 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:07 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:15:07.625 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:07 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:07.664 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:15:07.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:15:07.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2108800041' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:15:07.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[212939]: osdmap e184: 8 total, 8 up, 8 in 2023-09-23T23:15:07.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[212939]: failed to encode map e184 with expected crc 2023-09-23T23:15:07.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[212939]: failed to encode map e184 with expected crc 2023-09-23T23:15:07.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[212939]: failed to encode map e184 with expected crc 2023-09-23T23:15:07.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[212939]: failed to encode map e184 with expected crc 2023-09-23T23:15:07.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[212939]: failed to encode map e184 with expected crc 2023-09-23T23:15:07.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:15:07.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:15:07.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:07 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:07.725 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:07.904 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:15:07.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:07.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:15:08.012 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:15:08.058 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:08.204 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:15:08.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:08.435 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:08 smithi110 ceph-mon[212939]: pgmap v602: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s 2023-09-23T23:15:08.435 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:15:08.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:15:08.537 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:15:08.583 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:08.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:08 smithi138 ceph-mon[193606]: pgmap v602: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s 2023-09-23T23:15:08.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:08 smithi110 ceph-mon[221102]: pgmap v602: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s 2023-09-23T23:15:08.939 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:15:08.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:08.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:15:08.962 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:15:09.044 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:09.091 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:15:09.134 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:15:09.140 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:09.169 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:15:09.224 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:15:09.265 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:15:09.299 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:15:09.315 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:09.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:09.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:09.340 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:15:09.375 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:15:09.385 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:15:09.385 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:15:09.386 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:15:09.386 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:15:09.386 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:15:09.386 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:15:09.386 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:15:09.386 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:15:09.419 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:15:09.446 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:09.488 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:15:09.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:09.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:15:09.593 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:09.621 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:09.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:09 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4058322274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:09.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:09 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4058322274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:09.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:09 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4058322274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:10.520 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:15:10.694 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:15:10.694 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:15:10.694 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-09-23T23:15:10.694 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:15:10.699 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:10 smithi138 ceph-mon[193606]: pgmap v603: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 39 MiB/s wr, 814 op/s 2023-09-23T23:15:10.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:10 smithi110 ceph-mon[221102]: pgmap v603: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 39 MiB/s wr, 814 op/s 2023-09-23T23:15:10.711 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:10 smithi110 ceph-mon[212939]: pgmap v603: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 39 MiB/s wr, 814 op/s 2023-09-23T23:15:11.229 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:11.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:11.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:11.336 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:11.383 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:11.590 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:11.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:11.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:11.616 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:15:11.712 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:15:11.788 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:11.881 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:15:11.942 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:15:11.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:11.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:15:12.057 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:12.119 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:12.201 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:12.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:12.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:12.230 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:15:12.312 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:12.361 INFO:teuthology.orchestra.run.smithi110.stdout:moved 0 / 297 (0%) 2023-09-23T23:15:12.362 INFO:teuthology.orchestra.run.smithi110.stdout:avg 37.125 2023-09-23T23:15:12.362 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 7.38982 -> 7.38982 (expected baseline 5.69951) 2023-09-23T23:15:12.362 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.5 with 25 -> 25 pgs (0.673401 -> 0.673401 * mean) 2023-09-23T23:15:12.362 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.3 with 47 -> 47 pgs (1.26599 -> 1.26599 * mean) 2023-09-23T23:15:12.362 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:15:12.362 INFO:teuthology.orchestra.run.smithi110.stdout:oload 1000 2023-09-23T23:15:12.362 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 0.05 2023-09-23T23:15:12.362 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:15:12.362 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0316 2023-09-23T23:15:12.362 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.3156 2023-09-23T23:15:12.365 INFO:teuthology.orchestra.run.smithi110.stderr:no change 2023-09-23T23:15:12.388 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:12.494 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:12 smithi110 ceph-mon[221102]: pgmap v604: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 813 op/s 2023-09-23T23:15:12.495 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:12 smithi110 ceph-mon[212939]: pgmap v604: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 813 op/s 2023-09-23T23:15:12.548 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:12.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:12.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:12.663 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:12.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:12 smithi138 ceph-mon[193606]: pgmap v604: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 813 op/s 2023-09-23T23:15:12.712 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:12.866 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:12.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:12.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:12.972 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:15:12.977 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:15:13.025 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:13.111 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:15:13.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:13.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:15:13.221 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:13.288 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:13.343 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:13.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:13.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:13.447 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:13.500 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:13.607 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:13 smithi110 ceph-mon[221102]: from='client.45248 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:15:13.608 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:13.608 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:15:13.608 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:13 smithi110 ceph-mon[212939]: from='client.45248 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:15:13.609 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:13.609 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:15:13.704 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:13.706 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:13 smithi138 ceph-mon[193606]: from='client.45248 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:15:13.706 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:13.706 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:15:13.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:13.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:13.809 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:15:13.846 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:14.269 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:15:14.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:14.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:15:14.339 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:15:14.339 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:15:14.339 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:15:14.339 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:15:14.339 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:15:14.374 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:15:14.450 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:14.473 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T23:15:14.473 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T23:15:14.551 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:14 smithi110 ceph-mon[212939]: pgmap v605: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 802 op/s 2023-09-23T23:15:14.558 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:14 smithi110 ceph-mon[221102]: pgmap v605: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 802 op/s 2023-09-23T23:15:14.634 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:15:14.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:14.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:15:14.659 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:15:14.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:14 smithi138 ceph-mon[193606]: pgmap v605: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 802 op/s 2023-09-23T23:15:14.746 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:15:14.778 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:15.041 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:15:15.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:15.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:15:15.165 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:15:15.248 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:15.255 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:15:15.383 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:15:15.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:15.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:15:15.499 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:15:15.613 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:15.699 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:15:15.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:15.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:15:15.810 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:15:15.812 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:15.840 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:15.883 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:15.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:15.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:15.987 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:16.045 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:16.156 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:16.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:16.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:16.262 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:16.331 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:15:16.335 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:16.457 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:16.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:16.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:16.528 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:15:16.564 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:15:16.654 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:16 smithi110 ceph-mon[212939]: pgmap v606: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 13 MiB/s wr, 774 op/s 2023-09-23T23:15:16.655 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:16 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1196326883' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:16.655 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:16 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:16.655 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:16 smithi110 ceph-mon[221102]: pgmap v606: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 13 MiB/s wr, 774 op/s 2023-09-23T23:15:16.655 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:16 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1196326883' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:16.655 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:16 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:16.655 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:16.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:16 smithi138 ceph-mon[193606]: pgmap v606: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 13 MiB/s wr, 774 op/s 2023-09-23T23:15:16.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:16 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1196326883' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:16.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:16 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:16.955 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:15:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:15:16] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.43.0" 2023-09-23T23:15:17.470 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:15:17.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:17.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:15:17.578 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:17.606 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:17.747 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:15:17.747 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:15:17.747 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:15:17.747 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:15:17.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.747 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34269.0:168463 7.1 7:834322bf:::benchmark_data_smithi110_236826_object55755:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e103)", 2023-09-23T23:15:17.747 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:06:19.088978+0000", 2023-09-23T23:15:17.747 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 538.65394701299999, 2023-09-23T23:15:17.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.068973072999999996, 2023-09-23T23:15:17.748 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.748 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.748 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:17.748 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34269", 2023-09-23T23:15:17.748 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:15:17.748 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 168463 2023-09-23T23:15:17.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.748 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:06:19.088978+0000", 2023-09-23T23:15:17.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.749 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:06:19.088978+0000", 2023-09-23T23:15:17.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.749 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:06:19.088976+0000", 2023-09-23T23:15:17.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:17.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:06:19.088982+0000", 2023-09-23T23:15:17.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1e-06 2023-09-23T23:15:17.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:06:19.088996+0000", 2023-09-23T23:15:17.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3818999999999999e-05 2023-09-23T23:15:17.751 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:06:19.089002+0000", 2023-09-23T23:15:17.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0229999999999998e-06 2023-09-23T23:15:17.751 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:06:19.089040+0000", 2023-09-23T23:15:17.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.756e-05 2023-09-23T23:15:17.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.752 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:06:19.089173+0000", 2023-09-23T23:15:17.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000133163 2023-09-23T23:15:17.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.752 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:15:17.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:06:19.089295+0000", 2023-09-23T23:15:17.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000122003 2023-09-23T23:15:17.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:17.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:06:19.096927+0000", 2023-09-23T23:15:17.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0076322100000000004 2023-09-23T23:15:17.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:06:19.101303+0000", 2023-09-23T23:15:17.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043756450000000001 2023-09-23T23:15:17.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:06:19.157879+0000", 2023-09-23T23:15:17.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.056576424 2023-09-23T23:15:17.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:06:19.157917+0000", 2023-09-23T23:15:17.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7515999999999998e-05 2023-09-23T23:15:17.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:06:19.157951+0000", 2023-09-23T23:15:17.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3831000000000002e-05 2023-09-23T23:15:17.755 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.755 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.755 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.755 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.755 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:54180 8.2 8:452ad8e4:::benchmark_data_smithi110_258128_object54179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-23T23:15:17.755 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:08:23.952894+0000", 2023-09-23T23:15:17.755 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 413.79003033700002, 2023-09-23T23:15:17.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.214200627, 2023-09-23T23:15:17.756 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.756 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.756 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:17.756 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:17.756 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:17.756 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 54180 2023-09-23T23:15:17.756 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.756 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.756 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.756 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.756 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:23.952894+0000", 2023-09-23T23:15:17.756 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:23.952894+0000", 2023-09-23T23:15:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:23.952893+0000", 2023-09-23T23:15:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:15:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:23.952988+0000", 2023-09-23T23:15:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5299999999999999e-05 2023-09-23T23:15:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:23.953003+0000", 2023-09-23T23:15:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5038999999999999e-05 2023-09-23T23:15:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:23.953010+0000", 2023-09-23T23:15:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3169999999999998e-06 2023-09-23T23:15:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:23.953058+0000", 2023-09-23T23:15:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7831999999999999e-05 2023-09-23T23:15:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:23.953194+0000", 2023-09-23T23:15:17.760 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000136305 2023-09-23T23:15:17.760 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.760 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.760 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:15:17.760 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:23.953330+0000", 2023-09-23T23:15:17.760 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013606499999999999 2023-09-23T23:15:17.760 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.760 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.760 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:17.760 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:23.955236+0000", 2023-09-23T23:15:17.760 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0019058759999999999 2023-09-23T23:15:17.760 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.761 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.761 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:23.956397+0000", 2023-09-23T23:15:17.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001160779 2023-09-23T23:15:17.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.761 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.761 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:24.167001+0000", 2023-09-23T23:15:17.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21060429 2023-09-23T23:15:17.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.761 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.761 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:24.167054+0000", 2023-09-23T23:15:17.762 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2882e-05 2023-09-23T23:15:17.762 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.762 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.762 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.762 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:24.167095+0000", 2023-09-23T23:15:17.762 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1099000000000001e-05 2023-09-23T23:15:17.762 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.762 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.762 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.762 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.762 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.762 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:58042 8.2 8:42c972b5:::benchmark_data_smithi110_258128_object58041:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-09-23T23:15:17.763 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:08:27.644162+0000", 2023-09-23T23:15:17.763 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 410.098763099, 2023-09-23T23:15:17.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.083133068000000004, 2023-09-23T23:15:17.763 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.763 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.763 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:17.763 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:17.763 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:17.763 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58042 2023-09-23T23:15:17.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.763 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.763 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.763 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.764 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.644162+0000", 2023-09-23T23:15:17.764 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.764 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.764 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.764 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.764 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.644162+0000", 2023-09-23T23:15:17.764 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.764 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.764 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.764 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.764 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.644160+0000", 2023-09-23T23:15:17.764 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:15:17.764 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.765 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.765 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.765 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.644194+0000", 2023-09-23T23:15:17.765 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4517999999999997e-05 2023-09-23T23:15:17.765 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.765 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.765 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.765 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.644212+0000", 2023-09-23T23:15:17.765 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7561e-05 2023-09-23T23:15:17.765 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.765 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.765 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.766 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.644218+0000", 2023-09-23T23:15:17.766 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4649999999999999e-06 2023-09-23T23:15:17.766 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.766 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.766 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.766 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.644293+0000", 2023-09-23T23:15:17.766 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4993999999999994e-05 2023-09-23T23:15:17.766 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.766 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.766 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.767 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.644413+0000", 2023-09-23T23:15:17.767 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000119714 2023-09-23T23:15:17.767 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.767 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.767 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:15:17.767 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.644536+0000", 2023-09-23T23:15:17.767 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000123091 2023-09-23T23:15:17.767 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.767 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.767 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:17.767 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.649392+0000", 2023-09-23T23:15:17.768 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048561200000000002 2023-09-23T23:15:17.768 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.768 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.768 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.768 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.653321+0000", 2023-09-23T23:15:17.768 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039283540000000002 2023-09-23T23:15:17.768 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.768 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.768 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.768 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.727204+0000", 2023-09-23T23:15:17.768 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.073883004000000002 2023-09-23T23:15:17.769 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.769 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.769 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.769 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.727253+0000", 2023-09-23T23:15:17.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8955999999999997e-05 2023-09-23T23:15:17.769 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.769 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.769 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.769 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.727295+0000", 2023-09-23T23:15:17.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1989000000000001e-05 2023-09-23T23:15:17.769 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.769 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.769 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.770 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.770 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.770 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:58051 8.2 8:4267fc49:::benchmark_data_smithi110_258128_object58050:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-09-23T23:15:17.770 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:08:27.646808+0000", 2023-09-23T23:15:17.770 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 410.09611698399999, 2023-09-23T23:15:17.770 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.080647341999999997, 2023-09-23T23:15:17.770 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.770 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.770 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:17.770 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:17.770 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:17.773 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58051 2023-09-23T23:15:17.773 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.773 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.773 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.773 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.773 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.646808+0000", 2023-09-23T23:15:17.773 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.773 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.773 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.773 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.774 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.646808+0000", 2023-09-23T23:15:17.774 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.774 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.774 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.646806+0000", 2023-09-23T23:15:17.774 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:17.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.774 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.774 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.646882+0000", 2023-09-23T23:15:17.774 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5685000000000005e-05 2023-09-23T23:15:17.775 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.646911+0000", 2023-09-23T23:15:17.775 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9142999999999999e-05 2023-09-23T23:15:17.775 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.646918+0000", 2023-09-23T23:15:17.775 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6259999999999997e-06 2023-09-23T23:15:17.775 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.776 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.646985+0000", 2023-09-23T23:15:17.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7481e-05 2023-09-23T23:15:17.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.776 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.776 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.776 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.647123+0000", 2023-09-23T23:15:17.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000138164 2023-09-23T23:15:17.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.776 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.776 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:15:17.776 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.647263+0000", 2023-09-23T23:15:17.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014010199999999999 2023-09-23T23:15:17.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:17.777 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.649398+0000", 2023-09-23T23:15:17.777 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021347419999999998 2023-09-23T23:15:17.777 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.777 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.653352+0000", 2023-09-23T23:15:17.777 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039538949999999998 2023-09-23T23:15:17.777 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.778 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.727360+0000", 2023-09-23T23:15:17.778 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.074008097999999994 2023-09-23T23:15:17.778 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.778 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.778 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.778 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.727420+0000", 2023-09-23T23:15:17.778 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9914000000000003e-05 2023-09-23T23:15:17.778 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.778 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.778 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.778 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.727455+0000", 2023-09-23T23:15:17.778 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4932999999999999e-05 2023-09-23T23:15:17.779 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.779 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.779 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.779 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.779 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.779 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:58054 8.2 8:4d176544:::benchmark_data_smithi110_258128_object58053:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-09-23T23:15:17.779 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:08:27.649639+0000", 2023-09-23T23:15:17.779 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 410.09328524699998, 2023-09-23T23:15:17.779 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.077930803000000007, 2023-09-23T23:15:17.779 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.779 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.779 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:17.780 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:17.780 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:17.780 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58054 2023-09-23T23:15:17.780 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.780 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.780 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.780 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.780 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.649639+0000", 2023-09-23T23:15:17.780 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.780 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.780 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.780 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.781 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.649639+0000", 2023-09-23T23:15:17.781 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.781 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.781 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.781 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.781 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.649638+0000", 2023-09-23T23:15:17.781 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:17.781 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.781 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.781 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.781 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.649687+0000", 2023-09-23T23:15:17.781 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9014999999999997e-05 2023-09-23T23:15:17.782 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.782 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.782 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.782 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.649703+0000", 2023-09-23T23:15:17.782 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6521e-05 2023-09-23T23:15:17.782 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.782 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.782 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.782 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.649710+0000", 2023-09-23T23:15:17.782 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6730000000000003e-06 2023-09-23T23:15:17.782 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.782 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.783 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.783 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.649761+0000", 2023-09-23T23:15:17.783 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0834000000000002e-05 2023-09-23T23:15:17.783 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.783 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.783 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.783 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.649882+0000", 2023-09-23T23:15:17.783 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012066999999999999 2023-09-23T23:15:17.783 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.783 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.783 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:15:17.783 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.650007+0000", 2023-09-23T23:15:17.783 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012549800000000001 2023-09-23T23:15:17.784 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.784 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.784 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:17.784 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.652158+0000", 2023-09-23T23:15:17.784 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002150908 2023-09-23T23:15:17.784 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.784 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.784 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.784 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.653379+0000", 2023-09-23T23:15:17.785 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012210770000000001 2023-09-23T23:15:17.785 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.785 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.785 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.785 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.727506+0000", 2023-09-23T23:15:17.785 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.074126782000000002 2023-09-23T23:15:17.785 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.785 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.785 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.785 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.727537+0000", 2023-09-23T23:15:17.785 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1423000000000001e-05 2023-09-23T23:15:17.786 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.786 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.786 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.786 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.727570+0000", 2023-09-23T23:15:17.786 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2801e-05 2023-09-23T23:15:17.786 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.786 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.786 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.786 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.786 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.787 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:58067 8.2 8:49643016:::benchmark_data_smithi110_258128_object58066:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-09-23T23:15:17.787 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:08:27.662567+0000", 2023-09-23T23:15:17.787 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 410.080357978, 2023-09-23T23:15:17.787 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.065168377, 2023-09-23T23:15:17.787 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.787 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.787 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:17.787 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:17.787 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:17.787 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58067 2023-09-23T23:15:17.787 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.787 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.788 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.788 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.788 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.662567+0000", 2023-09-23T23:15:17.788 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.788 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.788 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.788 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.788 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.662567+0000", 2023-09-23T23:15:17.788 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.788 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.788 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.788 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.788 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.662566+0000", 2023-09-23T23:15:17.789 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:15:17.789 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.789 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.789 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.789 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.662623+0000", 2023-09-23T23:15:17.789 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7837999999999997e-05 2023-09-23T23:15:17.789 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.789 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.789 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.789 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.662635+0000", 2023-09-23T23:15:17.789 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2006e-05 2023-09-23T23:15:17.789 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.790 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.790 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.790 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.662640+0000", 2023-09-23T23:15:17.790 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8999999999999997e-06 2023-09-23T23:15:17.790 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.790 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.790 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.790 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.662673+0000", 2023-09-23T23:15:17.790 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3039e-05 2023-09-23T23:15:17.790 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.791 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.662764+0000", 2023-09-23T23:15:17.791 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0387e-05 2023-09-23T23:15:17.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.791 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:15:17.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.662858+0000", 2023-09-23T23:15:17.791 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4798999999999998e-05 2023-09-23T23:15:17.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.791 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:17.792 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.664532+0000", 2023-09-23T23:15:17.792 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0016738250000000001 2023-09-23T23:15:17.792 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.792 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.792 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.792 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.664663+0000", 2023-09-23T23:15:17.792 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000131149 2023-09-23T23:15:17.792 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.792 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.792 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.792 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.727601+0000", 2023-09-23T23:15:17.792 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.062937940999999997 2023-09-23T23:15:17.793 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.793 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.793 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.793 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.727688+0000", 2023-09-23T23:15:17.793 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6996999999999994e-05 2023-09-23T23:15:17.793 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.793 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.793 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.793 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:27.727735+0000", 2023-09-23T23:15:17.793 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6616999999999998e-05 2023-09-23T23:15:17.793 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.793 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.793 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.794 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.794 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.794 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:125997 8.2 8:456aa582:::benchmark_data_smithi110_258128_object7179:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T23:15:17.794 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:09:38.986688+0000", 2023-09-23T23:15:17.794 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 338.756237091, 2023-09-23T23:15:17.794 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.082326331000000003, 2023-09-23T23:15:17.794 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.794 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.794 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:17.794 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:17.794 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:17.794 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 125997 2023-09-23T23:15:17.795 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.795 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.795 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.795 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.795 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:38.986688+0000", 2023-09-23T23:15:17.795 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.795 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.795 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.795 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.795 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:38.986688+0000", 2023-09-23T23:15:17.795 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.795 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.796 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.796 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.796 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:38.986686+0000", 2023-09-23T23:15:17.796 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:17.796 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.796 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.796 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.796 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:38.986693+0000", 2023-09-23T23:15:17.796 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5039999999999999e-06 2023-09-23T23:15:17.796 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.796 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.796 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.796 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:38.986710+0000", 2023-09-23T23:15:17.797 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7195999999999999e-05 2023-09-23T23:15:17.797 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.797 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.797 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.797 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:38.986718+0000", 2023-09-23T23:15:17.797 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4649999999999994e-06 2023-09-23T23:15:17.797 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.797 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.797 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.797 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:38.986768+0000", 2023-09-23T23:15:17.798 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9604000000000001e-05 2023-09-23T23:15:17.798 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.798 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.798 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.798 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:38.986877+0000", 2023-09-23T23:15:17.799 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108868 2023-09-23T23:15:17.799 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.799 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.799 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:15:17.799 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:38.986996+0000", 2023-09-23T23:15:17.799 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011929 2023-09-23T23:15:17.799 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.799 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.799 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:17.800 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:38.991016+0000", 2023-09-23T23:15:17.800 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040199520000000002 2023-09-23T23:15:17.800 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.800 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.800 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.800 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:38.991158+0000", 2023-09-23T23:15:17.800 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014224800000000001 2023-09-23T23:15:17.801 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.801 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.801 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.801 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:39.068934+0000", 2023-09-23T23:15:17.801 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.077776087999999993 2023-09-23T23:15:17.801 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.801 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.801 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.801 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:39.068978+0000", 2023-09-23T23:15:17.801 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4057999999999997e-05 2023-09-23T23:15:17.802 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.802 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.802 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.802 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:39.069014+0000", 2023-09-23T23:15:17.802 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5410999999999997e-05 2023-09-23T23:15:17.802 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.802 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.802 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.802 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.802 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.802 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:140605 8.2 8:4a89fd53:::benchmark_data_smithi110_258128_object21787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T23:15:17.802 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:09:58.144208+0000", 2023-09-23T23:15:17.803 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 319.59871659499998, 2023-09-23T23:15:17.803 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16901797199999999, 2023-09-23T23:15:17.803 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.803 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.803 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:17.803 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:17.803 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:17.803 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 140605 2023-09-23T23:15:17.803 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.803 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.804 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.804 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.804 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:58.144208+0000", 2023-09-23T23:15:17.804 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.804 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.804 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.804 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.804 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:58.144208+0000", 2023-09-23T23:15:17.804 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.805 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.805 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.805 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.805 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:58.144206+0000", 2023-09-23T23:15:17.805 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:15:17.805 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.805 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.805 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.805 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:58.144216+0000", 2023-09-23T23:15:17.805 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3470000000000005e-06 2023-09-23T23:15:17.805 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.805 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.806 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.806 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:58.144232+0000", 2023-09-23T23:15:17.806 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5998000000000002e-05 2023-09-23T23:15:17.806 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.806 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.806 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.806 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:58.144240+0000", 2023-09-23T23:15:17.806 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3580000000000003e-06 2023-09-23T23:15:17.806 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.806 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.806 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.806 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:58.148330+0000", 2023-09-23T23:15:17.806 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004089545 2023-09-23T23:15:17.807 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.807 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.807 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.807 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:58.148448+0000", 2023-09-23T23:15:17.807 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118257 2023-09-23T23:15:17.807 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.807 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.807 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:15:17.807 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:58.148563+0000", 2023-09-23T23:15:17.807 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011526700000000001 2023-09-23T23:15:17.807 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.807 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.808 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:17.808 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:58.153219+0000", 2023-09-23T23:15:17.808 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0046561179999999999 2023-09-23T23:15:17.808 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.808 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.808 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.808 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:58.164349+0000", 2023-09-23T23:15:17.808 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011129527 2023-09-23T23:15:17.808 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.808 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.808 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.808 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:58.313117+0000", 2023-09-23T23:15:17.808 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.14876789800000001 2023-09-23T23:15:17.809 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.809 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.809 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.809 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:58.313180+0000", 2023-09-23T23:15:17.809 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2992999999999998e-05 2023-09-23T23:15:17.809 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.809 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.809 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.809 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:09:58.313226+0000", 2023-09-23T23:15:17.809 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6418999999999997e-05 2023-09-23T23:15:17.809 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.809 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.809 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.810 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.810 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.810 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:171302 8.b 8:d70babc6:::benchmark_data_smithi110_258128_object52484:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T23:15:17.810 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:39.128915+0000", 2023-09-23T23:15:17.810 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 278.61400971099999, 2023-09-23T23:15:17.810 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2014737069999999, 2023-09-23T23:15:17.810 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.810 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.810 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:17.810 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:17.810 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:17.810 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 171302 2023-09-23T23:15:17.811 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.811 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.811 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.811 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.811 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128915+0000", 2023-09-23T23:15:17.811 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.811 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.811 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.811 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.811 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128915+0000", 2023-09-23T23:15:17.811 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.811 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.811 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.812 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.812 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128913+0000", 2023-09-23T23:15:17.812 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:17.812 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.812 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.812 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.812 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128920+0000", 2023-09-23T23:15:17.812 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4930000000000003e-06 2023-09-23T23:15:17.812 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.812 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.812 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.812 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128944+0000", 2023-09-23T23:15:17.812 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4297000000000001e-05 2023-09-23T23:15:17.813 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.813 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.813 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.813 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128952+0000", 2023-09-23T23:15:17.813 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.712e-06 2023-09-23T23:15:17.813 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.813 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.813 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.813 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.129003+0000", 2023-09-23T23:15:17.813 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0538999999999998e-05 2023-09-23T23:15:17.813 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.813 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.814 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:17.814 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.129013+0000", 2023-09-23T23:15:17.814 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0188e-05 2023-09-23T23:15:17.814 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.814 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.814 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.814 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.146049+0000", 2023-09-23T23:15:17.814 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0170364089999999 2023-09-23T23:15:17.814 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.814 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.814 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.814 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.148680+0000", 2023-09-23T23:15:17.814 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0026305450000000002 2023-09-23T23:15:17.815 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.815 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.815 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1", 2023-09-23T23:15:17.815 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.148979+0000", 2023-09-23T23:15:17.815 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00029888000000000001 2023-09-23T23:15:17.815 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.815 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.815 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:17.815 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.158433+0000", 2023-09-23T23:15:17.815 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0094546939999999996 2023-09-23T23:15:17.815 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.816 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.816 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.816 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.330258+0000", 2023-09-23T23:15:17.816 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.171825009 2023-09-23T23:15:17.818 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.818 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.818 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.818 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.330303+0000", 2023-09-23T23:15:17.818 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4895999999999998e-05 2023-09-23T23:15:17.818 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.818 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.818 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.818 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.330389+0000", 2023-09-23T23:15:17.818 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5438000000000004e-05 2023-09-23T23:15:17.818 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.818 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.819 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.819 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.819 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.819 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:171308 8.b 8:d1ffb63e:::benchmark_data_smithi110_258128_object52490:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T23:15:17.819 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:39.134289+0000", 2023-09-23T23:15:17.819 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 278.608635628, 2023-09-23T23:15:17.819 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1993630239999999, 2023-09-23T23:15:17.819 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.819 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.819 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:17.819 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:17.819 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:17.820 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 171308 2023-09-23T23:15:17.820 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.820 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.820 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.820 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.820 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134289+0000", 2023-09-23T23:15:17.820 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.820 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.820 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.820 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.820 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134289+0000", 2023-09-23T23:15:17.820 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.820 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.821 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.821 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.821 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134287+0000", 2023-09-23T23:15:17.821 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:17.821 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.821 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.821 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.821 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134295+0000", 2023-09-23T23:15:17.821 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.187e-06 2023-09-23T23:15:17.821 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.821 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.821 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.822 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134313+0000", 2023-09-23T23:15:17.822 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8102e-05 2023-09-23T23:15:17.822 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.822 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.822 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.822 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134320+0000", 2023-09-23T23:15:17.822 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9859999999999997e-06 2023-09-23T23:15:17.822 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.822 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.822 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.822 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134380+0000", 2023-09-23T23:15:17.822 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9979999999999998e-05 2023-09-23T23:15:17.823 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.823 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:17.823 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134387+0000", 2023-09-23T23:15:17.823 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0450000000000003e-06 2023-09-23T23:15:17.823 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.823 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.823 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.153349+0000", 2023-09-23T23:15:17.823 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0189619759999999 2023-09-23T23:15:17.823 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.824 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.824 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.153445+0000", 2023-09-23T23:15:17.824 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6444e-05 2023-09-23T23:15:17.824 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.824 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.824 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1", 2023-09-23T23:15:17.824 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.153529+0000", 2023-09-23T23:15:17.824 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3825000000000005e-05 2023-09-23T23:15:17.824 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.824 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.824 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:17.824 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.158442+0000", 2023-09-23T23:15:17.825 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049127169999999996 2023-09-23T23:15:17.825 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.825 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.825 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.825 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.333511+0000", 2023-09-23T23:15:17.825 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.175069639 2023-09-23T23:15:17.825 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.826 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.826 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.826 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.333570+0000", 2023-09-23T23:15:17.826 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8323999999999999e-05 2023-09-23T23:15:17.826 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.826 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.826 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.826 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.333652+0000", 2023-09-23T23:15:17.826 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2400999999999999e-05 2023-09-23T23:15:17.826 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.826 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.826 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.827 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.827 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.827 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:171309 8.b 8:d16ada6c:::benchmark_data_smithi110_258128_object52491:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T23:15:17.827 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:39.134332+0000", 2023-09-23T23:15:17.827 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 278.608592321, 2023-09-23T23:15:17.827 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.199510887, 2023-09-23T23:15:17.827 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.827 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.827 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:17.827 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:17.827 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:17.827 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 171309 2023-09-23T23:15:17.828 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.828 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.828 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.828 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.828 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134332+0000", 2023-09-23T23:15:17.828 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.828 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.828 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.828 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134332+0000", 2023-09-23T23:15:17.829 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.829 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.829 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.829 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134332+0000", 2023-09-23T23:15:17.829 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:15:17.829 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.829 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.829 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134335+0000", 2023-09-23T23:15:17.829 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1360000000000001e-06 2023-09-23T23:15:17.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.830 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.830 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.830 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134341+0000", 2023-09-23T23:15:17.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7760000000000003e-06 2023-09-23T23:15:17.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.830 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.830 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.830 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134345+0000", 2023-09-23T23:15:17.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7579999999999998e-06 2023-09-23T23:15:17.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.830 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.831 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.831 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134404+0000", 2023-09-23T23:15:17.831 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9416999999999997e-05 2023-09-23T23:15:17.831 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.831 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.831 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:17.831 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134409+0000", 2023-09-23T23:15:17.831 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6120000000000001e-06 2023-09-23T23:15:17.831 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.831 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.831 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.831 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.156116+0000", 2023-09-23T23:15:17.832 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0217073729999999 2023-09-23T23:15:17.832 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.832 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.832 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.832 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.156208+0000", 2023-09-23T23:15:17.832 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2495000000000001e-05 2023-09-23T23:15:17.832 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.832 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.832 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1", 2023-09-23T23:15:17.832 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.156285+0000", 2023-09-23T23:15:17.832 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6478999999999997e-05 2023-09-23T23:15:17.832 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.832 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.833 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:17.833 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.158580+0000", 2023-09-23T23:15:17.833 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022948370000000001 2023-09-23T23:15:17.833 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.833 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.833 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.833 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.333697+0000", 2023-09-23T23:15:17.833 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17511763599999999 2023-09-23T23:15:17.833 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.833 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.833 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.833 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.333755+0000", 2023-09-23T23:15:17.834 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7218e-05 2023-09-23T23:15:17.834 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.834 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.834 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.834 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.333843+0000", 2023-09-23T23:15:17.834 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8541000000000005e-05 2023-09-23T23:15:17.834 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.834 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.834 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.835 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.835 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.835 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:173079 8.b 8:df2b19bb:::benchmark_data_smithi110_258128_object54261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T23:15:17.835 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:42.318557+0000", 2023-09-23T23:15:17.835 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 275.42436718099998, 2023-09-23T23:15:17.835 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.36297001299999998, 2023-09-23T23:15:17.835 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.835 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.835 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:17.835 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:17.835 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:17.835 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 173079 2023-09-23T23:15:17.836 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.836 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.836 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.836 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.836 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.318557+0000", 2023-09-23T23:15:17.836 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.836 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.836 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.836 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.836 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.318557+0000", 2023-09-23T23:15:17.836 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.836 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.836 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.837 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.837 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.318556+0000", 2023-09-23T23:15:17.837 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:17.837 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.837 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.837 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.837 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.318561+0000", 2023-09-23T23:15:17.837 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1309999999999997e-06 2023-09-23T23:15:17.837 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.837 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.837 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.837 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.318571+0000", 2023-09-23T23:15:17.838 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0339999999999999e-05 2023-09-23T23:15:17.838 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.838 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.838 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.838 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.318577+0000", 2023-09-23T23:15:17.838 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2700000000000004e-06 2023-09-23T23:15:17.838 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.838 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.838 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.838 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.321177+0000", 2023-09-23T23:15:17.838 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00259991 2023-09-23T23:15:17.838 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.839 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.839 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.839 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.321296+0000", 2023-09-23T23:15:17.839 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011894499999999999 2023-09-23T23:15:17.839 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.839 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.839 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1", 2023-09-23T23:15:17.839 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.321413+0000", 2023-09-23T23:15:17.839 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011746199999999999 2023-09-23T23:15:17.839 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.839 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.839 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:17.839 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.327193+0000", 2023-09-23T23:15:17.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00577978 2023-09-23T23:15:17.840 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.840 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.840 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.681380+0000", 2023-09-23T23:15:17.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.35418686599999999 2023-09-23T23:15:17.840 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.840 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.840 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.681429+0000", 2023-09-23T23:15:17.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9186e-05 2023-09-23T23:15:17.840 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.841 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.841 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.841 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.681527+0000", 2023-09-23T23:15:17.841 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8640000000000004e-05 2023-09-23T23:15:17.841 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.841 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.841 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.841 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.841 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.841 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:173096 8.b 8:d0616f0c:::benchmark_data_smithi110_258128_object54278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T23:15:17.841 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:42.337603+0000", 2023-09-23T23:15:17.841 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 275.40532192500001, 2023-09-23T23:15:17.841 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.34871130900000002, 2023-09-23T23:15:17.842 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.842 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.842 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:17.842 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:17.842 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:17.842 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 173096 2023-09-23T23:15:17.842 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.842 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.842 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.842 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.842 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337603+0000", 2023-09-23T23:15:17.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.843 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337603+0000", 2023-09-23T23:15:17.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.843 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337601+0000", 2023-09-23T23:15:17.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:15:17.843 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.844 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.844 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337609+0000", 2023-09-23T23:15:17.844 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0360000000000006e-06 2023-09-23T23:15:17.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.844 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.844 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.844 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337624+0000", 2023-09-23T23:15:17.844 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4965e-05 2023-09-23T23:15:17.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.844 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.844 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.844 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337630+0000", 2023-09-23T23:15:17.845 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.665e-06 2023-09-23T23:15:17.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337679+0000", 2023-09-23T23:15:17.845 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9227999999999997e-05 2023-09-23T23:15:17.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337780+0000", 2023-09-23T23:15:17.845 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101327 2023-09-23T23:15:17.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1", 2023-09-23T23:15:17.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337859+0000", 2023-09-23T23:15:17.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9330000000000001e-05 2023-09-23T23:15:17.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:17.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.341195+0000", 2023-09-23T23:15:17.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0033355820000000001 2023-09-23T23:15:17.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.686177+0000", 2023-09-23T23:15:17.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.344981922 2023-09-23T23:15:17.847 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.847 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.847 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.686225+0000", 2023-09-23T23:15:17.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8201999999999997e-05 2023-09-23T23:15:17.847 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.847 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.847 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.686314+0000", 2023-09-23T23:15:17.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8801000000000006e-05 2023-09-23T23:15:17.847 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.847 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.848 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.848 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.848 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.848 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:175297 8.b 8:db4658e4:::benchmark_data_smithi110_258128_object56479:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T23:15:17.848 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:45.168880+0000", 2023-09-23T23:15:17.848 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 272.57404477199998, 2023-09-23T23:15:17.848 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21835064700000001, 2023-09-23T23:15:17.848 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.848 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.848 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:17.848 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:17.848 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:17.849 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 175297 2023-09-23T23:15:17.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.849 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.849 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.849 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.849 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:45.168880+0000", 2023-09-23T23:15:17.849 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.849 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.849 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.849 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:45.168880+0000", 2023-09-23T23:15:17.849 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.850 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.850 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.850 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:45.168879+0000", 2023-09-23T23:15:17.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:15:17.850 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.850 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.850 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.850 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:45.168883+0000", 2023-09-23T23:15:17.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3900000000000003e-06 2023-09-23T23:15:17.850 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.850 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.850 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:45.168894+0000", 2023-09-23T23:15:17.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1003e-05 2023-09-23T23:15:17.851 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.851 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:45.168899+0000", 2023-09-23T23:15:17.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.634e-06 2023-09-23T23:15:17.851 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.851 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:45.168935+0000", 2023-09-23T23:15:17.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5726999999999999e-05 2023-09-23T23:15:17.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.852 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:45.169015+0000", 2023-09-23T23:15:17.852 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0249999999999999e-05 2023-09-23T23:15:17.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1", 2023-09-23T23:15:17.852 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:45.169085+0000", 2023-09-23T23:15:17.852 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9812000000000002e-05 2023-09-23T23:15:17.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:17.853 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:45.172177+0000", 2023-09-23T23:15:17.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003092278 2023-09-23T23:15:17.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.853 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.853 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:45.387082+0000", 2023-09-23T23:15:17.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21490527500000001 2023-09-23T23:15:17.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.853 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.853 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:45.387144+0000", 2023-09-23T23:15:17.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1668999999999995e-05 2023-09-23T23:15:17.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.854 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:45.387230+0000", 2023-09-23T23:15:17.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6539000000000006e-05 2023-09-23T23:15:17.854 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.854 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.854 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.854 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.854 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:196029 8.2 8:466336e2:::benchmark_data_smithi110_258128_object77211:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T23:15:17.854 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:11:14.183060+0000", 2023-09-23T23:15:17.854 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 243.55986492299999, 2023-09-23T23:15:17.855 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25712443899999998, 2023-09-23T23:15:17.855 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.855 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.855 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:17.855 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:17.855 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:17.855 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 196029 2023-09-23T23:15:17.855 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.855 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.855 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.855 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.855 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.183060+0000", 2023-09-23T23:15:17.856 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.856 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.856 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.856 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.856 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.183060+0000", 2023-09-23T23:15:17.856 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.856 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.856 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.856 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.856 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.183059+0000", 2023-09-23T23:15:17.856 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:15:17.856 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.183063+0000", 2023-09-23T23:15:17.857 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.839e-06 2023-09-23T23:15:17.857 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.183073+0000", 2023-09-23T23:15:17.857 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0013999999999999e-05 2023-09-23T23:15:17.857 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.183078+0000", 2023-09-23T23:15:17.858 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8039999999999998e-06 2023-09-23T23:15:17.858 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.858 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.858 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.858 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.183404+0000", 2023-09-23T23:15:17.858 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00032584999999999998 2023-09-23T23:15:17.858 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.858 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.858 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.858 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.183536+0000", 2023-09-23T23:15:17.858 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013172300000000001 2023-09-23T23:15:17.858 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.859 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:15:17.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.183662+0000", 2023-09-23T23:15:17.859 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000125795 2023-09-23T23:15:17.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.859 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:17.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.192207+0000", 2023-09-23T23:15:17.859 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085454810000000006 2023-09-23T23:15:17.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.859 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.198386+0000", 2023-09-23T23:15:17.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0061786020000000001 2023-09-23T23:15:17.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.440100+0000", 2023-09-23T23:15:17.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24171458100000001 2023-09-23T23:15:17.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.440143+0000", 2023-09-23T23:15:17.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2113999999999997e-05 2023-09-23T23:15:17.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.861 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.861 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.440184+0000", 2023-09-23T23:15:17.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1561999999999999e-05 2023-09-23T23:15:17.861 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.861 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.861 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.861 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.861 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.861 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:196044 8.2 8:44317770:::benchmark_data_smithi110_258128_object77226:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T23:15:17.861 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:11:14.206568+0000", 2023-09-23T23:15:17.861 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 243.53635699099999, 2023-09-23T23:15:17.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.233870513, 2023-09-23T23:15:17.862 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.862 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.862 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:17.862 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:17.862 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:17.862 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 196044 2023-09-23T23:15:17.862 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.862 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.862 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.862 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.862 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.206568+0000", 2023-09-23T23:15:17.862 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.863 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.863 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.863 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.863 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.206568+0000", 2023-09-23T23:15:17.863 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.863 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.863 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.863 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.863 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.206566+0000", 2023-09-23T23:15:17.863 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:17.863 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.863 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.863 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.864 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.206572+0000", 2023-09-23T23:15:17.864 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.409e-06 2023-09-23T23:15:17.864 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.864 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.864 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.864 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.206585+0000", 2023-09-23T23:15:17.864 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2876e-05 2023-09-23T23:15:17.864 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.864 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.864 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.864 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.206592+0000", 2023-09-23T23:15:17.864 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4989999999999999e-06 2023-09-23T23:15:17.865 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.865 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.865 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.206641+0000", 2023-09-23T23:15:17.865 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9081e-05 2023-09-23T23:15:17.865 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.865 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.865 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.206762+0000", 2023-09-23T23:15:17.865 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120585 2023-09-23T23:15:17.865 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.866 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:15:17.866 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.206882+0000", 2023-09-23T23:15:17.866 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120318 2023-09-23T23:15:17.866 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.866 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.866 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:17.866 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.211042+0000", 2023-09-23T23:15:17.866 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041602840000000002 2023-09-23T23:15:17.866 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.866 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.866 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.866 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.211467+0000", 2023-09-23T23:15:17.867 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00042466900000000002 2023-09-23T23:15:17.867 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.867 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:17.867 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.440344+0000", 2023-09-23T23:15:17.867 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22887693000000001 2023-09-23T23:15:17.867 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.867 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.867 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.440397+0000", 2023-09-23T23:15:17.867 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3473000000000001e-05 2023-09-23T23:15:17.867 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.868 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.868 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.868 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:14.440438+0000", 2023-09-23T23:15:17.868 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0933999999999998e-05 2023-09-23T23:15:17.868 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.868 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.868 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.868 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.868 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.868 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45164.0:115448 9.e e184/164 9:7589b6f3:::benchmark_data_smithi110_295904_object4267:head v 184'7231, mlcod=184'7226)", 2023-09-23T23:15:17.868 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:11.128813+0000", 2023-09-23T23:15:17.868 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 6.6141111239999999, 2023-09-23T23:15:17.869 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.059964375, 2023-09-23T23:15:17.869 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.869 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.869 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.869 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.869 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.869 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:11.128813+0000", 2023-09-23T23:15:17.869 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.869 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.869 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.869 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.869 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:11.128813+0000", 2023-09-23T23:15:17.869 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.870 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.870 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.870 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.870 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:11.128812+0000", 2023-09-23T23:15:17.870 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:15:17.870 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.870 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.870 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.870 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:11.128821+0000", 2023-09-23T23:15:17.870 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4009999999999997e-06 2023-09-23T23:15:17.870 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.871 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.871 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:11.172883+0000", 2023-09-23T23:15:17.871 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0440626 2023-09-23T23:15:17.871 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.871 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.871 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:11.172900+0000", 2023-09-23T23:15:17.871 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6436e-05 2023-09-23T23:15:17.871 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.871 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.871 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:11.177192+0000", 2023-09-23T23:15:17.872 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0042921529999999999 2023-09-23T23:15:17.872 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.872 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.872 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.872 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:11.177213+0000", 2023-09-23T23:15:17.872 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1073e-05 2023-09-23T23:15:17.872 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.872 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.872 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.872 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:11.188745+0000", 2023-09-23T23:15:17.872 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011532258 2023-09-23T23:15:17.872 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.873 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.873 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:11.188778+0000", 2023-09-23T23:15:17.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2462000000000002e-05 2023-09-23T23:15:17.873 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.873 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.873 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.873 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45164.0:118780 9.4 e184/164 9:240248fa:::benchmark_data_smithi110_295904_object7599:head v 184'7633, mlcod=184'7627)", 2023-09-23T23:15:17.873 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:15.554280+0000", 2023-09-23T23:15:17.873 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.1886446789999998, 2023-09-23T23:15:17.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.023167245999999999, 2023-09-23T23:15:17.874 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.874 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.874 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:15.554280+0000", 2023-09-23T23:15:17.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.874 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:15.554280+0000", 2023-09-23T23:15:17.875 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.875 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.875 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.875 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:15.554279+0000", 2023-09-23T23:15:17.875 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:15:17.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.875 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.875 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.875 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:15.554286+0000", 2023-09-23T23:15:17.875 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1149999999999996e-06 2023-09-23T23:15:17.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.876 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.876 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.876 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:15.554301+0000", 2023-09-23T23:15:17.876 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4168999999999999e-05 2023-09-23T23:15:17.876 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.876 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.876 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.876 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:15.554306+0000", 2023-09-23T23:15:17.876 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6250000000000004e-06 2023-09-23T23:15:17.876 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.876 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.876 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.876 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:15.560147+0000", 2023-09-23T23:15:17.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005840797 2023-09-23T23:15:17.877 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.877 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.877 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.877 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:15.560166+0000", 2023-09-23T23:15:17.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8943000000000001e-05 2023-09-23T23:15:17.877 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.877 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.877 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.877 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:15.577401+0000", 2023-09-23T23:15:17.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017235449999999999 2023-09-23T23:15:17.877 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.878 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.878 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:15.577447+0000", 2023-09-23T23:15:17.878 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5846000000000001e-05 2023-09-23T23:15:17.878 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.878 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.878 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.878 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45164.0:120345 9.c e185/164 9:3ce67898:::benchmark_data_smithi110_295904_object9164:head v 185'7521, mlcod=185'7518)", 2023-09-23T23:15:17.878 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:17.511874+0000", 2023-09-23T23:15:17.878 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.23105102699999999, 2023-09-23T23:15:17.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.025772186999999998, 2023-09-23T23:15:17.879 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.879 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.879 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.879 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.879 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.879 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.511874+0000", 2023-09-23T23:15:17.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.879 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.879 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.879 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.879 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.511874+0000", 2023-09-23T23:15:17.880 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.880 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.880 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.511872+0000", 2023-09-23T23:15:17.880 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:15:17.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.880 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.880 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.511882+0000", 2023-09-23T23:15:17.880 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9510000000000001e-06 2023-09-23T23:15:17.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.511905+0000", 2023-09-23T23:15:17.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2821999999999999e-05 2023-09-23T23:15:17.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.511914+0000", 2023-09-23T23:15:17.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1989999999999995e-06 2023-09-23T23:15:17.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.515724+0000", 2023-09-23T23:15:17.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003809172 2023-09-23T23:15:17.882 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.882 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.882 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.882 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.515746+0000", 2023-09-23T23:15:17.882 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2390000000000001e-05 2023-09-23T23:15:17.882 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.882 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.882 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.882 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.537590+0000", 2023-09-23T23:15:17.882 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.021843508000000001 2023-09-23T23:15:17.882 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.883 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.883 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.537646+0000", 2023-09-23T23:15:17.883 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6233000000000002e-05 2023-09-23T23:15:17.883 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.883 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.883 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.883 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.883 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.883 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45164.0:120342 9.4 e185/164 9:28613bd8:::benchmark_data_smithi110_295904_object9161:head v 185'7722, mlcod=185'7719)", 2023-09-23T23:15:17.883 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:17.514653+0000", 2023-09-23T23:15:17.883 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.22827125600000001, 2023-09-23T23:15:17.883 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.023023981999999998, 2023-09-23T23:15:17.884 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:17.884 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:17.884 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:17.884 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.884 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:17.884 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.514653+0000", 2023-09-23T23:15:17.884 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.884 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.884 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.884 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:17.884 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.514653+0000", 2023-09-23T23:15:17.884 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:17.885 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.885 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.885 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:17.885 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.514652+0000", 2023-09-23T23:15:17.885 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:15:17.885 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.885 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.885 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:17.885 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.514661+0000", 2023-09-23T23:15:17.885 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2660000000000001e-06 2023-09-23T23:15:17.885 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.885 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.885 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:17.886 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.514674+0000", 2023-09-23T23:15:17.886 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3366999999999999e-05 2023-09-23T23:15:17.886 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.886 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.886 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:17.886 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.514680+0000", 2023-09-23T23:15:17.886 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8780000000000003e-06 2023-09-23T23:15:17.886 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.886 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.886 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:17.886 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.517529+0000", 2023-09-23T23:15:17.886 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00284877 2023-09-23T23:15:17.887 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.887 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.887 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:17.887 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.517551+0000", 2023-09-23T23:15:17.887 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2461000000000001e-05 2023-09-23T23:15:17.887 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.887 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.887 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:17.887 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.537650+0000", 2023-09-23T23:15:17.887 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020099215 2023-09-23T23:15:17.887 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:17.887 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:17.887 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:17.888 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:17.537677+0000", 2023-09-23T23:15:17.888 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6930000000000001e-05 2023-09-23T23:15:17.888 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.888 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.888 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.888 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:17.888 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:17.888 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:15:17.889 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:17 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:15:17.889 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:17 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:15:17.889 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:17 smithi110 ceph-mon[212939]: osdmap e185: 8 total, 8 up, 8 in 2023-09-23T23:15:17.889 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:17 smithi110 ceph-mon[212939]: failed to encode map e185 with expected crc 2023-09-23T23:15:17.889 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:17 smithi110 ceph-mon[212939]: failed to encode map e185 with expected crc 2023-09-23T23:15:17.889 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:17 smithi110 ceph-mon[212939]: failed to encode map e185 with expected crc 2023-09-23T23:15:17.889 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:17 smithi110 ceph-mon[212939]: failed to encode map e185 with expected crc 2023-09-23T23:15:17.890 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:17 smithi110 ceph-mon[212939]: failed to encode map e185 with expected crc 2023-09-23T23:15:17.890 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:17 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:15:17.890 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:17 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:15:17.890 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:17 smithi138 ceph-mon[193606]: osdmap e185: 8 total, 8 up, 8 in 2023-09-23T23:15:17.890 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:17 smithi138 ceph-mon[193606]: failed to encode map e185 with expected crc 2023-09-23T23:15:17.890 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:17 smithi138 ceph-mon[193606]: failed to encode map e185 with expected crc 2023-09-23T23:15:17.890 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:17 smithi138 ceph-mon[193606]: failed to encode map e185 with expected crc 2023-09-23T23:15:17.890 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:17 smithi138 ceph-mon[193606]: failed to encode map e185 with expected crc 2023-09-23T23:15:17.890 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:17 smithi138 ceph-mon[193606]: failed to encode map e185 with expected crc 2023-09-23T23:15:17.890 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:17 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:15:17.891 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:17 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:15:17.891 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:17 smithi110 ceph-mon[221102]: osdmap e185: 8 total, 8 up, 8 in 2023-09-23T23:15:17.891 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:17 smithi110 ceph-mon[221102]: failed to encode map e185 with expected crc 2023-09-23T23:15:17.891 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:17 smithi110 ceph-mon[221102]: failed to encode map e185 with expected crc 2023-09-23T23:15:17.891 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:17 smithi110 ceph-mon[221102]: failed to encode map e185 with expected crc 2023-09-23T23:15:17.891 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:17 smithi110 ceph-mon[221102]: failed to encode map e185 with expected crc 2023-09-23T23:15:17.891 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:17 smithi110 ceph-mon[221102]: failed to encode map e185 with expected crc 2023-09-23T23:15:18.246 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:18.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:18.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:18.349 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:15:18.413 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:18.425 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:15:18.425 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:15:18.425 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:15:18.425 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:15:18.425 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:15:18.425 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:15:18.425 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:15:18.425 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:15:18.426 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:15:18.426 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:15:18.426 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:15:18.426 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:15:18.426 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:15:18.426 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:15:18.426 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:15:18.426 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:15:18.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:18 smithi138 ceph-mon[193606]: pgmap v608: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 8.1 MiB/s wr, 764 op/s 2023-09-23T23:15:18.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1360118723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:18.726 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:15:18.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:18.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:15:18.748 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:15:18.835 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:15:18.863 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:18 smithi110 ceph-mon[212939]: pgmap v608: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 8.1 MiB/s wr, 764 op/s 2023-09-23T23:15:18.863 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1360118723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:18.864 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:18 smithi110 ceph-mon[221102]: pgmap v608: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 8.1 MiB/s wr, 764 op/s 2023-09-23T23:15:18.864 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1360118723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:18.865 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:19.288 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:15:19.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:19.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:15:19.324 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:15:19.395 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:15:19.465 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:19.579 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:15:19.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:19.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:15:19.684 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:19.733 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:19.813 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:19.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:19.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:19.916 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:19.993 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:20.110 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:20.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:20.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:20.205 INFO:teuthology.orchestra.run.smithi110.stdout:moved 52 / 297 (17.5084%) 2023-09-23T23:15:20.206 INFO:teuthology.orchestra.run.smithi110.stdout:avg 37.125 2023-09-23T23:15:20.206 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 7.38982 -> 7.7045 (expected baseline 5.69951) 2023-09-23T23:15:20.206 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.5 with 25 -> 29 pgs (0.673401 -> 0.781145 * mean) 2023-09-23T23:15:20.206 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.3 with 47 -> 43 pgs (1.26599 -> 1.15825 * mean) 2023-09-23T23:15:20.206 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:15:20.207 INFO:teuthology.orchestra.run.smithi110.stdout:oload 110 2023-09-23T23:15:20.207 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 1 2023-09-23T23:15:20.207 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:15:20.207 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0298 2023-09-23T23:15:20.208 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.0328 2023-09-23T23:15:20.208 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 weight 1.0000 -> 0.6329 2023-09-23T23:15:20.208 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 weight 1.0000 -> 0.6938 2023-09-23T23:15:20.208 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 weight 1.0000 -> 0.6987 2023-09-23T23:15:20.208 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 weight 0.7527 -> 0.6487 2023-09-23T23:15:20.222 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:15:20.231 INFO:teuthology.orchestra.run.smithi110.stderr:no change 2023-09-23T23:15:20.260 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:20.263 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:15:20 smithi110 conmon[198813]: 2023-09-23T23:15:20.200+0000 7ff0150b6700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-09-23T23:15:20.318 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:15:20.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:20.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:15:20.425 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:20.469 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:20.530 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:20 smithi110 ceph-mon[212939]: pgmap v609: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 38 KiB/s wr, 732 op/s 2023-09-23T23:15:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:20 smithi138 ceph-mon[193606]: pgmap v609: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 38 KiB/s wr, 732 op/s 2023-09-23T23:15:20.792 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:20.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:20.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:20.897 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:15:20.944 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:20 smithi110 ceph-mon[221102]: pgmap v609: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 38 KiB/s wr, 732 op/s 2023-09-23T23:15:20.954 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:21.135 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:15:21.295 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:15:21.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:21.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:15:21.330 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T23:15:21.330 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T23:15:21.412 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:21.413 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:15:21.413 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:15:21.430 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:15:21.457 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:21.649 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:21.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:21.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:21.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:21 smithi138 ceph-mon[193606]: from='client.45266 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:15:21.757 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:21.793 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:21.836 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:21.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:21.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:21.860 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:15:21.939 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:21.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:21 smithi110 ceph-mon[221102]: from='client.45266 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:15:21.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:21 smithi110 ceph-mon[212939]: from='client.45266 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:15:21.995 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:22.317 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:22.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:22.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:22.328 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:15:22.428 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:15:22.445 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:15:22.482 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:22.492 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:15:22.614 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:15:22.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:22.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:15:22.623 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:22 smithi138 ceph-mon[193606]: pgmap v610: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2023-09-23T23:15:22.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:22 smithi110 ceph-mon[221102]: pgmap v610: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2023-09-23T23:15:22.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:22 smithi110 ceph-mon[212939]: pgmap v610: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2023-09-23T23:15:22.722 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:15:22.802 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:22.887 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:15:24.074 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:15:24.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:24.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:15:24.178 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:15:24.247 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:24.495 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:24 smithi110 ceph-mon[212939]: pgmap v611: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-09-23T23:15:24.495 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:24 smithi110 ceph-mon[221102]: pgmap v611: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-09-23T23:15:24.654 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:24 smithi138 ceph-mon[193606]: pgmap v611: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-09-23T23:15:24.664 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:15:24.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:24.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:15:24.779 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:24.849 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:24.938 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:15:24.938 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:15:25.909 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:15:26.116 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:26.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:26.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:26.231 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:15:26.286 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:26.367 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:15:26.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:26.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:15:26.409 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:26 smithi110 ceph-mon[212939]: pgmap v612: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-23T23:15:26.409 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:26 smithi110 ceph-mon[221102]: pgmap v612: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-23T23:15:26.485 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:26.539 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:26 smithi138 ceph-mon[193606]: pgmap v612: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-23T23:15:26.543 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:26.562 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:15:26.565 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:15:26.598 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:15:26.636 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:15:26.650 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:26.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:26.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:26.674 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:15:26.727 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:15:26.755 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:26.781 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:15:26.803 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:26.815 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:15:26.853 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:15:26.856 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:15:26.856 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:15:26.856 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:15:26.856 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:15:26.856 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:15:26.856 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:15:26.856 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:15:26.856 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:15:26.892 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:26.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:26.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:26.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:15:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:15:26] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.43.0" 2023-09-23T23:15:26.998 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:27.082 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:27.204 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:27.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:27.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:27.313 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:27.382 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:27.413 INFO:teuthology.orchestra.run.smithi110.stdout:moved 10 / 297 (3.367%) 2023-09-23T23:15:27.414 INFO:teuthology.orchestra.run.smithi110.stdout:avg 37.125 2023-09-23T23:15:27.414 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 7.38982 -> 7.68826 (expected baseline 5.69951) 2023-09-23T23:15:27.414 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.5 with 25 -> 23 pgs (0.673401 -> 0.619529 * mean) 2023-09-23T23:15:27.414 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.3 with 47 -> 49 pgs (1.26599 -> 1.31987 * mean) 2023-09-23T23:15:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:15:27.414 INFO:teuthology.orchestra.run.smithi110.stdout:oload 110 2023-09-23T23:15:27.414 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 0.05 2023-09-23T23:15:27.414 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:15:27.414 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0288 2023-09-23T23:15:27.414 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.0317 2023-09-23T23:15:27.415 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 weight 1.0000 -> 0.9500 2023-09-23T23:15:27.415 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 weight 1.0000 -> 0.9500 2023-09-23T23:15:27.415 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 weight 1.0000 -> 0.9500 2023-09-23T23:15:27.415 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 weight 0.7527 -> 0.7027 2023-09-23T23:15:27.417 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:15:27.426 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:27.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:27.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:27.530 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:27.587 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:27.640 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:27.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:27.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:27.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:27 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3228227949' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:27.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:27 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2562837831' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:15:27.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:27 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:15:27.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:27 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'46052','7':'62260'}"}]: dispatch 2023-09-23T23:15:27.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:27 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'46052','7':'62260'}"}]: dispatch 2023-09-23T23:15:27.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:27 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3228227949' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:27.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:27 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2562837831' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:15:27.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:27 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:15:27.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:27 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'46052','7':'62260'}"}]: dispatch 2023-09-23T23:15:27.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:27 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'46052','7':'62260'}"}]: dispatch 2023-09-23T23:15:27.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:27 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3228227949' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:27.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:27 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2562837831' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:15:27.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:27 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:15:27.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:27 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'46052','7':'62260'}"}]: dispatch 2023-09-23T23:15:27.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:27 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'46052','7':'62260'}"}]: dispatch 2023-09-23T23:15:27.744 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:27.770 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:28.059 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:28.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:28.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:28.164 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:28.191 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:28.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[212939]: pgmap v613: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-09-23T23:15:28.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[212939]: from='client.35307 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:15:28.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:15:28.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:15:28.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'46052','7':'62260'}"}]': finished 2023-09-23T23:15:28.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[212939]: osdmap e186: 8 total, 8 up, 8 in 2023-09-23T23:15:28.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[212939]: failed to encode map e186 with expected crc 2023-09-23T23:15:28.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[212939]: failed to encode map e186 with expected crc 2023-09-23T23:15:28.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[212939]: failed to encode map e186 with expected crc 2023-09-23T23:15:28.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[212939]: failed to encode map e186 with expected crc 2023-09-23T23:15:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:15:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[212939]: osdmap e187: 8 total, 8 up, 8 in 2023-09-23T23:15:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[221102]: pgmap v613: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-09-23T23:15:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[221102]: from='client.35307 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:15:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:15:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:15:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'46052','7':'62260'}"}]': finished 2023-09-23T23:15:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[221102]: osdmap e186: 8 total, 8 up, 8 in 2023-09-23T23:15:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[221102]: failed to encode map e186 with expected crc 2023-09-23T23:15:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[221102]: failed to encode map e186 with expected crc 2023-09-23T23:15:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[221102]: failed to encode map e186 with expected crc 2023-09-23T23:15:28.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[221102]: failed to encode map e186 with expected crc 2023-09-23T23:15:28.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:15:28.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:28 smithi110 ceph-mon[221102]: osdmap e187: 8 total, 8 up, 8 in 2023-09-23T23:15:28.585 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:15:28.585 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:15:28.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:15:28.585 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:15:28.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.585 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34269.0:216429 7.0 7:0f58a1f3:::benchmark_data_smithi110_236826_object103721:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-09-23T23:15:28.585 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:07:29.069337+0000", 2023-09-23T23:15:28.585 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 479.51194156000003, 2023-09-23T23:15:28.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3713624520000001, 2023-09-23T23:15:28.586 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.586 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.586 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.586 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34269", 2023-09-23T23:15:28.586 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:15:28.586 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 216429 2023-09-23T23:15:28.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.586 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:29.069337+0000", 2023-09-23T23:15:28.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:29.069337+0000", 2023-09-23T23:15:28.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:29.069336+0000", 2023-09-23T23:15:28.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:28.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:29.069342+0000", 2023-09-23T23:15:28.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7169999999999996e-06 2023-09-23T23:15:28.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:29.069351+0000", 2023-09-23T23:15:28.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5230000000000001e-06 2023-09-23T23:15:28.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:29.069358+0000", 2023-09-23T23:15:28.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4579999999999998e-06 2023-09-23T23:15:28.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:29.069649+0000", 2023-09-23T23:15:28.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00029184600000000001 2023-09-23T23:15:28.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:29.069660+0000", 2023-09-23T23:15:28.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0784999999999999e-05 2023-09-23T23:15:28.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.065338+0000", 2023-09-23T23:15:28.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99567820600000001 2023-09-23T23:15:28.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.065351+0000", 2023-09-23T23:15:28.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2500999999999999e-05 2023-09-23T23:15:28.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.426230+0000", 2023-09-23T23:15:28.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.360879282 2023-09-23T23:15:28.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.426417+0000", 2023-09-23T23:15:28.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018662999999999999 2023-09-23T23:15:28.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:15:28.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.426590+0000", 2023-09-23T23:15:28.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000173182 2023-09-23T23:15:28.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.435046+0000", 2023-09-23T23:15:28.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0084556419999999993 2023-09-23T23:15:28.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.435361+0000", 2023-09-23T23:15:28.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00031581900000000001 2023-09-23T23:15:28.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.440498+0000", 2023-09-23T23:15:28.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005136013 2023-09-23T23:15:28.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.440553+0000", 2023-09-23T23:15:28.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5040999999999997e-05 2023-09-23T23:15:28.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.440700+0000", 2023-09-23T23:15:28.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014729200000000001 2023-09-23T23:15:28.594 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.595 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.595 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.595 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34269.0:216450 7.0 7:0195d78c:::benchmark_data_smithi110_236826_object103742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-09-23T23:15:28.595 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:07:29.127066+0000", 2023-09-23T23:15:28.595 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 479.45421334000002, 2023-09-23T23:15:28.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.318362665, 2023-09-23T23:15:28.595 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.595 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.595 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.596 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34269", 2023-09-23T23:15:28.596 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:15:28.596 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 216450 2023-09-23T23:15:28.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.596 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:29.127066+0000", 2023-09-23T23:15:28.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:29.127066+0000", 2023-09-23T23:15:28.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:29.127064+0000", 2023-09-23T23:15:28.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:28.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:29.127071+0000", 2023-09-23T23:15:28.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7340000000000002e-06 2023-09-23T23:15:28.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:29.127085+0000", 2023-09-23T23:15:28.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4019000000000001e-05 2023-09-23T23:15:28.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:29.127090+0000", 2023-09-23T23:15:28.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3249999999999998e-06 2023-09-23T23:15:28.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:29.127162+0000", 2023-09-23T23:15:28.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1929999999999997e-05 2023-09-23T23:15:28.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:29.127173+0000", 2023-09-23T23:15:28.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0413e-05 2023-09-23T23:15:28.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.065382+0000", 2023-09-23T23:15:28.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.93820955100000003 2023-09-23T23:15:28.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.065390+0000", 2023-09-23T23:15:28.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4340000000000003e-06 2023-09-23T23:15:28.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.430940+0000", 2023-09-23T23:15:28.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.36555016 2023-09-23T23:15:28.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.431031+0000", 2023-09-23T23:15:28.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1221e-05 2023-09-23T23:15:28.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:15:28.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.431115+0000", 2023-09-23T23:15:28.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4355000000000001e-05 2023-09-23T23:15:28.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.440374+0000", 2023-09-23T23:15:28.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0092588010000000005 2023-09-23T23:15:28.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.440733+0000", 2023-09-23T23:15:28.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00035841200000000001 2023-09-23T23:15:28.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.445283+0000", 2023-09-23T23:15:28.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0045500219999999999 2023-09-23T23:15:28.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.445325+0000", 2023-09-23T23:15:28.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2602999999999999e-05 2023-09-23T23:15:28.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.445428+0000", 2023-09-23T23:15:28.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103056 2023-09-23T23:15:28.604 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.604 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.604 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.604 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34269.0:216447 7.b 7:d1cadb9e:::benchmark_data_smithi110_236826_object103739:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e113)", 2023-09-23T23:15:28.604 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:07:30.057032+0000", 2023-09-23T23:15:28.604 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 478.52424685900002, 2023-09-23T23:15:28.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3826280500000001, 2023-09-23T23:15:28.605 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.605 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.605 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.605 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34269", 2023-09-23T23:15:28.605 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:15:28.605 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 216447 2023-09-23T23:15:28.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.605 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.057032+0000", 2023-09-23T23:15:28.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.057032+0000", 2023-09-23T23:15:28.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.057030+0000", 2023-09-23T23:15:28.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:28.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.057041+0000", 2023-09-23T23:15:28.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0168e-05 2023-09-23T23:15:28.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.057055+0000", 2023-09-23T23:15:28.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4894e-05 2023-09-23T23:15:28.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.057064+0000", 2023-09-23T23:15:28.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0560000000000005e-06 2023-09-23T23:15:28.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.057145+0000", 2023-09-23T23:15:28.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1841000000000005e-05 2023-09-23T23:15:28.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op must wait for map", 2023-09-23T23:15:28.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.057157+0000", 2023-09-23T23:15:28.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1659e-05 2023-09-23T23:15:28.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.067736+0000", 2023-09-23T23:15:28.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010578974999999999 2023-09-23T23:15:28.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.067749+0000", 2023-09-23T23:15:28.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2982999999999999e-05 2023-09-23T23:15:28.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.429793+0000", 2023-09-23T23:15:28.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3620436490000001 2023-09-23T23:15:28.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.429949+0000", 2023-09-23T23:15:28.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000155954 2023-09-23T23:15:28.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6", 2023-09-23T23:15:28.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.430050+0000", 2023-09-23T23:15:28.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101383 2023-09-23T23:15:28.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.438217+0000", 2023-09-23T23:15:28.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0081669280000000004 2023-09-23T23:15:28.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.438254+0000", 2023-09-23T23:15:28.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6711999999999997e-05 2023-09-23T23:15:28.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.439515+0000", 2023-09-23T23:15:28.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012612439999999999 2023-09-23T23:15:28.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.439563+0000", 2023-09-23T23:15:28.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8238000000000001e-05 2023-09-23T23:15:28.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.439660+0000", 2023-09-23T23:15:28.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7027000000000005e-05 2023-09-23T23:15:28.613 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.613 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.614 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.614 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34269.0:216449 7.b 7:dae894dd:::benchmark_data_smithi110_236826_object103741:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e113)", 2023-09-23T23:15:28.614 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:07:30.057082+0000", 2023-09-23T23:15:28.614 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 478.52419691199998, 2023-09-23T23:15:28.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3827471, 2023-09-23T23:15:28.614 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.614 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.614 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.614 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34269", 2023-09-23T23:15:28.615 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:15:28.615 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 216449 2023-09-23T23:15:28.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.615 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.057082+0000", 2023-09-23T23:15:28.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.057082+0000", 2023-09-23T23:15:28.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.057081+0000", 2023-09-23T23:15:28.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:15:28.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.057085+0000", 2023-09-23T23:15:28.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0679999999999998e-06 2023-09-23T23:15:28.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.057092+0000", 2023-09-23T23:15:28.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.029e-06 2023-09-23T23:15:28.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.057098+0000", 2023-09-23T23:15:28.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.541e-06 2023-09-23T23:15:28.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.057218+0000", 2023-09-23T23:15:28.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000119799 2023-09-23T23:15:28.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:15:28.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.057229+0000", 2023-09-23T23:15:28.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1262e-05 2023-09-23T23:15:28.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.067785+0000", 2023-09-23T23:15:28.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0105562 2023-09-23T23:15:28.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:30.067795+0000", 2023-09-23T23:15:28.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2830000000000007e-06 2023-09-23T23:15:28.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.434547+0000", 2023-09-23T23:15:28.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3667525389999999 2023-09-23T23:15:28.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.434678+0000", 2023-09-23T23:15:28.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013118099999999999 2023-09-23T23:15:28.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6", 2023-09-23T23:15:28.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.434814+0000", 2023-09-23T23:15:28.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013607300000000001 2023-09-23T23:15:28.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.438248+0000", 2023-09-23T23:15:28.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034338039999999999 2023-09-23T23:15:28.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.438259+0000", 2023-09-23T23:15:28.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0898e-05 2023-09-23T23:15:28.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.439705+0000", 2023-09-23T23:15:28.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014454750000000001 2023-09-23T23:15:28.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.439738+0000", 2023-09-23T23:15:28.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3259999999999997e-05 2023-09-23T23:15:28.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:07:31.439829+0000", 2023-09-23T23:15:28.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1346000000000003e-05 2023-09-23T23:15:28.623 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.623 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.623 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.623 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:79748 8.4 8:20294c17:::benchmark_data_smithi110_258128_object79747:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T23:15:28.623 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:08:44.103297+0000", 2023-09-23T23:15:28.623 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 404.477982134, 2023-09-23T23:15:28.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1079633360000001, 2023-09-23T23:15:28.624 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.624 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:15:28.624 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.624 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:28.624 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:28.624 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 79748 2023-09-23T23:15:28.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.624 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103297+0000", 2023-09-23T23:15:28.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103297+0000", 2023-09-23T23:15:28.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103295+0000", 2023-09-23T23:15:28.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:28.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103328+0000", 2023-09-23T23:15:28.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3098000000000001e-05 2023-09-23T23:15:28.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103342+0000", 2023-09-23T23:15:28.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4131000000000001e-05 2023-09-23T23:15:28.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103350+0000", 2023-09-23T23:15:28.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4490000000000002e-06 2023-09-23T23:15:28.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103485+0000", 2023-09-23T23:15:28.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013490999999999999 2023-09-23T23:15:28.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103606+0000", 2023-09-23T23:15:28.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000121727 2023-09-23T23:15:28.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4", 2023-09-23T23:15:28.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103741+0000", 2023-09-23T23:15:28.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013452700000000001 2023-09-23T23:15:28.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.113503+0000", 2023-09-23T23:15:28.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0097624410000000002 2023-09-23T23:15:28.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.113515+0000", 2023-09-23T23:15:28.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1352000000000001e-05 2023-09-23T23:15:28.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.108207+0000", 2023-09-23T23:15:28.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99469248300000002 2023-09-23T23:15:28.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.108219+0000", 2023-09-23T23:15:28.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1865000000000001e-05 2023-09-23T23:15:28.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.197996+0000", 2023-09-23T23:15:28.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0897770739999999 2023-09-23T23:15:28.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-09-23T23:15:28.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.198757+0000", 2023-09-23T23:15:28.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000760941 2023-09-23T23:15:28.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.211195+0000", 2023-09-23T23:15:28.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012438015 2023-09-23T23:15:28.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.211260+0000", 2023-09-23T23:15:28.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4974999999999994e-05 2023-09-23T23:15:28.631 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.631 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.631 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.632 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:79761 8.d 8:b0e3b43c:::benchmark_data_smithi110_258128_object79760:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T23:15:28.632 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:08:44.127083+0000", 2023-09-23T23:15:28.632 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 404.45419626099999, 2023-09-23T23:15:28.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0759149159999999, 2023-09-23T23:15:28.632 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.632 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.632 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.632 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:28.632 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:28.632 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 79761 2023-09-23T23:15:28.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.633 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127083+0000", 2023-09-23T23:15:28.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127083+0000", 2023-09-23T23:15:28.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127081+0000", 2023-09-23T23:15:28.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:28.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127142+0000", 2023-09-23T23:15:28.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.037e-05 2023-09-23T23:15:28.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127156+0000", 2023-09-23T23:15:28.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4411e-05 2023-09-23T23:15:28.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127163+0000", 2023-09-23T23:15:28.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9210000000000003e-06 2023-09-23T23:15:28.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127226+0000", 2023-09-23T23:15:28.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2739999999999999e-05 2023-09-23T23:15:28.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127236+0000", 2023-09-23T23:15:28.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0417999999999999e-05 2023-09-23T23:15:28.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.106383+0000", 2023-09-23T23:15:28.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97914661700000005 2023-09-23T23:15:28.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.106399+0000", 2023-09-23T23:15:28.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6728999999999999e-05 2023-09-23T23:15:28.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.188815+0000", 2023-09-23T23:15:28.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0824155440000001 2023-09-23T23:15:28.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.189005+0000", 2023-09-23T23:15:28.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000190354 2023-09-23T23:15:28.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-09-23T23:15:28.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.189214+0000", 2023-09-23T23:15:28.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020901900000000001 2023-09-23T23:15:28.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.195871+0000", 2023-09-23T23:15:28.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066568920000000002 2023-09-23T23:15:28.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.195882+0000", 2023-09-23T23:15:28.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1206000000000001e-05 2023-09-23T23:15:28.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.201793+0000", 2023-09-23T23:15:28.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0059103649999999999 2023-09-23T23:15:28.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.201868+0000", 2023-09-23T23:15:28.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4945999999999997e-05 2023-09-23T23:15:28.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.202998+0000", 2023-09-23T23:15:28.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00112987 2023-09-23T23:15:28.640 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.640 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.640 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.640 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:79762 8.d 8:b39b1064:::benchmark_data_smithi110_258128_object79761:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T23:15:28.640 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:08:44.127180+0000", 2023-09-23T23:15:28.641 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 404.45409910900003, 2023-09-23T23:15:28.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0765545470000002, 2023-09-23T23:15:28.641 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.641 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.641 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.641 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:28.641 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:28.641 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 79762 2023-09-23T23:15:28.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.641 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127180+0000", 2023-09-23T23:15:28.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127180+0000", 2023-09-23T23:15:28.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127179+0000", 2023-09-23T23:15:28.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:15:28.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127213+0000", 2023-09-23T23:15:28.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3992000000000003e-05 2023-09-23T23:15:28.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127220+0000", 2023-09-23T23:15:28.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7279999999999998e-06 2023-09-23T23:15:28.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127225+0000", 2023-09-23T23:15:28.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5909999999999998e-06 2023-09-23T23:15:28.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127275+0000", 2023-09-23T23:15:28.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0458999999999999e-05 2023-09-23T23:15:28.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127282+0000", 2023-09-23T23:15:28.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3490000000000003e-06 2023-09-23T23:15:28.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.106425+0000", 2023-09-23T23:15:28.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97914234899999997 2023-09-23T23:15:28.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.106431+0000", 2023-09-23T23:15:28.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7229999999999998e-06 2023-09-23T23:15:28.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.193155+0000", 2023-09-23T23:15:28.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0867236360000001 2023-09-23T23:15:28.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.193297+0000", 2023-09-23T23:15:28.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000141843 2023-09-23T23:15:28.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-09-23T23:15:28.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.193419+0000", 2023-09-23T23:15:28.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012232200000000001 2023-09-23T23:15:28.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.198570+0000", 2023-09-23T23:15:28.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051506369999999996 2023-09-23T23:15:28.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.201744+0000", 2023-09-23T23:15:28.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031740499999999999 2023-09-23T23:15:28.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.203163+0000", 2023-09-23T23:15:28.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001419575 2023-09-23T23:15:28.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.203203+0000", 2023-09-23T23:15:28.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9728000000000003e-05 2023-09-23T23:15:28.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.203734+0000", 2023-09-23T23:15:28.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00053115499999999995 2023-09-23T23:15:28.649 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.649 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.649 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.649 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:79763 8.d 8:b52536a9:::benchmark_data_smithi110_258128_object79762:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T23:15:28.649 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:08:44.127232+0000", 2023-09-23T23:15:28.649 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 404.45404646499998, 2023-09-23T23:15:28.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0832427529999999, 2023-09-23T23:15:28.649 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.650 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.650 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.650 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:28.650 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:28.650 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 79763 2023-09-23T23:15:28.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.650 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127232+0000", 2023-09-23T23:15:28.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127232+0000", 2023-09-23T23:15:28.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127232+0000", 2023-09-23T23:15:28.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:15:28.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127262+0000", 2023-09-23T23:15:28.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.04e-05 2023-09-23T23:15:28.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127269+0000", 2023-09-23T23:15:28.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8759999999999999e-06 2023-09-23T23:15:28.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127274+0000", 2023-09-23T23:15:28.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.357e-06 2023-09-23T23:15:28.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127309+0000", 2023-09-23T23:15:28.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4946000000000001e-05 2023-09-23T23:15:28.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127318+0000", 2023-09-23T23:15:28.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9269999999999994e-06 2023-09-23T23:15:28.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.106461+0000", 2023-09-23T23:15:28.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97914303499999999 2023-09-23T23:15:28.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.106464+0000", 2023-09-23T23:15:28.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.743e-06 2023-09-23T23:15:28.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.196067+0000", 2023-09-23T23:15:28.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.089602677 2023-09-23T23:15:28.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.196197+0000", 2023-09-23T23:15:28.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013014400000000001 2023-09-23T23:15:28.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-09-23T23:15:28.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.196329+0000", 2023-09-23T23:15:28.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013207200000000001 2023-09-23T23:15:28.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.201686+0000", 2023-09-23T23:15:28.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0053572860000000002 2023-09-23T23:15:28.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.203099+0000", 2023-09-23T23:15:28.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001412558 2023-09-23T23:15:28.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.209540+0000", 2023-09-23T23:15:28.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064405979999999996 2023-09-23T23:15:28.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.209606+0000", 2023-09-23T23:15:28.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6402999999999994e-05 2023-09-23T23:15:28.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.210475+0000", 2023-09-23T23:15:28.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00086918300000000004 2023-09-23T23:15:28.657 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.657 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.658 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.658 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:79775 8.d 8:b42b6b80:::benchmark_data_smithi110_258128_object79774:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T23:15:28.658 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:08:44.135988+0000", 2023-09-23T23:15:28.658 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 404.44529095899998, 2023-09-23T23:15:28.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0751020269999998, 2023-09-23T23:15:28.658 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.658 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.658 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.658 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:28.658 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:28.658 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 79775 2023-09-23T23:15:28.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.659 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.135988+0000", 2023-09-23T23:15:28.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.135988+0000", 2023-09-23T23:15:28.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.135987+0000", 2023-09-23T23:15:28.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:15:28.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136044+0000", 2023-09-23T23:15:28.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6675999999999997e-05 2023-09-23T23:15:28.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136053+0000", 2023-09-23T23:15:28.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9609999999999994e-06 2023-09-23T23:15:28.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136057+0000", 2023-09-23T23:15:28.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9580000000000001e-06 2023-09-23T23:15:28.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136098+0000", 2023-09-23T23:15:28.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1551999999999998e-05 2023-09-23T23:15:28.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136105+0000", 2023-09-23T23:15:28.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6080000000000001e-06 2023-09-23T23:15:28.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.106481+0000", 2023-09-23T23:15:28.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97037604700000002 2023-09-23T23:15:28.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.106488+0000", 2023-09-23T23:15:28.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1489999999999997e-06 2023-09-23T23:15:28.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.198642+0000", 2023-09-23T23:15:28.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.092154125 2023-09-23T23:15:28.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.198803+0000", 2023-09-23T23:15:28.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000161087 2023-09-23T23:15:28.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-09-23T23:15:28.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.198989+0000", 2023-09-23T23:15:28.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018580499999999999 2023-09-23T23:15:28.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.203112+0000", 2023-09-23T23:15:28.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004122863 2023-09-23T23:15:28.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.203791+0000", 2023-09-23T23:15:28.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00067863899999999998 2023-09-23T23:15:28.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.210523+0000", 2023-09-23T23:15:28.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0067325019999999996 2023-09-23T23:15:28.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.210562+0000", 2023-09-23T23:15:28.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9252000000000003e-05 2023-09-23T23:15:28.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.211090+0000", 2023-09-23T23:15:28.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00052762200000000005 2023-09-23T23:15:28.666 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.666 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.666 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.666 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:79776 8.4 8:299b8268:::benchmark_data_smithi110_258128_object79775:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T23:15:28.666 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:08:44.136068+0000", 2023-09-23T23:15:28.666 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 404.44521077600001, 2023-09-23T23:15:28.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0756627179999998, 2023-09-23T23:15:28.667 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.667 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.667 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.667 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:28.667 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:28.667 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 79776 2023-09-23T23:15:28.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.667 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136068+0000", 2023-09-23T23:15:28.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136068+0000", 2023-09-23T23:15:28.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136068+0000", 2023-09-23T23:15:28.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:15:28.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136143+0000", 2023-09-23T23:15:28.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5354999999999999e-05 2023-09-23T23:15:28.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136150+0000", 2023-09-23T23:15:28.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9940000000000003e-06 2023-09-23T23:15:28.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136153+0000", 2023-09-23T23:15:28.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.016e-06 2023-09-23T23:15:28.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136190+0000", 2023-09-23T23:15:28.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6891999999999998e-05 2023-09-23T23:15:28.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136196+0000", 2023-09-23T23:15:28.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.507e-06 2023-09-23T23:15:28.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.108263+0000", 2023-09-23T23:15:28.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97206721200000001 2023-09-23T23:15:28.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.108272+0000", 2023-09-23T23:15:28.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6899999999999998e-06 2023-09-23T23:15:28.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.199665+0000", 2023-09-23T23:15:28.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.09139388 2023-09-23T23:15:28.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.199765+0000", 2023-09-23T23:15:28.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9269000000000001e-05 2023-09-23T23:15:28.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:15:28.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.199873+0000", 2023-09-23T23:15:28.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108823 2023-09-23T23:15:28.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.202326+0000", 2023-09-23T23:15:28.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024524310000000001 2023-09-23T23:15:28.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.204652+0000", 2023-09-23T23:15:28.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023259560000000001 2023-09-23T23:15:28.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.211305+0000", 2023-09-23T23:15:28.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066528059999999998 2023-09-23T23:15:28.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.211339+0000", 2023-09-23T23:15:28.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4777999999999998e-05 2023-09-23T23:15:28.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.211731+0000", 2023-09-23T23:15:28.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00039142099999999998 2023-09-23T23:15:28.674 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.675 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.675 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.675 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:171303 8.0 8:00d2d297:::benchmark_data_smithi110_258128_object52485:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T23:15:28.675 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:39.128982+0000", 2023-09-23T23:15:28.675 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 289.45229683399998, 2023-09-23T23:15:28.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1937137520000001, 2023-09-23T23:15:28.675 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.675 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.675 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.676 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:28.676 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:28.676 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 171303 2023-09-23T23:15:28.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.676 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128982+0000", 2023-09-23T23:15:28.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128982+0000", 2023-09-23T23:15:28.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128982+0000", 2023-09-23T23:15:28.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:15:28.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128984+0000", 2023-09-23T23:15:28.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8650000000000001e-06 2023-09-23T23:15:28.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128991+0000", 2023-09-23T23:15:28.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9319999999999999e-06 2023-09-23T23:15:28.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128997+0000", 2023-09-23T23:15:28.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6570000000000002e-06 2023-09-23T23:15:28.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.129048+0000", 2023-09-23T23:15:28.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0855e-05 2023-09-23T23:15:28.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.129059+0000", 2023-09-23T23:15:28.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0896999999999999e-05 2023-09-23T23:15:28.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.170835+0000", 2023-09-23T23:15:28.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.04177663 2023-09-23T23:15:28.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.172847+0000", 2023-09-23T23:15:28.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020116550000000002 2023-09-23T23:15:28.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1", 2023-09-23T23:15:28.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.172928+0000", 2023-09-23T23:15:28.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0867999999999999e-05 2023-09-23T23:15:28.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.180050+0000", 2023-09-23T23:15:28.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0071224879999999997 2023-09-23T23:15:28.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.322596+0000", 2023-09-23T23:15:28.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.14254508599999999 2023-09-23T23:15:28.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.322635+0000", 2023-09-23T23:15:28.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9564000000000003e-05 2023-09-23T23:15:28.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.322696+0000", 2023-09-23T23:15:28.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0732000000000001e-05 2023-09-23T23:15:28.682 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.682 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.682 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.683 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:186176 8.8 8:12f1ade7:::benchmark_data_smithi110_258128_object67358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T23:15:28.683 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:58.477323+0000", 2023-09-23T23:15:28.683 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 270.10395577600002, 2023-09-23T23:15:28.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4157779150000001, 2023-09-23T23:15:28.683 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.683 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:15:28.683 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.683 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:28.683 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:28.683 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 186176 2023-09-23T23:15:28.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.684 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477323+0000", 2023-09-23T23:15:28.684 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.684 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477323+0000", 2023-09-23T23:15:28.684 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.684 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477322+0000", 2023-09-23T23:15:28.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:15:28.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477327+0000", 2023-09-23T23:15:28.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7049999999999998e-06 2023-09-23T23:15:28.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477336+0000", 2023-09-23T23:15:28.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0299999999999999e-06 2023-09-23T23:15:28.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.686 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.686 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477342+0000", 2023-09-23T23:15:28.686 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7300000000000002e-06 2023-09-23T23:15:28.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.686 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.686 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477395+0000", 2023-09-23T23:15:28.686 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3256e-05 2023-09-23T23:15:28.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.686 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.687 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477519+0000", 2023-09-23T23:15:28.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012338599999999999 2023-09-23T23:15:28.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.687 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.687 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 7", 2023-09-23T23:15:28.687 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477633+0000", 2023-09-23T23:15:28.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000114461 2023-09-23T23:15:28.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.687 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.687 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.687 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.487194+0000", 2023-09-23T23:15:28.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0095614959999999992 2023-09-23T23:15:28.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.490656+0000", 2023-09-23T23:15:28.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003461689 2023-09-23T23:15:28.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.490665+0000", 2023-09-23T23:15:28.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0159999999999997e-06 2023-09-23T23:15:28.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489508+0000", 2023-09-23T23:15:28.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99884329500000002 2023-09-23T23:15:28.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489523+0000", 2023-09-23T23:15:28.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4565e-05 2023-09-23T23:15:28.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.868588+0000", 2023-09-23T23:15:28.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3790654570000001 2023-09-23T23:15:28.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-09-23T23:15:28.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.868704+0000", 2023-09-23T23:15:28.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000115458 2023-09-23T23:15:28.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893017+0000", 2023-09-23T23:15:28.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.024312963999999999 2023-09-23T23:15:28.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893101+0000", 2023-09-23T23:15:28.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4171000000000004e-05 2023-09-23T23:15:28.691 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.691 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.691 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.691 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.691 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:186189 8.8 8:14e93b65:::benchmark_data_smithi110_258128_object67371:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T23:15:28.691 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:58.504373+0000", 2023-09-23T23:15:28.691 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 270.07690557400002, 2023-09-23T23:15:28.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.388920556, 2023-09-23T23:15:28.691 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.691 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.692 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.692 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:28.692 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:28.692 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 186189 2023-09-23T23:15:28.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.692 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504373+0000", 2023-09-23T23:15:28.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.693 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504373+0000", 2023-09-23T23:15:28.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.693 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.693 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504372+0000", 2023-09-23T23:15:28.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:28.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.693 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.693 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504379+0000", 2023-09-23T23:15:28.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7580000000000003e-06 2023-09-23T23:15:28.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.694 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.694 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504390+0000", 2023-09-23T23:15:28.694 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1239e-05 2023-09-23T23:15:28.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.694 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.694 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504397+0000", 2023-09-23T23:15:28.694 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3509999999999996e-06 2023-09-23T23:15:28.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.694 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.695 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504473+0000", 2023-09-23T23:15:28.695 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5463000000000006e-05 2023-09-23T23:15:28.695 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.695 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.695 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.695 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504483+0000", 2023-09-23T23:15:28.695 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0202e-05 2023-09-23T23:15:28.695 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.695 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.695 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.695 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489551+0000", 2023-09-23T23:15:28.695 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98506775000000002 2023-09-23T23:15:28.695 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489558+0000", 2023-09-23T23:15:28.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4170000000000003e-06 2023-09-23T23:15:28.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.868754+0000", 2023-09-23T23:15:28.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3791954120000001 2023-09-23T23:15:28.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.868858+0000", 2023-09-23T23:15:28.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103983 2023-09-23T23:15:28.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:15:28.697 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.868960+0000", 2023-09-23T23:15:28.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010265800000000001 2023-09-23T23:15:28.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.697 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.877827+0000", 2023-09-23T23:15:28.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088667039999999996 2023-09-23T23:15:28.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.698 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.877878+0000", 2023-09-23T23:15:28.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1452000000000001e-05 2023-09-23T23:15:28.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.698 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893139+0000", 2023-09-23T23:15:28.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015260272 2023-09-23T23:15:28.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.698 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893179+0000", 2023-09-23T23:15:28.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9881e-05 2023-09-23T23:15:28.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.699 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.699 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.699 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893294+0000", 2023-09-23T23:15:28.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000115392 2023-09-23T23:15:28.699 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.699 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.699 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.699 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.699 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:186216 8.8 8:19876885:::benchmark_data_smithi110_258128_object67398:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T23:15:28.699 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:58.548467+0000", 2023-09-23T23:15:28.700 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 270.03281184999997, 2023-09-23T23:15:28.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3450009590000001, 2023-09-23T23:15:28.700 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.700 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.700 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.700 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:28.700 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:28.700 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 186216 2023-09-23T23:15:28.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.700 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548467+0000", 2023-09-23T23:15:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548467+0000", 2023-09-23T23:15:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548466+0000", 2023-09-23T23:15:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.702 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.702 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548473+0000", 2023-09-23T23:15:28.702 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0339999999999999e-06 2023-09-23T23:15:28.702 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.702 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.702 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.702 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548485+0000", 2023-09-23T23:15:28.702 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.223e-05 2023-09-23T23:15:28.702 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.702 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.702 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.702 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548492+0000", 2023-09-23T23:15:28.702 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5509999999999996e-06 2023-09-23T23:15:28.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.703 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.703 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548545+0000", 2023-09-23T23:15:28.703 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3434999999999999e-05 2023-09-23T23:15:28.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.703 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.703 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548552+0000", 2023-09-23T23:15:28.703 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8750000000000002e-06 2023-09-23T23:15:28.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.703 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.704 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489851+0000", 2023-09-23T23:15:28.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94129886100000004 2023-09-23T23:15:28.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.704 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.704 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489860+0000", 2023-09-23T23:15:28.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7079999999999992e-06 2023-09-23T23:15:28.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.704 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.704 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.873113+0000", 2023-09-23T23:15:28.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3832525150000001 2023-09-23T23:15:28.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.705 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.873221+0000", 2023-09-23T23:15:28.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010844300000000001 2023-09-23T23:15:28.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:15:28.705 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.873337+0000", 2023-09-23T23:15:28.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000115621 2023-09-23T23:15:28.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.706 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.877887+0000", 2023-09-23T23:15:28.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0045499099999999999 2023-09-23T23:15:28.706 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.706 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.706 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.706 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.877920+0000", 2023-09-23T23:15:28.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3314e-05 2023-09-23T23:15:28.706 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.706 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.706 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.706 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893326+0000", 2023-09-23T23:15:28.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015405911 2023-09-23T23:15:28.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893363+0000", 2023-09-23T23:15:28.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7240000000000003e-05 2023-09-23T23:15:28.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893468+0000", 2023-09-23T23:15:28.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010466700000000001 2023-09-23T23:15:28.707 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.707 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.707 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.708 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:212259 8.8 8:1ea4165f:::benchmark_data_smithi110_258128_object93441:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T23:15:28.708 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:11:36.633704+0000", 2023-09-23T23:15:28.708 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 231.947575296, 2023-09-23T23:15:28.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1054476879999999, 2023-09-23T23:15:28.708 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.708 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.708 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.708 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:28.708 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:28.708 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 212259 2023-09-23T23:15:28.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.709 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.709 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.633704+0000", 2023-09-23T23:15:28.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.709 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.633704+0000", 2023-09-23T23:15:28.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.633702+0000", 2023-09-23T23:15:28.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:15:28.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.633710+0000", 2023-09-23T23:15:28.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5599999999999994e-06 2023-09-23T23:15:28.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.633725+0000", 2023-09-23T23:15:28.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4715999999999999e-05 2023-09-23T23:15:28.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.711 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.711 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.633733+0000", 2023-09-23T23:15:28.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2740000000000007e-06 2023-09-23T23:15:28.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.711 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.711 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.633868+0000", 2023-09-23T23:15:28.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013501799999999999 2023-09-23T23:15:28.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.633880+0000", 2023-09-23T23:15:28.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1122e-05 2023-09-23T23:15:28.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.675476+0000", 2023-09-23T23:15:28.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.041596172 2023-09-23T23:15:28.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.713 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.675596+0000", 2023-09-23T23:15:28.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120145 2023-09-23T23:15:28.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.713 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:15:28.713 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.675684+0000", 2023-09-23T23:15:28.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8060999999999996e-05 2023-09-23T23:15:28.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.713 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.713 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.687555+0000", 2023-09-23T23:15:28.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011870798 2023-09-23T23:15:28.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739030+0000", 2023-09-23T23:15:28.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.051475198999999999 2023-09-23T23:15:28.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739068+0000", 2023-09-23T23:15:28.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8155000000000003e-05 2023-09-23T23:15:28.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739113+0000", 2023-09-23T23:15:28.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5328999999999999e-05 2023-09-23T23:15:28.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739151+0000", 2023-09-23T23:15:28.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7923000000000003e-05 2023-09-23T23:15:28.715 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.715 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.715 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.716 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.716 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:212262 8.8 8:1f06781a:::benchmark_data_smithi110_258128_object93444:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T23:15:28.716 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:11:36.633773+0000", 2023-09-23T23:15:28.716 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 231.94750618399999, 2023-09-23T23:15:28.716 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1055125969999999, 2023-09-23T23:15:28.716 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.716 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.716 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.716 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:28.716 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:28.716 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 212262 2023-09-23T23:15:28.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.717 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.633773+0000", 2023-09-23T23:15:28.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.633773+0000", 2023-09-23T23:15:28.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.633772+0000", 2023-09-23T23:15:28.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:15:28.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.633775+0000", 2023-09-23T23:15:28.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6259999999999999e-06 2023-09-23T23:15:28.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.633780+0000", 2023-09-23T23:15:28.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1479999999999997e-06 2023-09-23T23:15:28.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.719 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.633784+0000", 2023-09-23T23:15:28.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3170000000000002e-06 2023-09-23T23:15:28.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.719 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.633902+0000", 2023-09-23T23:15:28.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118261 2023-09-23T23:15:28.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.720 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.720 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.633908+0000", 2023-09-23T23:15:28.720 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8900000000000004e-06 2023-09-23T23:15:28.720 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.720 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.720 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.720 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.679933+0000", 2023-09-23T23:15:28.720 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0460249909999999 2023-09-23T23:15:28.720 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.720 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.720 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.720 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.680039+0000", 2023-09-23T23:15:28.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010610199999999999 2023-09-23T23:15:28.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.721 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:15:28.721 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.680187+0000", 2023-09-23T23:15:28.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014867600000000001 2023-09-23T23:15:28.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.721 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.721 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.696453+0000", 2023-09-23T23:15:28.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016265801999999999 2023-09-23T23:15:28.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.722 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739198+0000", 2023-09-23T23:15:28.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.042744618999999998 2023-09-23T23:15:28.722 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.722 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739226+0000", 2023-09-23T23:15:28.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7832000000000001e-05 2023-09-23T23:15:28.722 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739250+0000", 2023-09-23T23:15:28.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4076000000000001e-05 2023-09-23T23:15:28.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739285+0000", 2023-09-23T23:15:28.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5573e-05 2023-09-23T23:15:28.723 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.723 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.723 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.724 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:212265 8.8 8:13155f6c:::benchmark_data_smithi110_258128_object93447:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T23:15:28.724 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:11:36.642478+0000", 2023-09-23T23:15:28.724 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 231.93880060800001, 2023-09-23T23:15:28.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.096932985, 2023-09-23T23:15:28.724 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.724 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.724 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.724 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:28.724 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:28.724 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 212265 2023-09-23T23:15:28.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.724 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.642478+0000", 2023-09-23T23:15:28.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.642478+0000", 2023-09-23T23:15:28.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.642477+0000", 2023-09-23T23:15:28.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:28.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.642482+0000", 2023-09-23T23:15:28.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.356e-06 2023-09-23T23:15:28.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.642493+0000", 2023-09-23T23:15:28.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0682999999999999e-05 2023-09-23T23:15:28.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.642500+0000", 2023-09-23T23:15:28.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1069999999999999e-06 2023-09-23T23:15:28.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.642544+0000", 2023-09-23T23:15:28.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4044e-05 2023-09-23T23:15:28.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.642552+0000", 2023-09-23T23:15:28.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2940000000000006e-06 2023-09-23T23:15:28.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.728 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.728 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.683845+0000", 2023-09-23T23:15:28.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.041292892 2023-09-23T23:15:28.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.728 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.728 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.683972+0000", 2023-09-23T23:15:28.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012645700000000001 2023-09-23T23:15:28.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:15:28.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.684066+0000", 2023-09-23T23:15:28.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4745000000000002e-05 2023-09-23T23:15:28.729 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.704930+0000", 2023-09-23T23:15:28.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020863972000000001 2023-09-23T23:15:28.729 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739330+0000", 2023-09-23T23:15:28.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.034399777999999999 2023-09-23T23:15:28.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739357+0000", 2023-09-23T23:15:28.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7328999999999999e-05 2023-09-23T23:15:28.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739382+0000", 2023-09-23T23:15:28.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4244999999999999e-05 2023-09-23T23:15:28.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.731 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.731 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739411+0000", 2023-09-23T23:15:28.731 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9624e-05 2023-09-23T23:15:28.731 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.731 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.731 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.731 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:212287 8.8 8:121e75d6:::benchmark_data_smithi110_258128_object93469:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T23:15:28.731 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:11:36.669649+0000", 2023-09-23T23:15:28.731 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 231.91163042100001, 2023-09-23T23:15:28.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.06990479, 2023-09-23T23:15:28.732 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.732 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.732 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.732 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:28.732 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:28.732 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 212287 2023-09-23T23:15:28.732 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.732 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.669649+0000", 2023-09-23T23:15:28.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.669649+0000", 2023-09-23T23:15:28.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.669647+0000", 2023-09-23T23:15:28.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:28.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.669654+0000", 2023-09-23T23:15:28.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3300000000000004e-06 2023-09-23T23:15:28.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.669668+0000", 2023-09-23T23:15:28.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.466e-05 2023-09-23T23:15:28.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.669675+0000", 2023-09-23T23:15:28.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5479999999999998e-06 2023-09-23T23:15:28.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.735 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.735 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.669734+0000", 2023-09-23T23:15:28.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9644e-05 2023-09-23T23:15:28.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.735 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.735 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.669743+0000", 2023-09-23T23:15:28.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0240000000000003e-06 2023-09-23T23:15:28.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.687604+0000", 2023-09-23T23:15:28.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.017860327 2023-09-23T23:15:28.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.687742+0000", 2023-09-23T23:15:28.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013777599999999999 2023-09-23T23:15:28.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:15:28.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.687892+0000", 2023-09-23T23:15:28.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014991000000000001 2023-09-23T23:15:28.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.704935+0000", 2023-09-23T23:15:28.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017043729000000001 2023-09-23T23:15:28.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739435+0000", 2023-09-23T23:15:28.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.034499332000000001 2023-09-23T23:15:28.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.738 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739491+0000", 2023-09-23T23:15:28.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5949999999999998e-05 2023-09-23T23:15:28.738 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.738 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.738 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739519+0000", 2023-09-23T23:15:28.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.885e-05 2023-09-23T23:15:28.738 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.738 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.738 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739553+0000", 2023-09-23T23:15:28.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3951000000000001e-05 2023-09-23T23:15:28.739 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.739 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.739 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.739 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:212296 8.8 8:1ba68c7a:::benchmark_data_smithi110_258128_object93478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T23:15:28.739 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:11:36.676839+0000", 2023-09-23T23:15:28.739 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 231.904439865, 2023-09-23T23:15:28.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0628498610000001, 2023-09-23T23:15:28.739 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.739 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.739 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.740 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:28.740 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:28.740 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 212296 2023-09-23T23:15:28.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.740 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.676839+0000", 2023-09-23T23:15:28.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.676839+0000", 2023-09-23T23:15:28.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.676839+0000", 2023-09-23T23:15:28.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:15:28.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.676841+0000", 2023-09-23T23:15:28.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7049999999999999e-06 2023-09-23T23:15:28.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.676851+0000", 2023-09-23T23:15:28.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2080000000000006e-06 2023-09-23T23:15:28.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.676855+0000", 2023-09-23T23:15:28.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9129999999999996e-06 2023-09-23T23:15:28.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.676932+0000", 2023-09-23T23:15:28.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7460999999999996e-05 2023-09-23T23:15:28.743 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.676943+0000", 2023-09-23T23:15:28.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0514e-05 2023-09-23T23:15:28.743 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.692742+0000", 2023-09-23T23:15:28.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0157991159999999 2023-09-23T23:15:28.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.744 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.692822+0000", 2023-09-23T23:15:28.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0550000000000006e-05 2023-09-23T23:15:28.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.744 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:15:28.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.692904+0000", 2023-09-23T23:15:28.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1818999999999997e-05 2023-09-23T23:15:28.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.712646+0000", 2023-09-23T23:15:28.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019742297999999998 2023-09-23T23:15:28.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739598+0000", 2023-09-23T23:15:28.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.026951817999999999 2023-09-23T23:15:28.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.746 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739628+0000", 2023-09-23T23:15:28.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0185e-05 2023-09-23T23:15:28.746 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.746 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.746 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.746 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739656+0000", 2023-09-23T23:15:28.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7568000000000001e-05 2023-09-23T23:15:28.746 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.746 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.746 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.746 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739689+0000", 2023-09-23T23:15:28.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3080000000000002e-05 2023-09-23T23:15:28.747 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.747 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.747 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.747 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:212299 8.8 8:1e54462e:::benchmark_data_smithi110_258128_object93481:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T23:15:28.747 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:11:36.678486+0000", 2023-09-23T23:15:28.747 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 231.90279246399999, 2023-09-23T23:15:28.747 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.061328235, 2023-09-23T23:15:28.747 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:28.747 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:28.747 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:28.748 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:15:28.748 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:15:28.748 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 212299 2023-09-23T23:15:28.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.748 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:28.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:28.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.678486+0000", 2023-09-23T23:15:28.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:28.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.678486+0000", 2023-09-23T23:15:28.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:28.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.749 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:28.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.678486+0000", 2023-09-23T23:15:28.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:15:28.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.749 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:28.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.678491+0000", 2023-09-23T23:15:28.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2839999999999997e-06 2023-09-23T23:15:28.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:28.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.678504+0000", 2023-09-23T23:15:28.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2756e-05 2023-09-23T23:15:28.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:28.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.678510+0000", 2023-09-23T23:15:28.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8850000000000001e-06 2023-09-23T23:15:28.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.678563+0000", 2023-09-23T23:15:28.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2805e-05 2023-09-23T23:15:28.751 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:28.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:36.678574+0000", 2023-09-23T23:15:28.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0964e-05 2023-09-23T23:15:28.751 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:28.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.696490+0000", 2023-09-23T23:15:28.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0179157130000001 2023-09-23T23:15:28.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.752 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:28.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.696572+0000", 2023-09-23T23:15:28.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1578999999999999e-05 2023-09-23T23:15:28.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.752 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:15:28.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.696650+0000", 2023-09-23T23:15:28.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8308999999999999e-05 2023-09-23T23:15:28.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:28.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.712656+0000", 2023-09-23T23:15:28.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016005907 2023-09-23T23:15:28.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739729+0000", 2023-09-23T23:15:28.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.027073132999999999 2023-09-23T23:15:28.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:28.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739755+0000", 2023-09-23T23:15:28.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.658e-05 2023-09-23T23:15:28.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:28.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739783+0000", 2023-09-23T23:15:28.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7880000000000001e-05 2023-09-23T23:15:28.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:28.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:28.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:28.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:37.739815+0000", 2023-09-23T23:15:28.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1346000000000001e-05 2023-09-23T23:15:28.754 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.755 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.755 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.755 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:28.755 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:28.755 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:15:28.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:28 smithi138 ceph-mon[193606]: pgmap v613: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-09-23T23:15:28.756 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:28 smithi138 ceph-mon[193606]: from='client.35307 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:15:28.756 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:28 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:15:28.756 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:28 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:15:28.756 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'46052','7':'62260'}"}]': finished 2023-09-23T23:15:28.756 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:28 smithi138 ceph-mon[193606]: osdmap e186: 8 total, 8 up, 8 in 2023-09-23T23:15:28.756 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:28 smithi138 ceph-mon[193606]: failed to encode map e186 with expected crc 2023-09-23T23:15:28.756 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:28 smithi138 ceph-mon[193606]: failed to encode map e186 with expected crc 2023-09-23T23:15:28.756 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:28 smithi138 ceph-mon[193606]: failed to encode map e186 with expected crc 2023-09-23T23:15:28.756 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:28 smithi138 ceph-mon[193606]: failed to encode map e186 with expected crc 2023-09-23T23:15:28.756 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:15:28.757 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:28 smithi138 ceph-mon[193606]: osdmap e187: 8 total, 8 up, 8 in 2023-09-23T23:15:28.945 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:28.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:28.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:29.050 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:29.055 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T23:15:29.055 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-23T23:15:29.080 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:29.196 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:29.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:29.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:29.268 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:15:29.300 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:29.351 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:29.475 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:29.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:29.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:29.581 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:15:29.615 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:15:29.655 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[212939]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.655 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[212939]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.655 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[212939]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.655 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[212939]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.655 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[212939]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.655 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[212939]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.655 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[212939]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.655 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[212939]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.655 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[212939]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.656 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[212939]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.656 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[212939]: osdmap e188: 8 total, 8 up, 8 in 2023-09-23T23:15:29.658 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[221102]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.659 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[221102]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.659 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[221102]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.659 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[221102]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.659 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[221102]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.659 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[221102]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.659 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[221102]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.659 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[221102]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.659 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[221102]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.659 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[221102]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.659 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:29 smithi110 ceph-mon[221102]: osdmap e188: 8 total, 8 up, 8 in 2023-09-23T23:15:29.667 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:29 smithi138 ceph-mon[193606]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:29 smithi138 ceph-mon[193606]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:29 smithi138 ceph-mon[193606]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:29 smithi138 ceph-mon[193606]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:29 smithi138 ceph-mon[193606]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:29 smithi138 ceph-mon[193606]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:29 smithi138 ceph-mon[193606]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:29 smithi138 ceph-mon[193606]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:29 smithi138 ceph-mon[193606]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:29 smithi138 ceph-mon[193606]: failed to encode map e187 with expected crc 2023-09-23T23:15:29.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:29 smithi138 ceph-mon[193606]: osdmap e188: 8 total, 8 up, 8 in 2023-09-23T23:15:29.743 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:15:29.768 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:15:29.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:29.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:15:29.875 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:15:29.931 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:30.018 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:15:30.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:30.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:15:30.125 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:30.182 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:30.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[212939]: failed to encode map e188 with expected crc 2023-09-23T23:15:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[212939]: failed to encode map e188 with expected crc 2023-09-23T23:15:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[212939]: failed to encode map e188 with expected crc 2023-09-23T23:15:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[212939]: 9.8 starting backfill to osd.1 from (0'0,0'0] MAX to 185'7990 2023-09-23T23:15:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[212939]: failed to encode map e188 with expected crc 2023-09-23T23:15:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[212939]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 185'7929 2023-09-23T23:15:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[212939]: pgmap v617: 99 pgs: 1 remapped+peering, 98 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2023-09-23T23:15:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[212939]: failed to encode map e188 with expected crc 2023-09-23T23:15:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[212939]: osdmap e189: 8 total, 8 up, 8 in 2023-09-23T23:15:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[212939]: failed to encode map e189 with expected crc 2023-09-23T23:15:30.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[221102]: failed to encode map e188 with expected crc 2023-09-23T23:15:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[221102]: failed to encode map e188 with expected crc 2023-09-23T23:15:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[221102]: failed to encode map e188 with expected crc 2023-09-23T23:15:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[221102]: 9.8 starting backfill to osd.1 from (0'0,0'0] MAX to 185'7990 2023-09-23T23:15:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[221102]: failed to encode map e188 with expected crc 2023-09-23T23:15:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[221102]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 185'7929 2023-09-23T23:15:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[221102]: pgmap v617: 99 pgs: 1 remapped+peering, 98 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2023-09-23T23:15:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[221102]: failed to encode map e188 with expected crc 2023-09-23T23:15:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[221102]: osdmap e189: 8 total, 8 up, 8 in 2023-09-23T23:15:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:30 smithi110 ceph-mon[221102]: failed to encode map e189 with expected crc 2023-09-23T23:15:30.527 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:30.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:30.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:30.634 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:30.673 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:30 smithi138 ceph-mon[193606]: failed to encode map e188 with expected crc 2023-09-23T23:15:30.674 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:30 smithi138 ceph-mon[193606]: failed to encode map e188 with expected crc 2023-09-23T23:15:30.674 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:30 smithi138 ceph-mon[193606]: failed to encode map e188 with expected crc 2023-09-23T23:15:30.674 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:30 smithi138 ceph-mon[193606]: 9.8 starting backfill to osd.1 from (0'0,0'0] MAX to 185'7990 2023-09-23T23:15:30.674 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:30 smithi138 ceph-mon[193606]: failed to encode map e188 with expected crc 2023-09-23T23:15:30.674 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:30 smithi138 ceph-mon[193606]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 185'7929 2023-09-23T23:15:30.674 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:30 smithi138 ceph-mon[193606]: pgmap v617: 99 pgs: 1 remapped+peering, 98 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2023-09-23T23:15:30.674 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:30 smithi138 ceph-mon[193606]: failed to encode map e188 with expected crc 2023-09-23T23:15:30.674 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:30 smithi138 ceph-mon[193606]: osdmap e189: 8 total, 8 up, 8 in 2023-09-23T23:15:30.675 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:30 smithi138 ceph-mon[193606]: failed to encode map e189 with expected crc 2023-09-23T23:15:30.680 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:30.737 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:30.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:30.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:30.842 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:30.888 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:30.931 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:30.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:30.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:30.961 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:15:30.998 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:15:31.034 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:31.063 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:31.111 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:31.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:31.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:31.215 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:31.294 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:31.625 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:31 smithi110 ceph-mon[221102]: failed to encode map e189 with expected crc 2023-09-23T23:15:31.625 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:31 smithi110 ceph-mon[221102]: failed to encode map e189 with expected crc 2023-09-23T23:15:31.625 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:31 smithi110 ceph-mon[221102]: failed to encode map e189 with expected crc 2023-09-23T23:15:31.625 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:31 smithi110 ceph-mon[221102]: osdmap e190: 8 total, 8 up, 8 in 2023-09-23T23:15:31.629 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:31 smithi110 ceph-mon[212939]: failed to encode map e189 with expected crc 2023-09-23T23:15:31.629 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:31 smithi110 ceph-mon[212939]: failed to encode map e189 with expected crc 2023-09-23T23:15:31.629 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:31 smithi110 ceph-mon[212939]: failed to encode map e189 with expected crc 2023-09-23T23:15:31.630 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:31 smithi110 ceph-mon[212939]: osdmap e190: 8 total, 8 up, 8 in 2023-09-23T23:15:31.654 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:31.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:31.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:31.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:31 smithi138 ceph-mon[193606]: failed to encode map e189 with expected crc 2023-09-23T23:15:31.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:31 smithi138 ceph-mon[193606]: failed to encode map e189 with expected crc 2023-09-23T23:15:31.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:31 smithi138 ceph-mon[193606]: failed to encode map e189 with expected crc 2023-09-23T23:15:31.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:31 smithi138 ceph-mon[193606]: osdmap e190: 8 total, 8 up, 8 in 2023-09-23T23:15:31.759 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:31.805 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:31.865 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:15:31.925 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:31.925 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:15:31.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:31.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:32.021 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:15:32.021 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:15:32.021 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-09-23T23:15:32.021 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:15:32.032 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:32.104 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:32.265 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:32.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:32.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:32.377 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:32.475 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:32.607 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:32.623 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:32 smithi110 ceph-mon[212939]: failed to encode map e190 with expected crc 2023-09-23T23:15:32.623 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:32 smithi110 ceph-mon[212939]: failed to encode map e190 with expected crc 2023-09-23T23:15:32.623 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:32 smithi110 ceph-mon[212939]: pgmap v620: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 peering, 1 remapped+peering, 95 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 21 KiB/s rd, 56 KiB/s wr, 750 op/s; 17498/284193 objects misplaced (6.157%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T23:15:32.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:32 smithi110 ceph-mon[221102]: failed to encode map e190 with expected crc 2023-09-23T23:15:32.624 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:32 smithi110 ceph-mon[221102]: failed to encode map e190 with expected crc 2023-09-23T23:15:32.624 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:32 smithi110 ceph-mon[221102]: pgmap v620: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 peering, 1 remapped+peering, 95 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 21 KiB/s rd, 56 KiB/s wr, 750 op/s; 17498/284193 objects misplaced (6.157%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T23:15:32.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:32.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:32.727 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:15:32.752 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:32 smithi138 ceph-mon[193606]: failed to encode map e190 with expected crc 2023-09-23T23:15:32.752 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:32 smithi138 ceph-mon[193606]: failed to encode map e190 with expected crc 2023-09-23T23:15:32.753 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:32 smithi138 ceph-mon[193606]: pgmap v620: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 peering, 1 remapped+peering, 95 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 21 KiB/s rd, 56 KiB/s wr, 750 op/s; 17498/284193 objects misplaced (6.157%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T23:15:32.756 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:32.816 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:15:32.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:32.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:15:32.874 INFO:teuthology.orchestra.run.smithi110.stdout:moved 0 / 297 (0%) 2023-09-23T23:15:32.874 INFO:teuthology.orchestra.run.smithi110.stdout:avg 37.125 2023-09-23T23:15:32.874 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 7.68826 -> 7.68826 (expected baseline 5.69951) 2023-09-23T23:15:32.875 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.5 with 23 -> 23 pgs (0.619529 -> 0.619529 * mean) 2023-09-23T23:15:32.875 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.3 with 49 -> 49 pgs (1.31987 -> 1.31987 * mean) 2023-09-23T23:15:32.875 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:15:32.875 INFO:teuthology.orchestra.run.smithi110.stdout:oload 1000 2023-09-23T23:15:32.875 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 1 2023-09-23T23:15:32.875 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:15:32.875 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0280 2023-09-23T23:15:32.875 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.2798 2023-09-23T23:15:32.877 INFO:teuthology.orchestra.run.smithi110.stderr:no change 2023-09-23T23:15:32.921 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:32.971 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:33.273 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:33.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:33.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:33.284 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:15:33.384 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:15:33.457 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:33.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:33 smithi138 ceph-mon[193606]: from='client.35313 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:15:33.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:33 smithi110 ceph-mon[212939]: from='client.35313 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:15:33.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:33 smithi110 ceph-mon[221102]: from='client.35313 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:15:33.827 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:15:34.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:34 smithi138 ceph-mon[193606]: pgmap v621: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 peering, 1 remapped+peering, 95 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 16 KiB/s rd, 43 KiB/s wr, 512 op/s; 17498/283197 objects misplaced (6.179%); 968 KiB/s, 14 objects/s recovering 2023-09-23T23:15:34.849 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:34 smithi110 ceph-mon[212939]: pgmap v621: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 peering, 1 remapped+peering, 95 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 16 KiB/s rd, 43 KiB/s wr, 512 op/s; 17498/283197 objects misplaced (6.179%); 968 KiB/s, 14 objects/s recovering 2023-09-23T23:15:34.850 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:34 smithi110 ceph-mon[221102]: pgmap v621: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 peering, 1 remapped+peering, 95 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 16 KiB/s rd, 43 KiB/s wr, 512 op/s; 17498/283197 objects misplaced (6.179%); 968 KiB/s, 14 objects/s recovering 2023-09-23T23:15:35.604 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:15:35.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:35.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:15:35.714 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:35.786 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:36.095 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:36.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:36.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:36.124 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:15:36.124 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:15:36.124 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:15:36.124 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:15:36.124 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:15:36.214 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:15:36.260 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:36.316 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:15:36.340 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:15:36.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:36.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:15:36.363 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:15:36.402 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:15:36.430 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:15:36.444 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:15:36.467 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:15:36.481 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:36.504 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:15:36.539 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:15:36.548 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:15:36.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:36.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:15:36.588 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:15:36.614 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:15:36.614 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:15:36.614 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:15:36.614 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:15:36.614 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:15:36.615 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:15:36.615 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:15:36.615 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:15:36.652 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:15:36.678 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:36.721 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:15:36.722 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:36 smithi138 ceph-mon[193606]: pgmap v622: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 peering, 95 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 359 op/s; 21664/279612 objects misplaced (7.748%); 18 MiB/s, 279 objects/s recovering 2023-09-23T23:15:36.722 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:36 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2816147562' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:36.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:36.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:15:36.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:36 smithi110 ceph-mon[212939]: pgmap v622: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 peering, 95 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 359 op/s; 21664/279612 objects misplaced (7.748%); 18 MiB/s, 279 objects/s recovering 2023-09-23T23:15:36.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:36 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2816147562' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:36.804 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:15:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:15:36] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.43.0" 2023-09-23T23:15:36.805 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:36 smithi110 ceph-mon[221102]: pgmap v622: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 peering, 95 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 359 op/s; 21664/279612 objects misplaced (7.748%); 18 MiB/s, 279 objects/s recovering 2023-09-23T23:15:36.805 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:36 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2816147562' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:36.824 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:15:36.899 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:36.988 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:15:36.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:36.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:15:37.096 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:15:37.165 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:37.257 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:15:37.267 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:15:37.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:37.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:15:37.373 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:15:37.433 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:37.489 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:15:37.543 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:15:37.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:37.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:15:37.651 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:37.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:37 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3188314742' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:37.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:37 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:37.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:37 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3188314742' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:37.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:37 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:37.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:37 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3188314742' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:37.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:37 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:37.715 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:37.757 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:15:38.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:38 smithi138 ceph-mon[193606]: pgmap v623: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 350 op/s; 21209/277797 objects misplaced (7.635%); 15 MiB/s, 233 objects/s recovering 2023-09-23T23:15:38.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:38 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:15:38.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:38 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:15:38.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:38 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:15:38.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:38 smithi138 ceph-mon[193606]: osdmap e191: 8 total, 8 up, 8 in 2023-09-23T23:15:38.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:38 smithi138 ceph-mon[193606]: failed to encode map e191 with expected crc 2023-09-23T23:15:38.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:38 smithi138 ceph-mon[193606]: failed to encode map e191 with expected crc 2023-09-23T23:15:38.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:38 smithi138 ceph-mon[193606]: failed to encode map e191 with expected crc 2023-09-23T23:15:38.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:38 smithi138 ceph-mon[193606]: failed to encode map e191 with expected crc 2023-09-23T23:15:38.694 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:38.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:38.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:38.766 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[212939]: pgmap v623: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 350 op/s; 21209/277797 objects misplaced (7.635%); 15 MiB/s, 233 objects/s recovering 2023-09-23T23:15:38.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:15:38.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:15:38.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:15:38.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[212939]: osdmap e191: 8 total, 8 up, 8 in 2023-09-23T23:15:38.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[212939]: failed to encode map e191 with expected crc 2023-09-23T23:15:38.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[212939]: failed to encode map e191 with expected crc 2023-09-23T23:15:38.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[212939]: failed to encode map e191 with expected crc 2023-09-23T23:15:38.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[212939]: failed to encode map e191 with expected crc 2023-09-23T23:15:38.768 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[221102]: pgmap v623: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 350 op/s; 21209/277797 objects misplaced (7.635%); 15 MiB/s, 233 objects/s recovering 2023-09-23T23:15:38.768 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:15:38.768 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:15:38.768 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:15:38.768 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[221102]: osdmap e191: 8 total, 8 up, 8 in 2023-09-23T23:15:38.768 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[221102]: failed to encode map e191 with expected crc 2023-09-23T23:15:38.768 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[221102]: failed to encode map e191 with expected crc 2023-09-23T23:15:38.768 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[221102]: failed to encode map e191 with expected crc 2023-09-23T23:15:38.768 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:38 smithi110 ceph-mon[221102]: failed to encode map e191 with expected crc 2023-09-23T23:15:38.799 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:15:38.848 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:39.049 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:15:39.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:39.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:15:39.154 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:15:39.198 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:39.493 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:15:39.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:39.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:15:39.597 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:39.650 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:39.739 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:39.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:39.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:39.845 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:39.878 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:39.966 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:39.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:39.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:40.071 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:40.133 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:40.211 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:15:40.214 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:15:40.220 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:40.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:40.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:40.330 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:40.342 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T23:15:40.342 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:15:40.342 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T23:15:40.342 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T23:15:40.342 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T23:15:40.342 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T23:15:40.384 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:40.522 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:40.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:40.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:40.529 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:40 smithi138 ceph-mon[193606]: pgmap v625: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 19488/271887 objects misplaced (7.168%); 26 MiB/s, 423 objects/s recovering 2023-09-23T23:15:40.554 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:15:40.629 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:40.662 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:40.721 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:40.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:40.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:40.790 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:15:40.826 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:40.862 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:40.873 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:40 smithi110 ceph-mon[212939]: pgmap v625: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 19488/271887 objects misplaced (7.168%); 26 MiB/s, 423 objects/s recovering 2023-09-23T23:15:40.876 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:40 smithi110 ceph-mon[221102]: pgmap v625: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 19488/271887 objects misplaced (7.168%); 26 MiB/s, 423 objects/s recovering 2023-09-23T23:15:40.918 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:40.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:40.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:41.023 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:15:41.179 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:41.259 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:15:41.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:41.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:15:41.337 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:15:41.368 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:41.399 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:41.459 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:41.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:41.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:41.551 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:15:41.563 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:15:41.584 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:15:41.650 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:41.670 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:15:41.737 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:15:41.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:41.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:15:41.843 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:41.926 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:42.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:42 smithi138 ceph-mon[193606]: pgmap v626: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 18948/269688 objects misplaced (7.026%); 23 MiB/s, 361 objects/s recovering 2023-09-23T23:15:42.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:42 smithi110 ceph-mon[212939]: pgmap v626: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 18948/269688 objects misplaced (7.026%); 23 MiB/s, 361 objects/s recovering 2023-09-23T23:15:42.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:42 smithi110 ceph-mon[221102]: pgmap v626: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 18948/269688 objects misplaced (7.026%); 23 MiB/s, 361 objects/s recovering 2023-09-23T23:15:43.155 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:43.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:43.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:43.190 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:15:43.259 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:15:43.388 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:44.049 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:15:44.049 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T23:15:44.049 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:15:44.075 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:44.075 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:15:44.075 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:44.075 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:43 smithi138 ceph-mon[193606]: pgmap v627: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 18948/268272 objects misplaced (7.063%); 23 MiB/s, 361 objects/s recovering 2023-09-23T23:15:44.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:15:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:43 smithi110 ceph-mon[212939]: pgmap v627: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 18948/268272 objects misplaced (7.063%); 23 MiB/s, 361 objects/s recovering 2023-09-23T23:15:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:15:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:43 smithi110 ceph-mon[221102]: pgmap v627: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 18948/268272 objects misplaced (7.063%); 23 MiB/s, 361 objects/s recovering 2023-09-23T23:15:45.226 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:15:45.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:45.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:15:45.334 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:45.366 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:45.819 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:45.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:45.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:45.928 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:15:45.972 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:46.043 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:15:46.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:46.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:15:46.061 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:15:46.100 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:15:46.135 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:15:46.149 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:46.176 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:15:46.196 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:46.221 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:15:46.252 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:15:46.283 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:15:46.301 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:46.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:46.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:46.314 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:15:46.323 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:15:46.323 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:15:46.323 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:15:46.324 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:15:46.324 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:15:46.324 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:15:46.324 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:15:46.324 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:15:46.372 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:15:46.372 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:15:46.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:46.372 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45164.0:136451 9.7 e191/164 9:ea06f2fe:::benchmark_data_smithi110_295904_object25270:head v 191'8454, mlcod=191'8452)", 2023-09-23T23:15:46.372 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:46.369541+0000", 2023-09-23T23:15:46.372 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0014960900000000001, 2023-09-23T23:15:46.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015229600000000001, 2023-09-23T23:15:46.373 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:46.373 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:15:46.373 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:46.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:46.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:46.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.369541+0000", 2023-09-23T23:15:46.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:46.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:46.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:46.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:46.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.369541+0000", 2023-09-23T23:15:46.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:46.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:46.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:46.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:46.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.369540+0000", 2023-09-23T23:15:46.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:15:46.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:46.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:46.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:46.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.369547+0000", 2023-09-23T23:15:46.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2400000000000001e-06 2023-09-23T23:15:46.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:46.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:46.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:46.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.369561+0000", 2023-09-23T23:15:46.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3546999999999999e-05 2023-09-23T23:15:46.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:46.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:46.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:46.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.369567+0000", 2023-09-23T23:15:46.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8470000000000001e-06 2023-09-23T23:15:46.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:46.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:46.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:46.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.369612+0000", 2023-09-23T23:15:46.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5161999999999999e-05 2023-09-23T23:15:46.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:46.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:46.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:46.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.369631+0000", 2023-09-23T23:15:46.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8802999999999999e-05 2023-09-23T23:15:46.376 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:46.376 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:46.376 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:46.376 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:46.376 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-09-23T23:15:46.376 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 1 2023-09-23T23:15:46.377 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:15:46.405 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:46.474 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:46.597 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:46.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:46.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:46.612 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:46 smithi110 ceph-mon[212939]: pgmap v628: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 17202/263748 objects misplaced (6.522%); 23 MiB/s, 367 objects/s recovering 2023-09-23T23:15:46.612 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:46 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1685524569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:46.612 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:46 smithi110 ceph-mon[221102]: pgmap v628: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 17202/263748 objects misplaced (6.522%); 23 MiB/s, 367 objects/s recovering 2023-09-23T23:15:46.613 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:46 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1685524569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:46.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:46 smithi138 ceph-mon[193606]: pgmap v628: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 17202/263748 objects misplaced (6.522%); 23 MiB/s, 367 objects/s recovering 2023-09-23T23:15:46.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:46 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1685524569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:46.713 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:46.748 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:46.796 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:46.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:46.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:46.901 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:46.937 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:15:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:15:46] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.43.0" 2023-09-23T23:15:46.939 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:46.990 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:46.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:46.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:47.094 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:47.120 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:47.237 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:47.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:47.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:47.308 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T23:15:47.341 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:15:47.397 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:47.434 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:15:47.613 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:15:47.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:47.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:15:47.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:47 smithi138 ceph-mon[193606]: failed to encode map e192 with expected crc 2023-09-23T23:15:47.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:47 smithi138 ceph-mon[193606]: osdmap e192: 8 total, 8 up, 8 in 2023-09-23T23:15:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:47 smithi138 ceph-mon[193606]: failed to encode map e192 with expected crc 2023-09-23T23:15:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:47 smithi138 ceph-mon[193606]: failed to encode map e192 with expected crc 2023-09-23T23:15:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:47 smithi138 ceph-mon[193606]: failed to encode map e192 with expected crc 2023-09-23T23:15:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:47 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2714634820' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:15:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:47 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:15:47.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:47 smithi110 ceph-mon[212939]: failed to encode map e192 with expected crc 2023-09-23T23:15:47.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:47 smithi110 ceph-mon[212939]: osdmap e192: 8 total, 8 up, 8 in 2023-09-23T23:15:47.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:47 smithi110 ceph-mon[212939]: failed to encode map e192 with expected crc 2023-09-23T23:15:47.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:47 smithi110 ceph-mon[212939]: failed to encode map e192 with expected crc 2023-09-23T23:15:47.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:47 smithi110 ceph-mon[212939]: failed to encode map e192 with expected crc 2023-09-23T23:15:47.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:47 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2714634820' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:15:47.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:47 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:15:47.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:47 smithi110 ceph-mon[221102]: failed to encode map e192 with expected crc 2023-09-23T23:15:47.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:47 smithi110 ceph-mon[221102]: osdmap e192: 8 total, 8 up, 8 in 2023-09-23T23:15:47.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:47 smithi110 ceph-mon[221102]: failed to encode map e192 with expected crc 2023-09-23T23:15:47.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:47 smithi110 ceph-mon[221102]: failed to encode map e192 with expected crc 2023-09-23T23:15:47.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:47 smithi110 ceph-mon[221102]: failed to encode map e192 with expected crc 2023-09-23T23:15:47.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:47 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2714634820' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:15:47.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:47 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:15:47.730 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:47.759 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:15:47.815 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:47.909 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:47.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:47.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:48.014 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:15:48.039 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:48.119 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:15:48.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:48.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:15:48.231 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:48.270 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:48.419 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:48 smithi110 ceph-mon[212939]: pgmap v630: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 16684/261732 objects misplaced (6.374%); 23 MiB/s, 373 objects/s recovering 2023-09-23T23:15:48.419 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:48 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:15:48.419 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:48 smithi110 ceph-mon[212939]: failed to encode map e193 with expected crc 2023-09-23T23:15:48.419 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:48 smithi110 ceph-mon[212939]: failed to encode map e193 with expected crc 2023-09-23T23:15:48.419 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:48 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:15:48.419 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:48 smithi110 ceph-mon[212939]: osdmap e193: 8 total, 8 up, 8 in 2023-09-23T23:15:48.429 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:48.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:48.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:48.533 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:15:48.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:48 smithi138 ceph-mon[193606]: pgmap v630: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 16684/261732 objects misplaced (6.374%); 23 MiB/s, 373 objects/s recovering 2023-09-23T23:15:48.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:48 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:15:48.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:48 smithi138 ceph-mon[193606]: failed to encode map e193 with expected crc 2023-09-23T23:15:48.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:48 smithi138 ceph-mon[193606]: failed to encode map e193 with expected crc 2023-09-23T23:15:48.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:48 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:15:48.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:48 smithi138 ceph-mon[193606]: osdmap e193: 8 total, 8 up, 8 in 2023-09-23T23:15:48.570 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:48.644 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:15:48.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:48.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:15:48.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:48 smithi110 ceph-mon[221102]: pgmap v630: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 16684/261732 objects misplaced (6.374%); 23 MiB/s, 373 objects/s recovering 2023-09-23T23:15:48.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:48 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:15:48.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:48 smithi110 ceph-mon[221102]: failed to encode map e193 with expected crc 2023-09-23T23:15:48.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:48 smithi110 ceph-mon[221102]: failed to encode map e193 with expected crc 2023-09-23T23:15:48.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:48 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:15:48.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:48 smithi110 ceph-mon[221102]: osdmap e193: 8 total, 8 up, 8 in 2023-09-23T23:15:48.748 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:48.820 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:48.918 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:48.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:48.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:49.024 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:49.086 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:49.180 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:49.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:49.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:49.294 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:49.361 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:49.434 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:49.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:49.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:49.542 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:49.582 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:49.610 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:15:49.610 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:15:49.611 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:15:49.611 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:15:49.611 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:15:49.644 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:15:49.645 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:49.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:49.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:49.750 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:15:49.806 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:49.910 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:15:49.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:49.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:15:50.018 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:15:50.050 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:50.151 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:15:50.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:50.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:15:50.231 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:15:50.261 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:50.308 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:50.421 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:50 smithi110 ceph-mon[221102]: pgmap v632: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 15985/255942 objects misplaced (6.246%); 20 MiB/s, 319 objects/s recovering 2023-09-23T23:15:50.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:50 smithi110 ceph-mon[212939]: pgmap v632: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 15985/255942 objects misplaced (6.246%); 20 MiB/s, 319 objects/s recovering 2023-09-23T23:15:50.449 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:50.455 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T23:15:50.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:50.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:50.538 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:15:50.557 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:50 smithi138 ceph-mon[193606]: pgmap v632: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 15985/255942 objects misplaced (6.246%); 20 MiB/s, 319 objects/s recovering 2023-09-23T23:15:50.736 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:50.889 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T23:15:50.889 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:15:50.890 INFO:tasks.thrashosds.thrasher:Killing osd 2, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T23:15:50.890 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T23:15:50.890 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T23:15:50.983 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:15:51.001 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:51.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:51.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:51.108 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:15:51.141 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:51.260 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:15:51.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:51.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:15:51.365 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:15:51.402 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:51.454 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:15:51.455 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:15:51 smithi110 systemd[1]: Stopping Ceph osd.2 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:15:51.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:51.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:15:51.560 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:15:51.600 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:51.669 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:15:51.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:51.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:15:51.774 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:15:51.804 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:52.423 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:15:52.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:52.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:15:52.544 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:15:52.588 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:52.685 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:52 smithi138 ceph-mon[193606]: pgmap v633: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 15478/253941 objects misplaced (6.095%); 21 MiB/s, 341 objects/s recovering 2023-09-23T23:15:52.693 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:15:52.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:52.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:15:52.707 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:15:52 smithi110 conmon[260298]: 2023-09-23T23:15:52.661+0000 7ff8cd703700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:15:52.707 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:15:52 smithi110 conmon[260298]: 2023-09-23T23:15:52.661+0000 7ff8cd703700 -1 osd.2 193 *** Got signal Terminated *** 2023-09-23T23:15:52.707 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:15:52 smithi110 conmon[260298]: 2023-09-23T23:15:52.662+0000 7ff8cd703700 -1 osd.2 193 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:15:52.709 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:52 smithi110 ceph-mon[221102]: pgmap v633: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 15478/253941 objects misplaced (6.095%); 21 MiB/s, 341 objects/s recovering 2023-09-23T23:15:52.710 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:52 smithi110 ceph-mon[212939]: pgmap v633: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 15478/253941 objects misplaced (6.095%); 21 MiB/s, 341 objects/s recovering 2023-09-23T23:15:52.799 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:52.800 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:15:52.853 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:52.914 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:52.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:52.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:52.953 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:15:52.953 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:15:52.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:15:52.953 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:15:52.954 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.954 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34269.0:153984 7.5 7:accdd201:::benchmark_data_smithi110_236826_object41276:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-09-23T23:15:52.954 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:05:57.973188+0000", 2023-09-23T23:15:52.954 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 594.97487275699996, 2023-09-23T23:15:52.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0365887840000001, 2023-09-23T23:15:52.954 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:52.954 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:52.954 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:52.955 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34269", 2023-09-23T23:15:52.955 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:15:52.955 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 153984 2023-09-23T23:15:52.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.955 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:52.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:52.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.973188+0000", 2023-09-23T23:15:52.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:52.956 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.956 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:52.956 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.973188+0000", 2023-09-23T23:15:52.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:52.956 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.956 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:52.956 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.973187+0000", 2023-09-23T23:15:52.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:52.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:52.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.973194+0000", 2023-09-23T23:15:52.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1609999999999997e-06 2023-09-23T23:15:52.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:52.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.973206+0000", 2023-09-23T23:15:52.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2298e-05 2023-09-23T23:15:52.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:52.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.973212+0000", 2023-09-23T23:15:52.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2979999999999997e-06 2023-09-23T23:15:52.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.959 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:52.959 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.973411+0000", 2023-09-23T23:15:52.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000198481 2023-09-23T23:15:52.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.959 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.959 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:52.959 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.973419+0000", 2023-09-23T23:15:52.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0770000000000008e-06 2023-09-23T23:15:52.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:52.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.978784+0000", 2023-09-23T23:15:52.960 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.005364642 2023-09-23T23:15:52.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:52.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.978886+0000", 2023-09-23T23:15:52.960 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101916 2023-09-23T23:15:52.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-09-23T23:15:52.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.978974+0000", 2023-09-23T23:15:52.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.878e-05 2023-09-23T23:15:52.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:52.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.985647+0000", 2023-09-23T23:15:52.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066723889999999999 2023-09-23T23:15:52.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:52.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.009627+0000", 2023-09-23T23:15:52.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.023980756999999998 2023-09-23T23:15:52.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.963 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:52.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.009719+0000", 2023-09-23T23:15:52.963 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1695999999999998e-05 2023-09-23T23:15:52.963 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.963 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.963 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:52.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.009751+0000", 2023-09-23T23:15:52.963 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2209000000000003e-05 2023-09-23T23:15:52.963 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.964 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.964 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:52.964 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.009777+0000", 2023-09-23T23:15:52.964 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5576999999999999e-05 2023-09-23T23:15:52.964 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:52.964 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:52.964 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:52.964 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.964 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.965 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34269.0:153994 7.5 7:a75904cc:::benchmark_data_smithi110_236826_object41286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-09-23T23:15:52.965 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:05:57.994554+0000", 2023-09-23T23:15:52.965 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 594.95350674700001, 2023-09-23T23:15:52.965 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0153625580000001, 2023-09-23T23:15:52.965 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:52.965 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:52.965 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:52.965 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34269", 2023-09-23T23:15:52.966 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:15:52.966 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 153994 2023-09-23T23:15:52.966 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.966 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:52.966 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.966 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:52.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.994554+0000", 2023-09-23T23:15:52.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:52.966 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:52.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.994554+0000", 2023-09-23T23:15:52.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:52.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:52.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.994553+0000", 2023-09-23T23:15:52.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:52.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:52.968 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.994560+0000", 2023-09-23T23:15:52.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3830000000000003e-06 2023-09-23T23:15:52.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:52.968 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.994578+0000", 2023-09-23T23:15:52.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7691000000000001e-05 2023-09-23T23:15:52.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:52.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.994586+0000", 2023-09-23T23:15:52.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9349999999999994e-06 2023-09-23T23:15:52.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:52.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.994648+0000", 2023-09-23T23:15:52.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2044000000000004e-05 2023-09-23T23:15:52.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:52.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.994660+0000", 2023-09-23T23:15:52.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2004000000000001e-05 2023-09-23T23:15:52.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:52.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.981970+0000", 2023-09-23T23:15:52.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98731018000000004 2023-09-23T23:15:52.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:52.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.982036+0000", 2023-09-23T23:15:52.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6126999999999999e-05 2023-09-23T23:15:52.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-09-23T23:15:52.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.982105+0000", 2023-09-23T23:15:52.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9288999999999993e-05 2023-09-23T23:15:52.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:52.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.987493+0000", 2023-09-23T23:15:52.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0053878069999999997 2023-09-23T23:15:52.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:52.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.009806+0000", 2023-09-23T23:15:52.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.022313044000000001 2023-09-23T23:15:52.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:52.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.009861+0000", 2023-09-23T23:15:52.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4558000000000002e-05 2023-09-23T23:15:52.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.974 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:52.974 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.009894+0000", 2023-09-23T23:15:52.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2702e-05 2023-09-23T23:15:52.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.974 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:52.975 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.009917+0000", 2023-09-23T23:15:52.975 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3176999999999998e-05 2023-09-23T23:15:52.975 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:52.975 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:52.975 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:52.975 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.975 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.975 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34269.0:153997 7.5 7:a26f8a2e:::benchmark_data_smithi110_236826_object41289:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-09-23T23:15:52.976 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:05:57.994602+0000", 2023-09-23T23:15:52.976 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 594.95345929500002, 2023-09-23T23:15:52.976 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0153838639999999, 2023-09-23T23:15:52.976 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:52.976 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:52.976 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:52.976 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34269", 2023-09-23T23:15:52.976 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:15:52.976 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 153997 2023-09-23T23:15:52.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.977 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:52.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:52.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.994602+0000", 2023-09-23T23:15:52.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:52.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:52.978 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.994602+0000", 2023-09-23T23:15:52.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:52.978 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:52.978 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.994601+0000", 2023-09-23T23:15:52.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:15:52.978 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.979 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:52.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.994604+0000", 2023-09-23T23:15:52.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2899999999999998e-06 2023-09-23T23:15:52.979 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.979 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.979 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:52.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.994609+0000", 2023-09-23T23:15:52.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7060000000000003e-06 2023-09-23T23:15:52.979 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.980 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:52.980 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.994612+0000", 2023-09-23T23:15:52.980 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1070000000000001e-06 2023-09-23T23:15:52.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.980 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:52.980 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.994686+0000", 2023-09-23T23:15:52.980 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3674e-05 2023-09-23T23:15:52.981 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.981 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:52.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:57.994692+0000", 2023-09-23T23:15:52.981 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0909999999999998e-06 2023-09-23T23:15:52.981 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.981 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:52.982 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.983825+0000", 2023-09-23T23:15:52.982 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98913298599999999 2023-09-23T23:15:52.982 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.982 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.982 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:52.982 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.983895+0000", 2023-09-23T23:15:52.982 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0482999999999996e-05 2023-09-23T23:15:52.982 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.982 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.983 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-09-23T23:15:52.983 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.983964+0000", 2023-09-23T23:15:52.983 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8449999999999997e-05 2023-09-23T23:15:52.983 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.983 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.983 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:52.983 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.989326+0000", 2023-09-23T23:15:52.983 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0053619160000000004 2023-09-23T23:15:52.983 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.984 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.984 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:52.984 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.009835+0000", 2023-09-23T23:15:52.984 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020508713000000001 2023-09-23T23:15:52.984 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.984 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.984 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:52.984 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.009942+0000", 2023-09-23T23:15:52.984 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106965 2023-09-23T23:15:52.985 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.985 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.985 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:52.985 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.009966+0000", 2023-09-23T23:15:52.985 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4972000000000001e-05 2023-09-23T23:15:52.985 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.985 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.985 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:52.985 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.009985+0000", 2023-09-23T23:15:52.986 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8977000000000001e-05 2023-09-23T23:15:52.986 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:52.986 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:52.986 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:52.986 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.986 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.986 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34269.0:154034 7.5 7:a4f03803:::benchmark_data_smithi110_236826_object41326:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-09-23T23:15:52.986 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:05:58.048305+0000", 2023-09-23T23:15:52.986 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 594.89975627499996, 2023-09-23T23:15:52.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96190382399999996, 2023-09-23T23:15:52.987 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:52.987 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:52.987 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:52.987 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34269", 2023-09-23T23:15:52.987 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:15:52.987 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154034 2023-09-23T23:15:52.987 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.987 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:52.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.988 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:52.988 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.048305+0000", 2023-09-23T23:15:52.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:52.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.988 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:52.988 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.048305+0000", 2023-09-23T23:15:52.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:52.989 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.989 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:52.989 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.048303+0000", 2023-09-23T23:15:52.989 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:52.989 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.989 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:52.989 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.048309+0000", 2023-09-23T23:15:52.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3110000000000002e-06 2023-09-23T23:15:52.990 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.990 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.990 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:52.990 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.048322+0000", 2023-09-23T23:15:52.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3141e-05 2023-09-23T23:15:52.990 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.990 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.990 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:52.991 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.048328+0000", 2023-09-23T23:15:52.991 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1000000000000003e-06 2023-09-23T23:15:52.991 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.991 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.991 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:52.991 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.048384+0000", 2023-09-23T23:15:52.991 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6567999999999998e-05 2023-09-23T23:15:52.991 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.991 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:52.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.048395+0000", 2023-09-23T23:15:52.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1024e-05 2023-09-23T23:15:52.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:52.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.985665+0000", 2023-09-23T23:15:52.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.93727013100000001 2023-09-23T23:15:52.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:52.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.985733+0000", 2023-09-23T23:15:52.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7409999999999993e-05 2023-09-23T23:15:52.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-09-23T23:15:52.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.985799+0000", 2023-09-23T23:15:52.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6779999999999994e-05 2023-09-23T23:15:52.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.994 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:52.994 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.009426+0000", 2023-09-23T23:15:52.994 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.023626973999999999 2023-09-23T23:15:52.994 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.994 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:52.994 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.010055+0000", 2023-09-23T23:15:52.994 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00062873499999999995 2023-09-23T23:15:52.995 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.995 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.995 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:52.995 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.010115+0000", 2023-09-23T23:15:52.995 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0232000000000002e-05 2023-09-23T23:15:52.995 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.995 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.995 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:52.995 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.010169+0000", 2023-09-23T23:15:52.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4035e-05 2023-09-23T23:15:52.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.996 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:52.996 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.010208+0000", 2023-09-23T23:15:52.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9036999999999999e-05 2023-09-23T23:15:52.996 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:52.996 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:52.996 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:52.997 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.997 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.997 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34269.0:154035 7.5 7:a7f4b8b9:::benchmark_data_smithi110_236826_object41327:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-09-23T23:15:52.997 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:05:58.048342+0000", 2023-09-23T23:15:52.997 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 594.89971861699996, 2023-09-23T23:15:52.997 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96854074199999995, 2023-09-23T23:15:52.997 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:52.997 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:52.998 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:52.998 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34269", 2023-09-23T23:15:52.998 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:15:52.998 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154035 2023-09-23T23:15:52.998 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.998 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:52.998 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.998 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:52.998 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.048342+0000", 2023-09-23T23:15:52.999 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:52.999 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.999 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.999 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:52.999 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.048342+0000", 2023-09-23T23:15:52.999 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:52.999 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:52.999 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:52.999 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:53.000 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.048342+0000", 2023-09-23T23:15:53.000 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:15:53.000 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.000 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.000 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:53.000 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.048345+0000", 2023-09-23T23:15:53.000 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.021e-06 2023-09-23T23:15:53.000 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.000 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:53.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.048351+0000", 2023-09-23T23:15:53.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2879999999999998e-06 2023-09-23T23:15:53.001 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.001 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:53.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.048355+0000", 2023-09-23T23:15:53.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0820000000000001e-06 2023-09-23T23:15:53.001 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.002 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.002 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.002 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.048418+0000", 2023-09-23T23:15:53.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2807999999999999e-05 2023-09-23T23:15:53.002 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.002 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.002 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.002 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.048422+0000", 2023-09-23T23:15:53.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8260000000000003e-06 2023-09-23T23:15:53.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.003 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.003 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.003 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.987529+0000", 2023-09-23T23:15:53.003 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.93910689300000005 2023-09-23T23:15:53.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.003 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.003 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:53.003 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.987589+0000", 2023-09-23T23:15:53.004 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0553000000000001e-05 2023-09-23T23:15:53.004 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.004 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.004 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-09-23T23:15:53.004 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.987652+0000", 2023-09-23T23:15:53.004 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2409999999999994e-05 2023-09-23T23:15:53.004 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.004 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.004 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:53.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.009633+0000", 2023-09-23T23:15:53.005 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.021981460000000001 2023-09-23T23:15:53.005 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.005 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.010235+0000", 2023-09-23T23:15:53.005 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00060152299999999997 2023-09-23T23:15:53.005 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.016789+0000", 2023-09-23T23:15:53.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0065546579999999997 2023-09-23T23:15:53.006 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.006 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:53.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.016843+0000", 2023-09-23T23:15:53.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3906999999999997e-05 2023-09-23T23:15:53.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.007 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.007 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:53.007 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.016883+0000", 2023-09-23T23:15:53.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9725000000000003e-05 2023-09-23T23:15:53.007 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.007 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:53.007 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.008 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34269.0:154062 7.5 7:aa9f0c0e:::benchmark_data_smithi110_236826_object41354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-09-23T23:15:53.008 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:05:58.086962+0000", 2023-09-23T23:15:53.008 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 594.86109873500004, 2023-09-23T23:15:53.008 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.93006285499999997, 2023-09-23T23:15:53.008 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:53.008 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:53.008 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:53.008 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34269", 2023-09-23T23:15:53.009 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:15:53.009 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154062 2023-09-23T23:15:53.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.009 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:53.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.009 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:53.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.086962+0000", 2023-09-23T23:15:53.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.010 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:53.010 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.086962+0000", 2023-09-23T23:15:53.010 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.010 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.010 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:53.010 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.086961+0000", 2023-09-23T23:15:53.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:15:53.011 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:53.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.086966+0000", 2023-09-23T23:15:53.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.702e-06 2023-09-23T23:15:53.011 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:53.012 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.086979+0000", 2023-09-23T23:15:53.012 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2959000000000001e-05 2023-09-23T23:15:53.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.012 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:53.012 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.086985+0000", 2023-09-23T23:15:53.012 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1399999999999997e-06 2023-09-23T23:15:53.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.013 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.013 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.087055+0000", 2023-09-23T23:15:53.013 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9958999999999999e-05 2023-09-23T23:15:53.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.013 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.013 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.013 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.087067+0000", 2023-09-23T23:15:53.013 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1790999999999999e-05 2023-09-23T23:15:53.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.989361+0000", 2023-09-23T23:15:53.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.90229453199999998 2023-09-23T23:15:53.014 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:53.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.989444+0000", 2023-09-23T23:15:53.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3057000000000001e-05 2023-09-23T23:15:53.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.015 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-09-23T23:15:53.015 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.989516+0000", 2023-09-23T23:15:53.015 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2008000000000006e-05 2023-09-23T23:15:53.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.015 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:53.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.009637+0000", 2023-09-23T23:15:53.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020120583000000001 2023-09-23T23:15:53.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.010266+0000", 2023-09-23T23:15:53.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00062881000000000005 2023-09-23T23:15:53.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.017 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.017 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.016951+0000", 2023-09-23T23:15:53.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066856720000000001 2023-09-23T23:15:53.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.017 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:53.017 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.016992+0000", 2023-09-23T23:15:53.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0852999999999997e-05 2023-09-23T23:15:53.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.018 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.018 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:53.018 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.017025+0000", 2023-09-23T23:15:53.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2778999999999999e-05 2023-09-23T23:15:53.018 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.018 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:53.018 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.018 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.019 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.019 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34269.0:154103 7.5 7:adbf217f:::benchmark_data_smithi110_236826_object41395:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-09-23T23:15:53.019 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:05:58.147404+0000", 2023-09-23T23:15:53.019 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 594.80065663799996, 2023-09-23T23:15:53.019 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.874733867, 2023-09-23T23:15:53.019 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:53.019 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:53.019 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:53.019 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34269", 2023-09-23T23:15:53.020 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:15:53.020 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154103 2023-09-23T23:15:53.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.020 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:53.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.020 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:53.020 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.147404+0000", 2023-09-23T23:15:53.020 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.021 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:53.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.147404+0000", 2023-09-23T23:15:53.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.021 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.021 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:53.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.147403+0000", 2023-09-23T23:15:53.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:53.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.022 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:53.022 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.147408+0000", 2023-09-23T23:15:53.022 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5300000000000004e-06 2023-09-23T23:15:53.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.022 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:53.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.147421+0000", 2023-09-23T23:15:53.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2884000000000001e-05 2023-09-23T23:15:53.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.023 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:53.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.147427+0000", 2023-09-23T23:15:53.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7490000000000004e-06 2023-09-23T23:15:53.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.023 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.147487+0000", 2023-09-23T23:15:53.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0300000000000002e-05 2023-09-23T23:15:53.024 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.024 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.024 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.147498+0000", 2023-09-23T23:15:53.024 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0740000000000001e-05 2023-09-23T23:15:53.024 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.024 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.024 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.991186+0000", 2023-09-23T23:15:53.024 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.84368754000000001 2023-09-23T23:15:53.024 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.025 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:53.025 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.991253+0000", 2023-09-23T23:15:53.025 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7046999999999997e-05 2023-09-23T23:15:53.025 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.025 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.025 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-09-23T23:15:53.025 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.991319+0000", 2023-09-23T23:15:53.025 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6481000000000003e-05 2023-09-23T23:15:53.025 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.025 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.025 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:53.025 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.009642+0000", 2023-09-23T23:15:53.025 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018323332000000001 2023-09-23T23:15:53.026 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.026 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.026 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.026 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.017073+0000", 2023-09-23T23:15:53.026 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.007431076 2023-09-23T23:15:53.026 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.026 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.026 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.026 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.022040+0000", 2023-09-23T23:15:53.026 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049669470000000002 2023-09-23T23:15:53.026 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.026 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:53.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.022091+0000", 2023-09-23T23:15:53.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0738000000000001e-05 2023-09-23T23:15:53.027 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.027 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:53.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.022138+0000", 2023-09-23T23:15:53.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6941999999999999e-05 2023-09-23T23:15:53.027 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.027 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:53.027 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.029 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.029 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.030 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34269.0:154118 7.5 7:a5c92808:::benchmark_data_smithi110_236826_object41410:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-09-23T23:15:53.030 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:05:58.172316+0000", 2023-09-23T23:15:53.030 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 594.77574518599999, 2023-09-23T23:15:53.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.86454108500000004, 2023-09-23T23:15:53.030 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:53.030 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:53.030 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:53.030 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34269", 2023-09-23T23:15:53.030 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1687751556", 2023-09-23T23:15:53.030 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154118 2023-09-23T23:15:53.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.030 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:53.031 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.031 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:53.031 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.172316+0000", 2023-09-23T23:15:53.031 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.031 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.031 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.031 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:53.031 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.172316+0000", 2023-09-23T23:15:53.031 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.031 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.031 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.031 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:53.031 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.172315+0000", 2023-09-23T23:15:53.032 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:53.032 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.032 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.032 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:53.032 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.172321+0000", 2023-09-23T23:15:53.032 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9120000000000001e-06 2023-09-23T23:15:53.032 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.032 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.032 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:53.032 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.172338+0000", 2023-09-23T23:15:53.032 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6826000000000001e-05 2023-09-23T23:15:53.032 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.032 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.033 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:53.033 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.172344+0000", 2023-09-23T23:15:53.033 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6180000000000003e-06 2023-09-23T23:15:53.033 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.033 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.033 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.033 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.172402+0000", 2023-09-23T23:15:53.033 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7960000000000001e-05 2023-09-23T23:15:53.033 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.033 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.033 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.033 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.172412+0000", 2023-09-23T23:15:53.034 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0669999999999999e-05 2023-09-23T23:15:53.034 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.034 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.034 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.034 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.994862+0000", 2023-09-23T23:15:53.034 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.82244995600000004 2023-09-23T23:15:53.034 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.034 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.034 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:53.034 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.994922+0000", 2023-09-23T23:15:53.034 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9703000000000001e-05 2023-09-23T23:15:53.034 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.035 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.035 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-09-23T23:15:53.035 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:58.994991+0000", 2023-09-23T23:15:53.035 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8590999999999994e-05 2023-09-23T23:15:53.035 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.035 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.035 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:53.035 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.009653+0000", 2023-09-23T23:15:53.035 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0146622 2023-09-23T23:15:53.035 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.035 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.035 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.036 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.019381+0000", 2023-09-23T23:15:53.036 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0097281039999999996 2023-09-23T23:15:53.036 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.036 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.036 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.036 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.036791+0000", 2023-09-23T23:15:53.036 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017410271000000001 2023-09-23T23:15:53.036 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.036 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.036 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:53.036 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.036824+0000", 2023-09-23T23:15:53.036 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2468999999999997e-05 2023-09-23T23:15:53.037 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.037 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.037 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:53.037 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:05:59.036857+0000", 2023-09-23T23:15:53.037 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3011e-05 2023-09-23T23:15:53.037 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.037 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:53.037 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.037 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.037 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.037 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127467 9.d 9:b004077a:::benchmark_data_smithi110_295904_object16286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T23:15:53.037 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.526680+0000", 2023-09-23T23:15:53.037 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 25.421380689999999, 2023-09-23T23:15:53.038 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7990730689999999, 2023-09-23T23:15:53.038 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:53.038 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:15:53.038 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:53.038 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:15:53.038 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:15:53.038 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127467 2023-09-23T23:15:53.038 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.038 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:53.038 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.038 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:53.039 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526680+0000", 2023-09-23T23:15:53.039 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.039 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.039 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.039 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:53.039 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526680+0000", 2023-09-23T23:15:53.039 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.039 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.039 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.039 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:53.039 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526679+0000", 2023-09-23T23:15:53.039 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:15:53.039 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.040 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.040 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:53.040 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526685+0000", 2023-09-23T23:15:53.040 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9479999999999996e-06 2023-09-23T23:15:53.040 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.040 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.040 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:53.040 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526699+0000", 2023-09-23T23:15:53.040 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.398e-05 2023-09-23T23:15:53.040 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.040 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.040 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:53.041 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526704+0000", 2023-09-23T23:15:53.041 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4e-06 2023-09-23T23:15:53.041 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.041 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.041 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.041 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526763+0000", 2023-09-23T23:15:53.041 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8575000000000001e-05 2023-09-23T23:15:53.041 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.041 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.041 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:53.041 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526873+0000", 2023-09-23T23:15:53.041 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110303 2023-09-23T23:15:53.041 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.042 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.042 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,5", 2023-09-23T23:15:53.042 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526958+0000", 2023-09-23T23:15:53.042 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4518999999999995e-05 2023-09-23T23:15:53.042 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.042 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.042 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:53.042 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.530629+0000", 2023-09-23T23:15:53.042 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0036707380000000002 2023-09-23T23:15:53.042 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.042 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.042 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.043 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.530882+0000", 2023-09-23T23:15:53.043 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025305899999999998 2023-09-23T23:15:53.043 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.043 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.043 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.043 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.546584+0000", 2023-09-23T23:15:53.043 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015702391 2023-09-23T23:15:53.043 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.043 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.043 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.043 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.546595+0000", 2023-09-23T23:15:53.043 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.119e-05 2023-09-23T23:15:53.044 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.044 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.044 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.044 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222624+0000", 2023-09-23T23:15:53.044 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.67602865199999995 2023-09-23T23:15:53.044 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.044 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.044 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.044 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222636+0000", 2023-09-23T23:15:53.044 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2053e-05 2023-09-23T23:15:53.044 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.044 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.044 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.045 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.325683+0000", 2023-09-23T23:15:53.045 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.103046642 2023-09-23T23:15:53.045 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.045 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.045 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:53.045 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.325753+0000", 2023-09-23T23:15:53.045 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0722999999999994e-05 2023-09-23T23:15:53.045 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.045 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:53.045 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.045 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.045 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.045 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127477 9.d 9:be22822d:::benchmark_data_smithi110_295904_object16296:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T23:15:53.046 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.541829+0000", 2023-09-23T23:15:53.046 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 25.406231743999999, 2023-09-23T23:15:53.046 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.97042316, 2023-09-23T23:15:53.046 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:53.046 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:53.046 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:53.046 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:15:53.046 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:15:53.046 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127477 2023-09-23T23:15:53.046 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.046 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:53.046 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.047 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:53.047 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.541829+0000", 2023-09-23T23:15:53.047 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.047 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.047 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.047 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:53.047 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.541829+0000", 2023-09-23T23:15:53.047 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.047 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.047 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.047 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:53.047 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.541828+0000", 2023-09-23T23:15:53.048 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:15:53.048 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.048 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.048 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:53.048 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.541833+0000", 2023-09-23T23:15:53.048 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.707e-06 2023-09-23T23:15:53.048 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.048 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.048 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:53.048 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.541842+0000", 2023-09-23T23:15:53.048 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5210000000000008e-06 2023-09-23T23:15:53.048 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.048 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.049 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:53.049 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.541849+0000", 2023-09-23T23:15:53.049 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5710000000000003e-06 2023-09-23T23:15:53.049 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.049 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.049 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.049 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.546848+0000", 2023-09-23T23:15:53.049 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049989539999999999 2023-09-23T23:15:53.049 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.049 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.049 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.049 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.546855+0000", 2023-09-23T23:15:53.050 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6900000000000003e-06 2023-09-23T23:15:53.050 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.050 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.050 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222663+0000", 2023-09-23T23:15:53.050 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.67580857900000002 2023-09-23T23:15:53.050 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.050 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.050 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222670+0000", 2023-09-23T23:15:53.050 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2830000000000004e-06 2023-09-23T23:15:53.050 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.051 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.051 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.325792+0000", 2023-09-23T23:15:53.051 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.103121223 2023-09-23T23:15:53.051 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.051 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.051 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:53.051 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.325890+0000", 2023-09-23T23:15:53.051 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7864999999999999e-05 2023-09-23T23:15:53.051 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.051 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.051 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:15:53.051 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.325985+0000", 2023-09-23T23:15:53.052 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5896000000000005e-05 2023-09-23T23:15:53.052 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.052 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.052 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:53.052 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.334870+0000", 2023-09-23T23:15:53.052 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088846629999999992 2023-09-23T23:15:53.052 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.052 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.052 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.052 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.352987+0000", 2023-09-23T23:15:53.052 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018117366999999999 2023-09-23T23:15:53.052 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.053 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.053 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.053 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353011+0000", 2023-09-23T23:15:53.053 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3506000000000002e-05 2023-09-23T23:15:53.053 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.053 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.053 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.053 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.512087+0000", 2023-09-23T23:15:53.053 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.15907564299999999 2023-09-23T23:15:53.053 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.053 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.053 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:53.053 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.512151+0000", 2023-09-23T23:15:53.054 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4381000000000006e-05 2023-09-23T23:15:53.054 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.054 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.054 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:53.054 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.512252+0000", 2023-09-23T23:15:53.054 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101345 2023-09-23T23:15:53.054 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.054 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:53.054 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.054 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.054 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.054 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127490 9.d 9:b0fef0a1:::benchmark_data_smithi110_295904_object16309:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:15:53.055 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.559146+0000", 2023-09-23T23:15:53.055 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 25.388915147999999, 2023-09-23T23:15:53.055 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.955891351, 2023-09-23T23:15:53.055 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:53.055 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:53.055 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:53.055 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:15:53.055 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:15:53.055 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127490 2023-09-23T23:15:53.055 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.055 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:53.055 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:53.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.559146+0000", 2023-09-23T23:15:53.056 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.056 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.056 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:53.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.559146+0000", 2023-09-23T23:15:53.056 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.056 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.056 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:53.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.559145+0000", 2023-09-23T23:15:53.056 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:15:53.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:53.057 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.559148+0000", 2023-09-23T23:15:53.057 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.875e-06 2023-09-23T23:15:53.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:53.057 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.559161+0000", 2023-09-23T23:15:53.057 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2527000000000001e-05 2023-09-23T23:15:53.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:53.058 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.559165+0000", 2023-09-23T23:15:53.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3789999999999999e-06 2023-09-23T23:15:53.058 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.058 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.058 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.058 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.559227+0000", 2023-09-23T23:15:53.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2131000000000005e-05 2023-09-23T23:15:53.058 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.058 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.058 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.058 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.559236+0000", 2023-09-23T23:15:53.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9190000000000005e-06 2023-09-23T23:15:53.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.059 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222687+0000", 2023-09-23T23:15:53.059 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.66345047999999995 2023-09-23T23:15:53.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.059 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222691+0000", 2023-09-23T23:15:53.059 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0589999999999996e-06 2023-09-23T23:15:53.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.059 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.328857+0000", 2023-09-23T23:15:53.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1061658780000001 2023-09-23T23:15:53.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.060 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.060 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:53.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.329003+0000", 2023-09-23T23:15:53.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014625799999999999 2023-09-23T23:15:53.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.060 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.060 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:15:53.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.329170+0000", 2023-09-23T23:15:53.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016664499999999999 2023-09-23T23:15:53.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:53.061 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.337592+0000", 2023-09-23T23:15:53.061 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0084228219999999999 2023-09-23T23:15:53.061 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.061 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353036+0000", 2023-09-23T23:15:53.061 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015444049 2023-09-23T23:15:53.061 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.062 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353086+0000", 2023-09-23T23:15:53.062 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9873000000000002e-05 2023-09-23T23:15:53.062 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.062 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.062 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.062 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.514795+0000", 2023-09-23T23:15:53.062 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16170854800000001 2023-09-23T23:15:53.062 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.062 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.062 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:53.062 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.514873+0000", 2023-09-23T23:15:53.062 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7756000000000006e-05 2023-09-23T23:15:53.062 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.063 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.063 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:53.063 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515037+0000", 2023-09-23T23:15:53.063 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016452400000000001 2023-09-23T23:15:53.063 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.063 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:53.063 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.063 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.063 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.063 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127492 9.d 9:befecd6a:::benchmark_data_smithi110_295904_object16311:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:15:53.063 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.560155+0000", 2023-09-23T23:15:53.063 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 25.387905715999999, 2023-09-23T23:15:53.064 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9551049470000001, 2023-09-23T23:15:53.064 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:53.064 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:53.064 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:53.064 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:15:53.064 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:15:53.064 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127492 2023-09-23T23:15:53.064 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.064 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:53.064 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.064 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:53.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.560155+0000", 2023-09-23T23:15:53.064 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.065 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:53.065 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.560155+0000", 2023-09-23T23:15:53.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.065 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:53.065 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.560154+0000", 2023-09-23T23:15:53.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:15:53.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:53.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.560159+0000", 2023-09-23T23:15:53.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8409999999999999e-06 2023-09-23T23:15:53.066 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.066 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:53.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.560173+0000", 2023-09-23T23:15:53.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3359e-05 2023-09-23T23:15:53.066 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.066 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:53.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.560178+0000", 2023-09-23T23:15:53.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3569999999999997e-06 2023-09-23T23:15:53.067 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.067 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.067 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.067 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.560225+0000", 2023-09-23T23:15:53.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6548000000000002e-05 2023-09-23T23:15:53.067 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.067 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.067 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.067 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.560232+0000", 2023-09-23T23:15:53.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5279999999999998e-06 2023-09-23T23:15:53.067 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.068 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222702+0000", 2023-09-23T23:15:53.068 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.662470114 2023-09-23T23:15:53.068 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.068 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222706+0000", 2023-09-23T23:15:53.068 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4750000000000002e-06 2023-09-23T23:15:53.068 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.068 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.069 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.331907+0000", 2023-09-23T23:15:53.069 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1092014429999999 2023-09-23T23:15:53.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.069 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.069 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:53.069 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.332043+0000", 2023-09-23T23:15:53.069 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013605799999999999 2023-09-23T23:15:53.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.069 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.069 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:15:53.069 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.332202+0000", 2023-09-23T23:15:53.069 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015870700000000001 2023-09-23T23:15:53.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.070 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:53.070 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.343380+0000", 2023-09-23T23:15:53.070 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011177576999999999 2023-09-23T23:15:53.070 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.070 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.070 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353063+0000", 2023-09-23T23:15:53.070 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.009683199 2023-09-23T23:15:53.070 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.070 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.071 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353166+0000", 2023-09-23T23:15:53.071 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010363500000000001 2023-09-23T23:15:53.071 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.071 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.071 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.071 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515087+0000", 2023-09-23T23:15:53.071 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16192087399999999 2023-09-23T23:15:53.071 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.071 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.071 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:53.071 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515144+0000", 2023-09-23T23:15:53.071 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6725999999999999e-05 2023-09-23T23:15:53.072 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.072 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.072 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:53.072 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515260+0000", 2023-09-23T23:15:53.072 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000116186 2023-09-23T23:15:53.072 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.072 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:53.072 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.072 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.072 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.072 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127498 9.d 9:b0f15af0:::benchmark_data_smithi110_295904_object16317:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:15:53.072 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.566686+0000", 2023-09-23T23:15:53.072 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 25.381374863000001, 2023-09-23T23:15:53.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.948764122, 2023-09-23T23:15:53.073 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:53.073 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:53.073 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:53.073 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:15:53.073 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:15:53.073 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127498 2023-09-23T23:15:53.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.073 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:53.073 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.073 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:53.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.566686+0000", 2023-09-23T23:15:53.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.074 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.074 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:53.074 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.566686+0000", 2023-09-23T23:15:53.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.074 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.074 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:53.074 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.566686+0000", 2023-09-23T23:15:53.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:15:53.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.074 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.075 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:53.075 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.566689+0000", 2023-09-23T23:15:53.075 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5090000000000002e-06 2023-09-23T23:15:53.075 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.075 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.075 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:53.075 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.566696+0000", 2023-09-23T23:15:53.075 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8839999999999997e-06 2023-09-23T23:15:53.075 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.075 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.075 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:53.075 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.566700+0000", 2023-09-23T23:15:53.075 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0249999999999996e-06 2023-09-23T23:15:53.076 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.076 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.076 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.076 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.566748+0000", 2023-09-23T23:15:53.076 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7771000000000001e-05 2023-09-23T23:15:53.076 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.076 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.076 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.076 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.566754+0000", 2023-09-23T23:15:53.076 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8220000000000003e-06 2023-09-23T23:15:53.076 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.076 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.077 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.077 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222815+0000", 2023-09-23T23:15:53.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.65606194100000004 2023-09-23T23:15:53.077 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.077 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.077 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.077 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222825+0000", 2023-09-23T23:15:53.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.533e-06 2023-09-23T23:15:53.077 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.077 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.077 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.077 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.334889+0000", 2023-09-23T23:15:53.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.11206366 2023-09-23T23:15:53.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:53.078 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.334971+0000", 2023-09-23T23:15:53.078 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2327e-05 2023-09-23T23:15:53.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:15:53.078 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.335051+0000", 2023-09-23T23:15:53.078 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0158000000000001e-05 2023-09-23T23:15:53.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:53.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.352730+0000", 2023-09-23T23:15:53.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017678366000000001 2023-09-23T23:15:53.079 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.079 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353110+0000", 2023-09-23T23:15:53.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00038028399999999999 2023-09-23T23:15:53.079 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.079 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353214+0000", 2023-09-23T23:15:53.080 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010437699999999999 2023-09-23T23:15:53.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.080 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.080 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515295+0000", 2023-09-23T23:15:53.080 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.162080369 2023-09-23T23:15:53.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.080 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:53.080 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515338+0000", 2023-09-23T23:15:53.080 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3897999999999999e-05 2023-09-23T23:15:53.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.081 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:53.081 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515450+0000", 2023-09-23T23:15:53.081 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000111698 2023-09-23T23:15:53.081 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.081 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:53.081 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.081 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.081 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.081 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127508 9.d 9:b620324a:::benchmark_data_smithi110_295904_object16327:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:15:53.081 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.579645+0000", 2023-09-23T23:15:53.081 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 25.368415550000002, 2023-09-23T23:15:53.081 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.935992645, 2023-09-23T23:15:53.082 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:53.082 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:53.082 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:53.082 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:15:53.082 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:15:53.082 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127508 2023-09-23T23:15:53.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.082 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:53.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.082 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:53.082 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579645+0000", 2023-09-23T23:15:53.082 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.083 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.083 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.083 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:53.083 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579645+0000", 2023-09-23T23:15:53.083 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.083 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.083 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.083 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:53.083 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579644+0000", 2023-09-23T23:15:53.083 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:15:53.083 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.083 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.084 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:53.084 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579649+0000", 2023-09-23T23:15:53.084 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1930000000000001e-06 2023-09-23T23:15:53.084 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.084 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.084 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:53.084 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579661+0000", 2023-09-23T23:15:53.084 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1627999999999999e-05 2023-09-23T23:15:53.084 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.084 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.084 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:53.084 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579681+0000", 2023-09-23T23:15:53.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0066999999999999e-05 2023-09-23T23:15:53.085 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.085 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.085 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.085 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579768+0000", 2023-09-23T23:15:53.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6817999999999994e-05 2023-09-23T23:15:53.085 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.085 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.085 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.085 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579777+0000", 2023-09-23T23:15:53.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1849999999999993e-06 2023-09-23T23:15:53.085 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.085 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.086 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222866+0000", 2023-09-23T23:15:53.086 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.64308858499999999 2023-09-23T23:15:53.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.086 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.086 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222874+0000", 2023-09-23T23:15:53.086 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7749999999999998e-06 2023-09-23T23:15:53.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.086 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.086 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.337631+0000", 2023-09-23T23:15:53.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1147567780000001 2023-09-23T23:15:53.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:53.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.337737+0000", 2023-09-23T23:15:53.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105858 2023-09-23T23:15:53.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:15:53.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.337833+0000", 2023-09-23T23:15:53.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5952000000000006e-05 2023-09-23T23:15:53.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:53.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.352739+0000", 2023-09-23T23:15:53.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014905899 2023-09-23T23:15:53.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353141+0000", 2023-09-23T23:15:53.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00040202700000000001 2023-09-23T23:15:53.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353261+0000", 2023-09-23T23:15:53.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000119697 2023-09-23T23:15:53.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.089 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.089 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.089 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515485+0000", 2023-09-23T23:15:53.089 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16222382799999999 2023-09-23T23:15:53.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.089 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.089 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:53.089 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515527+0000", 2023-09-23T23:15:53.089 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2522999999999999e-05 2023-09-23T23:15:53.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.089 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.090 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:53.090 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515638+0000", 2023-09-23T23:15:53.090 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110931 2023-09-23T23:15:53.090 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.090 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:53.090 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.090 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.090 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.090 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127509 9.d 9:bda00fa7:::benchmark_data_smithi110_295904_object16328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:15:53.090 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.579703+0000", 2023-09-23T23:15:53.090 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 25.368358105999999, 2023-09-23T23:15:53.090 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9397497079999999, 2023-09-23T23:15:53.090 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:53.091 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:53.091 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:53.091 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:15:53.091 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:15:53.091 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127509 2023-09-23T23:15:53.091 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.091 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:53.091 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.091 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:53.091 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579703+0000", 2023-09-23T23:15:53.091 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.091 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.092 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.092 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:53.092 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579703+0000", 2023-09-23T23:15:53.092 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.092 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.092 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.092 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:53.092 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579702+0000", 2023-09-23T23:15:53.092 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:15:53.092 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.092 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.092 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:53.092 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579705+0000", 2023-09-23T23:15:53.093 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3e-06 2023-09-23T23:15:53.093 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.093 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.093 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:53.093 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579711+0000", 2023-09-23T23:15:53.093 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.601e-06 2023-09-23T23:15:53.093 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.093 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.093 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:53.093 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579714+0000", 2023-09-23T23:15:53.093 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9799999999999998e-06 2023-09-23T23:15:53.093 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.093 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.094 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.094 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579791+0000", 2023-09-23T23:15:53.094 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6923999999999997e-05 2023-09-23T23:15:53.094 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.094 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.094 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.094 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579795+0000", 2023-09-23T23:15:53.094 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5240000000000001e-06 2023-09-23T23:15:53.094 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.094 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.094 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.094 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222904+0000", 2023-09-23T23:15:53.094 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.643108825 2023-09-23T23:15:53.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.095 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.095 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.095 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222911+0000", 2023-09-23T23:15:53.095 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.233e-06 2023-09-23T23:15:53.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.095 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.095 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.095 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.340587+0000", 2023-09-23T23:15:53.095 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1176763380000001 2023-09-23T23:15:53.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.095 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.095 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:53.096 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.340704+0000", 2023-09-23T23:15:53.096 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011648 2023-09-23T23:15:53.096 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.096 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:15:53.096 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.340789+0000", 2023-09-23T23:15:53.096 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5054000000000003e-05 2023-09-23T23:15:53.096 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.096 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:53.096 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.352916+0000", 2023-09-23T23:15:53.096 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012127033000000001 2023-09-23T23:15:53.097 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.097 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.097 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.097 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353191+0000", 2023-09-23T23:15:53.097 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00027495599999999997 2023-09-23T23:15:53.097 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.097 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.097 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.097 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353283+0000", 2023-09-23T23:15:53.097 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2603999999999996e-05 2023-09-23T23:15:53.097 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.097 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.097 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.098 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.519298+0000", 2023-09-23T23:15:53.098 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16601465800000001 2023-09-23T23:15:53.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.098 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.098 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:53.098 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.519330+0000", 2023-09-23T23:15:53.098 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2190000000000002e-05 2023-09-23T23:15:53.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.098 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.098 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:53.098 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.519452+0000", 2023-09-23T23:15:53.098 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012235999999999999 2023-09-23T23:15:53.099 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.099 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:53.099 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.099 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.099 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.099 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127515 9.d 9:b8066f22:::benchmark_data_smithi110_295904_object16334:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:15:53.099 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.587604+0000", 2023-09-23T23:15:53.099 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 25.360456619000001, 2023-09-23T23:15:53.099 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9319918469999999, 2023-09-23T23:15:53.099 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:53.099 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:53.099 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:53.100 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:15:53.100 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:15:53.100 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127515 2023-09-23T23:15:53.100 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.100 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:53.100 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.100 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:53.100 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.587604+0000", 2023-09-23T23:15:53.100 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.100 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.100 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.100 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:53.100 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.587604+0000", 2023-09-23T23:15:53.101 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.101 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.101 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.101 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:53.101 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.587603+0000", 2023-09-23T23:15:53.101 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:15:53.101 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.101 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.101 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:53.101 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.587609+0000", 2023-09-23T23:15:53.101 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9089999999999997e-06 2023-09-23T23:15:53.101 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.101 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.102 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:53.102 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.587624+0000", 2023-09-23T23:15:53.102 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5162e-05 2023-09-23T23:15:53.102 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.102 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.102 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:53.102 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.587630+0000", 2023-09-23T23:15:53.102 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2060000000000004e-06 2023-09-23T23:15:53.102 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.102 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.102 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.102 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.587679+0000", 2023-09-23T23:15:53.102 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8223999999999998e-05 2023-09-23T23:15:53.103 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.103 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.103 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.103 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.587686+0000", 2023-09-23T23:15:53.103 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9940000000000003e-06 2023-09-23T23:15:53.103 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.103 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.103 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.103 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222948+0000", 2023-09-23T23:15:53.103 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.63526185199999996 2023-09-23T23:15:53.103 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.103 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.104 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222955+0000", 2023-09-23T23:15:53.104 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2880000000000004e-06 2023-09-23T23:15:53.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.104 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.104 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.343431+0000", 2023-09-23T23:15:53.104 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.120475756 2023-09-23T23:15:53.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.104 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.104 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:53.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.343552+0000", 2023-09-23T23:15:53.104 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000121666 2023-09-23T23:15:53.105 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.105 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:15:53.105 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.343632+0000", 2023-09-23T23:15:53.105 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9900000000000004e-05 2023-09-23T23:15:53.105 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.105 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:53.105 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.352930+0000", 2023-09-23T23:15:53.105 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.009298054 2023-09-23T23:15:53.105 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.106 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.106 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353238+0000", 2023-09-23T23:15:53.106 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000307569 2023-09-23T23:15:53.106 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.106 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.106 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.106 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353329+0000", 2023-09-23T23:15:53.106 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0870000000000002e-05 2023-09-23T23:15:53.106 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.106 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.106 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.106 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.519497+0000", 2023-09-23T23:15:53.106 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16616813799999999 2023-09-23T23:15:53.107 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.107 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.107 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:53.107 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.519523+0000", 2023-09-23T23:15:53.107 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.582e-05 2023-09-23T23:15:53.107 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.107 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.107 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:53.107 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.519596+0000", 2023-09-23T23:15:53.107 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3528000000000005e-05 2023-09-23T23:15:53.107 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.107 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:53.107 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.108 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.108 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.108 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127518 9.d 9:b6c2eec4:::benchmark_data_smithi110_295904_object16337:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:15:53.108 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.592077+0000", 2023-09-23T23:15:53.108 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 25.355983820999999, 2023-09-23T23:15:53.108 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.930210711, 2023-09-23T23:15:53.108 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:53.108 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:53.108 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:53.108 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:15:53.108 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:15:53.108 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127518 2023-09-23T23:15:53.108 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.109 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:53.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.109 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:53.109 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.592077+0000", 2023-09-23T23:15:53.109 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.109 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:53.109 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.592077+0000", 2023-09-23T23:15:53.109 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.109 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:53.110 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.592076+0000", 2023-09-23T23:15:53.110 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:15:53.110 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.110 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.110 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:53.110 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.592081+0000", 2023-09-23T23:15:53.110 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6569999999999996e-06 2023-09-23T23:15:53.110 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.110 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.110 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:53.110 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.592094+0000", 2023-09-23T23:15:53.110 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2619000000000001e-05 2023-09-23T23:15:53.111 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:53.111 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.592099+0000", 2023-09-23T23:15:53.111 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0780000000000003e-06 2023-09-23T23:15:53.111 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.111 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.592140+0000", 2023-09-23T23:15:53.111 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1115000000000002e-05 2023-09-23T23:15:53.111 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.592145+0000", 2023-09-23T23:15:53.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5899999999999998e-06 2023-09-23T23:15:53.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222978+0000", 2023-09-23T23:15:53.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.63083219899999998 2023-09-23T23:15:53.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222985+0000", 2023-09-23T23:15:53.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9600000000000003e-06 2023-09-23T23:15:53.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.346170+0000", 2023-09-23T23:15:53.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1231851390000001 2023-09-23T23:15:53.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:53.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.346304+0000", 2023-09-23T23:15:53.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013475999999999999 2023-09-23T23:15:53.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:15:53.114 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.346442+0000", 2023-09-23T23:15:53.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013756700000000001 2023-09-23T23:15:53.114 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.114 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:53.114 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.352939+0000", 2023-09-23T23:15:53.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064967480000000001 2023-09-23T23:15:53.114 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.114 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.114 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353307+0000", 2023-09-23T23:15:53.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00036789100000000002 2023-09-23T23:15:53.115 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.115 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.115 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.115 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.358342+0000", 2023-09-23T23:15:53.115 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050351759999999997 2023-09-23T23:15:53.115 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.115 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.115 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.115 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.522137+0000", 2023-09-23T23:15:53.115 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.163795633 2023-09-23T23:15:53.115 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.115 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.115 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:53.116 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.522184+0000", 2023-09-23T23:15:53.116 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6350999999999997e-05 2023-09-23T23:15:53.116 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.116 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.116 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:53.116 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.522288+0000", 2023-09-23T23:15:53.116 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010393800000000001 2023-09-23T23:15:53.116 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.116 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:53.116 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.116 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.116 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.117 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127533 9.d 9:b4a925a7:::benchmark_data_smithi110_295904_object16352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:15:53.117 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.618409+0000", 2023-09-23T23:15:53.117 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 25.329651500000001, 2023-09-23T23:15:53.117 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9040714919999999, 2023-09-23T23:15:53.117 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:53.117 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:53.117 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:53.117 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:15:53.117 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:15:53.117 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127533 2023-09-23T23:15:53.117 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.117 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:53.117 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.118 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:53.118 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.618409+0000", 2023-09-23T23:15:53.118 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.118 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.118 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.118 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:53.118 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.618409+0000", 2023-09-23T23:15:53.118 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.118 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.118 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.118 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:53.118 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.618408+0000", 2023-09-23T23:15:53.119 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:53.119 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.119 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.119 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:53.119 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.618413+0000", 2023-09-23T23:15:53.119 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4079999999999997e-06 2023-09-23T23:15:53.119 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.119 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.119 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:53.119 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.618426+0000", 2023-09-23T23:15:53.119 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2021000000000001e-05 2023-09-23T23:15:53.119 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.119 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.120 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:53.120 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.618430+0000", 2023-09-23T23:15:53.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8600000000000001e-06 2023-09-23T23:15:53.120 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.120 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.120 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.120 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.618497+0000", 2023-09-23T23:15:53.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6228000000000004e-05 2023-09-23T23:15:53.120 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.120 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.120 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.120 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.618506+0000", 2023-09-23T23:15:53.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4709999999999996e-06 2023-09-23T23:15:53.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.223043+0000", 2023-09-23T23:15:53.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.60453667499999997 2023-09-23T23:15:53.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.223052+0000", 2023-09-23T23:15:53.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.081e-06 2023-09-23T23:15:53.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.122 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.349615+0000", 2023-09-23T23:15:53.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1265627229999999 2023-09-23T23:15:53.122 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.122 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.122 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:53.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.349757+0000", 2023-09-23T23:15:53.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014230100000000001 2023-09-23T23:15:53.122 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.122 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.122 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:15:53.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.349908+0000", 2023-09-23T23:15:53.123 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000151323 2023-09-23T23:15:53.123 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.123 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.123 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:53.123 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353017+0000", 2023-09-23T23:15:53.123 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031085539999999999 2023-09-23T23:15:53.123 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.123 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.123 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.123 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353350+0000", 2023-09-23T23:15:53.123 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00033329399999999998 2023-09-23T23:15:53.123 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.123 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.124 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.124 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.358375+0000", 2023-09-23T23:15:53.124 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005024934 2023-09-23T23:15:53.124 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.124 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.124 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.124 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.522338+0000", 2023-09-23T23:15:53.124 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16396344700000001 2023-09-23T23:15:53.124 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.124 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.124 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:53.124 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.522385+0000", 2023-09-23T23:15:53.125 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6489999999999997e-05 2023-09-23T23:15:53.125 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.125 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.125 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:53.125 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.522481+0000", 2023-09-23T23:15:53.125 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5995999999999994e-05 2023-09-23T23:15:53.125 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.125 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:53.125 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.125 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.125 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.125 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:136466 9.d 9:bc0d042c:::benchmark_data_smithi110_295904_object25285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e192)", 2023-09-23T23:15:53.125 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:46.434300+0000", 2023-09-23T23:15:53.126 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 6.5137610219999997, 2023-09-23T23:15:53.126 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0567469730000001, 2023-09-23T23:15:53.126 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:53.126 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:53.126 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:53.126 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:15:53.126 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:15:53.126 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 136466 2023-09-23T23:15:53.126 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.126 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:53.126 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.126 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:53.126 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.434300+0000", 2023-09-23T23:15:53.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.127 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.127 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.127 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:53.127 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.434300+0000", 2023-09-23T23:15:53.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.127 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.127 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.127 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:53.127 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.434298+0000", 2023-09-23T23:15:53.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:53.128 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.128 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:53.128 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.434306+0000", 2023-09-23T23:15:53.128 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2509999999999997e-06 2023-09-23T23:15:53.128 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.128 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:53.128 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.434318+0000", 2023-09-23T23:15:53.128 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2771e-05 2023-09-23T23:15:53.128 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.128 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.129 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:53.129 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.434323+0000", 2023-09-23T23:15:53.129 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9660000000000004e-06 2023-09-23T23:15:53.129 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.129 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.129 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.129 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.434372+0000", 2023-09-23T23:15:53.129 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8775e-05 2023-09-23T23:15:53.129 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.129 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.129 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.129 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.434381+0000", 2023-09-23T23:15:53.129 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9290000000000004e-06 2023-09-23T23:15:53.130 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.130 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.130 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.130 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.444524+0000", 2023-09-23T23:15:53.130 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0101426259999999 2023-09-23T23:15:53.130 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.130 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.130 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:53.130 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.444677+0000", 2023-09-23T23:15:53.130 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015341700000000001 2023-09-23T23:15:53.130 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.130 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.131 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:15:53.131 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.444834+0000", 2023-09-23T23:15:53.131 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015701499999999999 2023-09-23T23:15:53.131 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.131 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.131 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:53.131 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.453710+0000", 2023-09-23T23:15:53.131 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088758589999999998 2023-09-23T23:15:53.131 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.131 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.131 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.131 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.490907+0000", 2023-09-23T23:15:53.131 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.037197075000000003 2023-09-23T23:15:53.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.132 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.132 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.132 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.490952+0000", 2023-09-23T23:15:53.132 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4617999999999999e-05 2023-09-23T23:15:53.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.132 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.132 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:53.132 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.491009+0000", 2023-09-23T23:15:53.132 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7163000000000001e-05 2023-09-23T23:15:53.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.132 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.133 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:53.133 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.491047+0000", 2023-09-23T23:15:53.133 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7910999999999997e-05 2023-09-23T23:15:53.133 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.133 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:53.133 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.133 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.133 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:136501 9.d 9:b0419d0f:::benchmark_data_smithi110_295904_object25320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e192)", 2023-09-23T23:15:53.133 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:46.547205+0000", 2023-09-23T23:15:53.133 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 6.400855569, 2023-09-23T23:15:53.133 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94398360299999995, 2023-09-23T23:15:53.133 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:15:53.134 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:15:53.134 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:15:53.134 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:15:53.134 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:15:53.134 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 136501 2023-09-23T23:15:53.134 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.134 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:15:53.134 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.134 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:15:53.134 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.547205+0000", 2023-09-23T23:15:53.134 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.134 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:15:53.135 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.547205+0000", 2023-09-23T23:15:53.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:15:53.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:15:53.135 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.547204+0000", 2023-09-23T23:15:53.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:15:53.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:15:53.136 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.547209+0000", 2023-09-23T23:15:53.136 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3959999999999997e-06 2023-09-23T23:15:53.136 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.136 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.136 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:15:53.136 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.547222+0000", 2023-09-23T23:15:53.136 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3185e-05 2023-09-23T23:15:53.136 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.136 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.136 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:15:53.136 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.547229+0000", 2023-09-23T23:15:53.136 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5690000000000001e-06 2023-09-23T23:15:53.136 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.137 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.137 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.137 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.547280+0000", 2023-09-23T23:15:53.137 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1319000000000001e-05 2023-09-23T23:15:53.137 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.137 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.137 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:15:53.137 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.547288+0000", 2023-09-23T23:15:53.137 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6120000000000001e-06 2023-09-23T23:15:53.137 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.137 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.137 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:15:53.137 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.447782+0000", 2023-09-23T23:15:53.138 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.90049416999999998 2023-09-23T23:15:53.138 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.138 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.138 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:15:53.138 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.447910+0000", 2023-09-23T23:15:53.138 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012784499999999999 2023-09-23T23:15:53.138 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.138 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.138 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:15:53.138 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.447993+0000", 2023-09-23T23:15:53.138 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3419999999999998e-05 2023-09-23T23:15:53.138 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.139 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.139 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:15:53.139 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.465774+0000", 2023-09-23T23:15:53.139 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017780267999999998 2023-09-23T23:15:53.139 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.139 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.139 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.139 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.491086+0000", 2023-09-23T23:15:53.139 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.025311951999999999 2023-09-23T23:15:53.139 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.139 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.139 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:15:53.140 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.491112+0000", 2023-09-23T23:15:53.140 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6370999999999999e-05 2023-09-23T23:15:53.140 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.140 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.140 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:15:53.140 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.491158+0000", 2023-09-23T23:15:53.140 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5908999999999997e-05 2023-09-23T23:15:53.140 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:15:53.140 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:15:53.140 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:15:53.140 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.491189+0000", 2023-09-23T23:15:53.140 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0997000000000002e-05 2023-09-23T23:15:53.140 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.141 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:53.141 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.141 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:15:53.141 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:15:53.141 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:15:53.142 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:53.177 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:53.242 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:53.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:53.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:53.334 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:15:53.346 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:15:53.376 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:15:53.386 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:53.414 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:15:53.444 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:15:53.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:53.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:15:53.451 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:15:53.491 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:15:53.520 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:15:53.549 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:15:53.560 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:15:53.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:53 smithi138 ceph-mon[193606]: osd.2 marked itself down and dead 2023-09-23T23:15:53.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:53 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1023717192' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:53.577 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:53.601 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:15:53.615 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:15:53.615 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:15:53.616 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:15:53.616 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:15:53.616 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:15:53.616 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:15:53.616 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:15:53.616 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:15:53.636 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:53 smithi110 ceph-mon[212939]: osd.2 marked itself down and dead 2023-09-23T23:15:53.637 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:53 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1023717192' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:53.637 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:53 smithi110 ceph-mon[221102]: osd.2 marked itself down and dead 2023-09-23T23:15:53.637 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:53 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1023717192' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:15:53.838 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:15:53.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:53.843 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:15:53.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:15:53.944 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:15:53.972 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:54.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:54 smithi110 ceph-mon[212939]: pgmap v634: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 97 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 15478/252387 objects misplaced (6.133%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T23:15:54.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:54 smithi110 ceph-mon[212939]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:15:54.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:54 smithi110 ceph-mon[212939]: osdmap e194: 8 total, 7 up, 8 in 2023-09-23T23:15:54.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:54 smithi110 ceph-mon[212939]: failed to encode map e194 with expected crc 2023-09-23T23:15:54.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:54 smithi110 ceph-mon[221102]: pgmap v634: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 97 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 15478/252387 objects misplaced (6.133%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T23:15:54.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:54 smithi110 ceph-mon[221102]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:15:54.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:54 smithi110 ceph-mon[221102]: osdmap e194: 8 total, 7 up, 8 in 2023-09-23T23:15:54.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:54 smithi110 ceph-mon[221102]: failed to encode map e194 with expected crc 2023-09-23T23:15:54.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:54 smithi138 ceph-mon[193606]: pgmap v634: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 97 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 15478/252387 objects misplaced (6.133%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T23:15:54.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:54 smithi138 ceph-mon[193606]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:15:54.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:54 smithi138 ceph-mon[193606]: osdmap e194: 8 total, 7 up, 8 in 2023-09-23T23:15:54.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:54 smithi138 ceph-mon[193606]: failed to encode map e194 with expected crc 2023-09-23T23:15:55.044 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:15:55.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:55.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:15:55.158 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:15:55.195 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:55.238 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:15:55.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:55.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:15:55.344 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:55.373 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:55.463 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:55.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:55.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:55.569 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:55.607 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:55.667 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:55.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:55.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:55.768 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:55 smithi110 ceph-mon[212939]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.769 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:55 smithi110 ceph-mon[212939]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.769 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:55 smithi110 ceph-mon[212939]: osdmap e195: 8 total, 7 up, 8 in 2023-09-23T23:15:55.769 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:55 smithi110 ceph-mon[212939]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.769 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:55 smithi110 ceph-mon[212939]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.769 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:55 smithi110 ceph-mon[212939]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.769 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:55 smithi110 ceph-mon[212939]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.769 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:55 smithi110 ceph-mon[212939]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.769 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:55 smithi110 ceph-mon[221102]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.769 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:55 smithi110 ceph-mon[221102]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.769 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:55 smithi110 ceph-mon[221102]: osdmap e195: 8 total, 7 up, 8 in 2023-09-23T23:15:55.770 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:55 smithi110 ceph-mon[221102]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.770 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:55 smithi110 ceph-mon[221102]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.770 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:55 smithi110 ceph-mon[221102]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.770 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:55 smithi110 ceph-mon[221102]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.770 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:55 smithi110 ceph-mon[221102]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.772 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:15:55.799 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:55 smithi138 ceph-mon[193606]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.799 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:55 smithi138 ceph-mon[193606]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.799 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:55 smithi138 ceph-mon[193606]: osdmap e195: 8 total, 7 up, 8 in 2023-09-23T23:15:55.799 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:55 smithi138 ceph-mon[193606]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.799 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:55 smithi138 ceph-mon[193606]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.799 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:55 smithi138 ceph-mon[193606]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.799 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:55 smithi138 ceph-mon[193606]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.799 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:55 smithi138 ceph-mon[193606]: failed to encode map e195 with expected crc 2023-09-23T23:15:55.801 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:55.846 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:15:55.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:55.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:15:55.952 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:55.980 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:56.035 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:56.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:56.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:56.140 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:56.166 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:56.201 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:15:55 smithi110 bash[323932]: 94b7643f91e8dfd098dfd2109d459edc6b49adfe015546cb45abae514b42d926 2023-09-23T23:15:56.207 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:56.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:56.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:56.313 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:56.378 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:56.634 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:15:56.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:56.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:15:56.645 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:56 smithi110 ceph-mon[221102]: pgmap v637: 99 pgs: 20 peering, 2 stale+active+clean, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 75 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 15478/248034 objects misplaced (6.240%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T23:15:56.645 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:56 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:15:56.646 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:56 smithi110 ceph-mon[212939]: pgmap v637: 99 pgs: 20 peering, 2 stale+active+clean, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 75 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 15478/248034 objects misplaced (6.240%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T23:15:56.646 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:56 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:15:56.745 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:15:56.790 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:56 smithi138 ceph-mon[193606]: pgmap v637: 99 pgs: 20 peering, 2 stale+active+clean, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 75 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 15478/248034 objects misplaced (6.240%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T23:15:56.790 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:56 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:15:56.792 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:56.852 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:15:56.852 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:15:56.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:56.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:15:56.956 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:15:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:15:56] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.43.0" 2023-09-23T23:15:56.958 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:56.992 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:57.046 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:57.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:57.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:57.151 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:57.188 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:57.245 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:57.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:57.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:57.353 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:15:57.407 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:57.713 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:15:57.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:57.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:15:57.822 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:15:57.858 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:57.904 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:15:57.917 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:15:57.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:57.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:15:58.024 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:58.051 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:58.112 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:58.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:58.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:58.221 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:15:58.247 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:58.287 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:15:58.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:58.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:15:58.392 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:58.422 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:58.477 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:58.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:58.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:58.536 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:15:58.588 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:58.620 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:58.673 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:58.678 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:58 smithi138 ceph-mon[193606]: pgmap v638: 99 pgs: 72 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized, 2 stale+active+clean, 20 peering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 402 op/s; 3/246273 objects degraded (0.001%); 15052/246273 objects misplaced (6.112%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:15:58.679 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:58 smithi138 ceph-mon[193606]: Health check failed: Degraded data redundancy: 3/246273 objects degraded (0.001%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:15:58.679 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:58 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3970063160' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:58.679 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:58 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:58.679 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:58.679 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:15:58.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:58.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:58.781 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:58.819 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:58.874 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:58.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:58.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:58.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:58 smithi110 ceph-mon[212939]: pgmap v638: 99 pgs: 72 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized, 2 stale+active+clean, 20 peering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 402 op/s; 3/246273 objects degraded (0.001%); 15052/246273 objects misplaced (6.112%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:15:58.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:58 smithi110 ceph-mon[212939]: Health check failed: Degraded data redundancy: 3/246273 objects degraded (0.001%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:15:58.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:58 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3970063160' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:58.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:58 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:58.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:58.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:15:58.963 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:58 smithi110 ceph-mon[221102]: pgmap v638: 99 pgs: 72 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized, 2 stale+active+clean, 20 peering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 402 op/s; 3/246273 objects degraded (0.001%); 15052/246273 objects misplaced (6.112%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:15:58.963 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:58 smithi110 ceph-mon[221102]: Health check failed: Degraded data redundancy: 3/246273 objects degraded (0.001%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:15:58.963 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:58 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3970063160' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:58.963 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:58 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:15:58.963 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:15:58.963 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:15:58.979 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:15:59.016 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:59.069 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:15:59.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:59.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:15:59.182 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:15:59.208 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:59.268 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:15:59.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:59.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:15:59.377 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:15:59.403 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:59.443 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:15:59.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:59.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:15:59.549 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:15:59.582 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:59 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:15:59.582 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:59 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:15:59.582 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:59 smithi138 ceph-mon[193606]: osdmap e196: 8 total, 7 up, 8 in 2023-09-23T23:15:59.582 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:59 smithi138 ceph-mon[193606]: failed to encode map e196 with expected crc 2023-09-23T23:15:59.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:59 smithi138 ceph-mon[193606]: failed to encode map e196 with expected crc 2023-09-23T23:15:59.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:59 smithi138 ceph-mon[193606]: failed to encode map e196 with expected crc 2023-09-23T23:15:59.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:59 smithi138 ceph-mon[193606]: failed to encode map e196 with expected crc 2023-09-23T23:15:59.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:15:59 smithi138 ceph-mon[193606]: failed to encode map e196 with expected crc 2023-09-23T23:15:59.587 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:15:59.639 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:15:59.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:15:59.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:15:59.744 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:15:59.825 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:59 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:15:59.825 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:59 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:15:59.825 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:59 smithi110 ceph-mon[212939]: osdmap e196: 8 total, 7 up, 8 in 2023-09-23T23:15:59.825 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:59 smithi110 ceph-mon[212939]: failed to encode map e196 with expected crc 2023-09-23T23:15:59.825 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:59 smithi110 ceph-mon[212939]: failed to encode map e196 with expected crc 2023-09-23T23:15:59.825 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:59 smithi110 ceph-mon[212939]: failed to encode map e196 with expected crc 2023-09-23T23:15:59.825 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:59 smithi110 ceph-mon[212939]: failed to encode map e196 with expected crc 2023-09-23T23:15:59.825 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:15:59 smithi110 ceph-mon[212939]: failed to encode map e196 with expected crc 2023-09-23T23:15:59.828 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:59 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:15:59.828 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:59 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:15:59.828 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:59 smithi110 ceph-mon[221102]: osdmap e196: 8 total, 7 up, 8 in 2023-09-23T23:15:59.828 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:59 smithi110 ceph-mon[221102]: failed to encode map e196 with expected crc 2023-09-23T23:15:59.828 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:59 smithi110 ceph-mon[221102]: failed to encode map e196 with expected crc 2023-09-23T23:15:59.828 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:59 smithi110 ceph-mon[221102]: failed to encode map e196 with expected crc 2023-09-23T23:15:59.828 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:59 smithi110 ceph-mon[221102]: failed to encode map e196 with expected crc 2023-09-23T23:15:59.828 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:15:59 smithi110 ceph-mon[221102]: failed to encode map e196 with expected crc 2023-09-23T23:15:59.845 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:00.136 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:00.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:00.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:00.240 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:00.272 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:00.333 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:00.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:00.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:00.447 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:00.477 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:00.518 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:00.521 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:00 smithi138 ceph-mon[193606]: pgmap v640: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 642 op/s; 15063/241008 objects degraded (6.250%); 15052/241008 objects misplaced (6.245%); 1.5 MiB/s, 22 objects/s recovering 2023-09-23T23:16:00.522 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:00 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:16:00.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:00.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:00.623 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:00.651 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:00.699 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:00.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:00.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:00.773 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:00 smithi110 ceph-mon[221102]: pgmap v640: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 642 op/s; 15063/241008 objects degraded (6.250%); 15052/241008 objects misplaced (6.245%); 1.5 MiB/s, 22 objects/s recovering 2023-09-23T23:16:00.774 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:00 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:16:00.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:00 smithi110 ceph-mon[212939]: pgmap v640: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 642 op/s; 15063/241008 objects degraded (6.250%); 15052/241008 objects misplaced (6.245%); 1.5 MiB/s, 22 objects/s recovering 2023-09-23T23:16:00.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:00 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:16:00.806 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:16:00.875 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:01.041 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:16:01.041 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:16:01.085 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:16:01.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:01.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:16:01.148 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T23:16:01.195 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:01.228 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:01.262 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:16:01.262 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T23:16:01.263 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:01.289 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:01.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:01.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:01.394 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:01.429 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:01.508 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:01.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:01.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:01.617 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:01.673 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:02.200 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:02.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:02.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:02.255 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:16:02.315 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:02.413 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:02.532 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:02.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:02.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:02.647 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:02.736 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:02 smithi110 ceph-mon[212939]: pgmap v641: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 11 KiB/s rd, 28 KiB/s wr, 572 op/s; 15063/239067 objects degraded (6.301%); 14517/239067 objects misplaced (6.072%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T23:16:02.736 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:02 smithi110 ceph-mon[221102]: pgmap v641: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 11 KiB/s rd, 28 KiB/s wr, 572 op/s; 15063/239067 objects degraded (6.301%); 14517/239067 objects misplaced (6.072%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T23:16:02.746 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:02.771 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:16:02.939 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:02.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:02 smithi138 ceph-mon[193606]: pgmap v641: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 11 KiB/s rd, 28 KiB/s wr, 572 op/s; 15063/239067 objects degraded (6.301%); 14517/239067 objects misplaced (6.072%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T23:16:02.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:02.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:03.050 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:16:03.079 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:03.134 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:16:03.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:03.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:16:03.242 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:03.288 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:03.334 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:03.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:03.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:03.443 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:03.468 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:03.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:03 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 15063/239067 objects degraded (6.301%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:03.840 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:03.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:03.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:03.846 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:03 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 15063/239067 objects degraded (6.301%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:03.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:03 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 15063/239067 objects degraded (6.301%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:03.946 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:04.025 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:04.154 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:04.170 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:04.221 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:04.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:04.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:04.325 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:04.399 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:04.658 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:04.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:04.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:04.764 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:04.793 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:04.824 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:04 smithi110 ceph-mon[212939]: pgmap v642: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 9.3 KiB/s rd, 25 KiB/s wr, 403 op/s; 14891/237483 objects degraded (6.270%); 14517/237483 objects misplaced (6.113%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:16:04.824 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:04 smithi110 ceph-mon[221102]: pgmap v642: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 9.3 KiB/s rd, 25 KiB/s wr, 403 op/s; 14891/237483 objects degraded (6.270%); 14517/237483 objects misplaced (6.113%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:16:04.844 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:04.847 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:04 smithi138 ceph-mon[193606]: pgmap v642: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 9.3 KiB/s rd, 25 KiB/s wr, 403 op/s; 14891/237483 objects degraded (6.270%); 14517/237483 objects misplaced (6.113%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:16:04.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:04.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:04.906 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:16:04.949 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:04.950 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:16:05.020 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:16:05.044 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:05.070 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:16:05.108 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:16:05.140 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:16:05.188 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:16:05.189 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:16:05.189 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:16:05.189 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T23:16:05.189 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:16:05.189 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:16:05.189 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:16:05.189 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:16:05.189 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:16:05.295 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:05.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:05.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:05.402 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:05.441 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:05.503 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:05.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:05.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:05.610 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:05.661 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:05.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:05 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1407296814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:16:05.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:05 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1407296814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:16:05.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:05 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1407296814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:16:05.738 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:05.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:05.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:05.851 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:05.935 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:06.217 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:06.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:06.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:06.263 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [2] live_osds: [0, 1, 3, 4, 5, 6, 7] 2023-09-23T23:16:06.264 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:16:06.264 INFO:tasks.thrashosds.thrasher:Reviving osd 2 2023-09-23T23:16:06.264 INFO:tasks.cephadm.osd.2:Restarting osd.2 (starting--it wasn't running)... 2023-09-23T23:16:06.264 DEBUG:teuthology.orchestra.run.smithi110:osd.2> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T23:16:06.267 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl start ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T23:16:06.323 INFO:journalctl@ceph.osd.2.smithi110.stdout:-- Logs begin at Sat 2023-09-23 22:30:57 UTC. -- 2023-09-23T23:16:06.334 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:06.376 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:06.443 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:06.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:06.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:06.549 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:06.561 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:16:06 smithi110 systemd[1]: Starting Ceph osd.2 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:16:06.590 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:06.634 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:06.639 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:06 smithi138 ceph-mon[193606]: pgmap v643: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8.3 KiB/s rd, 22 KiB/s wr, 520 op/s; 14560/232596 objects degraded (6.260%); 14517/232596 objects misplaced (6.241%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:16:06.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:06.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:06.741 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:06.799 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:06.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:06 smithi110 ceph-mon[212939]: pgmap v643: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8.3 KiB/s rd, 22 KiB/s wr, 520 op/s; 14560/232596 objects degraded (6.260%); 14517/232596 objects misplaced (6.241%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:16:06.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:16:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:16:06] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.43.0" 2023-09-23T23:16:06.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:06 smithi110 ceph-mon[221102]: pgmap v643: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8.3 KiB/s rd, 22 KiB/s wr, 520 op/s; 14560/232596 objects degraded (6.260%); 14517/232596 objects misplaced (6.241%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:16:07.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:07 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:16:07.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:07 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:16:07.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:07 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:16:08.259 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:08.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:08.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:08.370 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:08.415 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:08.420 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:16:08.421 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:16:08.483 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:08.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:08.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:08.590 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:16:08.665 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:08 smithi110 ceph-mon[212939]: pgmap v644: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8.3 KiB/s rd, 22 KiB/s wr, 537 op/s; 14560/230331 objects degraded (6.321%); 13970/230331 objects misplaced (6.065%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T23:16:08.665 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:08 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 14560/230331 objects degraded (6.321%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:08.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:08 smithi110 ceph-mon[221102]: pgmap v644: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8.3 KiB/s rd, 22 KiB/s wr, 537 op/s; 14560/230331 objects degraded (6.321%); 13970/230331 objects misplaced (6.065%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T23:16:08.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:08 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 14560/230331 objects degraded (6.321%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:08.682 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:08.697 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:08 smithi138 ceph-mon[193606]: pgmap v644: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8.3 KiB/s rd, 22 KiB/s wr, 537 op/s; 14560/230331 objects degraded (6.321%); 13970/230331 objects misplaced (6.065%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T23:16:08.697 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:08 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 14560/230331 objects degraded (6.321%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:08.976 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:16:08.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:08.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:16:09.000 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:16:09.001 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:16:09.001 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:16:09.001 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:16:09.001 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:16:09.084 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:09.138 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:09.235 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:09.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:09.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:09.345 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:09.448 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:09.606 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:16:09.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:09 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2716472646' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:16:09.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:09 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2716472646' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:16:09.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:09 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2716472646' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:16:10.422 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:16:10.786 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:16:10 smithi110 bash[327376]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T23:16:10.787 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:16:10 smithi110 bash[327376]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T23:16:10.787 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:16:10 smithi110 bash[327376]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T23:16:10.787 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:16:10 smithi110 bash[327376]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T23:16:10.787 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:16:10 smithi110 bash[327376]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T23:16:10.787 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:16:10 smithi110 bash[327376]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T23:16:10.787 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:16:10 smithi110 bash[327376]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T23:16:10.787 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:16:10 smithi110 bash[327376]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T23:16:10.787 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:10 smithi110 ceph-mon[212939]: pgmap v645: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 13964/223455 objects degraded (6.249%); 13970/223455 objects misplaced (6.252%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:16:10.787 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:10 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:16:10.788 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:10 smithi110 ceph-mon[212939]: failed to encode map e197 with expected crc 2023-09-23T23:16:10.788 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:10 smithi110 ceph-mon[212939]: failed to encode map e197 with expected crc 2023-09-23T23:16:10.788 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:10 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2716472646' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:16:10.788 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:10 smithi110 ceph-mon[212939]: osdmap e197: 8 total, 7 up, 8 in 2023-09-23T23:16:10.788 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:10 smithi110 ceph-mon[221102]: pgmap v645: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 13964/223455 objects degraded (6.249%); 13970/223455 objects misplaced (6.252%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:16:10.788 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:10 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:16:10.788 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:10 smithi110 ceph-mon[221102]: failed to encode map e197 with expected crc 2023-09-23T23:16:10.788 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:10 smithi110 ceph-mon[221102]: failed to encode map e197 with expected crc 2023-09-23T23:16:10.788 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:10 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2716472646' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:16:10.788 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:10 smithi110 ceph-mon[221102]: osdmap e197: 8 total, 7 up, 8 in 2023-09-23T23:16:10.833 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:10.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:10.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:10.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:10 smithi138 ceph-mon[193606]: pgmap v645: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 13964/223455 objects degraded (6.249%); 13970/223455 objects misplaced (6.252%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:16:10.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:10 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:16:10.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:10 smithi138 ceph-mon[193606]: failed to encode map e197 with expected crc 2023-09-23T23:16:10.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:10 smithi138 ceph-mon[193606]: failed to encode map e197 with expected crc 2023-09-23T23:16:10.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:10 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2716472646' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:16:10.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:10 smithi138 ceph-mon[193606]: osdmap e197: 8 total, 7 up, 8 in 2023-09-23T23:16:10.944 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:11.007 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:11.477 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:16:11.580 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:11.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:11.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:11.686 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:16:11.728 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:11.810 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:16:11.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:11.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:16:11.925 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:11.987 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:12.521 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:12.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:12.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:12.628 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:12.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:12 smithi138 ceph-mon[193606]: pgmap v647: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 13964/221244 objects degraded (6.312%); 13424/221244 objects misplaced (6.068%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:16:12.697 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:12 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:12.697 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:12 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:12.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:12 smithi110 ceph-mon[212939]: pgmap v647: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 13964/221244 objects degraded (6.312%); 13424/221244 objects misplaced (6.068%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:16:12.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:12 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:12.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:12 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:12.704 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:12.709 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:12 smithi110 ceph-mon[221102]: pgmap v647: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 13964/221244 objects degraded (6.312%); 13424/221244 objects misplaced (6.068%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:16:12.709 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:12 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:12.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:12 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:12.823 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:12.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:12.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:12.949 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:12.993 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:13.024 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:13.039 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:13.211 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:13.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:13.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:13.317 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:13.410 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:13.671 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:13.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:13.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:13.786 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:13.880 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:14.215 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:14.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:14.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:14.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:14.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:16:14.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:13 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 13964/221244 objects degraded (6.312%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:14.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:13 smithi110 ceph-mon[212939]: pgmap v648: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 13791/219564 objects degraded (6.281%); 13424/219564 objects misplaced (6.114%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:16:14.255 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:14.255 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:16:14.255 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:13 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 13964/221244 objects degraded (6.312%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:14.255 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:13 smithi110 ceph-mon[221102]: pgmap v648: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 13791/219564 objects degraded (6.281%); 13424/219564 objects misplaced (6.114%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:16:14.330 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:14.356 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:16:14.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:14.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:16:14.363 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:13 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 13964/221244 objects degraded (6.312%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:14.363 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:13 smithi138 ceph-mon[193606]: pgmap v648: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 13791/219564 objects degraded (6.281%); 13424/219564 objects misplaced (6.114%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:16:14.365 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:14.426 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:14.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:14.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:14.431 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:16:14.518 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:16:14 smithi110 bash[328694]: ebe9f18b6270c3d57f236c8f1c0da9938009320a4e1d397c3dace0938ae496e9 2023-09-23T23:16:14.519 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:16:14 smithi110 systemd[1]: Started Ceph osd.2 for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:16:14.532 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:16:14.564 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:14.616 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:16:14.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:14.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:16:14.719 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:16:14.752 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:14.806 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:16:14.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:14.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:16:14.919 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:14.951 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:16:14 smithi110 conmon[328908]: 2023-09-23T23:16:14.638+0000 7fa60359a640 -1 Falling back to public interface 2023-09-23T23:16:14.955 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:15.024 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:15.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:15.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:15.130 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:15.177 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:15.266 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:15.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:15.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:15.374 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:16:15.443 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:16.538 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:16:16.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:16.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:16:16.654 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:16.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:16 smithi138 ceph-mon[193606]: pgmap v649: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 13431/214578 objects degraded (6.259%); 13424/214578 objects misplaced (6.256%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:16:16.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:16 smithi110 ceph-mon[221102]: pgmap v649: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 13431/214578 objects degraded (6.259%); 13424/214578 objects misplaced (6.256%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:16:16.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:16 smithi110 ceph-mon[212939]: pgmap v649: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 13431/214578 objects degraded (6.259%); 13424/214578 objects misplaced (6.256%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:16:16.749 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:16.956 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:16:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:16:16] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.43.0" 2023-09-23T23:16:16.976 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:16.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:16.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:17.093 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:16:17.183 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:17.250 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:17.473 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:16:17.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:17.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:16:17.587 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:16:17.618 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:17.648 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:16:17.674 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:16:17.674 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:16:17.690 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:16:17.691 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:16:17.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:17.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:16:17.710 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:17.767 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:16:17.796 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:17.815 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:16:17.828 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:17.846 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:16:17.876 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:16:17.890 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:17.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:17.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:17.912 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:16:17.912 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:16:17.912 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:16:17.913 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T23:16:17.913 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:16:17.913 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:16:17.913 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:16:17.913 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:16:17.913 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:16:18.000 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:18.033 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:18.094 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:18.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:18.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:18.202 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:18.235 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:18.304 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:18.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:18.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:18.414 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:18.436 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:18 smithi110 ceph-mon[212939]: pgmap v650: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 13431/212496 objects degraded (6.321%); 12934/212496 objects misplaced (6.087%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:16:18.436 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/978021344' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:16:18.436 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:18 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 13431/212496 objects degraded (6.321%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:18.443 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:18.501 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:18.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:18.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:18.507 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:18 smithi138 ceph-mon[193606]: pgmap v650: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 13431/212496 objects degraded (6.321%); 12934/212496 objects misplaced (6.087%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:16:18.507 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/978021344' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:16:18.507 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:18 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 13431/212496 objects degraded (6.321%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:18.608 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:18.644 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:18.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:18 smithi110 ceph-mon[221102]: pgmap v650: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 13431/212496 objects degraded (6.321%); 12934/212496 objects misplaced (6.087%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:16:18.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/978021344' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:16:18.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:18 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 13431/212496 objects degraded (6.321%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:18.715 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:18.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:18.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:18.821 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:18.850 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:19.186 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:19.196 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:16:18 smithi110 conmon[328908]: 2023-09-23T23:16:18.879+0000 7fa60359a640 -1 osd.2 193 log_to_monitors true 2023-09-23T23:16:19.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:19.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:19.298 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:19.361 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:19.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:19 smithi110 ceph-mon[212939]: from='osd.2 [v2:172.21.15.110:6818/618908095,v1:172.21.15.110:6819/618908095]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T23:16:19.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:19 smithi110 ceph-mon[221102]: from='osd.2 [v2:172.21.15.110:6818/618908095,v1:172.21.15.110:6819/618908095]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T23:16:19.701 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:16:19 smithi110 conmon[328908]: 2023-09-23T23:16:19.481+0000 7fa5f09ff700 -1 osd.2 193 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T23:16:19.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:19 smithi138 ceph-mon[193606]: from='osd.2 [v2:172.21.15.110:6818/618908095,v1:172.21.15.110:6819/618908095]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T23:16:20.416 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:16:20.453 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:20.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:20.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:20.477 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:16:20.477 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:16:20.566 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:20.595 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:20.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:20 smithi110 ceph-mon[212939]: pgmap v651: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 12944/206904 objects degraded (6.256%); 12934/206904 objects misplaced (6.251%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:16:20.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:20 smithi110 ceph-mon[212939]: failed to encode map e198 with expected crc 2023-09-23T23:16:20.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:20 smithi110 ceph-mon[212939]: from='osd.2 [v2:172.21.15.110:6818/618908095,v1:172.21.15.110:6819/618908095]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T23:16:20.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:20 smithi110 ceph-mon[212939]: osdmap e198: 8 total, 7 up, 8 in 2023-09-23T23:16:20.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:20 smithi110 ceph-mon[212939]: failed to encode map e198 with expected crc 2023-09-23T23:16:20.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:20 smithi110 ceph-mon[212939]: from='osd.2 [v2:172.21.15.110:6818/618908095,v1:172.21.15.110:6819/618908095]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:16:20.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:20 smithi110 ceph-mon[212939]: failed to encode map e198 with expected crc 2023-09-23T23:16:20.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:20 smithi110 ceph-mon[221102]: pgmap v651: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 12944/206904 objects degraded (6.256%); 12934/206904 objects misplaced (6.251%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:16:20.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:20 smithi110 ceph-mon[221102]: failed to encode map e198 with expected crc 2023-09-23T23:16:20.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:20 smithi110 ceph-mon[221102]: from='osd.2 [v2:172.21.15.110:6818/618908095,v1:172.21.15.110:6819/618908095]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T23:16:20.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:20 smithi110 ceph-mon[221102]: osdmap e198: 8 total, 7 up, 8 in 2023-09-23T23:16:20.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:20 smithi110 ceph-mon[221102]: failed to encode map e198 with expected crc 2023-09-23T23:16:20.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:20 smithi110 ceph-mon[221102]: from='osd.2 [v2:172.21.15.110:6818/618908095,v1:172.21.15.110:6819/618908095]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:16:20.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:20 smithi110 ceph-mon[221102]: failed to encode map e198 with expected crc 2023-09-23T23:16:20.712 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:20.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:20.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:20.817 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:20.853 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:20.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:20 smithi138 ceph-mon[193606]: pgmap v651: 99 pgs: 69 active+clean, 1 active+recovering+undersized+remapped, 22 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 12944/206904 objects degraded (6.256%); 12934/206904 objects misplaced (6.251%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:16:20.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:20 smithi138 ceph-mon[193606]: failed to encode map e198 with expected crc 2023-09-23T23:16:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:20 smithi138 ceph-mon[193606]: from='osd.2 [v2:172.21.15.110:6818/618908095,v1:172.21.15.110:6819/618908095]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T23:16:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:20 smithi138 ceph-mon[193606]: osdmap e198: 8 total, 7 up, 8 in 2023-09-23T23:16:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:20 smithi138 ceph-mon[193606]: failed to encode map e198 with expected crc 2023-09-23T23:16:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:20 smithi138 ceph-mon[193606]: from='osd.2 [v2:172.21.15.110:6818/618908095,v1:172.21.15.110:6819/618908095]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-09-23T23:16:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:20 smithi138 ceph-mon[193606]: failed to encode map e198 with expected crc 2023-09-23T23:16:21.002 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:21.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:21.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:21.106 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:21.134 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:21.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:21 smithi138 ceph-mon[193606]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:16:21.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:21 smithi138 ceph-mon[193606]: osd.2 [v2:172.21.15.110:6818/618908095,v1:172.21.15.110:6819/618908095] boot 2023-09-23T23:16:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:21 smithi138 ceph-mon[193606]: osdmap e199: 8 total, 8 up, 8 in 2023-09-23T23:16:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:21 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T23:16:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:21 smithi138 ceph-mon[193606]: osdmap e200: 8 total, 8 up, 8 in 2023-09-23T23:16:21.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:21 smithi110 ceph-mon[221102]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:16:21.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:21 smithi110 ceph-mon[221102]: osd.2 [v2:172.21.15.110:6818/618908095,v1:172.21.15.110:6819/618908095] boot 2023-09-23T23:16:21.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:21 smithi110 ceph-mon[221102]: osdmap e199: 8 total, 8 up, 8 in 2023-09-23T23:16:21.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:21 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T23:16:21.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:21 smithi110 ceph-mon[221102]: osdmap e200: 8 total, 8 up, 8 in 2023-09-23T23:16:21.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:21 smithi110 ceph-mon[212939]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:16:21.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:21 smithi110 ceph-mon[212939]: osd.2 [v2:172.21.15.110:6818/618908095,v1:172.21.15.110:6819/618908095] boot 2023-09-23T23:16:21.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:21 smithi110 ceph-mon[212939]: osdmap e199: 8 total, 8 up, 8 in 2023-09-23T23:16:21.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:21 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T23:16:21.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:21 smithi110 ceph-mon[212939]: osdmap e200: 8 total, 8 up, 8 in 2023-09-23T23:16:22.478 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:16:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:22 smithi138 ceph-mon[193606]: pgmap v654: 99 pgs: 3 peering, 69 active+clean, 1 active+recovering+undersized+remapped, 21 active+undersized, 1 active+remapped+backfill_wait, 4 active+undersized+degraded; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 12941/205176 objects degraded (6.307%); 12500/205176 objects misplaced (6.092%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:16:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:22 smithi138 ceph-mon[193606]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:22 smithi138 ceph-mon[193606]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:22 smithi138 ceph-mon[193606]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:22 smithi138 ceph-mon[193606]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:22 smithi138 ceph-mon[193606]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:22 smithi138 ceph-mon[193606]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:22 smithi138 ceph-mon[193606]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:22 smithi138 ceph-mon[193606]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:22 smithi138 ceph-mon[193606]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:22 smithi138 ceph-mon[193606]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:22 smithi138 ceph-mon[193606]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:22 smithi138 ceph-mon[193606]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:22 smithi138 ceph-mon[193606]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:22 smithi138 ceph-mon[193606]: osdmap e201: 8 total, 8 up, 8 in 2023-09-23T23:16:22.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[221102]: pgmap v654: 99 pgs: 3 peering, 69 active+clean, 1 active+recovering+undersized+remapped, 21 active+undersized, 1 active+remapped+backfill_wait, 4 active+undersized+degraded; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 12941/205176 objects degraded (6.307%); 12500/205176 objects misplaced (6.092%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:16:22.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[221102]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[221102]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[221102]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[221102]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[221102]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[221102]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[221102]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[221102]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[221102]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[221102]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[221102]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[221102]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[221102]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[221102]: osdmap e201: 8 total, 8 up, 8 in 2023-09-23T23:16:22.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[212939]: pgmap v654: 99 pgs: 3 peering, 69 active+clean, 1 active+recovering+undersized+remapped, 21 active+undersized, 1 active+remapped+backfill_wait, 4 active+undersized+degraded; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 12941/205176 objects degraded (6.307%); 12500/205176 objects misplaced (6.092%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:16:22.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[212939]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[212939]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[212939]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[212939]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[212939]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[212939]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[212939]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[212939]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[212939]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[212939]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[212939]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[212939]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[212939]: failed to encode map e200 with expected crc 2023-09-23T23:16:22.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:22 smithi110 ceph-mon[212939]: osdmap e201: 8 total, 8 up, 8 in 2023-09-23T23:16:23.690 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:23.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:23.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:23.797 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:23.867 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:23 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:16:23.868 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:23 smithi110 ceph-mon[221102]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.868 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:23 smithi110 ceph-mon[221102]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.868 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:23 smithi110 ceph-mon[221102]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.868 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:23 smithi110 ceph-mon[221102]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.868 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:23 smithi110 ceph-mon[221102]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.868 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:23 smithi110 ceph-mon[221102]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.868 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:23 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 12941/205176 objects degraded (6.307%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:23.870 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:23.870 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:23 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:16:23.870 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:23 smithi110 ceph-mon[212939]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:23 smithi110 ceph-mon[212939]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:23 smithi110 ceph-mon[212939]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:23 smithi110 ceph-mon[212939]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:23 smithi110 ceph-mon[212939]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:23 smithi110 ceph-mon[212939]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:23 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 12941/205176 objects degraded (6.307%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:23 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:16:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:23 smithi138 ceph-mon[193606]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:23 smithi138 ceph-mon[193606]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:23 smithi138 ceph-mon[193606]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:23 smithi138 ceph-mon[193606]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:23 smithi138 ceph-mon[193606]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:23 smithi138 ceph-mon[193606]: failed to encode map e201 with expected crc 2023-09-23T23:16:23.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:23 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 12941/205176 objects degraded (6.307%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:24.010 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:24.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:24.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:24.128 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:24.260 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:24.545 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:24 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:16:24.545 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:24 smithi110 ceph-mon[212939]: pgmap v657: 99 pgs: 1 activating+undersized+degraded+remapped, 3 peering, 71 active+clean, 1 active+recovering+undersized+remapped, 19 active+undersized, 1 active+remapped+backfill_wait, 3 active+undersized+degraded; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 203 op/s; 9497/204453 objects degraded (4.645%); 12500/204453 objects misplaced (6.114%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T23:16:24.637 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:24.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:24.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:24.668 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:24.682 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:16:24.682 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:16:24.682 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-09-23T23:16:24.682 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:16:24.719 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:24.748 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:24.781 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:24.832 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:24.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:24.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:24.839 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:24 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:16:24.839 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:24 smithi138 ceph-mon[193606]: pgmap v657: 99 pgs: 1 activating+undersized+degraded+remapped, 3 peering, 71 active+clean, 1 active+recovering+undersized+remapped, 19 active+undersized, 1 active+remapped+backfill_wait, 3 active+undersized+degraded; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 203 op/s; 9497/204453 objects degraded (4.645%); 12500/204453 objects misplaced (6.114%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T23:16:24.940 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:24.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:24 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:16:24.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:24 smithi110 ceph-mon[221102]: pgmap v657: 99 pgs: 1 activating+undersized+degraded+remapped, 3 peering, 71 active+clean, 1 active+recovering+undersized+remapped, 19 active+undersized, 1 active+remapped+backfill_wait, 3 active+undersized+degraded; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 203 op/s; 9497/204453 objects degraded (4.645%); 12500/204453 objects misplaced (6.114%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T23:16:25.024 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:25.270 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:25.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:25.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:25.385 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:25.497 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:25.561 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:25 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4030170682' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:16:25.576 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:16:25.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:25 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4030170682' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:16:25.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:25 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:16:25.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:25 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:16:25.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:25 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4030170682' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:16:25.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:25 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:16:26.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:26 smithi138 ceph-mon[193606]: pgmap v658: 99 pgs: 1 activating+undersized+degraded+remapped, 3 peering, 91 active+clean, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 1849/201807 objects degraded (0.916%); 12500/201807 objects misplaced (6.194%); 1.6 MiB/s, 33 objects/s recovering 2023-09-23T23:16:26.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:26 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:16:26.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:26 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:16:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:26 smithi138 ceph-mon[193606]: osdmap e202: 8 total, 8 up, 8 in 2023-09-23T23:16:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:26 smithi138 ceph-mon[193606]: failed to encode map e202 with expected crc 2023-09-23T23:16:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:26 smithi138 ceph-mon[193606]: failed to encode map e202 with expected crc 2023-09-23T23:16:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:26 smithi138 ceph-mon[193606]: failed to encode map e202 with expected crc 2023-09-23T23:16:26.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:16:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:16:26] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.43.0" 2023-09-23T23:16:26.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:26 smithi110 ceph-mon[212939]: pgmap v658: 99 pgs: 1 activating+undersized+degraded+remapped, 3 peering, 91 active+clean, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 1849/201807 objects degraded (0.916%); 12500/201807 objects misplaced (6.194%); 1.6 MiB/s, 33 objects/s recovering 2023-09-23T23:16:26.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:26 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:16:26.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:26 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:16:26.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:26 smithi110 ceph-mon[212939]: osdmap e202: 8 total, 8 up, 8 in 2023-09-23T23:16:26.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:26 smithi110 ceph-mon[212939]: failed to encode map e202 with expected crc 2023-09-23T23:16:26.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:26 smithi110 ceph-mon[212939]: failed to encode map e202 with expected crc 2023-09-23T23:16:26.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:26 smithi110 ceph-mon[212939]: failed to encode map e202 with expected crc 2023-09-23T23:16:26.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:26 smithi110 ceph-mon[221102]: pgmap v658: 99 pgs: 1 activating+undersized+degraded+remapped, 3 peering, 91 active+clean, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 1849/201807 objects degraded (0.916%); 12500/201807 objects misplaced (6.194%); 1.6 MiB/s, 33 objects/s recovering 2023-09-23T23:16:26.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:26 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:16:26.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:26 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:16:26.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:26 smithi110 ceph-mon[221102]: osdmap e202: 8 total, 8 up, 8 in 2023-09-23T23:16:26.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:26 smithi110 ceph-mon[221102]: failed to encode map e202 with expected crc 2023-09-23T23:16:26.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:26 smithi110 ceph-mon[221102]: failed to encode map e202 with expected crc 2023-09-23T23:16:26.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:26 smithi110 ceph-mon[221102]: failed to encode map e202 with expected crc 2023-09-23T23:16:28.427 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:28.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:28.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:28.543 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:16:28.586 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:28.657 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:16:28.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:28.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:16:28.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:28 smithi110 ceph-mon[212939]: pgmap v660: 99 pgs: 1 activating+undersized+degraded+remapped, 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 1849/200604 objects degraded (0.922%); 12106/200604 objects misplaced (6.035%); 1.5 MiB/s, 31 objects/s recovering 2023-09-23T23:16:28.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:28.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:16:28.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:28 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1849/200604 objects degraded (0.922%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:28.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:28 smithi110 ceph-mon[221102]: pgmap v660: 99 pgs: 1 activating+undersized+degraded+remapped, 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 1849/200604 objects degraded (0.922%); 12106/200604 objects misplaced (6.035%); 1.5 MiB/s, 31 objects/s recovering 2023-09-23T23:16:28.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:28.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:16:28.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:28 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1849/200604 objects degraded (0.922%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:28.766 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:28.927 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:28 smithi138 ceph-mon[193606]: pgmap v660: 99 pgs: 1 activating+undersized+degraded+remapped, 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 1849/200604 objects degraded (0.922%); 12106/200604 objects misplaced (6.035%); 1.5 MiB/s, 31 objects/s recovering 2023-09-23T23:16:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:28.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:16:28.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:28 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1849/200604 objects degraded (0.922%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:16:29.307 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:29.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:29.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:29.412 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:29.443 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:29.495 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:29.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:29.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:29.603 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:16:29.630 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:29.680 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:16:29.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:29.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:16:29.790 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:29.825 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:29.883 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:29.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:29.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:29.993 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:16:30.082 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:30.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:30 smithi138 ceph-mon[193606]: pgmap v661: 99 pgs: 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 461 op/s; 1849/194244 objects degraded (0.952%); 12106/194244 objects misplaced (6.232%); 1.1 MiB/s, 67 objects/s recovering 2023-09-23T23:16:30.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:30 smithi110 ceph-mon[212939]: pgmap v661: 99 pgs: 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 461 op/s; 1849/194244 objects degraded (0.952%); 12106/194244 objects misplaced (6.232%); 1.1 MiB/s, 67 objects/s recovering 2023-09-23T23:16:30.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:30 smithi110 ceph-mon[221102]: pgmap v661: 99 pgs: 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 461 op/s; 1849/194244 objects degraded (0.952%); 12106/194244 objects misplaced (6.232%); 1.1 MiB/s, 67 objects/s recovering 2023-09-23T23:16:31.133 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:16:31.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:31.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:16:31.246 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:31.282 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:31.329 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:31.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:31.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:31.383 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:16:31.436 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:16:31.465 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:31.526 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:16:31.527 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:16:31.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:31.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:16:31.566 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:16:31.624 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:16:31.631 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:16:31.662 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:16:31.714 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:16:31.762 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:16:31.792 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:16:31.816 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:31.857 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:16:31.857 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:16:31.857 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:16:31.857 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:16:31.857 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:16:31.857 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:16:31.858 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:16:31.858 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:16:31.858 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:16:32.041 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:16:32.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:32.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:16:32.073 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:31 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:32.073 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:31 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:32.073 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:31 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/882022164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:16:32.078 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:31 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:32.078 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:31 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:32.079 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:31 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/882022164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:16:32.166 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:32.170 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:16:32.170 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:16:32.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:16:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPush(9.5 202/187 [PushOp(9:a8a90ef2:::benchmark_data_smithi110_295904_object48765:head, version: 170'3028, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:a8a90ef2:::benchmark_data_smithi110_295904_object48765:head@170'3028, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:16:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:16:32.149914+0000", 2023-09-23T23:16:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0032267250000000002, 2023-09-23T23:16:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032817240000000002, 2023-09-23T23:16:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:16:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:16:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:16:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:16:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:16:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:32.149914+0000", 2023-09-23T23:16:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:16:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:16:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:16:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:16:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:32.149914+0000", 2023-09-23T23:16:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:16:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:16:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:16:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:16:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:32.149913+0000", 2023-09-23T23:16:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:16:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:16:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:16:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:16:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:32.149951+0000", 2023-09-23T23:16:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7818999999999998e-05 2023-09-23T23:16:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:16:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:16:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:16:32.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:32.149966+0000", 2023-09-23T23:16:32.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4688000000000001e-05 2023-09-23T23:16:32.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:16:32.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:16:32.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:16:32.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:32.149980+0000", 2023-09-23T23:16:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4440999999999999e-05 2023-09-23T23:16:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:16:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:16:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:16:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:32.150097+0000", 2023-09-23T23:16:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117459 2023-09-23T23:16:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:16:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:16:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:16:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:32.150112+0000", 2023-09-23T23:16:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4409999999999999e-05 2023-09-23T23:16:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:16:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:16:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:16:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:16:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-09-23T23:16:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 1 2023-09-23T23:16:32.177 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:16:32.196 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:32.249 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:32.255 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:31 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:32.255 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:31 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:32.255 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:31 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/882022164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:16:32.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:32.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:32.357 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:32.391 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:32.452 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:32.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:32.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:32.558 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:32.595 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:32.658 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:32.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:32.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:32.770 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:32.804 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:32.873 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:32.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:32.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:32.985 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:16:33.047 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:33.104 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:16:33.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:33.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:16:33.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:32 smithi138 ceph-mon[193606]: pgmap v662: 99 pgs: 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 1849/192141 objects degraded (0.962%); 11740/192141 objects misplaced (6.110%); 1.7 MiB/s, 70 objects/s recovering 2023-09-23T23:16:33.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:32 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:16:33.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:32 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:16:33.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:32 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:33.212 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:16:33.320 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:32 smithi110 ceph-mon[212939]: pgmap v662: 99 pgs: 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 1849/192141 objects degraded (0.962%); 11740/192141 objects misplaced (6.110%); 1.7 MiB/s, 70 objects/s recovering 2023-09-23T23:16:33.320 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:32 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:16:33.320 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:32 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:16:33.320 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:32 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:33.320 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:32 smithi110 ceph-mon[221102]: pgmap v662: 99 pgs: 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 1849/192141 objects degraded (0.962%); 11740/192141 objects misplaced (6.110%); 1.7 MiB/s, 70 objects/s recovering 2023-09-23T23:16:33.320 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:32 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:16:33.320 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:32 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:16:33.320 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:32 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:33.333 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:34.265 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:16:34.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:34.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:16:34.272 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:33 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1849/192141 objects degraded (0.962%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:34.274 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:33 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1849/192141 objects degraded (0.962%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:34.373 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:34.442 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T23:16:34.442 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:16:34.443 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T23:16:34.443 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T23:16:34.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:33 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1849/192141 objects degraded (0.962%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:34.461 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:34.546 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:16:34.647 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:34.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:34.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:34.759 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:34.766 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:34.935 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:35.142 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:35.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:35.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:35.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:34 smithi110 ceph-mon[212939]: pgmap v663: 99 pgs: 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 1849/190485 objects degraded (0.971%); 11740/190485 objects misplaced (6.163%); 1.5 MiB/s, 63 objects/s recovering 2023-09-23T23:16:35.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:34 smithi110 ceph-mon[221102]: pgmap v663: 99 pgs: 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 1849/190485 objects degraded (0.971%); 11740/190485 objects misplaced (6.163%); 1.5 MiB/s, 63 objects/s recovering 2023-09-23T23:16:35.257 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:35.284 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:34 smithi138 ceph-mon[193606]: pgmap v663: 99 pgs: 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 1849/190485 objects degraded (0.971%); 11740/190485 objects misplaced (6.163%); 1.5 MiB/s, 63 objects/s recovering 2023-09-23T23:16:35.289 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:35.354 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:35.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:35.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:35.462 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:35.547 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:35.652 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:35.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:35.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:35.747 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:35.757 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:16:35.784 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:35.826 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:16:35.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:35.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:16:35.931 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:35.960 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:36.007 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:36.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:36.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:36.111 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:36.137 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:36.179 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:36.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:36.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:36.286 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:36.328 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:36.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:36 smithi138 ceph-mon[193606]: 1.8 deep-scrub starts 2023-09-23T23:16:36.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:36 smithi138 ceph-mon[193606]: osdmap e203: 8 total, 8 up, 8 in 2023-09-23T23:16:36.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:36 smithi138 ceph-mon[193606]: failed to encode map e203 with expected crc 2023-09-23T23:16:36.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:36 smithi138 ceph-mon[193606]: failed to encode map e203 with expected crc 2023-09-23T23:16:36.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:36 smithi138 ceph-mon[193606]: failed to encode map e203 with expected crc 2023-09-23T23:16:36.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:36 smithi138 ceph-mon[193606]: pgmap v665: 99 pgs: 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 1849/186087 objects degraded (0.994%); 11740/186087 objects misplaced (6.309%); 1.6 MiB/s, 99 objects/s recovering 2023-09-23T23:16:36.442 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:36.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:36.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:36.522 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:16:36.554 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:36.582 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:36 smithi110 ceph-mon[212939]: 1.8 deep-scrub starts 2023-09-23T23:16:36.583 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:36 smithi110 ceph-mon[212939]: osdmap e203: 8 total, 8 up, 8 in 2023-09-23T23:16:36.583 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:36 smithi110 ceph-mon[212939]: failed to encode map e203 with expected crc 2023-09-23T23:16:36.583 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:36 smithi110 ceph-mon[212939]: failed to encode map e203 with expected crc 2023-09-23T23:16:36.583 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:36 smithi110 ceph-mon[212939]: failed to encode map e203 with expected crc 2023-09-23T23:16:36.583 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:36 smithi110 ceph-mon[212939]: pgmap v665: 99 pgs: 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 1849/186087 objects degraded (0.994%); 11740/186087 objects misplaced (6.309%); 1.6 MiB/s, 99 objects/s recovering 2023-09-23T23:16:36.585 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:36 smithi110 ceph-mon[221102]: 1.8 deep-scrub starts 2023-09-23T23:16:36.586 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:36 smithi110 ceph-mon[221102]: osdmap e203: 8 total, 8 up, 8 in 2023-09-23T23:16:36.586 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:36 smithi110 ceph-mon[221102]: failed to encode map e203 with expected crc 2023-09-23T23:16:36.586 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:36 smithi110 ceph-mon[221102]: failed to encode map e203 with expected crc 2023-09-23T23:16:36.586 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:36 smithi110 ceph-mon[221102]: failed to encode map e203 with expected crc 2023-09-23T23:16:36.586 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:36 smithi110 ceph-mon[221102]: pgmap v665: 99 pgs: 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 1849/186087 objects degraded (0.994%); 11740/186087 objects misplaced (6.309%); 1.6 MiB/s, 99 objects/s recovering 2023-09-23T23:16:36.620 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:36.682 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:36.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:36.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:36.795 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:16:36.860 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:16:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:16:36] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.43.0" 2023-09-23T23:16:36.875 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:37.112 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:16:37.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:37.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:16:37.225 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:37.262 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:37.312 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:37.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:37.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:37.331 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:37.420 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:37.447 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:37.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:37 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:16:37.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:37 smithi110 ceph-mon[212939]: failed to encode map e204 with expected crc 2023-09-23T23:16:37.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:37 smithi110 ceph-mon[212939]: osdmap e204: 8 total, 8 up, 8 in 2023-09-23T23:16:37.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:37 smithi110 ceph-mon[212939]: failed to encode map e204 with expected crc 2023-09-23T23:16:37.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:37 smithi110 ceph-mon[212939]: failed to encode map e204 with expected crc 2023-09-23T23:16:37.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:37 smithi110 ceph-mon[212939]: failed to encode map e204 with expected crc 2023-09-23T23:16:37.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:37 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:16:37.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:37 smithi110 ceph-mon[221102]: failed to encode map e204 with expected crc 2023-09-23T23:16:37.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:37 smithi110 ceph-mon[221102]: osdmap e204: 8 total, 8 up, 8 in 2023-09-23T23:16:37.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:37 smithi110 ceph-mon[221102]: failed to encode map e204 with expected crc 2023-09-23T23:16:37.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:37 smithi110 ceph-mon[221102]: failed to encode map e204 with expected crc 2023-09-23T23:16:37.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:37 smithi110 ceph-mon[221102]: failed to encode map e204 with expected crc 2023-09-23T23:16:37.512 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:37.521 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:37 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:16:37.521 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:37 smithi138 ceph-mon[193606]: failed to encode map e204 with expected crc 2023-09-23T23:16:37.522 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:37 smithi138 ceph-mon[193606]: osdmap e204: 8 total, 8 up, 8 in 2023-09-23T23:16:37.522 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:37 smithi138 ceph-mon[193606]: failed to encode map e204 with expected crc 2023-09-23T23:16:37.522 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:37 smithi138 ceph-mon[193606]: failed to encode map e204 with expected crc 2023-09-23T23:16:37.522 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:37 smithi138 ceph-mon[193606]: failed to encode map e204 with expected crc 2023-09-23T23:16:37.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:37.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:37.624 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:37.651 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:37.706 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:37.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:37.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:37.817 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:37.853 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:37.922 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:37.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:37.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:38.031 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:38.059 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:38.127 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:38.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:38.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:38.233 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:16:38.269 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:38.325 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:16:38.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:38.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:16:38.437 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:38.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:38 smithi138 ceph-mon[193606]: pgmap v667: 99 pgs: 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 1849/184227 objects degraded (1.004%); 11266/184227 objects misplaced (6.115%); 2.1 MiB/s, 82 objects/s recovering 2023-09-23T23:16:38.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:38 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1849/186087 objects degraded (0.994%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:38.470 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:38.540 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:38.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:38.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:38.645 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:38.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:38 smithi110 ceph-mon[212939]: pgmap v667: 99 pgs: 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 1849/184227 objects degraded (1.004%); 11266/184227 objects misplaced (6.115%); 2.1 MiB/s, 82 objects/s recovering 2023-09-23T23:16:38.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:38 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1849/186087 objects degraded (0.994%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:38.711 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:38 smithi110 ceph-mon[221102]: pgmap v667: 99 pgs: 94 active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 1849/184227 objects degraded (1.004%); 11266/184227 objects misplaced (6.115%); 2.1 MiB/s, 82 objects/s recovering 2023-09-23T23:16:38.711 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:38 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1849/186087 objects degraded (0.994%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:38.714 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:39.192 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:39.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:39.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:39.270 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:39.304 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:39 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:16:39.305 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:39 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:16:39.312 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:39.341 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:39.400 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:39.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:39.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:39.511 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:39.597 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:39.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:39 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:16:40.302 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:16:40.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:40 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:16:40.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:40 smithi138 ceph-mon[193606]: pgmap v668: 99 pgs: 95 active+clean, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 1849/178239 objects degraded (1.037%); 11266/178239 objects misplaced (6.321%); 1.3 MiB/s, 80 objects/s recovering 2023-09-23T23:16:40.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:40 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1611904111' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:16:40.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:40 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:16:40.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:40 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:16:40.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:40 smithi110 ceph-mon[221102]: pgmap v668: 99 pgs: 95 active+clean, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 1849/178239 objects degraded (1.037%); 11266/178239 objects misplaced (6.321%); 1.3 MiB/s, 80 objects/s recovering 2023-09-23T23:16:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:40 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1611904111' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:16:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:40 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:16:40.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:40 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:16:40.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:40 smithi110 ceph-mon[212939]: pgmap v668: 99 pgs: 95 active+clean, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 1849/178239 objects degraded (1.037%); 11266/178239 objects misplaced (6.321%); 1.3 MiB/s, 80 objects/s recovering 2023-09-23T23:16:40.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:40 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1611904111' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:16:40.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:40 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:16:40.951 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:40.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:40.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:41.065 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:41.105 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:41.174 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:41.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:41.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:41.280 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:41.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:41 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:16:41.307 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:41 smithi110 ceph-mon[212939]: failed to encode map e205 with expected crc 2023-09-23T23:16:41.358 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:41.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:41 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:16:41.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:41 smithi138 ceph-mon[193606]: failed to encode map e205 with expected crc 2023-09-23T23:16:41.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:41 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:16:41.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:41 smithi138 ceph-mon[193606]: osdmap e205: 8 total, 8 up, 8 in 2023-09-23T23:16:41.657 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:41 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:16:41.657 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:41 smithi110 ceph-mon[212939]: osdmap e205: 8 total, 8 up, 8 in 2023-09-23T23:16:41.657 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:41 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:16:41.657 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:41 smithi110 ceph-mon[221102]: failed to encode map e205 with expected crc 2023-09-23T23:16:41.657 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:41 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:16:41.657 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:41 smithi110 ceph-mon[221102]: osdmap e205: 8 total, 8 up, 8 in 2023-09-23T23:16:42.132 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:42.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:42.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:42.251 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:42.348 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:42 smithi110 ceph-mon[212939]: pgmap v670: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 94 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 1849/176235 objects degraded (1.049%); 10747/176235 objects misplaced (6.098%); 3.4 MiB/s, 68 objects/s recovering 2023-09-23T23:16:42.356 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:42 smithi110 ceph-mon[221102]: pgmap v670: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 94 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 1849/176235 objects degraded (1.049%); 10747/176235 objects misplaced (6.098%); 3.4 MiB/s, 68 objects/s recovering 2023-09-23T23:16:42.356 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:42.577 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:42.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:42.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:42.685 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:42.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:42 smithi138 ceph-mon[193606]: pgmap v670: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 94 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 1849/176235 objects degraded (1.049%); 10747/176235 objects misplaced (6.098%); 3.4 MiB/s, 68 objects/s recovering 2023-09-23T23:16:42.726 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:42.778 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:42.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:42.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:42.885 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:42.956 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:43.012 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:16:43.012 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:16:43.012 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:16:43.012 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:16:43.012 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:16:43.064 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:43.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:43.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:43.177 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:43.247 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:43.290 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:43.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:43.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:43.335 INFO:teuthology.orchestra.run.smithi110.stdout:moved 8 / 297 (2.6936%) 2023-09-23T23:16:43.335 INFO:teuthology.orchestra.run.smithi110.stdout:avg 37.125 2023-09-23T23:16:43.335 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 7.68826 -> 6.54671 (expected baseline 5.69951) 2023-09-23T23:16:43.335 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.5 with 23 -> 24 pgs (0.619529 -> 0.646465 * mean) 2023-09-23T23:16:43.335 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.3 with 49 -> 47 pgs (1.31987 -> 1.26599 * mean) 2023-09-23T23:16:43.335 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:16:43.335 INFO:teuthology.orchestra.run.smithi110.stdout:oload 110 2023-09-23T23:16:43.335 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 0.05 2023-09-23T23:16:43.335 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:16:43.335 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0197 2023-09-23T23:16:43.335 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.0216 2023-09-23T23:16:43.335 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 weight 0.9500 -> 0.9000 2023-09-23T23:16:43.336 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 weight 0.9500 -> 0.9000 2023-09-23T23:16:43.336 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 weight 0.9500 -> 0.9000 2023-09-23T23:16:43.336 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 weight 0.7027 -> 0.6527 2023-09-23T23:16:43.393 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:16:43.442 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:43.511 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:16:43.516 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:43 smithi110 ceph-mon[212939]: from='client.55300 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:16:43.516 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'58984','3':'58984','5':'42776','7':'58984'}"}]: dispatch 2023-09-23T23:16:43.517 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'58984','3':'58984','5':'42776','7':'58984'}"}]: dispatch 2023-09-23T23:16:43.517 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:43.517 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:16:43.517 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:43 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1849/176235 objects degraded (1.049%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:43.517 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:43 smithi110 ceph-mon[221102]: from='client.55300 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:16:43.517 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'58984','3':'58984','5':'42776','7':'58984'}"}]: dispatch 2023-09-23T23:16:43.517 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'58984','3':'58984','5':'42776','7':'58984'}"}]: dispatch 2023-09-23T23:16:43.518 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:43.518 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:16:43.518 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:43 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1849/176235 objects degraded (1.049%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:43.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:43.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:16:43.619 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:43.642 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:43 smithi138 ceph-mon[193606]: from='client.55300 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:16:43.643 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'58984','3':'58984','5':'42776','7':'58984'}"}]: dispatch 2023-09-23T23:16:43.643 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'58984','3':'58984','5':'42776','7':'58984'}"}]: dispatch 2023-09-23T23:16:43.643 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:43.643 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:16:43.643 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:43 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1849/176235 objects degraded (1.049%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:43.645 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:43.687 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:43.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:43.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:43.796 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:43.822 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:43.863 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:43.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:43.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:43.970 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:43.995 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:44.037 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:44.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:44.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:44.146 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:44.171 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:44.212 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:44.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:44.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:44.229 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:16:44.318 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:44.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:44 smithi138 ceph-mon[193606]: failed to encode map e206 with expected crc 2023-09-23T23:16:44.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:44 smithi138 ceph-mon[193606]: failed to encode map e206 with expected crc 2023-09-23T23:16:44.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:44 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'58984','3':'58984','5':'42776','7':'58984'}"}]': finished 2023-09-23T23:16:44.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:44 smithi138 ceph-mon[193606]: osdmap e206: 8 total, 8 up, 8 in 2023-09-23T23:16:44.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:44 smithi138 ceph-mon[193606]: pgmap v672: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 94 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 1849/174732 objects degraded (1.058%); 10747/174732 objects misplaced (6.151%); 1.5 MiB/s, 36 objects/s recovering 2023-09-23T23:16:44.348 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:44.389 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:44.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:44.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:44.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:44 smithi110 ceph-mon[221102]: failed to encode map e206 with expected crc 2023-09-23T23:16:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:44 smithi110 ceph-mon[221102]: failed to encode map e206 with expected crc 2023-09-23T23:16:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:44 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'58984','3':'58984','5':'42776','7':'58984'}"}]': finished 2023-09-23T23:16:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:44 smithi110 ceph-mon[221102]: osdmap e206: 8 total, 8 up, 8 in 2023-09-23T23:16:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:44 smithi110 ceph-mon[221102]: pgmap v672: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 94 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 1849/174732 objects degraded (1.058%); 10747/174732 objects misplaced (6.151%); 1.5 MiB/s, 36 objects/s recovering 2023-09-23T23:16:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:44 smithi110 ceph-mon[212939]: failed to encode map e206 with expected crc 2023-09-23T23:16:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:44 smithi110 ceph-mon[212939]: failed to encode map e206 with expected crc 2023-09-23T23:16:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:44 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'58984','3':'58984','5':'42776','7':'58984'}"}]': finished 2023-09-23T23:16:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:44 smithi110 ceph-mon[212939]: osdmap e206: 8 total, 8 up, 8 in 2023-09-23T23:16:44.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:44 smithi110 ceph-mon[212939]: pgmap v672: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 94 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 1849/174732 objects degraded (1.058%); 10747/174732 objects misplaced (6.151%); 1.5 MiB/s, 36 objects/s recovering 2023-09-23T23:16:44.493 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:16:44.642 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:44.921 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:16:44.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:44.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:16:45.028 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:45.057 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:45.102 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:45.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:45.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:45.207 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:45.234 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:45.276 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:45.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:45.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:45.381 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:45.411 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:45.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:45 smithi138 ceph-mon[193606]: osdmap e207: 8 total, 8 up, 8 in 2023-09-23T23:16:45.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:45 smithi138 ceph-mon[193606]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:45 smithi138 ceph-mon[193606]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:45 smithi138 ceph-mon[193606]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:45 smithi138 ceph-mon[193606]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:45 smithi138 ceph-mon[193606]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:45 smithi138 ceph-mon[193606]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:45 smithi138 ceph-mon[193606]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:45 smithi110 ceph-mon[212939]: osdmap e207: 8 total, 8 up, 8 in 2023-09-23T23:16:45.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:45 smithi110 ceph-mon[212939]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:45 smithi110 ceph-mon[212939]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:45 smithi110 ceph-mon[212939]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:45 smithi110 ceph-mon[212939]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:45 smithi110 ceph-mon[212939]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:45 smithi110 ceph-mon[212939]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:45 smithi110 ceph-mon[212939]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:45 smithi110 ceph-mon[221102]: osdmap e207: 8 total, 8 up, 8 in 2023-09-23T23:16:45.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:45 smithi110 ceph-mon[221102]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:45 smithi110 ceph-mon[221102]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:45 smithi110 ceph-mon[221102]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:45 smithi110 ceph-mon[221102]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:45 smithi110 ceph-mon[221102]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:45 smithi110 ceph-mon[221102]: failed to encode map e207 with expected crc 2023-09-23T23:16:45.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:45 smithi110 ceph-mon[221102]: failed to encode map e207 with expected crc 2023-09-23T23:16:46.103 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:46.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:46.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:46.148 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T23:16:46.148 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-09-23T23:16:46.210 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:16:46.218 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:46.379 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:46.588 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:46 smithi110 ceph-mon[212939]: failed to encode map e208 with expected crc 2023-09-23T23:16:46.588 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:46 smithi110 ceph-mon[212939]: failed to encode map e208 with expected crc 2023-09-23T23:16:46.588 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:46 smithi110 ceph-mon[212939]: osdmap e208: 8 total, 8 up, 8 in 2023-09-23T23:16:46.588 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:46 smithi110 ceph-mon[212939]: failed to encode map e208 with expected crc 2023-09-23T23:16:46.588 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:46 smithi110 ceph-mon[212939]: failed to encode map e208 with expected crc 2023-09-23T23:16:46.588 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:46 smithi110 ceph-mon[212939]: 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 205'10556 2023-09-23T23:16:46.588 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:46 smithi110 ceph-mon[212939]: 9.7 starting backfill to osd.2 from (0'0,0'0] MAX to 205'10197 2023-09-23T23:16:46.588 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:46 smithi110 ceph-mon[212939]: pgmap v675: 99 pgs: 4 peering, 1 active+recovering+undersized+remapped, 91 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 1849/170709 objects degraded (1.083%); 10747/170709 objects misplaced (6.296%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T23:16:46.588 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:46 smithi110 ceph-mon[221102]: failed to encode map e208 with expected crc 2023-09-23T23:16:46.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:46 smithi110 ceph-mon[221102]: failed to encode map e208 with expected crc 2023-09-23T23:16:46.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:46 smithi110 ceph-mon[221102]: osdmap e208: 8 total, 8 up, 8 in 2023-09-23T23:16:46.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:46 smithi110 ceph-mon[221102]: failed to encode map e208 with expected crc 2023-09-23T23:16:46.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:46 smithi110 ceph-mon[221102]: failed to encode map e208 with expected crc 2023-09-23T23:16:46.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:46 smithi110 ceph-mon[221102]: 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 205'10556 2023-09-23T23:16:46.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:46 smithi110 ceph-mon[221102]: 9.7 starting backfill to osd.2 from (0'0,0'0] MAX to 205'10197 2023-09-23T23:16:46.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:46 smithi110 ceph-mon[221102]: pgmap v675: 99 pgs: 4 peering, 1 active+recovering+undersized+remapped, 91 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 1849/170709 objects degraded (1.083%); 10747/170709 objects misplaced (6.296%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T23:16:46.672 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:46.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:46 smithi138 ceph-mon[193606]: failed to encode map e208 with expected crc 2023-09-23T23:16:46.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:46 smithi138 ceph-mon[193606]: failed to encode map e208 with expected crc 2023-09-23T23:16:46.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:46 smithi138 ceph-mon[193606]: osdmap e208: 8 total, 8 up, 8 in 2023-09-23T23:16:46.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:46 smithi138 ceph-mon[193606]: failed to encode map e208 with expected crc 2023-09-23T23:16:46.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:46 smithi138 ceph-mon[193606]: failed to encode map e208 with expected crc 2023-09-23T23:16:46.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:46 smithi138 ceph-mon[193606]: 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 205'10556 2023-09-23T23:16:46.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:46 smithi138 ceph-mon[193606]: 9.7 starting backfill to osd.2 from (0'0,0'0] MAX to 205'10197 2023-09-23T23:16:46.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:46 smithi138 ceph-mon[193606]: pgmap v675: 99 pgs: 4 peering, 1 active+recovering+undersized+remapped, 91 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 1849/170709 objects degraded (1.083%); 10747/170709 objects misplaced (6.296%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T23:16:46.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:46.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:46.746 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:46.795 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:46.822 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:46.877 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:46.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:46.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:46.916 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:16:46.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:16:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:16:46] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.43.0" 2023-09-23T23:16:46.964 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:16:46.991 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:46.997 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:16:47.020 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:47.051 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:16:47.084 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:47.088 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:16:47.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:47.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:47.145 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:16:47.191 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:47.199 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:16:47.228 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:47.235 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:16:47.247 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:16:47.247 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:16:47.248 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:16:47.248 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:16:47.248 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:16:47.248 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:16:47.248 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:16:47.248 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:16:47.295 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:47.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:47.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:47.373 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:47 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:16:47.373 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:47 smithi138 ceph-mon[193606]: osdmap e209: 8 total, 8 up, 8 in 2023-09-23T23:16:47.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:47 smithi138 ceph-mon[193606]: failed to encode map e209 with expected crc 2023-09-23T23:16:47.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:47 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/941404021' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:16:47.391 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:47.405 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:47.433 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:47.495 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:47.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:47.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:47.603 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:47.625 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:47 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:16:47.625 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:47 smithi110 ceph-mon[212939]: osdmap e209: 8 total, 8 up, 8 in 2023-09-23T23:16:47.625 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:47 smithi110 ceph-mon[212939]: failed to encode map e209 with expected crc 2023-09-23T23:16:47.625 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:47 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/941404021' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:16:47.625 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:47 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:16:47.625 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:47 smithi110 ceph-mon[221102]: osdmap e209: 8 total, 8 up, 8 in 2023-09-23T23:16:47.625 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:47 smithi110 ceph-mon[221102]: failed to encode map e209 with expected crc 2023-09-23T23:16:47.625 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:47 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/941404021' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:16:47.662 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:47.727 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:47.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:47.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:47.834 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:47.874 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:47.941 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:47.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:47.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:48.050 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:16:48.148 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:48.500 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:48 smithi110 ceph-mon[221102]: osdmap e210: 8 total, 8 up, 8 in 2023-09-23T23:16:48.501 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:48 smithi110 ceph-mon[221102]: pgmap v677: 99 pgs: 1 active+recovering+undersized+remapped, 4 peering, 91 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 1849/169452 objects degraded (1.091%); 10261/169452 objects misplaced (6.055%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T23:16:48.501 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:48 smithi110 ceph-mon[221102]: failed to encode map e210 with expected crc 2023-09-23T23:16:48.501 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:48 smithi110 ceph-mon[221102]: failed to encode map e210 with expected crc 2023-09-23T23:16:48.501 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:48 smithi110 ceph-mon[221102]: failed to encode map e210 with expected crc 2023-09-23T23:16:48.501 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:48 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1849/170709 objects degraded (1.083%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:48.501 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:48 smithi110 ceph-mon[212939]: osdmap e210: 8 total, 8 up, 8 in 2023-09-23T23:16:48.502 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:48 smithi110 ceph-mon[212939]: pgmap v677: 99 pgs: 1 active+recovering+undersized+remapped, 4 peering, 91 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 1849/169452 objects degraded (1.091%); 10261/169452 objects misplaced (6.055%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T23:16:48.502 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:48 smithi110 ceph-mon[212939]: failed to encode map e210 with expected crc 2023-09-23T23:16:48.502 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:48 smithi110 ceph-mon[212939]: failed to encode map e210 with expected crc 2023-09-23T23:16:48.502 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:48 smithi110 ceph-mon[212939]: failed to encode map e210 with expected crc 2023-09-23T23:16:48.502 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:48 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1849/170709 objects degraded (1.083%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:48.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:48 smithi138 ceph-mon[193606]: osdmap e210: 8 total, 8 up, 8 in 2023-09-23T23:16:48.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:48 smithi138 ceph-mon[193606]: pgmap v677: 99 pgs: 1 active+recovering+undersized+remapped, 4 peering, 91 active+clean, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 1849/169452 objects degraded (1.091%); 10261/169452 objects misplaced (6.055%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T23:16:48.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:48 smithi138 ceph-mon[193606]: failed to encode map e210 with expected crc 2023-09-23T23:16:48.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:48 smithi138 ceph-mon[193606]: failed to encode map e210 with expected crc 2023-09-23T23:16:48.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:48 smithi138 ceph-mon[193606]: failed to encode map e210 with expected crc 2023-09-23T23:16:48.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:48 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1849/170709 objects degraded (1.083%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:49.270 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:16:49.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:49.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:16:49.378 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:49.413 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:49.491 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:49.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:49.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:49.597 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:49.674 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:49.691 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:49.818 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:49.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:49.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:49.924 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:49.958 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:50.025 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:50.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:50.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:50.132 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:16:50.238 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:50.484 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:16:50.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:50.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:16:50.599 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:50.631 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:50.689 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:50.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:50.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:50.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:50 smithi110 ceph-mon[221102]: pgmap v679: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 1849/165378 objects degraded (1.118%); 17189/165378 objects misplaced (10.394%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T23:16:50.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:50 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T23:16:50.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:50 smithi110 ceph-mon[212939]: pgmap v679: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 1849/165378 objects degraded (1.118%); 17189/165378 objects misplaced (10.394%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T23:16:50.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:50 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T23:16:50.801 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:50.869 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:50.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:50 smithi138 ceph-mon[193606]: pgmap v679: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 1849/165378 objects degraded (1.118%); 17189/165378 objects misplaced (10.394%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T23:16:50.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:50 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T23:16:51.647 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:16:52.655 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:52.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:52.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:52.694 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:16:52.732 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:52 smithi110 ceph-mon[221102]: pgmap v680: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 1849/163269 objects degraded (1.132%); 16808/163269 objects misplaced (10.295%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:16:52.732 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:52 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1685969709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:16:52.732 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:52 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:16:52.733 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:52 smithi110 ceph-mon[212939]: pgmap v680: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 1849/163269 objects degraded (1.132%); 16808/163269 objects misplaced (10.295%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:16:52.733 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:52 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1685969709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:16:52.733 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:52 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:16:52.770 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:52.803 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:52 smithi138 ceph-mon[193606]: pgmap v680: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 1849/163269 objects degraded (1.132%); 16808/163269 objects misplaced (10.295%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:16:52.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:52 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1685969709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:16:52.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:52 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:16:52.808 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:52.867 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:52.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:52.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:52.973 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:53.007 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:53.072 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:53.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:53.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:53.179 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:53.270 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:53.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:53 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:16:53.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:53 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:16:53.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:53 smithi110 ceph-mon[221102]: osdmap e211: 8 total, 8 up, 8 in 2023-09-23T23:16:53.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:53 smithi110 ceph-mon[221102]: failed to encode map e211 with expected crc 2023-09-23T23:16:53.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:53 smithi110 ceph-mon[221102]: failed to encode map e211 with expected crc 2023-09-23T23:16:53.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:53 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1849/163269 objects degraded (1.132%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:53.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:53 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:16:53.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:53 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:16:53.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:53 smithi110 ceph-mon[212939]: osdmap e211: 8 total, 8 up, 8 in 2023-09-23T23:16:53.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:53 smithi110 ceph-mon[212939]: failed to encode map e211 with expected crc 2023-09-23T23:16:53.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:53 smithi110 ceph-mon[212939]: failed to encode map e211 with expected crc 2023-09-23T23:16:53.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:53 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1849/163269 objects degraded (1.132%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:53.769 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:53.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:53.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:53.882 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:16:53.914 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:53 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:16:53.914 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:53 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:16:53.914 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:53 smithi138 ceph-mon[193606]: osdmap e211: 8 total, 8 up, 8 in 2023-09-23T23:16:53.914 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:53 smithi138 ceph-mon[193606]: failed to encode map e211 with expected crc 2023-09-23T23:16:53.914 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:53 smithi138 ceph-mon[193606]: failed to encode map e211 with expected crc 2023-09-23T23:16:53.914 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:53 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1849/163269 objects degraded (1.132%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:53.916 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:53.964 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:16:53.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:53.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:16:54.072 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:54.169 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:54.215 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:54.363 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:16:54.363 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:16:54.364 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:16:54.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:54.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:16:54.469 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:54.566 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:54.639 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:54 smithi110 ceph-mon[221102]: pgmap v682: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 1849/162294 objects degraded (1.139%); 16808/162294 objects misplaced (10.357%); 948 KiB/s, 14 objects/s recovering 2023-09-23T23:16:54.639 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:54 smithi110 ceph-mon[212939]: pgmap v682: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 1849/162294 objects degraded (1.139%); 16808/162294 objects misplaced (10.357%); 948 KiB/s, 14 objects/s recovering 2023-09-23T23:16:54.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:54 smithi138 ceph-mon[193606]: pgmap v682: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 1849/162294 objects degraded (1.139%); 16808/162294 objects misplaced (10.357%); 948 KiB/s, 14 objects/s recovering 2023-09-23T23:16:55.504 INFO:teuthology.orchestra.run.smithi110.stdout:moved 40 / 297 (13.468%) 2023-09-23T23:16:55.504 INFO:teuthology.orchestra.run.smithi110.stdout:avg 37.125 2023-09-23T23:16:55.504 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 6.54671 -> 8.63767 (expected baseline 5.69951) 2023-09-23T23:16:55.504 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.5 with 24 -> 30 pgs (0.646465 -> 0.808081 * mean) 2023-09-23T23:16:55.504 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.3 with 47 -> 44 pgs (1.26599 -> 1.18519 * mean) 2023-09-23T23:16:55.504 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:16:55.504 INFO:teuthology.orchestra.run.smithi110.stdout:oload 110 2023-09-23T23:16:55.504 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 3 2023-09-23T23:16:55.504 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:16:55.504 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0184 2023-09-23T23:16:55.505 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.0202 2023-09-23T23:16:55.505 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 weight 0.9000 -> 0.5625 2023-09-23T23:16:55.505 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 weight 0.9000 -> 0.6534 2023-09-23T23:16:55.505 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 weight 0.9000 -> 0.7057 2023-09-23T23:16:55.505 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 weight 0.6527 -> 0.5298 2023-09-23T23:16:55.844 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:55.846 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:55 smithi110 ceph-mon[212939]: from='client.45356 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:16:55.846 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:55 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'42819','3':'46251','5':'34720','7':'36861'}"}]: dispatch 2023-09-23T23:16:55.846 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:55 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'42819','3':'46251','5':'34720','7':'36861'}"}]: dispatch 2023-09-23T23:16:55.847 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:55 smithi110 ceph-mon[221102]: from='client.45356 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:16:55.847 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:55 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'42819','3':'46251','5':'34720','7':'36861'}"}]: dispatch 2023-09-23T23:16:55.847 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:55 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'42819','3':'46251','5':'34720','7':'36861'}"}]: dispatch 2023-09-23T23:16:55.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:55.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:55.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:55 smithi138 ceph-mon[193606]: from='client.45356 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:16:55.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:55 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'42819','3':'46251','5':'34720','7':'36861'}"}]: dispatch 2023-09-23T23:16:55.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:55 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'42819','3':'46251','5':'34720','7':'36861'}"}]: dispatch 2023-09-23T23:16:55.948 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:16:55.961 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:16:55.974 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:56.016 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:16:56.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:56.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:16:56.121 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:56.147 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:56.189 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:56.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:56.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:56.296 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:56.322 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:56.364 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:56.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:56.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:56.472 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:56.497 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:56.563 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:56.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:56.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:56.671 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:56.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:56 smithi138 ceph-mon[193606]: pgmap v683: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 1849/157044 objects degraded (1.177%); 16426/157044 objects misplaced (10.459%); 832 KiB/s, 12 objects/s recovering 2023-09-23T23:16:56.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:56 smithi138 ceph-mon[193606]: failed to encode map e212 with expected crc 2023-09-23T23:16:56.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:56 smithi138 ceph-mon[193606]: failed to encode map e212 with expected crc 2023-09-23T23:16:56.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:56 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'42819','3':'46251','5':'34720','7':'36861'}"}]': finished 2023-09-23T23:16:56.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:56 smithi138 ceph-mon[193606]: osdmap e212: 8 total, 8 up, 8 in 2023-09-23T23:16:56.697 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:56.738 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:56.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:56.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:56.802 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:56 smithi110 ceph-mon[212939]: pgmap v683: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 1849/157044 objects degraded (1.177%); 16426/157044 objects misplaced (10.459%); 832 KiB/s, 12 objects/s recovering 2023-09-23T23:16:56.802 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:56 smithi110 ceph-mon[212939]: failed to encode map e212 with expected crc 2023-09-23T23:16:56.802 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:56 smithi110 ceph-mon[212939]: failed to encode map e212 with expected crc 2023-09-23T23:16:56.802 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:56 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'42819','3':'46251','5':'34720','7':'36861'}"}]': finished 2023-09-23T23:16:56.802 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:56 smithi110 ceph-mon[212939]: osdmap e212: 8 total, 8 up, 8 in 2023-09-23T23:16:56.802 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:56 smithi110 ceph-mon[221102]: pgmap v683: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 1849/157044 objects degraded (1.177%); 16426/157044 objects misplaced (10.459%); 832 KiB/s, 12 objects/s recovering 2023-09-23T23:16:56.802 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:56 smithi110 ceph-mon[221102]: failed to encode map e212 with expected crc 2023-09-23T23:16:56.802 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:56 smithi110 ceph-mon[221102]: failed to encode map e212 with expected crc 2023-09-23T23:16:56.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:56 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'42819','3':'46251','5':'34720','7':'36861'}"}]': finished 2023-09-23T23:16:56.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:56 smithi110 ceph-mon[221102]: osdmap e212: 8 total, 8 up, 8 in 2023-09-23T23:16:56.843 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:56.869 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:56.910 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:56.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:56.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:57.016 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:16:57.042 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:57.060 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:16:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:16:56] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.43.0" 2023-09-23T23:16:57.081 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T23:16:57.081 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T23:16:57.083 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:16:57.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:57.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:16:57.157 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:16:57.191 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:16:57.232 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:57.335 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:16:57.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:57.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:16:57.439 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:16:57.465 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:57.506 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:16:57.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:57.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:16:57.611 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:57.655 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:57.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[212939]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[212939]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[212939]: osdmap e213: 8 total, 8 up, 8 in 2023-09-23T23:16:57.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[212939]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[212939]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[212939]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[212939]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[212939]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[212939]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[212939]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[212939]: 9.7 starting backfill to osd.1 from (0'0,0'0] MAX to 211'10583 2023-09-23T23:16:57.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[212939]: 9.7 continuing backfill to osd.2 from (168'500,211'10583] MIN to 211'10583 2023-09-23T23:16:57.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[221102]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[221102]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[221102]: osdmap e213: 8 total, 8 up, 8 in 2023-09-23T23:16:57.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[221102]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[221102]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[221102]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[221102]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[221102]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[221102]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[221102]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[221102]: 9.7 starting backfill to osd.1 from (0'0,0'0] MAX to 211'10583 2023-09-23T23:16:57.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:57 smithi110 ceph-mon[221102]: 9.7 continuing backfill to osd.2 from (168'500,211'10583] MIN to 211'10583 2023-09-23T23:16:57.767 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:57.782 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:57 smithi138 ceph-mon[193606]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.782 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:57 smithi138 ceph-mon[193606]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.782 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:57 smithi138 ceph-mon[193606]: osdmap e213: 8 total, 8 up, 8 in 2023-09-23T23:16:57.782 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:57 smithi138 ceph-mon[193606]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.783 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:57 smithi138 ceph-mon[193606]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.783 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:57 smithi138 ceph-mon[193606]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.783 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:57 smithi138 ceph-mon[193606]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.783 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:57 smithi138 ceph-mon[193606]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.783 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:57 smithi138 ceph-mon[193606]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.783 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:57 smithi138 ceph-mon[193606]: failed to encode map e213 with expected crc 2023-09-23T23:16:57.783 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:57 smithi138 ceph-mon[193606]: 9.7 starting backfill to osd.1 from (0'0,0'0] MAX to 211'10583 2023-09-23T23:16:57.783 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:57 smithi138 ceph-mon[193606]: 9.7 continuing backfill to osd.2 from (168'500,211'10583] MIN to 211'10583 2023-09-23T23:16:57.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:57.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:57.885 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:16:57.930 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:58.022 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:16:58.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:58.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:16:58.127 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:16:58.157 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:58.208 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:16:58.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:58.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:16:58.312 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:16:58.360 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:58.424 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:16:58.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:58.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:16:58.541 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:16:58.624 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:58.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: pgmap v686: 99 pgs: 1 active+recovering+undersized+remapped, 5 peering, 1 unknown, 1 active+recovery_wait+degraded+remapped, 88 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 469 op/s; 1849/155004 objects degraded (1.193%); 15868/155004 objects misplaced (10.237%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T23:16:58.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:16:58.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 211'10781 2023-09-23T23:16:58.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: 9.9 starting backfill to osd.6 from (0'0,0'0] MAX to 211'10646 2023-09-23T23:16:58.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: osdmap e214: 8 total, 8 up, 8 in 2023-09-23T23:16:58.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 211'10777 2023-09-23T23:16:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: 9.0 starting backfill to osd.0 from (0'0,0'0] MAX to 211'10603 2023-09-23T23:16:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: 9.d starting backfill to osd.6 from (0'0,0'0] MAX to 211'10658 2023-09-23T23:16:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:16:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1849/155004 objects degraded (1.193%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: osdmap e215: 8 total, 8 up, 8 in 2023-09-23T23:16:58.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:58 smithi138 ceph-mon[193606]: failed to encode map e215 with expected crc 2023-09-23T23:16:58.844 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: pgmap v686: 99 pgs: 1 active+recovering+undersized+remapped, 5 peering, 1 unknown, 1 active+recovery_wait+degraded+remapped, 88 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 469 op/s; 1849/155004 objects degraded (1.193%); 15868/155004 objects misplaced (10.237%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T23:16:58.844 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:16:58.844 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 211'10781 2023-09-23T23:16:58.844 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: 9.9 starting backfill to osd.6 from (0'0,0'0] MAX to 211'10646 2023-09-23T23:16:58.844 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.844 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.844 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.844 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: osdmap e214: 8 total, 8 up, 8 in 2023-09-23T23:16:58.844 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.845 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.845 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.845 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 211'10777 2023-09-23T23:16:58.845 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: 9.0 starting backfill to osd.0 from (0'0,0'0] MAX to 211'10603 2023-09-23T23:16:58.845 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: 9.d starting backfill to osd.6 from (0'0,0'0] MAX to 211'10658 2023-09-23T23:16:58.845 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:58.845 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:16:58.845 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1849/155004 objects degraded (1.193%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:58.845 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: osdmap e215: 8 total, 8 up, 8 in 2023-09-23T23:16:58.845 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[212939]: failed to encode map e215 with expected crc 2023-09-23T23:16:58.845 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: pgmap v686: 99 pgs: 1 active+recovering+undersized+remapped, 5 peering, 1 unknown, 1 active+recovery_wait+degraded+remapped, 88 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 469 op/s; 1849/155004 objects degraded (1.193%); 15868/155004 objects misplaced (10.237%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T23:16:58.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:16:58.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 211'10781 2023-09-23T23:16:58.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: 9.9 starting backfill to osd.6 from (0'0,0'0] MAX to 211'10646 2023-09-23T23:16:58.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: osdmap e214: 8 total, 8 up, 8 in 2023-09-23T23:16:58.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: failed to encode map e214 with expected crc 2023-09-23T23:16:58.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 211'10777 2023-09-23T23:16:58.847 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: 9.0 starting backfill to osd.0 from (0'0,0'0] MAX to 211'10603 2023-09-23T23:16:58.847 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: 9.d starting backfill to osd.6 from (0'0,0'0] MAX to 211'10658 2023-09-23T23:16:58.847 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:16:58.847 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:16:58.847 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1849/155004 objects degraded (1.193%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:16:58.847 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: osdmap e215: 8 total, 8 up, 8 in 2023-09-23T23:16:58.847 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:58 smithi110 ceph-mon[221102]: failed to encode map e215 with expected crc 2023-09-23T23:16:58.850 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:58.875 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:16:58.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:58.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:16:58.988 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:16:59.077 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:59.101 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:59.185 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:16:59.413 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:16:59.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:59.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:16:59.522 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:16:59.556 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:59.601 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:59 smithi110 ceph-mon[212939]: failed to encode map e215 with expected crc 2023-09-23T23:16:59.601 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:59 smithi110 ceph-mon[212939]: failed to encode map e215 with expected crc 2023-09-23T23:16:59.601 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:59 smithi110 ceph-mon[212939]: failed to encode map e215 with expected crc 2023-09-23T23:16:59.601 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:16:59 smithi110 ceph-mon[212939]: osdmap e216: 8 total, 8 up, 8 in 2023-09-23T23:16:59.602 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:59 smithi110 ceph-mon[221102]: failed to encode map e215 with expected crc 2023-09-23T23:16:59.602 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:59 smithi110 ceph-mon[221102]: failed to encode map e215 with expected crc 2023-09-23T23:16:59.602 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:59 smithi110 ceph-mon[221102]: failed to encode map e215 with expected crc 2023-09-23T23:16:59.602 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:16:59 smithi110 ceph-mon[221102]: osdmap e216: 8 total, 8 up, 8 in 2023-09-23T23:16:59.620 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:16:59.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:16:59.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:16:59.730 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:16:59.810 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:16:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:59 smithi138 ceph-mon[193606]: failed to encode map e215 with expected crc 2023-09-23T23:16:59.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:59 smithi138 ceph-mon[193606]: failed to encode map e215 with expected crc 2023-09-23T23:16:59.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:59 smithi138 ceph-mon[193606]: failed to encode map e215 with expected crc 2023-09-23T23:16:59.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:16:59 smithi138 ceph-mon[193606]: osdmap e216: 8 total, 8 up, 8 in 2023-09-23T23:17:00.005 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:17:00.041 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:17:00.092 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:17:00.103 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:17:00.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:00.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:17:00.162 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:17:00.200 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:17:00.219 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:00.255 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:17:00.300 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:17:00.325 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:00.331 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:17:00.362 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:17:00.362 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:17:00.363 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:17:00.363 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:17:00.363 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:17:00.363 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:17:00.363 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:17:00.363 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:17:00.443 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:00.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:00.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:00.553 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:17:00.597 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:00.683 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:17:00.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:00.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:17:00.789 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:17:00.865 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:00 smithi110 ceph-mon[221102]: failed to encode map e216 with expected crc 2023-09-23T23:17:00.865 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:00 smithi110 ceph-mon[221102]: failed to encode map e216 with expected crc 2023-09-23T23:17:00.866 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:00 smithi110 ceph-mon[221102]: pgmap v690: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 7 peering, 1 unknown, 81 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+remapped+backfill_wait; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 21 KiB/s rd, 56 KiB/s wr, 513 op/s; 923/151248 objects degraded (0.610%); 28100/151248 objects misplaced (18.579%); 6.1 MiB/s, 200 objects/s recovering 2023-09-23T23:17:00.866 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:00 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1568859384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:17:00.866 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:00 smithi110 ceph-mon[212939]: failed to encode map e216 with expected crc 2023-09-23T23:17:00.866 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:00 smithi110 ceph-mon[212939]: failed to encode map e216 with expected crc 2023-09-23T23:17:00.866 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:00 smithi110 ceph-mon[212939]: pgmap v690: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 7 peering, 1 unknown, 81 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+remapped+backfill_wait; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 21 KiB/s rd, 56 KiB/s wr, 513 op/s; 923/151248 objects degraded (0.610%); 28100/151248 objects misplaced (18.579%); 6.1 MiB/s, 200 objects/s recovering 2023-09-23T23:17:00.866 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:00 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1568859384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:17:00.876 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:00.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:00 smithi138 ceph-mon[193606]: failed to encode map e216 with expected crc 2023-09-23T23:17:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:00 smithi138 ceph-mon[193606]: failed to encode map e216 with expected crc 2023-09-23T23:17:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:00 smithi138 ceph-mon[193606]: pgmap v690: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 7 peering, 1 unknown, 81 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+remapped+backfill_wait; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 21 KiB/s rd, 56 KiB/s wr, 513 op/s; 923/151248 objects degraded (0.610%); 28100/151248 objects misplaced (18.579%); 6.1 MiB/s, 200 objects/s recovering 2023-09-23T23:17:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:00 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1568859384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:17:01.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:01 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:17:01.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:01 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:17:01.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:01 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:17:02.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:02 smithi138 ceph-mon[193606]: pgmap v691: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfilling, 2 peering, 86 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+remapped+backfill_wait; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 346 op/s; 9057/149928 objects degraded (6.041%); 24791/149928 objects misplaced (16.535%); 3.5 MiB/s, 138 objects/s recovering 2023-09-23T23:17:02.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:02 smithi110 ceph-mon[212939]: pgmap v691: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfilling, 2 peering, 86 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+remapped+backfill_wait; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 346 op/s; 9057/149928 objects degraded (6.041%); 24791/149928 objects misplaced (16.535%); 3.5 MiB/s, 138 objects/s recovering 2023-09-23T23:17:02.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:02 smithi110 ceph-mon[221102]: pgmap v691: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfilling, 2 peering, 86 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+remapped+backfill_wait; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 346 op/s; 9057/149928 objects degraded (6.041%); 24791/149928 objects misplaced (16.535%); 3.5 MiB/s, 138 objects/s recovering 2023-09-23T23:17:03.626 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:03 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 9057/149928 objects degraded (6.041%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:17:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:03 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 9057/149928 objects degraded (6.041%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:17:03.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:03 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 9057/149928 objects degraded (6.041%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:17:04.204 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:17:04.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:04.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:17:04.312 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:17:04.350 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:04.436 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:17:04.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:04.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:17:04.541 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:17:04.628 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:04.634 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:04 smithi138 ceph-mon[193606]: pgmap v692: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfilling, 2 peering, 87 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 9057/148836 objects degraded (6.085%); 23834/148836 objects misplaced (16.014%); 11 MiB/s, 245 objects/s recovering 2023-09-23T23:17:04.851 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:04 smithi110 ceph-mon[221102]: pgmap v692: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfilling, 2 peering, 87 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 9057/148836 objects degraded (6.085%); 23834/148836 objects misplaced (16.014%); 11 MiB/s, 245 objects/s recovering 2023-09-23T23:17:04.854 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:04 smithi110 ceph-mon[212939]: pgmap v692: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfilling, 2 peering, 87 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 9057/148836 objects degraded (6.085%); 23834/148836 objects misplaced (16.014%); 11 MiB/s, 245 objects/s recovering 2023-09-23T23:17:04.882 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:17:04.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:04.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:17:04.995 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:05.162 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:05.294 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:17:05.295 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:17:05.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.295 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45164.0:175544 9.4 e216/164 9:2ddbda5f:::benchmark_data_smithi110_295904_object64363:head v 216'11299, mlcod=216'11297)", 2023-09-23T23:17:05.295 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:05.282645+0000", 2023-09-23T23:17:05.295 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0097545559999999993, 2023-09-23T23:17:05.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0097861949999999993, 2023-09-23T23:17:05.295 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:05.295 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:17:05.295 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:05.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:05.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.282645+0000", 2023-09-23T23:17:05.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:05.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:05.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.282645+0000", 2023-09-23T23:17:05.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:05.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:05.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.282644+0000", 2023-09-23T23:17:05.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:17:05.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:05.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.282651+0000", 2023-09-23T23:17:05.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0509999999999999e-06 2023-09-23T23:17:05.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:05.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.282663+0000", 2023-09-23T23:17:05.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2e-05 2023-09-23T23:17:05.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:05.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.282670+0000", 2023-09-23T23:17:05.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4060000000000002e-06 2023-09-23T23:17:05.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:05.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.282752+0000", 2023-09-23T23:17:05.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2490999999999993e-05 2023-09-23T23:17:05.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:05.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.282777+0000", 2023-09-23T23:17:05.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4664999999999998e-05 2023-09-23T23:17:05.299 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:05.299 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:05.299 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:05.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.299 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:175548 9.c 9:37ba0817:::benchmark_data_smithi110_295904_object64367:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e216)", 2023-09-23T23:17:05.299 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:05.284857+0000", 2023-09-23T23:17:05.299 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0075417710000000001, 2023-09-23T23:17:05.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076449179999999997, 2023-09-23T23:17:05.300 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:05.300 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:17:05.300 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:05.300 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:17:05.300 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:17:05.300 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175548 2023-09-23T23:17:05.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.300 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:05.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:05.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.284857+0000", 2023-09-23T23:17:05.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:05.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:05.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.284857+0000", 2023-09-23T23:17:05.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:05.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:05.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.284856+0000", 2023-09-23T23:17:05.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:17:05.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:05.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.284864+0000", 2023-09-23T23:17:05.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5499999999999995e-06 2023-09-23T23:17:05.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:05.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.284883+0000", 2023-09-23T23:17:05.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8943000000000001e-05 2023-09-23T23:17:05.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:05.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.284893+0000", 2023-09-23T23:17:05.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4350000000000003e-06 2023-09-23T23:17:05.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:05.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.285462+0000", 2023-09-23T23:17:05.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000569657 2023-09-23T23:17:05.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:05.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.285600+0000", 2023-09-23T23:17:05.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013789799999999999 2023-09-23T23:17:05.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,7", 2023-09-23T23:17:05.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.285724+0000", 2023-09-23T23:17:05.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012342599999999999 2023-09-23T23:17:05.304 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:05.304 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:05.304 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:05.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.304 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:175556 9.c 9:38a7a5a8:::benchmark_data_smithi110_295904_object64375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e216)", 2023-09-23T23:17:05.304 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:05.284998+0000", 2023-09-23T23:17:05.304 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.007401721, 2023-09-23T23:17:05.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075918039999999997, 2023-09-23T23:17:05.304 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:05.305 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:17:05.305 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:05.305 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:17:05.305 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:17:05.305 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175556 2023-09-23T23:17:05.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.305 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:05.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:05.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.284998+0000", 2023-09-23T23:17:05.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:05.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:05.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.284998+0000", 2023-09-23T23:17:05.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:05.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:05.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.284997+0000", 2023-09-23T23:17:05.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:17:05.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:05.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.285002+0000", 2023-09-23T23:17:05.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.972e-06 2023-09-23T23:17:05.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:05.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.285012+0000", 2023-09-23T23:17:05.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0352e-05 2023-09-23T23:17:05.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:05.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.285028+0000", 2023-09-23T23:17:05.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6229999999999999e-05 2023-09-23T23:17:05.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:05.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.288946+0000", 2023-09-23T23:17:05.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039181889999999999 2023-09-23T23:17:05.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:05.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.289085+0000", 2023-09-23T23:17:05.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013852 2023-09-23T23:17:05.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,7", 2023-09-23T23:17:05.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.289215+0000", 2023-09-23T23:17:05.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013005499999999999 2023-09-23T23:17:05.309 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:05.309 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:05.309 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:05.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.309 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45164.0:175549 9.7 e216/212 9:e9742a3d:::benchmark_data_smithi110_295904_object64368:head v 216'10860, mlcod=216'10858)", 2023-09-23T23:17:05.309 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:05.287851+0000", 2023-09-23T23:17:05.309 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0045480290000000003, 2023-09-23T23:17:05.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048115229999999998, 2023-09-23T23:17:05.310 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:05.310 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:17:05.310 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:05.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:05.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.287851+0000", 2023-09-23T23:17:05.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:05.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:05.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.287851+0000", 2023-09-23T23:17:05.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:05.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:05.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.287850+0000", 2023-09-23T23:17:05.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:17:05.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:05.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.287862+0000", 2023-09-23T23:17:05.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1545999999999999e-05 2023-09-23T23:17:05.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:05.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.287873+0000", 2023-09-23T23:17:05.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0791999999999999e-05 2023-09-23T23:17:05.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:05.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.287879+0000", 2023-09-23T23:17:05.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6889999999999998e-06 2023-09-23T23:17:05.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:05.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.287930+0000", 2023-09-23T23:17:05.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0331000000000003e-05 2023-09-23T23:17:05.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:05.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.287947+0000", 2023-09-23T23:17:05.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7283999999999999e-05 2023-09-23T23:17:05.313 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:05.313 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:05.313 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:05.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.313 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.45164.0:175548 9.c e216/164)", 2023-09-23T23:17:05.313 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:05.289312+0000", 2023-09-23T23:17:05.314 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0030874370000000002, 2023-09-23T23:17:05.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034081390000000001, 2023-09-23T23:17:05.314 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:05.314 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:17:05.314 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:05.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:05.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.289312+0000", 2023-09-23T23:17:05.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:05.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:05.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.289312+0000", 2023-09-23T23:17:05.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:05.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:05.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.289311+0000", 2023-09-23T23:17:05.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:17:05.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:05.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.289316+0000", 2023-09-23T23:17:05.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3090000000000003e-06 2023-09-23T23:17:05.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:05.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.289327+0000", 2023-09-23T23:17:05.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0634e-05 2023-09-23T23:17:05.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:05.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.289334+0000", 2023-09-23T23:17:05.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3320000000000003e-06 2023-09-23T23:17:05.316 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:05.316 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:05.316 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:05.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.317 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45164.0:175555 9.7 e216/212 9:ecda626e:::benchmark_data_smithi110_295904_object64374:head v 216'10861, mlcod=216'10858)", 2023-09-23T23:17:05.317 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:05.290126+0000", 2023-09-23T23:17:05.317 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0022732009999999999, 2023-09-23T23:17:05.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026420089999999999, 2023-09-23T23:17:05.317 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:05.317 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:17:05.317 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:05.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:05.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.290126+0000", 2023-09-23T23:17:05.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:05.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:05.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.290126+0000", 2023-09-23T23:17:05.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:05.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:05.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.290125+0000", 2023-09-23T23:17:05.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:17:05.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:05.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.290137+0000", 2023-09-23T23:17:05.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2098e-05 2023-09-23T23:17:05.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:05.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.290154+0000", 2023-09-23T23:17:05.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7050000000000001e-05 2023-09-23T23:17:05.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:05.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.290162+0000", 2023-09-23T23:17:05.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9980000000000003e-06 2023-09-23T23:17:05.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:05.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.292441+0000", 2023-09-23T23:17:05.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022786149999999999 2023-09-23T23:17:05.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:05.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:05.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:05.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:05.292467+0000", 2023-09-23T23:17:05.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6574999999999999e-05 2023-09-23T23:17:05.321 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:05.321 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:05.321 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:05.321 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:05.321 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-09-23T23:17:05.321 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 6 2023-09-23T23:17:05.321 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:17:05.337 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:05.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:05.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:05.447 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:17:05.481 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:05.546 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:17:05.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:05.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:17:05.656 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:17:05.674 INFO:teuthology.orchestra.run.smithi110.stdout:moved 38 / 297 (12.7946%) 2023-09-23T23:17:05.674 INFO:teuthology.orchestra.run.smithi110.stdout:avg 37.125 2023-09-23T23:17:05.675 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 8.63767 -> 13.986 (expected baseline 5.69951) 2023-09-23T23:17:05.675 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.7 with 19 -> 12 pgs (0.511784 -> 0.323232 * mean) 2023-09-23T23:17:05.675 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.4 with 47 -> 52 pgs (1.26599 -> 1.40067 * mean) 2023-09-23T23:17:05.675 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:17:05.675 INFO:teuthology.orchestra.run.smithi110.stdout:oload 110 2023-09-23T23:17:05.675 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 1 2023-09-23T23:17:05.675 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:17:05.675 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0169 2023-09-23T23:17:05.675 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.0186 2023-09-23T23:17:05.675 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 weight 0.5625 -> 0.3567 2023-09-23T23:17:05.675 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 weight 0.6534 -> 0.4798 2023-09-23T23:17:05.675 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 weight 0.7057 -> 0.5531 2023-09-23T23:17:05.676 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 weight 0.5298 -> 0.4312 2023-09-23T23:17:05.693 INFO:teuthology.orchestra.run.smithi110.stderr:no change 2023-09-23T23:17:05.707 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:17:05.729 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:05 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2710578238' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:17:05.729 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:05 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:17:05.729 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:17:05 smithi110 conmon[198813]: 2023-09-23T23:17:05.667+0000 7ff0150b6700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-09-23T23:17:05.730 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:05 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2710578238' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:17:05.730 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:05 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:17:05.740 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:05.866 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:17:05.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:05.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:17:05.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:05 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2710578238' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:17:05.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:05 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:17:05.978 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:17:06.073 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:06.210 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:17:06.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:06 smithi138 ceph-mon[193606]: pgmap v693: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfilling, 89 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 9057/143808 objects degraded (6.298%); 22268/143808 objects misplaced (15.485%); 14 MiB/s, 318 objects/s recovering 2023-09-23T23:17:06.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:06 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:17:06.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:06 smithi138 ceph-mon[193606]: failed to encode map e217 with expected crc 2023-09-23T23:17:06.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:06 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:17:06.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:06 smithi138 ceph-mon[193606]: osdmap e217: 8 total, 8 up, 8 in 2023-09-23T23:17:06.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:06 smithi138 ceph-mon[193606]: failed to encode map e217 with expected crc 2023-09-23T23:17:06.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:06 smithi138 ceph-mon[193606]: from='client.55336 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:17:06.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:06 smithi138 ceph-mon[193606]: failed to encode map e217 with expected crc 2023-09-23T23:17:06.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:06 smithi138 ceph-mon[193606]: failed to encode map e217 with expected crc 2023-09-23T23:17:06.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[221102]: pgmap v693: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfilling, 89 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 9057/143808 objects degraded (6.298%); 22268/143808 objects misplaced (15.485%); 14 MiB/s, 318 objects/s recovering 2023-09-23T23:17:06.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:17:06.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[221102]: failed to encode map e217 with expected crc 2023-09-23T23:17:06.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:17:06.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[221102]: osdmap e217: 8 total, 8 up, 8 in 2023-09-23T23:17:06.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[221102]: failed to encode map e217 with expected crc 2023-09-23T23:17:06.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[221102]: from='client.55336 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:17:06.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[221102]: failed to encode map e217 with expected crc 2023-09-23T23:17:06.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[221102]: failed to encode map e217 with expected crc 2023-09-23T23:17:06.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[212939]: pgmap v693: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfilling, 89 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 9057/143808 objects degraded (6.298%); 22268/143808 objects misplaced (15.485%); 14 MiB/s, 318 objects/s recovering 2023-09-23T23:17:06.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:17:06.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[212939]: failed to encode map e217 with expected crc 2023-09-23T23:17:06.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:17:06.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[212939]: osdmap e217: 8 total, 8 up, 8 in 2023-09-23T23:17:06.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[212939]: failed to encode map e217 with expected crc 2023-09-23T23:17:06.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[212939]: from='client.55336 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:17:06.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[212939]: failed to encode map e217 with expected crc 2023-09-23T23:17:06.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:06 smithi110 ceph-mon[212939]: failed to encode map e217 with expected crc 2023-09-23T23:17:06.955 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:17:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:17:06] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-09-23T23:17:07.549 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:17:07.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:07.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:17:07.661 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:07.773 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:07.992 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:08.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:08.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:08.115 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:17:08.192 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:08.407 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T23:17:08.408 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T23:17:08.441 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 23:17:07 smithi138 conmon[172649]: logger=cleanup t=2023-09-23T23:17:07.950383247Z level=info msg="Completed cleanup jobs" duration=14.611161ms 2023-09-23T23:17:08.525 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:17:08.608 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:17:08.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:08.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:17:08.713 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:17:08.722 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:17:08.744 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:08.803 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:17:08.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:08.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:17:08.911 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:17:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:08 smithi138 ceph-mon[193606]: pgmap v695: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfilling, 89 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 402 op/s; 8727/141777 objects degraded (6.155%); 21912/141777 objects misplaced (15.455%); 15 MiB/s, 276 objects/s recovering 2023-09-23T23:17:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:08 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 8727/141777 objects degraded (6.155%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:17:08.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:08 smithi110 ceph-mon[212939]: pgmap v695: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfilling, 89 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 402 op/s; 8727/141777 objects degraded (6.155%); 21912/141777 objects misplaced (15.455%); 15 MiB/s, 276 objects/s recovering 2023-09-23T23:17:08.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:08 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 8727/141777 objects degraded (6.155%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:17:08.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:08 smithi110 ceph-mon[221102]: pgmap v695: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfilling, 89 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 402 op/s; 8727/141777 objects degraded (6.155%); 21912/141777 objects misplaced (15.455%); 15 MiB/s, 276 objects/s recovering 2023-09-23T23:17:08.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:08 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 8727/141777 objects degraded (6.155%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:17:09.028 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:09.229 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:17:09.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:09.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:17:09.340 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:09.442 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:09.634 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:09.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:09.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:09.751 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:09.806 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:09.954 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:09.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:09.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:10.061 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:10.140 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:10.278 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:10.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:10.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:10.388 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:10.416 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:10.465 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:10.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:10.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:10.484 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:17:10.569 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:17:10.602 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:10.733 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:17:10.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:10.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:17:10.845 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:10.931 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:10 smithi110 ceph-mon[221102]: pgmap v696: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfilling, 89 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 7804/136446 objects degraded (5.719%); 22132/136446 objects misplaced (16.220%); 23 MiB/s, 442 objects/s recovering 2023-09-23T23:17:10.933 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:10 smithi110 ceph-mon[212939]: pgmap v696: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfilling, 89 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 7804/136446 objects degraded (5.719%); 22132/136446 objects misplaced (16.220%); 23 MiB/s, 442 objects/s recovering 2023-09-23T23:17:10.935 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:10.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:10 smithi138 ceph-mon[193606]: pgmap v696: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfilling, 89 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 7804/136446 objects degraded (5.719%); 22132/136446 objects misplaced (16.220%); 23 MiB/s, 442 objects/s recovering 2023-09-23T23:17:11.066 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:17:11.069 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:17:11.087 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:11.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:11.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:11.200 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:17:11.246 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:11.307 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:17:11.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:11.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:17:11.414 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:17:11.505 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:11.773 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:17:11.786 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:17:11.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:11.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:17:11.907 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:17:11.978 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:12.531 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:17:12.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:12.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:17:12.638 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:12.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:12 smithi110 ceph-mon[212939]: pgmap v697: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfilling, 89 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 7490/134763 objects degraded (5.558%); 21848/134763 objects misplaced (16.212%); 23 MiB/s, 449 objects/s recovering 2023-09-23T23:17:12.706 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:12.757 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:12.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:12.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:12.861 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:12.890 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:12.941 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:12.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:12.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:12.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:12 smithi110 ceph-mon[221102]: pgmap v697: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfilling, 89 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 7490/134763 objects degraded (5.558%); 21848/134763 objects misplaced (16.212%); 23 MiB/s, 449 objects/s recovering 2023-09-23T23:17:13.046 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:13.114 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:13.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:12 smithi138 ceph-mon[193606]: pgmap v697: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfilling, 89 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 7490/134763 objects degraded (5.558%); 21848/134763 objects misplaced (16.212%); 23 MiB/s, 449 objects/s recovering 2023-09-23T23:17:13.370 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:13.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:13.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:13.488 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:17:13.520 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:13.567 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:17:13.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:13.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:17:13.594 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:17:13.674 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:17:13.691 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:17:13.702 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:13.749 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:17:13.769 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:17:13.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:13.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:17:13.817 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:17:13.867 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:17:13.876 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:17:13.941 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:17:13.975 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:13.976 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:17:14.036 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:17:14.043 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:17:14.043 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:17:14.043 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:17:14.043 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:17:14.043 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:17:14.043 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:17:14.043 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:17:14.043 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:17:14.072 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:14.072 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:17:14.072 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:13 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 7490/134763 objects degraded (5.558%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:17:14.072 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:13 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1179645694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:17:14.075 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:14.075 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:17:14.075 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:13 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 7490/134763 objects degraded (5.558%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:17:14.075 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:13 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1179645694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:17:14.196 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:14.197 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:17:14.197 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:13 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 7490/134763 objects degraded (5.558%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:17:14.197 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:13 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1179645694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:17:14.337 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:17:14.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:14.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:17:14.448 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:17:14.480 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:14.552 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:17:14.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:14.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:17:14.663 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:17:14.696 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:14.747 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:17:14.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:14.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:17:14.854 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:17:14.885 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:14.936 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:17:14.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:14.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:17:15.042 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:17:15.075 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:14 smithi138 ceph-mon[193606]: pgmap v698: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfilling, 89 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 7490/133941 objects degraded (5.592%); 20783/133941 objects misplaced (15.517%); 24 MiB/s, 471 objects/s recovering 2023-09-23T23:17:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:14 smithi138 ceph-mon[193606]: osdmap e218: 8 total, 8 up, 8 in 2023-09-23T23:17:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:14 smithi138 ceph-mon[193606]: failed to encode map e218 with expected crc 2023-09-23T23:17:15.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:14 smithi138 ceph-mon[193606]: failed to encode map e218 with expected crc 2023-09-23T23:17:15.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:14 smithi110 ceph-mon[212939]: pgmap v698: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfilling, 89 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 7490/133941 objects degraded (5.592%); 20783/133941 objects misplaced (15.517%); 24 MiB/s, 471 objects/s recovering 2023-09-23T23:17:15.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:14 smithi110 ceph-mon[212939]: osdmap e218: 8 total, 8 up, 8 in 2023-09-23T23:17:15.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:14 smithi110 ceph-mon[212939]: failed to encode map e218 with expected crc 2023-09-23T23:17:15.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:14 smithi110 ceph-mon[212939]: failed to encode map e218 with expected crc 2023-09-23T23:17:15.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:14 smithi110 ceph-mon[221102]: pgmap v698: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfilling, 89 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 7490/133941 objects degraded (5.592%); 20783/133941 objects misplaced (15.517%); 24 MiB/s, 471 objects/s recovering 2023-09-23T23:17:15.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:14 smithi110 ceph-mon[221102]: osdmap e218: 8 total, 8 up, 8 in 2023-09-23T23:17:15.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:14 smithi110 ceph-mon[221102]: failed to encode map e218 with expected crc 2023-09-23T23:17:15.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:14 smithi110 ceph-mon[221102]: failed to encode map e218 with expected crc 2023-09-23T23:17:16.101 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:17:16.104 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:15 smithi110 ceph-mon[212939]: failed to encode map e218 with expected crc 2023-09-23T23:17:16.104 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:15 smithi110 ceph-mon[212939]: failed to encode map e218 with expected crc 2023-09-23T23:17:16.104 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:15 smithi110 ceph-mon[212939]: osdmap e219: 8 total, 8 up, 8 in 2023-09-23T23:17:16.104 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:15 smithi110 ceph-mon[212939]: failed to encode map e219 with expected crc 2023-09-23T23:17:16.104 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:15 smithi110 ceph-mon[212939]: failed to encode map e219 with expected crc 2023-09-23T23:17:16.104 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:15 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:17:16.104 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:15 smithi110 ceph-mon[221102]: failed to encode map e218 with expected crc 2023-09-23T23:17:16.104 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:15 smithi110 ceph-mon[221102]: failed to encode map e218 with expected crc 2023-09-23T23:17:16.105 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:15 smithi110 ceph-mon[221102]: osdmap e219: 8 total, 8 up, 8 in 2023-09-23T23:17:16.105 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:15 smithi110 ceph-mon[221102]: failed to encode map e219 with expected crc 2023-09-23T23:17:16.105 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:15 smithi110 ceph-mon[221102]: failed to encode map e219 with expected crc 2023-09-23T23:17:16.105 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:15 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:17:16.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:16.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:17:16.206 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:17:16.234 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:16.283 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:17:16.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:16.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:17:16.289 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:15 smithi138 ceph-mon[193606]: failed to encode map e218 with expected crc 2023-09-23T23:17:16.289 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:15 smithi138 ceph-mon[193606]: failed to encode map e218 with expected crc 2023-09-23T23:17:16.289 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:15 smithi138 ceph-mon[193606]: osdmap e219: 8 total, 8 up, 8 in 2023-09-23T23:17:16.289 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:15 smithi138 ceph-mon[193606]: failed to encode map e219 with expected crc 2023-09-23T23:17:16.289 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:15 smithi138 ceph-mon[193606]: failed to encode map e219 with expected crc 2023-09-23T23:17:16.289 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:15 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:17:16.389 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:17:16.420 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:17.065 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:17:17.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:17.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:17:17.068 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:16 smithi110 ceph-mon[212939]: failed to encode map e219 with expected crc 2023-09-23T23:17:17.068 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:16 smithi110 ceph-mon[212939]: failed to encode map e219 with expected crc 2023-09-23T23:17:17.068 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:16 smithi110 ceph-mon[212939]: pgmap v701: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfilling, 89 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 7490/129546 objects degraded (5.782%); 18734/129546 objects misplaced (14.461%); 34 MiB/s, 656 objects/s recovering 2023-09-23T23:17:17.068 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:16 smithi110 ceph-mon[212939]: osdmap e220: 8 total, 8 up, 8 in 2023-09-23T23:17:17.068 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:17:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:17:16] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2023-09-23T23:17:17.069 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:17 smithi110 ceph-mon[221102]: failed to encode map e219 with expected crc 2023-09-23T23:17:17.069 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:17 smithi110 ceph-mon[221102]: failed to encode map e219 with expected crc 2023-09-23T23:17:17.069 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:17 smithi110 ceph-mon[221102]: pgmap v701: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfilling, 89 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 7490/129546 objects degraded (5.782%); 18734/129546 objects misplaced (14.461%); 34 MiB/s, 656 objects/s recovering 2023-09-23T23:17:17.069 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:17 smithi110 ceph-mon[221102]: osdmap e220: 8 total, 8 up, 8 in 2023-09-23T23:17:17.169 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:17.189 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:17:17.189 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:17:17.262 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:17.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:16 smithi138 ceph-mon[193606]: failed to encode map e219 with expected crc 2023-09-23T23:17:17.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:16 smithi138 ceph-mon[193606]: failed to encode map e219 with expected crc 2023-09-23T23:17:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:16 smithi138 ceph-mon[193606]: pgmap v701: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfilling, 89 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 7490/129546 objects degraded (5.782%); 18734/129546 objects misplaced (14.461%); 34 MiB/s, 656 objects/s recovering 2023-09-23T23:17:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:16 smithi138 ceph-mon[193606]: osdmap e220: 8 total, 8 up, 8 in 2023-09-23T23:17:17.546 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:17.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:17.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:17.658 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:17:17.689 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:17.745 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:17:17.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:17.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:17:17.856 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:17:17.885 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:17.939 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:17:17.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:17.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:17:18.045 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:17:18.074 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:18.126 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:17:18.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:18.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:17:18.232 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:17:18.257 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:17 smithi138 ceph-mon[193606]: failed to encode map e220 with expected crc 2023-09-23T23:17:18.258 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:17 smithi138 ceph-mon[193606]: failed to encode map e220 with expected crc 2023-09-23T23:17:18.258 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:17 smithi138 ceph-mon[193606]: failed to encode map e220 with expected crc 2023-09-23T23:17:18.258 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:17 smithi138 ceph-mon[193606]: osdmap e221: 8 total, 8 up, 8 in 2023-09-23T23:17:18.258 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:17 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:17:18.260 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:18.308 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:17:18.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:18.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:17:18.348 INFO:teuthology.orchestra.run.smithi110.stdout:moved 31 / 297 (10.4377%) 2023-09-23T23:17:18.348 INFO:teuthology.orchestra.run.smithi110.stdout:avg 37.125 2023-09-23T23:17:18.348 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 8.63767 -> 13.4018 (expected baseline 5.69951) 2023-09-23T23:17:18.348 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.7 with 19 -> 11 pgs (0.511784 -> 0.296296 * mean) 2023-09-23T23:17:18.348 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.4 with 47 -> 52 pgs (1.26599 -> 1.40067 * mean) 2023-09-23T23:17:18.348 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:17:18.348 INFO:teuthology.orchestra.run.smithi110.stdout:oload 110 2023-09-23T23:17:18.348 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 1 2023-09-23T23:17:18.348 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:17:18.348 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0161 2023-09-23T23:17:18.349 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.0177 2023-09-23T23:17:18.349 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 weight 0.5625 -> 0.3519 2023-09-23T23:17:18.349 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 weight 0.6534 -> 0.4862 2023-09-23T23:17:18.349 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 weight 0.5298 -> 0.4318 2023-09-23T23:17:18.349 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 weight 0.7057 -> 0.5877 2023-09-23T23:17:18.349 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:17 smithi110 ceph-mon[212939]: failed to encode map e220 with expected crc 2023-09-23T23:17:18.349 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:17 smithi110 ceph-mon[212939]: failed to encode map e220 with expected crc 2023-09-23T23:17:18.349 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:17 smithi110 ceph-mon[212939]: failed to encode map e220 with expected crc 2023-09-23T23:17:18.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:17 smithi110 ceph-mon[212939]: osdmap e221: 8 total, 8 up, 8 in 2023-09-23T23:17:18.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:17 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:17:18.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:17 smithi110 ceph-mon[221102]: failed to encode map e220 with expected crc 2023-09-23T23:17:18.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:17 smithi110 ceph-mon[221102]: failed to encode map e220 with expected crc 2023-09-23T23:17:18.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:17 smithi110 ceph-mon[221102]: failed to encode map e220 with expected crc 2023-09-23T23:17:18.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:17 smithi110 ceph-mon[221102]: osdmap e221: 8 total, 8 up, 8 in 2023-09-23T23:17:18.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:17 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:17:18.351 INFO:teuthology.orchestra.run.smithi110.stderr:no change 2023-09-23T23:17:18.413 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:17:18.453 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:18.512 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:17:18.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:18.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:17:18.622 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:17:18.656 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:18.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:17:18 smithi110 conmon[198813]: 2023-09-23T23:17:18.346+0000 7ff0150b6700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-09-23T23:17:18.752 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:17:18.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:18.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:17:18.863 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:17:18.973 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:17:18.987 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:19.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:18 smithi138 ceph-mon[193606]: failed to encode map e221 with expected crc 2023-09-23T23:17:19.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:18 smithi138 ceph-mon[193606]: pgmap v704: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+remapped+backfilling, 89 active+clean, 5 active+remapped+backfill_wait; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 23572/128022 objects misplaced (18.412%); 23 MiB/s, 475 objects/s recovering 2023-09-23T23:17:19.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:18 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 7490/129546 objects degraded (5.782%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:17:19.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:18 smithi138 ceph-mon[193606]: osdmap e222: 8 total, 8 up, 8 in 2023-09-23T23:17:19.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:18 smithi138 ceph-mon[193606]: failed to encode map e222 with expected crc 2023-09-23T23:17:19.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:18 smithi138 ceph-mon[193606]: failed to encode map e222 with expected crc 2023-09-23T23:17:19.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:18 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:17:19.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:18 smithi110 ceph-mon[212939]: failed to encode map e221 with expected crc 2023-09-23T23:17:19.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:18 smithi110 ceph-mon[212939]: pgmap v704: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+remapped+backfilling, 89 active+clean, 5 active+remapped+backfill_wait; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 23572/128022 objects misplaced (18.412%); 23 MiB/s, 475 objects/s recovering 2023-09-23T23:17:19.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:18 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 7490/129546 objects degraded (5.782%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:17:19.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:18 smithi110 ceph-mon[212939]: osdmap e222: 8 total, 8 up, 8 in 2023-09-23T23:17:19.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:18 smithi110 ceph-mon[212939]: failed to encode map e222 with expected crc 2023-09-23T23:17:19.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:18 smithi110 ceph-mon[212939]: failed to encode map e222 with expected crc 2023-09-23T23:17:19.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:18 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:17:19.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:18 smithi110 ceph-mon[221102]: failed to encode map e221 with expected crc 2023-09-23T23:17:19.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:18 smithi110 ceph-mon[221102]: pgmap v704: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+remapped+backfilling, 89 active+clean, 5 active+remapped+backfill_wait; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 23572/128022 objects misplaced (18.412%); 23 MiB/s, 475 objects/s recovering 2023-09-23T23:17:19.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:18 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 7490/129546 objects degraded (5.782%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:17:19.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:18 smithi110 ceph-mon[221102]: osdmap e222: 8 total, 8 up, 8 in 2023-09-23T23:17:19.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:18 smithi110 ceph-mon[221102]: failed to encode map e222 with expected crc 2023-09-23T23:17:19.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:18 smithi110 ceph-mon[221102]: failed to encode map e222 with expected crc 2023-09-23T23:17:19.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:18 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:17:19.858 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:17:19.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:19.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:17:19.962 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:17:19.989 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:20.034 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:17:20.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:20.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:17:20.138 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:17:20.166 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:19 smithi110 ceph-mon[212939]: failed to encode map e222 with expected crc 2023-09-23T23:17:20.167 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:19 smithi110 ceph-mon[212939]: from='client.55345 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:17:20.167 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:19 smithi110 ceph-mon[212939]: osdmap e223: 8 total, 8 up, 8 in 2023-09-23T23:17:20.167 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:19 smithi110 ceph-mon[212939]: failed to encode map e223 with expected crc 2023-09-23T23:17:20.167 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:19 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:17:20.167 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:19 smithi110 ceph-mon[221102]: failed to encode map e222 with expected crc 2023-09-23T23:17:20.167 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:19 smithi110 ceph-mon[221102]: from='client.55345 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:17:20.167 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:19 smithi110 ceph-mon[221102]: osdmap e223: 8 total, 8 up, 8 in 2023-09-23T23:17:20.167 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:19 smithi110 ceph-mon[221102]: failed to encode map e223 with expected crc 2023-09-23T23:17:20.168 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:19 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:17:20.170 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:20.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:19 smithi138 ceph-mon[193606]: failed to encode map e222 with expected crc 2023-09-23T23:17:20.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:19 smithi138 ceph-mon[193606]: from='client.55345 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:17:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:19 smithi138 ceph-mon[193606]: osdmap e223: 8 total, 8 up, 8 in 2023-09-23T23:17:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:19 smithi138 ceph-mon[193606]: failed to encode map e223 with expected crc 2023-09-23T23:17:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:19 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:17:21.397 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:17:21.402 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:20 smithi110 ceph-mon[221102]: failed to encode map e223 with expected crc 2023-09-23T23:17:21.403 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:20 smithi110 ceph-mon[221102]: pgmap v707: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 20032/123816 objects misplaced (16.179%); 50 MiB/s, 862 objects/s recovering 2023-09-23T23:17:21.403 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:20 smithi110 ceph-mon[221102]: osdmap e224: 8 total, 8 up, 8 in 2023-09-23T23:17:21.403 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:20 smithi110 ceph-mon[221102]: failed to encode map e224 with expected crc 2023-09-23T23:17:21.403 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:20 smithi110 ceph-mon[221102]: failed to encode map e224 with expected crc 2023-09-23T23:17:21.404 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:20 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:17:21.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:21.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:17:21.405 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:20 smithi110 ceph-mon[212939]: failed to encode map e223 with expected crc 2023-09-23T23:17:21.406 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:20 smithi110 ceph-mon[212939]: pgmap v707: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 20032/123816 objects misplaced (16.179%); 50 MiB/s, 862 objects/s recovering 2023-09-23T23:17:21.406 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:20 smithi110 ceph-mon[212939]: osdmap e224: 8 total, 8 up, 8 in 2023-09-23T23:17:21.406 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:20 smithi110 ceph-mon[212939]: failed to encode map e224 with expected crc 2023-09-23T23:17:21.406 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:20 smithi110 ceph-mon[212939]: failed to encode map e224 with expected crc 2023-09-23T23:17:21.407 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:20 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:17:21.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:20 smithi138 ceph-mon[193606]: failed to encode map e223 with expected crc 2023-09-23T23:17:21.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:20 smithi138 ceph-mon[193606]: pgmap v707: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 20032/123816 objects misplaced (16.179%); 50 MiB/s, 862 objects/s recovering 2023-09-23T23:17:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:20 smithi138 ceph-mon[193606]: osdmap e224: 8 total, 8 up, 8 in 2023-09-23T23:17:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:20 smithi138 ceph-mon[193606]: failed to encode map e224 with expected crc 2023-09-23T23:17:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:20 smithi138 ceph-mon[193606]: failed to encode map e224 with expected crc 2023-09-23T23:17:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:20 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:17:21.506 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:21.555 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:21.678 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:21.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:21.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:21.783 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:17:21.791 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:17:21.818 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:21.844 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T23:17:21.844 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T23:17:21.899 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:17:21.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:21.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:17:22.013 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:17:22.118 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:22.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:22 smithi138 ceph-mon[193606]: failed to encode map e224 with expected crc 2023-09-23T23:17:22.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:22 smithi138 ceph-mon[193606]: failed to encode map e224 with expected crc 2023-09-23T23:17:22.194 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:17:22.211 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:22 smithi110 ceph-mon[212939]: failed to encode map e224 with expected crc 2023-09-23T23:17:22.211 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:22 smithi110 ceph-mon[212939]: failed to encode map e224 with expected crc 2023-09-23T23:17:22.230 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:22 smithi110 ceph-mon[221102]: failed to encode map e224 with expected crc 2023-09-23T23:17:22.231 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:22 smithi110 ceph-mon[221102]: failed to encode map e224 with expected crc 2023-09-23T23:17:22.281 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:17:22.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:22.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:17:22.386 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:17:22.427 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:22.535 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:17:22.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:22.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:17:22.642 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:17:22.721 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:22.974 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:17:22.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:22.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:17:23.040 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:17:23.078 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:23.168 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:23.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:23 smithi138 ceph-mon[193606]: pgmap v709: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 468 op/s; 19449/122268 objects misplaced (15.907%); 49 MiB/s, 841 objects/s recovering 2023-09-23T23:17:23.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:23 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2360854444' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:17:23.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:23 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:17:23.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:23 smithi110 ceph-mon[212939]: pgmap v709: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 468 op/s; 19449/122268 objects misplaced (15.907%); 49 MiB/s, 841 objects/s recovering 2023-09-23T23:17:23.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:23 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2360854444' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:17:23.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:23 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:17:23.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:23 smithi110 ceph-mon[221102]: pgmap v709: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 468 op/s; 19449/122268 objects misplaced (15.907%); 49 MiB/s, 841 objects/s recovering 2023-09-23T23:17:23.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:23 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2360854444' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:17:23.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:23 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:17:23.922 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:23.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:23.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:24.029 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:17:24.072 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:24.145 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:17:24.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:24.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:17:24.157 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:24 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:17:24.158 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:24 smithi110 ceph-mon[212939]: failed to encode map e225 with expected crc 2023-09-23T23:17:24.158 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:24 smithi110 ceph-mon[212939]: failed to encode map e225 with expected crc 2023-09-23T23:17:24.158 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:24 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:17:24.158 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:24 smithi110 ceph-mon[212939]: osdmap e225: 8 total, 8 up, 8 in 2023-09-23T23:17:24.158 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:24 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-09-23T23:17:24.158 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:24 smithi110 ceph-mon[212939]: pgmap v711: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 379 op/s; 19449/121518 objects misplaced (16.005%); 38 MiB/s, 645 objects/s recovering 2023-09-23T23:17:24.158 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:24 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:17:24.158 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:24 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:17:24.158 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:24 smithi110 ceph-mon[221102]: failed to encode map e225 with expected crc 2023-09-23T23:17:24.159 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:24 smithi110 ceph-mon[221102]: failed to encode map e225 with expected crc 2023-09-23T23:17:24.159 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:24 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:17:24.159 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:24 smithi110 ceph-mon[221102]: osdmap e225: 8 total, 8 up, 8 in 2023-09-23T23:17:24.159 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:24 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-09-23T23:17:24.159 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:24 smithi110 ceph-mon[221102]: pgmap v711: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 379 op/s; 19449/121518 objects misplaced (16.005%); 38 MiB/s, 645 objects/s recovering 2023-09-23T23:17:24.159 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:24 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:17:24.252 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:17:24.302 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:24 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:17:24.302 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:24 smithi138 ceph-mon[193606]: failed to encode map e225 with expected crc 2023-09-23T23:17:24.302 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:24 smithi138 ceph-mon[193606]: failed to encode map e225 with expected crc 2023-09-23T23:17:24.302 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:24 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:17:24.302 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:24 smithi138 ceph-mon[193606]: osdmap e225: 8 total, 8 up, 8 in 2023-09-23T23:17:24.303 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:24 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-09-23T23:17:24.303 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:24 smithi138 ceph-mon[193606]: pgmap v711: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 379 op/s; 19449/121518 objects misplaced (16.005%); 38 MiB/s, 645 objects/s recovering 2023-09-23T23:17:24.303 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:24 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:17:24.305 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:24.395 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:17:24.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:24.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:17:24.509 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:17:24.563 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:24.595 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:17:24.629 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:17:24.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:24.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:17:24.741 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:17:24.793 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:24.876 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:17:24.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:24.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:17:24.983 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:17:25.027 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:25.075 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:17:25.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:25.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:17:25.179 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:17:25.223 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:25.294 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:17:25.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:25.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:17:25.410 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:17:25.457 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:25.525 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:17:25.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:25.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:17:25.631 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:17:25.718 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:25.926 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:17:25.961 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:17:25.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:25.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:17:26.076 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:26.155 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:26.502 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:17:26.773 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:26.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:26.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:26.787 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:26 smithi110 ceph-mon[212939]: pgmap v712: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 18104/117222 objects misplaced (15.444%); 17 MiB/s, 277 objects/s recovering 2023-09-23T23:17:26.788 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:26 smithi110 ceph-mon[221102]: pgmap v712: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 18104/117222 objects misplaced (15.444%); 17 MiB/s, 277 objects/s recovering 2023-09-23T23:17:26.887 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:17:26.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:26 smithi138 ceph-mon[193606]: pgmap v712: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 18104/117222 objects misplaced (15.444%); 17 MiB/s, 277 objects/s recovering 2023-09-23T23:17:27.011 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:27.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:17:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:17:26] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2023-09-23T23:17:27.894 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:27 smithi110 ceph-mon[212939]: 1.8 deep-scrub starts 2023-09-23T23:17:27.894 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:27 smithi110 ceph-mon[221102]: 1.8 deep-scrub starts 2023-09-23T23:17:27.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:27 smithi138 ceph-mon[193606]: 1.8 deep-scrub starts 2023-09-23T23:17:27.986 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:17:28.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:28.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:17:28.101 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:17:28.266 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:28.344 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:17:28.367 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:17:28.402 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:17:28.439 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:17:28.494 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:17:28.535 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:17:28.577 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:17:28.622 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:17:28.636 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:17:28.636 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:17:28.636 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:17:28.636 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:17:28.636 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:17:28.636 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:17:28.637 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:17:28.637 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:17:29.430 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:17:29.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:29 smithi138 ceph-mon[193606]: pgmap v713: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 17556/115116 objects misplaced (15.251%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T23:17:29.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:29 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:29.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:29 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:17:29.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:29 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3131251733' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:17:29.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:29 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:17:29.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:29 smithi110 ceph-mon[212939]: pgmap v713: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 17556/115116 objects misplaced (15.251%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T23:17:29.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:29 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:29.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:29 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:17:29.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:29 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3131251733' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:17:29.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:29 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:17:29.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:29 smithi110 ceph-mon[221102]: pgmap v713: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 17556/115116 objects misplaced (15.251%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T23:17:29.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:29 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:29.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:29 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:17:29.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:29 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3131251733' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:17:29.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:29 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:17:29.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:29.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:17:29.544 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:17:29.636 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:30.430 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:17:30.434 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:17:30.434 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:17:30.434 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:17:30.434 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:17:30.434 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.434 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:196597 8.b 8:d7986642:::benchmark_data_smithi110_258128_object77779:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T23:17:30.435 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:15.143183+0000", 2023-09-23T23:17:30.435 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 375.28555881099999, 2023-09-23T23:17:30.435 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.056248751, 2023-09-23T23:17:30.435 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.435 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:30.435 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.435 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.435 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.435 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196597 2023-09-23T23:17:30.435 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.436 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.436 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.436 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.143183+0000", 2023-09-23T23:17:30.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.436 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.436 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.436 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.143183+0000", 2023-09-23T23:17:30.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.437 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.143181+0000", 2023-09-23T23:17:30.437 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:17:30.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.437 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.143191+0000", 2023-09-23T23:17:30.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0002e-05 2023-09-23T23:17:30.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.438 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.143210+0000", 2023-09-23T23:17:30.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8980000000000001e-05 2023-09-23T23:17:30.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.438 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.143219+0000", 2023-09-23T23:17:30.439 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7110000000000001e-06 2023-09-23T23:17:30.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.439 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.439 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.439 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.143270+0000", 2023-09-23T23:17:30.439 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0998999999999997e-05 2023-09-23T23:17:30.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.439 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.439 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.440 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.143281+0000", 2023-09-23T23:17:30.440 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0983e-05 2023-09-23T23:17:30.440 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.441 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.441 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.184844+0000", 2023-09-23T23:17:30.441 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0415627949999999 2023-09-23T23:17:30.441 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.442 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:30.442 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.184986+0000", 2023-09-23T23:17:30.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000142905 2023-09-23T23:17:30.442 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.442 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.442 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:17:30.442 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.185126+0000", 2023-09-23T23:17:30.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014002900000000001 2023-09-23T23:17:30.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:17:30.443 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.191958+0000", 2023-09-23T23:17:30.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068318399999999996 2023-09-23T23:17:30.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.443 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.194121+0000", 2023-09-23T23:17:30.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021628049999999998 2023-09-23T23:17:30.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.444 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.199343+0000", 2023-09-23T23:17:30.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052219040000000003 2023-09-23T23:17:30.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.444 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:17:30.445 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.199392+0000", 2023-09-23T23:17:30.445 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8751999999999997e-05 2023-09-23T23:17:30.445 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.445 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.445 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.445 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.199432+0000", 2023-09-23T23:17:30.445 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9780999999999997e-05 2023-09-23T23:17:30.445 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.445 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.446 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.446 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.446 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:196598 8.b 8:d50b802c:::benchmark_data_smithi110_258128_object77780:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T23:17:30.446 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:15.143230+0000", 2023-09-23T23:17:30.446 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 375.28551121599997, 2023-09-23T23:17:30.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.065703801, 2023-09-23T23:17:30.447 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.447 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:30.447 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.447 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.447 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.447 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196598 2023-09-23T23:17:30.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.447 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.143230+0000", 2023-09-23T23:17:30.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.143230+0000", 2023-09-23T23:17:30.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.143230+0000", 2023-09-23T23:17:30.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:17:30.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.143234+0000", 2023-09-23T23:17:30.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.318e-06 2023-09-23T23:17:30.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.450 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.143241+0000", 2023-09-23T23:17:30.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2840000000000003e-06 2023-09-23T23:17:30.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.450 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.451 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.143246+0000", 2023-09-23T23:17:30.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0679999999999996e-06 2023-09-23T23:17:30.451 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.451 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.451 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.451 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.143318+0000", 2023-09-23T23:17:30.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2096999999999998e-05 2023-09-23T23:17:30.451 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.451 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.143328+0000", 2023-09-23T23:17:30.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0583e-05 2023-09-23T23:17:30.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.187494+0000", 2023-09-23T23:17:30.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0441661390000001 2023-09-23T23:17:30.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:30.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.187607+0000", 2023-09-23T23:17:30.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000113009 2023-09-23T23:17:30.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:17:30.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.187714+0000", 2023-09-23T23:17:30.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000106297 2023-09-23T23:17:30.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:17:30.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.194080+0000", 2023-09-23T23:17:30.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063663260000000003 2023-09-23T23:17:30.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.199298+0000", 2023-09-23T23:17:30.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052174689999999998 2023-09-23T23:17:30.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.208842+0000", 2023-09-23T23:17:30.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.009544594 2023-09-23T23:17:30.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:17:30.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.208893+0000", 2023-09-23T23:17:30.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0489999999999999e-05 2023-09-23T23:17:30.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.456 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.208934+0000", 2023-09-23T23:17:30.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1593e-05 2023-09-23T23:17:30.456 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.456 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.456 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.457 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.457 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:196618 8.b 8:db88eba6:::benchmark_data_smithi110_258128_object77800:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T23:17:30.457 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:15.174965+0000", 2023-09-23T23:17:30.457 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 375.25377689700002, 2023-09-23T23:17:30.457 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0341465940000001, 2023-09-23T23:17:30.457 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.457 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:30.457 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.457 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.458 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.458 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196618 2023-09-23T23:17:30.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.458 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.458 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.458 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.458 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.174965+0000", 2023-09-23T23:17:30.458 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.459 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.174965+0000", 2023-09-23T23:17:30.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.459 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.459 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.174954+0000", 2023-09-23T23:17:30.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999989 2023-09-23T23:17:30.460 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.460 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.460 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.460 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.174972+0000", 2023-09-23T23:17:30.460 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.863e-05 2023-09-23T23:17:30.460 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.460 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.460 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.460 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.174993+0000", 2023-09-23T23:17:30.460 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0758e-05 2023-09-23T23:17:30.461 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.461 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.175004+0000", 2023-09-23T23:17:30.461 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1012000000000001e-05 2023-09-23T23:17:30.461 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.461 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.175078+0000", 2023-09-23T23:17:30.461 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3758999999999996e-05 2023-09-23T23:17:30.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.175090+0000", 2023-09-23T23:17:30.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1866e-05 2023-09-23T23:17:30.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.189766+0000", 2023-09-23T23:17:30.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0146757319999999 2023-09-23T23:17:30.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.463 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:30.463 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.189873+0000", 2023-09-23T23:17:30.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107612 2023-09-23T23:17:30.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.463 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:17:30.463 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.189946+0000", 2023-09-23T23:17:30.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2774000000000005e-05 2023-09-23T23:17:30.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:17:30.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.196449+0000", 2023-09-23T23:17:30.464 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.006503099 2023-09-23T23:17:30.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.208993+0000", 2023-09-23T23:17:30.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012544384 2023-09-23T23:17:30.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.465 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.465 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.209032+0000", 2023-09-23T23:17:30.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8311e-05 2023-09-23T23:17:30.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.465 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:17:30.465 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.209075+0000", 2023-09-23T23:17:30.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3488000000000001e-05 2023-09-23T23:17:30.466 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.466 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.466 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.466 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.209111+0000", 2023-09-23T23:17:30.466 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6041000000000003e-05 2023-09-23T23:17:30.466 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.466 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.466 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.466 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.466 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.467 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:196620 8.b 8:d8eb6342:::benchmark_data_smithi110_258128_object77802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T23:17:30.467 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:15.175304+0000", 2023-09-23T23:17:30.467 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 375.25343725599998, 2023-09-23T23:17:30.467 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0339445359999999, 2023-09-23T23:17:30.467 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.467 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:30.467 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.467 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.467 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.467 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196620 2023-09-23T23:17:30.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.468 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.468 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.468 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.175304+0000", 2023-09-23T23:17:30.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.468 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.468 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.468 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.175304+0000", 2023-09-23T23:17:30.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.468 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.469 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.469 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.469 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.175303+0000", 2023-09-23T23:17:30.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:17:30.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.469 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.469 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.469 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.175309+0000", 2023-09-23T23:17:30.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7239999999999997e-06 2023-09-23T23:17:30.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.175320+0000", 2023-09-23T23:17:30.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1054e-05 2023-09-23T23:17:30.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.175328+0000", 2023-09-23T23:17:30.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0420000000000003e-06 2023-09-23T23:17:30.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.175441+0000", 2023-09-23T23:17:30.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000113137 2023-09-23T23:17:30.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.175451+0000", 2023-09-23T23:17:30.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6500000000000008e-06 2023-09-23T23:17:30.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.192020+0000", 2023-09-23T23:17:30.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.016568505 2023-09-23T23:17:30.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:30.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.192119+0000", 2023-09-23T23:17:30.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9807000000000002e-05 2023-09-23T23:17:30.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:17:30.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.192189+0000", 2023-09-23T23:17:30.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9561999999999996e-05 2023-09-23T23:17:30.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:17:30.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.199310+0000", 2023-09-23T23:17:30.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071212589999999996 2023-09-23T23:17:30.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.209148+0000", 2023-09-23T23:17:30.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0098378470000000003 2023-09-23T23:17:30.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.209177+0000", 2023-09-23T23:17:30.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8493999999999999e-05 2023-09-23T23:17:30.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:17:30.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.209217+0000", 2023-09-23T23:17:30.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0014e-05 2023-09-23T23:17:30.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.209249+0000", 2023-09-23T23:17:30.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2378000000000001e-05 2023-09-23T23:17:30.475 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.475 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.475 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.476 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:196640 8.b 8:dffb3e86:::benchmark_data_smithi110_258128_object77822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T23:17:30.476 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:15.205642+0000", 2023-09-23T23:17:30.476 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 375.22309970800001, 2023-09-23T23:17:30.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0126917559999999, 2023-09-23T23:17:30.476 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.476 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:30.476 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.476 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.477 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.477 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196640 2023-09-23T23:17:30.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.477 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.477 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.477 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.477 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.205642+0000", 2023-09-23T23:17:30.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.478 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.478 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.205642+0000", 2023-09-23T23:17:30.478 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.478 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.478 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.205640+0000", 2023-09-23T23:17:30.478 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:17:30.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.205650+0000", 2023-09-23T23:17:30.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2380000000000003e-06 2023-09-23T23:17:30.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.205662+0000", 2023-09-23T23:17:30.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2176e-05 2023-09-23T23:17:30.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.205670+0000", 2023-09-23T23:17:30.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2050000000000002e-06 2023-09-23T23:17:30.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.205739+0000", 2023-09-23T23:17:30.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9187e-05 2023-09-23T23:17:30.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.205754+0000", 2023-09-23T23:17:30.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4527e-05 2023-09-23T23:17:30.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.194147+0000", 2023-09-23T23:17:30.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98839315299999997 2023-09-23T23:17:30.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:30.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.194234+0000", 2023-09-23T23:17:30.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6724999999999994e-05 2023-09-23T23:17:30.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:17:30.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.194298+0000", 2023-09-23T23:17:30.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4942000000000003e-05 2023-09-23T23:17:30.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:17:30.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.202545+0000", 2023-09-23T23:17:30.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0082460769999999992 2023-09-23T23:17:30.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.209283+0000", 2023-09-23T23:17:30.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067389090000000004 2023-09-23T23:17:30.484 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.484 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.484 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.218247+0000", 2023-09-23T23:17:30.484 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0089633339999999999 2023-09-23T23:17:30.484 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.484 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:17:30.484 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.218295+0000", 2023-09-23T23:17:30.484 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8201999999999997e-05 2023-09-23T23:17:30.485 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.485 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.485 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.485 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.218334+0000", 2023-09-23T23:17:30.485 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8671000000000003e-05 2023-09-23T23:17:30.485 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.485 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.485 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.485 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.486 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:196655 8.b 8:d69484ec:::benchmark_data_smithi110_258128_object77837:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T23:17:30.486 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:15.225278+0000", 2023-09-23T23:17:30.486 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 375.203463414, 2023-09-23T23:17:30.486 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99317870399999997, 2023-09-23T23:17:30.486 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.486 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:30.486 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.486 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.487 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.487 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196655 2023-09-23T23:17:30.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.487 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.225278+0000", 2023-09-23T23:17:30.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.225278+0000", 2023-09-23T23:17:30.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.225276+0000", 2023-09-23T23:17:30.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:17:30.489 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.489 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.489 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.225284+0000", 2023-09-23T23:17:30.489 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9729999999999997e-06 2023-09-23T23:17:30.489 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.489 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.489 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.225302+0000", 2023-09-23T23:17:30.489 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7711e-05 2023-09-23T23:17:30.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.490 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.490 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.490 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.225311+0000", 2023-09-23T23:17:30.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9209999999999998e-06 2023-09-23T23:17:30.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.490 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.490 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.490 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.225375+0000", 2023-09-23T23:17:30.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3626000000000004e-05 2023-09-23T23:17:30.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.225388+0000", 2023-09-23T23:17:30.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3077e-05 2023-09-23T23:17:30.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.196496+0000", 2023-09-23T23:17:30.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97110877699999998 2023-09-23T23:17:30.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.492 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:30.492 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.196649+0000", 2023-09-23T23:17:30.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015254600000000001 2023-09-23T23:17:30.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.492 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:17:30.492 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.196772+0000", 2023-09-23T23:17:30.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000123416 2023-09-23T23:17:30.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.493 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:17:30.493 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.202551+0000", 2023-09-23T23:17:30.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0057781459999999996 2023-09-23T23:17:30.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.493 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.494 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.209311+0000", 2023-09-23T23:17:30.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067608210000000002 2023-09-23T23:17:30.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.494 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.494 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.218397+0000", 2023-09-23T23:17:30.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0090853559999999993 2023-09-23T23:17:30.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.494 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:17:30.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.218429+0000", 2023-09-23T23:17:30.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1792000000000003e-05 2023-09-23T23:17:30.495 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.495 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.218457+0000", 2023-09-23T23:17:30.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8410000000000001e-05 2023-09-23T23:17:30.496 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.496 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.496 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.496 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:196665 8.b 8:d4e0e962:::benchmark_data_smithi110_258128_object77847:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T23:17:30.496 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:15.238482+0000", 2023-09-23T23:17:30.496 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 375.19025913199999, 2023-09-23T23:17:30.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.986870629, 2023-09-23T23:17:30.496 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.497 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:30.497 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.497 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.497 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.497 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196665 2023-09-23T23:17:30.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.497 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.238482+0000", 2023-09-23T23:17:30.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.498 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.498 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.498 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.498 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.238482+0000", 2023-09-23T23:17:30.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.498 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.498 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.498 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.498 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.238481+0000", 2023-09-23T23:17:30.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:17:30.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.499 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.238488+0000", 2023-09-23T23:17:30.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4399999999999999e-06 2023-09-23T23:17:30.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.499 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.238505+0000", 2023-09-23T23:17:30.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6535999999999999e-05 2023-09-23T23:17:30.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.238514+0000", 2023-09-23T23:17:30.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6200000000000005e-06 2023-09-23T23:17:30.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.242800+0000", 2023-09-23T23:17:30.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042864089999999997 2023-09-23T23:17:30.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:15.242811+0000", 2023-09-23T23:17:30.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1112e-05 2023-09-23T23:17:30.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.199479+0000", 2023-09-23T23:17:30.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95666757499999999 2023-09-23T23:17:30.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.502 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:30.502 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.199609+0000", 2023-09-23T23:17:30.502 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013049399999999999 2023-09-23T23:17:30.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.502 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:17:30.502 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.199726+0000", 2023-09-23T23:17:30.502 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117081 2023-09-23T23:17:30.503 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.503 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:17:30.503 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.208951+0000", 2023-09-23T23:17:30.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0092247319999999994 2023-09-23T23:17:30.503 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.503 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.503 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.209342+0000", 2023-09-23T23:17:30.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039065500000000001 2023-09-23T23:17:30.504 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.504 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.504 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.504 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.225274+0000", 2023-09-23T23:17:30.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015932153000000001 2023-09-23T23:17:30.504 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.504 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.504 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:17:30.504 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.225317+0000", 2023-09-23T23:17:30.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2762000000000002e-05 2023-09-23T23:17:30.504 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.505 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.505 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.505 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:16.225353+0000", 2023-09-23T23:17:30.505 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6539999999999999e-05 2023-09-23T23:17:30.505 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.505 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.505 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.505 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.505 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.505 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:218450 8.0 8:0398ec19:::benchmark_data_smithi110_258128_object99632:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T23:17:30.506 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:45.173862+0000", 2023-09-23T23:17:30.506 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 345.254879571, 2023-09-23T23:17:30.506 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.033071034, 2023-09-23T23:17:30.506 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.506 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:17:30.506 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.506 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.506 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.506 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 218450 2023-09-23T23:17:30.506 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.507 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.507 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.507 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.173862+0000", 2023-09-23T23:17:30.507 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.507 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.507 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.173862+0000", 2023-09-23T23:17:30.507 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.173861+0000", 2023-09-23T23:17:30.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:17:30.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.173869+0000", 2023-09-23T23:17:30.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0949999999999996e-06 2023-09-23T23:17:30.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.509 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.509 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.173882+0000", 2023-09-23T23:17:30.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3444999999999999e-05 2023-09-23T23:17:30.509 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.509 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.509 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.173891+0000", 2023-09-23T23:17:30.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1009999999999998e-06 2023-09-23T23:17:30.509 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.510 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.510 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.173934+0000", 2023-09-23T23:17:30.510 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2392999999999999e-05 2023-09-23T23:17:30.510 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.510 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.510 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.510 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.173940+0000", 2023-09-23T23:17:30.510 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2559999999999999e-06 2023-09-23T23:17:30.510 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.510 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.511 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.511 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.206863+0000", 2023-09-23T23:17:30.511 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.032923246 2023-09-23T23:17:30.511 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.511 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.511 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.511 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.206933+0000", 2023-09-23T23:17:30.511 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0048000000000004e-05 2023-09-23T23:17:30.511 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.511 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.511 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.512 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.512 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.512 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:218481 8.0 8:0f1c8feb:::benchmark_data_smithi110_258128_object99663:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T23:17:30.512 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:45.193718+0000", 2023-09-23T23:17:30.512 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 345.23502322000002, 2023-09-23T23:17:30.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0292697770000001, 2023-09-23T23:17:30.512 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.512 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:30.512 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.512 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.512 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.513 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 218481 2023-09-23T23:17:30.513 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.513 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.513 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.513 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.513 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.193718+0000", 2023-09-23T23:17:30.513 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.513 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.513 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.513 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.514 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.193718+0000", 2023-09-23T23:17:30.514 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.514 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.514 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.514 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.514 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.193717+0000", 2023-09-23T23:17:30.514 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:17:30.514 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.514 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.514 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.515 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.193724+0000", 2023-09-23T23:17:30.515 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7739999999999999e-06 2023-09-23T23:17:30.515 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.515 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.515 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.515 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.193736+0000", 2023-09-23T23:17:30.515 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2497000000000001e-05 2023-09-23T23:17:30.515 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.515 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.515 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.515 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.193745+0000", 2023-09-23T23:17:30.516 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3890000000000005e-06 2023-09-23T23:17:30.516 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.516 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.516 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.516 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.197105+0000", 2023-09-23T23:17:30.516 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033608739999999998 2023-09-23T23:17:30.516 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.516 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.516 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.516 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.197117+0000", 2023-09-23T23:17:30.517 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1421e-05 2023-09-23T23:17:30.517 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.517 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.517 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.517 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.207308+0000", 2023-09-23T23:17:30.517 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0101911320000001 2023-09-23T23:17:30.517 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.517 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.517 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:30.517 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.207409+0000", 2023-09-23T23:17:30.517 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000101345 2023-09-23T23:17:30.518 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.518 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.518 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-09-23T23:17:30.518 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.207499+0000", 2023-09-23T23:17:30.518 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9486000000000003e-05 2023-09-23T23:17:30.518 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.518 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.518 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.518 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.222750+0000", 2023-09-23T23:17:30.519 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015250873 2023-09-23T23:17:30.519 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.519 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.519 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:17:30.519 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.222831+0000", 2023-09-23T23:17:30.519 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1211999999999994e-05 2023-09-23T23:17:30.519 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.519 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.519 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.519 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.222909+0000", 2023-09-23T23:17:30.520 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8434000000000002e-05 2023-09-23T23:17:30.520 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.520 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.520 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:17:30.520 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.222953+0000", 2023-09-23T23:17:30.520 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3872000000000003e-05 2023-09-23T23:17:30.520 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.520 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.520 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.521 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.222988+0000", 2023-09-23T23:17:30.521 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4938999999999999e-05 2023-09-23T23:17:30.521 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.521 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.521 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.521 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.521 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.521 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:218510 8.0 8:0b58be7b:::benchmark_data_smithi110_258128_object99692:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T23:17:30.521 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:45.234661+0000", 2023-09-23T23:17:30.521 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 345.19408055500003, 2023-09-23T23:17:30.522 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.988485854, 2023-09-23T23:17:30.522 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.522 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:30.522 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.522 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.522 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.522 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 218510 2023-09-23T23:17:30.522 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.522 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.522 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.522 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.523 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.234661+0000", 2023-09-23T23:17:30.523 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.523 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.523 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.234661+0000", 2023-09-23T23:17:30.523 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.523 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.234660+0000", 2023-09-23T23:17:30.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:17:30.524 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.234668+0000", 2023-09-23T23:17:30.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0579999999999998e-06 2023-09-23T23:17:30.524 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.525 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.234683+0000", 2023-09-23T23:17:30.525 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5478e-05 2023-09-23T23:17:30.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.525 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.525 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.234692+0000", 2023-09-23T23:17:30.525 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.081e-06 2023-09-23T23:17:30.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.525 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.526 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.234759+0000", 2023-09-23T23:17:30.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6680999999999994e-05 2023-09-23T23:17:30.526 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.526 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.526 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.526 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.234771+0000", 2023-09-23T23:17:30.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.207e-05 2023-09-23T23:17:30.526 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.526 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.526 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.527 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.209725+0000", 2023-09-23T23:17:30.527 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97495447499999999 2023-09-23T23:17:30.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.527 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.527 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:30.527 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.209820+0000", 2023-09-23T23:17:30.527 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4957999999999994e-05 2023-09-23T23:17:30.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.527 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.527 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-09-23T23:17:30.528 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.209890+0000", 2023-09-23T23:17:30.528 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9152999999999993e-05 2023-09-23T23:17:30.528 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.528 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.528 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.222783+0000", 2023-09-23T23:17:30.528 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012893457000000001 2023-09-23T23:17:30.528 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.528 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:17:30.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.223004+0000", 2023-09-23T23:17:30.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022087100000000001 2023-09-23T23:17:30.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.529 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.223065+0000", 2023-09-23T23:17:30.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0696999999999998e-05 2023-09-23T23:17:30.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.529 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:17:30.530 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.223110+0000", 2023-09-23T23:17:30.530 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5670999999999997e-05 2023-09-23T23:17:30.530 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.530 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.530 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.530 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.223147+0000", 2023-09-23T23:17:30.530 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6720000000000001e-05 2023-09-23T23:17:30.530 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.530 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.530 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.530 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.531 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:218522 8.0 8:00f9effd:::benchmark_data_smithi110_258128_object99704:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T23:17:30.531 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:45.248790+0000", 2023-09-23T23:17:30.531 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 345.17995159999998, 2023-09-23T23:17:30.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.974496472, 2023-09-23T23:17:30.531 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.531 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:30.531 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.531 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.531 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.531 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 218522 2023-09-23T23:17:30.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.532 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.248790+0000", 2023-09-23T23:17:30.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.533 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.248790+0000", 2023-09-23T23:17:30.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.533 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.533 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.248789+0000", 2023-09-23T23:17:30.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:17:30.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.533 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.534 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.248795+0000", 2023-09-23T23:17:30.534 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2589999999999997e-06 2023-09-23T23:17:30.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.534 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.248808+0000", 2023-09-23T23:17:30.534 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3125000000000001e-05 2023-09-23T23:17:30.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.248817+0000", 2023-09-23T23:17:30.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6750000000000008e-06 2023-09-23T23:17:30.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.248879+0000", 2023-09-23T23:17:30.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2014000000000006e-05 2023-09-23T23:17:30.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:45.248893+0000", 2023-09-23T23:17:30.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4049999999999999e-05 2023-09-23T23:17:30.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.212561+0000", 2023-09-23T23:17:30.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96366821199999997 2023-09-23T23:17:30.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:30.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.212689+0000", 2023-09-23T23:17:30.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012816300000000001 2023-09-23T23:17:30.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-09-23T23:17:30.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.212810+0000", 2023-09-23T23:17:30.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012073700000000001 2023-09-23T23:17:30.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.222866+0000", 2023-09-23T23:17:30.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010056559 2023-09-23T23:17:30.538 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.538 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:17:30.538 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.223009+0000", 2023-09-23T23:17:30.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014288500000000001 2023-09-23T23:17:30.538 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.538 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.538 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.223204+0000", 2023-09-23T23:17:30.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019503500000000001 2023-09-23T23:17:30.539 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.539 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:17:30.539 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.223247+0000", 2023-09-23T23:17:30.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2901999999999997e-05 2023-09-23T23:17:30.539 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.539 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.539 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:46.223287+0000", 2023-09-23T23:17:30.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9317000000000003e-05 2023-09-23T23:17:30.540 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.540 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.540 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.540 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.540 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.540 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:224391 8.f 8:f8666af0:::benchmark_data_smithi110_258128_object105573:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T23:17:30.540 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:53.144810+0000", 2023-09-23T23:17:30.540 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 337.28393150800002, 2023-09-23T23:17:30.540 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0748160179999999, 2023-09-23T23:17:30.540 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.541 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:17:30.541 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.541 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.541 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.541 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 224391 2023-09-23T23:17:30.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.541 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.541 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.541 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.144810+0000", 2023-09-23T23:17:30.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.144810+0000", 2023-09-23T23:17:30.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.144810+0000", 2023-09-23T23:17:30.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:17:30.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.144812+0000", 2023-09-23T23:17:30.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.757e-06 2023-09-23T23:17:30.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.144823+0000", 2023-09-23T23:17:30.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0618000000000001e-05 2023-09-23T23:17:30.544 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.544 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.544 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.144829+0000", 2023-09-23T23:17:30.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6130000000000003e-06 2023-09-23T23:17:30.544 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.544 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.544 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.145750+0000", 2023-09-23T23:17:30.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00092100000000000005 2023-09-23T23:17:30.545 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op must wait for map", 2023-09-23T23:17:30.545 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.145757+0000", 2023-09-23T23:17:30.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.447e-06 2023-09-23T23:17:30.545 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.546 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.546 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.546 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.154209+0000", 2023-09-23T23:17:30.546 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084515219999999995 2023-09-23T23:17:30.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.546 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.546 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.546 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.154218+0000", 2023-09-23T23:17:30.546 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.884e-06 2023-09-23T23:17:30.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.547 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.547 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.219514+0000", 2023-09-23T23:17:30.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.065296808 2023-09-23T23:17:30.547 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.547 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.547 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.219626+0000", 2023-09-23T23:17:30.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011182400000000001 2023-09-23T23:17:30.547 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.547 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.548 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.548 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.548 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.548 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:224393 8.f 8:fb15a681:::benchmark_data_smithi110_258128_object105575:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T23:17:30.548 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:53.144834+0000", 2023-09-23T23:17:30.548 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 337.283907828, 2023-09-23T23:17:30.548 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.089953307, 2023-09-23T23:17:30.548 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.548 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:17:30.548 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.549 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.549 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.549 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 224393 2023-09-23T23:17:30.549 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.549 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.549 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.549 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.549 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.144834+0000", 2023-09-23T23:17:30.549 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.549 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.550 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.144834+0000", 2023-09-23T23:17:30.550 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.550 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.550 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.144833+0000", 2023-09-23T23:17:30.550 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:17:30.550 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.144836+0000", 2023-09-23T23:17:30.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.638e-06 2023-09-23T23:17:30.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.144842+0000", 2023-09-23T23:17:30.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7409999999999998e-06 2023-09-23T23:17:30.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.144845+0000", 2023-09-23T23:17:30.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.965e-06 2023-09-23T23:17:30.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.145804+0000", 2023-09-23T23:17:30.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00095892099999999995 2023-09-23T23:17:30.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.553 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:17:30.553 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.145809+0000", 2023-09-23T23:17:30.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5500000000000002e-06 2023-09-23T23:17:30.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.553 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.553 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.154238+0000", 2023-09-23T23:17:30.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084283959999999995 2023-09-23T23:17:30.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.554 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.554 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.554 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.154245+0000", 2023-09-23T23:17:30.554 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0480000000000001e-06 2023-09-23T23:17:30.554 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.554 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.554 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.554 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.219891+0000", 2023-09-23T23:17:30.554 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.06564598 2023-09-23T23:17:30.554 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.555 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:17:30.555 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.219976+0000", 2023-09-23T23:17:30.555 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5667999999999994e-05 2023-09-23T23:17:30.555 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.555 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.555 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.234706+0000", 2023-09-23T23:17:30.555 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014729358 2023-09-23T23:17:30.555 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.556 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.234787+0000", 2023-09-23T23:17:30.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1509999999999997e-05 2023-09-23T23:17:30.556 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.556 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.556 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.556 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.556 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:224430 8.f 8:ff5e0772:::benchmark_data_smithi110_258128_object105612:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T23:17:30.556 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:53.171531+0000", 2023-09-23T23:17:30.557 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 337.25721050999999, 2023-09-23T23:17:30.557 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.062844669, 2023-09-23T23:17:30.557 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.557 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:30.557 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.557 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.557 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.557 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 224430 2023-09-23T23:17:30.557 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.557 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.557 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.558 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.558 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.171531+0000", 2023-09-23T23:17:30.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.558 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.558 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.558 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.171531+0000", 2023-09-23T23:17:30.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.558 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.171530+0000", 2023-09-23T23:17:30.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:17:30.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.171540+0000", 2023-09-23T23:17:30.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4949999999999997e-06 2023-09-23T23:17:30.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.560 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.171550+0000", 2023-09-23T23:17:30.560 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7340000000000003e-06 2023-09-23T23:17:30.560 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.560 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.560 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.560 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.171556+0000", 2023-09-23T23:17:30.560 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7050000000000002e-06 2023-09-23T23:17:30.560 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.560 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.560 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.561 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.171614+0000", 2023-09-23T23:17:30.561 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7788999999999998e-05 2023-09-23T23:17:30.561 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.561 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.561 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.561 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.171623+0000", 2023-09-23T23:17:30.561 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8349999999999993e-06 2023-09-23T23:17:30.561 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.561 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.561 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.220038+0000", 2023-09-23T23:17:30.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.048414884 2023-09-23T23:17:30.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.562 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:30.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.220147+0000", 2023-09-23T23:17:30.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109417 2023-09-23T23:17:30.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.562 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:17:30.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.220235+0000", 2023-09-23T23:17:30.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8162000000000001e-05 2023-09-23T23:17:30.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:17:30.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.228062+0000", 2023-09-23T23:17:30.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078268590000000002 2023-09-23T23:17:30.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.234157+0000", 2023-09-23T23:17:30.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060951909999999998 2023-09-23T23:17:30.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.234224+0000", 2023-09-23T23:17:30.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6719999999999998e-05 2023-09-23T23:17:30.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:17:30.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.234278+0000", 2023-09-23T23:17:30.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.401e-05 2023-09-23T23:17:30.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.565 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.565 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.234376+0000", 2023-09-23T23:17:30.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7612e-05 2023-09-23T23:17:30.565 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.565 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.565 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.566 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:224444 8.f 8:fffda5f4:::benchmark_data_smithi110_258128_object105626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T23:17:30.566 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:53.194940+0000", 2023-09-23T23:17:30.566 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 337.23380198400002, 2023-09-23T23:17:30.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0400248409999999, 2023-09-23T23:17:30.566 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.566 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:30.566 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.566 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.566 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.566 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 224444 2023-09-23T23:17:30.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.567 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.567 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.567 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.194940+0000", 2023-09-23T23:17:30.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.567 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.194940+0000", 2023-09-23T23:17:30.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.194938+0000", 2023-09-23T23:17:30.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:17:30.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.570 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.194945+0000", 2023-09-23T23:17:30.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3999999999999997e-06 2023-09-23T23:17:30.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.570 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.194958+0000", 2023-09-23T23:17:30.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3361e-05 2023-09-23T23:17:30.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.194964+0000", 2023-09-23T23:17:30.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1310000000000003e-06 2023-09-23T23:17:30.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.195014+0000", 2023-09-23T23:17:30.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9877999999999999e-05 2023-09-23T23:17:30.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.195027+0000", 2023-09-23T23:17:30.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3145e-05 2023-09-23T23:17:30.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.223309+0000", 2023-09-23T23:17:30.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0282815970000001 2023-09-23T23:17:30.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:30.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.223446+0000", 2023-09-23T23:17:30.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013690499999999999 2023-09-23T23:17:30.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:17:30.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.223589+0000", 2023-09-23T23:17:30.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000143299 2023-09-23T23:17:30.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:17:30.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.234090+0000", 2023-09-23T23:17:30.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010501056 2023-09-23T23:17:30.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.234825+0000", 2023-09-23T23:17:30.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00073435199999999996 2023-09-23T23:17:30.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.234855+0000", 2023-09-23T23:17:30.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0870000000000001e-05 2023-09-23T23:17:30.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.575 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:17:30.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.234906+0000", 2023-09-23T23:17:30.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1087000000000001e-05 2023-09-23T23:17:30.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.575 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.576 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.234964+0000", 2023-09-23T23:17:30.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7998000000000003e-05 2023-09-23T23:17:30.576 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.576 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.576 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.576 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.576 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:224450 8.f 8:fa18f54b:::benchmark_data_smithi110_258128_object105632:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T23:17:30.576 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:53.204692+0000", 2023-09-23T23:17:30.576 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 337.22405012399997, 2023-09-23T23:17:30.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0447475589999999, 2023-09-23T23:17:30.577 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.577 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:30.577 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.577 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.577 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.577 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 224450 2023-09-23T23:17:30.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.577 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.204692+0000", 2023-09-23T23:17:30.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.204692+0000", 2023-09-23T23:17:30.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.204690+0000", 2023-09-23T23:17:30.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:17:30.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.204697+0000", 2023-09-23T23:17:30.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2269999999999998e-06 2023-09-23T23:17:30.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.204709+0000", 2023-09-23T23:17:30.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2573e-05 2023-09-23T23:17:30.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.204717+0000", 2023-09-23T23:17:30.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7559999999999999e-06 2023-09-23T23:17:30.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.204775+0000", 2023-09-23T23:17:30.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8313999999999998e-05 2023-09-23T23:17:30.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.204783+0000", 2023-09-23T23:17:30.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9459999999999998e-06 2023-09-23T23:17:30.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.225806+0000", 2023-09-23T23:17:30.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0210233200000001 2023-09-23T23:17:30.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:30.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.225939+0000", 2023-09-23T23:17:30.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000132659 2023-09-23T23:17:30.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:17:30.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.226060+0000", 2023-09-23T23:17:30.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000121247 2023-09-23T23:17:30.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:17:30.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.234393+0000", 2023-09-23T23:17:30.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083325859999999995 2023-09-23T23:17:30.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.235006+0000", 2023-09-23T23:17:30.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00061269899999999997 2023-09-23T23:17:30.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.249284+0000", 2023-09-23T23:17:30.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014278236 2023-09-23T23:17:30.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:17:30.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.249340+0000", 2023-09-23T23:17:30.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6373999999999999e-05 2023-09-23T23:17:30.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.585 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.585 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.249439+0000", 2023-09-23T23:17:30.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8801999999999994e-05 2023-09-23T23:17:30.585 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.585 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.585 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.585 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:224452 8.f 8:f28fc460:::benchmark_data_smithi110_258128_object105634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T23:17:30.585 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:53.204821+0000", 2023-09-23T23:17:30.585 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 337.22392049899997, 2023-09-23T23:17:30.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0593580490000001, 2023-09-23T23:17:30.586 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.586 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:30.586 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.586 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.586 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.586 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 224452 2023-09-23T23:17:30.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.586 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.204821+0000", 2023-09-23T23:17:30.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.204821+0000", 2023-09-23T23:17:30.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.204820+0000", 2023-09-23T23:17:30.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:17:30.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.204825+0000", 2023-09-23T23:17:30.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3200000000000001e-06 2023-09-23T23:17:30.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.204832+0000", 2023-09-23T23:17:30.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6890000000000004e-06 2023-09-23T23:17:30.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.204839+0000", 2023-09-23T23:17:30.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4019999999999999e-06 2023-09-23T23:17:30.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.205018+0000", 2023-09-23T23:17:30.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017965900000000001 2023-09-23T23:17:30.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.205029+0000", 2023-09-23T23:17:30.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0793e-05 2023-09-23T23:17:30.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.228112+0000", 2023-09-23T23:17:30.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0230827629999999 2023-09-23T23:17:30.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:30.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.228247+0000", 2023-09-23T23:17:30.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000134821 2023-09-23T23:17:30.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:17:30.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.228383+0000", 2023-09-23T23:17:30.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000136014 2023-09-23T23:17:30.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:17:30.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.234401+0000", 2023-09-23T23:17:30.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060178330000000002 2023-09-23T23:17:30.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.249480+0000", 2023-09-23T23:17:30.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01507932 2023-09-23T23:17:30.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.264054+0000", 2023-09-23T23:17:30.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014574214 2023-09-23T23:17:30.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:17:30.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.264089+0000", 2023-09-23T23:17:30.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4480999999999997e-05 2023-09-23T23:17:30.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.264179+0000", 2023-09-23T23:17:30.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0511000000000001e-05 2023-09-23T23:17:30.594 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.594 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.594 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.595 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:224470 8.f 8:fac719e1:::benchmark_data_smithi110_258128_object105652:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T23:17:30.595 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:53.225843+0000", 2023-09-23T23:17:30.595 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 337.20289828599999, 2023-09-23T23:17:30.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.038760616, 2023-09-23T23:17:30.595 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.595 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:30.595 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.595 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.596 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.596 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 224470 2023-09-23T23:17:30.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.596 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.225843+0000", 2023-09-23T23:17:30.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.225843+0000", 2023-09-23T23:17:30.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.225842+0000", 2023-09-23T23:17:30.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:17:30.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.225850+0000", 2023-09-23T23:17:30.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1079999999999993e-06 2023-09-23T23:17:30.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.225867+0000", 2023-09-23T23:17:30.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7173e-05 2023-09-23T23:17:30.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.225876+0000", 2023-09-23T23:17:30.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3160000000000003e-06 2023-09-23T23:17:30.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.228113+0000", 2023-09-23T23:17:30.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022368269999999998 2023-09-23T23:17:30.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.228126+0000", 2023-09-23T23:17:30.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3067999999999999e-05 2023-09-23T23:17:30.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.231323+0000", 2023-09-23T23:17:30.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.003196945 2023-09-23T23:17:30.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:30.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.231461+0000", 2023-09-23T23:17:30.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013737999999999999 2023-09-23T23:17:30.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:17:30.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.231597+0000", 2023-09-23T23:17:30.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000136277 2023-09-23T23:17:30.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:17:30.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.243394+0000", 2023-09-23T23:17:30.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011797636 2023-09-23T23:17:30.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.249517+0000", 2023-09-23T23:17:30.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061227749999999996 2023-09-23T23:17:30.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.264527+0000", 2023-09-23T23:17:30.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015009542000000001 2023-09-23T23:17:30.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:17:30.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.264576+0000", 2023-09-23T23:17:30.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9131000000000001e-05 2023-09-23T23:17:30.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.264604+0000", 2023-09-23T23:17:30.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8036999999999999e-05 2023-09-23T23:17:30.603 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.603 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.604 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.604 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.44642.0:224480 8.f 8:f6864e7f:::benchmark_data_smithi110_258128_object105662:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T23:17:30.604 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:11:53.241369+0000", 2023-09-23T23:17:30.604 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 337.18737232400002, 2023-09-23T23:17:30.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0233701719999999, 2023-09-23T23:17:30.604 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.604 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:30.604 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.605 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.44642", 2023-09-23T23:17:30.605 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:30.605 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 224480 2023-09-23T23:17:30.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.605 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.241369+0000", 2023-09-23T23:17:30.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.241369+0000", 2023-09-23T23:17:30.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.241368+0000", 2023-09-23T23:17:30.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:17:30.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.241374+0000", 2023-09-23T23:17:30.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1720000000000004e-06 2023-09-23T23:17:30.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.241386+0000", 2023-09-23T23:17:30.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2283000000000001e-05 2023-09-23T23:17:30.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.241393+0000", 2023-09-23T23:17:30.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4810000000000003e-06 2023-09-23T23:17:30.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.241445+0000", 2023-09-23T23:17:30.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2244000000000003e-05 2023-09-23T23:17:30.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:53.241457+0000", 2023-09-23T23:17:30.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2077e-05 2023-09-23T23:17:30.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.235032+0000", 2023-09-23T23:17:30.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99357511200000004 2023-09-23T23:17:30.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:17:30.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.235106+0000", 2023-09-23T23:17:30.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3318999999999994e-05 2023-09-23T23:17:30.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:17:30.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.235175+0000", 2023-09-23T23:17:30.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9426999999999998e-05 2023-09-23T23:17:30.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:17:30.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.243467+0000", 2023-09-23T23:17:30.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0082920730000000005 2023-09-23T23:17:30.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.258854+0000", 2023-09-23T23:17:30.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015387002 2023-09-23T23:17:30.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:30.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.264669+0000", 2023-09-23T23:17:30.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058146220000000002 2023-09-23T23:17:30.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:17:30.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.264705+0000", 2023-09-23T23:17:30.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6028999999999997e-05 2023-09-23T23:17:30.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:11:54.264739+0000", 2023-09-23T23:17:30.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4610999999999998e-05 2023-09-23T23:17:30.613 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.613 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.613 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.613 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:165790 9.2 9.7f7f48e2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T23:17:30.613 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:16:43.243805+0000", 2023-09-23T23:17:30.613 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 47.184936397999998, 2023-09-23T23:17:30.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0918856130000001, 2023-09-23T23:17:30.613 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:17:30.613 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:17:30.614 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:17:30.614 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:17:30.614 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:17:30.614 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 165790 2023-09-23T23:17:30.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.614 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:17:30.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:17:30.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:43.243805+0000", 2023-09-23T23:17:30.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:17:30.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:43.243805+0000", 2023-09-23T23:17:30.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:17:30.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:17:30.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:43.243803+0000", 2023-09-23T23:17:30.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:17:30.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:17:30.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:43.243813+0000", 2023-09-23T23:17:30.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.577e-06 2023-09-23T23:17:30.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:17:30.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:43.341916+0000", 2023-09-23T23:17:30.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.098103459000000004 2023-09-23T23:17:30.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:17:30.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:43.341928+0000", 2023-09-23T23:17:30.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1507e-05 2023-09-23T23:17:30.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:43.350708+0000", 2023-09-23T23:17:30.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0087804530000000006 2023-09-23T23:17:30.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:17:30.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:43.350723+0000", 2023-09-23T23:17:30.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5098e-05 2023-09-23T23:17:30.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:17:30.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:44.335677+0000", 2023-09-23T23:17:30.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98495317299999996 2023-09-23T23:17:30.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:17:30.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:17:30.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:17:30.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:16:44.335691+0000", 2023-09-23T23:17:30.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4201e-05 2023-09-23T23:17:30.620 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.620 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.620 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.620 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:17:30.620 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:17:30.620 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:17:30.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:30.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:17:30.722 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:17:30.752 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:30.812 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:17:30.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:30.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:17:30.919 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:17:31.009 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:31.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:31 smithi110 ceph-mon[221102]: pgmap v714: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 92 active+clean, 4 active+remapped+backfill_wait; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 16542/109353 objects misplaced (15.127%); 11 MiB/s, 169 objects/s recovering 2023-09-23T23:17:31.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:31 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:17:31.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:31 smithi110 ceph-mon[212939]: pgmap v714: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 92 active+clean, 4 active+remapped+backfill_wait; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 16542/109353 objects misplaced (15.127%); 11 MiB/s, 169 objects/s recovering 2023-09-23T23:17:31.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:31 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:17:31.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:31 smithi138 ceph-mon[193606]: pgmap v714: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 92 active+clean, 4 active+remapped+backfill_wait; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 16542/109353 objects misplaced (15.127%); 11 MiB/s, 169 objects/s recovering 2023-09-23T23:17:31.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:31 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:17:31.721 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:17:31.724 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:17:31.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:31.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:17:31.744 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:17:31.833 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:17:31.938 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:32.156 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:17:32.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:32.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:17:32.182 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:32 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:17:32.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:32 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:17:32.262 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:17:32.299 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:32.384 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:17:32.389 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:32 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:17:32.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:32.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:17:32.491 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:17:32.523 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:17:32.548 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:32.630 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:17:32.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:32.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:17:32.738 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:17:32.797 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:32.835 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:17:32.926 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:17:32.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:32.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:17:33.029 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:17:33.102 INFO:teuthology.orchestra.run.smithi110.stdout:moved 5 / 297 (1.6835%) 2023-09-23T23:17:33.103 INFO:teuthology.orchestra.run.smithi110.stdout:avg 37.125 2023-09-23T23:17:33.103 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 8.63767 -> 9.08897 (expected baseline 5.69951) 2023-09-23T23:17:33.103 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.7 with 19 -> 19 pgs (0.511784 -> 0.511784 * mean) 2023-09-23T23:17:33.103 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.4 with 47 -> 48 pgs (1.26599 -> 1.29293 * mean) 2023-09-23T23:17:33.103 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:17:33.103 INFO:teuthology.orchestra.run.smithi110.stdout:oload 110 2023-09-23T23:17:33.103 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 0.05 2023-09-23T23:17:33.103 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:17:33.103 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0148 2023-09-23T23:17:33.103 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.0162 2023-09-23T23:17:33.103 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 weight 0.5625 -> 0.5125 2023-09-23T23:17:33.103 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 weight 0.6534 -> 0.6034 2023-09-23T23:17:33.104 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 weight 0.5298 -> 0.4798 2023-09-23T23:17:33.104 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 weight 0.7057 -> 0.6557 2023-09-23T23:17:33.131 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:33.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:33 smithi110 ceph-mon[212939]: pgmap v715: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 15919/107577 objects misplaced (14.798%); 9.7 MiB/s, 154 objects/s recovering 2023-09-23T23:17:33.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:33 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:17:33.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:33 smithi110 ceph-mon[212939]: from='client.35406 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:17:33.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:33 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'39543','3':'42975','5':'31444','7':'33585'}"}]: dispatch 2023-09-23T23:17:33.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:33 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'39543','3':'42975','5':'31444','7':'33585'}"}]: dispatch 2023-09-23T23:17:33.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:33 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:17:33.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:33 smithi110 ceph-mon[221102]: pgmap v715: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 15919/107577 objects misplaced (14.798%); 9.7 MiB/s, 154 objects/s recovering 2023-09-23T23:17:33.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:33 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:17:33.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:33 smithi110 ceph-mon[221102]: from='client.35406 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:17:33.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:33 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'39543','3':'42975','5':'31444','7':'33585'}"}]: dispatch 2023-09-23T23:17:33.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:33 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'39543','3':'42975','5':'31444','7':'33585'}"}]: dispatch 2023-09-23T23:17:33.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:33 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:17:33.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:33 smithi138 ceph-mon[193606]: pgmap v715: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 15919/107577 objects misplaced (14.798%); 9.7 MiB/s, 154 objects/s recovering 2023-09-23T23:17:33.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:33 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:17:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:33 smithi138 ceph-mon[193606]: from='client.35406 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:17:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:33 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'39543','3':'42975','5':'31444','7':'33585'}"}]: dispatch 2023-09-23T23:17:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:33 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'39543','3':'42975','5':'31444','7':'33585'}"}]: dispatch 2023-09-23T23:17:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:33 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:17:33.963 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:17:33.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:33.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:17:34.069 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:34.100 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:34.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'39543','3':'42975','5':'31444','7':'33585'}"}]': finished 2023-09-23T23:17:34.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[212939]: osdmap e226: 8 total, 8 up, 8 in 2023-09-23T23:17:34.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[212939]: failed to encode map e226 with expected crc 2023-09-23T23:17:34.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[212939]: failed to encode map e226 with expected crc 2023-09-23T23:17:34.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[212939]: osdmap e227: 8 total, 8 up, 8 in 2023-09-23T23:17:34.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[212939]: failed to encode map e227 with expected crc 2023-09-23T23:17:34.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[212939]: failed to encode map e227 with expected crc 2023-09-23T23:17:34.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[212939]: pgmap v718: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 443 op/s; 15919/106584 objects misplaced (14.936%); 5.9 MiB/s, 95 objects/s recovering 2023-09-23T23:17:34.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:17:34.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'39543','3':'42975','5':'31444','7':'33585'}"}]': finished 2023-09-23T23:17:34.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[221102]: osdmap e226: 8 total, 8 up, 8 in 2023-09-23T23:17:34.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[221102]: failed to encode map e226 with expected crc 2023-09-23T23:17:34.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[221102]: failed to encode map e226 with expected crc 2023-09-23T23:17:34.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[221102]: osdmap e227: 8 total, 8 up, 8 in 2023-09-23T23:17:34.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[221102]: failed to encode map e227 with expected crc 2023-09-23T23:17:34.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[221102]: failed to encode map e227 with expected crc 2023-09-23T23:17:34.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[221102]: pgmap v718: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 443 op/s; 15919/106584 objects misplaced (14.936%); 5.9 MiB/s, 95 objects/s recovering 2023-09-23T23:17:34.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:34 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:17:34.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:34 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'39543','3':'42975','5':'31444','7':'33585'}"}]': finished 2023-09-23T23:17:34.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:34 smithi138 ceph-mon[193606]: osdmap e226: 8 total, 8 up, 8 in 2023-09-23T23:17:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:34 smithi138 ceph-mon[193606]: failed to encode map e226 with expected crc 2023-09-23T23:17:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:34 smithi138 ceph-mon[193606]: failed to encode map e226 with expected crc 2023-09-23T23:17:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:34 smithi138 ceph-mon[193606]: osdmap e227: 8 total, 8 up, 8 in 2023-09-23T23:17:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:34 smithi138 ceph-mon[193606]: failed to encode map e227 with expected crc 2023-09-23T23:17:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:34 smithi138 ceph-mon[193606]: failed to encode map e227 with expected crc 2023-09-23T23:17:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:34 smithi138 ceph-mon[193606]: pgmap v718: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 92 active+clean, 4 active+remapped+backfill_wait; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 443 op/s; 15919/106584 objects misplaced (14.936%); 5.9 MiB/s, 95 objects/s recovering 2023-09-23T23:17:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:34 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:17:34.829 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:34.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:34.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:34.933 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:34.968 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:35.025 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:35.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:35.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:35.090 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:35 smithi110 ceph-mon[212939]: failed to encode map e227 with expected crc 2023-09-23T23:17:35.090 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:35 smithi110 ceph-mon[212939]: failed to encode map e228 with expected crc 2023-09-23T23:17:35.090 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:35 smithi110 ceph-mon[212939]: osdmap e228: 8 total, 8 up, 8 in 2023-09-23T23:17:35.090 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:35 smithi110 ceph-mon[212939]: failed to encode map e228 with expected crc 2023-09-23T23:17:35.090 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:35 smithi110 ceph-mon[212939]: failed to encode map e228 with expected crc 2023-09-23T23:17:35.091 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:35 smithi110 ceph-mon[212939]: failed to encode map e228 with expected crc 2023-09-23T23:17:35.091 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:35 smithi110 ceph-mon[212939]: 9.e starting backfill to osd.5 from (0'0,0'0] MAX to 225'11842 2023-09-23T23:17:35.091 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:35 smithi110 ceph-mon[221102]: failed to encode map e227 with expected crc 2023-09-23T23:17:35.091 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:35 smithi110 ceph-mon[221102]: failed to encode map e228 with expected crc 2023-09-23T23:17:35.091 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:35 smithi110 ceph-mon[221102]: osdmap e228: 8 total, 8 up, 8 in 2023-09-23T23:17:35.091 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:35 smithi110 ceph-mon[221102]: failed to encode map e228 with expected crc 2023-09-23T23:17:35.091 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:35 smithi110 ceph-mon[221102]: failed to encode map e228 with expected crc 2023-09-23T23:17:35.091 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:35 smithi110 ceph-mon[221102]: failed to encode map e228 with expected crc 2023-09-23T23:17:35.091 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:35 smithi110 ceph-mon[221102]: 9.e starting backfill to osd.5 from (0'0,0'0] MAX to 225'11842 2023-09-23T23:17:35.131 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:17:35.165 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:35.221 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:17:35.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:35.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:17:35.287 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:17:35.324 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:17:35.390 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:35.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:35 smithi138 ceph-mon[193606]: failed to encode map e227 with expected crc 2023-09-23T23:17:35.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:35 smithi138 ceph-mon[193606]: failed to encode map e228 with expected crc 2023-09-23T23:17:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:35 smithi138 ceph-mon[193606]: osdmap e228: 8 total, 8 up, 8 in 2023-09-23T23:17:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:35 smithi138 ceph-mon[193606]: failed to encode map e228 with expected crc 2023-09-23T23:17:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:35 smithi138 ceph-mon[193606]: failed to encode map e228 with expected crc 2023-09-23T23:17:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:35 smithi138 ceph-mon[193606]: failed to encode map e228 with expected crc 2023-09-23T23:17:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:35 smithi138 ceph-mon[193606]: 9.e starting backfill to osd.5 from (0'0,0'0] MAX to 225'11842 2023-09-23T23:17:35.456 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:17:35.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:35.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:17:35.562 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:17:35.590 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:35.637 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:17:35.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:35.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:17:35.740 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:17:35.766 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:35.807 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:17:35.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:35.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:17:35.911 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:17:35.995 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:36.257 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:17:36.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:36.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:17:36.271 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[212939]: failed to encode map e228 with expected crc 2023-09-23T23:17:36.271 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[212939]: failed to encode map e228 with expected crc 2023-09-23T23:17:36.271 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[212939]: failed to encode map e228 with expected crc 2023-09-23T23:17:36.271 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[212939]: osdmap e229: 8 total, 8 up, 8 in 2023-09-23T23:17:36.271 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[212939]: failed to encode map e229 with expected crc 2023-09-23T23:17:36.271 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[212939]: 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 225'11579 2023-09-23T23:17:36.271 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[212939]: pgmap v721: 99 pgs: 1 activating+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 90 active+clean, 3 active+remapped+backfill_wait; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 16651/102018 objects misplaced (16.322%); 13 MiB/s, 215 objects/s recovering 2023-09-23T23:17:36.271 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/240514737' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:17:36.272 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:17:36.272 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:17:36.272 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[221102]: failed to encode map e228 with expected crc 2023-09-23T23:17:36.272 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[221102]: failed to encode map e228 with expected crc 2023-09-23T23:17:36.272 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[221102]: failed to encode map e228 with expected crc 2023-09-23T23:17:36.272 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[221102]: osdmap e229: 8 total, 8 up, 8 in 2023-09-23T23:17:36.272 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[221102]: failed to encode map e229 with expected crc 2023-09-23T23:17:36.272 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[221102]: 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 225'11579 2023-09-23T23:17:36.272 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[221102]: pgmap v721: 99 pgs: 1 activating+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 90 active+clean, 3 active+remapped+backfill_wait; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 16651/102018 objects misplaced (16.322%); 13 MiB/s, 215 objects/s recovering 2023-09-23T23:17:36.272 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/240514737' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:17:36.273 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:17:36.273 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:36 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:17:36.360 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:17:36.369 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:17:36.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:36 smithi138 ceph-mon[193606]: failed to encode map e228 with expected crc 2023-09-23T23:17:36.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:36 smithi138 ceph-mon[193606]: failed to encode map e228 with expected crc 2023-09-23T23:17:36.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:36 smithi138 ceph-mon[193606]: failed to encode map e228 with expected crc 2023-09-23T23:17:36.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:36 smithi138 ceph-mon[193606]: osdmap e229: 8 total, 8 up, 8 in 2023-09-23T23:17:36.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:36 smithi138 ceph-mon[193606]: failed to encode map e229 with expected crc 2023-09-23T23:17:36.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:36 smithi138 ceph-mon[193606]: 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 225'11579 2023-09-23T23:17:36.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:36 smithi138 ceph-mon[193606]: pgmap v721: 99 pgs: 1 activating+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 90 active+clean, 3 active+remapped+backfill_wait; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 16651/102018 objects misplaced (16.322%); 13 MiB/s, 215 objects/s recovering 2023-09-23T23:17:36.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:36 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/240514737' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:17:36.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:36 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:17:36.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:36 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:17:36.459 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:37.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:37 smithi110 ceph-mon[212939]: failed to encode map e229 with expected crc 2023-09-23T23:17:37.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:37 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:17:37.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:37 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:17:37.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:37 smithi110 ceph-mon[212939]: osdmap e230: 8 total, 8 up, 8 in 2023-09-23T23:17:37.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:37 smithi110 ceph-mon[212939]: failed to encode map e230 with expected crc 2023-09-23T23:17:37.108 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:37 smithi110 ceph-mon[212939]: failed to encode map e230 with expected crc 2023-09-23T23:17:37.108 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:17:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:17:36] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.43.0" 2023-09-23T23:17:37.119 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:17:37.119 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:17:37.119 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:37.120 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45164.0:189432 9.4 e230/164 9:2c65906e:::benchmark_data_smithi110_295904_object78251:head v 230'12184, mlcod=230'12181)", 2023-09-23T23:17:37.120 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.116025+0000", 2023-09-23T23:17:37.120 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0022155130000000001, 2023-09-23T23:17:37.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022471320000000002, 2023-09-23T23:17:37.120 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:37.120 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:17:37.120 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:37.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:37.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:37.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.116025+0000", 2023-09-23T23:17:37.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:37.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:37.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:37.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:37.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.116025+0000", 2023-09-23T23:17:37.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:37.122 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:37.122 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:37.122 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:37.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.116024+0000", 2023-09-23T23:17:37.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:17:37.122 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:37.122 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:37.123 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:37.123 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.116036+0000", 2023-09-23T23:17:37.123 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1790999999999999e-05 2023-09-23T23:17:37.123 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:37.123 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:37.123 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:37.123 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.116051+0000", 2023-09-23T23:17:37.123 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4737e-05 2023-09-23T23:17:37.123 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:37.123 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:37.124 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:37.124 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.116057+0000", 2023-09-23T23:17:37.124 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9800000000000003e-06 2023-09-23T23:17:37.124 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:37.124 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:37.124 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:37.124 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.116120+0000", 2023-09-23T23:17:37.124 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3161000000000001e-05 2023-09-23T23:17:37.124 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:37.125 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:37.125 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:37.125 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.116145+0000", 2023-09-23T23:17:37.125 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5299999999999998e-05 2023-09-23T23:17:37.125 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:37.125 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:37.125 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:37.125 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:37.125 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-09-23T23:17:37.125 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 1 2023-09-23T23:17:37.125 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:17:37.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:37 smithi138 ceph-mon[193606]: failed to encode map e229 with expected crc 2023-09-23T23:17:37.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:37 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:17:37.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:37 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:17:37.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:37 smithi138 ceph-mon[193606]: osdmap e230: 8 total, 8 up, 8 in 2023-09-23T23:17:37.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:37 smithi138 ceph-mon[193606]: failed to encode map e230 with expected crc 2023-09-23T23:17:37.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:37 smithi138 ceph-mon[193606]: failed to encode map e230 with expected crc 2023-09-23T23:17:37.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:37 smithi110 ceph-mon[221102]: failed to encode map e229 with expected crc 2023-09-23T23:17:37.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:37 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:17:37.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:37 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:17:37.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:37 smithi110 ceph-mon[221102]: osdmap e230: 8 total, 8 up, 8 in 2023-09-23T23:17:37.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:37 smithi110 ceph-mon[221102]: failed to encode map e230 with expected crc 2023-09-23T23:17:37.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:37 smithi110 ceph-mon[221102]: failed to encode map e230 with expected crc 2023-09-23T23:17:38.006 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:17:38.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:38.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:17:38.111 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:38.141 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:38.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:38 smithi138 ceph-mon[193606]: failed to encode map e230 with expected crc 2023-09-23T23:17:38.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:38 smithi138 ceph-mon[193606]: failed to encode map e230 with expected crc 2023-09-23T23:17:38.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:38 smithi138 ceph-mon[193606]: pgmap v723: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+clean+remapped, 1 remapped+peering, 90 active+clean, 3 active+remapped+backfill_wait; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 16053/101034 objects misplaced (15.889%); 16 MiB/s, 261 objects/s recovering 2023-09-23T23:17:38.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:38 smithi110 ceph-mon[212939]: failed to encode map e230 with expected crc 2023-09-23T23:17:38.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:38 smithi110 ceph-mon[212939]: failed to encode map e230 with expected crc 2023-09-23T23:17:38.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:38 smithi110 ceph-mon[212939]: pgmap v723: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+clean+remapped, 1 remapped+peering, 90 active+clean, 3 active+remapped+backfill_wait; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 16053/101034 objects misplaced (15.889%); 16 MiB/s, 261 objects/s recovering 2023-09-23T23:17:38.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:38 smithi110 ceph-mon[221102]: failed to encode map e230 with expected crc 2023-09-23T23:17:38.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:38 smithi110 ceph-mon[221102]: failed to encode map e230 with expected crc 2023-09-23T23:17:38.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:38 smithi110 ceph-mon[221102]: pgmap v723: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+clean+remapped, 1 remapped+peering, 90 active+clean, 3 active+remapped+backfill_wait; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 16053/101034 objects misplaced (15.889%); 16 MiB/s, 261 objects/s recovering 2023-09-23T23:17:38.704 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:17:38.732 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:38.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:38.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:38.843 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:17:38.905 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:39.068 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:17:39.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:39.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:17:39.173 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:17:39.253 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:39.355 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:17:39.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:39.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:17:39.460 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:17:39.496 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:17:39.541 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:39.544 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:17:39.544 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:17:39.593 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:17:39.642 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:17:39.676 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:17:39.679 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:17:39.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:39.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:17:39.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:39 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1817120361' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:17:39.741 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:17:39.790 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:17:39.791 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:17:39.837 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:17:39.862 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:39 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1817120361' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:17:39.863 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:39 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1817120361' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:17:39.866 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:17:39.866 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:17:39.866 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:17:39.866 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:17:39.867 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:17:39.867 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:17:39.867 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:17:39.867 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:17:39.872 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:39.963 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T23:17:39.963 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:17:39.963 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T23:17:39.963 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd out 2 2023-09-23T23:17:40.198 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:17:40.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:40.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:17:40.319 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:40.383 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:40.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:40 smithi138 ceph-mon[193606]: pgmap v724: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 15009/97386 objects misplaced (15.412%); 40 MiB/s, 637 objects/s recovering 2023-09-23T23:17:40.862 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:17:40.902 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:40 smithi110 ceph-mon[212939]: pgmap v724: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 15009/97386 objects misplaced (15.412%); 40 MiB/s, 637 objects/s recovering 2023-09-23T23:17:40.902 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:40 smithi110 ceph-mon[221102]: pgmap v724: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 15009/97386 objects misplaced (15.412%); 40 MiB/s, 637 objects/s recovering 2023-09-23T23:17:41.104 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:41.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:41.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:41.208 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:41.269 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:41.379 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:41.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:41.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:41.489 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:17:41.526 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:41 smithi110 ceph-mon[212939]: failed to encode map e231 with expected crc 2023-09-23T23:17:41.527 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:41 smithi110 ceph-mon[212939]: osdmap e231: 8 total, 8 up, 8 in 2023-09-23T23:17:41.527 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:41 smithi110 ceph-mon[212939]: failed to encode map e231 with expected crc 2023-09-23T23:17:41.527 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:41 smithi110 ceph-mon[221102]: failed to encode map e231 with expected crc 2023-09-23T23:17:41.527 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:41 smithi110 ceph-mon[221102]: osdmap e231: 8 total, 8 up, 8 in 2023-09-23T23:17:41.528 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:41 smithi110 ceph-mon[221102]: failed to encode map e231 with expected crc 2023-09-23T23:17:41.528 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:41.658 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:17:41.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:41.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:17:41.736 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:17:41.777 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:17:41.812 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:41 smithi138 ceph-mon[193606]: failed to encode map e231 with expected crc 2023-09-23T23:17:41.812 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:41 smithi138 ceph-mon[193606]: osdmap e231: 8 total, 8 up, 8 in 2023-09-23T23:17:41.812 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:41 smithi138 ceph-mon[193606]: failed to encode map e231 with expected crc 2023-09-23T23:17:41.816 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:41.841 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:17:41.869 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:17:41.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:41.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:17:41.973 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:42.083 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:42.269 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:42.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:42.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:42.332 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:17:42.332 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:17:42.380 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:17:42.459 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:42.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:42 smithi138 ceph-mon[193606]: pgmap v726: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 338 op/s; 14491/95793 objects misplaced (15.127%); 32 MiB/s, 505 objects/s recovering 2023-09-23T23:17:42.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:42 smithi138 ceph-mon[193606]: failed to encode map e232 with expected crc 2023-09-23T23:17:42.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:42 smithi138 ceph-mon[193606]: osdmap e232: 8 total, 8 up, 8 in 2023-09-23T23:17:42.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:42 smithi138 ceph-mon[193606]: failed to encode map e232 with expected crc 2023-09-23T23:17:42.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:42 smithi110 ceph-mon[212939]: pgmap v726: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 338 op/s; 14491/95793 objects misplaced (15.127%); 32 MiB/s, 505 objects/s recovering 2023-09-23T23:17:42.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:42 smithi110 ceph-mon[212939]: failed to encode map e232 with expected crc 2023-09-23T23:17:42.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:42 smithi110 ceph-mon[212939]: osdmap e232: 8 total, 8 up, 8 in 2023-09-23T23:17:42.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:42 smithi110 ceph-mon[212939]: failed to encode map e232 with expected crc 2023-09-23T23:17:42.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:42 smithi110 ceph-mon[221102]: pgmap v726: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 338 op/s; 14491/95793 objects misplaced (15.127%); 32 MiB/s, 505 objects/s recovering 2023-09-23T23:17:42.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:42 smithi110 ceph-mon[221102]: failed to encode map e232 with expected crc 2023-09-23T23:17:42.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:42 smithi110 ceph-mon[221102]: osdmap e232: 8 total, 8 up, 8 in 2023-09-23T23:17:42.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:42 smithi110 ceph-mon[221102]: failed to encode map e232 with expected crc 2023-09-23T23:17:43.359 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:17:43.398 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:17:43.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:43 smithi138 ceph-mon[193606]: osdmap e233: 8 total, 8 up, 8 in 2023-09-23T23:17:43.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:43 smithi138 ceph-mon[193606]: failed to encode map e233 with expected crc 2023-09-23T23:17:43.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:43 smithi138 ceph-mon[193606]: failed to encode map e233 with expected crc 2023-09-23T23:17:43.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:43.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:17:43.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:43 smithi110 ceph-mon[212939]: osdmap e233: 8 total, 8 up, 8 in 2023-09-23T23:17:43.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:43 smithi110 ceph-mon[212939]: failed to encode map e233 with expected crc 2023-09-23T23:17:43.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:43 smithi110 ceph-mon[212939]: failed to encode map e233 with expected crc 2023-09-23T23:17:43.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:43.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:17:43.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:43 smithi110 ceph-mon[221102]: osdmap e233: 8 total, 8 up, 8 in 2023-09-23T23:17:43.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:43 smithi110 ceph-mon[221102]: failed to encode map e233 with expected crc 2023-09-23T23:17:43.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:43 smithi110 ceph-mon[221102]: failed to encode map e233 with expected crc 2023-09-23T23:17:43.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:43.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:17:43.870 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:17:43.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:43.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:17:43.903 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:17:43.979 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:44.084 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:44.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:44 smithi138 ceph-mon[193606]: pgmap v729: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 1.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 335 op/s; 14491/95001 objects misplaced (15.254%); 30 MiB/s, 480 objects/s recovering 2023-09-23T23:17:44.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:44 smithi138 ceph-mon[193606]: failed to encode map e234 with expected crc 2023-09-23T23:17:44.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:44 smithi138 ceph-mon[193606]: osdmap e234: 8 total, 8 up, 8 in 2023-09-23T23:17:44.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:44 smithi138 ceph-mon[193606]: failed to encode map e234 with expected crc 2023-09-23T23:17:44.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:44 smithi110 ceph-mon[212939]: pgmap v729: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 1.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 335 op/s; 14491/95001 objects misplaced (15.254%); 30 MiB/s, 480 objects/s recovering 2023-09-23T23:17:44.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:44 smithi110 ceph-mon[212939]: failed to encode map e234 with expected crc 2023-09-23T23:17:44.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:44 smithi110 ceph-mon[212939]: osdmap e234: 8 total, 8 up, 8 in 2023-09-23T23:17:44.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:44 smithi110 ceph-mon[212939]: failed to encode map e234 with expected crc 2023-09-23T23:17:44.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:44 smithi110 ceph-mon[221102]: pgmap v729: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 1.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 335 op/s; 14491/95001 objects misplaced (15.254%); 30 MiB/s, 480 objects/s recovering 2023-09-23T23:17:44.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:44 smithi110 ceph-mon[221102]: failed to encode map e234 with expected crc 2023-09-23T23:17:44.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:44 smithi110 ceph-mon[221102]: osdmap e234: 8 total, 8 up, 8 in 2023-09-23T23:17:44.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:44 smithi110 ceph-mon[221102]: failed to encode map e234 with expected crc 2023-09-23T23:17:45.275 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:45.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:45.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:45.393 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:17:45.497 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:46.199 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:17:46.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:46.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:17:46.309 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:17:46.358 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:46.374 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:17:46.374 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:17:46.441 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:17:46.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:46.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:17:46.549 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:17:46.607 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:46.883 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:17:46.930 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:17:46.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:46.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:17:47.053 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:17:47.062 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:46 smithi110 ceph-mon[212939]: pgmap v731: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 1.9 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 404 op/s; 12999/91434 objects misplaced (14.217%); 15 MiB/s, 244 objects/s recovering 2023-09-23T23:17:47.062 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:46 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:47.062 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:46 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:47.063 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:17:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:17:46] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.43.0" 2023-09-23T23:17:47.063 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:46 smithi110 ceph-mon[221102]: pgmap v731: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 1.9 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 404 op/s; 12999/91434 objects misplaced (14.217%); 15 MiB/s, 244 objects/s recovering 2023-09-23T23:17:47.063 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:46 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:47.063 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:46 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:47.187 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:47.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:46 smithi138 ceph-mon[193606]: pgmap v731: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 1.9 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 404 op/s; 12999/91434 objects misplaced (14.217%); 15 MiB/s, 244 objects/s recovering 2023-09-23T23:17:47.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:46 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:47.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:46 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:47.418 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:17:47.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:47 smithi110 conmon[212914]: 2023-09-23T23:17:47.355+0000 7f29aa464700 -1 mon.a@0(leader).osd e234 definitely_dead 0 2023-09-23T23:17:47.669 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:17:47.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:47.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:17:47.777 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:17:47.856 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:47.889 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:17:47.936 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:47 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2472748594' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T23:17:47.936 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:47 smithi110 ceph-mon[212939]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T23:17:47.936 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:47.936 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:47.936 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:47 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/907343361' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:17:47.936 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:47 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:17:47.936 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:47 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2472748594' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T23:17:47.936 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:47 smithi110 ceph-mon[221102]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T23:17:47.937 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:47 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:47.937 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:47 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:47.937 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:47 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/907343361' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:17:47.937 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:47 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:17:47.969 INFO:teuthology.orchestra.run.smithi110.stderr:marked out osd.2. 2023-09-23T23:17:48.012 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:17:48.013 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:17:48.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:48.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:17:48.127 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:48.147 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:47 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2472748594' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T23:17:48.147 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:47 smithi138 ceph-mon[193606]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T23:17:48.147 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:47 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:48.147 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:47 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:48.148 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:47 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/907343361' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:17:48.148 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:47 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:17:48.181 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:49.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:48 smithi138 ceph-mon[193606]: pgmap v732: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 1.8 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 337 op/s; 12380/89718 objects misplaced (13.799%); 13 MiB/s, 203 objects/s recovering 2023-09-23T23:17:49.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:48 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:17:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:48 smithi138 ceph-mon[193606]: failed to encode map e235 with expected crc 2023-09-23T23:17:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:48 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2472748594' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T23:17:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:48 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:17:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:48 smithi138 ceph-mon[193606]: osdmap e235: 8 total, 8 up, 7 in 2023-09-23T23:17:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:48 smithi138 ceph-mon[193606]: failed to encode map e235 with expected crc 2023-09-23T23:17:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:48 smithi138 ceph-mon[193606]: failed to encode map e236 with expected crc 2023-09-23T23:17:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:48 smithi138 ceph-mon[193606]: osdmap e236: 8 total, 8 up, 7 in 2023-09-23T23:17:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:48 smithi138 ceph-mon[193606]: failed to encode map e236 with expected crc 2023-09-23T23:17:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:48 smithi138 ceph-mon[193606]: failed to encode map e236 with expected crc 2023-09-23T23:17:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:48 smithi138 ceph-mon[193606]: failed to encode map e236 with expected crc 2023-09-23T23:17:49.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[212939]: pgmap v732: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 1.8 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 337 op/s; 12380/89718 objects misplaced (13.799%); 13 MiB/s, 203 objects/s recovering 2023-09-23T23:17:49.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:17:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[212939]: failed to encode map e235 with expected crc 2023-09-23T23:17:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2472748594' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T23:17:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:17:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[212939]: osdmap e235: 8 total, 8 up, 7 in 2023-09-23T23:17:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[212939]: failed to encode map e235 with expected crc 2023-09-23T23:17:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[212939]: failed to encode map e236 with expected crc 2023-09-23T23:17:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[212939]: osdmap e236: 8 total, 8 up, 7 in 2023-09-23T23:17:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[212939]: failed to encode map e236 with expected crc 2023-09-23T23:17:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[212939]: failed to encode map e236 with expected crc 2023-09-23T23:17:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[212939]: failed to encode map e236 with expected crc 2023-09-23T23:17:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[221102]: pgmap v732: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 1.8 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 337 op/s; 12380/89718 objects misplaced (13.799%); 13 MiB/s, 203 objects/s recovering 2023-09-23T23:17:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:17:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[221102]: failed to encode map e235 with expected crc 2023-09-23T23:17:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2472748594' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T23:17:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:17:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[221102]: osdmap e235: 8 total, 8 up, 7 in 2023-09-23T23:17:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[221102]: failed to encode map e235 with expected crc 2023-09-23T23:17:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[221102]: failed to encode map e236 with expected crc 2023-09-23T23:17:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[221102]: osdmap e236: 8 total, 8 up, 7 in 2023-09-23T23:17:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[221102]: failed to encode map e236 with expected crc 2023-09-23T23:17:49.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[221102]: failed to encode map e236 with expected crc 2023-09-23T23:17:49.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:48 smithi110 ceph-mon[221102]: failed to encode map e236 with expected crc 2023-09-23T23:17:49.903 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:49 smithi138 ceph-mon[193606]: failed to encode map e236 with expected crc 2023-09-23T23:17:49.903 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:49 smithi138 ceph-mon[193606]: failed to encode map e236 with expected crc 2023-09-23T23:17:49.904 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:49 smithi138 ceph-mon[193606]: failed to encode map e236 with expected crc 2023-09-23T23:17:49.904 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:49 smithi138 ceph-mon[193606]: 9.7 continuing backfill to osd.1 from (169'1993,234'12043] MIN to 234'12043 2023-09-23T23:17:49.904 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:49 smithi138 ceph-mon[193606]: 9.7 starting backfill to osd.6 from (0'0,0'0] MAX to 234'12043 2023-09-23T23:17:49.904 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:49 smithi138 ceph-mon[193606]: failed to encode map e237 with expected crc 2023-09-23T23:17:49.904 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:49 smithi138 ceph-mon[193606]: osdmap e237: 8 total, 8 up, 7 in 2023-09-23T23:17:49.904 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:49 smithi138 ceph-mon[193606]: failed to encode map e237 with expected crc 2023-09-23T23:17:49.904 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:49 smithi138 ceph-mon[193606]: failed to encode map e237 with expected crc 2023-09-23T23:17:49.904 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:49 smithi138 ceph-mon[193606]: 9.2 starting backfill to osd.6 from (0'0,0'0] MAX to 234'12343 2023-09-23T23:17:49.904 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:49 smithi138 ceph-mon[193606]: 9.0 starting backfill to osd.4 from (0'0,0'0] MAX to 234'11968 2023-09-23T23:17:49.906 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:17:49.906 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:17:50.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[212939]: failed to encode map e236 with expected crc 2023-09-23T23:17:50.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[212939]: failed to encode map e236 with expected crc 2023-09-23T23:17:50.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[212939]: failed to encode map e236 with expected crc 2023-09-23T23:17:50.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[212939]: 9.7 continuing backfill to osd.1 from (169'1993,234'12043] MIN to 234'12043 2023-09-23T23:17:50.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[212939]: 9.7 starting backfill to osd.6 from (0'0,0'0] MAX to 234'12043 2023-09-23T23:17:50.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[212939]: failed to encode map e237 with expected crc 2023-09-23T23:17:50.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[212939]: osdmap e237: 8 total, 8 up, 7 in 2023-09-23T23:17:50.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[212939]: failed to encode map e237 with expected crc 2023-09-23T23:17:50.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[212939]: failed to encode map e237 with expected crc 2023-09-23T23:17:50.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[212939]: 9.2 starting backfill to osd.6 from (0'0,0'0] MAX to 234'12343 2023-09-23T23:17:50.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[212939]: 9.0 starting backfill to osd.4 from (0'0,0'0] MAX to 234'11968 2023-09-23T23:17:50.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[221102]: failed to encode map e236 with expected crc 2023-09-23T23:17:50.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[221102]: failed to encode map e236 with expected crc 2023-09-23T23:17:50.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[221102]: failed to encode map e236 with expected crc 2023-09-23T23:17:50.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[221102]: 9.7 continuing backfill to osd.1 from (169'1993,234'12043] MIN to 234'12043 2023-09-23T23:17:50.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[221102]: 9.7 starting backfill to osd.6 from (0'0,0'0] MAX to 234'12043 2023-09-23T23:17:50.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[221102]: failed to encode map e237 with expected crc 2023-09-23T23:17:50.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[221102]: osdmap e237: 8 total, 8 up, 7 in 2023-09-23T23:17:50.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[221102]: failed to encode map e237 with expected crc 2023-09-23T23:17:50.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[221102]: failed to encode map e237 with expected crc 2023-09-23T23:17:50.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[221102]: 9.2 starting backfill to osd.6 from (0'0,0'0] MAX to 234'12343 2023-09-23T23:17:50.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:49 smithi110 ceph-mon[221102]: 9.0 starting backfill to osd.4 from (0'0,0'0] MAX to 234'11968 2023-09-23T23:17:51.084 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:17:51.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:50 smithi138 ceph-mon[193606]: failed to encode map e237 with expected crc 2023-09-23T23:17:51.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:50 smithi138 ceph-mon[193606]: failed to encode map e237 with expected crc 2023-09-23T23:17:51.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:50 smithi138 ceph-mon[193606]: failed to encode map e237 with expected crc 2023-09-23T23:17:51.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:50 smithi138 ceph-mon[193606]: 9.6 starting backfill to osd.6 from (0'0,0'0] MAX to 234'12303 2023-09-23T23:17:51.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:50 smithi138 ceph-mon[193606]: pgmap v736: 99 pgs: 1 active+remapped+backfilling, 7 remapped+peering, 1 active+recovering+undersized+remapped, 86 active+clean, 1 active+undersized+remapped, 3 active+remapped+backfill_wait; 1.7 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 11716/84747 objects misplaced (13.825%); 16 MiB/s, 1 keys/s, 253 objects/s recovering 2023-09-23T23:17:51.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:50 smithi138 ceph-mon[193606]: osdmap e238: 8 total, 8 up, 7 in 2023-09-23T23:17:51.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:50 smithi138 ceph-mon[193606]: failed to encode map e238 with expected crc 2023-09-23T23:17:51.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:50 smithi138 ceph-mon[193606]: failed to encode map e238 with expected crc 2023-09-23T23:17:51.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:50 smithi138 ceph-mon[193606]: failed to encode map e238 with expected crc 2023-09-23T23:17:51.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[221102]: failed to encode map e237 with expected crc 2023-09-23T23:17:51.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[221102]: failed to encode map e237 with expected crc 2023-09-23T23:17:51.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[221102]: failed to encode map e237 with expected crc 2023-09-23T23:17:51.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[221102]: 9.6 starting backfill to osd.6 from (0'0,0'0] MAX to 234'12303 2023-09-23T23:17:51.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[221102]: pgmap v736: 99 pgs: 1 active+remapped+backfilling, 7 remapped+peering, 1 active+recovering+undersized+remapped, 86 active+clean, 1 active+undersized+remapped, 3 active+remapped+backfill_wait; 1.7 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 11716/84747 objects misplaced (13.825%); 16 MiB/s, 1 keys/s, 253 objects/s recovering 2023-09-23T23:17:51.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[221102]: osdmap e238: 8 total, 8 up, 7 in 2023-09-23T23:17:51.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[221102]: failed to encode map e238 with expected crc 2023-09-23T23:17:51.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[221102]: failed to encode map e238 with expected crc 2023-09-23T23:17:51.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[221102]: failed to encode map e238 with expected crc 2023-09-23T23:17:51.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[212939]: failed to encode map e237 with expected crc 2023-09-23T23:17:51.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[212939]: failed to encode map e237 with expected crc 2023-09-23T23:17:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[212939]: failed to encode map e237 with expected crc 2023-09-23T23:17:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[212939]: 9.6 starting backfill to osd.6 from (0'0,0'0] MAX to 234'12303 2023-09-23T23:17:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[212939]: pgmap v736: 99 pgs: 1 active+remapped+backfilling, 7 remapped+peering, 1 active+recovering+undersized+remapped, 86 active+clean, 1 active+undersized+remapped, 3 active+remapped+backfill_wait; 1.7 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 11716/84747 objects misplaced (13.825%); 16 MiB/s, 1 keys/s, 253 objects/s recovering 2023-09-23T23:17:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[212939]: osdmap e238: 8 total, 8 up, 7 in 2023-09-23T23:17:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[212939]: failed to encode map e238 with expected crc 2023-09-23T23:17:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[212939]: failed to encode map e238 with expected crc 2023-09-23T23:17:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:50 smithi110 ceph-mon[212939]: failed to encode map e238 with expected crc 2023-09-23T23:17:51.577 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:17:51.878 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:51.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:51.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:51.983 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:17:52.031 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:52.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:51 smithi138 ceph-mon[193606]: failed to encode map e238 with expected crc 2023-09-23T23:17:52.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:51 smithi138 ceph-mon[193606]: failed to encode map e238 with expected crc 2023-09-23T23:17:52.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:51 smithi138 ceph-mon[193606]: failed to encode map e238 with expected crc 2023-09-23T23:17:52.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:51 smithi138 ceph-mon[193606]: failed to encode map e238 with expected crc 2023-09-23T23:17:52.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:51 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:17:52.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:51 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:17:52.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:51 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:52.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:51 smithi138 ceph-mon[193606]: failed to encode map e239 with expected crc 2023-09-23T23:17:52.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:51 smithi138 ceph-mon[193606]: osdmap e239: 8 total, 8 up, 7 in 2023-09-23T23:17:52.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:51 smithi138 ceph-mon[193606]: failed to encode map e239 with expected crc 2023-09-23T23:17:52.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:51 smithi138 ceph-mon[193606]: failed to encode map e239 with expected crc 2023-09-23T23:17:52.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:51 smithi138 ceph-mon[193606]: failed to encode map e239 with expected crc 2023-09-23T23:17:52.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[221102]: failed to encode map e238 with expected crc 2023-09-23T23:17:52.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[221102]: failed to encode map e238 with expected crc 2023-09-23T23:17:52.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[221102]: failed to encode map e238 with expected crc 2023-09-23T23:17:52.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[221102]: failed to encode map e238 with expected crc 2023-09-23T23:17:52.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:17:52.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:17:52.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:52.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[221102]: failed to encode map e239 with expected crc 2023-09-23T23:17:52.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[221102]: osdmap e239: 8 total, 8 up, 7 in 2023-09-23T23:17:52.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[221102]: failed to encode map e239 with expected crc 2023-09-23T23:17:52.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[221102]: failed to encode map e239 with expected crc 2023-09-23T23:17:52.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[221102]: failed to encode map e239 with expected crc 2023-09-23T23:17:52.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[212939]: failed to encode map e238 with expected crc 2023-09-23T23:17:52.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[212939]: failed to encode map e238 with expected crc 2023-09-23T23:17:52.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[212939]: failed to encode map e238 with expected crc 2023-09-23T23:17:52.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[212939]: failed to encode map e238 with expected crc 2023-09-23T23:17:52.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:17:52.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:17:52.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:17:52.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[212939]: failed to encode map e239 with expected crc 2023-09-23T23:17:52.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[212939]: osdmap e239: 8 total, 8 up, 7 in 2023-09-23T23:17:52.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[212939]: failed to encode map e239 with expected crc 2023-09-23T23:17:52.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[212939]: failed to encode map e239 with expected crc 2023-09-23T23:17:52.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:51 smithi110 ceph-mon[212939]: failed to encode map e239 with expected crc 2023-09-23T23:17:52.295 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:17:52.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:52.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:17:52.400 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:52.523 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:52.724 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:52.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:52.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:52.829 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:17:52.877 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:53.086 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:17:53.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:53.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:17:53.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:52 smithi110 ceph-mon[212939]: failed to encode map e239 with expected crc 2023-09-23T23:17:53.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:52 smithi110 ceph-mon[212939]: failed to encode map e239 with expected crc 2023-09-23T23:17:53.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:52 smithi110 ceph-mon[212939]: failed to encode map e239 with expected crc 2023-09-23T23:17:53.100 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:52 smithi110 ceph-mon[212939]: pgmap v739: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 7 remapped+peering, 86 active+clean, 1 active+undersized+remapped, 3 active+remapped+backfill_wait; 1.7 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 11105/83529 objects misplaced (13.295%); 8.3 MiB/s, 2 keys/s, 134 objects/s recovering 2023-09-23T23:17:53.100 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:52 smithi110 ceph-mon[221102]: failed to encode map e239 with expected crc 2023-09-23T23:17:53.100 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:52 smithi110 ceph-mon[221102]: failed to encode map e239 with expected crc 2023-09-23T23:17:53.100 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:52 smithi110 ceph-mon[221102]: failed to encode map e239 with expected crc 2023-09-23T23:17:53.100 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:52 smithi110 ceph-mon[221102]: pgmap v739: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 7 remapped+peering, 86 active+clean, 1 active+undersized+remapped, 3 active+remapped+backfill_wait; 1.7 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 11105/83529 objects misplaced (13.295%); 8.3 MiB/s, 2 keys/s, 134 objects/s recovering 2023-09-23T23:17:53.178 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:17:53.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:52 smithi138 ceph-mon[193606]: failed to encode map e239 with expected crc 2023-09-23T23:17:53.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:52 smithi138 ceph-mon[193606]: failed to encode map e239 with expected crc 2023-09-23T23:17:53.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:52 smithi138 ceph-mon[193606]: failed to encode map e239 with expected crc 2023-09-23T23:17:53.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:52 smithi138 ceph-mon[193606]: pgmap v739: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 7 remapped+peering, 86 active+clean, 1 active+undersized+remapped, 3 active+remapped+backfill_wait; 1.7 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 11105/83529 objects misplaced (13.295%); 8.3 MiB/s, 2 keys/s, 134 objects/s recovering 2023-09-23T23:17:53.199 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:17:53.222 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:17:53.267 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:17:53.304 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:53.328 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:17:53.361 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:17:53.408 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:17:53.462 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:17:53.488 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:17:53.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:53.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:17:53.512 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:17:53.519 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:17:53.519 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:17:53.519 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:17:53.519 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:17:53.519 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:17:53.519 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:17:53.519 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:17:53.519 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:17:53.604 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:17:53.654 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:53.759 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:17:53.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:53.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:17:53.865 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:17:53.940 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:54.035 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:17:54.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:54.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:17:54.064 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:17:54.064 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:17:54.064 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:17:54.064 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:17:54.064 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:17:54.144 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:17:54.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:53 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3059217242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:17:54.195 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:54.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:53 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3059217242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:17:54.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:53 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3059217242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:17:54.392 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:17:54.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:54.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:17:54.498 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:17:54.529 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:54.550 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:17:54.580 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:17:54.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:54.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:17:54.683 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:17:54.822 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:54.942 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:17:54.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:54.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:17:54.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:54 smithi110 ceph-mon[212939]: pgmap v740: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 7 remapped+peering, 86 active+clean, 1 active+undersized+remapped, 3 active+remapped+backfill_wait; 1.7 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 11105/82881 objects misplaced (13.399%); 5.7 MiB/s, 1 keys/s, 91 objects/s recovering 2023-09-23T23:17:55.049 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:55.163 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:55.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:54 smithi110 ceph-mon[221102]: pgmap v740: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 7 remapped+peering, 86 active+clean, 1 active+undersized+remapped, 3 active+remapped+backfill_wait; 1.7 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 11105/82881 objects misplaced (13.399%); 5.7 MiB/s, 1 keys/s, 91 objects/s recovering 2023-09-23T23:17:55.375 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:55.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:55.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:55.400 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:54 smithi138 ceph-mon[193606]: pgmap v740: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 7 remapped+peering, 86 active+clean, 1 active+undersized+remapped, 3 active+remapped+backfill_wait; 1.7 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 11105/82881 objects misplaced (13.399%); 5.7 MiB/s, 1 keys/s, 91 objects/s recovering 2023-09-23T23:17:55.431 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:17:55.482 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:17:55.527 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:17:55.528 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:55.614 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:17:55.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:55.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:17:55.725 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:17:55.799 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:55.926 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:17:55.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:55.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:17:56.037 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:56.048 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:17:56.148 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:56.392 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:56.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:56.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:56.504 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:17:56.581 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:56.719 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:17:56.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:56.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:17:56.828 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:17:56.919 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:56.959 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:56 smithi138 ceph-mon[193606]: pgmap v741: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 91 active+clean, 4 active+remapped+backfill_wait; 1.6 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 334 op/s; 13614/78924 objects misplaced (17.250%); 24 MiB/s, 385 objects/s recovering 2023-09-23T23:17:57.044 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:17:57.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:57.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:17:57.046 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:17:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:17:56] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.43.0" 2023-09-23T23:17:57.046 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:56 smithi110 ceph-mon[221102]: pgmap v741: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 91 active+clean, 4 active+remapped+backfill_wait; 1.6 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 334 op/s; 13614/78924 objects misplaced (17.250%); 24 MiB/s, 385 objects/s recovering 2023-09-23T23:17:57.048 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:56 smithi110 ceph-mon[212939]: pgmap v741: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 91 active+clean, 4 active+remapped+backfill_wait; 1.6 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 334 op/s; 13614/78924 objects misplaced (17.250%); 24 MiB/s, 385 objects/s recovering 2023-09-23T23:17:57.147 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:17:57.175 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:57.228 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:17:57.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:57.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:17:57.331 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:17:57.365 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:57.366 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:17:57.508 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7] out_osds: [2] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T23:17:57.508 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:17:57.508 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T23:17:57.508 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:17:57.672 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:17:57.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:57.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:17:57.781 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:17:57.812 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:57.964 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:17:57.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:57.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:17:58.069 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:17:58.114 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:58.219 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:17:58.219 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:17:58.219 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:17:58.219 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:17:58.219 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.219 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:79748 8.4 8:20294c17:::benchmark_data_smithi110_258128_object79747:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T23:17:58.219 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:08:44.103297+0000", 2023-09-23T23:17:58.220 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 554.113281809, 2023-09-23T23:17:58.220 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1079633360000001, 2023-09-23T23:17:58.220 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.220 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:17:58.220 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.220 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:17:58.220 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:58.220 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 79748 2023-09-23T23:17:58.220 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.220 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.220 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.220 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.221 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103297+0000", 2023-09-23T23:17:58.221 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.221 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.221 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.221 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.221 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103297+0000", 2023-09-23T23:17:58.221 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.221 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.221 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.221 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.221 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103295+0000", 2023-09-23T23:17:58.221 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:17:58.221 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.222 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.222 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.222 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103328+0000", 2023-09-23T23:17:58.222 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3098000000000001e-05 2023-09-23T23:17:58.222 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.222 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.222 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.222 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103342+0000", 2023-09-23T23:17:58.222 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4131000000000001e-05 2023-09-23T23:17:58.222 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.222 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.222 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.223 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103350+0000", 2023-09-23T23:17:58.223 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4490000000000002e-06 2023-09-23T23:17:58.223 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.223 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.223 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.223 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103485+0000", 2023-09-23T23:17:58.223 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013490999999999999 2023-09-23T23:17:58.223 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.223 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.223 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.223 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103606+0000", 2023-09-23T23:17:58.223 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000121727 2023-09-23T23:17:58.224 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.224 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.224 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4", 2023-09-23T23:17:58.224 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.103741+0000", 2023-09-23T23:17:58.224 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013452700000000001 2023-09-23T23:17:58.224 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.224 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.224 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.224 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.113503+0000", 2023-09-23T23:17:58.224 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0097624410000000002 2023-09-23T23:17:58.224 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.224 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.225 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.225 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.113515+0000", 2023-09-23T23:17:58.225 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1352000000000001e-05 2023-09-23T23:17:58.225 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.225 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.225 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.225 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.108207+0000", 2023-09-23T23:17:58.225 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99469248300000002 2023-09-23T23:17:58.225 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.225 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.225 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.225 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.108219+0000", 2023-09-23T23:17:58.225 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1865000000000001e-05 2023-09-23T23:17:58.226 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.226 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.226 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.226 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.197996+0000", 2023-09-23T23:17:58.226 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0897770739999999 2023-09-23T23:17:58.226 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.226 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.226 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-09-23T23:17:58.226 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.198757+0000", 2023-09-23T23:17:58.226 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000760941 2023-09-23T23:17:58.226 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.226 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.227 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.227 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.211195+0000", 2023-09-23T23:17:58.227 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012438015 2023-09-23T23:17:58.227 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.227 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.227 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.227 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.211260+0000", 2023-09-23T23:17:58.227 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4974999999999994e-05 2023-09-23T23:17:58.227 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.227 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.227 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.227 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.227 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.228 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:79761 8.d 8:b0e3b43c:::benchmark_data_smithi110_258128_object79760:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T23:17:58.228 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:08:44.127083+0000", 2023-09-23T23:17:58.228 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 554.08949593600005, 2023-09-23T23:17:58.228 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0759149159999999, 2023-09-23T23:17:58.228 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.228 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.228 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.228 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:17:58.228 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:58.228 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 79761 2023-09-23T23:17:58.228 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.228 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.229 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.229 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.229 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127083+0000", 2023-09-23T23:17:58.229 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.229 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.229 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.229 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.229 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127083+0000", 2023-09-23T23:17:58.229 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.229 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.229 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.229 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.230 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127081+0000", 2023-09-23T23:17:58.230 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:17:58.230 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.230 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.230 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.230 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127142+0000", 2023-09-23T23:17:58.230 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.037e-05 2023-09-23T23:17:58.230 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.230 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.230 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.230 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127156+0000", 2023-09-23T23:17:58.230 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4411e-05 2023-09-23T23:17:58.231 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.231 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.231 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.231 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127163+0000", 2023-09-23T23:17:58.231 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9210000000000003e-06 2023-09-23T23:17:58.231 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.231 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.231 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.231 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127226+0000", 2023-09-23T23:17:58.231 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2739999999999999e-05 2023-09-23T23:17:58.231 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.231 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.231 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.232 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127236+0000", 2023-09-23T23:17:58.232 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0417999999999999e-05 2023-09-23T23:17:58.232 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.232 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.232 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.232 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.106383+0000", 2023-09-23T23:17:58.232 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97914661700000005 2023-09-23T23:17:58.232 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.232 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.232 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.232 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.106399+0000", 2023-09-23T23:17:58.232 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6728999999999999e-05 2023-09-23T23:17:58.233 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.233 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.233 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.233 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.188815+0000", 2023-09-23T23:17:58.233 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0824155440000001 2023-09-23T23:17:58.233 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.233 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.233 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.233 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.189005+0000", 2023-09-23T23:17:58.233 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000190354 2023-09-23T23:17:58.233 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.233 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.234 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-09-23T23:17:58.234 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.189214+0000", 2023-09-23T23:17:58.234 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020901900000000001 2023-09-23T23:17:58.234 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.234 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.234 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.234 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.195871+0000", 2023-09-23T23:17:58.234 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066568920000000002 2023-09-23T23:17:58.234 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.235 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:17:58.236 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.236 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.236 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.195882+0000", 2023-09-23T23:17:58.236 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1206000000000001e-05 2023-09-23T23:17:58.236 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.236 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.237 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.237 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.201793+0000", 2023-09-23T23:17:58.237 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0059103649999999999 2023-09-23T23:17:58.237 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.237 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.237 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.237 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.201868+0000", 2023-09-23T23:17:58.237 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4945999999999997e-05 2023-09-23T23:17:58.237 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.237 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.237 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.237 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.202998+0000", 2023-09-23T23:17:58.238 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00112987 2023-09-23T23:17:58.238 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.238 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.238 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.238 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.238 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.238 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:79762 8.d 8:b39b1064:::benchmark_data_smithi110_258128_object79761:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T23:17:58.238 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:08:44.127180+0000", 2023-09-23T23:17:58.238 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 554.08939878399997, 2023-09-23T23:17:58.238 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0765545470000002, 2023-09-23T23:17:58.238 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.238 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.239 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.239 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:17:58.239 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:58.239 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 79762 2023-09-23T23:17:58.239 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.239 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.239 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.239 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.239 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127180+0000", 2023-09-23T23:17:58.239 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.239 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.239 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.240 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.240 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127180+0000", 2023-09-23T23:17:58.240 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.240 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.240 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.240 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.240 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127179+0000", 2023-09-23T23:17:58.240 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:17:58.240 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.240 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.240 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.240 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127213+0000", 2023-09-23T23:17:58.240 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3992000000000003e-05 2023-09-23T23:17:58.241 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.241 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.241 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.241 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127220+0000", 2023-09-23T23:17:58.241 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7279999999999998e-06 2023-09-23T23:17:58.241 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.241 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.241 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.241 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127225+0000", 2023-09-23T23:17:58.241 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5909999999999998e-06 2023-09-23T23:17:58.241 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.241 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.242 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.242 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127275+0000", 2023-09-23T23:17:58.242 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0458999999999999e-05 2023-09-23T23:17:58.242 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.242 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.242 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.242 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127282+0000", 2023-09-23T23:17:58.242 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3490000000000003e-06 2023-09-23T23:17:58.242 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.242 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.242 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.242 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.106425+0000", 2023-09-23T23:17:58.243 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97914234899999997 2023-09-23T23:17:58.243 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.243 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.243 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.243 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.106431+0000", 2023-09-23T23:17:58.243 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7229999999999998e-06 2023-09-23T23:17:58.243 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.243 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.243 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.243 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.193155+0000", 2023-09-23T23:17:58.243 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0867236360000001 2023-09-23T23:17:58.243 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.243 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.244 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.193297+0000", 2023-09-23T23:17:58.244 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000141843 2023-09-23T23:17:58.244 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.244 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-09-23T23:17:58.244 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.193419+0000", 2023-09-23T23:17:58.244 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012232200000000001 2023-09-23T23:17:58.244 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.244 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.244 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.198570+0000", 2023-09-23T23:17:58.244 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051506369999999996 2023-09-23T23:17:58.245 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.245 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.245 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.245 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.201744+0000", 2023-09-23T23:17:58.245 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031740499999999999 2023-09-23T23:17:58.245 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.245 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.245 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.245 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.203163+0000", 2023-09-23T23:17:58.245 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001419575 2023-09-23T23:17:58.245 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.245 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.246 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.246 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.203203+0000", 2023-09-23T23:17:58.246 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9728000000000003e-05 2023-09-23T23:17:58.246 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.246 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.246 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.246 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.203734+0000", 2023-09-23T23:17:58.246 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00053115499999999995 2023-09-23T23:17:58.246 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.246 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.246 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.246 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.246 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.247 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:79763 8.d 8:b52536a9:::benchmark_data_smithi110_258128_object79762:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T23:17:58.247 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:08:44.127232+0000", 2023-09-23T23:17:58.247 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 554.08934613999998, 2023-09-23T23:17:58.247 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0832427529999999, 2023-09-23T23:17:58.247 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.247 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.247 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.247 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:17:58.247 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:58.247 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 79763 2023-09-23T23:17:58.247 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.247 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.248 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.248 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.248 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127232+0000", 2023-09-23T23:17:58.248 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.248 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.248 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.248 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.248 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127232+0000", 2023-09-23T23:17:58.248 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.248 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.248 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.248 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.248 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127232+0000", 2023-09-23T23:17:58.249 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:17:58.249 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.249 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.249 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.249 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127262+0000", 2023-09-23T23:17:58.249 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.04e-05 2023-09-23T23:17:58.249 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.249 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.249 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.249 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127269+0000", 2023-09-23T23:17:58.249 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8759999999999999e-06 2023-09-23T23:17:58.249 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.249 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.250 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.250 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127274+0000", 2023-09-23T23:17:58.250 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.357e-06 2023-09-23T23:17:58.250 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.250 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.250 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.250 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127309+0000", 2023-09-23T23:17:58.250 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4946000000000001e-05 2023-09-23T23:17:58.250 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.250 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.250 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.250 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.127318+0000", 2023-09-23T23:17:58.251 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9269999999999994e-06 2023-09-23T23:17:58.251 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.251 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.251 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.251 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.106461+0000", 2023-09-23T23:17:58.251 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97914303499999999 2023-09-23T23:17:58.251 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.251 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.251 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.251 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.106464+0000", 2023-09-23T23:17:58.251 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.743e-06 2023-09-23T23:17:58.251 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.252 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.252 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.252 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.196067+0000", 2023-09-23T23:17:58.252 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.089602677 2023-09-23T23:17:58.252 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.252 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.252 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.252 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.196197+0000", 2023-09-23T23:17:58.252 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013014400000000001 2023-09-23T23:17:58.252 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.252 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.252 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-09-23T23:17:58.252 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.196329+0000", 2023-09-23T23:17:58.253 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013207200000000001 2023-09-23T23:17:58.253 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.253 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.253 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.253 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.201686+0000", 2023-09-23T23:17:58.253 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0053572860000000002 2023-09-23T23:17:58.253 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.253 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.253 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.253 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.203099+0000", 2023-09-23T23:17:58.253 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001412558 2023-09-23T23:17:58.253 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.253 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.254 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.254 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.209540+0000", 2023-09-23T23:17:58.254 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064405979999999996 2023-09-23T23:17:58.254 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.254 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.254 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.254 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.209606+0000", 2023-09-23T23:17:58.254 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6402999999999994e-05 2023-09-23T23:17:58.254 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.254 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.254 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.254 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.210475+0000", 2023-09-23T23:17:58.255 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00086918300000000004 2023-09-23T23:17:58.255 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.255 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.255 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.255 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.255 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.255 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:79775 8.d 8:b42b6b80:::benchmark_data_smithi110_258128_object79774:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T23:17:58.255 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:08:44.135988+0000", 2023-09-23T23:17:58.255 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 554.08059063400003, 2023-09-23T23:17:58.255 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0751020269999998, 2023-09-23T23:17:58.255 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.255 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.256 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.256 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:17:58.256 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:58.256 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 79775 2023-09-23T23:17:58.256 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.256 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.256 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.256 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.256 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.135988+0000", 2023-09-23T23:17:58.256 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.256 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.256 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.256 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.257 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.135988+0000", 2023-09-23T23:17:58.257 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.257 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.257 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.257 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.257 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.135987+0000", 2023-09-23T23:17:58.257 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:17:58.257 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.257 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.257 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.257 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136044+0000", 2023-09-23T23:17:58.257 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6675999999999997e-05 2023-09-23T23:17:58.257 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.258 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.258 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.258 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136053+0000", 2023-09-23T23:17:58.258 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9609999999999994e-06 2023-09-23T23:17:58.258 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.258 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.258 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.258 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136057+0000", 2023-09-23T23:17:58.258 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9580000000000001e-06 2023-09-23T23:17:58.258 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.258 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.258 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.259 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136098+0000", 2023-09-23T23:17:58.259 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1551999999999998e-05 2023-09-23T23:17:58.259 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.259 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.259 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.259 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136105+0000", 2023-09-23T23:17:58.259 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6080000000000001e-06 2023-09-23T23:17:58.259 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.259 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.259 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.259 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.106481+0000", 2023-09-23T23:17:58.259 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97037604700000002 2023-09-23T23:17:58.259 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.260 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.260 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.260 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.106488+0000", 2023-09-23T23:17:58.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1489999999999997e-06 2023-09-23T23:17:58.260 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.260 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.260 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.260 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.198642+0000", 2023-09-23T23:17:58.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.092154125 2023-09-23T23:17:58.260 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.260 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.260 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.198803+0000", 2023-09-23T23:17:58.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000161087 2023-09-23T23:17:58.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-09-23T23:17:58.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.198989+0000", 2023-09-23T23:17:58.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018580499999999999 2023-09-23T23:17:58.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.203112+0000", 2023-09-23T23:17:58.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004122863 2023-09-23T23:17:58.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.262 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.262 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.203791+0000", 2023-09-23T23:17:58.262 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00067863899999999998 2023-09-23T23:17:58.262 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.262 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.262 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.210523+0000", 2023-09-23T23:17:58.262 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0067325019999999996 2023-09-23T23:17:58.262 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.262 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.263 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.210562+0000", 2023-09-23T23:17:58.263 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9252000000000003e-05 2023-09-23T23:17:58.263 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.263 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.263 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.263 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.211090+0000", 2023-09-23T23:17:58.263 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00052762200000000005 2023-09-23T23:17:58.263 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.263 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.263 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.263 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.263 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.263 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:79776 8.4 8:299b8268:::benchmark_data_smithi110_258128_object79775:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T23:17:58.264 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:08:44.136068+0000", 2023-09-23T23:17:58.264 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 554.08051045100001, 2023-09-23T23:17:58.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0756627179999998, 2023-09-23T23:17:58.264 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.264 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.264 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.264 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:17:58.264 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:58.264 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 79776 2023-09-23T23:17:58.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.264 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.264 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.265 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136068+0000", 2023-09-23T23:17:58.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.265 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.265 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.265 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136068+0000", 2023-09-23T23:17:58.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.265 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.265 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.265 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136068+0000", 2023-09-23T23:17:58.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:17:58.266 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.266 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.266 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.266 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136143+0000", 2023-09-23T23:17:58.266 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5354999999999999e-05 2023-09-23T23:17:58.266 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.266 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.266 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.266 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136150+0000", 2023-09-23T23:17:58.266 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9940000000000003e-06 2023-09-23T23:17:58.266 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.266 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.266 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136153+0000", 2023-09-23T23:17:58.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.016e-06 2023-09-23T23:17:58.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.267 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136190+0000", 2023-09-23T23:17:58.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6891999999999998e-05 2023-09-23T23:17:58.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.267 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:44.136196+0000", 2023-09-23T23:17:58.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.507e-06 2023-09-23T23:17:58.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.268 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.108263+0000", 2023-09-23T23:17:58.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97206721200000001 2023-09-23T23:17:58.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.268 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:45.108272+0000", 2023-09-23T23:17:58.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6899999999999998e-06 2023-09-23T23:17:58.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.199665+0000", 2023-09-23T23:17:58.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.09139388 2023-09-23T23:17:58.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.199765+0000", 2023-09-23T23:17:58.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9269000000000001e-05 2023-09-23T23:17:58.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:17:58.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.199873+0000", 2023-09-23T23:17:58.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108823 2023-09-23T23:17:58.270 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.270 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.270 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.202326+0000", 2023-09-23T23:17:58.270 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024524310000000001 2023-09-23T23:17:58.270 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.270 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.270 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.204652+0000", 2023-09-23T23:17:58.270 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023259560000000001 2023-09-23T23:17:58.270 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.270 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.271 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.211305+0000", 2023-09-23T23:17:58.271 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066528059999999998 2023-09-23T23:17:58.271 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.271 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.271 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.271 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.211339+0000", 2023-09-23T23:17:58.271 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4777999999999998e-05 2023-09-23T23:17:58.271 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.271 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.271 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.271 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:08:46.211731+0000", 2023-09-23T23:17:58.271 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00039142099999999998 2023-09-23T23:17:58.272 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.272 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.272 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.272 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.272 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:186176 8.8 8:12f1ade7:::benchmark_data_smithi110_258128_object67358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T23:17:58.272 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:58.477323+0000", 2023-09-23T23:17:58.272 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 419.73925545100002, 2023-09-23T23:17:58.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4157779150000001, 2023-09-23T23:17:58.272 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.272 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:17:58.272 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.273 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:17:58.273 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:58.273 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 186176 2023-09-23T23:17:58.273 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.273 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.273 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.273 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477323+0000", 2023-09-23T23:17:58.273 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.273 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.273 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477323+0000", 2023-09-23T23:17:58.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.274 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.274 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.274 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477322+0000", 2023-09-23T23:17:58.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:17:58.274 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.274 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.274 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477327+0000", 2023-09-23T23:17:58.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7049999999999998e-06 2023-09-23T23:17:58.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.275 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.275 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477336+0000", 2023-09-23T23:17:58.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0299999999999999e-06 2023-09-23T23:17:58.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.275 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.275 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477342+0000", 2023-09-23T23:17:58.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7300000000000002e-06 2023-09-23T23:17:58.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477395+0000", 2023-09-23T23:17:58.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3256e-05 2023-09-23T23:17:58.276 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477519+0000", 2023-09-23T23:17:58.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012338599999999999 2023-09-23T23:17:58.276 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 7", 2023-09-23T23:17:58.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477633+0000", 2023-09-23T23:17:58.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000114461 2023-09-23T23:17:58.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.277 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.277 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.277 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.487194+0000", 2023-09-23T23:17:58.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0095614959999999992 2023-09-23T23:17:58.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.277 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.277 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.277 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.490656+0000", 2023-09-23T23:17:58.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003461689 2023-09-23T23:17:58.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.277 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.277 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.278 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.490665+0000", 2023-09-23T23:17:58.278 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0159999999999997e-06 2023-09-23T23:17:58.278 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.278 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.278 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489508+0000", 2023-09-23T23:17:58.278 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99884329500000002 2023-09-23T23:17:58.278 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.278 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.278 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489523+0000", 2023-09-23T23:17:58.278 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4565e-05 2023-09-23T23:17:58.278 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.279 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.868588+0000", 2023-09-23T23:17:58.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3790654570000001 2023-09-23T23:17:58.279 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-09-23T23:17:58.279 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.868704+0000", 2023-09-23T23:17:58.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000115458 2023-09-23T23:17:58.279 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.280 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893017+0000", 2023-09-23T23:17:58.280 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.024312963999999999 2023-09-23T23:17:58.280 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.280 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.280 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.280 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893101+0000", 2023-09-23T23:17:58.280 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4171000000000004e-05 2023-09-23T23:17:58.280 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.280 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.280 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.280 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.280 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.281 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:186189 8.8 8:14e93b65:::benchmark_data_smithi110_258128_object67371:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T23:17:58.281 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:58.504373+0000", 2023-09-23T23:17:58.281 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 419.71220524900002, 2023-09-23T23:17:58.281 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.388920556, 2023-09-23T23:17:58.281 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.281 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.281 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.281 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:17:58.281 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:58.281 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 186189 2023-09-23T23:17:58.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.281 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.282 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.282 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504373+0000", 2023-09-23T23:17:58.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.282 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.282 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.282 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504373+0000", 2023-09-23T23:17:58.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.282 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.282 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.282 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504372+0000", 2023-09-23T23:17:58.283 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:17:58.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.283 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.283 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.283 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504379+0000", 2023-09-23T23:17:58.283 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7580000000000003e-06 2023-09-23T23:17:58.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.283 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.283 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.283 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504390+0000", 2023-09-23T23:17:58.283 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1239e-05 2023-09-23T23:17:58.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504397+0000", 2023-09-23T23:17:58.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3509999999999996e-06 2023-09-23T23:17:58.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504473+0000", 2023-09-23T23:17:58.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5463000000000006e-05 2023-09-23T23:17:58.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504483+0000", 2023-09-23T23:17:58.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0202e-05 2023-09-23T23:17:58.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489551+0000", 2023-09-23T23:17:58.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98506775000000002 2023-09-23T23:17:58.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489558+0000", 2023-09-23T23:17:58.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4170000000000003e-06 2023-09-23T23:17:58.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.286 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.286 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.286 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.868754+0000", 2023-09-23T23:17:58.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3791954120000001 2023-09-23T23:17:58.286 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.286 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.286 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.286 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.868858+0000", 2023-09-23T23:17:58.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103983 2023-09-23T23:17:58.286 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.286 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.286 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:17:58.287 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.868960+0000", 2023-09-23T23:17:58.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010265800000000001 2023-09-23T23:17:58.287 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.287 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.877827+0000", 2023-09-23T23:17:58.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088667039999999996 2023-09-23T23:17:58.287 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.287 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.877878+0000", 2023-09-23T23:17:58.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1452000000000001e-05 2023-09-23T23:17:58.287 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.288 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.288 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893139+0000", 2023-09-23T23:17:58.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015260272 2023-09-23T23:17:58.288 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.288 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.288 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893179+0000", 2023-09-23T23:17:58.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9881e-05 2023-09-23T23:17:58.288 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.288 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.288 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893294+0000", 2023-09-23T23:17:58.289 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000115392 2023-09-23T23:17:58.289 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.289 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.289 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.289 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:186216 8.8 8:19876885:::benchmark_data_smithi110_258128_object67398:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T23:17:58.289 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:58.548467+0000", 2023-09-23T23:17:58.289 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 419.66811152499997, 2023-09-23T23:17:58.289 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3450009590000001, 2023-09-23T23:17:58.289 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.289 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.290 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.290 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:17:58.290 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:17:58.290 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 186216 2023-09-23T23:17:58.290 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.290 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.290 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.290 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.290 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548467+0000", 2023-09-23T23:17:58.290 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.290 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.290 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.291 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548467+0000", 2023-09-23T23:17:58.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.291 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.291 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.291 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548466+0000", 2023-09-23T23:17:58.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:17:58.291 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.291 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.291 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548473+0000", 2023-09-23T23:17:58.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0339999999999999e-06 2023-09-23T23:17:58.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.292 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.292 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.292 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548485+0000", 2023-09-23T23:17:58.292 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.223e-05 2023-09-23T23:17:58.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.292 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.292 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.292 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548492+0000", 2023-09-23T23:17:58.292 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5509999999999996e-06 2023-09-23T23:17:58.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.292 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.292 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.293 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548545+0000", 2023-09-23T23:17:58.293 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3434999999999999e-05 2023-09-23T23:17:58.293 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.293 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.293 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548552+0000", 2023-09-23T23:17:58.293 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8750000000000002e-06 2023-09-23T23:17:58.293 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.293 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.293 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489851+0000", 2023-09-23T23:17:58.293 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94129886100000004 2023-09-23T23:17:58.293 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.294 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.294 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489860+0000", 2023-09-23T23:17:58.294 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7079999999999992e-06 2023-09-23T23:17:58.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.294 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.294 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.873113+0000", 2023-09-23T23:17:58.294 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3832525150000001 2023-09-23T23:17:58.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.294 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.294 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.873221+0000", 2023-09-23T23:17:58.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010844300000000001 2023-09-23T23:17:58.295 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.295 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:17:58.295 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.873337+0000", 2023-09-23T23:17:58.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000115621 2023-09-23T23:17:58.295 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.295 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.295 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.877887+0000", 2023-09-23T23:17:58.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0045499099999999999 2023-09-23T23:17:58.295 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.296 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.877920+0000", 2023-09-23T23:17:58.296 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3314e-05 2023-09-23T23:17:58.296 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.296 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.296 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893326+0000", 2023-09-23T23:17:58.296 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015405911 2023-09-23T23:17:58.296 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.296 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.296 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893363+0000", 2023-09-23T23:17:58.297 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7240000000000003e-05 2023-09-23T23:17:58.297 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.297 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.297 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.297 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893468+0000", 2023-09-23T23:17:58.297 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010466700000000001 2023-09-23T23:17:58.297 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.297 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.297 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.297 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.297 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.297 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165813 9.7 9:ef096715:::benchmark_data_smithi110_295904_object54632:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:17:58.297 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.377865+0000", 2023-09-23T23:17:58.298 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 74.838713323999997, 2023-09-23T23:17:58.298 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1834216419999999, 2023-09-23T23:17:58.298 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.298 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.298 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.298 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:17:58.298 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:17:58.298 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165813 2023-09-23T23:17:58.298 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.298 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.298 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.298 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.299 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377865+0000", 2023-09-23T23:17:58.299 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.299 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.299 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377865+0000", 2023-09-23T23:17:58.299 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.299 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.299 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377864+0000", 2023-09-23T23:17:58.299 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:17:58.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.300 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377872+0000", 2023-09-23T23:17:58.300 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9179999999999994e-06 2023-09-23T23:17:58.300 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.300 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377889+0000", 2023-09-23T23:17:58.300 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7413000000000001e-05 2023-09-23T23:17:58.300 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.300 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377897+0000", 2023-09-23T23:17:58.301 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0309999999999998e-06 2023-09-23T23:17:58.301 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.301 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.301 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.301 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377972+0000", 2023-09-23T23:17:58.301 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4659999999999993e-05 2023-09-23T23:17:58.301 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.301 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.301 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.301 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377984+0000", 2023-09-23T23:17:58.301 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2333e-05 2023-09-23T23:17:58.301 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.302 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.302 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.302 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340836+0000", 2023-09-23T23:17:58.302 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96285172900000005 2023-09-23T23:17:58.302 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.302 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.302 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.302 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340849+0000", 2023-09-23T23:17:58.302 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3107e-05 2023-09-23T23:17:58.302 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.302 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.302 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.303 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.529796+0000", 2023-09-23T23:17:58.303 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.188947368 2023-09-23T23:17:58.303 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.303 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.303 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.303 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.529935+0000", 2023-09-23T23:17:58.303 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013898799999999999 2023-09-23T23:17:58.303 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.303 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.303 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:17:58.303 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.530085+0000", 2023-09-23T23:17:58.303 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015011199999999999 2023-09-23T23:17:58.303 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.304 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.304 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.304 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.557076+0000", 2023-09-23T23:17:58.304 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.026990584000000001 2023-09-23T23:17:58.304 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.304 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.304 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.304 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.560991+0000", 2023-09-23T23:17:58.304 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039146149999999998 2023-09-23T23:17:58.304 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.304 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.304 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.304 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561072+0000", 2023-09-23T23:17:58.305 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1526000000000005e-05 2023-09-23T23:17:58.305 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.305 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.305 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.305 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561185+0000", 2023-09-23T23:17:58.305 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011253599999999999 2023-09-23T23:17:58.305 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.305 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.305 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.305 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561222+0000", 2023-09-23T23:17:58.305 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6940000000000002e-05 2023-09-23T23:17:58.305 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.305 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.306 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.306 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561287+0000", 2023-09-23T23:17:58.306 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5339000000000006e-05 2023-09-23T23:17:58.306 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.306 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.306 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.306 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.306 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.306 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165818 9.7 9:edd24351:::benchmark_data_smithi110_295904_object54637:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:17:58.306 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.384734+0000", 2023-09-23T23:17:58.306 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 74.831844399999994, 2023-09-23T23:17:58.306 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1767809169999999, 2023-09-23T23:17:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:17:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:17:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165818 2023-09-23T23:17:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384734+0000", 2023-09-23T23:17:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384734+0000", 2023-09-23T23:17:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384733+0000", 2023-09-23T23:17:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:17:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384738+0000", 2023-09-23T23:17:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1370000000000001e-06 2023-09-23T23:17:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384750+0000", 2023-09-23T23:17:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.198e-05 2023-09-23T23:17:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384773+0000", 2023-09-23T23:17:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2255999999999998e-05 2023-09-23T23:17:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384816+0000", 2023-09-23T23:17:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.35e-05 2023-09-23T23:17:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384822+0000", 2023-09-23T23:17:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1630000000000001e-06 2023-09-23T23:17:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340872+0000", 2023-09-23T23:17:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95605004699999996 2023-09-23T23:17:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340880+0000", 2023-09-23T23:17:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3610000000000003e-06 2023-09-23T23:17:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.533907+0000", 2023-09-23T23:17:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.193027515 2023-09-23T23:17:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.534044+0000", 2023-09-23T23:17:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013652 2023-09-23T23:17:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:17:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.534213+0000", 2023-09-23T23:17:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016906000000000001 2023-09-23T23:17:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.557086+0000", 2023-09-23T23:17:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.022873668999999999 2023-09-23T23:17:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561026+0000", 2023-09-23T23:17:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039397240000000004 2023-09-23T23:17:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561103+0000", 2023-09-23T23:17:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6483000000000006e-05 2023-09-23T23:17:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561335+0000", 2023-09-23T23:17:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000231967 2023-09-23T23:17:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561390+0000", 2023-09-23T23:17:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5334999999999998e-05 2023-09-23T23:17:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561515+0000", 2023-09-23T23:17:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012529199999999999 2023-09-23T23:17:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165854 9.7 9:e376a013:::benchmark_data_smithi110_295904_object54673:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:17:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.463878+0000", 2023-09-23T23:17:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 74.752700367000003, 2023-09-23T23:17:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0978617750000002, 2023-09-23T23:17:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:17:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:17:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165854 2023-09-23T23:17:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463878+0000", 2023-09-23T23:17:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463878+0000", 2023-09-23T23:17:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463877+0000", 2023-09-23T23:17:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:17:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463883+0000", 2023-09-23T23:17:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5269999999999998e-06 2023-09-23T23:17:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463893+0000", 2023-09-23T23:17:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0327e-05 2023-09-23T23:17:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463898+0000", 2023-09-23T23:17:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0509999999999996e-06 2023-09-23T23:17:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463961+0000", 2023-09-23T23:17:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2624000000000002e-05 2023-09-23T23:17:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463972+0000", 2023-09-23T23:17:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0920999999999999e-05 2023-09-23T23:17:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340900+0000", 2023-09-23T23:17:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.87692793800000002 2023-09-23T23:17:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340907+0000", 2023-09-23T23:17:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0990000000000001e-06 2023-09-23T23:17:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.537799+0000", 2023-09-23T23:17:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1968927309999999 2023-09-23T23:17:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.537907+0000", 2023-09-23T23:17:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107235 2023-09-23T23:17:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:17:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.538020+0000", 2023-09-23T23:17:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011348700000000001 2023-09-23T23:17:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.557096+0000", 2023-09-23T23:17:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019076085999999999 2023-09-23T23:17:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561153+0000", 2023-09-23T23:17:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040572120000000001 2023-09-23T23:17:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561540+0000", 2023-09-23T23:17:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00038614200000000002 2023-09-23T23:17:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561584+0000", 2023-09-23T23:17:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3961999999999997e-05 2023-09-23T23:17:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561633+0000", 2023-09-23T23:17:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9116999999999997e-05 2023-09-23T23:17:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561740+0000", 2023-09-23T23:17:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107328 2023-09-23T23:17:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165860 9.7 9:e0c9fcc3:::benchmark_data_smithi110_295904_object54679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:17:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.472345+0000", 2023-09-23T23:17:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 74.744233596000001, 2023-09-23T23:17:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0897597270000001, 2023-09-23T23:17:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:17:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:17:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165860 2023-09-23T23:17:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472345+0000", 2023-09-23T23:17:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472345+0000", 2023-09-23T23:17:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472345+0000", 2023-09-23T23:17:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:17:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472348+0000", 2023-09-23T23:17:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9739999999999998e-06 2023-09-23T23:17:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472356+0000", 2023-09-23T23:17:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1419999999999993e-06 2023-09-23T23:17:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472359+0000", 2023-09-23T23:17:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3560000000000001e-06 2023-09-23T23:17:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472409+0000", 2023-09-23T23:17:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9982999999999999e-05 2023-09-23T23:17:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472418+0000", 2023-09-23T23:17:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7660000000000004e-06 2023-09-23T23:17:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340926+0000", 2023-09-23T23:17:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.86850790200000005 2023-09-23T23:17:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340933+0000", 2023-09-23T23:17:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2139999999999999e-06 2023-09-23T23:17:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.541418+0000", 2023-09-23T23:17:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2004846490000001 2023-09-23T23:17:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.541552+0000", 2023-09-23T23:17:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000134566 2023-09-23T23:17:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:17:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.542163+0000", 2023-09-23T23:17:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00061078700000000001 2023-09-23T23:17:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.560793+0000", 2023-09-23T23:17:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018630225 2023-09-23T23:17:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561763+0000", 2023-09-23T23:17:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00097026299999999996 2023-09-23T23:17:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561803+0000", 2023-09-23T23:17:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9400000000000002e-05 2023-09-23T23:17:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561930+0000", 2023-09-23T23:17:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000127367 2023-09-23T23:17:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561986+0000", 2023-09-23T23:17:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5379e-05 2023-09-23T23:17:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562105+0000", 2023-09-23T23:17:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000119218 2023-09-23T23:17:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165866 9.7 9:e8d58a1b:::benchmark_data_smithi110_295904_object54685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:17:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.486204+0000", 2023-09-23T23:17:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 74.730374783000002, 2023-09-23T23:17:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0760496399999999, 2023-09-23T23:17:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:17:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:17:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165866 2023-09-23T23:17:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486204+0000", 2023-09-23T23:17:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486204+0000", 2023-09-23T23:17:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486203+0000", 2023-09-23T23:17:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:17:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486207+0000", 2023-09-23T23:17:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.236e-06 2023-09-23T23:17:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486215+0000", 2023-09-23T23:17:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7199999999999995e-06 2023-09-23T23:17:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486219+0000", 2023-09-23T23:17:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5839999999999999e-06 2023-09-23T23:17:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486260+0000", 2023-09-23T23:17:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1323999999999999e-05 2023-09-23T23:17:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486269+0000", 2023-09-23T23:17:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5390000000000007e-06 2023-09-23T23:17:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340952+0000", 2023-09-23T23:17:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.85468296899999996 2023-09-23T23:17:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340959+0000", 2023-09-23T23:17:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0419999999999997e-06 2023-09-23T23:17:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.545682+0000", 2023-09-23T23:17:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2047230959999999 2023-09-23T23:17:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.545818+0000", 2023-09-23T23:17:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000136527 2023-09-23T23:17:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:17:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.545972+0000", 2023-09-23T23:17:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015338600000000001 2023-09-23T23:17:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.560804+0000", 2023-09-23T23:17:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014832051000000001 2023-09-23T23:17:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561831+0000", 2023-09-23T23:17:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001027395 2023-09-23T23:17:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561871+0000", 2023-09-23T23:17:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9718999999999997e-05 2023-09-23T23:17:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562134+0000", 2023-09-23T23:17:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00026311599999999998 2023-09-23T23:17:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562161+0000", 2023-09-23T23:17:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7259000000000001e-05 2023-09-23T23:17:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562253+0000", 2023-09-23T23:17:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2167e-05 2023-09-23T23:17:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165894 9.7 9:e099b5a1:::benchmark_data_smithi110_295904_object54713:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:17:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.692918+0000", 2023-09-23T23:17:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 74.523660196999998, 2023-09-23T23:17:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.869631313, 2023-09-23T23:17:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:17:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:17:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165894 2023-09-23T23:17:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692918+0000", 2023-09-23T23:17:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692918+0000", 2023-09-23T23:17:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692918+0000", 2023-09-23T23:17:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:17:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692923+0000", 2023-09-23T23:17:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0499999999999999e-06 2023-09-23T23:17:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692933+0000", 2023-09-23T23:17:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0591000000000001e-05 2023-09-23T23:17:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692938+0000", 2023-09-23T23:17:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1440000000000002e-06 2023-09-23T23:17:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692991+0000", 2023-09-23T23:17:58.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2522000000000003e-05 2023-09-23T23:17:58.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.693000+0000", 2023-09-23T23:17:58.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2229999999999996e-06 2023-09-23T23:17:58.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340977+0000", 2023-09-23T23:17:58.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.64797712799999996 2023-09-23T23:17:58.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340984+0000", 2023-09-23T23:17:58.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0360000000000001e-06 2023-09-23T23:17:58.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.549530+0000", 2023-09-23T23:17:58.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.208546127 2023-09-23T23:17:58.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.549658+0000", 2023-09-23T23:17:58.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012757799999999999 2023-09-23T23:17:58.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:17:58.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.549806+0000", 2023-09-23T23:17:58.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014840400000000001 2023-09-23T23:17:58.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.560819+0000", 2023-09-23T23:17:58.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011012752000000001 2023-09-23T23:17:58.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.348 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.348 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561890+0000", 2023-09-23T23:17:58.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001071383 2023-09-23T23:17:58.348 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.348 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.348 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562314+0000", 2023-09-23T23:17:58.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00042307100000000002 2023-09-23T23:17:58.348 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.348 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.348 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562371+0000", 2023-09-23T23:17:58.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7435999999999997e-05 2023-09-23T23:17:58.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562424+0000", 2023-09-23T23:17:58.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3461999999999997e-05 2023-09-23T23:17:58.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562550+0000", 2023-09-23T23:17:58.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000125277 2023-09-23T23:17:58.349 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.349 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.350 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.350 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165909 9.7 9:e5c10464:::benchmark_data_smithi110_295904_object54728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:17:58.350 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.748819+0000", 2023-09-23T23:17:58.350 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 74.467759333999993, 2023-09-23T23:17:58.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.814135311, 2023-09-23T23:17:58.350 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.350 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.350 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.350 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:17:58.350 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:17:58.350 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165909 2023-09-23T23:17:58.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.351 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748819+0000", 2023-09-23T23:17:58.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748819+0000", 2023-09-23T23:17:58.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748819+0000", 2023-09-23T23:17:58.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:17:58.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748823+0000", 2023-09-23T23:17:58.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.668e-06 2023-09-23T23:17:58.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748837+0000", 2023-09-23T23:17:58.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.363e-05 2023-09-23T23:17:58.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748842+0000", 2023-09-23T23:17:58.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8790000000000002e-06 2023-09-23T23:17:58.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748893+0000", 2023-09-23T23:17:58.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1456000000000003e-05 2023-09-23T23:17:58.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748901+0000", 2023-09-23T23:17:58.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7530000000000001e-06 2023-09-23T23:17:58.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.341003+0000", 2023-09-23T23:17:58.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.59210227299999996 2023-09-23T23:17:58.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:17:58.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.341010+0000", 2023-09-23T23:17:58.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5599999999999999e-06 2023-09-23T23:17:58.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.553256+0000", 2023-09-23T23:17:58.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2122462409999999 2023-09-23T23:17:58.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.553394+0000", 2023-09-23T23:17:58.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013755000000000001 2023-09-23T23:17:58.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:17:58.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.553548+0000", 2023-09-23T23:17:58.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000154449 2023-09-23T23:17:58.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561003+0000", 2023-09-23T23:17:58.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0074545380000000001 2023-09-23T23:17:58.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562606+0000", 2023-09-23T23:17:58.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0016038770000000001 2023-09-23T23:17:58.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562661+0000", 2023-09-23T23:17:58.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4122000000000001e-05 2023-09-23T23:17:58.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562787+0000", 2023-09-23T23:17:58.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012642 2023-09-23T23:17:58.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.358 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.358 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562837+0000", 2023-09-23T23:17:58.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0565000000000001e-05 2023-09-23T23:17:58.358 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.358 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.358 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562955+0000", 2023-09-23T23:17:58.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117106 2023-09-23T23:17:58.358 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.358 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.358 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.358 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.359 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:188501 9.6 9:679feb10:::benchmark_data_smithi110_295904_object77320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-09-23T23:17:58.359 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:33.113667+0000", 2023-09-23T23:17:58.359 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 25.102911124999999, 2023-09-23T23:17:58.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.30803306400000002, 2023-09-23T23:17:58.359 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.359 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.359 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.359 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:17:58.359 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:17:58.359 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 188501 2023-09-23T23:17:58.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.359 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113667+0000", 2023-09-23T23:17:58.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113667+0000", 2023-09-23T23:17:58.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113666+0000", 2023-09-23T23:17:58.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:17:58.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113673+0000", 2023-09-23T23:17:58.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7889999999999997e-06 2023-09-23T23:17:58.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113688+0000", 2023-09-23T23:17:58.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5165e-05 2023-09-23T23:17:58.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113695+0000", 2023-09-23T23:17:58.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6630000000000004e-06 2023-09-23T23:17:58.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113737+0000", 2023-09-23T23:17:58.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1814000000000003e-05 2023-09-23T23:17:58.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113855+0000", 2023-09-23T23:17:58.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118145 2023-09-23T23:17:58.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:17:58.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113978+0000", 2023-09-23T23:17:58.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000123103 2023-09-23T23:17:58.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.117927+0000", 2023-09-23T23:17:58.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039487050000000003 2023-09-23T23:17:58.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.126736+0000", 2023-09-23T23:17:58.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088093419999999995 2023-09-23T23:17:58.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.421639+0000", 2023-09-23T23:17:58.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.29490354099999999 2023-09-23T23:17:58.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.421673+0000", 2023-09-23T23:17:58.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3896000000000002e-05 2023-09-23T23:17:58.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.365 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.365 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.421701+0000", 2023-09-23T23:17:58.365 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7203e-05 2023-09-23T23:17:58.365 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.365 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.365 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.365 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.365 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:188507 9.f 9:fef08f68:::benchmark_data_smithi110_295904_object77326:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-09-23T23:17:58.365 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:33.132731+0000", 2023-09-23T23:17:58.365 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 25.083847885000001, 2023-09-23T23:17:58.365 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.30463230400000002, 2023-09-23T23:17:58.366 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.366 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.366 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.366 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:17:58.366 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:17:58.366 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 188507 2023-09-23T23:17:58.366 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.366 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.366 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.366 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.366 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.132731+0000", 2023-09-23T23:17:58.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.132731+0000", 2023-09-23T23:17:58.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.132730+0000", 2023-09-23T23:17:58.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:17:58.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.132735+0000", 2023-09-23T23:17:58.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4240000000000001e-06 2023-09-23T23:17:58.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.132747+0000", 2023-09-23T23:17:58.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1979e-05 2023-09-23T23:17:58.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.132752+0000", 2023-09-23T23:17:58.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1619999999999999e-06 2023-09-23T23:17:58.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.132785+0000", 2023-09-23T23:17:58.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2388999999999998e-05 2023-09-23T23:17:58.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.132880+0000", 2023-09-23T23:17:58.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5057999999999997e-05 2023-09-23T23:17:58.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:17:58.370 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.132956+0000", 2023-09-23T23:17:58.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6810000000000005e-05 2023-09-23T23:17:58.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.370 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.137194+0000", 2023-09-23T23:17:58.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0042370680000000001 2023-09-23T23:17:58.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.370 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.433640+0000", 2023-09-23T23:17:58.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.29644681499999997 2023-09-23T23:17:58.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.437282+0000", 2023-09-23T23:17:58.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0036414809999999998 2023-09-23T23:17:58.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.437322+0000", 2023-09-23T23:17:58.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0277000000000001e-05 2023-09-23T23:17:58.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.437363+0000", 2023-09-23T23:17:58.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0905000000000002e-05 2023-09-23T23:17:58.372 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.372 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.372 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.372 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:196771 9.f 9:f57a96f9:::benchmark_data_smithi110_295904_object85590:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e239)", 2023-09-23T23:17:58.372 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:54.434404+0000", 2023-09-23T23:17:58.372 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 3.782174656, 2023-09-23T23:17:58.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26140648900000002, 2023-09-23T23:17:58.372 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.373 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.373 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.373 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:17:58.373 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:17:58.373 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 196771 2023-09-23T23:17:58.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.373 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:54.434404+0000", 2023-09-23T23:17:58.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:54.434404+0000", 2023-09-23T23:17:58.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:54.434403+0000", 2023-09-23T23:17:58.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:17:58.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:54.434406+0000", 2023-09-23T23:17:58.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6960000000000001e-06 2023-09-23T23:17:58.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:54.434414+0000", 2023-09-23T23:17:58.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2230000000000007e-06 2023-09-23T23:17:58.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:54.434418+0000", 2023-09-23T23:17:58.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0230000000000003e-06 2023-09-23T23:17:58.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:54.439095+0000", 2023-09-23T23:17:58.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0046762100000000001 2023-09-23T23:17:58.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:54.439257+0000", 2023-09-23T23:17:58.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016274700000000001 2023-09-23T23:17:58.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:17:58.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:54.439385+0000", 2023-09-23T23:17:58.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012772900000000001 2023-09-23T23:17:58.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:54.444335+0000", 2023-09-23T23:17:58.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049498160000000001 2023-09-23T23:17:58.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:54.449152+0000", 2023-09-23T23:17:58.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048174730000000001 2023-09-23T23:17:58.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:54.695733+0000", 2023-09-23T23:17:58.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24658053499999999 2023-09-23T23:17:58.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:54.695772+0000", 2023-09-23T23:17:58.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9028e-05 2023-09-23T23:17:58.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:54.695810+0000", 2023-09-23T23:17:58.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8495000000000003e-05 2023-09-23T23:17:58.379 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.379 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.379 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.379 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.379 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:198262 9.6 9:617ef860:::benchmark_data_smithi110_295904_object87081:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e240)", 2023-09-23T23:17:58.379 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:57.025270+0000", 2023-09-23T23:17:58.379 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.191308542, 2023-09-23T23:17:58.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25577254999999999, 2023-09-23T23:17:58.379 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:17:58.379 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:17:58.379 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:17:58.379 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:17:58.380 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:17:58.380 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 198262 2023-09-23T23:17:58.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.380 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:17:58.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:17:58.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:57.025270+0000", 2023-09-23T23:17:58.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:17:58.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:57.025270+0000", 2023-09-23T23:17:58.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:17:58.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:17:58.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:57.025269+0000", 2023-09-23T23:17:58.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:17:58.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:17:58.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:57.025281+0000", 2023-09-23T23:17:58.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2714999999999999e-05 2023-09-23T23:17:58.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:17:58.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:57.025295+0000", 2023-09-23T23:17:58.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3737000000000001e-05 2023-09-23T23:17:58.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:17:58.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:57.025303+0000", 2023-09-23T23:17:58.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8089999999999993e-06 2023-09-23T23:17:58.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:17:58.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:57.025347+0000", 2023-09-23T23:17:58.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4060000000000002e-05 2023-09-23T23:17:58.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:17:58.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:57.025445+0000", 2023-09-23T23:17:58.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7936000000000006e-05 2023-09-23T23:17:58.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,6", 2023-09-23T23:17:58.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:57.025522+0000", 2023-09-23T23:17:58.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7650000000000004e-05 2023-09-23T23:17:58.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:57.029082+0000", 2023-09-23T23:17:58.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003560002 2023-09-23T23:17:58.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.384 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:57.029166+0000", 2023-09-23T23:17:58.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3510999999999994e-05 2023-09-23T23:17:58.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.384 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:17:58.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:57.029315+0000", 2023-09-23T23:17:58.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000149519 2023-09-23T23:17:58.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:17:58.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:57.280871+0000", 2023-09-23T23:17:58.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25155522800000002 2023-09-23T23:17:58.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:17:58.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:57.280926+0000", 2023-09-23T23:17:58.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5359999999999999e-05 2023-09-23T23:17:58.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:17:58.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:17:58.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:17:58.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:57.281043+0000", 2023-09-23T23:17:58.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011654000000000001 2023-09-23T23:17:58.386 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.386 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.386 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.386 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:17:58.386 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:17:58.386 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:17:58.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:58.387 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:17:58.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:17:58.387 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:57 smithi138 ceph-mon[193606]: failed to encode map e240 with expected crc 2023-09-23T23:17:58.388 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:57 smithi138 ceph-mon[193606]: osdmap e240: 8 total, 8 up, 7 in 2023-09-23T23:17:58.388 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:57 smithi138 ceph-mon[193606]: failed to encode map e240 with expected crc 2023-09-23T23:17:58.388 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:57 smithi138 ceph-mon[193606]: failed to encode map e240 with expected crc 2023-09-23T23:17:58.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:57 smithi110 ceph-mon[212939]: failed to encode map e240 with expected crc 2023-09-23T23:17:58.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:57 smithi110 ceph-mon[212939]: osdmap e240: 8 total, 8 up, 7 in 2023-09-23T23:17:58.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:57 smithi110 ceph-mon[212939]: failed to encode map e240 with expected crc 2023-09-23T23:17:58.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:57 smithi110 ceph-mon[212939]: failed to encode map e240 with expected crc 2023-09-23T23:17:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:57 smithi110 ceph-mon[221102]: failed to encode map e240 with expected crc 2023-09-23T23:17:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:57 smithi110 ceph-mon[221102]: osdmap e240: 8 total, 8 up, 7 in 2023-09-23T23:17:58.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:57 smithi110 ceph-mon[221102]: failed to encode map e240 with expected crc 2023-09-23T23:17:58.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:57 smithi110 ceph-mon[221102]: failed to encode map e240 with expected crc 2023-09-23T23:17:58.488 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:17:58.531 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:58.599 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:17:58.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:58.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:17:58.704 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:17:58.823 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:58.911 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:17:59.095 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:17:59.095 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:17:59.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:17:59.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:17:59.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[212939]: pgmap v743: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 91 active+clean, 4 active+remapped+backfill_wait; 1.6 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 367 op/s; 13005/76920 objects misplaced (16.907%); 22 MiB/s, 353 objects/s recovering 2023-09-23T23:17:59.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:17:59.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[212939]: failed to encode map e241 with expected crc 2023-09-23T23:17:59.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[212939]: failed to encode map e241 with expected crc 2023-09-23T23:17:59.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[212939]: osdmap e241: 8 total, 8 up, 7 in 2023-09-23T23:17:59.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[212939]: failed to encode map e241 with expected crc 2023-09-23T23:17:59.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[212939]: failed to encode map e241 with expected crc 2023-09-23T23:17:59.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[212939]: failed to encode map e241 with expected crc 2023-09-23T23:17:59.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3159316308' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:17:59.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:17:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[221102]: pgmap v743: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 91 active+clean, 4 active+remapped+backfill_wait; 1.6 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 367 op/s; 13005/76920 objects misplaced (16.907%); 22 MiB/s, 353 objects/s recovering 2023-09-23T23:17:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:17:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[221102]: failed to encode map e241 with expected crc 2023-09-23T23:17:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[221102]: failed to encode map e241 with expected crc 2023-09-23T23:17:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[221102]: osdmap e241: 8 total, 8 up, 7 in 2023-09-23T23:17:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[221102]: failed to encode map e241 with expected crc 2023-09-23T23:17:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[221102]: failed to encode map e241 with expected crc 2023-09-23T23:17:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[221102]: failed to encode map e241 with expected crc 2023-09-23T23:17:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3159316308' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:17:59.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:58 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:17:59.207 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:17:59.278 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:17:59.362 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:17:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:58 smithi138 ceph-mon[193606]: pgmap v743: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 91 active+clean, 4 active+remapped+backfill_wait; 1.6 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 367 op/s; 13005/76920 objects misplaced (16.907%); 22 MiB/s, 353 objects/s recovering 2023-09-23T23:17:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:17:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:58 smithi138 ceph-mon[193606]: failed to encode map e241 with expected crc 2023-09-23T23:17:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:58 smithi138 ceph-mon[193606]: failed to encode map e241 with expected crc 2023-09-23T23:17:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:58 smithi138 ceph-mon[193606]: osdmap e241: 8 total, 8 up, 7 in 2023-09-23T23:17:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:58 smithi138 ceph-mon[193606]: failed to encode map e241 with expected crc 2023-09-23T23:17:59.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:58 smithi138 ceph-mon[193606]: failed to encode map e241 with expected crc 2023-09-23T23:17:59.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:58 smithi138 ceph-mon[193606]: failed to encode map e241 with expected crc 2023-09-23T23:17:59.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:58 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3159316308' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:17:59.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:58 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:17:59.648 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:18:00.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:59 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:18:00.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:59 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:18:00.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:59 smithi138 ceph-mon[193606]: osdmap e242: 8 total, 8 up, 7 in 2023-09-23T23:18:00.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:17:59 smithi138 ceph-mon[193606]: failed to encode map e242 with expected crc 2023-09-23T23:18:00.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:59 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:18:00.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:59 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:18:00.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:59 smithi110 ceph-mon[221102]: osdmap e242: 8 total, 8 up, 7 in 2023-09-23T23:18:00.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:17:59 smithi110 ceph-mon[221102]: failed to encode map e242 with expected crc 2023-09-23T23:18:00.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:59 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:18:00.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:59 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:18:00.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:59 smithi110 ceph-mon[212939]: osdmap e242: 8 total, 8 up, 7 in 2023-09-23T23:18:00.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:17:59 smithi110 ceph-mon[212939]: failed to encode map e242 with expected crc 2023-09-23T23:18:01.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:00 smithi138 ceph-mon[193606]: failed to encode map e242 with expected crc 2023-09-23T23:18:01.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:00 smithi138 ceph-mon[193606]: failed to encode map e242 with expected crc 2023-09-23T23:18:01.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:00 smithi138 ceph-mon[193606]: pgmap v746: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 11061/71052 objects misplaced (15.567%); 38 MiB/s, 604 objects/s recovering 2023-09-23T23:18:01.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:00 smithi138 ceph-mon[193606]: failed to encode map e243 with expected crc 2023-09-23T23:18:01.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:00 smithi138 ceph-mon[193606]: osdmap e243: 8 total, 8 up, 7 in 2023-09-23T23:18:01.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:00 smithi138 ceph-mon[193606]: failed to encode map e243 with expected crc 2023-09-23T23:18:01.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:00 smithi110 ceph-mon[221102]: failed to encode map e242 with expected crc 2023-09-23T23:18:01.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:00 smithi110 ceph-mon[221102]: failed to encode map e242 with expected crc 2023-09-23T23:18:01.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:00 smithi110 ceph-mon[221102]: pgmap v746: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 11061/71052 objects misplaced (15.567%); 38 MiB/s, 604 objects/s recovering 2023-09-23T23:18:01.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:00 smithi110 ceph-mon[221102]: failed to encode map e243 with expected crc 2023-09-23T23:18:01.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:00 smithi110 ceph-mon[221102]: osdmap e243: 8 total, 8 up, 7 in 2023-09-23T23:18:01.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:00 smithi110 ceph-mon[221102]: failed to encode map e243 with expected crc 2023-09-23T23:18:01.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:00 smithi110 ceph-mon[212939]: failed to encode map e242 with expected crc 2023-09-23T23:18:01.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:00 smithi110 ceph-mon[212939]: failed to encode map e242 with expected crc 2023-09-23T23:18:01.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:00 smithi110 ceph-mon[212939]: pgmap v746: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 11061/71052 objects misplaced (15.567%); 38 MiB/s, 604 objects/s recovering 2023-09-23T23:18:01.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:00 smithi110 ceph-mon[212939]: failed to encode map e243 with expected crc 2023-09-23T23:18:01.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:00 smithi110 ceph-mon[212939]: osdmap e243: 8 total, 8 up, 7 in 2023-09-23T23:18:01.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:00 smithi110 ceph-mon[212939]: failed to encode map e243 with expected crc 2023-09-23T23:18:01.772 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:18:01.772 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:18:01.773 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:01.773 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:200352 9.0 9:0ef21073:::benchmark_data_smithi110_295904_object89171:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-09-23T23:18:01.773 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:18:01.724411+0000", 2023-09-23T23:18:01.773 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.047418185000000002, 2023-09-23T23:18:01.773 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.047447384000000002, 2023-09-23T23:18:01.773 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:01.773 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:18:01.773 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:01.773 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:01.773 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:01.773 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 200352 2023-09-23T23:18:01.773 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:01.774 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:01.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:01.774 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:01.774 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:01.724411+0000", 2023-09-23T23:18:01.774 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:01.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:01.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:01.774 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:01.774 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:01.724411+0000", 2023-09-23T23:18:01.774 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:01.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:01.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:01.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:01.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:01.724410+0000", 2023-09-23T23:18:01.775 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:18:01.775 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:01.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:01.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:01.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:01.724419+0000", 2023-09-23T23:18:01.775 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6880000000000005e-06 2023-09-23T23:18:01.775 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:01.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:01.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:01.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:01.724432+0000", 2023-09-23T23:18:01.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3699999999999999e-05 2023-09-23T23:18:01.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:01.776 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:01.776 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:01.776 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:01.724437+0000", 2023-09-23T23:18:01.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1819999999999997e-06 2023-09-23T23:18:01.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:01.776 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:01.776 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:01.776 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:01.724479+0000", 2023-09-23T23:18:01.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1464999999999997e-05 2023-09-23T23:18:01.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:01.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:01.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:01.777 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:01.724590+0000", 2023-09-23T23:18:01.777 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011065600000000001 2023-09-23T23:18:01.777 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:01.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:01.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:18:01.777 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:01.724692+0000", 2023-09-23T23:18:01.777 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010213900000000001 2023-09-23T23:18:01.777 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:01.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:01.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:01.778 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:01.727334+0000", 2023-09-23T23:18:01.778 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0026426729999999999 2023-09-23T23:18:01.778 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:01.778 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:01.778 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:01.778 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:01.727410+0000", 2023-09-23T23:18:01.778 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5673999999999994e-05 2023-09-23T23:18:01.778 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:01.778 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:01.778 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:01.778 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:01.778 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-09-23T23:18:01.779 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 1 2023-09-23T23:18:01.779 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:18:01.996 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:02.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:02.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:02.102 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:18:02.176 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:02.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:01 smithi138 ceph-mon[193606]: failed to encode map e243 with expected crc 2023-09-23T23:18:02.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:01 smithi138 ceph-mon[193606]: failed to encode map e243 with expected crc 2023-09-23T23:18:02.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:01 smithi110 ceph-mon[212939]: failed to encode map e243 with expected crc 2023-09-23T23:18:02.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:01 smithi110 ceph-mon[212939]: failed to encode map e243 with expected crc 2023-09-23T23:18:02.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:01 smithi110 ceph-mon[221102]: failed to encode map e243 with expected crc 2023-09-23T23:18:02.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:01 smithi110 ceph-mon[221102]: failed to encode map e243 with expected crc 2023-09-23T23:18:02.598 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:18:02.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:02.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:18:02.705 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:18:02.766 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:02.845 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:18:02.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:02.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:18:02.855 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:18:02.951 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:03.053 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:02 smithi138 ceph-mon[193606]: pgmap v748: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 1.4 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 10313/69402 objects misplaced (14.860%); 22 MiB/s, 349 objects/s recovering 2023-09-23T23:18:03.434 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:18:03.441 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:03.441 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:18:03.443 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:02 smithi110 ceph-mon[212939]: pgmap v748: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 1.4 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 10313/69402 objects misplaced (14.860%); 22 MiB/s, 349 objects/s recovering 2023-09-23T23:18:03.444 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:02 smithi110 ceph-mon[221102]: pgmap v748: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 1.4 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 10313/69402 objects misplaced (14.860%); 22 MiB/s, 349 objects/s recovering 2023-09-23T23:18:03.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:03.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:03.445 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:18:03.488 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:18:03.535 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:18:03.545 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:03.563 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:18:03.596 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:18:03.620 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:03.628 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:18:03.675 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:18:03.704 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:18:03.704 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:18:03.704 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:18:03.704 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:18:03.704 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:18:03.704 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:18:03.704 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:18:03.704 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:18:03.755 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:03.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:03.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:03.868 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:18:03.938 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:04.065 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:18:04.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:04.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:18:04.182 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:18:04.210 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:04 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3372133819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:04 smithi138 ceph-mon[193606]: pgmap v749: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 1.4 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 10313/68493 objects misplaced (15.057%); 16 MiB/s, 259 objects/s recovering 2023-09-23T23:18:04.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:04 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3372133819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:04.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:04 smithi110 ceph-mon[212939]: pgmap v749: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 1.4 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 10313/68493 objects misplaced (15.057%); 16 MiB/s, 259 objects/s recovering 2023-09-23T23:18:04.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:04 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3372133819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:04.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:04 smithi110 ceph-mon[221102]: pgmap v749: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 92 active+clean, 3 active+remapped+backfill_wait; 1.4 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 10313/68493 objects misplaced (15.057%); 16 MiB/s, 259 objects/s recovering 2023-09-23T23:18:04.499 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:18:05.367 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:18:05.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:05.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:18:05.479 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:18:05.482 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:18:05.483 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:18:05.558 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:05.715 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:18:05.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:05.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:18:05.819 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:18:05.861 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:05.941 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:05.948 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:18:05.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:05.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:18:06.055 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:06.173 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:06.348 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:06.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:06.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:06.440 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:18:06.460 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:06.549 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:06.652 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:06.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:06.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:06 smithi138 ceph-mon[193606]: pgmap v750: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 93 active+clean, 3 active+remapped+backfill_wait; 1.3 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 8921/63582 objects misplaced (14.031%); 20 MiB/s, 313 objects/s recovering 2023-09-23T23:18:06.710 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:06 smithi110 ceph-mon[212939]: pgmap v750: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 93 active+clean, 3 active+remapped+backfill_wait; 1.3 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 8921/63582 objects misplaced (14.031%); 20 MiB/s, 313 objects/s recovering 2023-09-23T23:18:06.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:06 smithi110 ceph-mon[221102]: pgmap v750: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 93 active+clean, 3 active+remapped+backfill_wait; 1.3 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 8921/63582 objects misplaced (14.031%); 20 MiB/s, 313 objects/s recovering 2023-09-23T23:18:06.769 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:06.846 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:06.958 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:18:06.961 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:18:06.965 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:18:07.000 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:07.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:07.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:07.116 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:18:07.182 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:18:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:18:06] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-09-23T23:18:07.189 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:07.739 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:18:07.823 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:07 smithi110 ceph-mon[212939]: from='client.55375 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:18:07.823 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:07 smithi110 ceph-mon[221102]: from='client.55375 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:18:07.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:07 smithi138 ceph-mon[193606]: from='client.55375 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:18:07.989 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:18:07.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:08.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:18:08.101 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:18:08.157 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:08.428 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:18:08.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:08.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:18:08.546 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:18:08.586 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:08.629 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:18:08.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:08.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:18:08.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:08 smithi138 ceph-mon[193606]: pgmap v751: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 93 active+clean, 3 active+remapped+backfill_wait; 1.3 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 408 op/s; 8238/61584 objects misplaced (13.377%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T23:18:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:08 smithi138 ceph-mon[193606]: osdmap e244: 8 total, 8 up, 7 in 2023-09-23T23:18:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:08 smithi138 ceph-mon[193606]: failed to encode map e244 with expected crc 2023-09-23T23:18:08.732 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:08.769 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:08.891 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:08 smithi110 ceph-mon[212939]: pgmap v751: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 93 active+clean, 3 active+remapped+backfill_wait; 1.3 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 408 op/s; 8238/61584 objects misplaced (13.377%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T23:18:08.891 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:08 smithi110 ceph-mon[212939]: osdmap e244: 8 total, 8 up, 7 in 2023-09-23T23:18:08.891 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:08 smithi110 ceph-mon[212939]: failed to encode map e244 with expected crc 2023-09-23T23:18:08.892 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:08 smithi110 ceph-mon[221102]: pgmap v751: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 93 active+clean, 3 active+remapped+backfill_wait; 1.3 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 408 op/s; 8238/61584 objects misplaced (13.377%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T23:18:08.892 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:08 smithi110 ceph-mon[221102]: osdmap e244: 8 total, 8 up, 7 in 2023-09-23T23:18:08.892 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:08 smithi110 ceph-mon[221102]: failed to encode map e244 with expected crc 2023-09-23T23:18:08.928 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:18:08.928 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:18:08.945 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:08.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:08.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:08.957 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":750,"stamp":"2023-09-23T23:18:05.424794+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1387825801,"num_objects":21194,"num_object_clones":0,"num_object_copies":63582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":8921,"num_objects_unfound":0,"num_objects_dirty":21194,"num_whiteouts":0,"num_read":134,"num_read_kb":249,"num_write":201314,"num_write_kb":7116080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32740,"num_bytes_recovered":1967481894,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116102,"ondisk_log_size":116102,"up":297,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":311,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8296116,"kb_used_data":5454908,"kb_used_omap":12,"kb_used_meta":2840883,"kb_avail":647919948,"statfs":{"total":671965249536,"available":663470026752,"internally_reserved":0,"allocated":5585825792,"data_stored":5579880769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12739,"internal_metadata":2909064765},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,1,3,1,0,0,0,8,2,12],"upper_bound":2048},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-269418496,"num_objects":-4111,"num_object_clones":0,"num_object_copies":-12333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-4084,"num_objects_unfound":0,"num_objects_dirty":-4111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4111,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2336,"num_bytes_recovered":153026560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.452447"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":249,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.353900+0000","last_change":"2023-09-23T23:17:48.320558+0000","last_active":"2023-09-23T23:17:59.353900+0000","last_peered":"2023-09-23T23:17:59.353900+0000","last_clean":"2023-09-23T23:17:59.353900+0000","last_became_active":"2023-09-23T23:17:48.320205+0000","last_became_peered":"2023-09-23T23:17:48.320205+0000","last_unstale":"2023-09-23T23:17:59.353900+0000","last_undegraded":"2023-09-23T23:17:59.353900+0000","last_fullsized":"2023-09-23T23:17:59.353900+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:24.708401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":182,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.365883+0000","last_change":"2023-09-23T23:10:59.514211+0000","last_active":"2023-09-23T23:17:59.365883+0000","last_peered":"2023-09-23T23:17:59.365883+0000","last_clean":"2023-09-23T23:17:59.365883+0000","last_became_active":"2023-09-23T23:10:59.513917+0000","last_became_peered":"2023-09-23T23:10:59.513917+0000","last_unstale":"2023-09-23T23:17:59.365883+0000","last_undegraded":"2023-09-23T23:17:59.365883+0000","last_fullsized":"2023-09-23T23:17:59.365883+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:55.533586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":213,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.345258+0000","last_change":"2023-09-23T23:10:59.505490+0000","last_active":"2023-09-23T23:17:59.345258+0000","last_peered":"2023-09-23T23:17:59.345258+0000","last_clean":"2023-09-23T23:17:59.345258+0000","last_became_active":"2023-09-23T23:10:59.504874+0000","last_became_peered":"2023-09-23T23:10:59.504874+0000","last_unstale":"2023-09-23T23:17:59.345258+0000","last_undegraded":"2023-09-23T23:17:59.345258+0000","last_fullsized":"2023-09-23T23:17:59.345258+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:15.279668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":212,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.350896+0000","last_change":"2023-09-23T23:17:33.437874+0000","last_active":"2023-09-23T23:17:59.350896+0000","last_peered":"2023-09-23T23:17:59.350896+0000","last_clean":"2023-09-23T23:17:59.350896+0000","last_became_active":"2023-09-23T23:17:33.437272+0000","last_became_peered":"2023-09-23T23:17:33.437272+0000","last_unstale":"2023-09-23T23:17:59.350896+0000","last_undegraded":"2023-09-23T23:17:59.350896+0000","last_fullsized":"2023-09-23T23:17:59.350896+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:34.600738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":218,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.364057+0000","last_change":"2023-09-23T23:17:48.305487+0000","last_active":"2023-09-23T23:17:59.364057+0000","last_peered":"2023-09-23T23:17:59.364057+0000","last_clean":"2023-09-23T23:17:59.364057+0000","last_became_active":"2023-09-23T23:17:48.305118+0000","last_became_peered":"2023-09-23T23:17:48.305118+0000","last_unstale":"2023-09-23T23:17:59.364057+0000","last_undegraded":"2023-09-23T23:17:59.364057+0000","last_fullsized":"2023-09-23T23:17:59.364057+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:07.958152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":203,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.353332+0000","last_change":"2023-09-23T23:07:30.078179+0000","last_active":"2023-09-23T23:17:59.353332+0000","last_peered":"2023-09-23T23:17:59.353332+0000","last_clean":"2023-09-23T23:17:59.353332+0000","last_became_active":"2023-09-23T23:07:30.077810+0000","last_became_peered":"2023-09-23T23:07:30.077810+0000","last_unstale":"2023-09-23T23:17:59.353332+0000","last_undegraded":"2023-09-23T23:17:59.353332+0000","last_fullsized":"2023-09-23T23:17:59.353332+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:19.218333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":193,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.349976+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:17:59.349976+0000","last_peered":"2023-09-23T23:17:59.349976+0000","last_clean":"2023-09-23T23:17:59.349976+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:17:59.349976+0000","last_undegraded":"2023-09-23T23:17:59.349976+0000","last_fullsized":"2023-09-23T23:17:59.349976+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":183,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.363981+0000","last_change":"2023-09-23T23:10:59.515840+0000","last_active":"2023-09-23T23:17:59.363981+0000","last_peered":"2023-09-23T23:17:59.363981+0000","last_clean":"2023-09-23T23:17:59.363981+0000","last_became_active":"2023-09-23T23:10:59.515000+0000","last_became_peered":"2023-09-23T23:10:59.515000+0000","last_unstale":"2023-09-23T23:17:59.363981+0000","last_undegraded":"2023-09-23T23:17:59.363981+0000","last_fullsized":"2023-09-23T23:17:59.363981+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:47.355624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":20,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.350642+0000","last_change":"2023-09-23T23:17:48.308746+0000","last_active":"2023-09-23T23:17:59.350642+0000","last_peered":"2023-09-23T23:17:59.350642+0000","last_clean":"2023-09-23T23:17:59.350642+0000","last_became_active":"2023-09-23T23:17:48.307974+0000","last_became_peered":"2023-09-23T23:17:48.307974+0000","last_unstale":"2023-09-23T23:17:59.350642+0000","last_undegraded":"2023-09-23T23:17:59.350642+0000","last_fullsized":"2023-09-23T23:17:59.350642+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:52.915506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":171,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.376351+0000","last_change":"2023-09-23T23:16:56.528057+0000","last_active":"2023-09-23T23:17:59.376351+0000","last_peered":"2023-09-23T23:17:59.376351+0000","last_clean":"2023-09-23T23:17:59.376351+0000","last_became_active":"2023-09-23T23:16:56.527755+0000","last_became_peered":"2023-09-23T23:16:56.527755+0000","last_unstale":"2023-09-23T23:17:59.376351+0000","last_undegraded":"2023-09-23T23:17:59.376351+0000","last_fullsized":"2023-09-23T23:17:59.376351+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:47.396406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":59,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.366058+0000","last_change":"2023-09-23T23:16:56.567128+0000","last_active":"2023-09-23T23:17:59.366058+0000","last_peered":"2023-09-23T23:17:59.366058+0000","last_clean":"2023-09-23T23:17:59.366058+0000","last_became_active":"2023-09-23T23:16:56.551484+0000","last_became_peered":"2023-09-23T23:16:56.551484+0000","last_unstale":"2023-09-23T23:17:59.366058+0000","last_undegraded":"2023-09-23T23:17:59.366058+0000","last_fullsized":"2023-09-23T23:17:59.366058+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:14.981307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":183,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.351784+0000","last_change":"2023-09-23T23:17:48.321527+0000","last_active":"2023-09-23T23:17:59.351784+0000","last_peered":"2023-09-23T23:17:59.351784+0000","last_clean":"2023-09-23T23:17:59.351784+0000","last_became_active":"2023-09-23T23:17:48.321198+0000","last_became_peered":"2023-09-23T23:17:48.321198+0000","last_unstale":"2023-09-23T23:17:59.351784+0000","last_undegraded":"2023-09-23T23:17:59.351784+0000","last_fullsized":"2023-09-23T23:17:59.351784+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:11.289469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":209,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.354329+0000","last_change":"2023-09-23T22:53:16.891930+0000","last_active":"2023-09-23T23:17:59.354329+0000","last_peered":"2023-09-23T23:17:59.354329+0000","last_clean":"2023-09-23T23:17:59.354329+0000","last_became_active":"2023-09-23T22:53:16.891658+0000","last_became_peered":"2023-09-23T22:53:16.891658+0000","last_unstale":"2023-09-23T23:17:59.354329+0000","last_undegraded":"2023-09-23T23:17:59.354329+0000","last_fullsized":"2023-09-23T23:17:59.354329+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:19.762775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":203,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.355993+0000","last_change":"2023-09-23T23:07:30.082476+0000","last_active":"2023-09-23T23:17:59.355993+0000","last_peered":"2023-09-23T23:17:59.355993+0000","last_clean":"2023-09-23T23:17:59.355993+0000","last_became_active":"2023-09-23T23:07:30.082201+0000","last_became_peered":"2023-09-23T23:07:30.082201+0000","last_unstale":"2023-09-23T23:17:59.355993+0000","last_undegraded":"2023-09-23T23:17:59.355993+0000","last_fullsized":"2023-09-23T23:17:59.355993+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:08.809163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":239,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.345453+0000","last_change":"2023-09-23T23:17:48.325540+0000","last_active":"2023-09-23T23:17:59.345453+0000","last_peered":"2023-09-23T23:17:59.345453+0000","last_clean":"2023-09-23T23:17:59.345453+0000","last_became_active":"2023-09-23T23:17:48.301566+0000","last_became_peered":"2023-09-23T23:17:48.301566+0000","last_unstale":"2023-09-23T23:17:59.345453+0000","last_undegraded":"2023-09-23T23:17:59.345453+0000","last_fullsized":"2023-09-23T23:17:59.345453+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:14.833073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":201,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.367013+0000","last_change":"2023-09-23T23:17:48.304131+0000","last_active":"2023-09-23T23:17:59.367013+0000","last_peered":"2023-09-23T23:17:59.367013+0000","last_clean":"2023-09-23T23:17:59.367013+0000","last_became_active":"2023-09-23T23:17:48.303720+0000","last_became_peered":"2023-09-23T23:17:48.303720+0000","last_unstale":"2023-09-23T23:17:59.367013+0000","last_undegraded":"2023-09-23T23:17:59.367013+0000","last_fullsized":"2023-09-23T23:17:59.367013+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:04.726467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":211,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.352128+0000","last_change":"2023-09-23T23:17:48.303810+0000","last_active":"2023-09-23T23:17:59.352128+0000","last_peered":"2023-09-23T23:17:59.352128+0000","last_clean":"2023-09-23T23:17:59.352128+0000","last_became_active":"2023-09-23T23:17:48.303505+0000","last_became_peered":"2023-09-23T23:17:48.303505+0000","last_unstale":"2023-09-23T23:17:59.352128+0000","last_undegraded":"2023-09-23T23:17:59.352128+0000","last_fullsized":"2023-09-23T23:17:59.352128+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:31.494085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":190,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.345865+0000","last_change":"2023-09-23T23:13:29.360403+0000","last_active":"2023-09-23T23:17:59.345865+0000","last_peered":"2023-09-23T23:17:59.345865+0000","last_clean":"2023-09-23T23:17:59.345865+0000","last_became_active":"2023-09-23T23:13:29.359519+0000","last_became_peered":"2023-09-23T23:13:29.359519+0000","last_unstale":"2023-09-23T23:17:59.345865+0000","last_undegraded":"2023-09-23T23:17:59.345865+0000","last_fullsized":"2023-09-23T23:17:59.345865+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:27.911252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":187,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.345344+0000","last_change":"2023-09-23T22:53:16.889362+0000","last_active":"2023-09-23T23:17:59.345344+0000","last_peered":"2023-09-23T23:17:59.345344+0000","last_clean":"2023-09-23T23:17:59.345344+0000","last_became_active":"2023-09-23T22:53:16.889081+0000","last_became_peered":"2023-09-23T22:53:16.889081+0000","last_unstale":"2023-09-23T23:17:59.345344+0000","last_undegraded":"2023-09-23T23:17:59.345344+0000","last_fullsized":"2023-09-23T23:17:59.345344+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:11.321151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":211,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.351034+0000","last_change":"2023-09-23T23:17:48.308900+0000","last_active":"2023-09-23T23:17:59.351034+0000","last_peered":"2023-09-23T23:17:59.351034+0000","last_clean":"2023-09-23T23:17:59.351034+0000","last_became_active":"2023-09-23T23:17:48.308475+0000","last_became_peered":"2023-09-23T23:17:48.308475+0000","last_unstale":"2023-09-23T23:17:59.351034+0000","last_undegraded":"2023-09-23T23:17:59.351034+0000","last_fullsized":"2023-09-23T23:17:59.351034+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:38.158435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":232,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.351999+0000","last_change":"2023-09-23T23:13:29.368175+0000","last_active":"2023-09-23T23:17:59.351999+0000","last_peered":"2023-09-23T23:17:59.351999+0000","last_clean":"2023-09-23T23:17:59.351999+0000","last_became_active":"2023-09-23T23:13:29.363603+0000","last_became_peered":"2023-09-23T23:13:29.363603+0000","last_unstale":"2023-09-23T23:17:59.351999+0000","last_undegraded":"2023-09-23T23:17:59.351999+0000","last_fullsized":"2023-09-23T23:17:59.351999+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:25.277871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":128,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.347859+0000","last_change":"2023-09-23T23:17:48.306755+0000","last_active":"2023-09-23T23:17:59.347859+0000","last_peered":"2023-09-23T23:17:59.347859+0000","last_clean":"2023-09-23T23:17:59.347859+0000","last_became_active":"2023-09-23T23:17:48.306455+0000","last_became_peered":"2023-09-23T23:17:48.306455+0000","last_unstale":"2023-09-23T23:17:59.347859+0000","last_undegraded":"2023-09-23T23:17:59.347859+0000","last_fullsized":"2023-09-23T23:17:59.347859+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:05.630787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.5","version":"243'12495","reported_seq":27228,"reported_epoch":243,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T23:18:01.386059+0000","last_change":"2023-09-23T23:17:48.367302+0000","last_active":"2023-09-23T23:18:01.386059+0000","last_peered":"2023-09-23T23:18:01.386059+0000","last_clean":"2023-09-23T23:15:27.538246+0000","last_became_active":"2023-09-23T23:15:29.635703+0000","last_became_peered":"2023-09-23T23:15:29.635703+0000","last_unstale":"2023-09-23T23:18:01.386059+0000","last_undegraded":"2023-09-23T23:18:01.386059+0000","last_fullsized":"2023-09-23T23:15:29.226857+0000","mapping_epoch":187,"log_start":"182'6063","ondisk_log_start":"182'6063","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":6432,"log_dups_size":0,"ondisk_log_size":6432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:32:33.086246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89587712,"num_objects":1367,"num_object_clones":0,"num_object_copies":4101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2050,"num_objects_unfound":0,"num_objects_dirty":1367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12495,"num_write_kb":443584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4875,"num_bytes_recovered":319488000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,1","objects":342},{"shards":"0,5,7","objects":1025}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":58,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.354468+0000","last_change":"2023-09-23T23:16:56.548535+0000","last_active":"2023-09-23T23:17:59.354468+0000","last_peered":"2023-09-23T23:17:59.354468+0000","last_clean":"2023-09-23T23:17:59.354468+0000","last_became_active":"2023-09-23T23:16:56.546115+0000","last_became_peered":"2023-09-23T23:16:56.546115+0000","last_unstale":"2023-09-23T23:17:59.354468+0000","last_undegraded":"2023-09-23T23:17:59.354468+0000","last_fullsized":"2023-09-23T23:17:59.354468+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:02.279715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":48,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.352088+0000","last_change":"2023-09-23T23:16:56.525479+0000","last_active":"2023-09-23T23:17:59.352088+0000","last_peered":"2023-09-23T23:17:59.352088+0000","last_clean":"2023-09-23T23:17:59.352088+0000","last_became_active":"2023-09-23T23:16:56.525147+0000","last_became_peered":"2023-09-23T23:16:56.525147+0000","last_unstale":"2023-09-23T23:17:59.352088+0000","last_undegraded":"2023-09-23T23:17:59.352088+0000","last_fullsized":"2023-09-23T23:17:59.352088+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:06.165300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"243'12989","reported_seq":13079,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T23:18:03.824045+0000","last_change":"2023-09-23T23:13:28.360159+0000","last_active":"2023-09-23T23:18:03.824045+0000","last_peered":"2023-09-23T23:18:03.824045+0000","last_clean":"2023-09-23T23:18:03.824045+0000","last_became_active":"2023-09-23T23:13:28.359915+0000","last_became_peered":"2023-09-23T23:13:28.359915+0000","last_unstale":"2023-09-23T23:18:03.824045+0000","last_undegraded":"2023-09-23T23:18:03.824045+0000","last_fullsized":"2023-09-23T23:18:03.824045+0000","mapping_epoch":164,"log_start":"177'5775","ondisk_log_start":"177'5775","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":7214,"log_dups_size":0,"ondisk_log_size":7214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:48.297395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86573056,"num_objects":1321,"num_object_clones":0,"num_object_copies":3963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12989,"num_write_kb":457920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":58,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.355714+0000","last_change":"2023-09-23T23:16:56.532271+0000","last_active":"2023-09-23T23:17:59.355714+0000","last_peered":"2023-09-23T23:17:59.355714+0000","last_clean":"2023-09-23T23:17:59.355714+0000","last_became_active":"2023-09-23T23:16:56.527347+0000","last_became_peered":"2023-09-23T23:16:56.527347+0000","last_unstale":"2023-09-23T23:17:59.355714+0000","last_undegraded":"2023-09-23T23:17:59.355714+0000","last_fullsized":"2023-09-23T23:17:59.355714+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:30:26.607017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":230,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.356099+0000","last_change":"2023-09-23T23:16:56.527215+0000","last_active":"2023-09-23T23:17:59.356099+0000","last_peered":"2023-09-23T23:17:59.356099+0000","last_clean":"2023-09-23T23:17:59.356099+0000","last_became_active":"2023-09-23T23:16:56.526551+0000","last_became_peered":"2023-09-23T23:16:56.526551+0000","last_unstale":"2023-09-23T23:17:59.356099+0000","last_undegraded":"2023-09-23T23:17:59.356099+0000","last_fullsized":"2023-09-23T23:17:59.356099+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:56:36.991437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"243'12550","reported_seq":12669,"reported_epoch":243,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:18:03.937362+0000","last_change":"2023-09-23T23:17:48.493805+0000","last_active":"2023-09-23T23:18:03.937362+0000","last_peered":"2023-09-23T23:18:03.937362+0000","last_clean":"2023-09-23T23:16:43.190306+0000","last_became_active":"2023-09-23T23:17:48.445289+0000","last_became_peered":"2023-09-23T23:17:48.445289+0000","last_unstale":"2023-09-23T23:18:03.937362+0000","last_undegraded":"2023-09-23T23:18:03.937362+0000","last_fullsized":"2023-09-23T23:18:03.937362+0000","mapping_epoch":235,"log_start":"172'3975","ondisk_log_start":"172'3975","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":8575,"log_dups_size":0,"ondisk_log_size":8575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:24.220993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82575360,"num_objects":1260,"num_object_clones":0,"num_object_copies":3780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2520,"num_objects_unfound":0,"num_objects_dirty":1260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12550,"num_write_kb":441920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,3,7],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1260}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":202,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.345670+0000","last_change":"2023-09-23T23:13:29.362820+0000","last_active":"2023-09-23T23:17:59.345670+0000","last_peered":"2023-09-23T23:17:59.345670+0000","last_clean":"2023-09-23T23:17:59.345670+0000","last_became_active":"2023-09-23T23:13:29.360151+0000","last_became_peered":"2023-09-23T23:13:29.360151+0000","last_unstale":"2023-09-23T23:17:59.345670+0000","last_undegraded":"2023-09-23T23:17:59.345670+0000","last_fullsized":"2023-09-23T23:17:59.345670+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:36.242850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":250,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.352021+0000","last_change":"2023-09-23T23:17:48.317798+0000","last_active":"2023-09-23T23:17:59.352021+0000","last_peered":"2023-09-23T23:17:59.352021+0000","last_clean":"2023-09-23T23:17:59.352021+0000","last_became_active":"2023-09-23T23:17:48.303959+0000","last_became_peered":"2023-09-23T23:17:48.303959+0000","last_unstale":"2023-09-23T23:17:59.352021+0000","last_undegraded":"2023-09-23T23:17:59.352021+0000","last_fullsized":"2023-09-23T23:17:59.352021+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:18.888131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"243'12755","reported_seq":14731,"reported_epoch":243,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T23:18:03.956533+0000","last_change":"2023-09-23T23:17:49.552705+0000","last_active":"2023-09-23T23:18:03.956533+0000","last_peered":"2023-09-23T23:18:03.956533+0000","last_clean":"2023-09-23T23:17:47.874547+0000","last_became_active":"2023-09-23T23:17:49.442103+0000","last_became_peered":"2023-09-23T23:17:49.442103+0000","last_unstale":"2023-09-23T23:18:03.956533+0000","last_undegraded":"2023-09-23T23:18:03.956533+0000","last_fullsized":"2023-09-23T23:18:03.956533+0000","mapping_epoch":236,"log_start":"172'4132","ondisk_log_start":"172'4132","created":164,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":8623,"log_dups_size":0,"ondisk_log_size":8623,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:18.526356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87228416,"num_objects":1331,"num_object_clones":0,"num_object_copies":3993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1331,"num_objects_unfound":0,"num_objects_dirty":1331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12755,"num_write_kb":450752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":925,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":1331}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":204,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.343912+0000","last_change":"2023-09-23T23:10:59.502426+0000","last_active":"2023-09-23T23:17:59.343912+0000","last_peered":"2023-09-23T23:17:59.343912+0000","last_clean":"2023-09-23T23:17:59.343912+0000","last_became_active":"2023-09-23T23:10:59.502074+0000","last_became_peered":"2023-09-23T23:10:59.502074+0000","last_unstale":"2023-09-23T23:17:59.343912+0000","last_undegraded":"2023-09-23T23:17:59.343912+0000","last_fullsized":"2023-09-23T23:17:59.343912+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:46.029347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":197,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.348814+0000","last_change":"2023-09-23T23:04:11.912493+0000","last_active":"2023-09-23T23:17:59.348814+0000","last_peered":"2023-09-23T23:17:59.348814+0000","last_clean":"2023-09-23T23:17:59.348814+0000","last_became_active":"2023-09-23T23:04:11.906712+0000","last_became_peered":"2023-09-23T23:04:11.906712+0000","last_unstale":"2023-09-23T23:17:59.348814+0000","last_undegraded":"2023-09-23T23:17:59.348814+0000","last_fullsized":"2023-09-23T23:17:59.348814+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:58.932610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":661,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.345438+0000","last_change":"2023-09-23T23:17:51.351926+0000","last_active":"2023-09-23T23:17:59.345438+0000","last_peered":"2023-09-23T23:17:59.345438+0000","last_clean":"2023-09-23T23:17:59.345438+0000","last_became_active":"2023-09-23T23:17:51.351586+0000","last_became_peered":"2023-09-23T23:17:51.351586+0000","last_unstale":"2023-09-23T23:17:59.345438+0000","last_undegraded":"2023-09-23T23:17:59.345438+0000","last_fullsized":"2023-09-23T23:17:59.345438+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:46.150686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:46.150686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:52.403435+0000","scrub_duration":0.0088912260000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.1","version":"243'12649","reported_seq":18634,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T23:18:02.982554+0000","last_change":"2023-09-23T23:17:15.459960+0000","last_active":"2023-09-23T23:18:02.982554+0000","last_peered":"2023-09-23T23:18:02.982554+0000","last_clean":"2023-09-23T23:18:02.982554+0000","last_became_active":"2023-09-23T23:17:15.424151+0000","last_became_peered":"2023-09-23T23:17:15.424151+0000","last_unstale":"2023-09-23T23:18:02.982554+0000","last_undegraded":"2023-09-23T23:18:02.982554+0000","last_fullsized":"2023-09-23T23:18:02.982554+0000","mapping_epoch":218,"log_start":"182'6247","ondisk_log_start":"182'6247","created":164,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":6402,"log_dups_size":0,"ondisk_log_size":6402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:21.856597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87752704,"num_objects":1339,"num_object_clones":0,"num_object_copies":4017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12649,"num_write_kb":447616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2935,"num_bytes_recovered":192348160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":207,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.373810+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:17:59.373810+0000","last_peered":"2023-09-23T23:17:59.373810+0000","last_clean":"2023-09-23T23:17:59.373810+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:17:59.373810+0000","last_undegraded":"2023-09-23T23:17:59.373810+0000","last_fullsized":"2023-09-23T23:17:59.373810+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":203,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.354270+0000","last_change":"2023-09-23T23:10:59.512363+0000","last_active":"2023-09-23T23:17:59.354270+0000","last_peered":"2023-09-23T23:17:59.354270+0000","last_clean":"2023-09-23T23:17:59.354270+0000","last_became_active":"2023-09-23T23:10:59.512022+0000","last_became_peered":"2023-09-23T23:10:59.512022+0000","last_unstale":"2023-09-23T23:17:59.354270+0000","last_undegraded":"2023-09-23T23:17:59.354270+0000","last_fullsized":"2023-09-23T23:17:59.354270+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:45.523432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":442,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.363906+0000","last_change":"2023-09-23T23:17:51.331237+0000","last_active":"2023-09-23T23:17:59.363906+0000","last_peered":"2023-09-23T23:17:59.363906+0000","last_clean":"2023-09-23T23:17:59.363906+0000","last_became_active":"2023-09-23T23:17:51.330912+0000","last_became_peered":"2023-09-23T23:17:51.330912+0000","last_unstale":"2023-09-23T23:17:59.363906+0000","last_undegraded":"2023-09-23T23:17:59.363906+0000","last_fullsized":"2023-09-23T23:17:59.363906+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:47.047289+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:47.047289+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.0","version":"243'12442","reported_seq":22557,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T23:18:04.169454+0000","last_change":"2023-09-23T23:17:58.043771+0000","last_active":"2023-09-23T23:18:04.169454+0000","last_peered":"2023-09-23T23:18:04.169454+0000","last_clean":"2023-09-23T23:18:04.169454+0000","last_became_active":"2023-09-23T23:17:57.984332+0000","last_became_peered":"2023-09-23T23:17:57.984332+0000","last_unstale":"2023-09-23T23:18:04.169454+0000","last_undegraded":"2023-09-23T23:18:04.169454+0000","last_fullsized":"2023-09-23T23:18:04.169454+0000","mapping_epoch":240,"log_start":"184'7105","ondisk_log_start":"184'7105","created":164,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":5337,"log_dups_size":0,"ondisk_log_size":5337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:02.161575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87031808,"num_objects":1328,"num_object_clones":0,"num_object_copies":3984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12442,"num_write_kb":440640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4976,"num_bytes_recovered":269615104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":214,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.352092+0000","last_change":"2023-09-23T23:16:56.525904+0000","last_active":"2023-09-23T23:17:59.352092+0000","last_peered":"2023-09-23T23:17:59.352092+0000","last_clean":"2023-09-23T23:17:59.352092+0000","last_became_active":"2023-09-23T23:16:56.525372+0000","last_became_peered":"2023-09-23T23:16:56.525372+0000","last_unstale":"2023-09-23T23:17:59.352092+0000","last_undegraded":"2023-09-23T23:17:59.352092+0000","last_fullsized":"2023-09-23T23:17:59.352092+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:17.070117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":247,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.354463+0000","last_change":"2023-09-23T23:17:48.319835+0000","last_active":"2023-09-23T23:17:59.354463+0000","last_peered":"2023-09-23T23:17:59.354463+0000","last_clean":"2023-09-23T23:17:59.354463+0000","last_became_active":"2023-09-23T23:17:48.319535+0000","last_became_peered":"2023-09-23T23:17:48.319535+0000","last_unstale":"2023-09-23T23:17:59.354463+0000","last_undegraded":"2023-09-23T23:17:59.354463+0000","last_fullsized":"2023-09-23T23:17:59.354463+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:45:06.176314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"243'12597","reported_seq":12684,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T23:18:03.971467+0000","last_change":"2023-09-23T23:13:28.356430+0000","last_active":"2023-09-23T23:18:03.971467+0000","last_peered":"2023-09-23T23:18:03.971467+0000","last_clean":"2023-09-23T23:18:03.971467+0000","last_became_active":"2023-09-23T23:13:28.356172+0000","last_became_peered":"2023-09-23T23:13:28.356172+0000","last_unstale":"2023-09-23T23:18:03.971467+0000","last_undegraded":"2023-09-23T23:18:03.971467+0000","last_fullsized":"2023-09-23T23:18:03.971467+0000","mapping_epoch":164,"log_start":"171'3700","ondisk_log_start":"171'3700","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":8897,"log_dups_size":0,"ondisk_log_size":8897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:06.046311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84475904,"num_objects":1289,"num_object_clones":0,"num_object_copies":3867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12597,"num_write_kb":444352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":231,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.355986+0000","last_change":"2023-09-23T23:17:48.319410+0000","last_active":"2023-09-23T23:17:59.355986+0000","last_peered":"2023-09-23T23:17:59.355986+0000","last_clean":"2023-09-23T23:17:59.355986+0000","last_became_active":"2023-09-23T23:17:48.303382+0000","last_became_peered":"2023-09-23T23:17:48.303382+0000","last_unstale":"2023-09-23T23:17:59.355986+0000","last_undegraded":"2023-09-23T23:17:59.355986+0000","last_fullsized":"2023-09-23T23:17:59.355986+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:09.313879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":213,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.355778+0000","last_change":"2023-09-23T23:15:28.228439+0000","last_active":"2023-09-23T23:17:59.355778+0000","last_peered":"2023-09-23T23:17:59.355778+0000","last_clean":"2023-09-23T23:17:59.355778+0000","last_became_active":"2023-09-23T23:15:28.228095+0000","last_became_peered":"2023-09-23T23:15:28.228095+0000","last_unstale":"2023-09-23T23:17:59.355778+0000","last_undegraded":"2023-09-23T23:17:59.355778+0000","last_fullsized":"2023-09-23T23:17:59.355778+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:37.667201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":523,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.343638+0000","last_change":"2023-09-23T23:17:51.348000+0000","last_active":"2023-09-23T23:17:59.343638+0000","last_peered":"2023-09-23T23:17:59.343638+0000","last_clean":"2023-09-23T23:17:59.343638+0000","last_became_active":"2023-09-23T23:17:51.347662+0000","last_became_peered":"2023-09-23T23:17:51.347662+0000","last_unstale":"2023-09-23T23:17:59.343638+0000","last_undegraded":"2023-09-23T23:17:59.343638+0000","last_fullsized":"2023-09-23T23:17:59.343638+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:51.734652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:51.734652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:38.179876+0000","scrub_duration":0.023100959000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"243'12769","reported_seq":22555,"reported_epoch":243,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T23:18:03.899024+0000","last_change":"2023-09-23T23:18:00.694350+0000","last_active":"2023-09-23T23:18:03.899024+0000","last_peered":"2023-09-23T23:18:03.899024+0000","last_clean":"2023-09-23T23:17:47.921683+0000","last_became_active":"2023-09-23T23:17:49.412072+0000","last_became_peered":"2023-09-23T23:17:49.412072+0000","last_unstale":"2023-09-23T23:18:03.899024+0000","last_undegraded":"2023-09-23T23:18:03.899024+0000","last_fullsized":"2023-09-23T23:18:03.899024+0000","mapping_epoch":236,"log_start":"177'5601","ondisk_log_start":"177'5601","created":164,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":7168,"log_dups_size":0,"ondisk_log_size":7168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:03.784250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89456640,"num_objects":1365,"num_object_clones":0,"num_object_copies":4095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":316,"num_objects_unfound":0,"num_objects_dirty":1365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12769,"num_write_kb":452288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4819,"num_bytes_recovered":255066112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,2],"avail_no_missing":["1","2","3","7"],"object_location_counts":[{"shards":"1,2,7","objects":1365}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":120,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.345487+0000","last_change":"2023-09-23T23:16:56.534749+0000","last_active":"2023-09-23T23:17:59.345487+0000","last_peered":"2023-09-23T23:17:59.345487+0000","last_clean":"2023-09-23T23:17:59.345487+0000","last_became_active":"2023-09-23T23:16:56.534346+0000","last_became_peered":"2023-09-23T23:16:56.534346+0000","last_unstale":"2023-09-23T23:17:59.345487+0000","last_undegraded":"2023-09-23T23:17:59.345487+0000","last_fullsized":"2023-09-23T23:17:59.345487+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:55.720486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":212,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.347775+0000","last_change":"2023-09-23T23:10:59.503483+0000","last_active":"2023-09-23T23:17:59.347775+0000","last_peered":"2023-09-23T23:17:59.347775+0000","last_clean":"2023-09-23T23:17:59.347775+0000","last_became_active":"2023-09-23T23:10:59.502851+0000","last_became_peered":"2023-09-23T23:10:59.502851+0000","last_unstale":"2023-09-23T23:17:59.347775+0000","last_undegraded":"2023-09-23T23:17:59.347775+0000","last_fullsized":"2023-09-23T23:17:59.347775+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:20.388467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":576,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.353266+0000","last_change":"2023-09-23T23:17:51.355247+0000","last_active":"2023-09-23T23:17:59.353266+0000","last_peered":"2023-09-23T23:17:59.353266+0000","last_clean":"2023-09-23T23:17:59.353266+0000","last_became_active":"2023-09-23T23:17:51.354856+0000","last_became_peered":"2023-09-23T23:17:51.354856+0000","last_unstale":"2023-09-23T23:17:59.353266+0000","last_undegraded":"2023-09-23T23:17:59.353266+0000","last_fullsized":"2023-09-23T23:17:59.353266+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:47.797085+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:47.797085+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:55.878838+0000","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.f","version":"243'12565","reported_seq":12652,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T23:18:03.885455+0000","last_change":"2023-09-23T23:13:28.355012+0000","last_active":"2023-09-23T23:18:03.885455+0000","last_peered":"2023-09-23T23:18:03.885455+0000","last_clean":"2023-09-23T23:18:03.885455+0000","last_became_active":"2023-09-23T23:13:28.354788+0000","last_became_peered":"2023-09-23T23:13:28.354788+0000","last_unstale":"2023-09-23T23:18:03.885455+0000","last_undegraded":"2023-09-23T23:18:03.885455+0000","last_fullsized":"2023-09-23T23:18:03.885455+0000","mapping_epoch":164,"log_start":"172'3933","ondisk_log_start":"172'3933","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":8632,"log_dups_size":0,"ondisk_log_size":8632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:04.013664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83296256,"num_objects":1271,"num_object_clones":0,"num_object_copies":3813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12565,"num_write_kb":442752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":205,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.347786+0000","last_change":"2023-09-23T23:07:30.069993+0000","last_active":"2023-09-23T23:17:59.347786+0000","last_peered":"2023-09-23T23:17:59.347786+0000","last_clean":"2023-09-23T23:17:59.347786+0000","last_became_active":"2023-09-23T23:07:30.069634+0000","last_became_peered":"2023-09-23T23:07:30.069634+0000","last_unstale":"2023-09-23T23:17:59.347786+0000","last_undegraded":"2023-09-23T23:17:59.347786+0000","last_fullsized":"2023-09-23T23:17:59.347786+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:24.608905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":248,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.346945+0000","last_change":"2023-09-23T23:17:48.318576+0000","last_active":"2023-09-23T23:17:59.346945+0000","last_peered":"2023-09-23T23:17:59.346945+0000","last_clean":"2023-09-23T23:17:59.346945+0000","last_became_active":"2023-09-23T23:17:48.302180+0000","last_became_peered":"2023-09-23T23:17:48.302180+0000","last_unstale":"2023-09-23T23:17:59.346945+0000","last_undegraded":"2023-09-23T23:17:59.346945+0000","last_fullsized":"2023-09-23T23:17:59.346945+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:34:56.665520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":177,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.353949+0000","last_change":"2023-09-23T23:17:48.320421+0000","last_active":"2023-09-23T23:17:59.353949+0000","last_peered":"2023-09-23T23:17:59.353949+0000","last_clean":"2023-09-23T23:17:59.353949+0000","last_became_active":"2023-09-23T23:17:48.320112+0000","last_became_peered":"2023-09-23T23:17:48.320112+0000","last_unstale":"2023-09-23T23:17:59.353949+0000","last_undegraded":"2023-09-23T23:17:59.353949+0000","last_fullsized":"2023-09-23T23:17:59.353949+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:32.901735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":523,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.343618+0000","last_change":"2023-09-23T23:17:51.342323+0000","last_active":"2023-09-23T23:17:59.343618+0000","last_peered":"2023-09-23T23:17:59.343618+0000","last_clean":"2023-09-23T23:17:59.343618+0000","last_became_active":"2023-09-23T23:17:51.341930+0000","last_became_peered":"2023-09-23T23:17:51.341930+0000","last_unstale":"2023-09-23T23:17:59.343618+0000","last_undegraded":"2023-09-23T23:17:59.343618+0000","last_fullsized":"2023-09-23T23:17:59.343618+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:49.726740+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:49.726740+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:16:57.092869+0000","scrub_duration":0.0013902070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"243'12432","reported_seq":12534,"reported_epoch":243,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:18:01.334907+0000","last_change":"2023-09-23T23:17:48.347691+0000","last_active":"2023-09-23T23:18:01.334907+0000","last_peered":"2023-09-23T23:18:01.334907+0000","last_clean":"2023-09-23T23:16:55.485111+0000","last_became_active":"2023-09-23T23:16:57.639573+0000","last_became_peered":"2023-09-23T23:16:57.639573+0000","last_unstale":"2023-09-23T23:18:01.334907+0000","last_undegraded":"2023-09-23T23:18:01.334907+0000","last_fullsized":"2023-09-23T23:18:01.334907+0000","mapping_epoch":213,"log_start":"182'5990","ondisk_log_start":"182'5990","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":6442,"log_dups_size":0,"ondisk_log_size":6442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:00.341194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89653248,"num_objects":1368,"num_object_clones":0,"num_object_copies":4104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1368,"num_objects_unfound":0,"num_objects_dirty":1368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12432,"num_write_kb":441600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":1368}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":208,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.343770+0000","last_change":"2023-09-23T23:07:30.075093+0000","last_active":"2023-09-23T23:17:59.343770+0000","last_peered":"2023-09-23T23:17:59.343770+0000","last_clean":"2023-09-23T23:17:59.343770+0000","last_became_active":"2023-09-23T23:07:30.074729+0000","last_became_peered":"2023-09-23T23:07:30.074729+0000","last_unstale":"2023-09-23T23:17:59.343770+0000","last_undegraded":"2023-09-23T23:17:59.343770+0000","last_fullsized":"2023-09-23T23:17:59.343770+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:34.502057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":204,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.355917+0000","last_change":"2023-09-23T23:04:11.514703+0000","last_active":"2023-09-23T23:17:59.355917+0000","last_peered":"2023-09-23T23:17:59.355917+0000","last_clean":"2023-09-23T23:17:59.355917+0000","last_became_active":"2023-09-23T23:04:11.514403+0000","last_became_peered":"2023-09-23T23:04:11.514403+0000","last_unstale":"2023-09-23T23:17:59.355917+0000","last_undegraded":"2023-09-23T23:17:59.355917+0000","last_fullsized":"2023-09-23T23:17:59.355917+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:26.374346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":211,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.366941+0000","last_change":"2023-09-23T23:17:48.300618+0000","last_active":"2023-09-23T23:17:59.366941+0000","last_peered":"2023-09-23T23:17:59.366941+0000","last_clean":"2023-09-23T23:17:59.366941+0000","last_became_active":"2023-09-23T23:17:48.300279+0000","last_became_peered":"2023-09-23T23:17:48.300279+0000","last_unstale":"2023-09-23T23:17:59.366941+0000","last_undegraded":"2023-09-23T23:17:59.366941+0000","last_fullsized":"2023-09-23T23:17:59.366941+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:53.227630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":508,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.349294+0000","last_change":"2023-09-23T23:17:51.342338+0000","last_active":"2023-09-23T23:17:59.349294+0000","last_peered":"2023-09-23T23:17:59.349294+0000","last_clean":"2023-09-23T23:17:59.349294+0000","last_became_active":"2023-09-23T23:17:51.342020+0000","last_became_peered":"2023-09-23T23:17:51.342020+0000","last_unstale":"2023-09-23T23:17:59.349294+0000","last_undegraded":"2023-09-23T23:17:59.349294+0000","last_fullsized":"2023-09-23T23:17:59.349294+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.183948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.183948+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"243'12417","reported_seq":15857,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T23:18:03.874714+0000","last_change":"2023-09-23T23:18:00.465911+0000","last_active":"2023-09-23T23:18:03.874714+0000","last_peered":"2023-09-23T23:18:03.874714+0000","last_clean":"2023-09-23T23:18:03.874714+0000","last_became_active":"2023-09-23T23:18:00.352183+0000","last_became_peered":"2023-09-23T23:18:00.352183+0000","last_unstale":"2023-09-23T23:18:03.874714+0000","last_undegraded":"2023-09-23T23:18:03.874714+0000","last_fullsized":"2023-09-23T23:18:03.874714+0000","mapping_epoch":242,"log_start":"174'5213","ondisk_log_start":"174'5213","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":7204,"log_dups_size":0,"ondisk_log_size":7204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:09.071393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83165184,"num_objects":1269,"num_object_clones":0,"num_object_copies":3807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12417,"num_write_kb":437952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1658,"num_bytes_recovered":108527616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":171,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T23:18:00.344151+0000","last_change":"2023-09-23T23:17:48.300243+0000","last_active":"2023-09-23T23:18:00.344151+0000","last_peered":"2023-09-23T23:18:00.344151+0000","last_clean":"2023-09-23T23:18:00.344151+0000","last_became_active":"2023-09-23T23:17:48.299953+0000","last_became_peered":"2023-09-23T23:17:48.299953+0000","last_unstale":"2023-09-23T23:18:00.344151+0000","last_undegraded":"2023-09-23T23:18:00.344151+0000","last_fullsized":"2023-09-23T23:18:00.344151+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:44.381159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":238,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.345378+0000","last_change":"2023-09-23T23:17:48.305230+0000","last_active":"2023-09-23T23:17:59.345378+0000","last_peered":"2023-09-23T23:17:59.345378+0000","last_clean":"2023-09-23T23:17:59.345378+0000","last_became_active":"2023-09-23T23:17:48.304847+0000","last_became_peered":"2023-09-23T23:17:48.304847+0000","last_unstale":"2023-09-23T23:17:59.345378+0000","last_undegraded":"2023-09-23T23:17:59.345378+0000","last_fullsized":"2023-09-23T23:17:59.345378+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:51.308840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":135,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.347124+0000","last_change":"2023-09-23T23:16:56.545792+0000","last_active":"2023-09-23T23:17:59.347124+0000","last_peered":"2023-09-23T23:17:59.347124+0000","last_clean":"2023-09-23T23:17:59.347124+0000","last_became_active":"2023-09-23T23:16:56.545447+0000","last_became_peered":"2023-09-23T23:16:56.545447+0000","last_unstale":"2023-09-23T23:17:59.347124+0000","last_undegraded":"2023-09-23T23:17:59.347124+0000","last_fullsized":"2023-09-23T23:17:59.347124+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:47.781440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"243'12487","reported_seq":12695,"reported_epoch":243,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:18:01.374064+0000","last_change":"2023-09-23T23:17:48.365484+0000","last_active":"2023-09-23T23:18:01.374064+0000","last_peered":"2023-09-23T23:18:01.374064+0000","last_clean":"2023-09-23T23:15:27.279123+0000","last_became_active":"2023-09-23T23:15:29.334529+0000","last_became_peered":"2023-09-23T23:15:29.334529+0000","last_unstale":"2023-09-23T23:18:01.374064+0000","last_undegraded":"2023-09-23T23:18:01.374064+0000","last_fullsized":"2023-09-23T23:18:01.374064+0000","mapping_epoch":187,"log_start":"182'6042","ondisk_log_start":"182'6042","created":164,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":6445,"log_dups_size":0,"ondisk_log_size":6445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:47:05.361853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88014848,"num_objects":1343,"num_object_clones":0,"num_object_copies":4029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1336,"num_objects_unfound":0,"num_objects_dirty":1343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12487,"num_write_kb":442560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":3342336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":1343}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":245,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.343841+0000","last_change":"2023-09-23T23:17:48.302538+0000","last_active":"2023-09-23T23:17:59.343841+0000","last_peered":"2023-09-23T23:17:59.343841+0000","last_clean":"2023-09-23T23:17:59.343841+0000","last_became_active":"2023-09-23T23:17:48.302203+0000","last_became_peered":"2023-09-23T23:17:48.302203+0000","last_unstale":"2023-09-23T23:17:59.343841+0000","last_undegraded":"2023-09-23T23:17:59.343841+0000","last_fullsized":"2023-09-23T23:17:59.343841+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:25.857118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":449,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.349115+0000","last_change":"2023-09-23T23:16:59.290902+0000","last_active":"2023-09-23T23:17:59.349115+0000","last_peered":"2023-09-23T23:17:59.349115+0000","last_clean":"2023-09-23T23:17:59.349115+0000","last_became_active":"2023-09-23T23:16:59.290535+0000","last_became_peered":"2023-09-23T23:16:59.290535+0000","last_unstale":"2023-09-23T23:17:59.349115+0000","last_undegraded":"2023-09-23T23:17:59.349115+0000","last_fullsized":"2023-09-23T23:17:59.349115+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:43.147316+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:43.147316+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":170,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.374711+0000","last_change":"2023-09-23T23:16:56.525298+0000","last_active":"2023-09-23T23:17:59.374711+0000","last_peered":"2023-09-23T23:17:59.374711+0000","last_clean":"2023-09-23T23:17:59.374711+0000","last_became_active":"2023-09-23T23:16:56.524974+0000","last_became_peered":"2023-09-23T23:16:56.524974+0000","last_unstale":"2023-09-23T23:17:59.374711+0000","last_undegraded":"2023-09-23T23:17:59.374711+0000","last_fullsized":"2023-09-23T23:17:59.374711+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:08.860790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":186,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.352157+0000","last_change":"2023-09-23T22:53:18.075922+0000","last_active":"2023-09-23T23:17:59.352157+0000","last_peered":"2023-09-23T23:17:59.352157+0000","last_clean":"2023-09-23T23:17:59.352157+0000","last_became_active":"2023-09-23T22:53:18.075621+0000","last_became_peered":"2023-09-23T22:53:18.075621+0000","last_unstale":"2023-09-23T23:17:59.352157+0000","last_undegraded":"2023-09-23T23:17:59.352157+0000","last_fullsized":"2023-09-23T23:17:59.352157+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:01.490333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":488,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.345297+0000","last_change":"2023-09-23T23:14:44.629928+0000","last_active":"2023-09-23T23:17:59.345297+0000","last_peered":"2023-09-23T23:17:59.345297+0000","last_clean":"2023-09-23T23:17:59.345297+0000","last_became_active":"2023-09-23T23:14:44.445304+0000","last_became_peered":"2023-09-23T23:14:44.445304+0000","last_unstale":"2023-09-23T23:17:59.345297+0000","last_undegraded":"2023-09-23T23:17:59.345297+0000","last_fullsized":"2023-09-23T23:17:59.345297+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:44.629771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:44.629771+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:07.493381+0000","scrub_duration":0.011659790999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.b","version":"243'12276","reported_seq":12366,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T23:18:01.374013+0000","last_change":"2023-09-23T23:13:28.357808+0000","last_active":"2023-09-23T23:18:01.374013+0000","last_peered":"2023-09-23T23:18:01.374013+0000","last_clean":"2023-09-23T23:18:01.374013+0000","last_became_active":"2023-09-23T23:13:28.357474+0000","last_became_peered":"2023-09-23T23:13:28.357474+0000","last_unstale":"2023-09-23T23:18:01.374013+0000","last_undegraded":"2023-09-23T23:18:01.374013+0000","last_fullsized":"2023-09-23T23:18:01.374013+0000","mapping_epoch":164,"log_start":"182'5833","ondisk_log_start":"182'5833","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":6443,"log_dups_size":0,"ondisk_log_size":6443,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:08.042264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89522176,"num_objects":1366,"num_object_clones":0,"num_object_copies":4098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12276,"num_write_kb":436544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":235,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.345277+0000","last_change":"2023-09-23T23:16:56.523954+0000","last_active":"2023-09-23T23:17:59.345277+0000","last_peered":"2023-09-23T23:17:59.345277+0000","last_clean":"2023-09-23T23:17:59.345277+0000","last_became_active":"2023-09-23T23:16:56.523679+0000","last_became_peered":"2023-09-23T23:16:56.523679+0000","last_unstale":"2023-09-23T23:17:59.345277+0000","last_undegraded":"2023-09-23T23:17:59.345277+0000","last_fullsized":"2023-09-23T23:17:59.345277+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:51.443997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":203,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.350351+0000","last_change":"2023-09-23T23:16:56.525850+0000","last_active":"2023-09-23T23:17:59.350351+0000","last_peered":"2023-09-23T23:17:59.350351+0000","last_clean":"2023-09-23T23:17:59.350351+0000","last_became_active":"2023-09-23T23:16:56.524998+0000","last_became_peered":"2023-09-23T23:16:56.524998+0000","last_unstale":"2023-09-23T23:17:59.350351+0000","last_undegraded":"2023-09-23T23:17:59.350351+0000","last_fullsized":"2023-09-23T23:17:59.350351+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:54.317877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":233,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.350962+0000","last_change":"2023-09-23T23:17:48.308925+0000","last_active":"2023-09-23T23:17:59.350962+0000","last_peered":"2023-09-23T23:17:59.350962+0000","last_clean":"2023-09-23T23:17:59.350962+0000","last_became_active":"2023-09-23T23:17:48.308628+0000","last_became_peered":"2023-09-23T23:17:48.308628+0000","last_unstale":"2023-09-23T23:17:59.350962+0000","last_undegraded":"2023-09-23T23:17:59.350962+0000","last_fullsized":"2023-09-23T23:17:59.350962+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:58:28.247724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":516,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.352585+0000","last_change":"2023-09-23T23:16:59.295000+0000","last_active":"2023-09-23T23:17:59.352585+0000","last_peered":"2023-09-23T23:17:59.352585+0000","last_clean":"2023-09-23T23:17:59.352585+0000","last_became_active":"2023-09-23T23:16:59.294607+0000","last_became_peered":"2023-09-23T23:16:59.294607+0000","last_unstale":"2023-09-23T23:17:59.352585+0000","last_undegraded":"2023-09-23T23:17:59.352585+0000","last_fullsized":"2023-09-23T23:17:59.352585+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.802642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.802642+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:48.429859+0000","scrub_duration":0.0056570290000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.e","version":"243'12629","reported_seq":22418,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T23:18:04.173036+0000","last_change":"2023-09-23T23:17:43.594560+0000","last_active":"2023-09-23T23:18:04.173036+0000","last_peered":"2023-09-23T23:18:04.173036+0000","last_clean":"2023-09-23T23:18:04.173036+0000","last_became_active":"2023-09-23T23:17:43.563581+0000","last_became_peered":"2023-09-23T23:17:43.563581+0000","last_unstale":"2023-09-23T23:18:04.173036+0000","last_undegraded":"2023-09-23T23:18:04.173036+0000","last_fullsized":"2023-09-23T23:18:04.173036+0000","mapping_epoch":233,"log_start":"184'7227","ondisk_log_start":"184'7227","created":164,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":5402,"log_dups_size":0,"ondisk_log_size":5402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:39.419351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86310912,"num_objects":1317,"num_object_clones":0,"num_object_copies":3951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12629,"num_write_kb":446272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4826,"num_bytes_recovered":316276736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":208,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.345748+0000","last_change":"2023-09-23T23:04:11.900743+0000","last_active":"2023-09-23T23:17:59.345748+0000","last_peered":"2023-09-23T23:17:59.345748+0000","last_clean":"2023-09-23T23:17:59.345748+0000","last_became_active":"2023-09-23T23:04:11.900418+0000","last_became_peered":"2023-09-23T23:04:11.900418+0000","last_unstale":"2023-09-23T23:17:59.345748+0000","last_undegraded":"2023-09-23T23:17:59.345748+0000","last_fullsized":"2023-09-23T23:17:59.345748+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:45:31.893513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":247,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.347790+0000","last_change":"2023-09-23T23:17:48.305869+0000","last_active":"2023-09-23T23:17:59.347790+0000","last_peered":"2023-09-23T23:17:59.347790+0000","last_clean":"2023-09-23T23:17:59.347790+0000","last_became_active":"2023-09-23T23:17:48.305574+0000","last_became_peered":"2023-09-23T23:17:48.305574+0000","last_unstale":"2023-09-23T23:17:59.347790+0000","last_undegraded":"2023-09-23T23:17:59.347790+0000","last_fullsized":"2023-09-23T23:17:59.347790+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:55.580919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":188,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.349057+0000","last_change":"2023-09-23T23:17:33.437620+0000","last_active":"2023-09-23T23:17:59.349057+0000","last_peered":"2023-09-23T23:17:59.349057+0000","last_clean":"2023-09-23T23:17:59.349057+0000","last_became_active":"2023-09-23T23:17:33.437274+0000","last_became_peered":"2023-09-23T23:17:33.437274+0000","last_unstale":"2023-09-23T23:17:59.349057+0000","last_undegraded":"2023-09-23T23:17:59.349057+0000","last_fullsized":"2023-09-23T23:17:59.349057+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:01.385833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":523,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.354313+0000","last_change":"2023-09-23T23:17:51.331967+0000","last_active":"2023-09-23T23:17:59.354313+0000","last_peered":"2023-09-23T23:17:59.354313+0000","last_clean":"2023-09-23T23:17:59.354313+0000","last_became_active":"2023-09-23T23:17:51.331604+0000","last_became_peered":"2023-09-23T23:17:51.331604+0000","last_unstale":"2023-09-23T23:17:59.354313+0000","last_undegraded":"2023-09-23T23:17:59.354313+0000","last_fullsized":"2023-09-23T23:17:59.354313+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:51.489074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:51.489074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:38:55.870389+0000","scrub_duration":0.00519324,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.d","version":"243'12583","reported_seq":28049,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T23:18:03.977481+0000","last_change":"2023-09-23T23:17:41.579641+0000","last_active":"2023-09-23T23:18:03.977481+0000","last_peered":"2023-09-23T23:18:03.977481+0000","last_clean":"2023-09-23T23:18:03.977481+0000","last_became_active":"2023-09-23T23:17:41.540249+0000","last_became_peered":"2023-09-23T23:17:41.540249+0000","last_unstale":"2023-09-23T23:18:03.977481+0000","last_undegraded":"2023-09-23T23:18:03.977481+0000","last_fullsized":"2023-09-23T23:18:03.977481+0000","mapping_epoch":231,"log_start":"172'4000","ondisk_log_start":"172'4000","created":164,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":8583,"log_dups_size":0,"ondisk_log_size":8583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:15:49.731942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83230744,"num_objects":1271,"num_object_clones":0,"num_object_copies":3813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1271,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12583,"num_write_kb":443265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7658,"num_bytes_recovered":501743664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":220,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.364518+0000","last_change":"2023-09-23T23:16:56.561147+0000","last_active":"2023-09-23T23:17:59.364518+0000","last_peered":"2023-09-23T23:17:59.364518+0000","last_clean":"2023-09-23T23:17:59.364518+0000","last_became_active":"2023-09-23T23:16:56.547808+0000","last_became_peered":"2023-09-23T23:16:56.547808+0000","last_unstale":"2023-09-23T23:17:59.364518+0000","last_undegraded":"2023-09-23T23:17:59.364518+0000","last_fullsized":"2023-09-23T23:17:59.364518+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:13.666957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":98,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.352729+0000","last_change":"2023-09-23T23:16:56.526187+0000","last_active":"2023-09-23T23:17:59.352729+0000","last_peered":"2023-09-23T23:17:59.352729+0000","last_clean":"2023-09-23T23:17:59.352729+0000","last_became_active":"2023-09-23T23:16:56.525874+0000","last_became_peered":"2023-09-23T23:16:56.525874+0000","last_unstale":"2023-09-23T23:17:59.352729+0000","last_undegraded":"2023-09-23T23:17:59.352729+0000","last_fullsized":"2023-09-23T23:17:59.352729+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:32:07.101559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":140,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.349064+0000","last_change":"2023-09-23T23:16:56.528347+0000","last_active":"2023-09-23T23:17:59.349064+0000","last_peered":"2023-09-23T23:17:59.349064+0000","last_clean":"2023-09-23T23:17:59.349064+0000","last_became_active":"2023-09-23T23:16:56.527965+0000","last_became_peered":"2023-09-23T23:16:56.527965+0000","last_unstale":"2023-09-23T23:17:59.349064+0000","last_undegraded":"2023-09-23T23:17:59.349064+0000","last_fullsized":"2023-09-23T23:17:59.349064+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:46.313356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":509,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.367142+0000","last_change":"2023-09-23T23:16:59.285629+0000","last_active":"2023-09-23T23:17:59.367142+0000","last_peered":"2023-09-23T23:17:59.367142+0000","last_clean":"2023-09-23T23:17:59.367142+0000","last_became_active":"2023-09-23T23:16:59.285208+0000","last_became_peered":"2023-09-23T23:16:59.285208+0000","last_unstale":"2023-09-23T23:17:59.367142+0000","last_undegraded":"2023-09-23T23:17:59.367142+0000","last_fullsized":"2023-09-23T23:17:59.367142+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.522632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.522632+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:04:15.985273+0000","scrub_duration":0.018152603999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"243'12553","reported_seq":12643,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T23:18:02.899264+0000","last_change":"2023-09-23T23:13:28.359953+0000","last_active":"2023-09-23T23:18:02.899264+0000","last_peered":"2023-09-23T23:18:02.899264+0000","last_clean":"2023-09-23T23:18:02.899264+0000","last_became_active":"2023-09-23T23:13:28.359690+0000","last_became_peered":"2023-09-23T23:13:28.359690+0000","last_unstale":"2023-09-23T23:18:02.899264+0000","last_undegraded":"2023-09-23T23:18:02.899264+0000","last_fullsized":"2023-09-23T23:18:02.899264+0000","mapping_epoch":164,"log_start":"182'6076","ondisk_log_start":"182'6076","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":6477,"log_dups_size":0,"ondisk_log_size":6477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:56.963155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89456640,"num_objects":1365,"num_object_clones":0,"num_object_copies":4095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12553,"num_write_kb":445376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":245,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.353186+0000","last_change":"2023-09-23T23:17:48.303246+0000","last_active":"2023-09-23T23:17:59.353186+0000","last_peered":"2023-09-23T23:17:59.353186+0000","last_clean":"2023-09-23T23:17:59.353186+0000","last_became_active":"2023-09-23T23:17:48.302984+0000","last_became_peered":"2023-09-23T23:17:48.302984+0000","last_unstale":"2023-09-23T23:17:59.353186+0000","last_undegraded":"2023-09-23T23:17:59.353186+0000","last_fullsized":"2023-09-23T23:17:59.353186+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:11.999957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":198,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.350825+0000","last_change":"2023-09-23T23:07:30.072388+0000","last_active":"2023-09-23T23:17:59.350825+0000","last_peered":"2023-09-23T23:17:59.350825+0000","last_clean":"2023-09-23T23:17:59.350825+0000","last_became_active":"2023-09-23T23:07:30.072099+0000","last_became_peered":"2023-09-23T23:07:30.072099+0000","last_unstale":"2023-09-23T23:17:59.350825+0000","last_undegraded":"2023-09-23T23:17:59.350825+0000","last_fullsized":"2023-09-23T23:17:59.350825+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:19.758734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":186,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.355863+0000","last_change":"2023-09-23T22:53:17.905706+0000","last_active":"2023-09-23T23:17:59.355863+0000","last_peered":"2023-09-23T23:17:59.355863+0000","last_clean":"2023-09-23T23:17:59.355863+0000","last_became_active":"2023-09-23T22:53:17.905309+0000","last_became_peered":"2023-09-23T22:53:17.905309+0000","last_unstale":"2023-09-23T23:17:59.355863+0000","last_undegraded":"2023-09-23T23:17:59.355863+0000","last_fullsized":"2023-09-23T23:17:59.355863+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:06.105426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":182,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.348976+0000","last_change":"2023-09-23T23:04:11.900795+0000","last_active":"2023-09-23T23:17:59.348976+0000","last_peered":"2023-09-23T23:17:59.348976+0000","last_clean":"2023-09-23T23:17:59.348976+0000","last_became_active":"2023-09-23T23:04:11.899751+0000","last_became_peered":"2023-09-23T23:04:11.899751+0000","last_unstale":"2023-09-23T23:17:59.348976+0000","last_undegraded":"2023-09-23T23:17:59.348976+0000","last_fullsized":"2023-09-23T23:17:59.348976+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:41.206493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":185,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.347714+0000","last_change":"2023-09-23T23:17:48.306469+0000","last_active":"2023-09-23T23:17:59.347714+0000","last_peered":"2023-09-23T23:17:59.347714+0000","last_clean":"2023-09-23T23:17:59.347714+0000","last_became_active":"2023-09-23T23:17:48.306156+0000","last_became_peered":"2023-09-23T23:17:48.306156+0000","last_unstale":"2023-09-23T23:17:59.347714+0000","last_undegraded":"2023-09-23T23:17:59.347714+0000","last_fullsized":"2023-09-23T23:17:59.347714+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:07.516611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":206,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.343869+0000","last_change":"2023-09-23T23:07:30.075311+0000","last_active":"2023-09-23T23:17:59.343869+0000","last_peered":"2023-09-23T23:17:59.343869+0000","last_clean":"2023-09-23T23:17:59.343869+0000","last_became_active":"2023-09-23T23:07:30.074952+0000","last_became_peered":"2023-09-23T23:07:30.074952+0000","last_unstale":"2023-09-23T23:17:59.343869+0000","last_undegraded":"2023-09-23T23:17:59.343869+0000","last_fullsized":"2023-09-23T23:17:59.343869+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:16.724083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":240,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.355842+0000","last_change":"2023-09-23T23:17:48.300748+0000","last_active":"2023-09-23T23:17:59.355842+0000","last_peered":"2023-09-23T23:17:59.355842+0000","last_clean":"2023-09-23T23:17:59.355842+0000","last_became_active":"2023-09-23T23:17:48.300443+0000","last_became_peered":"2023-09-23T23:17:48.300443+0000","last_unstale":"2023-09-23T23:17:59.355842+0000","last_undegraded":"2023-09-23T23:17:59.355842+0000","last_fullsized":"2023-09-23T23:17:59.355842+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:11.537165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":189,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.347036+0000","last_change":"2023-09-23T22:53:16.891297+0000","last_active":"2023-09-23T23:17:59.347036+0000","last_peered":"2023-09-23T23:17:59.347036+0000","last_clean":"2023-09-23T23:17:59.347036+0000","last_became_active":"2023-09-23T22:53:16.891025+0000","last_became_peered":"2023-09-23T22:53:16.891025+0000","last_unstale":"2023-09-23T23:17:59.347036+0000","last_undegraded":"2023-09-23T23:17:59.347036+0000","last_fullsized":"2023-09-23T23:17:59.347036+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:40:50.608339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":220,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.353744+0000","last_change":"2023-09-23T23:10:59.512247+0000","last_active":"2023-09-23T23:17:59.353744+0000","last_peered":"2023-09-23T23:17:59.353744+0000","last_clean":"2023-09-23T23:17:59.353744+0000","last_became_active":"2023-09-23T23:10:59.511865+0000","last_became_peered":"2023-09-23T23:10:59.511865+0000","last_unstale":"2023-09-23T23:17:59.353744+0000","last_undegraded":"2023-09-23T23:17:59.353744+0000","last_fullsized":"2023-09-23T23:17:59.353744+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:20.981239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":227,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.364013+0000","last_change":"2023-09-23T23:17:48.304879+0000","last_active":"2023-09-23T23:17:59.364013+0000","last_peered":"2023-09-23T23:17:59.364013+0000","last_clean":"2023-09-23T23:17:59.364013+0000","last_became_active":"2023-09-23T23:17:48.304506+0000","last_became_peered":"2023-09-23T23:17:48.304506+0000","last_unstale":"2023-09-23T23:17:59.364013+0000","last_undegraded":"2023-09-23T23:17:59.364013+0000","last_fullsized":"2023-09-23T23:17:59.364013+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:28.380655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":241,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.376495+0000","last_change":"2023-09-23T23:17:48.303036+0000","last_active":"2023-09-23T23:17:59.376495+0000","last_peered":"2023-09-23T23:17:59.376495+0000","last_clean":"2023-09-23T23:17:59.376495+0000","last_became_active":"2023-09-23T23:17:48.302706+0000","last_became_peered":"2023-09-23T23:17:48.302706+0000","last_unstale":"2023-09-23T23:17:59.376495+0000","last_undegraded":"2023-09-23T23:17:59.376495+0000","last_fullsized":"2023-09-23T23:17:59.376495+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:46:22.942097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":228,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.349845+0000","last_change":"2023-09-23T23:17:48.338955+0000","last_active":"2023-09-23T23:17:59.349845+0000","last_peered":"2023-09-23T23:17:59.349845+0000","last_clean":"2023-09-23T23:17:59.349845+0000","last_became_active":"2023-09-23T23:17:48.332511+0000","last_became_peered":"2023-09-23T23:17:48.332511+0000","last_unstale":"2023-09-23T23:17:59.349845+0000","last_undegraded":"2023-09-23T23:17:59.349845+0000","last_fullsized":"2023-09-23T23:17:59.349845+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:12.303751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":240,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T23:17:59.353829+0000","last_change":"2023-09-23T23:17:48.319814+0000","last_active":"2023-09-23T23:17:59.353829+0000","last_peered":"2023-09-23T23:17:59.353829+0000","last_clean":"2023-09-23T23:17:59.353829+0000","last_became_active":"2023-09-23T23:17:48.303463+0000","last_became_peered":"2023-09-23T23:17:48.303463+0000","last_unstale":"2023-09-23T23:17:59.353829+0000","last_undegraded":"2023-09-23T23:17:59.353829+0000","last_fullsized":"2023-09-23T23:17:59.353829+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:33.861565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":770,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":133,"num_read_kb":248,"num_write":65,"num_write_kb":658,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1073396,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1387331608,"num_objects":21170,"num_object_clones":0,"num_object_copies":63510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":8921,"num_objects_unfound":0,"num_objects_dirty":21170,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":201188,"num_write_kb":7115393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32723,"num_bytes_recovered":1966407728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5773676544,"data_stored":5773656184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114276,"ondisk_log_size":114276,"up":48,"acting":47,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234304,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1668836,"kb_used_data":1228764,"kb_used_omap":0,"kb_used_meta":440064,"kb_avail":92076316,"statfs":{"total":95995035648,"available":94286147584,"internally_reserved":0,"allocated":1258254336,"data_stored":1257530357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":450625536},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:17:34 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":4.8099999999999996,"15min":3.9239999999999999},"min":{"1min":0.23000000000000001,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":3.5760000000000001,"5min":216.40899999999999,"15min":230.249},"last":1.0840000000000001},{"interface":"front","average":{"1min":1.758,"5min":5.3010000000000002,"15min":4.1420000000000003},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.24199999999999999},"max":{"1min":10.398999999999999,"5min":216.43700000000001,"15min":230.261},"last":11.327999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:17:56 2023","interfaces":[{"interface":"back","average":{"1min":2.6150000000000002,"5min":2.9700000000000002,"15min":4.2359999999999998},"min":{"1min":0.39100000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.798,"5min":121.39400000000001,"15min":121.39400000000001},"last":1.319},{"interface":"front","average":{"1min":2.2989999999999999,"5min":2.9790000000000001,"15min":4.077},"min":{"1min":0.27000000000000002,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":10.034000000000001,"5min":121.381,"15min":121.381},"last":0.23899999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:17:24 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":2.0139999999999998,"15min":2.0139999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":9.5589999999999993,"5min":9.5589999999999993,"15min":9.5589999999999993},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.6830000000000001,"5min":1.6830000000000001,"15min":1.6830000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":9.5700000000000003,"5min":9.5700000000000003,"15min":9.5700000000000003},"last":1.6339999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:17:07 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":5.3230000000000004,"15min":3.4169999999999998},"min":{"1min":0.373,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.29,"5min":144.50299999999999,"15min":144.50299999999999},"last":1.744},{"interface":"front","average":{"1min":1.732,"5min":5.2439999999999998,"15min":3.46},"min":{"1min":0.441,"5min":0.307,"15min":0.307},"max":{"1min":6.2779999999999996,"5min":144.48699999999999,"15min":144.48699999999999},"last":0.29699999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:17:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":0.68000000000000005,"15min":1.335},"min":{"1min":0.30399999999999999,"5min":0.254,"15min":0.23599999999999999},"max":{"1min":5.2000000000000002,"5min":5.2000000000000002,"15min":92.781000000000006},"last":0.27300000000000002},{"interface":"front","average":{"1min":1.3460000000000001,"5min":1.075,"15min":1.4390000000000001},"min":{"1min":0.33200000000000002,"5min":0.28599999999999998,"15min":0.254},"max":{"1min":6.04,"5min":9.7530000000000001,"15min":92.626999999999995},"last":0.37}]},{"osd":5,"last update":"Sat Sep 23 23:17:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":0.96199999999999997,"15min":1.5680000000000001},"min":{"1min":0.29099999999999998,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":8.8979999999999997,"5min":8.8979999999999997,"15min":93.135000000000005},"last":1.2849999999999999},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.68700000000000006,"15min":1.3140000000000001},"min":{"1min":0.35699999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.601,"5min":3.9239999999999999,"15min":92.885000000000005},"last":0.55300000000000005}]},{"osd":6,"last update":"Sat Sep 23 23:17:24 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":0.94899999999999995,"15min":1.4410000000000001},"min":{"1min":0.318,"5min":0.30199999999999999,"15min":0.27300000000000002},"max":{"1min":9.9909999999999997,"5min":9.9909999999999997,"15min":93.105999999999995},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.395,"5min":1.0680000000000001,"15min":1.506},"min":{"1min":0.377,"5min":0.32400000000000001,"15min":0.28399999999999997},"max":{"1min":6.1050000000000004,"5min":6.1050000000000004,"15min":93.198999999999998},"last":0.435}]}]},{"osd":6,"up_from":46,"seq":197568495940,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":803624,"kb_used_data":484064,"kb_used_omap":0,"kb_used_meta":319552,"kb_avail":92941528,"statfs":{"total":95995035648,"available":95172124672,"internally_reserved":0,"allocated":495681536,"data_stored":494954294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":327221248},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:17:46 2023","interfaces":[{"interface":"back","average":{"1min":15.551,"5min":6.1399999999999997,"15min":4.1079999999999997},"min":{"1min":0.317,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":161.64599999999999,"5min":161.64599999999999,"15min":161.64599999999999},"last":1.175},{"interface":"front","average":{"1min":14.704000000000001,"5min":6.3449999999999998,"15min":4.484},"min":{"1min":0.26500000000000001,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":161.38499999999999,"5min":161.38499999999999,"15min":161.38499999999999},"last":1.962}]},{"osd":1,"last update":"Sat Sep 23 23:17:46 2023","interfaces":[{"interface":"back","average":{"1min":18.859999999999999,"5min":6.5510000000000002,"15min":3.9009999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.24099999999999999},"max":{"1min":210.869,"5min":210.869,"15min":210.869},"last":7.4489999999999998},{"interface":"front","average":{"1min":18.619,"5min":6.726,"15min":4.1630000000000003},"min":{"1min":0.36399999999999999,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":210.89699999999999,"5min":210.89699999999999,"15min":210.89699999999999},"last":6.0590000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:17:20 2023","interfaces":[{"interface":"back","average":{"1min":1.7889999999999999,"5min":1.7889999999999999,"15min":1.7889999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.867,"5min":7.867,"15min":7.867},"last":6.0209999999999999},{"interface":"front","average":{"1min":1.2649999999999999,"5min":1.2649999999999999,"15min":1.2649999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.1079999999999997,"5min":5.1079999999999997,"15min":5.1079999999999997},"last":0.32400000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:17:12 2023","interfaces":[{"interface":"back","average":{"1min":5.0730000000000004,"5min":4.5830000000000002,"15min":3.3170000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":31.564,"5min":109.98699999999999,"15min":109.98699999999999},"last":7.2880000000000003},{"interface":"front","average":{"1min":4.0110000000000001,"5min":4.4649999999999999,"15min":3.5760000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":32.366,"5min":110.024,"15min":110.024},"last":2.4950000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:18:01 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.76100000000000001,"15min":1.0569999999999999},"min":{"1min":0.32500000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.7120000000000002,"5min":5.306,"15min":11.890000000000001},"last":1.929},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.84999999999999998,"15min":1.111},"min":{"1min":0.32200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.7590000000000003,"5min":5.7590000000000003,"15min":11.728},"last":5.7590000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:18:01 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":0.89600000000000002,"15min":1.266},"min":{"1min":0.35999999999999999,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":2.7400000000000002,"5min":6.8310000000000004,"15min":26.158999999999999},"last":2.0179999999999998},{"interface":"front","average":{"1min":1.014,"5min":0.82299999999999995,"15min":1.1339999999999999},"min":{"1min":0.35099999999999998,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":3.9380000000000002,"5min":4.4080000000000004,"15min":25.786000000000001},"last":2.0990000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:17:26 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.84499999999999997,"15min":1.1619999999999999},"min":{"1min":0.34000000000000002,"5min":0.32300000000000001,"15min":0.32100000000000001},"max":{"1min":2.4409999999999998,"5min":4.4889999999999999,"15min":10.968999999999999},"last":5.8170000000000002},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.624,"15min":0.997},"min":{"1min":0.33000000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.3079999999999998,"5min":4.3079999999999998,"15min":8.0530000000000008},"last":0.73899999999999999}]}]},{"osd":1,"up_from":112,"seq":481036337281,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1349944,"kb_used_data":913712,"kb_used_omap":4,"kb_used_meta":436219,"kb_avail":92395208,"statfs":{"total":95995035648,"available":94612692992,"internally_reserved":0,"allocated":935641088,"data_stored":934605795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":446689166},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,0,0,2,2,12],"upper_bound":2048},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:17:39 2023","interfaces":[{"interface":"back","average":{"1min":4.6459999999999999,"5min":2.3279999999999998,"15min":1.95},"min":{"1min":0.26200000000000001,"5min":0.20100000000000001,"15min":0.17899999999999999},"max":{"1min":18.754000000000001,"5min":25.771000000000001,"15min":25.771000000000001},"last":0.20300000000000001},{"interface":"front","average":{"1min":6.8929999999999998,"5min":3.109,"15min":2.0649999999999999},"min":{"1min":0.46899999999999997,"5min":0.192,"15min":0.192},"max":{"1min":20.373999999999999,"5min":20.373999999999999,"15min":20.373999999999999},"last":0.24199999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:17:20 2023","interfaces":[{"interface":"back","average":{"1min":6.484,"5min":6.484,"15min":6.484},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":25.744,"5min":25.744,"15min":25.744},"last":0.26900000000000002},{"interface":"front","average":{"1min":3.7570000000000001,"5min":3.7570000000000001,"15min":3.7570000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":16.870000000000001,"5min":16.870000000000001,"15min":16.870000000000001},"last":2.726}]},{"osd":3,"last update":"Sat Sep 23 23:17:14 2023","interfaces":[{"interface":"back","average":{"1min":7.0700000000000003,"5min":3.879,"15min":3.0950000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.22700000000000001},"max":{"1min":30.565999999999999,"5min":70.956999999999994,"15min":70.956999999999994},"last":0.28899999999999998},{"interface":"front","average":{"1min":5.5099999999999998,"5min":3.5230000000000001,"15min":2.7010000000000001},"min":{"1min":0.32700000000000001,"5min":0.29399999999999998,"15min":0.26800000000000002},"max":{"1min":16.081,"5min":70.921000000000006,"15min":70.921000000000006},"last":0.34799999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:17:39 2023","interfaces":[{"interface":"back","average":{"1min":2.6949999999999998,"5min":1.7310000000000001,"15min":1.6599999999999999},"min":{"1min":0.312,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":16.748999999999999,"5min":16.748999999999999,"15min":16.748999999999999},"last":2.7810000000000001},{"interface":"front","average":{"1min":3.1019999999999999,"5min":2.2549999999999999,"15min":1.804},"min":{"1min":0.317,"5min":0.317,"15min":0.29099999999999998},"max":{"1min":13.177,"5min":25.710999999999999,"15min":25.710999999999999},"last":0.33600000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:17:39 2023","interfaces":[{"interface":"back","average":{"1min":4.3099999999999996,"5min":2.4929999999999999,"15min":3.2389999999999999},"min":{"1min":0.377,"5min":0.318,"15min":0.26400000000000001},"max":{"1min":20.344999999999999,"5min":20.344999999999999,"15min":67.497},"last":0.378},{"interface":"front","average":{"1min":2.8460000000000001,"5min":1.7849999999999999,"15min":3.056},"min":{"1min":0.36399999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":16.818000000000001,"5min":16.818000000000001,"15min":67.540999999999997},"last":2.8679999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:17:39 2023","interfaces":[{"interface":"back","average":{"1min":4.4989999999999997,"5min":2.6040000000000001,"15min":1.9359999999999999},"min":{"1min":0.41799999999999998,"5min":0.35199999999999998,"15min":0.33500000000000002},"max":{"1min":20.289999999999999,"5min":20.289999999999999,"15min":20.289999999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":2.7949999999999999,"5min":1.8680000000000001,"15min":1.841},"min":{"1min":0.36699999999999999,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":16.843,"5min":16.843,"15min":16.843},"last":2.8119999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:17:39 2023","interfaces":[{"interface":"back","average":{"1min":2.8340000000000001,"5min":1.9650000000000001,"15min":1.8879999999999999},"min":{"1min":0.44,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":16.789000000000001,"5min":16.789000000000001,"15min":16.789000000000001},"last":2.8410000000000002},{"interface":"front","average":{"1min":2.9710000000000001,"5min":2.1269999999999998,"15min":1.825},"min":{"1min":0.35499999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":13.140000000000001,"5min":25.667000000000002,"15min":25.667000000000002},"last":0.42199999999999999}]}]},{"osd":0,"up_from":91,"seq":390842024104,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1028280,"kb_used_data":542384,"kb_used_omap":5,"kb_used_meta":485690,"kb_avail":92716872,"statfs":{"total":95995035648,"available":94942076928,"internally_reserved":0,"allocated":555401216,"data_stored":554364653,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":497347194},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,6],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:17:49 2023","interfaces":[{"interface":"back","average":{"1min":5.1079999999999997,"5min":5.1630000000000003,"15min":3.3130000000000002},"min":{"1min":0.19700000000000001,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":26.754999999999999,"5min":155.91999999999999,"15min":155.91999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":3.2690000000000001,"5min":4.2839999999999998,"15min":2.9409999999999998},"min":{"1min":0.20399999999999999,"5min":0.19,"15min":0.189},"max":{"1min":9.0280000000000005,"5min":153.28899999999999,"15min":153.28899999999999},"last":0.35799999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:17:21 2023","interfaces":[{"interface":"back","average":{"1min":3.1379999999999999,"5min":3.1379999999999999,"15min":3.1379999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":20.495999999999999,"5min":20.495999999999999,"15min":20.495999999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":4.1840000000000002,"5min":4.1840000000000002,"15min":4.1840000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":36.383000000000003,"5min":36.383000000000003,"15min":36.383000000000003},"last":0.315}]},{"osd":3,"last update":"Sat Sep 23 23:17:04 2023","interfaces":[{"interface":"back","average":{"1min":6.7880000000000003,"5min":3.6699999999999999,"15min":3.5289999999999999},"min":{"1min":0.314,"5min":0.23200000000000001,"15min":0.22700000000000001},"max":{"1min":66.501000000000005,"5min":82.691000000000003,"15min":82.691000000000003},"last":0.28299999999999997},{"interface":"front","average":{"1min":5.1580000000000004,"5min":2.8940000000000001,"15min":2.6339999999999999},"min":{"1min":0.371,"5min":0.25,"15min":0.24199999999999999},"max":{"1min":66.230000000000004,"5min":82.429000000000002,"15min":82.429000000000002},"last":0.505}]},{"osd":4,"last update":"Sat Sep 23 23:17:31 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.2709999999999999,"15min":1.9450000000000001},"min":{"1min":0.318,"5min":0.27900000000000003,"15min":0.245},"max":{"1min":6.0999999999999996,"5min":7.0759999999999996,"15min":28.376999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":3.1720000000000002,"5min":1.581,"15min":2.3090000000000002},"min":{"1min":0.34399999999999997,"5min":0.32700000000000001,"15min":0.29099999999999998},"max":{"1min":13.787000000000001,"5min":19.167000000000002,"15min":30.706},"last":0.53000000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:17:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6179999999999999,"5min":1.6040000000000001,"15min":2.3340000000000001},"min":{"1min":0.40799999999999997,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":13.832000000000001,"5min":13.832000000000001,"15min":30.873999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.6100000000000001,"5min":1.4750000000000001,"15min":1.857},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29499999999999998},"max":{"1min":5.1790000000000003,"5min":7.9000000000000004,"15min":10.117000000000001},"last":0.42899999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:17:31 2023","interfaces":[{"interface":"back","average":{"1min":3.1970000000000001,"5min":1.6799999999999999,"15min":2.3809999999999998},"min":{"1min":0.378,"5min":0.314,"15min":0.314},"max":{"1min":13.859999999999999,"5min":13.859999999999999,"15min":21.047999999999998},"last":26.135999999999999},{"interface":"front","average":{"1min":1.829,"5min":1.673,"15min":2.0089999999999999},"min":{"1min":0.377,"5min":0.36499999999999999,"15min":0.30499999999999999},"max":{"1min":5.1459999999999999,"5min":8.0419999999999998,"15min":10.414},"last":26.067}]},{"osd":7,"last update":"Sat Sep 23 23:17:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":1.615,"15min":2.0129999999999999},"min":{"1min":0.35599999999999998,"5min":0.34499999999999997,"15min":0.29299999999999998},"max":{"1min":6.9340000000000002,"5min":8.4849999999999994,"15min":10.145},"last":26.413},{"interface":"front","average":{"1min":1.5649999999999999,"5min":1.4119999999999999,"15min":1.9490000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.313},"max":{"1min":6.0750000000000002,"5min":7.9480000000000004,"15min":10.513999999999999},"last":26.437999999999999}]}]},{"osd":2,"up_from":0,"seq":854698491926,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":148,"seq":635655159895,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1135216,"kb_used_data":820520,"kb_used_omap":2,"kb_used_meta":314685,"kb_avail":92609936,"statfs":{"total":95995035648,"available":94832574464,"internally_reserved":0,"allocated":840212480,"data_stored":839241787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":322237493},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:17:06 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":3.6099999999999999,"15min":2.1299999999999999},"min":{"1min":0.20399999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":9.6370000000000005,"5min":72.778999999999996,"15min":72.778999999999996},"last":0.25600000000000001},{"interface":"front","average":{"1min":1.891,"5min":3.0990000000000002,"15min":2.4900000000000002},"min":{"1min":0.23799999999999999,"5min":0.186,"15min":0.17999999999999999},"max":{"1min":6.6890000000000001,"5min":72.719999999999999,"15min":72.719999999999999},"last":1.5589999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:17:06 2023","interfaces":[{"interface":"back","average":{"1min":4.2190000000000003,"5min":3.7789999999999999,"15min":2.7229999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.19600000000000001},"max":{"1min":13.028,"5min":76.322999999999993,"15min":76.322999999999993},"last":1.0489999999999999},{"interface":"front","average":{"1min":3.9500000000000002,"5min":3.907,"15min":2.6030000000000002},"min":{"1min":0.23499999999999999,"5min":0.20499999999999999,"15min":0.185},"max":{"1min":13.970000000000001,"5min":76.444000000000003,"15min":76.444000000000003},"last":0.50700000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:17:23 2023","interfaces":[{"interface":"back","average":{"1min":3.7469999999999999,"5min":3.7469999999999999,"15min":3.7469999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":18.742999999999999,"5min":18.742999999999999,"15min":18.742999999999999},"last":0.80500000000000005},{"interface":"front","average":{"1min":1.6950000000000001,"5min":1.6950000000000001,"15min":1.6950000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.9290000000000003,"5min":5.9290000000000003,"15min":5.9290000000000003},"last":0.47699999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.5800000000000001,"15min":1.663},"min":{"1min":0.32300000000000001,"5min":0.315,"15min":0.29999999999999999},"max":{"1min":7.282,"5min":9.9749999999999996,"15min":9.9749999999999996},"last":2.7349999999999999},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.0740000000000001,"15min":2.548},"min":{"1min":0.32500000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.5569999999999999,"5min":8.0239999999999991,"15min":13.532999999999999},"last":3.766}]},{"osd":5,"last update":"Sat Sep 23 23:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.8220000000000001,"15min":2.4820000000000002},"min":{"1min":0.35899999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.5219999999999998,"5min":75.733000000000004,"15min":75.733000000000004},"last":1.5920000000000001},{"interface":"front","average":{"1min":2.1579999999999999,"5min":2.395,"15min":2.2549999999999999},"min":{"1min":0.44400000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.8119999999999994,"5min":75.686999999999998,"15min":75.686999999999998},"last":0.98799999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.145,"15min":2.3490000000000002},"min":{"1min":0.38,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.5790000000000002,"5min":8.0549999999999997,"15min":14.492000000000001},"last":1.631},{"interface":"front","average":{"1min":2.1469999999999998,"5min":1.522,"15min":2.0670000000000002},"min":{"1min":0.40200000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.8379999999999992,"5min":9.8379999999999992,"15min":9.8379999999999992},"last":1.079}]},{"osd":7,"last update":"Sat Sep 23 23:17:06 2023","interfaces":[{"interface":"back","average":{"1min":2.1619999999999999,"5min":2.6040000000000001,"15min":2.5699999999999998},"min":{"1min":0.38500000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.565,"5min":130.839,"15min":130.839},"last":0.66400000000000003},{"interface":"front","average":{"1min":1.806,"5min":2.5430000000000001,"15min":2.6509999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":14.778,"5min":130.90000000000001,"15min":130.90000000000001},"last":2.7170000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019213,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":913008,"kb_used_data":508392,"kb_used_omap":0,"kb_used_meta":404544,"kb_avail":92832144,"statfs":{"total":95995035648,"available":95060115456,"internally_reserved":0,"allocated":520593408,"data_stored":519858050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":414253056},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:17:34 2023","interfaces":[{"interface":"back","average":{"1min":2.7149999999999999,"5min":2.8900000000000001,"15min":3.004},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20799999999999999},"max":{"1min":15.071999999999999,"5min":102.12,"15min":238.62200000000001},"last":0.26600000000000001},{"interface":"front","average":{"1min":2.2629999999999999,"5min":2.694,"15min":3.3799999999999999},"min":{"1min":0.251,"5min":0.23499999999999999,"15min":0.23300000000000001},"max":{"1min":19.942,"5min":104.46899999999999,"15min":238.58199999999999},"last":3.403}]},{"osd":1,"last update":"Sat Sep 23 23:17:52 2023","interfaces":[{"interface":"back","average":{"1min":2.6539999999999999,"5min":6.5869999999999997,"15min":4.1600000000000001},"min":{"1min":0.253,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":12.348000000000001,"5min":113.25,"15min":175.00899999999999},"last":0.29699999999999999},{"interface":"front","average":{"1min":3.0699999999999998,"5min":6.7699999999999996,"15min":4.2519999999999998},"min":{"1min":0.28199999999999997,"5min":0.26200000000000001,"15min":0.25800000000000001},"max":{"1min":9.4779999999999998,"5min":113.321,"15min":174.821},"last":1.9950000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:17:21 2023","interfaces":[{"interface":"back","average":{"1min":3.5289999999999999,"5min":3.5289999999999999,"15min":3.5289999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":39.308,"5min":39.308,"15min":39.308},"last":0.57399999999999995},{"interface":"front","average":{"1min":2.6589999999999998,"5min":2.6589999999999998,"15min":2.6589999999999998},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":14.287000000000001,"5min":14.287000000000001,"15min":14.287000000000001},"last":1.282}]},{"osd":3,"last update":"Sat Sep 23 23:17:13 2023","interfaces":[{"interface":"back","average":{"1min":23.222000000000001,"5min":7.4770000000000003,"15min":3.71},"min":{"1min":0.314,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":303.41500000000002,"5min":303.41500000000002,"15min":303.41500000000002},"last":3.4199999999999999},{"interface":"front","average":{"1min":23.463999999999999,"5min":7.6299999999999999,"15min":4.117},"min":{"1min":0.32300000000000001,"5min":0.311,"15min":0.311},"max":{"1min":303.428,"5min":303.428,"15min":303.428},"last":2.1019999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:17:52 2023","interfaces":[{"interface":"back","average":{"1min":8.7690000000000001,"5min":2.4449999999999998,"15min":1.411},"min":{"1min":0.29899999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":160.79499999999999,"5min":160.79499999999999,"15min":160.79499999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":8.6760000000000002,"5min":2.6070000000000002,"15min":1.7230000000000001},"min":{"1min":0.317,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":160.61099999999999,"5min":160.61099999999999,"15min":160.61099999999999},"last":2.0649999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:17:13 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.0600000000000001,"15min":1.1359999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.253},"max":{"1min":6.0519999999999996,"5min":7.5499999999999998,"15min":14.4},"last":0.60199999999999998},{"interface":"front","average":{"1min":1.3400000000000001,"5min":1.0569999999999999,"15min":1.079},"min":{"1min":0.315,"5min":0.315,"15min":0.27300000000000002},"max":{"1min":7.2469999999999999,"5min":7.4459999999999997,"15min":13.888999999999999},"last":0.36399999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:17:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.3640000000000001,"15min":1.264},"min":{"1min":0.36199999999999999,"5min":0.315,"15min":0.30499999999999999},"max":{"1min":6.5640000000000001,"5min":14.842000000000001,"15min":14.842000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.875,"5min":1.222,"15min":1.319},"min":{"1min":0.379,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.3650000000000002,"5min":14.561,"15min":14.561},"last":0.63300000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757576,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1397208,"kb_used_data":957072,"kb_used_omap":0,"kb_used_meta":440128,"kb_avail":92347944,"statfs":{"total":95995035648,"available":94564294656,"internally_reserved":0,"allocated":980041728,"data_stored":979325833,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":450691072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:17:32 2023","interfaces":[{"interface":"back","average":{"1min":2.7599999999999998,"5min":2.254,"15min":3.681},"min":{"1min":0.24399999999999999,"5min":0.23000000000000001,"15min":0.21099999999999999},"max":{"1min":15.057,"5min":97.405000000000001,"15min":190.483},"last":6.5140000000000002},{"interface":"front","average":{"1min":1.663,"5min":2.278,"15min":4.2270000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.369999999999999,"5min":97.447000000000003,"15min":194.11099999999999},"last":1.0629999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:17:48 2023","interfaces":[{"interface":"back","average":{"1min":5.7270000000000003,"5min":6.1689999999999996,"15min":3.3700000000000001},"min":{"1min":0.27400000000000002,"5min":0.25,"15min":0.246},"max":{"1min":91.518000000000001,"5min":193.584,"15min":193.584},"last":2.7280000000000002},{"interface":"front","average":{"1min":4.9870000000000001,"5min":6.226,"15min":3.5070000000000001},"min":{"1min":0.30199999999999999,"5min":0.27800000000000002,"15min":0.26400000000000001},"max":{"1min":87.718000000000004,"5min":194.03100000000001,"15min":194.03100000000001},"last":1.3009999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:17:21 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.452,"15min":1.452},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.6239999999999997,"5min":5.6239999999999997,"15min":5.6239999999999997},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.7310000000000001,"5min":1.7310000000000001,"15min":1.7310000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.391,"5min":9.391,"15min":9.391},"last":0.52600000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:17:09 2023","interfaces":[{"interface":"back","average":{"1min":18.870000000000001,"5min":5.9950000000000001,"15min":3.238},"min":{"1min":0.29999999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":123.553,"5min":123.553,"15min":123.553},"last":30.521999999999998},{"interface":"front","average":{"1min":19.047999999999998,"5min":6.1559999999999997,"15min":4.125},"min":{"1min":0.45000000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":123.42,"5min":123.42,"15min":123.42},"last":30.553999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:18:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6579999999999999,"5min":1.1140000000000001,"15min":1.2030000000000001},"min":{"1min":0.25800000000000001,"5min":0.23999999999999999,"15min":0.221},"max":{"1min":7.3399999999999999,"5min":10.012,"15min":14.538},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.79100000000000004,"15min":0.874},"min":{"1min":0.314,"5min":0.26100000000000001,"15min":0.25},"max":{"1min":11.923999999999999,"5min":11.923999999999999,"15min":11.923999999999999},"last":0.56899999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:18:00 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":0.92600000000000005,"15min":1.113},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.255},"max":{"1min":5.2370000000000001,"5min":5.9809999999999999,"15min":13.226000000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.829,"5min":1.1639999999999999,"15min":1.264},"min":{"1min":0.32000000000000001,"5min":0.307,"15min":0.29099999999999998},"max":{"1min":12.178000000000001,"5min":12.178000000000001,"15min":13.098000000000001},"last":0.40699999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:17:41 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.97399999999999998,"15min":1.3029999999999999},"min":{"1min":0.32900000000000001,"5min":0.32700000000000001,"15min":0.29199999999999998},"max":{"1min":4.4560000000000004,"5min":4.9050000000000002,"15min":14.929},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.131,"5min":1.1160000000000001,"15min":1.226},"min":{"1min":0.315,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":6.4950000000000001,"5min":6.7089999999999996,"15min":15.090999999999999},"last":0.49299999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":544473088,"data_stored":544473088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":919343104,"data_stored":919339032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":263585792,"data_stored":263585792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":831717376,"data_stored":831717376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":511905792,"data_stored":511901720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":977014784,"data_stored":977010712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":490016768,"data_stored":490012696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1235619840,"data_stored":1235615768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:18:08.965 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 11 2023-09-23T23:18:09.052 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:18:09.082 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:09.142 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:18:09.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:09.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:18:09.178 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:09.251 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:09.280 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:09.326 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:09.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:09.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:09.430 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:18:09.468 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:09.542 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:18:09.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:09.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:18:09.650 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:18:09.689 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:09.838 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:09 smithi138 ceph-mon[193606]: failed to encode map e244 with expected crc 2023-09-23T23:18:09.839 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:09 smithi138 ceph-mon[193606]: failed to encode map e244 with expected crc 2023-09-23T23:18:09.839 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:09 smithi138 ceph-mon[193606]: failed to encode map e244 with expected crc 2023-09-23T23:18:09.839 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:09 smithi138 ceph-mon[193606]: failed to encode map e245 with expected crc 2023-09-23T23:18:09.839 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:09 smithi138 ceph-mon[193606]: osdmap e245: 8 total, 8 up, 7 in 2023-09-23T23:18:09.845 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:18:09.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:09.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:18:09.860 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:18:09.883 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:09 smithi110 ceph-mon[212939]: failed to encode map e244 with expected crc 2023-09-23T23:18:09.884 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:09 smithi110 ceph-mon[212939]: failed to encode map e244 with expected crc 2023-09-23T23:18:09.884 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:09 smithi110 ceph-mon[212939]: failed to encode map e244 with expected crc 2023-09-23T23:18:09.884 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:09 smithi110 ceph-mon[212939]: failed to encode map e245 with expected crc 2023-09-23T23:18:09.884 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:09 smithi110 ceph-mon[212939]: osdmap e245: 8 total, 8 up, 7 in 2023-09-23T23:18:09.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:09 smithi110 ceph-mon[221102]: failed to encode map e244 with expected crc 2023-09-23T23:18:09.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:09 smithi110 ceph-mon[221102]: failed to encode map e244 with expected crc 2023-09-23T23:18:09.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:09 smithi110 ceph-mon[221102]: failed to encode map e244 with expected crc 2023-09-23T23:18:09.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:09 smithi110 ceph-mon[221102]: failed to encode map e245 with expected crc 2023-09-23T23:18:09.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:09 smithi110 ceph-mon[221102]: osdmap e245: 8 total, 8 up, 7 in 2023-09-23T23:18:09.960 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:18:10.083 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:10.198 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:18:10.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:10.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:18:10.311 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:18:10.363 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:10.490 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:18:10.521 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:18:10.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:10.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:18:10.567 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:10 smithi110 ceph-mon[212939]: failed to encode map e245 with expected crc 2023-09-23T23:18:10.568 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:10 smithi110 ceph-mon[212939]: failed to encode map e245 with expected crc 2023-09-23T23:18:10.568 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:10 smithi110 ceph-mon[212939]: failed to encode map e245 with expected crc 2023-09-23T23:18:10.568 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:10 smithi110 ceph-mon[212939]: pgmap v754: 99 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+remapped, 93 active+clean, 3 active+remapped+backfill_wait; 1.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 7487/55806 objects misplaced (13.416%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T23:18:10.568 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:10 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1491614406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:18:10.574 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:18:10.628 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:10.652 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:10 smithi138 ceph-mon[193606]: failed to encode map e245 with expected crc 2023-09-23T23:18:10.652 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:10 smithi138 ceph-mon[193606]: failed to encode map e245 with expected crc 2023-09-23T23:18:10.652 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:10 smithi138 ceph-mon[193606]: failed to encode map e245 with expected crc 2023-09-23T23:18:10.653 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:10 smithi138 ceph-mon[193606]: pgmap v754: 99 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+remapped, 93 active+clean, 3 active+remapped+backfill_wait; 1.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 7487/55806 objects misplaced (13.416%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T23:18:10.653 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:10 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1491614406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:18:10.655 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:10.696 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:10.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:10.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:10.800 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:10.838 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:10.927 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:10.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:10.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:10.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:10 smithi110 ceph-mon[221102]: failed to encode map e245 with expected crc 2023-09-23T23:18:10.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:10 smithi110 ceph-mon[221102]: failed to encode map e245 with expected crc 2023-09-23T23:18:10.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:10 smithi110 ceph-mon[221102]: failed to encode map e245 with expected crc 2023-09-23T23:18:10.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:10 smithi110 ceph-mon[221102]: pgmap v754: 99 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+remapped, 93 active+clean, 3 active+remapped+backfill_wait; 1.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 7487/55806 objects misplaced (13.416%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T23:18:10.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:10 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1491614406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:18:11.031 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:11.086 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:11.287 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:11.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:11.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:11.395 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:18:11.456 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:11.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:11 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:18:11.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:11 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1491614406' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:18:11.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:11 smithi110 ceph-mon[212939]: osdmap e246: 8 total, 8 up, 7 in 2023-09-23T23:18:11.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:11 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:18:11.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:11 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1491614406' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:18:11.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:11 smithi110 ceph-mon[221102]: osdmap e246: 8 total, 8 up, 7 in 2023-09-23T23:18:11.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:11 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:18:11.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:11 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1491614406' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:18:11.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:11 smithi138 ceph-mon[193606]: osdmap e246: 8 total, 8 up, 7 in 2023-09-23T23:18:11.997 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:12.185 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:18:12.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:12.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:18:12.299 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:18:12.355 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:12.625 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:18:12.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:12.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:18:12.734 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:12.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:12 smithi138 ceph-mon[193606]: failed to encode map e246 with expected crc 2023-09-23T23:18:12.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:12 smithi138 ceph-mon[193606]: pgmap v756: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 peering, 93 active+clean, 3 active+remapped+backfill_wait; 1.1 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 6765/53979 objects misplaced (12.533%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:18:12.823 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:18:12.823 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:18:12.826 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:12.837 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:12 smithi110 ceph-mon[212939]: failed to encode map e246 with expected crc 2023-09-23T23:18:12.838 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:12 smithi110 ceph-mon[212939]: pgmap v756: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 peering, 93 active+clean, 3 active+remapped+backfill_wait; 1.1 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 6765/53979 objects misplaced (12.533%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:18:12.838 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:12 smithi110 ceph-mon[221102]: failed to encode map e246 with expected crc 2023-09-23T23:18:12.838 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:12 smithi110 ceph-mon[221102]: pgmap v756: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 peering, 93 active+clean, 3 active+remapped+backfill_wait; 1.1 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 6765/53979 objects misplaced (12.533%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:18:13.025 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:13.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:13.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:13.132 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:18:13.195 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:13.249 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:18:13.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:13.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:18:13.352 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:18:13.377 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:13.418 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:18:13.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:13.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:18:13.522 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:18:13.634 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:13.797 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:18:13.797 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:18:13.797 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:18:13.830 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:18:13.842 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:18:13.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:13.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:18:13.874 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:18:13.901 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:18:13.923 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T23:18:13.923 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:18:13.941 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:18:13.942 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:18:13.942 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:18:13.942 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:18:13.942 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:18:13.943 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:18:13.943 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:18:13.943 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:18:13.943 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:18:13.949 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:18:13.992 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:14.125 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:18:14.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:14.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:18:14.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:14.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:18:14.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:13 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/290952553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:14.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:14.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:18:14.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:13 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/290952553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:14.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:14.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:18:14.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:13 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/290952553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:14.236 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:14.287 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:14.312 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:18:14.497 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:18:15.155 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:14 smithi138 ceph-mon[193606]: pgmap v757: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 peering, 93 active+clean, 3 active+remapped+backfill_wait; 1.1 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 6765/52983 objects misplaced (12.768%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:18:15.345 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:15.347 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:14 smithi110 ceph-mon[212939]: pgmap v757: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 peering, 93 active+clean, 3 active+remapped+backfill_wait; 1.1 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 6765/52983 objects misplaced (12.768%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:18:15.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:15.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:15.349 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:14 smithi110 ceph-mon[221102]: pgmap v757: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 peering, 93 active+clean, 3 active+remapped+backfill_wait; 1.1 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 6765/52983 objects misplaced (12.768%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:18:15.449 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:18:15.494 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:15.623 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:18:15.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:15.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:18:15.740 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:18:15.755 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:15.800 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:16.000 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:18:16.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:16.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:18:16.109 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:16.195 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:16.326 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:16.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:16.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:16.435 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:18:16.516 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:16.677 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:18:16.677 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:18:16.677 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:18:16.677 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:18:16.677 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:18:16.722 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:18:16.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:16.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:18:16.826 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:18:16.910 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:16.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:18:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:18:16] "GET /metrics HTTP/1.1" 200 36543 "" "Prometheus/2.43.0" 2023-09-23T23:18:17.009 INFO:teuthology.orchestra.run.smithi110.stderr:set pool 1 pgp_num to 11 2023-09-23T23:18:17.010 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:17 smithi138 ceph-mon[193606]: pgmap v758: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 992 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 6260/47685 objects misplaced (13.128%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T23:18:17.010 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:17 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2002650644' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T23:18:17.010 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:17 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T23:18:17.035 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:18:17.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:17.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:18:17.146 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:18:17.174 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:17.221 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:18:17.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:17.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:18:17.325 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:18:17.363 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:17 smithi110 ceph-mon[212939]: pgmap v758: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 992 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 6260/47685 objects misplaced (13.128%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T23:18:17.363 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:17 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2002650644' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T23:18:17.363 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:17 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T23:18:17.363 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:17 smithi110 ceph-mon[221102]: pgmap v758: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 992 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 6260/47685 objects misplaced (13.128%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T23:18:17.363 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:17 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2002650644' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T23:18:17.363 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:17 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T23:18:17.365 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:17.416 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:18:17.710 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T23:18:18.056 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:18:18.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:18.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:18:18.165 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:18:18.232 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:18.341 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:18:18.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:18 smithi138 ceph-mon[193606]: failed to encode map e247 with expected crc 2023-09-23T23:18:18.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:18 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T23:18:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:18 smithi138 ceph-mon[193606]: osdmap e247: 8 total, 8 up, 7 in 2023-09-23T23:18:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:18 smithi138 ceph-mon[193606]: failed to encode map e247 with expected crc 2023-09-23T23:18:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:18 smithi138 ceph-mon[193606]: failed to encode map e247 with expected crc 2023-09-23T23:18:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:18 smithi110 ceph-mon[221102]: failed to encode map e247 with expected crc 2023-09-23T23:18:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:18 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T23:18:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:18 smithi110 ceph-mon[221102]: osdmap e247: 8 total, 8 up, 7 in 2023-09-23T23:18:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:18 smithi110 ceph-mon[221102]: failed to encode map e247 with expected crc 2023-09-23T23:18:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:18 smithi110 ceph-mon[221102]: failed to encode map e247 with expected crc 2023-09-23T23:18:18.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:18 smithi110 ceph-mon[212939]: failed to encode map e247 with expected crc 2023-09-23T23:18:18.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:18 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T23:18:18.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:18 smithi110 ceph-mon[212939]: osdmap e247: 8 total, 8 up, 7 in 2023-09-23T23:18:18.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:18 smithi110 ceph-mon[212939]: failed to encode map e247 with expected crc 2023-09-23T23:18:18.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:18 smithi110 ceph-mon[212939]: failed to encode map e247 with expected crc 2023-09-23T23:18:19.001 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:18:19.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:19.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:18:19.128 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:18:19.190 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:19.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:19 smithi110 ceph-mon[221102]: pgmap v760: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 950 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 5374/45630 objects misplaced (11.777%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T23:18:19.211 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:19 smithi110 ceph-mon[212939]: pgmap v760: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 950 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 5374/45630 objects misplaced (11.777%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T23:18:19.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:19 smithi138 ceph-mon[193606]: pgmap v760: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 950 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 5374/45630 objects misplaced (11.777%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T23:18:19.492 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:19.566 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:18:19.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:19.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:18:19.677 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:19.741 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:19.998 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:20.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:20.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:20.089 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:18:20.109 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:20.117 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:20 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/883009529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:18:20.123 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:20 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/883009529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:18:20.184 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:20.371 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:20 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/883009529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:18:20.675 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:18:20.675 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:18:20.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:18:20.675 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:18:20.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.675 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127467 9.d 9:b004077a:::benchmark_data_smithi110_295904_object16286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T23:18:20.675 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.526680+0000", 2023-09-23T23:18:20.676 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 173.14058742, 2023-09-23T23:18:20.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7990730689999999, 2023-09-23T23:18:20.676 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.676 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:18:20.676 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:20.676 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:20.676 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:20.676 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127467 2023-09-23T23:18:20.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.676 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526680+0000", 2023-09-23T23:18:20.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526680+0000", 2023-09-23T23:18:20.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526679+0000", 2023-09-23T23:18:20.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:18:20.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526685+0000", 2023-09-23T23:18:20.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9479999999999996e-06 2023-09-23T23:18:20.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526699+0000", 2023-09-23T23:18:20.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.398e-05 2023-09-23T23:18:20.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526704+0000", 2023-09-23T23:18:20.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4e-06 2023-09-23T23:18:20.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526763+0000", 2023-09-23T23:18:20.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8575000000000001e-05 2023-09-23T23:18:20.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526873+0000", 2023-09-23T23:18:20.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110303 2023-09-23T23:18:20.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,5", 2023-09-23T23:18:20.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.526958+0000", 2023-09-23T23:18:20.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4518999999999995e-05 2023-09-23T23:18:20.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:20.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.530629+0000", 2023-09-23T23:18:20.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0036707380000000002 2023-09-23T23:18:20.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.530882+0000", 2023-09-23T23:18:20.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025305899999999998 2023-09-23T23:18:20.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.546584+0000", 2023-09-23T23:18:20.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015702391 2023-09-23T23:18:20.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.546595+0000", 2023-09-23T23:18:20.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.119e-05 2023-09-23T23:18:20.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222624+0000", 2023-09-23T23:18:20.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.67602865199999995 2023-09-23T23:18:20.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222636+0000", 2023-09-23T23:18:20.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2053e-05 2023-09-23T23:18:20.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.683 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.325683+0000", 2023-09-23T23:18:20.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.103046642 2023-09-23T23:18:20.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.683 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.325753+0000", 2023-09-23T23:18:20.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0722999999999994e-05 2023-09-23T23:18:20.683 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.683 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.683 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.684 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.684 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127477 9.d 9:be22822d:::benchmark_data_smithi110_295904_object16296:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T23:18:20.684 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.541829+0000", 2023-09-23T23:18:20.684 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 173.12543847399999, 2023-09-23T23:18:20.684 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.97042316, 2023-09-23T23:18:20.684 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.684 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.684 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:20.684 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:20.684 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:20.684 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127477 2023-09-23T23:18:20.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.685 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.541829+0000", 2023-09-23T23:18:20.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.541829+0000", 2023-09-23T23:18:20.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.686 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.686 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.541828+0000", 2023-09-23T23:18:20.686 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:18:20.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.686 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.686 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.541833+0000", 2023-09-23T23:18:20.686 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.707e-06 2023-09-23T23:18:20.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.686 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.687 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.541842+0000", 2023-09-23T23:18:20.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5210000000000008e-06 2023-09-23T23:18:20.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.687 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.687 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.687 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.541849+0000", 2023-09-23T23:18:20.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5710000000000003e-06 2023-09-23T23:18:20.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.687 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.687 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.687 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.546848+0000", 2023-09-23T23:18:20.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049989539999999999 2023-09-23T23:18:20.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.546855+0000", 2023-09-23T23:18:20.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6900000000000003e-06 2023-09-23T23:18:20.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222663+0000", 2023-09-23T23:18:20.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.67580857900000002 2023-09-23T23:18:20.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222670+0000", 2023-09-23T23:18:20.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2830000000000004e-06 2023-09-23T23:18:20.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.325792+0000", 2023-09-23T23:18:20.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.103121223 2023-09-23T23:18:20.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.325890+0000", 2023-09-23T23:18:20.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7864999999999999e-05 2023-09-23T23:18:20.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:18:20.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.325985+0000", 2023-09-23T23:18:20.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5896000000000005e-05 2023-09-23T23:18:20.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:20.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.334870+0000", 2023-09-23T23:18:20.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088846629999999992 2023-09-23T23:18:20.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.691 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.352987+0000", 2023-09-23T23:18:20.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018117366999999999 2023-09-23T23:18:20.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.691 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.691 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353011+0000", 2023-09-23T23:18:20.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3506000000000002e-05 2023-09-23T23:18:20.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.691 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.691 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.512087+0000", 2023-09-23T23:18:20.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.15907564299999999 2023-09-23T23:18:20.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.512151+0000", 2023-09-23T23:18:20.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4381000000000006e-05 2023-09-23T23:18:20.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.512252+0000", 2023-09-23T23:18:20.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101345 2023-09-23T23:18:20.692 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.693 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.693 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.693 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127490 9.d 9:b0fef0a1:::benchmark_data_smithi110_295904_object16309:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:18:20.693 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.559146+0000", 2023-09-23T23:18:20.693 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 173.10812187799999, 2023-09-23T23:18:20.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.955891351, 2023-09-23T23:18:20.693 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.693 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.693 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:20.693 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:20.693 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:20.694 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127490 2023-09-23T23:18:20.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.694 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.694 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.694 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.559146+0000", 2023-09-23T23:18:20.694 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.694 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.694 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.559146+0000", 2023-09-23T23:18:20.694 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.695 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.695 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.695 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.695 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.559145+0000", 2023-09-23T23:18:20.695 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:18:20.695 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.695 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.695 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.695 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.559148+0000", 2023-09-23T23:18:20.695 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.875e-06 2023-09-23T23:18:20.695 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.559161+0000", 2023-09-23T23:18:20.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2527000000000001e-05 2023-09-23T23:18:20.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.559165+0000", 2023-09-23T23:18:20.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3789999999999999e-06 2023-09-23T23:18:20.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.559227+0000", 2023-09-23T23:18:20.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2131000000000005e-05 2023-09-23T23:18:20.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.697 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.559236+0000", 2023-09-23T23:18:20.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9190000000000005e-06 2023-09-23T23:18:20.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.697 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222687+0000", 2023-09-23T23:18:20.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.66345047999999995 2023-09-23T23:18:20.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.698 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222691+0000", 2023-09-23T23:18:20.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0589999999999996e-06 2023-09-23T23:18:20.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.698 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.328857+0000", 2023-09-23T23:18:20.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1061658780000001 2023-09-23T23:18:20.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.698 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.699 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.329003+0000", 2023-09-23T23:18:20.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014625799999999999 2023-09-23T23:18:20.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.699 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.699 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:18:20.699 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.329170+0000", 2023-09-23T23:18:20.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016664499999999999 2023-09-23T23:18:20.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.699 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.699 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:20.699 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.337592+0000", 2023-09-23T23:18:20.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0084228219999999999 2023-09-23T23:18:20.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353036+0000", 2023-09-23T23:18:20.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015444049 2023-09-23T23:18:20.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353086+0000", 2023-09-23T23:18:20.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9873000000000002e-05 2023-09-23T23:18:20.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.514795+0000", 2023-09-23T23:18:20.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16170854800000001 2023-09-23T23:18:20.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.514873+0000", 2023-09-23T23:18:20.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7756000000000006e-05 2023-09-23T23:18:20.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515037+0000", 2023-09-23T23:18:20.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016452400000000001 2023-09-23T23:18:20.701 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.701 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.702 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.702 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.702 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.702 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127492 9.d 9:befecd6a:::benchmark_data_smithi110_295904_object16311:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:18:20.702 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.560155+0000", 2023-09-23T23:18:20.702 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 173.107112446, 2023-09-23T23:18:20.702 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9551049470000001, 2023-09-23T23:18:20.702 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.702 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.702 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:20.702 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:20.702 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:20.703 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127492 2023-09-23T23:18:20.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.703 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.703 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.703 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.560155+0000", 2023-09-23T23:18:20.703 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.703 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.703 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.560155+0000", 2023-09-23T23:18:20.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.704 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.704 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.560154+0000", 2023-09-23T23:18:20.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:18:20.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.704 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.704 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.560159+0000", 2023-09-23T23:18:20.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8409999999999999e-06 2023-09-23T23:18:20.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.705 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.560173+0000", 2023-09-23T23:18:20.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3359e-05 2023-09-23T23:18:20.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.705 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.560178+0000", 2023-09-23T23:18:20.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3569999999999997e-06 2023-09-23T23:18:20.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.705 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.560225+0000", 2023-09-23T23:18:20.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6548000000000002e-05 2023-09-23T23:18:20.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.706 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.706 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.706 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.560232+0000", 2023-09-23T23:18:20.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5279999999999998e-06 2023-09-23T23:18:20.706 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.706 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.706 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.706 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222702+0000", 2023-09-23T23:18:20.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.662470114 2023-09-23T23:18:20.706 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.706 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.706 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222706+0000", 2023-09-23T23:18:20.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4750000000000002e-06 2023-09-23T23:18:20.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.331907+0000", 2023-09-23T23:18:20.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1092014429999999 2023-09-23T23:18:20.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.332043+0000", 2023-09-23T23:18:20.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013605799999999999 2023-09-23T23:18:20.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:18:20.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.332202+0000", 2023-09-23T23:18:20.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015870700000000001 2023-09-23T23:18:20.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:20.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.343380+0000", 2023-09-23T23:18:20.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011177576999999999 2023-09-23T23:18:20.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353063+0000", 2023-09-23T23:18:20.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.009683199 2023-09-23T23:18:20.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.709 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353166+0000", 2023-09-23T23:18:20.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010363500000000001 2023-09-23T23:18:20.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.709 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515087+0000", 2023-09-23T23:18:20.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16192087399999999 2023-09-23T23:18:20.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515144+0000", 2023-09-23T23:18:20.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6725999999999999e-05 2023-09-23T23:18:20.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515260+0000", 2023-09-23T23:18:20.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000116186 2023-09-23T23:18:20.710 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.710 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.710 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.711 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.711 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127498 9.d 9:b0f15af0:::benchmark_data_smithi110_295904_object16317:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:18:20.711 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.566686+0000", 2023-09-23T23:18:20.711 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 173.10058159299999, 2023-09-23T23:18:20.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.948764122, 2023-09-23T23:18:20.711 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.711 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.711 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:20.711 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:20.711 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:20.711 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127498 2023-09-23T23:18:20.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.711 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.566686+0000", 2023-09-23T23:18:20.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.566686+0000", 2023-09-23T23:18:20.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.566686+0000", 2023-09-23T23:18:20.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:18:20.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.713 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.713 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.566689+0000", 2023-09-23T23:18:20.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5090000000000002e-06 2023-09-23T23:18:20.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.713 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.713 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.566696+0000", 2023-09-23T23:18:20.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8839999999999997e-06 2023-09-23T23:18:20.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.713 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.566700+0000", 2023-09-23T23:18:20.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0249999999999996e-06 2023-09-23T23:18:20.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.566748+0000", 2023-09-23T23:18:20.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7771000000000001e-05 2023-09-23T23:18:20.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.566754+0000", 2023-09-23T23:18:20.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8220000000000003e-06 2023-09-23T23:18:20.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222815+0000", 2023-09-23T23:18:20.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.65606194100000004 2023-09-23T23:18:20.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222825+0000", 2023-09-23T23:18:20.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.533e-06 2023-09-23T23:18:20.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.716 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.334889+0000", 2023-09-23T23:18:20.716 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.11206366 2023-09-23T23:18:20.716 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.716 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.334971+0000", 2023-09-23T23:18:20.716 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2327e-05 2023-09-23T23:18:20.716 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.716 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:18:20.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.335051+0000", 2023-09-23T23:18:20.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0158000000000001e-05 2023-09-23T23:18:20.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:20.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.352730+0000", 2023-09-23T23:18:20.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017678366000000001 2023-09-23T23:18:20.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353110+0000", 2023-09-23T23:18:20.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00038028399999999999 2023-09-23T23:18:20.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353214+0000", 2023-09-23T23:18:20.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010437699999999999 2023-09-23T23:18:20.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515295+0000", 2023-09-23T23:18:20.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.162080369 2023-09-23T23:18:20.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515338+0000", 2023-09-23T23:18:20.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3897999999999999e-05 2023-09-23T23:18:20.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.719 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515450+0000", 2023-09-23T23:18:20.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000111698 2023-09-23T23:18:20.719 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.719 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.719 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.719 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127508 9.d 9:b620324a:::benchmark_data_smithi110_295904_object16327:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:18:20.720 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.579645+0000", 2023-09-23T23:18:20.720 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 173.08762228000001, 2023-09-23T23:18:20.720 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.935992645, 2023-09-23T23:18:20.720 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.720 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.720 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:20.720 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:20.720 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:20.720 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127508 2023-09-23T23:18:20.720 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.720 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.720 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.721 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.721 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579645+0000", 2023-09-23T23:18:20.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.721 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.721 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579645+0000", 2023-09-23T23:18:20.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.721 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.721 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579644+0000", 2023-09-23T23:18:20.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:18:20.722 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.722 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579649+0000", 2023-09-23T23:18:20.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1930000000000001e-06 2023-09-23T23:18:20.722 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.722 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579661+0000", 2023-09-23T23:18:20.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1627999999999999e-05 2023-09-23T23:18:20.722 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579681+0000", 2023-09-23T23:18:20.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0066999999999999e-05 2023-09-23T23:18:20.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579768+0000", 2023-09-23T23:18:20.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6817999999999994e-05 2023-09-23T23:18:20.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579777+0000", 2023-09-23T23:18:20.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1849999999999993e-06 2023-09-23T23:18:20.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222866+0000", 2023-09-23T23:18:20.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.64308858499999999 2023-09-23T23:18:20.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222874+0000", 2023-09-23T23:18:20.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7749999999999998e-06 2023-09-23T23:18:20.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.337631+0000", 2023-09-23T23:18:20.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1147567780000001 2023-09-23T23:18:20.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.337737+0000", 2023-09-23T23:18:20.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105858 2023-09-23T23:18:20.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:18:20.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.337833+0000", 2023-09-23T23:18:20.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5952000000000006e-05 2023-09-23T23:18:20.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:20.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.352739+0000", 2023-09-23T23:18:20.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014905899 2023-09-23T23:18:20.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353141+0000", 2023-09-23T23:18:20.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00040202700000000001 2023-09-23T23:18:20.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353261+0000", 2023-09-23T23:18:20.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000119697 2023-09-23T23:18:20.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515485+0000", 2023-09-23T23:18:20.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16222382799999999 2023-09-23T23:18:20.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.728 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515527+0000", 2023-09-23T23:18:20.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2522999999999999e-05 2023-09-23T23:18:20.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.728 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.728 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.515638+0000", 2023-09-23T23:18:20.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110931 2023-09-23T23:18:20.728 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.728 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.728 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.728 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127509 9.d 9:bda00fa7:::benchmark_data_smithi110_295904_object16328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:18:20.729 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.579703+0000", 2023-09-23T23:18:20.729 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 173.08756483600001, 2023-09-23T23:18:20.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9397497079999999, 2023-09-23T23:18:20.729 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.729 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.729 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:20.729 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:20.729 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:20.729 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127509 2023-09-23T23:18:20.729 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.729 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579703+0000", 2023-09-23T23:18:20.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579703+0000", 2023-09-23T23:18:20.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579702+0000", 2023-09-23T23:18:20.731 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:18:20.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.731 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.731 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579705+0000", 2023-09-23T23:18:20.731 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3e-06 2023-09-23T23:18:20.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.731 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.731 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579711+0000", 2023-09-23T23:18:20.731 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.601e-06 2023-09-23T23:18:20.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579714+0000", 2023-09-23T23:18:20.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9799999999999998e-06 2023-09-23T23:18:20.732 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579791+0000", 2023-09-23T23:18:20.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6923999999999997e-05 2023-09-23T23:18:20.732 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.579795+0000", 2023-09-23T23:18:20.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5240000000000001e-06 2023-09-23T23:18:20.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222904+0000", 2023-09-23T23:18:20.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.643108825 2023-09-23T23:18:20.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222911+0000", 2023-09-23T23:18:20.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.233e-06 2023-09-23T23:18:20.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.340587+0000", 2023-09-23T23:18:20.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1176763380000001 2023-09-23T23:18:20.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.340704+0000", 2023-09-23T23:18:20.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011648 2023-09-23T23:18:20.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:18:20.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.340789+0000", 2023-09-23T23:18:20.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5054000000000003e-05 2023-09-23T23:18:20.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.735 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:20.735 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.352916+0000", 2023-09-23T23:18:20.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012127033000000001 2023-09-23T23:18:20.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.735 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.735 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353191+0000", 2023-09-23T23:18:20.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00027495599999999997 2023-09-23T23:18:20.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353283+0000", 2023-09-23T23:18:20.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2603999999999996e-05 2023-09-23T23:18:20.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.519298+0000", 2023-09-23T23:18:20.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16601465800000001 2023-09-23T23:18:20.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.519330+0000", 2023-09-23T23:18:20.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2190000000000002e-05 2023-09-23T23:18:20.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.519452+0000", 2023-09-23T23:18:20.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012235999999999999 2023-09-23T23:18:20.737 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.737 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.737 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.738 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127515 9.d 9:b8066f22:::benchmark_data_smithi110_295904_object16334:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:18:20.738 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.587604+0000", 2023-09-23T23:18:20.738 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 173.07966334899999, 2023-09-23T23:18:20.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9319918469999999, 2023-09-23T23:18:20.738 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.738 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.738 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:20.738 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:20.738 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:20.738 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127515 2023-09-23T23:18:20.738 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.739 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.739 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.739 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.587604+0000", 2023-09-23T23:18:20.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.739 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.739 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.587604+0000", 2023-09-23T23:18:20.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.739 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.739 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.587603+0000", 2023-09-23T23:18:20.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:18:20.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.587609+0000", 2023-09-23T23:18:20.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9089999999999997e-06 2023-09-23T23:18:20.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.587624+0000", 2023-09-23T23:18:20.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5162e-05 2023-09-23T23:18:20.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.587630+0000", 2023-09-23T23:18:20.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2060000000000004e-06 2023-09-23T23:18:20.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.587679+0000", 2023-09-23T23:18:20.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8223999999999998e-05 2023-09-23T23:18:20.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.587686+0000", 2023-09-23T23:18:20.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9940000000000003e-06 2023-09-23T23:18:20.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222948+0000", 2023-09-23T23:18:20.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.63526185199999996 2023-09-23T23:18:20.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222955+0000", 2023-09-23T23:18:20.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2880000000000004e-06 2023-09-23T23:18:20.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.343431+0000", 2023-09-23T23:18:20.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.120475756 2023-09-23T23:18:20.743 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.343552+0000", 2023-09-23T23:18:20.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000121666 2023-09-23T23:18:20.743 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:18:20.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.343632+0000", 2023-09-23T23:18:20.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9900000000000004e-05 2023-09-23T23:18:20.743 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.744 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:20.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.352930+0000", 2023-09-23T23:18:20.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.009298054 2023-09-23T23:18:20.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.744 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353238+0000", 2023-09-23T23:18:20.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000307569 2023-09-23T23:18:20.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.744 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353329+0000", 2023-09-23T23:18:20.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0870000000000002e-05 2023-09-23T23:18:20.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.519497+0000", 2023-09-23T23:18:20.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16616813799999999 2023-09-23T23:18:20.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.519523+0000", 2023-09-23T23:18:20.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.582e-05 2023-09-23T23:18:20.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.746 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.519596+0000", 2023-09-23T23:18:20.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3528000000000005e-05 2023-09-23T23:18:20.746 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.746 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.746 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.746 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.746 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.746 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127518 9.d 9:b6c2eec4:::benchmark_data_smithi110_295904_object16337:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:18:20.746 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.592077+0000", 2023-09-23T23:18:20.746 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 173.07519055099999, 2023-09-23T23:18:20.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.930210711, 2023-09-23T23:18:20.747 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.747 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.747 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:20.747 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:20.747 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:20.747 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127518 2023-09-23T23:18:20.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.747 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.747 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.747 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.592077+0000", 2023-09-23T23:18:20.747 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.592077+0000", 2023-09-23T23:18:20.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.592076+0000", 2023-09-23T23:18:20.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:18:20.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.592081+0000", 2023-09-23T23:18:20.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6569999999999996e-06 2023-09-23T23:18:20.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.749 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.592094+0000", 2023-09-23T23:18:20.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2619000000000001e-05 2023-09-23T23:18:20.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.749 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.592099+0000", 2023-09-23T23:18:20.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0780000000000003e-06 2023-09-23T23:18:20.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.592140+0000", 2023-09-23T23:18:20.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1115000000000002e-05 2023-09-23T23:18:20.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.592145+0000", 2023-09-23T23:18:20.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5899999999999998e-06 2023-09-23T23:18:20.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222978+0000", 2023-09-23T23:18:20.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.63083219899999998 2023-09-23T23:18:20.751 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.222985+0000", 2023-09-23T23:18:20.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9600000000000003e-06 2023-09-23T23:18:20.751 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.346170+0000", 2023-09-23T23:18:20.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1231851390000001 2023-09-23T23:18:20.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.346304+0000", 2023-09-23T23:18:20.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013475999999999999 2023-09-23T23:18:20.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:18:20.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.346442+0000", 2023-09-23T23:18:20.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013756700000000001 2023-09-23T23:18:20.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:20.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.352939+0000", 2023-09-23T23:18:20.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064967480000000001 2023-09-23T23:18:20.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353307+0000", 2023-09-23T23:18:20.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00036789100000000002 2023-09-23T23:18:20.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.358342+0000", 2023-09-23T23:18:20.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050351759999999997 2023-09-23T23:18:20.755 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.522137+0000", 2023-09-23T23:18:20.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.163795633 2023-09-23T23:18:20.755 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.522184+0000", 2023-09-23T23:18:20.756 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6350999999999997e-05 2023-09-23T23:18:20.756 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.756 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.756 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.756 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.522288+0000", 2023-09-23T23:18:20.756 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010393800000000001 2023-09-23T23:18:20.756 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.756 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.756 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.756 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.756 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.756 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:127533 9.d 9:b4a925a7:::benchmark_data_smithi110_295904_object16352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:18:20.757 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:27.618409+0000", 2023-09-23T23:18:20.757 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 173.04885823000001, 2023-09-23T23:18:20.757 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9040714919999999, 2023-09-23T23:18:20.757 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.757 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.757 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:20.757 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:20.757 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:20.757 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127533 2023-09-23T23:18:20.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.757 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.618409+0000", 2023-09-23T23:18:20.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.758 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.618409+0000", 2023-09-23T23:18:20.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.758 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.759 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.618408+0000", 2023-09-23T23:18:20.759 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:18:20.759 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.759 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.759 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.759 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.618413+0000", 2023-09-23T23:18:20.759 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4079999999999997e-06 2023-09-23T23:18:20.759 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.759 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.759 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.759 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.618426+0000", 2023-09-23T23:18:20.760 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2021000000000001e-05 2023-09-23T23:18:20.760 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.760 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.760 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.760 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.618430+0000", 2023-09-23T23:18:20.760 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8600000000000001e-06 2023-09-23T23:18:20.760 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.760 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.760 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.760 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.618497+0000", 2023-09-23T23:18:20.760 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6228000000000004e-05 2023-09-23T23:18:20.760 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.761 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.761 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:27.618506+0000", 2023-09-23T23:18:20.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4709999999999996e-06 2023-09-23T23:18:20.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.761 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.761 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.223043+0000", 2023-09-23T23:18:20.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.60453667499999997 2023-09-23T23:18:20.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.761 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.762 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:28.223052+0000", 2023-09-23T23:18:20.762 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.081e-06 2023-09-23T23:18:20.762 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.762 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.762 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.762 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.349615+0000", 2023-09-23T23:18:20.762 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1265627229999999 2023-09-23T23:18:20.762 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.762 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.762 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.762 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.349757+0000", 2023-09-23T23:18:20.762 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014230100000000001 2023-09-23T23:18:20.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.763 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.763 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:18:20.763 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.349908+0000", 2023-09-23T23:18:20.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000151323 2023-09-23T23:18:20.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.763 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.763 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:20.763 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353017+0000", 2023-09-23T23:18:20.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031085539999999999 2023-09-23T23:18:20.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.763 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.764 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.764 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.353350+0000", 2023-09-23T23:18:20.764 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00033329399999999998 2023-09-23T23:18:20.764 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.764 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.764 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.764 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.358375+0000", 2023-09-23T23:18:20.764 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005024934 2023-09-23T23:18:20.764 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.764 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.764 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.764 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.522338+0000", 2023-09-23T23:18:20.764 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16396344700000001 2023-09-23T23:18:20.765 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.765 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.765 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.765 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.522385+0000", 2023-09-23T23:18:20.765 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6489999999999997e-05 2023-09-23T23:18:20.765 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.765 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.765 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.765 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:29.522481+0000", 2023-09-23T23:18:20.765 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5995999999999994e-05 2023-09-23T23:18:20.765 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.765 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.766 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.766 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.766 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.766 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:136466 9.d 9:bc0d042c:::benchmark_data_smithi110_295904_object25285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e192)", 2023-09-23T23:18:20.766 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:46.434300+0000", 2023-09-23T23:18:20.766 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 154.23296775200001, 2023-09-23T23:18:20.766 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0567469730000001, 2023-09-23T23:18:20.766 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.766 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.766 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:20.767 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:20.767 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:20.767 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 136466 2023-09-23T23:18:20.767 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.767 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.767 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.767 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.767 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.434300+0000", 2023-09-23T23:18:20.767 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.767 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.767 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.767 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.767 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.434300+0000", 2023-09-23T23:18:20.768 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.768 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.768 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.768 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.768 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.434298+0000", 2023-09-23T23:18:20.768 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:18:20.768 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.768 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.768 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.768 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.434306+0000", 2023-09-23T23:18:20.768 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2509999999999997e-06 2023-09-23T23:18:20.768 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.768 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.769 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.769 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.434318+0000", 2023-09-23T23:18:20.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2771e-05 2023-09-23T23:18:20.769 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.769 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.769 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.769 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.434323+0000", 2023-09-23T23:18:20.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9660000000000004e-06 2023-09-23T23:18:20.769 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.769 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.769 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.769 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.434372+0000", 2023-09-23T23:18:20.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8775e-05 2023-09-23T23:18:20.770 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.770 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.770 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.770 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.434381+0000", 2023-09-23T23:18:20.770 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9290000000000004e-06 2023-09-23T23:18:20.770 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.770 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.770 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.770 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.444524+0000", 2023-09-23T23:18:20.770 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0101426259999999 2023-09-23T23:18:20.770 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.770 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.770 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.771 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.444677+0000", 2023-09-23T23:18:20.771 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015341700000000001 2023-09-23T23:18:20.771 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.771 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.771 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:18:20.771 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.444834+0000", 2023-09-23T23:18:20.771 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015701499999999999 2023-09-23T23:18:20.771 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.771 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.771 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:20.771 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.453710+0000", 2023-09-23T23:18:20.771 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088758589999999998 2023-09-23T23:18:20.771 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.772 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.772 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.772 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.490907+0000", 2023-09-23T23:18:20.772 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.037197075000000003 2023-09-23T23:18:20.772 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.772 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.772 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.772 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.490952+0000", 2023-09-23T23:18:20.772 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4617999999999999e-05 2023-09-23T23:18:20.772 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.772 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.772 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.772 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.491009+0000", 2023-09-23T23:18:20.773 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7163000000000001e-05 2023-09-23T23:18:20.773 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.773 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.773 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.773 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.491047+0000", 2023-09-23T23:18:20.773 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7910999999999997e-05 2023-09-23T23:18:20.773 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.773 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.773 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.773 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.773 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.773 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:136501 9.d 9:b0419d0f:::benchmark_data_smithi110_295904_object25320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e192)", 2023-09-23T23:18:20.774 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:15:46.547205+0000", 2023-09-23T23:18:20.774 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 154.12006229900001, 2023-09-23T23:18:20.774 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94398360299999995, 2023-09-23T23:18:20.774 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.774 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.774 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:20.774 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:20.774 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:20.774 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 136501 2023-09-23T23:18:20.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.774 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.547205+0000", 2023-09-23T23:18:20.775 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.775 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.547205+0000", 2023-09-23T23:18:20.775 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.775 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.547204+0000", 2023-09-23T23:18:20.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:18:20.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.776 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.776 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.776 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.547209+0000", 2023-09-23T23:18:20.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3959999999999997e-06 2023-09-23T23:18:20.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.776 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.776 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.776 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.547222+0000", 2023-09-23T23:18:20.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3185e-05 2023-09-23T23:18:20.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.777 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.547229+0000", 2023-09-23T23:18:20.777 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5690000000000001e-06 2023-09-23T23:18:20.777 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.777 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.547280+0000", 2023-09-23T23:18:20.777 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1319000000000001e-05 2023-09-23T23:18:20.777 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:18:20.778 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:46.547288+0000", 2023-09-23T23:18:20.778 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6120000000000001e-06 2023-09-23T23:18:20.778 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.778 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.778 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.778 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.447782+0000", 2023-09-23T23:18:20.778 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.90049416999999998 2023-09-23T23:18:20.778 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.778 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.778 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.778 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.447910+0000", 2023-09-23T23:18:20.778 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012784499999999999 2023-09-23T23:18:20.779 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.779 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.779 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:18:20.779 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.447993+0000", 2023-09-23T23:18:20.779 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3419999999999998e-05 2023-09-23T23:18:20.779 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.779 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.779 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:20.779 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.465774+0000", 2023-09-23T23:18:20.779 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017780267999999998 2023-09-23T23:18:20.779 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.779 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.780 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.780 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.491086+0000", 2023-09-23T23:18:20.780 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.025311951999999999 2023-09-23T23:18:20.780 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.780 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.780 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.780 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.491112+0000", 2023-09-23T23:18:20.780 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6370999999999999e-05 2023-09-23T23:18:20.780 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.780 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.780 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.780 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.491158+0000", 2023-09-23T23:18:20.780 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5908999999999997e-05 2023-09-23T23:18:20.781 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.781 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.781 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.781 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:15:47.491189+0000", 2023-09-23T23:18:20.781 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0997000000000002e-05 2023-09-23T23:18:20.781 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.781 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.781 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.781 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.781 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.781 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:164261 9.e 9:7a81427e:::benchmark_data_smithi110_295904_object53080:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T23:18:20.781 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:40.243097+0000", 2023-09-23T23:18:20.782 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 100.424171083, 2023-09-23T23:18:20.782 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.29852363799999998, 2023-09-23T23:18:20.782 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.782 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.782 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:20.782 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:20.782 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:20.782 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 164261 2023-09-23T23:18:20.782 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.782 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.782 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.783 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.783 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.243097+0000", 2023-09-23T23:18:20.783 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.783 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.783 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.783 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.783 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.243097+0000", 2023-09-23T23:18:20.783 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.783 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.783 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.783 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.783 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.243096+0000", 2023-09-23T23:18:20.784 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:18:20.784 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.784 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.784 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.784 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.243100+0000", 2023-09-23T23:18:20.784 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2950000000000004e-06 2023-09-23T23:18:20.784 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.784 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.784 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.784 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.243109+0000", 2023-09-23T23:18:20.784 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1840000000000005e-06 2023-09-23T23:18:20.784 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.785 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.785 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.785 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.243113+0000", 2023-09-23T23:18:20.785 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.755e-06 2023-09-23T23:18:20.785 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.785 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.785 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.785 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.243159+0000", 2023-09-23T23:18:20.785 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5781000000000001e-05 2023-09-23T23:18:20.785 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.785 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.785 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.786 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.243255+0000", 2023-09-23T23:18:20.786 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5902000000000004e-05 2023-09-23T23:18:20.786 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.786 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.786 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:18:20.786 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.243332+0000", 2023-09-23T23:18:20.786 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7113000000000005e-05 2023-09-23T23:18:20.786 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.786 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.786 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:20.786 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.246121+0000", 2023-09-23T23:18:20.786 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0027894880000000001 2023-09-23T23:18:20.787 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.787 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.787 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.787 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.246395+0000", 2023-09-23T23:18:20.787 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00027367800000000002 2023-09-23T23:18:20.787 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.787 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.787 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.787 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.541522+0000", 2023-09-23T23:18:20.787 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.29512748500000002 2023-09-23T23:18:20.787 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.787 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.788 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.788 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.541580+0000", 2023-09-23T23:18:20.788 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7389999999999998e-05 2023-09-23T23:18:20.788 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.788 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.788 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.788 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.541620+0000", 2023-09-23T23:18:20.788 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0410000000000001e-05 2023-09-23T23:18:20.788 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.788 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.788 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.788 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.788 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.788 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:164265 9.e 9:7844be11:::benchmark_data_smithi110_295904_object53084:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T23:18:20.789 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:40.259778+0000", 2023-09-23T23:18:20.789 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 100.407489423, 2023-09-23T23:18:20.789 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.28204643200000001, 2023-09-23T23:18:20.789 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.789 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.789 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:20.789 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:20.789 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:20.789 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 164265 2023-09-23T23:18:20.789 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.789 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.790 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.790 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.790 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.259778+0000", 2023-09-23T23:18:20.790 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.790 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.790 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.790 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.790 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.259778+0000", 2023-09-23T23:18:20.790 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.790 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.790 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.790 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.259777+0000", 2023-09-23T23:18:20.791 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:18:20.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.791 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.259781+0000", 2023-09-23T23:18:20.791 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0849999999999999e-06 2023-09-23T23:18:20.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.791 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.259790+0000", 2023-09-23T23:18:20.791 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7150000000000004e-06 2023-09-23T23:18:20.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.792 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.792 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.792 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.259794+0000", 2023-09-23T23:18:20.792 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8e-06 2023-09-23T23:18:20.792 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.792 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.792 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.792 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.259836+0000", 2023-09-23T23:18:20.792 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1566999999999997e-05 2023-09-23T23:18:20.792 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.792 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.792 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.792 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.259932+0000", 2023-09-23T23:18:20.793 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6347000000000004e-05 2023-09-23T23:18:20.793 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.793 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.793 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:18:20.793 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.260008+0000", 2023-09-23T23:18:20.793 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6008999999999997e-05 2023-09-23T23:18:20.793 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.793 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.793 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:20.793 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.262790+0000", 2023-09-23T23:18:20.793 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0027825039999999999 2023-09-23T23:18:20.793 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.793 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.793 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.794 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.263060+0000", 2023-09-23T23:18:20.794 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00026972500000000001 2023-09-23T23:18:20.794 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.794 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.794 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.794 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.541748+0000", 2023-09-23T23:18:20.794 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.27868807400000001 2023-09-23T23:18:20.794 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.794 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.794 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.794 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.541791+0000", 2023-09-23T23:18:20.794 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2750000000000002e-05 2023-09-23T23:18:20.795 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.795 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.795 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.795 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:40.541825+0000", 2023-09-23T23:18:20.795 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.366e-05 2023-09-23T23:18:20.795 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.795 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.795 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.795 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.795 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.795 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:189890 9.3 9:cf3d406b:::benchmark_data_smithi110_295904_object78709:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T23:18:20.795 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.914682+0000", 2023-09-23T23:18:20.795 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 42.752585306999997, 2023-09-23T23:18:20.796 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.28227372299999998, 2023-09-23T23:18:20.796 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.796 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.796 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:20.796 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:20.796 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:20.796 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 189890 2023-09-23T23:18:20.796 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.796 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.796 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.796 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.796 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914682+0000", 2023-09-23T23:18:20.797 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.797 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.797 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.797 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.797 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914682+0000", 2023-09-23T23:18:20.797 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.797 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.797 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.797 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.797 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914681+0000", 2023-09-23T23:18:20.797 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:18:20.798 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.798 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.798 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.798 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914687+0000", 2023-09-23T23:18:20.798 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4759999999999997e-06 2023-09-23T23:18:20.798 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.798 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.798 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.798 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914696+0000", 2023-09-23T23:18:20.798 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3829999999999998e-06 2023-09-23T23:18:20.798 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.798 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.799 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.799 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914701+0000", 2023-09-23T23:18:20.799 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8829999999999997e-06 2023-09-23T23:18:20.799 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.799 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.799 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.799 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914741+0000", 2023-09-23T23:18:20.799 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9585999999999997e-05 2023-09-23T23:18:20.799 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.799 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.799 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.799 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.186791+0000", 2023-09-23T23:18:20.800 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.27205005599999998 2023-09-23T23:18:20.800 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.800 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.800 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:18:20.800 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.186924+0000", 2023-09-23T23:18:20.800 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000133189 2023-09-23T23:18:20.800 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.800 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.800 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.800 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.196839+0000", 2023-09-23T23:18:20.800 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0099150479999999992 2023-09-23T23:18:20.800 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.801 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.801 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.801 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.196872+0000", 2023-09-23T23:18:20.801 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2616000000000001e-05 2023-09-23T23:18:20.801 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.801 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.801 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:20.801 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.196876+0000", 2023-09-23T23:18:20.801 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9010000000000002e-06 2023-09-23T23:18:20.801 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.801 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.801 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.802 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.196915+0000", 2023-09-23T23:18:20.802 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8535000000000003e-05 2023-09-23T23:18:20.802 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.802 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.802 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.802 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.196956+0000", 2023-09-23T23:18:20.802 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1099000000000001e-05 2023-09-23T23:18:20.802 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.802 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.802 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.802 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.803 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.803 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:189891 9.3 9:c71c9bd8:::benchmark_data_smithi110_295904_object78710:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T23:18:20.803 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.914712+0000", 2023-09-23T23:18:20.803 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 42.752555811000001, 2023-09-23T23:18:20.803 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.28611555500000002, 2023-09-23T23:18:20.803 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.803 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.803 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:20.803 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:20.803 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:20.803 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 189891 2023-09-23T23:18:20.803 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.804 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.804 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.804 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.804 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914712+0000", 2023-09-23T23:18:20.804 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.804 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.804 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.804 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.804 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914712+0000", 2023-09-23T23:18:20.804 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.804 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.804 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.805 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.805 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914711+0000", 2023-09-23T23:18:20.805 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:18:20.805 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.805 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.805 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.805 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914714+0000", 2023-09-23T23:18:20.805 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6410000000000002e-06 2023-09-23T23:18:20.805 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.805 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.805 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.805 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914720+0000", 2023-09-23T23:18:20.806 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2029999999999997e-06 2023-09-23T23:18:20.806 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.806 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.806 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.806 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914724+0000", 2023-09-23T23:18:20.806 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1540000000000004e-06 2023-09-23T23:18:20.806 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.806 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.806 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.806 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.190155+0000", 2023-09-23T23:18:20.806 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.27543076100000002 2023-09-23T23:18:20.806 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.807 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.807 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.807 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.190258+0000", 2023-09-23T23:18:20.807 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103194 2023-09-23T23:18:20.807 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.807 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.807 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:18:20.807 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.190336+0000", 2023-09-23T23:18:20.807 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7447000000000006e-05 2023-09-23T23:18:20.807 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.807 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.807 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:20.807 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.196965+0000", 2023-09-23T23:18:20.807 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066294079999999998 2023-09-23T23:18:20.808 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.808 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.808 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.808 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.197366+0000", 2023-09-23T23:18:20.808 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00040047900000000003 2023-09-23T23:18:20.808 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.808 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.808 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.808 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.200726+0000", 2023-09-23T23:18:20.808 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003360493 2023-09-23T23:18:20.808 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.808 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.808 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.809 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.200779+0000", 2023-09-23T23:18:20.809 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2843999999999998e-05 2023-09-23T23:18:20.809 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.809 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.809 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.809 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.200827+0000", 2023-09-23T23:18:20.809 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8322000000000003e-05 2023-09-23T23:18:20.809 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.809 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.809 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.809 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.810 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.810 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:189895 9.3 9:c8d7d829:::benchmark_data_smithi110_295904_object78714:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T23:18:20.810 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.914749+0000", 2023-09-23T23:18:20.810 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 42.752518522000003, 2023-09-23T23:18:20.810 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.286187623, 2023-09-23T23:18:20.810 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.810 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.810 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:18:20.810 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:18:20.810 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:18:20.810 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 189895 2023-09-23T23:18:20.810 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.811 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.811 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.811 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.811 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914749+0000", 2023-09-23T23:18:20.811 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.811 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.811 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.811 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.811 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914749+0000", 2023-09-23T23:18:20.811 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.811 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.811 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.811 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.812 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914749+0000", 2023-09-23T23:18:20.812 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:18:20.812 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.812 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.812 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.812 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914751+0000", 2023-09-23T23:18:20.812 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7779999999999999e-06 2023-09-23T23:18:20.812 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.812 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.812 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.812 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914758+0000", 2023-09-23T23:18:20.812 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6780000000000003e-06 2023-09-23T23:18:20.812 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.813 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.813 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.813 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.914762+0000", 2023-09-23T23:18:20.813 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7639999999999999e-06 2023-09-23T23:18:20.813 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.813 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.813 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.813 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.193394+0000", 2023-09-23T23:18:20.813 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.278632357 2023-09-23T23:18:20.813 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.813 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.813 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.814 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.193529+0000", 2023-09-23T23:18:20.814 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013442700000000001 2023-09-23T23:18:20.814 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.814 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.814 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:18:20.814 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.193661+0000", 2023-09-23T23:18:20.814 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013190399999999999 2023-09-23T23:18:20.814 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.814 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.814 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:18:20.814 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.196968+0000", 2023-09-23T23:18:20.814 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0033078600000000001 2023-09-23T23:18:20.815 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.815 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.815 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.815 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.197620+0000", 2023-09-23T23:18:20.815 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00065124200000000001 2023-09-23T23:18:20.815 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.815 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.815 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:18:20.815 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.200865+0000", 2023-09-23T23:18:20.815 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032455449999999999 2023-09-23T23:18:20.815 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.815 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.816 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.816 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.200898+0000", 2023-09-23T23:18:20.816 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2719999999999998e-05 2023-09-23T23:18:20.816 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.816 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.816 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.816 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.200937+0000", 2023-09-23T23:18:20.816 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8767999999999999e-05 2023-09-23T23:18:20.816 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.816 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.816 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.816 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.817 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.817 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45164.0:189892 9.a e230/228 9:53ab6c37:::benchmark_data_smithi110_295904_object78711:head v 230'11673, mlcod=230'11671)", 2023-09-23T23:18:20.817 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.915244+0000", 2023-09-23T23:18:20.817 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 42.752024112000001, 2023-09-23T23:18:20.817 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.275383039, 2023-09-23T23:18:20.817 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.817 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.817 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.817 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.817 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.817 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.915244+0000", 2023-09-23T23:18:20.817 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.817 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.817 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.818 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.818 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.915244+0000", 2023-09-23T23:18:20.818 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.818 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.818 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.818 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.818 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.915243+0000", 2023-09-23T23:18:20.818 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:18:20.818 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.818 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.818 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.818 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.915248+0000", 2023-09-23T23:18:20.819 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9699999999999998e-06 2023-09-23T23:18:20.819 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.819 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.819 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.819 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.186764+0000", 2023-09-23T23:18:20.819 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.27151633800000002 2023-09-23T23:18:20.819 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.819 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.819 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.819 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.186773+0000", 2023-09-23T23:18:20.819 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6000000000000007e-06 2023-09-23T23:18:20.819 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.820 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.820 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.820 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.186823+0000", 2023-09-23T23:18:20.820 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0578000000000003e-05 2023-09-23T23:18:20.820 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.820 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.820 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.820 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.186852+0000", 2023-09-23T23:18:20.820 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8765999999999999e-05 2023-09-23T23:18:20.820 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.820 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.820 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.821 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.190599+0000", 2023-09-23T23:18:20.821 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037467329999999999 2023-09-23T23:18:20.821 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.821 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.821 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.821 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.190627+0000", 2023-09-23T23:18:20.821 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7619000000000001e-05 2023-09-23T23:18:20.821 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.821 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.821 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.821 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.821 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.821 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45164.0:189889 9.c e230/164 9:389b7bed:::benchmark_data_smithi110_295904_object78708:head v 230'11869, mlcod=230'11865)", 2023-09-23T23:18:20.822 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.922590+0000", 2023-09-23T23:18:20.822 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 42.744677265, 2023-09-23T23:18:20.822 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26780416800000001, 2023-09-23T23:18:20.822 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.822 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.822 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.822 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.822 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.822 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.922590+0000", 2023-09-23T23:18:20.822 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.822 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.822 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.822 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.823 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.922590+0000", 2023-09-23T23:18:20.823 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.823 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.823 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.823 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.922589+0000", 2023-09-23T23:18:20.823 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:18:20.823 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.823 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.823 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.922597+0000", 2023-09-23T23:18:20.824 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2509999999999997e-06 2023-09-23T23:18:20.824 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.824 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.824 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.824 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.186760+0000", 2023-09-23T23:18:20.824 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26416325299999999 2023-09-23T23:18:20.824 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.824 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.824 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.824 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.186773+0000", 2023-09-23T23:18:20.824 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2585999999999999e-05 2023-09-23T23:18:20.824 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.825 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.825 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.825 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.186816+0000", 2023-09-23T23:18:20.825 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3902000000000001e-05 2023-09-23T23:18:20.825 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.825 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.825 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.825 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.186840+0000", 2023-09-23T23:18:20.825 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3632999999999999e-05 2023-09-23T23:18:20.825 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.825 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.825 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.826 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.190368+0000", 2023-09-23T23:18:20.826 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0035274690000000001 2023-09-23T23:18:20.826 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.826 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.826 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.826 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.190395+0000", 2023-09-23T23:18:20.826 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6950000000000001e-05 2023-09-23T23:18:20.826 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.826 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.826 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.826 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.826 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.827 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45164.0:199582 9.2 e243/236 9:40c2fbd8:::benchmark_data_smithi110_295904_object88401:head v 243'12643, mlcod=242'12641)", 2023-09-23T23:18:20.827 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:18:00.451618+0000", 2023-09-23T23:18:20.827 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 20.215649386999999, 2023-09-23T23:18:20.827 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24358705999999999, 2023-09-23T23:18:20.827 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:18:20.827 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:18:20.827 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:18:20.827 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.827 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:18:20.827 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:00.451618+0000", 2023-09-23T23:18:20.827 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.827 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.827 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.828 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:18:20.828 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:00.451618+0000", 2023-09-23T23:18:20.828 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:18:20.828 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.828 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.828 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:18:20.828 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:00.451617+0000", 2023-09-23T23:18:20.828 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:18:20.828 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.828 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.828 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:18:20.828 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:00.451625+0000", 2023-09-23T23:18:20.828 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0369999999999995e-06 2023-09-23T23:18:20.828 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.829 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.829 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:18:20.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:00.451635+0000", 2023-09-23T23:18:20.829 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7189999999999996e-06 2023-09-23T23:18:20.829 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.829 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.829 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:18:20.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:00.451640+0000", 2023-09-23T23:18:20.829 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9049999999999996e-06 2023-09-23T23:18:20.829 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.829 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.829 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:18:20.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:00.451673+0000", 2023-09-23T23:18:20.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3269000000000003e-05 2023-09-23T23:18:20.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.830 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.830 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:18:20.830 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:00.451690+0000", 2023-09-23T23:18:20.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7007000000000002e-05 2023-09-23T23:18:20.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.830 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.830 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:18:20.830 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:00.695166+0000", 2023-09-23T23:18:20.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24347601899999999 2023-09-23T23:18:20.831 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:18:20.831 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:18:20.831 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:18:20.831 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:00.695205+0000", 2023-09-23T23:18:20.831 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8964000000000002e-05 2023-09-23T23:18:20.831 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.831 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.831 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.831 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:18:20.831 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:18:20.831 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:18:21.074 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:21 smithi138 ceph-mon[193606]: pgmap v761: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 797 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 4733/38322 objects misplaced (12.351%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T23:18:21.074 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:21 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:18:21.075 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:21 smithi138 ceph-mon[193606]: failed to encode map e248 with expected crc 2023-09-23T23:18:21.075 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:21 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/883009529' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:18:21.075 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:21 smithi138 ceph-mon[193606]: osdmap e248: 8 total, 8 up, 7 in 2023-09-23T23:18:21.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:21 smithi110 ceph-mon[212939]: pgmap v761: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 797 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 4733/38322 objects misplaced (12.351%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T23:18:21.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:21 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:18:21.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:21 smithi110 ceph-mon[212939]: failed to encode map e248 with expected crc 2023-09-23T23:18:21.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:21 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/883009529' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:18:21.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:21 smithi110 ceph-mon[212939]: osdmap e248: 8 total, 8 up, 7 in 2023-09-23T23:18:21.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:21 smithi110 ceph-mon[221102]: pgmap v761: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 797 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 4733/38322 objects misplaced (12.351%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T23:18:21.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:21 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:18:21.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:21 smithi110 ceph-mon[221102]: failed to encode map e248 with expected crc 2023-09-23T23:18:21.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:21 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/883009529' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:18:21.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:21 smithi110 ceph-mon[221102]: osdmap e248: 8 total, 8 up, 7 in 2023-09-23T23:18:21.799 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:18:22.382 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:22.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:22.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:22.489 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:22.556 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:22.665 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:22.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:22.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:22.771 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:22.824 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:22.982 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:22.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:22.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:23.085 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:23 smithi110 ceph-mon[212939]: pgmap v763: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 742 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 3723/35646 objects misplaced (10.444%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T23:18:23.089 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:23.158 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:23.247 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:23.249 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:23.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:23.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:23.355 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:23.382 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:18:23.419 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:18:23.431 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:23 smithi110 ceph-mon[221102]: pgmap v763: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 742 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 3723/35646 objects misplaced (10.444%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T23:18:23.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:23 smithi138 ceph-mon[193606]: pgmap v763: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 742 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 3723/35646 objects misplaced (10.444%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T23:18:23.441 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:23.455 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:18:23.498 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:18:23.539 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:18:23.577 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:18:23.622 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:23.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:23.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:23.627 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:18:23.664 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:18:23.689 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:18:23.689 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:18:23.690 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:18:23.690 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:18:23.690 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:18:23.690 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:18:23.690 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:18:23.690 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:18:23.728 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:23.808 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:24.036 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7] out_osds: [2] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T23:18:24.036 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:18:24.037 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-09-23T23:18:24.037 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:18:24.348 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:24 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1818610321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:24.348 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:24 smithi110 ceph-mon[212939]: pgmap v764: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 713 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 3723/34257 objects misplaced (10.868%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T23:18:24.348 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:24 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1818610321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:24.349 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:24 smithi110 ceph-mon[221102]: pgmap v764: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 713 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 3723/34257 objects misplaced (10.868%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T23:18:24.535 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:24.544 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:18:24.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:24.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:24.649 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:18:24.684 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:24 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1818610321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:24.684 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:24 smithi138 ceph-mon[193606]: pgmap v764: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 713 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 3723/34257 objects misplaced (10.868%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T23:18:24.689 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:24.741 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:18:24.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:24.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:18:24.848 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:24.874 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:25.499 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:25.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:25.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:25.609 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:18:25.660 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:25.701 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:18:25.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:25.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:18:25.807 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:18:25.882 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:26.328 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:18:26.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:26.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:18:26.435 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:26.443 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:26.451 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:26.497 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:26.508 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:26 smithi110 ceph-mon[212939]: pgmap v765: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 569 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 3037/27342 objects misplaced (11.107%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:18:26.565 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:26.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:26.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:26.679 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:26.734 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:26.802 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:26 smithi110 ceph-mon[221102]: pgmap v765: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 569 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 3037/27342 objects misplaced (11.107%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:18:26.819 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:26.826 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:26 smithi138 ceph-mon[193606]: pgmap v765: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 569 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 3037/27342 objects misplaced (11.107%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:18:26.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:26.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:26.927 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:18:27.004 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:27.053 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:18:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:18:26] "GET /metrics HTTP/1.1" 200 36543 "" "Prometheus/2.43.0" 2023-09-23T23:18:27.181 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:18:27.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:27.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:18:27.292 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:18:27.302 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:18:27.302 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:18:27.354 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:27.442 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:18:27.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:27.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:18:27.475 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:18:27.556 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:18:27.662 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:29.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:28 smithi110 ceph-mon[212939]: pgmap v766: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 514 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 695 op/s; 2443/24720 objects misplaced (9.883%); 2.3 MiB/s, 50 objects/s recovering 2023-09-23T23:18:29.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:29.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:18:29.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:28 smithi110 ceph-mon[221102]: pgmap v766: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 514 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 695 op/s; 2443/24720 objects misplaced (9.883%); 2.3 MiB/s, 50 objects/s recovering 2023-09-23T23:18:29.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:29.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:18:29.347 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:18:29.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:29.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:18:29.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:28 smithi138 ceph-mon[193606]: pgmap v766: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 514 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 695 op/s; 2443/24720 objects misplaced (9.883%); 2.3 MiB/s, 50 objects/s recovering 2023-09-23T23:18:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:18:29.453 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:18:29.534 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:30.413 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:18:30.455 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:18:30.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:30.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:18:30.567 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:18:30.607 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:30.703 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:30.716 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:18:30.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:30.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:18:30.822 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:30.910 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:31.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:30 smithi110 ceph-mon[212939]: pgmap v767: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 345 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 723 op/s; 1852/16590 objects misplaced (11.163%); 2.3 MiB/s, 50 objects/s recovering 2023-09-23T23:18:31.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:30 smithi110 ceph-mon[221102]: pgmap v767: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 345 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 723 op/s; 1852/16590 objects misplaced (11.163%); 2.3 MiB/s, 50 objects/s recovering 2023-09-23T23:18:31.286 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:31.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:31.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:31.396 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:18:31.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:30 smithi138 ceph-mon[193606]: pgmap v767: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 345 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 723 op/s; 1852/16590 objects misplaced (11.163%); 2.3 MiB/s, 50 objects/s recovering 2023-09-23T23:18:31.453 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:31.530 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:18:31.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:31.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:18:31.639 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:18:31.680 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:31.716 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:18:31.736 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:18:31.738 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:18:31.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:31.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:18:31.844 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:31.905 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:31.952 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:31.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:31.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:32.005 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:18:32.061 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:18:32.120 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:32.204 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:18:32.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:32.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:18:32.326 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:18:32.384 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:31 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3677325311' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:18:32.391 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:32.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:31 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3677325311' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:18:32.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:31 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3677325311' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:18:32.494 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:18:32.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:32.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:18:32.613 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:18:32.689 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:32 smithi138 ceph-mon[193606]: pgmap v768: 99 pgs: 1 active+recovery_wait+remapped, 94 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 286 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 1384/13773 objects misplaced (10.049%); 2.0 MiB/s, 66 objects/s recovering 2023-09-23T23:18:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:32 smithi138 ceph-mon[193606]: from='client.45434 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:18:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:32 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:18:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:32 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:18:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:32 smithi138 ceph-mon[193606]: failed to encode map e249 with expected crc 2023-09-23T23:18:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:32 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3677325311' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:18:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:32 smithi138 ceph-mon[193606]: osdmap e249: 8 total, 8 up, 7 in 2023-09-23T23:18:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:32 smithi138 ceph-mon[193606]: failed to encode map e249 with expected crc 2023-09-23T23:18:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:32 smithi138 ceph-mon[193606]: failed to encode map e249 with expected crc 2023-09-23T23:18:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:32 smithi138 ceph-mon[193606]: failed to encode map e249 with expected crc 2023-09-23T23:18:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:32 smithi138 ceph-mon[193606]: failed to encode map e249 with expected crc 2023-09-23T23:18:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:32 smithi138 ceph-mon[193606]: failed to encode map e249 with expected crc 2023-09-23T23:18:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[221102]: pgmap v768: 99 pgs: 1 active+recovery_wait+remapped, 94 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 286 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 1384/13773 objects misplaced (10.049%); 2.0 MiB/s, 66 objects/s recovering 2023-09-23T23:18:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[221102]: from='client.45434 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:18:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:18:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:18:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[221102]: failed to encode map e249 with expected crc 2023-09-23T23:18:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3677325311' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:18:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[221102]: osdmap e249: 8 total, 8 up, 7 in 2023-09-23T23:18:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[221102]: failed to encode map e249 with expected crc 2023-09-23T23:18:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[221102]: failed to encode map e249 with expected crc 2023-09-23T23:18:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[221102]: failed to encode map e249 with expected crc 2023-09-23T23:18:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[221102]: failed to encode map e249 with expected crc 2023-09-23T23:18:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[221102]: failed to encode map e249 with expected crc 2023-09-23T23:18:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[212939]: pgmap v768: 99 pgs: 1 active+recovery_wait+remapped, 94 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 286 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 1384/13773 objects misplaced (10.049%); 2.0 MiB/s, 66 objects/s recovering 2023-09-23T23:18:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[212939]: from='client.45434 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:18:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:18:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:18:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[212939]: failed to encode map e249 with expected crc 2023-09-23T23:18:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3677325311' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:18:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[212939]: osdmap e249: 8 total, 8 up, 7 in 2023-09-23T23:18:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[212939]: failed to encode map e249 with expected crc 2023-09-23T23:18:33.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[212939]: failed to encode map e249 with expected crc 2023-09-23T23:18:33.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[212939]: failed to encode map e249 with expected crc 2023-09-23T23:18:33.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[212939]: failed to encode map e249 with expected crc 2023-09-23T23:18:33.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:32 smithi110 ceph-mon[212939]: failed to encode map e249 with expected crc 2023-09-23T23:18:34.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:34 smithi110 ceph-mon[221102]: pgmap v770: 99 pgs: 1 active+recovery_wait+remapped, 94 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 255 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 1384/12303 objects misplaced (11.249%); 512 KiB/s, 47 objects/s recovering 2023-09-23T23:18:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:34 smithi110 ceph-mon[212939]: pgmap v770: 99 pgs: 1 active+recovery_wait+remapped, 94 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 255 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 1384/12303 objects misplaced (11.249%); 512 KiB/s, 47 objects/s recovering 2023-09-23T23:18:34.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:34 smithi138 ceph-mon[193606]: pgmap v770: 99 pgs: 1 active+recovery_wait+remapped, 94 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 255 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 1384/12303 objects misplaced (11.249%); 512 KiB/s, 47 objects/s recovering 2023-09-23T23:18:34.748 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:18:34.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:34.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:18:34.855 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:34.914 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:34.936 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:35.004 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:35.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:35.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:35.098 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":768,"stamp":"2023-09-23T23:18:31.451404+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":299731593,"num_objects":4591,"num_object_clones":0,"num_object_copies":13773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1384,"num_objects_unfound":0,"num_objects_dirty":4591,"num_whiteouts":0,"num_read":134,"num_read_kb":249,"num_write":217917,"num_write_kb":7116080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34465,"num_bytes_recovered":2054513702,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117105,"ondisk_log_size":117105,"up":297,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":311,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5064612,"kb_used_data":2210284,"kb_used_omap":12,"kb_used_meta":2853555,"kb_avail":651151452,"statfs":{"total":671965249536,"available":666779086848,"internally_reserved":0,"allocated":2263330816,"data_stored":2257363169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12739,"internal_metadata":2922040893},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536281088,"num_objects":-8183,"num_object_clones":0,"num_object_copies":-24549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-3349,"num_objects_unfound":0,"num_objects_dirty":-8183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8183,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":758,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.387068"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":254,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.024602+0000","last_change":"2023-09-23T23:17:48.320558+0000","last_active":"2023-09-23T23:18:17.024602+0000","last_peered":"2023-09-23T23:18:17.024602+0000","last_clean":"2023-09-23T23:18:17.024602+0000","last_became_active":"2023-09-23T23:17:48.320205+0000","last_became_peered":"2023-09-23T23:17:48.320205+0000","last_unstale":"2023-09-23T23:18:17.024602+0000","last_undegraded":"2023-09-23T23:18:17.024602+0000","last_fullsized":"2023-09-23T23:18:17.024602+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:24.708401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":188,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T23:18:20.078771+0000","last_change":"2023-09-23T23:10:59.514211+0000","last_active":"2023-09-23T23:18:20.078771+0000","last_peered":"2023-09-23T23:18:20.078771+0000","last_clean":"2023-09-23T23:18:20.078771+0000","last_became_active":"2023-09-23T23:10:59.513917+0000","last_became_peered":"2023-09-23T23:10:59.513917+0000","last_unstale":"2023-09-23T23:18:20.078771+0000","last_undegraded":"2023-09-23T23:18:20.078771+0000","last_fullsized":"2023-09-23T23:18:20.078771+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:55.533586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":218,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.005512+0000","last_change":"2023-09-23T23:10:59.505490+0000","last_active":"2023-09-23T23:18:17.005512+0000","last_peered":"2023-09-23T23:18:17.005512+0000","last_clean":"2023-09-23T23:18:17.005512+0000","last_became_active":"2023-09-23T23:10:59.504874+0000","last_became_peered":"2023-09-23T23:10:59.504874+0000","last_unstale":"2023-09-23T23:18:17.005512+0000","last_undegraded":"2023-09-23T23:18:17.005512+0000","last_fullsized":"2023-09-23T23:18:17.005512+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:15.279668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":218,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.015775+0000","last_change":"2023-09-23T23:17:33.437874+0000","last_active":"2023-09-23T23:18:17.015775+0000","last_peered":"2023-09-23T23:18:17.015775+0000","last_clean":"2023-09-23T23:18:17.015775+0000","last_became_active":"2023-09-23T23:17:33.437272+0000","last_became_peered":"2023-09-23T23:17:33.437272+0000","last_unstale":"2023-09-23T23:18:17.015775+0000","last_undegraded":"2023-09-23T23:18:17.015775+0000","last_fullsized":"2023-09-23T23:18:17.015775+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:34.600738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":224,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.016368+0000","last_change":"2023-09-23T23:17:48.305487+0000","last_active":"2023-09-23T23:18:17.016368+0000","last_peered":"2023-09-23T23:18:17.016368+0000","last_clean":"2023-09-23T23:18:17.016368+0000","last_became_active":"2023-09-23T23:17:48.305118+0000","last_became_peered":"2023-09-23T23:17:48.305118+0000","last_unstale":"2023-09-23T23:18:17.016368+0000","last_undegraded":"2023-09-23T23:18:17.016368+0000","last_fullsized":"2023-09-23T23:18:17.016368+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:07.958152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":208,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.013787+0000","last_change":"2023-09-23T23:07:30.078179+0000","last_active":"2023-09-23T23:18:17.013787+0000","last_peered":"2023-09-23T23:18:17.013787+0000","last_clean":"2023-09-23T23:18:17.013787+0000","last_became_active":"2023-09-23T23:07:30.077810+0000","last_became_peered":"2023-09-23T23:07:30.077810+0000","last_unstale":"2023-09-23T23:18:17.013787+0000","last_undegraded":"2023-09-23T23:18:17.013787+0000","last_fullsized":"2023-09-23T23:18:17.013787+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:19.218333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":198,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.011053+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:18:17.011053+0000","last_peered":"2023-09-23T23:18:17.011053+0000","last_clean":"2023-09-23T23:18:17.011053+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:18:17.011053+0000","last_undegraded":"2023-09-23T23:18:17.011053+0000","last_fullsized":"2023-09-23T23:18:17.011053+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":188,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.016624+0000","last_change":"2023-09-23T23:10:59.515840+0000","last_active":"2023-09-23T23:18:17.016624+0000","last_peered":"2023-09-23T23:18:17.016624+0000","last_clean":"2023-09-23T23:18:17.016624+0000","last_became_active":"2023-09-23T23:10:59.515000+0000","last_became_peered":"2023-09-23T23:10:59.515000+0000","last_unstale":"2023-09-23T23:18:17.016624+0000","last_undegraded":"2023-09-23T23:18:17.016624+0000","last_fullsized":"2023-09-23T23:18:17.016624+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:47.355624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":25,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.010637+0000","last_change":"2023-09-23T23:17:48.308746+0000","last_active":"2023-09-23T23:18:17.010637+0000","last_peered":"2023-09-23T23:18:17.010637+0000","last_clean":"2023-09-23T23:18:17.010637+0000","last_became_active":"2023-09-23T23:17:48.307974+0000","last_became_peered":"2023-09-23T23:17:48.307974+0000","last_unstale":"2023-09-23T23:18:17.010637+0000","last_undegraded":"2023-09-23T23:18:17.010637+0000","last_fullsized":"2023-09-23T23:18:17.010637+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:52.915506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":176,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.016779+0000","last_change":"2023-09-23T23:16:56.528057+0000","last_active":"2023-09-23T23:18:17.016779+0000","last_peered":"2023-09-23T23:18:17.016779+0000","last_clean":"2023-09-23T23:18:17.016779+0000","last_became_active":"2023-09-23T23:16:56.527755+0000","last_became_peered":"2023-09-23T23:16:56.527755+0000","last_unstale":"2023-09-23T23:18:17.016779+0000","last_undegraded":"2023-09-23T23:18:17.016779+0000","last_fullsized":"2023-09-23T23:18:17.016779+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:47.396406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":64,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.022975+0000","last_change":"2023-09-23T23:16:56.567128+0000","last_active":"2023-09-23T23:18:17.022975+0000","last_peered":"2023-09-23T23:18:17.022975+0000","last_clean":"2023-09-23T23:18:17.022975+0000","last_became_active":"2023-09-23T23:16:56.551484+0000","last_became_peered":"2023-09-23T23:16:56.551484+0000","last_unstale":"2023-09-23T23:18:17.022975+0000","last_undegraded":"2023-09-23T23:18:17.022975+0000","last_fullsized":"2023-09-23T23:18:17.022975+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:14.981307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":188,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.007222+0000","last_change":"2023-09-23T23:17:48.321527+0000","last_active":"2023-09-23T23:18:17.007222+0000","last_peered":"2023-09-23T23:18:17.007222+0000","last_clean":"2023-09-23T23:18:17.007222+0000","last_became_active":"2023-09-23T23:17:48.321198+0000","last_became_peered":"2023-09-23T23:17:48.321198+0000","last_unstale":"2023-09-23T23:18:17.007222+0000","last_undegraded":"2023-09-23T23:18:17.007222+0000","last_fullsized":"2023-09-23T23:18:17.007222+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:11.289469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":214,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.021805+0000","last_change":"2023-09-23T22:53:16.891930+0000","last_active":"2023-09-23T23:18:17.021805+0000","last_peered":"2023-09-23T23:18:17.021805+0000","last_clean":"2023-09-23T23:18:17.021805+0000","last_became_active":"2023-09-23T22:53:16.891658+0000","last_became_peered":"2023-09-23T22:53:16.891658+0000","last_unstale":"2023-09-23T23:18:17.021805+0000","last_undegraded":"2023-09-23T23:18:17.021805+0000","last_fullsized":"2023-09-23T23:18:17.021805+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:19.762775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":208,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.021233+0000","last_change":"2023-09-23T23:07:30.082476+0000","last_active":"2023-09-23T23:18:17.021233+0000","last_peered":"2023-09-23T23:18:17.021233+0000","last_clean":"2023-09-23T23:18:17.021233+0000","last_became_active":"2023-09-23T23:07:30.082201+0000","last_became_peered":"2023-09-23T23:07:30.082201+0000","last_unstale":"2023-09-23T23:18:17.021233+0000","last_undegraded":"2023-09-23T23:18:17.021233+0000","last_fullsized":"2023-09-23T23:18:17.021233+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:08.809163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":244,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.018106+0000","last_change":"2023-09-23T23:17:48.325540+0000","last_active":"2023-09-23T23:18:17.018106+0000","last_peered":"2023-09-23T23:18:17.018106+0000","last_clean":"2023-09-23T23:18:17.018106+0000","last_became_active":"2023-09-23T23:17:48.301566+0000","last_became_peered":"2023-09-23T23:17:48.301566+0000","last_unstale":"2023-09-23T23:18:17.018106+0000","last_undegraded":"2023-09-23T23:18:17.018106+0000","last_fullsized":"2023-09-23T23:18:17.018106+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:14.833073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":206,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.021084+0000","last_change":"2023-09-23T23:17:48.304131+0000","last_active":"2023-09-23T23:18:17.021084+0000","last_peered":"2023-09-23T23:18:17.021084+0000","last_clean":"2023-09-23T23:18:17.021084+0000","last_became_active":"2023-09-23T23:17:48.303720+0000","last_became_peered":"2023-09-23T23:17:48.303720+0000","last_unstale":"2023-09-23T23:18:17.021084+0000","last_undegraded":"2023-09-23T23:18:17.021084+0000","last_fullsized":"2023-09-23T23:18:17.021084+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:04.726467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":217,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.012074+0000","last_change":"2023-09-23T23:17:48.303810+0000","last_active":"2023-09-23T23:18:17.012074+0000","last_peered":"2023-09-23T23:18:17.012074+0000","last_clean":"2023-09-23T23:18:17.012074+0000","last_became_active":"2023-09-23T23:17:48.303505+0000","last_became_peered":"2023-09-23T23:17:48.303505+0000","last_unstale":"2023-09-23T23:18:17.012074+0000","last_undegraded":"2023-09-23T23:18:17.012074+0000","last_fullsized":"2023-09-23T23:18:17.012074+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:31.494085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":195,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.018876+0000","last_change":"2023-09-23T23:13:29.360403+0000","last_active":"2023-09-23T23:18:17.018876+0000","last_peered":"2023-09-23T23:18:17.018876+0000","last_clean":"2023-09-23T23:18:17.018876+0000","last_became_active":"2023-09-23T23:13:29.359519+0000","last_became_peered":"2023-09-23T23:13:29.359519+0000","last_unstale":"2023-09-23T23:18:17.018876+0000","last_undegraded":"2023-09-23T23:18:17.018876+0000","last_fullsized":"2023-09-23T23:18:17.018876+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:27.911252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":192,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.018372+0000","last_change":"2023-09-23T22:53:16.889362+0000","last_active":"2023-09-23T23:18:17.018372+0000","last_peered":"2023-09-23T23:18:17.018372+0000","last_clean":"2023-09-23T23:18:17.018372+0000","last_became_active":"2023-09-23T22:53:16.889081+0000","last_became_peered":"2023-09-23T22:53:16.889081+0000","last_unstale":"2023-09-23T23:18:17.018372+0000","last_undegraded":"2023-09-23T23:18:17.018372+0000","last_fullsized":"2023-09-23T23:18:17.018372+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:11.321151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":216,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.015925+0000","last_change":"2023-09-23T23:17:48.308900+0000","last_active":"2023-09-23T23:18:17.015925+0000","last_peered":"2023-09-23T23:18:17.015925+0000","last_clean":"2023-09-23T23:18:17.015925+0000","last_became_active":"2023-09-23T23:17:48.308475+0000","last_became_peered":"2023-09-23T23:17:48.308475+0000","last_unstale":"2023-09-23T23:18:17.015925+0000","last_undegraded":"2023-09-23T23:18:17.015925+0000","last_fullsized":"2023-09-23T23:18:17.015925+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:38.158435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":237,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.006377+0000","last_change":"2023-09-23T23:13:29.368175+0000","last_active":"2023-09-23T23:18:17.006377+0000","last_peered":"2023-09-23T23:18:17.006377+0000","last_clean":"2023-09-23T23:18:17.006377+0000","last_became_active":"2023-09-23T23:13:29.363603+0000","last_became_peered":"2023-09-23T23:13:29.363603+0000","last_unstale":"2023-09-23T23:18:17.006377+0000","last_undegraded":"2023-09-23T23:18:17.006377+0000","last_fullsized":"2023-09-23T23:18:17.006377+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:25.277871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":133,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.005149+0000","last_change":"2023-09-23T23:17:48.306755+0000","last_active":"2023-09-23T23:18:17.005149+0000","last_peered":"2023-09-23T23:18:17.005149+0000","last_clean":"2023-09-23T23:18:17.005149+0000","last_became_active":"2023-09-23T23:17:48.306455+0000","last_became_peered":"2023-09-23T23:17:48.306455+0000","last_unstale":"2023-09-23T23:18:17.005149+0000","last_undegraded":"2023-09-23T23:18:17.005149+0000","last_fullsized":"2023-09-23T23:18:17.005149+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:05.630787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.5","version":"248'13655","reported_seq":32659,"reported_epoch":248,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:18:31.389137+0000","last_change":"2023-09-23T23:18:31.389137+0000","last_active":"2023-09-23T23:18:31.389137+0000","last_peered":"2023-09-23T23:18:31.389137+0000","last_clean":"2023-09-23T23:15:27.538246+0000","last_became_active":"2023-09-23T23:15:29.635703+0000","last_became_peered":"2023-09-23T23:15:29.635703+0000","last_unstale":"2023-09-23T23:18:31.389137+0000","last_undegraded":"2023-09-23T23:18:31.388907+0000","last_fullsized":"2023-09-23T23:15:29.226857+0000","mapping_epoch":187,"log_start":"184'7263","ondisk_log_start":"184'7263","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":6392,"log_dups_size":0,"ondisk_log_size":6392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:32:33.086246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13565952,"num_objects":207,"num_object_clones":0,"num_object_copies":621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13655,"num_write_kb":443584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6297,"num_bytes_recovered":386662400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,1","objects":207}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":63,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.025160+0000","last_change":"2023-09-23T23:16:56.548535+0000","last_active":"2023-09-23T23:18:17.025160+0000","last_peered":"2023-09-23T23:18:17.025160+0000","last_clean":"2023-09-23T23:18:17.025160+0000","last_became_active":"2023-09-23T23:16:56.546115+0000","last_became_peered":"2023-09-23T23:16:56.546115+0000","last_unstale":"2023-09-23T23:18:17.025160+0000","last_undegraded":"2023-09-23T23:18:17.025160+0000","last_fullsized":"2023-09-23T23:18:17.025160+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:02.279715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":53,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.006453+0000","last_change":"2023-09-23T23:16:56.525479+0000","last_active":"2023-09-23T23:18:17.006453+0000","last_peered":"2023-09-23T23:18:17.006453+0000","last_clean":"2023-09-23T23:18:17.006453+0000","last_became_active":"2023-09-23T23:16:56.525147+0000","last_became_peered":"2023-09-23T23:16:56.525147+0000","last_unstale":"2023-09-23T23:18:17.006453+0000","last_undegraded":"2023-09-23T23:18:17.006453+0000","last_fullsized":"2023-09-23T23:18:17.006453+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:06.165300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"248'14000","reported_seq":14095,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T23:18:29.040420+0000","last_change":"2023-09-23T23:13:28.360159+0000","last_active":"2023-09-23T23:18:29.040420+0000","last_peered":"2023-09-23T23:18:29.040420+0000","last_clean":"2023-09-23T23:18:29.040420+0000","last_became_active":"2023-09-23T23:13:28.359915+0000","last_became_peered":"2023-09-23T23:13:28.359915+0000","last_unstale":"2023-09-23T23:18:29.040420+0000","last_undegraded":"2023-09-23T23:18:29.040420+0000","last_fullsized":"2023-09-23T23:18:29.040420+0000","mapping_epoch":164,"log_start":"183'6775","ondisk_log_start":"183'6775","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":7225,"log_dups_size":0,"ondisk_log_size":7225,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:48.297395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20316160,"num_objects":310,"num_object_clones":0,"num_object_copies":930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14000,"num_write_kb":457920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":63,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.022201+0000","last_change":"2023-09-23T23:16:56.532271+0000","last_active":"2023-09-23T23:18:17.022201+0000","last_peered":"2023-09-23T23:18:17.022201+0000","last_clean":"2023-09-23T23:18:17.022201+0000","last_became_active":"2023-09-23T23:16:56.527347+0000","last_became_peered":"2023-09-23T23:16:56.527347+0000","last_unstale":"2023-09-23T23:18:17.022201+0000","last_undegraded":"2023-09-23T23:18:17.022201+0000","last_fullsized":"2023-09-23T23:18:17.022201+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:30:26.607017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":235,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.021542+0000","last_change":"2023-09-23T23:16:56.527215+0000","last_active":"2023-09-23T23:18:17.021542+0000","last_peered":"2023-09-23T23:18:17.021542+0000","last_clean":"2023-09-23T23:18:17.021542+0000","last_became_active":"2023-09-23T23:16:56.526551+0000","last_became_peered":"2023-09-23T23:16:56.526551+0000","last_unstale":"2023-09-23T23:18:17.021542+0000","last_undegraded":"2023-09-23T23:18:17.021542+0000","last_fullsized":"2023-09-23T23:18:17.021542+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:56:36.991437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"248'13480","reported_seq":13604,"reported_epoch":248,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:18:28.963791+0000","last_change":"2023-09-23T23:17:48.493805+0000","last_active":"2023-09-23T23:18:28.963791+0000","last_peered":"2023-09-23T23:18:28.963791+0000","last_clean":"2023-09-23T23:16:43.190306+0000","last_became_active":"2023-09-23T23:17:48.445289+0000","last_became_peered":"2023-09-23T23:17:48.445289+0000","last_unstale":"2023-09-23T23:18:28.963791+0000","last_undegraded":"2023-09-23T23:18:28.963791+0000","last_fullsized":"2023-09-23T23:18:28.963791+0000","mapping_epoch":235,"log_start":"173'4875","ondisk_log_start":"173'4875","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":8605,"log_dups_size":0,"ondisk_log_size":8605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:24.220993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21626880,"num_objects":330,"num_object_clones":0,"num_object_copies":990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":660,"num_objects_unfound":0,"num_objects_dirty":330,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13480,"num_write_kb":441920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,3,7],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":330}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":207,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.018680+0000","last_change":"2023-09-23T23:13:29.362820+0000","last_active":"2023-09-23T23:18:17.018680+0000","last_peered":"2023-09-23T23:18:17.018680+0000","last_clean":"2023-09-23T23:18:17.018680+0000","last_became_active":"2023-09-23T23:13:29.360151+0000","last_became_peered":"2023-09-23T23:13:29.360151+0000","last_unstale":"2023-09-23T23:18:17.018680+0000","last_undegraded":"2023-09-23T23:18:17.018680+0000","last_fullsized":"2023-09-23T23:18:17.018680+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:36.242850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":255,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.008282+0000","last_change":"2023-09-23T23:17:48.317798+0000","last_active":"2023-09-23T23:18:17.008282+0000","last_peered":"2023-09-23T23:18:17.008282+0000","last_clean":"2023-09-23T23:18:17.008282+0000","last_became_active":"2023-09-23T23:17:48.303959+0000","last_became_peered":"2023-09-23T23:17:48.303959+0000","last_unstale":"2023-09-23T23:18:17.008282+0000","last_undegraded":"2023-09-23T23:18:17.008282+0000","last_fullsized":"2023-09-23T23:18:17.008282+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:18.888131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"248'13753","reported_seq":15734,"reported_epoch":248,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T23:18:28.978695+0000","last_change":"2023-09-23T23:17:49.552705+0000","last_active":"2023-09-23T23:18:28.978695+0000","last_peered":"2023-09-23T23:18:28.978695+0000","last_clean":"2023-09-23T23:17:47.874547+0000","last_became_active":"2023-09-23T23:17:49.442103+0000","last_became_peered":"2023-09-23T23:17:49.442103+0000","last_unstale":"2023-09-23T23:18:28.978695+0000","last_undegraded":"2023-09-23T23:18:28.978695+0000","last_fullsized":"2023-09-23T23:18:28.978695+0000","mapping_epoch":236,"log_start":"174'5132","ondisk_log_start":"174'5132","created":164,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":8621,"log_dups_size":0,"ondisk_log_size":8621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:18.526356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21823488,"num_objects":333,"num_object_clones":0,"num_object_copies":999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":333,"num_objects_unfound":0,"num_objects_dirty":333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13753,"num_write_kb":450752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":925,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":333}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":209,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.019194+0000","last_change":"2023-09-23T23:10:59.502426+0000","last_active":"2023-09-23T23:18:17.019194+0000","last_peered":"2023-09-23T23:18:17.019194+0000","last_clean":"2023-09-23T23:18:17.019194+0000","last_became_active":"2023-09-23T23:10:59.502074+0000","last_became_peered":"2023-09-23T23:10:59.502074+0000","last_unstale":"2023-09-23T23:18:17.019194+0000","last_undegraded":"2023-09-23T23:18:17.019194+0000","last_fullsized":"2023-09-23T23:18:17.019194+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:46.029347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":202,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.010177+0000","last_change":"2023-09-23T23:04:11.912493+0000","last_active":"2023-09-23T23:18:17.010177+0000","last_peered":"2023-09-23T23:18:17.010177+0000","last_clean":"2023-09-23T23:18:17.010177+0000","last_became_active":"2023-09-23T23:04:11.906712+0000","last_became_peered":"2023-09-23T23:04:11.906712+0000","last_unstale":"2023-09-23T23:18:17.010177+0000","last_undegraded":"2023-09-23T23:18:17.010177+0000","last_fullsized":"2023-09-23T23:18:17.010177+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:58.932610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":666,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.018527+0000","last_change":"2023-09-23T23:17:51.351926+0000","last_active":"2023-09-23T23:18:17.018527+0000","last_peered":"2023-09-23T23:18:17.018527+0000","last_clean":"2023-09-23T23:18:17.018527+0000","last_became_active":"2023-09-23T23:17:51.351586+0000","last_became_peered":"2023-09-23T23:17:51.351586+0000","last_unstale":"2023-09-23T23:18:17.018527+0000","last_undegraded":"2023-09-23T23:18:17.018527+0000","last_fullsized":"2023-09-23T23:18:17.018527+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:46.150686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:46.150686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:52.403435+0000","scrub_duration":0.0088912260000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.1","version":"248'13624","reported_seq":19616,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T23:18:27.962425+0000","last_change":"2023-09-23T23:17:15.459960+0000","last_active":"2023-09-23T23:18:27.962425+0000","last_peered":"2023-09-23T23:18:27.962425+0000","last_clean":"2023-09-23T23:18:27.962425+0000","last_became_active":"2023-09-23T23:17:15.424151+0000","last_became_peered":"2023-09-23T23:17:15.424151+0000","last_unstale":"2023-09-23T23:18:27.962425+0000","last_undegraded":"2023-09-23T23:18:27.962425+0000","last_fullsized":"2023-09-23T23:18:27.962425+0000","mapping_epoch":218,"log_start":"184'7147","ondisk_log_start":"184'7147","created":164,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":6477,"log_dups_size":0,"ondisk_log_size":6477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:21.856597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23855104,"num_objects":364,"num_object_clones":0,"num_object_copies":1092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13624,"num_write_kb":447616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2935,"num_bytes_recovered":192348160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":212,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.019316+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:18:17.019316+0000","last_peered":"2023-09-23T23:18:17.019316+0000","last_clean":"2023-09-23T23:18:17.019316+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:18:17.019316+0000","last_undegraded":"2023-09-23T23:18:17.019316+0000","last_fullsized":"2023-09-23T23:18:17.019316+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":208,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.021826+0000","last_change":"2023-09-23T23:10:59.512363+0000","last_active":"2023-09-23T23:18:17.021826+0000","last_peered":"2023-09-23T23:18:17.021826+0000","last_clean":"2023-09-23T23:18:17.021826+0000","last_became_active":"2023-09-23T23:10:59.512022+0000","last_became_peered":"2023-09-23T23:10:59.512022+0000","last_unstale":"2023-09-23T23:18:17.021826+0000","last_undegraded":"2023-09-23T23:18:17.021826+0000","last_fullsized":"2023-09-23T23:18:17.021826+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:45.523432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":448,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.016251+0000","last_change":"2023-09-23T23:17:51.331237+0000","last_active":"2023-09-23T23:18:17.016251+0000","last_peered":"2023-09-23T23:18:17.016251+0000","last_clean":"2023-09-23T23:18:17.016251+0000","last_became_active":"2023-09-23T23:17:51.330912+0000","last_became_peered":"2023-09-23T23:17:51.330912+0000","last_unstale":"2023-09-23T23:18:17.016251+0000","last_undegraded":"2023-09-23T23:18:17.016251+0000","last_fullsized":"2023-09-23T23:18:17.016251+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:47.047289+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:47.047289+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.0","version":"248'13493","reported_seq":23613,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T23:18:29.270921+0000","last_change":"2023-09-23T23:17:58.043771+0000","last_active":"2023-09-23T23:18:29.270921+0000","last_peered":"2023-09-23T23:18:29.270921+0000","last_clean":"2023-09-23T23:18:29.270921+0000","last_became_active":"2023-09-23T23:17:57.984332+0000","last_became_peered":"2023-09-23T23:17:57.984332+0000","last_unstale":"2023-09-23T23:18:29.270921+0000","last_undegraded":"2023-09-23T23:18:29.270921+0000","last_fullsized":"2023-09-23T23:18:29.270921+0000","mapping_epoch":240,"log_start":"191'8205","ondisk_log_start":"191'8205","created":164,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":5288,"log_dups_size":0,"ondisk_log_size":5288,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:02.161575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18153472,"num_objects":277,"num_object_clones":0,"num_object_copies":831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13493,"num_write_kb":440640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4976,"num_bytes_recovered":269615104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":219,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.008366+0000","last_change":"2023-09-23T23:16:56.525904+0000","last_active":"2023-09-23T23:18:17.008366+0000","last_peered":"2023-09-23T23:18:17.008366+0000","last_clean":"2023-09-23T23:18:17.008366+0000","last_became_active":"2023-09-23T23:16:56.525372+0000","last_became_peered":"2023-09-23T23:16:56.525372+0000","last_unstale":"2023-09-23T23:18:17.008366+0000","last_undegraded":"2023-09-23T23:18:17.008366+0000","last_fullsized":"2023-09-23T23:18:17.008366+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:17.070117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":252,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.025603+0000","last_change":"2023-09-23T23:17:48.319835+0000","last_active":"2023-09-23T23:18:17.025603+0000","last_peered":"2023-09-23T23:18:17.025603+0000","last_clean":"2023-09-23T23:18:17.025603+0000","last_became_active":"2023-09-23T23:17:48.319535+0000","last_became_peered":"2023-09-23T23:17:48.319535+0000","last_unstale":"2023-09-23T23:18:17.025603+0000","last_undegraded":"2023-09-23T23:18:17.025603+0000","last_fullsized":"2023-09-23T23:18:17.025603+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:45:06.176314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"248'13597","reported_seq":13688,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T23:18:29.003454+0000","last_change":"2023-09-23T23:13:28.356430+0000","last_active":"2023-09-23T23:18:29.003454+0000","last_peered":"2023-09-23T23:18:29.003454+0000","last_clean":"2023-09-23T23:18:29.003454+0000","last_became_active":"2023-09-23T23:13:28.356172+0000","last_became_peered":"2023-09-23T23:13:28.356172+0000","last_unstale":"2023-09-23T23:18:29.003454+0000","last_undegraded":"2023-09-23T23:18:29.003454+0000","last_fullsized":"2023-09-23T23:18:29.003454+0000","mapping_epoch":164,"log_start":"171'3700","ondisk_log_start":"171'3700","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":9897,"log_dups_size":0,"ondisk_log_size":9897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:06.046311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18939904,"num_objects":289,"num_object_clones":0,"num_object_copies":867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13597,"num_write_kb":444352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":236,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.021475+0000","last_change":"2023-09-23T23:17:48.319410+0000","last_active":"2023-09-23T23:18:17.021475+0000","last_peered":"2023-09-23T23:18:17.021475+0000","last_clean":"2023-09-23T23:18:17.021475+0000","last_became_active":"2023-09-23T23:17:48.303382+0000","last_became_peered":"2023-09-23T23:17:48.303382+0000","last_unstale":"2023-09-23T23:18:17.021475+0000","last_undegraded":"2023-09-23T23:18:17.021475+0000","last_fullsized":"2023-09-23T23:18:17.021475+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:09.313879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":218,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.022277+0000","last_change":"2023-09-23T23:15:28.228439+0000","last_active":"2023-09-23T23:18:17.022277+0000","last_peered":"2023-09-23T23:18:17.022277+0000","last_clean":"2023-09-23T23:18:17.022277+0000","last_became_active":"2023-09-23T23:15:28.228095+0000","last_became_peered":"2023-09-23T23:15:28.228095+0000","last_unstale":"2023-09-23T23:18:17.022277+0000","last_undegraded":"2023-09-23T23:18:17.022277+0000","last_fullsized":"2023-09-23T23:18:17.022277+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:37.667201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":528,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.018587+0000","last_change":"2023-09-23T23:17:51.348000+0000","last_active":"2023-09-23T23:18:17.018587+0000","last_peered":"2023-09-23T23:18:17.018587+0000","last_clean":"2023-09-23T23:18:17.018587+0000","last_became_active":"2023-09-23T23:17:51.347662+0000","last_became_peered":"2023-09-23T23:17:51.347662+0000","last_unstale":"2023-09-23T23:18:17.018587+0000","last_undegraded":"2023-09-23T23:18:17.018587+0000","last_fullsized":"2023-09-23T23:18:17.018587+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:51.734652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:51.734652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:38.179876+0000","scrub_duration":0.023100959000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"248'13801","reported_seq":24210,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T23:18:29.040051+0000","last_change":"2023-09-23T23:18:09.360801+0000","last_active":"2023-09-23T23:18:29.040051+0000","last_peered":"2023-09-23T23:18:29.040051+0000","last_clean":"2023-09-23T23:18:29.040051+0000","last_became_active":"2023-09-23T23:18:09.323360+0000","last_became_peered":"2023-09-23T23:18:09.323360+0000","last_unstale":"2023-09-23T23:18:29.040051+0000","last_undegraded":"2023-09-23T23:18:29.040051+0000","last_fullsized":"2023-09-23T23:18:29.040051+0000","mapping_epoch":244,"log_start":"183'6601","ondisk_log_start":"183'6601","created":164,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":7200,"log_dups_size":0,"ondisk_log_size":7200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.127334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21823488,"num_objects":333,"num_object_clones":0,"num_object_copies":999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13801,"num_write_kb":452288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5122,"num_bytes_recovered":274923520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":125,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.005279+0000","last_change":"2023-09-23T23:16:56.534749+0000","last_active":"2023-09-23T23:18:17.005279+0000","last_peered":"2023-09-23T23:18:17.005279+0000","last_clean":"2023-09-23T23:18:17.005279+0000","last_became_active":"2023-09-23T23:16:56.534346+0000","last_became_peered":"2023-09-23T23:16:56.534346+0000","last_unstale":"2023-09-23T23:18:17.005279+0000","last_undegraded":"2023-09-23T23:18:17.005279+0000","last_fullsized":"2023-09-23T23:18:17.005279+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:55.720486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":217,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.005068+0000","last_change":"2023-09-23T23:10:59.503483+0000","last_active":"2023-09-23T23:18:17.005068+0000","last_peered":"2023-09-23T23:18:17.005068+0000","last_clean":"2023-09-23T23:18:17.005068+0000","last_became_active":"2023-09-23T23:10:59.502851+0000","last_became_peered":"2023-09-23T23:10:59.502851+0000","last_unstale":"2023-09-23T23:18:17.005068+0000","last_undegraded":"2023-09-23T23:18:17.005068+0000","last_fullsized":"2023-09-23T23:18:17.005068+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:20.388467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":581,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.013712+0000","last_change":"2023-09-23T23:17:51.355247+0000","last_active":"2023-09-23T23:18:17.013712+0000","last_peered":"2023-09-23T23:18:17.013712+0000","last_clean":"2023-09-23T23:18:17.013712+0000","last_became_active":"2023-09-23T23:17:51.354856+0000","last_became_peered":"2023-09-23T23:17:51.354856+0000","last_unstale":"2023-09-23T23:18:17.013712+0000","last_undegraded":"2023-09-23T23:18:17.013712+0000","last_fullsized":"2023-09-23T23:18:17.013712+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:47.797085+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:47.797085+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:55.878838+0000","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.f","version":"248'13552","reported_seq":13644,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T23:18:28.971798+0000","last_change":"2023-09-23T23:13:28.355012+0000","last_active":"2023-09-23T23:18:28.971798+0000","last_peered":"2023-09-23T23:18:28.971798+0000","last_clean":"2023-09-23T23:18:28.971798+0000","last_became_active":"2023-09-23T23:13:28.354788+0000","last_became_peered":"2023-09-23T23:13:28.354788+0000","last_unstale":"2023-09-23T23:18:28.971798+0000","last_undegraded":"2023-09-23T23:18:28.971798+0000","last_fullsized":"2023-09-23T23:18:28.971798+0000","mapping_epoch":164,"log_start":"173'4933","ondisk_log_start":"173'4933","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":8619,"log_dups_size":0,"ondisk_log_size":8619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:04.013664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18612224,"num_objects":284,"num_object_clones":0,"num_object_copies":852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13552,"num_write_kb":442752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":210,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.006950+0000","last_change":"2023-09-23T23:07:30.069993+0000","last_active":"2023-09-23T23:18:17.006950+0000","last_peered":"2023-09-23T23:18:17.006950+0000","last_clean":"2023-09-23T23:18:17.006950+0000","last_became_active":"2023-09-23T23:07:30.069634+0000","last_became_peered":"2023-09-23T23:07:30.069634+0000","last_unstale":"2023-09-23T23:18:17.006950+0000","last_undegraded":"2023-09-23T23:18:17.006950+0000","last_fullsized":"2023-09-23T23:18:17.006950+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:24.608905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":253,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.019042+0000","last_change":"2023-09-23T23:17:48.318576+0000","last_active":"2023-09-23T23:18:17.019042+0000","last_peered":"2023-09-23T23:18:17.019042+0000","last_clean":"2023-09-23T23:18:17.019042+0000","last_became_active":"2023-09-23T23:17:48.302180+0000","last_became_peered":"2023-09-23T23:17:48.302180+0000","last_unstale":"2023-09-23T23:18:17.019042+0000","last_undegraded":"2023-09-23T23:18:17.019042+0000","last_fullsized":"2023-09-23T23:18:17.019042+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:34:56.665520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":182,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.024680+0000","last_change":"2023-09-23T23:17:48.320421+0000","last_active":"2023-09-23T23:18:17.024680+0000","last_peered":"2023-09-23T23:18:17.024680+0000","last_clean":"2023-09-23T23:18:17.024680+0000","last_became_active":"2023-09-23T23:17:48.320112+0000","last_became_peered":"2023-09-23T23:17:48.320112+0000","last_unstale":"2023-09-23T23:18:17.024680+0000","last_undegraded":"2023-09-23T23:18:17.024680+0000","last_fullsized":"2023-09-23T23:18:17.024680+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:32.901735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":528,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.018566+0000","last_change":"2023-09-23T23:17:51.342323+0000","last_active":"2023-09-23T23:18:17.018566+0000","last_peered":"2023-09-23T23:18:17.018566+0000","last_clean":"2023-09-23T23:18:17.018566+0000","last_became_active":"2023-09-23T23:17:51.341930+0000","last_became_peered":"2023-09-23T23:17:51.341930+0000","last_unstale":"2023-09-23T23:18:17.018566+0000","last_undegraded":"2023-09-23T23:18:17.018566+0000","last_fullsized":"2023-09-23T23:18:17.018566+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:49.726740+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:49.726740+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:16:57.092869+0000","scrub_duration":0.0013902070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"248'13603","reported_seq":13710,"reported_epoch":248,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:18:31.371691+0000","last_change":"2023-09-23T23:17:48.347691+0000","last_active":"2023-09-23T23:18:31.371691+0000","last_peered":"2023-09-23T23:18:31.371691+0000","last_clean":"2023-09-23T23:16:55.485111+0000","last_became_active":"2023-09-23T23:16:57.639573+0000","last_became_peered":"2023-09-23T23:16:57.639573+0000","last_unstale":"2023-09-23T23:18:31.371691+0000","last_undegraded":"2023-09-23T23:18:31.371691+0000","last_fullsized":"2023-09-23T23:18:31.371691+0000","mapping_epoch":213,"log_start":"184'7190","ondisk_log_start":"184'7190","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":6413,"log_dups_size":0,"ondisk_log_size":6413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:00.341194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12910592,"num_objects":197,"num_object_clones":0,"num_object_copies":591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":197,"num_objects_unfound":0,"num_objects_dirty":197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13603,"num_write_kb":441600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":197}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":213,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.018725+0000","last_change":"2023-09-23T23:07:30.075093+0000","last_active":"2023-09-23T23:18:17.018725+0000","last_peered":"2023-09-23T23:18:17.018725+0000","last_clean":"2023-09-23T23:18:17.018725+0000","last_became_active":"2023-09-23T23:07:30.074729+0000","last_became_peered":"2023-09-23T23:07:30.074729+0000","last_unstale":"2023-09-23T23:18:17.018725+0000","last_undegraded":"2023-09-23T23:18:17.018725+0000","last_fullsized":"2023-09-23T23:18:17.018725+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:34.502057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":209,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.021081+0000","last_change":"2023-09-23T23:04:11.514703+0000","last_active":"2023-09-23T23:18:17.021081+0000","last_peered":"2023-09-23T23:18:17.021081+0000","last_clean":"2023-09-23T23:18:17.021081+0000","last_became_active":"2023-09-23T23:04:11.514403+0000","last_became_peered":"2023-09-23T23:04:11.514403+0000","last_unstale":"2023-09-23T23:18:17.021081+0000","last_undegraded":"2023-09-23T23:18:17.021081+0000","last_fullsized":"2023-09-23T23:18:17.021081+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:26.374346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":216,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.021020+0000","last_change":"2023-09-23T23:17:48.300618+0000","last_active":"2023-09-23T23:18:17.021020+0000","last_peered":"2023-09-23T23:18:17.021020+0000","last_clean":"2023-09-23T23:18:17.021020+0000","last_became_active":"2023-09-23T23:17:48.300279+0000","last_became_peered":"2023-09-23T23:17:48.300279+0000","last_unstale":"2023-09-23T23:18:17.021020+0000","last_undegraded":"2023-09-23T23:18:17.021020+0000","last_fullsized":"2023-09-23T23:18:17.021020+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:53.227630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":513,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.011993+0000","last_change":"2023-09-23T23:17:51.342338+0000","last_active":"2023-09-23T23:18:17.011993+0000","last_peered":"2023-09-23T23:18:17.011993+0000","last_clean":"2023-09-23T23:18:17.011993+0000","last_became_active":"2023-09-23T23:17:51.342020+0000","last_became_peered":"2023-09-23T23:17:51.342020+0000","last_unstale":"2023-09-23T23:18:17.011993+0000","last_undegraded":"2023-09-23T23:18:17.011993+0000","last_fullsized":"2023-09-23T23:18:17.011993+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.183948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.183948+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"248'13395","reported_seq":16840,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T23:18:29.019385+0000","last_change":"2023-09-23T23:18:00.465911+0000","last_active":"2023-09-23T23:18:29.019385+0000","last_peered":"2023-09-23T23:18:29.019385+0000","last_clean":"2023-09-23T23:18:29.019385+0000","last_became_active":"2023-09-23T23:18:00.352183+0000","last_became_peered":"2023-09-23T23:18:00.352183+0000","last_unstale":"2023-09-23T23:18:29.019385+0000","last_undegraded":"2023-09-23T23:18:29.019385+0000","last_fullsized":"2023-09-23T23:18:29.019385+0000","mapping_epoch":242,"log_start":"182'6213","ondisk_log_start":"182'6213","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":7182,"log_dups_size":0,"ondisk_log_size":7182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:09.071393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19070976,"num_objects":291,"num_object_clones":0,"num_object_copies":873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13395,"num_write_kb":437952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1658,"num_bytes_recovered":108527616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":176,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.016852+0000","last_change":"2023-09-23T23:17:48.300243+0000","last_active":"2023-09-23T23:18:17.016852+0000","last_peered":"2023-09-23T23:18:17.016852+0000","last_clean":"2023-09-23T23:18:17.016852+0000","last_became_active":"2023-09-23T23:17:48.299953+0000","last_became_peered":"2023-09-23T23:17:48.299953+0000","last_unstale":"2023-09-23T23:18:17.016852+0000","last_undegraded":"2023-09-23T23:18:17.016852+0000","last_fullsized":"2023-09-23T23:18:17.016852+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:44.381159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":243,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.005153+0000","last_change":"2023-09-23T23:17:48.305230+0000","last_active":"2023-09-23T23:18:17.005153+0000","last_peered":"2023-09-23T23:18:17.005153+0000","last_clean":"2023-09-23T23:18:17.005153+0000","last_became_active":"2023-09-23T23:17:48.304847+0000","last_became_peered":"2023-09-23T23:17:48.304847+0000","last_unstale":"2023-09-23T23:18:17.005153+0000","last_undegraded":"2023-09-23T23:18:17.005153+0000","last_fullsized":"2023-09-23T23:18:17.005153+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:51.308840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":140,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.022406+0000","last_change":"2023-09-23T23:16:56.545792+0000","last_active":"2023-09-23T23:18:17.022406+0000","last_peered":"2023-09-23T23:18:17.022406+0000","last_clean":"2023-09-23T23:18:17.022406+0000","last_became_active":"2023-09-23T23:16:56.545447+0000","last_became_peered":"2023-09-23T23:16:56.545447+0000","last_unstale":"2023-09-23T23:18:17.022406+0000","last_undegraded":"2023-09-23T23:18:17.022406+0000","last_fullsized":"2023-09-23T23:18:17.022406+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:47.781440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"248'13635","reported_seq":13849,"reported_epoch":248,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:18:31.387549+0000","last_change":"2023-09-23T23:17:48.365484+0000","last_active":"2023-09-23T23:18:31.387549+0000","last_peered":"2023-09-23T23:18:31.387549+0000","last_clean":"2023-09-23T23:15:27.279123+0000","last_became_active":"2023-09-23T23:15:29.334529+0000","last_became_peered":"2023-09-23T23:15:29.334529+0000","last_unstale":"2023-09-23T23:18:31.387549+0000","last_undegraded":"2023-09-23T23:18:31.387549+0000","last_fullsized":"2023-09-23T23:18:31.387549+0000","mapping_epoch":187,"log_start":"184'7242","ondisk_log_start":"184'7242","created":164,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":6393,"log_dups_size":0,"ondisk_log_size":6393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:47:05.361853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12779520,"num_objects":195,"num_object_clones":0,"num_object_copies":585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":194,"num_objects_unfound":0,"num_objects_dirty":195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13635,"num_write_kb":442560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":3342336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":195}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":250,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.018936+0000","last_change":"2023-09-23T23:17:48.302538+0000","last_active":"2023-09-23T23:18:17.018936+0000","last_peered":"2023-09-23T23:18:17.018936+0000","last_clean":"2023-09-23T23:18:17.018936+0000","last_became_active":"2023-09-23T23:17:48.302203+0000","last_became_peered":"2023-09-23T23:17:48.302203+0000","last_unstale":"2023-09-23T23:18:17.018936+0000","last_undegraded":"2023-09-23T23:18:17.018936+0000","last_fullsized":"2023-09-23T23:18:17.018936+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:25.857118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":454,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.008779+0000","last_change":"2023-09-23T23:16:59.290902+0000","last_active":"2023-09-23T23:18:17.008779+0000","last_peered":"2023-09-23T23:18:17.008779+0000","last_clean":"2023-09-23T23:18:17.008779+0000","last_became_active":"2023-09-23T23:16:59.290535+0000","last_became_peered":"2023-09-23T23:16:59.290535+0000","last_unstale":"2023-09-23T23:18:17.008779+0000","last_undegraded":"2023-09-23T23:18:17.008779+0000","last_fullsized":"2023-09-23T23:18:17.008779+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:43.147316+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:43.147316+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":175,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.019096+0000","last_change":"2023-09-23T23:16:56.525298+0000","last_active":"2023-09-23T23:18:17.019096+0000","last_peered":"2023-09-23T23:18:17.019096+0000","last_clean":"2023-09-23T23:18:17.019096+0000","last_became_active":"2023-09-23T23:16:56.524974+0000","last_became_peered":"2023-09-23T23:16:56.524974+0000","last_unstale":"2023-09-23T23:18:17.019096+0000","last_undegraded":"2023-09-23T23:18:17.019096+0000","last_fullsized":"2023-09-23T23:18:17.019096+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:08.860790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":191,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.013458+0000","last_change":"2023-09-23T22:53:18.075922+0000","last_active":"2023-09-23T23:18:17.013458+0000","last_peered":"2023-09-23T23:18:17.013458+0000","last_clean":"2023-09-23T23:18:17.013458+0000","last_became_active":"2023-09-23T22:53:18.075621+0000","last_became_peered":"2023-09-23T22:53:18.075621+0000","last_unstale":"2023-09-23T23:18:17.013458+0000","last_undegraded":"2023-09-23T23:18:17.013458+0000","last_fullsized":"2023-09-23T23:18:17.013458+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:01.490333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":493,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.009146+0000","last_change":"2023-09-23T23:14:44.629928+0000","last_active":"2023-09-23T23:18:17.009146+0000","last_peered":"2023-09-23T23:18:17.009146+0000","last_clean":"2023-09-23T23:18:17.009146+0000","last_became_active":"2023-09-23T23:14:44.445304+0000","last_became_peered":"2023-09-23T23:14:44.445304+0000","last_unstale":"2023-09-23T23:18:17.009146+0000","last_undegraded":"2023-09-23T23:18:17.009146+0000","last_fullsized":"2023-09-23T23:18:17.009146+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:44.629771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:44.629771+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:07.493381+0000","scrub_duration":0.011659790999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.b","version":"248'13462","reported_seq":13557,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T23:18:31.369684+0000","last_change":"2023-09-23T23:13:28.357808+0000","last_active":"2023-09-23T23:18:31.369684+0000","last_peered":"2023-09-23T23:18:31.369684+0000","last_clean":"2023-09-23T23:18:31.369684+0000","last_became_active":"2023-09-23T23:13:28.357474+0000","last_became_peered":"2023-09-23T23:13:28.357474+0000","last_unstale":"2023-09-23T23:18:31.369684+0000","last_undegraded":"2023-09-23T23:18:31.369684+0000","last_fullsized":"2023-09-23T23:18:31.369684+0000","mapping_epoch":164,"log_start":"184'7033","ondisk_log_start":"184'7033","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":6429,"log_dups_size":0,"ondisk_log_size":6429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:08.042264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11796480,"num_objects":180,"num_object_clones":0,"num_object_copies":540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13462,"num_write_kb":436544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":240,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.017974+0000","last_change":"2023-09-23T23:16:56.523954+0000","last_active":"2023-09-23T23:18:17.017974+0000","last_peered":"2023-09-23T23:18:17.017974+0000","last_clean":"2023-09-23T23:18:17.017974+0000","last_became_active":"2023-09-23T23:16:56.523679+0000","last_became_peered":"2023-09-23T23:16:56.523679+0000","last_unstale":"2023-09-23T23:18:17.017974+0000","last_undegraded":"2023-09-23T23:18:17.017974+0000","last_fullsized":"2023-09-23T23:18:17.017974+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:51.443997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":208,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.010550+0000","last_change":"2023-09-23T23:16:56.525850+0000","last_active":"2023-09-23T23:18:17.010550+0000","last_peered":"2023-09-23T23:18:17.010550+0000","last_clean":"2023-09-23T23:18:17.010550+0000","last_became_active":"2023-09-23T23:16:56.524998+0000","last_became_peered":"2023-09-23T23:16:56.524998+0000","last_unstale":"2023-09-23T23:18:17.010550+0000","last_undegraded":"2023-09-23T23:18:17.010550+0000","last_fullsized":"2023-09-23T23:18:17.010550+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:54.317877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":238,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.015849+0000","last_change":"2023-09-23T23:17:48.308925+0000","last_active":"2023-09-23T23:18:17.015849+0000","last_peered":"2023-09-23T23:18:17.015849+0000","last_clean":"2023-09-23T23:18:17.015849+0000","last_became_active":"2023-09-23T23:17:48.308628+0000","last_became_peered":"2023-09-23T23:17:48.308628+0000","last_unstale":"2023-09-23T23:18:17.015849+0000","last_undegraded":"2023-09-23T23:18:17.015849+0000","last_fullsized":"2023-09-23T23:18:17.015849+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:58:28.247724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":521,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.013985+0000","last_change":"2023-09-23T23:16:59.295000+0000","last_active":"2023-09-23T23:18:17.013985+0000","last_peered":"2023-09-23T23:18:17.013985+0000","last_clean":"2023-09-23T23:18:17.013985+0000","last_became_active":"2023-09-23T23:16:59.294607+0000","last_became_peered":"2023-09-23T23:16:59.294607+0000","last_unstale":"2023-09-23T23:18:17.013985+0000","last_undegraded":"2023-09-23T23:18:17.013985+0000","last_fullsized":"2023-09-23T23:18:17.013985+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.802642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.802642+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:48.429859+0000","scrub_duration":0.0056570290000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.e","version":"248'13661","reported_seq":23455,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T23:18:29.400871+0000","last_change":"2023-09-23T23:17:43.594560+0000","last_active":"2023-09-23T23:18:29.400871+0000","last_peered":"2023-09-23T23:18:29.400871+0000","last_clean":"2023-09-23T23:18:29.400871+0000","last_became_active":"2023-09-23T23:17:43.563581+0000","last_became_peered":"2023-09-23T23:17:43.563581+0000","last_unstale":"2023-09-23T23:18:29.400871+0000","last_undegraded":"2023-09-23T23:18:29.400871+0000","last_fullsized":"2023-09-23T23:18:29.400871+0000","mapping_epoch":233,"log_start":"190'8227","ondisk_log_start":"190'8227","created":164,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":5434,"log_dups_size":0,"ondisk_log_size":5434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:39.419351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18677760,"num_objects":285,"num_object_clones":0,"num_object_copies":855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13661,"num_write_kb":446272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4826,"num_bytes_recovered":316276736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":213,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.018754+0000","last_change":"2023-09-23T23:04:11.900743+0000","last_active":"2023-09-23T23:18:17.018754+0000","last_peered":"2023-09-23T23:18:17.018754+0000","last_clean":"2023-09-23T23:18:17.018754+0000","last_became_active":"2023-09-23T23:04:11.900418+0000","last_became_peered":"2023-09-23T23:04:11.900418+0000","last_unstale":"2023-09-23T23:18:17.018754+0000","last_undegraded":"2023-09-23T23:18:17.018754+0000","last_fullsized":"2023-09-23T23:18:17.018754+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:45:31.893513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":252,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.006882+0000","last_change":"2023-09-23T23:17:48.305869+0000","last_active":"2023-09-23T23:18:17.006882+0000","last_peered":"2023-09-23T23:18:17.006882+0000","last_clean":"2023-09-23T23:18:17.006882+0000","last_became_active":"2023-09-23T23:17:48.305574+0000","last_became_peered":"2023-09-23T23:17:48.305574+0000","last_unstale":"2023-09-23T23:18:17.006882+0000","last_undegraded":"2023-09-23T23:18:17.006882+0000","last_fullsized":"2023-09-23T23:18:17.006882+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:55.580919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":193,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.010300+0000","last_change":"2023-09-23T23:17:33.437620+0000","last_active":"2023-09-23T23:18:17.010300+0000","last_peered":"2023-09-23T23:18:17.010300+0000","last_clean":"2023-09-23T23:18:17.010300+0000","last_became_active":"2023-09-23T23:17:33.437274+0000","last_became_peered":"2023-09-23T23:17:33.437274+0000","last_unstale":"2023-09-23T23:18:17.010300+0000","last_undegraded":"2023-09-23T23:18:17.010300+0000","last_fullsized":"2023-09-23T23:18:17.010300+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:01.385833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":528,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.025408+0000","last_change":"2023-09-23T23:17:51.331967+0000","last_active":"2023-09-23T23:18:17.025408+0000","last_peered":"2023-09-23T23:18:17.025408+0000","last_clean":"2023-09-23T23:18:17.025408+0000","last_became_active":"2023-09-23T23:17:51.331604+0000","last_became_peered":"2023-09-23T23:17:51.331604+0000","last_unstale":"2023-09-23T23:18:17.025408+0000","last_undegraded":"2023-09-23T23:18:17.025408+0000","last_fullsized":"2023-09-23T23:18:17.025408+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:51.489074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:51.489074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:38:55.870389+0000","scrub_duration":0.00519324,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.d","version":"248'13543","reported_seq":29013,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T23:18:29.005419+0000","last_change":"2023-09-23T23:17:41.579641+0000","last_active":"2023-09-23T23:18:29.005419+0000","last_peered":"2023-09-23T23:18:29.005419+0000","last_clean":"2023-09-23T23:18:29.005419+0000","last_became_active":"2023-09-23T23:17:41.540249+0000","last_became_peered":"2023-09-23T23:17:41.540249+0000","last_unstale":"2023-09-23T23:18:29.005419+0000","last_undegraded":"2023-09-23T23:18:29.005419+0000","last_fullsized":"2023-09-23T23:18:29.005419+0000","mapping_epoch":231,"log_start":"173'4900","ondisk_log_start":"173'4900","created":164,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":8643,"log_dups_size":0,"ondisk_log_size":8643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:15:49.731942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20316184,"num_objects":311,"num_object_clones":0,"num_object_copies":933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":311,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13543,"num_write_kb":443265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7658,"num_bytes_recovered":501743664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":225,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.019212+0000","last_change":"2023-09-23T23:16:56.561147+0000","last_active":"2023-09-23T23:18:17.019212+0000","last_peered":"2023-09-23T23:18:17.019212+0000","last_clean":"2023-09-23T23:18:17.019212+0000","last_became_active":"2023-09-23T23:16:56.547808+0000","last_became_peered":"2023-09-23T23:16:56.547808+0000","last_unstale":"2023-09-23T23:18:17.019212+0000","last_undegraded":"2023-09-23T23:18:17.019212+0000","last_fullsized":"2023-09-23T23:18:17.019212+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:13.666957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":103,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.014119+0000","last_change":"2023-09-23T23:16:56.526187+0000","last_active":"2023-09-23T23:18:17.014119+0000","last_peered":"2023-09-23T23:18:17.014119+0000","last_clean":"2023-09-23T23:18:17.014119+0000","last_became_active":"2023-09-23T23:16:56.525874+0000","last_became_peered":"2023-09-23T23:16:56.525874+0000","last_unstale":"2023-09-23T23:18:17.014119+0000","last_undegraded":"2023-09-23T23:18:17.014119+0000","last_fullsized":"2023-09-23T23:18:17.014119+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:32:07.101559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":145,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.008679+0000","last_change":"2023-09-23T23:16:56.528347+0000","last_active":"2023-09-23T23:18:17.008679+0000","last_peered":"2023-09-23T23:18:17.008679+0000","last_clean":"2023-09-23T23:18:17.008679+0000","last_became_active":"2023-09-23T23:16:56.527965+0000","last_became_peered":"2023-09-23T23:16:56.527965+0000","last_unstale":"2023-09-23T23:18:17.008679+0000","last_undegraded":"2023-09-23T23:18:17.008679+0000","last_fullsized":"2023-09-23T23:18:17.008679+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:46.313356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":514,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.021211+0000","last_change":"2023-09-23T23:16:59.285629+0000","last_active":"2023-09-23T23:18:17.021211+0000","last_peered":"2023-09-23T23:18:17.021211+0000","last_clean":"2023-09-23T23:18:17.021211+0000","last_became_active":"2023-09-23T23:16:59.285208+0000","last_became_peered":"2023-09-23T23:16:59.285208+0000","last_unstale":"2023-09-23T23:18:17.021211+0000","last_undegraded":"2023-09-23T23:18:17.021211+0000","last_fullsized":"2023-09-23T23:18:17.021211+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.522632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.522632+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:04:15.985273+0000","scrub_duration":0.018152603999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"248'13537","reported_seq":13632,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T23:18:27.978768+0000","last_change":"2023-09-23T23:13:28.359953+0000","last_active":"2023-09-23T23:18:27.978768+0000","last_peered":"2023-09-23T23:18:27.978768+0000","last_clean":"2023-09-23T23:18:27.978768+0000","last_became_active":"2023-09-23T23:13:28.359690+0000","last_became_peered":"2023-09-23T23:13:28.359690+0000","last_unstale":"2023-09-23T23:18:27.978768+0000","last_undegraded":"2023-09-23T23:18:27.978768+0000","last_fullsized":"2023-09-23T23:18:27.978768+0000","mapping_epoch":164,"log_start":"184'7076","ondisk_log_start":"184'7076","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:13:27.335981+0000","last_clean_scrub_stamp":"2023-09-23T23:13:27.335981+0000","objects_scrubbed":0,"log_size":6461,"log_dups_size":0,"ondisk_log_size":6461,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:56.963155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24969216,"num_objects":381,"num_object_clones":0,"num_object_copies":1143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13537,"num_write_kb":445376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":250,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.013611+0000","last_change":"2023-09-23T23:17:48.303246+0000","last_active":"2023-09-23T23:18:17.013611+0000","last_peered":"2023-09-23T23:18:17.013611+0000","last_clean":"2023-09-23T23:18:17.013611+0000","last_became_active":"2023-09-23T23:17:48.302984+0000","last_became_peered":"2023-09-23T23:17:48.302984+0000","last_unstale":"2023-09-23T23:18:17.013611+0000","last_undegraded":"2023-09-23T23:18:17.013611+0000","last_fullsized":"2023-09-23T23:18:17.013611+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:11.999957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":203,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.015691+0000","last_change":"2023-09-23T23:07:30.072388+0000","last_active":"2023-09-23T23:18:17.015691+0000","last_peered":"2023-09-23T23:18:17.015691+0000","last_clean":"2023-09-23T23:18:17.015691+0000","last_became_active":"2023-09-23T23:07:30.072099+0000","last_became_peered":"2023-09-23T23:07:30.072099+0000","last_unstale":"2023-09-23T23:18:17.015691+0000","last_undegraded":"2023-09-23T23:18:17.015691+0000","last_fullsized":"2023-09-23T23:18:17.015691+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:19.758734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":191,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.021405+0000","last_change":"2023-09-23T22:53:17.905706+0000","last_active":"2023-09-23T23:18:17.021405+0000","last_peered":"2023-09-23T23:18:17.021405+0000","last_clean":"2023-09-23T23:18:17.021405+0000","last_became_active":"2023-09-23T22:53:17.905309+0000","last_became_peered":"2023-09-23T22:53:17.905309+0000","last_unstale":"2023-09-23T23:18:17.021405+0000","last_undegraded":"2023-09-23T23:18:17.021405+0000","last_fullsized":"2023-09-23T23:18:17.021405+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:06.105426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":187,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.010725+0000","last_change":"2023-09-23T23:04:11.900795+0000","last_active":"2023-09-23T23:18:17.010725+0000","last_peered":"2023-09-23T23:18:17.010725+0000","last_clean":"2023-09-23T23:18:17.010725+0000","last_became_active":"2023-09-23T23:04:11.899751+0000","last_became_peered":"2023-09-23T23:04:11.899751+0000","last_unstale":"2023-09-23T23:18:17.010725+0000","last_undegraded":"2023-09-23T23:18:17.010725+0000","last_fullsized":"2023-09-23T23:18:17.010725+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:41.206493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":191,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T23:18:20.081856+0000","last_change":"2023-09-23T23:17:48.306469+0000","last_active":"2023-09-23T23:18:20.081856+0000","last_peered":"2023-09-23T23:18:20.081856+0000","last_clean":"2023-09-23T23:18:20.081856+0000","last_became_active":"2023-09-23T23:17:48.306156+0000","last_became_peered":"2023-09-23T23:17:48.306156+0000","last_unstale":"2023-09-23T23:18:20.081856+0000","last_undegraded":"2023-09-23T23:18:20.081856+0000","last_fullsized":"2023-09-23T23:18:20.081856+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:07.516611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":211,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.019105+0000","last_change":"2023-09-23T23:07:30.075311+0000","last_active":"2023-09-23T23:18:17.019105+0000","last_peered":"2023-09-23T23:18:17.019105+0000","last_clean":"2023-09-23T23:18:17.019105+0000","last_became_active":"2023-09-23T23:07:30.074952+0000","last_became_peered":"2023-09-23T23:07:30.074952+0000","last_unstale":"2023-09-23T23:18:17.019105+0000","last_undegraded":"2023-09-23T23:18:17.019105+0000","last_fullsized":"2023-09-23T23:18:17.019105+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:16.724083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":245,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.020946+0000","last_change":"2023-09-23T23:17:48.300748+0000","last_active":"2023-09-23T23:18:17.020946+0000","last_peered":"2023-09-23T23:18:17.020946+0000","last_clean":"2023-09-23T23:18:17.020946+0000","last_became_active":"2023-09-23T23:17:48.300443+0000","last_became_peered":"2023-09-23T23:17:48.300443+0000","last_unstale":"2023-09-23T23:18:17.020946+0000","last_undegraded":"2023-09-23T23:18:17.020946+0000","last_fullsized":"2023-09-23T23:18:17.020946+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:11.537165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":194,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.019116+0000","last_change":"2023-09-23T22:53:16.891297+0000","last_active":"2023-09-23T23:18:17.019116+0000","last_peered":"2023-09-23T23:18:17.019116+0000","last_clean":"2023-09-23T23:18:17.019116+0000","last_became_active":"2023-09-23T22:53:16.891025+0000","last_became_peered":"2023-09-23T22:53:16.891025+0000","last_unstale":"2023-09-23T23:18:17.019116+0000","last_undegraded":"2023-09-23T23:18:17.019116+0000","last_fullsized":"2023-09-23T23:18:17.019116+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:40:50.608339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":225,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.022696+0000","last_change":"2023-09-23T23:10:59.512247+0000","last_active":"2023-09-23T23:18:17.022696+0000","last_peered":"2023-09-23T23:18:17.022696+0000","last_clean":"2023-09-23T23:18:17.022696+0000","last_became_active":"2023-09-23T23:10:59.511865+0000","last_became_peered":"2023-09-23T23:10:59.511865+0000","last_unstale":"2023-09-23T23:18:17.022696+0000","last_undegraded":"2023-09-23T23:18:17.022696+0000","last_fullsized":"2023-09-23T23:18:17.022696+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:20.981239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":232,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.019420+0000","last_change":"2023-09-23T23:17:48.304879+0000","last_active":"2023-09-23T23:18:17.019420+0000","last_peered":"2023-09-23T23:18:17.019420+0000","last_clean":"2023-09-23T23:18:17.019420+0000","last_became_active":"2023-09-23T23:17:48.304506+0000","last_became_peered":"2023-09-23T23:17:48.304506+0000","last_unstale":"2023-09-23T23:18:17.019420+0000","last_undegraded":"2023-09-23T23:18:17.019420+0000","last_fullsized":"2023-09-23T23:18:17.019420+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:28.380655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":247,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.016937+0000","last_change":"2023-09-23T23:17:48.303036+0000","last_active":"2023-09-23T23:18:17.016937+0000","last_peered":"2023-09-23T23:18:17.016937+0000","last_clean":"2023-09-23T23:18:17.016937+0000","last_became_active":"2023-09-23T23:17:48.302706+0000","last_became_peered":"2023-09-23T23:17:48.302706+0000","last_unstale":"2023-09-23T23:18:17.016937+0000","last_undegraded":"2023-09-23T23:18:17.016937+0000","last_fullsized":"2023-09-23T23:18:17.016937+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:46:22.942097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":233,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.010978+0000","last_change":"2023-09-23T23:17:48.338955+0000","last_active":"2023-09-23T23:18:17.010978+0000","last_peered":"2023-09-23T23:18:17.010978+0000","last_clean":"2023-09-23T23:18:17.010978+0000","last_became_active":"2023-09-23T23:17:48.332511+0000","last_became_peered":"2023-09-23T23:17:48.332511+0000","last_unstale":"2023-09-23T23:18:17.010978+0000","last_undegraded":"2023-09-23T23:18:17.010978+0000","last_fullsized":"2023-09-23T23:18:17.010978+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:12.303751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":245,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T23:18:17.020738+0000","last_change":"2023-09-23T23:17:48.319814+0000","last_active":"2023-09-23T23:18:17.020738+0000","last_peered":"2023-09-23T23:18:17.020738+0000","last_clean":"2023-09-23T23:18:17.020738+0000","last_became_active":"2023-09-23T23:17:48.303463+0000","last_became_peered":"2023-09-23T23:17:48.303463+0000","last_unstale":"2023-09-23T23:18:17.020738+0000","last_undegraded":"2023-09-23T23:18:17.020738+0000","last_fullsized":"2023-09-23T23:18:17.020738+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:33.861565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":770,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":133,"num_read_kb":248,"num_write":65,"num_write_kb":658,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1073396,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":299237400,"num_objects":4567,"num_object_clones":0,"num_object_copies":13701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1384,"num_objects_unfound":0,"num_objects_dirty":4567,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":217791,"num_write_kb":7115393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34448,"num_bytes_recovered":2053439536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2245013504,"data_stored":2245001288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115279,"ondisk_log_size":115279,"up":48,"acting":47,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234309,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1092680,"kb_used_data":654272,"kb_used_omap":0,"kb_used_meta":438016,"kb_avail":92652472,"statfs":{"total":95995035648,"available":94876131328,"internally_reserved":0,"allocated":669974528,"data_stored":669254526,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":448528384},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:17:34 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":4.8099999999999996,"15min":3.9239999999999999},"min":{"1min":0.23000000000000001,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":3.5760000000000001,"5min":216.40899999999999,"15min":230.249},"last":0.434},{"interface":"front","average":{"1min":1.758,"5min":5.3010000000000002,"15min":4.1420000000000003},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.24199999999999999},"max":{"1min":10.398999999999999,"5min":216.43700000000001,"15min":230.261},"last":0.51800000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:17:56 2023","interfaces":[{"interface":"back","average":{"1min":2.6150000000000002,"5min":2.9700000000000002,"15min":4.2359999999999998},"min":{"1min":0.39100000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.798,"5min":121.39400000000001,"15min":121.39400000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":2.2989999999999999,"5min":2.9790000000000001,"15min":4.077},"min":{"1min":0.27000000000000002,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":10.034000000000001,"5min":121.381,"15min":121.381},"last":0.39800000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0339999999999998,"5min":2.0179999999999998,"15min":2.0150000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.375,"5min":9.5589999999999993,"15min":9.5589999999999993},"last":0.59299999999999997},{"interface":"front","average":{"1min":2.5600000000000001,"5min":1.8580000000000001,"15min":1.7410000000000001},"min":{"1min":0.36899999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.561999999999999,"5min":11.561999999999999,"15min":11.561999999999999},"last":0.46999999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:18:11 2023","interfaces":[{"interface":"back","average":{"1min":3.1400000000000001,"5min":5.7140000000000004,"15min":3.4620000000000002},"min":{"1min":0.27800000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":17.724,"5min":144.50299999999999,"15min":144.50299999999999},"last":9.5380000000000003},{"interface":"front","average":{"1min":2.6829999999999998,"5min":5.5990000000000002,"15min":3.468},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":17.712,"5min":144.48699999999999,"15min":144.48699999999999},"last":9.5250000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":0.83099999999999996,"15min":1.3200000000000001},"min":{"1min":0.27300000000000002,"5min":0.254,"15min":0.23599999999999999},"max":{"1min":7.0570000000000004,"5min":7.0570000000000004,"15min":92.781000000000006},"last":0.505},{"interface":"front","average":{"1min":1.208,"5min":1.2110000000000001,"15min":1.4219999999999999},"min":{"1min":0.29499999999999998,"5min":0.28599999999999998,"15min":0.254},"max":{"1min":3.9569999999999999,"5min":9.7530000000000001,"15min":92.626999999999995},"last":0.55400000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":13.24,"5min":3.488,"15min":2.3759999999999999},"min":{"1min":0.28799999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":185.321,"5min":185.321,"15min":185.321},"last":0.60899999999999999},{"interface":"front","average":{"1min":11.488,"5min":2.851,"15min":1.97},"min":{"1min":0.33900000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":185.47499999999999,"5min":185.47499999999999,"15min":185.47499999999999},"last":0.56999999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":1.292,"15min":1.4970000000000001},"min":{"1min":0.376,"5min":0.30199999999999999,"15min":0.27300000000000002},"max":{"1min":19.782,"5min":19.782,"15min":93.105999999999995},"last":1.47},{"interface":"front","average":{"1min":2.7349999999999999,"5min":1.492,"15min":1.599},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28399999999999997},"max":{"1min":20.064,"5min":20.064,"15min":93.198999999999998},"last":0.71699999999999997}]}]},{"osd":6,"up_from":46,"seq":197568495945,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":459024,"kb_used_data":131912,"kb_used_omap":0,"kb_used_meta":327104,"kb_avail":93286128,"statfs":{"total":95995035648,"available":95524995072,"internally_reserved":0,"allocated":135077888,"data_stored":134350551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":334954496},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:17:46 2023","interfaces":[{"interface":"back","average":{"1min":15.551,"5min":6.1399999999999997,"15min":4.1079999999999997},"min":{"1min":0.317,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":161.64599999999999,"5min":161.64599999999999,"15min":161.64599999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":14.704000000000001,"5min":6.3449999999999998,"15min":4.484},"min":{"1min":0.26500000000000001,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":161.38499999999999,"5min":161.38499999999999,"15min":161.38499999999999},"last":1.242}]},{"osd":1,"last update":"Sat Sep 23 23:17:46 2023","interfaces":[{"interface":"back","average":{"1min":18.859999999999999,"5min":6.5510000000000002,"15min":3.9009999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.24099999999999999},"max":{"1min":210.869,"5min":210.869,"15min":210.869},"last":0.56799999999999995},{"interface":"front","average":{"1min":18.619,"5min":6.726,"15min":4.1630000000000003},"min":{"1min":0.36399999999999999,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":210.89699999999999,"5min":210.89699999999999,"15min":210.89699999999999},"last":1.278}]},{"osd":2,"last update":"Sat Sep 23 23:18:22 2023","interfaces":[{"interface":"back","average":{"1min":3.3279999999999998,"5min":2.0960000000000001,"15min":1.891},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":11.804,"5min":11.804,"15min":11.804},"last":2.6629999999999998},{"interface":"front","average":{"1min":2.819,"5min":1.575,"15min":1.3680000000000001},"min":{"1min":0.32400000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.685,"5min":10.685,"15min":10.685},"last":2.7229999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:18:15 2023","interfaces":[{"interface":"back","average":{"1min":2.8220000000000001,"5min":5.0049999999999999,"15min":3.327},"min":{"1min":0.35699999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.452,"5min":109.98699999999999,"15min":109.98699999999999},"last":10.528},{"interface":"front","average":{"1min":2.512,"5min":4.7549999999999999,"15min":3.5350000000000001},"min":{"1min":0.33400000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":14.573,"5min":110.024,"15min":110.024},"last":0.81499999999999995}]},{"osd":4,"last update":"Sat Sep 23 23:18:01 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.76100000000000001,"15min":1.0569999999999999},"min":{"1min":0.32500000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.7120000000000002,"5min":5.306,"15min":11.890000000000001},"last":1.302},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.84999999999999998,"15min":1.111},"min":{"1min":0.32200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.7590000000000003,"5min":5.7590000000000003,"15min":11.728},"last":10.569000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:18:01 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":0.89600000000000002,"15min":1.266},"min":{"1min":0.35999999999999999,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":2.7400000000000002,"5min":6.8310000000000004,"15min":26.158999999999999},"last":3.6699999999999999},{"interface":"front","average":{"1min":1.014,"5min":0.82299999999999995,"15min":1.1339999999999999},"min":{"1min":0.35099999999999998,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":3.9380000000000002,"5min":4.4080000000000004,"15min":25.786000000000001},"last":10.593999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:18:28 2023","interfaces":[{"interface":"back","average":{"1min":2.7250000000000001,"5min":1.2689999999999999,"15min":1.27},"min":{"1min":0.36799999999999999,"5min":0.32300000000000001,"15min":0.32100000000000001},"max":{"1min":13.823,"5min":13.823,"15min":13.823},"last":10.627000000000001},{"interface":"front","average":{"1min":1.5609999999999999,"5min":0.79900000000000004,"15min":0.98799999999999999},"min":{"1min":0.35999999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":13.103,"5min":13.103,"15min":13.103},"last":0.55200000000000005}]}]},{"osd":1,"up_from":112,"seq":481036337286,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":821672,"kb_used_data":385440,"kb_used_omap":4,"kb_used_meta":436219,"kb_avail":92923480,"statfs":{"total":95995035648,"available":95153643520,"internally_reserved":0,"allocated":394690560,"data_stored":393648182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":446689166},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:17:39 2023","interfaces":[{"interface":"back","average":{"1min":4.6459999999999999,"5min":2.3279999999999998,"15min":1.95},"min":{"1min":0.26200000000000001,"5min":0.20100000000000001,"15min":0.17899999999999999},"max":{"1min":18.754000000000001,"5min":25.771000000000001,"15min":25.771000000000001},"last":0.70399999999999996},{"interface":"front","average":{"1min":6.8929999999999998,"5min":3.109,"15min":2.0649999999999999},"min":{"1min":0.46899999999999997,"5min":0.192,"15min":0.192},"max":{"1min":20.373999999999999,"5min":20.373999999999999,"15min":20.373999999999999},"last":0.317}]},{"osd":2,"last update":"Sat Sep 23 23:18:20 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":5.7350000000000003,"15min":6.234},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":12.506,"5min":25.744,"15min":25.744},"last":0.68000000000000005},{"interface":"front","average":{"1min":5.0990000000000002,"5min":4.0250000000000004,"15min":3.8460000000000001},"min":{"1min":0.27500000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":25.998999999999999,"5min":25.998999999999999,"15min":25.998999999999999},"last":0.79800000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:18:15 2023","interfaces":[{"interface":"back","average":{"1min":9.0220000000000002,"5min":5.5609999999999999,"15min":3.516},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":45.584000000000003,"5min":70.956999999999994,"15min":70.956999999999994},"last":0.47999999999999998},{"interface":"front","average":{"1min":9.9030000000000005,"5min":5.3869999999999996,"15min":3.2090000000000001},"min":{"1min":0.317,"5min":0.307,"15min":0.26800000000000002},"max":{"1min":44.646000000000001,"5min":70.921000000000006,"15min":70.921000000000006},"last":0.93500000000000005}]},{"osd":4,"last update":"Sat Sep 23 23:17:39 2023","interfaces":[{"interface":"back","average":{"1min":2.6949999999999998,"5min":1.7310000000000001,"15min":1.6599999999999999},"min":{"1min":0.312,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":16.748999999999999,"5min":16.748999999999999,"15min":16.748999999999999},"last":2.464},{"interface":"front","average":{"1min":3.1019999999999999,"5min":2.2549999999999999,"15min":1.804},"min":{"1min":0.317,"5min":0.317,"15min":0.29099999999999998},"max":{"1min":13.177,"5min":25.710999999999999,"15min":25.710999999999999},"last":0.629}]},{"osd":5,"last update":"Sat Sep 23 23:17:39 2023","interfaces":[{"interface":"back","average":{"1min":4.3099999999999996,"5min":2.4929999999999999,"15min":3.2389999999999999},"min":{"1min":0.377,"5min":0.318,"15min":0.26400000000000001},"max":{"1min":20.344999999999999,"5min":20.344999999999999,"15min":67.497},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.8460000000000001,"5min":1.7849999999999999,"15min":3.056},"min":{"1min":0.36399999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":16.818000000000001,"5min":16.818000000000001,"15min":67.540999999999997},"last":0.73999999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:17:39 2023","interfaces":[{"interface":"back","average":{"1min":4.4989999999999997,"5min":2.6040000000000001,"15min":1.9359999999999999},"min":{"1min":0.41799999999999998,"5min":0.35199999999999998,"15min":0.33500000000000002},"max":{"1min":20.289999999999999,"5min":20.289999999999999,"15min":20.289999999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":2.7949999999999999,"5min":1.8680000000000001,"15min":1.841},"min":{"1min":0.36699999999999999,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":16.843,"5min":16.843,"15min":16.843},"last":0.81899999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:17:39 2023","interfaces":[{"interface":"back","average":{"1min":2.8340000000000001,"5min":1.9650000000000001,"15min":1.8879999999999999},"min":{"1min":0.44,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":16.789000000000001,"5min":16.789000000000001,"15min":16.789000000000001},"last":0.77500000000000002},{"interface":"front","average":{"1min":2.9710000000000001,"5min":2.1269999999999998,"15min":1.825},"min":{"1min":0.35499999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":13.140000000000001,"5min":25.667000000000002,"15min":25.667000000000002},"last":0.65700000000000003}]}]},{"osd":0,"up_from":91,"seq":390842024110,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":575976,"kb_used_data":80416,"kb_used_omap":5,"kb_used_meta":495546,"kb_avail":93169176,"statfs":{"total":95995035648,"available":95405236224,"internally_reserved":0,"allocated":82345984,"data_stored":81301717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":507439738},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:17:49 2023","interfaces":[{"interface":"back","average":{"1min":5.1079999999999997,"5min":5.1630000000000003,"15min":3.3130000000000002},"min":{"1min":0.19700000000000001,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":26.754999999999999,"5min":155.91999999999999,"15min":155.91999999999999},"last":0.29899999999999999},{"interface":"front","average":{"1min":3.2690000000000001,"5min":4.2839999999999998,"15min":2.9409999999999998},"min":{"1min":0.20399999999999999,"5min":0.19,"15min":0.189},"max":{"1min":9.0280000000000005,"5min":153.28899999999999,"15min":153.28899999999999},"last":0.56599999999999995}]},{"osd":2,"last update":"Sat Sep 23 23:18:22 2023","interfaces":[{"interface":"back","average":{"1min":3.4359999999999999,"5min":3.1970000000000001,"15min":3.157},"min":{"1min":0.23000000000000001,"5min":0.222,"15min":0.222},"max":{"1min":20.259,"5min":20.495999999999999,"15min":20.495999999999999},"last":0.749},{"interface":"front","average":{"1min":3.2999999999999998,"5min":4.0069999999999997,"15min":4.125},"min":{"1min":0.23699999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":15.061999999999999,"5min":36.383000000000003,"15min":36.383000000000003},"last":0.499}]},{"osd":3,"last update":"Sat Sep 23 23:18:06 2023","interfaces":[{"interface":"back","average":{"1min":5.6840000000000002,"5min":4.6779999999999999,"15min":3.677},"min":{"1min":0.26400000000000001,"5min":0.23200000000000001,"15min":0.22700000000000001},"max":{"1min":28.995000000000001,"5min":82.691000000000003,"15min":82.691000000000003},"last":0.84599999999999997},{"interface":"front","average":{"1min":5.2309999999999999,"5min":3.8439999999999999,"15min":2.8149999999999999},"min":{"1min":0.43099999999999999,"5min":0.26100000000000001,"15min":0.24199999999999999},"max":{"1min":43.006999999999998,"5min":82.429000000000002,"15min":82.429000000000002},"last":0.54100000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:17:31 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.2709999999999999,"15min":1.9450000000000001},"min":{"1min":0.318,"5min":0.27900000000000003,"15min":0.245},"max":{"1min":6.0999999999999996,"5min":7.0759999999999996,"15min":28.376999999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":3.1720000000000002,"5min":1.581,"15min":2.3090000000000002},"min":{"1min":0.34399999999999997,"5min":0.32700000000000001,"15min":0.29099999999999998},"max":{"1min":13.787000000000001,"5min":19.167000000000002,"15min":30.706},"last":3.7890000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:17:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6179999999999999,"5min":1.6040000000000001,"15min":2.3340000000000001},"min":{"1min":0.40799999999999997,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":13.832000000000001,"5min":13.832000000000001,"15min":30.873999999999999},"last":4.6500000000000004},{"interface":"front","average":{"1min":1.6100000000000001,"5min":1.4750000000000001,"15min":1.857},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29499999999999998},"max":{"1min":5.1790000000000003,"5min":7.9000000000000004,"15min":10.117000000000001},"last":0.80800000000000005}]},{"osd":6,"last update":"Sat Sep 23 23:17:31 2023","interfaces":[{"interface":"back","average":{"1min":3.1970000000000001,"5min":1.6799999999999999,"15min":2.3809999999999998},"min":{"1min":0.378,"5min":0.314,"15min":0.314},"max":{"1min":13.859999999999999,"5min":13.859999999999999,"15min":21.047999999999998},"last":4.6130000000000004},{"interface":"front","average":{"1min":1.829,"5min":1.673,"15min":2.0089999999999999},"min":{"1min":0.377,"5min":0.36499999999999999,"15min":0.30499999999999999},"max":{"1min":5.1459999999999999,"5min":8.0419999999999998,"15min":10.414},"last":0.82799999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:17:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":1.615,"15min":2.0129999999999999},"min":{"1min":0.35599999999999998,"5min":0.34499999999999997,"15min":0.29299999999999998},"max":{"1min":6.9340000000000002,"5min":8.4849999999999994,"15min":10.145},"last":0.78000000000000003},{"interface":"front","average":{"1min":1.5649999999999999,"5min":1.4119999999999999,"15min":1.9490000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.313},"max":{"1min":6.0750000000000002,"5min":7.9480000000000004,"15min":10.513999999999999},"last":0.625}]}]},{"osd":2,"up_from":0,"seq":854698491931,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":148,"seq":635655159900,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":607972,"kb_used_data":297116,"kb_used_omap":2,"kb_used_meta":310589,"kb_avail":93137180,"statfs":{"total":95995035648,"available":95372472320,"internally_reserved":0,"allocated":304246784,"data_stored":303264564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":318043189},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:18:09 2023","interfaces":[{"interface":"back","average":{"1min":2.9399999999999999,"5min":4.1130000000000004,"15min":2.2330000000000001},"min":{"1min":0.20300000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":21.427,"5min":72.778999999999996,"15min":72.778999999999996},"last":0.21099999999999999},{"interface":"front","average":{"1min":4.6799999999999997,"5min":3.9710000000000001,"15min":2.6560000000000001},"min":{"1min":0.27100000000000002,"5min":0.186,"15min":0.17999999999999999},"max":{"1min":23.742999999999999,"5min":72.719999999999999,"15min":72.719999999999999},"last":0.40300000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:18:09 2023","interfaces":[{"interface":"back","average":{"1min":5.0670000000000002,"5min":4.6790000000000003,"15min":2.915},"min":{"1min":0.23499999999999999,"5min":0.215,"15min":0.19600000000000001},"max":{"1min":20.138999999999999,"5min":76.322999999999993,"15min":76.322999999999993},"last":3.4350000000000001},{"interface":"front","average":{"1min":3.4580000000000002,"5min":4.5250000000000004,"15min":2.7029999999999998},"min":{"1min":0.25900000000000001,"5min":0.20499999999999999,"15min":0.185},"max":{"1min":13.112,"5min":76.444000000000003,"15min":76.444000000000003},"last":1.605}]},{"osd":2,"last update":"Sat Sep 23 23:18:25 2023","interfaces":[{"interface":"back","average":{"1min":2.7959999999999998,"5min":3.556,"15min":3.6829999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.5359999999999996,"5min":18.742999999999999,"15min":18.742999999999999},"last":3.6699999999999999},{"interface":"front","average":{"1min":2.331,"5min":1.8220000000000001,"15min":1.7370000000000001},"min":{"1min":0.35399999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":14.711,"5min":14.711,"15min":14.711},"last":3.6469999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:18:09 2023","interfaces":[{"interface":"back","average":{"1min":2.6080000000000001,"5min":1.9219999999999999,"15min":1.7230000000000001},"min":{"1min":0.373,"5min":0.315,"15min":0.29999999999999999},"max":{"1min":14.754,"5min":14.754,"15min":14.754},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.5470000000000002,"5min":1.403,"15min":2.4990000000000001},"min":{"1min":0.38400000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.4299999999999997,"5min":8.0239999999999991,"15min":13.532999999999999},"last":0.503}]},{"osd":5,"last update":"Sat Sep 23 23:18:09 2023","interfaces":[{"interface":"back","average":{"1min":2.722,"5min":2.2469999999999999,"15min":2.476},"min":{"1min":0.32900000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.3920000000000003,"5min":75.733000000000004,"15min":75.733000000000004},"last":0.34399999999999997},{"interface":"front","average":{"1min":2.879,"5min":2.8069999999999999,"15min":2.3010000000000002},"min":{"1min":0.42099999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.3460000000000001,"5min":75.686999999999998,"15min":75.686999999999998},"last":0.46400000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:18:09 2023","interfaces":[{"interface":"back","average":{"1min":2.73,"5min":1.5640000000000001,"15min":2.3340000000000001},"min":{"1min":0.38800000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.4870000000000001,"5min":8.0549999999999997,"15min":14.492000000000001},"last":0.437},{"interface":"front","average":{"1min":2.669,"5min":1.821,"15min":2.089},"min":{"1min":0.42299999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":8.3889999999999993,"5min":9.8379999999999992,"15min":9.8379999999999992},"last":1.24}]},{"osd":7,"last update":"Sat Sep 23 23:18:09 2023","interfaces":[{"interface":"back","average":{"1min":2.198,"5min":2.915,"15min":2.5459999999999998},"min":{"1min":0.40200000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":9.4280000000000008,"5min":130.839,"15min":130.839},"last":1.4279999999999999},{"interface":"front","average":{"1min":2.9129999999999998,"5min":2.992,"15min":2.665},"min":{"1min":0.434,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":8.8190000000000008,"5min":130.90000000000001,"15min":130.90000000000001},"last":0.48099999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019218,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":520088,"kb_used_data":123920,"kb_used_omap":0,"kb_used_meta":396160,"kb_avail":93225064,"statfs":{"total":95995035648,"available":95462465536,"internally_reserved":0,"allocated":126894080,"data_stored":126158627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":405667840},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:17:34 2023","interfaces":[{"interface":"back","average":{"1min":2.7149999999999999,"5min":2.8900000000000001,"15min":3.004},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20799999999999999},"max":{"1min":15.071999999999999,"5min":102.12,"15min":238.62200000000001},"last":0.70599999999999996},{"interface":"front","average":{"1min":2.2629999999999999,"5min":2.694,"15min":3.3799999999999999},"min":{"1min":0.251,"5min":0.23499999999999999,"15min":0.23300000000000001},"max":{"1min":19.942,"5min":104.46899999999999,"15min":238.58199999999999},"last":6.1790000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:17:52 2023","interfaces":[{"interface":"back","average":{"1min":2.6539999999999999,"5min":6.5869999999999997,"15min":4.1600000000000001},"min":{"1min":0.253,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":12.348000000000001,"5min":113.25,"15min":175.00899999999999},"last":5.1059999999999999},{"interface":"front","average":{"1min":3.0699999999999998,"5min":6.7699999999999996,"15min":4.2519999999999998},"min":{"1min":0.28199999999999997,"5min":0.26200000000000001,"15min":0.25800000000000001},"max":{"1min":9.4779999999999998,"5min":113.321,"15min":174.821},"last":6.21}]},{"osd":2,"last update":"Sat Sep 23 23:18:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2909999999999999,"5min":3.2810000000000001,"15min":3.4460000000000002},"min":{"1min":0.318,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":10.75,"5min":39.308,"15min":39.308},"last":5.2130000000000001},{"interface":"front","average":{"1min":2.4929999999999999,"5min":2.625,"15min":2.6469999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":7.8490000000000002,"5min":14.287000000000001,"15min":14.287000000000001},"last":0.65200000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:18:16 2023","interfaces":[{"interface":"back","average":{"1min":9.952,"5min":9.3520000000000003,"15min":4.2519999999999998},"min":{"1min":0.29299999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":152.04499999999999,"5min":303.41500000000002,"15min":303.41500000000002},"last":5.2370000000000001},{"interface":"front","average":{"1min":9.4979999999999993,"5min":9.3469999999999995,"15min":4.593},"min":{"1min":0.317,"5min":0.311,"15min":0.311},"max":{"1min":151.75299999999999,"5min":303.428,"15min":303.428},"last":6.0960000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:17:52 2023","interfaces":[{"interface":"back","average":{"1min":8.7690000000000001,"5min":2.4449999999999998,"15min":1.411},"min":{"1min":0.29899999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":160.79499999999999,"5min":160.79499999999999,"15min":160.79499999999999},"last":5.1440000000000001},{"interface":"front","average":{"1min":8.6760000000000002,"5min":2.6070000000000002,"15min":1.7230000000000001},"min":{"1min":0.317,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":160.61099999999999,"5min":160.61099999999999,"15min":160.61099999999999},"last":6.2690000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:18:16 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":1.145,"15min":1.1419999999999999},"min":{"1min":0.311,"5min":0.28000000000000003,"15min":0.27200000000000002},"max":{"1min":4.569,"5min":7.5499999999999998,"15min":14.4},"last":5.181},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.119,"15min":1.091},"min":{"1min":0.36399999999999999,"5min":0.315,"15min":0.27500000000000002},"max":{"1min":5.4950000000000001,"5min":7.4459999999999997,"15min":13.888999999999999},"last":0.72599999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:17:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.3640000000000001,"15min":1.264},"min":{"1min":0.36199999999999999,"5min":0.315,"15min":0.30499999999999999},"max":{"1min":6.5640000000000001,"5min":14.842000000000001,"15min":14.842000000000001},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.875,"5min":1.222,"15min":1.319},"min":{"1min":0.379,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.3650000000000002,"5min":14.561,"15min":14.561},"last":0.752}]}]},{"osd":5,"up_from":38,"seq":163208757581,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":987200,"kb_used_data":537208,"kb_used_omap":0,"kb_used_meta":449920,"kb_avail":92757952,"statfs":{"total":95995035648,"available":94984142848,"internally_reserved":0,"allocated":550100992,"data_stored":549385002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":460718080},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:17:32 2023","interfaces":[{"interface":"back","average":{"1min":2.7599999999999998,"5min":2.254,"15min":3.681},"min":{"1min":0.24399999999999999,"5min":0.23000000000000001,"15min":0.21099999999999999},"max":{"1min":15.057,"5min":97.405000000000001,"15min":190.483},"last":3.569},{"interface":"front","average":{"1min":1.663,"5min":2.278,"15min":4.2270000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.369999999999999,"5min":97.447000000000003,"15min":194.11099999999999},"last":0.33600000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:17:48 2023","interfaces":[{"interface":"back","average":{"1min":5.7270000000000003,"5min":6.1689999999999996,"15min":3.3700000000000001},"min":{"1min":0.27400000000000002,"5min":0.25,"15min":0.246},"max":{"1min":91.518000000000001,"5min":193.584,"15min":193.584},"last":3.25},{"interface":"front","average":{"1min":4.9870000000000001,"5min":6.226,"15min":3.5070000000000001},"min":{"1min":0.30199999999999999,"5min":0.27800000000000002,"15min":0.26400000000000001},"max":{"1min":87.718000000000004,"5min":194.03100000000001,"15min":194.03100000000001},"last":9.2210000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:18:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0899999999999999,"5min":1.579,"15min":1.494},"min":{"1min":0.40200000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.5700000000000003,"5min":6.5700000000000003,"15min":6.5700000000000003},"last":5.3380000000000001},{"interface":"front","average":{"1min":1.893,"5min":1.7629999999999999,"15min":1.7410000000000001},"min":{"1min":0.35299999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.6040000000000001,"5min":9.391,"15min":9.391},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:18:12 2023","interfaces":[{"interface":"back","average":{"1min":10.567,"5min":7.9459999999999997,"15min":3.8180000000000001},"min":{"1min":0.35199999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":159.53299999999999,"5min":159.53299999999999,"15min":159.53299999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":11.516,"5min":8.2590000000000003,"15min":4.6849999999999996},"min":{"1min":0.505,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":159.54900000000001,"5min":159.54900000000001,"15min":159.54900000000001},"last":0.59299999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:18:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6579999999999999,"5min":1.1140000000000001,"15min":1.2030000000000001},"min":{"1min":0.25800000000000001,"5min":0.23999999999999999,"15min":0.221},"max":{"1min":7.3399999999999999,"5min":10.012,"15min":14.538},"last":5.1280000000000001},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.79100000000000004,"15min":0.874},"min":{"1min":0.314,"5min":0.26100000000000001,"15min":0.25},"max":{"1min":11.923999999999999,"5min":11.923999999999999,"15min":11.923999999999999},"last":0.31900000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:18:00 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":0.92600000000000005,"15min":1.113},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.255},"max":{"1min":5.2370000000000001,"5min":5.9809999999999999,"15min":13.226000000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.829,"5min":1.1639999999999999,"15min":1.264},"min":{"1min":0.32000000000000001,"5min":0.307,"15min":0.29099999999999998},"max":{"1min":12.178000000000001,"5min":12.178000000000001,"15min":13.098000000000001},"last":1.085}]},{"osd":7,"last update":"Sat Sep 23 23:17:41 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.97399999999999998,"15min":1.3029999999999999},"min":{"1min":0.32900000000000001,"5min":0.32700000000000001,"15min":0.29199999999999998},"max":{"1min":4.4560000000000004,"5min":4.9050000000000002,"15min":14.929},"last":0.497},{"interface":"front","average":{"1min":1.131,"5min":1.1160000000000001,"15min":1.226},"min":{"1min":0.315,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":6.4950000000000001,"5min":6.7089999999999996,"15min":15.090999999999999},"last":5.7350000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":74514432,"data_stored":74514432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":373686272,"data_stored":373686272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":97648640,"data_stored":97648640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":297992192,"data_stored":297992192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":113446912,"data_stored":113442840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":529928192,"data_stored":529924120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":117116928,"data_stored":117112856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":640679936,"data_stored":640679936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:18:35.106 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-09-23T23:18:35.117 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:18:35.184 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:35.273 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:18:35.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:35.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:18:35.382 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:35.454 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:35.527 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:35.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:35.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:35.607 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:18:35.635 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:18:35.664 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:18:35.691 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:35.701 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:18:35.756 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:18:35.787 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:18:35.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:35.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:18:35.805 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:18:35.868 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:18:35.903 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:18:35.932 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:18:35.968 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:18:35.971 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:18:35.971 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:18:35.971 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:18:35.972 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:18:35.972 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:18:35.972 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:18:35.972 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:18:35.972 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:18:35.972 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Removed 111178 objects 2023-09-23T23:18:35.972 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Clean up completed and total clean up time :210.398 2023-09-23T23:18:35.978 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:36.530 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:18:36.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:36.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:18:36.634 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:18:36.664 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:36.712 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:18:36.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:36.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:18:36.801 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:36 smithi110 ceph-mon[212939]: pgmap v771: 99 pgs: 1 active+recovery_wait+remapped, 94 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 118 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 682/5715 objects misplaced (11.934%); 512 KiB/s, 47 objects/s recovering 2023-09-23T23:18:36.801 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:36 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1669406339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:36.802 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:36 smithi110 ceph-mon[221102]: pgmap v771: 99 pgs: 1 active+recovery_wait+remapped, 94 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 118 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 682/5715 objects misplaced (11.934%); 512 KiB/s, 47 objects/s recovering 2023-09-23T23:18:36.802 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:36 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1669406339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:36.819 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:36.849 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:36.900 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:36.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:36.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:36.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:36 smithi138 ceph-mon[193606]: pgmap v771: 99 pgs: 1 active+recovery_wait+remapped, 94 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 118 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 682/5715 objects misplaced (11.934%); 512 KiB/s, 47 objects/s recovering 2023-09-23T23:18:36.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:36 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1669406339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:36.989 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:18:37.004 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:18:37.057 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:18:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:18:36] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.43.0" 2023-09-23T23:18:37.059 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:37.757 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:18:37.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:37.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:18:37.856 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:37.861 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:18:37.890 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:37 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-09-23T23:18:37.893 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:37.941 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:18:37.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:37.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:18:37.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:37 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-09-23T23:18:37.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:37 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-09-23T23:18:38.005 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:38.048 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:18:38.074 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:38.120 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:18:38.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:38.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:18:38.226 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:18:38.255 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:38.297 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:18:38.297 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:18:38.297 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:18:38.297 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:18:38.297 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:18:38.592 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:18:38.594 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:38 smithi110 ceph-mon[212939]: pgmap v772: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 69 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 711 op/s; 291/3378 objects misplaced (8.615%); 0 B/s, 61 objects/s recovering 2023-09-23T23:18:38.594 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:38 smithi110 ceph-mon[212939]: osdmap e250: 8 total, 8 up, 7 in 2023-09-23T23:18:38.594 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:38 smithi110 ceph-mon[221102]: pgmap v772: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 69 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 711 op/s; 291/3378 objects misplaced (8.615%); 0 B/s, 61 objects/s recovering 2023-09-23T23:18:38.594 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:38 smithi110 ceph-mon[221102]: osdmap e250: 8 total, 8 up, 7 in 2023-09-23T23:18:38.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:38.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:18:38.696 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:18:38.723 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:38.765 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:18:38.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:38.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:18:38.873 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:18:38.897 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:38 smithi138 ceph-mon[193606]: pgmap v772: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 94 active+clean, 3 active+remapped+backfill_wait; 69 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 711 op/s; 291/3378 objects misplaced (8.615%); 0 B/s, 61 objects/s recovering 2023-09-23T23:18:38.897 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:38 smithi138 ceph-mon[193606]: osdmap e250: 8 total, 8 up, 7 in 2023-09-23T23:18:38.899 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:38.941 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:18:38.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:38.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:18:39.045 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:18:39.070 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:39.112 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:18:39.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:39.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:18:39.217 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:18:39.247 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:39.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:39 smithi110 ceph-mon[212939]: failed to encode map e250 with expected crc 2023-09-23T23:18:39.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:39 smithi110 ceph-mon[212939]: failed to encode map e250 with expected crc 2023-09-23T23:18:39.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:39 smithi110 ceph-mon[212939]: failed to encode map e250 with expected crc 2023-09-23T23:18:39.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:39 smithi110 ceph-mon[212939]: failed to encode map e250 with expected crc 2023-09-23T23:18:39.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:39 smithi110 ceph-mon[212939]: osdmap e251: 8 total, 8 up, 7 in 2023-09-23T23:18:39.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:39 smithi110 ceph-mon[221102]: failed to encode map e250 with expected crc 2023-09-23T23:18:39.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:39 smithi110 ceph-mon[221102]: failed to encode map e250 with expected crc 2023-09-23T23:18:39.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:39 smithi110 ceph-mon[221102]: failed to encode map e250 with expected crc 2023-09-23T23:18:39.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:39 smithi110 ceph-mon[221102]: failed to encode map e250 with expected crc 2023-09-23T23:18:39.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:39 smithi110 ceph-mon[221102]: osdmap e251: 8 total, 8 up, 7 in 2023-09-23T23:18:39.772 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:18:39.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:39 smithi138 ceph-mon[193606]: failed to encode map e250 with expected crc 2023-09-23T23:18:39.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:39 smithi138 ceph-mon[193606]: failed to encode map e250 with expected crc 2023-09-23T23:18:39.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:39 smithi138 ceph-mon[193606]: failed to encode map e250 with expected crc 2023-09-23T23:18:39.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:39 smithi138 ceph-mon[193606]: failed to encode map e250 with expected crc 2023-09-23T23:18:39.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:39 smithi138 ceph-mon[193606]: osdmap e251: 8 total, 8 up, 7 in 2023-09-23T23:18:40.149 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:18:40.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:40.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:18:40.178 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:18:40.254 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:40.285 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:40.335 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:40.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:40.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:40.440 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:18:40.466 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:40.769 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:18:40.771 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:40 smithi110 ceph-mon[212939]: failed to encode map e251 with expected crc 2023-09-23T23:18:40.771 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:40 smithi110 ceph-mon[212939]: failed to encode map e251 with expected crc 2023-09-23T23:18:40.771 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:40 smithi110 ceph-mon[212939]: failed to encode map e251 with expected crc 2023-09-23T23:18:40.771 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:40 smithi110 ceph-mon[212939]: failed to encode map e251 with expected crc 2023-09-23T23:18:40.771 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:40 smithi110 ceph-mon[212939]: pgmap v775: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 94 active+clean, 2 active+remapped+backfill_wait; 483 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 0 B/s, 48 objects/s recovering 2023-09-23T23:18:40.772 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:40 smithi110 ceph-mon[221102]: failed to encode map e251 with expected crc 2023-09-23T23:18:40.772 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:40 smithi110 ceph-mon[221102]: failed to encode map e251 with expected crc 2023-09-23T23:18:40.772 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:40 smithi110 ceph-mon[221102]: failed to encode map e251 with expected crc 2023-09-23T23:18:40.772 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:40 smithi110 ceph-mon[221102]: failed to encode map e251 with expected crc 2023-09-23T23:18:40.772 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:40 smithi110 ceph-mon[221102]: pgmap v775: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 94 active+clean, 2 active+remapped+backfill_wait; 483 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 0 B/s, 48 objects/s recovering 2023-09-23T23:18:40.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:40.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:18:40.873 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:18:40.902 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:40 smithi138 ceph-mon[193606]: failed to encode map e251 with expected crc 2023-09-23T23:18:40.902 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:40 smithi138 ceph-mon[193606]: failed to encode map e251 with expected crc 2023-09-23T23:18:40.903 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:40 smithi138 ceph-mon[193606]: failed to encode map e251 with expected crc 2023-09-23T23:18:40.903 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:40 smithi138 ceph-mon[193606]: failed to encode map e251 with expected crc 2023-09-23T23:18:40.903 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:40 smithi138 ceph-mon[193606]: pgmap v775: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 94 active+clean, 2 active+remapped+backfill_wait; 483 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 0 B/s, 48 objects/s recovering 2023-09-23T23:18:40.905 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:40.954 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:18:40.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:40.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:18:41.059 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:18:41.087 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:41.132 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:18:41.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:41.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:18:41.163 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:41.237 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:41.263 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:41.305 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:41.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:41.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:41.410 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:41.442 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:41.466 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:41.579 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:18:41.732 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:41.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:41.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:41.836 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:41 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/31990712' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:18:41.836 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:41 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:18:41.836 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:41 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/31990712' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:18:41.836 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:41 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:18:41.837 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:18:41.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:41 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/31990712' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:18:41.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:41 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:18:41.863 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:41.903 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:18:41.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:41.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:18:42.008 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:42.037 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:42.587 INFO:teuthology.orchestra.run.smithi110.stderr:set pool 9 pgp_num to 16 2023-09-23T23:18:42.784 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:42.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:42.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:42.888 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:18:42.913 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[212939]: pgmap v776: 99 pgs: 2 peering, 97 active+clean; 483 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 0 B/s, 79 objects/s recovering 2023-09-23T23:18:42.913 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:18:42.913 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[212939]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-09-23T23:18:42.913 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[212939]: failed to encode map e252 with expected crc 2023-09-23T23:18:42.913 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:18:42.913 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[212939]: osdmap e252: 8 total, 8 up, 7 in 2023-09-23T23:18:42.914 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1148511286' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T23:18:42.914 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T23:18:42.914 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[212939]: 1.0 scrub starts 2023-09-23T23:18:42.914 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[212939]: 1.0 scrub ok 2023-09-23T23:18:42.914 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[221102]: pgmap v776: 99 pgs: 2 peering, 97 active+clean; 483 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 0 B/s, 79 objects/s recovering 2023-09-23T23:18:42.914 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:18:42.914 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[221102]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-09-23T23:18:42.914 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[221102]: failed to encode map e252 with expected crc 2023-09-23T23:18:42.914 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:18:42.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[221102]: osdmap e252: 8 total, 8 up, 7 in 2023-09-23T23:18:42.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1148511286' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T23:18:42.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T23:18:42.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[221102]: 1.0 scrub starts 2023-09-23T23:18:42.915 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:42 smithi110 ceph-mon[221102]: 1.0 scrub ok 2023-09-23T23:18:42.916 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:42.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:42 smithi138 ceph-mon[193606]: pgmap v776: 99 pgs: 2 peering, 97 active+clean; 483 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 0 B/s, 79 objects/s recovering 2023-09-23T23:18:42.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:42 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:18:42.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:42 smithi138 ceph-mon[193606]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-09-23T23:18:42.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:42 smithi138 ceph-mon[193606]: failed to encode map e252 with expected crc 2023-09-23T23:18:42.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:42 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:18:42.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:42 smithi138 ceph-mon[193606]: osdmap e252: 8 total, 8 up, 7 in 2023-09-23T23:18:42.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:42 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1148511286' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T23:18:42.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:42 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T23:18:42.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:42 smithi138 ceph-mon[193606]: 1.0 scrub starts 2023-09-23T23:18:42.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:42 smithi138 ceph-mon[193606]: 1.0 scrub ok 2023-09-23T23:18:43.010 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:18:43.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:43.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:18:43.114 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:18:43.140 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:43 smithi138 ceph-mon[193606]: failed to encode map e252 with expected crc 2023-09-23T23:18:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:43 smithi138 ceph-mon[193606]: failed to encode map e252 with expected crc 2023-09-23T23:18:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:43 smithi138 ceph-mon[193606]: failed to encode map e252 with expected crc 2023-09-23T23:18:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:43 smithi138 ceph-mon[193606]: failed to encode map e253 with expected crc 2023-09-23T23:18:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:43 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T23:18:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:43 smithi138 ceph-mon[193606]: osdmap e253: 8 total, 8 up, 7 in 2023-09-23T23:18:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:43 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:18:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:43 smithi138 ceph-mon[193606]: 1.8 scrub ok 2023-09-23T23:18:43.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:43.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:18:43.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[212939]: failed to encode map e252 with expected crc 2023-09-23T23:18:43.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[212939]: failed to encode map e252 with expected crc 2023-09-23T23:18:43.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[212939]: failed to encode map e252 with expected crc 2023-09-23T23:18:43.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[212939]: failed to encode map e253 with expected crc 2023-09-23T23:18:43.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T23:18:43.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[212939]: osdmap e253: 8 total, 8 up, 7 in 2023-09-23T23:18:43.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:18:43.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[212939]: 1.8 scrub ok 2023-09-23T23:18:43.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:43.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:18:43.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[221102]: failed to encode map e252 with expected crc 2023-09-23T23:18:43.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[221102]: failed to encode map e252 with expected crc 2023-09-23T23:18:43.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[221102]: failed to encode map e252 with expected crc 2023-09-23T23:18:43.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[221102]: failed to encode map e253 with expected crc 2023-09-23T23:18:43.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T23:18:43.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[221102]: osdmap e253: 8 total, 8 up, 7 in 2023-09-23T23:18:43.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:18:43.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[221102]: 1.8 scrub ok 2023-09-23T23:18:43.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:43.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:18:44.390 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:18:44.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:44.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:18:44.494 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:44.523 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:44.677 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:44 smithi110 ceph-mon[221102]: pgmap v779: 99 pgs: 2 peering, 97 active+clean; 687 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 160 op/s; 0 B/s, 52 objects/s recovering 2023-09-23T23:18:44.679 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:44 smithi110 ceph-mon[212939]: pgmap v779: 99 pgs: 2 peering, 97 active+clean; 687 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 160 op/s; 0 B/s, 52 objects/s recovering 2023-09-23T23:18:44.696 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-09-23T23:18:44.696 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-09-23T23:18:44.714 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:18:44.732 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:44.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:44.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:44.836 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:18:44.870 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:44.936 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:18:44.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:44.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:18:44.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:44 smithi138 ceph-mon[193606]: pgmap v779: 99 pgs: 2 peering, 97 active+clean; 687 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 160 op/s; 0 B/s, 52 objects/s recovering 2023-09-23T23:18:45.005 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:18:45.024 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:18:45.041 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:18:45.045 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:18:45.067 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:18:45.071 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:45.096 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:18:45.119 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:18:45.138 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:18:45.159 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:18:45.161 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:18:45.162 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:18:45.162 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:18:45.162 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:18:45.162 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:18:45.162 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:18:45.162 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:18:45.162 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:18:45.177 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:18:45.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:45.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:18:45.280 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:18:45.312 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:45.359 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:18:45.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:45.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:18:45.383 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:18:45.383 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:18:45.467 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:18:45.494 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:45.592 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:18:45.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:45.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:18:45.695 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:18:45.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:45 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2646652828' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:45.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:45 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2646652828' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:45.727 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:45.776 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:18:45.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:45.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:18:45.880 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:18:45.907 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:45.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:45 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2646652828' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:46.150 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:18:46.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:46.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:18:46.254 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:18:46.285 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:46.334 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:18:46.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:46.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:18:46.434 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:18:46.441 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:46.504 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:46.614 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:46.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:46.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:46.655 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:46.683 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:46 smithi110 ceph-mon[212939]: pgmap v780: 99 pgs: 99 active+clean; 687 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 0 B/s, 44 objects/s recovering 2023-09-23T23:18:46.683 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:46 smithi110 ceph-mon[221102]: pgmap v780: 99 pgs: 99 active+clean; 687 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 0 B/s, 44 objects/s recovering 2023-09-23T23:18:46.706 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:46.718 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:18:46.749 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:46.799 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:18:46.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:46.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:18:46.906 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:18:46.930 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:46 smithi138 ceph-mon[193606]: pgmap v780: 99 pgs: 99 active+clean; 687 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 0 B/s, 44 objects/s recovering 2023-09-23T23:18:46.933 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:46.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:18:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:18:46] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.43.0" 2023-09-23T23:18:46.975 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:18:46.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:46.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:18:47.082 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:47.108 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:47.144 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:47.150 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:47.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:47.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:47.255 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:18:47.281 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:47.323 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:18:47.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:47.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:18:47.429 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:18:47.459 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:47.916 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:47 smithi110 ceph-mon[212939]: 1.2 scrub starts 2023-09-23T23:18:47.917 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:47 smithi110 ceph-mon[212939]: 1.2 scrub ok 2023-09-23T23:18:47.917 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:47 smithi110 ceph-mon[221102]: 1.2 scrub starts 2023-09-23T23:18:47.917 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:47 smithi110 ceph-mon[221102]: 1.2 scrub ok 2023-09-23T23:18:47.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:47 smithi138 ceph-mon[193606]: 1.2 scrub starts 2023-09-23T23:18:47.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:47 smithi138 ceph-mon[193606]: 1.2 scrub ok 2023-09-23T23:18:47.952 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:18:47.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:47.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:18:48.055 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:18:48.083 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:48.633 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:18:48.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:48.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:18:48.685 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:18:48.736 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:18:48.765 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:48.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:48 smithi138 ceph-mon[193606]: pgmap v781: 99 pgs: 99 active+clean; 634 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 33 objects/s recovering 2023-09-23T23:18:48.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:48 smithi110 ceph-mon[212939]: pgmap v781: 99 pgs: 99 active+clean; 634 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 33 objects/s recovering 2023-09-23T23:18:48.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:48 smithi110 ceph-mon[221102]: pgmap v781: 99 pgs: 99 active+clean; 634 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 33 objects/s recovering 2023-09-23T23:18:49.381 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:18:49.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:49.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:18:49.485 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:18:49.516 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:49.564 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:18:49.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:49.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:18:49.671 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:49.696 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7] out_osds: [2] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T23:18:49.696 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:18:49.696 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T23:18:49.697 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T23:18:49.697 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T23:18:49.697 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T23:18:49.702 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:49.744 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:49.965 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:49.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:49.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:50.069 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:18:50.097 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:50.140 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:18:50.397 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:18:50.397 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:18:50.449 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:18:50.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:50.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:18:50.554 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:50.582 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:50.609 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:18:50.624 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_4' removed 2023-09-23T23:18:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:50 smithi138 ceph-mon[193606]: pgmap v782: 99 pgs: 99 active+clean; 634 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:18:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:50 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2790618063' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:18:50.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:50 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3936016696' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:18:50.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:50 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:18:50.784 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:50.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:50.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:50.887 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:18:50.928 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:50.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:50 smithi110 ceph-mon[212939]: pgmap v782: 99 pgs: 99 active+clean; 634 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:18:50.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:50 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2790618063' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:18:50.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:50 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3936016696' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:18:50.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:50 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:18:50.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:50 smithi110 ceph-mon[221102]: pgmap v782: 99 pgs: 99 active+clean; 634 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:18:50.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:50 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2790618063' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:18:50.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:50 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3936016696' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:18:50.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:50 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:18:50.971 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:18:50.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:50.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:18:51.074 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:18:51.102 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:51.644 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:51 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:18:51.645 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:51 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:18:51.645 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:51 smithi138 ceph-mon[193606]: failed to encode map e254 with expected crc 2023-09-23T23:18:51.645 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:51 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2790618063' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:18:51.645 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:51 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:18:51.645 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:51 smithi138 ceph-mon[193606]: osdmap e254: 8 total, 8 up, 7 in 2023-09-23T23:18:51.645 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:51 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:18:51.645 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:51 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:51.645 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:51 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:51.869 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:18:51.869 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "" 2023-09-23T23:18:51.869 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:18:51.934 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:18:51.934 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:18:51.934 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[212939]: failed to encode map e254 with expected crc 2023-09-23T23:18:51.934 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2790618063' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:18:51.934 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:18:51.934 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[212939]: osdmap e254: 8 total, 8 up, 7 in 2023-09-23T23:18:51.934 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:18:51.935 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:51.935 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:51.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:18:51.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:18:51.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[221102]: failed to encode map e254 with expected crc 2023-09-23T23:18:51.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2790618063' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:18:51.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:18:51.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[221102]: osdmap e254: 8 total, 8 up, 7 in 2023-09-23T23:18:51.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:18:51.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:51.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:51 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:52.056 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:18:52.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:52.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:18:52.071 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:18:52.161 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:18:52.191 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:52.608 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:18:52.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:52.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:18:52.712 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:18:52.715 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T23:18:52.716 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T23:18:52.716 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-09-23T23:18:52.716 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-09-23T23:18:52.740 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:52.794 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:18:52.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:52.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:18:52.901 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:18:52.910 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:52 smithi110 ceph-mon[221102]: pgmap v784: 83 pgs: 83 active+clean; 582 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:18:52.910 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:52 smithi110 ceph-mon[221102]: failed to encode map e254 with expected crc 2023-09-23T23:18:52.910 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:52 smithi110 ceph-mon[221102]: failed to encode map e254 with expected crc 2023-09-23T23:18:52.910 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:52 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:52.910 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:52 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:52.910 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:52 smithi110 ceph-mon[212939]: pgmap v784: 83 pgs: 83 active+clean; 582 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:18:52.910 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:52 smithi110 ceph-mon[212939]: failed to encode map e254 with expected crc 2023-09-23T23:18:52.911 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:52 smithi110 ceph-mon[212939]: failed to encode map e254 with expected crc 2023-09-23T23:18:52.911 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:52 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:52.911 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:52 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:52.938 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:52.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:52 smithi138 ceph-mon[193606]: pgmap v784: 83 pgs: 83 active+clean; 582 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:18:52.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:52 smithi138 ceph-mon[193606]: failed to encode map e254 with expected crc 2023-09-23T23:18:52.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:52 smithi138 ceph-mon[193606]: failed to encode map e254 with expected crc 2023-09-23T23:18:52.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:52 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:52.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:52 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:53.093 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:18:53.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:53.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:18:53.138 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:18:53.145 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:53.165 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:18:53.193 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:18:53.197 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:18:53.217 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:18:53.226 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:53.237 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:18:53.256 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:18:53.274 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:18:53.296 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:18:53.297 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:18:53.297 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:18:53.297 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:18:53.297 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:18:53.297 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:18:53.297 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:18:53.297 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:18:53.298 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:18:53.301 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:53.325 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:18:53.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:53.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:18:53.431 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:18:53.461 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:53.509 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:18:53.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:53.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:18:53.616 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:18:53.642 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:53.736 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:18:53.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:53.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:18:53.841 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:18:53.868 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:53 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1879436531' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:53.868 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:53 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:53.870 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:53.915 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:18:53.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:53.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:18:53.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:53 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1879436531' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:53.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:53 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:53.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:53 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1879436531' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:18:53.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:53 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:54.021 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:54.050 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:54.355 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:54.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:54.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:54.459 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:54.486 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:54.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:54 smithi110 ceph-mon[212939]: pgmap v785: 83 pgs: 83 active+clean; 582 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:18:54.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:54 smithi110 ceph-mon[221102]: pgmap v785: 83 pgs: 83 active+clean; 582 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:18:54.817 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:18:54.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:54 smithi138 ceph-mon[193606]: pgmap v785: 83 pgs: 83 active+clean; 582 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:18:55.132 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:55.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:55.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:55.245 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:18:55.276 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:55.471 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:18:55.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:55.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:18:55.576 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:55.606 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:55.864 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:55.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:55.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:55.967 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:55.994 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:56.037 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:56.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:56.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:56.143 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:18:56.173 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:56.213 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:56.220 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:18:56.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:56.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:18:56.283 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:18:56.283 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:18:56.283 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:18:56.283 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:18:56.283 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:18:56.325 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:56.351 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:56.391 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:56.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:56.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:56.498 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:56.524 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:56.567 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:56.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:56.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:56.673 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:18:56.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:56 smithi138 ceph-mon[193606]: pgmap v786: 83 pgs: 83 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:18:56.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:56 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:56.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:56 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:56.699 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:56.741 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:18:56.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:56.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:18:56.801 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:56 smithi110 ceph-mon[212939]: pgmap v786: 83 pgs: 83 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:18:56.801 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:56 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:56.801 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:56 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:56.801 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:56 smithi110 ceph-mon[221102]: pgmap v786: 83 pgs: 83 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:18:56.802 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:56 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:56.802 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:56 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:56.848 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:18:56.874 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:56.916 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:18:56.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:56.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:18:57.021 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:18:57.030 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:18:57.051 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:57.077 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:18:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:18:56] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.43.0" 2023-09-23T23:18:57.277 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:18:57.303 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:18:57.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:57.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:18:57.408 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:18:57.439 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:57.483 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:18:57.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:57.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:18:57.572 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_5' created 2023-09-23T23:18:57.589 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:18:57.593 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:57 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4005993444' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T23:18:57.593 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:57 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4005993444' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T23:18:57.615 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:57.657 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:18:57.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:57.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:18:57.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:57 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4005993444' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T23:18:57.698 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7] out_osds: [2] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T23:18:57.698 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:18:57.698 INFO:tasks.thrashosds.thrasher:Killing osd 7, live_osds are [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T23:18:57.698 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-09-23T23:18:57.698 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T23:18:57.763 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:18:57.791 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:58.191 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:18:57 smithi138 systemd[1]: Stopping Ceph osd.7 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:18:58.192 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:18:57 smithi138 conmon[139964]: 2023-09-23T23:18:57.961+0000 7fb42d7b2700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:18:58.192 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:18:57 smithi138 conmon[139964]: 2023-09-23T23:18:57.961+0000 7fb42d7b2700 -1 osd.7 255 *** Got signal Terminated *** 2023-09-23T23:18:58.192 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:18:57 smithi138 conmon[139964]: 2023-09-23T23:18:57.961+0000 7fb42d7b2700 -1 osd.7 255 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:18:58.646 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-09-23T23:18:58.664 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:18:58.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:58.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:18:58.717 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:58.733 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:18:58.770 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:58.800 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:58.845 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:58.847 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:58 smithi138 ceph-mon[193606]: pgmap v787: 83 pgs: 83 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:18:58.847 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:58 smithi138 ceph-mon[193606]: failed to encode map e255 with expected crc 2023-09-23T23:18:58.847 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:58 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4005993444' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T23:18:58.847 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:58 smithi138 ceph-mon[193606]: osdmap e255: 8 total, 8 up, 7 in 2023-09-23T23:18:58.847 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:58 smithi138 ceph-mon[193606]: osd.7 marked itself down and dead 2023-09-23T23:18:58.847 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:58.847 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:18:58.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:58.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:58.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:58 smithi110 ceph-mon[212939]: pgmap v787: 83 pgs: 83 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:18:58.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:58 smithi110 ceph-mon[212939]: failed to encode map e255 with expected crc 2023-09-23T23:18:58.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:58 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4005993444' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T23:18:58.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:58 smithi110 ceph-mon[212939]: osdmap e255: 8 total, 8 up, 7 in 2023-09-23T23:18:58.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:58 smithi110 ceph-mon[212939]: osd.7 marked itself down and dead 2023-09-23T23:18:58.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:58.948 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:18:58.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:58 smithi110 ceph-mon[221102]: pgmap v787: 83 pgs: 83 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:18:58.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:58 smithi110 ceph-mon[221102]: failed to encode map e255 with expected crc 2023-09-23T23:18:58.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:58 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4005993444' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T23:18:58.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:58 smithi110 ceph-mon[221102]: osdmap e255: 8 total, 8 up, 7 in 2023-09-23T23:18:58.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:58 smithi110 ceph-mon[221102]: osd.7 marked itself down and dead 2023-09-23T23:18:58.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:18:58.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:18:58.950 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:18:58.980 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:59.146 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:18:59.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:59.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:18:59.250 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:18:59.278 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:59.373 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:18:59.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:18:59.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:18:59.477 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:18:59.506 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:18:59.908 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:59 smithi110 ceph-mon[221102]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:18:59.908 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:59 smithi110 ceph-mon[221102]: osdmap e256: 8 total, 7 up, 7 in 2023-09-23T23:18:59.908 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:59 smithi110 ceph-mon[221102]: failed to encode map e256 with expected crc 2023-09-23T23:18:59.908 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:59 smithi110 ceph-mon[221102]: failed to encode map e256 with expected crc 2023-09-23T23:18:59.908 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:59 smithi110 ceph-mon[221102]: failed to encode map e256 with expected crc 2023-09-23T23:18:59.909 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:59 smithi110 ceph-mon[221102]: failed to encode map e256 with expected crc 2023-09-23T23:18:59.909 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:18:59 smithi110 ceph-mon[221102]: failed to encode map e256 with expected crc 2023-09-23T23:18:59.909 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:59 smithi110 ceph-mon[212939]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:18:59.909 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:59 smithi110 ceph-mon[212939]: osdmap e256: 8 total, 7 up, 7 in 2023-09-23T23:18:59.909 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:59 smithi110 ceph-mon[212939]: failed to encode map e256 with expected crc 2023-09-23T23:18:59.909 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:59 smithi110 ceph-mon[212939]: failed to encode map e256 with expected crc 2023-09-23T23:18:59.909 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:59 smithi110 ceph-mon[212939]: failed to encode map e256 with expected crc 2023-09-23T23:18:59.909 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:59 smithi110 ceph-mon[212939]: failed to encode map e256 with expected crc 2023-09-23T23:18:59.909 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:18:59 smithi110 ceph-mon[212939]: failed to encode map e256 with expected crc 2023-09-23T23:18:59.917 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:59 smithi138 ceph-mon[193606]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T23:18:59.918 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:59 smithi138 ceph-mon[193606]: osdmap e256: 8 total, 7 up, 7 in 2023-09-23T23:18:59.918 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:59 smithi138 ceph-mon[193606]: failed to encode map e256 with expected crc 2023-09-23T23:18:59.918 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:59 smithi138 ceph-mon[193606]: failed to encode map e256 with expected crc 2023-09-23T23:18:59.918 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:59 smithi138 ceph-mon[193606]: failed to encode map e256 with expected crc 2023-09-23T23:18:59.918 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:59 smithi138 ceph-mon[193606]: failed to encode map e256 with expected crc 2023-09-23T23:18:59.918 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:18:59 smithi138 ceph-mon[193606]: failed to encode map e256 with expected crc 2023-09-23T23:19:00.191 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:18:59 smithi138 bash[247743]: 161d1d2d0445b21e6ea340460034bf828d810cdfe935ed0000aca84a7021a010 2023-09-23T23:19:00.552 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:00.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:00.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:00.582 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:19:00.658 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:19:00.690 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:00.785 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[212939]: pgmap v790: 99 pgs: 9 peering, 4 stale+active+clean, 2 creating+peering, 3 unknown, 81 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:19:00.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:19:00.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[212939]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[212939]: osdmap e257: 8 total, 7 up, 7 in 2023-09-23T23:19:00.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[212939]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[212939]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[212939]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[212939]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[212939]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.787 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[212939]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.787 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/323971144' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:19:00.790 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[221102]: pgmap v790: 99 pgs: 9 peering, 4 stale+active+clean, 2 creating+peering, 3 unknown, 81 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:19:00.790 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:19:00.790 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[221102]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.790 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[221102]: osdmap e257: 8 total, 7 up, 7 in 2023-09-23T23:19:00.790 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[221102]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.790 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[221102]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.790 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[221102]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.790 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[221102]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.790 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[221102]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.791 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[221102]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.791 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:00 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/323971144' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:19:00.793 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:19:00.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:00.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:19:00.902 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:19:00.931 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:00 smithi138 ceph-mon[193606]: pgmap v790: 99 pgs: 9 peering, 4 stale+active+clean, 2 creating+peering, 3 unknown, 81 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:19:00.931 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:00 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:19:00.931 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:00 smithi138 ceph-mon[193606]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.931 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:00 smithi138 ceph-mon[193606]: osdmap e257: 8 total, 7 up, 7 in 2023-09-23T23:19:00.931 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:00 smithi138 ceph-mon[193606]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.931 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:00 smithi138 ceph-mon[193606]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.931 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:00 smithi138 ceph-mon[193606]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.931 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:00 smithi138 ceph-mon[193606]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.932 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:00 smithi138 ceph-mon[193606]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.932 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:00 smithi138 ceph-mon[193606]: failed to encode map e257 with expected crc 2023-09-23T23:19:00.932 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:00 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/323971144' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:19:00.935 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:00.984 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:19:00.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:00.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:19:01.088 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:19:01.116 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:01.261 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:19:01.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:01.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:19:01.364 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:01.392 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:01 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:01 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/323971144' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:19:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:01 smithi110 ceph-mon[212939]: osdmap e258: 8 total, 7 up, 7 in 2023-09-23T23:19:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:01 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:01 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/323971144' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:19:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:01 smithi110 ceph-mon[221102]: osdmap e258: 8 total, 7 up, 7 in 2023-09-23T23:19:01.779 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.7.service' 2023-09-23T23:19:01.833 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:19:01.833 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-09-23T23:19:01.887 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:01.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:01.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:01.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:01 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:01.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:01 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/323971144' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:19:01.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:01 smithi138 ceph-mon[193606]: osdmap e258: 8 total, 7 up, 7 in 2023-09-23T23:19:01.991 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:02.024 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:02.046 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:19:02.066 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:19:02.075 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:02.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:02.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:02.087 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:19:02.108 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:19:02.136 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:19:02.155 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:19:02.176 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:19:02.177 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:19:02.178 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:19:02.178 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:19:02.178 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:19:02.178 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:19:02.178 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:19:02.178 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:19:02.178 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:19:02.180 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:02.217 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:02.419 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:19:02.419 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:19:02.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:02 smithi110 ceph-mon[212939]: pgmap v793: 99 pgs: 4 active+undersized, 9 peering, 2 stale+active+clean, 2 creating+peering, 82 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:19:02.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:02 smithi110 ceph-mon[212939]: failed to encode map e258 with expected crc 2023-09-23T23:19:02.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:02 smithi110 ceph-mon[212939]: failed to encode map e258 with expected crc 2023-09-23T23:19:02.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:02 smithi110 ceph-mon[212939]: failed to encode map e258 with expected crc 2023-09-23T23:19:02.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:02 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2841782622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:02.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:02 smithi110 ceph-mon[221102]: pgmap v793: 99 pgs: 4 active+undersized, 9 peering, 2 stale+active+clean, 2 creating+peering, 82 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:19:02.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:02 smithi110 ceph-mon[221102]: failed to encode map e258 with expected crc 2023-09-23T23:19:02.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:02 smithi110 ceph-mon[221102]: failed to encode map e258 with expected crc 2023-09-23T23:19:02.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:02 smithi110 ceph-mon[221102]: failed to encode map e258 with expected crc 2023-09-23T23:19:02.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:02 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2841782622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:02.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:02 smithi138 ceph-mon[193606]: pgmap v793: 99 pgs: 4 active+undersized, 9 peering, 2 stale+active+clean, 2 creating+peering, 82 active+clean; 582 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T23:19:02.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:02 smithi138 ceph-mon[193606]: failed to encode map e258 with expected crc 2023-09-23T23:19:02.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:02 smithi138 ceph-mon[193606]: failed to encode map e258 with expected crc 2023-09-23T23:19:02.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:02 smithi138 ceph-mon[193606]: failed to encode map e258 with expected crc 2023-09-23T23:19:02.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:02 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2841782622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:03.599 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:03.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:03.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:03.703 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:03.731 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:03.934 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:03 smithi110 ceph-mon[212939]: failed to encode map e258 with expected crc 2023-09-23T23:19:03.934 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:03 smithi110 ceph-mon[212939]: failed to encode map e258 with expected crc 2023-09-23T23:19:03.934 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:03 smithi110 ceph-mon[212939]: failed to encode map e258 with expected crc 2023-09-23T23:19:03.934 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:03 smithi110 ceph-mon[212939]: failed to encode map e258 with expected crc 2023-09-23T23:19:03.934 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:03 smithi110 ceph-mon[221102]: failed to encode map e258 with expected crc 2023-09-23T23:19:03.934 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:03 smithi110 ceph-mon[221102]: failed to encode map e258 with expected crc 2023-09-23T23:19:03.934 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:03 smithi110 ceph-mon[221102]: failed to encode map e258 with expected crc 2023-09-23T23:19:03.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:03 smithi110 ceph-mon[221102]: failed to encode map e258 with expected crc 2023-09-23T23:19:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:03 smithi138 ceph-mon[193606]: failed to encode map e258 with expected crc 2023-09-23T23:19:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:03 smithi138 ceph-mon[193606]: failed to encode map e258 with expected crc 2023-09-23T23:19:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:03 smithi138 ceph-mon[193606]: failed to encode map e258 with expected crc 2023-09-23T23:19:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:03 smithi138 ceph-mon[193606]: failed to encode map e258 with expected crc 2023-09-23T23:19:03.952 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:19:03.953 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:19:04.239 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:04.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:04.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:04.337 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:19:04.343 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:19:04.375 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:04.403 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:04.552 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:19:04.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:04.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:19:04.656 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:04.684 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:04.781 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:04.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:04.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:04.813 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:04 smithi110 ceph-mon[212939]: pgmap v794: 99 pgs: 7 active+undersized, 9 peering, 2 creating+peering, 81 active+clean; 634 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 11 op/s; 27 KiB/s, 0 objects/s recovering 2023-09-23T23:19:04.813 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:04 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:19:04.813 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:04 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:19:04.813 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:04 smithi110 ceph-mon[221102]: pgmap v794: 99 pgs: 7 active+undersized, 9 peering, 2 creating+peering, 81 active+clean; 634 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 11 op/s; 27 KiB/s, 0 objects/s recovering 2023-09-23T23:19:04.813 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:04 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:19:04.813 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:04 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:19:04.885 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:04.915 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:04 smithi138 ceph-mon[193606]: pgmap v794: 99 pgs: 7 active+undersized, 9 peering, 2 creating+peering, 81 active+clean; 634 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 11 op/s; 27 KiB/s, 0 objects/s recovering 2023-09-23T23:19:04.915 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:04 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:19:04.915 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:04 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:19:04.918 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:04.968 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:04.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:04.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:05.075 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:19:05.077 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:05.109 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:05.254 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:19:05.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:05.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:19:05.357 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:19:05.386 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:05.412 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:19:05.630 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:19:05.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:05.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:19:05.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:05 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:19:05.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:05 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:19:05.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:05 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:19:05.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:05 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:19:05.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:05 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:19:05.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:05 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:19:05.734 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:19:05.767 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:05.816 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:19:05.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:05.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:19:05.922 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:05.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:05 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:19:05.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:05 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:19:05.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:05 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:19:05.950 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:05.954 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:19:06.097 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:06.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:06.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:06.202 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:19:06.233 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:06.282 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:19:06.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:06.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:19:06.387 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:06.415 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:06.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:06 smithi110 ceph-mon[212939]: pgmap v795: 99 pgs: 3 active+undersized+degraded, 15 active+undersized, 81 active+clean; 634 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 12 KiB/s rd, 32 KiB/s wr, 9 op/s; 7/72 objects degraded (9.722%); 23 KiB/s, 0 objects/s recovering 2023-09-23T23:19:06.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:06 smithi110 ceph-mon[212939]: Health check failed: Degraded data redundancy: 7/72 objects degraded (9.722%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:06.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:06 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2023-09-23T23:19:06.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:06 smithi110 ceph-mon[221102]: pgmap v795: 99 pgs: 3 active+undersized+degraded, 15 active+undersized, 81 active+clean; 634 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 12 KiB/s rd, 32 KiB/s wr, 9 op/s; 7/72 objects degraded (9.722%); 23 KiB/s, 0 objects/s recovering 2023-09-23T23:19:06.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:06 smithi110 ceph-mon[221102]: Health check failed: Degraded data redundancy: 7/72 objects degraded (9.722%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:06.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:06 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2023-09-23T23:19:06.833 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7] out_osds: [2] dead_osds: [7] live_osds: [0, 1, 3, 4, 5, 6, 2] 2023-09-23T23:19:06.834 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:19:06.834 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 3, 4, 5, 6, 7] 2023-09-23T23:19:06.834 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd out 1 2023-09-23T23:19:06.866 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:06.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:06.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:06.932 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:06.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:06 smithi138 ceph-mon[193606]: pgmap v795: 99 pgs: 3 active+undersized+degraded, 15 active+undersized, 81 active+clean; 634 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 12 KiB/s rd, 32 KiB/s wr, 9 op/s; 7/72 objects degraded (9.722%); 23 KiB/s, 0 objects/s recovering 2023-09-23T23:19:06.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:06 smithi138 ceph-mon[193606]: Health check failed: Degraded data redundancy: 7/72 objects degraded (9.722%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:06.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:06 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2023-09-23T23:19:06.974 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:07.003 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:07.165 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:19:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:19:06] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.43.0" 2023-09-23T23:19:07.202 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:07.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:07.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:07.306 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:07.334 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:07.370 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:07.428 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:19:07.428 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:19:07.487 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:07.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:07.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:07.592 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:07.605 INFO:teuthology.orchestra.run.smithi110.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-09-23T23:19:07.629 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:07.673 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:07.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:07.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:07.777 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:07.800 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:07 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/574186263' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:19:07.800 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:07 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:19:07.801 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:07 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/574186263' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:19:07.801 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:07 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:19:07.803 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:07.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:07 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/574186263' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:19:07.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:07 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:19:08.868 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:08 smithi110 ceph-mon[212939]: pgmap v796: 99 pgs: 3 active+undersized+degraded, 15 active+undersized, 81 active+clean; 634 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 8 op/s; 7/72 objects degraded (9.722%); 20 KiB/s, 0 objects/s recovering 2023-09-23T23:19:08.868 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:08 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:19:08.869 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:08 smithi110 ceph-mon[212939]: osdmap e259: 8 total, 7 up, 7 in 2023-09-23T23:19:08.869 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:08 smithi110 ceph-mon[212939]: failed to encode map e259 with expected crc 2023-09-23T23:19:08.869 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:08 smithi110 ceph-mon[212939]: failed to encode map e259 with expected crc 2023-09-23T23:19:08.869 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:08 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:19:08.869 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:08 smithi110 ceph-mon[221102]: pgmap v796: 99 pgs: 3 active+undersized+degraded, 15 active+undersized, 81 active+clean; 634 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 8 op/s; 7/72 objects degraded (9.722%); 20 KiB/s, 0 objects/s recovering 2023-09-23T23:19:08.869 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:08 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:19:08.869 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:08 smithi110 ceph-mon[221102]: osdmap e259: 8 total, 7 up, 7 in 2023-09-23T23:19:08.869 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:08 smithi110 ceph-mon[221102]: failed to encode map e259 with expected crc 2023-09-23T23:19:08.870 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:08 smithi110 ceph-mon[221102]: failed to encode map e259 with expected crc 2023-09-23T23:19:08.870 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:08 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:19:08.871 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:19:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:08 smithi138 ceph-mon[193606]: pgmap v796: 99 pgs: 3 active+undersized+degraded, 15 active+undersized, 81 active+clean; 634 KiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 8 op/s; 7/72 objects degraded (9.722%); 20 KiB/s, 0 objects/s recovering 2023-09-23T23:19:08.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:08 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:19:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:08 smithi138 ceph-mon[193606]: osdmap e259: 8 total, 7 up, 7 in 2023-09-23T23:19:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:08 smithi138 ceph-mon[193606]: failed to encode map e259 with expected crc 2023-09-23T23:19:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:08 smithi138 ceph-mon[193606]: failed to encode map e259 with expected crc 2023-09-23T23:19:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:08 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:19:08.968 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:08.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:08.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:09.072 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:09.098 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:09.196 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:09.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:09.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:09.300 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:09.327 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:09.720 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:09.769 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:09.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:09.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:09.873 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:19:09.903 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:09 smithi138 ceph-mon[193606]: failed to encode map e259 with expected crc 2023-09-23T23:19:09.903 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:09 smithi138 ceph-mon[193606]: failed to encode map e259 with expected crc 2023-09-23T23:19:09.908 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:09.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:09 smithi110 ceph-mon[212939]: failed to encode map e259 with expected crc 2023-09-23T23:19:09.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:09 smithi110 ceph-mon[212939]: failed to encode map e259 with expected crc 2023-09-23T23:19:09.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:09 smithi110 ceph-mon[221102]: failed to encode map e259 with expected crc 2023-09-23T23:19:09.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:09 smithi110 ceph-mon[221102]: failed to encode map e259 with expected crc 2023-09-23T23:19:09.957 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:19:09.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:09.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:19:10.062 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:10.091 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:10.743 DEBUG:teuthology.orchestra.run.smithi110:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-09-23T23:19:10.746 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T23:19:10.746 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T23:19:10.808 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:hints = 1 2023-09-23T23:19:10.808 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T23:19:10.808 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object prefix: benchmark_data_smithi110_366450 2023-09-23T23:19:10.809 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:19:10.809 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 0 0 0 0 0 0 - 0 2023-09-23T23:19:10.838 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:10.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:10.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:10.869 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:19:10.872 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:10 smithi110 ceph-mon[212939]: pgmap v798: 99 pgs: 3 active+undersized+degraded, 15 active+undersized, 81 active+clean; 634 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 9.5 KiB/s rd, 25 KiB/s wr, 7 op/s; 7/72 objects degraded (9.722%); 18 KiB/s, 0 objects/s recovering 2023-09-23T23:19:10.872 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:10 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1596356521' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:19:10.872 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:10 smithi110 ceph-mon[221102]: pgmap v798: 99 pgs: 3 active+undersized+degraded, 15 active+undersized, 81 active+clean; 634 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 9.5 KiB/s rd, 25 KiB/s wr, 7 op/s; 7/72 objects degraded (9.722%); 18 KiB/s, 0 objects/s recovering 2023-09-23T23:19:10.872 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:10 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1596356521' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:19:10.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:10 smithi138 ceph-mon[193606]: pgmap v798: 99 pgs: 3 active+undersized+degraded, 15 active+undersized, 81 active+clean; 634 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 9.5 KiB/s rd, 25 KiB/s wr, 7 op/s; 7/72 objects degraded (9.722%); 18 KiB/s, 0 objects/s recovering 2023-09-23T23:19:10.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:10 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1596356521' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:19:10.944 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:11.010 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:11.159 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:11.809 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 1 16 3130 3114 194.612 194.625 0.00338737 0.00512263 2023-09-23T23:19:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:11 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:11 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1596356521' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:19:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:11 smithi138 ceph-mon[193606]: osdmap e260: 8 total, 7 up, 7 in 2023-09-23T23:19:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:11 smithi138 ceph-mon[193606]: failed to encode map e260 with expected crc 2023-09-23T23:19:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:11 smithi138 ceph-mon[193606]: failed to encode map e260 with expected crc 2023-09-23T23:19:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:11 smithi138 ceph-mon[193606]: failed to encode map e260 with expected crc 2023-09-23T23:19:11.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:11 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:11.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:11 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1596356521' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:19:11.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:11 smithi110 ceph-mon[212939]: osdmap e260: 8 total, 7 up, 7 in 2023-09-23T23:19:11.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:11 smithi110 ceph-mon[212939]: failed to encode map e260 with expected crc 2023-09-23T23:19:11.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:11 smithi110 ceph-mon[212939]: failed to encode map e260 with expected crc 2023-09-23T23:19:11.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:11 smithi110 ceph-mon[212939]: failed to encode map e260 with expected crc 2023-09-23T23:19:11.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:11 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:11.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:11 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1596356521' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:19:11.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:11 smithi110 ceph-mon[221102]: osdmap e260: 8 total, 7 up, 7 in 2023-09-23T23:19:11.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:11 smithi110 ceph-mon[221102]: failed to encode map e260 with expected crc 2023-09-23T23:19:11.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:11 smithi110 ceph-mon[221102]: failed to encode map e260 with expected crc 2023-09-23T23:19:11.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:11 smithi110 ceph-mon[221102]: failed to encode map e260 with expected crc 2023-09-23T23:19:11.999 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:12.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:12.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:12.104 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:12.150 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:12.809 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 2 16 6513 6497 203.016 211.438 0.00501815 0.00491913 2023-09-23T23:19:12.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:12 smithi110 ceph-mon[212939]: failed to encode map e260 with expected crc 2023-09-23T23:19:12.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:12 smithi110 ceph-mon[212939]: failed to encode map e260 with expected crc 2023-09-23T23:19:12.907 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:12 smithi110 ceph-mon[212939]: pgmap v800: 99 pgs: 3 active+undersized+degraded, 15 active+undersized, 81 active+clean; 634 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 7/72 objects degraded (9.722%) 2023-09-23T23:19:12.908 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:12 smithi110 ceph-mon[221102]: failed to encode map e260 with expected crc 2023-09-23T23:19:12.908 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:12 smithi110 ceph-mon[221102]: failed to encode map e260 with expected crc 2023-09-23T23:19:12.908 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:12 smithi110 ceph-mon[221102]: pgmap v800: 99 pgs: 3 active+undersized+degraded, 15 active+undersized, 81 active+clean; 634 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 7/72 objects degraded (9.722%) 2023-09-23T23:19:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:12 smithi138 ceph-mon[193606]: failed to encode map e260 with expected crc 2023-09-23T23:19:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:12 smithi138 ceph-mon[193606]: failed to encode map e260 with expected crc 2023-09-23T23:19:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:12 smithi138 ceph-mon[193606]: pgmap v800: 99 pgs: 3 active+undersized+degraded, 15 active+undersized, 81 active+clean; 634 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 7/72 objects degraded (9.722%) 2023-09-23T23:19:12.985 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:12.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:12.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:13.088 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:13.140 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:13.237 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:13.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:13.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:13.346 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:13.399 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:13.566 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:19:13.599 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:19:13.631 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:19:13.667 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:19:13.700 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:19:13.730 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:19:13.765 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:19:13.766 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:19:13.766 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:19:13.766 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:19:13.766 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:19:13.766 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:19:13.767 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:19:13.767 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:19:13.767 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:19:13.809 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 3 16 9778 9762 203.359 204.062 0.00469001 0.00490934 2023-09-23T23:19:13.883 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:13.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:13.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:13.988 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:14.034 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:14.138 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:19:14.138 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:19:14.138 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:13 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3327864504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:14.139 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:19:14.139 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:19:14.139 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:13 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3327864504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:14.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:19:14.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:19:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:13 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3327864504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:14.753 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:14.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:14.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:14.809 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 4 16 12244 12228 191.047 154.125 0.00440933 0.005227 2023-09-23T23:19:14.859 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:14.915 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:15.020 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:15.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:15.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:15.029 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:19:15.061 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:14 smithi110 ceph-mon[212939]: pgmap v801: 99 pgs: 3 active+undersized+degraded, 15 active+undersized, 81 active+clean; 634 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 7/72 objects degraded (9.722%) 2023-09-23T23:19:15.062 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:14 smithi110 ceph-mon[221102]: pgmap v801: 99 pgs: 3 active+undersized+degraded, 15 active+undersized, 81 active+clean; 634 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 7/72 objects degraded (9.722%) 2023-09-23T23:19:15.128 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:15.176 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:14 smithi138 ceph-mon[193606]: pgmap v801: 99 pgs: 3 active+undersized+degraded, 15 active+undersized, 81 active+clean; 634 KiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 7/72 objects degraded (9.722%) 2023-09-23T23:19:15.182 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:15.275 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:15.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:15.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:15.322 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:15 smithi110 conmon[212914]: 2023-09-23T23:19:15.117+0000 7f29aa464700 -1 mon.a@0(leader).osd e260 definitely_dead 0 2023-09-23T23:19:15.382 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:15.429 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:15.809 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 5 16 14564 14548 181.835 145 0.0068931 0.00549423 2023-09-23T23:19:16.010 INFO:teuthology.orchestra.run.smithi110.stderr:marked out osd.1. 2023-09-23T23:19:16.264 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:15 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2454082527' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T23:19:16.264 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:15 smithi110 ceph-mon[212939]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T23:19:16.264 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:15 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2454082527' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T23:19:16.264 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:15 smithi110 ceph-mon[221102]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T23:19:16.293 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:16.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:16.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:16.375 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:19:16.376 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:19:16.397 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:16.421 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:15 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2454082527' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T23:19:16.421 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:15 smithi138 ceph-mon[193606]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T23:19:16.423 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:16.464 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:16.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:16.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:16.570 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:16.596 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:16.638 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:16.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:16.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:16.745 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:19:16.771 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:16.809 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 6 16 15251 15235 158.683 42.9375 0.001472 0.00548834 2023-09-23T23:19:16.812 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:19:16.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:16.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:19:16.919 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:19:16.945 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:19:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:19:16] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.43.0" 2023-09-23T23:19:16.948 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:17.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:16 smithi138 ceph-mon[193606]: pgmap v802: 99 pgs: 5 active+undersized+degraded, 13 active+undersized, 81 active+clean; 713 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 1476/34254 objects degraded (4.309%) 2023-09-23T23:19:17.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:16 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1476/34254 objects degraded (4.309%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:17.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:16 smithi138 ceph-mon[193606]: failed to encode map e261 with expected crc 2023-09-23T23:19:17.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:16 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2454082527' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T23:19:17.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:16 smithi138 ceph-mon[193606]: osdmap e261: 8 total, 7 up, 6 in 2023-09-23T23:19:17.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:16 smithi138 ceph-mon[193606]: failed to encode map e261 with expected crc 2023-09-23T23:19:17.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:16 smithi110 ceph-mon[212939]: pgmap v802: 99 pgs: 5 active+undersized+degraded, 13 active+undersized, 81 active+clean; 713 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 1476/34254 objects degraded (4.309%) 2023-09-23T23:19:17.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:16 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1476/34254 objects degraded (4.309%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:17.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:16 smithi110 ceph-mon[212939]: failed to encode map e261 with expected crc 2023-09-23T23:19:17.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:16 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2454082527' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T23:19:17.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:16 smithi110 ceph-mon[212939]: osdmap e261: 8 total, 7 up, 6 in 2023-09-23T23:19:17.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:16 smithi110 ceph-mon[212939]: failed to encode map e261 with expected crc 2023-09-23T23:19:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:16 smithi110 ceph-mon[221102]: pgmap v802: 99 pgs: 5 active+undersized+degraded, 13 active+undersized, 81 active+clean; 713 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 1476/34254 objects degraded (4.309%) 2023-09-23T23:19:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:16 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1476/34254 objects degraded (4.309%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:16 smithi110 ceph-mon[221102]: failed to encode map e261 with expected crc 2023-09-23T23:19:17.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:16 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2454082527' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T23:19:17.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:16 smithi110 ceph-mon[221102]: osdmap e261: 8 total, 7 up, 6 in 2023-09-23T23:19:17.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:16 smithi110 ceph-mon[221102]: failed to encode map e261 with expected crc 2023-09-23T23:19:17.271 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:19:17.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:17.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:19:17.375 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:17.402 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:17.484 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:17.497 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:17.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:17.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:17.600 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:17.609 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:19:17.609 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:19:17.609 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:19:17.609 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:19:17.609 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:19:17.629 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:17.673 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:17.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:17.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:17.779 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:17.809 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:17.809 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 7 16 15251 15235 136.014 0 - 0.00548834 2023-09-23T23:19:17.852 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:17.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:17.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:17.956 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:17.982 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:18.238 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:18 smithi110 ceph-mon[212939]: osdmap e262: 8 total, 7 up, 6 in 2023-09-23T23:19:18.238 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:18 smithi110 ceph-mon[212939]: failed to encode map e262 with expected crc 2023-09-23T23:19:18.238 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:18 smithi110 ceph-mon[212939]: failed to encode map e262 with expected crc 2023-09-23T23:19:18.238 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:18 smithi110 ceph-mon[212939]: failed to encode map e262 with expected crc 2023-09-23T23:19:18.239 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:18 smithi110 ceph-mon[212939]: failed to encode map e262 with expected crc 2023-09-23T23:19:18.239 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:18 smithi110 ceph-mon[212939]: failed to encode map e262 with expected crc 2023-09-23T23:19:18.239 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:18 smithi110 ceph-mon[221102]: osdmap e262: 8 total, 7 up, 6 in 2023-09-23T23:19:18.239 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:18 smithi110 ceph-mon[221102]: failed to encode map e262 with expected crc 2023-09-23T23:19:18.239 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:18 smithi110 ceph-mon[221102]: failed to encode map e262 with expected crc 2023-09-23T23:19:18.239 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:18 smithi110 ceph-mon[221102]: failed to encode map e262 with expected crc 2023-09-23T23:19:18.239 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:18 smithi110 ceph-mon[221102]: failed to encode map e262 with expected crc 2023-09-23T23:19:18.239 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:18 smithi110 ceph-mon[221102]: failed to encode map e262 with expected crc 2023-09-23T23:19:18.293 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:18.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:18.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:18.377 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:19:18.397 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:18.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:18 smithi138 ceph-mon[193606]: osdmap e262: 8 total, 7 up, 6 in 2023-09-23T23:19:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:18 smithi138 ceph-mon[193606]: failed to encode map e262 with expected crc 2023-09-23T23:19:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:18 smithi138 ceph-mon[193606]: failed to encode map e262 with expected crc 2023-09-23T23:19:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:18 smithi138 ceph-mon[193606]: failed to encode map e262 with expected crc 2023-09-23T23:19:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:18 smithi138 ceph-mon[193606]: failed to encode map e262 with expected crc 2023-09-23T23:19:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:18 smithi138 ceph-mon[193606]: failed to encode map e262 with expected crc 2023-09-23T23:19:18.446 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:18.810 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 8 16 16427 16411 128.198 36.75 0.00228801 0.00774378 2023-09-23T23:19:19.252 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:19:19.286 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[212939]: pgmap v805: 99 pgs: 5 active+undersized+degraded, 13 active+undersized, 81 active+clean; 713 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 104 MiB/s wr, 1.66k op/s; 1476/34254 objects degraded (4.309%) 2023-09-23T23:19:19.286 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[212939]: failed to encode map e263 with expected crc 2023-09-23T23:19:19.286 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[212939]: osdmap e263: 8 total, 7 up, 6 in 2023-09-23T23:19:19.286 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[212939]: failed to encode map e263 with expected crc 2023-09-23T23:19:19.287 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[212939]: failed to encode map e263 with expected crc 2023-09-23T23:19:19.287 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[212939]: failed to encode map e263 with expected crc 2023-09-23T23:19:19.287 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[212939]: failed to encode map e264 with expected crc 2023-09-23T23:19:19.287 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[212939]: failed to encode map e264 with expected crc 2023-09-23T23:19:19.287 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[212939]: osdmap e264: 8 total, 7 up, 6 in 2023-09-23T23:19:19.287 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[212939]: failed to encode map e264 with expected crc 2023-09-23T23:19:19.288 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[221102]: pgmap v805: 99 pgs: 5 active+undersized+degraded, 13 active+undersized, 81 active+clean; 713 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 104 MiB/s wr, 1.66k op/s; 1476/34254 objects degraded (4.309%) 2023-09-23T23:19:19.288 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[221102]: failed to encode map e263 with expected crc 2023-09-23T23:19:19.288 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[221102]: osdmap e263: 8 total, 7 up, 6 in 2023-09-23T23:19:19.288 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[221102]: failed to encode map e263 with expected crc 2023-09-23T23:19:19.288 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[221102]: failed to encode map e263 with expected crc 2023-09-23T23:19:19.288 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[221102]: failed to encode map e263 with expected crc 2023-09-23T23:19:19.288 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[221102]: failed to encode map e264 with expected crc 2023-09-23T23:19:19.288 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[221102]: failed to encode map e264 with expected crc 2023-09-23T23:19:19.288 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[221102]: osdmap e264: 8 total, 7 up, 6 in 2023-09-23T23:19:19.288 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:19 smithi110 ceph-mon[221102]: failed to encode map e264 with expected crc 2023-09-23T23:19:19.315 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:19.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:19.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:19.426 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:19.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:19 smithi138 ceph-mon[193606]: pgmap v805: 99 pgs: 5 active+undersized+degraded, 13 active+undersized, 81 active+clean; 713 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 104 MiB/s wr, 1.66k op/s; 1476/34254 objects degraded (4.309%) 2023-09-23T23:19:19.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:19 smithi138 ceph-mon[193606]: failed to encode map e263 with expected crc 2023-09-23T23:19:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:19 smithi138 ceph-mon[193606]: osdmap e263: 8 total, 7 up, 6 in 2023-09-23T23:19:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:19 smithi138 ceph-mon[193606]: failed to encode map e263 with expected crc 2023-09-23T23:19:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:19 smithi138 ceph-mon[193606]: failed to encode map e263 with expected crc 2023-09-23T23:19:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:19 smithi138 ceph-mon[193606]: failed to encode map e263 with expected crc 2023-09-23T23:19:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:19 smithi138 ceph-mon[193606]: failed to encode map e264 with expected crc 2023-09-23T23:19:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:19 smithi138 ceph-mon[193606]: failed to encode map e264 with expected crc 2023-09-23T23:19:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:19 smithi138 ceph-mon[193606]: osdmap e264: 8 total, 7 up, 6 in 2023-09-23T23:19:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:19 smithi138 ceph-mon[193606]: failed to encode map e264 with expected crc 2023-09-23T23:19:19.484 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:19.570 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:19.588 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:19.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:19.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:19.694 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:19.748 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:19.810 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 9 16 18660 18644 129.455 139.562 0.00945449 0.00771803 2023-09-23T23:19:19.846 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:19.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:19.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:19.961 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:20.025 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:20.209 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:20.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:20.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:20.314 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:20.352 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:19:20.372 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:20.436 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:20.478 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:20.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:20.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:20.585 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:19:20.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[212939]: osdmap e265: 8 total, 7 up, 6 in 2023-09-23T23:19:20.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[212939]: failed to encode map e265 with expected crc 2023-09-23T23:19:20.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[212939]: failed to encode map e265 with expected crc 2023-09-23T23:19:20.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[212939]: failed to encode map e265 with expected crc 2023-09-23T23:19:20.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[212939]: failed to encode map e265 with expected crc 2023-09-23T23:19:20.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[212939]: failed to encode map e265 with expected crc 2023-09-23T23:19:20.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[212939]: pgmap v809: 99 pgs: 1 peering, 3 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 12 active+undersized, 76 active+clean; 1.0 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 24 KiB/s rd, 91 MiB/s wr, 1.48k op/s; 3222/49410 objects degraded (6.521%); 846/49410 objects misplaced (1.712%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:19:20.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2186005965' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:19:20.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:19:20.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[221102]: osdmap e265: 8 total, 7 up, 6 in 2023-09-23T23:19:20.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[221102]: failed to encode map e265 with expected crc 2023-09-23T23:19:20.624 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[221102]: failed to encode map e265 with expected crc 2023-09-23T23:19:20.624 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[221102]: failed to encode map e265 with expected crc 2023-09-23T23:19:20.624 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[221102]: failed to encode map e265 with expected crc 2023-09-23T23:19:20.624 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[221102]: failed to encode map e265 with expected crc 2023-09-23T23:19:20.624 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[221102]: pgmap v809: 99 pgs: 1 peering, 3 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 12 active+undersized, 76 active+clean; 1.0 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 24 KiB/s rd, 91 MiB/s wr, 1.48k op/s; 3222/49410 objects degraded (6.521%); 846/49410 objects misplaced (1.712%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:19:20.624 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2186005965' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:19:20.624 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:20 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:19:20.628 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:20.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:20 smithi138 ceph-mon[193606]: osdmap e265: 8 total, 7 up, 6 in 2023-09-23T23:19:20.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:20 smithi138 ceph-mon[193606]: failed to encode map e265 with expected crc 2023-09-23T23:19:20.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:20 smithi138 ceph-mon[193606]: failed to encode map e265 with expected crc 2023-09-23T23:19:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:20 smithi138 ceph-mon[193606]: failed to encode map e265 with expected crc 2023-09-23T23:19:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:20 smithi138 ceph-mon[193606]: failed to encode map e265 with expected crc 2023-09-23T23:19:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:20 smithi138 ceph-mon[193606]: failed to encode map e265 with expected crc 2023-09-23T23:19:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:20 smithi138 ceph-mon[193606]: pgmap v809: 99 pgs: 1 peering, 3 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 12 active+undersized, 76 active+clean; 1.0 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 24 KiB/s rd, 91 MiB/s wr, 1.48k op/s; 3222/49410 objects degraded (6.521%); 846/49410 objects misplaced (1.712%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:19:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:20 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2186005965' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:19:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:20 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:19:20.810 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 10 16 20841 20825 130.139 136.312 0.00627852 0.00766061 2023-09-23T23:19:21.334 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:19:21.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:21.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:19:21.438 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:21.463 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:21 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:21 smithi138 ceph-mon[193606]: failed to encode map e266 with expected crc 2023-09-23T23:19:21.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:21 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:19:21.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:21 smithi138 ceph-mon[193606]: osdmap e266: 8 total, 7 up, 6 in 2023-09-23T23:19:21.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:21 smithi138 ceph-mon[193606]: failed to encode map e266 with expected crc 2023-09-23T23:19:21.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:21 smithi138 ceph-mon[193606]: failed to encode map e266 with expected crc 2023-09-23T23:19:21.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:21 smithi138 ceph-mon[193606]: failed to encode map e266 with expected crc 2023-09-23T23:19:21.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:21 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:21.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:21 smithi110 ceph-mon[212939]: failed to encode map e266 with expected crc 2023-09-23T23:19:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:21 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:19:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:21 smithi110 ceph-mon[212939]: osdmap e266: 8 total, 7 up, 6 in 2023-09-23T23:19:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:21 smithi110 ceph-mon[212939]: failed to encode map e266 with expected crc 2023-09-23T23:19:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:21 smithi110 ceph-mon[212939]: failed to encode map e266 with expected crc 2023-09-23T23:19:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:21 smithi110 ceph-mon[212939]: failed to encode map e266 with expected crc 2023-09-23T23:19:21.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:21 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:21.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:21 smithi110 ceph-mon[221102]: failed to encode map e266 with expected crc 2023-09-23T23:19:21.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:21 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:19:21.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:21 smithi110 ceph-mon[221102]: osdmap e266: 8 total, 7 up, 6 in 2023-09-23T23:19:21.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:21 smithi110 ceph-mon[221102]: failed to encode map e266 with expected crc 2023-09-23T23:19:21.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:21 smithi110 ceph-mon[221102]: failed to encode map e266 with expected crc 2023-09-23T23:19:21.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:21 smithi110 ceph-mon[221102]: failed to encode map e266 with expected crc 2023-09-23T23:19:21.810 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 11 16 22834 22818 129.631 124.562 0.00282165 0.00770304 2023-09-23T23:19:22.336 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7] out_osds: [2, 1] dead_osds: [7] live_osds: [0, 1, 3, 4, 5, 6, 2] 2023-09-23T23:19:22.336 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:19:22.337 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T23:19:22.337 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:19:22.488 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:22.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:22.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:22.593 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:22.641 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:22 smithi138 ceph-mon[193606]: pgmap v811: 99 pgs: 1 peering, 5 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 76 active+clean; 1.1 GiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 21 KiB/s rd, 98 MiB/s wr, 1.58k op/s; 3222/53007 objects degraded (6.078%); 846/53007 objects misplaced (1.596%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:19:22.644 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:22.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:22 smithi110 ceph-mon[212939]: pgmap v811: 99 pgs: 1 peering, 5 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 76 active+clean; 1.1 GiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 21 KiB/s rd, 98 MiB/s wr, 1.58k op/s; 3222/53007 objects degraded (6.078%); 846/53007 objects misplaced (1.596%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:19:22.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:22 smithi110 ceph-mon[221102]: pgmap v811: 99 pgs: 1 peering, 5 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 76 active+clean; 1.1 GiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 21 KiB/s rd, 98 MiB/s wr, 1.58k op/s; 3222/53007 objects degraded (6.078%); 846/53007 objects misplaced (1.596%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:19:22.741 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:22.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:22.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:22.811 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 12 16 24539 24523 127.706 106.562 0.0097426 0.0078257 2023-09-23T23:19:22.853 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:22.921 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:23.029 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:23.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:23.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:23.139 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:23.177 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:23.221 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:23.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:23.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:23.325 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:23.362 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:23.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:23 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 3222/53007 objects degraded (6.078%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:23.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:23 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 3222/53007 objects degraded (6.078%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:23.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:23 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 3222/53007 objects degraded (6.078%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:23.791 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:23.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:23.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:23.810 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 13 16 26424 26408 126.944 117.812 0.00619718 0.00787196 2023-09-23T23:19:23.896 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:23.950 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:24.025 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:24.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:24.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:24.068 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:24.130 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:24.161 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:19:24.190 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:19:24.220 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:19:24.251 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:19:24.282 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:19:24.316 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:24.317 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:19:24.348 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:19:24.348 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:19:24.348 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:19:24.349 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:19:24.349 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:19:24.349 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:19:24.349 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:19:24.349 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:19:24.349 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:19:24.420 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:24.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:24.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:24.498 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:19:24.529 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:24.542 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:24 smithi110 ceph-mon[221102]: pgmap v812: 99 pgs: 1 peering, 5 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 76 active+clean; 1.1 GiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 15 KiB/s rd, 72 MiB/s wr, 1.16k op/s; 3222/53007 objects degraded (6.078%); 846/53007 objects misplaced (1.596%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:19:24.542 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:24 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/576087408' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:24.542 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:24 smithi110 ceph-mon[212939]: pgmap v812: 99 pgs: 1 peering, 5 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 76 active+clean; 1.1 GiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 15 KiB/s rd, 72 MiB/s wr, 1.16k op/s; 3222/53007 objects degraded (6.078%); 846/53007 objects misplaced (1.596%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:19:24.542 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:24 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/576087408' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:24.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:24 smithi138 ceph-mon[193606]: pgmap v812: 99 pgs: 1 peering, 5 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 76 active+clean; 1.1 GiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 15 KiB/s rd, 72 MiB/s wr, 1.16k op/s; 3222/53007 objects degraded (6.078%); 846/53007 objects misplaced (1.596%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:19:24.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:24 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/576087408' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:24.581 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:24.588 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:19:24.588 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:19:24.664 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:24.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:24.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:24.773 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:19:24.810 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:24.810 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 14 16 28256 28240 126.055 114.5 0.0153933 0.00792568 2023-09-23T23:19:24.941 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:19:24.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:24.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:19:25.046 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:25.108 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:25.203 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:25.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:25.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:25.314 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:25.349 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:25.811 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 15 16 29497 29481 122.821 77.5625 0.00191057 0.00802193 2023-09-23T23:19:26.372 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:26 smithi110 ceph-mon[221102]: osdmap e267: 8 total, 7 up, 6 in 2023-09-23T23:19:26.373 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:26 smithi110 ceph-mon[221102]: failed to encode map e267 with expected crc 2023-09-23T23:19:26.373 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:26 smithi110 ceph-mon[221102]: failed to encode map e267 with expected crc 2023-09-23T23:19:26.373 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:26 smithi110 ceph-mon[221102]: pgmap v814: 99 pgs: 5 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 77 active+clean; 1.7 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 132 MiB/s wr, 2.12k op/s; 4442/82383 objects degraded (5.392%); 131/82383 objects misplaced (0.159%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T23:19:26.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:26 smithi138 ceph-mon[193606]: osdmap e267: 8 total, 7 up, 6 in 2023-09-23T23:19:26.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:26 smithi138 ceph-mon[193606]: failed to encode map e267 with expected crc 2023-09-23T23:19:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:26 smithi138 ceph-mon[193606]: failed to encode map e267 with expected crc 2023-09-23T23:19:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:26 smithi138 ceph-mon[193606]: pgmap v814: 99 pgs: 5 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 77 active+clean; 1.7 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 132 MiB/s wr, 2.12k op/s; 4442/82383 objects degraded (5.392%); 131/82383 objects misplaced (0.159%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T23:19:26.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:26 smithi110 ceph-mon[212939]: osdmap e267: 8 total, 7 up, 6 in 2023-09-23T23:19:26.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:26 smithi110 ceph-mon[212939]: failed to encode map e267 with expected crc 2023-09-23T23:19:26.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:26 smithi110 ceph-mon[212939]: failed to encode map e267 with expected crc 2023-09-23T23:19:26.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:26 smithi110 ceph-mon[212939]: pgmap v814: 99 pgs: 5 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 77 active+clean; 1.7 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 132 MiB/s wr, 2.12k op/s; 4442/82383 objects degraded (5.392%); 131/82383 objects misplaced (0.159%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T23:19:26.803 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:26.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:26.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:26.811 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 16 16 30058 30042 117.337 35.0625 0.010114 0.00851697 2023-09-23T23:19:26.889 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:19:26.907 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:19:26.965 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:27.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:19:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:19:26] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.43.0" 2023-09-23T23:19:27.254 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:19:27.260 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:19:27.261 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:19:27.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:27.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:19:27.362 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:27.426 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:27.546 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:27.551 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:27.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:27.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:27.652 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:27.653 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:27.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:27 smithi138 ceph-mon[193606]: failed to encode map e268 with expected crc 2023-09-23T23:19:27.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:27 smithi138 ceph-mon[193606]: osdmap e268: 8 total, 7 up, 6 in 2023-09-23T23:19:27.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:27 smithi138 ceph-mon[193606]: failed to encode map e268 with expected crc 2023-09-23T23:19:27.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:27 smithi138 ceph-mon[193606]: failed to encode map e268 with expected crc 2023-09-23T23:19:27.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:27 smithi110 ceph-mon[221102]: failed to encode map e268 with expected crc 2023-09-23T23:19:27.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:27 smithi110 ceph-mon[221102]: osdmap e268: 8 total, 7 up, 6 in 2023-09-23T23:19:27.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:27 smithi110 ceph-mon[221102]: failed to encode map e268 with expected crc 2023-09-23T23:19:27.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:27 smithi110 ceph-mon[221102]: failed to encode map e268 with expected crc 2023-09-23T23:19:27.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:27 smithi110 ceph-mon[212939]: failed to encode map e268 with expected crc 2023-09-23T23:19:27.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:27 smithi110 ceph-mon[212939]: osdmap e268: 8 total, 7 up, 6 in 2023-09-23T23:19:27.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:27 smithi110 ceph-mon[212939]: failed to encode map e268 with expected crc 2023-09-23T23:19:27.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:27 smithi110 ceph-mon[212939]: failed to encode map e268 with expected crc 2023-09-23T23:19:27.708 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:27.805 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:27.811 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 17 16 31825 31809 116.93 110.438 0.00993019 0.00854736 2023-09-23T23:19:27.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:27.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:27.914 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:27.964 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:27.986 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:19:27.991 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:19:28.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:28 smithi138 ceph-mon[193606]: pgmap v816: 99 pgs: 5 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 77 active+clean; 1.7 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 86 MiB/s wr, 1.37k op/s; 4442/82383 objects degraded (5.392%); 131/82383 objects misplaced (0.159%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T23:19:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:28 smithi138 ceph-mon[193606]: from='client.45545 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:19:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:19:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:28 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 4442/82383 objects degraded (5.392%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:28.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:28 smithi110 ceph-mon[212939]: pgmap v816: 99 pgs: 5 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 77 active+clean; 1.7 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 86 MiB/s wr, 1.37k op/s; 4442/82383 objects degraded (5.392%); 131/82383 objects misplaced (0.159%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T23:19:28.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:28 smithi110 ceph-mon[212939]: from='client.45545 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:19:28.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:19:28.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:28 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 4442/82383 objects degraded (5.392%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:28.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:28 smithi110 ceph-mon[221102]: pgmap v816: 99 pgs: 5 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 77 active+clean; 1.7 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 86 MiB/s wr, 1.37k op/s; 4442/82383 objects degraded (5.392%); 131/82383 objects misplaced (0.159%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T23:19:28.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:28 smithi110 ceph-mon[221102]: from='client.45545 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:19:28.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:19:28.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:28 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 4442/82383 objects degraded (5.392%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:28.811 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 18 16 33438 33422 116.034 100.812 0.00985485 0.00858315 2023-09-23T23:19:29.262 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:19:29.298 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:29.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:29.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:29.408 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:19:29.440 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:29.811 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 19 16 35019 35003 115.127 98.8125 0.00486489 0.00867353 2023-09-23T23:19:30.431 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":816,"stamp":"2023-09-23T23:19:27.464729+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1798814444,"num_objects":27461,"num_object_clones":0,"num_object_copies":82383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4442,"num_objects_misplaced":131,"num_objects_unfound":0,"num_objects_dirty":27461,"num_whiteouts":0,"num_read":89,"num_read_kb":167,"num_write":27541,"num_write_kb":1756436,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":859,"num_bytes_recovered":55897213,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":29263,"ondisk_log_size":29263,"up":277,"acting":280,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":296,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":8058856,"kb_used_data":5478840,"kb_used_omap":8,"kb_used_meta":2579959,"kb_avail":554412056,"statfs":{"total":575970213888,"available":567717945344,"internally_reserved":0,"allocated":5610332160,"data_stored":5605167366,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8529,"internal_metadata":2641878703},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[18,6,0,2,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":641728512,"num_objects":9792,"num_object_clones":0,"num_object_copies":29376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1220,"num_objects_misplaced":-715,"num_objects_unfound":0,"num_objects_dirty":9792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9792,"num_write_kb":626688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":715,"num_bytes_recovered":46858240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.123128"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":281,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.345182+0000","last_change":"2023-09-23T23:19:17.016873+0000","last_active":"2023-09-23T23:19:19.345182+0000","last_peered":"2023-09-23T23:19:19.345182+0000","last_clean":"2023-09-23T23:19:19.345182+0000","last_became_active":"2023-09-23T23:19:17.016504+0000","last_became_peered":"2023-09-23T23:19:17.016504+0000","last_unstale":"2023-09-23T23:19:19.345182+0000","last_undegraded":"2023-09-23T23:19:19.345182+0000","last_fullsized":"2023-09-23T23:19:19.345182+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:36.198889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":215,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.340265+0000","last_change":"2023-09-23T23:19:17.015302+0000","last_active":"2023-09-23T23:19:19.340265+0000","last_peered":"2023-09-23T23:19:19.340265+0000","last_clean":"2023-09-23T23:19:19.340265+0000","last_became_active":"2023-09-23T23:19:17.014986+0000","last_became_peered":"2023-09-23T23:19:17.014986+0000","last_unstale":"2023-09-23T23:19:19.340265+0000","last_undegraded":"2023-09-23T23:19:19.340265+0000","last_fullsized":"2023-09-23T23:19:19.340265+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:57.782719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":123,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.340165+0000","last_change":"2023-09-23T23:19:17.011302+0000","last_active":"2023-09-23T23:19:19.340165+0000","last_peered":"2023-09-23T23:19:19.340165+0000","last_clean":"2023-09-23T23:19:19.340165+0000","last_became_active":"2023-09-23T23:19:17.010980+0000","last_became_peered":"2023-09-23T23:19:17.010980+0000","last_unstale":"2023-09-23T23:19:19.340165+0000","last_undegraded":"2023-09-23T23:19:19.340165+0000","last_fullsized":"2023-09-23T23:19:19.340165+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:41:55.794769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":245,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.336922+0000","last_change":"2023-09-23T23:19:17.012647+0000","last_active":"2023-09-23T23:19:19.336922+0000","last_peered":"2023-09-23T23:19:19.336922+0000","last_clean":"2023-09-23T23:19:19.336922+0000","last_became_active":"2023-09-23T23:19:17.011436+0000","last_became_peered":"2023-09-23T23:19:17.011436+0000","last_unstale":"2023-09-23T23:19:19.336922+0000","last_undegraded":"2023-09-23T23:19:19.336922+0000","last_fullsized":"2023-09-23T23:19:19.336922+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:15.251482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":242,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.339982+0000","last_change":"2023-09-23T23:17:48.305487+0000","last_active":"2023-09-23T23:19:19.339982+0000","last_peered":"2023-09-23T23:19:19.339982+0000","last_clean":"2023-09-23T23:19:19.339982+0000","last_became_active":"2023-09-23T23:17:48.305118+0000","last_became_peered":"2023-09-23T23:17:48.305118+0000","last_unstale":"2023-09-23T23:19:19.339982+0000","last_undegraded":"2023-09-23T23:19:19.339982+0000","last_fullsized":"2023-09-23T23:19:19.339982+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:07.958152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":236,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.342898+0000","last_change":"2023-09-23T23:19:17.015076+0000","last_active":"2023-09-23T23:19:19.342898+0000","last_peered":"2023-09-23T23:19:19.342898+0000","last_clean":"2023-09-23T23:19:19.342898+0000","last_became_active":"2023-09-23T23:19:17.014808+0000","last_became_peered":"2023-09-23T23:19:17.014808+0000","last_unstale":"2023-09-23T23:19:19.342898+0000","last_undegraded":"2023-09-23T23:19:19.342898+0000","last_fullsized":"2023-09-23T23:19:19.342898+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:14.865292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":217,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.336302+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:19:19.336302+0000","last_peered":"2023-09-23T23:19:19.336302+0000","last_clean":"2023-09-23T23:19:19.336302+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:19:19.336302+0000","last_undegraded":"2023-09-23T23:19:19.336302+0000","last_fullsized":"2023-09-23T23:19:19.336302+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":215,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.339735+0000","last_change":"2023-09-23T23:19:17.011518+0000","last_active":"2023-09-23T23:19:19.339735+0000","last_peered":"2023-09-23T23:19:19.339735+0000","last_clean":"2023-09-23T23:19:19.339735+0000","last_became_active":"2023-09-23T23:19:17.011244+0000","last_became_peered":"2023-09-23T23:19:17.011244+0000","last_unstale":"2023-09-23T23:19:19.339735+0000","last_undegraded":"2023-09-23T23:19:19.339735+0000","last_fullsized":"2023-09-23T23:19:19.339735+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:13.621271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":43,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.336672+0000","last_change":"2023-09-23T23:17:48.308746+0000","last_active":"2023-09-23T23:19:19.336672+0000","last_peered":"2023-09-23T23:19:19.336672+0000","last_clean":"2023-09-23T23:19:19.336672+0000","last_became_active":"2023-09-23T23:17:48.307974+0000","last_became_peered":"2023-09-23T23:17:48.307974+0000","last_unstale":"2023-09-23T23:19:19.336672+0000","last_undegraded":"2023-09-23T23:19:19.336672+0000","last_fullsized":"2023-09-23T23:19:19.336672+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:52.915506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":194,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.339863+0000","last_change":"2023-09-23T23:16:56.528057+0000","last_active":"2023-09-23T23:19:19.339863+0000","last_peered":"2023-09-23T23:19:19.339863+0000","last_clean":"2023-09-23T23:19:19.339863+0000","last_became_active":"2023-09-23T23:16:56.527755+0000","last_became_peered":"2023-09-23T23:16:56.527755+0000","last_unstale":"2023-09-23T23:19:19.339863+0000","last_undegraded":"2023-09-23T23:19:19.339863+0000","last_fullsized":"2023-09-23T23:19:19.339863+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:47.396406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":83,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T23:19:20.347811+0000","last_change":"2023-09-23T23:16:56.567128+0000","last_active":"2023-09-23T23:19:20.347811+0000","last_peered":"2023-09-23T23:19:20.347811+0000","last_clean":"2023-09-23T23:19:20.347811+0000","last_became_active":"2023-09-23T23:16:56.551484+0000","last_became_peered":"2023-09-23T23:16:56.551484+0000","last_unstale":"2023-09-23T23:19:20.347811+0000","last_undegraded":"2023-09-23T23:19:20.347811+0000","last_fullsized":"2023-09-23T23:19:20.347811+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:14.981307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":206,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.342507+0000","last_change":"2023-09-23T23:17:48.321527+0000","last_active":"2023-09-23T23:19:19.342507+0000","last_peered":"2023-09-23T23:19:19.342507+0000","last_clean":"2023-09-23T23:19:19.342507+0000","last_became_active":"2023-09-23T23:17:48.321198+0000","last_became_peered":"2023-09-23T23:17:48.321198+0000","last_unstale":"2023-09-23T23:19:19.342507+0000","last_undegraded":"2023-09-23T23:19:19.342507+0000","last_fullsized":"2023-09-23T23:19:19.342507+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:11.289469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":238,"reported_epoch":265,"state":"active+undersized+degraded","last_fresh":"2023-09-23T23:19:19.346821+0000","last_change":"2023-09-23T23:18:59.587055+0000","last_active":"2023-09-23T23:19:19.346821+0000","last_peered":"2023-09-23T23:19:19.346821+0000","last_clean":"2023-09-23T23:18:57.571526+0000","last_became_active":"2023-09-23T23:18:59.587055+0000","last_became_peered":"2023-09-23T23:18:59.587055+0000","last_unstale":"2023-09-23T23:19:19.346821+0000","last_undegraded":"2023-09-23T23:18:59.583427+0000","last_fullsized":"2023-09-23T23:18:59.583313+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:20.891248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":235,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.347287+0000","last_change":"2023-09-23T23:19:17.017877+0000","last_active":"2023-09-23T23:19:19.347287+0000","last_peered":"2023-09-23T23:19:19.347287+0000","last_clean":"2023-09-23T23:19:19.347287+0000","last_became_active":"2023-09-23T23:19:17.017590+0000","last_became_peered":"2023-09-23T23:19:17.017590+0000","last_unstale":"2023-09-23T23:19:19.347287+0000","last_undegraded":"2023-09-23T23:19:19.347287+0000","last_fullsized":"2023-09-23T23:19:19.347287+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:41.420908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":30,"reported_epoch":265,"state":"active+undersized+degraded","last_fresh":"2023-09-23T23:19:19.347155+0000","last_change":"2023-09-23T23:18:59.587104+0000","last_active":"2023-09-23T23:19:19.347155+0000","last_peered":"2023-09-23T23:19:19.347155+0000","last_clean":"2023-09-23T22:53:20.392856+0000","last_became_active":"2023-09-23T23:18:59.587104+0000","last_became_peered":"2023-09-23T23:18:59.587104+0000","last_unstale":"2023-09-23T23:19:19.347155+0000","last_undegraded":"2023-09-23T23:18:59.583778+0000","last_fullsized":"2023-09-23T23:18:59.583694+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:57:18.977523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":230,"reported_epoch":265,"state":"active+undersized","last_fresh":"2023-09-23T23:19:19.340188+0000","last_change":"2023-09-23T23:18:59.589458+0000","last_active":"2023-09-23T23:19:19.340188+0000","last_peered":"2023-09-23T23:19:19.340188+0000","last_clean":"2023-09-23T23:18:57.564268+0000","last_became_active":"2023-09-23T23:18:59.589458+0000","last_became_peered":"2023-09-23T23:18:59.589458+0000","last_unstale":"2023-09-23T23:19:19.340188+0000","last_undegraded":"2023-09-23T23:19:19.340188+0000","last_fullsized":"2023-09-23T23:18:59.579757+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:35.461328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":235,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.335794+0000","last_change":"2023-09-23T23:17:48.303810+0000","last_active":"2023-09-23T23:19:19.335794+0000","last_peered":"2023-09-23T23:19:19.335794+0000","last_clean":"2023-09-23T23:19:19.335794+0000","last_became_active":"2023-09-23T23:17:48.303505+0000","last_became_peered":"2023-09-23T23:17:48.303505+0000","last_unstale":"2023-09-23T23:19:19.335794+0000","last_undegraded":"2023-09-23T23:19:19.335794+0000","last_fullsized":"2023-09-23T23:19:19.335794+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:31.494085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":122,"reported_epoch":265,"state":"active+undersized","last_fresh":"2023-09-23T23:19:19.340356+0000","last_change":"2023-09-23T23:18:59.586926+0000","last_active":"2023-09-23T23:19:19.340356+0000","last_peered":"2023-09-23T23:19:19.340356+0000","last_clean":"2023-09-23T23:10:34.157578+0000","last_became_active":"2023-09-23T23:18:59.586926+0000","last_became_peered":"2023-09-23T23:18:59.586926+0000","last_unstale":"2023-09-23T23:19:19.340356+0000","last_undegraded":"2023-09-23T23:19:19.340356+0000","last_fullsized":"2023-09-23T23:18:59.579446+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:29.301032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":19,"reported_epoch":265,"state":"active+undersized","last_fresh":"2023-09-23T23:19:19.345044+0000","last_change":"2023-09-23T23:18:59.587971+0000","last_active":"2023-09-23T23:19:19.345044+0000","last_peered":"2023-09-23T23:19:19.345044+0000","last_clean":"2023-09-23T22:53:15.863481+0000","last_became_active":"2023-09-23T23:18:59.587971+0000","last_became_peered":"2023-09-23T23:18:59.587971+0000","last_unstale":"2023-09-23T23:19:19.345044+0000","last_undegraded":"2023-09-23T23:19:19.345044+0000","last_fullsized":"2023-09-23T23:18:59.584213+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:17.017106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":235,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.337082+0000","last_change":"2023-09-23T23:17:48.308900+0000","last_active":"2023-09-23T23:19:19.337082+0000","last_peered":"2023-09-23T23:19:19.337082+0000","last_clean":"2023-09-23T23:19:19.337082+0000","last_became_active":"2023-09-23T23:17:48.308475+0000","last_became_peered":"2023-09-23T23:17:48.308475+0000","last_unstale":"2023-09-23T23:19:19.337082+0000","last_undegraded":"2023-09-23T23:19:19.337082+0000","last_fullsized":"2023-09-23T23:19:19.337082+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:38.158435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":255,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.344386+0000","last_change":"2023-09-23T23:13:29.368175+0000","last_active":"2023-09-23T23:19:19.344386+0000","last_peered":"2023-09-23T23:19:19.344386+0000","last_clean":"2023-09-23T23:19:19.344386+0000","last_became_active":"2023-09-23T23:13:29.363603+0000","last_became_peered":"2023-09-23T23:13:29.363603+0000","last_unstale":"2023-09-23T23:19:19.344386+0000","last_undegraded":"2023-09-23T23:19:19.344386+0000","last_fullsized":"2023-09-23T23:19:19.344386+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:25.277871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":129,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.343110+0000","last_change":"2023-09-23T23:19:17.014811+0000","last_active":"2023-09-23T23:19:19.343110+0000","last_peered":"2023-09-23T23:19:19.343110+0000","last_clean":"2023-09-23T23:19:19.343110+0000","last_became_active":"2023-09-23T23:19:17.014567+0000","last_became_peered":"2023-09-23T23:19:17.014567+0000","last_unstale":"2023-09-23T23:19:19.343110+0000","last_undegraded":"2023-09-23T23:19:19.343110+0000","last_fullsized":"2023-09-23T23:19:19.343110+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:51.955182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"266'1689","reported_seq":1708,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T23:19:24.011609+0000","last_change":"2023-09-23T23:18:58.587565+0000","last_active":"2023-09-23T23:19:24.011609+0000","last_peered":"2023-09-23T23:19:24.011609+0000","last_clean":"2023-09-23T23:19:24.011609+0000","last_became_active":"2023-09-23T23:18:58.587251+0000","last_became_peered":"2023-09-23T23:18:58.587251+0000","last_unstale":"2023-09-23T23:19:24.011609+0000","last_undegraded":"2023-09-23T23:19:24.011609+0000","last_fullsized":"2023-09-23T23:19:24.011609+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":1689,"log_dups_size":0,"ondisk_log_size":1689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:30.475182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110690304,"num_objects":1689,"num_object_clones":0,"num_object_copies":5067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1689,"num_write_kb":108096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":80,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.351043+0000","last_change":"2023-09-23T23:16:56.548535+0000","last_active":"2023-09-23T23:19:19.351043+0000","last_peered":"2023-09-23T23:19:19.351043+0000","last_clean":"2023-09-23T23:19:19.351043+0000","last_became_active":"2023-09-23T23:16:56.546115+0000","last_became_peered":"2023-09-23T23:16:56.546115+0000","last_unstale":"2023-09-23T23:19:19.351043+0000","last_undegraded":"2023-09-23T23:19:19.351043+0000","last_fullsized":"2023-09-23T23:19:19.351043+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:02.279715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":71,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.344477+0000","last_change":"2023-09-23T23:16:56.525479+0000","last_active":"2023-09-23T23:19:19.344477+0000","last_peered":"2023-09-23T23:19:19.344477+0000","last_clean":"2023-09-23T23:19:19.344477+0000","last_became_active":"2023-09-23T23:16:56.525147+0000","last_became_peered":"2023-09-23T23:16:56.525147+0000","last_unstale":"2023-09-23T23:19:19.344477+0000","last_undegraded":"2023-09-23T23:19:19.344477+0000","last_fullsized":"2023-09-23T23:19:19.344477+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:06.165300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.7","version":"266'1669","reported_seq":1688,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T23:19:24.276398+0000","last_change":"2023-09-23T23:18:58.587718+0000","last_active":"2023-09-23T23:19:24.276398+0000","last_peered":"2023-09-23T23:19:24.276398+0000","last_clean":"2023-09-23T23:19:24.276398+0000","last_became_active":"2023-09-23T23:18:58.587406+0000","last_became_peered":"2023-09-23T23:18:58.587406+0000","last_unstale":"2023-09-23T23:19:24.276398+0000","last_undegraded":"2023-09-23T23:19:24.276398+0000","last_fullsized":"2023-09-23T23:19:24.276398+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":1669,"log_dups_size":0,"ondisk_log_size":1669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:15.209187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109379584,"num_objects":1669,"num_object_clones":0,"num_object_copies":5007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1669,"num_write_kb":106816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":94,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.350050+0000","last_change":"2023-09-23T23:19:17.021093+0000","last_active":"2023-09-23T23:19:19.350050+0000","last_peered":"2023-09-23T23:19:19.350050+0000","last_clean":"2023-09-23T23:19:19.350050+0000","last_became_active":"2023-09-23T23:19:17.012434+0000","last_became_peered":"2023-09-23T23:19:17.012434+0000","last_unstale":"2023-09-23T23:19:19.350050+0000","last_undegraded":"2023-09-23T23:19:19.350050+0000","last_fullsized":"2023-09-23T23:19:19.350050+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:06.993778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":262,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.345123+0000","last_change":"2023-09-23T23:19:17.017714+0000","last_active":"2023-09-23T23:19:19.345123+0000","last_peered":"2023-09-23T23:19:19.345123+0000","last_clean":"2023-09-23T23:19:19.345123+0000","last_became_active":"2023-09-23T23:19:17.017400+0000","last_became_peered":"2023-09-23T23:19:17.017400+0000","last_unstale":"2023-09-23T23:19:19.345123+0000","last_undegraded":"2023-09-23T23:19:19.345123+0000","last_fullsized":"2023-09-23T23:19:19.345123+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:43.604654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"266'1718","reported_seq":1738,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T23:19:24.511710+0000","last_change":"2023-09-23T23:18:58.585739+0000","last_active":"2023-09-23T23:19:24.511710+0000","last_peered":"2023-09-23T23:19:24.511710+0000","last_clean":"2023-09-23T23:19:24.511710+0000","last_became_active":"2023-09-23T23:18:58.585475+0000","last_became_peered":"2023-09-23T23:18:58.585475+0000","last_unstale":"2023-09-23T23:19:24.511710+0000","last_undegraded":"2023-09-23T23:19:24.511710+0000","last_fullsized":"2023-09-23T23:19:24.511710+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":1718,"log_dups_size":0,"ondisk_log_size":1718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:14.419790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112590848,"num_objects":1718,"num_object_clones":0,"num_object_copies":5154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1718,"num_write_kb":109952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":30,"reported_epoch":265,"state":"active+undersized+degraded","last_fresh":"2023-09-23T23:19:19.342268+0000","last_change":"2023-09-23T23:18:59.587709+0000","last_active":"2023-09-23T23:19:19.342268+0000","last_peered":"2023-09-23T23:19:19.342268+0000","last_clean":"2023-09-23T22:53:20.393182+0000","last_became_active":"2023-09-23T23:18:59.587709+0000","last_became_peered":"2023-09-23T23:18:59.587709+0000","last_unstale":"2023-09-23T23:19:19.342268+0000","last_undegraded":"2023-09-23T23:18:59.585429+0000","last_fullsized":"2023-09-23T23:18:59.585311+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:58.759914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":283,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.342840+0000","last_change":"2023-09-23T23:19:17.013984+0000","last_active":"2023-09-23T23:19:19.342840+0000","last_peered":"2023-09-23T23:19:19.342840+0000","last_clean":"2023-09-23T23:19:19.342840+0000","last_became_active":"2023-09-23T23:19:17.013418+0000","last_became_peered":"2023-09-23T23:19:17.013418+0000","last_unstale":"2023-09-23T23:19:19.342840+0000","last_undegraded":"2023-09-23T23:19:19.342840+0000","last_fullsized":"2023-09-23T23:19:19.342840+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:08.483999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"266'1796","reported_seq":1816,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T23:19:24.510602+0000","last_change":"2023-09-23T23:18:58.585588+0000","last_active":"2023-09-23T23:19:24.510602+0000","last_peered":"2023-09-23T23:19:24.510602+0000","last_clean":"2023-09-23T23:19:24.510602+0000","last_became_active":"2023-09-23T23:18:58.585303+0000","last_became_peered":"2023-09-23T23:18:58.585303+0000","last_unstale":"2023-09-23T23:19:24.510602+0000","last_undegraded":"2023-09-23T23:19:24.510602+0000","last_fullsized":"2023-09-23T23:19:24.510602+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":1796,"log_dups_size":0,"ondisk_log_size":1796,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:56.689193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117702656,"num_objects":1796,"num_object_clones":0,"num_object_copies":5388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1796,"num_write_kb":114944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":227,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.346365+0000","last_change":"2023-09-23T23:10:59.502426+0000","last_active":"2023-09-23T23:19:19.346365+0000","last_peered":"2023-09-23T23:19:19.346365+0000","last_clean":"2023-09-23T23:19:19.346365+0000","last_became_active":"2023-09-23T23:10:59.502074+0000","last_became_peered":"2023-09-23T23:10:59.502074+0000","last_unstale":"2023-09-23T23:19:19.346365+0000","last_undegraded":"2023-09-23T23:19:19.346365+0000","last_fullsized":"2023-09-23T23:19:19.346365+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:46.029347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":229,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.335960+0000","last_change":"2023-09-23T23:19:17.005947+0000","last_active":"2023-09-23T23:19:19.335960+0000","last_peered":"2023-09-23T23:19:19.335960+0000","last_clean":"2023-09-23T23:19:19.335960+0000","last_became_active":"2023-09-23T23:19:17.005632+0000","last_became_peered":"2023-09-23T23:19:17.005632+0000","last_unstale":"2023-09-23T23:19:19.335960+0000","last_undegraded":"2023-09-23T23:19:19.335960+0000","last_fullsized":"2023-09-23T23:19:19.335960+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:53.270709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"266'1774","reported_seq":1793,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T23:19:24.401684+0000","last_change":"2023-09-23T23:18:58.587100+0000","last_active":"2023-09-23T23:19:24.401684+0000","last_peered":"2023-09-23T23:19:24.401684+0000","last_clean":"2023-09-23T23:19:24.401684+0000","last_became_active":"2023-09-23T23:18:58.586837+0000","last_became_peered":"2023-09-23T23:18:58.586837+0000","last_unstale":"2023-09-23T23:19:24.401684+0000","last_undegraded":"2023-09-23T23:19:24.401684+0000","last_fullsized":"2023-09-23T23:19:24.401684+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":1774,"log_dups_size":0,"ondisk_log_size":1774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:56.131536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116260864,"num_objects":1774,"num_object_clones":0,"num_object_copies":5322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1774,"num_write_kb":113536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":552,"reported_epoch":265,"state":"active+undersized","last_fresh":"2023-09-23T23:19:19.339737+0000","last_change":"2023-09-23T23:18:59.589553+0000","last_active":"2023-09-23T23:19:19.339737+0000","last_peered":"2023-09-23T23:19:19.339737+0000","last_clean":"2023-09-23T23:14:37.277228+0000","last_became_active":"2023-09-23T23:18:59.589553+0000","last_became_peered":"2023-09-23T23:18:59.589553+0000","last_unstale":"2023-09-23T23:19:19.339737+0000","last_undegraded":"2023-09-23T23:19:19.339737+0000","last_fullsized":"2023-09-23T23:18:59.579101+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:46.150686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:46.150686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":230,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.340036+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:19:19.340036+0000","last_peered":"2023-09-23T23:19:19.340036+0000","last_clean":"2023-09-23T23:19:19.340036+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:19:19.340036+0000","last_undegraded":"2023-09-23T23:19:19.340036+0000","last_fullsized":"2023-09-23T23:19:19.340036+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":225,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.346945+0000","last_change":"2023-09-23T23:10:59.512363+0000","last_active":"2023-09-23T23:19:19.346945+0000","last_peered":"2023-09-23T23:19:19.346945+0000","last_clean":"2023-09-23T23:19:19.346945+0000","last_became_active":"2023-09-23T23:10:59.512022+0000","last_became_peered":"2023-09-23T23:10:59.512022+0000","last_unstale":"2023-09-23T23:19:19.346945+0000","last_undegraded":"2023-09-23T23:19:19.346945+0000","last_fullsized":"2023-09-23T23:19:19.346945+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:45.523432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"266'1687","reported_seq":1706,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T23:19:24.004171+0000","last_change":"2023-09-23T23:18:58.589210+0000","last_active":"2023-09-23T23:19:24.004171+0000","last_peered":"2023-09-23T23:19:24.004171+0000","last_clean":"2023-09-23T23:19:24.004171+0000","last_became_active":"2023-09-23T23:18:58.587940+0000","last_became_peered":"2023-09-23T23:18:58.587940+0000","last_unstale":"2023-09-23T23:19:24.004171+0000","last_undegraded":"2023-09-23T23:19:24.004171+0000","last_fullsized":"2023-09-23T23:19:24.004171+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":1687,"log_dups_size":0,"ondisk_log_size":1687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:28.175000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110559232,"num_objects":1687,"num_object_clones":0,"num_object_copies":5061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1687,"num_write_kb":107968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":478,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.339722+0000","last_change":"2023-09-23T23:18:42.871236+0000","last_active":"2023-09-23T23:19:19.339722+0000","last_peered":"2023-09-23T23:19:19.339722+0000","last_clean":"2023-09-23T23:19:19.339722+0000","last_became_active":"2023-09-23T23:17:51.330912+0000","last_became_peered":"2023-09-23T23:17:51.330912+0000","last_unstale":"2023-09-23T23:19:19.339722+0000","last_undegraded":"2023-09-23T23:19:19.339722+0000","last_fullsized":"2023-09-23T23:19:19.339722+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:42.871134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:42.871134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:55:02.683599+0000","scrub_duration":0.0023428590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":237,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.342912+0000","last_change":"2023-09-23T23:16:56.525904+0000","last_active":"2023-09-23T23:19:19.342912+0000","last_peered":"2023-09-23T23:19:19.342912+0000","last_clean":"2023-09-23T23:19:19.342912+0000","last_became_active":"2023-09-23T23:16:56.525372+0000","last_became_peered":"2023-09-23T23:16:56.525372+0000","last_unstale":"2023-09-23T23:19:19.342912+0000","last_undegraded":"2023-09-23T23:19:19.342912+0000","last_fullsized":"2023-09-23T23:19:19.342912+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:17.070117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":279,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.349573+0000","last_change":"2023-09-23T23:19:17.017772+0000","last_active":"2023-09-23T23:19:19.349573+0000","last_peered":"2023-09-23T23:19:19.349573+0000","last_clean":"2023-09-23T23:19:19.349573+0000","last_became_active":"2023-09-23T23:19:17.017447+0000","last_became_peered":"2023-09-23T23:19:17.017447+0000","last_unstale":"2023-09-23T23:19:19.349573+0000","last_undegraded":"2023-09-23T23:19:19.349573+0000","last_fullsized":"2023-09-23T23:19:19.349573+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:13.658417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"266'1691","reported_seq":3406,"reported_epoch":266,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:19:24.281319+0000","last_change":"2023-09-23T23:19:18.050908+0000","last_active":"2023-09-23T23:19:24.281319+0000","last_peered":"2023-09-23T23:19:24.281319+0000","last_clean":"2023-09-23T23:19:15.985850+0000","last_became_active":"2023-09-23T23:19:18.038344+0000","last_became_peered":"2023-09-23T23:19:18.038344+0000","last_unstale":"2023-09-23T23:19:24.281319+0000","last_undegraded":"2023-09-23T23:19:18.012962+0000","last_fullsized":"2023-09-23T23:19:18.006408+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":1691,"log_dups_size":0,"ondisk_log_size":1691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:22.371443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110821376,"num_objects":1691,"num_object_clones":0,"num_object_copies":5073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":131,"num_objects_unfound":0,"num_objects_dirty":1691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1691,"num_write_kb":108224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":843,"num_bytes_recovered":55246848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":131},{"shards":"4,6","objects":1559}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":272,"reported_epoch":265,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:19:19.344993+0000","last_change":"2023-09-23T23:19:18.046973+0000","last_active":"2023-09-23T23:19:19.344993+0000","last_peered":"2023-09-23T23:19:19.344993+0000","last_clean":"2023-09-23T23:19:19.344993+0000","last_became_active":"2023-09-23T23:19:18.046575+0000","last_became_peered":"2023-09-23T23:19:18.046575+0000","last_unstale":"2023-09-23T23:19:19.344993+0000","last_undegraded":"2023-09-23T23:19:19.344993+0000","last_fullsized":"2023-09-23T23:19:19.344993+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:26.884470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5,1],"avail_no_missing":["4","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":235,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.350068+0000","last_change":"2023-09-23T23:15:28.228439+0000","last_active":"2023-09-23T23:19:19.350068+0000","last_peered":"2023-09-23T23:19:19.350068+0000","last_clean":"2023-09-23T23:19:19.350068+0000","last_became_active":"2023-09-23T23:15:28.228095+0000","last_became_peered":"2023-09-23T23:15:28.228095+0000","last_unstale":"2023-09-23T23:19:19.350068+0000","last_undegraded":"2023-09-23T23:19:19.350068+0000","last_fullsized":"2023-09-23T23:19:19.350068+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:37.667201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"266'1719","reported_seq":1737,"reported_epoch":266,"state":"active+undersized+degraded","last_fresh":"2023-09-23T23:19:24.429746+0000","last_change":"2023-09-23T23:19:10.823960+0000","last_active":"2023-09-23T23:19:24.429746+0000","last_peered":"2023-09-23T23:19:24.429746+0000","last_clean":"2023-09-23T23:18:57.558582+0000","last_became_active":"2023-09-23T23:18:59.585863+0000","last_became_peered":"2023-09-23T23:18:59.585863+0000","last_unstale":"2023-09-23T23:19:24.429746+0000","last_undegraded":"2023-09-23T23:19:10.823681+0000","last_fullsized":"2023-09-23T23:18:59.582693+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":255,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":1719,"log_dups_size":0,"ondisk_log_size":1719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:26.738056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112656384,"num_objects":1719,"num_object_clones":0,"num_object_copies":5157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1719,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1719,"num_write_kb":110016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":1719}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":546,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.343685+0000","last_change":"2023-09-23T23:17:51.348000+0000","last_active":"2023-09-23T23:19:19.343685+0000","last_peered":"2023-09-23T23:19:19.343685+0000","last_clean":"2023-09-23T23:19:19.343685+0000","last_became_active":"2023-09-23T23:17:51.347662+0000","last_became_peered":"2023-09-23T23:17:51.347662+0000","last_unstale":"2023-09-23T23:19:19.343685+0000","last_undegraded":"2023-09-23T23:19:19.343685+0000","last_fullsized":"2023-09-23T23:19:19.343685+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:51.734652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:51.734652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:38.179876+0000","scrub_duration":0.023100959000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":97,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.350727+0000","last_change":"2023-09-23T23:19:17.016882+0000","last_active":"2023-09-23T23:19:19.350727+0000","last_peered":"2023-09-23T23:19:19.350727+0000","last_clean":"2023-09-23T23:19:19.350727+0000","last_became_active":"2023-09-23T23:19:17.016552+0000","last_became_peered":"2023-09-23T23:19:17.016552+0000","last_unstale":"2023-09-23T23:19:19.350727+0000","last_undegraded":"2023-09-23T23:19:19.350727+0000","last_fullsized":"2023-09-23T23:19:19.350727+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:58.565086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":123,"reported_epoch":265,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:19:19.340656+0000","last_change":"2023-09-23T23:19:18.010787+0000","last_active":"2023-09-23T23:19:19.340656+0000","last_peered":"2023-09-23T23:19:19.340656+0000","last_clean":"2023-09-23T23:19:19.340656+0000","last_became_active":"2023-09-23T23:19:18.010376+0000","last_became_peered":"2023-09-23T23:19:18.010376+0000","last_unstale":"2023-09-23T23:19:19.340656+0000","last_undegraded":"2023-09-23T23:19:19.340656+0000","last_fullsized":"2023-09-23T23:19:19.340656+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:43:05.036819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4,1],"avail_no_missing":["3","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"266'1799","reported_seq":1814,"reported_epoch":266,"state":"active+undersized+degraded","last_fresh":"2023-09-23T23:19:24.436784+0000","last_change":"2023-09-23T23:19:10.829313+0000","last_active":"2023-09-23T23:19:24.436784+0000","last_peered":"2023-09-23T23:19:24.436784+0000","last_clean":"2023-09-23T23:18:57.558582+0000","last_became_active":"2023-09-23T23:18:59.585882+0000","last_became_peered":"2023-09-23T23:18:59.585882+0000","last_unstale":"2023-09-23T23:19:24.436784+0000","last_undegraded":"2023-09-23T23:19:10.822953+0000","last_fullsized":"2023-09-23T23:18:59.582773+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":255,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":1799,"log_dups_size":0,"ondisk_log_size":1799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:12.870079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117899264,"num_objects":1799,"num_object_clones":0,"num_object_copies":5397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1799,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1799,"num_write_kb":115136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":1799}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":599,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.342992+0000","last_change":"2023-09-23T23:17:51.355247+0000","last_active":"2023-09-23T23:19:19.342992+0000","last_peered":"2023-09-23T23:19:19.342992+0000","last_clean":"2023-09-23T23:19:19.342992+0000","last_became_active":"2023-09-23T23:17:51.354856+0000","last_became_peered":"2023-09-23T23:17:51.354856+0000","last_unstale":"2023-09-23T23:19:19.342992+0000","last_undegraded":"2023-09-23T23:19:19.342992+0000","last_fullsized":"2023-09-23T23:19:19.342992+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:47.797085+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:47.797085+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:55.878838+0000","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":236,"reported_epoch":265,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:19:19.511609+0000","last_change":"2023-09-23T23:19:18.009849+0000","last_active":"2023-09-23T23:19:19.511609+0000","last_peered":"2023-09-23T23:19:19.511609+0000","last_clean":"2023-09-23T23:19:19.511609+0000","last_became_active":"2023-09-23T23:19:18.009412+0000","last_became_peered":"2023-09-23T23:19:18.009412+0000","last_unstale":"2023-09-23T23:19:19.511609+0000","last_undegraded":"2023-09-23T23:19:19.511609+0000","last_fullsized":"2023-09-23T23:19:19.511609+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:47.883653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4,1],"avail_no_missing":["0","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":271,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.343525+0000","last_change":"2023-09-23T23:17:48.318576+0000","last_active":"2023-09-23T23:19:19.343525+0000","last_peered":"2023-09-23T23:19:19.343525+0000","last_clean":"2023-09-23T23:19:19.343525+0000","last_became_active":"2023-09-23T23:17:48.302180+0000","last_became_peered":"2023-09-23T23:17:48.302180+0000","last_unstale":"2023-09-23T23:19:19.343525+0000","last_undegraded":"2023-09-23T23:19:19.343525+0000","last_fullsized":"2023-09-23T23:19:19.343525+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:34:56.665520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":206,"reported_epoch":265,"state":"active+undersized","last_fresh":"2023-09-23T23:19:19.345250+0000","last_change":"2023-09-23T23:18:59.585186+0000","last_active":"2023-09-23T23:19:19.345250+0000","last_peered":"2023-09-23T23:19:19.345250+0000","last_clean":"2023-09-23T23:18:57.571469+0000","last_became_active":"2023-09-23T23:18:59.585186+0000","last_became_peered":"2023-09-23T23:18:59.585186+0000","last_unstale":"2023-09-23T23:19:19.345250+0000","last_undegraded":"2023-09-23T23:19:19.345250+0000","last_fullsized":"2023-09-23T23:18:59.582686+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:16.235263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.a","version":"266'1673","reported_seq":1692,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T23:19:24.267844+0000","last_change":"2023-09-23T23:18:58.585142+0000","last_active":"2023-09-23T23:19:24.267844+0000","last_peered":"2023-09-23T23:19:24.267844+0000","last_clean":"2023-09-23T23:19:24.267844+0000","last_became_active":"2023-09-23T23:18:58.584860+0000","last_became_peered":"2023-09-23T23:18:58.584860+0000","last_unstale":"2023-09-23T23:19:24.267844+0000","last_undegraded":"2023-09-23T23:19:24.267844+0000","last_fullsized":"2023-09-23T23:19:24.267844+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":1673,"log_dups_size":0,"ondisk_log_size":1673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:14.574839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109641728,"num_objects":1673,"num_object_clones":0,"num_object_copies":5019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1673,"num_write_kb":107072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":546,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.343486+0000","last_change":"2023-09-23T23:17:51.342323+0000","last_active":"2023-09-23T23:19:19.343486+0000","last_peered":"2023-09-23T23:19:19.343486+0000","last_clean":"2023-09-23T23:19:19.343486+0000","last_became_active":"2023-09-23T23:17:51.341930+0000","last_became_peered":"2023-09-23T23:17:51.341930+0000","last_unstale":"2023-09-23T23:19:19.343486+0000","last_undegraded":"2023-09-23T23:19:19.343486+0000","last_fullsized":"2023-09-23T23:19:19.343486+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:49.726740+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:49.726740+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:16:57.092869+0000","scrub_duration":0.0013902070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":241,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.343889+0000","last_change":"2023-09-23T23:19:17.011744+0000","last_active":"2023-09-23T23:19:19.343889+0000","last_peered":"2023-09-23T23:19:19.343889+0000","last_clean":"2023-09-23T23:19:19.343889+0000","last_became_active":"2023-09-23T23:19:17.011490+0000","last_became_peered":"2023-09-23T23:19:17.011490+0000","last_unstale":"2023-09-23T23:19:19.343889+0000","last_undegraded":"2023-09-23T23:19:19.343889+0000","last_fullsized":"2023-09-23T23:19:19.343889+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:03.744588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":226,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.347049+0000","last_change":"2023-09-23T23:04:11.514703+0000","last_active":"2023-09-23T23:19:19.347049+0000","last_peered":"2023-09-23T23:19:19.347049+0000","last_clean":"2023-09-23T23:19:19.347049+0000","last_became_active":"2023-09-23T23:04:11.514403+0000","last_became_peered":"2023-09-23T23:04:11.514403+0000","last_unstale":"2023-09-23T23:19:19.347049+0000","last_undegraded":"2023-09-23T23:19:19.347049+0000","last_fullsized":"2023-09-23T23:19:19.347049+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:26.374346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":234,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.340107+0000","last_change":"2023-09-23T23:17:48.300618+0000","last_active":"2023-09-23T23:19:19.340107+0000","last_peered":"2023-09-23T23:19:19.340107+0000","last_clean":"2023-09-23T23:19:19.340107+0000","last_became_active":"2023-09-23T23:17:48.300279+0000","last_became_peered":"2023-09-23T23:17:48.300279+0000","last_unstale":"2023-09-23T23:19:19.340107+0000","last_undegraded":"2023-09-23T23:19:19.340107+0000","last_fullsized":"2023-09-23T23:19:19.340107+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:53.227630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.9","version":"266'1688","reported_seq":1706,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T23:19:24.439341+0000","last_change":"2023-09-23T23:18:58.584371+0000","last_active":"2023-09-23T23:19:24.439341+0000","last_peered":"2023-09-23T23:19:24.439341+0000","last_clean":"2023-09-23T23:19:24.439341+0000","last_became_active":"2023-09-23T23:18:58.584076+0000","last_became_peered":"2023-09-23T23:18:58.584076+0000","last_unstale":"2023-09-23T23:19:24.439341+0000","last_undegraded":"2023-09-23T23:19:24.439341+0000","last_fullsized":"2023-09-23T23:19:24.439341+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":1688,"log_dups_size":0,"ondisk_log_size":1688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:47:02.949295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110624768,"num_objects":1688,"num_object_clones":0,"num_object_copies":5064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1688,"num_write_kb":108032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":543,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.335878+0000","last_change":"2023-09-23T23:18:47.277223+0000","last_active":"2023-09-23T23:19:19.335878+0000","last_peered":"2023-09-23T23:19:19.335878+0000","last_clean":"2023-09-23T23:19:19.335878+0000","last_became_active":"2023-09-23T23:17:51.342020+0000","last_became_peered":"2023-09-23T23:17:51.342020+0000","last_unstale":"2023-09-23T23:19:19.335878+0000","last_undegraded":"2023-09-23T23:19:19.335878+0000","last_fullsized":"2023-09-23T23:19:19.335878+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:47.277080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:47.277080+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:42.406644+0000","scrub_duration":0.001458104,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":203,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.340025+0000","last_change":"2023-09-23T23:19:17.011488+0000","last_active":"2023-09-23T23:19:19.340025+0000","last_peered":"2023-09-23T23:19:19.340025+0000","last_clean":"2023-09-23T23:19:19.340025+0000","last_became_active":"2023-09-23T23:19:17.010995+0000","last_became_peered":"2023-09-23T23:19:17.010995+0000","last_unstale":"2023-09-23T23:19:19.340025+0000","last_undegraded":"2023-09-23T23:19:19.340025+0000","last_fullsized":"2023-09-23T23:19:19.340025+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:41.325675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":239,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.344700+0000","last_change":"2023-09-23T23:19:17.012899+0000","last_active":"2023-09-23T23:19:19.344700+0000","last_peered":"2023-09-23T23:19:19.344700+0000","last_clean":"2023-09-23T23:19:19.344700+0000","last_became_active":"2023-09-23T23:19:17.011998+0000","last_became_peered":"2023-09-23T23:19:17.011998+0000","last_unstale":"2023-09-23T23:19:19.344700+0000","last_undegraded":"2023-09-23T23:19:19.344700+0000","last_fullsized":"2023-09-23T23:19:19.344700+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:23.556143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":158,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.343806+0000","last_change":"2023-09-23T23:16:56.545792+0000","last_active":"2023-09-23T23:19:19.343806+0000","last_peered":"2023-09-23T23:19:19.343806+0000","last_clean":"2023-09-23T23:19:19.343806+0000","last_became_active":"2023-09-23T23:16:56.545447+0000","last_became_peered":"2023-09-23T23:16:56.545447+0000","last_unstale":"2023-09-23T23:19:19.343806+0000","last_undegraded":"2023-09-23T23:19:19.343806+0000","last_fullsized":"2023-09-23T23:19:19.343806+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:47.781440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"266'1660","reported_seq":1688,"reported_epoch":266,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:19:24.276370+0000","last_change":"2023-09-23T23:19:18.046035+0000","last_active":"2023-09-23T23:19:24.276370+0000","last_peered":"2023-09-23T23:19:24.276370+0000","last_clean":"2023-09-23T23:19:15.994777+0000","last_became_active":"2023-09-23T23:19:18.035908+0000","last_became_peered":"2023-09-23T23:19:18.035908+0000","last_unstale":"2023-09-23T23:19:24.276370+0000","last_undegraded":"2023-09-23T23:19:18.013376+0000","last_fullsized":"2023-09-23T23:19:18.005966+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":1660,"log_dups_size":0,"ondisk_log_size":1660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:06.444716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108789760,"num_objects":1660,"num_object_clones":0,"num_object_copies":4980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":916,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1660,"num_write_kb":106240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0],"avail_no_missing":["6","0","1"],"object_location_counts":[{"shards":"0,1,6","objects":915},{"shards":"0,6","objects":744}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":268,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.343295+0000","last_change":"2023-09-23T23:17:48.302538+0000","last_active":"2023-09-23T23:19:19.343295+0000","last_peered":"2023-09-23T23:19:19.343295+0000","last_clean":"2023-09-23T23:19:19.343295+0000","last_became_active":"2023-09-23T23:17:48.302203+0000","last_became_peered":"2023-09-23T23:17:48.302203+0000","last_unstale":"2023-09-23T23:19:19.343295+0000","last_undegraded":"2023-09-23T23:19:19.343295+0000","last_fullsized":"2023-09-23T23:19:19.343295+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:25.857118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":502,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.355448+0000","last_change":"2023-09-23T23:19:19.353335+0000","last_active":"2023-09-23T23:19:19.355448+0000","last_peered":"2023-09-23T23:19:19.355448+0000","last_clean":"2023-09-23T23:19:19.355448+0000","last_became_active":"2023-09-23T23:19:19.353004+0000","last_became_peered":"2023-09-23T23:19:19.353004+0000","last_unstale":"2023-09-23T23:19:19.355448+0000","last_undegraded":"2023-09-23T23:19:19.355448+0000","last_fullsized":"2023-09-23T23:19:19.355448+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:42.190868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:42.190868+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:08:33.637016+0000","scrub_duration":0.0049256480000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":194,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.339891+0000","last_change":"2023-09-23T23:16:56.525298+0000","last_active":"2023-09-23T23:19:19.339891+0000","last_peered":"2023-09-23T23:19:19.339891+0000","last_clean":"2023-09-23T23:19:19.339891+0000","last_became_active":"2023-09-23T23:16:56.524974+0000","last_became_peered":"2023-09-23T23:16:56.524974+0000","last_unstale":"2023-09-23T23:19:19.339891+0000","last_undegraded":"2023-09-23T23:19:19.339891+0000","last_fullsized":"2023-09-23T23:19:19.339891+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:08.860790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":216,"reported_epoch":265,"state":"active+undersized","last_fresh":"2023-09-23T23:19:19.342143+0000","last_change":"2023-09-23T23:18:59.587941+0000","last_active":"2023-09-23T23:19:19.342143+0000","last_peered":"2023-09-23T23:19:19.342143+0000","last_clean":"2023-09-23T23:18:57.574024+0000","last_became_active":"2023-09-23T23:18:59.587941+0000","last_became_peered":"2023-09-23T23:18:59.587941+0000","last_unstale":"2023-09-23T23:19:19.342143+0000","last_undegraded":"2023-09-23T23:19:19.342143+0000","last_fullsized":"2023-09-23T23:18:59.585813+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:14:05.018221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"266'1785","reported_seq":1805,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T23:19:24.504712+0000","last_change":"2023-09-23T23:18:58.585721+0000","last_active":"2023-09-23T23:19:24.504712+0000","last_peered":"2023-09-23T23:19:24.504712+0000","last_clean":"2023-09-23T23:19:24.504712+0000","last_became_active":"2023-09-23T23:18:58.585264+0000","last_became_peered":"2023-09-23T23:18:58.585264+0000","last_unstale":"2023-09-23T23:19:24.504712+0000","last_undegraded":"2023-09-23T23:19:24.504712+0000","last_fullsized":"2023-09-23T23:19:24.504712+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":1785,"log_dups_size":0,"ondisk_log_size":1785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:02:49.312053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116981760,"num_objects":1785,"num_object_clones":0,"num_object_copies":5355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1785,"num_write_kb":114240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":426,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.353859+0000","last_change":"2023-09-23T23:19:19.353561+0000","last_active":"2023-09-23T23:19:19.353859+0000","last_peered":"2023-09-23T23:19:19.353859+0000","last_clean":"2023-09-23T23:19:19.353859+0000","last_became_active":"2023-09-23T23:19:19.353289+0000","last_became_peered":"2023-09-23T23:19:19.353289+0000","last_unstale":"2023-09-23T23:19:19.353859+0000","last_undegraded":"2023-09-23T23:19:19.353859+0000","last_fullsized":"2023-09-23T23:19:19.353859+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:44.629771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:44.629771+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":214,"reported_epoch":265,"state":"active+undersized","last_fresh":"2023-09-23T23:19:19.336493+0000","last_change":"2023-09-23T23:18:59.580395+0000","last_active":"2023-09-23T23:19:19.336493+0000","last_peered":"2023-09-23T23:19:19.336493+0000","last_clean":"2023-09-23T23:16:52.490944+0000","last_became_active":"2023-09-23T23:18:59.580395+0000","last_became_peered":"2023-09-23T23:18:59.580395+0000","last_unstale":"2023-09-23T23:19:19.336493+0000","last_undegraded":"2023-09-23T23:19:19.336493+0000","last_fullsized":"2023-09-23T23:18:59.577313+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:36.323937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":227,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.336176+0000","last_change":"2023-09-23T23:16:56.525850+0000","last_active":"2023-09-23T23:19:19.336176+0000","last_peered":"2023-09-23T23:19:19.336176+0000","last_clean":"2023-09-23T23:19:19.336176+0000","last_became_active":"2023-09-23T23:16:56.524998+0000","last_became_peered":"2023-09-23T23:16:56.524998+0000","last_unstale":"2023-09-23T23:19:19.336176+0000","last_undegraded":"2023-09-23T23:19:19.336176+0000","last_fullsized":"2023-09-23T23:19:19.336176+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:54.317877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":266,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.511716+0000","last_change":"2023-09-23T23:19:17.012781+0000","last_active":"2023-09-23T23:19:19.511716+0000","last_peered":"2023-09-23T23:19:19.511716+0000","last_clean":"2023-09-23T23:19:19.511716+0000","last_became_active":"2023-09-23T23:19:17.011835+0000","last_became_peered":"2023-09-23T23:19:17.011835+0000","last_unstale":"2023-09-23T23:19:19.511716+0000","last_undegraded":"2023-09-23T23:19:19.511716+0000","last_fullsized":"2023-09-23T23:19:19.511716+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:26.810456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"266'1700","reported_seq":1719,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T23:19:24.269588+0000","last_change":"2023-09-23T23:18:58.587441+0000","last_active":"2023-09-23T23:19:24.269588+0000","last_peered":"2023-09-23T23:19:24.269588+0000","last_clean":"2023-09-23T23:19:24.269588+0000","last_became_active":"2023-09-23T23:18:58.587188+0000","last_became_peered":"2023-09-23T23:18:58.587188+0000","last_unstale":"2023-09-23T23:19:24.269588+0000","last_undegraded":"2023-09-23T23:19:24.269588+0000","last_fullsized":"2023-09-23T23:19:24.269588+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":1700,"log_dups_size":0,"ondisk_log_size":1700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:54.227096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111411200,"num_objects":1700,"num_object_clones":0,"num_object_copies":5100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1700,"num_write_kb":108800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":539,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.342516+0000","last_change":"2023-09-23T23:16:59.295000+0000","last_active":"2023-09-23T23:19:19.342516+0000","last_peered":"2023-09-23T23:19:19.342516+0000","last_clean":"2023-09-23T23:19:19.342516+0000","last_became_active":"2023-09-23T23:16:59.294607+0000","last_became_peered":"2023-09-23T23:16:59.294607+0000","last_unstale":"2023-09-23T23:19:19.342516+0000","last_undegraded":"2023-09-23T23:19:19.342516+0000","last_fullsized":"2023-09-23T23:19:19.342516+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.802642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.802642+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:48.429859+0000","scrub_duration":0.0056570290000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":66,"reported_epoch":265,"state":"active+undersized","last_fresh":"2023-09-23T23:19:19.336624+0000","last_change":"2023-09-23T23:18:59.585438+0000","last_active":"2023-09-23T23:19:19.336624+0000","last_peered":"2023-09-23T23:19:19.336624+0000","last_clean":"2023-09-23T23:03:28.874460+0000","last_became_active":"2023-09-23T23:18:59.585438+0000","last_became_peered":"2023-09-23T23:18:59.585438+0000","last_unstale":"2023-09-23T23:19:19.336624+0000","last_undegraded":"2023-09-23T23:19:19.336624+0000","last_fullsized":"2023-09-23T23:18:59.577149+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:37.419384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":246,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.511591+0000","last_change":"2023-09-23T23:19:17.011430+0000","last_active":"2023-09-23T23:19:19.511591+0000","last_peered":"2023-09-23T23:19:19.511591+0000","last_clean":"2023-09-23T23:19:19.511591+0000","last_became_active":"2023-09-23T23:19:17.011105+0000","last_became_peered":"2023-09-23T23:19:17.011105+0000","last_unstale":"2023-09-23T23:19:19.511591+0000","last_undegraded":"2023-09-23T23:19:19.511591+0000","last_fullsized":"2023-09-23T23:19:19.511591+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:41.770486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":228,"reported_epoch":265,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:19:19.336289+0000","last_change":"2023-09-23T23:19:18.009218+0000","last_active":"2023-09-23T23:19:19.336289+0000","last_peered":"2023-09-23T23:19:19.336289+0000","last_clean":"2023-09-23T23:19:19.336289+0000","last_became_active":"2023-09-23T23:19:18.008887+0000","last_became_peered":"2023-09-23T23:19:18.008887+0000","last_unstale":"2023-09-23T23:19:19.336289+0000","last_undegraded":"2023-09-23T23:19:19.336289+0000","last_fullsized":"2023-09-23T23:19:19.336289+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:02.574282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4,1],"avail_no_missing":["0","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"266'1721","reported_seq":1740,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T23:19:24.437313+0000","last_change":"2023-09-23T23:18:58.586309+0000","last_active":"2023-09-23T23:19:24.437313+0000","last_peered":"2023-09-23T23:19:24.437313+0000","last_clean":"2023-09-23T23:19:24.437313+0000","last_became_active":"2023-09-23T23:18:58.586026+0000","last_became_peered":"2023-09-23T23:18:58.586026+0000","last_unstale":"2023-09-23T23:19:24.437313+0000","last_undegraded":"2023-09-23T23:19:24.437313+0000","last_fullsized":"2023-09-23T23:19:24.437313+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":1721,"log_dups_size":0,"ondisk_log_size":1721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:50.254164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112787456,"num_objects":1721,"num_object_clones":0,"num_object_copies":5163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1721,"num_write_kb":110144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":545,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.345617+0000","last_change":"2023-09-23T23:17:51.331967+0000","last_active":"2023-09-23T23:19:19.345617+0000","last_peered":"2023-09-23T23:19:19.345617+0000","last_clean":"2023-09-23T23:19:19.345617+0000","last_became_active":"2023-09-23T23:17:51.331604+0000","last_became_peered":"2023-09-23T23:17:51.331604+0000","last_unstale":"2023-09-23T23:19:19.345617+0000","last_undegraded":"2023-09-23T23:19:19.345617+0000","last_fullsized":"2023-09-23T23:19:19.345617+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:51.489074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:51.489074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:38:55.870389+0000","scrub_duration":0.00519324,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":243,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.340569+0000","last_change":"2023-09-23T23:16:56.561147+0000","last_active":"2023-09-23T23:19:19.340569+0000","last_peered":"2023-09-23T23:19:19.340569+0000","last_clean":"2023-09-23T23:19:19.340569+0000","last_became_active":"2023-09-23T23:16:56.547808+0000","last_became_peered":"2023-09-23T23:16:56.547808+0000","last_unstale":"2023-09-23T23:19:19.340569+0000","last_undegraded":"2023-09-23T23:19:19.340569+0000","last_fullsized":"2023-09-23T23:19:19.340569+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:13.666957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":121,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.342724+0000","last_change":"2023-09-23T23:16:56.526187+0000","last_active":"2023-09-23T23:19:19.342724+0000","last_peered":"2023-09-23T23:19:19.342724+0000","last_clean":"2023-09-23T23:19:19.342724+0000","last_became_active":"2023-09-23T23:16:56.525874+0000","last_became_peered":"2023-09-23T23:16:56.525874+0000","last_unstale":"2023-09-23T23:19:19.342724+0000","last_undegraded":"2023-09-23T23:19:19.342724+0000","last_fullsized":"2023-09-23T23:19:19.342724+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:32:07.101559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":173,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.336228+0000","last_change":"2023-09-23T23:19:17.011572+0000","last_active":"2023-09-23T23:19:19.336228+0000","last_peered":"2023-09-23T23:19:19.336228+0000","last_clean":"2023-09-23T23:19:19.336228+0000","last_became_active":"2023-09-23T23:19:17.011271+0000","last_became_peered":"2023-09-23T23:19:17.011271+0000","last_unstale":"2023-09-23T23:19:19.336228+0000","last_undegraded":"2023-09-23T23:19:19.336228+0000","last_fullsized":"2023-09-23T23:19:19.336228+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:27:25.286537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.f","version":"266'1668","reported_seq":1687,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T23:19:24.239540+0000","last_change":"2023-09-23T23:18:58.587628+0000","last_active":"2023-09-23T23:19:24.239540+0000","last_peered":"2023-09-23T23:19:24.239540+0000","last_clean":"2023-09-23T23:19:24.239540+0000","last_became_active":"2023-09-23T23:18:58.587032+0000","last_became_peered":"2023-09-23T23:18:58.587032+0000","last_unstale":"2023-09-23T23:19:24.239540+0000","last_undegraded":"2023-09-23T23:19:24.239540+0000","last_fullsized":"2023-09-23T23:19:24.239540+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":1668,"log_dups_size":0,"ondisk_log_size":1668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:35.329395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109314048,"num_objects":1668,"num_object_clones":0,"num_object_copies":5004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1668,"num_write_kb":106752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":533,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.340264+0000","last_change":"2023-09-23T23:16:59.285629+0000","last_active":"2023-09-23T23:19:19.340264+0000","last_peered":"2023-09-23T23:19:19.340264+0000","last_clean":"2023-09-23T23:19:19.340264+0000","last_became_active":"2023-09-23T23:16:59.285208+0000","last_became_peered":"2023-09-23T23:16:59.285208+0000","last_unstale":"2023-09-23T23:19:19.340264+0000","last_undegraded":"2023-09-23T23:19:19.340264+0000","last_fullsized":"2023-09-23T23:19:19.340264+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.522632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.522632+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:04:15.985273+0000","scrub_duration":0.018152603999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":268,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.343176+0000","last_change":"2023-09-23T23:17:48.303246+0000","last_active":"2023-09-23T23:19:19.343176+0000","last_peered":"2023-09-23T23:19:19.343176+0000","last_clean":"2023-09-23T23:19:19.343176+0000","last_became_active":"2023-09-23T23:17:48.302984+0000","last_became_peered":"2023-09-23T23:17:48.302984+0000","last_unstale":"2023-09-23T23:19:19.343176+0000","last_undegraded":"2023-09-23T23:19:19.343176+0000","last_fullsized":"2023-09-23T23:19:19.343176+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:11.999957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":235,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.511693+0000","last_change":"2023-09-23T23:19:17.023159+0000","last_active":"2023-09-23T23:19:19.511693+0000","last_peered":"2023-09-23T23:19:19.511693+0000","last_clean":"2023-09-23T23:19:19.511693+0000","last_became_active":"2023-09-23T23:19:17.012292+0000","last_became_peered":"2023-09-23T23:19:17.012292+0000","last_unstale":"2023-09-23T23:19:19.511693+0000","last_undegraded":"2023-09-23T23:19:19.511693+0000","last_fullsized":"2023-09-23T23:19:19.511693+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:06.870287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":208,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.344915+0000","last_change":"2023-09-23T22:53:17.905706+0000","last_active":"2023-09-23T23:19:19.344915+0000","last_peered":"2023-09-23T23:19:19.344915+0000","last_clean":"2023-09-23T23:19:19.344915+0000","last_became_active":"2023-09-23T22:53:17.905309+0000","last_became_peered":"2023-09-23T22:53:17.905309+0000","last_unstale":"2023-09-23T23:19:19.344915+0000","last_undegraded":"2023-09-23T23:19:19.344915+0000","last_fullsized":"2023-09-23T23:19:19.344915+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:06.105426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":212,"reported_epoch":265,"state":"active+undersized","last_fresh":"2023-09-23T23:19:19.336148+0000","last_change":"2023-09-23T23:18:59.585406+0000","last_active":"2023-09-23T23:19:19.336148+0000","last_peered":"2023-09-23T23:19:19.336148+0000","last_clean":"2023-09-23T23:18:57.571344+0000","last_became_active":"2023-09-23T23:18:59.585406+0000","last_became_peered":"2023-09-23T23:18:59.585406+0000","last_unstale":"2023-09-23T23:19:19.336148+0000","last_undegraded":"2023-09-23T23:19:19.336148+0000","last_fullsized":"2023-09-23T23:18:59.576862+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:58.868950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":185,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.340183+0000","last_change":"2023-09-23T23:19:17.006371+0000","last_active":"2023-09-23T23:19:19.340183+0000","last_peered":"2023-09-23T23:19:19.340183+0000","last_clean":"2023-09-23T23:19:19.340183+0000","last_became_active":"2023-09-23T23:19:17.006046+0000","last_became_peered":"2023-09-23T23:19:17.006046+0000","last_unstale":"2023-09-23T23:19:19.340183+0000","last_undegraded":"2023-09-23T23:19:19.340183+0000","last_fullsized":"2023-09-23T23:19:19.340183+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:22.256902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":243,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.346281+0000","last_change":"2023-09-23T23:19:17.021097+0000","last_active":"2023-09-23T23:19:19.346281+0000","last_peered":"2023-09-23T23:19:19.346281+0000","last_clean":"2023-09-23T23:19:19.346281+0000","last_became_active":"2023-09-23T23:19:17.011502+0000","last_became_peered":"2023-09-23T23:19:17.011502+0000","last_unstale":"2023-09-23T23:19:19.346281+0000","last_undegraded":"2023-09-23T23:19:19.346281+0000","last_fullsized":"2023-09-23T23:19:19.346281+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:09.356441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":269,"reported_epoch":265,"state":"active+undersized","last_fresh":"2023-09-23T23:19:19.346792+0000","last_change":"2023-09-23T23:18:59.585632+0000","last_active":"2023-09-23T23:19:19.346792+0000","last_peered":"2023-09-23T23:19:19.346792+0000","last_clean":"2023-09-23T23:18:57.571707+0000","last_became_active":"2023-09-23T23:18:59.585632+0000","last_became_peered":"2023-09-23T23:18:59.585632+0000","last_unstale":"2023-09-23T23:19:19.346792+0000","last_undegraded":"2023-09-23T23:19:19.346792+0000","last_fullsized":"2023-09-23T23:18:59.583032+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:02:31.440901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":212,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.343603+0000","last_change":"2023-09-23T22:53:16.891297+0000","last_active":"2023-09-23T23:19:19.343603+0000","last_peered":"2023-09-23T23:19:19.343603+0000","last_clean":"2023-09-23T23:19:19.343603+0000","last_became_active":"2023-09-23T22:53:16.891025+0000","last_became_peered":"2023-09-23T22:53:16.891025+0000","last_unstale":"2023-09-23T23:19:19.343603+0000","last_undegraded":"2023-09-23T23:19:19.343603+0000","last_fullsized":"2023-09-23T23:19:19.343603+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:40:50.608339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":242,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.344891+0000","last_change":"2023-09-23T23:10:59.512247+0000","last_active":"2023-09-23T23:19:19.344891+0000","last_peered":"2023-09-23T23:19:19.344891+0000","last_clean":"2023-09-23T23:19:19.344891+0000","last_became_active":"2023-09-23T23:10:59.511865+0000","last_became_peered":"2023-09-23T23:10:59.511865+0000","last_unstale":"2023-09-23T23:19:19.344891+0000","last_undegraded":"2023-09-23T23:19:19.344891+0000","last_fullsized":"2023-09-23T23:19:19.344891+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:20.981239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":261,"reported_epoch":265,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:19:19.340495+0000","last_change":"2023-09-23T23:19:18.010390+0000","last_active":"2023-09-23T23:19:19.340495+0000","last_peered":"2023-09-23T23:19:19.340495+0000","last_clean":"2023-09-23T23:19:19.340495+0000","last_became_active":"2023-09-23T23:19:18.009902+0000","last_became_peered":"2023-09-23T23:19:18.009902+0000","last_unstale":"2023-09-23T23:19:19.340495+0000","last_undegraded":"2023-09-23T23:19:19.340495+0000","last_fullsized":"2023-09-23T23:19:19.340495+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:47.860456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6,1],"avail_no_missing":["3","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":265,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.340094+0000","last_change":"2023-09-23T23:17:48.303036+0000","last_active":"2023-09-23T23:19:19.340094+0000","last_peered":"2023-09-23T23:19:19.340094+0000","last_clean":"2023-09-23T23:19:19.340094+0000","last_became_active":"2023-09-23T23:17:48.302706+0000","last_became_peered":"2023-09-23T23:17:48.302706+0000","last_unstale":"2023-09-23T23:19:19.340094+0000","last_undegraded":"2023-09-23T23:19:19.340094+0000","last_fullsized":"2023-09-23T23:19:19.340094+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:46:22.942097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":252,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.336147+0000","last_change":"2023-09-23T23:17:48.338955+0000","last_active":"2023-09-23T23:19:19.336147+0000","last_peered":"2023-09-23T23:19:19.336147+0000","last_clean":"2023-09-23T23:19:19.336147+0000","last_became_active":"2023-09-23T23:17:48.332511+0000","last_became_peered":"2023-09-23T23:17:48.332511+0000","last_unstale":"2023-09-23T23:19:19.336147+0000","last_undegraded":"2023-09-23T23:19:19.336147+0000","last_fullsized":"2023-09-23T23:19:19.336147+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:12.303751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":272,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T23:19:19.350643+0000","last_change":"2023-09-23T23:19:17.016921+0000","last_active":"2023-09-23T23:19:19.350643+0000","last_peered":"2023-09-23T23:19:19.350643+0000","last_clean":"2023-09-23T23:19:19.350643+0000","last_became_active":"2023-09-23T23:19:17.016654+0000","last_became_peered":"2023-09-23T23:19:17.016654+0000","last_unstale":"2023-09-23T23:19:19.350643+0000","last_undegraded":"2023-09-23T23:19:19.350643+0000","last_fullsized":"2023-09-23T23:19:19.350643+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:35.905548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":89,"acting":91,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1798111232,"num_objects":27437,"num_object_clones":0,"num_object_copies":82311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4435,"num_objects_misplaced":131,"num_objects_unfound":0,"num_objects_dirty":27437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":27437,"num_write_kb":1755968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":843,"num_bytes_recovered":55246848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5197201408,"data_stored":5197201408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":27437,"ondisk_log_size":27437,"up":46,"acting":44,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":666,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":89,"acting":91,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":697708,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":89,"num_read_kb":167,"num_write":43,"num_write_kb":439,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":649699,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":32,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":21,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234314,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":803516,"kb_used_data":352116,"kb_used_omap":0,"kb_used_meta":451392,"kb_avail":92941636,"statfs":{"total":95995035648,"available":95172235264,"internally_reserved":0,"allocated":360566784,"data_stored":359841767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":462225408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:18:39 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":4.9820000000000002,"15min":3.9239999999999999},"min":{"1min":0.22,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":6.9249999999999998,"5min":216.40899999999999,"15min":230.249},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.7320000000000002,"5min":5.7359999999999998,"15min":4.2290000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24199999999999999},"max":{"1min":11.327999999999999,"5min":216.43700000000001,"15min":230.261},"last":0.40899999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:17:56 2023","interfaces":[{"interface":"back","average":{"1min":2.6150000000000002,"5min":2.9700000000000002,"15min":4.2359999999999998},"min":{"1min":0.39100000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.798,"5min":121.39400000000001,"15min":121.39400000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.2989999999999999,"5min":2.9790000000000001,"15min":4.077},"min":{"1min":0.27000000000000002,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":10.034000000000001,"5min":121.381,"15min":121.381},"last":0.42099999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0339999999999998,"5min":2.0179999999999998,"15min":2.0150000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.375,"5min":9.5589999999999993,"15min":9.5589999999999993},"last":0.435},{"interface":"front","average":{"1min":2.5600000000000001,"5min":1.8580000000000001,"15min":1.7410000000000001},"min":{"1min":0.36899999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.561999999999999,"5min":11.561999999999999,"15min":11.561999999999999},"last":0.47999999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:18:11 2023","interfaces":[{"interface":"back","average":{"1min":3.1400000000000001,"5min":5.7140000000000004,"15min":3.4620000000000002},"min":{"1min":0.27800000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":17.724,"5min":144.50299999999999,"15min":144.50299999999999},"last":0.755},{"interface":"front","average":{"1min":2.6829999999999998,"5min":5.5990000000000002,"15min":3.468},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":17.712,"5min":144.48699999999999,"15min":144.48699999999999},"last":0.75}]},{"osd":4,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":0.83099999999999996,"15min":1.3200000000000001},"min":{"1min":0.27300000000000002,"5min":0.254,"15min":0.23599999999999999},"max":{"1min":7.0570000000000004,"5min":7.0570000000000004,"15min":92.781000000000006},"last":0.372},{"interface":"front","average":{"1min":1.208,"5min":1.2110000000000001,"15min":1.4219999999999999},"min":{"1min":0.29499999999999998,"5min":0.28599999999999998,"15min":0.254},"max":{"1min":3.9569999999999999,"5min":9.7530000000000001,"15min":92.626999999999995},"last":0.39100000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":13.24,"5min":3.488,"15min":2.3759999999999999},"min":{"1min":0.28799999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":185.321,"5min":185.321,"15min":185.321},"last":0.44600000000000001},{"interface":"front","average":{"1min":11.488,"5min":2.851,"15min":1.97},"min":{"1min":0.33900000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":185.47499999999999,"5min":185.47499999999999,"15min":185.47499999999999},"last":0.49199999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":1.292,"15min":1.4970000000000001},"min":{"1min":0.376,"5min":0.30199999999999999,"15min":0.27300000000000002},"max":{"1min":19.782,"5min":19.782,"15min":93.105999999999995},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.7349999999999999,"5min":1.492,"15min":1.599},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28399999999999997},"max":{"1min":20.064,"5min":20.064,"15min":93.198999999999998},"last":0.46800000000000003}]}]},{"osd":6,"up_from":46,"seq":197568495956,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1196468,"kb_used_data":850092,"kb_used_omap":0,"kb_used_meta":346368,"kb_avail":92548684,"statfs":{"total":95995035648,"available":94769852416,"internally_reserved":0,"allocated":870494208,"data_stored":869744313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":354680832},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[11,2,0,2,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:18:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4039999999999999,"5min":6.5099999999999998,"15min":3.839},"min":{"1min":0.245,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":13,"5min":161.64599999999999,"15min":161.64599999999999},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.4019999999999999,"5min":6.5220000000000002,"15min":4.0789999999999997},"min":{"1min":0.28899999999999998,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":4.5149999999999997,"5min":161.38499999999999,"15min":161.38499999999999},"last":0.29299999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:18:50 2023","interfaces":[{"interface":"back","average":{"1min":2.758,"5min":6.9829999999999997,"15min":4.024},"min":{"1min":0.26300000000000001,"5min":0.249,"15min":0.24099999999999999},"max":{"1min":13.074999999999999,"5min":210.869,"15min":210.869},"last":0.68300000000000005},{"interface":"front","average":{"1min":2.2810000000000001,"5min":7.0700000000000003,"15min":4.2249999999999996},"min":{"1min":0.26800000000000002,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":11.840999999999999,"5min":210.89699999999999,"15min":210.89699999999999},"last":3.5950000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:18:22 2023","interfaces":[{"interface":"back","average":{"1min":3.3279999999999998,"5min":2.0960000000000001,"15min":1.891},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":11.804,"5min":11.804,"15min":11.804},"last":1.073},{"interface":"front","average":{"1min":2.819,"5min":1.575,"15min":1.3680000000000001},"min":{"1min":0.32400000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.685,"5min":10.685,"15min":10.685},"last":2.3940000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:19:15 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":5.0869999999999997,"15min":3.2530000000000001},"min":{"1min":0.35499999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":10.528,"5min":109.98699999999999,"15min":109.98699999999999},"last":2.0089999999999999},{"interface":"front","average":{"1min":1.9550000000000001,"5min":4.9249999999999998,"15min":3.456},"min":{"1min":0.28100000000000003,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":13.039,"5min":110.024,"15min":110.024},"last":0.71899999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:19:02 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":0.94499999999999995,"15min":1.113},"min":{"1min":0.32800000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.0190000000000001,"5min":7.0190000000000001,"15min":11.890000000000001},"last":0.752},{"interface":"front","average":{"1min":2.4100000000000001,"5min":1.1990000000000001,"15min":1.2290000000000001},"min":{"1min":0.33200000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":11.057,"5min":11.057,"15min":11.728},"last":0.54500000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:19:02 2023","interfaces":[{"interface":"back","average":{"1min":6.1929999999999996,"5min":2.0150000000000001,"15min":1.6140000000000001},"min":{"1min":0.32600000000000001,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":91.899000000000001,"5min":91.899000000000001,"15min":91.899000000000001},"last":3.5579999999999998},{"interface":"front","average":{"1min":5.9569999999999999,"5min":1.8879999999999999,"15min":1.484},"min":{"1min":0.40999999999999998,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":94.174000000000007,"5min":94.174000000000007,"15min":94.174000000000007},"last":0.45100000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:18:28 2023","interfaces":[{"interface":"back","average":{"1min":2.7250000000000001,"5min":1.2689999999999999,"15min":1.27},"min":{"1min":0.36799999999999999,"5min":0.32300000000000001,"15min":0.32100000000000001},"max":{"1min":13.823,"5min":13.823,"15min":13.823},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.5609999999999999,"5min":0.79900000000000004,"15min":0.98799999999999999},"min":{"1min":0.35999999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":13.103,"5min":13.103,"15min":13.103},"last":0.54400000000000004}]}]},{"osd":1,"up_from":0,"seq":481036337297,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":91,"seq":390842024121,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2037968,"kb_used_data":1502216,"kb_used_omap":5,"kb_used_meta":535738,"kb_avail":91707184,"statfs":{"total":95995035648,"available":93908156416,"internally_reserved":0,"allocated":1538269184,"data_stored":1537139426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":548596346},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:18:50 2023","interfaces":[{"interface":"back","average":{"1min":3.2509999999999999,"5min":5.7279999999999998,"15min":3.3570000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":10.303000000000001,"5min":155.91999999999999,"15min":155.91999999999999},"last":0.54700000000000004},{"interface":"front","average":{"1min":3.0670000000000002,"5min":4.7990000000000004,"15min":2.9929999999999999},"min":{"1min":0.22600000000000001,"5min":0.19,"15min":0.189},"max":{"1min":16.591000000000001,"5min":153.28899999999999,"15min":153.28899999999999},"last":0.56399999999999995}]},{"osd":2,"last update":"Sat Sep 23 23:19:22 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":2.762,"15min":3.012},"min":{"1min":0.23999999999999999,"5min":0.222,"15min":0.222},"max":{"1min":10.332000000000001,"5min":20.495999999999999,"15min":20.495999999999999},"last":0.23999999999999999},{"interface":"front","average":{"1min":0.499,"5min":3.27,"15min":3.879},"min":{"1min":0.318,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.1519999999999999,"5min":36.383000000000003,"15min":36.383000000000003},"last":1.1519999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:19:07 2023","interfaces":[{"interface":"back","average":{"1min":2.3679999999999999,"5min":4.9690000000000003,"15min":3.6040000000000001},"min":{"1min":0.26800000000000002,"5min":0.23200000000000001,"15min":0.22700000000000001},"max":{"1min":15.037000000000001,"5min":82.691000000000003,"15min":82.691000000000003},"last":0.27200000000000002},{"interface":"front","average":{"1min":1.093,"5min":3.9710000000000001,"15min":2.7210000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.8090000000000002,"5min":82.429000000000002,"15min":82.429000000000002},"last":0.32400000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:18:32 2023","interfaces":[{"interface":"back","average":{"1min":4.5419999999999998,"5min":2.0129999999999999,"15min":2.1040000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.245},"max":{"1min":30.097000000000001,"5min":30.097000000000001,"15min":30.097000000000001},"last":4.4189999999999996},{"interface":"front","average":{"1min":4.3449999999999998,"5min":2.3420000000000001,"15min":2.4359999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29099999999999998},"max":{"1min":30.157,"5min":30.157,"15min":30.706},"last":1.7150000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:18:32 2023","interfaces":[{"interface":"back","average":{"1min":3.7589999999999999,"5min":2.2480000000000002,"15min":2.4220000000000002},"min":{"1min":0.42599999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":15.923,"5min":15.923,"15min":30.873999999999999},"last":1.266},{"interface":"front","average":{"1min":2.9180000000000001,"5min":1.9470000000000001,"15min":1.9330000000000001},"min":{"1min":0.39200000000000002,"5min":0.32100000000000001,"15min":0.29499999999999998},"max":{"1min":15.929,"5min":15.929,"15min":15.929},"last":0.57699999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:18:32 2023","interfaces":[{"interface":"back","average":{"1min":6.109,"5min":2.7879999999999998,"15min":2.6259999999999999},"min":{"1min":0.40999999999999998,"5min":0.314,"15min":0.314},"max":{"1min":30.651,"5min":30.651,"15min":30.651},"last":2.298},{"interface":"front","average":{"1min":5.1829999999999998,"5min":2.593,"15min":2.238},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.30499999999999999},"max":{"1min":30.673999999999999,"5min":30.673999999999999,"15min":30.673999999999999},"last":2.286}]},{"osd":7,"last update":"Sat Sep 23 23:18:32 2023","interfaces":[{"interface":"back","average":{"1min":4.306,"5min":2.3620000000000001,"15min":2.1829999999999998},"min":{"1min":0.372,"5min":0.34499999999999997,"15min":0.29299999999999998},"max":{"1min":26.413,"5min":26.413,"15min":26.413},"last":0.42899999999999999},{"interface":"front","average":{"1min":4.7960000000000003,"5min":2.1920000000000002,"15min":2.1219999999999999},"min":{"1min":0.372,"5min":0.36499999999999999,"15min":0.313},"max":{"1min":26.437999999999999,"5min":26.437999999999999,"15min":26.437999999999999},"last":0.52600000000000002}]}]},{"osd":2,"up_from":0,"seq":854698491942,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":148,"seq":635655159911,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":731912,"kb_used_data":395072,"kb_used_omap":2,"kb_used_meta":336829,"kb_avail":93013240,"statfs":{"total":95995035648,"available":95245557760,"internally_reserved":0,"allocated":404553728,"data_stored":403490163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":344912949},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:19:13 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":4.1600000000000001,"15min":2.1930000000000001},"min":{"1min":0.21099999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":5.2169999999999996,"5min":72.778999999999996,"15min":72.778999999999996},"last":0.38},{"interface":"front","average":{"1min":0.74299999999999999,"5min":3.98,"15min":2.5600000000000001},"min":{"1min":0.20499999999999999,"5min":0.186,"15min":0.17999999999999999},"max":{"1min":5.2309999999999999,"5min":72.719999999999999,"15min":72.719999999999999},"last":0.28899999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:19:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":3.9089999999999998,"15min":2.8370000000000002},"min":{"1min":0.22500000000000001,"5min":0.215,"15min":0.19600000000000001},"max":{"1min":4.0670000000000002,"5min":76.322999999999993,"15min":76.322999999999993},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.67900000000000005,"5min":3.6779999999999999,"15min":2.6179999999999999},"min":{"1min":0.28999999999999998,"5min":0.20499999999999999,"15min":0.185},"max":{"1min":3.5760000000000001,"5min":76.444000000000003,"15min":76.444000000000003},"last":0.47499999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:18:25 2023","interfaces":[{"interface":"back","average":{"1min":2.7959999999999998,"5min":3.556,"15min":3.6829999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.5359999999999996,"5min":18.742999999999999,"15min":18.742999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.331,"5min":1.8220000000000001,"15min":1.7370000000000001},"min":{"1min":0.35399999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":14.711,"5min":14.711,"15min":14.711},"last":0.42499999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:19:13 2023","interfaces":[{"interface":"back","average":{"1min":1.141,"5min":2.0049999999999999,"15min":1.6859999999999999},"min":{"1min":0.435,"5min":0.315,"15min":0.29999999999999999},"max":{"1min":3.7759999999999998,"5min":14.754,"15min":14.754},"last":1.2330000000000001},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.5089999999999999,"15min":2.363},"min":{"1min":0.45400000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.7199999999999998,"5min":8.0239999999999991,"15min":13.532999999999999},"last":0.57699999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:19:13 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":2.278,"15min":2.3450000000000002},"min":{"1min":0.34399999999999997,"5min":0.29299999999999998,"15min":0.28199999999999997},"max":{"1min":5.3949999999999996,"5min":75.733000000000004,"15min":75.733000000000004},"last":0.501},{"interface":"front","average":{"1min":1.345,"5min":2.8170000000000002,"15min":2.2450000000000001},"min":{"1min":0.46300000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.0750000000000002,"5min":75.686999999999998,"15min":75.686999999999998},"last":0.56000000000000005}]},{"osd":6,"last update":"Sat Sep 23 23:19:13 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.589,"15min":2.198},"min":{"1min":0.42199999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.2800000000000002,"5min":8.0549999999999997,"15min":14.492000000000001},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.7290000000000001,"5min":1.9119999999999999,"15min":2.048},"min":{"1min":0.44900000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.1539999999999999,"5min":9.8379999999999992,"15min":9.8379999999999992},"last":0.40500000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:18:09 2023","interfaces":[{"interface":"back","average":{"1min":2.198,"5min":2.915,"15min":2.5459999999999998},"min":{"1min":0.40200000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":9.4280000000000008,"5min":130.839,"15min":130.839},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.9129999999999998,"5min":2.992,"15min":2.665},"min":{"1min":0.434,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":8.8190000000000008,"5min":130.90000000000001,"15min":130.90000000000001},"last":0.57799999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019229,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1997308,"kb_used_data":1554612,"kb_used_omap":0,"kb_used_meta":442688,"kb_avail":91747844,"statfs":{"total":95995035648,"available":93949792256,"internally_reserved":0,"allocated":1591922688,"data_stored":1591164677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":453312512},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:18:35 2023","interfaces":[{"interface":"back","average":{"1min":16.757000000000001,"5min":6.1289999999999996,"15min":4.0250000000000004},"min":{"1min":0.223,"5min":0.223,"15min":0.20799999999999999},"max":{"1min":199.64400000000001,"5min":199.64400000000001,"15min":238.62200000000001},"last":0.82499999999999996},{"interface":"front","average":{"1min":17.088000000000001,"5min":6.0190000000000001,"15min":4.4039999999999999},"min":{"1min":0.32700000000000001,"5min":0.23499999999999999,"15min":0.23300000000000001},"max":{"1min":195.649,"5min":195.649,"15min":238.58199999999999},"last":5.7919999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:18:52 2023","interfaces":[{"interface":"back","average":{"1min":11.435,"5min":8.7639999999999993,"15min":4.8399999999999999},"min":{"1min":0.23899999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":161.923,"5min":161.923,"15min":175.00899999999999},"last":1.9319999999999999},{"interface":"front","average":{"1min":11.101000000000001,"5min":8.8770000000000007,"15min":4.9219999999999997},"min":{"1min":0.26500000000000001,"5min":0.26200000000000001,"15min":0.25800000000000001},"max":{"1min":161.39699999999999,"5min":161.39699999999999,"15min":174.821},"last":3.4100000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:18:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2909999999999999,"5min":3.2810000000000001,"15min":3.4460000000000002},"min":{"1min":0.318,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":10.75,"5min":39.308,"15min":39.308},"last":1.095},{"interface":"front","average":{"1min":2.4929999999999999,"5min":2.625,"15min":2.6469999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":7.8490000000000002,"5min":14.287000000000001,"15min":14.287000000000001},"last":23.138000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:19:17 2023","interfaces":[{"interface":"back","average":{"1min":1.3300000000000001,"5min":8.7629999999999999,"15min":4.2190000000000003},"min":{"1min":0.35199999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.0529999999999999,"5min":303.41500000000002,"15min":303.41500000000002},"last":2.3210000000000002},{"interface":"front","average":{"1min":1.6719999999999999,"5min":8.7840000000000007,"15min":4.5469999999999997},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":7.5359999999999996,"5min":303.428,"15min":303.428},"last":8.7159999999999993}]},{"osd":5,"last update":"Sat Sep 23 23:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.571,"5min":2.6469999999999998,"15min":1.478},"min":{"1min":0.312,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.8330000000000002,"5min":160.79499999999999,"15min":160.79499999999999},"last":0.76400000000000001},{"interface":"front","average":{"1min":1.5129999999999999,"5min":2.766,"15min":1.7310000000000001},"min":{"1min":0.29199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.2690000000000001,"5min":160.61099999999999,"15min":160.61099999999999},"last":3.3439999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:19:17 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":1.2150000000000001,"15min":1.123},"min":{"1min":0.30099999999999999,"5min":0.28000000000000003,"15min":0.27200000000000002},"max":{"1min":6.0910000000000002,"5min":7.5499999999999998,"15min":9.3350000000000009},"last":2.286},{"interface":"front","average":{"1min":0.82799999999999996,"5min":1.0680000000000001,"15min":1.101},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27500000000000002},"max":{"1min":4.484,"5min":7.4459999999999997,"15min":13.888999999999999},"last":0.79900000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:18:35 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.5609999999999999,"15min":1.3140000000000001},"min":{"1min":0.36299999999999999,"5min":0.315,"15min":0.30499999999999999},"max":{"1min":7.4850000000000003,"5min":14.842000000000001,"15min":14.842000000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.0169999999999999,"5min":1.502,"15min":1.341},"min":{"1min":0.433,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.882,"5min":14.561,"15min":14.561},"last":0.378}]}]},{"osd":5,"up_from":38,"seq":163208757592,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1291684,"kb_used_data":824732,"kb_used_omap":0,"kb_used_meta":466944,"kb_avail":92453468,"statfs":{"total":95995035648,"available":94672351232,"internally_reserved":0,"allocated":844525568,"data_stored":843787020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":478150656},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:18:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4910000000000001,"5min":2.456,"15min":3.0350000000000001},"min":{"1min":0.24399999999999999,"5min":0.23000000000000001,"15min":0.21099999999999999},"max":{"1min":6.5140000000000002,"5min":97.405000000000001,"15min":190.483},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.8170000000000002,"5min":2.7410000000000001,"15min":3.6150000000000002},"min":{"1min":0.315,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":12.42,"5min":97.447000000000003,"15min":194.11099999999999},"last":1.3400000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:18:48 2023","interfaces":[{"interface":"back","average":{"1min":6.0179999999999998,"5min":7.2590000000000003,"15min":3.7000000000000002},"min":{"1min":0.311,"5min":0.25,"15min":0.246},"max":{"1min":52.582000000000001,"5min":193.584,"15min":193.584},"last":0.76800000000000002},{"interface":"front","average":{"1min":5.923,"5min":7.3019999999999996,"15min":3.8290000000000002},"min":{"1min":0.437,"5min":0.27800000000000002,"15min":0.26400000000000001},"max":{"1min":54.814999999999998,"5min":194.03100000000001,"15min":194.03100000000001},"last":0.81299999999999994}]},{"osd":2,"last update":"Sat Sep 23 23:19:23 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.4970000000000001,"15min":1.4670000000000001},"min":{"1min":0.35499999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.3380000000000001,"5min":6.5700000000000003,"15min":6.5700000000000003},"last":0.85399999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.5269999999999999,"15min":1.663},"min":{"1min":0.34499999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.80500000000000005,"5min":9.391,"15min":9.391},"last":0.503}]},{"osd":3,"last update":"Sat Sep 23 23:19:14 2023","interfaces":[{"interface":"back","average":{"1min":6.673,"5min":9.1530000000000005,"15min":4.1390000000000002},"min":{"1min":0.378,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":58.264000000000003,"5min":159.53299999999999,"15min":159.53299999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":6.9660000000000002,"5min":9.5030000000000001,"15min":4.9420000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":58.137999999999998,"5min":159.54900000000001,"15min":159.54900000000001},"last":1.375}]},{"osd":4,"last update":"Sat Sep 23 23:19:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":1.246,"15min":1.2410000000000001},"min":{"1min":0.28199999999999997,"5min":0.23999999999999999,"15min":0.221},"max":{"1min":5.1280000000000001,"5min":10.012,"15min":14.538},"last":1.5429999999999999},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.81399999999999995,"15min":0.872},"min":{"1min":0.28999999999999998,"5min":0.26100000000000001,"15min":0.25},"max":{"1min":2.3039999999999998,"5min":11.923999999999999,"15min":11.923999999999999},"last":0.56699999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:19:05 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.008,"15min":1.1419999999999999},"min":{"1min":0.34399999999999997,"5min":0.28299999999999997,"15min":0.255},"max":{"1min":3.8039999999999998,"5min":5.9809999999999999,"15min":13.226000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.216,"15min":1.286},"min":{"1min":0.34000000000000002,"5min":0.307,"15min":0.29099999999999998},"max":{"1min":3.524,"5min":12.178000000000001,"15min":13.098000000000001},"last":1.4059999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:18:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2010000000000001,"5min":1.286,"15min":1.3700000000000001},"min":{"1min":0.36699999999999999,"5min":0.32700000000000001,"15min":0.29199999999999998},"max":{"1min":12.09,"5min":12.09,"15min":14.929},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.319,"15min":1.276},"min":{"1min":0.371,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":5.7350000000000003,"5min":6.7089999999999996,"15min":15.090999999999999},"last":0.54400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1588330496,"data_stored":1588330496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":123797504,"data_stored":123797504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":407568384,"data_stored":407568384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1639776256,"data_stored":1639776256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":550764544,"data_stored":550764544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":886964224,"data_stored":886964224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:19:30.437 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 11 2023-09-23T23:19:30.606 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:19:30.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:30.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:19:30.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:30 smithi110 ceph-mon[212939]: pgmap v817: 99 pgs: 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 78 active+clean; 2.0 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 121 MiB/s wr, 1.93k op/s; 4356/99342 objects degraded (4.385%); 133/99342 objects misplaced (0.134%); 13 MiB/s, 203 objects/s recovering 2023-09-23T23:19:30.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:30 smithi110 ceph-mon[221102]: pgmap v817: 99 pgs: 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 78 active+clean; 2.0 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 121 MiB/s wr, 1.93k op/s; 4356/99342 objects degraded (4.385%); 133/99342 objects misplaced (0.134%); 13 MiB/s, 203 objects/s recovering 2023-09-23T23:19:30.710 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:30.736 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:30.778 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:30.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:30.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:30.811 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:19:30.811257+0000 min lat: 0.000955869 max lat: 2.04249 avg lat: 0.00870488 2023-09-23T23:19:30.811 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:19:30.811 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 20 16 36316 36300 113.424 81.0625 0.00203219 0.00870488 2023-09-23T23:19:30.884 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:19:30.885 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:30.908 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:30 smithi138 ceph-mon[193606]: pgmap v817: 99 pgs: 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 78 active+clean; 2.0 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 121 MiB/s wr, 1.93k op/s; 4356/99342 objects degraded (4.385%); 133/99342 objects misplaced (0.134%); 13 MiB/s, 203 objects/s recovering 2023-09-23T23:19:30.910 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:30.951 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:19:30.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:30.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:19:31.059 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:31.091 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:31.138 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:31.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:31.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:31.243 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:31.270 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:31.311 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:31.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:31.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:31.334 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:19:31.334 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:19:31.417 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:31.443 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:31.484 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:31.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:31.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:31.547 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:19:31.556 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:31.590 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:31.615 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:31.656 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:31.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:31.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:31.677 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:31 smithi110 ceph-mon[212939]: failed to encode map e269 with expected crc 2023-09-23T23:19:31.677 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:31 smithi110 ceph-mon[212939]: osdmap e269: 8 total, 7 up, 6 in 2023-09-23T23:19:31.677 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:31 smithi110 ceph-mon[212939]: failed to encode map e269 with expected crc 2023-09-23T23:19:31.677 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:31 smithi110 ceph-mon[212939]: failed to encode map e269 with expected crc 2023-09-23T23:19:31.677 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:31 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2092583033' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:19:31.677 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:31 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:19:31.678 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:31 smithi110 ceph-mon[221102]: failed to encode map e269 with expected crc 2023-09-23T23:19:31.678 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:31 smithi110 ceph-mon[221102]: osdmap e269: 8 total, 7 up, 6 in 2023-09-23T23:19:31.678 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:31 smithi110 ceph-mon[221102]: failed to encode map e269 with expected crc 2023-09-23T23:19:31.678 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:31 smithi110 ceph-mon[221102]: failed to encode map e269 with expected crc 2023-09-23T23:19:31.678 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:31 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2092583033' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:19:31.678 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:31 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:19:31.762 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:31.794 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:31.811 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 21 16 36316 36300 108.023 0 - 0.00870488 2023-09-23T23:19:31.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:31 smithi138 ceph-mon[193606]: failed to encode map e269 with expected crc 2023-09-23T23:19:31.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:31 smithi138 ceph-mon[193606]: osdmap e269: 8 total, 7 up, 6 in 2023-09-23T23:19:31.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:31 smithi138 ceph-mon[193606]: failed to encode map e269 with expected crc 2023-09-23T23:19:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:31 smithi138 ceph-mon[193606]: failed to encode map e269 with expected crc 2023-09-23T23:19:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:31 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2092583033' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:19:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:31 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:19:32.339 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:32.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:32.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:32.445 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:19:32.497 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:32.603 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:32 smithi110 ceph-mon[221102]: 1.3 scrub starts 2023-09-23T23:19:32.603 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:32 smithi110 ceph-mon[221102]: 1.3 scrub ok 2023-09-23T23:19:32.603 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:32 smithi110 ceph-mon[221102]: pgmap v819: 99 pgs: 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 78 active+clean; 2.1 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s; 4356/103125 objects degraded (4.224%); 133/103125 objects misplaced (0.129%); 9.4 MiB/s, 149 objects/s recovering 2023-09-23T23:19:32.604 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:32 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:19:32.604 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:32 smithi110 ceph-mon[221102]: failed to encode map e270 with expected crc 2023-09-23T23:19:32.604 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:32 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:19:32.604 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:32 smithi110 ceph-mon[221102]: osdmap e270: 8 total, 7 up, 6 in 2023-09-23T23:19:32.604 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:32 smithi110 ceph-mon[221102]: failed to encode map e270 with expected crc 2023-09-23T23:19:32.604 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:32 smithi110 ceph-mon[212939]: 1.3 scrub starts 2023-09-23T23:19:32.604 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:32 smithi110 ceph-mon[212939]: 1.3 scrub ok 2023-09-23T23:19:32.605 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:32 smithi110 ceph-mon[212939]: pgmap v819: 99 pgs: 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 78 active+clean; 2.1 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s; 4356/103125 objects degraded (4.224%); 133/103125 objects misplaced (0.129%); 9.4 MiB/s, 149 objects/s recovering 2023-09-23T23:19:32.605 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:32 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:19:32.605 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:32 smithi110 ceph-mon[212939]: failed to encode map e270 with expected crc 2023-09-23T23:19:32.605 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:32 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:19:32.605 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:32 smithi110 ceph-mon[212939]: osdmap e270: 8 total, 7 up, 6 in 2023-09-23T23:19:32.605 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:32 smithi110 ceph-mon[212939]: failed to encode map e270 with expected crc 2023-09-23T23:19:32.605 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:19:32.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:32.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:19:32.720 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:32.744 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:32 smithi138 ceph-mon[193606]: 1.3 scrub starts 2023-09-23T23:19:32.744 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:32 smithi138 ceph-mon[193606]: 1.3 scrub ok 2023-09-23T23:19:32.744 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:32 smithi138 ceph-mon[193606]: pgmap v819: 99 pgs: 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 78 active+clean; 2.1 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s; 4356/103125 objects degraded (4.224%); 133/103125 objects misplaced (0.129%); 9.4 MiB/s, 149 objects/s recovering 2023-09-23T23:19:32.744 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:32 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:19:32.744 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:32 smithi138 ceph-mon[193606]: failed to encode map e270 with expected crc 2023-09-23T23:19:32.744 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:32 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:19:32.745 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:32 smithi138 ceph-mon[193606]: osdmap e270: 8 total, 7 up, 6 in 2023-09-23T23:19:32.745 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:32 smithi138 ceph-mon[193606]: failed to encode map e270 with expected crc 2023-09-23T23:19:32.745 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:32.811 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 22 16 37366 37350 106.095 32.8125 0.00594072 0.00942079 2023-09-23T23:19:32.825 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:32.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:32.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:32.937 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:32.987 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:33.038 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:33.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:33.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:33.153 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:19:33.190 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:33.678 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:19:33.681 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:19:33.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:33.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:19:33.786 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:33.811 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 23 16 39006 38990 105.939 102.5 0.00427061 0.00943562 2023-09-23T23:19:33.834 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:33 smithi110 ceph-mon[212939]: failed to encode map e270 with expected crc 2023-09-23T23:19:33.834 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:33 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 4356/103125 objects degraded (4.224%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:33.834 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:33 smithi110 ceph-mon[221102]: failed to encode map e270 with expected crc 2023-09-23T23:19:33.834 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:33 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 4356/103125 objects degraded (4.224%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:33.852 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:33.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:33 smithi138 ceph-mon[193606]: failed to encode map e270 with expected crc 2023-09-23T23:19:33.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:33 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 4356/103125 objects degraded (4.224%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:34.053 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:34.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:34.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:34.158 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:34.207 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:34.387 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:34.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:34.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:34.493 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:34.552 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:34.615 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:34.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:34.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:34.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:34 smithi110 ceph-mon[212939]: pgmap v821: 99 pgs: 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 78 active+clean; 2.1 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 61 MiB/s wr, 974 op/s; 4356/103125 objects degraded (4.224%); 133/103125 objects misplaced (0.129%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T23:19:34.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:34 smithi110 ceph-mon[221102]: pgmap v821: 99 pgs: 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 78 active+clean; 2.1 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 61 MiB/s wr, 974 op/s; 4356/103125 objects degraded (4.224%); 133/103125 objects misplaced (0.129%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T23:19:34.724 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:34.753 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:34.811 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 24 16 40422 40406 105.212 88.5 0.00287248 0.00947301 2023-09-23T23:19:34.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:34 smithi138 ceph-mon[193606]: pgmap v821: 99 pgs: 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 10 active+undersized, 78 active+clean; 2.1 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 61 MiB/s wr, 974 op/s; 4356/103125 objects degraded (4.224%); 133/103125 objects misplaced (0.129%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T23:19:34.948 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:19:34.949 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:19:34.949 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:19:34.980 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:19:35.012 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:19:35.046 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:19:35.081 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:19:35.082 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:19:35.082 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:19:35.082 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:19:35.082 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:19:35.082 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:19:35.083 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:19:35.083 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:19:35.083 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:19:35.440 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:35.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:35.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:35.508 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:19:35.547 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:35.624 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:35.685 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:35.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:35.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:35.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:35 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3980928784' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:35.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:35 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3980928784' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:35.790 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:35.812 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 25 16 41871 41855 104.626 90.5625 0.00734779 0.00955285 2023-09-23T23:19:35.828 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:35.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:35 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3980928784' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:36.077 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:36.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:36.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:36.181 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:36.221 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:36.294 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:19:36.295 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:19:36.403 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:36.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:36.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:36.510 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:36.556 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:36.633 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:36.697 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:36.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:36 smithi110 ceph-mon[212939]: pgmap v822: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s; 5014/119859 objects degraded (4.183%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T23:19:36.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:36 smithi110 ceph-mon[221102]: pgmap v822: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s; 5014/119859 objects degraded (4.183%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T23:19:36.705 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:36.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:36.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:36.811 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:36.812 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 26 16 43586 43570 104.724 107.188 0.00807633 0.00954391 2023-09-23T23:19:36.876 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:36 smithi138 ceph-mon[193606]: pgmap v822: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s; 5014/119859 objects degraded (4.183%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T23:19:36.974 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:19:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:19:36] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.43.0" 2023-09-23T23:19:37.014 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:37.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:37.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:37.118 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:19:37.166 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:37.582 INFO:teuthology.orchestra.run.smithi110.stderr:set pool 1 pgp_num to 11 2023-09-23T23:19:37.812 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 27 16 45297 45281 104.805 106.938 0.00650426 0.009537 2023-09-23T23:19:37.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:37 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/805787018' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T23:19:37.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:37 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T23:19:37.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:37 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/805787018' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T23:19:37.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:37 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T23:19:37.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:37 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/805787018' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T23:19:37.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:37 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T23:19:38.296 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:19:38.588 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:38 smithi110 ceph-mon[212939]: pgmap v823: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 53 MiB/s wr, 851 op/s; 5014/119859 objects degraded (4.183%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T23:19:38.588 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:38 smithi110 ceph-mon[212939]: failed to encode map e271 with expected crc 2023-09-23T23:19:38.588 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:38 smithi110 ceph-mon[212939]: failed to encode map e271 with expected crc 2023-09-23T23:19:38.589 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:38 smithi110 ceph-mon[212939]: failed to encode map e271 with expected crc 2023-09-23T23:19:38.589 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:38 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T23:19:38.589 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:38 smithi110 ceph-mon[212939]: osdmap e271: 8 total, 7 up, 6 in 2023-09-23T23:19:38.589 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:38 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 5014/119859 objects degraded (4.183%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:38.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:38 smithi110 ceph-mon[221102]: pgmap v823: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 53 MiB/s wr, 851 op/s; 5014/119859 objects degraded (4.183%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T23:19:38.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:38 smithi110 ceph-mon[221102]: failed to encode map e271 with expected crc 2023-09-23T23:19:38.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:38 smithi110 ceph-mon[221102]: failed to encode map e271 with expected crc 2023-09-23T23:19:38.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:38 smithi110 ceph-mon[221102]: failed to encode map e271 with expected crc 2023-09-23T23:19:38.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:38 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T23:19:38.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:38 smithi110 ceph-mon[221102]: osdmap e271: 8 total, 7 up, 6 in 2023-09-23T23:19:38.590 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:38 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 5014/119859 objects degraded (4.183%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:38.689 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:19:38.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:38.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:19:38.794 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:38.812 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 28 16 46942 46926 104.734 102.812 0.00975697 0.00954283 2023-09-23T23:19:38.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:38 smithi138 ceph-mon[193606]: pgmap v823: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 53 MiB/s wr, 851 op/s; 5014/119859 objects degraded (4.183%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T23:19:38.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:38 smithi138 ceph-mon[193606]: failed to encode map e271 with expected crc 2023-09-23T23:19:38.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:38 smithi138 ceph-mon[193606]: failed to encode map e271 with expected crc 2023-09-23T23:19:38.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:38 smithi138 ceph-mon[193606]: failed to encode map e271 with expected crc 2023-09-23T23:19:38.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:38 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T23:19:38.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:38 smithi138 ceph-mon[193606]: osdmap e271: 8 total, 7 up, 6 in 2023-09-23T23:19:38.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:38 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 5014/119859 objects degraded (4.183%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:38.869 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:38.963 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:38.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:38.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:39.072 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:39.143 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:39.264 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:39.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:39.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:39.387 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:39.463 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:39.562 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:39.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:39.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:39.677 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:39.741 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:39.812 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 29 16 48442 48426 104.354 93.75 0.0128731 0.00957875 2023-09-23T23:19:39.840 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:39.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:39.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:39.922 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:39.950 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:39.995 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:40.111 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:40.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:40.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:40.220 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:40.262 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:40.648 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:40.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:40.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:40.712 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:19:40.712 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:19:40.712 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:19:40.712 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:19:40.712 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:19:40.763 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:40.813 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 30 16 49960 49944 104.038 94.875 0.0102354 0.00960657 2023-09-23T23:19:40.829 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:40 smithi110 ceph-mon[212939]: pgmap v825: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s; 5983/138852 objects degraded (4.309%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T23:19:40.830 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:40 smithi110 ceph-mon[221102]: pgmap v825: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s; 5983/138852 objects degraded (4.309%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T23:19:40.838 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:40 smithi138 ceph-mon[193606]: pgmap v825: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s; 5983/138852 objects degraded (4.309%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T23:19:40.839 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:40.949 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:40.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:40.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:41.059 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:41.125 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:41.223 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:41.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:41.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:41.326 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:41.396 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:41.437 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:41.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:41.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:41.542 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:41.574 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:41.622 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:19:41.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:41 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/793482164' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:19:41.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:41 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:19:41.812 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 31 16 51220 51204 103.222 78.75 0.0111148 0.00968308 2023-09-23T23:19:41.877 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:41 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/793482164' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:19:41.878 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:41 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:19:41.880 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:41 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/793482164' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:19:41.880 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:41 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:19:42.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:42 smithi110 ceph-mon[212939]: pgmap v826: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s; 5983/143277 objects degraded (4.176%); 854 KiB/s, 13 objects/s recovering 2023-09-23T23:19:42.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:42 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:42.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:42 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:19:42.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:42 smithi110 ceph-mon[212939]: osdmap e272: 8 total, 7 up, 6 in 2023-09-23T23:19:42.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:42 smithi110 ceph-mon[221102]: pgmap v826: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s; 5983/143277 objects degraded (4.176%); 854 KiB/s, 13 objects/s recovering 2023-09-23T23:19:42.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:42 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:42.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:42 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:19:42.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:42 smithi110 ceph-mon[221102]: osdmap e272: 8 total, 7 up, 6 in 2023-09-23T23:19:42.812 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 32 16 52330 52314 102.164 69.375 0.0091288 0.00978226 2023-09-23T23:19:42.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:42 smithi138 ceph-mon[193606]: pgmap v826: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s; 5983/143277 objects degraded (4.176%); 854 KiB/s, 13 objects/s recovering 2023-09-23T23:19:42.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:42 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:42.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:42 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:19:42.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:42 smithi138 ceph-mon[193606]: osdmap e272: 8 total, 7 up, 6 in 2023-09-23T23:19:43.110 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:43.119 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:19:43.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:43.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:43.223 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:43.259 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:43.442 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:43.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:43.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:43.548 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:43.588 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:43.683 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:43.686 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:43 smithi110 ceph-mon[212939]: failed to encode map e272 with expected crc 2023-09-23T23:19:43.687 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:43 smithi110 ceph-mon[212939]: failed to encode map e272 with expected crc 2023-09-23T23:19:43.687 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:43 smithi110 ceph-mon[212939]: failed to encode map e272 with expected crc 2023-09-23T23:19:43.687 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:19:43.687 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:43 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 5983/143277 objects degraded (4.176%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:43.687 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:43 smithi110 ceph-mon[221102]: failed to encode map e272 with expected crc 2023-09-23T23:19:43.687 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:43 smithi110 ceph-mon[221102]: failed to encode map e272 with expected crc 2023-09-23T23:19:43.687 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:43 smithi110 ceph-mon[221102]: failed to encode map e272 with expected crc 2023-09-23T23:19:43.687 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:19:43.687 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:43 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 5983/143277 objects degraded (4.176%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:43.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:43.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:43.789 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:43.800 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:43.813 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 33 16 53798 53782 101.848 91.75 0.0117186 0.00981376 2023-09-23T23:19:43.848 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:43 smithi138 ceph-mon[193606]: failed to encode map e272 with expected crc 2023-09-23T23:19:43.848 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:43 smithi138 ceph-mon[193606]: failed to encode map e272 with expected crc 2023-09-23T23:19:43.848 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:43 smithi138 ceph-mon[193606]: failed to encode map e272 with expected crc 2023-09-23T23:19:43.848 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:19:43.848 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:43 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 5983/143277 objects degraded (4.176%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:43.851 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:43.870 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:19:43.902 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:19:43.927 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:19:43.953 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:19:43.956 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:43.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:43.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:43.980 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:19:44.010 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:19:44.039 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:19:44.039 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:19:44.039 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:19:44.039 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:19:44.039 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:19:44.040 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:19:44.040 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:19:44.040 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:19:44.040 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:19:44.068 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:44.101 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:44.621 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7] out_osds: [2, 1] dead_osds: [7] live_osds: [0, 1, 3, 4, 5, 6, 2] 2023-09-23T23:19:44.622 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:19:44.622 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T23:19:44.622 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.782736 2023-09-23T23:19:44.622 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 0.7827364279852448 2023-09-23T23:19:44.813 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 34 16 55166 55150 101.367 85.5 0.012253 0.00986089 2023-09-23T23:19:44.840 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:44.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:44.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:44.944 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:44.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:44 smithi138 ceph-mon[193606]: pgmap v828: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 61 MiB/s wr, 975 op/s; 5983/143277 objects degraded (4.176%) 2023-09-23T23:19:44.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:44 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2904272903' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:44.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:44 smithi110 ceph-mon[212939]: pgmap v828: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 61 MiB/s wr, 975 op/s; 5983/143277 objects degraded (4.176%) 2023-09-23T23:19:44.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:44 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2904272903' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:44.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:44 smithi110 ceph-mon[221102]: pgmap v828: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 61 MiB/s wr, 975 op/s; 5983/143277 objects degraded (4.176%) 2023-09-23T23:19:44.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:44 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2904272903' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:44.985 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:45.094 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:19:45.378 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:45.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:45.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:45.483 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:45.527 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:45.813 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 35 16 56522 56506 100.892 84.75 0.01095 0.00990701 2023-09-23T23:19:45.917 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:19:45.917 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:19:46.066 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:46.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:46.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:46.170 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:19:46.234 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:46.293 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:46.314 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:46.380 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:19:46.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:46.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:19:46.486 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:19:46.559 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:46.650 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:46 smithi110 ceph-mon[212939]: pgmap v829: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 114 MiB/s wr, 1.83k op/s; 6829/163758 objects degraded (4.170%) 2023-09-23T23:19:46.650 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:46 smithi110 ceph-mon[221102]: pgmap v829: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 114 MiB/s wr, 1.83k op/s; 6829/163758 objects degraded (4.170%) 2023-09-23T23:19:46.673 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:19:46.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:46.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:19:46.784 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:46.813 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 36 16 57888 57872 100.461 85.375 0.00993927 0.00994847 2023-09-23T23:19:46.854 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:46 smithi138 ceph-mon[193606]: pgmap v829: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 114 MiB/s wr, 1.83k op/s; 6829/163758 objects degraded (4.170%) 2023-09-23T23:19:46.860 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:46.946 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:46.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:46.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:47.058 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:47.079 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:19:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:19:46] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.43.0" 2023-09-23T23:19:47.088 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:19:47.089 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:19:47.118 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:47.252 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:47.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:47.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:47.361 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:47.406 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:47.724 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:47.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:47.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:47.813 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 37 16 59206 59190 99.9719 82.375 0.00903854 0.00998504 2023-09-23T23:19:47.827 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:19:47.853 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:47.894 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:19:47.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:47.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:19:47.919 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:19:47.998 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:48.062 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:48.153 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:48.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:48.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:48.265 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:48.302 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:48.424 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:48.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:48.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:48.533 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:19:48.590 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:48.686 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:19:48.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:48 smithi138 ceph-mon[193606]: pgmap v830: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 92 MiB/s wr, 1.48k op/s; 6829/163758 objects degraded (4.170%) 2023-09-23T23:19:48.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:48 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 6829/163758 objects degraded (4.170%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:48.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:48.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:19:48.796 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:19:48.813 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 38 16 60213 60197 98.9972 62.9375 0.00595665 0.0100797 2023-09-23T23:19:48.821 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:48 smithi110 ceph-mon[212939]: pgmap v830: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 92 MiB/s wr, 1.48k op/s; 6829/163758 objects degraded (4.170%) 2023-09-23T23:19:48.821 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:48 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 6829/163758 objects degraded (4.170%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:48.821 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:48 smithi110 ceph-mon[221102]: pgmap v830: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 92 MiB/s wr, 1.48k op/s; 6829/163758 objects degraded (4.170%) 2023-09-23T23:19:48.821 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:48 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 6829/163758 objects degraded (4.170%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:48.824 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:49.726 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:19:49.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:49.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:19:49.740 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:19:49.813 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 39 16 61363 61347 98.3016 71.875 0.00902333 0.010169 2023-09-23T23:19:49.835 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:49.890 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:50.024 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:50.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:50.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:50.130 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:19:50.150 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:50.180 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:50.261 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:19:50.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:50.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:19:50.373 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:50.417 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:50.494 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:50.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:50.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:50.521 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:19:50.601 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:50.673 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:50.813 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:19:50.813249+0000 min lat: 0.000955869 max lat: 2.04249 avg lat: 0.0102274 2023-09-23T23:19:50.814 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:19:50.814 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 40 16 62575 62559 97.7377 75.75 0.00566515 0.0102274 2023-09-23T23:19:50.847 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:50 smithi110 ceph-mon[221102]: pgmap v831: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 83 MiB/s wr, 1.33k op/s; 7549/178854 objects degraded (4.221%) 2023-09-23T23:19:50.849 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:50 smithi110 ceph-mon[212939]: pgmap v831: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 83 MiB/s wr, 1.33k op/s; 7549/178854 objects degraded (4.221%) 2023-09-23T23:19:50.850 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:50.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:50.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:50.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:50 smithi138 ceph-mon[193606]: pgmap v831: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 83 MiB/s wr, 1.33k op/s; 7549/178854 objects degraded (4.221%) 2023-09-23T23:19:50.955 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:19:51.051 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:51.199 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:19:51.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:51.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:19:51.307 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:51.335 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:51.377 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:51.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:51.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:51.480 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:51.557 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:51.613 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:51 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1628413659' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:19:51.613 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:51 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2356977412' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7827364279852448}]: dispatch 2023-09-23T23:19:51.613 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:51 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7827364279852448}]: dispatch 2023-09-23T23:19:51.629 INFO:teuthology.orchestra.run.smithi110.stderr:set osd.0 primary-affinity to 0.782736 (8512972) 2023-09-23T23:19:51.629 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:19:51.653 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:51.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:51.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:51.764 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:51.804 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:51.813 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 41 16 63593 63577 96.9055 63.625 0.00417862 0.0103045 2023-09-23T23:19:51.941 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:51 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1628413659' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:19:51.941 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:51 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2356977412' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7827364279852448}]: dispatch 2023-09-23T23:19:51.942 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:51 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7827364279852448}]: dispatch 2023-09-23T23:19:51.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:51 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1628413659' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:19:51.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:51 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2356977412' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7827364279852448}]: dispatch 2023-09-23T23:19:51.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:51 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7827364279852448}]: dispatch 2023-09-23T23:19:52.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:52 smithi110 ceph-mon[212939]: pgmap v832: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 81 MiB/s wr, 1.30k op/s; 7549/182349 objects degraded (4.140%) 2023-09-23T23:19:52.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:52 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:52.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:52 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1628413659' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:19:52.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:52 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7827364279852448}]': finished 2023-09-23T23:19:52.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:52 smithi110 ceph-mon[212939]: osdmap e273: 8 total, 7 up, 6 in 2023-09-23T23:19:52.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:52 smithi110 ceph-mon[212939]: failed to encode map e273 with expected crc 2023-09-23T23:19:52.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:52 smithi110 ceph-mon[212939]: failed to encode map e273 with expected crc 2023-09-23T23:19:52.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:52 smithi110 ceph-mon[212939]: failed to encode map e273 with expected crc 2023-09-23T23:19:52.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:52 smithi110 ceph-mon[221102]: pgmap v832: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 81 MiB/s wr, 1.30k op/s; 7549/182349 objects degraded (4.140%) 2023-09-23T23:19:52.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:52 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:52.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:52 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1628413659' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:19:52.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:52 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7827364279852448}]': finished 2023-09-23T23:19:52.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:52 smithi110 ceph-mon[221102]: osdmap e273: 8 total, 7 up, 6 in 2023-09-23T23:19:52.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:52 smithi110 ceph-mon[221102]: failed to encode map e273 with expected crc 2023-09-23T23:19:52.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:52 smithi110 ceph-mon[221102]: failed to encode map e273 with expected crc 2023-09-23T23:19:52.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:52 smithi110 ceph-mon[221102]: failed to encode map e273 with expected crc 2023-09-23T23:19:52.813 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 42 16 64006 63990 95.2128 25.8125 0.0069097 0.0104982 2023-09-23T23:19:52.914 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:52.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:52.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:52.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:52 smithi138 ceph-mon[193606]: pgmap v832: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 81 MiB/s wr, 1.30k op/s; 7549/182349 objects degraded (4.140%) 2023-09-23T23:19:52.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:52 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:52.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:52 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1628413659' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:19:52.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:52 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7827364279852448}]': finished 2023-09-23T23:19:52.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:52 smithi138 ceph-mon[193606]: osdmap e273: 8 total, 7 up, 6 in 2023-09-23T23:19:52.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:52 smithi138 ceph-mon[193606]: failed to encode map e273 with expected crc 2023-09-23T23:19:52.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:52 smithi138 ceph-mon[193606]: failed to encode map e273 with expected crc 2023-09-23T23:19:52.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:52 smithi138 ceph-mon[193606]: failed to encode map e273 with expected crc 2023-09-23T23:19:53.022 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:53.130 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:53.217 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:53.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:53.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:53.323 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:19:53.480 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:53.533 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:19:53.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:53.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:19:53.581 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:19:53.606 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:19:53.636 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:19:53.638 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:19:53.670 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:19:53.698 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:19:53.713 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:53.730 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:19:53.762 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:19:53.763 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:19:53.763 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:19:53.763 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:19:53.763 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:19:53.763 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:19:53.763 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:19:53.763 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:19:53.764 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:19:53.807 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:19:53.813 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 43 16 65205 65189 94.7411 74.9375 0.00972911 0.0105501 2023-09-23T23:19:53.814 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:53 smithi138 ceph-mon[193606]: failed to encode map e274 with expected crc 2023-09-23T23:19:53.814 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:53 smithi138 ceph-mon[193606]: failed to encode map e274 with expected crc 2023-09-23T23:19:53.814 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:53 smithi138 ceph-mon[193606]: osdmap e274: 8 total, 7 up, 6 in 2023-09-23T23:19:53.814 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:53 smithi138 ceph-mon[193606]: failed to encode map e274 with expected crc 2023-09-23T23:19:53.814 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:53 smithi138 ceph-mon[193606]: failed to encode map e274 with expected crc 2023-09-23T23:19:53.815 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:53 smithi138 ceph-mon[193606]: failed to encode map e274 with expected crc 2023-09-23T23:19:53.815 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:53 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 7549/182349 objects degraded (4.140%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:53.815 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:53 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1209911512' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:53.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:53.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:19:53.917 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:19:53.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:53 smithi110 ceph-mon[221102]: failed to encode map e274 with expected crc 2023-09-23T23:19:53.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:53 smithi110 ceph-mon[221102]: failed to encode map e274 with expected crc 2023-09-23T23:19:53.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:53 smithi110 ceph-mon[221102]: osdmap e274: 8 total, 7 up, 6 in 2023-09-23T23:19:53.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:53 smithi110 ceph-mon[221102]: failed to encode map e274 with expected crc 2023-09-23T23:19:53.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:53 smithi110 ceph-mon[221102]: failed to encode map e274 with expected crc 2023-09-23T23:19:53.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:53 smithi110 ceph-mon[221102]: failed to encode map e274 with expected crc 2023-09-23T23:19:53.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:53 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 7549/182349 objects degraded (4.140%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:53.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:53 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1209911512' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:53.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:53 smithi110 ceph-mon[212939]: failed to encode map e274 with expected crc 2023-09-23T23:19:53.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:53 smithi110 ceph-mon[212939]: failed to encode map e274 with expected crc 2023-09-23T23:19:53.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:53 smithi110 ceph-mon[212939]: osdmap e274: 8 total, 7 up, 6 in 2023-09-23T23:19:53.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:53 smithi110 ceph-mon[212939]: failed to encode map e274 with expected crc 2023-09-23T23:19:53.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:53 smithi110 ceph-mon[212939]: failed to encode map e274 with expected crc 2023-09-23T23:19:53.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:53 smithi110 ceph-mon[212939]: failed to encode map e274 with expected crc 2023-09-23T23:19:53.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:53 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 7549/182349 objects degraded (4.140%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:53.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:53 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1209911512' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:19:53.986 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:54.079 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:19:54.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:54.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:19:54.185 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:54.214 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:54.259 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:54.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:54.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:54.362 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:19:54.398 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:54.634 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:54 smithi110 ceph-mon[212939]: pgmap v835: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 48 MiB/s wr, 774 op/s; 7549/182349 objects degraded (4.140%) 2023-09-23T23:19:54.634 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:54 smithi110 ceph-mon[212939]: osdmap e275: 8 total, 7 up, 6 in 2023-09-23T23:19:54.634 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:54 smithi110 ceph-mon[212939]: failed to encode map e275 with expected crc 2023-09-23T23:19:54.634 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:54 smithi110 ceph-mon[212939]: failed to encode map e275 with expected crc 2023-09-23T23:19:54.636 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:54 smithi110 ceph-mon[221102]: pgmap v835: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 48 MiB/s wr, 774 op/s; 7549/182349 objects degraded (4.140%) 2023-09-23T23:19:54.636 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:54 smithi110 ceph-mon[221102]: osdmap e275: 8 total, 7 up, 6 in 2023-09-23T23:19:54.636 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:54 smithi110 ceph-mon[221102]: failed to encode map e275 with expected crc 2023-09-23T23:19:54.636 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:54 smithi110 ceph-mon[221102]: failed to encode map e275 with expected crc 2023-09-23T23:19:54.676 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:19:54.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:54.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:19:54.781 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:19:54.785 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:19:54.786 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:19:54.813 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 44 16 66256 66240 94.0807 65.6875 0.00854851 0.0106242 2023-09-23T23:19:54.840 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:54.919 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:19:54.939 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:19:54.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:54 smithi138 ceph-mon[193606]: pgmap v835: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 48 MiB/s wr, 774 op/s; 7549/182349 objects degraded (4.140%) 2023-09-23T23:19:54.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:54 smithi138 ceph-mon[193606]: osdmap e275: 8 total, 7 up, 6 in 2023-09-23T23:19:54.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:54 smithi138 ceph-mon[193606]: failed to encode map e275 with expected crc 2023-09-23T23:19:54.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:54 smithi138 ceph-mon[193606]: failed to encode map e275 with expected crc 2023-09-23T23:19:54.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:54.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:19:55.049 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:55.092 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:55.158 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:55.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:55.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:55.264 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:19:55.320 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:55.434 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:55.455 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:19:55.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:55.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:19:55.561 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:55.601 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:55.669 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:55.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:55.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:55.775 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:55.811 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:55.813 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 45 16 67501 67485 93.719 77.8125 0.0153275 0.0106656 2023-09-23T23:19:55.928 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:19:55.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:55.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:19:56.032 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:56.277 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:56.319 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:56.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:56.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:56.423 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:19:56.450 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:56.513 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:19:56.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:56.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:19:56.617 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:56.639 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:56 smithi138 ceph-mon[193606]: pgmap v837: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 8149/197055 objects degraded (4.135%) 2023-09-23T23:19:56.654 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:56.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:56 smithi110 ceph-mon[212939]: pgmap v837: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 8149/197055 objects degraded (4.135%) 2023-09-23T23:19:56.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:56 smithi110 ceph-mon[221102]: pgmap v837: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 8149/197055 objects degraded (4.135%) 2023-09-23T23:19:56.787 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:19:56.804 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:56.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:56.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:56.814 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 46 16 68525 68509 93.0719 64 0.0170214 0.0107394 2023-09-23T23:19:56.913 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:56.973 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:57.130 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:19:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:19:56] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.43.0" 2023-09-23T23:19:57.156 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:57.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:57.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:57.261 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:57.355 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:57.454 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:19:57.508 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:57.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:57.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:57.633 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:19:57.685 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:57.757 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:19:57.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:57.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:19:57.814 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 47 16 69704 69688 92.6593 73.6875 0.0195453 0.0107882 2023-09-23T23:19:57.864 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:19:57.947 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:58.009 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7] out_osds: [2, 1] dead_osds: [7] live_osds: [0, 1, 3, 4, 5, 6, 2] 2023-09-23T23:19:58.009 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:19:58.009 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T23:19:58.009 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T23:19:58.009 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T23:19:58.009 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T23:19:58.044 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:19:58.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:58.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:19:58.159 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:58.251 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:58.299 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:58.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:58.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:58.307 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:19:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:19:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:70107 10.1 10:8fffcc93:::benchmark_data_smithi110_366450_object70106:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T23:19:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:58.216055+0000", 2023-09-23T23:19:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.088973450999999995, 2023-09-23T23:19:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.089008580000000004, 2023-09-23T23:19:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:19:58.307 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:19:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:19:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:19:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:19:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 70107 2023-09-23T23:19:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:19:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:19:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.216055+0000", 2023-09-23T23:19:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:19:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:19:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.216055+0000", 2023-09-23T23:19:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:19:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:19:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.216054+0000", 2023-09-23T23:19:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:19:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:19:58.309 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.216089+0000", 2023-09-23T23:19:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5379999999999997e-05 2023-09-23T23:19:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:19:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.216105+0000", 2023-09-23T23:19:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6308e-05 2023-09-23T23:19:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:19:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.216113+0000", 2023-09-23T23:19:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5979999999999999e-06 2023-09-23T23:19:58.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:19:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.218031+0000", 2023-09-23T23:19:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0019180180000000001 2023-09-23T23:19:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:19:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.218186+0000", 2023-09-23T23:19:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015535000000000001 2023-09-23T23:19:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0", 2023-09-23T23:19:58.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.218353+0000", 2023-09-23T23:19:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016709599999999999 2023-09-23T23:19:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:19:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.227010+0000", 2023-09-23T23:19:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086569539999999997 2023-09-23T23:19:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:19:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:19:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:19:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:70126 10.9 10:94366167:::benchmark_data_smithi110_366450_object70125:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T23:19:58.312 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:58.245716+0000", 2023-09-23T23:19:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.059312045000000001, 2023-09-23T23:19:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.059403656999999999, 2023-09-23T23:19:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:19:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:19:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:19:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:19:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:19:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 70126 2023-09-23T23:19:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:19:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:19:58.313 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.245716+0000", 2023-09-23T23:19:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:19:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:19:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.245716+0000", 2023-09-23T23:19:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:19:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:19:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.245714+0000", 2023-09-23T23:19:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:19:58.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:19:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.245769+0000", 2023-09-23T23:19:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4289000000000001e-05 2023-09-23T23:19:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:19:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.245783+0000", 2023-09-23T23:19:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4083000000000001e-05 2023-09-23T23:19:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:19:58.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.245788+0000", 2023-09-23T23:19:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4210000000000003e-06 2023-09-23T23:19:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:19:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.245833+0000", 2023-09-23T23:19:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5185000000000002e-05 2023-09-23T23:19:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:19:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.245985+0000", 2023-09-23T23:19:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015178699999999999 2023-09-23T23:19:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:19:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.246120+0000", 2023-09-23T23:19:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013456699999999999 2023-09-23T23:19:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:19:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.249009+0000", 2023-09-23T23:19:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028893030000000002 2023-09-23T23:19:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:19:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.252507+0000", 2023-09-23T23:19:58.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034977530000000001 2023-09-23T23:19:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:19:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:19:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:19:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:70129 10.c 10:361a318e:::benchmark_data_smithi110_366450_object70128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T23:19:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:58.248019+0000", 2023-09-23T23:19:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.057009414000000001, 2023-09-23T23:19:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.057173264000000001, 2023-09-23T23:19:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:19:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:19:58.318 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:19:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:19:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:19:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 70129 2023-09-23T23:19:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:19:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:19:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.248019+0000", 2023-09-23T23:19:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:19:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:19:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.248019+0000", 2023-09-23T23:19:58.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:19:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:19:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.248018+0000", 2023-09-23T23:19:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:19:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:19:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.248091+0000", 2023-09-23T23:19:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3275000000000006e-05 2023-09-23T23:19:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:19:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.248105+0000", 2023-09-23T23:19:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3569e-05 2023-09-23T23:19:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:19:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.248110+0000", 2023-09-23T23:19:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7899999999999996e-06 2023-09-23T23:19:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:19:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.248159+0000", 2023-09-23T23:19:58.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8199999999999999e-05 2023-09-23T23:19:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:19:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.248248+0000", 2023-09-23T23:19:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9809999999999996e-05 2023-09-23T23:19:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0", 2023-09-23T23:19:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.248398+0000", 2023-09-23T23:19:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014975700000000001 2023-09-23T23:19:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:19:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.251029+0000", 2023-09-23T23:19:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0026304140000000002 2023-09-23T23:19:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:19:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:19:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:19:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:70112 10.c 10:3e7b33ec:::benchmark_data_smithi110_366450_object70111:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T23:19:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:58.221121+0000", 2023-09-23T23:19:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.083907344999999994, 2023-09-23T23:19:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.084118680000000001, 2023-09-23T23:19:58.323 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:19:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:19:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:19:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:19:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:19:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 70112 2023-09-23T23:19:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:19:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:19:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.221121+0000", 2023-09-23T23:19:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:19:58.324 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:19:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.221121+0000", 2023-09-23T23:19:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:19:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:19:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.221120+0000", 2023-09-23T23:19:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:19:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.325 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:19:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.221426+0000", 2023-09-23T23:19:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000305655 2023-09-23T23:19:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:19:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.221439+0000", 2023-09-23T23:19:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3291e-05 2023-09-23T23:19:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:19:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.221446+0000", 2023-09-23T23:19:58.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3420000000000002e-06 2023-09-23T23:19:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:19:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.221942+0000", 2023-09-23T23:19:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00049560299999999995 2023-09-23T23:19:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:19:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.222080+0000", 2023-09-23T23:19:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013823700000000001 2023-09-23T23:19:58.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0", 2023-09-23T23:19:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.222245+0000", 2023-09-23T23:19:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016517300000000001 2023-09-23T23:19:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:19:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.224924+0000", 2023-09-23T23:19:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0026785839999999999 2023-09-23T23:19:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:19:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:19:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:19:58.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:70114 10.c 10:374f7a67:::benchmark_data_smithi110_366450_object70113:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T23:19:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:58.225304+0000", 2023-09-23T23:19:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.079724381999999996, 2023-09-23T23:19:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0799821, 2023-09-23T23:19:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:19:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:19:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:19:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:19:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:19:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 70114 2023-09-23T23:19:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:19:58.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:19:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.225304+0000", 2023-09-23T23:19:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:19:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:19:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.225304+0000", 2023-09-23T23:19:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:19:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:19:58.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.225302+0000", 2023-09-23T23:19:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:19:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:19:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.225370+0000", 2023-09-23T23:19:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7892000000000007e-05 2023-09-23T23:19:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:19:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.225382+0000", 2023-09-23T23:19:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1644999999999999e-05 2023-09-23T23:19:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.331 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:19:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.225388+0000", 2023-09-23T23:19:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1630000000000001e-06 2023-09-23T23:19:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:19:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.226253+0000", 2023-09-23T23:19:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00086489300000000004 2023-09-23T23:19:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:19:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.226374+0000", 2023-09-23T23:19:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012120099999999999 2023-09-23T23:19:58.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0", 2023-09-23T23:19:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.226522+0000", 2023-09-23T23:19:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000147622 2023-09-23T23:19:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:19:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.230618+0000", 2023-09-23T23:19:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040957119999999996 2023-09-23T23:19:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:19:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:19:58.333 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:19:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:70118 10.2 10:4c72c417:::benchmark_data_smithi110_366450_object70117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T23:19:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:58.228461+0000", 2023-09-23T23:19:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.076567449999999995, 2023-09-23T23:19:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.076870928000000005, 2023-09-23T23:19:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:19:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:19:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:19:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:19:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:19:58.334 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 70118 2023-09-23T23:19:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:19:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:19:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.228461+0000", 2023-09-23T23:19:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:19:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:19:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.228461+0000", 2023-09-23T23:19:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:19:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:19:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.228460+0000", 2023-09-23T23:19:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:19:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:19:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.228512+0000", 2023-09-23T23:19:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2079000000000001e-05 2023-09-23T23:19:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:19:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.228523+0000", 2023-09-23T23:19:58.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0934000000000001e-05 2023-09-23T23:19:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:19:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.228528+0000", 2023-09-23T23:19:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7639999999999996e-06 2023-09-23T23:19:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:19:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.228561+0000", 2023-09-23T23:19:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3256000000000001e-05 2023-09-23T23:19:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:19:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.228667+0000", 2023-09-23T23:19:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106291 2023-09-23T23:19:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-09-23T23:19:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.228756+0000", 2023-09-23T23:19:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8735999999999999e-05 2023-09-23T23:19:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:19:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.230765+0000", 2023-09-23T23:19:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020089819999999999 2023-09-23T23:19:58.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:19:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.239046+0000", 2023-09-23T23:19:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0082812249999999997 2023-09-23T23:19:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:19:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:19:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:19:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:70119 10.2 10:46539b7a:::benchmark_data_smithi110_366450_object70118:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T23:19:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:58.232339+0000", 2023-09-23T23:19:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.072689045999999993, 2023-09-23T23:19:58.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.073042791999999995, 2023-09-23T23:19:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:19:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:19:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:19:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:19:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:19:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 70119 2023-09-23T23:19:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:19:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:19:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.232339+0000", 2023-09-23T23:19:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:19:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:19:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.232339+0000", 2023-09-23T23:19:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:19:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:19:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.232338+0000", 2023-09-23T23:19:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:19:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:19:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.232368+0000", 2023-09-23T23:19:58.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0345000000000002e-05 2023-09-23T23:19:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:19:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.232385+0000", 2023-09-23T23:19:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6761000000000001e-05 2023-09-23T23:19:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:19:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.232391+0000", 2023-09-23T23:19:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3289999999999999e-06 2023-09-23T23:19:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:19:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.232488+0000", 2023-09-23T23:19:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7115999999999996e-05 2023-09-23T23:19:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:19:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.232619+0000", 2023-09-23T23:19:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013079500000000001 2023-09-23T23:19:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-09-23T23:19:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.232828+0000", 2023-09-23T23:19:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000208562 2023-09-23T23:19:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:19:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.238011+0000", 2023-09-23T23:19:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051831860000000002 2023-09-23T23:19:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:19:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:19:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:19:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:58.239079+0000", 2023-09-23T23:19:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010682129999999999 2023-09-23T23:19:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:19:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:19:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:19:58.344 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:19:58.345 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-09-23T23:19:58.345 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 7 2023-09-23T23:19:58.345 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:19:58.403 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:58.436 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:58.653 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:19:58.697 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:58.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:58.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:58.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:58 smithi110 ceph-mon[221102]: pgmap v838: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 51 MiB/s wr, 816 op/s; 8149/197055 objects degraded (4.135%) 2023-09-23T23:19:58.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:58 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3642122678' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:19:58.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:58 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:19:58.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:19:58.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:58 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 8149/197055 objects degraded (4.135%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:58.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:58 smithi110 ceph-mon[212939]: pgmap v838: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 51 MiB/s wr, 816 op/s; 8149/197055 objects degraded (4.135%) 2023-09-23T23:19:58.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:58 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3642122678' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:19:58.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:58 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:19:58.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:19:58.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:58 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 8149/197055 objects degraded (4.135%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:58.805 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:19:58.814 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 48 16 70644 70628 91.9528 58.75 0.0172328 0.0108702 2023-09-23T23:19:58.850 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:58.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:58 smithi138 ceph-mon[193606]: pgmap v838: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 51 MiB/s wr, 816 op/s; 8149/197055 objects degraded (4.135%) 2023-09-23T23:19:58.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:58 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3642122678' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:19:58.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:58 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:19:58.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:19:58.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:58 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 8149/197055 objects degraded (4.135%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:19:59.241 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:19:59.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:59.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:19:59.344 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:19:59.374 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:59.479 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:19:59.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:59.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:19:59.588 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:19:59.677 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:19:59.814 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 49 16 71390 71374 91.0276 46.625 0.00584513 0.0109801 2023-09-23T23:19:59.824 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:19:59.834 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:19:59.838 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:19:59.848 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:19:59.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:19:59.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:19:59.876 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:19:59.902 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:19:59.915 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:19:59.915 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:59 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:59.915 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:59 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:19:59.915 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:19:59 smithi138 ceph-mon[193606]: osdmap e276: 8 total, 7 up, 6 in 2023-09-23T23:19:59.936 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:19:59.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:59 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:59.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:59 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:19:59.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:19:59 smithi110 ceph-mon[221102]: osdmap e276: 8 total, 7 up, 6 in 2023-09-23T23:19:59.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:59 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:19:59.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:59 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:19:59.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:19:59 smithi110 ceph-mon[212939]: osdmap e276: 8 total, 7 up, 6 in 2023-09-23T23:19:59.952 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:19:59.972 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:19:59.990 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:00.008 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:20:00.008 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:20:00.008 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:20:00.008 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:20:00.009 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:20:00.009 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:20:00.009 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:20:00.009 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:20:00.009 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:20:00.050 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:00.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:00.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:00.154 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:00.193 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:00.266 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:00.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:00.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:00.370 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:00.400 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:00.458 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:00.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:00.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:00.564 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:20:00.594 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:00.800 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:20:00.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:00.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:20:00.814 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 50 16 72436 72420 90.5145 65.375 0.00915532 0.0110439 2023-09-23T23:20:00.855 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[221102]: failed to encode map e276 with expected crc 2023-09-23T23:20:00.855 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[221102]: failed to encode map e276 with expected crc 2023-09-23T23:20:00.855 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[221102]: pgmap v840: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s; 8149/206001 objects degraded (3.956%) 2023-09-23T23:20:00.855 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/593332415' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:00.855 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[221102]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 8149/206001 objects degraded (3.956%), 5 pgs degraded 2023-09-23T23:20:00.855 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[221102]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T23:20:00.855 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[221102]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T23:20:00.855 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[221102]: osd.7 (root=default,host=smithi138) is down 2023-09-23T23:20:00.855 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[221102]: [WRN] PG_DEGRADED: Degraded data redundancy: 8149/206001 objects degraded (3.956%), 5 pgs degraded 2023-09-23T23:20:00.856 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[221102]: pg 3.d is active+undersized+degraded, acting [6,3] 2023-09-23T23:20:00.856 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[221102]: pg 3.13 is active+undersized+degraded, acting [4,6] 2023-09-23T23:20:00.856 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[221102]: pg 3.14 is active+undersized+degraded, acting [4,6] 2023-09-23T23:20:00.856 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[221102]: pg 10.1 is active+undersized+degraded, acting [4,0] 2023-09-23T23:20:00.856 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[221102]: pg 10.c is active+undersized+degraded, acting [4,0] 2023-09-23T23:20:00.856 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:20:00.857 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:20:00.857 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[212939]: failed to encode map e276 with expected crc 2023-09-23T23:20:00.857 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[212939]: failed to encode map e276 with expected crc 2023-09-23T23:20:00.857 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[212939]: pgmap v840: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s; 8149/206001 objects degraded (3.956%) 2023-09-23T23:20:00.857 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/593332415' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:00.858 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[212939]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 8149/206001 objects degraded (3.956%), 5 pgs degraded 2023-09-23T23:20:00.858 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[212939]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T23:20:00.858 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[212939]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T23:20:00.858 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[212939]: osd.7 (root=default,host=smithi138) is down 2023-09-23T23:20:00.858 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[212939]: [WRN] PG_DEGRADED: Degraded data redundancy: 8149/206001 objects degraded (3.956%), 5 pgs degraded 2023-09-23T23:20:00.858 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[212939]: pg 3.d is active+undersized+degraded, acting [6,3] 2023-09-23T23:20:00.858 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[212939]: pg 3.13 is active+undersized+degraded, acting [4,6] 2023-09-23T23:20:00.858 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[212939]: pg 3.14 is active+undersized+degraded, acting [4,6] 2023-09-23T23:20:00.858 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[212939]: pg 10.1 is active+undersized+degraded, acting [4,0] 2023-09-23T23:20:00.858 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:00 smithi110 ceph-mon[212939]: pg 10.c is active+undersized+degraded, acting [4,0] 2023-09-23T23:20:00.905 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:00 smithi138 ceph-mon[193606]: failed to encode map e276 with expected crc 2023-09-23T23:20:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:00 smithi138 ceph-mon[193606]: failed to encode map e276 with expected crc 2023-09-23T23:20:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:00 smithi138 ceph-mon[193606]: pgmap v840: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s; 8149/206001 objects degraded (3.956%) 2023-09-23T23:20:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:00 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/593332415' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:00 smithi138 ceph-mon[193606]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 8149/206001 objects degraded (3.956%), 5 pgs degraded 2023-09-23T23:20:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:00 smithi138 ceph-mon[193606]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T23:20:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:00 smithi138 ceph-mon[193606]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T23:20:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:00 smithi138 ceph-mon[193606]: osd.7 (root=default,host=smithi138) is down 2023-09-23T23:20:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:00 smithi138 ceph-mon[193606]: [WRN] PG_DEGRADED: Degraded data redundancy: 8149/206001 objects degraded (3.956%), 5 pgs degraded 2023-09-23T23:20:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:00 smithi138 ceph-mon[193606]: pg 3.d is active+undersized+degraded, acting [6,3] 2023-09-23T23:20:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:00 smithi138 ceph-mon[193606]: pg 3.13 is active+undersized+degraded, acting [4,6] 2023-09-23T23:20:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:00 smithi138 ceph-mon[193606]: pg 3.14 is active+undersized+degraded, acting [4,6] 2023-09-23T23:20:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:00 smithi138 ceph-mon[193606]: pg 10.1 is active+undersized+degraded, acting [4,0] 2023-09-23T23:20:00.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:00 smithi138 ceph-mon[193606]: pg 10.c is active+undersized+degraded, acting [4,0] 2023-09-23T23:20:00.981 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:01.077 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:01.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:01.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:01.182 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:01.209 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:01.251 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:01.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:01.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:01.357 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:01.370 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:20:01.391 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:01.471 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:01.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:01.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:01.586 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:01.629 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:01.684 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:20:01.692 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:01.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:01.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:01.801 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:20:01.815 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 51 16 73337 73321 89.8438 56.3125 0.0222278 0.0111258 2023-09-23T23:20:01.873 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:01.880 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:02.112 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:20:02.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:02.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:20:02.224 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:02.261 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:02.378 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:02.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:02.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:02.485 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:02.577 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:02.698 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:20:02.698 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "" 2023-09-23T23:20:02.698 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:20:02.766 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:02.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:02.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:02.815 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 52 16 74385 74369 89.3756 65.5 0.00619173 0.0111842 2023-09-23T23:20:02.858 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:20:02.878 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:02.943 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:02 smithi110 ceph-mon[212939]: pgmap v841: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 80 MiB/s wr, 1.27k op/s; 8827/212961 objects degraded (4.145%) 2023-09-23T23:20:02.943 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:02 smithi110 ceph-mon[221102]: pgmap v841: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 80 MiB/s wr, 1.27k op/s; 8827/212961 objects degraded (4.145%) 2023-09-23T23:20:02.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:02 smithi138 ceph-mon[193606]: pgmap v841: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 80 MiB/s wr, 1.27k op/s; 8827/212961 objects degraded (4.145%) 2023-09-23T23:20:02.946 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:03.321 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:03.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:03.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:03.429 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:03.481 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:03.542 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:03.562 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:03.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:03.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:03.666 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:20:03.669 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:03 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 8827/212961 objects degraded (4.145%), 5 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:03.709 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:03.815 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 53 16 75218 75202 88.6715 52.0625 0.0652449 0.0112682 2023-09-23T23:20:03.848 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:20:03.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:03.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:20:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:03 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 8827/212961 objects degraded (4.145%), 5 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:03.951 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:03 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 8827/212961 objects degraded (4.145%), 5 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:03.953 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:04.012 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:04.322 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:04.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:04.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:04.432 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:04.487 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:04.534 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:04.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:04.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:04.639 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:04.671 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:04 smithi110 ceph-mon[221102]: pgmap v842: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s; 8827/212961 objects degraded (4.145%) 2023-09-23T23:20:04.671 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:04 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/343525244' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:20:04.692 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:20:04.728 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:04.790 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:20:04.790 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:20:04.815 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 54 16 76120 76104 88.0733 56.375 0.0261952 0.0113492 2023-09-23T23:20:04.842 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:04.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:04.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:04.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:04 smithi138 ceph-mon[193606]: pgmap v842: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s; 8827/212961 objects degraded (4.145%) 2023-09-23T23:20:04.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:04 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/343525244' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:20:04.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:04 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:20:04.949 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:04.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:04 smithi110 ceph-mon[212939]: pgmap v842: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s; 8827/212961 objects degraded (4.145%) 2023-09-23T23:20:04.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:04 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/343525244' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:20:04.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:04 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:20:04.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:04 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:20:05.032 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:05.088 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:05.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:05.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:05.192 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:05.265 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:05.436 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:05.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:05.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:05.556 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:20:05.582 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:05.818 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 55 16 76944 76928 87.4078 51.5 0.0101094 0.0114354 2023-09-23T23:20:05.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:05 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:20:05.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:05 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:20:05.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:05 smithi138 ceph-mon[193606]: osdmap e277: 8 total, 7 up, 6 in 2023-09-23T23:20:05.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:05 smithi138 ceph-mon[193606]: failed to encode map e277 with expected crc 2023-09-23T23:20:05.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:05 smithi138 ceph-mon[193606]: failed to encode map e277 with expected crc 2023-09-23T23:20:05.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:05 smithi138 ceph-mon[193606]: failed to encode map e277 with expected crc 2023-09-23T23:20:05.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:05 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:20:05.949 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:20:05.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:05 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:20:05.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:05 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:20:05.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:05 smithi110 ceph-mon[212939]: osdmap e277: 8 total, 7 up, 6 in 2023-09-23T23:20:05.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:05 smithi110 ceph-mon[212939]: failed to encode map e277 with expected crc 2023-09-23T23:20:05.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:05 smithi110 ceph-mon[212939]: failed to encode map e277 with expected crc 2023-09-23T23:20:05.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:05 smithi110 ceph-mon[212939]: failed to encode map e277 with expected crc 2023-09-23T23:20:05.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:05 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:20:05.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:05 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:20:05.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:05 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:20:05.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:05 smithi110 ceph-mon[221102]: osdmap e277: 8 total, 7 up, 6 in 2023-09-23T23:20:05.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:05 smithi110 ceph-mon[221102]: failed to encode map e277 with expected crc 2023-09-23T23:20:05.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:05 smithi110 ceph-mon[221102]: failed to encode map e277 with expected crc 2023-09-23T23:20:05.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:05 smithi110 ceph-mon[221102]: failed to encode map e277 with expected crc 2023-09-23T23:20:05.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:05 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:20:05.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:05.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:20:05.960 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:20:05.989 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:20:06.024 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:20:06.053 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:20:06.057 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:06.087 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:20:06.095 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:20:06.100 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:06.120 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:20:06.164 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:20:06.166 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:20:06.166 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:20:06.167 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:20:06.167 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:20:06.167 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:20:06.167 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:20:06.167 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:20:06.167 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:20:06.228 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:06.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:06.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:06.335 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:06.461 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:06.607 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:06.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:06.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:06.711 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:20:06.738 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:06.815 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 56 16 77810 77794 86.8134 54.125 0.0120901 0.011514 2023-09-23T23:20:06.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:06 smithi138 ceph-mon[193606]: pgmap v844: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s; 9449/227478 objects degraded (4.154%) 2023-09-23T23:20:06.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:06 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1202474080' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:06.945 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:20:06.951 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:20:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:20:06] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.43.0" 2023-09-23T23:20:06.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:06 smithi110 ceph-mon[221102]: pgmap v844: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s; 9449/227478 objects degraded (4.154%) 2023-09-23T23:20:06.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:06 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1202474080' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:06.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:06 smithi110 ceph-mon[212939]: pgmap v844: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s; 9449/227478 objects degraded (4.154%) 2023-09-23T23:20:06.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:06 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1202474080' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:07.155 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:20:07.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:07.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:20:07.239 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:20:07.239 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:20:07.260 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:07.304 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:07.364 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:07.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:07.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:07.472 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:20:07.473 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:07.578 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:07.816 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 57 16 78843 78827 86.4229 64.5625 0.0119773 0.0115656 2023-09-23T23:20:07.829 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:07.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:07.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:07.936 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:08.003 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:08.004 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:08.816 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 58 16 79764 79748 85.9253 57.5625 0.0148309 0.0116338 2023-09-23T23:20:08.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:08 smithi110 ceph-mon[212939]: pgmap v845: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s; 9449/227478 objects degraded (4.154%) 2023-09-23T23:20:08.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:08 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 9449/227478 objects degraded (4.154%), 5 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:08.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:08 smithi110 ceph-mon[221102]: pgmap v845: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s; 9449/227478 objects degraded (4.154%) 2023-09-23T23:20:08.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:08 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 9449/227478 objects degraded (4.154%), 5 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:08.981 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:08.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:08.991 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:08 smithi138 ceph-mon[193606]: pgmap v845: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s; 9449/227478 objects degraded (4.154%) 2023-09-23T23:20:08.991 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:08 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 9449/227478 objects degraded (4.154%), 5 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:08.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:09.092 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:09.157 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:09.240 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:20:09.308 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:09.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:09.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:09.335 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7] out_osds: [2, 1] dead_osds: [7] live_osds: [0, 1, 3, 4, 5, 6, 2] 2023-09-23T23:20:09.335 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:20:09.335 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-09-23T23:20:09.336 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd in 2 2023-09-23T23:20:09.418 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:20:09.456 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:09.816 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 59 16 80761 80745 85.525 62.3125 0.0228405 0.0116868 2023-09-23T23:20:09.827 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:20:09.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:09.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:20:09.879 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:20:09.879 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:20:09.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:20:09.879 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:20:09.879 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.880 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:171302 8.b 8:d70babc6:::benchmark_data_smithi110_258128_object52484:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T23:20:09.880 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:39.128915+0000", 2023-09-23T23:20:09.880 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 570.74003066800003, 2023-09-23T23:20:09.880 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2014737069999999, 2023-09-23T23:20:09.880 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:09.880 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:09.880 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:09.880 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:20:09.880 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:20:09.880 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 171302 2023-09-23T23:20:09.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.880 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:09.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:09.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128915+0000", 2023-09-23T23:20:09.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:09.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128915+0000", 2023-09-23T23:20:09.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:09.882 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128913+0000", 2023-09-23T23:20:09.882 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:09.882 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.882 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.882 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:09.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128920+0000", 2023-09-23T23:20:09.883 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4930000000000003e-06 2023-09-23T23:20:09.883 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.883 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.883 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:09.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128944+0000", 2023-09-23T23:20:09.883 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4297000000000001e-05 2023-09-23T23:20:09.883 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.883 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.883 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:09.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.128952+0000", 2023-09-23T23:20:09.884 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.712e-06 2023-09-23T23:20:09.884 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.884 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.884 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.884 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.129003+0000", 2023-09-23T23:20:09.884 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0538999999999998e-05 2023-09-23T23:20:09.884 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.884 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.884 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:09.884 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.129013+0000", 2023-09-23T23:20:09.884 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0188e-05 2023-09-23T23:20:09.884 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.885 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.885 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.885 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.146049+0000", 2023-09-23T23:20:09.885 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0170364089999999 2023-09-23T23:20:09.885 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.885 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.885 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:09.885 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.148680+0000", 2023-09-23T23:20:09.885 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0026305450000000002 2023-09-23T23:20:09.885 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.885 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.885 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1", 2023-09-23T23:20:09.885 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.148979+0000", 2023-09-23T23:20:09.886 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00029888000000000001 2023-09-23T23:20:09.886 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.886 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.886 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:09.886 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.158433+0000", 2023-09-23T23:20:09.886 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0094546939999999996 2023-09-23T23:20:09.886 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.886 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.886 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.886 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.330258+0000", 2023-09-23T23:20:09.887 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.171825009 2023-09-23T23:20:09.887 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.887 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.887 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:09.887 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.330303+0000", 2023-09-23T23:20:09.887 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4895999999999998e-05 2023-09-23T23:20:09.887 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.887 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.887 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:09.887 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.330389+0000", 2023-09-23T23:20:09.887 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5438000000000004e-05 2023-09-23T23:20:09.887 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.887 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:09.888 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.888 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.888 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.888 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:171308 8.b 8:d1ffb63e:::benchmark_data_smithi110_258128_object52490:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T23:20:09.888 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:39.134289+0000", 2023-09-23T23:20:09.888 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 570.73465658500004, 2023-09-23T23:20:09.888 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1993630239999999, 2023-09-23T23:20:09.888 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:09.888 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:09.888 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:09.888 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:20:09.888 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:20:09.889 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 171308 2023-09-23T23:20:09.889 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.889 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:09.889 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.889 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:09.889 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134289+0000", 2023-09-23T23:20:09.889 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.889 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.889 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.889 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:09.889 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134289+0000", 2023-09-23T23:20:09.889 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.889 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.890 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.890 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:09.890 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134287+0000", 2023-09-23T23:20:09.890 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:09.890 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.890 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.890 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:09.890 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134295+0000", 2023-09-23T23:20:09.890 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.187e-06 2023-09-23T23:20:09.890 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.890 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.891 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:09.891 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134313+0000", 2023-09-23T23:20:09.891 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8102e-05 2023-09-23T23:20:09.891 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.891 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.891 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:09.891 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134320+0000", 2023-09-23T23:20:09.891 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9859999999999997e-06 2023-09-23T23:20:09.891 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.891 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.891 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.891 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134380+0000", 2023-09-23T23:20:09.892 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9979999999999998e-05 2023-09-23T23:20:09.892 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.892 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.892 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:09.892 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134387+0000", 2023-09-23T23:20:09.892 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0450000000000003e-06 2023-09-23T23:20:09.892 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.892 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.892 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.892 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.153349+0000", 2023-09-23T23:20:09.892 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0189619759999999 2023-09-23T23:20:09.892 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.893 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.893 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:09.893 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.153445+0000", 2023-09-23T23:20:09.893 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6444e-05 2023-09-23T23:20:09.893 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.893 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.893 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1", 2023-09-23T23:20:09.893 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.153529+0000", 2023-09-23T23:20:09.893 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3825000000000005e-05 2023-09-23T23:20:09.893 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.893 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.893 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:09.893 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.158442+0000", 2023-09-23T23:20:09.894 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049127169999999996 2023-09-23T23:20:09.894 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.894 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.894 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.894 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.333511+0000", 2023-09-23T23:20:09.894 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.175069639 2023-09-23T23:20:09.894 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.894 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.894 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:09.894 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.333570+0000", 2023-09-23T23:20:09.895 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8323999999999999e-05 2023-09-23T23:20:09.895 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.895 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.895 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:09.895 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.333652+0000", 2023-09-23T23:20:09.895 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2400999999999999e-05 2023-09-23T23:20:09.895 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.895 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:09.895 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.895 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.895 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.895 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:171309 8.b 8:d16ada6c:::benchmark_data_smithi110_258128_object52491:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T23:20:09.895 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:39.134332+0000", 2023-09-23T23:20:09.896 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 570.73461327799998, 2023-09-23T23:20:09.896 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.199510887, 2023-09-23T23:20:09.896 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:09.896 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:09.896 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:09.896 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:20:09.896 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:20:09.896 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 171309 2023-09-23T23:20:09.896 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.896 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:09.896 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.897 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:09.897 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134332+0000", 2023-09-23T23:20:09.897 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.897 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.897 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.897 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:09.897 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134332+0000", 2023-09-23T23:20:09.897 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.897 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.897 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.897 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:09.897 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134332+0000", 2023-09-23T23:20:09.897 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:20:09.898 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.898 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.898 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:09.898 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134335+0000", 2023-09-23T23:20:09.898 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1360000000000001e-06 2023-09-23T23:20:09.898 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.898 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.898 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:09.898 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134341+0000", 2023-09-23T23:20:09.898 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7760000000000003e-06 2023-09-23T23:20:09.898 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.898 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.899 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:09.899 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134345+0000", 2023-09-23T23:20:09.899 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7579999999999998e-06 2023-09-23T23:20:09.899 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.899 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.899 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.899 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134404+0000", 2023-09-23T23:20:09.899 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9416999999999997e-05 2023-09-23T23:20:09.899 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.899 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.899 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:09.899 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:39.134409+0000", 2023-09-23T23:20:09.899 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6120000000000001e-06 2023-09-23T23:20:09.900 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.900 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.900 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.900 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.156116+0000", 2023-09-23T23:20:09.900 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0217073729999999 2023-09-23T23:20:09.900 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.900 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.900 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:09.900 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.156208+0000", 2023-09-23T23:20:09.900 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2495000000000001e-05 2023-09-23T23:20:09.900 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.900 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.901 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1", 2023-09-23T23:20:09.901 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.156285+0000", 2023-09-23T23:20:09.901 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6478999999999997e-05 2023-09-23T23:20:09.901 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.901 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.901 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:09.901 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.158580+0000", 2023-09-23T23:20:09.901 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022948370000000001 2023-09-23T23:20:09.901 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.901 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.901 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.901 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.333697+0000", 2023-09-23T23:20:09.902 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17511763599999999 2023-09-23T23:20:09.902 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.902 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.902 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:09.902 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.333755+0000", 2023-09-23T23:20:09.902 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7218e-05 2023-09-23T23:20:09.902 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.902 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.902 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:09.902 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:40.333843+0000", 2023-09-23T23:20:09.902 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8541000000000005e-05 2023-09-23T23:20:09.902 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.903 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:09.903 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.903 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.903 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.903 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:173079 8.b 8:df2b19bb:::benchmark_data_smithi110_258128_object54261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T23:20:09.903 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:42.318557+0000", 2023-09-23T23:20:09.903 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 567.55038813800002, 2023-09-23T23:20:09.903 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.36297001299999998, 2023-09-23T23:20:09.903 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:09.903 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:09.903 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:09.903 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:20:09.903 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:20:09.904 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 173079 2023-09-23T23:20:09.904 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.904 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:09.904 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.904 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:09.904 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.318557+0000", 2023-09-23T23:20:09.904 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.904 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.904 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.904 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:09.904 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.318557+0000", 2023-09-23T23:20:09.904 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.904 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.905 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.905 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:09.905 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.318556+0000", 2023-09-23T23:20:09.905 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:09.905 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.905 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.905 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:09.905 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.318561+0000", 2023-09-23T23:20:09.905 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1309999999999997e-06 2023-09-23T23:20:09.905 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.905 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.905 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:09.905 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.318571+0000", 2023-09-23T23:20:09.906 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0339999999999999e-05 2023-09-23T23:20:09.906 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.906 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.906 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:09.906 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.318577+0000", 2023-09-23T23:20:09.906 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2700000000000004e-06 2023-09-23T23:20:09.906 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.906 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.906 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.906 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.321177+0000", 2023-09-23T23:20:09.906 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00259991 2023-09-23T23:20:09.907 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.907 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.907 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:09.907 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.321296+0000", 2023-09-23T23:20:09.907 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011894499999999999 2023-09-23T23:20:09.907 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.907 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.907 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1", 2023-09-23T23:20:09.907 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.321413+0000", 2023-09-23T23:20:09.907 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011746199999999999 2023-09-23T23:20:09.907 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.907 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.908 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:09.908 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.327193+0000", 2023-09-23T23:20:09.908 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00577978 2023-09-23T23:20:09.908 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.908 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.908 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.908 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.681380+0000", 2023-09-23T23:20:09.908 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.35418686599999999 2023-09-23T23:20:09.908 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.908 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.908 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:09.908 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.681429+0000", 2023-09-23T23:20:09.909 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9186e-05 2023-09-23T23:20:09.909 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.909 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.909 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:09.909 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.681527+0000", 2023-09-23T23:20:09.909 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8640000000000004e-05 2023-09-23T23:20:09.909 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.909 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:09.909 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.909 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.909 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.909 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:173096 8.b 8:d0616f0c:::benchmark_data_smithi110_258128_object54278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T23:20:09.909 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:42.337603+0000", 2023-09-23T23:20:09.910 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 567.53134288199999, 2023-09-23T23:20:09.910 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.34871130900000002, 2023-09-23T23:20:09.910 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:09.910 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:09.910 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:09.910 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:20:09.910 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:20:09.910 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 173096 2023-09-23T23:20:09.910 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.910 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:09.910 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.910 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:09.911 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337603+0000", 2023-09-23T23:20:09.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.911 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.911 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.911 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:09.911 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337603+0000", 2023-09-23T23:20:09.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.911 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.911 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.911 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:09.911 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337601+0000", 2023-09-23T23:20:09.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:20:09.912 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.912 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.912 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:09.912 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337609+0000", 2023-09-23T23:20:09.912 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0360000000000006e-06 2023-09-23T23:20:09.912 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.912 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.912 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:09.912 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337624+0000", 2023-09-23T23:20:09.912 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4965e-05 2023-09-23T23:20:09.912 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.912 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.912 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:09.913 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337630+0000", 2023-09-23T23:20:09.913 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.665e-06 2023-09-23T23:20:09.913 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.913 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.913 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.913 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337679+0000", 2023-09-23T23:20:09.913 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9227999999999997e-05 2023-09-23T23:20:09.913 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.913 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.913 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:09.913 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337780+0000", 2023-09-23T23:20:09.913 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101327 2023-09-23T23:20:09.913 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.914 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.914 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1", 2023-09-23T23:20:09.914 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.337859+0000", 2023-09-23T23:20:09.914 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9330000000000001e-05 2023-09-23T23:20:09.914 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.914 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.914 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:09.914 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.341195+0000", 2023-09-23T23:20:09.914 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0033355820000000001 2023-09-23T23:20:09.914 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.914 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.914 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.915 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.686177+0000", 2023-09-23T23:20:09.915 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.344981922 2023-09-23T23:20:09.915 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.915 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.915 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:09.915 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.686225+0000", 2023-09-23T23:20:09.915 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8201999999999997e-05 2023-09-23T23:20:09.915 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.915 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.915 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:09.915 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:42.686314+0000", 2023-09-23T23:20:09.915 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8801000000000006e-05 2023-09-23T23:20:09.915 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.916 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:09.916 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.916 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.916 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.916 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:177977 9.e 9:7f4c5a4f:::benchmark_data_smithi110_295904_object66796:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-09-23T23:20:09.916 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:10.280640+0000", 2023-09-23T23:20:09.916 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 179.58830543299999, 2023-09-23T23:20:09.916 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.35592362900000002, 2023-09-23T23:20:09.916 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:09.916 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:09.916 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:09.916 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:09.917 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:09.917 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 177977 2023-09-23T23:20:09.917 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.917 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:09.917 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.917 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:09.917 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.280640+0000", 2023-09-23T23:20:09.917 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.917 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.917 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.917 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:09.917 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.280640+0000", 2023-09-23T23:20:09.917 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.918 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.918 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.918 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:09.918 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.280639+0000", 2023-09-23T23:20:09.918 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:20:09.918 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.918 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.918 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:09.918 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.280644+0000", 2023-09-23T23:20:09.918 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7199999999999997e-06 2023-09-23T23:20:09.918 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.918 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.918 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:09.919 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.280655+0000", 2023-09-23T23:20:09.919 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1131000000000001e-05 2023-09-23T23:20:09.919 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.919 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.919 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:09.919 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.280661+0000", 2023-09-23T23:20:09.919 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3219999999999998e-06 2023-09-23T23:20:09.919 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.919 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.919 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.919 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.281007+0000", 2023-09-23T23:20:09.919 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0003456 2023-09-23T23:20:09.919 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.920 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:09.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.281171+0000", 2023-09-23T23:20:09.920 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016447200000000001 2023-09-23T23:20:09.920 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.920 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T23:20:09.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.281319+0000", 2023-09-23T23:20:09.920 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014733300000000001 2023-09-23T23:20:09.920 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.920 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:09.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.284890+0000", 2023-09-23T23:20:09.921 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003571603 2023-09-23T23:20:09.921 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.921 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.921 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.921 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.287513+0000", 2023-09-23T23:20:09.921 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002622732 2023-09-23T23:20:09.921 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.921 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.921 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.921 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.291419+0000", 2023-09-23T23:20:09.921 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003906456 2023-09-23T23:20:09.921 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.921 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.922 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.922 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.636405+0000", 2023-09-23T23:20:09.922 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.34498575100000001 2023-09-23T23:20:09.922 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.922 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.922 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:09.922 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.636462+0000", 2023-09-23T23:20:09.922 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6975000000000003e-05 2023-09-23T23:20:09.922 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.922 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.922 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:09.922 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.636564+0000", 2023-09-23T23:20:09.922 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101568 2023-09-23T23:20:09.923 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.923 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:09.923 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.923 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.923 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.923 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:188568 9.e 9:71d3eae7:::benchmark_data_smithi110_295904_object77387:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:20:09.923 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:33.524654+0000", 2023-09-23T23:20:09.923 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 156.344291145, 2023-09-23T23:20:09.923 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.89632690699999995, 2023-09-23T23:20:09.923 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:09.923 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:09.923 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:09.924 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:09.924 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:09.924 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 188568 2023-09-23T23:20:09.924 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.924 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:09.924 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.924 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:09.924 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.524654+0000", 2023-09-23T23:20:09.924 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.924 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.924 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.924 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:09.924 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.524654+0000", 2023-09-23T23:20:09.925 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.925 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.925 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.925 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:09.925 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.524653+0000", 2023-09-23T23:20:09.925 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:09.925 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.925 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.925 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:09.925 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.524660+0000", 2023-09-23T23:20:09.926 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.861e-06 2023-09-23T23:20:09.926 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.926 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.926 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:09.926 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.524676+0000", 2023-09-23T23:20:09.926 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6052000000000001e-05 2023-09-23T23:20:09.926 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.926 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.926 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:09.926 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.524683+0000", 2023-09-23T23:20:09.926 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0940000000000002e-06 2023-09-23T23:20:09.926 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.927 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.927 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.524759+0000", 2023-09-23T23:20:09.927 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6048000000000001e-05 2023-09-23T23:20:09.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.927 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:09.927 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.524774+0000", 2023-09-23T23:20:09.927 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4982e-05 2023-09-23T23:20:09.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.927 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.928 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.415935+0000", 2023-09-23T23:20:09.928 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.89116085199999995 2023-09-23T23:20:09.928 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.928 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.928 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:09.928 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.416036+0000", 2023-09-23T23:20:09.928 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101493 2023-09-23T23:20:09.928 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.928 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.928 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T23:20:09.928 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.416475+0000", 2023-09-23T23:20:09.928 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000439018 2023-09-23T23:20:09.929 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.929 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.929 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.929 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.420801+0000", 2023-09-23T23:20:09.929 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043260030000000001 2023-09-23T23:20:09.929 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.929 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.929 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.929 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.420845+0000", 2023-09-23T23:20:09.929 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3825000000000001e-05 2023-09-23T23:20:09.929 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.929 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.930 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:09.930 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.420853+0000", 2023-09-23T23:20:09.930 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1450000000000008e-06 2023-09-23T23:20:09.930 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.930 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.930 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.930 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.420880+0000", 2023-09-23T23:20:09.930 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6478999999999998e-05 2023-09-23T23:20:09.930 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.930 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.930 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:09.930 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.420916+0000", 2023-09-23T23:20:09.931 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5964000000000003e-05 2023-09-23T23:20:09.931 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.931 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.931 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:09.931 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.420981+0000", 2023-09-23T23:20:09.931 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5512000000000006e-05 2023-09-23T23:20:09.931 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.931 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:09.931 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.931 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.931 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.931 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:188601 9.e 9:7a546467:::benchmark_data_smithi110_295904_object77420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:20:09.931 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:33.597788+0000", 2023-09-23T23:20:09.932 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 156.27115718499999, 2023-09-23T23:20:09.932 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.83494150099999997, 2023-09-23T23:20:09.932 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:09.932 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:09.932 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:09.932 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:09.932 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:09.932 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 188601 2023-09-23T23:20:09.932 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.932 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:09.932 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.932 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:09.933 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.597788+0000", 2023-09-23T23:20:09.933 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.933 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.933 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:09.933 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.597788+0000", 2023-09-23T23:20:09.933 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.933 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.933 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:09.933 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.597786+0000", 2023-09-23T23:20:09.933 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:20:09.934 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.934 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.934 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:09.934 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.597796+0000", 2023-09-23T23:20:09.934 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9499999999999996e-06 2023-09-23T23:20:09.934 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.934 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.934 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:09.934 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.597819+0000", 2023-09-23T23:20:09.934 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2379e-05 2023-09-23T23:20:09.934 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.934 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.935 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:09.935 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.597827+0000", 2023-09-23T23:20:09.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6570000000000003e-06 2023-09-23T23:20:09.935 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.935 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.935 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.935 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.597905+0000", 2023-09-23T23:20:09.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7583000000000006e-05 2023-09-23T23:20:09.935 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.935 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.935 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:09.935 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.597920+0000", 2023-09-23T23:20:09.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.521e-05 2023-09-23T23:20:09.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.936 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.936 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.421138+0000", 2023-09-23T23:20:09.936 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.82321820300000004 2023-09-23T23:20:09.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.936 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:09.936 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.421237+0000", 2023-09-23T23:20:09.936 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8747999999999997e-05 2023-09-23T23:20:09.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T23:20:09.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.421311+0000", 2023-09-23T23:20:09.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3942000000000005e-05 2023-09-23T23:20:09.937 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:09.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.425900+0000", 2023-09-23T23:20:09.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004589063 2023-09-23T23:20:09.937 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.425956+0000", 2023-09-23T23:20:09.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6047000000000001e-05 2023-09-23T23:20:09.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.938 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.425991+0000", 2023-09-23T23:20:09.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4814000000000002e-05 2023-09-23T23:20:09.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.938 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.432596+0000", 2023-09-23T23:20:09.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066052410000000004 2023-09-23T23:20:09.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.938 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:09.939 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.432632+0000", 2023-09-23T23:20:09.939 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.54e-05 2023-09-23T23:20:09.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.939 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:09.939 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.432730+0000", 2023-09-23T23:20:09.939 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8204999999999999e-05 2023-09-23T23:20:09.939 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.939 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:09.939 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.939 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:188618 9.e 9:7259fd00:::benchmark_data_smithi110_295904_object77437:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:20:09.940 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:33.650613+0000", 2023-09-23T23:20:09.940 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 156.21833221399999, 2023-09-23T23:20:09.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.782310116, 2023-09-23T23:20:09.940 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:09.940 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:09.940 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:09.940 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:09.940 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:09.940 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 188618 2023-09-23T23:20:09.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.940 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:09.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:09.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.650613+0000", 2023-09-23T23:20:09.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.941 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:09.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.650613+0000", 2023-09-23T23:20:09.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.941 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:09.942 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.650612+0000", 2023-09-23T23:20:09.942 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:09.942 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.942 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.942 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:09.942 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.650618+0000", 2023-09-23T23:20:09.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0769999999999996e-06 2023-09-23T23:20:09.943 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.943 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:09.943 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.650646+0000", 2023-09-23T23:20:09.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7568000000000001e-05 2023-09-23T23:20:09.944 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.944 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.944 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:09.944 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.650653+0000", 2023-09-23T23:20:09.944 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.109e-06 2023-09-23T23:20:09.944 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.945 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.650716+0000", 2023-09-23T23:20:09.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3427000000000001e-05 2023-09-23T23:20:09.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.946 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:09.946 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.650739+0000", 2023-09-23T23:20:09.946 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.279e-05 2023-09-23T23:20:09.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.947 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.423559+0000", 2023-09-23T23:20:09.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.77282010300000004 2023-09-23T23:20:09.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:09.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.423668+0000", 2023-09-23T23:20:09.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108614 2023-09-23T23:20:09.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T23:20:09.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.423751+0000", 2023-09-23T23:20:09.949 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3353e-05 2023-09-23T23:20:09.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.950 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.950 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.426022+0000", 2023-09-23T23:20:09.950 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022710930000000001 2023-09-23T23:20:09.950 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.950 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.951 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.951 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.426056+0000", 2023-09-23T23:20:09.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4044999999999997e-05 2023-09-23T23:20:09.951 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.951 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.951 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:09.952 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.426068+0000", 2023-09-23T23:20:09.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1486e-05 2023-09-23T23:20:09.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.952 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.952 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.432784+0000", 2023-09-23T23:20:09.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0067167909999999997 2023-09-23T23:20:09.953 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:09.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.432832+0000", 2023-09-23T23:20:09.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7457999999999999e-05 2023-09-23T23:20:09.954 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.954 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.954 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:09.954 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.432923+0000", 2023-09-23T23:20:09.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1658999999999998e-05 2023-09-23T23:20:09.955 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.955 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:09.955 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.956 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:189866 9.e 9:7a5298a8:::benchmark_data_smithi110_295904_object78685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T23:20:09.956 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.873989+0000", 2023-09-23T23:20:09.956 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 151.99495619199999, 2023-09-23T23:20:09.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.40595450399999999, 2023-09-23T23:20:09.956 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:09.956 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:09.957 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:09.957 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:09.957 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:09.957 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 189866 2023-09-23T23:20:09.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.957 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:09.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:09.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.873989+0000", 2023-09-23T23:20:09.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.959 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.959 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:09.959 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.873989+0000", 2023-09-23T23:20:09.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.959 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:09.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.873988+0000", 2023-09-23T23:20:09.960 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:09.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:09.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.873993+0000", 2023-09-23T23:20:09.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6179999999999997e-06 2023-09-23T23:20:09.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:09.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.874003+0000", 2023-09-23T23:20:09.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8949999999999993e-06 2023-09-23T23:20:09.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:09.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.874009+0000", 2023-09-23T23:20:09.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8309999999999997e-06 2023-09-23T23:20:09.963 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.963 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.963 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.874255+0000", 2023-09-23T23:20:09.963 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00024681099999999999 2023-09-23T23:20:09.963 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.964 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.964 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:09.964 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.874371+0000", 2023-09-23T23:20:09.964 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011542199999999999 2023-09-23T23:20:09.964 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.964 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.965 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T23:20:09.965 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.874494+0000", 2023-09-23T23:20:09.965 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012323299999999999 2023-09-23T23:20:09.965 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.965 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.965 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:09.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.878670+0000", 2023-09-23T23:20:09.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041756409999999999 2023-09-23T23:20:09.966 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.966 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.966 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.883011+0000", 2023-09-23T23:20:09.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043409989999999999 2023-09-23T23:20:09.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.888583+0000", 2023-09-23T23:20:09.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0055726439999999999 2023-09-23T23:20:09.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.968 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279764+0000", 2023-09-23T23:20:09.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39118028599999999 2023-09-23T23:20:09.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:09.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279809+0000", 2023-09-23T23:20:09.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5655000000000002e-05 2023-09-23T23:20:09.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:09.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279944+0000", 2023-09-23T23:20:09.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000134671 2023-09-23T23:20:09.970 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.970 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:09.970 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.971 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:189870 9.e 9:70f28e9e:::benchmark_data_smithi110_295904_object78689:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T23:20:09.971 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.879664+0000", 2023-09-23T23:20:09.971 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 151.98928157700001, 2023-09-23T23:20:09.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.40061539299999999, 2023-09-23T23:20:09.972 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:09.972 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:09.972 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:09.972 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:09.972 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:09.972 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 189870 2023-09-23T23:20:09.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.973 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:09.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:09.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879664+0000", 2023-09-23T23:20:09.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.974 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:09.974 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879664+0000", 2023-09-23T23:20:09.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.975 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.975 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.975 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:09.975 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879662+0000", 2023-09-23T23:20:09.975 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:20:09.975 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.976 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.976 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:09.976 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879668+0000", 2023-09-23T23:20:09.976 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1530000000000002e-06 2023-09-23T23:20:09.976 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.976 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:09.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879680+0000", 2023-09-23T23:20:09.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1406000000000001e-05 2023-09-23T23:20:09.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:09.978 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879686+0000", 2023-09-23T23:20:09.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8600000000000004e-06 2023-09-23T23:20:09.978 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.978 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879733+0000", 2023-09-23T23:20:09.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6013000000000001e-05 2023-09-23T23:20:09.979 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.979 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.979 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:09.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879859+0000", 2023-09-23T23:20:09.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012643800000000001 2023-09-23T23:20:09.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.980 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T23:20:09.980 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879989+0000", 2023-09-23T23:20:09.980 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000130028 2023-09-23T23:20:09.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.981 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:09.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.883117+0000", 2023-09-23T23:20:09.981 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003128058 2023-09-23T23:20:09.981 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.982 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.982 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.982 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.888632+0000", 2023-09-23T23:20:09.982 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0055148460000000003 2023-09-23T23:20:09.982 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.983 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.983 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.983 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279989+0000", 2023-09-23T23:20:09.983 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39135692999999999 2023-09-23T23:20:09.983 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.984 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.984 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.984 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.280159+0000", 2023-09-23T23:20:09.984 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017025300000000001 2023-09-23T23:20:09.984 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.984 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.984 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:09.985 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.280205+0000", 2023-09-23T23:20:09.985 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6001000000000002e-05 2023-09-23T23:20:09.985 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.985 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.985 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:09.985 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.280279+0000", 2023-09-23T23:20:09.986 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4320999999999995e-05 2023-09-23T23:20:09.986 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.986 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:09.986 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:09.986 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.986 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.987 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:189877 9.e 9:731f13fe:::benchmark_data_smithi110_295904_object78696:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T23:20:09.987 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.884881+0000", 2023-09-23T23:20:09.987 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 151.98406480200001, 2023-09-23T23:20:09.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39560695299999998, 2023-09-23T23:20:09.987 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:09.988 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:09.988 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:09.988 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:09.988 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:09.988 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 189877 2023-09-23T23:20:09.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.989 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:09.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.989 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:09.989 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.884881+0000", 2023-09-23T23:20:09.989 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.989 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.990 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.990 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:09.990 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.884881+0000", 2023-09-23T23:20:09.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:09.990 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.991 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.991 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:09.991 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.884879+0000", 2023-09-23T23:20:09.991 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:09.991 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.991 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:09.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.884886+0000", 2023-09-23T23:20:09.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2740000000000002e-06 2023-09-23T23:20:09.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:09.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.884901+0000", 2023-09-23T23:20:09.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4231e-05 2023-09-23T23:20:09.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.994 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:09.994 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.884908+0000", 2023-09-23T23:20:09.994 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.81e-06 2023-09-23T23:20:09.994 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.995 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:09.995 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.884957+0000", 2023-09-23T23:20:09.995 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9024999999999999e-05 2023-09-23T23:20:09.995 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.995 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.995 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:09.996 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.885086+0000", 2023-09-23T23:20:09.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012970700000000001 2023-09-23T23:20:09.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.996 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T23:20:09.997 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.885200+0000", 2023-09-23T23:20:09.997 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000113587 2023-09-23T23:20:09.997 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.997 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.997 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.888654+0000", 2023-09-23T23:20:09.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034543899999999999 2023-09-23T23:20:09.998 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.998 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.998 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:09.998 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.888660+0000", 2023-09-23T23:20:09.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1700000000000002e-06 2023-09-23T23:20:09.999 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:09.999 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:09.999 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:09.999 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.280023+0000", 2023-09-23T23:20:09.999 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39136293700000002 2023-09-23T23:20:09.999 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.000 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.000 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:10.000 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.280332+0000", 2023-09-23T23:20:10.000 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00030889200000000001 2023-09-23T23:20:10.000 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.000 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:10.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.280374+0000", 2023-09-23T23:20:10.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2070000000000002e-05 2023-09-23T23:20:10.001 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.001 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.002 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:10.002 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.280488+0000", 2023-09-23T23:20:10.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000113399 2023-09-23T23:20:10.002 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.002 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:10.003 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.003 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.003 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45164.0:189880 9.1 e230/218 9:87969838:::benchmark_data_smithi110_295904_object78699:head v 230'11922, mlcod=230'11920)", 2023-09-23T23:20:10.003 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.895576+0000", 2023-09-23T23:20:10.003 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 151.97336913800001, 2023-09-23T23:20:10.004 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.384357911, 2023-09-23T23:20:10.004 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:10.004 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:10.004 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:10.004 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.004 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:10.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.895576+0000", 2023-09-23T23:20:10.005 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:10.005 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.005 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:10.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.895576+0000", 2023-09-23T23:20:10.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:10.006 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.006 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:10.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.895576+0000", 2023-09-23T23:20:10.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:20:10.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.007 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.007 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:10.007 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.895583+0000", 2023-09-23T23:20:10.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9589999999999998e-06 2023-09-23T23:20:10.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.007 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.007 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:10.008 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.895595+0000", 2023-09-23T23:20:10.008 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2418e-05 2023-09-23T23:20:10.008 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:10.008 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.895601+0000", 2023-09-23T23:20:10.008 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1419999999999998e-06 2023-09-23T23:20:10.008 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:10.008 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.895689+0000", 2023-09-23T23:20:10.008 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7548999999999998e-05 2023-09-23T23:20:10.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.009 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:10.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.895709+0000", 2023-09-23T23:20:10.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9868e-05 2023-09-23T23:20:10.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.009 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:10.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279904+0000", 2023-09-23T23:20:10.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.384195071 2023-09-23T23:20:10.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.010 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:10.010 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279934+0000", 2023-09-23T23:20:10.010 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0558000000000001e-05 2023-09-23T23:20:10.010 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.010 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:10.010 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.010 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.010 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45164.0:189879 9.c e230/164 9:3a3914a2:::benchmark_data_smithi110_295904_object78698:head v 230'11867, mlcod=230'11864)", 2023-09-23T23:20:10.010 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.896307+0000", 2023-09-23T23:20:10.010 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 151.97263875300001, 2023-09-23T23:20:10.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.383339608, 2023-09-23T23:20:10.011 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:10.011 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:10.011 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:10.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:10.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.896307+0000", 2023-09-23T23:20:10.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:10.011 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:10.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.896307+0000", 2023-09-23T23:20:10.012 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:10.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.012 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:10.012 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.896306+0000", 2023-09-23T23:20:10.012 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:20:10.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.012 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:10.012 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.896313+0000", 2023-09-23T23:20:10.012 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9619999999999996e-06 2023-09-23T23:20:10.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.013 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:10.013 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.896323+0000", 2023-09-23T23:20:10.013 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4490000000000005e-06 2023-09-23T23:20:10.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.013 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.013 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:10.013 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.896328+0000", 2023-09-23T23:20:10.013 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3290000000000001e-06 2023-09-23T23:20:10.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.013 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.013 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:10.013 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.896373+0000", 2023-09-23T23:20:10.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4653000000000001e-05 2023-09-23T23:20:10.014 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:10.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.896386+0000", 2023-09-23T23:20:10.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3088000000000001e-05 2023-09-23T23:20:10.014 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:10.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279611+0000", 2023-09-23T23:20:10.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38322504400000001 2023-09-23T23:20:10.014 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.015 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:10.015 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279646+0000", 2023-09-23T23:20:10.015 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5778000000000002e-05 2023-09-23T23:20:10.015 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.015 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:10.015 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.015 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.45164.0:189866 9.e e230/227 ondisk, result = 0)", 2023-09-23T23:20:10.015 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.899217+0000", 2023-09-23T23:20:10.015 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 151.96972818, 2023-09-23T23:20:10.015 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38073507499999998, 2023-09-23T23:20:10.016 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:10.016 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:20:10.016 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:10.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:10.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.899217+0000", 2023-09-23T23:20:10.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:10.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:10.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.899217+0000", 2023-09-23T23:20:10.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:10.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.017 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:10.017 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.899217+0000", 2023-09-23T23:20:10.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:20:10.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.017 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:10.017 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.899225+0000", 2023-09-23T23:20:10.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9610000000000005e-06 2023-09-23T23:20:10.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.018 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:10.018 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279591+0000", 2023-09-23T23:20:10.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38036651300000002 2023-09-23T23:20:10.018 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.018 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.018 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:10.018 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279604+0000", 2023-09-23T23:20:10.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3256e-05 2023-09-23T23:20:10.018 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.018 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.018 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:10.018 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279754+0000", 2023-09-23T23:20:10.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014917399999999999 2023-09-23T23:20:10.019 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.019 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.019 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:10.019 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279759+0000", 2023-09-23T23:20:10.019 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5809999999999996e-06 2023-09-23T23:20:10.019 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.019 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.019 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:10.019 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279952+0000", 2023-09-23T23:20:10.019 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019326900000000001 2023-09-23T23:20:10.019 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.019 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:10.020 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.020 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45164.0:189883 9.c e230/164 9:3785bba4:::benchmark_data_smithi110_295904_object78702:head v 230'11868, mlcod=230'11865)", 2023-09-23T23:20:10.020 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.900642+0000", 2023-09-23T23:20:10.020 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 151.96830376700001, 2023-09-23T23:20:10.020 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.388835292, 2023-09-23T23:20:10.020 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:10.020 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:10.020 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:10.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.020 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:10.020 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.900642+0000", 2023-09-23T23:20:10.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:10.021 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.021 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:10.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.900642+0000", 2023-09-23T23:20:10.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:10.021 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.021 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:10.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.900641+0000", 2023-09-23T23:20:10.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:20:10.021 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.022 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:10.022 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.900648+0000", 2023-09-23T23:20:10.022 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4569999999999999e-06 2023-09-23T23:20:10.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.022 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:10.022 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279592+0000", 2023-09-23T23:20:10.022 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.37894372799999998 2023-09-23T23:20:10.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.022 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:10.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279604+0000", 2023-09-23T23:20:10.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1575e-05 2023-09-23T23:20:10.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.023 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:10.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279672+0000", 2023-09-23T23:20:10.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8100000000000002e-05 2023-09-23T23:20:10.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.023 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:10.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279693+0000", 2023-09-23T23:20:10.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0786000000000001e-05 2023-09-23T23:20:10.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.024 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:10.024 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.289430+0000", 2023-09-23T23:20:10.024 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0097376070000000006 2023-09-23T23:20:10.024 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.024 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:10.024 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.289477+0000", 2023-09-23T23:20:10.024 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6825e-05 2023-09-23T23:20:10.024 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.024 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:10.024 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.024 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.025 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.025 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:207686 9.e 9:7c3a69f3:::benchmark_data_smithi110_295904_object96505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-23T23:20:10.025 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:18:15.313700+0000", 2023-09-23T23:20:10.025 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 114.555245505, 2023-09-23T23:20:10.025 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.29027254800000002, 2023-09-23T23:20:10.025 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:10.025 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:10.025 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:10.025 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:10.025 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:10.025 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 207686 2023-09-23T23:20:10.025 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.026 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:10.026 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.026 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:10.026 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313700+0000", 2023-09-23T23:20:10.026 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:10.026 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.026 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.026 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:10.026 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313700+0000", 2023-09-23T23:20:10.026 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:10.026 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.026 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:10.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313698+0000", 2023-09-23T23:20:10.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-09-23T23:20:10.027 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.027 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:10.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313712+0000", 2023-09-23T23:20:10.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.393e-05 2023-09-23T23:20:10.027 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.027 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:10.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313746+0000", 2023-09-23T23:20:10.028 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4085000000000003e-05 2023-09-23T23:20:10.028 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.028 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.028 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:10.028 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313760+0000", 2023-09-23T23:20:10.028 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.435e-05 2023-09-23T23:20:10.028 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.028 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.028 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:10.028 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313818+0000", 2023-09-23T23:20:10.028 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8167000000000001e-05 2023-09-23T23:20:10.029 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.029 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.029 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:10.029 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313913+0000", 2023-09-23T23:20:10.029 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5057999999999997e-05 2023-09-23T23:20:10.029 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.029 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.029 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:20:10.029 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313987+0000", 2023-09-23T23:20:10.029 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4214000000000005e-05 2023-09-23T23:20:10.029 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.029 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.029 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:10.030 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.316356+0000", 2023-09-23T23:20:10.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002368462 2023-09-23T23:20:10.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.030 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.030 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:10.030 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.316611+0000", 2023-09-23T23:20:10.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025519300000000002 2023-09-23T23:20:10.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.030 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.030 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:10.030 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.603884+0000", 2023-09-23T23:20:10.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.287272841 2023-09-23T23:20:10.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.031 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.031 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:10.031 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.603934+0000", 2023-09-23T23:20:10.031 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9842999999999997e-05 2023-09-23T23:20:10.031 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.031 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.031 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:10.031 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.603973+0000", 2023-09-23T23:20:10.031 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8831000000000001e-05 2023-09-23T23:20:10.031 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.031 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:10.031 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.031 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.032 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.032 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:70902 10.2 10:4adf2976:::benchmark_data_smithi110_366450_object70901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T23:20:10.032 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:59.013259+0000", 2023-09-23T23:20:10.032 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 10.855686144, 2023-09-23T23:20:10.032 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39337688999999998, 2023-09-23T23:20:10.032 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:10.032 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:10.032 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:10.032 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:20:10.032 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:20:10.032 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 70902 2023-09-23T23:20:10.032 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.033 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:10.033 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.033 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:10.033 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013259+0000", 2023-09-23T23:20:10.033 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:10.033 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.033 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.033 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:10.033 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013259+0000", 2023-09-23T23:20:10.033 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:10.033 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.033 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.033 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:10.034 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013258+0000", 2023-09-23T23:20:10.034 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:20:10.034 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.034 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.034 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:10.034 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013323+0000", 2023-09-23T23:20:10.034 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4944999999999996e-05 2023-09-23T23:20:10.034 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.034 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.034 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:10.034 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013334+0000", 2023-09-23T23:20:10.034 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0142999999999999e-05 2023-09-23T23:20:10.034 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.035 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.035 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:10.035 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013341+0000", 2023-09-23T23:20:10.035 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0979999999999996e-06 2023-09-23T23:20:10.035 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.035 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.035 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:10.035 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013383+0000", 2023-09-23T23:20:10.035 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2188999999999999e-05 2023-09-23T23:20:10.035 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.035 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.035 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:10.036 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013498+0000", 2023-09-23T23:20:10.036 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011552099999999999 2023-09-23T23:20:10.036 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.036 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.036 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-09-23T23:20:10.036 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013642+0000", 2023-09-23T23:20:10.036 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014336500000000001 2023-09-23T23:20:10.036 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.036 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.036 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:10.036 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.017890+0000", 2023-09-23T23:20:10.036 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0042483140000000004 2023-09-23T23:20:10.036 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.037 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.037 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:10.037 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.020865+0000", 2023-09-23T23:20:10.037 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029748169999999998 2023-09-23T23:20:10.037 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.037 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.037 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:10.037 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.406538+0000", 2023-09-23T23:20:10.037 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38567305200000002 2023-09-23T23:20:10.037 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.037 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.037 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:10.038 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.406592+0000", 2023-09-23T23:20:10.038 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4456999999999997e-05 2023-09-23T23:20:10.038 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.038 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.038 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:10.038 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.406636+0000", 2023-09-23T23:20:10.038 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3968999999999999e-05 2023-09-23T23:20:10.038 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.038 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:10.038 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.038 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.038 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.038 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:70912 10.e 10:709d2b27:::benchmark_data_smithi110_366450_object70911:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T23:20:10.039 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:59.024249+0000", 2023-09-23T23:20:10.039 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 10.844696900000001, 2023-09-23T23:20:10.039 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39032808699999999, 2023-09-23T23:20:10.039 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:10.039 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:10.039 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:10.039 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:20:10.039 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:20:10.039 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 70912 2023-09-23T23:20:10.039 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.039 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:10.039 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.040 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:10.040 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024249+0000", 2023-09-23T23:20:10.040 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:10.040 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.040 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.040 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:10.040 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024249+0000", 2023-09-23T23:20:10.040 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:10.040 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.040 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.041 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:10.041 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024247+0000", 2023-09-23T23:20:10.041 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:10.041 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.041 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.041 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:10.041 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024326+0000", 2023-09-23T23:20:10.041 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8335000000000001e-05 2023-09-23T23:20:10.041 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.041 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.041 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:10.041 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024337+0000", 2023-09-23T23:20:10.041 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1573000000000001e-05 2023-09-23T23:20:10.042 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.042 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.042 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:10.042 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024343+0000", 2023-09-23T23:20:10.042 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.088e-06 2023-09-23T23:20:10.042 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.042 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.042 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:10.042 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024379+0000", 2023-09-23T23:20:10.042 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5998999999999999e-05 2023-09-23T23:20:10.042 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.042 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.042 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:10.043 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024488+0000", 2023-09-23T23:20:10.043 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109059 2023-09-23T23:20:10.043 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.043 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.043 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:20:10.043 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024622+0000", 2023-09-23T23:20:10.043 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013397800000000001 2023-09-23T23:20:10.043 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.043 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.043 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:10.043 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.029840+0000", 2023-09-23T23:20:10.044 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0052171960000000003 2023-09-23T23:20:10.044 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.044 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.044 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:10.044 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.030243+0000", 2023-09-23T23:20:10.044 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00040324000000000002 2023-09-23T23:20:10.044 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.044 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.044 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:10.044 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.414483+0000", 2023-09-23T23:20:10.044 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38424069100000002 2023-09-23T23:20:10.044 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.044 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.045 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:10.045 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.414536+0000", 2023-09-23T23:20:10.045 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2833000000000001e-05 2023-09-23T23:20:10.045 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.045 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.045 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:10.045 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.414577+0000", 2023-09-23T23:20:10.045 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0436999999999999e-05 2023-09-23T23:20:10.045 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.045 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:10.045 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.045 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.046 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.046 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:70917 10.2 10:4e149dab:::benchmark_data_smithi110_366450_object70916:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T23:20:10.046 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:59.027511+0000", 2023-09-23T23:20:10.046 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 10.84143441, 2023-09-23T23:20:10.046 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39079427300000003, 2023-09-23T23:20:10.046 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:10.046 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:10.046 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:10.046 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:20:10.046 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:20:10.046 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 70917 2023-09-23T23:20:10.046 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.046 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:10.047 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.047 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:10.047 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027511+0000", 2023-09-23T23:20:10.047 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:10.047 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.047 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.047 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:10.047 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027511+0000", 2023-09-23T23:20:10.047 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:10.047 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.047 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.047 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:10.047 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027510+0000", 2023-09-23T23:20:10.048 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:20:10.048 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.048 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.048 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:10.048 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027577+0000", 2023-09-23T23:20:10.048 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6310999999999996e-05 2023-09-23T23:20:10.048 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.048 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.048 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:10.049 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027589+0000", 2023-09-23T23:20:10.049 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2481e-05 2023-09-23T23:20:10.049 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.049 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.049 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:10.049 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027595+0000", 2023-09-23T23:20:10.049 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1940000000000003e-06 2023-09-23T23:20:10.049 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.049 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.049 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:10.049 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027650+0000", 2023-09-23T23:20:10.049 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4588e-05 2023-09-23T23:20:10.049 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.050 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:10.050 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027761+0000", 2023-09-23T23:20:10.050 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110839 2023-09-23T23:20:10.050 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.050 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-09-23T23:20:10.050 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027897+0000", 2023-09-23T23:20:10.050 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000136737 2023-09-23T23:20:10.050 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.050 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:10.050 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.030294+0000", 2023-09-23T23:20:10.051 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023962549999999999 2023-09-23T23:20:10.051 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.051 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.051 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:10.051 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.032258+0000", 2023-09-23T23:20:10.051 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0019644300000000001 2023-09-23T23:20:10.051 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.051 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.051 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:10.051 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.418216+0000", 2023-09-23T23:20:10.051 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38595784999999999 2023-09-23T23:20:10.051 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.052 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.052 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:10.052 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.418267+0000", 2023-09-23T23:20:10.052 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0973999999999997e-05 2023-09-23T23:20:10.052 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:10.052 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:10.052 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:10.052 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.418305+0000", 2023-09-23T23:20:10.052 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8534000000000001e-05 2023-09-23T23:20:10.052 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.052 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:10.052 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.052 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:10.053 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:10.053 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:20:10.053 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:10.054 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:10.098 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:10.138 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:10.459 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:10.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:10.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:10.572 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:10.602 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:10.816 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:20:10.815810+0000 min lat: 0.000955869 max lat: 2.04249 avg lat: 0.0117757 2023-09-23T23:20:10.816 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:20:10.816 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 60 16 81469 81453 84.837 44.25 0.00726216 0.0117757 2023-09-23T23:20:10.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:10 smithi110 ceph-mon[212939]: pgmap v846: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 61 MiB/s wr, 970 op/s; 9449/235134 objects degraded (4.019%) 2023-09-23T23:20:10.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:10 smithi110 ceph-mon[221102]: pgmap v846: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 61 MiB/s wr, 970 op/s; 9449/235134 objects degraded (4.019%) 2023-09-23T23:20:11.066 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:10 smithi138 ceph-mon[193606]: pgmap v846: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 61 MiB/s wr, 970 op/s; 9449/235134 objects degraded (4.019%) 2023-09-23T23:20:11.092 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:11.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:11.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:11.107 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:20:11.197 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:11.295 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:11.479 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:11.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:11.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:11.585 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:11.620 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:11.718 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:20:11.816 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 61 15 82341 82326 84.3407 54.5625 0.0104423 0.0118523 2023-09-23T23:20:11.916 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:20:11.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:11 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1562271546' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:20:11.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:11 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1562271546' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:20:11.982 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:11.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:11.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:12.087 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:12.137 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:12.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:11 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1562271546' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:20:12.359 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:12.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:12.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:12.465 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:20:12.495 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:12.816 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 62 16 83076 83060 83.7202 45.875 0.0122047 0.0119242 2023-09-23T23:20:12.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:12 smithi138 ceph-mon[193606]: pgmap v847: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 59 MiB/s wr, 936 op/s; 10037/241062 objects degraded (4.164%) 2023-09-23T23:20:12.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:12 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:20:12.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:12 smithi138 ceph-mon[193606]: failed to encode map e278 with expected crc 2023-09-23T23:20:12.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:12 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1562271546' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:20:12.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:12 smithi138 ceph-mon[193606]: osdmap e278: 8 total, 7 up, 6 in 2023-09-23T23:20:12.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:12 smithi138 ceph-mon[193606]: failed to encode map e278 with expected crc 2023-09-23T23:20:12.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:12 smithi138 ceph-mon[193606]: failed to encode map e278 with expected crc 2023-09-23T23:20:12.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:12 smithi138 ceph-mon[193606]: failed to encode map e278 with expected crc 2023-09-23T23:20:12.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:12 smithi138 ceph-mon[193606]: failed to encode map e278 with expected crc 2023-09-23T23:20:12.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:12 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:12.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:12 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:12.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[221102]: pgmap v847: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 59 MiB/s wr, 936 op/s; 10037/241062 objects degraded (4.164%) 2023-09-23T23:20:12.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:20:12.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[221102]: failed to encode map e278 with expected crc 2023-09-23T23:20:12.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1562271546' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:20:12.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[221102]: osdmap e278: 8 total, 7 up, 6 in 2023-09-23T23:20:12.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[221102]: failed to encode map e278 with expected crc 2023-09-23T23:20:12.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[221102]: failed to encode map e278 with expected crc 2023-09-23T23:20:12.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[221102]: failed to encode map e278 with expected crc 2023-09-23T23:20:12.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[221102]: failed to encode map e278 with expected crc 2023-09-23T23:20:12.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:12.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:12.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[212939]: pgmap v847: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 59 MiB/s wr, 936 op/s; 10037/241062 objects degraded (4.164%) 2023-09-23T23:20:12.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:20:12.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[212939]: failed to encode map e278 with expected crc 2023-09-23T23:20:12.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1562271546' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:20:12.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[212939]: osdmap e278: 8 total, 7 up, 6 in 2023-09-23T23:20:12.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[212939]: failed to encode map e278 with expected crc 2023-09-23T23:20:12.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[212939]: failed to encode map e278 with expected crc 2023-09-23T23:20:12.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[212939]: failed to encode map e278 with expected crc 2023-09-23T23:20:12.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[212939]: failed to encode map e278 with expected crc 2023-09-23T23:20:12.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:12.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:12 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:13.818 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 63 16 83886 83870 83.1927 50.625 0.00802048 0.0120165 2023-09-23T23:20:13.906 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:20:13.941 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:20:13.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:20:13.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:13 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 10037/241062 objects degraded (4.164%), 5 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:13.972 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:20:14.001 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:20:14.012 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:20:14.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:14.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:20:14.039 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:20:14.068 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:20:14.110 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:20:14.110 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:20:14.110 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:20:14.110 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:20:14.110 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:20:14.110 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:20:14.111 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:20:14.111 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:20:14.111 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:20:14.120 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:14.132 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:20:14.132 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:13 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 10037/241062 objects degraded (4.164%), 5 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:14.132 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:20:14.132 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:13 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 10037/241062 objects degraded (4.164%), 5 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:14.161 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:14.338 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:14.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:14.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:14.445 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:14.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:14 smithi110 conmon[212914]: 2023-09-23T23:20:14.290+0000 7f29aa464700 -1 mon.a@0(leader).osd e278 definitely_dead 0 2023-09-23T23:20:14.472 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:14.514 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:14.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:14.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:14.619 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:14.651 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:14.730 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:14 smithi110 ceph-mon[212939]: pgmap v849: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 32 MiB/s wr, 514 op/s; 10037/241062 objects degraded (4.164%) 2023-09-23T23:20:14.730 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:14 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/493088749' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:14.730 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:14 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/336982812' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T23:20:14.730 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:14 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T23:20:14.730 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:14 smithi110 ceph-mon[221102]: pgmap v849: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 32 MiB/s wr, 514 op/s; 10037/241062 objects degraded (4.164%) 2023-09-23T23:20:14.731 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:14 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/493088749' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:14.731 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:14 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/336982812' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T23:20:14.731 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:14 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T23:20:14.735 INFO:teuthology.orchestra.run.smithi110.stderr:marked in osd.2. 2023-09-23T23:20:14.818 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 64 16 84687 84671 82.675 50.0625 0.00274905 0.0120793 2023-09-23T23:20:15.042 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:20:15.042 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:20:15.070 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:14 smithi138 ceph-mon[193606]: pgmap v849: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 32 MiB/s wr, 514 op/s; 10037/241062 objects degraded (4.164%) 2023-09-23T23:20:15.070 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:14 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/493088749' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:15.070 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:14 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/336982812' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T23:20:15.070 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:14 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T23:20:15.290 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:20:15.291 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:20:15.312 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:20:15.357 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:15.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:15.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:15.464 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:15.490 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:15.555 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-09-23T23:20:15.586 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:15.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:15.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:15.687 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:15.692 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:15.727 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:15.818 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 65 16 84687 84671 81.4031 0 - 0.0120793 2023-09-23T23:20:15.825 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:15.827 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[221102]: failed to encode map e279 with expected crc 2023-09-23T23:20:15.827 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T23:20:15.827 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[221102]: osdmap e279: 8 total, 7 up, 7 in 2023-09-23T23:20:15.827 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[221102]: failed to encode map e279 with expected crc 2023-09-23T23:20:15.827 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[221102]: failed to encode map e279 with expected crc 2023-09-23T23:20:15.827 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[221102]: failed to encode map e279 with expected crc 2023-09-23T23:20:15.827 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:15.827 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:15.828 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[221102]: osdmap e280: 8 total, 7 up, 7 in 2023-09-23T23:20:15.828 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[212939]: failed to encode map e279 with expected crc 2023-09-23T23:20:15.828 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T23:20:15.828 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[212939]: osdmap e279: 8 total, 7 up, 7 in 2023-09-23T23:20:15.828 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[212939]: failed to encode map e279 with expected crc 2023-09-23T23:20:15.828 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[212939]: failed to encode map e279 with expected crc 2023-09-23T23:20:15.828 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[212939]: failed to encode map e279 with expected crc 2023-09-23T23:20:15.828 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:15.828 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:15.828 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:15 smithi110 ceph-mon[212939]: osdmap e280: 8 total, 7 up, 7 in 2023-09-23T23:20:15.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:15.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:15.930 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:15.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:15 smithi138 ceph-mon[193606]: failed to encode map e279 with expected crc 2023-09-23T23:20:15.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:15 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T23:20:15.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:15 smithi138 ceph-mon[193606]: osdmap e279: 8 total, 7 up, 7 in 2023-09-23T23:20:15.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:15 smithi138 ceph-mon[193606]: failed to encode map e279 with expected crc 2023-09-23T23:20:15.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:15 smithi138 ceph-mon[193606]: failed to encode map e279 with expected crc 2023-09-23T23:20:15.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:15 smithi138 ceph-mon[193606]: failed to encode map e279 with expected crc 2023-09-23T23:20:15.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:15 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:15.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:15 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:15.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:15 smithi138 ceph-mon[193606]: osdmap e280: 8 total, 7 up, 7 in 2023-09-23T23:20:15.961 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:16.014 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:16.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:16.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:16.085 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:20:16.116 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:20:16.143 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:16.186 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:20:16.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:16.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:20:16.290 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:16.317 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:16.360 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:16.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:16.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:16.427 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:20:16.463 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:16.491 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:16.533 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:16.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:16.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:16.637 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:16.663 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:16.804 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:20:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:20:16] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.43.0" 2023-09-23T23:20:16.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[212939]: pgmap v851: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 67 MiB/s wr, 1.08k op/s; 10512/253332 objects degraded (4.149%) 2023-09-23T23:20:16.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[212939]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[212939]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[212939]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[212939]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[212939]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[212939]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[212939]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.805 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:20:16.805 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:20:16.805 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:16.805 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[212939]: osdmap e281: 8 total, 7 up, 7 in 2023-09-23T23:20:16.807 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[221102]: pgmap v851: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 67 MiB/s wr, 1.08k op/s; 10512/253332 objects degraded (4.149%) 2023-09-23T23:20:16.808 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[221102]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.808 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[221102]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.808 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[221102]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.808 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[221102]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.808 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[221102]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.808 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[221102]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.808 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[221102]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.808 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:20:16.808 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:20:16.808 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:16.808 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:16 smithi110 ceph-mon[221102]: osdmap e281: 8 total, 7 up, 7 in 2023-09-23T23:20:16.819 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 66 16 84687 84671 80.168 0 - 0.0120793 2023-09-23T23:20:16.875 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:16.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:16.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:16.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:16 smithi138 ceph-mon[193606]: pgmap v851: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 67 MiB/s wr, 1.08k op/s; 10512/253332 objects degraded (4.149%) 2023-09-23T23:20:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:16 smithi138 ceph-mon[193606]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:16 smithi138 ceph-mon[193606]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:16 smithi138 ceph-mon[193606]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:16 smithi138 ceph-mon[193606]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:16 smithi138 ceph-mon[193606]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:16 smithi138 ceph-mon[193606]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:16 smithi138 ceph-mon[193606]: failed to encode map e280 with expected crc 2023-09-23T23:20:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:16 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:20:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:16 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:20:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:16 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:16.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:16 smithi138 ceph-mon[193606]: osdmap e281: 8 total, 7 up, 7 in 2023-09-23T23:20:16.985 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:17.060 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:17.218 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:17.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:17.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:17.292 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:20:17.328 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:20:17.380 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:17.652 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:20:17.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:17.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:20:17.761 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:17.803 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:17.820 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 67 16 85413 85397 79.6487 15.125 0.0317522 0.0125499 2023-09-23T23:20:17.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:17 smithi138 ceph-mon[193606]: failed to encode map e281 with expected crc 2023-09-23T23:20:17.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:17 smithi138 ceph-mon[193606]: 10.9 starting backfill to osd.2 from (0'0,0'0] MAX to 278'5261 2023-09-23T23:20:17.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:17 smithi138 ceph-mon[193606]: 10.5 starting backfill to osd.2 from (0'0,0'0] MAX to 278'5275 2023-09-23T23:20:17.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:17 smithi138 ceph-mon[193606]: failed to encode map e281 with expected crc 2023-09-23T23:20:17.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:17 smithi138 ceph-mon[193606]: failed to encode map e281 with expected crc 2023-09-23T23:20:17.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:17 smithi138 ceph-mon[193606]: failed to encode map e281 with expected crc 2023-09-23T23:20:17.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:17 smithi138 ceph-mon[193606]: failed to encode map e281 with expected crc 2023-09-23T23:20:17.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:17 smithi138 ceph-mon[193606]: failed to encode map e281 with expected crc 2023-09-23T23:20:17.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:17 smithi138 ceph-mon[193606]: 10.e starting backfill to osd.2 from (0'0,0'0] MAX to 278'5276 2023-09-23T23:20:17.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:17 smithi138 ceph-mon[193606]: 10.7 starting backfill to osd.2 from (0'0,0'0] MAX to 278'5211 2023-09-23T23:20:17.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:17 smithi138 ceph-mon[193606]: 10.f starting backfill to osd.2 from (0'0,0'0] MAX to 278'5142 2023-09-23T23:20:17.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:17 smithi138 ceph-mon[193606]: osdmap e282: 8 total, 7 up, 7 in 2023-09-23T23:20:17.979 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:17.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:17.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:18.045 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:18.114 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:20:18.182 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[212939]: failed to encode map e281 with expected crc 2023-09-23T23:20:18.182 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[212939]: 10.9 starting backfill to osd.2 from (0'0,0'0] MAX to 278'5261 2023-09-23T23:20:18.182 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[212939]: 10.5 starting backfill to osd.2 from (0'0,0'0] MAX to 278'5275 2023-09-23T23:20:18.182 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[212939]: failed to encode map e281 with expected crc 2023-09-23T23:20:18.182 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[212939]: failed to encode map e281 with expected crc 2023-09-23T23:20:18.182 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[212939]: failed to encode map e281 with expected crc 2023-09-23T23:20:18.182 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[212939]: failed to encode map e281 with expected crc 2023-09-23T23:20:18.183 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[212939]: failed to encode map e281 with expected crc 2023-09-23T23:20:18.183 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[212939]: 10.e starting backfill to osd.2 from (0'0,0'0] MAX to 278'5276 2023-09-23T23:20:18.183 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[212939]: 10.7 starting backfill to osd.2 from (0'0,0'0] MAX to 278'5211 2023-09-23T23:20:18.183 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[212939]: 10.f starting backfill to osd.2 from (0'0,0'0] MAX to 278'5142 2023-09-23T23:20:18.183 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[212939]: osdmap e282: 8 total, 7 up, 7 in 2023-09-23T23:20:18.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[221102]: failed to encode map e281 with expected crc 2023-09-23T23:20:18.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[221102]: 10.9 starting backfill to osd.2 from (0'0,0'0] MAX to 278'5261 2023-09-23T23:20:18.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[221102]: 10.5 starting backfill to osd.2 from (0'0,0'0] MAX to 278'5275 2023-09-23T23:20:18.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[221102]: failed to encode map e281 with expected crc 2023-09-23T23:20:18.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[221102]: failed to encode map e281 with expected crc 2023-09-23T23:20:18.184 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[221102]: failed to encode map e281 with expected crc 2023-09-23T23:20:18.184 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[221102]: failed to encode map e281 with expected crc 2023-09-23T23:20:18.184 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[221102]: failed to encode map e281 with expected crc 2023-09-23T23:20:18.184 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[221102]: 10.e starting backfill to osd.2 from (0'0,0'0] MAX to 278'5276 2023-09-23T23:20:18.184 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[221102]: 10.7 starting backfill to osd.2 from (0'0,0'0] MAX to 278'5211 2023-09-23T23:20:18.184 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[221102]: 10.f starting backfill to osd.2 from (0'0,0'0] MAX to 278'5142 2023-09-23T23:20:18.184 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:17 smithi110 ceph-mon[221102]: osdmap e282: 8 total, 7 up, 7 in 2023-09-23T23:20:18.185 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:18.302 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:20:18.302 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:20:18.311 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:20:18.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:18.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:20:18.419 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:18.480 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:18.744 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:18.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:18.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:18.759 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:20:18.819 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 68 16 86314 86298 79.3054 56.3125 0.0115578 0.0126053 2023-09-23T23:20:18.851 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:18.878 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:18.952 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:18.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:18.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:19.060 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:19.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[221102]: pgmap v854: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 44 MiB/s wr, 707 op/s; 10512/253332 objects degraded (4.149%) 2023-09-23T23:20:19.094 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[221102]: failed to encode map e282 with expected crc 2023-09-23T23:20:19.094 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[221102]: failed to encode map e282 with expected crc 2023-09-23T23:20:19.094 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[221102]: failed to encode map e282 with expected crc 2023-09-23T23:20:19.094 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[221102]: failed to encode map e282 with expected crc 2023-09-23T23:20:19.094 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2613615731' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:20:19.094 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:20:19.094 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 10512/253332 objects degraded (4.149%), 5 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:19.094 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:20:19.094 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:20:19.094 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[221102]: osdmap e283: 8 total, 7 up, 7 in 2023-09-23T23:20:19.095 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[212939]: pgmap v854: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 44 MiB/s wr, 707 op/s; 10512/253332 objects degraded (4.149%) 2023-09-23T23:20:19.095 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[212939]: failed to encode map e282 with expected crc 2023-09-23T23:20:19.095 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[212939]: failed to encode map e282 with expected crc 2023-09-23T23:20:19.095 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[212939]: failed to encode map e282 with expected crc 2023-09-23T23:20:19.095 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[212939]: failed to encode map e282 with expected crc 2023-09-23T23:20:19.095 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2613615731' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:20:19.095 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:20:19.095 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 10512/253332 objects degraded (4.149%), 5 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:19.096 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:20:19.096 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:20:19.096 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:18 smithi110 ceph-mon[212939]: osdmap e283: 8 total, 7 up, 7 in 2023-09-23T23:20:19.096 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:19.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:18 smithi138 ceph-mon[193606]: pgmap v854: 99 pgs: 5 active+clean+remapped, 5 active+undersized+degraded, 10 active+undersized, 79 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 44 MiB/s wr, 707 op/s; 10512/253332 objects degraded (4.149%) 2023-09-23T23:20:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:18 smithi138 ceph-mon[193606]: failed to encode map e282 with expected crc 2023-09-23T23:20:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:18 smithi138 ceph-mon[193606]: failed to encode map e282 with expected crc 2023-09-23T23:20:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:18 smithi138 ceph-mon[193606]: failed to encode map e282 with expected crc 2023-09-23T23:20:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:18 smithi138 ceph-mon[193606]: failed to encode map e282 with expected crc 2023-09-23T23:20:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2613615731' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:20:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:18 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:20:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:18 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 10512/253332 objects degraded (4.149%), 5 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:18 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:20:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:18 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:20:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:18 smithi138 ceph-mon[193606]: osdmap e283: 8 total, 7 up, 7 in 2023-09-23T23:20:19.448 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:20:19.820 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 69 16 87176 87160 78.9368 53.875 0.0333036 0.012663 2023-09-23T23:20:20.067 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:19 smithi110 ceph-mon[212939]: failed to encode map e283 with expected crc 2023-09-23T23:20:20.067 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:19 smithi110 ceph-mon[212939]: failed to encode map e283 with expected crc 2023-09-23T23:20:20.067 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:19 smithi110 ceph-mon[212939]: failed to encode map e283 with expected crc 2023-09-23T23:20:20.067 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:19 smithi110 ceph-mon[212939]: failed to encode map e283 with expected crc 2023-09-23T23:20:20.070 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:19 smithi110 ceph-mon[221102]: failed to encode map e283 with expected crc 2023-09-23T23:20:20.071 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:19 smithi110 ceph-mon[221102]: failed to encode map e283 with expected crc 2023-09-23T23:20:20.071 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:19 smithi110 ceph-mon[221102]: failed to encode map e283 with expected crc 2023-09-23T23:20:20.071 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:19 smithi110 ceph-mon[221102]: failed to encode map e283 with expected crc 2023-09-23T23:20:20.108 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:20:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:19 smithi138 ceph-mon[193606]: failed to encode map e283 with expected crc 2023-09-23T23:20:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:19 smithi138 ceph-mon[193606]: failed to encode map e283 with expected crc 2023-09-23T23:20:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:19 smithi138 ceph-mon[193606]: failed to encode map e283 with expected crc 2023-09-23T23:20:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:19 smithi138 ceph-mon[193606]: failed to encode map e283 with expected crc 2023-09-23T23:20:20.305 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:20.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:20.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:20.414 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:20.463 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:20.557 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 2] out_osds: [1] dead_osds: [7] live_osds: [0, 1, 3, 4, 5, 6, 2] 2023-09-23T23:20:20.557 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:20:20.557 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T23:20:20.557 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T23:20:20.769 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:20 smithi110 ceph-mon[212939]: pgmap v857: 99 pgs: 1 remapped+peering, 5 active+clean+remapped, 2 active+remapped+backfill_wait, 5 active+undersized+degraded, 8 active+undersized, 78 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 16 MiB/s wr, 251 op/s; 10512/256356 objects degraded (4.101%); 10591/256356 objects misplaced (4.131%); 432 KiB/s, 6 objects/s recovering 2023-09-23T23:20:20.769 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:20 smithi110 ceph-mon[221102]: pgmap v857: 99 pgs: 1 remapped+peering, 5 active+clean+remapped, 2 active+remapped+backfill_wait, 5 active+undersized+degraded, 8 active+undersized, 78 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 16 MiB/s wr, 251 op/s; 10512/256356 objects degraded (4.101%); 10591/256356 objects misplaced (4.131%); 432 KiB/s, 6 objects/s recovering 2023-09-23T23:20:20.820 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 70 16 88051 88035 78.5904 54.6875 0.0147311 0.012719 2023-09-23T23:20:20.871 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:20.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:20.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:20.976 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:20:21.089 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:21.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:20 smithi138 ceph-mon[193606]: pgmap v857: 99 pgs: 1 remapped+peering, 5 active+clean+remapped, 2 active+remapped+backfill_wait, 5 active+undersized+degraded, 8 active+undersized, 78 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 16 MiB/s wr, 251 op/s; 10512/256356 objects degraded (4.101%); 10591/256356 objects misplaced (4.131%); 432 KiB/s, 6 objects/s recovering 2023-09-23T23:20:21.214 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:20:21.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:21.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:20:21.322 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:21.370 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:21.453 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:21.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:21.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:21.530 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:21.559 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:21.590 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:20:21.627 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:21.636 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:20:21.685 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:20:21.726 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:20:21.769 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:20:21.785 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:21.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:21.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:21.810 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:20:21.820 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 71 16 88912 88896 78.2413 53.8125 0.0135152 0.0127757 2023-09-23T23:20:21.853 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:20:21.857 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:20:21.857 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:20:21.857 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:20:21.857 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:20:21.857 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:20:21.858 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:20:21.858 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:20:21.858 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:20:21.887 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:21 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2597333264' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:21.890 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:21 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2597333264' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:21.900 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:21.969 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:22.159 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:21 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2597333264' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:22.169 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:22.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:22.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:22.279 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:22.338 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:22.475 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:22.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:22.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:22.490 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:20:22.491 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:20:22.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:20:22.491 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:20:22.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.491 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:186176 8.8 8:12f1ade7:::benchmark_data_smithi110_258128_object67358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T23:20:22.491 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:58.477323+0000", 2023-09-23T23:20:22.491 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 564.00856867799996, 2023-09-23T23:20:22.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4157779150000001, 2023-09-23T23:20:22.492 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.492 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:20:22.492 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.492 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:20:22.492 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:20:22.492 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 186176 2023-09-23T23:20:22.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.492 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477323+0000", 2023-09-23T23:20:22.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477323+0000", 2023-09-23T23:20:22.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477322+0000", 2023-09-23T23:20:22.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:20:22.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477327+0000", 2023-09-23T23:20:22.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7049999999999998e-06 2023-09-23T23:20:22.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477336+0000", 2023-09-23T23:20:22.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0299999999999999e-06 2023-09-23T23:20:22.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477342+0000", 2023-09-23T23:20:22.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7300000000000002e-06 2023-09-23T23:20:22.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477395+0000", 2023-09-23T23:20:22.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3256e-05 2023-09-23T23:20:22.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477519+0000", 2023-09-23T23:20:22.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012338599999999999 2023-09-23T23:20:22.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 7", 2023-09-23T23:20:22.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.477633+0000", 2023-09-23T23:20:22.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000114461 2023-09-23T23:20:22.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.487194+0000", 2023-09-23T23:20:22.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0095614959999999992 2023-09-23T23:20:22.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.490656+0000", 2023-09-23T23:20:22.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003461689 2023-09-23T23:20:22.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.490665+0000", 2023-09-23T23:20:22.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0159999999999997e-06 2023-09-23T23:20:22.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489508+0000", 2023-09-23T23:20:22.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99884329500000002 2023-09-23T23:20:22.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489523+0000", 2023-09-23T23:20:22.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4565e-05 2023-09-23T23:20:22.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.868588+0000", 2023-09-23T23:20:22.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3790654570000001 2023-09-23T23:20:22.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-09-23T23:20:22.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.868704+0000", 2023-09-23T23:20:22.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000115458 2023-09-23T23:20:22.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893017+0000", 2023-09-23T23:20:22.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.024312963999999999 2023-09-23T23:20:22.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893101+0000", 2023-09-23T23:20:22.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4171000000000004e-05 2023-09-23T23:20:22.502 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.502 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.502 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.503 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:186189 8.8 8:14e93b65:::benchmark_data_smithi110_258128_object67371:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T23:20:22.503 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:58.504373+0000", 2023-09-23T23:20:22.503 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 563.98151847600002, 2023-09-23T23:20:22.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.388920556, 2023-09-23T23:20:22.503 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.503 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.503 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.503 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:20:22.503 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:20:22.504 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 186189 2023-09-23T23:20:22.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.504 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504373+0000", 2023-09-23T23:20:22.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504373+0000", 2023-09-23T23:20:22.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504372+0000", 2023-09-23T23:20:22.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:22.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504379+0000", 2023-09-23T23:20:22.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7580000000000003e-06 2023-09-23T23:20:22.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504390+0000", 2023-09-23T23:20:22.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1239e-05 2023-09-23T23:20:22.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504397+0000", 2023-09-23T23:20:22.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3509999999999996e-06 2023-09-23T23:20:22.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504473+0000", 2023-09-23T23:20:22.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5463000000000006e-05 2023-09-23T23:20:22.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.504483+0000", 2023-09-23T23:20:22.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0202e-05 2023-09-23T23:20:22.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489551+0000", 2023-09-23T23:20:22.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98506775000000002 2023-09-23T23:20:22.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489558+0000", 2023-09-23T23:20:22.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4170000000000003e-06 2023-09-23T23:20:22.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.868754+0000", 2023-09-23T23:20:22.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3791954120000001 2023-09-23T23:20:22.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.868858+0000", 2023-09-23T23:20:22.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103983 2023-09-23T23:20:22.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.510 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:20:22.510 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.868960+0000", 2023-09-23T23:20:22.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010265800000000001 2023-09-23T23:20:22.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.510 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.510 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.877827+0000", 2023-09-23T23:20:22.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088667039999999996 2023-09-23T23:20:22.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.510 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.877878+0000", 2023-09-23T23:20:22.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1452000000000001e-05 2023-09-23T23:20:22.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893139+0000", 2023-09-23T23:20:22.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015260272 2023-09-23T23:20:22.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893179+0000", 2023-09-23T23:20:22.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9881e-05 2023-09-23T23:20:22.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.512 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.512 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893294+0000", 2023-09-23T23:20:22.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000115392 2023-09-23T23:20:22.512 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.512 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.512 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.512 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.512 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.44642.0:186216 8.8 8:19876885:::benchmark_data_smithi110_258128_object67398:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T23:20:22.512 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:10:58.548467+0000", 2023-09-23T23:20:22.512 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 563.93742475199997, 2023-09-23T23:20:22.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3450009590000001, 2023-09-23T23:20:22.513 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.513 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.513 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.513 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.44642", 2023-09-23T23:20:22.513 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2599750445", 2023-09-23T23:20:22.513 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 186216 2023-09-23T23:20:22.513 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.513 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.513 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.513 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548467+0000", 2023-09-23T23:20:22.514 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.514 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.514 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.514 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.514 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548467+0000", 2023-09-23T23:20:22.514 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.514 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.514 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.514 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.514 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548466+0000", 2023-09-23T23:20:22.514 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:22.514 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.515 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.515 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.515 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548473+0000", 2023-09-23T23:20:22.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0339999999999999e-06 2023-09-23T23:20:22.515 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.515 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.515 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.515 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548485+0000", 2023-09-23T23:20:22.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.223e-05 2023-09-23T23:20:22.515 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.515 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.515 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.515 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548492+0000", 2023-09-23T23:20:22.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5509999999999996e-06 2023-09-23T23:20:22.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548545+0000", 2023-09-23T23:20:22.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3434999999999999e-05 2023-09-23T23:20:22.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:58.548552+0000", 2023-09-23T23:20:22.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8750000000000002e-06 2023-09-23T23:20:22.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489851+0000", 2023-09-23T23:20:22.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94129886100000004 2023-09-23T23:20:22.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:10:59.489860+0000", 2023-09-23T23:20:22.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7079999999999992e-06 2023-09-23T23:20:22.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.518 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.873113+0000", 2023-09-23T23:20:22.518 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3832525150000001 2023-09-23T23:20:22.518 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.518 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.518 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.518 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.873221+0000", 2023-09-23T23:20:22.518 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010844300000000001 2023-09-23T23:20:22.518 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.518 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.518 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:20:22.518 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.873337+0000", 2023-09-23T23:20:22.518 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000115621 2023-09-23T23:20:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.877887+0000", 2023-09-23T23:20:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0045499099999999999 2023-09-23T23:20:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.877920+0000", 2023-09-23T23:20:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3314e-05 2023-09-23T23:20:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893326+0000", 2023-09-23T23:20:22.520 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015405911 2023-09-23T23:20:22.520 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.520 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893363+0000", 2023-09-23T23:20:22.520 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7240000000000003e-05 2023-09-23T23:20:22.520 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.520 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:11:00.893468+0000", 2023-09-23T23:20:22.520 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010466700000000001 2023-09-23T23:20:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165813 9.7 9:ef096715:::benchmark_data_smithi110_295904_object54632:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:20:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.377865+0000", 2023-09-23T23:20:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 219.10802655099999, 2023-09-23T23:20:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1834216419999999, 2023-09-23T23:20:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165813 2023-09-23T23:20:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377865+0000", 2023-09-23T23:20:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377865+0000", 2023-09-23T23:20:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377864+0000", 2023-09-23T23:20:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377872+0000", 2023-09-23T23:20:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9179999999999994e-06 2023-09-23T23:20:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377889+0000", 2023-09-23T23:20:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7413000000000001e-05 2023-09-23T23:20:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377897+0000", 2023-09-23T23:20:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0309999999999998e-06 2023-09-23T23:20:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377972+0000", 2023-09-23T23:20:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4659999999999993e-05 2023-09-23T23:20:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377984+0000", 2023-09-23T23:20:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2333e-05 2023-09-23T23:20:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340836+0000", 2023-09-23T23:20:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96285172900000005 2023-09-23T23:20:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340849+0000", 2023-09-23T23:20:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3107e-05 2023-09-23T23:20:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.529796+0000", 2023-09-23T23:20:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.188947368 2023-09-23T23:20:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.529935+0000", 2023-09-23T23:20:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013898799999999999 2023-09-23T23:20:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:20:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.530085+0000", 2023-09-23T23:20:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015011199999999999 2023-09-23T23:20:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.557076+0000", 2023-09-23T23:20:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.026990584000000001 2023-09-23T23:20:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.560991+0000", 2023-09-23T23:20:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039146149999999998 2023-09-23T23:20:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561072+0000", 2023-09-23T23:20:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1526000000000005e-05 2023-09-23T23:20:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561185+0000", 2023-09-23T23:20:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011253599999999999 2023-09-23T23:20:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561222+0000", 2023-09-23T23:20:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6940000000000002e-05 2023-09-23T23:20:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561287+0000", 2023-09-23T23:20:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5339000000000006e-05 2023-09-23T23:20:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165818 9.7 9:edd24351:::benchmark_data_smithi110_295904_object54637:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:20:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.384734+0000", 2023-09-23T23:20:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 219.10115762699999, 2023-09-23T23:20:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1767809169999999, 2023-09-23T23:20:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165818 2023-09-23T23:20:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384734+0000", 2023-09-23T23:20:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384734+0000", 2023-09-23T23:20:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384733+0000", 2023-09-23T23:20:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:20:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384738+0000", 2023-09-23T23:20:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1370000000000001e-06 2023-09-23T23:20:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384750+0000", 2023-09-23T23:20:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.198e-05 2023-09-23T23:20:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384773+0000", 2023-09-23T23:20:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2255999999999998e-05 2023-09-23T23:20:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384816+0000", 2023-09-23T23:20:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.35e-05 2023-09-23T23:20:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384822+0000", 2023-09-23T23:20:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1630000000000001e-06 2023-09-23T23:20:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340872+0000", 2023-09-23T23:20:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95605004699999996 2023-09-23T23:20:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340880+0000", 2023-09-23T23:20:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3610000000000003e-06 2023-09-23T23:20:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.533907+0000", 2023-09-23T23:20:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.193027515 2023-09-23T23:20:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.534044+0000", 2023-09-23T23:20:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013652 2023-09-23T23:20:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:20:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.534213+0000", 2023-09-23T23:20:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016906000000000001 2023-09-23T23:20:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.557086+0000", 2023-09-23T23:20:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.022873668999999999 2023-09-23T23:20:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561026+0000", 2023-09-23T23:20:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039397240000000004 2023-09-23T23:20:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561103+0000", 2023-09-23T23:20:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6483000000000006e-05 2023-09-23T23:20:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561335+0000", 2023-09-23T23:20:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000231967 2023-09-23T23:20:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561390+0000", 2023-09-23T23:20:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5334999999999998e-05 2023-09-23T23:20:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561515+0000", 2023-09-23T23:20:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012529199999999999 2023-09-23T23:20:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165854 9.7 9:e376a013:::benchmark_data_smithi110_295904_object54673:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:20:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.463878+0000", 2023-09-23T23:20:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 219.02201359399999, 2023-09-23T23:20:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0978617750000002, 2023-09-23T23:20:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165854 2023-09-23T23:20:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463878+0000", 2023-09-23T23:20:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463878+0000", 2023-09-23T23:20:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463877+0000", 2023-09-23T23:20:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:20:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463883+0000", 2023-09-23T23:20:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5269999999999998e-06 2023-09-23T23:20:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463893+0000", 2023-09-23T23:20:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0327e-05 2023-09-23T23:20:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463898+0000", 2023-09-23T23:20:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0509999999999996e-06 2023-09-23T23:20:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463961+0000", 2023-09-23T23:20:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2624000000000002e-05 2023-09-23T23:20:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463972+0000", 2023-09-23T23:20:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0920999999999999e-05 2023-09-23T23:20:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340900+0000", 2023-09-23T23:20:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.87692793800000002 2023-09-23T23:20:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340907+0000", 2023-09-23T23:20:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0990000000000001e-06 2023-09-23T23:20:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.537799+0000", 2023-09-23T23:20:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1968927309999999 2023-09-23T23:20:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.537907+0000", 2023-09-23T23:20:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107235 2023-09-23T23:20:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:20:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.538020+0000", 2023-09-23T23:20:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011348700000000001 2023-09-23T23:20:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.557096+0000", 2023-09-23T23:20:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019076085999999999 2023-09-23T23:20:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561153+0000", 2023-09-23T23:20:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040572120000000001 2023-09-23T23:20:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561540+0000", 2023-09-23T23:20:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00038614200000000002 2023-09-23T23:20:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561584+0000", 2023-09-23T23:20:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3961999999999997e-05 2023-09-23T23:20:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561633+0000", 2023-09-23T23:20:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9116999999999997e-05 2023-09-23T23:20:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561740+0000", 2023-09-23T23:20:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107328 2023-09-23T23:20:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165860 9.7 9:e0c9fcc3:::benchmark_data_smithi110_295904_object54679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:20:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.472345+0000", 2023-09-23T23:20:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 219.01354682300001, 2023-09-23T23:20:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0897597270000001, 2023-09-23T23:20:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165860 2023-09-23T23:20:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472345+0000", 2023-09-23T23:20:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472345+0000", 2023-09-23T23:20:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472345+0000", 2023-09-23T23:20:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:20:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472348+0000", 2023-09-23T23:20:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9739999999999998e-06 2023-09-23T23:20:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472356+0000", 2023-09-23T23:20:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1419999999999993e-06 2023-09-23T23:20:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472359+0000", 2023-09-23T23:20:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3560000000000001e-06 2023-09-23T23:20:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472409+0000", 2023-09-23T23:20:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9982999999999999e-05 2023-09-23T23:20:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472418+0000", 2023-09-23T23:20:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7660000000000004e-06 2023-09-23T23:20:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340926+0000", 2023-09-23T23:20:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.86850790200000005 2023-09-23T23:20:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340933+0000", 2023-09-23T23:20:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2139999999999999e-06 2023-09-23T23:20:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.541418+0000", 2023-09-23T23:20:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2004846490000001 2023-09-23T23:20:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.541552+0000", 2023-09-23T23:20:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000134566 2023-09-23T23:20:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:20:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.542163+0000", 2023-09-23T23:20:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00061078700000000001 2023-09-23T23:20:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.560793+0000", 2023-09-23T23:20:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018630225 2023-09-23T23:20:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561763+0000", 2023-09-23T23:20:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00097026299999999996 2023-09-23T23:20:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561803+0000", 2023-09-23T23:20:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9400000000000002e-05 2023-09-23T23:20:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561930+0000", 2023-09-23T23:20:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000127367 2023-09-23T23:20:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561986+0000", 2023-09-23T23:20:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5379e-05 2023-09-23T23:20:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.558 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562105+0000", 2023-09-23T23:20:22.558 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000119218 2023-09-23T23:20:22.558 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.558 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.558 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.558 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.558 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.558 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165866 9.7 9:e8d58a1b:::benchmark_data_smithi110_295904_object54685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:20:22.558 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.486204+0000", 2023-09-23T23:20:22.559 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 218.99968801, 2023-09-23T23:20:22.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0760496399999999, 2023-09-23T23:20:22.559 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.559 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.559 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.559 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:22.559 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:22.559 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165866 2023-09-23T23:20:22.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486204+0000", 2023-09-23T23:20:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486204+0000", 2023-09-23T23:20:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486203+0000", 2023-09-23T23:20:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:20:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486207+0000", 2023-09-23T23:20:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.236e-06 2023-09-23T23:20:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486215+0000", 2023-09-23T23:20:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7199999999999995e-06 2023-09-23T23:20:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486219+0000", 2023-09-23T23:20:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5839999999999999e-06 2023-09-23T23:20:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486260+0000", 2023-09-23T23:20:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1323999999999999e-05 2023-09-23T23:20:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486269+0000", 2023-09-23T23:20:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5390000000000007e-06 2023-09-23T23:20:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340952+0000", 2023-09-23T23:20:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.85468296899999996 2023-09-23T23:20:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340959+0000", 2023-09-23T23:20:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0419999999999997e-06 2023-09-23T23:20:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.545682+0000", 2023-09-23T23:20:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2047230959999999 2023-09-23T23:20:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.545818+0000", 2023-09-23T23:20:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000136527 2023-09-23T23:20:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:20:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.545972+0000", 2023-09-23T23:20:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015338600000000001 2023-09-23T23:20:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.560804+0000", 2023-09-23T23:20:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014832051000000001 2023-09-23T23:20:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561831+0000", 2023-09-23T23:20:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001027395 2023-09-23T23:20:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561871+0000", 2023-09-23T23:20:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9718999999999997e-05 2023-09-23T23:20:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562134+0000", 2023-09-23T23:20:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00026311599999999998 2023-09-23T23:20:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562161+0000", 2023-09-23T23:20:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7259000000000001e-05 2023-09-23T23:20:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562253+0000", 2023-09-23T23:20:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2167e-05 2023-09-23T23:20:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165894 9.7 9:e099b5a1:::benchmark_data_smithi110_295904_object54713:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:20:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.692918+0000", 2023-09-23T23:20:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 218.792973424, 2023-09-23T23:20:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.869631313, 2023-09-23T23:20:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165894 2023-09-23T23:20:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692918+0000", 2023-09-23T23:20:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692918+0000", 2023-09-23T23:20:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692918+0000", 2023-09-23T23:20:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:20:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692923+0000", 2023-09-23T23:20:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0499999999999999e-06 2023-09-23T23:20:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692933+0000", 2023-09-23T23:20:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0591000000000001e-05 2023-09-23T23:20:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692938+0000", 2023-09-23T23:20:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1440000000000002e-06 2023-09-23T23:20:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692991+0000", 2023-09-23T23:20:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2522000000000003e-05 2023-09-23T23:20:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.693000+0000", 2023-09-23T23:20:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2229999999999996e-06 2023-09-23T23:20:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340977+0000", 2023-09-23T23:20:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.64797712799999996 2023-09-23T23:20:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340984+0000", 2023-09-23T23:20:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0360000000000001e-06 2023-09-23T23:20:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.549530+0000", 2023-09-23T23:20:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.208546127 2023-09-23T23:20:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.549658+0000", 2023-09-23T23:20:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012757799999999999 2023-09-23T23:20:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:20:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.549806+0000", 2023-09-23T23:20:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014840400000000001 2023-09-23T23:20:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.560819+0000", 2023-09-23T23:20:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011012752000000001 2023-09-23T23:20:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561890+0000", 2023-09-23T23:20:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001071383 2023-09-23T23:20:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562314+0000", 2023-09-23T23:20:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00042307100000000002 2023-09-23T23:20:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562371+0000", 2023-09-23T23:20:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7435999999999997e-05 2023-09-23T23:20:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562424+0000", 2023-09-23T23:20:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3461999999999997e-05 2023-09-23T23:20:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562550+0000", 2023-09-23T23:20:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000125277 2023-09-23T23:20:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165909 9.7 9:e5c10464:::benchmark_data_smithi110_295904_object54728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:20:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.748819+0000", 2023-09-23T23:20:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 218.73707256099999, 2023-09-23T23:20:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.814135311, 2023-09-23T23:20:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165909 2023-09-23T23:20:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748819+0000", 2023-09-23T23:20:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748819+0000", 2023-09-23T23:20:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748819+0000", 2023-09-23T23:20:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:20:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748823+0000", 2023-09-23T23:20:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.668e-06 2023-09-23T23:20:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748837+0000", 2023-09-23T23:20:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.363e-05 2023-09-23T23:20:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748842+0000", 2023-09-23T23:20:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8790000000000002e-06 2023-09-23T23:20:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748893+0000", 2023-09-23T23:20:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1456000000000003e-05 2023-09-23T23:20:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748901+0000", 2023-09-23T23:20:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7530000000000001e-06 2023-09-23T23:20:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.341003+0000", 2023-09-23T23:20:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.59210227299999996 2023-09-23T23:20:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.341010+0000", 2023-09-23T23:20:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5599999999999999e-06 2023-09-23T23:20:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.553256+0000", 2023-09-23T23:20:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2122462409999999 2023-09-23T23:20:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.553394+0000", 2023-09-23T23:20:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013755000000000001 2023-09-23T23:20:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:20:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.553548+0000", 2023-09-23T23:20:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000154449 2023-09-23T23:20:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561003+0000", 2023-09-23T23:20:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0074545380000000001 2023-09-23T23:20:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562606+0000", 2023-09-23T23:20:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0016038770000000001 2023-09-23T23:20:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562661+0000", 2023-09-23T23:20:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4122000000000001e-05 2023-09-23T23:20:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562787+0000", 2023-09-23T23:20:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012642 2023-09-23T23:20:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562837+0000", 2023-09-23T23:20:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0565000000000001e-05 2023-09-23T23:20:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562955+0000", 2023-09-23T23:20:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117106 2023-09-23T23:20:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:188501 9.6 9:679feb10:::benchmark_data_smithi110_295904_object77320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-09-23T23:20:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:33.113667+0000", 2023-09-23T23:20:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 169.37222435199999, 2023-09-23T23:20:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.30803306400000002, 2023-09-23T23:20:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 188501 2023-09-23T23:20:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113667+0000", 2023-09-23T23:20:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113667+0000", 2023-09-23T23:20:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113666+0000", 2023-09-23T23:20:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113673+0000", 2023-09-23T23:20:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7889999999999997e-06 2023-09-23T23:20:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113688+0000", 2023-09-23T23:20:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5165e-05 2023-09-23T23:20:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113695+0000", 2023-09-23T23:20:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6630000000000004e-06 2023-09-23T23:20:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113737+0000", 2023-09-23T23:20:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1814000000000003e-05 2023-09-23T23:20:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113855+0000", 2023-09-23T23:20:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118145 2023-09-23T23:20:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:20:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113978+0000", 2023-09-23T23:20:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000123103 2023-09-23T23:20:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.117927+0000", 2023-09-23T23:20:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039487050000000003 2023-09-23T23:20:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.126736+0000", 2023-09-23T23:20:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088093419999999995 2023-09-23T23:20:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.421639+0000", 2023-09-23T23:20:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.29490354099999999 2023-09-23T23:20:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.421673+0000", 2023-09-23T23:20:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3896000000000002e-05 2023-09-23T23:20:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.421701+0000", 2023-09-23T23:20:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7203e-05 2023-09-23T23:20:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:207662 9.d 9:b22a6364:::benchmark_data_smithi110_295904_object96481:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-23T23:20:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:18:15.286195+0000", 2023-09-23T23:20:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 127.199696562, 2023-09-23T23:20:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.31124376799999998, 2023-09-23T23:20:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 207662 2023-09-23T23:20:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.286195+0000", 2023-09-23T23:20:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.286195+0000", 2023-09-23T23:20:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.286195+0000", 2023-09-23T23:20:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:20:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.286197+0000", 2023-09-23T23:20:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.43e-06 2023-09-23T23:20:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.286203+0000", 2023-09-23T23:20:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6330000000000001e-06 2023-09-23T23:20:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.286207+0000", 2023-09-23T23:20:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7469999999999999e-06 2023-09-23T23:20:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.289084+0000", 2023-09-23T23:20:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028777339999999998 2023-09-23T23:20:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.289209+0000", 2023-09-23T23:20:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000124578 2023-09-23T23:20:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-09-23T23:20:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.289323+0000", 2023-09-23T23:20:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000114287 2023-09-23T23:20:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.293339+0000", 2023-09-23T23:20:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040155709999999999 2023-09-23T23:20:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.294526+0000", 2023-09-23T23:20:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001187032 2023-09-23T23:20:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.597339+0000", 2023-09-23T23:20:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.30281330299999998 2023-09-23T23:20:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.597387+0000", 2023-09-23T23:20:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8254000000000003e-05 2023-09-23T23:20:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.597439+0000", 2023-09-23T23:20:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1609e-05 2023-09-23T23:20:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:207668 9.7 9:e7915572:::benchmark_data_smithi110_295904_object96487:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-23T23:20:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:18:15.295609+0000", 2023-09-23T23:20:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 127.190283071, 2023-09-23T23:20:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.30867075199999999, 2023-09-23T23:20:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 207668 2023-09-23T23:20:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295609+0000", 2023-09-23T23:20:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295609+0000", 2023-09-23T23:20:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295608+0000", 2023-09-23T23:20:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:20:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295612+0000", 2023-09-23T23:20:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3730000000000003e-06 2023-09-23T23:20:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295619+0000", 2023-09-23T23:20:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0310000000000001e-06 2023-09-23T23:20:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295624+0000", 2023-09-23T23:20:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8729999999999998e-06 2023-09-23T23:20:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295665+0000", 2023-09-23T23:20:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0583000000000001e-05 2023-09-23T23:20:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295780+0000", 2023-09-23T23:20:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011491 2023-09-23T23:20:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3,6,7", 2023-09-23T23:20:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295908+0000", 2023-09-23T23:20:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000128539 2023-09-23T23:20:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.603885+0000", 2023-09-23T23:20:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.30797660999999998 2023-09-23T23:20:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.603914+0000", 2023-09-23T23:20:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9269999999999999e-05 2023-09-23T23:20:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.603943+0000", 2023-09-23T23:20:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8564999999999998e-05 2023-09-23T23:20:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.604073+0000", 2023-09-23T23:20:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000130648 2023-09-23T23:20:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.604083+0000", 2023-09-23T23:20:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8770000000000005e-06 2023-09-23T23:20:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.604151+0000", 2023-09-23T23:20:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8336000000000004e-05 2023-09-23T23:20:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.604280+0000", 2023-09-23T23:20:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012813200000000001 2023-09-23T23:20:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:207670 9.d 9:b8c349e6:::benchmark_data_smithi110_295904_object96489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-23T23:20:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:18:15.295643+0000", 2023-09-23T23:20:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 127.190248571, 2023-09-23T23:20:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.31272043100000002, 2023-09-23T23:20:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:20:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:20:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 207670 2023-09-23T23:20:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295643+0000", 2023-09-23T23:20:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295643+0000", 2023-09-23T23:20:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295643+0000", 2023-09-23T23:20:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:20:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295646+0000", 2023-09-23T23:20:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3009999999999998e-06 2023-09-23T23:20:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295656+0000", 2023-09-23T23:20:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0145e-05 2023-09-23T23:20:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295661+0000", 2023-09-23T23:20:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6210000000000003e-06 2023-09-23T23:20:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295702+0000", 2023-09-23T23:20:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0877999999999997e-05 2023-09-23T23:20:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295809+0000", 2023-09-23T23:20:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107404 2023-09-23T23:20:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-09-23T23:20:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.589180+0000", 2023-09-23T23:20:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.29337100700000002 2023-09-23T23:20:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.608245+0000", 2023-09-23T23:20:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019064349000000001 2023-09-23T23:20:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.608287+0000", 2023-09-23T23:20:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2691000000000002e-05 2023-09-23T23:20:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.608319+0000", 2023-09-23T23:20:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1467999999999998e-05 2023-09-23T23:20:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.608342+0000", 2023-09-23T23:20:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2982000000000001e-05 2023-09-23T23:20:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.608364+0000", 2023-09-23T23:20:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2045999999999999e-05 2023-09-23T23:20:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:14293 10.6 10:62a31385:::benchmark_data_smithi110_366450_object14292:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2023-09-23T23:20:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:15.521478+0000", 2023-09-23T23:20:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 66.964413535999995, 2023-09-23T23:20:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20660231800000001, 2023-09-23T23:20:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:20:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:20:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 14293 2023-09-23T23:20:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521478+0000", 2023-09-23T23:20:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521478+0000", 2023-09-23T23:20:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521477+0000", 2023-09-23T23:20:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521511+0000", 2023-09-23T23:20:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3936999999999997e-05 2023-09-23T23:20:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521525+0000", 2023-09-23T23:20:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4412e-05 2023-09-23T23:20:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521532+0000", 2023-09-23T23:20:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7109999999999998e-06 2023-09-23T23:20:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521598+0000", 2023-09-23T23:20:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5606999999999997e-05 2023-09-23T23:20:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521743+0000", 2023-09-23T23:20:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014537099999999999 2023-09-23T23:20:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:20:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521888+0000", 2023-09-23T23:20:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014454800000000001 2023-09-23T23:20:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.523492+0000", 2023-09-23T23:20:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001604255 2023-09-23T23:20:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.524401+0000", 2023-09-23T23:20:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000908733 2023-09-23T23:20:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.727985+0000", 2023-09-23T23:20:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.203584864 2023-09-23T23:20:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.728040+0000", 2023-09-23T23:20:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4092999999999999e-05 2023-09-23T23:20:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.728081+0000", 2023-09-23T23:20:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1118999999999997e-05 2023-09-23T23:20:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:14298 10.6 10:65bb9d0b:::benchmark_data_smithi110_366450_object14297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2023-09-23T23:20:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:15.523563+0000", 2023-09-23T23:20:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 66.962328399, 2023-09-23T23:20:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20462076600000001, 2023-09-23T23:20:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:20:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:20:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 14298 2023-09-23T23:20:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.523563+0000", 2023-09-23T23:20:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.523563+0000", 2023-09-23T23:20:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.523562+0000", 2023-09-23T23:20:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.523633+0000", 2023-09-23T23:20:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1057999999999996e-05 2023-09-23T23:20:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.523648+0000", 2023-09-23T23:20:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4554e-05 2023-09-23T23:20:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.523657+0000", 2023-09-23T23:20:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8009999999999993e-06 2023-09-23T23:20:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.523696+0000", 2023-09-23T23:20:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9530000000000003e-05 2023-09-23T23:20:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.523803+0000", 2023-09-23T23:20:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107266 2023-09-23T23:20:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:20:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.523900+0000", 2023-09-23T23:20:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6881999999999998e-05 2023-09-23T23:20:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.524506+0000", 2023-09-23T23:20:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00060584699999999996 2023-09-23T23:20:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.524659+0000", 2023-09-23T23:20:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015319199999999999 2023-09-23T23:20:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.728121+0000", 2023-09-23T23:20:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.203462064 2023-09-23T23:20:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.728163+0000", 2023-09-23T23:20:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1529000000000002e-05 2023-09-23T23:20:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.728184+0000", 2023-09-23T23:20:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1237999999999999e-05 2023-09-23T23:20:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:81407 10.8 10:19cd79ee:::benchmark_data_smithi110_366450_object81406:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-09-23T23:20:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:10.477387+0000", 2023-09-23T23:20:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 12.008504635, 2023-09-23T23:20:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.27317791600000002, 2023-09-23T23:20:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:20:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:20:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 81407 2023-09-23T23:20:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477387+0000", 2023-09-23T23:20:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477387+0000", 2023-09-23T23:20:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477386+0000", 2023-09-23T23:20:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477420+0000", 2023-09-23T23:20:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4446999999999997e-05 2023-09-23T23:20:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477434+0000", 2023-09-23T23:20:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3482000000000001e-05 2023-09-23T23:20:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477440+0000", 2023-09-23T23:20:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9159999999999998e-06 2023-09-23T23:20:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477546+0000", 2023-09-23T23:20:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106925 2023-09-23T23:20:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477672+0000", 2023-09-23T23:20:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012528699999999999 2023-09-23T23:20:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:20:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477800+0000", 2023-09-23T23:20:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000128057 2023-09-23T23:20:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.483553+0000", 2023-09-23T23:20:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0057529360000000002 2023-09-23T23:20:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.492111+0000", 2023-09-23T23:20:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085582790000000002 2023-09-23T23:20:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.750471+0000", 2023-09-23T23:20:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25835971200000002 2023-09-23T23:20:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.750526+0000", 2023-09-23T23:20:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5138e-05 2023-09-23T23:20:22.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.750565+0000", 2023-09-23T23:20:22.635 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:22.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9224000000000003e-05 2023-09-23T23:20:22.636 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.636 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.636 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.636 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:81410 10.8 10:1b5bc57b:::benchmark_data_smithi110_366450_object81409:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-09-23T23:20:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:10.477455+0000", 2023-09-23T23:20:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 12.008436638999999, 2023-09-23T23:20:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.273493826, 2023-09-23T23:20:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:20:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:20:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 81410 2023-09-23T23:20:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477455+0000", 2023-09-23T23:20:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477455+0000", 2023-09-23T23:20:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477455+0000", 2023-09-23T23:20:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:20:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477485+0000", 2023-09-23T23:20:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0553999999999999e-05 2023-09-23T23:20:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477492+0000", 2023-09-23T23:20:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7449999999999998e-06 2023-09-23T23:20:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477496+0000", 2023-09-23T23:20:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2390000000000004e-06 2023-09-23T23:20:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.480642+0000", 2023-09-23T23:20:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031452889999999999 2023-09-23T23:20:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.480752+0000", 2023-09-23T23:20:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110807 2023-09-23T23:20:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:20:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.480876+0000", 2023-09-23T23:20:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012367699999999999 2023-09-23T23:20:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.483813+0000", 2023-09-23T23:20:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029365559999999999 2023-09-23T23:20:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.492143+0000", 2023-09-23T23:20:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0083307559999999999 2023-09-23T23:20:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.750865+0000", 2023-09-23T23:20:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25872173199999998 2023-09-23T23:20:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.750912+0000", 2023-09-23T23:20:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6473e-05 2023-09-23T23:20:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.750949+0000", 2023-09-23T23:20:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7456000000000002e-05 2023-09-23T23:20:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:81425 10.3 10:c25ee332:::benchmark_data_smithi110_366450_object81424:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-09-23T23:20:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:10.492942+0000", 2023-09-23T23:20:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 11.992949647, 2023-09-23T23:20:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26416036700000001, 2023-09-23T23:20:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:20:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:20:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 81425 2023-09-23T23:20:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.492942+0000", 2023-09-23T23:20:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.492942+0000", 2023-09-23T23:20:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.492941+0000", 2023-09-23T23:20:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.492984+0000", 2023-09-23T23:20:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3312000000000001e-05 2023-09-23T23:20:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.492998+0000", 2023-09-23T23:20:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4088e-05 2023-09-23T23:20:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.493004+0000", 2023-09-23T23:20:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0430000000000004e-06 2023-09-23T23:20:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.493053+0000", 2023-09-23T23:20:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9057000000000001e-05 2023-09-23T23:20:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.493193+0000", 2023-09-23T23:20:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013915999999999999 2023-09-23T23:20:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:20:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.493324+0000", 2023-09-23T23:20:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000131338 2023-09-23T23:20:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.498816+0000", 2023-09-23T23:20:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0054921220000000003 2023-09-23T23:20:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.498836+0000", 2023-09-23T23:20:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9504000000000002e-05 2023-09-23T23:20:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.757006+0000", 2023-09-23T23:20:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25817092600000002 2023-09-23T23:20:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.757059+0000", 2023-09-23T23:20:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2842e-05 2023-09-23T23:20:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.757103+0000", 2023-09-23T23:20:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3282000000000003e-05 2023-09-23T23:20:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:81429 10.3 10:c438e427:::benchmark_data_smithi110_366450_object81428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-09-23T23:20:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:10.495410+0000", 2023-09-23T23:20:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 11.990481905999999, 2023-09-23T23:20:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26733140399999999, 2023-09-23T23:20:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:20:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:20:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:20:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 81429 2023-09-23T23:20:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.495410+0000", 2023-09-23T23:20:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.495410+0000", 2023-09-23T23:20:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.495409+0000", 2023-09-23T23:20:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.495475+0000", 2023-09-23T23:20:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6433000000000006e-05 2023-09-23T23:20:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.495485+0000", 2023-09-23T23:20:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5759999999999995e-06 2023-09-23T23:20:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.495490+0000", 2023-09-23T23:20:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6119999999999998e-06 2023-09-23T23:20:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.495924+0000", 2023-09-23T23:20:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00043414300000000001 2023-09-23T23:20:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:20:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.496026+0000", 2023-09-23T23:20:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101579 2023-09-23T23:20:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:20:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.496151+0000", 2023-09-23T23:20:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000124547 2023-09-23T23:20:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:20:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.499122+0000", 2023-09-23T23:20:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029714630000000001 2023-09-23T23:20:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.501414+0000", 2023-09-23T23:20:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022915019999999999 2023-09-23T23:20:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:20:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.762668+0000", 2023-09-23T23:20:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26125429999999999 2023-09-23T23:20:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:20:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.762703+0000", 2023-09-23T23:20:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5621000000000003e-05 2023-09-23T23:20:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:20:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.762741+0000", 2023-09-23T23:20:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7861000000000002e-05 2023-09-23T23:20:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:22.657 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:20:22.716 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:22.788 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:22 smithi138 ceph-mon[193606]: pgmap v858: 99 pgs: 1 active+recovery_wait+remapped, 1 remapped+peering, 4 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 77 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 26 MiB/s wr, 417 op/s; 10794/260565 objects degraded (4.143%); 37453/260565 objects misplaced (14.374%); 1.1 MiB/s, 1 keys/s, 17 objects/s recovering 2023-09-23T23:20:22.820 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 72 16 89865 89849 77.9819 59.5625 0.0084974 0.0128187 2023-09-23T23:20:22.862 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:22.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:22.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:22.967 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:20:23.017 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:20:23.021 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:23.062 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:22 smithi110 ceph-mon[212939]: pgmap v858: 99 pgs: 1 active+recovery_wait+remapped, 1 remapped+peering, 4 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 77 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 26 MiB/s wr, 417 op/s; 10794/260565 objects degraded (4.143%); 37453/260565 objects misplaced (14.374%); 1.1 MiB/s, 1 keys/s, 17 objects/s recovering 2023-09-23T23:20:23.062 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:22 smithi110 ceph-mon[221102]: pgmap v858: 99 pgs: 1 active+recovery_wait+remapped, 1 remapped+peering, 4 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 77 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 26 MiB/s wr, 417 op/s; 10794/260565 objects degraded (4.143%); 37453/260565 objects misplaced (14.374%); 1.1 MiB/s, 1 keys/s, 17 objects/s recovering 2023-09-23T23:20:23.075 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:20:23.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:23.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:20:23.186 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:20:23.227 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:23.317 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:20:23.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:23.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:20:23.423 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:23.478 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:23.693 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:23.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:23.700 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:20:23.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:23.802 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:23.821 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 73 16 90695 90679 77.6226 51.875 0.0123864 0.0128775 2023-09-23T23:20:23.838 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:20:23.839 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:20:23.867 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:24.080 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:23 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 10794/260565 objects degraded (4.143%), 4 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:24.080 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:23 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 10794/260565 objects degraded (4.143%), 4 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:24.148 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:24.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:24.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:24.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:23 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 10794/260565 objects degraded (4.143%), 4 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:24.223 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:24.259 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:24.313 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:20:24.314 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:24.403 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:24.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:24.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:24.513 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:24.586 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:24.754 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:24.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:24.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:24.821 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 74 16 91518 91502 77.2687 51.4375 0.0137889 0.0129372 2023-09-23T23:20:24.861 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:24.918 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:24 smithi138 ceph-mon[193606]: pgmap v859: 99 pgs: 1 active+recovery_wait+remapped, 5 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 77 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 22 MiB/s wr, 356 op/s; 10794/260565 objects degraded (4.143%); 37453/260565 objects misplaced (14.374%); 936 KiB/s, 1 keys/s, 14 objects/s recovering 2023-09-23T23:20:24.928 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:25.061 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:25.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:25.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:25.168 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:25.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:24 smithi110 ceph-mon[212939]: pgmap v859: 99 pgs: 1 active+recovery_wait+remapped, 5 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 77 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 22 MiB/s wr, 356 op/s; 10794/260565 objects degraded (4.143%); 37453/260565 objects misplaced (14.374%); 936 KiB/s, 1 keys/s, 14 objects/s recovering 2023-09-23T23:20:25.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:24 smithi110 ceph-mon[221102]: pgmap v859: 99 pgs: 1 active+recovery_wait+remapped, 5 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 77 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 22 MiB/s wr, 356 op/s; 10794/260565 objects degraded (4.143%); 37453/260565 objects misplaced (14.374%); 936 KiB/s, 1 keys/s, 14 objects/s recovering 2023-09-23T23:20:25.243 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:25.341 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:25.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:25.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:25.448 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:25.475 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:25.522 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:25.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:25.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:25.626 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:25.671 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:25.822 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 75 15 92204 92189 76.811 42.9375 0.00633746 0.0130151 2023-09-23T23:20:25.839 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:20:25.843 INFO:teuthology.orchestra.run.smithi110.stdout:moved 6 / 297 (2.0202%) 2023-09-23T23:20:25.843 INFO:teuthology.orchestra.run.smithi110.stdout:avg 49.5 2023-09-23T23:20:25.843 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 11.7296 -> 11.3248 (expected baseline 6.42262) 2023-09-23T23:20:25.843 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.5 with 25 -> 25 pgs (0.50505 -> 0.50505 * mean) 2023-09-23T23:20:25.843 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.4 with 57 -> 54 pgs (1.15152 -> 1.09091 * mean) 2023-09-23T23:20:25.843 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:20:25.843 INFO:teuthology.orchestra.run.smithi110.stdout:oload 110 2023-09-23T23:20:25.843 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 0.05 2023-09-23T23:20:25.844 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:20:25.844 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0297 2023-09-23T23:20:25.844 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.0327 2023-09-23T23:20:25.844 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 weight 1.0000 -> 0.9500 2023-09-23T23:20:25.844 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 weight 1.0000 -> 0.9500 2023-09-23T23:20:25.844 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 weight 0.5125 -> 0.5625 2023-09-23T23:20:25.844 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 weight 1.0000 -> 0.9500 2023-09-23T23:20:26.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:25 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','4':'62260','6':'62260','7':'36861'}"}]: dispatch 2023-09-23T23:20:26.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:25 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','4':'62260','6':'62260','7':'36861'}"}]: dispatch 2023-09-23T23:20:26.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:25 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','4':'62260','6':'62260','7':'36861'}"}]: dispatch 2023-09-23T23:20:26.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:25 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','4':'62260','6':'62260','7':'36861'}"}]: dispatch 2023-09-23T23:20:26.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:25 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','4':'62260','6':'62260','7':'36861'}"}]: dispatch 2023-09-23T23:20:26.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:25 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','4':'62260','6':'62260','7':'36861'}"}]: dispatch 2023-09-23T23:20:26.257 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:26.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:26.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:26.361 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:26.388 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:26.471 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:20:26.471 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:20:26.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.471 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:92240 10.0 10.355bc5d0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:20:26.471 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:25.873807+0000", 2023-09-23T23:20:26.471 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.59606102100000002, 2023-09-23T23:20:26.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.59608086000000005, 2023-09-23T23:20:26.471 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:26.472 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "delayed", 2023-09-23T23:20:26.472 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:26.472 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:20:26.472 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:20:26.472 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92240 2023-09-23T23:20:26.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.472 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:26.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:26.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873807+0000", 2023-09-23T23:20:26.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:26.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:26.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873807+0000", 2023-09-23T23:20:26.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:26.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:26.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873806+0000", 2023-09-23T23:20:26.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:26.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:26.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873834+0000", 2023-09-23T23:20:26.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8551000000000002e-05 2023-09-23T23:20:26.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:26.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873844+0000", 2023-09-23T23:20:26.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0769999999999997e-06 2023-09-23T23:20:26.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:26.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873849+0000", 2023-09-23T23:20:26.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3639999999999998e-06 2023-09-23T23:20:26.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.475 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:26.475 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873908+0000", 2023-09-23T23:20:26.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8962000000000002e-05 2023-09-23T23:20:26.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.475 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:26.475 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873916+0000", 2023-09-23T23:20:26.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8210000000000002e-06 2023-09-23T23:20:26.475 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:26.475 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:26.476 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:26.476 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.476 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.476 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:92247 10.0 10.d6996e10 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:20:26.476 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:25.877726+0000", 2023-09-23T23:20:26.476 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.59214220699999998, 2023-09-23T23:20:26.476 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.59220793299999996, 2023-09-23T23:20:26.476 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:26.476 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "delayed", 2023-09-23T23:20:26.476 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:26.476 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:20:26.476 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:20:26.477 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92247 2023-09-23T23:20:26.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.477 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:26.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:26.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877726+0000", 2023-09-23T23:20:26.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:26.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:26.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877726+0000", 2023-09-23T23:20:26.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:26.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:26.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877726+0000", 2023-09-23T23:20:26.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:20:26.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:26.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877760+0000", 2023-09-23T23:20:26.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.413e-05 2023-09-23T23:20:26.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:26.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877767+0000", 2023-09-23T23:20:26.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7910000000000005e-06 2023-09-23T23:20:26.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:26.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877773+0000", 2023-09-23T23:20:26.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3190000000000002e-06 2023-09-23T23:20:26.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:26.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877855+0000", 2023-09-23T23:20:26.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.242e-05 2023-09-23T23:20:26.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:26.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877864+0000", 2023-09-23T23:20:26.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8929999999999994e-06 2023-09-23T23:20:26.480 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:26.480 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:26.480 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:26.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.481 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:92334 10.0 10.6cc21130 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:20:26.481 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:25.993847+0000", 2023-09-23T23:20:26.481 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.47602156000000001, 2023-09-23T23:20:26.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.47612623399999998, 2023-09-23T23:20:26.481 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:26.481 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "delayed", 2023-09-23T23:20:26.481 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:26.481 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:20:26.481 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:20:26.481 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92334 2023-09-23T23:20:26.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.482 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:26.482 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.482 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:26.482 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.993847+0000", 2023-09-23T23:20:26.482 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:26.482 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.482 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.482 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:26.482 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.993847+0000", 2023-09-23T23:20:26.482 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:26.482 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:26.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.993845+0000", 2023-09-23T23:20:26.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:26.483 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:26.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.993935+0000", 2023-09-23T23:20:26.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9233000000000004e-05 2023-09-23T23:20:26.483 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:26.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.993948+0000", 2023-09-23T23:20:26.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3242e-05 2023-09-23T23:20:26.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.484 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:26.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.993953+0000", 2023-09-23T23:20:26.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3029999999999999e-06 2023-09-23T23:20:26.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.484 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:26.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.994011+0000", 2023-09-23T23:20:26.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7568999999999997e-05 2023-09-23T23:20:26.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:26.485 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.994019+0000", 2023-09-23T23:20:26.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4570000000000005e-06 2023-09-23T23:20:26.485 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:26.485 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:26.485 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:26.485 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.485 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:92255 10.0 10.93217e20 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:20:26.485 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:25.889232+0000", 2023-09-23T23:20:26.485 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.58063664800000003, 2023-09-23T23:20:26.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.58077866700000003, 2023-09-23T23:20:26.486 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:26.486 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "delayed", 2023-09-23T23:20:26.486 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:26.486 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:20:26.486 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:20:26.486 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92255 2023-09-23T23:20:26.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.486 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:26.486 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:26.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889232+0000", 2023-09-23T23:20:26.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:26.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:26.487 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889232+0000", 2023-09-23T23:20:26.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:26.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:26.487 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889230+0000", 2023-09-23T23:20:26.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:20:26.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:26.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889267+0000", 2023-09-23T23:20:26.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7274000000000003e-05 2023-09-23T23:20:26.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:26.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889274+0000", 2023-09-23T23:20:26.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1899999999999998e-06 2023-09-23T23:20:26.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:26.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889279+0000", 2023-09-23T23:20:26.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9640000000000002e-06 2023-09-23T23:20:26.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:26.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889353+0000", 2023-09-23T23:20:26.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3460000000000005e-05 2023-09-23T23:20:26.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:26.490 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889362+0000", 2023-09-23T23:20:26.490 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2520000000000005e-06 2023-09-23T23:20:26.490 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:26.490 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:26.490 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:26.490 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.490 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:92265 10.0 10.e7cc730 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:20:26.490 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:25.898486+0000", 2023-09-23T23:20:26.490 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.57138187600000001, 2023-09-23T23:20:26.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.57156169000000001, 2023-09-23T23:20:26.491 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:26.491 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "delayed", 2023-09-23T23:20:26.491 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:26.491 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:20:26.491 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:20:26.491 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92265 2023-09-23T23:20:26.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.491 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:26.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.491 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:26.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.898486+0000", 2023-09-23T23:20:26.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:26.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:26.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.898486+0000", 2023-09-23T23:20:26.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:26.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:26.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.898486+0000", 2023-09-23T23:20:26.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:20:26.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:26.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.898519+0000", 2023-09-23T23:20:26.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3756e-05 2023-09-23T23:20:26.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:26.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.898530+0000", 2023-09-23T23:20:26.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0318e-05 2023-09-23T23:20:26.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:26.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.898535+0000", 2023-09-23T23:20:26.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8230000000000002e-06 2023-09-23T23:20:26.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:26.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.898606+0000", 2023-09-23T23:20:26.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1013999999999995e-05 2023-09-23T23:20:26.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:26.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.898623+0000", 2023-09-23T23:20:26.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7302000000000001e-05 2023-09-23T23:20:26.495 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:26.495 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:26.495 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:26.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.495 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:92306 10.0 10.6e593d80 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:20:26.495 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:25.949542+0000", 2023-09-23T23:20:26.495 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.52032568400000001, 2023-09-23T23:20:26.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.52054417100000006, 2023-09-23T23:20:26.495 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:20:26.495 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "delayed", 2023-09-23T23:20:26.495 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:20:26.496 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:20:26.496 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:20:26.496 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92306 2023-09-23T23:20:26.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.496 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:20:26.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:20:26.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.949542+0000", 2023-09-23T23:20:26.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:26.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:20:26.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.949542+0000", 2023-09-23T23:20:26.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:20:26.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:20:26.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.949542+0000", 2023-09-23T23:20:26.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:20:26.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:20:26.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.949572+0000", 2023-09-23T23:20:26.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0073000000000002e-05 2023-09-23T23:20:26.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:20:26.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.949579+0000", 2023-09-23T23:20:26.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7209999999999997e-06 2023-09-23T23:20:26.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:20:26.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.949583+0000", 2023-09-23T23:20:26.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2910000000000001e-06 2023-09-23T23:20:26.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:20:26.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.949766+0000", 2023-09-23T23:20:26.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018346399999999999 2023-09-23T23:20:26.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:20:26.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:20:26.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:20:26.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.949777+0000", 2023-09-23T23:20:26.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0081999999999999e-05 2023-09-23T23:20:26.499 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:26.499 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:20:26.499 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:26.499 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:20:26.500 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-09-23T23:20:26.500 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 6 2023-09-23T23:20:26.500 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:20:26.822 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 76 16 92340 92324 75.9114 8.4375 0.00334266 0.0130172 2023-09-23T23:20:26.862 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:26.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:26.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:26.926 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:26 smithi110 ceph-mon[221102]: from='client.45632 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:20:26.926 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:26 smithi110 ceph-mon[221102]: pgmap v860: 99 pgs: 1 active+recovery_wait+remapped, 5 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 77 active+clean; 5.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 852 op/s; 11309/273798 objects degraded (4.130%); 38733/273798 objects misplaced (14.147%); 1.9 MiB/s, 1 keys/s, 30 objects/s recovering 2023-09-23T23:20:26.926 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:26 smithi110 ceph-mon[221102]: failed to encode map e284 with expected crc 2023-09-23T23:20:26.926 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:26 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','4':'62260','6':'62260','7':'36861'}"}]': finished 2023-09-23T23:20:26.926 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:26 smithi110 ceph-mon[221102]: osdmap e284: 8 total, 7 up, 7 in 2023-09-23T23:20:26.926 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:26 smithi110 ceph-mon[221102]: failed to encode map e284 with expected crc 2023-09-23T23:20:26.926 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:26 smithi110 ceph-mon[221102]: failed to encode map e284 with expected crc 2023-09-23T23:20:26.927 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:26 smithi110 ceph-mon[212939]: from='client.45632 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:20:26.927 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:26 smithi110 ceph-mon[212939]: pgmap v860: 99 pgs: 1 active+recovery_wait+remapped, 5 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 77 active+clean; 5.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 852 op/s; 11309/273798 objects degraded (4.130%); 38733/273798 objects misplaced (14.147%); 1.9 MiB/s, 1 keys/s, 30 objects/s recovering 2023-09-23T23:20:26.927 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:26 smithi110 ceph-mon[212939]: failed to encode map e284 with expected crc 2023-09-23T23:20:26.927 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:26 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','4':'62260','6':'62260','7':'36861'}"}]': finished 2023-09-23T23:20:26.927 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:26 smithi110 ceph-mon[212939]: osdmap e284: 8 total, 7 up, 7 in 2023-09-23T23:20:26.927 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:26 smithi110 ceph-mon[212939]: failed to encode map e284 with expected crc 2023-09-23T23:20:26.927 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:26 smithi110 ceph-mon[212939]: failed to encode map e284 with expected crc 2023-09-23T23:20:26.927 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:20:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:20:26] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.43.0" 2023-09-23T23:20:26.952 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T23:20:26.953 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-09-23T23:20:26.968 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:26.999 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:27.057 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:27.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:27.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:27.087 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:27.160 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:27.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:26 smithi138 ceph-mon[193606]: from='client.45632 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:20:27.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:26 smithi138 ceph-mon[193606]: pgmap v860: 99 pgs: 1 active+recovery_wait+remapped, 5 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 77 active+clean; 5.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 852 op/s; 11309/273798 objects degraded (4.130%); 38733/273798 objects misplaced (14.147%); 1.9 MiB/s, 1 keys/s, 30 objects/s recovering 2023-09-23T23:20:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:26 smithi138 ceph-mon[193606]: failed to encode map e284 with expected crc 2023-09-23T23:20:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:26 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','4':'62260','6':'62260','7':'36861'}"}]': finished 2023-09-23T23:20:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:26 smithi138 ceph-mon[193606]: osdmap e284: 8 total, 7 up, 7 in 2023-09-23T23:20:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:26 smithi138 ceph-mon[193606]: failed to encode map e284 with expected crc 2023-09-23T23:20:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:26 smithi138 ceph-mon[193606]: failed to encode map e284 with expected crc 2023-09-23T23:20:27.197 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:27.373 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:27.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:27.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:27.407 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:20:27.477 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:27.509 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:27.560 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:27.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:27.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:27.665 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:27.692 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:27.738 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:27.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:27.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:27.751 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:20:27.822 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 77 16 92506 92490 75.0603 10.375 0.00487563 0.0131099 2023-09-23T23:20:27.843 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:27.849 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:20:27.874 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:27.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:27 smithi138 ceph-mon[193606]: failed to encode map e285 with expected crc 2023-09-23T23:20:27.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:27 smithi138 ceph-mon[193606]: osdmap e285: 8 total, 7 up, 7 in 2023-09-23T23:20:27.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:27 smithi138 ceph-mon[193606]: failed to encode map e285 with expected crc 2023-09-23T23:20:27.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:27 smithi138 ceph-mon[193606]: failed to encode map e285 with expected crc 2023-09-23T23:20:27.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:27 smithi138 ceph-mon[193606]: 10.9 continuing backfill to osd.2 from (264'1000,283'5742] MIN to 283'5742 2023-09-23T23:20:27.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:27 smithi138 ceph-mon[193606]: failed to encode map e285 with expected crc 2023-09-23T23:20:27.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:27 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2325845727' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:20:27.954 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:27.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:27.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:28.059 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:28.111 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:28.209 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:27 smithi110 ceph-mon[212939]: failed to encode map e285 with expected crc 2023-09-23T23:20:28.209 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:27 smithi110 ceph-mon[212939]: osdmap e285: 8 total, 7 up, 7 in 2023-09-23T23:20:28.209 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:27 smithi110 ceph-mon[212939]: failed to encode map e285 with expected crc 2023-09-23T23:20:28.209 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:27 smithi110 ceph-mon[212939]: failed to encode map e285 with expected crc 2023-09-23T23:20:28.210 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:27 smithi110 ceph-mon[212939]: 10.9 continuing backfill to osd.2 from (264'1000,283'5742] MIN to 283'5742 2023-09-23T23:20:28.210 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:27 smithi110 ceph-mon[212939]: failed to encode map e285 with expected crc 2023-09-23T23:20:28.210 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:27 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2325845727' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:20:28.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:27 smithi110 ceph-mon[221102]: failed to encode map e285 with expected crc 2023-09-23T23:20:28.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:27 smithi110 ceph-mon[221102]: osdmap e285: 8 total, 7 up, 7 in 2023-09-23T23:20:28.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:27 smithi110 ceph-mon[221102]: failed to encode map e285 with expected crc 2023-09-23T23:20:28.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:27 smithi110 ceph-mon[221102]: failed to encode map e285 with expected crc 2023-09-23T23:20:28.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:27 smithi110 ceph-mon[221102]: 10.9 continuing backfill to osd.2 from (264'1000,283'5742] MIN to 283'5742 2023-09-23T23:20:28.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:27 smithi110 ceph-mon[221102]: failed to encode map e285 with expected crc 2023-09-23T23:20:28.211 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:27 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2325845727' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:20:28.246 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:28.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:28.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:28.358 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:28.405 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:28.822 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 78 16 93124 93108 74.5932 38.625 0.0162346 0.0134022 2023-09-23T23:20:28.951 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:28.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:28.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:29.055 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:20:29.081 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:29.102 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:28 smithi138 ceph-mon[193606]: pgmap v863: 99 pgs: 1 active+recovery_wait+remapped, 5 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 77 active+clean; 5.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 45 MiB/s wr, 726 op/s; 11309/273798 objects degraded (4.130%); 38733/273798 objects misplaced (14.147%); 1.7 MiB/s, 1 keys/s, 27 objects/s recovering 2023-09-23T23:20:29.103 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:28 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:20:29.103 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:28 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2325845727' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:20:29.103 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:28 smithi138 ceph-mon[193606]: osdmap e286: 8 total, 7 up, 7 in 2023-09-23T23:20:29.103 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:28 smithi138 ceph-mon[193606]: failed to encode map e286 with expected crc 2023-09-23T23:20:29.103 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:28 smithi138 ceph-mon[193606]: failed to encode map e286 with expected crc 2023-09-23T23:20:29.103 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:20:29.103 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:28 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 11309/273798 objects degraded (4.130%), 4 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:29.145 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:20:29.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:29.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:20:29.162 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:28 smithi110 ceph-mon[212939]: pgmap v863: 99 pgs: 1 active+recovery_wait+remapped, 5 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 77 active+clean; 5.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 45 MiB/s wr, 726 op/s; 11309/273798 objects degraded (4.130%); 38733/273798 objects misplaced (14.147%); 1.7 MiB/s, 1 keys/s, 27 objects/s recovering 2023-09-23T23:20:29.163 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:28 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:20:29.163 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:28 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2325845727' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:20:29.163 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:28 smithi110 ceph-mon[212939]: osdmap e286: 8 total, 7 up, 7 in 2023-09-23T23:20:29.163 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:28 smithi110 ceph-mon[212939]: failed to encode map e286 with expected crc 2023-09-23T23:20:29.163 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:28 smithi110 ceph-mon[212939]: failed to encode map e286 with expected crc 2023-09-23T23:20:29.163 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:20:29.163 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:28 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 11309/273798 objects degraded (4.130%), 4 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:29.163 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:28 smithi110 ceph-mon[221102]: pgmap v863: 99 pgs: 1 active+recovery_wait+remapped, 5 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 77 active+clean; 5.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 45 MiB/s wr, 726 op/s; 11309/273798 objects degraded (4.130%); 38733/273798 objects misplaced (14.147%); 1.7 MiB/s, 1 keys/s, 27 objects/s recovering 2023-09-23T23:20:29.163 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:28 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:20:29.164 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:28 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2325845727' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:20:29.164 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:28 smithi110 ceph-mon[221102]: osdmap e286: 8 total, 7 up, 7 in 2023-09-23T23:20:29.164 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:28 smithi110 ceph-mon[221102]: failed to encode map e286 with expected crc 2023-09-23T23:20:29.164 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:28 smithi110 ceph-mon[221102]: failed to encode map e286 with expected crc 2023-09-23T23:20:29.164 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:20:29.164 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:28 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 11309/273798 objects degraded (4.130%), 4 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:29.255 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:29.282 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:29.703 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:20:29.703 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:20:29.703 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:20:29.703 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:20:29.704 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:20:29.822 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 79 16 93774 93758 74.1632 40.625 0.016393 0.0134783 2023-09-23T23:20:29.885 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:29.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:29.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:29.952 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:29.992 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:30.063 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:30.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:29 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:20:30.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:29 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:20:30.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:29 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:20:30.259 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:30.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:30.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:30.363 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:30.428 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:30.622 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:30.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:30.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:30.713 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:20:30.733 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:20:30.815 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:30.822 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:20:30.822145+0000 min lat: 0.000955869 max lat: 2.26865 avg lat: 0.0135488 2023-09-23T23:20:30.822 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:20:30.822 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 80 16 94471 94455 73.7807 43.5625 0.0186435 0.0135488 2023-09-23T23:20:30.869 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:30 smithi110 ceph-mon[221102]: pgmap v865: 99 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 76 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 907 op/s; 11309/276912 objects degraded (4.084%); 44905/276912 objects misplaced (16.216%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:20:30.869 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:30 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:20:30.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:30 smithi138 ceph-mon[193606]: pgmap v865: 99 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 76 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 907 op/s; 11309/276912 objects degraded (4.084%); 44905/276912 objects misplaced (16.216%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:20:30.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:30 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:20:30.954 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:20:30.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:30.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:20:31.063 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:31.076 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:20:31.076 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:20:31.076 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:20:31.076 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:20:31.102 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:20:31.115 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:31.137 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:20:31.180 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:20:31.187 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:20:31.187 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:20:31.188 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:20:31.188 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:20:31.188 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:20:31.188 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:20:31.188 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:20:31.188 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:20:31.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:30 smithi110 ceph-mon[212939]: pgmap v865: 99 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 76 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 907 op/s; 11309/276912 objects degraded (4.084%); 44905/276912 objects misplaced (16.216%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:20:31.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:30 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:20:31.224 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:31.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:31.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:31.278 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:20:31.334 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:31.359 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:31.400 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:31.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:31.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:31.504 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:20:31.558 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:31.822 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 81 16 95135 95119 73.3821 41.5 0.0110705 0.0136218 2023-09-23T23:20:31.934 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:20:31.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:31 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3920108901' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:31.944 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:20:31.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:31.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:20:32.049 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:32.114 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:32.156 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:32.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:32.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:32.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:31 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3920108901' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:32.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:31 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3920108901' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:32.259 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:32.287 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:32.578 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:32.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:32.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:32.685 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:32.713 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:20:32.713 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:20:32.767 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:32.828 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 82 16 95802 95786 72.9905 41.6875 0.0145468 0.0136949 2023-09-23T23:20:32.949 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:32.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:32.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:33.044 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:33.060 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:32 smithi138 ceph-mon[193606]: pgmap v866: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 5 active+undersized, 75 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 24 MiB/s wr, 378 op/s; 11567/280605 objects degraded (4.122%); 50888/280605 objects misplaced (18.135%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:20:33.065 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:20:33.136 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:32 smithi110 ceph-mon[212939]: pgmap v866: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 5 active+undersized, 75 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 24 MiB/s wr, 378 op/s; 11567/280605 objects degraded (4.122%); 50888/280605 objects misplaced (18.135%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:20:33.137 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:32 smithi110 ceph-mon[221102]: pgmap v866: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 5 active+undersized, 75 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 24 MiB/s wr, 378 op/s; 11567/280605 objects degraded (4.122%); 50888/280605 objects misplaced (18.135%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:20:33.138 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:33.268 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:20:33.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:33.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:20:33.363 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:20:33.364 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:20:33.376 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:33.441 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:33.828 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 83 16 96625 96609 72.7308 51.4375 0.0200459 0.0137438 2023-09-23T23:20:34.089 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:34.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:34.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:34.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:33 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 11567/280605 objects degraded (4.122%), 4 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:34.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:33 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'46251','5':'34720','7':'40137'}"}]: dispatch 2023-09-23T23:20:34.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:33 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'46251','5':'34720','7':'40137'}"}]: dispatch 2023-09-23T23:20:34.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:33 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 11567/280605 objects degraded (4.122%), 4 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:34.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:33 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'46251','5':'34720','7':'40137'}"}]: dispatch 2023-09-23T23:20:34.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:33 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'46251','5':'34720','7':'40137'}"}]: dispatch 2023-09-23T23:20:34.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:33 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 11567/280605 objects degraded (4.122%), 4 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:34.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:33 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'46251','5':'34720','7':'40137'}"}]: dispatch 2023-09-23T23:20:34.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:33 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'46251','5':'34720','7':'40137'}"}]: dispatch 2023-09-23T23:20:34.204 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:34.239 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:34.282 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:34.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:34.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:34.375 INFO:teuthology.orchestra.run.smithi110.stdout:moved 5 / 297 (1.6835%) 2023-09-23T23:20:34.375 INFO:teuthology.orchestra.run.smithi110.stdout:avg 49.5 2023-09-23T23:20:34.375 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 11.3248 -> 10.5475 (expected baseline 6.42262) 2023-09-23T23:20:34.375 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.5 with 25 -> 27 pgs (0.50505 -> 0.545455 * mean) 2023-09-23T23:20:34.376 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.6 with 57 -> 56 pgs (1.15152 -> 1.13131 * mean) 2023-09-23T23:20:34.376 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:20:34.376 INFO:teuthology.orchestra.run.smithi110.stdout:oload 1000 2023-09-23T23:20:34.376 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 0.05 2023-09-23T23:20:34.376 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:20:34.376 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0318 2023-09-23T23:20:34.376 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.3181 2023-09-23T23:20:34.376 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 weight 0.5625 -> 0.6124 2023-09-23T23:20:34.376 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 weight 0.6557 -> 0.7057 2023-09-23T23:20:34.376 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 weight 0.4798 -> 0.5298 2023-09-23T23:20:34.387 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:34.414 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:34.459 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:34.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:34.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:34.563 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:20:34.607 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:34.631 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:20:34.666 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:20:34.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:34.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:20:34.714 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:20:34.774 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:34.815 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:34.828 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 84 16 97142 97126 72.2494 32.3125 0.00421371 0.0138089 2023-09-23T23:20:34.929 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:34 smithi110 ceph-mon[212939]: pgmap v867: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 1 active+recovering+undersized+remapped, 5 active+undersized, 75 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 19 MiB/s wr, 296 op/s; 11567/280605 objects degraded (4.122%); 50888/280605 objects misplaced (18.135%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:20:34.929 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:34 smithi110 ceph-mon[212939]: from='client.55576 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:20:34.929 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:34 smithi110 ceph-mon[212939]: failed to encode map e287 with expected crc 2023-09-23T23:20:34.929 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:34 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'46251','5':'34720','7':'40137'}"}]': finished 2023-09-23T23:20:34.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:34 smithi110 ceph-mon[212939]: osdmap e287: 8 total, 7 up, 7 in 2023-09-23T23:20:34.930 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:34 smithi110 ceph-mon[221102]: pgmap v867: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 1 active+recovering+undersized+remapped, 5 active+undersized, 75 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 19 MiB/s wr, 296 op/s; 11567/280605 objects degraded (4.122%); 50888/280605 objects misplaced (18.135%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:20:34.930 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:34 smithi110 ceph-mon[221102]: from='client.55576 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:20:34.930 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:34 smithi110 ceph-mon[221102]: failed to encode map e287 with expected crc 2023-09-23T23:20:34.930 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:34 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'46251','5':'34720','7':'40137'}"}]': finished 2023-09-23T23:20:34.930 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:34 smithi110 ceph-mon[221102]: osdmap e287: 8 total, 7 up, 7 in 2023-09-23T23:20:35.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:34 smithi138 ceph-mon[193606]: pgmap v867: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 1 active+recovering+undersized+remapped, 5 active+undersized, 75 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 19 MiB/s wr, 296 op/s; 11567/280605 objects degraded (4.122%); 50888/280605 objects misplaced (18.135%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:20:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:34 smithi138 ceph-mon[193606]: from='client.55576 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:20:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:34 smithi138 ceph-mon[193606]: failed to encode map e287 with expected crc 2023-09-23T23:20:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:34 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'46251','5':'34720','7':'40137'}"}]': finished 2023-09-23T23:20:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:34 smithi138 ceph-mon[193606]: osdmap e287: 8 total, 7 up, 7 in 2023-09-23T23:20:35.614 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:35.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:35.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:35.720 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:35.739 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T23:20:35.740 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-09-23T23:20:35.800 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:35.833 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 85 16 97368 97352 71.5624 14.125 0.009787 0.0139674 2023-09-23T23:20:35.952 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:35.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:35.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:36.061 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:36.098 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:36.108 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:36.156 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:36.176 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:36.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:36.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:36.287 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:36.331 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:36.396 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:36 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:20:36.396 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:36 smithi138 ceph-mon[193606]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.396 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:36 smithi138 ceph-mon[193606]: osdmap e288: 8 total, 7 up, 7 in 2023-09-23T23:20:36.396 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:36 smithi138 ceph-mon[193606]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.397 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:36 smithi138 ceph-mon[193606]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.397 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:36 smithi138 ceph-mon[193606]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.397 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:36 smithi138 ceph-mon[193606]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.397 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:36 smithi138 ceph-mon[193606]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.397 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:36 smithi138 ceph-mon[193606]: pgmap v870: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 2 remapped+peering, 3 peering, 7 active+clean+remapped, 4 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 71 active+clean; 5.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 749 op/s; 11977/291003 objects degraded (4.116%); 47498/291003 objects misplaced (16.322%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T23:20:36.399 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:20:36.399 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[212939]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.399 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[212939]: osdmap e288: 8 total, 7 up, 7 in 2023-09-23T23:20:36.399 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[212939]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.399 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[212939]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.399 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[212939]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.399 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[212939]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.399 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[212939]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.399 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[212939]: pgmap v870: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 2 remapped+peering, 3 peering, 7 active+clean+remapped, 4 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 71 active+clean; 5.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 749 op/s; 11977/291003 objects degraded (4.116%); 47498/291003 objects misplaced (16.322%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T23:20:36.400 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:36.400 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:20:36.400 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[221102]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.400 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[221102]: osdmap e288: 8 total, 7 up, 7 in 2023-09-23T23:20:36.400 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[221102]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.400 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[221102]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.401 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[221102]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.401 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[221102]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.401 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[221102]: failed to encode map e288 with expected crc 2023-09-23T23:20:36.401 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:36 smithi110 ceph-mon[221102]: pgmap v870: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 2 remapped+peering, 3 peering, 7 active+clean+remapped, 4 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 71 active+clean; 5.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 749 op/s; 11977/291003 objects degraded (4.116%); 47498/291003 objects misplaced (16.322%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T23:20:36.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:36.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:36.506 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:36.592 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:36.770 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:36.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:36.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:36.833 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 86 16 98098 98082 71.2608 45.625 0.0165193 0.0140274 2023-09-23T23:20:36.879 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:36.929 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:36.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:20:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:20:36] "GET /metrics HTTP/1.1" 200 36626 "" "Prometheus/2.43.0" 2023-09-23T23:20:36.999 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:37.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:37.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:37.108 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:20:37.174 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:37.394 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:37 smithi110 ceph-mon[212939]: osdmap e289: 8 total, 7 up, 7 in 2023-09-23T23:20:37.394 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:37 smithi110 ceph-mon[212939]: failed to encode map e289 with expected crc 2023-09-23T23:20:37.394 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:37 smithi110 ceph-mon[212939]: failed to encode map e289 with expected crc 2023-09-23T23:20:37.394 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:37 smithi110 ceph-mon[212939]: failed to encode map e289 with expected crc 2023-09-23T23:20:37.394 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:37 smithi110 ceph-mon[212939]: failed to encode map e289 with expected crc 2023-09-23T23:20:37.394 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:37 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3844216459' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:20:37.395 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:37 smithi110 ceph-mon[221102]: osdmap e289: 8 total, 7 up, 7 in 2023-09-23T23:20:37.396 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:37 smithi110 ceph-mon[221102]: failed to encode map e289 with expected crc 2023-09-23T23:20:37.396 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:37 smithi110 ceph-mon[221102]: failed to encode map e289 with expected crc 2023-09-23T23:20:37.396 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:37 smithi110 ceph-mon[221102]: failed to encode map e289 with expected crc 2023-09-23T23:20:37.396 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:37 smithi110 ceph-mon[221102]: failed to encode map e289 with expected crc 2023-09-23T23:20:37.396 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:37 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3844216459' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:20:37.422 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:20:37.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:37 smithi138 ceph-mon[193606]: osdmap e289: 8 total, 7 up, 7 in 2023-09-23T23:20:37.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:37 smithi138 ceph-mon[193606]: failed to encode map e289 with expected crc 2023-09-23T23:20:37.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:37 smithi138 ceph-mon[193606]: failed to encode map e289 with expected crc 2023-09-23T23:20:37.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:37 smithi138 ceph-mon[193606]: failed to encode map e289 with expected crc 2023-09-23T23:20:37.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:37 smithi138 ceph-mon[193606]: failed to encode map e289 with expected crc 2023-09-23T23:20:37.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:37 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3844216459' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:20:37.833 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 87 16 98949 98933 71.053 53.1875 0.0165269 0.0140694 2023-09-23T23:20:38.426 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:20:38.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:38.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:20:38.533 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:20:38.603 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:38.681 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:20:38.689 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:38 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:20:38.689 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:38 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3844216459' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:20:38.689 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:38 smithi138 ceph-mon[193606]: osdmap e290: 8 total, 7 up, 7 in 2023-09-23T23:20:38.689 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:38 smithi138 ceph-mon[193606]: failed to encode map e290 with expected crc 2023-09-23T23:20:38.690 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:38 smithi138 ceph-mon[193606]: failed to encode map e290 with expected crc 2023-09-23T23:20:38.690 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:38 smithi138 ceph-mon[193606]: failed to encode map e290 with expected crc 2023-09-23T23:20:38.690 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:38 smithi138 ceph-mon[193606]: failed to encode map e290 with expected crc 2023-09-23T23:20:38.690 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:38 smithi138 ceph-mon[193606]: pgmap v873: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 2 remapped+peering, 3 peering, 7 active+clean+remapped, 4 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 71 active+clean; 5.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 866 op/s; 11977/291003 objects degraded (4.116%); 47498/291003 objects misplaced (16.322%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T23:20:38.690 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:38 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 11977/291003 objects degraded (4.116%), 4 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:38.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:38.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:20:38.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:20:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3844216459' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:20:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[221102]: osdmap e290: 8 total, 7 up, 7 in 2023-09-23T23:20:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[221102]: failed to encode map e290 with expected crc 2023-09-23T23:20:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[221102]: failed to encode map e290 with expected crc 2023-09-23T23:20:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[221102]: failed to encode map e290 with expected crc 2023-09-23T23:20:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[221102]: failed to encode map e290 with expected crc 2023-09-23T23:20:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[221102]: pgmap v873: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 2 remapped+peering, 3 peering, 7 active+clean+remapped, 4 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 71 active+clean; 5.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 866 op/s; 11977/291003 objects degraded (4.116%); 47498/291003 objects misplaced (16.322%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T23:20:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 11977/291003 objects degraded (4.116%), 4 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:20:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3844216459' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:20:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[212939]: osdmap e290: 8 total, 7 up, 7 in 2023-09-23T23:20:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[212939]: failed to encode map e290 with expected crc 2023-09-23T23:20:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[212939]: failed to encode map e290 with expected crc 2023-09-23T23:20:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[212939]: failed to encode map e290 with expected crc 2023-09-23T23:20:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[212939]: failed to encode map e290 with expected crc 2023-09-23T23:20:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[212939]: pgmap v873: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 2 remapped+peering, 3 peering, 7 active+clean+remapped, 4 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 71 active+clean; 5.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 866 op/s; 11977/291003 objects degraded (4.116%); 47498/291003 objects misplaced (16.322%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T23:20:38.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:38 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 11977/291003 objects degraded (4.116%), 4 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:38.792 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:38.819 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:38.833 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 88 16 99730 99714 70.8003 48.8125 0.00932192 0.0141066 2023-09-23T23:20:39.601 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:39.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:39.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:39.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:39 smithi138 ceph-mon[193606]: osdmap e291: 8 total, 7 up, 7 in 2023-09-23T23:20:39.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:39 smithi138 ceph-mon[193606]: failed to encode map e291 with expected crc 2023-09-23T23:20:39.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:39 smithi138 ceph-mon[193606]: failed to encode map e291 with expected crc 2023-09-23T23:20:39.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:39 smithi138 ceph-mon[193606]: failed to encode map e291 with expected crc 2023-09-23T23:20:39.701 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:39.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:39 smithi110 ceph-mon[221102]: osdmap e291: 8 total, 7 up, 7 in 2023-09-23T23:20:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:39 smithi110 ceph-mon[221102]: failed to encode map e291 with expected crc 2023-09-23T23:20:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:39 smithi110 ceph-mon[221102]: failed to encode map e291 with expected crc 2023-09-23T23:20:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:39 smithi110 ceph-mon[221102]: failed to encode map e291 with expected crc 2023-09-23T23:20:39.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:39 smithi110 ceph-mon[212939]: osdmap e291: 8 total, 7 up, 7 in 2023-09-23T23:20:39.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:39 smithi110 ceph-mon[212939]: failed to encode map e291 with expected crc 2023-09-23T23:20:39.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:39 smithi110 ceph-mon[212939]: failed to encode map e291 with expected crc 2023-09-23T23:20:39.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:39 smithi110 ceph-mon[212939]: failed to encode map e291 with expected crc 2023-09-23T23:20:39.713 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:20:39.811 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:39.840 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 89 16 100388 100372 70.4613 41.125 0.0210219 0.0141853 2023-09-23T23:20:39.914 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:20:39.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:39.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:20:40.022 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:40.070 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:40.152 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:40.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:40.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:40.265 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:20:40.312 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:40.404 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:20:40.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:40.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:20:40.514 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:40.551 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:40.579 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:20:40.618 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:20:40.642 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:40.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:40.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:40.654 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:20:40.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:40 smithi138 ceph-mon[193606]: pgmap v875: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 remapped+peering, 2 peering, 8 active+clean+remapped, 4 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 73 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 21 MiB/s wr, 335 op/s; 11977/295149 objects degraded (4.058%); 48099/295149 objects misplaced (16.297%) 2023-09-23T23:20:40.692 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:20:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:40 smithi110 ceph-mon[212939]: pgmap v875: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 remapped+peering, 2 peering, 8 active+clean+remapped, 4 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 73 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 21 MiB/s wr, 335 op/s; 11977/295149 objects degraded (4.058%); 48099/295149 objects misplaced (16.297%) 2023-09-23T23:20:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:40 smithi110 ceph-mon[221102]: pgmap v875: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 remapped+peering, 2 peering, 8 active+clean+remapped, 4 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 73 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 21 MiB/s wr, 335 op/s; 11977/295149 objects degraded (4.058%); 48099/295149 objects misplaced (16.297%) 2023-09-23T23:20:40.731 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:20:40.751 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:40.767 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:20:40.814 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:20:40.815 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:20:40.815 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:20:40.815 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:20:40.815 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:20:40.815 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:20:40.815 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:20:40.815 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:20:40.815 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:20:40.828 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:40.840 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total time run: 90.0101 2023-09-23T23:20:40.840 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total writes made: 101205 2023-09-23T23:20:40.840 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Write size: 65536 2023-09-23T23:20:40.840 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object size: 65536 2023-09-23T23:20:40.840 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Bandwidth (MB/sec): 70.2734 2023-09-23T23:20:40.840 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Bandwidth: 41.37 2023-09-23T23:20:40.840 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max bandwidth (MB/sec): 211.438 2023-09-23T23:20:40.841 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min bandwidth (MB/sec): 0 2023-09-23T23:20:40.841 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average IOPS: 1124 2023-09-23T23:20:40.841 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev IOPS: 661.92 2023-09-23T23:20:40.841 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max IOPS: 3383 2023-09-23T23:20:40.841 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min IOPS: 0 2023-09-23T23:20:40.841 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average Latency(s): 0.0142263 2023-09-23T23:20:40.841 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Latency(s): 0.0530156 2023-09-23T23:20:40.841 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max latency(s): 2.26865 2023-09-23T23:20:40.841 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min latency(s): 0.000955869 2023-09-23T23:20:40.846 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T23:20:40.918 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:40.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:40.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:41.022 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:20:41.081 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:41.173 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:20:41.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:41.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:20:41.281 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:41.342 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:41.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:41 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1627766039' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:41.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:41 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1627766039' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:41.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:41 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1627766039' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:41.747 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:20:41.748 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:20:41.748 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-09-23T23:20:41.748 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:20:41.754 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:20:42.776 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:42.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:42.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:42.789 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:42 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:20:42.789 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:42 smithi110 ceph-mon[221102]: pgmap v876: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 33 MiB/s wr, 524 op/s; 18760/300441 objects degraded (6.244%); 53447/300441 objects misplaced (17.790%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T23:20:42.790 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:42 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:20:42.790 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:42 smithi110 ceph-mon[212939]: pgmap v876: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 33 MiB/s wr, 524 op/s; 18760/300441 objects degraded (6.244%); 53447/300441 objects misplaced (17.790%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T23:20:42.890 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:42.936 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:42 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:20:42.936 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:42 smithi138 ceph-mon[193606]: pgmap v876: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 33 MiB/s wr, 524 op/s; 18760/300441 objects degraded (6.244%); 53447/300441 objects misplaced (17.790%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T23:20:42.944 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:43.049 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:43.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:43.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:43.162 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:43.256 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:20:43.256 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:20:43.292 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:43.307 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:20:43.535 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:43.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:43.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:43.658 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:43.697 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:43.721 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:43.770 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:43.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:43.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:43.875 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:43.954 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:44.052 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:44.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:44.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:44.159 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:44.199 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:44.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:43 smithi110 ceph-mon[212939]: 1.9 deep-scrub starts 2023-09-23T23:20:44.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:44.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:20:44.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:43 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 18760/300441 objects degraded (6.244%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:44.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:43 smithi110 ceph-mon[221102]: 1.9 deep-scrub starts 2023-09-23T23:20:44.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:44.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:20:44.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:43 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 18760/300441 objects degraded (6.244%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:44.271 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:44.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:44.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:44.279 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:43 smithi138 ceph-mon[193606]: 1.9 deep-scrub starts 2023-09-23T23:20:44.279 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:44.279 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:20:44.279 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:43 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 18760/300441 objects degraded (6.244%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:44.323 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:44.337 INFO:teuthology.orchestra.run.smithi110.stdout:moved 13 / 297 (4.3771%) 2023-09-23T23:20:44.337 INFO:teuthology.orchestra.run.smithi110.stdout:avg 49.5 2023-09-23T23:20:44.338 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 10.5475 -> 10.3722 (expected baseline 6.42262) 2023-09-23T23:20:44.338 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.5 with 27 -> 26 pgs (0.545455 -> 0.525253 * mean) 2023-09-23T23:20:44.338 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.6 with 56 -> 54 pgs (1.13131 -> 1.09091 * mean) 2023-09-23T23:20:44.338 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:20:44.338 INFO:teuthology.orchestra.run.smithi110.stdout:oload 110 2023-09-23T23:20:44.338 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 0.05 2023-09-23T23:20:44.338 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:20:44.338 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0337 2023-09-23T23:20:44.338 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.0371 2023-09-23T23:20:44.338 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4 weight 0.9500 -> 0.9000 2023-09-23T23:20:44.338 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 weight 0.9500 -> 0.9000 2023-09-23T23:20:44.339 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 weight 0.6124 -> 0.6624 2023-09-23T23:20:44.339 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6 weight 0.9500 -> 0.9000 2023-09-23T23:20:44.339 INFO:teuthology.orchestra.run.smithi110.stderr:no change 2023-09-23T23:20:44.379 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:44.453 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:44.701 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:20:44 smithi110 conmon[198813]: 2023-09-23T23:20:44.329+0000 7ff0150b6700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-09-23T23:20:45.257 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:20:45.289 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:44 smithi110 ceph-mon[212939]: pgmap v877: 99 pgs: 1 active+clean+remapped+scrubbing+deep, 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 28 MiB/s wr, 443 op/s; 18760/300441 objects degraded (6.244%); 53447/300441 objects misplaced (17.790%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T23:20:45.289 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:44 smithi110 ceph-mon[221102]: pgmap v877: 99 pgs: 1 active+clean+remapped+scrubbing+deep, 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 28 MiB/s wr, 443 op/s; 18760/300441 objects degraded (6.244%); 53447/300441 objects misplaced (17.790%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T23:20:45.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:44 smithi138 ceph-mon[193606]: pgmap v877: 99 pgs: 1 active+clean+remapped+scrubbing+deep, 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 28 MiB/s wr, 443 op/s; 18760/300441 objects degraded (6.244%); 53447/300441 objects misplaced (17.790%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T23:20:45.470 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:45.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:45.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:45.579 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:20:45.623 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:45.721 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:20:45.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:45.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:20:45.831 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:45.857 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:45.940 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:45.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:45.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:46.052 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:46.131 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:46.285 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:46 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:20:46.286 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:46 smithi110 ceph-mon[212939]: from='client.45653 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:20:46.286 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:46 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:20:46.286 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:46 smithi110 ceph-mon[221102]: from='client.45653 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:20:46.319 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T23:20:46.319 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-09-23T23:20:46.356 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:46.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:46.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:46 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:20:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:46 smithi138 ceph-mon[193606]: from='client.45653 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:20:46.467 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:20:46.607 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:46.843 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:20:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:20:46] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.43.0" 2023-09-23T23:20:46.916 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:47.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:47 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:20:47.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:47 smithi110 ceph-mon[221102]: pgmap v878: 99 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped+scrubbing+deep, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 126 B/s rd, 33 MiB/s wr, 889 op/s; 18374/294786 objects degraded (6.233%); 52450/294786 objects misplaced (17.793%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:20:47.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:47 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:20:47.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:47 smithi110 ceph-mon[212939]: pgmap v878: 99 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped+scrubbing+deep, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 126 B/s rd, 33 MiB/s wr, 889 op/s; 18374/294786 objects degraded (6.233%); 52450/294786 objects misplaced (17.793%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:20:47.391 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:20:47.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:47.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:20:47.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:47 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:20:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:47 smithi138 ceph-mon[193606]: pgmap v878: 99 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped+scrubbing+deep, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 126 B/s rd, 33 MiB/s wr, 889 op/s; 18374/294786 objects degraded (6.233%); 52450/294786 objects misplaced (17.793%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:20:47.500 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:47.526 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:47.568 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:47.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:47.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:47.675 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:47.715 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:47.801 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:47.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:47.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:47.910 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:47.965 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:48.216 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:48.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:48.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:48.330 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:48.374 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:48.464 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:48.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:48.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:48.582 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:48.589 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:48.684 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:49.103 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:49.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:49.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:49.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:49 smithi110 ceph-mon[221102]: pgmap v879: 99 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped+scrubbing+deep, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 26 MiB/s wr, 719 op/s; 18374/294786 objects degraded (6.233%); 52450/294786 objects misplaced (17.793%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:20:49.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:49 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 18374/294786 objects degraded (6.233%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:49.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:49 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/133975500' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:20:49.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:49 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:20:49.208 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:49 smithi110 ceph-mon[212939]: pgmap v879: 99 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped+scrubbing+deep, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 26 MiB/s wr, 719 op/s; 18374/294786 objects degraded (6.233%); 52450/294786 objects misplaced (17.793%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:20:49.208 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:49 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 18374/294786 objects degraded (6.233%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:49.208 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:49 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/133975500' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:20:49.208 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:49 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:20:49.212 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:49.281 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:49.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:49 smithi138 ceph-mon[193606]: pgmap v879: 99 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped+scrubbing+deep, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 26 MiB/s wr, 719 op/s; 18374/294786 objects degraded (6.233%); 52450/294786 objects misplaced (17.793%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:20:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:49 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 18374/294786 objects degraded (6.233%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:49 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/133975500' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:20:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:49 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:20:49.476 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:20:50.320 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:50 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:20:50.320 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:50 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:20:50.320 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:50 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:20:50.321 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:50 smithi110 ceph-mon[221102]: osdmap e292: 8 total, 7 up, 7 in 2023-09-23T23:20:50.321 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:50 smithi110 ceph-mon[221102]: failed to encode map e292 with expected crc 2023-09-23T23:20:50.321 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:50 smithi110 ceph-mon[221102]: failed to encode map e292 with expected crc 2023-09-23T23:20:50.321 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:50 smithi110 ceph-mon[221102]: failed to encode map e292 with expected crc 2023-09-23T23:20:50.321 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:50 smithi110 ceph-mon[221102]: failed to encode map e292 with expected crc 2023-09-23T23:20:50.321 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:50 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:20:50.321 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:50 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:20:50.321 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:50 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:20:50.321 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:50 smithi110 ceph-mon[212939]: osdmap e292: 8 total, 7 up, 7 in 2023-09-23T23:20:50.321 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:50 smithi110 ceph-mon[212939]: failed to encode map e292 with expected crc 2023-09-23T23:20:50.322 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:50 smithi110 ceph-mon[212939]: failed to encode map e292 with expected crc 2023-09-23T23:20:50.322 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:50 smithi110 ceph-mon[212939]: failed to encode map e292 with expected crc 2023-09-23T23:20:50.322 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:50 smithi110 ceph-mon[212939]: failed to encode map e292 with expected crc 2023-09-23T23:20:50.435 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:50 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:20:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:50 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:20:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:50 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:20:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:50 smithi138 ceph-mon[193606]: osdmap e292: 8 total, 7 up, 7 in 2023-09-23T23:20:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:50 smithi138 ceph-mon[193606]: failed to encode map e292 with expected crc 2023-09-23T23:20:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:50 smithi138 ceph-mon[193606]: failed to encode map e292 with expected crc 2023-09-23T23:20:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:50 smithi138 ceph-mon[193606]: failed to encode map e292 with expected crc 2023-09-23T23:20:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:50 smithi138 ceph-mon[193606]: failed to encode map e292 with expected crc 2023-09-23T23:20:50.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:50.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:50.502 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:20:50.502 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:20:50.502 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:20:50.502 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:20:50.502 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:20:50.545 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:20:50.617 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:51.315 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:51 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:20:51.315 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:51 smithi110 ceph-mon[221102]: pgmap v881: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 18 MiB/s wr, 759 op/s; 18374/289425 objects degraded (6.348%); 51675/289425 objects misplaced (17.854%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:20:51.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:51 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:20:51.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:51 smithi110 ceph-mon[212939]: pgmap v881: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 18 MiB/s wr, 759 op/s; 18374/289425 objects degraded (6.348%); 51675/289425 objects misplaced (17.854%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:20:51.434 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:20:51.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:51 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:20:51.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:51 smithi138 ceph-mon[193606]: pgmap v881: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 18 MiB/s wr, 759 op/s; 18374/289425 objects degraded (6.348%); 51675/289425 objects misplaced (17.854%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:20:51.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:51.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:20:51.545 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:51.587 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:52.301 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:52.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:52.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:52.337 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:20:52.415 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:52.560 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:52.756 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:52.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:52.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:52.862 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:20:52.913 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:20:52.932 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:52.950 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:20:53.000 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:20:53.043 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:20:53.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:53 smithi110 ceph-mon[212939]: pgmap v882: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 6.8 MiB/s wr, 824 op/s; 17580/282201 objects degraded (6.230%); 50491/282201 objects misplaced (17.892%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:20:53.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:53 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3822784906' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:53.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:53 smithi110 ceph-mon[221102]: pgmap v882: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 6.8 MiB/s wr, 824 op/s; 17580/282201 objects degraded (6.230%); 50491/282201 objects misplaced (17.892%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:20:53.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:53 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3822784906' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:53.286 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:20:53.286 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:20:53.286 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:20:53.286 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:20:53.286 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:20:53.286 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:20:53.286 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:20:53.286 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:20:53.287 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:20:53.287 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:20:53.287 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:20:53.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:53 smithi138 ceph-mon[193606]: pgmap v882: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 6.8 MiB/s wr, 824 op/s; 17580/282201 objects degraded (6.230%); 50491/282201 objects misplaced (17.892%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:20:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:53 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3822784906' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:20:53.661 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:20:53.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:53.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:20:53.766 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:20:53.847 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:54.242 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:20:54.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:54 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 17580/282201 objects degraded (6.230%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:54.345 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:54 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 17580/282201 objects degraded (6.230%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:54.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:54 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 17580/282201 objects degraded (6.230%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:54.625 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:20:54.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:54.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:20:54.672 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:20:54.673 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:20:54.730 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:54.778 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:54.853 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:54.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:54.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:54.960 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:20:55.016 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:55.062 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:55 smithi138 ceph-mon[193606]: pgmap v883: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 6.8 MiB/s wr, 824 op/s; 17580/282201 objects degraded (6.230%); 50491/282201 objects misplaced (17.892%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:20:55.079 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:55.094 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:20:55.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:55.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:20:55.200 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:55.212 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:55.254 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:55.316 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:55.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:55.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:55.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:55 smithi110 ceph-mon[212939]: pgmap v883: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 6.8 MiB/s wr, 824 op/s; 17580/282201 objects degraded (6.230%); 50491/282201 objects misplaced (17.892%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:20:55.421 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:55 smithi110 ceph-mon[221102]: pgmap v883: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 6.8 MiB/s wr, 824 op/s; 17580/282201 objects degraded (6.230%); 50491/282201 objects misplaced (17.892%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:20:55.431 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:20:55.487 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:55.571 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:20:55.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:55.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:20:55.585 INFO:teuthology.orchestra.run.smithi110.stdout:moved 5 / 297 (1.6835%) 2023-09-23T23:20:55.585 INFO:teuthology.orchestra.run.smithi110.stdout:avg 49.5 2023-09-23T23:20:55.585 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 10.5475 -> 10.6732 (expected baseline 6.42262) 2023-09-23T23:20:55.585 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.5 with 27 -> 26 pgs (0.545455 -> 0.525253 * mean) 2023-09-23T23:20:55.585 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.6 with 56 -> 54 pgs (1.13131 -> 1.09091 * mean) 2023-09-23T23:20:55.585 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:20:55.586 INFO:teuthology.orchestra.run.smithi110.stdout:oload 1000 2023-09-23T23:20:55.586 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 0.05 2023-09-23T23:20:55.586 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:20:55.586 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0312 2023-09-23T23:20:55.586 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.3118 2023-09-23T23:20:55.586 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 weight 0.6124 -> 0.6624 2023-09-23T23:20:55.586 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 weight 0.7057 -> 0.7557 2023-09-23T23:20:55.586 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 weight 0.5298 -> 0.5798 2023-09-23T23:20:55.605 INFO:teuthology.orchestra.run.smithi110.stderr:no change 2023-09-23T23:20:55.672 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:20:55 smithi110 conmon[198813]: 2023-09-23T23:20:55.579+0000 7ff0150b6700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T23:20:55.676 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:55.766 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:56.673 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:20:57.025 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:20:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:20:56] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.43.0" 2023-09-23T23:20:57.441 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T23:20:57.441 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-09-23T23:20:57.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:57 smithi110 ceph-mon[212939]: pgmap v884: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 806 op/s; 16867/270570 objects degraded (6.234%); 48704/270570 objects misplaced (18.001%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T23:20:57.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:57 smithi110 ceph-mon[212939]: from='client.35679 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:20:57.442 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:57 smithi110 ceph-mon[221102]: pgmap v884: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 806 op/s; 16867/270570 objects degraded (6.234%); 48704/270570 objects misplaced (18.001%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T23:20:57.442 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:57 smithi110 ceph-mon[221102]: from='client.35679 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:20:57.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:57 smithi138 ceph-mon[193606]: pgmap v884: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 806 op/s; 16867/270570 objects degraded (6.234%); 48704/270570 objects misplaced (18.001%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T23:20:57.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:57 smithi138 ceph-mon[193606]: from='client.35679 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:20:57.754 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:57.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:57.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:57.866 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:20:57.932 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:58.498 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:20:58.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:58.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:20:58.607 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:58.650 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:58.744 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:58.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:58.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:58.852 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:20:58.931 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:59.043 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:59.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:59 smithi110 ceph-mon[212939]: pgmap v885: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 806 op/s; 16867/270570 objects degraded (6.234%); 48704/270570 objects misplaced (18.001%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T23:20:59.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:59 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:59.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:59 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:20:59.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:20:59 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 16867/270570 objects degraded (6.234%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:59.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:59 smithi110 ceph-mon[221102]: pgmap v885: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 806 op/s; 16867/270570 objects degraded (6.234%); 48704/270570 objects misplaced (18.001%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T23:20:59.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:59 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:59.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:59 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:20:59.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:20:59 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 16867/270570 objects degraded (6.234%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:59.292 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:20:59.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:59.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:20:59.400 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:20:59.436 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:58 smithi138 ceph-mon[193606]: pgmap v885: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 806 op/s; 16867/270570 objects degraded (6.234%); 48704/270570 objects misplaced (18.001%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T23:20:59.436 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:20:59.436 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:20:59.436 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:20:58 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 16867/270570 objects degraded (6.234%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:20:59.448 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:59.530 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:20:59.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:59.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:20:59.646 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:20:59.683 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:20:59.683 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:20:59.734 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:20:59.734 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:20:59.748 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:20:59.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:20:59.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:20:59.856 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:00.083 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:00.361 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:00.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:00.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:00.466 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:00.511 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:00.601 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:00.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:00.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:00.708 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:00.752 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:00.847 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:00.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:00.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:00.957 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:01.026 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:01.055 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:21:01.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:01 smithi138 ceph-mon[193606]: pgmap v886: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 16317/261873 objects degraded (6.231%); 47624/261873 objects misplaced (18.186%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T23:21:01.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:01 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/409069980' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:21:01.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:01 smithi110 ceph-mon[212939]: pgmap v886: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 16317/261873 objects degraded (6.231%); 47624/261873 objects misplaced (18.186%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T23:21:01.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:01 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/409069980' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:21:01.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:01 smithi110 ceph-mon[221102]: pgmap v886: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 16317/261873 objects degraded (6.231%); 47624/261873 objects misplaced (18.186%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T23:21:01.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:01 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/409069980' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:21:01.596 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:01.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:01.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:01.703 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:01.753 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:01.838 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:01.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:01.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:01.947 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:01.993 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:02.073 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:02.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:02.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:02.179 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:02.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:02 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:21:02.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:02 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/409069980' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:21:02.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:02 smithi110 ceph-mon[221102]: osdmap e293: 8 total, 7 up, 7 in 2023-09-23T23:21:02.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:02 smithi110 ceph-mon[221102]: failed to encode map e293 with expected crc 2023-09-23T23:21:02.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:02 smithi110 ceph-mon[221102]: failed to encode map e293 with expected crc 2023-09-23T23:21:02.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:02 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:21:02.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:02 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/409069980' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:21:02.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:02 smithi110 ceph-mon[212939]: osdmap e293: 8 total, 7 up, 7 in 2023-09-23T23:21:02.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:02 smithi110 ceph-mon[212939]: failed to encode map e293 with expected crc 2023-09-23T23:21:02.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:02 smithi110 ceph-mon[212939]: failed to encode map e293 with expected crc 2023-09-23T23:21:02.215 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:02.319 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:02.335 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:02 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:21:02.335 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:02 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/409069980' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:21:02.335 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:02 smithi138 ceph-mon[193606]: osdmap e293: 8 total, 7 up, 7 in 2023-09-23T23:21:02.335 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:02 smithi138 ceph-mon[193606]: failed to encode map e293 with expected crc 2023-09-23T23:21:02.335 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:02 smithi138 ceph-mon[193606]: failed to encode map e293 with expected crc 2023-09-23T23:21:02.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:02.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:02.437 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:02.501 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:02.603 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:02.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:02.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:02.640 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:21:02.709 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:02.778 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:03.396 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:03.400 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:03 smithi110 ceph-mon[221102]: pgmap v888: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 16070/258156 objects degraded (6.225%); 46772/258156 objects misplaced (18.118%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T23:21:03.401 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:03 smithi110 ceph-mon[212939]: pgmap v888: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 16070/258156 objects degraded (6.225%); 46772/258156 objects misplaced (18.118%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T23:21:03.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:03.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:03 smithi138 ceph-mon[193606]: pgmap v888: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 16070/258156 objects degraded (6.225%); 46772/258156 objects misplaced (18.118%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T23:21:03.502 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:21:03.600 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:03.684 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:21:03.714 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:21:03.732 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:21:03.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:03.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:21:03.755 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:21:03.788 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:21:03.817 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:21:03.837 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:03.856 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:21:03.896 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:03.973 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:03.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:03.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:04.022 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:21:04.023 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:21:04.023 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:21:04.023 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:21:04.023 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:21:04.023 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:21:04.023 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:21:04.023 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:21:04.023 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:21:04.041 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:04 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 16070/258156 objects degraded (6.225%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:04.041 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:04 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3657631413' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:04.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:04 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 16070/258156 objects degraded (6.225%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:04.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:04 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3657631413' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:04.078 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:04.122 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:04.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:04 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 16070/258156 objects degraded (6.225%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:04.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:04 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3657631413' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:04.200 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:04.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:04.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:04.308 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:04.390 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:04.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:04.808 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:04.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:04.815 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:21:04.909 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:04.986 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:05.190 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:05 smithi110 ceph-mon[212939]: pgmap v889: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 16070/258156 objects degraded (6.225%); 46772/258156 objects misplaced (18.118%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T23:21:05.190 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:05 smithi110 ceph-mon[221102]: pgmap v889: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 16070/258156 objects degraded (6.225%); 46772/258156 objects misplaced (18.118%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T23:21:05.202 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:05.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:05.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:05.213 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:21:05.213 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:21:05.284 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:05.309 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:05.388 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:05.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:05 smithi138 ceph-mon[193606]: pgmap v889: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 16070/258156 objects degraded (6.225%); 46772/258156 objects misplaced (18.118%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T23:21:05.554 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:05.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:05.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:05.670 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:21:05.757 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:06.037 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:21:06.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:06.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:21:06.146 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:06.194 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:06.284 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:06.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:06.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:06.357 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:21:06 smithi110 conmon[198813]: 2023-09-23T23:21:06.353+0000 7ff0150b6700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T23:21:06.357 INFO:teuthology.orchestra.run.smithi110.stdout:moved 35 / 297 (11.7845%) 2023-09-23T23:21:06.357 INFO:teuthology.orchestra.run.smithi110.stdout:avg 49.5 2023-09-23T23:21:06.357 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 10.5475 -> 13.1498 (expected baseline 6.42262) 2023-09-23T23:21:06.357 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.5 with 27 -> 20 pgs (0.545455 -> 0.40404 * mean) 2023-09-23T23:21:06.357 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.6 with 56 -> 49 pgs (1.13131 -> 0.989899 * mean) 2023-09-23T23:21:06.357 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:21:06.358 INFO:teuthology.orchestra.run.smithi110.stdout:oload 1000 2023-09-23T23:21:06.358 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 3 2023-09-23T23:21:06.358 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:21:06.358 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0289 2023-09-23T23:21:06.358 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.2893 2023-09-23T23:21:06.358 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7 weight 0.6124 -> 1.0000 2023-09-23T23:21:06.358 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 weight 0.7057 -> 1.0000 2023-09-23T23:21:06.358 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5 weight 0.5298 -> 0.6329 2023-09-23T23:21:06.385 INFO:teuthology.orchestra.run.smithi110.stderr:no change 2023-09-23T23:21:06.395 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:06.462 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:06.565 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:06.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:06.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:06.671 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:06.734 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:07.065 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:21:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:21:06] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.43.0" 2023-09-23T23:21:07.214 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:21:07.215 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:07.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:07.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:07.320 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:07.377 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:07 smithi110 ceph-mon[212939]: pgmap v890: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 833 op/s; 15250/245571 objects degraded (6.210%); 44828/245571 objects misplaced (18.255%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:07.378 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:07 smithi110 ceph-mon[221102]: pgmap v890: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 833 op/s; 15250/245571 objects degraded (6.210%); 44828/245571 objects misplaced (18.255%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:07.386 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:07.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:07 smithi138 ceph-mon[193606]: pgmap v890: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 833 op/s; 15250/245571 objects degraded (6.210%); 44828/245571 objects misplaced (18.255%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:07.854 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:07.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:07.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:07.963 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:07.982 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:07.990 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:08.045 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:08.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:08.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:08.151 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:21:08.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:08 smithi110 ceph-mon[212939]: from='client.45674 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:21:08.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:08 smithi110 ceph-mon[221102]: from='client.45674 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:21:08.240 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:08.433 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:21:08.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:08.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:21:08.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:08 smithi138 ceph-mon[193606]: from='client.45674 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:21:08.538 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:21:08.568 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:08.630 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:08.814 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:21:08.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:08.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:21:08.927 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:21:08.998 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:09.176 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:21:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:21:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35529.0:124448 10.a e293/255 10:5bf36135:::benchmark_data_smithi110_366450_object23240:head v 293'7825, mlcod=293'7823)", 2023-09-23T23:21:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:09.166444+0000", 2023-09-23T23:21:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0073940669999999998, 2023-09-23T23:21:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074149699999999999, 2023-09-23T23:21:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:21:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166444+0000", 2023-09-23T23:21:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166444+0000", 2023-09-23T23:21:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166444+0000", 2023-09-23T23:21:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:21:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166457+0000", 2023-09-23T23:21:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2955e-05 2023-09-23T23:21:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166474+0000", 2023-09-23T23:21:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7825e-05 2023-09-23T23:21:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166482+0000", 2023-09-23T23:21:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.379e-06 2023-09-23T23:21:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166528+0000", 2023-09-23T23:21:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6026000000000003e-05 2023-09-23T23:21:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166560+0000", 2023-09-23T23:21:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2112e-05 2023-09-23T23:21:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35529.0:124452 10.0 e293/285 10:02267d74:::benchmark_data_smithi110_366450_object23244:head v 293'7844, mlcod=293'7842)", 2023-09-23T23:21:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:09.166496+0000", 2023-09-23T23:21:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0073426899999999998, 2023-09-23T23:21:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074058780000000003, 2023-09-23T23:21:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:21:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166496+0000", 2023-09-23T23:21:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166496+0000", 2023-09-23T23:21:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166495+0000", 2023-09-23T23:21:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:21:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166500+0000", 2023-09-23T23:21:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5390000000000001e-06 2023-09-23T23:21:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166506+0000", 2023-09-23T23:21:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3890000000000002e-06 2023-09-23T23:21:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166511+0000", 2023-09-23T23:21:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.583e-06 2023-09-23T23:21:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166629+0000", 2023-09-23T23:21:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118026 2023-09-23T23:21:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166648+0000", 2023-09-23T23:21:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9168e-05 2023-09-23T23:21:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35529.0:124455 10.b e293/269 10:d61f4da8:::benchmark_data_smithi110_366450_object23247:head v 293'7835, mlcod=293'7833)", 2023-09-23T23:21:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:09.166516+0000", 2023-09-23T23:21:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0073226389999999997, 2023-09-23T23:21:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074239600000000003, 2023-09-23T23:21:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:21:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166516+0000", 2023-09-23T23:21:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166516+0000", 2023-09-23T23:21:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166515+0000", 2023-09-23T23:21:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-09-23T23:21:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166519+0000", 2023-09-23T23:21:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5499999999999999e-06 2023-09-23T23:21:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166531+0000", 2023-09-23T23:21:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1749999999999999e-05 2023-09-23T23:21:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166548+0000", 2023-09-23T23:21:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6773999999999999e-05 2023-09-23T23:21:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166584+0000", 2023-09-23T23:21:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6875000000000002e-05 2023-09-23T23:21:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.166598+0000", 2023-09-23T23:21:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4039000000000001e-05 2023-09-23T23:21:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:124447 10.5 10:ac78c3de:::benchmark_data_smithi110_366450_object23239:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-09-23T23:21:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:09.167637+0000", 2023-09-23T23:21:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0062013260000000001, 2023-09-23T23:21:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063402930000000003, 2023-09-23T23:21:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:21:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 124447 2023-09-23T23:21:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167637+0000", 2023-09-23T23:21:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167637+0000", 2023-09-23T23:21:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167635+0000", 2023-09-23T23:21:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:21:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167642+0000", 2023-09-23T23:21:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7839999999999998e-06 2023-09-23T23:21:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167658+0000", 2023-09-23T23:21:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5905000000000001e-05 2023-09-23T23:21:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167666+0000", 2023-09-23T23:21:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3629999999999994e-06 2023-09-23T23:21:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167765+0000", 2023-09-23T23:21:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9135000000000005e-05 2023-09-23T23:21:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167924+0000", 2023-09-23T23:21:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000158312 2023-09-23T23:21:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:21:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.168088+0000", 2023-09-23T23:21:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016461199999999999 2023-09-23T23:21:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:124449 10.8 10:1616aeb0:::benchmark_data_smithi110_366450_object23241:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-09-23T23:21:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:09.167678+0000", 2023-09-23T23:21:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0061603250000000003, 2023-09-23T23:21:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063487999999999999, 2023-09-23T23:21:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:21:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 124449 2023-09-23T23:21:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167678+0000", 2023-09-23T23:21:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167678+0000", 2023-09-23T23:21:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167677+0000", 2023-09-23T23:21:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:21:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167681+0000", 2023-09-23T23:21:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1659999999999998e-06 2023-09-23T23:21:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167687+0000", 2023-09-23T23:21:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0769999999999996e-06 2023-09-23T23:21:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167691+0000", 2023-09-23T23:21:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6319999999999999e-06 2023-09-23T23:21:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.171201+0000", 2023-09-23T23:21:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035096939999999998 2023-09-23T23:21:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.171292+0000", 2023-09-23T23:21:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0730999999999996e-05 2023-09-23T23:21:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:21:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.171421+0000", 2023-09-23T23:21:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012932899999999999 2023-09-23T23:21:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:124451 10.5 10:aa10413c:::benchmark_data_smithi110_366450_object23243:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-09-23T23:21:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:09.167699+0000", 2023-09-23T23:21:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0061392390000000003, 2023-09-23T23:21:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063706090000000002, 2023-09-23T23:21:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:21:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 124451 2023-09-23T23:21:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167699+0000", 2023-09-23T23:21:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167699+0000", 2023-09-23T23:21:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167699+0000", 2023-09-23T23:21:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:21:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167701+0000", 2023-09-23T23:21:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2850000000000001e-06 2023-09-23T23:21:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167707+0000", 2023-09-23T23:21:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8130000000000001e-06 2023-09-23T23:21:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167711+0000", 2023-09-23T23:21:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5510000000000001e-06 2023-09-23T23:21:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.170615+0000", 2023-09-23T23:21:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029032110000000002 2023-09-23T23:21:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.170748+0000", 2023-09-23T23:21:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000133208 2023-09-23T23:21:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:21:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.170882+0000", 2023-09-23T23:21:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013435699999999999 2023-09-23T23:21:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:124453 10.5 10:a2b9cbf6:::benchmark_data_smithi110_366450_object23245:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-09-23T23:21:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:09.167716+0000", 2023-09-23T23:21:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0061221260000000003, 2023-09-23T23:21:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063958039999999997, 2023-09-23T23:21:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:21:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 124453 2023-09-23T23:21:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167716+0000", 2023-09-23T23:21:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167716+0000", 2023-09-23T23:21:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167716+0000", 2023-09-23T23:21:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:21:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167718+0000", 2023-09-23T23:21:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.306e-06 2023-09-23T23:21:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167724+0000", 2023-09-23T23:21:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.429e-06 2023-09-23T23:21:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.167728+0000", 2023-09-23T23:21:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2780000000000004e-06 2023-09-23T23:21:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.173133+0000", 2023-09-23T23:21:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054054960000000001 2023-09-23T23:21:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.173262+0000", 2023-09-23T23:21:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012831299999999999 2023-09-23T23:21:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:21:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.173390+0000", 2023-09-23T23:21:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000127933 2023-09-23T23:21:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35529.0:124456 10.6 e293/255 10:6f0dedfa:::benchmark_data_smithi110_366450_object23248:head v 293'7781, mlcod=293'7779)", 2023-09-23T23:21:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:09.168117+0000", 2023-09-23T23:21:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0057211010000000001, 2023-09-23T23:21:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060386249999999997, 2023-09-23T23:21:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:21:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.168117+0000", 2023-09-23T23:21:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.168117+0000", 2023-09-23T23:21:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.168117+0000", 2023-09-23T23:21:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:21:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.168123+0000", 2023-09-23T23:21:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0929999999999999e-06 2023-09-23T23:21:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.168131+0000", 2023-09-23T23:21:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3839999999999997e-06 2023-09-23T23:21:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.168137+0000", 2023-09-23T23:21:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1990000000000003e-06 2023-09-23T23:21:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.169304+0000", 2023-09-23T23:21:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011667489999999999 2023-09-23T23:21:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.169321+0000", 2023-09-23T23:21:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7312e-05 2023-09-23T23:21:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.35529.0:124439 10.5 e293/280)", 2023-09-23T23:21:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:09.172589+0000", 2023-09-23T23:21:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0012492709999999999, 2023-09-23T23:21:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016001170000000001, 2023-09-23T23:21:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:21:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172589+0000", 2023-09-23T23:21:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172589+0000", 2023-09-23T23:21:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172588+0000", 2023-09-23T23:21:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:21:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172593+0000", 2023-09-23T23:21:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.758e-06 2023-09-23T23:21:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172605+0000", 2023-09-23T23:21:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1440999999999999e-05 2023-09-23T23:21:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172610+0000", 2023-09-23T23:21:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.643e-06 2023-09-23T23:21:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.35529.0:124441 10.5 e293/280)", 2023-09-23T23:21:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:09.172620+0000", 2023-09-23T23:21:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0012181749999999999, 2023-09-23T23:21:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001595387, 2023-09-23T23:21:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:21:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172620+0000", 2023-09-23T23:21:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172620+0000", 2023-09-23T23:21:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172620+0000", 2023-09-23T23:21:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-09-23T23:21:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172622+0000", 2023-09-23T23:21:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7370000000000001e-06 2023-09-23T23:21:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172625+0000", 2023-09-23T23:21:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2729999999999998e-06 2023-09-23T23:21:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172627+0000", 2023-09-23T23:21:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5170000000000002e-06 2023-09-23T23:21:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.35529.0:124446 10.5 e293/280)", 2023-09-23T23:21:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:09.172630+0000", 2023-09-23T23:21:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0012086639999999999, 2023-09-23T23:21:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016119870000000001, 2023-09-23T23:21:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:21:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172630+0000", 2023-09-23T23:21:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172630+0000", 2023-09-23T23:21:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172630+0000", 2023-09-23T23:21:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-09-23T23:21:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172631+0000", 2023-09-23T23:21:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2619999999999999e-06 2023-09-23T23:21:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172633+0000", 2023-09-23T23:21:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5560000000000002e-06 2023-09-23T23:21:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.172636+0000", 2023-09-23T23:21:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1900000000000002e-06 2023-09-23T23:21:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.35529.0:124449 10.8 e293/287)", 2023-09-23T23:21:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:09.173980+0000", 2023-09-23T23:21:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 4294967295.9998584, 2023-09-23T23:21:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028809199999999999, 2023-09-23T23:21:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:21:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.173980+0000", 2023-09-23T23:21:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.173980+0000", 2023-09-23T23:21:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.173980+0000", 2023-09-23T23:21:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:21:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.173986+0000", 2023-09-23T23:21:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7129999999999999e-06 2023-09-23T23:21:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.173995+0000", 2023-09-23T23:21:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3059999999999998e-06 2023-09-23T23:21:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.174001+0000", 2023-09-23T23:21:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8639999999999998e-06 2023-09-23T23:21:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:124439 10.5 10:a7427206:::benchmark_data_smithi110_366450_object23231:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-09-23T23:21:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:09.156136+0000", 2023-09-23T23:21:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.017702518, 2023-09-23T23:21:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018162612000000002, 2023-09-23T23:21:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:21:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 124439 2023-09-23T23:21:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156136+0000", 2023-09-23T23:21:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156136+0000", 2023-09-23T23:21:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156135+0000", 2023-09-23T23:21:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:21:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156141+0000", 2023-09-23T23:21:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2380000000000002e-06 2023-09-23T23:21:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156155+0000", 2023-09-23T23:21:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3624e-05 2023-09-23T23:21:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156163+0000", 2023-09-23T23:21:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8480000000000007e-06 2023-09-23T23:21:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156223+0000", 2023-09-23T23:21:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9549999999999997e-05 2023-09-23T23:21:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156411+0000", 2023-09-23T23:21:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018815699999999999 2023-09-23T23:21:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:21:09.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156627+0000", 2023-09-23T23:21:09.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021557400000000001 2023-09-23T23:21:09.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.164470+0000", 2023-09-23T23:21:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007842814 2023-09-23T23:21:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.164503+0000", 2023-09-23T23:21:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3741000000000001e-05 2023-09-23T23:21:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.164589+0000", 2023-09-23T23:21:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5653000000000001e-05 2023-09-23T23:21:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:124441 10.5 10:a8ee4907:::benchmark_data_smithi110_366450_object23233:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-09-23T23:21:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:09.156178+0000", 2023-09-23T23:21:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.017660823999999999, 2023-09-23T23:21:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018178462999999999, 2023-09-23T23:21:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:21:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 124441 2023-09-23T23:21:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156178+0000", 2023-09-23T23:21:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156178+0000", 2023-09-23T23:21:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156177+0000", 2023-09-23T23:21:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:21:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156180+0000", 2023-09-23T23:21:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6850000000000001e-06 2023-09-23T23:21:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156191+0000", 2023-09-23T23:21:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1728e-05 2023-09-23T23:21:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156197+0000", 2023-09-23T23:21:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.587e-06 2023-09-23T23:21:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.159364+0000", 2023-09-23T23:21:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031668730000000002 2023-09-23T23:21:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.159502+0000", 2023-09-23T23:21:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013816999999999999 2023-09-23T23:21:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:21:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.159655+0000", 2023-09-23T23:21:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000153299 2023-09-23T23:21:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.164512+0000", 2023-09-23T23:21:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048562520000000001 2023-09-23T23:21:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.164556+0000", 2023-09-23T23:21:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3983000000000002e-05 2023-09-23T23:21:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:124446 10.5 10:a98fea4f:::benchmark_data_smithi110_366450_object23238:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-09-23T23:21:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:09.156226+0000", 2023-09-23T23:21:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.017612383999999998, 2023-09-23T23:21:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018180053000000002, 2023-09-23T23:21:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:21:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 124446 2023-09-23T23:21:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156226+0000", 2023-09-23T23:21:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156226+0000", 2023-09-23T23:21:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156226+0000", 2023-09-23T23:21:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:21:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156228+0000", 2023-09-23T23:21:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7530000000000002e-06 2023-09-23T23:21:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156245+0000", 2023-09-23T23:21:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6322000000000001e-05 2023-09-23T23:21:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.156251+0000", 2023-09-23T23:21:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2389999999999999e-06 2023-09-23T23:21:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.162121+0000", 2023-09-23T23:21:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058700619999999997 2023-09-23T23:21:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.162257+0000", 2023-09-23T23:21:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013650000000000001 2023-09-23T23:21:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:21:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.162367+0000", 2023-09-23T23:21:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010988399999999999 2023-09-23T23:21:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:09.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.164739+0000", 2023-09-23T23:21:09.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023714019999999999 2023-09-23T23:21:09.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:09.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:09.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:09.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:09.164871+0000", 2023-09-23T23:21:09.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013197799999999999 2023-09-23T23:21:09.246 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.246 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:09.246 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.247 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:09.247 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-09-23T23:21:09.247 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 15 2023-09-23T23:21:09.247 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:21:09.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:09 smithi138 ceph-mon[193606]: pgmap v891: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 833 op/s; 15250/245571 objects degraded (6.210%); 44828/245571 objects misplaced (18.255%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:09.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:09 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 15250/245571 objects degraded (6.210%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:09.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:09 smithi110 ceph-mon[212939]: pgmap v891: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 833 op/s; 15250/245571 objects degraded (6.210%); 44828/245571 objects misplaced (18.255%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:09.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:09 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 15250/245571 objects degraded (6.210%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:09.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:09 smithi110 ceph-mon[221102]: pgmap v891: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 833 op/s; 15250/245571 objects degraded (6.210%); 44828/245571 objects misplaced (18.255%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:09.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:09 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 15250/245571 objects degraded (6.210%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:09.811 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:21:09.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:09.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:21:09.918 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:09.969 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:10.054 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:10.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:10.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:10.162 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:10.225 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:10.308 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:10.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:10.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:10.421 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:10.467 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:10.565 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:10.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:10.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:10.674 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:21:10.763 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:10.902 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:21:10.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:10.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:21:10.976 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:21:11.011 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:11.053 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:11.138 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:11.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:11.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:11.158 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:11 smithi110 ceph-mon[221102]: pgmap v892: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 724 op/s; 15250/240138 objects degraded (6.351%); 44067/240138 objects misplaced (18.351%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:11.160 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:11 smithi110 ceph-mon[212939]: pgmap v892: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 724 op/s; 15250/240138 objects degraded (6.351%); 44067/240138 objects misplaced (18.351%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:11.244 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:11.299 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:11 smithi138 ceph-mon[193606]: pgmap v892: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 724 op/s; 15250/240138 objects degraded (6.351%); 44067/240138 objects misplaced (18.351%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:11.306 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:11.382 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:11.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:11.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:11.490 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:11.516 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:11.574 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:11.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:11.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:11.686 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:11.750 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:11.969 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:11.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:11.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:12.075 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:12.115 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:12.192 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:12.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:12.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:12.302 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:12.361 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:12.477 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:12.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:12.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:12.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:12.701 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:12.734 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:12.807 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:12.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:12.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:12.881 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:21:12.911 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:21:12.924 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:12.959 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:21:12.981 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:13.005 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:21:13.044 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 2] out_osds: [1] dead_osds: [7] live_osds: [0, 1, 3, 4, 5, 6, 2] 2023-09-23T23:21:13.044 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:21:13.044 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T23:21:13.044 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.490234 2023-09-23T23:21:13.044 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 0.49023425359608697 2023-09-23T23:21:13.057 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:21:13.085 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:13.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:13.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:13.117 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:21:13.122 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:21:13.156 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:21:13.165 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:21:13.165 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:21:13.166 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:21:13.166 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:21:13.166 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:21:13.166 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:21:13.166 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:21:13.166 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:21:13.197 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:13.228 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:13 smithi138 ceph-mon[193606]: pgmap v893: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 805 op/s; 14420/232827 objects degraded (6.193%); 42886/232827 objects misplaced (18.420%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T23:21:13.228 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:13 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3996554760' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:21:13.228 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:13 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:21:13.228 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:13 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/308070352' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:13.228 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:21:13.230 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:13.302 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:13.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:13.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:13.419 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:13.429 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:13 smithi110 ceph-mon[212939]: pgmap v893: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 805 op/s; 14420/232827 objects degraded (6.193%); 42886/232827 objects misplaced (18.420%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T23:21:13.429 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:13 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3996554760' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:21:13.429 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:13 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:21:13.429 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:13 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/308070352' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:13.429 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:21:13.430 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:13 smithi110 ceph-mon[221102]: pgmap v893: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 805 op/s; 14420/232827 objects degraded (6.193%); 42886/232827 objects misplaced (18.420%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T23:21:13.430 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:13 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3996554760' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:21:13.430 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:13 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:21:13.430 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:13 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/308070352' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:13.430 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:21:13.468 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:13.541 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:13.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:13.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:13.647 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:13.704 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:14.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:14 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:21:14.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:14 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:21:14.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:14 smithi138 ceph-mon[193606]: osdmap e294: 8 total, 7 up, 7 in 2023-09-23T23:21:14.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:14 smithi138 ceph-mon[193606]: failed to encode map e294 with expected crc 2023-09-23T23:21:14.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:14 smithi138 ceph-mon[193606]: failed to encode map e294 with expected crc 2023-09-23T23:21:14.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:14 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 14420/232827 objects degraded (6.193%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:14.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:14 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:21:14.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:14 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:21:14.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:14 smithi110 ceph-mon[212939]: osdmap e294: 8 total, 7 up, 7 in 2023-09-23T23:21:14.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:14 smithi110 ceph-mon[212939]: failed to encode map e294 with expected crc 2023-09-23T23:21:14.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:14 smithi110 ceph-mon[212939]: failed to encode map e294 with expected crc 2023-09-23T23:21:14.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:14 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 14420/232827 objects degraded (6.193%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:14.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:14 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:21:14.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:14 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:21:14.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:14 smithi110 ceph-mon[221102]: osdmap e294: 8 total, 7 up, 7 in 2023-09-23T23:21:14.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:14 smithi110 ceph-mon[221102]: failed to encode map e294 with expected crc 2023-09-23T23:21:14.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:14 smithi110 ceph-mon[221102]: failed to encode map e294 with expected crc 2023-09-23T23:21:14.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:14 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 14420/232827 objects degraded (6.193%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:15.121 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:15 smithi110 ceph-mon[212939]: pgmap v895: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 844 op/s; 14420/232827 objects degraded (6.193%); 42886/232827 objects misplaced (18.420%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:21:15.171 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:15.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:15.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:15.278 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:15.307 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:15.386 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:15.403 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:15 smithi138 ceph-mon[193606]: pgmap v895: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 844 op/s; 14420/232827 objects degraded (6.193%); 42886/232827 objects misplaced (18.420%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:21:15.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:15.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:15.447 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:15 smithi110 ceph-mon[221102]: pgmap v895: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 844 op/s; 14420/232827 objects degraded (6.193%); 42886/232827 objects misplaced (18.420%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:21:15.505 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:15.544 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:15.620 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:15.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:15.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:15.733 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:15.787 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:15.890 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:15.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:15.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:15.998 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:16.071 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:16.198 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:21:16.199 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:21:16.300 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:16.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:16.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:16.413 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:16.475 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:16.544 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:16.559 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:16.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:16.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:16.668 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:16.726 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:16.858 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:21:16.859 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:21:16.984 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:21:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:21:16] "GET /metrics HTTP/1.1" 200 36632 "" "Prometheus/2.43.0" 2023-09-23T23:21:17.014 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:17.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:17.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:17.122 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:17.190 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:17.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:17 smithi138 ceph-mon[193606]: pgmap v896: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 845 op/s; 13623/220185 objects degraded (6.187%); 40943/220185 objects misplaced (18.595%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:17.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:17 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:21:17.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:17 smithi110 ceph-mon[221102]: pgmap v896: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 845 op/s; 13623/220185 objects degraded (6.187%); 40943/220185 objects misplaced (18.595%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:17.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:17 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:21:17.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:17 smithi110 ceph-mon[212939]: pgmap v896: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 845 op/s; 13623/220185 objects degraded (6.187%); 40943/220185 objects misplaced (18.595%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:17.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:17 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:21:17.470 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:21:17.492 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:17.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:17.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:17.596 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:17.658 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:17.769 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:17.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:17.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:17.879 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:17.935 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:18.072 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:18.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:18.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:18.177 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:18.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:18 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:21:18.200 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:21:18.246 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:18.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:18 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:21:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:18 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:21:18.726 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:21:18.854 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:18.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:18.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:18.966 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:19.105 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:19.152 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:19.184 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:19.249 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:19.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:19.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:19.260 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:19 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:21:19.261 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:19 smithi110 ceph-mon[212939]: pgmap v897: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 845 op/s; 13623/220185 objects degraded (6.187%); 40943/220185 objects misplaced (18.595%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:19.261 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:19 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 13623/220185 objects degraded (6.187%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:19.261 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:19 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:21:19.261 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:19 smithi110 ceph-mon[221102]: pgmap v897: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 845 op/s; 13623/220185 objects degraded (6.187%); 40943/220185 objects misplaced (18.595%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:19.261 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:19 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 13623/220185 objects degraded (6.187%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:19.360 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:19.400 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:19.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:19 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:21:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:19 smithi138 ceph-mon[193606]: pgmap v897: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 845 op/s; 13623/220185 objects degraded (6.187%); 40943/220185 objects misplaced (18.595%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:19 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 13623/220185 objects degraded (6.187%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:19.562 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:19.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:19.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:19.666 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:21:19.730 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:20.247 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:21:20.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:20.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:21:20.354 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:21:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:20 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:21:20.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:20 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:21:20.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:20 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:21:20.486 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:20.639 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:21:20.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:20.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:21:20.747 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:20.816 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:20.918 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:20.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:20.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:21.022 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:21.100 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:21.209 INFO:teuthology.orchestra.run.smithi110.stderr:set osd.5 primary-affinity to 0.490234 (8321272) 2023-09-23T23:21:21.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:21 smithi110 ceph-mon[212939]: pgmap v898: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 844 op/s; 13623/214761 objects degraded (6.343%); 40180/214761 objects misplaced (18.709%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:21.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:21 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:21:21.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:21 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3726073010' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.49023425359608697}]: dispatch 2023-09-23T23:21:21.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:21 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.49023425359608697}]: dispatch 2023-09-23T23:21:21.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:21 smithi110 ceph-mon[221102]: pgmap v898: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 844 op/s; 13623/214761 objects degraded (6.343%); 40180/214761 objects misplaced (18.709%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:21.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:21 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:21:21.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:21 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3726073010' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.49023425359608697}]: dispatch 2023-09-23T23:21:21.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:21 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.49023425359608697}]: dispatch 2023-09-23T23:21:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:21 smithi138 ceph-mon[193606]: pgmap v898: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 844 op/s; 13623/214761 objects degraded (6.343%); 40180/214761 objects misplaced (18.709%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:21:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:21 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:21:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:21 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3726073010' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.49023425359608697}]: dispatch 2023-09-23T23:21:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:21 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.49023425359608697}]: dispatch 2023-09-23T23:21:22.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:22 smithi110 ceph-mon[212939]: failed to encode map e295 with expected crc 2023-09-23T23:21:22.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:22 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.49023425359608697}]': finished 2023-09-23T23:21:22.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:22 smithi110 ceph-mon[212939]: osdmap e295: 8 total, 7 up, 7 in 2023-09-23T23:21:22.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:22 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:22.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:22 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:22.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:22 smithi110 ceph-mon[212939]: pgmap v900: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 981 op/s; 12888/208050 objects degraded (6.195%); 39031/208050 objects misplaced (18.760%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T23:21:22.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:22 smithi110 ceph-mon[212939]: osdmap e296: 8 total, 7 up, 7 in 2023-09-23T23:21:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:22 smithi110 ceph-mon[221102]: failed to encode map e295 with expected crc 2023-09-23T23:21:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:22 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.49023425359608697}]': finished 2023-09-23T23:21:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:22 smithi110 ceph-mon[221102]: osdmap e295: 8 total, 7 up, 7 in 2023-09-23T23:21:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:22 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:22 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:22 smithi110 ceph-mon[221102]: pgmap v900: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 981 op/s; 12888/208050 objects degraded (6.195%); 39031/208050 objects misplaced (18.760%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T23:21:22.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:22 smithi110 ceph-mon[221102]: osdmap e296: 8 total, 7 up, 7 in 2023-09-23T23:21:22.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:22 smithi138 ceph-mon[193606]: failed to encode map e295 with expected crc 2023-09-23T23:21:22.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:22 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.49023425359608697}]': finished 2023-09-23T23:21:22.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:22 smithi138 ceph-mon[193606]: osdmap e295: 8 total, 7 up, 7 in 2023-09-23T23:21:22.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:22 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:22.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:22 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:22.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:22 smithi138 ceph-mon[193606]: pgmap v900: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 981 op/s; 12888/208050 objects degraded (6.195%); 39031/208050 objects misplaced (18.760%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T23:21:22.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:22 smithi138 ceph-mon[193606]: osdmap e296: 8 total, 7 up, 7 in 2023-09-23T23:21:22.884 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:22.954 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:22.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:22.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:23.064 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:21:23.173 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:23.438 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:23 smithi110 ceph-mon[212939]: failed to encode map e296 with expected crc 2023-09-23T23:21:23.438 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:23 smithi110 ceph-mon[212939]: failed to encode map e296 with expected crc 2023-09-23T23:21:23.438 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:23 smithi110 ceph-mon[212939]: failed to encode map e296 with expected crc 2023-09-23T23:21:23.438 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:23 smithi110 ceph-mon[221102]: failed to encode map e296 with expected crc 2023-09-23T23:21:23.438 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:23 smithi110 ceph-mon[221102]: failed to encode map e296 with expected crc 2023-09-23T23:21:23.439 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:23 smithi110 ceph-mon[221102]: failed to encode map e296 with expected crc 2023-09-23T23:21:23.451 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:21:23.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:23.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:21:23.558 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:23.595 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:23 smithi138 ceph-mon[193606]: failed to encode map e296 with expected crc 2023-09-23T23:21:23.595 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:23 smithi138 ceph-mon[193606]: failed to encode map e296 with expected crc 2023-09-23T23:21:23.595 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:23 smithi138 ceph-mon[193606]: failed to encode map e296 with expected crc 2023-09-23T23:21:23.606 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:23.668 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:23.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:23.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:23.779 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:23.874 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:24.014 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:21:24.058 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:21:24.106 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:21:24.160 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:21:24.175 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:24.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:24.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:24.198 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:21:24.229 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:21:24.277 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:21:24.282 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:21:24.282 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:21:24.283 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:21:24.283 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:21:24.283 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:21:24.283 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:21:24.283 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:21:24.283 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:21:24.296 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:24.350 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:24.430 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:21:24.446 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:24.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:24.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:24.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:24 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 12888/208050 objects degraded (6.195%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:24.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:24 smithi110 ceph-mon[212939]: pgmap v902: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 12888/208050 objects degraded (6.195%); 39031/208050 objects misplaced (18.760%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T23:21:24.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:24 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3562989296' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:24.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:24 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2213295367' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:21:24.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:24 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:21:24.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:24 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 12888/208050 objects degraded (6.195%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:24.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:24 smithi110 ceph-mon[221102]: pgmap v902: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 12888/208050 objects degraded (6.195%); 39031/208050 objects misplaced (18.760%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T23:21:24.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:24 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3562989296' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:24.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:24 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2213295367' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:21:24.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:24 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:21:24.551 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:21:24.610 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:24.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:24 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 12888/208050 objects degraded (6.195%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:24 smithi138 ceph-mon[193606]: pgmap v902: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 12888/208050 objects degraded (6.195%); 39031/208050 objects misplaced (18.760%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T23:21:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:24 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3562989296' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:24 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2213295367' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:21:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:24 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:21:25.394 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:21:25.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:25.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:21:25.500 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:25.526 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:25.566 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:25.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:25.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:25.674 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:21:25.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:25 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:21:25.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:25 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:21:25.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:25 smithi138 ceph-mon[193606]: osdmap e297: 8 total, 7 up, 7 in 2023-09-23T23:21:25.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:25 smithi138 ceph-mon[193606]: failed to encode map e297 with expected crc 2023-09-23T23:21:25.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:25 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:21:25.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:25 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:21:25.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:25 smithi110 ceph-mon[212939]: osdmap e297: 8 total, 7 up, 7 in 2023-09-23T23:21:25.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:25 smithi110 ceph-mon[212939]: failed to encode map e297 with expected crc 2023-09-23T23:21:25.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:25 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:21:25.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:25 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:21:25.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:25 smithi110 ceph-mon[221102]: osdmap e297: 8 total, 7 up, 7 in 2023-09-23T23:21:25.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:25 smithi110 ceph-mon[221102]: failed to encode map e297 with expected crc 2023-09-23T23:21:25.737 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:26.573 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:21:26.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:26.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:21:26.657 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:26 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:21:26.657 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:26 smithi110 ceph-mon[221102]: pgmap v904: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 957 op/s; 12279/197712 objects degraded (6.211%); 37323/197712 objects misplaced (18.877%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T23:21:26.658 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:26 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:21:26.658 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:26 smithi110 ceph-mon[212939]: pgmap v904: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 957 op/s; 12279/197712 objects degraded (6.211%); 37323/197712 objects misplaced (18.877%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T23:21:26.681 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:21:26.682 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:21:26.683 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:21:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:26 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:21:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:26 smithi138 ceph-mon[193606]: pgmap v904: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 957 op/s; 12279/197712 objects degraded (6.211%); 37323/197712 objects misplaced (18.877%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T23:21:26.794 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:26.953 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:21:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:21:26] "GET /metrics HTTP/1.1" 200 36632 "" "Prometheus/2.43.0" 2023-09-23T23:21:27.084 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:21:27.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:27.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:21:27.194 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:21:27.269 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:27.347 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:21:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:21:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:21:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:21:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:127485 9.5 9:a9ed1e17:::benchmark_data_smithi110_295904_object16304:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:21:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:15:27.556380+0000", 2023-09-23T23:21:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 359.78492689900003, 2023-09-23T23:21:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2403889330000002, 2023-09-23T23:21:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 127485 2023-09-23T23:21:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.556380+0000", 2023-09-23T23:21:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.556380+0000", 2023-09-23T23:21:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.556380+0000", 2023-09-23T23:21:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:21:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.556383+0000", 2023-09-23T23:21:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8499999999999998e-06 2023-09-23T23:21:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.556388+0000", 2023-09-23T23:21:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8270000000000003e-06 2023-09-23T23:21:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.556394+0000", 2023-09-23T23:21:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3430000000000003e-06 2023-09-23T23:21:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.558560+0000", 2023-09-23T23:21:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021662249999999999 2023-09-23T23:21:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.558572+0000", 2023-09-23T23:21:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1765e-05 2023-09-23T23:21:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:28.218551+0000", 2023-09-23T23:21:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.65997934300000005 2023-09-23T23:21:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:28.218559+0000", 2023-09-23T23:21:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4279999999999996e-06 2023-09-23T23:21:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.659043+0000", 2023-09-23T23:21:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.440483993 2023-09-23T23:21:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:21:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.663812+0000", 2023-09-23T23:21:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.004768652 2023-09-23T23:21:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.688195+0000", 2023-09-23T23:21:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024382404 2023-09-23T23:21:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.688307+0000", 2023-09-23T23:21:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011260399999999999 2023-09-23T23:21:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:21:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.688451+0000", 2023-09-23T23:21:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014346199999999999 2023-09-23T23:21:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.712991+0000", 2023-09-23T23:21:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024540859000000002 2023-09-23T23:21:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.732323+0000", 2023-09-23T23:21:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019331596 2023-09-23T23:21:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.792003+0000", 2023-09-23T23:21:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.059679667999999998 2023-09-23T23:21:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.792062+0000", 2023-09-23T23:21:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9694999999999997e-05 2023-09-23T23:21:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.796769+0000", 2023-09-23T23:21:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.004706666 2023-09-23T23:21:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:127494 9.5 9:aafc87b8:::benchmark_data_smithi110_295904_object16313:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:21:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:15:27.562760+0000", 2023-09-23T23:21:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 359.778547355, 2023-09-23T23:21:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2389521440000002, 2023-09-23T23:21:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 127494 2023-09-23T23:21:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.562760+0000", 2023-09-23T23:21:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.562760+0000", 2023-09-23T23:21:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.562758+0000", 2023-09-23T23:21:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:21:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.562765+0000", 2023-09-23T23:21:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4529999999999996e-06 2023-09-23T23:21:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.562781+0000", 2023-09-23T23:21:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.607e-05 2023-09-23T23:21:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.562790+0000", 2023-09-23T23:21:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7499999999999992e-06 2023-09-23T23:21:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.562851+0000", 2023-09-23T23:21:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1179999999999994e-05 2023-09-23T23:21:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.562863+0000", 2023-09-23T23:21:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2089999999999999e-05 2023-09-23T23:21:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:28.218579+0000", 2023-09-23T23:21:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.65571592000000001 2023-09-23T23:21:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:28.218586+0000", 2023-09-23T23:21:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5250000000000002e-06 2023-09-23T23:21:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.668845+0000", 2023-09-23T23:21:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4502595409999999 2023-09-23T23:21:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:21:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.669390+0000", 2023-09-23T23:21:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00054490400000000003 2023-09-23T23:21:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.727901+0000", 2023-09-23T23:21:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.058511191999999997 2023-09-23T23:21:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.728021+0000", 2023-09-23T23:21:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000119465 2023-09-23T23:21:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:21:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.728195+0000", 2023-09-23T23:21:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017422499999999999 2023-09-23T23:21:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.751453+0000", 2023-09-23T23:21:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023257554999999999 2023-09-23T23:21:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.796981+0000", 2023-09-23T23:21:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.045528668000000001 2023-09-23T23:21:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.797048+0000", 2023-09-23T23:21:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6575000000000005e-05 2023-09-23T23:21:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.797105+0000", 2023-09-23T23:21:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6721000000000001e-05 2023-09-23T23:21:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.801712+0000", 2023-09-23T23:21:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046071330000000002 2023-09-23T23:21:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:127502 9.8 9:13d7207e:::benchmark_data_smithi110_295904_object16321:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:21:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:15:27.571754+0000", 2023-09-23T23:21:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 359.76955331800002, 2023-09-23T23:21:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8011237840000001, 2023-09-23T23:21:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 127502 2023-09-23T23:21:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.571754+0000", 2023-09-23T23:21:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.571754+0000", 2023-09-23T23:21:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.571752+0000", 2023-09-23T23:21:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:21:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.571759+0000", 2023-09-23T23:21:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8120000000000002e-06 2023-09-23T23:21:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.571775+0000", 2023-09-23T23:21:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5722e-05 2023-09-23T23:21:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.571782+0000", 2023-09-23T23:21:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8499999999999994e-06 2023-09-23T23:21:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.571843+0000", 2023-09-23T23:21:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0736999999999998e-05 2023-09-23T23:21:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.571852+0000", 2023-09-23T23:21:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2699999999999993e-06 2023-09-23T23:21:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:28.217915+0000", 2023-09-23T23:21:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.64606270899999996 2023-09-23T23:21:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:28.217928+0000", 2023-09-23T23:21:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2928999999999999e-05 2023-09-23T23:21:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.348743+0000", 2023-09-23T23:21:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.130814505 2023-09-23T23:21:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.351342+0000", 2023-09-23T23:21:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025997720000000002 2023-09-23T23:21:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T23:21:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.351435+0000", 2023-09-23T23:21:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2293999999999994e-05 2023-09-23T23:21:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.355150+0000", 2023-09-23T23:21:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037150350000000002 2023-09-23T23:21:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.366406+0000", 2023-09-23T23:21:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011256281 2023-09-23T23:21:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.372679+0000", 2023-09-23T23:21:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0062734499999999999 2023-09-23T23:21:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.372726+0000", 2023-09-23T23:21:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6820000000000002e-05 2023-09-23T23:21:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.372777+0000", 2023-09-23T23:21:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1306e-05 2023-09-23T23:21:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.372877+0000", 2023-09-23T23:21:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9990999999999999e-05 2023-09-23T23:21:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:127516 9.8 9:191f49d7:::benchmark_data_smithi110_295904_object16335:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-09-23T23:21:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:15:27.587621+0000", 2023-09-23T23:21:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 359.75368595100002, 2023-09-23T23:21:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7853615619999998, 2023-09-23T23:21:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 127516 2023-09-23T23:21:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.587621+0000", 2023-09-23T23:21:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.587621+0000", 2023-09-23T23:21:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.587619+0000", 2023-09-23T23:21:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:21:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.587629+0000", 2023-09-23T23:21:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5219999999999996e-06 2023-09-23T23:21:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.587653+0000", 2023-09-23T23:21:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3991000000000001e-05 2023-09-23T23:21:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.587663+0000", 2023-09-23T23:21:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.961e-06 2023-09-23T23:21:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.587732+0000", 2023-09-23T23:21:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9380000000000003e-05 2023-09-23T23:21:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:27.587749+0000", 2023-09-23T23:21:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6534000000000001e-05 2023-09-23T23:21:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:28.218149+0000", 2023-09-23T23:21:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.630400191 2023-09-23T23:21:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:28.218159+0000", 2023-09-23T23:21:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0010000000000001e-05 2023-09-23T23:21:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.355166+0000", 2023-09-23T23:21:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.137007042 2023-09-23T23:21:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.355256+0000", 2023-09-23T23:21:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0274999999999999e-05 2023-09-23T23:21:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T23:21:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.355337+0000", 2023-09-23T23:21:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1101999999999997e-05 2023-09-23T23:21:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.366340+0000", 2023-09-23T23:21:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011002803 2023-09-23T23:21:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.372670+0000", 2023-09-23T23:21:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063301260000000002 2023-09-23T23:21:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.372685+0000", 2023-09-23T23:21:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4915e-05 2023-09-23T23:21:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.372903+0000", 2023-09-23T23:21:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021804999999999999 2023-09-23T23:21:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.372928+0000", 2023-09-23T23:21:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4422000000000001e-05 2023-09-23T23:21:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:15:29.372983+0000", 2023-09-23T23:21:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5074000000000002e-05 2023-09-23T23:21:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188491 9.8 9:1de3f163:::benchmark_data_smithi110_295904_object77310:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2023-09-23T23:21:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.075734+0000", 2023-09-23T23:21:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 234.26557260999999, 2023-09-23T23:21:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.34710358600000002, 2023-09-23T23:21:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188491 2023-09-23T23:21:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.075734+0000", 2023-09-23T23:21:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.075734+0000", 2023-09-23T23:21:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.075732+0000", 2023-09-23T23:21:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:21:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.075743+0000", 2023-09-23T23:21:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0647e-05 2023-09-23T23:21:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.075779+0000", 2023-09-23T23:21:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6322999999999998e-05 2023-09-23T23:21:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.075791+0000", 2023-09-23T23:21:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1256e-05 2023-09-23T23:21:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.075846+0000", 2023-09-23T23:21:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5170999999999997e-05 2023-09-23T23:21:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.076007+0000", 2023-09-23T23:21:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000161022 2023-09-23T23:21:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T23:21:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.076148+0000", 2023-09-23T23:21:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014125100000000001 2023-09-23T23:21:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.080661+0000", 2023-09-23T23:21:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045133329999999996 2023-09-23T23:21:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.087289+0000", 2023-09-23T23:21:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0066279420000000004 2023-09-23T23:21:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.092632+0000", 2023-09-23T23:21:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053428199999999999 2023-09-23T23:21:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.422667+0000", 2023-09-23T23:21:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.33003437200000002 2023-09-23T23:21:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.422717+0000", 2023-09-23T23:21:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9993000000000001e-05 2023-09-23T23:21:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.422838+0000", 2023-09-23T23:21:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000121439 2023-09-23T23:21:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188497 9.8 9:1c4c4f68:::benchmark_data_smithi110_295904_object77316:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2023-09-23T23:21:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.099409+0000", 2023-09-23T23:21:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 234.241898507, 2023-09-23T23:21:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.332170831, 2023-09-23T23:21:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188497 2023-09-23T23:21:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.099409+0000", 2023-09-23T23:21:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.099409+0000", 2023-09-23T23:21:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.099407+0000", 2023-09-23T23:21:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:21:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.099415+0000", 2023-09-23T23:21:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5399999999999998e-06 2023-09-23T23:21:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.099430+0000", 2023-09-23T23:21:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5509e-05 2023-09-23T23:21:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.099440+0000", 2023-09-23T23:21:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.73e-06 2023-09-23T23:21:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.099513+0000", 2023-09-23T23:21:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3153999999999998e-05 2023-09-23T23:21:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.099693+0000", 2023-09-23T23:21:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017979000000000001 2023-09-23T23:21:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T23:21:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.099861+0000", 2023-09-23T23:21:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000168536 2023-09-23T23:21:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.110117+0000", 2023-09-23T23:21:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010256026 2023-09-23T23:21:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.112116+0000", 2023-09-23T23:21:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019982250000000002 2023-09-23T23:21:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.424595+0000", 2023-09-23T23:21:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.31247914799999998 2023-09-23T23:21:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.431377+0000", 2023-09-23T23:21:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067819209999999998 2023-09-23T23:21:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.431432+0000", 2023-09-23T23:21:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.524e-05 2023-09-23T23:21:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.431579+0000", 2023-09-23T23:21:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014750000000000001 2023-09-23T23:21:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188502 9.5 9:a9629de3:::benchmark_data_smithi110_295904_object77321:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-09-23T23:21:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.113855+0000", 2023-09-23T23:21:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 234.22745179500001, 2023-09-23T23:21:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.33515815900000001, 2023-09-23T23:21:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188502 2023-09-23T23:21:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.113855+0000", 2023-09-23T23:21:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.113855+0000", 2023-09-23T23:21:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.113853+0000", 2023-09-23T23:21:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:21:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.113863+0000", 2023-09-23T23:21:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3759999999999997e-06 2023-09-23T23:21:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.113879+0000", 2023-09-23T23:21:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6286999999999998e-05 2023-09-23T23:21:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.113890+0000", 2023-09-23T23:21:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0752e-05 2023-09-23T23:21:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.120983+0000", 2023-09-23T23:21:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007093635 2023-09-23T23:21:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.121099+0000", 2023-09-23T23:21:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000115328 2023-09-23T23:21:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:21:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.121255+0000", 2023-09-23T23:21:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015590299999999999 2023-09-23T23:21:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.125015+0000", 2023-09-23T23:21:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037603279999999998 2023-09-23T23:21:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.439283+0000", 2023-09-23T23:21:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.31426850699999997 2023-09-23T23:21:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.447334+0000", 2023-09-23T23:21:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080506099999999997 2023-09-23T23:21:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.447368+0000", 2023-09-23T23:21:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4369000000000002e-05 2023-09-23T23:21:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.449013+0000", 2023-09-23T23:21:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001645007 2023-09-23T23:21:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:92235 10.9 10:9dad52bb:::benchmark_data_smithi110_366450_object92234:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:21:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:20:25.861770+0000", 2023-09-23T23:21:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 61.479536805999999, 2023-09-23T23:21:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99164348099999999, 2023-09-23T23:21:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 92235 2023-09-23T23:21:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.861770+0000", 2023-09-23T23:21:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.861770+0000", 2023-09-23T23:21:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.861770+0000", 2023-09-23T23:21:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:21:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.861790+0000", 2023-09-23T23:21:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.056e-05 2023-09-23T23:21:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.861797+0000", 2023-09-23T23:21:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7039999999999997e-06 2023-09-23T23:21:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.861802+0000", 2023-09-23T23:21:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8149999999999996e-06 2023-09-23T23:21:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.861905+0000", 2023-09-23T23:21:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102859 2023-09-23T23:21:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.861913+0000", 2023-09-23T23:21:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6419999999999996e-06 2023-09-23T23:21:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.849646+0000", 2023-09-23T23:21:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98773304799999995 2023-09-23T23:21:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.849824+0000", 2023-09-23T23:21:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017723899999999999 2023-09-23T23:21:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:21:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.850014+0000", 2023-09-23T23:21:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019071499999999999 2023-09-23T23:21:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853198+0000", 2023-09-23T23:21:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031833450000000002 2023-09-23T23:21:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853224+0000", 2023-09-23T23:21:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6503e-05 2023-09-23T23:21:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853231+0000", 2023-09-23T23:21:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9870000000000002e-06 2023-09-23T23:21:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853269+0000", 2023-09-23T23:21:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7620999999999998e-05 2023-09-23T23:21:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853308+0000", 2023-09-23T23:21:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9610999999999997e-05 2023-09-23T23:21:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853414+0000", 2023-09-23T23:21:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105293 2023-09-23T23:21:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:92276 10.9 10:967c7a97:::benchmark_data_smithi110_366450_object92275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:21:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:20:25.911070+0000", 2023-09-23T23:21:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 61.430236999000002, 2023-09-23T23:21:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94893306799999999, 2023-09-23T23:21:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 92276 2023-09-23T23:21:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911070+0000", 2023-09-23T23:21:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911070+0000", 2023-09-23T23:21:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911068+0000", 2023-09-23T23:21:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:21:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911093+0000", 2023-09-23T23:21:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4618999999999999e-05 2023-09-23T23:21:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911106+0000", 2023-09-23T23:21:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.271e-05 2023-09-23T23:21:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911112+0000", 2023-09-23T23:21:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8019999999999997e-06 2023-09-23T23:21:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911148+0000", 2023-09-23T23:21:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6596e-05 2023-09-23T23:21:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911158+0000", 2023-09-23T23:21:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6989999999999998e-06 2023-09-23T23:21:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853463+0000", 2023-09-23T23:21:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94230513900000001 2023-09-23T23:21:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853641+0000", 2023-09-23T23:21:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000178438 2023-09-23T23:21:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:21:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853829+0000", 2023-09-23T23:21:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018803099999999999 2023-09-23T23:21:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.859666+0000", 2023-09-23T23:21:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058363540000000002 2023-09-23T23:21:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.859695+0000", 2023-09-23T23:21:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8878e-05 2023-09-23T23:21:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.859743+0000", 2023-09-23T23:21:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8288000000000003e-05 2023-09-23T23:21:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.859809+0000", 2023-09-23T23:21:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5925000000000003e-05 2023-09-23T23:21:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.859876+0000", 2023-09-23T23:21:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7123000000000001e-05 2023-09-23T23:21:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.860003+0000", 2023-09-23T23:21:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000127057 2023-09-23T23:21:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:92277 10.9 10:98a3a044:::benchmark_data_smithi110_366450_object92276:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:21:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:20:25.911200+0000", 2023-09-23T23:21:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 61.430106610000003, 2023-09-23T23:21:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94902298299999999, 2023-09-23T23:21:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 92277 2023-09-23T23:21:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911200+0000", 2023-09-23T23:21:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911200+0000", 2023-09-23T23:21:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911200+0000", 2023-09-23T23:21:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:21:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911217+0000", 2023-09-23T23:21:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.77e-05 2023-09-23T23:21:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911227+0000", 2023-09-23T23:21:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9550000000000004e-06 2023-09-23T23:21:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911232+0000", 2023-09-23T23:21:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6099999999999999e-06 2023-09-23T23:21:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.424 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.424 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911257+0000", 2023-09-23T23:21:27.424 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5199e-05 2023-09-23T23:21:27.424 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.424 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.424 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.424 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911262+0000", 2023-09-23T23:21:27.424 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7820000000000001e-06 2023-09-23T23:21:27.424 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.424 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.424 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.424 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.856640+0000", 2023-09-23T23:21:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94537845200000004 2023-09-23T23:21:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.856798+0000", 2023-09-23T23:21:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000157787 2023-09-23T23:21:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:21:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.856914+0000", 2023-09-23T23:21:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011568700000000001 2023-09-23T23:21:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.859731+0000", 2023-09-23T23:21:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028171210000000001 2023-09-23T23:21:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.859764+0000", 2023-09-23T23:21:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2990000000000001e-05 2023-09-23T23:21:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.860018+0000", 2023-09-23T23:21:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025431299999999999 2023-09-23T23:21:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.860076+0000", 2023-09-23T23:21:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8088999999999999e-05 2023-09-23T23:21:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.860114+0000", 2023-09-23T23:21:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7311000000000002e-05 2023-09-23T23:21:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.860223+0000", 2023-09-23T23:21:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109764 2023-09-23T23:21:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:92309 10.9 10:933f833e:::benchmark_data_smithi110_366450_object92308:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:21:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:20:25.956927+0000", 2023-09-23T23:21:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 61.384380221999997, 2023-09-23T23:21:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.90963400599999999, 2023-09-23T23:21:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 92309 2023-09-23T23:21:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.956927+0000", 2023-09-23T23:21:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.956927+0000", 2023-09-23T23:21:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.956926+0000", 2023-09-23T23:21:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:21:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.956948+0000", 2023-09-23T23:21:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2098000000000001e-05 2023-09-23T23:21:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.956958+0000", 2023-09-23T23:21:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0258000000000001e-05 2023-09-23T23:21:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.956965+0000", 2023-09-23T23:21:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9059999999999996e-06 2023-09-23T23:21:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.957037+0000", 2023-09-23T23:21:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2584999999999997e-05 2023-09-23T23:21:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.957048+0000", 2023-09-23T23:21:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0543999999999999e-05 2023-09-23T23:21:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.860278+0000", 2023-09-23T23:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.90323057299999998 2023-09-23T23:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.860456+0000", 2023-09-23T23:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000178013 2023-09-23T23:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.860661+0000", 2023-09-23T23:21:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000204515 2023-09-23T23:21:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.866282+0000", 2023-09-23T23:21:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056214480000000002 2023-09-23T23:21:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.866293+0000", 2023-09-23T23:21:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0771000000000001e-05 2023-09-23T23:21:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.866333+0000", 2023-09-23T23:21:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9311999999999999e-05 2023-09-23T23:21:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.866369+0000", 2023-09-23T23:21:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6155999999999997e-05 2023-09-23T23:21:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.866428+0000", 2023-09-23T23:21:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9670000000000003e-05 2023-09-23T23:21:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.866561+0000", 2023-09-23T23:21:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013244999999999999 2023-09-23T23:21:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:92311 10.9 10:9da1d2b5:::benchmark_data_smithi110_366450_object92310:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:21:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:20:25.957309+0000", 2023-09-23T23:21:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 61.383997721, 2023-09-23T23:21:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.91510362499999998, 2023-09-23T23:21:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 92311 2023-09-23T23:21:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.957309+0000", 2023-09-23T23:21:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.957309+0000", 2023-09-23T23:21:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.957308+0000", 2023-09-23T23:21:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:21:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.957341+0000", 2023-09-23T23:21:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2684000000000001e-05 2023-09-23T23:21:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.957351+0000", 2023-09-23T23:21:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0169999999999999e-05 2023-09-23T23:21:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.957358+0000", 2023-09-23T23:21:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7730000000000002e-06 2023-09-23T23:21:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.959492+0000", 2023-09-23T23:21:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021336390000000001 2023-09-23T23:21:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.959502+0000", 2023-09-23T23:21:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0040999999999999e-05 2023-09-23T23:21:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.863331+0000", 2023-09-23T23:21:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.90382913499999995 2023-09-23T23:21:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.863487+0000", 2023-09-23T23:21:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015611800000000001 2023-09-23T23:21:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:21:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.863663+0000", 2023-09-23T23:21:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017590300000000001 2023-09-23T23:21:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.866602+0000", 2023-09-23T23:21:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029394270000000001 2023-09-23T23:21:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.866610+0000", 2023-09-23T23:21:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8210000000000002e-06 2023-09-23T23:21:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.866646+0000", 2023-09-23T23:21:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5716999999999997e-05 2023-09-23T23:21:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.872240+0000", 2023-09-23T23:21:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055940190000000004 2023-09-23T23:21:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.872294+0000", 2023-09-23T23:21:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4360000000000001e-05 2023-09-23T23:21:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.872413+0000", 2023-09-23T23:21:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118626 2023-09-23T23:21:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:92317 10.9 10:9fd1e9b2:::benchmark_data_smithi110_366450_object92316:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:21:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:20:25.966738+0000", 2023-09-23T23:21:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 61.374569456000003, 2023-09-23T23:21:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.90597842900000003, 2023-09-23T23:21:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 92317 2023-09-23T23:21:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.966738+0000", 2023-09-23T23:21:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.966738+0000", 2023-09-23T23:21:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.966737+0000", 2023-09-23T23:21:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:21:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.966761+0000", 2023-09-23T23:21:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3645000000000001e-05 2023-09-23T23:21:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.966777+0000", 2023-09-23T23:21:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6589e-05 2023-09-23T23:21:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.966784+0000", 2023-09-23T23:21:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8260000000000004e-06 2023-09-23T23:21:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.966842+0000", 2023-09-23T23:21:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8343e-05 2023-09-23T23:21:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.966853+0000", 2023-09-23T23:21:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0548e-05 2023-09-23T23:21:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.866678+0000", 2023-09-23T23:21:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.89982456 2023-09-23T23:21:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.866839+0000", 2023-09-23T23:21:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016121199999999999 2023-09-23T23:21:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:21:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.867025+0000", 2023-09-23T23:21:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018594799999999999 2023-09-23T23:21:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.872475+0000", 2023-09-23T23:21:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054506279999999999 2023-09-23T23:21:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.872487+0000", 2023-09-23T23:21:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1511999999999999e-05 2023-09-23T23:21:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.872519+0000", 2023-09-23T23:21:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2283000000000002e-05 2023-09-23T23:21:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.872565+0000", 2023-09-23T23:21:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5521e-05 2023-09-23T23:21:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.872613+0000", 2023-09-23T23:21:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8850000000000002e-05 2023-09-23T23:21:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.872716+0000", 2023-09-23T23:21:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102513 2023-09-23T23:21:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:92319 10.9 10:9c45a5c6:::benchmark_data_smithi110_366450_object92318:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:21:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:20:25.968630+0000", 2023-09-23T23:21:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 61.372676996999999, 2023-09-23T23:21:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.91702418399999996, 2023-09-23T23:21:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 92319 2023-09-23T23:21:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.968630+0000", 2023-09-23T23:21:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.968630+0000", 2023-09-23T23:21:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.968629+0000", 2023-09-23T23:21:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:21:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.968655+0000", 2023-09-23T23:21:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6279e-05 2023-09-23T23:21:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.968669+0000", 2023-09-23T23:21:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4133e-05 2023-09-23T23:21:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.968678+0000", 2023-09-23T23:21:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.935e-06 2023-09-23T23:21:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.969227+0000", 2023-09-23T23:21:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00054943600000000004 2023-09-23T23:21:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.457 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.457 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.969239+0000", 2023-09-23T23:21:27.457 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1241e-05 2023-09-23T23:21:27.457 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.457 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.457 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.457 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.869285+0000", 2023-09-23T23:21:27.457 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.90004693700000005 2023-09-23T23:21:27.457 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.457 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.457 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.457 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.869432+0000", 2023-09-23T23:21:27.457 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000146763 2023-09-23T23:21:27.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.458 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.458 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:21:27.458 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.869556+0000", 2023-09-23T23:21:27.458 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012423100000000001 2023-09-23T23:21:27.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.458 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.458 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.458 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.872754+0000", 2023-09-23T23:21:27.458 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031975990000000002 2023-09-23T23:21:27.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.458 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.458 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.872787+0000", 2023-09-23T23:21:27.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3175e-05 2023-09-23T23:21:27.459 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.459 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.879551+0000", 2023-09-23T23:21:27.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067641400000000001 2023-09-23T23:21:27.459 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.459 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.885483+0000", 2023-09-23T23:21:27.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059316799999999999 2023-09-23T23:21:27.459 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.460 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.460 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.460 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.885532+0000", 2023-09-23T23:21:27.460 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8573999999999999e-05 2023-09-23T23:21:27.460 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.460 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.460 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.460 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.885654+0000", 2023-09-23T23:21:27.460 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012256600000000001 2023-09-23T23:21:27.460 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.460 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.460 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.461 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.461 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:92330 10.9 10:921187c1:::benchmark_data_smithi110_366450_object92329:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:21:27.461 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:20:25.984003+0000", 2023-09-23T23:21:27.461 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 61.357304251000002, 2023-09-23T23:21:27.461 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.90292867499999996, 2023-09-23T23:21:27.461 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.461 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.461 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.461 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:27.461 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:27.461 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 92330 2023-09-23T23:21:27.461 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.462 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.984003+0000", 2023-09-23T23:21:27.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.984003+0000", 2023-09-23T23:21:27.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.463 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.984001+0000", 2023-09-23T23:21:27.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:21:27.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.463 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.463 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.984020+0000", 2023-09-23T23:21:27.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8936e-05 2023-09-23T23:21:27.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.463 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.463 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.984032+0000", 2023-09-23T23:21:27.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1576999999999999e-05 2023-09-23T23:21:27.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.984038+0000", 2023-09-23T23:21:27.464 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0870000000000003e-06 2023-09-23T23:21:27.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.984077+0000", 2023-09-23T23:21:27.464 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8902999999999997e-05 2023-09-23T23:21:27.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.984083+0000", 2023-09-23T23:21:27.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5209999999999999e-06 2023-09-23T23:21:27.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.465 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.465 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.872810+0000", 2023-09-23T23:21:27.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.88872694799999996 2023-09-23T23:21:27.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.465 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.465 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.872945+0000", 2023-09-23T23:21:27.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013525899999999999 2023-09-23T23:21:27.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.466 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:21:27.466 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.873099+0000", 2023-09-23T23:21:27.466 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015367399999999999 2023-09-23T23:21:27.466 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.466 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.466 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.466 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.879514+0000", 2023-09-23T23:21:27.466 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0064144570000000001 2023-09-23T23:21:27.466 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.466 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.466 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.466 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.885711+0000", 2023-09-23T23:21:27.466 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061975479999999998 2023-09-23T23:21:27.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.467 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.467 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.885725+0000", 2023-09-23T23:21:27.467 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3370000000000001e-05 2023-09-23T23:21:27.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.467 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.467 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.886810+0000", 2023-09-23T23:21:27.467 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001085382 2023-09-23T23:21:27.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.468 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.468 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.886848+0000", 2023-09-23T23:21:27.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8415999999999999e-05 2023-09-23T23:21:27.468 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.468 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.468 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.886931+0000", 2023-09-23T23:21:27.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3149e-05 2023-09-23T23:21:27.468 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.468 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.468 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.468 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.469 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:92336 10.9 10:9ce661fb:::benchmark_data_smithi110_366450_object92335:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:21:27.469 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:20:26.001808+0000", 2023-09-23T23:21:27.469 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 61.339498978999998, 2023-09-23T23:21:27.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.88908786699999998, 2023-09-23T23:21:27.469 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.469 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.469 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.469 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:27.469 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:27.469 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 92336 2023-09-23T23:21:27.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.469 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.469 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.001808+0000", 2023-09-23T23:21:27.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.001808+0000", 2023-09-23T23:21:27.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.001806+0000", 2023-09-23T23:21:27.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:21:27.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.001831+0000", 2023-09-23T23:21:27.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4828e-05 2023-09-23T23:21:27.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.001848+0000", 2023-09-23T23:21:27.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7486000000000002e-05 2023-09-23T23:21:27.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.001861+0000", 2023-09-23T23:21:27.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2343e-05 2023-09-23T23:21:27.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.001912+0000", 2023-09-23T23:21:27.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0869999999999999e-05 2023-09-23T23:21:27.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.001924+0000", 2023-09-23T23:21:27.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.199e-05 2023-09-23T23:21:27.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.876251+0000", 2023-09-23T23:21:27.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.87432723599999995 2023-09-23T23:21:27.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.876429+0000", 2023-09-23T23:21:27.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017851799999999999 2023-09-23T23:21:27.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:21:27.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.876621+0000", 2023-09-23T23:21:27.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000191886 2023-09-23T23:21:27.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.879583+0000", 2023-09-23T23:21:27.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002961469 2023-09-23T23:21:27.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.885731+0000", 2023-09-23T23:21:27.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061482389999999998 2023-09-23T23:21:27.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.886975+0000", 2023-09-23T23:21:27.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012436190000000001 2023-09-23T23:21:27.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.890728+0000", 2023-09-23T23:21:27.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003753583 2023-09-23T23:21:27.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.890783+0000", 2023-09-23T23:21:27.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5180999999999999e-05 2023-09-23T23:21:27.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.890896+0000", 2023-09-23T23:21:27.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011259700000000001 2023-09-23T23:21:27.476 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.476 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.476 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.477 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:92340 10.9 10:9d88a2c3:::benchmark_data_smithi110_366450_object92339:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:21:27.477 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:20:26.026740+0000", 2023-09-23T23:21:27.477 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 61.314567089000001, 2023-09-23T23:21:27.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86433311199999996, 2023-09-23T23:21:27.477 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.477 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.477 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.477 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:27.477 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:27.477 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 92340 2023-09-23T23:21:27.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.477 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.478 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.478 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.026740+0000", 2023-09-23T23:21:27.478 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.478 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.478 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.026740+0000", 2023-09-23T23:21:27.478 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.478 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.026738+0000", 2023-09-23T23:21:27.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:21:27.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.026764+0000", 2023-09-23T23:21:27.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5728000000000002e-05 2023-09-23T23:21:27.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.026780+0000", 2023-09-23T23:21:27.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6466000000000001e-05 2023-09-23T23:21:27.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.026790+0000", 2023-09-23T23:21:27.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0064999999999999e-05 2023-09-23T23:21:27.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.026849+0000", 2023-09-23T23:21:27.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8162999999999999e-05 2023-09-23T23:21:27.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:27.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.026860+0000", 2023-09-23T23:21:27.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1216999999999999e-05 2023-09-23T23:21:27.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.879601+0000", 2023-09-23T23:21:27.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.85274127799999999 2023-09-23T23:21:27.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.879706+0000", 2023-09-23T23:21:27.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010467999999999999 2023-09-23T23:21:27.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:21:27.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.879856+0000", 2023-09-23T23:21:27.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015055299999999999 2023-09-23T23:21:27.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.885465+0000", 2023-09-23T23:21:27.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056087180000000004 2023-09-23T23:21:27.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.885738+0000", 2023-09-23T23:21:27.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000272716 2023-09-23T23:21:27.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.887016+0000", 2023-09-23T23:21:27.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012777629999999999 2023-09-23T23:21:27.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.890932+0000", 2023-09-23T23:21:27.484 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039163690000000003 2023-09-23T23:21:27.484 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.484 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.484 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.890971+0000", 2023-09-23T23:21:27.484 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9388999999999998e-05 2023-09-23T23:21:27.484 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.484 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.484 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.891073+0000", 2023-09-23T23:21:27.484 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000101717 2023-09-23T23:21:27.484 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.484 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.485 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.485 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.485 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.485 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:137699 10.8 10:17e6a929:::benchmark_data_smithi110_366450_object36491:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2023-09-23T23:21:27.485 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:26.008507+0000", 2023-09-23T23:21:27.485 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 1.332800446, 2023-09-23T23:21:27.485 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28491336699999997, 2023-09-23T23:21:27.485 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.485 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.485 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.485 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:27.485 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:27.486 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 137699 2023-09-23T23:21:27.486 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.486 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.486 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.486 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.008507+0000", 2023-09-23T23:21:27.486 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.486 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.486 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.486 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.008507+0000", 2023-09-23T23:21:27.486 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.486 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.008505+0000", 2023-09-23T23:21:27.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:21:27.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.008510+0000", 2023-09-23T23:21:27.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4649999999999996e-06 2023-09-23T23:21:27.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.008518+0000", 2023-09-23T23:21:27.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7009999999999996e-06 2023-09-23T23:21:27.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.008523+0000", 2023-09-23T23:21:27.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9540000000000001e-06 2023-09-23T23:21:27.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.015769+0000", 2023-09-23T23:21:27.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007245675 2023-09-23T23:21:27.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.489 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.489 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.015862+0000", 2023-09-23T23:21:27.489 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3058000000000002e-05 2023-09-23T23:21:27.489 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.489 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:21:27.489 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.015992+0000", 2023-09-23T23:21:27.489 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013017399999999999 2023-09-23T23:21:27.489 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.489 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.489 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.029216+0000", 2023-09-23T23:21:27.489 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013223591999999999 2023-09-23T23:21:27.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.490 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.490 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.490 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.029256+0000", 2023-09-23T23:21:27.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0534000000000002e-05 2023-09-23T23:21:27.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.490 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.490 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.490 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.290545+0000", 2023-09-23T23:21:27.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.26128892199999998 2023-09-23T23:21:27.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.490 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.490 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.290608+0000", 2023-09-23T23:21:27.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2637000000000004e-05 2023-09-23T23:21:27.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.293420+0000", 2023-09-23T23:21:27.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028118890000000001 2023-09-23T23:21:27.491 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.491 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.491 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.491 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:137709 10.8 10:13f4b2eb:::benchmark_data_smithi110_366450_object36501:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2023-09-23T23:21:27.492 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:26.016582+0000", 2023-09-23T23:21:27.492 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 1.324724665, 2023-09-23T23:21:27.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28210891500000002, 2023-09-23T23:21:27.492 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.492 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.492 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.492 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:27.492 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:27.492 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 137709 2023-09-23T23:21:27.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.492 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.492 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.493 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.016582+0000", 2023-09-23T23:21:27.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.493 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.493 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.016582+0000", 2023-09-23T23:21:27.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.493 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.493 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.016581+0000", 2023-09-23T23:21:27.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:21:27.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.494 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.494 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.016586+0000", 2023-09-23T23:21:27.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3390000000000003e-06 2023-09-23T23:21:27.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.494 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.494 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.016596+0000", 2023-09-23T23:21:27.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0035e-05 2023-09-23T23:21:27.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.494 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.016602+0000", 2023-09-23T23:21:27.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2899999999999999e-06 2023-09-23T23:21:27.495 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.495 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.026084+0000", 2023-09-23T23:21:27.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0094815980000000008 2023-09-23T23:21:27.495 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.495 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.026171+0000", 2023-09-23T23:21:27.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7626000000000004e-05 2023-09-23T23:21:27.495 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:21:27.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.026300+0000", 2023-09-23T23:21:27.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000129102 2023-09-23T23:21:27.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.276992+0000", 2023-09-23T23:21:27.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.25069158400000002 2023-09-23T23:21:27.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.277045+0000", 2023-09-23T23:21:27.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2787000000000001e-05 2023-09-23T23:21:27.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.293430+0000", 2023-09-23T23:21:27.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016385705 2023-09-23T23:21:27.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.293481+0000", 2023-09-23T23:21:27.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0954e-05 2023-09-23T23:21:27.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.498 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.498 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.298691+0000", 2023-09-23T23:21:27.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052098960000000003 2023-09-23T23:21:27.498 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.498 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.498 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.498 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.498 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.498 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:137716 10.8 10:145f7a45:::benchmark_data_smithi110_366450_object36508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2023-09-23T23:21:27.498 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:26.016646+0000", 2023-09-23T23:21:27.498 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 1.324660747, 2023-09-23T23:21:27.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.29654835299999999, 2023-09-23T23:21:27.499 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:27.499 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:27.499 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:27.499 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:21:27.499 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:21:27.499 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 137716 2023-09-23T23:21:27.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.499 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:27.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:27.499 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.016646+0000", 2023-09-23T23:21:27.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:27.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.016646+0000", 2023-09-23T23:21:27.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:27.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:27.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.016646+0000", 2023-09-23T23:21:27.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-09-23T23:21:27.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:27.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.016648+0000", 2023-09-23T23:21:27.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.762e-06 2023-09-23T23:21:27.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:27.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.016651+0000", 2023-09-23T23:21:27.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0580000000000002e-06 2023-09-23T23:21:27.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:27.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.016654+0000", 2023-09-23T23:21:27.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9689999999999999e-06 2023-09-23T23:21:27.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.502 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:27.502 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.029282+0000", 2023-09-23T23:21:27.502 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012628189999999999 2023-09-23T23:21:27.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.502 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:27.502 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.029385+0000", 2023-09-23T23:21:27.502 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103106 2023-09-23T23:21:27.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.502 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:21:27.503 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.029531+0000", 2023-09-23T23:21:27.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014576199999999999 2023-09-23T23:21:27.503 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.503 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.503 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.282284+0000", 2023-09-23T23:21:27.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.25275262599999998 2023-09-23T23:21:27.503 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.503 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:27.503 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.282326+0000", 2023-09-23T23:21:27.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2559999999999999e-05 2023-09-23T23:21:27.503 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.504 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:27.504 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.310326+0000", 2023-09-23T23:21:27.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028000377 2023-09-23T23:21:27.504 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.504 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.504 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:27.504 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.310380+0000", 2023-09-23T23:21:27.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3641999999999999e-05 2023-09-23T23:21:27.504 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:27.504 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:27.504 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:27.504 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:26.313195+0000", 2023-09-23T23:21:27.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028145309999999999 2023-09-23T23:21:27.505 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.505 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.505 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.505 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:27.505 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:27.505 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:21:27.513 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:21:27.520 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:27 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:21:27.520 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:27 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:21:27.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:27.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:21:27.622 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:27.687 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:27.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:27 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:21:28.160 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 2] out_osds: [1] dead_osds: [7] live_osds: [0, 1, 3, 4, 5, 6, 2] 2023-09-23T23:21:28.161 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:21:28.161 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T23:21:28.161 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T23:21:28.161 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T23:21:28.161 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T23:21:28.315 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:28.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:28.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:28.427 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:28.436 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:21:28.464 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:28.547 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:28.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:28.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:28.660 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:28.684 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:21:28.697 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:28.738 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:21:28.758 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:28.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:28.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:28.864 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:28.973 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:29.010 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:29.106 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:29.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:29.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:29.134 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:28 smithi110 ceph-mon[212939]: pgmap v905: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 13 KiB/s rd, 35 KiB/s wr, 554 op/s; 12279/197712 objects degraded (6.211%); 37323/197712 objects misplaced (18.877%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:21:29.134 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:29.134 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:21:29.134 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:28 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 12279/197712 objects degraded (6.211%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:29.134 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:28 smithi110 ceph-mon[221102]: pgmap v905: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 13 KiB/s rd, 35 KiB/s wr, 554 op/s; 12279/197712 objects degraded (6.211%); 37323/197712 objects misplaced (18.877%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:21:29.134 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:29.134 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:21:29.135 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:28 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 12279/197712 objects degraded (6.211%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:29.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:28 smithi138 ceph-mon[193606]: pgmap v905: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 13 KiB/s rd, 35 KiB/s wr, 554 op/s; 12279/197712 objects degraded (6.211%); 37323/197712 objects misplaced (18.877%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:21:29.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:29.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:21:29.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:28 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 12279/197712 objects degraded (6.211%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:29.234 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:29.324 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:29.457 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:29.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:29.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:29.572 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:29.672 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:29.721 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:30.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:29 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:21:30.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:29 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:21:30.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:29 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:21:31.126 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:31.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:31.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:31.234 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:31.243 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:31.269 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:31 smithi138 ceph-mon[193606]: pgmap v906: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 612 op/s; 12279/193536 objects degraded (6.345%); 36634/193536 objects misplaced (18.929%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:21:31.269 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:31 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:31.269 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:31 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:31.272 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:31.329 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:31.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:31.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:31.356 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:31 smithi110 ceph-mon[212939]: pgmap v906: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 612 op/s; 12279/193536 objects degraded (6.345%); 36634/193536 objects misplaced (18.929%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:21:31.356 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:31 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:31.356 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:31 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:31.357 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:31 smithi110 ceph-mon[221102]: pgmap v906: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 612 op/s; 12279/193536 objects degraded (6.345%); 36634/193536 objects misplaced (18.929%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:21:31.357 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:31 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:31.357 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:31 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:31.413 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:31.436 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:31.480 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:31.547 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:31.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:31.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:31.652 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:31.760 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:32.038 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:32 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:21:32.038 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:32 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:21:32.038 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:32 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:32.038 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:32 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:21:32.038 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:32 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:21:32.038 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:32 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:32.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:32 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:21:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:32 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:21:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:32 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:33.180 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:33.184 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:33 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:21:33.184 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:33 smithi110 ceph-mon[221102]: pgmap v907: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 8.9 KiB/s rd, 24 KiB/s wr, 779 op/s; 11565/186393 objects degraded (6.205%); 35527/186393 objects misplaced (19.060%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:21:33.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:33 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:21:33.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:33 smithi110 ceph-mon[212939]: pgmap v907: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 8.9 KiB/s rd, 24 KiB/s wr, 779 op/s; 11565/186393 objects degraded (6.205%); 35527/186393 objects misplaced (19.060%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:21:33.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:33.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:33.286 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:33.327 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:33 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:21:33.327 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:33 smithi138 ceph-mon[193606]: pgmap v907: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 8.9 KiB/s rd, 24 KiB/s wr, 779 op/s; 11565/186393 objects degraded (6.205%); 35527/186393 objects misplaced (19.060%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:21:33.337 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:33.407 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:33.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:33.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:33.519 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:21:33.551 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:33.762 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:21:33.762 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "" 2023-09-23T23:21:33.762 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:21:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:34 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 11565/186393 objects degraded (6.205%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:34 smithi138 ceph-mon[193606]: pgmap v908: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 8.3 KiB/s rd, 22 KiB/s wr, 728 op/s; 11565/186393 objects degraded (6.205%); 35527/186393 objects misplaced (19.060%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:21:34.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:34 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 11565/186393 objects degraded (6.205%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:34.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:34 smithi110 ceph-mon[212939]: pgmap v908: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 8.3 KiB/s rd, 22 KiB/s wr, 728 op/s; 11565/186393 objects degraded (6.205%); 35527/186393 objects misplaced (19.060%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:21:34.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:34 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 11565/186393 objects degraded (6.205%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:34.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:34 smithi110 ceph-mon[221102]: pgmap v908: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 8.3 KiB/s rd, 22 KiB/s wr, 728 op/s; 11565/186393 objects degraded (6.205%); 35527/186393 objects misplaced (19.060%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:21:34.870 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:21:34.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:34.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:21:34.978 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:21:35.042 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:36.001 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:21:36.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:36.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:21:36.123 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:21:36.186 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:36.255 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:21:36.256 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:21:36.256 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-09-23T23:21:36.256 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:21:36.442 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:21:36.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:36.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:21:36.547 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:36.586 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:36.640 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:36.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:36.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:36.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:36 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:21:36.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:36 smithi110 ceph-mon[212939]: pgmap v909: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 7.5 KiB/s rd, 20 KiB/s wr, 1.03k op/s; 10798/174003 objects degraded (6.206%); 33609/174003 objects misplaced (19.315%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T23:21:36.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:36 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:21:36.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:36 smithi110 ceph-mon[221102]: pgmap v909: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 7.5 KiB/s rd, 20 KiB/s wr, 1.03k op/s; 10798/174003 objects degraded (6.206%); 33609/174003 objects misplaced (19.315%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T23:21:36.748 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:36.802 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:36.877 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:36.887 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:36 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:21:36.887 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:36 smithi138 ceph-mon[193606]: pgmap v909: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 7.5 KiB/s rd, 20 KiB/s wr, 1.03k op/s; 10798/174003 objects degraded (6.206%); 33609/174003 objects misplaced (19.315%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T23:21:36.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:36.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:36.988 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:37.026 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:37.086 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:37.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:37.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:37.192 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:37.205 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:21:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:21:36] "GET /metrics HTTP/1.1" 200 36640 "" "Prometheus/2.43.0" 2023-09-23T23:21:37.275 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:37.568 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:37.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:37.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:37.628 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:21:37.684 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:21:37.726 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:21:37.771 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:21:37.779 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:37.800 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:21:37.813 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:21:37.848 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:21:37.876 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:21:37.879 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:37 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3038141316' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:21:37.879 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:37 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3038141316' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:21:37.924 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:21:37.933 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:21:37.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:37.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:21:37.970 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:21:37.970 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:21:37.970 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:21:37.970 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:21:37.970 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:21:37.970 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:21:37.970 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:21:37.971 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:21:37.971 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:21:38.040 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:38.050 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:37 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3038141316' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:21:38.092 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:38.180 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:38.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:38.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:38.286 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:38.358 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:38 smithi110 ceph-mon[212939]: pgmap v910: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 10798/174003 objects degraded (6.206%); 33609/174003 objects misplaced (19.315%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T23:21:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:38 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:21:38.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:38 smithi110 ceph-mon[212939]: failed to encode map e298 with expected crc 2023-09-23T23:21:38.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:38 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3038141316' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:21:38.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:38 smithi110 ceph-mon[212939]: osdmap e298: 8 total, 7 up, 7 in 2023-09-23T23:21:38.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:38 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3616314284' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:38.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:38 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 10798/174003 objects degraded (6.206%), 5 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:38.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:38 smithi110 ceph-mon[221102]: pgmap v910: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 10798/174003 objects degraded (6.206%); 33609/174003 objects misplaced (19.315%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T23:21:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:38 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:21:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:38 smithi110 ceph-mon[221102]: failed to encode map e298 with expected crc 2023-09-23T23:21:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:38 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3038141316' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:21:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:38 smithi110 ceph-mon[221102]: osdmap e298: 8 total, 7 up, 7 in 2023-09-23T23:21:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:38 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3616314284' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:38.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:38 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 10798/174003 objects degraded (6.206%), 5 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:38.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:38 smithi138 ceph-mon[193606]: pgmap v910: 99 pgs: 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 5 active+undersized, 75 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 10798/174003 objects degraded (6.206%); 33609/174003 objects misplaced (19.315%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T23:21:38.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:38 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:21:38.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:38 smithi138 ceph-mon[193606]: failed to encode map e298 with expected crc 2023-09-23T23:21:38.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:38 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3038141316' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:21:38.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:38 smithi138 ceph-mon[193606]: osdmap e298: 8 total, 7 up, 7 in 2023-09-23T23:21:38.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:38 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3616314284' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:38.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:38 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 10798/174003 objects degraded (6.206%), 5 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:39.255 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:39.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:39.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:39.363 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:21:39.375 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:21:39.377 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:21:39.441 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:39.493 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:39.541 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:21:39.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:39.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:21:39.657 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:39.705 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:39.781 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:39.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:39.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:39.891 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:39.974 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:40.134 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:40.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:40.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:40.239 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:40.299 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:40.407 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:40.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:40.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:40.516 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:40.578 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:40.673 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:40.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:40.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:40.780 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:40.829 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:40 smithi110 ceph-mon[221102]: pgmap v912: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 805 op/s; 10798/169350 objects degraded (6.376%); 32821/169350 objects misplaced (19.381%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T23:21:40.831 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:40 smithi110 ceph-mon[212939]: pgmap v912: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 805 op/s; 10798/169350 objects degraded (6.376%); 32821/169350 objects misplaced (19.381%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T23:21:40.833 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:40.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:40 smithi138 ceph-mon[193606]: pgmap v912: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 805 op/s; 10798/169350 objects degraded (6.376%); 32821/169350 objects misplaced (19.381%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T23:21:41.045 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:41.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:41.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:41.168 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:41.213 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:41.225 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 2] out_osds: [1] dead_osds: [7] live_osds: [0, 1, 3, 4, 5, 6, 2] 2023-09-23T23:21:41.226 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:21:41.226 INFO:tasks.thrashosds.thrasher:Reweighting osd 4 to 0.5578114180620021 2023-09-23T23:21:41.226 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 0.5578114180620021 2023-09-23T23:21:41.268 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:21:41.292 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:41.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:41.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:41.377 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:21:41.398 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:41.429 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:41.500 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:41.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:41.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:41.607 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:21:41.726 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:41.758 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:21:41.758 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:21:41.984 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:21:41.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:41.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:21:42.097 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:42.150 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:42.254 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:42.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:42.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:42.364 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:42.412 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:42.492 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:42.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:42.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:42.601 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:42.649 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:42.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:42 smithi138 ceph-mon[193606]: pgmap v913: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 10004/161553 objects degraded (6.192%); 31646/161553 objects misplaced (19.589%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:21:42.717 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:42.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:42.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:42.824 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:42.886 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:42.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:42 smithi110 ceph-mon[212939]: pgmap v913: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 10004/161553 objects degraded (6.192%); 31646/161553 objects misplaced (19.589%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:21:42.952 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:42.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:42 smithi110 ceph-mon[221102]: pgmap v913: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 10004/161553 objects degraded (6.192%); 31646/161553 objects misplaced (19.589%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:21:42.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:42.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:42.979 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:21:43.057 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:43.090 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:43.176 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:43.182 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:43.202 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:43.334 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:43.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:43.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:43.441 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:43.490 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:43.550 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:43.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:43.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:43.654 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:21:43.748 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:43.919 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:21:43.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:43.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:21:44.029 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:44.073 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:44.171 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:44.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:44.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:44.278 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:44.333 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:44.333 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:21:44.333 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:43 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 10004/161553 objects degraded (6.192%), 5 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:44.336 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:44.417 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:44.423 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:44.423 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:21:44.423 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:43 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 10004/161553 objects degraded (6.192%), 5 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:44.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:21:44.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:21:44.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:43 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 10004/161553 objects degraded (6.192%), 5 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:44.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:44.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:44.531 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:44.586 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:45.413 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:44 smithi110 ceph-mon[212939]: pgmap v914: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 10004/161553 objects degraded (6.192%); 31646/161553 objects misplaced (19.589%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:21:45.414 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:44 smithi110 ceph-mon[221102]: pgmap v914: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 10004/161553 objects degraded (6.192%); 31646/161553 objects misplaced (19.589%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:21:45.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:44 smithi138 ceph-mon[193606]: pgmap v914: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 10004/161553 objects degraded (6.192%); 31646/161553 objects misplaced (19.589%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:21:46.930 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:46.932 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:21:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:21:46] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.43.0" 2023-09-23T23:21:46.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:46.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:47.035 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:47.118 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:47.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:47 smithi110 ceph-mon[212939]: pgmap v915: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 9208/148470 objects degraded (6.202%); 29567/148470 objects misplaced (19.914%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:21:47.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:47 smithi110 ceph-mon[221102]: pgmap v915: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 9208/148470 objects degraded (6.202%); 29567/148470 objects misplaced (19.914%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:21:47.323 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:47.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:47.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:47.432 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:47.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:47 smithi138 ceph-mon[193606]: pgmap v915: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 9208/148470 objects degraded (6.202%); 29567/148470 objects misplaced (19.914%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:21:47.470 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:47.541 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:47.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:47.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:47.611 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:21:47.644 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:21:47.649 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:47.688 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:21:47.735 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:21:47.745 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:47.760 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:47.813 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:21:47.853 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:21:47.872 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:47.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:47.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:47.895 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:21:47.906 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:21:47.907 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:21:47.907 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:21:47.907 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:21:47.907 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:21:47.907 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:21:47.907 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:21:47.907 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:21:47.981 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:48.032 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:21:48.083 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:48.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:48 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1549871581' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:48.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:48 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/431927541' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:21:48.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:48 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1549871581' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:48.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:48 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/431927541' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:21:48.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:48 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1549871581' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:48 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/431927541' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:21:48.595 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:48.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:48.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:48.700 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:48.767 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:49.046 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.4 to 0.557811 (8ecc) 2023-09-23T23:21:49.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:49 smithi138 ceph-mon[193606]: pgmap v916: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 9208/148470 objects degraded (6.202%); 29567/148470 objects misplaced (19.914%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:21:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:49 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:21:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:49 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/431927541' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:21:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:49 smithi138 ceph-mon[193606]: osdmap e299: 8 total, 7 up, 7 in 2023-09-23T23:21:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:49 smithi138 ceph-mon[193606]: failed to encode map e299 with expected crc 2023-09-23T23:21:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:49 smithi138 ceph-mon[193606]: failed to encode map e299 with expected crc 2023-09-23T23:21:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:49 smithi138 ceph-mon[193606]: failed to encode map e299 with expected crc 2023-09-23T23:21:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:49 smithi138 ceph-mon[193606]: failed to encode map e299 with expected crc 2023-09-23T23:21:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:49 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2756529815' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.5578114180620021}]: dispatch 2023-09-23T23:21:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:49 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.5578114180620021}]: dispatch 2023-09-23T23:21:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:49 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 9208/148470 objects degraded (6.202%), 5 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:49.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[212939]: pgmap v916: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 9208/148470 objects degraded (6.202%); 29567/148470 objects misplaced (19.914%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:21:49.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:21:49.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/431927541' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:21:49.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[212939]: osdmap e299: 8 total, 7 up, 7 in 2023-09-23T23:21:49.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[212939]: failed to encode map e299 with expected crc 2023-09-23T23:21:49.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[212939]: failed to encode map e299 with expected crc 2023-09-23T23:21:49.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[212939]: failed to encode map e299 with expected crc 2023-09-23T23:21:49.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[212939]: failed to encode map e299 with expected crc 2023-09-23T23:21:49.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2756529815' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.5578114180620021}]: dispatch 2023-09-23T23:21:49.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.5578114180620021}]: dispatch 2023-09-23T23:21:49.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 9208/148470 objects degraded (6.202%), 5 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[221102]: pgmap v916: 99 pgs: 1 active+recovery_wait+remapped, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+undersized+degraded, 5 active+undersized, 75 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 9208/148470 objects degraded (6.202%); 29567/148470 objects misplaced (19.914%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:21:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:21:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/431927541' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:21:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[221102]: osdmap e299: 8 total, 7 up, 7 in 2023-09-23T23:21:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[221102]: failed to encode map e299 with expected crc 2023-09-23T23:21:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[221102]: failed to encode map e299 with expected crc 2023-09-23T23:21:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[221102]: failed to encode map e299 with expected crc 2023-09-23T23:21:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[221102]: failed to encode map e299 with expected crc 2023-09-23T23:21:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2756529815' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.5578114180620021}]: dispatch 2023-09-23T23:21:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.5578114180620021}]: dispatch 2023-09-23T23:21:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:49 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 9208/148470 objects degraded (6.202%), 5 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:50.087 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:21:50.087 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:21:50.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:50 smithi110 ceph-mon[221102]: failed to encode map e300 with expected crc 2023-09-23T23:21:50.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:50 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.5578114180620021}]': finished 2023-09-23T23:21:50.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:50 smithi110 ceph-mon[221102]: osdmap e300: 8 total, 7 up, 7 in 2023-09-23T23:21:50.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:50 smithi110 ceph-mon[221102]: failed to encode map e300 with expected crc 2023-09-23T23:21:50.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:50 smithi110 ceph-mon[221102]: failed to encode map e300 with expected crc 2023-09-23T23:21:50.218 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:50 smithi110 ceph-mon[221102]: osdmap e301: 8 total, 7 up, 7 in 2023-09-23T23:21:50.218 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:50 smithi110 ceph-mon[212939]: failed to encode map e300 with expected crc 2023-09-23T23:21:50.218 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:50 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.5578114180620021}]': finished 2023-09-23T23:21:50.219 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:50 smithi110 ceph-mon[212939]: osdmap e300: 8 total, 7 up, 7 in 2023-09-23T23:21:50.219 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:50 smithi110 ceph-mon[212939]: failed to encode map e300 with expected crc 2023-09-23T23:21:50.219 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:50 smithi110 ceph-mon[212939]: failed to encode map e300 with expected crc 2023-09-23T23:21:50.219 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:50 smithi110 ceph-mon[212939]: osdmap e301: 8 total, 7 up, 7 in 2023-09-23T23:21:50.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:50 smithi138 ceph-mon[193606]: failed to encode map e300 with expected crc 2023-09-23T23:21:50.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:50 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.5578114180620021}]': finished 2023-09-23T23:21:50.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:50 smithi138 ceph-mon[193606]: osdmap e300: 8 total, 7 up, 7 in 2023-09-23T23:21:50.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:50 smithi138 ceph-mon[193606]: failed to encode map e300 with expected crc 2023-09-23T23:21:50.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:50 smithi138 ceph-mon[193606]: failed to encode map e300 with expected crc 2023-09-23T23:21:50.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:50 smithi138 ceph-mon[193606]: osdmap e301: 8 total, 7 up, 7 in 2023-09-23T23:21:50.737 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:50.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:50.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:50.841 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:21:50.870 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:51.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:51 smithi138 ceph-mon[193606]: pgmap v919: 99 pgs: 1 remapped+peering, 5 peering, 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 3 active+undersized+degraded, 4 active+undersized, 72 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 9204/144018 objects degraded (6.391%); 25976/144018 objects misplaced (18.037%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:21:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:51 smithi138 ceph-mon[193606]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:51 smithi138 ceph-mon[193606]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:51 smithi138 ceph-mon[193606]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:51 smithi138 ceph-mon[193606]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:51 smithi138 ceph-mon[193606]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:51 smithi138 ceph-mon[193606]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:51 smithi138 ceph-mon[193606]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:51 smithi138 ceph-mon[193606]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:51 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:21:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:51 smithi138 ceph-mon[193606]: osdmap e302: 8 total, 7 up, 7 in 2023-09-23T23:21:51.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[221102]: pgmap v919: 99 pgs: 1 remapped+peering, 5 peering, 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 3 active+undersized+degraded, 4 active+undersized, 72 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 9204/144018 objects degraded (6.391%); 25976/144018 objects misplaced (18.037%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:21:51.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[221102]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[221102]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[221102]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[221102]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[221102]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[221102]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[221102]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[221102]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:21:51.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[221102]: osdmap e302: 8 total, 7 up, 7 in 2023-09-23T23:21:51.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[212939]: pgmap v919: 99 pgs: 1 remapped+peering, 5 peering, 1 active+recovery_wait+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 3 active+undersized+degraded, 4 active+undersized, 72 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 9204/144018 objects degraded (6.391%); 25976/144018 objects misplaced (18.037%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:21:51.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[212939]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[212939]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[212939]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[212939]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[212939]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[212939]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[212939]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[212939]: failed to encode map e301 with expected crc 2023-09-23T23:21:51.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:21:51.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:51 smithi110 ceph-mon[212939]: osdmap e302: 8 total, 7 up, 7 in 2023-09-23T23:21:51.536 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:21:51.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:51.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:21:51.640 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:51.667 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:51.710 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:51.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:51.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:51.817 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:21:51.846 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:52.054 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:21:52.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:52.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:21:52.088 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:21:52.117 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:52 smithi110 ceph-mon[221102]: failed to encode map e302 with expected crc 2023-09-23T23:21:52.117 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:52 smithi110 ceph-mon[221102]: failed to encode map e302 with expected crc 2023-09-23T23:21:52.117 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:52 smithi110 ceph-mon[221102]: failed to encode map e302 with expected crc 2023-09-23T23:21:52.117 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:52 smithi110 ceph-mon[221102]: 10.1 starting backfill to osd.2 from (0'0,0'0] MAX to 299'9616 2023-09-23T23:21:52.117 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:52 smithi110 ceph-mon[221102]: failed to encode map e302 with expected crc 2023-09-23T23:21:52.117 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:52 smithi110 ceph-mon[221102]: failed to encode map e302 with expected crc 2023-09-23T23:21:52.118 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:52 smithi110 ceph-mon[221102]: osdmap e303: 8 total, 7 up, 7 in 2023-09-23T23:21:52.118 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:52 smithi110 ceph-mon[212939]: failed to encode map e302 with expected crc 2023-09-23T23:21:52.118 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:52 smithi110 ceph-mon[212939]: failed to encode map e302 with expected crc 2023-09-23T23:21:52.118 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:52 smithi110 ceph-mon[212939]: failed to encode map e302 with expected crc 2023-09-23T23:21:52.118 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:52 smithi110 ceph-mon[212939]: 10.1 starting backfill to osd.2 from (0'0,0'0] MAX to 299'9616 2023-09-23T23:21:52.118 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:52 smithi110 ceph-mon[212939]: failed to encode map e302 with expected crc 2023-09-23T23:21:52.118 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:52 smithi110 ceph-mon[212939]: failed to encode map e302 with expected crc 2023-09-23T23:21:52.119 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:52 smithi110 ceph-mon[212939]: osdmap e303: 8 total, 7 up, 7 in 2023-09-23T23:21:52.126 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:21:52.160 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:52.186 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:52.229 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:52.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:52.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:52.335 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:52.371 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:52.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:52 smithi138 ceph-mon[193606]: failed to encode map e302 with expected crc 2023-09-23T23:21:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:52 smithi138 ceph-mon[193606]: failed to encode map e302 with expected crc 2023-09-23T23:21:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:52 smithi138 ceph-mon[193606]: failed to encode map e302 with expected crc 2023-09-23T23:21:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:52 smithi138 ceph-mon[193606]: 10.1 starting backfill to osd.2 from (0'0,0'0] MAX to 299'9616 2023-09-23T23:21:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:52 smithi138 ceph-mon[193606]: failed to encode map e302 with expected crc 2023-09-23T23:21:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:52 smithi138 ceph-mon[193606]: failed to encode map e302 with expected crc 2023-09-23T23:21:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:52 smithi138 ceph-mon[193606]: osdmap e303: 8 total, 7 up, 7 in 2023-09-23T23:21:52.482 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:52.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:52.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:52.587 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:52.613 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:52.655 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:52.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:52.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:52.703 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:52.759 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:21:52.760 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:21:52.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:21:52.760 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:21:52.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.760 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188489 9.a 9:5bc5f789:::benchmark_data_smithi110_295904_object77308:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2023-09-23T23:21:52.760 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.088144+0000", 2023-09-23T23:21:52.760 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 259.66827473400002, 2023-09-23T23:21:52.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.404339309, 2023-09-23T23:21:52.760 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:52.760 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:21:52.760 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:52.761 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:52.761 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:52.761 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188489 2023-09-23T23:21:52.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.761 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:52.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:52.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088144+0000", 2023-09-23T23:21:52.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:52.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088144+0000", 2023-09-23T23:21:52.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:52.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088143+0000", 2023-09-23T23:21:52.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:21:52.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:52.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088149+0000", 2023-09-23T23:21:52.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5010000000000001e-06 2023-09-23T23:21:52.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:52.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088167+0000", 2023-09-23T23:21:52.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8244999999999999e-05 2023-09-23T23:21:52.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:52.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088175+0000", 2023-09-23T23:21:52.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7940000000000003e-06 2023-09-23T23:21:52.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088467+0000", 2023-09-23T23:21:52.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00029206099999999998 2023-09-23T23:21:52.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:52.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088606+0000", 2023-09-23T23:21:52.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013849199999999999 2023-09-23T23:21:52.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:21:52.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088732+0000", 2023-09-23T23:21:52.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012622999999999999 2023-09-23T23:21:52.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.121963+0000", 2023-09-23T23:21:52.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.033230955999999999 2023-09-23T23:21:52.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.121975+0000", 2023-09-23T23:21:52.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2269999999999999e-05 2023-09-23T23:21:52.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291169+0000", 2023-09-23T23:21:52.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.169194251 2023-09-23T23:21:52.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.766 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291184+0000", 2023-09-23T23:21:52.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4622e-05 2023-09-23T23:21:52.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.766 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.466395+0000", 2023-09-23T23:21:52.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.175211303 2023-09-23T23:21:52.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:21:52.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.466591+0000", 2023-09-23T23:21:52.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019571700000000001 2023-09-23T23:21:52.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492418+0000", 2023-09-23T23:21:52.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.025827039 2023-09-23T23:21:52.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:52.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492483+0000", 2023-09-23T23:21:52.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5488999999999996e-05 2023-09-23T23:21:52.768 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.768 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:52.768 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.768 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.768 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188535 9.a 9:556d73ae:::benchmark_data_smithi110_295904_object77354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:21:52.768 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.479923+0000", 2023-09-23T23:21:52.768 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 259.27649544299999, 2023-09-23T23:21:52.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.195745337, 2023-09-23T23:21:52.768 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:52.768 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:52.768 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:52.768 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:52.769 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:52.769 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188535 2023-09-23T23:21:52.769 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.769 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:52.769 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.769 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:52.769 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.479923+0000", 2023-09-23T23:21:52.769 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.769 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.769 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.769 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:52.769 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.479923+0000", 2023-09-23T23:21:52.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.770 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.770 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:52.770 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.479922+0000", 2023-09-23T23:21:52.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:21:52.770 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.770 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:52.770 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.479929+0000", 2023-09-23T23:21:52.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.277e-06 2023-09-23T23:21:52.770 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:52.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.479944+0000", 2023-09-23T23:21:52.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5027e-05 2023-09-23T23:21:52.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:52.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.479954+0000", 2023-09-23T23:21:52.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9280000000000005e-06 2023-09-23T23:21:52.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.484298+0000", 2023-09-23T23:21:52.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043436960000000002 2023-09-23T23:21:52.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.772 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.772 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.484306+0000", 2023-09-23T23:21:52.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2390000000000002e-06 2023-09-23T23:21:52.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291218+0000", 2023-09-23T23:21:52.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.80691163300000002 2023-09-23T23:21:52.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291226+0000", 2023-09-23T23:21:52.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5539999999999998e-06 2023-09-23T23:21:52.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.466903+0000", 2023-09-23T23:21:52.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.175676763 2023-09-23T23:21:52.774 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:52.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.467051+0000", 2023-09-23T23:21:52.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014825299999999999 2023-09-23T23:21:52.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:21:52.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.467178+0000", 2023-09-23T23:21:52.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000126332 2023-09-23T23:21:52.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484120+0000", 2023-09-23T23:21:52.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016942282999999999 2023-09-23T23:21:52.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484174+0000", 2023-09-23T23:21:52.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4039000000000002e-05 2023-09-23T23:21:52.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:52.777 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484241+0000", 2023-09-23T23:21:52.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7428999999999994e-05 2023-09-23T23:21:52.777 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.777 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.777 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.675434+0000", 2023-09-23T23:21:52.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.19119288500000001 2023-09-23T23:21:52.777 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:52.778 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.675510+0000", 2023-09-23T23:21:52.778 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5970000000000006e-05 2023-09-23T23:21:52.778 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.778 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:52.778 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.675669+0000", 2023-09-23T23:21:52.778 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015847499999999999 2023-09-23T23:21:52.779 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.779 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:52.779 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.779 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188546 9.a 9:58b06a3d:::benchmark_data_smithi110_295904_object77365:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:21:52.779 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.494309+0000", 2023-09-23T23:21:52.779 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 259.26211014299997, 2023-09-23T23:21:52.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1816013459999999, 2023-09-23T23:21:52.779 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:52.779 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:52.780 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:52.780 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:52.780 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:52.780 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188546 2023-09-23T23:21:52.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.780 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:52.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:52.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.494309+0000", 2023-09-23T23:21:52.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:52.782 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.494309+0000", 2023-09-23T23:21:52.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:52.782 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.494307+0000", 2023-09-23T23:21:52.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:21:52.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:52.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.494315+0000", 2023-09-23T23:21:52.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7389999999999999e-06 2023-09-23T23:21:52.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:52.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.494334+0000", 2023-09-23T23:21:52.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8660000000000001e-05 2023-09-23T23:21:52.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:52.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.494343+0000", 2023-09-23T23:21:52.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4199999999999996e-06 2023-09-23T23:21:52.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.494380+0000", 2023-09-23T23:21:52.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7329000000000001e-05 2023-09-23T23:21:52.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.494388+0000", 2023-09-23T23:21:52.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8970000000000008e-06 2023-09-23T23:21:52.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291261+0000", 2023-09-23T23:21:52.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.79687277199999995 2023-09-23T23:21:52.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291269+0000", 2023-09-23T23:21:52.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2349999999999999e-06 2023-09-23T23:21:52.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.471001+0000", 2023-09-23T23:21:52.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1797320760000001 2023-09-23T23:21:52.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:52.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.471127+0000", 2023-09-23T23:21:52.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012590499999999999 2023-09-23T23:21:52.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:21:52.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.471217+0000", 2023-09-23T23:21:52.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0203000000000004e-05 2023-09-23T23:21:52.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484224+0000", 2023-09-23T23:21:52.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013006676 2023-09-23T23:21:52.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:52.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484255+0000", 2023-09-23T23:21:52.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0657999999999997e-05 2023-09-23T23:21:52.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492151+0000", 2023-09-23T23:21:52.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078959400000000006 2023-09-23T23:21:52.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.675738+0000", 2023-09-23T23:21:52.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.18358733599999999 2023-09-23T23:21:52.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:52.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.675793+0000", 2023-09-23T23:21:52.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4852999999999998e-05 2023-09-23T23:21:52.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:52.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.675910+0000", 2023-09-23T23:21:52.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011725100000000001 2023-09-23T23:21:52.790 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.790 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:52.791 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.791 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188551 9.a 9:538d3980:::benchmark_data_smithi110_295904_object77370:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:21:52.791 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.499666+0000", 2023-09-23T23:21:52.791 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 259.25675258799998, 2023-09-23T23:21:52.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1800013950000001, 2023-09-23T23:21:52.791 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:52.791 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:52.791 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:52.791 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:52.792 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:52.792 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188551 2023-09-23T23:21:52.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.792 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:52.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:52.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.499666+0000", 2023-09-23T23:21:52.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:52.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.499666+0000", 2023-09-23T23:21:52.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:52.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.499665+0000", 2023-09-23T23:21:52.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:21:52.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:52.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.499671+0000", 2023-09-23T23:21:52.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.648e-06 2023-09-23T23:21:52.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:52.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.499681+0000", 2023-09-23T23:21:52.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0913e-05 2023-09-23T23:21:52.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:52.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.499689+0000", 2023-09-23T23:21:52.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2609999999999996e-06 2023-09-23T23:21:52.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.500993+0000", 2023-09-23T23:21:52.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013044109999999999 2023-09-23T23:21:52.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.501004+0000", 2023-09-23T23:21:52.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0645999999999999e-05 2023-09-23T23:21:52.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291296+0000", 2023-09-23T23:21:52.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.790291878 2023-09-23T23:21:52.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291304+0000", 2023-09-23T23:21:52.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3780000000000001e-06 2023-09-23T23:21:52.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.474986+0000", 2023-09-23T23:21:52.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.183682377 2023-09-23T23:21:52.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:52.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.475121+0000", 2023-09-23T23:21:52.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000134664 2023-09-23T23:21:52.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:21:52.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.475271+0000", 2023-09-23T23:21:52.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015008900000000001 2023-09-23T23:21:52.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:52.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484271+0000", 2023-09-23T23:21:52.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0090001539999999998 2023-09-23T23:21:52.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484340+0000", 2023-09-23T23:21:52.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8305000000000004e-05 2023-09-23T23:21:52.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492192+0000", 2023-09-23T23:21:52.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078518979999999995 2023-09-23T23:21:52.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.679470+0000", 2023-09-23T23:21:52.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.187278429 2023-09-23T23:21:52.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:52.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.679534+0000", 2023-09-23T23:21:52.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3664999999999994e-05 2023-09-23T23:21:52.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.801 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:52.801 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.679668+0000", 2023-09-23T23:21:52.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013406099999999999 2023-09-23T23:21:52.801 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.801 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:52.801 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.801 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188554 9.a 9:5586f631:::benchmark_data_smithi110_295904_object77373:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:21:52.801 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.509609+0000", 2023-09-23T23:21:52.802 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 259.24680952799997, 2023-09-23T23:21:52.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1702691110000001, 2023-09-23T23:21:52.802 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:52.802 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:52.802 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:52.802 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:52.802 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:52.802 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188554 2023-09-23T23:21:52.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.802 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:52.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:52.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.509609+0000", 2023-09-23T23:21:52.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:52.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.509609+0000", 2023-09-23T23:21:52.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:52.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.509607+0000", 2023-09-23T23:21:52.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:21:52.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:52.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.509616+0000", 2023-09-23T23:21:52.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2749999999999995e-06 2023-09-23T23:21:52.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:52.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.509633+0000", 2023-09-23T23:21:52.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6799e-05 2023-09-23T23:21:52.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:52.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.509641+0000", 2023-09-23T23:21:52.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4360000000000002e-06 2023-09-23T23:21:52.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.509692+0000", 2023-09-23T23:21:52.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0918000000000003e-05 2023-09-23T23:21:52.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.509702+0000", 2023-09-23T23:21:52.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0227000000000001e-05 2023-09-23T23:21:52.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291329+0000", 2023-09-23T23:21:52.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.781626771 2023-09-23T23:21:52.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291337+0000", 2023-09-23T23:21:52.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0539999999999995e-06 2023-09-23T23:21:52.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.479463+0000", 2023-09-23T23:21:52.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1881260490000001 2023-09-23T23:21:52.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:52.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.479640+0000", 2023-09-23T23:21:52.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017681899999999999 2023-09-23T23:21:52.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:21:52.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.479809+0000", 2023-09-23T23:21:52.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000168968 2023-09-23T23:21:52.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492506+0000", 2023-09-23T23:21:52.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012697721 2023-09-23T23:21:52.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508609+0000", 2023-09-23T23:21:52.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016102103 2023-09-23T23:21:52.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:52.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.516984+0000", 2023-09-23T23:21:52.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083757399999999996 2023-09-23T23:21:52.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.679712+0000", 2023-09-23T23:21:52.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.162728014 2023-09-23T23:21:52.810 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.810 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:52.810 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.679758+0000", 2023-09-23T23:21:52.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5176999999999998e-05 2023-09-23T23:21:52.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:52.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.679878+0000", 2023-09-23T23:21:52.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012095900000000001 2023-09-23T23:21:52.811 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.811 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:52.811 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.811 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188555 9.a 9:532f4ecb:::benchmark_data_smithi110_295904_object77374:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:21:52.812 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.511168+0000", 2023-09-23T23:21:52.812 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 259.245250559, 2023-09-23T23:21:52.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.168916372, 2023-09-23T23:21:52.812 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:52.812 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:52.812 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:52.812 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:52.813 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:52.813 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188555 2023-09-23T23:21:52.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.813 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:52.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:52.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.511168+0000", 2023-09-23T23:21:52.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:52.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.511168+0000", 2023-09-23T23:21:52.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:52.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.511167+0000", 2023-09-23T23:21:52.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:21:52.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:52.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.511174+0000", 2023-09-23T23:21:52.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6320000000000002e-06 2023-09-23T23:21:52.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:52.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.511190+0000", 2023-09-23T23:21:52.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6449000000000001e-05 2023-09-23T23:21:52.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:52.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.511198+0000", 2023-09-23T23:21:52.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3270000000000001e-06 2023-09-23T23:21:52.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.511252+0000", 2023-09-23T23:21:52.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3882999999999999e-05 2023-09-23T23:21:52.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.511263+0000", 2023-09-23T23:21:52.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0502e-05 2023-09-23T23:21:52.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291361+0000", 2023-09-23T23:21:52.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.78009846000000005 2023-09-23T23:21:52.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291376+0000", 2023-09-23T23:21:52.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4992e-05 2023-09-23T23:21:52.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484374+0000", 2023-09-23T23:21:52.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.192997358 2023-09-23T23:21:52.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:52.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484535+0000", 2023-09-23T23:21:52.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000160972 2023-09-23T23:21:52.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.819 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:21:52.819 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484666+0000", 2023-09-23T23:21:52.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013110300000000001 2023-09-23T23:21:52.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.819 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492530+0000", 2023-09-23T23:21:52.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078639350000000007 2023-09-23T23:21:52.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508670+0000", 2023-09-23T23:21:52.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016140815999999999 2023-09-23T23:21:52.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:52.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.517035+0000", 2023-09-23T23:21:52.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083644519999999997 2023-09-23T23:21:52.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.679921+0000", 2023-09-23T23:21:52.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.162886436 2023-09-23T23:21:52.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:52.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.679967+0000", 2023-09-23T23:21:52.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5833e-05 2023-09-23T23:21:52.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:52.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.680085+0000", 2023-09-23T23:21:52.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117537 2023-09-23T23:21:52.824 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.824 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:52.824 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.824 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188563 9.a 9:50891094:::benchmark_data_smithi110_295904_object77382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:21:52.824 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.519697+0000", 2023-09-23T23:21:52.825 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 259.23672191899999, 2023-09-23T23:21:52.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.169432821, 2023-09-23T23:21:52.825 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:52.825 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:52.825 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:52.825 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:52.825 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:52.825 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188563 2023-09-23T23:21:52.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.825 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:52.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:52.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.519697+0000", 2023-09-23T23:21:52.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:52.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.519697+0000", 2023-09-23T23:21:52.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:52.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.519696+0000", 2023-09-23T23:21:52.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:21:52.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:52.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.519702+0000", 2023-09-23T23:21:52.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9110000000000004e-06 2023-09-23T23:21:52.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:52.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.519717+0000", 2023-09-23T23:21:52.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4688999999999999e-05 2023-09-23T23:21:52.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:52.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.519725+0000", 2023-09-23T23:21:52.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8550000000000002e-06 2023-09-23T23:21:52.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.524712+0000", 2023-09-23T23:21:52.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0049872980000000003 2023-09-23T23:21:52.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.524724+0000", 2023-09-23T23:21:52.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1639e-05 2023-09-23T23:21:52.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291402+0000", 2023-09-23T23:21:52.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.76667763499999997 2023-09-23T23:21:52.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291410+0000", 2023-09-23T23:21:52.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1519999999999992e-06 2023-09-23T23:21:52.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.488548+0000", 2023-09-23T23:21:52.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1971378480000001 2023-09-23T23:21:52.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:52.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.488682+0000", 2023-09-23T23:21:52.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000134857 2023-09-23T23:21:52.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:21:52.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.488774+0000", 2023-09-23T23:21:52.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2015999999999994e-05 2023-09-23T23:21:52.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508531+0000", 2023-09-23T23:21:52.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019756756 2023-09-23T23:21:52.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508729+0000", 2023-09-23T23:21:52.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019776599999999999 2023-09-23T23:21:52.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:52.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.517041+0000", 2023-09-23T23:21:52.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083118629999999992 2023-09-23T23:21:52.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.688958+0000", 2023-09-23T23:21:52.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.17191702 2023-09-23T23:21:52.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:52.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689015+0000", 2023-09-23T23:21:52.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6950999999999997e-05 2023-09-23T23:21:52.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:52.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689130+0000", 2023-09-23T23:21:52.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000114984 2023-09-23T23:21:52.833 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.833 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:52.833 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.833 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188579 9.a 9:5f580cc6:::benchmark_data_smithi110_295904_object77398:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:21:52.833 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.551569+0000", 2023-09-23T23:21:52.833 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 259.204849905, 2023-09-23T23:21:52.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1377349300000001, 2023-09-23T23:21:52.834 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:52.834 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:52.834 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:52.834 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:52.834 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:52.834 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188579 2023-09-23T23:21:52.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.834 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:52.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:52.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.551569+0000", 2023-09-23T23:21:52.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:52.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.551569+0000", 2023-09-23T23:21:52.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:52.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.551567+0000", 2023-09-23T23:21:52.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:21:52.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:52.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.551575+0000", 2023-09-23T23:21:52.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0220000000000004e-06 2023-09-23T23:21:52.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:52.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.551593+0000", 2023-09-23T23:21:52.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7995999999999998e-05 2023-09-23T23:21:52.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:52.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.551601+0000", 2023-09-23T23:21:52.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1300000000000001e-06 2023-09-23T23:21:52.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.551649+0000", 2023-09-23T23:21:52.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8032999999999999e-05 2023-09-23T23:21:52.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.551660+0000", 2023-09-23T23:21:52.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0773e-05 2023-09-23T23:21:52.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291436+0000", 2023-09-23T23:21:52.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.73977552599999996 2023-09-23T23:21:52.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291444+0000", 2023-09-23T23:21:52.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1329999999999999e-06 2023-09-23T23:21:52.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492557+0000", 2023-09-23T23:21:52.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.201113463 2023-09-23T23:21:52.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:52.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492654+0000", 2023-09-23T23:21:52.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6564999999999994e-05 2023-09-23T23:21:52.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:21:52.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492804+0000", 2023-09-23T23:21:52.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015046800000000001 2023-09-23T23:21:52.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508572+0000", 2023-09-23T23:21:52.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015767664000000001 2023-09-23T23:21:52.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508764+0000", 2023-09-23T23:21:52.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000191919 2023-09-23T23:21:52.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:52.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.517047+0000", 2023-09-23T23:21:52.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0082832940000000001 2023-09-23T23:21:52.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.841 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.841 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689167+0000", 2023-09-23T23:21:52.841 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.17211952 2023-09-23T23:21:52.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.841 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:52.841 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689205+0000", 2023-09-23T23:21:52.841 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8515e-05 2023-09-23T23:21:52.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.841 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:52.842 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689304+0000", 2023-09-23T23:21:52.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8722999999999997e-05 2023-09-23T23:21:52.842 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.842 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:52.842 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.842 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.842 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188599 9.a 9:586bac8c:::benchmark_data_smithi110_295904_object77418:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:21:52.842 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.599792+0000", 2023-09-23T23:21:52.842 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 259.15662738100002, 2023-09-23T23:21:52.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0896802550000002, 2023-09-23T23:21:52.842 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:52.842 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:52.843 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:52.843 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:52.843 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:52.843 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188599 2023-09-23T23:21:52.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.843 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:52.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:52.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.599792+0000", 2023-09-23T23:21:52.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:52.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.599792+0000", 2023-09-23T23:21:52.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.844 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:52.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.599790+0000", 2023-09-23T23:21:52.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:21:52.844 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:52.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.599799+0000", 2023-09-23T23:21:52.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1730000000000001e-06 2023-09-23T23:21:52.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:52.845 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.599814+0000", 2023-09-23T23:21:52.845 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5143000000000001e-05 2023-09-23T23:21:52.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:52.845 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.599823+0000", 2023-09-23T23:21:52.845 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4779999999999997e-06 2023-09-23T23:21:52.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.599896+0000", 2023-09-23T23:21:52.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2936999999999996e-05 2023-09-23T23:21:52.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.846 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.599909+0000", 2023-09-23T23:21:52.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2845e-05 2023-09-23T23:21:52.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.846 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291469+0000", 2023-09-23T23:21:52.847 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.69155943600000003 2023-09-23T23:21:52.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.847 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.847 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291476+0000", 2023-09-23T23:21:52.847 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8869999999999992e-06 2023-09-23T23:21:52.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.496771+0000", 2023-09-23T23:21:52.848 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2052949530000001 2023-09-23T23:21:52.848 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.848 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:52.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.496894+0000", 2023-09-23T23:21:52.848 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012217400000000001 2023-09-23T23:21:52.848 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.848 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:21:52.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.497021+0000", 2023-09-23T23:21:52.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012702899999999999 2023-09-23T23:21:52.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.849 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.849 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.849 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508642+0000", 2023-09-23T23:21:52.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011621154 2023-09-23T23:21:52.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:52.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.517053+0000", 2023-09-23T23:21:52.850 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084112340000000001 2023-09-23T23:21:52.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.530496+0000", 2023-09-23T23:21:52.850 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013442568 2023-09-23T23:21:52.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.851 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689338+0000", 2023-09-23T23:21:52.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.158842757 2023-09-23T23:21:52.851 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.851 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.851 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:52.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689376+0000", 2023-09-23T23:21:52.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7685000000000003e-05 2023-09-23T23:21:52.852 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.852 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.852 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:52.852 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689472+0000", 2023-09-23T23:21:52.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5816000000000006e-05 2023-09-23T23:21:52.852 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.852 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:52.852 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.852 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.852 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.853 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188633 9.a 9:5d39851e:::benchmark_data_smithi110_295904_object77452:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:21:52.853 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.683953+0000", 2023-09-23T23:21:52.853 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 259.072465405, 2023-09-23T23:21:52.853 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0056920740000002, 2023-09-23T23:21:52.853 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:52.853 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:52.853 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:52.854 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:52.854 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:52.854 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188633 2023-09-23T23:21:52.854 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.854 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:52.854 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:52.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.683953+0000", 2023-09-23T23:21:52.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.854 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.854 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.855 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:52.855 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.683953+0000", 2023-09-23T23:21:52.855 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.855 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.855 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.855 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:52.855 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.683952+0000", 2023-09-23T23:21:52.856 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:21:52.856 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.856 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.856 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:52.856 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.683960+0000", 2023-09-23T23:21:52.856 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.157e-06 2023-09-23T23:21:52.856 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.856 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.857 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:52.857 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.683976+0000", 2023-09-23T23:21:52.857 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6602000000000001e-05 2023-09-23T23:21:52.857 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.857 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.857 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:52.857 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.683985+0000", 2023-09-23T23:21:52.857 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3839999999999997e-06 2023-09-23T23:21:52.857 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.857 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.858 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.858 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.684042+0000", 2023-09-23T23:21:52.858 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7247999999999998e-05 2023-09-23T23:21:52.858 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.858 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.858 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.684053+0000", 2023-09-23T23:21:52.859 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1321999999999999e-05 2023-09-23T23:21:52.859 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.859 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291502+0000", 2023-09-23T23:21:52.859 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.60744907299999995 2023-09-23T23:21:52.859 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.859 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291510+0000", 2023-09-23T23:21:52.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0059999999999992e-06 2023-09-23T23:21:52.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.501330+0000", 2023-09-23T23:21:52.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.209819821 2023-09-23T23:21:52.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:52.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.501459+0000", 2023-09-23T23:21:52.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012884999999999999 2023-09-23T23:21:52.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:21:52.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.501594+0000", 2023-09-23T23:21:52.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013470999999999999 2023-09-23T23:21:52.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508698+0000", 2023-09-23T23:21:52.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071044080000000004 2023-09-23T23:21:52.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:52.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.517059+0000", 2023-09-23T23:21:52.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083607079999999997 2023-09-23T23:21:52.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.530597+0000", 2023-09-23T23:21:52.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013538197 2023-09-23T23:21:52.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689506+0000", 2023-09-23T23:21:52.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.158908456 2023-09-23T23:21:52.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:52.863 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689549+0000", 2023-09-23T23:21:52.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3043000000000001e-05 2023-09-23T23:21:52.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:52.863 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689646+0000", 2023-09-23T23:21:52.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6972999999999995e-05 2023-09-23T23:21:52.863 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.863 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:52.863 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.864 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.864 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.864 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188637 9.a 9:5fdc95bd:::benchmark_data_smithi110_295904_object77456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:21:52.864 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.693643+0000", 2023-09-23T23:21:52.864 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 259.062775961, 2023-09-23T23:21:52.864 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0039752800000001, 2023-09-23T23:21:52.864 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:52.864 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:52.864 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:21:52.864 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:21:52.864 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:21:52.864 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188637 2023-09-23T23:21:52.864 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.865 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:52.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:52.865 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.693643+0000", 2023-09-23T23:21:52.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.865 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:52.865 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.693643+0000", 2023-09-23T23:21:52.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.865 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:52.866 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.693641+0000", 2023-09-23T23:21:52.866 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:21:52.866 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.866 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.866 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:52.866 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.693648+0000", 2023-09-23T23:21:52.866 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2709999999999998e-06 2023-09-23T23:21:52.866 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.866 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.866 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:52.866 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.693660+0000", 2023-09-23T23:21:52.867 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2296999999999999e-05 2023-09-23T23:21:52.867 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.867 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.867 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:52.867 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.693666+0000", 2023-09-23T23:21:52.867 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1099999999999999e-06 2023-09-23T23:21:52.867 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.867 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.867 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.867 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.693733+0000", 2023-09-23T23:21:52.867 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7353000000000004e-05 2023-09-23T23:21:52.867 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.867 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.868 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.693747+0000", 2023-09-23T23:21:52.868 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3808999999999999e-05 2023-09-23T23:21:52.868 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.868 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291534+0000", 2023-09-23T23:21:52.868 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.59778706299999995 2023-09-23T23:21:52.868 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:21:52.868 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291556+0000", 2023-09-23T23:21:52.868 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1708999999999998e-05 2023-09-23T23:21:52.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.869 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.869 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.869 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508787+0000", 2023-09-23T23:21:52.869 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2172306980000001 2023-09-23T23:21:52.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.869 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.869 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:52.869 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508915+0000", 2023-09-23T23:21:52.869 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012779299999999999 2023-09-23T23:21:52.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.869 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.870 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:21:52.870 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.509041+0000", 2023-09-23T23:21:52.870 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012688700000000001 2023-09-23T23:21:52.870 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.870 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.870 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.870 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.517027+0000", 2023-09-23T23:21:52.870 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079852910000000003 2023-09-23T23:21:52.870 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.870 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.870 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:21:52.870 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.517064+0000", 2023-09-23T23:21:52.871 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7722000000000002e-05 2023-09-23T23:21:52.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.871 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.871 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.534755+0000", 2023-09-23T23:21:52.871 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017690251000000001 2023-09-23T23:21:52.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.871 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:21:52.871 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.697445+0000", 2023-09-23T23:21:52.871 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.16269038 2023-09-23T23:21:52.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.872 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.872 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:52.872 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.697499+0000", 2023-09-23T23:21:52.872 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3807000000000002e-05 2023-09-23T23:21:52.872 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.872 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.872 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:52.872 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.697618+0000", 2023-09-23T23:21:52.872 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011933 2023-09-23T23:21:52.872 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.872 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:52.872 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.872 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.873 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35529.0:9158 10.b e260/255 10:d91ff547:::benchmark_data_smithi110_366450_object9157:head v 260'558, mlcod=260'556)", 2023-09-23T23:21:52.873 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:19:13.619071+0000", 2023-09-23T23:21:52.873 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 159.13734794699999, 2023-09-23T23:21:52.873 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086555910000000007, 2023-09-23T23:21:52.873 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:52.873 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:52.873 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:52.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.873 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:52.873 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.619071+0000", 2023-09-23T23:21:52.873 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.874 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.874 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:52.874 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.619071+0000", 2023-09-23T23:21:52.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.874 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.874 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:52.874 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.619070+0000", 2023-09-23T23:21:52.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:21:52.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.874 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.875 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:52.875 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.619127+0000", 2023-09-23T23:21:52.875 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6745e-05 2023-09-23T23:21:52.875 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.875 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.875 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:52.875 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.619146+0000", 2023-09-23T23:21:52.875 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8882e-05 2023-09-23T23:21:52.875 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.875 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.875 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:52.875 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.619154+0000", 2023-09-23T23:21:52.875 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8789999999999992e-06 2023-09-23T23:21:52.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.619249+0000", 2023-09-23T23:21:52.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4442000000000001e-05 2023-09-23T23:21:52.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:52.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.619273+0000", 2023-09-23T23:21:52.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4216999999999999e-05 2023-09-23T23:21:52.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:52.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.627681+0000", 2023-09-23T23:21:52.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084078439999999994 2023-09-23T23:21:52.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.877 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:52.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.627727+0000", 2023-09-23T23:21:52.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5649000000000003e-05 2023-09-23T23:21:52.877 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.877 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:52.877 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.877 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35529.0:9161 10.b e260/255 10:d6cf8d91:::benchmark_data_smithi110_366450_object9160:head v 260'559, mlcod=260'556)", 2023-09-23T23:21:52.878 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:19:13.620863+0000", 2023-09-23T23:21:52.878 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 159.13555588899999, 2023-09-23T23:21:52.878 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070633919999999999, 2023-09-23T23:21:52.878 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:52.878 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:52.878 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:52.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:52.878 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.620863+0000", 2023-09-23T23:21:52.878 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.878 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.879 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:52.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.620863+0000", 2023-09-23T23:21:52.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.879 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.879 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:52.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.620862+0000", 2023-09-23T23:21:52.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:21:52.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.879 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.879 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:52.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.620909+0000", 2023-09-23T23:21:52.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7027000000000002e-05 2023-09-23T23:21:52.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.880 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:52.880 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.620925+0000", 2023-09-23T23:21:52.880 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.605e-05 2023-09-23T23:21:52.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.880 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:52.880 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.620933+0000", 2023-09-23T23:21:52.880 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4340000000000003e-06 2023-09-23T23:21:52.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.880 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.620977+0000", 2023-09-23T23:21:52.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4416000000000003e-05 2023-09-23T23:21:52.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.881 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.881 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:52.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.620998+0000", 2023-09-23T23:21:52.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1098e-05 2023-09-23T23:21:52.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.881 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.881 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:52.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.627852+0000", 2023-09-23T23:21:52.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068537490000000001 2023-09-23T23:21:52.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.882 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:52.882 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:13.627926+0000", 2023-09-23T23:21:52.882 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4400999999999995e-05 2023-09-23T23:21:52.882 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.882 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:52.882 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.882 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35529.0:11535 10.b e260/255 10:de0a3f12:::benchmark_data_smithi110_366450_object11534:head v 260'696, mlcod=260'693)", 2023-09-23T23:21:52.882 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:19:14.339194+0000", 2023-09-23T23:21:52.882 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 158.41722494300001, 2023-09-23T23:21:52.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.125612269, 2023-09-23T23:21:52.883 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:52.883 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:52.883 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:52.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:52.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.339194+0000", 2023-09-23T23:21:52.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:52.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.339194+0000", 2023-09-23T23:21:52.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.884 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.884 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:52.884 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.339194+0000", 2023-09-23T23:21:52.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:21:52.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:52.885 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.339223+0000", 2023-09-23T23:21:52.885 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8826000000000001e-05 2023-09-23T23:21:52.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:52.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.339230+0000", 2023-09-23T23:21:52.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1280000000000002e-06 2023-09-23T23:21:52.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:52.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.339233+0000", 2023-09-23T23:21:52.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.738e-06 2023-09-23T23:21:52.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.339684+0000", 2023-09-23T23:21:52.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045091899999999998 2023-09-23T23:21:52.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:52.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.339703+0000", 2023-09-23T23:21:52.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8898000000000001e-05 2023-09-23T23:21:52.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:52.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.340454+0000", 2023-09-23T23:21:52.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00075089700000000004 2023-09-23T23:21:52.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:52.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.464806+0000", 2023-09-23T23:21:52.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.124352113 2023-09-23T23:21:52.889 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.889 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:52.889 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.890 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35529.0:11539 10.b e260/255 10:d74eed84:::benchmark_data_smithi110_366450_object11538:head v 260'697, mlcod=260'693)", 2023-09-23T23:21:52.890 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:19:14.340345+0000", 2023-09-23T23:21:52.890 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 158.41607406399999, 2023-09-23T23:21:52.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12542535299999999, 2023-09-23T23:21:52.890 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:21:52.890 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:21:52.891 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:21:52.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:21:52.891 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.340345+0000", 2023-09-23T23:21:52.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:21:52.891 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.340345+0000", 2023-09-23T23:21:52.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:21:52.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:21:52.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.340344+0000", 2023-09-23T23:21:52.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:21:52.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:21:52.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.340417+0000", 2023-09-23T23:21:52.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2565999999999996e-05 2023-09-23T23:21:52.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:21:52.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.340429+0000", 2023-09-23T23:21:52.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2459e-05 2023-09-23T23:21:52.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:21:52.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.340436+0000", 2023-09-23T23:21:52.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3509999999999996e-06 2023-09-23T23:21:52.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:21:52.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.464834+0000", 2023-09-23T23:21:52.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.124397717 2023-09-23T23:21:52.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:21:52.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.464852+0000", 2023-09-23T23:21:52.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8298e-05 2023-09-23T23:21:52.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:21:52.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.465740+0000", 2023-09-23T23:21:52.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00088736999999999996 2023-09-23T23:21:52.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:21:52.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:21:52.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:21:52.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:19:14.465770+0000", 2023-09-23T23:21:52.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0318999999999999e-05 2023-09-23T23:21:52.895 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.895 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:52.895 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.895 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:21:52.895 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:21:52.895 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:21:52.895 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:52.895 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:21:52.928 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:52.978 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:21:52.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:52.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:21:53.058 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:53 smithi110 ceph-mon[212939]: pgmap v922: 99 pgs: 3 unknown, 4 remapped+peering, 9 peering, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 66 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 931 op/s; 5737/137283 objects degraded (4.179%); 19633/137283 objects misplaced (14.301%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T23:21:53.059 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:53 smithi110 ceph-mon[212939]: failed to encode map e303 with expected crc 2023-09-23T23:21:53.059 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:53 smithi110 ceph-mon[212939]: failed to encode map e303 with expected crc 2023-09-23T23:21:53.059 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:53 smithi110 ceph-mon[212939]: failed to encode map e303 with expected crc 2023-09-23T23:21:53.059 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:53 smithi110 ceph-mon[212939]: failed to encode map e303 with expected crc 2023-09-23T23:21:53.059 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:53 smithi110 ceph-mon[212939]: failed to encode map e303 with expected crc 2023-09-23T23:21:53.059 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:53 smithi110 ceph-mon[212939]: osdmap e304: 8 total, 7 up, 7 in 2023-09-23T23:21:53.059 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:53 smithi110 ceph-mon[221102]: pgmap v922: 99 pgs: 3 unknown, 4 remapped+peering, 9 peering, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 66 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 931 op/s; 5737/137283 objects degraded (4.179%); 19633/137283 objects misplaced (14.301%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T23:21:53.059 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:53 smithi110 ceph-mon[221102]: failed to encode map e303 with expected crc 2023-09-23T23:21:53.059 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:53 smithi110 ceph-mon[221102]: failed to encode map e303 with expected crc 2023-09-23T23:21:53.059 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:53 smithi110 ceph-mon[221102]: failed to encode map e303 with expected crc 2023-09-23T23:21:53.060 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:53 smithi110 ceph-mon[221102]: failed to encode map e303 with expected crc 2023-09-23T23:21:53.060 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:53 smithi110 ceph-mon[221102]: failed to encode map e303 with expected crc 2023-09-23T23:21:53.060 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:53 smithi110 ceph-mon[221102]: osdmap e304: 8 total, 7 up, 7 in 2023-09-23T23:21:53.083 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:53.109 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:53.151 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:53.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:53.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:53.259 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:53.291 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:53.339 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:53.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:53.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:53.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:53 smithi138 ceph-mon[193606]: pgmap v922: 99 pgs: 3 unknown, 4 remapped+peering, 9 peering, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 66 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 931 op/s; 5737/137283 objects degraded (4.179%); 19633/137283 objects misplaced (14.301%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T23:21:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:53 smithi138 ceph-mon[193606]: failed to encode map e303 with expected crc 2023-09-23T23:21:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:53 smithi138 ceph-mon[193606]: failed to encode map e303 with expected crc 2023-09-23T23:21:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:53 smithi138 ceph-mon[193606]: failed to encode map e303 with expected crc 2023-09-23T23:21:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:53 smithi138 ceph-mon[193606]: failed to encode map e303 with expected crc 2023-09-23T23:21:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:53 smithi138 ceph-mon[193606]: failed to encode map e303 with expected crc 2023-09-23T23:21:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:53 smithi138 ceph-mon[193606]: osdmap e304: 8 total, 7 up, 7 in 2023-09-23T23:21:53.442 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:53.469 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:53.522 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:53.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:53.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:53.628 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:53.678 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:53.970 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:53.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:53.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:54.074 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:21:54.109 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:54.148 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:21:54.228 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:54 smithi110 ceph-mon[212939]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.228 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:54 smithi110 ceph-mon[212939]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.228 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:54 smithi110 ceph-mon[212939]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.228 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:54 smithi110 ceph-mon[212939]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.228 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:54 smithi110 ceph-mon[212939]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.229 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:54 smithi110 ceph-mon[212939]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.229 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:54 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 5737/137283 objects degraded (4.179%), 3 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:54.229 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:54 smithi110 ceph-mon[221102]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.229 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:54 smithi110 ceph-mon[221102]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.229 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:54 smithi110 ceph-mon[221102]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.229 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:54 smithi110 ceph-mon[221102]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.229 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:54 smithi110 ceph-mon[221102]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.229 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:54 smithi110 ceph-mon[221102]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.229 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:54 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 5737/137283 objects degraded (4.179%), 3 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:54.329 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:21:54.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:54.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:21:54.434 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:21:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:54 smithi138 ceph-mon[193606]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:54 smithi138 ceph-mon[193606]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:54 smithi138 ceph-mon[193606]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:54 smithi138 ceph-mon[193606]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:54 smithi138 ceph-mon[193606]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:54 smithi138 ceph-mon[193606]: failed to encode map e304 with expected crc 2023-09-23T23:21:54.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:54 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 5737/137283 objects degraded (4.179%), 3 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:54.492 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:54.764 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:21:54.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:54.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:21:54.871 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:54.897 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:54.941 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:54.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:54.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:55.048 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:21:55.060 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:55 smithi110 ceph-mon[212939]: pgmap v925: 99 pgs: 3 unknown, 4 remapped+peering, 9 peering, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 65 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 5737/137283 objects degraded (4.179%); 19633/137283 objects misplaced (14.301%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T23:21:55.060 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:55 smithi110 ceph-mon[221102]: pgmap v925: 99 pgs: 3 unknown, 4 remapped+peering, 9 peering, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 65 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 5737/137283 objects degraded (4.179%); 19633/137283 objects misplaced (14.301%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T23:21:55.093 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:55.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:55 smithi138 ceph-mon[193606]: pgmap v925: 99 pgs: 3 unknown, 4 remapped+peering, 9 peering, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 65 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 5737/137283 objects degraded (4.179%); 19633/137283 objects misplaced (14.301%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T23:21:55.611 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:21:55.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:55.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:21:55.715 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:55.743 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:55.803 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:55.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:55.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:55.910 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:55.936 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:55.990 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:55.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:55.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:56.097 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:21:56.123 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:56.165 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:21:56.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:56.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:21:56.270 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:21:56.304 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:56 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 4 pgs peering) 2023-09-23T23:21:56.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:56 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 4 pgs peering) 2023-09-23T23:21:56.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:56 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 4 pgs peering) 2023-09-23T23:21:56.652 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:21:56.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:56.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:21:56.759 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:56.786 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:56.832 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:56.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:56.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:56.850 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:21:56.918 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:21:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:21:56] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.43.0" 2023-09-23T23:21:56.921 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 2] out_osds: [1] dead_osds: [7] live_osds: [0, 1, 3, 4, 5, 6, 2] 2023-09-23T23:21:56.921 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:21:56.921 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T23:21:56.921 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:21:56.939 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:57.040 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:57.178 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:21:57.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:57.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:21:57.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:57 smithi110 ceph-mon[212939]: pgmap v926: 99 pgs: 1 active+recovery_wait+degraded, 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 108 op/s; 13591/135486 objects degraded (10.031%); 27330/135486 objects misplaced (20.172%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T23:21:57.191 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:57 smithi110 ceph-mon[221102]: pgmap v926: 99 pgs: 1 active+recovery_wait+degraded, 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 108 op/s; 13591/135486 objects degraded (10.031%); 27330/135486 objects misplaced (20.172%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T23:21:57.280 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:21:57.292 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:21:57.310 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:21:57.344 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:21:57.368 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:57.385 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:21:57.418 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:21:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:57 smithi138 ceph-mon[193606]: pgmap v926: 99 pgs: 1 active+recovery_wait+degraded, 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 108 op/s; 13591/135486 objects degraded (10.031%); 27330/135486 objects misplaced (20.172%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T23:21:57.452 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:21:57.489 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:21:57.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:21:57.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:21:57.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:21:57.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:21:57.491 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:21:57.491 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:21:57.491 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:21:57.491 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:21:58.144 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:58 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2328430399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:58.144 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:58 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/636999039' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:21:58.144 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:21:58.149 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:21:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:58 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2328430399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:58 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/636999039' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:21:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:21:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:58 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2328430399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:21:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:58 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/636999039' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:21:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:21:58.735 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:21:58.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:58.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:21:58.840 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:58.868 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:58.914 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:58.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:58.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:59.025 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:59.057 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:59.123 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:59.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:59.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:59.141 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:59 smithi110 ceph-mon[221102]: pgmap v927: 99 pgs: 1 active+recovery_wait+degraded, 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 92 op/s; 13591/135486 objects degraded (10.031%); 27330/135486 objects misplaced (20.172%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T23:21:59.142 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:59 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:21:59.142 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:59 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/636999039' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:21:59.142 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:59 smithi110 ceph-mon[221102]: osdmap e305: 8 total, 7 up, 7 in 2023-09-23T23:21:59.142 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:59 smithi110 ceph-mon[221102]: failed to encode map e305 with expected crc 2023-09-23T23:21:59.142 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:59 smithi110 ceph-mon[221102]: failed to encode map e305 with expected crc 2023-09-23T23:21:59.142 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:21:59 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 13591/135486 objects degraded (10.031%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:59.232 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:21:59.258 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:59.300 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:21:59.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:59.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:21:59.407 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:21:59.430 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:59 smithi138 ceph-mon[193606]: pgmap v927: 99 pgs: 1 active+recovery_wait+degraded, 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 92 op/s; 13591/135486 objects degraded (10.031%); 27330/135486 objects misplaced (20.172%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T23:21:59.431 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:59 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:21:59.431 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:59 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/636999039' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:21:59.431 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:59 smithi138 ceph-mon[193606]: osdmap e305: 8 total, 7 up, 7 in 2023-09-23T23:21:59.431 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:59 smithi138 ceph-mon[193606]: failed to encode map e305 with expected crc 2023-09-23T23:21:59.431 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:59 smithi138 ceph-mon[193606]: failed to encode map e305 with expected crc 2023-09-23T23:21:59.431 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:21:59 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 13591/135486 objects degraded (10.031%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:59.433 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:21:59.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:59 smithi110 ceph-mon[212939]: pgmap v927: 99 pgs: 1 active+recovery_wait+degraded, 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 92 op/s; 13591/135486 objects degraded (10.031%); 27330/135486 objects misplaced (20.172%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T23:21:59.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:59 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:21:59.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:59 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/636999039' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:21:59.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:59 smithi110 ceph-mon[212939]: osdmap e305: 8 total, 7 up, 7 in 2023-09-23T23:21:59.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:59 smithi110 ceph-mon[212939]: failed to encode map e305 with expected crc 2023-09-23T23:21:59.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:59 smithi110 ceph-mon[212939]: failed to encode map e305 with expected crc 2023-09-23T23:21:59.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:21:59 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 13591/135486 objects degraded (10.031%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T23:21:59.475 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:21:59.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:21:59.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:21:59.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:21:59.634 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:00.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:00 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:22:00.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:00 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:22:00.334 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:22:00.335 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:22:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:00 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:22:00.486 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:00.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:00.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:00.592 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:00.615 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:00.622 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:00.670 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:00.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:00.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:00.780 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:22:00.813 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:00.876 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:22:00.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:00.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:22:00.953 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:22:00.953 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:22:00.981 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:01.008 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:01.051 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:01.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:01.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:01.155 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:01.181 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:01.227 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:01.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:01.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:01.335 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:01.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:01 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:22:01.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:01 smithi138 ceph-mon[193606]: pgmap v929: 99 pgs: 1 active+recovery_wait+degraded, 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 157 op/s; 13591/133725 objects degraded (10.163%); 26961/133725 objects misplaced (20.162%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T23:22:01.361 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:01.406 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:01.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:01.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:01.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:01 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:22:01.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:01 smithi110 ceph-mon[212939]: pgmap v929: 99 pgs: 1 active+recovery_wait+degraded, 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 157 op/s; 13591/133725 objects degraded (10.163%); 26961/133725 objects misplaced (20.162%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T23:22:01.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:01 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:22:01.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:01 smithi110 ceph-mon[221102]: pgmap v929: 99 pgs: 1 active+recovery_wait+degraded, 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 157 op/s; 13591/133725 objects degraded (10.163%); 26961/133725 objects misplaced (20.162%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T23:22:01.513 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:01.538 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:01.583 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:01.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:01.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:01.689 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:01.715 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:01.756 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:01.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:01.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:01.862 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:01.898 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:02.307 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:02.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:02.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:02.336 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:22:02.412 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:02.438 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:02.658 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:22:02.932 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:02.935 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:22:02.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:02.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:03.042 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:03.070 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:03.118 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:03.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:03.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:03.220 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:03.223 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:03.250 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:03.300 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:03.304 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:03.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:03.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:03.381 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:03 smithi110 ceph-mon[212939]: pgmap v930: 99 pgs: 1 active+recovery_wait+degraded, 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 254 op/s; 13217/130788 objects degraded (10.106%); 25864/130788 objects misplaced (19.776%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T23:22:03.384 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:03 smithi110 ceph-mon[221102]: pgmap v930: 99 pgs: 1 active+recovery_wait+degraded, 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 254 op/s; 13217/130788 objects degraded (10.106%); 25864/130788 objects misplaced (19.776%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T23:22:03.409 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:03.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:03 smithi138 ceph-mon[193606]: pgmap v930: 99 pgs: 1 active+recovery_wait+degraded, 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 254 op/s; 13217/130788 objects degraded (10.106%); 25864/130788 objects misplaced (19.776%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T23:22:03.464 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:03.845 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:03.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:03.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:03.952 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:03.980 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:04.020 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:22:04.022 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:22:04.034 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:04.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:04.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:04.140 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:04.199 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:04.334 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:04 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 13217/130788 objects degraded (10.106%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:04.335 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:04 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 13217/130788 objects degraded (10.106%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:04.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:04 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 13217/130788 objects degraded (10.106%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:05.401 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:05 smithi110 ceph-mon[212939]: pgmap v931: 99 pgs: 1 active+recovery_wait+degraded, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 216 op/s; 13217/130788 objects degraded (10.106%); 25864/130788 objects misplaced (19.776%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:22:05.402 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:05 smithi110 ceph-mon[212939]: from='client.35727 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:22:05.406 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:05 smithi110 ceph-mon[221102]: pgmap v931: 99 pgs: 1 active+recovery_wait+degraded, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 216 op/s; 13217/130788 objects degraded (10.106%); 25864/130788 objects misplaced (19.776%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:22:05.406 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:05 smithi110 ceph-mon[221102]: from='client.35727 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:22:05.422 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:05.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:05.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:05.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:05 smithi138 ceph-mon[193606]: pgmap v931: 99 pgs: 1 active+recovery_wait+degraded, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 216 op/s; 13217/130788 objects degraded (10.106%); 25864/130788 objects misplaced (19.776%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:22:05.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:05 smithi138 ceph-mon[193606]: from='client.35727 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:22:05.530 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:22:05.565 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:06.275 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":931,"stamp":"2023-09-23T23:22:03.542356+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2856106756,"num_objects":43596,"num_object_clones":0,"num_object_copies":130788,"num_objects_missing_on_primary":5116,"num_objects_missing":5116,"num_objects_degraded":13217,"num_objects_misplaced":25864,"num_objects_unfound":0,"num_objects_dirty":43596,"num_whiteouts":0,"num_read":91,"num_read_kb":168,"num_write":158945,"num_write_kb":6477589,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6259,"num_bytes_recovered":409994053,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83255,"ondisk_log_size":83255,"up":273,"acting":289,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":317,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12102740,"kb_used_data":8792668,"kb_used_omap":16,"kb_used_meta":3309999,"kb_avail":644113324,"statfs":{"total":671965249536,"available":659572043776,"internally_reserved":0,"allocated":9003692032,"data_stored":8996501189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16640,"internal_metadata":3389439744},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,1,0,0,0,2,0,0,4,4,8],"upper_bound":2048},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-141885440,"num_objects":-2165,"num_object_clones":0,"num_object_copies":-6495,"num_objects_missing_on_primary":5116,"num_objects_missing":5116,"num_objects_degraded":7480,"num_objects_misplaced":6231,"num_objects_unfound":0,"num_objects_dirty":-2165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2165,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1300,"num_bytes_recovered":85004892,"num_keys_recovered":-10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003695"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":330,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.074283+0000","last_change":"2023-09-23T23:20:15.786245+0000","last_active":"2023-09-23T23:21:53.074283+0000","last_peered":"2023-09-23T23:21:53.074283+0000","last_clean":"2023-09-23T23:21:53.074283+0000","last_became_active":"2023-09-23T23:20:15.785657+0000","last_became_peered":"2023-09-23T23:20:15.785657+0000","last_unstale":"2023-09-23T23:21:53.074283+0000","last_undegraded":"2023-09-23T23:21:53.074283+0000","last_fullsized":"2023-09-23T23:21:53.074283+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:44.141283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":255,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.039487+0000","last_change":"2023-09-23T23:19:17.015302+0000","last_active":"2023-09-23T23:21:53.039487+0000","last_peered":"2023-09-23T23:21:53.039487+0000","last_clean":"2023-09-23T23:21:53.039487+0000","last_became_active":"2023-09-23T23:19:17.014986+0000","last_became_peered":"2023-09-23T23:19:17.014986+0000","last_unstale":"2023-09-23T23:21:53.039487+0000","last_undegraded":"2023-09-23T23:21:53.039487+0000","last_fullsized":"2023-09-23T23:21:53.039487+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:57.782719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":172,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.038891+0000","last_change":"2023-09-23T23:21:50.045549+0000","last_active":"2023-09-23T23:21:53.038891+0000","last_peered":"2023-09-23T23:21:53.038891+0000","last_clean":"2023-09-23T23:21:53.038891+0000","last_became_active":"2023-09-23T23:21:50.045215+0000","last_became_peered":"2023-09-23T23:21:50.045215+0000","last_unstale":"2023-09-23T23:21:53.038891+0000","last_undegraded":"2023-09-23T23:21:53.038891+0000","last_fullsized":"2023-09-23T23:21:53.038891+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:01.093934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":294,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.039100+0000","last_change":"2023-09-23T23:20:35.383019+0000","last_active":"2023-09-23T23:21:53.039100+0000","last_peered":"2023-09-23T23:21:53.039100+0000","last_clean":"2023-09-23T23:21:53.039100+0000","last_became_active":"2023-09-23T23:20:35.381937+0000","last_became_peered":"2023-09-23T23:20:35.381937+0000","last_unstale":"2023-09-23T23:21:53.039100+0000","last_undegraded":"2023-09-23T23:21:53.039100+0000","last_fullsized":"2023-09-23T23:21:53.039100+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:37.764701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":291,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.038258+0000","last_change":"2023-09-23T23:20:15.738434+0000","last_active":"2023-09-23T23:21:53.038258+0000","last_peered":"2023-09-23T23:21:53.038258+0000","last_clean":"2023-09-23T23:21:53.038258+0000","last_became_active":"2023-09-23T23:20:15.738195+0000","last_became_peered":"2023-09-23T23:20:15.738195+0000","last_unstale":"2023-09-23T23:21:53.038258+0000","last_undegraded":"2023-09-23T23:21:53.038258+0000","last_fullsized":"2023-09-23T23:21:53.038258+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:15.224045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":275,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.037724+0000","last_change":"2023-09-23T23:19:17.015076+0000","last_active":"2023-09-23T23:21:53.037724+0000","last_peered":"2023-09-23T23:21:53.037724+0000","last_clean":"2023-09-23T23:21:53.037724+0000","last_became_active":"2023-09-23T23:19:17.014808+0000","last_became_peered":"2023-09-23T23:19:17.014808+0000","last_unstale":"2023-09-23T23:21:53.037724+0000","last_undegraded":"2023-09-23T23:21:53.037724+0000","last_fullsized":"2023-09-23T23:21:53.037724+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:14.865292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":257,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.038738+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:21:53.038738+0000","last_peered":"2023-09-23T23:21:53.038738+0000","last_clean":"2023-09-23T23:21:53.038738+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:21:53.038738+0000","last_undegraded":"2023-09-23T23:21:53.038738+0000","last_fullsized":"2023-09-23T23:21:53.038738+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":264,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.038182+0000","last_change":"2023-09-23T23:21:50.045655+0000","last_active":"2023-09-23T23:21:53.038182+0000","last_peered":"2023-09-23T23:21:53.038182+0000","last_clean":"2023-09-23T23:21:53.038182+0000","last_became_active":"2023-09-23T23:21:50.045234+0000","last_became_peered":"2023-09-23T23:21:50.045234+0000","last_unstale":"2023-09-23T23:21:53.038182+0000","last_undegraded":"2023-09-23T23:21:53.038182+0000","last_fullsized":"2023-09-23T23:21:53.038182+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:58:57.630594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":45,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.036255+0000","last_change":"2023-09-23T23:20:26.841909+0000","last_active":"2023-09-23T23:21:53.036255+0000","last_peered":"2023-09-23T23:21:53.036255+0000","last_clean":"2023-09-23T23:21:53.036255+0000","last_became_active":"2023-09-23T23:20:26.841631+0000","last_became_peered":"2023-09-23T23:20:26.841631+0000","last_unstale":"2023-09-23T23:21:53.036255+0000","last_undegraded":"2023-09-23T23:21:53.036255+0000","last_fullsized":"2023-09-23T23:21:53.036255+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:07.380539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":234,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.038600+0000","last_change":"2023-09-23T23:16:56.528057+0000","last_active":"2023-09-23T23:21:53.038600+0000","last_peered":"2023-09-23T23:21:53.038600+0000","last_clean":"2023-09-23T23:21:53.038600+0000","last_became_active":"2023-09-23T23:16:56.527755+0000","last_became_peered":"2023-09-23T23:16:56.527755+0000","last_unstale":"2023-09-23T23:21:53.038600+0000","last_undegraded":"2023-09-23T23:21:53.038600+0000","last_fullsized":"2023-09-23T23:21:53.038600+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:47.396406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":123,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.039884+0000","last_change":"2023-09-23T23:16:56.567128+0000","last_active":"2023-09-23T23:21:53.039884+0000","last_peered":"2023-09-23T23:21:53.039884+0000","last_clean":"2023-09-23T23:21:53.039884+0000","last_became_active":"2023-09-23T23:16:56.551484+0000","last_became_peered":"2023-09-23T23:16:56.551484+0000","last_unstale":"2023-09-23T23:21:53.039884+0000","last_undegraded":"2023-09-23T23:21:53.039884+0000","last_fullsized":"2023-09-23T23:21:53.039884+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:14.981307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":255,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.037084+0000","last_change":"2023-09-23T23:20:15.781637+0000","last_active":"2023-09-23T23:21:53.037084+0000","last_peered":"2023-09-23T23:21:53.037084+0000","last_clean":"2023-09-23T23:21:53.037084+0000","last_became_active":"2023-09-23T23:20:15.781308+0000","last_became_peered":"2023-09-23T23:20:15.781308+0000","last_unstale":"2023-09-23T23:21:53.037084+0000","last_undegraded":"2023-09-23T23:21:53.037084+0000","last_fullsized":"2023-09-23T23:21:53.037084+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:47.963872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":75,"reported_epoch":304,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:21:53.037385+0000","last_change":"2023-09-23T23:21:51.058999+0000","last_active":"2023-09-23T23:21:53.037385+0000","last_peered":"2023-09-23T23:21:53.037385+0000","last_clean":"2023-09-23T23:21:53.037385+0000","last_became_active":"2023-09-23T23:21:51.044013+0000","last_became_peered":"2023-09-23T23:21:51.044013+0000","last_unstale":"2023-09-23T23:21:53.037385+0000","last_undegraded":"2023-09-23T23:21:53.037385+0000","last_fullsized":"2023-09-23T23:21:53.037385+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:47.100334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2,4],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":274,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.074589+0000","last_change":"2023-09-23T23:19:17.017877+0000","last_active":"2023-09-23T23:21:53.074589+0000","last_peered":"2023-09-23T23:21:53.074589+0000","last_clean":"2023-09-23T23:21:53.074589+0000","last_became_active":"2023-09-23T23:19:17.017590+0000","last_became_peered":"2023-09-23T23:19:17.017590+0000","last_unstale":"2023-09-23T23:21:53.074589+0000","last_undegraded":"2023-09-23T23:21:53.074589+0000","last_fullsized":"2023-09-23T23:21:53.074589+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:41.420908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":63,"reported_epoch":304,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:21:54.612144+0000","last_change":"2023-09-23T23:21:51.055871+0000","last_active":"2023-09-23T23:21:54.612144+0000","last_peered":"2023-09-23T23:21:54.612144+0000","last_clean":"2023-09-23T23:21:54.612144+0000","last_became_active":"2023-09-23T23:21:51.055372+0000","last_became_peered":"2023-09-23T23:21:51.055372+0000","last_unstale":"2023-09-23T23:21:54.612144+0000","last_undegraded":"2023-09-23T23:21:54.612144+0000","last_fullsized":"2023-09-23T23:21:54.612144+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:11.929868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":280,"reported_epoch":304,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:21:53.039224+0000","last_change":"2023-09-23T23:20:16.730841+0000","last_active":"2023-09-23T23:21:53.039224+0000","last_peered":"2023-09-23T23:21:53.039224+0000","last_clean":"2023-09-23T23:21:53.039224+0000","last_became_active":"2023-09-23T23:20:16.730440+0000","last_became_peered":"2023-09-23T23:20:16.730440+0000","last_unstale":"2023-09-23T23:21:53.039224+0000","last_undegraded":"2023-09-23T23:21:53.039224+0000","last_fullsized":"2023-09-23T23:21:53.039224+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:12.839390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,5],"avail_no_missing":["3","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":36,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.036201+0000","last_change":"2023-09-23T23:20:15.781738+0000","last_active":"2023-09-23T23:21:53.036201+0000","last_peered":"2023-09-23T23:21:53.036201+0000","last_clean":"2023-09-23T23:21:53.036201+0000","last_became_active":"2023-09-23T23:20:15.781394+0000","last_became_peered":"2023-09-23T23:20:15.781394+0000","last_unstale":"2023-09-23T23:21:53.036201+0000","last_undegraded":"2023-09-23T23:21:53.036201+0000","last_fullsized":"2023-09-23T23:21:53.036201+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:04.696118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":162,"reported_epoch":304,"state":"active+undersized","last_fresh":"2023-09-23T23:21:53.039597+0000","last_change":"2023-09-23T23:18:59.586926+0000","last_active":"2023-09-23T23:21:53.039597+0000","last_peered":"2023-09-23T23:21:53.039597+0000","last_clean":"2023-09-23T23:10:34.157578+0000","last_became_active":"2023-09-23T23:18:59.586926+0000","last_became_peered":"2023-09-23T23:18:59.586926+0000","last_unstale":"2023-09-23T23:21:53.039597+0000","last_undegraded":"2023-09-23T23:21:53.039597+0000","last_fullsized":"2023-09-23T23:18:59.579446+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:29.301032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":58,"reported_epoch":304,"state":"active+undersized","last_fresh":"2023-09-23T23:21:53.074330+0000","last_change":"2023-09-23T23:18:59.587971+0000","last_active":"2023-09-23T23:21:53.074330+0000","last_peered":"2023-09-23T23:21:53.074330+0000","last_clean":"2023-09-23T22:53:15.863481+0000","last_became_active":"2023-09-23T23:18:59.587971+0000","last_became_peered":"2023-09-23T23:18:59.587971+0000","last_unstale":"2023-09-23T23:21:53.074330+0000","last_undegraded":"2023-09-23T23:21:53.074330+0000","last_fullsized":"2023-09-23T23:18:59.584213+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:17.017106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":284,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.039244+0000","last_change":"2023-09-23T23:20:15.737685+0000","last_active":"2023-09-23T23:21:53.039244+0000","last_peered":"2023-09-23T23:21:53.039244+0000","last_clean":"2023-09-23T23:21:53.039244+0000","last_became_active":"2023-09-23T23:20:15.737311+0000","last_became_peered":"2023-09-23T23:20:15.737311+0000","last_unstale":"2023-09-23T23:21:53.039244+0000","last_undegraded":"2023-09-23T23:21:53.039244+0000","last_fullsized":"2023-09-23T23:21:53.039244+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:15:21.213270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":294,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.036952+0000","last_change":"2023-09-23T23:13:29.368175+0000","last_active":"2023-09-23T23:21:53.036952+0000","last_peered":"2023-09-23T23:21:53.036952+0000","last_clean":"2023-09-23T23:21:53.036952+0000","last_became_active":"2023-09-23T23:13:29.363603+0000","last_became_peered":"2023-09-23T23:13:29.363603+0000","last_unstale":"2023-09-23T23:21:53.036952+0000","last_undegraded":"2023-09-23T23:21:53.036952+0000","last_fullsized":"2023-09-23T23:21:53.036952+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:25.277871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":36,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.036459+0000","last_change":"2023-09-23T23:20:15.786871+0000","last_active":"2023-09-23T23:21:53.036459+0000","last_peered":"2023-09-23T23:21:53.036459+0000","last_clean":"2023-09-23T23:21:53.036459+0000","last_became_active":"2023-09-23T23:20:15.786556+0000","last_became_peered":"2023-09-23T23:20:15.786556+0000","last_unstale":"2023-09-23T23:21:53.036459+0000","last_undegraded":"2023-09-23T23:21:53.036459+0000","last_fullsized":"2023-09-23T23:21:53.036459+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:33.978101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.6","version":"305'9974","reported_seq":10040,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T23:21:59.936434+0000","last_change":"2023-09-23T23:21:22.237069+0000","last_active":"2023-09-23T23:21:59.936434+0000","last_peered":"2023-09-23T23:21:59.936434+0000","last_clean":"2023-09-23T23:21:59.936434+0000","last_became_active":"2023-09-23T23:21:22.194375+0000","last_became_peered":"2023-09-23T23:21:22.194375+0000","last_unstale":"2023-09-23T23:21:59.936434+0000","last_undegraded":"2023-09-23T23:21:59.936434+0000","last_fullsized":"2023-09-23T23:21:59.936434+0000","mapping_epoch":295,"log_start":"277'5086","ondisk_log_start":"277'5086","created":255,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4888,"log_dups_size":0,"ondisk_log_size":4888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:56:59.831180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174850048,"num_objects":2668,"num_object_clones":0,"num_object_copies":8004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9974,"num_write_kb":404544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":119,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.074770+0000","last_change":"2023-09-23T23:16:56.548535+0000","last_active":"2023-09-23T23:21:53.074770+0000","last_peered":"2023-09-23T23:21:53.074770+0000","last_clean":"2023-09-23T23:21:53.074770+0000","last_became_active":"2023-09-23T23:16:56.546115+0000","last_became_peered":"2023-09-23T23:16:56.546115+0000","last_unstale":"2023-09-23T23:21:53.074770+0000","last_undegraded":"2023-09-23T23:21:53.074770+0000","last_fullsized":"2023-09-23T23:21:53.074770+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:02.279715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":110,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.037021+0000","last_change":"2023-09-23T23:16:56.525479+0000","last_active":"2023-09-23T23:21:53.037021+0000","last_peered":"2023-09-23T23:21:53.037021+0000","last_clean":"2023-09-23T23:21:53.037021+0000","last_became_active":"2023-09-23T23:16:56.525147+0000","last_became_peered":"2023-09-23T23:16:56.525147+0000","last_unstale":"2023-09-23T23:21:53.037021+0000","last_undegraded":"2023-09-23T23:21:53.037021+0000","last_fullsized":"2023-09-23T23:21:53.037021+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:06.165300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.7","version":"305'9807","reported_seq":9886,"reported_epoch":305,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:21:59.233775+0000","last_change":"2023-09-23T23:20:16.868499+0000","last_active":"2023-09-23T23:21:59.233775+0000","last_peered":"2023-09-23T23:21:59.233775+0000","last_clean":"2023-09-23T23:20:14.713000+0000","last_became_active":"2023-09-23T23:20:16.845351+0000","last_became_peered":"2023-09-23T23:20:16.845351+0000","last_unstale":"2023-09-23T23:21:59.233775+0000","last_undegraded":"2023-09-23T23:21:59.233775+0000","last_fullsized":"2023-09-23T23:21:59.233775+0000","mapping_epoch":280,"log_start":"276'4602","ondisk_log_start":"276'4602","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":5205,"log_dups_size":0,"ondisk_log_size":5205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:02.881237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178323456,"num_objects":2721,"num_object_clones":0,"num_object_copies":8163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2720,"num_objects_unfound":0,"num_objects_dirty":2721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9807,"num_write_kb":400896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,4,0],"avail_no_missing":["6","0","4"],"object_location_counts":[{"shards":"0,4,6","objects":2721}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":35,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.038800+0000","last_change":"2023-09-23T23:21:51.064202+0000","last_active":"2023-09-23T23:21:53.038800+0000","last_peered":"2023-09-23T23:21:53.038800+0000","last_clean":"2023-09-23T23:21:53.038800+0000","last_became_active":"2023-09-23T23:21:50.039929+0000","last_became_peered":"2023-09-23T23:21:50.039929+0000","last_unstale":"2023-09-23T23:21:53.038800+0000","last_undegraded":"2023-09-23T23:21:53.038800+0000","last_fullsized":"2023-09-23T23:21:53.038800+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:00.184800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":301,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.074526+0000","last_change":"2023-09-23T23:19:17.017714+0000","last_active":"2023-09-23T23:21:53.074526+0000","last_peered":"2023-09-23T23:21:53.074526+0000","last_clean":"2023-09-23T23:21:53.074526+0000","last_became_active":"2023-09-23T23:19:17.017400+0000","last_became_peered":"2023-09-23T23:19:17.017400+0000","last_unstale":"2023-09-23T23:21:53.074526+0000","last_undegraded":"2023-09-23T23:21:53.074526+0000","last_fullsized":"2023-09-23T23:21:53.074526+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:43.604654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"305'9843","reported_seq":9903,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T23:21:59.942609+0000","last_change":"2023-09-23T23:18:58.585739+0000","last_active":"2023-09-23T23:21:59.942609+0000","last_peered":"2023-09-23T23:21:59.942609+0000","last_clean":"2023-09-23T23:21:59.942609+0000","last_became_active":"2023-09-23T23:18:58.585475+0000","last_became_peered":"2023-09-23T23:18:58.585475+0000","last_unstale":"2023-09-23T23:21:59.942609+0000","last_undegraded":"2023-09-23T23:21:59.942609+0000","last_fullsized":"2023-09-23T23:21:59.942609+0000","mapping_epoch":255,"log_start":"277'4961","ondisk_log_start":"277'4961","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4882,"log_dups_size":0,"ondisk_log_size":4882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:14.419790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174260224,"num_objects":2659,"num_object_clones":0,"num_object_copies":7977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9843,"num_write_kb":400064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":69,"reported_epoch":304,"state":"active+undersized+degraded","last_fresh":"2023-09-23T23:21:53.037588+0000","last_change":"2023-09-23T23:18:59.587709+0000","last_active":"2023-09-23T23:21:53.037588+0000","last_peered":"2023-09-23T23:21:53.037588+0000","last_clean":"2023-09-23T22:53:20.393182+0000","last_became_active":"2023-09-23T23:18:59.587709+0000","last_became_peered":"2023-09-23T23:18:59.587709+0000","last_unstale":"2023-09-23T23:21:53.037588+0000","last_undegraded":"2023-09-23T23:18:59.585429+0000","last_fullsized":"2023-09-23T23:18:59.585311+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:58.759914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":322,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.037232+0000","last_change":"2023-09-23T23:19:17.013984+0000","last_active":"2023-09-23T23:21:53.037232+0000","last_peered":"2023-09-23T23:21:53.037232+0000","last_clean":"2023-09-23T23:21:53.037232+0000","last_became_active":"2023-09-23T23:19:17.013418+0000","last_became_peered":"2023-09-23T23:19:17.013418+0000","last_unstale":"2023-09-23T23:21:53.037232+0000","last_undegraded":"2023-09-23T23:21:53.037232+0000","last_fullsized":"2023-09-23T23:21:53.037232+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:08.483999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"305'9883","reported_seq":9949,"reported_epoch":305,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:21:59.847692+0000","last_change":"2023-09-23T23:20:16.873566+0000","last_active":"2023-09-23T23:21:59.847692+0000","last_peered":"2023-09-23T23:21:59.847692+0000","last_clean":"2023-09-23T23:20:14.681831+0000","last_became_active":"2023-09-23T23:20:16.868495+0000","last_became_peered":"2023-09-23T23:20:16.868495+0000","last_unstale":"2023-09-23T23:21:59.847692+0000","last_undegraded":"2023-09-23T23:21:59.847692+0000","last_fullsized":"2023-09-23T23:21:59.847692+0000","mapping_epoch":280,"log_start":"277'5004","ondisk_log_start":"277'5004","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4879,"log_dups_size":0,"ondisk_log_size":4879,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:04.491169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174784512,"num_objects":2667,"num_object_clones":0,"num_object_copies":8001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2667,"num_objects_unfound":0,"num_objects_dirty":2667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9883,"num_write_kb":401600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,3,4],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3,4","objects":2667}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":122,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.039095+0000","last_change":"2023-09-23T23:21:22.192374+0000","last_active":"2023-09-23T23:21:53.039095+0000","last_peered":"2023-09-23T23:21:53.039095+0000","last_clean":"2023-09-23T23:21:53.039095+0000","last_became_active":"2023-09-23T23:21:22.192086+0000","last_became_peered":"2023-09-23T23:21:22.192086+0000","last_unstale":"2023-09-23T23:21:53.039095+0000","last_undegraded":"2023-09-23T23:21:53.039095+0000","last_fullsized":"2023-09-23T23:21:53.039095+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:20.605133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":269,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.039674+0000","last_change":"2023-09-23T23:19:17.005947+0000","last_active":"2023-09-23T23:21:53.039674+0000","last_peered":"2023-09-23T23:21:53.039674+0000","last_clean":"2023-09-23T23:21:53.039674+0000","last_became_active":"2023-09-23T23:19:17.005632+0000","last_became_peered":"2023-09-23T23:19:17.005632+0000","last_unstale":"2023-09-23T23:21:53.039674+0000","last_undegraded":"2023-09-23T23:21:53.039674+0000","last_fullsized":"2023-09-23T23:21:53.039674+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:53.270709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"305'10041","reported_seq":10099,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T23:21:59.161548+0000","last_change":"2023-09-23T23:18:58.587100+0000","last_active":"2023-09-23T23:21:59.161548+0000","last_peered":"2023-09-23T23:21:59.161548+0000","last_clean":"2023-09-23T23:21:59.161548+0000","last_became_active":"2023-09-23T23:18:58.586837+0000","last_became_peered":"2023-09-23T23:18:58.586837+0000","last_unstale":"2023-09-23T23:21:59.161548+0000","last_undegraded":"2023-09-23T23:21:59.161548+0000","last_fullsized":"2023-09-23T23:21:59.161548+0000","mapping_epoch":255,"log_start":"277'4800","ondisk_log_start":"277'4800","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":5241,"log_dups_size":0,"ondisk_log_size":5241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:56.131536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181469184,"num_objects":2769,"num_object_clones":0,"num_object_copies":8307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10041,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":629,"reported_epoch":305,"state":"active+clean+remapped+scrubbing","last_fresh":"2023-09-23T23:21:58.278851+0000","last_change":"2023-09-23T23:21:58.278851+0000","last_active":"2023-09-23T23:21:58.278851+0000","last_peered":"2023-09-23T23:21:58.278851+0000","last_clean":"2023-09-23T23:21:58.278851+0000","last_became_active":"2023-09-23T23:20:18.750219+0000","last_became_peered":"2023-09-23T23:20:18.750219+0000","last_unstale":"2023-09-23T23:21:58.278851+0000","last_undegraded":"2023-09-23T23:21:58.278851+0000","last_fullsized":"2023-09-23T23:21:58.278851+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:46.150686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:46.150686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,5],"avail_no_missing":["3","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":270,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.037894+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:21:53.037894+0000","last_peered":"2023-09-23T23:21:53.037894+0000","last_clean":"2023-09-23T23:21:53.037894+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:21:53.037894+0000","last_undegraded":"2023-09-23T23:21:53.037894+0000","last_fullsized":"2023-09-23T23:21:53.037894+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":264,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.074806+0000","last_change":"2023-09-23T23:10:59.512363+0000","last_active":"2023-09-23T23:21:53.074806+0000","last_peered":"2023-09-23T23:21:53.074806+0000","last_clean":"2023-09-23T23:21:53.074806+0000","last_became_active":"2023-09-23T23:10:59.512022+0000","last_became_peered":"2023-09-23T23:10:59.512022+0000","last_unstale":"2023-09-23T23:21:53.074806+0000","last_undegraded":"2023-09-23T23:21:53.074806+0000","last_fullsized":"2023-09-23T23:21:53.074806+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:45.523432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"305'9922","reported_seq":10363,"reported_epoch":305,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:21:59.635883+0000","last_change":"2023-09-23T23:21:52.849662+0000","last_active":"2023-09-23T23:21:59.635883+0000","last_peered":"2023-09-23T23:21:59.635883+0000","last_clean":"2023-09-23T23:20:14.670806+0000","last_became_active":"2023-09-23T23:21:51.082780+0000","last_became_peered":"2023-09-23T23:21:51.082780+0000","last_unstale":"2023-09-23T23:21:59.635883+0000","last_undegraded":"2023-09-23T23:21:51.046443+0000","last_fullsized":"2023-09-23T23:21:59.635883+0000","mapping_epoch":301,"log_start":"276'4722","ondisk_log_start":"276'4722","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":5200,"log_dups_size":0,"ondisk_log_size":5200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:37.352090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178520064,"num_objects":2724,"num_object_clones":0,"num_object_copies":8172,"num_objects_missing_on_primary":5116,"num_objects_missing":5116,"num_objects_degraded":5116,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9922,"num_write_kb":404672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":180,"num_bytes_recovered":11796480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":["0","4","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":10},{"shards":"0,2,5","objects":1018},{"shards":"0,4,5","objects":1696}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":526,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.035856+0000","last_change":"2023-09-23T23:20:18.748975+0000","last_active":"2023-09-23T23:21:53.035856+0000","last_peered":"2023-09-23T23:21:53.035856+0000","last_clean":"2023-09-23T23:21:53.035856+0000","last_became_active":"2023-09-23T23:20:18.748608+0000","last_became_peered":"2023-09-23T23:20:18.748608+0000","last_unstale":"2023-09-23T23:21:53.035856+0000","last_undegraded":"2023-09-23T23:21:53.035856+0000","last_fullsized":"2023-09-23T23:21:53.035856+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:42.871134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:42.871134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:51.672826+0000","scrub_duration":0.0023428590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":276,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.037283+0000","last_change":"2023-09-23T23:16:56.525904+0000","last_active":"2023-09-23T23:21:53.037283+0000","last_peered":"2023-09-23T23:21:53.037283+0000","last_clean":"2023-09-23T23:21:53.037283+0000","last_became_active":"2023-09-23T23:16:56.525372+0000","last_became_peered":"2023-09-23T23:16:56.525372+0000","last_unstale":"2023-09-23T23:21:53.037283+0000","last_undegraded":"2023-09-23T23:21:53.037283+0000","last_fullsized":"2023-09-23T23:21:53.037283+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:17.070117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":310,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.036876+0000","last_change":"2023-09-23T23:21:50.044815+0000","last_active":"2023-09-23T23:21:53.036876+0000","last_peered":"2023-09-23T23:21:53.036876+0000","last_clean":"2023-09-23T23:21:53.036876+0000","last_became_active":"2023-09-23T23:21:50.044530+0000","last_became_peered":"2023-09-23T23:21:50.044530+0000","last_unstale":"2023-09-23T23:21:53.036876+0000","last_undegraded":"2023-09-23T23:21:53.036876+0000","last_fullsized":"2023-09-23T23:21:53.036876+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:56.926102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"305'10007","reported_seq":12058,"reported_epoch":305,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:21:59.350795+0000","last_change":"2023-09-23T23:21:51.076069+0000","last_active":"2023-09-23T23:21:59.350795+0000","last_peered":"2023-09-23T23:21:59.350795+0000","last_clean":"2023-09-23T23:21:59.350795+0000","last_became_active":"2023-09-23T23:21:51.041885+0000","last_became_peered":"2023-09-23T23:21:51.041885+0000","last_unstale":"2023-09-23T23:21:59.350795+0000","last_undegraded":"2023-09-23T23:21:59.350795+0000","last_fullsized":"2023-09-23T23:21:59.350795+0000","mapping_epoch":301,"log_start":"277'4819","ondisk_log_start":"277'4819","created":255,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":5188,"log_dups_size":0,"ondisk_log_size":5188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:56.176084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182779904,"num_objects":2789,"num_object_clones":0,"num_object_copies":8367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2789,"num_objects_unfound":0,"num_objects_dirty":2789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10007,"num_write_kb":409472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":975,"num_bytes_recovered":63897600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,4],"avail_no_missing":["6","0","4"],"object_location_counts":[{"shards":"0,4,6","objects":2789}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":305,"reported_epoch":304,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:21:54.612054+0000","last_change":"2023-09-23T23:21:51.059706+0000","last_active":"2023-09-23T23:21:54.612054+0000","last_peered":"2023-09-23T23:21:54.612054+0000","last_clean":"2023-09-23T23:21:54.612054+0000","last_became_active":"2023-09-23T23:21:51.059218+0000","last_became_peered":"2023-09-23T23:21:51.059218+0000","last_unstale":"2023-09-23T23:21:54.612054+0000","last_undegraded":"2023-09-23T23:21:54.612054+0000","last_fullsized":"2023-09-23T23:21:54.612054+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:08.869124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5,4],"avail_no_missing":["2","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":274,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.074899+0000","last_change":"2023-09-23T23:15:28.228439+0000","last_active":"2023-09-23T23:21:53.074899+0000","last_peered":"2023-09-23T23:21:53.074899+0000","last_clean":"2023-09-23T23:21:53.074899+0000","last_became_active":"2023-09-23T23:15:28.228095+0000","last_became_peered":"2023-09-23T23:15:28.228095+0000","last_unstale":"2023-09-23T23:21:53.074899+0000","last_undegraded":"2023-09-23T23:21:53.074899+0000","last_fullsized":"2023-09-23T23:21:53.074899+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:37.667201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"305'9751","reported_seq":9815,"reported_epoch":305,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T23:21:59.931074+0000","last_change":"2023-09-23T23:21:51.156649+0000","last_active":"2023-09-23T23:21:59.931074+0000","last_peered":"2023-09-23T23:21:59.931074+0000","last_clean":"2023-09-23T23:18:57.558582+0000","last_became_active":"2023-09-23T23:21:51.125394+0000","last_became_peered":"2023-09-23T23:21:51.125394+0000","last_unstale":"2023-09-23T23:21:59.931074+0000","last_undegraded":"2023-09-23T23:21:51.051589+0000","last_fullsized":"2023-09-23T23:21:51.036385+0000","mapping_epoch":301,"log_start":"277'4878","ondisk_log_start":"277'4878","created":255,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4873,"log_dups_size":0,"ondisk_log_size":4873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:57.087854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172687360,"num_objects":2635,"num_object_clones":0,"num_object_copies":7905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2635,"num_objects_misplaced":2635,"num_objects_unfound":0,"num_objects_dirty":2635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9751,"num_write_kb":396352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":2635}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":585,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.035950+0000","last_change":"2023-09-23T23:21:22.194486+0000","last_active":"2023-09-23T23:21:53.035950+0000","last_peered":"2023-09-23T23:21:53.035950+0000","last_clean":"2023-09-23T23:21:53.035950+0000","last_became_active":"2023-09-23T23:21:22.194192+0000","last_became_peered":"2023-09-23T23:21:22.194192+0000","last_unstale":"2023-09-23T23:21:53.035950+0000","last_undegraded":"2023-09-23T23:21:53.035950+0000","last_fullsized":"2023-09-23T23:21:53.035950+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:51.734652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:51.734652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:51.641289+0000","scrub_duration":0.023100959000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":136,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.074046+0000","last_change":"2023-09-23T23:19:17.016882+0000","last_active":"2023-09-23T23:21:53.074046+0000","last_peered":"2023-09-23T23:21:53.074046+0000","last_clean":"2023-09-23T23:21:53.074046+0000","last_became_active":"2023-09-23T23:19:17.016552+0000","last_became_peered":"2023-09-23T23:19:17.016552+0000","last_unstale":"2023-09-23T23:21:53.074046+0000","last_undegraded":"2023-09-23T23:21:53.074046+0000","last_fullsized":"2023-09-23T23:21:53.074046+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:58.565086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":163,"reported_epoch":304,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:21:53.039142+0000","last_change":"2023-09-23T23:19:18.010787+0000","last_active":"2023-09-23T23:21:53.039142+0000","last_peered":"2023-09-23T23:21:53.039142+0000","last_clean":"2023-09-23T23:21:53.039142+0000","last_became_active":"2023-09-23T23:19:18.010376+0000","last_became_peered":"2023-09-23T23:19:18.010376+0000","last_unstale":"2023-09-23T23:21:53.039142+0000","last_undegraded":"2023-09-23T23:21:53.039142+0000","last_fullsized":"2023-09-23T23:21:53.039142+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:43:05.036819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4,1],"avail_no_missing":["3","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"305'9925","reported_seq":9979,"reported_epoch":305,"state":"active+undersized+degraded","last_fresh":"2023-09-23T23:21:59.200190+0000","last_change":"2023-09-23T23:19:10.829313+0000","last_active":"2023-09-23T23:21:59.200190+0000","last_peered":"2023-09-23T23:21:59.200190+0000","last_clean":"2023-09-23T23:18:57.558582+0000","last_became_active":"2023-09-23T23:18:59.585882+0000","last_became_peered":"2023-09-23T23:18:59.585882+0000","last_unstale":"2023-09-23T23:21:59.200190+0000","last_undegraded":"2023-09-23T23:19:10.822953+0000","last_fullsized":"2023-09-23T23:18:59.582773+0000","mapping_epoch":256,"log_start":"276'4700","ondisk_log_start":"276'4700","created":255,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":5225,"log_dups_size":0,"ondisk_log_size":5225,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:12.870079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177668096,"num_objects":2711,"num_object_clones":0,"num_object_copies":8133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2711,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9925,"num_write_kb":404352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":2711}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":678,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.037824+0000","last_change":"2023-09-23T23:20:38.433883+0000","last_active":"2023-09-23T23:21:53.037824+0000","last_peered":"2023-09-23T23:21:53.037824+0000","last_clean":"2023-09-23T23:21:53.037824+0000","last_became_active":"2023-09-23T23:20:38.433500+0000","last_became_peered":"2023-09-23T23:20:38.433500+0000","last_unstale":"2023-09-23T23:21:53.037824+0000","last_undegraded":"2023-09-23T23:21:53.037824+0000","last_fullsized":"2023-09-23T23:21:53.037824+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:47.797085+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:47.797085+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:35:59.895118+0000","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":35,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.036706+0000","last_change":"2023-09-23T23:20:16.732474+0000","last_active":"2023-09-23T23:21:53.036706+0000","last_peered":"2023-09-23T23:21:53.036706+0000","last_clean":"2023-09-23T23:21:53.036706+0000","last_became_active":"2023-09-23T23:20:16.732087+0000","last_became_peered":"2023-09-23T23:20:16.732087+0000","last_unstale":"2023-09-23T23:21:53.036706+0000","last_undegraded":"2023-09-23T23:21:53.036706+0000","last_fullsized":"2023-09-23T23:21:53.036706+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:12.741506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":320,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.042096+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:21:53.042096+0000","last_peered":"2023-09-23T23:21:53.042096+0000","last_clean":"2023-09-23T23:21:53.042096+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:21:53.042096+0000","last_undegraded":"2023-09-23T23:21:53.042096+0000","last_fullsized":"2023-09-23T23:21:53.042096+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":238,"reported_epoch":304,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:21:53.038816+0000","last_change":"2023-09-23T23:21:51.043740+0000","last_active":"2023-09-23T23:21:53.038816+0000","last_peered":"2023-09-23T23:21:53.038816+0000","last_clean":"2023-09-23T23:21:53.038816+0000","last_became_active":"2023-09-23T23:21:51.043279+0000","last_became_peered":"2023-09-23T23:21:51.043279+0000","last_unstale":"2023-09-23T23:21:53.038816+0000","last_undegraded":"2023-09-23T23:21:53.038816+0000","last_fullsized":"2023-09-23T23:21:53.038816+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:57:07.926321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2,4],"avail_no_missing":["0","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.a","version":"305'10069","reported_seq":10127,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T23:21:59.210380+0000","last_change":"2023-09-23T23:18:58.585142+0000","last_active":"2023-09-23T23:21:59.210380+0000","last_peered":"2023-09-23T23:21:59.210380+0000","last_clean":"2023-09-23T23:21:59.210380+0000","last_became_active":"2023-09-23T23:18:58.584860+0000","last_became_peered":"2023-09-23T23:18:58.584860+0000","last_unstale":"2023-09-23T23:21:59.210380+0000","last_undegraded":"2023-09-23T23:21:59.210380+0000","last_fullsized":"2023-09-23T23:21:59.210380+0000","mapping_epoch":255,"log_start":"277'4841","ondisk_log_start":"277'4841","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":5228,"log_dups_size":0,"ondisk_log_size":5228,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:14.574839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180420608,"num_objects":2753,"num_object_clones":0,"num_object_copies":8259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10069,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":428,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.037749+0000","last_change":"2023-09-23T23:21:22.194755+0000","last_active":"2023-09-23T23:21:53.037749+0000","last_peered":"2023-09-23T23:21:53.037749+0000","last_clean":"2023-09-23T23:21:53.037749+0000","last_became_active":"2023-09-23T23:21:22.194412+0000","last_became_peered":"2023-09-23T23:21:22.194412+0000","last_unstale":"2023-09-23T23:21:53.037749+0000","last_undegraded":"2023-09-23T23:21:53.037749+0000","last_fullsized":"2023-09-23T23:21:53.037749+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:49.726740+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:49.726740+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":250,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.037226+0000","last_change":"2023-09-23T23:21:22.193479+0000","last_active":"2023-09-23T23:21:53.037226+0000","last_peered":"2023-09-23T23:21:53.037226+0000","last_clean":"2023-09-23T23:21:53.037226+0000","last_became_active":"2023-09-23T23:21:22.193220+0000","last_became_peered":"2023-09-23T23:21:22.193220+0000","last_unstale":"2023-09-23T23:21:53.037226+0000","last_undegraded":"2023-09-23T23:21:53.037226+0000","last_fullsized":"2023-09-23T23:21:53.037226+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:36.709733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":265,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.074484+0000","last_change":"2023-09-23T23:04:11.514703+0000","last_active":"2023-09-23T23:21:53.074484+0000","last_peered":"2023-09-23T23:21:53.074484+0000","last_clean":"2023-09-23T23:21:53.074484+0000","last_became_active":"2023-09-23T23:04:11.514403+0000","last_became_peered":"2023-09-23T23:04:11.514403+0000","last_unstale":"2023-09-23T23:21:53.074484+0000","last_undegraded":"2023-09-23T23:21:53.074484+0000","last_fullsized":"2023-09-23T23:21:53.074484+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:26.374346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":36,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.036414+0000","last_change":"2023-09-23T23:20:15.781844+0000","last_active":"2023-09-23T23:21:53.036414+0000","last_peered":"2023-09-23T23:21:53.036414+0000","last_clean":"2023-09-23T23:21:53.036414+0000","last_became_active":"2023-09-23T23:20:15.781573+0000","last_became_peered":"2023-09-23T23:20:15.781573+0000","last_unstale":"2023-09-23T23:21:53.036414+0000","last_undegraded":"2023-09-23T23:21:53.036414+0000","last_fullsized":"2023-09-23T23:21:53.036414+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:25:50.816486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"305'9874","reported_seq":9947,"reported_epoch":305,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:21:59.854377+0000","last_change":"2023-09-23T23:20:26.853101+0000","last_active":"2023-09-23T23:21:59.854377+0000","last_peered":"2023-09-23T23:21:59.854377+0000","last_clean":"2023-09-23T23:20:14.650424+0000","last_became_active":"2023-09-23T23:20:26.846172+0000","last_became_peered":"2023-09-23T23:20:26.846172+0000","last_unstale":"2023-09-23T23:21:59.854377+0000","last_undegraded":"2023-09-23T23:21:59.854377+0000","last_fullsized":"2023-09-23T23:21:59.854377+0000","mapping_epoch":284,"log_start":"277'4975","ondisk_log_start":"277'4975","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4899,"log_dups_size":0,"ondisk_log_size":4899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:44.923109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182190080,"num_objects":2780,"num_object_clones":0,"num_object_copies":8340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5560,"num_objects_unfound":0,"num_objects_dirty":2780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9874,"num_write_kb":404928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[0,4,3],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3,4","objects":2780}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":618,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.043092+0000","last_change":"2023-09-23T23:21:53.042842+0000","last_active":"2023-09-23T23:21:53.043092+0000","last_peered":"2023-09-23T23:21:53.043092+0000","last_clean":"2023-09-23T23:21:53.043092+0000","last_became_active":"2023-09-23T23:21:53.042505+0000","last_became_peered":"2023-09-23T23:21:53.042505+0000","last_unstale":"2023-09-23T23:21:53.043092+0000","last_undegraded":"2023-09-23T23:21:53.043092+0000","last_fullsized":"2023-09-23T23:21:53.043092+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:47.277080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:47.277080+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:59.372845+0000","scrub_duration":0.001458104,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":36,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.035946+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:21:53.035946+0000","last_peered":"2023-09-23T23:21:53.035946+0000","last_clean":"2023-09-23T23:21:53.035946+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:21:53.035946+0000","last_undegraded":"2023-09-23T23:21:53.035946+0000","last_fullsized":"2023-09-23T23:21:53.035946+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":36,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.035776+0000","last_change":"2023-09-23T23:20:15.781833+0000","last_active":"2023-09-23T23:21:53.035776+0000","last_peered":"2023-09-23T23:21:53.035776+0000","last_clean":"2023-09-23T23:21:53.035776+0000","last_became_active":"2023-09-23T23:20:15.781528+0000","last_became_peered":"2023-09-23T23:20:15.781528+0000","last_unstale":"2023-09-23T23:21:53.035776+0000","last_undegraded":"2023-09-23T23:21:53.035776+0000","last_fullsized":"2023-09-23T23:21:53.035776+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:00.011882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":117,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.038643+0000","last_change":"2023-09-23T23:21:22.194721+0000","last_active":"2023-09-23T23:21:53.038643+0000","last_peered":"2023-09-23T23:21:53.038643+0000","last_clean":"2023-09-23T23:21:53.038643+0000","last_became_active":"2023-09-23T23:21:22.194333+0000","last_became_peered":"2023-09-23T23:21:22.194333+0000","last_unstale":"2023-09-23T23:21:53.038643+0000","last_undegraded":"2023-09-23T23:21:53.038643+0000","last_fullsized":"2023-09-23T23:21:53.038643+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:46.823997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"305'10080","reported_seq":12005,"reported_epoch":305,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:21:59.389037+0000","last_change":"2023-09-23T23:21:51.084503+0000","last_active":"2023-09-23T23:21:59.389037+0000","last_peered":"2023-09-23T23:21:59.389037+0000","last_clean":"2023-09-23T23:21:59.389037+0000","last_became_active":"2023-09-23T23:21:51.043860+0000","last_became_peered":"2023-09-23T23:21:51.043860+0000","last_unstale":"2023-09-23T23:21:59.389037+0000","last_undegraded":"2023-09-23T23:21:59.389037+0000","last_fullsized":"2023-09-23T23:21:59.389037+0000","mapping_epoch":301,"log_start":"277'4865","ondisk_log_start":"277'4865","created":255,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":5215,"log_dups_size":0,"ondisk_log_size":5215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:12.094453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185335808,"num_objects":2828,"num_object_clones":0,"num_object_copies":8484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2828,"num_objects_unfound":0,"num_objects_dirty":2828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10080,"num_write_kb":413056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":917,"num_bytes_recovered":60096512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,4],"avail_no_missing":["6","0","4"],"object_location_counts":[{"shards":"0,4,6","objects":2828}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":304,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.042466+0000","last_change":"2023-09-23T23:21:22.193754+0000","last_active":"2023-09-23T23:21:53.042466+0000","last_peered":"2023-09-23T23:21:53.042466+0000","last_clean":"2023-09-23T23:21:53.042466+0000","last_became_active":"2023-09-23T23:21:22.193409+0000","last_became_peered":"2023-09-23T23:21:22.193409+0000","last_unstale":"2023-09-23T23:21:53.042466+0000","last_undegraded":"2023-09-23T23:21:53.042466+0000","last_fullsized":"2023-09-23T23:21:53.042466+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:53.635813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":559,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.038474+0000","last_change":"2023-09-23T23:20:18.766932+0000","last_active":"2023-09-23T23:21:53.038474+0000","last_peered":"2023-09-23T23:21:53.038474+0000","last_clean":"2023-09-23T23:21:53.038474+0000","last_became_active":"2023-09-23T23:20:18.766599+0000","last_became_peered":"2023-09-23T23:20:18.766599+0000","last_unstale":"2023-09-23T23:21:53.038474+0000","last_undegraded":"2023-09-23T23:21:53.038474+0000","last_fullsized":"2023-09-23T23:21:53.038474+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:42.190868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:42.190868+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:47.989449+0000","scrub_duration":0.0049256480000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":234,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.037496+0000","last_change":"2023-09-23T23:16:56.525298+0000","last_active":"2023-09-23T23:21:53.037496+0000","last_peered":"2023-09-23T23:21:53.037496+0000","last_clean":"2023-09-23T23:21:53.037496+0000","last_became_active":"2023-09-23T23:16:56.524974+0000","last_became_peered":"2023-09-23T23:16:56.524974+0000","last_unstale":"2023-09-23T23:21:53.037496+0000","last_undegraded":"2023-09-23T23:21:53.037496+0000","last_fullsized":"2023-09-23T23:21:53.037496+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:08.860790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":267,"reported_epoch":304,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:21:53.038068+0000","last_change":"2023-09-23T23:21:51.046606+0000","last_active":"2023-09-23T23:21:53.038068+0000","last_peered":"2023-09-23T23:21:53.038068+0000","last_clean":"2023-09-23T23:21:53.038068+0000","last_became_active":"2023-09-23T23:21:51.046103+0000","last_became_peered":"2023-09-23T23:21:51.046103+0000","last_unstale":"2023-09-23T23:21:53.038068+0000","last_undegraded":"2023-09-23T23:21:53.038068+0000","last_fullsized":"2023-09-23T23:21:53.038068+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:05.999638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,4],"avail_no_missing":["6","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"305'10142","reported_seq":18392,"reported_epoch":305,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:21:59.946055+0000","last_change":"2023-09-23T23:21:51.067385+0000","last_active":"2023-09-23T23:21:59.946055+0000","last_peered":"2023-09-23T23:21:59.946055+0000","last_clean":"2023-09-23T23:20:14.651555+0000","last_became_active":"2023-09-23T23:20:35.497961+0000","last_became_peered":"2023-09-23T23:20:35.497961+0000","last_unstale":"2023-09-23T23:21:59.946055+0000","last_undegraded":"2023-09-23T23:20:35.491209+0000","last_fullsized":"2023-09-23T23:20:35.375324+0000","mapping_epoch":287,"log_start":"278'5259","ondisk_log_start":"278'5259","created":255,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4883,"log_dups_size":0,"ondisk_log_size":4883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:38.877968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180486144,"num_objects":2754,"num_object_clones":0,"num_object_copies":8262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2754,"num_objects_misplaced":1295,"num_objects_unfound":0,"num_objects_dirty":2754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10142,"num_write_kb":412672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4082,"num_bytes_recovered":267517952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[0,5],"avail_no_missing":["0","4","5"],"object_location_counts":[{"shards":"0,4,5","objects":516},{"shards":"0,5","objects":1459}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":495,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.042207+0000","last_change":"2023-09-23T23:21:53.041705+0000","last_active":"2023-09-23T23:21:53.042207+0000","last_peered":"2023-09-23T23:21:53.042207+0000","last_clean":"2023-09-23T23:21:53.042207+0000","last_became_active":"2023-09-23T23:21:53.041337+0000","last_became_peered":"2023-09-23T23:21:53.041337+0000","last_unstale":"2023-09-23T23:21:53.042207+0000","last_undegraded":"2023-09-23T23:21:53.042207+0000","last_fullsized":"2023-09-23T23:21:53.042207+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:19:31.012484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:19:31.012484+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:16.377672+0000","scrub_duration":0.0014650920000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":254,"reported_epoch":304,"state":"active+undersized","last_fresh":"2023-09-23T23:21:53.039978+0000","last_change":"2023-09-23T23:18:59.580395+0000","last_active":"2023-09-23T23:21:53.039978+0000","last_peered":"2023-09-23T23:21:53.039978+0000","last_clean":"2023-09-23T23:16:52.490944+0000","last_became_active":"2023-09-23T23:18:59.580395+0000","last_became_peered":"2023-09-23T23:18:59.580395+0000","last_unstale":"2023-09-23T23:21:53.039978+0000","last_undegraded":"2023-09-23T23:21:53.039978+0000","last_fullsized":"2023-09-23T23:18:59.577313+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:36.323937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":277,"reported_epoch":304,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:21:53.037534+0000","last_change":"2023-09-23T23:20:27.898330+0000","last_active":"2023-09-23T23:21:53.037534+0000","last_peered":"2023-09-23T23:21:53.037534+0000","last_clean":"2023-09-23T23:21:53.037534+0000","last_became_active":"2023-09-23T23:20:27.897982+0000","last_became_peered":"2023-09-23T23:20:27.897982+0000","last_unstale":"2023-09-23T23:21:53.037534+0000","last_undegraded":"2023-09-23T23:21:53.037534+0000","last_fullsized":"2023-09-23T23:21:53.037534+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:28.564415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,4],"avail_no_missing":["0","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":324,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.039174+0000","last_change":"2023-09-23T23:21:50.040301+0000","last_active":"2023-09-23T23:21:53.039174+0000","last_peered":"2023-09-23T23:21:53.039174+0000","last_clean":"2023-09-23T23:21:53.039174+0000","last_became_active":"2023-09-23T23:21:50.039967+0000","last_became_peered":"2023-09-23T23:21:50.039967+0000","last_unstale":"2023-09-23T23:21:53.039174+0000","last_undegraded":"2023-09-23T23:21:53.039174+0000","last_fullsized":"2023-09-23T23:21:53.039174+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:09.380818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"305'9925","reported_seq":9984,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T23:21:59.171520+0000","last_change":"2023-09-23T23:18:58.587441+0000","last_active":"2023-09-23T23:21:59.171520+0000","last_peered":"2023-09-23T23:21:59.171520+0000","last_clean":"2023-09-23T23:21:59.171520+0000","last_became_active":"2023-09-23T23:18:58.587188+0000","last_became_peered":"2023-09-23T23:18:58.587188+0000","last_unstale":"2023-09-23T23:21:59.171520+0000","last_undegraded":"2023-09-23T23:21:59.171520+0000","last_fullsized":"2023-09-23T23:21:59.171520+0000","mapping_epoch":255,"log_start":"276'4720","ondisk_log_start":"276'4720","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":5205,"log_dups_size":0,"ondisk_log_size":5205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:54.227096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177340440,"num_objects":2707,"num_object_clones":0,"num_object_copies":8121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2707,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9925,"num_write_kb":404161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":578,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.037702+0000","last_change":"2023-09-23T23:16:59.295000+0000","last_active":"2023-09-23T23:21:53.037702+0000","last_peered":"2023-09-23T23:21:53.037702+0000","last_clean":"2023-09-23T23:21:53.037702+0000","last_became_active":"2023-09-23T23:16:59.294607+0000","last_became_peered":"2023-09-23T23:16:59.294607+0000","last_unstale":"2023-09-23T23:21:53.037702+0000","last_undegraded":"2023-09-23T23:21:53.037702+0000","last_fullsized":"2023-09-23T23:21:53.037702+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.802642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.802642+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:48.429859+0000","scrub_duration":0.0056570290000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":106,"reported_epoch":304,"state":"active+undersized","last_fresh":"2023-09-23T23:21:53.037803+0000","last_change":"2023-09-23T23:18:59.585438+0000","last_active":"2023-09-23T23:21:53.037803+0000","last_peered":"2023-09-23T23:21:53.037803+0000","last_clean":"2023-09-23T23:03:28.874460+0000","last_became_active":"2023-09-23T23:18:59.585438+0000","last_became_peered":"2023-09-23T23:18:59.585438+0000","last_unstale":"2023-09-23T23:21:53.037803+0000","last_undegraded":"2023-09-23T23:21:53.037803+0000","last_fullsized":"2023-09-23T23:18:59.577149+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:37.419384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":36,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.036491+0000","last_change":"2023-09-23T23:20:15.781789+0000","last_active":"2023-09-23T23:21:53.036491+0000","last_peered":"2023-09-23T23:21:53.036491+0000","last_clean":"2023-09-23T23:21:53.036491+0000","last_became_active":"2023-09-23T23:20:15.781479+0000","last_became_peered":"2023-09-23T23:20:15.781479+0000","last_unstale":"2023-09-23T23:21:53.036491+0000","last_undegraded":"2023-09-23T23:21:53.036491+0000","last_fullsized":"2023-09-23T23:21:53.036491+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:58.256791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":26,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.042033+0000","last_change":"2023-09-23T23:20:36.429855+0000","last_active":"2023-09-23T23:21:53.042033+0000","last_peered":"2023-09-23T23:21:53.042033+0000","last_clean":"2023-09-23T23:21:53.042033+0000","last_became_active":"2023-09-23T23:20:36.427234+0000","last_became_peered":"2023-09-23T23:20:36.427234+0000","last_unstale":"2023-09-23T23:21:53.042033+0000","last_undegraded":"2023-09-23T23:21:53.042033+0000","last_fullsized":"2023-09-23T23:21:53.042033+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:39.297055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"305'9935","reported_seq":10122,"reported_epoch":305,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:21:59.223806+0000","last_change":"2023-09-23T23:20:34.547612+0000","last_active":"2023-09-23T23:21:59.223806+0000","last_peered":"2023-09-23T23:21:59.223806+0000","last_clean":"2023-09-23T23:20:14.708957+0000","last_became_active":"2023-09-23T23:20:16.860045+0000","last_became_peered":"2023-09-23T23:20:16.860045+0000","last_unstale":"2023-09-23T23:21:59.223806+0000","last_undegraded":"2023-09-23T23:21:59.223806+0000","last_fullsized":"2023-09-23T23:21:59.223806+0000","mapping_epoch":280,"log_start":"276'4700","ondisk_log_start":"276'4700","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":5235,"log_dups_size":0,"ondisk_log_size":5235,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:31.359924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178585600,"num_objects":2725,"num_object_clones":0,"num_object_copies":8175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2694,"num_objects_unfound":0,"num_objects_dirty":2725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9935,"num_write_kb":405120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":3670016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,5],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":2725}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":587,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.043375+0000","last_change":"2023-09-23T23:21:53.042685+0000","last_active":"2023-09-23T23:21:53.043375+0000","last_peered":"2023-09-23T23:21:53.043375+0000","last_clean":"2023-09-23T23:21:53.043375+0000","last_became_active":"2023-09-23T23:21:53.042278+0000","last_became_peered":"2023-09-23T23:21:53.042278+0000","last_unstale":"2023-09-23T23:21:53.043375+0000","last_undegraded":"2023-09-23T23:21:53.043375+0000","last_fullsized":"2023-09-23T23:21:53.043375+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:51.489074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:51.489074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:34:39.708914+0000","scrub_duration":0.00519324,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":284,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.039059+0000","last_change":"2023-09-23T23:16:56.561147+0000","last_active":"2023-09-23T23:21:53.039059+0000","last_peered":"2023-09-23T23:21:53.039059+0000","last_clean":"2023-09-23T23:21:53.039059+0000","last_became_active":"2023-09-23T23:16:56.547808+0000","last_became_peered":"2023-09-23T23:16:56.547808+0000","last_unstale":"2023-09-23T23:21:53.039059+0000","last_undegraded":"2023-09-23T23:21:53.039059+0000","last_fullsized":"2023-09-23T23:21:53.039059+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:13.666957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":160,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.037959+0000","last_change":"2023-09-23T23:16:56.526187+0000","last_active":"2023-09-23T23:21:53.037959+0000","last_peered":"2023-09-23T23:21:53.037959+0000","last_clean":"2023-09-23T23:21:53.037959+0000","last_became_active":"2023-09-23T23:16:56.525874+0000","last_became_peered":"2023-09-23T23:16:56.525874+0000","last_unstale":"2023-09-23T23:21:53.037959+0000","last_undegraded":"2023-09-23T23:21:53.037959+0000","last_fullsized":"2023-09-23T23:21:53.037959+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:32:07.101559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":186,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.036833+0000","last_change":"2023-09-23T23:21:22.199540+0000","last_active":"2023-09-23T23:21:53.036833+0000","last_peered":"2023-09-23T23:21:53.036833+0000","last_clean":"2023-09-23T23:21:53.036833+0000","last_became_active":"2023-09-23T23:21:22.199276+0000","last_became_peered":"2023-09-23T23:21:22.199276+0000","last_unstale":"2023-09-23T23:21:53.036833+0000","last_undegraded":"2023-09-23T23:21:53.036833+0000","last_fullsized":"2023-09-23T23:21:53.036833+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:35:05.831440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.f","version":"305'9663","reported_seq":9784,"reported_epoch":305,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:21:59.247595+0000","last_change":"2023-09-23T23:20:16.959133+0000","last_active":"2023-09-23T23:21:59.247595+0000","last_peered":"2023-09-23T23:21:59.247595+0000","last_clean":"2023-09-23T23:20:14.713919+0000","last_became_active":"2023-09-23T23:20:16.853529+0000","last_became_peered":"2023-09-23T23:20:16.853529+0000","last_unstale":"2023-09-23T23:21:59.247595+0000","last_undegraded":"2023-09-23T23:21:59.247595+0000","last_fullsized":"2023-09-23T23:21:59.247595+0000","mapping_epoch":280,"log_start":"276'4480","ondisk_log_start":"276'4480","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":5183,"log_dups_size":0,"ondisk_log_size":5183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:52.048771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175702016,"num_objects":2681,"num_object_clones":0,"num_object_copies":8043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2670,"num_objects_unfound":0,"num_objects_dirty":2681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9663,"num_write_kb":395008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":1572864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,4],"avail_no_missing":["6","3","4"],"object_location_counts":[{"shards":"3,4,6","objects":2681}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":594,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.043493+0000","last_change":"2023-09-23T23:21:53.042966+0000","last_active":"2023-09-23T23:21:53.043493+0000","last_peered":"2023-09-23T23:21:53.043493+0000","last_clean":"2023-09-23T23:21:53.043493+0000","last_became_active":"2023-09-23T23:21:53.042508+0000","last_became_peered":"2023-09-23T23:21:53.042508+0000","last_unstale":"2023-09-23T23:21:53.043493+0000","last_undegraded":"2023-09-23T23:21:53.043493+0000","last_fullsized":"2023-09-23T23:21:53.043493+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.522632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.522632+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:02.496014+0000","scrub_duration":0.018152603999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":317,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.037886+0000","last_change":"2023-09-23T23:20:15.786693+0000","last_active":"2023-09-23T23:21:53.037886+0000","last_peered":"2023-09-23T23:21:53.037886+0000","last_clean":"2023-09-23T23:21:53.037886+0000","last_became_active":"2023-09-23T23:20:15.786360+0000","last_became_peered":"2023-09-23T23:20:15.786360+0000","last_unstale":"2023-09-23T23:21:53.037886+0000","last_undegraded":"2023-09-23T23:21:53.037886+0000","last_fullsized":"2023-09-23T23:21:53.037886+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:00.707946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":275,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.039018+0000","last_change":"2023-09-23T23:19:17.023159+0000","last_active":"2023-09-23T23:21:53.039018+0000","last_peered":"2023-09-23T23:21:53.039018+0000","last_clean":"2023-09-23T23:21:53.039018+0000","last_became_active":"2023-09-23T23:19:17.012292+0000","last_became_peered":"2023-09-23T23:19:17.012292+0000","last_unstale":"2023-09-23T23:21:53.039018+0000","last_undegraded":"2023-09-23T23:21:53.039018+0000","last_fullsized":"2023-09-23T23:21:53.039018+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:06.870287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":18,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.041832+0000","last_change":"2023-09-23T23:21:50.045682+0000","last_active":"2023-09-23T23:21:53.041832+0000","last_peered":"2023-09-23T23:21:53.041832+0000","last_clean":"2023-09-23T23:21:53.041832+0000","last_became_active":"2023-09-23T23:21:50.045339+0000","last_became_peered":"2023-09-23T23:21:50.045339+0000","last_unstale":"2023-09-23T23:21:53.041832+0000","last_undegraded":"2023-09-23T23:21:53.041832+0000","last_fullsized":"2023-09-23T23:21:53.041832+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:55.802091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":45,"reported_epoch":304,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:21:53.074435+0000","last_change":"2023-09-23T23:20:27.899913+0000","last_active":"2023-09-23T23:21:53.074435+0000","last_peered":"2023-09-23T23:21:53.074435+0000","last_clean":"2023-09-23T23:21:53.074435+0000","last_became_active":"2023-09-23T23:20:27.899525+0000","last_became_peered":"2023-09-23T23:20:27.899525+0000","last_unstale":"2023-09-23T23:21:53.074435+0000","last_undegraded":"2023-09-23T23:21:53.074435+0000","last_fullsized":"2023-09-23T23:21:53.074435+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:41.775325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,0],"avail_no_missing":["4","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":36,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.036562+0000","last_change":"2023-09-23T23:20:15.740081+0000","last_active":"2023-09-23T23:21:53.036562+0000","last_peered":"2023-09-23T23:21:53.036562+0000","last_clean":"2023-09-23T23:21:53.036562+0000","last_became_active":"2023-09-23T23:20:15.738447+0000","last_became_peered":"2023-09-23T23:20:15.738447+0000","last_unstale":"2023-09-23T23:21:53.036562+0000","last_undegraded":"2023-09-23T23:21:53.036562+0000","last_fullsized":"2023-09-23T23:21:53.036562+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:55.593712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":33,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.074871+0000","last_change":"2023-09-23T23:21:22.194626+0000","last_active":"2023-09-23T23:21:53.074871+0000","last_peered":"2023-09-23T23:21:53.074871+0000","last_clean":"2023-09-23T23:21:53.074871+0000","last_became_active":"2023-09-23T23:21:22.194300+0000","last_became_peered":"2023-09-23T23:21:22.194300+0000","last_unstale":"2023-09-23T23:21:53.074871+0000","last_undegraded":"2023-09-23T23:21:53.074871+0000","last_fullsized":"2023-09-23T23:21:53.074871+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:21.103294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":301,"reported_epoch":304,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:21:53.158067+0000","last_change":"2023-09-23T23:21:51.056449+0000","last_active":"2023-09-23T23:21:53.158067+0000","last_peered":"2023-09-23T23:21:53.158067+0000","last_clean":"2023-09-23T23:21:53.158067+0000","last_became_active":"2023-09-23T23:21:51.056092+0000","last_became_peered":"2023-09-23T23:21:51.056092+0000","last_unstale":"2023-09-23T23:21:53.158067+0000","last_undegraded":"2023-09-23T23:21:53.158067+0000","last_fullsized":"2023-09-23T23:21:53.158067+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:33.836469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,4],"avail_no_missing":["3","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":33,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.075011+0000","last_change":"2023-09-23T23:21:22.194730+0000","last_active":"2023-09-23T23:21:53.075011+0000","last_peered":"2023-09-23T23:21:53.075011+0000","last_clean":"2023-09-23T23:21:53.075011+0000","last_became_active":"2023-09-23T23:21:22.194420+0000","last_became_peered":"2023-09-23T23:21:22.194420+0000","last_unstale":"2023-09-23T23:21:53.075011+0000","last_undegraded":"2023-09-23T23:21:53.075011+0000","last_fullsized":"2023-09-23T23:21:53.075011+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:31.083059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":149,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.037421+0000","last_change":"2023-09-23T23:20:26.841878+0000","last_active":"2023-09-23T23:21:53.037421+0000","last_peered":"2023-09-23T23:21:53.037421+0000","last_clean":"2023-09-23T23:21:53.037421+0000","last_became_active":"2023-09-23T23:20:26.841590+0000","last_became_peered":"2023-09-23T23:20:26.841590+0000","last_unstale":"2023-09-23T23:21:53.037421+0000","last_undegraded":"2023-09-23T23:21:53.037421+0000","last_fullsized":"2023-09-23T23:21:53.037421+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:23.904011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":311,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.038598+0000","last_change":"2023-09-23T23:20:16.730934+0000","last_active":"2023-09-23T23:21:53.038598+0000","last_peered":"2023-09-23T23:21:53.038598+0000","last_clean":"2023-09-23T23:21:53.038598+0000","last_became_active":"2023-09-23T23:20:16.730571+0000","last_became_peered":"2023-09-23T23:20:16.730571+0000","last_unstale":"2023-09-23T23:21:53.038598+0000","last_undegraded":"2023-09-23T23:21:53.038598+0000","last_fullsized":"2023-09-23T23:21:53.038598+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:17.993278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":314,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.038805+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:21:53.038805+0000","last_peered":"2023-09-23T23:21:53.038805+0000","last_clean":"2023-09-23T23:21:53.038805+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:21:53.038805+0000","last_undegraded":"2023-09-23T23:21:53.038805+0000","last_fullsized":"2023-09-23T23:21:53.038805+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":307,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.038531+0000","last_change":"2023-09-23T23:20:15.786279+0000","last_active":"2023-09-23T23:21:53.038531+0000","last_peered":"2023-09-23T23:21:53.038531+0000","last_clean":"2023-09-23T23:21:53.038531+0000","last_became_active":"2023-09-23T23:20:15.782731+0000","last_became_peered":"2023-09-23T23:20:15.782731+0000","last_unstale":"2023-09-23T23:21:53.038531+0000","last_undegraded":"2023-09-23T23:21:53.038531+0000","last_fullsized":"2023-09-23T23:21:53.038531+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:58.011396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":331,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T23:21:53.073985+0000","last_change":"2023-09-23T23:20:35.563189+0000","last_active":"2023-09-23T23:21:53.073985+0000","last_peered":"2023-09-23T23:21:53.073985+0000","last_clean":"2023-09-23T23:21:53.073985+0000","last_became_active":"2023-09-23T23:20:35.562911+0000","last_became_peered":"2023-09-23T23:20:35.562911+0000","last_unstale":"2023-09-23T23:21:53.073985+0000","last_undegraded":"2023-09-23T23:21:53.073985+0000","last_fullsized":"2023-09-23T23:21:53.073985+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:38.887436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":89,"acting":95,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2855403544,"num_objects":43571,"num_object_clones":0,"num_object_copies":130713,"num_objects_missing_on_primary":5116,"num_objects_missing":5116,"num_objects_degraded":13216,"num_objects_misplaced":25858,"num_objects_unfound":0,"num_objects_dirty":43571,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":158841,"num_write_kb":6477121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6237,"num_bytes_recovered":408748032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8605282304,"data_stored":8605270088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":81429,"ondisk_log_size":81429,"up":42,"acting":45,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":6,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5954,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":89,"acting":94,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":697708,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":90,"num_read_kb":167,"num_write":43,"num_write_kb":439,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1240067,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":32,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":21,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234314,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":803516,"kb_used_data":352116,"kb_used_omap":0,"kb_used_meta":451392,"kb_avail":92941636,"statfs":{"total":95995035648,"available":95172235264,"internally_reserved":0,"allocated":360566784,"data_stored":359841767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":462225408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:18:39 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":4.9820000000000002,"15min":3.9239999999999999},"min":{"1min":0.22,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":6.9249999999999998,"5min":216.40899999999999,"15min":230.249},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.7320000000000002,"5min":5.7359999999999998,"15min":4.2290000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24199999999999999},"max":{"1min":11.327999999999999,"5min":216.43700000000001,"15min":230.261},"last":0.40899999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:17:56 2023","interfaces":[{"interface":"back","average":{"1min":2.6150000000000002,"5min":2.9700000000000002,"15min":4.2359999999999998},"min":{"1min":0.39100000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.798,"5min":121.39400000000001,"15min":121.39400000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.2989999999999999,"5min":2.9790000000000001,"15min":4.077},"min":{"1min":0.27000000000000002,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":10.034000000000001,"5min":121.381,"15min":121.381},"last":0.42099999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0339999999999998,"5min":2.0179999999999998,"15min":2.0150000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.375,"5min":9.5589999999999993,"15min":9.5589999999999993},"last":0.435},{"interface":"front","average":{"1min":2.5600000000000001,"5min":1.8580000000000001,"15min":1.7410000000000001},"min":{"1min":0.36899999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.561999999999999,"5min":11.561999999999999,"15min":11.561999999999999},"last":0.47999999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:18:11 2023","interfaces":[{"interface":"back","average":{"1min":3.1400000000000001,"5min":5.7140000000000004,"15min":3.4620000000000002},"min":{"1min":0.27800000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":17.724,"5min":144.50299999999999,"15min":144.50299999999999},"last":0.755},{"interface":"front","average":{"1min":2.6829999999999998,"5min":5.5990000000000002,"15min":3.468},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":17.712,"5min":144.48699999999999,"15min":144.48699999999999},"last":0.75}]},{"osd":4,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":0.83099999999999996,"15min":1.3200000000000001},"min":{"1min":0.27300000000000002,"5min":0.254,"15min":0.23599999999999999},"max":{"1min":7.0570000000000004,"5min":7.0570000000000004,"15min":92.781000000000006},"last":0.372},{"interface":"front","average":{"1min":1.208,"5min":1.2110000000000001,"15min":1.4219999999999999},"min":{"1min":0.29499999999999998,"5min":0.28599999999999998,"15min":0.254},"max":{"1min":3.9569999999999999,"5min":9.7530000000000001,"15min":92.626999999999995},"last":0.39100000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":13.24,"5min":3.488,"15min":2.3759999999999999},"min":{"1min":0.28799999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":185.321,"5min":185.321,"15min":185.321},"last":0.44600000000000001},{"interface":"front","average":{"1min":11.488,"5min":2.851,"15min":1.97},"min":{"1min":0.33900000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":185.47499999999999,"5min":185.47499999999999,"15min":185.47499999999999},"last":0.49199999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":1.292,"15min":1.4970000000000001},"min":{"1min":0.376,"5min":0.30199999999999999,"15min":0.27300000000000002},"max":{"1min":19.782,"5min":19.782,"15min":93.105999999999995},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.7349999999999999,"5min":1.492,"15min":1.599},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28399999999999997},"max":{"1min":20.064,"5min":20.064,"15min":93.198999999999998},"last":0.46800000000000003}]}]},{"osd":6,"up_from":46,"seq":197568495987,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1853908,"kb_used_data":1417804,"kb_used_omap":0,"kb_used_meta":436096,"kb_avail":91891244,"statfs":{"total":95995035648,"available":94096633856,"internally_reserved":0,"allocated":1451831296,"data_stored":1450984963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":446562304},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:21:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0049999999999999,"5min":7.4279999999999999,"15min":4.1600000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":23.481000000000002,"5min":161.64599999999999,"15min":161.64599999999999},"last":4.8780000000000001},{"interface":"front","average":{"1min":3.7069999999999999,"5min":6.7370000000000001,"15min":4.0640000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":23.454000000000001,"5min":161.38499999999999,"15min":161.38499999999999},"last":2.347}]},{"osd":1,"last update":"Sat Sep 23 23:21:52 2023","interfaces":[{"interface":"back","average":{"1min":1.798,"5min":5.0599999999999996,"15min":4.0880000000000001},"min":{"1min":0.29299999999999998,"5min":0.249,"15min":0.24099999999999999},"max":{"1min":8.5210000000000008,"5min":210.869,"15min":210.869},"last":1.865},{"interface":"front","average":{"1min":1.6579999999999999,"5min":4.9500000000000002,"15min":4.2140000000000004},"min":{"1min":0.27500000000000002,"5min":0.26800000000000002,"15min":0.26200000000000001},"max":{"1min":7.6849999999999996,"5min":210.89699999999999,"15min":210.89699999999999},"last":2.206}]},{"osd":2,"last update":"Sat Sep 23 23:21:31 2023","interfaces":[{"interface":"back","average":{"1min":2.081,"5min":1.915,"15min":1.831},"min":{"1min":0.308,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":8.6430000000000007,"5min":11.804,"15min":11.804},"last":2.125},{"interface":"front","average":{"1min":2.2549999999999999,"5min":1.8420000000000001,"15min":1.4570000000000001},"min":{"1min":0.36399999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.2290000000000001,"5min":10.685,"15min":10.685},"last":2.109}]},{"osd":3,"last update":"Sat Sep 23 23:21:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8290000000000002,"5min":2.6970000000000001,"15min":3.1619999999999999},"min":{"1min":0.33200000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":10.237,"5min":31.564,"15min":109.98699999999999},"last":1.9910000000000001},{"interface":"front","average":{"1min":2.1240000000000001,"5min":2.4630000000000001,"15min":3.2949999999999999},"min":{"1min":0.33000000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.4539999999999997,"5min":32.366,"15min":110.024},"last":2.472}]},{"osd":4,"last update":"Sat Sep 23 23:21:03 2023","interfaces":[{"interface":"back","average":{"1min":9.3170000000000002,"5min":2.5939999999999999,"15min":1.6539999999999999},"min":{"1min":0.31900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":166.05199999999999,"5min":166.05199999999999,"15min":166.05199999999999},"last":2.0609999999999999},{"interface":"front","average":{"1min":11.108000000000001,"5min":3.3610000000000002,"15min":1.9470000000000001},"min":{"1min":0.32000000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":166.631,"5min":166.631,"15min":166.631},"last":0.318}]},{"osd":5,"last update":"Sat Sep 23 23:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":2.0129999999999999,"15min":1.6020000000000001},"min":{"1min":0.33300000000000002,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":3.1779999999999999,"5min":91.899000000000001,"15min":91.899000000000001},"last":2.0979999999999999},{"interface":"front","average":{"1min":1.996,"5min":2.214,"15min":1.605},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":8.4090000000000007,"5min":94.174000000000007,"15min":94.174000000000007},"last":0.28799999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:18:28 2023","interfaces":[{"interface":"back","average":{"1min":2.7250000000000001,"5min":1.2689999999999999,"15min":1.27},"min":{"1min":0.36799999999999999,"5min":0.32300000000000001,"15min":0.32100000000000001},"max":{"1min":13.823,"5min":13.823,"15min":13.823},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.5609999999999999,"5min":0.79900000000000004,"15min":0.98799999999999999},"min":{"1min":0.35999999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":13.103,"5min":13.103,"15min":13.103},"last":0.54400000000000004}]}]},{"osd":1,"up_from":0,"seq":481036337328,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":91,"seq":390842024152,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3205556,"kb_used_data":2450220,"kb_used_omap":5,"kb_used_meta":755322,"kb_avail":90539596,"statfs":{"total":95995035648,"available":92712546304,"internally_reserved":0,"allocated":2509025280,"data_stored":2507676166,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":773450362},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:21:58 2023","interfaces":[{"interface":"back","average":{"1min":4.5069999999999997,"5min":3.1200000000000001,"15min":3.3239999999999998},"min":{"1min":0.20599999999999999,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":21.585999999999999,"5min":26.754999999999999,"15min":155.91999999999999},"last":0.20599999999999999},{"interface":"front","average":{"1min":2.5739999999999998,"5min":2.3069999999999999,"15min":2.8820000000000001},"min":{"1min":0.26000000000000001,"5min":0.20399999999999999,"15min":0.189},"max":{"1min":9.0449999999999999,"5min":16.591000000000001,"15min":153.28899999999999},"last":0.35999999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:21:27 2023","interfaces":[{"interface":"back","average":{"1min":4.0389999999999997,"5min":2.5550000000000002,"15min":2.9430000000000001},"min":{"1min":0.23200000000000001,"5min":0.222,"15min":0.222},"max":{"1min":33.447000000000003,"5min":33.447000000000003,"15min":33.447000000000003},"last":0.29899999999999999},{"interface":"front","average":{"1min":4.8159999999999998,"5min":2.8159999999999998,"15min":3.7280000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":33.186,"5min":36.383000000000003,"15min":36.383000000000003},"last":0.27900000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:21:08 2023","interfaces":[{"interface":"back","average":{"1min":3.7360000000000002,"5min":3.8540000000000001,"15min":3.4390000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.22700000000000001},"max":{"1min":35.148000000000003,"5min":66.501000000000005,"15min":82.691000000000003},"last":0.26500000000000001},{"interface":"front","average":{"1min":4.1909999999999998,"5min":3.3410000000000002,"15min":2.7360000000000002},"min":{"1min":0.28299999999999997,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":35.186999999999998,"5min":66.230000000000004,"15min":82.429000000000002},"last":0.316}]},{"osd":4,"last update":"Sat Sep 23 23:21:36 2023","interfaces":[{"interface":"back","average":{"1min":2.367,"5min":3.1190000000000002,"15min":2.2949999999999999},"min":{"1min":0.32600000000000001,"5min":0.27000000000000002,"15min":0.245},"max":{"1min":9.0079999999999991,"5min":86.915999999999997,"15min":86.915999999999997},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.5189999999999999,"5min":3.2719999999999998,"15min":2.4809999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":8.859,"5min":86.433999999999997,"15min":86.433999999999997},"last":0.48299999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:21:36 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":2.2370000000000001,"15min":2.1280000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":8.891,"5min":15.923,"15min":30.873999999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":2.2759999999999998,"5min":1.738,"15min":1.796},"min":{"1min":0.34799999999999998,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":5.6529999999999996,"5min":15.929,"15min":15.929},"last":0.47099999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:21:36 2023","interfaces":[{"interface":"back","average":{"1min":1.8,"5min":2.726,"15min":2.3359999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.314},"max":{"1min":8.9309999999999992,"5min":30.651,"15min":30.651},"last":0.53100000000000003},{"interface":"front","average":{"1min":2.5569999999999999,"5min":2.46,"15min":2.1549999999999998},"min":{"1min":0.39000000000000001,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":5.6239999999999997,"5min":30.673999999999999,"15min":30.673999999999999},"last":0.435}]},{"osd":7,"last update":"Sat Sep 23 23:18:32 2023","interfaces":[{"interface":"back","average":{"1min":4.306,"5min":2.3620000000000001,"15min":2.1829999999999998},"min":{"1min":0.372,"5min":0.34499999999999997,"15min":0.29299999999999998},"max":{"1min":26.413,"5min":26.413,"15min":26.413},"last":0.42899999999999999},{"interface":"front","average":{"1min":4.7960000000000003,"5min":2.1920000000000002,"15min":2.1219999999999999},"min":{"1min":0.372,"5min":0.36499999999999999,"15min":0.313},"max":{"1min":26.437999999999999,"5min":26.437999999999999,"15min":26.437999999999999},"last":0.52600000000000002}]}]},{"osd":2,"up_from":199,"seq":854698491973,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":381644,"kb_used_data":212676,"kb_used_omap":7,"kb_used_meta":168952,"kb_avail":93363508,"statfs":{"total":95995035648,"available":95604232192,"internally_reserved":0,"allocated":217780224,"data_stored":216473874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":173006929},"hb_peers":[0,1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,0,2,0,0,4,4,8],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:21:33 2023","interfaces":[{"interface":"back","average":{"1min":4.3109999999999999,"5min":3.8450000000000002,"15min":4.9710000000000001},"min":{"1min":0.19,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":41.137,"5min":97.733000000000004,"15min":97.733000000000004},"last":0.189},{"interface":"front","average":{"1min":3.649,"5min":4.0549999999999997,"15min":5.3840000000000003},"min":{"1min":0.20599999999999999,"5min":0.185,"15min":0.185},"max":{"1min":40.954000000000001,"5min":98.379000000000005,"15min":98.379000000000005},"last":0.20899999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:21:33 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":4.5,"15min":11.388},"min":{"1min":0.29099999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.6890000000000001,"5min":130.02799999999999,"15min":130.02799999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.962,"5min":4.4859999999999998,"15min":10.673999999999999},"min":{"1min":0.28499999999999998,"5min":0.245,"15min":0.245},"max":{"1min":9.5790000000000006,"5min":130.13399999999999,"15min":130.13399999999999},"last":0.29499999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:21:33 2023","interfaces":[{"interface":"back","average":{"1min":1.7989999999999999,"5min":3.7069999999999999,"15min":6.4790000000000001},"min":{"1min":0.27800000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.1440000000000001,"5min":72.459999999999994,"15min":72.459999999999994},"last":0.51100000000000001},{"interface":"front","average":{"1min":2.415,"5min":3.581,"15min":5.8650000000000002},"min":{"1min":0.26700000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.6300000000000008,"5min":72.412999999999997,"15min":72.412999999999997},"last":0.44800000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:21:33 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":6.96,"15min":3.6520000000000001},"min":{"1min":0.27800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.2300000000000004,"5min":332.98099999999999,"15min":332.98099999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.1259999999999999,"5min":6.6150000000000002,"15min":3.581},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.5009999999999999,"5min":332.96199999999999,"15min":332.96199999999999},"last":0.66700000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:21:33 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":1.913,"15min":1.9710000000000001},"min":{"1min":0.30499999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.2699999999999996,"5min":20.581,"15min":20.581},"last":0.629},{"interface":"front","average":{"1min":1.2410000000000001,"5min":1.772,"15min":1.992},"min":{"1min":0.377,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.7149999999999999,"5min":20.609000000000002,"15min":20.609000000000002},"last":0.69399999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:21:33 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":7.1139999999999999,"15min":3.8159999999999998},"min":{"1min":0.32600000000000001,"5min":0.31,"15min":0.31},"max":{"1min":5.2549999999999999,"5min":349.32799999999997,"15min":349.32799999999997},"last":0.64600000000000002},{"interface":"front","average":{"1min":1.4039999999999999,"5min":6.9720000000000004,"15min":3.7240000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.532,"5min":349.17700000000002,"15min":349.17700000000002},"last":0.71599999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:18:22 2023","interfaces":[{"interface":"back","average":{"1min":23.006,"5min":6.4870000000000001,"15min":3.734},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":239.768,"5min":239.768,"15min":239.768},"last":0.61499999999999999},{"interface":"front","average":{"1min":22.75,"5min":5.71,"15min":2.8700000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":239.78899999999999,"5min":239.78899999999999,"15min":239.78899999999999},"last":0.54400000000000004}]}]},{"osd":3,"up_from":148,"seq":635655159943,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1108324,"kb_used_data":685340,"kb_used_omap":2,"kb_used_meta":422973,"kb_avail":92636828,"statfs":{"total":95995035648,"available":94860111872,"internally_reserved":0,"allocated":701788160,"data_stored":700513026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":433124405},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:21:20 2023","interfaces":[{"interface":"back","average":{"1min":3.448,"5min":2.0990000000000002,"15min":2.2989999999999999},"min":{"1min":0.158,"5min":0.158,"15min":0.158},"max":{"1min":13.609999999999999,"5min":21.427,"15min":72.778999999999996},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.476,"5min":1.837,"15min":2.3929999999999998},"min":{"1min":0.20200000000000001,"5min":0.193,"15min":0.17999999999999999},"max":{"1min":7.9509999999999996,"5min":23.742999999999999,"15min":72.719999999999999},"last":0.218}]},{"osd":1,"last update":"Sat Sep 23 23:21:20 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":2.5619999999999998,"15min":2.7109999999999999},"min":{"1min":0.216,"5min":0.215,"15min":0.19600000000000001},"max":{"1min":7.4649999999999999,"5min":20.138999999999999,"15min":76.322999999999993},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.2190000000000001,"5min":1.9370000000000001,"15min":2.4649999999999999},"min":{"1min":0.24399999999999999,"5min":0.23499999999999999,"15min":0.185},"max":{"1min":4.9859999999999998,"5min":13.970000000000001,"15min":76.444000000000003},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:21:31 2023","interfaces":[{"interface":"back","average":{"1min":4.1449999999999996,"5min":2.6400000000000001,"15min":3.3780000000000001},"min":{"1min":0.245,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":22.768999999999998,"5min":22.768999999999998,"15min":22.768999999999998},"last":0.38100000000000001},{"interface":"front","average":{"1min":3.8610000000000002,"5min":1.9550000000000001,"15min":1.7809999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":22.800999999999998,"5min":22.800999999999998,"15min":22.800999999999998},"last":0.41799999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:21:20 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":1.6910000000000001,"15min":1.649},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.2089999999999996,"5min":14.754,"15min":14.754},"last":0.56799999999999995},{"interface":"front","average":{"1min":2.4340000000000002,"5min":1.7529999999999999,"15min":2.1909999999999998},"min":{"1min":0.42199999999999999,"5min":0.32500000000000001,"15min":0.28499999999999998},"max":{"1min":9.5960000000000001,"5min":9.5960000000000001,"15min":13.532999999999999},"last":0.68000000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:21:20 2023","interfaces":[{"interface":"back","average":{"1min":2.044,"5min":1.5820000000000001,"15min":2.1869999999999998},"min":{"1min":0.42799999999999999,"5min":0.32900000000000001,"15min":0.28199999999999997},"max":{"1min":10.531000000000001,"5min":10.531000000000001,"15min":75.733000000000004},"last":0.55600000000000005},{"interface":"front","average":{"1min":2.0009999999999999,"5min":1.9199999999999999,"15min":2.169},"min":{"1min":0.36899999999999999,"5min":0.36499999999999999,"15min":0.28399999999999997},"max":{"1min":6.4269999999999996,"5min":9.8119999999999994,"15min":75.686999999999998},"last":0.62}]},{"osd":6,"last update":"Sat Sep 23 23:21:20 2023","interfaces":[{"interface":"back","average":{"1min":2.669,"5min":1.778,"15min":2.0870000000000002},"min":{"1min":0.40699999999999997,"5min":0.38,"15min":0.32200000000000001},"max":{"1min":10.593999999999999,"5min":10.593999999999999,"15min":14.492000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":2.0609999999999999,"5min":2.097,"15min":1.9990000000000001},"min":{"1min":0.45000000000000001,"5min":0.40200000000000002,"15min":0.33300000000000002},"max":{"1min":6.4729999999999999,"5min":9.8379999999999992,"15min":9.8379999999999992},"last":0.70899999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:18:09 2023","interfaces":[{"interface":"back","average":{"1min":2.198,"5min":2.915,"15min":2.5459999999999998},"min":{"1min":0.40200000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":9.4280000000000008,"5min":130.839,"15min":130.839},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.9129999999999998,"5min":2.992,"15min":2.665},"min":{"1min":0.434,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":8.8190000000000008,"5min":130.90000000000001,"15min":130.90000000000001},"last":0.57799999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019260,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3353884,"kb_used_data":2793620,"kb_used_omap":0,"kb_used_meta":560256,"kb_avail":90391268,"statfs":{"total":95995035648,"available":92560658432,"internally_reserved":0,"allocated":2860666880,"data_stored":2859812441,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":573702144},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:21:42 2023","interfaces":[{"interface":"back","average":{"1min":4.1630000000000003,"5min":7.9489999999999998,"15min":5.0659999999999998},"min":{"1min":0.245,"5min":0.223,"15min":0.20799999999999999},"max":{"1min":52.356000000000002,"5min":199.64400000000001,"15min":238.62200000000001},"last":0.31900000000000001},{"interface":"front","average":{"1min":5.3579999999999997,"5min":8.3079999999999998,"15min":5.5309999999999997},"min":{"1min":0.245,"5min":0.245,"15min":0.23300000000000001},"max":{"1min":52.546999999999997,"5min":195.649,"15min":238.58199999999999},"last":0.51500000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:21:58 2023","interfaces":[{"interface":"back","average":{"1min":1.694,"5min":3.5870000000000002,"15min":4.851},"min":{"1min":0.307,"5min":0.23200000000000001,"15min":0.23100000000000001},"max":{"1min":7.5110000000000001,"5min":161.923,"15min":175.00899999999999},"last":0.496},{"interface":"front","average":{"1min":1.8560000000000001,"5min":3.7050000000000001,"15min":5.0030000000000001},"min":{"1min":0.29299999999999998,"5min":0.26500000000000001,"15min":0.25800000000000001},"max":{"1min":6.1210000000000004,"5min":161.39699999999999,"15min":174.821},"last":0.41499999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:21:31 2023","interfaces":[{"interface":"back","average":{"1min":2.375,"5min":2.2370000000000001,"15min":3.0979999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":8.2810000000000006,"5min":39.308,"15min":39.308},"last":2.423},{"interface":"front","average":{"1min":2.2480000000000002,"5min":2.073,"15min":2.4630000000000001},"min":{"1min":0.34999999999999998,"5min":0.316,"15min":0.316},"max":{"1min":4.0819999999999999,"5min":23.138000000000002,"15min":23.138000000000002},"last":0.44500000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:21:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":7.3140000000000001,"15min":4.1130000000000004},"min":{"1min":0.29699999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.5170000000000003,"5min":303.41500000000002,"15min":303.41500000000002},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.722,"5min":7.79,"15min":4.5199999999999996},"min":{"1min":0.36199999999999999,"5min":0.308,"15min":0.308},"max":{"1min":8.5760000000000005,"5min":303.428,"15min":303.428},"last":0.378}]},{"osd":5,"last update":"Sat Sep 23 23:21:58 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":2.7149999999999999,"15min":1.5449999999999999},"min":{"1min":0.33500000000000002,"5min":0.26500000000000001,"15min":0.23699999999999999},"max":{"1min":4.8319999999999999,"5min":160.79499999999999,"15min":160.79499999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.7030000000000001,"5min":2.895,"15min":1.821},"min":{"1min":0.30399999999999999,"5min":0.27400000000000002,"15min":0.27200000000000002},"max":{"1min":6.1710000000000003,"5min":160.61099999999999,"15min":160.61099999999999},"last":0.46100000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.93100000000000005,"15min":1.109},"min":{"1min":0.313,"5min":0.28000000000000003,"15min":0.27200000000000002},"max":{"1min":3.1059999999999999,"5min":6.0910000000000002,"15min":9.3350000000000009},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.6910000000000001,"5min":1.3009999999999999,"15min":1.1990000000000001},"min":{"1min":0.32600000000000001,"5min":0.30599999999999999,"15min":0.27500000000000002},"max":{"1min":4.6509999999999998,"5min":7.2469999999999999,"15min":13.888999999999999},"last":0.53800000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:18:35 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.5609999999999999,"15min":1.3140000000000001},"min":{"1min":0.36299999999999999,"5min":0.315,"15min":0.30499999999999999},"max":{"1min":7.4850000000000003,"5min":14.842000000000001,"15min":14.842000000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.0169999999999999,"5min":1.502,"15min":1.341},"min":{"1min":0.433,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.882,"5min":14.561,"15min":14.561},"last":0.378}]}]},{"osd":5,"up_from":38,"seq":163208757623,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1395908,"kb_used_data":880892,"kb_used_omap":0,"kb_used_meta":515008,"kb_avail":92349244,"statfs":{"total":95995035648,"available":94565625856,"internally_reserved":0,"allocated":902033408,"data_stored":901198952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":527368192},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:21:42 2023","interfaces":[{"interface":"back","average":{"1min":9.7050000000000001,"5min":3.9660000000000002,"15min":2.4769999999999999},"min":{"1min":0.251,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":70.393000000000001,"5min":70.393000000000001,"15min":97.405000000000001},"last":0.24399999999999999},{"interface":"front","average":{"1min":8.6920000000000002,"5min":3.8730000000000002,"15min":2.859},"min":{"1min":0.25800000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":67.721000000000004,"5min":67.721000000000004,"15min":97.447000000000003},"last":0.32300000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:21:51 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":3.2759999999999998,"15min":3.7959999999999998},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.246},"max":{"1min":8.0899999999999999,"5min":91.518000000000001,"15min":193.584},"last":0.28100000000000003},{"interface":"front","average":{"1min":3.3929999999999998,"5min":3.4159999999999999,"15min":4.0190000000000001},"min":{"1min":0.39000000000000001,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":16.312999999999999,"5min":87.718000000000004,"15min":194.03100000000001},"last":0.38500000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:21:26 2023","interfaces":[{"interface":"back","average":{"1min":2.0899999999999999,"5min":1.6579999999999999,"15min":1.52},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.6390000000000002,"5min":6.6840000000000002,"15min":6.6840000000000002},"last":0.56899999999999995},{"interface":"front","average":{"1min":2.1190000000000002,"5min":1.407,"15min":1.623},"min":{"1min":0.30599999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.3249999999999993,"5min":9.391,"15min":9.391},"last":0.45200000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:21:22 2023","interfaces":[{"interface":"back","average":{"1min":1.9019999999999999,"5min":7.8529999999999998,"15min":4.1020000000000003},"min":{"1min":0.36699999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":8.1419999999999995,"5min":159.53299999999999,"15min":159.53299999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":2.0230000000000001,"5min":8.3219999999999992,"15min":4.7999999999999998},"min":{"1min":0.378,"5min":0.312,"15min":0.312},"max":{"1min":6.6689999999999996,"5min":159.54900000000001,"15min":159.54900000000001},"last":0.48999999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:21:08 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":1.216,"15min":1.2849999999999999},"min":{"1min":0.24299999999999999,"5min":0.23999999999999999,"15min":0.221},"max":{"1min":3.101,"5min":7.3399999999999999,"15min":14.538},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.331,"5min":0.98899999999999999,"15min":0.93999999999999995},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.25},"max":{"1min":8.1829999999999998,"5min":11.923999999999999,"15min":11.923999999999999},"last":0.36899999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2.456,"5min":4.3860000000000001,"15min":2.27},"min":{"1min":0.315,"5min":0.28299999999999997,"15min":0.255},"max":{"1min":17.515999999999998,"5min":310.68799999999999,"15min":310.68799999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":2.6890000000000001,"5min":4.7670000000000003,"15min":2.5099999999999998},"min":{"1min":0.30399999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":20.335000000000001,"5min":311.45400000000001,"15min":311.45400000000001},"last":0.46100000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:18:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2010000000000001,"5min":1.286,"15min":1.3700000000000001},"min":{"1min":0.36699999999999999,"5min":0.32700000000000001,"15min":0.29199999999999998},"max":{"1min":12.09,"5min":12.09,"15min":14.929},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.319,"15min":1.276},"min":{"1min":0.371,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":5.7350000000000003,"5min":6.7089999999999996,"15min":15.090999999999999},"last":0.54400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2496925696,"data_stored":2496921624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":6946816,"data_stored":6946816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":214302720,"data_stored":214302720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":693960704,"data_stored":693960704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2856980480,"data_stored":2856976408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":896204800,"data_stored":896204800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1439961088,"data_stored":1439957016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:22:06.281 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T23:22:06.282 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 21 2023-09-23T23:22:06.324 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:22:06.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:06.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:22:06.428 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:06.474 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:06.517 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:06.579 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:06.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:06.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:06.688 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:06.726 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:06.849 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:22:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:22:06] "GET /metrics HTTP/1.1" 200 36645 "" "Prometheus/2.43.0" 2023-09-23T23:22:06.890 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:06.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:06.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:06.996 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:07.025 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:07.074 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:07.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:07.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:07.155 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:07 smithi110 ceph-mon[212939]: pgmap v932: 99 pgs: 1 active+recovery_wait+degraded, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.6 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 294 op/s; 12875/126639 objects degraded (10.167%); 24344/126639 objects misplaced (19.223%); 8.0 MiB/s, 148 objects/s recovering 2023-09-23T23:22:07.181 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:22:07.259 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:07.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:07 smithi138 ceph-mon[193606]: pgmap v932: 99 pgs: 1 active+recovery_wait+degraded, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.6 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 294 op/s; 12875/126639 objects degraded (10.167%); 24344/126639 objects misplaced (19.223%); 8.0 MiB/s, 148 objects/s recovering 2023-09-23T23:22:07.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:07 smithi110 ceph-mon[221102]: pgmap v932: 99 pgs: 1 active+recovery_wait+degraded, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.6 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 294 op/s; 12875/126639 objects degraded (10.167%); 24344/126639 objects misplaced (19.223%); 8.0 MiB/s, 148 objects/s recovering 2023-09-23T23:22:07.701 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:22:07.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:07.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:22:07.807 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:07.833 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:07.874 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:07.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:07.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:07.979 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:08.005 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:08.048 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:08.048 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:22:08.049 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:22:08.049 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:22:08.049 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:22:08.049 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:22:08.049 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:22:08.049 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:22:08.049 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:22:08.049 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:22:08.049 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:22:08.049 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:22:08.050 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:22:08.050 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:22:08.050 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:22:08.050 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:22:08.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:08.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:08.154 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:08.173 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:08 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/924932047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:08.173 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:08 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2715183030' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:22:08.173 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:08 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:22:08.174 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:08 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/924932047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:08.174 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:08 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2715183030' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:22:08.174 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:08 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:22:08.189 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:22:08.236 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:08.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:08 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/924932047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:08.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:08 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2715183030' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:22:08.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:08 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:22:09.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:09 smithi138 ceph-mon[193606]: pgmap v933: 99 pgs: 1 active+recovery_wait+degraded, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.6 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 294 op/s; 12875/126639 objects degraded (10.167%); 24344/126639 objects misplaced (19.223%); 8.0 MiB/s, 148 objects/s recovering 2023-09-23T23:22:09.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:09 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:22:09.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:09 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:22:09.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:09 smithi138 ceph-mon[193606]: osdmap e306: 8 total, 7 up, 7 in 2023-09-23T23:22:09.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:09 smithi138 ceph-mon[193606]: failed to encode map e306 with expected crc 2023-09-23T23:22:09.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:09 smithi138 ceph-mon[193606]: failed to encode map e306 with expected crc 2023-09-23T23:22:09.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:09 smithi138 ceph-mon[193606]: failed to encode map e306 with expected crc 2023-09-23T23:22:09.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:09 smithi138 ceph-mon[193606]: failed to encode map e306 with expected crc 2023-09-23T23:22:09.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:09 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 12875/126639 objects degraded (10.167%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:09.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[212939]: pgmap v933: 99 pgs: 1 active+recovery_wait+degraded, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.6 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 294 op/s; 12875/126639 objects degraded (10.167%); 24344/126639 objects misplaced (19.223%); 8.0 MiB/s, 148 objects/s recovering 2023-09-23T23:22:09.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:22:09.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:22:09.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[212939]: osdmap e306: 8 total, 7 up, 7 in 2023-09-23T23:22:09.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[212939]: failed to encode map e306 with expected crc 2023-09-23T23:22:09.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[212939]: failed to encode map e306 with expected crc 2023-09-23T23:22:09.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[212939]: failed to encode map e306 with expected crc 2023-09-23T23:22:09.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[212939]: failed to encode map e306 with expected crc 2023-09-23T23:22:09.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 12875/126639 objects degraded (10.167%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:09.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[221102]: pgmap v933: 99 pgs: 1 active+recovery_wait+degraded, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.6 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 294 op/s; 12875/126639 objects degraded (10.167%); 24344/126639 objects misplaced (19.223%); 8.0 MiB/s, 148 objects/s recovering 2023-09-23T23:22:09.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:22:09.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:22:09.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[221102]: osdmap e306: 8 total, 7 up, 7 in 2023-09-23T23:22:09.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[221102]: failed to encode map e306 with expected crc 2023-09-23T23:22:09.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[221102]: failed to encode map e306 with expected crc 2023-09-23T23:22:09.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[221102]: failed to encode map e306 with expected crc 2023-09-23T23:22:09.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[221102]: failed to encode map e306 with expected crc 2023-09-23T23:22:09.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:09 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 12875/126639 objects degraded (10.167%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:09.615 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:09.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:09.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:09.722 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:09.754 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:09.799 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:09.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:09.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:09.832 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:22:09.832 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:22:09.908 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:09.935 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:09.978 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:09.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:09.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:10.065 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:10.083 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:10.111 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:10.160 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:10.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:10.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:10.267 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:10.306 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:10.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:10 smithi138 ceph-mon[193606]: failed to encode map e307 with expected crc 2023-09-23T23:22:10.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:10 smithi138 ceph-mon[193606]: osdmap e307: 8 total, 7 up, 7 in 2023-09-23T23:22:10.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:10 smithi138 ceph-mon[193606]: failed to encode map e307 with expected crc 2023-09-23T23:22:10.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:10 smithi110 ceph-mon[221102]: failed to encode map e307 with expected crc 2023-09-23T23:22:10.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:10 smithi110 ceph-mon[221102]: osdmap e307: 8 total, 7 up, 7 in 2023-09-23T23:22:10.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:10 smithi110 ceph-mon[221102]: failed to encode map e307 with expected crc 2023-09-23T23:22:10.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:10 smithi110 ceph-mon[212939]: failed to encode map e307 with expected crc 2023-09-23T23:22:10.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:10 smithi110 ceph-mon[212939]: osdmap e307: 8 total, 7 up, 7 in 2023-09-23T23:22:10.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:10 smithi110 ceph-mon[212939]: failed to encode map e307 with expected crc 2023-09-23T23:22:10.576 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:10.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:10.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:10.680 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:10.705 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:22:10.705 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:22:10.706 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:22:10.706 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:22:10.706 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:22:10.706 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:10.748 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:10.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:10.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:10.853 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:10.882 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:10.927 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:10.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:10.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:11.033 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:11.062 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:11.117 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:11.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:11.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:11.224 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:11.250 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:11.295 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:11.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:11.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:11.403 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:22:11.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:11 smithi138 ceph-mon[193606]: pgmap v936: 99 pgs: 1 active+recovery_wait+degraded, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 282 op/s; 12798/124002 objects degraded (10.321%); 23847/124002 objects misplaced (19.231%); 4.7 MiB/s, 102 objects/s recovering 2023-09-23T23:22:11.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:11 smithi110 ceph-mon[212939]: pgmap v936: 99 pgs: 1 active+recovery_wait+degraded, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 282 op/s; 12798/124002 objects degraded (10.321%); 23847/124002 objects misplaced (19.231%); 4.7 MiB/s, 102 objects/s recovering 2023-09-23T23:22:11.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:11 smithi110 ceph-mon[221102]: pgmap v936: 99 pgs: 1 active+recovery_wait+degraded, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 72 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 282 op/s; 12798/124002 objects degraded (10.321%); 23847/124002 objects misplaced (19.231%); 4.7 MiB/s, 102 objects/s recovering 2023-09-23T23:22:11.451 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:11.833 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:22:12.096 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:22:12.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:12 smithi110 ceph-mon[212939]: pgmap v937: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 13 active+clean+remapped, 1 active+recovering, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 3978/121851 objects degraded (3.265%); 26953/121851 objects misplaced (22.120%); 14 MiB/s, 315 objects/s recovering 2023-09-23T23:22:12.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:12 smithi110 ceph-mon[221102]: pgmap v937: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 13 active+clean+remapped, 1 active+recovering, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 3978/121851 objects degraded (3.265%); 26953/121851 objects misplaced (22.120%); 14 MiB/s, 315 objects/s recovering 2023-09-23T23:22:12.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:12 smithi138 ceph-mon[193606]: pgmap v937: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 13 active+clean+remapped, 1 active+recovering, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 3978/121851 objects degraded (3.265%); 26953/121851 objects misplaced (22.120%); 14 MiB/s, 315 objects/s recovering 2023-09-23T23:22:12.880 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:22:12.936 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:22:12.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:12.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:22:13.042 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:13.072 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:13.126 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:13.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:13.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:13.147 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:13.228 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:13.231 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:13.260 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:13.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:13.488 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:13.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:13.589 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:13.626 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:13.688 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:13.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:13.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:13.792 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:13.839 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:14.057 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:14.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:14.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:14.163 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:14.192 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:14.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:22:14.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:14.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:22:14.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 3978/121851 objects degraded (3.265%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:14.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[212939]: failed to encode map e308 with expected crc 2023-09-23T23:22:14.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[212939]: osdmap e308: 8 total, 7 up, 7 in 2023-09-23T23:22:14.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[212939]: failed to encode map e308 with expected crc 2023-09-23T23:22:14.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[212939]: failed to encode map e308 with expected crc 2023-09-23T23:22:14.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/359917272' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T23:22:14.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:22:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:22:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 3978/121851 objects degraded (3.265%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[221102]: failed to encode map e308 with expected crc 2023-09-23T23:22:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[221102]: osdmap e308: 8 total, 7 up, 7 in 2023-09-23T23:22:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[221102]: failed to encode map e308 with expected crc 2023-09-23T23:22:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[221102]: failed to encode map e308 with expected crc 2023-09-23T23:22:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:13 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/359917272' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T23:22:14.429 INFO:teuthology.orchestra.run.smithi110.stderr:set pool 1 pg_num to 21 2023-09-23T23:22:14.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:13 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:22:14.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:22:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:13 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 3978/121851 objects degraded (3.265%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:13 smithi138 ceph-mon[193606]: failed to encode map e308 with expected crc 2023-09-23T23:22:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:13 smithi138 ceph-mon[193606]: osdmap e308: 8 total, 7 up, 7 in 2023-09-23T23:22:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:13 smithi138 ceph-mon[193606]: failed to encode map e308 with expected crc 2023-09-23T23:22:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:13 smithi138 ceph-mon[193606]: failed to encode map e308 with expected crc 2023-09-23T23:22:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:13 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/359917272' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T23:22:14.657 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:14.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:14.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:14.764 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:22:14.827 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:14 smithi110 ceph-mon[212939]: pgmap v939: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 13 active+clean+remapped, 1 active+recovering, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 265 op/s; 3978/121851 objects degraded (3.265%); 26953/121851 objects misplaced (22.120%); 12 MiB/s, 284 objects/s recovering 2023-09-23T23:22:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:14 smithi110 ceph-mon[212939]: failed to encode map e309 with expected crc 2023-09-23T23:22:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:14 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/359917272' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T23:22:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:14 smithi110 ceph-mon[212939]: osdmap e309: 8 total, 7 up, 7 in 2023-09-23T23:22:15.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:14 smithi110 ceph-mon[221102]: pgmap v939: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 13 active+clean+remapped, 1 active+recovering, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 265 op/s; 3978/121851 objects degraded (3.265%); 26953/121851 objects misplaced (22.120%); 12 MiB/s, 284 objects/s recovering 2023-09-23T23:22:15.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:14 smithi110 ceph-mon[221102]: failed to encode map e309 with expected crc 2023-09-23T23:22:15.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:14 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/359917272' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T23:22:15.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:14 smithi110 ceph-mon[221102]: osdmap e309: 8 total, 7 up, 7 in 2023-09-23T23:22:15.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:14 smithi138 ceph-mon[193606]: pgmap v939: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 13 active+clean+remapped, 1 active+recovering, 5 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 265 op/s; 3978/121851 objects degraded (3.265%); 26953/121851 objects misplaced (22.120%); 12 MiB/s, 284 objects/s recovering 2023-09-23T23:22:15.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:14 smithi138 ceph-mon[193606]: failed to encode map e309 with expected crc 2023-09-23T23:22:15.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:14 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/359917272' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T23:22:15.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:14 smithi138 ceph-mon[193606]: osdmap e309: 8 total, 7 up, 7 in 2023-09-23T23:22:15.694 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:22:15.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:15.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:22:15.799 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:22:15.826 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:16.035 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:16.066 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:22:16.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:16.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:22:16.185 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:16.238 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:16.258 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:15 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:22:16.258 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:15 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T23:22:16.259 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:15 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T23:22:16.259 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:15 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:22:16.259 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:15 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T23:22:16.259 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:15 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T23:22:16.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:15 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:22:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:15 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T23:22:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:15 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T23:22:16.645 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:16.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:16.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:16.751 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:16.821 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:16.934 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:22:16.953 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:22:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:22:16] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.43.0" 2023-09-23T23:22:16.964 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:22:16.994 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:22:17.014 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:22:17.027 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:22:17.053 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:22:17.079 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:22:17.099 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:17.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:17.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:17.108 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:22:17.108 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:22:17.109 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:22:17.109 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:22:17.109 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:22:17.109 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:22:17.109 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:22:17.109 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:22:17.109 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:22:17.204 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:17.234 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:17.290 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:17.295 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:17 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:22:17.295 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:17 smithi138 ceph-mon[193606]: pgmap v941: 99 pgs: 1 active+remapped+backfilling, 14 active+clean+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 2475/118260 objects degraded (2.093%); 25431/118260 objects misplaced (21.504%); 33 MiB/s, 615 objects/s recovering 2023-09-23T23:22:17.295 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:17 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T23:22:17.295 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:17 smithi138 ceph-mon[193606]: osdmap e310: 8 total, 7 up, 7 in 2023-09-23T23:22:17.295 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:17 smithi138 ceph-mon[193606]: failed to encode map e310 with expected crc 2023-09-23T23:22:17.295 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:17 smithi138 ceph-mon[193606]: failed to encode map e310 with expected crc 2023-09-23T23:22:17.295 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:17 smithi138 ceph-mon[193606]: failed to encode map e310 with expected crc 2023-09-23T23:22:17.295 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:17 smithi138 ceph-mon[193606]: failed to encode map e310 with expected crc 2023-09-23T23:22:17.295 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:17 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/195858504' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:22:17.296 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:17 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/557661651' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:17.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:17.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:17.397 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:17.425 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:16 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:22:17.425 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:16 smithi110 ceph-mon[212939]: pgmap v941: 99 pgs: 1 active+remapped+backfilling, 14 active+clean+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 2475/118260 objects degraded (2.093%); 25431/118260 objects misplaced (21.504%); 33 MiB/s, 615 objects/s recovering 2023-09-23T23:22:17.425 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:16 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T23:22:17.425 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:16 smithi110 ceph-mon[212939]: osdmap e310: 8 total, 7 up, 7 in 2023-09-23T23:22:17.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:16 smithi110 ceph-mon[212939]: failed to encode map e310 with expected crc 2023-09-23T23:22:17.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:16 smithi110 ceph-mon[212939]: failed to encode map e310 with expected crc 2023-09-23T23:22:17.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:16 smithi110 ceph-mon[212939]: failed to encode map e310 with expected crc 2023-09-23T23:22:17.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:16 smithi110 ceph-mon[212939]: failed to encode map e310 with expected crc 2023-09-23T23:22:17.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:17 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/195858504' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:22:17.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:17 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/557661651' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:17.427 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:17 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:22:17.427 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:17 smithi110 ceph-mon[221102]: pgmap v941: 99 pgs: 1 active+remapped+backfilling, 14 active+clean+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 2475/118260 objects degraded (2.093%); 25431/118260 objects misplaced (21.504%); 33 MiB/s, 615 objects/s recovering 2023-09-23T23:22:17.427 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:17 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T23:22:17.427 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:17 smithi110 ceph-mon[221102]: osdmap e310: 8 total, 7 up, 7 in 2023-09-23T23:22:17.427 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:17 smithi110 ceph-mon[221102]: failed to encode map e310 with expected crc 2023-09-23T23:22:17.427 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:17 smithi110 ceph-mon[221102]: failed to encode map e310 with expected crc 2023-09-23T23:22:17.428 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:17 smithi110 ceph-mon[221102]: failed to encode map e310 with expected crc 2023-09-23T23:22:17.428 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:17 smithi110 ceph-mon[221102]: failed to encode map e310 with expected crc 2023-09-23T23:22:17.428 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:17 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/195858504' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:22:17.428 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:17 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/557661651' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:17.428 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:18.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:18 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:22:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/195858504' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:22:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:18 smithi138 ceph-mon[193606]: osdmap e311: 8 total, 7 up, 7 in 2023-09-23T23:22:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:18 smithi138 ceph-mon[193606]: failed to encode map e311 with expected crc 2023-09-23T23:22:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:18 smithi138 ceph-mon[193606]: failed to encode map e311 with expected crc 2023-09-23T23:22:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:18 smithi138 ceph-mon[193606]: failed to encode map e311 with expected crc 2023-09-23T23:22:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:18 smithi138 ceph-mon[193606]: failed to encode map e311 with expected crc 2023-09-23T23:22:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:18 smithi138 ceph-mon[193606]: failed to encode map e311 with expected crc 2023-09-23T23:22:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:18 smithi138 ceph-mon[193606]: failed to encode map e311 with expected crc 2023-09-23T23:22:18.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:22:18.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/195858504' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:22:18.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[212939]: osdmap e311: 8 total, 7 up, 7 in 2023-09-23T23:22:18.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[212939]: failed to encode map e311 with expected crc 2023-09-23T23:22:18.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[212939]: failed to encode map e311 with expected crc 2023-09-23T23:22:18.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[212939]: failed to encode map e311 with expected crc 2023-09-23T23:22:18.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[212939]: failed to encode map e311 with expected crc 2023-09-23T23:22:18.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[212939]: failed to encode map e311 with expected crc 2023-09-23T23:22:18.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[212939]: failed to encode map e311 with expected crc 2023-09-23T23:22:18.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:22:18.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/195858504' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:22:18.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[221102]: osdmap e311: 8 total, 7 up, 7 in 2023-09-23T23:22:18.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[221102]: failed to encode map e311 with expected crc 2023-09-23T23:22:18.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[221102]: failed to encode map e311 with expected crc 2023-09-23T23:22:18.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[221102]: failed to encode map e311 with expected crc 2023-09-23T23:22:18.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[221102]: failed to encode map e311 with expected crc 2023-09-23T23:22:18.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[221102]: failed to encode map e311 with expected crc 2023-09-23T23:22:18.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:18 smithi110 ceph-mon[221102]: failed to encode map e311 with expected crc 2023-09-23T23:22:19.096 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:19.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:19.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:19.109 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:22:19.109 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:22:19.203 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:19.247 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:19 smithi110 ceph-mon[221102]: pgmap v944: 109 pgs: 10 unknown, 1 active+remapped+backfilling, 14 active+clean+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 289 op/s; 2475/118260 objects degraded (2.093%); 25431/118260 objects misplaced (21.504%); 33 MiB/s, 535 objects/s recovering 2023-09-23T23:22:19.247 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:19 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2475/118260 objects degraded (2.093%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:19.248 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:19 smithi110 ceph-mon[212939]: pgmap v944: 109 pgs: 10 unknown, 1 active+remapped+backfilling, 14 active+clean+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 289 op/s; 2475/118260 objects degraded (2.093%); 25431/118260 objects misplaced (21.504%); 33 MiB/s, 535 objects/s recovering 2023-09-23T23:22:19.248 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:19 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2475/118260 objects degraded (2.093%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:19.252 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:19.416 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:19.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:19.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:19.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:19 smithi138 ceph-mon[193606]: pgmap v944: 109 pgs: 10 unknown, 1 active+remapped+backfilling, 14 active+clean+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 2 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 4 active+undersized, 72 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 289 op/s; 2475/118260 objects degraded (2.093%); 25431/118260 objects misplaced (21.504%); 33 MiB/s, 535 objects/s recovering 2023-09-23T23:22:19.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:19 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2475/118260 objects degraded (2.093%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:19.524 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:19.567 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:19.637 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:19.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:19.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:19.646 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:22:19.646 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:22:19.747 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:22:19.781 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:19.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:19.841 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:22:19.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:22:19.942 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:19.972 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:20.022 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:20.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:20.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:20.127 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:20.152 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:20.195 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:20.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:20.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:20.301 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:20.332 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:20.419 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:20 smithi110 ceph-mon[212939]: osdmap e312: 8 total, 7 up, 7 in 2023-09-23T23:22:20.419 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:20 smithi110 ceph-mon[212939]: failed to encode map e312 with expected crc 2023-09-23T23:22:20.419 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:20 smithi110 ceph-mon[212939]: failed to encode map e312 with expected crc 2023-09-23T23:22:20.419 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:20 smithi110 ceph-mon[212939]: failed to encode map e312 with expected crc 2023-09-23T23:22:20.419 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:20 smithi110 ceph-mon[212939]: pgmap v946: 109 pgs: 4 unknown, 2 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 78 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 370 op/s; 2475/116157 objects degraded (2.131%); 24682/116157 objects misplaced (21.249%); 30 MiB/s, 487 objects/s recovering 2023-09-23T23:22:20.419 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:20 smithi110 ceph-mon[221102]: osdmap e312: 8 total, 7 up, 7 in 2023-09-23T23:22:20.420 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:20 smithi110 ceph-mon[221102]: failed to encode map e312 with expected crc 2023-09-23T23:22:20.420 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:20 smithi110 ceph-mon[221102]: failed to encode map e312 with expected crc 2023-09-23T23:22:20.420 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:20 smithi110 ceph-mon[221102]: failed to encode map e312 with expected crc 2023-09-23T23:22:20.420 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:20 smithi110 ceph-mon[221102]: pgmap v946: 109 pgs: 4 unknown, 2 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 78 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 370 op/s; 2475/116157 objects degraded (2.131%); 24682/116157 objects misplaced (21.249%); 30 MiB/s, 487 objects/s recovering 2023-09-23T23:22:20.539 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:20.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:20.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:20.606 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:22:20.646 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:20.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:20 smithi138 ceph-mon[193606]: osdmap e312: 8 total, 7 up, 7 in 2023-09-23T23:22:20.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:20 smithi138 ceph-mon[193606]: failed to encode map e312 with expected crc 2023-09-23T23:22:20.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:20 smithi138 ceph-mon[193606]: failed to encode map e312 with expected crc 2023-09-23T23:22:20.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:20 smithi138 ceph-mon[193606]: failed to encode map e312 with expected crc 2023-09-23T23:22:20.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:20 smithi138 ceph-mon[193606]: pgmap v946: 109 pgs: 4 unknown, 2 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 78 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 370 op/s; 2475/116157 objects degraded (2.131%); 24682/116157 objects misplaced (21.249%); 30 MiB/s, 487 objects/s recovering 2023-09-23T23:22:20.694 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:20.741 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:20.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:20.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:20.848 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:20.883 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:20.939 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:20.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:20.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:21.007 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:22:21.046 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:21.060 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:22:21.071 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 2] out_osds: [1] dead_osds: [7] live_osds: [0, 1, 3, 4, 5, 6, 2] 2023-09-23T23:22:21.071 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:22:21.072 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T23:22:21.072 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T23:22:21.072 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T23:22:21.072 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T23:22:21.083 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:21.111 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:22:21.146 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:21.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:21.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:21.251 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:21.311 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:21.400 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:21.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:21.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:21.514 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:21.546 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:21.582 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:21.632 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:21 smithi138 ceph-mon[193606]: failed to encode map e313 with expected crc 2023-09-23T23:22:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:21 smithi138 ceph-mon[193606]: osdmap e313: 8 total, 7 up, 7 in 2023-09-23T23:22:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:21 smithi138 ceph-mon[193606]: failed to encode map e313 with expected crc 2023-09-23T23:22:21.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:21 smithi110 ceph-mon[212939]: failed to encode map e313 with expected crc 2023-09-23T23:22:21.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:21 smithi110 ceph-mon[212939]: osdmap e313: 8 total, 7 up, 7 in 2023-09-23T23:22:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:21 smithi110 ceph-mon[212939]: failed to encode map e313 with expected crc 2023-09-23T23:22:21.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:21 smithi110 ceph-mon[221102]: failed to encode map e313 with expected crc 2023-09-23T23:22:21.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:21 smithi110 ceph-mon[221102]: osdmap e313: 8 total, 7 up, 7 in 2023-09-23T23:22:21.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:21 smithi110 ceph-mon[221102]: failed to encode map e313 with expected crc 2023-09-23T23:22:21.755 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:21.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:21.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:21.861 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:21.909 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:21.969 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:21.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:21.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:22.075 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:22.104 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:22.125 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:22.153 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:22.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:22.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:22.256 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:22.286 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:22.334 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:22.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:22.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:22.438 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:22.469 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:22.523 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:22.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:22.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:22.627 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:22.657 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:22.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:22 smithi138 ceph-mon[193606]: pgmap v948: 109 pgs: 1 peering, 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 82 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 300 op/s; 2374/113247 objects degraded (2.096%); 22361/113247 objects misplaced (19.745%); 23 MiB/s, 374 objects/s recovering 2023-09-23T23:22:22.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:22 smithi110 ceph-mon[212939]: pgmap v948: 109 pgs: 1 peering, 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 82 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 300 op/s; 2374/113247 objects degraded (2.096%); 22361/113247 objects misplaced (19.745%); 23 MiB/s, 374 objects/s recovering 2023-09-23T23:22:22.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:22 smithi110 ceph-mon[221102]: pgmap v948: 109 pgs: 1 peering, 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 82 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 300 op/s; 2374/113247 objects degraded (2.096%); 22361/113247 objects misplaced (19.745%); 23 MiB/s, 374 objects/s recovering 2023-09-23T23:22:22.703 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:22.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:22.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:22.807 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:22.840 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:23.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:23 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2374/113247 objects degraded (2.096%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:23.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:23 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2374/113247 objects degraded (2.096%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:23.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:23 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2374/113247 objects degraded (2.096%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:23.895 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:23.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:23.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:24.006 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:24.059 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:24.166 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:24.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:24.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:24.272 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:24.319 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:24.436 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:24 smithi110 ceph-mon[212939]: pgmap v949: 109 pgs: 1 peering, 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 82 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 254 op/s; 2374/113247 objects degraded (2.096%); 22361/113247 objects misplaced (19.745%); 20 MiB/s, 317 objects/s recovering 2023-09-23T23:22:24.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:24 smithi138 ceph-mon[193606]: pgmap v949: 109 pgs: 1 peering, 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 82 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 254 op/s; 2374/113247 objects degraded (2.096%); 22361/113247 objects misplaced (19.745%); 20 MiB/s, 317 objects/s recovering 2023-09-23T23:22:24.701 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:24 smithi110 ceph-mon[221102]: pgmap v949: 109 pgs: 1 peering, 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 82 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 254 op/s; 2374/113247 objects degraded (2.096%); 22361/113247 objects misplaced (19.745%); 20 MiB/s, 317 objects/s recovering 2023-09-23T23:22:25.232 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:25.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:25.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:25.337 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:22:25.368 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:25.429 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:22:25.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:25.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:22:25.459 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:22:25.534 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:25.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:25 smithi138 ceph-mon[193606]: failed to encode map e314 with expected crc 2023-09-23T23:22:25.562 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:25 smithi138 ceph-mon[193606]: failed to encode map e314 with expected crc 2023-09-23T23:22:25.562 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:25 smithi138 ceph-mon[193606]: osdmap e314: 8 total, 7 up, 7 in 2023-09-23T23:22:25.562 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:25 smithi138 ceph-mon[193606]: failed to encode map e314 with expected crc 2023-09-23T23:22:25.562 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:25 smithi138 ceph-mon[193606]: failed to encode map e314 with expected crc 2023-09-23T23:22:25.562 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:25 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/801266002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:22:25.562 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:25 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:22:25.601 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:25.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:25 smithi110 ceph-mon[212939]: failed to encode map e314 with expected crc 2023-09-23T23:22:25.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:25 smithi110 ceph-mon[212939]: failed to encode map e314 with expected crc 2023-09-23T23:22:25.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:25 smithi110 ceph-mon[212939]: osdmap e314: 8 total, 7 up, 7 in 2023-09-23T23:22:25.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:25 smithi110 ceph-mon[212939]: failed to encode map e314 with expected crc 2023-09-23T23:22:25.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:25 smithi110 ceph-mon[212939]: failed to encode map e314 with expected crc 2023-09-23T23:22:25.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:25 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/801266002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:22:25.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:25 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:22:25.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:25 smithi110 ceph-mon[221102]: failed to encode map e314 with expected crc 2023-09-23T23:22:25.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:25 smithi110 ceph-mon[221102]: failed to encode map e314 with expected crc 2023-09-23T23:22:25.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:25 smithi110 ceph-mon[221102]: osdmap e314: 8 total, 7 up, 7 in 2023-09-23T23:22:25.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:25 smithi110 ceph-mon[221102]: failed to encode map e314 with expected crc 2023-09-23T23:22:25.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:25 smithi110 ceph-mon[221102]: failed to encode map e314 with expected crc 2023-09-23T23:22:25.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:25 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/801266002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:22:25.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:25 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:22:25.772 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:25.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:25.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:25.816 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:22:25.816 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:22:25.878 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:25.911 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:26.102 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:22:26.102 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T23:22:26.102 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:22:26.108 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:26.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:26.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:26.213 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:22:26.245 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:26.433 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:26 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:22:26.434 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:26 smithi138 ceph-mon[193606]: failed to encode map e315 with expected crc 2023-09-23T23:22:26.434 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:26 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:22:26.434 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:26 smithi138 ceph-mon[193606]: osdmap e315: 8 total, 7 up, 7 in 2023-09-23T23:22:26.434 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:26 smithi138 ceph-mon[193606]: failed to encode map e315 with expected crc 2023-09-23T23:22:26.434 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:26 smithi138 ceph-mon[193606]: pgmap v952: 109 pgs: 1 peering, 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 83 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 452 op/s; 2259/108006 objects degraded (2.092%); 19225/108006 objects misplaced (17.800%); 41 MiB/s, 663 objects/s recovering 2023-09-23T23:22:26.435 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:26 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:22:26.435 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:26 smithi110 ceph-mon[212939]: failed to encode map e315 with expected crc 2023-09-23T23:22:26.435 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:26 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:22:26.435 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:26 smithi110 ceph-mon[212939]: osdmap e315: 8 total, 7 up, 7 in 2023-09-23T23:22:26.435 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:26 smithi110 ceph-mon[212939]: failed to encode map e315 with expected crc 2023-09-23T23:22:26.435 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:26 smithi110 ceph-mon[212939]: pgmap v952: 109 pgs: 1 peering, 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 83 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 452 op/s; 2259/108006 objects degraded (2.092%); 19225/108006 objects misplaced (17.800%); 41 MiB/s, 663 objects/s recovering 2023-09-23T23:22:26.435 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:26 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:22:26.436 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:26 smithi110 ceph-mon[221102]: failed to encode map e315 with expected crc 2023-09-23T23:22:26.436 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:26 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:22:26.436 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:26 smithi110 ceph-mon[221102]: osdmap e315: 8 total, 7 up, 7 in 2023-09-23T23:22:26.436 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:26 smithi110 ceph-mon[221102]: failed to encode map e315 with expected crc 2023-09-23T23:22:26.436 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:26 smithi110 ceph-mon[221102]: pgmap v952: 109 pgs: 1 peering, 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 83 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 452 op/s; 2259/108006 objects degraded (2.092%); 19225/108006 objects misplaced (17.800%); 41 MiB/s, 663 objects/s recovering 2023-09-23T23:22:27.034 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:22:27.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:27.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:22:27.057 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:22:27.126 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:22:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:22:26] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.43.0" 2023-09-23T23:22:27.139 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:22:27.176 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:27.233 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:22:27.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:27.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:22:27.341 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:27.386 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:27.448 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:27.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:27.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:27.468 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:22:27.488 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:22:27.514 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:22:27.535 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:22:27.552 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:27.556 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:22:27.581 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:22:27.583 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:27.606 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:22:27.633 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:22:27.633 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:22:27.633 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:22:27.633 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:22:27.633 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:22:27.634 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:22:27.634 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:22:27.634 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:22:27.634 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:22:27.652 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:27 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1830820082' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:27.653 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:27 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1830820082' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:27.745 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:27.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:27.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:27.765 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:27 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1830820082' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:27.855 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:27.895 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:27.957 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:27.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:27.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:28.060 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:28.086 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:28.129 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:28.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:28.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:28.238 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:28.271 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:28.317 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:28.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:28.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:28.420 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:28.448 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:28.475 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:22:28.476 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:22:28.497 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:28.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:28.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:28.603 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:28.650 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:28.848 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:22:28.932 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:28.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:28.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:29.037 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:29.073 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:29.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:28 smithi138 ceph-mon[193606]: pgmap v953: 109 pgs: 1 peering, 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 83 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 244 op/s; 2259/108006 objects degraded (2.092%); 19225/108006 objects misplaced (17.800%); 19 MiB/s, 306 objects/s recovering 2023-09-23T23:22:29.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:29.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:22:29.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:28 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2259/108006 objects degraded (2.092%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:29.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:28 smithi110 ceph-mon[221102]: pgmap v953: 109 pgs: 1 peering, 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 83 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 244 op/s; 2259/108006 objects degraded (2.092%); 19225/108006 objects misplaced (17.800%); 19 MiB/s, 306 objects/s recovering 2023-09-23T23:22:29.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:29.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:22:29.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:28 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2259/108006 objects degraded (2.092%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:29.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:28 smithi110 ceph-mon[212939]: pgmap v953: 109 pgs: 1 peering, 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 83 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 244 op/s; 2259/108006 objects degraded (2.092%); 19225/108006 objects misplaced (17.800%); 19 MiB/s, 306 objects/s recovering 2023-09-23T23:22:29.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:29.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:22:29.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:28 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2259/108006 objects degraded (2.092%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:29.231 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:29.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:29.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:29.297 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:29.335 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:29.370 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:29.424 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:29.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:29.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:29.498 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:22:29.498 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:22:29.499 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:22:29.499 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:22:29.499 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:22:29.528 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:29.575 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:29.636 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:29.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:29.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:29.741 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:29.770 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:29.823 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:29.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:29.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:29.927 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:29.955 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:30.003 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:30.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:30.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:30.107 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:30.132 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:30.181 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:30.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:30.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:30.284 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:30.310 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:30.476 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:22:30.498 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:30.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:30.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:30.567 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:22:30.602 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:22:30.637 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:30.876 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:22:30.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:30.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:22:30.930 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:22:30.985 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:31.001 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:31.014 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:31.076 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:31.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:31.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:31.181 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:31.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:30 smithi110 ceph-mon[212939]: pgmap v954: 109 pgs: 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 84 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 2162/105477 objects degraded (2.050%); 17111/105477 objects misplaced (16.222%); 32 MiB/s, 507 objects/s recovering 2023-09-23T23:22:31.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:30 smithi110 ceph-mon[212939]: osdmap e316: 8 total, 7 up, 7 in 2023-09-23T23:22:31.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:30 smithi110 ceph-mon[212939]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:30 smithi110 ceph-mon[212939]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:30 smithi110 ceph-mon[212939]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:30 smithi110 ceph-mon[212939]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:30 smithi110 ceph-mon[212939]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.184 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:30 smithi110 ceph-mon[212939]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.185 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:30 smithi110 ceph-mon[221102]: pgmap v954: 109 pgs: 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 84 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 2162/105477 objects degraded (2.050%); 17111/105477 objects misplaced (16.222%); 32 MiB/s, 507 objects/s recovering 2023-09-23T23:22:31.185 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:30 smithi110 ceph-mon[221102]: osdmap e316: 8 total, 7 up, 7 in 2023-09-23T23:22:31.185 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:30 smithi110 ceph-mon[221102]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.185 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:30 smithi110 ceph-mon[221102]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.185 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:30 smithi110 ceph-mon[221102]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.185 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:30 smithi110 ceph-mon[221102]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.185 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:30 smithi110 ceph-mon[221102]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.185 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:30 smithi110 ceph-mon[221102]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.225 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:31.249 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:30 smithi138 ceph-mon[193606]: pgmap v954: 109 pgs: 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 84 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 2162/105477 objects degraded (2.050%); 17111/105477 objects misplaced (16.222%); 32 MiB/s, 507 objects/s recovering 2023-09-23T23:22:31.250 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:30 smithi138 ceph-mon[193606]: osdmap e316: 8 total, 7 up, 7 in 2023-09-23T23:22:31.250 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:30 smithi138 ceph-mon[193606]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.250 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:30 smithi138 ceph-mon[193606]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.250 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:30 smithi138 ceph-mon[193606]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.250 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:30 smithi138 ceph-mon[193606]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.250 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:30 smithi138 ceph-mon[193606]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.250 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:30 smithi138 ceph-mon[193606]: failed to encode map e316 with expected crc 2023-09-23T23:22:31.289 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:31.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:31.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:31.393 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:31.429 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:31.549 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:31.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:31.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:31.656 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:31.700 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:31.760 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:31.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:31.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:31.865 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:22:31.902 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:31.955 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:22:31.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:31.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:22:32.014 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:22:32.059 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:32.098 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:32.153 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 2] out_osds: [1] dead_osds: [7] live_osds: [0, 1, 3, 4, 5, 6, 2] 2023-09-23T23:22:32.153 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:22:32.153 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-09-23T23:22:32.153 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd in 1 2023-09-23T23:22:32.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:32 smithi110 ceph-mon[212939]: failed to encode map e317 with expected crc 2023-09-23T23:22:32.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:32 smithi110 ceph-mon[212939]: osdmap e317: 8 total, 7 up, 7 in 2023-09-23T23:22:32.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:32 smithi110 ceph-mon[212939]: failed to encode map e317 with expected crc 2023-09-23T23:22:32.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:32 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3956956860' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:22:32.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:32 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:22:32.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:32 smithi110 ceph-mon[221102]: failed to encode map e317 with expected crc 2023-09-23T23:22:32.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:32 smithi110 ceph-mon[221102]: osdmap e317: 8 total, 7 up, 7 in 2023-09-23T23:22:32.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:32 smithi110 ceph-mon[221102]: failed to encode map e317 with expected crc 2023-09-23T23:22:32.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:32 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3956956860' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:22:32.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:32 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:22:32.268 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:32.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:32.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:32.274 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:32 smithi138 ceph-mon[193606]: failed to encode map e317 with expected crc 2023-09-23T23:22:32.274 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:32 smithi138 ceph-mon[193606]: osdmap e317: 8 total, 7 up, 7 in 2023-09-23T23:22:32.274 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:32 smithi138 ceph-mon[193606]: failed to encode map e317 with expected crc 2023-09-23T23:22:32.274 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:32 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3956956860' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:22:32.274 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:32 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:22:32.374 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:32.445 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:32.815 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:22:32.815 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:22:32.815 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:22:32.816 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:22:32.816 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.816 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:177977 9.e 9:7f4c5a4f:::benchmark_data_smithi110_295904_object66796:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-09-23T23:22:32.816 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:10.280640+0000", 2023-09-23T23:22:32.816 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 322.52027182799998, 2023-09-23T23:22:32.816 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.35592362900000002, 2023-09-23T23:22:32.816 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.816 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.816 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:22:32.816 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:22:32.816 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:22:32.816 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 177977 2023-09-23T23:22:32.817 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.817 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.817 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.817 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.817 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.280640+0000", 2023-09-23T23:22:32.817 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.817 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.817 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.817 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.817 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.280640+0000", 2023-09-23T23:22:32.817 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.817 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.818 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.818 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.818 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.280639+0000", 2023-09-23T23:22:32.818 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:22:32.818 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.818 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.818 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.818 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.280644+0000", 2023-09-23T23:22:32.818 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7199999999999997e-06 2023-09-23T23:22:32.818 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.818 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.818 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.819 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.280655+0000", 2023-09-23T23:22:32.819 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1131000000000001e-05 2023-09-23T23:22:32.819 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.819 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.819 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.819 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.280661+0000", 2023-09-23T23:22:32.819 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3219999999999998e-06 2023-09-23T23:22:32.819 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.819 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.819 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.819 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.281007+0000", 2023-09-23T23:22:32.820 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0003456 2023-09-23T23:22:32.820 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.820 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.820 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.820 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.281171+0000", 2023-09-23T23:22:32.820 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016447200000000001 2023-09-23T23:22:32.820 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.820 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.820 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T23:22:32.820 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.281319+0000", 2023-09-23T23:22:32.820 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014733300000000001 2023-09-23T23:22:32.820 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.821 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.821 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:22:32.821 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.284890+0000", 2023-09-23T23:22:32.821 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003571603 2023-09-23T23:22:32.821 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.821 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.821 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.821 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.287513+0000", 2023-09-23T23:22:32.821 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002622732 2023-09-23T23:22:32.821 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.821 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.822 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.822 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.291419+0000", 2023-09-23T23:22:32.822 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003906456 2023-09-23T23:22:32.822 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.822 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.822 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.822 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.636405+0000", 2023-09-23T23:22:32.822 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.34498575100000001 2023-09-23T23:22:32.822 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.822 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.822 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.822 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.636462+0000", 2023-09-23T23:22:32.823 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6975000000000003e-05 2023-09-23T23:22:32.823 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.823 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.823 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:10.636564+0000", 2023-09-23T23:22:32.823 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101568 2023-09-23T23:22:32.823 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.823 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.823 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.823 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.823 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:188568 9.e 9:71d3eae7:::benchmark_data_smithi110_295904_object77387:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:22:32.824 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:33.524654+0000", 2023-09-23T23:22:32.824 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 299.27625754000002, 2023-09-23T23:22:32.824 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.89632690699999995, 2023-09-23T23:22:32.824 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.824 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.824 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:22:32.824 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:22:32.824 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:22:32.824 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 188568 2023-09-23T23:22:32.824 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.824 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.824 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.825 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.825 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.524654+0000", 2023-09-23T23:22:32.825 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.825 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.825 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.825 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.825 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.524654+0000", 2023-09-23T23:22:32.825 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.825 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.825 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.825 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.825 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.524653+0000", 2023-09-23T23:22:32.825 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:22:32.826 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.826 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.826 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.826 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.524660+0000", 2023-09-23T23:22:32.826 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.861e-06 2023-09-23T23:22:32.826 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.826 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.826 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.826 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.524676+0000", 2023-09-23T23:22:32.826 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6052000000000001e-05 2023-09-23T23:22:32.826 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.826 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.827 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.827 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.524683+0000", 2023-09-23T23:22:32.827 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0940000000000002e-06 2023-09-23T23:22:32.827 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.827 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.827 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.827 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.524759+0000", 2023-09-23T23:22:32.827 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6048000000000001e-05 2023-09-23T23:22:32.827 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.827 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.827 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:22:32.827 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.524774+0000", 2023-09-23T23:22:32.828 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4982e-05 2023-09-23T23:22:32.828 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.828 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.828 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.828 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.415935+0000", 2023-09-23T23:22:32.828 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.89116085199999995 2023-09-23T23:22:32.828 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.828 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.828 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.828 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.416036+0000", 2023-09-23T23:22:32.828 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101493 2023-09-23T23:22:32.828 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.829 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.829 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T23:22:32.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.416475+0000", 2023-09-23T23:22:32.829 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000439018 2023-09-23T23:22:32.829 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.829 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.829 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.420801+0000", 2023-09-23T23:22:32.829 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043260030000000001 2023-09-23T23:22:32.829 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.829 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.829 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.830 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.420845+0000", 2023-09-23T23:22:32.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3825000000000001e-05 2023-09-23T23:22:32.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.830 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.830 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:22:32.830 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.420853+0000", 2023-09-23T23:22:32.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1450000000000008e-06 2023-09-23T23:22:32.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.830 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.830 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.830 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.420880+0000", 2023-09-23T23:22:32.831 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6478999999999998e-05 2023-09-23T23:22:32.831 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.831 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.831 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.831 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.420916+0000", 2023-09-23T23:22:32.831 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5964000000000003e-05 2023-09-23T23:22:32.831 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.831 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.831 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.831 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.420981+0000", 2023-09-23T23:22:32.831 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5512000000000006e-05 2023-09-23T23:22:32.831 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.832 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.832 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.832 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.832 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.832 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:188601 9.e 9:7a546467:::benchmark_data_smithi110_295904_object77420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:22:32.832 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:33.597788+0000", 2023-09-23T23:22:32.832 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 299.20312358000001, 2023-09-23T23:22:32.832 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.83494150099999997, 2023-09-23T23:22:32.832 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.832 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.832 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:22:32.832 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:22:32.833 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:22:32.833 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 188601 2023-09-23T23:22:32.833 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.833 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.833 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.833 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.833 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.597788+0000", 2023-09-23T23:22:32.833 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.833 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.833 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.833 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.833 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.597788+0000", 2023-09-23T23:22:32.833 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.834 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.834 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.834 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.834 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.597786+0000", 2023-09-23T23:22:32.834 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:22:32.834 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.834 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.834 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.834 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.597796+0000", 2023-09-23T23:22:32.834 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9499999999999996e-06 2023-09-23T23:22:32.834 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.835 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.835 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.835 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.597819+0000", 2023-09-23T23:22:32.835 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2379e-05 2023-09-23T23:22:32.835 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.835 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.835 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.835 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.597827+0000", 2023-09-23T23:22:32.835 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6570000000000003e-06 2023-09-23T23:22:32.835 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.835 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.835 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.836 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.597905+0000", 2023-09-23T23:22:32.836 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7583000000000006e-05 2023-09-23T23:22:32.836 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.836 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.836 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:22:32.836 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.597920+0000", 2023-09-23T23:22:32.836 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.521e-05 2023-09-23T23:22:32.836 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.836 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.836 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.836 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.421138+0000", 2023-09-23T23:22:32.836 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.82321820300000004 2023-09-23T23:22:32.837 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.837 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.837 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.837 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.421237+0000", 2023-09-23T23:22:32.837 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8747999999999997e-05 2023-09-23T23:22:32.837 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.837 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.837 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T23:22:32.837 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.421311+0000", 2023-09-23T23:22:32.837 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3942000000000005e-05 2023-09-23T23:22:32.837 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.837 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.838 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:22:32.838 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.425900+0000", 2023-09-23T23:22:32.838 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004589063 2023-09-23T23:22:32.838 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.838 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.838 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.838 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.425956+0000", 2023-09-23T23:22:32.838 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6047000000000001e-05 2023-09-23T23:22:32.838 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.838 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.838 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.838 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.425991+0000", 2023-09-23T23:22:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4814000000000002e-05 2023-09-23T23:22:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.432596+0000", 2023-09-23T23:22:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066052410000000004 2023-09-23T23:22:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.432632+0000", 2023-09-23T23:22:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.54e-05 2023-09-23T23:22:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.432730+0000", 2023-09-23T23:22:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8204999999999999e-05 2023-09-23T23:22:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:188618 9.e 9:7259fd00:::benchmark_data_smithi110_295904_object77437:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:22:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:33.650613+0000", 2023-09-23T23:22:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 299.150298609, 2023-09-23T23:22:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.782310116, 2023-09-23T23:22:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:22:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:22:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:22:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 188618 2023-09-23T23:22:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.650613+0000", 2023-09-23T23:22:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.650613+0000", 2023-09-23T23:22:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.650612+0000", 2023-09-23T23:22:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:22:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.650618+0000", 2023-09-23T23:22:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0769999999999996e-06 2023-09-23T23:22:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.650646+0000", 2023-09-23T23:22:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7568000000000001e-05 2023-09-23T23:22:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.650653+0000", 2023-09-23T23:22:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.109e-06 2023-09-23T23:22:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.650716+0000", 2023-09-23T23:22:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3427000000000001e-05 2023-09-23T23:22:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:22:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.650739+0000", 2023-09-23T23:22:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.279e-05 2023-09-23T23:22:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.423559+0000", 2023-09-23T23:22:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.77282010300000004 2023-09-23T23:22:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.423668+0000", 2023-09-23T23:22:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108614 2023-09-23T23:22:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T23:22:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.423751+0000", 2023-09-23T23:22:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3353e-05 2023-09-23T23:22:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.426022+0000", 2023-09-23T23:22:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022710930000000001 2023-09-23T23:22:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.426056+0000", 2023-09-23T23:22:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4044999999999997e-05 2023-09-23T23:22:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:22:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.426068+0000", 2023-09-23T23:22:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1486e-05 2023-09-23T23:22:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.432784+0000", 2023-09-23T23:22:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0067167909999999997 2023-09-23T23:22:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.432832+0000", 2023-09-23T23:22:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7457999999999999e-05 2023-09-23T23:22:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:34.432923+0000", 2023-09-23T23:22:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1658999999999998e-05 2023-09-23T23:22:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:189866 9.e 9:7a5298a8:::benchmark_data_smithi110_295904_object78685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T23:22:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.873989+0000", 2023-09-23T23:22:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 294.92692258699998, 2023-09-23T23:22:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.40595450399999999, 2023-09-23T23:22:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:22:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:22:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:22:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 189866 2023-09-23T23:22:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.873989+0000", 2023-09-23T23:22:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.873989+0000", 2023-09-23T23:22:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.873988+0000", 2023-09-23T23:22:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:22:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.873993+0000", 2023-09-23T23:22:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6179999999999997e-06 2023-09-23T23:22:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.874003+0000", 2023-09-23T23:22:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8949999999999993e-06 2023-09-23T23:22:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.874009+0000", 2023-09-23T23:22:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8309999999999997e-06 2023-09-23T23:22:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.874255+0000", 2023-09-23T23:22:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00024681099999999999 2023-09-23T23:22:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.874371+0000", 2023-09-23T23:22:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011542199999999999 2023-09-23T23:22:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T23:22:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.874494+0000", 2023-09-23T23:22:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012323299999999999 2023-09-23T23:22:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:22:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.878670+0000", 2023-09-23T23:22:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041756409999999999 2023-09-23T23:22:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.883011+0000", 2023-09-23T23:22:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043409989999999999 2023-09-23T23:22:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.888583+0000", 2023-09-23T23:22:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0055726439999999999 2023-09-23T23:22:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279764+0000", 2023-09-23T23:22:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39118028599999999 2023-09-23T23:22:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279809+0000", 2023-09-23T23:22:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5655000000000002e-05 2023-09-23T23:22:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279944+0000", 2023-09-23T23:22:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000134671 2023-09-23T23:22:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:189870 9.e 9:70f28e9e:::benchmark_data_smithi110_295904_object78689:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T23:22:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.879664+0000", 2023-09-23T23:22:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 294.921247972, 2023-09-23T23:22:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.40061539299999999, 2023-09-23T23:22:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:22:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:22:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:22:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 189870 2023-09-23T23:22:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879664+0000", 2023-09-23T23:22:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879664+0000", 2023-09-23T23:22:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879662+0000", 2023-09-23T23:22:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:22:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879668+0000", 2023-09-23T23:22:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1530000000000002e-06 2023-09-23T23:22:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879680+0000", 2023-09-23T23:22:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1406000000000001e-05 2023-09-23T23:22:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879686+0000", 2023-09-23T23:22:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8600000000000004e-06 2023-09-23T23:22:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879733+0000", 2023-09-23T23:22:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6013000000000001e-05 2023-09-23T23:22:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879859+0000", 2023-09-23T23:22:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012643800000000001 2023-09-23T23:22:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T23:22:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.879989+0000", 2023-09-23T23:22:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000130028 2023-09-23T23:22:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:22:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.883117+0000", 2023-09-23T23:22:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003128058 2023-09-23T23:22:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.888632+0000", 2023-09-23T23:22:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0055148460000000003 2023-09-23T23:22:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279989+0000", 2023-09-23T23:22:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39135692999999999 2023-09-23T23:22:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.280159+0000", 2023-09-23T23:22:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017025300000000001 2023-09-23T23:22:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.280205+0000", 2023-09-23T23:22:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6001000000000002e-05 2023-09-23T23:22:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.280279+0000", 2023-09-23T23:22:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4320999999999995e-05 2023-09-23T23:22:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:189877 9.e 9:731f13fe:::benchmark_data_smithi110_295904_object78696:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T23:22:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.884881+0000", 2023-09-23T23:22:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 294.916031197, 2023-09-23T23:22:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39560695299999998, 2023-09-23T23:22:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:22:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:22:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:22:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 189877 2023-09-23T23:22:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.884881+0000", 2023-09-23T23:22:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.884881+0000", 2023-09-23T23:22:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.884879+0000", 2023-09-23T23:22:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:22:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.884886+0000", 2023-09-23T23:22:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2740000000000002e-06 2023-09-23T23:22:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.884901+0000", 2023-09-23T23:22:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4231e-05 2023-09-23T23:22:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.884908+0000", 2023-09-23T23:22:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.81e-06 2023-09-23T23:22:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.884957+0000", 2023-09-23T23:22:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9024999999999999e-05 2023-09-23T23:22:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.885086+0000", 2023-09-23T23:22:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012970700000000001 2023-09-23T23:22:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T23:22:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.885200+0000", 2023-09-23T23:22:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000113587 2023-09-23T23:22:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.888654+0000", 2023-09-23T23:22:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034543899999999999 2023-09-23T23:22:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:22:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.888660+0000", 2023-09-23T23:22:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1700000000000002e-06 2023-09-23T23:22:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.280023+0000", 2023-09-23T23:22:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39136293700000002 2023-09-23T23:22:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.280332+0000", 2023-09-23T23:22:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00030889200000000001 2023-09-23T23:22:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.280374+0000", 2023-09-23T23:22:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2070000000000002e-05 2023-09-23T23:22:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.280488+0000", 2023-09-23T23:22:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000113399 2023-09-23T23:22:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45164.0:189880 9.1 e230/218 9:87969838:::benchmark_data_smithi110_295904_object78699:head v 230'11922, mlcod=230'11920)", 2023-09-23T23:22:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.895576+0000", 2023-09-23T23:22:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 294.90533553300003, 2023-09-23T23:22:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.384357911, 2023-09-23T23:22:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.895576+0000", 2023-09-23T23:22:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.895576+0000", 2023-09-23T23:22:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.895576+0000", 2023-09-23T23:22:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:22:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.895583+0000", 2023-09-23T23:22:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9589999999999998e-06 2023-09-23T23:22:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.895595+0000", 2023-09-23T23:22:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2418e-05 2023-09-23T23:22:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.895601+0000", 2023-09-23T23:22:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1419999999999998e-06 2023-09-23T23:22:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.895689+0000", 2023-09-23T23:22:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7548999999999998e-05 2023-09-23T23:22:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.895709+0000", 2023-09-23T23:22:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9868e-05 2023-09-23T23:22:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279904+0000", 2023-09-23T23:22:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.384195071 2023-09-23T23:22:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279934+0000", 2023-09-23T23:22:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0558000000000001e-05 2023-09-23T23:22:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45164.0:189879 9.c e230/164 9:3a3914a2:::benchmark_data_smithi110_295904_object78698:head v 230'11867, mlcod=230'11864)", 2023-09-23T23:22:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.896307+0000", 2023-09-23T23:22:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 294.90460514799997, 2023-09-23T23:22:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.383339608, 2023-09-23T23:22:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.896307+0000", 2023-09-23T23:22:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.896307+0000", 2023-09-23T23:22:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.896306+0000", 2023-09-23T23:22:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:22:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.896313+0000", 2023-09-23T23:22:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9619999999999996e-06 2023-09-23T23:22:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.896323+0000", 2023-09-23T23:22:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4490000000000005e-06 2023-09-23T23:22:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.896328+0000", 2023-09-23T23:22:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3290000000000001e-06 2023-09-23T23:22:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.896373+0000", 2023-09-23T23:22:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4653000000000001e-05 2023-09-23T23:22:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.896386+0000", 2023-09-23T23:22:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3088000000000001e-05 2023-09-23T23:22:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279611+0000", 2023-09-23T23:22:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38322504400000001 2023-09-23T23:22:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279646+0000", 2023-09-23T23:22:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5778000000000002e-05 2023-09-23T23:22:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.45164.0:189866 9.e e230/227 ondisk, result = 0)", 2023-09-23T23:22:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.899217+0000", 2023-09-23T23:22:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 294.90169457500002, 2023-09-23T23:22:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38073507499999998, 2023-09-23T23:22:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:22:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.899217+0000", 2023-09-23T23:22:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.899217+0000", 2023-09-23T23:22:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.899217+0000", 2023-09-23T23:22:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:22:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.899225+0000", 2023-09-23T23:22:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9610000000000005e-06 2023-09-23T23:22:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279591+0000", 2023-09-23T23:22:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38036651300000002 2023-09-23T23:22:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279604+0000", 2023-09-23T23:22:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3256e-05 2023-09-23T23:22:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279754+0000", 2023-09-23T23:22:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014917399999999999 2023-09-23T23:22:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279759+0000", 2023-09-23T23:22:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5809999999999996e-06 2023-09-23T23:22:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279952+0000", 2023-09-23T23:22:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019326900000000001 2023-09-23T23:22:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45164.0:189883 9.c e230/164 9:3785bba4:::benchmark_data_smithi110_295904_object78702:head v 230'11868, mlcod=230'11865)", 2023-09-23T23:22:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:37.900642+0000", 2023-09-23T23:22:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 294.90027016200003, 2023-09-23T23:22:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.388835292, 2023-09-23T23:22:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.900642+0000", 2023-09-23T23:22:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.900642+0000", 2023-09-23T23:22:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.900641+0000", 2023-09-23T23:22:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:22:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:37.900648+0000", 2023-09-23T23:22:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4569999999999999e-06 2023-09-23T23:22:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279592+0000", 2023-09-23T23:22:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.37894372799999998 2023-09-23T23:22:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279604+0000", 2023-09-23T23:22:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1575e-05 2023-09-23T23:22:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279672+0000", 2023-09-23T23:22:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8100000000000002e-05 2023-09-23T23:22:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.279693+0000", 2023-09-23T23:22:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0786000000000001e-05 2023-09-23T23:22:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.289430+0000", 2023-09-23T23:22:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0097376070000000006 2023-09-23T23:22:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:38.289477+0000", 2023-09-23T23:22:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6825e-05 2023-09-23T23:22:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:207686 9.e 9:7c3a69f3:::benchmark_data_smithi110_295904_object96505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-23T23:22:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:18:15.313700+0000", 2023-09-23T23:22:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 257.48721189999998, 2023-09-23T23:22:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.29027254800000002, 2023-09-23T23:22:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:22:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:22:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:22:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 207686 2023-09-23T23:22:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313700+0000", 2023-09-23T23:22:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313700+0000", 2023-09-23T23:22:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313698+0000", 2023-09-23T23:22:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-09-23T23:22:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313712+0000", 2023-09-23T23:22:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.393e-05 2023-09-23T23:22:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313746+0000", 2023-09-23T23:22:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4085000000000003e-05 2023-09-23T23:22:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313760+0000", 2023-09-23T23:22:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.435e-05 2023-09-23T23:22:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313818+0000", 2023-09-23T23:22:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8167000000000001e-05 2023-09-23T23:22:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313913+0000", 2023-09-23T23:22:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5057999999999997e-05 2023-09-23T23:22:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:22:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.313987+0000", 2023-09-23T23:22:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4214000000000005e-05 2023-09-23T23:22:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:22:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.316356+0000", 2023-09-23T23:22:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002368462 2023-09-23T23:22:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.316611+0000", 2023-09-23T23:22:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025519300000000002 2023-09-23T23:22:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.603884+0000", 2023-09-23T23:22:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.287272841 2023-09-23T23:22:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.603934+0000", 2023-09-23T23:22:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9842999999999997e-05 2023-09-23T23:22:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.603973+0000", 2023-09-23T23:22:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8831000000000001e-05 2023-09-23T23:22:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:70902 10.2 10:4adf2976:::benchmark_data_smithi110_366450_object70901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T23:22:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:59.013259+0000", 2023-09-23T23:22:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 153.78765253899999, 2023-09-23T23:22:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39337688999999998, 2023-09-23T23:22:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:22:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:22:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:22:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 70902 2023-09-23T23:22:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013259+0000", 2023-09-23T23:22:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013259+0000", 2023-09-23T23:22:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013258+0000", 2023-09-23T23:22:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:22:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013323+0000", 2023-09-23T23:22:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4944999999999996e-05 2023-09-23T23:22:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013334+0000", 2023-09-23T23:22:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0142999999999999e-05 2023-09-23T23:22:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013341+0000", 2023-09-23T23:22:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0979999999999996e-06 2023-09-23T23:22:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013383+0000", 2023-09-23T23:22:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2188999999999999e-05 2023-09-23T23:22:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013498+0000", 2023-09-23T23:22:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011552099999999999 2023-09-23T23:22:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-09-23T23:22:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.013642+0000", 2023-09-23T23:22:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014336500000000001 2023-09-23T23:22:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:22:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.017890+0000", 2023-09-23T23:22:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0042483140000000004 2023-09-23T23:22:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.020865+0000", 2023-09-23T23:22:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029748169999999998 2023-09-23T23:22:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.406538+0000", 2023-09-23T23:22:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38567305200000002 2023-09-23T23:22:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.406592+0000", 2023-09-23T23:22:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4456999999999997e-05 2023-09-23T23:22:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.406636+0000", 2023-09-23T23:22:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3968999999999999e-05 2023-09-23T23:22:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:70912 10.e 10:709d2b27:::benchmark_data_smithi110_366450_object70911:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T23:22:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:59.024249+0000", 2023-09-23T23:22:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 153.77666329499999, 2023-09-23T23:22:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39032808699999999, 2023-09-23T23:22:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:22:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:22:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:22:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 70912 2023-09-23T23:22:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024249+0000", 2023-09-23T23:22:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024249+0000", 2023-09-23T23:22:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024247+0000", 2023-09-23T23:22:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:22:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024326+0000", 2023-09-23T23:22:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8335000000000001e-05 2023-09-23T23:22:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024337+0000", 2023-09-23T23:22:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1573000000000001e-05 2023-09-23T23:22:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024343+0000", 2023-09-23T23:22:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.088e-06 2023-09-23T23:22:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024379+0000", 2023-09-23T23:22:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5998999999999999e-05 2023-09-23T23:22:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024488+0000", 2023-09-23T23:22:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109059 2023-09-23T23:22:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:22:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.024622+0000", 2023-09-23T23:22:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013397800000000001 2023-09-23T23:22:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:22:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.029840+0000", 2023-09-23T23:22:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0052171960000000003 2023-09-23T23:22:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.030243+0000", 2023-09-23T23:22:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00040324000000000002 2023-09-23T23:22:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.414483+0000", 2023-09-23T23:22:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38424069100000002 2023-09-23T23:22:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.414536+0000", 2023-09-23T23:22:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2833000000000001e-05 2023-09-23T23:22:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.414577+0000", 2023-09-23T23:22:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0436999999999999e-05 2023-09-23T23:22:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:70917 10.2 10:4e149dab:::benchmark_data_smithi110_366450_object70916:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T23:22:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:59.027511+0000", 2023-09-23T23:22:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 153.77340080499999, 2023-09-23T23:22:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39079427300000003, 2023-09-23T23:22:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:22:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:22:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:22:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 70917 2023-09-23T23:22:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027511+0000", 2023-09-23T23:22:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027511+0000", 2023-09-23T23:22:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027510+0000", 2023-09-23T23:22:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:22:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027577+0000", 2023-09-23T23:22:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6310999999999996e-05 2023-09-23T23:22:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027589+0000", 2023-09-23T23:22:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2481e-05 2023-09-23T23:22:32.916 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027595+0000", 2023-09-23T23:22:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1940000000000003e-06 2023-09-23T23:22:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027650+0000", 2023-09-23T23:22:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4588e-05 2023-09-23T23:22:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027761+0000", 2023-09-23T23:22:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110839 2023-09-23T23:22:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-09-23T23:22:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.027897+0000", 2023-09-23T23:22:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000136737 2023-09-23T23:22:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.030294+0000", 2023-09-23T23:22:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023962549999999999 2023-09-23T23:22:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:22:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.032258+0000", 2023-09-23T23:22:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0019644300000000001 2023-09-23T23:22:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.418216+0000", 2023-09-23T23:22:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38595784999999999 2023-09-23T23:22:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.418267+0000", 2023-09-23T23:22:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0973999999999997e-05 2023-09-23T23:22:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:59.418305+0000", 2023-09-23T23:22:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8534000000000001e-05 2023-09-23T23:22:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:109531 10.2 10:4b91e526:::benchmark_data_smithi110_366450_object8323:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e292)", 2023-09-23T23:22:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:50.765997+0000", 2023-09-23T23:22:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 102.034915302, 2023-09-23T23:22:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23547553700000001, 2023-09-23T23:22:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:22:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:22:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:22:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109531 2023-09-23T23:22:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.765997+0000", 2023-09-23T23:22:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.765997+0000", 2023-09-23T23:22:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.765996+0000", 2023-09-23T23:22:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:22:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.766001+0000", 2023-09-23T23:22:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.87e-06 2023-09-23T23:22:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.766014+0000", 2023-09-23T23:22:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.314e-05 2023-09-23T23:22:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.766020+0000", 2023-09-23T23:22:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3779999999999998e-06 2023-09-23T23:22:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.766073+0000", 2023-09-23T23:22:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2728000000000001e-05 2023-09-23T23:22:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.766199+0000", 2023-09-23T23:22:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0001261 2023-09-23T23:22:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-09-23T23:22:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.766313+0000", 2023-09-23T23:22:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011400699999999999 2023-09-23T23:22:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:22:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.770916+0000", 2023-09-23T23:22:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0046032349999999998 2023-09-23T23:22:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.771454+0000", 2023-09-23T23:22:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00053829099999999999 2023-09-23T23:22:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:51.001382+0000", 2023-09-23T23:22:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22992722800000001 2023-09-23T23:22:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:51.001432+0000", 2023-09-23T23:22:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0503000000000001e-05 2023-09-23T23:22:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:51.001472+0000", 2023-09-23T23:22:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9978999999999998e-05 2023-09-23T23:22:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:137717 10.c 10:341caf6d:::benchmark_data_smithi110_366450_object36509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2023-09-23T23:22:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:26.017015+0000", 2023-09-23T23:22:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 66.783896470000002, 2023-09-23T23:22:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25778899599999999, 2023-09-23T23:22:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:22:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:22:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:22:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 137717 2023-09-23T23:22:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.017015+0000", 2023-09-23T23:22:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.017015+0000", 2023-09-23T23:22:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.017014+0000", 2023-09-23T23:22:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:22:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.017020+0000", 2023-09-23T23:22:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6699999999999999e-06 2023-09-23T23:22:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.017035+0000", 2023-09-23T23:22:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4619e-05 2023-09-23T23:22:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.017041+0000", 2023-09-23T23:22:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0599999999999996e-06 2023-09-23T23:22:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.020545+0000", 2023-09-23T23:22:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0035044870000000001 2023-09-23T23:22:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.020663+0000", 2023-09-23T23:22:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117977 2023-09-23T23:22:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0", 2023-09-23T23:22:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.020768+0000", 2023-09-23T23:22:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105025 2023-09-23T23:22:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:22:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.023087+0000", 2023-09-23T23:22:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023181870000000002 2023-09-23T23:22:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.274628+0000", 2023-09-23T23:22:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.251541185 2023-09-23T23:22:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.274683+0000", 2023-09-23T23:22:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4997999999999998e-05 2023-09-23T23:22:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.274804+0000", 2023-09-23T23:22:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000121777 2023-09-23T23:22:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:137719 10.1 10:88c0f33b:::benchmark_data_smithi110_366450_object36511:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2023-09-23T23:22:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:26.022464+0000", 2023-09-23T23:22:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 66.778448410999999, 2023-09-23T23:22:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25234093899999999, 2023-09-23T23:22:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:22:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:22:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:22:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 137719 2023-09-23T23:22:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.022464+0000", 2023-09-23T23:22:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.022464+0000", 2023-09-23T23:22:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.022462+0000", 2023-09-23T23:22:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:22:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.022470+0000", 2023-09-23T23:22:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3129999999999999e-06 2023-09-23T23:22:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.022484+0000", 2023-09-23T23:22:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3603e-05 2023-09-23T23:22:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.022491+0000", 2023-09-23T23:22:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4789999999999999e-06 2023-09-23T23:22:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.022976+0000", 2023-09-23T23:22:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000485222 2023-09-23T23:22:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.023090+0000", 2023-09-23T23:22:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011338200000000001 2023-09-23T23:22:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0", 2023-09-23T23:22:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.023206+0000", 2023-09-23T23:22:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0001161 2023-09-23T23:22:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:22:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.027383+0000", 2023-09-23T23:22:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041771869999999997 2023-09-23T23:22:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.274624+0000", 2023-09-23T23:22:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24724068199999999 2023-09-23T23:22:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.274685+0000", 2023-09-23T23:22:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1636000000000004e-05 2023-09-23T23:22:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.274804+0000", 2023-09-23T23:22:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011914299999999999 2023-09-23T23:22:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:137725 10.2 10:41dffa09:::benchmark_data_smithi110_366450_object36517:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2023-09-23T23:22:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:26.031618+0000", 2023-09-23T23:22:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 66.769293972, 2023-09-23T23:22:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.252242986, 2023-09-23T23:22:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:22:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:22:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:22:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 137725 2023-09-23T23:22:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031618+0000", 2023-09-23T23:22:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031618+0000", 2023-09-23T23:22:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031617+0000", 2023-09-23T23:22:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:22:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031622+0000", 2023-09-23T23:22:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3909999999999997e-06 2023-09-23T23:22:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031634+0000", 2023-09-23T23:22:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1506e-05 2023-09-23T23:22:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031656+0000", 2023-09-23T23:22:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1866e-05 2023-09-23T23:22:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031696+0000", 2023-09-23T23:22:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9922000000000002e-05 2023-09-23T23:22:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031794+0000", 2023-09-23T23:22:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8245000000000006e-05 2023-09-23T23:22:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-09-23T23:22:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031870+0000", 2023-09-23T23:22:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6217000000000006e-05 2023-09-23T23:22:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:22:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.034584+0000", 2023-09-23T23:22:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0027140340000000002 2023-09-23T23:22:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.034642+0000", 2023-09-23T23:22:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7911999999999997e-05 2023-09-23T23:22:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.283768+0000", 2023-09-23T23:22:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.249126075 2023-09-23T23:22:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.283819+0000", 2023-09-23T23:22:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1279000000000002e-05 2023-09-23T23:22:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.283861+0000", 2023-09-23T23:22:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1641000000000003e-05 2023-09-23T23:22:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:137728 10.1 10:8f45f659:::benchmark_data_smithi110_366450_object36520:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2023-09-23T23:22:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:26.031665+0000", 2023-09-23T23:22:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 66.769246480000007, 2023-09-23T23:22:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24922878300000001, 2023-09-23T23:22:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:22:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:22:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:22:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:22:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 137728 2023-09-23T23:22:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031665+0000", 2023-09-23T23:22:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031665+0000", 2023-09-23T23:22:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031665+0000", 2023-09-23T23:22:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:22:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031668+0000", 2023-09-23T23:22:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9380000000000001e-06 2023-09-23T23:22:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031674+0000", 2023-09-23T23:22:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1530000000000002e-06 2023-09-23T23:22:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031678+0000", 2023-09-23T23:22:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9779999999999999e-06 2023-09-23T23:22:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031719+0000", 2023-09-23T23:22:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0478000000000001e-05 2023-09-23T23:22:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031795+0000", 2023-09-23T23:22:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5981000000000003e-05 2023-09-23T23:22:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0", 2023-09-23T23:22:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.031870+0000", 2023-09-23T23:22:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5300000000000001e-05 2023-09-23T23:22:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:22:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.034225+0000", 2023-09-23T23:22:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023554050000000001 2023-09-23T23:22:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:22:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.280757+0000", 2023-09-23T23:22:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24653182200000001 2023-09-23T23:22:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:22:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.280809+0000", 2023-09-23T23:22:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1566000000000001e-05 2023-09-23T23:22:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:22:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:26.280894+0000", 2023-09-23T23:22:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5604000000000002e-05 2023-09-23T23:22:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.956 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.956 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.956 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:32.956 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:32.956 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:22:33.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:33 smithi138 ceph-mon[193606]: pgmap v957: 109 pgs: 2 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 84 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 246 op/s; 2162/103494 objects degraded (2.089%); 16443/103494 objects misplaced (15.888%); 21 MiB/s, 335 objects/s recovering 2023-09-23T23:22:33.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:33 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:22:33.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:33 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:22:33.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:33 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:22:33.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:33 smithi138 ceph-mon[193606]: osdmap e318: 8 total, 7 up, 7 in 2023-09-23T23:22:33.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:33 smithi138 ceph-mon[193606]: failed to encode map e318 with expected crc 2023-09-23T23:22:33.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:33 smithi110 ceph-mon[221102]: pgmap v957: 109 pgs: 2 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 84 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 246 op/s; 2162/103494 objects degraded (2.089%); 16443/103494 objects misplaced (15.888%); 21 MiB/s, 335 objects/s recovering 2023-09-23T23:22:33.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:33 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:22:33.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:33 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:22:33.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:33 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:22:33.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:33 smithi110 ceph-mon[221102]: osdmap e318: 8 total, 7 up, 7 in 2023-09-23T23:22:33.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:33 smithi110 ceph-mon[221102]: failed to encode map e318 with expected crc 2023-09-23T23:22:33.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:33 smithi110 ceph-mon[212939]: pgmap v957: 109 pgs: 2 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 84 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 246 op/s; 2162/103494 objects degraded (2.089%); 16443/103494 objects misplaced (15.888%); 21 MiB/s, 335 objects/s recovering 2023-09-23T23:22:33.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:33 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:22:33.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:33 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:22:33.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:33 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:22:33.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:33 smithi110 ceph-mon[212939]: osdmap e318: 8 total, 7 up, 7 in 2023-09-23T23:22:33.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:33 smithi110 ceph-mon[212939]: failed to encode map e318 with expected crc 2023-09-23T23:22:33.323 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:33.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:33.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:33.428 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:33.462 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:33.995 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:34.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:34.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:34.018 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:22:34.102 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:34.156 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:34 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:22:34.156 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:34 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2162/103494 objects degraded (2.089%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:34.156 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:34 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:22:34.156 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:34 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2162/103494 objects degraded (2.089%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:34.160 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:34.179 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:34.245 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:34.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:34.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:34.311 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:22:34.341 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:22:34.350 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:34.375 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:22:34.385 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:34 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:22:34.386 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:34 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2162/103494 objects degraded (2.089%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:34.388 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:34.407 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:22:34.439 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:22:34.447 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:34.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:34.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:34.467 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:22:34.494 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:22:34.499 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:22:34.500 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:22:34.500 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:22:34.500 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:22:34.500 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:22:34.500 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:22:34.500 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:22:34.500 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:22:34.551 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:22:34.590 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:34.626 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:22:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:35 smithi138 ceph-mon[193606]: pgmap v959: 109 pgs: 2 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 84 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 250 op/s; 2162/103494 objects degraded (2.089%); 16443/103494 objects misplaced (15.888%); 21 MiB/s, 341 objects/s recovering 2023-09-23T23:22:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:35 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3609805277' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:35 smithi138 ceph-mon[193606]: 1.7 scrub starts 2023-09-23T23:22:35.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:35 smithi110 ceph-mon[212939]: pgmap v959: 109 pgs: 2 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 84 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 250 op/s; 2162/103494 objects degraded (2.089%); 16443/103494 objects misplaced (15.888%); 21 MiB/s, 341 objects/s recovering 2023-09-23T23:22:35.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:35 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3609805277' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:35.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:35 smithi110 ceph-mon[212939]: 1.7 scrub starts 2023-09-23T23:22:35.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:35 smithi110 ceph-mon[221102]: pgmap v959: 109 pgs: 2 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 84 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 250 op/s; 2162/103494 objects degraded (2.089%); 16443/103494 objects misplaced (15.888%); 21 MiB/s, 341 objects/s recovering 2023-09-23T23:22:35.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:35 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3609805277' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:35.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:35 smithi110 ceph-mon[221102]: 1.7 scrub starts 2023-09-23T23:22:35.662 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:22:36.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:36 smithi138 ceph-mon[193606]: 1.4 scrub starts 2023-09-23T23:22:36.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:36 smithi110 ceph-mon[212939]: 1.4 scrub starts 2023-09-23T23:22:36.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:36 smithi110 ceph-mon[221102]: 1.4 scrub starts 2023-09-23T23:22:36.654 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:22:36.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:36.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:22:36.762 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:36.783 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:22:36.783 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:22:36.795 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:36.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:22:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:22:36] "GET /metrics HTTP/1.1" 200 36634 "" "Prometheus/2.43.0" 2023-09-23T23:22:36.992 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:36.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:36.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:37.013 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:22:37.013 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:22:37.087 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:37 smithi138 ceph-mon[193606]: pgmap v960: 109 pgs: 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 85 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 404 op/s; 2057/98187 objects degraded (2.095%); 13480/98187 objects misplaced (13.729%); 25 MiB/s, 402 objects/s recovering 2023-09-23T23:22:37.088 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:37 smithi138 ceph-mon[193606]: 1.3 scrub starts 2023-09-23T23:22:37.097 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:37.143 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:37.164 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:37.225 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:37.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:37.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:37.230 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:37 smithi110 ceph-mon[221102]: pgmap v960: 109 pgs: 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 85 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 404 op/s; 2057/98187 objects degraded (2.095%); 13480/98187 objects misplaced (13.729%); 25 MiB/s, 402 objects/s recovering 2023-09-23T23:22:37.230 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:37 smithi110 ceph-mon[221102]: 1.3 scrub starts 2023-09-23T23:22:37.230 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:37 smithi110 ceph-mon[212939]: pgmap v960: 109 pgs: 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 85 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 404 op/s; 2057/98187 objects degraded (2.095%); 13480/98187 objects misplaced (13.729%); 25 MiB/s, 402 objects/s recovering 2023-09-23T23:22:37.230 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:37 smithi110 ceph-mon[212939]: 1.3 scrub starts 2023-09-23T23:22:37.330 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:37.357 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:37.806 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:37.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:37.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:37.898 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:22:37.910 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:37.954 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:38.053 INFO:teuthology.orchestra.run.smithi110.stderr:marked in osd.1. 2023-09-23T23:22:38.062 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:38.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:38.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:38.167 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:38.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:38 smithi138 ceph-mon[193606]: osdmap e319: 8 total, 7 up, 7 in 2023-09-23T23:22:38.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:38 smithi138 ceph-mon[193606]: failed to encode map e319 with expected crc 2023-09-23T23:22:38.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:38 smithi138 ceph-mon[193606]: failed to encode map e319 with expected crc 2023-09-23T23:22:38.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:38 smithi138 ceph-mon[193606]: failed to encode map e319 with expected crc 2023-09-23T23:22:38.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:38 smithi138 ceph-mon[193606]: failed to encode map e319 with expected crc 2023-09-23T23:22:38.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:38 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3708491072' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T23:22:38.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:37 smithi110 conmon[212914]: 2023-09-23T23:22:37.990+0000 7f29aa464700 -1 mon.a@0(leader).osd e319 definitely_dead 0 2023-09-23T23:22:38.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:38 smithi110 ceph-mon[212939]: osdmap e319: 8 total, 7 up, 7 in 2023-09-23T23:22:38.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:38 smithi110 ceph-mon[212939]: failed to encode map e319 with expected crc 2023-09-23T23:22:38.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:38 smithi110 ceph-mon[212939]: failed to encode map e319 with expected crc 2023-09-23T23:22:38.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:38 smithi110 ceph-mon[212939]: failed to encode map e319 with expected crc 2023-09-23T23:22:38.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:38 smithi110 ceph-mon[212939]: failed to encode map e319 with expected crc 2023-09-23T23:22:38.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:38 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3708491072' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T23:22:38.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:38 smithi110 ceph-mon[221102]: osdmap e319: 8 total, 7 up, 7 in 2023-09-23T23:22:38.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:38 smithi110 ceph-mon[221102]: failed to encode map e319 with expected crc 2023-09-23T23:22:38.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:38 smithi110 ceph-mon[221102]: failed to encode map e319 with expected crc 2023-09-23T23:22:38.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:38 smithi110 ceph-mon[221102]: failed to encode map e319 with expected crc 2023-09-23T23:22:38.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:38 smithi110 ceph-mon[221102]: failed to encode map e319 with expected crc 2023-09-23T23:22:38.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:38 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3708491072' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T23:22:38.205 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:38.467 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:22:39.014 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:22:39.278 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:39.281 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: 1.c scrub starts 2023-09-23T23:22:39.281 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: pgmap v962: 109 pgs: 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 85 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 270 op/s; 2057/98187 objects degraded (2.095%); 13480/98187 objects misplaced (13.729%); 21 MiB/s, 341 objects/s recovering 2023-09-23T23:22:39.281 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.281 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3708491072' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T23:22:39.281 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: osdmap e320: 8 total, 7 up, 8 in 2023-09-23T23:22:39.281 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.281 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.281 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.281 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2057/98187 objects degraded (2.095%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:39.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: failed to encode map e321 with expected crc 2023-09-23T23:22:39.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: osdmap e321: 8 total, 7 up, 8 in 2023-09-23T23:22:39.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: failed to encode map e321 with expected crc 2023-09-23T23:22:39.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: failed to encode map e321 with expected crc 2023-09-23T23:22:39.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: failed to encode map e321 with expected crc 2023-09-23T23:22:39.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: failed to encode map e321 with expected crc 2023-09-23T23:22:39.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: 10.0 starting backfill to osd.1 from (0'0,0'0] MAX to 319'10780 2023-09-23T23:22:39.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: 10.b starting backfill to osd.1 from (0'0,0'0] MAX to 319'10848 2023-09-23T23:22:39.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:39.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:39.283 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: 1.c scrub starts 2023-09-23T23:22:39.283 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: pgmap v962: 109 pgs: 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 85 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 270 op/s; 2057/98187 objects degraded (2.095%); 13480/98187 objects misplaced (13.729%); 21 MiB/s, 341 objects/s recovering 2023-09-23T23:22:39.283 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.283 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3708491072' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T23:22:39.283 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: osdmap e320: 8 total, 7 up, 8 in 2023-09-23T23:22:39.283 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.283 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.283 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.283 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.284 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.284 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.284 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2057/98187 objects degraded (2.095%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:39.284 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: failed to encode map e321 with expected crc 2023-09-23T23:22:39.284 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: osdmap e321: 8 total, 7 up, 8 in 2023-09-23T23:22:39.284 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: failed to encode map e321 with expected crc 2023-09-23T23:22:39.284 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: failed to encode map e321 with expected crc 2023-09-23T23:22:39.284 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: failed to encode map e321 with expected crc 2023-09-23T23:22:39.284 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: failed to encode map e321 with expected crc 2023-09-23T23:22:39.284 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: 10.0 starting backfill to osd.1 from (0'0,0'0] MAX to 319'10780 2023-09-23T23:22:39.284 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: 10.b starting backfill to osd.1 from (0'0,0'0] MAX to 319'10848 2023-09-23T23:22:39.284 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:39.284 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:39 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:39.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:39.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:39.386 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:39.428 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:39.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: 1.c scrub starts 2023-09-23T23:22:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: pgmap v962: 109 pgs: 1 active+remapped+backfilling, 15 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 85 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 270 op/s; 2057/98187 objects degraded (2.095%); 13480/98187 objects misplaced (13.729%); 21 MiB/s, 341 objects/s recovering 2023-09-23T23:22:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3708491072' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T23:22:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: osdmap e320: 8 total, 7 up, 8 in 2023-09-23T23:22:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: failed to encode map e320 with expected crc 2023-09-23T23:22:39.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2057/98187 objects degraded (2.095%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:39.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: failed to encode map e321 with expected crc 2023-09-23T23:22:39.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: osdmap e321: 8 total, 7 up, 8 in 2023-09-23T23:22:39.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: failed to encode map e321 with expected crc 2023-09-23T23:22:39.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: failed to encode map e321 with expected crc 2023-09-23T23:22:39.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: failed to encode map e321 with expected crc 2023-09-23T23:22:39.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: failed to encode map e321 with expected crc 2023-09-23T23:22:39.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: 10.0 starting backfill to osd.1 from (0'0,0'0] MAX to 319'10780 2023-09-23T23:22:39.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: 10.b starting backfill to osd.1 from (0'0,0'0] MAX to 319'10848 2023-09-23T23:22:39.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:39.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:39 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:40.139 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:40 smithi138 ceph-mon[193606]: 1.14 scrub starts 2023-09-23T23:22:40.139 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:40 smithi138 ceph-mon[193606]: 10.1 starting backfill to osd.1 from (0'0,0'0] MAX to 319'10485 2023-09-23T23:22:40.139 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:40 smithi138 ceph-mon[193606]: osdmap e322: 8 total, 7 up, 8 in 2023-09-23T23:22:40.139 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:40 smithi138 ceph-mon[193606]: failed to encode map e322 with expected crc 2023-09-23T23:22:40.139 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:40 smithi138 ceph-mon[193606]: failed to encode map e322 with expected crc 2023-09-23T23:22:40.139 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:40 smithi138 ceph-mon[193606]: failed to encode map e322 with expected crc 2023-09-23T23:22:40.140 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:40 smithi138 ceph-mon[193606]: failed to encode map e322 with expected crc 2023-09-23T23:22:40.354 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:40 smithi110 ceph-mon[212939]: 1.14 scrub starts 2023-09-23T23:22:40.354 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:40 smithi110 ceph-mon[212939]: 10.1 starting backfill to osd.1 from (0'0,0'0] MAX to 319'10485 2023-09-23T23:22:40.355 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:40 smithi110 ceph-mon[212939]: osdmap e322: 8 total, 7 up, 8 in 2023-09-23T23:22:40.355 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:40 smithi110 ceph-mon[212939]: failed to encode map e322 with expected crc 2023-09-23T23:22:40.355 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:40 smithi110 ceph-mon[212939]: failed to encode map e322 with expected crc 2023-09-23T23:22:40.355 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:40 smithi110 ceph-mon[212939]: failed to encode map e322 with expected crc 2023-09-23T23:22:40.355 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:40 smithi110 ceph-mon[212939]: failed to encode map e322 with expected crc 2023-09-23T23:22:40.355 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:40 smithi110 ceph-mon[221102]: 1.14 scrub starts 2023-09-23T23:22:40.355 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:40 smithi110 ceph-mon[221102]: 10.1 starting backfill to osd.1 from (0'0,0'0] MAX to 319'10485 2023-09-23T23:22:40.355 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:40 smithi110 ceph-mon[221102]: osdmap e322: 8 total, 7 up, 8 in 2023-09-23T23:22:40.355 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:40 smithi110 ceph-mon[221102]: failed to encode map e322 with expected crc 2023-09-23T23:22:40.355 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:40 smithi110 ceph-mon[221102]: failed to encode map e322 with expected crc 2023-09-23T23:22:40.356 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:40 smithi110 ceph-mon[221102]: failed to encode map e322 with expected crc 2023-09-23T23:22:40.356 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:40 smithi110 ceph-mon[221102]: failed to encode map e322 with expected crc 2023-09-23T23:22:40.358 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-09-23T23:22:40.362 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:40.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:40.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:40.375 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:22:40.375 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:22:40.469 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:40.502 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:40.603 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:40.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:40.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:40.662 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:40.707 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:40.733 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:40.776 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:40.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:40.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:40.879 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:40.913 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:41.036 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:41 smithi138 ceph-mon[193606]: pgmap v966: 109 pgs: 4 peering, 1 active+remapped+backfilling, 12 active+clean+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 82 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 267 op/s; 1911/94971 objects degraded (2.012%); 15244/94971 objects misplaced (16.051%); 0 B/s, 0 objects/s recovering 2023-09-23T23:22:41.036 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:41 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:22:41.037 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:41 smithi138 ceph-mon[193606]: osdmap e323: 8 total, 7 up, 8 in 2023-09-23T23:22:41.037 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:41 smithi138 ceph-mon[193606]: failed to encode map e323 with expected crc 2023-09-23T23:22:41.037 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:41 smithi138 ceph-mon[193606]: failed to encode map e323 with expected crc 2023-09-23T23:22:41.170 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:41.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:41.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:41.274 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:41.280 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:22:41.301 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:41.359 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:41.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:41.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:41.417 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:41 smithi110 ceph-mon[221102]: pgmap v966: 109 pgs: 4 peering, 1 active+remapped+backfilling, 12 active+clean+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 82 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 267 op/s; 1911/94971 objects degraded (2.012%); 15244/94971 objects misplaced (16.051%); 0 B/s, 0 objects/s recovering 2023-09-23T23:22:41.417 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:41 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:22:41.418 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:41 smithi110 ceph-mon[221102]: osdmap e323: 8 total, 7 up, 8 in 2023-09-23T23:22:41.418 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:41 smithi110 ceph-mon[221102]: failed to encode map e323 with expected crc 2023-09-23T23:22:41.418 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:41 smithi110 ceph-mon[221102]: failed to encode map e323 with expected crc 2023-09-23T23:22:41.418 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:41 smithi110 ceph-mon[212939]: pgmap v966: 109 pgs: 4 peering, 1 active+remapped+backfilling, 12 active+clean+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 82 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 267 op/s; 1911/94971 objects degraded (2.012%); 15244/94971 objects misplaced (16.051%); 0 B/s, 0 objects/s recovering 2023-09-23T23:22:41.418 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:41 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:22:41.418 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:41 smithi110 ceph-mon[212939]: osdmap e323: 8 total, 7 up, 8 in 2023-09-23T23:22:41.418 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:41 smithi110 ceph-mon[212939]: failed to encode map e323 with expected crc 2023-09-23T23:22:41.418 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:41 smithi110 ceph-mon[212939]: failed to encode map e323 with expected crc 2023-09-23T23:22:41.466 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:41.531 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:41.710 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:41.711 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:22:41.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:41.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:41.814 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:41.843 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:41.951 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:41.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:41.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:42.056 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:42.084 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:42.180 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:42.182 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:42 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:22:42.182 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:42 smithi138 ceph-mon[193606]: osdmap e324: 8 total, 7 up, 8 in 2023-09-23T23:22:42.182 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:42 smithi138 ceph-mon[193606]: failed to encode map e324 with expected crc 2023-09-23T23:22:42.182 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:42 smithi138 ceph-mon[193606]: failed to encode map e324 with expected crc 2023-09-23T23:22:42.182 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:42 smithi138 ceph-mon[193606]: failed to encode map e324 with expected crc 2023-09-23T23:22:42.182 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:42 smithi138 ceph-mon[193606]: failed to encode map e324 with expected crc 2023-09-23T23:22:42.182 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:42 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1305108226' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:22:42.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:42.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:42.284 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:42.319 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:42 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:22:42.319 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:42 smithi110 ceph-mon[212939]: osdmap e324: 8 total, 7 up, 8 in 2023-09-23T23:22:42.320 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:42 smithi110 ceph-mon[212939]: failed to encode map e324 with expected crc 2023-09-23T23:22:42.320 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:42 smithi110 ceph-mon[212939]: failed to encode map e324 with expected crc 2023-09-23T23:22:42.320 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:42 smithi110 ceph-mon[212939]: failed to encode map e324 with expected crc 2023-09-23T23:22:42.320 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:42 smithi110 ceph-mon[212939]: failed to encode map e324 with expected crc 2023-09-23T23:22:42.320 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:42 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1305108226' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:22:42.320 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:42 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:22:42.321 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:42 smithi110 ceph-mon[221102]: osdmap e324: 8 total, 7 up, 8 in 2023-09-23T23:22:42.321 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:42 smithi110 ceph-mon[221102]: failed to encode map e324 with expected crc 2023-09-23T23:22:42.321 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:42 smithi110 ceph-mon[221102]: failed to encode map e324 with expected crc 2023-09-23T23:22:42.321 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:42 smithi110 ceph-mon[221102]: failed to encode map e324 with expected crc 2023-09-23T23:22:42.321 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:42 smithi110 ceph-mon[221102]: failed to encode map e324 with expected crc 2023-09-23T23:22:42.321 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:42 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1305108226' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:22:42.323 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:42.461 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:22:43.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:43 smithi138 ceph-mon[193606]: pgmap v969: 109 pgs: 4 peering, 2 active+remapped+backfilling, 12 active+clean+remapped, 1 active+recovering, 5 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 77 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 24 KiB/s rd, 62 KiB/s wr, 566 op/s; 1911/92388 objects degraded (2.068%); 14483/92388 objects misplaced (15.676%); 39 MiB/s, 616 objects/s recovering 2023-09-23T23:22:43.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:43 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:22:43.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:43 smithi138 ceph-mon[193606]: failed to encode map e325 with expected crc 2023-09-23T23:22:43.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:43 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1305108226' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:22:43.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:43 smithi138 ceph-mon[193606]: osdmap e325: 8 total, 7 up, 8 in 2023-09-23T23:22:43.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:43 smithi138 ceph-mon[193606]: failed to encode map e325 with expected crc 2023-09-23T23:22:43.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:43 smithi138 ceph-mon[193606]: failed to encode map e325 with expected crc 2023-09-23T23:22:43.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:43 smithi138 ceph-mon[193606]: failed to encode map e325 with expected crc 2023-09-23T23:22:43.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:43.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:22:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[221102]: pgmap v969: 109 pgs: 4 peering, 2 active+remapped+backfilling, 12 active+clean+remapped, 1 active+recovering, 5 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 77 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 24 KiB/s rd, 62 KiB/s wr, 566 op/s; 1911/92388 objects degraded (2.068%); 14483/92388 objects misplaced (15.676%); 39 MiB/s, 616 objects/s recovering 2023-09-23T23:22:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:22:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[221102]: failed to encode map e325 with expected crc 2023-09-23T23:22:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1305108226' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:22:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[221102]: osdmap e325: 8 total, 7 up, 8 in 2023-09-23T23:22:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[221102]: failed to encode map e325 with expected crc 2023-09-23T23:22:43.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[221102]: failed to encode map e325 with expected crc 2023-09-23T23:22:43.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[221102]: failed to encode map e325 with expected crc 2023-09-23T23:22:43.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:43.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:22:43.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[212939]: pgmap v969: 109 pgs: 4 peering, 2 active+remapped+backfilling, 12 active+clean+remapped, 1 active+recovering, 5 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 77 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 24 KiB/s rd, 62 KiB/s wr, 566 op/s; 1911/92388 objects degraded (2.068%); 14483/92388 objects misplaced (15.676%); 39 MiB/s, 616 objects/s recovering 2023-09-23T23:22:43.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:22:43.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[212939]: failed to encode map e325 with expected crc 2023-09-23T23:22:43.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1305108226' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:22:43.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[212939]: osdmap e325: 8 total, 7 up, 8 in 2023-09-23T23:22:43.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[212939]: failed to encode map e325 with expected crc 2023-09-23T23:22:43.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[212939]: failed to encode map e325 with expected crc 2023-09-23T23:22:43.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[212939]: failed to encode map e325 with expected crc 2023-09-23T23:22:43.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:43.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:22:43.502 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:22:43.502 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:22:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:44 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1911/92388 objects degraded (2.068%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:44 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1911/92388 objects degraded (2.068%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:44 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1911/92388 objects degraded (2.068%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:44.607 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:22:44.837 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:44.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:44.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:44.950 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:44.999 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:45.019 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:22:45.046 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:45 smithi110 ceph-mon[212939]: pgmap v971: 109 pgs: 2 peering, 2 active+remapped+backfilling, 12 active+clean+remapped, 1 active+recovering, 5 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 79 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 20 KiB/s rd, 53 KiB/s wr, 483 op/s; 1911/92388 objects degraded (2.068%); 14483/92388 objects misplaced (15.676%); 33 MiB/s, 525 objects/s recovering 2023-09-23T23:22:45.047 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:45 smithi110 ceph-mon[212939]: failed to encode map e326 with expected crc 2023-09-23T23:22:45.047 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:45 smithi110 ceph-mon[212939]: osdmap e326: 8 total, 7 up, 8 in 2023-09-23T23:22:45.047 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:45 smithi110 ceph-mon[212939]: failed to encode map e326 with expected crc 2023-09-23T23:22:45.048 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:45 smithi110 ceph-mon[212939]: failed to encode map e326 with expected crc 2023-09-23T23:22:45.048 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:45 smithi110 ceph-mon[221102]: pgmap v971: 109 pgs: 2 peering, 2 active+remapped+backfilling, 12 active+clean+remapped, 1 active+recovering, 5 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 79 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 20 KiB/s rd, 53 KiB/s wr, 483 op/s; 1911/92388 objects degraded (2.068%); 14483/92388 objects misplaced (15.676%); 33 MiB/s, 525 objects/s recovering 2023-09-23T23:22:45.048 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:45 smithi110 ceph-mon[221102]: failed to encode map e326 with expected crc 2023-09-23T23:22:45.049 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:45 smithi110 ceph-mon[221102]: osdmap e326: 8 total, 7 up, 8 in 2023-09-23T23:22:45.049 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:45 smithi110 ceph-mon[221102]: failed to encode map e326 with expected crc 2023-09-23T23:22:45.049 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:45 smithi110 ceph-mon[221102]: failed to encode map e326 with expected crc 2023-09-23T23:22:45.067 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:45.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:45.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:45.172 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:45.207 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:45.262 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:45.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:45.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:45.299 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:45 smithi138 ceph-mon[193606]: pgmap v971: 109 pgs: 2 peering, 2 active+remapped+backfilling, 12 active+clean+remapped, 1 active+recovering, 5 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 79 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 20 KiB/s rd, 53 KiB/s wr, 483 op/s; 1911/92388 objects degraded (2.068%); 14483/92388 objects misplaced (15.676%); 33 MiB/s, 525 objects/s recovering 2023-09-23T23:22:45.299 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:45 smithi138 ceph-mon[193606]: failed to encode map e326 with expected crc 2023-09-23T23:22:45.299 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:45 smithi138 ceph-mon[193606]: osdmap e326: 8 total, 7 up, 8 in 2023-09-23T23:22:45.299 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:45 smithi138 ceph-mon[193606]: failed to encode map e326 with expected crc 2023-09-23T23:22:45.299 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:45 smithi138 ceph-mon[193606]: failed to encode map e326 with expected crc 2023-09-23T23:22:45.358 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 2, 1] out_osds: [] dead_osds: [7] live_osds: [0, 1, 3, 4, 5, 6, 2] 2023-09-23T23:22:45.359 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:22:45.359 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T23:22:45.359 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:22:45.367 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:45.395 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:45.408 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:22:45.436 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:22:45.474 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:22:45.522 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:22:45.576 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:22:45.599 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:45.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:45.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:45.617 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:22:45.651 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:22:45.652 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:22:45.652 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:22:45.652 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:22:45.653 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:22:45.653 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:22:45.653 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:22:45.653 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:22:45.653 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:22:45.703 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:45.799 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:45.919 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:45.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:45.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:46.032 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:46.073 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:46.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:46 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2648919144' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:46.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:46 smithi110 ceph-mon[212939]: osdmap e327: 8 total, 7 up, 8 in 2023-09-23T23:22:46.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:46 smithi110 ceph-mon[212939]: failed to encode map e327 with expected crc 2023-09-23T23:22:46.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:46 smithi110 ceph-mon[212939]: failed to encode map e327 with expected crc 2023-09-23T23:22:46.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:46 smithi110 ceph-mon[212939]: failed to encode map e327 with expected crc 2023-09-23T23:22:46.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:46 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:46.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:46 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:46.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:46 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2648919144' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:46.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:46 smithi110 ceph-mon[221102]: osdmap e327: 8 total, 7 up, 8 in 2023-09-23T23:22:46.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:46 smithi110 ceph-mon[221102]: failed to encode map e327 with expected crc 2023-09-23T23:22:46.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:46 smithi110 ceph-mon[221102]: failed to encode map e327 with expected crc 2023-09-23T23:22:46.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:46 smithi110 ceph-mon[221102]: failed to encode map e327 with expected crc 2023-09-23T23:22:46.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:46 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:46.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:46 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:46 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2648919144' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:46 smithi138 ceph-mon[193606]: osdmap e327: 8 total, 7 up, 8 in 2023-09-23T23:22:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:46 smithi138 ceph-mon[193606]: failed to encode map e327 with expected crc 2023-09-23T23:22:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:46 smithi138 ceph-mon[193606]: failed to encode map e327 with expected crc 2023-09-23T23:22:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:46 smithi138 ceph-mon[193606]: failed to encode map e327 with expected crc 2023-09-23T23:22:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:46 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:46 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:46.497 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:46.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:46.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:46.603 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:46.620 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:22:46.620 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:22:46.648 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:46.717 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:46.758 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:46.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:46.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:46.852 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:22:46.866 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:22:46.962 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:47.021 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:22:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:22:46] "GET /metrics HTTP/1.1" 200 36658 "" "Prometheus/2.43.0" 2023-09-23T23:22:47.049 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:22:47.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:47.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:22:47.130 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:22:47.131 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:22:47.131 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:47.131 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:174408 10.d 10:becf7093:::benchmark_data_smithi110_366450_object73200:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T23:22:47.131 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:22:47.128103+0000", 2023-09-23T23:22:47.131 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0011027070000000001, 2023-09-23T23:22:47.131 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001124634, 2023-09-23T23:22:47.131 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:22:47.131 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:22:47.131 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:22:47.131 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:22:47.132 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:22:47.132 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 174408 2023-09-23T23:22:47.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:47.132 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:22:47.132 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:47.132 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:22:47.132 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:47.128103+0000", 2023-09-23T23:22:47.132 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:47.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:47.132 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:47.132 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:22:47.132 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:47.128103+0000", 2023-09-23T23:22:47.133 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:22:47.133 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:47.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:47.133 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:22:47.133 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:47.128102+0000", 2023-09-23T23:22:47.133 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:22:47.133 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:47.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:47.133 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:22:47.133 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:47.128107+0000", 2023-09-23T23:22:47.133 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6359999999999999e-06 2023-09-23T23:22:47.133 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:47.134 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:47.134 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:22:47.134 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:47.128116+0000", 2023-09-23T23:22:47.134 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3549999999999994e-06 2023-09-23T23:22:47.134 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:47.134 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:47.134 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:22:47.134 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:47.128121+0000", 2023-09-23T23:22:47.134 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5940000000000004e-06 2023-09-23T23:22:47.134 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:47.134 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:47.134 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:22:47.135 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:47.128171+0000", 2023-09-23T23:22:47.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0389999999999997e-05 2023-09-23T23:22:47.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:47.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:47.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:22:47.135 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:47.128259+0000", 2023-09-23T23:22:47.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7581000000000001e-05 2023-09-23T23:22:47.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:22:47.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:22:47.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:22:47.135 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:47.128327+0000", 2023-09-23T23:22:47.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8008000000000003e-05 2023-09-23T23:22:47.136 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:47.136 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:22:47.136 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:47.136 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:22:47.136 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-09-23T23:22:47.136 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 1 2023-09-23T23:22:47.136 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:22:47.152 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:47 smithi138 ceph-mon[193606]: pgmap v974: 109 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 12 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 87 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 1785/86958 objects degraded (2.053%); 9612/86958 objects misplaced (11.054%); 37 MiB/s, 588 objects/s recovering 2023-09-23T23:22:47.152 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:47 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T23:22:47.152 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:47 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:22:47.152 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:47 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:22:47.152 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:47 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:47.159 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:47.192 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:47.318 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:47 smithi110 ceph-mon[212939]: pgmap v974: 109 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 12 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 87 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 1785/86958 objects degraded (2.053%); 9612/86958 objects misplaced (11.054%); 37 MiB/s, 588 objects/s recovering 2023-09-23T23:22:47.319 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:47 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T23:22:47.319 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:47 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:22:47.319 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:47 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:22:47.319 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:47.319 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:47 smithi110 ceph-mon[221102]: pgmap v974: 109 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 12 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 87 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 1785/86958 objects degraded (2.053%); 9612/86958 objects misplaced (11.054%); 37 MiB/s, 588 objects/s recovering 2023-09-23T23:22:47.319 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:47 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T23:22:47.319 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:47 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:22:47.319 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:47 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:22:47.319 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:47 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:47.454 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:47.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:47.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:47.558 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:47.586 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:47.629 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:47.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:47.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:47.733 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:47.761 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:47.811 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:47.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:47.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:47.919 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:47.970 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:48.075 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:48.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:48.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:48.180 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:48.194 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:48.219 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:48.263 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:22:48.389 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:48.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:48.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:48.493 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:48.538 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:48.621 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:22:48.693 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:22:48.774 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:48.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:48.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:48.882 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:48.915 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:48.970 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:48.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:48.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:49.034 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:22:49.036 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:22:49.073 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:49.099 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:49 smithi138 ceph-mon[193606]: pgmap v975: 109 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 12 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 87 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 1785/86958 objects degraded (2.053%); 9612/86958 objects misplaced (11.054%); 25 MiB/s, 403 objects/s recovering 2023-09-23T23:22:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:49 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1785/86958 objects degraded (2.053%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:49.319 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:49 smithi110 ceph-mon[212939]: pgmap v975: 109 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 12 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 87 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 1785/86958 objects degraded (2.053%); 9612/86958 objects misplaced (11.054%); 25 MiB/s, 403 objects/s recovering 2023-09-23T23:22:49.320 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:49 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1785/86958 objects degraded (2.053%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:49.320 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:49 smithi110 ceph-mon[221102]: pgmap v975: 109 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 12 active+clean+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+undersized+degraded, 4 active+undersized, 87 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 1785/86958 objects degraded (2.053%); 9612/86958 objects misplaced (11.054%); 25 MiB/s, 403 objects/s recovering 2023-09-23T23:22:49.320 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:49 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1785/86958 objects degraded (2.053%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:49.481 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:49.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:49.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:49.587 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:49.617 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:50.130 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:50.131 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":975,"stamp":"2023-09-23T23:22:47.557040+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1898679466,"num_objects":28986,"num_object_clones":0,"num_object_copies":86958,"num_objects_missing_on_primary":4189,"num_objects_missing":4189,"num_objects_degraded":1785,"num_objects_misplaced":9612,"num_objects_unfound":0,"num_objects_dirty":28986,"num_whiteouts":0,"num_read":135,"num_read_kb":251,"num_write":173577,"num_write_kb":6477809,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24142,"num_bytes_recovered":1530755127,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85255,"ondisk_log_size":85255,"up":305,"acting":321,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":355,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10830748,"kb_used_data":6944284,"kb_used_omap":20,"kb_used_meta":3886379,"kb_avail":739130468,"statfs":{"total":767960285184,"available":756869599232,"internally_reserved":0,"allocated":7110946816,"data_stored":7101661815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":3979652750},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-118620160,"num_objects":-1810,"num_object_clones":0,"num_object_copies":-5430,"num_objects_missing_on_primary":-119,"num_objects_missing":-119,"num_objects_degraded":-126,"num_objects_misplaced":-4871,"num_objects_unfound":0,"num_objects_dirty":-1810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1810,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2421,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.003545"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":361,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.512264+0000","last_change":"2023-09-23T23:22:38.439629+0000","last_active":"2023-09-23T23:22:42.512264+0000","last_peered":"2023-09-23T23:22:42.512264+0000","last_clean":"2023-09-23T23:22:42.512264+0000","last_became_active":"2023-09-23T23:22:38.439285+0000","last_became_peered":"2023-09-23T23:22:38.439285+0000","last_unstale":"2023-09-23T23:22:42.512264+0000","last_undegraded":"2023-09-23T23:22:42.512264+0000","last_fullsized":"2023-09-23T23:22:42.512264+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:34.431221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":284,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.487700+0000","last_change":"2023-09-23T23:22:38.436372+0000","last_active":"2023-09-23T23:22:41.487700+0000","last_peered":"2023-09-23T23:22:41.487700+0000","last_clean":"2023-09-23T23:22:41.487700+0000","last_became_active":"2023-09-23T23:22:38.436054+0000","last_became_peered":"2023-09-23T23:22:38.436054+0000","last_unstale":"2023-09-23T23:22:41.487700+0000","last_undegraded":"2023-09-23T23:22:41.487700+0000","last_fullsized":"2023-09-23T23:22:41.487700+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:36:41.158712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.461120+0000","last_change":"2023-09-23T23:22:38.436383+0000","last_active":"2023-09-23T23:22:41.461120+0000","last_peered":"2023-09-23T23:22:41.461120+0000","last_clean":"2023-09-23T23:22:41.461120+0000","last_became_active":"2023-09-23T23:22:38.436062+0000","last_became_peered":"2023-09-23T23:22:38.436062+0000","last_unstale":"2023-09-23T23:22:41.461120+0000","last_undegraded":"2023-09-23T23:22:41.461120+0000","last_fullsized":"2023-09-23T23:22:41.461120+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:07.142420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":325,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.453944+0000","last_change":"2023-09-23T23:22:38.438239+0000","last_active":"2023-09-23T23:22:42.453944+0000","last_peered":"2023-09-23T23:22:42.453944+0000","last_clean":"2023-09-23T23:22:42.453944+0000","last_became_active":"2023-09-23T23:22:38.437359+0000","last_became_peered":"2023-09-23T23:22:38.437359+0000","last_unstale":"2023-09-23T23:22:42.453944+0000","last_undegraded":"2023-09-23T23:22:42.453944+0000","last_fullsized":"2023-09-23T23:22:42.453944+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:15:40.886267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":311,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.485235+0000","last_change":"2023-09-23T23:20:15.738434+0000","last_active":"2023-09-23T23:22:41.485235+0000","last_peered":"2023-09-23T23:22:41.485235+0000","last_clean":"2023-09-23T23:22:41.485235+0000","last_became_active":"2023-09-23T23:20:15.738195+0000","last_became_peered":"2023-09-23T23:20:15.738195+0000","last_unstale":"2023-09-23T23:22:41.485235+0000","last_undegraded":"2023-09-23T23:22:41.485235+0000","last_fullsized":"2023-09-23T23:22:41.485235+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:15.224045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":306,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.597243+0000","last_change":"2023-09-23T23:22:38.436482+0000","last_active":"2023-09-23T23:22:42.597243+0000","last_peered":"2023-09-23T23:22:42.597243+0000","last_clean":"2023-09-23T23:22:42.597243+0000","last_became_active":"2023-09-23T23:22:38.436147+0000","last_became_peered":"2023-09-23T23:22:38.436147+0000","last_unstale":"2023-09-23T23:22:42.597243+0000","last_undegraded":"2023-09-23T23:22:42.597243+0000","last_fullsized":"2023-09-23T23:22:42.597243+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:30.542674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":278,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.454441+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:22:42.454441+0000","last_peered":"2023-09-23T23:22:42.454441+0000","last_clean":"2023-09-23T23:22:42.454441+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:22:42.454441+0000","last_undegraded":"2023-09-23T23:22:42.454441+0000","last_fullsized":"2023-09-23T23:22:42.454441+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":294,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.485153+0000","last_change":"2023-09-23T23:22:38.450254+0000","last_active":"2023-09-23T23:22:41.485153+0000","last_peered":"2023-09-23T23:22:41.485153+0000","last_clean":"2023-09-23T23:22:41.485153+0000","last_became_active":"2023-09-23T23:22:38.449879+0000","last_became_peered":"2023-09-23T23:22:38.449879+0000","last_unstale":"2023-09-23T23:22:41.485153+0000","last_undegraded":"2023-09-23T23:22:41.485153+0000","last_fullsized":"2023-09-23T23:22:41.485153+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:02.777916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":624,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.486969+0000","last_change":"2023-09-23T23:22:38.429461+0000","last_active":"2023-09-23T23:22:41.486969+0000","last_peered":"2023-09-23T23:22:41.486969+0000","last_clean":"2023-09-23T23:22:41.486969+0000","last_became_active":"2023-09-23T23:22:17.011652+0000","last_became_peered":"2023-09-23T23:22:17.011652+0000","last_unstale":"2023-09-23T23:22:41.486969+0000","last_undegraded":"2023-09-23T23:22:41.486969+0000","last_fullsized":"2023-09-23T23:22:41.486969+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.522632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.522632+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.018152603999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":67,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T23:22:44.443052+0000","last_change":"2023-09-23T23:20:26.841909+0000","last_active":"2023-09-23T23:22:44.443052+0000","last_peered":"2023-09-23T23:22:44.443052+0000","last_clean":"2023-09-23T23:22:44.443052+0000","last_became_active":"2023-09-23T23:20:26.841631+0000","last_became_peered":"2023-09-23T23:20:26.841631+0000","last_unstale":"2023-09-23T23:22:44.443052+0000","last_undegraded":"2023-09-23T23:22:44.443052+0000","last_fullsized":"2023-09-23T23:22:44.443052+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:07.380539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":255,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.484910+0000","last_change":"2023-09-23T23:16:56.528057+0000","last_active":"2023-09-23T23:22:41.484910+0000","last_peered":"2023-09-23T23:22:41.484910+0000","last_clean":"2023-09-23T23:22:41.484910+0000","last_became_active":"2023-09-23T23:16:56.527755+0000","last_became_peered":"2023-09-23T23:16:56.527755+0000","last_unstale":"2023-09-23T23:22:41.484910+0000","last_undegraded":"2023-09-23T23:22:41.484910+0000","last_fullsized":"2023-09-23T23:22:41.484910+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:47.396406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":143,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.487976+0000","last_change":"2023-09-23T23:16:56.567128+0000","last_active":"2023-09-23T23:22:41.487976+0000","last_peered":"2023-09-23T23:22:41.487976+0000","last_clean":"2023-09-23T23:22:41.487976+0000","last_became_active":"2023-09-23T23:16:56.551484+0000","last_became_peered":"2023-09-23T23:16:56.551484+0000","last_unstale":"2023-09-23T23:22:41.487976+0000","last_undegraded":"2023-09-23T23:22:41.487976+0000","last_fullsized":"2023-09-23T23:22:41.487976+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:14.981307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":276,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.450314+0000","last_change":"2023-09-23T23:20:15.781637+0000","last_active":"2023-09-23T23:22:42.450314+0000","last_peered":"2023-09-23T23:22:42.450314+0000","last_clean":"2023-09-23T23:22:42.450314+0000","last_became_active":"2023-09-23T23:20:15.781308+0000","last_became_peered":"2023-09-23T23:20:15.781308+0000","last_unstale":"2023-09-23T23:22:42.450314+0000","last_undegraded":"2023-09-23T23:22:42.450314+0000","last_fullsized":"2023-09-23T23:22:42.450314+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:47.963872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":96,"reported_epoch":325,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:42.596338+0000","last_change":"2023-09-23T23:21:51.058999+0000","last_active":"2023-09-23T23:22:42.596338+0000","last_peered":"2023-09-23T23:22:42.596338+0000","last_clean":"2023-09-23T23:22:42.596338+0000","last_became_active":"2023-09-23T23:21:51.044013+0000","last_became_peered":"2023-09-23T23:21:51.044013+0000","last_unstale":"2023-09-23T23:22:42.596338+0000","last_undegraded":"2023-09-23T23:22:42.596338+0000","last_fullsized":"2023-09-23T23:22:42.596338+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:47.100334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2,4],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":305,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.598530+0000","last_change":"2023-09-23T23:22:38.443826+0000","last_active":"2023-09-23T23:22:42.598530+0000","last_peered":"2023-09-23T23:22:42.598530+0000","last_clean":"2023-09-23T23:22:42.598530+0000","last_became_active":"2023-09-23T23:22:38.443494+0000","last_became_peered":"2023-09-23T23:22:38.443494+0000","last_unstale":"2023-09-23T23:22:42.598530+0000","last_undegraded":"2023-09-23T23:22:42.598530+0000","last_fullsized":"2023-09-23T23:22:42.598530+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:55.630019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":458,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.453160+0000","last_change":"2023-09-23T23:22:17.008359+0000","last_active":"2023-09-23T23:22:42.453160+0000","last_peered":"2023-09-23T23:22:42.453160+0000","last_clean":"2023-09-23T23:22:42.453160+0000","last_became_active":"2023-09-23T23:22:17.008045+0000","last_became_peered":"2023-09-23T23:22:17.008045+0000","last_unstale":"2023-09-23T23:22:42.453160+0000","last_undegraded":"2023-09-23T23:22:42.453160+0000","last_fullsized":"2023-09-23T23:22:42.453160+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:49.726740+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:49.726740+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":84,"reported_epoch":325,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:42.458311+0000","last_change":"2023-09-23T23:21:51.055871+0000","last_active":"2023-09-23T23:22:42.458311+0000","last_peered":"2023-09-23T23:22:42.458311+0000","last_clean":"2023-09-23T23:22:42.458311+0000","last_became_active":"2023-09-23T23:21:51.055372+0000","last_became_peered":"2023-09-23T23:21:51.055372+0000","last_unstale":"2023-09-23T23:22:42.458311+0000","last_undegraded":"2023-09-23T23:22:42.458311+0000","last_fullsized":"2023-09-23T23:22:42.458311+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:11.929868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":301,"reported_epoch":324,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:41.487351+0000","last_change":"2023-09-23T23:20:16.730841+0000","last_active":"2023-09-23T23:22:41.487351+0000","last_peered":"2023-09-23T23:22:41.487351+0000","last_clean":"2023-09-23T23:22:41.487351+0000","last_became_active":"2023-09-23T23:20:16.730440+0000","last_became_peered":"2023-09-23T23:20:16.730440+0000","last_unstale":"2023-09-23T23:22:41.487351+0000","last_undegraded":"2023-09-23T23:22:41.487351+0000","last_fullsized":"2023-09-23T23:22:41.487351+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:12.839390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,5],"avail_no_missing":["3","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":606,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.452512+0000","last_change":"2023-09-23T23:22:41.462393+0000","last_active":"2023-09-23T23:22:42.452512+0000","last_peered":"2023-09-23T23:22:42.452512+0000","last_clean":"2023-09-23T23:22:42.452512+0000","last_became_active":"2023-09-23T23:22:41.462082+0000","last_became_peered":"2023-09-23T23:22:41.462082+0000","last_unstale":"2023-09-23T23:22:42.452512+0000","last_undegraded":"2023-09-23T23:22:42.452512+0000","last_fullsized":"2023-09-23T23:22:42.452512+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:42.190868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:42.190868+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0049256480000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":58,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.452886+0000","last_change":"2023-09-23T23:20:15.781738+0000","last_active":"2023-09-23T23:22:42.452886+0000","last_peered":"2023-09-23T23:22:42.452886+0000","last_clean":"2023-09-23T23:22:42.452886+0000","last_became_active":"2023-09-23T23:20:15.781394+0000","last_became_peered":"2023-09-23T23:20:15.781394+0000","last_unstale":"2023-09-23T23:22:42.452886+0000","last_undegraded":"2023-09-23T23:22:42.452886+0000","last_fullsized":"2023-09-23T23:22:42.452886+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:04.696118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":182,"reported_epoch":324,"state":"active+undersized","last_fresh":"2023-09-23T23:22:41.487784+0000","last_change":"2023-09-23T23:18:59.586926+0000","last_active":"2023-09-23T23:22:41.487784+0000","last_peered":"2023-09-23T23:22:41.487784+0000","last_clean":"2023-09-23T23:10:34.157578+0000","last_became_active":"2023-09-23T23:18:59.586926+0000","last_became_peered":"2023-09-23T23:18:59.586926+0000","last_unstale":"2023-09-23T23:22:41.487784+0000","last_undegraded":"2023-09-23T23:22:41.487784+0000","last_fullsized":"2023-09-23T23:18:59.579446+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:29.301032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":422,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.450223+0000","last_change":"2023-09-23T23:22:41.468429+0000","last_active":"2023-09-23T23:22:42.450223+0000","last_peered":"2023-09-23T23:22:42.450223+0000","last_clean":"2023-09-23T23:22:42.450223+0000","last_became_active":"2023-09-23T23:22:41.467838+0000","last_became_peered":"2023-09-23T23:22:41.467838+0000","last_unstale":"2023-09-23T23:22:42.450223+0000","last_undegraded":"2023-09-23T23:22:42.450223+0000","last_fullsized":"2023-09-23T23:22:42.450223+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:19:31.012484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:19:31.012484+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":79,"reported_epoch":325,"state":"active+undersized","last_fresh":"2023-09-23T23:22:42.599917+0000","last_change":"2023-09-23T23:18:59.587971+0000","last_active":"2023-09-23T23:22:42.599917+0000","last_peered":"2023-09-23T23:22:42.599917+0000","last_clean":"2023-09-23T22:53:15.863481+0000","last_became_active":"2023-09-23T23:18:59.587971+0000","last_became_peered":"2023-09-23T23:18:59.587971+0000","last_unstale":"2023-09-23T23:22:42.599917+0000","last_undegraded":"2023-09-23T23:22:42.599917+0000","last_fullsized":"2023-09-23T23:18:59.584213+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:17.017106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":305,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.454036+0000","last_change":"2023-09-23T23:20:15.737685+0000","last_active":"2023-09-23T23:22:42.454036+0000","last_peered":"2023-09-23T23:22:42.454036+0000","last_clean":"2023-09-23T23:22:42.454036+0000","last_became_active":"2023-09-23T23:20:15.737311+0000","last_became_peered":"2023-09-23T23:20:15.737311+0000","last_unstale":"2023-09-23T23:22:42.454036+0000","last_undegraded":"2023-09-23T23:22:42.454036+0000","last_fullsized":"2023-09-23T23:22:42.454036+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:15:21.213270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":648,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.453185+0000","last_change":"2023-09-23T23:22:17.006817+0000","last_active":"2023-09-23T23:22:42.453185+0000","last_peered":"2023-09-23T23:22:42.453185+0000","last_clean":"2023-09-23T23:22:42.453185+0000","last_became_active":"2023-09-23T23:22:17.006326+0000","last_became_peered":"2023-09-23T23:22:17.006326+0000","last_unstale":"2023-09-23T23:22:42.453185+0000","last_undegraded":"2023-09-23T23:22:42.453185+0000","last_fullsized":"2023-09-23T23:22:42.453185+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:47.277080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:47.277080+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001458104,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":315,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.450176+0000","last_change":"2023-09-23T23:13:29.368175+0000","last_active":"2023-09-23T23:22:42.450176+0000","last_peered":"2023-09-23T23:22:42.450176+0000","last_clean":"2023-09-23T23:22:42.450176+0000","last_became_active":"2023-09-23T23:13:29.363603+0000","last_became_peered":"2023-09-23T23:13:29.363603+0000","last_unstale":"2023-09-23T23:22:42.450176+0000","last_undegraded":"2023-09-23T23:22:42.450176+0000","last_fullsized":"2023-09-23T23:22:42.450176+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:25.277871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":15,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.455114+0000","last_change":"2023-09-23T23:22:38.452672+0000","last_active":"2023-09-23T23:22:41.455114+0000","last_peered":"2023-09-23T23:22:41.455114+0000","last_clean":"2023-09-23T23:22:41.455114+0000","last_became_active":"2023-09-23T23:22:38.452374+0000","last_became_peered":"2023-09-23T23:22:38.452374+0000","last_unstale":"2023-09-23T23:22:41.455114+0000","last_undegraded":"2023-09-23T23:22:41.455114+0000","last_fullsized":"2023-09-23T23:22:41.455114+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:29.874430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":617,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.453557+0000","last_change":"2023-09-23T23:22:17.008670+0000","last_active":"2023-09-23T23:22:42.453557+0000","last_peered":"2023-09-23T23:22:42.453557+0000","last_clean":"2023-09-23T23:22:42.453557+0000","last_became_active":"2023-09-23T23:22:17.008338+0000","last_became_peered":"2023-09-23T23:22:17.008338+0000","last_unstale":"2023-09-23T23:22:42.453557+0000","last_undegraded":"2023-09-23T23:22:42.453557+0000","last_fullsized":"2023-09-23T23:22:42.453557+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:51.489074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:51.489074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00519324,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.6","version":"326'10853","reported_seq":10940,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T23:22:44.967915+0000","last_change":"2023-09-23T23:21:22.237069+0000","last_active":"2023-09-23T23:22:44.967915+0000","last_peered":"2023-09-23T23:22:44.967915+0000","last_clean":"2023-09-23T23:22:44.967915+0000","last_became_active":"2023-09-23T23:21:22.194375+0000","last_became_peered":"2023-09-23T23:21:22.194375+0000","last_unstale":"2023-09-23T23:22:44.967915+0000","last_undegraded":"2023-09-23T23:22:44.967915+0000","last_fullsized":"2023-09-23T23:22:44.967915+0000","mapping_epoch":295,"log_start":"290'6161","ondisk_log_start":"290'6161","created":255,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4692,"log_dups_size":0,"ondisk_log_size":4692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:56:59.831180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117243904,"num_objects":1789,"num_object_clones":0,"num_object_copies":5367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10853,"num_write_kb":404544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":140,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.599629+0000","last_change":"2023-09-23T23:16:56.548535+0000","last_active":"2023-09-23T23:22:42.599629+0000","last_peered":"2023-09-23T23:22:42.599629+0000","last_clean":"2023-09-23T23:22:42.599629+0000","last_became_active":"2023-09-23T23:16:56.546115+0000","last_became_peered":"2023-09-23T23:16:56.546115+0000","last_unstale":"2023-09-23T23:22:42.599629+0000","last_undegraded":"2023-09-23T23:22:42.599629+0000","last_fullsized":"2023-09-23T23:22:42.599629+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:02.279715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":131,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.450264+0000","last_change":"2023-09-23T23:16:56.525479+0000","last_active":"2023-09-23T23:22:42.450264+0000","last_peered":"2023-09-23T23:22:42.450264+0000","last_clean":"2023-09-23T23:22:42.450264+0000","last_became_active":"2023-09-23T23:16:56.525147+0000","last_became_peered":"2023-09-23T23:16:56.525147+0000","last_unstale":"2023-09-23T23:22:42.450264+0000","last_undegraded":"2023-09-23T23:22:42.450264+0000","last_fullsized":"2023-09-23T23:22:42.450264+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:06.165300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":625,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.487055+0000","last_change":"2023-09-23T23:22:38.040298+0000","last_active":"2023-09-23T23:22:41.487055+0000","last_peered":"2023-09-23T23:22:41.487055+0000","last_clean":"2023-09-23T23:22:41.487055+0000","last_became_active":"2023-09-23T23:22:17.013093+0000","last_became_peered":"2023-09-23T23:22:17.013093+0000","last_unstale":"2023-09-23T23:22:41.487055+0000","last_undegraded":"2023-09-23T23:22:41.487055+0000","last_fullsized":"2023-09-23T23:22:41.487055+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.522632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.522632+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.018152603999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218454,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.7","version":"326'10728","reported_seq":15208,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T23:22:44.442583+0000","last_change":"2023-09-23T23:22:31.086081+0000","last_active":"2023-09-23T23:22:44.442583+0000","last_peered":"2023-09-23T23:22:44.442583+0000","last_clean":"2023-09-23T23:22:44.442583+0000","last_became_active":"2023-09-23T23:22:31.023167+0000","last_became_peered":"2023-09-23T23:22:31.023167+0000","last_unstale":"2023-09-23T23:22:44.442583+0000","last_undegraded":"2023-09-23T23:22:44.442583+0000","last_fullsized":"2023-09-23T23:22:44.442583+0000","mapping_epoch":316,"log_start":"286'5968","ondisk_log_start":"286'5968","created":255,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4760,"log_dups_size":0,"ondisk_log_size":4760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:35.081883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117964800,"num_objects":1800,"num_object_clones":0,"num_object_copies":5400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10728,"num_write_kb":400896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2184,"num_bytes_recovered":143065088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":19,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.454978+0000","last_change":"2023-09-23T23:22:38.474057+0000","last_active":"2023-09-23T23:22:41.454978+0000","last_peered":"2023-09-23T23:22:41.454978+0000","last_clean":"2023-09-23T23:22:41.454978+0000","last_became_active":"2023-09-23T23:22:38.444672+0000","last_became_peered":"2023-09-23T23:22:38.444672+0000","last_unstale":"2023-09-23T23:22:41.454978+0000","last_undegraded":"2023-09-23T23:22:41.454978+0000","last_fullsized":"2023-09-23T23:22:41.454978+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:00.966999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":332,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.600040+0000","last_change":"2023-09-23T23:22:38.439077+0000","last_active":"2023-09-23T23:22:42.600040+0000","last_peered":"2023-09-23T23:22:42.600040+0000","last_clean":"2023-09-23T23:22:42.600040+0000","last_became_active":"2023-09-23T23:22:38.438638+0000","last_became_peered":"2023-09-23T23:22:38.438638+0000","last_unstale":"2023-09-23T23:22:42.600040+0000","last_undegraded":"2023-09-23T23:22:42.600040+0000","last_fullsized":"2023-09-23T23:22:42.600040+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:57.067702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":708,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.597077+0000","last_change":"2023-09-23T23:22:17.012087+0000","last_active":"2023-09-23T23:22:42.597077+0000","last_peered":"2023-09-23T23:22:42.597077+0000","last_clean":"2023-09-23T23:22:42.597077+0000","last_became_active":"2023-09-23T23:22:17.011761+0000","last_became_peered":"2023-09-23T23:22:17.011761+0000","last_unstale":"2023-09-23T23:22:42.597077+0000","last_undegraded":"2023-09-23T23:22:42.597077+0000","last_fullsized":"2023-09-23T23:22:42.597077+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:47.797085+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:47.797085+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.4","version":"326'10782","reported_seq":10863,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T23:22:45.090830+0000","last_change":"2023-09-23T23:18:58.585739+0000","last_active":"2023-09-23T23:22:45.090830+0000","last_peered":"2023-09-23T23:22:45.090830+0000","last_clean":"2023-09-23T23:22:45.090830+0000","last_became_active":"2023-09-23T23:18:58.585475+0000","last_became_peered":"2023-09-23T23:18:58.585475+0000","last_unstale":"2023-09-23T23:22:45.090830+0000","last_undegraded":"2023-09-23T23:22:45.090830+0000","last_fullsized":"2023-09-23T23:22:45.090830+0000","mapping_epoch":255,"log_start":"289'6082","ondisk_log_start":"289'6082","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4700,"log_dups_size":0,"ondisk_log_size":4700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:14.419790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112721920,"num_objects":1720,"num_object_clones":0,"num_object_copies":5160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10782,"num_write_kb":400064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":90,"reported_epoch":325,"state":"active+undersized+degraded","last_fresh":"2023-09-23T23:22:42.603911+0000","last_change":"2023-09-23T23:18:59.587709+0000","last_active":"2023-09-23T23:22:42.603911+0000","last_peered":"2023-09-23T23:22:42.603911+0000","last_clean":"2023-09-23T22:53:20.393182+0000","last_became_active":"2023-09-23T23:18:59.587709+0000","last_became_peered":"2023-09-23T23:18:59.587709+0000","last_unstale":"2023-09-23T23:22:42.603911+0000","last_undegraded":"2023-09-23T23:18:59.585429+0000","last_fullsized":"2023-09-23T23:18:59.585311+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:58.759914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":353,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.596292+0000","last_change":"2023-09-23T23:22:38.439331+0000","last_active":"2023-09-23T23:22:42.596292+0000","last_peered":"2023-09-23T23:22:42.596292+0000","last_clean":"2023-09-23T23:22:42.596292+0000","last_became_active":"2023-09-23T23:22:38.436293+0000","last_became_peered":"2023-09-23T23:22:38.436293+0000","last_unstale":"2023-09-23T23:22:42.596292+0000","last_undegraded":"2023-09-23T23:22:42.596292+0000","last_fullsized":"2023-09-23T23:22:42.596292+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:53.668614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":608,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.596831+0000","last_change":"2023-09-23T23:22:17.006673+0000","last_active":"2023-09-23T23:22:42.596831+0000","last_peered":"2023-09-23T23:22:42.596831+0000","last_clean":"2023-09-23T23:22:42.596831+0000","last_became_active":"2023-09-23T23:22:17.006428+0000","last_became_peered":"2023-09-23T23:22:17.006428+0000","last_unstale":"2023-09-23T23:22:42.596831+0000","last_undegraded":"2023-09-23T23:22:42.596831+0000","last_fullsized":"2023-09-23T23:22:42.596831+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.802642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.802642+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0056570290000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"326'10811","reported_seq":14794,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T23:22:45.132649+0000","last_change":"2023-09-23T23:22:42.499876+0000","last_active":"2023-09-23T23:22:45.132649+0000","last_peered":"2023-09-23T23:22:45.132649+0000","last_clean":"2023-09-23T23:22:45.132649+0000","last_became_active":"2023-09-23T23:22:42.457163+0000","last_became_peered":"2023-09-23T23:22:42.457163+0000","last_unstale":"2023-09-23T23:22:45.132649+0000","last_undegraded":"2023-09-23T23:22:45.132649+0000","last_fullsized":"2023-09-23T23:22:45.132649+0000","mapping_epoch":324,"log_start":"289'6084","ondisk_log_start":"289'6084","created":255,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4727,"log_dups_size":0,"ondisk_log_size":4727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:41.352974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113967104,"num_objects":1739,"num_object_clones":0,"num_object_copies":5217,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1739,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10811,"num_write_kb":401600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1938,"num_bytes_recovered":127008768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":142,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.487231+0000","last_change":"2023-09-23T23:21:22.192374+0000","last_active":"2023-09-23T23:22:41.487231+0000","last_peered":"2023-09-23T23:22:41.487231+0000","last_clean":"2023-09-23T23:22:41.487231+0000","last_became_active":"2023-09-23T23:21:22.192086+0000","last_became_peered":"2023-09-23T23:21:22.192086+0000","last_unstale":"2023-09-23T23:22:41.487231+0000","last_undegraded":"2023-09-23T23:22:41.487231+0000","last_fullsized":"2023-09-23T23:22:41.487231+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:20.605133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":300,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.454691+0000","last_change":"2023-09-23T23:22:38.438390+0000","last_active":"2023-09-23T23:22:42.454691+0000","last_peered":"2023-09-23T23:22:42.454691+0000","last_clean":"2023-09-23T23:22:42.454691+0000","last_became_active":"2023-09-23T23:22:38.438065+0000","last_became_peered":"2023-09-23T23:22:38.438065+0000","last_unstale":"2023-09-23T23:22:42.454691+0000","last_undegraded":"2023-09-23T23:22:42.454691+0000","last_fullsized":"2023-09-23T23:22:42.454691+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:51.254227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"326'10946","reported_seq":11025,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T23:22:44.493334+0000","last_change":"2023-09-23T23:18:58.587100+0000","last_active":"2023-09-23T23:22:44.493334+0000","last_peered":"2023-09-23T23:22:44.493334+0000","last_clean":"2023-09-23T23:22:44.493334+0000","last_became_active":"2023-09-23T23:18:58.586837+0000","last_became_peered":"2023-09-23T23:18:58.586837+0000","last_unstale":"2023-09-23T23:22:44.493334+0000","last_undegraded":"2023-09-23T23:22:44.493334+0000","last_fullsized":"2023-09-23T23:22:44.493334+0000","mapping_epoch":255,"log_start":"277'4800","ondisk_log_start":"277'4800","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":6146,"log_dups_size":0,"ondisk_log_size":6146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:56.131536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122159104,"num_objects":1864,"num_object_clones":0,"num_object_copies":5592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10946,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":654,"reported_epoch":324,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:41.484720+0000","last_change":"2023-09-23T23:22:41.484720+0000","last_active":"2023-09-23T23:22:41.484720+0000","last_peered":"2023-09-23T23:22:41.484720+0000","last_clean":"2023-09-23T23:22:41.484720+0000","last_became_active":"2023-09-23T23:20:18.750219+0000","last_became_peered":"2023-09-23T23:20:18.750219+0000","last_unstale":"2023-09-23T23:22:41.484720+0000","last_undegraded":"2023-09-23T23:22:41.484720+0000","last_fullsized":"2023-09-23T23:22:41.484720+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:46.150686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:46.150686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,5],"avail_no_missing":["3","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":291,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.486795+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:22:41.486795+0000","last_peered":"2023-09-23T23:22:41.486795+0000","last_clean":"2023-09-23T23:22:41.486795+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:22:41.486795+0000","last_undegraded":"2023-09-23T23:22:41.486795+0000","last_fullsized":"2023-09-23T23:22:41.486795+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":285,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.512428+0000","last_change":"2023-09-23T23:10:59.512363+0000","last_active":"2023-09-23T23:22:42.512428+0000","last_peered":"2023-09-23T23:22:42.512428+0000","last_clean":"2023-09-23T23:22:42.512428+0000","last_became_active":"2023-09-23T23:10:59.512022+0000","last_became_peered":"2023-09-23T23:10:59.512022+0000","last_unstale":"2023-09-23T23:22:42.512428+0000","last_undegraded":"2023-09-23T23:22:42.512428+0000","last_fullsized":"2023-09-23T23:22:42.512428+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:45.523432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"326'10855","reported_seq":13180,"reported_epoch":326,"state":"active+recovering","last_fresh":"2023-09-23T23:22:45.124343+0000","last_change":"2023-09-23T23:22:38.517002+0000","last_active":"2023-09-23T23:22:45.124343+0000","last_peered":"2023-09-23T23:22:45.124343+0000","last_clean":"2023-09-23T23:20:14.670806+0000","last_became_active":"2023-09-23T23:21:51.082780+0000","last_became_peered":"2023-09-23T23:21:51.082780+0000","last_unstale":"2023-09-23T23:22:45.124343+0000","last_undegraded":"2023-09-23T23:22:45.124343+0000","last_fullsized":"2023-09-23T23:22:45.124343+0000","mapping_epoch":301,"log_start":"278'5222","ondisk_log_start":"278'5222","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":5633,"log_dups_size":0,"ondisk_log_size":5633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:37.352090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117374976,"num_objects":1791,"num_object_clones":0,"num_object_copies":5373,"num_objects_missing_on_primary":4189,"num_objects_missing":4189,"num_objects_degraded":0,"num_objects_misplaced":769,"num_objects_unfound":0,"num_objects_dirty":1791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10855,"num_write_kb":404672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1113,"num_bytes_recovered":72941568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":["0","4","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":4},{"shards":"0,2,5","objects":1018},{"shards":"0,4,5","objects":769}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":548,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.452360+0000","last_change":"2023-09-23T23:20:18.748975+0000","last_active":"2023-09-23T23:22:42.452360+0000","last_peered":"2023-09-23T23:22:42.452360+0000","last_clean":"2023-09-23T23:22:42.452360+0000","last_became_active":"2023-09-23T23:20:18.748608+0000","last_became_peered":"2023-09-23T23:20:18.748608+0000","last_unstale":"2023-09-23T23:22:42.452360+0000","last_undegraded":"2023-09-23T23:22:42.452360+0000","last_fullsized":"2023-09-23T23:22:42.452360+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:42.871134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:42.871134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:51.672826+0000","scrub_duration":0.0023428590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":297,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.596307+0000","last_change":"2023-09-23T23:16:56.525904+0000","last_active":"2023-09-23T23:22:42.596307+0000","last_peered":"2023-09-23T23:22:42.596307+0000","last_clean":"2023-09-23T23:22:42.596307+0000","last_became_active":"2023-09-23T23:16:56.525372+0000","last_became_peered":"2023-09-23T23:16:56.525372+0000","last_unstale":"2023-09-23T23:22:42.596307+0000","last_undegraded":"2023-09-23T23:22:42.596307+0000","last_fullsized":"2023-09-23T23:22:42.596307+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:17.070117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":340,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.453716+0000","last_change":"2023-09-23T23:22:38.439737+0000","last_active":"2023-09-23T23:22:42.453716+0000","last_peered":"2023-09-23T23:22:42.453716+0000","last_clean":"2023-09-23T23:22:42.453716+0000","last_became_active":"2023-09-23T23:22:38.439246+0000","last_became_peered":"2023-09-23T23:22:38.439246+0000","last_unstale":"2023-09-23T23:22:42.453716+0000","last_undegraded":"2023-09-23T23:22:42.453716+0000","last_fullsized":"2023-09-23T23:22:42.453716+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:40.042010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":422,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.450442+0000","last_change":"2023-09-23T23:22:41.467037+0000","last_active":"2023-09-23T23:22:42.450442+0000","last_peered":"2023-09-23T23:22:42.450442+0000","last_clean":"2023-09-23T23:22:42.450442+0000","last_became_active":"2023-09-23T23:22:41.466017+0000","last_became_peered":"2023-09-23T23:22:41.466017+0000","last_unstale":"2023-09-23T23:22:42.450442+0000","last_undegraded":"2023-09-23T23:22:42.450442+0000","last_fullsized":"2023-09-23T23:22:42.450442+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:19:31.012484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:19:31.012484+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"325'10923","reported_seq":13004,"reported_epoch":325,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:22:44.376205+0000","last_change":"2023-09-23T23:22:38.579988+0000","last_active":"2023-09-23T23:22:44.376205+0000","last_peered":"2023-09-23T23:22:44.376205+0000","last_clean":"2023-09-23T23:22:37.867954+0000","last_became_active":"2023-09-23T23:22:38.539075+0000","last_became_peered":"2023-09-23T23:22:38.539075+0000","last_unstale":"2023-09-23T23:22:44.376205+0000","last_undegraded":"2023-09-23T23:22:44.376205+0000","last_fullsized":"2023-09-23T23:22:44.376205+0000","mapping_epoch":320,"log_start":"288'6143","ondisk_log_start":"288'6143","created":255,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4780,"log_dups_size":0,"ondisk_log_size":4780,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:16.329520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122748928,"num_objects":1873,"num_object_clones":0,"num_object_copies":5619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3746,"num_objects_unfound":0,"num_objects_dirty":1873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10923,"num_write_kb":409472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":975,"num_bytes_recovered":63897600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,0,4],"avail_no_missing":["6","0","4"],"object_location_counts":[{"shards":"0,4,6","objects":1873}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":336,"reported_epoch":325,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:42.452848+0000","last_change":"2023-09-23T23:22:39.449144+0000","last_active":"2023-09-23T23:22:42.452848+0000","last_peered":"2023-09-23T23:22:42.452848+0000","last_clean":"2023-09-23T23:22:42.452848+0000","last_became_active":"2023-09-23T23:22:39.448587+0000","last_became_peered":"2023-09-23T23:22:39.448587+0000","last_unstale":"2023-09-23T23:22:42.452848+0000","last_undegraded":"2023-09-23T23:22:42.452848+0000","last_fullsized":"2023-09-23T23:22:42.452848+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:37.516445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1,5],"avail_no_missing":["2","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":295,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.600092+0000","last_change":"2023-09-23T23:15:28.228439+0000","last_active":"2023-09-23T23:22:42.600092+0000","last_peered":"2023-09-23T23:22:42.600092+0000","last_clean":"2023-09-23T23:22:42.600092+0000","last_became_active":"2023-09-23T23:15:28.228095+0000","last_became_peered":"2023-09-23T23:15:28.228095+0000","last_unstale":"2023-09-23T23:22:42.600092+0000","last_undegraded":"2023-09-23T23:22:42.600092+0000","last_fullsized":"2023-09-23T23:22:42.600092+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:37.667201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"325'10639","reported_seq":19281,"reported_epoch":326,"state":"remapped+peering","last_fresh":"2023-09-23T23:22:44.422839+0000","last_change":"2023-09-23T23:22:44.423021+0000","last_active":"2023-09-23T23:22:44.422839+0000","last_peered":"2023-09-23T23:22:44.344014+0000","last_clean":"2023-09-23T23:22:44.344014+0000","last_became_active":"2023-09-23T23:22:38.521189+0000","last_became_peered":"2023-09-23T23:22:38.521189+0000","last_unstale":"2023-09-23T23:22:44.422839+0000","last_undegraded":"2023-09-23T23:22:44.422839+0000","last_fullsized":"2023-09-23T23:22:44.422839+0000","mapping_epoch":326,"log_start":"288'5947","ondisk_log_start":"288'5947","created":255,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4692,"log_dups_size":0,"ondisk_log_size":4692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:27.224477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114491392,"num_objects":1747,"num_object_clones":0,"num_object_copies":5241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10639,"num_write_kb":396352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4265,"num_bytes_recovered":279511040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,4],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":606,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.452618+0000","last_change":"2023-09-23T23:21:22.194486+0000","last_active":"2023-09-23T23:22:42.452618+0000","last_peered":"2023-09-23T23:22:42.452618+0000","last_clean":"2023-09-23T23:22:42.452618+0000","last_became_active":"2023-09-23T23:21:22.194192+0000","last_became_peered":"2023-09-23T23:21:22.194192+0000","last_unstale":"2023-09-23T23:22:42.452618+0000","last_undegraded":"2023-09-23T23:22:42.452618+0000","last_fullsized":"2023-09-23T23:22:42.452618+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:51.734652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:51.734652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:51.641289+0000","scrub_duration":0.023100959000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.453018+0000","last_change":"2023-09-23T23:22:38.434606+0000","last_active":"2023-09-23T23:22:41.453018+0000","last_peered":"2023-09-23T23:22:41.453018+0000","last_clean":"2023-09-23T23:22:41.453018+0000","last_became_active":"2023-09-23T23:22:38.434196+0000","last_became_peered":"2023-09-23T23:22:38.434196+0000","last_unstale":"2023-09-23T23:22:41.453018+0000","last_undegraded":"2023-09-23T23:22:41.453018+0000","last_fullsized":"2023-09-23T23:22:41.453018+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:14.702902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":244,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.455178+0000","last_change":"2023-09-23T23:22:39.489325+0000","last_active":"2023-09-23T23:22:41.455178+0000","last_peered":"2023-09-23T23:22:41.455178+0000","last_clean":"2023-09-23T23:22:41.455178+0000","last_became_active":"2023-09-23T23:22:39.489048+0000","last_became_peered":"2023-09-23T23:22:39.489048+0000","last_unstale":"2023-09-23T23:22:41.455178+0000","last_undegraded":"2023-09-23T23:22:41.455178+0000","last_fullsized":"2023-09-23T23:22:41.455178+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:29.984948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"326'10852","reported_seq":10927,"reported_epoch":326,"state":"active+undersized+degraded","last_fresh":"2023-09-23T23:22:44.525974+0000","last_change":"2023-09-23T23:19:10.829313+0000","last_active":"2023-09-23T23:22:44.525974+0000","last_peered":"2023-09-23T23:22:44.525974+0000","last_clean":"2023-09-23T23:18:57.558582+0000","last_became_active":"2023-09-23T23:18:59.585882+0000","last_became_peered":"2023-09-23T23:18:59.585882+0000","last_unstale":"2023-09-23T23:22:44.525974+0000","last_undegraded":"2023-09-23T23:19:10.822953+0000","last_fullsized":"2023-09-23T23:18:59.582773+0000","mapping_epoch":256,"log_start":"276'4700","ondisk_log_start":"276'4700","created":255,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":6152,"log_dups_size":0,"ondisk_log_size":6152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:12.870079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116916224,"num_objects":1784,"num_object_clones":0,"num_object_copies":5352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1784,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10852,"num_write_kb":404352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":1784}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":712,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.597685+0000","last_change":"2023-09-23T23:22:34.724848+0000","last_active":"2023-09-23T23:22:42.597685+0000","last_peered":"2023-09-23T23:22:42.597685+0000","last_clean":"2023-09-23T23:22:42.597685+0000","last_became_active":"2023-09-23T23:22:17.011704+0000","last_became_peered":"2023-09-23T23:22:17.011704+0000","last_unstale":"2023-09-23T23:22:42.597685+0000","last_undegraded":"2023-09-23T23:22:42.597685+0000","last_fullsized":"2023-09-23T23:22:42.597685+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:47.797085+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:47.797085+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.454471+0000","last_change":"2023-09-23T23:22:39.426465+0000","last_active":"2023-09-23T23:22:41.454471+0000","last_peered":"2023-09-23T23:22:41.454471+0000","last_clean":"2023-09-23T23:22:41.454471+0000","last_became_active":"2023-09-23T23:22:39.425837+0000","last_became_peered":"2023-09-23T23:22:39.425837+0000","last_unstale":"2023-09-23T23:22:41.454471+0000","last_undegraded":"2023-09-23T23:22:41.454471+0000","last_fullsized":"2023-09-23T23:22:41.454471+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:36.873867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":340,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.454249+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:22:41.454249+0000","last_peered":"2023-09-23T23:22:41.454249+0000","last_clean":"2023-09-23T23:22:41.454249+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:22:41.454249+0000","last_undegraded":"2023-09-23T23:22:41.454249+0000","last_fullsized":"2023-09-23T23:22:41.454249+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":259,"reported_epoch":325,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:42.453355+0000","last_change":"2023-09-23T23:21:51.043740+0000","last_active":"2023-09-23T23:22:42.453355+0000","last_peered":"2023-09-23T23:22:42.453355+0000","last_clean":"2023-09-23T23:22:42.453355+0000","last_became_active":"2023-09-23T23:21:51.043279+0000","last_became_peered":"2023-09-23T23:21:51.043279+0000","last_unstale":"2023-09-23T23:22:42.453355+0000","last_undegraded":"2023-09-23T23:22:42.453355+0000","last_fullsized":"2023-09-23T23:22:42.453355+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:57:07.926321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2,4],"avail_no_missing":["0","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.a","version":"326'10963","reported_seq":11042,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T23:22:44.425115+0000","last_change":"2023-09-23T23:18:58.585142+0000","last_active":"2023-09-23T23:22:44.425115+0000","last_peered":"2023-09-23T23:22:44.425115+0000","last_clean":"2023-09-23T23:22:44.425115+0000","last_became_active":"2023-09-23T23:18:58.584860+0000","last_became_peered":"2023-09-23T23:18:58.584860+0000","last_unstale":"2023-09-23T23:22:44.425115+0000","last_undegraded":"2023-09-23T23:22:44.425115+0000","last_fullsized":"2023-09-23T23:22:44.425115+0000","mapping_epoch":255,"log_start":"288'6207","ondisk_log_start":"288'6207","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4756,"log_dups_size":0,"ondisk_log_size":4756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:14.574839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121831424,"num_objects":1859,"num_object_clones":0,"num_object_copies":5577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10963,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":458,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.453610+0000","last_change":"2023-09-23T23:22:17.002317+0000","last_active":"2023-09-23T23:22:42.453610+0000","last_peered":"2023-09-23T23:22:42.453610+0000","last_clean":"2023-09-23T23:22:42.453610+0000","last_became_active":"2023-09-23T23:22:17.001911+0000","last_became_peered":"2023-09-23T23:22:17.001911+0000","last_unstale":"2023-09-23T23:22:42.453610+0000","last_undegraded":"2023-09-23T23:22:42.453610+0000","last_fullsized":"2023-09-23T23:22:42.453610+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:49.726740+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:49.726740+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.454323+0000","last_change":"2023-09-23T23:22:38.433755+0000","last_active":"2023-09-23T23:22:41.454323+0000","last_peered":"2023-09-23T23:22:41.454323+0000","last_clean":"2023-09-23T23:22:41.454323+0000","last_became_active":"2023-09-23T23:22:38.433442+0000","last_became_peered":"2023-09-23T23:22:38.433442+0000","last_unstale":"2023-09-23T23:22:41.454323+0000","last_undegraded":"2023-09-23T23:22:41.454323+0000","last_fullsized":"2023-09-23T23:22:41.454323+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:25.409363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":286,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.598369+0000","last_change":"2023-09-23T23:04:11.514703+0000","last_active":"2023-09-23T23:22:42.598369+0000","last_peered":"2023-09-23T23:22:42.598369+0000","last_clean":"2023-09-23T23:22:42.598369+0000","last_became_active":"2023-09-23T23:04:11.514403+0000","last_became_peered":"2023-09-23T23:04:11.514403+0000","last_unstale":"2023-09-23T23:22:42.598369+0000","last_undegraded":"2023-09-23T23:22:42.598369+0000","last_fullsized":"2023-09-23T23:22:42.598369+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:26.374346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":58,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.453169+0000","last_change":"2023-09-23T23:20:15.781844+0000","last_active":"2023-09-23T23:22:42.453169+0000","last_peered":"2023-09-23T23:22:42.453169+0000","last_clean":"2023-09-23T23:22:42.453169+0000","last_became_active":"2023-09-23T23:20:15.781573+0000","last_became_peered":"2023-09-23T23:20:15.781573+0000","last_unstale":"2023-09-23T23:22:42.453169+0000","last_undegraded":"2023-09-23T23:22:42.453169+0000","last_fullsized":"2023-09-23T23:22:42.453169+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:25:50.816486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"326'10836","reported_seq":15126,"reported_epoch":326,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:44.993561+0000","last_change":"2023-09-23T23:22:38.105753+0000","last_active":"2023-09-23T23:22:44.993561+0000","last_peered":"2023-09-23T23:22:44.993561+0000","last_clean":"2023-09-23T23:22:44.993561+0000","last_became_active":"2023-09-23T23:22:38.065297+0000","last_became_peered":"2023-09-23T23:22:38.065297+0000","last_unstale":"2023-09-23T23:22:44.993561+0000","last_undegraded":"2023-09-23T23:22:44.993561+0000","last_fullsized":"2023-09-23T23:22:44.993561+0000","mapping_epoch":319,"log_start":"288'6087","ondisk_log_start":"288'6087","created":255,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4749,"log_dups_size":0,"ondisk_log_size":4749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:25.482919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119144448,"num_objects":1818,"num_object_clones":0,"num_object_copies":5454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1818,"num_objects_unfound":0,"num_objects_dirty":1818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10836,"num_write_kb":404928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2090,"num_bytes_recovered":136970240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4,0],"avail_no_missing":["2","0","4"],"object_location_counts":[{"shards":"0,2,4","objects":1818}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":648,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.453373+0000","last_change":"2023-09-23T23:22:17.001078+0000","last_active":"2023-09-23T23:22:42.453373+0000","last_peered":"2023-09-23T23:22:42.453373+0000","last_clean":"2023-09-23T23:22:42.453373+0000","last_became_active":"2023-09-23T23:22:17.000797+0000","last_became_peered":"2023-09-23T23:22:17.000797+0000","last_unstale":"2023-09-23T23:22:42.453373+0000","last_undegraded":"2023-09-23T23:22:42.453373+0000","last_fullsized":"2023-09-23T23:22:42.453373+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:47.277080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:47.277080+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001458104,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":58,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.452765+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:22:42.452765+0000","last_peered":"2023-09-23T23:22:42.452765+0000","last_clean":"2023-09-23T23:22:42.452765+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:22:42.452765+0000","last_undegraded":"2023-09-23T23:22:42.452765+0000","last_fullsized":"2023-09-23T23:22:42.452765+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.452942+0000","last_change":"2023-09-23T23:22:38.480385+0000","last_active":"2023-09-23T23:22:41.452942+0000","last_peered":"2023-09-23T23:22:41.452942+0000","last_clean":"2023-09-23T23:22:41.452942+0000","last_became_active":"2023-09-23T23:22:38.480071+0000","last_became_peered":"2023-09-23T23:22:38.480071+0000","last_unstale":"2023-09-23T23:22:41.452942+0000","last_undegraded":"2023-09-23T23:22:41.452942+0000","last_fullsized":"2023-09-23T23:22:41.452942+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:15.348789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":138,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.454381+0000","last_change":"2023-09-23T23:21:22.194721+0000","last_active":"2023-09-23T23:22:42.454381+0000","last_peered":"2023-09-23T23:22:42.454381+0000","last_clean":"2023-09-23T23:22:42.454381+0000","last_became_active":"2023-09-23T23:21:22.194333+0000","last_became_peered":"2023-09-23T23:21:22.194333+0000","last_unstale":"2023-09-23T23:22:42.454381+0000","last_undegraded":"2023-09-23T23:22:42.454381+0000","last_fullsized":"2023-09-23T23:22:42.454381+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:46.823997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"325'11020","reported_seq":13969,"reported_epoch":326,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T23:22:44.480050+0000","last_change":"2023-09-23T23:22:43.455950+0000","last_active":"2023-09-23T23:22:44.480050+0000","last_peered":"2023-09-23T23:22:44.480050+0000","last_clean":"2023-09-23T23:22:37.921766+0000","last_became_active":"2023-09-23T23:22:38.526179+0000","last_became_peered":"2023-09-23T23:22:38.526179+0000","last_unstale":"2023-09-23T23:22:44.480050+0000","last_undegraded":"2023-09-23T23:22:44.480050+0000","last_fullsized":"2023-09-23T23:22:44.480050+0000","mapping_epoch":320,"log_start":"288'6215","ondisk_log_start":"288'6215","created":255,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4805,"log_dups_size":0,"ondisk_log_size":4805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:33.352068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123731968,"num_objects":1888,"num_object_clones":0,"num_object_copies":5664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1393,"num_objects_unfound":0,"num_objects_dirty":1888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11020,"num_write_kb":413056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1413,"num_bytes_recovered":92602368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,0,4],"avail_no_missing":["6","0","4"],"object_location_counts":[{"shards":"0,4,6","objects":1888}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":325,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.458120+0000","last_change":"2023-09-23T23:21:22.193754+0000","last_active":"2023-09-23T23:22:42.458120+0000","last_peered":"2023-09-23T23:22:42.458120+0000","last_clean":"2023-09-23T23:22:42.458120+0000","last_became_active":"2023-09-23T23:21:22.193409+0000","last_became_peered":"2023-09-23T23:21:22.193409+0000","last_unstale":"2023-09-23T23:22:42.458120+0000","last_undegraded":"2023-09-23T23:22:42.458120+0000","last_fullsized":"2023-09-23T23:22:42.458120+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:53.635813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":606,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.452652+0000","last_change":"2023-09-23T23:22:41.462946+0000","last_active":"2023-09-23T23:22:42.452652+0000","last_peered":"2023-09-23T23:22:42.452652+0000","last_clean":"2023-09-23T23:22:42.452652+0000","last_became_active":"2023-09-23T23:22:41.462616+0000","last_became_peered":"2023-09-23T23:22:41.462616+0000","last_unstale":"2023-09-23T23:22:42.452652+0000","last_undegraded":"2023-09-23T23:22:42.452652+0000","last_fullsized":"2023-09-23T23:22:42.452652+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:42.190868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:42.190868+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0049256480000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":254,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.485895+0000","last_change":"2023-09-23T23:16:56.525298+0000","last_active":"2023-09-23T23:22:41.485895+0000","last_peered":"2023-09-23T23:22:41.485895+0000","last_clean":"2023-09-23T23:22:41.485895+0000","last_became_active":"2023-09-23T23:16:56.524974+0000","last_became_peered":"2023-09-23T23:16:56.524974+0000","last_unstale":"2023-09-23T23:22:41.485895+0000","last_undegraded":"2023-09-23T23:22:41.485895+0000","last_fullsized":"2023-09-23T23:22:41.485895+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:08.860790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":288,"reported_epoch":325,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:42.603760+0000","last_change":"2023-09-23T23:21:51.046606+0000","last_active":"2023-09-23T23:22:42.603760+0000","last_peered":"2023-09-23T23:22:42.603760+0000","last_clean":"2023-09-23T23:22:42.603760+0000","last_became_active":"2023-09-23T23:21:51.046103+0000","last_became_peered":"2023-09-23T23:21:51.046103+0000","last_unstale":"2023-09-23T23:22:42.603760+0000","last_undegraded":"2023-09-23T23:22:42.603760+0000","last_fullsized":"2023-09-23T23:22:42.603760+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:05.999638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,4],"avail_no_missing":["6","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"326'11016","reported_seq":21888,"reported_epoch":326,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:44.940101+0000","last_change":"2023-09-23T23:22:09.217110+0000","last_active":"2023-09-23T23:22:44.940101+0000","last_peered":"2023-09-23T23:22:44.940101+0000","last_clean":"2023-09-23T23:22:44.940101+0000","last_became_active":"2023-09-23T23:22:09.186713+0000","last_became_peered":"2023-09-23T23:22:09.186713+0000","last_unstale":"2023-09-23T23:22:44.940101+0000","last_undegraded":"2023-09-23T23:22:44.940101+0000","last_fullsized":"2023-09-23T23:22:44.940101+0000","mapping_epoch":306,"log_start":"282'5459","ondisk_log_start":"282'5459","created":255,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":5557,"log_dups_size":0,"ondisk_log_size":5557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:00.893419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123207680,"num_objects":1880,"num_object_clones":0,"num_object_copies":5640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1880,"num_objects_unfound":0,"num_objects_dirty":1880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11016,"num_write_kb":412672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5377,"num_bytes_recovered":301334528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0,5],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2,5","objects":1880}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":422,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.450360+0000","last_change":"2023-09-23T23:22:41.467338+0000","last_active":"2023-09-23T23:22:42.450360+0000","last_peered":"2023-09-23T23:22:42.450360+0000","last_clean":"2023-09-23T23:22:42.450360+0000","last_became_active":"2023-09-23T23:22:41.466645+0000","last_became_peered":"2023-09-23T23:22:41.466645+0000","last_unstale":"2023-09-23T23:22:42.450360+0000","last_undegraded":"2023-09-23T23:22:42.450360+0000","last_fullsized":"2023-09-23T23:22:42.450360+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:19:31.012484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:19:31.012484+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":275,"reported_epoch":325,"state":"active+undersized","last_fresh":"2023-09-23T23:22:42.452778+0000","last_change":"2023-09-23T23:18:59.580395+0000","last_active":"2023-09-23T23:22:42.452778+0000","last_peered":"2023-09-23T23:22:42.452778+0000","last_clean":"2023-09-23T23:16:52.490944+0000","last_became_active":"2023-09-23T23:18:59.580395+0000","last_became_peered":"2023-09-23T23:18:59.580395+0000","last_unstale":"2023-09-23T23:22:42.452778+0000","last_undegraded":"2023-09-23T23:22:42.452778+0000","last_fullsized":"2023-09-23T23:18:59.577313+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:36.323937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":298,"reported_epoch":325,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:42.453473+0000","last_change":"2023-09-23T23:20:27.898330+0000","last_active":"2023-09-23T23:22:42.453473+0000","last_peered":"2023-09-23T23:22:42.453473+0000","last_clean":"2023-09-23T23:22:42.453473+0000","last_became_active":"2023-09-23T23:20:27.897982+0000","last_became_peered":"2023-09-23T23:20:27.897982+0000","last_unstale":"2023-09-23T23:22:42.453473+0000","last_undegraded":"2023-09-23T23:22:42.453473+0000","last_fullsized":"2023-09-23T23:22:42.453473+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:28.564415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,4],"avail_no_missing":["0","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":354,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.453982+0000","last_change":"2023-09-23T23:22:38.438448+0000","last_active":"2023-09-23T23:22:42.453982+0000","last_peered":"2023-09-23T23:22:42.453982+0000","last_clean":"2023-09-23T23:22:42.453982+0000","last_became_active":"2023-09-23T23:22:38.437495+0000","last_became_peered":"2023-09-23T23:22:38.437495+0000","last_unstale":"2023-09-23T23:22:42.453982+0000","last_undegraded":"2023-09-23T23:22:42.453982+0000","last_fullsized":"2023-09-23T23:22:42.453982+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:49.587056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"325'10837","reported_seq":10917,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:44.337698+0000","last_change":"2023-09-23T23:18:58.587441+0000","last_active":"2023-09-23T23:22:44.337698+0000","last_peered":"2023-09-23T23:22:44.337698+0000","last_clean":"2023-09-23T23:22:44.337698+0000","last_became_active":"2023-09-23T23:18:58.587188+0000","last_became_peered":"2023-09-23T23:18:58.587188+0000","last_unstale":"2023-09-23T23:22:44.337698+0000","last_undegraded":"2023-09-23T23:22:44.337698+0000","last_fullsized":"2023-09-23T23:22:44.337698+0000","mapping_epoch":255,"log_start":"287'6072","ondisk_log_start":"287'6072","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4765,"log_dups_size":0,"ondisk_log_size":4765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:54.227096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117571608,"num_objects":1795,"num_object_clones":0,"num_object_copies":5385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1795,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10837,"num_write_kb":404161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":609,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.597040+0000","last_change":"2023-09-23T23:22:17.006681+0000","last_active":"2023-09-23T23:22:42.597040+0000","last_peered":"2023-09-23T23:22:42.597040+0000","last_clean":"2023-09-23T23:22:42.597040+0000","last_became_active":"2023-09-23T23:22:17.006437+0000","last_became_peered":"2023-09-23T23:22:17.006437+0000","last_unstale":"2023-09-23T23:22:42.597040+0000","last_undegraded":"2023-09-23T23:22:42.597040+0000","last_fullsized":"2023-09-23T23:22:42.597040+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.802642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.802642+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0056570290000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":127,"reported_epoch":325,"state":"active+undersized","last_fresh":"2023-09-23T23:22:42.458113+0000","last_change":"2023-09-23T23:18:59.585438+0000","last_active":"2023-09-23T23:22:42.458113+0000","last_peered":"2023-09-23T23:22:42.458113+0000","last_clean":"2023-09-23T23:03:28.874460+0000","last_became_active":"2023-09-23T23:18:59.585438+0000","last_became_peered":"2023-09-23T23:18:59.585438+0000","last_unstale":"2023-09-23T23:22:42.458113+0000","last_undegraded":"2023-09-23T23:22:42.458113+0000","last_fullsized":"2023-09-23T23:18:59.577149+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:37.419384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.460767+0000","last_change":"2023-09-23T23:22:38.453323+0000","last_active":"2023-09-23T23:22:41.460767+0000","last_peered":"2023-09-23T23:22:41.460767+0000","last_clean":"2023-09-23T23:22:41.460767+0000","last_became_active":"2023-09-23T23:22:38.452858+0000","last_became_peered":"2023-09-23T23:22:38.452858+0000","last_unstale":"2023-09-23T23:22:41.460767+0000","last_undegraded":"2023-09-23T23:22:41.460767+0000","last_fullsized":"2023-09-23T23:22:41.460767+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:31.988045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":46,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.453360+0000","last_change":"2023-09-23T23:20:36.429855+0000","last_active":"2023-09-23T23:22:41.453360+0000","last_peered":"2023-09-23T23:22:41.453360+0000","last_clean":"2023-09-23T23:22:41.453360+0000","last_became_active":"2023-09-23T23:20:36.427234+0000","last_became_peered":"2023-09-23T23:20:36.427234+0000","last_unstale":"2023-09-23T23:22:41.453360+0000","last_undegraded":"2023-09-23T23:22:41.453360+0000","last_fullsized":"2023-09-23T23:22:41.453360+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:39.297055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"326'10822","reported_seq":15866,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T23:22:44.457581+0000","last_change":"2023-09-23T23:22:20.455214+0000","last_active":"2023-09-23T23:22:44.457581+0000","last_peered":"2023-09-23T23:22:44.457581+0000","last_clean":"2023-09-23T23:22:44.457581+0000","last_became_active":"2023-09-23T23:22:20.427281+0000","last_became_peered":"2023-09-23T23:22:20.427281+0000","last_unstale":"2023-09-23T23:22:44.457581+0000","last_undegraded":"2023-09-23T23:22:44.457581+0000","last_fullsized":"2023-09-23T23:22:44.457581+0000","mapping_epoch":312,"log_start":"276'4700","ondisk_log_start":"276'4700","created":255,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":6122,"log_dups_size":0,"ondisk_log_size":6122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:50.223603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120455168,"num_objects":1838,"num_object_clones":0,"num_object_copies":5514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10822,"num_write_kb":405120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2467,"num_bytes_recovered":161677312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":617,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.453641+0000","last_change":"2023-09-23T23:22:17.005178+0000","last_active":"2023-09-23T23:22:42.453641+0000","last_peered":"2023-09-23T23:22:42.453641+0000","last_clean":"2023-09-23T23:22:42.453641+0000","last_became_active":"2023-09-23T23:22:17.004855+0000","last_became_peered":"2023-09-23T23:22:17.004855+0000","last_unstale":"2023-09-23T23:22:42.453641+0000","last_undegraded":"2023-09-23T23:22:42.453641+0000","last_fullsized":"2023-09-23T23:22:42.453641+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:51.489074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:51.489074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00519324,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":304,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.487066+0000","last_change":"2023-09-23T23:16:56.561147+0000","last_active":"2023-09-23T23:22:41.487066+0000","last_peered":"2023-09-23T23:22:41.487066+0000","last_clean":"2023-09-23T23:22:41.487066+0000","last_became_active":"2023-09-23T23:16:56.547808+0000","last_became_peered":"2023-09-23T23:16:56.547808+0000","last_unstale":"2023-09-23T23:22:41.487066+0000","last_undegraded":"2023-09-23T23:22:41.487066+0000","last_fullsized":"2023-09-23T23:22:41.487066+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:13.666957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":181,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.597237+0000","last_change":"2023-09-23T23:16:56.526187+0000","last_active":"2023-09-23T23:22:42.597237+0000","last_peered":"2023-09-23T23:22:42.597237+0000","last_clean":"2023-09-23T23:22:42.597237+0000","last_became_active":"2023-09-23T23:16:56.525874+0000","last_became_peered":"2023-09-23T23:16:56.525874+0000","last_unstale":"2023-09-23T23:22:42.597237+0000","last_undegraded":"2023-09-23T23:22:42.597237+0000","last_fullsized":"2023-09-23T23:22:42.597237+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:32:07.101559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":15,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.452795+0000","last_change":"2023-09-23T23:22:38.444291+0000","last_active":"2023-09-23T23:22:41.452795+0000","last_peered":"2023-09-23T23:22:41.452795+0000","last_clean":"2023-09-23T23:22:41.452795+0000","last_became_active":"2023-09-23T23:22:38.443996+0000","last_became_peered":"2023-09-23T23:22:38.443996+0000","last_unstale":"2023-09-23T23:22:41.452795+0000","last_undegraded":"2023-09-23T23:22:41.452795+0000","last_fullsized":"2023-09-23T23:22:41.452795+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:57.255970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.f","version":"325'10568","reported_seq":15264,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:44.366901+0000","last_change":"2023-09-23T23:22:25.497617+0000","last_active":"2023-09-23T23:22:44.366901+0000","last_peered":"2023-09-23T23:22:44.366901+0000","last_clean":"2023-09-23T23:22:44.366901+0000","last_became_active":"2023-09-23T23:22:25.453489+0000","last_became_peered":"2023-09-23T23:22:25.453489+0000","last_unstale":"2023-09-23T23:22:44.366901+0000","last_undegraded":"2023-09-23T23:22:44.366901+0000","last_fullsized":"2023-09-23T23:22:44.366901+0000","mapping_epoch":314,"log_start":"286'5785","ondisk_log_start":"286'5785","created":255,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4783,"log_dups_size":0,"ondisk_log_size":4783,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:58.199474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116391936,"num_objects":1776,"num_object_clones":0,"num_object_copies":5328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10568,"num_write_kb":395008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2294,"num_bytes_recovered":150339584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":624,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.487522+0000","last_change":"2023-09-23T23:22:37.037700+0000","last_active":"2023-09-23T23:22:41.487522+0000","last_peered":"2023-09-23T23:22:41.487522+0000","last_clean":"2023-09-23T23:22:41.487522+0000","last_became_active":"2023-09-23T23:22:17.009092+0000","last_became_peered":"2023-09-23T23:22:17.009092+0000","last_unstale":"2023-09-23T23:22:41.487522+0000","last_undegraded":"2023-09-23T23:22:41.487522+0000","last_fullsized":"2023-09-23T23:22:41.487522+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.522632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.522632+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.018152603999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":338,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.597758+0000","last_change":"2023-09-23T23:20:15.786693+0000","last_active":"2023-09-23T23:22:42.597758+0000","last_peered":"2023-09-23T23:22:42.597758+0000","last_clean":"2023-09-23T23:22:42.597758+0000","last_became_active":"2023-09-23T23:20:15.786360+0000","last_became_peered":"2023-09-23T23:20:15.786360+0000","last_unstale":"2023-09-23T23:22:42.597758+0000","last_undegraded":"2023-09-23T23:22:42.597758+0000","last_fullsized":"2023-09-23T23:22:42.597758+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:00.707946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":309,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.453900+0000","last_change":"2023-09-23T23:22:38.482705+0000","last_active":"2023-09-23T23:22:42.453900+0000","last_peered":"2023-09-23T23:22:42.453900+0000","last_clean":"2023-09-23T23:22:42.453900+0000","last_became_active":"2023-09-23T23:22:38.438239+0000","last_became_peered":"2023-09-23T23:22:38.438239+0000","last_unstale":"2023-09-23T23:22:42.453900+0000","last_undegraded":"2023-09-23T23:22:42.453900+0000","last_fullsized":"2023-09-23T23:22:42.453900+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:52.006349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":38,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.453508+0000","last_change":"2023-09-23T23:21:50.045682+0000","last_active":"2023-09-23T23:22:41.453508+0000","last_peered":"2023-09-23T23:22:41.453508+0000","last_clean":"2023-09-23T23:22:41.453508+0000","last_became_active":"2023-09-23T23:21:50.045339+0000","last_became_peered":"2023-09-23T23:21:50.045339+0000","last_unstale":"2023-09-23T23:22:41.453508+0000","last_undegraded":"2023-09-23T23:22:41.453508+0000","last_fullsized":"2023-09-23T23:22:41.453508+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:55.802091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":66,"reported_epoch":325,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:42.512409+0000","last_change":"2023-09-23T23:20:27.899913+0000","last_active":"2023-09-23T23:22:42.512409+0000","last_peered":"2023-09-23T23:22:42.512409+0000","last_clean":"2023-09-23T23:22:42.512409+0000","last_became_active":"2023-09-23T23:20:27.899525+0000","last_became_peered":"2023-09-23T23:20:27.899525+0000","last_unstale":"2023-09-23T23:22:42.512409+0000","last_undegraded":"2023-09-23T23:22:42.512409+0000","last_fullsized":"2023-09-23T23:22:42.512409+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:41.775325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,0],"avail_no_missing":["4","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":66,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.453290+0000","last_change":"2023-09-23T23:22:38.439602+0000","last_active":"2023-09-23T23:22:42.453290+0000","last_peered":"2023-09-23T23:22:42.453290+0000","last_clean":"2023-09-23T23:22:42.453290+0000","last_became_active":"2023-09-23T23:22:38.439261+0000","last_became_peered":"2023-09-23T23:22:38.439261+0000","last_unstale":"2023-09-23T23:22:42.453290+0000","last_undegraded":"2023-09-23T23:22:42.453290+0000","last_fullsized":"2023-09-23T23:22:42.453290+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:03.293816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":68,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.600575+0000","last_change":"2023-09-23T23:22:38.452968+0000","last_active":"2023-09-23T23:22:42.600575+0000","last_peered":"2023-09-23T23:22:42.600575+0000","last_clean":"2023-09-23T23:22:42.600575+0000","last_became_active":"2023-09-23T23:22:38.437986+0000","last_became_peered":"2023-09-23T23:22:38.437986+0000","last_unstale":"2023-09-23T23:22:42.600575+0000","last_undegraded":"2023-09-23T23:22:42.600575+0000","last_fullsized":"2023-09-23T23:22:42.600575+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:10.333271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":321,"reported_epoch":324,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:41.486582+0000","last_change":"2023-09-23T23:21:51.056449+0000","last_active":"2023-09-23T23:22:41.486582+0000","last_peered":"2023-09-23T23:22:41.486582+0000","last_clean":"2023-09-23T23:22:41.486582+0000","last_became_active":"2023-09-23T23:21:51.056092+0000","last_became_peered":"2023-09-23T23:21:51.056092+0000","last_unstale":"2023-09-23T23:22:41.486582+0000","last_undegraded":"2023-09-23T23:22:41.486582+0000","last_fullsized":"2023-09-23T23:22:41.486582+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:33.836469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,4],"avail_no_missing":["3","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":54,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.599697+0000","last_change":"2023-09-23T23:21:22.194730+0000","last_active":"2023-09-23T23:22:42.599697+0000","last_peered":"2023-09-23T23:22:42.599697+0000","last_clean":"2023-09-23T23:22:42.599697+0000","last_became_active":"2023-09-23T23:21:22.194420+0000","last_became_peered":"2023-09-23T23:21:22.194420+0000","last_unstale":"2023-09-23T23:22:42.599697+0000","last_undegraded":"2023-09-23T23:22:42.599697+0000","last_fullsized":"2023-09-23T23:22:42.599697+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:31.083059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":169,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.485816+0000","last_change":"2023-09-23T23:20:26.841878+0000","last_active":"2023-09-23T23:22:41.485816+0000","last_peered":"2023-09-23T23:22:41.485816+0000","last_clean":"2023-09-23T23:22:41.485816+0000","last_became_active":"2023-09-23T23:20:26.841590+0000","last_became_peered":"2023-09-23T23:20:26.841590+0000","last_unstale":"2023-09-23T23:22:41.485816+0000","last_undegraded":"2023-09-23T23:22:41.485816+0000","last_fullsized":"2023-09-23T23:22:41.485816+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:23.904011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":331,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.486753+0000","last_change":"2023-09-23T23:20:16.730934+0000","last_active":"2023-09-23T23:22:41.486753+0000","last_peered":"2023-09-23T23:22:41.486753+0000","last_clean":"2023-09-23T23:22:41.486753+0000","last_became_active":"2023-09-23T23:20:16.730571+0000","last_became_peered":"2023-09-23T23:20:16.730571+0000","last_unstale":"2023-09-23T23:22:41.486753+0000","last_undegraded":"2023-09-23T23:22:41.486753+0000","last_fullsized":"2023-09-23T23:22:41.486753+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:17.993278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":334,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T23:22:41.486109+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:22:41.486109+0000","last_peered":"2023-09-23T23:22:41.486109+0000","last_clean":"2023-09-23T23:22:41.486109+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:22:41.486109+0000","last_undegraded":"2023-09-23T23:22:41.486109+0000","last_fullsized":"2023-09-23T23:22:41.486109+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":328,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.454309+0000","last_change":"2023-09-23T23:20:15.786279+0000","last_active":"2023-09-23T23:22:42.454309+0000","last_peered":"2023-09-23T23:22:42.454309+0000","last_clean":"2023-09-23T23:22:42.454309+0000","last_became_active":"2023-09-23T23:20:15.782731+0000","last_became_peered":"2023-09-23T23:20:15.782731+0000","last_unstale":"2023-09-23T23:22:42.454309+0000","last_undegraded":"2023-09-23T23:22:42.454309+0000","last_fullsized":"2023-09-23T23:22:42.454309+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:58.011396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":362,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T23:22:42.512068+0000","last_change":"2023-09-23T23:22:38.447350+0000","last_active":"2023-09-23T23:22:42.512068+0000","last_peered":"2023-09-23T23:22:42.512068+0000","last_clean":"2023-09-23T23:22:42.512068+0000","last_became_active":"2023-09-23T23:22:38.447081+0000","last_became_peered":"2023-09-23T23:22:38.447081+0000","last_unstale":"2023-09-23T23:22:42.512068+0000","last_undegraded":"2023-09-23T23:22:42.512068+0000","last_fullsized":"2023-09-23T23:22:42.512068+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:39:12.306545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":90,"acting":95,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1897922584,"num_objects":28961,"num_object_clones":0,"num_object_copies":86883,"num_objects_missing_on_primary":4189,"num_objects_missing":4189,"num_objects_degraded":1784,"num_objects_misplaced":9606,"num_objects_unfound":0,"num_objects_dirty":28961,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":173451,"num_write_kb":6477121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24116,"num_bytes_recovered":1529348096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6696218624,"data_stored":6696206408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":81819,"ondisk_log_size":81819,"up":43,"acting":47,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":6,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5954,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":89,"acting":94,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":751378,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":134,"num_read_kb":250,"num_write":65,"num_write_kb":659,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1401077,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":62,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":21,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234314,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":803516,"kb_used_data":352116,"kb_used_omap":0,"kb_used_meta":451392,"kb_avail":92941636,"statfs":{"total":95995035648,"available":95172235264,"internally_reserved":0,"allocated":360566784,"data_stored":359841767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":462225408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:18:39 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":4.9820000000000002,"15min":3.9239999999999999},"min":{"1min":0.22,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":6.9249999999999998,"5min":216.40899999999999,"15min":230.249},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.7320000000000002,"5min":5.7359999999999998,"15min":4.2290000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24199999999999999},"max":{"1min":11.327999999999999,"5min":216.43700000000001,"15min":230.261},"last":0.40899999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:17:56 2023","interfaces":[{"interface":"back","average":{"1min":2.6150000000000002,"5min":2.9700000000000002,"15min":4.2359999999999998},"min":{"1min":0.39100000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.798,"5min":121.39400000000001,"15min":121.39400000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.2989999999999999,"5min":2.9790000000000001,"15min":4.077},"min":{"1min":0.27000000000000002,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":10.034000000000001,"5min":121.381,"15min":121.381},"last":0.42099999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0339999999999998,"5min":2.0179999999999998,"15min":2.0150000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.375,"5min":9.5589999999999993,"15min":9.5589999999999993},"last":0.435},{"interface":"front","average":{"1min":2.5600000000000001,"5min":1.8580000000000001,"15min":1.7410000000000001},"min":{"1min":0.36899999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.561999999999999,"5min":11.561999999999999,"15min":11.561999999999999},"last":0.47999999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:18:11 2023","interfaces":[{"interface":"back","average":{"1min":3.1400000000000001,"5min":5.7140000000000004,"15min":3.4620000000000002},"min":{"1min":0.27800000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":17.724,"5min":144.50299999999999,"15min":144.50299999999999},"last":0.755},{"interface":"front","average":{"1min":2.6829999999999998,"5min":5.5990000000000002,"15min":3.468},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":17.712,"5min":144.48699999999999,"15min":144.48699999999999},"last":0.75}]},{"osd":4,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":0.83099999999999996,"15min":1.3200000000000001},"min":{"1min":0.27300000000000002,"5min":0.254,"15min":0.23599999999999999},"max":{"1min":7.0570000000000004,"5min":7.0570000000000004,"15min":92.781000000000006},"last":0.372},{"interface":"front","average":{"1min":1.208,"5min":1.2110000000000001,"15min":1.4219999999999999},"min":{"1min":0.29499999999999998,"5min":0.28599999999999998,"15min":0.254},"max":{"1min":3.9569999999999999,"5min":9.7530000000000001,"15min":92.626999999999995},"last":0.39100000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":13.24,"5min":3.488,"15min":2.3759999999999999},"min":{"1min":0.28799999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":185.321,"5min":185.321,"15min":185.321},"last":0.44600000000000001},{"interface":"front","average":{"1min":11.488,"5min":2.851,"15min":1.97},"min":{"1min":0.33900000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":185.47499999999999,"5min":185.47499999999999,"15min":185.47499999999999},"last":0.49199999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":1.292,"15min":1.4970000000000001},"min":{"1min":0.376,"5min":0.30199999999999999,"15min":0.27300000000000002},"max":{"1min":19.782,"5min":19.782,"15min":93.105999999999995},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.7349999999999999,"5min":1.492,"15min":1.599},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28399999999999997},"max":{"1min":20.064,"5min":20.064,"15min":93.198999999999998},"last":0.46800000000000003}]}]},{"osd":6,"up_from":46,"seq":197568495996,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1389700,"kb_used_data":949948,"kb_used_omap":0,"kb_used_meta":439744,"kb_avail":92355452,"statfs":{"total":95995035648,"available":94571982848,"internally_reserved":0,"allocated":972746752,"data_stored":971828350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":450297856},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:21:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0049999999999999,"5min":7.4279999999999999,"15min":4.1600000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":23.481000000000002,"5min":161.64599999999999,"15min":161.64599999999999},"last":0.23300000000000001},{"interface":"front","average":{"1min":3.7069999999999999,"5min":6.7370000000000001,"15min":4.0640000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":23.454000000000001,"5min":161.38499999999999,"15min":161.38499999999999},"last":0.437}]},{"osd":1,"last update":"Sat Sep 23 23:21:52 2023","interfaces":[{"interface":"back","average":{"1min":1.798,"5min":5.0599999999999996,"15min":4.0880000000000001},"min":{"1min":0.29299999999999998,"5min":0.249,"15min":0.24099999999999999},"max":{"1min":8.5210000000000008,"5min":210.869,"15min":210.869},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.6579999999999999,"5min":4.9500000000000002,"15min":4.2140000000000004},"min":{"1min":0.27500000000000002,"5min":0.26800000000000002,"15min":0.26200000000000001},"max":{"1min":7.6849999999999996,"5min":210.89699999999999,"15min":210.89699999999999},"last":0.28199999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":1.712,"15min":1.7629999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.7349999999999999,"5min":11.804,"15min":11.804},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.73799999999999999,"5min":1.7370000000000001,"15min":1.4219999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.7559999999999998,"5min":10.685,"15min":10.685},"last":0.26500000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.8109999999999999,"15min":3.0259999999999998},"min":{"1min":0.31900000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.9910000000000001,"5min":10.528,"15min":109.98699999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.885,"15min":3.161},"min":{"1min":0.32700000000000001,"5min":0.28100000000000003,"15min":0.24199999999999999},"max":{"1min":4.1970000000000001,"5min":14.573,"15min":110.024},"last":0.36299999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":2.6120000000000001,"15min":1.653},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.22900000000000001},"max":{"1min":3.0779999999999998,"5min":166.05199999999999,"15min":166.05199999999999},"last":0.316},{"interface":"front","average":{"1min":0.70799999999999996,"5min":3.367,"15min":1.9470000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":2.319,"5min":166.631,"15min":166.631},"last":0.34599999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":1.9339999999999999,"15min":1.5860000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.29999999999999999},"max":{"1min":3.113,"5min":91.899000000000001,"15min":91.899000000000001},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.65900000000000003,"5min":2.234,"15min":1.5740000000000001},"min":{"1min":0.28799999999999998,"5min":0.249,"15min":0.249},"max":{"1min":2.3559999999999999,"5min":94.174000000000007,"15min":94.174000000000007},"last":0.42799999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:18:28 2023","interfaces":[{"interface":"back","average":{"1min":2.7250000000000001,"5min":1.2689999999999999,"15min":1.27},"min":{"1min":0.36799999999999999,"5min":0.32300000000000001,"15min":0.32100000000000001},"max":{"1min":13.823,"5min":13.823,"15min":13.823},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.5609999999999999,"5min":0.79900000000000004,"15min":0.98799999999999999},"min":{"1min":0.35999999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":13.103,"5min":13.103,"15min":13.103},"last":0.54400000000000004}]}]},{"osd":1,"up_from":112,"seq":481036337337,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":601508,"kb_used_data":128988,"kb_used_omap":4,"kb_used_meta":472507,"kb_avail":93143644,"statfs":{"total":95995035648,"available":95379091456,"internally_reserved":0,"allocated":132083712,"data_stored":130605089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":483848078},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:21:50 2023","interfaces":[{"interface":"back","average":{"1min":4.0359999999999996,"5min":4.008,"15min":2.5739999999999998},"min":{"1min":0.19800000000000001,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":23.138999999999999,"5min":68.540000000000006,"15min":68.540000000000006},"last":0.16800000000000001},{"interface":"front","average":{"1min":3.7269999999999999,"5min":4.2949999999999999,"15min":2.6720000000000002},"min":{"1min":0.28499999999999998,"5min":0.19600000000000001,"15min":0.192},"max":{"1min":22.741,"5min":67.007000000000005,"15min":67.007000000000005},"last":1.4510000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:22:30 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":1.125,"15min":4.6970000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.984,"5min":12.506,"15min":25.744},"last":0.20799999999999999},{"interface":"front","average":{"1min":1.052,"5min":1.853,"15min":3.1219999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":6.5659999999999998,"5min":25.998999999999999,"15min":25.998999999999999},"last":1.508}]},{"osd":3,"last update":"Sat Sep 23 23:22:24 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":4.3369999999999997,"15min":3.6400000000000001},"min":{"1min":0.27300000000000002,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":3.0840000000000001,"5min":145.88499999999999,"15min":145.88499999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":1.117,"5min":4.3239999999999998,"15min":3.379},"min":{"1min":0.30599999999999999,"5min":0.245,"15min":0.245},"max":{"1min":4.8490000000000002,"5min":145.93000000000001,"15min":145.93000000000001},"last":0.35399999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:21:50 2023","interfaces":[{"interface":"back","average":{"1min":1.536,"5min":1.7170000000000001,"15min":1.677},"min":{"1min":0.307,"5min":0.28799999999999998,"15min":0.28000000000000003},"max":{"1min":6.3170000000000002,"5min":18.474,"15min":18.474},"last":1.5629999999999999},{"interface":"front","average":{"1min":0.90700000000000003,"5min":1.5529999999999999,"15min":1.7729999999999999},"min":{"1min":0.34799999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.5350000000000001,"5min":13.177,"15min":25.710999999999999},"last":0.30399999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:21:50 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":6.2489999999999997,"15min":3.7909999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.26400000000000001},"max":{"1min":4.827,"5min":250.33000000000001,"15min":250.33000000000001},"last":0.443},{"interface":"front","average":{"1min":1.6599999999999999,"5min":6.2510000000000003,"15min":3.706},"min":{"1min":0.34499999999999997,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":5.0389999999999997,"5min":250.21199999999999,"15min":250.21199999999999},"last":1.611}]},{"osd":6,"last update":"Sat Sep 23 23:21:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":3.0209999999999999,"15min":2.2349999999999999},"min":{"1min":0.36899999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.3540000000000001,"5min":49.171999999999997,"15min":49.171999999999997},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.6639999999999999,"5min":2.7469999999999999,"15min":2.202},"min":{"1min":0.38900000000000001,"5min":0.32800000000000001,"15min":0.32700000000000001},"max":{"1min":5.016,"5min":49.222000000000001,"15min":49.222000000000001},"last":1.5900000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:18:40 2023","interfaces":[{"interface":"back","average":{"1min":3.0710000000000002,"5min":2.4540000000000002,"15min":1.9970000000000001},"min":{"1min":0.47699999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":18.532,"5min":18.532,"15min":18.532},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.3380000000000001,"15min":1.8440000000000001},"min":{"1min":0.42199999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.9710000000000001,"5min":25.667000000000002,"15min":25.667000000000002},"last":0.5}]}]},{"osd":0,"up_from":91,"seq":390842024161,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2400812,"kb_used_data":1627428,"kb_used_omap":5,"kb_used_meta":773370,"kb_avail":91344340,"statfs":{"total":95995035648,"available":93536604160,"internally_reserved":0,"allocated":1666486272,"data_stored":1664997875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":791931514},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:21:58 2023","interfaces":[{"interface":"back","average":{"1min":4.5069999999999997,"5min":3.1200000000000001,"15min":3.3239999999999998},"min":{"1min":0.20599999999999999,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":21.585999999999999,"5min":26.754999999999999,"15min":155.91999999999999},"last":0.22800000000000001},{"interface":"front","average":{"1min":2.5739999999999998,"5min":2.3069999999999999,"15min":2.8820000000000001},"min":{"1min":0.26000000000000001,"5min":0.20399999999999999,"15min":0.189},"max":{"1min":9.0449999999999999,"5min":16.591000000000001,"15min":153.28899999999999},"last":1.359}]},{"osd":2,"last update":"Sat Sep 23 23:22:29 2023","interfaces":[{"interface":"back","average":{"1min":12.528,"5min":4.4329999999999998,"15min":3.569},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":154.952,"5min":154.952,"15min":154.952},"last":0.32700000000000001},{"interface":"front","average":{"1min":13.58,"5min":4.6950000000000003,"15min":4.3540000000000001},"min":{"1min":0.27900000000000003,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":154.922,"5min":154.922,"15min":154.922},"last":0.27500000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:22:08 2023","interfaces":[{"interface":"back","average":{"1min":3.5329999999999999,"5min":3.2029999999999998,"15min":3.444},"min":{"1min":0.26500000000000001,"5min":0.245,"15min":0.22700000000000001},"max":{"1min":13.736000000000001,"5min":35.148000000000003,"15min":82.691000000000003},"last":0.26300000000000001},{"interface":"front","average":{"1min":2.6840000000000002,"5min":2.8460000000000001,"15min":2.7480000000000002},"min":{"1min":0.30599999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.4889999999999999,"5min":43.006999999999998,"15min":82.429000000000002},"last":1.391}]},{"osd":4,"last update":"Sat Sep 23 23:22:37 2023","interfaces":[{"interface":"back","average":{"1min":1.8360000000000001,"5min":3.1640000000000001,"15min":2.2879999999999998},"min":{"1min":0.39200000000000002,"5min":0.27000000000000002,"15min":0.247},"max":{"1min":10.803000000000001,"5min":86.915999999999997,"15min":86.915999999999997},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.091,"5min":2.8559999999999999,"15min":2.2280000000000002},"min":{"1min":0.34300000000000003,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":8.2919999999999998,"5min":86.433999999999997,"15min":86.433999999999997},"last":2.593}]},{"osd":5,"last update":"Sat Sep 23 23:22:37 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.7430000000000001,"15min":1.8660000000000001},"min":{"1min":0.33300000000000002,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":8.3300000000000001,"5min":15.923,"15min":15.923},"last":2.5590000000000002},{"interface":"front","average":{"1min":1.28,"5min":1.6719999999999999,"15min":1.7390000000000001},"min":{"1min":0.42999999999999999,"5min":0.34300000000000003,"15min":0.30299999999999999},"max":{"1min":7.1769999999999996,"5min":15.929,"15min":15.929},"last":0.44700000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:22:37 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":2.3719999999999999,"15min":2.1200000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.4209999999999994,"5min":30.651,"15min":30.651},"last":2.6240000000000001},{"interface":"front","average":{"1min":1.3919999999999999,"5min":2.3719999999999999,"15min":2.1200000000000001},"min":{"1min":0.39900000000000002,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":7.218,"5min":30.673999999999999,"15min":30.673999999999999},"last":0.47999999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:18:32 2023","interfaces":[{"interface":"back","average":{"1min":4.306,"5min":2.3620000000000001,"15min":2.1829999999999998},"min":{"1min":0.372,"5min":0.34499999999999997,"15min":0.29299999999999998},"max":{"1min":26.413,"5min":26.413,"15min":26.413},"last":0.42899999999999999},{"interface":"front","average":{"1min":4.7960000000000003,"5min":2.1920000000000002,"15min":2.1219999999999999},"min":{"1min":0.372,"5min":0.36499999999999999,"15min":0.313},"max":{"1min":26.437999999999999,"5min":26.437999999999999,"15min":26.437999999999999},"last":0.52600000000000002}]}]},{"osd":2,"up_from":199,"seq":854698491982,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1118476,"kb_used_data":876740,"kb_used_omap":7,"kb_used_meta":241720,"kb_avail":92626676,"statfs":{"total":95995035648,"available":94849716224,"internally_reserved":0,"allocated":897781760,"data_stored":896335922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":247521361},"hb_peers":[0,1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:22:33 2023","interfaces":[{"interface":"back","average":{"1min":10.635,"5min":4.8650000000000002,"15min":5.3109999999999999},"min":{"1min":0.16700000000000001,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":119.277,"5min":119.277,"15min":119.277},"last":0.52700000000000002},{"interface":"front","average":{"1min":10.326000000000001,"5min":4.9109999999999996,"15min":5.6689999999999996},"min":{"1min":0.19800000000000001,"5min":0.185,"15min":0.185},"max":{"1min":119.56100000000001,"5min":119.56100000000001,"15min":119.56100000000001},"last":0.23499999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:22:35 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":1.8180000000000001,"15min":10.494},"min":{"1min":0.23000000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":6.7839999999999998,"5min":18.888000000000002,"15min":130.02799999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.212,"5min":1.9750000000000001,"15min":9.8369999999999997},"min":{"1min":0.26300000000000001,"5min":0.245,"15min":0.245},"max":{"1min":6.7350000000000003,"5min":18.196999999999999,"15min":130.13399999999999},"last":0.49099999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:22:35 2023","interfaces":[{"interface":"back","average":{"1min":6.952,"5min":3.5249999999999999,"15min":6.4180000000000001},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":119.366,"5min":119.366,"15min":119.366},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.99299999999999999,"5min":2.3780000000000001,"15min":5.4640000000000004},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.5590000000000002,"5min":72.412999999999997,"15min":72.412999999999997},"last":0.56699999999999995}]},{"osd":4,"last update":"Sat Sep 23 23:22:35 2023","interfaces":[{"interface":"back","average":{"1min":7.016,"5min":7.9640000000000004,"15min":3.9860000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":119.88200000000001,"5min":332.98099999999999,"15min":332.98099999999999},"last":0.64600000000000002},{"interface":"front","average":{"1min":1.3089999999999999,"5min":6.4640000000000004,"15min":3.5299999999999998},"min":{"1min":0.33800000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.4290000000000003,"5min":332.96199999999999,"15min":332.96199999999999},"last":0.55000000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:22:35 2023","interfaces":[{"interface":"back","average":{"1min":8.1159999999999997,"5min":3.1360000000000001,"15min":2.3780000000000001},"min":{"1min":0.29499999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":119.919,"5min":119.919,"15min":119.919},"last":0.67900000000000005},{"interface":"front","average":{"1min":8.2759999999999998,"5min":3.0070000000000001,"15min":2.403},"min":{"1min":0.35499999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":119.306,"5min":119.306,"15min":119.306},"last":0.58699999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:22:35 2023","interfaces":[{"interface":"back","average":{"1min":9.0850000000000009,"5min":8.4969999999999999,"15min":4.2770000000000001},"min":{"1min":0.35799999999999998,"5min":0.31,"15min":0.31},"max":{"1min":119.943,"5min":349.32799999999997,"15min":349.32799999999997},"last":0.61599999999999999},{"interface":"front","average":{"1min":3.359,"5min":7.2240000000000002,"15min":3.8079999999999998},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":39.856999999999999,"5min":349.17700000000002,"15min":349.17700000000002},"last":0.629}]},{"osd":7,"last update":"Sat Sep 23 23:18:22 2023","interfaces":[{"interface":"back","average":{"1min":23.006,"5min":6.4870000000000001,"15min":3.734},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":239.768,"5min":239.768,"15min":239.768},"last":0.61499999999999999},{"interface":"front","average":{"1min":22.75,"5min":5.71,"15min":2.8700000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":239.78899999999999,"5min":239.78899999999999,"15min":239.78899999999999},"last":0.54400000000000004}]}]},{"osd":3,"up_from":148,"seq":635655159951,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":888640,"kb_used_data":469752,"kb_used_omap":2,"kb_used_meta":418877,"kb_avail":92856512,"statfs":{"total":95995035648,"available":95085068288,"internally_reserved":0,"allocated":481026048,"data_stored":479630357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":428930101},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:22:21 2023","interfaces":[{"interface":"back","average":{"1min":1.948,"5min":2.008,"15min":2.3359999999999999},"min":{"1min":0.17999999999999999,"5min":0.158,"15min":0.158},"max":{"1min":5.8949999999999996,"5min":21.427,"15min":72.778999999999996},"last":4.1379999999999999},{"interface":"front","average":{"1min":1.528,"5min":1.764,"15min":2.3500000000000001},"min":{"1min":0.218,"5min":0.193,"15min":0.17999999999999999},"max":{"1min":7.2489999999999997,"5min":23.742999999999999,"15min":72.719999999999999},"last":1.994}]},{"osd":1,"last update":"Sat Sep 23 23:22:21 2023","interfaces":[{"interface":"back","average":{"1min":1.871,"5min":2.0920000000000001,"15min":2.6890000000000001},"min":{"1min":0.28000000000000003,"5min":0.216,"15min":0.19600000000000001},"max":{"1min":5.7939999999999996,"5min":20.138999999999999,"15min":76.322999999999993},"last":4.1070000000000002},{"interface":"front","average":{"1min":1.456,"5min":1.4379999999999999,"15min":2.4319999999999999},"min":{"1min":0.28599999999999998,"5min":0.24399999999999999,"15min":0.185},"max":{"1min":5.8220000000000001,"5min":13.112,"15min":76.444000000000003},"last":0.65400000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:22:33 2023","interfaces":[{"interface":"back","average":{"1min":10.446999999999999,"5min":3.98,"15min":3.8239999999999998},"min":{"1min":0.23999999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":101.279,"5min":101.279,"15min":101.279},"last":7.4909999999999997},{"interface":"front","average":{"1min":10.196999999999999,"5min":3.6560000000000001,"15min":2.3479999999999999},"min":{"1min":0.308,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":101.294,"5min":101.294,"15min":101.294},"last":6.0289999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:22:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.552,"15min":1.6120000000000001},"min":{"1min":0.31900000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.8620000000000001,"5min":14.754,"15min":14.754},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.5900000000000001,"5min":1.875,"15min":2.0779999999999998},"min":{"1min":0.38900000000000001,"5min":0.38400000000000001,"15min":0.28499999999999998},"max":{"1min":8.1099999999999994,"5min":9.5960000000000001,"15min":13.532999999999999},"last":2.149}]},{"osd":5,"last update":"Sat Sep 23 23:22:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":1.6499999999999999,"15min":2.0950000000000002},"min":{"1min":0.35899999999999999,"5min":0.32900000000000001,"15min":0.28199999999999997},"max":{"1min":7.3049999999999997,"5min":10.531000000000001,"15min":75.733000000000004},"last":2.0960000000000001},{"interface":"front","average":{"1min":1.7909999999999999,"5min":1.847,"15min":2.1419999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.28399999999999997},"max":{"1min":5.3769999999999998,"5min":8.3460000000000001,"15min":75.686999999999998},"last":4.1619999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:22:21 2023","interfaces":[{"interface":"back","average":{"1min":1.3580000000000001,"5min":1.8460000000000001,"15min":1.98},"min":{"1min":0.39100000000000001,"5min":0.38800000000000001,"15min":0.32200000000000001},"max":{"1min":7.569,"5min":10.593999999999999,"15min":14.492000000000001},"last":2.0619999999999998},{"interface":"front","average":{"1min":1.9590000000000001,"5min":2.0600000000000001,"15min":1.974},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.33300000000000002},"max":{"1min":6.7069999999999999,"5min":8.3889999999999993,"15min":9.8379999999999992},"last":4.2050000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:18:09 2023","interfaces":[{"interface":"back","average":{"1min":2.198,"5min":2.915,"15min":2.5459999999999998},"min":{"1min":0.40200000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":9.4280000000000008,"5min":130.839,"15min":130.839},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.9129999999999998,"5min":2.992,"15min":2.665},"min":{"1min":0.434,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":8.8190000000000008,"5min":130.90000000000001,"15min":130.90000000000001},"last":0.57799999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019269,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2524812,"kb_used_data":1955652,"kb_used_omap":0,"kb_used_meta":569152,"kb_avail":91220340,"statfs":{"total":95995035648,"available":93409628160,"internally_reserved":0,"allocated":2002587648,"data_stored":2001661140,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":582811648},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:21:42 2023","interfaces":[{"interface":"back","average":{"1min":4.1630000000000003,"5min":7.9489999999999998,"15min":5.0659999999999998},"min":{"1min":0.245,"5min":0.223,"15min":0.20799999999999999},"max":{"1min":52.356000000000002,"5min":199.64400000000001,"15min":238.62200000000001},"last":0.29399999999999998},{"interface":"front","average":{"1min":5.3579999999999997,"5min":8.3079999999999998,"15min":5.5309999999999997},"min":{"1min":0.245,"5min":0.245,"15min":0.23300000000000001},"max":{"1min":52.546999999999997,"5min":195.649,"15min":238.58199999999999},"last":0.32200000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:21:58 2023","interfaces":[{"interface":"back","average":{"1min":1.694,"5min":3.5870000000000002,"15min":4.851},"min":{"1min":0.307,"5min":0.23200000000000001,"15min":0.23100000000000001},"max":{"1min":7.5110000000000001,"5min":161.923,"15min":175.00899999999999},"last":0.252},{"interface":"front","average":{"1min":1.8560000000000001,"5min":3.7050000000000001,"15min":5.0030000000000001},"min":{"1min":0.29299999999999998,"5min":0.26500000000000001,"15min":0.25800000000000001},"max":{"1min":6.1210000000000004,"5min":161.39699999999999,"15min":174.821},"last":0.27900000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:22:31 2023","interfaces":[{"interface":"back","average":{"1min":2.1749999999999998,"5min":1.9670000000000001,"15min":3.008},"min":{"1min":0.27500000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":27.148,"5min":27.148,"15min":39.308},"last":0.76700000000000002},{"interface":"front","average":{"1min":2.6640000000000001,"5min":2.0739999999999998,"15min":2.464},"min":{"1min":0.32500000000000001,"5min":0.316,"15min":0.316},"max":{"1min":27.117000000000001,"5min":27.117000000000001,"15min":27.117000000000001},"last":0.34100000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:22:25 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":2.9889999999999999,"15min":4.0979999999999999},"min":{"1min":0.32800000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.165,"5min":152.04499999999999,"15min":303.41500000000002},"last":0.442},{"interface":"front","average":{"1min":1.6799999999999999,"5min":3.4329999999999998,"15min":4.4740000000000002},"min":{"1min":0.317,"5min":0.308,"15min":0.308},"max":{"1min":6.2789999999999999,"5min":151.75299999999999,"15min":303.428},"last":0.45000000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:21:58 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":2.7149999999999999,"15min":1.5449999999999999},"min":{"1min":0.33500000000000002,"5min":0.26500000000000001,"15min":0.23699999999999999},"max":{"1min":4.8319999999999999,"5min":160.79499999999999,"15min":160.79499999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.7030000000000001,"5min":2.895,"15min":1.821},"min":{"1min":0.30399999999999999,"5min":0.27400000000000002,"15min":0.27200000000000002},"max":{"1min":6.1710000000000003,"5min":160.61099999999999,"15min":160.61099999999999},"last":0.39500000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.89700000000000002,"15min":1.0629999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.8820000000000001,"5min":6.0910000000000002,"15min":9.3350000000000009},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.093,"5min":1.252,"15min":1.224},"min":{"1min":0.32700000000000001,"5min":0.30599999999999999,"15min":0.27500000000000002},"max":{"1min":7.1219999999999999,"5min":7.1219999999999999,"15min":13.888999999999999},"last":0.42199999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:18:35 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.5609999999999999,"15min":1.3140000000000001},"min":{"1min":0.36299999999999999,"5min":0.315,"15min":0.30499999999999999},"max":{"1min":7.4850000000000003,"5min":14.842000000000001,"15min":14.842000000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.0169999999999999,"5min":1.502,"15min":1.341},"min":{"1min":0.433,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.882,"5min":14.561,"15min":14.561},"last":0.378}]}]},{"osd":5,"up_from":38,"seq":163208757632,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1103284,"kb_used_data":583660,"kb_used_omap":0,"kb_used_meta":519616,"kb_avail":92641868,"statfs":{"total":95995035648,"available":94865272832,"internally_reserved":0,"allocated":597667840,"data_stored":596761315,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":532086784},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:22:42 2023","interfaces":[{"interface":"back","average":{"1min":17.654,"5min":6.9450000000000003,"15min":3.5430000000000001},"min":{"1min":0.23499999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":113.869,"5min":113.869,"15min":113.869},"last":37.371000000000002},{"interface":"front","average":{"1min":17.544,"5min":7.0490000000000004,"15min":3.798},"min":{"1min":0.26000000000000001,"5min":0.23000000000000001,"15min":0.22700000000000001},"max":{"1min":113.398,"5min":113.398,"15min":113.398},"last":37.286999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:21:51 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":3.2759999999999998,"15min":3.7959999999999998},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.246},"max":{"1min":8.0899999999999999,"5min":91.518000000000001,"15min":193.584},"last":0.27200000000000002},{"interface":"front","average":{"1min":3.3929999999999998,"5min":3.4159999999999999,"15min":4.0190000000000001},"min":{"1min":0.39000000000000001,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":16.312999999999999,"5min":87.718000000000004,"15min":194.03100000000001},"last":0.35299999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:22:31 2023","interfaces":[{"interface":"back","average":{"1min":5.8209999999999997,"5min":2.532,"15min":1.8120000000000001},"min":{"1min":0.32200000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":86.765000000000001,"5min":86.765000000000001,"15min":86.765000000000001},"last":0.309},{"interface":"front","average":{"1min":5.4500000000000002,"5min":2.1509999999999998,"15min":1.871},"min":{"1min":0.32100000000000001,"5min":0.30099999999999999,"15min":0.25800000000000001},"max":{"1min":86.837000000000003,"5min":86.837000000000003,"15min":86.837000000000003},"last":0.45100000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:22:26 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":4.4009999999999998,"15min":4.085},"min":{"1min":0.40899999999999997,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":13.664,"5min":159.53299999999999,"15min":159.53299999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.625,"5min":4.8369999999999997,"15min":4.7009999999999996},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.312},"max":{"1min":13.715999999999999,"5min":159.54900000000001,"15min":159.54900000000001},"last":0.36899999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:22:09 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.224,"15min":1.3149999999999999},"min":{"1min":0.29999999999999999,"5min":0.24299999999999999,"15min":0.221},"max":{"1min":8.1210000000000004,"5min":8.1210000000000004,"15min":14.538},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.89700000000000002,"15min":0.92400000000000004},"min":{"1min":0.26400000000000001,"5min":0.26100000000000001,"15min":0.25},"max":{"1min":1.6180000000000001,"5min":11.923999999999999,"15min":11.923999999999999},"last":0.32300000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:22:09 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":4.282,"15min":2.2690000000000001},"min":{"1min":0.33500000000000002,"5min":0.28299999999999997,"15min":0.255},"max":{"1min":1.5940000000000001,"5min":310.68799999999999,"15min":310.68799999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.0880000000000001,"5min":4.7990000000000004,"15min":2.5150000000000001},"min":{"1min":0.28599999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.3230000000000004,"5min":311.45400000000001,"15min":311.45400000000001},"last":0.40699999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:18:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2010000000000001,"5min":1.286,"15min":1.3700000000000001},"min":{"1min":0.36699999999999999,"5min":0.32700000000000001,"15min":0.29199999999999998},"max":{"1min":12.09,"5min":12.09,"15min":14.929},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.319,"15min":1.276},"min":{"1min":0.371,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":5.7350000000000003,"5min":6.7089999999999996,"15min":15.090999999999999},"last":0.54400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1651314688,"data_stored":1651310616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":133562368,"data_stored":133562368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":890372096,"data_stored":890372096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":469172224,"data_stored":469172224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1997410304,"data_stored":1997406232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":591331328,"data_stored":591331328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":963055616,"data_stored":963051544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:22:50.138 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T23:22:50.138 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 31 2023-09-23T23:22:50.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:50.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:50.241 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:50.273 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:50.274 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:50.374 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:50 smithi110 ceph-mon[212939]: 1.4 scrub starts 2023-09-23T23:22:50.374 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:50 smithi110 ceph-mon[212939]: from='client.55723 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:22:50.374 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:50 smithi110 ceph-mon[212939]: osdmap e328: 8 total, 7 up, 8 in 2023-09-23T23:22:50.374 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:50 smithi110 ceph-mon[212939]: failed to encode map e328 with expected crc 2023-09-23T23:22:50.374 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:50 smithi110 ceph-mon[212939]: failed to encode map e328 with expected crc 2023-09-23T23:22:50.375 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:50 smithi110 ceph-mon[212939]: failed to encode map e328 with expected crc 2023-09-23T23:22:50.375 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:50 smithi110 ceph-mon[221102]: 1.4 scrub starts 2023-09-23T23:22:50.375 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:50 smithi110 ceph-mon[221102]: from='client.55723 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:22:50.375 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:50 smithi110 ceph-mon[221102]: osdmap e328: 8 total, 7 up, 8 in 2023-09-23T23:22:50.375 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:50 smithi110 ceph-mon[221102]: failed to encode map e328 with expected crc 2023-09-23T23:22:50.375 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:50 smithi110 ceph-mon[221102]: failed to encode map e328 with expected crc 2023-09-23T23:22:50.375 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:50 smithi110 ceph-mon[221102]: failed to encode map e328 with expected crc 2023-09-23T23:22:50.408 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:50.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:50.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:50.433 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:50 smithi138 ceph-mon[193606]: 1.4 scrub starts 2023-09-23T23:22:50.433 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:50 smithi138 ceph-mon[193606]: from='client.55723 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:22:50.433 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:50 smithi138 ceph-mon[193606]: osdmap e328: 8 total, 7 up, 8 in 2023-09-23T23:22:50.434 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:50 smithi138 ceph-mon[193606]: failed to encode map e328 with expected crc 2023-09-23T23:22:50.434 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:50 smithi138 ceph-mon[193606]: failed to encode map e328 with expected crc 2023-09-23T23:22:50.434 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:50 smithi138 ceph-mon[193606]: failed to encode map e328 with expected crc 2023-09-23T23:22:50.511 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:50.561 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:50.644 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:50.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:50.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:50.720 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:22:50.720 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:22:50.720 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:22:50.720 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:22:50.720 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:22:50.752 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:50.824 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:51.061 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:51 smithi138 ceph-mon[193606]: pgmap v977: 109 pgs: 1 peering, 1 active+remapped+backfilling, 1 remapped+peering, 12 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 87 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 1670/83895 objects degraded (1.991%); 7765/83895 objects misplaced (9.256%); 42 MiB/s, 665 objects/s recovering 2023-09-23T23:22:51.061 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:51 smithi138 ceph-mon[193606]: failed to encode map e329 with expected crc 2023-09-23T23:22:51.061 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:51 smithi138 ceph-mon[193606]: osdmap e329: 8 total, 7 up, 8 in 2023-09-23T23:22:51.061 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:51 smithi138 ceph-mon[193606]: failed to encode map e329 with expected crc 2023-09-23T23:22:51.061 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:51 smithi138 ceph-mon[193606]: failed to encode map e329 with expected crc 2023-09-23T23:22:51.062 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:51 smithi138 ceph-mon[193606]: failed to encode map e329 with expected crc 2023-09-23T23:22:51.076 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:51.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:51.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:51.180 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:51.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:51 smithi110 ceph-mon[221102]: pgmap v977: 109 pgs: 1 peering, 1 active+remapped+backfilling, 1 remapped+peering, 12 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 87 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 1670/83895 objects degraded (1.991%); 7765/83895 objects misplaced (9.256%); 42 MiB/s, 665 objects/s recovering 2023-09-23T23:22:51.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:51 smithi110 ceph-mon[221102]: failed to encode map e329 with expected crc 2023-09-23T23:22:51.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:51 smithi110 ceph-mon[221102]: osdmap e329: 8 total, 7 up, 8 in 2023-09-23T23:22:51.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:51 smithi110 ceph-mon[221102]: failed to encode map e329 with expected crc 2023-09-23T23:22:51.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:51 smithi110 ceph-mon[221102]: failed to encode map e329 with expected crc 2023-09-23T23:22:51.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:51 smithi110 ceph-mon[221102]: failed to encode map e329 with expected crc 2023-09-23T23:22:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:51 smithi110 ceph-mon[212939]: pgmap v977: 109 pgs: 1 peering, 1 active+remapped+backfilling, 1 remapped+peering, 12 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 87 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 1670/83895 objects degraded (1.991%); 7765/83895 objects misplaced (9.256%); 42 MiB/s, 665 objects/s recovering 2023-09-23T23:22:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:51 smithi110 ceph-mon[212939]: failed to encode map e329 with expected crc 2023-09-23T23:22:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:51 smithi110 ceph-mon[212939]: osdmap e329: 8 total, 7 up, 8 in 2023-09-23T23:22:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:51 smithi110 ceph-mon[212939]: failed to encode map e329 with expected crc 2023-09-23T23:22:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:51 smithi110 ceph-mon[212939]: failed to encode map e329 with expected crc 2023-09-23T23:22:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:51 smithi110 ceph-mon[212939]: failed to encode map e329 with expected crc 2023-09-23T23:22:51.217 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:51.268 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:51.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:51.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:51.374 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:51.467 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:51.766 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:51.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:51.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:51.858 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:22:51.870 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:51.901 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:51.934 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:51.996 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:52.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:52.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:52.103 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:22:52.107 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:52.147 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:52.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:52 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2232379936' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:22:52.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:52 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:22:52.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:52 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2232379936' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:22:52.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:52 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:22:52.218 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:52.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:52.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:52.331 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:22:52.340 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:22:52.368 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:52 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2232379936' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:22:52.368 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:52 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:22:52.371 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:52.434 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:22:52.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:52.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:22:52.543 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:52.587 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:52.956 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:52.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:52.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:53.062 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:53.094 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:53.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:53 smithi138 ceph-mon[193606]: pgmap v979: 109 pgs: 1 peering, 1 active+remapped+backfilling, 13 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 87 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 297 op/s; 1670/81495 objects degraded (2.049%); 9061/81495 objects misplaced (11.118%); 17 MiB/s, 278 objects/s recovering 2023-09-23T23:22:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:53 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:22:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:53 smithi138 ceph-mon[193606]: failed to encode map e330 with expected crc 2023-09-23T23:22:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:53 smithi138 ceph-mon[193606]: failed to encode map e330 with expected crc 2023-09-23T23:22:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:53 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:22:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:53 smithi138 ceph-mon[193606]: osdmap e330: 8 total, 7 up, 8 in 2023-09-23T23:22:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:53 smithi138 ceph-mon[193606]: failed to encode map e330 with expected crc 2023-09-23T23:22:53.298 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:53 smithi110 ceph-mon[221102]: pgmap v979: 109 pgs: 1 peering, 1 active+remapped+backfilling, 13 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 87 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 297 op/s; 1670/81495 objects degraded (2.049%); 9061/81495 objects misplaced (11.118%); 17 MiB/s, 278 objects/s recovering 2023-09-23T23:22:53.298 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:53 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:22:53.298 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:53 smithi110 ceph-mon[221102]: failed to encode map e330 with expected crc 2023-09-23T23:22:53.298 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:53 smithi110 ceph-mon[221102]: failed to encode map e330 with expected crc 2023-09-23T23:22:53.299 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:53 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:22:53.299 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:53 smithi110 ceph-mon[221102]: osdmap e330: 8 total, 7 up, 8 in 2023-09-23T23:22:53.299 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:53 smithi110 ceph-mon[221102]: failed to encode map e330 with expected crc 2023-09-23T23:22:53.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:53 smithi110 ceph-mon[212939]: pgmap v979: 109 pgs: 1 peering, 1 active+remapped+backfilling, 13 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 87 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 297 op/s; 1670/81495 objects degraded (2.049%); 9061/81495 objects misplaced (11.118%); 17 MiB/s, 278 objects/s recovering 2023-09-23T23:22:53.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:53 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:22:53.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:53 smithi110 ceph-mon[212939]: failed to encode map e330 with expected crc 2023-09-23T23:22:53.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:53 smithi110 ceph-mon[212939]: failed to encode map e330 with expected crc 2023-09-23T23:22:53.300 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:53 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:22:53.300 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:53 smithi110 ceph-mon[212939]: osdmap e330: 8 total, 7 up, 8 in 2023-09-23T23:22:53.300 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:53 smithi110 ceph-mon[212939]: failed to encode map e330 with expected crc 2023-09-23T23:22:53.465 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:22:53.490 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:22:53.512 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:53.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:53.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:53.525 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:22:53.621 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:53.648 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:53.691 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:53.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:53.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:53.711 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:22:53.711 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:22:53.711 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:22:53.711 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:22:53.712 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:22:53.712 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:22:53.712 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:22:53.712 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:22:53.712 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:22:53.712 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:22:53.712 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:22:53.712 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:22:53.794 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:53.865 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:54.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:54 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1670/81495 objects degraded (2.049%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:54.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:54 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1582324166' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:54.293 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:54.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:54.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:54.396 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:54.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:54 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1670/81495 objects degraded (2.049%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:54.447 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:54 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1582324166' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:54.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:54 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1670/81495 objects degraded (2.049%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:54.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:54 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1582324166' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:22:54.452 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:54.526 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:22:54.526 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:22:54.836 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:22:54.836 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:22:54.836 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:54.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:54.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:54.941 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:54.977 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:55.074 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:55 smithi110 ceph-mon[212939]: pgmap v981: 109 pgs: 1 peering, 1 active+remapped+backfilling, 13 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 87 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 1670/81495 objects degraded (2.049%); 9061/81495 objects misplaced (11.118%); 18 MiB/s, 284 objects/s recovering 2023-09-23T23:22:55.074 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:55 smithi110 ceph-mon[212939]: osdmap e331: 8 total, 7 up, 8 in 2023-09-23T23:22:55.074 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:55 smithi110 ceph-mon[212939]: failed to encode map e331 with expected crc 2023-09-23T23:22:55.127 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:55.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:55.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:55.232 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:55.262 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:55.310 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:55.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:55.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:55.326 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:22:55.378 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:55 smithi110 ceph-mon[221102]: pgmap v981: 109 pgs: 1 peering, 1 active+remapped+backfilling, 13 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 87 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 1670/81495 objects degraded (2.049%); 9061/81495 objects misplaced (11.118%); 18 MiB/s, 284 objects/s recovering 2023-09-23T23:22:55.379 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:55 smithi110 ceph-mon[221102]: osdmap e331: 8 total, 7 up, 8 in 2023-09-23T23:22:55.379 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:55 smithi110 ceph-mon[221102]: failed to encode map e331 with expected crc 2023-09-23T23:22:55.414 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:55.420 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:55 smithi138 ceph-mon[193606]: pgmap v981: 109 pgs: 1 peering, 1 active+remapped+backfilling, 13 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 87 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 1670/81495 objects degraded (2.049%); 9061/81495 objects misplaced (11.118%); 18 MiB/s, 284 objects/s recovering 2023-09-23T23:22:55.420 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:55 smithi138 ceph-mon[193606]: osdmap e331: 8 total, 7 up, 8 in 2023-09-23T23:22:55.420 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:55 smithi138 ceph-mon[193606]: failed to encode map e331 with expected crc 2023-09-23T23:22:55.441 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:22:55.441 INFO:teuthology.orchestra.run.smithi110.stderr:set pool 1 pg_num to 31 2023-09-23T23:22:55.450 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:55.499 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:55.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:55.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:55.607 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:22:55.702 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:56.254 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:22:56.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:56.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:22:56.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1492804315' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T23:22:56.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T23:22:56.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[212939]: failed to encode map e332 with expected crc 2023-09-23T23:22:56.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T23:22:56.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[212939]: osdmap e332: 8 total, 7 up, 8 in 2023-09-23T23:22:56.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[212939]: failed to encode map e332 with expected crc 2023-09-23T23:22:56.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[212939]: failed to encode map e332 with expected crc 2023-09-23T23:22:56.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[212939]: failed to encode map e332 with expected crc 2023-09-23T23:22:56.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T23:22:56.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T23:22:56.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1492804315' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T23:22:56.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T23:22:56.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[221102]: failed to encode map e332 with expected crc 2023-09-23T23:22:56.352 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T23:22:56.352 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[221102]: osdmap e332: 8 total, 7 up, 8 in 2023-09-23T23:22:56.352 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[221102]: failed to encode map e332 with expected crc 2023-09-23T23:22:56.352 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[221102]: failed to encode map e332 with expected crc 2023-09-23T23:22:56.352 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[221102]: failed to encode map e332 with expected crc 2023-09-23T23:22:56.352 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T23:22:56.352 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:56 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T23:22:56.366 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:56.393 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:56 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1492804315' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T23:22:56.394 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:56 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T23:22:56.394 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:56 smithi138 ceph-mon[193606]: failed to encode map e332 with expected crc 2023-09-23T23:22:56.394 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:56 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T23:22:56.394 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:56 smithi138 ceph-mon[193606]: osdmap e332: 8 total, 7 up, 8 in 2023-09-23T23:22:56.394 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:56 smithi138 ceph-mon[193606]: failed to encode map e332 with expected crc 2023-09-23T23:22:56.394 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:56 smithi138 ceph-mon[193606]: failed to encode map e332 with expected crc 2023-09-23T23:22:56.394 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:56 smithi138 ceph-mon[193606]: failed to encode map e332 with expected crc 2023-09-23T23:22:56.394 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:56 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T23:22:56.394 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:56 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T23:22:56.397 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:56.453 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:56.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:56.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:56.505 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:56.560 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:56.579 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:56.639 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:56.823 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:56.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:56.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:56.837 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:22:56.884 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:22:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:22:56] "GET /metrics HTTP/1.1" 200 36658 "" "Prometheus/2.43.0" 2023-09-23T23:22:56.933 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:56.998 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:57.142 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:57.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:57.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:57.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:57 smithi110 ceph-mon[212939]: pgmap v984: 109 pgs: 1 remapped+peering, 13 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 88 active+clean; 1.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 455 op/s; 1508/74478 objects degraded (2.025%); 5218/74478 objects misplaced (7.006%); 20 MiB/s, 318 objects/s recovering 2023-09-23T23:22:57.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:57 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T23:22:57.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:57 smithi110 ceph-mon[212939]: osdmap e333: 8 total, 7 up, 8 in 2023-09-23T23:22:57.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:57 smithi110 ceph-mon[212939]: failed to encode map e333 with expected crc 2023-09-23T23:22:57.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:57 smithi110 ceph-mon[212939]: failed to encode map e333 with expected crc 2023-09-23T23:22:57.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:57 smithi110 ceph-mon[212939]: failed to encode map e333 with expected crc 2023-09-23T23:22:57.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:57 smithi110 ceph-mon[212939]: failed to encode map e333 with expected crc 2023-09-23T23:22:57.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:57 smithi110 ceph-mon[212939]: failed to encode map e333 with expected crc 2023-09-23T23:22:57.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:57 smithi110 ceph-mon[221102]: pgmap v984: 109 pgs: 1 remapped+peering, 13 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 88 active+clean; 1.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 455 op/s; 1508/74478 objects degraded (2.025%); 5218/74478 objects misplaced (7.006%); 20 MiB/s, 318 objects/s recovering 2023-09-23T23:22:57.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:57 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T23:22:57.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:57 smithi110 ceph-mon[221102]: osdmap e333: 8 total, 7 up, 8 in 2023-09-23T23:22:57.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:57 smithi110 ceph-mon[221102]: failed to encode map e333 with expected crc 2023-09-23T23:22:57.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:57 smithi110 ceph-mon[221102]: failed to encode map e333 with expected crc 2023-09-23T23:22:57.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:57 smithi110 ceph-mon[221102]: failed to encode map e333 with expected crc 2023-09-23T23:22:57.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:57 smithi110 ceph-mon[221102]: failed to encode map e333 with expected crc 2023-09-23T23:22:57.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:57 smithi110 ceph-mon[221102]: failed to encode map e333 with expected crc 2023-09-23T23:22:57.252 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:57.279 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:57.326 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:57.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:57.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:57.435 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:22:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:57 smithi138 ceph-mon[193606]: pgmap v984: 109 pgs: 1 remapped+peering, 13 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 88 active+clean; 1.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 455 op/s; 1508/74478 objects degraded (2.025%); 5218/74478 objects misplaced (7.006%); 20 MiB/s, 318 objects/s recovering 2023-09-23T23:22:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:57 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T23:22:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:57 smithi138 ceph-mon[193606]: osdmap e333: 8 total, 7 up, 8 in 2023-09-23T23:22:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:57 smithi138 ceph-mon[193606]: failed to encode map e333 with expected crc 2023-09-23T23:22:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:57 smithi138 ceph-mon[193606]: failed to encode map e333 with expected crc 2023-09-23T23:22:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:57 smithi138 ceph-mon[193606]: failed to encode map e333 with expected crc 2023-09-23T23:22:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:57 smithi138 ceph-mon[193606]: failed to encode map e333 with expected crc 2023-09-23T23:22:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:57 smithi138 ceph-mon[193606]: failed to encode map e333 with expected crc 2023-09-23T23:22:57.475 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:57.518 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:22:57.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:57.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:22:57.624 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:22:57.651 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:58.309 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:22:58.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:58.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:22:58.413 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:22:58.441 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:58.493 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:22:58.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:58.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:22:58.598 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:58.646 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:58 smithi138 ceph-mon[193606]: osdmap e334: 8 total, 7 up, 8 in 2023-09-23T23:22:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:58 smithi138 ceph-mon[193606]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:58 smithi138 ceph-mon[193606]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:58 smithi138 ceph-mon[193606]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:58 smithi138 ceph-mon[193606]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:58 smithi138 ceph-mon[193606]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:58 smithi138 ceph-mon[193606]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:58 smithi138 ceph-mon[193606]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:58 smithi138 ceph-mon[193606]: pgmap v987: 119 pgs: 10 unknown, 1 remapped+peering, 13 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 88 active+clean; 1.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 1508/74478 objects degraded (2.025%); 5218/74478 objects misplaced (7.006%); 26 MiB/s, 408 objects/s recovering 2023-09-23T23:22:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:22:58.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:58 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1508/74478 objects degraded (2.025%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:58.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:58 smithi138 ceph-mon[193606]: osdmap e335: 8 total, 7 up, 8 in 2023-09-23T23:22:58.784 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[221102]: osdmap e334: 8 total, 7 up, 8 in 2023-09-23T23:22:58.784 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[221102]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.784 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[221102]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.784 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[221102]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.784 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[221102]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.784 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[221102]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.784 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[221102]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.785 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[221102]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.785 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[221102]: pgmap v987: 119 pgs: 10 unknown, 1 remapped+peering, 13 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 88 active+clean; 1.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 1508/74478 objects degraded (2.025%); 5218/74478 objects misplaced (7.006%); 26 MiB/s, 408 objects/s recovering 2023-09-23T23:22:58.785 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:58.785 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:22:58.785 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1508/74478 objects degraded (2.025%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:58.785 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[221102]: osdmap e335: 8 total, 7 up, 8 in 2023-09-23T23:22:58.785 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[212939]: osdmap e334: 8 total, 7 up, 8 in 2023-09-23T23:22:58.785 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[212939]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[212939]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[212939]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[212939]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[212939]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[212939]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[212939]: failed to encode map e334 with expected crc 2023-09-23T23:22:58.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[212939]: pgmap v987: 119 pgs: 10 unknown, 1 remapped+peering, 13 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 88 active+clean; 1.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 1508/74478 objects degraded (2.025%); 5218/74478 objects misplaced (7.006%); 26 MiB/s, 408 objects/s recovering 2023-09-23T23:22:58.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:22:58.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:22:58.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1508/74478 objects degraded (2.025%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:22:58.786 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:58 smithi110 ceph-mon[212939]: osdmap e335: 8 total, 7 up, 8 in 2023-09-23T23:22:58.803 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:58.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:58.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:58.876 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:22:58.916 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:58.962 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:59.320 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:22:59.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:59.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:22:59.425 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:22:59.500 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:59.611 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:22:59.611 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:22:59.634 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:59 smithi110 ceph-mon[221102]: failed to encode map e335 with expected crc 2023-09-23T23:22:59.634 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:59 smithi110 ceph-mon[221102]: failed to encode map e335 with expected crc 2023-09-23T23:22:59.634 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:22:59 smithi110 ceph-mon[221102]: osdmap e336: 8 total, 7 up, 8 in 2023-09-23T23:22:59.634 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:59 smithi110 ceph-mon[212939]: failed to encode map e335 with expected crc 2023-09-23T23:22:59.634 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:59 smithi110 ceph-mon[212939]: failed to encode map e335 with expected crc 2023-09-23T23:22:59.634 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:22:59 smithi110 ceph-mon[212939]: osdmap e336: 8 total, 7 up, 8 in 2023-09-23T23:22:59.635 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:22:59.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:22:59.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:22:59.739 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:22:59.785 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:22:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:59 smithi138 ceph-mon[193606]: failed to encode map e335 with expected crc 2023-09-23T23:22:59.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:59 smithi138 ceph-mon[193606]: failed to encode map e335 with expected crc 2023-09-23T23:22:59.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:22:59 smithi138 ceph-mon[193606]: osdmap e336: 8 total, 7 up, 8 in 2023-09-23T23:23:00.163 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:00.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:00.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:00.267 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:00.317 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:00.499 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:23:00.567 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:00 smithi110 ceph-mon[212939]: failed to encode map e336 with expected crc 2023-09-23T23:23:00.568 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:00 smithi110 ceph-mon[212939]: failed to encode map e336 with expected crc 2023-09-23T23:23:00.568 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:00 smithi110 ceph-mon[212939]: pgmap v990: 119 pgs: 5 unknown, 14 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 1332/70398 objects degraded (1.892%); 6668/70398 objects misplaced (9.472%) 2023-09-23T23:23:00.568 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:00 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2855725125' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:00.568 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:00 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:00.568 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:00 smithi110 ceph-mon[221102]: failed to encode map e336 with expected crc 2023-09-23T23:23:00.568 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:00 smithi110 ceph-mon[221102]: failed to encode map e336 with expected crc 2023-09-23T23:23:00.568 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:00 smithi110 ceph-mon[221102]: pgmap v990: 119 pgs: 5 unknown, 14 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 1332/70398 objects degraded (1.892%); 6668/70398 objects misplaced (9.472%) 2023-09-23T23:23:00.569 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:00 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2855725125' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:00.569 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:00 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:00.656 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:00.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:00.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:00.762 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:00.790 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:00 smithi138 ceph-mon[193606]: failed to encode map e336 with expected crc 2023-09-23T23:23:00.790 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:00 smithi138 ceph-mon[193606]: failed to encode map e336 with expected crc 2023-09-23T23:23:00.790 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:00 smithi138 ceph-mon[193606]: pgmap v990: 119 pgs: 5 unknown, 14 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 93 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 1332/70398 objects degraded (1.892%); 6668/70398 objects misplaced (9.472%) 2023-09-23T23:23:00.791 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:00 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2855725125' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:00.791 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:00 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:00.793 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:00.840 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:23:00.843 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:00.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:00.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:00.948 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:00.976 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:01.353 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 2, 1] out_osds: [] dead_osds: [7] live_osds: [0, 1, 3, 4, 5, 6, 2] 2023-09-23T23:23:01.354 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:23:01.354 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T23:23:01.354 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:23:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:01 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:23:01.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:01 smithi110 ceph-mon[212939]: failed to encode map e337 with expected crc 2023-09-23T23:23:01.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:01 smithi110 ceph-mon[212939]: failed to encode map e337 with expected crc 2023-09-23T23:23:01.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:01 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:23:01.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:01 smithi110 ceph-mon[212939]: osdmap e337: 8 total, 7 up, 8 in 2023-09-23T23:23:01.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:01 smithi110 ceph-mon[212939]: failed to encode map e337 with expected crc 2023-09-23T23:23:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:01 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:23:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:01 smithi110 ceph-mon[221102]: failed to encode map e337 with expected crc 2023-09-23T23:23:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:01 smithi110 ceph-mon[221102]: failed to encode map e337 with expected crc 2023-09-23T23:23:01.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:01 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:23:01.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:01 smithi110 ceph-mon[221102]: osdmap e337: 8 total, 7 up, 8 in 2023-09-23T23:23:01.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:01 smithi110 ceph-mon[221102]: failed to encode map e337 with expected crc 2023-09-23T23:23:01.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:01 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:23:01.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:01 smithi138 ceph-mon[193606]: failed to encode map e337 with expected crc 2023-09-23T23:23:01.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:01 smithi138 ceph-mon[193606]: failed to encode map e337 with expected crc 2023-09-23T23:23:01.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:01 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:23:01.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:01 smithi138 ceph-mon[193606]: osdmap e337: 8 total, 7 up, 8 in 2023-09-23T23:23:01.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:01 smithi138 ceph-mon[193606]: failed to encode map e337 with expected crc 2023-09-23T23:23:02.107 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:02.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:02.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:02.212 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:23:02.275 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:02.361 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:02.390 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:02.432 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:23:02.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:02.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:23:02.537 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:02.595 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:02.752 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:02.763 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:02 smithi110 ceph-mon[221102]: pgmap v992: 119 pgs: 1 unknown, 14 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 1332/66684 objects degraded (1.997%); 5945/66684 objects misplaced (8.915%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T23:23:02.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:02.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:02.765 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:02 smithi110 ceph-mon[212939]: pgmap v992: 119 pgs: 1 unknown, 14 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 1332/66684 objects degraded (1.997%); 5945/66684 objects misplaced (8.915%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T23:23:02.865 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:23:02.889 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:02 smithi138 ceph-mon[193606]: pgmap v992: 119 pgs: 1 unknown, 14 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 1332/66684 objects degraded (1.997%); 5945/66684 objects misplaced (8.915%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T23:23:02.891 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:02.941 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:23:02.942 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:23:02.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:02.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:23:02.968 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:23:02.996 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:23:03.035 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:23:03.048 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:03.054 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:23:03.085 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:03.087 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:23:03.113 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:23:03.113 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:23:03.113 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:23:03.113 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:23:03.114 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:23:03.114 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:23:03.114 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:23:03.114 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:23:03.114 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:23:03.140 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:03.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:03.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:03.247 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:23:03.281 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:03.328 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:23:03.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:03.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:23:03.434 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:23:03.461 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:03.513 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:23:03.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:03.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:23:03.620 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:23:03.669 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:03.932 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:23:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:03 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1464786809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:03 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1332/66684 objects degraded (1.997%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:03.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:03 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1464786809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:03.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:03 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1332/66684 objects degraded (1.997%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:03.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:03 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1464786809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:03.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:03 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1332/66684 objects degraded (1.997%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:04.918 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:04 smithi110 ceph-mon[212939]: pgmap v993: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 1.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 1332/66684 objects degraded (1.997%); 5945/66684 objects misplaced (8.915%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:23:04.918 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:04 smithi110 ceph-mon[221102]: pgmap v993: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 1.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 1332/66684 objects degraded (1.997%); 5945/66684 objects misplaced (8.915%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:23:04.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:04 smithi138 ceph-mon[193606]: pgmap v993: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 1.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 1332/66684 objects degraded (1.997%); 5945/66684 objects misplaced (8.915%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:23:04.944 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:23:04.945 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:23:05.171 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:23:05.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:05.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:23:05.277 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:23:05.306 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:05.358 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:23:05.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:05.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:23:05.469 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:05.516 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:05.531 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:05.717 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:05.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:05.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:05.823 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:05.856 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:05.911 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:05.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:05.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:06.002 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:23:06.217 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:23:06.217 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:23:06.217 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:23:06.217 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:23:06.217 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:23:06.250 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:06.307 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:23:06.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:06.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:23:06.412 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:06.477 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:06.555 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:23:06.575 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:23:06.624 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:06 smithi138 ceph-mon[193606]: pgmap v994: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 1.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 1146/57621 objects degraded (1.989%); 4970/57621 objects misplaced (8.625%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T23:23:06.945 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:23:06.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:06 smithi110 ceph-mon[212939]: pgmap v994: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 1.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 1146/57621 objects degraded (1.989%); 4970/57621 objects misplaced (8.625%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T23:23:06.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:23:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:23:06] "GET /metrics HTTP/1.1" 200 36646 "" "Prometheus/2.43.0" 2023-09-23T23:23:06.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:06 smithi110 ceph-mon[221102]: pgmap v994: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 1.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 1146/57621 objects degraded (1.989%); 4970/57621 objects misplaced (8.625%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T23:23:07.731 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:07.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:07.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:07.835 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:07.879 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:07.880 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:07 smithi138 ceph-mon[193606]: from='client.35823 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:23:07.889 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:07 smithi110 ceph-mon[212939]: from='client.35823 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:23:07.890 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:07 smithi110 ceph-mon[221102]: from='client.35823 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:23:07.929 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:23:07.940 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:07.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:07.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:08.048 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:08.083 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:08.519 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":994,"stamp":"2023-09-23T23:23:05.566034+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1257802922,"num_objects":19207,"num_object_clones":0,"num_object_copies":57621,"num_objects_missing_on_primary":3558,"num_objects_missing":3558,"num_objects_degraded":1146,"num_objects_misplaced":4970,"num_objects_unfound":0,"num_objects_dirty":19207,"num_whiteouts":0,"num_read":135,"num_read_kb":251,"num_write":183356,"num_write_kb":6477809,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27831,"num_bytes_recovered":1772517431,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85116,"ondisk_log_size":85116,"up":334,"acting":351,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":382,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8994588,"kb_used_data":5070428,"kb_used_omap":20,"kb_used_meta":3924075,"kb_avail":740966628,"statfs":{"total":767960285184,"available":758749827072,"internally_reserved":0,"allocated":5192118272,"data_stored":5182384122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4018253454},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,3,2,0,0,8,3,3],"upper_bound":1024},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-368246784,"num_objects":-5619,"num_object_clones":0,"num_object_copies":-16857,"num_objects_missing_on_primary":-352,"num_objects_missing":-352,"num_objects_degraded":-362,"num_objects_misplaced":-248,"num_objects_unfound":0,"num_objects_dirty":-5619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5619,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":346,"num_bytes_recovered":22675456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.146896"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":569,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:23:00.158271+0000","last_change":"2023-09-23T23:22:57.471044+0000","last_active":"2023-09-23T23:23:00.158271+0000","last_peered":"2023-09-23T23:23:00.158271+0000","last_clean":"2023-09-23T23:23:00.158271+0000","last_became_active":"2023-09-23T23:22:57.470772+0000","last_became_peered":"2023-09-23T23:22:57.470772+0000","last_unstale":"2023-09-23T23:23:00.158271+0000","last_undegraded":"2023-09-23T23:23:00.158271+0000","last_fullsized":"2023-09-23T23:23:00.158271+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:42.871134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:42.871134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023428590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":131344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":372,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.429833+0000","last_change":"2023-09-23T23:22:38.439629+0000","last_active":"2023-09-23T23:22:59.429833+0000","last_peered":"2023-09-23T23:22:59.429833+0000","last_clean":"2023-09-23T23:22:59.429833+0000","last_became_active":"2023-09-23T23:22:38.439285+0000","last_became_peered":"2023-09-23T23:22:38.439285+0000","last_unstale":"2023-09-23T23:22:59.429833+0000","last_undegraded":"2023-09-23T23:22:59.429833+0000","last_fullsized":"2023-09-23T23:22:59.429833+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:34.431221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":296,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.432577+0000","last_change":"2023-09-23T23:22:38.436372+0000","last_active":"2023-09-23T23:22:59.432577+0000","last_peered":"2023-09-23T23:22:59.432577+0000","last_clean":"2023-09-23T23:22:59.432577+0000","last_became_active":"2023-09-23T23:22:38.436054+0000","last_became_peered":"2023-09-23T23:22:38.436054+0000","last_unstale":"2023-09-23T23:22:59.432577+0000","last_undegraded":"2023-09-23T23:22:59.432577+0000","last_fullsized":"2023-09-23T23:22:59.432577+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:36:41.158712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":443,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.438520+0000","last_change":"2023-09-23T23:22:57.775559+0000","last_active":"2023-09-23T23:22:59.438520+0000","last_peered":"2023-09-23T23:22:59.438520+0000","last_clean":"2023-09-23T23:22:59.438520+0000","last_became_active":"2023-09-23T23:22:57.775215+0000","last_became_peered":"2023-09-23T23:22:57.775215+0000","last_unstale":"2023-09-23T23:22:59.438520+0000","last_undegraded":"2023-09-23T23:22:59.438520+0000","last_fullsized":"2023-09-23T23:22:59.438520+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:19:31.012484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:19:31.012484+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":27,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.439403+0000","last_change":"2023-09-23T23:22:38.436383+0000","last_active":"2023-09-23T23:22:59.439403+0000","last_peered":"2023-09-23T23:22:59.439403+0000","last_clean":"2023-09-23T23:22:59.439403+0000","last_became_active":"2023-09-23T23:22:38.436062+0000","last_became_peered":"2023-09-23T23:22:38.436062+0000","last_unstale":"2023-09-23T23:22:59.439403+0000","last_undegraded":"2023-09-23T23:22:59.439403+0000","last_fullsized":"2023-09-23T23:22:59.439403+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:07.142420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":336,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.430914+0000","last_change":"2023-09-23T23:22:38.438239+0000","last_active":"2023-09-23T23:22:59.430914+0000","last_peered":"2023-09-23T23:22:59.430914+0000","last_clean":"2023-09-23T23:22:59.430914+0000","last_became_active":"2023-09-23T23:22:38.437359+0000","last_became_peered":"2023-09-23T23:22:38.437359+0000","last_unstale":"2023-09-23T23:22:59.430914+0000","last_undegraded":"2023-09-23T23:22:59.430914+0000","last_fullsized":"2023-09-23T23:22:59.430914+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:15:40.886267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":626,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:23:00.158340+0000","last_change":"2023-09-23T23:22:57.474856+0000","last_active":"2023-09-23T23:23:00.158340+0000","last_peered":"2023-09-23T23:23:00.158340+0000","last_clean":"2023-09-23T23:23:00.158340+0000","last_became_active":"2023-09-23T23:22:57.474442+0000","last_became_peered":"2023-09-23T23:22:57.474442+0000","last_unstale":"2023-09-23T23:23:00.158340+0000","last_undegraded":"2023-09-23T23:23:00.158340+0000","last_fullsized":"2023-09-23T23:23:00.158340+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:51.734652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:51.734652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.023100959000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":324,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T23:23:00.485122+0000","last_change":"2023-09-23T23:20:15.738434+0000","last_active":"2023-09-23T23:23:00.485122+0000","last_peered":"2023-09-23T23:23:00.485122+0000","last_clean":"2023-09-23T23:23:00.485122+0000","last_became_active":"2023-09-23T23:20:15.738195+0000","last_became_peered":"2023-09-23T23:20:15.738195+0000","last_unstale":"2023-09-23T23:23:00.485122+0000","last_undegraded":"2023-09-23T23:23:00.485122+0000","last_fullsized":"2023-09-23T23:23:00.485122+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:15.224045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":317,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.434176+0000","last_change":"2023-09-23T23:22:38.436482+0000","last_active":"2023-09-23T23:22:59.434176+0000","last_peered":"2023-09-23T23:22:59.434176+0000","last_clean":"2023-09-23T23:22:59.434176+0000","last_became_active":"2023-09-23T23:22:38.436147+0000","last_became_peered":"2023-09-23T23:22:38.436147+0000","last_unstale":"2023-09-23T23:22:59.434176+0000","last_undegraded":"2023-09-23T23:22:59.434176+0000","last_fullsized":"2023-09-23T23:22:59.434176+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:30.542674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":637,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:23:00.173717+0000","last_change":"2023-09-23T23:22:57.469609+0000","last_active":"2023-09-23T23:23:00.173717+0000","last_peered":"2023-09-23T23:23:00.173717+0000","last_clean":"2023-09-23T23:23:00.173717+0000","last_became_active":"2023-09-23T23:22:57.469240+0000","last_became_peered":"2023-09-23T23:22:57.469240+0000","last_unstale":"2023-09-23T23:23:00.173717+0000","last_undegraded":"2023-09-23T23:23:00.173717+0000","last_fullsized":"2023-09-23T23:23:00.173717+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:51.489074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:51.489074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00519324,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":289,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.431620+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:22:59.431620+0000","last_peered":"2023-09-23T23:22:59.431620+0000","last_clean":"2023-09-23T23:22:59.431620+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:22:59.431620+0000","last_undegraded":"2023-09-23T23:22:59.431620+0000","last_fullsized":"2023-09-23T23:22:59.431620+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":306,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.430309+0000","last_change":"2023-09-23T23:22:38.450254+0000","last_active":"2023-09-23T23:22:59.430309+0000","last_peered":"2023-09-23T23:22:59.430309+0000","last_clean":"2023-09-23T23:22:59.430309+0000","last_became_active":"2023-09-23T23:22:38.449879+0000","last_became_peered":"2023-09-23T23:22:38.449879+0000","last_unstale":"2023-09-23T23:22:59.430309+0000","last_undegraded":"2023-09-23T23:22:59.430309+0000","last_fullsized":"2023-09-23T23:22:59.430309+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:02.777916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":636,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.431890+0000","last_change":"2023-09-23T23:22:38.429461+0000","last_active":"2023-09-23T23:22:59.431890+0000","last_peered":"2023-09-23T23:22:59.431890+0000","last_clean":"2023-09-23T23:22:59.431890+0000","last_became_active":"2023-09-23T23:22:17.011652+0000","last_became_peered":"2023-09-23T23:22:17.011652+0000","last_unstale":"2023-09-23T23:22:59.431890+0000","last_undegraded":"2023-09-23T23:22:59.431890+0000","last_fullsized":"2023-09-23T23:22:59.431890+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.522632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.522632+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.018152603999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":78,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.428311+0000","last_change":"2023-09-23T23:20:26.841909+0000","last_active":"2023-09-23T23:22:59.428311+0000","last_peered":"2023-09-23T23:22:59.428311+0000","last_clean":"2023-09-23T23:22:59.428311+0000","last_became_active":"2023-09-23T23:20:26.841631+0000","last_became_peered":"2023-09-23T23:20:26.841631+0000","last_unstale":"2023-09-23T23:22:59.428311+0000","last_undegraded":"2023-09-23T23:22:59.428311+0000","last_fullsized":"2023-09-23T23:22:59.428311+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:07.380539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":267,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.430356+0000","last_change":"2023-09-23T23:16:56.528057+0000","last_active":"2023-09-23T23:22:59.430356+0000","last_peered":"2023-09-23T23:22:59.430356+0000","last_clean":"2023-09-23T23:22:59.430356+0000","last_became_active":"2023-09-23T23:16:56.527755+0000","last_became_peered":"2023-09-23T23:16:56.527755+0000","last_unstale":"2023-09-23T23:22:59.430356+0000","last_undegraded":"2023-09-23T23:22:59.430356+0000","last_fullsized":"2023-09-23T23:22:59.430356+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:47.396406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":732,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.434333+0000","last_change":"2023-09-23T23:22:57.471100+0000","last_active":"2023-09-23T23:22:59.434333+0000","last_peered":"2023-09-23T23:22:59.434333+0000","last_clean":"2023-09-23T23:22:59.434333+0000","last_became_active":"2023-09-23T23:22:57.470779+0000","last_became_peered":"2023-09-23T23:22:57.470779+0000","last_unstale":"2023-09-23T23:22:59.434333+0000","last_undegraded":"2023-09-23T23:22:59.434333+0000","last_fullsized":"2023-09-23T23:22:59.434333+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:47.797085+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:47.797085+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":155,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.432895+0000","last_change":"2023-09-23T23:16:56.567128+0000","last_active":"2023-09-23T23:22:59.432895+0000","last_peered":"2023-09-23T23:22:59.432895+0000","last_clean":"2023-09-23T23:22:59.432895+0000","last_became_active":"2023-09-23T23:16:56.551484+0000","last_became_peered":"2023-09-23T23:16:56.551484+0000","last_unstale":"2023-09-23T23:22:59.432895+0000","last_undegraded":"2023-09-23T23:22:59.432895+0000","last_fullsized":"2023-09-23T23:22:59.432895+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:14.981307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":287,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.434023+0000","last_change":"2023-09-23T23:20:15.781637+0000","last_active":"2023-09-23T23:22:59.434023+0000","last_peered":"2023-09-23T23:22:59.434023+0000","last_clean":"2023-09-23T23:22:59.434023+0000","last_became_active":"2023-09-23T23:20:15.781308+0000","last_became_peered":"2023-09-23T23:20:15.781308+0000","last_unstale":"2023-09-23T23:22:59.434023+0000","last_undegraded":"2023-09-23T23:22:59.434023+0000","last_fullsized":"2023-09-23T23:22:59.434023+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:47.963872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":629,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.433706+0000","last_change":"2023-09-23T23:22:57.765391+0000","last_active":"2023-09-23T23:22:59.433706+0000","last_peered":"2023-09-23T23:22:59.433706+0000","last_clean":"2023-09-23T23:22:59.433706+0000","last_became_active":"2023-09-23T23:22:57.765098+0000","last_became_peered":"2023-09-23T23:22:57.765098+0000","last_unstale":"2023-09-23T23:22:59.433706+0000","last_undegraded":"2023-09-23T23:22:59.433706+0000","last_fullsized":"2023-09-23T23:22:59.433706+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.802642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.802642+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0056570290000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":107,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:59.433621+0000","last_change":"2023-09-23T23:21:51.058999+0000","last_active":"2023-09-23T23:22:59.433621+0000","last_peered":"2023-09-23T23:22:59.433621+0000","last_clean":"2023-09-23T23:22:59.433621+0000","last_became_active":"2023-09-23T23:21:51.044013+0000","last_became_peered":"2023-09-23T23:21:51.044013+0000","last_unstale":"2023-09-23T23:22:59.433621+0000","last_undegraded":"2023-09-23T23:22:59.433621+0000","last_fullsized":"2023-09-23T23:22:59.433621+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:47.100334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2,4],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":316,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.430232+0000","last_change":"2023-09-23T23:22:38.443826+0000","last_active":"2023-09-23T23:22:59.430232+0000","last_peered":"2023-09-23T23:22:59.430232+0000","last_clean":"2023-09-23T23:22:59.430232+0000","last_became_active":"2023-09-23T23:22:38.443494+0000","last_became_peered":"2023-09-23T23:22:38.443494+0000","last_unstale":"2023-09-23T23:22:59.430232+0000","last_undegraded":"2023-09-23T23:22:59.430232+0000","last_fullsized":"2023-09-23T23:22:59.430232+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:55.630019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":469,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.433983+0000","last_change":"2023-09-23T23:22:17.008359+0000","last_active":"2023-09-23T23:22:59.433983+0000","last_peered":"2023-09-23T23:22:59.433983+0000","last_clean":"2023-09-23T23:22:59.433983+0000","last_became_active":"2023-09-23T23:22:17.008045+0000","last_became_peered":"2023-09-23T23:22:17.008045+0000","last_unstale":"2023-09-23T23:22:59.433983+0000","last_undegraded":"2023-09-23T23:22:59.433983+0000","last_fullsized":"2023-09-23T23:22:59.433983+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:49.726740+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:49.726740+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":95,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:59.440356+0000","last_change":"2023-09-23T23:21:51.055871+0000","last_active":"2023-09-23T23:22:59.440356+0000","last_peered":"2023-09-23T23:22:59.440356+0000","last_clean":"2023-09-23T23:22:59.440356+0000","last_became_active":"2023-09-23T23:21:51.055372+0000","last_became_peered":"2023-09-23T23:21:51.055372+0000","last_unstale":"2023-09-23T23:22:59.440356+0000","last_undegraded":"2023-09-23T23:22:59.440356+0000","last_fullsized":"2023-09-23T23:22:59.440356+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:11.929868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":313,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:59.432195+0000","last_change":"2023-09-23T23:20:16.730841+0000","last_active":"2023-09-23T23:22:59.432195+0000","last_peered":"2023-09-23T23:22:59.432195+0000","last_clean":"2023-09-23T23:22:59.432195+0000","last_became_active":"2023-09-23T23:20:16.730440+0000","last_became_peered":"2023-09-23T23:20:16.730440+0000","last_unstale":"2023-09-23T23:22:59.432195+0000","last_undegraded":"2023-09-23T23:22:59.432195+0000","last_fullsized":"2023-09-23T23:22:59.432195+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:12.839390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,5],"avail_no_missing":["3","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":617,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.431485+0000","last_change":"2023-09-23T23:22:41.462393+0000","last_active":"2023-09-23T23:22:59.431485+0000","last_peered":"2023-09-23T23:22:59.431485+0000","last_clean":"2023-09-23T23:22:59.431485+0000","last_became_active":"2023-09-23T23:22:41.462082+0000","last_became_peered":"2023-09-23T23:22:41.462082+0000","last_unstale":"2023-09-23T23:22:59.431485+0000","last_undegraded":"2023-09-23T23:22:59.431485+0000","last_fullsized":"2023-09-23T23:22:59.431485+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:42.190868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:42.190868+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0049256480000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":69,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.425940+0000","last_change":"2023-09-23T23:20:15.781738+0000","last_active":"2023-09-23T23:22:59.425940+0000","last_peered":"2023-09-23T23:22:59.425940+0000","last_clean":"2023-09-23T23:22:59.425940+0000","last_became_active":"2023-09-23T23:20:15.781394+0000","last_became_peered":"2023-09-23T23:20:15.781394+0000","last_unstale":"2023-09-23T23:22:59.425940+0000","last_undegraded":"2023-09-23T23:22:59.425940+0000","last_fullsized":"2023-09-23T23:22:59.425940+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:04.696118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":194,"reported_epoch":336,"state":"active+undersized","last_fresh":"2023-09-23T23:22:59.432660+0000","last_change":"2023-09-23T23:18:59.586926+0000","last_active":"2023-09-23T23:22:59.432660+0000","last_peered":"2023-09-23T23:22:59.432660+0000","last_clean":"2023-09-23T23:10:34.157578+0000","last_became_active":"2023-09-23T23:18:59.586926+0000","last_became_peered":"2023-09-23T23:18:59.586926+0000","last_unstale":"2023-09-23T23:22:59.432660+0000","last_undegraded":"2023-09-23T23:22:59.432660+0000","last_fullsized":"2023-09-23T23:18:59.579446+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:29.301032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":434,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.438852+0000","last_change":"2023-09-23T23:22:41.468429+0000","last_active":"2023-09-23T23:22:59.438852+0000","last_peered":"2023-09-23T23:22:59.438852+0000","last_clean":"2023-09-23T23:22:59.438852+0000","last_became_active":"2023-09-23T23:22:41.467838+0000","last_became_peered":"2023-09-23T23:22:41.467838+0000","last_unstale":"2023-09-23T23:22:59.438852+0000","last_undegraded":"2023-09-23T23:22:59.438852+0000","last_fullsized":"2023-09-23T23:22:59.438852+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:19:31.012484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:19:31.012484+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":90,"reported_epoch":336,"state":"active+undersized","last_fresh":"2023-09-23T23:22:59.429801+0000","last_change":"2023-09-23T23:18:59.587971+0000","last_active":"2023-09-23T23:22:59.429801+0000","last_peered":"2023-09-23T23:22:59.429801+0000","last_clean":"2023-09-23T22:53:15.863481+0000","last_became_active":"2023-09-23T23:18:59.587971+0000","last_became_peered":"2023-09-23T23:18:59.587971+0000","last_unstale":"2023-09-23T23:22:59.429801+0000","last_undegraded":"2023-09-23T23:22:59.429801+0000","last_fullsized":"2023-09-23T23:18:59.584213+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:17.017106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":316,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.431074+0000","last_change":"2023-09-23T23:20:15.737685+0000","last_active":"2023-09-23T23:22:59.431074+0000","last_peered":"2023-09-23T23:22:59.431074+0000","last_clean":"2023-09-23T23:22:59.431074+0000","last_became_active":"2023-09-23T23:20:15.737311+0000","last_became_peered":"2023-09-23T23:20:15.737311+0000","last_unstale":"2023-09-23T23:22:59.431074+0000","last_undegraded":"2023-09-23T23:22:59.431074+0000","last_fullsized":"2023-09-23T23:22:59.431074+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:15:21.213270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":659,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.431792+0000","last_change":"2023-09-23T23:22:17.006817+0000","last_active":"2023-09-23T23:22:59.431792+0000","last_peered":"2023-09-23T23:22:59.431792+0000","last_clean":"2023-09-23T23:22:59.431792+0000","last_became_active":"2023-09-23T23:22:17.006326+0000","last_became_peered":"2023-09-23T23:22:17.006326+0000","last_unstale":"2023-09-23T23:22:59.431792+0000","last_undegraded":"2023-09-23T23:22:59.431792+0000","last_fullsized":"2023-09-23T23:22:59.431792+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:47.277080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:47.277080+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001458104,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":326,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.434853+0000","last_change":"2023-09-23T23:13:29.368175+0000","last_active":"2023-09-23T23:22:59.434853+0000","last_peered":"2023-09-23T23:22:59.434853+0000","last_clean":"2023-09-23T23:22:59.434853+0000","last_became_active":"2023-09-23T23:13:29.363603+0000","last_became_peered":"2023-09-23T23:13:29.363603+0000","last_unstale":"2023-09-23T23:22:59.434853+0000","last_undegraded":"2023-09-23T23:22:59.434853+0000","last_fullsized":"2023-09-23T23:22:59.434853+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:25.277871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":27,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.438363+0000","last_change":"2023-09-23T23:22:38.452672+0000","last_active":"2023-09-23T23:22:59.438363+0000","last_peered":"2023-09-23T23:22:59.438363+0000","last_clean":"2023-09-23T23:22:59.438363+0000","last_became_active":"2023-09-23T23:22:38.452374+0000","last_became_peered":"2023-09-23T23:22:38.452374+0000","last_unstale":"2023-09-23T23:22:59.438363+0000","last_undegraded":"2023-09-23T23:22:59.438363+0000","last_fullsized":"2023-09-23T23:22:59.438363+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:29.874430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":637,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.427926+0000","last_change":"2023-09-23T23:22:57.466003+0000","last_active":"2023-09-23T23:22:59.427926+0000","last_peered":"2023-09-23T23:22:59.427926+0000","last_clean":"2023-09-23T23:22:59.427926+0000","last_became_active":"2023-09-23T23:22:57.465520+0000","last_became_peered":"2023-09-23T23:22:57.465520+0000","last_unstale":"2023-09-23T23:22:59.427926+0000","last_undegraded":"2023-09-23T23:22:59.427926+0000","last_fullsized":"2023-09-23T23:22:59.427926+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:51.489074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:51.489074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00519324,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.6","version":"337'11480","reported_seq":11578,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T23:23:05.112421+0000","last_change":"2023-09-23T23:21:22.237069+0000","last_active":"2023-09-23T23:23:05.112421+0000","last_peered":"2023-09-23T23:23:05.112421+0000","last_clean":"2023-09-23T23:23:05.112421+0000","last_became_active":"2023-09-23T23:21:22.194375+0000","last_became_peered":"2023-09-23T23:21:22.194375+0000","last_unstale":"2023-09-23T23:23:05.112421+0000","last_undegraded":"2023-09-23T23:23:05.112421+0000","last_fullsized":"2023-09-23T23:23:05.112421+0000","mapping_epoch":295,"log_start":"291'6561","ondisk_log_start":"291'6561","created":255,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4919,"log_dups_size":0,"ondisk_log_size":4919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:56:59.831180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76152832,"num_objects":1162,"num_object_clones":0,"num_object_copies":3486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11480,"num_write_kb":404544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":151,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.430567+0000","last_change":"2023-09-23T23:16:56.548535+0000","last_active":"2023-09-23T23:22:59.430567+0000","last_peered":"2023-09-23T23:22:59.430567+0000","last_clean":"2023-09-23T23:22:59.430567+0000","last_became_active":"2023-09-23T23:16:56.546115+0000","last_became_peered":"2023-09-23T23:16:56.546115+0000","last_unstale":"2023-09-23T23:22:59.430567+0000","last_undegraded":"2023-09-23T23:22:59.430567+0000","last_fullsized":"2023-09-23T23:22:59.430567+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:02.279715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":142,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.434917+0000","last_change":"2023-09-23T23:16:56.525479+0000","last_active":"2023-09-23T23:22:59.434917+0000","last_peered":"2023-09-23T23:22:59.434917+0000","last_clean":"2023-09-23T23:22:59.434917+0000","last_became_active":"2023-09-23T23:16:56.525147+0000","last_became_peered":"2023-09-23T23:16:56.525147+0000","last_unstale":"2023-09-23T23:22:59.434917+0000","last_undegraded":"2023-09-23T23:22:59.434917+0000","last_fullsized":"2023-09-23T23:22:59.434917+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:06.165300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":647,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.431976+0000","last_change":"2023-09-23T23:22:57.468857+0000","last_active":"2023-09-23T23:22:59.431976+0000","last_peered":"2023-09-23T23:22:59.431976+0000","last_clean":"2023-09-23T23:22:59.431976+0000","last_became_active":"2023-09-23T23:22:57.468451+0000","last_became_peered":"2023-09-23T23:22:57.468451+0000","last_unstale":"2023-09-23T23:22:59.431976+0000","last_undegraded":"2023-09-23T23:22:59.431976+0000","last_fullsized":"2023-09-23T23:22:59.431976+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.522632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.522632+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.018152603999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54613,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":109227,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.7","version":"337'11304","reported_seq":15795,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T23:23:04.470300+0000","last_change":"2023-09-23T23:22:31.086081+0000","last_active":"2023-09-23T23:23:04.470300+0000","last_peered":"2023-09-23T23:23:04.470300+0000","last_clean":"2023-09-23T23:23:04.470300+0000","last_became_active":"2023-09-23T23:22:31.023167+0000","last_became_peered":"2023-09-23T23:22:31.023167+0000","last_unstale":"2023-09-23T23:23:04.470300+0000","last_undegraded":"2023-09-23T23:23:04.470300+0000","last_fullsized":"2023-09-23T23:23:04.470300+0000","mapping_epoch":316,"log_start":"292'6820","ondisk_log_start":"292'6820","created":255,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4484,"log_dups_size":0,"ondisk_log_size":4484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:35.081883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80216064,"num_objects":1224,"num_object_clones":0,"num_object_copies":3672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11304,"num_write_kb":400896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2184,"num_bytes_recovered":143065088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":31,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.439679+0000","last_change":"2023-09-23T23:22:38.474057+0000","last_active":"2023-09-23T23:22:59.439679+0000","last_peered":"2023-09-23T23:22:59.439679+0000","last_clean":"2023-09-23T23:22:59.439679+0000","last_became_active":"2023-09-23T23:22:38.444672+0000","last_became_peered":"2023-09-23T23:22:38.444672+0000","last_unstale":"2023-09-23T23:22:59.439679+0000","last_undegraded":"2023-09-23T23:22:59.439679+0000","last_fullsized":"2023-09-23T23:22:59.439679+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:00.966999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":343,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.429847+0000","last_change":"2023-09-23T23:22:38.439077+0000","last_active":"2023-09-23T23:22:59.429847+0000","last_peered":"2023-09-23T23:22:59.429847+0000","last_clean":"2023-09-23T23:22:59.429847+0000","last_became_active":"2023-09-23T23:22:38.438638+0000","last_became_peered":"2023-09-23T23:22:38.438638+0000","last_unstale":"2023-09-23T23:22:59.429847+0000","last_undegraded":"2023-09-23T23:22:59.429847+0000","last_fullsized":"2023-09-23T23:22:59.429847+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:57.067702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":719,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.434014+0000","last_change":"2023-09-23T23:22:17.012087+0000","last_active":"2023-09-23T23:22:59.434014+0000","last_peered":"2023-09-23T23:22:59.434014+0000","last_clean":"2023-09-23T23:22:59.434014+0000","last_became_active":"2023-09-23T23:22:17.011761+0000","last_became_peered":"2023-09-23T23:22:17.011761+0000","last_unstale":"2023-09-23T23:22:59.434014+0000","last_undegraded":"2023-09-23T23:22:59.434014+0000","last_fullsized":"2023-09-23T23:22:59.434014+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:47.797085+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:47.797085+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.4","version":"337'11369","reported_seq":11461,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T23:23:05.039216+0000","last_change":"2023-09-23T23:18:58.585739+0000","last_active":"2023-09-23T23:23:05.039216+0000","last_peered":"2023-09-23T23:23:05.039216+0000","last_clean":"2023-09-23T23:23:05.039216+0000","last_became_active":"2023-09-23T23:18:58.585475+0000","last_became_peered":"2023-09-23T23:18:58.585475+0000","last_unstale":"2023-09-23T23:23:05.039216+0000","last_undegraded":"2023-09-23T23:23:05.039216+0000","last_fullsized":"2023-09-23T23:23:05.039216+0000","mapping_epoch":255,"log_start":"291'6382","ondisk_log_start":"291'6382","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4987,"log_dups_size":0,"ondisk_log_size":4987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:14.419790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74252288,"num_objects":1133,"num_object_clones":0,"num_object_copies":3399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11369,"num_write_kb":400064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":101,"reported_epoch":336,"state":"active+undersized+degraded","last_fresh":"2023-09-23T23:22:59.434434+0000","last_change":"2023-09-23T23:18:59.587709+0000","last_active":"2023-09-23T23:22:59.434434+0000","last_peered":"2023-09-23T23:22:59.434434+0000","last_clean":"2023-09-23T22:53:20.393182+0000","last_became_active":"2023-09-23T23:18:59.587709+0000","last_became_peered":"2023-09-23T23:18:59.587709+0000","last_unstale":"2023-09-23T23:22:59.434434+0000","last_undegraded":"2023-09-23T23:18:59.585429+0000","last_fullsized":"2023-09-23T23:18:59.585311+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:58.759914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":364,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.434401+0000","last_change":"2023-09-23T23:22:38.439331+0000","last_active":"2023-09-23T23:22:59.434401+0000","last_peered":"2023-09-23T23:22:59.434401+0000","last_clean":"2023-09-23T23:22:59.434401+0000","last_became_active":"2023-09-23T23:22:38.436293+0000","last_became_peered":"2023-09-23T23:22:38.436293+0000","last_unstale":"2023-09-23T23:22:59.434401+0000","last_undegraded":"2023-09-23T23:22:59.434401+0000","last_fullsized":"2023-09-23T23:22:59.434401+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:53.668614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":629,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.433790+0000","last_change":"2023-09-23T23:22:57.759984+0000","last_active":"2023-09-23T23:22:59.433790+0000","last_peered":"2023-09-23T23:22:59.433790+0000","last_clean":"2023-09-23T23:22:59.433790+0000","last_became_active":"2023-09-23T23:22:57.759564+0000","last_became_peered":"2023-09-23T23:22:57.759564+0000","last_unstale":"2023-09-23T23:22:59.433790+0000","last_undegraded":"2023-09-23T23:22:59.433790+0000","last_fullsized":"2023-09-23T23:22:59.433790+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.802642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.802642+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0056570290000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"337'11411","reported_seq":15404,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T23:23:05.166013+0000","last_change":"2023-09-23T23:22:42.499876+0000","last_active":"2023-09-23T23:23:05.166013+0000","last_peered":"2023-09-23T23:23:05.166013+0000","last_clean":"2023-09-23T23:23:05.166013+0000","last_became_active":"2023-09-23T23:22:42.457163+0000","last_became_peered":"2023-09-23T23:22:42.457163+0000","last_unstale":"2023-09-23T23:23:05.166013+0000","last_undegraded":"2023-09-23T23:23:05.166013+0000","last_fullsized":"2023-09-23T23:23:05.166013+0000","mapping_epoch":324,"log_start":"291'6484","ondisk_log_start":"291'6484","created":255,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4927,"log_dups_size":0,"ondisk_log_size":4927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:41.352974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74645504,"num_objects":1139,"num_object_clones":0,"num_object_copies":3417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11411,"num_write_kb":401600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1938,"num_bytes_recovered":127008768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":154,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.432125+0000","last_change":"2023-09-23T23:21:22.192374+0000","last_active":"2023-09-23T23:22:59.432125+0000","last_peered":"2023-09-23T23:22:59.432125+0000","last_clean":"2023-09-23T23:22:59.432125+0000","last_became_active":"2023-09-23T23:21:22.192086+0000","last_became_peered":"2023-09-23T23:21:22.192086+0000","last_unstale":"2023-09-23T23:22:59.432125+0000","last_undegraded":"2023-09-23T23:22:59.432125+0000","last_fullsized":"2023-09-23T23:22:59.432125+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:20.605133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":311,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.430727+0000","last_change":"2023-09-23T23:22:38.438390+0000","last_active":"2023-09-23T23:22:59.430727+0000","last_peered":"2023-09-23T23:22:59.430727+0000","last_clean":"2023-09-23T23:22:59.430727+0000","last_became_active":"2023-09-23T23:22:38.438065+0000","last_became_peered":"2023-09-23T23:22:38.438065+0000","last_unstale":"2023-09-23T23:22:59.430727+0000","last_undegraded":"2023-09-23T23:22:59.430727+0000","last_fullsized":"2023-09-23T23:22:59.430727+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:51.254227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"337'11580","reported_seq":11670,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T23:23:04.518464+0000","last_change":"2023-09-23T23:18:58.587100+0000","last_active":"2023-09-23T23:23:04.518464+0000","last_peered":"2023-09-23T23:23:04.518464+0000","last_clean":"2023-09-23T23:23:04.518464+0000","last_became_active":"2023-09-23T23:18:58.586837+0000","last_became_peered":"2023-09-23T23:18:58.586837+0000","last_unstale":"2023-09-23T23:23:04.518464+0000","last_undegraded":"2023-09-23T23:23:04.518464+0000","last_fullsized":"2023-09-23T23:23:04.518464+0000","mapping_epoch":255,"log_start":"282'5444","ondisk_log_start":"282'5444","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":6136,"log_dups_size":0,"ondisk_log_size":6136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:56.131536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80609280,"num_objects":1230,"num_object_clones":0,"num_object_copies":3690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11580,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":675,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:59.430223+0000","last_change":"2023-09-23T23:22:57.466372+0000","last_active":"2023-09-23T23:22:59.430223+0000","last_peered":"2023-09-23T23:22:59.430223+0000","last_clean":"2023-09-23T23:22:59.430223+0000","last_became_active":"2023-09-23T23:22:57.465950+0000","last_became_peered":"2023-09-23T23:22:57.465950+0000","last_unstale":"2023-09-23T23:22:59.430223+0000","last_undegraded":"2023-09-23T23:22:59.430223+0000","last_fullsized":"2023-09-23T23:22:59.430223+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:46.150686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:46.150686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,5],"avail_no_missing":["3","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":303,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.431137+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:22:59.431137+0000","last_peered":"2023-09-23T23:22:59.431137+0000","last_clean":"2023-09-23T23:22:59.431137+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:22:59.431137+0000","last_undegraded":"2023-09-23T23:22:59.431137+0000","last_fullsized":"2023-09-23T23:22:59.431137+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":296,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.430332+0000","last_change":"2023-09-23T23:10:59.512363+0000","last_active":"2023-09-23T23:22:59.430332+0000","last_peered":"2023-09-23T23:22:59.430332+0000","last_clean":"2023-09-23T23:22:59.430332+0000","last_became_active":"2023-09-23T23:10:59.512022+0000","last_became_peered":"2023-09-23T23:10:59.512022+0000","last_unstale":"2023-09-23T23:22:59.430332+0000","last_undegraded":"2023-09-23T23:22:59.430332+0000","last_fullsized":"2023-09-23T23:22:59.430332+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:45.523432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"337'11484","reported_seq":15079,"reported_epoch":337,"state":"active+recovering","last_fresh":"2023-09-23T23:23:05.187705+0000","last_change":"2023-09-23T23:22:38.517002+0000","last_active":"2023-09-23T23:23:05.187705+0000","last_peered":"2023-09-23T23:23:05.187705+0000","last_clean":"2023-09-23T23:20:14.670806+0000","last_became_active":"2023-09-23T23:21:51.082780+0000","last_became_peered":"2023-09-23T23:21:51.082780+0000","last_unstale":"2023-09-23T23:23:05.187705+0000","last_undegraded":"2023-09-23T23:23:05.187705+0000","last_fullsized":"2023-09-23T23:23:05.187705+0000","mapping_epoch":301,"log_start":"291'6544","ondisk_log_start":"291'6544","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4940,"log_dups_size":0,"ondisk_log_size":4940,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:37.352090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76152832,"num_objects":1162,"num_object_clones":0,"num_object_copies":3486,"num_objects_missing_on_primary":3558,"num_objects_missing":3558,"num_objects_degraded":0,"num_objects_misplaced":138,"num_objects_unfound":0,"num_objects_dirty":1162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11484,"num_write_kb":404672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1741,"num_bytes_recovered":114098176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":["0","4","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":7},{"shards":"0,2,5","objects":1017},{"shards":"0,4,5","objects":138}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":568,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.426608+0000","last_change":"2023-09-23T23:22:57.464403+0000","last_active":"2023-09-23T23:22:59.426608+0000","last_peered":"2023-09-23T23:22:59.426608+0000","last_clean":"2023-09-23T23:22:59.426608+0000","last_became_active":"2023-09-23T23:22:57.463887+0000","last_became_peered":"2023-09-23T23:22:57.463887+0000","last_unstale":"2023-09-23T23:22:59.426608+0000","last_undegraded":"2023-09-23T23:22:59.426608+0000","last_fullsized":"2023-09-23T23:22:59.426608+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:42.871134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:42.871134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023428590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":131344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":308,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.434478+0000","last_change":"2023-09-23T23:16:56.525904+0000","last_active":"2023-09-23T23:22:59.434478+0000","last_peered":"2023-09-23T23:22:59.434478+0000","last_clean":"2023-09-23T23:22:59.434478+0000","last_became_active":"2023-09-23T23:16:56.525372+0000","last_became_peered":"2023-09-23T23:16:56.525372+0000","last_unstale":"2023-09-23T23:22:59.434478+0000","last_undegraded":"2023-09-23T23:22:59.434478+0000","last_fullsized":"2023-09-23T23:22:59.434478+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:17.070117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":351,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.427836+0000","last_change":"2023-09-23T23:22:38.439737+0000","last_active":"2023-09-23T23:22:59.427836+0000","last_peered":"2023-09-23T23:22:59.427836+0000","last_clean":"2023-09-23T23:22:59.427836+0000","last_became_active":"2023-09-23T23:22:38.439246+0000","last_became_peered":"2023-09-23T23:22:38.439246+0000","last_unstale":"2023-09-23T23:22:59.427836+0000","last_undegraded":"2023-09-23T23:22:59.427836+0000","last_fullsized":"2023-09-23T23:22:59.427836+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:40.042010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":443,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.438603+0000","last_change":"2023-09-23T23:22:57.469688+0000","last_active":"2023-09-23T23:22:59.438603+0000","last_peered":"2023-09-23T23:22:59.438603+0000","last_clean":"2023-09-23T23:22:59.438603+0000","last_became_active":"2023-09-23T23:22:57.469369+0000","last_became_peered":"2023-09-23T23:22:57.469369+0000","last_unstale":"2023-09-23T23:22:59.438603+0000","last_undegraded":"2023-09-23T23:22:59.438603+0000","last_fullsized":"2023-09-23T23:22:59.438603+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:19:31.012484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:19:31.012484+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"337'11541","reported_seq":17070,"reported_epoch":337,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:23:04.469248+0000","last_change":"2023-09-23T23:22:55.504811+0000","last_active":"2023-09-23T23:23:04.469248+0000","last_peered":"2023-09-23T23:23:04.469248+0000","last_clean":"2023-09-23T23:23:04.469248+0000","last_became_active":"2023-09-23T23:22:55.444331+0000","last_became_peered":"2023-09-23T23:22:55.444331+0000","last_unstale":"2023-09-23T23:23:04.469248+0000","last_undegraded":"2023-09-23T23:23:04.469248+0000","last_fullsized":"2023-09-23T23:23:04.469248+0000","mapping_epoch":331,"log_start":"292'7034","ondisk_log_start":"292'7034","created":255,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4507,"log_dups_size":0,"ondisk_log_size":4507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:50.040548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82247680,"num_objects":1255,"num_object_clones":0,"num_object_copies":3765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1255,"num_objects_unfound":0,"num_objects_dirty":1255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11541,"num_write_kb":409472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2686,"num_bytes_recovered":176029696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1,0],"avail_no_missing":["6","0","1"],"object_location_counts":[{"shards":"0,1,6","objects":1255}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":347,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:59.426270+0000","last_change":"2023-09-23T23:22:39.449144+0000","last_active":"2023-09-23T23:22:59.426270+0000","last_peered":"2023-09-23T23:22:59.426270+0000","last_clean":"2023-09-23T23:22:59.426270+0000","last_became_active":"2023-09-23T23:22:39.448587+0000","last_became_peered":"2023-09-23T23:22:39.448587+0000","last_unstale":"2023-09-23T23:22:59.426270+0000","last_undegraded":"2023-09-23T23:22:59.426270+0000","last_fullsized":"2023-09-23T23:22:59.426270+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:37.516445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1,5],"avail_no_missing":["2","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":306,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.430654+0000","last_change":"2023-09-23T23:15:28.228439+0000","last_active":"2023-09-23T23:22:59.430654+0000","last_peered":"2023-09-23T23:22:59.430654+0000","last_clean":"2023-09-23T23:22:59.430654+0000","last_became_active":"2023-09-23T23:15:28.228095+0000","last_became_peered":"2023-09-23T23:15:28.228095+0000","last_unstale":"2023-09-23T23:22:59.430654+0000","last_undegraded":"2023-09-23T23:22:59.430654+0000","last_fullsized":"2023-09-23T23:22:59.430654+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:37.667201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"337'11247","reported_seq":19908,"reported_epoch":337,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:23:05.078931+0000","last_change":"2023-09-23T23:22:45.478727+0000","last_active":"2023-09-23T23:23:05.078931+0000","last_peered":"2023-09-23T23:23:05.078931+0000","last_clean":"2023-09-23T23:23:05.078931+0000","last_became_active":"2023-09-23T23:22:45.432387+0000","last_became_peered":"2023-09-23T23:22:45.432387+0000","last_unstale":"2023-09-23T23:23:05.078931+0000","last_undegraded":"2023-09-23T23:23:05.078931+0000","last_fullsized":"2023-09-23T23:23:05.078931+0000","mapping_epoch":326,"log_start":"291'6247","ondisk_log_start":"291'6247","created":255,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":5000,"log_dups_size":0,"ondisk_log_size":5000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:19.983401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74645504,"num_objects":1139,"num_object_clones":0,"num_object_copies":3417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1139,"num_objects_unfound":0,"num_objects_dirty":1139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11247,"num_write_kb":396352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4265,"num_bytes_recovered":279511040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,2],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":1139}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":626,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.425690+0000","last_change":"2023-09-23T23:22:57.468520+0000","last_active":"2023-09-23T23:22:59.425690+0000","last_peered":"2023-09-23T23:22:59.425690+0000","last_clean":"2023-09-23T23:22:59.425690+0000","last_became_active":"2023-09-23T23:22:57.468211+0000","last_became_peered":"2023-09-23T23:22:57.468211+0000","last_unstale":"2023-09-23T23:22:59.425690+0000","last_undegraded":"2023-09-23T23:22:59.425690+0000","last_fullsized":"2023-09-23T23:22:59.425690+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:51.734652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:51.734652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.023100959000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":27,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.438106+0000","last_change":"2023-09-23T23:22:38.434606+0000","last_active":"2023-09-23T23:22:59.438106+0000","last_peered":"2023-09-23T23:22:59.438106+0000","last_clean":"2023-09-23T23:22:59.438106+0000","last_became_active":"2023-09-23T23:22:38.434196+0000","last_became_peered":"2023-09-23T23:22:38.434196+0000","last_unstale":"2023-09-23T23:22:59.438106+0000","last_undegraded":"2023-09-23T23:22:59.438106+0000","last_fullsized":"2023-09-23T23:22:59.438106+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:14.702902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":256,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.438431+0000","last_change":"2023-09-23T23:22:39.489325+0000","last_active":"2023-09-23T23:22:59.438431+0000","last_peered":"2023-09-23T23:22:59.438431+0000","last_clean":"2023-09-23T23:22:59.438431+0000","last_became_active":"2023-09-23T23:22:39.489048+0000","last_became_peered":"2023-09-23T23:22:39.489048+0000","last_unstale":"2023-09-23T23:22:59.438431+0000","last_undegraded":"2023-09-23T23:22:59.438431+0000","last_fullsized":"2023-09-23T23:22:59.438431+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:29.984948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"337'11491","reported_seq":11577,"reported_epoch":337,"state":"active+undersized+degraded","last_fresh":"2023-09-23T23:23:04.540989+0000","last_change":"2023-09-23T23:19:10.829313+0000","last_active":"2023-09-23T23:23:04.540989+0000","last_peered":"2023-09-23T23:23:04.540989+0000","last_clean":"2023-09-23T23:18:57.558582+0000","last_became_active":"2023-09-23T23:18:59.585882+0000","last_became_peered":"2023-09-23T23:18:59.585882+0000","last_unstale":"2023-09-23T23:23:04.540989+0000","last_undegraded":"2023-09-23T23:19:10.822953+0000","last_fullsized":"2023-09-23T23:18:59.582773+0000","mapping_epoch":256,"log_start":"281'5351","ondisk_log_start":"281'5351","created":255,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":6140,"log_dups_size":0,"ondisk_log_size":6140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:12.870079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75038720,"num_objects":1145,"num_object_clones":0,"num_object_copies":3435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1145,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11491,"num_write_kb":404352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":1145}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":733,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.434254+0000","last_change":"2023-09-23T23:22:57.470406+0000","last_active":"2023-09-23T23:22:59.434254+0000","last_peered":"2023-09-23T23:22:59.434254+0000","last_clean":"2023-09-23T23:22:59.434254+0000","last_became_active":"2023-09-23T23:22:57.470024+0000","last_became_peered":"2023-09-23T23:22:57.470024+0000","last_unstale":"2023-09-23T23:22:59.434254+0000","last_undegraded":"2023-09-23T23:22:59.434254+0000","last_fullsized":"2023-09-23T23:22:59.434254+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:47.797085+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:47.797085+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":26,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.438106+0000","last_change":"2023-09-23T23:22:39.426465+0000","last_active":"2023-09-23T23:22:59.438106+0000","last_peered":"2023-09-23T23:22:59.438106+0000","last_clean":"2023-09-23T23:22:59.438106+0000","last_became_active":"2023-09-23T23:22:39.425837+0000","last_became_peered":"2023-09-23T23:22:39.425837+0000","last_unstale":"2023-09-23T23:22:59.438106+0000","last_undegraded":"2023-09-23T23:22:59.438106+0000","last_fullsized":"2023-09-23T23:22:59.438106+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:36.873867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":352,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.436410+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:22:59.436410+0000","last_peered":"2023-09-23T23:22:59.436410+0000","last_clean":"2023-09-23T23:22:59.436410+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:22:59.436410+0000","last_undegraded":"2023-09-23T23:22:59.436410+0000","last_fullsized":"2023-09-23T23:22:59.436410+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":270,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:59.432094+0000","last_change":"2023-09-23T23:21:51.043740+0000","last_active":"2023-09-23T23:22:59.432094+0000","last_peered":"2023-09-23T23:22:59.432094+0000","last_clean":"2023-09-23T23:22:59.432094+0000","last_became_active":"2023-09-23T23:21:51.043279+0000","last_became_peered":"2023-09-23T23:21:51.043279+0000","last_unstale":"2023-09-23T23:22:59.432094+0000","last_undegraded":"2023-09-23T23:22:59.432094+0000","last_fullsized":"2023-09-23T23:22:59.432094+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:57:07.926321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2,4],"avail_no_missing":["0","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.a","version":"337'11573","reported_seq":11663,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T23:23:04.478259+0000","last_change":"2023-09-23T23:18:58.585142+0000","last_active":"2023-09-23T23:23:04.478259+0000","last_peered":"2023-09-23T23:23:04.478259+0000","last_clean":"2023-09-23T23:23:04.478259+0000","last_became_active":"2023-09-23T23:18:58.584860+0000","last_became_peered":"2023-09-23T23:18:58.584860+0000","last_unstale":"2023-09-23T23:23:04.478259+0000","last_undegraded":"2023-09-23T23:23:04.478259+0000","last_fullsized":"2023-09-23T23:23:04.478259+0000","mapping_epoch":255,"log_start":"292'7063","ondisk_log_start":"292'7063","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4510,"log_dups_size":0,"ondisk_log_size":4510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:14.574839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81854464,"num_objects":1249,"num_object_clones":0,"num_object_copies":3747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11573,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":469,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.434407+0000","last_change":"2023-09-23T23:22:17.002317+0000","last_active":"2023-09-23T23:22:59.434407+0000","last_peered":"2023-09-23T23:22:59.434407+0000","last_clean":"2023-09-23T23:22:59.434407+0000","last_became_active":"2023-09-23T23:22:17.001911+0000","last_became_peered":"2023-09-23T23:22:17.001911+0000","last_unstale":"2023-09-23T23:22:59.434407+0000","last_undegraded":"2023-09-23T23:22:59.434407+0000","last_fullsized":"2023-09-23T23:22:59.434407+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:49.726740+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:49.726740+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":27,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.438493+0000","last_change":"2023-09-23T23:22:38.433755+0000","last_active":"2023-09-23T23:22:59.438493+0000","last_peered":"2023-09-23T23:22:59.438493+0000","last_clean":"2023-09-23T23:22:59.438493+0000","last_became_active":"2023-09-23T23:22:38.433442+0000","last_became_peered":"2023-09-23T23:22:38.433442+0000","last_unstale":"2023-09-23T23:22:59.438493+0000","last_undegraded":"2023-09-23T23:22:59.438493+0000","last_fullsized":"2023-09-23T23:22:59.438493+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:25.409363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":297,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.430154+0000","last_change":"2023-09-23T23:04:11.514703+0000","last_active":"2023-09-23T23:22:59.430154+0000","last_peered":"2023-09-23T23:22:59.430154+0000","last_clean":"2023-09-23T23:22:59.430154+0000","last_became_active":"2023-09-23T23:04:11.514403+0000","last_became_peered":"2023-09-23T23:04:11.514403+0000","last_unstale":"2023-09-23T23:22:59.430154+0000","last_undegraded":"2023-09-23T23:22:59.430154+0000","last_fullsized":"2023-09-23T23:22:59.430154+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:26.374346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":69,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.426240+0000","last_change":"2023-09-23T23:20:15.781844+0000","last_active":"2023-09-23T23:22:59.426240+0000","last_peered":"2023-09-23T23:22:59.426240+0000","last_clean":"2023-09-23T23:22:59.426240+0000","last_became_active":"2023-09-23T23:20:15.781573+0000","last_became_peered":"2023-09-23T23:20:15.781573+0000","last_unstale":"2023-09-23T23:22:59.426240+0000","last_undegraded":"2023-09-23T23:22:59.426240+0000","last_fullsized":"2023-09-23T23:22:59.426240+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:25:50.816486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"337'11456","reported_seq":15756,"reported_epoch":337,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:23:05.172068+0000","last_change":"2023-09-23T23:22:38.105753+0000","last_active":"2023-09-23T23:23:05.172068+0000","last_peered":"2023-09-23T23:23:05.172068+0000","last_clean":"2023-09-23T23:23:05.172068+0000","last_became_active":"2023-09-23T23:22:38.065297+0000","last_became_peered":"2023-09-23T23:22:38.065297+0000","last_unstale":"2023-09-23T23:23:05.172068+0000","last_undegraded":"2023-09-23T23:23:05.172068+0000","last_fullsized":"2023-09-23T23:23:05.172068+0000","mapping_epoch":319,"log_start":"291'6510","ondisk_log_start":"291'6510","created":255,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4946,"log_dups_size":0,"ondisk_log_size":4946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:25.482919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78512128,"num_objects":1198,"num_object_clones":0,"num_object_copies":3594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1198,"num_objects_unfound":0,"num_objects_dirty":1198,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11456,"num_write_kb":404928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2090,"num_bytes_recovered":136970240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4,0],"avail_no_missing":["2","0","4"],"object_location_counts":[{"shards":"0,2,4","objects":1198}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":659,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.431990+0000","last_change":"2023-09-23T23:22:17.001078+0000","last_active":"2023-09-23T23:22:59.431990+0000","last_peered":"2023-09-23T23:22:59.431990+0000","last_clean":"2023-09-23T23:22:59.431990+0000","last_became_active":"2023-09-23T23:22:17.000797+0000","last_became_peered":"2023-09-23T23:22:17.000797+0000","last_unstale":"2023-09-23T23:22:59.431990+0000","last_undegraded":"2023-09-23T23:22:59.431990+0000","last_fullsized":"2023-09-23T23:22:59.431990+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:47.277080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:47.277080+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001458104,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":69,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.426059+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:22:59.426059+0000","last_peered":"2023-09-23T23:22:59.426059+0000","last_clean":"2023-09-23T23:22:59.426059+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:22:59.426059+0000","last_undegraded":"2023-09-23T23:22:59.426059+0000","last_fullsized":"2023-09-23T23:22:59.426059+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":27,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.438308+0000","last_change":"2023-09-23T23:22:38.480385+0000","last_active":"2023-09-23T23:22:59.438308+0000","last_peered":"2023-09-23T23:22:59.438308+0000","last_clean":"2023-09-23T23:22:59.438308+0000","last_became_active":"2023-09-23T23:22:38.480071+0000","last_became_peered":"2023-09-23T23:22:38.480071+0000","last_unstale":"2023-09-23T23:22:59.438308+0000","last_undegraded":"2023-09-23T23:22:59.438308+0000","last_fullsized":"2023-09-23T23:22:59.438308+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:15.348789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":149,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.431516+0000","last_change":"2023-09-23T23:21:22.194721+0000","last_active":"2023-09-23T23:22:59.431516+0000","last_peered":"2023-09-23T23:22:59.431516+0000","last_clean":"2023-09-23T23:22:59.431516+0000","last_became_active":"2023-09-23T23:21:22.194333+0000","last_became_peered":"2023-09-23T23:21:22.194333+0000","last_unstale":"2023-09-23T23:22:59.431516+0000","last_undegraded":"2023-09-23T23:22:59.431516+0000","last_fullsized":"2023-09-23T23:22:59.431516+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:46.823997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"337'11645","reported_seq":17318,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T23:23:04.448775+0000","last_change":"2023-09-23T23:22:50.492608+0000","last_active":"2023-09-23T23:23:04.448775+0000","last_peered":"2023-09-23T23:23:04.448775+0000","last_clean":"2023-09-23T23:23:04.448775+0000","last_became_active":"2023-09-23T23:22:50.436512+0000","last_became_peered":"2023-09-23T23:22:50.436512+0000","last_unstale":"2023-09-23T23:23:04.448775+0000","last_undegraded":"2023-09-23T23:23:04.448775+0000","last_fullsized":"2023-09-23T23:23:04.448775+0000","mapping_epoch":328,"log_start":"292'7224","ondisk_log_start":"292'7224","created":255,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4421,"log_dups_size":0,"ondisk_log_size":4421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:28.237856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82771968,"num_objects":1263,"num_object_clones":0,"num_object_copies":3789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11645,"num_write_kb":413056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2763,"num_bytes_recovered":181075968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":336,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.440128+0000","last_change":"2023-09-23T23:21:22.193754+0000","last_active":"2023-09-23T23:22:59.440128+0000","last_peered":"2023-09-23T23:22:59.440128+0000","last_clean":"2023-09-23T23:22:59.440128+0000","last_became_active":"2023-09-23T23:21:22.193409+0000","last_became_peered":"2023-09-23T23:21:22.193409+0000","last_unstale":"2023-09-23T23:22:59.440128+0000","last_undegraded":"2023-09-23T23:22:59.440128+0000","last_fullsized":"2023-09-23T23:22:59.440128+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:53.635813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":618,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:23:00.122825+0000","last_change":"2023-09-23T23:22:41.462946+0000","last_active":"2023-09-23T23:23:00.122825+0000","last_peered":"2023-09-23T23:23:00.122825+0000","last_clean":"2023-09-23T23:23:00.122825+0000","last_became_active":"2023-09-23T23:22:41.462616+0000","last_became_peered":"2023-09-23T23:22:41.462616+0000","last_unstale":"2023-09-23T23:23:00.122825+0000","last_undegraded":"2023-09-23T23:23:00.122825+0000","last_fullsized":"2023-09-23T23:23:00.122825+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:42.190868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:42.190868+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0049256480000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":266,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.430795+0000","last_change":"2023-09-23T23:16:56.525298+0000","last_active":"2023-09-23T23:22:59.430795+0000","last_peered":"2023-09-23T23:22:59.430795+0000","last_clean":"2023-09-23T23:22:59.430795+0000","last_became_active":"2023-09-23T23:16:56.524974+0000","last_became_peered":"2023-09-23T23:16:56.524974+0000","last_unstale":"2023-09-23T23:22:59.430795+0000","last_undegraded":"2023-09-23T23:22:59.430795+0000","last_fullsized":"2023-09-23T23:22:59.430795+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:08.860790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":299,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:59.434369+0000","last_change":"2023-09-23T23:21:51.046606+0000","last_active":"2023-09-23T23:22:59.434369+0000","last_peered":"2023-09-23T23:22:59.434369+0000","last_clean":"2023-09-23T23:22:59.434369+0000","last_became_active":"2023-09-23T23:21:51.046103+0000","last_became_peered":"2023-09-23T23:21:51.046103+0000","last_unstale":"2023-09-23T23:22:59.434369+0000","last_undegraded":"2023-09-23T23:22:59.434369+0000","last_fullsized":"2023-09-23T23:22:59.434369+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:05.999638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,4],"avail_no_missing":["6","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"337'11662","reported_seq":22545,"reported_epoch":337,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:23:05.190367+0000","last_change":"2023-09-23T23:22:09.217110+0000","last_active":"2023-09-23T23:23:05.190367+0000","last_peered":"2023-09-23T23:23:05.190367+0000","last_clean":"2023-09-23T23:23:05.190367+0000","last_became_active":"2023-09-23T23:22:09.186713+0000","last_became_peered":"2023-09-23T23:22:09.186713+0000","last_unstale":"2023-09-23T23:23:05.190367+0000","last_undegraded":"2023-09-23T23:23:05.190367+0000","last_fullsized":"2023-09-23T23:23:05.190367+0000","mapping_epoch":306,"log_start":"291'6674","ondisk_log_start":"291'6674","created":255,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4988,"log_dups_size":0,"ondisk_log_size":4988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:00.893419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80871424,"num_objects":1234,"num_object_clones":0,"num_object_copies":3702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1234,"num_objects_unfound":0,"num_objects_dirty":1234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11662,"num_write_kb":412672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5377,"num_bytes_recovered":301334528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0,5],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2,5","objects":1234}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":434,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.438682+0000","last_change":"2023-09-23T23:22:41.467338+0000","last_active":"2023-09-23T23:22:59.438682+0000","last_peered":"2023-09-23T23:22:59.438682+0000","last_clean":"2023-09-23T23:22:59.438682+0000","last_became_active":"2023-09-23T23:22:41.466645+0000","last_became_peered":"2023-09-23T23:22:41.466645+0000","last_unstale":"2023-09-23T23:22:59.438682+0000","last_undegraded":"2023-09-23T23:22:59.438682+0000","last_fullsized":"2023-09-23T23:22:59.438682+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:19:31.012484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:19:31.012484+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":287,"reported_epoch":337,"state":"active+undersized","last_fresh":"2023-09-23T23:23:00.472896+0000","last_change":"2023-09-23T23:18:59.580395+0000","last_active":"2023-09-23T23:23:00.472896+0000","last_peered":"2023-09-23T23:23:00.472896+0000","last_clean":"2023-09-23T23:16:52.490944+0000","last_became_active":"2023-09-23T23:18:59.580395+0000","last_became_peered":"2023-09-23T23:18:59.580395+0000","last_unstale":"2023-09-23T23:23:00.472896+0000","last_undegraded":"2023-09-23T23:23:00.472896+0000","last_fullsized":"2023-09-23T23:18:59.577313+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:36.323937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":310,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:59.434285+0000","last_change":"2023-09-23T23:20:27.898330+0000","last_active":"2023-09-23T23:22:59.434285+0000","last_peered":"2023-09-23T23:22:59.434285+0000","last_clean":"2023-09-23T23:22:59.434285+0000","last_became_active":"2023-09-23T23:20:27.897982+0000","last_became_peered":"2023-09-23T23:20:27.897982+0000","last_unstale":"2023-09-23T23:22:59.434285+0000","last_undegraded":"2023-09-23T23:22:59.434285+0000","last_fullsized":"2023-09-23T23:22:59.434285+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:28.564415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,4],"avail_no_missing":["0","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":365,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.430995+0000","last_change":"2023-09-23T23:22:38.438448+0000","last_active":"2023-09-23T23:22:59.430995+0000","last_peered":"2023-09-23T23:22:59.430995+0000","last_clean":"2023-09-23T23:22:59.430995+0000","last_became_active":"2023-09-23T23:22:38.437495+0000","last_became_peered":"2023-09-23T23:22:38.437495+0000","last_unstale":"2023-09-23T23:22:59.430995+0000","last_undegraded":"2023-09-23T23:22:59.430995+0000","last_fullsized":"2023-09-23T23:22:59.430995+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:49.587056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"337'11427","reported_seq":11518,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T23:23:04.417888+0000","last_change":"2023-09-23T23:18:58.587441+0000","last_active":"2023-09-23T23:23:04.417888+0000","last_peered":"2023-09-23T23:23:04.417888+0000","last_clean":"2023-09-23T23:23:04.417888+0000","last_became_active":"2023-09-23T23:18:58.587188+0000","last_became_peered":"2023-09-23T23:18:58.587188+0000","last_unstale":"2023-09-23T23:23:04.417888+0000","last_undegraded":"2023-09-23T23:23:04.417888+0000","last_fullsized":"2023-09-23T23:23:04.417888+0000","mapping_epoch":255,"log_start":"292'6930","ondisk_log_start":"292'6930","created":255,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4497,"log_dups_size":0,"ondisk_log_size":4497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:54.227096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78905368,"num_objects":1205,"num_object_clones":0,"num_object_copies":3615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1205,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11427,"num_write_kb":404161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":630,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.433996+0000","last_change":"2023-09-23T23:22:57.759898+0000","last_active":"2023-09-23T23:22:59.433996+0000","last_peered":"2023-09-23T23:22:59.433996+0000","last_clean":"2023-09-23T23:22:59.433996+0000","last_became_active":"2023-09-23T23:22:57.759601+0000","last_became_peered":"2023-09-23T23:22:57.759601+0000","last_unstale":"2023-09-23T23:22:59.433996+0000","last_undegraded":"2023-09-23T23:22:59.433996+0000","last_fullsized":"2023-09-23T23:22:59.433996+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.802642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.802642+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0056570290000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":138,"reported_epoch":336,"state":"active+undersized","last_fresh":"2023-09-23T23:22:59.431972+0000","last_change":"2023-09-23T23:18:59.585438+0000","last_active":"2023-09-23T23:22:59.431972+0000","last_peered":"2023-09-23T23:22:59.431972+0000","last_clean":"2023-09-23T23:03:28.874460+0000","last_became_active":"2023-09-23T23:18:59.585438+0000","last_became_peered":"2023-09-23T23:18:59.585438+0000","last_unstale":"2023-09-23T23:22:59.431972+0000","last_undegraded":"2023-09-23T23:22:59.431972+0000","last_fullsized":"2023-09-23T23:18:59.577149+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:37.419384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":27,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.438233+0000","last_change":"2023-09-23T23:22:38.453323+0000","last_active":"2023-09-23T23:22:59.438233+0000","last_peered":"2023-09-23T23:22:59.438233+0000","last_clean":"2023-09-23T23:22:59.438233+0000","last_became_active":"2023-09-23T23:22:38.452858+0000","last_became_peered":"2023-09-23T23:22:38.452858+0000","last_unstale":"2023-09-23T23:22:59.438233+0000","last_undegraded":"2023-09-23T23:22:59.438233+0000","last_fullsized":"2023-09-23T23:22:59.438233+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:31.988045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":58,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.435965+0000","last_change":"2023-09-23T23:20:36.429855+0000","last_active":"2023-09-23T23:22:59.435965+0000","last_peered":"2023-09-23T23:22:59.435965+0000","last_clean":"2023-09-23T23:22:59.435965+0000","last_became_active":"2023-09-23T23:20:36.427234+0000","last_became_peered":"2023-09-23T23:20:36.427234+0000","last_unstale":"2023-09-23T23:22:59.435965+0000","last_undegraded":"2023-09-23T23:22:59.435965+0000","last_fullsized":"2023-09-23T23:22:59.435965+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:39.297055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"337'11412","reported_seq":16467,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T23:23:04.483187+0000","last_change":"2023-09-23T23:22:20.455214+0000","last_active":"2023-09-23T23:23:04.483187+0000","last_peered":"2023-09-23T23:23:04.483187+0000","last_clean":"2023-09-23T23:23:04.483187+0000","last_became_active":"2023-09-23T23:22:20.427281+0000","last_became_peered":"2023-09-23T23:22:20.427281+0000","last_unstale":"2023-09-23T23:23:04.483187+0000","last_undegraded":"2023-09-23T23:23:04.483187+0000","last_fullsized":"2023-09-23T23:23:04.483187+0000","mapping_epoch":312,"log_start":"278'5222","ondisk_log_start":"278'5222","created":255,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":6190,"log_dups_size":0,"ondisk_log_size":6190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:50.223603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81788928,"num_objects":1248,"num_object_clones":0,"num_object_copies":3744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11412,"num_write_kb":405120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2467,"num_bytes_recovered":161677312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":637,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.427888+0000","last_change":"2023-09-23T23:22:57.465829+0000","last_active":"2023-09-23T23:22:59.427888+0000","last_peered":"2023-09-23T23:22:59.427888+0000","last_clean":"2023-09-23T23:22:59.427888+0000","last_became_active":"2023-09-23T23:22:57.465086+0000","last_became_peered":"2023-09-23T23:22:57.465086+0000","last_unstale":"2023-09-23T23:22:59.427888+0000","last_undegraded":"2023-09-23T23:22:59.427888+0000","last_fullsized":"2023-09-23T23:22:59.427888+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:51.489074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:51.489074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00519324,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":316,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.431525+0000","last_change":"2023-09-23T23:16:56.561147+0000","last_active":"2023-09-23T23:22:59.431525+0000","last_peered":"2023-09-23T23:22:59.431525+0000","last_clean":"2023-09-23T23:22:59.431525+0000","last_became_active":"2023-09-23T23:16:56.547808+0000","last_became_peered":"2023-09-23T23:16:56.547808+0000","last_unstale":"2023-09-23T23:22:59.431525+0000","last_undegraded":"2023-09-23T23:22:59.431525+0000","last_fullsized":"2023-09-23T23:22:59.431525+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:13.666957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":192,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.434204+0000","last_change":"2023-09-23T23:16:56.526187+0000","last_active":"2023-09-23T23:22:59.434204+0000","last_peered":"2023-09-23T23:22:59.434204+0000","last_clean":"2023-09-23T23:22:59.434204+0000","last_became_active":"2023-09-23T23:16:56.525874+0000","last_became_peered":"2023-09-23T23:16:56.525874+0000","last_unstale":"2023-09-23T23:22:59.434204+0000","last_undegraded":"2023-09-23T23:22:59.434204+0000","last_fullsized":"2023-09-23T23:22:59.434204+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:32:07.101559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":27,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.438184+0000","last_change":"2023-09-23T23:22:38.444291+0000","last_active":"2023-09-23T23:22:59.438184+0000","last_peered":"2023-09-23T23:22:59.438184+0000","last_clean":"2023-09-23T23:22:59.438184+0000","last_became_active":"2023-09-23T23:22:38.443996+0000","last_became_peered":"2023-09-23T23:22:38.443996+0000","last_unstale":"2023-09-23T23:22:59.438184+0000","last_undegraded":"2023-09-23T23:22:59.438184+0000","last_fullsized":"2023-09-23T23:22:59.438184+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:57.255970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.f","version":"337'11148","reported_seq":15855,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T23:23:04.445960+0000","last_change":"2023-09-23T23:22:25.497617+0000","last_active":"2023-09-23T23:23:04.445960+0000","last_peered":"2023-09-23T23:23:04.445960+0000","last_clean":"2023-09-23T23:23:04.445960+0000","last_became_active":"2023-09-23T23:22:25.453489+0000","last_became_peered":"2023-09-23T23:22:25.453489+0000","last_unstale":"2023-09-23T23:23:04.445960+0000","last_undegraded":"2023-09-23T23:23:04.445960+0000","last_fullsized":"2023-09-23T23:23:04.445960+0000","mapping_epoch":314,"log_start":"292'6670","ondisk_log_start":"292'6670","created":255,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:18:57.558582+0000","last_clean_scrub_stamp":"2023-09-23T23:18:57.558582+0000","objects_scrubbed":0,"log_size":4478,"log_dups_size":0,"ondisk_log_size":4478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:58.199474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78381056,"num_objects":1196,"num_object_clones":0,"num_object_copies":3588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11148,"num_write_kb":395008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2294,"num_bytes_recovered":150339584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":637,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.432335+0000","last_change":"2023-09-23T23:22:49.450190+0000","last_active":"2023-09-23T23:22:59.432335+0000","last_peered":"2023-09-23T23:22:59.432335+0000","last_clean":"2023-09-23T23:22:59.432335+0000","last_became_active":"2023-09-23T23:22:17.009092+0000","last_became_peered":"2023-09-23T23:22:17.009092+0000","last_unstale":"2023-09-23T23:22:59.432335+0000","last_undegraded":"2023-09-23T23:22:59.432335+0000","last_fullsized":"2023-09-23T23:22:59.432335+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.522632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.522632+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.018152603999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":218453,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":349,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.434483+0000","last_change":"2023-09-23T23:20:15.786693+0000","last_active":"2023-09-23T23:22:59.434483+0000","last_peered":"2023-09-23T23:22:59.434483+0000","last_clean":"2023-09-23T23:22:59.434483+0000","last_became_active":"2023-09-23T23:20:15.786360+0000","last_became_peered":"2023-09-23T23:20:15.786360+0000","last_unstale":"2023-09-23T23:22:59.434483+0000","last_undegraded":"2023-09-23T23:22:59.434483+0000","last_fullsized":"2023-09-23T23:22:59.434483+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:00.707946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":320,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.430825+0000","last_change":"2023-09-23T23:22:38.482705+0000","last_active":"2023-09-23T23:22:59.430825+0000","last_peered":"2023-09-23T23:22:59.430825+0000","last_clean":"2023-09-23T23:22:59.430825+0000","last_became_active":"2023-09-23T23:22:38.438239+0000","last_became_peered":"2023-09-23T23:22:38.438239+0000","last_unstale":"2023-09-23T23:22:59.430825+0000","last_undegraded":"2023-09-23T23:22:59.430825+0000","last_fullsized":"2023-09-23T23:22:59.430825+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:52.006349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":50,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.436048+0000","last_change":"2023-09-23T23:21:50.045682+0000","last_active":"2023-09-23T23:22:59.436048+0000","last_peered":"2023-09-23T23:22:59.436048+0000","last_clean":"2023-09-23T23:22:59.436048+0000","last_became_active":"2023-09-23T23:21:50.045339+0000","last_became_peered":"2023-09-23T23:21:50.045339+0000","last_unstale":"2023-09-23T23:22:59.436048+0000","last_undegraded":"2023-09-23T23:22:59.436048+0000","last_fullsized":"2023-09-23T23:22:59.436048+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:55.802091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":676,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:59.449718+0000","last_change":"2023-09-23T23:22:59.445554+0000","last_active":"2023-09-23T23:22:59.449718+0000","last_peered":"2023-09-23T23:22:59.449718+0000","last_clean":"2023-09-23T23:22:59.449718+0000","last_became_active":"2023-09-23T23:22:59.445108+0000","last_became_peered":"2023-09-23T23:22:59.445108+0000","last_unstale":"2023-09-23T23:22:59.449718+0000","last_undegraded":"2023-09-23T23:22:59.449718+0000","last_fullsized":"2023-09-23T23:22:59.449718+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:46.150686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:46.150686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,5],"avail_no_missing":["3","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":77,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:59.430072+0000","last_change":"2023-09-23T23:20:27.899913+0000","last_active":"2023-09-23T23:22:59.430072+0000","last_peered":"2023-09-23T23:22:59.430072+0000","last_clean":"2023-09-23T23:22:59.430072+0000","last_became_active":"2023-09-23T23:20:27.899525+0000","last_became_peered":"2023-09-23T23:20:27.899525+0000","last_unstale":"2023-09-23T23:22:59.430072+0000","last_undegraded":"2023-09-23T23:22:59.430072+0000","last_fullsized":"2023-09-23T23:22:59.430072+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:41.775325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,0],"avail_no_missing":["4","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":78,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.426367+0000","last_change":"2023-09-23T23:22:38.439602+0000","last_active":"2023-09-23T23:22:59.426367+0000","last_peered":"2023-09-23T23:22:59.426367+0000","last_clean":"2023-09-23T23:22:59.426367+0000","last_became_active":"2023-09-23T23:22:38.439261+0000","last_became_peered":"2023-09-23T23:22:38.439261+0000","last_unstale":"2023-09-23T23:22:59.426367+0000","last_undegraded":"2023-09-23T23:22:59.426367+0000","last_fullsized":"2023-09-23T23:22:59.426367+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:03.293816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":628,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.433620+0000","last_change":"2023-09-23T23:22:57.765002+0000","last_active":"2023-09-23T23:22:59.433620+0000","last_peered":"2023-09-23T23:22:59.433620+0000","last_clean":"2023-09-23T23:22:59.433620+0000","last_became_active":"2023-09-23T23:22:57.764718+0000","last_became_peered":"2023-09-23T23:22:57.764718+0000","last_unstale":"2023-09-23T23:22:59.433620+0000","last_undegraded":"2023-09-23T23:22:59.433620+0000","last_fullsized":"2023-09-23T23:22:59.433620+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.802642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.802642+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0056570290000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":79,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.430431+0000","last_change":"2023-09-23T23:22:38.452968+0000","last_active":"2023-09-23T23:22:59.430431+0000","last_peered":"2023-09-23T23:22:59.430431+0000","last_clean":"2023-09-23T23:22:59.430431+0000","last_became_active":"2023-09-23T23:22:38.437986+0000","last_became_peered":"2023-09-23T23:22:38.437986+0000","last_unstale":"2023-09-23T23:22:59.430431+0000","last_undegraded":"2023-09-23T23:22:59.430431+0000","last_fullsized":"2023-09-23T23:22:59.430431+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:10.333271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":333,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:22:59.430936+0000","last_change":"2023-09-23T23:21:51.056449+0000","last_active":"2023-09-23T23:22:59.430936+0000","last_peered":"2023-09-23T23:22:59.430936+0000","last_clean":"2023-09-23T23:22:59.430936+0000","last_became_active":"2023-09-23T23:21:51.056092+0000","last_became_peered":"2023-09-23T23:21:51.056092+0000","last_unstale":"2023-09-23T23:22:59.430936+0000","last_undegraded":"2023-09-23T23:22:59.430936+0000","last_fullsized":"2023-09-23T23:22:59.430936+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:33.836469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,4],"avail_no_missing":["3","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":65,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.430398+0000","last_change":"2023-09-23T23:21:22.194730+0000","last_active":"2023-09-23T23:22:59.430398+0000","last_peered":"2023-09-23T23:22:59.430398+0000","last_clean":"2023-09-23T23:22:59.430398+0000","last_became_active":"2023-09-23T23:21:22.194420+0000","last_became_peered":"2023-09-23T23:21:22.194420+0000","last_unstale":"2023-09-23T23:22:59.430398+0000","last_undegraded":"2023-09-23T23:22:59.430398+0000","last_fullsized":"2023-09-23T23:22:59.430398+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:31.083059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":182,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.430711+0000","last_change":"2023-09-23T23:20:26.841878+0000","last_active":"2023-09-23T23:22:59.430711+0000","last_peered":"2023-09-23T23:22:59.430711+0000","last_clean":"2023-09-23T23:22:59.430711+0000","last_became_active":"2023-09-23T23:20:26.841590+0000","last_became_peered":"2023-09-23T23:20:26.841590+0000","last_unstale":"2023-09-23T23:22:59.430711+0000","last_undegraded":"2023-09-23T23:22:59.430711+0000","last_fullsized":"2023-09-23T23:22:59.430711+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:23.904011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":646,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.431799+0000","last_change":"2023-09-23T23:22:57.472784+0000","last_active":"2023-09-23T23:22:59.431799+0000","last_peered":"2023-09-23T23:22:59.431799+0000","last_clean":"2023-09-23T23:22:59.431799+0000","last_became_active":"2023-09-23T23:22:57.472476+0000","last_became_peered":"2023-09-23T23:22:57.472476+0000","last_unstale":"2023-09-23T23:22:59.431799+0000","last_undegraded":"2023-09-23T23:22:59.431799+0000","last_fullsized":"2023-09-23T23:22:59.431799+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.522632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.522632+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.018152603999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54614,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":109227,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":344,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.430882+0000","last_change":"2023-09-23T23:20:16.730934+0000","last_active":"2023-09-23T23:22:59.430882+0000","last_peered":"2023-09-23T23:22:59.430882+0000","last_clean":"2023-09-23T23:22:59.430882+0000","last_became_active":"2023-09-23T23:20:16.730571+0000","last_became_peered":"2023-09-23T23:20:16.730571+0000","last_unstale":"2023-09-23T23:22:59.430882+0000","last_undegraded":"2023-09-23T23:22:59.430882+0000","last_fullsized":"2023-09-23T23:22:59.430882+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:17.993278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":347,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T23:23:00.485554+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:23:00.485554+0000","last_peered":"2023-09-23T23:23:00.485554+0000","last_clean":"2023-09-23T23:23:00.485554+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:23:00.485554+0000","last_undegraded":"2023-09-23T23:23:00.485554+0000","last_fullsized":"2023-09-23T23:23:00.485554+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":637,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:23:00.173678+0000","last_change":"2023-09-23T23:22:57.469656+0000","last_active":"2023-09-23T23:23:00.173678+0000","last_peered":"2023-09-23T23:23:00.173678+0000","last_clean":"2023-09-23T23:23:00.173678+0000","last_became_active":"2023-09-23T23:22:57.469358+0000","last_became_peered":"2023-09-23T23:22:57.469358+0000","last_unstale":"2023-09-23T23:23:00.173678+0000","last_undegraded":"2023-09-23T23:23:00.173678+0000","last_fullsized":"2023-09-23T23:23:00.173678+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:51.489074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:51.489074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00519324,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":340,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.431686+0000","last_change":"2023-09-23T23:20:15.786279+0000","last_active":"2023-09-23T23:22:59.431686+0000","last_peered":"2023-09-23T23:22:59.431686+0000","last_clean":"2023-09-23T23:22:59.431686+0000","last_became_active":"2023-09-23T23:20:15.782731+0000","last_became_peered":"2023-09-23T23:20:15.782731+0000","last_unstale":"2023-09-23T23:22:59.431686+0000","last_undegraded":"2023-09-23T23:22:59.431686+0000","last_fullsized":"2023-09-23T23:22:59.431686+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:58.011396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":373,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T23:22:59.429605+0000","last_change":"2023-09-23T23:22:38.447350+0000","last_active":"2023-09-23T23:22:59.429605+0000","last_peered":"2023-09-23T23:22:59.429605+0000","last_clean":"2023-09-23T23:22:59.429605+0000","last_became_active":"2023-09-23T23:22:38.447081+0000","last_became_peered":"2023-09-23T23:22:38.447081+0000","last_unstale":"2023-09-23T23:22:59.429605+0000","last_undegraded":"2023-09-23T23:22:59.429605+0000","last_fullsized":"2023-09-23T23:22:59.429605+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:39:12.306545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":90,"acting":95,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1257046040,"num_objects":19182,"num_object_clones":0,"num_object_copies":57546,"num_objects_missing_on_primary":3558,"num_objects_missing":3558,"num_objects_degraded":1145,"num_objects_misplaced":4964,"num_objects_unfound":0,"num_objects_dirty":19182,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":183230,"num_write_kb":6477121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27805,"num_bytes_recovered":1771110400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4748488704,"data_stored":4748476488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":80070,"ondisk_log_size":80070,"up":43,"acting":47,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":6,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5954,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":89,"acting":94,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":751378,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":134,"num_read_kb":250,"num_write":65,"num_write_kb":659,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1401077,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":91,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":21,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234314,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":803516,"kb_used_data":352116,"kb_used_omap":0,"kb_used_meta":451392,"kb_avail":92941636,"statfs":{"total":95995035648,"available":95172235264,"internally_reserved":0,"allocated":360566784,"data_stored":359841767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":462225408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:18:39 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":4.9820000000000002,"15min":3.9239999999999999},"min":{"1min":0.22,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":6.9249999999999998,"5min":216.40899999999999,"15min":230.249},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.7320000000000002,"5min":5.7359999999999998,"15min":4.2290000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24199999999999999},"max":{"1min":11.327999999999999,"5min":216.43700000000001,"15min":230.261},"last":0.40899999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:17:56 2023","interfaces":[{"interface":"back","average":{"1min":2.6150000000000002,"5min":2.9700000000000002,"15min":4.2359999999999998},"min":{"1min":0.39100000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.798,"5min":121.39400000000001,"15min":121.39400000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.2989999999999999,"5min":2.9790000000000001,"15min":4.077},"min":{"1min":0.27000000000000002,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":10.034000000000001,"5min":121.381,"15min":121.381},"last":0.42099999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0339999999999998,"5min":2.0179999999999998,"15min":2.0150000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.375,"5min":9.5589999999999993,"15min":9.5589999999999993},"last":0.435},{"interface":"front","average":{"1min":2.5600000000000001,"5min":1.8580000000000001,"15min":1.7410000000000001},"min":{"1min":0.36899999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.561999999999999,"5min":11.561999999999999,"15min":11.561999999999999},"last":0.47999999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:18:11 2023","interfaces":[{"interface":"back","average":{"1min":3.1400000000000001,"5min":5.7140000000000004,"15min":3.4620000000000002},"min":{"1min":0.27800000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":17.724,"5min":144.50299999999999,"15min":144.50299999999999},"last":0.755},{"interface":"front","average":{"1min":2.6829999999999998,"5min":5.5990000000000002,"15min":3.468},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":17.712,"5min":144.48699999999999,"15min":144.48699999999999},"last":0.75}]},{"osd":4,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":0.83099999999999996,"15min":1.3200000000000001},"min":{"1min":0.27300000000000002,"5min":0.254,"15min":0.23599999999999999},"max":{"1min":7.0570000000000004,"5min":7.0570000000000004,"15min":92.781000000000006},"last":0.372},{"interface":"front","average":{"1min":1.208,"5min":1.2110000000000001,"15min":1.4219999999999999},"min":{"1min":0.29499999999999998,"5min":0.28599999999999998,"15min":0.254},"max":{"1min":3.9569999999999999,"5min":9.7530000000000001,"15min":92.626999999999995},"last":0.39100000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":13.24,"5min":3.488,"15min":2.3759999999999999},"min":{"1min":0.28799999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":185.321,"5min":185.321,"15min":185.321},"last":0.44600000000000001},{"interface":"front","average":{"1min":11.488,"5min":2.851,"15min":1.97},"min":{"1min":0.33900000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":185.47499999999999,"5min":185.47499999999999,"15min":185.47499999999999},"last":0.49199999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:18:27 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":1.292,"15min":1.4970000000000001},"min":{"1min":0.376,"5min":0.30199999999999999,"15min":0.27300000000000002},"max":{"1min":19.782,"5min":19.782,"15min":93.105999999999995},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.7349999999999999,"5min":1.492,"15min":1.599},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28399999999999997},"max":{"1min":20.064,"5min":20.064,"15min":93.198999999999998},"last":0.46800000000000003}]}]},{"osd":6,"up_from":46,"seq":197568496000,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1101972,"kb_used_data":654284,"kb_used_omap":0,"kb_used_meta":447680,"kb_avail":92643180,"statfs":{"total":95995035648,"available":94866616320,"internally_reserved":0,"allocated":669986816,"data_stored":669023647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":458424320},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:22:52 2023","interfaces":[{"interface":"back","average":{"1min":8.843,"5min":6.0869999999999997,"15min":4.6440000000000001},"min":{"1min":0.214,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":120.392,"5min":120.392,"15min":161.64599999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":8.6180000000000003,"5min":5.5199999999999996,"15min":4.516},"min":{"1min":0.23599999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":119.917,"5min":119.917,"15min":161.38499999999999},"last":0.253}]},{"osd":1,"last update":"Sat Sep 23 23:22:52 2023","interfaces":[{"interface":"back","average":{"1min":3.234,"5min":1.9350000000000001,"15min":4.2430000000000003},"min":{"1min":0.253,"5min":0.253,"15min":0.24099999999999999},"max":{"1min":58.295999999999999,"5min":58.295999999999999,"15min":210.869},"last":0.45700000000000002},{"interface":"front","average":{"1min":3.238,"5min":1.873,"15min":4.3410000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26200000000000001},"max":{"1min":58.326999999999998,"5min":58.326999999999998,"15min":210.89699999999999},"last":0.52800000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":1.712,"15min":1.7629999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.7349999999999999,"5min":11.804,"15min":11.804},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.73799999999999999,"5min":1.7370000000000001,"15min":1.4219999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.7559999999999998,"5min":10.685,"15min":10.685},"last":0.38200000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.8109999999999999,"15min":3.0259999999999998},"min":{"1min":0.31900000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.9910000000000001,"5min":10.528,"15min":109.98699999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.885,"15min":3.161},"min":{"1min":0.32700000000000001,"5min":0.28100000000000003,"15min":0.24199999999999999},"max":{"1min":4.1970000000000001,"5min":14.573,"15min":110.024},"last":1.3839999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":2.6120000000000001,"15min":1.653},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.22900000000000001},"max":{"1min":3.0779999999999998,"5min":166.05199999999999,"15min":166.05199999999999},"last":0.318},{"interface":"front","average":{"1min":0.70799999999999996,"5min":3.367,"15min":1.9470000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":2.319,"5min":166.631,"15min":166.631},"last":0.53900000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":1.9339999999999999,"15min":1.5860000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.29999999999999999},"max":{"1min":3.113,"5min":91.899000000000001,"15min":91.899000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.65900000000000003,"5min":2.234,"15min":1.5740000000000001},"min":{"1min":0.28799999999999998,"5min":0.249,"15min":0.249},"max":{"1min":2.3559999999999999,"5min":94.174000000000007,"15min":94.174000000000007},"last":0.38900000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:18:28 2023","interfaces":[{"interface":"back","average":{"1min":2.7250000000000001,"5min":1.2689999999999999,"15min":1.27},"min":{"1min":0.36799999999999999,"5min":0.32300000000000001,"15min":0.32100000000000001},"max":{"1min":13.823,"5min":13.823,"15min":13.823},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.5609999999999999,"5min":0.79900000000000004,"15min":0.98799999999999999},"min":{"1min":0.35999999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":13.103,"5min":13.103,"15min":13.103},"last":0.54400000000000004}]}]},{"osd":1,"up_from":112,"seq":481036337341,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":715340,"kb_used_data":242820,"kb_used_omap":4,"kb_used_meta":472507,"kb_avail":93029812,"statfs":{"total":95995035648,"available":95262527488,"internally_reserved":0,"allocated":248647680,"data_stored":247101711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":483848078},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:22:52 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":3.3140000000000001,"15min":2.5510000000000002},"min":{"1min":0.16800000000000001,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":3.9620000000000002,"5min":68.540000000000006,"15min":68.540000000000006},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.60399999999999998,"5min":3.0369999999999999,"15min":2.621},"min":{"1min":0.20799999999999999,"5min":0.19600000000000001,"15min":0.192},"max":{"1min":1.6850000000000001,"5min":67.007000000000005,"15min":67.007000000000005},"last":0.40300000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:22:30 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":1.125,"15min":4.6970000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.984,"5min":12.506,"15min":25.744},"last":0.28799999999999998},{"interface":"front","average":{"1min":1.052,"5min":1.853,"15min":3.1219999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":6.5659999999999998,"5min":25.998999999999999,"15min":25.998999999999999},"last":3.3610000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:22:24 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":4.3369999999999997,"15min":3.6400000000000001},"min":{"1min":0.27300000000000002,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":3.0840000000000001,"5min":145.88499999999999,"15min":145.88499999999999},"last":1.331},{"interface":"front","average":{"1min":1.117,"5min":4.3239999999999998,"15min":3.379},"min":{"1min":0.30599999999999999,"5min":0.245,"15min":0.245},"max":{"1min":4.8490000000000002,"5min":145.93000000000001,"15min":145.93000000000001},"last":0.51300000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:22:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.3979999999999999,"15min":1.657},"min":{"1min":0.376,"5min":0.28799999999999998,"15min":0.28000000000000003},"max":{"1min":6.6120000000000001,"5min":18.474,"15min":18.474},"last":3.4380000000000002},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.171,"15min":1.766},"min":{"1min":0.30399999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.2439999999999998,"5min":12.411,"15min":25.710999999999999},"last":0.52300000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:22:52 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":5.5449999999999999,"15min":3.5329999999999999},"min":{"1min":0.40000000000000002,"5min":0.34300000000000003,"15min":0.26400000000000001},"max":{"1min":2.5219999999999998,"5min":250.33000000000001,"15min":250.33000000000001},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.131,"5min":5.9080000000000004,"15min":3.4700000000000002},"min":{"1min":0.41099999999999998,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":6.6379999999999999,"5min":250.21199999999999,"15min":250.21199999999999},"last":3.403}]},{"osd":6,"last update":"Sat Sep 23 23:22:52 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":2.27,"15min":2.1819999999999999},"min":{"1min":0.41499999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.4969999999999999,"5min":49.171999999999997,"15min":49.171999999999997},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.0509999999999999,"5min":2.3980000000000001,"15min":2.1800000000000002},"min":{"1min":0.39100000000000001,"5min":0.32800000000000001,"15min":0.32700000000000001},"max":{"1min":6.6600000000000001,"5min":49.222000000000001,"15min":49.222000000000001},"last":3.468}]},{"osd":7,"last update":"Sat Sep 23 23:18:40 2023","interfaces":[{"interface":"back","average":{"1min":3.0710000000000002,"5min":2.4540000000000002,"15min":1.9970000000000001},"min":{"1min":0.47699999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":18.532,"5min":18.532,"15min":18.532},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.3380000000000001,"15min":1.8440000000000001},"min":{"1min":0.42199999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.9710000000000001,"5min":25.667000000000002,"15min":25.667000000000002},"last":0.5}]}]},{"osd":0,"up_from":91,"seq":390842024165,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1849512,"kb_used_data":1062176,"kb_used_omap":5,"kb_used_meta":787322,"kb_avail":91895640,"statfs":{"total":95995035648,"available":94101135360,"internally_reserved":0,"allocated":1087668224,"data_stored":1086093603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":806218362},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:23:02 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":2.286,"15min":3.214},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":6.3499999999999996,"5min":21.585999999999999,"15min":155.91999999999999},"last":0.21199999999999999},{"interface":"front","average":{"1min":1.079,"5min":1.869,"15min":2.8010000000000002},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":7.1150000000000002,"5min":16.591000000000001,"15min":153.28899999999999},"last":0.25}]},{"osd":2,"last update":"Sat Sep 23 23:22:29 2023","interfaces":[{"interface":"back","average":{"1min":12.528,"5min":4.4329999999999998,"15min":3.569},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":154.952,"5min":154.952,"15min":154.952},"last":0.23999999999999999},{"interface":"front","average":{"1min":13.58,"5min":4.6950000000000003,"15min":4.3540000000000001},"min":{"1min":0.27900000000000003,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":154.922,"5min":154.922,"15min":154.922},"last":0.39200000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:22:08 2023","interfaces":[{"interface":"back","average":{"1min":3.5329999999999999,"5min":3.2029999999999998,"15min":3.444},"min":{"1min":0.26500000000000001,"5min":0.245,"15min":0.22700000000000001},"max":{"1min":13.736000000000001,"5min":35.148000000000003,"15min":82.691000000000003},"last":0.29499999999999998},{"interface":"front","average":{"1min":2.6840000000000002,"5min":2.8460000000000001,"15min":2.7480000000000002},"min":{"1min":0.30599999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.4889999999999999,"5min":43.006999999999998,"15min":82.429000000000002},"last":2.169}]},{"osd":4,"last update":"Sat Sep 23 23:22:37 2023","interfaces":[{"interface":"back","average":{"1min":1.8360000000000001,"5min":3.1640000000000001,"15min":2.2879999999999998},"min":{"1min":0.39200000000000002,"5min":0.27000000000000002,"15min":0.247},"max":{"1min":10.803000000000001,"5min":86.915999999999997,"15min":86.915999999999997},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.091,"5min":2.8559999999999999,"15min":2.2280000000000002},"min":{"1min":0.34300000000000003,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":8.2919999999999998,"5min":86.433999999999997,"15min":86.433999999999997},"last":0.377}]},{"osd":5,"last update":"Sat Sep 23 23:22:37 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.7430000000000001,"15min":1.8660000000000001},"min":{"1min":0.33300000000000002,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":8.3300000000000001,"5min":15.923,"15min":15.923},"last":0.32700000000000001},{"interface":"front","average":{"1min":1.28,"5min":1.6719999999999999,"15min":1.7390000000000001},"min":{"1min":0.42999999999999999,"5min":0.34300000000000003,"15min":0.30299999999999999},"max":{"1min":7.1769999999999996,"5min":15.929,"15min":15.929},"last":0.434}]},{"osd":6,"last update":"Sat Sep 23 23:22:37 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":2.3719999999999999,"15min":2.1200000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.4209999999999994,"5min":30.651,"15min":30.651},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.3919999999999999,"5min":2.3719999999999999,"15min":2.1200000000000001},"min":{"1min":0.39900000000000002,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":7.218,"5min":30.673999999999999,"15min":30.673999999999999},"last":0.40500000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:18:32 2023","interfaces":[{"interface":"back","average":{"1min":4.306,"5min":2.3620000000000001,"15min":2.1829999999999998},"min":{"1min":0.372,"5min":0.34499999999999997,"15min":0.29299999999999998},"max":{"1min":26.413,"5min":26.413,"15min":26.413},"last":0.42899999999999999},{"interface":"front","average":{"1min":4.7960000000000003,"5min":2.1920000000000002,"15min":2.1219999999999999},"min":{"1min":0.372,"5min":0.36499999999999999,"15min":0.313},"max":{"1min":26.437999999999999,"5min":26.437999999999999,"15min":26.437999999999999},"last":0.52600000000000002}]}]},{"osd":2,"up_from":199,"seq":854698491986,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":852536,"kb_used_data":606704,"kb_used_omap":7,"kb_used_meta":245816,"kb_avail":92892616,"statfs":{"total":95995035648,"available":95122038784,"internally_reserved":0,"allocated":621264896,"data_stored":619740020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":251715665},"hb_peers":[0,1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,0,0,8,3,3],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:22:33 2023","interfaces":[{"interface":"back","average":{"1min":10.635,"5min":4.8650000000000002,"15min":5.3109999999999999},"min":{"1min":0.16700000000000001,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":119.277,"5min":119.277,"15min":119.277},"last":0.25},{"interface":"front","average":{"1min":10.326000000000001,"5min":4.9109999999999996,"15min":5.6689999999999996},"min":{"1min":0.19800000000000001,"5min":0.185,"15min":0.185},"max":{"1min":119.56100000000001,"5min":119.56100000000001,"15min":119.56100000000001},"last":0.222}]},{"osd":1,"last update":"Sat Sep 23 23:22:35 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":1.8180000000000001,"15min":10.494},"min":{"1min":0.23000000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":6.7839999999999998,"5min":18.888000000000002,"15min":130.02799999999999},"last":0.20699999999999999},{"interface":"front","average":{"1min":1.212,"5min":1.9750000000000001,"15min":9.8369999999999997},"min":{"1min":0.26300000000000001,"5min":0.245,"15min":0.245},"max":{"1min":6.7350000000000003,"5min":18.196999999999999,"15min":130.13399999999999},"last":0.38700000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:22:35 2023","interfaces":[{"interface":"back","average":{"1min":6.952,"5min":3.5249999999999999,"15min":6.4180000000000001},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":119.366,"5min":119.366,"15min":119.366},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.99299999999999999,"5min":2.3780000000000001,"15min":5.4640000000000004},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.5590000000000002,"5min":72.412999999999997,"15min":72.412999999999997},"last":0.439}]},{"osd":4,"last update":"Sat Sep 23 23:22:35 2023","interfaces":[{"interface":"back","average":{"1min":7.016,"5min":7.9640000000000004,"15min":3.9860000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":119.88200000000001,"5min":332.98099999999999,"15min":332.98099999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.3089999999999999,"5min":6.4640000000000004,"15min":3.5299999999999998},"min":{"1min":0.33800000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.4290000000000003,"5min":332.96199999999999,"15min":332.96199999999999},"last":0.42799999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:22:35 2023","interfaces":[{"interface":"back","average":{"1min":8.1159999999999997,"5min":3.1360000000000001,"15min":2.3780000000000001},"min":{"1min":0.29499999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":119.919,"5min":119.919,"15min":119.919},"last":0.36499999999999999},{"interface":"front","average":{"1min":8.2759999999999998,"5min":3.0070000000000001,"15min":2.403},"min":{"1min":0.35499999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":119.306,"5min":119.306,"15min":119.306},"last":0.45800000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:22:35 2023","interfaces":[{"interface":"back","average":{"1min":9.0850000000000009,"5min":8.4969999999999999,"15min":4.2770000000000001},"min":{"1min":0.35799999999999998,"5min":0.31,"15min":0.31},"max":{"1min":119.943,"5min":349.32799999999997,"15min":349.32799999999997},"last":2.1520000000000001},{"interface":"front","average":{"1min":3.359,"5min":7.2240000000000002,"15min":3.8079999999999998},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":39.856999999999999,"5min":349.17700000000002,"15min":349.17700000000002},"last":0.48199999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:18:22 2023","interfaces":[{"interface":"back","average":{"1min":23.006,"5min":6.4870000000000001,"15min":3.734},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":239.768,"5min":239.768,"15min":239.768},"last":0.61499999999999999},{"interface":"front","average":{"1min":22.75,"5min":5.71,"15min":2.8700000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":239.78899999999999,"5min":239.78899999999999,"15min":239.78899999999999},"last":0.54400000000000004}]}]},{"osd":3,"up_from":148,"seq":635655159955,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":750464,"kb_used_data":323384,"kb_used_omap":2,"kb_used_meta":427069,"kb_avail":92994688,"statfs":{"total":95995035648,"available":95226560512,"internally_reserved":0,"allocated":331145216,"data_stored":329667285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":437318709},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:22:21 2023","interfaces":[{"interface":"back","average":{"1min":1.948,"5min":2.008,"15min":2.3359999999999999},"min":{"1min":0.17999999999999999,"5min":0.158,"15min":0.158},"max":{"1min":5.8949999999999996,"5min":21.427,"15min":72.778999999999996},"last":0.60099999999999998},{"interface":"front","average":{"1min":1.528,"5min":1.764,"15min":2.3500000000000001},"min":{"1min":0.218,"5min":0.193,"15min":0.17999999999999999},"max":{"1min":7.2489999999999997,"5min":23.742999999999999,"15min":72.719999999999999},"last":0.248}]},{"osd":1,"last update":"Sat Sep 23 23:22:21 2023","interfaces":[{"interface":"back","average":{"1min":1.871,"5min":2.0920000000000001,"15min":2.6890000000000001},"min":{"1min":0.28000000000000003,"5min":0.216,"15min":0.19600000000000001},"max":{"1min":5.7939999999999996,"5min":20.138999999999999,"15min":76.322999999999993},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.456,"5min":1.4379999999999999,"15min":2.4319999999999999},"min":{"1min":0.28599999999999998,"5min":0.24399999999999999,"15min":0.185},"max":{"1min":5.8220000000000001,"5min":13.112,"15min":76.444000000000003},"last":0.441}]},{"osd":2,"last update":"Sat Sep 23 23:22:33 2023","interfaces":[{"interface":"back","average":{"1min":10.446999999999999,"5min":3.98,"15min":3.8239999999999998},"min":{"1min":0.23999999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":101.279,"5min":101.279,"15min":101.279},"last":4.6509999999999998},{"interface":"front","average":{"1min":10.196999999999999,"5min":3.6560000000000001,"15min":2.3479999999999999},"min":{"1min":0.308,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":101.294,"5min":101.294,"15min":101.294},"last":2.738}]},{"osd":4,"last update":"Sat Sep 23 23:22:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.552,"15min":1.6120000000000001},"min":{"1min":0.31900000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.8620000000000001,"5min":14.754,"15min":14.754},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.5900000000000001,"5min":1.875,"15min":2.0779999999999998},"min":{"1min":0.38900000000000001,"5min":0.38400000000000001,"15min":0.28499999999999998},"max":{"1min":8.1099999999999994,"5min":9.5960000000000001,"15min":13.532999999999999},"last":0.33600000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:22:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":1.6499999999999999,"15min":2.0950000000000002},"min":{"1min":0.35899999999999999,"5min":0.32900000000000001,"15min":0.28199999999999997},"max":{"1min":7.3049999999999997,"5min":10.531000000000001,"15min":75.733000000000004},"last":0.307},{"interface":"front","average":{"1min":1.7909999999999999,"5min":1.847,"15min":2.1419999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.28399999999999997},"max":{"1min":5.3769999999999998,"5min":8.3460000000000001,"15min":75.686999999999998},"last":0.67100000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:22:21 2023","interfaces":[{"interface":"back","average":{"1min":1.3580000000000001,"5min":1.8460000000000001,"15min":1.98},"min":{"1min":0.39100000000000001,"5min":0.38800000000000001,"15min":0.32200000000000001},"max":{"1min":7.569,"5min":10.593999999999999,"15min":14.492000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.9590000000000001,"5min":2.0600000000000001,"15min":1.974},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.33300000000000002},"max":{"1min":6.7069999999999999,"5min":8.3889999999999993,"15min":9.8379999999999992},"last":0.64900000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:18:09 2023","interfaces":[{"interface":"back","average":{"1min":2.198,"5min":2.915,"15min":2.5459999999999998},"min":{"1min":0.40200000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":9.4280000000000008,"5min":130.839,"15min":130.839},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.9129999999999998,"5min":2.992,"15min":2.665},"min":{"1min":0.434,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":8.8190000000000008,"5min":130.90000000000001,"15min":130.90000000000001},"last":0.57799999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019273,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2009244,"kb_used_data":1431124,"kb_used_omap":0,"kb_used_meta":578112,"kb_avail":91735908,"statfs":{"total":95995035648,"available":93937569792,"internally_reserved":0,"allocated":1465470976,"data_stored":1464499701,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":591986688},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:22:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":7.5209999999999999,"15min":4.766},"min":{"1min":0.218,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":4.0229999999999997,"5min":199.64400000000001,"15min":238.62200000000001},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.877,"5min":8.0310000000000006,"15min":5.202},"min":{"1min":0.245,"5min":0.245,"15min":0.23300000000000001},"max":{"1min":4.0709999999999997,"5min":195.649,"15min":238.58199999999999},"last":0.98999999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:22:59 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":3.1930000000000001,"15min":4.8150000000000004},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.1840000000000002,"5min":161.923,"15min":175.00899999999999},"last":0.23100000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":3.1960000000000002,"15min":4.968},"min":{"1min":0.27500000000000002,"5min":0.26500000000000001,"15min":0.25800000000000001},"max":{"1min":2.04,"5min":161.39699999999999,"15min":174.821},"last":0.34399999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:22:31 2023","interfaces":[{"interface":"back","average":{"1min":2.1749999999999998,"5min":1.9670000000000001,"15min":3.008},"min":{"1min":0.27500000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":27.148,"5min":27.148,"15min":39.308},"last":0.39300000000000002},{"interface":"front","average":{"1min":2.6640000000000001,"5min":2.0739999999999998,"15min":2.464},"min":{"1min":0.32500000000000001,"5min":0.316,"15min":0.316},"max":{"1min":27.117000000000001,"5min":27.117000000000001,"15min":27.117000000000001},"last":0.42799999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:22:25 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":2.9889999999999999,"15min":4.0979999999999999},"min":{"1min":0.32800000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.165,"5min":152.04499999999999,"15min":303.41500000000002},"last":1.375},{"interface":"front","average":{"1min":1.6799999999999999,"5min":3.4329999999999998,"15min":4.4740000000000002},"min":{"1min":0.317,"5min":0.308,"15min":0.308},"max":{"1min":6.2789999999999999,"5min":151.75299999999999,"15min":303.428},"last":1.1040000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:22:59 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":1.0389999999999999,"15min":1.5369999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.23699999999999999},"max":{"1min":0.50600000000000001,"5min":5.8330000000000002,"15min":160.79499999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.51300000000000001,"5min":1.262,"15min":1.794},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":2.121,"5min":7.9669999999999996,"15min":160.61099999999999},"last":0.38}]},{"osd":6,"last update":"Sat Sep 23 23:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.89700000000000002,"15min":1.0629999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.8820000000000001,"5min":6.0910000000000002,"15min":9.3350000000000009},"last":0.44},{"interface":"front","average":{"1min":1.093,"5min":1.252,"15min":1.224},"min":{"1min":0.32700000000000001,"5min":0.30599999999999999,"15min":0.27500000000000002},"max":{"1min":7.1219999999999999,"5min":7.1219999999999999,"15min":13.888999999999999},"last":0.40999999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:18:35 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.5609999999999999,"15min":1.3140000000000001},"min":{"1min":0.36299999999999999,"5min":0.315,"15min":0.30499999999999999},"max":{"1min":7.4850000000000003,"5min":14.842000000000001,"15min":14.842000000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.0169999999999999,"5min":1.502,"15min":1.341},"min":{"1min":0.433,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.882,"5min":14.561,"15min":14.561},"last":0.378}]}]},{"osd":5,"up_from":38,"seq":163208757636,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":912004,"kb_used_data":397820,"kb_used_omap":0,"kb_used_meta":514176,"kb_avail":92833148,"statfs":{"total":95995035648,"available":95061143552,"internally_reserved":0,"allocated":407367680,"data_stored":406416388,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":526516224},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:22:42 2023","interfaces":[{"interface":"back","average":{"1min":17.654,"5min":6.9450000000000003,"15min":3.5430000000000001},"min":{"1min":0.23499999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":113.869,"5min":113.869,"15min":113.869},"last":0.27500000000000002},{"interface":"front","average":{"1min":17.544,"5min":7.0490000000000004,"15min":3.798},"min":{"1min":0.26000000000000001,"5min":0.23000000000000001,"15min":0.22700000000000001},"max":{"1min":113.398,"5min":113.398,"15min":113.398},"last":0.39000000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:22:53 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":2.2200000000000002,"15min":3.7549999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.246},"max":{"1min":1.8300000000000001,"5min":52.582000000000001,"15min":193.584},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.63600000000000001,"5min":2.5459999999999998,"15min":3.9870000000000001},"min":{"1min":0.29399999999999998,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":2.9260000000000002,"5min":54.814999999999998,"15min":194.03100000000001},"last":0.25900000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:22:31 2023","interfaces":[{"interface":"back","average":{"1min":5.8209999999999997,"5min":2.532,"15min":1.8120000000000001},"min":{"1min":0.32200000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":86.765000000000001,"5min":86.765000000000001,"15min":86.765000000000001},"last":0.29599999999999999},{"interface":"front","average":{"1min":5.4500000000000002,"5min":2.1509999999999998,"15min":1.871},"min":{"1min":0.32100000000000001,"5min":0.30099999999999999,"15min":0.25800000000000001},"max":{"1min":86.837000000000003,"5min":86.837000000000003,"15min":86.837000000000003},"last":0.377}]},{"osd":3,"last update":"Sat Sep 23 23:22:26 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":4.4009999999999998,"15min":4.085},"min":{"1min":0.40899999999999997,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":13.664,"5min":159.53299999999999,"15min":159.53299999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.625,"5min":4.8369999999999997,"15min":4.7009999999999996},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.312},"max":{"1min":13.715999999999999,"5min":159.54900000000001,"15min":159.54900000000001},"last":0.43099999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:22:09 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.224,"15min":1.3149999999999999},"min":{"1min":0.29999999999999999,"5min":0.24299999999999999,"15min":0.221},"max":{"1min":8.1210000000000004,"5min":8.1210000000000004,"15min":14.538},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.89700000000000002,"15min":0.92400000000000004},"min":{"1min":0.26400000000000001,"5min":0.26100000000000001,"15min":0.25},"max":{"1min":1.6180000000000001,"5min":11.923999999999999,"15min":11.923999999999999},"last":0.315}]},{"osd":6,"last update":"Sat Sep 23 23:22:09 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":4.282,"15min":2.2690000000000001},"min":{"1min":0.33500000000000002,"5min":0.28299999999999997,"15min":0.255},"max":{"1min":1.5940000000000001,"5min":310.68799999999999,"15min":310.68799999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.0880000000000001,"5min":4.7990000000000004,"15min":2.5150000000000001},"min":{"1min":0.28599999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.3230000000000004,"5min":311.45400000000001,"15min":311.45400000000001},"last":0.34799999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:18:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2010000000000001,"5min":1.286,"15min":1.3700000000000001},"min":{"1min":0.36699999999999999,"5min":0.32700000000000001,"15min":0.29199999999999998},"max":{"1min":12.09,"5min":12.09,"15min":14.929},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.319,"15min":1.276},"min":{"1min":0.371,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":5.7350000000000003,"5min":6.7089999999999996,"15min":15.090999999999999},"last":0.54400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1081872384,"data_stored":1081868312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":244121600,"data_stored":244121600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":610598912,"data_stored":610598912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":319815680,"data_stored":319815680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1449529344,"data_stored":1449525272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":395640832,"data_stored":395640832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":646909952,"data_stored":646905880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:23:08.528 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 31 2023-09-23T23:23:08.551 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:08.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:08.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:08.677 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:23:08.787 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:08.813 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:08 smithi110 ceph-mon[221102]: pgmap v995: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 1.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 1146/57621 objects degraded (1.989%); 4970/57621 objects misplaced (8.625%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:23:08.814 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:08 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1146/57621 objects degraded (1.989%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:08.814 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:08 smithi110 ceph-mon[212939]: pgmap v995: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 1.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 1146/57621 objects degraded (1.989%); 4970/57621 objects misplaced (8.625%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:23:08.814 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:08 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1146/57621 objects degraded (1.989%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:08.820 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:08.868 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:08.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:08 smithi138 ceph-mon[193606]: pgmap v995: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 1.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 1146/57621 objects degraded (1.989%); 4970/57621 objects misplaced (8.625%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:23:08.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:08 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1146/57621 objects degraded (1.989%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:09.007 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:23:09.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:09.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:23:09.116 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:23:09.149 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:09.218 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:23:09.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:09.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:23:09.325 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:09.401 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:09.574 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:09.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:09.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:09.680 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:09.779 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:09.972 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:09.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:09.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:10.089 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:10.152 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:10.417 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:10.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:10.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:10.522 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:23:10.550 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:10.604 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:23:10.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:10.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:23:10.676 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:23:10.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:10 smithi110 ceph-mon[212939]: pgmap v996: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 1.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 937/51333 objects degraded (1.825%); 4734/51333 objects misplaced (9.222%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:23:10.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:10 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/691174561' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:23:10.694 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:10 smithi110 ceph-mon[221102]: pgmap v996: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 1.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 937/51333 objects degraded (1.825%); 4734/51333 objects misplaced (9.222%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:23:10.694 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:10 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/691174561' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:23:10.711 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:10.741 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:10.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:10 smithi138 ceph-mon[193606]: pgmap v996: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 1.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 937/51333 objects degraded (1.825%); 4734/51333 objects misplaced (9.222%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:23:10.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:10 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/691174561' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:23:11.758 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:11.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:11.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:11.867 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:11.940 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:11 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:23:11.940 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:11 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/691174561' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:23:11.941 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:11 smithi110 ceph-mon[221102]: osdmap e338: 8 total, 7 up, 8 in 2023-09-23T23:23:11.941 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:11 smithi110 ceph-mon[221102]: failed to encode map e338 with expected crc 2023-09-23T23:23:11.941 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:11 smithi110 ceph-mon[221102]: failed to encode map e338 with expected crc 2023-09-23T23:23:11.941 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:11 smithi110 ceph-mon[221102]: failed to encode map e338 with expected crc 2023-09-23T23:23:11.941 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:11 smithi110 ceph-mon[221102]: 1.f scrub starts 2023-09-23T23:23:11.941 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:11 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:23:11.941 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:11 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/691174561' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:23:11.941 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:11 smithi110 ceph-mon[212939]: osdmap e338: 8 total, 7 up, 8 in 2023-09-23T23:23:11.941 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:11 smithi110 ceph-mon[212939]: failed to encode map e338 with expected crc 2023-09-23T23:23:11.942 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:11 smithi110 ceph-mon[212939]: failed to encode map e338 with expected crc 2023-09-23T23:23:11.942 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:11 smithi110 ceph-mon[212939]: failed to encode map e338 with expected crc 2023-09-23T23:23:11.942 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:11 smithi110 ceph-mon[212939]: 1.f scrub starts 2023-09-23T23:23:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:11 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:23:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:11 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/691174561' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:23:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:11 smithi138 ceph-mon[193606]: osdmap e338: 8 total, 7 up, 8 in 2023-09-23T23:23:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:11 smithi138 ceph-mon[193606]: failed to encode map e338 with expected crc 2023-09-23T23:23:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:11 smithi138 ceph-mon[193606]: failed to encode map e338 with expected crc 2023-09-23T23:23:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:11 smithi138 ceph-mon[193606]: failed to encode map e338 with expected crc 2023-09-23T23:23:11.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:11 smithi138 ceph-mon[193606]: 1.f scrub starts 2023-09-23T23:23:11.967 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:12.188 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:12.253 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:12.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:12.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:12.371 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:12.443 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:12.647 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:12.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:12.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:12.708 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:23:12.708 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:23:12.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:23:12.708 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:23:12.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156658 10.3 10:c8345f3f:::benchmark_data_smithi110_366450_object55450:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e301)", 2023-09-23T23:23:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:50.036433+0000", 2023-09-23T23:23:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 82.666748948999995, 2023-09-23T23:23:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2122256870000001, 2023-09-23T23:23:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156658 2023-09-23T23:23:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036433+0000", 2023-09-23T23:23:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036433+0000", 2023-09-23T23:23:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036432+0000", 2023-09-23T23:23:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:23:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036438+0000", 2023-09-23T23:23:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2260000000000002e-06 2023-09-23T23:23:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036447+0000", 2023-09-23T23:23:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2599999999999994e-06 2023-09-23T23:23:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036454+0000", 2023-09-23T23:23:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2180000000000002e-06 2023-09-23T23:23:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038082+0000", 2023-09-23T23:23:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016271840000000001 2023-09-23T23:23:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:23:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038092+0000", 2023-09-23T23:23:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0312999999999999e-05 2023-09-23T23:23:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.132541+0000", 2023-09-23T23:23:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.094449464 2023-09-23T23:23:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.135895+0000", 2023-09-23T23:23:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033539080000000001 2023-09-23T23:23:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.880055+0000", 2023-09-23T23:23:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.744159365 2023-09-23T23:23:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.883944+0000", 2023-09-23T23:23:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038893600000000001 2023-09-23T23:23:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.224607+0000", 2023-09-23T23:23:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.34066281199999998 2023-09-23T23:23:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.227336+0000", 2023-09-23T23:23:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002729553 2023-09-23T23:23:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.232121+0000", 2023-09-23T23:23:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047848830000000002 2023-09-23T23:23:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.238887+0000", 2023-09-23T23:23:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.006765971 2023-09-23T23:23:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.240149+0000", 2023-09-23T23:23:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001261543 2023-09-23T23:23:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.240604+0000", 2023-09-23T23:23:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045544900000000002 2023-09-23T23:23:12.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.242329+0000", 2023-09-23T23:23:12.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001724864 2023-09-23T23:23:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.248659+0000", 2023-09-23T23:23:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063300179999999998 2023-09-23T23:23:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156665 10.3 10:ce3a78fd:::benchmark_data_smithi110_366450_object55457:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e301)", 2023-09-23T23:23:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:50.036463+0000", 2023-09-23T23:23:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 82.666718880999994, 2023-09-23T23:23:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2684505270000002, 2023-09-23T23:23:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156665 2023-09-23T23:23:12.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.721 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036463+0000", 2023-09-23T23:23:12.721 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036463+0000", 2023-09-23T23:23:12.721 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.722 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.722 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.722 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036463+0000", 2023-09-23T23:23:12.722 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:23:12.722 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.722 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.722 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.722 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036465+0000", 2023-09-23T23:23:12.722 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4990000000000001e-06 2023-09-23T23:23:12.722 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.722 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.722 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.723 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036471+0000", 2023-09-23T23:23:12.723 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8850000000000004e-06 2023-09-23T23:23:12.723 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.723 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.723 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.723 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036477+0000", 2023-09-23T23:23:12.723 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.857e-06 2023-09-23T23:23:12.723 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.723 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.723 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.724 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038117+0000", 2023-09-23T23:23:12.724 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016396080000000001 2023-09-23T23:23:12.724 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.724 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.724 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:23:12.724 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038124+0000", 2023-09-23T23:23:12.724 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8190000000000003e-06 2023-09-23T23:23:12.724 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.724 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.724 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.724 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.887950+0000", 2023-09-23T23:23:12.724 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8498266559999998 2023-09-23T23:23:12.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.891672+0000", 2023-09-23T23:23:12.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037213459999999999 2023-09-23T23:23:12.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.909743+0000", 2023-09-23T23:23:12.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018071329000000001 2023-09-23T23:23:12.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.913176+0000", 2023-09-23T23:23:12.726 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034328200000000001 2023-09-23T23:23:12.726 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.256612+0000", 2023-09-23T23:23:12.726 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.343436199 2023-09-23T23:23:12.726 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.259680+0000", 2023-09-23T23:23:12.727 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030675870000000001 2023-09-23T23:23:12.727 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.727 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.727 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.727 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.265203+0000", 2023-09-23T23:23:12.727 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055235350000000004 2023-09-23T23:23:12.727 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.727 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.727 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.727 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.293752+0000", 2023-09-23T23:23:12.727 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028548720999999999 2023-09-23T23:23:12.727 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.728 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.728 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.728 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.295411+0000", 2023-09-23T23:23:12.728 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001659326 2023-09-23T23:23:12.728 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.728 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.728 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.728 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.296790+0000", 2023-09-23T23:23:12.728 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013783339999999999 2023-09-23T23:23:12.728 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.728 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.728 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.298340+0000", 2023-09-23T23:23:12.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015504679999999999 2023-09-23T23:23:12.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.304914+0000", 2023-09-23T23:23:12.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00657399 2023-09-23T23:23:12.729 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.729 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.729 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.730 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156677 10.3 10:c754b68c:::benchmark_data_smithi110_366450_object55469:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e301)", 2023-09-23T23:23:12.730 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:50.036483+0000", 2023-09-23T23:23:12.730 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 82.666699027999996, 2023-09-23T23:23:12.730 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2794486950000001, 2023-09-23T23:23:12.730 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.730 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.730 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.730 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.730 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.730 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156677 2023-09-23T23:23:12.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.730 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036483+0000", 2023-09-23T23:23:12.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036483+0000", 2023-09-23T23:23:12.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.732 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036483+0000", 2023-09-23T23:23:12.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:23:12.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.732 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.732 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036485+0000", 2023-09-23T23:23:12.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1509999999999998e-06 2023-09-23T23:23:12.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.732 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.732 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036491+0000", 2023-09-23T23:23:12.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1029999999999998e-06 2023-09-23T23:23:12.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036497+0000", 2023-09-23T23:23:12.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6699999999999999e-06 2023-09-23T23:23:12.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038146+0000", 2023-09-23T23:23:12.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001649614 2023-09-23T23:23:12.734 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.734 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:23:12.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038153+0000", 2023-09-23T23:23:12.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7120000000000003e-06 2023-09-23T23:23:12.734 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.734 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.892021+0000", 2023-09-23T23:23:12.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8538682949999998 2023-09-23T23:23:12.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.895469+0000", 2023-09-23T23:23:12.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034472249999999999 2023-09-23T23:23:12.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.925797+0000", 2023-09-23T23:23:12.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.030328755999999998 2023-09-23T23:23:12.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.929104+0000", 2023-09-23T23:23:12.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033061599999999998 2023-09-23T23:23:12.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.279422+0000", 2023-09-23T23:23:12.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.35031821000000002 2023-09-23T23:23:12.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.282430+0000", 2023-09-23T23:23:12.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030082590000000001 2023-09-23T23:23:12.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.737 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.737 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.287816+0000", 2023-09-23T23:23:12.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053855860000000004 2023-09-23T23:23:12.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.737 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.737 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.305266+0000", 2023-09-23T23:23:12.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017450195000000002 2023-09-23T23:23:12.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.306434+0000", 2023-09-23T23:23:12.738 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011678719999999999 2023-09-23T23:23:12.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.307444+0000", 2023-09-23T23:23:12.738 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010101159999999999 2023-09-23T23:23:12.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.309131+0000", 2023-09-23T23:23:12.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001687207 2023-09-23T23:23:12.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.739 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.315932+0000", 2023-09-23T23:23:12.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068009699999999999 2023-09-23T23:23:12.739 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.739 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.739 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.739 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156689 10.3 10:c3cc188a:::benchmark_data_smithi110_366450_object55481:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e301)", 2023-09-23T23:23:12.740 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:50.036502+0000", 2023-09-23T23:23:12.740 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 82.666680650999993, 2023-09-23T23:23:12.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.470265768, 2023-09-23T23:23:12.740 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.740 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.740 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.740 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.740 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.740 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156689 2023-09-23T23:23:12.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.740 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.740 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.741 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.741 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036502+0000", 2023-09-23T23:23:12.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.741 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.741 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036502+0000", 2023-09-23T23:23:12.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.741 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.741 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036501+0000", 2023-09-23T23:23:12.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:23:12.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036503+0000", 2023-09-23T23:23:12.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1349999999999999e-06 2023-09-23T23:23:12.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036509+0000", 2023-09-23T23:23:12.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9970000000000004e-06 2023-09-23T23:23:12.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036515+0000", 2023-09-23T23:23:12.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7819999999999999e-06 2023-09-23T23:23:12.743 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038348+0000", 2023-09-23T23:23:12.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018323930000000001 2023-09-23T23:23:12.743 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:23:12.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038356+0000", 2023-09-23T23:23:12.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5450000000000003e-06 2023-09-23T23:23:12.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.930347+0000", 2023-09-23T23:23:12.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.891991215 2023-09-23T23:23:12.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.933759+0000", 2023-09-23T23:23:12.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034111269999999999 2023-09-23T23:23:12.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.956094+0000", 2023-09-23T23:23:12.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022335738000000001 2023-09-23T23:23:12.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.960338+0000", 2023-09-23T23:23:12.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042442030000000002 2023-09-23T23:23:12.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.746 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.746 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.464888+0000", 2023-09-23T23:23:12.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.50454982800000003 2023-09-23T23:23:12.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.746 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.746 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.467869+0000", 2023-09-23T23:23:12.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029807459999999998 2023-09-23T23:23:12.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.473841+0000", 2023-09-23T23:23:12.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059724180000000002 2023-09-23T23:23:12.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.496377+0000", 2023-09-23T23:23:12.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022535192999999999 2023-09-23T23:23:12.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.497644+0000", 2023-09-23T23:23:12.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00126756 2023-09-23T23:23:12.748 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.748 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.748 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.498689+0000", 2023-09-23T23:23:12.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010452650000000001 2023-09-23T23:23:12.748 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.748 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.748 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.500547+0000", 2023-09-23T23:23:12.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018575410000000001 2023-09-23T23:23:12.748 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.749 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.506767+0000", 2023-09-23T23:23:12.749 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0062204340000000004 2023-09-23T23:23:12.749 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.749 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.749 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.749 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156699 10.3 10:c7e78881:::benchmark_data_smithi110_366450_object55491:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:23:12.749 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.085948+0000", 2023-09-23T23:23:12.749 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.617234202000006, 2023-09-23T23:23:12.749 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.459542565, 2023-09-23T23:23:12.750 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.750 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.750 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.750 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.750 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.750 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156699 2023-09-23T23:23:12.750 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.750 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.750 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.750 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085948+0000", 2023-09-23T23:23:12.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.751 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085948+0000", 2023-09-23T23:23:12.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.751 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085946+0000", 2023-09-23T23:23:12.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:23:12.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.752 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085954+0000", 2023-09-23T23:23:12.752 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8690000000000004e-06 2023-09-23T23:23:12.752 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.752 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.752 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.752 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085969+0000", 2023-09-23T23:23:12.752 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4579999999999999e-05 2023-09-23T23:23:12.752 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.752 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.752 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.752 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085979+0000", 2023-09-23T23:23:12.753 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6730000000000004e-06 2023-09-23T23:23:12.753 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.753 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.753 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.753 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.934188+0000", 2023-09-23T23:23:12.753 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.848209207 2023-09-23T23:23:12.753 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.753 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.753 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.753 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.938192+0000", 2023-09-23T23:23:12.754 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040045879999999999 2023-09-23T23:23:12.754 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.754 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.974891+0000", 2023-09-23T23:23:12.754 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036698716999999999 2023-09-23T23:23:12.754 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.754 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.978909+0000", 2023-09-23T23:23:12.754 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040178669999999996 2023-09-23T23:23:12.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.755 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.755 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.755 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.515734+0000", 2023-09-23T23:23:12.755 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.53682527300000005 2023-09-23T23:23:12.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.755 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.755 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.755 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.518894+0000", 2023-09-23T23:23:12.755 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031600869999999998 2023-09-23T23:23:12.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.755 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.756 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.756 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.524144+0000", 2023-09-23T23:23:12.756 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052494250000000003 2023-09-23T23:23:12.756 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.756 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.756 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.756 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.535043+0000", 2023-09-23T23:23:12.756 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010899067 2023-09-23T23:23:12.756 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.756 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.756 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.756 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.535410+0000", 2023-09-23T23:23:12.757 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036688299999999998 2023-09-23T23:23:12.757 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.757 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.757 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.757 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.536594+0000", 2023-09-23T23:23:12.757 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001184095 2023-09-23T23:23:12.757 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.757 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.757 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.757 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.538475+0000", 2023-09-23T23:23:12.757 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018812639999999999 2023-09-23T23:23:12.757 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.758 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.758 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.758 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.545491+0000", 2023-09-23T23:23:12.758 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070156009999999998 2023-09-23T23:23:12.758 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.758 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.758 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.758 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.758 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.758 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156707 10.3 10:ca34f172:::benchmark_data_smithi110_366450_object55499:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:23:12.758 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.101757+0000", 2023-09-23T23:23:12.759 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.601425763999998, 2023-09-23T23:23:12.759 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5204141440000001, 2023-09-23T23:23:12.759 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.759 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.759 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.759 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.759 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.759 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156707 2023-09-23T23:23:12.759 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.759 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.759 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.759 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101757+0000", 2023-09-23T23:23:12.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.760 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101757+0000", 2023-09-23T23:23:12.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.760 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101754+0000", 2023-09-23T23:23:12.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-09-23T23:23:12.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101763+0000", 2023-09-23T23:23:12.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1020000000000003e-06 2023-09-23T23:23:12.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101778+0000", 2023-09-23T23:23:12.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4268e-05 2023-09-23T23:23:12.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101787+0000", 2023-09-23T23:23:12.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7580000000000004e-06 2023-09-23T23:23:12.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.939584+0000", 2023-09-23T23:23:12.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8377970509999999 2023-09-23T23:23:12.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.943370+0000", 2023-09-23T23:23:12.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037859840000000001 2023-09-23T23:23:12.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.991414+0000", 2023-09-23T23:23:12.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048043296999999999 2023-09-23T23:23:12.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.994689+0000", 2023-09-23T23:23:12.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032749989999999998 2023-09-23T23:23:12.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.572368+0000", 2023-09-23T23:23:12.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.57767970499999999 2023-09-23T23:23:12.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.575376+0000", 2023-09-23T23:23:12.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030079059999999999 2023-09-23T23:23:12.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.580798+0000", 2023-09-23T23:23:12.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054212740000000002 2023-09-23T23:23:12.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.612099+0000", 2023-09-23T23:23:12.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031301739000000002 2023-09-23T23:23:12.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.612481+0000", 2023-09-23T23:23:12.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00038158799999999998 2023-09-23T23:23:12.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.766 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.613567+0000", 2023-09-23T23:23:12.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010858490000000001 2023-09-23T23:23:12.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.766 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.615343+0000", 2023-09-23T23:23:12.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017767620000000001 2023-09-23T23:23:12.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.622171+0000", 2023-09-23T23:23:12.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068272410000000004 2023-09-23T23:23:12.767 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.767 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.767 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.767 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPush(10.3 302/301 [PushOp(10:c8345f3f:::benchmark_data_smithi110_366450_object55450:head, version: 272'3485, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:c8345f3f:::benchmark_data_smithi110_366450_object55450:head@272'3485, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:23:12.767 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.136769+0000", 2023-09-23T23:23:12.768 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.566413402999999, 2023-09-23T23:23:12.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7428203010000001, 2023-09-23T23:23:12.768 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.768 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:23:12.768 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.768 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.768 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.768 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136769+0000", 2023-09-23T23:23:12.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.768 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.768 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.769 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136769+0000", 2023-09-23T23:23:12.769 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.769 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.769 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.769 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.769 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136765+0000", 2023-09-23T23:23:12.769 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999962 2023-09-23T23:23:12.769 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.769 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.769 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.769 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136850+0000", 2023-09-23T23:23:12.769 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5081999999999996e-05 2023-09-23T23:23:12.769 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.770 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.770 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136868+0000", 2023-09-23T23:23:12.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8357000000000001e-05 2023-09-23T23:23:12.770 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.770 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.770 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136884+0000", 2023-09-23T23:23:12.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5732000000000001e-05 2023-09-23T23:23:12.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.867402+0000", 2023-09-23T23:23:12.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7305173149999999 2023-09-23T23:23:12.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.867705+0000", 2023-09-23T23:23:12.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030383999999999999 2023-09-23T23:23:12.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.879589+0000", 2023-09-23T23:23:12.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011883874000000001 2023-09-23T23:23:12.772 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.773 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.773 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.773 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156724 10.3 10:ca0db88d:::benchmark_data_smithi110_366450_object55516:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:23:12.773 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.159299+0000", 2023-09-23T23:23:12.773 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.543882947, 2023-09-23T23:23:12.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.473675901, 2023-09-23T23:23:12.773 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.774 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.774 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.774 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.774 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.774 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156724 2023-09-23T23:23:12.774 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.774 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159299+0000", 2023-09-23T23:23:12.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159299+0000", 2023-09-23T23:23:12.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159297+0000", 2023-09-23T23:23:12.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:23:12.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159310+0000", 2023-09-23T23:23:12.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2174000000000001e-05 2023-09-23T23:23:12.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.777 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.777 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159324+0000", 2023-09-23T23:23:12.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4511999999999999e-05 2023-09-23T23:23:12.777 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.778 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.778 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159334+0000", 2023-09-23T23:23:12.778 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7480000000000005e-06 2023-09-23T23:23:12.778 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.996054+0000", 2023-09-23T23:23:12.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8367205470000001 2023-09-23T23:23:12.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.999367+0000", 2023-09-23T23:23:12.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033122659999999999 2023-09-23T23:23:12.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.780 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.019886+0000", 2023-09-23T23:23:12.780 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02051973 2023-09-23T23:23:12.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.780 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.023203+0000", 2023-09-23T23:23:12.780 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003316865 2023-09-23T23:23:12.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.780 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.595994+0000", 2023-09-23T23:23:12.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.57279082400000003 2023-09-23T23:23:12.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.598978+0000", 2023-09-23T23:23:12.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002984062 2023-09-23T23:23:12.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.604389+0000", 2023-09-23T23:23:12.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054107959999999998 2023-09-23T23:23:12.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.782 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.622507+0000", 2023-09-23T23:23:12.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018117985 2023-09-23T23:23:12.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.782 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.623486+0000", 2023-09-23T23:23:12.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0009786860000000001 2023-09-23T23:23:12.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.624625+0000", 2023-09-23T23:23:12.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011391330000000001 2023-09-23T23:23:12.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.626256+0000", 2023-09-23T23:23:12.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016308449999999999 2023-09-23T23:23:12.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.632975+0000", 2023-09-23T23:23:12.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.006719727 2023-09-23T23:23:12.784 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.784 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.784 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.784 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156745 10.3 10:c3a25812:::benchmark_data_smithi110_366450_object55537:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:23:12.784 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.191485+0000", 2023-09-23T23:23:12.784 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.511697346999995, 2023-09-23T23:23:12.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4810129569999999, 2023-09-23T23:23:12.784 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.784 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.784 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.785 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.785 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.785 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156745 2023-09-23T23:23:12.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.785 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191485+0000", 2023-09-23T23:23:12.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191485+0000", 2023-09-23T23:23:12.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191483+0000", 2023-09-23T23:23:12.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:23:12.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191491+0000", 2023-09-23T23:23:12.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1079999999999993e-06 2023-09-23T23:23:12.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191512+0000", 2023-09-23T23:23:12.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0267000000000001e-05 2023-09-23T23:23:12.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191521+0000", 2023-09-23T23:23:12.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2480000000000002e-06 2023-09-23T23:23:12.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.999765+0000", 2023-09-23T23:23:12.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8082440449999999 2023-09-23T23:23:12.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.003124+0000", 2023-09-23T23:23:12.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033588789999999999 2023-09-23T23:23:12.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.036625+0000", 2023-09-23T23:23:12.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.03350144 2023-09-23T23:23:12.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.040838+0000", 2023-09-23T23:23:12.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042131260000000002 2023-09-23T23:23:12.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.641997+0000", 2023-09-23T23:23:12.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.60115844900000004 2023-09-23T23:23:12.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.645336+0000", 2023-09-23T23:23:12.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003339482 2023-09-23T23:23:12.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.651259+0000", 2023-09-23T23:23:12.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059230419999999999 2023-09-23T23:23:12.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.661663+0000", 2023-09-23T23:23:12.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010403302 2023-09-23T23:23:12.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.791 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.662676+0000", 2023-09-23T23:23:12.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010129189999999999 2023-09-23T23:23:12.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.791 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.663859+0000", 2023-09-23T23:23:12.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011834149999999999 2023-09-23T23:23:12.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.665740+0000", 2023-09-23T23:23:12.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001880545 2023-09-23T23:23:12.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.672498+0000", 2023-09-23T23:23:12.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067584250000000002 2023-09-23T23:23:12.792 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.792 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.792 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.793 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156758 10.3 10:c6bd9813:::benchmark_data_smithi110_366450_object55550:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:23:12.793 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.205665+0000", 2023-09-23T23:23:12.793 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.497517385999998, 2023-09-23T23:23:12.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6487948870000002, 2023-09-23T23:23:12.793 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.793 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.793 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.793 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.793 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.793 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156758 2023-09-23T23:23:12.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.794 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205665+0000", 2023-09-23T23:23:12.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205665+0000", 2023-09-23T23:23:12.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205664+0000", 2023-09-23T23:23:12.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:23:12.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205670+0000", 2023-09-23T23:23:12.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3929999999999996e-06 2023-09-23T23:23:12.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205683+0000", 2023-09-23T23:23:12.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3288999999999999e-05 2023-09-23T23:23:12.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205690+0000", 2023-09-23T23:23:12.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1210000000000001e-06 2023-09-23T23:23:12.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.004368+0000", 2023-09-23T23:23:12.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.798677251 2023-09-23T23:23:12.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.007617+0000", 2023-09-23T23:23:12.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032493769999999999 2023-09-23T23:23:12.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.056183+0000", 2023-09-23T23:23:12.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048565482 2023-09-23T23:23:12.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.060370+0000", 2023-09-23T23:23:12.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041874099999999999 2023-09-23T23:23:12.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.811482+0000", 2023-09-23T23:23:12.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.751111588 2023-09-23T23:23:12.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.814815+0000", 2023-09-23T23:23:12.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033331239999999998 2023-09-23T23:23:12.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.819734+0000", 2023-09-23T23:23:12.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0049189960000000001 2023-09-23T23:23:12.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.844633+0000", 2023-09-23T23:23:12.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024899787999999999 2023-09-23T23:23:12.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.845534+0000", 2023-09-23T23:23:12.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00090030399999999999 2023-09-23T23:23:12.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.846942+0000", 2023-09-23T23:23:12.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001408573 2023-09-23T23:23:12.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.848519+0000", 2023-09-23T23:23:12.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001576521 2023-09-23T23:23:12.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.854460+0000", 2023-09-23T23:23:12.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059410060000000004 2023-09-23T23:23:12.801 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.801 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.801 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.801 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156771 10.3 10:c6f4a9de:::benchmark_data_smithi110_366450_object55563:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:23:12.801 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.219050+0000", 2023-09-23T23:23:12.801 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.484132509000005, 2023-09-23T23:23:12.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6446795810000001, 2023-09-23T23:23:12.801 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.801 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.801 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.802 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.802 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.802 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156771 2023-09-23T23:23:12.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.802 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.802 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.802 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219050+0000", 2023-09-23T23:23:12.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.802 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.802 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219050+0000", 2023-09-23T23:23:12.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219048+0000", 2023-09-23T23:23:12.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:23:12.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219054+0000", 2023-09-23T23:23:12.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8950000000000003e-06 2023-09-23T23:23:12.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219068+0000", 2023-09-23T23:23:12.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4149e-05 2023-09-23T23:23:12.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219076+0000", 2023-09-23T23:23:12.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8960000000000003e-06 2023-09-23T23:23:12.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.070783+0000", 2023-09-23T23:23:12.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8517066980000001 2023-09-23T23:23:12.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.074340+0000", 2023-09-23T23:23:12.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003557185 2023-09-23T23:23:12.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.095850+0000", 2023-09-23T23:23:12.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021510344000000001 2023-09-23T23:23:12.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.099086+0000", 2023-09-23T23:23:12.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032357319999999998 2023-09-23T23:23:12.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.831720+0000", 2023-09-23T23:23:12.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.732633639 2023-09-23T23:23:12.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.834388+0000", 2023-09-23T23:23:12.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026684980000000001 2023-09-23T23:23:12.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.839278+0000", 2023-09-23T23:23:12.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048894430000000003 2023-09-23T23:23:12.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.846624+0000", 2023-09-23T23:23:12.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0073457380000000001 2023-09-23T23:23:12.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.854781+0000", 2023-09-23T23:23:12.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081573540000000003 2023-09-23T23:23:12.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.855781+0000", 2023-09-23T23:23:12.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00099972499999999992 2023-09-23T23:23:12.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.857550+0000", 2023-09-23T23:23:12.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001769135 2023-09-23T23:23:12.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.863729+0000", 2023-09-23T23:23:12.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061796489999999997 2023-09-23T23:23:12.809 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.809 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.809 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.810 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156792 10.3 10:c9793c14:::benchmark_data_smithi110_366450_object55584:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:23:12.810 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.244861+0000", 2023-09-23T23:23:12.810 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.458321118000001, 2023-09-23T23:23:12.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.66926905, 2023-09-23T23:23:12.810 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.810 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.810 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.810 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.810 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.810 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156792 2023-09-23T23:23:12.810 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.811 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244861+0000", 2023-09-23T23:23:12.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244861+0000", 2023-09-23T23:23:12.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244859+0000", 2023-09-23T23:23:12.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:23:12.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244868+0000", 2023-09-23T23:23:12.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.551e-06 2023-09-23T23:23:12.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244881+0000", 2023-09-23T23:23:12.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3215e-05 2023-09-23T23:23:12.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244888+0000", 2023-09-23T23:23:12.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.199e-06 2023-09-23T23:23:12.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.074685+0000", 2023-09-23T23:23:12.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.829796778 2023-09-23T23:23:12.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.078783+0000", 2023-09-23T23:23:12.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040977440000000004 2023-09-23T23:23:12.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.112127+0000", 2023-09-23T23:23:12.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.033344602000000001 2023-09-23T23:23:12.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.115665+0000", 2023-09-23T23:23:12.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035371859999999999 2023-09-23T23:23:12.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.871651+0000", 2023-09-23T23:23:12.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.75598617700000004 2023-09-23T23:23:12.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.874503+0000", 2023-09-23T23:23:12.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028525780000000001 2023-09-23T23:23:12.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.879361+0000", 2023-09-23T23:23:12.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048573160000000004 2023-09-23T23:23:12.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.905198+0000", 2023-09-23T23:23:12.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.025837373 2023-09-23T23:23:12.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.905568+0000", 2023-09-23T23:23:12.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000370021 2023-09-23T23:23:12.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.906504+0000", 2023-09-23T23:23:12.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00093603000000000002 2023-09-23T23:23:12.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.907999+0000", 2023-09-23T23:23:12.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001495213 2023-09-23T23:23:12.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.914130+0000", 2023-09-23T23:23:12.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061308609999999996 2023-09-23T23:23:12.818 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.818 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.818 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.819 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156798 10.3 10:c087de90:::benchmark_data_smithi110_366450_object55590:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:23:12.819 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.251881+0000", 2023-09-23T23:23:12.819 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.451301665000003, 2023-09-23T23:23:12.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6721864360000001, 2023-09-23T23:23:12.819 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.819 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.819 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.819 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.819 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.819 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156798 2023-09-23T23:23:12.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.819 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251881+0000", 2023-09-23T23:23:12.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251881+0000", 2023-09-23T23:23:12.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251879+0000", 2023-09-23T23:23:12.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:23:12.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251886+0000", 2023-09-23T23:23:12.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4769999999999997e-06 2023-09-23T23:23:12.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251899+0000", 2023-09-23T23:23:12.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2914999999999999e-05 2023-09-23T23:23:12.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251908+0000", 2023-09-23T23:23:12.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8259999999999999e-06 2023-09-23T23:23:12.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.079210+0000", 2023-09-23T23:23:12.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8273024840000001 2023-09-23T23:23:12.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.082825+0000", 2023-09-23T23:23:12.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036146450000000001 2023-09-23T23:23:12.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.129346+0000", 2023-09-23T23:23:12.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.046521376000000003 2023-09-23T23:23:12.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.132533+0000", 2023-09-23T23:23:12.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031862040000000002 2023-09-23T23:23:12.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.892035+0000", 2023-09-23T23:23:12.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.75950217900000005 2023-09-23T23:23:12.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.894859+0000", 2023-09-23T23:23:12.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002824287 2023-09-23T23:23:12.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.899648+0000", 2023-09-23T23:23:12.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047886220000000002 2023-09-23T23:23:12.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.915092+0000", 2023-09-23T23:23:12.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015443738 2023-09-23T23:23:12.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.915395+0000", 2023-09-23T23:23:12.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000303252 2023-09-23T23:23:12.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.916288+0000", 2023-09-23T23:23:12.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00089328499999999996 2023-09-23T23:23:12.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.917880+0000", 2023-09-23T23:23:12.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001592336 2023-09-23T23:23:12.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.924067+0000", 2023-09-23T23:23:12.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061867060000000001 2023-09-23T23:23:12.827 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.827 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.827 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.827 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156815 10.3 10:c745291f:::benchmark_data_smithi110_366450_object55607:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:23:12.827 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.282940+0000", 2023-09-23T23:23:12.827 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.420242169000005, 2023-09-23T23:23:12.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7155916270000002, 2023-09-23T23:23:12.828 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.828 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.828 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.828 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.828 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.828 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156815 2023-09-23T23:23:12.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.828 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282940+0000", 2023-09-23T23:23:12.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282940+0000", 2023-09-23T23:23:12.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282938+0000", 2023-09-23T23:23:12.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:23:12.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282946+0000", 2023-09-23T23:23:12.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6690000000000006e-06 2023-09-23T23:23:12.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282960+0000", 2023-09-23T23:23:12.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4154e-05 2023-09-23T23:23:12.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282969+0000", 2023-09-23T23:23:12.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4290000000000001e-06 2023-09-23T23:23:12.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.132962+0000", 2023-09-23T23:23:12.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.849993564 2023-09-23T23:23:12.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.136239+0000", 2023-09-23T23:23:12.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032766829999999999 2023-09-23T23:23:12.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.153725+0000", 2023-09-23T23:23:12.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017485941000000001 2023-09-23T23:23:12.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.157066+0000", 2023-09-23T23:23:12.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003341283 2023-09-23T23:23:12.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.940774+0000", 2023-09-23T23:23:12.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.78370831600000002 2023-09-23T23:23:12.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.943618+0000", 2023-09-23T23:23:12.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002843632 2023-09-23T23:23:12.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.948942+0000", 2023-09-23T23:23:12.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053240989999999997 2023-09-23T23:23:12.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.984632+0000", 2023-09-23T23:23:12.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.035689569999999997 2023-09-23T23:23:12.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.990058+0000", 2023-09-23T23:23:12.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054259690000000001 2023-09-23T23:23:12.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.991453+0000", 2023-09-23T23:23:12.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013957710000000001 2023-09-23T23:23:12.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.992873+0000", 2023-09-23T23:23:12.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014191939999999999 2023-09-23T23:23:12.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.998532+0000", 2023-09-23T23:23:12.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056591080000000004 2023-09-23T23:23:12.835 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.836 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.836 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.836 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156870 10.3 10:cbcf9498:::benchmark_data_smithi110_366450_object55662:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:23:12.836 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.374775+0000", 2023-09-23T23:23:12.836 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.328407456999997, 2023-09-23T23:23:12.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6969683209999999, 2023-09-23T23:23:12.836 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.836 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.836 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.836 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.837 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.837 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156870 2023-09-23T23:23:12.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.837 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374775+0000", 2023-09-23T23:23:12.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374775+0000", 2023-09-23T23:23:12.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374773+0000", 2023-09-23T23:23:12.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:23:12.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374781+0000", 2023-09-23T23:23:12.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6690000000000006e-06 2023-09-23T23:23:12.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374798+0000", 2023-09-23T23:23:12.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7326e-05 2023-09-23T23:23:12.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374807+0000", 2023-09-23T23:23:12.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8920000000000006e-06 2023-09-23T23:23:12.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.137607+0000", 2023-09-23T23:23:12.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.762799918 2023-09-23T23:23:12.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.140841+0000", 2023-09-23T23:23:12.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032347109999999999 2023-09-23T23:23:12.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.170460+0000", 2023-09-23T23:23:12.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029618499999999999 2023-09-23T23:23:12.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.841 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.841 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.173817+0000", 2023-09-23T23:23:12.841 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033569440000000002 2023-09-23T23:23:12.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.841 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.841 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.015338+0000", 2023-09-23T23:23:12.841 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.841521509 2023-09-23T23:23:12.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.842 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.842 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.017988+0000", 2023-09-23T23:23:12.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026499359999999999 2023-09-23T23:23:12.842 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.842 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.842 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.022695+0000", 2023-09-23T23:23:12.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047064819999999997 2023-09-23T23:23:12.842 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.842 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.042997+0000", 2023-09-23T23:23:12.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020301784999999999 2023-09-23T23:23:12.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.062327+0000", 2023-09-23T23:23:12.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019330791 2023-09-23T23:23:12.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.064213+0000", 2023-09-23T23:23:12.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018860210000000001 2023-09-23T23:23:12.844 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.065676+0000", 2023-09-23T23:23:12.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001462968 2023-09-23T23:23:12.844 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.071743+0000", 2023-09-23T23:23:12.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060667660000000003 2023-09-23T23:23:12.844 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.844 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.844 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.845 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156880 10.3 10:ca35dff2:::benchmark_data_smithi110_366450_object55672:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:23:12.845 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.396748+0000", 2023-09-23T23:23:12.845 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.306434167999996, 2023-09-23T23:23:12.845 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7345815240000002, 2023-09-23T23:23:12.845 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.845 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.845 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.845 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.845 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.845 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156880 2023-09-23T23:23:12.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.846 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.846 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396748+0000", 2023-09-23T23:23:12.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.846 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396748+0000", 2023-09-23T23:23:12.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.847 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.847 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396747+0000", 2023-09-23T23:23:12.847 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:23:12.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.847 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.847 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396753+0000", 2023-09-23T23:23:12.847 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3829999999999997e-06 2023-09-23T23:23:12.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396765+0000", 2023-09-23T23:23:12.848 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1868999999999999e-05 2023-09-23T23:23:12.848 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.848 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396771+0000", 2023-09-23T23:23:12.848 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2299999999999996e-06 2023-09-23T23:23:12.848 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.848 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.174176+0000", 2023-09-23T23:23:12.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.777405116 2023-09-23T23:23:12.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.849 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.849 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.849 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.177553+0000", 2023-09-23T23:23:12.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033766130000000001 2023-09-23T23:23:12.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.849 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.849 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.196534+0000", 2023-09-23T23:23:12.850 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018980770000000001 2023-09-23T23:23:12.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.200001+0000", 2023-09-23T23:23:12.850 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003467159 2023-09-23T23:23:12.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.851 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.851 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.084438+0000", 2023-09-23T23:23:12.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.88443721799999997 2023-09-23T23:23:12.851 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.851 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.851 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.087726+0000", 2023-09-23T23:23:12.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003287805 2023-09-23T23:23:12.851 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.852 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.852 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.852 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.093532+0000", 2023-09-23T23:23:12.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058060129999999996 2023-09-23T23:23:12.852 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.852 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.852 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.852 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.120102+0000", 2023-09-23T23:23:12.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.026570277999999999 2023-09-23T23:23:12.852 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.853 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.853 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.121067+0000", 2023-09-23T23:23:12.853 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00096454899999999998 2023-09-23T23:23:12.853 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.853 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.853 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.122429+0000", 2023-09-23T23:23:12.853 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001362399 2023-09-23T23:23:12.853 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.124160+0000", 2023-09-23T23:23:12.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001731291 2023-09-23T23:23:12.854 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.854 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.131330+0000", 2023-09-23T23:23:12.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071693939999999999 2023-09-23T23:23:12.854 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.854 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.855 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.855 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.855 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.855 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156887 10.3 10:c3285ada:::benchmark_data_smithi110_366450_object55679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:23:12.855 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.419195+0000", 2023-09-23T23:23:12.855 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.283987749000005, 2023-09-23T23:23:12.855 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.750874418, 2023-09-23T23:23:12.855 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.855 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.855 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.855 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.856 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.856 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156887 2023-09-23T23:23:12.856 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.856 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.856 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.856 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.856 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419195+0000", 2023-09-23T23:23:12.856 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.856 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.856 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.856 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419195+0000", 2023-09-23T23:23:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419193+0000", 2023-09-23T23:23:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:23:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419199+0000", 2023-09-23T23:23:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1539999999999999e-06 2023-09-23T23:23:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419211+0000", 2023-09-23T23:23:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1651000000000001e-05 2023-09-23T23:23:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419219+0000", 2023-09-23T23:23:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0220000000000004e-06 2023-09-23T23:23:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.178884+0000", 2023-09-23T23:23:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7596647829999998 2023-09-23T23:23:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.182782+0000", 2023-09-23T23:23:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038980529999999998 2023-09-23T23:23:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.212229+0000", 2023-09-23T23:23:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029447465999999999 2023-09-23T23:23:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.215590+0000", 2023-09-23T23:23:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033608779999999999 2023-09-23T23:23:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.141066+0000", 2023-09-23T23:23:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.92547559800000001 2023-09-23T23:23:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.144556+0000", 2023-09-23T23:23:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034902330000000001 2023-09-23T23:23:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.149839+0000", 2023-09-23T23:23:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052831730000000004 2023-09-23T23:23:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.160290+0000", 2023-09-23T23:23:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010451251999999999 2023-09-23T23:23:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.160640+0000", 2023-09-23T23:23:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034921699999999997 2023-09-23T23:23:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.161546+0000", 2023-09-23T23:23:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00090665099999999996 2023-09-23T23:23:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.163299+0000", 2023-09-23T23:23:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017525590000000001 2023-09-23T23:23:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.170069+0000", 2023-09-23T23:23:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067702309999999998 2023-09-23T23:23:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:162491 10.3 10:c27b7a62:::benchmark_data_smithi110_366450_object61283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T23:23:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:22:12.428302+0000", 2023-09-23T23:23:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.274880467999999, 2023-09-23T23:23:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.190000068, 2023-09-23T23:23:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 162491 2023-09-23T23:23:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428302+0000", 2023-09-23T23:23:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428302+0000", 2023-09-23T23:23:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428301+0000", 2023-09-23T23:23:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:23:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428306+0000", 2023-09-23T23:23:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3000000000000001e-06 2023-09-23T23:23:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428315+0000", 2023-09-23T23:23:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2450000000000004e-06 2023-09-23T23:23:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428321+0000", 2023-09-23T23:23:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0780000000000001e-06 2023-09-23T23:23:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.484840+0000", 2023-09-23T23:23:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.056518688999999997 2023-09-23T23:23:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.486346+0000", 2023-09-23T23:23:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015063489999999999 2023-09-23T23:23:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.928898+0000", 2023-09-23T23:23:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.44255191999999999 2023-09-23T23:23:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.932431+0000", 2023-09-23T23:23:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035331630000000002 2023-09-23T23:23:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.575704+0000", 2023-09-23T23:23:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.64327284799999995 2023-09-23T23:23:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.578243+0000", 2023-09-23T23:23:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025391559999999999 2023-09-23T23:23:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.870 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.870 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.870 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.582742+0000", 2023-09-23T23:23:12.870 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044984200000000004 2023-09-23T23:23:12.870 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.870 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.870 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.870 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.610331+0000", 2023-09-23T23:23:12.870 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.027589460999999999 2023-09-23T23:23:12.870 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.870 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.870 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.610601+0000", 2023-09-23T23:23:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026943799999999999 2023-09-23T23:23:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.611382+0000", 2023-09-23T23:23:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00078106699999999998 2023-09-23T23:23:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.612748+0000", 2023-09-23T23:23:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013659690000000001 2023-09-23T23:23:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.618302+0000", 2023-09-23T23:23:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055542489999999998 2023-09-23T23:23:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:163210 10.3 10:cceb657e:::benchmark_data_smithi110_366450_object62002:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-09-23T23:23:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:22:15.221166+0000", 2023-09-23T23:23:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 57.482015979000003, 2023-09-23T23:23:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.360785763, 2023-09-23T23:23:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 163210 2023-09-23T23:23:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221166+0000", 2023-09-23T23:23:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221166+0000", 2023-09-23T23:23:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221165+0000", 2023-09-23T23:23:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:23:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221170+0000", 2023-09-23T23:23:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3000000000000003e-06 2023-09-23T23:23:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221180+0000", 2023-09-23T23:23:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0513000000000001e-05 2023-09-23T23:23:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221186+0000", 2023-09-23T23:23:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3789999999999997e-06 2023-09-23T23:23:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.236027+0000", 2023-09-23T23:23:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014841425 2023-09-23T23:23:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.237325+0000", 2023-09-23T23:23:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012978829999999999 2023-09-23T23:23:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.914366+0000", 2023-09-23T23:23:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.677041059 2023-09-23T23:23:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.917400+0000", 2023-09-23T23:23:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003033507 2023-09-23T23:23:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.556394+0000", 2023-09-23T23:23:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.63899485600000006 2023-09-23T23:23:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.558747+0000", 2023-09-23T23:23:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023522899999999999 2023-09-23T23:23:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.563392+0000", 2023-09-23T23:23:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046450199999999997 2023-09-23T23:23:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.572010+0000", 2023-09-23T23:23:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086180040000000003 2023-09-23T23:23:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.572357+0000", 2023-09-23T23:23:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034713000000000002 2023-09-23T23:23:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.573395+0000", 2023-09-23T23:23:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010380890000000001 2023-09-23T23:23:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.575115+0000", 2023-09-23T23:23:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017195960000000001 2023-09-23T23:23:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.581952+0000", 2023-09-23T23:23:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068375789999999999 2023-09-23T23:23:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:168464 10.3 10:c5beec16:::benchmark_data_smithi110_366450_object67256:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T23:23:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:22:31.505622+0000", 2023-09-23T23:23:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 41.197560299000003, 2023-09-23T23:23:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1552177969999999, 2023-09-23T23:23:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:23:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:23:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:23:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:23:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:23:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 168464 2023-09-23T23:23:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:23:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:23:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505622+0000", 2023-09-23T23:23:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:23:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505622+0000", 2023-09-23T23:23:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:23:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:23:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505621+0000", 2023-09-23T23:23:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:23:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:23:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505627+0000", 2023-09-23T23:23:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8950000000000003e-06 2023-09-23T23:23:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:23:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505646+0000", 2023-09-23T23:23:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9667e-05 2023-09-23T23:23:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:23:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505657+0000", 2023-09-23T23:23:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1141000000000001e-05 2023-09-23T23:23:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.560164+0000", 2023-09-23T23:23:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.054506720000000002 2023-09-23T23:23:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:23:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.561430+0000", 2023-09-23T23:23:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001266267 2023-09-23T23:23:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.028996+0000", 2023-09-23T23:23:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.46756556300000002 2023-09-23T23:23:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:23:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.031880+0000", 2023-09-23T23:23:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028842189999999999 2023-09-23T23:23:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:23:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.517916+0000", 2023-09-23T23:23:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.486035885 2023-09-23T23:23:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:23:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.519830+0000", 2023-09-23T23:23:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019141340000000001 2023-09-23T23:23:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:23:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.523467+0000", 2023-09-23T23:23:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036371060000000002 2023-09-23T23:23:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.530246+0000", 2023-09-23T23:23:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067788550000000003 2023-09-23T23:23:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:23:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.532994+0000", 2023-09-23T23:23:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027481290000000002 2023-09-23T23:23:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:23:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.654294+0000", 2023-09-23T23:23:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.121299324 2023-09-23T23:23:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:23:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.655550+0000", 2023-09-23T23:23:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012558789999999999 2023-09-23T23:23:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:23:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:23:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:23:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.660840+0000", 2023-09-23T23:23:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052902979999999997 2023-09-23T23:23:12.890 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.890 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.890 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.890 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:23:12.890 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:23:12.890 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:23:12.890 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:12.923 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:12 smithi110 ceph-mon[212939]: pgmap v998: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 967 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 937/46473 objects degraded (2.016%); 3894/46473 objects misplaced (8.379%); 2.1 MiB/s, 43 objects/s recovering 2023-09-23T23:23:12.924 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:12 smithi110 ceph-mon[221102]: pgmap v998: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 967 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 937/46473 objects degraded (2.016%); 3894/46473 objects misplaced (8.379%); 2.1 MiB/s, 43 objects/s recovering 2023-09-23T23:23:12.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:12 smithi138 ceph-mon[193606]: pgmap v998: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 967 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 937/46473 objects degraded (2.016%); 3894/46473 objects misplaced (8.379%); 2.1 MiB/s, 43 objects/s recovering 2023-09-23T23:23:12.973 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:13.265 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:13.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:13.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:13.373 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:13.431 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:13.727 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:23:13.727 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:23:13.727 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:23:13.727 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:23:13.727 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:23:13.727 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:23:13.727 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:23:13.728 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:23:13.728 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:23:13.728 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:23:13.728 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:23:13.728 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:23:13.728 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:23:13.728 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:23:13.728 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:23:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:13 smithi138 ceph-mon[193606]: 1.3 scrub starts 2023-09-23T23:23:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:13 smithi138 ceph-mon[193606]: 1.7 scrub starts 2023-09-23T23:23:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:23:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:13 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/622501366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:13 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 937/46473 objects degraded (2.016%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:13.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:13 smithi110 ceph-mon[212939]: 1.3 scrub starts 2023-09-23T23:23:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:13 smithi110 ceph-mon[212939]: 1.7 scrub starts 2023-09-23T23:23:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:23:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:13 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/622501366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:13 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 937/46473 objects degraded (2.016%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:13 smithi110 ceph-mon[221102]: 1.3 scrub starts 2023-09-23T23:23:13.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:13 smithi110 ceph-mon[221102]: 1.7 scrub starts 2023-09-23T23:23:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:23:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:13 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/622501366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:13 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 937/46473 objects degraded (2.016%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:14.195 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:23:14.518 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:14.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:14.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:14.630 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:23:14.663 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:14.736 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:23:14.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:14.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:23:14.841 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:14.876 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:14 smithi110 ceph-mon[221102]: pgmap v999: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 967 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 937/46473 objects degraded (2.016%); 3894/46473 objects misplaced (8.379%); 2.1 MiB/s, 43 objects/s recovering 2023-09-23T23:23:14.876 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:14 smithi110 ceph-mon[212939]: pgmap v999: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 967 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 937/46473 objects degraded (2.016%); 3894/46473 objects misplaced (8.379%); 2.1 MiB/s, 43 objects/s recovering 2023-09-23T23:23:14.887 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:14.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:14 smithi138 ceph-mon[193606]: pgmap v999: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 967 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 937/46473 objects degraded (2.016%); 3894/46473 objects misplaced (8.379%); 2.1 MiB/s, 43 objects/s recovering 2023-09-23T23:23:14.998 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:23:15.295 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:23:15.296 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:23:15.374 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:15.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:15.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:15.485 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:23:15.518 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:15.577 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:23:15.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:15.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:23:15.689 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:15.693 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:15.769 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:15.907 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:15.964 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:15.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:15.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:16.069 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:23:16.103 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:16.174 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:23:16.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:16.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:23:16.282 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:16.315 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:16.368 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:16.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:16.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:16.480 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:16.543 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:16.708 INFO:teuthology.orchestra.run.smithi110.stderr:set pool 1 pgp_num to 31 2023-09-23T23:23:16.751 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:16 smithi110 ceph-mon[212939]: pgmap v1000: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 696 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 687/33462 objects degraded (2.053%); 2781/33462 objects misplaced (8.311%); 927 KiB/s, 54 objects/s recovering 2023-09-23T23:23:16.751 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:16 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1531842157' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T23:23:16.752 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:16 smithi110 ceph-mon[221102]: pgmap v1000: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 696 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 687/33462 objects degraded (2.053%); 2781/33462 objects misplaced (8.311%); 927 KiB/s, 54 objects/s recovering 2023-09-23T23:23:16.752 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:16 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1531842157' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T23:23:16.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:16 smithi138 ceph-mon[193606]: pgmap v1000: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 696 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 687/33462 objects degraded (2.053%); 2781/33462 objects misplaced (8.311%); 927 KiB/s, 54 objects/s recovering 2023-09-23T23:23:16.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:16 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1531842157' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T23:23:17.201 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:23:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:23:16] "GET /metrics HTTP/1.1" 200 36636 "" "Prometheus/2.43.0" 2023-09-23T23:23:17.297 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:23:17.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:17 smithi110 ceph-mon[221102]: 1.6 scrub starts 2023-09-23T23:23:17.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:17 smithi110 ceph-mon[221102]: 1.13 scrub starts 2023-09-23T23:23:17.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:17 smithi110 ceph-mon[221102]: 1.4 scrub starts 2023-09-23T23:23:17.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:17 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1531842157' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-23T23:23:17.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:17 smithi110 ceph-mon[221102]: osdmap e339: 8 total, 7 up, 8 in 2023-09-23T23:23:17.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:17 smithi110 ceph-mon[221102]: failed to encode map e339 with expected crc 2023-09-23T23:23:17.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:17 smithi110 ceph-mon[221102]: failed to encode map e339 with expected crc 2023-09-23T23:23:17.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:17 smithi110 ceph-mon[221102]: failed to encode map e339 with expected crc 2023-09-23T23:23:17.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:17 smithi110 ceph-mon[212939]: 1.6 scrub starts 2023-09-23T23:23:17.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:17 smithi110 ceph-mon[212939]: 1.13 scrub starts 2023-09-23T23:23:17.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:17 smithi110 ceph-mon[212939]: 1.4 scrub starts 2023-09-23T23:23:17.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:17 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1531842157' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-23T23:23:17.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:17 smithi110 ceph-mon[212939]: osdmap e339: 8 total, 7 up, 8 in 2023-09-23T23:23:17.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:17 smithi110 ceph-mon[212939]: failed to encode map e339 with expected crc 2023-09-23T23:23:17.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:17 smithi110 ceph-mon[212939]: failed to encode map e339 with expected crc 2023-09-23T23:23:17.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:17 smithi110 ceph-mon[212939]: failed to encode map e339 with expected crc 2023-09-23T23:23:18.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:17 smithi138 ceph-mon[193606]: 1.6 scrub starts 2023-09-23T23:23:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:17 smithi138 ceph-mon[193606]: 1.13 scrub starts 2023-09-23T23:23:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:17 smithi138 ceph-mon[193606]: 1.4 scrub starts 2023-09-23T23:23:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:17 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1531842157' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-23T23:23:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:17 smithi138 ceph-mon[193606]: osdmap e339: 8 total, 7 up, 8 in 2023-09-23T23:23:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:17 smithi138 ceph-mon[193606]: failed to encode map e339 with expected crc 2023-09-23T23:23:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:17 smithi138 ceph-mon[193606]: failed to encode map e339 with expected crc 2023-09-23T23:23:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:17 smithi138 ceph-mon[193606]: failed to encode map e339 with expected crc 2023-09-23T23:23:18.728 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:18.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:18.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:18.833 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:18.935 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:18.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:18 smithi110 ceph-mon[221102]: pgmap v1002: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 696 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 687/33462 objects degraded (2.053%); 2781/33462 objects misplaced (8.311%); 1.1 MiB/s, 68 objects/s recovering 2023-09-23T23:23:18.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:18 smithi110 ceph-mon[221102]: 1.14 scrub starts 2023-09-23T23:23:18.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:18 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 687/33462 objects degraded (2.053%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:18.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:18 smithi110 ceph-mon[212939]: pgmap v1002: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 696 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 687/33462 objects degraded (2.053%); 2781/33462 objects misplaced (8.311%); 1.1 MiB/s, 68 objects/s recovering 2023-09-23T23:23:18.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:18 smithi110 ceph-mon[212939]: 1.14 scrub starts 2023-09-23T23:23:18.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:18 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 687/33462 objects degraded (2.053%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:19.030 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:19.107 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:19.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:19.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:18 smithi138 ceph-mon[193606]: pgmap v1002: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 696 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 687/33462 objects degraded (2.053%); 2781/33462 objects misplaced (8.311%); 1.1 MiB/s, 68 objects/s recovering 2023-09-23T23:23:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:18 smithi138 ceph-mon[193606]: 1.14 scrub starts 2023-09-23T23:23:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:18 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 687/33462 objects degraded (2.053%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:19.221 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:19.306 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:19.519 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:19.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:19.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:19.629 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:19.730 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:19.921 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:23:19.921 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:23:19.922 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:19.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:19.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:20.041 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:23:20.113 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:20.325 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:23:20.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:20.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:23:20.434 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:23:20.483 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:20.562 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:23:20.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:20.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:23:20.668 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:20.696 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:20.716 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:23:20.743 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:20.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:20.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:20.847 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:20.876 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:20.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:20 smithi138 ceph-mon[193606]: pgmap v1003: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 553 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 459/26580 objects degraded (1.727%); 2534/26580 objects misplaced (9.533%); 0 B/s, 34 objects/s recovering 2023-09-23T23:23:20.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:20 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:23:20.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:20 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/91328798' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:20.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:20 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:20.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:20 smithi110 ceph-mon[212939]: pgmap v1003: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 553 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 459/26580 objects degraded (1.727%); 2534/26580 objects misplaced (9.533%); 0 B/s, 34 objects/s recovering 2023-09-23T23:23:20.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:20 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:23:20.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:20 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/91328798' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:20.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:20 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:20.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:20 smithi110 ceph-mon[221102]: pgmap v1003: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 553 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 459/26580 objects degraded (1.727%); 2534/26580 objects misplaced (9.533%); 0 B/s, 34 objects/s recovering 2023-09-23T23:23:20.962 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:20 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:23:20.962 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:20 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/91328798' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:20.962 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:20 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:21.595 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:21.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:21.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:21.697 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:23:21.705 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:21.742 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:21.801 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:21.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:21.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:21.908 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:21.934 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:21 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:23:21.934 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:21 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:23:21.934 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:21 smithi110 ceph-mon[221102]: osdmap e340: 8 total, 7 up, 8 in 2023-09-23T23:23:21.934 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:21 smithi110 ceph-mon[221102]: failed to encode map e340 with expected crc 2023-09-23T23:23:21.934 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:21 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:23:21.935 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:21 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:23:21.935 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:21 smithi110 ceph-mon[212939]: osdmap e340: 8 total, 7 up, 8 in 2023-09-23T23:23:21.935 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:21 smithi110 ceph-mon[212939]: failed to encode map e340 with expected crc 2023-09-23T23:23:22.001 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:22.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:21 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:23:22.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:21 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:23:22.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:21 smithi138 ceph-mon[193606]: osdmap e340: 8 total, 7 up, 8 in 2023-09-23T23:23:22.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:21 smithi138 ceph-mon[193606]: failed to encode map e340 with expected crc 2023-09-23T23:23:22.477 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:22.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:22.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:22.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:23:22.650 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:22.782 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:23:22.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:22.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:23:22.879 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:22.884 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:22.947 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:23.046 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:23.053 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:22 smithi110 ceph-mon[212939]: 1.e scrub starts 2023-09-23T23:23:23.053 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:22 smithi110 ceph-mon[212939]: failed to encode map e340 with expected crc 2023-09-23T23:23:23.053 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:22 smithi110 ceph-mon[212939]: failed to encode map e340 with expected crc 2023-09-23T23:23:23.053 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:22 smithi110 ceph-mon[212939]: pgmap v1005: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 441 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 459/21228 objects degraded (2.162%); 1761/21228 objects misplaced (8.296%); 0 B/s, 75 objects/s recovering 2023-09-23T23:23:23.054 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:22 smithi110 ceph-mon[221102]: 1.e scrub starts 2023-09-23T23:23:23.054 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:22 smithi110 ceph-mon[221102]: failed to encode map e340 with expected crc 2023-09-23T23:23:23.054 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:22 smithi110 ceph-mon[221102]: failed to encode map e340 with expected crc 2023-09-23T23:23:23.054 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:22 smithi110 ceph-mon[221102]: pgmap v1005: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 441 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 459/21228 objects degraded (2.162%); 1761/21228 objects misplaced (8.296%); 0 B/s, 75 objects/s recovering 2023-09-23T23:23:23.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:23.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:23.116 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:23:23.150 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:23:23.155 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:23:23.189 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:23:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:22 smithi138 ceph-mon[193606]: 1.e scrub starts 2023-09-23T23:23:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:22 smithi138 ceph-mon[193606]: failed to encode map e340 with expected crc 2023-09-23T23:23:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:22 smithi138 ceph-mon[193606]: failed to encode map e340 with expected crc 2023-09-23T23:23:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:22 smithi138 ceph-mon[193606]: pgmap v1005: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 441 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 459/21228 objects degraded (2.162%); 1761/21228 objects misplaced (8.296%); 0 B/s, 75 objects/s recovering 2023-09-23T23:23:23.212 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:23.221 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:23:23.260 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:23:23.277 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:23:23.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:23.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:23:23.299 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:23:23.332 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:23:23.333 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:23:23.333 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:23:23.333 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:23:23.333 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:23:23.333 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:23:23.333 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:23:23.333 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:23:23.333 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:23:23.384 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:23.454 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:23.840 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 2, 1] out_osds: [] dead_osds: [7] live_osds: [0, 1, 3, 4, 5, 6, 2] 2023-09-23T23:23:23.840 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:23:23.840 INFO:tasks.thrashosds.thrasher:Reviving osd 7 2023-09-23T23:23:23.841 INFO:tasks.cephadm.osd.7:Restarting osd.7 (starting--it wasn't running)... 2023-09-23T23:23:23.841 DEBUG:teuthology.orchestra.run.smithi138:osd.7> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.7.service 2023-09-23T23:23:23.843 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl start ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T23:23:23.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:23 smithi110 ceph-mon[212939]: 1.17 scrub starts 2023-09-23T23:23:23.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:23 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2831330639' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:23.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:23 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 459/21228 objects degraded (2.162%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:23.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:23 smithi110 ceph-mon[221102]: 1.17 scrub starts 2023-09-23T23:23:23.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:23 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2831330639' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:23.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:23 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 459/21228 objects degraded (2.162%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:23.964 INFO:journalctl@ceph.osd.7.smithi138.stdout:-- Logs begin at Sat 2023-09-23 22:31:02 UTC. -- 2023-09-23T23:23:24.160 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:24.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:24.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:24.192 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:23:23 smithi138 systemd[1]: Starting Ceph osd.7 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:23:24.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:23 smithi138 ceph-mon[193606]: 1.17 scrub starts 2023-09-23T23:23:24.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:23 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2831330639' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:24.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:23 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 459/21228 objects degraded (2.162%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:24.281 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:23:24.352 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:24.715 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:23:24.840 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:23:24.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:24.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:23:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:24 smithi138 ceph-mon[193606]: 1.b scrub starts 2023-09-23T23:23:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:24 smithi138 ceph-mon[193606]: pgmap v1006: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 441 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 459/21228 objects degraded (2.162%); 1761/21228 objects misplaced (8.296%); 0 B/s, 37 objects/s recovering 2023-09-23T23:23:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:24 smithi138 ceph-mon[193606]: 1.1e scrub starts 2023-09-23T23:23:24.950 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:25.013 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:24 smithi110 ceph-mon[212939]: 1.b scrub starts 2023-09-23T23:23:25.013 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:24 smithi110 ceph-mon[212939]: pgmap v1006: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 441 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 459/21228 objects degraded (2.162%); 1761/21228 objects misplaced (8.296%); 0 B/s, 37 objects/s recovering 2023-09-23T23:23:25.013 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:24 smithi110 ceph-mon[212939]: 1.1e scrub starts 2023-09-23T23:23:25.013 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:24 smithi110 ceph-mon[221102]: 1.b scrub starts 2023-09-23T23:23:25.014 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:24 smithi110 ceph-mon[221102]: pgmap v1006: 119 pgs: 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 97 active+clean; 441 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 459/21228 objects degraded (2.162%); 1761/21228 objects misplaced (8.296%); 0 B/s, 37 objects/s recovering 2023-09-23T23:23:25.014 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:24 smithi110 ceph-mon[221102]: 1.1e scrub starts 2023-09-23T23:23:25.028 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:25.053 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:23:25.053 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:23:25.161 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:25.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:25.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:25.278 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:23:25.330 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:25.435 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:23:25 smithi138 bash[274531]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T23:23:25.435 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:23:25 smithi138 bash[274531]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-09-23T23:23:25.436 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:23:25 smithi138 bash[274531]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-7 --no-mon-config 2023-09-23T23:23:25.436 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:23:25 smithi138 bash[274531]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-7/block 2023-09-23T23:23:25.436 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:23:25 smithi138 bash[274531]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-7/block 2023-09-23T23:23:25.436 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:23:25 smithi138 bash[274531]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-23T23:23:25.436 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:23:25 smithi138 bash[274531]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-09-23T23:23:25.436 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:23:25 smithi138 bash[274531]: --> ceph-volume lvm activate successful for osd ID: 7 2023-09-23T23:23:25.454 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:23:25.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:25.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:23:25.559 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:25.634 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:25.683 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:25.753 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:25.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:25.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:25.837 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:23:25.837 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:23:25.837 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:23:25.837 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:23:25.838 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:23:25.863 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:25.953 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:26.120 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:26.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:26.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:26.233 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:23:26.308 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:27.019 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:23:26 smithi138 ceph-osd[275039]: -- 172.21.15.138:0/3808850126 <== mon.1 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d3a46b2b60 con 0x55d3a4679000 2023-09-23T23:23:27.019 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:23:27 smithi138 bash[274975]: 641f92ed2e0cb2c7a905aa5a77568d8f0649db39dd871f4497f6909ec25f5ca8 2023-09-23T23:23:27.020 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:26 smithi138 ceph-mon[193606]: pgmap v1007: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 184 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 214/8868 objects degraded (2.413%); 731/8868 objects misplaced (8.243%); 0 B/s, 75 objects/s recovering 2023-09-23T23:23:27.020 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:26 smithi138 ceph-mon[193606]: 1.c deep-scrub starts 2023-09-23T23:23:27.025 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:23:27.054 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:23:27.112 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:23:27.113 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:26 smithi110 ceph-mon[221102]: pgmap v1007: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 184 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 214/8868 objects degraded (2.413%); 731/8868 objects misplaced (8.243%); 0 B/s, 75 objects/s recovering 2023-09-23T23:23:27.113 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:26 smithi110 ceph-mon[221102]: 1.c deep-scrub starts 2023-09-23T23:23:27.114 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:26 smithi110 ceph-mon[212939]: pgmap v1007: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 184 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 214/8868 objects degraded (2.413%); 731/8868 objects misplaced (8.243%); 0 B/s, 75 objects/s recovering 2023-09-23T23:23:27.114 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:26 smithi110 ceph-mon[212939]: 1.c deep-scrub starts 2023-09-23T23:23:27.114 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:23:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:23:26] "GET /metrics HTTP/1.1" 200 36636 "" "Prometheus/2.43.0" 2023-09-23T23:23:27.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:27.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:23:27.218 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:27.284 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:27.441 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:23:27 smithi138 systemd[1]: Started Ceph osd.7 for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:23:27.473 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:23:27.663 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:23:27.774 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:27.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:27.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:27.803 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:23:27 smithi138 conmon[275015]: 2023-09-23T23:23:27.521+0000 7f0c8778d540 -1 Falling back to public interface 2023-09-23T23:23:27.803 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:27 smithi138 ceph-mon[193606]: 1.1b scrub starts 2023-09-23T23:23:27.884 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:27.936 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:27.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:27 smithi110 ceph-mon[221102]: 1.1b scrub starts 2023-09-23T23:23:27.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:27 smithi110 ceph-mon[212939]: 1.1b scrub starts 2023-09-23T23:23:28.049 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:28.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:28.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:28.087 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:28.154 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:28.251 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:28.258 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Removed 101205 objects 2023-09-23T23:23:28.258 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Clean up completed and total clean up time :167.412 2023-09-23T23:23:28.312 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:28.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:28.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:28.416 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:28.447 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:28.600 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:28.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:28.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:28.708 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:28.718 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:28.737 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:29.028 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:29.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:29.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:29.137 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:29.166 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:29.266 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:28 smithi138 ceph-mon[193606]: pgmap v1008: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 184 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 214/8868 objects degraded (2.413%); 731/8868 objects misplaced (8.243%); 0 B/s, 66 objects/s recovering 2023-09-23T23:23:29.266 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:29.267 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:23:29.267 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:28 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 214/8868 objects degraded (2.413%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:29.267 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:28 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/124146577' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:23:29.437 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:23:29.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:28 smithi110 ceph-mon[221102]: pgmap v1008: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 184 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 214/8868 objects degraded (2.413%); 731/8868 objects misplaced (8.243%); 0 B/s, 66 objects/s recovering 2023-09-23T23:23:29.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:29.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:23:29.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:28 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 214/8868 objects degraded (2.413%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:29.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:28 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/124146577' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:23:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:28 smithi110 ceph-mon[212939]: pgmap v1008: 119 pgs: 1 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 96 active+clean; 184 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 214/8868 objects degraded (2.413%); 731/8868 objects misplaced (8.243%); 0 B/s, 66 objects/s recovering 2023-09-23T23:23:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:23:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:28 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 214/8868 objects degraded (2.413%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:28 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/124146577' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:23:29.471 INFO:teuthology.orchestra.run.smithi138.stderr:no valid command found; 10 closest matches: 2023-09-23T23:23:29.471 INFO:teuthology.orchestra.run.smithi138.stderr:0 2023-09-23T23:23:29.471 INFO:teuthology.orchestra.run.smithi138.stderr:1 2023-09-23T23:23:29.471 INFO:teuthology.orchestra.run.smithi138.stderr:2 2023-09-23T23:23:29.472 INFO:teuthology.orchestra.run.smithi138.stderr:abort 2023-09-23T23:23:29.472 INFO:teuthology.orchestra.run.smithi138.stderr:assert 2023-09-23T23:23:29.472 INFO:teuthology.orchestra.run.smithi138.stderr:config diff 2023-09-23T23:23:29.472 INFO:teuthology.orchestra.run.smithi138.stderr:config diff get 2023-09-23T23:23:29.472 INFO:teuthology.orchestra.run.smithi138.stderr:config get 2023-09-23T23:23:29.472 INFO:teuthology.orchestra.run.smithi138.stderr:config help [] 2023-09-23T23:23:29.472 INFO:teuthology.orchestra.run.smithi138.stderr:config set ... 2023-09-23T23:23:29.472 INFO:teuthology.orchestra.run.smithi138.stderr:admin_socket: invalid command 2023-09-23T23:23:29.493 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:29.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:29.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:29.597 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:29.629 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:29.723 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:29.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:29.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:29.828 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:23:29.868 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:29.911 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:23:29.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:29.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:23:30.016 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:23:30.045 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:30.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:29 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:23:30.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:29 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/124146577' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:23:30.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:29 smithi138 ceph-mon[193606]: osdmap e341: 8 total, 7 up, 8 in 2023-09-23T23:23:30.194 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:23:30.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:30.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:23:30.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:29 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:23:30.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:29 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/124146577' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:23:30.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:29 smithi110 ceph-mon[212939]: osdmap e341: 8 total, 7 up, 8 in 2023-09-23T23:23:30.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:29 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:23:30.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:29 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/124146577' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:23:30.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:29 smithi110 ceph-mon[221102]: osdmap e341: 8 total, 7 up, 8 in 2023-09-23T23:23:30.249 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T23:23:30.249 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-7, ['dump_ops_in_flight'] 2023-09-23T23:23:30.296 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:30.335 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:30.489 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:30.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:30.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:30.593 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:23:30.624 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:30.668 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:23:30.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:30.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:23:30.759 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:23:30.778 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:30.780 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:23:30.799 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:23:30.809 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:30.828 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:23:30.851 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:23:30.859 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:30.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:30.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:30.873 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:23:30.894 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:23:30.894 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:23:30.894 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:23:30.895 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:23:30.895 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:23:30.895 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:23:30.895 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:23:30.895 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:23:30.895 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T23:23:30.964 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:23:30.995 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:31.041 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:23:31.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:31.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:23:31.145 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:23:31.173 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:31.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[212939]: pgmap v1010: 119 pgs: 2 active+clean+scrubbing, 15 active+clean+remapped, 1 active+clean+scrubbing+deep, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 94 active+clean; 73 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 1/3561 objects degraded (0.028%); 518/3561 objects misplaced (14.546%); 0 B/s, 41 objects/s recovering 2023-09-23T23:23:31.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[212939]: 1.1c scrub starts 2023-09-23T23:23:31.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[212939]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[212939]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[212939]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[212939]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[212939]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[212939]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2934835959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:31.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[221102]: pgmap v1010: 119 pgs: 2 active+clean+scrubbing, 15 active+clean+remapped, 1 active+clean+scrubbing+deep, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 94 active+clean; 73 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 1/3561 objects degraded (0.028%); 518/3561 objects misplaced (14.546%); 0 B/s, 41 objects/s recovering 2023-09-23T23:23:31.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[221102]: 1.1c scrub starts 2023-09-23T23:23:31.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[221102]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[221102]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[221102]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[221102]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[221102]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[221102]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:30 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2934835959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:31.219 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:23:31.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:31.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:23:31.330 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:31.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:30 smithi138 ceph-mon[193606]: pgmap v1010: 119 pgs: 2 active+clean+scrubbing, 15 active+clean+remapped, 1 active+clean+scrubbing+deep, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 94 active+clean; 73 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 1/3561 objects degraded (0.028%); 518/3561 objects misplaced (14.546%); 0 B/s, 41 objects/s recovering 2023-09-23T23:23:31.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:30 smithi138 ceph-mon[193606]: 1.1c scrub starts 2023-09-23T23:23:31.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:30 smithi138 ceph-mon[193606]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:30 smithi138 ceph-mon[193606]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:30 smithi138 ceph-mon[193606]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:30 smithi138 ceph-mon[193606]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:30 smithi138 ceph-mon[193606]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:30 smithi138 ceph-mon[193606]: failed to encode map e341 with expected crc 2023-09-23T23:23:31.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:30 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2934835959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:31.359 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:31.406 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:31.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:31.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:31.514 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:31.542 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:31.642 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:31.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:31.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:31.748 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:31.776 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:32.089 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:32.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:32.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:32.193 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:23:32.223 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:32.226 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:23:32.226 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:23:32.321 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:23:32.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:32.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:23:32.370 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:23:32.426 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:23:32.473 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:32.518 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:23:32.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:32.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:23:32.552 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:23:32.552 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:23:32.627 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:32.655 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:32.704 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:32.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:32.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:32.811 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:32.840 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:32.886 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:32.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:32.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:32.990 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:33.019 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:33.071 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:33.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:33.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:33.100 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-09-23T23:23:33.100 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-09-23T23:23:33.125 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:32 smithi110 ceph-mon[221102]: pgmap v1011: 119 pgs: 2 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 1 active+clean+scrubbing+deep, 2 active+undersized+degraded, 4 active+undersized, 94 active+clean; 739 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 1/75 objects degraded (1.333%); 6/75 objects misplaced (8.000%); 0 B/s, 115 objects/s recovering 2023-09-23T23:23:33.126 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:32 smithi110 ceph-mon[221102]: from='osd.7 [v2:172.21.15.138:6824/675905607,v1:172.21.15.138:6825/675905607]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T23:23:33.126 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:32 smithi110 ceph-mon[221102]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T23:23:33.126 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:32 smithi110 ceph-mon[212939]: pgmap v1011: 119 pgs: 2 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 1 active+clean+scrubbing+deep, 2 active+undersized+degraded, 4 active+undersized, 94 active+clean; 739 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 1/75 objects degraded (1.333%); 6/75 objects misplaced (8.000%); 0 B/s, 115 objects/s recovering 2023-09-23T23:23:33.126 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:32 smithi110 ceph-mon[212939]: from='osd.7 [v2:172.21.15.138:6824/675905607,v1:172.21.15.138:6825/675905607]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T23:23:33.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:32 smithi110 ceph-mon[212939]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T23:23:33.177 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:33.191 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:23:32 smithi138 conmon[275015]: 2023-09-23T23:23:32.879+0000 7f0c8778d540 -1 osd.7 255 log_to_monitors true 2023-09-23T23:23:33.191 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:23:33 smithi138 conmon[275015]: 2023-09-23T23:23:33.049+0000 7f0c76d6a700 -1 osd.7 342 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T23:23:33.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:32 smithi138 ceph-mon[193606]: pgmap v1011: 119 pgs: 2 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 1 active+clean+scrubbing+deep, 2 active+undersized+degraded, 4 active+undersized, 94 active+clean; 739 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 1/75 objects degraded (1.333%); 6/75 objects misplaced (8.000%); 0 B/s, 115 objects/s recovering 2023-09-23T23:23:33.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:32 smithi138 ceph-mon[193606]: from='osd.7 [v2:172.21.15.138:6824/675905607,v1:172.21.15.138:6825/675905607]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T23:23:33.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:32 smithi138 ceph-mon[193606]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T23:23:33.204 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:33.364 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:33.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:33.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:33.369 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:23:33.417 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:33.468 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:23:33.496 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:33.540 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:23:33.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:33.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:23:33.547 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:33.648 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:33.688 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:34.227 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:23:34.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:34 smithi138 ceph-mon[193606]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T23:23:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:34 smithi138 ceph-mon[193606]: osdmap e342: 8 total, 7 up, 8 in 2023-09-23T23:23:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:34 smithi138 ceph-mon[193606]: from='osd.7 [v2:172.21.15.138:6824/675905607,v1:172.21.15.138:6825/675905607]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T23:23:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:34 smithi138 ceph-mon[193606]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T23:23:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:34 smithi138 ceph-mon[193606]: failed to encode map e342 with expected crc 2023-09-23T23:23:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:34 smithi138 ceph-mon[193606]: failed to encode map e342 with expected crc 2023-09-23T23:23:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:34 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1/75 objects degraded (1.333%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:34.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:34 smithi110 ceph-mon[212939]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T23:23:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:34 smithi110 ceph-mon[212939]: osdmap e342: 8 total, 7 up, 8 in 2023-09-23T23:23:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:34 smithi110 ceph-mon[212939]: from='osd.7 [v2:172.21.15.138:6824/675905607,v1:172.21.15.138:6825/675905607]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T23:23:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:34 smithi110 ceph-mon[212939]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T23:23:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:34 smithi110 ceph-mon[212939]: failed to encode map e342 with expected crc 2023-09-23T23:23:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:34 smithi110 ceph-mon[212939]: failed to encode map e342 with expected crc 2023-09-23T23:23:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:34 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1/75 objects degraded (1.333%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:34.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:34 smithi110 ceph-mon[221102]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T23:23:34.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:34 smithi110 ceph-mon[221102]: osdmap e342: 8 total, 7 up, 8 in 2023-09-23T23:23:34.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:34 smithi110 ceph-mon[221102]: from='osd.7 [v2:172.21.15.138:6824/675905607,v1:172.21.15.138:6825/675905607]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T23:23:34.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:34 smithi110 ceph-mon[221102]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T23:23:34.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:34 smithi110 ceph-mon[221102]: failed to encode map e342 with expected crc 2023-09-23T23:23:34.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:34 smithi110 ceph-mon[221102]: failed to encode map e342 with expected crc 2023-09-23T23:23:34.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:34 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1/75 objects degraded (1.333%), 2 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:34.644 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:34.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:34.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:34.748 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:34.779 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:34.832 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:35.085 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:35.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:35.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:35.188 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:35.213 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[212939]: pgmap v1013: 119 pgs: 2 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 95 active+clean; 739 KiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 366 op/s; 1/75 objects degraded (1.333%); 6/75 objects misplaced (8.000%); 0 B/s, 98 objects/s recovering 2023-09-23T23:23:35.213 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[212939]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:23:35.213 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[212939]: osd.7 [v2:172.21.15.138:6824/675905607,v1:172.21.15.138:6825/675905607] boot 2023-09-23T23:23:35.213 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[212939]: osdmap e343: 8 total, 8 up, 8 in 2023-09-23T23:23:35.213 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T23:23:35.213 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[212939]: failed to encode map e343 with expected crc 2023-09-23T23:23:35.213 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[212939]: failed to encode map e342 with expected crc 2023-09-23T23:23:35.214 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[212939]: failed to encode map e342 with expected crc 2023-09-23T23:23:35.214 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[212939]: failed to encode map e342 with expected crc 2023-09-23T23:23:35.214 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[212939]: osdmap e344: 8 total, 8 up, 8 in 2023-09-23T23:23:35.214 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[221102]: pgmap v1013: 119 pgs: 2 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 95 active+clean; 739 KiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 366 op/s; 1/75 objects degraded (1.333%); 6/75 objects misplaced (8.000%); 0 B/s, 98 objects/s recovering 2023-09-23T23:23:35.214 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[221102]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:23:35.214 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[221102]: osd.7 [v2:172.21.15.138:6824/675905607,v1:172.21.15.138:6825/675905607] boot 2023-09-23T23:23:35.214 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[221102]: osdmap e343: 8 total, 8 up, 8 in 2023-09-23T23:23:35.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T23:23:35.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[221102]: failed to encode map e343 with expected crc 2023-09-23T23:23:35.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[221102]: failed to encode map e342 with expected crc 2023-09-23T23:23:35.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[221102]: failed to encode map e342 with expected crc 2023-09-23T23:23:35.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[221102]: failed to encode map e342 with expected crc 2023-09-23T23:23:35.215 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:35 smithi110 ceph-mon[221102]: osdmap e344: 8 total, 8 up, 8 in 2023-09-23T23:23:35.216 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:35.250 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:23:35.388 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:35.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:35.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:35.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:35 smithi138 ceph-mon[193606]: pgmap v1013: 119 pgs: 2 active+clean+scrubbing, 15 active+clean+remapped, 1 active+recovering, 2 active+undersized+degraded, 4 active+undersized, 95 active+clean; 739 KiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 366 op/s; 1/75 objects degraded (1.333%); 6/75 objects misplaced (8.000%); 0 B/s, 98 objects/s recovering 2023-09-23T23:23:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:35 smithi138 ceph-mon[193606]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T23:23:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:35 smithi138 ceph-mon[193606]: osd.7 [v2:172.21.15.138:6824/675905607,v1:172.21.15.138:6825/675905607] boot 2023-09-23T23:23:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:35 smithi138 ceph-mon[193606]: osdmap e343: 8 total, 8 up, 8 in 2023-09-23T23:23:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:35 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T23:23:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:35 smithi138 ceph-mon[193606]: failed to encode map e343 with expected crc 2023-09-23T23:23:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:35 smithi138 ceph-mon[193606]: failed to encode map e342 with expected crc 2023-09-23T23:23:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:35 smithi138 ceph-mon[193606]: failed to encode map e342 with expected crc 2023-09-23T23:23:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:35 smithi138 ceph-mon[193606]: failed to encode map e342 with expected crc 2023-09-23T23:23:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:35 smithi138 ceph-mon[193606]: osdmap e344: 8 total, 8 up, 8 in 2023-09-23T23:23:35.491 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:35.520 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:35.603 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:23:35.671 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:35.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:35.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:35.727 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:35.775 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:23:35.805 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:35.852 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:23:35.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:35.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:23:35.956 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:35.986 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:36.065 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: failed to encode map e256 with expected crc 2023-09-23T23:23:36.066 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: failed to encode map e296 with expected crc 2023-09-23T23:23:36.066 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: failed to encode map e336 with expected crc 2023-09-23T23:23:36.066 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: failed to encode map e343 with expected crc 2023-09-23T23:23:36.066 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: failed to encode map e343 with expected crc 2023-09-23T23:23:36.066 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.066 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.066 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.066 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.066 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.066 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.067 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.067 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: 10.1 starting backfill to osd.7 from (0'0,0'0] MAX to 340'12386 2023-09-23T23:23:36.067 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: 10.0 starting backfill to osd.7 from (0'0,0'0] MAX to 340'12796 2023-09-23T23:23:36.067 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.067 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: 10.9 starting backfill to osd.7 from (0'0,0'0] MAX to 340'12654 2023-09-23T23:23:36.067 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: 10.8 starting backfill to osd.7 from (0'0,0'0] MAX to 340'12896 2023-09-23T23:23:36.067 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.067 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:36 smithi138 ceph-mon[193606]: osdmap e345: 8 total, 8 up, 8 in 2023-09-23T23:23:36.173 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:36.173 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: failed to encode map e256 with expected crc 2023-09-23T23:23:36.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: failed to encode map e296 with expected crc 2023-09-23T23:23:36.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: failed to encode map e336 with expected crc 2023-09-23T23:23:36.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: failed to encode map e343 with expected crc 2023-09-23T23:23:36.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: failed to encode map e343 with expected crc 2023-09-23T23:23:36.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.174 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.175 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.175 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: 10.1 starting backfill to osd.7 from (0'0,0'0] MAX to 340'12386 2023-09-23T23:23:36.175 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: 10.0 starting backfill to osd.7 from (0'0,0'0] MAX to 340'12796 2023-09-23T23:23:36.175 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.175 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: 10.9 starting backfill to osd.7 from (0'0,0'0] MAX to 340'12654 2023-09-23T23:23:36.175 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: 10.8 starting backfill to osd.7 from (0'0,0'0] MAX to 340'12896 2023-09-23T23:23:36.175 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.175 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[212939]: osdmap e345: 8 total, 8 up, 8 in 2023-09-23T23:23:36.175 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: failed to encode map e256 with expected crc 2023-09-23T23:23:36.175 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: failed to encode map e296 with expected crc 2023-09-23T23:23:36.175 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: failed to encode map e336 with expected crc 2023-09-23T23:23:36.176 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: failed to encode map e343 with expected crc 2023-09-23T23:23:36.176 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: failed to encode map e343 with expected crc 2023-09-23T23:23:36.176 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.176 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.176 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.176 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.176 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.176 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.176 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.176 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: 10.1 starting backfill to osd.7 from (0'0,0'0] MAX to 340'12386 2023-09-23T23:23:36.176 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: 10.0 starting backfill to osd.7 from (0'0,0'0] MAX to 340'12796 2023-09-23T23:23:36.176 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.176 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: 10.9 starting backfill to osd.7 from (0'0,0'0] MAX to 340'12654 2023-09-23T23:23:36.177 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: 10.8 starting backfill to osd.7 from (0'0,0'0] MAX to 340'12896 2023-09-23T23:23:36.177 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: failed to encode map e344 with expected crc 2023-09-23T23:23:36.177 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:36 smithi110 ceph-mon[221102]: osdmap e345: 8 total, 8 up, 8 in 2023-09-23T23:23:36.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:36.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:36.278 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:36.308 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:36.351 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:36.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:36.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:36.455 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:36.481 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:36.522 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:36.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:36.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:36.626 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:36.655 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:37.018 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:23:37.064 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[212939]: pgmap v1016: 119 pgs: 3 peering, 7 remapped+peering, 7 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+degraded, 4 active+undersized, 96 active+clean; 739 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 193 op/s; 1/75 objects degraded (1.333%); 2/75 objects misplaced (2.667%); 0 B/s, 391 objects/s recovering 2023-09-23T23:23:37.064 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[212939]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.064 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[212939]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.064 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[212939]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.065 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[212939]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.065 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[212939]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.065 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[212939]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.065 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[212939]: 10.c starting backfill to osd.7 from (0'0,0'0] MAX to 340'12636 2023-09-23T23:23:37.065 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1855525007' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:37.065 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:37.065 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:23:37.065 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:23:37.065 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[212939]: osdmap e346: 8 total, 8 up, 8 in 2023-09-23T23:23:37.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[221102]: pgmap v1016: 119 pgs: 3 peering, 7 remapped+peering, 7 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+degraded, 4 active+undersized, 96 active+clean; 739 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 193 op/s; 1/75 objects degraded (1.333%); 2/75 objects misplaced (2.667%); 0 B/s, 391 objects/s recovering 2023-09-23T23:23:37.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[221102]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[221102]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[221102]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[221102]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[221102]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[221102]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[221102]: 10.c starting backfill to osd.7 from (0'0,0'0] MAX to 340'12636 2023-09-23T23:23:37.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1855525007' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:37.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:37.066 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:23:37.067 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:23:37.067 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:37 smithi110 ceph-mon[221102]: osdmap e346: 8 total, 8 up, 8 in 2023-09-23T23:23:37.067 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:23:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:23:36] "GET /metrics HTTP/1.1" 200 36635 "" "Prometheus/2.43.0" 2023-09-23T23:23:37.114 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:37.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:37.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:37.156 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:37 smithi138 ceph-mon[193606]: pgmap v1016: 119 pgs: 3 peering, 7 remapped+peering, 7 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+degraded, 4 active+undersized, 96 active+clean; 739 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 193 op/s; 1/75 objects degraded (1.333%); 2/75 objects misplaced (2.667%); 0 B/s, 391 objects/s recovering 2023-09-23T23:23:37.156 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:37 smithi138 ceph-mon[193606]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.156 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:37 smithi138 ceph-mon[193606]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.156 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:37 smithi138 ceph-mon[193606]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.156 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:37 smithi138 ceph-mon[193606]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.156 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:37 smithi138 ceph-mon[193606]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.156 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:37 smithi138 ceph-mon[193606]: failed to encode map e345 with expected crc 2023-09-23T23:23:37.156 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:37 smithi138 ceph-mon[193606]: 10.c starting backfill to osd.7 from (0'0,0'0] MAX to 340'12636 2023-09-23T23:23:37.156 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:37 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1855525007' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:37.157 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:37 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:37.157 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:37 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:23:37.157 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:37 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:23:37.157 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:37 smithi138 ceph-mon[193606]: osdmap e346: 8 total, 8 up, 8 in 2023-09-23T23:23:37.228 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:37.256 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:37.366 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:23:37.366 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:23:37.366 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-09-23T23:23:37.366 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:23:37.854 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:37.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:37.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:37.957 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:37.987 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:38.023 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_5' removed 2023-09-23T23:23:38.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[212939]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[212939]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[212939]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[212939]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[212939]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[212939]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[212939]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1942920772' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:23:38.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1942920772' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:23:38.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[212939]: osdmap e347: 8 total, 8 up, 8 in 2023-09-23T23:23:38.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[221102]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[221102]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[221102]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[221102]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[221102]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[221102]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[221102]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1942920772' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:23:38.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1942920772' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:23:38.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:38 smithi110 ceph-mon[221102]: osdmap e347: 8 total, 8 up, 8 in 2023-09-23T23:23:38.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:38 smithi138 ceph-mon[193606]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:38 smithi138 ceph-mon[193606]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:38 smithi138 ceph-mon[193606]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:38 smithi138 ceph-mon[193606]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:38 smithi138 ceph-mon[193606]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:38 smithi138 ceph-mon[193606]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:38 smithi138 ceph-mon[193606]: failed to encode map e346 with expected crc 2023-09-23T23:23:38.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:38 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1942920772' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:23:38.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:38 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1942920772' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:23:38.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:38 smithi138 ceph-mon[193606]: osdmap e347: 8 total, 8 up, 8 in 2023-09-23T23:23:38.525 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:38.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:38.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:38.629 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:38.656 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:38.857 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:38.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:38.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:38.960 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:38.988 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:39.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:39 smithi110 ceph-mon[212939]: pgmap v1019: 119 pgs: 3 peering, 7 remapped+peering, 7 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+degraded, 4 active+undersized, 96 active+clean; 739 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 1/75 objects degraded (1.333%); 2/75 objects misplaced (2.667%); 0 B/s, 390 objects/s recovering 2023-09-23T23:23:39.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:39 smithi110 ceph-mon[212939]: failed to encode map e347 with expected crc 2023-09-23T23:23:39.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:39 smithi110 ceph-mon[212939]: failed to encode map e347 with expected crc 2023-09-23T23:23:39.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:39 smithi110 ceph-mon[212939]: failed to encode map e347 with expected crc 2023-09-23T23:23:39.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:39 smithi110 ceph-mon[212939]: failed to encode map e347 with expected crc 2023-09-23T23:23:39.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:39 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1/75 objects degraded (1.333%), 1 pg degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:39.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:39 smithi110 ceph-mon[221102]: pgmap v1019: 119 pgs: 3 peering, 7 remapped+peering, 7 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+degraded, 4 active+undersized, 96 active+clean; 739 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 1/75 objects degraded (1.333%); 2/75 objects misplaced (2.667%); 0 B/s, 390 objects/s recovering 2023-09-23T23:23:39.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:39 smithi110 ceph-mon[221102]: failed to encode map e347 with expected crc 2023-09-23T23:23:39.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:39 smithi110 ceph-mon[221102]: failed to encode map e347 with expected crc 2023-09-23T23:23:39.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:39 smithi110 ceph-mon[221102]: failed to encode map e347 with expected crc 2023-09-23T23:23:39.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:39 smithi110 ceph-mon[221102]: failed to encode map e347 with expected crc 2023-09-23T23:23:39.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:39 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1/75 objects degraded (1.333%), 1 pg degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:39.300 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:23:39.300 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:23:39.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:39 smithi138 ceph-mon[193606]: pgmap v1019: 119 pgs: 3 peering, 7 remapped+peering, 7 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+degraded, 4 active+undersized, 96 active+clean; 739 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 1/75 objects degraded (1.333%); 2/75 objects misplaced (2.667%); 0 B/s, 390 objects/s recovering 2023-09-23T23:23:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:39 smithi138 ceph-mon[193606]: failed to encode map e347 with expected crc 2023-09-23T23:23:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:39 smithi138 ceph-mon[193606]: failed to encode map e347 with expected crc 2023-09-23T23:23:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:39 smithi138 ceph-mon[193606]: failed to encode map e347 with expected crc 2023-09-23T23:23:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:39 smithi138 ceph-mon[193606]: failed to encode map e347 with expected crc 2023-09-23T23:23:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:39 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1/75 objects degraded (1.333%), 1 pg degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:23:39.443 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T23:23:39.443 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T23:23:39.444 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_6 2023-09-23T23:23:39.444 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_6 16 2023-09-23T23:23:39.529 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:39.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:39.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:39.634 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:23:39.666 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:39.712 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:23:39.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:39.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:23:39.818 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:23:39.844 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:39.887 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:23:39.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:39.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:23:39.986 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:23:39.994 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:23:40.020 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:40.061 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:23:40.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:40.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:23:40.108 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:23:40.169 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:23:40.207 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:40.240 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:40.273 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:23:40.276 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:40 smithi110 ceph-mon[212939]: failed to encode map e347 with expected crc 2023-09-23T23:23:40.276 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:40 smithi110 ceph-mon[221102]: failed to encode map e347 with expected crc 2023-09-23T23:23:40.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:40.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:23:40.286 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:23:40.311 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:23:40.349 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:23:40.373 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:23:40.377 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:23:40.396 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:23:40.403 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:40 smithi138 ceph-mon[193606]: failed to encode map e347 with expected crc 2023-09-23T23:23:40.405 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:40.416 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:23:40.444 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:23:40.451 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:23:40.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:40.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:23:40.467 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:23:40.468 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:23:40.468 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:23:40.468 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:23:40.469 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:23:40.469 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:23:40.469 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:23:40.469 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:23:40.470 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:23:40.556 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:23:40.583 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:40.624 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:23:40.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:40.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:23:40.724 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:40.733 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:40.769 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:40.916 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:40.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:40.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:41.020 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:41.046 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:41.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:41 smithi138 ceph-mon[193606]: pgmap v1021: 103 pgs: 3 peering, 100 active+clean; 739 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2023-09-23T23:23:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:41 smithi138 ceph-mon[193606]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/75 objects degraded (1.333%), 1 pg degraded, 5 pgs undersized) 2023-09-23T23:23:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:41 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1728699809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:41.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:41 smithi110 ceph-mon[212939]: pgmap v1021: 103 pgs: 3 peering, 100 active+clean; 739 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2023-09-23T23:23:41.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:41 smithi110 ceph-mon[212939]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/75 objects degraded (1.333%), 1 pg degraded, 5 pgs undersized) 2023-09-23T23:23:41.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:41 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1728699809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:41.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:41 smithi110 ceph-mon[221102]: pgmap v1021: 103 pgs: 3 peering, 100 active+clean; 739 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2023-09-23T23:23:41.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:41 smithi110 ceph-mon[221102]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/75 objects degraded (1.333%), 1 pg degraded, 5 pgs undersized) 2023-09-23T23:23:41.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:41 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1728699809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:41.637 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:41.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:41.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:41.741 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:41.770 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:41.869 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:41.869 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:41.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:41.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:41.976 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:42.006 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:42.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:42 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T23:23:42.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:42 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T23:23:42.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:42 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T23:23:42.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:42 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T23:23:42.412 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:42.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:42.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:42 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T23:23:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:42 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T23:23:42.518 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:42.545 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:43.041 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:43.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:43.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:43.086 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_6' created 2023-09-23T23:23:43.111 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 2, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:23:43.111 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:23:43.145 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:23:43.178 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:43.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: pgmap v1022: 103 pgs: 103 active+clean; 739 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2023-09-23T23:23:43.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T23:23:43.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: osdmap e348: 8 total, 8 up, 8 in 2023-09-23T23:23:43.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.346 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.346 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1939439840' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-09-23T23:23:43.346 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [6, 4]}]: dispatch 2023-09-23T23:23:43.346 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T23:23:43.346 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [6, 4]}]: dispatch 2023-09-23T23:23:43.346 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T23:23:43.346 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:43.346 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:23:43.346 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: pgmap v1022: 103 pgs: 103 active+clean; 739 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2023-09-23T23:23:43.346 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T23:23:43.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: osdmap e348: 8 total, 8 up, 8 in 2023-09-23T23:23:43.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1939439840' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-09-23T23:23:43.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [6, 4]}]: dispatch 2023-09-23T23:23:43.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T23:23:43.348 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [6, 4]}]: dispatch 2023-09-23T23:23:43.348 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T23:23:43.348 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:43.348 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:23:43.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: pgmap v1022: 103 pgs: 103 active+clean; 739 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2023-09-23T23:23:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T23:23:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: osdmap e348: 8 total, 8 up, 8 in 2023-09-23T23:23:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: failed to encode map e348 with expected crc 2023-09-23T23:23:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1939439840' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-09-23T23:23:43.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [6, 4]}]: dispatch 2023-09-23T23:23:43.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T23:23:43.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [6, 4]}]: dispatch 2023-09-23T23:23:43.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T23:23:43.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:43.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:23:43.794 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:23:44.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[212939]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[212939]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[212939]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1939439840' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-09-23T23:23:44.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [6, 4]}]': finished 2023-09-23T23:23:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T23:23:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[212939]: osdmap e349: 8 total, 8 up, 8 in 2023-09-23T23:23:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[212939]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[212939]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[212939]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[212939]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[212939]: failed to encode map e350 with expected crc 2023-09-23T23:23:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[212939]: failed to encode map e350 with expected crc 2023-09-23T23:23:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[212939]: osdmap e350: 8 total, 8 up, 8 in 2023-09-23T23:23:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[212939]: failed to encode map e350 with expected crc 2023-09-23T23:23:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[212939]: failed to encode map e350 with expected crc 2023-09-23T23:23:44.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[212939]: failed to encode map e350 with expected crc 2023-09-23T23:23:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[221102]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[221102]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[221102]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1939439840' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-09-23T23:23:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [6, 4]}]': finished 2023-09-23T23:23:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T23:23:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[221102]: osdmap e349: 8 total, 8 up, 8 in 2023-09-23T23:23:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[221102]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[221102]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[221102]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[221102]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[221102]: failed to encode map e350 with expected crc 2023-09-23T23:23:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[221102]: failed to encode map e350 with expected crc 2023-09-23T23:23:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[221102]: osdmap e350: 8 total, 8 up, 8 in 2023-09-23T23:23:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[221102]: failed to encode map e350 with expected crc 2023-09-23T23:23:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[221102]: failed to encode map e350 with expected crc 2023-09-23T23:23:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:44 smithi110 ceph-mon[221102]: failed to encode map e350 with expected crc 2023-09-23T23:23:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:44 smithi138 ceph-mon[193606]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:44 smithi138 ceph-mon[193606]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:44 smithi138 ceph-mon[193606]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:44 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1939439840' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-09-23T23:23:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:44 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [6, 4]}]': finished 2023-09-23T23:23:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:44 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T23:23:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:44 smithi138 ceph-mon[193606]: osdmap e349: 8 total, 8 up, 8 in 2023-09-23T23:23:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:44 smithi138 ceph-mon[193606]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:44 smithi138 ceph-mon[193606]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:44 smithi138 ceph-mon[193606]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:44 smithi138 ceph-mon[193606]: failed to encode map e349 with expected crc 2023-09-23T23:23:44.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:44 smithi138 ceph-mon[193606]: failed to encode map e350 with expected crc 2023-09-23T23:23:44.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:44 smithi138 ceph-mon[193606]: failed to encode map e350 with expected crc 2023-09-23T23:23:44.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:44 smithi138 ceph-mon[193606]: osdmap e350: 8 total, 8 up, 8 in 2023-09-23T23:23:44.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:44 smithi138 ceph-mon[193606]: failed to encode map e350 with expected crc 2023-09-23T23:23:44.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:44 smithi138 ceph-mon[193606]: failed to encode map e350 with expected crc 2023-09-23T23:23:44.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:44 smithi138 ceph-mon[193606]: failed to encode map e350 with expected crc 2023-09-23T23:23:44.582 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:23:44.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:44.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:23:44.687 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:44.699 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_6 rados --yes-i-really-mean-it || true 2023-09-23T23:23:44.718 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:44.781 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:44.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:44.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:44.854 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:44.885 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:23:44.916 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:44.966 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:23:44.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:44.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:23:45.072 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:45.102 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:45.174 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:23:45.174 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:23:45.174 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:23:45.174 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:23:45.174 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:23:45.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:45 smithi110 ceph-mon[212939]: pgmap v1026: 119 pgs: 1 peering, 2 remapped+peering, 3 creating+peering, 13 unknown, 100 active+clean; 739 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T23:23:45.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:45 smithi110 ceph-mon[212939]: failed to encode map e351 with expected crc 2023-09-23T23:23:45.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:45 smithi110 ceph-mon[212939]: osdmap e351: 8 total, 8 up, 8 in 2023-09-23T23:23:45.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:45 smithi110 ceph-mon[212939]: failed to encode map e351 with expected crc 2023-09-23T23:23:45.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:45 smithi110 ceph-mon[212939]: failed to encode map e351 with expected crc 2023-09-23T23:23:45.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:45 smithi110 ceph-mon[212939]: failed to encode map e351 with expected crc 2023-09-23T23:23:45.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:45 smithi110 ceph-mon[212939]: failed to encode map e351 with expected crc 2023-09-23T23:23:45.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:45 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3490826174' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:23:45.187 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:45 smithi110 ceph-mon[221102]: pgmap v1026: 119 pgs: 1 peering, 2 remapped+peering, 3 creating+peering, 13 unknown, 100 active+clean; 739 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T23:23:45.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:45 smithi110 ceph-mon[221102]: failed to encode map e351 with expected crc 2023-09-23T23:23:45.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:45 smithi110 ceph-mon[221102]: osdmap e351: 8 total, 8 up, 8 in 2023-09-23T23:23:45.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:45 smithi110 ceph-mon[221102]: failed to encode map e351 with expected crc 2023-09-23T23:23:45.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:45 smithi110 ceph-mon[221102]: failed to encode map e351 with expected crc 2023-09-23T23:23:45.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:45 smithi110 ceph-mon[221102]: failed to encode map e351 with expected crc 2023-09-23T23:23:45.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:45 smithi110 ceph-mon[221102]: failed to encode map e351 with expected crc 2023-09-23T23:23:45.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:45 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3490826174' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:23:45.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:45 smithi138 ceph-mon[193606]: pgmap v1026: 119 pgs: 1 peering, 2 remapped+peering, 3 creating+peering, 13 unknown, 100 active+clean; 739 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T23:23:45.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:45 smithi138 ceph-mon[193606]: failed to encode map e351 with expected crc 2023-09-23T23:23:45.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:45 smithi138 ceph-mon[193606]: osdmap e351: 8 total, 8 up, 8 in 2023-09-23T23:23:45.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:45 smithi138 ceph-mon[193606]: failed to encode map e351 with expected crc 2023-09-23T23:23:45.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:45 smithi138 ceph-mon[193606]: failed to encode map e351 with expected crc 2023-09-23T23:23:45.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:45 smithi138 ceph-mon[193606]: failed to encode map e351 with expected crc 2023-09-23T23:23:45.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:45 smithi138 ceph-mon[193606]: failed to encode map e351 with expected crc 2023-09-23T23:23:45.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:45 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3490826174' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:23:45.201 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:45.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:45.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:45.305 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:23:45.333 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:45.378 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:23:45.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:45.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:23:45.454 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:23:45.484 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:23:45.510 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:45.553 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:23:45.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:45.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:23:45.658 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:23:45.686 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:46 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:23:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:46 smithi138 ceph-mon[193606]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:46 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3490826174' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:23:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:46 smithi138 ceph-mon[193606]: osdmap e352: 8 total, 8 up, 8 in 2023-09-23T23:23:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:46 smithi138 ceph-mon[193606]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:46 smithi138 ceph-mon[193606]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:46 smithi138 ceph-mon[193606]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:46 smithi138 ceph-mon[193606]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:46 smithi138 ceph-mon[193606]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:46 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:23:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:46 smithi138 ceph-mon[193606]: 1.9 scrub ok 2023-09-23T23:23:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:23:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[212939]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3490826174' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:23:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[212939]: osdmap e352: 8 total, 8 up, 8 in 2023-09-23T23:23:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[212939]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[212939]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[212939]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[212939]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[212939]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:23:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[212939]: 1.9 scrub ok 2023-09-23T23:23:46.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:23:46.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[221102]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3490826174' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:23:46.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[221102]: osdmap e352: 8 total, 8 up, 8 in 2023-09-23T23:23:46.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[221102]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[221102]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[221102]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[221102]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[221102]: failed to encode map e352 with expected crc 2023-09-23T23:23:46.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:23:46.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:46 smithi110 ceph-mon[221102]: 1.9 scrub ok 2023-09-23T23:23:46.884 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:23:46.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:46.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:23:46.989 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:47.017 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:47.063 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:47.066 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:23:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:23:46] "GET /metrics HTTP/1.1" 200 34219 "" "Prometheus/2.43.0" 2023-09-23T23:23:47.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:47.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:47.169 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:47.171 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:23:47.198 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:47.222 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:47.244 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:47.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:47.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:47.349 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:47.373 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:47 smithi138 ceph-mon[193606]: pgmap v1029: 119 pgs: 4 active+undersized+remapped, 5 peering, 2 remapped+peering, 3 creating+peering, 2 unknown, 103 active+clean; 739 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 27 KiB/s, 0 objects/s recovering 2023-09-23T23:23:47.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:47 smithi138 ceph-mon[193606]: 1.4 scrub starts 2023-09-23T23:23:47.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:47 smithi138 ceph-mon[193606]: 1.4 scrub ok 2023-09-23T23:23:47.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:47 smithi138 ceph-mon[193606]: 1.3 scrub starts 2023-09-23T23:23:47.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:47 smithi138 ceph-mon[193606]: 1.3 scrub ok 2023-09-23T23:23:47.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:47 smithi138 ceph-mon[193606]: osdmap e353: 8 total, 8 up, 8 in 2023-09-23T23:23:47.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:47 smithi138 ceph-mon[193606]: failed to encode map e353 with expected crc 2023-09-23T23:23:47.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:47 smithi138 ceph-mon[193606]: failed to encode map e353 with expected crc 2023-09-23T23:23:47.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:47 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:23:47.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:47 smithi138 ceph-mon[193606]: failed to encode map e353 with expected crc 2023-09-23T23:23:47.375 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:47 smithi138 ceph-mon[193606]: 1.15 scrub starts 2023-09-23T23:23:47.375 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:47 smithi138 ceph-mon[193606]: 1.15 scrub ok 2023-09-23T23:23:47.375 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:47.387 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[212939]: pgmap v1029: 119 pgs: 4 active+undersized+remapped, 5 peering, 2 remapped+peering, 3 creating+peering, 2 unknown, 103 active+clean; 739 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 27 KiB/s, 0 objects/s recovering 2023-09-23T23:23:47.387 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[212939]: 1.4 scrub starts 2023-09-23T23:23:47.387 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[212939]: 1.4 scrub ok 2023-09-23T23:23:47.387 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[212939]: 1.3 scrub starts 2023-09-23T23:23:47.387 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[212939]: 1.3 scrub ok 2023-09-23T23:23:47.387 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[212939]: osdmap e353: 8 total, 8 up, 8 in 2023-09-23T23:23:47.387 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[212939]: failed to encode map e353 with expected crc 2023-09-23T23:23:47.387 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[212939]: failed to encode map e353 with expected crc 2023-09-23T23:23:47.388 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:23:47.388 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[212939]: failed to encode map e353 with expected crc 2023-09-23T23:23:47.388 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[212939]: 1.15 scrub starts 2023-09-23T23:23:47.388 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[212939]: 1.15 scrub ok 2023-09-23T23:23:47.388 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[221102]: pgmap v1029: 119 pgs: 4 active+undersized+remapped, 5 peering, 2 remapped+peering, 3 creating+peering, 2 unknown, 103 active+clean; 739 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 27 KiB/s, 0 objects/s recovering 2023-09-23T23:23:47.388 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[221102]: 1.4 scrub starts 2023-09-23T23:23:47.388 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[221102]: 1.4 scrub ok 2023-09-23T23:23:47.388 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[221102]: 1.3 scrub starts 2023-09-23T23:23:47.389 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[221102]: 1.3 scrub ok 2023-09-23T23:23:47.389 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[221102]: osdmap e353: 8 total, 8 up, 8 in 2023-09-23T23:23:47.389 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[221102]: failed to encode map e353 with expected crc 2023-09-23T23:23:47.389 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[221102]: failed to encode map e353 with expected crc 2023-09-23T23:23:47.389 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:23:47.389 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[221102]: failed to encode map e353 with expected crc 2023-09-23T23:23:47.390 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[221102]: 1.15 scrub starts 2023-09-23T23:23:47.390 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:47 smithi110 ceph-mon[221102]: 1.15 scrub ok 2023-09-23T23:23:47.490 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:47.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:47.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:47.593 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:47.620 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:47.625 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:23:47.648 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:23:47.665 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:47.668 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:23:47.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:47.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:47.689 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:23:47.717 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:23:47.738 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:23:47.760 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:23:47.771 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:47.781 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:23:47.782 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:23:47.783 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:23:47.783 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:23:47.783 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:23:47.783 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:23:47.783 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:23:47.783 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:23:47.783 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:23:47.814 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:48.060 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:48.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:48.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:48.164 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:48.191 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:48.353 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:48 smithi110 ceph-mon[212939]: failed to encode map e353 with expected crc 2023-09-23T23:23:48.353 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:48 smithi110 ceph-mon[212939]: 1.5 scrub starts 2023-09-23T23:23:48.353 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:48 smithi110 ceph-mon[212939]: 1.5 scrub ok 2023-09-23T23:23:48.353 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:48 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3569245870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:48.354 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:48 smithi110 ceph-mon[221102]: failed to encode map e353 with expected crc 2023-09-23T23:23:48.354 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:48 smithi110 ceph-mon[221102]: 1.5 scrub starts 2023-09-23T23:23:48.354 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:48 smithi110 ceph-mon[221102]: 1.5 scrub ok 2023-09-23T23:23:48.354 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:48 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3569245870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:48.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:48 smithi138 ceph-mon[193606]: failed to encode map e353 with expected crc 2023-09-23T23:23:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:48 smithi138 ceph-mon[193606]: 1.5 scrub starts 2023-09-23T23:23:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:48 smithi138 ceph-mon[193606]: 1.5 scrub ok 2023-09-23T23:23:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:48 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3569245870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:23:48.935 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:48.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:48.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:49.040 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:49.067 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:49.123 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:23:49.161 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:49.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:49.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:49 smithi110 ceph-mon[212939]: pgmap v1031: 119 pgs: 4 active+undersized+remapped, 5 peering, 2 remapped+peering, 3 creating+peering, 2 unknown, 103 active+clean; 739 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s, 0 objects/s recovering 2023-09-23T23:23:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:49 smithi110 ceph-mon[212939]: 1.7 scrub starts 2023-09-23T23:23:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:49 smithi110 ceph-mon[212939]: 1.7 scrub ok 2023-09-23T23:23:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:49 smithi110 ceph-mon[212939]: 1.12 scrub starts 2023-09-23T23:23:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:49 smithi110 ceph-mon[212939]: 1.12 scrub ok 2023-09-23T23:23:49.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:49 smithi110 ceph-mon[221102]: pgmap v1031: 119 pgs: 4 active+undersized+remapped, 5 peering, 2 remapped+peering, 3 creating+peering, 2 unknown, 103 active+clean; 739 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s, 0 objects/s recovering 2023-09-23T23:23:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:49 smithi110 ceph-mon[221102]: 1.7 scrub starts 2023-09-23T23:23:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:49 smithi110 ceph-mon[221102]: 1.7 scrub ok 2023-09-23T23:23:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:49 smithi110 ceph-mon[221102]: 1.12 scrub starts 2023-09-23T23:23:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:49 smithi110 ceph-mon[221102]: 1.12 scrub ok 2023-09-23T23:23:49.224 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:49.266 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:49.313 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:49.418 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:49.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:49.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:49.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:49 smithi138 ceph-mon[193606]: pgmap v1031: 119 pgs: 4 active+undersized+remapped, 5 peering, 2 remapped+peering, 3 creating+peering, 2 unknown, 103 active+clean; 739 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s, 0 objects/s recovering 2023-09-23T23:23:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:49 smithi138 ceph-mon[193606]: 1.7 scrub starts 2023-09-23T23:23:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:49 smithi138 ceph-mon[193606]: 1.7 scrub ok 2023-09-23T23:23:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:49 smithi138 ceph-mon[193606]: 1.12 scrub starts 2023-09-23T23:23:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:49 smithi138 ceph-mon[193606]: 1.12 scrub ok 2023-09-23T23:23:49.522 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:49.554 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:49.600 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:49.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:49.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:49.704 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:49.734 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:49.785 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:49.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:49.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:49.888 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:49.916 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:50.066 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:50.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:50.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:50.082 INFO:teuthology.orchestra.run.smithi110.stderr:enabled application 'rados' on pool 'unique_pool_6' 2023-09-23T23:23:50.170 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:23:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:50 smithi110 ceph-mon[221102]: 1.c scrub starts 2023-09-23T23:23:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:50 smithi110 ceph-mon[221102]: 1.c scrub ok 2023-09-23T23:23:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:50 smithi110 ceph-mon[221102]: 1.1a scrub starts 2023-09-23T23:23:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:50 smithi110 ceph-mon[221102]: 1.1a scrub ok 2023-09-23T23:23:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:50 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2404124865' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:23:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:50 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:50.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:50 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:50.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:50 smithi110 ceph-mon[212939]: 1.c scrub starts 2023-09-23T23:23:50.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:50 smithi110 ceph-mon[212939]: 1.c scrub ok 2023-09-23T23:23:50.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:50 smithi110 ceph-mon[212939]: 1.1a scrub starts 2023-09-23T23:23:50.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:50 smithi110 ceph-mon[212939]: 1.1a scrub ok 2023-09-23T23:23:50.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:50 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2404124865' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:23:50.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:50 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:50.197 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:50 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:50.198 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:50.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:50 smithi138 ceph-mon[193606]: 1.c scrub starts 2023-09-23T23:23:50.827 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:50 smithi138 ceph-mon[193606]: 1.c scrub ok 2023-09-23T23:23:50.828 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:50 smithi138 ceph-mon[193606]: 1.1a scrub starts 2023-09-23T23:23:50.828 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:50 smithi138 ceph-mon[193606]: 1.1a scrub ok 2023-09-23T23:23:50.828 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:50 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2404124865' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:23:50.828 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:50 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:50.828 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:50 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:50.923 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:23:50.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:50.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:23:50.980 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:51.027 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:51.059 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:51.105 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:51.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:51.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:51.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[212939]: 1.1 scrub starts 2023-09-23T23:23:51.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[212939]: 1.1 scrub ok 2023-09-23T23:23:51.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[212939]: pgmap v1032: 119 pgs: 2 peering, 117 active+clean; 628 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 32 KiB/s, 0 objects/s recovering 2023-09-23T23:23:51.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[212939]: failed to encode map e354 with expected crc 2023-09-23T23:23:51.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2404124865' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:23:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[212939]: osdmap e354: 8 total, 8 up, 8 in 2023-09-23T23:23:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[212939]: 1.b scrub starts 2023-09-23T23:23:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[212939]: 1.b scrub ok 2023-09-23T23:23:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[212939]: 1.1b scrub starts 2023-09-23T23:23:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[212939]: failed to encode map e354 with expected crc 2023-09-23T23:23:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[212939]: 1.1b scrub ok 2023-09-23T23:23:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[212939]: failed to encode map e354 with expected crc 2023-09-23T23:23:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[212939]: failed to encode map e354 with expected crc 2023-09-23T23:23:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[212939]: failed to encode map e354 with expected crc 2023-09-23T23:23:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[212939]: 1.10 scrub starts 2023-09-23T23:23:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[212939]: 1.10 scrub ok 2023-09-23T23:23:51.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[221102]: 1.1 scrub starts 2023-09-23T23:23:51.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[221102]: 1.1 scrub ok 2023-09-23T23:23:51.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[221102]: pgmap v1032: 119 pgs: 2 peering, 117 active+clean; 628 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 32 KiB/s, 0 objects/s recovering 2023-09-23T23:23:51.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[221102]: failed to encode map e354 with expected crc 2023-09-23T23:23:51.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2404124865' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:23:51.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[221102]: osdmap e354: 8 total, 8 up, 8 in 2023-09-23T23:23:51.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[221102]: 1.b scrub starts 2023-09-23T23:23:51.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[221102]: 1.b scrub ok 2023-09-23T23:23:51.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[221102]: 1.1b scrub starts 2023-09-23T23:23:51.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[221102]: failed to encode map e354 with expected crc 2023-09-23T23:23:51.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[221102]: 1.1b scrub ok 2023-09-23T23:23:51.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[221102]: failed to encode map e354 with expected crc 2023-09-23T23:23:51.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[221102]: failed to encode map e354 with expected crc 2023-09-23T23:23:51.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[221102]: failed to encode map e354 with expected crc 2023-09-23T23:23:51.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[221102]: 1.10 scrub starts 2023-09-23T23:23:51.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:51 smithi110 ceph-mon[221102]: 1.10 scrub ok 2023-09-23T23:23:51.212 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:23:51.239 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:51.245 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:23:51.281 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:23:51.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:51.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:23:51.388 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:23:51.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:51 smithi138 ceph-mon[193606]: 1.1 scrub starts 2023-09-23T23:23:51.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:51 smithi138 ceph-mon[193606]: 1.1 scrub ok 2023-09-23T23:23:51.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:51 smithi138 ceph-mon[193606]: pgmap v1032: 119 pgs: 2 peering, 117 active+clean; 628 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 32 KiB/s, 0 objects/s recovering 2023-09-23T23:23:51.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:51 smithi138 ceph-mon[193606]: failed to encode map e354 with expected crc 2023-09-23T23:23:51.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:51 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2404124865' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:23:51.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:51 smithi138 ceph-mon[193606]: osdmap e354: 8 total, 8 up, 8 in 2023-09-23T23:23:51.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:51 smithi138 ceph-mon[193606]: 1.b scrub starts 2023-09-23T23:23:51.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:51 smithi138 ceph-mon[193606]: 1.b scrub ok 2023-09-23T23:23:51.413 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:51 smithi138 ceph-mon[193606]: 1.1b scrub starts 2023-09-23T23:23:51.413 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:51 smithi138 ceph-mon[193606]: failed to encode map e354 with expected crc 2023-09-23T23:23:51.413 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:51 smithi138 ceph-mon[193606]: 1.1b scrub ok 2023-09-23T23:23:51.413 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:51 smithi138 ceph-mon[193606]: failed to encode map e354 with expected crc 2023-09-23T23:23:51.413 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:51 smithi138 ceph-mon[193606]: failed to encode map e354 with expected crc 2023-09-23T23:23:51.413 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:51 smithi138 ceph-mon[193606]: failed to encode map e354 with expected crc 2023-09-23T23:23:51.413 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:51 smithi138 ceph-mon[193606]: 1.10 scrub starts 2023-09-23T23:23:51.413 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:51 smithi138 ceph-mon[193606]: 1.10 scrub ok 2023-09-23T23:23:51.414 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:51.457 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:23:51.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:51.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:23:51.562 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:23:51.588 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:51.630 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:23:51.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:51.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:23:51.739 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:51.777 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:51.878 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:51.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:51.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:51.912 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:23:51.912 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:23:51.983 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:52.011 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:52.061 DEBUG:teuthology.orchestra.run.smithi110:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_6 bench 90 write' 2023-09-23T23:23:52.061 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T23:23:52.062 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:23:52.064 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T23:23:52.064 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T23:23:52.081 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:52 smithi110 ceph-mon[212939]: 1.8 scrub starts 2023-09-23T23:23:52.081 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:52 smithi110 ceph-mon[212939]: 1.8 scrub ok 2023-09-23T23:23:52.082 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:52 smithi110 ceph-mon[212939]: 1.19 scrub starts 2023-09-23T23:23:52.082 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:52 smithi110 ceph-mon[212939]: 1.19 scrub ok 2023-09-23T23:23:52.082 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:52 smithi110 ceph-mon[212939]: 1.6 scrub starts 2023-09-23T23:23:52.082 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:52 smithi110 ceph-mon[212939]: 1.6 scrub ok 2023-09-23T23:23:52.082 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:52 smithi110 ceph-mon[212939]: 1.1c scrub starts 2023-09-23T23:23:52.082 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:52 smithi110 ceph-mon[212939]: 1.1c scrub ok 2023-09-23T23:23:52.111 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:hints = 1 2023-09-23T23:23:52.112 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T23:23:52.112 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object prefix: benchmark_data_smithi110_433746 2023-09-23T23:23:52.112 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:23:52.112 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 0 0 0 0 0 0 - 0 2023-09-23T23:23:52.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:52 smithi138 ceph-mon[193606]: 1.8 scrub starts 2023-09-23T23:23:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:52 smithi138 ceph-mon[193606]: 1.8 scrub ok 2023-09-23T23:23:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:52 smithi138 ceph-mon[193606]: 1.19 scrub starts 2023-09-23T23:23:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:52 smithi138 ceph-mon[193606]: 1.19 scrub ok 2023-09-23T23:23:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:52 smithi138 ceph-mon[193606]: 1.6 scrub starts 2023-09-23T23:23:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:52 smithi138 ceph-mon[193606]: 1.6 scrub ok 2023-09-23T23:23:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:52 smithi138 ceph-mon[193606]: 1.1c scrub starts 2023-09-23T23:23:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:52 smithi138 ceph-mon[193606]: 1.1c scrub ok 2023-09-23T23:23:52.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:52 smithi110 ceph-mon[221102]: 1.8 scrub starts 2023-09-23T23:23:52.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:52 smithi110 ceph-mon[221102]: 1.8 scrub ok 2023-09-23T23:23:52.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:52 smithi110 ceph-mon[221102]: 1.19 scrub starts 2023-09-23T23:23:52.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:52 smithi110 ceph-mon[221102]: 1.19 scrub ok 2023-09-23T23:23:52.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:52 smithi110 ceph-mon[221102]: 1.6 scrub starts 2023-09-23T23:23:52.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:52 smithi110 ceph-mon[221102]: 1.6 scrub ok 2023-09-23T23:23:52.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:52 smithi110 ceph-mon[221102]: 1.1c scrub starts 2023-09-23T23:23:52.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:52 smithi110 ceph-mon[221102]: 1.1c scrub ok 2023-09-23T23:23:53.112 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 1 16 2490 2474 154.608 154.625 0.00136734 0.00592917 2023-09-23T23:23:53.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:53 smithi138 ceph-mon[193606]: pgmap v1034: 119 pgs: 119 active+clean; 602 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.4 KiB/s, 0 objects/s recovering 2023-09-23T23:23:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:53 smithi138 ceph-mon[193606]: 1.a scrub starts 2023-09-23T23:23:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:53 smithi138 ceph-mon[193606]: 1.a scrub ok 2023-09-23T23:23:53.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:53 smithi110 ceph-mon[221102]: pgmap v1034: 119 pgs: 119 active+clean; 602 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.4 KiB/s, 0 objects/s recovering 2023-09-23T23:23:53.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:53 smithi110 ceph-mon[221102]: 1.a scrub starts 2023-09-23T23:23:53.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:53 smithi110 ceph-mon[221102]: 1.a scrub ok 2023-09-23T23:23:53.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:53 smithi110 ceph-mon[212939]: pgmap v1034: 119 pgs: 119 active+clean; 602 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.4 KiB/s, 0 objects/s recovering 2023-09-23T23:23:53.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:53 smithi110 ceph-mon[212939]: 1.a scrub starts 2023-09-23T23:23:53.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:53 smithi110 ceph-mon[212939]: 1.a scrub ok 2023-09-23T23:23:53.484 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:53.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:53.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:53.595 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:53.630 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:53.665 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:23:53.669 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:53.902 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:53.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:53.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:54.020 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:23:54.090 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:54 smithi110 ceph-mon[212939]: 1.2 scrub starts 2023-09-23T23:23:54.090 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:54 smithi110 ceph-mon[212939]: 1.2 scrub ok 2023-09-23T23:23:54.098 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:54 smithi110 ceph-mon[221102]: 1.2 scrub starts 2023-09-23T23:23:54.098 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:54 smithi110 ceph-mon[221102]: 1.2 scrub ok 2023-09-23T23:23:54.113 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 2 16 4741 4725 147.642 140.688 0.00870659 0.00674395 2023-09-23T23:23:54.113 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:54.113 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:54.224 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:23:54.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:54.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:23:54.334 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:23:54.394 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:54 smithi138 ceph-mon[193606]: 1.2 scrub starts 2023-09-23T23:23:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:54 smithi138 ceph-mon[193606]: 1.2 scrub ok 2023-09-23T23:23:54.839 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:23:54.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:54.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:23:54.946 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:23:54.987 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:55.062 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:23:55.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:55.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:23:55.114 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 3 16 7438 7422 154.504 168.562 0.00263384 0.00646434 2023-09-23T23:23:55.169 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:23:55.224 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:55.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:55 smithi110 ceph-mon[212939]: pgmap v1035: 119 pgs: 119 active+clean; 51 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.2 MiB/s wr, 97 op/s 2023-09-23T23:23:55.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:55 smithi110 ceph-mon[212939]: 1.11 scrub starts 2023-09-23T23:23:55.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:55 smithi110 ceph-mon[212939]: 1.11 scrub ok 2023-09-23T23:23:55.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:55 smithi110 ceph-mon[212939]: 1.17 scrub starts 2023-09-23T23:23:55.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:55 smithi110 ceph-mon[212939]: 1.17 scrub ok 2023-09-23T23:23:55.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:55 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/14163411' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:55.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:55 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:55.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:55 smithi110 ceph-mon[221102]: pgmap v1035: 119 pgs: 119 active+clean; 51 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.2 MiB/s wr, 97 op/s 2023-09-23T23:23:55.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:55 smithi110 ceph-mon[221102]: 1.11 scrub starts 2023-09-23T23:23:55.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:55 smithi110 ceph-mon[221102]: 1.11 scrub ok 2023-09-23T23:23:55.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:55 smithi110 ceph-mon[221102]: 1.17 scrub starts 2023-09-23T23:23:55.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:55 smithi110 ceph-mon[221102]: 1.17 scrub ok 2023-09-23T23:23:55.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:55 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/14163411' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:55.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:55 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:55.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:55 smithi138 ceph-mon[193606]: pgmap v1035: 119 pgs: 119 active+clean; 51 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.2 MiB/s wr, 97 op/s 2023-09-23T23:23:55.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:55 smithi138 ceph-mon[193606]: 1.11 scrub starts 2023-09-23T23:23:55.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:55 smithi138 ceph-mon[193606]: 1.11 scrub ok 2023-09-23T23:23:55.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:55 smithi138 ceph-mon[193606]: 1.17 scrub starts 2023-09-23T23:23:55.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:55 smithi138 ceph-mon[193606]: 1.17 scrub ok 2023-09-23T23:23:55.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:55 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/14163411' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:55.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:55 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:23:55.463 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:23:56.114 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 4 16 10300 10284 160.589 178.875 0.00322602 0.00606857 2023-09-23T23:23:56.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:56 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:23:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:56 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:23:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:56 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:23:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:56 smithi138 ceph-mon[193606]: osdmap e355: 8 total, 8 up, 8 in 2023-09-23T23:23:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:56 smithi138 ceph-mon[193606]: failed to encode map e355 with expected crc 2023-09-23T23:23:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:56 smithi138 ceph-mon[193606]: failed to encode map e355 with expected crc 2023-09-23T23:23:56.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:56 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:23:56.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:56 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:23:56.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:56 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:23:56.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:56 smithi110 ceph-mon[221102]: osdmap e355: 8 total, 8 up, 8 in 2023-09-23T23:23:56.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:56 smithi110 ceph-mon[221102]: failed to encode map e355 with expected crc 2023-09-23T23:23:56.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:56 smithi110 ceph-mon[221102]: failed to encode map e355 with expected crc 2023-09-23T23:23:56.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:56 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:23:56.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:56 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:23:56.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:56 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:23:56.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:56 smithi110 ceph-mon[212939]: osdmap e355: 8 total, 8 up, 8 in 2023-09-23T23:23:56.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:56 smithi110 ceph-mon[212939]: failed to encode map e355 with expected crc 2023-09-23T23:23:56.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:56 smithi110 ceph-mon[212939]: failed to encode map e355 with expected crc 2023-09-23T23:23:57.093 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:23:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:23:56] "GET /metrics HTTP/1.1" 200 34219 "" "Prometheus/2.43.0" 2023-09-23T23:23:57.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:57 smithi110 ceph-mon[221102]: 1.0 scrub starts 2023-09-23T23:23:57.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:57 smithi110 ceph-mon[221102]: 1.0 scrub ok 2023-09-23T23:23:57.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:57 smithi110 ceph-mon[221102]: pgmap v1037: 119 pgs: 119 active+clean; 309 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 38 MiB/s wr, 612 op/s 2023-09-23T23:23:57.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:57 smithi110 ceph-mon[221102]: 1.14 scrub starts 2023-09-23T23:23:57.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:57 smithi110 ceph-mon[221102]: 1.14 scrub ok 2023-09-23T23:23:57.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:57 smithi110 ceph-mon[221102]: 1.1d scrub starts 2023-09-23T23:23:57.094 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:57 smithi110 ceph-mon[221102]: 1.1d scrub ok 2023-09-23T23:23:57.114 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 5 16 13364 13348 166.765 191.5 0.00784286 0.00599091 2023-09-23T23:23:57.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:57 smithi138 ceph-mon[193606]: 1.0 scrub starts 2023-09-23T23:23:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:57 smithi138 ceph-mon[193606]: 1.0 scrub ok 2023-09-23T23:23:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:57 smithi138 ceph-mon[193606]: pgmap v1037: 119 pgs: 119 active+clean; 309 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 38 MiB/s wr, 612 op/s 2023-09-23T23:23:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:57 smithi138 ceph-mon[193606]: 1.14 scrub starts 2023-09-23T23:23:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:57 smithi138 ceph-mon[193606]: 1.14 scrub ok 2023-09-23T23:23:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:57 smithi138 ceph-mon[193606]: 1.1d scrub starts 2023-09-23T23:23:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:57 smithi138 ceph-mon[193606]: 1.1d scrub ok 2023-09-23T23:23:57.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:57 smithi110 ceph-mon[212939]: 1.0 scrub starts 2023-09-23T23:23:57.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:57 smithi110 ceph-mon[212939]: 1.0 scrub ok 2023-09-23T23:23:57.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:57 smithi110 ceph-mon[212939]: pgmap v1037: 119 pgs: 119 active+clean; 309 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 38 MiB/s wr, 612 op/s 2023-09-23T23:23:57.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:57 smithi110 ceph-mon[212939]: 1.14 scrub starts 2023-09-23T23:23:57.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:57 smithi110 ceph-mon[212939]: 1.14 scrub ok 2023-09-23T23:23:57.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:57 smithi110 ceph-mon[212939]: 1.1d scrub starts 2023-09-23T23:23:57.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:57 smithi110 ceph-mon[212939]: 1.1d scrub ok 2023-09-23T23:23:58.041 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:23:58.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:58.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:23:58.114 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 6 16 16077 16061 167.228 169.562 0.00942917 0.0059736 2023-09-23T23:23:58.149 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:23:58.176 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:58.433 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:58 smithi110 ceph-mon[212939]: 1.f scrub starts 2023-09-23T23:23:58.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:58 smithi110 ceph-mon[212939]: 1.f scrub ok 2023-09-23T23:23:58.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:58 smithi110 ceph-mon[212939]: 1.d scrub starts 2023-09-23T23:23:58.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:58 smithi110 ceph-mon[212939]: 1.d scrub ok 2023-09-23T23:23:58.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:58.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:23:58.434 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:58 smithi110 ceph-mon[221102]: 1.f scrub starts 2023-09-23T23:23:58.435 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:58 smithi110 ceph-mon[221102]: 1.f scrub ok 2023-09-23T23:23:58.435 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:58 smithi110 ceph-mon[221102]: 1.d scrub starts 2023-09-23T23:23:58.435 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:58 smithi110 ceph-mon[221102]: 1.d scrub ok 2023-09-23T23:23:58.435 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:58.435 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:23:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:58 smithi138 ceph-mon[193606]: 1.f scrub starts 2023-09-23T23:23:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:58 smithi138 ceph-mon[193606]: 1.f scrub ok 2023-09-23T23:23:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:58 smithi138 ceph-mon[193606]: 1.d scrub starts 2023-09-23T23:23:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:58 smithi138 ceph-mon[193606]: 1.d scrub ok 2023-09-23T23:23:58.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:58.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:23:59.114 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 7 16 17601 17585 156.947 95.25 0.00640833 0.00636297 2023-09-23T23:23:59.193 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:23:59.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:59.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:23:59.300 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:23:59.313 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:23:59.397 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:59 smithi110 ceph-mon[212939]: pgmap v1038: 119 pgs: 119 active+clean; 309 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 39 MiB/s wr, 612 op/s 2023-09-23T23:23:59.397 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:59 smithi110 ceph-mon[212939]: 1.18 scrub starts 2023-09-23T23:23:59.397 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:59 smithi110 ceph-mon[212939]: 1.18 scrub ok 2023-09-23T23:23:59.397 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:23:59 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:59.397 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:59 smithi110 ceph-mon[221102]: pgmap v1038: 119 pgs: 119 active+clean; 309 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 39 MiB/s wr, 612 op/s 2023-09-23T23:23:59.397 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:59 smithi110 ceph-mon[221102]: 1.18 scrub starts 2023-09-23T23:23:59.397 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:59 smithi110 ceph-mon[221102]: 1.18 scrub ok 2023-09-23T23:23:59.398 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:23:59 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:59.402 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:59 smithi138 ceph-mon[193606]: pgmap v1038: 119 pgs: 119 active+clean; 309 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 39 MiB/s wr, 612 op/s 2023-09-23T23:23:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:59 smithi138 ceph-mon[193606]: 1.18 scrub starts 2023-09-23T23:23:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:59 smithi138 ceph-mon[193606]: 1.18 scrub ok 2023-09-23T23:23:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:23:59 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:23:59.592 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:23:59.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:59.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:23:59.698 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:23:59.746 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:23:59.816 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:23:59.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:23:59.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:23:59.922 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:23:59.959 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:00.029 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:00.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:00.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:00.118 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:00 smithi110 ceph-mon[212939]: 1.1e scrub starts 2023-09-23T23:24:00.118 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:00 smithi110 ceph-mon[212939]: 1.1e scrub ok 2023-09-23T23:24:00.121 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 8 16 19703 19687 153.675 131.375 0.00661594 0.00649926 2023-09-23T23:24:00.134 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:00.171 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:00.237 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:00.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:00.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:00.347 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:00.390 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:00 smithi110 ceph-mon[221102]: 1.1e scrub starts 2023-09-23T23:24:00.391 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:00 smithi110 ceph-mon[221102]: 1.1e scrub ok 2023-09-23T23:24:00.437 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:00 smithi138 ceph-mon[193606]: 1.1e scrub starts 2023-09-23T23:24:00.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:00 smithi138 ceph-mon[193606]: 1.1e scrub ok 2023-09-23T23:24:00.767 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:24:00.791 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:24:01.119 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 9 16 21642 21626 150.051 121.188 0.0171417 0.00665738 2023-09-23T23:24:01.132 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:01 smithi110 ceph-mon[212939]: pgmap v1039: 119 pgs: 119 active+clean; 973 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T23:24:01.132 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:01 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3041347184' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:01.140 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:24:01.140 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:24:01.179 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:24:01.221 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:24:01.257 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:24:01.292 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:24:01.323 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:24:01.329 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:24:01.349 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:24:01.360 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:24:01.360 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:24:01.360 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:24:01.360 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:24:01.360 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:24:01.360 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:24:01.360 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:24:01.360 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:24:01.394 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:01 smithi110 ceph-mon[221102]: pgmap v1039: 119 pgs: 119 active+clean; 973 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T23:24:01.394 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:01 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3041347184' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:01.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:01 smithi138 ceph-mon[193606]: pgmap v1039: 119 pgs: 119 active+clean; 973 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T23:24:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:01 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3041347184' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:01.648 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:01.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:01.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:01.754 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:01.781 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:02.119 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 10 16 23332 23316 145.611 105.625 0.00453273 0.00686218 2023-09-23T23:24:02.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:02 smithi138 ceph-mon[193606]: from='client.45905 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:24:02.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:02 smithi110 ceph-mon[212939]: from='client.45905 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:24:02.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:02 smithi110 ceph-mon[221102]: from='client.45905 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:24:03.120 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 11 16 24637 24621 139.791 81.5625 0.00331826 0.00714062 2023-09-23T23:24:03.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:03 smithi138 ceph-mon[193606]: pgmap v1040: 119 pgs: 119 active+clean; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T23:24:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:03 smithi138 ceph-mon[193606]: 1.16 scrub starts 2023-09-23T23:24:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:03 smithi138 ceph-mon[193606]: 1.16 scrub ok 2023-09-23T23:24:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:03 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:03 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:03.450 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:03 smithi110 ceph-mon[221102]: pgmap v1040: 119 pgs: 119 active+clean; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T23:24:03.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:03 smithi110 ceph-mon[221102]: 1.16 scrub starts 2023-09-23T23:24:03.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:03 smithi110 ceph-mon[221102]: 1.16 scrub ok 2023-09-23T23:24:03.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:03 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:03.451 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:03 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:03.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:03 smithi110 ceph-mon[212939]: pgmap v1040: 119 pgs: 119 active+clean; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T23:24:03.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:03 smithi110 ceph-mon[212939]: 1.16 scrub starts 2023-09-23T23:24:03.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:03 smithi110 ceph-mon[212939]: 1.16 scrub ok 2023-09-23T23:24:03.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:03 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:03.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:03 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:03.938 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:03.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:03.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:04.043 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:04.089 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:04.120 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 12 16 26281 26265 136.705 102.75 0.00771406 0.00730844 2023-09-23T23:24:04.154 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:04.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:04.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:04.261 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:04.280 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1039,"stamp":"2023-09-23T23:23:59.592174+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1019913306,"num_objects":15577,"num_object_clones":0,"num_object_copies":46731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15577,"num_whiteouts":0,"num_read":48,"num_read_kb":91,"num_write":15638,"num_write_kb":995659,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":678906,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":20599,"ondisk_log_size":20599,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":360,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6660176,"kb_used_data":3012880,"kb_used_omap":29,"kb_used_meta":3647202,"kb_avail":743301040,"statfs":{"total":767960285184,"available":761140264960,"internally_reserved":0,"allocated":3085189120,"data_stored":3074754301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":3734735571},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1019270122,"num_objects":15552,"num_object_clones":0,"num_object_copies":46656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15552,"num_whiteouts":0,"num_read":-83,"num_read_kb":-153,"num_write":15512,"num_write_kb":994988,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-4,"num_bytes_recovered":-918050,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.517997"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":608,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.448795+0000","last_change":"2023-09-23T23:23:45.457313+0000","last_active":"2023-09-23T23:23:46.448795+0000","last_peered":"2023-09-23T23:23:46.448795+0000","last_clean":"2023-09-23T23:23:46.448795+0000","last_became_active":"2023-09-23T23:23:45.456902+0000","last_became_peered":"2023-09-23T23:23:45.456902+0000","last_unstale":"2023-09-23T23:23:46.448795+0000","last_undegraded":"2023-09-23T23:23:46.448795+0000","last_fullsized":"2023-09-23T23:23:46.448795+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:42.871134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:42.871134+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023428590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":389,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.517708+0000","last_change":"2023-09-23T23:22:38.439629+0000","last_active":"2023-09-23T23:23:50.517708+0000","last_peered":"2023-09-23T23:23:50.517708+0000","last_clean":"2023-09-23T23:23:50.517708+0000","last_became_active":"2023-09-23T23:22:38.439285+0000","last_became_peered":"2023-09-23T23:22:38.439285+0000","last_unstale":"2023-09-23T23:23:50.517708+0000","last_undegraded":"2023-09-23T23:23:50.517708+0000","last_fullsized":"2023-09-23T23:23:50.517708+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:34.431221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":323,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.517007+0000","last_change":"2023-09-23T23:23:43.511912+0000","last_active":"2023-09-23T23:23:50.517007+0000","last_peered":"2023-09-23T23:23:50.517007+0000","last_clean":"2023-09-23T23:23:50.517007+0000","last_became_active":"2023-09-23T23:23:43.511574+0000","last_became_peered":"2023-09-23T23:23:43.511574+0000","last_unstale":"2023-09-23T23:23:50.517007+0000","last_undegraded":"2023-09-23T23:23:50.517007+0000","last_fullsized":"2023-09-23T23:23:50.517007+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:16.286926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":444,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.461525+0000","last_change":"2023-09-23T23:23:50.459634+0000","last_active":"2023-09-23T23:23:50.461525+0000","last_peered":"2023-09-23T23:23:50.461525+0000","last_clean":"2023-09-23T23:23:50.461525+0000","last_became_active":"2023-09-23T23:23:45.453958+0000","last_became_peered":"2023-09-23T23:23:45.453958+0000","last_unstale":"2023-09-23T23:23:50.461525+0000","last_undegraded":"2023-09-23T23:23:50.461525+0000","last_fullsized":"2023-09-23T23:23:50.461525+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:56.563201+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":45,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.177572+0000","last_change":"2023-09-23T23:22:38.436383+0000","last_active":"2023-09-23T23:23:50.177572+0000","last_peered":"2023-09-23T23:23:50.177572+0000","last_clean":"2023-09-23T23:23:50.177572+0000","last_became_active":"2023-09-23T23:22:38.436062+0000","last_became_peered":"2023-09-23T23:22:38.436062+0000","last_unstale":"2023-09-23T23:23:50.177572+0000","last_undegraded":"2023-09-23T23:23:50.177572+0000","last_fullsized":"2023-09-23T23:23:50.177572+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:07.142420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":352,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.450213+0000","last_change":"2023-09-23T23:22:38.438239+0000","last_active":"2023-09-23T23:23:46.450213+0000","last_peered":"2023-09-23T23:23:46.450213+0000","last_clean":"2023-09-23T23:23:46.450213+0000","last_became_active":"2023-09-23T23:22:38.437359+0000","last_became_peered":"2023-09-23T23:22:38.437359+0000","last_unstale":"2023-09-23T23:23:46.450213+0000","last_undegraded":"2023-09-23T23:23:46.450213+0000","last_fullsized":"2023-09-23T23:23:46.450213+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:15:40.886267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":664,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.079051+0000","last_change":"2023-09-23T23:23:49.842338+0000","last_active":"2023-09-23T23:23:50.079051+0000","last_peered":"2023-09-23T23:23:50.079051+0000","last_clean":"2023-09-23T23:23:50.079051+0000","last_became_active":"2023-09-23T23:23:45.459515+0000","last_became_peered":"2023-09-23T23:23:45.459515+0000","last_unstale":"2023-09-23T23:23:50.079051+0000","last_undegraded":"2023-09-23T23:23:50.079051+0000","last_fullsized":"2023-09-23T23:23:50.079051+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.842272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.842272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:13:00.500804+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":341,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.516477+0000","last_change":"2023-09-23T23:20:15.738434+0000","last_active":"2023-09-23T23:23:50.516477+0000","last_peered":"2023-09-23T23:23:50.516477+0000","last_clean":"2023-09-23T23:23:50.516477+0000","last_became_active":"2023-09-23T23:20:15.738195+0000","last_became_peered":"2023-09-23T23:20:15.738195+0000","last_unstale":"2023-09-23T23:23:50.516477+0000","last_undegraded":"2023-09-23T23:23:50.516477+0000","last_fullsized":"2023-09-23T23:23:50.516477+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:15.224045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":334,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.517788+0000","last_change":"2023-09-23T23:22:38.436482+0000","last_active":"2023-09-23T23:23:50.517788+0000","last_peered":"2023-09-23T23:23:50.517788+0000","last_clean":"2023-09-23T23:23:50.517788+0000","last_became_active":"2023-09-23T23:22:38.436147+0000","last_became_peered":"2023-09-23T23:22:38.436147+0000","last_unstale":"2023-09-23T23:23:50.517788+0000","last_undegraded":"2023-09-23T23:23:50.517788+0000","last_fullsized":"2023-09-23T23:23:50.517788+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:30.542674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":675,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.079012+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:23:50.079012+0000","last_peered":"2023-09-23T23:23:50.079012+0000","last_clean":"2023-09-23T23:23:50.079012+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:23:50.079012+0000","last_undegraded":"2023-09-23T23:23:50.079012+0000","last_fullsized":"2023-09-23T23:23:50.079012+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":306,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.451321+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:23:46.451321+0000","last_peered":"2023-09-23T23:23:46.451321+0000","last_clean":"2023-09-23T23:23:46.451321+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:23:46.451321+0000","last_undegraded":"2023-09-23T23:23:46.451321+0000","last_fullsized":"2023-09-23T23:23:46.451321+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":323,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.516257+0000","last_change":"2023-09-23T23:22:38.450254+0000","last_active":"2023-09-23T23:23:50.516257+0000","last_peered":"2023-09-23T23:23:50.516257+0000","last_clean":"2023-09-23T23:23:50.516257+0000","last_became_active":"2023-09-23T23:22:38.449879+0000","last_became_peered":"2023-09-23T23:22:38.449879+0000","last_unstale":"2023-09-23T23:23:50.516257+0000","last_undegraded":"2023-09-23T23:23:50.516257+0000","last_fullsized":"2023-09-23T23:23:50.516257+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:02.777916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":596,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.450165+0000","last_change":"2023-09-23T23:23:45.459697+0000","last_active":"2023-09-23T23:23:46.450165+0000","last_peered":"2023-09-23T23:23:46.450165+0000","last_clean":"2023-09-23T23:23:46.450165+0000","last_became_active":"2023-09-23T23:23:45.459314+0000","last_became_peered":"2023-09-23T23:23:45.459314+0000","last_unstale":"2023-09-23T23:23:46.450165+0000","last_undegraded":"2023-09-23T23:23:46.450165+0000","last_fullsized":"2023-09-23T23:23:46.450165+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.522632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.522632+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.018152603999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":109227,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":95,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.451180+0000","last_change":"2023-09-23T23:20:26.841909+0000","last_active":"2023-09-23T23:23:46.451180+0000","last_peered":"2023-09-23T23:23:46.451180+0000","last_clean":"2023-09-23T23:23:46.451180+0000","last_became_active":"2023-09-23T23:20:26.841631+0000","last_became_peered":"2023-09-23T23:20:26.841631+0000","last_unstale":"2023-09-23T23:23:46.451180+0000","last_undegraded":"2023-09-23T23:23:46.451180+0000","last_fullsized":"2023-09-23T23:23:46.451180+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:07.380539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":284,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.516715+0000","last_change":"2023-09-23T23:16:56.528057+0000","last_active":"2023-09-23T23:23:50.516715+0000","last_peered":"2023-09-23T23:23:50.516715+0000","last_clean":"2023-09-23T23:23:50.516715+0000","last_became_active":"2023-09-23T23:16:56.527755+0000","last_became_peered":"2023-09-23T23:16:56.527755+0000","last_unstale":"2023-09-23T23:23:50.516715+0000","last_undegraded":"2023-09-23T23:23:50.516715+0000","last_fullsized":"2023-09-23T23:23:50.516715+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:47.396406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":782,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:54.095179+0000","last_change":"2023-09-23T23:23:54.095122+0000","last_active":"2023-09-23T23:23:54.095179+0000","last_peered":"2023-09-23T23:23:54.095179+0000","last_clean":"2023-09-23T23:23:54.095179+0000","last_became_active":"2023-09-23T23:23:45.458110+0000","last_became_peered":"2023-09-23T23:23:45.458110+0000","last_unstale":"2023-09-23T23:23:54.095179+0000","last_undegraded":"2023-09-23T23:23:54.095179+0000","last_fullsized":"2023-09-23T23:23:54.095179+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:54.095054+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:54.095054+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:50.195093+0000","scrub_duration":0.0020267950000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":172,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.517124+0000","last_change":"2023-09-23T23:16:56.567128+0000","last_active":"2023-09-23T23:23:50.517124+0000","last_peered":"2023-09-23T23:23:50.517124+0000","last_clean":"2023-09-23T23:23:50.517124+0000","last_became_active":"2023-09-23T23:16:56.551484+0000","last_became_peered":"2023-09-23T23:16:56.551484+0000","last_unstale":"2023-09-23T23:23:50.517124+0000","last_undegraded":"2023-09-23T23:23:50.517124+0000","last_fullsized":"2023-09-23T23:23:50.517124+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:14.981307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":304,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.518764+0000","last_change":"2023-09-23T23:20:15.781637+0000","last_active":"2023-09-23T23:23:50.518764+0000","last_peered":"2023-09-23T23:23:50.518764+0000","last_clean":"2023-09-23T23:23:50.518764+0000","last_became_active":"2023-09-23T23:20:15.781308+0000","last_became_peered":"2023-09-23T23:20:15.781308+0000","last_unstale":"2023-09-23T23:23:50.518764+0000","last_undegraded":"2023-09-23T23:23:50.518764+0000","last_fullsized":"2023-09-23T23:23:50.518764+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:47.963872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":416,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.177779+0000","last_change":"2023-09-23T23:23:45.463780+0000","last_active":"2023-09-23T23:23:50.177779+0000","last_peered":"2023-09-23T23:23:50.177779+0000","last_clean":"2023-09-23T23:23:50.177779+0000","last_became_active":"2023-09-23T23:23:45.463106+0000","last_became_peered":"2023-09-23T23:23:45.463106+0000","last_unstale":"2023-09-23T23:23:50.177779+0000","last_undegraded":"2023-09-23T23:23:50.177779+0000","last_fullsized":"2023-09-23T23:23:50.177779+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.802642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.802642+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":70,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.079268+0000","last_change":"2023-09-23T23:23:36.017911+0000","last_active":"2023-09-23T23:23:50.079268+0000","last_peered":"2023-09-23T23:23:50.079268+0000","last_clean":"2023-09-23T23:23:50.079268+0000","last_became_active":"2023-09-23T23:23:36.017507+0000","last_became_peered":"2023-09-23T23:23:36.017507+0000","last_unstale":"2023-09-23T23:23:50.079268+0000","last_undegraded":"2023-09-23T23:23:50.079268+0000","last_fullsized":"2023-09-23T23:23:50.079268+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:49.095779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":333,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.518333+0000","last_change":"2023-09-23T23:22:38.443826+0000","last_active":"2023-09-23T23:23:50.518333+0000","last_peered":"2023-09-23T23:23:50.518333+0000","last_clean":"2023-09-23T23:23:50.518333+0000","last_became_active":"2023-09-23T23:22:38.443494+0000","last_became_peered":"2023-09-23T23:22:38.443494+0000","last_unstale":"2023-09-23T23:23:50.518333+0000","last_undegraded":"2023-09-23T23:23:50.518333+0000","last_fullsized":"2023-09-23T23:23:50.518333+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:55.630019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":507,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:53.828451+0000","last_change":"2023-09-23T23:23:53.828451+0000","last_active":"2023-09-23T23:23:53.828451+0000","last_peered":"2023-09-23T23:23:53.828451+0000","last_clean":"2023-09-23T23:23:53.828451+0000","last_became_active":"2023-09-23T23:23:45.459682+0000","last_became_peered":"2023-09-23T23:23:45.459682+0000","last_unstale":"2023-09-23T23:23:53.828451+0000","last_undegraded":"2023-09-23T23:23:53.828451+0000","last_fullsized":"2023-09-23T23:23:53.828451+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:53.828311+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:53.828311+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:19.729551+0000","scrub_duration":0.0047553140000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":255,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.078960+0000","last_change":"2023-09-23T23:23:36.026037+0000","last_active":"2023-09-23T23:23:50.078960+0000","last_peered":"2023-09-23T23:23:50.078960+0000","last_clean":"2023-09-23T23:23:50.078960+0000","last_became_active":"2023-09-23T23:23:36.025726+0000","last_became_peered":"2023-09-23T23:23:36.025726+0000","last_unstale":"2023-09-23T23:23:50.078960+0000","last_undegraded":"2023-09-23T23:23:50.078960+0000","last_fullsized":"2023-09-23T23:23:50.078960+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:28:48.287606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":340,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.517111+0000","last_change":"2023-09-23T23:23:36.032487+0000","last_active":"2023-09-23T23:23:50.517111+0000","last_peered":"2023-09-23T23:23:50.517111+0000","last_clean":"2023-09-23T23:23:50.517111+0000","last_became_active":"2023-09-23T23:23:36.032200+0000","last_became_peered":"2023-09-23T23:23:36.032200+0000","last_unstale":"2023-09-23T23:23:50.517111+0000","last_undegraded":"2023-09-23T23:23:50.517111+0000","last_fullsized":"2023-09-23T23:23:50.517111+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:29.843272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":654,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.876931+0000","last_change":"2023-09-23T23:23:50.876882+0000","last_active":"2023-09-23T23:23:50.876931+0000","last_peered":"2023-09-23T23:23:50.876931+0000","last_clean":"2023-09-23T23:23:50.876931+0000","last_became_active":"2023-09-23T23:23:45.457157+0000","last_became_peered":"2023-09-23T23:23:45.457157+0000","last_unstale":"2023-09-23T23:23:50.876931+0000","last_undegraded":"2023-09-23T23:23:50.876931+0000","last_fullsized":"2023-09-23T23:23:50.876931+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:30.732737+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":86,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.449542+0000","last_change":"2023-09-23T23:20:15.781738+0000","last_active":"2023-09-23T23:23:46.449542+0000","last_peered":"2023-09-23T23:23:46.449542+0000","last_clean":"2023-09-23T23:23:46.449542+0000","last_became_active":"2023-09-23T23:20:15.781394+0000","last_became_peered":"2023-09-23T23:20:15.781394+0000","last_unstale":"2023-09-23T23:23:46.449542+0000","last_undegraded":"2023-09-23T23:23:46.449542+0000","last_fullsized":"2023-09-23T23:23:46.449542+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:04.696118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":221,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.079072+0000","last_change":"2023-09-23T23:23:35.033379+0000","last_active":"2023-09-23T23:23:50.079072+0000","last_peered":"2023-09-23T23:23:50.079072+0000","last_clean":"2023-09-23T23:23:50.079072+0000","last_became_active":"2023-09-23T23:23:35.033069+0000","last_became_peered":"2023-09-23T23:23:35.033069+0000","last_unstale":"2023-09-23T23:23:50.079072+0000","last_undegraded":"2023-09-23T23:23:50.079072+0000","last_fullsized":"2023-09-23T23:23:50.079072+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:47.957081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":470,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.178223+0000","last_change":"2023-09-23T23:23:45.457500+0000","last_active":"2023-09-23T23:23:50.178223+0000","last_peered":"2023-09-23T23:23:50.178223+0000","last_clean":"2023-09-23T23:23:50.178223+0000","last_became_active":"2023-09-23T23:23:45.456982+0000","last_became_peered":"2023-09-23T23:23:45.456982+0000","last_unstale":"2023-09-23T23:23:50.178223+0000","last_undegraded":"2023-09-23T23:23:50.178223+0000","last_fullsized":"2023-09-23T23:23:50.178223+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:19:31.012484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:19:31.012484+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":199,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.078887+0000","last_change":"2023-09-23T23:23:35.028958+0000","last_active":"2023-09-23T23:23:50.078887+0000","last_peered":"2023-09-23T23:23:50.078887+0000","last_clean":"2023-09-23T23:23:50.078887+0000","last_became_active":"2023-09-23T23:23:35.028478+0000","last_became_peered":"2023-09-23T23:23:35.028478+0000","last_unstale":"2023-09-23T23:23:50.078887+0000","last_undegraded":"2023-09-23T23:23:50.078887+0000","last_fullsized":"2023-09-23T23:23:50.078887+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:44.960119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":333,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.450365+0000","last_change":"2023-09-23T23:20:15.737685+0000","last_active":"2023-09-23T23:23:46.450365+0000","last_peered":"2023-09-23T23:23:46.450365+0000","last_clean":"2023-09-23T23:23:46.450365+0000","last_became_active":"2023-09-23T23:20:15.737311+0000","last_became_peered":"2023-09-23T23:20:15.737311+0000","last_unstale":"2023-09-23T23:23:46.450365+0000","last_undegraded":"2023-09-23T23:23:46.450365+0000","last_fullsized":"2023-09-23T23:23:46.450365+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:15:21.213270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":638,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.461730+0000","last_change":"2023-09-23T23:23:48.533516+0000","last_active":"2023-09-23T23:23:50.461730+0000","last_peered":"2023-09-23T23:23:50.461730+0000","last_clean":"2023-09-23T23:23:50.461730+0000","last_became_active":"2023-09-23T23:23:45.458037+0000","last_became_peered":"2023-09-23T23:23:45.458037+0000","last_unstale":"2023-09-23T23:23:50.461730+0000","last_undegraded":"2023-09-23T23:23:50.461730+0000","last_fullsized":"2023-09-23T23:23:50.461730+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:47.822898+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":343,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.517794+0000","last_change":"2023-09-23T23:13:29.368175+0000","last_active":"2023-09-23T23:23:50.517794+0000","last_peered":"2023-09-23T23:23:50.517794+0000","last_clean":"2023-09-23T23:23:50.517794+0000","last_became_active":"2023-09-23T23:13:29.363603+0000","last_became_peered":"2023-09-23T23:13:29.363603+0000","last_unstale":"2023-09-23T23:23:50.517794+0000","last_undegraded":"2023-09-23T23:23:50.517794+0000","last_fullsized":"2023-09-23T23:23:50.517794+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:25.277871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":45,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.177648+0000","last_change":"2023-09-23T23:22:38.452672+0000","last_active":"2023-09-23T23:23:50.177648+0000","last_peered":"2023-09-23T23:23:50.177648+0000","last_clean":"2023-09-23T23:23:50.177648+0000","last_became_active":"2023-09-23T23:22:38.452374+0000","last_became_peered":"2023-09-23T23:22:38.452374+0000","last_unstale":"2023-09-23T23:23:50.177648+0000","last_undegraded":"2023-09-23T23:23:50.177648+0000","last_fullsized":"2023-09-23T23:23:50.177648+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:29.874430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.7","version":"355'1163","reported_seq":1177,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-23T23:23:59.558014+0000","last_change":"2023-09-23T23:23:43.505723+0000","last_active":"2023-09-23T23:23:59.558014+0000","last_peered":"2023-09-23T23:23:59.558014+0000","last_clean":"2023-09-23T23:23:59.558014+0000","last_became_active":"2023-09-23T23:23:43.505340+0000","last_became_peered":"2023-09-23T23:23:43.505340+0000","last_unstale":"2023-09-23T23:23:59.558014+0000","last_undegraded":"2023-09-23T23:23:59.558014+0000","last_fullsized":"2023-09-23T23:23:59.558014+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":1163,"log_dups_size":0,"ondisk_log_size":1163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:33.987175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76218368,"num_objects":1163,"num_object_clones":0,"num_object_copies":3489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1163,"num_write_kb":74432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":676,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-23T23:23:57.267004+0000","last_change":"2023-09-23T23:23:57.267117+0000","last_active":"2023-09-23T23:23:57.267004+0000","last_peered":"2023-09-23T23:23:57.267004+0000","last_clean":"2023-09-23T23:23:57.267004+0000","last_became_active":"2023-09-23T23:23:45.458275+0000","last_became_peered":"2023-09-23T23:23:45.458275+0000","last_unstale":"2023-09-23T23:23:57.267004+0000","last_undegraded":"2023-09-23T23:23:57.267004+0000","last_fullsized":"2023-09-23T23:23:57.267004+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:47:09.088899+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":168,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.518231+0000","last_change":"2023-09-23T23:16:56.548535+0000","last_active":"2023-09-23T23:23:50.518231+0000","last_peered":"2023-09-23T23:23:50.518231+0000","last_clean":"2023-09-23T23:23:50.518231+0000","last_became_active":"2023-09-23T23:16:56.546115+0000","last_became_peered":"2023-09-23T23:16:56.546115+0000","last_unstale":"2023-09-23T23:23:50.518231+0000","last_undegraded":"2023-09-23T23:23:50.518231+0000","last_fullsized":"2023-09-23T23:23:50.518231+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:02.279715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":159,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.517793+0000","last_change":"2023-09-23T23:16:56.525479+0000","last_active":"2023-09-23T23:23:50.517793+0000","last_peered":"2023-09-23T23:23:50.517793+0000","last_clean":"2023-09-23T23:23:50.517793+0000","last_became_active":"2023-09-23T23:16:56.525147+0000","last_became_peered":"2023-09-23T23:16:56.525147+0000","last_unstale":"2023-09-23T23:23:50.517793+0000","last_undegraded":"2023-09-23T23:23:50.517793+0000","last_fullsized":"2023-09-23T23:23:50.517793+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:06.165300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.6","version":"354'444","reported_seq":458,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:55.159204+0000","last_change":"2023-09-23T23:23:43.514932+0000","last_active":"2023-09-23T23:23:55.159204+0000","last_peered":"2023-09-23T23:23:55.159204+0000","last_clean":"2023-09-23T23:23:55.159204+0000","last_became_active":"2023-09-23T23:23:43.513416+0000","last_became_peered":"2023-09-23T23:23:43.513416+0000","last_unstale":"2023-09-23T23:23:55.159204+0000","last_undegraded":"2023-09-23T23:23:55.159204+0000","last_fullsized":"2023-09-23T23:23:55.159204+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":444,"log_dups_size":0,"ondisk_log_size":444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:41.201456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29097984,"num_objects":444,"num_object_clones":0,"num_object_copies":1332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":444,"num_write_kb":28416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":698,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.516938+0000","last_change":"2023-09-23T23:23:48.906779+0000","last_active":"2023-09-23T23:23:50.516938+0000","last_peered":"2023-09-23T23:23:50.516938+0000","last_clean":"2023-09-23T23:23:50.516938+0000","last_became_active":"2023-09-23T23:23:45.454967+0000","last_became_peered":"2023-09-23T23:23:45.454967+0000","last_unstale":"2023-09-23T23:23:50.516938+0000","last_undegraded":"2023-09-23T23:23:50.516938+0000","last_fullsized":"2023-09-23T23:23:50.516938+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:02.570612+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":49,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.177955+0000","last_change":"2023-09-23T23:22:38.474057+0000","last_active":"2023-09-23T23:23:50.177955+0000","last_peered":"2023-09-23T23:23:50.177955+0000","last_clean":"2023-09-23T23:23:50.177955+0000","last_became_active":"2023-09-23T23:22:38.444672+0000","last_became_peered":"2023-09-23T23:22:38.444672+0000","last_unstale":"2023-09-23T23:23:50.177955+0000","last_undegraded":"2023-09-23T23:23:50.177955+0000","last_fullsized":"2023-09-23T23:23:50.177955+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:00.966999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":360,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.517999+0000","last_change":"2023-09-23T23:22:38.439077+0000","last_active":"2023-09-23T23:23:50.517999+0000","last_peered":"2023-09-23T23:23:50.517999+0000","last_clean":"2023-09-23T23:23:50.517999+0000","last_became_active":"2023-09-23T23:22:38.438638+0000","last_became_peered":"2023-09-23T23:22:38.438638+0000","last_unstale":"2023-09-23T23:23:50.517999+0000","last_undegraded":"2023-09-23T23:23:50.517999+0000","last_fullsized":"2023-09-23T23:23:50.517999+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:57.067702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"355'1125","reported_seq":1139,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-23T23:23:59.470740+0000","last_change":"2023-09-23T23:23:43.514636+0000","last_active":"2023-09-23T23:23:59.470740+0000","last_peered":"2023-09-23T23:23:59.470740+0000","last_clean":"2023-09-23T23:23:59.470740+0000","last_became_active":"2023-09-23T23:23:43.514352+0000","last_became_peered":"2023-09-23T23:23:43.514352+0000","last_unstale":"2023-09-23T23:23:59.470740+0000","last_undegraded":"2023-09-23T23:23:59.470740+0000","last_fullsized":"2023-09-23T23:23:59.470740+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":1125,"log_dups_size":0,"ondisk_log_size":1125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:06.752989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73728000,"num_objects":1125,"num_object_clones":0,"num_object_copies":3375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1125,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1125,"num_write_kb":72000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":769,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-23T23:23:57.042714+0000","last_change":"2023-09-23T23:23:57.042667+0000","last_active":"2023-09-23T23:23:57.042714+0000","last_peered":"2023-09-23T23:23:57.042714+0000","last_clean":"2023-09-23T23:23:57.042714+0000","last_became_active":"2023-09-23T23:23:45.454067+0000","last_became_peered":"2023-09-23T23:23:45.454067+0000","last_unstale":"2023-09-23T23:23:57.042714+0000","last_undegraded":"2023-09-23T23:23:57.042714+0000","last_fullsized":"2023-09-23T23:23:57.042714+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.042580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.042580+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:02:09.561802+0000","scrub_duration":0.0044137760000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":233,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.079188+0000","last_change":"2023-09-23T23:23:35.033371+0000","last_active":"2023-09-23T23:23:50.079188+0000","last_peered":"2023-09-23T23:23:50.079188+0000","last_clean":"2023-09-23T23:23:50.079188+0000","last_became_active":"2023-09-23T23:23:35.033032+0000","last_became_peered":"2023-09-23T23:23:35.033032+0000","last_unstale":"2023-09-23T23:23:50.079188+0000","last_undegraded":"2023-09-23T23:23:50.079188+0000","last_fullsized":"2023-09-23T23:23:50.079188+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:33:17.856914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":381,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.517585+0000","last_change":"2023-09-23T23:22:38.439331+0000","last_active":"2023-09-23T23:23:50.517585+0000","last_peered":"2023-09-23T23:23:50.517585+0000","last_clean":"2023-09-23T23:23:50.517585+0000","last_became_active":"2023-09-23T23:22:38.436293+0000","last_became_peered":"2023-09-23T23:22:38.436293+0000","last_unstale":"2023-09-23T23:23:50.517585+0000","last_undegraded":"2023-09-23T23:23:50.517585+0000","last_fullsized":"2023-09-23T23:23:50.517585+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:53.668614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"355'999","reported_seq":1013,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-23T23:23:57.958767+0000","last_change":"2023-09-23T23:23:43.513124+0000","last_active":"2023-09-23T23:23:57.958767+0000","last_peered":"2023-09-23T23:23:57.958767+0000","last_clean":"2023-09-23T23:23:57.958767+0000","last_became_active":"2023-09-23T23:23:43.512782+0000","last_became_peered":"2023-09-23T23:23:43.512782+0000","last_unstale":"2023-09-23T23:23:57.958767+0000","last_undegraded":"2023-09-23T23:23:57.958767+0000","last_fullsized":"2023-09-23T23:23:57.958767+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":999,"log_dups_size":0,"ondisk_log_size":999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:16:06.664727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65470464,"num_objects":999,"num_object_clones":0,"num_object_copies":2997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":999,"num_write_kb":63936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":417,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.517133+0000","last_change":"2023-09-23T23:23:45.460816+0000","last_active":"2023-09-23T23:23:50.517133+0000","last_peered":"2023-09-23T23:23:50.517133+0000","last_clean":"2023-09-23T23:23:50.517133+0000","last_became_active":"2023-09-23T23:23:45.460444+0000","last_became_peered":"2023-09-23T23:23:45.460444+0000","last_unstale":"2023-09-23T23:23:50.517133+0000","last_undegraded":"2023-09-23T23:23:50.517133+0000","last_fullsized":"2023-09-23T23:23:50.517133+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.802642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.802642+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":172,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.517049+0000","last_change":"2023-09-23T23:21:22.192374+0000","last_active":"2023-09-23T23:23:50.517049+0000","last_peered":"2023-09-23T23:23:50.517049+0000","last_clean":"2023-09-23T23:23:50.517049+0000","last_became_active":"2023-09-23T23:21:22.192086+0000","last_became_peered":"2023-09-23T23:21:22.192086+0000","last_unstale":"2023-09-23T23:23:50.517049+0000","last_undegraded":"2023-09-23T23:23:50.517049+0000","last_fullsized":"2023-09-23T23:23:50.517049+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:20.605133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":327,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.450619+0000","last_change":"2023-09-23T23:22:38.438390+0000","last_active":"2023-09-23T23:23:46.450619+0000","last_peered":"2023-09-23T23:23:46.450619+0000","last_clean":"2023-09-23T23:23:46.450619+0000","last_became_active":"2023-09-23T23:22:38.438065+0000","last_became_peered":"2023-09-23T23:22:38.438065+0000","last_unstale":"2023-09-23T23:23:46.450619+0000","last_undegraded":"2023-09-23T23:23:46.450619+0000","last_fullsized":"2023-09-23T23:23:46.450619+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:51.254227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"355'1170","reported_seq":1184,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-23T23:23:59.480517+0000","last_change":"2023-09-23T23:23:43.512187+0000","last_active":"2023-09-23T23:23:59.480517+0000","last_peered":"2023-09-23T23:23:59.480517+0000","last_clean":"2023-09-23T23:23:59.480517+0000","last_became_active":"2023-09-23T23:23:43.511874+0000","last_became_peered":"2023-09-23T23:23:43.511874+0000","last_unstale":"2023-09-23T23:23:59.480517+0000","last_undegraded":"2023-09-23T23:23:59.480517+0000","last_fullsized":"2023-09-23T23:23:59.480517+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":1170,"log_dups_size":0,"ondisk_log_size":1170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:38.052106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76677120,"num_objects":1170,"num_object_clones":0,"num_object_copies":3510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1170,"num_write_kb":74880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":690,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.078888+0000","last_change":"2023-09-23T23:23:45.830384+0000","last_active":"2023-09-23T23:23:50.078888+0000","last_peered":"2023-09-23T23:23:50.078888+0000","last_clean":"2023-09-23T23:23:50.078888+0000","last_became_active":"2023-09-23T23:23:36.025724+0000","last_became_peered":"2023-09-23T23:23:36.025724+0000","last_unstale":"2023-09-23T23:23:50.078888+0000","last_undegraded":"2023-09-23T23:23:50.078888+0000","last_fullsized":"2023-09-23T23:23:50.078888+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:34.345528+0000","scrub_duration":0.001659259,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":320,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.516769+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:23:50.516769+0000","last_peered":"2023-09-23T23:23:50.516769+0000","last_clean":"2023-09-23T23:23:50.516769+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:23:50.516769+0000","last_undegraded":"2023-09-23T23:23:50.516769+0000","last_fullsized":"2023-09-23T23:23:50.516769+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":313,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.518544+0000","last_change":"2023-09-23T23:10:59.512363+0000","last_active":"2023-09-23T23:23:50.518544+0000","last_peered":"2023-09-23T23:23:50.518544+0000","last_clean":"2023-09-23T23:23:50.518544+0000","last_became_active":"2023-09-23T23:10:59.512022+0000","last_became_peered":"2023-09-23T23:10:59.512022+0000","last_unstale":"2023-09-23T23:23:50.518544+0000","last_undegraded":"2023-09-23T23:23:50.518544+0000","last_fullsized":"2023-09-23T23:23:50.518544+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:45.523432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"354'509","reported_seq":523,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:55.217799+0000","last_change":"2023-09-23T23:23:43.503924+0000","last_active":"2023-09-23T23:23:55.217799+0000","last_peered":"2023-09-23T23:23:55.217799+0000","last_clean":"2023-09-23T23:23:55.217799+0000","last_became_active":"2023-09-23T23:23:43.503698+0000","last_became_peered":"2023-09-23T23:23:43.503698+0000","last_unstale":"2023-09-23T23:23:55.217799+0000","last_undegraded":"2023-09-23T23:23:55.217799+0000","last_fullsized":"2023-09-23T23:23:55.217799+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":509,"log_dups_size":0,"ondisk_log_size":509,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:14.346656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33357824,"num_objects":509,"num_object_clones":0,"num_object_copies":1527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":509,"num_write_kb":32576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":597,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.862579+0000","last_change":"2023-09-23T23:23:50.862663+0000","last_active":"2023-09-23T23:23:50.862579+0000","last_peered":"2023-09-23T23:23:50.862579+0000","last_clean":"2023-09-23T23:23:50.862579+0000","last_became_active":"2023-09-23T23:22:57.463887+0000","last_became_peered":"2023-09-23T23:22:57.463887+0000","last_unstale":"2023-09-23T23:23:50.862579+0000","last_undegraded":"2023-09-23T23:23:50.862579+0000","last_fullsized":"2023-09-23T23:23:50.862579+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:46:51.271983+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":325,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.519082+0000","last_change":"2023-09-23T23:16:56.525904+0000","last_active":"2023-09-23T23:23:50.519082+0000","last_peered":"2023-09-23T23:23:50.519082+0000","last_clean":"2023-09-23T23:23:50.519082+0000","last_became_active":"2023-09-23T23:16:56.525372+0000","last_became_peered":"2023-09-23T23:16:56.525372+0000","last_unstale":"2023-09-23T23:23:50.519082+0000","last_undegraded":"2023-09-23T23:23:50.519082+0000","last_fullsized":"2023-09-23T23:23:50.519082+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:17.070117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":368,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.451002+0000","last_change":"2023-09-23T23:22:38.439737+0000","last_active":"2023-09-23T23:23:46.451002+0000","last_peered":"2023-09-23T23:23:46.451002+0000","last_clean":"2023-09-23T23:23:46.451002+0000","last_became_active":"2023-09-23T23:22:38.439246+0000","last_became_peered":"2023-09-23T23:22:38.439246+0000","last_unstale":"2023-09-23T23:23:46.451002+0000","last_undegraded":"2023-09-23T23:23:46.451002+0000","last_fullsized":"2023-09-23T23:23:46.451002+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:40.042010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"355'1137","reported_seq":1151,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-23T23:23:59.479647+0000","last_change":"2023-09-23T23:23:43.513972+0000","last_active":"2023-09-23T23:23:59.479647+0000","last_peered":"2023-09-23T23:23:59.479647+0000","last_clean":"2023-09-23T23:23:59.479647+0000","last_became_active":"2023-09-23T23:23:43.513615+0000","last_became_peered":"2023-09-23T23:23:43.513615+0000","last_unstale":"2023-09-23T23:23:59.479647+0000","last_undegraded":"2023-09-23T23:23:59.479647+0000","last_fullsized":"2023-09-23T23:23:59.479647+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":1137,"log_dups_size":0,"ondisk_log_size":1137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:27.822029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74514432,"num_objects":1137,"num_object_clones":0,"num_object_copies":3411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1137,"num_write_kb":72768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":491,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.178725+0000","last_change":"2023-09-23T23:23:50.178725+0000","last_active":"2023-09-23T23:23:50.178725+0000","last_peered":"2023-09-23T23:23:50.178725+0000","last_clean":"2023-09-23T23:23:50.178725+0000","last_became_active":"2023-09-23T23:23:45.458319+0000","last_became_peered":"2023-09-23T23:23:45.458319+0000","last_unstale":"2023-09-23T23:23:50.178725+0000","last_undegraded":"2023-09-23T23:23:50.178725+0000","last_fullsized":"2023-09-23T23:23:50.178725+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:46.518884+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":374,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.450640+0000","last_change":"2023-09-23T23:23:36.030470+0000","last_active":"2023-09-23T23:23:46.450640+0000","last_peered":"2023-09-23T23:23:46.450640+0000","last_clean":"2023-09-23T23:23:46.450640+0000","last_became_active":"2023-09-23T23:23:36.030157+0000","last_became_peered":"2023-09-23T23:23:36.030157+0000","last_unstale":"2023-09-23T23:23:46.450640+0000","last_undegraded":"2023-09-23T23:23:46.450640+0000","last_fullsized":"2023-09-23T23:23:46.450640+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:08.848362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":323,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.518113+0000","last_change":"2023-09-23T23:15:28.228439+0000","last_active":"2023-09-23T23:23:50.518113+0000","last_peered":"2023-09-23T23:23:50.518113+0000","last_clean":"2023-09-23T23:23:50.518113+0000","last_became_active":"2023-09-23T23:15:28.228095+0000","last_became_peered":"2023-09-23T23:15:28.228095+0000","last_unstale":"2023-09-23T23:23:50.518113+0000","last_undegraded":"2023-09-23T23:23:50.518113+0000","last_fullsized":"2023-09-23T23:23:50.518113+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:37.667201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"355'1097","reported_seq":1112,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-23T23:23:59.165224+0000","last_change":"2023-09-23T23:23:43.513796+0000","last_active":"2023-09-23T23:23:59.165224+0000","last_peered":"2023-09-23T23:23:59.165224+0000","last_clean":"2023-09-23T23:23:59.165224+0000","last_became_active":"2023-09-23T23:23:43.513512+0000","last_became_peered":"2023-09-23T23:23:43.513512+0000","last_unstale":"2023-09-23T23:23:59.165224+0000","last_undegraded":"2023-09-23T23:23:59.165224+0000","last_fullsized":"2023-09-23T23:23:59.165224+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":1097,"log_dups_size":0,"ondisk_log_size":1097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:10.363938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71892992,"num_objects":1097,"num_object_clones":0,"num_object_copies":3291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1097,"num_write_kb":70208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":656,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:51.876906+0000","last_change":"2023-09-23T23:23:51.877015+0000","last_active":"2023-09-23T23:23:51.876906+0000","last_peered":"2023-09-23T23:23:51.876906+0000","last_clean":"2023-09-23T23:23:51.876906+0000","last_became_active":"2023-09-23T23:22:57.468211+0000","last_became_peered":"2023-09-23T23:22:57.468211+0000","last_unstale":"2023-09-23T23:23:51.876906+0000","last_undegraded":"2023-09-23T23:23:51.876906+0000","last_fullsized":"2023-09-23T23:23:51.876906+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:42.443440+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":45,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.177453+0000","last_change":"2023-09-23T23:22:38.434606+0000","last_active":"2023-09-23T23:23:50.177453+0000","last_peered":"2023-09-23T23:23:50.177453+0000","last_clean":"2023-09-23T23:23:50.177453+0000","last_became_active":"2023-09-23T23:22:38.434196+0000","last_became_peered":"2023-09-23T23:22:38.434196+0000","last_unstale":"2023-09-23T23:23:50.177453+0000","last_undegraded":"2023-09-23T23:23:50.177453+0000","last_fullsized":"2023-09-23T23:23:50.177453+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:14.702902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":274,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.177726+0000","last_change":"2023-09-23T23:22:39.489325+0000","last_active":"2023-09-23T23:23:50.177726+0000","last_peered":"2023-09-23T23:23:50.177726+0000","last_clean":"2023-09-23T23:23:50.177726+0000","last_became_active":"2023-09-23T23:22:39.489048+0000","last_became_peered":"2023-09-23T23:22:39.489048+0000","last_unstale":"2023-09-23T23:23:50.177726+0000","last_undegraded":"2023-09-23T23:23:50.177726+0000","last_fullsized":"2023-09-23T23:23:50.177726+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:29.984948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"354'525","reported_seq":539,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:55.226915+0000","last_change":"2023-09-23T23:23:43.507963+0000","last_active":"2023-09-23T23:23:55.226915+0000","last_peered":"2023-09-23T23:23:55.226915+0000","last_clean":"2023-09-23T23:23:55.226915+0000","last_became_active":"2023-09-23T23:23:43.507503+0000","last_became_peered":"2023-09-23T23:23:43.507503+0000","last_unstale":"2023-09-23T23:23:55.226915+0000","last_undegraded":"2023-09-23T23:23:55.226915+0000","last_fullsized":"2023-09-23T23:23:55.226915+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":525,"log_dups_size":0,"ondisk_log_size":525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:39.595255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34406400,"num_objects":525,"num_object_clones":0,"num_object_copies":1575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":525,"num_write_kb":33600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":763,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.518859+0000","last_change":"2023-09-23T23:23:48.063145+0000","last_active":"2023-09-23T23:23:50.518859+0000","last_peered":"2023-09-23T23:23:50.518859+0000","last_clean":"2023-09-23T23:23:50.518859+0000","last_became_active":"2023-09-23T23:22:57.470024+0000","last_became_peered":"2023-09-23T23:22:57.470024+0000","last_unstale":"2023-09-23T23:23:50.518859+0000","last_undegraded":"2023-09-23T23:23:50.518859+0000","last_fullsized":"2023-09-23T23:23:50.518859+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.063071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.063071+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:51.217545+0000","scrub_duration":0.0015554690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":54,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.177678+0000","last_change":"2023-09-23T23:23:36.048889+0000","last_active":"2023-09-23T23:23:50.177678+0000","last_peered":"2023-09-23T23:23:50.177678+0000","last_clean":"2023-09-23T23:23:50.177678+0000","last_became_active":"2023-09-23T23:23:36.048572+0000","last_became_peered":"2023-09-23T23:23:36.048572+0000","last_unstale":"2023-09-23T23:23:50.177678+0000","last_undegraded":"2023-09-23T23:23:50.177678+0000","last_fullsized":"2023-09-23T23:23:50.177678+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:55.581146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":370,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.461667+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:23:50.461667+0000","last_peered":"2023-09-23T23:23:50.461667+0000","last_clean":"2023-09-23T23:23:50.461667+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:23:50.461667+0000","last_undegraded":"2023-09-23T23:23:50.461667+0000","last_fullsized":"2023-09-23T23:23:50.461667+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":297,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.180061+0000","last_change":"2023-09-23T23:23:36.030569+0000","last_active":"2023-09-23T23:23:50.180061+0000","last_peered":"2023-09-23T23:23:50.180061+0000","last_clean":"2023-09-23T23:23:50.180061+0000","last_became_active":"2023-09-23T23:23:36.030257+0000","last_became_peered":"2023-09-23T23:23:36.030257+0000","last_unstale":"2023-09-23T23:23:50.180061+0000","last_undegraded":"2023-09-23T23:23:50.180061+0000","last_fullsized":"2023-09-23T23:23:50.180061+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:53.906813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.b","version":"355'1043","reported_seq":1058,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-23T23:23:58.121551+0000","last_change":"2023-09-23T23:23:43.509408+0000","last_active":"2023-09-23T23:23:58.121551+0000","last_peered":"2023-09-23T23:23:58.121551+0000","last_clean":"2023-09-23T23:23:58.121551+0000","last_became_active":"2023-09-23T23:23:43.509146+0000","last_became_peered":"2023-09-23T23:23:43.509146+0000","last_unstale":"2023-09-23T23:23:58.121551+0000","last_undegraded":"2023-09-23T23:23:58.121551+0000","last_fullsized":"2023-09-23T23:23:58.121551+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":1043,"log_dups_size":0,"ondisk_log_size":1043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:51.961026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68354048,"num_objects":1043,"num_object_clones":0,"num_object_copies":3129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1043,"num_write_kb":66752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":498,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:49.320969+0000","last_change":"2023-09-23T23:23:49.320969+0000","last_active":"2023-09-23T23:23:49.320969+0000","last_peered":"2023-09-23T23:23:49.320969+0000","last_clean":"2023-09-23T23:23:49.320969+0000","last_became_active":"2023-09-23T23:22:17.001911+0000","last_became_peered":"2023-09-23T23:22:17.001911+0000","last_unstale":"2023-09-23T23:23:49.320969+0000","last_undegraded":"2023-09-23T23:23:49.320969+0000","last_fullsized":"2023-09-23T23:23:49.320969+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.320823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.320823+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:07.000984+0000","scrub_duration":0.0018140630000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":45,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.177586+0000","last_change":"2023-09-23T23:22:38.433755+0000","last_active":"2023-09-23T23:23:50.177586+0000","last_peered":"2023-09-23T23:23:50.177586+0000","last_clean":"2023-09-23T23:23:50.177586+0000","last_became_active":"2023-09-23T23:22:38.433442+0000","last_became_peered":"2023-09-23T23:22:38.433442+0000","last_unstale":"2023-09-23T23:23:50.177586+0000","last_undegraded":"2023-09-23T23:23:50.177586+0000","last_fullsized":"2023-09-23T23:23:50.177586+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:25.409363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":314,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.518780+0000","last_change":"2023-09-23T23:04:11.514703+0000","last_active":"2023-09-23T23:23:50.518780+0000","last_peered":"2023-09-23T23:23:50.518780+0000","last_clean":"2023-09-23T23:23:50.518780+0000","last_became_active":"2023-09-23T23:04:11.514403+0000","last_became_peered":"2023-09-23T23:04:11.514403+0000","last_unstale":"2023-09-23T23:23:50.518780+0000","last_undegraded":"2023-09-23T23:23:50.518780+0000","last_fullsized":"2023-09-23T23:23:50.518780+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:26.374346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":86,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.450388+0000","last_change":"2023-09-23T23:20:15.781844+0000","last_active":"2023-09-23T23:23:46.450388+0000","last_peered":"2023-09-23T23:23:46.450388+0000","last_clean":"2023-09-23T23:23:46.450388+0000","last_became_active":"2023-09-23T23:20:15.781573+0000","last_became_peered":"2023-09-23T23:20:15.781573+0000","last_unstale":"2023-09-23T23:23:46.450388+0000","last_undegraded":"2023-09-23T23:23:46.450388+0000","last_fullsized":"2023-09-23T23:23:46.450388+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:25:50.816486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"355'1134","reported_seq":1148,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-23T23:23:59.477701+0000","last_change":"2023-09-23T23:23:43.508971+0000","last_active":"2023-09-23T23:23:59.477701+0000","last_peered":"2023-09-23T23:23:59.477701+0000","last_clean":"2023-09-23T23:23:59.477701+0000","last_became_active":"2023-09-23T23:23:43.508648+0000","last_became_peered":"2023-09-23T23:23:43.508648+0000","last_unstale":"2023-09-23T23:23:59.477701+0000","last_undegraded":"2023-09-23T23:23:59.477701+0000","last_fullsized":"2023-09-23T23:23:59.477701+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":1134,"log_dups_size":0,"ondisk_log_size":1134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:43.036229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74317824,"num_objects":1134,"num_object_clones":0,"num_object_copies":3402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1134,"num_write_kb":72576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":688,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:52.342139+0000","last_change":"2023-09-23T23:23:52.342245+0000","last_active":"2023-09-23T23:23:52.342139+0000","last_peered":"2023-09-23T23:23:52.342139+0000","last_clean":"2023-09-23T23:23:52.342139+0000","last_became_active":"2023-09-23T23:22:17.000797+0000","last_became_peered":"2023-09-23T23:22:17.000797+0000","last_unstale":"2023-09-23T23:23:52.342139+0000","last_undegraded":"2023-09-23T23:23:52.342139+0000","last_fullsized":"2023-09-23T23:23:52.342139+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:13.663896+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":86,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.450488+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:23:46.450488+0000","last_peered":"2023-09-23T23:23:46.450488+0000","last_clean":"2023-09-23T23:23:46.450488+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:23:46.450488+0000","last_undegraded":"2023-09-23T23:23:46.450488+0000","last_fullsized":"2023-09-23T23:23:46.450488+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":45,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.177557+0000","last_change":"2023-09-23T23:22:38.480385+0000","last_active":"2023-09-23T23:23:50.177557+0000","last_peered":"2023-09-23T23:23:50.177557+0000","last_clean":"2023-09-23T23:23:50.177557+0000","last_became_active":"2023-09-23T23:22:38.480071+0000","last_became_peered":"2023-09-23T23:22:38.480071+0000","last_unstale":"2023-09-23T23:23:50.177557+0000","last_undegraded":"2023-09-23T23:23:50.177557+0000","last_fullsized":"2023-09-23T23:23:50.177557+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:15.348789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":166,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.451241+0000","last_change":"2023-09-23T23:21:22.194721+0000","last_active":"2023-09-23T23:23:46.451241+0000","last_peered":"2023-09-23T23:23:46.451241+0000","last_clean":"2023-09-23T23:23:46.451241+0000","last_became_active":"2023-09-23T23:21:22.194333+0000","last_became_peered":"2023-09-23T23:21:22.194333+0000","last_unstale":"2023-09-23T23:23:46.451241+0000","last_undegraded":"2023-09-23T23:23:46.451241+0000","last_fullsized":"2023-09-23T23:23:46.451241+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:46.823997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"355'975","reported_seq":990,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-23T23:23:58.131789+0000","last_change":"2023-09-23T23:23:43.510145+0000","last_active":"2023-09-23T23:23:58.131789+0000","last_peered":"2023-09-23T23:23:58.131789+0000","last_clean":"2023-09-23T23:23:58.131789+0000","last_became_active":"2023-09-23T23:23:43.509875+0000","last_became_peered":"2023-09-23T23:23:43.509875+0000","last_unstale":"2023-09-23T23:23:58.131789+0000","last_undegraded":"2023-09-23T23:23:58.131789+0000","last_fullsized":"2023-09-23T23:23:58.131789+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":975,"log_dups_size":0,"ondisk_log_size":975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:37.678046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63897600,"num_objects":975,"num_object_clones":0,"num_object_copies":2925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":975,"num_write_kb":62400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":354,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.449685+0000","last_change":"2023-09-23T23:21:22.193754+0000","last_active":"2023-09-23T23:23:46.449685+0000","last_peered":"2023-09-23T23:23:46.449685+0000","last_clean":"2023-09-23T23:23:46.449685+0000","last_became_active":"2023-09-23T23:21:22.193409+0000","last_became_peered":"2023-09-23T23:21:22.193409+0000","last_unstale":"2023-09-23T23:23:46.449685+0000","last_undegraded":"2023-09-23T23:23:46.449685+0000","last_fullsized":"2023-09-23T23:23:46.449685+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:53.635813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":634,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.449721+0000","last_change":"2023-09-23T23:22:41.462946+0000","last_active":"2023-09-23T23:23:46.449721+0000","last_peered":"2023-09-23T23:23:46.449721+0000","last_clean":"2023-09-23T23:23:46.449721+0000","last_became_active":"2023-09-23T23:22:41.462616+0000","last_became_peered":"2023-09-23T23:22:41.462616+0000","last_unstale":"2023-09-23T23:23:46.449721+0000","last_undegraded":"2023-09-23T23:23:46.449721+0000","last_fullsized":"2023-09-23T23:23:46.449721+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:18:42.190868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:18:42.190868+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0049256480000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":284,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.516726+0000","last_change":"2023-09-23T23:16:56.525298+0000","last_active":"2023-09-23T23:23:50.516726+0000","last_peered":"2023-09-23T23:23:50.516726+0000","last_clean":"2023-09-23T23:23:50.516726+0000","last_became_active":"2023-09-23T23:16:56.524974+0000","last_became_peered":"2023-09-23T23:16:56.524974+0000","last_unstale":"2023-09-23T23:23:50.516726+0000","last_undegraded":"2023-09-23T23:23:50.516726+0000","last_fullsized":"2023-09-23T23:23:50.516726+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:08.860790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":328,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.518658+0000","last_change":"2023-09-23T23:23:36.022440+0000","last_active":"2023-09-23T23:23:50.518658+0000","last_peered":"2023-09-23T23:23:50.518658+0000","last_clean":"2023-09-23T23:23:50.518658+0000","last_became_active":"2023-09-23T23:23:36.022065+0000","last_became_peered":"2023-09-23T23:23:36.022065+0000","last_unstale":"2023-09-23T23:23:50.518658+0000","last_undegraded":"2023-09-23T23:23:50.518658+0000","last_fullsized":"2023-09-23T23:23:50.518658+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:43:27.098960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"355'1009","reported_seq":1024,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-23T23:23:58.132095+0000","last_change":"2023-09-23T23:23:43.510366+0000","last_active":"2023-09-23T23:23:58.132095+0000","last_peered":"2023-09-23T23:23:58.132095+0000","last_clean":"2023-09-23T23:23:58.132095+0000","last_became_active":"2023-09-23T23:23:43.510117+0000","last_became_peered":"2023-09-23T23:23:43.510117+0000","last_unstale":"2023-09-23T23:23:58.132095+0000","last_undegraded":"2023-09-23T23:23:58.132095+0000","last_fullsized":"2023-09-23T23:23:58.132095+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":1009,"log_dups_size":0,"ondisk_log_size":1009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:05.709562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66125824,"num_objects":1009,"num_object_clones":0,"num_object_copies":3027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1009,"num_write_kb":64576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":465,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.178120+0000","last_change":"2023-09-23T23:23:46.214371+0000","last_active":"2023-09-23T23:23:50.178120+0000","last_peered":"2023-09-23T23:23:50.178120+0000","last_clean":"2023-09-23T23:23:50.178120+0000","last_became_active":"2023-09-23T23:22:41.466645+0000","last_became_peered":"2023-09-23T23:22:41.466645+0000","last_unstale":"2023-09-23T23:23:50.178120+0000","last_undegraded":"2023-09-23T23:23:50.178120+0000","last_fullsized":"2023-09-23T23:23:50.178120+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:14.444467+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":272,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.078858+0000","last_change":"2023-09-23T23:23:35.021697+0000","last_active":"2023-09-23T23:23:50.078858+0000","last_peered":"2023-09-23T23:23:50.078858+0000","last_clean":"2023-09-23T23:23:50.078858+0000","last_became_active":"2023-09-23T23:23:35.021376+0000","last_became_peered":"2023-09-23T23:23:35.021376+0000","last_unstale":"2023-09-23T23:23:50.078858+0000","last_undegraded":"2023-09-23T23:23:50.078858+0000","last_fullsized":"2023-09-23T23:23:50.078858+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:42.630129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":336,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.449589+0000","last_change":"2023-09-23T23:23:36.031618+0000","last_active":"2023-09-23T23:23:46.449589+0000","last_peered":"2023-09-23T23:23:46.449589+0000","last_clean":"2023-09-23T23:23:46.449589+0000","last_became_active":"2023-09-23T23:23:36.031322+0000","last_became_peered":"2023-09-23T23:23:36.031322+0000","last_unstale":"2023-09-23T23:23:46.449589+0000","last_undegraded":"2023-09-23T23:23:46.449589+0000","last_fullsized":"2023-09-23T23:23:46.449589+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:07.871195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":382,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.450286+0000","last_change":"2023-09-23T23:22:38.438448+0000","last_active":"2023-09-23T23:23:46.450286+0000","last_peered":"2023-09-23T23:23:46.450286+0000","last_clean":"2023-09-23T23:23:46.450286+0000","last_became_active":"2023-09-23T23:22:38.437495+0000","last_became_peered":"2023-09-23T23:22:38.437495+0000","last_unstale":"2023-09-23T23:23:46.450286+0000","last_undegraded":"2023-09-23T23:23:46.450286+0000","last_fullsized":"2023-09-23T23:23:46.450286+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:49.587056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"355'1084","reported_seq":1099,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-23T23:23:59.174754+0000","last_change":"2023-09-23T23:23:43.512003+0000","last_active":"2023-09-23T23:23:59.174754+0000","last_peered":"2023-09-23T23:23:59.174754+0000","last_clean":"2023-09-23T23:23:59.174754+0000","last_became_active":"2023-09-23T23:23:43.511728+0000","last_became_peered":"2023-09-23T23:23:43.511728+0000","last_unstale":"2023-09-23T23:23:59.174754+0000","last_undegraded":"2023-09-23T23:23:59.174754+0000","last_fullsized":"2023-09-23T23:23:59.174754+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":1084,"log_dups_size":0,"ondisk_log_size":1084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:33.464148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71041024,"num_objects":1084,"num_object_clones":0,"num_object_copies":3252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1084,"num_write_kb":69376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":660,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:51.048830+0000","last_change":"2023-09-23T23:23:51.048787+0000","last_active":"2023-09-23T23:23:51.048830+0000","last_peered":"2023-09-23T23:23:51.048830+0000","last_clean":"2023-09-23T23:23:51.048830+0000","last_became_active":"2023-09-23T23:22:57.759601+0000","last_became_peered":"2023-09-23T23:22:57.759601+0000","last_unstale":"2023-09-23T23:23:51.048830+0000","last_undegraded":"2023-09-23T23:23:51.048830+0000","last_fullsized":"2023-09-23T23:23:51.048830+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.048718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.048718+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:18.385599+0000","scrub_duration":0.0016072149999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":206,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.079220+0000","last_change":"2023-09-23T23:23:35.037351+0000","last_active":"2023-09-23T23:23:50.079220+0000","last_peered":"2023-09-23T23:23:50.079220+0000","last_clean":"2023-09-23T23:23:50.079220+0000","last_became_active":"2023-09-23T23:23:35.036841+0000","last_became_peered":"2023-09-23T23:23:35.036841+0000","last_unstale":"2023-09-23T23:23:50.079220+0000","last_undegraded":"2023-09-23T23:23:50.079220+0000","last_fullsized":"2023-09-23T23:23:50.079220+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:20.618292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":45,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.177810+0000","last_change":"2023-09-23T23:22:38.453323+0000","last_active":"2023-09-23T23:23:50.177810+0000","last_peered":"2023-09-23T23:23:50.177810+0000","last_clean":"2023-09-23T23:23:50.177810+0000","last_became_active":"2023-09-23T23:22:38.452858+0000","last_became_peered":"2023-09-23T23:22:38.452858+0000","last_unstale":"2023-09-23T23:23:50.177810+0000","last_undegraded":"2023-09-23T23:23:50.177810+0000","last_fullsized":"2023-09-23T23:23:50.177810+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:31.988045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":88,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.461626+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:23:50.461626+0000","last_peered":"2023-09-23T23:23:50.461626+0000","last_clean":"2023-09-23T23:23:50.461626+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:23:50.461626+0000","last_undegraded":"2023-09-23T23:23:50.461626+0000","last_fullsized":"2023-09-23T23:23:50.461626+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"355'995","reported_seq":1009,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-23T23:23:58.036599+0000","last_change":"2023-09-23T23:23:43.505852+0000","last_active":"2023-09-23T23:23:58.036599+0000","last_peered":"2023-09-23T23:23:58.036599+0000","last_clean":"2023-09-23T23:23:58.036599+0000","last_became_active":"2023-09-23T23:23:43.505473+0000","last_became_peered":"2023-09-23T23:23:43.505473+0000","last_unstale":"2023-09-23T23:23:58.036599+0000","last_undegraded":"2023-09-23T23:23:58.036599+0000","last_fullsized":"2023-09-23T23:23:58.036599+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":995,"log_dups_size":0,"ondisk_log_size":995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:31.385610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65208320,"num_objects":995,"num_object_clones":0,"num_object_copies":2985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":995,"num_write_kb":63680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":667,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.077996+0000","last_change":"2023-09-23T23:23:46.892667+0000","last_active":"2023-09-23T23:23:50.077996+0000","last_peered":"2023-09-23T23:23:50.077996+0000","last_clean":"2023-09-23T23:23:50.077996+0000","last_became_active":"2023-09-23T23:22:57.465086+0000","last_became_peered":"2023-09-23T23:22:57.465086+0000","last_unstale":"2023-09-23T23:23:50.077996+0000","last_undegraded":"2023-09-23T23:23:50.077996+0000","last_fullsized":"2023-09-23T23:23:50.077996+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:55.262932+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":334,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-23T23:23:55.458550+0000","last_change":"2023-09-23T23:16:56.561147+0000","last_active":"2023-09-23T23:23:55.458550+0000","last_peered":"2023-09-23T23:23:55.458550+0000","last_clean":"2023-09-23T23:23:55.458550+0000","last_became_active":"2023-09-23T23:16:56.547808+0000","last_became_peered":"2023-09-23T23:16:56.547808+0000","last_unstale":"2023-09-23T23:23:55.458550+0000","last_undegraded":"2023-09-23T23:23:55.458550+0000","last_fullsized":"2023-09-23T23:23:55.458550+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:13.666957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":209,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.518567+0000","last_change":"2023-09-23T23:16:56.526187+0000","last_active":"2023-09-23T23:23:50.518567+0000","last_peered":"2023-09-23T23:23:50.518567+0000","last_clean":"2023-09-23T23:23:50.518567+0000","last_became_active":"2023-09-23T23:16:56.525874+0000","last_became_peered":"2023-09-23T23:16:56.525874+0000","last_unstale":"2023-09-23T23:23:50.518567+0000","last_undegraded":"2023-09-23T23:23:50.518567+0000","last_fullsized":"2023-09-23T23:23:50.518567+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:32:07.101559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":45,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.177453+0000","last_change":"2023-09-23T23:22:38.444291+0000","last_active":"2023-09-23T23:23:50.177453+0000","last_peered":"2023-09-23T23:23:50.177453+0000","last_clean":"2023-09-23T23:23:50.177453+0000","last_became_active":"2023-09-23T23:22:38.443996+0000","last_became_peered":"2023-09-23T23:22:38.443996+0000","last_unstale":"2023-09-23T23:23:50.177453+0000","last_undegraded":"2023-09-23T23:23:50.177453+0000","last_fullsized":"2023-09-23T23:23:50.177453+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:57.255970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"355'1144","reported_seq":1159,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-23T23:23:59.199333+0000","last_change":"2023-09-23T23:23:43.510714+0000","last_active":"2023-09-23T23:23:59.199333+0000","last_peered":"2023-09-23T23:23:59.199333+0000","last_clean":"2023-09-23T23:23:59.199333+0000","last_became_active":"2023-09-23T23:23:43.510441+0000","last_became_peered":"2023-09-23T23:23:43.510441+0000","last_unstale":"2023-09-23T23:23:59.199333+0000","last_undegraded":"2023-09-23T23:23:59.199333+0000","last_fullsized":"2023-09-23T23:23:59.199333+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":1144,"log_dups_size":0,"ondisk_log_size":1144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:54.915575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74973184,"num_objects":1144,"num_object_clones":0,"num_object_copies":3432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1144,"num_write_kb":73216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":668,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.517168+0000","last_change":"2023-09-23T23:23:45.832267+0000","last_active":"2023-09-23T23:23:50.517168+0000","last_peered":"2023-09-23T23:23:50.517168+0000","last_clean":"2023-09-23T23:23:50.517168+0000","last_became_active":"2023-09-23T23:22:17.009092+0000","last_became_peered":"2023-09-23T23:22:17.009092+0000","last_unstale":"2023-09-23T23:23:50.517168+0000","last_undegraded":"2023-09-23T23:23:50.517168+0000","last_fullsized":"2023-09-23T23:23:50.517168+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:42.979434+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":366,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.518958+0000","last_change":"2023-09-23T23:20:15.786693+0000","last_active":"2023-09-23T23:23:50.518958+0000","last_peered":"2023-09-23T23:23:50.518958+0000","last_clean":"2023-09-23T23:23:50.518958+0000","last_became_active":"2023-09-23T23:20:15.786360+0000","last_became_peered":"2023-09-23T23:20:15.786360+0000","last_unstale":"2023-09-23T23:23:50.518958+0000","last_undegraded":"2023-09-23T23:23:50.518958+0000","last_fullsized":"2023-09-23T23:23:50.518958+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:00.707946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":337,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.450123+0000","last_change":"2023-09-23T23:22:38.482705+0000","last_active":"2023-09-23T23:23:46.450123+0000","last_peered":"2023-09-23T23:23:46.450123+0000","last_clean":"2023-09-23T23:23:46.450123+0000","last_became_active":"2023-09-23T23:22:38.438239+0000","last_became_peered":"2023-09-23T23:22:38.438239+0000","last_unstale":"2023-09-23T23:23:46.450123+0000","last_undegraded":"2023-09-23T23:23:46.450123+0000","last_fullsized":"2023-09-23T23:23:46.450123+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:52.006349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":68,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.461380+0000","last_change":"2023-09-23T23:21:50.045682+0000","last_active":"2023-09-23T23:23:50.461380+0000","last_peered":"2023-09-23T23:23:50.461380+0000","last_clean":"2023-09-23T23:23:50.461380+0000","last_became_active":"2023-09-23T23:21:50.045339+0000","last_became_peered":"2023-09-23T23:21:50.045339+0000","last_unstale":"2023-09-23T23:23:50.461380+0000","last_undegraded":"2023-09-23T23:23:50.461380+0000","last_fullsized":"2023-09-23T23:23:50.461380+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:55.802091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":718,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.872439+0000","last_change":"2023-09-23T23:23:50.872439+0000","last_active":"2023-09-23T23:23:50.872439+0000","last_peered":"2023-09-23T23:23:50.872439+0000","last_clean":"2023-09-23T23:23:50.872439+0000","last_became_active":"2023-09-23T23:23:45.454920+0000","last_became_peered":"2023-09-23T23:23:45.454920+0000","last_unstale":"2023-09-23T23:23:50.872439+0000","last_undegraded":"2023-09-23T23:23:50.872439+0000","last_fullsized":"2023-09-23T23:23:50.872439+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:30.422526+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":106,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.518700+0000","last_change":"2023-09-23T23:23:36.026729+0000","last_active":"2023-09-23T23:23:50.518700+0000","last_peered":"2023-09-23T23:23:50.518700+0000","last_clean":"2023-09-23T23:23:50.518700+0000","last_became_active":"2023-09-23T23:23:36.026345+0000","last_became_peered":"2023-09-23T23:23:36.026345+0000","last_unstale":"2023-09-23T23:23:50.518700+0000","last_undegraded":"2023-09-23T23:23:50.518700+0000","last_fullsized":"2023-09-23T23:23:50.518700+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.463936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":95,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.450590+0000","last_change":"2023-09-23T23:22:38.439602+0000","last_active":"2023-09-23T23:23:46.450590+0000","last_peered":"2023-09-23T23:23:46.450590+0000","last_clean":"2023-09-23T23:23:46.450590+0000","last_became_active":"2023-09-23T23:22:38.439261+0000","last_became_peered":"2023-09-23T23:22:38.439261+0000","last_unstale":"2023-09-23T23:23:46.450590+0000","last_undegraded":"2023-09-23T23:23:46.450590+0000","last_fullsized":"2023-09-23T23:23:46.450590+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:03.293816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":417,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.517283+0000","last_change":"2023-09-23T23:23:45.457584+0000","last_active":"2023-09-23T23:23:50.517283+0000","last_peered":"2023-09-23T23:23:50.517283+0000","last_clean":"2023-09-23T23:23:50.517283+0000","last_became_active":"2023-09-23T23:23:45.457175+0000","last_became_peered":"2023-09-23T23:23:45.457175+0000","last_unstale":"2023-09-23T23:23:50.517283+0000","last_undegraded":"2023-09-23T23:23:50.517283+0000","last_fullsized":"2023-09-23T23:23:50.517283+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:14:45.802642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:14:45.802642+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":96,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.518385+0000","last_change":"2023-09-23T23:22:38.452968+0000","last_active":"2023-09-23T23:23:50.518385+0000","last_peered":"2023-09-23T23:23:50.518385+0000","last_clean":"2023-09-23T23:23:50.518385+0000","last_became_active":"2023-09-23T23:22:38.437986+0000","last_became_peered":"2023-09-23T23:22:38.437986+0000","last_unstale":"2023-09-23T23:23:50.518385+0000","last_undegraded":"2023-09-23T23:23:50.518385+0000","last_fullsized":"2023-09-23T23:23:50.518385+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:10.333271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":360,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.516681+0000","last_change":"2023-09-23T23:23:36.032480+0000","last_active":"2023-09-23T23:23:50.516681+0000","last_peered":"2023-09-23T23:23:50.516681+0000","last_clean":"2023-09-23T23:23:50.516681+0000","last_became_active":"2023-09-23T23:23:36.032169+0000","last_became_peered":"2023-09-23T23:23:36.032169+0000","last_unstale":"2023-09-23T23:23:50.516681+0000","last_undegraded":"2023-09-23T23:23:50.516681+0000","last_fullsized":"2023-09-23T23:23:50.516681+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:52.893674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":82,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.517978+0000","last_change":"2023-09-23T23:21:22.194730+0000","last_active":"2023-09-23T23:23:50.517978+0000","last_peered":"2023-09-23T23:23:50.517978+0000","last_clean":"2023-09-23T23:23:50.517978+0000","last_became_active":"2023-09-23T23:21:22.194420+0000","last_became_peered":"2023-09-23T23:21:22.194420+0000","last_unstale":"2023-09-23T23:23:50.517978+0000","last_undegraded":"2023-09-23T23:23:50.517978+0000","last_fullsized":"2023-09-23T23:23:50.517978+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:31.083059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":199,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.516632+0000","last_change":"2023-09-23T23:20:26.841878+0000","last_active":"2023-09-23T23:23:50.516632+0000","last_peered":"2023-09-23T23:23:50.516632+0000","last_clean":"2023-09-23T23:23:50.516632+0000","last_became_active":"2023-09-23T23:20:26.841590+0000","last_became_peered":"2023-09-23T23:20:26.841590+0000","last_unstale":"2023-09-23T23:23:50.516632+0000","last_undegraded":"2023-09-23T23:23:50.516632+0000","last_fullsized":"2023-09-23T23:23:50.516632+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:23.904011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":601,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:51.871796+0000","last_change":"2023-09-23T23:23:51.871725+0000","last_active":"2023-09-23T23:23:51.871796+0000","last_peered":"2023-09-23T23:23:51.871796+0000","last_clean":"2023-09-23T23:23:51.871796+0000","last_became_active":"2023-09-23T23:23:46.456342+0000","last_became_peered":"2023-09-23T23:23:46.456342+0000","last_unstale":"2023-09-23T23:23:51.871796+0000","last_undegraded":"2023-09-23T23:23:51.871796+0000","last_fullsized":"2023-09-23T23:23:51.871796+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:02.041716+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":361,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.517225+0000","last_change":"2023-09-23T23:20:16.730934+0000","last_active":"2023-09-23T23:23:50.517225+0000","last_peered":"2023-09-23T23:23:50.517225+0000","last_clean":"2023-09-23T23:23:50.517225+0000","last_became_active":"2023-09-23T23:20:16.730571+0000","last_became_peered":"2023-09-23T23:20:16.730571+0000","last_unstale":"2023-09-23T23:23:50.517225+0000","last_undegraded":"2023-09-23T23:23:50.517225+0000","last_fullsized":"2023-09-23T23:23:50.517225+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:17.993278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":364,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.516792+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:23:50.516792+0000","last_peered":"2023-09-23T23:23:50.516792+0000","last_clean":"2023-09-23T23:23:50.516792+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:23:50.516792+0000","last_undegraded":"2023-09-23T23:23:50.516792+0000","last_fullsized":"2023-09-23T23:23:50.516792+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":676,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-09-23T23:23:56.246070+0000","last_change":"2023-09-23T23:23:56.246175+0000","last_active":"2023-09-23T23:23:56.246070+0000","last_peered":"2023-09-23T23:23:56.246070+0000","last_clean":"2023-09-23T23:23:56.246070+0000","last_became_active":"2023-09-23T23:23:45.458251+0000","last_became_peered":"2023-09-23T23:23:45.458251+0000","last_unstale":"2023-09-23T23:23:56.246070+0000","last_undegraded":"2023-09-23T23:23:56.246070+0000","last_fullsized":"2023-09-23T23:23:56.246070+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:41.637297+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":356,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T23:23:46.451152+0000","last_change":"2023-09-23T23:20:15.786279+0000","last_active":"2023-09-23T23:23:46.451152+0000","last_peered":"2023-09-23T23:23:46.451152+0000","last_clean":"2023-09-23T23:23:46.451152+0000","last_became_active":"2023-09-23T23:20:15.782731+0000","last_became_peered":"2023-09-23T23:20:15.782731+0000","last_unstale":"2023-09-23T23:23:46.451152+0000","last_undegraded":"2023-09-23T23:23:46.451152+0000","last_fullsized":"2023-09-23T23:23:46.451152+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:58.011396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":390,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T23:23:50.517736+0000","last_change":"2023-09-23T23:22:38.447350+0000","last_active":"2023-09-23T23:23:50.517736+0000","last_peered":"2023-09-23T23:23:50.517736+0000","last_clean":"2023-09-23T23:23:50.517736+0000","last_became_active":"2023-09-23T23:22:38.447081+0000","last_became_peered":"2023-09-23T23:22:38.447081+0000","last_unstale":"2023-09-23T23:23:50.517736+0000","last_undegraded":"2023-09-23T23:23:50.517736+0000","last_fullsized":"2023-09-23T23:23:50.517736+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:39:12.306545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1019281408,"num_objects":15553,"num_object_clones":0,"num_object_copies":46659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15553,"num_write_kb":995392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2659123200,"data_stored":2659123200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":15553,"ondisk_log_size":15553,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":1220,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":626394,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":48,"num_read_kb":91,"num_write":24,"num_write_kb":238,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":677686,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782534,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":774472,"kb_used_data":729536,"kb_used_omap":8,"kb_used_meta":44919,"kb_avail":92970680,"statfs":{"total":95995035648,"available":95201976320,"internally_reserved":0,"allocated":747044864,"data_stored":746046627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":45997125},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50800000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.86199999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":5.0659999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.438}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38700000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45400000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496011,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1087616,"kb_used_data":633272,"kb_used_omap":0,"kb_used_meta":454336,"kb_avail":92657536,"statfs":{"total":95995035648,"available":94881316864,"internally_reserved":0,"allocated":648470528,"data_stored":647479674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":465240064},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:23:53 2023","interfaces":[{"interface":"back","average":{"1min":8.3539999999999992,"5min":7.2770000000000001,"15min":5.0810000000000004},"min":{"1min":0.21099999999999999,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":112.767,"5min":120.392,"15min":161.64599999999999},"last":33.219999999999999},{"interface":"front","average":{"1min":8.1630000000000003,"5min":6.8719999999999999,"15min":4.9619999999999997},"min":{"1min":0.23899999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":109.38200000000001,"5min":119.917,"15min":161.38499999999999},"last":33.235999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":1.5009999999999999,"15min":4.2220000000000004},"min":{"1min":0.245,"5min":0.245,"15min":0.24099999999999999},"max":{"1min":1.5029999999999999,"5min":58.295999999999999,"15min":210.869},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.70899999999999996,"5min":1.5589999999999999,"15min":4.298},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":2.3279999999999998,"5min":58.326999999999998,"15min":210.89699999999999},"last":0.45500000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:23:34 2023","interfaces":[{"interface":"back","average":{"1min":5.6189999999999998,"5min":2.1699999999999999,"15min":2.0179999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":59.631999999999998,"5min":59.631999999999998,"15min":59.631999999999998},"last":0.55600000000000005},{"interface":"front","average":{"1min":5.7160000000000002,"5min":2.3159999999999998,"15min":1.7190000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":59.750999999999998,"5min":59.750999999999998,"15min":59.750999999999998},"last":0.47499999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:23:27 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.4159999999999999,"15min":2.9039999999999999},"min":{"1min":0.28699999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.0569999999999999,"5min":10.528,"15min":109.98699999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.5740000000000001,"15min":3.016},"min":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.24199999999999999},"max":{"1min":5.54,"5min":13.039,"15min":110.024},"last":0.58899999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:23:06 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":2.5569999999999999,"15min":1.5900000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.22900000000000001},"max":{"1min":2.363,"5min":166.05199999999999,"15min":166.05199999999999},"last":0.376},{"interface":"front","average":{"1min":0.59199999999999997,"5min":3.2810000000000001,"15min":1.9410000000000001},"min":{"1min":0.28100000000000003,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":3.02,"5min":166.631,"15min":166.631},"last":0.39700000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:23:06 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":1.8360000000000001,"15min":1.512},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.383,"5min":91.899000000000001,"15min":91.899000000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":2.1579999999999999,"15min":1.5349999999999999},"min":{"1min":0.33900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":3.0939999999999999,"5min":94.174000000000007,"15min":94.174000000000007},"last":0.35799999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:18:28 2023","interfaces":[{"interface":"back","average":{"1min":2.7250000000000001,"5min":1.2689999999999999,"15min":1.27},"min":{"1min":0.36799999999999999,"5min":0.32300000000000001,"15min":0.32100000000000001},"max":{"1min":13.823,"5min":13.823,"15min":13.823},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.5609999999999999,"5min":0.79900000000000004,"15min":0.98799999999999999},"min":{"1min":0.35999999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":13.103,"5min":13.103,"15min":13.103},"last":0.40899999999999997}]}]},{"osd":1,"up_from":112,"seq":481036337352,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":792556,"kb_used_data":288036,"kb_used_omap":4,"kb_used_meta":504507,"kb_avail":92952596,"statfs":{"total":95995035648,"available":95183458304,"internally_reserved":0,"allocated":294948864,"data_stored":293329938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":516616078},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:23:52 2023","interfaces":[{"interface":"back","average":{"1min":4.3529999999999998,"5min":3.806,"15min":2.7410000000000001},"min":{"1min":0.189,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":75.823999999999998,"5min":75.823999999999998,"15min":75.823999999999998},"last":0.83099999999999996},{"interface":"front","average":{"1min":4.5419999999999998,"5min":3.6200000000000001,"15min":2.8330000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.192},"max":{"1min":75.840999999999994,"5min":75.840999999999994,"15min":75.840999999999994},"last":0.33300000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:23:34 2023","interfaces":[{"interface":"back","average":{"1min":4.6230000000000002,"5min":1.502,"15min":4.5730000000000004},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":78.950000000000003,"5min":78.950000000000003,"15min":78.950000000000003},"last":2.343},{"interface":"front","average":{"1min":5.266,"5min":1.8859999999999999,"15min":3.2229999999999999},"min":{"1min":0.25,"5min":0.218,"15min":0.218},"max":{"1min":78.694000000000003,"5min":78.694000000000003,"15min":78.694000000000003},"last":1.03}]},{"osd":3,"last update":"Sat Sep 23 23:23:27 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":2.7040000000000002,"15min":3.5169999999999999},"min":{"1min":0.22600000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":3.5070000000000001,"5min":145.88499999999999,"15min":145.88499999999999},"last":1.087},{"interface":"front","average":{"1min":0.94499999999999995,"5min":2.5329999999999999,"15min":3.29},"min":{"1min":0.252,"5min":0.245,"15min":0.245},"max":{"1min":7.399,"5min":145.93000000000001,"15min":145.93000000000001},"last":0.84999999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:23:52 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.0469999999999999,"15min":1.637},"min":{"1min":0.29199999999999998,"5min":0.28799999999999998,"15min":0.28000000000000003},"max":{"1min":5.2750000000000004,"5min":6.6120000000000001,"15min":18.474},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.081,"5min":0.999,"15min":1.7529999999999999},"min":{"1min":0.32500000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.2789999999999999,"5min":7.2439999999999998,"15min":25.710999999999999},"last":0.85999999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.92700000000000005,"15min":3.2730000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.26400000000000001},"max":{"1min":4.5640000000000001,"5min":4.827,"15min":250.33000000000001},"last":0.69399999999999995},{"interface":"front","average":{"1min":1.603,"5min":1.2390000000000001,"15min":3.266},"min":{"1min":0.34899999999999998,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":10.92,"5min":10.92,"15min":250.21199999999999},"last":0.75800000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":1.5820000000000001,"15min":2.1349999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.32400000000000001},"max":{"1min":4.601,"5min":49.171999999999997,"15min":49.171999999999997},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.496,"5min":1.76,"15min":2.1869999999999998},"min":{"1min":0.36699999999999999,"5min":0.32800000000000001,"15min":0.32700000000000001},"max":{"1min":5.9699999999999998,"5min":49.222000000000001,"15min":49.222000000000001},"last":0.625}]},{"osd":7,"last update":"Sat Sep 23 23:18:40 2023","interfaces":[{"interface":"back","average":{"1min":3.0710000000000002,"5min":2.4540000000000002,"15min":1.9970000000000001},"min":{"1min":0.47699999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":18.532,"5min":18.532,"15min":18.532},"last":0.79800000000000004},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.3380000000000001,"15min":1.8440000000000001},"min":{"1min":0.42199999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.9710000000000001,"5min":25.667000000000002,"15min":25.667000000000002},"last":0.72499999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024175,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1003856,"kb_used_data":194376,"kb_used_omap":5,"kb_used_meta":809466,"kb_avail":92741296,"statfs":{"total":95995035648,"available":94967087104,"internally_reserved":0,"allocated":199041024,"data_stored":197377888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":828893818},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:23:02 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":2.286,"15min":3.214},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":6.3499999999999996,"5min":21.585999999999999,"15min":155.91999999999999},"last":0.64900000000000002},{"interface":"front","average":{"1min":1.079,"5min":1.869,"15min":2.8010000000000002},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":7.1150000000000002,"5min":16.591000000000001,"15min":153.28899999999999},"last":0.434}]},{"osd":2,"last update":"Sat Sep 23 23:23:32 2023","interfaces":[{"interface":"back","average":{"1min":10.75,"5min":5.8959999999999999,"15min":4.077},"min":{"1min":0.23999999999999999,"5min":0.217,"15min":0.217},"max":{"1min":171.15100000000001,"5min":171.15100000000001,"15min":171.15100000000001},"last":0.21199999999999999},{"interface":"front","average":{"1min":10.177,"5min":6.0700000000000003,"15min":4.7539999999999996},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":173.04900000000001,"5min":173.04900000000001,"15min":173.04900000000001},"last":0.46300000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:23:10 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":2.2410000000000001,"15min":3.2709999999999999},"min":{"1min":0.25600000000000001,"5min":0.245,"15min":0.22700000000000001},"max":{"1min":2.5670000000000002,"5min":35.148000000000003,"15min":82.691000000000003},"last":0.84799999999999998},{"interface":"front","average":{"1min":1.1819999999999999,"5min":2.036,"15min":2.6589999999999998},"min":{"1min":0.26500000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.7480000000000002,"5min":35.186999999999998,"15min":82.429000000000002},"last":0.34899999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:23:37 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":2.4390000000000001,"15min":2.2389999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.247},"max":{"1min":6.899,"5min":86.915999999999997,"15min":86.915999999999997},"last":0.40100000000000002},{"interface":"front","average":{"1min":9.9120000000000008,"5min":3.9689999999999999,"15min":2.7440000000000002},"min":{"1min":0.35599999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":171.251,"5min":171.251,"15min":171.251},"last":0.41699999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:23:37 2023","interfaces":[{"interface":"back","average":{"1min":10.173,"5min":3.0249999999999999,"15min":2.3740000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29099999999999998},"max":{"1min":171.30699999999999,"5min":171.30699999999999,"15min":171.30699999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.226,"5min":1.3340000000000001,"15min":1.6850000000000001},"min":{"1min":0.40799999999999997,"5min":0.34300000000000003,"15min":0.30299999999999999},"max":{"1min":3.9470000000000001,"5min":10.231,"15min":15.929},"last":0.53600000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:23:37 2023","interfaces":[{"interface":"back","average":{"1min":9.907,"5min":3.1309999999999998,"15min":2.6040000000000001},"min":{"1min":0.35499999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":171.33600000000001,"5min":171.33600000000001,"15min":171.33600000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.222,"5min":1.5800000000000001,"15min":2.069},"min":{"1min":0.40500000000000003,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.9830000000000001,"5min":10.256,"15min":30.673999999999999},"last":0.56399999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:18:32 2023","interfaces":[{"interface":"back","average":{"1min":4.306,"5min":2.3620000000000001,"15min":2.1829999999999998},"min":{"1min":0.372,"5min":0.34499999999999997,"15min":0.29299999999999998},"max":{"1min":26.413,"5min":26.413,"15min":26.413},"last":0.49399999999999999},{"interface":"front","average":{"1min":4.7960000000000003,"5min":2.1920000000000002,"15min":2.1219999999999999},"min":{"1min":0.372,"5min":0.36499999999999999,"15min":0.313},"max":{"1min":26.437999999999999,"5min":26.437999999999999,"15min":26.437999999999999},"last":0.58099999999999996}]}]},{"osd":2,"up_from":199,"seq":854698491996,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":596328,"kb_used_data":296352,"kb_used_omap":7,"kb_used_meta":299960,"kb_avail":93148824,"statfs":{"total":95995035648,"available":95384395776,"internally_reserved":0,"allocated":303464448,"data_stored":301844275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":307159121},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:23:36 2023","interfaces":[{"interface":"back","average":{"1min":6.0979999999999999,"5min":4.5469999999999997,"15min":5.3490000000000002},"min":{"1min":0.17699999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":95.587999999999994,"5min":119.277,"15min":119.277},"last":0.23999999999999999},{"interface":"front","average":{"1min":4.9740000000000002,"5min":4.1660000000000004,"15min":5.5979999999999999},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":95.796000000000006,"5min":119.56100000000001,"15min":119.56100000000001},"last":0.28399999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:23:36 2023","interfaces":[{"interface":"back","average":{"1min":8.4019999999999992,"5min":2.73,"15min":10.066000000000001},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":157.98699999999999,"5min":157.98699999999999,"15min":157.98699999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":9.4629999999999992,"5min":3.1440000000000001,"15min":9.5500000000000007},"min":{"1min":0.29299999999999998,"5min":0.25800000000000001,"15min":0.245},"max":{"1min":158.03800000000001,"5min":158.03800000000001,"15min":158.03800000000001},"last":0.34300000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:23:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":2.964,"15min":5.9710000000000001},"min":{"1min":0.248,"5min":0.223,"15min":0.223},"max":{"1min":11.852,"5min":119.366,"15min":119.366},"last":0.314},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.2429999999999999,"15min":5.1470000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":12.047000000000001,"5min":72.412999999999997,"15min":72.412999999999997},"last":0.30299999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:23:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":2.3660000000000001,"15min":3.9319999999999999},"min":{"1min":0.28000000000000003,"5min":0.25,"15min":0.25},"max":{"1min":7.7939999999999996,"5min":119.88200000000001,"15min":332.98099999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.92000000000000004,"15min":3.431},"min":{"1min":0.36199999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.4079999999999999,"5min":5.4290000000000003,"15min":332.96199999999999},"last":0.45100000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:23:36 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":2.7050000000000001,"15min":2.3260000000000001},"min":{"1min":0.33700000000000002,"5min":0.29499999999999998,"15min":0.27500000000000002},"max":{"1min":7.8410000000000002,"5min":119.919,"15min":119.919},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.81200000000000006,"5min":2.468,"15min":2.3170000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":5.6509999999999998,"5min":119.306,"15min":119.306},"last":0.39300000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:23:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3580000000000001,"5min":2.9329999999999998,"15min":4.2229999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.31},"max":{"1min":7.8799999999999999,"5min":119.943,"15min":349.32799999999997},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.63800000000000001,"5min":1.6040000000000001,"15min":3.71},"min":{"1min":0.36699999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.4350000000000001,"5min":39.856999999999999,"15min":349.17700000000002},"last":0.42199999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:18:22 2023","interfaces":[{"interface":"back","average":{"1min":23.006,"5min":6.4870000000000001,"15min":3.734},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":239.768,"5min":239.768,"15min":239.768},"last":0.36899999999999999},{"interface":"front","average":{"1min":22.75,"5min":5.71,"15min":2.8700000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":239.78899999999999,"5min":239.78899999999999,"15min":239.78899999999999},"last":0.40699999999999997}]}]},{"osd":3,"up_from":148,"seq":635655159966,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":665584,"kb_used_data":246696,"kb_used_omap":2,"kb_used_meta":418877,"kb_avail":93079568,"statfs":{"total":95995035648,"available":95313477632,"internally_reserved":0,"allocated":252616704,"data_stored":251050708,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":428930101},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:23:25 2023","interfaces":[{"interface":"back","average":{"1min":7.2720000000000002,"5min":2.875,"15min":2.7280000000000002},"min":{"1min":0.20200000000000001,"5min":0.158,"15min":0.158},"max":{"1min":103.866,"5min":103.866,"15min":103.866},"last":0.27100000000000002},{"interface":"front","average":{"1min":6.6909999999999998,"5min":2.1659999999999999,"15min":2.6499999999999999},"min":{"1min":0.20799999999999999,"5min":0.193,"15min":0.17999999999999999},"max":{"1min":101.884,"5min":101.884,"15min":101.884},"last":0.72299999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:23:25 2023","interfaces":[{"interface":"back","average":{"1min":2.4910000000000001,"5min":1.577,"15min":2.7090000000000001},"min":{"1min":0.26600000000000001,"5min":0.216,"15min":0.19600000000000001},"max":{"1min":14.427,"5min":14.427,"15min":76.322999999999993},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.6160000000000001,"5min":1.069,"15min":2.4100000000000001},"min":{"1min":0.254,"5min":0.24399999999999999,"15min":0.185},"max":{"1min":7.9180000000000001,"5min":7.9180000000000001,"15min":76.444000000000003},"last":0.36799999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:23:35 2023","interfaces":[{"interface":"back","average":{"1min":3.161,"5min":4.0529999999999999,"15min":3.7850000000000001},"min":{"1min":0.24299999999999999,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":9.3819999999999997,"5min":101.279,"15min":101.279},"last":0.68500000000000005},{"interface":"front","average":{"1min":2.4550000000000001,"5min":3.681,"15min":2.399},"min":{"1min":0.34499999999999997,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.4640000000000004,"5min":101.294,"15min":101.294},"last":0.55300000000000005}]},{"osd":4,"last update":"Sat Sep 23 23:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.238,"15min":1.5680000000000001},"min":{"1min":0.29999999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":7.2610000000000001,"5min":9.2089999999999996,"15min":14.754},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.3939999999999999,"5min":1.645,"15min":1.952},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.28499999999999998},"max":{"1min":7.9660000000000002,"5min":9.5960000000000001,"15min":13.532999999999999},"last":0.38800000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.3089999999999999,"15min":1.9750000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.28199999999999997},"max":{"1min":3.206,"5min":10.531000000000001,"15min":75.733000000000004},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.7709999999999999,"5min":1.625,"15min":2.1150000000000002},"min":{"1min":0.37,"5min":0.33800000000000002,"15min":0.28399999999999997},"max":{"1min":5.6470000000000002,"5min":7.0750000000000002,"15min":75.686999999999998},"last":0.66100000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":1.585,"15min":1.879},"min":{"1min":0.38,"5min":0.38,"15min":0.32200000000000001},"max":{"1min":8.0050000000000008,"5min":10.593999999999999,"15min":14.492000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.726,"5min":1.871,"15min":1.9330000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.33300000000000002},"max":{"1min":5.6879999999999997,"5min":7.1539999999999999,"15min":9.8379999999999992},"last":0.71499999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:18:09 2023","interfaces":[{"interface":"back","average":{"1min":2.198,"5min":2.915,"15min":2.5459999999999998},"min":{"1min":0.40200000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":9.4280000000000008,"5min":130.839,"15min":130.839},"last":0.58599999999999997},{"interface":"front","average":{"1min":2.9129999999999998,"5min":2.992,"15min":2.665},"min":{"1min":0.434,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":8.8190000000000008,"5min":130.90000000000001,"15min":130.90000000000001},"last":0.48299999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019284,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1085372,"kb_used_data":488564,"kb_used_omap":0,"kb_used_meta":596800,"kb_avail":92659780,"statfs":{"total":95995035648,"available":94883614720,"internally_reserved":0,"allocated":500289536,"data_stored":499298640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":611123200},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:23:45 2023","interfaces":[{"interface":"back","average":{"1min":7.9169999999999998,"5min":5.7530000000000001,"15min":5.2329999999999997},"min":{"1min":0.22800000000000001,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":92.141999999999996,"5min":92.141999999999996,"15min":238.62200000000001},"last":0.37},{"interface":"front","average":{"1min":8.0999999999999996,"5min":6.234,"15min":5.6669999999999998},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":92.284000000000006,"5min":92.284000000000006,"15min":238.58199999999999},"last":0.68000000000000005}]},{"osd":1,"last update":"Sat Sep 23 23:22:59 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":3.1930000000000001,"15min":4.8150000000000004},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.1840000000000002,"5min":161.923,"15min":175.00899999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":3.1960000000000002,"15min":4.968},"min":{"1min":0.27500000000000002,"5min":0.26500000000000001,"15min":0.25800000000000001},"max":{"1min":2.04,"5min":161.39699999999999,"15min":174.821},"last":0.30299999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:23:35 2023","interfaces":[{"interface":"back","average":{"1min":6.6710000000000003,"5min":2.843,"15min":3.2170000000000001},"min":{"1min":0.30099999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":63.113999999999997,"5min":63.113999999999997,"15min":63.113999999999997},"last":1.2250000000000001},{"interface":"front","average":{"1min":6.0149999999999997,"5min":2.7789999999999999,"15min":2.6880000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.316},"max":{"1min":63.125,"5min":63.125,"15min":63.125},"last":0.90600000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:23:27 2023","interfaces":[{"interface":"back","average":{"1min":4.2990000000000004,"5min":1.8580000000000001,"15min":4.2619999999999996},"min":{"1min":0.34100000000000003,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":65.855999999999995,"5min":65.855999999999995,"15min":303.41500000000002},"last":0.27200000000000002},{"interface":"front","average":{"1min":4.2530000000000001,"5min":2.3839999999999999,"15min":4.601},"min":{"1min":0.317,"5min":0.308,"15min":0.308},"max":{"1min":66.293000000000006,"5min":66.293000000000006,"15min":303.428},"last":0.40699999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:22:59 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":1.0389999999999999,"15min":1.5369999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.23699999999999999},"max":{"1min":0.50600000000000001,"5min":5.8330000000000002,"15min":160.79499999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.51300000000000001,"5min":1.262,"15min":1.794},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":2.121,"5min":7.9669999999999996,"15min":160.61099999999999},"last":0.35299999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:23:27 2023","interfaces":[{"interface":"back","average":{"1min":4.7240000000000002,"5min":1.6539999999999999,"15min":1.3380000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":90.346999999999994,"5min":90.346999999999994,"15min":90.346999999999994},"last":0.31900000000000001},{"interface":"front","average":{"1min":4.8570000000000002,"5min":2.02,"15min":1.478},"min":{"1min":0.31900000000000001,"5min":0.30599999999999999,"15min":0.27500000000000002},"max":{"1min":90.733000000000004,"5min":90.733000000000004,"15min":90.733000000000004},"last":0.46300000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:18:35 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.5609999999999999,"15min":1.3140000000000001},"min":{"1min":0.36299999999999999,"5min":0.315,"15min":0.30499999999999999},"max":{"1min":7.4850000000000003,"5min":14.842000000000001,"15min":14.842000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.0169999999999999,"5min":1.502,"15min":1.341},"min":{"1min":0.433,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.882,"5min":14.561,"15min":14.561},"last":0.48999999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757647,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":654392,"kb_used_data":136048,"kb_used_omap":0,"kb_used_meta":518336,"kb_avail":93090760,"statfs":{"total":95995035648,"available":95324938240,"internally_reserved":0,"allocated":139313152,"data_stored":138326551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":530776064},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:23:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":6.859,"15min":3.5489999999999999},"min":{"1min":0.24299999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":5.1479999999999997,"5min":113.869,"15min":113.869},"last":0.35599999999999998},{"interface":"front","average":{"1min":7.4809999999999999,"5min":7.9820000000000002,"15min":4.2270000000000003},"min":{"1min":0.23799999999999999,"5min":0.23000000000000001,"15min":0.22700000000000001},"max":{"1min":138.55199999999999,"5min":138.55199999999999,"15min":138.55199999999999},"last":0.38100000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:23:57 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.1419999999999999,"15min":3.726},"min":{"1min":0.28199999999999997,"5min":0.27200000000000002,"15min":0.246},"max":{"1min":4.1699999999999999,"5min":8.0899999999999999,"15min":193.584},"last":0.496},{"interface":"front","average":{"1min":0.76400000000000001,"5min":1.514,"15min":3.9649999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.9109999999999996,"5min":16.312999999999999,"15min":194.03100000000001},"last":0.79500000000000004}]},{"osd":2,"last update":"Sat Sep 23 23:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":2.2970000000000002,"15min":1.776},"min":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":3.1150000000000002,"5min":86.765000000000001,"15min":86.765000000000001},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.83199999999999996,"5min":1.9379999999999999,"15min":1.8109999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.25800000000000001},"max":{"1min":3.4279999999999999,"5min":86.837000000000003,"15min":86.837000000000003},"last":0.44500000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:23:26 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":2.488,"15min":4.0279999999999996},"min":{"1min":0.34200000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.1179999999999999,"5min":58.264000000000003,"15min":159.53299999999999},"last":1.51},{"interface":"front","average":{"1min":1.054,"5min":2.7450000000000001,"15min":4.5640000000000001},"min":{"1min":0.34200000000000003,"5min":0.312,"15min":0.312},"max":{"1min":4,"5min":58.137999999999998,"15min":159.54900000000001},"last":1.5329999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:23:09 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.98299999999999998,"15min":1.2749999999999999},"min":{"1min":0.27200000000000002,"5min":0.24299999999999999,"15min":0.23999999999999999},"max":{"1min":1.248,"5min":8.1210000000000004,"15min":14.538},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.443,"5min":0.79300000000000004,"15min":0.89100000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":1.236,"5min":8.1829999999999998,"15min":11.923999999999999},"last":0.53200000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:23:09 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":4.1500000000000004,"15min":2.2010000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.1659999999999999,"5min":310.68799999999999,"15min":310.68799999999999},"last":1.4390000000000001},{"interface":"front","average":{"1min":0.502,"5min":4.5339999999999998,"15min":2.5099999999999998},"min":{"1min":0.26000000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.411,"5min":311.45400000000001,"15min":311.45400000000001},"last":0.85299999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:18:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2010000000000001,"5min":1.286,"15min":1.3700000000000001},"min":{"1min":0.36699999999999999,"5min":0.32700000000000001,"15min":0.29199999999999998},"max":{"1min":12.09,"5min":12.09,"15min":14.929},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.319,"15min":1.276},"min":{"1min":0.371,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":5.7350000000000003,"5min":6.7089999999999996,"15min":15.090999999999999},"last":0.51900000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":282329088,"data_stored":282329088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":291307520,"data_stored":291307520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":319356928,"data_stored":319356928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":263585792,"data_stored":263585792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":228065280,"data_stored":228065280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":143065088,"data_stored":143065088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":677249024,"data_stored":677249024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":454164480,"data_stored":454164480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:24:04.288 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T23:24:04.288 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 41 2023-09-23T23:24:04.360 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:04.492 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:04 smithi110 ceph-mon[212939]: pgmap v1041: 119 pgs: 119 active+clean; 1.3 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T23:24:04.492 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:04 smithi110 ceph-mon[221102]: pgmap v1041: 119 pgs: 119 active+clean; 1.3 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T23:24:04.669 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:04.681 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:04.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:04.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:04.794 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:04.822 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:04 smithi138 ceph-mon[193606]: pgmap v1041: 119 pgs: 119 active+clean; 1.3 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T23:24:04.825 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:04.826 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:24:04.826 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:24:04.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.826 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:27334 11.a 11:5fcb9b9d:::benchmark_data_smithi110_433746_object27333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2023-09-23T23:24:04.827 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:24:04.822746+0000", 2023-09-23T23:24:04.827 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.002090799, 2023-09-23T23:24:04.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021019200000000002, 2023-09-23T23:24:04.827 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:24:04.827 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:04.827 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:24:04.827 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:24:04.827 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:24:04.827 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 27334 2023-09-23T23:24:04.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.828 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:24:04.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:24:04.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.822746+0000", 2023-09-23T23:24:04.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:24:04.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:24:04.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.822746+0000", 2023-09-23T23:24:04.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:24:04.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:24:04.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.822744+0000", 2023-09-23T23:24:04.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:24:04.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:24:04.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.822771+0000", 2023-09-23T23:24:04.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6834000000000001e-05 2023-09-23T23:24:04.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:24:04.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.822788+0000", 2023-09-23T23:24:04.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7549000000000001e-05 2023-09-23T23:24:04.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:24:04.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.822797+0000", 2023-09-23T23:24:04.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7229999999999993e-06 2023-09-23T23:24:04.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:24:04.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.822853+0000", 2023-09-23T23:24:04.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5622999999999999e-05 2023-09-23T23:24:04.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:24:04.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.822999+0000", 2023-09-23T23:24:04.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000146501 2023-09-23T23:24:04.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:24:04.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.823169+0000", 2023-09-23T23:24:04.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017012400000000001 2023-09-23T23:24:04.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:24:04.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.824464+0000", 2023-09-23T23:24:04.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012945229999999999 2023-09-23T23:24:04.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:04.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.824731+0000", 2023-09-23T23:24:04.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026686100000000002 2023-09-23T23:24:04.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:04.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.824761+0000", 2023-09-23T23:24:04.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0531000000000003e-05 2023-09-23T23:24:04.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:24:04.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.824812+0000", 2023-09-23T23:24:04.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0435000000000001e-05 2023-09-23T23:24:04.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:24:04.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.824847+0000", 2023-09-23T23:24:04.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5800000000000003e-05 2023-09-23T23:24:04.835 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:24:04.835 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:24:04.835 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:24:04.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.836 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.35904.0:27334 11.a e355/349 ondisk, result = 0)", 2023-09-23T23:24:04.836 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:24:04.824708+0000", 2023-09-23T23:24:04.836 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.000128155, 2023-09-23T23:24:04.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00029964999999999999, 2023-09-23T23:24:04.836 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:24:04.836 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:24:04.836 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:24:04.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:24:04.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.824708+0000", 2023-09-23T23:24:04.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:24:04.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:24:04.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.824708+0000", 2023-09-23T23:24:04.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:24:04.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:24:04.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.824708+0000", 2023-09-23T23:24:04.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:24:04.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:24:04.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.824713+0000", 2023-09-23T23:24:04.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0610000000000003e-06 2023-09-23T23:24:04.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:24:04.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.824722+0000", 2023-09-23T23:24:04.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5370000000000003e-06 2023-09-23T23:24:04.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:24:04.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.824729+0000", 2023-09-23T23:24:04.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6209999999999998e-06 2023-09-23T23:24:04.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:24:04.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.824751+0000", 2023-09-23T23:24:04.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2245000000000001e-05 2023-09-23T23:24:04.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:24:04.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.824757+0000", 2023-09-23T23:24:04.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7930000000000003e-06 2023-09-23T23:24:04.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:24:04.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:24:04.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:24:04.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:04.825009+0000", 2023-09-23T23:24:04.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025209100000000002 2023-09-23T23:24:04.840 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:24:04.840 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:24:04.841 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:24:04.841 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:24:04.841 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-09-23T23:24:04.841 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 2 2023-09-23T23:24:04.841 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:24:04.899 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:04.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:04.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:05.006 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:05.076 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:05.120 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 13 16 27873 27857 133.843 99.5 0.0124876 0.00746327 2023-09-23T23:24:05.189 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:05.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:05.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:05.295 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:05.353 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:05.462 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:05.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:05.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:05.566 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:05.592 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:05.633 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:05.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:05.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:05.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:05 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:24:05.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:05 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:24:05.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:05 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:05.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:05 smithi110 ceph-mon[212939]: 1.e scrub starts 2023-09-23T23:24:05.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:05 smithi110 ceph-mon[212939]: 1.e scrub ok 2023-09-23T23:24:05.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:05 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:24:05.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:05 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:24:05.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:05 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:05.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:05 smithi110 ceph-mon[221102]: 1.e scrub starts 2023-09-23T23:24:05.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:05 smithi110 ceph-mon[221102]: 1.e scrub ok 2023-09-23T23:24:05.740 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:05.769 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:05.823 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:05.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:05.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:05.927 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:05.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:05 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:24:05.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:05 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:24:05.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:05 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:05.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:05 smithi138 ceph-mon[193606]: 1.e scrub starts 2023-09-23T23:24:05.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:05 smithi138 ceph-mon[193606]: 1.e scrub ok 2023-09-23T23:24:05.968 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:05.999 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:24:06.051 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:06.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:06.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:06.120 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 14 16 29160 29144 130.03 80.4375 0.00600211 0.00768463 2023-09-23T23:24:06.160 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:24:06.196 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:06.272 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:24:06.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:06.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:24:06.355 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:24:06.380 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:24:06.417 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:06.480 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:24:06.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:06.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:24:06.586 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:06.643 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:06 smithi110 ceph-mon[221102]: 1.13 scrub starts 2023-09-23T23:24:06.643 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:06 smithi110 ceph-mon[221102]: 1.13 scrub ok 2023-09-23T23:24:06.643 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:06 smithi110 ceph-mon[221102]: pgmap v1042: 119 pgs: 119 active+clean; 1.6 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-09-23T23:24:06.643 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:06 smithi110 ceph-mon[212939]: 1.13 scrub starts 2023-09-23T23:24:06.644 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:06 smithi110 ceph-mon[212939]: 1.13 scrub ok 2023-09-23T23:24:06.644 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:06 smithi110 ceph-mon[212939]: pgmap v1042: 119 pgs: 119 active+clean; 1.6 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-09-23T23:24:06.647 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:06.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:06 smithi138 ceph-mon[193606]: 1.13 scrub starts 2023-09-23T23:24:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:06 smithi138 ceph-mon[193606]: 1.13 scrub ok 2023-09-23T23:24:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:06 smithi138 ceph-mon[193606]: pgmap v1042: 119 pgs: 119 active+clean; 1.6 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-09-23T23:24:06.953 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:24:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:24:06] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.43.0" 2023-09-23T23:24:07.120 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 15 16 30593 30577 127.332 89.5625 0.00745249 0.00784903 2023-09-23T23:24:07.156 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:07.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:07.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:07.232 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:07.262 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:07.297 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:07.352 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:07.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:07.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:07.440 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:07.458 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:07.520 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:07.668 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:07.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:07.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:07.772 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:24:07.799 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:08.076 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:24:08.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:08.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:24:08.120 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 16 16 31993 31977 124.843 87.5 0.00444578 0.00800527 2023-09-23T23:24:08.182 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:08.207 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:08.218 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:08.273 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:08.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:08.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:08.379 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:24:08.413 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:08.481 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:24:08.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:08.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:24:08.589 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:24:08.646 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:08.848 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:08 smithi110 ceph-mon[212939]: pgmap v1043: 119 pgs: 119 active+clean; 1.6 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T23:24:08.849 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:08 smithi110 ceph-mon[221102]: pgmap v1043: 119 pgs: 119 active+clean; 1.6 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T23:24:08.866 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:24:08.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:08.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:24:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:08 smithi138 ceph-mon[193606]: pgmap v1043: 119 pgs: 119 active+clean; 1.6 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T23:24:08.977 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:09.041 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:09.120 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 17 16 33722 33706 123.856 108.062 0.0124196 0.00806663 2023-09-23T23:24:09.148 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:09.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:09.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:09.255 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:09.309 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:09.702 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:24:09.767 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:09.770 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:09 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/479166197' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:24:09.770 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:09 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:24:09.771 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:09 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/479166197' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:24:09.771 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:09 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:24:09.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:09.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:09.872 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:09.919 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:09 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/479166197' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:24:09.920 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:09 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:24:09.926 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:09.995 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:10.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:10.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:10.104 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:10.121 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 18 16 35498 35482 123.141 111 0.00653807 0.00811621 2023-09-23T23:24:10.144 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:10.216 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:10.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:10.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:10.325 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:10.351 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:10.402 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:10.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:10.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:10.508 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:24:10.548 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:10.599 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:24:10.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:10.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:24:10.705 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:10.746 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:10.831 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:10.837 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:10 smithi138 ceph-mon[193606]: pgmap v1044: 119 pgs: 119 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-09-23T23:24:10.837 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:10 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:24:10.837 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:10 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:24:10.837 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:10 smithi138 ceph-mon[193606]: osdmap e356: 8 total, 8 up, 8 in 2023-09-23T23:24:10.837 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:10 smithi138 ceph-mon[193606]: failed to encode map e356 with expected crc 2023-09-23T23:24:10.837 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:10 smithi138 ceph-mon[193606]: failed to encode map e356 with expected crc 2023-09-23T23:24:10.837 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:10 smithi138 ceph-mon[193606]: failed to encode map e356 with expected crc 2023-09-23T23:24:10.837 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:10 smithi138 ceph-mon[193606]: failed to encode map e356 with expected crc 2023-09-23T23:24:10.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:10.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:10.939 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:10.951 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:10 smithi110 ceph-mon[212939]: pgmap v1044: 119 pgs: 119 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-09-23T23:24:10.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:10 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:24:10.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:10 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:24:10.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:10 smithi110 ceph-mon[212939]: osdmap e356: 8 total, 8 up, 8 in 2023-09-23T23:24:10.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:10 smithi110 ceph-mon[212939]: failed to encode map e356 with expected crc 2023-09-23T23:24:10.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:10 smithi110 ceph-mon[212939]: failed to encode map e356 with expected crc 2023-09-23T23:24:10.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:10 smithi110 ceph-mon[212939]: failed to encode map e356 with expected crc 2023-09-23T23:24:10.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:10 smithi110 ceph-mon[212939]: failed to encode map e356 with expected crc 2023-09-23T23:24:10.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:10 smithi110 ceph-mon[221102]: pgmap v1044: 119 pgs: 119 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-09-23T23:24:10.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:10 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:24:10.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:10 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:24:10.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:10 smithi110 ceph-mon[221102]: osdmap e356: 8 total, 8 up, 8 in 2023-09-23T23:24:10.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:10 smithi110 ceph-mon[221102]: failed to encode map e356 with expected crc 2023-09-23T23:24:10.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:10 smithi110 ceph-mon[221102]: failed to encode map e356 with expected crc 2023-09-23T23:24:10.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:10 smithi110 ceph-mon[221102]: failed to encode map e356 with expected crc 2023-09-23T23:24:10.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:10 smithi110 ceph-mon[221102]: failed to encode map e356 with expected crc 2023-09-23T23:24:10.969 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:11.121 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 19 16 36849 36833 121.102 84.4375 0.00537732 0.00825265 2023-09-23T23:24:11.506 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:11.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:11.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:11.618 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:11.668 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:11.743 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:11.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:11.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:11.851 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:11.889 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:11.964 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:11.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:11.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:12.071 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:12.123 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:24:12.123342+0000 min lat: 0.00111746 max lat: 0.34576 avg lat: 0.0083001 2023-09-23T23:24:12.124 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:24:12.124 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 20 16 38566 38550 120.397 107.312 0.0127736 0.0083001 2023-09-23T23:24:12.158 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:12.560 INFO:teuthology.orchestra.run.smithi110.stderr:Error EINVAL: specified pg_num 41 < pg_num_max 32 2023-09-23T23:24:12.888 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:12.898 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:12 smithi110 ceph-mon[221102]: pgmap v1046: 119 pgs: 119 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-09-23T23:24:12.898 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:12 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1470426673' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T23:24:12.898 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:12 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T23:24:12.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:12.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:12.900 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:12 smithi110 ceph-mon[212939]: pgmap v1046: 119 pgs: 119 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-09-23T23:24:12.900 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:12 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1470426673' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T23:24:12.900 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:12 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T23:24:12.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:12 smithi138 ceph-mon[193606]: pgmap v1046: 119 pgs: 119 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-09-23T23:24:12.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:12 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1470426673' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T23:24:12.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:12 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T23:24:13.000 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:24:13.029 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:13.124 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 21 16 39687 39671 118.001 70.0625 0.0023155 0.00841364 2023-09-23T23:24:13.685 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:24:13.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:13.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:24:13.794 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:13.826 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T23:24:13.828 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:13.905 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:13.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:13.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:14.014 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:24:14.072 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:14.124 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 22 16 41102 41086 116.657 88.4375 0.013679 0.00856592 2023-09-23T23:24:14.136 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:24:14.177 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:24:14.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:14.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:24:14.220 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:24:14.244 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:24:14.248 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:14.248 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:24:14.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:14.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:24:14.260 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:14.260 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:24:14.265 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:24:14.302 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:24:14.337 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:24:14.351 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:24:14.370 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:24:14.387 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:14.405 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:24:14.414 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:24:14.415 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:24:14.415 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:24:14.415 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:24:14.415 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:24:14.415 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:24:14.415 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:24:14.415 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:24:14.461 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:24:14.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:14.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:24:14.550 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:24:14.550 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:24:14.567 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:14.606 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:14.622 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:24:14.669 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:14.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:14.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:14.776 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:14.885 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:15.124 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 23 16 42564 42548 115.558 91.375 0.00750523 0.00864855 2023-09-23T23:24:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:14 smithi110 ceph-mon[212939]: pgmap v1047: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-09-23T23:24:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:14 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3696902535' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:15.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:14 smithi110 ceph-mon[221102]: pgmap v1047: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-09-23T23:24:15.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:14 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3696902535' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:15.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:14 smithi138 ceph-mon[193606]: pgmap v1047: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-09-23T23:24:15.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:14 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3696902535' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:15.698 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:15.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:15.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:15.808 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:24:15.836 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:16.125 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 24 16 43800 43784 113.962 77.25 0.00264592 0.00876965 2023-09-23T23:24:16.306 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:24:16.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:16.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:24:16.354 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:24:16.418 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:16.456 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:16.523 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:16.529 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:16.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:16.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:16.648 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:24:16.715 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:16.829 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:24:16.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:16.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:24:16.941 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:24:16.976 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:17.006 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:24:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:24:16] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.43.0" 2023-09-23T23:24:17.047 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:24:17.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:17.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:24:17.125 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 25 16 45238 45222 112.996 89.875 0.00516503 0.00884467 2023-09-23T23:24:17.154 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:24:17.226 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:17.326 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:24:17.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:17.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:24:17.334 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:17 smithi110 ceph-mon[221102]: pgmap v1048: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T23:24:17.335 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:17 smithi110 ceph-mon[212939]: pgmap v1048: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T23:24:17.434 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:17 smithi138 ceph-mon[193606]: pgmap v1048: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T23:24:17.505 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:17.591 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:17.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:17.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:17.695 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:17.721 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:17.762 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:17.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:17.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:17.866 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:24:17.928 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:18.126 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 26 16 46409 46393 111.459 73.1875 0.0140191 0.00896682 2023-09-23T23:24:18.206 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:18.234 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:24:18.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:18.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:24:18.328 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:24:18.359 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:24:18.450 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:18.786 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:24:18.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:18.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:24:18.827 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 2, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:24:18.828 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:24:18.828 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T23:24:18.828 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T23:24:18.828 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T23:24:18.828 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T23:24:18.895 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:18.935 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:19.023 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:19.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:19.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:19.126 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 27 16 47694 47678 110.305 80.3125 0.0149478 0.00906135 2023-09-23T23:24:19.132 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:19.172 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:19 smithi138 ceph-mon[193606]: pgmap v1049: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T23:24:19.177 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:19.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:19 smithi110 ceph-mon[212939]: pgmap v1049: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T23:24:19.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:19 smithi110 ceph-mon[221102]: pgmap v1049: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T23:24:19.256 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:19.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:19.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:19.361 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:19.378 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:24:19.422 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:19.502 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:19.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:19.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:19.606 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:24:19.635 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:19.722 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:24:19.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:19.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:24:19.826 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:19.878 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:19.947 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:19.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:19.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:20.042 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:24:20.054 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:20.103 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:20.129 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 28 16 48832 48816 108.899 71.125 0.0155538 0.00917666 2023-09-23T23:24:20.168 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:20.170 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:20 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3964530470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:24:20.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:20.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:20.272 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:24:20.312 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:20.361 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:24:20.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:20.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:24:20.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:20 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3964530470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:24:20.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:20 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3964530470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:24:20.464 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:24:20.489 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:20.532 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:24:20.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:20.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:24:20.635 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:24:20.677 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:20.756 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:24:20.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:20.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:24:20.860 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:20.901 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:20.957 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:20.973 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:20.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:20.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:21.078 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:24:21.137 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 29 16 49982 49966 107.596 71.875 0.00769296 0.00928841 2023-09-23T23:24:21.143 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:21.328 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:21 smithi110 ceph-mon[221102]: pgmap v1050: 119 pgs: 119 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T23:24:21.328 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:21 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:24:21.329 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:21 smithi110 ceph-mon[221102]: failed to encode map e357 with expected crc 2023-09-23T23:24:21.329 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:21 smithi110 ceph-mon[221102]: failed to encode map e357 with expected crc 2023-09-23T23:24:21.329 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:21 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3964530470' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:24:21.329 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:21 smithi110 ceph-mon[221102]: osdmap e357: 8 total, 8 up, 8 in 2023-09-23T23:24:21.330 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:21 smithi110 ceph-mon[212939]: pgmap v1050: 119 pgs: 119 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T23:24:21.330 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:21 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:24:21.330 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:21 smithi110 ceph-mon[212939]: failed to encode map e357 with expected crc 2023-09-23T23:24:21.330 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:21 smithi110 ceph-mon[212939]: failed to encode map e357 with expected crc 2023-09-23T23:24:21.330 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:21 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3964530470' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:24:21.330 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:21 smithi110 ceph-mon[212939]: osdmap e357: 8 total, 8 up, 8 in 2023-09-23T23:24:21.441 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:24:21.441 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:24:21.442 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:24:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:21 smithi138 ceph-mon[193606]: pgmap v1050: 119 pgs: 119 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T23:24:21.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:21 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:24:21.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:21 smithi138 ceph-mon[193606]: failed to encode map e357 with expected crc 2023-09-23T23:24:21.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:21 smithi138 ceph-mon[193606]: failed to encode map e357 with expected crc 2023-09-23T23:24:21.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:21 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3964530470' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:24:21.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:21 smithi138 ceph-mon[193606]: osdmap e357: 8 total, 8 up, 8 in 2023-09-23T23:24:21.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:21.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:24:21.551 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:21.603 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:21.684 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:21.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:21.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:21.787 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:24:21.821 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:21.880 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:24:21.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:21.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:24:21.985 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:22.048 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:22.136 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 30 16 51353 51337 106.866 85.6875 0.0128346 0.00935305 2023-09-23T23:24:22.966 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:24:23.141 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 31 16 52265 52249 105.242 57 0.00555825 0.00949583 2023-09-23T23:24:23.170 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:23.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:23.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:23.197 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:24:23.275 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:23.320 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:23 smithi138 ceph-mon[193606]: pgmap v1052: 119 pgs: 119 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T23:24:23.323 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:23.422 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:23.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:23.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:23.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:23 smithi110 ceph-mon[212939]: pgmap v1052: 119 pgs: 119 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T23:24:23.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:23 smithi110 ceph-mon[221102]: pgmap v1052: 119 pgs: 119 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T23:24:23.528 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:23.604 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:23.714 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:23.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:23.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:23.822 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:23.847 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:23.880 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:23.972 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:24:23.994 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:24:24.013 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:24:24.032 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:24:24.060 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:24:24.097 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:24:24.132 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:24:24.141 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 32 16 53354 53338 104.081 68.0625 0.0160106 0.00960338 2023-09-23T23:24:24.172 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:24:24.190 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:24:24.190 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:24:24.190 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:24:24.190 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:24:24.190 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:24:24.190 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:24:24.190 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:24:24.190 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:24:24.414 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:24.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:24.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:24.518 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:24:24.573 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:24.638 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:24:24.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:24 smithi138 ceph-mon[193606]: pgmap v1053: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T23:24:24.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:24 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3204391803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:24.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:24 smithi110 ceph-mon[212939]: pgmap v1053: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T23:24:24.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:24 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3204391803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:24.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:24 smithi110 ceph-mon[221102]: pgmap v1053: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T23:24:24.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:24 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3204391803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:25.141 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 33 16 54663 54647 103.406 81.8125 0.0122633 0.00965066 2023-09-23T23:24:25.173 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:24:25.174 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T23:24:25.174 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:24:25.330 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:24:25.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:25.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:24:25.433 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:25.497 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:26.141 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 34 16 55486 55470 101.878 51.4375 0.00493057 0.00981164 2023-09-23T23:24:26.634 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:26.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:26.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:26.738 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:24:26.782 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:26.806 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:26 smithi110 ceph-mon[221102]: pgmap v1054: 119 pgs: 119 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T23:24:26.807 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:26 smithi110 ceph-mon[212939]: pgmap v1054: 119 pgs: 119 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T23:24:26.807 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:24:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:24:26] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.43.0" 2023-09-23T23:24:26.853 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:24:26.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:26.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:24:26.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:26 smithi138 ceph-mon[193606]: pgmap v1054: 119 pgs: 119 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T23:24:26.963 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:27.005 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:27.141 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 35 16 56697 56681 101.131 75.6875 0.0130623 0.0098832 2023-09-23T23:24:27.195 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:27.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:27.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:27.301 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:27.370 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:27.421 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:24:27.421 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:24:27.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:27.421 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:57076 11.7 11:ec08d171:::benchmark_data_smithi110_433746_object57075:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-09-23T23:24:27.421 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:24:27.414906+0000", 2023-09-23T23:24:27.421 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0047184599999999998, 2023-09-23T23:24:27.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0047553700000000001, 2023-09-23T23:24:27.421 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:27.421 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:24:27.421 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:27.422 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:24:27.422 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:24:27.422 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 57076 2023-09-23T23:24:27.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:27.422 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:27.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:27.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:27.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:24:27.414906+0000", 2023-09-23T23:24:27.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:27.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:27.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:27.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:27.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:24:27.414906+0000", 2023-09-23T23:24:27.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:27.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:27.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:27.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:27.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:24:27.414904+0000", 2023-09-23T23:24:27.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:24:27.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:27.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:27.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:27.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:24:27.414954+0000", 2023-09-23T23:24:27.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9314000000000003e-05 2023-09-23T23:24:27.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:27.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:27.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:27.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:24:27.414970+0000", 2023-09-23T23:24:27.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6099e-05 2023-09-23T23:24:27.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:27.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:27.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:27.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:24:27.414976+0000", 2023-09-23T23:24:27.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6209999999999998e-06 2023-09-23T23:24:27.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:27.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:27.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:27.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:24:27.415022+0000", 2023-09-23T23:24:27.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5986999999999999e-05 2023-09-23T23:24:27.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:27.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:27.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:27.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:24:27.415179+0000", 2023-09-23T23:24:27.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000157143 2023-09-23T23:24:27.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:27.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:27.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:24:27.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:24:27.415339+0000", 2023-09-23T23:24:27.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000159421 2023-09-23T23:24:27.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:27.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:27.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:27.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:24:27.417544+0000", 2023-09-23T23:24:27.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002204658 2023-09-23T23:24:27.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:27.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:27.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:27.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:24:27.418123+0000", 2023-09-23T23:24:27.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00057894100000000003 2023-09-23T23:24:27.426 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:27.426 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:27.426 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:27.427 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:27.427 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-09-23T23:24:27.427 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 1 2023-09-23T23:24:27.427 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:24:27.478 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:27.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:27.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:27.584 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:24:27.639 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:27.715 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:24:27.747 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:24:27.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:27.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:24:27.854 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:24:27.880 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:27 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/918691340' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:24:27.880 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:27 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:24:27.882 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:27.913 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:27 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/918691340' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:24:27.913 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:27 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:24:27.914 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:27 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/918691340' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:24:27.914 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:27 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:24:27.926 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:24:27.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:27.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:24:28.030 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:28.074 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:28.121 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:24:28.141 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 36 16 57761 57745 100.169 66.5 0.0174198 0.00997721 2023-09-23T23:24:28.181 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:28.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:28.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:28.287 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:28.349 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:28.453 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:24:28.679 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:28.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:28.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:28 smithi138 ceph-mon[193606]: pgmap v1055: 119 pgs: 119 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T23:24:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:28 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:24:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:28 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:24:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:28 smithi138 ceph-mon[193606]: osdmap e358: 8 total, 8 up, 8 in 2023-09-23T23:24:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:28 smithi138 ceph-mon[193606]: failed to encode map e358 with expected crc 2023-09-23T23:24:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:28 smithi138 ceph-mon[193606]: failed to encode map e358 with expected crc 2023-09-23T23:24:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:28 smithi138 ceph-mon[193606]: failed to encode map e358 with expected crc 2023-09-23T23:24:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:24:28.789 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:28.855 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:28.898 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:24:28.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:28 smithi110 ceph-mon[212939]: pgmap v1055: 119 pgs: 119 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T23:24:28.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:28 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:24:28.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:28 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:24:28.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:28 smithi110 ceph-mon[212939]: osdmap e358: 8 total, 8 up, 8 in 2023-09-23T23:24:28.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:28 smithi110 ceph-mon[212939]: failed to encode map e358 with expected crc 2023-09-23T23:24:28.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:28 smithi110 ceph-mon[212939]: failed to encode map e358 with expected crc 2023-09-23T23:24:28.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:28 smithi110 ceph-mon[212939]: failed to encode map e358 with expected crc 2023-09-23T23:24:28.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:24:28.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:28 smithi110 ceph-mon[221102]: pgmap v1055: 119 pgs: 119 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T23:24:28.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:28 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:24:28.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:28 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:24:28.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:28 smithi110 ceph-mon[221102]: osdmap e358: 8 total, 8 up, 8 in 2023-09-23T23:24:28.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:28 smithi110 ceph-mon[221102]: failed to encode map e358 with expected crc 2023-09-23T23:24:28.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:28 smithi110 ceph-mon[221102]: failed to encode map e358 with expected crc 2023-09-23T23:24:28.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:28 smithi110 ceph-mon[221102]: failed to encode map e358 with expected crc 2023-09-23T23:24:28.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:24:29.010 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:29.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:29.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:29.118 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:29.141 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 37 16 58684 58668 99.0216 57.6875 0.0114895 0.0100931 2023-09-23T23:24:29.176 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:29.204 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:29.300 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:29.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:29.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:29.408 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:24:29.462 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:29.536 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:24:29.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:29.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:24:29.646 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:24:29.698 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:29.755 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:24:29.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:29.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:24:29.859 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:24:29.887 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:30.051 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:24:30.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:30.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:24:30.141 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 38 16 59710 59694 98.1038 64.125 0.0071142 0.0101891 2023-09-23T23:24:30.163 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:30.207 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:30.361 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:30.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:30.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:30.468 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:24:30.537 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:30.821 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:30 smithi110 ceph-mon[212939]: pgmap v1057: 119 pgs: 119 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T23:24:30.821 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:30 smithi110 ceph-mon[221102]: pgmap v1057: 119 pgs: 119 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T23:24:30.842 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:24:30.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:30.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:24:30.856 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:24:30.856 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:24:30.856 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:24:30.856 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:24:30.857 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:24:30.884 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:30 smithi138 ceph-mon[193606]: pgmap v1057: 119 pgs: 119 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T23:24:30.954 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:24:31.010 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:31.028 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:24:31.081 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:24:31.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:31.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:24:31.148 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 39 16 60707 60691 97.1712 62.3125 0.0203898 0.0102861 2023-09-23T23:24:31.185 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:31.245 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:31.348 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:31.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:31.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:31.452 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:31.512 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:31.515 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:24:31.551 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:24:31.577 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:31.597 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:24:31.634 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 2, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:24:31.634 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:24:31.635 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T23:24:31.635 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:24:31.654 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:24:31.666 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:31.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:31.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:31.691 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:24:31.729 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:24:31.776 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:24:31.783 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:24:31.798 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:24:31.820 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:24:31.822 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:24:31.822 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:24:31.822 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:24:31.822 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:24:31.822 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:24:31.822 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:24:31.823 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:24:31.823 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:24:31.882 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:31.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:31 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/579111755' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:31.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:31 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/579111755' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:32.095 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:24:32.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:32.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:24:32.151 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:24:32.150680+0000 min lat: 0.00111746 max lat: 0.34576 avg lat: 0.0104083 2023-09-23T23:24:32.151 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:24:32.151 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 40 16 61536 61520 96.0308 51.8125 0.0131251 0.0104083 2023-09-23T23:24:32.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:31 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/579111755' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:32.204 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:32.265 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:32.312 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:24:32.357 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:32.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:32.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:32.469 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:32.538 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:32.628 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:32.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:32.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:32.697 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:32.733 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:32.770 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:32.866 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:32.867 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:32 smithi138 ceph-mon[193606]: pgmap v1058: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T23:24:32.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:32.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:32.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:32 smithi110 ceph-mon[212939]: pgmap v1058: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T23:24:32.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:32 smithi110 ceph-mon[221102]: pgmap v1058: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T23:24:32.969 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:24:33.019 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:33.120 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:24:33.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:33.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:24:33.151 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 41 16 62712 62696 95.4816 73.5 0.0108986 0.0104681 2023-09-23T23:24:33.226 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:33.267 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:33.320 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:33.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:33.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:33.423 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:24:33.455 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:33.978 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:24:33.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:33.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:24:34.091 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:34.130 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:34.151 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 42 16 63726 63710 94.7176 63.375 0.012497 0.0105532 2023-09-23T23:24:34.214 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:34.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:34.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:34.322 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:24:34.372 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:34.457 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:24:34.457 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:24:34.460 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:24:34.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:34.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:24:34.524 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:34 smithi110 ceph-mon[212939]: pgmap v1059: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-09-23T23:24:34.528 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:34 smithi110 ceph-mon[221102]: pgmap v1059: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-09-23T23:24:34.566 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:24:34.617 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:34.705 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:24:34.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:34.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:24:34.813 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:24:34.840 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:34 smithi138 ceph-mon[193606]: pgmap v1059: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-09-23T23:24:34.842 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:34.883 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:24:34.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:34.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:24:34.987 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:24:35.011 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:35.051 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:24:35.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:35.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:24:35.151 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 43 16 64661 64645 93.8745 58.4375 0.00416425 0.0106393 2023-09-23T23:24:35.154 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:35.205 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:35.502 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:35.503 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:24:35.506 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:24:35.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:35.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:35.610 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:35.683 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:35.789 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:35.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:35.798 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:35 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2250309845' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:24:35.799 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:35 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:24:35.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:35.900 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:24:35.924 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:35 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2250309845' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:24:35.924 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:35 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:24:35.924 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:35 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2250309845' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:24:35.924 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:35 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:24:35.927 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:35.933 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:24:35.967 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:24:36.151 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 44 16 65616 65600 93.0973 59.6875 0.0126157 0.0107367 2023-09-23T23:24:36.354 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:24:36.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:36.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:24:36.461 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:36.541 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:36.652 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:24:36.652 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:24:36.653 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:24:36.653 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[212939]: osdmap e359: 8 total, 8 up, 8 in 2023-09-23T23:24:36.653 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[212939]: failed to encode map e359 with expected crc 2023-09-23T23:24:36.653 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[212939]: failed to encode map e359 with expected crc 2023-09-23T23:24:36.653 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[212939]: failed to encode map e359 with expected crc 2023-09-23T23:24:36.653 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[212939]: failed to encode map e359 with expected crc 2023-09-23T23:24:36.653 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[212939]: pgmap v1061: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T23:24:36.653 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:24:36.653 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:24:36.654 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:24:36.654 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[221102]: osdmap e359: 8 total, 8 up, 8 in 2023-09-23T23:24:36.654 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[221102]: failed to encode map e359 with expected crc 2023-09-23T23:24:36.654 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[221102]: failed to encode map e359 with expected crc 2023-09-23T23:24:36.654 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[221102]: failed to encode map e359 with expected crc 2023-09-23T23:24:36.654 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[221102]: failed to encode map e359 with expected crc 2023-09-23T23:24:36.654 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:36 smithi110 ceph-mon[221102]: pgmap v1061: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T23:24:36.687 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:36.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:36.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:36.801 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:36.830 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:36.861 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:36.877 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:36.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:36.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:36 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:24:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:36 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:24:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:36 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:24:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:36 smithi138 ceph-mon[193606]: osdmap e359: 8 total, 8 up, 8 in 2023-09-23T23:24:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:36 smithi138 ceph-mon[193606]: failed to encode map e359 with expected crc 2023-09-23T23:24:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:36 smithi138 ceph-mon[193606]: failed to encode map e359 with expected crc 2023-09-23T23:24:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:36 smithi138 ceph-mon[193606]: failed to encode map e359 with expected crc 2023-09-23T23:24:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:36 smithi138 ceph-mon[193606]: failed to encode map e359 with expected crc 2023-09-23T23:24:36.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:36 smithi138 ceph-mon[193606]: pgmap v1061: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T23:24:36.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:24:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:24:36] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.43.0" 2023-09-23T23:24:36.981 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:24:37.008 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:37.131 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:24:37.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:37.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:24:37.152 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 45 16 66443 66427 92.176 51.6875 0.00692014 0.0108447 2023-09-23T23:24:37.177 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:24:37.195 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:24:37.241 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:37.283 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:37.448 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:37.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:37.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:37.558 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:37.612 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:37 smithi110 ceph-mon[212939]: from='client.45959 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:24:37.612 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:37 smithi110 ceph-mon[221102]: from='client.45959 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:24:37.617 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:37.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:37 smithi138 ceph-mon[193606]: from='client.45959 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:24:38.035 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:24:38.035 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:24:38.152 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 46 16 67521 67505 91.6372 67.375 0.0120415 0.0109079 2023-09-23T23:24:38.471 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:24:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:38 smithi110 ceph-mon[212939]: pgmap v1062: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T23:24:38.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:38 smithi110 ceph-mon[221102]: pgmap v1062: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T23:24:38.788 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:38.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:38.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:38.841 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1061,"stamp":"2023-09-23T23:24:35.604866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4181989280,"num_objects":63827,"num_object_clones":0,"num_object_copies":191481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":63864,"num_write_kb":4083421,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":1220,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":68849,"ondisk_log_size":68849,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":360,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16470708,"kb_used_data":12615988,"kb_used_omap":29,"kb_used_meta":3854626,"kb_avail":733490508,"statfs":{"total":767960285184,"available":751094280192,"internally_reserved":0,"allocated":12918771712,"data_stored":12908314829,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":3947137747},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":679608320,"num_objects":10370,"num_object_clones":0,"num_object_copies":31110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10370,"num_write_kb":663680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003607"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":624,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.039149+0000","last_change":"2023-09-23T23:23:58.058868+0000","last_active":"2023-09-23T23:24:20.039149+0000","last_peered":"2023-09-23T23:24:20.039149+0000","last_clean":"2023-09-23T23:24:20.039149+0000","last_became_active":"2023-09-23T23:23:45.456902+0000","last_became_peered":"2023-09-23T23:23:45.456902+0000","last_unstale":"2023-09-23T23:24:20.039149+0000","last_undegraded":"2023-09-23T23:24:20.039149+0000","last_fullsized":"2023-09-23T23:24:20.039149+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.058755+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.058755+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:52.935253+0000","scrub_duration":0.0039029870000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":392,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.044157+0000","last_change":"2023-09-23T23:22:38.439629+0000","last_active":"2023-09-23T23:24:20.044157+0000","last_peered":"2023-09-23T23:24:20.044157+0000","last_clean":"2023-09-23T23:24:20.044157+0000","last_became_active":"2023-09-23T23:22:38.439285+0000","last_became_peered":"2023-09-23T23:22:38.439285+0000","last_unstale":"2023-09-23T23:24:20.044157+0000","last_undegraded":"2023-09-23T23:24:20.044157+0000","last_fullsized":"2023-09-23T23:24:20.044157+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:34.431221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":326,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.037886+0000","last_change":"2023-09-23T23:23:43.511912+0000","last_active":"2023-09-23T23:24:20.037886+0000","last_peered":"2023-09-23T23:24:20.037886+0000","last_clean":"2023-09-23T23:24:20.037886+0000","last_became_active":"2023-09-23T23:23:43.511574+0000","last_became_peered":"2023-09-23T23:23:43.511574+0000","last_unstale":"2023-09-23T23:24:20.037886+0000","last_undegraded":"2023-09-23T23:24:20.037886+0000","last_fullsized":"2023-09-23T23:24:20.037886+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:16.286926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":447,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.033212+0000","last_change":"2023-09-23T23:23:50.459634+0000","last_active":"2023-09-23T23:24:20.033212+0000","last_peered":"2023-09-23T23:24:20.033212+0000","last_clean":"2023-09-23T23:24:20.033212+0000","last_became_active":"2023-09-23T23:23:45.453958+0000","last_became_peered":"2023-09-23T23:23:45.453958+0000","last_unstale":"2023-09-23T23:24:20.033212+0000","last_undegraded":"2023-09-23T23:24:20.033212+0000","last_fullsized":"2023-09-23T23:24:20.033212+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:56.563201+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.035591+0000","last_change":"2023-09-23T23:22:38.436383+0000","last_active":"2023-09-23T23:24:20.035591+0000","last_peered":"2023-09-23T23:24:20.035591+0000","last_clean":"2023-09-23T23:24:20.035591+0000","last_became_active":"2023-09-23T23:22:38.436062+0000","last_became_peered":"2023-09-23T23:22:38.436062+0000","last_unstale":"2023-09-23T23:24:20.035591+0000","last_undegraded":"2023-09-23T23:24:20.035591+0000","last_fullsized":"2023-09-23T23:24:20.035591+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:07.142420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":357,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.042207+0000","last_change":"2023-09-23T23:22:38.438239+0000","last_active":"2023-09-23T23:24:20.042207+0000","last_peered":"2023-09-23T23:24:20.042207+0000","last_clean":"2023-09-23T23:24:20.042207+0000","last_became_active":"2023-09-23T23:22:38.437359+0000","last_became_peered":"2023-09-23T23:22:38.437359+0000","last_unstale":"2023-09-23T23:24:20.042207+0000","last_undegraded":"2023-09-23T23:24:20.042207+0000","last_fullsized":"2023-09-23T23:24:20.042207+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:15:40.886267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":667,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.030564+0000","last_change":"2023-09-23T23:23:49.842338+0000","last_active":"2023-09-23T23:24:20.030564+0000","last_peered":"2023-09-23T23:24:20.030564+0000","last_clean":"2023-09-23T23:24:20.030564+0000","last_became_active":"2023-09-23T23:23:45.459515+0000","last_became_peered":"2023-09-23T23:23:45.459515+0000","last_unstale":"2023-09-23T23:24:20.030564+0000","last_undegraded":"2023-09-23T23:24:20.030564+0000","last_fullsized":"2023-09-23T23:24:20.030564+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.842272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.842272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:13:00.500804+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":344,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.041065+0000","last_change":"2023-09-23T23:20:15.738434+0000","last_active":"2023-09-23T23:24:20.041065+0000","last_peered":"2023-09-23T23:24:20.041065+0000","last_clean":"2023-09-23T23:24:20.041065+0000","last_became_active":"2023-09-23T23:20:15.738195+0000","last_became_peered":"2023-09-23T23:20:15.738195+0000","last_unstale":"2023-09-23T23:24:20.041065+0000","last_undegraded":"2023-09-23T23:24:20.041065+0000","last_fullsized":"2023-09-23T23:24:20.041065+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:15.224045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":337,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.034079+0000","last_change":"2023-09-23T23:22:38.436482+0000","last_active":"2023-09-23T23:24:20.034079+0000","last_peered":"2023-09-23T23:24:20.034079+0000","last_clean":"2023-09-23T23:24:20.034079+0000","last_became_active":"2023-09-23T23:22:38.436147+0000","last_became_peered":"2023-09-23T23:22:38.436147+0000","last_unstale":"2023-09-23T23:24:20.034079+0000","last_undegraded":"2023-09-23T23:24:20.034079+0000","last_fullsized":"2023-09-23T23:24:20.034079+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:30.542674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":678,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.030277+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:24:20.030277+0000","last_peered":"2023-09-23T23:24:20.030277+0000","last_clean":"2023-09-23T23:24:20.030277+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:24:20.030277+0000","last_undegraded":"2023-09-23T23:24:20.030277+0000","last_fullsized":"2023-09-23T23:24:20.030277+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":310,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.039768+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:24:20.039768+0000","last_peered":"2023-09-23T23:24:20.039768+0000","last_clean":"2023-09-23T23:24:20.039768+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:24:20.039768+0000","last_undegraded":"2023-09-23T23:24:20.039768+0000","last_fullsized":"2023-09-23T23:24:20.039768+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":326,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.041019+0000","last_change":"2023-09-23T23:22:38.450254+0000","last_active":"2023-09-23T23:24:20.041019+0000","last_peered":"2023-09-23T23:24:20.041019+0000","last_clean":"2023-09-23T23:24:20.041019+0000","last_became_active":"2023-09-23T23:22:38.449879+0000","last_became_peered":"2023-09-23T23:22:38.449879+0000","last_unstale":"2023-09-23T23:24:20.041019+0000","last_undegraded":"2023-09-23T23:24:20.041019+0000","last_fullsized":"2023-09-23T23:24:20.041019+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:02.777916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":612,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.052973+0000","last_change":"2023-09-23T23:23:56.137571+0000","last_active":"2023-09-23T23:24:20.052973+0000","last_peered":"2023-09-23T23:24:20.052973+0000","last_clean":"2023-09-23T23:24:20.052973+0000","last_became_active":"2023-09-23T23:23:45.459314+0000","last_became_peered":"2023-09-23T23:23:45.459314+0000","last_unstale":"2023-09-23T23:24:20.052973+0000","last_undegraded":"2023-09-23T23:24:20.052973+0000","last_fullsized":"2023-09-23T23:24:20.052973+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:14.751052+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":99,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.050818+0000","last_change":"2023-09-23T23:20:26.841909+0000","last_active":"2023-09-23T23:24:20.050818+0000","last_peered":"2023-09-23T23:24:20.050818+0000","last_clean":"2023-09-23T23:24:20.050818+0000","last_became_active":"2023-09-23T23:20:26.841631+0000","last_became_peered":"2023-09-23T23:20:26.841631+0000","last_unstale":"2023-09-23T23:24:20.050818+0000","last_undegraded":"2023-09-23T23:24:20.050818+0000","last_fullsized":"2023-09-23T23:24:20.050818+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:07.380539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":287,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.037762+0000","last_change":"2023-09-23T23:16:56.528057+0000","last_active":"2023-09-23T23:24:20.037762+0000","last_peered":"2023-09-23T23:24:20.037762+0000","last_clean":"2023-09-23T23:24:20.037762+0000","last_became_active":"2023-09-23T23:16:56.527755+0000","last_became_peered":"2023-09-23T23:16:56.527755+0000","last_unstale":"2023-09-23T23:24:20.037762+0000","last_undegraded":"2023-09-23T23:24:20.037762+0000","last_fullsized":"2023-09-23T23:24:20.037762+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:47.396406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":785,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.035383+0000","last_change":"2023-09-23T23:23:54.095122+0000","last_active":"2023-09-23T23:24:20.035383+0000","last_peered":"2023-09-23T23:24:20.035383+0000","last_clean":"2023-09-23T23:24:20.035383+0000","last_became_active":"2023-09-23T23:23:45.458110+0000","last_became_peered":"2023-09-23T23:23:45.458110+0000","last_unstale":"2023-09-23T23:24:20.035383+0000","last_undegraded":"2023-09-23T23:24:20.035383+0000","last_fullsized":"2023-09-23T23:24:20.035383+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:54.095054+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:54.095054+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:50.195093+0000","scrub_duration":0.0020267950000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":176,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.038064+0000","last_change":"2023-09-23T23:16:56.567128+0000","last_active":"2023-09-23T23:24:20.038064+0000","last_peered":"2023-09-23T23:24:20.038064+0000","last_clean":"2023-09-23T23:24:20.038064+0000","last_became_active":"2023-09-23T23:16:56.551484+0000","last_became_peered":"2023-09-23T23:16:56.551484+0000","last_unstale":"2023-09-23T23:24:20.038064+0000","last_undegraded":"2023-09-23T23:24:20.038064+0000","last_fullsized":"2023-09-23T23:24:20.038064+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:14.981307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":307,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.032647+0000","last_change":"2023-09-23T23:20:15.781637+0000","last_active":"2023-09-23T23:24:20.032647+0000","last_peered":"2023-09-23T23:24:20.032647+0000","last_clean":"2023-09-23T23:24:20.032647+0000","last_became_active":"2023-09-23T23:20:15.781308+0000","last_became_peered":"2023-09-23T23:20:15.781308+0000","last_unstale":"2023-09-23T23:24:20.032647+0000","last_undegraded":"2023-09-23T23:24:20.032647+0000","last_fullsized":"2023-09-23T23:24:20.032647+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:47.963872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":431,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.035709+0000","last_change":"2023-09-23T23:24:02.268079+0000","last_active":"2023-09-23T23:24:20.035709+0000","last_peered":"2023-09-23T23:24:20.035709+0000","last_clean":"2023-09-23T23:24:20.035709+0000","last_became_active":"2023-09-23T23:23:45.463106+0000","last_became_peered":"2023-09-23T23:23:45.463106+0000","last_unstale":"2023-09-23T23:24:20.035709+0000","last_undegraded":"2023-09-23T23:24:20.035709+0000","last_fullsized":"2023-09-23T23:24:20.035709+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.128813+0000","scrub_duration":0.0018535310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":73,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.030644+0000","last_change":"2023-09-23T23:23:36.017911+0000","last_active":"2023-09-23T23:24:20.030644+0000","last_peered":"2023-09-23T23:24:20.030644+0000","last_clean":"2023-09-23T23:24:20.030644+0000","last_became_active":"2023-09-23T23:23:36.017507+0000","last_became_peered":"2023-09-23T23:23:36.017507+0000","last_unstale":"2023-09-23T23:24:20.030644+0000","last_undegraded":"2023-09-23T23:24:20.030644+0000","last_fullsized":"2023-09-23T23:24:20.030644+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:49.095779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":336,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.044440+0000","last_change":"2023-09-23T23:22:38.443826+0000","last_active":"2023-09-23T23:24:20.044440+0000","last_peered":"2023-09-23T23:24:20.044440+0000","last_clean":"2023-09-23T23:24:20.044440+0000","last_became_active":"2023-09-23T23:22:38.443494+0000","last_became_peered":"2023-09-23T23:22:38.443494+0000","last_unstale":"2023-09-23T23:24:20.044440+0000","last_undegraded":"2023-09-23T23:24:20.044440+0000","last_fullsized":"2023-09-23T23:24:20.044440+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:55.630019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":510,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.037559+0000","last_change":"2023-09-23T23:23:53.828451+0000","last_active":"2023-09-23T23:24:20.037559+0000","last_peered":"2023-09-23T23:24:20.037559+0000","last_clean":"2023-09-23T23:24:20.037559+0000","last_became_active":"2023-09-23T23:23:45.459682+0000","last_became_peered":"2023-09-23T23:23:45.459682+0000","last_unstale":"2023-09-23T23:24:20.037559+0000","last_undegraded":"2023-09-23T23:24:20.037559+0000","last_fullsized":"2023-09-23T23:24:20.037559+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:53.828311+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:53.828311+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:19.729551+0000","scrub_duration":0.0047553140000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":258,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.030217+0000","last_change":"2023-09-23T23:23:36.026037+0000","last_active":"2023-09-23T23:24:20.030217+0000","last_peered":"2023-09-23T23:24:20.030217+0000","last_clean":"2023-09-23T23:24:20.030217+0000","last_became_active":"2023-09-23T23:23:36.025726+0000","last_became_peered":"2023-09-23T23:23:36.025726+0000","last_unstale":"2023-09-23T23:24:20.030217+0000","last_undegraded":"2023-09-23T23:24:20.030217+0000","last_fullsized":"2023-09-23T23:24:20.030217+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:28:48.287606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":343,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.037931+0000","last_change":"2023-09-23T23:23:36.032487+0000","last_active":"2023-09-23T23:24:20.037931+0000","last_peered":"2023-09-23T23:24:20.037931+0000","last_clean":"2023-09-23T23:24:20.037931+0000","last_became_active":"2023-09-23T23:23:36.032200+0000","last_became_peered":"2023-09-23T23:23:36.032200+0000","last_unstale":"2023-09-23T23:24:20.037931+0000","last_undegraded":"2023-09-23T23:24:20.037931+0000","last_fullsized":"2023-09-23T23:24:20.037931+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:29.843272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":657,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.030179+0000","last_change":"2023-09-23T23:23:50.876882+0000","last_active":"2023-09-23T23:24:20.030179+0000","last_peered":"2023-09-23T23:24:20.030179+0000","last_clean":"2023-09-23T23:24:20.030179+0000","last_became_active":"2023-09-23T23:23:45.457157+0000","last_became_peered":"2023-09-23T23:23:45.457157+0000","last_unstale":"2023-09-23T23:24:20.030179+0000","last_undegraded":"2023-09-23T23:24:20.030179+0000","last_fullsized":"2023-09-23T23:24:20.030179+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:30.732737+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":90,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.042170+0000","last_change":"2023-09-23T23:20:15.781738+0000","last_active":"2023-09-23T23:24:20.042170+0000","last_peered":"2023-09-23T23:24:20.042170+0000","last_clean":"2023-09-23T23:24:20.042170+0000","last_became_active":"2023-09-23T23:20:15.781394+0000","last_became_peered":"2023-09-23T23:20:15.781394+0000","last_unstale":"2023-09-23T23:24:20.042170+0000","last_undegraded":"2023-09-23T23:24:20.042170+0000","last_fullsized":"2023-09-23T23:24:20.042170+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:04.696118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":224,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.030365+0000","last_change":"2023-09-23T23:23:35.033379+0000","last_active":"2023-09-23T23:24:20.030365+0000","last_peered":"2023-09-23T23:24:20.030365+0000","last_clean":"2023-09-23T23:24:20.030365+0000","last_became_active":"2023-09-23T23:23:35.033069+0000","last_became_peered":"2023-09-23T23:23:35.033069+0000","last_unstale":"2023-09-23T23:24:20.030365+0000","last_undegraded":"2023-09-23T23:24:20.030365+0000","last_fullsized":"2023-09-23T23:24:20.030365+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:47.957081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":485,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.036149+0000","last_change":"2023-09-23T23:24:05.304978+0000","last_active":"2023-09-23T23:24:20.036149+0000","last_peered":"2023-09-23T23:24:20.036149+0000","last_clean":"2023-09-23T23:24:20.036149+0000","last_became_active":"2023-09-23T23:23:45.456982+0000","last_became_peered":"2023-09-23T23:23:45.456982+0000","last_unstale":"2023-09-23T23:24:20.036149+0000","last_undegraded":"2023-09-23T23:24:20.036149+0000","last_fullsized":"2023-09-23T23:24:20.036149+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:20.760502+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":202,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.030099+0000","last_change":"2023-09-23T23:23:35.028958+0000","last_active":"2023-09-23T23:24:20.030099+0000","last_peered":"2023-09-23T23:24:20.030099+0000","last_clean":"2023-09-23T23:24:20.030099+0000","last_became_active":"2023-09-23T23:23:35.028478+0000","last_became_peered":"2023-09-23T23:23:35.028478+0000","last_unstale":"2023-09-23T23:24:20.030099+0000","last_undegraded":"2023-09-23T23:24:20.030099+0000","last_fullsized":"2023-09-23T23:24:20.030099+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:44.960119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":337,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.042345+0000","last_change":"2023-09-23T23:20:15.737685+0000","last_active":"2023-09-23T23:24:20.042345+0000","last_peered":"2023-09-23T23:24:20.042345+0000","last_clean":"2023-09-23T23:24:20.042345+0000","last_became_active":"2023-09-23T23:20:15.737311+0000","last_became_peered":"2023-09-23T23:20:15.737311+0000","last_unstale":"2023-09-23T23:24:20.042345+0000","last_undegraded":"2023-09-23T23:24:20.042345+0000","last_fullsized":"2023-09-23T23:24:20.042345+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:15:21.213270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":641,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.031729+0000","last_change":"2023-09-23T23:23:48.533516+0000","last_active":"2023-09-23T23:24:20.031729+0000","last_peered":"2023-09-23T23:24:20.031729+0000","last_clean":"2023-09-23T23:24:20.031729+0000","last_became_active":"2023-09-23T23:23:45.458037+0000","last_became_peered":"2023-09-23T23:23:45.458037+0000","last_unstale":"2023-09-23T23:24:20.031729+0000","last_undegraded":"2023-09-23T23:24:20.031729+0000","last_fullsized":"2023-09-23T23:24:20.031729+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:47.822898+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":346,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.032458+0000","last_change":"2023-09-23T23:13:29.368175+0000","last_active":"2023-09-23T23:24:20.032458+0000","last_peered":"2023-09-23T23:24:20.032458+0000","last_clean":"2023-09-23T23:24:20.032458+0000","last_became_active":"2023-09-23T23:13:29.363603+0000","last_became_peered":"2023-09-23T23:13:29.363603+0000","last_unstale":"2023-09-23T23:24:20.032458+0000","last_undegraded":"2023-09-23T23:24:20.032458+0000","last_fullsized":"2023-09-23T23:24:20.032458+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:25.277871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":48,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.035986+0000","last_change":"2023-09-23T23:22:38.452672+0000","last_active":"2023-09-23T23:24:20.035986+0000","last_peered":"2023-09-23T23:24:20.035986+0000","last_clean":"2023-09-23T23:24:20.035986+0000","last_became_active":"2023-09-23T23:22:38.452374+0000","last_became_peered":"2023-09-23T23:22:38.452374+0000","last_unstale":"2023-09-23T23:24:20.035986+0000","last_undegraded":"2023-09-23T23:24:20.035986+0000","last_fullsized":"2023-09-23T23:24:20.035986+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:29.874430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.7","version":"358'4000","reported_seq":4017,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:34.508972+0000","last_change":"2023-09-23T23:23:43.505723+0000","last_active":"2023-09-23T23:24:34.508972+0000","last_peered":"2023-09-23T23:24:34.508972+0000","last_clean":"2023-09-23T23:24:34.508972+0000","last_became_active":"2023-09-23T23:23:43.505340+0000","last_became_peered":"2023-09-23T23:23:43.505340+0000","last_unstale":"2023-09-23T23:24:34.508972+0000","last_undegraded":"2023-09-23T23:24:34.508972+0000","last_fullsized":"2023-09-23T23:24:34.508972+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4000,"log_dups_size":0,"ondisk_log_size":4000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:33.987175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262144000,"num_objects":4000,"num_object_clones":0,"num_object_copies":12000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4000,"num_write_kb":256000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":679,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.036060+0000","last_change":"2023-09-23T23:23:57.267117+0000","last_active":"2023-09-23T23:24:20.036060+0000","last_peered":"2023-09-23T23:24:20.036060+0000","last_clean":"2023-09-23T23:24:20.036060+0000","last_became_active":"2023-09-23T23:23:45.458275+0000","last_became_peered":"2023-09-23T23:23:45.458275+0000","last_unstale":"2023-09-23T23:24:20.036060+0000","last_undegraded":"2023-09-23T23:24:20.036060+0000","last_fullsized":"2023-09-23T23:24:20.036060+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:47:09.088899+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":171,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.044563+0000","last_change":"2023-09-23T23:16:56.548535+0000","last_active":"2023-09-23T23:24:20.044563+0000","last_peered":"2023-09-23T23:24:20.044563+0000","last_clean":"2023-09-23T23:24:20.044563+0000","last_became_active":"2023-09-23T23:16:56.546115+0000","last_became_peered":"2023-09-23T23:16:56.546115+0000","last_unstale":"2023-09-23T23:24:20.044563+0000","last_undegraded":"2023-09-23T23:24:20.044563+0000","last_fullsized":"2023-09-23T23:24:20.044563+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:02.279715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":162,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.032500+0000","last_change":"2023-09-23T23:16:56.525479+0000","last_active":"2023-09-23T23:24:20.032500+0000","last_peered":"2023-09-23T23:24:20.032500+0000","last_clean":"2023-09-23T23:24:20.032500+0000","last_became_active":"2023-09-23T23:16:56.525147+0000","last_became_peered":"2023-09-23T23:16:56.525147+0000","last_unstale":"2023-09-23T23:24:20.032500+0000","last_undegraded":"2023-09-23T23:24:20.032500+0000","last_fullsized":"2023-09-23T23:24:20.032500+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:06.165300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.6","version":"358'4021","reported_seq":4039,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:35.095549+0000","last_change":"2023-09-23T23:23:43.514932+0000","last_active":"2023-09-23T23:24:35.095549+0000","last_peered":"2023-09-23T23:24:35.095549+0000","last_clean":"2023-09-23T23:24:35.095549+0000","last_became_active":"2023-09-23T23:23:43.513416+0000","last_became_peered":"2023-09-23T23:23:43.513416+0000","last_unstale":"2023-09-23T23:24:35.095549+0000","last_undegraded":"2023-09-23T23:24:35.095549+0000","last_fullsized":"2023-09-23T23:24:35.095549+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4021,"log_dups_size":0,"ondisk_log_size":4021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:41.201456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263520256,"num_objects":4021,"num_object_clones":0,"num_object_copies":12063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4021,"num_write_kb":257344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":702,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:27.682718+0000","last_change":"2023-09-23T23:23:48.906779+0000","last_active":"2023-09-23T23:24:27.682718+0000","last_peered":"2023-09-23T23:24:27.682718+0000","last_clean":"2023-09-23T23:24:27.682718+0000","last_became_active":"2023-09-23T23:23:45.454967+0000","last_became_peered":"2023-09-23T23:23:45.454967+0000","last_unstale":"2023-09-23T23:24:27.682718+0000","last_undegraded":"2023-09-23T23:24:27.682718+0000","last_fullsized":"2023-09-23T23:24:27.682718+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:02.570612+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":52,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.035849+0000","last_change":"2023-09-23T23:22:38.474057+0000","last_active":"2023-09-23T23:24:20.035849+0000","last_peered":"2023-09-23T23:24:20.035849+0000","last_clean":"2023-09-23T23:24:20.035849+0000","last_became_active":"2023-09-23T23:22:38.444672+0000","last_became_peered":"2023-09-23T23:22:38.444672+0000","last_unstale":"2023-09-23T23:24:20.035849+0000","last_undegraded":"2023-09-23T23:24:20.035849+0000","last_fullsized":"2023-09-23T23:24:20.035849+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:00.966999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":363,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.044086+0000","last_change":"2023-09-23T23:22:38.439077+0000","last_active":"2023-09-23T23:24:20.044086+0000","last_peered":"2023-09-23T23:24:20.044086+0000","last_clean":"2023-09-23T23:24:20.044086+0000","last_became_active":"2023-09-23T23:22:38.438638+0000","last_became_peered":"2023-09-23T23:22:38.438638+0000","last_unstale":"2023-09-23T23:24:20.044086+0000","last_undegraded":"2023-09-23T23:24:20.044086+0000","last_fullsized":"2023-09-23T23:24:20.044086+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:57.067702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"358'4042","reported_seq":4059,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:34.494159+0000","last_change":"2023-09-23T23:23:43.514636+0000","last_active":"2023-09-23T23:24:34.494159+0000","last_peered":"2023-09-23T23:24:34.494159+0000","last_clean":"2023-09-23T23:24:34.494159+0000","last_became_active":"2023-09-23T23:23:43.514352+0000","last_became_peered":"2023-09-23T23:23:43.514352+0000","last_unstale":"2023-09-23T23:24:34.494159+0000","last_undegraded":"2023-09-23T23:24:34.494159+0000","last_fullsized":"2023-09-23T23:24:34.494159+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4042,"log_dups_size":0,"ondisk_log_size":4042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:06.752989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264896512,"num_objects":4042,"num_object_clones":0,"num_object_copies":12126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4042,"num_write_kb":258688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":772,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.034004+0000","last_change":"2023-09-23T23:23:57.042667+0000","last_active":"2023-09-23T23:24:20.034004+0000","last_peered":"2023-09-23T23:24:20.034004+0000","last_clean":"2023-09-23T23:24:20.034004+0000","last_became_active":"2023-09-23T23:23:45.454067+0000","last_became_peered":"2023-09-23T23:23:45.454067+0000","last_unstale":"2023-09-23T23:24:20.034004+0000","last_undegraded":"2023-09-23T23:24:20.034004+0000","last_fullsized":"2023-09-23T23:24:20.034004+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.042580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.042580+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:02:09.561802+0000","scrub_duration":0.0044137760000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":236,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.030563+0000","last_change":"2023-09-23T23:23:35.033371+0000","last_active":"2023-09-23T23:24:20.030563+0000","last_peered":"2023-09-23T23:24:20.030563+0000","last_clean":"2023-09-23T23:24:20.030563+0000","last_became_active":"2023-09-23T23:23:35.033032+0000","last_became_peered":"2023-09-23T23:23:35.033032+0000","last_unstale":"2023-09-23T23:24:20.030563+0000","last_undegraded":"2023-09-23T23:24:20.030563+0000","last_fullsized":"2023-09-23T23:24:20.030563+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:33:17.856914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":384,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.032833+0000","last_change":"2023-09-23T23:22:38.439331+0000","last_active":"2023-09-23T23:24:20.032833+0000","last_peered":"2023-09-23T23:24:20.032833+0000","last_clean":"2023-09-23T23:24:20.032833+0000","last_became_active":"2023-09-23T23:22:38.436293+0000","last_became_peered":"2023-09-23T23:22:38.436293+0000","last_unstale":"2023-09-23T23:24:20.032833+0000","last_undegraded":"2023-09-23T23:24:20.032833+0000","last_fullsized":"2023-09-23T23:24:20.032833+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:53.668614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"358'3972","reported_seq":3987,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:33.047533+0000","last_change":"2023-09-23T23:23:43.513124+0000","last_active":"2023-09-23T23:24:33.047533+0000","last_peered":"2023-09-23T23:24:33.047533+0000","last_clean":"2023-09-23T23:24:33.047533+0000","last_became_active":"2023-09-23T23:23:43.512782+0000","last_became_peered":"2023-09-23T23:23:43.512782+0000","last_unstale":"2023-09-23T23:24:33.047533+0000","last_undegraded":"2023-09-23T23:24:33.047533+0000","last_fullsized":"2023-09-23T23:24:33.047533+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":3972,"log_dups_size":0,"ondisk_log_size":3972,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:16:06.664727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260308992,"num_objects":3972,"num_object_clones":0,"num_object_copies":11916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3972,"num_write_kb":254208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":432,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.038201+0000","last_change":"2023-09-23T23:24:04.832826+0000","last_active":"2023-09-23T23:24:20.038201+0000","last_peered":"2023-09-23T23:24:20.038201+0000","last_clean":"2023-09-23T23:24:20.038201+0000","last_became_active":"2023-09-23T23:23:45.460444+0000","last_became_peered":"2023-09-23T23:23:45.460444+0000","last_unstale":"2023-09-23T23:24:20.038201+0000","last_undegraded":"2023-09-23T23:24:20.038201+0000","last_fullsized":"2023-09-23T23:24:20.038201+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:04.832687+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:04.832687+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:02.707716+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":175,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.037810+0000","last_change":"2023-09-23T23:21:22.192374+0000","last_active":"2023-09-23T23:24:20.037810+0000","last_peered":"2023-09-23T23:24:20.037810+0000","last_clean":"2023-09-23T23:24:20.037810+0000","last_became_active":"2023-09-23T23:21:22.192086+0000","last_became_peered":"2023-09-23T23:21:22.192086+0000","last_unstale":"2023-09-23T23:24:20.037810+0000","last_undegraded":"2023-09-23T23:24:20.037810+0000","last_fullsized":"2023-09-23T23:24:20.037810+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:20.605133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":332,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:25.189194+0000","last_change":"2023-09-23T23:22:38.438390+0000","last_active":"2023-09-23T23:24:25.189194+0000","last_peered":"2023-09-23T23:24:25.189194+0000","last_clean":"2023-09-23T23:24:25.189194+0000","last_became_active":"2023-09-23T23:22:38.438065+0000","last_became_peered":"2023-09-23T23:22:38.438065+0000","last_unstale":"2023-09-23T23:24:25.189194+0000","last_undegraded":"2023-09-23T23:24:25.189194+0000","last_fullsized":"2023-09-23T23:24:25.189194+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:51.254227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"358'3998","reported_seq":4015,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:34.482348+0000","last_change":"2023-09-23T23:23:43.512187+0000","last_active":"2023-09-23T23:24:34.482348+0000","last_peered":"2023-09-23T23:24:34.482348+0000","last_clean":"2023-09-23T23:24:34.482348+0000","last_became_active":"2023-09-23T23:23:43.511874+0000","last_became_peered":"2023-09-23T23:23:43.511874+0000","last_unstale":"2023-09-23T23:24:34.482348+0000","last_undegraded":"2023-09-23T23:24:34.482348+0000","last_fullsized":"2023-09-23T23:24:34.482348+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":3998,"log_dups_size":0,"ondisk_log_size":3998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:38.052106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262012928,"num_objects":3998,"num_object_clones":0,"num_object_copies":11994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3998,"num_write_kb":255872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":693,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.030181+0000","last_change":"2023-09-23T23:23:45.830384+0000","last_active":"2023-09-23T23:24:20.030181+0000","last_peered":"2023-09-23T23:24:20.030181+0000","last_clean":"2023-09-23T23:24:20.030181+0000","last_became_active":"2023-09-23T23:23:36.025724+0000","last_became_peered":"2023-09-23T23:23:36.025724+0000","last_unstale":"2023-09-23T23:24:20.030181+0000","last_undegraded":"2023-09-23T23:24:20.030181+0000","last_fullsized":"2023-09-23T23:24:20.030181+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:34.345528+0000","scrub_duration":0.001659259,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":324,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:27.683622+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:24:27.683622+0000","last_peered":"2023-09-23T23:24:27.683622+0000","last_clean":"2023-09-23T23:24:27.683622+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:24:27.683622+0000","last_undegraded":"2023-09-23T23:24:27.683622+0000","last_fullsized":"2023-09-23T23:24:27.683622+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":316,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.044263+0000","last_change":"2023-09-23T23:10:59.512363+0000","last_active":"2023-09-23T23:24:20.044263+0000","last_peered":"2023-09-23T23:24:20.044263+0000","last_clean":"2023-09-23T23:24:20.044263+0000","last_became_active":"2023-09-23T23:10:59.512022+0000","last_became_peered":"2023-09-23T23:10:59.512022+0000","last_unstale":"2023-09-23T23:24:20.044263+0000","last_undegraded":"2023-09-23T23:24:20.044263+0000","last_fullsized":"2023-09-23T23:24:20.044263+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:45.523432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"358'4073","reported_seq":4091,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:35.437736+0000","last_change":"2023-09-23T23:23:43.503924+0000","last_active":"2023-09-23T23:24:35.437736+0000","last_peered":"2023-09-23T23:24:35.437736+0000","last_clean":"2023-09-23T23:24:35.437736+0000","last_became_active":"2023-09-23T23:23:43.503698+0000","last_became_peered":"2023-09-23T23:23:43.503698+0000","last_unstale":"2023-09-23T23:24:35.437736+0000","last_undegraded":"2023-09-23T23:24:35.437736+0000","last_fullsized":"2023-09-23T23:24:35.437736+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4073,"log_dups_size":0,"ondisk_log_size":4073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:14.346656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266928128,"num_objects":4073,"num_object_clones":0,"num_object_copies":12219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4073,"num_write_kb":260672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":601,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.039232+0000","last_change":"2023-09-23T23:23:50.862663+0000","last_active":"2023-09-23T23:24:20.039232+0000","last_peered":"2023-09-23T23:24:20.039232+0000","last_clean":"2023-09-23T23:24:20.039232+0000","last_became_active":"2023-09-23T23:22:57.463887+0000","last_became_peered":"2023-09-23T23:22:57.463887+0000","last_unstale":"2023-09-23T23:24:20.039232+0000","last_undegraded":"2023-09-23T23:24:20.039232+0000","last_fullsized":"2023-09-23T23:24:20.039232+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:46:51.271983+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":328,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.032902+0000","last_change":"2023-09-23T23:16:56.525904+0000","last_active":"2023-09-23T23:24:20.032902+0000","last_peered":"2023-09-23T23:24:20.032902+0000","last_clean":"2023-09-23T23:24:20.032902+0000","last_became_active":"2023-09-23T23:16:56.525372+0000","last_became_peered":"2023-09-23T23:16:56.525372+0000","last_unstale":"2023-09-23T23:24:20.032902+0000","last_undegraded":"2023-09-23T23:24:20.032902+0000","last_fullsized":"2023-09-23T23:24:20.032902+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:17.070117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":373,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.049804+0000","last_change":"2023-09-23T23:22:38.439737+0000","last_active":"2023-09-23T23:24:20.049804+0000","last_peered":"2023-09-23T23:24:20.049804+0000","last_clean":"2023-09-23T23:24:20.049804+0000","last_became_active":"2023-09-23T23:22:38.439246+0000","last_became_peered":"2023-09-23T23:22:38.439246+0000","last_unstale":"2023-09-23T23:24:20.049804+0000","last_undegraded":"2023-09-23T23:24:20.049804+0000","last_fullsized":"2023-09-23T23:24:20.049804+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:40.042010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"358'4002","reported_seq":4019,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:34.482198+0000","last_change":"2023-09-23T23:23:43.513972+0000","last_active":"2023-09-23T23:24:34.482198+0000","last_peered":"2023-09-23T23:24:34.482198+0000","last_clean":"2023-09-23T23:24:34.482198+0000","last_became_active":"2023-09-23T23:23:43.513615+0000","last_became_peered":"2023-09-23T23:23:43.513615+0000","last_unstale":"2023-09-23T23:24:34.482198+0000","last_undegraded":"2023-09-23T23:24:34.482198+0000","last_fullsized":"2023-09-23T23:24:34.482198+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4002,"log_dups_size":0,"ondisk_log_size":4002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:27.822029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262275072,"num_objects":4002,"num_object_clones":0,"num_object_copies":12006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4002,"num_write_kb":256128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":494,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.035860+0000","last_change":"2023-09-23T23:23:50.178725+0000","last_active":"2023-09-23T23:24:20.035860+0000","last_peered":"2023-09-23T23:24:20.035860+0000","last_clean":"2023-09-23T23:24:20.035860+0000","last_became_active":"2023-09-23T23:23:45.458319+0000","last_became_peered":"2023-09-23T23:23:45.458319+0000","last_unstale":"2023-09-23T23:24:20.035860+0000","last_undegraded":"2023-09-23T23:24:20.035860+0000","last_fullsized":"2023-09-23T23:24:20.035860+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:46.518884+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":379,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.041506+0000","last_change":"2023-09-23T23:23:36.030470+0000","last_active":"2023-09-23T23:24:20.041506+0000","last_peered":"2023-09-23T23:24:20.041506+0000","last_clean":"2023-09-23T23:24:20.041506+0000","last_became_active":"2023-09-23T23:23:36.030157+0000","last_became_peered":"2023-09-23T23:23:36.030157+0000","last_unstale":"2023-09-23T23:24:20.041506+0000","last_undegraded":"2023-09-23T23:24:20.041506+0000","last_fullsized":"2023-09-23T23:24:20.041506+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:08.848362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":326,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.044991+0000","last_change":"2023-09-23T23:15:28.228439+0000","last_active":"2023-09-23T23:24:20.044991+0000","last_peered":"2023-09-23T23:24:20.044991+0000","last_clean":"2023-09-23T23:24:20.044991+0000","last_became_active":"2023-09-23T23:15:28.228095+0000","last_became_peered":"2023-09-23T23:15:28.228095+0000","last_unstale":"2023-09-23T23:24:20.044991+0000","last_undegraded":"2023-09-23T23:24:20.044991+0000","last_fullsized":"2023-09-23T23:24:20.044991+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:37.667201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"358'3940","reported_seq":3958,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:34.214399+0000","last_change":"2023-09-23T23:23:43.513796+0000","last_active":"2023-09-23T23:24:34.214399+0000","last_peered":"2023-09-23T23:24:34.214399+0000","last_clean":"2023-09-23T23:24:34.214399+0000","last_became_active":"2023-09-23T23:23:43.513512+0000","last_became_peered":"2023-09-23T23:23:43.513512+0000","last_unstale":"2023-09-23T23:24:34.214399+0000","last_undegraded":"2023-09-23T23:24:34.214399+0000","last_fullsized":"2023-09-23T23:24:34.214399+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":3940,"log_dups_size":0,"ondisk_log_size":3940,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:10.363938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258211840,"num_objects":3940,"num_object_clones":0,"num_object_copies":11820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3940,"num_write_kb":252160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":660,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.042025+0000","last_change":"2023-09-23T23:23:51.877015+0000","last_active":"2023-09-23T23:24:20.042025+0000","last_peered":"2023-09-23T23:24:20.042025+0000","last_clean":"2023-09-23T23:24:20.042025+0000","last_became_active":"2023-09-23T23:22:57.468211+0000","last_became_peered":"2023-09-23T23:22:57.468211+0000","last_unstale":"2023-09-23T23:24:20.042025+0000","last_undegraded":"2023-09-23T23:24:20.042025+0000","last_fullsized":"2023-09-23T23:24:20.042025+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:42.443440+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":48,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.035570+0000","last_change":"2023-09-23T23:22:38.434606+0000","last_active":"2023-09-23T23:24:20.035570+0000","last_peered":"2023-09-23T23:24:20.035570+0000","last_clean":"2023-09-23T23:24:20.035570+0000","last_became_active":"2023-09-23T23:22:38.434196+0000","last_became_peered":"2023-09-23T23:22:38.434196+0000","last_unstale":"2023-09-23T23:24:20.035570+0000","last_undegraded":"2023-09-23T23:24:20.035570+0000","last_fullsized":"2023-09-23T23:24:20.035570+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:14.702902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":277,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.036100+0000","last_change":"2023-09-23T23:22:39.489325+0000","last_active":"2023-09-23T23:24:20.036100+0000","last_peered":"2023-09-23T23:24:20.036100+0000","last_clean":"2023-09-23T23:24:20.036100+0000","last_became_active":"2023-09-23T23:22:39.489048+0000","last_became_peered":"2023-09-23T23:22:39.489048+0000","last_unstale":"2023-09-23T23:24:20.036100+0000","last_undegraded":"2023-09-23T23:24:20.036100+0000","last_fullsized":"2023-09-23T23:24:20.036100+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:29.984948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"358'4088","reported_seq":4106,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:35.456832+0000","last_change":"2023-09-23T23:23:43.507963+0000","last_active":"2023-09-23T23:24:35.456832+0000","last_peered":"2023-09-23T23:24:35.456832+0000","last_clean":"2023-09-23T23:24:35.456832+0000","last_became_active":"2023-09-23T23:23:43.507503+0000","last_became_peered":"2023-09-23T23:23:43.507503+0000","last_unstale":"2023-09-23T23:24:35.456832+0000","last_undegraded":"2023-09-23T23:24:35.456832+0000","last_fullsized":"2023-09-23T23:24:35.456832+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4088,"log_dups_size":0,"ondisk_log_size":4088,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:39.595255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267911168,"num_objects":4088,"num_object_clones":0,"num_object_copies":12264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4088,"num_write_kb":261632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":766,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.035403+0000","last_change":"2023-09-23T23:23:48.063145+0000","last_active":"2023-09-23T23:24:20.035403+0000","last_peered":"2023-09-23T23:24:20.035403+0000","last_clean":"2023-09-23T23:24:20.035403+0000","last_became_active":"2023-09-23T23:22:57.470024+0000","last_became_peered":"2023-09-23T23:22:57.470024+0000","last_unstale":"2023-09-23T23:24:20.035403+0000","last_undegraded":"2023-09-23T23:24:20.035403+0000","last_fullsized":"2023-09-23T23:24:20.035403+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.063071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.063071+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:51.217545+0000","scrub_duration":0.0015554690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":57,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.036409+0000","last_change":"2023-09-23T23:23:36.048889+0000","last_active":"2023-09-23T23:24:20.036409+0000","last_peered":"2023-09-23T23:24:20.036409+0000","last_clean":"2023-09-23T23:24:20.036409+0000","last_became_active":"2023-09-23T23:23:36.048572+0000","last_became_peered":"2023-09-23T23:23:36.048572+0000","last_unstale":"2023-09-23T23:24:20.036409+0000","last_undegraded":"2023-09-23T23:24:20.036409+0000","last_fullsized":"2023-09-23T23:24:20.036409+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:55.581146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":373,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.031605+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:24:20.031605+0000","last_peered":"2023-09-23T23:24:20.031605+0000","last_clean":"2023-09-23T23:24:20.031605+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:24:20.031605+0000","last_undegraded":"2023-09-23T23:24:20.031605+0000","last_fullsized":"2023-09-23T23:24:20.031605+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":301,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.040363+0000","last_change":"2023-09-23T23:23:36.030569+0000","last_active":"2023-09-23T23:24:20.040363+0000","last_peered":"2023-09-23T23:24:20.040363+0000","last_clean":"2023-09-23T23:24:20.040363+0000","last_became_active":"2023-09-23T23:23:36.030257+0000","last_became_peered":"2023-09-23T23:23:36.030257+0000","last_unstale":"2023-09-23T23:24:20.040363+0000","last_undegraded":"2023-09-23T23:24:20.040363+0000","last_fullsized":"2023-09-23T23:24:20.040363+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:53.906813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.b","version":"358'4011","reported_seq":4029,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:33.279045+0000","last_change":"2023-09-23T23:23:43.509408+0000","last_active":"2023-09-23T23:24:33.279045+0000","last_peered":"2023-09-23T23:24:33.279045+0000","last_clean":"2023-09-23T23:24:33.279045+0000","last_became_active":"2023-09-23T23:23:43.509146+0000","last_became_peered":"2023-09-23T23:23:43.509146+0000","last_unstale":"2023-09-23T23:24:33.279045+0000","last_undegraded":"2023-09-23T23:24:33.279045+0000","last_fullsized":"2023-09-23T23:24:33.279045+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4011,"log_dups_size":0,"ondisk_log_size":4011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:51.961026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262864896,"num_objects":4011,"num_object_clones":0,"num_object_copies":12033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4011,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4011,"num_write_kb":256704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":502,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.040172+0000","last_change":"2023-09-23T23:23:49.320969+0000","last_active":"2023-09-23T23:24:20.040172+0000","last_peered":"2023-09-23T23:24:20.040172+0000","last_clean":"2023-09-23T23:24:20.040172+0000","last_became_active":"2023-09-23T23:22:17.001911+0000","last_became_peered":"2023-09-23T23:22:17.001911+0000","last_unstale":"2023-09-23T23:24:20.040172+0000","last_undegraded":"2023-09-23T23:24:20.040172+0000","last_fullsized":"2023-09-23T23:24:20.040172+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.320823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.320823+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:07.000984+0000","scrub_duration":0.0018140630000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":48,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.035762+0000","last_change":"2023-09-23T23:22:38.433755+0000","last_active":"2023-09-23T23:24:20.035762+0000","last_peered":"2023-09-23T23:24:20.035762+0000","last_clean":"2023-09-23T23:24:20.035762+0000","last_became_active":"2023-09-23T23:22:38.433442+0000","last_became_peered":"2023-09-23T23:22:38.433442+0000","last_unstale":"2023-09-23T23:24:20.035762+0000","last_undegraded":"2023-09-23T23:24:20.035762+0000","last_fullsized":"2023-09-23T23:24:20.035762+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:25.409363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":317,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.044455+0000","last_change":"2023-09-23T23:04:11.514703+0000","last_active":"2023-09-23T23:24:20.044455+0000","last_peered":"2023-09-23T23:24:20.044455+0000","last_clean":"2023-09-23T23:24:20.044455+0000","last_became_active":"2023-09-23T23:04:11.514403+0000","last_became_peered":"2023-09-23T23:04:11.514403+0000","last_unstale":"2023-09-23T23:24:20.044455+0000","last_undegraded":"2023-09-23T23:24:20.044455+0000","last_fullsized":"2023-09-23T23:24:20.044455+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:26.374346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":91,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:27.694251+0000","last_change":"2023-09-23T23:20:15.781844+0000","last_active":"2023-09-23T23:24:27.694251+0000","last_peered":"2023-09-23T23:24:27.694251+0000","last_clean":"2023-09-23T23:24:27.694251+0000","last_became_active":"2023-09-23T23:20:15.781573+0000","last_became_peered":"2023-09-23T23:20:15.781573+0000","last_unstale":"2023-09-23T23:24:27.694251+0000","last_undegraded":"2023-09-23T23:24:27.694251+0000","last_fullsized":"2023-09-23T23:24:27.694251+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:25:50.816486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"358'3982","reported_seq":3999,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:34.499264+0000","last_change":"2023-09-23T23:23:43.508971+0000","last_active":"2023-09-23T23:24:34.499264+0000","last_peered":"2023-09-23T23:24:34.499264+0000","last_clean":"2023-09-23T23:24:34.499264+0000","last_became_active":"2023-09-23T23:23:43.508648+0000","last_became_peered":"2023-09-23T23:23:43.508648+0000","last_unstale":"2023-09-23T23:24:34.499264+0000","last_undegraded":"2023-09-23T23:24:34.499264+0000","last_fullsized":"2023-09-23T23:24:34.499264+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":3982,"log_dups_size":0,"ondisk_log_size":3982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:43.036229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260964352,"num_objects":3982,"num_object_clones":0,"num_object_copies":11946,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3982,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3982,"num_write_kb":254848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":692,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.040247+0000","last_change":"2023-09-23T23:23:52.342245+0000","last_active":"2023-09-23T23:24:20.040247+0000","last_peered":"2023-09-23T23:24:20.040247+0000","last_clean":"2023-09-23T23:24:20.040247+0000","last_became_active":"2023-09-23T23:22:17.000797+0000","last_became_peered":"2023-09-23T23:22:17.000797+0000","last_unstale":"2023-09-23T23:24:20.040247+0000","last_undegraded":"2023-09-23T23:24:20.040247+0000","last_fullsized":"2023-09-23T23:24:20.040247+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:13.663896+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":91,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:27.693842+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:24:27.693842+0000","last_peered":"2023-09-23T23:24:27.693842+0000","last_clean":"2023-09-23T23:24:27.693842+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:24:27.693842+0000","last_undegraded":"2023-09-23T23:24:27.693842+0000","last_fullsized":"2023-09-23T23:24:27.693842+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.035688+0000","last_change":"2023-09-23T23:22:38.480385+0000","last_active":"2023-09-23T23:24:20.035688+0000","last_peered":"2023-09-23T23:24:20.035688+0000","last_clean":"2023-09-23T23:24:20.035688+0000","last_became_active":"2023-09-23T23:22:38.480071+0000","last_became_peered":"2023-09-23T23:22:38.480071+0000","last_unstale":"2023-09-23T23:24:20.035688+0000","last_undegraded":"2023-09-23T23:24:20.035688+0000","last_fullsized":"2023-09-23T23:24:20.035688+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:15.348789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":170,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.039808+0000","last_change":"2023-09-23T23:21:22.194721+0000","last_active":"2023-09-23T23:24:20.039808+0000","last_peered":"2023-09-23T23:24:20.039808+0000","last_clean":"2023-09-23T23:24:20.039808+0000","last_became_active":"2023-09-23T23:21:22.194333+0000","last_became_peered":"2023-09-23T23:21:22.194333+0000","last_unstale":"2023-09-23T23:24:20.039808+0000","last_undegraded":"2023-09-23T23:24:20.039808+0000","last_fullsized":"2023-09-23T23:24:20.039808+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:46.823997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"358'3899","reported_seq":3917,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:33.286971+0000","last_change":"2023-09-23T23:23:43.510145+0000","last_active":"2023-09-23T23:24:33.286971+0000","last_peered":"2023-09-23T23:24:33.286971+0000","last_clean":"2023-09-23T23:24:33.286971+0000","last_became_active":"2023-09-23T23:23:43.509875+0000","last_became_peered":"2023-09-23T23:23:43.509875+0000","last_unstale":"2023-09-23T23:24:33.286971+0000","last_undegraded":"2023-09-23T23:24:33.286971+0000","last_fullsized":"2023-09-23T23:24:33.286971+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":3899,"log_dups_size":0,"ondisk_log_size":3899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:37.678046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255524864,"num_objects":3899,"num_object_clones":0,"num_object_copies":11697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3899,"num_write_kb":249536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":358,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.043221+0000","last_change":"2023-09-23T23:21:22.193754+0000","last_active":"2023-09-23T23:24:20.043221+0000","last_peered":"2023-09-23T23:24:20.043221+0000","last_clean":"2023-09-23T23:24:20.043221+0000","last_became_active":"2023-09-23T23:21:22.193409+0000","last_became_peered":"2023-09-23T23:21:22.193409+0000","last_unstale":"2023-09-23T23:24:20.043221+0000","last_undegraded":"2023-09-23T23:24:20.043221+0000","last_fullsized":"2023-09-23T23:24:20.043221+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:53.635813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":650,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.040189+0000","last_change":"2023-09-23T23:23:55.343042+0000","last_active":"2023-09-23T23:24:20.040189+0000","last_peered":"2023-09-23T23:24:20.040189+0000","last_clean":"2023-09-23T23:24:20.040189+0000","last_became_active":"2023-09-23T23:22:41.462616+0000","last_became_peered":"2023-09-23T23:22:41.462616+0000","last_unstale":"2023-09-23T23:24:20.040189+0000","last_undegraded":"2023-09-23T23:24:20.040189+0000","last_fullsized":"2023-09-23T23:24:20.040189+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:35.897918+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":287,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.038887+0000","last_change":"2023-09-23T23:16:56.525298+0000","last_active":"2023-09-23T23:24:20.038887+0000","last_peered":"2023-09-23T23:24:20.038887+0000","last_clean":"2023-09-23T23:24:20.038887+0000","last_became_active":"2023-09-23T23:16:56.524974+0000","last_became_peered":"2023-09-23T23:16:56.524974+0000","last_unstale":"2023-09-23T23:24:20.038887+0000","last_undegraded":"2023-09-23T23:24:20.038887+0000","last_fullsized":"2023-09-23T23:24:20.038887+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:08.860790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":331,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.034039+0000","last_change":"2023-09-23T23:23:36.022440+0000","last_active":"2023-09-23T23:24:20.034039+0000","last_peered":"2023-09-23T23:24:20.034039+0000","last_clean":"2023-09-23T23:24:20.034039+0000","last_became_active":"2023-09-23T23:23:36.022065+0000","last_became_peered":"2023-09-23T23:23:36.022065+0000","last_unstale":"2023-09-23T23:24:20.034039+0000","last_undegraded":"2023-09-23T23:24:20.034039+0000","last_fullsized":"2023-09-23T23:24:20.034039+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:43:27.098960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"358'3944","reported_seq":3962,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:33.280232+0000","last_change":"2023-09-23T23:23:43.510366+0000","last_active":"2023-09-23T23:24:33.280232+0000","last_peered":"2023-09-23T23:24:33.280232+0000","last_clean":"2023-09-23T23:24:33.280232+0000","last_became_active":"2023-09-23T23:23:43.510117+0000","last_became_peered":"2023-09-23T23:23:43.510117+0000","last_unstale":"2023-09-23T23:24:33.280232+0000","last_undegraded":"2023-09-23T23:24:33.280232+0000","last_fullsized":"2023-09-23T23:24:33.280232+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":3944,"log_dups_size":0,"ondisk_log_size":3944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:05.709562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258473984,"num_objects":3944,"num_object_clones":0,"num_object_copies":11832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3944,"num_write_kb":252416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":468,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.035942+0000","last_change":"2023-09-23T23:23:46.214371+0000","last_active":"2023-09-23T23:24:20.035942+0000","last_peered":"2023-09-23T23:24:20.035942+0000","last_clean":"2023-09-23T23:24:20.035942+0000","last_became_active":"2023-09-23T23:22:41.466645+0000","last_became_peered":"2023-09-23T23:22:41.466645+0000","last_unstale":"2023-09-23T23:24:20.035942+0000","last_undegraded":"2023-09-23T23:24:20.035942+0000","last_fullsized":"2023-09-23T23:24:20.035942+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:14.444467+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":275,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.030259+0000","last_change":"2023-09-23T23:23:35.021697+0000","last_active":"2023-09-23T23:24:20.030259+0000","last_peered":"2023-09-23T23:24:20.030259+0000","last_clean":"2023-09-23T23:24:20.030259+0000","last_became_active":"2023-09-23T23:23:35.021376+0000","last_became_peered":"2023-09-23T23:23:35.021376+0000","last_unstale":"2023-09-23T23:24:20.030259+0000","last_undegraded":"2023-09-23T23:24:20.030259+0000","last_fullsized":"2023-09-23T23:24:20.030259+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:42.630129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":340,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.040091+0000","last_change":"2023-09-23T23:23:36.031618+0000","last_active":"2023-09-23T23:24:20.040091+0000","last_peered":"2023-09-23T23:24:20.040091+0000","last_clean":"2023-09-23T23:24:20.040091+0000","last_became_active":"2023-09-23T23:23:36.031322+0000","last_became_peered":"2023-09-23T23:23:36.031322+0000","last_unstale":"2023-09-23T23:24:20.040091+0000","last_undegraded":"2023-09-23T23:24:20.040091+0000","last_fullsized":"2023-09-23T23:24:20.040091+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:07.871195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":386,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.042285+0000","last_change":"2023-09-23T23:22:38.438448+0000","last_active":"2023-09-23T23:24:20.042285+0000","last_peered":"2023-09-23T23:24:20.042285+0000","last_clean":"2023-09-23T23:24:20.042285+0000","last_became_active":"2023-09-23T23:22:38.437495+0000","last_became_peered":"2023-09-23T23:22:38.437495+0000","last_unstale":"2023-09-23T23:24:20.042285+0000","last_undegraded":"2023-09-23T23:24:20.042285+0000","last_fullsized":"2023-09-23T23:24:20.042285+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:49.587056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"358'3867","reported_seq":3885,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:34.229979+0000","last_change":"2023-09-23T23:23:43.512003+0000","last_active":"2023-09-23T23:24:34.229979+0000","last_peered":"2023-09-23T23:24:34.229979+0000","last_clean":"2023-09-23T23:24:34.229979+0000","last_became_active":"2023-09-23T23:23:43.511728+0000","last_became_peered":"2023-09-23T23:23:43.511728+0000","last_unstale":"2023-09-23T23:24:34.229979+0000","last_undegraded":"2023-09-23T23:24:34.229979+0000","last_fullsized":"2023-09-23T23:24:34.229979+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":3867,"log_dups_size":0,"ondisk_log_size":3867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:33.464148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253427712,"num_objects":3867,"num_object_clones":0,"num_object_copies":11601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3867,"num_write_kb":247488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":663,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.034686+0000","last_change":"2023-09-23T23:23:51.048787+0000","last_active":"2023-09-23T23:24:20.034686+0000","last_peered":"2023-09-23T23:24:20.034686+0000","last_clean":"2023-09-23T23:24:20.034686+0000","last_became_active":"2023-09-23T23:22:57.759601+0000","last_became_peered":"2023-09-23T23:22:57.759601+0000","last_unstale":"2023-09-23T23:24:20.034686+0000","last_undegraded":"2023-09-23T23:24:20.034686+0000","last_fullsized":"2023-09-23T23:24:20.034686+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.048718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.048718+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:18.385599+0000","scrub_duration":0.0016072149999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":209,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.030622+0000","last_change":"2023-09-23T23:23:35.037351+0000","last_active":"2023-09-23T23:24:20.030622+0000","last_peered":"2023-09-23T23:24:20.030622+0000","last_clean":"2023-09-23T23:24:20.030622+0000","last_became_active":"2023-09-23T23:23:35.036841+0000","last_became_peered":"2023-09-23T23:23:35.036841+0000","last_unstale":"2023-09-23T23:24:20.030622+0000","last_undegraded":"2023-09-23T23:24:20.030622+0000","last_fullsized":"2023-09-23T23:24:20.030622+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:20.618292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.036629+0000","last_change":"2023-09-23T23:22:38.453323+0000","last_active":"2023-09-23T23:24:20.036629+0000","last_peered":"2023-09-23T23:24:20.036629+0000","last_clean":"2023-09-23T23:24:20.036629+0000","last_became_active":"2023-09-23T23:22:38.452858+0000","last_became_peered":"2023-09-23T23:22:38.452858+0000","last_unstale":"2023-09-23T23:24:20.036629+0000","last_undegraded":"2023-09-23T23:24:20.036629+0000","last_fullsized":"2023-09-23T23:24:20.036629+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:31.988045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":91,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.033262+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:24:20.033262+0000","last_peered":"2023-09-23T23:24:20.033262+0000","last_clean":"2023-09-23T23:24:20.033262+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:24:20.033262+0000","last_undegraded":"2023-09-23T23:24:20.033262+0000","last_fullsized":"2023-09-23T23:24:20.033262+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"358'3920","reported_seq":3937,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:33.057518+0000","last_change":"2023-09-23T23:23:43.505852+0000","last_active":"2023-09-23T23:24:33.057518+0000","last_peered":"2023-09-23T23:24:33.057518+0000","last_clean":"2023-09-23T23:24:33.057518+0000","last_became_active":"2023-09-23T23:23:43.505473+0000","last_became_peered":"2023-09-23T23:23:43.505473+0000","last_unstale":"2023-09-23T23:24:33.057518+0000","last_undegraded":"2023-09-23T23:24:33.057518+0000","last_fullsized":"2023-09-23T23:24:33.057518+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":3920,"log_dups_size":0,"ondisk_log_size":3920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:31.385610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256901120,"num_objects":3920,"num_object_clones":0,"num_object_copies":11760,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3920,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3920,"num_write_kb":250880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":671,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.049881+0000","last_change":"2023-09-23T23:23:46.892667+0000","last_active":"2023-09-23T23:24:20.049881+0000","last_peered":"2023-09-23T23:24:20.049881+0000","last_clean":"2023-09-23T23:24:20.049881+0000","last_became_active":"2023-09-23T23:22:57.465086+0000","last_became_peered":"2023-09-23T23:22:57.465086+0000","last_unstale":"2023-09-23T23:24:20.049881+0000","last_undegraded":"2023-09-23T23:24:20.049881+0000","last_fullsized":"2023-09-23T23:24:20.049881+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:55.262932+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":337,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.038870+0000","last_change":"2023-09-23T23:16:56.561147+0000","last_active":"2023-09-23T23:24:20.038870+0000","last_peered":"2023-09-23T23:24:20.038870+0000","last_clean":"2023-09-23T23:24:20.038870+0000","last_became_active":"2023-09-23T23:16:56.547808+0000","last_became_peered":"2023-09-23T23:16:56.547808+0000","last_unstale":"2023-09-23T23:24:20.038870+0000","last_undegraded":"2023-09-23T23:24:20.038870+0000","last_fullsized":"2023-09-23T23:24:20.038870+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:13.666957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":212,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.034803+0000","last_change":"2023-09-23T23:16:56.526187+0000","last_active":"2023-09-23T23:24:20.034803+0000","last_peered":"2023-09-23T23:24:20.034803+0000","last_clean":"2023-09-23T23:24:20.034803+0000","last_became_active":"2023-09-23T23:16:56.525874+0000","last_became_peered":"2023-09-23T23:16:56.525874+0000","last_unstale":"2023-09-23T23:24:20.034803+0000","last_undegraded":"2023-09-23T23:24:20.034803+0000","last_fullsized":"2023-09-23T23:24:20.034803+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:32:07.101559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":48,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.035651+0000","last_change":"2023-09-23T23:22:38.444291+0000","last_active":"2023-09-23T23:24:20.035651+0000","last_peered":"2023-09-23T23:24:20.035651+0000","last_clean":"2023-09-23T23:24:20.035651+0000","last_became_active":"2023-09-23T23:22:38.443996+0000","last_became_peered":"2023-09-23T23:22:38.443996+0000","last_unstale":"2023-09-23T23:24:20.035651+0000","last_undegraded":"2023-09-23T23:24:20.035651+0000","last_fullsized":"2023-09-23T23:24:20.035651+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:57.255970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"358'4044","reported_seq":4062,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T23:24:34.213031+0000","last_change":"2023-09-23T23:23:43.510714+0000","last_active":"2023-09-23T23:24:34.213031+0000","last_peered":"2023-09-23T23:24:34.213031+0000","last_clean":"2023-09-23T23:24:34.213031+0000","last_became_active":"2023-09-23T23:23:43.510441+0000","last_became_peered":"2023-09-23T23:23:43.510441+0000","last_unstale":"2023-09-23T23:24:34.213031+0000","last_undegraded":"2023-09-23T23:24:34.213031+0000","last_fullsized":"2023-09-23T23:24:34.213031+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4044,"log_dups_size":0,"ondisk_log_size":4044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:54.915575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265027584,"num_objects":4044,"num_object_clones":0,"num_object_copies":12132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4044,"num_write_kb":258816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":671,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.038059+0000","last_change":"2023-09-23T23:23:45.832267+0000","last_active":"2023-09-23T23:24:20.038059+0000","last_peered":"2023-09-23T23:24:20.038059+0000","last_clean":"2023-09-23T23:24:20.038059+0000","last_became_active":"2023-09-23T23:22:17.009092+0000","last_became_peered":"2023-09-23T23:22:17.009092+0000","last_unstale":"2023-09-23T23:24:20.038059+0000","last_undegraded":"2023-09-23T23:24:20.038059+0000","last_fullsized":"2023-09-23T23:24:20.038059+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:42.979434+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":369,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.035471+0000","last_change":"2023-09-23T23:20:15.786693+0000","last_active":"2023-09-23T23:24:20.035471+0000","last_peered":"2023-09-23T23:24:20.035471+0000","last_clean":"2023-09-23T23:24:20.035471+0000","last_became_active":"2023-09-23T23:20:15.786360+0000","last_became_peered":"2023-09-23T23:20:15.786360+0000","last_unstale":"2023-09-23T23:24:20.035471+0000","last_undegraded":"2023-09-23T23:24:20.035471+0000","last_fullsized":"2023-09-23T23:24:20.035471+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:00.707946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":341,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.042120+0000","last_change":"2023-09-23T23:22:38.482705+0000","last_active":"2023-09-23T23:24:20.042120+0000","last_peered":"2023-09-23T23:24:20.042120+0000","last_clean":"2023-09-23T23:24:20.042120+0000","last_became_active":"2023-09-23T23:22:38.438239+0000","last_became_peered":"2023-09-23T23:22:38.438239+0000","last_unstale":"2023-09-23T23:24:20.042120+0000","last_undegraded":"2023-09-23T23:24:20.042120+0000","last_fullsized":"2023-09-23T23:24:20.042120+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:52.006349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":71,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.031331+0000","last_change":"2023-09-23T23:21:50.045682+0000","last_active":"2023-09-23T23:24:20.031331+0000","last_peered":"2023-09-23T23:24:20.031331+0000","last_clean":"2023-09-23T23:24:20.031331+0000","last_became_active":"2023-09-23T23:21:50.045339+0000","last_became_peered":"2023-09-23T23:21:50.045339+0000","last_unstale":"2023-09-23T23:24:20.031331+0000","last_undegraded":"2023-09-23T23:24:20.031331+0000","last_fullsized":"2023-09-23T23:24:20.031331+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:55.802091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":721,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.037631+0000","last_change":"2023-09-23T23:23:50.872439+0000","last_active":"2023-09-23T23:24:20.037631+0000","last_peered":"2023-09-23T23:24:20.037631+0000","last_clean":"2023-09-23T23:24:20.037631+0000","last_became_active":"2023-09-23T23:23:45.454920+0000","last_became_peered":"2023-09-23T23:23:45.454920+0000","last_unstale":"2023-09-23T23:24:20.037631+0000","last_undegraded":"2023-09-23T23:24:20.037631+0000","last_fullsized":"2023-09-23T23:24:20.037631+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:30.422526+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":109,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.044387+0000","last_change":"2023-09-23T23:23:36.026729+0000","last_active":"2023-09-23T23:24:20.044387+0000","last_peered":"2023-09-23T23:24:20.044387+0000","last_clean":"2023-09-23T23:24:20.044387+0000","last_became_active":"2023-09-23T23:23:36.026345+0000","last_became_peered":"2023-09-23T23:23:36.026345+0000","last_unstale":"2023-09-23T23:24:20.044387+0000","last_undegraded":"2023-09-23T23:24:20.044387+0000","last_fullsized":"2023-09-23T23:24:20.044387+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.463936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":99,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.053292+0000","last_change":"2023-09-23T23:22:38.439602+0000","last_active":"2023-09-23T23:24:20.053292+0000","last_peered":"2023-09-23T23:24:20.053292+0000","last_clean":"2023-09-23T23:24:20.053292+0000","last_became_active":"2023-09-23T23:22:38.439261+0000","last_became_peered":"2023-09-23T23:22:38.439261+0000","last_unstale":"2023-09-23T23:24:20.053292+0000","last_undegraded":"2023-09-23T23:24:20.053292+0000","last_fullsized":"2023-09-23T23:24:20.053292+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:03.293816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":432,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.038395+0000","last_change":"2023-09-23T23:23:58.725636+0000","last_active":"2023-09-23T23:24:20.038395+0000","last_peered":"2023-09-23T23:24:20.038395+0000","last_clean":"2023-09-23T23:24:20.038395+0000","last_became_active":"2023-09-23T23:23:45.457175+0000","last_became_peered":"2023-09-23T23:23:45.457175+0000","last_unstale":"2023-09-23T23:24:20.038395+0000","last_undegraded":"2023-09-23T23:24:20.038395+0000","last_fullsized":"2023-09-23T23:24:20.038395+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:02.529300+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":99,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.044323+0000","last_change":"2023-09-23T23:22:38.452968+0000","last_active":"2023-09-23T23:24:20.044323+0000","last_peered":"2023-09-23T23:24:20.044323+0000","last_clean":"2023-09-23T23:24:20.044323+0000","last_became_active":"2023-09-23T23:22:38.437986+0000","last_became_peered":"2023-09-23T23:22:38.437986+0000","last_unstale":"2023-09-23T23:24:20.044323+0000","last_undegraded":"2023-09-23T23:24:20.044323+0000","last_fullsized":"2023-09-23T23:24:20.044323+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:10.333271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":364,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:23.295414+0000","last_change":"2023-09-23T23:23:36.032480+0000","last_active":"2023-09-23T23:24:23.295414+0000","last_peered":"2023-09-23T23:24:23.295414+0000","last_clean":"2023-09-23T23:24:23.295414+0000","last_became_active":"2023-09-23T23:23:36.032169+0000","last_became_peered":"2023-09-23T23:23:36.032169+0000","last_unstale":"2023-09-23T23:24:23.295414+0000","last_undegraded":"2023-09-23T23:24:23.295414+0000","last_fullsized":"2023-09-23T23:24:23.295414+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:52.893674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":85,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.044283+0000","last_change":"2023-09-23T23:21:22.194730+0000","last_active":"2023-09-23T23:24:20.044283+0000","last_peered":"2023-09-23T23:24:20.044283+0000","last_clean":"2023-09-23T23:24:20.044283+0000","last_became_active":"2023-09-23T23:21:22.194420+0000","last_became_peered":"2023-09-23T23:21:22.194420+0000","last_unstale":"2023-09-23T23:24:20.044283+0000","last_undegraded":"2023-09-23T23:24:20.044283+0000","last_fullsized":"2023-09-23T23:24:20.044283+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:31.083059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":202,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.038709+0000","last_change":"2023-09-23T23:20:26.841878+0000","last_active":"2023-09-23T23:24:20.038709+0000","last_peered":"2023-09-23T23:24:20.038709+0000","last_clean":"2023-09-23T23:24:20.038709+0000","last_became_active":"2023-09-23T23:20:26.841590+0000","last_became_peered":"2023-09-23T23:20:26.841590+0000","last_unstale":"2023-09-23T23:24:20.038709+0000","last_undegraded":"2023-09-23T23:24:20.038709+0000","last_fullsized":"2023-09-23T23:24:20.038709+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:23.904011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":604,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.030797+0000","last_change":"2023-09-23T23:23:51.871725+0000","last_active":"2023-09-23T23:24:20.030797+0000","last_peered":"2023-09-23T23:24:20.030797+0000","last_clean":"2023-09-23T23:24:20.030797+0000","last_became_active":"2023-09-23T23:23:46.456342+0000","last_became_peered":"2023-09-23T23:23:46.456342+0000","last_unstale":"2023-09-23T23:24:20.030797+0000","last_undegraded":"2023-09-23T23:24:20.030797+0000","last_fullsized":"2023-09-23T23:24:20.030797+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:02.041716+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":364,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.038310+0000","last_change":"2023-09-23T23:20:16.730934+0000","last_active":"2023-09-23T23:24:20.038310+0000","last_peered":"2023-09-23T23:24:20.038310+0000","last_clean":"2023-09-23T23:24:20.038310+0000","last_became_active":"2023-09-23T23:20:16.730571+0000","last_became_peered":"2023-09-23T23:20:16.730571+0000","last_unstale":"2023-09-23T23:24:20.038310+0000","last_undegraded":"2023-09-23T23:24:20.038310+0000","last_fullsized":"2023-09-23T23:24:20.038310+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:17.993278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":367,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.038220+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:24:20.038220+0000","last_peered":"2023-09-23T23:24:20.038220+0000","last_clean":"2023-09-23T23:24:20.038220+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:24:20.038220+0000","last_undegraded":"2023-09-23T23:24:20.038220+0000","last_fullsized":"2023-09-23T23:24:20.038220+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":679,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.036149+0000","last_change":"2023-09-23T23:23:56.246175+0000","last_active":"2023-09-23T23:24:20.036149+0000","last_peered":"2023-09-23T23:24:20.036149+0000","last_clean":"2023-09-23T23:24:20.036149+0000","last_became_active":"2023-09-23T23:23:45.458251+0000","last_became_peered":"2023-09-23T23:23:45.458251+0000","last_unstale":"2023-09-23T23:24:20.036149+0000","last_undegraded":"2023-09-23T23:24:20.036149+0000","last_fullsized":"2023-09-23T23:24:20.036149+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:41.637297+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":360,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.039627+0000","last_change":"2023-09-23T23:20:15.786279+0000","last_active":"2023-09-23T23:24:20.039627+0000","last_peered":"2023-09-23T23:24:20.039627+0000","last_clean":"2023-09-23T23:24:20.039627+0000","last_became_active":"2023-09-23T23:20:15.782731+0000","last_became_peered":"2023-09-23T23:20:15.782731+0000","last_unstale":"2023-09-23T23:24:20.039627+0000","last_undegraded":"2023-09-23T23:24:20.039627+0000","last_fullsized":"2023-09-23T23:24:20.039627+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:58.011396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":393,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T23:24:20.044106+0000","last_change":"2023-09-23T23:22:38.447350+0000","last_active":"2023-09-23T23:24:20.044106+0000","last_peered":"2023-09-23T23:24:20.044106+0000","last_clean":"2023-09-23T23:24:20.044106+0000","last_became_active":"2023-09-23T23:22:38.447081+0000","last_became_peered":"2023-09-23T23:22:38.447081+0000","last_unstale":"2023-09-23T23:24:20.044106+0000","last_undegraded":"2023-09-23T23:24:20.044106+0000","last_fullsized":"2023-09-23T23:24:20.044106+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:39:12.306545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":4181393408,"num_objects":63803,"num_object_clones":0,"num_object_copies":191409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":63803,"num_write_kb":4083392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12596084736,"data_stored":12596084736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":63803,"ondisk_log_size":63803,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":1220,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782541,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2022368,"kb_used_data":1965464,"kb_used_omap":8,"kb_used_meta":56887,"kb_avail":91722784,"statfs":{"total":95995035648,"available":93924130816,"internally_reserved":0,"allocated":2012635136,"data_stored":2011636542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":58252357},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.5099999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.5469999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.431}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.86599999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29699999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.3719999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47799999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496018,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2810392,"kb_used_data":2321872,"kb_used_omap":0,"kb_used_meta":488512,"kb_avail":90934760,"statfs":{"total":95995035648,"available":93117194240,"internally_reserved":0,"allocated":2377596928,"data_stored":2376605717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":500236288},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:23:53 2023","interfaces":[{"interface":"back","average":{"1min":8.3539999999999992,"5min":7.2770000000000001,"15min":5.0810000000000004},"min":{"1min":0.21099999999999999,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":112.767,"5min":120.392,"15min":161.64599999999999},"last":2.4750000000000001},{"interface":"front","average":{"1min":8.1630000000000003,"5min":6.8719999999999999,"15min":4.9619999999999997},"min":{"1min":0.23899999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":109.38200000000001,"5min":119.917,"15min":161.38499999999999},"last":2.8919999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":1.5009999999999999,"15min":4.2220000000000004},"min":{"1min":0.245,"5min":0.245,"15min":0.24099999999999999},"max":{"1min":1.5029999999999999,"5min":58.295999999999999,"15min":210.869},"last":3.9209999999999998},{"interface":"front","average":{"1min":0.70899999999999996,"5min":1.5589999999999999,"15min":4.298},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":2.3279999999999998,"5min":58.326999999999998,"15min":210.89699999999999},"last":3.0409999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:23:34 2023","interfaces":[{"interface":"back","average":{"1min":5.6189999999999998,"5min":2.1699999999999999,"15min":2.0179999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":59.631999999999998,"5min":59.631999999999998,"15min":59.631999999999998},"last":3.0720000000000001},{"interface":"front","average":{"1min":5.7160000000000002,"5min":2.3159999999999998,"15min":1.7190000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":59.750999999999998,"5min":59.750999999999998,"15min":59.750999999999998},"last":4.3369999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:24:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":1.2869999999999999,"15min":2.7869999999999999},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.7090000000000001,"5min":10.237,"15min":109.98699999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.371,"15min":2.8690000000000002},"min":{"1min":0.35799999999999998,"5min":0.28799999999999998,"15min":0.24199999999999999},"max":{"1min":5.4189999999999996,"5min":6.4539999999999997,"15min":110.024},"last":3.8220000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:24:10 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":2.3559999999999999,"15min":1.5680000000000001},"min":{"1min":0.28499999999999998,"5min":0.23400000000000001,"15min":0.22900000000000001},"max":{"1min":1.1000000000000001,"5min":166.05199999999999,"15min":166.05199999999999},"last":2.944},{"interface":"front","average":{"1min":0.61399999999999999,"5min":2.9209999999999998,"15min":1.9139999999999999},"min":{"1min":0.31900000000000001,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":3.6619999999999999,"5min":166.631,"15min":166.631},"last":0.627}]},{"osd":5,"last update":"Sat Sep 23 23:24:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.69999999999999996,"15min":1.4099999999999999},"min":{"1min":0.316,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.329,"5min":3.5579999999999998,"15min":91.899000000000001},"last":3.0099999999999998},{"interface":"front","average":{"1min":0.60599999999999998,"5min":1.0880000000000001,"15min":1.4299999999999999},"min":{"1min":0.35099999999999998,"5min":0.249,"15min":0.249},"max":{"1min":3.7490000000000001,"5min":8.4090000000000007,"15min":94.174000000000007},"last":0.59599999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:18:28 2023","interfaces":[{"interface":"back","average":{"1min":2.7250000000000001,"5min":1.2689999999999999,"15min":1.27},"min":{"1min":0.36799999999999999,"5min":0.32300000000000001,"15min":0.32100000000000001},"max":{"1min":13.823,"5min":13.823,"15min":13.823},"last":2.8460000000000001},{"interface":"front","average":{"1min":1.5609999999999999,"5min":0.79900000000000004,"15min":0.98799999999999999},"min":{"1min":0.35999999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":13.103,"5min":13.103,"15min":13.103},"last":2.9790000000000001}]}]},{"osd":1,"up_from":112,"seq":481036337359,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1516040,"kb_used_data":1007424,"kb_used_omap":4,"kb_used_meta":508603,"kb_avail":92229112,"statfs":{"total":95995035648,"available":94442610688,"internally_reserved":0,"allocated":1031602176,"data_stored":1029979099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":520810382},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:23:52 2023","interfaces":[{"interface":"back","average":{"1min":4.3529999999999998,"5min":3.806,"15min":2.7410000000000001},"min":{"1min":0.189,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":75.823999999999998,"5min":75.823999999999998,"15min":75.823999999999998},"last":1.655},{"interface":"front","average":{"1min":4.5419999999999998,"5min":3.6200000000000001,"15min":2.8330000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.192},"max":{"1min":75.840999999999994,"5min":75.840999999999994,"15min":75.840999999999994},"last":1.617}]},{"osd":2,"last update":"Sat Sep 23 23:23:34 2023","interfaces":[{"interface":"back","average":{"1min":4.6230000000000002,"5min":1.502,"15min":4.5730000000000004},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":78.950000000000003,"5min":78.950000000000003,"15min":78.950000000000003},"last":1.6819999999999999},{"interface":"front","average":{"1min":5.266,"5min":1.8859999999999999,"15min":3.2229999999999999},"min":{"1min":0.25,"5min":0.218,"15min":0.218},"max":{"1min":78.694000000000003,"5min":78.694000000000003,"15min":78.694000000000003},"last":0.53700000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:24:32 2023","interfaces":[{"interface":"back","average":{"1min":7.4960000000000004,"5min":2.5609999999999999,"15min":3.8359999999999999},"min":{"1min":0.25700000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":157.90899999999999,"5min":157.90899999999999,"15min":157.90899999999999},"last":0.33600000000000002},{"interface":"front","average":{"1min":7.4550000000000001,"5min":2.3879999999999999,"15min":3.6339999999999999},"min":{"1min":0.30099999999999999,"5min":0.245,"15min":0.245},"max":{"1min":156.27000000000001,"5min":156.27000000000001,"15min":156.27000000000001},"last":1.3120000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:23:52 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.0469999999999999,"15min":1.637},"min":{"1min":0.29199999999999998,"5min":0.28799999999999998,"15min":0.28000000000000003},"max":{"1min":5.2750000000000004,"5min":6.6120000000000001,"15min":18.474},"last":3.9740000000000002},{"interface":"front","average":{"1min":1.081,"5min":0.999,"15min":1.7529999999999999},"min":{"1min":0.32500000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.2789999999999999,"5min":7.2439999999999998,"15min":25.710999999999999},"last":4.0620000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.92700000000000005,"15min":3.2730000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.26400000000000001},"max":{"1min":4.5640000000000001,"5min":4.827,"15min":250.33000000000001},"last":2.1240000000000001},{"interface":"front","average":{"1min":1.603,"5min":1.2390000000000001,"15min":3.266},"min":{"1min":0.34899999999999998,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":10.92,"5min":10.92,"15min":250.21199999999999},"last":0.501}]},{"osd":6,"last update":"Sat Sep 23 23:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":1.5820000000000001,"15min":2.1349999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.32400000000000001},"max":{"1min":4.601,"5min":49.171999999999997,"15min":49.171999999999997},"last":2.4329999999999998},{"interface":"front","average":{"1min":1.496,"5min":1.76,"15min":2.1869999999999998},"min":{"1min":0.36699999999999999,"5min":0.32800000000000001,"15min":0.32700000000000001},"max":{"1min":5.9699999999999998,"5min":49.222000000000001,"15min":49.222000000000001},"last":1.29}]},{"osd":7,"last update":"Sat Sep 23 23:18:40 2023","interfaces":[{"interface":"back","average":{"1min":3.0710000000000002,"5min":2.4540000000000002,"15min":1.9970000000000001},"min":{"1min":0.47699999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":18.532,"5min":18.532,"15min":18.532},"last":4.0119999999999996},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.3380000000000001,"15min":1.8440000000000001},"min":{"1min":0.42199999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.9710000000000001,"5min":25.667000000000002,"15min":25.667000000000002},"last":0.92200000000000004}]}]},{"osd":0,"up_from":91,"seq":390842024183,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3184180,"kb_used_data":2320556,"kb_used_omap":5,"kb_used_meta":863610,"kb_avail":90560972,"statfs":{"total":95995035648,"available":92734435328,"internally_reserved":0,"allocated":2376249344,"data_stored":2374581938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":884337274},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:24:05 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.835,"15min":3.1080000000000001},"min":{"1min":0.221,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":3.923,"5min":21.585999999999999,"15min":155.91999999999999},"last":2.802},{"interface":"front","average":{"1min":1.3819999999999999,"5min":1.532,"15min":2.7410000000000001},"min":{"1min":0.221,"5min":0.183,"15min":0.183},"max":{"1min":6.9379999999999997,"5min":9.6980000000000004,"15min":153.28899999999999},"last":2.4449999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:24:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1589999999999998,"5min":6.335,"15min":4.0780000000000003},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":29.007000000000001,"5min":171.15100000000001,"15min":171.15100000000001},"last":2.8279999999999998},{"interface":"front","average":{"1min":2.77,"5min":6.5250000000000004,"15min":4.6589999999999998},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":28.988,"5min":173.04900000000001,"15min":173.04900000000001},"last":2.6539999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:24:15 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.9650000000000001,"15min":3.1070000000000002},"min":{"1min":0.27300000000000002,"5min":0.245,"15min":0.22700000000000001},"max":{"1min":3.8889999999999998,"5min":35.148000000000003,"15min":82.691000000000003},"last":2.9209999999999998},{"interface":"front","average":{"1min":1.1579999999999999,"5min":2.0489999999999999,"15min":2.5699999999999998},"min":{"1min":0.29799999999999999,"5min":0.26500000000000001,"15min":0.23200000000000001},"max":{"1min":6.9850000000000003,"5min":35.186999999999998,"15min":82.429000000000002},"last":2.4740000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:23:37 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":2.4390000000000001,"15min":2.2389999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.247},"max":{"1min":6.899,"5min":86.915999999999997,"15min":86.915999999999997},"last":3.3650000000000002},{"interface":"front","average":{"1min":9.9120000000000008,"5min":3.9689999999999999,"15min":2.7440000000000002},"min":{"1min":0.35599999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":171.251,"5min":171.251,"15min":171.251},"last":0.35399999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:23:37 2023","interfaces":[{"interface":"back","average":{"1min":10.173,"5min":3.0249999999999999,"15min":2.3740000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29099999999999998},"max":{"1min":171.30699999999999,"5min":171.30699999999999,"15min":171.30699999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.226,"5min":1.3340000000000001,"15min":1.6850000000000001},"min":{"1min":0.40799999999999997,"5min":0.34300000000000003,"15min":0.30299999999999999},"max":{"1min":3.9470000000000001,"5min":10.231,"15min":15.929},"last":2.774}]},{"osd":6,"last update":"Sat Sep 23 23:23:37 2023","interfaces":[{"interface":"back","average":{"1min":9.907,"5min":3.1309999999999998,"15min":2.6040000000000001},"min":{"1min":0.35499999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":171.33600000000001,"5min":171.33600000000001,"15min":171.33600000000001},"last":0.433},{"interface":"front","average":{"1min":1.222,"5min":1.5800000000000001,"15min":2.069},"min":{"1min":0.40500000000000003,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.9830000000000001,"5min":10.256,"15min":30.673999999999999},"last":2.7410000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:24:34 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.341,"15min":1.341},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":9.843,"5min":9.843,"15min":9.843},"last":9.843},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":2.7090000000000001,"5min":2.7090000000000001,"15min":2.7090000000000001},"last":2.7090000000000001}]}]},{"osd":2,"up_from":199,"seq":854698492004,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2916492,"kb_used_data":2580420,"kb_used_omap":7,"kb_used_meta":336056,"kb_avail":90828660,"statfs":{"total":95995035648,"available":93008547840,"internally_reserved":0,"allocated":2642350080,"data_stored":2640725637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":344121425},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:23:36 2023","interfaces":[{"interface":"back","average":{"1min":6.0979999999999999,"5min":4.5469999999999997,"15min":5.3490000000000002},"min":{"1min":0.17699999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":95.587999999999994,"5min":119.277,"15min":119.277},"last":2.1640000000000001},{"interface":"front","average":{"1min":4.9740000000000002,"5min":4.1660000000000004,"15min":5.5979999999999999},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":95.796000000000006,"5min":119.56100000000001,"15min":119.56100000000001},"last":2.335}]},{"osd":1,"last update":"Sat Sep 23 23:23:36 2023","interfaces":[{"interface":"back","average":{"1min":8.4019999999999992,"5min":2.73,"15min":10.066000000000001},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":157.98699999999999,"5min":157.98699999999999,"15min":157.98699999999999},"last":0.23899999999999999},{"interface":"front","average":{"1min":9.4629999999999992,"5min":3.1440000000000001,"15min":9.5500000000000007},"min":{"1min":0.29299999999999998,"5min":0.25800000000000001,"15min":0.245},"max":{"1min":158.03800000000001,"5min":158.03800000000001,"15min":158.03800000000001},"last":2.1930000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:23:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":2.964,"15min":5.9710000000000001},"min":{"1min":0.248,"5min":0.223,"15min":0.223},"max":{"1min":11.852,"5min":119.366,"15min":119.366},"last":0.26100000000000001},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.2429999999999999,"15min":5.1470000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":12.047000000000001,"5min":72.412999999999997,"15min":72.412999999999997},"last":3.8599999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:23:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":2.3660000000000001,"15min":3.9319999999999999},"min":{"1min":0.28000000000000003,"5min":0.25,"15min":0.25},"max":{"1min":7.7939999999999996,"5min":119.88200000000001,"15min":332.98099999999999},"last":2.4060000000000001},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.92000000000000004,"15min":3.431},"min":{"1min":0.36199999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.4079999999999999,"5min":5.4290000000000003,"15min":332.96199999999999},"last":0.317}]},{"osd":5,"last update":"Sat Sep 23 23:23:36 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":2.7050000000000001,"15min":2.3260000000000001},"min":{"1min":0.33700000000000002,"5min":0.29499999999999998,"15min":0.27500000000000002},"max":{"1min":7.8410000000000002,"5min":119.919,"15min":119.919},"last":2.4569999999999999},{"interface":"front","average":{"1min":0.81200000000000006,"5min":2.468,"15min":2.3170000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":5.6509999999999998,"5min":119.306,"15min":119.306},"last":0.43099999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:23:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3580000000000001,"5min":2.9329999999999998,"15min":4.2229999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.31},"max":{"1min":7.8799999999999999,"5min":119.943,"15min":349.32799999999997},"last":2.3690000000000002},{"interface":"front","average":{"1min":0.63800000000000001,"5min":1.6040000000000001,"15min":3.71},"min":{"1min":0.36699999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.4350000000000001,"5min":39.856999999999999,"15min":349.17700000000002},"last":0.47899999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:18:22 2023","interfaces":[{"interface":"back","average":{"1min":23.006,"5min":6.4870000000000001,"15min":3.734},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":239.768,"5min":239.768,"15min":239.768},"last":2.2469999999999999},{"interface":"front","average":{"1min":22.75,"5min":5.71,"15min":2.8700000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":239.78899999999999,"5min":239.78899999999999,"15min":239.78899999999999},"last":2.2690000000000001}]}]},{"osd":3,"up_from":148,"seq":635655159973,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1471440,"kb_used_data":1008264,"kb_used_omap":2,"kb_used_meta":463165,"kb_avail":92273712,"statfs":{"total":95995035648,"available":94488281088,"internally_reserved":0,"allocated":1032462336,"data_stored":1030888395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":474281013},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:24:28 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":2.976,"15min":2.722},"min":{"1min":0.23100000000000001,"5min":0.158,"15min":0.158},"max":{"1min":5.2640000000000002,"5min":103.866,"15min":103.866},"last":7.3150000000000004},{"interface":"front","average":{"1min":0.82299999999999995,"5min":2.1819999999999999,"15min":2.5590000000000002},"min":{"1min":0.23400000000000001,"5min":0.193,"15min":0.17999999999999999},"max":{"1min":4.6589999999999998,"5min":101.884,"15min":101.884},"last":6.6589999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:24:28 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":1.6459999999999999,"15min":2.6539999999999999},"min":{"1min":0.253,"5min":0.216,"15min":0.19600000000000001},"max":{"1min":5.7350000000000003,"5min":14.427,"15min":76.322999999999993},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.137,"15min":2.347},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.185},"max":{"1min":4.0449999999999999,"5min":7.9180000000000001,"15min":76.444000000000003},"last":0.44500000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:23:35 2023","interfaces":[{"interface":"back","average":{"1min":3.161,"5min":4.0529999999999999,"15min":3.7850000000000001},"min":{"1min":0.24299999999999999,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":9.3819999999999997,"5min":101.279,"15min":101.279},"last":1.262},{"interface":"front","average":{"1min":2.4550000000000001,"5min":3.681,"15min":2.399},"min":{"1min":0.34499999999999997,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.4640000000000004,"5min":101.294,"15min":101.294},"last":1.371}]},{"osd":4,"last update":"Sat Sep 23 23:24:28 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":1.165,"15min":1.506},"min":{"1min":0.317,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.016,"5min":9.2089999999999996,"15min":14.754},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.5940000000000001,"15min":1.7989999999999999},"min":{"1min":0.377,"5min":0.33600000000000002,"15min":0.28499999999999998},"max":{"1min":4.7759999999999998,"5min":9.5960000000000001,"15min":13.532999999999999},"last":5.0579999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:24:28 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.355,"15min":1.859},"min":{"1min":0.42099999999999999,"5min":0.307,"15min":0.28199999999999997},"max":{"1min":4.726,"5min":10.531000000000001,"15min":75.733000000000004},"last":5.1269999999999998},{"interface":"front","average":{"1min":1.371,"5min":1.6299999999999999,"15min":2.0600000000000001},"min":{"1min":0.36499999999999999,"5min":0.33800000000000002,"15min":0.28399999999999997},"max":{"1min":5.6699999999999999,"5min":6.4269999999999996,"15min":75.686999999999998},"last":0.46000000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:24:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":1.6160000000000001,"15min":1.7529999999999999},"min":{"1min":0.45500000000000002,"5min":0.38,"15min":0.32200000000000001},"max":{"1min":4.7000000000000002,"5min":10.593999999999999,"15min":14.492000000000001},"last":5.1020000000000003},{"interface":"front","average":{"1min":1.2909999999999999,"5min":1.784,"15min":1.863},"min":{"1min":0.39000000000000001,"5min":0.36599999999999999,"15min":0.33300000000000002},"max":{"1min":5.7050000000000001,"5min":6.7069999999999999,"15min":9.8379999999999992},"last":0.49399999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:18:09 2023","interfaces":[{"interface":"back","average":{"1min":2.198,"5min":2.915,"15min":2.5459999999999998},"min":{"1min":0.40200000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":9.4280000000000008,"5min":130.839,"15min":130.839},"last":0.504},{"interface":"front","average":{"1min":2.9129999999999998,"5min":2.992,"15min":2.665},"min":{"1min":0.434,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":8.8190000000000008,"5min":130.90000000000001,"15min":130.90000000000001},"last":5.149}]}]},{"osd":4,"up_from":30,"seq":128849019291,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1511380,"kb_used_data":905868,"kb_used_omap":0,"kb_used_meta":605504,"kb_avail":92233772,"statfs":{"total":95995035648,"available":94447382528,"internally_reserved":0,"allocated":927608832,"data_stored":926617579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":620036096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:23:45 2023","interfaces":[{"interface":"back","average":{"1min":7.9169999999999998,"5min":5.7530000000000001,"15min":5.2329999999999997},"min":{"1min":0.22800000000000001,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":92.141999999999996,"5min":92.141999999999996,"15min":238.62200000000001},"last":0.24299999999999999},{"interface":"front","average":{"1min":8.0999999999999996,"5min":6.234,"15min":5.6669999999999998},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":92.284000000000006,"5min":92.284000000000006,"15min":238.58199999999999},"last":0.43099999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.077,"15min":4.79},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.6550000000000002,"5min":7.5110000000000001,"15min":175.00899999999999},"last":2.0009999999999999},{"interface":"front","average":{"1min":1.7490000000000001,"5min":1.325,"15min":5.0140000000000002},"min":{"1min":0.28399999999999997,"5min":0.27500000000000002,"15min":0.25800000000000001},"max":{"1min":9.7690000000000001,"5min":9.7690000000000001,"15min":174.821},"last":1.8240000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:23:35 2023","interfaces":[{"interface":"back","average":{"1min":6.6710000000000003,"5min":2.843,"15min":3.2170000000000001},"min":{"1min":0.30099999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":63.113999999999997,"5min":63.113999999999997,"15min":63.113999999999997},"last":0.49099999999999999},{"interface":"front","average":{"1min":6.0149999999999997,"5min":2.7789999999999999,"15min":2.6880000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.316},"max":{"1min":63.125,"5min":63.125,"15min":63.125},"last":0.50800000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:24:29 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":1.9650000000000001,"15min":4.2649999999999997},"min":{"1min":0.27200000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":23.696000000000002,"5min":65.855999999999995,"15min":303.41500000000002},"last":2.3740000000000001},{"interface":"front","average":{"1min":1.8169999999999999,"5min":2.4129999999999998,"15min":4.5640000000000001},"min":{"1min":0.36599999999999999,"5min":0.317,"15min":0.308},"max":{"1min":23.756,"5min":66.293000000000006,"15min":303.428},"last":0.57799999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.82499999999999996,"15min":1.5349999999999999},"min":{"1min":0.26700000000000002,"5min":0.26100000000000001,"15min":0.23699999999999999},"max":{"1min":2.278,"5min":4.8319999999999999,"15min":160.79499999999999},"last":2.0459999999999998},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.0880000000000001,"15min":1.7929999999999999},"min":{"1min":0.31,"5min":0.249,"15min":0.249},"max":{"1min":2.577,"5min":7.9669999999999996,"15min":160.61099999999999},"last":2.0209999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:24:29 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":1.581,"15min":1.3480000000000001},"min":{"1min":0.31900000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.0449999999999999,"5min":90.346999999999994,"15min":90.346999999999994},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.9710000000000001,"15min":1.4419999999999999},"min":{"1min":0.40799999999999997,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":0.81899999999999995,"5min":90.733000000000004,"15min":90.733000000000004},"last":0.63300000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:18:35 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.5609999999999999,"15min":1.3140000000000001},"min":{"1min":0.36299999999999999,"5min":0.315,"15min":0.30499999999999999},"max":{"1min":7.4850000000000003,"5min":14.842000000000001,"15min":14.842000000000001},"last":0.70699999999999996},{"interface":"front","average":{"1min":2.0169999999999999,"5min":1.502,"15min":1.341},"min":{"1min":0.433,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.882,"5min":14.561,"15min":14.561},"last":0.47699999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757654,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1038416,"kb_used_data":506120,"kb_used_omap":0,"kb_used_meta":532288,"kb_avail":92706736,"statfs":{"total":95995035648,"available":94931697664,"internally_reserved":0,"allocated":518266880,"data_stored":517279922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":545062912},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:23:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":6.859,"15min":3.5489999999999999},"min":{"1min":0.24299999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":5.1479999999999997,"5min":113.869,"15min":113.869},"last":0.40999999999999998},{"interface":"front","average":{"1min":7.4809999999999999,"5min":7.9820000000000002,"15min":4.2270000000000003},"min":{"1min":0.23799999999999999,"5min":0.23000000000000001,"15min":0.22700000000000001},"max":{"1min":138.55199999999999,"5min":138.55199999999999,"15min":138.55199999999999},"last":0.36599999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:23:57 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.1419999999999999,"15min":3.726},"min":{"1min":0.28199999999999997,"5min":0.27200000000000002,"15min":0.246},"max":{"1min":4.1699999999999999,"5min":8.0899999999999999,"15min":193.584},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.76400000000000001,"5min":1.514,"15min":3.9649999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.9109999999999996,"5min":16.312999999999999,"15min":194.03100000000001},"last":0.34300000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":2.2970000000000002,"15min":1.776},"min":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":3.1150000000000002,"5min":86.765000000000001,"15min":86.765000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.83199999999999996,"5min":1.9379999999999999,"15min":1.8109999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.25800000000000001},"max":{"1min":3.4279999999999999,"5min":86.837000000000003,"15min":86.837000000000003},"last":0.35399999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:24:26 2023","interfaces":[{"interface":"back","average":{"1min":3.6960000000000002,"5min":1.8919999999999999,"15min":4.1500000000000004},"min":{"1min":0.315,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":57.283999999999999,"5min":57.283999999999999,"15min":159.53299999999999},"last":0.438},{"interface":"front","average":{"1min":3.9409999999999998,"5min":2.1400000000000001,"15min":4.6189999999999998},"min":{"1min":0.34999999999999998,"5min":0.33100000000000002,"15min":0.312},"max":{"1min":57.334000000000003,"5min":57.334000000000003,"15min":159.54900000000001},"last":0.39700000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:24:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.82199999999999995,"15min":1.2310000000000001},"min":{"1min":0.251,"5min":0.24299999999999999,"15min":0.23999999999999999},"max":{"1min":0.58799999999999997,"5min":8.1210000000000004,"15min":14.538},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.75600000000000001,"15min":0.877},"min":{"1min":0.28100000000000003,"5min":0.245,"15min":0.245},"max":{"1min":1.075,"5min":8.1829999999999998,"15min":11.923999999999999},"last":0.29699999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:24:09 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":4.0899999999999999,"15min":2.1890000000000001},"min":{"1min":0.32200000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.391,"5min":310.68799999999999,"15min":310.68799999999999},"last":68.456999999999994},{"interface":"front","average":{"1min":0.503,"5min":4.4550000000000001,"15min":2.4769999999999999},"min":{"1min":0.312,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.997,"5min":311.45400000000001,"15min":311.45400000000001},"last":66.721999999999994}]},{"osd":7,"last update":"Sat Sep 23 23:18:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2010000000000001,"5min":1.286,"15min":1.3700000000000001},"min":{"1min":0.36699999999999999,"5min":0.32700000000000001,"15min":0.29199999999999998},"max":{"1min":12.09,"5min":12.09,"15min":14.929},"last":66.372},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.319,"15min":1.276},"min":{"1min":0.371,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":5.7350000000000003,"5min":6.7089999999999996,"15min":15.090999999999999},"last":66.558999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2392719360,"data_stored":2392719360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1038942208,"data_stored":1038942208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2659844096,"data_stored":2659844096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1031667712,"data_stored":1031667712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":788201472,"data_stored":788201472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":518651904,"data_stored":518651904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2374238208,"data_stored":2374238208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1791819776,"data_stored":1791819776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:24:38.849 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T23:24:38.849 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 51 2023-09-23T23:24:38.895 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:24:38.933 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:38 smithi138 ceph-mon[193606]: pgmap v1062: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T23:24:38.938 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:39.056 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:24:39.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:39.068 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:24:39.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:24:39.157 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 47 16 68443 68427 90.9054 57.625 0.012876 0.0109939 2023-09-23T23:24:39.169 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:24:39.237 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:39.286 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:39.348 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:24:39.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:39.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:24:39.464 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:24:39.515 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:39.590 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:24:39.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:39.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:24:39.650 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:24:39.695 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:39.747 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:39.854 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:39.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:39.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:39.960 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:24:40.005 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:40.024 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:24:40.082 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:24:40.123 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:24:40.158 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 48 16 69479 69463 90.3608 64.75 0.0181865 0.0110617 2023-09-23T23:24:40.176 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:24:40.189 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:24:40.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:40.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:24:40.205 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:24:40.244 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:24:40.287 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:24:40.294 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:40.320 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:24:40.328 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:24:40.328 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:24:40.328 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:24:40.328 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:24:40.328 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:24:40.328 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:24:40.329 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:24:40.329 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:24:40.342 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:40.403 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:40.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:40.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:40.506 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:40.532 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:40.575 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:40.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:40.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:40.668 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:40 smithi138 ceph-mon[193606]: pgmap v1063: 119 pgs: 119 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2023-09-23T23:24:40.668 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:40 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2800692035' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:40.671 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:40 smithi110 ceph-mon[221102]: pgmap v1063: 119 pgs: 119 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2023-09-23T23:24:40.671 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:40 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2800692035' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:40.678 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:40.714 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:40.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:40 smithi110 ceph-mon[212939]: pgmap v1063: 119 pgs: 119 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2023-09-23T23:24:40.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:40 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2800692035' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:41.157 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 49 16 70273 70257 89.5301 49.625 0.00696009 0.0111643 2023-09-23T23:24:41.518 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:41.730 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:41.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:41.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:41.756 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:24:41.756 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:24:41.756 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:24:41.756 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:24:41.756 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:24:41.846 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:41.917 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:42.157 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 50 16 71365 71349 89.1047 68.25 0.00555759 0.0112059 2023-09-23T23:24:42.158 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:42.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:42.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:42.263 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:42.315 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:42.463 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:42.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:42.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:42.572 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:24:42.641 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:42.735 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:24:42.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:42.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:24:42.740 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:24:42.840 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:24:42.898 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:42 smithi110 ceph-mon[212939]: pgmap v1064: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2023-09-23T23:24:42.899 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:42 smithi110 ceph-mon[221102]: pgmap v1064: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2023-09-23T23:24:42.905 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:42.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:42 smithi138 ceph-mon[193606]: pgmap v1064: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2023-09-23T23:24:43.157 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 51 16 72305 72289 88.5099 58.75 0.0143915 0.0112936 2023-09-23T23:24:43.235 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:24:43.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:43.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:24:43.294 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:24:43.348 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:43.415 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:43.632 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:43.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:43.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:43.729 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:24:43.738 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:24:43.812 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:24:43.824 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:43.893 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:43 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3861818241' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:24:43.893 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:43 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:24:43.893 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:43.893 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:24:43.894 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:43 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3861818241' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:24:43.894 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:43 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:24:43.894 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:43.894 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:24:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:43 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3861818241' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:24:43.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:43 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:24:43.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:43.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:24:44.132 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:24:44.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:44.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:24:44.163 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 52 16 73301 73285 87.9998 62.25 0.0145634 0.0113578 2023-09-23T23:24:44.241 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:24:44.271 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:44.341 INFO:teuthology.orchestra.run.smithi110.stderr:Error EINVAL: specified pg_num 51 < pg_num_max 32 2023-09-23T23:24:44.934 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:24:44.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:44.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:24:45.038 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:24:45.077 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:44 smithi138 ceph-mon[193606]: pgmap v1065: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2023-09-23T23:24:45.077 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:44 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:24:45.077 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:44 smithi138 ceph-mon[193606]: failed to encode map e360 with expected crc 2023-09-23T23:24:45.077 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:44 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:24:45.077 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:44 smithi138 ceph-mon[193606]: osdmap e360: 8 total, 8 up, 8 in 2023-09-23T23:24:45.077 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:44 smithi138 ceph-mon[193606]: failed to encode map e360 with expected crc 2023-09-23T23:24:45.077 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:44 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1822849100' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T23:24:45.079 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:45.125 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:24:45.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:45.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:24:45.162 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 53 16 74108 74092 87.2916 50.4375 0.00394397 0.011431 2023-09-23T23:24:45.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:44 smithi110 ceph-mon[212939]: pgmap v1065: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2023-09-23T23:24:45.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:44 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:24:45.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:44 smithi110 ceph-mon[212939]: failed to encode map e360 with expected crc 2023-09-23T23:24:45.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:44 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:24:45.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:44 smithi110 ceph-mon[212939]: osdmap e360: 8 total, 8 up, 8 in 2023-09-23T23:24:45.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:44 smithi110 ceph-mon[212939]: failed to encode map e360 with expected crc 2023-09-23T23:24:45.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:44 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1822849100' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T23:24:45.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:44 smithi110 ceph-mon[221102]: pgmap v1065: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2023-09-23T23:24:45.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:44 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:24:45.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:44 smithi110 ceph-mon[221102]: failed to encode map e360 with expected crc 2023-09-23T23:24:45.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:44 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:24:45.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:44 smithi110 ceph-mon[221102]: osdmap e360: 8 total, 8 up, 8 in 2023-09-23T23:24:45.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:44 smithi110 ceph-mon[221102]: failed to encode map e360 with expected crc 2023-09-23T23:24:45.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:44 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1822849100' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T23:24:45.228 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:45.256 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:45.333 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:24:45.334 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:24:45.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:24:45.334 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:24:45.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.334 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165813 9.7 9:ef096715:::benchmark_data_smithi110_295904_object54632:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:24:45.334 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.377865+0000", 2023-09-23T23:24:45.334 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 481.95203486700001, 2023-09-23T23:24:45.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1834216419999999, 2023-09-23T23:24:45.334 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.335 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.335 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.335 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:24:45.335 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:24:45.335 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165813 2023-09-23T23:24:45.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.335 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377865+0000", 2023-09-23T23:24:45.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377865+0000", 2023-09-23T23:24:45.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377864+0000", 2023-09-23T23:24:45.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:24:45.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377872+0000", 2023-09-23T23:24:45.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9179999999999994e-06 2023-09-23T23:24:45.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377889+0000", 2023-09-23T23:24:45.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7413000000000001e-05 2023-09-23T23:24:45.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377897+0000", 2023-09-23T23:24:45.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0309999999999998e-06 2023-09-23T23:24:45.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377972+0000", 2023-09-23T23:24:45.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4659999999999993e-05 2023-09-23T23:24:45.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:45.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.377984+0000", 2023-09-23T23:24:45.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2333e-05 2023-09-23T23:24:45.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340836+0000", 2023-09-23T23:24:45.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96285172900000005 2023-09-23T23:24:45.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:45.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340849+0000", 2023-09-23T23:24:45.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3107e-05 2023-09-23T23:24:45.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.529796+0000", 2023-09-23T23:24:45.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.188947368 2023-09-23T23:24:45.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.529935+0000", 2023-09-23T23:24:45.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013898799999999999 2023-09-23T23:24:45.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:24:45.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.530085+0000", 2023-09-23T23:24:45.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015011199999999999 2023-09-23T23:24:45.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.557076+0000", 2023-09-23T23:24:45.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.026990584000000001 2023-09-23T23:24:45.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.560991+0000", 2023-09-23T23:24:45.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039146149999999998 2023-09-23T23:24:45.348 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561072+0000", 2023-09-23T23:24:45.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1526000000000005e-05 2023-09-23T23:24:45.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561185+0000", 2023-09-23T23:24:45.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011253599999999999 2023-09-23T23:24:45.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561222+0000", 2023-09-23T23:24:45.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6940000000000002e-05 2023-09-23T23:24:45.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561287+0000", 2023-09-23T23:24:45.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5339000000000006e-05 2023-09-23T23:24:45.358 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.358 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.359 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.359 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.359 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165818 9.7 9:edd24351:::benchmark_data_smithi110_295904_object54637:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:24:45.359 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.384734+0000", 2023-09-23T23:24:45.359 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 481.94516594300001, 2023-09-23T23:24:45.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1767809169999999, 2023-09-23T23:24:45.359 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.359 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.359 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.359 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:24:45.359 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:24:45.360 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165818 2023-09-23T23:24:45.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.360 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384734+0000", 2023-09-23T23:24:45.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384734+0000", 2023-09-23T23:24:45.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384733+0000", 2023-09-23T23:24:45.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:24:45.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384738+0000", 2023-09-23T23:24:45.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1370000000000001e-06 2023-09-23T23:24:45.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384750+0000", 2023-09-23T23:24:45.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.198e-05 2023-09-23T23:24:45.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384773+0000", 2023-09-23T23:24:45.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2255999999999998e-05 2023-09-23T23:24:45.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384816+0000", 2023-09-23T23:24:45.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.35e-05 2023-09-23T23:24:45.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:45.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.384822+0000", 2023-09-23T23:24:45.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1630000000000001e-06 2023-09-23T23:24:45.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340872+0000", 2023-09-23T23:24:45.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95605004699999996 2023-09-23T23:24:45.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:45.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340880+0000", 2023-09-23T23:24:45.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3610000000000003e-06 2023-09-23T23:24:45.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.533907+0000", 2023-09-23T23:24:45.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.193027515 2023-09-23T23:24:45.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.534044+0000", 2023-09-23T23:24:45.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013652 2023-09-23T23:24:45.365 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.365 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:24:45.365 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.534213+0000", 2023-09-23T23:24:45.365 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016906000000000001 2023-09-23T23:24:45.365 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.365 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.365 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.557086+0000", 2023-09-23T23:24:45.365 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.022873668999999999 2023-09-23T23:24:45.365 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.366 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.366 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561026+0000", 2023-09-23T23:24:45.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039397240000000004 2023-09-23T23:24:45.366 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.366 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.366 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.366 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561103+0000", 2023-09-23T23:24:45.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6483000000000006e-05 2023-09-23T23:24:45.366 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.366 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.366 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.366 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561335+0000", 2023-09-23T23:24:45.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000231967 2023-09-23T23:24:45.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561390+0000", 2023-09-23T23:24:45.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5334999999999998e-05 2023-09-23T23:24:45.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561515+0000", 2023-09-23T23:24:45.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012529199999999999 2023-09-23T23:24:45.367 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.367 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.368 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.368 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165854 9.7 9:e376a013:::benchmark_data_smithi110_295904_object54673:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:24:45.368 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.463878+0000", 2023-09-23T23:24:45.368 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 481.86602190999997, 2023-09-23T23:24:45.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0978617750000002, 2023-09-23T23:24:45.368 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.368 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.368 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.368 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:24:45.368 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:24:45.369 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165854 2023-09-23T23:24:45.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.369 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463878+0000", 2023-09-23T23:24:45.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463878+0000", 2023-09-23T23:24:45.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.370 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463877+0000", 2023-09-23T23:24:45.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:24:45.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.370 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463883+0000", 2023-09-23T23:24:45.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5269999999999998e-06 2023-09-23T23:24:45.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463893+0000", 2023-09-23T23:24:45.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0327e-05 2023-09-23T23:24:45.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463898+0000", 2023-09-23T23:24:45.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0509999999999996e-06 2023-09-23T23:24:45.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463961+0000", 2023-09-23T23:24:45.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2624000000000002e-05 2023-09-23T23:24:45.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:45.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.463972+0000", 2023-09-23T23:24:45.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0920999999999999e-05 2023-09-23T23:24:45.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340900+0000", 2023-09-23T23:24:45.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.87692793800000002 2023-09-23T23:24:45.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:45.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340907+0000", 2023-09-23T23:24:45.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0990000000000001e-06 2023-09-23T23:24:45.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.537799+0000", 2023-09-23T23:24:45.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1968927309999999 2023-09-23T23:24:45.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.537907+0000", 2023-09-23T23:24:45.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107235 2023-09-23T23:24:45.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:24:45.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.538020+0000", 2023-09-23T23:24:45.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011348700000000001 2023-09-23T23:24:45.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.557096+0000", 2023-09-23T23:24:45.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019076085999999999 2023-09-23T23:24:45.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561153+0000", 2023-09-23T23:24:45.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040572120000000001 2023-09-23T23:24:45.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561540+0000", 2023-09-23T23:24:45.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00038614200000000002 2023-09-23T23:24:45.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561584+0000", 2023-09-23T23:24:45.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3961999999999997e-05 2023-09-23T23:24:45.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561633+0000", 2023-09-23T23:24:45.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9116999999999997e-05 2023-09-23T23:24:45.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561740+0000", 2023-09-23T23:24:45.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107328 2023-09-23T23:24:45.376 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.377 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.377 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.377 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165860 9.7 9:e0c9fcc3:::benchmark_data_smithi110_295904_object54679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:24:45.377 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.472345+0000", 2023-09-23T23:24:45.377 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 481.857555139, 2023-09-23T23:24:45.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0897597270000001, 2023-09-23T23:24:45.377 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.377 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.377 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.377 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:24:45.377 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:24:45.378 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165860 2023-09-23T23:24:45.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.378 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472345+0000", 2023-09-23T23:24:45.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472345+0000", 2023-09-23T23:24:45.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.379 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.379 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472345+0000", 2023-09-23T23:24:45.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:24:45.379 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.379 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.379 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472348+0000", 2023-09-23T23:24:45.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9739999999999998e-06 2023-09-23T23:24:45.379 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.379 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472356+0000", 2023-09-23T23:24:45.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1419999999999993e-06 2023-09-23T23:24:45.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472359+0000", 2023-09-23T23:24:45.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3560000000000001e-06 2023-09-23T23:24:45.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472409+0000", 2023-09-23T23:24:45.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9982999999999999e-05 2023-09-23T23:24:45.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:45.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.472418+0000", 2023-09-23T23:24:45.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7660000000000004e-06 2023-09-23T23:24:45.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340926+0000", 2023-09-23T23:24:45.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.86850790200000005 2023-09-23T23:24:45.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:45.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340933+0000", 2023-09-23T23:24:45.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2139999999999999e-06 2023-09-23T23:24:45.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.541418+0000", 2023-09-23T23:24:45.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2004846490000001 2023-09-23T23:24:45.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.541552+0000", 2023-09-23T23:24:45.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000134566 2023-09-23T23:24:45.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:24:45.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.542163+0000", 2023-09-23T23:24:45.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00061078700000000001 2023-09-23T23:24:45.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.560793+0000", 2023-09-23T23:24:45.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018630225 2023-09-23T23:24:45.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561763+0000", 2023-09-23T23:24:45.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00097026299999999996 2023-09-23T23:24:45.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.384 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561803+0000", 2023-09-23T23:24:45.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9400000000000002e-05 2023-09-23T23:24:45.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.384 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561930+0000", 2023-09-23T23:24:45.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000127367 2023-09-23T23:24:45.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561986+0000", 2023-09-23T23:24:45.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5379e-05 2023-09-23T23:24:45.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562105+0000", 2023-09-23T23:24:45.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000119218 2023-09-23T23:24:45.385 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.385 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.385 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.386 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.386 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165866 9.7 9:e8d58a1b:::benchmark_data_smithi110_295904_object54685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:24:45.386 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.486204+0000", 2023-09-23T23:24:45.386 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 481.84369632599999, 2023-09-23T23:24:45.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0760496399999999, 2023-09-23T23:24:45.386 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.386 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.386 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.386 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:24:45.386 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:24:45.387 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165866 2023-09-23T23:24:45.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.387 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486204+0000", 2023-09-23T23:24:45.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486204+0000", 2023-09-23T23:24:45.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486203+0000", 2023-09-23T23:24:45.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:24:45.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486207+0000", 2023-09-23T23:24:45.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.236e-06 2023-09-23T23:24:45.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486215+0000", 2023-09-23T23:24:45.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7199999999999995e-06 2023-09-23T23:24:45.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486219+0000", 2023-09-23T23:24:45.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5839999999999999e-06 2023-09-23T23:24:45.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486260+0000", 2023-09-23T23:24:45.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1323999999999999e-05 2023-09-23T23:24:45.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:45.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.486269+0000", 2023-09-23T23:24:45.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5390000000000007e-06 2023-09-23T23:24:45.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340952+0000", 2023-09-23T23:24:45.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.85468296899999996 2023-09-23T23:24:45.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:45.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340959+0000", 2023-09-23T23:24:45.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0419999999999997e-06 2023-09-23T23:24:45.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.545682+0000", 2023-09-23T23:24:45.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2047230959999999 2023-09-23T23:24:45.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.545818+0000", 2023-09-23T23:24:45.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000136527 2023-09-23T23:24:45.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:24:45.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.545972+0000", 2023-09-23T23:24:45.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015338600000000001 2023-09-23T23:24:45.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.560804+0000", 2023-09-23T23:24:45.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014832051000000001 2023-09-23T23:24:45.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561831+0000", 2023-09-23T23:24:45.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001027395 2023-09-23T23:24:45.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.393 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.393 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561871+0000", 2023-09-23T23:24:45.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9718999999999997e-05 2023-09-23T23:24:45.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.393 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.393 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562134+0000", 2023-09-23T23:24:45.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00026311599999999998 2023-09-23T23:24:45.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562161+0000", 2023-09-23T23:24:45.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7259000000000001e-05 2023-09-23T23:24:45.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562253+0000", 2023-09-23T23:24:45.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2167e-05 2023-09-23T23:24:45.394 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.394 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.394 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.395 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165894 9.7 9:e099b5a1:::benchmark_data_smithi110_295904_object54713:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:24:45.395 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.692918+0000", 2023-09-23T23:24:45.395 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 481.63698174000001, 2023-09-23T23:24:45.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.869631313, 2023-09-23T23:24:45.395 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.395 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.395 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.395 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:24:45.395 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:24:45.395 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165894 2023-09-23T23:24:45.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.396 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692918+0000", 2023-09-23T23:24:45.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692918+0000", 2023-09-23T23:24:45.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.397 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692918+0000", 2023-09-23T23:24:45.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:24:45.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.397 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.397 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692923+0000", 2023-09-23T23:24:45.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0499999999999999e-06 2023-09-23T23:24:45.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.397 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.397 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692933+0000", 2023-09-23T23:24:45.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0591000000000001e-05 2023-09-23T23:24:45.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.398 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.398 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692938+0000", 2023-09-23T23:24:45.398 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1440000000000002e-06 2023-09-23T23:24:45.398 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.398 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.398 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.692991+0000", 2023-09-23T23:24:45.398 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2522000000000003e-05 2023-09-23T23:24:45.398 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:45.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.693000+0000", 2023-09-23T23:24:45.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2229999999999996e-06 2023-09-23T23:24:45.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340977+0000", 2023-09-23T23:24:45.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.64797712799999996 2023-09-23T23:24:45.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:45.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.340984+0000", 2023-09-23T23:24:45.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0360000000000001e-06 2023-09-23T23:24:45.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.549530+0000", 2023-09-23T23:24:45.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.208546127 2023-09-23T23:24:45.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.549658+0000", 2023-09-23T23:24:45.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012757799999999999 2023-09-23T23:24:45.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:24:45.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.549806+0000", 2023-09-23T23:24:45.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014840400000000001 2023-09-23T23:24:45.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.560819+0000", 2023-09-23T23:24:45.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011012752000000001 2023-09-23T23:24:45.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561890+0000", 2023-09-23T23:24:45.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001071383 2023-09-23T23:24:45.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562314+0000", 2023-09-23T23:24:45.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00042307100000000002 2023-09-23T23:24:45.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562371+0000", 2023-09-23T23:24:45.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7435999999999997e-05 2023-09-23T23:24:45.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.403 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562424+0000", 2023-09-23T23:24:45.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3461999999999997e-05 2023-09-23T23:24:45.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.403 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.403 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562550+0000", 2023-09-23T23:24:45.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000125277 2023-09-23T23:24:45.403 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.403 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.403 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.404 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:165909 9.7 9:e5c10464:::benchmark_data_smithi110_295904_object54728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T23:24:45.404 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:16:43.748819+0000", 2023-09-23T23:24:45.404 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 481.58108087699998, 2023-09-23T23:24:45.404 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.814135311, 2023-09-23T23:24:45.404 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.404 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.404 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.404 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:24:45.404 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:24:45.404 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165909 2023-09-23T23:24:45.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.404 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748819+0000", 2023-09-23T23:24:45.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748819+0000", 2023-09-23T23:24:45.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748819+0000", 2023-09-23T23:24:45.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:24:45.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.406 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.406 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748823+0000", 2023-09-23T23:24:45.406 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.668e-06 2023-09-23T23:24:45.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.406 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.406 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748837+0000", 2023-09-23T23:24:45.406 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.363e-05 2023-09-23T23:24:45.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748842+0000", 2023-09-23T23:24:45.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8790000000000002e-06 2023-09-23T23:24:45.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748893+0000", 2023-09-23T23:24:45.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1456000000000003e-05 2023-09-23T23:24:45.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:45.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:43.748901+0000", 2023-09-23T23:24:45.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7530000000000001e-06 2023-09-23T23:24:45.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.341003+0000", 2023-09-23T23:24:45.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.59210227299999996 2023-09-23T23:24:45.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:45.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:44.341010+0000", 2023-09-23T23:24:45.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5599999999999999e-06 2023-09-23T23:24:45.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.553256+0000", 2023-09-23T23:24:45.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2122462409999999 2023-09-23T23:24:45.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.553394+0000", 2023-09-23T23:24:45.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013755000000000001 2023-09-23T23:24:45.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T23:24:45.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.553548+0000", 2023-09-23T23:24:45.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000154449 2023-09-23T23:24:45.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.410 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.561003+0000", 2023-09-23T23:24:45.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0074545380000000001 2023-09-23T23:24:45.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.410 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562606+0000", 2023-09-23T23:24:45.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0016038770000000001 2023-09-23T23:24:45.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.411 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562661+0000", 2023-09-23T23:24:45.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4122000000000001e-05 2023-09-23T23:24:45.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.411 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562787+0000", 2023-09-23T23:24:45.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012642 2023-09-23T23:24:45.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.411 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562837+0000", 2023-09-23T23:24:45.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0565000000000001e-05 2023-09-23T23:24:45.412 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.412 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.412 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:16:45.562955+0000", 2023-09-23T23:24:45.412 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117106 2023-09-23T23:24:45.412 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.412 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.412 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.412 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.412 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.412 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:188501 9.6 9:679feb10:::benchmark_data_smithi110_295904_object77320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-09-23T23:24:45.412 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:17:33.113667+0000", 2023-09-23T23:24:45.412 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 432.21623266799998, 2023-09-23T23:24:45.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.30803306400000002, 2023-09-23T23:24:45.413 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.413 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.413 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.413 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:24:45.413 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:24:45.413 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 188501 2023-09-23T23:24:45.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.413 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113667+0000", 2023-09-23T23:24:45.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113667+0000", 2023-09-23T23:24:45.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113666+0000", 2023-09-23T23:24:45.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:24:45.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113673+0000", 2023-09-23T23:24:45.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7889999999999997e-06 2023-09-23T23:24:45.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113688+0000", 2023-09-23T23:24:45.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5165e-05 2023-09-23T23:24:45.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113695+0000", 2023-09-23T23:24:45.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6630000000000004e-06 2023-09-23T23:24:45.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.416 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113737+0000", 2023-09-23T23:24:45.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1814000000000003e-05 2023-09-23T23:24:45.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.416 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113855+0000", 2023-09-23T23:24:45.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118145 2023-09-23T23:24:45.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:24:45.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.113978+0000", 2023-09-23T23:24:45.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000123103 2023-09-23T23:24:45.417 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.117927+0000", 2023-09-23T23:24:45.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039487050000000003 2023-09-23T23:24:45.417 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.126736+0000", 2023-09-23T23:24:45.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088093419999999995 2023-09-23T23:24:45.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.421639+0000", 2023-09-23T23:24:45.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.29490354099999999 2023-09-23T23:24:45.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.421673+0000", 2023-09-23T23:24:45.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3896000000000002e-05 2023-09-23T23:24:45.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:17:33.421701+0000", 2023-09-23T23:24:45.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7203e-05 2023-09-23T23:24:45.419 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.419 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.419 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.419 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:207662 9.d 9:b22a6364:::benchmark_data_smithi110_295904_object96481:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-23T23:24:45.419 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:18:15.286195+0000", 2023-09-23T23:24:45.419 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 390.04370487800003, 2023-09-23T23:24:45.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.31124376799999998, 2023-09-23T23:24:45.419 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.420 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.420 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.420 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:24:45.420 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:24:45.420 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 207662 2023-09-23T23:24:45.420 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.420 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.420 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.286195+0000", 2023-09-23T23:24:45.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.420 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.421 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.421 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.286195+0000", 2023-09-23T23:24:45.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.421 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.421 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.286195+0000", 2023-09-23T23:24:45.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:24:45.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.421 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.421 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.286197+0000", 2023-09-23T23:24:45.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.43e-06 2023-09-23T23:24:45.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.286203+0000", 2023-09-23T23:24:45.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6330000000000001e-06 2023-09-23T23:24:45.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.286207+0000", 2023-09-23T23:24:45.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7469999999999999e-06 2023-09-23T23:24:45.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.289084+0000", 2023-09-23T23:24:45.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028777339999999998 2023-09-23T23:24:45.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.289209+0000", 2023-09-23T23:24:45.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000124578 2023-09-23T23:24:45.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-09-23T23:24:45.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.289323+0000", 2023-09-23T23:24:45.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000114287 2023-09-23T23:24:45.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.293339+0000", 2023-09-23T23:24:45.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040155709999999999 2023-09-23T23:24:45.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.294526+0000", 2023-09-23T23:24:45.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001187032 2023-09-23T23:24:45.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.597339+0000", 2023-09-23T23:24:45.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.30281330299999998 2023-09-23T23:24:45.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.597387+0000", 2023-09-23T23:24:45.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8254000000000003e-05 2023-09-23T23:24:45.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.597439+0000", 2023-09-23T23:24:45.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1609e-05 2023-09-23T23:24:45.427 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.427 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.427 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.427 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.428 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:207668 9.7 9:e7915572:::benchmark_data_smithi110_295904_object96487:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-23T23:24:45.428 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:18:15.295609+0000", 2023-09-23T23:24:45.428 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 390.034291387, 2023-09-23T23:24:45.428 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.30867075199999999, 2023-09-23T23:24:45.428 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.428 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.428 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.428 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:24:45.428 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:24:45.428 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 207668 2023-09-23T23:24:45.428 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.429 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295609+0000", 2023-09-23T23:24:45.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295609+0000", 2023-09-23T23:24:45.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.430 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295608+0000", 2023-09-23T23:24:45.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:24:45.430 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.430 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.430 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.430 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295612+0000", 2023-09-23T23:24:45.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3730000000000003e-06 2023-09-23T23:24:45.430 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.430 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.430 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.430 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295619+0000", 2023-09-23T23:24:45.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0310000000000001e-06 2023-09-23T23:24:45.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.431 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.431 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.431 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295624+0000", 2023-09-23T23:24:45.431 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8729999999999998e-06 2023-09-23T23:24:45.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.431 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.431 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.431 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295665+0000", 2023-09-23T23:24:45.431 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0583000000000001e-05 2023-09-23T23:24:45.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.431 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295780+0000", 2023-09-23T23:24:45.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011491 2023-09-23T23:24:45.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3,6,7", 2023-09-23T23:24:45.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295908+0000", 2023-09-23T23:24:45.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000128539 2023-09-23T23:24:45.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.603885+0000", 2023-09-23T23:24:45.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.30797660999999998 2023-09-23T23:24:45.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.433 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.603914+0000", 2023-09-23T23:24:45.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9269999999999999e-05 2023-09-23T23:24:45.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.433 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.603943+0000", 2023-09-23T23:24:45.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8564999999999998e-05 2023-09-23T23:24:45.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.604073+0000", 2023-09-23T23:24:45.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000130648 2023-09-23T23:24:45.434 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.604083+0000", 2023-09-23T23:24:45.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8770000000000005e-06 2023-09-23T23:24:45.434 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.604151+0000", 2023-09-23T23:24:45.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8336000000000004e-05 2023-09-23T23:24:45.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.435 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.435 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.604280+0000", 2023-09-23T23:24:45.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012813200000000001 2023-09-23T23:24:45.435 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.435 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.435 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.435 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45164.0:207670 9.d 9:b8c349e6:::benchmark_data_smithi110_295904_object96489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-23T23:24:45.436 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:18:15.295643+0000", 2023-09-23T23:24:45.436 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 390.03425688700003, 2023-09-23T23:24:45.436 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.31272043100000002, 2023-09-23T23:24:45.436 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.436 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.436 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.436 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45164", 2023-09-23T23:24:45.436 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:24:45.436 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 207670 2023-09-23T23:24:45.436 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.436 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.436 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.436 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295643+0000", 2023-09-23T23:24:45.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295643+0000", 2023-09-23T23:24:45.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295643+0000", 2023-09-23T23:24:45.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:24:45.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.438 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.438 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.438 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295646+0000", 2023-09-23T23:24:45.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3009999999999998e-06 2023-09-23T23:24:45.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.438 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.438 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.438 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295656+0000", 2023-09-23T23:24:45.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0145e-05 2023-09-23T23:24:45.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.438 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.438 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.439 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295661+0000", 2023-09-23T23:24:45.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6210000000000003e-06 2023-09-23T23:24:45.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.439 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295702+0000", 2023-09-23T23:24:45.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0877999999999997e-05 2023-09-23T23:24:45.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.439 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.295809+0000", 2023-09-23T23:24:45.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107404 2023-09-23T23:24:45.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-09-23T23:24:45.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.589180+0000", 2023-09-23T23:24:45.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.29337100700000002 2023-09-23T23:24:45.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.608245+0000", 2023-09-23T23:24:45.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019064349000000001 2023-09-23T23:24:45.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.608287+0000", 2023-09-23T23:24:45.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2691000000000002e-05 2023-09-23T23:24:45.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.608319+0000", 2023-09-23T23:24:45.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1467999999999998e-05 2023-09-23T23:24:45.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.608342+0000", 2023-09-23T23:24:45.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2982000000000001e-05 2023-09-23T23:24:45.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:18:15.608364+0000", 2023-09-23T23:24:45.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2045999999999999e-05 2023-09-23T23:24:45.442 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.442 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.442 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.442 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:14293 10.6 10:62a31385:::benchmark_data_smithi110_366450_object14292:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2023-09-23T23:24:45.442 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:15.521478+0000", 2023-09-23T23:24:45.442 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 329.80842185199998, 2023-09-23T23:24:45.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20660231800000001, 2023-09-23T23:24:45.443 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.443 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.443 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.443 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:45.443 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:45.443 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 14293 2023-09-23T23:24:45.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.443 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521478+0000", 2023-09-23T23:24:45.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521478+0000", 2023-09-23T23:24:45.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521477+0000", 2023-09-23T23:24:45.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:24:45.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521511+0000", 2023-09-23T23:24:45.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3936999999999997e-05 2023-09-23T23:24:45.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521525+0000", 2023-09-23T23:24:45.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4412e-05 2023-09-23T23:24:45.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521532+0000", 2023-09-23T23:24:45.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7109999999999998e-06 2023-09-23T23:24:45.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521598+0000", 2023-09-23T23:24:45.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5606999999999997e-05 2023-09-23T23:24:45.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521743+0000", 2023-09-23T23:24:45.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014537099999999999 2023-09-23T23:24:45.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:45.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.521888+0000", 2023-09-23T23:24:45.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014454800000000001 2023-09-23T23:24:45.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.523492+0000", 2023-09-23T23:24:45.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001604255 2023-09-23T23:24:45.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.524401+0000", 2023-09-23T23:24:45.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000908733 2023-09-23T23:24:45.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.448 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.448 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.727985+0000", 2023-09-23T23:24:45.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.203584864 2023-09-23T23:24:45.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.448 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.448 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.728040+0000", 2023-09-23T23:24:45.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4092999999999999e-05 2023-09-23T23:24:45.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.449 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.449 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.728081+0000", 2023-09-23T23:24:45.449 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1118999999999997e-05 2023-09-23T23:24:45.449 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.449 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.449 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.449 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.449 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:81407 10.8 10:19cd79ee:::benchmark_data_smithi110_366450_object81406:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-09-23T23:24:45.449 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:10.477387+0000", 2023-09-23T23:24:45.449 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 274.85251295099999, 2023-09-23T23:24:45.449 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.27317791600000002, 2023-09-23T23:24:45.449 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.450 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.450 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.450 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:45.450 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:45.450 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 81407 2023-09-23T23:24:45.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.450 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477387+0000", 2023-09-23T23:24:45.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477387+0000", 2023-09-23T23:24:45.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477386+0000", 2023-09-23T23:24:45.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:24:45.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477420+0000", 2023-09-23T23:24:45.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4446999999999997e-05 2023-09-23T23:24:45.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477434+0000", 2023-09-23T23:24:45.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3482000000000001e-05 2023-09-23T23:24:45.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477440+0000", 2023-09-23T23:24:45.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9159999999999998e-06 2023-09-23T23:24:45.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477546+0000", 2023-09-23T23:24:45.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106925 2023-09-23T23:24:45.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477672+0000", 2023-09-23T23:24:45.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012528699999999999 2023-09-23T23:24:45.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:45.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477800+0000", 2023-09-23T23:24:45.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000128057 2023-09-23T23:24:45.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.454 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.483553+0000", 2023-09-23T23:24:45.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0057529360000000002 2023-09-23T23:24:45.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.454 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.492111+0000", 2023-09-23T23:24:45.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085582790000000002 2023-09-23T23:24:45.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.455 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.750471+0000", 2023-09-23T23:24:45.455 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25835971200000002 2023-09-23T23:24:45.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.455 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.750526+0000", 2023-09-23T23:24:45.455 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5138e-05 2023-09-23T23:24:45.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.455 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.750565+0000", 2023-09-23T23:24:45.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9224000000000003e-05 2023-09-23T23:24:45.456 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.456 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.456 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.456 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:81410 10.8 10:1b5bc57b:::benchmark_data_smithi110_366450_object81409:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-09-23T23:24:45.456 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:10.477455+0000", 2023-09-23T23:24:45.456 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 274.85244495500001, 2023-09-23T23:24:45.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.273493826, 2023-09-23T23:24:45.456 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.456 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.456 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.457 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:45.457 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:45.457 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 81410 2023-09-23T23:24:45.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.457 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477455+0000", 2023-09-23T23:24:45.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.458 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477455+0000", 2023-09-23T23:24:45.458 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.458 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.458 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.458 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.458 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477455+0000", 2023-09-23T23:24:45.458 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:24:45.458 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.458 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.458 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.458 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477485+0000", 2023-09-23T23:24:45.458 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0553999999999999e-05 2023-09-23T23:24:45.458 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.459 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.459 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.459 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477492+0000", 2023-09-23T23:24:45.459 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7449999999999998e-06 2023-09-23T23:24:45.459 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.459 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.459 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.460 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.477496+0000", 2023-09-23T23:24:45.460 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2390000000000004e-06 2023-09-23T23:24:45.460 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.460 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.460 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.460 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.480642+0000", 2023-09-23T23:24:45.460 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031452889999999999 2023-09-23T23:24:45.460 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.461 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.461 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.461 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.480752+0000", 2023-09-23T23:24:45.461 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110807 2023-09-23T23:24:45.461 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.461 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.461 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:45.461 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.480876+0000", 2023-09-23T23:24:45.462 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012367699999999999 2023-09-23T23:24:45.462 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.462 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.462 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.462 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.483813+0000", 2023-09-23T23:24:45.462 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029365559999999999 2023-09-23T23:24:45.462 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.463 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.492143+0000", 2023-09-23T23:24:45.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0083307559999999999 2023-09-23T23:24:45.463 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.463 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.750865+0000", 2023-09-23T23:24:45.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25872173199999998 2023-09-23T23:24:45.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.464 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.464 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.750912+0000", 2023-09-23T23:24:45.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6473e-05 2023-09-23T23:24:45.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.465 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.465 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.750949+0000", 2023-09-23T23:24:45.465 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7456000000000002e-05 2023-09-23T23:24:45.465 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.465 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.465 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.465 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.466 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.466 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:81425 10.3 10:c25ee332:::benchmark_data_smithi110_366450_object81424:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-09-23T23:24:45.466 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:10.492942+0000", 2023-09-23T23:24:45.466 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 274.83695796299997, 2023-09-23T23:24:45.466 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26416036700000001, 2023-09-23T23:24:45.466 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.466 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.467 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.467 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:45.467 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:45.467 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 81425 2023-09-23T23:24:45.467 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.467 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.467 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.467 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.468 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.492942+0000", 2023-09-23T23:24:45.468 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.468 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.468 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.468 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.468 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.492942+0000", 2023-09-23T23:24:45.468 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.468 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.469 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.469 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.469 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.492941+0000", 2023-09-23T23:24:45.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:24:45.469 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.469 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.469 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.469 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.492984+0000", 2023-09-23T23:24:45.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3312000000000001e-05 2023-09-23T23:24:45.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.470 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.470 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.492998+0000", 2023-09-23T23:24:45.470 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4088e-05 2023-09-23T23:24:45.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.493004+0000", 2023-09-23T23:24:45.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0430000000000004e-06 2023-09-23T23:24:45.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.493053+0000", 2023-09-23T23:24:45.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9057000000000001e-05 2023-09-23T23:24:45.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.493193+0000", 2023-09-23T23:24:45.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013915999999999999 2023-09-23T23:24:45.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:45.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.493324+0000", 2023-09-23T23:24:45.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000131338 2023-09-23T23:24:45.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.498816+0000", 2023-09-23T23:24:45.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0054921220000000003 2023-09-23T23:24:45.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.498836+0000", 2023-09-23T23:24:45.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9504000000000002e-05 2023-09-23T23:24:45.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.757006+0000", 2023-09-23T23:24:45.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25817092600000002 2023-09-23T23:24:45.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.757059+0000", 2023-09-23T23:24:45.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2842e-05 2023-09-23T23:24:45.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.757103+0000", 2023-09-23T23:24:45.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3282000000000003e-05 2023-09-23T23:24:45.474 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.474 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.474 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.475 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:81429 10.3 10:c438e427:::benchmark_data_smithi110_366450_object81428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-09-23T23:24:45.475 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:10.495410+0000", 2023-09-23T23:24:45.475 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 274.834490222, 2023-09-23T23:24:45.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26733140399999999, 2023-09-23T23:24:45.475 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.475 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.475 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.475 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:45.475 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:45.475 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 81429 2023-09-23T23:24:45.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.475 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.476 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.476 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.476 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.495410+0000", 2023-09-23T23:24:45.476 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.476 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.476 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.476 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.476 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.495410+0000", 2023-09-23T23:24:45.476 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.476 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.476 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.476 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.495409+0000", 2023-09-23T23:24:45.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:24:45.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.495475+0000", 2023-09-23T23:24:45.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6433000000000006e-05 2023-09-23T23:24:45.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.495485+0000", 2023-09-23T23:24:45.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5759999999999995e-06 2023-09-23T23:24:45.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.495490+0000", 2023-09-23T23:24:45.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6119999999999998e-06 2023-09-23T23:24:45.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.495924+0000", 2023-09-23T23:24:45.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00043414300000000001 2023-09-23T23:24:45.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.496026+0000", 2023-09-23T23:24:45.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101579 2023-09-23T23:24:45.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:45.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.496151+0000", 2023-09-23T23:24:45.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000124547 2023-09-23T23:24:45.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.499122+0000", 2023-09-23T23:24:45.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029714630000000001 2023-09-23T23:24:45.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.501414+0000", 2023-09-23T23:24:45.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022915019999999999 2023-09-23T23:24:45.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.762668+0000", 2023-09-23T23:24:45.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26125429999999999 2023-09-23T23:24:45.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.481 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.481 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.762703+0000", 2023-09-23T23:24:45.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5621000000000003e-05 2023-09-23T23:24:45.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.481 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.481 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:10.762741+0000", 2023-09-23T23:24:45.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7861000000000002e-05 2023-09-23T23:24:45.481 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.481 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.481 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.482 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:109530 10.6 10:65e1b6cb:::benchmark_data_smithi110_366450_object8322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e292)", 2023-09-23T23:24:45.482 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:50.764889+0000", 2023-09-23T23:24:45.482 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 234.56501151500001, 2023-09-23T23:24:45.482 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.236142881, 2023-09-23T23:24:45.482 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.482 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.482 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.482 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:45.482 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:45.482 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109530 2023-09-23T23:24:45.482 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.483 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.764889+0000", 2023-09-23T23:24:45.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.483 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.764889+0000", 2023-09-23T23:24:45.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.483 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.764887+0000", 2023-09-23T23:24:45.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:24:45.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.484 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.764893+0000", 2023-09-23T23:24:45.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9939999999999997e-06 2023-09-23T23:24:45.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.484 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.764907+0000", 2023-09-23T23:24:45.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4260000000000001e-05 2023-09-23T23:24:45.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.485 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.764913+0000", 2023-09-23T23:24:45.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1140000000000002e-06 2023-09-23T23:24:45.485 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.485 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.764957+0000", 2023-09-23T23:24:45.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3943999999999998e-05 2023-09-23T23:24:45.485 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.765054+0000", 2023-09-23T23:24:45.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6651999999999996e-05 2023-09-23T23:24:45.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.486 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:45.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.765133+0000", 2023-09-23T23:24:45.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9400000000000006e-05 2023-09-23T23:24:45.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.486 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.768903+0000", 2023-09-23T23:24:45.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037692939999999999 2023-09-23T23:24:45.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.487 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:50.771263+0000", 2023-09-23T23:24:45.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023606999999999999 2023-09-23T23:24:45.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.487 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:51.000961+0000", 2023-09-23T23:24:45.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.229697447 2023-09-23T23:24:45.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:51.001004+0000", 2023-09-23T23:24:45.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2858000000000002e-05 2023-09-23T23:24:45.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:51.001032+0000", 2023-09-23T23:24:45.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7851000000000002e-05 2023-09-23T23:24:45.488 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.488 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.488 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.489 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:123006 10.6 10:6deaf92d:::benchmark_data_smithi110_366450_object21798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-09-23T23:24:45.489 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:07.382697+0000", 2023-09-23T23:24:45.489 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 217.94720315000001, 2023-09-23T23:24:45.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.142037464, 2023-09-23T23:24:45.489 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.489 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.489 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.489 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:45.489 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:45.489 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 123006 2023-09-23T23:24:45.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.489 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.490 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.382697+0000", 2023-09-23T23:24:45.490 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.490 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.490 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.382697+0000", 2023-09-23T23:24:45.490 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.490 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.490 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.382692+0000", 2023-09-23T23:24:45.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999948 2023-09-23T23:24:45.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.491 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.382702+0000", 2023-09-23T23:24:45.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0312000000000001e-05 2023-09-23T23:24:45.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.491 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.382715+0000", 2023-09-23T23:24:45.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2537999999999999e-05 2023-09-23T23:24:45.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.382720+0000", 2023-09-23T23:24:45.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7870000000000001e-06 2023-09-23T23:24:45.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.382759+0000", 2023-09-23T23:24:45.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9079000000000003e-05 2023-09-23T23:24:45.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.382857+0000", 2023-09-23T23:24:45.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8263000000000004e-05 2023-09-23T23:24:45.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:45.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.382931+0000", 2023-09-23T23:24:45.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4408999999999999e-05 2023-09-23T23:24:45.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.390472+0000", 2023-09-23T23:24:45.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0075405289999999998 2023-09-23T23:24:45.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.390507+0000", 2023-09-23T23:24:45.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4724999999999997e-05 2023-09-23T23:24:45.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.524647+0000", 2023-09-23T23:24:45.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.13414045399999999 2023-09-23T23:24:45.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.524692+0000", 2023-09-23T23:24:45.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4617999999999999e-05 2023-09-23T23:24:45.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.524734+0000", 2023-09-23T23:24:45.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2843000000000003e-05 2023-09-23T23:24:45.495 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.495 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.495 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.495 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:123008 10.6 10:63fd2d1e:::benchmark_data_smithi110_366450_object21800:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-09-23T23:24:45.496 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:07.382729+0000", 2023-09-23T23:24:45.496 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 217.94717118200001, 2023-09-23T23:24:45.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.14213300100000001, 2023-09-23T23:24:45.496 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.496 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.496 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:45.496 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:45.496 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:45.496 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 123008 2023-09-23T23:24:45.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.496 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.382729+0000", 2023-09-23T23:24:45.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.382729+0000", 2023-09-23T23:24:45.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.382729+0000", 2023-09-23T23:24:45.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:24:45.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.382731+0000", 2023-09-23T23:24:45.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1660000000000001e-06 2023-09-23T23:24:45.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.382734+0000", 2023-09-23T23:24:45.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6260000000000001e-06 2023-09-23T23:24:45.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.382737+0000", 2023-09-23T23:24:45.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4710000000000001e-06 2023-09-23T23:24:45.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.386843+0000", 2023-09-23T23:24:45.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041061400000000003 2023-09-23T23:24:45.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.386978+0000", 2023-09-23T23:24:45.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000135241 2023-09-23T23:24:45.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:45.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.387097+0000", 2023-09-23T23:24:45.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118582 2023-09-23T23:24:45.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:45.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.390577+0000", 2023-09-23T23:24:45.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034797640000000002 2023-09-23T23:24:45.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.390650+0000", 2023-09-23T23:24:45.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3622999999999996e-05 2023-09-23T23:24:45.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:45.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.524803+0000", 2023-09-23T23:24:45.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.13415318400000001 2023-09-23T23:24:45.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.524841+0000", 2023-09-23T23:24:45.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7146e-05 2023-09-23T23:24:45.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:07.524862+0000", 2023-09-23T23:24:45.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1440000000000001e-05 2023-09-23T23:24:45.502 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.502 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.502 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.502 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35529.0:167903 10.3 e316/301 10:c8e41c3e:::benchmark_data_smithi110_366450_object66695:head v 316'10510, mlcod=0'0)", 2023-09-23T23:24:45.502 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:22:30.049157+0000", 2023-09-23T23:24:45.502 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 135.28074280600001, 2023-09-23T23:24:45.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.116484247, 2023-09-23T23:24:45.503 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:45.503 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:45.503 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:45.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:45.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:30.049157+0000", 2023-09-23T23:24:45.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:45.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:30.049157+0000", 2023-09-23T23:24:45.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:45.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:45.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:30.049156+0000", 2023-09-23T23:24:45.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:24:45.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:45.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:30.049168+0000", 2023-09-23T23:24:45.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2133000000000001e-05 2023-09-23T23:24:45.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:45.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:30.157108+0000", 2023-09-23T23:24:45.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.107939666 2023-09-23T23:24:45.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:45.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:30.157119+0000", 2023-09-23T23:24:45.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.137e-05 2023-09-23T23:24:45.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:45.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:30.157168+0000", 2023-09-23T23:24:45.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8479000000000001e-05 2023-09-23T23:24:45.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:45.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:30.157207+0000", 2023-09-23T23:24:45.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9458000000000001e-05 2023-09-23T23:24:45.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:45.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:30.165603+0000", 2023-09-23T23:24:45.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0083955539999999995 2023-09-23T23:24:45.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:45.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:45.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:45.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:22:30.165642+0000", 2023-09-23T23:24:45.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9020000000000002e-05 2023-09-23T23:24:45.507 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.507 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.507 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.507 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:45.507 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:45.507 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:24:45.518 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:45.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:45.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:45.628 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:24:45.638 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T23:24:45.695 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:45.742 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:45.796 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:24:45.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:45.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:24:45.900 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:45.943 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:46.161 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 54 16 75006 74990 86.7149 56.125 0.00857454 0.0115277 2023-09-23T23:24:46.247 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:46.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:46.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:46.259 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:24:46.359 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:24:46.435 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:46.562 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:24:46.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:46.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:24:46.675 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:46.741 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:46.824 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:46.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:46.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:46.849 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:24:46.935 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:46.990 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:46 smithi110 ceph-mon[212939]: pgmap v1067: 119 pgs: 119 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T23:24:46.992 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:46 smithi110 ceph-mon[221102]: pgmap v1067: 119 pgs: 119 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T23:24:46.994 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:24:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:24:46] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.43.0" 2023-09-23T23:24:46.999 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:47.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:46 smithi138 ceph-mon[193606]: pgmap v1067: 119 pgs: 119 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T23:24:47.161 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 55 16 76133 76117 86.419 70.4375 0.0140378 0.0115677 2023-09-23T23:24:47.292 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:47.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:47.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:47.410 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:47.460 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:47.509 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:24:47.539 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:47.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:47.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:47.648 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:24:47.726 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:47.910 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:24:47.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:47.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:24:48.022 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:24:48.033 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:24:48.060 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:48.068 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:24:48.114 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:24:48.168 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:24:48.168 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 56 16 77048 77032 85.8968 57.1875 0.0159428 0.0116384 2023-09-23T23:24:48.220 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:24:48.237 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:48.255 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:24:48.283 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:24:48.292 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:24:48.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:48.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:24:48.325 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:24:48.342 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:24:48.342 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:24:48.342 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:24:48.342 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:24:48.342 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:24:48.343 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:24:48.343 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:24:48.343 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:24:48.397 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:24:48.440 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:48.572 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:24:48.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:48.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:24:48.680 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:48.760 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:48.846 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:24:48.847 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:24:48.860 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:48.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:48.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:48.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:48 smithi110 ceph-mon[212939]: pgmap v1068: 119 pgs: 119 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2023-09-23T23:24:48.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:48 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3603443728' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:48.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:48 smithi110 ceph-mon[221102]: pgmap v1068: 119 pgs: 119 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2023-09-23T23:24:48.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:48 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3603443728' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:48.966 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:48.991 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:49.009 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:48 smithi138 ceph-mon[193606]: pgmap v1068: 119 pgs: 119 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2023-09-23T23:24:49.009 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:48 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3603443728' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:49.033 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:49.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:49.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:49.136 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:49.162 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 57 16 77879 77863 85.3013 51.9375 0.0054378 0.0116767 2023-09-23T23:24:49.165 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:49.209 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:49.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:49.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:49.312 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:49.380 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:49.661 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:24:49.703 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:49.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:49.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:49.811 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:24:49.864 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:49.942 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:24:49.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:49.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:24:50.053 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:24:50.098 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:50.162 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 58 16 78855 78839 84.8826 61 0.0158161 0.0117766 2023-09-23T23:24:50.206 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:24:50.207 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:24:50.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:50.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:24:50.313 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:50.362 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:50.490 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:50.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:50.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:50.597 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:24:50.637 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:50.639 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 2, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:24:50.639 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:24:50.639 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T23:24:50.639 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T23:24:50.639 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T23:24:50.640 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T23:24:50.928 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:24:50.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:50.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:24:50.933 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:50 smithi138 ceph-mon[193606]: pgmap v1069: 119 pgs: 119 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2023-09-23T23:24:50.950 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:50 smithi110 ceph-mon[221102]: pgmap v1069: 119 pgs: 119 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2023-09-23T23:24:50.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:50 smithi110 ceph-mon[212939]: pgmap v1069: 119 pgs: 119 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2023-09-23T23:24:51.034 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:51.113 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:51.162 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 59 16 79894 79878 84.5447 64.9375 0.00322355 0.0118125 2023-09-23T23:24:51.162 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:51.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:51.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:51.266 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:51.299 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:51.405 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:51.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:51.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:51.515 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:24:51.559 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:51.705 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:24:51.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:51.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:24:51.727 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:24:51.775 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:24:51.812 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:24:51.827 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:51.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:51 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/546493013' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:24:51.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:51 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:24:51.942 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:52.036 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:51 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/546493013' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:24:52.036 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:51 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:24:52.036 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:51 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/546493013' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:24:52.037 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:51 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:24:52.136 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:52.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:52.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:52.162 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:24:52.161632+0000 min lat: 0.00111746 max lat: 0.34576 avg lat: 0.0118956 2023-09-23T23:24:52.162 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:24:52.162 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 60 16 80751 80735 84.0287 53.5625 0.0171769 0.0118956 2023-09-23T23:24:52.243 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:52.275 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:24:52.276 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:24:52.313 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:52.851 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:52.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:52.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:52.938 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:52 smithi138 ceph-mon[193606]: pgmap v1070: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s 2023-09-23T23:24:52.938 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:52 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:24:52.939 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:52 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:24:52.939 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:52 smithi138 ceph-mon[193606]: osdmap e361: 8 total, 8 up, 8 in 2023-09-23T23:24:52.939 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:52 smithi138 ceph-mon[193606]: failed to encode map e361 with expected crc 2023-09-23T23:24:52.939 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:52 smithi138 ceph-mon[193606]: failed to encode map e361 with expected crc 2023-09-23T23:24:52.939 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:52 smithi138 ceph-mon[193606]: failed to encode map e361 with expected crc 2023-09-23T23:24:52.955 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:24:53.010 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:52 smithi110 ceph-mon[221102]: pgmap v1070: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s 2023-09-23T23:24:53.011 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:52 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:24:53.011 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:52 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:24:53.011 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:52 smithi110 ceph-mon[221102]: osdmap e361: 8 total, 8 up, 8 in 2023-09-23T23:24:53.011 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:52 smithi110 ceph-mon[221102]: failed to encode map e361 with expected crc 2023-09-23T23:24:53.011 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:52 smithi110 ceph-mon[221102]: failed to encode map e361 with expected crc 2023-09-23T23:24:53.011 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:52 smithi110 ceph-mon[221102]: failed to encode map e361 with expected crc 2023-09-23T23:24:53.012 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:52 smithi110 ceph-mon[212939]: pgmap v1070: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s 2023-09-23T23:24:53.012 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:52 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:24:53.012 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:52 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:24:53.012 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:52 smithi110 ceph-mon[212939]: osdmap e361: 8 total, 8 up, 8 in 2023-09-23T23:24:53.012 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:52 smithi110 ceph-mon[212939]: failed to encode map e361 with expected crc 2023-09-23T23:24:53.012 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:52 smithi110 ceph-mon[212939]: failed to encode map e361 with expected crc 2023-09-23T23:24:53.012 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:52 smithi110 ceph-mon[212939]: failed to encode map e361 with expected crc 2023-09-23T23:24:53.019 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:53.136 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:53.160 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:24:53.163 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 61 16 81410 81394 83.3255 41.1875 0.00920527 0.0119975 2023-09-23T23:24:53.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:53.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:24:53.273 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:53.346 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:53.482 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:53.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:53.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:53.607 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:24:53.663 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:24:53.677 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:53.909 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:24:53.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:53.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:24:54.016 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:54.085 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:54.163 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 62 16 82399 82383 82.9787 61.8125 0.00952064 0.0120459 2023-09-23T23:24:54.288 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:54.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:54.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:54.398 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:24:54.470 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:54.519 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:24:54.542 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:24:54.542 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "" 2023-09-23T23:24:54.542 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:24:54.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:54 smithi138 ceph-mon[193606]: pgmap v1072: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2023-09-23T23:24:54.617 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:24:54.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:54.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:24:54.727 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:54.788 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:54.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:54 smithi110 ceph-mon[212939]: pgmap v1072: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2023-09-23T23:24:54.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:54 smithi110 ceph-mon[221102]: pgmap v1072: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2023-09-23T23:24:54.963 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:24:55.163 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 63 16 83421 83405 82.6756 63.875 0.0179579 0.0120889 2023-09-23T23:24:55.286 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:24:55.320 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:24:55.368 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:24:55.409 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:24:55.447 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:24:55.482 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:24:55.529 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:24:55.544 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:55.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:55.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:55.558 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:24:55.564 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:55.586 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:24:55.586 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:24:55.586 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:24:55.586 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:24:55.587 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:24:55.587 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:24:55.587 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:24:55.587 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:24:55.649 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:55.728 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:55 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/677983640' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:55.728 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:55 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/677983640' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:55.732 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:55.811 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:55 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/677983640' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:24:56.163 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 64 16 84143 84127 82.0892 45.125 0.0164496 0.0121766 2023-09-23T23:24:56.683 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:56.687 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:56 smithi110 ceph-mon[221102]: pgmap v1073: 119 pgs: 119 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2023-09-23T23:24:56.687 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:56 smithi110 ceph-mon[212939]: pgmap v1073: 119 pgs: 119 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2023-09-23T23:24:56.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:56.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:56.789 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:24:56.856 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:56.884 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:56 smithi138 ceph-mon[193606]: pgmap v1073: 119 pgs: 119 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2023-09-23T23:24:56.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:24:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:24:56] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.43.0" 2023-09-23T23:24:57.163 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 65 16 85038 85022 81.6871 55.9375 0.0237095 0.0122361 2023-09-23T23:24:57.170 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:24:57.170 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:24:57.170 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:24:57.170 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:24:57.170 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.170 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:15109 10.0 10:0529bd2b:::benchmark_data_smithi110_366450_object15108:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2023-09-23T23:24:57.171 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:15.997974+0000", 2023-09-23T23:24:57.171 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.166593261, 2023-09-23T23:24:57.171 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0422259139999999, 2023-09-23T23:24:57.171 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.171 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:24:57.171 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.171 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.172 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.172 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 15109 2023-09-23T23:24:57.172 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.172 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.172 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.172 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.172 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.997974+0000", 2023-09-23T23:24:57.172 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.172 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.172 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.173 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.173 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.997974+0000", 2023-09-23T23:24:57.173 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.173 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.173 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.173 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.173 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.997973+0000", 2023-09-23T23:24:57.173 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:24:57.174 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.174 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.174 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.174 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.998030+0000", 2023-09-23T23:24:57.174 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7120999999999997e-05 2023-09-23T23:24:57.174 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.174 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.174 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.174 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.998043+0000", 2023-09-23T23:24:57.175 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2982999999999999e-05 2023-09-23T23:24:57.175 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.175 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.175 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.175 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.998049+0000", 2023-09-23T23:24:57.175 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4500000000000001e-06 2023-09-23T23:24:57.175 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.175 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.175 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.176 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.998091+0000", 2023-09-23T23:24:57.176 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.18e-05 2023-09-23T23:24:57.176 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.176 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.176 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.176 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.998237+0000", 2023-09-23T23:24:57.176 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000145317 2023-09-23T23:24:57.176 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.177 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.177 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:24:57.177 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.998379+0000", 2023-09-23T23:24:57.177 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014215300000000001 2023-09-23T23:24:57.177 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.177 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.177 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.178 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.999399+0000", 2023-09-23T23:24:57.178 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010200890000000001 2023-09-23T23:24:57.178 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.178 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.178 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.178 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.000529+0000", 2023-09-23T23:24:57.178 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011305989999999999 2023-09-23T23:24:57.179 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.179 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.179 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.179 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.005513+0000", 2023-09-23T23:24:57.179 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049837220000000003 2023-09-23T23:24:57.179 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.179 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.179 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.179 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.008829+0000", 2023-09-23T23:24:57.180 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003315527 2023-09-23T23:24:57.180 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.180 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.180 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.180 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.008971+0000", 2023-09-23T23:24:57.180 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.000142369 2023-09-23T23:24:57.180 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.180 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.180 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.180 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.008986+0000", 2023-09-23T23:24:57.180 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4664999999999999e-05 2023-09-23T23:24:57.181 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.181 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.181 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.181 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.040109+0000", 2023-09-23T23:24:57.181 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.031123134 2023-09-23T23:24:57.181 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.181 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.181 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.181 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.040200+0000", 2023-09-23T23:24:57.181 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0754000000000006e-05 2023-09-23T23:24:57.182 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.182 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.182 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.182 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.182 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.182 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:15114 10.b 10:d7a399aa:::benchmark_data_smithi110_366450_object15113:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2023-09-23T23:24:57.182 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:15.998063+0000", 2023-09-23T23:24:57.182 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.16650372599997, 2023-09-23T23:24:57.182 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.039241804, 2023-09-23T23:24:57.182 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.183 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:24:57.183 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.183 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.183 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.183 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 15114 2023-09-23T23:24:57.183 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.183 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.183 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.183 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.183 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.998063+0000", 2023-09-23T23:24:57.183 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.184 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.184 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.184 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.184 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.998063+0000", 2023-09-23T23:24:57.184 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.184 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.184 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.184 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.184 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.998063+0000", 2023-09-23T23:24:57.185 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:24:57.185 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.185 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.185 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.185 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.998102+0000", 2023-09-23T23:24:57.185 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9458000000000001e-05 2023-09-23T23:24:57.185 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.185 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.185 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.185 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.998111+0000", 2023-09-23T23:24:57.185 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8610000000000004e-06 2023-09-23T23:24:57.186 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.186 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.186 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.186 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.998116+0000", 2023-09-23T23:24:57.186 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1649999999999997e-06 2023-09-23T23:24:57.186 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.186 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.186 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.186 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.998333+0000", 2023-09-23T23:24:57.186 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021725 2023-09-23T23:24:57.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.187 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.187 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.998451+0000", 2023-09-23T23:24:57.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117639 2023-09-23T23:24:57.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.187 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:24:57.187 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.998584+0000", 2023-09-23T23:24:57.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013266600000000001 2023-09-23T23:24:57.188 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.188 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.188 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:15.999468+0000", 2023-09-23T23:24:57.188 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00088399799999999999 2023-09-23T23:24:57.188 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.188 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.188 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.008134+0000", 2023-09-23T23:24:57.188 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086660550000000006 2023-09-23T23:24:57.189 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.189 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.189 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.189 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.008143+0000", 2023-09-23T23:24:57.189 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9830000000000002e-06 2023-09-23T23:24:57.189 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.189 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.189 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.189 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.010532+0000", 2023-09-23T23:24:57.189 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0023890129999999 2023-09-23T23:24:57.189 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.190 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.190 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.190 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.010541+0000", 2023-09-23T23:24:57.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3060000000000003e-06 2023-09-23T23:24:57.190 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.190 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.190 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.190 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.037228+0000", 2023-09-23T23:24:57.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.026687313 2023-09-23T23:24:57.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.191 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.037305+0000", 2023-09-23T23:24:57.191 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6705999999999994e-05 2023-09-23T23:24:57.191 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.191 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.191 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.191 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:15133 10.0 10:02222896:::benchmark_data_smithi110_366450_object15132:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-09-23T23:24:57.192 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:16.019437+0000", 2023-09-23T23:24:57.192 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.14512978300002, 2023-09-23T23:24:57.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0265013839999999, 2023-09-23T23:24:57.192 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.192 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:57.192 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.192 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.192 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.192 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 15133 2023-09-23T23:24:57.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.193 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.193 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.193 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.019437+0000", 2023-09-23T23:24:57.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.193 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.193 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.193 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.019437+0000", 2023-09-23T23:24:57.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.194 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.194 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.194 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.194 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.019436+0000", 2023-09-23T23:24:57.194 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:24:57.194 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.194 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.194 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.194 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.019496+0000", 2023-09-23T23:24:57.194 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9886999999999998e-05 2023-09-23T23:24:57.195 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.195 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.195 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.195 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.019506+0000", 2023-09-23T23:24:57.195 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8220000000000002e-06 2023-09-23T23:24:57.195 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.195 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.195 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.195 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.019510+0000", 2023-09-23T23:24:57.195 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4089999999999996e-06 2023-09-23T23:24:57.195 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.196 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.196 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.019577+0000", 2023-09-23T23:24:57.196 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6248000000000007e-05 2023-09-23T23:24:57.196 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.196 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.196 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.019590+0000", 2023-09-23T23:24:57.196 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3190000000000001e-05 2023-09-23T23:24:57.196 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.197 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.197 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.009030+0000", 2023-09-23T23:24:57.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98943987499999997 2023-09-23T23:24:57.197 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.197 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.197 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.009040+0000", 2023-09-23T23:24:57.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0536999999999999e-05 2023-09-23T23:24:57.197 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.198 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.198 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.198 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.040225+0000", 2023-09-23T23:24:57.198 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.031184643 2023-09-23T23:24:57.198 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.198 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.198 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.198 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.040364+0000", 2023-09-23T23:24:57.198 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013942199999999999 2023-09-23T23:24:57.198 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.198 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.199 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.199 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.040521+0000", 2023-09-23T23:24:57.199 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015684100000000001 2023-09-23T23:24:57.199 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.199 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.199 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.199 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.045119+0000", 2023-09-23T23:24:57.199 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0045975599999999997 2023-09-23T23:24:57.199 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.199 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.200 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.200 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.045190+0000", 2023-09-23T23:24:57.200 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1740000000000001e-05 2023-09-23T23:24:57.200 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.200 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.200 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.200 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.045312+0000", 2023-09-23T23:24:57.200 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000121469 2023-09-23T23:24:57.200 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.200 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.201 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:57.201 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.045375+0000", 2023-09-23T23:24:57.201 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2717000000000003e-05 2023-09-23T23:24:57.201 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.201 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.201 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.201 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.045939+0000", 2023-09-23T23:24:57.201 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00056385500000000004 2023-09-23T23:24:57.201 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.202 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.202 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.202 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.202 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.202 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:15143 10.b 10:d1b130b9:::benchmark_data_smithi110_366450_object15142:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-09-23T23:24:57.202 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:16.023340+0000", 2023-09-23T23:24:57.202 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.141227362, 2023-09-23T23:24:57.202 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0249427469999999, 2023-09-23T23:24:57.203 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.203 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:57.203 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.203 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.203 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.203 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 15143 2023-09-23T23:24:57.203 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.203 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.203 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.203 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.204 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.023340+0000", 2023-09-23T23:24:57.204 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.204 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.204 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.204 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.204 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.023340+0000", 2023-09-23T23:24:57.204 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.205 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.205 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.205 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.205 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.023338+0000", 2023-09-23T23:24:57.205 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:24:57.205 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.205 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.205 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.205 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.023412+0000", 2023-09-23T23:24:57.205 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3697999999999998e-05 2023-09-23T23:24:57.206 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.206 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.206 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.206 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.023428+0000", 2023-09-23T23:24:57.206 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.609e-05 2023-09-23T23:24:57.206 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.206 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.206 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.207 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.023435+0000", 2023-09-23T23:24:57.207 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5089999999999999e-06 2023-09-23T23:24:57.207 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.207 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.207 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.207 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.023788+0000", 2023-09-23T23:24:57.207 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00035343199999999999 2023-09-23T23:24:57.207 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.208 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.208 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.208 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.023798+0000", 2023-09-23T23:24:57.208 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8020000000000003e-06 2023-09-23T23:24:57.208 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.208 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.208 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.208 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.010566+0000", 2023-09-23T23:24:57.208 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98676765200000005 2023-09-23T23:24:57.209 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.209 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.209 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.209 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.010573+0000", 2023-09-23T23:24:57.209 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0690000000000004e-06 2023-09-23T23:24:57.209 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.209 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.209 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.210 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.037324+0000", 2023-09-23T23:24:57.210 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.026750853 2023-09-23T23:24:57.210 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.210 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.210 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.210 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.037428+0000", 2023-09-23T23:24:57.210 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000104645 2023-09-23T23:24:57.210 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.210 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.211 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.211 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.037551+0000", 2023-09-23T23:24:57.211 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012318400000000001 2023-09-23T23:24:57.211 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.211 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.211 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.211 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.041738+0000", 2023-09-23T23:24:57.212 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041862410000000003 2023-09-23T23:24:57.212 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.212 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.212 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.212 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.046742+0000", 2023-09-23T23:24:57.212 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050042580000000001 2023-09-23T23:24:57.212 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.213 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.213 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.213 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.046890+0000", 2023-09-23T23:24:57.213 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014804099999999999 2023-09-23T23:24:57.213 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.213 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.213 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:57.213 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.046944+0000", 2023-09-23T23:24:57.214 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3582000000000003e-05 2023-09-23T23:24:57.214 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.214 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.214 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.214 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.048282+0000", 2023-09-23T23:24:57.214 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00133876 2023-09-23T23:24:57.214 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.214 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.214 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.215 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.215 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.215 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:15154 10.b 10:dcd972d2:::benchmark_data_smithi110_366450_object15153:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-09-23T23:24:57.215 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:16.025757+0000", 2023-09-23T23:24:57.215 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.138810369, 2023-09-23T23:24:57.215 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.023298236, 2023-09-23T23:24:57.215 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.215 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:57.215 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.215 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.216 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.216 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 15154 2023-09-23T23:24:57.216 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.216 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.216 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.216 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.216 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.025757+0000", 2023-09-23T23:24:57.216 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.216 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.216 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.217 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.217 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.025757+0000", 2023-09-23T23:24:57.217 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.217 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.217 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.217 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.217 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.025756+0000", 2023-09-23T23:24:57.217 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:24:57.217 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.217 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.218 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.218 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.025818+0000", 2023-09-23T23:24:57.218 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2464000000000004e-05 2023-09-23T23:24:57.218 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.218 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.218 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.218 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.025825+0000", 2023-09-23T23:24:57.218 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7329999999999997e-06 2023-09-23T23:24:57.218 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.218 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.218 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.219 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.025828+0000", 2023-09-23T23:24:57.219 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2329999999999998e-06 2023-09-23T23:24:57.219 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.219 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.219 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.219 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.025885+0000", 2023-09-23T23:24:57.219 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6217000000000001e-05 2023-09-23T23:24:57.219 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.219 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.219 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.220 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.025894+0000", 2023-09-23T23:24:57.220 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1530000000000003e-06 2023-09-23T23:24:57.220 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.220 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.220 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.220 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.010595+0000", 2023-09-23T23:24:57.220 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98470119199999995 2023-09-23T23:24:57.220 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.220 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.220 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.221 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.010602+0000", 2023-09-23T23:24:57.221 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9199999999999998e-06 2023-09-23T23:24:57.221 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.221 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.221 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.221 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.038209+0000", 2023-09-23T23:24:57.221 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0276072110000001 2023-09-23T23:24:57.221 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.221 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.221 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.222 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.038449+0000", 2023-09-23T23:24:57.222 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000239721 2023-09-23T23:24:57.222 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.222 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.222 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.222 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.038581+0000", 2023-09-23T23:24:57.222 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000132399 2023-09-23T23:24:57.222 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.222 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.222 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.223 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.042604+0000", 2023-09-23T23:24:57.223 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040224730000000004 2023-09-23T23:24:57.223 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.223 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.223 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.223 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.046817+0000", 2023-09-23T23:24:57.223 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0042128240000000004 2023-09-23T23:24:57.223 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.223 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.223 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.224 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.048443+0000", 2023-09-23T23:24:57.224 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0016259860000000001 2023-09-23T23:24:57.224 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.224 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.224 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:57.224 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.048583+0000", 2023-09-23T23:24:57.224 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000140292 2023-09-23T23:24:57.224 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.224 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.224 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.225 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.049055+0000", 2023-09-23T23:24:57.225 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000471946 2023-09-23T23:24:57.225 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.225 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.225 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.225 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.225 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.225 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:15156 10.b 10:dfec54d1:::benchmark_data_smithi110_366450_object15155:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-09-23T23:24:57.225 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:16.028469+0000", 2023-09-23T23:24:57.225 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.13609796499998, 2023-09-23T23:24:57.226 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0211131560000002, 2023-09-23T23:24:57.226 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.226 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:57.226 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.226 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.226 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.226 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 15156 2023-09-23T23:24:57.226 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.226 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.227 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.227 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.227 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.028469+0000", 2023-09-23T23:24:57.227 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.227 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.227 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.227 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.227 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.028469+0000", 2023-09-23T23:24:57.227 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.227 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.227 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.228 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.228 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.028468+0000", 2023-09-23T23:24:57.228 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:24:57.228 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.228 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.228 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.228 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.028497+0000", 2023-09-23T23:24:57.228 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8707000000000001e-05 2023-09-23T23:24:57.228 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.228 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.229 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.229 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.028504+0000", 2023-09-23T23:24:57.229 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1439999999999997e-06 2023-09-23T23:24:57.229 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.229 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.229 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.229 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.028509+0000", 2023-09-23T23:24:57.229 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6380000000000003e-06 2023-09-23T23:24:57.229 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.229 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.229 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.230 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.028554+0000", 2023-09-23T23:24:57.230 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4842999999999997e-05 2023-09-23T23:24:57.230 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.230 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.230 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.230 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.028562+0000", 2023-09-23T23:24:57.230 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0730000000000005e-06 2023-09-23T23:24:57.230 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.230 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.231 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.231 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.010626+0000", 2023-09-23T23:24:57.231 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98206406999999996 2023-09-23T23:24:57.231 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.231 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.231 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.231 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.010633+0000", 2023-09-23T23:24:57.231 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6969999999999996e-06 2023-09-23T23:24:57.231 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.231 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.231 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.232 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.039173+0000", 2023-09-23T23:24:57.232 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.028539979 2023-09-23T23:24:57.232 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.232 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.232 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.232 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.039293+0000", 2023-09-23T23:24:57.232 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012015699999999999 2023-09-23T23:24:57.232 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.232 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.232 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.233 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.039472+0000", 2023-09-23T23:24:57.233 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017955599999999999 2023-09-23T23:24:57.233 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.233 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.233 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.233 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.042608+0000", 2023-09-23T23:24:57.233 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031361470000000002 2023-09-23T23:24:57.233 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.233 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.233 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.234 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.046854+0000", 2023-09-23T23:24:57.234 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0042457249999999997 2023-09-23T23:24:57.234 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.234 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.234 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.234 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.049104+0000", 2023-09-23T23:24:57.235 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002249399 2023-09-23T23:24:57.235 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.235 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.235 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:57.235 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.049173+0000", 2023-09-23T23:24:57.235 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9838000000000005e-05 2023-09-23T23:24:57.235 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.235 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.235 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.235 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.049582+0000", 2023-09-23T23:24:57.235 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00040876500000000001 2023-09-23T23:24:57.236 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.236 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.236 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.236 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.236 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.236 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:15168 10.b 10:d4e254f1:::benchmark_data_smithi110_366450_object15167:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-09-23T23:24:57.236 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:16.032401+0000", 2023-09-23T23:24:57.236 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.13216546400002, 2023-09-23T23:24:57.237 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0176874630000001, 2023-09-23T23:24:57.237 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.237 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:57.237 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.237 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.237 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.237 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 15168 2023-09-23T23:24:57.237 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.237 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.237 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.238 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.238 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.032401+0000", 2023-09-23T23:24:57.238 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.238 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.238 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.238 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.238 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.032401+0000", 2023-09-23T23:24:57.239 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.239 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.239 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.239 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.239 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.032401+0000", 2023-09-23T23:24:57.239 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:24:57.239 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.239 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.239 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.239 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.032532+0000", 2023-09-23T23:24:57.239 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000131505 2023-09-23T23:24:57.240 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.240 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.240 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.240 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.032541+0000", 2023-09-23T23:24:57.240 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4530000000000002e-06 2023-09-23T23:24:57.240 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.240 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.241 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.241 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.032546+0000", 2023-09-23T23:24:57.241 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0549999999999999e-06 2023-09-23T23:24:57.241 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.241 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.241 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.241 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.032608+0000", 2023-09-23T23:24:57.241 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2310999999999993e-05 2023-09-23T23:24:57.241 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.242 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.242 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.242 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.032617+0000", 2023-09-23T23:24:57.242 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6219999999999998e-06 2023-09-23T23:24:57.242 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.242 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.242 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.242 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.010655+0000", 2023-09-23T23:24:57.243 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97803820799999996 2023-09-23T23:24:57.243 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.243 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.243 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.243 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.010662+0000", 2023-09-23T23:24:57.243 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8109999999999997e-06 2023-09-23T23:24:57.243 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.244 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.244 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.039985+0000", 2023-09-23T23:24:57.244 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.029323027 2023-09-23T23:24:57.244 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.244 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.245 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.040120+0000", 2023-09-23T23:24:57.245 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013592699999999999 2023-09-23T23:24:57.245 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.245 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.245 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.245 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.040318+0000", 2023-09-23T23:24:57.245 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000197111 2023-09-23T23:24:57.246 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.246 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.246 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.246 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.042612+0000", 2023-09-23T23:24:57.246 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00229444 2023-09-23T23:24:57.246 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.246 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.246 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.246 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.048352+0000", 2023-09-23T23:24:57.247 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0057402110000000003 2023-09-23T23:24:57.247 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.247 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.247 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.247 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.049666+0000", 2023-09-23T23:24:57.247 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001314207 2023-09-23T23:24:57.247 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.247 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.247 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:57.247 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.049700+0000", 2023-09-23T23:24:57.248 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3315000000000002e-05 2023-09-23T23:24:57.248 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.248 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.248 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.248 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.050089+0000", 2023-09-23T23:24:57.248 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00038913499999999998 2023-09-23T23:24:57.248 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.248 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.248 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.248 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.249 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.249 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:15177 10.b 10:d912c341:::benchmark_data_smithi110_366450_object15176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-09-23T23:24:57.249 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:16.036616+0000", 2023-09-23T23:24:57.249 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.12795073699999, 2023-09-23T23:24:57.249 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0139463360000001, 2023-09-23T23:24:57.249 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.249 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:57.249 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.249 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.249 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.249 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 15177 2023-09-23T23:24:57.250 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.250 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.250 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.250 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.250 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.036616+0000", 2023-09-23T23:24:57.250 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.250 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.250 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.250 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.250 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.036616+0000", 2023-09-23T23:24:57.251 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.251 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.251 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.251 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.251 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.036614+0000", 2023-09-23T23:24:57.251 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:24:57.251 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.251 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.251 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.251 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.036653+0000", 2023-09-23T23:24:57.251 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8695999999999997e-05 2023-09-23T23:24:57.252 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.252 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.252 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.252 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.036664+0000", 2023-09-23T23:24:57.252 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0779e-05 2023-09-23T23:24:57.252 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.252 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.252 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.252 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.036670+0000", 2023-09-23T23:24:57.252 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0440000000000001e-06 2023-09-23T23:24:57.253 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.253 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.253 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.253 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.036729+0000", 2023-09-23T23:24:57.253 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9119000000000001e-05 2023-09-23T23:24:57.253 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.253 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.253 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.253 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.036739+0000", 2023-09-23T23:24:57.253 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6560000000000004e-06 2023-09-23T23:24:57.253 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.254 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.254 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.254 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.010688+0000", 2023-09-23T23:24:57.254 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97394888099999999 2023-09-23T23:24:57.254 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.254 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.254 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.254 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.010695+0000", 2023-09-23T23:24:57.254 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6569999999999997e-06 2023-09-23T23:24:57.254 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.255 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.255 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.255 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.040937+0000", 2023-09-23T23:24:57.255 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0302416190000001 2023-09-23T23:24:57.255 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.255 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.255 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.255 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.041072+0000", 2023-09-23T23:24:57.255 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013539300000000001 2023-09-23T23:24:57.255 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.255 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.256 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.256 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.041219+0000", 2023-09-23T23:24:57.256 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000146732 2023-09-23T23:24:57.256 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.256 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.256 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.256 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.043396+0000", 2023-09-23T23:24:57.256 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021767829999999998 2023-09-23T23:24:57.256 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.256 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.257 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.257 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.048401+0000", 2023-09-23T23:24:57.257 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050048710000000001 2023-09-23T23:24:57.257 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.257 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.257 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.257 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.050142+0000", 2023-09-23T23:24:57.257 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0017414380000000001 2023-09-23T23:24:57.257 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.257 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.257 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:57.258 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.050196+0000", 2023-09-23T23:24:57.258 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4373000000000003e-05 2023-09-23T23:24:57.258 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.258 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.258 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.258 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.050563+0000", 2023-09-23T23:24:57.258 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00036611499999999997 2023-09-23T23:24:57.258 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.258 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.258 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.258 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.259 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.259 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:15184 10.b 10:de14126f:::benchmark_data_smithi110_366450_object15183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-09-23T23:24:57.259 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:16.037310+0000", 2023-09-23T23:24:57.259 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.12725663499998, 2023-09-23T23:24:57.259 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0140428030000002, 2023-09-23T23:24:57.259 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.259 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:57.259 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.259 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.259 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.260 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 15184 2023-09-23T23:24:57.260 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.260 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.260 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.260 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.260 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.037310+0000", 2023-09-23T23:24:57.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.260 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.260 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.260 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.260 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.037310+0000", 2023-09-23T23:24:57.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.037310+0000", 2023-09-23T23:24:57.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:24:57.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.037374+0000", 2023-09-23T23:24:57.262 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4197999999999998e-05 2023-09-23T23:24:57.262 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.262 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.262 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.037379+0000", 2023-09-23T23:24:57.262 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4299999999999997e-06 2023-09-23T23:24:57.262 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.262 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.262 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.037383+0000", 2023-09-23T23:24:57.263 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2499999999999998e-06 2023-09-23T23:24:57.263 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.263 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.263 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.263 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.037428+0000", 2023-09-23T23:24:57.263 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5544000000000003e-05 2023-09-23T23:24:57.263 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.263 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.263 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.263 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.037436+0000", 2023-09-23T23:24:57.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0600000000000008e-06 2023-09-23T23:24:57.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.264 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.264 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.010719+0000", 2023-09-23T23:24:57.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97328315499999996 2023-09-23T23:24:57.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.264 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.264 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.010726+0000", 2023-09-23T23:24:57.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1250000000000004e-06 2023-09-23T23:24:57.265 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.265 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.265 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.041771+0000", 2023-09-23T23:24:57.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0310450120000001 2023-09-23T23:24:57.265 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.266 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.266 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.041922+0000", 2023-09-23T23:24:57.266 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015087600000000001 2023-09-23T23:24:57.266 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.266 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.266 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.266 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.042083+0000", 2023-09-23T23:24:57.266 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000160903 2023-09-23T23:24:57.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.267 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.046751+0000", 2023-09-23T23:24:57.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0046679929999999996 2023-09-23T23:24:57.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.267 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.049628+0000", 2023-09-23T23:24:57.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028766769999999998 2023-09-23T23:24:57.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.268 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.050846+0000", 2023-09-23T23:24:57.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012184030000000001 2023-09-23T23:24:57.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:57.268 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.050914+0000", 2023-09-23T23:24:57.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8145000000000006e-05 2023-09-23T23:24:57.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.051353+0000", 2023-09-23T23:24:57.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00043867199999999998 2023-09-23T23:24:57.269 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.269 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.269 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.270 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:15196 10.0 10:042eccec:::benchmark_data_smithi110_366450_object15195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-09-23T23:24:57.270 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:19:16.041688+0000", 2023-09-23T23:24:57.270 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.12287932599997, 2023-09-23T23:24:57.270 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.005164663, 2023-09-23T23:24:57.270 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.270 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:57.270 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.271 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.271 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.271 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 15196 2023-09-23T23:24:57.271 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.271 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.271 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.271 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.271 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.041688+0000", 2023-09-23T23:24:57.271 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.272 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.272 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.272 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.041688+0000", 2023-09-23T23:24:57.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.272 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.273 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.273 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.041687+0000", 2023-09-23T23:24:57.273 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:24:57.273 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.273 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.273 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.041799+0000", 2023-09-23T23:24:57.273 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011260900000000001 2023-09-23T23:24:57.273 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.274 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.041809+0000", 2023-09-23T23:24:57.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3400000000000004e-06 2023-09-23T23:24:57.274 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.274 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.274 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.041814+0000", 2023-09-23T23:24:57.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4389999999999999e-06 2023-09-23T23:24:57.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.275 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.275 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.041879+0000", 2023-09-23T23:24:57.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5485000000000001e-05 2023-09-23T23:24:57.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:16.041895+0000", 2023-09-23T23:24:57.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5469e-05 2023-09-23T23:24:57.276 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.009078+0000", 2023-09-23T23:24:57.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96718263800000004 2023-09-23T23:24:57.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.277 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.277 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.277 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:17.009087+0000", 2023-09-23T23:24:57.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3549999999999994e-06 2023-09-23T23:24:57.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.277 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.278 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.278 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.041324+0000", 2023-09-23T23:24:57.278 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0322372390000001 2023-09-23T23:24:57.278 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.278 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.278 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.041460+0000", 2023-09-23T23:24:57.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013556199999999999 2023-09-23T23:24:57.279 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.279 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.041645+0000", 2023-09-23T23:24:57.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018549899999999999 2023-09-23T23:24:57.279 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.280 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.280 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.045237+0000", 2023-09-23T23:24:57.280 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003591463 2023-09-23T23:24:57.280 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.280 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.280 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.280 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.046090+0000", 2023-09-23T23:24:57.280 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00085317300000000004 2023-09-23T23:24:57.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.281 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.281 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.281 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.046285+0000", 2023-09-23T23:24:57.281 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019532300000000001 2023-09-23T23:24:57.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.281 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.281 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:57.281 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.046337+0000", 2023-09-23T23:24:57.281 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1409000000000002e-05 2023-09-23T23:24:57.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.282 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.282 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:19:18.046852+0000", 2023-09-23T23:24:57.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00051566999999999997 2023-09-23T23:24:57.282 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.282 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.282 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.282 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.282 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:92240 10.0 10:0ba3daac:::benchmark_data_smithi110_366450_object92239:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:24:57.283 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:25.873807+0000", 2023-09-23T23:24:57.283 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 271.29075977899998, 2023-09-23T23:24:57.283 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0193927490000001, 2023-09-23T23:24:57.283 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.283 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:57.283 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.283 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.283 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.283 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92240 2023-09-23T23:24:57.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.284 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873807+0000", 2023-09-23T23:24:57.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873807+0000", 2023-09-23T23:24:57.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873806+0000", 2023-09-23T23:24:57.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:24:57.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873834+0000", 2023-09-23T23:24:57.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8551000000000002e-05 2023-09-23T23:24:57.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.286 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.286 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873844+0000", 2023-09-23T23:24:57.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0769999999999997e-06 2023-09-23T23:24:57.286 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.286 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.286 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.286 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873849+0000", 2023-09-23T23:24:57.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3639999999999998e-06 2023-09-23T23:24:57.286 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.286 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.287 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873908+0000", 2023-09-23T23:24:57.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8962000000000002e-05 2023-09-23T23:24:57.287 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.287 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873916+0000", 2023-09-23T23:24:57.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8210000000000002e-06 2023-09-23T23:24:57.287 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:26.840320+0000", 2023-09-23T23:24:57.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96640448599999995 2023-09-23T23:24:57.288 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.288 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.288 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:26.840332+0000", 2023-09-23T23:24:57.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1807000000000001e-05 2023-09-23T23:24:57.288 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.288 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.288 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.289 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.862906+0000", 2023-09-23T23:24:57.289 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0225740700000001 2023-09-23T23:24:57.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.289 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.289 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.863009+0000", 2023-09-23T23:24:57.289 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103318 2023-09-23T23:24:57.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.289 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.289 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.863135+0000", 2023-09-23T23:24:57.290 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000125955 2023-09-23T23:24:57.290 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.290 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.290 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.290 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.872367+0000", 2023-09-23T23:24:57.290 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0092317479999999997 2023-09-23T23:24:57.290 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.290 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.290 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.892992+0000", 2023-09-23T23:24:57.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020624462 2023-09-23T23:24:57.291 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.291 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.291 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893090+0000", 2023-09-23T23:24:57.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8231000000000002e-05 2023-09-23T23:24:57.291 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.291 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.291 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:57.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893126+0000", 2023-09-23T23:24:57.292 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6010000000000003e-05 2023-09-23T23:24:57.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.292 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.292 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.292 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893200+0000", 2023-09-23T23:24:57.292 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4126000000000001e-05 2023-09-23T23:24:57.292 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.292 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.292 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.293 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:92247 10.0 10:0876996b:::benchmark_data_smithi110_366450_object92246:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:24:57.293 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:25.877726+0000", 2023-09-23T23:24:57.293 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 271.28684096500001, 2023-09-23T23:24:57.293 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.01571897, 2023-09-23T23:24:57.293 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.293 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:57.293 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.293 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.293 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.294 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92247 2023-09-23T23:24:57.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.294 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.294 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.294 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877726+0000", 2023-09-23T23:24:57.294 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.294 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.295 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877726+0000", 2023-09-23T23:24:57.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.295 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.295 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.295 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877726+0000", 2023-09-23T23:24:57.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:24:57.295 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.295 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.295 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877760+0000", 2023-09-23T23:24:57.296 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.413e-05 2023-09-23T23:24:57.296 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.296 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.296 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877767+0000", 2023-09-23T23:24:57.296 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7910000000000005e-06 2023-09-23T23:24:57.296 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.296 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.296 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877773+0000", 2023-09-23T23:24:57.296 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3190000000000002e-06 2023-09-23T23:24:57.297 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.297 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.297 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.297 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877855+0000", 2023-09-23T23:24:57.297 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.242e-05 2023-09-23T23:24:57.297 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.297 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.297 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.297 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877864+0000", 2023-09-23T23:24:57.297 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8929999999999994e-06 2023-09-23T23:24:57.298 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.298 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.298 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.298 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:26.840358+0000", 2023-09-23T23:24:57.298 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96249378399999996 2023-09-23T23:24:57.298 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.298 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.298 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.298 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:26.840365+0000", 2023-09-23T23:24:57.298 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0960000000000003e-06 2023-09-23T23:24:57.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.299 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.299 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.865238+0000", 2023-09-23T23:24:57.299 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0248726539999999 2023-09-23T23:24:57.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.299 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.299 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.865348+0000", 2023-09-23T23:24:57.299 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110773 2023-09-23T23:24:57.300 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.300 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.865452+0000", 2023-09-23T23:24:57.300 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103787 2023-09-23T23:24:57.300 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.300 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.301 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.872373+0000", 2023-09-23T23:24:57.301 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069208259999999997 2023-09-23T23:24:57.301 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.301 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.301 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.301 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893034+0000", 2023-09-23T23:24:57.301 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020660586000000002 2023-09-23T23:24:57.301 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.301 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.301 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.302 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893351+0000", 2023-09-23T23:24:57.302 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00031712999999999999 2023-09-23T23:24:57.302 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.302 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.302 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:57.302 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893383+0000", 2023-09-23T23:24:57.302 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2360000000000002e-05 2023-09-23T23:24:57.302 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.303 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.303 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.303 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893445+0000", 2023-09-23T23:24:57.303 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1867999999999998e-05 2023-09-23T23:24:57.303 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.303 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.303 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.303 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.303 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.304 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:92255 10.0 10:047e84c9:::benchmark_data_smithi110_366450_object92254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:24:57.304 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:25.889232+0000", 2023-09-23T23:24:57.304 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 271.27533540600001, 2023-09-23T23:24:57.304 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0043332720000002, 2023-09-23T23:24:57.304 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.304 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:57.304 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.304 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.304 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.305 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92255 2023-09-23T23:24:57.305 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.305 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.305 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.305 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.305 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889232+0000", 2023-09-23T23:24:57.305 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.305 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.305 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.306 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.306 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889232+0000", 2023-09-23T23:24:57.306 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.306 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.306 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.306 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.306 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889230+0000", 2023-09-23T23:24:57.307 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:24:57.307 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.307 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.307 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.307 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889267+0000", 2023-09-23T23:24:57.307 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7274000000000003e-05 2023-09-23T23:24:57.307 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.307 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.307 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.308 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889274+0000", 2023-09-23T23:24:57.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1899999999999998e-06 2023-09-23T23:24:57.308 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.308 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.308 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889279+0000", 2023-09-23T23:24:57.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9640000000000002e-06 2023-09-23T23:24:57.308 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.309 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.309 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889353+0000", 2023-09-23T23:24:57.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3460000000000005e-05 2023-09-23T23:24:57.309 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.309 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.309 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.309 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889362+0000", 2023-09-23T23:24:57.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2520000000000005e-06 2023-09-23T23:24:57.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.310 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.310 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:26.840390+0000", 2023-09-23T23:24:57.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95102806500000003 2023-09-23T23:24:57.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:26.840398+0000", 2023-09-23T23:24:57.311 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2840000000000006e-06 2023-09-23T23:24:57.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.867281+0000", 2023-09-23T23:24:57.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.026882686 2023-09-23T23:24:57.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.312 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.867395+0000", 2023-09-23T23:24:57.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000113608 2023-09-23T23:24:57.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.313 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.313 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.867496+0000", 2023-09-23T23:24:57.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101588 2023-09-23T23:24:57.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.313 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.313 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.876926+0000", 2023-09-23T23:24:57.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0094297870000000002 2023-09-23T23:24:57.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893056+0000", 2023-09-23T23:24:57.314 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016130228999999999 2023-09-23T23:24:57.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893478+0000", 2023-09-23T23:24:57.314 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00042173999999999998 2023-09-23T23:24:57.315 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.315 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:57.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893511+0000", 2023-09-23T23:24:57.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3319999999999999e-05 2023-09-23T23:24:57.315 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.315 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893565+0000", 2023-09-23T23:24:57.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3349e-05 2023-09-23T23:24:57.315 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.316 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.316 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.316 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:156641 10.0 10:00af6c9f:::benchmark_data_smithi110_366450_object55433:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T23:24:57.316 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:49.023609+0000", 2023-09-23T23:24:57.316 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 188.1409578, 2023-09-23T23:24:57.316 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0481008119999999, 2023-09-23T23:24:57.316 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.316 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:24:57.317 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.317 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.317 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.317 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 156641 2023-09-23T23:24:57.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.317 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.317 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.023609+0000", 2023-09-23T23:24:57.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.318 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.318 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.318 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.023609+0000", 2023-09-23T23:24:57.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.318 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.318 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.318 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.318 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.023609+0000", 2023-09-23T23:24:57.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:24:57.318 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.319 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.319 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.023611+0000", 2023-09-23T23:24:57.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0980000000000001e-06 2023-09-23T23:24:57.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.319 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.319 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.023616+0000", 2023-09-23T23:24:57.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4310000000000002e-06 2023-09-23T23:24:57.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.023620+0000", 2023-09-23T23:24:57.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8460000000000001e-06 2023-09-23T23:24:57.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.026676+0000", 2023-09-23T23:24:57.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030557929999999998 2023-09-23T23:24:57.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.026794+0000", 2023-09-23T23:24:57.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011783200000000001 2023-09-23T23:24:57.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.026924+0000", 2023-09-23T23:24:57.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012984199999999999 2023-09-23T23:24:57.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.030162+0000", 2023-09-23T23:24:57.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003238239 2023-09-23T23:24:57.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.039529+0000", 2023-09-23T23:24:57.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0093669070000000007 2023-09-23T23:24:57.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.039554+0000", 2023-09-23T23:24:57.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5494999999999999e-05 2023-09-23T23:24:57.323 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.323 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.323 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.323 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.036786+0000", 2023-09-23T23:24:57.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99723147700000003 2023-09-23T23:24:57.323 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.323 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.323 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.323 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.036802+0000", 2023-09-23T23:24:57.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6588000000000001e-05 2023-09-23T23:24:57.323 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.324 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.324 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.324 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071642+0000", 2023-09-23T23:24:57.324 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0348391619999999 2023-09-23T23:24:57.324 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.324 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.325 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.325 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071710+0000", 2023-09-23T23:24:57.325 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8406999999999997e-05 2023-09-23T23:24:57.325 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.325 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.325 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.325 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.325 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.326 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:156645 10.0 10:09d04cd1:::benchmark_data_smithi110_366450_object55437:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T23:24:57.326 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:49.033190+0000", 2023-09-23T23:24:57.326 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 188.13137673899999, 2023-09-23T23:24:57.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0386241300000001, 2023-09-23T23:24:57.326 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.326 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:24:57.326 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.326 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.326 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.327 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 156645 2023-09-23T23:24:57.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.327 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033190+0000", 2023-09-23T23:24:57.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033190+0000", 2023-09-23T23:24:57.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033189+0000", 2023-09-23T23:24:57.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:24:57.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033195+0000", 2023-09-23T23:24:57.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5199999999999997e-06 2023-09-23T23:24:57.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.329 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.329 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033206+0000", 2023-09-23T23:24:57.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1522000000000001e-05 2023-09-23T23:24:57.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.329 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.329 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033213+0000", 2023-09-23T23:24:57.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5540000000000003e-06 2023-09-23T23:24:57.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033436+0000", 2023-09-23T23:24:57.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00022373599999999999 2023-09-23T23:24:57.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033545+0000", 2023-09-23T23:24:57.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108587 2023-09-23T23:24:57.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.331 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.331 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033657+0000", 2023-09-23T23:24:57.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112044 2023-09-23T23:24:57.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.331 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.331 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.039584+0000", 2023-09-23T23:24:57.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0059267770000000003 2023-09-23T23:24:57.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.039591+0000", 2023-09-23T23:24:57.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6930000000000007e-06 2023-09-23T23:24:57.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.036888+0000", 2023-09-23T23:24:57.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99729603 2023-09-23T23:24:57.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.036897+0000", 2023-09-23T23:24:57.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3370000000000006e-06 2023-09-23T23:24:57.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071737+0000", 2023-09-23T23:24:57.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0348404870000001 2023-09-23T23:24:57.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071814+0000", 2023-09-23T23:24:57.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6959999999999995e-05 2023-09-23T23:24:57.334 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.335 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.335 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.335 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:156648 10.b 10:d34329b9:::benchmark_data_smithi110_366450_object55440:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T23:24:57.335 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:49.042527+0000", 2023-09-23T23:24:57.335 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 188.12203961399999, 2023-09-23T23:24:57.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0423278210000002, 2023-09-23T23:24:57.336 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.336 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:57.336 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.336 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.336 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.336 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 156648 2023-09-23T23:24:57.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.336 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.042527+0000", 2023-09-23T23:24:57.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.042527+0000", 2023-09-23T23:24:57.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.042526+0000", 2023-09-23T23:24:57.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:24:57.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.042530+0000", 2023-09-23T23:24:57.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3000000000000002e-06 2023-09-23T23:24:57.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.042538+0000", 2023-09-23T23:24:57.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1200000000000002e-06 2023-09-23T23:24:57.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.042543+0000", 2023-09-23T23:24:57.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8069999999999999e-06 2023-09-23T23:24:57.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.340 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.050117+0000", 2023-09-23T23:24:57.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0075745989999999996 2023-09-23T23:24:57.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.340 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.050123+0000", 2023-09-23T23:24:57.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.942e-06 2023-09-23T23:24:57.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.038821+0000", 2023-09-23T23:24:57.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98869784299999997 2023-09-23T23:24:57.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.038828+0000", 2023-09-23T23:24:57.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1910000000000003e-06 2023-09-23T23:24:57.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.067513+0000", 2023-09-23T23:24:57.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0286844479999999 2023-09-23T23:24:57.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.067622+0000", 2023-09-23T23:24:57.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109442 2023-09-23T23:24:57.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.068137+0000", 2023-09-23T23:24:57.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00051531799999999996 2023-09-23T23:24:57.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.083787+0000", 2023-09-23T23:24:57.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015649676000000001 2023-09-23T23:24:57.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.084665+0000", 2023-09-23T23:24:57.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000877664 2023-09-23T23:24:57.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.084720+0000", 2023-09-23T23:24:57.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5492999999999999e-05 2023-09-23T23:24:57.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:57.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.084768+0000", 2023-09-23T23:24:57.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7715e-05 2023-09-23T23:24:57.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.084855+0000", 2023-09-23T23:24:57.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7238999999999996e-05 2023-09-23T23:24:57.347 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.347 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.348 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.348 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.348 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:156656 10.0 10:03429159:::benchmark_data_smithi110_366450_object55448:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:24:57.348 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:49.049592+0000", 2023-09-23T23:24:57.348 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 188.114974553, 2023-09-23T23:24:57.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.030975679, 2023-09-23T23:24:57.348 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.348 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:57.348 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.349 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.349 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.349 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 156656 2023-09-23T23:24:57.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.349 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.049592+0000", 2023-09-23T23:24:57.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.350 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.350 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.049592+0000", 2023-09-23T23:24:57.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.350 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.350 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.049591+0000", 2023-09-23T23:24:57.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:24:57.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.049597+0000", 2023-09-23T23:24:57.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.331e-06 2023-09-23T23:24:57.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.049609+0000", 2023-09-23T23:24:57.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2109e-05 2023-09-23T23:24:57.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.049614+0000", 2023-09-23T23:24:57.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.13e-06 2023-09-23T23:24:57.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.050086+0000", 2023-09-23T23:24:57.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00047189900000000002 2023-09-23T23:24:57.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.050095+0000", 2023-09-23T23:24:57.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4069999999999993e-06 2023-09-23T23:24:57.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.037453+0000", 2023-09-23T23:24:57.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98735800600000001 2023-09-23T23:24:57.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.037468+0000", 2023-09-23T23:24:57.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4931e-05 2023-09-23T23:24:57.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071833+0000", 2023-09-23T23:24:57.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0343654170000001 2023-09-23T23:24:57.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071917+0000", 2023-09-23T23:24:57.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4394000000000005e-05 2023-09-23T23:24:57.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.072419+0000", 2023-09-23T23:24:57.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00050144499999999999 2023-09-23T23:24:57.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.079021+0000", 2023-09-23T23:24:57.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066016110000000003 2023-09-23T23:24:57.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.079064+0000", 2023-09-23T23:24:57.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3686999999999997e-05 2023-09-23T23:24:57.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.080466+0000", 2023-09-23T23:24:57.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014013280000000001 2023-09-23T23:24:57.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:57.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.080504+0000", 2023-09-23T23:24:57.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8420000000000001e-05 2023-09-23T23:24:57.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.080568+0000", 2023-09-23T23:24:57.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4061999999999998e-05 2023-09-23T23:24:57.358 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.358 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.358 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.358 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.358 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:156674 10.b 10:d8537923:::benchmark_data_smithi110_366450_object55466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:24:57.358 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:49.070250+0000", 2023-09-23T23:24:57.358 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 188.09431653600001, 2023-09-23T23:24:57.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0147600799999998, 2023-09-23T23:24:57.358 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.359 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:57.359 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.359 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.359 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.359 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 156674 2023-09-23T23:24:57.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.359 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.359 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.359 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.359 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.070250+0000", 2023-09-23T23:24:57.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.070250+0000", 2023-09-23T23:24:57.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.070250+0000", 2023-09-23T23:24:57.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:24:57.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.070254+0000", 2023-09-23T23:24:57.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5349999999999998e-06 2023-09-23T23:24:57.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.070265+0000", 2023-09-23T23:24:57.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0645000000000001e-05 2023-09-23T23:24:57.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.070271+0000", 2023-09-23T23:24:57.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1720000000000004e-06 2023-09-23T23:24:57.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.070342+0000", 2023-09-23T23:24:57.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0698999999999995e-05 2023-09-23T23:24:57.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.070350+0000", 2023-09-23T23:24:57.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4679999999999992e-06 2023-09-23T23:24:57.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.038859+0000", 2023-09-23T23:24:57.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96850860999999999 2023-09-23T23:24:57.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.038867+0000", 2023-09-23T23:24:57.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4319999999999999e-06 2023-09-23T23:24:57.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071622+0000", 2023-09-23T23:24:57.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.032754452 2023-09-23T23:24:57.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.365 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071765+0000", 2023-09-23T23:24:57.365 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014328400000000001 2023-09-23T23:24:57.365 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.365 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.365 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071909+0000", 2023-09-23T23:24:57.365 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014438999999999999 2023-09-23T23:24:57.365 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.365 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.365 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.083795+0000", 2023-09-23T23:24:57.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01188547 2023-09-23T23:24:57.366 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.366 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.366 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.366 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.084693+0000", 2023-09-23T23:24:57.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00089860500000000002 2023-09-23T23:24:57.366 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.366 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.366 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.366 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.084890+0000", 2023-09-23T23:24:57.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019686399999999999 2023-09-23T23:24:57.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:57.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.084929+0000", 2023-09-23T23:24:57.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8413e-05 2023-09-23T23:24:57.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085010+0000", 2023-09-23T23:24:57.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1774999999999995e-05 2023-09-23T23:24:57.368 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.368 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.368 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.368 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:156675 10.b 10:d38caa2d:::benchmark_data_smithi110_366450_object55467:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:24:57.368 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:49.071507+0000", 2023-09-23T23:24:57.369 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 188.09305970599999, 2023-09-23T23:24:57.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0138194789999999, 2023-09-23T23:24:57.369 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.369 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:57.369 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.369 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 156675 2023-09-23T23:24:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.071507+0000", 2023-09-23T23:24:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.071507+0000", 2023-09-23T23:24:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.071507+0000", 2023-09-23T23:24:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:24:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.071510+0000", 2023-09-23T23:24:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.343e-06 2023-09-23T23:24:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.071518+0000", 2023-09-23T23:24:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2379999999999997e-06 2023-09-23T23:24:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.071522+0000", 2023-09-23T23:24:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6789999999999998e-06 2023-09-23T23:24:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.071556+0000", 2023-09-23T23:24:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3973999999999997e-05 2023-09-23T23:24:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.071561+0000", 2023-09-23T23:24:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8400000000000002e-06 2023-09-23T23:24:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.038901+0000", 2023-09-23T23:24:57.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96734012999999996 2023-09-23T23:24:57.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.038906+0000", 2023-09-23T23:24:57.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1560000000000003e-06 2023-09-23T23:24:57.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.074645+0000", 2023-09-23T23:24:57.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0357391819999999 2023-09-23T23:24:57.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.074755+0000", 2023-09-23T23:24:57.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109843 2023-09-23T23:24:57.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.074883+0000", 2023-09-23T23:24:57.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012784600000000001 2023-09-23T23:24:57.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.083844+0000", 2023-09-23T23:24:57.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0089609440000000002 2023-09-23T23:24:57.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.379 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.379 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085122+0000", 2023-09-23T23:24:57.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012780440000000001 2023-09-23T23:24:57.379 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.379 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085193+0000", 2023-09-23T23:24:57.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.114e-05 2023-09-23T23:24:57.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:57.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085236+0000", 2023-09-23T23:24:57.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3235000000000002e-05 2023-09-23T23:24:57.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085327+0000", 2023-09-23T23:24:57.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0464000000000007e-05 2023-09-23T23:24:57.381 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.381 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.381 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.381 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:156680 10.b 10:dfe97b19:::benchmark_data_smithi110_366450_object55472:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:24:57.381 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:49.075711+0000", 2023-09-23T23:24:57.382 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 188.08885558899999, 2023-09-23T23:24:57.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.009800915, 2023-09-23T23:24:57.382 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:24:57.382 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:24:57.382 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:24:57.382 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:24:57.382 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:24:57.382 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 156680 2023-09-23T23:24:57.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.382 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:24:57.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:24:57.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.075711+0000", 2023-09-23T23:24:57.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:24:57.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.075711+0000", 2023-09-23T23:24:57.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:24:57.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.384 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:24:57.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.075710+0000", 2023-09-23T23:24:57.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:24:57.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.384 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:24:57.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.075715+0000", 2023-09-23T23:24:57.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8489999999999997e-06 2023-09-23T23:24:57.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:24:57.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.075725+0000", 2023-09-23T23:24:57.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3479999999999993e-06 2023-09-23T23:24:57.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:24:57.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.075736+0000", 2023-09-23T23:24:57.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1620999999999999e-05 2023-09-23T23:24:57.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.386 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.386 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.075800+0000", 2023-09-23T23:24:57.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3490000000000004e-05 2023-09-23T23:24:57.386 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.386 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.386 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.075811+0000", 2023-09-23T23:24:57.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0876000000000001e-05 2023-09-23T23:24:57.386 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.038937+0000", 2023-09-23T23:24:57.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96312656399999996 2023-09-23T23:24:57.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:24:57.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.038945+0000", 2023-09-23T23:24:57.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6040000000000004e-06 2023-09-23T23:24:57.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:24:57.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.077861+0000", 2023-09-23T23:24:57.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0389166460000001 2023-09-23T23:24:57.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:24:57.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.078000+0000", 2023-09-23T23:24:57.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000139025 2023-09-23T23:24:57.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:24:57.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.078146+0000", 2023-09-23T23:24:57.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014589299999999999 2023-09-23T23:24:57.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:24:57.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.083849+0000", 2023-09-23T23:24:57.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0057028219999999998 2023-09-23T23:24:57.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085164+0000", 2023-09-23T23:24:57.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0013147160000000001 2023-09-23T23:24:57.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:24:57.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085392+0000", 2023-09-23T23:24:57.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000228641 2023-09-23T23:24:57.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:24:57.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085430+0000", 2023-09-23T23:24:57.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7920000000000003e-05 2023-09-23T23:24:57.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:24:57.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:24:57.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:24:57.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085512+0000", 2023-09-23T23:24:57.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1898999999999996e-05 2023-09-23T23:24:57.391 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.391 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.392 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.392 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:24:57.392 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:24:57.392 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:24:57.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:57.394 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:24:57.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:24:57.495 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:57.544 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:57.621 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:57.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:57.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:57.725 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:57.754 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:58.064 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:58.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:58.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:58.169 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 66 16 85938 85922 81.3001 56.25 0.00953483 0.0122951 2023-09-23T23:24:58.171 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:58.208 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:24:58.213 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:58.286 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:58.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:58.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:58.392 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:58.511 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:58.726 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:58.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:58.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:58.836 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:24:58.921 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:59.012 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:59 smithi110 ceph-mon[212939]: pgmap v1074: 119 pgs: 119 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-09-23T23:24:59.012 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:59 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:59.012 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:24:59 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:24:59.013 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:59 smithi110 ceph-mon[221102]: pgmap v1074: 119 pgs: 119 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-09-23T23:24:59.013 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:59 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:59.013 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:24:59 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:24:59.057 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:24:59.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:59.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:24:59.108 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:24:59.167 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 67 16 86813 86797 80.9009 54.6875 0.0132477 0.0123561 2023-09-23T23:24:59.171 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:24:59.203 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:59.271 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:24:59.277 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:59 smithi138 ceph-mon[193606]: pgmap v1074: 119 pgs: 119 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-09-23T23:24:59.277 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:59 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:24:59.277 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:24:59 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:24:59.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:59.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:24:59.378 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:24:59.394 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:24:59.447 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:59.583 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:24:59.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:59.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:24:59.698 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:24:59.733 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:24:59.798 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:24:59.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:24:59.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:24:59.907 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:24:59.985 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:00.009 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:00 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1458948759' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:25:00.010 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:00 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:25:00.057 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:25:00.168 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 68 16 87723 87707 80.5478 56.875 0.0135236 0.0124101 2023-09-23T23:25:00.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:00 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1458948759' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:25:00.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:00 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:25:00.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:00 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1458948759' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:25:00.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:00 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:25:00.710 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 2, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:25:00.710 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:25:00.710 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_6 2023-09-23T23:25:00.710 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:25:00.767 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:25:00.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:00.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:25:00.876 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:00.903 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:00.945 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:00.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:00.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:01.017 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:01 smithi110 ceph-mon[221102]: pgmap v1075: 119 pgs: 119 active+clean; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2023-09-23T23:25:01.018 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:01 smithi110 ceph-mon[212939]: pgmap v1075: 119 pgs: 119 active+clean; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2023-09-23T23:25:01.018 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:01 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:25:01.018 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:01 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:25:01.018 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:01 smithi110 ceph-mon[212939]: osdmap e362: 8 total, 8 up, 8 in 2023-09-23T23:25:01.018 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:01 smithi110 ceph-mon[212939]: failed to encode map e362 with expected crc 2023-09-23T23:25:01.018 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:01 smithi110 ceph-mon[212939]: failed to encode map e362 with expected crc 2023-09-23T23:25:01.019 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:01 smithi110 ceph-mon[212939]: failed to encode map e362 with expected crc 2023-09-23T23:25:01.019 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:01 smithi110 ceph-mon[212939]: failed to encode map e362 with expected crc 2023-09-23T23:25:01.052 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:25:01.245 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:01 smithi138 ceph-mon[193606]: pgmap v1075: 119 pgs: 119 active+clean; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2023-09-23T23:25:01.245 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:01 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:25:01.246 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:01 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:25:01.246 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:01 smithi138 ceph-mon[193606]: osdmap e362: 8 total, 8 up, 8 in 2023-09-23T23:25:01.246 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:01 smithi138 ceph-mon[193606]: failed to encode map e362 with expected crc 2023-09-23T23:25:01.246 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:01 smithi138 ceph-mon[193606]: failed to encode map e362 with expected crc 2023-09-23T23:25:01.246 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:01 smithi138 ceph-mon[193606]: failed to encode map e362 with expected crc 2023-09-23T23:25:01.246 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:01 smithi138 ceph-mon[193606]: failed to encode map e362 with expected crc 2023-09-23T23:25:01.247 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 69 16 88618 88602 80.1913 55.9375 0.00866983 0.0124657 2023-09-23T23:25:01.297 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:01.374 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:25:01.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:01.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:25:01.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:01 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:25:01.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:01 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:25:01.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:01 smithi110 ceph-mon[221102]: osdmap e362: 8 total, 8 up, 8 in 2023-09-23T23:25:01.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:01 smithi110 ceph-mon[221102]: failed to encode map e362 with expected crc 2023-09-23T23:25:01.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:01 smithi110 ceph-mon[221102]: failed to encode map e362 with expected crc 2023-09-23T23:25:01.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:01 smithi110 ceph-mon[221102]: failed to encode map e362 with expected crc 2023-09-23T23:25:01.459 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:01 smithi110 ceph-mon[221102]: failed to encode map e362 with expected crc 2023-09-23T23:25:01.494 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:01.528 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:01.585 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:01.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:01.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:01.694 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:25:01.737 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:01.751 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:01.807 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:25:01.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:01.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:25:01.915 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:25:01.996 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:02.086 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:02.168 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 70 16 89533 89517 79.8619 57.1875 0.0171298 0.0125156 2023-09-23T23:25:02.229 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:25:02.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:02.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:25:02.267 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:25:02.267 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:25:02.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.267 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35904.0:89593 11.4 e362/349 11:2f1cc2fb:::benchmark_data_smithi110_433746_object89592:head v 362'5705, mlcod=362'5703)", 2023-09-23T23:25:02.267 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:02.246362+0000", 2023-09-23T23:25:02.267 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0056118069999999999, 2023-09-23T23:25:02.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.005649055, 2023-09-23T23:25:02.267 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:02.268 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:25:02.268 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:02.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:02.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.246362+0000", 2023-09-23T23:25:02.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:02.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:02.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.246362+0000", 2023-09-23T23:25:02.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:02.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:02.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.246361+0000", 2023-09-23T23:25:02.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:25:02.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:02.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.246411+0000", 2023-09-23T23:25:02.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0269000000000003e-05 2023-09-23T23:25:02.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:02.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.246431+0000", 2023-09-23T23:25:02.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0316e-05 2023-09-23T23:25:02.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:02.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.246440+0000", 2023-09-23T23:25:02.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7320000000000004e-06 2023-09-23T23:25:02.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:02.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.249960+0000", 2023-09-23T23:25:02.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035202229999999998 2023-09-23T23:25:02.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:02.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.249986+0000", 2023-09-23T23:25:02.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5459000000000001e-05 2023-09-23T23:25:02.271 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:02.271 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:02.271 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:02.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.271 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35904.0:89595 11.4 e362/349)", 2023-09-23T23:25:02.272 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:02.246527+0000", 2023-09-23T23:25:02.272 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0054459950000000003, 2023-09-23T23:25:02.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055445240000000003, 2023-09-23T23:25:02.273 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:02.273 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:25:02.273 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:02.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:02.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.246527+0000", 2023-09-23T23:25:02.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:02.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:02.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.246527+0000", 2023-09-23T23:25:02.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:02.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:02.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.246527+0000", 2023-09-23T23:25:02.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:25:02.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:02.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.246561+0000", 2023-09-23T23:25:02.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4186000000000001e-05 2023-09-23T23:25:02.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:02.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.246567+0000", 2023-09-23T23:25:02.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.054e-06 2023-09-23T23:25:02.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:02.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.246572+0000", 2023-09-23T23:25:02.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4399999999999998e-06 2023-09-23T23:25:02.276 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:02.276 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:02.276 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:02.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.276 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35904.0:89596 11.c e362/349)", 2023-09-23T23:25:02.276 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:02.251121+0000", 2023-09-23T23:25:02.276 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00085220700000000001, 2023-09-23T23:25:02.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00099453200000000001, 2023-09-23T23:25:02.276 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:02.276 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:25:02.276 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:02.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.277 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:02.277 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.251121+0000", 2023-09-23T23:25:02.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:02.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.277 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:02.277 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.251121+0000", 2023-09-23T23:25:02.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:02.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.277 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:02.277 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.251120+0000", 2023-09-23T23:25:02.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:25:02.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:02.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.251143+0000", 2023-09-23T23:25:02.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2756e-05 2023-09-23T23:25:02.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:02.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.251158+0000", 2023-09-23T23:25:02.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5461999999999999e-05 2023-09-23T23:25:02.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:02.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:02.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:02.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:02.251166+0000", 2023-09-23T23:25:02.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2690000000000002e-06 2023-09-23T23:25:02.279 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:02.279 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:02.279 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:02.279 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:02.279 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-09-23T23:25:02.279 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 3 2023-09-23T23:25:02.279 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:25:02.339 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:02.408 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:03.168 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 71 16 90479 90463 79.5699 59.125 0.0345978 0.0125624 2023-09-23T23:25:03.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:03 smithi110 ceph-mon[212939]: pgmap v1077: 119 pgs: 119 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-09-23T23:25:03.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:03 smithi110 ceph-mon[221102]: pgmap v1077: 119 pgs: 119 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-09-23T23:25:03.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:03 smithi138 ceph-mon[193606]: pgmap v1077: 119 pgs: 119 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-09-23T23:25:03.540 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:25:03.752 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:03.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:03.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:03.860 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:03.946 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:04.168 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 72 16 91242 91226 79.1274 47.6875 0.0130731 0.0126338 2023-09-23T23:25:04.261 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:25:04.534 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:04.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:04.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:04.650 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:04.698 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:04.780 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:04.786 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:04 smithi138 ceph-mon[193606]: pgmap v1078: 119 pgs: 119 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s 2023-09-23T23:25:04.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:04.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:04.889 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:04.915 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:04.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:04 smithi110 ceph-mon[212939]: pgmap v1078: 119 pgs: 119 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s 2023-09-23T23:25:04.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:04 smithi110 ceph-mon[221102]: pgmap v1078: 119 pgs: 119 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s 2023-09-23T23:25:05.021 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:05.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:05.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:05.131 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:25:05.168 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 73 16 92032 92016 78.72 49.375 0.0256479 0.0126979 2023-09-23T23:25:05.183 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:05.286 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:25:05.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:05.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:25:05.399 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:25:05.508 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:05.654 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:25:05.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:05 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:25:05.662 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:05 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:25:05.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:05.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:25:05.767 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:25:05.857 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:05.914 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:05.922 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:05.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:05 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:25:05.979 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:25:05.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:05.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:25:06.018 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:25:06.060 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:25:06.090 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:25:06.108 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:25:06.147 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:25:06.169 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 74 16 92821 92805 78.3222 49.3125 0.0252358 0.0127628 2023-09-23T23:25:06.205 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:06.205 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:25:06.257 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:25:06.304 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:25:06.342 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:25:06.351 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:25:06.352 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:25:06.352 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:25:06.352 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:25:06.352 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:25:06.352 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:25:06.352 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:25:06.352 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:25:06.708 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:25:06.719 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:06 smithi110 ceph-mon[221102]: pgmap v1079: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-09-23T23:25:06.720 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:06 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/206180191' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:25:06.720 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:06 smithi110 ceph-mon[212939]: pgmap v1079: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-09-23T23:25:06.720 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:06 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/206180191' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:25:06.720 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:25:06.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:06 smithi138 ceph-mon[193606]: pgmap v1079: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-09-23T23:25:06.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:06 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/206180191' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:25:07.172 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 75 16 93607 93591 77.9298 49.125 0.00853875 0.0128274 2023-09-23T23:25:07.207 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:25:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:25:06] "GET /metrics HTTP/1.1" 200 36619 "" "Prometheus/2.43.0" 2023-09-23T23:25:07.804 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:25:07.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:07.810 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:07 smithi110 ceph-mon[212939]: from='client.35979 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:25:07.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:25:07.811 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:07 smithi110 ceph-mon[221102]: from='client.35979 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:25:07.912 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:07.949 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:07 smithi138 ceph-mon[193606]: from='client.35979 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:25:07.976 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:08.172 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 76 16 94510 94494 77.6471 56.4375 0.0210525 0.0128745 2023-09-23T23:25:08.216 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:08.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:08.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:08.324 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:08.414 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:08.524 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:08 smithi138 ceph-mon[193606]: pgmap v1080: 119 pgs: 119 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s 2023-09-23T23:25:08.594 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:08 smithi110 ceph-mon[212939]: pgmap v1080: 119 pgs: 119 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s 2023-09-23T23:25:08.594 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:08 smithi110 ceph-mon[221102]: pgmap v1080: 119 pgs: 119 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s 2023-09-23T23:25:09.037 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:09.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:09.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:09.101 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1079,"stamp":"2023-09-23T23:25:05.626393+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5949298592,"num_objects":90794,"num_object_clones":0,"num_object_copies":272382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":90831,"num_write_kb":5809309,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":1220,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92616,"ondisk_log_size":92616,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":360,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21537644,"kb_used_data":17536428,"kb_used_omap":29,"kb_used_meta":4001122,"kb_avail":728423572,"statfs":{"total":767960285184,"available":745905737728,"internally_reserved":0,"allocated":17957302272,"data_stored":17946788465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":4097149651},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,2,6,5,6,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":581566464,"num_objects":8874,"num_object_clones":0,"num_object_copies":26622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8874,"num_write_kb":567936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006982"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":628,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.769164+0000","last_change":"2023-09-23T23:23:58.058868+0000","last_active":"2023-09-23T23:24:51.769164+0000","last_peered":"2023-09-23T23:24:51.769164+0000","last_clean":"2023-09-23T23:24:51.769164+0000","last_became_active":"2023-09-23T23:23:45.456902+0000","last_became_peered":"2023-09-23T23:23:45.456902+0000","last_unstale":"2023-09-23T23:24:51.769164+0000","last_undegraded":"2023-09-23T23:24:51.769164+0000","last_fullsized":"2023-09-23T23:24:51.769164+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.058755+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.058755+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:52.935253+0000","scrub_duration":0.0039029870000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":396,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.769769+0000","last_change":"2023-09-23T23:22:38.439629+0000","last_active":"2023-09-23T23:24:51.769769+0000","last_peered":"2023-09-23T23:24:51.769769+0000","last_clean":"2023-09-23T23:24:51.769769+0000","last_became_active":"2023-09-23T23:22:38.439285+0000","last_became_peered":"2023-09-23T23:22:38.439285+0000","last_unstale":"2023-09-23T23:24:51.769769+0000","last_undegraded":"2023-09-23T23:24:51.769769+0000","last_fullsized":"2023-09-23T23:24:51.769769+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:34.431221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":330,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.755728+0000","last_change":"2023-09-23T23:23:43.511912+0000","last_active":"2023-09-23T23:24:51.755728+0000","last_peered":"2023-09-23T23:24:51.755728+0000","last_clean":"2023-09-23T23:24:51.755728+0000","last_became_active":"2023-09-23T23:23:43.511574+0000","last_became_peered":"2023-09-23T23:23:43.511574+0000","last_unstale":"2023-09-23T23:24:51.755728+0000","last_undegraded":"2023-09-23T23:24:51.755728+0000","last_fullsized":"2023-09-23T23:24:51.755728+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:16.286926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":451,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.765856+0000","last_change":"2023-09-23T23:23:50.459634+0000","last_active":"2023-09-23T23:24:51.765856+0000","last_peered":"2023-09-23T23:24:51.765856+0000","last_clean":"2023-09-23T23:24:51.765856+0000","last_became_active":"2023-09-23T23:23:45.453958+0000","last_became_peered":"2023-09-23T23:23:45.453958+0000","last_unstale":"2023-09-23T23:24:51.765856+0000","last_undegraded":"2023-09-23T23:24:51.765856+0000","last_fullsized":"2023-09-23T23:24:51.765856+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:56.563201+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":53,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:00.046230+0000","last_change":"2023-09-23T23:22:38.436383+0000","last_active":"2023-09-23T23:25:00.046230+0000","last_peered":"2023-09-23T23:25:00.046230+0000","last_clean":"2023-09-23T23:25:00.046230+0000","last_became_active":"2023-09-23T23:22:38.436062+0000","last_became_peered":"2023-09-23T23:22:38.436062+0000","last_unstale":"2023-09-23T23:25:00.046230+0000","last_undegraded":"2023-09-23T23:25:00.046230+0000","last_fullsized":"2023-09-23T23:25:00.046230+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:07.142420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":361,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.759085+0000","last_change":"2023-09-23T23:22:38.438239+0000","last_active":"2023-09-23T23:24:51.759085+0000","last_peered":"2023-09-23T23:24:51.759085+0000","last_clean":"2023-09-23T23:24:51.759085+0000","last_became_active":"2023-09-23T23:22:38.437359+0000","last_became_peered":"2023-09-23T23:22:38.437359+0000","last_unstale":"2023-09-23T23:24:51.759085+0000","last_undegraded":"2023-09-23T23:24:51.759085+0000","last_fullsized":"2023-09-23T23:24:51.759085+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:15:40.886267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":671,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.754203+0000","last_change":"2023-09-23T23:23:49.842338+0000","last_active":"2023-09-23T23:24:51.754203+0000","last_peered":"2023-09-23T23:24:51.754203+0000","last_clean":"2023-09-23T23:24:51.754203+0000","last_became_active":"2023-09-23T23:23:45.459515+0000","last_became_peered":"2023-09-23T23:23:45.459515+0000","last_unstale":"2023-09-23T23:24:51.754203+0000","last_undegraded":"2023-09-23T23:24:51.754203+0000","last_fullsized":"2023-09-23T23:24:51.754203+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.842272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.842272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:13:00.500804+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":348,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.755398+0000","last_change":"2023-09-23T23:20:15.738434+0000","last_active":"2023-09-23T23:24:51.755398+0000","last_peered":"2023-09-23T23:24:51.755398+0000","last_clean":"2023-09-23T23:24:51.755398+0000","last_became_active":"2023-09-23T23:20:15.738195+0000","last_became_peered":"2023-09-23T23:20:15.738195+0000","last_unstale":"2023-09-23T23:24:51.755398+0000","last_undegraded":"2023-09-23T23:24:51.755398+0000","last_fullsized":"2023-09-23T23:24:51.755398+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:15.224045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":341,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.763901+0000","last_change":"2023-09-23T23:22:38.436482+0000","last_active":"2023-09-23T23:24:51.763901+0000","last_peered":"2023-09-23T23:24:51.763901+0000","last_clean":"2023-09-23T23:24:51.763901+0000","last_became_active":"2023-09-23T23:22:38.436147+0000","last_became_peered":"2023-09-23T23:22:38.436147+0000","last_unstale":"2023-09-23T23:24:51.763901+0000","last_undegraded":"2023-09-23T23:24:51.763901+0000","last_fullsized":"2023-09-23T23:24:51.763901+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:30.542674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":682,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.754608+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:24:51.754608+0000","last_peered":"2023-09-23T23:24:51.754608+0000","last_clean":"2023-09-23T23:24:51.754608+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:24:51.754608+0000","last_undegraded":"2023-09-23T23:24:51.754608+0000","last_fullsized":"2023-09-23T23:24:51.754608+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":314,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.759461+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:24:51.759461+0000","last_peered":"2023-09-23T23:24:51.759461+0000","last_clean":"2023-09-23T23:24:51.759461+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:24:51.759461+0000","last_undegraded":"2023-09-23T23:24:51.759461+0000","last_fullsized":"2023-09-23T23:24:51.759461+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":331,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:58.314607+0000","last_change":"2023-09-23T23:22:38.450254+0000","last_active":"2023-09-23T23:24:58.314607+0000","last_peered":"2023-09-23T23:24:58.314607+0000","last_clean":"2023-09-23T23:24:58.314607+0000","last_became_active":"2023-09-23T23:22:38.449879+0000","last_became_peered":"2023-09-23T23:22:38.449879+0000","last_unstale":"2023-09-23T23:24:58.314607+0000","last_undegraded":"2023-09-23T23:24:58.314607+0000","last_fullsized":"2023-09-23T23:24:58.314607+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:02.777916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":616,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.769586+0000","last_change":"2023-09-23T23:23:56.137571+0000","last_active":"2023-09-23T23:24:51.769586+0000","last_peered":"2023-09-23T23:24:51.769586+0000","last_clean":"2023-09-23T23:24:51.769586+0000","last_became_active":"2023-09-23T23:23:45.459314+0000","last_became_peered":"2023-09-23T23:23:45.459314+0000","last_unstale":"2023-09-23T23:24:51.769586+0000","last_undegraded":"2023-09-23T23:24:51.769586+0000","last_fullsized":"2023-09-23T23:24:51.769586+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:14.751052+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":103,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.768237+0000","last_change":"2023-09-23T23:20:26.841909+0000","last_active":"2023-09-23T23:24:51.768237+0000","last_peered":"2023-09-23T23:24:51.768237+0000","last_clean":"2023-09-23T23:24:51.768237+0000","last_became_active":"2023-09-23T23:20:26.841631+0000","last_became_peered":"2023-09-23T23:20:26.841631+0000","last_unstale":"2023-09-23T23:24:51.768237+0000","last_undegraded":"2023-09-23T23:24:51.768237+0000","last_fullsized":"2023-09-23T23:24:51.768237+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:07.380539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":292,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:58.314646+0000","last_change":"2023-09-23T23:16:56.528057+0000","last_active":"2023-09-23T23:24:58.314646+0000","last_peered":"2023-09-23T23:24:58.314646+0000","last_clean":"2023-09-23T23:24:58.314646+0000","last_became_active":"2023-09-23T23:16:56.527755+0000","last_became_peered":"2023-09-23T23:16:56.527755+0000","last_unstale":"2023-09-23T23:24:58.314646+0000","last_undegraded":"2023-09-23T23:24:58.314646+0000","last_fullsized":"2023-09-23T23:24:58.314646+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:47.396406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":789,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.766179+0000","last_change":"2023-09-23T23:23:54.095122+0000","last_active":"2023-09-23T23:24:51.766179+0000","last_peered":"2023-09-23T23:24:51.766179+0000","last_clean":"2023-09-23T23:24:51.766179+0000","last_became_active":"2023-09-23T23:23:45.458110+0000","last_became_peered":"2023-09-23T23:23:45.458110+0000","last_unstale":"2023-09-23T23:24:51.766179+0000","last_undegraded":"2023-09-23T23:24:51.766179+0000","last_fullsized":"2023-09-23T23:24:51.766179+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:54.095054+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:54.095054+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:50.195093+0000","scrub_duration":0.0020267950000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":180,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.755887+0000","last_change":"2023-09-23T23:16:56.567128+0000","last_active":"2023-09-23T23:24:51.755887+0000","last_peered":"2023-09-23T23:24:51.755887+0000","last_clean":"2023-09-23T23:24:51.755887+0000","last_became_active":"2023-09-23T23:16:56.551484+0000","last_became_peered":"2023-09-23T23:16:56.551484+0000","last_unstale":"2023-09-23T23:24:51.755887+0000","last_undegraded":"2023-09-23T23:24:51.755887+0000","last_fullsized":"2023-09-23T23:24:51.755887+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:14.981307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":311,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.764927+0000","last_change":"2023-09-23T23:20:15.781637+0000","last_active":"2023-09-23T23:24:51.764927+0000","last_peered":"2023-09-23T23:24:51.764927+0000","last_clean":"2023-09-23T23:24:51.764927+0000","last_became_active":"2023-09-23T23:20:15.781308+0000","last_became_peered":"2023-09-23T23:20:15.781308+0000","last_unstale":"2023-09-23T23:24:51.764927+0000","last_undegraded":"2023-09-23T23:24:51.764927+0000","last_fullsized":"2023-09-23T23:24:51.764927+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:47.963872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":435,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.768277+0000","last_change":"2023-09-23T23:24:02.268079+0000","last_active":"2023-09-23T23:24:51.768277+0000","last_peered":"2023-09-23T23:24:51.768277+0000","last_clean":"2023-09-23T23:24:51.768277+0000","last_became_active":"2023-09-23T23:23:45.463106+0000","last_became_peered":"2023-09-23T23:23:45.463106+0000","last_unstale":"2023-09-23T23:24:51.768277+0000","last_undegraded":"2023-09-23T23:24:51.768277+0000","last_fullsized":"2023-09-23T23:24:51.768277+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.128813+0000","scrub_duration":0.0018535310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":77,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.754299+0000","last_change":"2023-09-23T23:23:36.017911+0000","last_active":"2023-09-23T23:24:51.754299+0000","last_peered":"2023-09-23T23:24:51.754299+0000","last_clean":"2023-09-23T23:24:51.754299+0000","last_became_active":"2023-09-23T23:23:36.017507+0000","last_became_peered":"2023-09-23T23:23:36.017507+0000","last_unstale":"2023-09-23T23:24:51.754299+0000","last_undegraded":"2023-09-23T23:24:51.754299+0000","last_fullsized":"2023-09-23T23:24:51.754299+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:49.095779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":340,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.770315+0000","last_change":"2023-09-23T23:22:38.443826+0000","last_active":"2023-09-23T23:24:51.770315+0000","last_peered":"2023-09-23T23:24:51.770315+0000","last_clean":"2023-09-23T23:24:51.770315+0000","last_became_active":"2023-09-23T23:22:38.443494+0000","last_became_peered":"2023-09-23T23:22:38.443494+0000","last_unstale":"2023-09-23T23:24:51.770315+0000","last_undegraded":"2023-09-23T23:24:51.770315+0000","last_fullsized":"2023-09-23T23:24:51.770315+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:55.630019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":514,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.755083+0000","last_change":"2023-09-23T23:23:53.828451+0000","last_active":"2023-09-23T23:24:51.755083+0000","last_peered":"2023-09-23T23:24:51.755083+0000","last_clean":"2023-09-23T23:24:51.755083+0000","last_became_active":"2023-09-23T23:23:45.459682+0000","last_became_peered":"2023-09-23T23:23:45.459682+0000","last_unstale":"2023-09-23T23:24:51.755083+0000","last_undegraded":"2023-09-23T23:24:51.755083+0000","last_fullsized":"2023-09-23T23:24:51.755083+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:53.828311+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:53.828311+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:19.729551+0000","scrub_duration":0.0047553140000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":262,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.754126+0000","last_change":"2023-09-23T23:23:36.026037+0000","last_active":"2023-09-23T23:24:51.754126+0000","last_peered":"2023-09-23T23:24:51.754126+0000","last_clean":"2023-09-23T23:24:51.754126+0000","last_became_active":"2023-09-23T23:23:36.025726+0000","last_became_peered":"2023-09-23T23:23:36.025726+0000","last_unstale":"2023-09-23T23:24:51.754126+0000","last_undegraded":"2023-09-23T23:24:51.754126+0000","last_fullsized":"2023-09-23T23:24:51.754126+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:28:48.287606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":348,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:58.314703+0000","last_change":"2023-09-23T23:23:36.032487+0000","last_active":"2023-09-23T23:24:58.314703+0000","last_peered":"2023-09-23T23:24:58.314703+0000","last_clean":"2023-09-23T23:24:58.314703+0000","last_became_active":"2023-09-23T23:23:36.032200+0000","last_became_peered":"2023-09-23T23:23:36.032200+0000","last_unstale":"2023-09-23T23:24:58.314703+0000","last_undegraded":"2023-09-23T23:24:58.314703+0000","last_fullsized":"2023-09-23T23:24:58.314703+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:29.843272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":661,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.753795+0000","last_change":"2023-09-23T23:23:50.876882+0000","last_active":"2023-09-23T23:24:51.753795+0000","last_peered":"2023-09-23T23:24:51.753795+0000","last_clean":"2023-09-23T23:24:51.753795+0000","last_became_active":"2023-09-23T23:23:45.457157+0000","last_became_peered":"2023-09-23T23:23:45.457157+0000","last_unstale":"2023-09-23T23:24:51.753795+0000","last_undegraded":"2023-09-23T23:24:51.753795+0000","last_fullsized":"2023-09-23T23:24:51.753795+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:30.732737+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":95,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:00.022684+0000","last_change":"2023-09-23T23:20:15.781738+0000","last_active":"2023-09-23T23:25:00.022684+0000","last_peered":"2023-09-23T23:25:00.022684+0000","last_clean":"2023-09-23T23:25:00.022684+0000","last_became_active":"2023-09-23T23:20:15.781394+0000","last_became_peered":"2023-09-23T23:20:15.781394+0000","last_unstale":"2023-09-23T23:25:00.022684+0000","last_undegraded":"2023-09-23T23:25:00.022684+0000","last_fullsized":"2023-09-23T23:25:00.022684+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:04.696118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":228,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.754673+0000","last_change":"2023-09-23T23:23:35.033379+0000","last_active":"2023-09-23T23:24:51.754673+0000","last_peered":"2023-09-23T23:24:51.754673+0000","last_clean":"2023-09-23T23:24:51.754673+0000","last_became_active":"2023-09-23T23:23:35.033069+0000","last_became_peered":"2023-09-23T23:23:35.033069+0000","last_unstale":"2023-09-23T23:24:51.754673+0000","last_undegraded":"2023-09-23T23:24:51.754673+0000","last_fullsized":"2023-09-23T23:24:51.754673+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:47.957081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":490,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:00.046985+0000","last_change":"2023-09-23T23:24:05.304978+0000","last_active":"2023-09-23T23:25:00.046985+0000","last_peered":"2023-09-23T23:25:00.046985+0000","last_clean":"2023-09-23T23:25:00.046985+0000","last_became_active":"2023-09-23T23:23:45.456982+0000","last_became_peered":"2023-09-23T23:23:45.456982+0000","last_unstale":"2023-09-23T23:25:00.046985+0000","last_undegraded":"2023-09-23T23:25:00.046985+0000","last_fullsized":"2023-09-23T23:25:00.046985+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:20.760502+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":206,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.753955+0000","last_change":"2023-09-23T23:23:35.028958+0000","last_active":"2023-09-23T23:24:51.753955+0000","last_peered":"2023-09-23T23:24:51.753955+0000","last_clean":"2023-09-23T23:24:51.753955+0000","last_became_active":"2023-09-23T23:23:35.028478+0000","last_became_peered":"2023-09-23T23:23:35.028478+0000","last_unstale":"2023-09-23T23:24:51.753955+0000","last_undegraded":"2023-09-23T23:24:51.753955+0000","last_fullsized":"2023-09-23T23:24:51.753955+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:44.960119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":341,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.759219+0000","last_change":"2023-09-23T23:20:15.737685+0000","last_active":"2023-09-23T23:24:51.759219+0000","last_peered":"2023-09-23T23:24:51.759219+0000","last_clean":"2023-09-23T23:24:51.759219+0000","last_became_active":"2023-09-23T23:20:15.737311+0000","last_became_peered":"2023-09-23T23:20:15.737311+0000","last_unstale":"2023-09-23T23:24:51.759219+0000","last_undegraded":"2023-09-23T23:24:51.759219+0000","last_fullsized":"2023-09-23T23:24:51.759219+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:15:21.213270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":645,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.766299+0000","last_change":"2023-09-23T23:23:48.533516+0000","last_active":"2023-09-23T23:24:51.766299+0000","last_peered":"2023-09-23T23:24:51.766299+0000","last_clean":"2023-09-23T23:24:51.766299+0000","last_became_active":"2023-09-23T23:23:45.458037+0000","last_became_peered":"2023-09-23T23:23:45.458037+0000","last_unstale":"2023-09-23T23:24:51.766299+0000","last_undegraded":"2023-09-23T23:24:51.766299+0000","last_fullsized":"2023-09-23T23:24:51.766299+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:47.822898+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":350,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.763323+0000","last_change":"2023-09-23T23:13:29.368175+0000","last_active":"2023-09-23T23:24:51.763323+0000","last_peered":"2023-09-23T23:24:51.763323+0000","last_clean":"2023-09-23T23:24:51.763323+0000","last_became_active":"2023-09-23T23:13:29.363603+0000","last_became_peered":"2023-09-23T23:13:29.363603+0000","last_unstale":"2023-09-23T23:24:51.763323+0000","last_undegraded":"2023-09-23T23:24:51.763323+0000","last_fullsized":"2023-09-23T23:24:51.763323+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:25.277871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":53,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:00.046760+0000","last_change":"2023-09-23T23:22:38.452672+0000","last_active":"2023-09-23T23:25:00.046760+0000","last_peered":"2023-09-23T23:25:00.046760+0000","last_clean":"2023-09-23T23:25:00.046760+0000","last_became_active":"2023-09-23T23:22:38.452374+0000","last_became_peered":"2023-09-23T23:22:38.452374+0000","last_unstale":"2023-09-23T23:25:00.046760+0000","last_undegraded":"2023-09-23T23:25:00.046760+0000","last_fullsized":"2023-09-23T23:25:00.046760+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:29.874430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.7","version":"362'5698","reported_seq":5719,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:04.552895+0000","last_change":"2023-09-23T23:23:43.505723+0000","last_active":"2023-09-23T23:25:04.552895+0000","last_peered":"2023-09-23T23:25:04.552895+0000","last_clean":"2023-09-23T23:25:04.552895+0000","last_became_active":"2023-09-23T23:23:43.505340+0000","last_became_peered":"2023-09-23T23:23:43.505340+0000","last_unstale":"2023-09-23T23:25:04.552895+0000","last_undegraded":"2023-09-23T23:25:04.552895+0000","last_fullsized":"2023-09-23T23:25:04.552895+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5698,"log_dups_size":0,"ondisk_log_size":5698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:33.987175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373424128,"num_objects":5698,"num_object_clones":0,"num_object_copies":17094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5698,"num_write_kb":364672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":683,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.767973+0000","last_change":"2023-09-23T23:23:57.267117+0000","last_active":"2023-09-23T23:24:51.767973+0000","last_peered":"2023-09-23T23:24:51.767973+0000","last_clean":"2023-09-23T23:24:51.767973+0000","last_became_active":"2023-09-23T23:23:45.458275+0000","last_became_peered":"2023-09-23T23:23:45.458275+0000","last_unstale":"2023-09-23T23:24:51.767973+0000","last_undegraded":"2023-09-23T23:24:51.767973+0000","last_fullsized":"2023-09-23T23:24:51.767973+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:47:09.088899+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":175,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.770474+0000","last_change":"2023-09-23T23:16:56.548535+0000","last_active":"2023-09-23T23:24:51.770474+0000","last_peered":"2023-09-23T23:24:51.770474+0000","last_clean":"2023-09-23T23:24:51.770474+0000","last_became_active":"2023-09-23T23:16:56.546115+0000","last_became_peered":"2023-09-23T23:16:56.546115+0000","last_unstale":"2023-09-23T23:24:51.770474+0000","last_undegraded":"2023-09-23T23:24:51.770474+0000","last_fullsized":"2023-09-23T23:24:51.770474+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:02.279715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":166,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.763409+0000","last_change":"2023-09-23T23:16:56.525479+0000","last_active":"2023-09-23T23:24:51.763409+0000","last_peered":"2023-09-23T23:24:51.763409+0000","last_clean":"2023-09-23T23:24:51.763409+0000","last_became_active":"2023-09-23T23:16:56.525147+0000","last_became_peered":"2023-09-23T23:16:56.525147+0000","last_unstale":"2023-09-23T23:24:51.763409+0000","last_undegraded":"2023-09-23T23:24:51.763409+0000","last_fullsized":"2023-09-23T23:24:51.763409+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:06.165300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.6","version":"362'5749","reported_seq":5770,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:05.201370+0000","last_change":"2023-09-23T23:23:43.514932+0000","last_active":"2023-09-23T23:25:05.201370+0000","last_peered":"2023-09-23T23:25:05.201370+0000","last_clean":"2023-09-23T23:25:05.201370+0000","last_became_active":"2023-09-23T23:23:43.513416+0000","last_became_peered":"2023-09-23T23:23:43.513416+0000","last_unstale":"2023-09-23T23:25:05.201370+0000","last_undegraded":"2023-09-23T23:25:05.201370+0000","last_fullsized":"2023-09-23T23:25:05.201370+0000","mapping_epoch":349,"log_start":"354'300","ondisk_log_start":"354'300","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5449,"log_dups_size":0,"ondisk_log_size":5449,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:41.201456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376766464,"num_objects":5749,"num_object_clones":0,"num_object_copies":17247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5749,"num_write_kb":367936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":706,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.755259+0000","last_change":"2023-09-23T23:23:48.906779+0000","last_active":"2023-09-23T23:24:51.755259+0000","last_peered":"2023-09-23T23:24:51.755259+0000","last_clean":"2023-09-23T23:24:51.755259+0000","last_became_active":"2023-09-23T23:23:45.454967+0000","last_became_peered":"2023-09-23T23:23:45.454967+0000","last_unstale":"2023-09-23T23:24:51.755259+0000","last_undegraded":"2023-09-23T23:24:51.755259+0000","last_fullsized":"2023-09-23T23:24:51.755259+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:02.570612+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":57,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:54.238063+0000","last_change":"2023-09-23T23:22:38.474057+0000","last_active":"2023-09-23T23:24:54.238063+0000","last_peered":"2023-09-23T23:24:54.238063+0000","last_clean":"2023-09-23T23:24:54.238063+0000","last_became_active":"2023-09-23T23:22:38.444672+0000","last_became_peered":"2023-09-23T23:22:38.444672+0000","last_unstale":"2023-09-23T23:24:54.238063+0000","last_undegraded":"2023-09-23T23:24:54.238063+0000","last_fullsized":"2023-09-23T23:24:54.238063+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:00.966999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":367,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.769631+0000","last_change":"2023-09-23T23:22:38.439077+0000","last_active":"2023-09-23T23:24:51.769631+0000","last_peered":"2023-09-23T23:24:51.769631+0000","last_clean":"2023-09-23T23:24:51.769631+0000","last_became_active":"2023-09-23T23:22:38.438638+0000","last_became_peered":"2023-09-23T23:22:38.438638+0000","last_unstale":"2023-09-23T23:24:51.769631+0000","last_undegraded":"2023-09-23T23:24:51.769631+0000","last_fullsized":"2023-09-23T23:24:51.769631+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:57.067702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"362'5726","reported_seq":5747,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:04.491832+0000","last_change":"2023-09-23T23:23:43.514636+0000","last_active":"2023-09-23T23:25:04.491832+0000","last_peered":"2023-09-23T23:25:04.491832+0000","last_clean":"2023-09-23T23:25:04.491832+0000","last_became_active":"2023-09-23T23:23:43.514352+0000","last_became_peered":"2023-09-23T23:23:43.514352+0000","last_unstale":"2023-09-23T23:25:04.491832+0000","last_undegraded":"2023-09-23T23:25:04.491832+0000","last_fullsized":"2023-09-23T23:25:04.491832+0000","mapping_epoch":349,"log_start":"355'800","ondisk_log_start":"355'800","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4926,"log_dups_size":0,"ondisk_log_size":4926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:06.752989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375259136,"num_objects":5726,"num_object_clones":0,"num_object_copies":17178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5726,"num_write_kb":366464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":776,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.763828+0000","last_change":"2023-09-23T23:23:57.042667+0000","last_active":"2023-09-23T23:24:51.763828+0000","last_peered":"2023-09-23T23:24:51.763828+0000","last_clean":"2023-09-23T23:24:51.763828+0000","last_became_active":"2023-09-23T23:23:45.454067+0000","last_became_peered":"2023-09-23T23:23:45.454067+0000","last_unstale":"2023-09-23T23:24:51.763828+0000","last_undegraded":"2023-09-23T23:24:51.763828+0000","last_fullsized":"2023-09-23T23:24:51.763828+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.042580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.042580+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:02:09.561802+0000","scrub_duration":0.0044137760000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":240,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.754767+0000","last_change":"2023-09-23T23:23:35.033371+0000","last_active":"2023-09-23T23:24:51.754767+0000","last_peered":"2023-09-23T23:24:51.754767+0000","last_clean":"2023-09-23T23:24:51.754767+0000","last_became_active":"2023-09-23T23:23:35.033032+0000","last_became_peered":"2023-09-23T23:23:35.033032+0000","last_unstale":"2023-09-23T23:24:51.754767+0000","last_undegraded":"2023-09-23T23:24:51.754767+0000","last_fullsized":"2023-09-23T23:24:51.754767+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:33:17.856914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":388,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.765247+0000","last_change":"2023-09-23T23:22:38.439331+0000","last_active":"2023-09-23T23:24:51.765247+0000","last_peered":"2023-09-23T23:24:51.765247+0000","last_clean":"2023-09-23T23:24:51.765247+0000","last_became_active":"2023-09-23T23:22:38.436293+0000","last_became_peered":"2023-09-23T23:22:38.436293+0000","last_unstale":"2023-09-23T23:24:51.765247+0000","last_undegraded":"2023-09-23T23:24:51.765247+0000","last_fullsized":"2023-09-23T23:24:51.765247+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:53.668614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"362'5762","reported_seq":5783,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:03.055620+0000","last_change":"2023-09-23T23:23:43.513124+0000","last_active":"2023-09-23T23:25:03.055620+0000","last_peered":"2023-09-23T23:25:03.055620+0000","last_clean":"2023-09-23T23:25:03.055620+0000","last_became_active":"2023-09-23T23:23:43.512782+0000","last_became_peered":"2023-09-23T23:23:43.512782+0000","last_unstale":"2023-09-23T23:25:03.055620+0000","last_undegraded":"2023-09-23T23:25:03.055620+0000","last_fullsized":"2023-09-23T23:25:03.055620+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5762,"log_dups_size":0,"ondisk_log_size":5762,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:16:06.664727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377618432,"num_objects":5762,"num_object_clones":0,"num_object_copies":17286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5762,"num_write_kb":368768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":436,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.755887+0000","last_change":"2023-09-23T23:24:04.832826+0000","last_active":"2023-09-23T23:24:51.755887+0000","last_peered":"2023-09-23T23:24:51.755887+0000","last_clean":"2023-09-23T23:24:51.755887+0000","last_became_active":"2023-09-23T23:23:45.460444+0000","last_became_peered":"2023-09-23T23:23:45.460444+0000","last_unstale":"2023-09-23T23:24:51.755887+0000","last_undegraded":"2023-09-23T23:24:51.755887+0000","last_fullsized":"2023-09-23T23:24:51.755887+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:04.832687+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:04.832687+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:02.707716+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":179,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.755396+0000","last_change":"2023-09-23T23:21:22.192374+0000","last_active":"2023-09-23T23:24:51.755396+0000","last_peered":"2023-09-23T23:24:51.755396+0000","last_clean":"2023-09-23T23:24:51.755396+0000","last_became_active":"2023-09-23T23:21:22.192086+0000","last_became_peered":"2023-09-23T23:21:22.192086+0000","last_unstale":"2023-09-23T23:24:51.755396+0000","last_undegraded":"2023-09-23T23:24:51.755396+0000","last_fullsized":"2023-09-23T23:24:51.755396+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:20.605133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":336,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.758566+0000","last_change":"2023-09-23T23:22:38.438390+0000","last_active":"2023-09-23T23:24:51.758566+0000","last_peered":"2023-09-23T23:24:51.758566+0000","last_clean":"2023-09-23T23:24:51.758566+0000","last_became_active":"2023-09-23T23:22:38.438065+0000","last_became_peered":"2023-09-23T23:22:38.438065+0000","last_unstale":"2023-09-23T23:24:51.758566+0000","last_undegraded":"2023-09-23T23:24:51.758566+0000","last_fullsized":"2023-09-23T23:24:51.758566+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:51.254227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"362'5717","reported_seq":5738,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:04.481006+0000","last_change":"2023-09-23T23:23:43.512187+0000","last_active":"2023-09-23T23:25:04.481006+0000","last_peered":"2023-09-23T23:25:04.481006+0000","last_clean":"2023-09-23T23:25:04.481006+0000","last_became_active":"2023-09-23T23:23:43.511874+0000","last_became_peered":"2023-09-23T23:23:43.511874+0000","last_unstale":"2023-09-23T23:25:04.481006+0000","last_undegraded":"2023-09-23T23:25:04.481006+0000","last_fullsized":"2023-09-23T23:25:04.481006+0000","mapping_epoch":349,"log_start":"355'700","ondisk_log_start":"355'700","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5017,"log_dups_size":0,"ondisk_log_size":5017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:38.052106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374669312,"num_objects":5717,"num_object_clones":0,"num_object_copies":17151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5717,"num_write_kb":365888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":697,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.753938+0000","last_change":"2023-09-23T23:23:45.830384+0000","last_active":"2023-09-23T23:24:51.753938+0000","last_peered":"2023-09-23T23:24:51.753938+0000","last_clean":"2023-09-23T23:24:51.753938+0000","last_became_active":"2023-09-23T23:23:36.025724+0000","last_became_peered":"2023-09-23T23:23:36.025724+0000","last_unstale":"2023-09-23T23:24:51.753938+0000","last_undegraded":"2023-09-23T23:24:51.753938+0000","last_fullsized":"2023-09-23T23:24:51.753938+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:34.345528+0000","scrub_duration":0.001659259,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":328,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.755725+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:24:51.755725+0000","last_peered":"2023-09-23T23:24:51.755725+0000","last_clean":"2023-09-23T23:24:51.755725+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:24:51.755725+0000","last_undegraded":"2023-09-23T23:24:51.755725+0000","last_fullsized":"2023-09-23T23:24:51.755725+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":320,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.770404+0000","last_change":"2023-09-23T23:10:59.512363+0000","last_active":"2023-09-23T23:24:51.770404+0000","last_peered":"2023-09-23T23:24:51.770404+0000","last_clean":"2023-09-23T23:24:51.770404+0000","last_became_active":"2023-09-23T23:10:59.512022+0000","last_became_peered":"2023-09-23T23:10:59.512022+0000","last_unstale":"2023-09-23T23:24:51.770404+0000","last_undegraded":"2023-09-23T23:24:51.770404+0000","last_fullsized":"2023-09-23T23:24:51.770404+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:45.523432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"362'5533","reported_seq":5555,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:00.654878+0000","last_change":"2023-09-23T23:23:43.503924+0000","last_active":"2023-09-23T23:25:00.654878+0000","last_peered":"2023-09-23T23:25:00.654878+0000","last_clean":"2023-09-23T23:25:00.654878+0000","last_became_active":"2023-09-23T23:23:43.503698+0000","last_became_peered":"2023-09-23T23:23:43.503698+0000","last_unstale":"2023-09-23T23:25:00.654878+0000","last_undegraded":"2023-09-23T23:25:00.654878+0000","last_fullsized":"2023-09-23T23:25:00.654878+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5533,"log_dups_size":0,"ondisk_log_size":5533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:14.346656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362610688,"num_objects":5533,"num_object_clones":0,"num_object_copies":16599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5533,"num_write_kb":354112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":606,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.769252+0000","last_change":"2023-09-23T23:23:50.862663+0000","last_active":"2023-09-23T23:24:51.769252+0000","last_peered":"2023-09-23T23:24:51.769252+0000","last_clean":"2023-09-23T23:24:51.769252+0000","last_became_active":"2023-09-23T23:22:57.463887+0000","last_became_peered":"2023-09-23T23:22:57.463887+0000","last_unstale":"2023-09-23T23:24:51.769252+0000","last_undegraded":"2023-09-23T23:24:51.769252+0000","last_fullsized":"2023-09-23T23:24:51.769252+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:46:51.271983+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":332,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.765325+0000","last_change":"2023-09-23T23:16:56.525904+0000","last_active":"2023-09-23T23:24:51.765325+0000","last_peered":"2023-09-23T23:24:51.765325+0000","last_clean":"2023-09-23T23:24:51.765325+0000","last_became_active":"2023-09-23T23:16:56.525372+0000","last_became_peered":"2023-09-23T23:16:56.525372+0000","last_unstale":"2023-09-23T23:24:51.765325+0000","last_undegraded":"2023-09-23T23:24:51.765325+0000","last_fullsized":"2023-09-23T23:24:51.765325+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:17.070117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":377,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.770594+0000","last_change":"2023-09-23T23:22:38.439737+0000","last_active":"2023-09-23T23:24:51.770594+0000","last_peered":"2023-09-23T23:24:51.770594+0000","last_clean":"2023-09-23T23:24:51.770594+0000","last_became_active":"2023-09-23T23:22:38.439246+0000","last_became_peered":"2023-09-23T23:22:38.439246+0000","last_unstale":"2023-09-23T23:24:51.770594+0000","last_undegraded":"2023-09-23T23:24:51.770594+0000","last_fullsized":"2023-09-23T23:24:51.770594+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:40.042010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"362'5712","reported_seq":5733,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:04.412402+0000","last_change":"2023-09-23T23:23:43.513972+0000","last_active":"2023-09-23T23:25:04.412402+0000","last_peered":"2023-09-23T23:25:04.412402+0000","last_clean":"2023-09-23T23:25:04.412402+0000","last_became_active":"2023-09-23T23:23:43.513615+0000","last_became_peered":"2023-09-23T23:23:43.513615+0000","last_unstale":"2023-09-23T23:25:04.412402+0000","last_undegraded":"2023-09-23T23:25:04.412402+0000","last_fullsized":"2023-09-23T23:25:04.412402+0000","mapping_epoch":349,"log_start":"355'700","ondisk_log_start":"355'700","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5012,"log_dups_size":0,"ondisk_log_size":5012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:27.822029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374341632,"num_objects":5712,"num_object_clones":0,"num_object_copies":17136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5712,"num_write_kb":365568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":499,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:59.238635+0000","last_change":"2023-09-23T23:23:50.178725+0000","last_active":"2023-09-23T23:24:59.238635+0000","last_peered":"2023-09-23T23:24:59.238635+0000","last_clean":"2023-09-23T23:24:59.238635+0000","last_became_active":"2023-09-23T23:23:45.458319+0000","last_became_peered":"2023-09-23T23:23:45.458319+0000","last_unstale":"2023-09-23T23:24:59.238635+0000","last_undegraded":"2023-09-23T23:24:59.238635+0000","last_fullsized":"2023-09-23T23:24:59.238635+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:46.518884+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":383,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.768828+0000","last_change":"2023-09-23T23:23:36.030470+0000","last_active":"2023-09-23T23:24:51.768828+0000","last_peered":"2023-09-23T23:24:51.768828+0000","last_clean":"2023-09-23T23:24:51.768828+0000","last_became_active":"2023-09-23T23:23:36.030157+0000","last_became_peered":"2023-09-23T23:23:36.030157+0000","last_unstale":"2023-09-23T23:24:51.768828+0000","last_undegraded":"2023-09-23T23:24:51.768828+0000","last_fullsized":"2023-09-23T23:24:51.768828+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:08.848362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":330,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.770527+0000","last_change":"2023-09-23T23:15:28.228439+0000","last_active":"2023-09-23T23:24:51.770527+0000","last_peered":"2023-09-23T23:24:51.770527+0000","last_clean":"2023-09-23T23:24:51.770527+0000","last_became_active":"2023-09-23T23:15:28.228095+0000","last_became_peered":"2023-09-23T23:15:28.228095+0000","last_unstale":"2023-09-23T23:24:51.770527+0000","last_undegraded":"2023-09-23T23:24:51.770527+0000","last_fullsized":"2023-09-23T23:24:51.770527+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:37.667201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"362'5616","reported_seq":5638,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:04.219154+0000","last_change":"2023-09-23T23:23:43.513796+0000","last_active":"2023-09-23T23:25:04.219154+0000","last_peered":"2023-09-23T23:25:04.219154+0000","last_clean":"2023-09-23T23:25:04.219154+0000","last_became_active":"2023-09-23T23:23:43.513512+0000","last_became_peered":"2023-09-23T23:23:43.513512+0000","last_unstale":"2023-09-23T23:25:04.219154+0000","last_undegraded":"2023-09-23T23:25:04.219154+0000","last_fullsized":"2023-09-23T23:25:04.219154+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5616,"log_dups_size":0,"ondisk_log_size":5616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:10.363938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368050176,"num_objects":5616,"num_object_clones":0,"num_object_copies":16848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5616,"num_write_kb":359424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":664,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.768653+0000","last_change":"2023-09-23T23:23:51.877015+0000","last_active":"2023-09-23T23:24:51.768653+0000","last_peered":"2023-09-23T23:24:51.768653+0000","last_clean":"2023-09-23T23:24:51.768653+0000","last_became_active":"2023-09-23T23:22:57.468211+0000","last_became_peered":"2023-09-23T23:22:57.468211+0000","last_unstale":"2023-09-23T23:24:51.768653+0000","last_undegraded":"2023-09-23T23:24:51.768653+0000","last_fullsized":"2023-09-23T23:24:51.768653+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:42.443440+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":53,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.772065+0000","last_change":"2023-09-23T23:22:38.434606+0000","last_active":"2023-09-23T23:24:51.772065+0000","last_peered":"2023-09-23T23:24:51.772065+0000","last_clean":"2023-09-23T23:24:51.772065+0000","last_became_active":"2023-09-23T23:22:38.434196+0000","last_became_peered":"2023-09-23T23:22:38.434196+0000","last_unstale":"2023-09-23T23:24:51.772065+0000","last_undegraded":"2023-09-23T23:24:51.772065+0000","last_fullsized":"2023-09-23T23:24:51.772065+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:14.702902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":282,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:54.238093+0000","last_change":"2023-09-23T23:22:39.489325+0000","last_active":"2023-09-23T23:24:54.238093+0000","last_peered":"2023-09-23T23:24:54.238093+0000","last_clean":"2023-09-23T23:24:54.238093+0000","last_became_active":"2023-09-23T23:22:39.489048+0000","last_became_peered":"2023-09-23T23:22:39.489048+0000","last_unstale":"2023-09-23T23:24:54.238093+0000","last_undegraded":"2023-09-23T23:24:54.238093+0000","last_fullsized":"2023-09-23T23:24:54.238093+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:29.984948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"362'5529","reported_seq":5551,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:00.651795+0000","last_change":"2023-09-23T23:23:43.507963+0000","last_active":"2023-09-23T23:25:00.651795+0000","last_peered":"2023-09-23T23:25:00.651795+0000","last_clean":"2023-09-23T23:25:00.651795+0000","last_became_active":"2023-09-23T23:23:43.507503+0000","last_became_peered":"2023-09-23T23:23:43.507503+0000","last_unstale":"2023-09-23T23:25:00.651795+0000","last_undegraded":"2023-09-23T23:25:00.651795+0000","last_fullsized":"2023-09-23T23:25:00.651795+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5529,"log_dups_size":0,"ondisk_log_size":5529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:39.595255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362348544,"num_objects":5529,"num_object_clones":0,"num_object_copies":16587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5529,"num_write_kb":353856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":770,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.764007+0000","last_change":"2023-09-23T23:23:48.063145+0000","last_active":"2023-09-23T23:24:51.764007+0000","last_peered":"2023-09-23T23:24:51.764007+0000","last_clean":"2023-09-23T23:24:51.764007+0000","last_became_active":"2023-09-23T23:22:57.470024+0000","last_became_peered":"2023-09-23T23:22:57.470024+0000","last_unstale":"2023-09-23T23:24:51.764007+0000","last_undegraded":"2023-09-23T23:24:51.764007+0000","last_fullsized":"2023-09-23T23:24:51.764007+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.063071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.063071+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:51.217545+0000","scrub_duration":0.0015554690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":62,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.769894+0000","last_change":"2023-09-23T23:23:36.048889+0000","last_active":"2023-09-23T23:24:51.769894+0000","last_peered":"2023-09-23T23:24:51.769894+0000","last_clean":"2023-09-23T23:24:51.769894+0000","last_became_active":"2023-09-23T23:23:36.048572+0000","last_became_peered":"2023-09-23T23:23:36.048572+0000","last_unstale":"2023-09-23T23:24:51.769894+0000","last_undegraded":"2023-09-23T23:24:51.769894+0000","last_fullsized":"2023-09-23T23:24:51.769894+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:55.581146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":377,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.765578+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:24:51.765578+0000","last_peered":"2023-09-23T23:24:51.765578+0000","last_clean":"2023-09-23T23:24:51.765578+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:24:51.765578+0000","last_undegraded":"2023-09-23T23:24:51.765578+0000","last_fullsized":"2023-09-23T23:24:51.765578+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":305,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.758253+0000","last_change":"2023-09-23T23:23:36.030569+0000","last_active":"2023-09-23T23:24:51.758253+0000","last_peered":"2023-09-23T23:24:51.758253+0000","last_clean":"2023-09-23T23:24:51.758253+0000","last_became_active":"2023-09-23T23:23:36.030257+0000","last_became_peered":"2023-09-23T23:23:36.030257+0000","last_unstale":"2023-09-23T23:24:51.758253+0000","last_undegraded":"2023-09-23T23:24:51.758253+0000","last_fullsized":"2023-09-23T23:24:51.758253+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:53.906813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.b","version":"362'5731","reported_seq":5753,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:03.263909+0000","last_change":"2023-09-23T23:23:43.509408+0000","last_active":"2023-09-23T23:25:03.263909+0000","last_peered":"2023-09-23T23:25:03.263909+0000","last_clean":"2023-09-23T23:25:03.263909+0000","last_became_active":"2023-09-23T23:23:43.509146+0000","last_became_peered":"2023-09-23T23:23:43.509146+0000","last_unstale":"2023-09-23T23:25:03.263909+0000","last_undegraded":"2023-09-23T23:25:03.263909+0000","last_fullsized":"2023-09-23T23:25:03.263909+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5731,"log_dups_size":0,"ondisk_log_size":5731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:51.961026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375586816,"num_objects":5731,"num_object_clones":0,"num_object_copies":17193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5731,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":506,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.758552+0000","last_change":"2023-09-23T23:23:49.320969+0000","last_active":"2023-09-23T23:24:51.758552+0000","last_peered":"2023-09-23T23:24:51.758552+0000","last_clean":"2023-09-23T23:24:51.758552+0000","last_became_active":"2023-09-23T23:22:17.001911+0000","last_became_peered":"2023-09-23T23:22:17.001911+0000","last_unstale":"2023-09-23T23:24:51.758552+0000","last_undegraded":"2023-09-23T23:24:51.758552+0000","last_fullsized":"2023-09-23T23:24:51.758552+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.320823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.320823+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:07.000984+0000","scrub_duration":0.0018140630000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":53,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.767815+0000","last_change":"2023-09-23T23:22:38.433755+0000","last_active":"2023-09-23T23:24:51.767815+0000","last_peered":"2023-09-23T23:24:51.767815+0000","last_clean":"2023-09-23T23:24:51.767815+0000","last_became_active":"2023-09-23T23:22:38.433442+0000","last_became_peered":"2023-09-23T23:22:38.433442+0000","last_unstale":"2023-09-23T23:24:51.767815+0000","last_undegraded":"2023-09-23T23:24:51.767815+0000","last_fullsized":"2023-09-23T23:24:51.767815+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:25.409363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":321,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.770247+0000","last_change":"2023-09-23T23:04:11.514703+0000","last_active":"2023-09-23T23:24:51.770247+0000","last_peered":"2023-09-23T23:24:51.770247+0000","last_clean":"2023-09-23T23:24:51.770247+0000","last_became_active":"2023-09-23T23:04:11.514403+0000","last_became_peered":"2023-09-23T23:04:11.514403+0000","last_unstale":"2023-09-23T23:24:51.770247+0000","last_undegraded":"2023-09-23T23:24:51.770247+0000","last_fullsized":"2023-09-23T23:24:51.770247+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:26.374346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":95,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.769723+0000","last_change":"2023-09-23T23:20:15.781844+0000","last_active":"2023-09-23T23:24:51.769723+0000","last_peered":"2023-09-23T23:24:51.769723+0000","last_clean":"2023-09-23T23:24:51.769723+0000","last_became_active":"2023-09-23T23:20:15.781573+0000","last_became_peered":"2023-09-23T23:20:15.781573+0000","last_unstale":"2023-09-23T23:24:51.769723+0000","last_undegraded":"2023-09-23T23:24:51.769723+0000","last_fullsized":"2023-09-23T23:24:51.769723+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:25:50.816486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"362'5673","reported_seq":5694,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:04.439833+0000","last_change":"2023-09-23T23:23:43.508971+0000","last_active":"2023-09-23T23:25:04.439833+0000","last_peered":"2023-09-23T23:25:04.439833+0000","last_clean":"2023-09-23T23:25:04.439833+0000","last_became_active":"2023-09-23T23:23:43.508648+0000","last_became_peered":"2023-09-23T23:23:43.508648+0000","last_unstale":"2023-09-23T23:25:04.439833+0000","last_undegraded":"2023-09-23T23:25:04.439833+0000","last_fullsized":"2023-09-23T23:25:04.439833+0000","mapping_epoch":349,"log_start":"355'700","ondisk_log_start":"355'700","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4973,"log_dups_size":0,"ondisk_log_size":4973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:43.036229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371785728,"num_objects":5673,"num_object_clones":0,"num_object_copies":17019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5673,"num_write_kb":363072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":696,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.758162+0000","last_change":"2023-09-23T23:23:52.342245+0000","last_active":"2023-09-23T23:24:51.758162+0000","last_peered":"2023-09-23T23:24:51.758162+0000","last_clean":"2023-09-23T23:24:51.758162+0000","last_became_active":"2023-09-23T23:22:17.000797+0000","last_became_peered":"2023-09-23T23:22:17.000797+0000","last_unstale":"2023-09-23T23:24:51.758162+0000","last_undegraded":"2023-09-23T23:24:51.758162+0000","last_fullsized":"2023-09-23T23:24:51.758162+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:13.663896+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":95,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.768688+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:24:51.768688+0000","last_peered":"2023-09-23T23:24:51.768688+0000","last_clean":"2023-09-23T23:24:51.768688+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:24:51.768688+0000","last_undegraded":"2023-09-23T23:24:51.768688+0000","last_fullsized":"2023-09-23T23:24:51.768688+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":53,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.772245+0000","last_change":"2023-09-23T23:22:38.480385+0000","last_active":"2023-09-23T23:24:51.772245+0000","last_peered":"2023-09-23T23:24:51.772245+0000","last_clean":"2023-09-23T23:24:51.772245+0000","last_became_active":"2023-09-23T23:22:38.480071+0000","last_became_peered":"2023-09-23T23:22:38.480071+0000","last_unstale":"2023-09-23T23:24:51.772245+0000","last_undegraded":"2023-09-23T23:24:51.772245+0000","last_fullsized":"2023-09-23T23:24:51.772245+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:15.348789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":174,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.759394+0000","last_change":"2023-09-23T23:21:22.194721+0000","last_active":"2023-09-23T23:24:51.759394+0000","last_peered":"2023-09-23T23:24:51.759394+0000","last_clean":"2023-09-23T23:24:51.759394+0000","last_became_active":"2023-09-23T23:21:22.194333+0000","last_became_peered":"2023-09-23T23:21:22.194333+0000","last_unstale":"2023-09-23T23:24:51.759394+0000","last_undegraded":"2023-09-23T23:24:51.759394+0000","last_fullsized":"2023-09-23T23:24:51.759394+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:46.823997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"362'5610","reported_seq":5632,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:03.252792+0000","last_change":"2023-09-23T23:23:43.510145+0000","last_active":"2023-09-23T23:25:03.252792+0000","last_peered":"2023-09-23T23:25:03.252792+0000","last_clean":"2023-09-23T23:25:03.252792+0000","last_became_active":"2023-09-23T23:23:43.509875+0000","last_became_peered":"2023-09-23T23:23:43.509875+0000","last_unstale":"2023-09-23T23:25:03.252792+0000","last_undegraded":"2023-09-23T23:25:03.252792+0000","last_fullsized":"2023-09-23T23:25:03.252792+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5610,"log_dups_size":0,"ondisk_log_size":5610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:37.678046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367656960,"num_objects":5610,"num_object_clones":0,"num_object_copies":16830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5610,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":362,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.770961+0000","last_change":"2023-09-23T23:21:22.193754+0000","last_active":"2023-09-23T23:24:51.770961+0000","last_peered":"2023-09-23T23:24:51.770961+0000","last_clean":"2023-09-23T23:24:51.770961+0000","last_became_active":"2023-09-23T23:21:22.193409+0000","last_became_peered":"2023-09-23T23:21:22.193409+0000","last_unstale":"2023-09-23T23:24:51.770961+0000","last_undegraded":"2023-09-23T23:24:51.770961+0000","last_fullsized":"2023-09-23T23:24:51.770961+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:53.635813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":655,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:00.033259+0000","last_change":"2023-09-23T23:23:55.343042+0000","last_active":"2023-09-23T23:25:00.033259+0000","last_peered":"2023-09-23T23:25:00.033259+0000","last_clean":"2023-09-23T23:25:00.033259+0000","last_became_active":"2023-09-23T23:22:41.462616+0000","last_became_peered":"2023-09-23T23:22:41.462616+0000","last_unstale":"2023-09-23T23:25:00.033259+0000","last_undegraded":"2023-09-23T23:25:00.033259+0000","last_fullsized":"2023-09-23T23:25:00.033259+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:35.897918+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":291,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.756119+0000","last_change":"2023-09-23T23:16:56.525298+0000","last_active":"2023-09-23T23:24:51.756119+0000","last_peered":"2023-09-23T23:24:51.756119+0000","last_clean":"2023-09-23T23:24:51.756119+0000","last_became_active":"2023-09-23T23:16:56.524974+0000","last_became_peered":"2023-09-23T23:16:56.524974+0000","last_unstale":"2023-09-23T23:24:51.756119+0000","last_undegraded":"2023-09-23T23:24:51.756119+0000","last_fullsized":"2023-09-23T23:24:51.756119+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:08.860790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":335,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.765135+0000","last_change":"2023-09-23T23:23:36.022440+0000","last_active":"2023-09-23T23:24:51.765135+0000","last_peered":"2023-09-23T23:24:51.765135+0000","last_clean":"2023-09-23T23:24:51.765135+0000","last_became_active":"2023-09-23T23:23:36.022065+0000","last_became_peered":"2023-09-23T23:23:36.022065+0000","last_unstale":"2023-09-23T23:24:51.765135+0000","last_undegraded":"2023-09-23T23:24:51.765135+0000","last_fullsized":"2023-09-23T23:24:51.765135+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:43:27.098960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"362'5723","reported_seq":5745,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:03.269667+0000","last_change":"2023-09-23T23:23:43.510366+0000","last_active":"2023-09-23T23:25:03.269667+0000","last_peered":"2023-09-23T23:25:03.269667+0000","last_clean":"2023-09-23T23:25:03.269667+0000","last_became_active":"2023-09-23T23:23:43.510117+0000","last_became_peered":"2023-09-23T23:23:43.510117+0000","last_unstale":"2023-09-23T23:25:03.269667+0000","last_undegraded":"2023-09-23T23:25:03.269667+0000","last_fullsized":"2023-09-23T23:25:03.269667+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5723,"log_dups_size":0,"ondisk_log_size":5723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:05.709562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375062528,"num_objects":5723,"num_object_clones":0,"num_object_copies":17169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5723,"num_write_kb":366272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":473,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.769403+0000","last_change":"2023-09-23T23:23:46.214371+0000","last_active":"2023-09-23T23:24:51.769403+0000","last_peered":"2023-09-23T23:24:51.769403+0000","last_clean":"2023-09-23T23:24:51.769403+0000","last_became_active":"2023-09-23T23:22:41.466645+0000","last_became_peered":"2023-09-23T23:22:41.466645+0000","last_unstale":"2023-09-23T23:24:51.769403+0000","last_undegraded":"2023-09-23T23:24:51.769403+0000","last_fullsized":"2023-09-23T23:24:51.769403+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:14.444467+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":279,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.753805+0000","last_change":"2023-09-23T23:23:35.021697+0000","last_active":"2023-09-23T23:24:51.753805+0000","last_peered":"2023-09-23T23:24:51.753805+0000","last_clean":"2023-09-23T23:24:51.753805+0000","last_became_active":"2023-09-23T23:23:35.021376+0000","last_became_peered":"2023-09-23T23:23:35.021376+0000","last_unstale":"2023-09-23T23:24:51.753805+0000","last_undegraded":"2023-09-23T23:24:51.753805+0000","last_fullsized":"2023-09-23T23:24:51.753805+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:42.630129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":345,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:00.033715+0000","last_change":"2023-09-23T23:23:36.031618+0000","last_active":"2023-09-23T23:25:00.033715+0000","last_peered":"2023-09-23T23:25:00.033715+0000","last_clean":"2023-09-23T23:25:00.033715+0000","last_became_active":"2023-09-23T23:23:36.031322+0000","last_became_peered":"2023-09-23T23:23:36.031322+0000","last_unstale":"2023-09-23T23:25:00.033715+0000","last_undegraded":"2023-09-23T23:25:00.033715+0000","last_fullsized":"2023-09-23T23:25:00.033715+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:07.871195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":390,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.759151+0000","last_change":"2023-09-23T23:22:38.438448+0000","last_active":"2023-09-23T23:24:51.759151+0000","last_peered":"2023-09-23T23:24:51.759151+0000","last_clean":"2023-09-23T23:24:51.759151+0000","last_became_active":"2023-09-23T23:22:38.437495+0000","last_became_peered":"2023-09-23T23:22:38.437495+0000","last_unstale":"2023-09-23T23:24:51.759151+0000","last_undegraded":"2023-09-23T23:24:51.759151+0000","last_fullsized":"2023-09-23T23:24:51.759151+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:49.587056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"362'5554","reported_seq":5576,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:04.191893+0000","last_change":"2023-09-23T23:23:43.512003+0000","last_active":"2023-09-23T23:25:04.191893+0000","last_peered":"2023-09-23T23:25:04.191893+0000","last_clean":"2023-09-23T23:25:04.191893+0000","last_became_active":"2023-09-23T23:23:43.511728+0000","last_became_peered":"2023-09-23T23:23:43.511728+0000","last_unstale":"2023-09-23T23:25:04.191893+0000","last_undegraded":"2023-09-23T23:25:04.191893+0000","last_fullsized":"2023-09-23T23:25:04.191893+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5554,"log_dups_size":0,"ondisk_log_size":5554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:33.464148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363986944,"num_objects":5554,"num_object_clones":0,"num_object_copies":16662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5554,"num_write_kb":355456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":667,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.763609+0000","last_change":"2023-09-23T23:23:51.048787+0000","last_active":"2023-09-23T23:24:51.763609+0000","last_peered":"2023-09-23T23:24:51.763609+0000","last_clean":"2023-09-23T23:24:51.763609+0000","last_became_active":"2023-09-23T23:22:57.759601+0000","last_became_peered":"2023-09-23T23:22:57.759601+0000","last_unstale":"2023-09-23T23:24:51.763609+0000","last_undegraded":"2023-09-23T23:24:51.763609+0000","last_fullsized":"2023-09-23T23:24:51.763609+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.048718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.048718+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:18.385599+0000","scrub_duration":0.0016072149999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":213,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.754824+0000","last_change":"2023-09-23T23:23:35.037351+0000","last_active":"2023-09-23T23:24:51.754824+0000","last_peered":"2023-09-23T23:24:51.754824+0000","last_clean":"2023-09-23T23:24:51.754824+0000","last_became_active":"2023-09-23T23:23:35.036841+0000","last_became_peered":"2023-09-23T23:23:35.036841+0000","last_unstale":"2023-09-23T23:24:51.754824+0000","last_undegraded":"2023-09-23T23:24:51.754824+0000","last_fullsized":"2023-09-23T23:24:51.754824+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:20.618292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":53,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.770107+0000","last_change":"2023-09-23T23:22:38.453323+0000","last_active":"2023-09-23T23:24:51.770107+0000","last_peered":"2023-09-23T23:24:51.770107+0000","last_clean":"2023-09-23T23:24:51.770107+0000","last_became_active":"2023-09-23T23:22:38.452858+0000","last_became_peered":"2023-09-23T23:22:38.452858+0000","last_unstale":"2023-09-23T23:24:51.770107+0000","last_undegraded":"2023-09-23T23:24:51.770107+0000","last_fullsized":"2023-09-23T23:24:51.770107+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:31.988045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":95,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.765924+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:24:51.765924+0000","last_peered":"2023-09-23T23:24:51.765924+0000","last_clean":"2023-09-23T23:24:51.765924+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:24:51.765924+0000","last_undegraded":"2023-09-23T23:24:51.765924+0000","last_fullsized":"2023-09-23T23:24:51.765924+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"362'5720","reported_seq":5741,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:03.070913+0000","last_change":"2023-09-23T23:23:43.505852+0000","last_active":"2023-09-23T23:25:03.070913+0000","last_peered":"2023-09-23T23:25:03.070913+0000","last_clean":"2023-09-23T23:25:03.070913+0000","last_became_active":"2023-09-23T23:23:43.505473+0000","last_became_peered":"2023-09-23T23:23:43.505473+0000","last_unstale":"2023-09-23T23:25:03.070913+0000","last_undegraded":"2023-09-23T23:25:03.070913+0000","last_fullsized":"2023-09-23T23:25:03.070913+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5720,"log_dups_size":0,"ondisk_log_size":5720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:31.385610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374865920,"num_objects":5720,"num_object_clones":0,"num_object_copies":17160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5720,"num_write_kb":366080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":675,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.770678+0000","last_change":"2023-09-23T23:23:46.892667+0000","last_active":"2023-09-23T23:24:51.770678+0000","last_peered":"2023-09-23T23:24:51.770678+0000","last_clean":"2023-09-23T23:24:51.770678+0000","last_became_active":"2023-09-23T23:22:57.465086+0000","last_became_peered":"2023-09-23T23:22:57.465086+0000","last_unstale":"2023-09-23T23:24:51.770678+0000","last_undegraded":"2023-09-23T23:24:51.770678+0000","last_fullsized":"2023-09-23T23:24:51.770678+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:55.262932+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":341,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.756338+0000","last_change":"2023-09-23T23:16:56.561147+0000","last_active":"2023-09-23T23:24:51.756338+0000","last_peered":"2023-09-23T23:24:51.756338+0000","last_clean":"2023-09-23T23:24:51.756338+0000","last_became_active":"2023-09-23T23:16:56.547808+0000","last_became_peered":"2023-09-23T23:16:56.547808+0000","last_unstale":"2023-09-23T23:24:51.756338+0000","last_undegraded":"2023-09-23T23:24:51.756338+0000","last_fullsized":"2023-09-23T23:24:51.756338+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:13.666957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":216,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.763740+0000","last_change":"2023-09-23T23:16:56.526187+0000","last_active":"2023-09-23T23:24:51.763740+0000","last_peered":"2023-09-23T23:24:51.763740+0000","last_clean":"2023-09-23T23:24:51.763740+0000","last_became_active":"2023-09-23T23:16:56.525874+0000","last_became_peered":"2023-09-23T23:16:56.525874+0000","last_unstale":"2023-09-23T23:24:51.763740+0000","last_undegraded":"2023-09-23T23:24:51.763740+0000","last_fullsized":"2023-09-23T23:24:51.763740+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:32:07.101559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":53,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.772135+0000","last_change":"2023-09-23T23:22:38.444291+0000","last_active":"2023-09-23T23:24:51.772135+0000","last_peered":"2023-09-23T23:24:51.772135+0000","last_clean":"2023-09-23T23:24:51.772135+0000","last_became_active":"2023-09-23T23:22:38.443996+0000","last_became_peered":"2023-09-23T23:22:38.443996+0000","last_unstale":"2023-09-23T23:24:51.772135+0000","last_undegraded":"2023-09-23T23:24:51.772135+0000","last_fullsized":"2023-09-23T23:24:51.772135+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:57.255970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"362'5717","reported_seq":5739,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:04.199531+0000","last_change":"2023-09-23T23:23:43.510714+0000","last_active":"2023-09-23T23:25:04.199531+0000","last_peered":"2023-09-23T23:25:04.199531+0000","last_clean":"2023-09-23T23:25:04.199531+0000","last_became_active":"2023-09-23T23:23:43.510441+0000","last_became_peered":"2023-09-23T23:23:43.510441+0000","last_unstale":"2023-09-23T23:25:04.199531+0000","last_undegraded":"2023-09-23T23:25:04.199531+0000","last_fullsized":"2023-09-23T23:25:04.199531+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5717,"log_dups_size":0,"ondisk_log_size":5717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:54.915575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374669312,"num_objects":5717,"num_object_clones":0,"num_object_copies":17151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5717,"num_write_kb":365888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":675,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.755629+0000","last_change":"2023-09-23T23:23:45.832267+0000","last_active":"2023-09-23T23:24:51.755629+0000","last_peered":"2023-09-23T23:24:51.755629+0000","last_clean":"2023-09-23T23:24:51.755629+0000","last_became_active":"2023-09-23T23:22:17.009092+0000","last_became_peered":"2023-09-23T23:22:17.009092+0000","last_unstale":"2023-09-23T23:24:51.755629+0000","last_undegraded":"2023-09-23T23:24:51.755629+0000","last_fullsized":"2023-09-23T23:24:51.755629+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:42.979434+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":373,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.766263+0000","last_change":"2023-09-23T23:20:15.786693+0000","last_active":"2023-09-23T23:24:51.766263+0000","last_peered":"2023-09-23T23:24:51.766263+0000","last_clean":"2023-09-23T23:24:51.766263+0000","last_became_active":"2023-09-23T23:20:15.786360+0000","last_became_peered":"2023-09-23T23:20:15.786360+0000","last_unstale":"2023-09-23T23:24:51.766263+0000","last_undegraded":"2023-09-23T23:24:51.766263+0000","last_fullsized":"2023-09-23T23:24:51.766263+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:00.707946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":345,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.759013+0000","last_change":"2023-09-23T23:22:38.482705+0000","last_active":"2023-09-23T23:24:51.759013+0000","last_peered":"2023-09-23T23:24:51.759013+0000","last_clean":"2023-09-23T23:24:51.759013+0000","last_became_active":"2023-09-23T23:22:38.438239+0000","last_became_peered":"2023-09-23T23:22:38.438239+0000","last_unstale":"2023-09-23T23:24:51.759013+0000","last_undegraded":"2023-09-23T23:24:51.759013+0000","last_fullsized":"2023-09-23T23:24:51.759013+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:52.006349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":75,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.765167+0000","last_change":"2023-09-23T23:21:50.045682+0000","last_active":"2023-09-23T23:24:51.765167+0000","last_peered":"2023-09-23T23:24:51.765167+0000","last_clean":"2023-09-23T23:24:51.765167+0000","last_became_active":"2023-09-23T23:21:50.045339+0000","last_became_peered":"2023-09-23T23:21:50.045339+0000","last_unstale":"2023-09-23T23:24:51.765167+0000","last_undegraded":"2023-09-23T23:24:51.765167+0000","last_fullsized":"2023-09-23T23:24:51.765167+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:55.802091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":725,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.755122+0000","last_change":"2023-09-23T23:23:50.872439+0000","last_active":"2023-09-23T23:24:51.755122+0000","last_peered":"2023-09-23T23:24:51.755122+0000","last_clean":"2023-09-23T23:24:51.755122+0000","last_became_active":"2023-09-23T23:23:45.454920+0000","last_became_peered":"2023-09-23T23:23:45.454920+0000","last_unstale":"2023-09-23T23:24:51.755122+0000","last_undegraded":"2023-09-23T23:24:51.755122+0000","last_fullsized":"2023-09-23T23:24:51.755122+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:30.422526+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":113,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.770102+0000","last_change":"2023-09-23T23:23:36.026729+0000","last_active":"2023-09-23T23:24:51.770102+0000","last_peered":"2023-09-23T23:24:51.770102+0000","last_clean":"2023-09-23T23:24:51.770102+0000","last_became_active":"2023-09-23T23:23:36.026345+0000","last_became_peered":"2023-09-23T23:23:36.026345+0000","last_unstale":"2023-09-23T23:24:51.770102+0000","last_undegraded":"2023-09-23T23:24:51.770102+0000","last_fullsized":"2023-09-23T23:24:51.770102+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.463936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":103,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.770428+0000","last_change":"2023-09-23T23:22:38.439602+0000","last_active":"2023-09-23T23:24:51.770428+0000","last_peered":"2023-09-23T23:24:51.770428+0000","last_clean":"2023-09-23T23:24:51.770428+0000","last_became_active":"2023-09-23T23:22:38.439261+0000","last_became_peered":"2023-09-23T23:22:38.439261+0000","last_unstale":"2023-09-23T23:24:51.770428+0000","last_undegraded":"2023-09-23T23:24:51.770428+0000","last_fullsized":"2023-09-23T23:24:51.770428+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:03.293816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":436,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.756076+0000","last_change":"2023-09-23T23:23:58.725636+0000","last_active":"2023-09-23T23:24:51.756076+0000","last_peered":"2023-09-23T23:24:51.756076+0000","last_clean":"2023-09-23T23:24:51.756076+0000","last_became_active":"2023-09-23T23:23:45.457175+0000","last_became_peered":"2023-09-23T23:23:45.457175+0000","last_unstale":"2023-09-23T23:24:51.756076+0000","last_undegraded":"2023-09-23T23:24:51.756076+0000","last_fullsized":"2023-09-23T23:24:51.756076+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:02.529300+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":103,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.770502+0000","last_change":"2023-09-23T23:22:38.452968+0000","last_active":"2023-09-23T23:24:51.770502+0000","last_peered":"2023-09-23T23:24:51.770502+0000","last_clean":"2023-09-23T23:24:51.770502+0000","last_became_active":"2023-09-23T23:22:38.437986+0000","last_became_peered":"2023-09-23T23:22:38.437986+0000","last_unstale":"2023-09-23T23:24:51.770502+0000","last_undegraded":"2023-09-23T23:24:51.770502+0000","last_fullsized":"2023-09-23T23:24:51.770502+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:10.333271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":368,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.755621+0000","last_change":"2023-09-23T23:23:36.032480+0000","last_active":"2023-09-23T23:24:51.755621+0000","last_peered":"2023-09-23T23:24:51.755621+0000","last_clean":"2023-09-23T23:24:51.755621+0000","last_became_active":"2023-09-23T23:23:36.032169+0000","last_became_peered":"2023-09-23T23:23:36.032169+0000","last_unstale":"2023-09-23T23:24:51.755621+0000","last_undegraded":"2023-09-23T23:24:51.755621+0000","last_fullsized":"2023-09-23T23:24:51.755621+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:52.893674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":89,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.774575+0000","last_change":"2023-09-23T23:21:22.194730+0000","last_active":"2023-09-23T23:24:51.774575+0000","last_peered":"2023-09-23T23:24:51.774575+0000","last_clean":"2023-09-23T23:24:51.774575+0000","last_became_active":"2023-09-23T23:21:22.194420+0000","last_became_peered":"2023-09-23T23:21:22.194420+0000","last_unstale":"2023-09-23T23:24:51.774575+0000","last_undegraded":"2023-09-23T23:24:51.774575+0000","last_fullsized":"2023-09-23T23:24:51.774575+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:31.083059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":206,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.755968+0000","last_change":"2023-09-23T23:20:26.841878+0000","last_active":"2023-09-23T23:24:51.755968+0000","last_peered":"2023-09-23T23:24:51.755968+0000","last_clean":"2023-09-23T23:24:51.755968+0000","last_became_active":"2023-09-23T23:20:26.841590+0000","last_became_peered":"2023-09-23T23:20:26.841590+0000","last_unstale":"2023-09-23T23:24:51.755968+0000","last_undegraded":"2023-09-23T23:24:51.755968+0000","last_fullsized":"2023-09-23T23:24:51.755968+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:23.904011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":608,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.754533+0000","last_change":"2023-09-23T23:23:51.871725+0000","last_active":"2023-09-23T23:24:51.754533+0000","last_peered":"2023-09-23T23:24:51.754533+0000","last_clean":"2023-09-23T23:24:51.754533+0000","last_became_active":"2023-09-23T23:23:46.456342+0000","last_became_peered":"2023-09-23T23:23:46.456342+0000","last_unstale":"2023-09-23T23:24:51.754533+0000","last_undegraded":"2023-09-23T23:24:51.754533+0000","last_fullsized":"2023-09-23T23:24:51.754533+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:02.041716+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":369,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:58.314734+0000","last_change":"2023-09-23T23:20:16.730934+0000","last_active":"2023-09-23T23:24:58.314734+0000","last_peered":"2023-09-23T23:24:58.314734+0000","last_clean":"2023-09-23T23:24:58.314734+0000","last_became_active":"2023-09-23T23:20:16.730571+0000","last_became_peered":"2023-09-23T23:20:16.730571+0000","last_unstale":"2023-09-23T23:24:58.314734+0000","last_undegraded":"2023-09-23T23:24:58.314734+0000","last_fullsized":"2023-09-23T23:24:58.314734+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:17.993278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":371,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.755486+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:24:51.755486+0000","last_peered":"2023-09-23T23:24:51.755486+0000","last_clean":"2023-09-23T23:24:51.755486+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:24:51.755486+0000","last_undegraded":"2023-09-23T23:24:51.755486+0000","last_fullsized":"2023-09-23T23:24:51.755486+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":683,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.768056+0000","last_change":"2023-09-23T23:23:56.246175+0000","last_active":"2023-09-23T23:24:51.768056+0000","last_peered":"2023-09-23T23:24:51.768056+0000","last_clean":"2023-09-23T23:24:51.768056+0000","last_became_active":"2023-09-23T23:23:45.458251+0000","last_became_peered":"2023-09-23T23:23:45.458251+0000","last_unstale":"2023-09-23T23:24:51.768056+0000","last_undegraded":"2023-09-23T23:24:51.768056+0000","last_fullsized":"2023-09-23T23:24:51.768056+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:41.637297+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":365,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T23:25:00.034147+0000","last_change":"2023-09-23T23:20:15.786279+0000","last_active":"2023-09-23T23:25:00.034147+0000","last_peered":"2023-09-23T23:25:00.034147+0000","last_clean":"2023-09-23T23:25:00.034147+0000","last_became_active":"2023-09-23T23:20:15.782731+0000","last_became_peered":"2023-09-23T23:20:15.782731+0000","last_unstale":"2023-09-23T23:25:00.034147+0000","last_undegraded":"2023-09-23T23:25:00.034147+0000","last_fullsized":"2023-09-23T23:25:00.034147+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:58.011396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":397,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-09-23T23:24:51.769479+0000","last_change":"2023-09-23T23:22:38.447350+0000","last_active":"2023-09-23T23:24:51.769479+0000","last_peered":"2023-09-23T23:24:51.769479+0000","last_clean":"2023-09-23T23:24:51.769479+0000","last_became_active":"2023-09-23T23:22:38.447081+0000","last_became_peered":"2023-09-23T23:22:38.447081+0000","last_unstale":"2023-09-23T23:24:51.769479+0000","last_undegraded":"2023-09-23T23:24:51.769479+0000","last_fullsized":"2023-09-23T23:24:51.769479+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:39:12.306545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":5948702720,"num_objects":90770,"num_object_clones":0,"num_object_copies":272310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":90770,"num_write_kb":5809280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17825988608,"data_stored":17825988608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87570,"ondisk_log_size":87570,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":1220,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782547,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2764160,"kb_used_data":2683000,"kb_used_omap":8,"kb_used_meta":81143,"kb_avail":90980992,"statfs":{"total":95995035648,"available":93164535808,"internally_reserved":0,"allocated":2747392000,"data_stored":2746392930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":83090501},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:24:37 2023","interfaces":[{"interface":"back","average":{"1min":1.486,"5min":1.486,"15min":1.486},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.7629999999999999,"5min":6.7629999999999999,"15min":6.7629999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.1609999999999996,"5min":4.1609999999999996,"15min":4.1609999999999996},"last":0.55900000000000005}]},{"osd":1,"last update":"Sat Sep 23 23:24:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.0389999999999999,"15min":1.0389999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.8129999999999997,"5min":6.8129999999999997,"15min":6.8129999999999997},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.4690000000000001,"15min":1.4690000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.3860000000000001,"5min":5.3860000000000001,"15min":5.3860000000000001},"last":0.45700000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:24:37 2023","interfaces":[{"interface":"back","average":{"1min":1.833,"5min":1.833,"15min":1.833},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":6.8579999999999997,"5min":6.8579999999999997,"15min":6.8579999999999997},"last":1.5960000000000001},{"interface":"front","average":{"1min":1.1919999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.1799999999999997,"5min":4.1799999999999997,"15min":4.1799999999999997},"last":4.1529999999999996}]},{"osd":3,"last update":"Sat Sep 23 23:24:37 2023","interfaces":[{"interface":"back","average":{"1min":1.49,"5min":1.49,"15min":1.49},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":5.0659999999999998,"5min":5.0659999999999998,"15min":5.0659999999999998},"last":1.1619999999999999},{"interface":"front","average":{"1min":1.294,"5min":1.294,"15min":1.294},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.8890000000000002,"5min":6.8890000000000002,"15min":6.8890000000000002},"last":6.0339999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:24:37 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.8380000000000001,"5min":6.8380000000000001,"15min":6.8380000000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.9039999999999999,"5min":3.9039999999999999,"15min":3.9039999999999999},"last":0.504}]},{"osd":5,"last update":"Sat Sep 23 23:24:37 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":1.087,"15min":1.087},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":3.8660000000000001,"5min":3.8660000000000001,"15min":3.8660000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.383,"5min":2.383,"15min":2.383},"last":0.85499999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:24:37 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.755,"15min":0.755},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":2.4590000000000001,"5min":2.4590000000000001,"15min":2.4590000000000001},"last":0.88400000000000001},{"interface":"front","average":{"1min":1.089,"5min":1.089,"15min":1.089},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.9219999999999997,"5min":6.9219999999999997,"15min":6.9219999999999997},"last":0.61099999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496024,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3808120,"kb_used_data":3300144,"kb_used_omap":0,"kb_used_meta":507968,"kb_avail":89937032,"statfs":{"total":95995035648,"available":92095520768,"internally_reserved":0,"allocated":3379347456,"data_stored":3378355769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":520159232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,2,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:24:54 2023","interfaces":[{"interface":"back","average":{"1min":8.2720000000000002,"5min":6.2709999999999999,"15min":5.5579999999999998},"min":{"1min":0.26400000000000001,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":121.697,"5min":121.697,"15min":161.64599999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":7.7039999999999997,"5min":5.7830000000000004,"15min":5.3860000000000001},"min":{"1min":0.252,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":121.667,"5min":121.667,"15min":161.38499999999999},"last":0.313}]},{"osd":1,"last update":"Sat Sep 23 23:24:54 2023","interfaces":[{"interface":"back","average":{"1min":4.2039999999999997,"5min":2.1850000000000001,"15min":4.3959999999999999},"min":{"1min":0.29999999999999999,"5min":0.245,"15min":0.24099999999999999},"max":{"1min":67.503,"5min":67.503,"15min":210.869},"last":0.35599999999999998},{"interface":"front","average":{"1min":4.25,"5min":2.198,"15min":4.4320000000000004},"min":{"1min":0.26300000000000001,"5min":0.247,"15min":0.247},"max":{"1min":67.474999999999994,"5min":67.474999999999994,"15min":210.89699999999999},"last":0.41699999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:24:37 2023","interfaces":[{"interface":"back","average":{"1min":10.061999999999999,"5min":3.984,"15min":2.5699999999999998},"min":{"1min":0.42599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":127.44499999999999,"5min":127.44499999999999,"15min":127.44499999999999},"last":5.3559999999999999},{"interface":"front","average":{"1min":10.121,"5min":4.0810000000000004,"15min":2.3090000000000002},"min":{"1min":0.28199999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":125.175,"5min":125.175,"15min":125.175},"last":5.4450000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:24:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":1.2869999999999999,"15min":2.7869999999999999},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.7090000000000001,"5min":10.237,"15min":109.98699999999999},"last":0.374},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.371,"15min":2.8690000000000002},"min":{"1min":0.35799999999999998,"5min":0.28799999999999998,"15min":0.24199999999999999},"max":{"1min":5.4189999999999996,"5min":6.4539999999999997,"15min":110.024},"last":0.442}]},{"osd":4,"last update":"Sat Sep 23 23:24:10 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":2.3559999999999999,"15min":1.5680000000000001},"min":{"1min":0.28499999999999998,"5min":0.23400000000000001,"15min":0.22900000000000001},"max":{"1min":1.1000000000000001,"5min":166.05199999999999,"15min":166.05199999999999},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.61399999999999999,"5min":2.9209999999999998,"15min":1.9139999999999999},"min":{"1min":0.31900000000000001,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":3.6619999999999999,"5min":166.631,"15min":166.631},"last":0.42999999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:24:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.69999999999999996,"15min":1.4099999999999999},"min":{"1min":0.316,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.329,"5min":3.5579999999999998,"15min":91.899000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.60599999999999998,"5min":1.0880000000000001,"15min":1.4299999999999999},"min":{"1min":0.35099999999999998,"5min":0.249,"15min":0.249},"max":{"1min":3.7490000000000001,"5min":8.4090000000000007,"15min":94.174000000000007},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:24:37 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":3.8079999999999998,"5min":3.8079999999999998,"15min":3.8079999999999998},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.9790000000000001,"5min":2.9790000000000001,"15min":2.9790000000000001},"last":0.501}]}]},{"osd":1,"up_from":112,"seq":481036337365,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1980788,"kb_used_data":1445932,"kb_used_omap":4,"kb_used_meta":534843,"kb_avail":91764364,"statfs":{"total":95995035648,"available":93966708736,"internally_reserved":0,"allocated":1480634368,"data_stored":1478999433,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":547680142},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:24:53 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":2.7559999999999998,"15min":2.6600000000000001},"min":{"1min":0.23100000000000001,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":4.8920000000000003,"5min":75.823999999999998,"15min":75.823999999999998},"last":8.4619999999999997},{"interface":"front","average":{"1min":0.84699999999999998,"5min":2.4420000000000002,"15min":2.7650000000000001},"min":{"1min":0.21299999999999999,"5min":0.193,"15min":0.192},"max":{"1min":3.6480000000000001,"5min":75.840999999999994,"15min":75.840999999999994},"last":3.8290000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:24:35 2023","interfaces":[{"interface":"back","average":{"1min":10.827,"5min":3.532,"15min":4.8630000000000004},"min":{"1min":0.25800000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":209.84200000000001,"5min":209.84200000000001,"15min":209.84200000000001},"last":8.5009999999999994},{"interface":"front","average":{"1min":10.446999999999999,"5min":3.8490000000000002,"15min":3.669},"min":{"1min":0.23899999999999999,"5min":0.218,"15min":0.218},"max":{"1min":209.11099999999999,"5min":209.11099999999999,"15min":209.11099999999999},"last":5.8200000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:24:32 2023","interfaces":[{"interface":"back","average":{"1min":7.4960000000000004,"5min":2.5609999999999999,"15min":3.8359999999999999},"min":{"1min":0.25700000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":157.90899999999999,"5min":157.90899999999999,"15min":157.90899999999999},"last":3.899},{"interface":"front","average":{"1min":7.4550000000000001,"5min":2.3879999999999999,"15min":3.6339999999999999},"min":{"1min":0.30099999999999999,"5min":0.245,"15min":0.245},"max":{"1min":156.27000000000001,"5min":156.27000000000001,"15min":156.27000000000001},"last":0.378}]},{"osd":4,"last update":"Sat Sep 23 23:24:53 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.1439999999999999,"15min":1.546},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":3.9740000000000002,"5min":6.6120000000000001,"15min":18.474},"last":2.3050000000000002},{"interface":"front","average":{"1min":1.786,"5min":1.2330000000000001,"15min":1.7769999999999999},"min":{"1min":0.35099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.7729999999999997,"5min":7.2439999999999998,"15min":25.710999999999999},"last":4.4610000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:24:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":0.98699999999999999,"15min":3.2069999999999999},"min":{"1min":0.31900000000000001,"5min":0.31,"15min":0.26400000000000001},"max":{"1min":2.706,"5min":4.827,"15min":250.33000000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.347,"5min":1.367,"15min":3.1619999999999999},"min":{"1min":0.38800000000000001,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":3.5880000000000001,"5min":10.92,"15min":250.21199999999999},"last":2.3290000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:24:53 2023","interfaces":[{"interface":"back","average":{"1min":6.3280000000000003,"5min":2.677,"15min":2.4319999999999999},"min":{"1min":0.36699999999999999,"5min":0.33800000000000002,"15min":0.32400000000000001},"max":{"1min":74.087999999999994,"5min":74.087999999999994,"15min":74.087999999999994},"last":3.859},{"interface":"front","average":{"1min":6.0540000000000003,"5min":2.8149999999999999,"15min":2.4060000000000001},"min":{"1min":0.33100000000000002,"5min":0.32800000000000001,"15min":0.32700000000000001},"max":{"1min":73.974999999999994,"5min":73.974999999999994,"15min":73.974999999999994},"last":2.2749999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:24:35 2023","interfaces":[{"interface":"back","average":{"1min":4.6159999999999997,"5min":4.6159999999999997,"15min":4.6159999999999997},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":78.308000000000007,"5min":78.308000000000007,"15min":78.308000000000007},"last":4.4939999999999998},{"interface":"front","average":{"1min":4.069,"5min":4.069,"15min":4.069},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":77.204999999999998,"5min":77.204999999999998,"15min":77.204999999999998},"last":2.246}]}]},{"osd":0,"up_from":91,"seq":390842024189,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4236068,"kb_used_data":3302044,"kb_used_omap":5,"kb_used_meta":934010,"kb_avail":89509084,"statfs":{"total":95995035648,"available":91657302016,"internally_reserved":0,"allocated":3381293056,"data_stored":3379609998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":956426874},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:24:05 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.835,"15min":3.1080000000000001},"min":{"1min":0.221,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":3.923,"5min":21.585999999999999,"15min":155.91999999999999},"last":139.30000000000001},{"interface":"front","average":{"1min":1.3819999999999999,"5min":1.532,"15min":2.7410000000000001},"min":{"1min":0.221,"5min":0.183,"15min":0.183},"max":{"1min":6.9379999999999997,"5min":9.6980000000000004,"15min":153.28899999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:24:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1589999999999998,"5min":6.335,"15min":4.0780000000000003},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":29.007000000000001,"5min":171.15100000000001,"15min":171.15100000000001},"last":139.471},{"interface":"front","average":{"1min":2.77,"5min":6.5250000000000004,"15min":4.6589999999999998},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":28.988,"5min":173.04900000000001,"15min":173.04900000000001},"last":139.04400000000001}]},{"osd":3,"last update":"Sat Sep 23 23:24:15 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.9650000000000001,"15min":3.1070000000000002},"min":{"1min":0.27300000000000002,"5min":0.245,"15min":0.22700000000000001},"max":{"1min":3.8889999999999998,"5min":35.148000000000003,"15min":82.691000000000003},"last":139.34899999999999},{"interface":"front","average":{"1min":1.1579999999999999,"5min":2.0489999999999999,"15min":2.5699999999999998},"min":{"1min":0.29799999999999999,"5min":0.26500000000000001,"15min":0.23200000000000001},"max":{"1min":6.9850000000000003,"5min":35.186999999999998,"15min":82.429000000000002},"last":138.739}]},{"osd":4,"last update":"Sat Sep 23 23:24:38 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":2.4769999999999999,"15min":2.1709999999999998},"min":{"1min":0.40100000000000002,"5min":0.29099999999999998,"15min":0.247},"max":{"1min":3.3650000000000002,"5min":86.915999999999997,"15min":86.915999999999997},"last":138.81299999999999},{"interface":"front","average":{"1min":1.2869999999999999,"5min":4.0369999999999999,"15min":2.7490000000000001},"min":{"1min":0.31,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.375,"5min":171.251,"15min":171.251},"last":139.19800000000001}]},{"osd":5,"last update":"Sat Sep 23 23:24:38 2023","interfaces":[{"interface":"back","average":{"1min":1.2170000000000001,"5min":3.0950000000000002,"15min":2.3420000000000001},"min":{"1min":0.33200000000000002,"5min":0.32100000000000001,"15min":0.29099999999999998},"max":{"1min":5.4480000000000004,"5min":171.30699999999999,"15min":171.30699999999999},"last":139.119},{"interface":"front","average":{"1min":0.81000000000000005,"5min":1.26,"15min":1.645},"min":{"1min":0.35199999999999998,"5min":0.34300000000000003,"15min":0.32100000000000001},"max":{"1min":2.774,"5min":7.1769999999999996,"15min":15.929},"last":139.40799999999999}]},{"osd":6,"last update":"Sat Sep 23 23:24:38 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":3.234,"15min":2.5960000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":5.4199999999999999,"5min":171.33600000000001,"15min":171.33600000000001},"last":139.16399999999999},{"interface":"front","average":{"1min":0.95599999999999996,"5min":1.5229999999999999,"15min":2.0289999999999999},"min":{"1min":0.38500000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":2.7410000000000001,"5min":7.218,"15min":30.673999999999999},"last":139.43899999999999}]},{"osd":7,"last update":"Sat Sep 23 23:24:34 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.341,"15min":1.341},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":9.843,"5min":9.843,"15min":9.843},"last":138.77699999999999},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":2.7090000000000001,"5min":2.7090000000000001,"15min":2.7090000000000001},"last":139.37899999999999}]}]},{"osd":2,"up_from":199,"seq":854698492009,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3857976,"kb_used_data":3512048,"kb_used_omap":7,"kb_used_meta":345912,"kb_avail":89887176,"statfs":{"total":95995035648,"available":92044468224,"internally_reserved":0,"allocated":3596337152,"data_stored":3594700851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":354213969},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,3,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:24:36 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":4.8099999999999996,"15min":5.0899999999999999},"min":{"1min":0.19900000000000001,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":8.6440000000000001,"5min":119.277,"15min":119.277},"last":5.1139999999999999},{"interface":"front","average":{"1min":1.266,"5min":4.3440000000000003,"15min":5.2789999999999999},"min":{"1min":0.222,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":5.8440000000000003,"5min":119.56100000000001,"15min":119.56100000000001},"last":6.7130000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:24:36 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":2.6110000000000002,"15min":9.1289999999999996},"min":{"1min":0.23899999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":1.9550000000000001,"5min":157.98699999999999,"15min":157.98699999999999},"last":3.0249999999999999},{"interface":"front","average":{"1min":1.2689999999999999,"5min":3.1150000000000002,"15min":8.7159999999999993},"min":{"1min":0.26400000000000001,"5min":0.26300000000000001,"15min":0.245},"max":{"1min":5.5640000000000001,"5min":158.03800000000001,"15min":158.03800000000001},"last":5.1710000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:24:36 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":2.3140000000000001,"15min":5.5},"min":{"1min":0.23499999999999999,"5min":0.223,"15min":0.223},"max":{"1min":5.9249999999999998,"5min":119.366,"15min":119.366},"last":3.0009999999999999},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.71,"15min":4.7709999999999999},"min":{"1min":0.28399999999999997,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.5739999999999998,"5min":12.047000000000001,"15min":72.412999999999997},"last":2.8519999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:24:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":2.4889999999999999,"15min":3.8769999999999998},"min":{"1min":0.41799999999999998,"5min":0.25,"15min":0.25},"max":{"1min":4.2560000000000002,"5min":119.88200000000001,"15min":332.98099999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.94599999999999995,"15min":3.3410000000000002},"min":{"1min":0.317,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.2879999999999998,"5min":5.4290000000000003,"15min":332.96199999999999},"last":2.8969999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:24:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":2.8279999999999998,"15min":2.2730000000000001},"min":{"1min":0.39600000000000002,"5min":0.29499999999999998,"15min":0.27500000000000002},"max":{"1min":4.1909999999999998,"5min":119.919,"15min":119.919},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.755,"5min":2.4950000000000001,"15min":2.2269999999999999},"min":{"1min":0.39300000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.1789999999999998,"5min":119.306,"15min":119.306},"last":3.355}]},{"osd":6,"last update":"Sat Sep 23 23:24:36 2023","interfaces":[{"interface":"back","average":{"1min":1.226,"5min":3.0539999999999998,"15min":4.1609999999999996},"min":{"1min":0.40899999999999997,"5min":0.32000000000000001,"15min":0.31},"max":{"1min":4.1479999999999997,"5min":119.943,"15min":349.32799999999997},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.78100000000000003,"5min":1.5980000000000001,"15min":3.6219999999999999},"min":{"1min":0.40400000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.9339999999999999,"5min":39.856999999999999,"15min":349.17700000000002},"last":2.9649999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:24:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":1.1759999999999999,"15min":1.1759999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":5.742,"5min":5.742,"15min":5.742},"last":0.66800000000000004},{"interface":"front","average":{"1min":1.266,"5min":1.266,"15min":1.266},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":5.7069999999999999,"5min":5.7069999999999999,"15min":5.7069999999999999},"last":0.624}]}]},{"osd":3,"up_from":148,"seq":635655159979,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1906944,"kb_used_data":1451960,"kb_used_omap":2,"kb_used_meta":454973,"kb_avail":91838208,"statfs":{"total":95995035648,"available":94042324992,"internally_reserved":0,"allocated":1486807040,"data_stored":1485217447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":465892405},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:24:28 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":2.976,"15min":2.722},"min":{"1min":0.23100000000000001,"5min":0.158,"15min":0.158},"max":{"1min":5.2640000000000002,"5min":103.866,"15min":103.866},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.82299999999999995,"5min":2.1819999999999999,"15min":2.5590000000000002},"min":{"1min":0.23400000000000001,"5min":0.193,"15min":0.17999999999999999},"max":{"1min":4.6589999999999998,"5min":101.884,"15min":101.884},"last":0.40600000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:24:28 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":1.6459999999999999,"15min":2.6539999999999999},"min":{"1min":0.253,"5min":0.216,"15min":0.19600000000000001},"max":{"1min":5.7350000000000003,"5min":14.427,"15min":76.322999999999993},"last":0.29499999999999998},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.137,"15min":2.347},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.185},"max":{"1min":4.0449999999999999,"5min":7.9180000000000001,"15min":76.444000000000003},"last":3.306}]},{"osd":2,"last update":"Sat Sep 23 23:24:38 2023","interfaces":[{"interface":"back","average":{"1min":1.617,"5min":4.2290000000000001,"15min":3.6429999999999998},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.292999999999999,"5min":101.279,"15min":101.279},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.833,"5min":3.8780000000000001,"15min":2.4079999999999999},"min":{"1min":0.309,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":11.217000000000001,"5min":101.294,"15min":101.294},"last":0.89600000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:24:28 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":1.165,"15min":1.506},"min":{"1min":0.317,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.016,"5min":9.2089999999999996,"15min":14.754},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.5940000000000001,"15min":1.7989999999999999},"min":{"1min":0.377,"5min":0.33600000000000002,"15min":0.28499999999999998},"max":{"1min":4.7759999999999998,"5min":9.5960000000000001,"15min":13.532999999999999},"last":0.84899999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:24:28 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.355,"15min":1.859},"min":{"1min":0.42099999999999999,"5min":0.307,"15min":0.28199999999999997},"max":{"1min":4.726,"5min":10.531000000000001,"15min":75.733000000000004},"last":4.1349999999999998},{"interface":"front","average":{"1min":1.371,"5min":1.6299999999999999,"15min":2.0600000000000001},"min":{"1min":0.36499999999999999,"5min":0.33800000000000002,"15min":0.28399999999999997},"max":{"1min":5.6699999999999999,"5min":6.4269999999999996,"15min":75.686999999999998},"last":0.50700000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:24:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":1.6160000000000001,"15min":1.7529999999999999},"min":{"1min":0.45500000000000002,"5min":0.38,"15min":0.32200000000000001},"max":{"1min":4.7000000000000002,"5min":10.593999999999999,"15min":14.492000000000001},"last":0.63},{"interface":"front","average":{"1min":1.2909999999999999,"5min":1.784,"15min":1.863},"min":{"1min":0.39000000000000001,"5min":0.36599999999999999,"15min":0.33300000000000002},"max":{"1min":5.7050000000000001,"5min":6.7069999999999999,"15min":9.8379999999999992},"last":0.55900000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:24:35 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":3.9740000000000002,"5min":3.9740000000000002,"15min":3.9740000000000002},"last":0.95499999999999996},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":5.149,"5min":5.149,"15min":5.149},"last":1.756}]}]},{"osd":4,"up_from":30,"seq":128849019297,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1724212,"kb_used_data":1113580,"kb_used_omap":0,"kb_used_meta":610624,"kb_avail":92020940,"statfs":{"total":95995035648,"available":94229442560,"internally_reserved":0,"allocated":1140305920,"data_stored":1139314191,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":625278976},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:24:46 2023","interfaces":[{"interface":"back","average":{"1min":3.782,"5min":4.7460000000000004,"15min":5.4269999999999996},"min":{"1min":0.24299999999999999,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":56.279000000000003,"5min":92.141999999999996,"15min":238.62200000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":3.6949999999999998,"5min":5.0860000000000003,"15min":5.7519999999999998},"min":{"1min":0.27600000000000002,"5min":0.216,"15min":0.216},"max":{"1min":56.386000000000003,"5min":92.284000000000006,"15min":238.58199999999999},"last":1.9379999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.077,"15min":4.79},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.6550000000000002,"5min":7.5110000000000001,"15min":175.00899999999999},"last":0.251},{"interface":"front","average":{"1min":1.7490000000000001,"5min":1.325,"15min":5.0140000000000002},"min":{"1min":0.28399999999999997,"5min":0.27500000000000002,"15min":0.25800000000000001},"max":{"1min":9.7690000000000001,"5min":9.7690000000000001,"15min":174.821},"last":0.42999999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:24:39 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":2.6030000000000002,"15min":3.0470000000000002},"min":{"1min":0.35899999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.085,"5min":63.113999999999997,"15min":63.113999999999997},"last":1.5},{"interface":"front","average":{"1min":1.0269999999999999,"5min":2.6219999999999999,"15min":2.5790000000000002},"min":{"1min":0.32600000000000001,"5min":0.317,"15min":0.316},"max":{"1min":3.4470000000000001,"5min":63.125,"15min":63.125},"last":1.6299999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:24:29 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":1.9650000000000001,"15min":4.2649999999999997},"min":{"1min":0.27200000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":23.696000000000002,"5min":65.855999999999995,"15min":303.41500000000002},"last":8},{"interface":"front","average":{"1min":1.8169999999999999,"5min":2.4129999999999998,"15min":4.5640000000000001},"min":{"1min":0.36599999999999999,"5min":0.317,"15min":0.308},"max":{"1min":23.756,"5min":66.293000000000006,"15min":303.428},"last":0.46300000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.82499999999999996,"15min":1.5349999999999999},"min":{"1min":0.26700000000000002,"5min":0.26100000000000001,"15min":0.23699999999999999},"max":{"1min":2.278,"5min":4.8319999999999999,"15min":160.79499999999999},"last":1.246},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.0880000000000001,"15min":1.7929999999999999},"min":{"1min":0.31,"5min":0.249,"15min":0.249},"max":{"1min":2.577,"5min":7.9669999999999996,"15min":160.61099999999999},"last":1.2749999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:24:29 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":1.581,"15min":1.3480000000000001},"min":{"1min":0.31900000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.0449999999999999,"5min":90.346999999999994,"15min":90.346999999999994},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.9710000000000001,"15min":1.4419999999999999},"min":{"1min":0.40799999999999997,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":0.81899999999999995,"5min":90.733000000000004,"15min":90.733000000000004},"last":0.45100000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:24:39 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":4.7910000000000004,"5min":4.7910000000000004,"15min":4.7910000000000004},"last":1.798}]}]},{"osd":5,"up_from":38,"seq":163208757660,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1259376,"kb_used_data":727720,"kb_used_omap":0,"kb_used_meta":531648,"kb_avail":92485776,"statfs":{"total":95995035648,"available":94705434624,"internally_reserved":0,"allocated":745185280,"data_stored":744197846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":544407552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:24:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1339999999999999,"5min":6.8419999999999996,"15min":3.5339999999999998},"min":{"1min":0.26100000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":4.4580000000000002,"5min":113.869,"15min":113.869},"last":1.044},{"interface":"front","average":{"1min":1.3560000000000001,"5min":7.9720000000000004,"15min":4.1769999999999996},"min":{"1min":0.36099999999999999,"5min":0.23000000000000001,"15min":0.22700000000000001},"max":{"1min":4.7009999999999996,"5min":138.55199999999999,"15min":138.55199999999999},"last":0.22900000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:24:58 2023","interfaces":[{"interface":"back","average":{"1min":5.0590000000000002,"5min":1.962,"15min":3.9350000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.246},"max":{"1min":56.634,"5min":56.634,"15min":193.584},"last":0.33900000000000002},{"interface":"front","average":{"1min":5.9870000000000001,"5min":2.5219999999999998,"15min":4.2160000000000002},"min":{"1min":0.34300000000000003,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":55.143999999999998,"5min":55.143999999999998,"15min":194.03100000000001},"last":1.361}]},{"osd":2,"last update":"Sat Sep 23 23:24:34 2023","interfaces":[{"interface":"back","average":{"1min":3.7480000000000002,"5min":2.839,"15min":1.929},"min":{"1min":0.35299999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":56.744,"5min":86.765000000000001,"15min":86.765000000000001},"last":6.7320000000000002},{"interface":"front","average":{"1min":3.98,"5min":2.6240000000000001,"15min":1.9610000000000001},"min":{"1min":0.35399999999999998,"5min":0.29599999999999999,"15min":0.25800000000000001},"max":{"1min":57.354999999999997,"5min":86.837000000000003,"15min":86.837000000000003},"last":0.81399999999999995}]},{"osd":3,"last update":"Sat Sep 23 23:24:26 2023","interfaces":[{"interface":"back","average":{"1min":3.6960000000000002,"5min":1.8919999999999999,"15min":4.1500000000000004},"min":{"1min":0.315,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":57.283999999999999,"5min":57.283999999999999,"15min":159.53299999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":3.9409999999999998,"5min":2.1400000000000001,"15min":4.6189999999999998},"min":{"1min":0.34999999999999998,"5min":0.33100000000000002,"15min":0.312},"max":{"1min":57.334000000000003,"5min":57.334000000000003,"15min":159.54900000000001},"last":0.504}]},{"osd":4,"last update":"Sat Sep 23 23:24:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.82199999999999995,"15min":1.2310000000000001},"min":{"1min":0.251,"5min":0.24299999999999999,"15min":0.23999999999999999},"max":{"1min":0.58799999999999997,"5min":8.1210000000000004,"15min":14.538},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.75600000000000001,"15min":0.877},"min":{"1min":0.28100000000000003,"5min":0.245,"15min":0.245},"max":{"1min":1.075,"5min":8.1829999999999998,"15min":11.923999999999999},"last":0.29699999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:24:09 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":4.0899999999999999,"15min":2.1890000000000001},"min":{"1min":0.32200000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.391,"5min":310.68799999999999,"15min":310.68799999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.503,"5min":4.4550000000000001,"15min":2.4769999999999999},"min":{"1min":0.312,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.997,"5min":311.45400000000001,"15min":311.45400000000001},"last":0.315}]},{"osd":7,"last update":"Sat Sep 23 23:24:34 2023","interfaces":[{"interface":"back","average":{"1min":4.1470000000000002,"5min":4.1470000000000002,"15min":4.1470000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":66.372,"5min":66.372,"15min":66.372},"last":0.40100000000000002},{"interface":"front","average":{"1min":4.3360000000000003,"5min":4.3360000000000003,"15min":4.3360000000000003},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":66.558999999999997,"5min":66.558999999999997,"15min":66.558999999999997},"last":0.46899999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3400728576,"data_stored":3400728576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1485766656,"data_stored":1485766656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3614048256,"data_stored":3614048256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1483341824,"data_stored":1483341824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1125384192,"data_stored":1125384192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":742850560,"data_stored":742850560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3384475648,"data_stored":3384475648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2589392896,"data_stored":2589392896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:25:09.110 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T23:25:09.110 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_6 pg_num 26 2023-09-23T23:25:09.148 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:09.172 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 77 16 95288 95272 77.2704 48.625 0.00989548 0.0129368 2023-09-23T23:25:09.189 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:09.259 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:09.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:09.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:09.367 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:25:09.430 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:09.545 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:25:09.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:09.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:25:09.650 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:25:09.695 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:09.766 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:25:09.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:09.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:25:09.874 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:09.930 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:10.021 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:10.031 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:09 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:10.031 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:09 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:10.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:10.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:10.135 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:25:10.172 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 78 16 96234 96218 77.0378 59.125 0.0127673 0.0129761 2023-09-23T23:25:10.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:09 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:10.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:09 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:10.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:09 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:10.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:09 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:10.224 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:10.325 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:25:10.360 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:25:10.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:10.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:25:10.478 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:10.509 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:10.569 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:10.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:10.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:10.674 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:25:10.750 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:10.757 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:11.023 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:10 smithi110 ceph-mon[212939]: pgmap v1081: 119 pgs: 119 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2023-09-23T23:25:11.027 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:10 smithi110 ceph-mon[221102]: pgmap v1081: 119 pgs: 119 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2023-09-23T23:25:11.172 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 79 16 96961 96945 76.638 45.4375 0.0110497 0.013044 2023-09-23T23:25:11.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:10 smithi138 ceph-mon[193606]: pgmap v1081: 119 pgs: 119 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2023-09-23T23:25:11.358 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:25:11.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:11.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:25:11.470 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:11.481 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:25:11.481 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:25:11.512 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:11.567 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:11.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:11.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:11.672 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:25:11.722 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:11.786 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:25:11.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:11.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:25:11.891 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:25:11.928 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:12.014 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:25:12.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:12.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:25:12.123 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:12.153 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:12.173 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:25:12.172721+0000 min lat: 0.00111746 max lat: 0.34576 avg lat: 0.0131107 2023-09-23T23:25:12.173 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:25:12.173 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 80 16 97688 97672 76.2479 45.4375 0.0234956 0.0131107 2023-09-23T23:25:12.223 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:12.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:12.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:12.336 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:25:12.407 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:12.581 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:25:12.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:12.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:25:12.622 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:25:12.691 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:12.798 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:12.819 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:12.955 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:25:12.994 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:12 smithi110 ceph-mon[212939]: pgmap v1082: 119 pgs: 119 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s 2023-09-23T23:25:12.994 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:12 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/617078053' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:25:12.994 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:12 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:25:12.994 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:12 smithi110 ceph-mon[221102]: pgmap v1082: 119 pgs: 119 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s 2023-09-23T23:25:12.994 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:12 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/617078053' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:25:12.995 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:12 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:25:13.058 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:13.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:13.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:13.172 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:25:13.180 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 81 16 98520 98504 75.9452 52 0.0139973 0.0131628 2023-09-23T23:25:13.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:12 smithi138 ceph-mon[193606]: pgmap v1082: 119 pgs: 119 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s 2023-09-23T23:25:13.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:12 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/617078053' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:25:13.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:12 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:25:13.257 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:14.180 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 82 16 99282 99266 75.6 47.625 0.0068775 0.0132228 2023-09-23T23:25:14.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:13 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:25:14.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:13 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:25:14.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:13 smithi138 ceph-mon[193606]: failed to encode map e363 with expected crc 2023-09-23T23:25:14.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:13 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:25:14.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:13 smithi138 ceph-mon[193606]: osdmap e363: 8 total, 8 up, 8 in 2023-09-23T23:25:14.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:13 smithi138 ceph-mon[193606]: failed to encode map e363 with expected crc 2023-09-23T23:25:14.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:13 smithi138 ceph-mon[193606]: failed to encode map e363 with expected crc 2023-09-23T23:25:14.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:14.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:25:14.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:25:14.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:25:14.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[212939]: failed to encode map e363 with expected crc 2023-09-23T23:25:14.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:25:14.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[212939]: osdmap e363: 8 total, 8 up, 8 in 2023-09-23T23:25:14.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[212939]: failed to encode map e363 with expected crc 2023-09-23T23:25:14.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[212939]: failed to encode map e363 with expected crc 2023-09-23T23:25:14.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:14.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:25:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:25:14.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:25:14.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[221102]: failed to encode map e363 with expected crc 2023-09-23T23:25:14.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:25:14.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[221102]: osdmap e363: 8 total, 8 up, 8 in 2023-09-23T23:25:14.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[221102]: failed to encode map e363 with expected crc 2023-09-23T23:25:14.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[221102]: failed to encode map e363 with expected crc 2023-09-23T23:25:14.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:14.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:25:14.445 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:25:14.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:14.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:25:14.552 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:14.637 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:14.722 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:14.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:14.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:14.832 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:25:14.905 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:15.177 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 83 16 100156 100140 75.3474 54.625 0.00909191 0.013267 2023-09-23T23:25:15.186 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:25:15.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:14 smithi138 ceph-mon[193606]: pgmap v1084: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-09-23T23:25:15.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:14 smithi110 ceph-mon[221102]: pgmap v1084: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-09-23T23:25:15.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:15.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:25:15.193 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:14 smithi110 ceph-mon[212939]: pgmap v1084: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-09-23T23:25:15.293 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:25:15.336 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:15.405 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:25:15.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:15.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:25:15.512 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:15.587 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:16.177 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 84 16 101154 101138 75.193 62.375 0.0243657 0.0132946 2023-09-23T23:25:16.195 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:16.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:16.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:16.267 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:16.305 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:16.429 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:16.513 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:25:16.716 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:16.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:16.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:16.835 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:16.929 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:25:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:25:16] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.43.0" 2023-09-23T23:25:16.944 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:16.960 INFO:teuthology.orchestra.run.smithi110.stderr:set pool 11 pg_num to 26 2023-09-23T23:25:17.177 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 85 16 101918 101902 74.8703 47.75 0.0097141 0.013352 2023-09-23T23:25:17.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:16 smithi110 ceph-mon[212939]: pgmap v1085: 119 pgs: 119 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-09-23T23:25:17.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:16 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2951247219' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T23:25:17.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:16 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T23:25:17.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:16 smithi110 ceph-mon[221102]: pgmap v1085: 119 pgs: 119 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-09-23T23:25:17.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:16 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2951247219' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T23:25:17.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:16 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T23:25:17.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:16 smithi138 ceph-mon[193606]: pgmap v1085: 119 pgs: 119 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-09-23T23:25:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:16 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2951247219' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T23:25:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:16 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T23:25:18.177 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 86 16 102609 102593 74.5019 43.1875 0.160771 0.0133905 2023-09-23T23:25:18.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:17 smithi110 ceph-mon[212939]: failed to encode map e364 with expected crc 2023-09-23T23:25:18.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:17 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]': finished 2023-09-23T23:25:18.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:17 smithi110 ceph-mon[212939]: osdmap e364: 8 total, 8 up, 8 in 2023-09-23T23:25:18.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:17 smithi110 ceph-mon[212939]: failed to encode map e364 with expected crc 2023-09-23T23:25:18.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:17 smithi110 ceph-mon[212939]: failed to encode map e364 with expected crc 2023-09-23T23:25:18.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:17 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T23:25:18.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:17 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T23:25:18.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:17 smithi110 ceph-mon[221102]: failed to encode map e364 with expected crc 2023-09-23T23:25:18.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:17 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]': finished 2023-09-23T23:25:18.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:17 smithi110 ceph-mon[221102]: osdmap e364: 8 total, 8 up, 8 in 2023-09-23T23:25:18.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:17 smithi110 ceph-mon[221102]: failed to encode map e364 with expected crc 2023-09-23T23:25:18.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:17 smithi110 ceph-mon[221102]: failed to encode map e364 with expected crc 2023-09-23T23:25:18.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:17 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T23:25:18.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:17 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T23:25:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:17 smithi138 ceph-mon[193606]: failed to encode map e364 with expected crc 2023-09-23T23:25:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:17 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]': finished 2023-09-23T23:25:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:17 smithi138 ceph-mon[193606]: osdmap e364: 8 total, 8 up, 8 in 2023-09-23T23:25:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:17 smithi138 ceph-mon[193606]: failed to encode map e364 with expected crc 2023-09-23T23:25:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:17 smithi138 ceph-mon[193606]: failed to encode map e364 with expected crc 2023-09-23T23:25:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:17 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T23:25:18.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:17 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T23:25:18.453 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:25:18.472 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:25:18.491 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:25:18.512 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:25:18.531 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:25:18.553 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:25:18.573 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:25:18.593 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:25:18.594 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:25:18.594 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:25:18.594 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:25:18.594 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:25:18.594 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:25:18.594 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:25:18.594 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:25:18.595 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:25:18.856 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:18.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:18.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:18.962 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:19.047 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:19.145 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:19.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:19.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:19.187 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 87 15 102828 102813 73.8008 13.75 0.0111894 0.0135458 2023-09-23T23:25:19.209 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:18 smithi110 ceph-mon[212939]: pgmap v1087: 119 pgs: 119 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 605 op/s 2023-09-23T23:25:19.209 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:18 smithi110 ceph-mon[212939]: failed to encode map e365 with expected crc 2023-09-23T23:25:19.209 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:18 smithi110 ceph-mon[212939]: failed to encode map e365 with expected crc 2023-09-23T23:25:19.209 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:18 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T23:25:19.209 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:18 smithi110 ceph-mon[212939]: osdmap e365: 8 total, 8 up, 8 in 2023-09-23T23:25:19.209 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2562163025' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:25:19.209 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:18 smithi110 ceph-mon[221102]: pgmap v1087: 119 pgs: 119 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 605 op/s 2023-09-23T23:25:19.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:18 smithi110 ceph-mon[221102]: failed to encode map e365 with expected crc 2023-09-23T23:25:19.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:18 smithi110 ceph-mon[221102]: failed to encode map e365 with expected crc 2023-09-23T23:25:19.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:18 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T23:25:19.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:18 smithi110 ceph-mon[221102]: osdmap e365: 8 total, 8 up, 8 in 2023-09-23T23:25:19.210 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2562163025' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:25:19.252 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:19.359 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:18 smithi138 ceph-mon[193606]: pgmap v1087: 119 pgs: 119 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 605 op/s 2023-09-23T23:25:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:18 smithi138 ceph-mon[193606]: failed to encode map e365 with expected crc 2023-09-23T23:25:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:18 smithi138 ceph-mon[193606]: failed to encode map e365 with expected crc 2023-09-23T23:25:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:18 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T23:25:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:18 smithi138 ceph-mon[193606]: osdmap e365: 8 total, 8 up, 8 in 2023-09-23T23:25:19.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2562163025' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:25:20.186 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:20.186 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 88 16 103827 103811 73.6698 62.375 0.0215996 0.0135699 2023-09-23T23:25:20.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:20.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:20.277 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:19 smithi110 ceph-mon[212939]: failed to encode map e366 with expected crc 2023-09-23T23:25:20.278 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:19 smithi110 ceph-mon[212939]: failed to encode map e366 with expected crc 2023-09-23T23:25:20.278 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:19 smithi110 ceph-mon[212939]: osdmap e366: 8 total, 8 up, 8 in 2023-09-23T23:25:20.278 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:19 smithi110 ceph-mon[212939]: failed to encode map e366 with expected crc 2023-09-23T23:25:20.278 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:19 smithi110 ceph-mon[212939]: failed to encode map e366 with expected crc 2023-09-23T23:25:20.278 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:19 smithi110 ceph-mon[212939]: failed to encode map e366 with expected crc 2023-09-23T23:25:20.278 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:19 smithi110 ceph-mon[212939]: 11.0 scrub starts 2023-09-23T23:25:20.278 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:19 smithi110 ceph-mon[221102]: failed to encode map e366 with expected crc 2023-09-23T23:25:20.278 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:19 smithi110 ceph-mon[221102]: failed to encode map e366 with expected crc 2023-09-23T23:25:20.279 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:19 smithi110 ceph-mon[221102]: osdmap e366: 8 total, 8 up, 8 in 2023-09-23T23:25:20.279 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:19 smithi110 ceph-mon[221102]: failed to encode map e366 with expected crc 2023-09-23T23:25:20.279 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:19 smithi110 ceph-mon[221102]: failed to encode map e366 with expected crc 2023-09-23T23:25:20.279 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:19 smithi110 ceph-mon[221102]: failed to encode map e366 with expected crc 2023-09-23T23:25:20.279 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:19 smithi110 ceph-mon[221102]: 11.0 scrub starts 2023-09-23T23:25:20.292 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:20.441 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:19 smithi138 ceph-mon[193606]: failed to encode map e366 with expected crc 2023-09-23T23:25:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:19 smithi138 ceph-mon[193606]: failed to encode map e366 with expected crc 2023-09-23T23:25:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:19 smithi138 ceph-mon[193606]: osdmap e366: 8 total, 8 up, 8 in 2023-09-23T23:25:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:19 smithi138 ceph-mon[193606]: failed to encode map e366 with expected crc 2023-09-23T23:25:20.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:19 smithi138 ceph-mon[193606]: failed to encode map e366 with expected crc 2023-09-23T23:25:20.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:19 smithi138 ceph-mon[193606]: failed to encode map e366 with expected crc 2023-09-23T23:25:20.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:19 smithi138 ceph-mon[193606]: 11.0 scrub starts 2023-09-23T23:25:20.522 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:20.616 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:20.620 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:25:20.620 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:25:20.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:25:20.621 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:25:20.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.621 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156658 10.3 10:c8345f3f:::benchmark_data_smithi110_366450_object55450:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e301)", 2023-09-23T23:25:20.621 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:50.036433+0000", 2023-09-23T23:25:20.621 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 210.575138751, 2023-09-23T23:25:20.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2122256870000001, 2023-09-23T23:25:20.622 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.622 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.622 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.622 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.622 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.623 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156658 2023-09-23T23:25:20.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.623 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.624 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.624 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.624 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036433+0000", 2023-09-23T23:25:20.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.624 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036433+0000", 2023-09-23T23:25:20.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.626 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036432+0000", 2023-09-23T23:25:20.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:25:20.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.626 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.626 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036438+0000", 2023-09-23T23:25:20.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2260000000000002e-06 2023-09-23T23:25:20.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036447+0000", 2023-09-23T23:25:20.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2599999999999994e-06 2023-09-23T23:25:20.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036454+0000", 2023-09-23T23:25:20.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2180000000000002e-06 2023-09-23T23:25:20.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038082+0000", 2023-09-23T23:25:20.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016271840000000001 2023-09-23T23:25:20.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:25:20.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038092+0000", 2023-09-23T23:25:20.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0312999999999999e-05 2023-09-23T23:25:20.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.132541+0000", 2023-09-23T23:25:20.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.094449464 2023-09-23T23:25:20.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.135895+0000", 2023-09-23T23:25:20.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033539080000000001 2023-09-23T23:25:20.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.880055+0000", 2023-09-23T23:25:20.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.744159365 2023-09-23T23:25:20.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.883944+0000", 2023-09-23T23:25:20.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038893600000000001 2023-09-23T23:25:20.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.224607+0000", 2023-09-23T23:25:20.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.34066281199999998 2023-09-23T23:25:20.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.227336+0000", 2023-09-23T23:25:20.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002729553 2023-09-23T23:25:20.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.232121+0000", 2023-09-23T23:25:20.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047848830000000002 2023-09-23T23:25:20.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.238887+0000", 2023-09-23T23:25:20.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.006765971 2023-09-23T23:25:20.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.240149+0000", 2023-09-23T23:25:20.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001261543 2023-09-23T23:25:20.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.240604+0000", 2023-09-23T23:25:20.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045544900000000002 2023-09-23T23:25:20.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.242329+0000", 2023-09-23T23:25:20.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001724864 2023-09-23T23:25:20.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.248659+0000", 2023-09-23T23:25:20.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063300179999999998 2023-09-23T23:25:20.639 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.639 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.640 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.640 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156665 10.3 10:ce3a78fd:::benchmark_data_smithi110_366450_object55457:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e301)", 2023-09-23T23:25:20.640 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:50.036463+0000", 2023-09-23T23:25:20.640 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 210.575108683, 2023-09-23T23:25:20.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2684505270000002, 2023-09-23T23:25:20.641 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.641 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.641 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.641 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.641 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.641 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156665 2023-09-23T23:25:20.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.642 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.642 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.642 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036463+0000", 2023-09-23T23:25:20.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.642 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036463+0000", 2023-09-23T23:25:20.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036463+0000", 2023-09-23T23:25:20.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:25:20.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036465+0000", 2023-09-23T23:25:20.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4990000000000001e-06 2023-09-23T23:25:20.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036471+0000", 2023-09-23T23:25:20.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8850000000000004e-06 2023-09-23T23:25:20.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036477+0000", 2023-09-23T23:25:20.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.857e-06 2023-09-23T23:25:20.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038117+0000", 2023-09-23T23:25:20.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016396080000000001 2023-09-23T23:25:20.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:25:20.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038124+0000", 2023-09-23T23:25:20.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8190000000000003e-06 2023-09-23T23:25:20.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.887950+0000", 2023-09-23T23:25:20.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8498266559999998 2023-09-23T23:25:20.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.891672+0000", 2023-09-23T23:25:20.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037213459999999999 2023-09-23T23:25:20.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.909743+0000", 2023-09-23T23:25:20.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018071329000000001 2023-09-23T23:25:20.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.913176+0000", 2023-09-23T23:25:20.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034328200000000001 2023-09-23T23:25:20.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.256612+0000", 2023-09-23T23:25:20.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.343436199 2023-09-23T23:25:20.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.259680+0000", 2023-09-23T23:25:20.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030675870000000001 2023-09-23T23:25:20.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.265203+0000", 2023-09-23T23:25:20.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055235350000000004 2023-09-23T23:25:20.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.293752+0000", 2023-09-23T23:25:20.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028548720999999999 2023-09-23T23:25:20.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.650 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.295411+0000", 2023-09-23T23:25:20.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001659326 2023-09-23T23:25:20.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.650 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.296790+0000", 2023-09-23T23:25:20.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013783339999999999 2023-09-23T23:25:20.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.298340+0000", 2023-09-23T23:25:20.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015504679999999999 2023-09-23T23:25:20.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.304914+0000", 2023-09-23T23:25:20.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00657399 2023-09-23T23:25:20.651 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.651 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.651 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.652 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156677 10.3 10:c754b68c:::benchmark_data_smithi110_366450_object55469:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e301)", 2023-09-23T23:25:20.652 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:50.036483+0000", 2023-09-23T23:25:20.652 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 210.57508883, 2023-09-23T23:25:20.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2794486950000001, 2023-09-23T23:25:20.653 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.653 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.653 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.653 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.653 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.654 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156677 2023-09-23T23:25:20.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.654 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036483+0000", 2023-09-23T23:25:20.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036483+0000", 2023-09-23T23:25:20.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.656 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.656 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036483+0000", 2023-09-23T23:25:20.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:25:20.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.656 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.656 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036485+0000", 2023-09-23T23:25:20.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1509999999999998e-06 2023-09-23T23:25:20.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036491+0000", 2023-09-23T23:25:20.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1029999999999998e-06 2023-09-23T23:25:20.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036497+0000", 2023-09-23T23:25:20.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6699999999999999e-06 2023-09-23T23:25:20.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038146+0000", 2023-09-23T23:25:20.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001649614 2023-09-23T23:25:20.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:25:20.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038153+0000", 2023-09-23T23:25:20.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7120000000000003e-06 2023-09-23T23:25:20.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.892021+0000", 2023-09-23T23:25:20.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8538682949999998 2023-09-23T23:25:20.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.895469+0000", 2023-09-23T23:25:20.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034472249999999999 2023-09-23T23:25:20.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.925797+0000", 2023-09-23T23:25:20.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.030328755999999998 2023-09-23T23:25:20.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.929104+0000", 2023-09-23T23:25:20.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033061599999999998 2023-09-23T23:25:20.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.279422+0000", 2023-09-23T23:25:20.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.35031821000000002 2023-09-23T23:25:20.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.282430+0000", 2023-09-23T23:25:20.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030082590000000001 2023-09-23T23:25:20.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.287816+0000", 2023-09-23T23:25:20.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053855860000000004 2023-09-23T23:25:20.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.305266+0000", 2023-09-23T23:25:20.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017450195000000002 2023-09-23T23:25:20.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.306434+0000", 2023-09-23T23:25:20.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011678719999999999 2023-09-23T23:25:20.662 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.307444+0000", 2023-09-23T23:25:20.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010101159999999999 2023-09-23T23:25:20.662 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.309131+0000", 2023-09-23T23:25:20.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001687207 2023-09-23T23:25:20.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.663 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.315932+0000", 2023-09-23T23:25:20.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068009699999999999 2023-09-23T23:25:20.663 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.663 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.663 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.664 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156689 10.3 10:c3cc188a:::benchmark_data_smithi110_366450_object55481:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e301)", 2023-09-23T23:25:20.664 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:50.036502+0000", 2023-09-23T23:25:20.664 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 210.575070453, 2023-09-23T23:25:20.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.470265768, 2023-09-23T23:25:20.664 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.664 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.664 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.664 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.664 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.664 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156689 2023-09-23T23:25:20.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.664 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036502+0000", 2023-09-23T23:25:20.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036502+0000", 2023-09-23T23:25:20.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036501+0000", 2023-09-23T23:25:20.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:25:20.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036503+0000", 2023-09-23T23:25:20.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1349999999999999e-06 2023-09-23T23:25:20.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036509+0000", 2023-09-23T23:25:20.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9970000000000004e-06 2023-09-23T23:25:20.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.667 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036515+0000", 2023-09-23T23:25:20.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7819999999999999e-06 2023-09-23T23:25:20.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.667 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038348+0000", 2023-09-23T23:25:20.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018323930000000001 2023-09-23T23:25:20.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:25:20.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038356+0000", 2023-09-23T23:25:20.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5450000000000003e-06 2023-09-23T23:25:20.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.930347+0000", 2023-09-23T23:25:20.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.891991215 2023-09-23T23:25:20.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.933759+0000", 2023-09-23T23:25:20.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034111269999999999 2023-09-23T23:25:20.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.669 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.669 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.956094+0000", 2023-09-23T23:25:20.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022335738000000001 2023-09-23T23:25:20.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.669 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.669 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.960338+0000", 2023-09-23T23:25:20.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042442030000000002 2023-09-23T23:25:20.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.670 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.464888+0000", 2023-09-23T23:25:20.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.50454982800000003 2023-09-23T23:25:20.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.670 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.467869+0000", 2023-09-23T23:25:20.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029807459999999998 2023-09-23T23:25:20.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.473841+0000", 2023-09-23T23:25:20.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059724180000000002 2023-09-23T23:25:20.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.671 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.496377+0000", 2023-09-23T23:25:20.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022535192999999999 2023-09-23T23:25:20.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.671 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.497644+0000", 2023-09-23T23:25:20.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00126756 2023-09-23T23:25:20.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.672 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.498689+0000", 2023-09-23T23:25:20.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010452650000000001 2023-09-23T23:25:20.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.672 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.500547+0000", 2023-09-23T23:25:20.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018575410000000001 2023-09-23T23:25:20.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.506767+0000", 2023-09-23T23:25:20.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0062204340000000004 2023-09-23T23:25:20.673 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.673 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.673 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.673 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156699 10.3 10:c7e78881:::benchmark_data_smithi110_366450_object55491:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:25:20.673 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.085948+0000", 2023-09-23T23:25:20.673 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 209.52562400400001, 2023-09-23T23:25:20.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.459542565, 2023-09-23T23:25:20.673 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.674 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.674 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.674 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.674 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.674 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156699 2023-09-23T23:25:20.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.674 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085948+0000", 2023-09-23T23:25:20.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085948+0000", 2023-09-23T23:25:20.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085946+0000", 2023-09-23T23:25:20.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:25:20.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085954+0000", 2023-09-23T23:25:20.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8690000000000004e-06 2023-09-23T23:25:20.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.676 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.676 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085969+0000", 2023-09-23T23:25:20.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4579999999999999e-05 2023-09-23T23:25:20.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.676 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.676 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085979+0000", 2023-09-23T23:25:20.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6730000000000004e-06 2023-09-23T23:25:20.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.676 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.934188+0000", 2023-09-23T23:25:20.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.848209207 2023-09-23T23:25:20.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.938192+0000", 2023-09-23T23:25:20.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040045879999999999 2023-09-23T23:25:20.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.974891+0000", 2023-09-23T23:25:20.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036698716999999999 2023-09-23T23:25:20.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.678 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.978909+0000", 2023-09-23T23:25:20.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040178669999999996 2023-09-23T23:25:20.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.678 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.515734+0000", 2023-09-23T23:25:20.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.53682527300000005 2023-09-23T23:25:20.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.518894+0000", 2023-09-23T23:25:20.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031600869999999998 2023-09-23T23:25:20.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.524144+0000", 2023-09-23T23:25:20.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052494250000000003 2023-09-23T23:25:20.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.535043+0000", 2023-09-23T23:25:20.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010899067 2023-09-23T23:25:20.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.535410+0000", 2023-09-23T23:25:20.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036688299999999998 2023-09-23T23:25:20.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.536594+0000", 2023-09-23T23:25:20.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001184095 2023-09-23T23:25:20.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.538475+0000", 2023-09-23T23:25:20.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018812639999999999 2023-09-23T23:25:20.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.545491+0000", 2023-09-23T23:25:20.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070156009999999998 2023-09-23T23:25:20.681 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.681 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.681 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.682 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156707 10.3 10:ca34f172:::benchmark_data_smithi110_366450_object55499:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:25:20.682 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.101757+0000", 2023-09-23T23:25:20.682 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 209.50981556599999, 2023-09-23T23:25:20.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5204141440000001, 2023-09-23T23:25:20.682 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.682 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.682 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.682 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.682 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.682 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156707 2023-09-23T23:25:20.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.683 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101757+0000", 2023-09-23T23:25:20.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101757+0000", 2023-09-23T23:25:20.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.684 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.684 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101754+0000", 2023-09-23T23:25:20.684 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-09-23T23:25:20.684 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.684 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.684 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101763+0000", 2023-09-23T23:25:20.684 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1020000000000003e-06 2023-09-23T23:25:20.684 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.684 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101778+0000", 2023-09-23T23:25:20.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4268e-05 2023-09-23T23:25:20.685 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.685 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.685 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101787+0000", 2023-09-23T23:25:20.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7580000000000004e-06 2023-09-23T23:25:20.685 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.685 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.685 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.939584+0000", 2023-09-23T23:25:20.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8377970509999999 2023-09-23T23:25:20.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.686 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.686 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.943370+0000", 2023-09-23T23:25:20.686 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037859840000000001 2023-09-23T23:25:20.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.686 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.686 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.991414+0000", 2023-09-23T23:25:20.686 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048043296999999999 2023-09-23T23:25:20.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.994689+0000", 2023-09-23T23:25:20.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032749989999999998 2023-09-23T23:25:20.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.572368+0000", 2023-09-23T23:25:20.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.57767970499999999 2023-09-23T23:25:20.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.575376+0000", 2023-09-23T23:25:20.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030079059999999999 2023-09-23T23:25:20.688 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.688 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.688 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.688 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.580798+0000", 2023-09-23T23:25:20.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054212740000000002 2023-09-23T23:25:20.688 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.688 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.688 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.688 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.612099+0000", 2023-09-23T23:25:20.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031301739000000002 2023-09-23T23:25:20.688 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.689 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.612481+0000", 2023-09-23T23:25:20.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00038158799999999998 2023-09-23T23:25:20.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.689 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.613567+0000", 2023-09-23T23:25:20.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010858490000000001 2023-09-23T23:25:20.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.689 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.615343+0000", 2023-09-23T23:25:20.690 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017767620000000001 2023-09-23T23:25:20.690 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.690 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.690 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.622171+0000", 2023-09-23T23:25:20.690 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068272410000000004 2023-09-23T23:25:20.690 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.690 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.690 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.690 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.691 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPush(10.3 302/301 [PushOp(10:c8345f3f:::benchmark_data_smithi110_366450_object55450:head, version: 272'3485, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:c8345f3f:::benchmark_data_smithi110_366450_object55450:head@272'3485, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:25:20.691 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.136769+0000", 2023-09-23T23:25:20.691 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 209.474803205, 2023-09-23T23:25:20.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7428203010000001, 2023-09-23T23:25:20.691 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.691 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:25:20.691 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.691 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.691 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136769+0000", 2023-09-23T23:25:20.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.692 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.692 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136769+0000", 2023-09-23T23:25:20.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.692 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.692 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136765+0000", 2023-09-23T23:25:20.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999962 2023-09-23T23:25:20.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.692 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.692 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136850+0000", 2023-09-23T23:25:20.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5081999999999996e-05 2023-09-23T23:25:20.693 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.693 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.693 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.693 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136868+0000", 2023-09-23T23:25:20.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8357000000000001e-05 2023-09-23T23:25:20.693 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.693 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.693 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.693 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136884+0000", 2023-09-23T23:25:20.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5732000000000001e-05 2023-09-23T23:25:20.693 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.694 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.694 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.867402+0000", 2023-09-23T23:25:20.694 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7305173149999999 2023-09-23T23:25:20.694 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.694 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.694 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.867705+0000", 2023-09-23T23:25:20.694 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030383999999999999 2023-09-23T23:25:20.694 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.694 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.695 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.879589+0000", 2023-09-23T23:25:20.695 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011883874000000001 2023-09-23T23:25:20.695 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.695 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.695 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.695 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.695 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.695 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156724 10.3 10:ca0db88d:::benchmark_data_smithi110_366450_object55516:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:25:20.695 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.159299+0000", 2023-09-23T23:25:20.695 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 209.452272749, 2023-09-23T23:25:20.695 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.473675901, 2023-09-23T23:25:20.695 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.696 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.696 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.696 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.696 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.696 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156724 2023-09-23T23:25:20.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.696 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.696 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.696 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.696 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159299+0000", 2023-09-23T23:25:20.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.697 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.697 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159299+0000", 2023-09-23T23:25:20.697 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.697 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.697 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.697 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159297+0000", 2023-09-23T23:25:20.697 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:25:20.697 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.697 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.697 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159310+0000", 2023-09-23T23:25:20.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2174000000000001e-05 2023-09-23T23:25:20.698 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.698 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159324+0000", 2023-09-23T23:25:20.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4511999999999999e-05 2023-09-23T23:25:20.698 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.698 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159334+0000", 2023-09-23T23:25:20.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7480000000000005e-06 2023-09-23T23:25:20.698 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.699 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.699 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.996054+0000", 2023-09-23T23:25:20.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8367205470000001 2023-09-23T23:25:20.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.699 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.699 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.999367+0000", 2023-09-23T23:25:20.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033122659999999999 2023-09-23T23:25:20.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.699 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.019886+0000", 2023-09-23T23:25:20.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02051973 2023-09-23T23:25:20.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.023203+0000", 2023-09-23T23:25:20.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003316865 2023-09-23T23:25:20.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.595994+0000", 2023-09-23T23:25:20.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.57279082400000003 2023-09-23T23:25:20.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.598978+0000", 2023-09-23T23:25:20.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002984062 2023-09-23T23:25:20.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.604389+0000", 2023-09-23T23:25:20.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054107959999999998 2023-09-23T23:25:20.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.622507+0000", 2023-09-23T23:25:20.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018117985 2023-09-23T23:25:20.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.623486+0000", 2023-09-23T23:25:20.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0009786860000000001 2023-09-23T23:25:20.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.624625+0000", 2023-09-23T23:25:20.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011391330000000001 2023-09-23T23:25:20.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.626256+0000", 2023-09-23T23:25:20.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016308449999999999 2023-09-23T23:25:20.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.704 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.632975+0000", 2023-09-23T23:25:20.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.006719727 2023-09-23T23:25:20.704 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.704 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.704 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.705 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.705 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156745 10.3 10:c3a25812:::benchmark_data_smithi110_366450_object55537:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:25:20.705 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.191485+0000", 2023-09-23T23:25:20.705 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 209.42008714900001, 2023-09-23T23:25:20.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4810129569999999, 2023-09-23T23:25:20.705 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.706 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.706 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.706 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.706 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.706 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156745 2023-09-23T23:25:20.706 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.707 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191485+0000", 2023-09-23T23:25:20.707 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191485+0000", 2023-09-23T23:25:20.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191483+0000", 2023-09-23T23:25:20.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:25:20.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191491+0000", 2023-09-23T23:25:20.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1079999999999993e-06 2023-09-23T23:25:20.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191512+0000", 2023-09-23T23:25:20.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0267000000000001e-05 2023-09-23T23:25:20.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191521+0000", 2023-09-23T23:25:20.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2480000000000002e-06 2023-09-23T23:25:20.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.999765+0000", 2023-09-23T23:25:20.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8082440449999999 2023-09-23T23:25:20.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.713 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.713 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.003124+0000", 2023-09-23T23:25:20.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033588789999999999 2023-09-23T23:25:20.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.714 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.714 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.036625+0000", 2023-09-23T23:25:20.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.03350144 2023-09-23T23:25:20.714 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.040838+0000", 2023-09-23T23:25:20.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042131260000000002 2023-09-23T23:25:20.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.641997+0000", 2023-09-23T23:25:20.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.60115844900000004 2023-09-23T23:25:20.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.645336+0000", 2023-09-23T23:25:20.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003339482 2023-09-23T23:25:20.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.651259+0000", 2023-09-23T23:25:20.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059230419999999999 2023-09-23T23:25:20.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.661663+0000", 2023-09-23T23:25:20.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010403302 2023-09-23T23:25:20.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.662676+0000", 2023-09-23T23:25:20.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010129189999999999 2023-09-23T23:25:20.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.720 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.720 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.663859+0000", 2023-09-23T23:25:20.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011834149999999999 2023-09-23T23:25:20.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.720 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.665740+0000", 2023-09-23T23:25:20.721 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001880545 2023-09-23T23:25:20.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.672498+0000", 2023-09-23T23:25:20.722 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067584250000000002 2023-09-23T23:25:20.722 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.722 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.722 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.722 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.722 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.723 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156758 10.3 10:c6bd9813:::benchmark_data_smithi110_366450_object55550:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:25:20.723 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.205665+0000", 2023-09-23T23:25:20.723 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 209.40590718799999, 2023-09-23T23:25:20.723 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6487948870000002, 2023-09-23T23:25:20.723 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.723 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.724 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.724 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.724 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.724 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156758 2023-09-23T23:25:20.724 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.724 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205665+0000", 2023-09-23T23:25:20.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205665+0000", 2023-09-23T23:25:20.726 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.726 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.727 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.727 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205664+0000", 2023-09-23T23:25:20.727 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:25:20.727 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.727 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.727 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.728 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205670+0000", 2023-09-23T23:25:20.728 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3929999999999996e-06 2023-09-23T23:25:20.728 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.728 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.728 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.728 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205683+0000", 2023-09-23T23:25:20.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3288999999999999e-05 2023-09-23T23:25:20.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205690+0000", 2023-09-23T23:25:20.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1210000000000001e-06 2023-09-23T23:25:20.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.004368+0000", 2023-09-23T23:25:20.730 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.798677251 2023-09-23T23:25:20.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.007617+0000", 2023-09-23T23:25:20.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032493769999999999 2023-09-23T23:25:20.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.732 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.732 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.056183+0000", 2023-09-23T23:25:20.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048565482 2023-09-23T23:25:20.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.732 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.060370+0000", 2023-09-23T23:25:20.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041874099999999999 2023-09-23T23:25:20.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.811482+0000", 2023-09-23T23:25:20.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.751111588 2023-09-23T23:25:20.734 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.734 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.814815+0000", 2023-09-23T23:25:20.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033331239999999998 2023-09-23T23:25:20.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.819734+0000", 2023-09-23T23:25:20.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0049189960000000001 2023-09-23T23:25:20.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.844633+0000", 2023-09-23T23:25:20.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024899787999999999 2023-09-23T23:25:20.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.737 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.737 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.845534+0000", 2023-09-23T23:25:20.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00090030399999999999 2023-09-23T23:25:20.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.846942+0000", 2023-09-23T23:25:20.738 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001408573 2023-09-23T23:25:20.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.739 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.848519+0000", 2023-09-23T23:25:20.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001576521 2023-09-23T23:25:20.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.739 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.854460+0000", 2023-09-23T23:25:20.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059410060000000004 2023-09-23T23:25:20.740 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.740 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.740 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.740 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.741 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156771 10.3 10:c6f4a9de:::benchmark_data_smithi110_366450_object55563:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:25:20.741 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.219050+0000", 2023-09-23T23:25:20.741 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 209.39252231099999, 2023-09-23T23:25:20.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6446795810000001, 2023-09-23T23:25:20.741 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.741 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.742 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.742 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.742 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.742 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156771 2023-09-23T23:25:20.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.742 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219050+0000", 2023-09-23T23:25:20.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.743 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219050+0000", 2023-09-23T23:25:20.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219048+0000", 2023-09-23T23:25:20.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:25:20.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219054+0000", 2023-09-23T23:25:20.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8950000000000003e-06 2023-09-23T23:25:20.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.746 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.746 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219068+0000", 2023-09-23T23:25:20.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4149e-05 2023-09-23T23:25:20.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219076+0000", 2023-09-23T23:25:20.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8960000000000003e-06 2023-09-23T23:25:20.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.748 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.748 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.070783+0000", 2023-09-23T23:25:20.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8517066980000001 2023-09-23T23:25:20.748 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.749 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.074340+0000", 2023-09-23T23:25:20.749 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003557185 2023-09-23T23:25:20.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.095850+0000", 2023-09-23T23:25:20.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021510344000000001 2023-09-23T23:25:20.750 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.750 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.750 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.099086+0000", 2023-09-23T23:25:20.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032357319999999998 2023-09-23T23:25:20.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.751 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.831720+0000", 2023-09-23T23:25:20.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.732633639 2023-09-23T23:25:20.752 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.752 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.752 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.752 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.834388+0000", 2023-09-23T23:25:20.752 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026684980000000001 2023-09-23T23:25:20.753 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.753 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.753 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.753 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.839278+0000", 2023-09-23T23:25:20.753 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048894430000000003 2023-09-23T23:25:20.753 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.754 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.846624+0000", 2023-09-23T23:25:20.754 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0073457380000000001 2023-09-23T23:25:20.754 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.755 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.755 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.854781+0000", 2023-09-23T23:25:20.755 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081573540000000003 2023-09-23T23:25:20.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.755 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.755 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.756 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.855781+0000", 2023-09-23T23:25:20.756 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00099972499999999992 2023-09-23T23:25:20.756 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.756 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.756 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.756 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.857550+0000", 2023-09-23T23:25:20.757 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001769135 2023-09-23T23:25:20.757 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.757 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.757 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.757 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.863729+0000", 2023-09-23T23:25:20.757 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061796489999999997 2023-09-23T23:25:20.758 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.758 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.758 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.758 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.758 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.758 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156792 10.3 10:c9793c14:::benchmark_data_smithi110_366450_object55584:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:25:20.759 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.244861+0000", 2023-09-23T23:25:20.759 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 209.36671092, 2023-09-23T23:25:20.759 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.66926905, 2023-09-23T23:25:20.759 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.759 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.759 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.760 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.760 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.760 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156792 2023-09-23T23:25:20.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.760 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244861+0000", 2023-09-23T23:25:20.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244861+0000", 2023-09-23T23:25:20.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244859+0000", 2023-09-23T23:25:20.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:25:20.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244868+0000", 2023-09-23T23:25:20.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.551e-06 2023-09-23T23:25:20.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244881+0000", 2023-09-23T23:25:20.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3215e-05 2023-09-23T23:25:20.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244888+0000", 2023-09-23T23:25:20.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.199e-06 2023-09-23T23:25:20.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.074685+0000", 2023-09-23T23:25:20.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.829796778 2023-09-23T23:25:20.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.078783+0000", 2023-09-23T23:25:20.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040977440000000004 2023-09-23T23:25:20.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.112127+0000", 2023-09-23T23:25:20.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.033344602000000001 2023-09-23T23:25:20.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.115665+0000", 2023-09-23T23:25:20.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035371859999999999 2023-09-23T23:25:20.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.871651+0000", 2023-09-23T23:25:20.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.75598617700000004 2023-09-23T23:25:20.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.874503+0000", 2023-09-23T23:25:20.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028525780000000001 2023-09-23T23:25:20.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.766 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.879361+0000", 2023-09-23T23:25:20.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048573160000000004 2023-09-23T23:25:20.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.766 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.905198+0000", 2023-09-23T23:25:20.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.025837373 2023-09-23T23:25:20.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.905568+0000", 2023-09-23T23:25:20.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000370021 2023-09-23T23:25:20.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.906504+0000", 2023-09-23T23:25:20.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00093603000000000002 2023-09-23T23:25:20.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.768 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.768 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.907999+0000", 2023-09-23T23:25:20.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001495213 2023-09-23T23:25:20.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.768 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.768 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.768 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.914130+0000", 2023-09-23T23:25:20.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061308609999999996 2023-09-23T23:25:20.768 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.768 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.768 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.769 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.769 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156798 10.3 10:c087de90:::benchmark_data_smithi110_366450_object55590:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:25:20.769 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.251881+0000", 2023-09-23T23:25:20.769 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 209.359691467, 2023-09-23T23:25:20.769 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6721864360000001, 2023-09-23T23:25:20.769 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.769 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.769 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.769 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.769 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.769 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156798 2023-09-23T23:25:20.769 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.770 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.770 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.770 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251881+0000", 2023-09-23T23:25:20.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.770 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.770 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.770 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251881+0000", 2023-09-23T23:25:20.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.770 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251879+0000", 2023-09-23T23:25:20.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:25:20.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251886+0000", 2023-09-23T23:25:20.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4769999999999997e-06 2023-09-23T23:25:20.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251899+0000", 2023-09-23T23:25:20.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2914999999999999e-05 2023-09-23T23:25:20.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.772 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.772 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251908+0000", 2023-09-23T23:25:20.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8259999999999999e-06 2023-09-23T23:25:20.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.772 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.772 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.079210+0000", 2023-09-23T23:25:20.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8273024840000001 2023-09-23T23:25:20.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.082825+0000", 2023-09-23T23:25:20.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036146450000000001 2023-09-23T23:25:20.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.129346+0000", 2023-09-23T23:25:20.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.046521376000000003 2023-09-23T23:25:20.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.132533+0000", 2023-09-23T23:25:20.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031862040000000002 2023-09-23T23:25:20.774 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.892035+0000", 2023-09-23T23:25:20.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.75950217900000005 2023-09-23T23:25:20.774 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.894859+0000", 2023-09-23T23:25:20.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002824287 2023-09-23T23:25:20.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.899648+0000", 2023-09-23T23:25:20.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047886220000000002 2023-09-23T23:25:20.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.915092+0000", 2023-09-23T23:25:20.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015443738 2023-09-23T23:25:20.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.915395+0000", 2023-09-23T23:25:20.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000303252 2023-09-23T23:25:20.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.916288+0000", 2023-09-23T23:25:20.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00089328499999999996 2023-09-23T23:25:20.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.917880+0000", 2023-09-23T23:25:20.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001592336 2023-09-23T23:25:20.777 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.777 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.777 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.924067+0000", 2023-09-23T23:25:20.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061867060000000001 2023-09-23T23:25:20.777 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.777 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.777 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.777 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.777 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156815 10.3 10:c745291f:::benchmark_data_smithi110_366450_object55607:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:25:20.778 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.282940+0000", 2023-09-23T23:25:20.778 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 209.32863197099999, 2023-09-23T23:25:20.778 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7155916270000002, 2023-09-23T23:25:20.778 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.778 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.778 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.778 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.778 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.778 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156815 2023-09-23T23:25:20.778 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.778 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282940+0000", 2023-09-23T23:25:20.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282940+0000", 2023-09-23T23:25:20.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282938+0000", 2023-09-23T23:25:20.780 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:25:20.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.780 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282946+0000", 2023-09-23T23:25:20.780 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6690000000000006e-06 2023-09-23T23:25:20.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.780 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282960+0000", 2023-09-23T23:25:20.780 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4154e-05 2023-09-23T23:25:20.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282969+0000", 2023-09-23T23:25:20.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4290000000000001e-06 2023-09-23T23:25:20.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.132962+0000", 2023-09-23T23:25:20.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.849993564 2023-09-23T23:25:20.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.782 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.136239+0000", 2023-09-23T23:25:20.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032766829999999999 2023-09-23T23:25:20.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.782 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.153725+0000", 2023-09-23T23:25:20.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017485941000000001 2023-09-23T23:25:20.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.782 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.157066+0000", 2023-09-23T23:25:20.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003341283 2023-09-23T23:25:20.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.940774+0000", 2023-09-23T23:25:20.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.78370831600000002 2023-09-23T23:25:20.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.943618+0000", 2023-09-23T23:25:20.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002843632 2023-09-23T23:25:20.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.948942+0000", 2023-09-23T23:25:20.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053240989999999997 2023-09-23T23:25:20.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.984632+0000", 2023-09-23T23:25:20.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.035689569999999997 2023-09-23T23:25:20.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.990058+0000", 2023-09-23T23:25:20.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054259690000000001 2023-09-23T23:25:20.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.991453+0000", 2023-09-23T23:25:20.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013957710000000001 2023-09-23T23:25:20.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.992873+0000", 2023-09-23T23:25:20.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014191939999999999 2023-09-23T23:25:20.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.998532+0000", 2023-09-23T23:25:20.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056591080000000004 2023-09-23T23:25:20.786 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.786 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.786 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.786 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156870 10.3 10:cbcf9498:::benchmark_data_smithi110_366450_object55662:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:25:20.786 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.374775+0000", 2023-09-23T23:25:20.786 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 209.23679725900001, 2023-09-23T23:25:20.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6969683209999999, 2023-09-23T23:25:20.787 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.787 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.787 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.787 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.787 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.787 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156870 2023-09-23T23:25:20.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.787 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374775+0000", 2023-09-23T23:25:20.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374775+0000", 2023-09-23T23:25:20.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374773+0000", 2023-09-23T23:25:20.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:25:20.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374781+0000", 2023-09-23T23:25:20.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6690000000000006e-06 2023-09-23T23:25:20.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374798+0000", 2023-09-23T23:25:20.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7326e-05 2023-09-23T23:25:20.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374807+0000", 2023-09-23T23:25:20.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8920000000000006e-06 2023-09-23T23:25:20.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.137607+0000", 2023-09-23T23:25:20.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.762799918 2023-09-23T23:25:20.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.140841+0000", 2023-09-23T23:25:20.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032347109999999999 2023-09-23T23:25:20.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.791 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.170460+0000", 2023-09-23T23:25:20.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029618499999999999 2023-09-23T23:25:20.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.791 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.173817+0000", 2023-09-23T23:25:20.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033569440000000002 2023-09-23T23:25:20.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.015338+0000", 2023-09-23T23:25:20.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.841521509 2023-09-23T23:25:20.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.017988+0000", 2023-09-23T23:25:20.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026499359999999999 2023-09-23T23:25:20.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.022695+0000", 2023-09-23T23:25:20.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047064819999999997 2023-09-23T23:25:20.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.042997+0000", 2023-09-23T23:25:20.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020301784999999999 2023-09-23T23:25:20.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.062327+0000", 2023-09-23T23:25:20.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019330791 2023-09-23T23:25:20.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.064213+0000", 2023-09-23T23:25:20.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018860210000000001 2023-09-23T23:25:20.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.065676+0000", 2023-09-23T23:25:20.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001462968 2023-09-23T23:25:20.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.071743+0000", 2023-09-23T23:25:20.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060667660000000003 2023-09-23T23:25:20.795 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.795 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.795 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.795 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156880 10.3 10:ca35dff2:::benchmark_data_smithi110_366450_object55672:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:25:20.795 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.396748+0000", 2023-09-23T23:25:20.795 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 209.21482397, 2023-09-23T23:25:20.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7345815240000002, 2023-09-23T23:25:20.795 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.796 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.796 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.796 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.796 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.796 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156880 2023-09-23T23:25:20.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.796 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396748+0000", 2023-09-23T23:25:20.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396748+0000", 2023-09-23T23:25:20.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396747+0000", 2023-09-23T23:25:20.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:25:20.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396753+0000", 2023-09-23T23:25:20.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3829999999999997e-06 2023-09-23T23:25:20.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396765+0000", 2023-09-23T23:25:20.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1868999999999999e-05 2023-09-23T23:25:20.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396771+0000", 2023-09-23T23:25:20.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2299999999999996e-06 2023-09-23T23:25:20.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.174176+0000", 2023-09-23T23:25:20.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.777405116 2023-09-23T23:25:20.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.177553+0000", 2023-09-23T23:25:20.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033766130000000001 2023-09-23T23:25:20.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.196534+0000", 2023-09-23T23:25:20.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018980770000000001 2023-09-23T23:25:20.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.200001+0000", 2023-09-23T23:25:20.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003467159 2023-09-23T23:25:20.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.084438+0000", 2023-09-23T23:25:20.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.88443721799999997 2023-09-23T23:25:20.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.801 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.801 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.087726+0000", 2023-09-23T23:25:20.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003287805 2023-09-23T23:25:20.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.801 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.801 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.093532+0000", 2023-09-23T23:25:20.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058060129999999996 2023-09-23T23:25:20.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.801 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.802 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.120102+0000", 2023-09-23T23:25:20.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.026570277999999999 2023-09-23T23:25:20.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.802 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.802 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.121067+0000", 2023-09-23T23:25:20.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00096454899999999998 2023-09-23T23:25:20.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.802 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.802 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.122429+0000", 2023-09-23T23:25:20.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001362399 2023-09-23T23:25:20.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.124160+0000", 2023-09-23T23:25:20.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001731291 2023-09-23T23:25:20.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.131330+0000", 2023-09-23T23:25:20.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071693939999999999 2023-09-23T23:25:20.803 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.803 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.803 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.804 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156887 10.3 10:c3285ada:::benchmark_data_smithi110_366450_object55679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:25:20.804 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.419195+0000", 2023-09-23T23:25:20.804 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 209.19237755099999, 2023-09-23T23:25:20.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.750874418, 2023-09-23T23:25:20.804 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.804 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.804 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.804 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.804 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.804 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156887 2023-09-23T23:25:20.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.805 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419195+0000", 2023-09-23T23:25:20.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419195+0000", 2023-09-23T23:25:20.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419193+0000", 2023-09-23T23:25:20.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:25:20.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419199+0000", 2023-09-23T23:25:20.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1539999999999999e-06 2023-09-23T23:25:20.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419211+0000", 2023-09-23T23:25:20.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1651000000000001e-05 2023-09-23T23:25:20.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419219+0000", 2023-09-23T23:25:20.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0220000000000004e-06 2023-09-23T23:25:20.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.178884+0000", 2023-09-23T23:25:20.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7596647829999998 2023-09-23T23:25:20.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.182782+0000", 2023-09-23T23:25:20.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038980529999999998 2023-09-23T23:25:20.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.212229+0000", 2023-09-23T23:25:20.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029447465999999999 2023-09-23T23:25:20.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.215590+0000", 2023-09-23T23:25:20.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033608779999999999 2023-09-23T23:25:20.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.141066+0000", 2023-09-23T23:25:20.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.92547559800000001 2023-09-23T23:25:20.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.144556+0000", 2023-09-23T23:25:20.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034902330000000001 2023-09-23T23:25:20.810 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.810 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.810 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.149839+0000", 2023-09-23T23:25:20.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052831730000000004 2023-09-23T23:25:20.810 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.810 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.810 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.160290+0000", 2023-09-23T23:25:20.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010451251999999999 2023-09-23T23:25:20.810 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.160640+0000", 2023-09-23T23:25:20.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034921699999999997 2023-09-23T23:25:20.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.161546+0000", 2023-09-23T23:25:20.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00090665099999999996 2023-09-23T23:25:20.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.163299+0000", 2023-09-23T23:25:20.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017525590000000001 2023-09-23T23:25:20.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.170069+0000", 2023-09-23T23:25:20.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067702309999999998 2023-09-23T23:25:20.812 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.812 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.812 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.812 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:162491 10.3 10:c27b7a62:::benchmark_data_smithi110_366450_object61283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T23:25:20.812 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:22:12.428302+0000", 2023-09-23T23:25:20.813 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 188.18327027000001, 2023-09-23T23:25:20.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.190000068, 2023-09-23T23:25:20.813 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.813 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.813 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.813 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.813 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.813 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 162491 2023-09-23T23:25:20.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.813 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428302+0000", 2023-09-23T23:25:20.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428302+0000", 2023-09-23T23:25:20.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428301+0000", 2023-09-23T23:25:20.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:25:20.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428306+0000", 2023-09-23T23:25:20.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3000000000000001e-06 2023-09-23T23:25:20.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428315+0000", 2023-09-23T23:25:20.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2450000000000004e-06 2023-09-23T23:25:20.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428321+0000", 2023-09-23T23:25:20.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0780000000000001e-06 2023-09-23T23:25:20.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.484840+0000", 2023-09-23T23:25:20.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.056518688999999997 2023-09-23T23:25:20.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.486346+0000", 2023-09-23T23:25:20.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015063489999999999 2023-09-23T23:25:20.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.928898+0000", 2023-09-23T23:25:20.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.44255191999999999 2023-09-23T23:25:20.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.932431+0000", 2023-09-23T23:25:20.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035331630000000002 2023-09-23T23:25:20.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.575704+0000", 2023-09-23T23:25:20.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.64327284799999995 2023-09-23T23:25:20.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.578243+0000", 2023-09-23T23:25:20.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025391559999999999 2023-09-23T23:25:20.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.582742+0000", 2023-09-23T23:25:20.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044984200000000004 2023-09-23T23:25:20.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.819 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.819 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.610331+0000", 2023-09-23T23:25:20.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.027589460999999999 2023-09-23T23:25:20.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.819 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.819 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.610601+0000", 2023-09-23T23:25:20.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026943799999999999 2023-09-23T23:25:20.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.611382+0000", 2023-09-23T23:25:20.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00078106699999999998 2023-09-23T23:25:20.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.612748+0000", 2023-09-23T23:25:20.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013659690000000001 2023-09-23T23:25:20.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.618302+0000", 2023-09-23T23:25:20.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055542489999999998 2023-09-23T23:25:20.821 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.821 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.821 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.821 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:163210 10.3 10:cceb657e:::benchmark_data_smithi110_366450_object62002:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-09-23T23:25:20.821 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:22:15.221166+0000", 2023-09-23T23:25:20.821 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 185.390405781, 2023-09-23T23:25:20.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.360785763, 2023-09-23T23:25:20.821 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.821 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.821 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.822 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.822 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.822 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 163210 2023-09-23T23:25:20.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.822 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221166+0000", 2023-09-23T23:25:20.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221166+0000", 2023-09-23T23:25:20.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221165+0000", 2023-09-23T23:25:20.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:25:20.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221170+0000", 2023-09-23T23:25:20.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3000000000000003e-06 2023-09-23T23:25:20.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221180+0000", 2023-09-23T23:25:20.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0513000000000001e-05 2023-09-23T23:25:20.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221186+0000", 2023-09-23T23:25:20.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3789999999999997e-06 2023-09-23T23:25:20.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.236027+0000", 2023-09-23T23:25:20.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014841425 2023-09-23T23:25:20.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.237325+0000", 2023-09-23T23:25:20.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012978829999999999 2023-09-23T23:25:20.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.914366+0000", 2023-09-23T23:25:20.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.677041059 2023-09-23T23:25:20.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.917400+0000", 2023-09-23T23:25:20.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003033507 2023-09-23T23:25:20.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.556394+0000", 2023-09-23T23:25:20.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.63899485600000006 2023-09-23T23:25:20.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.558747+0000", 2023-09-23T23:25:20.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023522899999999999 2023-09-23T23:25:20.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.563392+0000", 2023-09-23T23:25:20.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046450199999999997 2023-09-23T23:25:20.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.572010+0000", 2023-09-23T23:25:20.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086180040000000003 2023-09-23T23:25:20.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.572357+0000", 2023-09-23T23:25:20.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034713000000000002 2023-09-23T23:25:20.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.573395+0000", 2023-09-23T23:25:20.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010380890000000001 2023-09-23T23:25:20.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.575115+0000", 2023-09-23T23:25:20.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017195960000000001 2023-09-23T23:25:20.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.581952+0000", 2023-09-23T23:25:20.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068375789999999999 2023-09-23T23:25:20.829 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.829 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.829 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.830 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:168464 10.3 10:c5beec16:::benchmark_data_smithi110_366450_object67256:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T23:25:20.830 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:22:31.505622+0000", 2023-09-23T23:25:20.830 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 169.10595010099999, 2023-09-23T23:25:20.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1552177969999999, 2023-09-23T23:25:20.830 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:20.830 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:25:20.830 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:20.830 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:25:20.830 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:25:20.830 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 168464 2023-09-23T23:25:20.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.831 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:20.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:20.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505622+0000", 2023-09-23T23:25:20.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:20.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505622+0000", 2023-09-23T23:25:20.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:20.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:20.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505621+0000", 2023-09-23T23:25:20.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:25:20.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:20.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505627+0000", 2023-09-23T23:25:20.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8950000000000003e-06 2023-09-23T23:25:20.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:20.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505646+0000", 2023-09-23T23:25:20.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9667e-05 2023-09-23T23:25:20.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:20.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505657+0000", 2023-09-23T23:25:20.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1141000000000001e-05 2023-09-23T23:25:20.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.560164+0000", 2023-09-23T23:25:20.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.054506720000000002 2023-09-23T23:25:20.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:25:20.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.561430+0000", 2023-09-23T23:25:20.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001266267 2023-09-23T23:25:20.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.028996+0000", 2023-09-23T23:25:20.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.46756556300000002 2023-09-23T23:25:20.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:25:20.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.031880+0000", 2023-09-23T23:25:20.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028842189999999999 2023-09-23T23:25:20.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:20.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.517916+0000", 2023-09-23T23:25:20.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.486035885 2023-09-23T23:25:20.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:20.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.519830+0000", 2023-09-23T23:25:20.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019141340000000001 2023-09-23T23:25:20.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:25:20.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.523467+0000", 2023-09-23T23:25:20.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036371060000000002 2023-09-23T23:25:20.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.530246+0000", 2023-09-23T23:25:20.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067788550000000003 2023-09-23T23:25:20.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:20.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.532994+0000", 2023-09-23T23:25:20.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027481290000000002 2023-09-23T23:25:20.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:20.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.654294+0000", 2023-09-23T23:25:20.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.121299324 2023-09-23T23:25:20.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:25:20.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.655550+0000", 2023-09-23T23:25:20.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012558789999999999 2023-09-23T23:25:20.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:20.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:20.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:25:20.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.660840+0000", 2023-09-23T23:25:20.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052902979999999997 2023-09-23T23:25:20.838 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.838 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.838 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.838 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:20.838 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:20.838 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:25:20.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:20.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:20.940 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:25:21.035 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:21.183 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 89 16 104762 104746 73.4987 58.4375 0.0205653 0.0136013 2023-09-23T23:25:21.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[221102]: 11.8 scrub starts 2023-09-23T23:25:21.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[221102]: 11.4 scrub starts 2023-09-23T23:25:21.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[221102]: pgmap v1090: 129 pgs: 1 active+clean+scrubbing, 2 peering, 4 unknown, 122 active+clean; 6.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 27 MiB/s wr, 427 op/s 2023-09-23T23:25:21.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:21.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:21.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[221102]: 11.0 scrub ok 2023-09-23T23:25:21.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[221102]: 11.10 scrub starts 2023-09-23T23:25:21.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:25:21.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:25:21.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:21.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[212939]: 11.8 scrub starts 2023-09-23T23:25:21.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[212939]: 11.4 scrub starts 2023-09-23T23:25:21.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[212939]: pgmap v1090: 129 pgs: 1 active+clean+scrubbing, 2 peering, 4 unknown, 122 active+clean; 6.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 27 MiB/s wr, 427 op/s 2023-09-23T23:25:21.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:21.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:21.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[212939]: 11.0 scrub ok 2023-09-23T23:25:21.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[212939]: 11.10 scrub starts 2023-09-23T23:25:21.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:25:21.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:25:21.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:21 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:21 smithi138 ceph-mon[193606]: 11.8 scrub starts 2023-09-23T23:25:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:21 smithi138 ceph-mon[193606]: 11.4 scrub starts 2023-09-23T23:25:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:21 smithi138 ceph-mon[193606]: pgmap v1090: 129 pgs: 1 active+clean+scrubbing, 2 peering, 4 unknown, 122 active+clean; 6.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 27 MiB/s wr, 427 op/s 2023-09-23T23:25:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:21 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:21 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:21 smithi138 ceph-mon[193606]: 11.0 scrub ok 2023-09-23T23:25:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:21 smithi138 ceph-mon[193606]: 11.10 scrub starts 2023-09-23T23:25:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:21 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:25:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:21 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:25:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:21 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:21.651 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:25:21.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:21.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:25:21.756 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:25:21.814 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:22.183 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total time run: 90.0061 2023-09-23T23:25:22.183 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total writes made: 105877 2023-09-23T23:25:22.183 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Write size: 65536 2023-09-23T23:25:22.183 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object size: 65536 2023-09-23T23:25:22.184 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Bandwidth (MB/sec): 73.5207 2023-09-23T23:25:22.184 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Bandwidth: 32.1923 2023-09-23T23:25:22.184 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max bandwidth (MB/sec): 191.5 2023-09-23T23:25:22.184 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min bandwidth (MB/sec): 13.75 2023-09-23T23:25:22.184 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average IOPS: 1176 2023-09-23T23:25:22.184 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev IOPS: 515.077 2023-09-23T23:25:22.184 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max IOPS: 3064 2023-09-23T23:25:22.184 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min IOPS: 220 2023-09-23T23:25:22.184 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average Latency(s): 0.0135983 2023-09-23T23:25:22.184 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Latency(s): 0.02198 2023-09-23T23:25:22.184 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max latency(s): 1.07916 2023-09-23T23:25:22.184 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min latency(s): 0.00111746 2023-09-23T23:25:22.187 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T23:25:22.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:22 smithi110 ceph-mon[212939]: 11.8 scrub ok 2023-09-23T23:25:22.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:22 smithi110 ceph-mon[212939]: 11.10 scrub ok 2023-09-23T23:25:22.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:22 smithi110 ceph-mon[221102]: 11.8 scrub ok 2023-09-23T23:25:22.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:22 smithi110 ceph-mon[221102]: 11.10 scrub ok 2023-09-23T23:25:22.210 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:25:22.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:22 smithi138 ceph-mon[193606]: 11.8 scrub ok 2023-09-23T23:25:22.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:22 smithi138 ceph-mon[193606]: 11.10 scrub ok 2023-09-23T23:25:23.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:23 smithi110 ceph-mon[212939]: 11.6 scrub starts 2023-09-23T23:25:23.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:23 smithi110 ceph-mon[212939]: pgmap v1091: 129 pgs: 1 active+clean+scrubbing, 2 peering, 2 unknown, 124 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 10 MiB/s wr, 164 op/s; 11 KiB/s, 0 objects/s recovering 2023-09-23T23:25:23.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:23 smithi110 ceph-mon[212939]: 11.9 scrub starts 2023-09-23T23:25:23.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:23 smithi110 ceph-mon[212939]: 11.6 scrub ok 2023-09-23T23:25:23.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:23 smithi110 ceph-mon[221102]: 11.6 scrub starts 2023-09-23T23:25:23.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:23 smithi110 ceph-mon[221102]: pgmap v1091: 129 pgs: 1 active+clean+scrubbing, 2 peering, 2 unknown, 124 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 10 MiB/s wr, 164 op/s; 11 KiB/s, 0 objects/s recovering 2023-09-23T23:25:23.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:23 smithi110 ceph-mon[221102]: 11.9 scrub starts 2023-09-23T23:25:23.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:23 smithi110 ceph-mon[221102]: 11.6 scrub ok 2023-09-23T23:25:23.389 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:25:23.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:23.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:25:23.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:23 smithi138 ceph-mon[193606]: 11.6 scrub starts 2023-09-23T23:25:23.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:23 smithi138 ceph-mon[193606]: pgmap v1091: 129 pgs: 1 active+clean+scrubbing, 2 peering, 2 unknown, 124 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 10 MiB/s wr, 164 op/s; 11 KiB/s, 0 objects/s recovering 2023-09-23T23:25:23.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:23 smithi138 ceph-mon[193606]: 11.9 scrub starts 2023-09-23T23:25:23.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:23 smithi138 ceph-mon[193606]: 11.6 scrub ok 2023-09-23T23:25:23.498 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:23.525 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:23.812 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:25:23.886 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:23.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:23.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:24.002 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:24.004 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:25:24.094 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:24.179 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:25:24.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:24.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:25:24.294 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:25:24.362 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:24.542 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:25:24.566 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:25:24.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:24.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:25:24.660 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:24 smithi110 ceph-mon[212939]: 11.14 scrub starts 2023-09-23T23:25:24.660 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:24 smithi110 ceph-mon[212939]: pgmap v1092: 129 pgs: 1 active+clean+scrubbing, 128 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 769 op/s; 19 KiB/s, 0 objects/s recovering 2023-09-23T23:25:24.660 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:24 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T23:25:24.660 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:24 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T23:25:24.660 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:24 smithi110 ceph-mon[212939]: 11.14 scrub ok 2023-09-23T23:25:24.660 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:24 smithi110 ceph-mon[221102]: 11.14 scrub starts 2023-09-23T23:25:24.661 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:24 smithi110 ceph-mon[221102]: pgmap v1092: 129 pgs: 1 active+clean+scrubbing, 128 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 769 op/s; 19 KiB/s, 0 objects/s recovering 2023-09-23T23:25:24.661 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:24 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T23:25:24.661 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:24 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T23:25:24.661 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:24 smithi110 ceph-mon[221102]: 11.14 scrub ok 2023-09-23T23:25:24.671 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:24.716 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:24 smithi138 ceph-mon[193606]: 11.14 scrub starts 2023-09-23T23:25:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:24 smithi138 ceph-mon[193606]: pgmap v1092: 129 pgs: 1 active+clean+scrubbing, 128 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 769 op/s; 19 KiB/s, 0 objects/s recovering 2023-09-23T23:25:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:24 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T23:25:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:24 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T23:25:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:24 smithi138 ceph-mon[193606]: 11.14 scrub ok 2023-09-23T23:25:25.364 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 2, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:25:25.365 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:25:25.365 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 3, 4, 5, 6, 7, 2, 1] 2023-09-23T23:25:25.365 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd out 6 2023-09-23T23:25:25.504 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[221102]: 11.16 scrub starts 2023-09-23T23:25:25.505 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[221102]: 11.3 scrub starts 2023-09-23T23:25:25.505 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[221102]: 11.4 scrub starts 2023-09-23T23:25:25.505 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[221102]: failed to encode map e367 with expected crc 2023-09-23T23:25:25.505 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T23:25:25.505 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[221102]: osdmap e367: 8 total, 8 up, 8 in 2023-09-23T23:25:25.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[221102]: failed to encode map e367 with expected crc 2023-09-23T23:25:25.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[221102]: failed to encode map e367 with expected crc 2023-09-23T23:25:25.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[221102]: failed to encode map e367 with expected crc 2023-09-23T23:25:25.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[221102]: 11.7 scrub starts 2023-09-23T23:25:25.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[221102]: 11.3 scrub ok 2023-09-23T23:25:25.506 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/60972666' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:25:25.507 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:25:25.507 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[212939]: 11.16 scrub starts 2023-09-23T23:25:25.507 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[212939]: 11.3 scrub starts 2023-09-23T23:25:25.507 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[212939]: 11.4 scrub starts 2023-09-23T23:25:25.508 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[212939]: failed to encode map e367 with expected crc 2023-09-23T23:25:25.508 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T23:25:25.508 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[212939]: osdmap e367: 8 total, 8 up, 8 in 2023-09-23T23:25:25.508 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[212939]: failed to encode map e367 with expected crc 2023-09-23T23:25:25.508 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[212939]: failed to encode map e367 with expected crc 2023-09-23T23:25:25.508 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[212939]: failed to encode map e367 with expected crc 2023-09-23T23:25:25.508 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[212939]: 11.7 scrub starts 2023-09-23T23:25:25.509 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[212939]: 11.3 scrub ok 2023-09-23T23:25:25.509 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/60972666' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:25:25.509 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:25 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:25:25.522 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:25:25.771 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:25.773 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:25 smithi138 ceph-mon[193606]: 11.16 scrub starts 2023-09-23T23:25:25.773 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:25 smithi138 ceph-mon[193606]: 11.3 scrub starts 2023-09-23T23:25:25.773 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:25 smithi138 ceph-mon[193606]: 11.4 scrub starts 2023-09-23T23:25:25.773 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:25 smithi138 ceph-mon[193606]: failed to encode map e367 with expected crc 2023-09-23T23:25:25.773 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:25 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T23:25:25.774 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:25 smithi138 ceph-mon[193606]: osdmap e367: 8 total, 8 up, 8 in 2023-09-23T23:25:25.774 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:25 smithi138 ceph-mon[193606]: failed to encode map e367 with expected crc 2023-09-23T23:25:25.774 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:25 smithi138 ceph-mon[193606]: failed to encode map e367 with expected crc 2023-09-23T23:25:25.774 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:25 smithi138 ceph-mon[193606]: failed to encode map e367 with expected crc 2023-09-23T23:25:25.774 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:25 smithi138 ceph-mon[193606]: 11.7 scrub starts 2023-09-23T23:25:25.774 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:25 smithi138 ceph-mon[193606]: 11.3 scrub ok 2023-09-23T23:25:25.774 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:25 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/60972666' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:25:25.774 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:25 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:25:25.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:25.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:25.876 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:25.901 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:25.942 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:25.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:25.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:26.046 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:25:26.072 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:26.572 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:25:26.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:26.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:25:26.679 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:25:26.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[221102]: 11.7 scrub ok 2023-09-23T23:25:26.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[221102]: 11.19 deep-scrub starts 2023-09-23T23:25:26.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[221102]: 11.4 scrub starts 2023-09-23T23:25:26.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:25:26.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[221102]: failed to encode map e368 with expected crc 2023-09-23T23:25:26.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:25:26.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[221102]: osdmap e368: 8 total, 8 up, 8 in 2023-09-23T23:25:26.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[221102]: failed to encode map e368 with expected crc 2023-09-23T23:25:26.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[221102]: pgmap v1095: 129 pgs: 1 unknown, 1 active+clean+scrubbing, 127 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 19 KiB/s, 0 objects/s recovering 2023-09-23T23:25:26.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[221102]: failed to encode map e368 with expected crc 2023-09-23T23:25:26.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[221102]: failed to encode map e368 with expected crc 2023-09-23T23:25:26.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[212939]: 11.7 scrub ok 2023-09-23T23:25:26.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[212939]: 11.19 deep-scrub starts 2023-09-23T23:25:26.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[212939]: 11.4 scrub starts 2023-09-23T23:25:26.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:25:26.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[212939]: failed to encode map e368 with expected crc 2023-09-23T23:25:26.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:25:26.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[212939]: osdmap e368: 8 total, 8 up, 8 in 2023-09-23T23:25:26.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[212939]: failed to encode map e368 with expected crc 2023-09-23T23:25:26.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[212939]: pgmap v1095: 129 pgs: 1 unknown, 1 active+clean+scrubbing, 127 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 19 KiB/s, 0 objects/s recovering 2023-09-23T23:25:26.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[212939]: failed to encode map e368 with expected crc 2023-09-23T23:25:26.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:26 smithi110 ceph-mon[212939]: failed to encode map e368 with expected crc 2023-09-23T23:25:26.725 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:26.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:26 smithi138 ceph-mon[193606]: 11.7 scrub ok 2023-09-23T23:25:26.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:26 smithi138 ceph-mon[193606]: 11.19 deep-scrub starts 2023-09-23T23:25:26.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:26 smithi138 ceph-mon[193606]: 11.4 scrub starts 2023-09-23T23:25:26.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:26 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:25:26.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:26 smithi138 ceph-mon[193606]: failed to encode map e368 with expected crc 2023-09-23T23:25:26.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:26 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:25:26.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:26 smithi138 ceph-mon[193606]: osdmap e368: 8 total, 8 up, 8 in 2023-09-23T23:25:26.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:26 smithi138 ceph-mon[193606]: failed to encode map e368 with expected crc 2023-09-23T23:25:26.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:26 smithi138 ceph-mon[193606]: pgmap v1095: 129 pgs: 1 unknown, 1 active+clean+scrubbing, 127 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 19 KiB/s, 0 objects/s recovering 2023-09-23T23:25:26.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:26 smithi138 ceph-mon[193606]: failed to encode map e368 with expected crc 2023-09-23T23:25:26.756 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:26 smithi138 ceph-mon[193606]: failed to encode map e368 with expected crc 2023-09-23T23:25:26.782 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:26.831 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:25:26.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:26.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:25:26.870 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:25:26.903 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:25:26.924 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:25:26.936 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:25:26.946 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:25:26.968 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:25:26.968 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:26.989 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:25:27.016 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:25:27.018 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:25:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:25:26] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.43.0" 2023-09-23T23:25:27.018 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:25:27.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:27.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:25:27.044 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:25:27.047 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:25:27.047 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:25:27.047 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:25:27.047 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:25:27.047 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:25:27.047 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:25:27.047 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:25:27.048 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:25:27.120 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:25:27.161 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:27.820 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:25:27.840 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:27 smithi110 ceph-mon[212939]: failed to encode map e369 with expected crc 2023-09-23T23:25:27.840 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:27 smithi110 ceph-mon[212939]: failed to encode map e369 with expected crc 2023-09-23T23:25:27.840 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:27 smithi110 ceph-mon[212939]: osdmap e369: 8 total, 8 up, 8 in 2023-09-23T23:25:27.841 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:27 smithi110 ceph-mon[212939]: failed to encode map e369 with expected crc 2023-09-23T23:25:27.841 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:27 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/655650361' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:25:27.841 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:27 smithi110 ceph-mon[221102]: failed to encode map e369 with expected crc 2023-09-23T23:25:27.841 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:27 smithi110 ceph-mon[221102]: failed to encode map e369 with expected crc 2023-09-23T23:25:27.841 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:27 smithi110 ceph-mon[221102]: osdmap e369: 8 total, 8 up, 8 in 2023-09-23T23:25:27.841 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:27 smithi110 ceph-mon[221102]: failed to encode map e369 with expected crc 2023-09-23T23:25:27.841 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:27 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/655650361' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:25:27.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:27.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:25:27.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:27 smithi138 ceph-mon[193606]: failed to encode map e369 with expected crc 2023-09-23T23:25:27.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:27 smithi138 ceph-mon[193606]: failed to encode map e369 with expected crc 2023-09-23T23:25:27.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:27 smithi138 ceph-mon[193606]: osdmap e369: 8 total, 8 up, 8 in 2023-09-23T23:25:27.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:27 smithi138 ceph-mon[193606]: failed to encode map e369 with expected crc 2023-09-23T23:25:27.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:27 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/655650361' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:25:27.942 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:25:27.982 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:28.026 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:25:28.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:28.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:25:28.130 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:28.221 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:28.426 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:25:28.923 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:28 smithi138 ceph-mon[193606]: failed to encode map e369 with expected crc 2023-09-23T23:25:28.923 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:28 smithi138 ceph-mon[193606]: pgmap v1097: 129 pgs: 1 unknown, 1 active+clean+scrubbing, 127 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 11 KiB/s, 0 objects/s recovering 2023-09-23T23:25:28.923 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:28.923 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:25:28.923 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:28.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:28 smithi110 ceph-mon[212939]: failed to encode map e369 with expected crc 2023-09-23T23:25:28.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:28 smithi110 ceph-mon[212939]: pgmap v1097: 129 pgs: 1 unknown, 1 active+clean+scrubbing, 127 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 11 KiB/s, 0 objects/s recovering 2023-09-23T23:25:28.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:28.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:25:28.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:28.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:28 smithi110 ceph-mon[221102]: failed to encode map e369 with expected crc 2023-09-23T23:25:28.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:28 smithi110 ceph-mon[221102]: pgmap v1097: 129 pgs: 1 unknown, 1 active+clean+scrubbing, 127 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 11 KiB/s, 0 objects/s recovering 2023-09-23T23:25:28.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:28.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:25:28.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:29.132 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:29.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:29.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:29.193 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:25:29.193 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:25:29.238 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:25:29.282 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:29.448 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:25:29.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:29.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:25:29.517 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:29.551 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:25:29.579 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:29.620 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:25:29.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:29.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:25:29.724 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:29.759 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:29.823 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:29.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:29.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:29.927 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:25:29.938 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:29 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T23:25:29.938 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:29 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T23:25:29.938 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:29 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T23:25:29.938 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:29 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T23:25:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:29 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T23:25:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:29 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T23:25:29.959 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:30.063 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:25:30.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:30.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:25:30.127 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T23:25:30.168 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:25:30.213 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:30.267 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:25:30.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:30.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:25:30.376 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:30.407 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:30.467 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:30.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:30.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:30.574 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:30.575 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:25:30.610 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:30.648 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:30 smithi110 conmon[212914]: 2023-09-23T23:25:30.331+0000 7f29aa464700 -1 mon.a@0(leader).osd e370 definitely_dead 0 2023-09-23T23:25:30.733 INFO:teuthology.orchestra.run.smithi110.stderr:marked out osd.6. 2023-09-23T23:25:30.736 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:30.738 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:30 smithi138 ceph-mon[193606]: pgmap v1098: 129 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 127 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 9366/305199 objects misplaced (3.069%); 661 KiB/s, 10 objects/s recovering 2023-09-23T23:25:30.738 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:30 smithi138 ceph-mon[193606]: failed to encode map e370 with expected crc 2023-09-23T23:25:30.738 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:30 smithi138 ceph-mon[193606]: failed to encode map e370 with expected crc 2023-09-23T23:25:30.738 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:30 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T23:25:30.738 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:30 smithi138 ceph-mon[193606]: osdmap e370: 8 total, 8 up, 8 in 2023-09-23T23:25:30.738 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:30 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1661661197' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T23:25:30.738 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:30 smithi138 ceph-mon[193606]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T23:25:30.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:30.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:30.840 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:25:30.872 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:30.916 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:25:30.917 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:30 smithi110 ceph-mon[212939]: pgmap v1098: 129 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 127 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 9366/305199 objects misplaced (3.069%); 661 KiB/s, 10 objects/s recovering 2023-09-23T23:25:30.918 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:30 smithi110 ceph-mon[212939]: failed to encode map e370 with expected crc 2023-09-23T23:25:30.918 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:30 smithi110 ceph-mon[212939]: failed to encode map e370 with expected crc 2023-09-23T23:25:30.918 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:30 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T23:25:30.918 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:30 smithi110 ceph-mon[212939]: osdmap e370: 8 total, 8 up, 8 in 2023-09-23T23:25:30.919 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:30 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1661661197' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T23:25:30.919 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:30 smithi110 ceph-mon[212939]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T23:25:30.919 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:30 smithi110 ceph-mon[221102]: pgmap v1098: 129 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 127 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 9366/305199 objects misplaced (3.069%); 661 KiB/s, 10 objects/s recovering 2023-09-23T23:25:30.919 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:30 smithi110 ceph-mon[221102]: failed to encode map e370 with expected crc 2023-09-23T23:25:30.919 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:30 smithi110 ceph-mon[221102]: failed to encode map e370 with expected crc 2023-09-23T23:25:30.920 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:30 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T23:25:30.920 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:30 smithi110 ceph-mon[221102]: osdmap e370: 8 total, 8 up, 8 in 2023-09-23T23:25:30.920 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:30 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1661661197' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T23:25:30.920 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:30 smithi110 ceph-mon[221102]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T23:25:30.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:30.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:25:31.024 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:25:31.053 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:31.201 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:25:31.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:31.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:25:31.242 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:25:31.305 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:25:31.336 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:31.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:31 smithi110 ceph-mon[212939]: failed to encode map e371 with expected crc 2023-09-23T23:25:31.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:31 smithi110 ceph-mon[212939]: failed to encode map e371 with expected crc 2023-09-23T23:25:31.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:31 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1661661197' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T23:25:31.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:31 smithi110 ceph-mon[212939]: osdmap e371: 8 total, 8 up, 7 in 2023-09-23T23:25:31.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:31 smithi110 ceph-mon[221102]: failed to encode map e371 with expected crc 2023-09-23T23:25:31.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:31 smithi110 ceph-mon[221102]: failed to encode map e371 with expected crc 2023-09-23T23:25:31.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:31 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1661661197' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T23:25:31.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:31 smithi110 ceph-mon[221102]: osdmap e371: 8 total, 8 up, 7 in 2023-09-23T23:25:32.005 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:25:32.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:32.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:25:32.097 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:31 smithi138 ceph-mon[193606]: failed to encode map e371 with expected crc 2023-09-23T23:25:32.098 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:31 smithi138 ceph-mon[193606]: failed to encode map e371 with expected crc 2023-09-23T23:25:32.098 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:31 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1661661197' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T23:25:32.098 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:31 smithi138 ceph-mon[193606]: osdmap e371: 8 total, 8 up, 7 in 2023-09-23T23:25:32.109 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:25:32.137 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:32.187 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:25:32.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:32.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:25:32.293 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:32.315 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:25:32.315 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:25:32.324 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:32.394 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:32.573 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:32.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:32.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:32.677 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:32.718 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:32 smithi138 ceph-mon[193606]: pgmap v1101: 129 pgs: 1 unknown, 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 126 active+clean; 6.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 170 B/s rd, 0 op/s; 9366/303993 objects misplaced (3.081%); 661 KiB/s, 10 objects/s recovering 2023-09-23T23:25:32.718 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:32 smithi138 ceph-mon[193606]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.718 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:32 smithi138 ceph-mon[193606]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.718 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:32 smithi138 ceph-mon[193606]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.718 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:32 smithi138 ceph-mon[193606]: osdmap e372: 8 total, 8 up, 7 in 2023-09-23T23:25:32.718 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:32 smithi138 ceph-mon[193606]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.718 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:32 smithi138 ceph-mon[193606]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.718 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:32 smithi138 ceph-mon[193606]: 11.11 starting backfill to osd.3 from (0'0,0'0] MAX to 369'6665 2023-09-23T23:25:32.762 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:32.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:32 smithi138 ceph-mon[193606]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:32 smithi138 ceph-mon[193606]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:32 smithi138 ceph-mon[193606]: 11.11 starting backfill to osd.5 from (0'0,0'0] MAX to 369'6665 2023-09-23T23:25:32.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[221102]: pgmap v1101: 129 pgs: 1 unknown, 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 126 active+clean; 6.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 170 B/s rd, 0 op/s; 9366/303993 objects misplaced (3.081%); 661 KiB/s, 10 objects/s recovering 2023-09-23T23:25:32.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[221102]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[221102]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[221102]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[221102]: osdmap e372: 8 total, 8 up, 7 in 2023-09-23T23:25:32.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[221102]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[221102]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[221102]: 11.11 starting backfill to osd.3 from (0'0,0'0] MAX to 369'6665 2023-09-23T23:25:32.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[221102]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[221102]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[221102]: 11.11 starting backfill to osd.5 from (0'0,0'0] MAX to 369'6665 2023-09-23T23:25:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[212939]: pgmap v1101: 129 pgs: 1 unknown, 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 126 active+clean; 6.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 170 B/s rd, 0 op/s; 9366/303993 objects misplaced (3.081%); 661 KiB/s, 10 objects/s recovering 2023-09-23T23:25:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[212939]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[212939]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[212939]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[212939]: osdmap e372: 8 total, 8 up, 7 in 2023-09-23T23:25:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[212939]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[212939]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[212939]: 11.11 starting backfill to osd.3 from (0'0,0'0] MAX to 369'6665 2023-09-23T23:25:32.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[212939]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[212939]: failed to encode map e372 with expected crc 2023-09-23T23:25:32.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:32 smithi110 ceph-mon[212939]: 11.11 starting backfill to osd.5 from (0'0,0'0] MAX to 369'6665 2023-09-23T23:25:32.976 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:32.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:32.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:33.042 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T23:25:33.083 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:33.124 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:33.524 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:25:33.579 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:25:33.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: failed to encode map e373 with expected crc 2023-09-23T23:25:33.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: osdmap e373: 8 total, 8 up, 7 in 2023-09-23T23:25:33.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: failed to encode map e373 with expected crc 2023-09-23T23:25:33.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: failed to encode map e373 with expected crc 2023-09-23T23:25:33.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: failed to encode map e373 with expected crc 2023-09-23T23:25:33.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: 11.3 starting backfill to osd.3 from (0'0,0'0] MAX to 370'6698 2023-09-23T23:25:33.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: 11.13 starting backfill to osd.3 from (0'0,0'0] MAX to 370'6700 2023-09-23T23:25:33.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: 11.d starting backfill to osd.1 from (0'0,0'0] MAX to 370'7024 2023-09-23T23:25:33.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: 11.1 starting backfill to osd.1 from (0'0,0'0] MAX to 370'6649 2023-09-23T23:25:33.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 370'6604 2023-09-23T23:25:33.930 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: 11.a starting backfill to osd.2 from (0'0,0'0] MAX to 370'6877 2023-09-23T23:25:33.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 370'6690 2023-09-23T23:25:33.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: 11.5 starting backfill to osd.4 from (0'0,0'0] MAX to 370'6660 2023-09-23T23:25:33.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: 11.8 starting backfill to osd.4 from (0'0,0'0] MAX to 370'6619 2023-09-23T23:25:33.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: 11.9 starting backfill to osd.7 from (0'0,0'0] MAX to 370'6715 2023-09-23T23:25:33.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: 11.b starting backfill to osd.7 from (0'0,0'0] MAX to 370'6976 2023-09-23T23:25:33.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: 11.19 starting backfill to osd.7 from (0'0,0'0] MAX to 370'8320 2023-09-23T23:25:33.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1229233640' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:25:33.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:25:33.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1229233640' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:25:33.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[212939]: osdmap e374: 8 total, 8 up, 7 in 2023-09-23T23:25:33.931 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: failed to encode map e373 with expected crc 2023-09-23T23:25:33.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: osdmap e373: 8 total, 8 up, 7 in 2023-09-23T23:25:33.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: failed to encode map e373 with expected crc 2023-09-23T23:25:33.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: failed to encode map e373 with expected crc 2023-09-23T23:25:33.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: failed to encode map e373 with expected crc 2023-09-23T23:25:33.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: 11.3 starting backfill to osd.3 from (0'0,0'0] MAX to 370'6698 2023-09-23T23:25:33.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: 11.13 starting backfill to osd.3 from (0'0,0'0] MAX to 370'6700 2023-09-23T23:25:33.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: 11.d starting backfill to osd.1 from (0'0,0'0] MAX to 370'7024 2023-09-23T23:25:33.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: 11.1 starting backfill to osd.1 from (0'0,0'0] MAX to 370'6649 2023-09-23T23:25:33.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 370'6604 2023-09-23T23:25:33.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: 11.a starting backfill to osd.2 from (0'0,0'0] MAX to 370'6877 2023-09-23T23:25:33.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 370'6690 2023-09-23T23:25:33.932 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: 11.5 starting backfill to osd.4 from (0'0,0'0] MAX to 370'6660 2023-09-23T23:25:33.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: 11.8 starting backfill to osd.4 from (0'0,0'0] MAX to 370'6619 2023-09-23T23:25:33.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: 11.9 starting backfill to osd.7 from (0'0,0'0] MAX to 370'6715 2023-09-23T23:25:33.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: 11.b starting backfill to osd.7 from (0'0,0'0] MAX to 370'6976 2023-09-23T23:25:33.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: 11.19 starting backfill to osd.7 from (0'0,0'0] MAX to 370'8320 2023-09-23T23:25:33.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1229233640' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:25:33.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:25:33.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1229233640' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:25:33.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:33 smithi110 ceph-mon[221102]: osdmap e374: 8 total, 8 up, 7 in 2023-09-23T23:25:33.978 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: failed to encode map e373 with expected crc 2023-09-23T23:25:33.978 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: osdmap e373: 8 total, 8 up, 7 in 2023-09-23T23:25:33.978 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: failed to encode map e373 with expected crc 2023-09-23T23:25:33.978 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: failed to encode map e373 with expected crc 2023-09-23T23:25:33.978 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: failed to encode map e373 with expected crc 2023-09-23T23:25:33.978 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: 11.3 starting backfill to osd.3 from (0'0,0'0] MAX to 370'6698 2023-09-23T23:25:33.979 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: 11.13 starting backfill to osd.3 from (0'0,0'0] MAX to 370'6700 2023-09-23T23:25:33.979 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: 11.d starting backfill to osd.1 from (0'0,0'0] MAX to 370'7024 2023-09-23T23:25:33.979 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: 11.1 starting backfill to osd.1 from (0'0,0'0] MAX to 370'6649 2023-09-23T23:25:33.979 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 370'6604 2023-09-23T23:25:33.979 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: 11.a starting backfill to osd.2 from (0'0,0'0] MAX to 370'6877 2023-09-23T23:25:33.979 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 370'6690 2023-09-23T23:25:33.979 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: 11.5 starting backfill to osd.4 from (0'0,0'0] MAX to 370'6660 2023-09-23T23:25:33.979 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: 11.8 starting backfill to osd.4 from (0'0,0'0] MAX to 370'6619 2023-09-23T23:25:33.979 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: 11.9 starting backfill to osd.7 from (0'0,0'0] MAX to 370'6715 2023-09-23T23:25:33.979 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: 11.b starting backfill to osd.7 from (0'0,0'0] MAX to 370'6976 2023-09-23T23:25:33.979 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: 11.19 starting backfill to osd.7 from (0'0,0'0] MAX to 370'8320 2023-09-23T23:25:33.979 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1229233640' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:25:33.980 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:25:33.980 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1229233640' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:25:33.980 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:33 smithi138 ceph-mon[193606]: osdmap e374: 8 total, 8 up, 7 in 2023-09-23T23:25:35.018 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:35.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:35.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:35.024 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[212939]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[212939]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[212939]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[212939]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[212939]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[212939]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[212939]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[212939]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[212939]: pgmap v1105: 129 pgs: 1 active+remapped+backfilling, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 1 active+recovering+undersized+remapped, 114 active+clean; 6.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 240 op/s; 3349/302355 objects degraded (1.108%); 31263/302355 objects misplaced (10.340%); 2.3 MiB/s, 2 keys/s, 39 objects/s recovering 2023-09-23T23:25:35.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[212939]: osdmap e375: 8 total, 8 up, 7 in 2023-09-23T23:25:35.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[212939]: Health check failed: Degraded data redundancy: 3349/302355 objects degraded (1.108%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:25:35.025 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[221102]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.026 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[221102]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.026 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[221102]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.026 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[221102]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.026 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[221102]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.026 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[221102]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.026 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[221102]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.026 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[221102]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.026 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[221102]: pgmap v1105: 129 pgs: 1 active+remapped+backfilling, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 1 active+recovering+undersized+remapped, 114 active+clean; 6.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 240 op/s; 3349/302355 objects degraded (1.108%); 31263/302355 objects misplaced (10.340%); 2.3 MiB/s, 2 keys/s, 39 objects/s recovering 2023-09-23T23:25:35.026 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[221102]: osdmap e375: 8 total, 8 up, 7 in 2023-09-23T23:25:35.026 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:34 smithi110 ceph-mon[221102]: Health check failed: Degraded data redundancy: 3349/302355 objects degraded (1.108%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:25:35.125 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:35.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:34 smithi138 ceph-mon[193606]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:34 smithi138 ceph-mon[193606]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:34 smithi138 ceph-mon[193606]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:34 smithi138 ceph-mon[193606]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:34 smithi138 ceph-mon[193606]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:34 smithi138 ceph-mon[193606]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:34 smithi138 ceph-mon[193606]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:34 smithi138 ceph-mon[193606]: failed to encode map e374 with expected crc 2023-09-23T23:25:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:34 smithi138 ceph-mon[193606]: pgmap v1105: 129 pgs: 1 active+remapped+backfilling, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 1 active+recovering+undersized+remapped, 114 active+clean; 6.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 240 op/s; 3349/302355 objects degraded (1.108%); 31263/302355 objects misplaced (10.340%); 2.3 MiB/s, 2 keys/s, 39 objects/s recovering 2023-09-23T23:25:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:34 smithi138 ceph-mon[193606]: osdmap e375: 8 total, 8 up, 7 in 2023-09-23T23:25:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:34 smithi138 ceph-mon[193606]: Health check failed: Degraded data redundancy: 3349/302355 objects degraded (1.108%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:25:35.203 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:35.310 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:35.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:35.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:35.415 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:25:35.448 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:35.518 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:25:35.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:35.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:25:35.634 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:25:35.673 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:35.794 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:25:35.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:35.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:25:35.902 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:25:35.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:35 smithi138 ceph-mon[193606]: failed to encode map e375 with expected crc 2023-09-23T23:25:35.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:35 smithi138 ceph-mon[193606]: failed to encode map e375 with expected crc 2023-09-23T23:25:35.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:35 smithi138 ceph-mon[193606]: failed to encode map e375 with expected crc 2023-09-23T23:25:35.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:35 smithi138 ceph-mon[193606]: failed to encode map e375 with expected crc 2023-09-23T23:25:35.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:35 smithi138 ceph-mon[193606]: failed to encode map e375 with expected crc 2023-09-23T23:25:35.948 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:35 smithi138 ceph-mon[193606]: failed to encode map e375 with expected crc 2023-09-23T23:25:35.950 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:35.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:35 smithi110 ceph-mon[212939]: failed to encode map e375 with expected crc 2023-09-23T23:25:35.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:35 smithi110 ceph-mon[212939]: failed to encode map e375 with expected crc 2023-09-23T23:25:35.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:35 smithi110 ceph-mon[212939]: failed to encode map e375 with expected crc 2023-09-23T23:25:35.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:35 smithi110 ceph-mon[212939]: failed to encode map e375 with expected crc 2023-09-23T23:25:35.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:35 smithi110 ceph-mon[212939]: failed to encode map e375 with expected crc 2023-09-23T23:25:35.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:35 smithi110 ceph-mon[212939]: failed to encode map e375 with expected crc 2023-09-23T23:25:35.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:35 smithi110 ceph-mon[221102]: failed to encode map e375 with expected crc 2023-09-23T23:25:35.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:35 smithi110 ceph-mon[221102]: failed to encode map e375 with expected crc 2023-09-23T23:25:35.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:35 smithi110 ceph-mon[221102]: failed to encode map e375 with expected crc 2023-09-23T23:25:35.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:35 smithi110 ceph-mon[221102]: failed to encode map e375 with expected crc 2023-09-23T23:25:35.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:35 smithi110 ceph-mon[221102]: failed to encode map e375 with expected crc 2023-09-23T23:25:35.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:35 smithi110 ceph-mon[221102]: failed to encode map e375 with expected crc 2023-09-23T23:25:36.026 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:25:36.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:36.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:25:36.065 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:25:36.065 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:25:36.130 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:25:36.160 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:36.208 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:25:36.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:36.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:25:36.312 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:25:36.373 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:36.727 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:25:36.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:36.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:25:36.838 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:25:36.880 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:36.941 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:25:36.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:36.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:25:36.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:25:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:25:36] "GET /metrics HTTP/1.1" 200 36632 "" "Prometheus/2.43.0" 2023-09-23T23:25:36.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:36 smithi110 ceph-mon[212939]: pgmap v1107: 129 pgs: 6 peering, 3 active+remapped+backfilling, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 101 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 9604/299559 objects degraded (3.206%); 45961/299559 objects misplaced (15.343%); 15 MiB/s, 2 keys/s, 237 objects/s recovering 2023-09-23T23:25:36.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:36 smithi110 ceph-mon[221102]: pgmap v1107: 129 pgs: 6 peering, 3 active+remapped+backfilling, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 101 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 9604/299559 objects degraded (3.206%); 45961/299559 objects misplaced (15.343%); 15 MiB/s, 2 keys/s, 237 objects/s recovering 2023-09-23T23:25:37.044 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:25:37.072 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:36 smithi138 ceph-mon[193606]: pgmap v1107: 129 pgs: 6 peering, 3 active+remapped+backfilling, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 101 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 9604/299559 objects degraded (3.206%); 45961/299559 objects misplaced (15.343%); 15 MiB/s, 2 keys/s, 237 objects/s recovering 2023-09-23T23:25:37.080 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:37.153 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:25:37.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:37.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:25:37.257 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:37.298 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:37.301 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 7, 2, 1] out_osds: [6] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:25:37.301 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T23:25:37.354 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:25:37.395 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:25:37.444 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:25:37.499 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:25:37.547 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:25:37.583 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:25:37.607 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:37.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:37.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:37.617 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:25:37.659 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:25:37.673 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:25:37.673 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:25:37.673 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:25:37.673 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:25:37.673 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:25:37.673 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:25:37.673 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:25:37.673 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:25:37.692 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:25:37.711 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:37.817 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:37.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:37 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/861874603' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:25:37.982 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:37.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:37.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:37.998 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:37 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/861874603' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:25:37.998 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:37 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/861874603' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:25:38.097 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:25:38.137 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:38.226 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:25:38.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:38.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:25:38.339 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:25:38.428 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:38.717 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:25:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:38 smithi138 ceph-mon[193606]: pgmap v1108: 129 pgs: 1 active+clean+remapped, 6 peering, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 99 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 300 op/s; 9604/298650 objects degraded (3.216%); 58210/298650 objects misplaced (19.491%); 11 MiB/s, 1 keys/s, 172 objects/s recovering 2023-09-23T23:25:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:38 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:39.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:38 smithi110 ceph-mon[212939]: pgmap v1108: 129 pgs: 1 active+clean+remapped, 6 peering, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 99 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 300 op/s; 9604/298650 objects degraded (3.216%); 58210/298650 objects misplaced (19.491%); 11 MiB/s, 1 keys/s, 172 objects/s recovering 2023-09-23T23:25:39.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:38 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:39.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:38 smithi110 ceph-mon[221102]: pgmap v1108: 129 pgs: 1 active+clean+remapped, 6 peering, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 99 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 300 op/s; 9604/298650 objects degraded (3.216%); 58210/298650 objects misplaced (19.491%); 11 MiB/s, 1 keys/s, 172 objects/s recovering 2023-09-23T23:25:39.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:38 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:39.486 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:25:39.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:39.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:25:39.597 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:25:39.674 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:39.788 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:39.788 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:25:39.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:39.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:25:39.807 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:39 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 9478/292422 objects degraded (3.241%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:25:39.807 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:39 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 9478/292422 objects degraded (3.241%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:25:39.830 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:39.877 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:39.904 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:25:39.936 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:40.006 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:25:40.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:40.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:25:40.112 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:25:40.145 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:39 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 9478/292422 objects degraded (3.241%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:25:40.148 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:40.215 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:25:40.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:40.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:25:40.322 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:25:40.363 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:40.425 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:25:40.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:40.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:25:40.531 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:25:40.571 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:40.635 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:25:40.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:40.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:25:40.742 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:40.808 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:40 smithi110 ceph-mon[212939]: pgmap v1109: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 9478/292422 objects degraded (3.241%); 55004/292422 objects misplaced (18.810%); 30 MiB/s, 1 keys/s, 487 objects/s recovering 2023-09-23T23:25:40.836 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:40.991 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:40.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:40.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:41.097 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:25:41.141 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:40 smithi138 ceph-mon[193606]: pgmap v1109: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 9478/292422 objects degraded (3.241%); 55004/292422 objects misplaced (18.810%); 30 MiB/s, 1 keys/s, 487 objects/s recovering 2023-09-23T23:25:41.144 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:41.184 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:40 smithi110 ceph-mon[221102]: pgmap v1109: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 9478/292422 objects degraded (3.241%); 55004/292422 objects misplaced (18.810%); 30 MiB/s, 1 keys/s, 487 objects/s recovering 2023-09-23T23:25:41.218 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:25:41.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:41.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:25:41.325 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:25:41.366 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:41.441 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:25:41.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:41.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:25:41.547 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:41.597 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:41.640 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:25:41.675 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:41.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:41.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:41.781 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:41.861 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:42.027 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:42.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:42.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:42.152 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:25:42.190 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:42.260 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:25:42.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:42.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:25:42.370 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:25:42.411 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:42.496 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:25:42.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:42.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:25:42.601 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:42.679 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:42.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:42 smithi110 ceph-mon[221102]: pgmap v1110: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 9478/288543 objects degraded (3.285%); 53012/288543 objects misplaced (18.372%); 33 MiB/s, 526 objects/s recovering 2023-09-23T23:25:42.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:42 smithi110 ceph-mon[212939]: pgmap v1110: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 9478/288543 objects degraded (3.285%); 53012/288543 objects misplaced (18.372%); 33 MiB/s, 526 objects/s recovering 2023-09-23T23:25:43.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:42 smithi138 ceph-mon[193606]: pgmap v1110: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 9478/288543 objects degraded (3.285%); 53012/288543 objects misplaced (18.372%); 33 MiB/s, 526 objects/s recovering 2023-09-23T23:25:43.562 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:43.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:43.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:43.666 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:25:43.695 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:43.745 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:25:43.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:43.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:25:43.855 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:43.891 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:43.946 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:43.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:43.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:44.053 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:44.081 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:44.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:44 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:44 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:25:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:44 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:44 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:25:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:44 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:44.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:44 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:25:44.880 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:44.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:44.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:44.986 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:44.998 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:25:45.019 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:45 smithi110 ceph-mon[212939]: pgmap v1111: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 9478/284937 objects degraded (3.326%); 51037/284937 objects misplaced (17.912%); 35 MiB/s, 564 objects/s recovering 2023-09-23T23:25:45.032 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:45.101 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:25:45.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:45.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:25:45.207 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:45.275 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:45 smithi110 ceph-mon[221102]: pgmap v1111: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 9478/284937 objects degraded (3.326%); 51037/284937 objects misplaced (17.912%); 35 MiB/s, 564 objects/s recovering 2023-09-23T23:25:45.314 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:45.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:45 smithi138 ceph-mon[193606]: pgmap v1111: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 9478/284937 objects degraded (3.326%); 51037/284937 objects misplaced (17.912%); 35 MiB/s, 564 objects/s recovering 2023-09-23T23:25:45.773 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:25:45.774 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:25:45.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.774 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:119898 11.18 11:1c2f8e68:::benchmark_data_smithi110_433746_object14018:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T23:25:45.774 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:45.759593+0000", 2023-09-23T23:25:45.774 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.010180206000000001, 2023-09-23T23:25:45.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010213206000000001, 2023-09-23T23:25:45.774 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:45.774 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:25:45.774 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:45.775 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:25:45.775 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:25:45.775 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 119898 2023-09-23T23:25:45.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.775 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:45.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:45.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.759593+0000", 2023-09-23T23:25:45.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:45.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.759593+0000", 2023-09-23T23:25:45.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:45.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.759591+0000", 2023-09-23T23:25:45.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:25:45.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.777 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:45.777 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.759598+0000", 2023-09-23T23:25:45.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5849999999999996e-06 2023-09-23T23:25:45.777 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.777 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:45.777 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.759612+0000", 2023-09-23T23:25:45.778 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4331999999999999e-05 2023-09-23T23:25:45.778 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.778 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:45.778 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.759621+0000", 2023-09-23T23:25:45.778 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7110000000000001e-06 2023-09-23T23:25:45.778 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.778 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:45.778 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.761580+0000", 2023-09-23T23:25:45.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019588000000000001 2023-09-23T23:25:45.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:45.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.761714+0000", 2023-09-23T23:25:45.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013418300000000001 2023-09-23T23:25:45.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,6", 2023-09-23T23:25:45.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.761958+0000", 2023-09-23T23:25:45.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000244398 2023-09-23T23:25:45.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:45.780 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.763578+0000", 2023-09-23T23:25:45.780 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016202230000000001 2023-09-23T23:25:45.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:25:45.780 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.767782+0000", 2023-09-23T23:25:45.780 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042039549999999997 2023-09-23T23:25:45.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:25:45.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.767823+0000", 2023-09-23T23:25:45.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0708999999999999e-05 2023-09-23T23:25:45.781 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.781 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:45.781 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.781 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:119902 11.1 11:8164acea:::benchmark_data_smithi110_433746_object14022:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T23:25:45.781 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:45.763118+0000", 2023-09-23T23:25:45.781 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0066547209999999997, 2023-09-23T23:25:45.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067896199999999997, 2023-09-23T23:25:45.781 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:45.782 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:25:45.782 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:45.782 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:25:45.782 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:25:45.782 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 119902 2023-09-23T23:25:45.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.782 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:45.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:45.782 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.763118+0000", 2023-09-23T23:25:45.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:45.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.763118+0000", 2023-09-23T23:25:45.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:45.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.763116+0000", 2023-09-23T23:25:45.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:25:45.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:45.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.763124+0000", 2023-09-23T23:25:45.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4749999999999996e-06 2023-09-23T23:25:45.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:45.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.763137+0000", 2023-09-23T23:25:45.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3047000000000001e-05 2023-09-23T23:25:45.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:45.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.763145+0000", 2023-09-23T23:25:45.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4309999999999994e-06 2023-09-23T23:25:45.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:45.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.763205+0000", 2023-09-23T23:25:45.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9858999999999997e-05 2023-09-23T23:25:45.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:45.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.763346+0000", 2023-09-23T23:25:45.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014123 2023-09-23T23:25:45.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,6", 2023-09-23T23:25:45.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.763730+0000", 2023-09-23T23:25:45.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00038357000000000002 2023-09-23T23:25:45.786 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.786 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:45.786 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.787 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:119903 11.18 11:1b37b9da:::benchmark_data_smithi110_433746_object14023:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T23:25:45.787 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:45.765437+0000", 2023-09-23T23:25:45.787 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0043357860000000003, 2023-09-23T23:25:45.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045462260000000004, 2023-09-23T23:25:45.787 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:45.787 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:25:45.787 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:45.787 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:25:45.787 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:25:45.787 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 119903 2023-09-23T23:25:45.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.788 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:45.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:45.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765437+0000", 2023-09-23T23:25:45.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:45.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765437+0000", 2023-09-23T23:25:45.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:45.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765436+0000", 2023-09-23T23:25:45.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:25:45.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:45.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765443+0000", 2023-09-23T23:25:45.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9940000000000003e-06 2023-09-23T23:25:45.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:45.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765456+0000", 2023-09-23T23:25:45.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3665e-05 2023-09-23T23:25:45.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:45.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765464+0000", 2023-09-23T23:25:45.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0749999999999998e-06 2023-09-23T23:25:45.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:45.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765652+0000", 2023-09-23T23:25:45.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018767 2023-09-23T23:25:45.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.791 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:45.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765779+0000", 2023-09-23T23:25:45.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000126997 2023-09-23T23:25:45.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.791 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,6", 2023-09-23T23:25:45.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.766097+0000", 2023-09-23T23:25:45.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00031816899999999998 2023-09-23T23:25:45.791 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.792 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:45.792 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.792 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:119904 11.18 11.f806d7d8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T23:25:45.792 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:45.765482+0000", 2023-09-23T23:25:45.792 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0042909869999999996, 2023-09-23T23:25:45.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045805849999999999, 2023-09-23T23:25:45.792 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:45.792 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:25:45.793 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:45.793 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:25:45.793 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:25:45.793 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 119904 2023-09-23T23:25:45.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.793 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:45.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:45.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765482+0000", 2023-09-23T23:25:45.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:45.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765482+0000", 2023-09-23T23:25:45.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:45.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765481+0000", 2023-09-23T23:25:45.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:25:45.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:45.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765486+0000", 2023-09-23T23:25:45.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0559999999999998e-06 2023-09-23T23:25:45.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:45.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765493+0000", 2023-09-23T23:25:45.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6389999999999992e-06 2023-09-23T23:25:45.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:45.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765499+0000", 2023-09-23T23:25:45.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.229e-06 2023-09-23T23:25:45.796 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.796 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:45.796 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.796 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:119910 11.6 11:670afb18:::benchmark_data_smithi110_433746_object14030:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T23:25:45.797 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:45.765505+0000", 2023-09-23T23:25:45.797 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0042677319999999998, 2023-09-23T23:25:45.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046145989999999996, 2023-09-23T23:25:45.797 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:45.797 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:25:45.797 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:25:45.797 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:25:45.797 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:25:45.797 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 119910 2023-09-23T23:25:45.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.797 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:45.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:45.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765505+0000", 2023-09-23T23:25:45.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:45.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765505+0000", 2023-09-23T23:25:45.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:45.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765505+0000", 2023-09-23T23:25:45.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:25:45.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:45.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765508+0000", 2023-09-23T23:25:45.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6740000000000001e-06 2023-09-23T23:25:45.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:45.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765514+0000", 2023-09-23T23:25:45.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7159999999999997e-06 2023-09-23T23:25:45.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:45.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765520+0000", 2023-09-23T23:25:45.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3739999999999997e-06 2023-09-23T23:25:45.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.801 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:45.801 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765609+0000", 2023-09-23T23:25:45.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9509e-05 2023-09-23T23:25:45.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.801 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:45.801 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765733+0000", 2023-09-23T23:25:45.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000124268 2023-09-23T23:25:45.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.802 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:25:45.802 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.766102+0000", 2023-09-23T23:25:45.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036887500000000001 2023-09-23T23:25:45.802 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.802 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:45.802 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.802 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35904.0:119905 11.11 e375/371 11:88ff6321:::benchmark_data_smithi110_433746_object14025:head v 375'6934, mlcod=375'6932)", 2023-09-23T23:25:45.803 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:45.765922+0000", 2023-09-23T23:25:45.803 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0038509569999999999, 2023-09-23T23:25:45.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042732569999999999, 2023-09-23T23:25:45.803 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:45.803 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:25:45.803 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:45.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:45.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765922+0000", 2023-09-23T23:25:45.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:45.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765922+0000", 2023-09-23T23:25:45.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:45.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765922+0000", 2023-09-23T23:25:45.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-09-23T23:25:45.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:45.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765925+0000", 2023-09-23T23:25:45.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.63e-06 2023-09-23T23:25:45.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:45.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765930+0000", 2023-09-23T23:25:45.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3109999999999999e-06 2023-09-23T23:25:45.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:45.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.765933+0000", 2023-09-23T23:25:45.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4709999999999999e-06 2023-09-23T23:25:45.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:45.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768699+0000", 2023-09-23T23:25:45.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002765769 2023-09-23T23:25:45.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:45.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768724+0000", 2023-09-23T23:25:45.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5457e-05 2023-09-23T23:25:45.806 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.806 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:45.806 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.807 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.35904.0:119902 11.1 e375/372)", 2023-09-23T23:25:45.807 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:45.767877+0000", 2023-09-23T23:25:45.807 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0018955809999999999, 2023-09-23T23:25:45.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023882539999999998, 2023-09-23T23:25:45.807 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:45.807 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:25:45.807 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:45.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:45.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.767877+0000", 2023-09-23T23:25:45.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:45.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.767877+0000", 2023-09-23T23:25:45.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:45.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.767877+0000", 2023-09-23T23:25:45.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:25:45.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:45.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.767887+0000", 2023-09-23T23:25:45.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0190000000000001e-05 2023-09-23T23:25:45.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:45.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.767896+0000", 2023-09-23T23:25:45.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6569999999999992e-06 2023-09-23T23:25:45.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.810 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:45.810 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.767904+0000", 2023-09-23T23:25:45.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.311e-06 2023-09-23T23:25:45.810 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.810 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:45.810 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.810 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.810 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35904.0:119908 11.11 e375/371)", 2023-09-23T23:25:45.810 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:45.768404+0000", 2023-09-23T23:25:45.810 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.001369358, 2023-09-23T23:25:45.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001910473, 2023-09-23T23:25:45.811 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:45.811 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:25:45.811 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:45.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:45.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768404+0000", 2023-09-23T23:25:45.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:45.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768404+0000", 2023-09-23T23:25:45.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:45.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768403+0000", 2023-09-23T23:25:45.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:25:45.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:45.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768410+0000", 2023-09-23T23:25:45.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8639999999999998e-06 2023-09-23T23:25:45.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:45.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768421+0000", 2023-09-23T23:25:45.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0655e-05 2023-09-23T23:25:45.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:45.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768427+0000", 2023-09-23T23:25:45.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.443e-06 2023-09-23T23:25:45.814 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.814 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:45.814 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.814 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35904.0:119909 11.17 e375/365 11:ea1d0474:::benchmark_data_smithi110_433746_object14029:head v 375'6928, mlcod=375'6926)", 2023-09-23T23:25:45.815 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:45.768449+0000", 2023-09-23T23:25:45.815 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.001323547, 2023-09-23T23:25:45.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019183329999999999, 2023-09-23T23:25:45.815 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:45.815 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:25:45.815 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:45.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:45.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768449+0000", 2023-09-23T23:25:45.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:45.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768449+0000", 2023-09-23T23:25:45.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:45.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768449+0000", 2023-09-23T23:25:45.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:25:45.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:45.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768457+0000", 2023-09-23T23:25:45.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2269999999999993e-06 2023-09-23T23:25:45.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:45.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768464+0000", 2023-09-23T23:25:45.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5069999999999997e-06 2023-09-23T23:25:45.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:45.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768468+0000", 2023-09-23T23:25:45.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7129999999999996e-06 2023-09-23T23:25:45.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:25:45.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.769353+0000", 2023-09-23T23:25:45.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00088416999999999999 2023-09-23T23:25:45.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:25:45.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.769377+0000", 2023-09-23T23:25:45.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4474999999999999e-05 2023-09-23T23:25:45.818 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.818 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:45.818 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.819 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.35904.0:119898 11.18 e375/372)", 2023-09-23T23:25:45.819 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:45.768613+0000", 2023-09-23T23:25:45.819 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0011602990000000001, 2023-09-23T23:25:45.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018122800000000001, 2023-09-23T23:25:45.819 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:45.819 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:25:45.819 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:45.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.819 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:45.819 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768613+0000", 2023-09-23T23:25:45.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:45.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768613+0000", 2023-09-23T23:25:45.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:45.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768612+0000", 2023-09-23T23:25:45.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:25:45.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:45.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768623+0000", 2023-09-23T23:25:45.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0767999999999999e-05 2023-09-23T23:25:45.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:45.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768631+0000", 2023-09-23T23:25:45.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0800000000000006e-06 2023-09-23T23:25:45.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:45.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.768637+0000", 2023-09-23T23:25:45.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4299999999999997e-06 2023-09-23T23:25:45.822 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.822 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:45.822 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.822 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.35904.0:119903 11.18 e375/372)", 2023-09-23T23:25:45.822 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:45.770256+0000", 2023-09-23T23:25:45.822 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 4294967295.9995174, 2023-09-23T23:25:45.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021569300000000001, 2023-09-23T23:25:45.822 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:45.822 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:25:45.822 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:45.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:45.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.770256+0000", 2023-09-23T23:25:45.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:45.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.770256+0000", 2023-09-23T23:25:45.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:45.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.770254+0000", 2023-09-23T23:25:45.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:25:45.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:45.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.770263+0000", 2023-09-23T23:25:45.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.346e-06 2023-09-23T23:25:45.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:45.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.770276+0000", 2023-09-23T23:25:45.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3071000000000001e-05 2023-09-23T23:25:45.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:45.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.770286+0000", 2023-09-23T23:25:45.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2029999999999998e-06 2023-09-23T23:25:45.825 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.825 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:45.825 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.825 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPushReply(11.15 375/372 [PushReplyOp(11:af00ee65:::benchmark_data_smithi110_433746_object42432:head)])", 2023-09-23T23:25:45.825 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:45.770433+0000", 2023-09-23T23:25:45.825 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 4294967295.9993401, 2023-09-23T23:25:45.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6379999999999996e-05, 2023-09-23T23:25:45.826 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:25:45.826 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:25:45.826 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:25:45.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:25:45.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.770433+0000", 2023-09-23T23:25:45.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:25:45.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.770433+0000", 2023-09-23T23:25:45.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:25:45.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:25:45.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.770432+0000", 2023-09-23T23:25:45.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:25:45.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:25:45.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.770436+0000", 2023-09-23T23:25:45.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6229999999999999e-06 2023-09-23T23:25:45.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:25:45.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.770444+0000", 2023-09-23T23:25:45.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.481e-06 2023-09-23T23:25:45.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:25:45.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:25:45.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:25:45.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:45.770450+0000", 2023-09-23T23:25:45.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.939e-06 2023-09-23T23:25:45.830 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.830 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:25:45.830 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.830 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:25:45.830 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-09-23T23:25:45.830 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 12 2023-09-23T23:25:45.830 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:25:45.854 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:45.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:45.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:45.967 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:46.083 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:46 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 9258/279111 objects degraded (3.317%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:25:46.083 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:46 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3238063432' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:25:46.083 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:46 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:25:46.086 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:46 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 9258/279111 objects degraded (3.317%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:25:46.087 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:46 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3238063432' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:25:46.087 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:46 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:25:46.118 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:46.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:46 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 9258/279111 objects degraded (3.317%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:25:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:46 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3238063432' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:25:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:46 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:25:46.925 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:46.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:46.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:46.934 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:25:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:25:46] "GET /metrics HTTP/1.1" 200 36638 "" "Prometheus/2.43.0" 2023-09-23T23:25:47.032 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:25:47.090 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:25:47.091 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T23:25:47.142 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:47.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:47 smithi110 ceph-mon[212939]: pgmap v1112: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 9258/279111 objects degraded (3.317%); 48551/279111 objects misplaced (17.395%); 41 MiB/s, 648 objects/s recovering 2023-09-23T23:25:47.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:47 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3657794311' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:25:47.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:47 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:25:47.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:47 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:25:47.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:47 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:25:47.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:47 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:25:47.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:47 smithi110 ceph-mon[212939]: osdmap e376: 8 total, 8 up, 7 in 2023-09-23T23:25:47.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:47 smithi110 ceph-mon[212939]: failed to encode map e376 with expected crc 2023-09-23T23:25:47.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:47 smithi110 ceph-mon[221102]: pgmap v1112: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 9258/279111 objects degraded (3.317%); 48551/279111 objects misplaced (17.395%); 41 MiB/s, 648 objects/s recovering 2023-09-23T23:25:47.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:47 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3657794311' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:25:47.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:47 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:25:47.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:47 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:25:47.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:47 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:25:47.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:47 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:25:47.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:47 smithi110 ceph-mon[221102]: osdmap e376: 8 total, 8 up, 7 in 2023-09-23T23:25:47.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:47 smithi110 ceph-mon[221102]: failed to encode map e376 with expected crc 2023-09-23T23:25:47.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:47 smithi138 ceph-mon[193606]: pgmap v1112: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 9258/279111 objects degraded (3.317%); 48551/279111 objects misplaced (17.395%); 41 MiB/s, 648 objects/s recovering 2023-09-23T23:25:47.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:47 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3657794311' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:25:47.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:47 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:25:47.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:47 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:25:47.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:47 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:25:47.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:47 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:25:47.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:47 smithi138 ceph-mon[193606]: osdmap e376: 8 total, 8 up, 7 in 2023-09-23T23:25:47.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:47 smithi138 ceph-mon[193606]: failed to encode map e376 with expected crc 2023-09-23T23:25:48.104 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:25:48.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:48.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:25:48.216 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:25:48.266 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:48.342 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:25:48.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:48.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:25:48.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:48 smithi138 ceph-mon[193606]: failed to encode map e376 with expected crc 2023-09-23T23:25:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:48 smithi138 ceph-mon[193606]: osdmap e377: 8 total, 8 up, 7 in 2023-09-23T23:25:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:48 smithi138 ceph-mon[193606]: failed to encode map e377 with expected crc 2023-09-23T23:25:48.447 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:25:48.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:48 smithi110 ceph-mon[212939]: failed to encode map e376 with expected crc 2023-09-23T23:25:48.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:48 smithi110 ceph-mon[212939]: osdmap e377: 8 total, 8 up, 7 in 2023-09-23T23:25:48.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:48 smithi110 ceph-mon[212939]: failed to encode map e377 with expected crc 2023-09-23T23:25:48.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:48 smithi110 ceph-mon[221102]: failed to encode map e376 with expected crc 2023-09-23T23:25:48.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:48 smithi110 ceph-mon[221102]: osdmap e377: 8 total, 8 up, 7 in 2023-09-23T23:25:48.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:48 smithi110 ceph-mon[221102]: failed to encode map e377 with expected crc 2023-09-23T23:25:48.483 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:49.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:49 smithi138 ceph-mon[193606]: pgmap v1114: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 9258/277551 objects degraded (3.336%); 48115/277551 objects misplaced (17.336%); 41 MiB/s, 660 objects/s recovering 2023-09-23T23:25:49.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:49 smithi138 ceph-mon[193606]: failed to encode map e377 with expected crc 2023-09-23T23:25:49.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:49 smithi138 ceph-mon[193606]: failed to encode map e377 with expected crc 2023-09-23T23:25:49.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:49 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:49.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:49 smithi138 ceph-mon[193606]: failed to encode map e378 with expected crc 2023-09-23T23:25:49.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:49 smithi138 ceph-mon[193606]: osdmap e378: 8 total, 8 up, 7 in 2023-09-23T23:25:49.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:49 smithi138 ceph-mon[193606]: failed to encode map e378 with expected crc 2023-09-23T23:25:49.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:49 smithi138 ceph-mon[193606]: failed to encode map e378 with expected crc 2023-09-23T23:25:49.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:49 smithi110 ceph-mon[212939]: pgmap v1114: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 9258/277551 objects degraded (3.336%); 48115/277551 objects misplaced (17.336%); 41 MiB/s, 660 objects/s recovering 2023-09-23T23:25:49.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:49 smithi110 ceph-mon[212939]: failed to encode map e377 with expected crc 2023-09-23T23:25:49.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:49 smithi110 ceph-mon[212939]: failed to encode map e377 with expected crc 2023-09-23T23:25:49.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:49 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:49.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:49 smithi110 ceph-mon[212939]: failed to encode map e378 with expected crc 2023-09-23T23:25:49.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:49 smithi110 ceph-mon[212939]: osdmap e378: 8 total, 8 up, 7 in 2023-09-23T23:25:49.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:49 smithi110 ceph-mon[212939]: failed to encode map e378 with expected crc 2023-09-23T23:25:49.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:49 smithi110 ceph-mon[212939]: failed to encode map e378 with expected crc 2023-09-23T23:25:49.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:49 smithi110 ceph-mon[221102]: pgmap v1114: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 9258/277551 objects degraded (3.336%); 48115/277551 objects misplaced (17.336%); 41 MiB/s, 660 objects/s recovering 2023-09-23T23:25:49.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:49 smithi110 ceph-mon[221102]: failed to encode map e377 with expected crc 2023-09-23T23:25:49.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:49 smithi110 ceph-mon[221102]: failed to encode map e377 with expected crc 2023-09-23T23:25:49.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:49 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:25:49.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:49 smithi110 ceph-mon[221102]: failed to encode map e378 with expected crc 2023-09-23T23:25:49.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:49 smithi110 ceph-mon[221102]: osdmap e378: 8 total, 8 up, 7 in 2023-09-23T23:25:49.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:49 smithi110 ceph-mon[221102]: failed to encode map e378 with expected crc 2023-09-23T23:25:49.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:49 smithi110 ceph-mon[221102]: failed to encode map e378 with expected crc 2023-09-23T23:25:50.189 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:25:50.397 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:50 smithi110 ceph-mon[221102]: failed to encode map e378 with expected crc 2023-09-23T23:25:50.397 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:50 smithi110 ceph-mon[221102]: failed to encode map e379 with expected crc 2023-09-23T23:25:50.397 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:50 smithi110 ceph-mon[221102]: failed to encode map e379 with expected crc 2023-09-23T23:25:50.397 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:50 smithi110 ceph-mon[221102]: osdmap e379: 8 total, 8 up, 7 in 2023-09-23T23:25:50.398 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:50 smithi110 ceph-mon[212939]: failed to encode map e378 with expected crc 2023-09-23T23:25:50.398 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:50 smithi110 ceph-mon[212939]: failed to encode map e379 with expected crc 2023-09-23T23:25:50.398 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:50 smithi110 ceph-mon[212939]: failed to encode map e379 with expected crc 2023-09-23T23:25:50.398 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:50 smithi110 ceph-mon[212939]: osdmap e379: 8 total, 8 up, 7 in 2023-09-23T23:25:50.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:50 smithi138 ceph-mon[193606]: failed to encode map e378 with expected crc 2023-09-23T23:25:50.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:50 smithi138 ceph-mon[193606]: failed to encode map e379 with expected crc 2023-09-23T23:25:50.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:50 smithi138 ceph-mon[193606]: failed to encode map e379 with expected crc 2023-09-23T23:25:50.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:50 smithi138 ceph-mon[193606]: osdmap e379: 8 total, 8 up, 7 in 2023-09-23T23:25:50.737 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:25:50.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:50.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:25:50.843 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:25:50.873 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:51.272 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:25:51.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:51.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:25:51.372 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:51 smithi110 ceph-mon[212939]: pgmap v1117: 129 pgs: 1 remapped+peering, 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 745 op/s; 9040/271503 objects degraded (3.330%); 45187/271503 objects misplaced (16.643%); 43 MiB/s, 689 objects/s recovering 2023-09-23T23:25:51.372 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:51 smithi110 ceph-mon[212939]: failed to encode map e379 with expected crc 2023-09-23T23:25:51.372 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:51 smithi110 ceph-mon[212939]: osdmap e380: 8 total, 8 up, 7 in 2023-09-23T23:25:51.372 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:51 smithi110 ceph-mon[212939]: failed to encode map e380 with expected crc 2023-09-23T23:25:51.373 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:51 smithi110 ceph-mon[221102]: pgmap v1117: 129 pgs: 1 remapped+peering, 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 745 op/s; 9040/271503 objects degraded (3.330%); 45187/271503 objects misplaced (16.643%); 43 MiB/s, 689 objects/s recovering 2023-09-23T23:25:51.373 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:51 smithi110 ceph-mon[221102]: failed to encode map e379 with expected crc 2023-09-23T23:25:51.373 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:51 smithi110 ceph-mon[221102]: osdmap e380: 8 total, 8 up, 7 in 2023-09-23T23:25:51.373 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:51 smithi110 ceph-mon[221102]: failed to encode map e380 with expected crc 2023-09-23T23:25:51.378 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:25:51.397 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T23:25:51.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:51 smithi138 ceph-mon[193606]: pgmap v1117: 129 pgs: 1 remapped+peering, 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 745 op/s; 9040/271503 objects degraded (3.330%); 45187/271503 objects misplaced (16.643%); 43 MiB/s, 689 objects/s recovering 2023-09-23T23:25:51.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:51 smithi138 ceph-mon[193606]: failed to encode map e379 with expected crc 2023-09-23T23:25:51.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:51 smithi138 ceph-mon[193606]: osdmap e380: 8 total, 8 up, 7 in 2023-09-23T23:25:51.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:51 smithi138 ceph-mon[193606]: failed to encode map e380 with expected crc 2023-09-23T23:25:51.415 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:51.481 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:25:51.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:51.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:25:51.587 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:51.623 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:51.681 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:51.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:51.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:51.765 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:51.788 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:25:51.807 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:25:51.847 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:25:51.849 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:51.882 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:25:51.914 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:25:51.947 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:25:51.978 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:25:52.011 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:25:52.041 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:25:52.041 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:25:52.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:52.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:25:52.047 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:25:52.047 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:25:52.047 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:25:52.047 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:25:52.047 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:25:52.047 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:25:52.047 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:25:52.047 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:25:52.147 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:25:52.192 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:52.271 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:25:52.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:52.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:25:52.377 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:52.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:52 smithi110 ceph-mon[212939]: failed to encode map e380 with expected crc 2023-09-23T23:25:52.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:52 smithi110 ceph-mon[212939]: failed to encode map e380 with expected crc 2023-09-23T23:25:52.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:52 smithi110 ceph-mon[212939]: failed to encode map e380 with expected crc 2023-09-23T23:25:52.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:52 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3009110323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:25:52.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:52 smithi110 ceph-mon[212939]: failed to encode map e381 with expected crc 2023-09-23T23:25:52.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:52 smithi110 ceph-mon[212939]: osdmap e381: 8 total, 8 up, 7 in 2023-09-23T23:25:52.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:52 smithi110 ceph-mon[221102]: failed to encode map e380 with expected crc 2023-09-23T23:25:52.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:52 smithi110 ceph-mon[221102]: failed to encode map e380 with expected crc 2023-09-23T23:25:52.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:52 smithi110 ceph-mon[221102]: failed to encode map e380 with expected crc 2023-09-23T23:25:52.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:52 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3009110323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:25:52.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:52 smithi110 ceph-mon[221102]: failed to encode map e381 with expected crc 2023-09-23T23:25:52.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:52 smithi110 ceph-mon[221102]: osdmap e381: 8 total, 8 up, 7 in 2023-09-23T23:25:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:52 smithi138 ceph-mon[193606]: failed to encode map e380 with expected crc 2023-09-23T23:25:52.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:52 smithi138 ceph-mon[193606]: failed to encode map e380 with expected crc 2023-09-23T23:25:52.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:52 smithi138 ceph-mon[193606]: failed to encode map e380 with expected crc 2023-09-23T23:25:52.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:52 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3009110323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:25:52.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:52 smithi138 ceph-mon[193606]: failed to encode map e381 with expected crc 2023-09-23T23:25:52.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:52 smithi138 ceph-mon[193606]: osdmap e381: 8 total, 8 up, 7 in 2023-09-23T23:25:52.462 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:53.114 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:25:53.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:53 smithi138 ceph-mon[193606]: pgmap v1120: 129 pgs: 1 active+clean+remapped, 1 remapped+peering, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 807 op/s; 9040/267855 objects degraded (3.375%); 43359/267855 objects misplaced (16.187%); 53 MiB/s, 852 objects/s recovering 2023-09-23T23:25:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:53 smithi138 ceph-mon[193606]: failed to encode map e381 with expected crc 2023-09-23T23:25:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:53 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 9040/267855 objects degraded (3.375%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:25:53.461 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:53 smithi110 ceph-mon[212939]: pgmap v1120: 129 pgs: 1 active+clean+remapped, 1 remapped+peering, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 807 op/s; 9040/267855 objects degraded (3.375%); 43359/267855 objects misplaced (16.187%); 53 MiB/s, 852 objects/s recovering 2023-09-23T23:25:53.461 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:53 smithi110 ceph-mon[212939]: failed to encode map e381 with expected crc 2023-09-23T23:25:53.461 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:53 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 9040/267855 objects degraded (3.375%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:25:53.461 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:53 smithi110 ceph-mon[221102]: pgmap v1120: 129 pgs: 1 active+clean+remapped, 1 remapped+peering, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 807 op/s; 9040/267855 objects degraded (3.375%); 43359/267855 objects misplaced (16.187%); 53 MiB/s, 852 objects/s recovering 2023-09-23T23:25:53.461 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:53 smithi110 ceph-mon[221102]: failed to encode map e381 with expected crc 2023-09-23T23:25:53.461 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:53 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 9040/267855 objects degraded (3.375%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:25:53.993 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:54.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:54.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:54.102 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:25:54.139 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:54.216 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:25:54.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:54.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:25:54.322 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:25:54.405 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:54.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:54 smithi110 ceph-mon[212939]: pgmap v1122: 129 pgs: 1 active+clean+remapped, 1 remapped+peering, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 9040/266670 objects degraded (3.390%); 42999/266670 objects misplaced (16.124%); 32 MiB/s, 506 objects/s recovering 2023-09-23T23:25:54.709 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:54 smithi110 ceph-mon[221102]: pgmap v1122: 129 pgs: 1 active+clean+remapped, 1 remapped+peering, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 9040/266670 objects degraded (3.390%); 42999/266670 objects misplaced (16.124%); 32 MiB/s, 506 objects/s recovering 2023-09-23T23:25:54.887 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:54.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:54 smithi138 ceph-mon[193606]: pgmap v1122: 129 pgs: 1 active+clean+remapped, 1 remapped+peering, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 9040/266670 objects degraded (3.390%); 42999/266670 objects misplaced (16.124%); 32 MiB/s, 506 objects/s recovering 2023-09-23T23:25:55.113 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:25:55.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:55.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:25:55.234 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:55.273 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:55.337 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:55.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:55.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:55.442 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:25:55.527 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:55.778 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:25:55.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:55.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:25:55.883 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:55.927 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:55.936 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:55.995 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:55.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:56.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:56.027 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:25:56.027 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:25:56.100 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:56.143 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:56.211 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:56.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:56.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:56.306 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:25:56.317 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:25:56.359 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:56.426 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:25:56.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:56.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:25:56.533 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:56.635 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:56.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:25:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:25:56] "GET /metrics HTTP/1.1" 200 36638 "" "Prometheus/2.43.0" 2023-09-23T23:25:56.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:56 smithi110 ceph-mon[212939]: pgmap v1123: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 8810/259200 objects degraded (3.399%); 38723/259200 objects misplaced (14.939%); 50 MiB/s, 805 objects/s recovering 2023-09-23T23:25:56.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:56 smithi110 ceph-mon[221102]: pgmap v1123: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 8810/259200 objects degraded (3.399%); 38723/259200 objects misplaced (14.939%); 50 MiB/s, 805 objects/s recovering 2023-09-23T23:25:57.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:56 smithi138 ceph-mon[193606]: pgmap v1123: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 8810/259200 objects degraded (3.399%); 38723/259200 objects misplaced (14.939%); 50 MiB/s, 805 objects/s recovering 2023-09-23T23:25:57.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:57 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 8810/257427 objects degraded (3.422%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:25:57.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:57 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 8810/257427 objects degraded (3.422%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:25:58.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:57 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 8810/257427 objects degraded (3.422%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:25:58.396 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:58.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:58.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:58.510 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:25:58.552 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:58.628 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:25:58.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:58.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:25:58.733 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:25:58.779 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:58.852 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:25:58.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:58 smithi138 ceph-mon[193606]: pgmap v1124: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 8810/257427 objects degraded (3.422%); 38268/257427 objects misplaced (14.866%); 40 MiB/s, 637 objects/s recovering 2023-09-23T23:25:58.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:25:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:25:58.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:58.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:25:58.959 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:25:58.968 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:25:59.040 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:58 smithi110 ceph-mon[212939]: pgmap v1124: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 8810/257427 objects degraded (3.422%); 38268/257427 objects misplaced (14.866%); 40 MiB/s, 637 objects/s recovering 2023-09-23T23:25:59.041 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:25:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:25:59.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:58 smithi110 ceph-mon[221102]: pgmap v1124: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 8810/257427 objects degraded (3.422%); 38268/257427 objects misplaced (14.866%); 40 MiB/s, 637 objects/s recovering 2023-09-23T23:25:59.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:25:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:25:59.042 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:25:59.308 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:25:59.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:25:59.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:25:59.387 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:25:59.420 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:25:59.557 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:00.854 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T23:26:01.062 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:26:01.063 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:00 smithi110 ceph-mon[212939]: pgmap v1125: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 8580/250386 objects degraded (3.427%); 35619/250386 objects misplaced (14.226%); 42 MiB/s, 679 objects/s recovering 2023-09-23T23:26:01.064 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:00 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2978726477' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:26:01.064 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:00 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:26:01.064 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:00 smithi110 ceph-mon[221102]: pgmap v1125: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 8580/250386 objects degraded (3.427%); 35619/250386 objects misplaced (14.226%); 42 MiB/s, 679 objects/s recovering 2023-09-23T23:26:01.064 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:00 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2978726477' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:26:01.064 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:00 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:26:01.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:01.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:26:01.166 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:01.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:00 smithi138 ceph-mon[193606]: pgmap v1125: 129 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 8580/250386 objects degraded (3.427%); 35619/250386 objects misplaced (14.226%); 42 MiB/s, 679 objects/s recovering 2023-09-23T23:26:01.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:00 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2978726477' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:26:01.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:00 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:26:01.196 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:01.244 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:01.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:01.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:01.349 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:26:01.379 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:01.795 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:26:01.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:01.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:26:01.899 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:01.927 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:01.974 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:01.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:01.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:02.081 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:26:02.113 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:01 smithi138 ceph-mon[193606]: 11.17 scrub starts 2023-09-23T23:26:02.113 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:01 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:26:02.113 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:01 smithi138 ceph-mon[193606]: osdmap e382: 8 total, 8 up, 7 in 2023-09-23T23:26:02.113 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:01 smithi138 ceph-mon[193606]: failed to encode map e382 with expected crc 2023-09-23T23:26:02.114 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:01 smithi138 ceph-mon[193606]: failed to encode map e382 with expected crc 2023-09-23T23:26:02.116 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:02.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:01 smithi110 ceph-mon[212939]: 11.17 scrub starts 2023-09-23T23:26:02.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:01 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:26:02.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:01 smithi110 ceph-mon[212939]: osdmap e382: 8 total, 8 up, 7 in 2023-09-23T23:26:02.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:01 smithi110 ceph-mon[212939]: failed to encode map e382 with expected crc 2023-09-23T23:26:02.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:01 smithi110 ceph-mon[212939]: failed to encode map e382 with expected crc 2023-09-23T23:26:02.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:01 smithi110 ceph-mon[221102]: 11.17 scrub starts 2023-09-23T23:26:02.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:01 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:26:02.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:01 smithi110 ceph-mon[221102]: osdmap e382: 8 total, 8 up, 7 in 2023-09-23T23:26:02.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:01 smithi110 ceph-mon[221102]: failed to encode map e382 with expected crc 2023-09-23T23:26:02.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:01 smithi110 ceph-mon[221102]: failed to encode map e382 with expected crc 2023-09-23T23:26:02.208 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:26:02.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:02.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:26:02.240 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T23:26:02.313 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:02.340 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:02.388 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:02.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:02.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:02.495 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:26:02.532 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:02.828 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[212939]: pgmap v1127: 129 pgs: 2 active+clean+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 747 op/s; 8580/246357 objects degraded (3.483%); 33849/246357 objects misplaced (13.740%); 44 MiB/s, 707 objects/s recovering 2023-09-23T23:26:02.828 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[212939]: failed to encode map e383 with expected crc 2023-09-23T23:26:02.829 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[212939]: failed to encode map e383 with expected crc 2023-09-23T23:26:02.829 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[212939]: osdmap e383: 8 total, 8 up, 7 in 2023-09-23T23:26:02.829 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[212939]: failed to encode map e383 with expected crc 2023-09-23T23:26:02.829 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[212939]: 11.18 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7210 2023-09-23T23:26:02.829 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[212939]: 11.8 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7226 2023-09-23T23:26:02.829 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[212939]: 11.1 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7277 2023-09-23T23:26:02.829 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[212939]: failed to encode map e383 with expected crc 2023-09-23T23:26:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:02 smithi138 ceph-mon[193606]: pgmap v1127: 129 pgs: 2 active+clean+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 747 op/s; 8580/246357 objects degraded (3.483%); 33849/246357 objects misplaced (13.740%); 44 MiB/s, 707 objects/s recovering 2023-09-23T23:26:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:02 smithi138 ceph-mon[193606]: failed to encode map e383 with expected crc 2023-09-23T23:26:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:02 smithi138 ceph-mon[193606]: failed to encode map e383 with expected crc 2023-09-23T23:26:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:02 smithi138 ceph-mon[193606]: osdmap e383: 8 total, 8 up, 7 in 2023-09-23T23:26:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:02 smithi138 ceph-mon[193606]: failed to encode map e383 with expected crc 2023-09-23T23:26:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:02 smithi138 ceph-mon[193606]: 11.18 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7210 2023-09-23T23:26:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:02 smithi138 ceph-mon[193606]: 11.8 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7226 2023-09-23T23:26:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:02 smithi138 ceph-mon[193606]: 11.1 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7277 2023-09-23T23:26:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:02 smithi138 ceph-mon[193606]: failed to encode map e383 with expected crc 2023-09-23T23:26:03.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[221102]: pgmap v1127: 129 pgs: 2 active+clean+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 747 op/s; 8580/246357 objects degraded (3.483%); 33849/246357 objects misplaced (13.740%); 44 MiB/s, 707 objects/s recovering 2023-09-23T23:26:03.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[221102]: failed to encode map e383 with expected crc 2023-09-23T23:26:03.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[221102]: failed to encode map e383 with expected crc 2023-09-23T23:26:03.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[221102]: osdmap e383: 8 total, 8 up, 7 in 2023-09-23T23:26:03.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[221102]: failed to encode map e383 with expected crc 2023-09-23T23:26:03.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[221102]: 11.18 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7210 2023-09-23T23:26:03.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[221102]: 11.8 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7226 2023-09-23T23:26:03.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[221102]: 11.1 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7277 2023-09-23T23:26:03.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:02 smithi110 ceph-mon[221102]: failed to encode map e383 with expected crc 2023-09-23T23:26:03.510 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:26:03.525 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:26:03.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:03.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:26:03.632 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:26:03.664 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:03.730 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:26:03.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:03.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:26:03.827 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[212939]: 11.17 scrub starts 2023-09-23T23:26:03.834 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[221102]: 11.17 scrub starts 2023-09-23T23:26:03.835 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[221102]: failed to encode map e384 with expected crc 2023-09-23T23:26:03.835 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[221102]: failed to encode map e384 with expected crc 2023-09-23T23:26:03.835 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[221102]: osdmap e384: 8 total, 8 up, 7 in 2023-09-23T23:26:03.835 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[221102]: 11.2 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7408 2023-09-23T23:26:03.835 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[221102]: failed to encode map e384 with expected crc 2023-09-23T23:26:03.835 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[221102]: 11.12 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7340 2023-09-23T23:26:03.835 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[221102]: failed to encode map e384 with expected crc 2023-09-23T23:26:03.835 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[221102]: 11.5 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7233 2023-09-23T23:26:03.835 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[221102]: 11.15 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7292 2023-09-23T23:26:03.835 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[221102]: failed to encode map e384 with expected crc 2023-09-23T23:26:03.835 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 8580/246357 objects degraded (3.483%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:03.836 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[221102]: osdmap e385: 8 total, 8 up, 7 in 2023-09-23T23:26:03.836 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[221102]: failed to encode map e385 with expected crc 2023-09-23T23:26:03.836 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[221102]: failed to encode map e385 with expected crc 2023-09-23T23:26:03.836 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3536347722' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:26:03.836 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:26:03.839 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:26:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:03 smithi138 ceph-mon[193606]: 11.17 scrub starts 2023-09-23T23:26:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:03 smithi138 ceph-mon[193606]: failed to encode map e384 with expected crc 2023-09-23T23:26:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:03 smithi138 ceph-mon[193606]: failed to encode map e384 with expected crc 2023-09-23T23:26:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:03 smithi138 ceph-mon[193606]: osdmap e384: 8 total, 8 up, 7 in 2023-09-23T23:26:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:03 smithi138 ceph-mon[193606]: 11.2 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7408 2023-09-23T23:26:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:03 smithi138 ceph-mon[193606]: failed to encode map e384 with expected crc 2023-09-23T23:26:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:03 smithi138 ceph-mon[193606]: 11.12 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7340 2023-09-23T23:26:03.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:03 smithi138 ceph-mon[193606]: failed to encode map e384 with expected crc 2023-09-23T23:26:03.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:03 smithi138 ceph-mon[193606]: 11.5 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7233 2023-09-23T23:26:03.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:03 smithi138 ceph-mon[193606]: 11.15 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7292 2023-09-23T23:26:03.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:03 smithi138 ceph-mon[193606]: failed to encode map e384 with expected crc 2023-09-23T23:26:03.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:03 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 8580/246357 objects degraded (3.483%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:03.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:03 smithi138 ceph-mon[193606]: osdmap e385: 8 total, 8 up, 7 in 2023-09-23T23:26:03.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:03 smithi138 ceph-mon[193606]: failed to encode map e385 with expected crc 2023-09-23T23:26:03.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:03 smithi138 ceph-mon[193606]: failed to encode map e385 with expected crc 2023-09-23T23:26:03.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:03 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3536347722' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:26:03.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:03 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:26:03.961 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:04.091 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[212939]: failed to encode map e384 with expected crc 2023-09-23T23:26:04.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[212939]: failed to encode map e384 with expected crc 2023-09-23T23:26:04.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[212939]: osdmap e384: 8 total, 8 up, 7 in 2023-09-23T23:26:04.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[212939]: 11.2 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7408 2023-09-23T23:26:04.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[212939]: failed to encode map e384 with expected crc 2023-09-23T23:26:04.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[212939]: 11.12 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7340 2023-09-23T23:26:04.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[212939]: failed to encode map e384 with expected crc 2023-09-23T23:26:04.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[212939]: 11.5 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7233 2023-09-23T23:26:04.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[212939]: 11.15 starting backfill to osd.3 from (0'0,0'0] MAX to 381'7292 2023-09-23T23:26:04.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[212939]: failed to encode map e384 with expected crc 2023-09-23T23:26:04.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 8580/246357 objects degraded (3.483%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:04.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[212939]: osdmap e385: 8 total, 8 up, 7 in 2023-09-23T23:26:04.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[212939]: failed to encode map e385 with expected crc 2023-09-23T23:26:04.093 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[212939]: failed to encode map e385 with expected crc 2023-09-23T23:26:04.093 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3536347722' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:26:04.093 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:03 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:26:04.148 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:26:04.206 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:26:04.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:04.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:26:04.314 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:04.346 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:04.405 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:04.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:04.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:04.429 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:26:04.473 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:26:04.509 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:26:04.512 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:26:04.547 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:04.568 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:26:04.570 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:26:04.608 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:26:04.610 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:26:04.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:04.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:26:04.651 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:26:04.684 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:26:04.714 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:26:04.716 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:26:04.725 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:26:04.725 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:26:04.725 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:26:04.725 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:26:04.726 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:26:04.726 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:26:04.726 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:26:04.726 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:26:04.756 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:04.817 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:26:04.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:04.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:26:04.929 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:26:04.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:04 smithi138 ceph-mon[193606]: failed to encode map e385 with expected crc 2023-09-23T23:26:04.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:04 smithi138 ceph-mon[193606]: failed to encode map e385 with expected crc 2023-09-23T23:26:04.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:04 smithi138 ceph-mon[193606]: failed to encode map e385 with expected crc 2023-09-23T23:26:04.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:04 smithi138 ceph-mon[193606]: 11.17 scrub starts 2023-09-23T23:26:04.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:04 smithi138 ceph-mon[193606]: pgmap v1131: 129 pgs: 2 active+clean+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 8580/245565 objects degraded (3.494%); 33281/245565 objects misplaced (13.553%); 24 MiB/s, 379 objects/s recovering 2023-09-23T23:26:04.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:04 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1762527969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:26:04.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:04 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:26:04.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:04 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:26:04.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:04 smithi138 ceph-mon[193606]: osdmap e386: 8 total, 8 up, 7 in 2023-09-23T23:26:04.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:04 smithi138 ceph-mon[193606]: failed to encode map e386 with expected crc 2023-09-23T23:26:05.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[212939]: failed to encode map e385 with expected crc 2023-09-23T23:26:05.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[212939]: failed to encode map e385 with expected crc 2023-09-23T23:26:05.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[212939]: failed to encode map e385 with expected crc 2023-09-23T23:26:05.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[212939]: 11.17 scrub starts 2023-09-23T23:26:05.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[212939]: pgmap v1131: 129 pgs: 2 active+clean+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 8580/245565 objects degraded (3.494%); 33281/245565 objects misplaced (13.553%); 24 MiB/s, 379 objects/s recovering 2023-09-23T23:26:05.026 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1762527969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:26:05.026 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:26:05.026 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:26:05.026 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[212939]: osdmap e386: 8 total, 8 up, 7 in 2023-09-23T23:26:05.026 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[212939]: failed to encode map e386 with expected crc 2023-09-23T23:26:05.026 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:05.027 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[221102]: failed to encode map e385 with expected crc 2023-09-23T23:26:05.027 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[221102]: failed to encode map e385 with expected crc 2023-09-23T23:26:05.027 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[221102]: failed to encode map e385 with expected crc 2023-09-23T23:26:05.027 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[221102]: 11.17 scrub starts 2023-09-23T23:26:05.027 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[221102]: pgmap v1131: 129 pgs: 2 active+clean+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 8580/245565 objects degraded (3.494%); 33281/245565 objects misplaced (13.553%); 24 MiB/s, 379 objects/s recovering 2023-09-23T23:26:05.027 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1762527969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:26:05.027 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:26:05.027 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:26:05.027 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[221102]: osdmap e386: 8 total, 8 up, 7 in 2023-09-23T23:26:05.028 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:04 smithi110 ceph-mon[221102]: failed to encode map e386 with expected crc 2023-09-23T23:26:06.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:05 smithi138 ceph-mon[193606]: failed to encode map e386 with expected crc 2023-09-23T23:26:06.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:05 smithi138 ceph-mon[193606]: failed to encode map e386 with expected crc 2023-09-23T23:26:06.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:05 smithi138 ceph-mon[193606]: failed to encode map e386 with expected crc 2023-09-23T23:26:06.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:05 smithi110 ceph-mon[212939]: failed to encode map e386 with expected crc 2023-09-23T23:26:06.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:05 smithi110 ceph-mon[212939]: failed to encode map e386 with expected crc 2023-09-23T23:26:06.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:05 smithi110 ceph-mon[212939]: failed to encode map e386 with expected crc 2023-09-23T23:26:06.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:05 smithi110 ceph-mon[221102]: failed to encode map e386 with expected crc 2023-09-23T23:26:06.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:05 smithi110 ceph-mon[221102]: failed to encode map e386 with expected crc 2023-09-23T23:26:06.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:05 smithi110 ceph-mon[221102]: failed to encode map e386 with expected crc 2023-09-23T23:26:06.689 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:26:06.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:06.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:26:06.802 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:26:06.882 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:06.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:06 smithi110 ceph-mon[212939]: 11.17 scrub starts 2023-09-23T23:26:06.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:06 smithi110 ceph-mon[212939]: pgmap v1133: 129 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 5.0 KiB/s rd, 14 KiB/s wr, 442 op/s; 8456/241089 objects degraded (3.507%); 36561/241089 objects misplaced (15.165%); 27 MiB/s, 430 objects/s recovering 2023-09-23T23:26:06.954 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:26:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:26:06] "GET /metrics HTTP/1.1" 200 36619 "" "Prometheus/2.43.0" 2023-09-23T23:26:06.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:06 smithi110 ceph-mon[221102]: 11.17 scrub starts 2023-09-23T23:26:06.961 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:06 smithi110 ceph-mon[221102]: pgmap v1133: 129 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 5.0 KiB/s rd, 14 KiB/s wr, 442 op/s; 8456/241089 objects degraded (3.507%); 36561/241089 objects misplaced (15.165%); 27 MiB/s, 430 objects/s recovering 2023-09-23T23:26:07.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:06 smithi138 ceph-mon[193606]: 11.17 scrub starts 2023-09-23T23:26:07.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:06 smithi138 ceph-mon[193606]: pgmap v1133: 129 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 5.0 KiB/s rd, 14 KiB/s wr, 442 op/s; 8456/241089 objects degraded (3.507%); 36561/241089 objects misplaced (15.165%); 27 MiB/s, 430 objects/s recovering 2023-09-23T23:26:07.727 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:26:07.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:07.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:26:07.840 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:26:07.869 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:08.730 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:26:08.844 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:26:08.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:08.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:26:08.957 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:26:09.064 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:08 smithi110 ceph-mon[221102]: pgmap v1134: 129 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.4 KiB/s rd, 9.4 KiB/s wr, 363 op/s; 8456/240048 objects degraded (3.523%); 40087/240048 objects misplaced (16.700%); 25 MiB/s, 404 objects/s recovering 2023-09-23T23:26:09.064 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:08 smithi110 ceph-mon[221102]: 1.f scrub starts 2023-09-23T23:26:09.064 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:08 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 8456/240048 objects degraded (3.523%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:09.070 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:08 smithi110 ceph-mon[212939]: pgmap v1134: 129 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.4 KiB/s rd, 9.4 KiB/s wr, 363 op/s; 8456/240048 objects degraded (3.523%); 40087/240048 objects misplaced (16.700%); 25 MiB/s, 404 objects/s recovering 2023-09-23T23:26:09.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:08 smithi110 ceph-mon[212939]: 1.f scrub starts 2023-09-23T23:26:09.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:08 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 8456/240048 objects degraded (3.523%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:09.071 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:09.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:08 smithi138 ceph-mon[193606]: pgmap v1134: 129 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.4 KiB/s rd, 9.4 KiB/s wr, 363 op/s; 8456/240048 objects degraded (3.523%); 40087/240048 objects misplaced (16.700%); 25 MiB/s, 404 objects/s recovering 2023-09-23T23:26:09.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:08 smithi138 ceph-mon[193606]: 1.f scrub starts 2023-09-23T23:26:09.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:08 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 8456/240048 objects degraded (3.523%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:09.282 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:26:09.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:09.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:26:09.330 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:26:09.330 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:26:09.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:26:09.330 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:26:09.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.330 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:92235 10.9 10:9dad52bb:::benchmark_data_smithi110_366450_object92234:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:26:09.330 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:20:25.861770+0000", 2023-09-23T23:26:09.331 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 343.45697053599997, 2023-09-23T23:26:09.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99164348099999999, 2023-09-23T23:26:09.331 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.331 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:09.331 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.331 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:26:09.331 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:26:09.331 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 92235 2023-09-23T23:26:09.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.331 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.861770+0000", 2023-09-23T23:26:09.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.861770+0000", 2023-09-23T23:26:09.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.861770+0000", 2023-09-23T23:26:09.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:26:09.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.861790+0000", 2023-09-23T23:26:09.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.056e-05 2023-09-23T23:26:09.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.861797+0000", 2023-09-23T23:26:09.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7039999999999997e-06 2023-09-23T23:26:09.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.335 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.335 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.861802+0000", 2023-09-23T23:26:09.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8149999999999996e-06 2023-09-23T23:26:09.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.335 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.335 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.861905+0000", 2023-09-23T23:26:09.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102859 2023-09-23T23:26:09.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.861913+0000", 2023-09-23T23:26:09.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6419999999999996e-06 2023-09-23T23:26:09.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.849646+0000", 2023-09-23T23:26:09.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98773304799999995 2023-09-23T23:26:09.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:09.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.849824+0000", 2023-09-23T23:26:09.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017723899999999999 2023-09-23T23:26:09.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:26:09.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.850014+0000", 2023-09-23T23:26:09.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019071499999999999 2023-09-23T23:26:09.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853198+0000", 2023-09-23T23:26:09.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031833450000000002 2023-09-23T23:26:09.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853224+0000", 2023-09-23T23:26:09.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6503e-05 2023-09-23T23:26:09.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:09.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853231+0000", 2023-09-23T23:26:09.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9870000000000002e-06 2023-09-23T23:26:09.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853269+0000", 2023-09-23T23:26:09.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7620999999999998e-05 2023-09-23T23:26:09.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:09.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853308+0000", 2023-09-23T23:26:09.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9610999999999997e-05 2023-09-23T23:26:09.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853414+0000", 2023-09-23T23:26:09.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105293 2023-09-23T23:26:09.340 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.340 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.340 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.340 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:92276 10.9 10:967c7a97:::benchmark_data_smithi110_366450_object92275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:26:09.340 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:20:25.911070+0000", 2023-09-23T23:26:09.340 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 343.40767072900002, 2023-09-23T23:26:09.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94893306799999999, 2023-09-23T23:26:09.340 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.340 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:09.340 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.340 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:26:09.341 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:26:09.341 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 92276 2023-09-23T23:26:09.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.341 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911070+0000", 2023-09-23T23:26:09.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911070+0000", 2023-09-23T23:26:09.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911068+0000", 2023-09-23T23:26:09.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:09.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911093+0000", 2023-09-23T23:26:09.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4618999999999999e-05 2023-09-23T23:26:09.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911106+0000", 2023-09-23T23:26:09.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.271e-05 2023-09-23T23:26:09.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911112+0000", 2023-09-23T23:26:09.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8019999999999997e-06 2023-09-23T23:26:09.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911148+0000", 2023-09-23T23:26:09.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6596e-05 2023-09-23T23:26:09.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911158+0000", 2023-09-23T23:26:09.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6989999999999998e-06 2023-09-23T23:26:09.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853463+0000", 2023-09-23T23:26:09.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94230513900000001 2023-09-23T23:26:09.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:09.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853641+0000", 2023-09-23T23:26:09.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000178438 2023-09-23T23:26:09.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:26:09.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.853829+0000", 2023-09-23T23:26:09.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018803099999999999 2023-09-23T23:26:09.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.859666+0000", 2023-09-23T23:26:09.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058363540000000002 2023-09-23T23:26:09.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.859695+0000", 2023-09-23T23:26:09.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8878e-05 2023-09-23T23:26:09.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:09.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.859743+0000", 2023-09-23T23:26:09.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8288000000000003e-05 2023-09-23T23:26:09.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.859809+0000", 2023-09-23T23:26:09.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5925000000000003e-05 2023-09-23T23:26:09.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:09.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.859876+0000", 2023-09-23T23:26:09.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7123000000000001e-05 2023-09-23T23:26:09.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.860003+0000", 2023-09-23T23:26:09.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000127057 2023-09-23T23:26:09.350 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.350 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.350 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.350 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:92277 10.9 10:98a3a044:::benchmark_data_smithi110_366450_object92276:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:26:09.351 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:20:25.911200+0000", 2023-09-23T23:26:09.351 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 343.40754034000003, 2023-09-23T23:26:09.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94902298299999999, 2023-09-23T23:26:09.351 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.351 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:09.352 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.352 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:26:09.352 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:26:09.352 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 92277 2023-09-23T23:26:09.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.352 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911200+0000", 2023-09-23T23:26:09.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911200+0000", 2023-09-23T23:26:09.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911200+0000", 2023-09-23T23:26:09.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:26:09.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911217+0000", 2023-09-23T23:26:09.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.77e-05 2023-09-23T23:26:09.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911227+0000", 2023-09-23T23:26:09.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9550000000000004e-06 2023-09-23T23:26:09.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911232+0000", 2023-09-23T23:26:09.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6099999999999999e-06 2023-09-23T23:26:09.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911257+0000", 2023-09-23T23:26:09.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5199e-05 2023-09-23T23:26:09.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:25.911262+0000", 2023-09-23T23:26:09.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7820000000000001e-06 2023-09-23T23:26:09.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.856640+0000", 2023-09-23T23:26:09.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94537845200000004 2023-09-23T23:26:09.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:09.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.856798+0000", 2023-09-23T23:26:09.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000157787 2023-09-23T23:26:09.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T23:26:09.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.856914+0000", 2023-09-23T23:26:09.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011568700000000001 2023-09-23T23:26:09.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.859731+0000", 2023-09-23T23:26:09.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028171210000000001 2023-09-23T23:26:09.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.859764+0000", 2023-09-23T23:26:09.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2990000000000001e-05 2023-09-23T23:26:09.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:09.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.860018+0000", 2023-09-23T23:26:09.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025431299999999999 2023-09-23T23:26:09.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.860076+0000", 2023-09-23T23:26:09.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8088999999999999e-05 2023-09-23T23:26:09.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:09.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.860114+0000", 2023-09-23T23:26:09.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7311000000000002e-05 2023-09-23T23:26:09.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:20:26.860223+0000", 2023-09-23T23:26:09.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109764 2023-09-23T23:26:09.361 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.361 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.361 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.361 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156651 10.1 10:8a72b73b:::benchmark_data_smithi110_366450_object55443:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:26:09.361 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:49.044180+0000", 2023-09-23T23:26:09.361 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 260.27456106800003, 2023-09-23T23:26:09.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.112146391, 2023-09-23T23:26:09.362 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.362 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:26:09.362 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.362 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:26:09.362 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:26:09.362 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156651 2023-09-23T23:26:09.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.362 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044180+0000", 2023-09-23T23:26:09.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044180+0000", 2023-09-23T23:26:09.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044179+0000", 2023-09-23T23:26:09.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:26:09.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044185+0000", 2023-09-23T23:26:09.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9699999999999996e-06 2023-09-23T23:26:09.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044197+0000", 2023-09-23T23:26:09.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2187e-05 2023-09-23T23:26:09.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044205+0000", 2023-09-23T23:26:09.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.678e-06 2023-09-23T23:26:09.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044265+0000", 2023-09-23T23:26:09.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9889000000000002e-05 2023-09-23T23:26:09.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op must wait for map", 2023-09-23T23:26:09.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044271+0000", 2023-09-23T23:26:09.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.455e-06 2023-09-23T23:26:09.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.052559+0000", 2023-09-23T23:26:09.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0082876549999999997 2023-09-23T23:26:09.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.052569+0000", 2023-09-23T23:26:09.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0039e-05 2023-09-23T23:26:09.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032597+0000", 2023-09-23T23:26:09.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98002873899999998 2023-09-23T23:26:09.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032607+0000", 2023-09-23T23:26:09.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4720000000000001e-06 2023-09-23T23:26:09.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.154569+0000", 2023-09-23T23:26:09.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.121962066 2023-09-23T23:26:09.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.156326+0000", 2023-09-23T23:26:09.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017571030000000001 2023-09-23T23:26:09.368 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.368 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.368 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.368 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156658 10.3 10.fcfa2c13 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:26:09.368 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:49.048969+0000", 2023-09-23T23:26:09.368 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 260.269771967, 2023-09-23T23:26:09.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98286358200000001, 2023-09-23T23:26:09.369 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.369 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:26:09.369 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.369 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:26:09.369 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:26:09.369 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156658 2023-09-23T23:26:09.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.369 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.048969+0000", 2023-09-23T23:26:09.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.048969+0000", 2023-09-23T23:26:09.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.048968+0000", 2023-09-23T23:26:09.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:26:09.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.048972+0000", 2023-09-23T23:26:09.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4879999999999999e-06 2023-09-23T23:26:09.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.048980+0000", 2023-09-23T23:26:09.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9689999999999994e-06 2023-09-23T23:26:09.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.048986+0000", 2023-09-23T23:26:09.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3489999999999997e-06 2023-09-23T23:26:09.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.065480+0000", 2023-09-23T23:26:09.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016493645000000001 2023-09-23T23:26:09.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.065490+0000", 2023-09-23T23:26:09.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0088000000000001e-05 2023-09-23T23:26:09.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.031815+0000", 2023-09-23T23:26:09.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96632486100000003 2023-09-23T23:26:09.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.031832+0000", 2023-09-23T23:26:09.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7717e-05 2023-09-23T23:26:09.373 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.374 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.374 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.374 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156665 10.3 10.bf1e5c73 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:26:09.374 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:49.064034+0000", 2023-09-23T23:26:09.374 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 260.25470684999999, 2023-09-23T23:26:09.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96784821899999995, 2023-09-23T23:26:09.377 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.377 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:26:09.378 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.378 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:26:09.378 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:26:09.378 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156665 2023-09-23T23:26:09.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.378 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064034+0000", 2023-09-23T23:26:09.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.379 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.379 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064034+0000", 2023-09-23T23:26:09.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.379 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.379 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064032+0000", 2023-09-23T23:26:09.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:09.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064039+0000", 2023-09-23T23:26:09.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.72e-06 2023-09-23T23:26:09.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064052+0000", 2023-09-23T23:26:09.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2439000000000001e-05 2023-09-23T23:26:09.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064060+0000", 2023-09-23T23:26:09.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6419999999999996e-06 2023-09-23T23:26:09.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.065512+0000", 2023-09-23T23:26:09.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001451761 2023-09-23T23:26:09.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.065520+0000", 2023-09-23T23:26:09.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.588e-06 2023-09-23T23:26:09.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.031862+0000", 2023-09-23T23:26:09.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96634209599999998 2023-09-23T23:26:09.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.031882+0000", 2023-09-23T23:26:09.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0483999999999999e-05 2023-09-23T23:26:09.384 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.384 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.384 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.384 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156668 10.1 10:8e70998a:::benchmark_data_smithi110_366450_object55460:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:26:09.385 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:49.064619+0000", 2023-09-23T23:26:09.385 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 260.25412143, 2023-09-23T23:26:09.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1088757770000002, 2023-09-23T23:26:09.385 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.385 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:09.385 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.385 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:26:09.385 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:26:09.385 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156668 2023-09-23T23:26:09.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.386 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064619+0000", 2023-09-23T23:26:09.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064619+0000", 2023-09-23T23:26:09.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064619+0000", 2023-09-23T23:26:09.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:26:09.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064623+0000", 2023-09-23T23:26:09.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4089999999999996e-06 2023-09-23T23:26:09.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064632+0000", 2023-09-23T23:26:09.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5830000000000007e-06 2023-09-23T23:26:09.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.389 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.389 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064637+0000", 2023-09-23T23:26:09.389 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0760000000000002e-06 2023-09-23T23:26:09.389 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.389 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.390 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.066069+0000", 2023-09-23T23:26:09.390 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014321799999999999 2023-09-23T23:26:09.390 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.390 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.390 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.390 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.066077+0000", 2023-09-23T23:26:09.390 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8390000000000007e-06 2023-09-23T23:26:09.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.391 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032631+0000", 2023-09-23T23:26:09.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96655426600000005 2023-09-23T23:26:09.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032638+0000", 2023-09-23T23:26:09.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7669999999999998e-06 2023-09-23T23:26:09.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.156763+0000", 2023-09-23T23:26:09.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.124125493 2023-09-23T23:26:09.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:09.393 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.157388+0000", 2023-09-23T23:26:09.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00062452500000000002 2023-09-23T23:26:09.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:26:09.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.157840+0000", 2023-09-23T23:26:09.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045213799999999999 2023-09-23T23:26:09.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.165314+0000", 2023-09-23T23:26:09.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074745469999999998 2023-09-23T23:26:09.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.165348+0000", 2023-09-23T23:26:09.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3593999999999997e-05 2023-09-23T23:26:09.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:09.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173335+0000", 2023-09-23T23:26:09.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079866030000000001 2023-09-23T23:26:09.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:09.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173385+0000", 2023-09-23T23:26:09.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0932e-05 2023-09-23T23:26:09.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.397 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.397 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173495+0000", 2023-09-23T23:26:09.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109603 2023-09-23T23:26:09.397 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.397 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.397 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.397 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.397 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.397 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156677 10.3 10.316d2ae3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:26:09.397 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:49.071554+0000", 2023-09-23T23:26:09.397 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 260.247186371, 2023-09-23T23:26:09.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96036474299999997, 2023-09-23T23:26:09.398 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.398 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:26:09.398 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.398 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:26:09.398 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:26:09.398 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156677 2023-09-23T23:26:09.398 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.398 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.398 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.398 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.398 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.071554+0000", 2023-09-23T23:26:09.398 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.399 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.399 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.071554+0000", 2023-09-23T23:26:09.399 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.399 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.399 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.071554+0000", 2023-09-23T23:26:09.399 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:26:09.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.399 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.071557+0000", 2023-09-23T23:26:09.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.835e-06 2023-09-23T23:26:09.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.400 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.400 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.071563+0000", 2023-09-23T23:26:09.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7240000000000003e-06 2023-09-23T23:26:09.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.400 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.400 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.071569+0000", 2023-09-23T23:26:09.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.587e-06 2023-09-23T23:26:09.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.071621+0000", 2023-09-23T23:26:09.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1851000000000002e-05 2023-09-23T23:26:09.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.071630+0000", 2023-09-23T23:26:09.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3190000000000001e-06 2023-09-23T23:26:09.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.031909+0000", 2023-09-23T23:26:09.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.960278768 2023-09-23T23:26:09.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.031919+0000", 2023-09-23T23:26:09.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0179e-05 2023-09-23T23:26:09.402 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.403 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.403 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.403 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156686 10.1 10:87df1f6d:::benchmark_data_smithi110_366450_object55478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:26:09.403 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:49.084621+0000", 2023-09-23T23:26:09.403 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 260.23412008899999, 2023-09-23T23:26:09.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0890410259999999, 2023-09-23T23:26:09.403 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.403 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:09.403 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.403 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:26:09.404 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:26:09.404 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156686 2023-09-23T23:26:09.404 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.404 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.404 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.404 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.404 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084621+0000", 2023-09-23T23:26:09.405 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.405 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.405 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.405 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.405 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084621+0000", 2023-09-23T23:26:09.405 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.405 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.405 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.405 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.405 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084620+0000", 2023-09-23T23:26:09.405 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:26:09.405 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.406 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.406 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.406 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084623+0000", 2023-09-23T23:26:09.406 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5040000000000001e-06 2023-09-23T23:26:09.406 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.406 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.406 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.406 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084634+0000", 2023-09-23T23:26:09.406 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1405e-05 2023-09-23T23:26:09.406 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.406 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.407 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.407 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084639+0000", 2023-09-23T23:26:09.407 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.729e-06 2023-09-23T23:26:09.407 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.407 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.407 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.407 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084684+0000", 2023-09-23T23:26:09.407 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5045e-05 2023-09-23T23:26:09.407 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.407 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.407 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.407 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084690+0000", 2023-09-23T23:26:09.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5650000000000001e-06 2023-09-23T23:26:09.408 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.408 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.408 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032662+0000", 2023-09-23T23:26:09.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94797212099999995 2023-09-23T23:26:09.408 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.408 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.408 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032668+0000", 2023-09-23T23:26:09.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6080000000000001e-06 2023-09-23T23:26:09.408 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.409 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.409 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.162444+0000", 2023-09-23T23:26:09.409 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1297754170000001 2023-09-23T23:26:09.409 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.409 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.409 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:09.409 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.162591+0000", 2023-09-23T23:26:09.409 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014782799999999999 2023-09-23T23:26:09.409 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.409 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.409 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:26:09.409 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.162685+0000", 2023-09-23T23:26:09.410 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3898000000000001e-05 2023-09-23T23:26:09.410 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.410 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.410 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.410 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.170322+0000", 2023-09-23T23:26:09.410 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007636866 2023-09-23T23:26:09.410 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.410 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.410 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.410 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.170366+0000", 2023-09-23T23:26:09.410 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3380000000000001e-05 2023-09-23T23:26:09.411 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.411 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:09.411 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173508+0000", 2023-09-23T23:26:09.411 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031422640000000001 2023-09-23T23:26:09.411 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.411 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:09.411 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173562+0000", 2023-09-23T23:26:09.411 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4169000000000003e-05 2023-09-23T23:26:09.412 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.412 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.412 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.412 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173662+0000", 2023-09-23T23:26:09.412 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9615000000000001e-05 2023-09-23T23:26:09.412 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.413 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.413 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.413 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.413 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.413 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156687 10.1 10:8bf04f95:::benchmark_data_smithi110_366450_object55479:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:26:09.413 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:49.084650+0000", 2023-09-23T23:26:09.414 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 260.23409091299999, 2023-09-23T23:26:09.414 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.089277176, 2023-09-23T23:26:09.414 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.414 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:09.414 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.415 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:26:09.415 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:26:09.415 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156687 2023-09-23T23:26:09.415 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.415 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.416 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.416 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.416 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084650+0000", 2023-09-23T23:26:09.416 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.416 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.416 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.417 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.417 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084650+0000", 2023-09-23T23:26:09.417 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.417 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.417 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.417 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.417 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084649+0000", 2023-09-23T23:26:09.418 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:26:09.418 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.418 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.418 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.418 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084652+0000", 2023-09-23T23:26:09.418 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2670000000000001e-06 2023-09-23T23:26:09.418 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.418 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.418 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.419 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084658+0000", 2023-09-23T23:26:09.419 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1279999999999996e-06 2023-09-23T23:26:09.419 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.419 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.419 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.419 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084662+0000", 2023-09-23T23:26:09.419 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1289999999999998e-06 2023-09-23T23:26:09.419 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.420 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.420 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.420 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084716+0000", 2023-09-23T23:26:09.420 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4110000000000002e-05 2023-09-23T23:26:09.420 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.420 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.420 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.420 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084725+0000", 2023-09-23T23:26:09.420 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6989999999999992e-06 2023-09-23T23:26:09.421 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.421 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.421 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.421 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032691+0000", 2023-09-23T23:26:09.421 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94796597400000004 2023-09-23T23:26:09.421 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.421 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.421 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.422 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032698+0000", 2023-09-23T23:26:09.422 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7479999999999996e-06 2023-09-23T23:26:09.422 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.422 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.422 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.422 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.165371+0000", 2023-09-23T23:26:09.422 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1326732019999999 2023-09-23T23:26:09.422 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.422 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.423 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:09.423 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.165505+0000", 2023-09-23T23:26:09.423 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013406900000000001 2023-09-23T23:26:09.423 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.424 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.424 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:26:09.424 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.165655+0000", 2023-09-23T23:26:09.424 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015065 2023-09-23T23:26:09.425 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.425 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.425 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:09.425 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173673+0000", 2023-09-23T23:26:09.425 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080170699999999994 2023-09-23T23:26:09.425 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.425 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.425 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.426 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173736+0000", 2023-09-23T23:26:09.426 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3116999999999999e-05 2023-09-23T23:26:09.426 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.426 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.426 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.426 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173781+0000", 2023-09-23T23:26:09.426 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5778000000000001e-05 2023-09-23T23:26:09.426 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.427 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.427 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:09.427 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173829+0000", 2023-09-23T23:26:09.427 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7182999999999999e-05 2023-09-23T23:26:09.427 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.427 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.427 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.427 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173927+0000", 2023-09-23T23:26:09.428 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8424000000000005e-05 2023-09-23T23:26:09.428 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.428 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.428 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.428 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.428 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.429 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156689 10.3 10.511833c3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:26:09.429 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:49.088136+0000", 2023-09-23T23:26:09.429 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 260.23060450999998, 2023-09-23T23:26:09.429 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94381797700000003, 2023-09-23T23:26:09.429 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.429 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:26:09.429 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.431 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:26:09.431 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:26:09.432 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156689 2023-09-23T23:26:09.432 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.435 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.435 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.435 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.436 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.088136+0000", 2023-09-23T23:26:09.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.436 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.436 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.436 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.088136+0000", 2023-09-23T23:26:09.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.436 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.436 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.437 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.088135+0000", 2023-09-23T23:26:09.437 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:26:09.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.437 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.088140+0000", 2023-09-23T23:26:09.437 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5260000000000004e-06 2023-09-23T23:26:09.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.437 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.088149+0000", 2023-09-23T23:26:09.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1700000000000003e-06 2023-09-23T23:26:09.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.438 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.088156+0000", 2023-09-23T23:26:09.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1409999999999999e-06 2023-09-23T23:26:09.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.438 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.088202+0000", 2023-09-23T23:26:09.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6097000000000003e-05 2023-09-23T23:26:09.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.439 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.439 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.439 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.088209+0000", 2023-09-23T23:26:09.439 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0099999999999998e-06 2023-09-23T23:26:09.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.439 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.439 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.439 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.031943+0000", 2023-09-23T23:26:09.439 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94373409399999997 2023-09-23T23:26:09.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.440 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.440 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.440 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.031954+0000", 2023-09-23T23:26:09.440 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.077e-05 2023-09-23T23:26:09.440 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.440 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.440 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.440 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.440 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.440 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:108667 11.10 11.cc161150 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T23:26:09.440 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:24.510844+0000", 2023-09-23T23:26:09.440 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 44.807896831000001, 2023-09-23T23:26:09.441 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0013290399999999, 2023-09-23T23:26:09.441 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.441 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:26:09.441 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.441 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:09.441 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:09.441 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 108667 2023-09-23T23:26:09.441 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.441 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.441 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.441 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.510844+0000", 2023-09-23T23:26:09.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.442 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.442 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.442 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.442 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.510844+0000", 2023-09-23T23:26:09.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.442 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.442 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.442 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.442 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.510843+0000", 2023-09-23T23:26:09.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:09.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.443 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.510849+0000", 2023-09-23T23:26:09.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.742e-06 2023-09-23T23:26:09.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.443 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.510867+0000", 2023-09-23T23:26:09.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7394e-05 2023-09-23T23:26:09.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.444 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.510876+0000", 2023-09-23T23:26:09.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5489999999999995e-06 2023-09-23T23:26:09.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.444 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.519715+0000", 2023-09-23T23:26:09.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0088387459999999998 2023-09-23T23:26:09.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.444 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.519723+0000", 2023-09-23T23:26:09.445 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2530000000000004e-06 2023-09-23T23:26:09.445 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.445 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.445 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.445 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:25.512161+0000", 2023-09-23T23:26:09.445 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99243793000000002 2023-09-23T23:26:09.445 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.445 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.445 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.445 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:25.512173+0000", 2023-09-23T23:26:09.445 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1836e-05 2023-09-23T23:26:09.445 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.446 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.446 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.446 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.446 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:108683 11.10 11.bfb9b70 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T23:26:09.446 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:24.517045+0000", 2023-09-23T23:26:09.446 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 44.801695887999998, 2023-09-23T23:26:09.446 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99516022599999998, 2023-09-23T23:26:09.446 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.446 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:26:09.446 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.446 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:09.447 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:09.447 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 108683 2023-09-23T23:26:09.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.447 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.447 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.447 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.447 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.517045+0000", 2023-09-23T23:26:09.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.447 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.447 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.447 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.517045+0000", 2023-09-23T23:26:09.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.517043+0000", 2023-09-23T23:26:09.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:09.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.517051+0000", 2023-09-23T23:26:09.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2989999999999999e-06 2023-09-23T23:26:09.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.517064+0000", 2023-09-23T23:26:09.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3926e-05 2023-09-23T23:26:09.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.517073+0000", 2023-09-23T23:26:09.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8910000000000001e-06 2023-09-23T23:26:09.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.519744+0000", 2023-09-23T23:26:09.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026710850000000001 2023-09-23T23:26:09.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.450 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.519750+0000", 2023-09-23T23:26:09.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8000000000000004e-06 2023-09-23T23:26:09.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.450 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:25.512197+0000", 2023-09-23T23:26:09.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99244725099999997 2023-09-23T23:26:09.451 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.451 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.451 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.451 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:25.512205+0000", 2023-09-23T23:26:09.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6369999999999999e-06 2023-09-23T23:26:09.451 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.451 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.451 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.451 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.451 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.451 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:108687 11.10 11.38d4ac50 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T23:26:09.452 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:24.522236+0000", 2023-09-23T23:26:09.452 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 44.796504812000002, 2023-09-23T23:26:09.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98999880299999998, 2023-09-23T23:26:09.452 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.452 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:26:09.452 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.452 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:09.452 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:09.452 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 108687 2023-09-23T23:26:09.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.452 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.522236+0000", 2023-09-23T23:26:09.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.522236+0000", 2023-09-23T23:26:09.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.522234+0000", 2023-09-23T23:26:09.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:09.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.522242+0000", 2023-09-23T23:26:09.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5490000000000001e-06 2023-09-23T23:26:09.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.522255+0000", 2023-09-23T23:26:09.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2847000000000001e-05 2023-09-23T23:26:09.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.522263+0000", 2023-09-23T23:26:09.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7080000000000003e-06 2023-09-23T23:26:09.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.522310+0000", 2023-09-23T23:26:09.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6306e-05 2023-09-23T23:26:09.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.456 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.522318+0000", 2023-09-23T23:26:09.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5650000000000002e-06 2023-09-23T23:26:09.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.456 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:25.512226+0000", 2023-09-23T23:26:09.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98990815300000001 2023-09-23T23:26:09.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.456 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:25.512235+0000", 2023-09-23T23:26:09.457 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.276e-06 2023-09-23T23:26:09.457 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.457 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.457 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.457 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.457 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.457 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:108693 11.10 11.b80100b0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T23:26:09.457 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:24.527096+0000", 2023-09-23T23:26:09.457 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 44.791644705000003, 2023-09-23T23:26:09.457 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98516907399999998, 2023-09-23T23:26:09.457 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.457 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:26:09.458 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.458 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:09.458 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:09.458 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 108693 2023-09-23T23:26:09.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.458 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.458 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.458 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.458 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.527096+0000", 2023-09-23T23:26:09.458 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.458 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.458 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.527096+0000", 2023-09-23T23:26:09.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.459 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.459 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.527096+0000", 2023-09-23T23:26:09.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:26:09.459 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.459 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.527098+0000", 2023-09-23T23:26:09.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7989999999999998e-06 2023-09-23T23:26:09.460 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.460 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.460 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.460 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.527106+0000", 2023-09-23T23:26:09.460 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6499999999999996e-06 2023-09-23T23:26:09.460 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.460 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.460 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.460 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.527111+0000", 2023-09-23T23:26:09.460 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6469999999999997e-06 2023-09-23T23:26:09.460 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.460 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.461 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.528457+0000", 2023-09-23T23:26:09.461 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001345873 2023-09-23T23:26:09.461 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.461 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.528464+0000", 2023-09-23T23:26:09.461 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.951e-06 2023-09-23T23:26:09.461 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.461 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:25.512258+0000", 2023-09-23T23:26:09.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98379410199999995 2023-09-23T23:26:09.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:25.512265+0000", 2023-09-23T23:26:09.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4479999999999997e-06 2023-09-23T23:26:09.462 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.462 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.462 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.462 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:108723 11.10 11.80c3b7d0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T23:26:09.463 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:24.568173+0000", 2023-09-23T23:26:09.463 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 44.750567416000003, 2023-09-23T23:26:09.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94411997299999995, 2023-09-23T23:26:09.463 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.463 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:26:09.463 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.463 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:09.463 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:09.463 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 108723 2023-09-23T23:26:09.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.463 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.463 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.568173+0000", 2023-09-23T23:26:09.464 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.568173+0000", 2023-09-23T23:26:09.464 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.568171+0000", 2023-09-23T23:26:09.464 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:26:09.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.465 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.465 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.568182+0000", 2023-09-23T23:26:09.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0847e-05 2023-09-23T23:26:09.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.465 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.465 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.568208+0000", 2023-09-23T23:26:09.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5269000000000002e-05 2023-09-23T23:26:09.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.465 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.466 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.568219+0000", 2023-09-23T23:26:09.466 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1002000000000001e-05 2023-09-23T23:26:09.466 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.466 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.466 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.466 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.568280+0000", 2023-09-23T23:26:09.466 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1595999999999998e-05 2023-09-23T23:26:09.466 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.466 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.466 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.466 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:24.568292+0000", 2023-09-23T23:26:09.466 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1888999999999999e-05 2023-09-23T23:26:09.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.467 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.467 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:25.512286+0000", 2023-09-23T23:26:09.467 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94399385300000005 2023-09-23T23:26:09.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.467 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.467 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:25.512293+0000", 2023-09-23T23:26:09.467 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.43e-06 2023-09-23T23:26:09.467 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.467 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.468 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.468 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.468 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:121565 11.15 11:af4917d1:::benchmark_data_smithi110_433746_object15685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e377)", 2023-09-23T23:26:09.468 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:48.192705+0000", 2023-09-23T23:26:09.468 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 21.126035528999999, 2023-09-23T23:26:09.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.88735869300000003, 2023-09-23T23:26:09.468 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.468 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:09.468 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.468 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:09.468 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:09.469 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 121565 2023-09-23T23:26:09.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.469 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.469 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.469 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.469 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.192705+0000", 2023-09-23T23:26:09.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.469 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.469 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.469 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.192705+0000", 2023-09-23T23:26:09.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.192704+0000", 2023-09-23T23:26:09.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:09.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.192710+0000", 2023-09-23T23:26:09.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.438e-06 2023-09-23T23:26:09.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.192724+0000", 2023-09-23T23:26:09.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4307999999999999e-05 2023-09-23T23:26:09.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.192731+0000", 2023-09-23T23:26:09.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7360000000000004e-06 2023-09-23T23:26:09.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.192774+0000", 2023-09-23T23:26:09.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2407999999999998e-05 2023-09-23T23:26:09.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.192784+0000", 2023-09-23T23:26:09.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0557000000000001e-05 2023-09-23T23:26:09.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.072455+0000", 2023-09-23T23:26:09.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.87967114400000002 2023-09-23T23:26:09.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:09.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.072583+0000", 2023-09-23T23:26:09.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012790100000000001 2023-09-23T23:26:09.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:26:09.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.072953+0000", 2023-09-23T23:26:09.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036960399999999999 2023-09-23T23:26:09.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.078217+0000", 2023-09-23T23:26:09.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052644709999999997 2023-09-23T23:26:09.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.078277+0000", 2023-09-23T23:26:09.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0069999999999999e-05 2023-09-23T23:26:09.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:09.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.079970+0000", 2023-09-23T23:26:09.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016925180000000001 2023-09-23T23:26:09.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:09.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.080023+0000", 2023-09-23T23:26:09.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2743e-05 2023-09-23T23:26:09.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.080064+0000", 2023-09-23T23:26:09.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1386e-05 2023-09-23T23:26:09.475 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.476 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.476 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.476 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:121669 11.15 11:ab2713ca:::benchmark_data_smithi110_433746_object15789:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e377)", 2023-09-23T23:26:09.476 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:48.317699+0000", 2023-09-23T23:26:09.476 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 21.001042170000002, 2023-09-23T23:26:09.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.76733598700000005, 2023-09-23T23:26:09.476 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.476 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:09.476 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.476 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:09.477 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:09.477 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 121669 2023-09-23T23:26:09.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.477 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.477 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.477 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.477 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.317699+0000", 2023-09-23T23:26:09.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.477 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.477 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.477 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.317699+0000", 2023-09-23T23:26:09.478 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.478 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.478 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.317698+0000", 2023-09-23T23:26:09.478 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:26:09.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.478 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.478 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.317701+0000", 2023-09-23T23:26:09.478 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1219999999999999e-06 2023-09-23T23:26:09.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.317707+0000", 2023-09-23T23:26:09.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7200000000000003e-06 2023-09-23T23:26:09.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.317712+0000", 2023-09-23T23:26:09.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.169e-06 2023-09-23T23:26:09.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.318594+0000", 2023-09-23T23:26:09.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00088194399999999996 2023-09-23T23:26:09.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.318608+0000", 2023-09-23T23:26:09.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3464e-05 2023-09-23T23:26:09.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.074814+0000", 2023-09-23T23:26:09.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.756206822 2023-09-23T23:26:09.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:09.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.074934+0000", 2023-09-23T23:26:09.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011976099999999999 2023-09-23T23:26:09.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:26:09.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.075149+0000", 2023-09-23T23:26:09.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021515299999999999 2023-09-23T23:26:09.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:09.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.080148+0000", 2023-09-23T23:26:09.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.004999135 2023-09-23T23:26:09.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.084862+0000", 2023-09-23T23:26:09.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047131639999999997 2023-09-23T23:26:09.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.084959+0000", 2023-09-23T23:26:09.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7063000000000002e-05 2023-09-23T23:26:09.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:09.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.085000+0000", 2023-09-23T23:26:09.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1229000000000001e-05 2023-09-23T23:26:09.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.484 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.085035+0000", 2023-09-23T23:26:09.484 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4654999999999999e-05 2023-09-23T23:26:09.484 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.484 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.484 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.484 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.484 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:121694 11.15 11:aea0ea64:::benchmark_data_smithi110_433746_object15814:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e377)", 2023-09-23T23:26:09.484 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:48.347624+0000", 2023-09-23T23:26:09.484 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 20.971116891000001, 2023-09-23T23:26:09.484 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.73754983600000001, 2023-09-23T23:26:09.484 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.485 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:09.485 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.485 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:09.485 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:09.485 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 121694 2023-09-23T23:26:09.485 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.485 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.485 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.485 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.485 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.347624+0000", 2023-09-23T23:26:09.485 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.486 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.486 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.486 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.347624+0000", 2023-09-23T23:26:09.486 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.486 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.486 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.486 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.347622+0000", 2023-09-23T23:26:09.486 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:09.486 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.347629+0000", 2023-09-23T23:26:09.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8519999999999998e-06 2023-09-23T23:26:09.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.347640+0000", 2023-09-23T23:26:09.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1078e-05 2023-09-23T23:26:09.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.347648+0000", 2023-09-23T23:26:09.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9219999999999997e-06 2023-09-23T23:26:09.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.347724+0000", 2023-09-23T23:26:09.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5933999999999995e-05 2023-09-23T23:26:09.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.347733+0000", 2023-09-23T23:26:09.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0359999999999995e-06 2023-09-23T23:26:09.489 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.489 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.489 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.076584+0000", 2023-09-23T23:26:09.489 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.72885044899999996 2023-09-23T23:26:09.489 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.489 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:09.490 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.076711+0000", 2023-09-23T23:26:09.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012794999999999999 2023-09-23T23:26:09.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.490 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.490 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:26:09.490 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.076941+0000", 2023-09-23T23:26:09.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022911399999999999 2023-09-23T23:26:09.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.490 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.490 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:09.490 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.081791+0000", 2023-09-23T23:26:09.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048504539999999997 2023-09-23T23:26:09.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.084916+0000", 2023-09-23T23:26:09.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031246630000000002 2023-09-23T23:26:09.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.085105+0000", 2023-09-23T23:26:09.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018959399999999999 2023-09-23T23:26:09.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.492 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:09.492 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.085141+0000", 2023-09-23T23:26:09.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6130000000000001e-05 2023-09-23T23:26:09.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.492 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.492 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.085174+0000", 2023-09-23T23:26:09.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2249000000000002e-05 2023-09-23T23:26:09.492 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.492 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.492 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.493 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:121724 11.15 11:ae459d45:::benchmark_data_smithi110_433746_object15844:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e377)", 2023-09-23T23:26:09.493 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:48.383263+0000", 2023-09-23T23:26:09.493 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 20.935478242999999, 2023-09-23T23:26:09.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.70204546499999998, 2023-09-23T23:26:09.493 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:09.493 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:09.493 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:09.493 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:09.493 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:09.493 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 121724 2023-09-23T23:26:09.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.494 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:09.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.494 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:09.494 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.383263+0000", 2023-09-23T23:26:09.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.494 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:09.494 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.383263+0000", 2023-09-23T23:26:09.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:09.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.495 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:09.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.383261+0000", 2023-09-23T23:26:09.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:09.495 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.495 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:09.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.383267+0000", 2023-09-23T23:26:09.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3160000000000002e-06 2023-09-23T23:26:09.495 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.495 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:09.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.383278+0000", 2023-09-23T23:26:09.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.009e-05 2023-09-23T23:26:09.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:09.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.383284+0000", 2023-09-23T23:26:09.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7560000000000002e-06 2023-09-23T23:26:09.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.383346+0000", 2023-09-23T23:26:09.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1934999999999996e-05 2023-09-23T23:26:09.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:09.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:48.383355+0000", 2023-09-23T23:26:09.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0729999999999994e-06 2023-09-23T23:26:09.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:09.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.078374+0000", 2023-09-23T23:26:09.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.69501876600000001 2023-09-23T23:26:09.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:09.498 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.078485+0000", 2023-09-23T23:26:09.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000111251 2023-09-23T23:26:09.498 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.498 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.498 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:26:09.498 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.078706+0000", 2023-09-23T23:26:09.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022111599999999999 2023-09-23T23:26:09.498 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.498 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.498 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:09.498 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.083366+0000", 2023-09-23T23:26:09.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046594729999999999 2023-09-23T23:26:09.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.499 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.085071+0000", 2023-09-23T23:26:09.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001705009 2023-09-23T23:26:09.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:09.499 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.085240+0000", 2023-09-23T23:26:09.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016871799999999999 2023-09-23T23:26:09.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:09.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.085277+0000", 2023-09-23T23:26:09.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7163000000000003e-05 2023-09-23T23:26:09.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:09.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:09.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:09.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:49.085308+0000", 2023-09-23T23:26:09.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1170000000000001e-05 2023-09-23T23:26:09.500 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.500 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.500 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.500 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:09.501 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:09.501 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:26:09.501 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:09.540 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:09.595 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:09.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:09.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:09.707 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:09.738 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:09.792 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:09.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:09.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:09.898 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:26:09.968 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:10.054 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:26:10.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:10.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:26:10.160 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:26:10.213 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:10.284 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:26:10.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:10.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:26:10.388 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:26:10.416 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:10.463 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:26:10.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:10.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:26:10.510 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:10 smithi110 ceph-mon[212939]: failed to encode map e387 with expected crc 2023-09-23T23:26:10.510 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:10 smithi110 ceph-mon[212939]: osdmap e387: 8 total, 8 up, 7 in 2023-09-23T23:26:10.510 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:10 smithi110 ceph-mon[212939]: failed to encode map e387 with expected crc 2023-09-23T23:26:10.511 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:10 smithi110 ceph-mon[212939]: pgmap v1136: 129 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.2 KiB/s rd, 8.9 KiB/s wr, 656 op/s; 8226/233499 objects degraded (3.523%); 38516/233499 objects misplaced (16.495%); 30 MiB/s, 473 objects/s recovering 2023-09-23T23:26:10.511 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:10 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1352363323' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:26:10.511 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:10 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:26:10.522 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T23:26:10.569 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:26:10.633 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:10.673 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:26:10.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:10 smithi138 ceph-mon[193606]: failed to encode map e387 with expected crc 2023-09-23T23:26:10.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:10 smithi138 ceph-mon[193606]: osdmap e387: 8 total, 8 up, 7 in 2023-09-23T23:26:10.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:10 smithi138 ceph-mon[193606]: failed to encode map e387 with expected crc 2023-09-23T23:26:10.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:10 smithi138 ceph-mon[193606]: pgmap v1136: 129 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.2 KiB/s rd, 8.9 KiB/s wr, 656 op/s; 8226/233499 objects degraded (3.523%); 38516/233499 objects misplaced (16.495%); 30 MiB/s, 473 objects/s recovering 2023-09-23T23:26:10.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:10 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1352363323' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:26:10.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:10 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:26:10.872 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:26:10.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:10.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:26:10.876 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:10 smithi110 ceph-mon[221102]: failed to encode map e387 with expected crc 2023-09-23T23:26:10.877 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:10 smithi110 ceph-mon[221102]: osdmap e387: 8 total, 8 up, 7 in 2023-09-23T23:26:10.877 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:10 smithi110 ceph-mon[221102]: failed to encode map e387 with expected crc 2023-09-23T23:26:10.877 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:10 smithi110 ceph-mon[221102]: pgmap v1136: 129 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.2 KiB/s rd, 8.9 KiB/s wr, 656 op/s; 8226/233499 objects degraded (3.523%); 38516/233499 objects misplaced (16.495%); 30 MiB/s, 473 objects/s recovering 2023-09-23T23:26:10.877 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:10 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1352363323' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:26:10.877 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:10 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:26:10.900 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:26:10.977 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:26:11.004 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:11.481 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:26:11.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:11.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:26:11.566 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:26:11.585 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:11.596 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T23:26:11.612 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:11.656 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:11.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:11.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:11.706 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:26:11.761 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:11.784 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:11 smithi138 ceph-mon[193606]: 11.17 scrub starts 2023-09-23T23:26:11.785 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:11 smithi138 ceph-mon[193606]: failed to encode map e388 with expected crc 2023-09-23T23:26:11.785 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:11 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:26:11.785 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:11 smithi138 ceph-mon[193606]: osdmap e388: 8 total, 8 up, 7 in 2023-09-23T23:26:11.785 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:11 smithi138 ceph-mon[193606]: failed to encode map e388 with expected crc 2023-09-23T23:26:11.785 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:11 smithi138 ceph-mon[193606]: failed to encode map e388 with expected crc 2023-09-23T23:26:11.785 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:11 smithi138 ceph-mon[193606]: osdmap e389: 8 total, 8 up, 7 in 2023-09-23T23:26:11.787 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:11.806 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:26:11.832 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:11.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:11.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:11.937 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:26:11.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:11 smithi110 ceph-mon[212939]: 11.17 scrub starts 2023-09-23T23:26:11.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:11 smithi110 ceph-mon[212939]: failed to encode map e388 with expected crc 2023-09-23T23:26:11.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:11 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:26:11.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:11 smithi110 ceph-mon[212939]: osdmap e388: 8 total, 8 up, 7 in 2023-09-23T23:26:11.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:11 smithi110 ceph-mon[212939]: failed to encode map e388 with expected crc 2023-09-23T23:26:11.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:11 smithi110 ceph-mon[212939]: failed to encode map e388 with expected crc 2023-09-23T23:26:11.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:11 smithi110 ceph-mon[212939]: osdmap e389: 8 total, 8 up, 7 in 2023-09-23T23:26:11.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:11 smithi110 ceph-mon[221102]: 11.17 scrub starts 2023-09-23T23:26:11.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:11 smithi110 ceph-mon[221102]: failed to encode map e388 with expected crc 2023-09-23T23:26:11.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:11 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:26:11.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:11 smithi110 ceph-mon[221102]: osdmap e388: 8 total, 8 up, 7 in 2023-09-23T23:26:11.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:11 smithi110 ceph-mon[221102]: failed to encode map e388 with expected crc 2023-09-23T23:26:11.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:11 smithi110 ceph-mon[221102]: failed to encode map e388 with expected crc 2023-09-23T23:26:11.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:11 smithi110 ceph-mon[221102]: osdmap e389: 8 total, 8 up, 7 in 2023-09-23T23:26:11.983 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:12.160 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:26:12.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:12.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:26:12.266 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:26:12.303 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:12.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[212939]: 11.13 scrub starts 2023-09-23T23:26:12.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[212939]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[212939]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[212939]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[212939]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[212939]: 11.b starting backfill to osd.4 from (0'0,0'0] MAX to 387'8555 2023-09-23T23:26:12.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[212939]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[212939]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[212939]: pgmap v1139: 129 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 8226/229863 objects degraded (3.579%); 37052/229863 objects misplaced (16.119%); 23 MiB/s, 362 objects/s recovering 2023-09-23T23:26:12.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[212939]: osdmap e390: 8 total, 8 up, 7 in 2023-09-23T23:26:12.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[221102]: 11.13 scrub starts 2023-09-23T23:26:12.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[221102]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[221102]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[221102]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[221102]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[221102]: 11.b starting backfill to osd.4 from (0'0,0'0] MAX to 387'8555 2023-09-23T23:26:12.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[221102]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[221102]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[221102]: pgmap v1139: 129 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 8226/229863 objects degraded (3.579%); 37052/229863 objects misplaced (16.119%); 23 MiB/s, 362 objects/s recovering 2023-09-23T23:26:12.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:12 smithi110 ceph-mon[221102]: osdmap e390: 8 total, 8 up, 7 in 2023-09-23T23:26:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:12 smithi138 ceph-mon[193606]: 11.13 scrub starts 2023-09-23T23:26:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:12 smithi138 ceph-mon[193606]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:12 smithi138 ceph-mon[193606]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:12 smithi138 ceph-mon[193606]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:12 smithi138 ceph-mon[193606]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:12 smithi138 ceph-mon[193606]: 11.b starting backfill to osd.4 from (0'0,0'0] MAX to 387'8555 2023-09-23T23:26:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:12 smithi138 ceph-mon[193606]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:12 smithi138 ceph-mon[193606]: failed to encode map e389 with expected crc 2023-09-23T23:26:12.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:12 smithi138 ceph-mon[193606]: pgmap v1139: 129 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 8226/229863 objects degraded (3.579%); 37052/229863 objects misplaced (16.119%); 23 MiB/s, 362 objects/s recovering 2023-09-23T23:26:12.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:12 smithi138 ceph-mon[193606]: osdmap e390: 8 total, 8 up, 7 in 2023-09-23T23:26:13.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:13 smithi138 ceph-mon[193606]: 11.17 scrub starts 2023-09-23T23:26:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:13 smithi138 ceph-mon[193606]: failed to encode map e390 with expected crc 2023-09-23T23:26:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:13 smithi138 ceph-mon[193606]: failed to encode map e390 with expected crc 2023-09-23T23:26:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:13 smithi138 ceph-mon[193606]: failed to encode map e390 with expected crc 2023-09-23T23:26:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:13 smithi138 ceph-mon[193606]: failed to encode map e390 with expected crc 2023-09-23T23:26:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:13 smithi138 ceph-mon[193606]: 11.c starting backfill to osd.4 from (0'0,0'0] MAX to 387'8261 2023-09-23T23:26:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:13 smithi138 ceph-mon[193606]: 11.e starting backfill to osd.4 from (0'0,0'0] MAX to 387'8502 2023-09-23T23:26:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:13.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:26:13.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:13 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 8226/229863 objects degraded (3.579%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:13.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:13 smithi138 ceph-mon[193606]: osdmap e391: 8 total, 8 up, 7 in 2023-09-23T23:26:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[212939]: 11.17 scrub starts 2023-09-23T23:26:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[212939]: failed to encode map e390 with expected crc 2023-09-23T23:26:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[212939]: failed to encode map e390 with expected crc 2023-09-23T23:26:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[212939]: failed to encode map e390 with expected crc 2023-09-23T23:26:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[212939]: failed to encode map e390 with expected crc 2023-09-23T23:26:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[212939]: 11.c starting backfill to osd.4 from (0'0,0'0] MAX to 387'8261 2023-09-23T23:26:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[212939]: 11.e starting backfill to osd.4 from (0'0,0'0] MAX to 387'8502 2023-09-23T23:26:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:13.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:26:13.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 8226/229863 objects degraded (3.579%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:13.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[212939]: osdmap e391: 8 total, 8 up, 7 in 2023-09-23T23:26:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[221102]: 11.17 scrub starts 2023-09-23T23:26:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[221102]: failed to encode map e390 with expected crc 2023-09-23T23:26:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[221102]: failed to encode map e390 with expected crc 2023-09-23T23:26:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[221102]: failed to encode map e390 with expected crc 2023-09-23T23:26:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[221102]: failed to encode map e390 with expected crc 2023-09-23T23:26:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[221102]: 11.c starting backfill to osd.4 from (0'0,0'0] MAX to 387'8261 2023-09-23T23:26:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[221102]: 11.e starting backfill to osd.4 from (0'0,0'0] MAX to 387'8502 2023-09-23T23:26:13.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:26:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 8226/229863 objects degraded (3.579%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:13 smithi110 ceph-mon[221102]: osdmap e391: 8 total, 8 up, 7 in 2023-09-23T23:26:14.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:14 smithi138 ceph-mon[193606]: failed to encode map e390 with expected crc 2023-09-23T23:26:14.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:14 smithi138 ceph-mon[193606]: failed to encode map e390 with expected crc 2023-09-23T23:26:14.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:14 smithi138 ceph-mon[193606]: failed to encode map e391 with expected crc 2023-09-23T23:26:14.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:14 smithi138 ceph-mon[193606]: failed to encode map e391 with expected crc 2023-09-23T23:26:14.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:14 smithi138 ceph-mon[193606]: failed to encode map e391 with expected crc 2023-09-23T23:26:14.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:14 smithi138 ceph-mon[193606]: failed to encode map e391 with expected crc 2023-09-23T23:26:14.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:14 smithi138 ceph-mon[193606]: pgmap v1142: 129 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+clean+remapped, 11 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 350 op/s; 8607/229293 objects degraded (3.754%); 36317/229293 objects misplaced (15.839%); 14 MiB/s, 2 keys/s, 230 objects/s recovering 2023-09-23T23:26:14.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:14 smithi138 ceph-mon[193606]: osdmap e392: 8 total, 8 up, 7 in 2023-09-23T23:26:14.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:14 smithi110 ceph-mon[212939]: failed to encode map e390 with expected crc 2023-09-23T23:26:14.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:14 smithi110 ceph-mon[212939]: failed to encode map e390 with expected crc 2023-09-23T23:26:14.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:14 smithi110 ceph-mon[212939]: failed to encode map e391 with expected crc 2023-09-23T23:26:14.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:14 smithi110 ceph-mon[212939]: failed to encode map e391 with expected crc 2023-09-23T23:26:14.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:14 smithi110 ceph-mon[212939]: failed to encode map e391 with expected crc 2023-09-23T23:26:14.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:14 smithi110 ceph-mon[212939]: failed to encode map e391 with expected crc 2023-09-23T23:26:14.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:14 smithi110 ceph-mon[212939]: pgmap v1142: 129 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+clean+remapped, 11 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 350 op/s; 8607/229293 objects degraded (3.754%); 36317/229293 objects misplaced (15.839%); 14 MiB/s, 2 keys/s, 230 objects/s recovering 2023-09-23T23:26:14.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:14 smithi110 ceph-mon[212939]: osdmap e392: 8 total, 8 up, 7 in 2023-09-23T23:26:14.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:14 smithi110 ceph-mon[221102]: failed to encode map e390 with expected crc 2023-09-23T23:26:14.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:14 smithi110 ceph-mon[221102]: failed to encode map e390 with expected crc 2023-09-23T23:26:14.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:14 smithi110 ceph-mon[221102]: failed to encode map e391 with expected crc 2023-09-23T23:26:14.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:14 smithi110 ceph-mon[221102]: failed to encode map e391 with expected crc 2023-09-23T23:26:14.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:14 smithi110 ceph-mon[221102]: failed to encode map e391 with expected crc 2023-09-23T23:26:14.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:14 smithi110 ceph-mon[221102]: failed to encode map e391 with expected crc 2023-09-23T23:26:14.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:14 smithi110 ceph-mon[221102]: pgmap v1142: 129 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+clean+remapped, 11 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 350 op/s; 8607/229293 objects degraded (3.754%); 36317/229293 objects misplaced (15.839%); 14 MiB/s, 2 keys/s, 230 objects/s recovering 2023-09-23T23:26:14.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:14 smithi110 ceph-mon[221102]: osdmap e392: 8 total, 8 up, 7 in 2023-09-23T23:26:15.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:15 smithi138 ceph-mon[193606]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:15 smithi138 ceph-mon[193606]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:15 smithi138 ceph-mon[193606]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:15 smithi138 ceph-mon[193606]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:15 smithi138 ceph-mon[193606]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:15 smithi138 ceph-mon[193606]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:15 smithi138 ceph-mon[193606]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:15 smithi110 ceph-mon[212939]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:15 smithi110 ceph-mon[212939]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:15 smithi110 ceph-mon[212939]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:15 smithi110 ceph-mon[212939]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:15 smithi110 ceph-mon[212939]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:15 smithi110 ceph-mon[212939]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:15 smithi110 ceph-mon[212939]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:15 smithi110 ceph-mon[221102]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:15 smithi110 ceph-mon[221102]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:15 smithi110 ceph-mon[221102]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:15 smithi110 ceph-mon[221102]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:15 smithi110 ceph-mon[221102]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:15 smithi110 ceph-mon[221102]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:15 smithi110 ceph-mon[221102]: failed to encode map e392 with expected crc 2023-09-23T23:26:15.956 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:26:15.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:15.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:26:16.064 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:16.104 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:16.180 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:16.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:16.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:16.287 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:26:16.318 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:16.379 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:26:16.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:16.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:26:16.484 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:26:16.644 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:16.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:16 smithi138 ceph-mon[193606]: pgmap v1144: 129 pgs: 3 active+remapped+backfilling, 8 peering, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 5.1 KiB/s rd, 13 KiB/s wr, 355 op/s; 3748/225483 objects degraded (1.662%); 49003/225483 objects misplaced (21.732%); 53 MiB/s, 2 keys/s, 851 objects/s recovering 2023-09-23T23:26:16.803 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:26:16.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:16.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:26:16.908 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:26:16.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:16 smithi110 ceph-mon[221102]: pgmap v1144: 129 pgs: 3 active+remapped+backfilling, 8 peering, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 5.1 KiB/s rd, 13 KiB/s wr, 355 op/s; 3748/225483 objects degraded (1.662%); 49003/225483 objects misplaced (21.732%); 53 MiB/s, 2 keys/s, 851 objects/s recovering 2023-09-23T23:26:16.963 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:16 smithi110 ceph-mon[212939]: pgmap v1144: 129 pgs: 3 active+remapped+backfilling, 8 peering, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 5.1 KiB/s rd, 13 KiB/s wr, 355 op/s; 3748/225483 objects degraded (1.662%); 49003/225483 objects misplaced (21.732%); 53 MiB/s, 2 keys/s, 851 objects/s recovering 2023-09-23T23:26:16.963 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:26:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:26:16] "GET /metrics HTTP/1.1" 200 36603 "" "Prometheus/2.43.0" 2023-09-23T23:26:17.032 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:17.174 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:26:17.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:17.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:26:17.282 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:17.310 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:17.358 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:17.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:17.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:17.463 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:26:17.497 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:17.560 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:26:17.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:17.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:26:17.664 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:26:17.697 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:17.758 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:26:17.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:17.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:26:17.863 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:26:18.043 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:18.639 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:26:18.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:18.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:26:18.754 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:18.785 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:18.838 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:18.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:18.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:18.849 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:18 smithi138 ceph-mon[193606]: osdmap e393: 8 total, 8 up, 7 in 2023-09-23T23:26:18.849 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:18 smithi138 ceph-mon[193606]: failed to encode map e393 with expected crc 2023-09-23T23:26:18.849 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:18 smithi138 ceph-mon[193606]: failed to encode map e393 with expected crc 2023-09-23T23:26:18.849 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:18 smithi138 ceph-mon[193606]: failed to encode map e393 with expected crc 2023-09-23T23:26:18.849 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:18 smithi138 ceph-mon[193606]: failed to encode map e393 with expected crc 2023-09-23T23:26:18.850 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:18 smithi138 ceph-mon[193606]: pgmap v1146: 129 pgs: 3 active+remapped+backfilling, 7 peering, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 4.1 KiB/s rd, 11 KiB/s wr, 353 op/s; 3748/224460 objects degraded (1.670%); 48616/224460 objects misplaced (21.659%); 43 MiB/s, 1 keys/s, 691 objects/s recovering 2023-09-23T23:26:18.850 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:18 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 3748/225483 objects degraded (1.662%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:18.949 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:26:18.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:18 smithi110 ceph-mon[212939]: osdmap e393: 8 total, 8 up, 7 in 2023-09-23T23:26:18.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:18 smithi110 ceph-mon[212939]: failed to encode map e393 with expected crc 2023-09-23T23:26:18.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:18 smithi110 ceph-mon[212939]: failed to encode map e393 with expected crc 2023-09-23T23:26:18.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:18 smithi110 ceph-mon[212939]: failed to encode map e393 with expected crc 2023-09-23T23:26:18.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:18 smithi110 ceph-mon[212939]: failed to encode map e393 with expected crc 2023-09-23T23:26:18.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:18 smithi110 ceph-mon[212939]: pgmap v1146: 129 pgs: 3 active+remapped+backfilling, 7 peering, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 4.1 KiB/s rd, 11 KiB/s wr, 353 op/s; 3748/224460 objects degraded (1.670%); 48616/224460 objects misplaced (21.659%); 43 MiB/s, 1 keys/s, 691 objects/s recovering 2023-09-23T23:26:18.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:18 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 3748/225483 objects degraded (1.662%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:18.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:18 smithi110 ceph-mon[221102]: osdmap e393: 8 total, 8 up, 7 in 2023-09-23T23:26:18.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:18 smithi110 ceph-mon[221102]: failed to encode map e393 with expected crc 2023-09-23T23:26:18.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:18 smithi110 ceph-mon[221102]: failed to encode map e393 with expected crc 2023-09-23T23:26:18.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:18 smithi110 ceph-mon[221102]: failed to encode map e393 with expected crc 2023-09-23T23:26:18.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:18 smithi110 ceph-mon[221102]: failed to encode map e393 with expected crc 2023-09-23T23:26:18.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:18 smithi110 ceph-mon[221102]: pgmap v1146: 129 pgs: 3 active+remapped+backfilling, 7 peering, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 4.1 KiB/s rd, 11 KiB/s wr, 353 op/s; 3748/224460 objects degraded (1.670%); 48616/224460 objects misplaced (21.659%); 43 MiB/s, 1 keys/s, 691 objects/s recovering 2023-09-23T23:26:18.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:18 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 3748/225483 objects degraded (1.662%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:19.005 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:19.152 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:26:19.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:19.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:26:19.266 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:19.300 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:19.360 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:19.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:19.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:19.467 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:26:19.539 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:19.604 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:19 smithi138 ceph-mon[193606]: failed to encode map e394 with expected crc 2023-09-23T23:26:19.604 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:19 smithi138 ceph-mon[193606]: failed to encode map e394 with expected crc 2023-09-23T23:26:19.604 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:19 smithi138 ceph-mon[193606]: osdmap e394: 8 total, 8 up, 7 in 2023-09-23T23:26:19.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:19 smithi138 ceph-mon[193606]: failed to encode map e394 with expected crc 2023-09-23T23:26:19.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:19 smithi138 ceph-mon[193606]: failed to encode map e394 with expected crc 2023-09-23T23:26:19.630 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:26:19.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:19.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:26:19.741 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:19.773 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:19.833 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:19.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:19.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:19.940 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:19.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:19 smithi110 ceph-mon[212939]: failed to encode map e394 with expected crc 2023-09-23T23:26:19.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:19 smithi110 ceph-mon[212939]: failed to encode map e394 with expected crc 2023-09-23T23:26:19.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:19 smithi110 ceph-mon[212939]: osdmap e394: 8 total, 8 up, 7 in 2023-09-23T23:26:19.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:19 smithi110 ceph-mon[212939]: failed to encode map e394 with expected crc 2023-09-23T23:26:19.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:19 smithi110 ceph-mon[212939]: failed to encode map e394 with expected crc 2023-09-23T23:26:19.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:19 smithi110 ceph-mon[221102]: failed to encode map e394 with expected crc 2023-09-23T23:26:19.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:19 smithi110 ceph-mon[221102]: failed to encode map e394 with expected crc 2023-09-23T23:26:19.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:19 smithi110 ceph-mon[221102]: osdmap e394: 8 total, 8 up, 7 in 2023-09-23T23:26:19.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:19 smithi110 ceph-mon[221102]: failed to encode map e394 with expected crc 2023-09-23T23:26:19.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:19 smithi110 ceph-mon[221102]: failed to encode map e394 with expected crc 2023-09-23T23:26:19.973 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:20.030 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:20.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:20.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:20.137 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:26:20.204 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:20.285 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:26:20.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:20.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:26:20.393 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:20.421 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:20.465 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:20.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:20.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:20.570 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:20.597 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:20.644 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:20.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:20.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:20 smithi138 ceph-mon[193606]: 11.16 scrub starts 2023-09-23T23:26:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:20 smithi138 ceph-mon[193606]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:20 smithi138 ceph-mon[193606]: osdmap e395: 8 total, 8 up, 7 in 2023-09-23T23:26:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:20 smithi138 ceph-mon[193606]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:20 smithi138 ceph-mon[193606]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:20 smithi138 ceph-mon[193606]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:20 smithi138 ceph-mon[193606]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:20 smithi138 ceph-mon[193606]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:20 smithi138 ceph-mon[193606]: pgmap v1149: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 4.1 KiB/s rd, 11 KiB/s wr, 591 op/s; 3748/218520 objects degraded (1.715%); 45854/218520 objects misplaced (20.984%); 43 MiB/s, 691 objects/s recovering 2023-09-23T23:26:20.750 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:20.785 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:20.832 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:20.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:20.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:20.937 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:26:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[212939]: 11.16 scrub starts 2023-09-23T23:26:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[212939]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[212939]: osdmap e395: 8 total, 8 up, 7 in 2023-09-23T23:26:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[212939]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[212939]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[212939]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[212939]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[212939]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[212939]: pgmap v1149: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 4.1 KiB/s rd, 11 KiB/s wr, 591 op/s; 3748/218520 objects degraded (1.715%); 45854/218520 objects misplaced (20.984%); 43 MiB/s, 691 objects/s recovering 2023-09-23T23:26:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[221102]: 11.16 scrub starts 2023-09-23T23:26:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[221102]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[221102]: osdmap e395: 8 total, 8 up, 7 in 2023-09-23T23:26:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[221102]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[221102]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[221102]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[221102]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[221102]: failed to encode map e395 with expected crc 2023-09-23T23:26:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:20 smithi110 ceph-mon[221102]: pgmap v1149: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 4.1 KiB/s rd, 11 KiB/s wr, 591 op/s; 3748/218520 objects degraded (1.715%); 45854/218520 objects misplaced (20.984%); 43 MiB/s, 691 objects/s recovering 2023-09-23T23:26:20.964 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:21.403 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:26:21.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:21.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:26:21.522 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:21.560 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:21.624 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:26:21.627 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:21.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:21.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:21.734 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:21.772 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:21.838 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:21.843 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:21 smithi138 ceph-mon[193606]: failed to encode map e396 with expected crc 2023-09-23T23:26:21.844 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:21 smithi138 ceph-mon[193606]: osdmap e396: 8 total, 8 up, 7 in 2023-09-23T23:26:21.844 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:21 smithi138 ceph-mon[193606]: failed to encode map e396 with expected crc 2023-09-23T23:26:21.844 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:21 smithi138 ceph-mon[193606]: failed to encode map e396 with expected crc 2023-09-23T23:26:21.844 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:21 smithi138 ceph-mon[193606]: failed to encode map e396 with expected crc 2023-09-23T23:26:21.844 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:21 smithi138 ceph-mon[193606]: failed to encode map e396 with expected crc 2023-09-23T23:26:21.844 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:21 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:26:21.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:21.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:21.932 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:21 smithi110 ceph-mon[212939]: failed to encode map e396 with expected crc 2023-09-23T23:26:21.933 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:21 smithi110 ceph-mon[212939]: osdmap e396: 8 total, 8 up, 7 in 2023-09-23T23:26:21.933 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:21 smithi110 ceph-mon[212939]: failed to encode map e396 with expected crc 2023-09-23T23:26:21.933 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:21 smithi110 ceph-mon[212939]: failed to encode map e396 with expected crc 2023-09-23T23:26:21.933 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:21 smithi110 ceph-mon[212939]: failed to encode map e396 with expected crc 2023-09-23T23:26:21.933 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:21 smithi110 ceph-mon[212939]: failed to encode map e396 with expected crc 2023-09-23T23:26:21.933 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:21 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:26:21.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:21 smithi110 ceph-mon[221102]: failed to encode map e396 with expected crc 2023-09-23T23:26:21.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:21 smithi110 ceph-mon[221102]: osdmap e396: 8 total, 8 up, 7 in 2023-09-23T23:26:21.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:21 smithi110 ceph-mon[221102]: failed to encode map e396 with expected crc 2023-09-23T23:26:21.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:21 smithi110 ceph-mon[221102]: failed to encode map e396 with expected crc 2023-09-23T23:26:21.934 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:21 smithi110 ceph-mon[221102]: failed to encode map e396 with expected crc 2023-09-23T23:26:21.934 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:21 smithi110 ceph-mon[221102]: failed to encode map e396 with expected crc 2023-09-23T23:26:21.934 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:21 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:26:21.945 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:26:22.036 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:22.380 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:26:22.418 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:26:22.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:22.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:26:22.452 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:26:22.525 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:26:22.529 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:22.561 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:22.593 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:26:22.639 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:26:22.714 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:26:22.718 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:22.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:22.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:22.746 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:26:22.771 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:26:22.771 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:26:22.826 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:26:22.829 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:26:22.849 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:26:22.849 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:26:22.849 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:26:22.849 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:26:22.850 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:26:22.850 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:26:22.850 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:26:22.850 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:26:22.872 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:22 smithi110 ceph-mon[212939]: pgmap v1151: 129 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 3748/214848 objects degraded (1.744%); 42535/214848 objects misplaced (19.798%); 62 MiB/s, 996 objects/s recovering 2023-09-23T23:26:22.872 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:22 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3472653867' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:26:22.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:22 smithi110 ceph-mon[221102]: pgmap v1151: 129 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 3748/214848 objects degraded (1.744%); 42535/214848 objects misplaced (19.798%); 62 MiB/s, 996 objects/s recovering 2023-09-23T23:26:22.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:22 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3472653867' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:26:22.934 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:22.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:22 smithi138 ceph-mon[193606]: pgmap v1151: 129 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 3748/214848 objects degraded (1.744%); 42535/214848 objects misplaced (19.798%); 62 MiB/s, 996 objects/s recovering 2023-09-23T23:26:22.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:22 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3472653867' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:26:23.209 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:26:23.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:23.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:26:23.317 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:26:23.399 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:23.705 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:26:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:23 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1377245366' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:26:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:23 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 3748/214848 objects degraded (1.744%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:23.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:23 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1377245366' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:26:23.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:23 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 3748/214848 objects degraded (1.744%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:23.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:23 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1377245366' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:26:23.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:23 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 3748/214848 objects degraded (1.744%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:24 smithi138 ceph-mon[193606]: 11.16 scrub starts 2023-09-23T23:26:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:24 smithi138 ceph-mon[193606]: pgmap v1152: 129 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 3748/213777 objects degraded (1.753%); 42535/213777 objects misplaced (19.897%); 42 MiB/s, 675 objects/s recovering 2023-09-23T23:26:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:24 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:26:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:24 smithi138 ceph-mon[193606]: failed to encode map e397 with expected crc 2023-09-23T23:26:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:24 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1377245366' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:26:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:24 smithi138 ceph-mon[193606]: osdmap e397: 8 total, 8 up, 7 in 2023-09-23T23:26:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:24 smithi138 ceph-mon[193606]: failed to encode map e397 with expected crc 2023-09-23T23:26:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:24 smithi138 ceph-mon[193606]: failed to encode map e397 with expected crc 2023-09-23T23:26:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:24 smithi138 ceph-mon[193606]: failed to encode map e397 with expected crc 2023-09-23T23:26:24.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:24 smithi138 ceph-mon[193606]: failed to encode map e397 with expected crc 2023-09-23T23:26:24.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[212939]: 11.16 scrub starts 2023-09-23T23:26:24.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[212939]: pgmap v1152: 129 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 3748/213777 objects degraded (1.753%); 42535/213777 objects misplaced (19.897%); 42 MiB/s, 675 objects/s recovering 2023-09-23T23:26:24.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:26:24.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[212939]: failed to encode map e397 with expected crc 2023-09-23T23:26:24.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1377245366' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:26:24.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[212939]: osdmap e397: 8 total, 8 up, 7 in 2023-09-23T23:26:24.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[212939]: failed to encode map e397 with expected crc 2023-09-23T23:26:24.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[212939]: failed to encode map e397 with expected crc 2023-09-23T23:26:24.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[212939]: failed to encode map e397 with expected crc 2023-09-23T23:26:24.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[212939]: failed to encode map e397 with expected crc 2023-09-23T23:26:24.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[221102]: 11.16 scrub starts 2023-09-23T23:26:24.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[221102]: pgmap v1152: 129 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 3748/213777 objects degraded (1.753%); 42535/213777 objects misplaced (19.897%); 42 MiB/s, 675 objects/s recovering 2023-09-23T23:26:24.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:26:24.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[221102]: failed to encode map e397 with expected crc 2023-09-23T23:26:24.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1377245366' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:26:24.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[221102]: osdmap e397: 8 total, 8 up, 7 in 2023-09-23T23:26:24.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[221102]: failed to encode map e397 with expected crc 2023-09-23T23:26:24.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[221102]: failed to encode map e397 with expected crc 2023-09-23T23:26:24.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[221102]: failed to encode map e397 with expected crc 2023-09-23T23:26:24.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:24 smithi110 ceph-mon[221102]: failed to encode map e397 with expected crc 2023-09-23T23:26:26.809 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:26 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:26.810 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:26 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:26.810 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:26 smithi110 ceph-mon[212939]: pgmap v1154: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 3748/206631 objects degraded (1.814%); 39553/206631 objects misplaced (19.142%); 40 MiB/s, 647 objects/s recovering 2023-09-23T23:26:26.810 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:26:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:26:26] "GET /metrics HTTP/1.1" 200 36603 "" "Prometheus/2.43.0" 2023-09-23T23:26:26.810 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:26 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:26.810 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:26 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:26.810 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:26 smithi110 ceph-mon[221102]: pgmap v1154: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 3748/206631 objects degraded (1.814%); 39553/206631 objects misplaced (19.142%); 40 MiB/s, 647 objects/s recovering 2023-09-23T23:26:26.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:26 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:26 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:26 smithi138 ceph-mon[193606]: pgmap v1154: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 3748/206631 objects degraded (1.814%); 39553/206631 objects misplaced (19.142%); 40 MiB/s, 647 objects/s recovering 2023-09-23T23:26:29.249 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:28 smithi110 ceph-mon[221102]: pgmap v1155: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 3748/204888 objects degraded (1.829%); 38904/204888 objects misplaced (18.988%); 30 MiB/s, 487 objects/s recovering 2023-09-23T23:26:29.249 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:29.249 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:26:29.249 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:28 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 3748/204888 objects degraded (1.829%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:29.252 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:28 smithi110 ceph-mon[212939]: pgmap v1155: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 3748/204888 objects degraded (1.829%); 38904/204888 objects misplaced (18.988%); 30 MiB/s, 487 objects/s recovering 2023-09-23T23:26:29.252 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:29.252 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:26:29.253 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:28 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 3748/204888 objects degraded (1.829%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:29.276 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:26:29.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:29.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:26:29.290 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:26:29.389 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:26:29.428 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:28 smithi138 ceph-mon[193606]: pgmap v1155: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 3748/204888 objects degraded (1.829%); 38904/204888 objects misplaced (18.988%); 30 MiB/s, 487 objects/s recovering 2023-09-23T23:26:29.429 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:29.429 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:26:29.429 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:28 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 3748/204888 objects degraded (1.829%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:29.438 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:29.491 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:26:29.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:29.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:26:29.596 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:26:29.739 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:30.126 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:26:30.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:30.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:26:30.239 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:26:30.337 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:30.572 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:26:30.624 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:26:30.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:30.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:26:30.743 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:30.775 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:30.833 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:30.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:30.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:30.940 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:26:31.011 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:31.042 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T23:26:31.093 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:26:31.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:31.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:26:31.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:31 smithi138 ceph-mon[193606]: pgmap v1156: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 3748/198291 objects degraded (1.890%); 37829/198291 objects misplaced (19.078%); 12 MiB/s, 192 objects/s recovering 2023-09-23T23:26:31.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:31 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2471804516' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:26:31.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:31 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:26:31.197 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:31.223 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:31.234 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:26:31.268 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:31.269 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:31 smithi110 ceph-mon[221102]: pgmap v1156: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 3748/198291 objects degraded (1.890%); 37829/198291 objects misplaced (19.078%); 12 MiB/s, 192 objects/s recovering 2023-09-23T23:26:31.269 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:31 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2471804516' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:26:31.269 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:31 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:26:31.270 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:31 smithi110 ceph-mon[212939]: pgmap v1156: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 3748/198291 objects degraded (1.890%); 37829/198291 objects misplaced (19.078%); 12 MiB/s, 192 objects/s recovering 2023-09-23T23:26:31.271 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:31 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2471804516' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:26:31.271 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:31 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:26:31.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:31.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:31.328 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:26:31.374 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:26:31.497 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:32.076 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:26:32.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:32.080 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:32 smithi110 ceph-mon[212939]: failed to encode map e398 with expected crc 2023-09-23T23:26:32.080 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:32 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:26:32.080 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:32 smithi110 ceph-mon[212939]: osdmap e398: 8 total, 8 up, 7 in 2023-09-23T23:26:32.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:26:32.081 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:32 smithi110 ceph-mon[221102]: failed to encode map e398 with expected crc 2023-09-23T23:26:32.081 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:32 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:26:32.082 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:32 smithi110 ceph-mon[221102]: osdmap e398: 8 total, 8 up, 7 in 2023-09-23T23:26:32.183 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:26:32.216 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:32.270 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:26:32.274 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:32 smithi138 ceph-mon[193606]: failed to encode map e398 with expected crc 2023-09-23T23:26:32.274 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:32 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:26:32.274 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:32 smithi138 ceph-mon[193606]: osdmap e398: 8 total, 8 up, 7 in 2023-09-23T23:26:32.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:32.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:26:32.376 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:32.403 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:32.461 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:32.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:32.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:32.565 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:26:32.593 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:32.640 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:26:32.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:32.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:26:32.746 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:26:32.782 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:33.028 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[212939]: pgmap v1158: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 3748/194094 objects degraded (1.931%); 35555/194094 objects misplaced (18.318%); 23 MiB/s, 372 objects/s recovering 2023-09-23T23:26:33.029 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[212939]: failed to encode map e399 with expected crc 2023-09-23T23:26:33.029 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[212939]: osdmap e399: 8 total, 8 up, 7 in 2023-09-23T23:26:33.029 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[212939]: failed to encode map e399 with expected crc 2023-09-23T23:26:33.029 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[212939]: failed to encode map e399 with expected crc 2023-09-23T23:26:33.029 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[212939]: failed to encode map e399 with expected crc 2023-09-23T23:26:33.029 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[212939]: 11.18 continuing backfill to osd.3 from (355'1400,397'7759] 11:1e4739d0:::benchmark_data_smithi110_433746_object61007:head to 397'7759 2023-09-23T23:26:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:33 smithi138 ceph-mon[193606]: pgmap v1158: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 3748/194094 objects degraded (1.931%); 35555/194094 objects misplaced (18.318%); 23 MiB/s, 372 objects/s recovering 2023-09-23T23:26:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:33 smithi138 ceph-mon[193606]: failed to encode map e399 with expected crc 2023-09-23T23:26:33.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:33 smithi138 ceph-mon[193606]: osdmap e399: 8 total, 8 up, 7 in 2023-09-23T23:26:33.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:33 smithi138 ceph-mon[193606]: failed to encode map e399 with expected crc 2023-09-23T23:26:33.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:33 smithi138 ceph-mon[193606]: failed to encode map e399 with expected crc 2023-09-23T23:26:33.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:33 smithi138 ceph-mon[193606]: failed to encode map e399 with expected crc 2023-09-23T23:26:33.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:33 smithi138 ceph-mon[193606]: 11.18 continuing backfill to osd.3 from (355'1400,397'7759] 11:1e4739d0:::benchmark_data_smithi110_433746_object61007:head to 397'7759 2023-09-23T23:26:33.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:33 smithi138 ceph-mon[193606]: 11.18 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7759 2023-09-23T23:26:33.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:33 smithi138 ceph-mon[193606]: failed to encode map e399 with expected crc 2023-09-23T23:26:33.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:33 smithi138 ceph-mon[193606]: 11.5 continuing backfill to osd.3 from (355'1400,397'7792] MIN to 397'7792 2023-09-23T23:26:33.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:33 smithi138 ceph-mon[193606]: 11.5 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7792 2023-09-23T23:26:33.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:33 smithi138 ceph-mon[193606]: 11.15 continuing backfill to osd.3 from (355'1400,397'7892] MIN to 397'7892 2023-09-23T23:26:33.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:33 smithi138 ceph-mon[193606]: 11.15 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7892 2023-09-23T23:26:33.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:33 smithi138 ceph-mon[193606]: failed to encode map e399 with expected crc 2023-09-23T23:26:33.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[212939]: 11.18 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7759 2023-09-23T23:26:33.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[212939]: failed to encode map e399 with expected crc 2023-09-23T23:26:33.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[212939]: 11.5 continuing backfill to osd.3 from (355'1400,397'7792] MIN to 397'7792 2023-09-23T23:26:33.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[212939]: 11.5 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7792 2023-09-23T23:26:33.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[212939]: 11.15 continuing backfill to osd.3 from (355'1400,397'7892] MIN to 397'7892 2023-09-23T23:26:33.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[212939]: 11.15 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7892 2023-09-23T23:26:33.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[212939]: failed to encode map e399 with expected crc 2023-09-23T23:26:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[221102]: pgmap v1158: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 3748/194094 objects degraded (1.931%); 35555/194094 objects misplaced (18.318%); 23 MiB/s, 372 objects/s recovering 2023-09-23T23:26:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[221102]: failed to encode map e399 with expected crc 2023-09-23T23:26:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[221102]: osdmap e399: 8 total, 8 up, 7 in 2023-09-23T23:26:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[221102]: failed to encode map e399 with expected crc 2023-09-23T23:26:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[221102]: failed to encode map e399 with expected crc 2023-09-23T23:26:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[221102]: failed to encode map e399 with expected crc 2023-09-23T23:26:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[221102]: 11.18 continuing backfill to osd.3 from (355'1400,397'7759] 11:1e4739d0:::benchmark_data_smithi110_433746_object61007:head to 397'7759 2023-09-23T23:26:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[221102]: 11.18 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7759 2023-09-23T23:26:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[221102]: failed to encode map e399 with expected crc 2023-09-23T23:26:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[221102]: 11.5 continuing backfill to osd.3 from (355'1400,397'7792] MIN to 397'7792 2023-09-23T23:26:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[221102]: 11.5 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7792 2023-09-23T23:26:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[221102]: 11.15 continuing backfill to osd.3 from (355'1400,397'7892] MIN to 397'7892 2023-09-23T23:26:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[221102]: 11.15 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7892 2023-09-23T23:26:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:33 smithi110 ceph-mon[221102]: failed to encode map e399 with expected crc 2023-09-23T23:26:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:34 smithi138 ceph-mon[193606]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:34 smithi138 ceph-mon[193606]: osdmap e400: 8 total, 8 up, 7 in 2023-09-23T23:26:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:34 smithi138 ceph-mon[193606]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:34 smithi138 ceph-mon[193606]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:34 smithi138 ceph-mon[193606]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:34 smithi138 ceph-mon[193606]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:34 smithi138 ceph-mon[193606]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:34 smithi138 ceph-mon[193606]: 11.8 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7760 2023-09-23T23:26:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:34 smithi138 ceph-mon[193606]: 11.7 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7770 2023-09-23T23:26:34.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:34 smithi138 ceph-mon[193606]: 11.17 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7812 2023-09-23T23:26:34.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:34 smithi138 ceph-mon[193606]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:34 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 3748/194094 objects degraded (1.931%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:34.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:34 smithi138 ceph-mon[193606]: osdmap e401: 8 total, 8 up, 7 in 2023-09-23T23:26:34.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:34 smithi138 ceph-mon[193606]: failed to encode map e401 with expected crc 2023-09-23T23:26:34.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:34 smithi138 ceph-mon[193606]: failed to encode map e401 with expected crc 2023-09-23T23:26:34.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[221102]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[221102]: osdmap e400: 8 total, 8 up, 7 in 2023-09-23T23:26:34.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[221102]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[221102]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[221102]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[221102]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[221102]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[221102]: 11.8 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7760 2023-09-23T23:26:34.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[221102]: 11.7 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7770 2023-09-23T23:26:34.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[221102]: 11.17 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7812 2023-09-23T23:26:34.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[221102]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 3748/194094 objects degraded (1.931%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:34.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[221102]: osdmap e401: 8 total, 8 up, 7 in 2023-09-23T23:26:34.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[221102]: failed to encode map e401 with expected crc 2023-09-23T23:26:34.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[221102]: failed to encode map e401 with expected crc 2023-09-23T23:26:34.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[212939]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[212939]: osdmap e400: 8 total, 8 up, 7 in 2023-09-23T23:26:34.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[212939]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[212939]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[212939]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[212939]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[212939]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[212939]: 11.8 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7760 2023-09-23T23:26:34.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[212939]: 11.7 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7770 2023-09-23T23:26:34.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[212939]: 11.17 starting backfill to osd.5 from (0'0,0'0] MAX to 397'7812 2023-09-23T23:26:34.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[212939]: failed to encode map e400 with expected crc 2023-09-23T23:26:34.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 3748/194094 objects degraded (1.931%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:26:34.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[212939]: osdmap e401: 8 total, 8 up, 7 in 2023-09-23T23:26:34.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[212939]: failed to encode map e401 with expected crc 2023-09-23T23:26:34.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:34 smithi110 ceph-mon[212939]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:35 smithi138 ceph-mon[193606]: failed to encode map e400 with expected crc 2023-09-23T23:26:35.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:35 smithi138 ceph-mon[193606]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:35 smithi138 ceph-mon[193606]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:35 smithi138 ceph-mon[193606]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:35 smithi138 ceph-mon[193606]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:35 smithi138 ceph-mon[193606]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:35 smithi138 ceph-mon[193606]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:35 smithi138 ceph-mon[193606]: pgmap v1162: 129 pgs: 1 activating+undersized+remapped, 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 113 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 411 op/s; 1555/193353 objects degraded (0.804%); 37471/193353 objects misplaced (19.380%); 21 MiB/s, 338 objects/s recovering 2023-09-23T23:26:35.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:35 smithi138 ceph-mon[193606]: osdmap e402: 8 total, 8 up, 7 in 2023-09-23T23:26:35.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:35 smithi138 ceph-mon[193606]: failed to encode map e402 with expected crc 2023-09-23T23:26:35.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:35 smithi138 ceph-mon[193606]: failed to encode map e402 with expected crc 2023-09-23T23:26:35.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:35 smithi138 ceph-mon[193606]: failed to encode map e402 with expected crc 2023-09-23T23:26:35.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[221102]: failed to encode map e400 with expected crc 2023-09-23T23:26:35.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[221102]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[221102]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[221102]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[221102]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[221102]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[221102]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[221102]: pgmap v1162: 129 pgs: 1 activating+undersized+remapped, 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 113 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 411 op/s; 1555/193353 objects degraded (0.804%); 37471/193353 objects misplaced (19.380%); 21 MiB/s, 338 objects/s recovering 2023-09-23T23:26:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[221102]: osdmap e402: 8 total, 8 up, 7 in 2023-09-23T23:26:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[221102]: failed to encode map e402 with expected crc 2023-09-23T23:26:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[221102]: failed to encode map e402 with expected crc 2023-09-23T23:26:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[221102]: failed to encode map e402 with expected crc 2023-09-23T23:26:35.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[212939]: failed to encode map e400 with expected crc 2023-09-23T23:26:35.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[212939]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[212939]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[212939]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[212939]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[212939]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[212939]: failed to encode map e401 with expected crc 2023-09-23T23:26:35.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[212939]: pgmap v1162: 129 pgs: 1 activating+undersized+remapped, 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 113 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 411 op/s; 1555/193353 objects degraded (0.804%); 37471/193353 objects misplaced (19.380%); 21 MiB/s, 338 objects/s recovering 2023-09-23T23:26:35.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[212939]: osdmap e402: 8 total, 8 up, 7 in 2023-09-23T23:26:35.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[212939]: failed to encode map e402 with expected crc 2023-09-23T23:26:35.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[212939]: failed to encode map e402 with expected crc 2023-09-23T23:26:35.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:35 smithi110 ceph-mon[212939]: failed to encode map e402 with expected crc 2023-09-23T23:26:35.701 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:26:35.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:35.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:26:35.808 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:36.008 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:36.091 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:36.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:36.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:36.199 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T23:26:36.203 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:26:36.238 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:36.241 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:26:36.300 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:26:36.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:36 smithi138 ceph-mon[193606]: failed to encode map e402 with expected crc 2023-09-23T23:26:36.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:36 smithi138 ceph-mon[193606]: failed to encode map e402 with expected crc 2023-09-23T23:26:36.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:36 smithi138 ceph-mon[193606]: failed to encode map e402 with expected crc 2023-09-23T23:26:36.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:36 smithi138 ceph-mon[193606]: failed to encode map e402 with expected crc 2023-09-23T23:26:36.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:36 smithi138 ceph-mon[193606]: failed to encode map e402 with expected crc 2023-09-23T23:26:36.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:36.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:26:36.407 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:36.441 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:36.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:36 smithi110 ceph-mon[221102]: failed to encode map e402 with expected crc 2023-09-23T23:26:36.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:36 smithi110 ceph-mon[221102]: failed to encode map e402 with expected crc 2023-09-23T23:26:36.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:36 smithi110 ceph-mon[221102]: failed to encode map e402 with expected crc 2023-09-23T23:26:36.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:36 smithi110 ceph-mon[221102]: failed to encode map e402 with expected crc 2023-09-23T23:26:36.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:36 smithi110 ceph-mon[221102]: failed to encode map e402 with expected crc 2023-09-23T23:26:36.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:36 smithi110 ceph-mon[212939]: failed to encode map e402 with expected crc 2023-09-23T23:26:36.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:36 smithi110 ceph-mon[212939]: failed to encode map e402 with expected crc 2023-09-23T23:26:36.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:36 smithi110 ceph-mon[212939]: failed to encode map e402 with expected crc 2023-09-23T23:26:36.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:36 smithi110 ceph-mon[212939]: failed to encode map e402 with expected crc 2023-09-23T23:26:36.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:36 smithi110 ceph-mon[212939]: failed to encode map e402 with expected crc 2023-09-23T23:26:36.504 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:26:36.504 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:36.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:36.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:36.615 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:36.652 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:36.724 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:36.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:36.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:36.829 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:26:36.872 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:36.927 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:26:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:26:36] "GET /metrics HTTP/1.1" 200 36607 "" "Prometheus/2.43.0" 2023-09-23T23:26:36.936 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:26:36.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:36.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:26:37.048 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:26:37.115 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:37.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:37 smithi110 ceph-mon[212939]: pgmap v1164: 129 pgs: 1 active+recovery_wait+remapped, 5 peering, 1 activating+undersized+remapped, 2 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 105 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 421 op/s; 4998/189141 objects degraded (2.642%); 42997/189141 objects misplaced (22.733%); 34 MiB/s, 537 objects/s recovering 2023-09-23T23:26:37.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:37 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:26:37.425 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:37 smithi110 ceph-mon[221102]: pgmap v1164: 129 pgs: 1 active+recovery_wait+remapped, 5 peering, 1 activating+undersized+remapped, 2 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 105 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 421 op/s; 4998/189141 objects degraded (2.642%); 42997/189141 objects misplaced (22.733%); 34 MiB/s, 537 objects/s recovering 2023-09-23T23:26:37.425 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:37 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:26:37.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:37 smithi138 ceph-mon[193606]: pgmap v1164: 129 pgs: 1 active+recovery_wait+remapped, 5 peering, 1 activating+undersized+remapped, 2 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 105 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 421 op/s; 4998/189141 objects degraded (2.642%); 42997/189141 objects misplaced (22.733%); 34 MiB/s, 537 objects/s recovering 2023-09-23T23:26:37.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:37 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:26:37.764 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:26:37.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:37.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:26:37.874 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:26:37.933 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:26:37.933 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:26:37.972 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:38.091 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:26:38.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:38.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:26:38.201 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:26:38.281 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:38.649 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:26:38.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:38.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:26:38.759 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:26:38.793 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:39 smithi138 ceph-mon[193606]: pgmap v1165: 129 pgs: 1 active+recovery_wait+remapped, 5 peering, 1 activating+undersized+remapped, 3 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 105 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 7.4 KiB/s rd, 19 KiB/s wr, 363 op/s; 4998/188046 objects degraded (2.658%); 41090/188046 objects misplaced (21.851%); 41 MiB/s, 659 objects/s recovering 2023-09-23T23:26:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:39 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 4998/188046 objects degraded (2.658%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:26:39.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:39 smithi110 ceph-mon[212939]: pgmap v1165: 129 pgs: 1 active+recovery_wait+remapped, 5 peering, 1 activating+undersized+remapped, 3 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 105 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 7.4 KiB/s rd, 19 KiB/s wr, 363 op/s; 4998/188046 objects degraded (2.658%); 41090/188046 objects misplaced (21.851%); 41 MiB/s, 659 objects/s recovering 2023-09-23T23:26:39.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:39 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 4998/188046 objects degraded (2.658%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:26:39.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:39 smithi110 ceph-mon[221102]: pgmap v1165: 129 pgs: 1 active+recovery_wait+remapped, 5 peering, 1 activating+undersized+remapped, 3 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 105 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 7.4 KiB/s rd, 19 KiB/s wr, 363 op/s; 4998/188046 objects degraded (2.658%); 41090/188046 objects misplaced (21.851%); 41 MiB/s, 659 objects/s recovering 2023-09-23T23:26:39.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:39 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 4998/188046 objects degraded (2.658%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:26:40.132 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:40 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T23:26:40.132 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:40 smithi110 ceph-mon[212939]: osdmap e403: 8 total, 8 up, 7 in 2023-09-23T23:26:40.132 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:40 smithi110 ceph-mon[212939]: failed to encode map e403 with expected crc 2023-09-23T23:26:40.329 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:26:40.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:40.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:26:40.438 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:26:40.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:40 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T23:26:40.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:40 smithi138 ceph-mon[193606]: osdmap e403: 8 total, 8 up, 7 in 2023-09-23T23:26:40.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:40 smithi138 ceph-mon[193606]: failed to encode map e403 with expected crc 2023-09-23T23:26:40.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:40 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T23:26:40.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:40 smithi110 ceph-mon[221102]: osdmap e403: 8 total, 8 up, 7 in 2023-09-23T23:26:40.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:40 smithi110 ceph-mon[221102]: failed to encode map e403 with expected crc 2023-09-23T23:26:40.496 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:41 smithi138 ceph-mon[193606]: pgmap v1166: 129 pgs: 1 active+recovery_wait+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 111 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 6.3 KiB/s rd, 17 KiB/s wr, 676 op/s; 4998/180708 objects degraded (2.766%); 37167/180708 objects misplaced (20.567%); 64 MiB/s, 1.02k objects/s recovering 2023-09-23T23:26:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:41 smithi138 ceph-mon[193606]: failed to encode map e403 with expected crc 2023-09-23T23:26:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:41 smithi138 ceph-mon[193606]: failed to encode map e404 with expected crc 2023-09-23T23:26:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:41 smithi138 ceph-mon[193606]: osdmap e404: 8 total, 8 up, 7 in 2023-09-23T23:26:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:41 smithi138 ceph-mon[193606]: failed to encode map e404 with expected crc 2023-09-23T23:26:41.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:41 smithi110 ceph-mon[212939]: pgmap v1166: 129 pgs: 1 active+recovery_wait+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 111 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 6.3 KiB/s rd, 17 KiB/s wr, 676 op/s; 4998/180708 objects degraded (2.766%); 37167/180708 objects misplaced (20.567%); 64 MiB/s, 1.02k objects/s recovering 2023-09-23T23:26:41.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:41 smithi110 ceph-mon[212939]: failed to encode map e403 with expected crc 2023-09-23T23:26:41.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:41 smithi110 ceph-mon[212939]: failed to encode map e404 with expected crc 2023-09-23T23:26:41.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:41 smithi110 ceph-mon[212939]: osdmap e404: 8 total, 8 up, 7 in 2023-09-23T23:26:41.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:41 smithi110 ceph-mon[212939]: failed to encode map e404 with expected crc 2023-09-23T23:26:41.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:41 smithi110 ceph-mon[221102]: pgmap v1166: 129 pgs: 1 active+recovery_wait+remapped, 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 111 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 6.3 KiB/s rd, 17 KiB/s wr, 676 op/s; 4998/180708 objects degraded (2.766%); 37167/180708 objects misplaced (20.567%); 64 MiB/s, 1.02k objects/s recovering 2023-09-23T23:26:41.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:41 smithi110 ceph-mon[221102]: failed to encode map e403 with expected crc 2023-09-23T23:26:41.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:41 smithi110 ceph-mon[221102]: failed to encode map e404 with expected crc 2023-09-23T23:26:41.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:41 smithi110 ceph-mon[221102]: osdmap e404: 8 total, 8 up, 7 in 2023-09-23T23:26:41.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:41 smithi110 ceph-mon[221102]: failed to encode map e404 with expected crc 2023-09-23T23:26:41.580 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:26:41.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:41.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:26:41.689 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:41.700 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:26:41.722 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:41.750 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:26:41.788 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:41.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:41.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:41.894 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:26:41.926 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:42.152 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:42 smithi138 ceph-mon[193606]: failed to encode map e404 with expected crc 2023-09-23T23:26:42.153 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:42 smithi138 ceph-mon[193606]: failed to encode map e404 with expected crc 2023-09-23T23:26:42.153 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:42 smithi138 ceph-mon[193606]: failed to encode map e404 with expected crc 2023-09-23T23:26:42.153 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:42 smithi138 ceph-mon[193606]: failed to encode map e405 with expected crc 2023-09-23T23:26:42.153 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:42 smithi138 ceph-mon[193606]: osdmap e405: 8 total, 8 up, 7 in 2023-09-23T23:26:42.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:42 smithi110 ceph-mon[212939]: failed to encode map e404 with expected crc 2023-09-23T23:26:42.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:42 smithi110 ceph-mon[212939]: failed to encode map e404 with expected crc 2023-09-23T23:26:42.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:42 smithi110 ceph-mon[212939]: failed to encode map e404 with expected crc 2023-09-23T23:26:42.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:42 smithi110 ceph-mon[212939]: failed to encode map e405 with expected crc 2023-09-23T23:26:42.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:42 smithi110 ceph-mon[212939]: osdmap e405: 8 total, 8 up, 7 in 2023-09-23T23:26:42.188 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:26:42.189 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:42 smithi110 ceph-mon[221102]: failed to encode map e404 with expected crc 2023-09-23T23:26:42.189 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:42 smithi110 ceph-mon[221102]: failed to encode map e404 with expected crc 2023-09-23T23:26:42.189 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:42 smithi110 ceph-mon[221102]: failed to encode map e404 with expected crc 2023-09-23T23:26:42.189 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:42 smithi110 ceph-mon[221102]: failed to encode map e405 with expected crc 2023-09-23T23:26:42.189 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:42 smithi110 ceph-mon[221102]: osdmap e405: 8 total, 8 up, 7 in 2023-09-23T23:26:42.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:42.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:26:42.297 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:26:42.377 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:42.948 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:26:43.002 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:26:43.037 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:26:43.046 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:26:43.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:43.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:26:43.092 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:26:43.133 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:26:43.154 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:43 smithi110 ceph-mon[221102]: pgmap v1169: 129 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 111 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 4998/176958 objects degraded (2.824%); 35084/176958 objects misplaced (19.826%); 59 MiB/s, 951 objects/s recovering 2023-09-23T23:26:43.155 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:43 smithi110 ceph-mon[221102]: failed to encode map e405 with expected crc 2023-09-23T23:26:43.155 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:43.156 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:43 smithi110 ceph-mon[212939]: pgmap v1169: 129 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 111 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 4998/176958 objects degraded (2.824%); 35084/176958 objects misplaced (19.826%); 59 MiB/s, 951 objects/s recovering 2023-09-23T23:26:43.156 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:43 smithi110 ceph-mon[212939]: failed to encode map e405 with expected crc 2023-09-23T23:26:43.156 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:43 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2763977251' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:26:43.156 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:43 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2319124106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:26:43.156 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:26:43.175 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:26:43.190 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:26:43.194 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:43.221 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:26:43.244 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:43.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:43.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:43.262 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:26:43.263 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:26:43.263 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:26:43.263 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:26:43.263 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:26:43.263 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:26:43.263 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:26:43.263 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:26:43.263 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:26:43.351 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:26:43.383 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:43 smithi138 ceph-mon[193606]: pgmap v1169: 129 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 111 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 4998/176958 objects degraded (2.824%); 35084/176958 objects misplaced (19.826%); 59 MiB/s, 951 objects/s recovering 2023-09-23T23:26:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:43 smithi138 ceph-mon[193606]: failed to encode map e405 with expected crc 2023-09-23T23:26:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:43 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2763977251' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:26:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:43 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2319124106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:26:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:26:43.449 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:26:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:43 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2763977251' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:26:43.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:43 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2319124106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:26:43.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:26:43.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:43.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:26:43.556 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:43.602 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:43.670 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:43.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:43.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:43.779 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:43.815 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:43.884 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:43.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:43.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:43.992 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:26:44.114 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:44.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:44 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:26:44.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:44 smithi138 ceph-mon[193606]: failed to encode map e406 with expected crc 2023-09-23T23:26:44.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:44 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2763977251' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:26:44.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:44 smithi138 ceph-mon[193606]: osdmap e406: 8 total, 8 up, 7 in 2023-09-23T23:26:44.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:44 smithi138 ceph-mon[193606]: failed to encode map e406 with expected crc 2023-09-23T23:26:44.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:44 smithi138 ceph-mon[193606]: failed to encode map e406 with expected crc 2023-09-23T23:26:44.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:44 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 4998/176958 objects degraded (2.824%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:26:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:44 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:26:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:44 smithi110 ceph-mon[212939]: failed to encode map e406 with expected crc 2023-09-23T23:26:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:44 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2763977251' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:26:44.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:44 smithi110 ceph-mon[212939]: osdmap e406: 8 total, 8 up, 7 in 2023-09-23T23:26:44.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:44 smithi110 ceph-mon[212939]: failed to encode map e406 with expected crc 2023-09-23T23:26:44.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:44 smithi110 ceph-mon[212939]: failed to encode map e406 with expected crc 2023-09-23T23:26:44.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:44 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 4998/176958 objects degraded (2.824%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:26:44.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:44 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:26:44.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:44 smithi110 ceph-mon[221102]: failed to encode map e406 with expected crc 2023-09-23T23:26:44.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:44 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2763977251' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:26:44.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:44 smithi110 ceph-mon[221102]: osdmap e406: 8 total, 8 up, 7 in 2023-09-23T23:26:44.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:44 smithi110 ceph-mon[221102]: failed to encode map e406 with expected crc 2023-09-23T23:26:44.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:44 smithi110 ceph-mon[221102]: failed to encode map e406 with expected crc 2023-09-23T23:26:44.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:44 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 4998/176958 objects degraded (2.824%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:26:44.605 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:26:44.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:44.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:26:44.721 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:26:44.810 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:45.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:45 smithi138 ceph-mon[193606]: pgmap v1172: 129 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 111 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 4998/175818 objects degraded (2.843%); 34760/175818 objects misplaced (19.770%); 18 MiB/s, 289 objects/s recovering 2023-09-23T23:26:45.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:45 smithi138 ceph-mon[193606]: failed to encode map e407 with expected crc 2023-09-23T23:26:45.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:45 smithi138 ceph-mon[193606]: osdmap e407: 8 total, 8 up, 7 in 2023-09-23T23:26:45.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:45 smithi138 ceph-mon[193606]: failed to encode map e407 with expected crc 2023-09-23T23:26:45.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:45 smithi138 ceph-mon[193606]: failed to encode map e407 with expected crc 2023-09-23T23:26:45.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:45 smithi138 ceph-mon[193606]: failed to encode map e407 with expected crc 2023-09-23T23:26:45.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:45 smithi110 ceph-mon[212939]: pgmap v1172: 129 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 111 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 4998/175818 objects degraded (2.843%); 34760/175818 objects misplaced (19.770%); 18 MiB/s, 289 objects/s recovering 2023-09-23T23:26:45.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:45 smithi110 ceph-mon[212939]: failed to encode map e407 with expected crc 2023-09-23T23:26:45.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:45 smithi110 ceph-mon[212939]: osdmap e407: 8 total, 8 up, 7 in 2023-09-23T23:26:45.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:45 smithi110 ceph-mon[212939]: failed to encode map e407 with expected crc 2023-09-23T23:26:45.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:45 smithi110 ceph-mon[212939]: failed to encode map e407 with expected crc 2023-09-23T23:26:45.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:45 smithi110 ceph-mon[212939]: failed to encode map e407 with expected crc 2023-09-23T23:26:45.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:45 smithi110 ceph-mon[221102]: pgmap v1172: 129 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 111 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 4998/175818 objects degraded (2.843%); 34760/175818 objects misplaced (19.770%); 18 MiB/s, 289 objects/s recovering 2023-09-23T23:26:45.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:45 smithi110 ceph-mon[221102]: failed to encode map e407 with expected crc 2023-09-23T23:26:45.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:45 smithi110 ceph-mon[221102]: osdmap e407: 8 total, 8 up, 7 in 2023-09-23T23:26:45.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:45 smithi110 ceph-mon[221102]: failed to encode map e407 with expected crc 2023-09-23T23:26:45.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:45 smithi110 ceph-mon[221102]: failed to encode map e407 with expected crc 2023-09-23T23:26:45.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:45 smithi110 ceph-mon[221102]: failed to encode map e407 with expected crc 2023-09-23T23:26:46.550 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:26:46.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:46.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:26:46.585 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:26:46.657 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:26:46.698 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:46.769 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:26:46.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:46.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:26:46.862 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:26:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:26:46] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.43.0" 2023-09-23T23:26:46.876 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:46.904 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:46.978 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:46.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:46.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:47.085 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:26:47.114 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:47.163 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:26:47.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:47.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:26:47.268 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:26:47.298 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:47.353 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:26:47.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:47.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:26:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:47 smithi138 ceph-mon[193606]: pgmap v1174: 129 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 113 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 4998/169929 objects degraded (2.941%); 30637/169929 objects misplaced (18.029%); 47 MiB/s, 747 objects/s recovering 2023-09-23T23:26:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:47 smithi138 ceph-mon[193606]: osdmap e408: 8 total, 8 up, 7 in 2023-09-23T23:26:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:47 smithi138 ceph-mon[193606]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:47 smithi138 ceph-mon[193606]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:47 smithi138 ceph-mon[193606]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:47 smithi138 ceph-mon[193606]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:47 smithi138 ceph-mon[193606]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:47 smithi138 ceph-mon[193606]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:47 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:47 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:47.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[221102]: pgmap v1174: 129 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 113 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 4998/169929 objects degraded (2.941%); 30637/169929 objects misplaced (18.029%); 47 MiB/s, 747 objects/s recovering 2023-09-23T23:26:47.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[221102]: osdmap e408: 8 total, 8 up, 7 in 2023-09-23T23:26:47.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[221102]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[221102]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[221102]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[221102]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[221102]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[221102]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:47.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:47.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[212939]: pgmap v1174: 129 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 113 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 4998/169929 objects degraded (2.941%); 30637/169929 objects misplaced (18.029%); 47 MiB/s, 747 objects/s recovering 2023-09-23T23:26:47.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[212939]: osdmap e408: 8 total, 8 up, 7 in 2023-09-23T23:26:47.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[212939]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[212939]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[212939]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[212939]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[212939]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[212939]: failed to encode map e408 with expected crc 2023-09-23T23:26:47.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:47.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:47.461 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:47.493 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:47.556 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:47.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:47.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:47.661 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:26:47.762 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:47.877 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:26:47.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:47.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:26:47.982 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:48.016 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:48.092 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:48.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:48.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:48.202 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:26:48.241 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:48.317 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:26:48.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:48.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:26:48.324 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:48 smithi138 ceph-mon[193606]: failed to encode map e409 with expected crc 2023-09-23T23:26:48.324 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:48 smithi138 ceph-mon[193606]: osdmap e409: 8 total, 8 up, 7 in 2023-09-23T23:26:48.325 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:48 smithi138 ceph-mon[193606]: failed to encode map e409 with expected crc 2023-09-23T23:26:48.325 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:48 smithi138 ceph-mon[193606]: failed to encode map e409 with expected crc 2023-09-23T23:26:48.325 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:48 smithi138 ceph-mon[193606]: failed to encode map e409 with expected crc 2023-09-23T23:26:48.325 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:48 smithi138 ceph-mon[193606]: failed to encode map e409 with expected crc 2023-09-23T23:26:48.325 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:48 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:26:48.325 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:48 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:26:48.325 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:48 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:48.325 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:48 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3891425878' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:26:48.345 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:26:48.361 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:26:48.425 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:26:48.445 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T23:26:48.457 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:48.478 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[221102]: failed to encode map e409 with expected crc 2023-09-23T23:26:48.478 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[221102]: osdmap e409: 8 total, 8 up, 7 in 2023-09-23T23:26:48.478 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[221102]: failed to encode map e409 with expected crc 2023-09-23T23:26:48.478 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[221102]: failed to encode map e409 with expected crc 2023-09-23T23:26:48.478 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[221102]: failed to encode map e409 with expected crc 2023-09-23T23:26:48.479 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[221102]: failed to encode map e409 with expected crc 2023-09-23T23:26:48.479 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:26:48.479 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:26:48.479 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:48.479 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3891425878' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:26:48.479 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[212939]: failed to encode map e409 with expected crc 2023-09-23T23:26:48.479 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[212939]: osdmap e409: 8 total, 8 up, 7 in 2023-09-23T23:26:48.479 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[212939]: failed to encode map e409 with expected crc 2023-09-23T23:26:48.480 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[212939]: failed to encode map e409 with expected crc 2023-09-23T23:26:48.480 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[212939]: failed to encode map e409 with expected crc 2023-09-23T23:26:48.480 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[212939]: failed to encode map e409 with expected crc 2023-09-23T23:26:48.480 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:26:48.480 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:26:48.480 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:26:48.480 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:48 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3891425878' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:26:48.511 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:26:48.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:48.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:26:48.617 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:26:48.647 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:48.698 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:26:48.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:48.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:26:48.805 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:26:48.839 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:49.589 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:26:49.592 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: pgmap v1177: 129 pgs: 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 114 active+clean; 3.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 4998/168621 objects degraded (2.964%); 29640/168621 objects misplaced (17.578%); 47 MiB/s, 759 objects/s recovering 2023-09-23T23:26:49.592 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: failed to encode map e410 with expected crc 2023-09-23T23:26:49.592 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3891425878' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:26:49.592 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: osdmap e410: 8 total, 8 up, 7 in 2023-09-23T23:26:49.592 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 4998/168621 objects degraded (2.964%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:26:49.592 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.592 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: osdmap e411: 8 total, 8 up, 7 in 2023-09-23T23:26:49.592 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.593 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.593 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.593 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.593 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.593 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: 11.11 continuing backfill to osd.3 from (355'1400,409'8118] 11:8dd28052:::benchmark_data_smithi110_433746_object102712:head to 409'8118 2023-09-23T23:26:49.593 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: 11.11 starting backfill to osd.7 from (0'0,0'0] MAX to 409'8118 2023-09-23T23:26:49.593 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.593 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: 11.5 continuing backfill to osd.3 from (355'1400,409'8092] MIN to 409'8092 2023-09-23T23:26:49.593 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: 11.5 continuing backfill to osd.5 from (355'1400,409'8092] MIN to 409'8092 2023-09-23T23:26:49.593 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: 11.5 starting backfill to osd.7 from (0'0,0'0] MAX to 409'8092 2023-09-23T23:26:49.593 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: 11.15 continuing backfill to osd.3 from (355'1434,409'8187] MIN to 409'8187 2023-09-23T23:26:49.593 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: 11.15 continuing backfill to osd.5 from (355'1434,409'8187] MIN to 409'8187 2023-09-23T23:26:49.594 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: 11.15 starting backfill to osd.7 from (0'0,0'0] MAX to 409'8187 2023-09-23T23:26:49.594 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[212939]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:49.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:26:49.595 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: pgmap v1177: 129 pgs: 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 114 active+clean; 3.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 4998/168621 objects degraded (2.964%); 29640/168621 objects misplaced (17.578%); 47 MiB/s, 759 objects/s recovering 2023-09-23T23:26:49.595 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: failed to encode map e410 with expected crc 2023-09-23T23:26:49.595 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3891425878' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:26:49.595 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: osdmap e410: 8 total, 8 up, 7 in 2023-09-23T23:26:49.595 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 4998/168621 objects degraded (2.964%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:26:49.595 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.595 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: osdmap e411: 8 total, 8 up, 7 in 2023-09-23T23:26:49.595 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.596 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.596 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.596 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.596 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.596 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: 11.11 continuing backfill to osd.3 from (355'1400,409'8118] 11:8dd28052:::benchmark_data_smithi110_433746_object102712:head to 409'8118 2023-09-23T23:26:49.596 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: 11.11 starting backfill to osd.7 from (0'0,0'0] MAX to 409'8118 2023-09-23T23:26:49.596 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.596 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: 11.5 continuing backfill to osd.3 from (355'1400,409'8092] MIN to 409'8092 2023-09-23T23:26:49.596 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: 11.5 continuing backfill to osd.5 from (355'1400,409'8092] MIN to 409'8092 2023-09-23T23:26:49.596 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: 11.5 starting backfill to osd.7 from (0'0,0'0] MAX to 409'8092 2023-09-23T23:26:49.596 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: 11.15 continuing backfill to osd.3 from (355'1434,409'8187] MIN to 409'8187 2023-09-23T23:26:49.596 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: 11.15 continuing backfill to osd.5 from (355'1434,409'8187] MIN to 409'8187 2023-09-23T23:26:49.597 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: 11.15 starting backfill to osd.7 from (0'0,0'0] MAX to 409'8187 2023-09-23T23:26:49.597 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:49 smithi110 ceph-mon[221102]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: pgmap v1177: 129 pgs: 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 114 active+clean; 3.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 4998/168621 objects degraded (2.964%); 29640/168621 objects misplaced (17.578%); 47 MiB/s, 759 objects/s recovering 2023-09-23T23:26:49.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: failed to encode map e410 with expected crc 2023-09-23T23:26:49.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3891425878' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:26:49.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: osdmap e410: 8 total, 8 up, 7 in 2023-09-23T23:26:49.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 4998/168621 objects degraded (2.964%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:26:49.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: osdmap e411: 8 total, 8 up, 7 in 2023-09-23T23:26:49.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: 11.11 continuing backfill to osd.3 from (355'1400,409'8118] 11:8dd28052:::benchmark_data_smithi110_433746_object102712:head to 409'8118 2023-09-23T23:26:49.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: 11.11 starting backfill to osd.7 from (0'0,0'0] MAX to 409'8118 2023-09-23T23:26:49.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: 11.5 continuing backfill to osd.3 from (355'1400,409'8092] MIN to 409'8092 2023-09-23T23:26:49.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: 11.5 continuing backfill to osd.5 from (355'1400,409'8092] MIN to 409'8092 2023-09-23T23:26:49.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: 11.5 starting backfill to osd.7 from (0'0,0'0] MAX to 409'8092 2023-09-23T23:26:49.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: 11.15 continuing backfill to osd.3 from (355'1434,409'8187] MIN to 409'8187 2023-09-23T23:26:49.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: 11.15 continuing backfill to osd.5 from (355'1434,409'8187] MIN to 409'8187 2023-09-23T23:26:49.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: 11.15 starting backfill to osd.7 from (0'0,0'0] MAX to 409'8187 2023-09-23T23:26:49.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:49 smithi138 ceph-mon[193606]: failed to encode map e411 with expected crc 2023-09-23T23:26:49.695 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:26:49.733 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:49.797 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:26:49.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:49.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:26:49.904 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:26:49.985 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:50.316 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:26:50.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:50.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:26:50.422 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:26:50.500 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:50.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:50 smithi110 ceph-mon[212939]: osdmap e412: 8 total, 8 up, 7 in 2023-09-23T23:26:50.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:50 smithi110 ceph-mon[212939]: failed to encode map e412 with expected crc 2023-09-23T23:26:50.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:50 smithi110 ceph-mon[212939]: failed to encode map e412 with expected crc 2023-09-23T23:26:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:50 smithi110 ceph-mon[212939]: 11.3 starting backfill to osd.7 from (0'0,0'0] MAX to 409'8141 2023-09-23T23:26:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:50 smithi110 ceph-mon[212939]: 11.13 starting backfill to osd.7 from (0'0,0'0] MAX to 409'8153 2023-09-23T23:26:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:50 smithi110 ceph-mon[212939]: failed to encode map e412 with expected crc 2023-09-23T23:26:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:50 smithi110 ceph-mon[212939]: pgmap v1181: 129 pgs: 4 remapped+peering, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 110 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 4986/164055 objects degraded (3.039%); 27638/164055 objects misplaced (16.847%); 29 MiB/s, 462 objects/s recovering 2023-09-23T23:26:50.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:50 smithi110 ceph-mon[221102]: osdmap e412: 8 total, 8 up, 7 in 2023-09-23T23:26:50.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:50 smithi110 ceph-mon[221102]: failed to encode map e412 with expected crc 2023-09-23T23:26:50.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:50 smithi110 ceph-mon[221102]: failed to encode map e412 with expected crc 2023-09-23T23:26:50.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:50 smithi110 ceph-mon[221102]: 11.3 starting backfill to osd.7 from (0'0,0'0] MAX to 409'8141 2023-09-23T23:26:50.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:50 smithi110 ceph-mon[221102]: 11.13 starting backfill to osd.7 from (0'0,0'0] MAX to 409'8153 2023-09-23T23:26:50.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:50 smithi110 ceph-mon[221102]: failed to encode map e412 with expected crc 2023-09-23T23:26:50.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:50 smithi110 ceph-mon[221102]: pgmap v1181: 129 pgs: 4 remapped+peering, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 110 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 4986/164055 objects degraded (3.039%); 27638/164055 objects misplaced (16.847%); 29 MiB/s, 462 objects/s recovering 2023-09-23T23:26:50.869 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T23:26:50.878 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:26:50.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:50.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:26:50.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:50 smithi138 ceph-mon[193606]: osdmap e412: 8 total, 8 up, 7 in 2023-09-23T23:26:50.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:50 smithi138 ceph-mon[193606]: failed to encode map e412 with expected crc 2023-09-23T23:26:50.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:50 smithi138 ceph-mon[193606]: failed to encode map e412 with expected crc 2023-09-23T23:26:50.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:50 smithi138 ceph-mon[193606]: 11.3 starting backfill to osd.7 from (0'0,0'0] MAX to 409'8141 2023-09-23T23:26:50.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:50 smithi138 ceph-mon[193606]: 11.13 starting backfill to osd.7 from (0'0,0'0] MAX to 409'8153 2023-09-23T23:26:50.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:50 smithi138 ceph-mon[193606]: failed to encode map e412 with expected crc 2023-09-23T23:26:50.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:50 smithi138 ceph-mon[193606]: pgmap v1181: 129 pgs: 4 remapped+peering, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 110 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 4986/164055 objects degraded (3.039%); 27638/164055 objects misplaced (16.847%); 29 MiB/s, 462 objects/s recovering 2023-09-23T23:26:50.994 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:26:51.051 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:26:51.087 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:26:51.134 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:51.332 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:26:51.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:51.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:26:51.441 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:51.470 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:51.532 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:51.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:51.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:51.636 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:26:51.724 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:51.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:51 smithi110 ceph-mon[212939]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:51 smithi110 ceph-mon[212939]: osdmap e413: 8 total, 8 up, 7 in 2023-09-23T23:26:51.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:51 smithi110 ceph-mon[212939]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:51 smithi110 ceph-mon[212939]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:51 smithi110 ceph-mon[212939]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:51 smithi110 ceph-mon[212939]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:51 smithi110 ceph-mon[212939]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.778 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:51 smithi110 ceph-mon[221102]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.778 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:51 smithi110 ceph-mon[221102]: osdmap e413: 8 total, 8 up, 7 in 2023-09-23T23:26:51.779 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:51 smithi110 ceph-mon[221102]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.779 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:51 smithi110 ceph-mon[221102]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.779 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:51 smithi110 ceph-mon[221102]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.779 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:51 smithi110 ceph-mon[221102]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.779 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:51 smithi110 ceph-mon[221102]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.782 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:26:51.782 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:26:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:26:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:26:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188489 9.a 9:5bc5f789:::benchmark_data_smithi110_295904_object77308:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2023-09-23T23:26:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.088144+0000", 2023-09-23T23:26:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 558.68737509100004, 2023-09-23T23:26:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.404339309, 2023-09-23T23:26:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:26:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:26:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:26:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188489 2023-09-23T23:26:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088144+0000", 2023-09-23T23:26:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088144+0000", 2023-09-23T23:26:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088143+0000", 2023-09-23T23:26:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088149+0000", 2023-09-23T23:26:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5010000000000001e-06 2023-09-23T23:26:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088167+0000", 2023-09-23T23:26:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8244999999999999e-05 2023-09-23T23:26:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088175+0000", 2023-09-23T23:26:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7940000000000003e-06 2023-09-23T23:26:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088467+0000", 2023-09-23T23:26:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00029206099999999998 2023-09-23T23:26:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088606+0000", 2023-09-23T23:26:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013849199999999999 2023-09-23T23:26:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,7", 2023-09-23T23:26:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.088732+0000", 2023-09-23T23:26:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012622999999999999 2023-09-23T23:26:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.121963+0000", 2023-09-23T23:26:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.033230955999999999 2023-09-23T23:26:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.121975+0000", 2023-09-23T23:26:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2269999999999999e-05 2023-09-23T23:26:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291169+0000", 2023-09-23T23:26:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.169194251 2023-09-23T23:26:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291184+0000", 2023-09-23T23:26:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4622e-05 2023-09-23T23:26:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.466395+0000", 2023-09-23T23:26:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.175211303 2023-09-23T23:26:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:26:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.466591+0000", 2023-09-23T23:26:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019571700000000001 2023-09-23T23:26:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492418+0000", 2023-09-23T23:26:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.025827039 2023-09-23T23:26:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492483+0000", 2023-09-23T23:26:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5488999999999996e-05 2023-09-23T23:26:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188535 9.a 9:556d73ae:::benchmark_data_smithi110_295904_object77354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:26:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.479923+0000", 2023-09-23T23:26:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 558.2955958, 2023-09-23T23:26:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.195745337, 2023-09-23T23:26:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:26:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:26:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188535 2023-09-23T23:26:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.479923+0000", 2023-09-23T23:26:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.479923+0000", 2023-09-23T23:26:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.479922+0000", 2023-09-23T23:26:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.479929+0000", 2023-09-23T23:26:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.277e-06 2023-09-23T23:26:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.479944+0000", 2023-09-23T23:26:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5027e-05 2023-09-23T23:26:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.479954+0000", 2023-09-23T23:26:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9280000000000005e-06 2023-09-23T23:26:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.484298+0000", 2023-09-23T23:26:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043436960000000002 2023-09-23T23:26:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.484306+0000", 2023-09-23T23:26:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2390000000000002e-06 2023-09-23T23:26:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291218+0000", 2023-09-23T23:26:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.80691163300000002 2023-09-23T23:26:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291226+0000", 2023-09-23T23:26:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5539999999999998e-06 2023-09-23T23:26:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.466903+0000", 2023-09-23T23:26:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.175676763 2023-09-23T23:26:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.467051+0000", 2023-09-23T23:26:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014825299999999999 2023-09-23T23:26:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:26:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.467178+0000", 2023-09-23T23:26:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000126332 2023-09-23T23:26:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484120+0000", 2023-09-23T23:26:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016942282999999999 2023-09-23T23:26:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484174+0000", 2023-09-23T23:26:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4039000000000002e-05 2023-09-23T23:26:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484241+0000", 2023-09-23T23:26:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7428999999999994e-05 2023-09-23T23:26:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.675434+0000", 2023-09-23T23:26:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.19119288500000001 2023-09-23T23:26:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.675510+0000", 2023-09-23T23:26:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5970000000000006e-05 2023-09-23T23:26:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.675669+0000", 2023-09-23T23:26:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015847499999999999 2023-09-23T23:26:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188546 9.a 9:58b06a3d:::benchmark_data_smithi110_295904_object77365:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:26:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.494309+0000", 2023-09-23T23:26:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 558.28121050000004, 2023-09-23T23:26:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1816013459999999, 2023-09-23T23:26:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:26:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:26:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188546 2023-09-23T23:26:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.494309+0000", 2023-09-23T23:26:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.494309+0000", 2023-09-23T23:26:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.494307+0000", 2023-09-23T23:26:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.494315+0000", 2023-09-23T23:26:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7389999999999999e-06 2023-09-23T23:26:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.494334+0000", 2023-09-23T23:26:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8660000000000001e-05 2023-09-23T23:26:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.494343+0000", 2023-09-23T23:26:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4199999999999996e-06 2023-09-23T23:26:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.494380+0000", 2023-09-23T23:26:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7329000000000001e-05 2023-09-23T23:26:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.494388+0000", 2023-09-23T23:26:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8970000000000008e-06 2023-09-23T23:26:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291261+0000", 2023-09-23T23:26:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.79687277199999995 2023-09-23T23:26:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291269+0000", 2023-09-23T23:26:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2349999999999999e-06 2023-09-23T23:26:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.471001+0000", 2023-09-23T23:26:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1797320760000001 2023-09-23T23:26:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.471127+0000", 2023-09-23T23:26:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012590499999999999 2023-09-23T23:26:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:26:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.471217+0000", 2023-09-23T23:26:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0203000000000004e-05 2023-09-23T23:26:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484224+0000", 2023-09-23T23:26:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013006676 2023-09-23T23:26:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484255+0000", 2023-09-23T23:26:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0657999999999997e-05 2023-09-23T23:26:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492151+0000", 2023-09-23T23:26:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078959400000000006 2023-09-23T23:26:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.675738+0000", 2023-09-23T23:26:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.18358733599999999 2023-09-23T23:26:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.675793+0000", 2023-09-23T23:26:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4852999999999998e-05 2023-09-23T23:26:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.675910+0000", 2023-09-23T23:26:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011725100000000001 2023-09-23T23:26:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188551 9.a 9:538d3980:::benchmark_data_smithi110_295904_object77370:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:26:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.499666+0000", 2023-09-23T23:26:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 558.275852945, 2023-09-23T23:26:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1800013950000001, 2023-09-23T23:26:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:26:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:26:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188551 2023-09-23T23:26:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.499666+0000", 2023-09-23T23:26:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.499666+0000", 2023-09-23T23:26:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.499665+0000", 2023-09-23T23:26:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.499671+0000", 2023-09-23T23:26:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.648e-06 2023-09-23T23:26:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.499681+0000", 2023-09-23T23:26:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0913e-05 2023-09-23T23:26:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.499689+0000", 2023-09-23T23:26:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2609999999999996e-06 2023-09-23T23:26:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.500993+0000", 2023-09-23T23:26:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013044109999999999 2023-09-23T23:26:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.501004+0000", 2023-09-23T23:26:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0645999999999999e-05 2023-09-23T23:26:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291296+0000", 2023-09-23T23:26:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.790291878 2023-09-23T23:26:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291304+0000", 2023-09-23T23:26:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3780000000000001e-06 2023-09-23T23:26:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.474986+0000", 2023-09-23T23:26:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.183682377 2023-09-23T23:26:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.475121+0000", 2023-09-23T23:26:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000134664 2023-09-23T23:26:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:26:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.475271+0000", 2023-09-23T23:26:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015008900000000001 2023-09-23T23:26:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484271+0000", 2023-09-23T23:26:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0090001539999999998 2023-09-23T23:26:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484340+0000", 2023-09-23T23:26:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8305000000000004e-05 2023-09-23T23:26:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492192+0000", 2023-09-23T23:26:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078518979999999995 2023-09-23T23:26:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.679470+0000", 2023-09-23T23:26:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.187278429 2023-09-23T23:26:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.679534+0000", 2023-09-23T23:26:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3664999999999994e-05 2023-09-23T23:26:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.679668+0000", 2023-09-23T23:26:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013406099999999999 2023-09-23T23:26:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188554 9.a 9:5586f631:::benchmark_data_smithi110_295904_object77373:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:26:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.509609+0000", 2023-09-23T23:26:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 558.26590988500004, 2023-09-23T23:26:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1702691110000001, 2023-09-23T23:26:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:26:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:26:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188554 2023-09-23T23:26:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.509609+0000", 2023-09-23T23:26:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.509609+0000", 2023-09-23T23:26:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.509607+0000", 2023-09-23T23:26:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:26:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.509616+0000", 2023-09-23T23:26:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2749999999999995e-06 2023-09-23T23:26:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.509633+0000", 2023-09-23T23:26:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6799e-05 2023-09-23T23:26:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.509641+0000", 2023-09-23T23:26:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4360000000000002e-06 2023-09-23T23:26:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.509692+0000", 2023-09-23T23:26:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0918000000000003e-05 2023-09-23T23:26:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.509702+0000", 2023-09-23T23:26:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0227000000000001e-05 2023-09-23T23:26:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291329+0000", 2023-09-23T23:26:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.781626771 2023-09-23T23:26:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291337+0000", 2023-09-23T23:26:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0539999999999995e-06 2023-09-23T23:26:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.479463+0000", 2023-09-23T23:26:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1881260490000001 2023-09-23T23:26:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.479640+0000", 2023-09-23T23:26:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017681899999999999 2023-09-23T23:26:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:26:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.479809+0000", 2023-09-23T23:26:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000168968 2023-09-23T23:26:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492506+0000", 2023-09-23T23:26:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012697721 2023-09-23T23:26:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508609+0000", 2023-09-23T23:26:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016102103 2023-09-23T23:26:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.516984+0000", 2023-09-23T23:26:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083757399999999996 2023-09-23T23:26:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.679712+0000", 2023-09-23T23:26:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.162728014 2023-09-23T23:26:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.679758+0000", 2023-09-23T23:26:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5176999999999998e-05 2023-09-23T23:26:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.679878+0000", 2023-09-23T23:26:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012095900000000001 2023-09-23T23:26:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188555 9.a 9:532f4ecb:::benchmark_data_smithi110_295904_object77374:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:26:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.511168+0000", 2023-09-23T23:26:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 558.26435091600001, 2023-09-23T23:26:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.168916372, 2023-09-23T23:26:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:26:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:26:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188555 2023-09-23T23:26:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.511168+0000", 2023-09-23T23:26:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.511168+0000", 2023-09-23T23:26:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.511167+0000", 2023-09-23T23:26:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.511174+0000", 2023-09-23T23:26:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6320000000000002e-06 2023-09-23T23:26:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.511190+0000", 2023-09-23T23:26:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6449000000000001e-05 2023-09-23T23:26:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.511198+0000", 2023-09-23T23:26:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3270000000000001e-06 2023-09-23T23:26:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.511252+0000", 2023-09-23T23:26:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3882999999999999e-05 2023-09-23T23:26:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.511263+0000", 2023-09-23T23:26:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0502e-05 2023-09-23T23:26:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291361+0000", 2023-09-23T23:26:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.78009846000000005 2023-09-23T23:26:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291376+0000", 2023-09-23T23:26:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4992e-05 2023-09-23T23:26:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484374+0000", 2023-09-23T23:26:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.192997358 2023-09-23T23:26:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484535+0000", 2023-09-23T23:26:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000160972 2023-09-23T23:26:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:26:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.484666+0000", 2023-09-23T23:26:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013110300000000001 2023-09-23T23:26:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492530+0000", 2023-09-23T23:26:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078639350000000007 2023-09-23T23:26:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508670+0000", 2023-09-23T23:26:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016140815999999999 2023-09-23T23:26:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.517035+0000", 2023-09-23T23:26:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083644519999999997 2023-09-23T23:26:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.679921+0000", 2023-09-23T23:26:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.162886436 2023-09-23T23:26:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.679967+0000", 2023-09-23T23:26:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5833e-05 2023-09-23T23:26:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.680085+0000", 2023-09-23T23:26:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117537 2023-09-23T23:26:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188563 9.a 9:50891094:::benchmark_data_smithi110_295904_object77382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:26:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.519697+0000", 2023-09-23T23:26:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 558.255822276, 2023-09-23T23:26:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.169432821, 2023-09-23T23:26:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:26:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:26:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188563 2023-09-23T23:26:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.519697+0000", 2023-09-23T23:26:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.519697+0000", 2023-09-23T23:26:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.519696+0000", 2023-09-23T23:26:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.519702+0000", 2023-09-23T23:26:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9110000000000004e-06 2023-09-23T23:26:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.519717+0000", 2023-09-23T23:26:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4688999999999999e-05 2023-09-23T23:26:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.519725+0000", 2023-09-23T23:26:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8550000000000002e-06 2023-09-23T23:26:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.524712+0000", 2023-09-23T23:26:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0049872980000000003 2023-09-23T23:26:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.524724+0000", 2023-09-23T23:26:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1639e-05 2023-09-23T23:26:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291402+0000", 2023-09-23T23:26:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.76667763499999997 2023-09-23T23:26:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291410+0000", 2023-09-23T23:26:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1519999999999992e-06 2023-09-23T23:26:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.488548+0000", 2023-09-23T23:26:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1971378480000001 2023-09-23T23:26:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.488682+0000", 2023-09-23T23:26:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000134857 2023-09-23T23:26:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:26:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.488774+0000", 2023-09-23T23:26:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2015999999999994e-05 2023-09-23T23:26:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508531+0000", 2023-09-23T23:26:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019756756 2023-09-23T23:26:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508729+0000", 2023-09-23T23:26:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019776599999999999 2023-09-23T23:26:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.517041+0000", 2023-09-23T23:26:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083118629999999992 2023-09-23T23:26:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.688958+0000", 2023-09-23T23:26:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.17191702 2023-09-23T23:26:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689015+0000", 2023-09-23T23:26:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6950999999999997e-05 2023-09-23T23:26:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689130+0000", 2023-09-23T23:26:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000114984 2023-09-23T23:26:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188579 9.a 9:5f580cc6:::benchmark_data_smithi110_295904_object77398:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:26:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.551569+0000", 2023-09-23T23:26:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 558.22395026200002, 2023-09-23T23:26:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1377349300000001, 2023-09-23T23:26:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:26:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:26:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188579 2023-09-23T23:26:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.551569+0000", 2023-09-23T23:26:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.551569+0000", 2023-09-23T23:26:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.551567+0000", 2023-09-23T23:26:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:26:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.551575+0000", 2023-09-23T23:26:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0220000000000004e-06 2023-09-23T23:26:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.551593+0000", 2023-09-23T23:26:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7995999999999998e-05 2023-09-23T23:26:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.551601+0000", 2023-09-23T23:26:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1300000000000001e-06 2023-09-23T23:26:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.551649+0000", 2023-09-23T23:26:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8032999999999999e-05 2023-09-23T23:26:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.551660+0000", 2023-09-23T23:26:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0773e-05 2023-09-23T23:26:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291436+0000", 2023-09-23T23:26:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.73977552599999996 2023-09-23T23:26:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291444+0000", 2023-09-23T23:26:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1329999999999999e-06 2023-09-23T23:26:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492557+0000", 2023-09-23T23:26:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.201113463 2023-09-23T23:26:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492654+0000", 2023-09-23T23:26:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6564999999999994e-05 2023-09-23T23:26:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:26:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.492804+0000", 2023-09-23T23:26:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015046800000000001 2023-09-23T23:26:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508572+0000", 2023-09-23T23:26:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015767664000000001 2023-09-23T23:26:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508764+0000", 2023-09-23T23:26:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000191919 2023-09-23T23:26:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.517047+0000", 2023-09-23T23:26:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0082832940000000001 2023-09-23T23:26:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689167+0000", 2023-09-23T23:26:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.17211952 2023-09-23T23:26:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689205+0000", 2023-09-23T23:26:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8515e-05 2023-09-23T23:26:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689304+0000", 2023-09-23T23:26:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8722999999999997e-05 2023-09-23T23:26:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188599 9.a 9:586bac8c:::benchmark_data_smithi110_295904_object77418:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:26:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.599792+0000", 2023-09-23T23:26:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 558.17572773799998, 2023-09-23T23:26:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0896802550000002, 2023-09-23T23:26:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:26:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:26:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188599 2023-09-23T23:26:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.599792+0000", 2023-09-23T23:26:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.599792+0000", 2023-09-23T23:26:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.599790+0000", 2023-09-23T23:26:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:26:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.599799+0000", 2023-09-23T23:26:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1730000000000001e-06 2023-09-23T23:26:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.599814+0000", 2023-09-23T23:26:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5143000000000001e-05 2023-09-23T23:26:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.599823+0000", 2023-09-23T23:26:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4779999999999997e-06 2023-09-23T23:26:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.599896+0000", 2023-09-23T23:26:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2936999999999996e-05 2023-09-23T23:26:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.599909+0000", 2023-09-23T23:26:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2845e-05 2023-09-23T23:26:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291469+0000", 2023-09-23T23:26:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.69155943600000003 2023-09-23T23:26:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291476+0000", 2023-09-23T23:26:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8869999999999992e-06 2023-09-23T23:26:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.496771+0000", 2023-09-23T23:26:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2052949530000001 2023-09-23T23:26:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.496894+0000", 2023-09-23T23:26:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012217400000000001 2023-09-23T23:26:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:26:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.497021+0000", 2023-09-23T23:26:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012702899999999999 2023-09-23T23:26:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508642+0000", 2023-09-23T23:26:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011621154 2023-09-23T23:26:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.517053+0000", 2023-09-23T23:26:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084112340000000001 2023-09-23T23:26:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.530496+0000", 2023-09-23T23:26:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013442568 2023-09-23T23:26:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689338+0000", 2023-09-23T23:26:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.158842757 2023-09-23T23:26:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689376+0000", 2023-09-23T23:26:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7685000000000003e-05 2023-09-23T23:26:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689472+0000", 2023-09-23T23:26:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5816000000000006e-05 2023-09-23T23:26:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188633 9.a 9:5d39851e:::benchmark_data_smithi110_295904_object77452:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:26:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.683953+0000", 2023-09-23T23:26:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 558.09156576199996, 2023-09-23T23:26:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0056920740000002, 2023-09-23T23:26:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:26:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:26:51.867 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188633 2023-09-23T23:26:51.867 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.867 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.867 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.867 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.867 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.683953+0000", 2023-09-23T23:26:51.867 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.867 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.867 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.867 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.867 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.683953+0000", 2023-09-23T23:26:51.867 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.867 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.868 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.683952+0000", 2023-09-23T23:26:51.868 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:26:51.868 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.868 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.683960+0000", 2023-09-23T23:26:51.868 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.157e-06 2023-09-23T23:26:51.868 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.683976+0000", 2023-09-23T23:26:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6602000000000001e-05 2023-09-23T23:26:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.683985+0000", 2023-09-23T23:26:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3839999999999997e-06 2023-09-23T23:26:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.684042+0000", 2023-09-23T23:26:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7247999999999998e-05 2023-09-23T23:26:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.684053+0000", 2023-09-23T23:26:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1321999999999999e-05 2023-09-23T23:26:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291502+0000", 2023-09-23T23:26:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.60744907299999995 2023-09-23T23:26:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291510+0000", 2023-09-23T23:26:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0059999999999992e-06 2023-09-23T23:26:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.501330+0000", 2023-09-23T23:26:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.209819821 2023-09-23T23:26:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.501459+0000", 2023-09-23T23:26:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012884999999999999 2023-09-23T23:26:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:26:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.501594+0000", 2023-09-23T23:26:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013470999999999999 2023-09-23T23:26:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508698+0000", 2023-09-23T23:26:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071044080000000004 2023-09-23T23:26:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.517059+0000", 2023-09-23T23:26:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083607079999999997 2023-09-23T23:26:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.530597+0000", 2023-09-23T23:26:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013538197 2023-09-23T23:26:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689506+0000", 2023-09-23T23:26:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.158908456 2023-09-23T23:26:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689549+0000", 2023-09-23T23:26:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3043000000000001e-05 2023-09-23T23:26:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.689646+0000", 2023-09-23T23:26:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6972999999999995e-05 2023-09-23T23:26:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45164.0:188637 9.a 9:5fdc95bd:::benchmark_data_smithi110_295904_object77456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-09-23T23:26:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:17:33.693643+0000", 2023-09-23T23:26:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 558.08187631800001, 2023-09-23T23:26:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0039752800000001, 2023-09-23T23:26:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45164", 2023-09-23T23:26:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1382353290", 2023-09-23T23:26:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 188637 2023-09-23T23:26:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.693643+0000", 2023-09-23T23:26:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.693643+0000", 2023-09-23T23:26:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.693641+0000", 2023-09-23T23:26:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.693648+0000", 2023-09-23T23:26:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2709999999999998e-06 2023-09-23T23:26:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.693660+0000", 2023-09-23T23:26:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2296999999999999e-05 2023-09-23T23:26:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.693666+0000", 2023-09-23T23:26:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1099999999999999e-06 2023-09-23T23:26:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.693733+0000", 2023-09-23T23:26:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7353000000000004e-05 2023-09-23T23:26:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:33.693747+0000", 2023-09-23T23:26:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3808999999999999e-05 2023-09-23T23:26:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291534+0000", 2023-09-23T23:26:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.59778706299999995 2023-09-23T23:26:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:34.291556+0000", 2023-09-23T23:26:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1708999999999998e-05 2023-09-23T23:26:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508787+0000", 2023-09-23T23:26:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2172306980000001 2023-09-23T23:26:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.508915+0000", 2023-09-23T23:26:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012779299999999999 2023-09-23T23:26:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T23:26:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.509041+0000", 2023-09-23T23:26:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012688700000000001 2023-09-23T23:26:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.517027+0000", 2023-09-23T23:26:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079852910000000003 2023-09-23T23:26:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.517064+0000", 2023-09-23T23:26:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7722000000000002e-05 2023-09-23T23:26:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.534755+0000", 2023-09-23T23:26:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017690251000000001 2023-09-23T23:26:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.697445+0000", 2023-09-23T23:26:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.16269038 2023-09-23T23:26:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.697499+0000", 2023-09-23T23:26:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3807000000000002e-05 2023-09-23T23:26:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:17:35.697618+0000", 2023-09-23T23:26:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011933 2023-09-23T23:26:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:28525 11.c 11:3128b4d2:::benchmark_data_smithi110_433746_object28524:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2023-09-23T23:26:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:24:05.446273+0000", 2023-09-23T23:26:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 166.32924653200001, 2023-09-23T23:26:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.34245724300000002, 2023-09-23T23:26:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 28525 2023-09-23T23:26:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:05.446273+0000", 2023-09-23T23:26:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:05.446273+0000", 2023-09-23T23:26:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:05.446271+0000", 2023-09-23T23:26:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:05.446298+0000", 2023-09-23T23:26:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6832999999999999e-05 2023-09-23T23:26:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:05.446310+0000", 2023-09-23T23:26:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1926000000000001e-05 2023-09-23T23:26:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:05.446318+0000", 2023-09-23T23:26:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.994e-06 2023-09-23T23:26:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:05.446359+0000", 2023-09-23T23:26:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1319000000000002e-05 2023-09-23T23:26:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:05.446563+0000", 2023-09-23T23:26:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000203227 2023-09-23T23:26:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,7", 2023-09-23T23:26:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:05.446730+0000", 2023-09-23T23:26:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000167379 2023-09-23T23:26:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:05.449507+0000", 2023-09-23T23:26:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027773759999999998 2023-09-23T23:26:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:05.451424+0000", 2023-09-23T23:26:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001916748 2023-09-23T23:26:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:05.788625+0000", 2023-09-23T23:26:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.337201166 2023-09-23T23:26:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:05.788687+0000", 2023-09-23T23:26:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1702e-05 2023-09-23T23:26:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:24:05.788730+0000", 2023-09-23T23:26:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2886999999999998e-05 2023-09-23T23:26:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:118711 11.10 11:0bd73574:::benchmark_data_smithi110_433746_object12831:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T23:26:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:25:43.442779+0000", 2023-09-23T23:26:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 68.332740268999999, 2023-09-23T23:26:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.31095083099999998, 2023-09-23T23:26:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 118711 2023-09-23T23:26:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:43.442779+0000", 2023-09-23T23:26:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:43.442779+0000", 2023-09-23T23:26:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:43.442777+0000", 2023-09-23T23:26:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:43.442784+0000", 2023-09-23T23:26:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2380000000000002e-06 2023-09-23T23:26:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:43.442796+0000", 2023-09-23T23:26:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2034e-05 2023-09-23T23:26:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:43.442803+0000", 2023-09-23T23:26:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7370000000000006e-06 2023-09-23T23:26:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:43.456720+0000", 2023-09-23T23:26:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013916411 2023-09-23T23:26:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:26:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:43.457247+0000", 2023-09-23T23:26:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000527164 2023-09-23T23:26:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:43.547823+0000", 2023-09-23T23:26:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.090575707000000005 2023-09-23T23:26:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:43.547925+0000", 2023-09-23T23:26:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102095 2023-09-23T23:26:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2,5", 2023-09-23T23:26:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:43.548278+0000", 2023-09-23T23:26:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035323400000000001 2023-09-23T23:26:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:43.554746+0000", 2023-09-23T23:26:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0064675260000000004 2023-09-23T23:26:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:43.555010+0000", 2023-09-23T23:26:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026395500000000002 2023-09-23T23:26:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:43.555211+0000", 2023-09-23T23:26:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000201864 2023-09-23T23:26:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:43.751548+0000", 2023-09-23T23:26:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.19633618799999999 2023-09-23T23:26:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:43.751605+0000", 2023-09-23T23:26:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7747000000000001e-05 2023-09-23T23:26:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:25:43.753730+0000", 2023-09-23T23:26:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021244340000000001 2023-09-23T23:26:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135622 11.c 11:38fc4d47:::benchmark_data_smithi110_433746_object29742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:26:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.568262+0000", 2023-09-23T23:26:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 41.207257667999997, 2023-09-23T23:26:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7526951390000001, 2023-09-23T23:26:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135622 2023-09-23T23:26:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568262+0000", 2023-09-23T23:26:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568262+0000", 2023-09-23T23:26:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568260+0000", 2023-09-23T23:26:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568267+0000", 2023-09-23T23:26:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5010000000000001e-06 2023-09-23T23:26:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568277+0000", 2023-09-23T23:26:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0165e-05 2023-09-23T23:26:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568284+0000", 2023-09-23T23:26:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5689999999999999e-06 2023-09-23T23:26:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568341+0000", 2023-09-23T23:26:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6894000000000001e-05 2023-09-23T23:26:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568349+0000", 2023-09-23T23:26:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3669999999999999e-06 2023-09-23T23:26:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505217+0000", 2023-09-23T23:26:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93686795 2023-09-23T23:26:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505229+0000", 2023-09-23T23:26:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2274e-05 2023-09-23T23:26:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.305074+0000", 2023-09-23T23:26:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.799845516 2023-09-23T23:26:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.305205+0000", 2023-09-23T23:26:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013038200000000001 2023-09-23T23:26:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:26:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.305293+0000", 2023-09-23T23:26:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8187000000000001e-05 2023-09-23T23:26:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.308879+0000", 2023-09-23T23:26:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035864999999999998 2023-09-23T23:26:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.308902+0000", 2023-09-23T23:26:51.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2161999999999999e-05 2023-09-23T23:26:51.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320708+0000", 2023-09-23T23:26:51.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011806535999999999 2023-09-23T23:26:51.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320814+0000", 2023-09-23T23:26:51.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105544 2023-09-23T23:26:51.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320848+0000", 2023-09-23T23:26:51.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4674e-05 2023-09-23T23:26:51.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320957+0000", 2023-09-23T23:26:51.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000108328 2023-09-23T23:26:51.907 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.907 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.908 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.908 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135633 11.c 11:3e1f1508:::benchmark_data_smithi110_433746_object29753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:26:51.908 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.573921+0000", 2023-09-23T23:26:51.909 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 41.201598148999999, 2023-09-23T23:26:51.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7472477729999998, 2023-09-23T23:26:51.909 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.909 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.909 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.909 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:51.909 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:51.909 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135633 2023-09-23T23:26:51.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.910 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573921+0000", 2023-09-23T23:26:51.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.910 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573921+0000", 2023-09-23T23:26:51.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.910 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.911 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573919+0000", 2023-09-23T23:26:51.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:26:51.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.911 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.911 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573926+0000", 2023-09-23T23:26:51.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2060000000000004e-06 2023-09-23T23:26:51.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.911 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.911 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573941+0000", 2023-09-23T23:26:51.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5325999999999999e-05 2023-09-23T23:26:51.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.912 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573949+0000", 2023-09-23T23:26:51.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1780000000000003e-06 2023-09-23T23:26:51.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.912 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.574200+0000", 2023-09-23T23:26:51.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025123000000000002 2023-09-23T23:26:51.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.913 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.574213+0000", 2023-09-23T23:26:51.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2498e-05 2023-09-23T23:26:51.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.913 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505252+0000", 2023-09-23T23:26:51.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93103919199999996 2023-09-23T23:26:51.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.913 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505260+0000", 2023-09-23T23:26:51.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2320000000000001e-06 2023-09-23T23:26:51.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.308916+0000", 2023-09-23T23:26:51.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.803655368 2023-09-23T23:26:51.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.309001+0000", 2023-09-23T23:26:51.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5617000000000004e-05 2023-09-23T23:26:51.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:26:51.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.309074+0000", 2023-09-23T23:26:51.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2853000000000002e-05 2023-09-23T23:26:51.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320731+0000", 2023-09-23T23:26:51.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011657351999999999 2023-09-23T23:26:51.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320771+0000", 2023-09-23T23:26:51.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9177000000000001e-05 2023-09-23T23:26:51.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320796+0000", 2023-09-23T23:26:51.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5031000000000001e-05 2023-09-23T23:26:51.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321024+0000", 2023-09-23T23:26:51.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000228156 2023-09-23T23:26:51.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321066+0000", 2023-09-23T23:26:51.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2126999999999998e-05 2023-09-23T23:26:51.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321169+0000", 2023-09-23T23:26:51.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010292 2023-09-23T23:26:51.917 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.917 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.917 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.918 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.918 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135638 11.c 11:32116872:::benchmark_data_smithi110_433746_object29758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:26:51.918 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.579560+0000", 2023-09-23T23:26:51.918 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 41.195959733999999, 2023-09-23T23:26:51.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7417568370000001, 2023-09-23T23:26:51.918 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.918 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.918 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.918 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:51.918 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:51.919 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135638 2023-09-23T23:26:51.919 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.919 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.919 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.919 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579560+0000", 2023-09-23T23:26:51.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.919 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.919 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.919 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579560+0000", 2023-09-23T23:26:51.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.919 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.920 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.920 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579558+0000", 2023-09-23T23:26:51.920 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:26:51.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.920 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.920 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579565+0000", 2023-09-23T23:26:51.920 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1459999999999999e-06 2023-09-23T23:26:51.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579578+0000", 2023-09-23T23:26:51.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3019e-05 2023-09-23T23:26:51.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579587+0000", 2023-09-23T23:26:51.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6999999999999997e-06 2023-09-23T23:26:51.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579670+0000", 2023-09-23T23:26:51.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2934000000000003e-05 2023-09-23T23:26:51.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579680+0000", 2023-09-23T23:26:51.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0742e-05 2023-09-23T23:26:51.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505282+0000", 2023-09-23T23:26:51.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.92560134699999996 2023-09-23T23:26:51.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505290+0000", 2023-09-23T23:26:51.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8380000000000002e-06 2023-09-23T23:26:51.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.312113+0000", 2023-09-23T23:26:51.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8068231020000001 2023-09-23T23:26:51.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.312237+0000", 2023-09-23T23:26:51.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012441899999999999 2023-09-23T23:26:51.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:26:51.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.312331+0000", 2023-09-23T23:26:51.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3719000000000002e-05 2023-09-23T23:26:51.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320776+0000", 2023-09-23T23:26:51.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084449720000000002 2023-09-23T23:26:51.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320988+0000", 2023-09-23T23:26:51.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021210700000000001 2023-09-23T23:26:51.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321006+0000", 2023-09-23T23:26:51.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8111999999999999e-05 2023-09-23T23:26:51.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321204+0000", 2023-09-23T23:26:51.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000197796 2023-09-23T23:26:51.926 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321233+0000", 2023-09-23T23:26:51.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9108999999999999e-05 2023-09-23T23:26:51.926 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321316+0000", 2023-09-23T23:26:51.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3517999999999996e-05 2023-09-23T23:26:51.926 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.927 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.927 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.927 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.927 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.927 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135640 11.c 11:3dfd0302:::benchmark_data_smithi110_433746_object29760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:26:51.927 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.585524+0000", 2023-09-23T23:26:51.927 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 41.189994765000002, 2023-09-23T23:26:51.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7460455509999999, 2023-09-23T23:26:51.927 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.927 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.927 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.927 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:51.928 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:51.928 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135640 2023-09-23T23:26:51.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.928 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.928 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.928 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585524+0000", 2023-09-23T23:26:51.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.928 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.928 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585524+0000", 2023-09-23T23:26:51.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585522+0000", 2023-09-23T23:26:51.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-09-23T23:26:51.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585530+0000", 2023-09-23T23:26:51.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0099999999999995e-06 2023-09-23T23:26:51.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585582+0000", 2023-09-23T23:26:51.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2074999999999999e-05 2023-09-23T23:26:51.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585593+0000", 2023-09-23T23:26:51.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1155000000000001e-05 2023-09-23T23:26:51.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585657+0000", 2023-09-23T23:26:51.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3563000000000001e-05 2023-09-23T23:26:51.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585669+0000", 2023-09-23T23:26:51.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2368000000000001e-05 2023-09-23T23:26:51.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505310+0000", 2023-09-23T23:26:51.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.91964102400000003 2023-09-23T23:26:51.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.932 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.932 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505318+0000", 2023-09-23T23:26:51.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7349999999999996e-06 2023-09-23T23:26:51.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.932 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.932 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.315785+0000", 2023-09-23T23:26:51.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.810466366 2023-09-23T23:26:51.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.932 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.932 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.315917+0000", 2023-09-23T23:26:51.933 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013256300000000001 2023-09-23T23:26:51.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:26:51.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.316041+0000", 2023-09-23T23:26:51.933 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012345100000000001 2023-09-23T23:26:51.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321343+0000", 2023-09-23T23:26:51.933 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053020849999999998 2023-09-23T23:26:51.934 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.934 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.934 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.934 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331351+0000", 2023-09-23T23:26:51.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010008193 2023-09-23T23:26:51.934 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.934 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.934 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.934 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331412+0000", 2023-09-23T23:26:51.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1023000000000002e-05 2023-09-23T23:26:51.934 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.934 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.934 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.935 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331438+0000", 2023-09-23T23:26:51.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6432999999999999e-05 2023-09-23T23:26:51.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.935 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331471+0000", 2023-09-23T23:26:51.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3127999999999999e-05 2023-09-23T23:26:51.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.935 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331570+0000", 2023-09-23T23:26:51.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8610999999999995e-05 2023-09-23T23:26:51.936 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.936 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.936 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.936 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135653 11.c 11:39fc45c5:::benchmark_data_smithi110_433746_object29773:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:26:51.936 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.612482+0000", 2023-09-23T23:26:51.936 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 41.163037746999997, 2023-09-23T23:26:51.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7224649520000002, 2023-09-23T23:26:51.936 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.936 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.936 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.937 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:51.937 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:51.937 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135653 2023-09-23T23:26:51.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.937 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612482+0000", 2023-09-23T23:26:51.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612482+0000", 2023-09-23T23:26:51.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612480+0000", 2023-09-23T23:26:51.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:51.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612487+0000", 2023-09-23T23:26:51.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2640000000000002e-06 2023-09-23T23:26:51.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612499+0000", 2023-09-23T23:26:51.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2116000000000001e-05 2023-09-23T23:26:51.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612509+0000", 2023-09-23T23:26:51.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6940000000000007e-06 2023-09-23T23:26:51.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612569+0000", 2023-09-23T23:26:51.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9738999999999998e-05 2023-09-23T23:26:51.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612579+0000", 2023-09-23T23:26:51.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0592e-05 2023-09-23T23:26:51.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505353+0000", 2023-09-23T23:26:51.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.89277419800000002 2023-09-23T23:26:51.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.941 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.941 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505365+0000", 2023-09-23T23:26:51.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1090999999999999e-05 2023-09-23T23:26:51.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.941 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.941 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321356+0000", 2023-09-23T23:26:51.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8159917220000001 2023-09-23T23:26:51.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.942 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.942 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321466+0000", 2023-09-23T23:26:51.942 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109347 2023-09-23T23:26:51.942 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.942 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:26:51.942 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321551+0000", 2023-09-23T23:26:51.942 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5669999999999998e-05 2023-09-23T23:26:51.942 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.943 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331357+0000", 2023-09-23T23:26:51.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00980549 2023-09-23T23:26:51.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.943 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331615+0000", 2023-09-23T23:26:51.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025800399999999999 2023-09-23T23:26:51.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.943 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331637+0000", 2023-09-23T23:26:51.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2036e-05 2023-09-23T23:26:51.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334781+0000", 2023-09-23T23:26:51.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031445589999999999 2023-09-23T23:26:51.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334836+0000", 2023-09-23T23:26:51.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4469999999999999e-05 2023-09-23T23:26:51.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334946+0000", 2023-09-23T23:26:51.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110579 2023-09-23T23:26:51.945 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.945 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.945 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.945 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135663 11.c 11:3bd42e62:::benchmark_data_smithi110_433746_object29783:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:26:51.945 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.646633+0000", 2023-09-23T23:26:51.946 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 41.128886596000001, 2023-09-23T23:26:51.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.691323304, 2023-09-23T23:26:51.946 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.946 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.946 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.946 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:51.946 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:51.946 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135663 2023-09-23T23:26:51.946 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.946 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646633+0000", 2023-09-23T23:26:51.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646633+0000", 2023-09-23T23:26:51.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646631+0000", 2023-09-23T23:26:51.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:51.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646638+0000", 2023-09-23T23:26:51.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9949999999999999e-06 2023-09-23T23:26:51.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646653+0000", 2023-09-23T23:26:51.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4892000000000001e-05 2023-09-23T23:26:51.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646661+0000", 2023-09-23T23:26:51.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9929999999999995e-06 2023-09-23T23:26:51.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646713+0000", 2023-09-23T23:26:51.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1544e-05 2023-09-23T23:26:51.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646725+0000", 2023-09-23T23:26:51.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.223e-05 2023-09-23T23:26:51.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505389+0000", 2023-09-23T23:26:51.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.858663756 2023-09-23T23:26:51.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505397+0000", 2023-09-23T23:26:51.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4670000000000004e-06 2023-09-23T23:26:51.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.324831+0000", 2023-09-23T23:26:51.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.819433461 2023-09-23T23:26:51.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.324942+0000", 2023-09-23T23:26:51.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011171900000000001 2023-09-23T23:26:51.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:26:51.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.325072+0000", 2023-09-23T23:26:51.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012980000000000001 2023-09-23T23:26:51.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331656+0000", 2023-09-23T23:26:51.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0065834639999999998 2023-09-23T23:26:51.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.332662+0000", 2023-09-23T23:26:51.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010060049999999999 2023-09-23T23:26:51.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.333742+0000", 2023-09-23T23:26:51.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010798170000000001 2023-09-23T23:26:51.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337799+0000", 2023-09-23T23:26:51.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040573579999999996 2023-09-23T23:26:51.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.954 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337850+0000", 2023-09-23T23:26:51.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0818e-05 2023-09-23T23:26:51.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.954 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337956+0000", 2023-09-23T23:26:51.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010621 2023-09-23T23:26:51.954 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.954 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.954 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.955 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135680 11.c 11:34dfa950:::benchmark_data_smithi110_433746_object29800:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:26:51.955 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.699865+0000", 2023-09-23T23:26:51.955 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 41.075654172, 2023-09-23T23:26:51.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6382614580000001, 2023-09-23T23:26:51.955 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:26:51.955 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:26:51.955 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:26:51.955 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:26:51.955 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:26:51.955 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135680 2023-09-23T23:26:51.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.956 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:26:51.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:26:51.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699865+0000", 2023-09-23T23:26:51.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:26:51.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699865+0000", 2023-09-23T23:26:51.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:26:51.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:26:51.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699864+0000", 2023-09-23T23:26:51.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:26:51.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:26:51.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699870+0000", 2023-09-23T23:26:51.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9329999999999998e-06 2023-09-23T23:26:51.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:26:51.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699886+0000", 2023-09-23T23:26:51.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6402e-05 2023-09-23T23:26:51.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:26:51.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699896+0000", 2023-09-23T23:26:51.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3889999999999994e-06 2023-09-23T23:26:51.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699953+0000", 2023-09-23T23:26:51.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7763999999999998e-05 2023-09-23T23:26:51.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699968+0000", 2023-09-23T23:26:51.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4749999999999999e-05 2023-09-23T23:26:51.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505806+0000", 2023-09-23T23:26:51.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.80583805100000006 2023-09-23T23:26:51.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:26:51.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505816+0000", 2023-09-23T23:26:51.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9320000000000008e-06 2023-09-23T23:26:51.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:26:51.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.328139+0000", 2023-09-23T23:26:51.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8223231129999999 2023-09-23T23:26:51.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:26:51.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.328247+0000", 2023-09-23T23:26:51.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010777600000000001 2023-09-23T23:26:51.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:26:51.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.328367+0000", 2023-09-23T23:26:51.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012031000000000001 2023-09-23T23:26:51.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331675+0000", 2023-09-23T23:26:51.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033076669999999998 2023-09-23T23:26:51.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.332701+0000", 2023-09-23T23:26:51.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010263959999999999 2023-09-23T23:26:51.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:26:51.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.333784+0000", 2023-09-23T23:26:51.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001082911 2023-09-23T23:26:51.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:26:51.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337990+0000", 2023-09-23T23:26:51.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042062970000000003 2023-09-23T23:26:51.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:26:51.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.338026+0000", 2023-09-23T23:26:51.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5335e-05 2023-09-23T23:26:51.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:26:51.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:26:51.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:26:51.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.338127+0000", 2023-09-23T23:26:51.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000100717 2023-09-23T23:26:51.964 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.964 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.964 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.964 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:26:51.964 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:26:51.964 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:26:51.965 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:51 smithi138 ceph-mon[193606]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.965 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:51 smithi138 ceph-mon[193606]: osdmap e413: 8 total, 8 up, 7 in 2023-09-23T23:26:51.965 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:51 smithi138 ceph-mon[193606]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.965 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:51 smithi138 ceph-mon[193606]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.965 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:51 smithi138 ceph-mon[193606]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.966 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:51 smithi138 ceph-mon[193606]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.966 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:51 smithi138 ceph-mon[193606]: failed to encode map e413 with expected crc 2023-09-23T23:26:51.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:51.966 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:26:51.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:26:52.067 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:26:52.162 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:52.938 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:26:52.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:52 smithi138 ceph-mon[193606]: failed to encode map e414 with expected crc 2023-09-23T23:26:52.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:52 smithi138 ceph-mon[193606]: osdmap e414: 8 total, 8 up, 7 in 2023-09-23T23:26:52.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:52 smithi138 ceph-mon[193606]: failed to encode map e414 with expected crc 2023-09-23T23:26:52.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:52 smithi138 ceph-mon[193606]: failed to encode map e414 with expected crc 2023-09-23T23:26:52.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:52 smithi138 ceph-mon[193606]: pgmap v1184: 129 pgs: 4 remapped+peering, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 108 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 4986/160722 objects degraded (3.102%); 33218/160722 objects misplaced (20.668%); 42 MiB/s, 677 objects/s recovering 2023-09-23T23:26:52.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:52 smithi110 ceph-mon[212939]: failed to encode map e414 with expected crc 2023-09-23T23:26:52.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:52 smithi110 ceph-mon[212939]: osdmap e414: 8 total, 8 up, 7 in 2023-09-23T23:26:52.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:52 smithi110 ceph-mon[212939]: failed to encode map e414 with expected crc 2023-09-23T23:26:52.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:52 smithi110 ceph-mon[212939]: failed to encode map e414 with expected crc 2023-09-23T23:26:52.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:52 smithi110 ceph-mon[212939]: pgmap v1184: 129 pgs: 4 remapped+peering, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 108 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 4986/160722 objects degraded (3.102%); 33218/160722 objects misplaced (20.668%); 42 MiB/s, 677 objects/s recovering 2023-09-23T23:26:52.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:52 smithi110 ceph-mon[221102]: failed to encode map e414 with expected crc 2023-09-23T23:26:52.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:52 smithi110 ceph-mon[221102]: osdmap e414: 8 total, 8 up, 7 in 2023-09-23T23:26:52.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:52 smithi110 ceph-mon[221102]: failed to encode map e414 with expected crc 2023-09-23T23:26:52.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:52 smithi110 ceph-mon[221102]: failed to encode map e414 with expected crc 2023-09-23T23:26:52.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:52 smithi110 ceph-mon[221102]: pgmap v1184: 129 pgs: 4 remapped+peering, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 108 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 4986/160722 objects degraded (3.102%); 33218/160722 objects misplaced (20.668%); 42 MiB/s, 677 objects/s recovering 2023-09-23T23:26:52.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:52.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:26:53.061 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:26:53.180 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:53.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:53 smithi138 ceph-mon[193606]: failed to encode map e415 with expected crc 2023-09-23T23:26:53.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:53 smithi138 ceph-mon[193606]: failed to encode map e415 with expected crc 2023-09-23T23:26:53.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:53 smithi138 ceph-mon[193606]: osdmap e415: 8 total, 8 up, 7 in 2023-09-23T23:26:53.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:53 smithi138 ceph-mon[193606]: failed to encode map e415 with expected crc 2023-09-23T23:26:53.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:53 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 4986/160722 objects degraded (3.102%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:26:53.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:53 smithi110 ceph-mon[212939]: failed to encode map e415 with expected crc 2023-09-23T23:26:53.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:53 smithi110 ceph-mon[212939]: failed to encode map e415 with expected crc 2023-09-23T23:26:53.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:53 smithi110 ceph-mon[212939]: osdmap e415: 8 total, 8 up, 7 in 2023-09-23T23:26:53.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:53 smithi110 ceph-mon[212939]: failed to encode map e415 with expected crc 2023-09-23T23:26:53.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:53 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 4986/160722 objects degraded (3.102%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:26:53.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:53 smithi110 ceph-mon[221102]: failed to encode map e415 with expected crc 2023-09-23T23:26:53.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:53 smithi110 ceph-mon[221102]: failed to encode map e415 with expected crc 2023-09-23T23:26:53.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:53 smithi110 ceph-mon[221102]: osdmap e415: 8 total, 8 up, 7 in 2023-09-23T23:26:53.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:53 smithi110 ceph-mon[221102]: failed to encode map e415 with expected crc 2023-09-23T23:26:53.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:53 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 4986/160722 objects degraded (3.102%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:26:54.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:54 smithi138 ceph-mon[193606]: pgmap v1186: 129 pgs: 4 remapped+peering, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 109 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 5717/159666 objects degraded (3.581%); 32245/159666 objects misplaced (20.195%); 18 MiB/s, 327 objects/s recovering 2023-09-23T23:26:54.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:54 smithi110 ceph-mon[212939]: pgmap v1186: 129 pgs: 4 remapped+peering, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 109 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 5717/159666 objects degraded (3.581%); 32245/159666 objects misplaced (20.195%); 18 MiB/s, 327 objects/s recovering 2023-09-23T23:26:54.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:54 smithi110 ceph-mon[221102]: pgmap v1186: 129 pgs: 4 remapped+peering, 1 active+recovery_wait+remapped, 2 active+remapped+backfilling, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 109 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 5717/159666 objects degraded (3.581%); 32245/159666 objects misplaced (20.195%); 18 MiB/s, 327 objects/s recovering 2023-09-23T23:26:55.824 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:26:55.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:55.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:26:55.928 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:26:56.101 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:56.653 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:26:56.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:56.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:26:56.763 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:26:56.791 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:56.851 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:26:56.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:56.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:26:56.953 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:26:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:26:56] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.43.0" 2023-09-23T23:26:56.957 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:56.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:56 smithi110 ceph-mon[212939]: pgmap v1187: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 115 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 2286/153057 objects degraded (1.494%); 32321/153057 objects misplaced (21.117%); 28 MiB/s, 471 objects/s recovering 2023-09-23T23:26:56.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:56 smithi110 ceph-mon[221102]: pgmap v1187: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 115 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 2286/153057 objects degraded (1.494%); 32321/153057 objects misplaced (21.117%); 28 MiB/s, 471 objects/s recovering 2023-09-23T23:26:56.990 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:57.054 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:57.062 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:56 smithi138 ceph-mon[193606]: pgmap v1187: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 115 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 2286/153057 objects degraded (1.494%); 32321/153057 objects misplaced (21.117%); 28 MiB/s, 471 objects/s recovering 2023-09-23T23:26:57.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:57.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:57.164 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:26:57.261 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:57.547 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:26:57.553 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:26:57.562 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:26:57.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:57.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:26:57.685 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:26:57.714 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:57.767 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:26:57.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:57.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:26:57.876 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:26:57.956 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:58.148 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:26:58.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:58.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:26:58.263 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:26:58.371 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:58.531 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:26:58.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:58.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:26:58.649 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:26:58.681 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:58.738 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:26:58.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:58.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:26:58.754 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:58 smithi110 ceph-mon[221102]: pgmap v1188: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 115 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 2286/150813 objects degraded (1.516%); 30734/150813 objects misplaced (20.379%); 32 MiB/s, 530 objects/s recovering 2023-09-23T23:26:58.754 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:26:58.754 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:58 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2286/150813 objects degraded (1.516%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:26:58.846 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:26:58.875 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:26:58.875 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:58.920 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:26:58.941 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:26:58.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:58 smithi138 ceph-mon[193606]: pgmap v1188: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 115 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 2286/150813 objects degraded (1.516%); 30734/150813 objects misplaced (20.379%); 32 MiB/s, 530 objects/s recovering 2023-09-23T23:26:58.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:26:58.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:58 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2286/150813 objects degraded (1.516%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:26:58.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:59.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:26:59.035 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:26:59.041 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:26:59.074 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:26:59.126 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:26:59.135 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:26:59.167 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:26:59.167 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:59.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:58 smithi110 ceph-mon[212939]: pgmap v1188: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 115 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 2286/150813 objects degraded (1.516%); 30734/150813 objects misplaced (20.379%); 32 MiB/s, 530 objects/s recovering 2023-09-23T23:26:59.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:26:59.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:58 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2286/150813 objects degraded (1.516%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:26:59.210 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:26:59.224 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:26:59.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:59.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:26:59.252 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:26:59.252 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:26:59.252 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:26:59.253 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:26:59.253 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:26:59.253 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:26:59.253 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:26:59.253 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:26:59.331 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:26:59.401 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:59.508 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:26:59.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:59.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:26:59.630 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:26:59.667 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:59.731 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:26:59.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:59.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:26:59.807 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:26:59.838 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:26:59.841 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:59 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4072607140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:26:59.842 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:26:59 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1877775629' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:26:59.843 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:59 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4072607140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:26:59.844 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:26:59 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1877775629' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:26:59.871 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:26:59.924 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:26:59.929 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:59 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4072607140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:26:59.929 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:26:59 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1877775629' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:26:59.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:26:59.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:00.031 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:27:00.067 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:00.124 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:27:00.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:00.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:27:00.230 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:00.386 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:01.124 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:00 smithi110 ceph-mon[221102]: pgmap v1189: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 115 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 2286/144321 objects degraded (1.584%); 29776/144321 objects misplaced (20.632%); 22 MiB/s, 369 objects/s recovering 2023-09-23T23:27:01.124 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:00 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:27:01.124 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:00 smithi110 ceph-mon[221102]: failed to encode map e416 with expected crc 2023-09-23T23:27:01.124 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:00 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1877775629' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:27:01.125 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:00 smithi110 ceph-mon[221102]: osdmap e416: 8 total, 8 up, 7 in 2023-09-23T23:27:01.125 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:00 smithi110 ceph-mon[221102]: failed to encode map e416 with expected crc 2023-09-23T23:27:01.125 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:00 smithi110 ceph-mon[221102]: failed to encode map e416 with expected crc 2023-09-23T23:27:01.126 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:00 smithi110 ceph-mon[212939]: pgmap v1189: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 115 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 2286/144321 objects degraded (1.584%); 29776/144321 objects misplaced (20.632%); 22 MiB/s, 369 objects/s recovering 2023-09-23T23:27:01.126 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:00 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:27:01.126 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:00 smithi110 ceph-mon[212939]: failed to encode map e416 with expected crc 2023-09-23T23:27:01.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:00 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1877775629' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:27:01.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:00 smithi110 ceph-mon[212939]: osdmap e416: 8 total, 8 up, 7 in 2023-09-23T23:27:01.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:00 smithi110 ceph-mon[212939]: failed to encode map e416 with expected crc 2023-09-23T23:27:01.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:00 smithi110 ceph-mon[212939]: failed to encode map e416 with expected crc 2023-09-23T23:27:01.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:00 smithi138 ceph-mon[193606]: pgmap v1189: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 115 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 2286/144321 objects degraded (1.584%); 29776/144321 objects misplaced (20.632%); 22 MiB/s, 369 objects/s recovering 2023-09-23T23:27:01.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:00 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:27:01.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:00 smithi138 ceph-mon[193606]: failed to encode map e416 with expected crc 2023-09-23T23:27:01.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:00 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1877775629' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:27:01.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:00 smithi138 ceph-mon[193606]: osdmap e416: 8 total, 8 up, 7 in 2023-09-23T23:27:01.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:00 smithi138 ceph-mon[193606]: failed to encode map e416 with expected crc 2023-09-23T23:27:01.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:00 smithi138 ceph-mon[193606]: failed to encode map e416 with expected crc 2023-09-23T23:27:02.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:01 smithi138 ceph-mon[193606]: osdmap e417: 8 total, 8 up, 7 in 2023-09-23T23:27:02.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:01 smithi138 ceph-mon[193606]: failed to encode map e417 with expected crc 2023-09-23T23:27:02.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:01 smithi138 ceph-mon[193606]: failed to encode map e417 with expected crc 2023-09-23T23:27:02.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:01 smithi138 ceph-mon[193606]: failed to encode map e417 with expected crc 2023-09-23T23:27:02.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:01 smithi110 ceph-mon[212939]: osdmap e417: 8 total, 8 up, 7 in 2023-09-23T23:27:02.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:01 smithi110 ceph-mon[212939]: failed to encode map e417 with expected crc 2023-09-23T23:27:02.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:01 smithi110 ceph-mon[212939]: failed to encode map e417 with expected crc 2023-09-23T23:27:02.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:01 smithi110 ceph-mon[212939]: failed to encode map e417 with expected crc 2023-09-23T23:27:02.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:01 smithi110 ceph-mon[221102]: osdmap e417: 8 total, 8 up, 7 in 2023-09-23T23:27:02.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:01 smithi110 ceph-mon[221102]: failed to encode map e417 with expected crc 2023-09-23T23:27:02.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:01 smithi110 ceph-mon[221102]: failed to encode map e417 with expected crc 2023-09-23T23:27:02.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:01 smithi110 ceph-mon[221102]: failed to encode map e417 with expected crc 2023-09-23T23:27:02.502 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:02.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:02.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:02.614 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:27:02.789 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:02 smithi110 ceph-mon[221102]: pgmap v1192: 129 pgs: 4 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 115 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 2286/140637 objects degraded (1.625%); 27576/140637 objects misplaced (19.608%); 28 MiB/s, 444 objects/s recovering 2023-09-23T23:27:02.789 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:02 smithi110 ceph-mon[221102]: 1.1a scrub starts 2023-09-23T23:27:02.789 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:02 smithi110 ceph-mon[221102]: failed to encode map e418 with expected crc 2023-09-23T23:27:02.789 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:02 smithi110 ceph-mon[221102]: failed to encode map e418 with expected crc 2023-09-23T23:27:02.790 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:02 smithi110 ceph-mon[221102]: osdmap e418: 8 total, 8 up, 7 in 2023-09-23T23:27:02.790 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:02 smithi110 ceph-mon[221102]: failed to encode map e418 with expected crc 2023-09-23T23:27:02.873 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:02 smithi138 ceph-mon[193606]: pgmap v1192: 129 pgs: 4 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 115 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 2286/140637 objects degraded (1.625%); 27576/140637 objects misplaced (19.608%); 28 MiB/s, 444 objects/s recovering 2023-09-23T23:27:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:02 smithi138 ceph-mon[193606]: 1.1a scrub starts 2023-09-23T23:27:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:02 smithi138 ceph-mon[193606]: failed to encode map e418 with expected crc 2023-09-23T23:27:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:02 smithi138 ceph-mon[193606]: failed to encode map e418 with expected crc 2023-09-23T23:27:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:02 smithi138 ceph-mon[193606]: osdmap e418: 8 total, 8 up, 7 in 2023-09-23T23:27:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:02 smithi138 ceph-mon[193606]: failed to encode map e418 with expected crc 2023-09-23T23:27:03.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:02 smithi110 ceph-mon[212939]: pgmap v1192: 129 pgs: 4 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 115 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 2286/140637 objects degraded (1.625%); 27576/140637 objects misplaced (19.608%); 28 MiB/s, 444 objects/s recovering 2023-09-23T23:27:03.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:02 smithi110 ceph-mon[212939]: 1.1a scrub starts 2023-09-23T23:27:03.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:02 smithi110 ceph-mon[212939]: failed to encode map e418 with expected crc 2023-09-23T23:27:03.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:02 smithi110 ceph-mon[212939]: failed to encode map e418 with expected crc 2023-09-23T23:27:03.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:02 smithi110 ceph-mon[212939]: osdmap e418: 8 total, 8 up, 7 in 2023-09-23T23:27:03.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:02 smithi110 ceph-mon[212939]: failed to encode map e418 with expected crc 2023-09-23T23:27:03.421 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:27:03.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:03.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:27:03.533 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:27:03.567 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:03.634 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:27:03.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:03.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:27:03.740 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:27:03.813 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:03.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:03 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2286/140637 objects degraded (1.625%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:27:04.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:03 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2286/140637 objects degraded (1.625%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:27:04.199 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:03 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2286/140637 objects degraded (1.625%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:27:04.333 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:27:04.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:04.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:27:04.396 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:27:04.442 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:04.474 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:04.537 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:04.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:04.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:04.643 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:27:04.672 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:04.729 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:27:04.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:04.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:27:04.835 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:27:04.905 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:04 smithi110 ceph-mon[212939]: pgmap v1194: 129 pgs: 3 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 116 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 2286/138870 objects degraded (1.646%); 26969/138870 objects misplaced (19.420%); 18 MiB/s, 290 objects/s recovering 2023-09-23T23:27:04.905 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:04 smithi110 ceph-mon[221102]: pgmap v1194: 129 pgs: 3 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 116 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 2286/138870 objects degraded (1.646%); 26969/138870 objects misplaced (19.420%); 18 MiB/s, 290 objects/s recovering 2023-09-23T23:27:04.913 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:05.140 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:27:05.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:05.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:27:05.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:04 smithi138 ceph-mon[193606]: pgmap v1194: 129 pgs: 3 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 116 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 2286/138870 objects degraded (1.646%); 26969/138870 objects misplaced (19.420%); 18 MiB/s, 290 objects/s recovering 2023-09-23T23:27:05.252 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:27:05.355 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:05.478 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:27:05.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:05.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:27:05.588 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:27:05.667 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:05.802 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:27:05.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:05.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:27:05.906 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T23:27:05.916 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:27:05.951 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:05.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:05 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4228949898' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:27:05.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:05 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:27:05.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:05 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4228949898' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:27:05.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:05 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:27:06.007 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:27:06.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:06.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:06.113 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:27:06.141 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:05 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4228949898' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:27:06.141 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:05 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:27:06.143 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:06.184 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:27:06.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:06.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:27:06.265 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:27:06.291 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:06.357 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:27:06.377 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:06.498 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:06.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:06.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:06.629 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:06.674 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:06.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:06 smithi110 ceph-mon[212939]: pgmap v1195: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 117 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 2286/132564 objects degraded (1.724%); 25002/132564 objects misplaced (18.860%); 19 MiB/s, 311 objects/s recovering 2023-09-23T23:27:06.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:06 smithi110 ceph-mon[212939]: 11.17 scrub starts 2023-09-23T23:27:06.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:06 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:27:06.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:06 smithi110 ceph-mon[212939]: osdmap e419: 8 total, 8 up, 7 in 2023-09-23T23:27:06.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:06 smithi110 ceph-mon[212939]: failed to encode map e419 with expected crc 2023-09-23T23:27:06.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:06 smithi110 ceph-mon[212939]: failed to encode map e419 with expected crc 2023-09-23T23:27:06.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:06 smithi110 ceph-mon[212939]: failed to encode map e419 with expected crc 2023-09-23T23:27:06.953 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:27:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:27:06] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.43.0" 2023-09-23T23:27:06.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:06 smithi110 ceph-mon[221102]: pgmap v1195: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 117 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 2286/132564 objects degraded (1.724%); 25002/132564 objects misplaced (18.860%); 19 MiB/s, 311 objects/s recovering 2023-09-23T23:27:06.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:06 smithi110 ceph-mon[221102]: 11.17 scrub starts 2023-09-23T23:27:06.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:06 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:27:06.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:06 smithi110 ceph-mon[221102]: osdmap e419: 8 total, 8 up, 7 in 2023-09-23T23:27:06.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:06 smithi110 ceph-mon[221102]: failed to encode map e419 with expected crc 2023-09-23T23:27:06.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:06 smithi110 ceph-mon[221102]: failed to encode map e419 with expected crc 2023-09-23T23:27:06.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:06 smithi110 ceph-mon[221102]: failed to encode map e419 with expected crc 2023-09-23T23:27:07.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:06 smithi138 ceph-mon[193606]: pgmap v1195: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 117 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 2286/132564 objects degraded (1.724%); 25002/132564 objects misplaced (18.860%); 19 MiB/s, 311 objects/s recovering 2023-09-23T23:27:07.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:06 smithi138 ceph-mon[193606]: 11.17 scrub starts 2023-09-23T23:27:07.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:06 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:27:07.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:06 smithi138 ceph-mon[193606]: osdmap e419: 8 total, 8 up, 7 in 2023-09-23T23:27:07.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:06 smithi138 ceph-mon[193606]: failed to encode map e419 with expected crc 2023-09-23T23:27:07.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:06 smithi138 ceph-mon[193606]: failed to encode map e419 with expected crc 2023-09-23T23:27:07.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:06 smithi138 ceph-mon[193606]: failed to encode map e419 with expected crc 2023-09-23T23:27:07.990 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:07.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:07.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:08.012 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T23:27:08.099 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:27:08.166 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[212939]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.166 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[212939]: osdmap e420: 8 total, 8 up, 7 in 2023-09-23T23:27:08.166 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[212939]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.166 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[212939]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.166 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[212939]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.166 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[212939]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.166 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[212939]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.167 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[212939]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.167 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[212939]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.167 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[212939]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.167 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[221102]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.167 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[221102]: osdmap e420: 8 total, 8 up, 7 in 2023-09-23T23:27:08.167 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[221102]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.167 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[221102]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.167 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[221102]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.167 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[221102]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.167 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[221102]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.168 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[221102]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.168 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[221102]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.168 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:07 smithi110 ceph-mon[221102]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:07 smithi138 ceph-mon[193606]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:07 smithi138 ceph-mon[193606]: osdmap e420: 8 total, 8 up, 7 in 2023-09-23T23:27:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:07 smithi138 ceph-mon[193606]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:07 smithi138 ceph-mon[193606]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:07 smithi138 ceph-mon[193606]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:07 smithi138 ceph-mon[193606]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:07 smithi138 ceph-mon[193606]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:07 smithi138 ceph-mon[193606]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:07 smithi138 ceph-mon[193606]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:07 smithi138 ceph-mon[193606]: failed to encode map e420 with expected crc 2023-09-23T23:27:08.206 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:08.382 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:27:08.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:08.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:27:08.493 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:27:08.527 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:08.586 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:27:08.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:08.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:27:08.687 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:27:08.692 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:27:08.722 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:08.733 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:27:08.781 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:27:08.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:08.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:08.855 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:27:08.856 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:27:08.865 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:08 smithi110 ceph-mon[212939]: pgmap v1198: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 117 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 2286/130947 objects degraded (1.746%); 23967/130947 objects misplaced (18.303%); 16 MiB/s, 256 objects/s recovering 2023-09-23T23:27:08.865 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:08 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2286/130947 objects degraded (1.746%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:27:08.865 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:08 smithi110 ceph-mon[221102]: pgmap v1198: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 117 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 2286/130947 objects degraded (1.746%); 23967/130947 objects misplaced (18.303%); 16 MiB/s, 256 objects/s recovering 2023-09-23T23:27:08.865 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:08 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2286/130947 objects degraded (1.746%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:27:08.889 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:27:08.987 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:09.012 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:27:09.192 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 23:27:08 smithi138 conmon[172649]: logger=cleanup t=2023-09-23T23:27:08.822060973Z level=info msg="Completed cleanup jobs" duration=886.820463ms 2023-09-23T23:27:09.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:08 smithi138 ceph-mon[193606]: pgmap v1198: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 117 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 2286/130947 objects degraded (1.746%); 23967/130947 objects misplaced (18.303%); 16 MiB/s, 256 objects/s recovering 2023-09-23T23:27:09.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:08 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2286/130947 objects degraded (1.746%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:27:10.531 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:27:10.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:10.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:27:10.638 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:27:10.669 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:10.853 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:27:10.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:10.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:27:10.958 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:11.042 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:11.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:10 smithi138 ceph-mon[193606]: pgmap v1199: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 118 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 2286/123618 objects degraded (1.849%); 21864/123618 objects misplaced (17.687%); 22 MiB/s, 351 objects/s recovering 2023-09-23T23:27:11.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:10 smithi138 ceph-mon[193606]: osdmap e421: 8 total, 8 up, 7 in 2023-09-23T23:27:11.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:10 smithi138 ceph-mon[193606]: failed to encode map e421 with expected crc 2023-09-23T23:27:11.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:10 smithi138 ceph-mon[193606]: failed to encode map e421 with expected crc 2023-09-23T23:27:11.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:10 smithi110 ceph-mon[212939]: pgmap v1199: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 118 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 2286/123618 objects degraded (1.849%); 21864/123618 objects misplaced (17.687%); 22 MiB/s, 351 objects/s recovering 2023-09-23T23:27:11.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:10 smithi110 ceph-mon[212939]: osdmap e421: 8 total, 8 up, 7 in 2023-09-23T23:27:11.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:10 smithi110 ceph-mon[212939]: failed to encode map e421 with expected crc 2023-09-23T23:27:11.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:10 smithi110 ceph-mon[212939]: failed to encode map e421 with expected crc 2023-09-23T23:27:11.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:10 smithi110 ceph-mon[221102]: pgmap v1199: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 118 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 2286/123618 objects degraded (1.849%); 21864/123618 objects misplaced (17.687%); 22 MiB/s, 351 objects/s recovering 2023-09-23T23:27:11.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:10 smithi110 ceph-mon[221102]: osdmap e421: 8 total, 8 up, 7 in 2023-09-23T23:27:11.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:10 smithi110 ceph-mon[221102]: failed to encode map e421 with expected crc 2023-09-23T23:27:11.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:10 smithi110 ceph-mon[221102]: failed to encode map e421 with expected crc 2023-09-23T23:27:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:11 smithi138 ceph-mon[193606]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:11 smithi138 ceph-mon[193606]: osdmap e422: 8 total, 8 up, 7 in 2023-09-23T23:27:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:11 smithi138 ceph-mon[193606]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:11 smithi138 ceph-mon[193606]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:11 smithi138 ceph-mon[193606]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:11 smithi138 ceph-mon[193606]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:11 smithi138 ceph-mon[193606]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:11 smithi110 ceph-mon[212939]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:11 smithi110 ceph-mon[212939]: osdmap e422: 8 total, 8 up, 7 in 2023-09-23T23:27:12.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:11 smithi110 ceph-mon[212939]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:11 smithi110 ceph-mon[212939]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:11 smithi110 ceph-mon[212939]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:11 smithi110 ceph-mon[212939]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:11 smithi110 ceph-mon[212939]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:11 smithi110 ceph-mon[221102]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:11 smithi110 ceph-mon[221102]: osdmap e422: 8 total, 8 up, 7 in 2023-09-23T23:27:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:11 smithi110 ceph-mon[221102]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:11 smithi110 ceph-mon[221102]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:11 smithi110 ceph-mon[221102]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:11 smithi110 ceph-mon[221102]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:11 smithi110 ceph-mon[221102]: failed to encode map e422 with expected crc 2023-09-23T23:27:12.638 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:12.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:12.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:12.747 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:27:12.791 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:27:12.810 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:12.895 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:12 smithi110 ceph-mon[212939]: pgmap v1202: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 119 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 2286/120597 objects degraded (1.896%); 20320/120597 objects misplaced (16.850%); 25 MiB/s, 392 objects/s recovering 2023-09-23T23:27:13.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:12 smithi138 ceph-mon[193606]: pgmap v1202: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 119 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 2286/120597 objects degraded (1.896%); 20320/120597 objects misplaced (16.850%); 25 MiB/s, 392 objects/s recovering 2023-09-23T23:27:13.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:12 smithi110 ceph-mon[221102]: pgmap v1202: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 119 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 2286/120597 objects degraded (1.896%); 20320/120597 objects misplaced (16.850%); 25 MiB/s, 392 objects/s recovering 2023-09-23T23:27:13.645 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:27:13.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:13.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:27:13.758 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:13.794 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:13.851 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:13.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:13.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:13.897 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:13 smithi110 ceph-mon[212939]: 1.e scrub starts 2023-09-23T23:27:13.898 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:27:13.898 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:13 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2286/120597 objects degraded (1.896%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:27:13.898 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:13 smithi110 ceph-mon[221102]: 1.e scrub starts 2023-09-23T23:27:13.898 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:27:13.898 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:13 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2286/120597 objects degraded (1.896%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:27:13.959 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:13.997 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:14.055 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:14.060 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:13 smithi138 ceph-mon[193606]: 1.e scrub starts 2023-09-23T23:27:14.060 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:27:14.060 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:13 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2286/120597 objects degraded (1.896%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:27:14.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:14.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:14.162 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:27:14.196 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:14.256 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:27:14.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:14.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:27:14.366 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:14.399 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:14.466 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:14.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:14.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:14.575 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:27:14.608 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:14.662 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:27:14.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:14.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:27:14.767 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:14.799 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:14.857 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:14.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:14.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:14.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:14 smithi138 ceph-mon[193606]: pgmap v1203: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 119 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 2286/118575 objects degraded (1.928%); 20320/118575 objects misplaced (17.137%); 15 MiB/s, 243 objects/s recovering 2023-09-23T23:27:14.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:14 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3279394223' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:27:14.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:14 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:27:14.966 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:15.060 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:15.150 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:27:15.150 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:27:15.172 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:27:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:14 smithi110 ceph-mon[212939]: pgmap v1203: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 119 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 2286/118575 objects degraded (1.928%); 20320/118575 objects misplaced (17.137%); 15 MiB/s, 243 objects/s recovering 2023-09-23T23:27:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:14 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3279394223' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:27:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:14 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:27:15.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:14 smithi110 ceph-mon[221102]: pgmap v1203: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 119 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 2286/118575 objects degraded (1.928%); 20320/118575 objects misplaced (17.137%); 15 MiB/s, 243 objects/s recovering 2023-09-23T23:27:15.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:14 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3279394223' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:27:15.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:14 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:27:15.208 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:27:15.213 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:15.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:15.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:15.246 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:27:15.273 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:27:15.295 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:27:15.324 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:27:15.328 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:27:15.356 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:27:15.356 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:27:15.356 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:27:15.356 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:27:15.356 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:27:15.356 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:27:15.356 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:27:15.356 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:27:15.357 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:15.431 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:27:15.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:15.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:27:15.540 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:15.576 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:27:15.580 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:15.640 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:15.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:15.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:15.747 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:27:15.828 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:15.898 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:15 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1341502731' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:27:15.899 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:15 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:27:15.902 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:15 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1341502731' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:27:15.902 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:15 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:27:15.902 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:15 smithi110 ceph-mon[212939]: failed to encode map e423 with expected crc 2023-09-23T23:27:15.903 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:15 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:27:15.903 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:15 smithi110 ceph-mon[212939]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T23:27:15.904 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:15 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1341502731' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:27:15.905 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:15 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:27:15.905 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:15 smithi110 ceph-mon[221102]: failed to encode map e423 with expected crc 2023-09-23T23:27:15.905 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:15 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:27:15.905 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:15 smithi110 ceph-mon[221102]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T23:27:16.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:15 smithi138 ceph-mon[193606]: failed to encode map e423 with expected crc 2023-09-23T23:27:16.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:15 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:27:16.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:15 smithi138 ceph-mon[193606]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T23:27:16.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:27:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:27:16] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.43.0" 2023-09-23T23:27:16.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:16 smithi110 ceph-mon[212939]: failed to encode map e423 with expected crc 2023-09-23T23:27:16.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:16 smithi110 ceph-mon[212939]: failed to encode map e423 with expected crc 2023-09-23T23:27:16.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:16 smithi110 ceph-mon[212939]: failed to encode map e423 with expected crc 2023-09-23T23:27:16.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:16 smithi110 ceph-mon[212939]: failed to encode map e423 with expected crc 2023-09-23T23:27:16.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:16 smithi110 ceph-mon[212939]: pgmap v1205: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 119 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 2286/112104 objects degraded (2.039%); 18366/112104 objects misplaced (16.383%); 7.4 MiB/s, 117 objects/s recovering 2023-09-23T23:27:16.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:16 smithi110 ceph-mon[221102]: failed to encode map e423 with expected crc 2023-09-23T23:27:16.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:16 smithi110 ceph-mon[221102]: failed to encode map e423 with expected crc 2023-09-23T23:27:16.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:16 smithi110 ceph-mon[221102]: failed to encode map e423 with expected crc 2023-09-23T23:27:16.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:16 smithi110 ceph-mon[221102]: failed to encode map e423 with expected crc 2023-09-23T23:27:16.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:16 smithi110 ceph-mon[221102]: pgmap v1205: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 119 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 2286/112104 objects degraded (2.039%); 18366/112104 objects misplaced (16.383%); 7.4 MiB/s, 117 objects/s recovering 2023-09-23T23:27:17.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:16 smithi138 ceph-mon[193606]: failed to encode map e423 with expected crc 2023-09-23T23:27:17.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:16 smithi138 ceph-mon[193606]: failed to encode map e423 with expected crc 2023-09-23T23:27:17.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:16 smithi138 ceph-mon[193606]: failed to encode map e423 with expected crc 2023-09-23T23:27:17.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:16 smithi138 ceph-mon[193606]: failed to encode map e423 with expected crc 2023-09-23T23:27:17.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:16 smithi138 ceph-mon[193606]: pgmap v1205: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 119 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 2286/112104 objects degraded (2.039%); 18366/112104 objects misplaced (16.383%); 7.4 MiB/s, 117 objects/s recovering 2023-09-23T23:27:17.251 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:27:17.453 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:27:17.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:17.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:27:17.567 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:17.655 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:17.924 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:17.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:17.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:18.036 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:18.096 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:18.251 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:18.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:18.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:18.357 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:27:18.398 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:18.458 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:27:18.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:18.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:18.565 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:18.598 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:18.642 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:18.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:18.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:18.747 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:27:18.773 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:18.826 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:27:18.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:18.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:18.933 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:27:18.969 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:19.036 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:27:19.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:19.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:27:19.144 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:27:19.177 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:18 smithi138 ceph-mon[193606]: pgmap v1206: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 119 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 2286/110250 objects degraded (2.073%); 17980/110250 objects misplaced (16.308%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T23:27:19.177 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:18 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2286/110250 objects degraded (2.073%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:27:19.177 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3934203221' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:27:19.177 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:18 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:27:19.179 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:19.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:18 smithi110 ceph-mon[212939]: pgmap v1206: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 119 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 2286/110250 objects degraded (2.073%); 17980/110250 objects misplaced (16.308%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T23:27:19.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:18 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2286/110250 objects degraded (2.073%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:27:19.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3934203221' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:27:19.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:18 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:27:19.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:18 smithi110 ceph-mon[221102]: pgmap v1206: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 119 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 2286/110250 objects degraded (2.073%); 17980/110250 objects misplaced (16.308%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T23:27:19.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:18 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2286/110250 objects degraded (2.073%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:27:19.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3934203221' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:27:19.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:18 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:27:19.242 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:27:19.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:19.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:27:19.290 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:27:19.353 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:27:19.470 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:27:19.501 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:19.570 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T23:27:19.640 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:27:19.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:19.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:27:19.745 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:27:19.771 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:19.814 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:27:19.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:19.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:19.920 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:19.947 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:19.990 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:19.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:19.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:20.094 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:27:20.127 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:20.275 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:27:20.275 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:27:20.275 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:27:20.276 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:27:20.276 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:27:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:20 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:27:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:20 smithi138 ceph-mon[193606]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T23:27:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:20 smithi138 ceph-mon[193606]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:20 smithi138 ceph-mon[193606]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:20 smithi138 ceph-mon[193606]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:20 smithi138 ceph-mon[193606]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:20 smithi138 ceph-mon[193606]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:20 smithi138 ceph-mon[193606]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:20 smithi138 ceph-mon[193606]: pgmap v1208: 129 pgs: 5 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 114 active+clean; 2.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 2286/103335 objects degraded (2.212%); 16937/103335 objects misplaced (16.390%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T23:27:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:27:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[212939]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T23:27:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[212939]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[212939]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[212939]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[212939]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[212939]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[212939]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[212939]: pgmap v1208: 129 pgs: 5 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 114 active+clean; 2.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 2286/103335 objects degraded (2.212%); 16937/103335 objects misplaced (16.390%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T23:27:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:27:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[221102]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T23:27:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[221102]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[221102]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[221102]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[221102]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[221102]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[221102]: failed to encode map e424 with expected crc 2023-09-23T23:27:20.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:20 smithi110 ceph-mon[221102]: pgmap v1208: 129 pgs: 5 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 114 active+clean; 2.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 2286/103335 objects degraded (2.212%); 16937/103335 objects misplaced (16.390%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T23:27:21.587 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:27:21.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:21.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:27:21.692 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:21.774 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:21 smithi110 ceph-mon[221102]: failed to encode map e425 with expected crc 2023-09-23T23:27:21.774 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:21 smithi110 ceph-mon[221102]: failed to encode map e425 with expected crc 2023-09-23T23:27:21.774 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:21 smithi110 ceph-mon[221102]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T23:27:21.774 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:21 smithi110 ceph-mon[221102]: failed to encode map e425 with expected crc 2023-09-23T23:27:21.775 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:21 smithi110 ceph-mon[221102]: failed to encode map e425 with expected crc 2023-09-23T23:27:21.775 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:21 smithi110 ceph-mon[221102]: 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 423'11061 2023-09-23T23:27:21.775 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:21 smithi110 ceph-mon[221102]: failed to encode map e425 with expected crc 2023-09-23T23:27:21.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:21 smithi110 ceph-mon[212939]: failed to encode map e425 with expected crc 2023-09-23T23:27:21.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:21 smithi110 ceph-mon[212939]: failed to encode map e425 with expected crc 2023-09-23T23:27:21.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:21 smithi110 ceph-mon[212939]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T23:27:21.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:21 smithi110 ceph-mon[212939]: failed to encode map e425 with expected crc 2023-09-23T23:27:21.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:21 smithi110 ceph-mon[212939]: failed to encode map e425 with expected crc 2023-09-23T23:27:21.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:21 smithi110 ceph-mon[212939]: 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 423'11061 2023-09-23T23:27:21.775 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:21 smithi110 ceph-mon[212939]: failed to encode map e425 with expected crc 2023-09-23T23:27:21.785 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:21 smithi138 ceph-mon[193606]: failed to encode map e425 with expected crc 2023-09-23T23:27:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:21 smithi138 ceph-mon[193606]: failed to encode map e425 with expected crc 2023-09-23T23:27:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:21 smithi138 ceph-mon[193606]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T23:27:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:21 smithi138 ceph-mon[193606]: failed to encode map e425 with expected crc 2023-09-23T23:27:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:21 smithi138 ceph-mon[193606]: failed to encode map e425 with expected crc 2023-09-23T23:27:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:21 smithi138 ceph-mon[193606]: 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 423'11061 2023-09-23T23:27:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:21 smithi138 ceph-mon[193606]: failed to encode map e425 with expected crc 2023-09-23T23:27:22.213 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T23:27:22.214 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:27:22.383 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:22.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:22.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:22.500 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:27:22.533 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:22.579 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:27:22.592 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:27:22.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:22.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:27:22.703 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:22.727 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:27:22.739 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:22.778 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:27:22.791 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:22.795 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:22 smithi138 ceph-mon[193606]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.795 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:22 smithi138 ceph-mon[193606]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T23:27:22.795 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:22 smithi138 ceph-mon[193606]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.795 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:22 smithi138 ceph-mon[193606]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.795 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:22 smithi138 ceph-mon[193606]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.795 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:22 smithi138 ceph-mon[193606]: 11.b starting backfill to osd.1 from (0'0,0'0] MAX to 423'11268 2023-09-23T23:27:22.795 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:22 smithi138 ceph-mon[193606]: 11.16 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8787 2023-09-23T23:27:22.796 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:22 smithi138 ceph-mon[193606]: 11.6 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8785 2023-09-23T23:27:22.796 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:22 smithi138 ceph-mon[193606]: 11.7 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8712 2023-09-23T23:27:22.796 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:22 smithi138 ceph-mon[193606]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.796 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:22 smithi138 ceph-mon[193606]: 11.19 starting backfill to osd.1 from (0'0,0'0] MAX to 423'10498 2023-09-23T23:27:22.796 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:22 smithi138 ceph-mon[193606]: 11.9 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8786 2023-09-23T23:27:22.796 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:22 smithi138 ceph-mon[193606]: 11.17 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8783 2023-09-23T23:27:22.796 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:22 smithi138 ceph-mon[193606]: 11.13 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8854 2023-09-23T23:27:22.796 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:22 smithi138 ceph-mon[193606]: 11.3 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8803 2023-09-23T23:27:22.796 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:22 smithi138 ceph-mon[193606]: pgmap v1211: 129 pgs: 4 remapped+peering, 5 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 110 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 2286/100173 objects degraded (2.282%); 16093/100173 objects misplaced (16.065%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:27:22.796 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:22 smithi138 ceph-mon[193606]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:22.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:22.898 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:27:22.930 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:22.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[212939]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[212939]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T23:27:22.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[212939]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[212939]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[212939]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[212939]: 11.b starting backfill to osd.1 from (0'0,0'0] MAX to 423'11268 2023-09-23T23:27:22.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[212939]: 11.16 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8787 2023-09-23T23:27:22.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[212939]: 11.6 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8785 2023-09-23T23:27:22.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[212939]: 11.7 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8712 2023-09-23T23:27:22.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[212939]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[212939]: 11.19 starting backfill to osd.1 from (0'0,0'0] MAX to 423'10498 2023-09-23T23:27:22.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[212939]: 11.9 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8786 2023-09-23T23:27:22.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[212939]: 11.17 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8783 2023-09-23T23:27:22.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[212939]: 11.13 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8854 2023-09-23T23:27:22.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[212939]: 11.3 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8803 2023-09-23T23:27:22.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[212939]: pgmap v1211: 129 pgs: 4 remapped+peering, 5 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 110 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 2286/100173 objects degraded (2.282%); 16093/100173 objects misplaced (16.065%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:27:22.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[212939]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[221102]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[221102]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T23:27:22.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[221102]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[221102]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[221102]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[221102]: 11.b starting backfill to osd.1 from (0'0,0'0] MAX to 423'11268 2023-09-23T23:27:22.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[221102]: 11.16 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8787 2023-09-23T23:27:22.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[221102]: 11.6 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8785 2023-09-23T23:27:22.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[221102]: 11.7 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8712 2023-09-23T23:27:22.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[221102]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[221102]: 11.19 starting backfill to osd.1 from (0'0,0'0] MAX to 423'10498 2023-09-23T23:27:22.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[221102]: 11.9 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8786 2023-09-23T23:27:22.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[221102]: 11.17 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8783 2023-09-23T23:27:22.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[221102]: 11.13 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8854 2023-09-23T23:27:22.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[221102]: 11.3 starting backfill to osd.1 from (0'0,0'0] MAX to 423'8803 2023-09-23T23:27:22.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[221102]: pgmap v1211: 129 pgs: 4 remapped+peering, 5 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 110 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 2286/100173 objects degraded (2.282%); 16093/100173 objects misplaced (16.065%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:27:22.958 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:22 smithi110 ceph-mon[221102]: failed to encode map e426 with expected crc 2023-09-23T23:27:22.984 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:27:22.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:22.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:27:23.094 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:23.210 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:23.406 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:23.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:23.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:23.491 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:27:23.515 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:23.548 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:23.610 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:23.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:23.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:23.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:23 smithi110 ceph-mon[221102]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T23:27:23.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:23 smithi110 ceph-mon[221102]: failed to encode map e427 with expected crc 2023-09-23T23:27:23.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:23 smithi110 ceph-mon[221102]: failed to encode map e427 with expected crc 2023-09-23T23:27:23.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:23 smithi110 ceph-mon[221102]: failed to encode map e427 with expected crc 2023-09-23T23:27:23.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:23 smithi110 ceph-mon[221102]: 1.e scrub starts 2023-09-23T23:27:23.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:23 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2286/100173 objects degraded (2.282%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:27:23.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:23 smithi110 ceph-mon[221102]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T23:27:23.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:23 smithi110 ceph-mon[212939]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T23:27:23.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:23 smithi110 ceph-mon[212939]: failed to encode map e427 with expected crc 2023-09-23T23:27:23.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:23 smithi110 ceph-mon[212939]: failed to encode map e427 with expected crc 2023-09-23T23:27:23.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:23 smithi110 ceph-mon[212939]: failed to encode map e427 with expected crc 2023-09-23T23:27:23.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:23 smithi110 ceph-mon[212939]: 1.e scrub starts 2023-09-23T23:27:23.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:23 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2286/100173 objects degraded (2.282%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:27:23.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:23 smithi110 ceph-mon[212939]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T23:27:23.720 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:27:23.793 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:23 smithi138 ceph-mon[193606]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T23:27:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:23 smithi138 ceph-mon[193606]: failed to encode map e427 with expected crc 2023-09-23T23:27:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:23 smithi138 ceph-mon[193606]: failed to encode map e427 with expected crc 2023-09-23T23:27:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:23 smithi138 ceph-mon[193606]: failed to encode map e427 with expected crc 2023-09-23T23:27:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:23 smithi138 ceph-mon[193606]: 1.e scrub starts 2023-09-23T23:27:23.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:23 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2286/100173 objects degraded (2.282%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:27:23.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:23 smithi138 ceph-mon[193606]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T23:27:24.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:24 smithi138 ceph-mon[193606]: failed to encode map e428 with expected crc 2023-09-23T23:27:24.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:24 smithi138 ceph-mon[193606]: failed to encode map e428 with expected crc 2023-09-23T23:27:24.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:24 smithi138 ceph-mon[193606]: failed to encode map e428 with expected crc 2023-09-23T23:27:24.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:24 smithi138 ceph-mon[193606]: pgmap v1214: 129 pgs: 4 remapped+peering, 5 peering, 2 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 108 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 359 op/s; 1173/99021 objects degraded (1.185%); 18985/99021 objects misplaced (19.173%); 160 KiB/s, 2 objects/s recovering 2023-09-23T23:27:24.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:24 smithi110 ceph-mon[212939]: failed to encode map e428 with expected crc 2023-09-23T23:27:24.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:24 smithi110 ceph-mon[212939]: failed to encode map e428 with expected crc 2023-09-23T23:27:24.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:24 smithi110 ceph-mon[212939]: failed to encode map e428 with expected crc 2023-09-23T23:27:24.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:24 smithi110 ceph-mon[212939]: pgmap v1214: 129 pgs: 4 remapped+peering, 5 peering, 2 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 108 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 359 op/s; 1173/99021 objects degraded (1.185%); 18985/99021 objects misplaced (19.173%); 160 KiB/s, 2 objects/s recovering 2023-09-23T23:27:24.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:24 smithi110 ceph-mon[221102]: failed to encode map e428 with expected crc 2023-09-23T23:27:24.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:24 smithi110 ceph-mon[221102]: failed to encode map e428 with expected crc 2023-09-23T23:27:24.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:24 smithi110 ceph-mon[221102]: failed to encode map e428 with expected crc 2023-09-23T23:27:24.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:24 smithi110 ceph-mon[221102]: pgmap v1214: 129 pgs: 4 remapped+peering, 5 peering, 2 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 108 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 359 op/s; 1173/99021 objects degraded (1.185%); 18985/99021 objects misplaced (19.173%); 160 KiB/s, 2 objects/s recovering 2023-09-23T23:27:25.357 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:27:25.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:25.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:27:25.462 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:25.494 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:25.555 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:25.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:25.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:25.661 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:27:25.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[221102]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[221102]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[221102]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[221102]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[221102]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T23:27:25.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[221102]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[221102]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[221102]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[221102]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[212939]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[212939]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[212939]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[212939]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[212939]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T23:27:25.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[212939]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[212939]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[212939]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:25 smithi110 ceph-mon[212939]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.737 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:25 smithi138 ceph-mon[193606]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:25 smithi138 ceph-mon[193606]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:25 smithi138 ceph-mon[193606]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:25 smithi138 ceph-mon[193606]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:25 smithi138 ceph-mon[193606]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T23:27:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:25 smithi138 ceph-mon[193606]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:25 smithi138 ceph-mon[193606]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:25 smithi138 ceph-mon[193606]: failed to encode map e429 with expected crc 2023-09-23T23:27:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:25 smithi138 ceph-mon[193606]: failed to encode map e429 with expected crc 2023-09-23T23:27:26.179 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:27:26.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:26.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:27:26.285 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:27:26.335 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:26.396 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:27:26.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:26.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:27:26.500 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:27:26.526 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:26.586 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:27:26.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:26.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:26.694 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:26.736 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:26.797 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:26.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:26.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:26 smithi138 ceph-mon[193606]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:26 smithi138 ceph-mon[193606]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T23:27:26.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:26 smithi138 ceph-mon[193606]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:26 smithi138 ceph-mon[193606]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:26 smithi138 ceph-mon[193606]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:26 smithi138 ceph-mon[193606]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:26 smithi138 ceph-mon[193606]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:26 smithi138 ceph-mon[193606]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:26 smithi138 ceph-mon[193606]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:26 smithi138 ceph-mon[193606]: pgmap v1217: 129 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 111 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 1155/95241 objects degraded (1.213%); 23763/95241 objects misplaced (24.950%); 24 MiB/s, 438 objects/s recovering 2023-09-23T23:27:26.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:26.906 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:26.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[221102]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[221102]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T23:27:26.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[221102]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[221102]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[221102]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[221102]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[221102]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[221102]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[221102]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[221102]: pgmap v1217: 129 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 111 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 1155/95241 objects degraded (1.213%); 23763/95241 objects misplaced (24.950%); 24 MiB/s, 438 objects/s recovering 2023-09-23T23:27:26.955 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:27:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:27:26] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.43.0" 2023-09-23T23:27:26.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[212939]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[212939]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T23:27:26.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[212939]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[212939]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[212939]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[212939]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[212939]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[212939]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.961 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[212939]: failed to encode map e430 with expected crc 2023-09-23T23:27:26.962 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:26 smithi110 ceph-mon[212939]: pgmap v1217: 129 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 111 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 1155/95241 objects degraded (1.213%); 23763/95241 objects misplaced (24.950%); 24 MiB/s, 438 objects/s recovering 2023-09-23T23:27:26.995 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:27.129 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:27.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:27.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:27.242 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:27.334 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:27.492 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:27:27.601 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:27.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:27.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:27.710 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:27.742 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:27.811 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:27.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:27.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:27.919 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:27:27.962 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:28.029 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:27:28.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:28.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:27:28.138 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:27:28.156 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:27:28.201 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:27:28.243 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:28.263 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:27:28.300 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:27:28.342 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:27:28.382 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:27:28.386 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:27:28.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:28.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:27:28.424 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:27:28.464 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:27:28.489 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:27:28.489 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:27:28.489 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:27:28.489 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:27:28.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:27:28.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:27:28.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:27:28.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:27:28.502 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:27:28.593 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:28.637 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:27:28.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:28.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:27:28.741 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:28.766 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:28.812 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:28.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:28.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:28.916 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:27:28.948 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:29.008 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:27:29.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:29.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:27:29.114 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:29.144 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:29 smithi110 ceph-mon[212939]: pgmap v1218: 129 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 111 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 394 op/s; 1155/94110 objects degraded (1.227%); 23529/94110 objects misplaced (25.002%); 19 MiB/s, 342 objects/s recovering 2023-09-23T23:27:29.144 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:29 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:27:29.144 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:29 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:27:29.144 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:29 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2660604304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:27:29.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:29 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1155/94110 objects degraded (1.227%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:27:29.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:29 smithi110 ceph-mon[212939]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T23:27:29.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:29 smithi110 ceph-mon[212939]: failed to encode map e431 with expected crc 2023-09-23T23:27:29.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:29 smithi110 ceph-mon[221102]: pgmap v1218: 129 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 111 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 394 op/s; 1155/94110 objects degraded (1.227%); 23529/94110 objects misplaced (25.002%); 19 MiB/s, 342 objects/s recovering 2023-09-23T23:27:29.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:29 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:27:29.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:29 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:27:29.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:29 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2660604304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:27:29.147 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:29 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1155/94110 objects degraded (1.227%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:27:29.147 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:29 smithi110 ceph-mon[221102]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T23:27:29.147 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:29 smithi110 ceph-mon[221102]: failed to encode map e431 with expected crc 2023-09-23T23:27:29.163 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:29 smithi138 ceph-mon[193606]: pgmap v1218: 129 pgs: 3 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 111 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 394 op/s; 1155/94110 objects degraded (1.227%); 23529/94110 objects misplaced (25.002%); 19 MiB/s, 342 objects/s recovering 2023-09-23T23:27:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:29 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:27:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:29 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:27:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:29 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2660604304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:27:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:29 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1155/94110 objects degraded (1.227%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:27:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:29 smithi138 ceph-mon[193606]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T23:27:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:29 smithi138 ceph-mon[193606]: failed to encode map e431 with expected crc 2023-09-23T23:27:30.401 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:30.404 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:30 smithi110 ceph-mon[212939]: failed to encode map e431 with expected crc 2023-09-23T23:27:30.404 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:30 smithi110 ceph-mon[212939]: failed to encode map e431 with expected crc 2023-09-23T23:27:30.404 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:30 smithi110 ceph-mon[212939]: failed to encode map e431 with expected crc 2023-09-23T23:27:30.404 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:30 smithi110 ceph-mon[212939]: failed to encode map e431 with expected crc 2023-09-23T23:27:30.404 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:30 smithi110 ceph-mon[212939]: osdmap e432: 8 total, 8 up, 7 in 2023-09-23T23:27:30.404 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:30 smithi110 ceph-mon[221102]: failed to encode map e431 with expected crc 2023-09-23T23:27:30.404 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:30 smithi110 ceph-mon[221102]: failed to encode map e431 with expected crc 2023-09-23T23:27:30.405 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:30 smithi110 ceph-mon[221102]: failed to encode map e431 with expected crc 2023-09-23T23:27:30.405 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:30 smithi110 ceph-mon[221102]: failed to encode map e431 with expected crc 2023-09-23T23:27:30.405 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:30 smithi110 ceph-mon[221102]: osdmap e432: 8 total, 8 up, 7 in 2023-09-23T23:27:30.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:30.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:30.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:30 smithi138 ceph-mon[193606]: failed to encode map e431 with expected crc 2023-09-23T23:27:30.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:30 smithi138 ceph-mon[193606]: failed to encode map e431 with expected crc 2023-09-23T23:27:30.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:30 smithi138 ceph-mon[193606]: failed to encode map e431 with expected crc 2023-09-23T23:27:30.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:30 smithi138 ceph-mon[193606]: failed to encode map e431 with expected crc 2023-09-23T23:27:30.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:30 smithi138 ceph-mon[193606]: osdmap e432: 8 total, 8 up, 7 in 2023-09-23T23:27:30.507 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:27:30.545 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:30.574 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:27:30.633 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:27:30.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:30.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:27:30.738 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:30.815 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:30.972 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:30.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:30.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:31.077 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:31.109 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:31.172 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:31.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:31.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:31.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[212939]: failed to encode map e432 with expected crc 2023-09-23T23:27:31.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[212939]: failed to encode map e432 with expected crc 2023-09-23T23:27:31.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[212939]: pgmap v1221: 129 pgs: 1 activating, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 112 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 1155/89130 objects degraded (1.296%); 21172/89130 objects misplaced (23.754%); 31 MiB/s, 528 objects/s recovering 2023-09-23T23:27:31.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[212939]: 1.f scrub starts 2023-09-23T23:27:31.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/415481330' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:27:31.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:27:31.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:27:31.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:27:31.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[212939]: osdmap e433: 8 total, 8 up, 7 in 2023-09-23T23:27:31.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[212939]: failed to encode map e433 with expected crc 2023-09-23T23:27:31.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[221102]: failed to encode map e432 with expected crc 2023-09-23T23:27:31.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[221102]: failed to encode map e432 with expected crc 2023-09-23T23:27:31.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[221102]: pgmap v1221: 129 pgs: 1 activating, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 112 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 1155/89130 objects degraded (1.296%); 21172/89130 objects misplaced (23.754%); 31 MiB/s, 528 objects/s recovering 2023-09-23T23:27:31.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[221102]: 1.f scrub starts 2023-09-23T23:27:31.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/415481330' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:27:31.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:27:31.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:27:31.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:27:31.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[221102]: osdmap e433: 8 total, 8 up, 7 in 2023-09-23T23:27:31.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:31 smithi110 ceph-mon[221102]: failed to encode map e433 with expected crc 2023-09-23T23:27:31.278 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:31.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:31 smithi138 ceph-mon[193606]: failed to encode map e432 with expected crc 2023-09-23T23:27:31.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:31 smithi138 ceph-mon[193606]: failed to encode map e432 with expected crc 2023-09-23T23:27:31.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:31 smithi138 ceph-mon[193606]: pgmap v1221: 129 pgs: 1 activating, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 112 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 1155/89130 objects degraded (1.296%); 21172/89130 objects misplaced (23.754%); 31 MiB/s, 528 objects/s recovering 2023-09-23T23:27:31.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:31 smithi138 ceph-mon[193606]: 1.f scrub starts 2023-09-23T23:27:31.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:31 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/415481330' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:27:31.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:31 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:27:31.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:31 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:27:31.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:31 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:27:31.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:31 smithi138 ceph-mon[193606]: osdmap e433: 8 total, 8 up, 7 in 2023-09-23T23:27:31.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:31 smithi138 ceph-mon[193606]: failed to encode map e433 with expected crc 2023-09-23T23:27:31.311 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:31.360 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:31.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:31.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:31.467 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:31.493 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:31.562 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:31.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:31.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:31.670 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:27:31.745 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:32.034 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:32 smithi110 ceph-mon[212939]: failed to encode map e433 with expected crc 2023-09-23T23:27:32.034 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:32 smithi110 ceph-mon[212939]: failed to encode map e433 with expected crc 2023-09-23T23:27:32.034 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:32 smithi110 ceph-mon[212939]: failed to encode map e433 with expected crc 2023-09-23T23:27:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:32 smithi138 ceph-mon[193606]: failed to encode map e433 with expected crc 2023-09-23T23:27:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:32 smithi138 ceph-mon[193606]: failed to encode map e433 with expected crc 2023-09-23T23:27:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:32 smithi138 ceph-mon[193606]: failed to encode map e433 with expected crc 2023-09-23T23:27:32.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:32 smithi110 ceph-mon[221102]: failed to encode map e433 with expected crc 2023-09-23T23:27:32.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:32 smithi110 ceph-mon[221102]: failed to encode map e433 with expected crc 2023-09-23T23:27:32.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:32 smithi110 ceph-mon[221102]: failed to encode map e433 with expected crc 2023-09-23T23:27:32.507 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:27:32.794 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:27:32.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:32.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:27:32.911 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:27:32.938 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:32.999 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:27:33.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:33.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:33.105 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:27:33.137 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:33.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:33 smithi138 ceph-mon[193606]: pgmap v1223: 129 pgs: 1 activating, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 112 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 1155/86136 objects degraded (1.341%); 19226/86136 objects misplaced (22.321%); 25 MiB/s, 400 objects/s recovering 2023-09-23T23:27:33.193 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:27:33.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:33.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:33.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:33 smithi110 ceph-mon[212939]: pgmap v1223: 129 pgs: 1 activating, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 112 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 1155/86136 objects degraded (1.341%); 19226/86136 objects misplaced (22.321%); 25 MiB/s, 400 objects/s recovering 2023-09-23T23:27:33.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:33 smithi110 ceph-mon[221102]: pgmap v1223: 129 pgs: 1 activating, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 112 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 1155/86136 objects degraded (1.341%); 19226/86136 objects misplaced (22.321%); 25 MiB/s, 400 objects/s recovering 2023-09-23T23:27:33.300 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:33.374 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:33.504 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:27:33.529 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:33.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:33.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:33.579 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:27:33.587 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:27:33.637 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:27:33.664 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:27:33.664 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:27:33.689 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:34 smithi138 ceph-mon[193606]: osdmap e434: 8 total, 8 up, 7 in 2023-09-23T23:27:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:34 smithi138 ceph-mon[193606]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:34 smithi138 ceph-mon[193606]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:34 smithi138 ceph-mon[193606]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:34 smithi138 ceph-mon[193606]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:34 smithi138 ceph-mon[193606]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:34 smithi138 ceph-mon[193606]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:34 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1155/86136 objects degraded (1.341%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:27:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:34 smithi110 ceph-mon[212939]: osdmap e434: 8 total, 8 up, 7 in 2023-09-23T23:27:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:34 smithi110 ceph-mon[212939]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:34 smithi110 ceph-mon[212939]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:34 smithi110 ceph-mon[212939]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:34 smithi110 ceph-mon[212939]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:34 smithi110 ceph-mon[212939]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:34 smithi110 ceph-mon[212939]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:34 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1155/86136 objects degraded (1.341%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:27:34.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:34 smithi110 ceph-mon[221102]: osdmap e434: 8 total, 8 up, 7 in 2023-09-23T23:27:34.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:34 smithi110 ceph-mon[221102]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:34 smithi110 ceph-mon[221102]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:34 smithi110 ceph-mon[221102]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:34 smithi110 ceph-mon[221102]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:34 smithi110 ceph-mon[221102]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:34 smithi110 ceph-mon[221102]: failed to encode map e434 with expected crc 2023-09-23T23:27:34.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:34 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1155/86136 objects degraded (1.341%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:27:34.907 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:27:35.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:35 smithi110 ceph-mon[212939]: from='client.36096 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:27:35.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:35 smithi110 ceph-mon[212939]: pgmap v1225: 129 pgs: 1 activating, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 112 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 1155/84567 objects degraded (1.366%); 18510/84567 objects misplaced (21.888%); 31 MiB/s, 497 objects/s recovering 2023-09-23T23:27:35.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:35 smithi110 ceph-mon[212939]: failed to encode map e435 with expected crc 2023-09-23T23:27:35.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:35 smithi110 ceph-mon[212939]: osdmap e435: 8 total, 8 up, 7 in 2023-09-23T23:27:35.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:35 smithi110 ceph-mon[221102]: from='client.36096 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:27:35.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:35 smithi110 ceph-mon[221102]: pgmap v1225: 129 pgs: 1 activating, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 112 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 1155/84567 objects degraded (1.366%); 18510/84567 objects misplaced (21.888%); 31 MiB/s, 497 objects/s recovering 2023-09-23T23:27:35.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:35 smithi110 ceph-mon[221102]: failed to encode map e435 with expected crc 2023-09-23T23:27:35.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:35 smithi110 ceph-mon[221102]: osdmap e435: 8 total, 8 up, 7 in 2023-09-23T23:27:35.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:35 smithi138 ceph-mon[193606]: from='client.36096 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:27:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:35 smithi138 ceph-mon[193606]: pgmap v1225: 129 pgs: 1 activating, 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 112 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 1155/84567 objects degraded (1.366%); 18510/84567 objects misplaced (21.888%); 31 MiB/s, 497 objects/s recovering 2023-09-23T23:27:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:35 smithi138 ceph-mon[193606]: failed to encode map e435 with expected crc 2023-09-23T23:27:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:35 smithi138 ceph-mon[193606]: osdmap e435: 8 total, 8 up, 7 in 2023-09-23T23:27:35.891 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1224,"stamp":"2023-09-23T23:27:33.043158+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1880483917,"num_objects":28712,"num_object_clones":0,"num_object_copies":86136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1155,"num_objects_misplaced":19226,"num_objects_unfound":0,"num_objects_dirty":28712,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":159882,"num_write_kb":5311354,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58095,"num_bytes_recovered":3782905907,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":169424,"ondisk_log_size":169424,"up":387,"acting":386,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":425,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11237412,"kb_used_data":6979820,"kb_used_omap":29,"kb_used_meta":4257506,"kb_avail":644978652,"statfs":{"total":671965249536,"available":660458139648,"internally_reserved":0,"allocated":7147335680,"data_stored":7135537150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":4359686867},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,4,3,7,0,0,1,0,3],"upper_bound":1024},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-174194688,"num_objects":-2658,"num_object_clones":0,"num_object_copies":-7974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-4303,"num_objects_unfound":0,"num_objects_dirty":-2658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2658,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2402,"num_bytes_recovered":157417472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.350698"},"pg_stats":[{"pgid":"11.12","version":"433'9023","reported_seq":13056,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-09-23T23:27:30.837865+0000","last_change":"2023-09-23T23:26:42.110093+0000","last_active":"2023-09-23T23:27:30.837865+0000","last_peered":"2023-09-23T23:27:30.837865+0000","last_clean":"2023-09-23T23:27:30.837865+0000","last_became_active":"2023-09-23T23:26:42.095788+0000","last_became_peered":"2023-09-23T23:26:42.095788+0000","last_unstale":"2023-09-23T23:27:30.837865+0000","last_undegraded":"2023-09-23T23:27:30.837865+0000","last_fullsized":"2023-09-23T23:27:30.837865+0000","mapping_epoch":404,"log_start":"355'800","ondisk_log_start":"355'800","created":365,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":8223,"log_dups_size":0,"ondisk_log_size":8223,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62291968,"num_objects":951,"num_object_clones":0,"num_object_copies":2853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5777,"num_write_kb":215264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1954,"num_bytes_recovered":128057344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":617,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.595669+0000","last_change":"2023-09-23T23:26:50.557273+0000","last_active":"2023-09-23T23:27:25.595669+0000","last_peered":"2023-09-23T23:27:25.595669+0000","last_clean":"2023-09-23T23:27:25.595669+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:27:25.595669+0000","last_undegraded":"2023-09-23T23:27:25.595669+0000","last_fullsized":"2023-09-23T23:27:25.595669+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.058755+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.058755+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023428590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":466,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.543830+0000","last_change":"2023-09-23T23:22:38.439629+0000","last_active":"2023-09-23T23:27:28.543830+0000","last_peered":"2023-09-23T23:27:28.543830+0000","last_clean":"2023-09-23T23:27:28.543830+0000","last_became_active":"2023-09-23T23:22:38.439285+0000","last_became_peered":"2023-09-23T23:22:38.439285+0000","last_unstale":"2023-09-23T23:27:28.543830+0000","last_undegraded":"2023-09-23T23:27:28.543830+0000","last_fullsized":"2023-09-23T23:27:28.543830+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:34.431221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":400,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.583457+0000","last_change":"2023-09-23T23:23:43.511912+0000","last_active":"2023-09-23T23:27:25.583457+0000","last_peered":"2023-09-23T23:27:25.583457+0000","last_clean":"2023-09-23T23:27:25.583457+0000","last_became_active":"2023-09-23T23:23:43.511574+0000","last_became_peered":"2023-09-23T23:23:43.511574+0000","last_unstale":"2023-09-23T23:27:25.583457+0000","last_undegraded":"2023-09-23T23:27:25.583457+0000","last_fullsized":"2023-09-23T23:27:25.583457+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:16.286926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"433'8940","reported_seq":15243,"reported_epoch":433,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T23:27:30.949759+0000","last_change":"2023-09-23T23:27:28.849679+0000","last_active":"2023-09-23T23:27:30.949759+0000","last_peered":"2023-09-23T23:27:30.949759+0000","last_clean":"2023-09-23T23:25:29.511093+0000","last_became_active":"2023-09-23T23:26:48.602256+0000","last_became_peered":"2023-09-23T23:26:48.602256+0000","last_unstale":"2023-09-23T23:27:30.949759+0000","last_undegraded":"2023-09-23T23:27:30.949759+0000","last_fullsized":"2023-09-23T23:27:30.949759+0000","mapping_epoch":410,"log_start":"355'1400","ondisk_log_start":"355'1400","created":365,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":7540,"log_dups_size":0,"ondisk_log_size":7540,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57769984,"num_objects":882,"num_object_clones":0,"num_object_copies":2646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":268,"num_objects_unfound":0,"num_objects_dirty":882,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5738,"num_write_kb":211808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2274,"num_bytes_recovered":149028864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,6,0],"avail_no_missing":["2","0","5","6"],"object_location_counts":[{"shards":"0,2,6","objects":882}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":520,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.595718+0000","last_change":"2023-09-23T23:23:50.459634+0000","last_active":"2023-09-23T23:27:25.595718+0000","last_peered":"2023-09-23T23:27:25.595718+0000","last_clean":"2023-09-23T23:27:25.595718+0000","last_became_active":"2023-09-23T23:23:45.453958+0000","last_became_peered":"2023-09-23T23:23:45.453958+0000","last_unstale":"2023-09-23T23:27:25.595718+0000","last_undegraded":"2023-09-23T23:27:25.595718+0000","last_fullsized":"2023-09-23T23:27:25.595718+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:56.563201+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":142,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.555841+0000","last_change":"2023-09-23T23:26:11.528846+0000","last_active":"2023-09-23T23:27:29.555841+0000","last_peered":"2023-09-23T23:27:29.555841+0000","last_clean":"2023-09-23T23:27:29.555841+0000","last_became_active":"2023-09-23T23:26:11.528529+0000","last_became_peered":"2023-09-23T23:26:11.528529+0000","last_unstale":"2023-09-23T23:27:29.555841+0000","last_undegraded":"2023-09-23T23:27:29.555841+0000","last_fullsized":"2023-09-23T23:27:29.555841+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:35.094583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":451,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.637445+0000","last_change":"2023-09-23T23:26:48.558967+0000","last_active":"2023-09-23T23:27:29.637445+0000","last_peered":"2023-09-23T23:27:29.637445+0000","last_clean":"2023-09-23T23:27:29.637445+0000","last_became_active":"2023-09-23T23:26:48.555271+0000","last_became_peered":"2023-09-23T23:26:48.555271+0000","last_unstale":"2023-09-23T23:27:29.637445+0000","last_undegraded":"2023-09-23T23:27:29.637445+0000","last_fullsized":"2023-09-23T23:27:29.637445+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:50:57.509734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"433'8856","reported_seq":22351,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-09-23T23:27:30.569532+0000","last_change":"2023-09-23T23:27:24.606995+0000","last_active":"2023-09-23T23:27:30.569532+0000","last_peered":"2023-09-23T23:27:30.569532+0000","last_clean":"2023-09-23T23:27:30.569532+0000","last_became_active":"2023-09-23T23:27:24.589817+0000","last_became_peered":"2023-09-23T23:27:24.589817+0000","last_unstale":"2023-09-23T23:27:30.569532+0000","last_undegraded":"2023-09-23T23:27:30.569532+0000","last_fullsized":"2023-09-23T23:27:30.569532+0000","mapping_epoch":428,"log_start":"356'2537","ondisk_log_start":"356'2537","created":365,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"366'6410","last_scrub_stamp":"2023-09-23T23:25:21.346991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:21.346991+0000","objects_scrubbed":3231,"log_size":6319,"log_dups_size":0,"ondisk_log_size":6319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:37.469662+0000","scrub_duration":0.84688343099999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55705600,"num_objects":850,"num_object_clones":0,"num_object_copies":2550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2461,"num_write_kb":2560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3371,"num_bytes_recovered":208666624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":722,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.543707+0000","last_change":"2023-09-23T23:27:23.551379+0000","last_active":"2023-09-23T23:27:28.543707+0000","last_peered":"2023-09-23T23:27:28.543707+0000","last_clean":"2023-09-23T23:27:28.543707+0000","last_became_active":"2023-09-23T23:27:23.551046+0000","last_became_peered":"2023-09-23T23:27:23.551046+0000","last_unstale":"2023-09-23T23:27:28.543707+0000","last_undegraded":"2023-09-23T23:27:28.543707+0000","last_fullsized":"2023-09-23T23:27:28.543707+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.842272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.842272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.023100959000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":427,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.582977+0000","last_change":"2023-09-23T23:27:20.554007+0000","last_active":"2023-09-23T23:27:25.582977+0000","last_peered":"2023-09-23T23:27:25.582977+0000","last_clean":"2023-09-23T23:27:25.582977+0000","last_became_active":"2023-09-23T23:27:20.553716+0000","last_became_peered":"2023-09-23T23:27:20.553716+0000","last_unstale":"2023-09-23T23:27:25.582977+0000","last_undegraded":"2023-09-23T23:27:25.582977+0000","last_fullsized":"2023-09-23T23:27:25.582977+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:38.911862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":364,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.540182+0000","last_change":"2023-09-23T23:25:31.735427+0000","last_active":"2023-09-23T23:27:28.540182+0000","last_peered":"2023-09-23T23:27:28.540182+0000","last_clean":"2023-09-23T23:27:28.540182+0000","last_became_active":"2023-09-23T23:25:31.734582+0000","last_became_peered":"2023-09-23T23:25:31.734582+0000","last_unstale":"2023-09-23T23:27:28.540182+0000","last_undegraded":"2023-09-23T23:27:28.540182+0000","last_fullsized":"2023-09-23T23:27:28.540182+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:39.075149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":750,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.586778+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:27:24.586778+0000","last_peered":"2023-09-23T23:27:24.586778+0000","last_clean":"2023-09-23T23:27:24.586778+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:27:24.586778+0000","last_undegraded":"2023-09-23T23:27:24.586778+0000","last_fullsized":"2023-09-23T23:27:24.586778+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":386,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.638397+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:27:29.638397+0000","last_peered":"2023-09-23T23:27:29.638397+0000","last_clean":"2023-09-23T23:27:29.638397+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:27:29.638397+0000","last_undegraded":"2023-09-23T23:27:29.638397+0000","last_fullsized":"2023-09-23T23:27:29.638397+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":419,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.582743+0000","last_change":"2023-09-23T23:26:11.528354+0000","last_active":"2023-09-23T23:27:25.582743+0000","last_peered":"2023-09-23T23:27:25.582743+0000","last_clean":"2023-09-23T23:27:25.582743+0000","last_became_active":"2023-09-23T23:26:11.528076+0000","last_became_peered":"2023-09-23T23:26:11.528076+0000","last_unstale":"2023-09-23T23:27:25.582743+0000","last_undegraded":"2023-09-23T23:27:25.582743+0000","last_fullsized":"2023-09-23T23:27:25.582743+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:16.100396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":705,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.639527+0000","last_change":"2023-09-23T23:26:14.521792+0000","last_active":"2023-09-23T23:27:29.639527+0000","last_peered":"2023-09-23T23:27:29.639527+0000","last_clean":"2023-09-23T23:27:29.639527+0000","last_became_active":"2023-09-23T23:26:14.521438+0000","last_became_peered":"2023-09-23T23:26:14.521438+0000","last_unstale":"2023-09-23T23:27:29.639527+0000","last_undegraded":"2023-09-23T23:27:29.639527+0000","last_fullsized":"2023-09-23T23:27:29.639527+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:09.103712+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":62,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.596554+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:27:25.596554+0000","last_peered":"2023-09-23T23:27:25.596554+0000","last_clean":"2023-09-23T23:27:25.596554+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:27:25.596554+0000","last_undegraded":"2023-09-23T23:27:25.596554+0000","last_fullsized":"2023-09-23T23:27:25.596554+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":380,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.583394+0000","last_change":"2023-09-23T23:26:48.538059+0000","last_active":"2023-09-23T23:27:25.583394+0000","last_peered":"2023-09-23T23:27:25.583394+0000","last_clean":"2023-09-23T23:27:25.583394+0000","last_became_active":"2023-09-23T23:26:48.537735+0000","last_became_peered":"2023-09-23T23:26:48.537735+0000","last_unstale":"2023-09-23T23:27:25.583394+0000","last_undegraded":"2023-09-23T23:27:25.583394+0000","last_fullsized":"2023-09-23T23:27:25.583394+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:32.878591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":815,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.596139+0000","last_change":"2023-09-23T23:26:51.563894+0000","last_active":"2023-09-23T23:27:25.596139+0000","last_peered":"2023-09-23T23:27:25.596139+0000","last_clean":"2023-09-23T23:27:25.596139+0000","last_became_active":"2023-09-23T23:26:51.563536+0000","last_became_peered":"2023-09-23T23:26:51.563536+0000","last_unstale":"2023-09-23T23:27:25.596139+0000","last_undegraded":"2023-09-23T23:27:25.596139+0000","last_fullsized":"2023-09-23T23:27:25.596139+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:54.095054+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:54.095054+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":35,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.586065+0000","last_change":"2023-09-23T23:26:48.669857+0000","last_active":"2023-09-23T23:27:24.586065+0000","last_peered":"2023-09-23T23:27:24.586065+0000","last_clean":"2023-09-23T23:27:24.586065+0000","last_became_active":"2023-09-23T23:26:48.538246+0000","last_became_peered":"2023-09-23T23:26:48.538246+0000","last_unstale":"2023-09-23T23:27:24.586065+0000","last_undegraded":"2023-09-23T23:27:24.586065+0000","last_fullsized":"2023-09-23T23:27:24.586065+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:19.039472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":18,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.538564+0000","last_change":"2023-09-23T23:27:20.551305+0000","last_active":"2023-09-23T23:27:28.538564+0000","last_peered":"2023-09-23T23:27:28.538564+0000","last_clean":"2023-09-23T23:27:28.538564+0000","last_became_active":"2023-09-23T23:27:20.551017+0000","last_became_peered":"2023-09-23T23:27:20.551017+0000","last_unstale":"2023-09-23T23:27:28.538564+0000","last_undegraded":"2023-09-23T23:27:28.538564+0000","last_fullsized":"2023-09-23T23:27:28.538564+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:27.026015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":506,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.538579+0000","last_change":"2023-09-23T23:24:02.268079+0000","last_active":"2023-09-23T23:27:28.538579+0000","last_peered":"2023-09-23T23:27:28.538579+0000","last_clean":"2023-09-23T23:27:28.538579+0000","last_became_active":"2023-09-23T23:23:45.463106+0000","last_became_peered":"2023-09-23T23:23:45.463106+0000","last_unstale":"2023-09-23T23:27:28.538579+0000","last_undegraded":"2023-09-23T23:27:28.538579+0000","last_fullsized":"2023-09-23T23:27:28.538579+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.128813+0000","scrub_duration":0.0018535310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":63,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.545276+0000","last_change":"2023-09-23T23:26:11.534727+0000","last_active":"2023-09-23T23:27:28.545276+0000","last_peered":"2023-09-23T23:27:28.545276+0000","last_clean":"2023-09-23T23:27:28.545276+0000","last_became_active":"2023-09-23T23:26:11.513694+0000","last_became_peered":"2023-09-23T23:26:11.513694+0000","last_unstale":"2023-09-23T23:27:28.545276+0000","last_undegraded":"2023-09-23T23:27:28.545276+0000","last_fullsized":"2023-09-23T23:27:28.545276+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:04.562837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":420,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.544402+0000","last_change":"2023-09-23T23:25:31.718910+0000","last_active":"2023-09-23T23:27:28.544402+0000","last_peered":"2023-09-23T23:27:28.544402+0000","last_clean":"2023-09-23T23:27:28.544402+0000","last_became_active":"2023-09-23T23:25:31.718567+0000","last_became_peered":"2023-09-23T23:25:31.718567+0000","last_unstale":"2023-09-23T23:27:28.544402+0000","last_undegraded":"2023-09-23T23:27:28.544402+0000","last_fullsized":"2023-09-23T23:27:28.544402+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:21.447944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":530,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.594913+0000","last_change":"2023-09-23T23:26:34.550496+0000","last_active":"2023-09-23T23:27:25.594913+0000","last_peered":"2023-09-23T23:27:25.594913+0000","last_clean":"2023-09-23T23:27:25.594913+0000","last_became_active":"2023-09-23T23:26:34.550151+0000","last_became_peered":"2023-09-23T23:26:34.550151+0000","last_unstale":"2023-09-23T23:27:25.594913+0000","last_undegraded":"2023-09-23T23:27:25.594913+0000","last_fullsized":"2023-09-23T23:27:25.594913+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:53.828311+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:53.828311+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":362,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.587230+0000","last_change":"2023-09-23T23:26:11.569228+0000","last_active":"2023-09-23T23:27:24.587230+0000","last_peered":"2023-09-23T23:27:24.587230+0000","last_clean":"2023-09-23T23:27:24.587230+0000","last_became_active":"2023-09-23T23:26:11.513159+0000","last_became_peered":"2023-09-23T23:26:11.513159+0000","last_unstale":"2023-09-23T23:27:24.587230+0000","last_undegraded":"2023-09-23T23:27:24.587230+0000","last_fullsized":"2023-09-23T23:27:24.587230+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:40.698257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":427,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.583120+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:27:25.583120+0000","last_peered":"2023-09-23T23:27:25.583120+0000","last_clean":"2023-09-23T23:27:25.583120+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:27:25.583120+0000","last_undegraded":"2023-09-23T23:27:25.583120+0000","last_fullsized":"2023-09-23T23:27:25.583120+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":769,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.585595+0000","last_change":"2023-09-23T23:26:34.551458+0000","last_active":"2023-09-23T23:27:24.585595+0000","last_peered":"2023-09-23T23:27:24.585595+0000","last_clean":"2023-09-23T23:27:24.585595+0000","last_became_active":"2023-09-23T23:26:34.551067+0000","last_became_peered":"2023-09-23T23:26:34.551067+0000","last_unstale":"2023-09-23T23:27:24.585595+0000","last_undegraded":"2023-09-23T23:27:24.585595+0000","last_fullsized":"2023-09-23T23:27:24.585595+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:05.788656+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":29,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.586318+0000","last_change":"2023-09-23T23:26:48.563723+0000","last_active":"2023-09-23T23:27:24.586318+0000","last_peered":"2023-09-23T23:27:24.586318+0000","last_clean":"2023-09-23T23:27:24.586318+0000","last_became_active":"2023-09-23T23:26:48.563340+0000","last_became_peered":"2023-09-23T23:26:48.563340+0000","last_unstale":"2023-09-23T23:27:24.586318+0000","last_undegraded":"2023-09-23T23:27:24.586318+0000","last_fullsized":"2023-09-23T23:27:24.586318+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:05.974131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":316,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.586541+0000","last_change":"2023-09-23T23:26:32.040316+0000","last_active":"2023-09-23T23:27:24.586541+0000","last_peered":"2023-09-23T23:27:24.586541+0000","last_clean":"2023-09-23T23:27:24.586541+0000","last_became_active":"2023-09-23T23:26:32.039758+0000","last_became_peered":"2023-09-23T23:26:32.039758+0000","last_unstale":"2023-09-23T23:27:24.586541+0000","last_undegraded":"2023-09-23T23:27:24.586541+0000","last_fullsized":"2023-09-23T23:27:24.586541+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:03.930331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.19","version":"430'10620","reported_seq":15719,"reported_epoch":430,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:28.532085+0000","last_change":"2023-09-23T23:27:21.715633+0000","last_active":"2023-09-23T23:27:28.532085+0000","last_peered":"2023-09-23T23:27:28.532085+0000","last_clean":"2023-09-23T23:27:19.516646+0000","last_became_active":"2023-09-23T23:27:21.687051+0000","last_became_peered":"2023-09-23T23:27:21.687051+0000","last_unstale":"2023-09-23T23:27:28.532085+0000","last_undegraded":"2023-09-23T23:27:28.532085+0000","last_fullsized":"2023-09-23T23:27:28.532085+0000","mapping_epoch":425,"log_start":"362'5929","ondisk_log_start":"362'5929","created":365,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":1949,"log_size":4691,"log_dups_size":0,"ondisk_log_size":4691,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56721408,"num_objects":866,"num_object_clones":0,"num_object_copies":2598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":866,"num_objects_unfound":0,"num_objects_dirty":866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5804,"num_write_kb":213408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2401,"num_bytes_recovered":156762112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,7],"avail_no_missing":["3","0","6","7"],"object_location_counts":[{"shards":"0,3,7","objects":866}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":595,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.538702+0000","last_change":"2023-09-23T23:26:14.527752+0000","last_active":"2023-09-23T23:27:28.538702+0000","last_peered":"2023-09-23T23:27:28.538702+0000","last_clean":"2023-09-23T23:27:28.538702+0000","last_became_active":"2023-09-23T23:26:14.525188+0000","last_became_peered":"2023-09-23T23:26:14.525188+0000","last_unstale":"2023-09-23T23:27:28.538702+0000","last_undegraded":"2023-09-23T23:27:28.538702+0000","last_fullsized":"2023-09-23T23:27:28.538702+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:10.617744+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":284,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.587026+0000","last_change":"2023-09-23T23:25:31.725776+0000","last_active":"2023-09-23T23:27:24.587026+0000","last_peered":"2023-09-23T23:27:24.587026+0000","last_clean":"2023-09-23T23:27:24.587026+0000","last_became_active":"2023-09-23T23:25:31.725429+0000","last_became_peered":"2023-09-23T23:25:31.725429+0000","last_unstale":"2023-09-23T23:27:24.587026+0000","last_undegraded":"2023-09-23T23:27:24.587026+0000","last_fullsized":"2023-09-23T23:27:24.587026+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:07.289251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":422,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.638824+0000","last_change":"2023-09-23T23:26:48.559268+0000","last_active":"2023-09-23T23:27:29.638824+0000","last_peered":"2023-09-23T23:27:29.638824+0000","last_clean":"2023-09-23T23:27:29.638824+0000","last_became_active":"2023-09-23T23:26:48.558563+0000","last_became_peered":"2023-09-23T23:26:48.558563+0000","last_unstale":"2023-09-23T23:27:29.638824+0000","last_undegraded":"2023-09-23T23:27:29.638824+0000","last_fullsized":"2023-09-23T23:27:29.638824+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:58.779288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"430'8837","reported_seq":13464,"reported_epoch":430,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T23:27:25.663858+0000","last_change":"2023-09-23T23:27:24.433117+0000","last_active":"2023-09-23T23:27:25.663858+0000","last_peered":"2023-09-23T23:27:25.663858+0000","last_clean":"2023-09-23T23:25:30.482221+0000","last_became_active":"2023-09-23T23:26:32.112583+0000","last_became_peered":"2023-09-23T23:26:32.112583+0000","last_unstale":"2023-09-23T23:27:25.663858+0000","last_undegraded":"2023-09-23T23:27:25.663858+0000","last_fullsized":"2023-09-23T23:27:25.663858+0000","mapping_epoch":398,"log_start":"355'1834","ondisk_log_start":"355'1834","created":365,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":7003,"log_dups_size":0,"ondisk_log_size":7003,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57769984,"num_objects":882,"num_object_clones":0,"num_object_copies":2646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":521,"num_objects_unfound":0,"num_objects_dirty":882,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5664,"num_write_kb":209440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2250,"num_bytes_recovered":147456000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[0,2,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":882}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":774,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.595379+0000","last_change":"2023-09-23T23:27:23.549144+0000","last_active":"2023-09-23T23:27:25.595379+0000","last_peered":"2023-09-23T23:27:25.595379+0000","last_clean":"2023-09-23T23:27:25.595379+0000","last_became_active":"2023-09-23T23:27:23.548774+0000","last_became_peered":"2023-09-23T23:27:23.548774+0000","last_unstale":"2023-09-23T23:27:25.595379+0000","last_undegraded":"2023-09-23T23:27:25.595379+0000","last_fullsized":"2023-09-23T23:27:25.595379+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:18.211722+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":113,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.555329+0000","last_change":"2023-09-23T23:26:32.139091+0000","last_active":"2023-09-23T23:27:29.555329+0000","last_peered":"2023-09-23T23:27:29.555329+0000","last_clean":"2023-09-23T23:27:29.555329+0000","last_became_active":"2023-09-23T23:26:32.053033+0000","last_became_peered":"2023-09-23T23:26:32.053033+0000","last_unstale":"2023-09-23T23:27:29.555329+0000","last_undegraded":"2023-09-23T23:27:29.555329+0000","last_fullsized":"2023-09-23T23:27:29.555329+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:32.625818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":68,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.583851+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:27:25.583851+0000","last_peered":"2023-09-23T23:27:25.583851+0000","last_clean":"2023-09-23T23:27:25.583851+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:27:25.583851+0000","last_undegraded":"2023-09-23T23:27:25.583851+0000","last_fullsized":"2023-09-23T23:27:25.583851+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"432'8869","reported_seq":12877,"reported_epoch":432,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:30.204185+0000","last_change":"2023-09-23T23:27:21.731801+0000","last_active":"2023-09-23T23:27:30.204185+0000","last_peered":"2023-09-23T23:27:30.204185+0000","last_clean":"2023-09-23T23:27:19.451010+0000","last_became_active":"2023-09-23T23:27:21.704869+0000","last_became_peered":"2023-09-23T23:27:21.704869+0000","last_unstale":"2023-09-23T23:27:30.204185+0000","last_undegraded":"2023-09-23T23:27:30.204185+0000","last_fullsized":"2023-09-23T23:27:30.204185+0000","mapping_epoch":425,"log_start":"355'1583","ondisk_log_start":"355'1583","created":349,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"367'6563","last_scrub_stamp":"2023-09-23T23:25:25.026782+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:25.026782+0000","objects_scrubbed":3175,"log_size":7286,"log_dups_size":0,"ondisk_log_size":7286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:37.076814+0000","scrub_duration":0.41131605999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56688640,"num_objects":865,"num_object_clones":0,"num_object_copies":2595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":865,"num_objects_unfound":0,"num_objects_dirty":865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2310,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1868,"num_bytes_recovered":122421248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[0,5,4],"avail_no_missing":["0","4","5","7"],"object_location_counts":[{"shards":"0,4,5","objects":865}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":757,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.586126+0000","last_change":"2023-09-23T23:26:51.563868+0000","last_active":"2023-09-23T23:27:24.586126+0000","last_peered":"2023-09-23T23:27:24.586126+0000","last_clean":"2023-09-23T23:27:24.586126+0000","last_became_active":"2023-09-23T23:26:51.563510+0000","last_became_peered":"2023-09-23T23:26:51.563510+0000","last_unstale":"2023-09-23T23:27:24.586126+0000","last_undegraded":"2023-09-23T23:27:24.586126+0000","last_fullsized":"2023-09-23T23:27:24.586126+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:34.244822+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":33,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.586643+0000","last_change":"2023-09-23T23:26:48.673221+0000","last_active":"2023-09-23T23:27:24.586643+0000","last_peered":"2023-09-23T23:27:24.586643+0000","last_clean":"2023-09-23T23:27:24.586643+0000","last_became_active":"2023-09-23T23:26:48.555920+0000","last_became_peered":"2023-09-23T23:26:48.555920+0000","last_unstale":"2023-09-23T23:27:24.586643+0000","last_undegraded":"2023-09-23T23:27:24.586643+0000","last_fullsized":"2023-09-23T23:27:24.586643+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:26.586824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.595585+0000","last_change":"2023-09-23T23:26:48.525190+0000","last_active":"2023-09-23T23:27:25.595585+0000","last_peered":"2023-09-23T23:27:25.595585+0000","last_clean":"2023-09-23T23:27:25.595585+0000","last_became_active":"2023-09-23T23:26:48.524874+0000","last_became_peered":"2023-09-23T23:26:48.524874+0000","last_unstale":"2023-09-23T23:27:25.595585+0000","last_undegraded":"2023-09-23T23:27:25.595585+0000","last_fullsized":"2023-09-23T23:27:25.595585+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:43.114769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"432'8942","reported_seq":9134,"reported_epoch":432,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:30.311329+0000","last_change":"2023-09-23T23:27:21.759725+0000","last_active":"2023-09-23T23:27:30.311329+0000","last_peered":"2023-09-23T23:27:30.311329+0000","last_clean":"2023-09-23T23:27:19.475740+0000","last_became_active":"2023-09-23T23:27:21.718647+0000","last_became_peered":"2023-09-23T23:27:21.718647+0000","last_unstale":"2023-09-23T23:27:30.311329+0000","last_undegraded":"2023-09-23T23:27:30.311329+0000","last_fullsized":"2023-09-23T23:27:30.311329+0000","mapping_epoch":425,"log_start":"355'1631","ondisk_log_start":"355'1631","created":349,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"366'6545","last_scrub_stamp":"2023-09-23T23:25:22.335514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:22.335514+0000","objects_scrubbed":3321,"log_size":7311,"log_dups_size":0,"ondisk_log_size":7311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:12.578946+0000","scrub_duration":0.75924003299999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60555264,"num_objects":924,"num_object_clones":0,"num_object_copies":2772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":924,"num_objects_unfound":0,"num_objects_dirty":924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2401,"num_write_kb":128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,4],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":924}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":818,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.582896+0000","last_change":"2023-09-23T23:26:14.530414+0000","last_active":"2023-09-23T23:27:25.582896+0000","last_peered":"2023-09-23T23:27:25.582896+0000","last_clean":"2023-09-23T23:27:25.582896+0000","last_became_active":"2023-09-23T23:26:14.530001+0000","last_became_peered":"2023-09-23T23:26:14.530001+0000","last_unstale":"2023-09-23T23:27:25.582896+0000","last_undegraded":"2023-09-23T23:27:25.582896+0000","last_fullsized":"2023-09-23T23:27:25.582896+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:37.443998+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":33,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.586717+0000","last_change":"2023-09-23T23:26:48.547146+0000","last_active":"2023-09-23T23:27:24.586717+0000","last_peered":"2023-09-23T23:27:24.586717+0000","last_clean":"2023-09-23T23:27:24.586717+0000","last_became_active":"2023-09-23T23:26:48.537682+0000","last_became_peered":"2023-09-23T23:26:48.537682+0000","last_unstale":"2023-09-23T23:27:24.586717+0000","last_undegraded":"2023-09-23T23:27:24.586717+0000","last_fullsized":"2023-09-23T23:27:24.586717+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:10.115563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":457,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.543502+0000","last_change":"2023-09-23T23:26:01.833889+0000","last_active":"2023-09-23T23:27:28.543502+0000","last_peered":"2023-09-23T23:27:28.543502+0000","last_clean":"2023-09-23T23:27:28.543502+0000","last_became_active":"2023-09-23T23:26:01.833626+0000","last_became_peered":"2023-09-23T23:26:01.833626+0000","last_unstale":"2023-09-23T23:27:28.543502+0000","last_undegraded":"2023-09-23T23:27:28.543502+0000","last_fullsized":"2023-09-23T23:27:28.543502+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:25.659671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"433'8882","reported_seq":14427,"reported_epoch":433,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:30.570819+0000","last_change":"2023-09-23T23:26:48.644080+0000","last_active":"2023-09-23T23:27:30.570819+0000","last_peered":"2023-09-23T23:27:30.570819+0000","last_clean":"2023-09-23T23:26:00.788738+0000","last_became_active":"2023-09-23T23:26:48.619251+0000","last_became_peered":"2023-09-23T23:26:48.619251+0000","last_unstale":"2023-09-23T23:27:30.570819+0000","last_undegraded":"2023-09-23T23:27:30.570819+0000","last_fullsized":"2023-09-23T23:27:30.570819+0000","mapping_epoch":410,"log_start":"355'1532","ondisk_log_start":"355'1532","created":349,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":7350,"log_dups_size":0,"ondisk_log_size":7350,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60981248,"num_objects":930,"num_object_clones":0,"num_object_copies":2790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2790,"num_objects_unfound":0,"num_objects_dirty":930,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5688,"num_write_kb":211808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2704,"num_bytes_recovered":177209344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[0,2,4],"avail_no_missing":["0","2","4","6"],"object_location_counts":[{"shards":"0,2,4","objects":930}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":750,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.596232+0000","last_change":"2023-09-23T23:26:14.521495+0000","last_active":"2023-09-23T23:27:25.596232+0000","last_peered":"2023-09-23T23:27:25.596232+0000","last_clean":"2023-09-23T23:27:25.596232+0000","last_became_active":"2023-09-23T23:26:14.521164+0000","last_became_peered":"2023-09-23T23:26:14.521164+0000","last_unstale":"2023-09-23T23:27:25.596232+0000","last_undegraded":"2023-09-23T23:27:25.596232+0000","last_fullsized":"2023-09-23T23:27:25.596232+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.042580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.042580+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":336,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.586394+0000","last_change":"2023-09-23T23:26:32.043926+0000","last_active":"2023-09-23T23:27:24.586394+0000","last_peered":"2023-09-23T23:27:24.586394+0000","last_clean":"2023-09-23T23:27:24.586394+0000","last_became_active":"2023-09-23T23:26:32.039282+0000","last_became_peered":"2023-09-23T23:26:32.039282+0000","last_unstale":"2023-09-23T23:27:24.586394+0000","last_undegraded":"2023-09-23T23:27:24.586394+0000","last_fullsized":"2023-09-23T23:27:24.586394+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:35.345086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":420,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.538765+0000","last_change":"2023-09-23T23:26:48.536996+0000","last_active":"2023-09-23T23:27:28.538765+0000","last_peered":"2023-09-23T23:27:28.538765+0000","last_clean":"2023-09-23T23:27:28.538765+0000","last_became_active":"2023-09-23T23:26:48.536638+0000","last_became_peered":"2023-09-23T23:26:48.536638+0000","last_unstale":"2023-09-23T23:27:28.538765+0000","last_undegraded":"2023-09-23T23:27:28.538765+0000","last_fullsized":"2023-09-23T23:27:28.538765+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:36.943318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.4","version":"430'9054","reported_seq":12116,"reported_epoch":430,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:27:28.145140+0000","last_change":"2023-09-23T23:27:22.936603+0000","last_active":"2023-09-23T23:27:28.145140+0000","last_peered":"2023-09-23T23:27:28.145140+0000","last_clean":"2023-09-23T23:25:30.577110+0000","last_became_active":"2023-09-23T23:25:32.991045+0000","last_became_peered":"2023-09-23T23:25:32.991045+0000","last_unstale":"2023-09-23T23:27:28.145140+0000","last_undegraded":"2023-09-23T23:25:32.735490+0000","last_fullsized":"2023-09-23T23:25:32.712272+0000","mapping_epoch":372,"log_start":"357'3152","ondisk_log_start":"357'3152","created":349,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5902,"log_dups_size":0,"ondisk_log_size":5902,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57376768,"num_objects":875,"num_object_clones":0,"num_object_copies":2625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1155,"num_objects_misplaced":559,"num_objects_unfound":0,"num_objects_dirty":875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5807,"num_write_kb":213856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1471,"num_bytes_recovered":96403456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":559},{"shards":"2,7","objects":-839}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":483,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.544883+0000","last_change":"2023-09-23T23:27:22.721185+0000","last_active":"2023-09-23T23:27:28.544883+0000","last_peered":"2023-09-23T23:27:28.544883+0000","last_clean":"2023-09-23T23:27:28.544883+0000","last_became_active":"2023-09-23T23:26:51.588385+0000","last_became_peered":"2023-09-23T23:26:51.588385+0000","last_unstale":"2023-09-23T23:27:28.544883+0000","last_undegraded":"2023-09-23T23:27:28.544883+0000","last_fullsized":"2023-09-23T23:27:28.544883+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:04.832687+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:04.832687+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":258,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.583675+0000","last_change":"2023-09-23T23:25:31.732607+0000","last_active":"2023-09-23T23:27:25.583675+0000","last_peered":"2023-09-23T23:27:25.583675+0000","last_clean":"2023-09-23T23:27:25.583675+0000","last_became_active":"2023-09-23T23:25:31.732099+0000","last_became_peered":"2023-09-23T23:25:31.732099+0000","last_unstale":"2023-09-23T23:27:25.583675+0000","last_undegraded":"2023-09-23T23:27:25.583675+0000","last_fullsized":"2023-09-23T23:27:25.583675+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:23.639936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":408,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.637121+0000","last_change":"2023-09-23T23:22:38.438390+0000","last_active":"2023-09-23T23:27:29.637121+0000","last_peered":"2023-09-23T23:27:29.637121+0000","last_clean":"2023-09-23T23:27:29.637121+0000","last_became_active":"2023-09-23T23:22:38.438065+0000","last_became_peered":"2023-09-23T23:22:38.438065+0000","last_unstale":"2023-09-23T23:27:29.637121+0000","last_undegraded":"2023-09-23T23:27:29.637121+0000","last_fullsized":"2023-09-23T23:27:29.637121+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:51.254227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"430'8915","reported_seq":17177,"reported_epoch":430,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:28.106014+0000","last_change":"2023-09-23T23:27:21.744029+0000","last_active":"2023-09-23T23:27:28.106014+0000","last_peered":"2023-09-23T23:27:28.106014+0000","last_clean":"2023-09-23T23:27:19.524907+0000","last_became_active":"2023-09-23T23:27:21.716446+0000","last_became_peered":"2023-09-23T23:27:21.716446+0000","last_unstale":"2023-09-23T23:27:28.106014+0000","last_undegraded":"2023-09-23T23:27:28.106014+0000","last_fullsized":"2023-09-23T23:27:28.106014+0000","mapping_epoch":425,"log_start":"356'2893","ondisk_log_start":"356'2893","created":349,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"367'6627","last_scrub_stamp":"2023-09-23T23:25:24.842424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:24.842424+0000","objects_scrubbed":3200,"log_size":6022,"log_dups_size":0,"ondisk_log_size":6022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:58.481655+0000","scrub_duration":0.68031426699999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59179008,"num_objects":903,"num_object_clones":0,"num_object_copies":2709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":903,"num_objects_unfound":0,"num_objects_dirty":903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2297,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3980,"num_bytes_recovered":260833280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,5,2],"avail_no_missing":["7","2","3","5","6"],"object_location_counts":[{"shards":"2,5,7","objects":903}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":765,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.587077+0000","last_change":"2023-09-23T23:23:45.830384+0000","last_active":"2023-09-23T23:27:24.587077+0000","last_peered":"2023-09-23T23:27:24.587077+0000","last_clean":"2023-09-23T23:27:24.587077+0000","last_became_active":"2023-09-23T23:23:36.025724+0000","last_became_peered":"2023-09-23T23:23:36.025724+0000","last_unstale":"2023-09-23T23:27:24.587077+0000","last_undegraded":"2023-09-23T23:27:24.587077+0000","last_fullsized":"2023-09-23T23:27:24.587077+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:34.345528+0000","scrub_duration":0.001659259,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":398,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.583201+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:27:25.583201+0000","last_peered":"2023-09-23T23:27:25.583201+0000","last_clean":"2023-09-23T23:27:25.583201+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:27:25.583201+0000","last_undegraded":"2023-09-23T23:27:25.583201+0000","last_fullsized":"2023-09-23T23:27:25.583201+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":400,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.545352+0000","last_change":"2023-09-23T23:25:31.731172+0000","last_active":"2023-09-23T23:27:28.545352+0000","last_peered":"2023-09-23T23:27:28.545352+0000","last_clean":"2023-09-23T23:27:28.545352+0000","last_became_active":"2023-09-23T23:25:31.730903+0000","last_became_peered":"2023-09-23T23:25:31.730903+0000","last_unstale":"2023-09-23T23:27:28.545352+0000","last_undegraded":"2023-09-23T23:27:28.545352+0000","last_fullsized":"2023-09-23T23:27:28.545352+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:35.442945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"433'9068","reported_seq":12367,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-09-23T23:27:30.935599+0000","last_change":"2023-09-23T23:27:10.906583+0000","last_active":"2023-09-23T23:27:30.935599+0000","last_peered":"2023-09-23T23:27:30.935599+0000","last_clean":"2023-09-23T23:27:30.935599+0000","last_became_active":"2023-09-23T23:27:10.888274+0000","last_became_peered":"2023-09-23T23:27:10.888274+0000","last_unstale":"2023-09-23T23:27:30.935599+0000","last_undegraded":"2023-09-23T23:27:30.935599+0000","last_fullsized":"2023-09-23T23:27:30.935599+0000","mapping_epoch":421,"log_start":"355'800","ondisk_log_start":"355'800","created":349,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":8268,"log_dups_size":0,"ondisk_log_size":8268,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58949632,"num_objects":899,"num_object_clones":0,"num_object_copies":2697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5821,"num_write_kb":215072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1584,"num_bytes_recovered":103809024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":712,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.640186+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:27:29.640186+0000","last_peered":"2023-09-23T23:27:29.640186+0000","last_clean":"2023-09-23T23:27:29.640186+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:27:29.640186+0000","last_undegraded":"2023-09-23T23:27:29.640186+0000","last_fullsized":"2023-09-23T23:27:29.640186+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":258,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.543814+0000","last_change":"2023-09-23T23:27:20.560074+0000","last_active":"2023-09-23T23:27:28.543814+0000","last_peered":"2023-09-23T23:27:28.543814+0000","last_clean":"2023-09-23T23:27:28.543814+0000","last_became_active":"2023-09-23T23:27:20.559764+0000","last_became_peered":"2023-09-23T23:27:20.559764+0000","last_unstale":"2023-09-23T23:27:28.543814+0000","last_undegraded":"2023-09-23T23:27:28.543814+0000","last_fullsized":"2023-09-23T23:27:28.543814+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:23.675227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":53,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.547204+0000","last_change":"2023-09-23T23:26:11.516475+0000","last_active":"2023-09-23T23:27:28.547204+0000","last_peered":"2023-09-23T23:27:28.547204+0000","last_clean":"2023-09-23T23:27:28.547204+0000","last_became_active":"2023-09-23T23:26:11.516198+0000","last_became_peered":"2023-09-23T23:26:11.516198+0000","last_unstale":"2023-09-23T23:27:28.547204+0000","last_undegraded":"2023-09-23T23:27:28.547204+0000","last_fullsized":"2023-09-23T23:27:28.547204+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:56.875103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"430'8911","reported_seq":12182,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.532223+0000","last_change":"2023-09-23T23:27:01.819921+0000","last_active":"2023-09-23T23:27:28.532223+0000","last_peered":"2023-09-23T23:27:28.532223+0000","last_clean":"2023-09-23T23:27:28.532223+0000","last_became_active":"2023-09-23T23:27:01.796624+0000","last_became_peered":"2023-09-23T23:27:01.796624+0000","last_unstale":"2023-09-23T23:27:28.532223+0000","last_undegraded":"2023-09-23T23:27:28.532223+0000","last_fullsized":"2023-09-23T23:27:28.532223+0000","mapping_epoch":417,"log_start":"356'2513","ondisk_log_start":"356'2513","created":349,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":6398,"log_dups_size":0,"ondisk_log_size":6398,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58097664,"num_objects":886,"num_object_clones":0,"num_object_copies":2658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5708,"num_write_kb":211040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1570,"num_bytes_recovered":102891520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":587,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.538404+0000","last_change":"2023-09-23T23:26:34.548211+0000","last_active":"2023-09-23T23:27:28.538404+0000","last_peered":"2023-09-23T23:27:28.538404+0000","last_clean":"2023-09-23T23:27:28.538404+0000","last_became_active":"2023-09-23T23:26:34.547815+0000","last_became_peered":"2023-09-23T23:26:34.547815+0000","last_unstale":"2023-09-23T23:27:28.538404+0000","last_undegraded":"2023-09-23T23:27:28.538404+0000","last_fullsized":"2023-09-23T23:27:28.538404+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:37.675417+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":53,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.543377+0000","last_change":"2023-09-23T23:26:11.512815+0000","last_active":"2023-09-23T23:27:28.543377+0000","last_peered":"2023-09-23T23:27:28.543377+0000","last_clean":"2023-09-23T23:27:28.543377+0000","last_became_active":"2023-09-23T23:26:11.512466+0000","last_became_peered":"2023-09-23T23:26:11.512466+0000","last_unstale":"2023-09-23T23:27:28.543377+0000","last_undegraded":"2023-09-23T23:27:28.543377+0000","last_fullsized":"2023-09-23T23:27:28.543377+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:10.324158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":430,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.545390+0000","last_change":"2023-09-23T23:27:20.558731+0000","last_active":"2023-09-23T23:27:28.545390+0000","last_peered":"2023-09-23T23:27:28.545390+0000","last_clean":"2023-09-23T23:27:28.545390+0000","last_became_active":"2023-09-23T23:27:20.558394+0000","last_became_peered":"2023-09-23T23:27:20.558394+0000","last_unstale":"2023-09-23T23:27:28.545390+0000","last_undegraded":"2023-09-23T23:27:28.545390+0000","last_fullsized":"2023-09-23T23:27:28.545390+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:25:00.408551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"431'8776","reported_seq":8955,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.309698+0000","last_change":"2023-09-23T23:25:20.332216+0000","last_active":"2023-09-23T23:27:29.309698+0000","last_peered":"2023-09-23T23:27:29.309698+0000","last_clean":"2023-09-23T23:27:29.309698+0000","last_became_active":"2023-09-23T23:25:18.988253+0000","last_became_peered":"2023-09-23T23:25:18.988253+0000","last_unstale":"2023-09-23T23:27:29.309698+0000","last_undegraded":"2023-09-23T23:27:29.309698+0000","last_fullsized":"2023-09-23T23:27:29.309698+0000","mapping_epoch":349,"log_start":"356'2401","ondisk_log_start":"356'2401","created":349,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"366'6383","last_scrub_stamp":"2023-09-23T23:25:20.331998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:20.331998+0000","objects_scrubbed":3194,"log_size":6375,"log_dups_size":0,"ondisk_log_size":6375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:37.230596+0000","scrub_duration":0.84482357799999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61603840,"num_objects":940,"num_object_clones":0,"num_object_copies":2820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2404,"num_write_kb":4800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":753,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.639704+0000","last_change":"2023-09-23T23:25:34.500038+0000","last_active":"2023-09-23T23:27:29.639704+0000","last_peered":"2023-09-23T23:27:29.639704+0000","last_clean":"2023-09-23T23:27:29.639704+0000","last_became_active":"2023-09-23T23:25:34.499705+0000","last_became_peered":"2023-09-23T23:25:34.499705+0000","last_unstale":"2023-09-23T23:27:29.639704+0000","last_undegraded":"2023-09-23T23:27:29.639704+0000","last_fullsized":"2023-09-23T23:27:29.639704+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:43.096696+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":68,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.582578+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:27:25.582578+0000","last_peered":"2023-09-23T23:27:25.582578+0000","last_clean":"2023-09-23T23:27:25.582578+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:27:25.582578+0000","last_undegraded":"2023-09-23T23:27:25.582578+0000","last_fullsized":"2023-09-23T23:27:25.582578+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":353,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.539953+0000","last_change":"2023-09-23T23:22:39.489325+0000","last_active":"2023-09-23T23:27:28.539953+0000","last_peered":"2023-09-23T23:27:28.539953+0000","last_clean":"2023-09-23T23:27:28.539953+0000","last_became_active":"2023-09-23T23:22:39.489048+0000","last_became_peered":"2023-09-23T23:22:39.489048+0000","last_unstale":"2023-09-23T23:27:28.539953+0000","last_undegraded":"2023-09-23T23:27:28.539953+0000","last_fullsized":"2023-09-23T23:27:28.539953+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:29.984948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"433'11563","reported_seq":11710,"reported_epoch":433,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:30.931180+0000","last_change":"2023-09-23T23:26:11.572743+0000","last_active":"2023-09-23T23:27:30.931180+0000","last_peered":"2023-09-23T23:27:30.931180+0000","last_clean":"2023-09-23T23:25:30.684619+0000","last_became_active":"2023-09-23T23:25:32.841731+0000","last_became_peered":"2023-09-23T23:25:32.841731+0000","last_unstale":"2023-09-23T23:27:30.931180+0000","last_undegraded":"2023-09-23T23:27:30.931180+0000","last_fullsized":"2023-09-23T23:27:30.931180+0000","mapping_epoch":372,"log_start":"360'4958","ondisk_log_start":"360'4958","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":6605,"log_dups_size":0,"ondisk_log_size":6605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:12.410433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117964824,"num_objects":1801,"num_object_clones":0,"num_object_copies":5403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1793,"num_objects_unfound":0,"num_objects_dirty":1801,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11563,"num_write_kb":427585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1179648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,6],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":1801}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":729,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.583748+0000","last_change":"2023-09-23T23:26:51.561734+0000","last_active":"2023-09-23T23:27:25.583748+0000","last_peered":"2023-09-23T23:27:25.583748+0000","last_clean":"2023-09-23T23:27:25.583748+0000","last_became_active":"2023-09-23T23:26:51.561394+0000","last_became_peered":"2023-09-23T23:26:51.561394+0000","last_unstale":"2023-09-23T23:27:25.583748+0000","last_undegraded":"2023-09-23T23:27:25.583748+0000","last_fullsized":"2023-09-23T23:27:25.583748+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.063071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.063071+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":133,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.540046+0000","last_change":"2023-09-23T23:23:36.048889+0000","last_active":"2023-09-23T23:27:28.540046+0000","last_peered":"2023-09-23T23:27:28.540046+0000","last_clean":"2023-09-23T23:27:28.540046+0000","last_became_active":"2023-09-23T23:23:36.048572+0000","last_became_peered":"2023-09-23T23:23:36.048572+0000","last_unstale":"2023-09-23T23:27:28.540046+0000","last_undegraded":"2023-09-23T23:27:28.540046+0000","last_fullsized":"2023-09-23T23:27:28.540046+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:55.581146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":446,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.596360+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:27:25.596360+0000","last_peered":"2023-09-23T23:27:25.596360+0000","last_clean":"2023-09-23T23:27:25.596360+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:27:25.596360+0000","last_undegraded":"2023-09-23T23:27:25.596360+0000","last_fullsized":"2023-09-23T23:27:25.596360+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":18,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.538643+0000","last_change":"2023-09-23T23:27:20.549904+0000","last_active":"2023-09-23T23:27:28.538643+0000","last_peered":"2023-09-23T23:27:28.538643+0000","last_clean":"2023-09-23T23:27:28.538643+0000","last_became_active":"2023-09-23T23:27:20.549645+0000","last_became_peered":"2023-09-23T23:27:20.549645+0000","last_unstale":"2023-09-23T23:27:28.538643+0000","last_undegraded":"2023-09-23T23:27:28.538643+0000","last_fullsized":"2023-09-23T23:27:28.538643+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:09.850640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"430'11506","reported_seq":26018,"reported_epoch":431,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:28.544710+0000","last_change":"2023-09-23T23:27:21.765359+0000","last_active":"2023-09-23T23:27:28.544710+0000","last_peered":"2023-09-23T23:27:28.544710+0000","last_clean":"2023-09-23T23:27:19.522554+0000","last_became_active":"2023-09-23T23:27:21.717182+0000","last_became_peered":"2023-09-23T23:27:21.717182+0000","last_unstale":"2023-09-23T23:27:28.544710+0000","last_undegraded":"2023-09-23T23:27:28.544710+0000","last_fullsized":"2023-09-23T23:27:28.544710+0000","mapping_epoch":425,"log_start":"374'7036","ondisk_log_start":"374'7036","created":349,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4470,"log_dups_size":0,"ondisk_log_size":4470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:29.188830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116916224,"num_objects":1784,"num_object_clones":0,"num_object_copies":5352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1784,"num_objects_unfound":0,"num_objects_dirty":1784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11506,"num_write_kb":425280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7186,"num_bytes_recovered":470941696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,0,4],"avail_no_missing":["3","0","4","6","7"],"object_location_counts":[{"shards":"0,3,4","objects":1784}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":578,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.635118+0000","last_change":"2023-09-23T23:23:49.320969+0000","last_active":"2023-09-23T23:27:29.635118+0000","last_peered":"2023-09-23T23:27:29.635118+0000","last_clean":"2023-09-23T23:27:29.635118+0000","last_became_active":"2023-09-23T23:22:17.001911+0000","last_became_peered":"2023-09-23T23:22:17.001911+0000","last_unstale":"2023-09-23T23:27:29.635118+0000","last_undegraded":"2023-09-23T23:27:29.635118+0000","last_fullsized":"2023-09-23T23:27:29.635118+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.320823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.320823+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:07.000984+0000","scrub_duration":0.0018140630000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":133,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.538996+0000","last_change":"2023-09-23T23:25:31.731396+0000","last_active":"2023-09-23T23:27:28.538996+0000","last_peered":"2023-09-23T23:27:28.538996+0000","last_clean":"2023-09-23T23:27:28.538996+0000","last_became_active":"2023-09-23T23:25:31.730931+0000","last_became_peered":"2023-09-23T23:25:31.730931+0000","last_unstale":"2023-09-23T23:27:28.538996+0000","last_undegraded":"2023-09-23T23:27:28.538996+0000","last_fullsized":"2023-09-23T23:27:28.538996+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:48.002628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":411,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.544326+0000","last_change":"2023-09-23T23:27:20.556916+0000","last_active":"2023-09-23T23:27:28.544326+0000","last_peered":"2023-09-23T23:27:28.544326+0000","last_clean":"2023-09-23T23:27:28.544326+0000","last_became_active":"2023-09-23T23:27:20.556651+0000","last_became_peered":"2023-09-23T23:27:20.556651+0000","last_unstale":"2023-09-23T23:27:28.544326+0000","last_undegraded":"2023-09-23T23:27:28.544326+0000","last_fullsized":"2023-09-23T23:27:28.544326+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:06.012543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":18,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.539671+0000","last_change":"2023-09-23T23:27:20.551221+0000","last_active":"2023-09-23T23:27:28.539671+0000","last_peered":"2023-09-23T23:27:28.539671+0000","last_clean":"2023-09-23T23:27:28.539671+0000","last_became_active":"2023-09-23T23:27:20.550900+0000","last_became_peered":"2023-09-23T23:27:20.550900+0000","last_unstale":"2023-09-23T23:27:28.539671+0000","last_undegraded":"2023-09-23T23:27:28.539671+0000","last_fullsized":"2023-09-23T23:27:28.539671+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:23.012149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"430'8783","reported_seq":16167,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.559365+0000","last_change":"2023-09-23T23:27:25.617962+0000","last_active":"2023-09-23T23:27:28.559365+0000","last_peered":"2023-09-23T23:27:28.559365+0000","last_clean":"2023-09-23T23:27:28.559365+0000","last_became_active":"2023-09-23T23:27:25.598086+0000","last_became_peered":"2023-09-23T23:27:25.598086+0000","last_unstale":"2023-09-23T23:27:28.559365+0000","last_undegraded":"2023-09-23T23:27:28.559365+0000","last_fullsized":"2023-09-23T23:27:28.559365+0000","mapping_epoch":429,"log_start":"356'2197","ondisk_log_start":"356'2197","created":349,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":0,"last_scrub":"366'6384","last_scrub_stamp":"2023-09-23T23:25:20.132415+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:20.132415+0000","objects_scrubbed":3102,"log_size":6586,"log_dups_size":0,"ondisk_log_size":6586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:22.673284+0000","scrub_duration":0.99622401299999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57016320,"num_objects":870,"num_object_clones":0,"num_object_copies":2610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2416,"num_write_kb":5888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3543,"num_bytes_recovered":232194048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":792,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.585868+0000","last_change":"2023-09-23T23:26:51.562598+0000","last_active":"2023-09-23T23:27:24.585868+0000","last_peered":"2023-09-23T23:27:24.585868+0000","last_clean":"2023-09-23T23:27:24.585868+0000","last_became_active":"2023-09-23T23:26:51.562167+0000","last_became_peered":"2023-09-23T23:26:51.562167+0000","last_unstale":"2023-09-23T23:27:24.585868+0000","last_undegraded":"2023-09-23T23:27:24.585868+0000","last_fullsized":"2023-09-23T23:27:24.585868+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:17.310285+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":167,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.639281+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:27:29.639281+0000","last_peered":"2023-09-23T23:27:29.639281+0000","last_clean":"2023-09-23T23:27:29.639281+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:27:29.639281+0000","last_undegraded":"2023-09-23T23:27:29.639281+0000","last_fullsized":"2023-09-23T23:27:29.639281+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":133,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.539791+0000","last_change":"2023-09-23T23:25:31.828908+0000","last_active":"2023-09-23T23:27:28.539791+0000","last_peered":"2023-09-23T23:27:28.539791+0000","last_clean":"2023-09-23T23:27:28.539791+0000","last_became_active":"2023-09-23T23:25:31.828582+0000","last_became_peered":"2023-09-23T23:25:31.828582+0000","last_unstale":"2023-09-23T23:27:28.539791+0000","last_undegraded":"2023-09-23T23:27:28.539791+0000","last_fullsized":"2023-09-23T23:27:28.539791+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:01.466593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":18,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.540244+0000","last_change":"2023-09-23T23:27:20.551574+0000","last_active":"2023-09-23T23:27:28.540244+0000","last_peered":"2023-09-23T23:27:28.540244+0000","last_clean":"2023-09-23T23:27:28.540244+0000","last_became_active":"2023-09-23T23:27:20.550820+0000","last_became_peered":"2023-09-23T23:27:20.550820+0000","last_unstale":"2023-09-23T23:27:28.540244+0000","last_undegraded":"2023-09-23T23:27:28.540244+0000","last_fullsized":"2023-09-23T23:27:28.540244+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:09.274552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"431'11312","reported_seq":11423,"reported_epoch":431,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:28.570056+0000","last_change":"2023-09-23T23:27:20.756116+0000","last_active":"2023-09-23T23:27:28.570056+0000","last_peered":"2023-09-23T23:27:28.570056+0000","last_clean":"2023-09-23T23:25:30.690916+0000","last_became_active":"2023-09-23T23:27:20.617962+0000","last_became_peered":"2023-09-23T23:27:20.617962+0000","last_unstale":"2023-09-23T23:27:28.570056+0000","last_undegraded":"2023-09-23T23:27:28.570056+0000","last_fullsized":"2023-09-23T23:27:28.570056+0000","mapping_epoch":424,"log_start":"369'6841","ondisk_log_start":"369'6841","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4471,"log_dups_size":0,"ondisk_log_size":4471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:02:52.104913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118358016,"num_objects":1806,"num_object_clones":0,"num_object_copies":5418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1806,"num_objects_unfound":0,"num_objects_dirty":1806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11312,"num_write_kb":419776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":1806}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":443,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.639591+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:27:29.639591+0000","last_peered":"2023-09-23T23:27:29.639591+0000","last_clean":"2023-09-23T23:27:29.639591+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:27:29.639591+0000","last_undegraded":"2023-09-23T23:27:29.639591+0000","last_fullsized":"2023-09-23T23:27:29.639591+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":750,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.585829+0000","last_change":"2023-09-23T23:26:51.564817+0000","last_active":"2023-09-23T23:27:24.585829+0000","last_peered":"2023-09-23T23:27:24.585829+0000","last_clean":"2023-09-23T23:27:24.585829+0000","last_became_active":"2023-09-23T23:26:51.564511+0000","last_became_peered":"2023-09-23T23:26:51.564511+0000","last_unstale":"2023-09-23T23:27:24.585829+0000","last_undegraded":"2023-09-23T23:27:24.585829+0000","last_fullsized":"2023-09-23T23:27:24.585829+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:02.415212+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":379,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.583938+0000","last_change":"2023-09-23T23:26:32.037619+0000","last_active":"2023-09-23T23:27:25.583938+0000","last_peered":"2023-09-23T23:27:25.583938+0000","last_clean":"2023-09-23T23:27:25.583938+0000","last_became_active":"2023-09-23T23:26:32.037045+0000","last_became_peered":"2023-09-23T23:26:32.037045+0000","last_unstale":"2023-09-23T23:27:25.583938+0000","last_undegraded":"2023-09-23T23:27:25.583938+0000","last_fullsized":"2023-09-23T23:27:25.583938+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:47.657527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":53,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.547278+0000","last_change":"2023-09-23T23:26:11.525845+0000","last_active":"2023-09-23T23:27:28.547278+0000","last_peered":"2023-09-23T23:27:28.547278+0000","last_clean":"2023-09-23T23:27:28.547278+0000","last_became_active":"2023-09-23T23:26:11.525507+0000","last_became_peered":"2023-09-23T23:26:11.525507+0000","last_unstale":"2023-09-23T23:27:28.547278+0000","last_undegraded":"2023-09-23T23:27:28.547278+0000","last_fullsized":"2023-09-23T23:27:28.547278+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:35.462874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"431'8918","reported_seq":14394,"reported_epoch":431,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:28.548944+0000","last_change":"2023-09-23T23:27:21.714698+0000","last_active":"2023-09-23T23:27:28.548944+0000","last_peered":"2023-09-23T23:27:28.548944+0000","last_clean":"2023-09-23T23:27:19.438581+0000","last_became_active":"2023-09-23T23:27:21.688910+0000","last_became_peered":"2023-09-23T23:27:21.688910+0000","last_unstale":"2023-09-23T23:27:28.548944+0000","last_undegraded":"2023-09-23T23:27:28.548944+0000","last_fullsized":"2023-09-23T23:27:28.548944+0000","mapping_epoch":425,"log_start":"356'2498","ondisk_log_start":"356'2498","created":349,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":6420,"log_dups_size":0,"ondisk_log_size":6420,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63602688,"num_objects":970,"num_object_clones":0,"num_object_copies":2910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":970,"num_objects_unfound":0,"num_objects_dirty":970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5682,"num_write_kb":212896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2670,"num_bytes_recovered":174981120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,7],"avail_no_missing":["3","0","6","7"],"object_location_counts":[{"shards":"0,3,7","objects":970}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":595,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.538486+0000","last_change":"2023-09-23T23:26:34.550276+0000","last_active":"2023-09-23T23:27:28.538486+0000","last_peered":"2023-09-23T23:27:28.538486+0000","last_clean":"2023-09-23T23:27:28.538486+0000","last_became_active":"2023-09-23T23:26:34.549923+0000","last_became_peered":"2023-09-23T23:26:34.549923+0000","last_unstale":"2023-09-23T23:27:28.538486+0000","last_undegraded":"2023-09-23T23:27:28.538486+0000","last_fullsized":"2023-09-23T23:27:28.538486+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:22.636211+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":357,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.585578+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:27:24.585578+0000","last_peered":"2023-09-23T23:27:24.585578+0000","last_clean":"2023-09-23T23:27:24.585578+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:27:24.585578+0000","last_undegraded":"2023-09-23T23:27:24.585578+0000","last_fullsized":"2023-09-23T23:27:24.585578+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":435,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.555809+0000","last_change":"2023-09-23T23:26:11.528036+0000","last_active":"2023-09-23T23:27:29.555809+0000","last_peered":"2023-09-23T23:27:29.555809+0000","last_clean":"2023-09-23T23:27:29.555809+0000","last_became_active":"2023-09-23T23:26:11.527688+0000","last_became_peered":"2023-09-23T23:26:11.527688+0000","last_unstale":"2023-09-23T23:27:29.555809+0000","last_undegraded":"2023-09-23T23:27:29.555809+0000","last_fullsized":"2023-09-23T23:27:29.555809+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:49.411897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":462,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.637509+0000","last_change":"2023-09-23T23:22:38.438448+0000","last_active":"2023-09-23T23:27:29.637509+0000","last_peered":"2023-09-23T23:27:29.637509+0000","last_clean":"2023-09-23T23:27:29.637509+0000","last_became_active":"2023-09-23T23:22:38.437495+0000","last_became_peered":"2023-09-23T23:22:38.437495+0000","last_unstale":"2023-09-23T23:27:29.637509+0000","last_undegraded":"2023-09-23T23:27:29.637509+0000","last_fullsized":"2023-09-23T23:27:29.637509+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:49.587056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"430'11097","reported_seq":15751,"reported_epoch":432,"state":"activating","last_fresh":"2023-09-23T23:27:29.638995+0000","last_change":"2023-09-23T23:27:29.638995+0000","last_active":"2023-09-23T23:27:28.542330+0000","last_peered":"2023-09-23T23:27:28.513656+0000","last_clean":"2023-09-23T23:27:28.513656+0000","last_became_active":"2023-09-23T23:26:13.289552+0000","last_became_peered":"2023-09-23T23:26:13.289552+0000","last_unstale":"2023-09-23T23:27:29.638995+0000","last_undegraded":"2023-09-23T23:27:29.638995+0000","last_fullsized":"2023-09-23T23:27:29.638995+0000","mapping_epoch":431,"log_start":"369'6683","ondisk_log_start":"369'6683","created":349,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4414,"log_dups_size":0,"ondisk_log_size":4414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:47.739467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119865344,"num_objects":1829,"num_object_clones":0,"num_object_copies":5487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11097,"num_write_kb":413632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2267,"num_bytes_recovered":148570112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":["1","2","4","7"],"object_location_counts":[{"shards":"1,2,4","objects":1829}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":459,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.583387+0000","last_change":"2023-09-23T23:26:04.560829+0000","last_active":"2023-09-23T23:27:25.583387+0000","last_peered":"2023-09-23T23:27:25.583387+0000","last_clean":"2023-09-23T23:27:25.583387+0000","last_became_active":"2023-09-23T23:26:04.560484+0000","last_became_peered":"2023-09-23T23:26:04.560484+0000","last_unstale":"2023-09-23T23:27:25.583387+0000","last_undegraded":"2023-09-23T23:27:25.583387+0000","last_fullsized":"2023-09-23T23:27:25.583387+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.048718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.048718+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":281,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.586216+0000","last_change":"2023-09-23T23:23:35.037351+0000","last_active":"2023-09-23T23:27:24.586216+0000","last_peered":"2023-09-23T23:27:24.586216+0000","last_clean":"2023-09-23T23:27:24.586216+0000","last_became_active":"2023-09-23T23:23:35.036841+0000","last_became_peered":"2023-09-23T23:23:35.036841+0000","last_unstale":"2023-09-23T23:27:24.586216+0000","last_undegraded":"2023-09-23T23:27:24.586216+0000","last_fullsized":"2023-09-23T23:27:24.586216+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:20.618292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":133,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.542707+0000","last_change":"2023-09-23T23:26:32.037099+0000","last_active":"2023-09-23T23:27:28.542707+0000","last_peered":"2023-09-23T23:27:28.542707+0000","last_clean":"2023-09-23T23:27:28.542707+0000","last_became_active":"2023-09-23T23:26:32.036681+0000","last_became_peered":"2023-09-23T23:26:32.036681+0000","last_unstale":"2023-09-23T23:27:28.542707+0000","last_undegraded":"2023-09-23T23:27:28.542707+0000","last_fullsized":"2023-09-23T23:27:28.542707+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:48.926576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":164,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.595845+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:27:25.595845+0000","last_peered":"2023-09-23T23:27:25.595845+0000","last_clean":"2023-09-23T23:27:25.595845+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:27:25.595845+0000","last_undegraded":"2023-09-23T23:27:25.595845+0000","last_fullsized":"2023-09-23T23:27:25.595845+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"430'11522","reported_seq":11611,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.133468+0000","last_change":"2023-09-23T23:23:43.505852+0000","last_active":"2023-09-23T23:27:28.133468+0000","last_peered":"2023-09-23T23:27:28.133468+0000","last_clean":"2023-09-23T23:27:28.133468+0000","last_became_active":"2023-09-23T23:23:43.505473+0000","last_became_peered":"2023-09-23T23:23:43.505473+0000","last_unstale":"2023-09-23T23:27:28.133468+0000","last_undegraded":"2023-09-23T23:27:28.133468+0000","last_fullsized":"2023-09-23T23:27:28.133468+0000","mapping_epoch":349,"log_start":"375'7266","ondisk_log_start":"375'7266","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4256,"log_dups_size":0,"ondisk_log_size":4256,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:31.385610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120586240,"num_objects":1840,"num_object_clones":0,"num_object_copies":5520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11522,"num_write_kb":427584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":743,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.547359+0000","last_change":"2023-09-23T23:26:14.518921+0000","last_active":"2023-09-23T23:27:28.547359+0000","last_peered":"2023-09-23T23:27:28.547359+0000","last_clean":"2023-09-23T23:27:28.547359+0000","last_became_active":"2023-09-23T23:26:14.518558+0000","last_became_peered":"2023-09-23T23:26:14.518558+0000","last_unstale":"2023-09-23T23:27:28.547359+0000","last_undegraded":"2023-09-23T23:27:28.547359+0000","last_fullsized":"2023-09-23T23:27:28.547359+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:14.360597+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":424,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.584071+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:27:25.584071+0000","last_peered":"2023-09-23T23:27:25.584071+0000","last_clean":"2023-09-23T23:27:25.584071+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:27:25.584071+0000","last_undegraded":"2023-09-23T23:27:25.584071+0000","last_fullsized":"2023-09-23T23:27:25.584071+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.538347+0000","last_change":"2023-09-23T23:27:20.562744+0000","last_active":"2023-09-23T23:27:28.538347+0000","last_peered":"2023-09-23T23:27:28.538347+0000","last_clean":"2023-09-23T23:27:28.538347+0000","last_became_active":"2023-09-23T23:27:20.562383+0000","last_became_peered":"2023-09-23T23:27:20.562383+0000","last_unstale":"2023-09-23T23:27:28.538347+0000","last_undegraded":"2023-09-23T23:27:28.538347+0000","last_fullsized":"2023-09-23T23:27:28.538347+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:01.085133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.582552+0000","last_change":"2023-09-23T23:26:48.562966+0000","last_active":"2023-09-23T23:27:25.582552+0000","last_peered":"2023-09-23T23:27:25.582552+0000","last_clean":"2023-09-23T23:27:25.582552+0000","last_became_active":"2023-09-23T23:26:48.562630+0000","last_became_peered":"2023-09-23T23:26:48.562630+0000","last_unstale":"2023-09-23T23:27:25.582552+0000","last_undegraded":"2023-09-23T23:27:25.582552+0000","last_fullsized":"2023-09-23T23:27:25.582552+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:40.163647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"431'11494","reported_seq":20049,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.320110+0000","last_change":"2023-09-23T23:26:33.547524+0000","last_active":"2023-09-23T23:27:29.320110+0000","last_peered":"2023-09-23T23:27:29.320110+0000","last_clean":"2023-09-23T23:27:29.320110+0000","last_became_active":"2023-09-23T23:26:33.521810+0000","last_became_peered":"2023-09-23T23:26:33.521810+0000","last_unstale":"2023-09-23T23:27:29.320110+0000","last_undegraded":"2023-09-23T23:27:29.320110+0000","last_fullsized":"2023-09-23T23:27:29.320110+0000","mapping_epoch":400,"log_start":"366'6705","ondisk_log_start":"366'6705","created":349,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4789,"log_dups_size":0,"ondisk_log_size":4789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:49.164061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111411200,"num_objects":1700,"num_object_clones":0,"num_object_copies":5100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11494,"num_write_kb":422208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4222,"num_bytes_recovered":276692992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":814,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.583259+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:27:25.583259+0000","last_peered":"2023-09-23T23:27:25.583259+0000","last_clean":"2023-09-23T23:27:25.583259+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:27:25.583259+0000","last_undegraded":"2023-09-23T23:27:25.583259+0000","last_fullsized":"2023-09-23T23:27:25.583259+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":29,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.586796+0000","last_change":"2023-09-23T23:26:48.544465+0000","last_active":"2023-09-23T23:27:24.586796+0000","last_peered":"2023-09-23T23:27:24.586796+0000","last_clean":"2023-09-23T23:27:24.586796+0000","last_became_active":"2023-09-23T23:26:48.544166+0000","last_became_peered":"2023-09-23T23:26:48.544166+0000","last_unstale":"2023-09-23T23:27:24.586796+0000","last_undegraded":"2023-09-23T23:27:24.586796+0000","last_fullsized":"2023-09-23T23:27:24.586796+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:38.844346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":417,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.637374+0000","last_change":"2023-09-23T23:22:38.482705+0000","last_active":"2023-09-23T23:27:29.637374+0000","last_peered":"2023-09-23T23:27:29.637374+0000","last_clean":"2023-09-23T23:27:29.637374+0000","last_became_active":"2023-09-23T23:22:38.438239+0000","last_became_peered":"2023-09-23T23:22:38.438239+0000","last_unstale":"2023-09-23T23:27:29.637374+0000","last_undegraded":"2023-09-23T23:27:29.637374+0000","last_fullsized":"2023-09-23T23:27:29.637374+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:52.006349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.543421+0000","last_change":"2023-09-23T23:26:11.522604+0000","last_active":"2023-09-23T23:27:28.543421+0000","last_peered":"2023-09-23T23:27:28.543421+0000","last_clean":"2023-09-23T23:27:28.543421+0000","last_became_active":"2023-09-23T23:26:11.522350+0000","last_became_peered":"2023-09-23T23:26:11.522350+0000","last_unstale":"2023-09-23T23:27:28.543421+0000","last_undegraded":"2023-09-23T23:27:28.543421+0000","last_fullsized":"2023-09-23T23:27:28.543421+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:20.164202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"430'8967","reported_seq":18217,"reported_epoch":430,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:28.106325+0000","last_change":"2023-09-23T23:27:21.744033+0000","last_active":"2023-09-23T23:27:28.106325+0000","last_peered":"2023-09-23T23:27:28.106325+0000","last_clean":"2023-09-23T23:27:19.485531+0000","last_became_active":"2023-09-23T23:27:21.716360+0000","last_became_peered":"2023-09-23T23:27:21.716360+0000","last_unstale":"2023-09-23T23:27:28.106325+0000","last_undegraded":"2023-09-23T23:27:28.106325+0000","last_fullsized":"2023-09-23T23:27:28.106325+0000","mapping_epoch":425,"log_start":"357'3117","ondisk_log_start":"357'3117","created":365,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5850,"log_dups_size":0,"ondisk_log_size":5850,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57737216,"num_objects":881,"num_object_clones":0,"num_object_copies":2643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":881,"num_objects_unfound":0,"num_objects_dirty":881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5755,"num_write_kb":212352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4546,"num_bytes_recovered":297926656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,5,2],"avail_no_missing":["7","2","3","5","6"],"object_location_counts":[{"shards":"2,5,7","objects":881}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":812,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.583253+0000","last_change":"2023-09-23T23:25:34.505042+0000","last_active":"2023-09-23T23:27:25.583253+0000","last_peered":"2023-09-23T23:27:25.583253+0000","last_clean":"2023-09-23T23:27:25.583253+0000","last_became_active":"2023-09-23T23:25:34.504687+0000","last_became_peered":"2023-09-23T23:25:34.504687+0000","last_unstale":"2023-09-23T23:27:25.583253+0000","last_undegraded":"2023-09-23T23:27:25.583253+0000","last_fullsized":"2023-09-23T23:27:25.583253+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:06.369262+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":67,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.637281+0000","last_change":"2023-09-23T23:25:48.082121+0000","last_active":"2023-09-23T23:27:29.637281+0000","last_peered":"2023-09-23T23:27:29.637281+0000","last_clean":"2023-09-23T23:27:29.637281+0000","last_became_active":"2023-09-23T23:25:48.081746+0000","last_became_peered":"2023-09-23T23:25:48.081746+0000","last_unstale":"2023-09-23T23:27:29.637281+0000","last_undegraded":"2023-09-23T23:27:29.637281+0000","last_fullsized":"2023-09-23T23:27:29.637281+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:00.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":175,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.639815+0000","last_change":"2023-09-23T23:22:38.439602+0000","last_active":"2023-09-23T23:27:29.639815+0000","last_peered":"2023-09-23T23:27:29.639815+0000","last_clean":"2023-09-23T23:27:29.639815+0000","last_became_active":"2023-09-23T23:22:38.439261+0000","last_became_peered":"2023-09-23T23:22:38.439261+0000","last_unstale":"2023-09-23T23:27:29.639815+0000","last_undegraded":"2023-09-23T23:27:29.639815+0000","last_fullsized":"2023-09-23T23:27:29.639815+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:03.293816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"430'8984","reported_seq":15947,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:27.868325+0000","last_change":"2023-09-23T23:26:52.609382+0000","last_active":"2023-09-23T23:27:27.868325+0000","last_peered":"2023-09-23T23:27:27.868325+0000","last_clean":"2023-09-23T23:27:27.868325+0000","last_became_active":"2023-09-23T23:26:52.578253+0000","last_became_peered":"2023-09-23T23:26:52.578253+0000","last_unstale":"2023-09-23T23:27:27.868325+0000","last_undegraded":"2023-09-23T23:27:27.868325+0000","last_fullsized":"2023-09-23T23:27:27.868325+0000","mapping_epoch":414,"log_start":"356'3026","ondisk_log_start":"356'3026","created":365,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":5,"last_scrub":"366'6668","last_scrub_stamp":"2023-09-23T23:25:23.925139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:23.925139+0000","objects_scrubbed":3290,"log_size":5958,"log_dups_size":0,"ondisk_log_size":5958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:07.350349+0000","scrub_duration":0.63215223600000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63242240,"num_objects":965,"num_object_clones":0,"num_object_copies":2895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2325,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3348,"num_bytes_recovered":209125376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":557,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.583890+0000","last_change":"2023-09-23T23:26:14.522158+0000","last_active":"2023-09-23T23:27:25.583890+0000","last_peered":"2023-09-23T23:27:25.583890+0000","last_clean":"2023-09-23T23:27:25.583890+0000","last_became_active":"2023-09-23T23:26:14.521785+0000","last_became_peered":"2023-09-23T23:26:14.521785+0000","last_unstale":"2023-09-23T23:27:25.583890+0000","last_undegraded":"2023-09-23T23:27:25.583890+0000","last_fullsized":"2023-09-23T23:27:25.583890+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:39.445601+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":173,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.545432+0000","last_change":"2023-09-23T23:22:38.452968+0000","last_active":"2023-09-23T23:27:28.545432+0000","last_peered":"2023-09-23T23:27:28.545432+0000","last_clean":"2023-09-23T23:27:28.545432+0000","last_became_active":"2023-09-23T23:22:38.437986+0000","last_became_peered":"2023-09-23T23:22:38.437986+0000","last_unstale":"2023-09-23T23:27:28.545432+0000","last_undegraded":"2023-09-23T23:27:28.545432+0000","last_fullsized":"2023-09-23T23:27:28.545432+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:10.333271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":63,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.543990+0000","last_change":"2023-09-23T23:27:20.558454+0000","last_active":"2023-09-23T23:27:28.543990+0000","last_peered":"2023-09-23T23:27:28.543990+0000","last_clean":"2023-09-23T23:27:28.543990+0000","last_became_active":"2023-09-23T23:27:20.558204+0000","last_became_peered":"2023-09-23T23:27:20.558204+0000","last_unstale":"2023-09-23T23:27:28.543990+0000","last_undegraded":"2023-09-23T23:27:28.543990+0000","last_fullsized":"2023-09-23T23:27:28.543990+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:15.079990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"433'8965","reported_seq":15151,"reported_epoch":433,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:30.571105+0000","last_change":"2023-09-23T23:26:48.644277+0000","last_active":"2023-09-23T23:27:30.571105+0000","last_peered":"2023-09-23T23:27:30.571105+0000","last_clean":"2023-09-23T23:26:00.820858+0000","last_became_active":"2023-09-23T23:26:48.622740+0000","last_became_peered":"2023-09-23T23:26:48.622740+0000","last_unstale":"2023-09-23T23:27:30.571105+0000","last_undegraded":"2023-09-23T23:27:30.571105+0000","last_fullsized":"2023-09-23T23:27:30.571105+0000","mapping_epoch":410,"log_start":"355'1735","ondisk_log_start":"355'1735","created":365,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":7230,"log_dups_size":0,"ondisk_log_size":7230,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55934976,"num_objects":854,"num_object_clones":0,"num_object_copies":2562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2562,"num_objects_unfound":0,"num_objects_dirty":854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5772,"num_write_kb":212000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3022,"num_bytes_recovered":198049792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[0,2,4],"avail_no_missing":["0","2","4","6"],"object_location_counts":[{"shards":"0,2,4","objects":854}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":173,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.547654+0000","last_change":"2023-09-23T23:25:31.740467+0000","last_active":"2023-09-23T23:27:28.547654+0000","last_peered":"2023-09-23T23:27:28.547654+0000","last_clean":"2023-09-23T23:27:28.547654+0000","last_became_active":"2023-09-23T23:25:31.729046+0000","last_became_peered":"2023-09-23T23:25:31.729046+0000","last_unstale":"2023-09-23T23:27:28.547654+0000","last_undegraded":"2023-09-23T23:27:28.547654+0000","last_fullsized":"2023-09-23T23:27:28.547654+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:31.222969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":53,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.543637+0000","last_change":"2023-09-23T23:26:11.527453+0000","last_active":"2023-09-23T23:27:28.543637+0000","last_peered":"2023-09-23T23:27:28.543637+0000","last_clean":"2023-09-23T23:27:28.543637+0000","last_became_active":"2023-09-23T23:26:11.527208+0000","last_became_peered":"2023-09-23T23:26:11.527208+0000","last_unstale":"2023-09-23T23:27:28.543637+0000","last_undegraded":"2023-09-23T23:27:28.543637+0000","last_fullsized":"2023-09-23T23:27:28.543637+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:15.083611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"432'8950","reported_seq":9068,"reported_epoch":432,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:30.301103+0000","last_change":"2023-09-23T23:27:21.759891+0000","last_active":"2023-09-23T23:27:30.301103+0000","last_peered":"2023-09-23T23:27:30.301103+0000","last_clean":"2023-09-23T23:27:19.534765+0000","last_became_active":"2023-09-23T23:27:21.718696+0000","last_became_peered":"2023-09-23T23:27:21.718696+0000","last_unstale":"2023-09-23T23:27:30.301103+0000","last_undegraded":"2023-09-23T23:27:30.301103+0000","last_fullsized":"2023-09-23T23:27:30.301103+0000","mapping_epoch":425,"log_start":"355'1600","ondisk_log_start":"355'1600","created":365,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":7350,"log_dups_size":0,"ondisk_log_size":7350,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58589184,"num_objects":894,"num_object_clones":0,"num_object_copies":2682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":894,"num_objects_unfound":0,"num_objects_dirty":894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5732,"num_write_kb":212032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,4],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":894}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":696,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T23:27:24.586265+0000","last_change":"2023-09-23T23:25:34.510468+0000","last_active":"2023-09-23T23:27:24.586265+0000","last_peered":"2023-09-23T23:27:24.586265+0000","last_clean":"2023-09-23T23:27:24.586265+0000","last_became_active":"2023-09-23T23:25:34.510106+0000","last_became_peered":"2023-09-23T23:25:34.510106+0000","last_unstale":"2023-09-23T23:27:24.586265+0000","last_undegraded":"2023-09-23T23:27:24.586265+0000","last_fullsized":"2023-09-23T23:27:24.586265+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:42.157293+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":448,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.583697+0000","last_change":"2023-09-23T23:25:31.746383+0000","last_active":"2023-09-23T23:27:25.583697+0000","last_peered":"2023-09-23T23:27:25.583697+0000","last_clean":"2023-09-23T23:27:25.583697+0000","last_became_active":"2023-09-23T23:25:31.745929+0000","last_became_peered":"2023-09-23T23:25:31.745929+0000","last_unstale":"2023-09-23T23:27:25.583697+0000","last_undegraded":"2023-09-23T23:27:25.583697+0000","last_fullsized":"2023-09-23T23:27:25.583697+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:21.696449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":441,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T23:27:25.583527+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:27:25.583527+0000","last_peered":"2023-09-23T23:27:25.583527+0000","last_clean":"2023-09-23T23:27:25.583527+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:27:25.583527+0000","last_undegraded":"2023-09-23T23:27:25.583527+0000","last_fullsized":"2023-09-23T23:27:25.583527+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"433'8924","reported_seq":12694,"reported_epoch":433,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:30.570458+0000","last_change":"2023-09-23T23:27:21.773301+0000","last_active":"2023-09-23T23:27:30.570458+0000","last_peered":"2023-09-23T23:27:30.570458+0000","last_clean":"2023-09-23T23:27:19.482304+0000","last_became_active":"2023-09-23T23:27:21.727527+0000","last_became_peered":"2023-09-23T23:27:21.727527+0000","last_unstale":"2023-09-23T23:27:30.570458+0000","last_undegraded":"2023-09-23T23:27:30.570458+0000","last_fullsized":"2023-09-23T23:27:30.570458+0000","mapping_epoch":425,"log_start":"355'1633","ondisk_log_start":"355'1633","created":365,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":7291,"log_dups_size":0,"ondisk_log_size":7291,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55050240,"num_objects":840,"num_object_clones":0,"num_object_copies":2520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":840,"num_objects_unfound":0,"num_objects_dirty":840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5744,"num_write_kb":210688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1807,"num_bytes_recovered":118423552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[0,5,4],"avail_no_missing":["0","4","5","7"],"object_location_counts":[{"shards":"0,4,5","objects":840}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":805,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.541830+0000","last_change":"2023-09-23T23:26:14.527390+0000","last_active":"2023-09-23T23:27:28.541830+0000","last_peered":"2023-09-23T23:27:28.541830+0000","last_clean":"2023-09-23T23:27:28.541830+0000","last_became_active":"2023-09-23T23:26:14.527008+0000","last_became_peered":"2023-09-23T23:26:14.527008+0000","last_unstale":"2023-09-23T23:27:28.541830+0000","last_undegraded":"2023-09-23T23:27:28.541830+0000","last_fullsized":"2023-09-23T23:27:28.541830+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:36.460827+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":437,"reported_epoch":432,"state":"active+clean","last_fresh":"2023-09-23T23:27:29.638246+0000","last_change":"2023-09-23T23:20:15.786279+0000","last_active":"2023-09-23T23:27:29.638246+0000","last_peered":"2023-09-23T23:27:29.638246+0000","last_clean":"2023-09-23T23:27:29.638246+0000","last_became_active":"2023-09-23T23:20:15.782731+0000","last_became_peered":"2023-09-23T23:20:15.782731+0000","last_unstale":"2023-09-23T23:27:29.638246+0000","last_undegraded":"2023-09-23T23:27:29.638246+0000","last_fullsized":"2023-09-23T23:27:29.638246+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:58.011396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":477,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-09-23T23:27:28.543370+0000","last_change":"2023-09-23T23:25:31.827217+0000","last_active":"2023-09-23T23:27:28.543370+0000","last_peered":"2023-09-23T23:27:28.543370+0000","last_clean":"2023-09-23T23:27:28.543370+0000","last_became_active":"2023-09-23T23:25:31.826915+0000","last_became_peered":"2023-09-23T23:25:31.826915+0000","last_unstale":"2023-09-23T23:27:28.543370+0000","last_undegraded":"2023-09-23T23:27:28.543370+0000","last_fullsized":"2023-09-23T23:27:28.543370+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:44.656199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":1879965720,"num_objects":28687,"num_object_clones":0,"num_object_copies":86061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1155,"num_objects_misplaced":19226,"num_objects_unfound":0,"num_objects_dirty":28687,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":159800,"num_write_kb":5311105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58057,"num_bytes_recovered":3781689344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8504750080,"data_stored":8504737864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":164378,"ondisk_log_size":164378,"up":78,"acting":77,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":6162,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":512693,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":21,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1210401,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782576,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1757780,"kb_used_data":1588492,"kb_used_omap":8,"kb_used_meta":169271,"kb_avail":91987372,"statfs":{"total":95995035648,"available":94195068928,"internally_reserved":0,"allocated":1626615808,"data_stored":1625389703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":173333573},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":2.5590000000000002,"15min":1.843},"min":{"1min":0.23100000000000001,"5min":0.221,"15min":0.221},"max":{"1min":6.9930000000000003,"5min":85.721000000000004,"15min":85.721000000000004},"last":3.1299999999999999},{"interface":"front","average":{"1min":1.825,"5min":2.601,"15min":1.597},"min":{"1min":0.26400000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":6.7350000000000003,"5min":85.965000000000003,"15min":85.965000000000003},"last":3.2909999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":1.645,"15min":1.2410000000000001},"min":{"1min":0.25600000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.5039999999999996,"5min":60.847000000000001,"15min":60.847000000000001},"last":2.468},{"interface":"front","average":{"1min":1.2929999999999999,"5min":1.9490000000000001,"15min":1.629},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.6310000000000002,"5min":59.512999999999998,"15min":59.512999999999998},"last":0.32300000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5189999999999999,"5min":2.3809999999999998,"15min":2.0150000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.7430000000000003,"5min":55.811999999999998,"15min":55.811999999999998},"last":1.163},{"interface":"front","average":{"1min":1.847,"5min":1.8740000000000001,"15min":1.419},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.5449999999999999,"5min":55.847999999999999,"15min":55.847999999999999},"last":0.28199999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":2.024,"5min":1.536,"15min":1.5049999999999999},"min":{"1min":0.34100000000000003,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":9.9960000000000004,"5min":9.9960000000000004,"15min":9.9960000000000004},"last":1.0780000000000001},{"interface":"front","average":{"1min":2.036,"5min":1.502,"15min":1.363},"min":{"1min":0.35999999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":9.9659999999999993,"5min":9.9659999999999993,"15min":9.9659999999999993},"last":6.5949999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.78600000000000003,"15min":0.83399999999999996},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.2270000000000003,"5min":6.8380000000000001,"15min":6.8380000000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.91000000000000003,"15min":0.92800000000000005},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.1030000000000002,"5min":5.242,"15min":5.242},"last":0.29599999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":1.0289999999999999,"15min":1.0669999999999999},"min":{"1min":0.29299999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":2.1579999999999999,"5min":5.2110000000000003,"15min":5.2110000000000003},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.80900000000000005,"15min":0.81000000000000005},"min":{"1min":0.29499999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.5249999999999999,"5min":6.0149999999999997,"15min":6.0149999999999997},"last":0.36799999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":1.389,"15min":0.96599999999999997},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.4720000000000004,"5min":62.213000000000001,"15min":62.213000000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.85399999999999998,"5min":1.629,"15min":1.2689999999999999},"min":{"1min":0.36899999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.2969999999999997,"5min":62.805999999999997,"15min":62.805999999999997},"last":0.435}]}]},{"osd":6,"up_from":0,"seq":197568496053,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":112,"seq":481036337394,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1178220,"kb_used_data":548772,"kb_used_omap":4,"kb_used_meta":629435,"kb_avail":92566932,"statfs":{"total":95995035648,"available":94788538368,"internally_reserved":0,"allocated":561942528,"data_stored":559939186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":644542350},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:26:57 2023","interfaces":[{"interface":"back","average":{"1min":4.0049999999999999,"5min":2.8580000000000001,"15min":2.8319999999999999},"min":{"1min":0.192,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":16.491,"5min":75.823999999999998,"15min":75.823999999999998},"last":130.768},{"interface":"front","average":{"1min":2.601,"5min":1.9730000000000001,"15min":2.73},"min":{"1min":0.26800000000000002,"5min":0.193,"15min":0.192},"max":{"1min":13.154,"5min":75.840999999999994,"15min":75.840999999999994},"last":134.929}]},{"osd":2,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":4.2949999999999999,"5min":5.4870000000000001,"15min":4.7409999999999997},"min":{"1min":0.22,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":14.538,"5min":209.84200000000001,"15min":209.84200000000001},"last":18.123999999999999},{"interface":"front","average":{"1min":2.931,"5min":5.4779999999999998,"15min":3.8759999999999999},"min":{"1min":0.34000000000000002,"5min":0.218,"15min":0.218},"max":{"1min":11.276999999999999,"5min":209.11099999999999,"15min":209.11099999999999},"last":17.997}]},{"osd":3,"last update":"Sat Sep 23 23:26:36 2023","interfaces":[{"interface":"back","average":{"1min":2.3029999999999999,"5min":2.6850000000000001,"15min":3.754},"min":{"1min":0.23200000000000001,"5min":0.22600000000000001,"15min":0.20799999999999999},"max":{"1min":10.388999999999999,"5min":157.90899999999999,"15min":157.90899999999999},"last":0.23799999999999999},{"interface":"front","average":{"1min":3.024,"5min":2.7330000000000001,"15min":3.605},"min":{"1min":0.255,"5min":0.245,"15min":0.245},"max":{"1min":10.419,"5min":156.27000000000001,"15min":156.27000000000001},"last":0.26500000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:26:57 2023","interfaces":[{"interface":"back","average":{"1min":2.5529999999999999,"5min":1.5229999999999999,"15min":1.5680000000000001},"min":{"1min":0.29099999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":9.3780000000000001,"5min":9.3780000000000001,"15min":18.474},"last":0.36399999999999999},{"interface":"front","average":{"1min":2.9500000000000002,"5min":1.804,"15min":1.758},"min":{"1min":0.36199999999999999,"5min":0.30399999999999999,"15min":0.29099999999999998},"max":{"1min":9.7460000000000004,"5min":9.7460000000000004,"15min":25.710999999999999},"last":0.41099999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:26:57 2023","interfaces":[{"interface":"back","average":{"1min":2.0979999999999999,"5min":1.0880000000000001,"15min":3.0640000000000001},"min":{"1min":0.34000000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":13.064,"5min":13.064,"15min":250.33000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":3.0219999999999998,"5min":1.8149999999999999,"15min":3.2160000000000002},"min":{"1min":0.40300000000000002,"5min":0.34899999999999998,"15min":0.30199999999999999},"max":{"1min":9.3450000000000006,"5min":10.92,"15min":250.21199999999999},"last":0.39000000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:26:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":2.407,"15min":2.4420000000000002},"min":{"1min":0.39300000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":13.119,"5min":74.087999999999994,"15min":74.087999999999994},"last":0.377},{"interface":"front","average":{"1min":2.6960000000000002,"5min":2.6859999999999999,"15min":2.3759999999999999},"min":{"1min":0.40699999999999997,"5min":0.33100000000000002,"15min":0.32800000000000001},"max":{"1min":9.4030000000000005,"5min":73.974999999999994,"15min":73.974999999999994},"last":0.5}]},{"osd":7,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":3.6389999999999998,"5min":3.859,"15min":4.3630000000000004},"min":{"1min":0.40699999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.7929999999999993,"5min":78.308000000000007,"15min":78.308000000000007},"last":0.35299999999999998},{"interface":"front","average":{"1min":2.1989999999999998,"5min":3.3580000000000001,"15min":3.8319999999999999},"min":{"1min":0.378,"5min":0.371,"15min":0.371},"max":{"1min":9.3109999999999999,"5min":77.204999999999998,"15min":77.204999999999998},"last":0.52000000000000002}]}]},{"osd":0,"up_from":91,"seq":390842024218,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2191620,"kb_used_data":1149308,"kb_used_omap":5,"kb_used_meta":1042298,"kb_avail":91553532,"statfs":{"total":95995035648,"available":93750816768,"internally_reserved":0,"allocated":1176891392,"data_stored":1174808601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1067313786},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:27:18 2023","interfaces":[{"interface":"back","average":{"1min":3.29,"5min":4.29,"15min":3.9260000000000002},"min":{"1min":0.17799999999999999,"5min":0.17499999999999999,"15min":0.16900000000000001},"max":{"1min":15.144,"5min":139.30000000000001,"15min":155.91999999999999},"last":2.6349999999999998},{"interface":"front","average":{"1min":3.488,"5min":1.998,"15min":2.742},"min":{"1min":0.22700000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":10.423,"5min":10.423,"15min":153.28899999999999},"last":5.1210000000000004}]},{"osd":2,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":3.2890000000000001,"5min":8.5579999999999998,"15min":4.75},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":11.326000000000001,"5min":171.15100000000001,"15min":171.15100000000001},"last":2.4710000000000001},{"interface":"front","average":{"1min":2.0800000000000001,"5min":8.2010000000000005,"15min":5.0670000000000002},"min":{"1min":0.218,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":11.882999999999999,"5min":173.04900000000001,"15min":173.04900000000001},"last":1.877}]},{"osd":3,"last update":"Sat Sep 23 23:27:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1019999999999999,"5min":4.6459999999999999,"15min":3.8399999999999999},"min":{"1min":0.32900000000000001,"5min":0.25600000000000001,"15min":0.23200000000000001},"max":{"1min":13.112,"5min":139.34899999999999,"15min":139.34899999999999},"last":8.5660000000000007},{"interface":"front","average":{"1min":3.2480000000000002,"5min":4.5229999999999997,"15min":3.4209999999999998},"min":{"1min":0.311,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":10.237,"5min":138.739,"15min":138.739},"last":1.915}]},{"osd":4,"last update":"Sat Sep 23 23:26:44 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":3.391,"15min":2.6309999999999998},"min":{"1min":0.315,"5min":0.29099999999999998,"15min":0.247},"max":{"1min":4.5019999999999998,"5min":138.81299999999999,"15min":138.81299999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":2.1709999999999998,"5min":5.3230000000000004,"15min":3.2930000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":18.004000000000001,"5min":171.251,"15min":171.251},"last":5.1870000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:26:44 2023","interfaces":[{"interface":"back","average":{"1min":2.1379999999999999,"5min":5.4199999999999999,"15min":2.9409999999999998},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.29099999999999998},"max":{"1min":18.045000000000002,"5min":171.30699999999999,"15min":171.30699999999999},"last":5.2210000000000001},{"interface":"front","average":{"1min":1.55,"5min":3.4710000000000001,"15min":2.2490000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":7.5810000000000004,"5min":139.40799999999999,"15min":139.40799999999999},"last":2.391}]},{"osd":6,"last update":"Sat Sep 23 23:26:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2109999999999999,"5min":5.399,"15min":3.1899999999999999},"min":{"1min":0.34499999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":21.276,"5min":171.33600000000001,"15min":171.33600000000001},"last":5.2480000000000002},{"interface":"front","average":{"1min":1.706,"5min":6.0579999999999998,"15min":3.4289999999999998},"min":{"1min":0.40200000000000002,"5min":0.38500000000000001,"15min":0.32700000000000001},"max":{"1min":7.5090000000000003,"5min":184.13499999999999,"15min":184.13499999999999},"last":2.4489999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":1.4079999999999999,"5min":3.5579999999999998,"15min":2.0800000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.4669999999999996,"5min":138.77699999999999,"15min":138.77699999999999},"last":0.432},{"interface":"front","average":{"1min":1.615,"5min":3.3900000000000001,"15min":1.718},"min":{"1min":0.44700000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":7.5389999999999997,"5min":139.37899999999999,"15min":139.37899999999999},"last":2.419}]}]},{"osd":2,"up_from":199,"seq":854698492039,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1706796,"kb_used_data":1209188,"kb_used_omap":7,"kb_used_meta":497592,"kb_avail":92038356,"statfs":{"total":95995035648,"available":94247276544,"internally_reserved":0,"allocated":1238208512,"data_stored":1236154600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":509534289},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":3.9489999999999998,"5min":5.8310000000000004,"15min":5.0700000000000003},"min":{"1min":0.20399999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":17.382000000000001,"5min":119.277,"15min":119.277},"last":5.3460000000000001},{"interface":"front","average":{"1min":4.0880000000000001,"5min":5.5049999999999999,"15min":5.2030000000000003},"min":{"1min":0.192,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":11.69,"5min":119.56100000000001,"15min":119.56100000000001},"last":12.454000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":10.321999999999999,"5min":4.8470000000000004,"15min":8.0600000000000005},"min":{"1min":0.23499999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":103.65900000000001,"5min":157.98699999999999,"15min":157.98699999999999},"last":9.3379999999999992},{"interface":"front","average":{"1min":8.9329999999999998,"5min":4.9189999999999996,"15min":7.7240000000000002},"min":{"1min":0.28499999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":100.182,"5min":158.03800000000001,"15min":158.03800000000001},"last":8.4979999999999993}]},{"osd":3,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":4.9260000000000002,"5min":4.3360000000000003,"15min":5.3029999999999999},"min":{"1min":0.27300000000000002,"5min":0.191,"15min":0.191},"max":{"1min":23.045000000000002,"5min":119.366,"15min":119.366},"last":9.1170000000000009},{"interface":"front","average":{"1min":2.6520000000000001,"5min":3.1320000000000001,"15min":4.5730000000000004},"min":{"1min":0.29699999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.8079999999999998,"5min":75.418000000000006,"15min":75.418000000000006},"last":8.532}]},{"osd":4,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":2.2480000000000002,"5min":2.7610000000000001,"15min":3.9060000000000001},"min":{"1min":0.29399999999999998,"5min":0.25,"15min":0.25},"max":{"1min":10.444000000000001,"5min":119.88200000000001,"15min":332.98099999999999},"last":10.157},{"interface":"front","average":{"1min":2.5449999999999999,"5min":1.4790000000000001,"15min":3.3860000000000001},"min":{"1min":0.36499999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.268000000000001,"5min":21.076000000000001,"15min":332.96199999999999},"last":8.6839999999999993}]},{"osd":5,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":2.9409999999999998,"15min":2.2839999999999998},"min":{"1min":0.318,"5min":0.29499999999999998,"15min":0.27500000000000002},"max":{"1min":7.5880000000000001,"5min":119.919,"15min":119.919},"last":10.192},{"interface":"front","average":{"1min":2.5110000000000001,"5min":2.9420000000000002,"15min":2.2719999999999998},"min":{"1min":0.39800000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.305,"5min":119.306,"15min":119.306},"last":8.7110000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":3.2109999999999999,"15min":4.1639999999999997},"min":{"1min":0.34000000000000002,"5min":0.32000000000000001,"15min":0.31},"max":{"1min":7.6369999999999996,"5min":119.943,"15min":349.32799999999997},"last":10.218},{"interface":"front","average":{"1min":2.532,"5min":1.925,"15min":3.6659999999999999},"min":{"1min":0.42499999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":10.202999999999999,"5min":39.856999999999999,"15min":349.17700000000002},"last":8.7300000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0750000000000002,"5min":1.6319999999999999,"15min":1.3280000000000001},"min":{"1min":0.40000000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.7999999999999998,"5min":20.350000000000001,"15min":20.350000000000001},"last":5.2839999999999998},{"interface":"front","average":{"1min":2.1880000000000002,"5min":1.7270000000000001,"15min":1.419},"min":{"1min":0.39500000000000002,"5min":0.38,"15min":0.38},"max":{"1min":9.6029999999999998,"5min":20.321000000000002,"15min":20.321000000000002},"last":5.3140000000000001}]}]},{"osd":3,"up_from":148,"seq":635655160008,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1695444,"kb_used_data":1073740,"kb_used_omap":2,"kb_used_meta":621693,"kb_avail":92049708,"statfs":{"total":95995035648,"available":94258900992,"internally_reserved":0,"allocated":1099509760,"data_stored":1097517365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":636613685},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1,0,3],"upper_bound":1024},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:26:32 2023","interfaces":[{"interface":"back","average":{"1min":3.3300000000000001,"5min":4.0039999999999996,"15min":3.1699999999999999},"min":{"1min":0.19900000000000001,"5min":0.17999999999999999,"15min":0.158},"max":{"1min":21.84,"5min":103.866,"15min":103.866},"last":2.5449999999999999},{"interface":"front","average":{"1min":4.3760000000000003,"5min":3.5710000000000002,"15min":2.855},"min":{"1min":0.26100000000000001,"5min":0.20799999999999999,"15min":0.17999999999999999},"max":{"1min":20.827000000000002,"5min":101.884,"15min":101.884},"last":0.191}]},{"osd":1,"last update":"Sat Sep 23 23:26:32 2023","interfaces":[{"interface":"back","average":{"1min":1.8899999999999999,"5min":2.3490000000000002,"15min":2.762},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":11.631,"5min":38.094000000000001,"15min":76.322999999999993},"last":1.8759999999999999},{"interface":"front","average":{"1min":2.2559999999999998,"5min":2.1400000000000001,"15min":2.528},"min":{"1min":0.28199999999999997,"5min":0.23699999999999999,"15min":0.185},"max":{"1min":8.6120000000000001,"5min":42.030999999999999,"15min":76.444000000000003},"last":2.0430000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:26:40 2023","interfaces":[{"interface":"back","average":{"1min":2.4569999999999999,"5min":4.8760000000000003,"15min":3.754},"min":{"1min":0.28100000000000003,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":9.7970000000000006,"5min":101.279,"15min":101.279},"last":7.5620000000000003},{"interface":"front","average":{"1min":2.4449999999999998,"5min":4.54,"15min":2.73},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":7.5469999999999997,"5min":101.294,"15min":101.294},"last":5.8719999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:26:32 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":1.103,"15min":1.4490000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":6.766,"5min":7.2610000000000001,"15min":14.754},"last":2.117},{"interface":"front","average":{"1min":2.3559999999999999,"5min":1.4990000000000001,"15min":1.5960000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.28499999999999998},"max":{"1min":15.619,"5min":15.619,"15min":15.619},"last":0.26800000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:26:32 2023","interfaces":[{"interface":"back","average":{"1min":2.609,"5min":1.502,"15min":1.75},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":15.762,"5min":15.762,"15min":75.733000000000004},"last":0.29599999999999999},{"interface":"front","average":{"1min":1.764,"5min":1.6970000000000001,"15min":2.0059999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.28399999999999997},"max":{"1min":11.568,"5min":13.864000000000001,"15min":75.686999999999998},"last":2.1520000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:26:32 2023","interfaces":[{"interface":"back","average":{"1min":2.6509999999999998,"5min":2.097,"15min":1.802},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.32200000000000001},"max":{"1min":15.840999999999999,"5min":53.057000000000002,"15min":53.057000000000002},"last":0.31900000000000001},{"interface":"front","average":{"1min":1.8,"5min":2.266,"15min":1.9750000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.33300000000000002},"max":{"1min":9.8930000000000007,"5min":53.088000000000001,"15min":53.088000000000001},"last":2.2189999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:26:40 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":1.0600000000000001,"15min":0.92800000000000005},"min":{"1min":0.40400000000000003,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":6.806,"5min":6.806,"15min":6.806},"last":2.0859999999999999},{"interface":"front","average":{"1min":2.6909999999999998,"5min":1.5309999999999999,"15min":1.3320000000000001},"min":{"1min":0.47299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":15.795,"5min":15.795,"15min":15.795},"last":0.37}]}]},{"osd":4,"up_from":30,"seq":128849019326,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1508960,"kb_used_data":812888,"kb_used_omap":0,"kb_used_meta":696064,"kb_avail":92236192,"statfs":{"total":95995035648,"available":94449860608,"internally_reserved":0,"allocated":832397312,"data_stored":831167220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":712769536},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:26:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":3.1850000000000001,"15min":4.6050000000000004},"min":{"1min":0.25900000000000001,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":17.712,"5min":92.141999999999996,"15min":199.64400000000001},"last":0.23599999999999999},{"interface":"front","average":{"1min":1.744,"5min":3.29,"15min":4.8159999999999998},"min":{"1min":0.224,"5min":0.216,"15min":0.216},"max":{"1min":16.666,"5min":92.284000000000006,"15min":195.649},"last":0.318}]},{"osd":1,"last update":"Sat Sep 23 23:27:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":1.585,"15min":3.8650000000000002},"min":{"1min":0.24299999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.8700000000000001,"5min":40.643000000000001,"15min":161.923},"last":0.30299999999999999},{"interface":"front","average":{"1min":2.2650000000000001,"5min":1.901,"15min":4.0259999999999998},"min":{"1min":0.307,"5min":0.27500000000000002,"15min":0.26200000000000001},"max":{"1min":10.657999999999999,"5min":35.362000000000002,"15min":161.39699999999999},"last":0.26400000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":4.157,"15min":3.3079999999999998},"min":{"1min":0.27900000000000003,"5min":0.27500000000000002,"15min":0.26300000000000001},"max":{"1min":7.3239999999999998,"5min":97.962000000000003,"15min":97.962000000000003},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.7889999999999999,"5min":4.2460000000000004,"15min":2.992},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.3929999999999998,"5min":98.137,"15min":98.137},"last":0.47199999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:26:34 2023","interfaces":[{"interface":"back","average":{"1min":14.739000000000001,"5min":6.0430000000000001,"15min":5.5179999999999998},"min":{"1min":0.39400000000000002,"5min":0.27200000000000002,"15min":0.26300000000000001},"max":{"1min":176.74100000000001,"5min":176.74100000000001,"15min":303.41500000000002},"last":0.48099999999999998},{"interface":"front","average":{"1min":14.337,"5min":5.8579999999999997,"15min":5.6859999999999999},"min":{"1min":0.432,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":176.87799999999999,"5min":176.87799999999999,"15min":303.428},"last":0.39400000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:27:09 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.65000000000000002,"15min":1.4379999999999999},"min":{"1min":0.29499999999999998,"5min":0.26100000000000001,"15min":0.23699999999999999},"max":{"1min":4.266,"5min":4.266,"15min":160.79499999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.879,"5min":0.746,"15min":1.6080000000000001},"min":{"1min":0.34999999999999998,"5min":0.249,"15min":0.249},"max":{"1min":4.4359999999999999,"5min":5.0789999999999997,"15min":160.61099999999999},"last":0.36299999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:26:34 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":1.5840000000000001,"15min":1.2569999999999999},"min":{"1min":0.34100000000000003,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.1790000000000003,"5min":90.346999999999994,"15min":90.346999999999994},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.157,"5min":1.728,"15min":1.3779999999999999},"min":{"1min":0.34100000000000003,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":4.298,"5min":90.733000000000004,"15min":90.733000000000004},"last":0.49399999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":1.5209999999999999,"15min":0.88500000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.3280000000000003,"5min":61.923999999999999,"15min":61.923999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.83499999999999996,"5min":1.581,"15min":1.0620000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":2.1520000000000001,"5min":61.990000000000002,"15min":61.990000000000002},"last":0.45800000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757689,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1198592,"kb_used_data":597432,"kb_used_omap":0,"kb_used_meta":601152,"kb_avail":92546560,"statfs":{"total":95995035648,"available":94767677440,"internally_reserved":0,"allocated":611770368,"data_stored":610560475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":615579648},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:26:52 2023","interfaces":[{"interface":"back","average":{"1min":5.0419999999999998,"5min":5.2549999999999999,"15min":3.7789999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.19500000000000001},"max":{"1min":63.741999999999997,"5min":113.869,"15min":113.869},"last":19.952000000000002},{"interface":"front","average":{"1min":4.8609999999999998,"5min":6.5490000000000004,"15min":4.2549999999999999},"min":{"1min":0.23699999999999999,"5min":0.22900000000000001,"15min":0.22700000000000001},"max":{"1min":63.259,"5min":138.55199999999999,"15min":138.55199999999999},"last":2.4489999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:27:02 2023","interfaces":[{"interface":"back","average":{"1min":9.375,"5min":3.3940000000000001,"15min":3.9710000000000001},"min":{"1min":0.25900000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":69.816999999999993,"5min":69.816999999999993,"15min":193.584},"last":0.374},{"interface":"front","average":{"1min":10.48,"5min":3.8639999999999999,"15min":4.2549999999999999},"min":{"1min":0.375,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":70.268000000000001,"5min":70.268000000000001,"15min":194.03100000000001},"last":0.441}]},{"osd":2,"last update":"Sat Sep 23 23:26:35 2023","interfaces":[{"interface":"back","average":{"1min":9.984,"5min":4.7009999999999996,"15min":2.6040000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":144.91200000000001,"5min":144.91200000000001,"15min":144.91200000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":10.863,"5min":4.54,"15min":2.5590000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":145.15799999999999,"5min":145.15799999999999,"15min":145.15799999999999},"last":3.133}]},{"osd":3,"last update":"Sat Sep 23 23:26:29 2023","interfaces":[{"interface":"back","average":{"1min":4.2039999999999997,"5min":2.4329999999999998,"15min":4.2930000000000001},"min":{"1min":0.373,"5min":0.315,"15min":0.27400000000000002},"max":{"1min":41.682000000000002,"5min":57.283999999999999,"15min":159.53299999999999},"last":1.909},{"interface":"front","average":{"1min":4.3390000000000004,"5min":2.5449999999999999,"15min":4.6120000000000001},"min":{"1min":0.39600000000000002,"5min":0.33100000000000002,"15min":0.312},"max":{"1min":41.732999999999997,"5min":57.334000000000003,"15min":159.54900000000001},"last":1.9330000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:27:10 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.71699999999999997,"15min":0.98299999999999998},"min":{"1min":0.26700000000000002,"5min":0.251,"15min":0.23999999999999999},"max":{"1min":1.1919999999999999,"5min":5.3739999999999997,"15min":10.012},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.60899999999999999,"15min":0.76300000000000001},"min":{"1min":0.30299999999999999,"5min":0.245,"15min":0.245},"max":{"1min":3.2530000000000001,"5min":3.2530000000000001,"15min":11.923999999999999},"last":0.38400000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:27:10 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.4770000000000001,"15min":2.2250000000000001},"min":{"1min":0.33000000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.2749999999999999,"5min":68.456999999999994,"15min":310.68799999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":1.452,"15min":2.4209999999999998},"min":{"1min":0.34799999999999998,"5min":0.26000000000000001,"15min":0.25900000000000001},"max":{"1min":1.2050000000000001,"5min":66.721999999999994,"15min":311.45400000000001},"last":0.48599999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:26:35 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":2.9209999999999998,"15min":3.738},"min":{"1min":0.38500000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":3.3500000000000001,"5min":66.372,"15min":66.372},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.76800000000000002,"5min":2.996,"15min":3.8889999999999998},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":3.3239999999999998,"5min":66.558999999999997,"15min":66.558999999999997},"last":0.505}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1158086656,"data_stored":1158086656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":556269568,"data_stored":556269568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1212223488,"data_stored":1212219416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1082523648,"data_stored":1082523648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":821235712,"data_stored":821231640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":601686016,"data_stored":601686016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1465323520,"data_stored":1465319448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1607401472,"data_stored":1607401472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:27:35.899 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:27:35.924 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:27:35.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:35.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:27:36.030 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:27:36.065 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:36.125 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:27:36.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:36.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:27:36.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:36 smithi110 ceph-mon[221102]: failed to encode map e435 with expected crc 2023-09-23T23:27:36.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:36 smithi110 ceph-mon[212939]: failed to encode map e435 with expected crc 2023-09-23T23:27:36.232 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:36.265 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:36.322 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:36.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:36.327 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:36 smithi138 ceph-mon[193606]: failed to encode map e435 with expected crc 2023-09-23T23:27:36.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:36.429 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:27:36.520 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:36.584 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:27:36.743 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:27:36.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:36.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:27:36.852 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:36.935 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:37.018 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:37.020 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:27:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:27:36] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.43.0" 2023-09-23T23:27:37.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:37.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:37.091 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:27:37.122 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:27:37.150 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:37.233 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:27:37.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:37.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:37.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:37 smithi138 ceph-mon[193606]: pgmap v1227: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 114 active+clean; 1.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 1155/78759 objects degraded (1.466%); 15219/78759 objects misplaced (19.324%); 23 MiB/s, 366 objects/s recovering 2023-09-23T23:27:37.340 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:27:37.403 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:37 smithi110 ceph-mon[212939]: pgmap v1227: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 114 active+clean; 1.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 1155/78759 objects degraded (1.466%); 15219/78759 objects misplaced (19.324%); 23 MiB/s, 366 objects/s recovering 2023-09-23T23:27:37.405 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:37.405 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:37 smithi110 ceph-mon[221102]: pgmap v1227: 129 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 114 active+clean; 1.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 1155/78759 objects degraded (1.466%); 15219/78759 objects misplaced (19.324%); 23 MiB/s, 366 objects/s recovering 2023-09-23T23:27:37.628 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:27:37.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:37.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:27:37.736 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:27:37.773 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:27:37.776 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:37.797 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:27:37.834 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:27:37.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:37.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:27:37.941 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:27:37.975 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:38.037 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:27:38.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:38.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:38.143 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:38.176 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:38.246 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:38.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:38.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:38.352 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:27:38.382 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:38.433 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:27:38.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:38.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:38.509 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:27:38.536 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:38.569 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:38.574 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:27:38.621 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:38.625 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:27:38.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:38.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:38.668 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:27:38.726 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:38.729 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:27:38.755 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:38.791 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:27:38.810 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:38.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:38.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:38.850 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:27:38.894 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:27:38.917 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:27:38.917 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:27:38.917 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:27:38.917 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:27:38.917 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:27:38.917 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:27:38.917 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:27:38.917 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:27:38.917 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:27:38.950 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:39.017 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:27:39.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:39.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:27:39.123 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:27:39.195 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:39.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:39 smithi110 ceph-mon[212939]: pgmap v1228: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 115 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 1155/77250 objects degraded (1.495%); 14752/77250 objects misplaced (19.096%); 20 MiB/s, 327 objects/s recovering 2023-09-23T23:27:39.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:39 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2121559680' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:27:39.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:39 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1155/77250 objects degraded (1.495%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:27:39.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:39 smithi110 ceph-mon[221102]: pgmap v1228: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 115 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 1155/77250 objects degraded (1.495%); 14752/77250 objects misplaced (19.096%); 20 MiB/s, 327 objects/s recovering 2023-09-23T23:27:39.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:39 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2121559680' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:27:39.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:39 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1155/77250 objects degraded (1.495%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:27:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:39 smithi138 ceph-mon[193606]: pgmap v1228: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 115 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 1155/77250 objects degraded (1.495%); 14752/77250 objects misplaced (19.096%); 20 MiB/s, 327 objects/s recovering 2023-09-23T23:27:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:39 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2121559680' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:27:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:39 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1155/77250 objects degraded (1.495%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:27:40.820 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:27:40.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:40.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:27:40.926 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:27:40.959 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:41 smithi138 ceph-mon[193606]: pgmap v1229: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 115 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 827/70788 objects degraded (1.168%); 13875/70788 objects misplaced (19.601%); 12 MiB/s, 259 objects/s recovering 2023-09-23T23:27:41.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:41 smithi110 ceph-mon[212939]: pgmap v1229: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 115 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 827/70788 objects degraded (1.168%); 13875/70788 objects misplaced (19.601%); 12 MiB/s, 259 objects/s recovering 2023-09-23T23:27:41.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:41 smithi110 ceph-mon[221102]: pgmap v1229: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 115 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 827/70788 objects degraded (1.168%); 13875/70788 objects misplaced (19.601%); 12 MiB/s, 259 objects/s recovering 2023-09-23T23:27:41.657 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:27:41.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:41.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:27:41.767 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:27:41.819 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:41.941 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:27:41.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:41.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:27:42.045 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:42.083 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:42.319 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:42.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:42.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:42.375 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:42 smithi110 ceph-mon[212939]: osdmap e436: 8 total, 8 up, 7 in 2023-09-23T23:27:42.376 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:42 smithi110 ceph-mon[212939]: failed to encode map e436 with expected crc 2023-09-23T23:27:42.376 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:42 smithi110 ceph-mon[221102]: osdmap e436: 8 total, 8 up, 7 in 2023-09-23T23:27:42.376 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:42 smithi110 ceph-mon[221102]: failed to encode map e436 with expected crc 2023-09-23T23:27:42.428 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:27:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:42 smithi138 ceph-mon[193606]: osdmap e436: 8 total, 8 up, 7 in 2023-09-23T23:27:42.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:42 smithi138 ceph-mon[193606]: failed to encode map e436 with expected crc 2023-09-23T23:27:42.520 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:42.667 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:27:42.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:42.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:27:42.728 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:27:42.740 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:27:42.776 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:27:42.842 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:42.994 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:27:43.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:43.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:27:43.113 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:27:43.193 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:43.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[212939]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[212939]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[212939]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[212939]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[212939]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[212939]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[212939]: pgmap v1231: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 115 active+clean; 1.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 733 op/s; 827/66954 objects degraded (1.235%); 11529/66954 objects misplaced (17.219%); 11 MiB/s, 241 objects/s recovering 2023-09-23T23:27:43.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[212939]: failed to encode map e437 with expected crc 2023-09-23T23:27:43.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[212939]: failed to encode map e437 with expected crc 2023-09-23T23:27:43.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[212939]: failed to encode map e437 with expected crc 2023-09-23T23:27:43.206 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[212939]: osdmap e437: 8 total, 8 up, 7 in 2023-09-23T23:27:43.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[212939]: failed to encode map e437 with expected crc 2023-09-23T23:27:43.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:27:43.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:27:43.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[221102]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[221102]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[221102]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[221102]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[221102]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[221102]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[221102]: pgmap v1231: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 115 active+clean; 1.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 733 op/s; 827/66954 objects degraded (1.235%); 11529/66954 objects misplaced (17.219%); 11 MiB/s, 241 objects/s recovering 2023-09-23T23:27:43.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[221102]: failed to encode map e437 with expected crc 2023-09-23T23:27:43.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[221102]: failed to encode map e437 with expected crc 2023-09-23T23:27:43.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[221102]: failed to encode map e437 with expected crc 2023-09-23T23:27:43.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[221102]: osdmap e437: 8 total, 8 up, 7 in 2023-09-23T23:27:43.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[221102]: failed to encode map e437 with expected crc 2023-09-23T23:27:43.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:27:43.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:27:43.325 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:27:43.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:43.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:27:43.434 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:27:43.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:43 smithi138 ceph-mon[193606]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:43 smithi138 ceph-mon[193606]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:43 smithi138 ceph-mon[193606]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:43 smithi138 ceph-mon[193606]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:43 smithi138 ceph-mon[193606]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:43 smithi138 ceph-mon[193606]: failed to encode map e436 with expected crc 2023-09-23T23:27:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:43 smithi138 ceph-mon[193606]: pgmap v1231: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 115 active+clean; 1.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 733 op/s; 827/66954 objects degraded (1.235%); 11529/66954 objects misplaced (17.219%); 11 MiB/s, 241 objects/s recovering 2023-09-23T23:27:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:43 smithi138 ceph-mon[193606]: failed to encode map e437 with expected crc 2023-09-23T23:27:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:43 smithi138 ceph-mon[193606]: failed to encode map e437 with expected crc 2023-09-23T23:27:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:43 smithi138 ceph-mon[193606]: failed to encode map e437 with expected crc 2023-09-23T23:27:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:43 smithi138 ceph-mon[193606]: osdmap e437: 8 total, 8 up, 7 in 2023-09-23T23:27:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:43 smithi138 ceph-mon[193606]: failed to encode map e437 with expected crc 2023-09-23T23:27:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:27:43.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:27:43.473 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:43.592 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:27:43.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:43.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:43.696 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:27:43.746 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:43.822 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:27:43.853 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:27:43.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:43.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:43.956 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:44.043 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:44.173 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:27:44.259 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:27:44.259 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:27:44.259 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.259 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:192406 11.e 11:7b924e16:::benchmark_data_smithi110_433746_object86526:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-09-23T23:27:44.260 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:44.252811+0000", 2023-09-23T23:27:44.260 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.00453543, 2023-09-23T23:27:44.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0045622639999999999, 2023-09-23T23:27:44.260 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:27:44.260 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:27:44.260 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:27:44.260 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:27:44.260 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:27:44.260 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 192406 2023-09-23T23:27:44.260 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.261 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:27:44.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:27:44.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.252811+0000", 2023-09-23T23:27:44.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:27:44.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:27:44.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.252811+0000", 2023-09-23T23:27:44.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:27:44.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.262 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:27:44.262 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.252809+0000", 2023-09-23T23:27:44.262 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:27:44.262 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.262 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:27:44.262 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.252816+0000", 2023-09-23T23:27:44.262 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2420000000000003e-06 2023-09-23T23:27:44.262 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.263 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:27:44.263 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.252836+0000", 2023-09-23T23:27:44.263 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9432e-05 2023-09-23T23:27:44.263 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.263 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.263 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:27:44.263 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.252843+0000", 2023-09-23T23:27:44.263 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9189999999999999e-06 2023-09-23T23:27:44.263 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.263 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.263 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:27:44.263 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.252892+0000", 2023-09-23T23:27:44.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8616999999999999e-05 2023-09-23T23:27:44.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.264 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:27:44.264 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.253017+0000", 2023-09-23T23:27:44.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000125124 2023-09-23T23:27:44.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.264 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:27:44.264 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.253152+0000", 2023-09-23T23:27:44.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013505200000000001 2023-09-23T23:27:44.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.265 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:27:44.265 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.256395+0000", 2023-09-23T23:27:44.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032429500000000001 2023-09-23T23:27:44.265 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.265 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:27:44.265 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.256687+0000", 2023-09-23T23:27:44.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00029221599999999998 2023-09-23T23:27:44.265 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:27:44.265 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:27:44.266 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:27:44.266 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.266 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.266 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:192413 11.e 11:7f53c779:::benchmark_data_smithi110_433746_object86533:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-09-23T23:27:44.266 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:44.256714+0000", 2023-09-23T23:27:44.266 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.00063224400000000001, 2023-09-23T23:27:44.266 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00076506700000000003, 2023-09-23T23:27:44.266 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:27:44.266 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:27:44.266 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:27:44.266 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:27:44.266 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:27:44.267 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 192413 2023-09-23T23:27:44.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.267 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:27:44.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.267 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:27:44.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.256714+0000", 2023-09-23T23:27:44.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:27:44.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.267 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:27:44.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.256714+0000", 2023-09-23T23:27:44.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:27:44.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:27:44.268 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.256713+0000", 2023-09-23T23:27:44.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:27:44.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:27:44.268 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.256718+0000", 2023-09-23T23:27:44.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1950000000000002e-06 2023-09-23T23:27:44.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:27:44.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.256733+0000", 2023-09-23T23:27:44.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5107e-05 2023-09-23T23:27:44.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:27:44.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.256739+0000", 2023-09-23T23:27:44.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6789999999999999e-06 2023-09-23T23:27:44.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:27:44.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.256780+0000", 2023-09-23T23:27:44.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0197999999999997e-05 2023-09-23T23:27:44.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.270 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:27:44.270 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.256913+0000", 2023-09-23T23:27:44.270 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013328799999999999 2023-09-23T23:27:44.270 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:27:44.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:27:44.270 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:27:44.270 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.257051+0000", 2023-09-23T23:27:44.270 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013795100000000001 2023-09-23T23:27:44.270 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:27:44.270 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:27:44.270 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:27:44.271 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:27:44.271 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-09-23T23:27:44.271 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 2 2023-09-23T23:27:44.271 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:27:44.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:44 smithi138 ceph-mon[193606]: failed to encode map e437 with expected crc 2023-09-23T23:27:44.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:44 smithi138 ceph-mon[193606]: from='client.46124 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:27:44.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:44 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/807079744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:27:44.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:44 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:27:44.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:44 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 827/66954 objects degraded (1.235%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:27:44.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:44 smithi110 ceph-mon[212939]: failed to encode map e437 with expected crc 2023-09-23T23:27:44.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:44 smithi110 ceph-mon[212939]: from='client.46124 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:27:44.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:44 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/807079744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:27:44.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:44 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:27:44.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:44 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 827/66954 objects degraded (1.235%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:27:44.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:44 smithi110 ceph-mon[221102]: failed to encode map e437 with expected crc 2023-09-23T23:27:44.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:44 smithi110 ceph-mon[221102]: from='client.46124 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:27:44.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:44 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/807079744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:27:44.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:44 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:27:44.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:44 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 827/66954 objects degraded (1.235%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:27:44.464 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:44.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:44.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:44.570 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:27:44.616 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:45.092 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1232,"stamp":"2023-09-23T23:27:42.105184+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1461446733,"num_objects":22318,"num_object_clones":0,"num_object_copies":66954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":827,"num_objects_misplaced":11529,"num_objects_unfound":0,"num_objects_dirty":22318,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":166276,"num_write_kb":5311354,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60577,"num_bytes_recovered":3913388083,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167352,"ondisk_log_size":167352,"up":387,"acting":386,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":425,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10204072,"kb_used_data":5911856,"kb_used_omap":29,"kb_used_meta":4291810,"kb_avail":646011992,"statfs":{"total":671965249536,"available":661516279808,"internally_reserved":0,"allocated":6053740544,"data_stored":6041794777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":4394814163},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-332201984,"num_objects":-5069,"num_object_clones":0,"num_object_copies":-15207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-328,"num_objects_misplaced":-5948,"num_objects_unfound":0,"num_objects_dirty":-5069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5069,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1462,"num_bytes_recovered":63635456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.054491"},"pg_stats":[{"pgid":"11.12","version":"435'9227","reported_seq":13263,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T23:27:40.881759+0000","last_change":"2023-09-23T23:26:42.110093+0000","last_active":"2023-09-23T23:27:40.881759+0000","last_peered":"2023-09-23T23:27:40.881759+0000","last_clean":"2023-09-23T23:27:40.881759+0000","last_became_active":"2023-09-23T23:26:42.095788+0000","last_became_peered":"2023-09-23T23:26:42.095788+0000","last_unstale":"2023-09-23T23:27:40.881759+0000","last_undegraded":"2023-09-23T23:27:40.881759+0000","last_fullsized":"2023-09-23T23:27:40.881759+0000","mapping_epoch":404,"log_start":"355'800","ondisk_log_start":"355'800","created":365,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":8427,"log_dups_size":0,"ondisk_log_size":8427,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48922624,"num_objects":747,"num_object_clones":0,"num_object_copies":2241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5981,"num_write_kb":215264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1954,"num_bytes_recovered":128057344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":621,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.056000+0000","last_change":"2023-09-23T23:26:50.557273+0000","last_active":"2023-09-23T23:27:33.056000+0000","last_peered":"2023-09-23T23:27:33.056000+0000","last_clean":"2023-09-23T23:27:33.056000+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:27:33.056000+0000","last_undegraded":"2023-09-23T23:27:33.056000+0000","last_fullsized":"2023-09-23T23:27:33.056000+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.058755+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.058755+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023428590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":469,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061974+0000","last_change":"2023-09-23T23:22:38.439629+0000","last_active":"2023-09-23T23:27:33.061974+0000","last_peered":"2023-09-23T23:27:33.061974+0000","last_clean":"2023-09-23T23:27:33.061974+0000","last_became_active":"2023-09-23T23:22:38.439285+0000","last_became_peered":"2023-09-23T23:22:38.439285+0000","last_unstale":"2023-09-23T23:27:33.061974+0000","last_undegraded":"2023-09-23T23:27:33.061974+0000","last_fullsized":"2023-09-23T23:27:33.061974+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:34.431221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":404,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.051139+0000","last_change":"2023-09-23T23:23:43.511912+0000","last_active":"2023-09-23T23:27:33.051139+0000","last_peered":"2023-09-23T23:27:33.051139+0000","last_clean":"2023-09-23T23:27:33.051139+0000","last_became_active":"2023-09-23T23:23:43.511574+0000","last_became_peered":"2023-09-23T23:23:43.511574+0000","last_unstale":"2023-09-23T23:27:33.051139+0000","last_undegraded":"2023-09-23T23:27:33.051139+0000","last_fullsized":"2023-09-23T23:27:33.051139+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:16.286926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"435'9157","reported_seq":15891,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T23:27:40.913069+0000","last_change":"2023-09-23T23:27:34.096906+0000","last_active":"2023-09-23T23:27:40.913069+0000","last_peered":"2023-09-23T23:27:40.913069+0000","last_clean":"2023-09-23T23:27:40.913069+0000","last_became_active":"2023-09-23T23:27:34.078923+0000","last_became_peered":"2023-09-23T23:27:34.078923+0000","last_unstale":"2023-09-23T23:27:40.913069+0000","last_undegraded":"2023-09-23T23:27:40.913069+0000","last_fullsized":"2023-09-23T23:27:40.913069+0000","mapping_epoch":434,"log_start":"355'1400","ondisk_log_start":"355'1400","created":365,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":7757,"log_dups_size":0,"ondisk_log_size":7757,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43548672,"num_objects":665,"num_object_clones":0,"num_object_copies":1995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5955,"num_write_kb":211808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2413,"num_bytes_recovered":158138368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":524,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.055661+0000","last_change":"2023-09-23T23:23:50.459634+0000","last_active":"2023-09-23T23:27:33.055661+0000","last_peered":"2023-09-23T23:27:33.055661+0000","last_clean":"2023-09-23T23:27:33.055661+0000","last_became_active":"2023-09-23T23:23:45.453958+0000","last_became_peered":"2023-09-23T23:23:45.453958+0000","last_unstale":"2023-09-23T23:27:33.055661+0000","last_undegraded":"2023-09-23T23:27:33.055661+0000","last_fullsized":"2023-09-23T23:27:33.055661+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:56.563201+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":145,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.060844+0000","last_change":"2023-09-23T23:26:11.528846+0000","last_active":"2023-09-23T23:27:33.060844+0000","last_peered":"2023-09-23T23:27:33.060844+0000","last_clean":"2023-09-23T23:27:33.060844+0000","last_became_active":"2023-09-23T23:26:11.528529+0000","last_became_peered":"2023-09-23T23:26:11.528529+0000","last_unstale":"2023-09-23T23:27:33.060844+0000","last_undegraded":"2023-09-23T23:27:33.060844+0000","last_fullsized":"2023-09-23T23:27:33.060844+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:35.094583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":453,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.068247+0000","last_change":"2023-09-23T23:26:48.558967+0000","last_active":"2023-09-23T23:27:33.068247+0000","last_peered":"2023-09-23T23:27:33.068247+0000","last_clean":"2023-09-23T23:27:33.068247+0000","last_became_active":"2023-09-23T23:26:48.555271+0000","last_became_peered":"2023-09-23T23:26:48.555271+0000","last_unstale":"2023-09-23T23:27:33.068247+0000","last_undegraded":"2023-09-23T23:27:33.068247+0000","last_fullsized":"2023-09-23T23:27:33.068247+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:50:57.509734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"435'9045","reported_seq":22542,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T23:27:40.654070+0000","last_change":"2023-09-23T23:27:24.606995+0000","last_active":"2023-09-23T23:27:40.654070+0000","last_peered":"2023-09-23T23:27:40.654070+0000","last_clean":"2023-09-23T23:27:40.654070+0000","last_became_active":"2023-09-23T23:27:24.589817+0000","last_became_peered":"2023-09-23T23:27:24.589817+0000","last_unstale":"2023-09-23T23:27:40.654070+0000","last_undegraded":"2023-09-23T23:27:40.654070+0000","last_fullsized":"2023-09-23T23:27:40.654070+0000","mapping_epoch":428,"log_start":"356'2537","ondisk_log_start":"356'2537","created":365,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"366'6410","last_scrub_stamp":"2023-09-23T23:25:21.346991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:21.346991+0000","objects_scrubbed":3231,"log_size":6508,"log_dups_size":0,"ondisk_log_size":6508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:37.469662+0000","scrub_duration":0.84688343099999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43319296,"num_objects":661,"num_object_clones":0,"num_object_copies":1983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2650,"num_write_kb":2560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3371,"num_bytes_recovered":208666624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":725,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061752+0000","last_change":"2023-09-23T23:27:23.551379+0000","last_active":"2023-09-23T23:27:33.061752+0000","last_peered":"2023-09-23T23:27:33.061752+0000","last_clean":"2023-09-23T23:27:33.061752+0000","last_became_active":"2023-09-23T23:27:23.551046+0000","last_became_peered":"2023-09-23T23:27:23.551046+0000","last_unstale":"2023-09-23T23:27:33.061752+0000","last_undegraded":"2023-09-23T23:27:33.061752+0000","last_fullsized":"2023-09-23T23:27:33.061752+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.842272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.842272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.023100959000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":431,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.050479+0000","last_change":"2023-09-23T23:27:20.554007+0000","last_active":"2023-09-23T23:27:33.050479+0000","last_peered":"2023-09-23T23:27:33.050479+0000","last_clean":"2023-09-23T23:27:33.050479+0000","last_became_active":"2023-09-23T23:27:20.553716+0000","last_became_peered":"2023-09-23T23:27:20.553716+0000","last_unstale":"2023-09-23T23:27:33.050479+0000","last_undegraded":"2023-09-23T23:27:33.050479+0000","last_fullsized":"2023-09-23T23:27:33.050479+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:38.911862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":368,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T23:27:34.052959+0000","last_change":"2023-09-23T23:25:31.735427+0000","last_active":"2023-09-23T23:27:34.052959+0000","last_peered":"2023-09-23T23:27:34.052959+0000","last_clean":"2023-09-23T23:27:34.052959+0000","last_became_active":"2023-09-23T23:25:31.734582+0000","last_became_peered":"2023-09-23T23:25:31.734582+0000","last_unstale":"2023-09-23T23:27:34.052959+0000","last_undegraded":"2023-09-23T23:27:34.052959+0000","last_fullsized":"2023-09-23T23:27:34.052959+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:39.075149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":755,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.058484+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:27:33.058484+0000","last_peered":"2023-09-23T23:27:33.058484+0000","last_clean":"2023-09-23T23:27:33.058484+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:27:33.058484+0000","last_undegraded":"2023-09-23T23:27:33.058484+0000","last_fullsized":"2023-09-23T23:27:33.058484+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":388,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.071172+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:27:33.071172+0000","last_peered":"2023-09-23T23:27:33.071172+0000","last_clean":"2023-09-23T23:27:33.071172+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:27:33.071172+0000","last_undegraded":"2023-09-23T23:27:33.071172+0000","last_fullsized":"2023-09-23T23:27:33.071172+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":423,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.050316+0000","last_change":"2023-09-23T23:26:11.528354+0000","last_active":"2023-09-23T23:27:33.050316+0000","last_peered":"2023-09-23T23:27:33.050316+0000","last_clean":"2023-09-23T23:27:33.050316+0000","last_became_active":"2023-09-23T23:26:11.528076+0000","last_became_peered":"2023-09-23T23:26:11.528076+0000","last_unstale":"2023-09-23T23:27:33.050316+0000","last_undegraded":"2023-09-23T23:27:33.050316+0000","last_fullsized":"2023-09-23T23:27:33.050316+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:16.100396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":707,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.048774+0000","last_change":"2023-09-23T23:26:14.521792+0000","last_active":"2023-09-23T23:27:33.048774+0000","last_peered":"2023-09-23T23:27:33.048774+0000","last_clean":"2023-09-23T23:27:33.048774+0000","last_became_active":"2023-09-23T23:26:14.521438+0000","last_became_peered":"2023-09-23T23:26:14.521438+0000","last_unstale":"2023-09-23T23:27:33.048774+0000","last_undegraded":"2023-09-23T23:27:33.048774+0000","last_fullsized":"2023-09-23T23:27:33.048774+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:09.103712+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":66,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.055179+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:27:33.055179+0000","last_peered":"2023-09-23T23:27:33.055179+0000","last_clean":"2023-09-23T23:27:33.055179+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:27:33.055179+0000","last_undegraded":"2023-09-23T23:27:33.055179+0000","last_fullsized":"2023-09-23T23:27:33.055179+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":384,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.053898+0000","last_change":"2023-09-23T23:26:48.538059+0000","last_active":"2023-09-23T23:27:33.053898+0000","last_peered":"2023-09-23T23:27:33.053898+0000","last_clean":"2023-09-23T23:27:33.053898+0000","last_became_active":"2023-09-23T23:26:48.537735+0000","last_became_peered":"2023-09-23T23:26:48.537735+0000","last_unstale":"2023-09-23T23:27:33.053898+0000","last_undegraded":"2023-09-23T23:27:33.053898+0000","last_fullsized":"2023-09-23T23:27:33.053898+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:32.878591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":819,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.055601+0000","last_change":"2023-09-23T23:26:51.563894+0000","last_active":"2023-09-23T23:27:33.055601+0000","last_peered":"2023-09-23T23:27:33.055601+0000","last_clean":"2023-09-23T23:27:33.055601+0000","last_became_active":"2023-09-23T23:26:51.563536+0000","last_became_peered":"2023-09-23T23:26:51.563536+0000","last_unstale":"2023-09-23T23:27:33.055601+0000","last_undegraded":"2023-09-23T23:27:33.055601+0000","last_fullsized":"2023-09-23T23:27:33.055601+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:54.095054+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:54.095054+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":40,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.057903+0000","last_change":"2023-09-23T23:26:48.669857+0000","last_active":"2023-09-23T23:27:33.057903+0000","last_peered":"2023-09-23T23:27:33.057903+0000","last_clean":"2023-09-23T23:27:33.057903+0000","last_became_active":"2023-09-23T23:26:48.538246+0000","last_became_peered":"2023-09-23T23:26:48.538246+0000","last_unstale":"2023-09-23T23:27:33.057903+0000","last_undegraded":"2023-09-23T23:27:33.057903+0000","last_fullsized":"2023-09-23T23:27:33.057903+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:19.039472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":21,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061092+0000","last_change":"2023-09-23T23:27:20.551305+0000","last_active":"2023-09-23T23:27:33.061092+0000","last_peered":"2023-09-23T23:27:33.061092+0000","last_clean":"2023-09-23T23:27:33.061092+0000","last_became_active":"2023-09-23T23:27:20.551017+0000","last_became_peered":"2023-09-23T23:27:20.551017+0000","last_unstale":"2023-09-23T23:27:33.061092+0000","last_undegraded":"2023-09-23T23:27:33.061092+0000","last_fullsized":"2023-09-23T23:27:33.061092+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:27.026015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":509,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.062316+0000","last_change":"2023-09-23T23:24:02.268079+0000","last_active":"2023-09-23T23:27:33.062316+0000","last_peered":"2023-09-23T23:27:33.062316+0000","last_clean":"2023-09-23T23:27:33.062316+0000","last_became_active":"2023-09-23T23:23:45.463106+0000","last_became_peered":"2023-09-23T23:23:45.463106+0000","last_unstale":"2023-09-23T23:27:33.062316+0000","last_undegraded":"2023-09-23T23:27:33.062316+0000","last_fullsized":"2023-09-23T23:27:33.062316+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.128813+0000","scrub_duration":0.0018535310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":66,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.062415+0000","last_change":"2023-09-23T23:26:11.534727+0000","last_active":"2023-09-23T23:27:33.062415+0000","last_peered":"2023-09-23T23:27:33.062415+0000","last_clean":"2023-09-23T23:27:33.062415+0000","last_became_active":"2023-09-23T23:26:11.513694+0000","last_became_peered":"2023-09-23T23:26:11.513694+0000","last_unstale":"2023-09-23T23:27:33.062415+0000","last_undegraded":"2023-09-23T23:27:33.062415+0000","last_fullsized":"2023-09-23T23:27:33.062415+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:04.562837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":423,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061945+0000","last_change":"2023-09-23T23:25:31.718910+0000","last_active":"2023-09-23T23:27:33.061945+0000","last_peered":"2023-09-23T23:27:33.061945+0000","last_clean":"2023-09-23T23:27:33.061945+0000","last_became_active":"2023-09-23T23:25:31.718567+0000","last_became_peered":"2023-09-23T23:25:31.718567+0000","last_unstale":"2023-09-23T23:27:33.061945+0000","last_undegraded":"2023-09-23T23:27:33.061945+0000","last_fullsized":"2023-09-23T23:27:33.061945+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:21.447944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":534,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.055119+0000","last_change":"2023-09-23T23:26:34.550496+0000","last_active":"2023-09-23T23:27:33.055119+0000","last_peered":"2023-09-23T23:27:33.055119+0000","last_clean":"2023-09-23T23:27:33.055119+0000","last_became_active":"2023-09-23T23:26:34.550151+0000","last_became_peered":"2023-09-23T23:26:34.550151+0000","last_unstale":"2023-09-23T23:27:33.055119+0000","last_undegraded":"2023-09-23T23:27:33.055119+0000","last_fullsized":"2023-09-23T23:27:33.055119+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:53.828311+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:53.828311+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":367,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.057970+0000","last_change":"2023-09-23T23:26:11.569228+0000","last_active":"2023-09-23T23:27:33.057970+0000","last_peered":"2023-09-23T23:27:33.057970+0000","last_clean":"2023-09-23T23:27:33.057970+0000","last_became_active":"2023-09-23T23:26:11.513159+0000","last_became_peered":"2023-09-23T23:26:11.513159+0000","last_unstale":"2023-09-23T23:27:33.057970+0000","last_undegraded":"2023-09-23T23:27:33.057970+0000","last_fullsized":"2023-09-23T23:27:33.057970+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:40.698257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":431,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.050690+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:27:33.050690+0000","last_peered":"2023-09-23T23:27:33.050690+0000","last_clean":"2023-09-23T23:27:33.050690+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:27:33.050690+0000","last_undegraded":"2023-09-23T23:27:33.050690+0000","last_fullsized":"2023-09-23T23:27:33.050690+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":774,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.057166+0000","last_change":"2023-09-23T23:26:34.551458+0000","last_active":"2023-09-23T23:27:33.057166+0000","last_peered":"2023-09-23T23:27:33.057166+0000","last_clean":"2023-09-23T23:27:33.057166+0000","last_became_active":"2023-09-23T23:26:34.551067+0000","last_became_peered":"2023-09-23T23:26:34.551067+0000","last_unstale":"2023-09-23T23:27:33.057166+0000","last_undegraded":"2023-09-23T23:27:33.057166+0000","last_fullsized":"2023-09-23T23:27:33.057166+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:05.788656+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":34,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.057672+0000","last_change":"2023-09-23T23:26:48.563723+0000","last_active":"2023-09-23T23:27:33.057672+0000","last_peered":"2023-09-23T23:27:33.057672+0000","last_clean":"2023-09-23T23:27:33.057672+0000","last_became_active":"2023-09-23T23:26:48.563340+0000","last_became_peered":"2023-09-23T23:26:48.563340+0000","last_unstale":"2023-09-23T23:27:33.057672+0000","last_undegraded":"2023-09-23T23:27:33.057672+0000","last_fullsized":"2023-09-23T23:27:33.057672+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:05.974131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":321,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.058413+0000","last_change":"2023-09-23T23:26:32.040316+0000","last_active":"2023-09-23T23:27:33.058413+0000","last_peered":"2023-09-23T23:27:33.058413+0000","last_clean":"2023-09-23T23:27:33.058413+0000","last_became_active":"2023-09-23T23:26:32.039758+0000","last_became_peered":"2023-09-23T23:26:32.039758+0000","last_unstale":"2023-09-23T23:27:33.058413+0000","last_undegraded":"2023-09-23T23:27:33.058413+0000","last_fullsized":"2023-09-23T23:27:33.058413+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:03.930331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.19","version":"435'10823","reported_seq":15926,"reported_epoch":435,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:38.677278+0000","last_change":"2023-09-23T23:27:21.715633+0000","last_active":"2023-09-23T23:27:38.677278+0000","last_peered":"2023-09-23T23:27:38.677278+0000","last_clean":"2023-09-23T23:27:19.516646+0000","last_became_active":"2023-09-23T23:27:21.687051+0000","last_became_peered":"2023-09-23T23:27:21.687051+0000","last_unstale":"2023-09-23T23:27:38.677278+0000","last_undegraded":"2023-09-23T23:27:38.677278+0000","last_fullsized":"2023-09-23T23:27:38.677278+0000","mapping_epoch":425,"log_start":"363'6321","ondisk_log_start":"363'6321","created":365,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":1949,"log_size":4502,"log_dups_size":0,"ondisk_log_size":4502,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43417600,"num_objects":663,"num_object_clones":0,"num_object_copies":1989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":663,"num_objects_unfound":0,"num_objects_dirty":663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6007,"num_write_kb":213408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2401,"num_bytes_recovered":156762112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,7],"avail_no_missing":["3","0","6","7"],"object_location_counts":[{"shards":"0,3,7","objects":663}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":598,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061813+0000","last_change":"2023-09-23T23:26:14.527752+0000","last_active":"2023-09-23T23:27:33.061813+0000","last_peered":"2023-09-23T23:27:33.061813+0000","last_clean":"2023-09-23T23:27:33.061813+0000","last_became_active":"2023-09-23T23:26:14.525188+0000","last_became_peered":"2023-09-23T23:26:14.525188+0000","last_unstale":"2023-09-23T23:27:33.061813+0000","last_undegraded":"2023-09-23T23:27:33.061813+0000","last_fullsized":"2023-09-23T23:27:33.061813+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:10.617744+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":289,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.057908+0000","last_change":"2023-09-23T23:25:31.725776+0000","last_active":"2023-09-23T23:27:33.057908+0000","last_peered":"2023-09-23T23:27:33.057908+0000","last_clean":"2023-09-23T23:27:33.057908+0000","last_became_active":"2023-09-23T23:25:31.725429+0000","last_became_peered":"2023-09-23T23:25:31.725429+0000","last_unstale":"2023-09-23T23:27:33.057908+0000","last_undegraded":"2023-09-23T23:27:33.057908+0000","last_fullsized":"2023-09-23T23:27:33.057908+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:07.289251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":424,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.068378+0000","last_change":"2023-09-23T23:26:48.559268+0000","last_active":"2023-09-23T23:27:33.068378+0000","last_peered":"2023-09-23T23:27:33.068378+0000","last_clean":"2023-09-23T23:27:33.068378+0000","last_became_active":"2023-09-23T23:26:48.558563+0000","last_became_peered":"2023-09-23T23:26:48.558563+0000","last_unstale":"2023-09-23T23:27:33.068378+0000","last_undegraded":"2023-09-23T23:27:33.068378+0000","last_fullsized":"2023-09-23T23:27:33.068378+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:58.779288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"435'9084","reported_seq":14892,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T23:27:38.764458+0000","last_change":"2023-09-23T23:27:30.609575+0000","last_active":"2023-09-23T23:27:38.764458+0000","last_peered":"2023-09-23T23:27:38.764458+0000","last_clean":"2023-09-23T23:27:38.764458+0000","last_became_active":"2023-09-23T23:27:30.579515+0000","last_became_peered":"2023-09-23T23:27:30.579515+0000","last_unstale":"2023-09-23T23:27:38.764458+0000","last_undegraded":"2023-09-23T23:27:38.764458+0000","last_fullsized":"2023-09-23T23:27:38.764458+0000","mapping_epoch":432,"log_start":"357'3148","ondisk_log_start":"357'3148","created":365,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5936,"log_dups_size":0,"ondisk_log_size":5936,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41582592,"num_objects":635,"num_object_clones":0,"num_object_copies":1905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5911,"num_write_kb":209440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2724,"num_bytes_recovered":178520064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":778,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.055745+0000","last_change":"2023-09-23T23:27:23.549144+0000","last_active":"2023-09-23T23:27:33.055745+0000","last_peered":"2023-09-23T23:27:33.055745+0000","last_clean":"2023-09-23T23:27:33.055745+0000","last_became_active":"2023-09-23T23:27:23.548774+0000","last_became_peered":"2023-09-23T23:27:23.548774+0000","last_unstale":"2023-09-23T23:27:33.055745+0000","last_undegraded":"2023-09-23T23:27:33.055745+0000","last_fullsized":"2023-09-23T23:27:33.055745+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:18.211722+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":115,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.068128+0000","last_change":"2023-09-23T23:26:32.139091+0000","last_active":"2023-09-23T23:27:33.068128+0000","last_peered":"2023-09-23T23:27:33.068128+0000","last_clean":"2023-09-23T23:27:33.068128+0000","last_became_active":"2023-09-23T23:26:32.053033+0000","last_became_peered":"2023-09-23T23:26:32.053033+0000","last_unstale":"2023-09-23T23:27:33.068128+0000","last_undegraded":"2023-09-23T23:27:33.068128+0000","last_fullsized":"2023-09-23T23:27:33.068128+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:32.625818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.049804+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:27:33.049804+0000","last_peered":"2023-09-23T23:27:33.049804+0000","last_clean":"2023-09-23T23:27:33.049804+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:27:33.049804+0000","last_undegraded":"2023-09-23T23:27:33.049804+0000","last_fullsized":"2023-09-23T23:27:33.049804+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"435'9089","reported_seq":13099,"reported_epoch":435,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:40.298211+0000","last_change":"2023-09-23T23:27:21.731801+0000","last_active":"2023-09-23T23:27:40.298211+0000","last_peered":"2023-09-23T23:27:40.298211+0000","last_clean":"2023-09-23T23:27:19.451010+0000","last_became_active":"2023-09-23T23:27:21.704869+0000","last_became_peered":"2023-09-23T23:27:21.704869+0000","last_unstale":"2023-09-23T23:27:40.298211+0000","last_undegraded":"2023-09-23T23:27:40.298211+0000","last_fullsized":"2023-09-23T23:27:40.298211+0000","mapping_epoch":425,"log_start":"355'1862","ondisk_log_start":"355'1862","created":349,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"367'6563","last_scrub_stamp":"2023-09-23T23:25:25.026782+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:25.026782+0000","objects_scrubbed":3175,"log_size":7227,"log_dups_size":0,"ondisk_log_size":7227,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:37.076814+0000","scrub_duration":0.41131605999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42270720,"num_objects":645,"num_object_clones":0,"num_object_copies":1935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":645,"num_objects_unfound":0,"num_objects_dirty":645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2530,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1868,"num_bytes_recovered":122421248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[0,5,4],"avail_no_missing":["0","4","5","7"],"object_location_counts":[{"shards":"0,4,5","objects":645}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":762,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.057990+0000","last_change":"2023-09-23T23:26:51.563868+0000","last_active":"2023-09-23T23:27:33.057990+0000","last_peered":"2023-09-23T23:27:33.057990+0000","last_clean":"2023-09-23T23:27:33.057990+0000","last_became_active":"2023-09-23T23:26:51.563510+0000","last_became_peered":"2023-09-23T23:26:51.563510+0000","last_unstale":"2023-09-23T23:27:33.057990+0000","last_undegraded":"2023-09-23T23:27:33.057990+0000","last_fullsized":"2023-09-23T23:27:33.057990+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:34.244822+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":38,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.059193+0000","last_change":"2023-09-23T23:26:48.673221+0000","last_active":"2023-09-23T23:27:33.059193+0000","last_peered":"2023-09-23T23:27:33.059193+0000","last_clean":"2023-09-23T23:27:33.059193+0000","last_became_active":"2023-09-23T23:26:48.555920+0000","last_became_peered":"2023-09-23T23:26:48.555920+0000","last_unstale":"2023-09-23T23:27:33.059193+0000","last_undegraded":"2023-09-23T23:27:33.059193+0000","last_fullsized":"2023-09-23T23:27:33.059193+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:26.586824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":56,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.055921+0000","last_change":"2023-09-23T23:26:48.525190+0000","last_active":"2023-09-23T23:27:33.055921+0000","last_peered":"2023-09-23T23:27:33.055921+0000","last_clean":"2023-09-23T23:27:33.055921+0000","last_became_active":"2023-09-23T23:26:48.524874+0000","last_became_peered":"2023-09-23T23:26:48.524874+0000","last_unstale":"2023-09-23T23:27:33.055921+0000","last_undegraded":"2023-09-23T23:27:33.055921+0000","last_fullsized":"2023-09-23T23:27:33.055921+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:43.114769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"435'9157","reported_seq":9350,"reported_epoch":435,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:40.657220+0000","last_change":"2023-09-23T23:27:21.759725+0000","last_active":"2023-09-23T23:27:40.657220+0000","last_peered":"2023-09-23T23:27:40.657220+0000","last_clean":"2023-09-23T23:27:19.475740+0000","last_became_active":"2023-09-23T23:27:21.718647+0000","last_became_peered":"2023-09-23T23:27:21.718647+0000","last_unstale":"2023-09-23T23:27:40.657220+0000","last_undegraded":"2023-09-23T23:27:40.657220+0000","last_fullsized":"2023-09-23T23:27:40.657220+0000","mapping_epoch":425,"log_start":"355'1896","ondisk_log_start":"355'1896","created":349,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"366'6545","last_scrub_stamp":"2023-09-23T23:25:22.335514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:22.335514+0000","objects_scrubbed":3321,"log_size":7261,"log_dups_size":0,"ondisk_log_size":7261,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:12.578946+0000","scrub_duration":0.75924003299999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46465024,"num_objects":709,"num_object_clones":0,"num_object_copies":2127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":709,"num_objects_unfound":0,"num_objects_dirty":709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2616,"num_write_kb":128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,4],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":709}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":822,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.050115+0000","last_change":"2023-09-23T23:26:14.530414+0000","last_active":"2023-09-23T23:27:33.050115+0000","last_peered":"2023-09-23T23:27:33.050115+0000","last_clean":"2023-09-23T23:27:33.050115+0000","last_became_active":"2023-09-23T23:26:14.530001+0000","last_became_peered":"2023-09-23T23:26:14.530001+0000","last_unstale":"2023-09-23T23:27:33.050115+0000","last_undegraded":"2023-09-23T23:27:33.050115+0000","last_fullsized":"2023-09-23T23:27:33.050115+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:37.443998+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":38,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.058860+0000","last_change":"2023-09-23T23:26:48.547146+0000","last_active":"2023-09-23T23:27:33.058860+0000","last_peered":"2023-09-23T23:27:33.058860+0000","last_clean":"2023-09-23T23:27:33.058860+0000","last_became_active":"2023-09-23T23:26:48.537682+0000","last_became_peered":"2023-09-23T23:26:48.537682+0000","last_unstale":"2023-09-23T23:27:33.058860+0000","last_undegraded":"2023-09-23T23:27:33.058860+0000","last_fullsized":"2023-09-23T23:27:33.058860+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:10.115563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":460,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061531+0000","last_change":"2023-09-23T23:26:01.833889+0000","last_active":"2023-09-23T23:27:33.061531+0000","last_peered":"2023-09-23T23:27:33.061531+0000","last_clean":"2023-09-23T23:27:33.061531+0000","last_became_active":"2023-09-23T23:26:01.833626+0000","last_became_peered":"2023-09-23T23:26:01.833626+0000","last_unstale":"2023-09-23T23:27:33.061531+0000","last_undegraded":"2023-09-23T23:27:33.061531+0000","last_fullsized":"2023-09-23T23:27:33.061531+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:25.659671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"435'9091","reported_seq":17787,"reported_epoch":435,"state":"active+clean+remapped","last_fresh":"2023-09-23T23:27:40.661372+0000","last_change":"2023-09-23T23:27:40.549085+0000","last_active":"2023-09-23T23:27:40.661372+0000","last_peered":"2023-09-23T23:27:40.661372+0000","last_clean":"2023-09-23T23:27:40.661372+0000","last_became_active":"2023-09-23T23:26:48.619251+0000","last_became_peered":"2023-09-23T23:26:48.619251+0000","last_unstale":"2023-09-23T23:27:40.661372+0000","last_undegraded":"2023-09-23T23:27:40.661372+0000","last_fullsized":"2023-09-23T23:27:40.661372+0000","mapping_epoch":410,"log_start":"355'1800","ondisk_log_start":"355'1800","created":349,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":7291,"log_dups_size":0,"ondisk_log_size":7291,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47284224,"num_objects":721,"num_object_clones":0,"num_object_copies":2163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5897,"num_write_kb":211808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3490,"num_bytes_recovered":228720640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[0,2,4],"avail_no_missing":["0","2","3","4","5","7"],"object_location_counts":[{"shards":"0,2,4","objects":721}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":757,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.055676+0000","last_change":"2023-09-23T23:27:30.256071+0000","last_active":"2023-09-23T23:27:33.055676+0000","last_peered":"2023-09-23T23:27:33.055676+0000","last_clean":"2023-09-23T23:27:33.055676+0000","last_became_active":"2023-09-23T23:26:14.521164+0000","last_became_peered":"2023-09-23T23:26:14.521164+0000","last_unstale":"2023-09-23T23:27:33.055676+0000","last_undegraded":"2023-09-23T23:27:33.055676+0000","last_fullsized":"2023-09-23T23:27:33.055676+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.042580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.042580+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":341,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.058298+0000","last_change":"2023-09-23T23:26:32.043926+0000","last_active":"2023-09-23T23:27:33.058298+0000","last_peered":"2023-09-23T23:27:33.058298+0000","last_clean":"2023-09-23T23:27:33.058298+0000","last_became_active":"2023-09-23T23:26:32.039282+0000","last_became_peered":"2023-09-23T23:26:32.039282+0000","last_unstale":"2023-09-23T23:27:33.058298+0000","last_undegraded":"2023-09-23T23:27:33.058298+0000","last_fullsized":"2023-09-23T23:27:33.058298+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:35.345086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":423,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061178+0000","last_change":"2023-09-23T23:26:48.536996+0000","last_active":"2023-09-23T23:27:33.061178+0000","last_peered":"2023-09-23T23:27:33.061178+0000","last_clean":"2023-09-23T23:27:33.061178+0000","last_became_active":"2023-09-23T23:26:48.536638+0000","last_became_peered":"2023-09-23T23:26:48.536638+0000","last_unstale":"2023-09-23T23:27:33.061178+0000","last_undegraded":"2023-09-23T23:27:33.061178+0000","last_fullsized":"2023-09-23T23:27:33.061178+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:36.943318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.4","version":"435'9244","reported_seq":14412,"reported_epoch":435,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:27:38.135673+0000","last_change":"2023-09-23T23:27:22.936603+0000","last_active":"2023-09-23T23:27:38.135673+0000","last_peered":"2023-09-23T23:27:38.135673+0000","last_clean":"2023-09-23T23:25:30.577110+0000","last_became_active":"2023-09-23T23:25:32.991045+0000","last_became_peered":"2023-09-23T23:25:32.991045+0000","last_unstale":"2023-09-23T23:27:38.135673+0000","last_undegraded":"2023-09-23T23:25:32.735490+0000","last_fullsized":"2023-09-23T23:25:32.712272+0000","mapping_epoch":372,"log_start":"357'3518","ondisk_log_start":"357'3518","created":349,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5726,"log_dups_size":0,"ondisk_log_size":5726,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44924928,"num_objects":685,"num_object_clones":0,"num_object_copies":2055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":827,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5997,"num_write_kb":213856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2521,"num_bytes_recovered":133038080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,7","objects":-142}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":486,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.062485+0000","last_change":"2023-09-23T23:27:22.721185+0000","last_active":"2023-09-23T23:27:33.062485+0000","last_peered":"2023-09-23T23:27:33.062485+0000","last_clean":"2023-09-23T23:27:33.062485+0000","last_became_active":"2023-09-23T23:26:51.588385+0000","last_became_peered":"2023-09-23T23:26:51.588385+0000","last_unstale":"2023-09-23T23:27:33.062485+0000","last_undegraded":"2023-09-23T23:27:33.062485+0000","last_fullsized":"2023-09-23T23:27:33.062485+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:04.832687+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:04.832687+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":262,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.050187+0000","last_change":"2023-09-23T23:25:31.732607+0000","last_active":"2023-09-23T23:27:33.050187+0000","last_peered":"2023-09-23T23:27:33.050187+0000","last_clean":"2023-09-23T23:27:33.050187+0000","last_became_active":"2023-09-23T23:25:31.732099+0000","last_became_peered":"2023-09-23T23:25:31.732099+0000","last_unstale":"2023-09-23T23:27:33.050187+0000","last_undegraded":"2023-09-23T23:27:33.050187+0000","last_fullsized":"2023-09-23T23:27:33.050187+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:23.639936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":410,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.067226+0000","last_change":"2023-09-23T23:22:38.438390+0000","last_active":"2023-09-23T23:27:33.067226+0000","last_peered":"2023-09-23T23:27:33.067226+0000","last_clean":"2023-09-23T23:27:33.067226+0000","last_became_active":"2023-09-23T23:22:38.438065+0000","last_became_peered":"2023-09-23T23:22:38.438065+0000","last_unstale":"2023-09-23T23:27:33.067226+0000","last_undegraded":"2023-09-23T23:27:33.067226+0000","last_fullsized":"2023-09-23T23:27:33.067226+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:51.254227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"435'9088","reported_seq":17355,"reported_epoch":435,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:38.059084+0000","last_change":"2023-09-23T23:27:21.744029+0000","last_active":"2023-09-23T23:27:38.059084+0000","last_peered":"2023-09-23T23:27:38.059084+0000","last_clean":"2023-09-23T23:27:19.524907+0000","last_became_active":"2023-09-23T23:27:21.716446+0000","last_became_peered":"2023-09-23T23:27:21.716446+0000","last_unstale":"2023-09-23T23:27:38.059084+0000","last_undegraded":"2023-09-23T23:27:38.059084+0000","last_fullsized":"2023-09-23T23:27:38.059084+0000","mapping_epoch":425,"log_start":"357'3242","ondisk_log_start":"357'3242","created":349,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"367'6627","last_scrub_stamp":"2023-09-23T23:25:24.842424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:24.842424+0000","objects_scrubbed":3200,"log_size":5846,"log_dups_size":0,"ondisk_log_size":5846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:58.481655+0000","scrub_duration":0.68031426699999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47841280,"num_objects":730,"num_object_clones":0,"num_object_copies":2190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":730,"num_objects_unfound":0,"num_objects_dirty":730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2470,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3980,"num_bytes_recovered":260833280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,5,2],"avail_no_missing":["7","2","3","5","6"],"object_location_counts":[{"shards":"2,5,7","objects":730}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":770,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.057993+0000","last_change":"2023-09-23T23:23:45.830384+0000","last_active":"2023-09-23T23:27:33.057993+0000","last_peered":"2023-09-23T23:27:33.057993+0000","last_clean":"2023-09-23T23:27:33.057993+0000","last_became_active":"2023-09-23T23:23:36.025724+0000","last_became_peered":"2023-09-23T23:23:36.025724+0000","last_unstale":"2023-09-23T23:27:33.057993+0000","last_undegraded":"2023-09-23T23:27:33.057993+0000","last_fullsized":"2023-09-23T23:27:33.057993+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:34.345528+0000","scrub_duration":0.001659259,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":402,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.050565+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:27:33.050565+0000","last_peered":"2023-09-23T23:27:33.050565+0000","last_clean":"2023-09-23T23:27:33.050565+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:27:33.050565+0000","last_undegraded":"2023-09-23T23:27:33.050565+0000","last_fullsized":"2023-09-23T23:27:33.050565+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":403,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.062489+0000","last_change":"2023-09-23T23:25:31.731172+0000","last_active":"2023-09-23T23:27:33.062489+0000","last_peered":"2023-09-23T23:27:33.062489+0000","last_clean":"2023-09-23T23:27:33.062489+0000","last_became_active":"2023-09-23T23:25:31.730903+0000","last_became_peered":"2023-09-23T23:25:31.730903+0000","last_unstale":"2023-09-23T23:27:33.062489+0000","last_undegraded":"2023-09-23T23:27:33.062489+0000","last_fullsized":"2023-09-23T23:27:33.062489+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:35.442945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"435'9279","reported_seq":12581,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T23:27:40.853728+0000","last_change":"2023-09-23T23:27:10.906583+0000","last_active":"2023-09-23T23:27:40.853728+0000","last_peered":"2023-09-23T23:27:40.853728+0000","last_clean":"2023-09-23T23:27:40.853728+0000","last_became_active":"2023-09-23T23:27:10.888274+0000","last_became_peered":"2023-09-23T23:27:10.888274+0000","last_unstale":"2023-09-23T23:27:40.853728+0000","last_undegraded":"2023-09-23T23:27:40.853728+0000","last_fullsized":"2023-09-23T23:27:40.853728+0000","mapping_epoch":421,"log_start":"355'800","ondisk_log_start":"355'800","created":349,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":8479,"log_dups_size":0,"ondisk_log_size":8479,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45121536,"num_objects":688,"num_object_clones":0,"num_object_copies":2064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6032,"num_write_kb":215072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1584,"num_bytes_recovered":103809024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":714,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.043638+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:27:33.043638+0000","last_peered":"2023-09-23T23:27:33.043638+0000","last_clean":"2023-09-23T23:27:33.043638+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:27:33.043638+0000","last_undegraded":"2023-09-23T23:27:33.043638+0000","last_fullsized":"2023-09-23T23:27:33.043638+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":261,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061893+0000","last_change":"2023-09-23T23:27:20.560074+0000","last_active":"2023-09-23T23:27:33.061893+0000","last_peered":"2023-09-23T23:27:33.061893+0000","last_clean":"2023-09-23T23:27:33.061893+0000","last_became_active":"2023-09-23T23:27:20.559764+0000","last_became_peered":"2023-09-23T23:27:20.559764+0000","last_unstale":"2023-09-23T23:27:33.061893+0000","last_undegraded":"2023-09-23T23:27:33.061893+0000","last_fullsized":"2023-09-23T23:27:33.061893+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:23.675227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":56,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.062132+0000","last_change":"2023-09-23T23:26:11.516475+0000","last_active":"2023-09-23T23:27:33.062132+0000","last_peered":"2023-09-23T23:27:33.062132+0000","last_clean":"2023-09-23T23:27:33.062132+0000","last_became_active":"2023-09-23T23:26:11.516198+0000","last_became_peered":"2023-09-23T23:26:11.516198+0000","last_unstale":"2023-09-23T23:27:33.062132+0000","last_undegraded":"2023-09-23T23:27:33.062132+0000","last_fullsized":"2023-09-23T23:27:33.062132+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:56.875103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"435'9077","reported_seq":12352,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T23:27:38.764652+0000","last_change":"2023-09-23T23:27:01.819921+0000","last_active":"2023-09-23T23:27:38.764652+0000","last_peered":"2023-09-23T23:27:38.764652+0000","last_clean":"2023-09-23T23:27:38.764652+0000","last_became_active":"2023-09-23T23:27:01.796624+0000","last_became_peered":"2023-09-23T23:27:01.796624+0000","last_unstale":"2023-09-23T23:27:38.764652+0000","last_undegraded":"2023-09-23T23:27:38.764652+0000","last_fullsized":"2023-09-23T23:27:38.764652+0000","mapping_epoch":417,"log_start":"356'2863","ondisk_log_start":"356'2863","created":349,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":6214,"log_dups_size":0,"ondisk_log_size":6214,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47218688,"num_objects":720,"num_object_clones":0,"num_object_copies":2160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5874,"num_write_kb":211040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1570,"num_bytes_recovered":102891520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":590,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061559+0000","last_change":"2023-09-23T23:26:34.548211+0000","last_active":"2023-09-23T23:27:33.061559+0000","last_peered":"2023-09-23T23:27:33.061559+0000","last_clean":"2023-09-23T23:27:33.061559+0000","last_became_active":"2023-09-23T23:26:34.547815+0000","last_became_peered":"2023-09-23T23:26:34.547815+0000","last_unstale":"2023-09-23T23:27:33.061559+0000","last_undegraded":"2023-09-23T23:27:33.061559+0000","last_fullsized":"2023-09-23T23:27:33.061559+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:37.675417+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":56,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061460+0000","last_change":"2023-09-23T23:26:11.512815+0000","last_active":"2023-09-23T23:27:33.061460+0000","last_peered":"2023-09-23T23:27:33.061460+0000","last_clean":"2023-09-23T23:27:33.061460+0000","last_became_active":"2023-09-23T23:26:11.512466+0000","last_became_peered":"2023-09-23T23:26:11.512466+0000","last_unstale":"2023-09-23T23:27:33.061460+0000","last_undegraded":"2023-09-23T23:27:33.061460+0000","last_fullsized":"2023-09-23T23:27:33.061460+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:10.324158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":433,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.062834+0000","last_change":"2023-09-23T23:27:20.558731+0000","last_active":"2023-09-23T23:27:33.062834+0000","last_peered":"2023-09-23T23:27:33.062834+0000","last_clean":"2023-09-23T23:27:33.062834+0000","last_became_active":"2023-09-23T23:27:20.558394+0000","last_became_peered":"2023-09-23T23:27:20.558394+0000","last_unstale":"2023-09-23T23:27:33.062834+0000","last_undegraded":"2023-09-23T23:27:33.062834+0000","last_fullsized":"2023-09-23T23:27:33.062834+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:25:00.408551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"435'8985","reported_seq":9167,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T23:27:39.592594+0000","last_change":"2023-09-23T23:25:20.332216+0000","last_active":"2023-09-23T23:27:39.592594+0000","last_peered":"2023-09-23T23:27:39.592594+0000","last_clean":"2023-09-23T23:27:39.592594+0000","last_became_active":"2023-09-23T23:25:18.988253+0000","last_became_peered":"2023-09-23T23:25:18.988253+0000","last_unstale":"2023-09-23T23:27:39.592594+0000","last_undegraded":"2023-09-23T23:27:39.592594+0000","last_fullsized":"2023-09-23T23:27:39.592594+0000","mapping_epoch":349,"log_start":"356'2842","ondisk_log_start":"356'2842","created":349,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"366'6383","last_scrub_stamp":"2023-09-23T23:25:20.331998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:20.331998+0000","objects_scrubbed":3194,"log_size":6143,"log_dups_size":0,"ondisk_log_size":6143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:37.230596+0000","scrub_duration":0.84482357799999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47906816,"num_objects":731,"num_object_clones":0,"num_object_copies":2193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2613,"num_write_kb":4800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":755,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.044285+0000","last_change":"2023-09-23T23:25:34.500038+0000","last_active":"2023-09-23T23:27:33.044285+0000","last_peered":"2023-09-23T23:27:33.044285+0000","last_clean":"2023-09-23T23:27:33.044285+0000","last_became_active":"2023-09-23T23:25:34.499705+0000","last_became_peered":"2023-09-23T23:25:34.499705+0000","last_unstale":"2023-09-23T23:27:33.044285+0000","last_undegraded":"2023-09-23T23:27:33.044285+0000","last_fullsized":"2023-09-23T23:27:33.044285+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:43.096696+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":72,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.049389+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:27:33.049389+0000","last_peered":"2023-09-23T23:27:33.049389+0000","last_clean":"2023-09-23T23:27:33.049389+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:27:33.049389+0000","last_undegraded":"2023-09-23T23:27:33.049389+0000","last_fullsized":"2023-09-23T23:27:33.049389+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":356,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.062606+0000","last_change":"2023-09-23T23:22:39.489325+0000","last_active":"2023-09-23T23:27:33.062606+0000","last_peered":"2023-09-23T23:27:33.062606+0000","last_clean":"2023-09-23T23:27:33.062606+0000","last_became_active":"2023-09-23T23:22:39.489048+0000","last_became_peered":"2023-09-23T23:22:39.489048+0000","last_unstale":"2023-09-23T23:27:33.062606+0000","last_undegraded":"2023-09-23T23:27:33.062606+0000","last_fullsized":"2023-09-23T23:27:33.062606+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:29.984948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"435'11973","reported_seq":12122,"reported_epoch":435,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:40.867008+0000","last_change":"2023-09-23T23:26:11.572743+0000","last_active":"2023-09-23T23:27:40.867008+0000","last_peered":"2023-09-23T23:27:40.867008+0000","last_clean":"2023-09-23T23:25:30.684619+0000","last_became_active":"2023-09-23T23:25:32.841731+0000","last_became_peered":"2023-09-23T23:25:32.841731+0000","last_unstale":"2023-09-23T23:27:40.867008+0000","last_undegraded":"2023-09-23T23:27:40.867008+0000","last_fullsized":"2023-09-23T23:27:40.867008+0000","mapping_epoch":372,"log_start":"361'5258","ondisk_log_start":"361'5258","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":6715,"log_dups_size":0,"ondisk_log_size":6715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:12.410433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91095064,"num_objects":1391,"num_object_clones":0,"num_object_copies":4173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1383,"num_objects_unfound":0,"num_objects_dirty":1391,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11973,"num_write_kb":427585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1179648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,6],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":1391}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":733,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.049732+0000","last_change":"2023-09-23T23:26:51.561734+0000","last_active":"2023-09-23T23:27:33.049732+0000","last_peered":"2023-09-23T23:27:33.049732+0000","last_clean":"2023-09-23T23:27:33.049732+0000","last_became_active":"2023-09-23T23:26:51.561394+0000","last_became_peered":"2023-09-23T23:26:51.561394+0000","last_unstale":"2023-09-23T23:27:33.049732+0000","last_undegraded":"2023-09-23T23:27:33.049732+0000","last_fullsized":"2023-09-23T23:27:33.049732+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.063071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.063071+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":136,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.063906+0000","last_change":"2023-09-23T23:23:36.048889+0000","last_active":"2023-09-23T23:27:33.063906+0000","last_peered":"2023-09-23T23:27:33.063906+0000","last_clean":"2023-09-23T23:27:33.063906+0000","last_became_active":"2023-09-23T23:23:36.048572+0000","last_became_peered":"2023-09-23T23:23:36.048572+0000","last_unstale":"2023-09-23T23:27:33.063906+0000","last_undegraded":"2023-09-23T23:27:33.063906+0000","last_fullsized":"2023-09-23T23:27:33.063906+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:55.581146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":450,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.056195+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:27:33.056195+0000","last_peered":"2023-09-23T23:27:33.056195+0000","last_clean":"2023-09-23T23:27:33.056195+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:27:33.056195+0000","last_undegraded":"2023-09-23T23:27:33.056195+0000","last_fullsized":"2023-09-23T23:27:33.056195+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":21,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061172+0000","last_change":"2023-09-23T23:27:20.549904+0000","last_active":"2023-09-23T23:27:33.061172+0000","last_peered":"2023-09-23T23:27:33.061172+0000","last_clean":"2023-09-23T23:27:33.061172+0000","last_became_active":"2023-09-23T23:27:20.549645+0000","last_became_peered":"2023-09-23T23:27:20.549645+0000","last_unstale":"2023-09-23T23:27:33.061172+0000","last_undegraded":"2023-09-23T23:27:33.061172+0000","last_fullsized":"2023-09-23T23:27:33.061172+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:09.850640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"435'11916","reported_seq":26432,"reported_epoch":435,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:38.786135+0000","last_change":"2023-09-23T23:27:21.765359+0000","last_active":"2023-09-23T23:27:38.786135+0000","last_peered":"2023-09-23T23:27:38.786135+0000","last_clean":"2023-09-23T23:27:19.522554+0000","last_became_active":"2023-09-23T23:27:21.717182+0000","last_became_peered":"2023-09-23T23:27:21.717182+0000","last_unstale":"2023-09-23T23:27:38.786135+0000","last_undegraded":"2023-09-23T23:27:38.786135+0000","last_fullsized":"2023-09-23T23:27:38.786135+0000","mapping_epoch":425,"log_start":"375'7436","ondisk_log_start":"375'7436","created":349,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4480,"log_dups_size":0,"ondisk_log_size":4480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:29.188830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90046464,"num_objects":1374,"num_object_clones":0,"num_object_copies":4122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1374,"num_objects_unfound":0,"num_objects_dirty":1374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11916,"num_write_kb":425280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7186,"num_bytes_recovered":470941696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,0,4],"avail_no_missing":["3","0","4","6","7"],"object_location_counts":[{"shards":"0,3,4","objects":1374}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":581,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T23:27:34.057258+0000","last_change":"2023-09-23T23:23:49.320969+0000","last_active":"2023-09-23T23:27:34.057258+0000","last_peered":"2023-09-23T23:27:34.057258+0000","last_clean":"2023-09-23T23:27:34.057258+0000","last_became_active":"2023-09-23T23:22:17.001911+0000","last_became_peered":"2023-09-23T23:22:17.001911+0000","last_unstale":"2023-09-23T23:27:34.057258+0000","last_undegraded":"2023-09-23T23:27:34.057258+0000","last_fullsized":"2023-09-23T23:27:34.057258+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.320823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.320823+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:07.000984+0000","scrub_duration":0.0018140630000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":136,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061256+0000","last_change":"2023-09-23T23:25:31.731396+0000","last_active":"2023-09-23T23:27:33.061256+0000","last_peered":"2023-09-23T23:27:33.061256+0000","last_clean":"2023-09-23T23:27:33.061256+0000","last_became_active":"2023-09-23T23:25:31.730931+0000","last_became_peered":"2023-09-23T23:25:31.730931+0000","last_unstale":"2023-09-23T23:27:33.061256+0000","last_undegraded":"2023-09-23T23:27:33.061256+0000","last_fullsized":"2023-09-23T23:27:33.061256+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:48.002628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":414,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061960+0000","last_change":"2023-09-23T23:27:20.556916+0000","last_active":"2023-09-23T23:27:33.061960+0000","last_peered":"2023-09-23T23:27:33.061960+0000","last_clean":"2023-09-23T23:27:33.061960+0000","last_became_active":"2023-09-23T23:27:20.556651+0000","last_became_peered":"2023-09-23T23:27:20.556651+0000","last_unstale":"2023-09-23T23:27:33.061960+0000","last_undegraded":"2023-09-23T23:27:33.061960+0000","last_fullsized":"2023-09-23T23:27:33.061960+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:06.012543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":21,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.063444+0000","last_change":"2023-09-23T23:27:20.551221+0000","last_active":"2023-09-23T23:27:33.063444+0000","last_peered":"2023-09-23T23:27:33.063444+0000","last_clean":"2023-09-23T23:27:33.063444+0000","last_became_active":"2023-09-23T23:27:20.550900+0000","last_became_peered":"2023-09-23T23:27:20.550900+0000","last_unstale":"2023-09-23T23:27:33.063444+0000","last_undegraded":"2023-09-23T23:27:33.063444+0000","last_fullsized":"2023-09-23T23:27:33.063444+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:23.012149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"435'8971","reported_seq":16359,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T23:27:38.698045+0000","last_change":"2023-09-23T23:27:25.617962+0000","last_active":"2023-09-23T23:27:38.698045+0000","last_peered":"2023-09-23T23:27:38.698045+0000","last_clean":"2023-09-23T23:27:38.698045+0000","last_became_active":"2023-09-23T23:27:25.598086+0000","last_became_peered":"2023-09-23T23:27:25.598086+0000","last_unstale":"2023-09-23T23:27:38.698045+0000","last_undegraded":"2023-09-23T23:27:38.698045+0000","last_fullsized":"2023-09-23T23:27:38.698045+0000","mapping_epoch":429,"log_start":"356'2604","ondisk_log_start":"356'2604","created":349,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":0,"last_scrub":"366'6384","last_scrub_stamp":"2023-09-23T23:25:20.132415+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:20.132415+0000","objects_scrubbed":3102,"log_size":6367,"log_dups_size":0,"ondisk_log_size":6367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:22.673284+0000","scrub_duration":0.99622401299999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44695552,"num_objects":682,"num_object_clones":0,"num_object_copies":2046,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":682,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2604,"num_write_kb":5888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3543,"num_bytes_recovered":232194048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":797,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.057296+0000","last_change":"2023-09-23T23:26:51.562598+0000","last_active":"2023-09-23T23:27:33.057296+0000","last_peered":"2023-09-23T23:27:33.057296+0000","last_clean":"2023-09-23T23:27:33.057296+0000","last_became_active":"2023-09-23T23:26:51.562167+0000","last_became_peered":"2023-09-23T23:26:51.562167+0000","last_unstale":"2023-09-23T23:27:33.057296+0000","last_undegraded":"2023-09-23T23:27:33.057296+0000","last_fullsized":"2023-09-23T23:27:33.057296+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:17.310285+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":169,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.045080+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:27:33.045080+0000","last_peered":"2023-09-23T23:27:33.045080+0000","last_clean":"2023-09-23T23:27:33.045080+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:27:33.045080+0000","last_undegraded":"2023-09-23T23:27:33.045080+0000","last_fullsized":"2023-09-23T23:27:33.045080+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":136,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.060987+0000","last_change":"2023-09-23T23:25:31.828908+0000","last_active":"2023-09-23T23:27:33.060987+0000","last_peered":"2023-09-23T23:27:33.060987+0000","last_clean":"2023-09-23T23:27:33.060987+0000","last_became_active":"2023-09-23T23:25:31.828582+0000","last_became_peered":"2023-09-23T23:25:31.828582+0000","last_unstale":"2023-09-23T23:27:33.060987+0000","last_undegraded":"2023-09-23T23:27:33.060987+0000","last_fullsized":"2023-09-23T23:27:33.060987+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:01.466593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":21,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.062909+0000","last_change":"2023-09-23T23:27:20.551574+0000","last_active":"2023-09-23T23:27:33.062909+0000","last_peered":"2023-09-23T23:27:33.062909+0000","last_clean":"2023-09-23T23:27:33.062909+0000","last_became_active":"2023-09-23T23:27:20.550820+0000","last_became_peered":"2023-09-23T23:27:20.550820+0000","last_unstale":"2023-09-23T23:27:33.062909+0000","last_undegraded":"2023-09-23T23:27:33.062909+0000","last_fullsized":"2023-09-23T23:27:33.062909+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:09.274552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"435'11664","reported_seq":11779,"reported_epoch":435,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:38.753210+0000","last_change":"2023-09-23T23:27:20.756116+0000","last_active":"2023-09-23T23:27:38.753210+0000","last_peered":"2023-09-23T23:27:38.753210+0000","last_clean":"2023-09-23T23:25:30.690916+0000","last_became_active":"2023-09-23T23:27:20.617962+0000","last_became_peered":"2023-09-23T23:27:20.617962+0000","last_unstale":"2023-09-23T23:27:38.753210+0000","last_undegraded":"2023-09-23T23:27:38.753210+0000","last_fullsized":"2023-09-23T23:27:38.753210+0000","mapping_epoch":424,"log_start":"375'7241","ondisk_log_start":"375'7241","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4423,"log_dups_size":0,"ondisk_log_size":4423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:02:52.104913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95289344,"num_objects":1454,"num_object_clones":0,"num_object_copies":4362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1454,"num_objects_unfound":0,"num_objects_dirty":1454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11664,"num_write_kb":419776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":1454}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":446,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T23:27:34.055160+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:27:34.055160+0000","last_peered":"2023-09-23T23:27:34.055160+0000","last_clean":"2023-09-23T23:27:34.055160+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:27:34.055160+0000","last_undegraded":"2023-09-23T23:27:34.055160+0000","last_fullsized":"2023-09-23T23:27:34.055160+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":755,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.057357+0000","last_change":"2023-09-23T23:26:51.564817+0000","last_active":"2023-09-23T23:27:33.057357+0000","last_peered":"2023-09-23T23:27:33.057357+0000","last_clean":"2023-09-23T23:27:33.057357+0000","last_became_active":"2023-09-23T23:26:51.564511+0000","last_became_peered":"2023-09-23T23:26:51.564511+0000","last_unstale":"2023-09-23T23:27:33.057357+0000","last_undegraded":"2023-09-23T23:27:33.057357+0000","last_fullsized":"2023-09-23T23:27:33.057357+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:02.415212+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":384,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T23:27:34.077016+0000","last_change":"2023-09-23T23:26:32.037619+0000","last_active":"2023-09-23T23:27:34.077016+0000","last_peered":"2023-09-23T23:27:34.077016+0000","last_clean":"2023-09-23T23:27:34.077016+0000","last_became_active":"2023-09-23T23:26:32.037045+0000","last_became_peered":"2023-09-23T23:26:32.037045+0000","last_unstale":"2023-09-23T23:27:34.077016+0000","last_undegraded":"2023-09-23T23:27:34.077016+0000","last_fullsized":"2023-09-23T23:27:34.077016+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:47.657527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":56,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.062179+0000","last_change":"2023-09-23T23:26:11.525845+0000","last_active":"2023-09-23T23:27:33.062179+0000","last_peered":"2023-09-23T23:27:33.062179+0000","last_clean":"2023-09-23T23:27:33.062179+0000","last_became_active":"2023-09-23T23:26:11.525507+0000","last_became_peered":"2023-09-23T23:26:11.525507+0000","last_unstale":"2023-09-23T23:27:33.062179+0000","last_undegraded":"2023-09-23T23:27:33.062179+0000","last_fullsized":"2023-09-23T23:27:33.062179+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:35.462874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"435'9134","reported_seq":14614,"reported_epoch":435,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:38.715701+0000","last_change":"2023-09-23T23:27:21.714698+0000","last_active":"2023-09-23T23:27:38.715701+0000","last_peered":"2023-09-23T23:27:38.715701+0000","last_clean":"2023-09-23T23:27:19.438581+0000","last_became_active":"2023-09-23T23:27:21.688910+0000","last_became_peered":"2023-09-23T23:27:21.688910+0000","last_unstale":"2023-09-23T23:27:38.715701+0000","last_undegraded":"2023-09-23T23:27:38.715701+0000","last_fullsized":"2023-09-23T23:27:38.715701+0000","mapping_epoch":425,"log_start":"356'2933","ondisk_log_start":"356'2933","created":349,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":6201,"log_dups_size":0,"ondisk_log_size":6201,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49446912,"num_objects":754,"num_object_clones":0,"num_object_copies":2262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":754,"num_objects_unfound":0,"num_objects_dirty":754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5898,"num_write_kb":212896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2670,"num_bytes_recovered":174981120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,7],"avail_no_missing":["3","0","6","7"],"object_location_counts":[{"shards":"0,3,7","objects":754}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":598,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061640+0000","last_change":"2023-09-23T23:26:34.550276+0000","last_active":"2023-09-23T23:27:33.061640+0000","last_peered":"2023-09-23T23:27:33.061640+0000","last_clean":"2023-09-23T23:27:33.061640+0000","last_became_active":"2023-09-23T23:26:34.549923+0000","last_became_peered":"2023-09-23T23:26:34.549923+0000","last_unstale":"2023-09-23T23:27:33.061640+0000","last_undegraded":"2023-09-23T23:27:33.061640+0000","last_fullsized":"2023-09-23T23:27:33.061640+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:22.636211+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":362,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.057121+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:27:33.057121+0000","last_peered":"2023-09-23T23:27:33.057121+0000","last_clean":"2023-09-23T23:27:33.057121+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:27:33.057121+0000","last_undegraded":"2023-09-23T23:27:33.057121+0000","last_fullsized":"2023-09-23T23:27:33.057121+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":437,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.070236+0000","last_change":"2023-09-23T23:26:11.528036+0000","last_active":"2023-09-23T23:27:33.070236+0000","last_peered":"2023-09-23T23:27:33.070236+0000","last_clean":"2023-09-23T23:27:33.070236+0000","last_became_active":"2023-09-23T23:26:11.527688+0000","last_became_peered":"2023-09-23T23:26:11.527688+0000","last_unstale":"2023-09-23T23:27:33.070236+0000","last_undegraded":"2023-09-23T23:27:33.070236+0000","last_fullsized":"2023-09-23T23:27:33.070236+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:49.411897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":464,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.068312+0000","last_change":"2023-09-23T23:22:38.438448+0000","last_active":"2023-09-23T23:27:33.068312+0000","last_peered":"2023-09-23T23:27:33.068312+0000","last_clean":"2023-09-23T23:27:33.068312+0000","last_became_active":"2023-09-23T23:22:38.437495+0000","last_became_peered":"2023-09-23T23:22:38.437495+0000","last_unstale":"2023-09-23T23:27:33.068312+0000","last_undegraded":"2023-09-23T23:27:33.068312+0000","last_fullsized":"2023-09-23T23:27:33.068312+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:49.587056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"435'11520","reported_seq":16183,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T23:27:39.605297+0000","last_change":"2023-09-23T23:27:29.668660+0000","last_active":"2023-09-23T23:27:39.605297+0000","last_peered":"2023-09-23T23:27:39.605297+0000","last_clean":"2023-09-23T23:27:39.605297+0000","last_became_active":"2023-09-23T23:27:29.641291+0000","last_became_peered":"2023-09-23T23:27:29.641291+0000","last_unstale":"2023-09-23T23:27:39.605297+0000","last_undegraded":"2023-09-23T23:27:39.605297+0000","last_fullsized":"2023-09-23T23:27:39.605297+0000","mapping_epoch":431,"log_start":"375'7083","ondisk_log_start":"375'7083","created":349,"last_epoch_clean":432,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4437,"log_dups_size":0,"ondisk_log_size":4437,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:47.739467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92143616,"num_objects":1406,"num_object_clones":0,"num_object_copies":4218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11520,"num_write_kb":413632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2267,"num_bytes_recovered":148570112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":463,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.052789+0000","last_change":"2023-09-23T23:26:04.560829+0000","last_active":"2023-09-23T23:27:33.052789+0000","last_peered":"2023-09-23T23:27:33.052789+0000","last_clean":"2023-09-23T23:27:33.052789+0000","last_became_active":"2023-09-23T23:26:04.560484+0000","last_became_peered":"2023-09-23T23:26:04.560484+0000","last_unstale":"2023-09-23T23:27:33.052789+0000","last_undegraded":"2023-09-23T23:27:33.052789+0000","last_fullsized":"2023-09-23T23:27:33.052789+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.048718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.048718+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":286,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.058146+0000","last_change":"2023-09-23T23:23:35.037351+0000","last_active":"2023-09-23T23:27:33.058146+0000","last_peered":"2023-09-23T23:27:33.058146+0000","last_clean":"2023-09-23T23:27:33.058146+0000","last_became_active":"2023-09-23T23:23:35.036841+0000","last_became_peered":"2023-09-23T23:23:35.036841+0000","last_unstale":"2023-09-23T23:27:33.058146+0000","last_undegraded":"2023-09-23T23:27:33.058146+0000","last_fullsized":"2023-09-23T23:27:33.058146+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:20.618292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":136,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.064122+0000","last_change":"2023-09-23T23:26:32.037099+0000","last_active":"2023-09-23T23:27:33.064122+0000","last_peered":"2023-09-23T23:27:33.064122+0000","last_clean":"2023-09-23T23:27:33.064122+0000","last_became_active":"2023-09-23T23:26:32.036681+0000","last_became_peered":"2023-09-23T23:26:32.036681+0000","last_unstale":"2023-09-23T23:27:33.064122+0000","last_undegraded":"2023-09-23T23:27:33.064122+0000","last_fullsized":"2023-09-23T23:27:33.064122+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:48.926576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":168,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.055800+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:27:33.055800+0000","last_peered":"2023-09-23T23:27:33.055800+0000","last_clean":"2023-09-23T23:27:33.055800+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:27:33.055800+0000","last_undegraded":"2023-09-23T23:27:33.055800+0000","last_fullsized":"2023-09-23T23:27:33.055800+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"435'11906","reported_seq":12000,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T23:27:38.118974+0000","last_change":"2023-09-23T23:23:43.505852+0000","last_active":"2023-09-23T23:27:38.118974+0000","last_peered":"2023-09-23T23:27:38.118974+0000","last_clean":"2023-09-23T23:27:38.118974+0000","last_became_active":"2023-09-23T23:23:43.505473+0000","last_became_peered":"2023-09-23T23:23:43.505473+0000","last_unstale":"2023-09-23T23:27:38.118974+0000","last_undegraded":"2023-09-23T23:27:38.118974+0000","last_fullsized":"2023-09-23T23:27:38.118974+0000","mapping_epoch":349,"log_start":"377'7666","ondisk_log_start":"377'7666","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4240,"log_dups_size":0,"ondisk_log_size":4240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:31.385610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95420416,"num_objects":1456,"num_object_clones":0,"num_object_copies":4368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11906,"num_write_kb":427584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":746,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.062225+0000","last_change":"2023-09-23T23:26:14.518921+0000","last_active":"2023-09-23T23:27:33.062225+0000","last_peered":"2023-09-23T23:27:33.062225+0000","last_clean":"2023-09-23T23:27:33.062225+0000","last_became_active":"2023-09-23T23:26:14.518558+0000","last_became_peered":"2023-09-23T23:26:14.518558+0000","last_unstale":"2023-09-23T23:27:33.062225+0000","last_undegraded":"2023-09-23T23:27:33.062225+0000","last_fullsized":"2023-09-23T23:27:33.062225+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:14.360597+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":428,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.050027+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:27:33.050027+0000","last_peered":"2023-09-23T23:27:33.050027+0000","last_clean":"2023-09-23T23:27:33.050027+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:27:33.050027+0000","last_undegraded":"2023-09-23T23:27:33.050027+0000","last_fullsized":"2023-09-23T23:27:33.050027+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":21,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.062204+0000","last_change":"2023-09-23T23:27:20.562744+0000","last_active":"2023-09-23T23:27:33.062204+0000","last_peered":"2023-09-23T23:27:33.062204+0000","last_clean":"2023-09-23T23:27:33.062204+0000","last_became_active":"2023-09-23T23:27:20.562383+0000","last_became_peered":"2023-09-23T23:27:20.562383+0000","last_unstale":"2023-09-23T23:27:33.062204+0000","last_undegraded":"2023-09-23T23:27:33.062204+0000","last_fullsized":"2023-09-23T23:27:33.062204+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:01.085133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":72,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.049310+0000","last_change":"2023-09-23T23:26:48.562966+0000","last_active":"2023-09-23T23:27:33.049310+0000","last_peered":"2023-09-23T23:27:33.049310+0000","last_clean":"2023-09-23T23:27:33.049310+0000","last_became_active":"2023-09-23T23:26:48.562630+0000","last_became_peered":"2023-09-23T23:26:48.562630+0000","last_unstale":"2023-09-23T23:27:33.049310+0000","last_undegraded":"2023-09-23T23:27:33.049310+0000","last_fullsized":"2023-09-23T23:27:33.049310+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:40.163647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"435'11850","reported_seq":20408,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T23:27:39.584930+0000","last_change":"2023-09-23T23:26:33.547524+0000","last_active":"2023-09-23T23:27:39.584930+0000","last_peered":"2023-09-23T23:27:39.584930+0000","last_clean":"2023-09-23T23:27:39.584930+0000","last_became_active":"2023-09-23T23:26:33.521810+0000","last_became_peered":"2023-09-23T23:26:33.521810+0000","last_unstale":"2023-09-23T23:27:39.584930+0000","last_undegraded":"2023-09-23T23:27:39.584930+0000","last_fullsized":"2023-09-23T23:27:39.584930+0000","mapping_epoch":400,"log_start":"375'7005","ondisk_log_start":"375'7005","created":349,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4845,"log_dups_size":0,"ondisk_log_size":4845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:49.164061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88080384,"num_objects":1344,"num_object_clones":0,"num_object_copies":4032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11850,"num_write_kb":422208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4222,"num_bytes_recovered":276692992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":818,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.050810+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:27:33.050810+0000","last_peered":"2023-09-23T23:27:33.050810+0000","last_clean":"2023-09-23T23:27:33.050810+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:27:33.050810+0000","last_undegraded":"2023-09-23T23:27:33.050810+0000","last_fullsized":"2023-09-23T23:27:33.050810+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":34,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.059405+0000","last_change":"2023-09-23T23:26:48.544465+0000","last_active":"2023-09-23T23:27:33.059405+0000","last_peered":"2023-09-23T23:27:33.059405+0000","last_clean":"2023-09-23T23:27:33.059405+0000","last_became_active":"2023-09-23T23:26:48.544166+0000","last_became_peered":"2023-09-23T23:26:48.544166+0000","last_unstale":"2023-09-23T23:27:33.059405+0000","last_undegraded":"2023-09-23T23:27:33.059405+0000","last_fullsized":"2023-09-23T23:27:33.059405+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:38.844346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":419,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.068172+0000","last_change":"2023-09-23T23:22:38.482705+0000","last_active":"2023-09-23T23:27:33.068172+0000","last_peered":"2023-09-23T23:27:33.068172+0000","last_clean":"2023-09-23T23:27:33.068172+0000","last_became_active":"2023-09-23T23:22:38.438239+0000","last_became_peered":"2023-09-23T23:22:38.438239+0000","last_unstale":"2023-09-23T23:27:33.068172+0000","last_undegraded":"2023-09-23T23:27:33.068172+0000","last_fullsized":"2023-09-23T23:27:33.068172+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:52.006349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":56,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061481+0000","last_change":"2023-09-23T23:26:11.522604+0000","last_active":"2023-09-23T23:27:33.061481+0000","last_peered":"2023-09-23T23:27:33.061481+0000","last_clean":"2023-09-23T23:27:33.061481+0000","last_became_active":"2023-09-23T23:26:11.522350+0000","last_became_peered":"2023-09-23T23:26:11.522350+0000","last_unstale":"2023-09-23T23:27:33.061481+0000","last_undegraded":"2023-09-23T23:27:33.061481+0000","last_fullsized":"2023-09-23T23:27:33.061481+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:20.164202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"435'9159","reported_seq":18414,"reported_epoch":435,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:38.107010+0000","last_change":"2023-09-23T23:27:21.744033+0000","last_active":"2023-09-23T23:27:38.107010+0000","last_peered":"2023-09-23T23:27:38.107010+0000","last_clean":"2023-09-23T23:27:19.485531+0000","last_became_active":"2023-09-23T23:27:21.716360+0000","last_became_peered":"2023-09-23T23:27:21.716360+0000","last_unstale":"2023-09-23T23:27:38.107010+0000","last_undegraded":"2023-09-23T23:27:38.107010+0000","last_fullsized":"2023-09-23T23:27:38.107010+0000","mapping_epoch":425,"log_start":"357'3474","ondisk_log_start":"357'3474","created":365,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5685,"log_dups_size":0,"ondisk_log_size":5685,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45154304,"num_objects":689,"num_object_clones":0,"num_object_copies":2067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":689,"num_objects_unfound":0,"num_objects_dirty":689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5947,"num_write_kb":212352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4546,"num_bytes_recovered":297926656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,5,2],"avail_no_missing":["7","2","3","5","6"],"object_location_counts":[{"shards":"2,5,7","objects":689}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":816,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.054134+0000","last_change":"2023-09-23T23:25:34.505042+0000","last_active":"2023-09-23T23:27:33.054134+0000","last_peered":"2023-09-23T23:27:33.054134+0000","last_clean":"2023-09-23T23:27:33.054134+0000","last_became_active":"2023-09-23T23:25:34.504687+0000","last_became_peered":"2023-09-23T23:25:34.504687+0000","last_unstale":"2023-09-23T23:27:33.054134+0000","last_undegraded":"2023-09-23T23:27:33.054134+0000","last_fullsized":"2023-09-23T23:27:33.054134+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:06.369262+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":69,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.067381+0000","last_change":"2023-09-23T23:25:48.082121+0000","last_active":"2023-09-23T23:27:33.067381+0000","last_peered":"2023-09-23T23:27:33.067381+0000","last_clean":"2023-09-23T23:27:33.067381+0000","last_became_active":"2023-09-23T23:25:48.081746+0000","last_became_peered":"2023-09-23T23:25:48.081746+0000","last_unstale":"2023-09-23T23:27:33.067381+0000","last_undegraded":"2023-09-23T23:27:33.067381+0000","last_fullsized":"2023-09-23T23:27:33.067381+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:00.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":177,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.049028+0000","last_change":"2023-09-23T23:22:38.439602+0000","last_active":"2023-09-23T23:27:33.049028+0000","last_peered":"2023-09-23T23:27:33.049028+0000","last_clean":"2023-09-23T23:27:33.049028+0000","last_became_active":"2023-09-23T23:22:38.439261+0000","last_became_peered":"2023-09-23T23:22:38.439261+0000","last_unstale":"2023-09-23T23:27:33.049028+0000","last_undegraded":"2023-09-23T23:27:33.049028+0000","last_fullsized":"2023-09-23T23:27:33.049028+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:03.293816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"435'9185","reported_seq":16153,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T23:27:38.094896+0000","last_change":"2023-09-23T23:26:52.609382+0000","last_active":"2023-09-23T23:27:38.094896+0000","last_peered":"2023-09-23T23:27:38.094896+0000","last_clean":"2023-09-23T23:27:38.094896+0000","last_became_active":"2023-09-23T23:26:52.578253+0000","last_became_peered":"2023-09-23T23:26:52.578253+0000","last_unstale":"2023-09-23T23:27:38.094896+0000","last_undegraded":"2023-09-23T23:27:38.094896+0000","last_fullsized":"2023-09-23T23:27:38.094896+0000","mapping_epoch":414,"log_start":"357'3448","ondisk_log_start":"357'3448","created":365,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":5,"last_scrub":"366'6668","last_scrub_stamp":"2023-09-23T23:25:23.925139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:23.925139+0000","objects_scrubbed":3290,"log_size":5737,"log_dups_size":0,"ondisk_log_size":5737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:07.350349+0000","scrub_duration":0.63215223600000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50069504,"num_objects":764,"num_object_clones":0,"num_object_copies":2292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2526,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3348,"num_bytes_recovered":209125376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":561,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.053158+0000","last_change":"2023-09-23T23:26:14.522158+0000","last_active":"2023-09-23T23:27:33.053158+0000","last_peered":"2023-09-23T23:27:33.053158+0000","last_clean":"2023-09-23T23:27:33.053158+0000","last_became_active":"2023-09-23T23:26:14.521785+0000","last_became_peered":"2023-09-23T23:26:14.521785+0000","last_unstale":"2023-09-23T23:27:33.053158+0000","last_undegraded":"2023-09-23T23:27:33.053158+0000","last_fullsized":"2023-09-23T23:27:33.053158+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:39.445601+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":176,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.062572+0000","last_change":"2023-09-23T23:22:38.452968+0000","last_active":"2023-09-23T23:27:33.062572+0000","last_peered":"2023-09-23T23:27:33.062572+0000","last_clean":"2023-09-23T23:27:33.062572+0000","last_became_active":"2023-09-23T23:22:38.437986+0000","last_became_peered":"2023-09-23T23:22:38.437986+0000","last_unstale":"2023-09-23T23:27:33.062572+0000","last_undegraded":"2023-09-23T23:27:33.062572+0000","last_fullsized":"2023-09-23T23:27:33.062572+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:10.333271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":66,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061750+0000","last_change":"2023-09-23T23:27:20.558454+0000","last_active":"2023-09-23T23:27:33.061750+0000","last_peered":"2023-09-23T23:27:33.061750+0000","last_clean":"2023-09-23T23:27:33.061750+0000","last_became_active":"2023-09-23T23:27:20.558204+0000","last_became_peered":"2023-09-23T23:27:20.558204+0000","last_unstale":"2023-09-23T23:27:33.061750+0000","last_undegraded":"2023-09-23T23:27:33.061750+0000","last_fullsized":"2023-09-23T23:27:33.061750+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:15.079990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"435'9195","reported_seq":15518,"reported_epoch":435,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T23:27:40.683856+0000","last_change":"2023-09-23T23:27:40.549301+0000","last_active":"2023-09-23T23:27:40.683856+0000","last_peered":"2023-09-23T23:27:40.683856+0000","last_clean":"2023-09-23T23:26:00.820858+0000","last_became_active":"2023-09-23T23:26:48.622740+0000","last_became_peered":"2023-09-23T23:26:48.622740+0000","last_unstale":"2023-09-23T23:27:40.683856+0000","last_undegraded":"2023-09-23T23:27:40.683856+0000","last_fullsized":"2023-09-23T23:27:40.683856+0000","mapping_epoch":410,"log_start":"355'1966","ondisk_log_start":"355'1966","created":365,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":7229,"log_dups_size":0,"ondisk_log_size":7229,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40861696,"num_objects":624,"num_object_clones":0,"num_object_copies":1872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1773,"num_objects_unfound":0,"num_objects_dirty":624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6002,"num_write_kb":212000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3055,"num_bytes_recovered":200212480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[0,2,4],"avail_no_missing":["0","2","4","6"],"object_location_counts":[{"shards":"0,2,4","objects":623},{"shards":"5","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":176,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.062514+0000","last_change":"2023-09-23T23:25:31.740467+0000","last_active":"2023-09-23T23:27:33.062514+0000","last_peered":"2023-09-23T23:27:33.062514+0000","last_clean":"2023-09-23T23:27:33.062514+0000","last_became_active":"2023-09-23T23:25:31.729046+0000","last_became_peered":"2023-09-23T23:25:31.729046+0000","last_unstale":"2023-09-23T23:27:33.062514+0000","last_undegraded":"2023-09-23T23:27:33.062514+0000","last_fullsized":"2023-09-23T23:27:33.062514+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:31.222969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":56,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061644+0000","last_change":"2023-09-23T23:26:11.527453+0000","last_active":"2023-09-23T23:27:33.061644+0000","last_peered":"2023-09-23T23:27:33.061644+0000","last_clean":"2023-09-23T23:27:33.061644+0000","last_became_active":"2023-09-23T23:26:11.527208+0000","last_became_peered":"2023-09-23T23:26:11.527208+0000","last_unstale":"2023-09-23T23:27:33.061644+0000","last_undegraded":"2023-09-23T23:27:33.061644+0000","last_fullsized":"2023-09-23T23:27:33.061644+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:15.083611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"435'9147","reported_seq":9268,"reported_epoch":435,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:40.630785+0000","last_change":"2023-09-23T23:27:21.759891+0000","last_active":"2023-09-23T23:27:40.630785+0000","last_peered":"2023-09-23T23:27:40.630785+0000","last_clean":"2023-09-23T23:27:19.534765+0000","last_became_active":"2023-09-23T23:27:21.718696+0000","last_became_peered":"2023-09-23T23:27:21.718696+0000","last_unstale":"2023-09-23T23:27:40.630785+0000","last_undegraded":"2023-09-23T23:27:40.630785+0000","last_fullsized":"2023-09-23T23:27:40.630785+0000","mapping_epoch":425,"log_start":"355'1830","ondisk_log_start":"355'1830","created":365,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":7317,"log_dups_size":0,"ondisk_log_size":7317,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45678592,"num_objects":697,"num_object_clones":0,"num_object_copies":2091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":697,"num_objects_unfound":0,"num_objects_dirty":697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5929,"num_write_kb":212032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,4],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":697}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":701,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.058192+0000","last_change":"2023-09-23T23:25:34.510468+0000","last_active":"2023-09-23T23:27:33.058192+0000","last_peered":"2023-09-23T23:27:33.058192+0000","last_clean":"2023-09-23T23:27:33.058192+0000","last_became_active":"2023-09-23T23:25:34.510106+0000","last_became_peered":"2023-09-23T23:25:34.510106+0000","last_unstale":"2023-09-23T23:27:33.058192+0000","last_undegraded":"2023-09-23T23:27:33.058192+0000","last_fullsized":"2023-09-23T23:27:33.058192+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:42.157293+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":452,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.053069+0000","last_change":"2023-09-23T23:25:31.746383+0000","last_active":"2023-09-23T23:27:33.053069+0000","last_peered":"2023-09-23T23:27:33.053069+0000","last_clean":"2023-09-23T23:27:33.053069+0000","last_became_active":"2023-09-23T23:25:31.745929+0000","last_became_peered":"2023-09-23T23:25:31.745929+0000","last_unstale":"2023-09-23T23:27:33.053069+0000","last_undegraded":"2023-09-23T23:27:33.053069+0000","last_fullsized":"2023-09-23T23:27:33.053069+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:21.696449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":445,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.054026+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:27:33.054026+0000","last_peered":"2023-09-23T23:27:33.054026+0000","last_clean":"2023-09-23T23:27:33.054026+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:27:33.054026+0000","last_undegraded":"2023-09-23T23:27:33.054026+0000","last_fullsized":"2023-09-23T23:27:33.054026+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"435'9106","reported_seq":12878,"reported_epoch":435,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:40.499704+0000","last_change":"2023-09-23T23:27:21.773301+0000","last_active":"2023-09-23T23:27:40.499704+0000","last_peered":"2023-09-23T23:27:40.499704+0000","last_clean":"2023-09-23T23:27:19.482304+0000","last_became_active":"2023-09-23T23:27:21.727527+0000","last_became_peered":"2023-09-23T23:27:21.727527+0000","last_unstale":"2023-09-23T23:27:40.499704+0000","last_undegraded":"2023-09-23T23:27:40.499704+0000","last_fullsized":"2023-09-23T23:27:40.499704+0000","mapping_epoch":425,"log_start":"355'1793","ondisk_log_start":"355'1793","created":365,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":7313,"log_dups_size":0,"ondisk_log_size":7313,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43122688,"num_objects":658,"num_object_clones":0,"num_object_copies":1974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":658,"num_objects_unfound":0,"num_objects_dirty":658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5926,"num_write_kb":210688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1807,"num_bytes_recovered":118423552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[0,5,4],"avail_no_missing":["0","4","5","7"],"object_location_counts":[{"shards":"0,4,5","objects":658}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":808,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.062055+0000","last_change":"2023-09-23T23:26:14.527390+0000","last_active":"2023-09-23T23:27:33.062055+0000","last_peered":"2023-09-23T23:27:33.062055+0000","last_clean":"2023-09-23T23:27:33.062055+0000","last_became_active":"2023-09-23T23:26:14.527008+0000","last_became_peered":"2023-09-23T23:26:14.527008+0000","last_unstale":"2023-09-23T23:27:33.062055+0000","last_undegraded":"2023-09-23T23:27:33.062055+0000","last_fullsized":"2023-09-23T23:27:33.062055+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:36.460827+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":439,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.071060+0000","last_change":"2023-09-23T23:20:15.786279+0000","last_active":"2023-09-23T23:27:33.071060+0000","last_peered":"2023-09-23T23:27:33.071060+0000","last_clean":"2023-09-23T23:27:33.071060+0000","last_became_active":"2023-09-23T23:20:15.782731+0000","last_became_peered":"2023-09-23T23:20:15.782731+0000","last_unstale":"2023-09-23T23:27:33.071060+0000","last_undegraded":"2023-09-23T23:27:33.071060+0000","last_fullsized":"2023-09-23T23:27:33.071060+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:58.011396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":480,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T23:27:33.061437+0000","last_change":"2023-09-23T23:25:31.827217+0000","last_active":"2023-09-23T23:27:33.061437+0000","last_peered":"2023-09-23T23:27:33.061437+0000","last_clean":"2023-09-23T23:27:33.061437+0000","last_became_active":"2023-09-23T23:25:31.826915+0000","last_became_peered":"2023-09-23T23:25:31.826915+0000","last_unstale":"2023-09-23T23:27:33.061437+0000","last_undegraded":"2023-09-23T23:27:33.061437+0000","last_fullsized":"2023-09-23T23:27:33.061437+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:44.656199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":1460928536,"num_objects":22293,"num_object_clones":0,"num_object_copies":66879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":827,"num_objects_misplaced":11529,"num_objects_unfound":0,"num_objects_dirty":22293,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":166194,"num_write_kb":5311105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60539,"num_bytes_recovered":3912171520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7242199040,"data_stored":7242186824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":162306,"ondisk_log_size":162306,"up":78,"acting":77,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":6162,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":512693,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":21,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1210401,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782578,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1715216,"kb_used_data":1534856,"kb_used_omap":8,"kb_used_meta":180343,"kb_avail":92029936,"statfs":{"total":95995035648,"available":94238654464,"internally_reserved":0,"allocated":1571692544,"data_stored":1570447641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":184671301},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":2.5590000000000002,"15min":1.843},"min":{"1min":0.23100000000000001,"5min":0.221,"15min":0.221},"max":{"1min":6.9930000000000003,"5min":85.721000000000004,"15min":85.721000000000004},"last":7.1840000000000002},{"interface":"front","average":{"1min":1.825,"5min":2.601,"15min":1.597},"min":{"1min":0.26400000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":6.7350000000000003,"5min":85.965000000000003,"15min":85.965000000000003},"last":5.0970000000000004}]},{"osd":1,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":1.645,"15min":1.2410000000000001},"min":{"1min":0.25600000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.5039999999999996,"5min":60.847000000000001,"15min":60.847000000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.2929999999999999,"5min":1.9490000000000001,"15min":1.629},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.6310000000000002,"5min":59.512999999999998,"15min":59.512999999999998},"last":1.4099999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5189999999999999,"5min":2.3809999999999998,"15min":2.0150000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.7430000000000003,"5min":55.811999999999998,"15min":55.811999999999998},"last":2.8820000000000001},{"interface":"front","average":{"1min":1.847,"5min":1.8740000000000001,"15min":1.419},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.5449999999999999,"5min":55.847999999999999,"15min":55.847999999999999},"last":1.155}]},{"osd":3,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":2.024,"5min":1.536,"15min":1.5049999999999999},"min":{"1min":0.34100000000000003,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":9.9960000000000004,"5min":9.9960000000000004,"15min":9.9960000000000004},"last":1.633},{"interface":"front","average":{"1min":2.036,"5min":1.502,"15min":1.363},"min":{"1min":0.35999999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":9.9659999999999993,"5min":9.9659999999999993,"15min":9.9659999999999993},"last":0.372}]},{"osd":4,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.78600000000000003,"15min":0.83399999999999996},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.2270000000000003,"5min":6.8380000000000001,"15min":6.8380000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.91000000000000003,"15min":0.92800000000000005},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.1030000000000002,"5min":5.242,"15min":5.242},"last":0.39100000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":1.0289999999999999,"15min":1.0669999999999999},"min":{"1min":0.29299999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":2.1579999999999999,"5min":5.2110000000000003,"15min":5.2110000000000003},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.80900000000000005,"15min":0.81000000000000005},"min":{"1min":0.29499999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.5249999999999999,"5min":6.0149999999999997,"15min":6.0149999999999997},"last":0.59399999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":1.389,"15min":0.96599999999999997},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.4720000000000004,"5min":62.213000000000001,"15min":62.213000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.85399999999999998,"5min":1.629,"15min":1.2689999999999999},"min":{"1min":0.36899999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.2969999999999997,"5min":62.805999999999997,"15min":62.805999999999997},"last":0.42299999999999999}]}]},{"osd":6,"up_from":0,"seq":197568496055,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":112,"seq":481036337396,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1067816,"kb_used_data":450656,"kb_used_omap":4,"kb_used_meta":617147,"kb_avail":92677336,"statfs":{"total":95995035648,"available":94901592064,"internally_reserved":0,"allocated":461471744,"data_stored":459441929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":631959438},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:26:57 2023","interfaces":[{"interface":"back","average":{"1min":4.0049999999999999,"5min":2.8580000000000001,"15min":2.8319999999999999},"min":{"1min":0.192,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":16.491,"5min":75.823999999999998,"15min":75.823999999999998},"last":1.6890000000000001},{"interface":"front","average":{"1min":2.601,"5min":1.9730000000000001,"15min":2.73},"min":{"1min":0.26800000000000002,"5min":0.193,"15min":0.192},"max":{"1min":13.154,"5min":75.840999999999994,"15min":75.840999999999994},"last":0.191}]},{"osd":2,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":4.2949999999999999,"5min":5.4870000000000001,"15min":4.7409999999999997},"min":{"1min":0.22,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":14.538,"5min":209.84200000000001,"15min":209.84200000000001},"last":1.8320000000000001},{"interface":"front","average":{"1min":2.931,"5min":5.4779999999999998,"15min":3.8759999999999999},"min":{"1min":0.34000000000000002,"5min":0.218,"15min":0.218},"max":{"1min":11.276999999999999,"5min":209.11099999999999,"15min":209.11099999999999},"last":4.6420000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:26:36 2023","interfaces":[{"interface":"back","average":{"1min":2.3029999999999999,"5min":2.6850000000000001,"15min":3.754},"min":{"1min":0.23200000000000001,"5min":0.22600000000000001,"15min":0.20799999999999999},"max":{"1min":10.388999999999999,"5min":157.90899999999999,"15min":157.90899999999999},"last":1.7070000000000001},{"interface":"front","average":{"1min":3.024,"5min":2.7330000000000001,"15min":3.605},"min":{"1min":0.255,"5min":0.245,"15min":0.245},"max":{"1min":10.419,"5min":156.27000000000001,"15min":156.27000000000001},"last":0.86399999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:26:57 2023","interfaces":[{"interface":"back","average":{"1min":2.5529999999999999,"5min":1.5229999999999999,"15min":1.5680000000000001},"min":{"1min":0.29099999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":9.3780000000000001,"5min":9.3780000000000001,"15min":18.474},"last":3.9980000000000002},{"interface":"front","average":{"1min":2.9500000000000002,"5min":1.804,"15min":1.758},"min":{"1min":0.36199999999999999,"5min":0.30399999999999999,"15min":0.29099999999999998},"max":{"1min":9.7460000000000004,"5min":9.7460000000000004,"15min":25.710999999999999},"last":1.617}]},{"osd":5,"last update":"Sat Sep 23 23:26:57 2023","interfaces":[{"interface":"back","average":{"1min":2.0979999999999999,"5min":1.0880000000000001,"15min":3.0640000000000001},"min":{"1min":0.34000000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":13.064,"5min":13.064,"15min":250.33000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":3.0219999999999998,"5min":1.8149999999999999,"15min":3.2160000000000002},"min":{"1min":0.40300000000000002,"5min":0.34899999999999998,"15min":0.30199999999999999},"max":{"1min":9.3450000000000006,"5min":10.92,"15min":250.21199999999999},"last":3.9340000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:26:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":2.407,"15min":2.4420000000000002},"min":{"1min":0.39300000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":13.119,"5min":74.087999999999994,"15min":74.087999999999994},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.6960000000000002,"5min":2.6859999999999999,"15min":2.3759999999999999},"min":{"1min":0.40699999999999997,"5min":0.33100000000000002,"15min":0.32800000000000001},"max":{"1min":9.4030000000000005,"5min":73.974999999999994,"15min":73.974999999999994},"last":4.1310000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":3.6389999999999998,"5min":3.859,"15min":4.3630000000000004},"min":{"1min":0.40699999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.7929999999999993,"5min":78.308000000000007,"15min":78.308000000000007},"last":1.728},{"interface":"front","average":{"1min":2.1989999999999998,"5min":3.3580000000000001,"15min":3.8319999999999999},"min":{"1min":0.378,"5min":0.371,"15min":0.371},"max":{"1min":9.3109999999999999,"5min":77.204999999999998,"15min":77.204999999999998},"last":3.9710000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024220,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1922800,"kb_used_data":888552,"kb_used_omap":5,"kb_used_meta":1034106,"kb_avail":91822352,"statfs":{"total":95995035648,"available":94026088448,"internally_reserved":0,"allocated":909877248,"data_stored":907775523,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1058925178},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:27:18 2023","interfaces":[{"interface":"back","average":{"1min":3.29,"5min":4.29,"15min":3.9260000000000002},"min":{"1min":0.17799999999999999,"5min":0.17499999999999999,"15min":0.16900000000000001},"max":{"1min":15.144,"5min":139.30000000000001,"15min":155.91999999999999},"last":80.774000000000001},{"interface":"front","average":{"1min":3.488,"5min":1.998,"15min":2.742},"min":{"1min":0.22700000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":10.423,"5min":10.423,"15min":153.28899999999999},"last":85.534999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:27:40 2023","interfaces":[{"interface":"back","average":{"1min":6.4240000000000004,"5min":7.3380000000000001,"15min":4.9690000000000003},"min":{"1min":0.22,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":80.736000000000004,"5min":171.15100000000001,"15min":171.15100000000001},"last":80.736000000000004},{"interface":"front","average":{"1min":7.1420000000000003,"5min":6.9139999999999997,"15min":5.2640000000000002},"min":{"1min":0.26400000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":85.375,"5min":173.04900000000001,"15min":173.04900000000001},"last":85.375}]},{"osd":3,"last update":"Sat Sep 23 23:27:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1019999999999999,"5min":4.6459999999999999,"15min":3.8399999999999999},"min":{"1min":0.32900000000000001,"5min":0.25600000000000001,"15min":0.23200000000000001},"max":{"1min":13.112,"5min":139.34899999999999,"15min":139.34899999999999},"last":0.26800000000000002},{"interface":"front","average":{"1min":3.2480000000000002,"5min":4.5229999999999997,"15min":3.4209999999999998},"min":{"1min":0.311,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":10.237,"5min":138.739,"15min":138.739},"last":0.317}]},{"osd":4,"last update":"Sat Sep 23 23:26:44 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":3.391,"15min":2.6309999999999998},"min":{"1min":0.315,"5min":0.29099999999999998,"15min":0.247},"max":{"1min":4.5019999999999998,"5min":138.81299999999999,"15min":138.81299999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.1709999999999998,"5min":5.3230000000000004,"15min":3.2930000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":18.004000000000001,"5min":171.251,"15min":171.251},"last":0.33200000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:26:44 2023","interfaces":[{"interface":"back","average":{"1min":2.1379999999999999,"5min":5.4199999999999999,"15min":2.9409999999999998},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.29099999999999998},"max":{"1min":18.045000000000002,"5min":171.30699999999999,"15min":171.30699999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.55,"5min":3.4710000000000001,"15min":2.2490000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":7.5810000000000004,"5min":139.40799999999999,"15min":139.40799999999999},"last":1.2090000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:26:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2109999999999999,"5min":5.399,"15min":3.1899999999999999},"min":{"1min":0.34499999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":21.276,"5min":171.33600000000001,"15min":171.33600000000001},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.706,"5min":6.0579999999999998,"15min":3.4289999999999998},"min":{"1min":0.40200000000000002,"5min":0.38500000000000001,"15min":0.32700000000000001},"max":{"1min":7.5090000000000003,"5min":184.13499999999999,"15min":184.13499999999999},"last":0.439}]},{"osd":7,"last update":"Sat Sep 23 23:27:40 2023","interfaces":[{"interface":"back","average":{"1min":2.375,"5min":3.7650000000000001,"15min":2.149},"min":{"1min":0.378,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":9.1739999999999995,"5min":138.77699999999999,"15min":138.77699999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":2.29,"5min":3.6720000000000002,"15min":1.8120000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":11.27,"5min":139.37899999999999,"15min":139.37899999999999},"last":0.58299999999999996}]}]},{"osd":2,"up_from":199,"seq":854698492041,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1448220,"kb_used_data":920148,"kb_used_omap":7,"kb_used_meta":527928,"kb_avail":92296932,"statfs":{"total":95995035648,"available":94512058368,"internally_reserved":0,"allocated":942231552,"data_stored":940154891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":540598353},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":3.9489999999999998,"5min":5.8310000000000004,"15min":5.0700000000000003},"min":{"1min":0.20399999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":17.382000000000001,"5min":119.277,"15min":119.277},"last":4.9580000000000002},{"interface":"front","average":{"1min":4.0880000000000001,"5min":5.5049999999999999,"15min":5.2030000000000003},"min":{"1min":0.192,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":11.69,"5min":119.56100000000001,"15min":119.56100000000001},"last":4.8250000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":10.321999999999999,"5min":4.8470000000000004,"15min":8.0600000000000005},"min":{"1min":0.23499999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":103.65900000000001,"5min":157.98699999999999,"15min":157.98699999999999},"last":5.1239999999999997},{"interface":"front","average":{"1min":8.9329999999999998,"5min":4.9189999999999996,"15min":7.7240000000000002},"min":{"1min":0.28499999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":100.182,"5min":158.03800000000001,"15min":158.03800000000001},"last":4.8959999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":4.9260000000000002,"5min":4.3360000000000003,"15min":5.3029999999999999},"min":{"1min":0.27300000000000002,"5min":0.191,"15min":0.191},"max":{"1min":23.045000000000002,"5min":119.366,"15min":119.366},"last":8.1319999999999997},{"interface":"front","average":{"1min":2.6520000000000001,"5min":3.1320000000000001,"15min":4.5730000000000004},"min":{"1min":0.29699999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.8079999999999998,"5min":75.418000000000006,"15min":75.418000000000006},"last":7.0830000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":2.2480000000000002,"5min":2.7610000000000001,"15min":3.9060000000000001},"min":{"1min":0.29399999999999998,"5min":0.25,"15min":0.25},"max":{"1min":10.444000000000001,"5min":119.88200000000001,"15min":332.98099999999999},"last":4.7859999999999996},{"interface":"front","average":{"1min":2.5449999999999999,"5min":1.4790000000000001,"15min":3.3860000000000001},"min":{"1min":0.36499999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.268000000000001,"5min":21.076000000000001,"15min":332.96199999999999},"last":5.0380000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":2.9409999999999998,"15min":2.2839999999999998},"min":{"1min":0.318,"5min":0.29499999999999998,"15min":0.27500000000000002},"max":{"1min":7.5880000000000001,"5min":119.919,"15min":119.919},"last":4.7450000000000001},{"interface":"front","average":{"1min":2.5110000000000001,"5min":2.9420000000000002,"15min":2.2719999999999998},"min":{"1min":0.39800000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.305,"5min":119.306,"15min":119.306},"last":5.0999999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":3.2109999999999999,"15min":4.1639999999999997},"min":{"1min":0.34000000000000002,"5min":0.32000000000000001,"15min":0.31},"max":{"1min":7.6369999999999996,"5min":119.943,"15min":349.32799999999997},"last":4.8079999999999998},{"interface":"front","average":{"1min":2.532,"5min":1.925,"15min":3.6659999999999999},"min":{"1min":0.42499999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":10.202999999999999,"5min":39.856999999999999,"15min":349.17700000000002},"last":5.0750000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:26:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0750000000000002,"5min":1.6319999999999999,"15min":1.3280000000000001},"min":{"1min":0.40000000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.7999999999999998,"5min":20.350000000000001,"15min":20.350000000000001},"last":4.8559999999999999},{"interface":"front","average":{"1min":2.1880000000000002,"5min":1.7270000000000001,"15min":1.419},"min":{"1min":0.39500000000000002,"5min":0.38,"15min":0.38},"max":{"1min":9.6029999999999998,"5min":20.321000000000002,"15min":20.321000000000002},"last":4.8719999999999999}]}]},{"osd":3,"up_from":148,"seq":635655160010,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1590360,"kb_used_data":954640,"kb_used_omap":2,"kb_used_meta":635645,"kb_avail":92154792,"statfs":{"total":95995035648,"available":94366507008,"internally_reserved":0,"allocated":977551360,"data_stored":975532558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":650900533},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:27:32 2023","interfaces":[{"interface":"back","average":{"1min":3.9369999999999998,"5min":4.4020000000000001,"15min":3.3399999999999999},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.158},"max":{"1min":40.901000000000003,"5min":103.866,"15min":103.866},"last":2.9700000000000002},{"interface":"front","average":{"1min":4.8689999999999998,"5min":4.2389999999999999,"15min":3.0339999999999998},"min":{"1min":0.191,"5min":0.191,"15min":0.186},"max":{"1min":46.204000000000001,"5min":101.884,"15min":101.884},"last":8.8219999999999992}]},{"osd":1,"last update":"Sat Sep 23 23:27:32 2023","interfaces":[{"interface":"back","average":{"1min":3.5379999999999998,"5min":2.6829999999999998,"15min":2.851},"min":{"1min":0.218,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":26.959,"5min":38.094000000000001,"15min":76.322999999999993},"last":0.69799999999999995},{"interface":"front","average":{"1min":5.0220000000000002,"5min":2.8530000000000002,"15min":2.7330000000000001},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.20499999999999999},"max":{"1min":27.321999999999999,"5min":42.030999999999999,"15min":76.444000000000003},"last":0.32700000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:26:40 2023","interfaces":[{"interface":"back","average":{"1min":2.4569999999999999,"5min":4.8760000000000003,"15min":3.754},"min":{"1min":0.28100000000000003,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":9.7970000000000006,"5min":101.279,"15min":101.279},"last":3.6829999999999998},{"interface":"front","average":{"1min":2.4449999999999998,"5min":4.54,"15min":2.73},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":7.5469999999999997,"5min":101.294,"15min":101.294},"last":2.9369999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:27:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":1.1699999999999999,"15min":1.4339999999999999},"min":{"1min":0.27500000000000002,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":6.8890000000000002,"5min":7.2610000000000001,"15min":14.754},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.5029999999999999,"5min":1.482,"15min":1.4770000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.2800000000000002,"5min":15.619,"15min":15.619},"last":0.52600000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:27:32 2023","interfaces":[{"interface":"back","average":{"1min":1.788,"5min":1.573,"15min":1.6819999999999999},"min":{"1min":0.29599999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.2619999999999996,"5min":15.762,"15min":75.733000000000004},"last":0.497},{"interface":"front","average":{"1min":1.6990000000000001,"5min":1.679,"15min":1.9730000000000001},"min":{"1min":0.318,"5min":0.316,"15min":0.28399999999999997},"max":{"1min":6.9880000000000004,"5min":13.864000000000001,"15min":75.686999999999998},"last":3.7189999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:27:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6120000000000001,"5min":2.1480000000000001,"15min":1.7130000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.3019999999999996,"5min":53.057000000000002,"15min":53.057000000000002},"last":3.5579999999999998},{"interface":"front","average":{"1min":1.6419999999999999,"5min":2.2029999999999998,"15min":1.9279999999999999},"min":{"1min":0.40000000000000002,"5min":0.35299999999999998,"15min":0.33300000000000002},"max":{"1min":7.2320000000000002,"5min":53.088000000000001,"15min":53.088000000000001},"last":3.7469999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:26:40 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":1.0600000000000001,"15min":0.92800000000000005},"min":{"1min":0.40400000000000003,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":6.806,"5min":6.806,"15min":6.806},"last":2.9940000000000002},{"interface":"front","average":{"1min":2.6909999999999998,"5min":1.5309999999999999,"15min":1.3320000000000001},"min":{"1min":0.47299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":15.795,"5min":15.795,"15min":15.795},"last":3.512}]}]},{"osd":4,"up_from":30,"seq":128849019328,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1336464,"kb_used_data":641288,"kb_used_omap":0,"kb_used_meta":695168,"kb_avail":92408688,"statfs":{"total":95995035648,"available":94626496512,"internally_reserved":0,"allocated":656678912,"data_stored":655433742,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":711852032},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:26:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":3.1850000000000001,"15min":4.6050000000000004},"min":{"1min":0.25900000000000001,"5min":0.218,"15min":0.20799999999999999},"max":{"1min":17.712,"5min":92.141999999999996,"15min":199.64400000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.744,"5min":3.29,"15min":4.8159999999999998},"min":{"1min":0.224,"5min":0.216,"15min":0.216},"max":{"1min":16.666,"5min":92.284000000000006,"15min":195.649},"last":0.40799999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:27:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":1.585,"15min":3.8650000000000002},"min":{"1min":0.24299999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.8700000000000001,"5min":40.643000000000001,"15min":161.923},"last":0.65400000000000003},{"interface":"front","average":{"1min":2.2650000000000001,"5min":1.901,"15min":4.0259999999999998},"min":{"1min":0.307,"5min":0.27500000000000002,"15min":0.26200000000000001},"max":{"1min":10.657999999999999,"5min":35.362000000000002,"15min":161.39699999999999},"last":0.61199999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":4.157,"15min":3.3079999999999998},"min":{"1min":0.27900000000000003,"5min":0.27500000000000002,"15min":0.26300000000000001},"max":{"1min":7.3239999999999998,"5min":97.962000000000003,"15min":97.962000000000003},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.7889999999999999,"5min":4.2460000000000004,"15min":2.992},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.3929999999999998,"5min":98.137,"15min":98.137},"last":0.497}]},{"osd":3,"last update":"Sat Sep 23 23:27:35 2023","interfaces":[{"interface":"back","average":{"1min":4.1859999999999999,"5min":6.5609999999999999,"15min":5.6760000000000002},"min":{"1min":0.33500000000000002,"5min":0.27200000000000002,"15min":0.26300000000000001},"max":{"1min":50.755000000000003,"5min":176.74100000000001,"15min":303.41500000000002},"last":50.755000000000003},{"interface":"front","average":{"1min":3.3660000000000001,"5min":6.1959999999999997,"15min":5.7530000000000001},"min":{"1min":0.39400000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":50.738999999999997,"5min":176.87799999999999,"15min":303.428},"last":50.738999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:27:09 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.65000000000000002,"15min":1.4379999999999999},"min":{"1min":0.29499999999999998,"5min":0.26100000000000001,"15min":0.23699999999999999},"max":{"1min":4.266,"5min":4.266,"15min":160.79499999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.879,"5min":0.746,"15min":1.6080000000000001},"min":{"1min":0.34999999999999998,"5min":0.249,"15min":0.249},"max":{"1min":4.4359999999999999,"5min":5.0789999999999997,"15min":160.61099999999999},"last":0.44400000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:27:35 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":1.55,"15min":1.169},"min":{"1min":0.29799999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.097,"5min":90.346999999999994,"15min":90.346999999999994},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":1.6220000000000001,"15min":1.3100000000000001},"min":{"1min":0.38300000000000001,"5min":0.31900000000000001,"15min":0.30599999999999999},"max":{"1min":1.0980000000000001,"5min":90.733000000000004,"15min":90.733000000000004},"last":0.53800000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:26:39 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":1.5209999999999999,"15min":0.88500000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.3280000000000003,"5min":61.923999999999999,"15min":61.923999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.83499999999999996,"5min":1.581,"15min":1.0620000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":2.1520000000000001,"5min":61.990000000000002,"15min":61.990000000000002},"last":0.47499999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757691,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1123196,"kb_used_data":521716,"kb_used_omap":0,"kb_used_meta":601472,"kb_avail":92621956,"statfs":{"total":95995035648,"available":94844882944,"internally_reserved":0,"allocated":534237184,"data_stored":533008493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":615907328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:26:52 2023","interfaces":[{"interface":"back","average":{"1min":5.0419999999999998,"5min":5.2549999999999999,"15min":3.7789999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.19500000000000001},"max":{"1min":63.741999999999997,"5min":113.869,"15min":113.869},"last":3.0550000000000002},{"interface":"front","average":{"1min":4.8609999999999998,"5min":6.5490000000000004,"15min":4.2549999999999999},"min":{"1min":0.23699999999999999,"5min":0.22900000000000001,"15min":0.22700000000000001},"max":{"1min":63.259,"5min":138.55199999999999,"15min":138.55199999999999},"last":2.8879999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:27:02 2023","interfaces":[{"interface":"back","average":{"1min":9.375,"5min":3.3940000000000001,"15min":3.9710000000000001},"min":{"1min":0.25900000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":69.816999999999993,"5min":69.816999999999993,"15min":193.584},"last":12.702},{"interface":"front","average":{"1min":10.48,"5min":3.8639999999999999,"15min":4.2549999999999999},"min":{"1min":0.375,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":70.268000000000001,"5min":70.268000000000001,"15min":194.03100000000001},"last":1.329}]},{"osd":2,"last update":"Sat Sep 23 23:27:35 2023","interfaces":[{"interface":"back","average":{"1min":2.0750000000000002,"5min":3.952,"15min":2.645},"min":{"1min":0.314,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.5640000000000001,"5min":144.91200000000001,"15min":144.91200000000001},"last":1.96},{"interface":"front","average":{"1min":1.776,"5min":3.806,"15min":2.5619999999999998},"min":{"1min":0.28799999999999998,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":5.8410000000000002,"5min":145.15799999999999,"15min":145.15799999999999},"last":1.7629999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:27:29 2023","interfaces":[{"interface":"back","average":{"1min":4.4880000000000004,"5min":3.008,"15min":4.468},"min":{"1min":0.32700000000000001,"5min":0.315,"15min":0.27400000000000002},"max":{"1min":62.612000000000002,"5min":62.612000000000002,"15min":159.53299999999999},"last":1.5860000000000001},{"interface":"front","average":{"1min":4.3970000000000002,"5min":3.1000000000000001,"15min":4.6980000000000004},"min":{"1min":0.318,"5min":0.318,"15min":0.312},"max":{"1min":62.639000000000003,"5min":62.639000000000003,"15min":159.54900000000001},"last":1.577}]},{"osd":4,"last update":"Sat Sep 23 23:27:10 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.71699999999999997,"15min":0.98299999999999998},"min":{"1min":0.26700000000000002,"5min":0.251,"15min":0.23999999999999999},"max":{"1min":1.1919999999999999,"5min":5.3739999999999997,"15min":10.012},"last":0.311},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.60899999999999999,"15min":0.76300000000000001},"min":{"1min":0.30299999999999999,"5min":0.245,"15min":0.245},"max":{"1min":3.2530000000000001,"5min":3.2530000000000001,"15min":11.923999999999999},"last":0.34499999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:27:10 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.4770000000000001,"15min":2.2250000000000001},"min":{"1min":0.33000000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.2749999999999999,"5min":68.456999999999994,"15min":310.68799999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":1.452,"15min":2.4209999999999998},"min":{"1min":0.34799999999999998,"5min":0.26000000000000001,"15min":0.25900000000000001},"max":{"1min":1.2050000000000001,"5min":66.721999999999994,"15min":311.45400000000001},"last":0.40000000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:27:35 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":2.2160000000000002,"15min":3.5030000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.0859999999999999,"5min":66.372,"15min":66.372},"last":0.371},{"interface":"front","average":{"1min":0.66400000000000003,"5min":2.262,"15min":3.6440000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.758,"5min":66.558999999999997,"15min":66.558999999999997},"last":0.433}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":902496256,"data_stored":902496256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":442171392,"data_stored":442171392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":911544320,"data_stored":911540248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":955777024,"data_stored":955777024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":633147392,"data_stored":633143320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":518520832,"data_stored":518520832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1321340928,"data_stored":1321336856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1557200896,"data_stored":1557200896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:27:45.131 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:27:45.289 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:27:45.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:45.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:27:45.395 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:27:45.396 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:45 smithi138 ceph-mon[193606]: pgmap v1233: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 116 active+clean; 1.3 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 316/64911 objects degraded (0.487%); 11343/64911 objects misplaced (17.475%); 4.3 MiB/s, 194 objects/s recovering 2023-09-23T23:27:45.396 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:45 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:27:45.396 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:45 smithi138 ceph-mon[193606]: failed to encode map e438 with expected crc 2023-09-23T23:27:45.396 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:45 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:27:45.396 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:45 smithi138 ceph-mon[193606]: osdmap e438: 8 total, 8 up, 7 in 2023-09-23T23:27:45.396 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:45 smithi138 ceph-mon[193606]: failed to encode map e438 with expected crc 2023-09-23T23:27:45.397 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:27:45.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:45 smithi110 ceph-mon[212939]: pgmap v1233: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 116 active+clean; 1.3 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 316/64911 objects degraded (0.487%); 11343/64911 objects misplaced (17.475%); 4.3 MiB/s, 194 objects/s recovering 2023-09-23T23:27:45.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:45 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:27:45.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:45 smithi110 ceph-mon[212939]: failed to encode map e438 with expected crc 2023-09-23T23:27:45.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:45 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:27:45.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:45 smithi110 ceph-mon[212939]: osdmap e438: 8 total, 8 up, 7 in 2023-09-23T23:27:45.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:45 smithi110 ceph-mon[212939]: failed to encode map e438 with expected crc 2023-09-23T23:27:45.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:45 smithi110 ceph-mon[221102]: pgmap v1233: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 116 active+clean; 1.3 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 316/64911 objects degraded (0.487%); 11343/64911 objects misplaced (17.475%); 4.3 MiB/s, 194 objects/s recovering 2023-09-23T23:27:45.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:45 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:27:45.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:45 smithi110 ceph-mon[221102]: failed to encode map e438 with expected crc 2023-09-23T23:27:45.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:45 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:27:45.425 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:45 smithi110 ceph-mon[221102]: osdmap e438: 8 total, 8 up, 7 in 2023-09-23T23:27:45.425 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:45 smithi110 ceph-mon[221102]: failed to encode map e438 with expected crc 2023-09-23T23:27:45.431 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:45.500 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:27:45.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:45.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:27:45.566 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:27:45.606 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:45.675 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:45.775 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:45.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:45.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:45.888 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:27:45.903 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:27:45.966 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:46.135 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:27:46.145 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:27:46.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:46.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:27:46.255 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:27:46.288 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:46.397 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:27:46.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:46.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:27:46.504 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:27:46.539 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:47.144 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:27:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:27:46] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.43.0" 2023-09-23T23:27:47.426 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:27:47.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:47 smithi138 ceph-mon[193606]: pgmap v1235: 129 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 116 active+clean; 1.2 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 316/58797 objects degraded (0.537%); 8513/58797 objects misplaced (14.479%); 10 MiB/s, 247 objects/s recovering 2023-09-23T23:27:47.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:47 smithi138 ceph-mon[193606]: failed to encode map e439 with expected crc 2023-09-23T23:27:47.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:47 smithi138 ceph-mon[193606]: osdmap e439: 8 total, 8 up, 7 in 2023-09-23T23:27:47.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:47 smithi138 ceph-mon[193606]: failed to encode map e439 with expected crc 2023-09-23T23:27:47.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:47 smithi138 ceph-mon[193606]: failed to encode map e439 with expected crc 2023-09-23T23:27:47.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:47 smithi110 ceph-mon[212939]: pgmap v1235: 129 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 116 active+clean; 1.2 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 316/58797 objects degraded (0.537%); 8513/58797 objects misplaced (14.479%); 10 MiB/s, 247 objects/s recovering 2023-09-23T23:27:47.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:47 smithi110 ceph-mon[212939]: failed to encode map e439 with expected crc 2023-09-23T23:27:47.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:47 smithi110 ceph-mon[212939]: osdmap e439: 8 total, 8 up, 7 in 2023-09-23T23:27:47.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:47 smithi110 ceph-mon[212939]: failed to encode map e439 with expected crc 2023-09-23T23:27:47.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:47 smithi110 ceph-mon[212939]: failed to encode map e439 with expected crc 2023-09-23T23:27:47.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:47 smithi110 ceph-mon[221102]: pgmap v1235: 129 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 116 active+clean; 1.2 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 316/58797 objects degraded (0.537%); 8513/58797 objects misplaced (14.479%); 10 MiB/s, 247 objects/s recovering 2023-09-23T23:27:47.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:47 smithi110 ceph-mon[221102]: failed to encode map e439 with expected crc 2023-09-23T23:27:47.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:47 smithi110 ceph-mon[221102]: osdmap e439: 8 total, 8 up, 7 in 2023-09-23T23:27:47.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:47 smithi110 ceph-mon[221102]: failed to encode map e439 with expected crc 2023-09-23T23:27:47.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:47 smithi110 ceph-mon[221102]: failed to encode map e439 with expected crc 2023-09-23T23:27:47.494 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:27:47.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:47.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:27:47.601 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:27:47.653 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:47.771 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:27:47.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:47.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:27:47.842 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:27:47.842 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:27:47.842 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:27:47.842 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:27:47.842 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:27:47.876 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:27:47.955 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:48.244 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:27:48.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:48.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:27:48.350 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:27:48.393 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:48.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:48 smithi138 ceph-mon[193606]: failed to encode map e440 with expected crc 2023-09-23T23:27:48.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:48 smithi138 ceph-mon[193606]: failed to encode map e440 with expected crc 2023-09-23T23:27:48.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:48 smithi138 ceph-mon[193606]: osdmap e440: 8 total, 8 up, 7 in 2023-09-23T23:27:48.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:48 smithi138 ceph-mon[193606]: failed to encode map e440 with expected crc 2023-09-23T23:27:48.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:48 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:27:48.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:48 smithi110 ceph-mon[212939]: failed to encode map e440 with expected crc 2023-09-23T23:27:48.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:48 smithi110 ceph-mon[212939]: failed to encode map e440 with expected crc 2023-09-23T23:27:48.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:48 smithi110 ceph-mon[212939]: osdmap e440: 8 total, 8 up, 7 in 2023-09-23T23:27:48.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:48 smithi110 ceph-mon[212939]: failed to encode map e440 with expected crc 2023-09-23T23:27:48.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:48 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:27:48.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:48 smithi110 ceph-mon[221102]: failed to encode map e440 with expected crc 2023-09-23T23:27:48.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:48 smithi110 ceph-mon[221102]: failed to encode map e440 with expected crc 2023-09-23T23:27:48.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:48 smithi110 ceph-mon[221102]: osdmap e440: 8 total, 8 up, 7 in 2023-09-23T23:27:48.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:48 smithi110 ceph-mon[221102]: failed to encode map e440 with expected crc 2023-09-23T23:27:48.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:48 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:27:48.679 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:27:48.720 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:27:48.722 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:27:48.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:48.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:27:48.738 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:27:48.797 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:27:48.827 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:27:48.839 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:27:48.887 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:27:48.900 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:48.929 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:27:48.964 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:27:49.008 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:27:49.041 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:27:49.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:49.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:27:49.147 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:27:49.159 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:49 smithi138 ceph-mon[193606]: pgmap v1238: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 116 active+clean; 1.2 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 316/57249 objects degraded (0.552%); 8304/57249 objects misplaced (14.505%); 6.5 MiB/s, 194 objects/s recovering 2023-09-23T23:27:49.159 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:49 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 316/57249 objects degraded (0.552%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:27:49.159 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:49 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3056160099' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:27:49.173 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:49.219 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:27:49.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:49.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:49.262 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:27:49.262 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:27:49.263 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:27:49.263 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:27:49.263 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:27:49.263 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:27:49.263 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:27:49.263 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:27:49.324 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:27:49.378 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:49.407 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:27:49.458 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:49 smithi110 ceph-mon[212939]: pgmap v1238: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 116 active+clean; 1.2 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 316/57249 objects degraded (0.552%); 8304/57249 objects misplaced (14.505%); 6.5 MiB/s, 194 objects/s recovering 2023-09-23T23:27:49.458 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:49 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 316/57249 objects degraded (0.552%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:27:49.458 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:49 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3056160099' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:27:49.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:49 smithi110 ceph-mon[221102]: pgmap v1238: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 116 active+clean; 1.2 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 316/57249 objects degraded (0.552%); 8304/57249 objects misplaced (14.505%); 6.5 MiB/s, 194 objects/s recovering 2023-09-23T23:27:49.459 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:49 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 316/57249 objects degraded (0.552%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:27:49.459 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:49 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3056160099' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:27:49.609 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:27:49.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:49.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:27:49.713 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:27:49.766 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:49.843 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:27:49.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:49.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:27:49.947 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:27:49.977 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:50.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:50 smithi138 ceph-mon[193606]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 316/57249 objects degraded (0.552%), 1 pg degraded, 1 pg undersized) 2023-09-23T23:27:50.498 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:50 smithi110 ceph-mon[221102]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 316/57249 objects degraded (0.552%), 1 pg degraded, 1 pg undersized) 2023-09-23T23:27:50.498 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:50 smithi110 ceph-mon[212939]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 316/57249 objects degraded (0.552%), 1 pg degraded, 1 pg undersized) 2023-09-23T23:27:50.686 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:27:50.858 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:27:50.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:50.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:27:50.964 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:27:51.001 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:51.115 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:27:51.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:51.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:51.222 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:27:51.301 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:51.376 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:51 smithi110 ceph-mon[212939]: pgmap v1239: 129 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 118 active+clean; 1.0 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 777 op/s; 7515/50907 objects misplaced (14.762%); 6.0 MiB/s, 149 objects/s recovering 2023-09-23T23:27:51.376 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:51 smithi110 ceph-mon[221102]: pgmap v1239: 129 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 118 active+clean; 1.0 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 777 op/s; 7515/50907 objects misplaced (14.762%); 6.0 MiB/s, 149 objects/s recovering 2023-09-23T23:27:51.377 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:51 smithi138 ceph-mon[193606]: pgmap v1239: 129 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 118 active+clean; 1.0 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 777 op/s; 7515/50907 objects misplaced (14.762%); 6.0 MiB/s, 149 objects/s recovering 2023-09-23T23:27:51.435 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:27:51.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:51.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:27:51.542 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:27:51.580 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:51.613 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:27:51.613 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:27:51.655 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:27:51.655 INFO:teuthology.orchestra.run.smithi110.stdout:{"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1549,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":440,"num_osds":8,"num_up_osds":8,"osd_up_since":1695511414,"num_in_osds":7,"osd_in_since":1695511530,"num_remapped_pgs":11},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":118},{"state_name":"active+remapped+backfill_wait","count":10},{"state_name":"active+remapped+backfilling","count":1}],"num_pgs":129,"num_pools":5,"num_objects":16969,"data_bytes":1110894669,"bytes_used":9598869504,"bytes_avail":662366380032,"bytes_total":671965249536,"misplaced_objects":7515,"misplaced_total":50907,"misplaced_ratio":0.1476221344805233,"recovering_objects_per_sec":149,"recovering_bytes_per_sec":6331281,"recovering_keys_per_sec":0,"num_objects_recovered":896,"num_bytes_recovered":38010880,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":777},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi110.fuglpi","status":"up:active","gid":24572}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.110:8443/","prometheus":"http://172.21.15.110:9283/"}},"servicemap":{"epoch":28,"modified":"2023-09-23T23:24:33.604549+0000","services":{}},"progress_events":{"833ef15d-07ae-4299-b4e4-2cf94e068147":{"message":"Upgrade to 18.2.0-460-gbb0daeff (10m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true},"95a38dfb-7905-4e6f-9481-693c25c07df3":{"message":"Global Recovery Event (2m)\n [=========================...] (remaining: 12s)","progress":0.90697675943374634,"add_to_ceph_s":true}}} 2023-09-23T23:27:52.370 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:27:52.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:52.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:27:52.476 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:27:52.478 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:27:52.508 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:52 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3781638465' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:27:52.508 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:52 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3781638465' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:27:52.511 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:52 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3781638465' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:27:52.513 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:52.525 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:27:52.912 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:27:53.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:53 smithi138 ceph-mon[193606]: pgmap v1240: 129 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 118 active+clean; 995 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 5994/47832 objects misplaced (12.531%); 14 MiB/s, 257 objects/s recovering 2023-09-23T23:27:53.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:53 smithi138 ceph-mon[193606]: osdmap e441: 8 total, 8 up, 7 in 2023-09-23T23:27:53.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:53 smithi138 ceph-mon[193606]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:53 smithi138 ceph-mon[193606]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:53 smithi138 ceph-mon[193606]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:53 smithi138 ceph-mon[193606]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:53 smithi138 ceph-mon[193606]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:53 smithi138 ceph-mon[193606]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:53 smithi138 ceph-mon[193606]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[212939]: pgmap v1240: 129 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 118 active+clean; 995 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 5994/47832 objects misplaced (12.531%); 14 MiB/s, 257 objects/s recovering 2023-09-23T23:27:53.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[212939]: osdmap e441: 8 total, 8 up, 7 in 2023-09-23T23:27:53.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[212939]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[212939]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[212939]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[212939]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[212939]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[212939]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[212939]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[221102]: pgmap v1240: 129 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 118 active+clean; 995 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 5994/47832 objects misplaced (12.531%); 14 MiB/s, 257 objects/s recovering 2023-09-23T23:27:53.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[221102]: osdmap e441: 8 total, 8 up, 7 in 2023-09-23T23:27:53.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[221102]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[221102]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[221102]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[221102]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[221102]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[221102]: failed to encode map e441 with expected crc 2023-09-23T23:27:53.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:53 smithi110 ceph-mon[221102]: failed to encode map e441 with expected crc 2023-09-23T23:27:54.397 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:27:54.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:54.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:27:54.414 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:54 smithi110 ceph-mon[212939]: osdmap e442: 8 total, 8 up, 7 in 2023-09-23T23:27:54.414 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:54 smithi110 ceph-mon[212939]: failed to encode map e442 with expected crc 2023-09-23T23:27:54.414 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:54 smithi110 ceph-mon[212939]: failed to encode map e442 with expected crc 2023-09-23T23:27:54.415 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:54 smithi110 ceph-mon[212939]: failed to encode map e442 with expected crc 2023-09-23T23:27:54.415 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:54 smithi110 ceph-mon[212939]: failed to encode map e442 with expected crc 2023-09-23T23:27:54.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:54 smithi110 ceph-mon[221102]: osdmap e442: 8 total, 8 up, 7 in 2023-09-23T23:27:54.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:54 smithi110 ceph-mon[221102]: failed to encode map e442 with expected crc 2023-09-23T23:27:54.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:54 smithi110 ceph-mon[221102]: failed to encode map e442 with expected crc 2023-09-23T23:27:54.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:54 smithi110 ceph-mon[221102]: failed to encode map e442 with expected crc 2023-09-23T23:27:54.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:54 smithi110 ceph-mon[221102]: failed to encode map e442 with expected crc 2023-09-23T23:27:54.430 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:27:54.430 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:27:54.515 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:27:54.547 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:54 smithi138 ceph-mon[193606]: osdmap e442: 8 total, 8 up, 7 in 2023-09-23T23:27:54.547 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:54 smithi138 ceph-mon[193606]: failed to encode map e442 with expected crc 2023-09-23T23:27:54.547 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:54 smithi138 ceph-mon[193606]: failed to encode map e442 with expected crc 2023-09-23T23:27:54.548 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:54 smithi138 ceph-mon[193606]: failed to encode map e442 with expected crc 2023-09-23T23:27:54.548 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:54 smithi138 ceph-mon[193606]: failed to encode map e442 with expected crc 2023-09-23T23:27:54.552 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:54.826 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:27:54.857 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:27:54.857 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:27:54.863 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:27:54.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:54.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:54.969 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:27:55.002 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:55.055 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:27:55.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:55.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:27:55.160 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:27:55.199 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:55.265 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:27:55.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:55.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:27:55.368 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:55.394 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:55.485 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:55 smithi110 ceph-mon[221102]: pgmap v1243: 129 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 118 active+clean; 944 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 5797/45369 objects misplaced (12.777%); 10 MiB/s, 208 objects/s recovering 2023-09-23T23:27:55.485 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:55 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:27:55.485 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:55 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:27:55.486 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:55 smithi110 ceph-mon[212939]: pgmap v1243: 129 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 118 active+clean; 944 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 5797/45369 objects misplaced (12.777%); 10 MiB/s, 208 objects/s recovering 2023-09-23T23:27:55.486 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:55 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:27:55.486 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:55 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:27:55.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:55 smithi138 ceph-mon[193606]: pgmap v1243: 129 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 118 active+clean; 944 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 5797/45369 objects misplaced (12.777%); 10 MiB/s, 208 objects/s recovering 2023-09-23T23:27:55.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:55 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:27:55.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:55 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:27:55.815 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:55.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:55.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:55.890 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:27:55.918 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:27:55.952 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:56.234 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:27:56.294 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:27:56.628 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:56 smithi138 ceph-mon[193606]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.628 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:56 smithi138 ceph-mon[193606]: osdmap e443: 8 total, 8 up, 7 in 2023-09-23T23:27:56.628 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:56 smithi138 ceph-mon[193606]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.628 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:56 smithi138 ceph-mon[193606]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.628 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:56 smithi138 ceph-mon[193606]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.628 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:56 smithi138 ceph-mon[193606]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.629 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:56 smithi138 ceph-mon[193606]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.629 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:56 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/98194070' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:27:56.629 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:56 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:27:56.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[221102]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[221102]: osdmap e443: 8 total, 8 up, 7 in 2023-09-23T23:27:56.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[221102]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[221102]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[221102]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[221102]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[221102]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/98194070' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:27:56.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:27:56.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[212939]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[212939]: osdmap e443: 8 total, 8 up, 7 in 2023-09-23T23:27:56.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[212939]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[212939]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[212939]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[212939]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[212939]: failed to encode map e443 with expected crc 2023-09-23T23:27:56.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/98194070' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:27:56.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:56 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:27:56.754 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:27:56.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:56.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:27:56.860 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:27:56.892 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:57.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:27:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:27:56] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.43.0" 2023-09-23T23:27:57.650 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:57 smithi138 ceph-mon[193606]: pgmap v1245: 129 pgs: 3 active+remapped+backfilling, 8 active+remapped+backfill_wait, 118 active+clean; 827 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 3834/39744 objects misplaced (9.647%); 22 MiB/s, 349 objects/s recovering 2023-09-23T23:27:57.650 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:57 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:27:57.651 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:57 smithi138 ceph-mon[193606]: failed to encode map e444 with expected crc 2023-09-23T23:27:57.651 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:57 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:27:57.651 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:57 smithi138 ceph-mon[193606]: osdmap e444: 8 total, 8 up, 7 in 2023-09-23T23:27:57.651 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:57 smithi138 ceph-mon[193606]: failed to encode map e444 with expected crc 2023-09-23T23:27:57.651 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:57 smithi138 ceph-mon[193606]: failed to encode map e444 with expected crc 2023-09-23T23:27:57.651 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:57 smithi138 ceph-mon[193606]: failed to encode map e444 with expected crc 2023-09-23T23:27:57.654 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:27:57.660 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:57 smithi110 ceph-mon[221102]: pgmap v1245: 129 pgs: 3 active+remapped+backfilling, 8 active+remapped+backfill_wait, 118 active+clean; 827 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 3834/39744 objects misplaced (9.647%); 22 MiB/s, 349 objects/s recovering 2023-09-23T23:27:57.660 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:57 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:27:57.660 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:57 smithi110 ceph-mon[221102]: failed to encode map e444 with expected crc 2023-09-23T23:27:57.660 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:57 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:27:57.660 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:57 smithi110 ceph-mon[221102]: osdmap e444: 8 total, 8 up, 7 in 2023-09-23T23:27:57.660 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:57 smithi110 ceph-mon[221102]: failed to encode map e444 with expected crc 2023-09-23T23:27:57.661 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:57 smithi110 ceph-mon[221102]: failed to encode map e444 with expected crc 2023-09-23T23:27:57.661 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:57 smithi110 ceph-mon[221102]: failed to encode map e444 with expected crc 2023-09-23T23:27:57.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:57 smithi110 ceph-mon[212939]: pgmap v1245: 129 pgs: 3 active+remapped+backfilling, 8 active+remapped+backfill_wait, 118 active+clean; 827 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 3834/39744 objects misplaced (9.647%); 22 MiB/s, 349 objects/s recovering 2023-09-23T23:27:57.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:57 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:27:57.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:57 smithi110 ceph-mon[212939]: failed to encode map e444 with expected crc 2023-09-23T23:27:57.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:57 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:27:57.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:57 smithi110 ceph-mon[212939]: osdmap e444: 8 total, 8 up, 7 in 2023-09-23T23:27:57.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:57 smithi110 ceph-mon[212939]: failed to encode map e444 with expected crc 2023-09-23T23:27:57.663 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:57 smithi110 ceph-mon[212939]: failed to encode map e444 with expected crc 2023-09-23T23:27:57.663 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:57 smithi110 ceph-mon[212939]: failed to encode map e444 with expected crc 2023-09-23T23:27:57.892 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:27:57.895 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:27:57.895 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:27:57.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:57.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:27:58.002 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:27:58.051 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:58.269 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:27:58.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:58.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:27:58.374 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:27:58.403 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:58.473 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:27:58.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:58.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:27:58.577 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:58.604 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:58 smithi138 ceph-mon[193606]: 11.12 scrub starts 2023-09-23T23:27:58.604 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:58 smithi138 ceph-mon[193606]: failed to encode map e445 with expected crc 2023-09-23T23:27:58.604 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:58 smithi138 ceph-mon[193606]: osdmap e445: 8 total, 8 up, 7 in 2023-09-23T23:27:58.604 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:58 smithi138 ceph-mon[193606]: failed to encode map e445 with expected crc 2023-09-23T23:27:58.604 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:58 smithi138 ceph-mon[193606]: failed to encode map e445 with expected crc 2023-09-23T23:27:58.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:58 smithi138 ceph-mon[193606]: failed to encode map e445 with expected crc 2023-09-23T23:27:58.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:58 smithi138 ceph-mon[193606]: failed to encode map e445 with expected crc 2023-09-23T23:27:58.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:58 smithi138 ceph-mon[193606]: 11.5 scrub starts 2023-09-23T23:27:58.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:27:58.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:27:58.607 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:58.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[212939]: 11.12 scrub starts 2023-09-23T23:27:58.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[212939]: failed to encode map e445 with expected crc 2023-09-23T23:27:58.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[212939]: osdmap e445: 8 total, 8 up, 7 in 2023-09-23T23:27:58.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[212939]: failed to encode map e445 with expected crc 2023-09-23T23:27:58.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[212939]: failed to encode map e445 with expected crc 2023-09-23T23:27:58.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[212939]: failed to encode map e445 with expected crc 2023-09-23T23:27:58.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[212939]: failed to encode map e445 with expected crc 2023-09-23T23:27:58.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[212939]: 11.5 scrub starts 2023-09-23T23:27:58.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:27:58.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:27:58.707 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:27:58.713 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[221102]: 11.12 scrub starts 2023-09-23T23:27:58.713 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[221102]: failed to encode map e445 with expected crc 2023-09-23T23:27:58.713 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[221102]: osdmap e445: 8 total, 8 up, 7 in 2023-09-23T23:27:58.713 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[221102]: failed to encode map e445 with expected crc 2023-09-23T23:27:58.713 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[221102]: failed to encode map e445 with expected crc 2023-09-23T23:27:58.713 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[221102]: failed to encode map e445 with expected crc 2023-09-23T23:27:58.714 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[221102]: failed to encode map e445 with expected crc 2023-09-23T23:27:58.714 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[221102]: 11.5 scrub starts 2023-09-23T23:27:58.714 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:27:58.714 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:27:58.739 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:58.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:58.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:58.757 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:27:58.768 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:27:58.792 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:27:58.835 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:27:58.850 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:27:58.869 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:27:58.885 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:58.900 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:27:58.933 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:27:58.941 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:27:58.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:58.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:27:58.965 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:27:58.984 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:27:58.984 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:27:58.984 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:27:58.984 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:27:58.984 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:27:58.984 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:27:58.984 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:27:58.984 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:27:59.050 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:27:59.076 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:59.123 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:27:59.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:59.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:27:59.201 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:27:59.229 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:27:59.256 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:59.352 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:27:59.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:27:59.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:27:59.458 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:27:59.530 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:27:59.618 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:59 smithi138 ceph-mon[193606]: pgmap v1248: 129 pgs: 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 119 active+clean; 797 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 3793/38295 objects misplaced (9.905%); 15 MiB/s, 243 objects/s recovering 2023-09-23T23:27:59.619 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:59 smithi138 ceph-mon[193606]: 11.2 scrub starts 2023-09-23T23:27:59.619 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:59 smithi138 ceph-mon[193606]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.619 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:59 smithi138 ceph-mon[193606]: osdmap e446: 8 total, 8 up, 7 in 2023-09-23T23:27:59.619 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:59 smithi138 ceph-mon[193606]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.619 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:59 smithi138 ceph-mon[193606]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.619 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:59 smithi138 ceph-mon[193606]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.619 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:59 smithi138 ceph-mon[193606]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.619 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:59 smithi138 ceph-mon[193606]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.619 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:59 smithi138 ceph-mon[193606]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.619 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:59 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2075293851' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:27:59.620 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:27:59 smithi138 ceph-mon[193606]: 1.e scrub starts 2023-09-23T23:27:59.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[212939]: pgmap v1248: 129 pgs: 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 119 active+clean; 797 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 3793/38295 objects misplaced (9.905%); 15 MiB/s, 243 objects/s recovering 2023-09-23T23:27:59.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[212939]: 11.2 scrub starts 2023-09-23T23:27:59.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[212939]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[212939]: osdmap e446: 8 total, 8 up, 7 in 2023-09-23T23:27:59.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[212939]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[212939]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[212939]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[212939]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[212939]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[212939]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2075293851' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:27:59.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[212939]: 1.e scrub starts 2023-09-23T23:27:59.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[221102]: pgmap v1248: 129 pgs: 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 119 active+clean; 797 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 3793/38295 objects misplaced (9.905%); 15 MiB/s, 243 objects/s recovering 2023-09-23T23:27:59.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[221102]: 11.2 scrub starts 2023-09-23T23:27:59.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[221102]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[221102]: osdmap e446: 8 total, 8 up, 7 in 2023-09-23T23:27:59.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[221102]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[221102]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[221102]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[221102]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[221102]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[221102]: failed to encode map e446 with expected crc 2023-09-23T23:27:59.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2075293851' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:27:59.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:27:59 smithi110 ceph-mon[221102]: 1.e scrub starts 2023-09-23T23:27:59.969 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:28:00.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:00 smithi110 ceph-mon[212939]: osdmap e447: 8 total, 8 up, 7 in 2023-09-23T23:28:00.307 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:00 smithi110 ceph-mon[212939]: failed to encode map e447 with expected crc 2023-09-23T23:28:00.307 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:00 smithi110 ceph-mon[212939]: failed to encode map e447 with expected crc 2023-09-23T23:28:00.307 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:00 smithi110 ceph-mon[212939]: failed to encode map e447 with expected crc 2023-09-23T23:28:00.307 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:00 smithi110 ceph-mon[212939]: failed to encode map e447 with expected crc 2023-09-23T23:28:00.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:00 smithi138 ceph-mon[193606]: osdmap e447: 8 total, 8 up, 7 in 2023-09-23T23:28:00.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:00 smithi138 ceph-mon[193606]: failed to encode map e447 with expected crc 2023-09-23T23:28:00.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:00 smithi138 ceph-mon[193606]: failed to encode map e447 with expected crc 2023-09-23T23:28:00.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:00 smithi138 ceph-mon[193606]: failed to encode map e447 with expected crc 2023-09-23T23:28:00.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:00 smithi138 ceph-mon[193606]: failed to encode map e447 with expected crc 2023-09-23T23:28:00.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:00 smithi110 ceph-mon[221102]: osdmap e447: 8 total, 8 up, 7 in 2023-09-23T23:28:00.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:00 smithi110 ceph-mon[221102]: failed to encode map e447 with expected crc 2023-09-23T23:28:00.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:00 smithi110 ceph-mon[221102]: failed to encode map e447 with expected crc 2023-09-23T23:28:00.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:00 smithi110 ceph-mon[221102]: failed to encode map e447 with expected crc 2023-09-23T23:28:00.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:00 smithi110 ceph-mon[221102]: failed to encode map e447 with expected crc 2023-09-23T23:28:01.045 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:28:01.045 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:28:01.046 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:28:01.046 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:28:01.046 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.046 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135076 11.3 11:c525619a:::benchmark_data_smithi110_433746_object29196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-09-23T23:28:01.046 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:09.525071+0000", 2023-09-23T23:28:01.046 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 111.516134249, 2023-09-23T23:28:01.046 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.024135834, 2023-09-23T23:28:01.046 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.046 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.046 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:01.046 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:01.047 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:01.047 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135076 2023-09-23T23:28:01.047 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.047 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.047 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.047 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.047 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.525071+0000", 2023-09-23T23:28:01.047 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.047 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.047 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.047 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.047 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.525071+0000", 2023-09-23T23:28:01.048 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.048 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.048 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.048 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.048 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.525069+0000", 2023-09-23T23:28:01.048 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-09-23T23:28:01.048 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.048 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.048 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.048 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.525078+0000", 2023-09-23T23:28:01.048 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8079999999999994e-06 2023-09-23T23:28:01.048 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.049 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.049 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.049 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.525098+0000", 2023-09-23T23:28:01.049 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9772e-05 2023-09-23T23:28:01.049 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.049 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.049 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.049 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.525105+0000", 2023-09-23T23:28:01.049 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2300000000000002e-06 2023-09-23T23:28:01.049 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.049 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.049 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.049 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.525211+0000", 2023-09-23T23:28:01.050 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105951 2023-09-23T23:28:01.050 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.050 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:01.050 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.525222+0000", 2023-09-23T23:28:01.050 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1214e-05 2023-09-23T23:28:01.050 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.050 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.050 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.530725+0000", 2023-09-23T23:28:01.050 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0055029 2023-09-23T23:28:01.050 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.051 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.051 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.051 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.530820+0000", 2023-09-23T23:28:01.051 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4856000000000001e-05 2023-09-23T23:28:01.051 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.051 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.051 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:28:01.051 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.530896+0000", 2023-09-23T23:28:01.051 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6354999999999997e-05 2023-09-23T23:28:01.051 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.051 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.051 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:01.052 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.536783+0000", 2023-09-23T23:28:01.052 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0058871749999999997 2023-09-23T23:28:01.052 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.052 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.052 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.052 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549066+0000", 2023-09-23T23:28:01.052 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012282288000000001 2023-09-23T23:28:01.052 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.052 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.052 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.052 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549108+0000", 2023-09-23T23:28:01.052 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1763e-05 2023-09-23T23:28:01.052 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.053 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.053 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.053 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549165+0000", 2023-09-23T23:28:01.053 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7158999999999999e-05 2023-09-23T23:28:01.053 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.053 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.053 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.053 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549207+0000", 2023-09-23T23:28:01.053 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2582e-05 2023-09-23T23:28:01.053 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.053 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.053 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.054 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.054 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.054 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135153 11.3 11:c4362d1b:::benchmark_data_smithi110_433746_object29273:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T23:28:01.054 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:09.620481+0000", 2023-09-23T23:28:01.054 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 111.420724613, 2023-09-23T23:28:01.054 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92919842699999999, 2023-09-23T23:28:01.054 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.054 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.054 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:01.054 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:01.054 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:01.054 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135153 2023-09-23T23:28:01.055 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.055 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.055 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.055 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.055 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.620481+0000", 2023-09-23T23:28:01.055 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.055 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.055 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.055 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.055 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.620481+0000", 2023-09-23T23:28:01.055 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.055 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.055 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.620480+0000", 2023-09-23T23:28:01.056 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:01.056 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.056 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.620487+0000", 2023-09-23T23:28:01.056 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5789999999999998e-06 2023-09-23T23:28:01.056 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.056 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.620506+0000", 2023-09-23T23:28:01.056 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8598999999999999e-05 2023-09-23T23:28:01.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.057 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.620512+0000", 2023-09-23T23:28:01.057 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5420000000000002e-06 2023-09-23T23:28:01.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.057 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.620551+0000", 2023-09-23T23:28:01.057 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8373e-05 2023-09-23T23:28:01.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.058 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:01.058 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.620558+0000", 2023-09-23T23:28:01.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5290000000000003e-06 2023-09-23T23:28:01.058 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.058 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.058 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.058 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.532287+0000", 2023-09-23T23:28:01.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91172838499999997 2023-09-23T23:28:01.058 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.058 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.058 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.058 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.532414+0000", 2023-09-23T23:28:01.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000127589 2023-09-23T23:28:01.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.059 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:28:01.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.532547+0000", 2023-09-23T23:28:01.059 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013264700000000001 2023-09-23T23:28:01.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.059 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:01.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.536792+0000", 2023-09-23T23:28:01.059 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0042455009999999996 2023-09-23T23:28:01.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.060 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549241+0000", 2023-09-23T23:28:01.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012449030999999999 2023-09-23T23:28:01.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.060 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.060 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549595+0000", 2023-09-23T23:28:01.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00035407899999999998 2023-09-23T23:28:01.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.060 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.060 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549643+0000", 2023-09-23T23:28:01.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7256000000000003e-05 2023-09-23T23:28:01.061 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.061 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549679+0000", 2023-09-23T23:28:01.061 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6837e-05 2023-09-23T23:28:01.061 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.061 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.061 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.061 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.061 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135186 11.3 11:c5ed0a82:::benchmark_data_smithi110_433746_object29306:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T23:28:01.061 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:09.653395+0000", 2023-09-23T23:28:01.062 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 111.387810441, 2023-09-23T23:28:01.062 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.89645744299999996, 2023-09-23T23:28:01.062 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.062 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.062 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:01.062 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:01.062 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:01.062 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135186 2023-09-23T23:28:01.062 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.062 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.062 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.062 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.062 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.653395+0000", 2023-09-23T23:28:01.063 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.063 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.063 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.063 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.063 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.653395+0000", 2023-09-23T23:28:01.063 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.063 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.063 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.063 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.063 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.653394+0000", 2023-09-23T23:28:01.063 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:01.063 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.064 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.064 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.653400+0000", 2023-09-23T23:28:01.064 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5250000000000002e-06 2023-09-23T23:28:01.064 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.064 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.064 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.653417+0000", 2023-09-23T23:28:01.064 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6078000000000001e-05 2023-09-23T23:28:01.064 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.064 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.064 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.653424+0000", 2023-09-23T23:28:01.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2740000000000002e-06 2023-09-23T23:28:01.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.065 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.065 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.653474+0000", 2023-09-23T23:28:01.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0639000000000001e-05 2023-09-23T23:28:01.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.065 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:01.065 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.653483+0000", 2023-09-23T23:28:01.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4430000000000003e-06 2023-09-23T23:28:01.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.533697+0000", 2023-09-23T23:28:01.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88021376699999998 2023-09-23T23:28:01.066 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.066 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.533813+0000", 2023-09-23T23:28:01.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000116264 2023-09-23T23:28:01.066 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.066 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:28:01.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.533929+0000", 2023-09-23T23:28:01.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000116486 2023-09-23T23:28:01.067 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.067 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.067 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:01.067 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.539237+0000", 2023-09-23T23:28:01.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0053076080000000001 2023-09-23T23:28:01.067 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.067 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.067 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.067 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549280+0000", 2023-09-23T23:28:01.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010042766999999999 2023-09-23T23:28:01.067 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.067 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.068 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549774+0000", 2023-09-23T23:28:01.068 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00049376300000000004 2023-09-23T23:28:01.068 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.068 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549819+0000", 2023-09-23T23:28:01.068 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5491999999999997e-05 2023-09-23T23:28:01.068 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.068 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549853+0000", 2023-09-23T23:28:01.068 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3646999999999998e-05 2023-09-23T23:28:01.069 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.069 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.069 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.069 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.069 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135202 11.3 11:c28fa593:::benchmark_data_smithi110_433746_object29322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T23:28:01.069 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:09.672755+0000", 2023-09-23T23:28:01.069 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 111.368450982, 2023-09-23T23:28:01.069 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.87717449400000003, 2023-09-23T23:28:01.069 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.069 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.069 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:01.069 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:01.070 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:01.070 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135202 2023-09-23T23:28:01.070 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.070 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.070 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.070 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.672755+0000", 2023-09-23T23:28:01.070 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.070 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.070 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.070 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.672755+0000", 2023-09-23T23:28:01.071 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.071 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.071 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.071 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.071 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.672753+0000", 2023-09-23T23:28:01.071 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:01.071 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.071 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.071 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.071 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.672761+0000", 2023-09-23T23:28:01.071 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8259999999999993e-06 2023-09-23T23:28:01.071 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.072 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.072 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.072 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.672777+0000", 2023-09-23T23:28:01.072 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6232999999999999e-05 2023-09-23T23:28:01.072 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.072 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.072 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.072 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.672785+0000", 2023-09-23T23:28:01.072 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4200000000000001e-06 2023-09-23T23:28:01.072 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.072 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.072 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.672826+0000", 2023-09-23T23:28:01.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1666999999999999e-05 2023-09-23T23:28:01.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.073 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.073 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:01.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.672835+0000", 2023-09-23T23:28:01.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3420000000000008e-06 2023-09-23T23:28:01.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.073 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.073 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.534793+0000", 2023-09-23T23:28:01.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.86195810299999998 2023-09-23T23:28:01.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.074 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.074 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.074 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.534873+0000", 2023-09-23T23:28:01.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0760000000000006e-05 2023-09-23T23:28:01.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.074 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.074 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:28:01.074 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.534941+0000", 2023-09-23T23:28:01.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7339e-05 2023-09-23T23:28:01.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.075 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.075 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:01.075 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.539246+0000", 2023-09-23T23:28:01.075 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043050290000000001 2023-09-23T23:28:01.075 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.075 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.075 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.075 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549296+0000", 2023-09-23T23:28:01.075 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010050534 2023-09-23T23:28:01.075 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.075 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.076 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.076 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549877+0000", 2023-09-23T23:28:01.076 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00058099499999999995 2023-09-23T23:28:01.076 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.076 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.076 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.076 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549906+0000", 2023-09-23T23:28:01.076 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8572999999999999e-05 2023-09-23T23:28:01.076 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.076 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.076 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.076 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549929+0000", 2023-09-23T23:28:01.076 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3247e-05 2023-09-23T23:28:01.077 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.077 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.077 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.077 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.077 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.077 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135230 11.3 11:c6d562a7:::benchmark_data_smithi110_433746_object29350:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T23:28:01.077 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:09.706365+0000", 2023-09-23T23:28:01.077 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 111.334841062, 2023-09-23T23:28:01.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.84368306900000001, 2023-09-23T23:28:01.077 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.077 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.077 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:01.078 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:01.078 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:01.078 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135230 2023-09-23T23:28:01.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.078 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.078 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.706365+0000", 2023-09-23T23:28:01.078 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.078 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.706365+0000", 2023-09-23T23:28:01.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.079 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.079 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.706363+0000", 2023-09-23T23:28:01.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:01.079 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.079 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.706370+0000", 2023-09-23T23:28:01.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.618e-06 2023-09-23T23:28:01.079 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.080 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.080 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.706383+0000", 2023-09-23T23:28:01.080 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3396e-05 2023-09-23T23:28:01.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.080 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.080 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.706389+0000", 2023-09-23T23:28:01.080 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6250000000000004e-06 2023-09-23T23:28:01.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.080 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.081 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.706445+0000", 2023-09-23T23:28:01.081 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6425000000000003e-05 2023-09-23T23:28:01.081 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.081 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.081 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:01.081 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.706455+0000", 2023-09-23T23:28:01.081 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7410000000000004e-06 2023-09-23T23:28:01.081 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.081 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.081 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.081 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.535774+0000", 2023-09-23T23:28:01.081 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.82931910099999995 2023-09-23T23:28:01.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.082 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.082 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.535885+0000", 2023-09-23T23:28:01.082 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011102100000000001 2023-09-23T23:28:01.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.082 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:28:01.082 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.536002+0000", 2023-09-23T23:28:01.082 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000116526 2023-09-23T23:28:01.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.083 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:01.083 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.541431+0000", 2023-09-23T23:28:01.083 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0054289020000000002 2023-09-23T23:28:01.083 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.083 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.083 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.083 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549394+0000", 2023-09-23T23:28:01.083 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0079628939999999999 2023-09-23T23:28:01.083 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.083 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.083 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.083 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549969+0000", 2023-09-23T23:28:01.084 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00057510899999999997 2023-09-23T23:28:01.084 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.084 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.084 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.084 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550012+0000", 2023-09-23T23:28:01.084 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2944000000000001e-05 2023-09-23T23:28:01.084 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.084 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.084 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.084 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550048+0000", 2023-09-23T23:28:01.084 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5985999999999997e-05 2023-09-23T23:28:01.084 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.084 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.085 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.085 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.085 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.085 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135279 11.3 11:c347edb6:::benchmark_data_smithi110_433746_object29399:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T23:28:01.085 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:09.771819+0000", 2023-09-23T23:28:01.085 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 111.269386758, 2023-09-23T23:28:01.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.77832863900000004, 2023-09-23T23:28:01.085 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.085 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.085 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:01.085 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:01.085 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:01.085 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135279 2023-09-23T23:28:01.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.086 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.086 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.086 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.771819+0000", 2023-09-23T23:28:01.086 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.086 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.086 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.771819+0000", 2023-09-23T23:28:01.086 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.771817+0000", 2023-09-23T23:28:01.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:28:01.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.771827+0000", 2023-09-23T23:28:01.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0224000000000001e-05 2023-09-23T23:28:01.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.771851+0000", 2023-09-23T23:28:01.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3907999999999999e-05 2023-09-23T23:28:01.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.771859+0000", 2023-09-23T23:28:01.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8790000000000003e-06 2023-09-23T23:28:01.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.771925+0000", 2023-09-23T23:28:01.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6264000000000001e-05 2023-09-23T23:28:01.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.089 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:01.089 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.771938+0000", 2023-09-23T23:28:01.089 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2835e-05 2023-09-23T23:28:01.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.089 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.089 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.089 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.536837+0000", 2023-09-23T23:28:01.089 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.76489877799999995 2023-09-23T23:28:01.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.089 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.089 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.090 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.536921+0000", 2023-09-23T23:28:01.090 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4248999999999999e-05 2023-09-23T23:28:01.090 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.090 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.090 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:28:01.090 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.536985+0000", 2023-09-23T23:28:01.090 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4088e-05 2023-09-23T23:28:01.090 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.090 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.090 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:01.090 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.541439+0000", 2023-09-23T23:28:01.090 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004453589 2023-09-23T23:28:01.091 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.091 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.091 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.091 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549438+0000", 2023-09-23T23:28:01.091 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0079992680000000004 2023-09-23T23:28:01.091 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.091 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.091 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.091 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550083+0000", 2023-09-23T23:28:01.091 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00064511799999999997 2023-09-23T23:28:01.091 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.091 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.091 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.092 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550110+0000", 2023-09-23T23:28:01.092 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6339e-05 2023-09-23T23:28:01.092 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.092 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.092 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.092 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550148+0000", 2023-09-23T23:28:01.092 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7922e-05 2023-09-23T23:28:01.092 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.092 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.092 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.092 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.092 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.093 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135286 11.3 11:c25c5609:::benchmark_data_smithi110_433746_object29406:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T23:28:01.093 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:09.782594+0000", 2023-09-23T23:28:01.093 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 111.25861170100001, 2023-09-23T23:28:01.093 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.76765006499999999, 2023-09-23T23:28:01.093 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.093 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.093 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:01.093 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:01.093 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:01.093 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135286 2023-09-23T23:28:01.093 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.093 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.094 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.094 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.094 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.782594+0000", 2023-09-23T23:28:01.094 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.094 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.094 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.094 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.094 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.782594+0000", 2023-09-23T23:28:01.094 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.094 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.094 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.095 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.095 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.782592+0000", 2023-09-23T23:28:01.095 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:28:01.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.095 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.095 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.095 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.782600+0000", 2023-09-23T23:28:01.095 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2090000000000005e-06 2023-09-23T23:28:01.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.095 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.095 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.095 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.782621+0000", 2023-09-23T23:28:01.096 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0112999999999999e-05 2023-09-23T23:28:01.096 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.096 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.096 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.782626+0000", 2023-09-23T23:28:01.096 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9449999999999998e-06 2023-09-23T23:28:01.096 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.096 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.096 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.782685+0000", 2023-09-23T23:28:01.096 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8786999999999998e-05 2023-09-23T23:28:01.096 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.097 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.097 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:01.097 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.782695+0000", 2023-09-23T23:28:01.097 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7599999999999997e-06 2023-09-23T23:28:01.097 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.097 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.097 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.097 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.537843+0000", 2023-09-23T23:28:01.097 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.75514778599999999 2023-09-23T23:28:01.097 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.097 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.097 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.097 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.537956+0000", 2023-09-23T23:28:01.098 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011293700000000001 2023-09-23T23:28:01.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.098 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.098 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:28:01.098 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.538072+0000", 2023-09-23T23:28:01.098 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000116073 2023-09-23T23:28:01.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.098 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.098 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:01.098 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.543500+0000", 2023-09-23T23:28:01.098 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0054284140000000003 2023-09-23T23:28:01.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.099 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.099 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.099 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549477+0000", 2023-09-23T23:28:01.099 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005976945 2023-09-23T23:28:01.099 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.099 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.099 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.099 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550176+0000", 2023-09-23T23:28:01.099 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00069912100000000003 2023-09-23T23:28:01.099 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.099 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.099 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.100 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550210+0000", 2023-09-23T23:28:01.100 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3250999999999997e-05 2023-09-23T23:28:01.100 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.100 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.100 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.100 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550244+0000", 2023-09-23T23:28:01.100 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4465000000000003e-05 2023-09-23T23:28:01.100 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.100 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.100 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.100 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.100 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.100 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135328 11.3 11:c5dfc486:::benchmark_data_smithi110_433746_object29448:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T23:28:01.101 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:09.834223+0000", 2023-09-23T23:28:01.101 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 111.206982729, 2023-09-23T23:28:01.101 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.71616003399999995, 2023-09-23T23:28:01.101 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.101 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.101 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:01.101 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:01.101 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:01.101 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135328 2023-09-23T23:28:01.101 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.101 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.101 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.101 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.102 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.834223+0000", 2023-09-23T23:28:01.102 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.102 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.102 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.102 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.102 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.834223+0000", 2023-09-23T23:28:01.102 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.102 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.102 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.102 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.102 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.834222+0000", 2023-09-23T23:28:01.102 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:01.103 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.103 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.103 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.103 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.834228+0000", 2023-09-23T23:28:01.103 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5230000000000001e-06 2023-09-23T23:28:01.103 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.103 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.103 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.103 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.834243+0000", 2023-09-23T23:28:01.103 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4831000000000001e-05 2023-09-23T23:28:01.103 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.103 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.103 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.834249+0000", 2023-09-23T23:28:01.104 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.528e-06 2023-09-23T23:28:01.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.104 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.104 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.834289+0000", 2023-09-23T23:28:01.104 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9616999999999997e-05 2023-09-23T23:28:01.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.104 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.104 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:01.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.834295+0000", 2023-09-23T23:28:01.104 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0290000000000002e-06 2023-09-23T23:28:01.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.105 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.105 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.539275+0000", 2023-09-23T23:28:01.105 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.70497945699999998 2023-09-23T23:28:01.105 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.105 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.105 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.539343+0000", 2023-09-23T23:28:01.105 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8753999999999999e-05 2023-09-23T23:28:01.105 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.105 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:28:01.106 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.539409+0000", 2023-09-23T23:28:01.106 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5784999999999994e-05 2023-09-23T23:28:01.106 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.106 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.106 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:01.106 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.543507+0000", 2023-09-23T23:28:01.106 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004097781 2023-09-23T23:28:01.106 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.106 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.106 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.106 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549551+0000", 2023-09-23T23:28:01.106 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0060443800000000002 2023-09-23T23:28:01.106 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.107 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.107 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.107 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550300+0000", 2023-09-23T23:28:01.107 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00074844600000000005 2023-09-23T23:28:01.107 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.107 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.107 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.107 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550348+0000", 2023-09-23T23:28:01.107 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7809000000000003e-05 2023-09-23T23:28:01.107 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.107 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.107 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.107 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550383+0000", 2023-09-23T23:28:01.108 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5454999999999999e-05 2023-09-23T23:28:01.108 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.108 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.108 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.108 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.108 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.108 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135336 11.3 11:c3c09805:::benchmark_data_smithi110_433746_object29456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T23:28:01.108 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:09.843819+0000", 2023-09-23T23:28:01.108 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 111.19738705899999, 2023-09-23T23:28:01.108 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.70670411099999997, 2023-09-23T23:28:01.108 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.108 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.109 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:01.109 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:01.109 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:01.109 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135336 2023-09-23T23:28:01.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.109 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.109 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.109 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.843819+0000", 2023-09-23T23:28:01.109 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.109 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.110 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.843819+0000", 2023-09-23T23:28:01.110 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.110 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.110 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.110 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.110 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.843817+0000", 2023-09-23T23:28:01.110 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:01.110 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.110 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.110 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.110 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.843825+0000", 2023-09-23T23:28:01.110 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1610000000000003e-06 2023-09-23T23:28:01.110 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.111 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.843844+0000", 2023-09-23T23:28:01.111 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8227999999999999e-05 2023-09-23T23:28:01.111 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.111 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.843850+0000", 2023-09-23T23:28:01.111 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.866e-06 2023-09-23T23:28:01.111 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.843900+0000", 2023-09-23T23:28:01.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9592999999999998e-05 2023-09-23T23:28:01.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:01.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.843908+0000", 2023-09-23T23:28:01.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9370000000000004e-06 2023-09-23T23:28:01.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.540299+0000", 2023-09-23T23:28:01.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.69639090199999998 2023-09-23T23:28:01.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.540414+0000", 2023-09-23T23:28:01.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000114874 2023-09-23T23:28:01.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:28:01.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.540536+0000", 2023-09-23T23:28:01.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012199199999999999 2023-09-23T23:28:01.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:01.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.545544+0000", 2023-09-23T23:28:01.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005008111 2023-09-23T23:28:01.114 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.114 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.114 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.549731+0000", 2023-09-23T23:28:01.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041870179999999998 2023-09-23T23:28:01.114 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.114 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.114 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550444+0000", 2023-09-23T23:28:01.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00071336999999999995 2023-09-23T23:28:01.114 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.115 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.115 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550485+0000", 2023-09-23T23:28:01.115 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0818000000000001e-05 2023-09-23T23:28:01.115 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.115 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.115 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.115 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550523+0000", 2023-09-23T23:28:01.115 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7629000000000002e-05 2023-09-23T23:28:01.115 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.115 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.115 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.115 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.115 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.116 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135337 11.3 11:c24713d2:::benchmark_data_smithi110_433746_object29457:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T23:28:01.116 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:09.844915+0000", 2023-09-23T23:28:01.116 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 111.196290862, 2023-09-23T23:28:01.116 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.70571332399999998, 2023-09-23T23:28:01.116 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.116 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.116 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:01.116 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:01.117 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:01.117 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135337 2023-09-23T23:28:01.117 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.117 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.117 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.117 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.117 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.844915+0000", 2023-09-23T23:28:01.117 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.117 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.117 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.117 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.117 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.844915+0000", 2023-09-23T23:28:01.117 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.118 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.118 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.118 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.118 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.844914+0000", 2023-09-23T23:28:01.118 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:01.118 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.118 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.118 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.118 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.844918+0000", 2023-09-23T23:28:01.118 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8089999999999999e-06 2023-09-23T23:28:01.118 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.118 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.118 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.119 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.844927+0000", 2023-09-23T23:28:01.119 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.935e-06 2023-09-23T23:28:01.119 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.119 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.119 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.119 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.844931+0000", 2023-09-23T23:28:01.119 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.827e-06 2023-09-23T23:28:01.119 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.119 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.119 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.119 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.844968+0000", 2023-09-23T23:28:01.119 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7039000000000002e-05 2023-09-23T23:28:01.120 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.120 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.120 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:01.120 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.844974+0000", 2023-09-23T23:28:01.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7239999999999997e-06 2023-09-23T23:28:01.120 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.120 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.120 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.120 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.541476+0000", 2023-09-23T23:28:01.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.69650237199999998 2023-09-23T23:28:01.120 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.120 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.120 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.541547+0000", 2023-09-23T23:28:01.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0704000000000006e-05 2023-09-23T23:28:01.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:28:01.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.541611+0000", 2023-09-23T23:28:01.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4686999999999999e-05 2023-09-23T23:28:01.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:01.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.545553+0000", 2023-09-23T23:28:01.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039414369999999999 2023-09-23T23:28:01.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.122 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550410+0000", 2023-09-23T23:28:01.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048575880000000004 2023-09-23T23:28:01.122 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.122 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.122 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550565+0000", 2023-09-23T23:28:01.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015446500000000001 2023-09-23T23:28:01.122 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.122 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.122 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550598+0000", 2023-09-23T23:28:01.123 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3287000000000002e-05 2023-09-23T23:28:01.123 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.123 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.123 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.123 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550628+0000", 2023-09-23T23:28:01.123 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0015e-05 2023-09-23T23:28:01.123 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.123 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.123 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.123 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.123 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.123 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135353 11.3 11:c4634112:::benchmark_data_smithi110_433746_object29473:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T23:28:01.123 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:09.868193+0000", 2023-09-23T23:28:01.124 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 111.17301232200001, 2023-09-23T23:28:01.124 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.68263527499999999, 2023-09-23T23:28:01.124 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.124 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.124 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:01.124 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:01.124 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:01.124 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135353 2023-09-23T23:28:01.124 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.124 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.124 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.124 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.124 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.868193+0000", 2023-09-23T23:28:01.125 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.125 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.125 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.125 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.125 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.868193+0000", 2023-09-23T23:28:01.125 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.125 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.125 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.125 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.125 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.868192+0000", 2023-09-23T23:28:01.125 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:28:01.125 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.126 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.126 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.126 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.868197+0000", 2023-09-23T23:28:01.126 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0359999999999997e-06 2023-09-23T23:28:01.126 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.126 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.126 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.126 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.868208+0000", 2023-09-23T23:28:01.126 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0774000000000001e-05 2023-09-23T23:28:01.126 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.126 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.126 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.126 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.868213+0000", 2023-09-23T23:28:01.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6990000000000002e-06 2023-09-23T23:28:01.127 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.127 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.127 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.127 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.868251+0000", 2023-09-23T23:28:01.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8306000000000002e-05 2023-09-23T23:28:01.127 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.127 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.127 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:01.127 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:09.868260+0000", 2023-09-23T23:28:01.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4249999999999998e-06 2023-09-23T23:28:01.127 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.127 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.128 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.542437+0000", 2023-09-23T23:28:01.128 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.67417735300000003 2023-09-23T23:28:01.128 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.128 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.128 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.542547+0000", 2023-09-23T23:28:01.128 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110018 2023-09-23T23:28:01.128 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.128 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:28:01.128 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.542665+0000", 2023-09-23T23:28:01.128 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118195 2023-09-23T23:28:01.129 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.129 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.129 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:01.129 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.547640+0000", 2023-09-23T23:28:01.129 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049750489999999996 2023-09-23T23:28:01.129 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.129 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.129 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.129 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550662+0000", 2023-09-23T23:28:01.129 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003021592 2023-09-23T23:28:01.129 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.129 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.129 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.130 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550758+0000", 2023-09-23T23:28:01.130 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6436999999999998e-05 2023-09-23T23:28:01.130 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.130 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.130 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.130 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550794+0000", 2023-09-23T23:28:01.130 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5759000000000001e-05 2023-09-23T23:28:01.130 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.130 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.130 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.130 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.550829+0000", 2023-09-23T23:28:01.130 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4607999999999998e-05 2023-09-23T23:28:01.131 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.131 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.131 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.131 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.131 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.131 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135460 11.3 11:c03612d0:::benchmark_data_smithi110_433746_object29580:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T23:28:01.131 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.060240+0000", 2023-09-23T23:28:01.131 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 110.980965613, 2023-09-23T23:28:01.131 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.49440505299999998, 2023-09-23T23:28:01.131 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.131 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.131 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:01.131 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:01.132 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:01.132 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135460 2023-09-23T23:28:01.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.132 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.132 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.132 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.132 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.060240+0000", 2023-09-23T23:28:01.132 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.132 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.132 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.132 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.060240+0000", 2023-09-23T23:28:01.132 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.133 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.133 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.133 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.060239+0000", 2023-09-23T23:28:01.133 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:01.133 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.133 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.133 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.060245+0000", 2023-09-23T23:28:01.133 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1820000000000003e-06 2023-09-23T23:28:01.133 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.133 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.134 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.060257+0000", 2023-09-23T23:28:01.134 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2429000000000001e-05 2023-09-23T23:28:01.134 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.134 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.134 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.134 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.060262+0000", 2023-09-23T23:28:01.134 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0649999999999998e-06 2023-09-23T23:28:01.134 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.134 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.134 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.134 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.060302+0000", 2023-09-23T23:28:01.134 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9495e-05 2023-09-23T23:28:01.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:01.135 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.060308+0000", 2023-09-23T23:28:01.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.297e-06 2023-09-23T23:28:01.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.135 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.543549+0000", 2023-09-23T23:28:01.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.483241268 2023-09-23T23:28:01.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.136 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.543649+0000", 2023-09-23T23:28:01.136 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9384999999999998e-05 2023-09-23T23:28:01.136 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.136 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.136 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:28:01.136 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.543720+0000", 2023-09-23T23:28:01.136 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1482000000000004e-05 2023-09-23T23:28:01.136 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.136 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.136 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:01.136 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.547647+0000", 2023-09-23T23:28:01.137 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039265250000000002 2023-09-23T23:28:01.137 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.137 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.137 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.137 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.551938+0000", 2023-09-23T23:28:01.137 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0042914709999999998 2023-09-23T23:28:01.137 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.137 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.137 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.137 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.554560+0000", 2023-09-23T23:28:01.138 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0026220179999999998 2023-09-23T23:28:01.138 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.138 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.138 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.138 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.554607+0000", 2023-09-23T23:28:01.138 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7188000000000003e-05 2023-09-23T23:28:01.138 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.138 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.138 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.138 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.554645+0000", 2023-09-23T23:28:01.138 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7737999999999997e-05 2023-09-23T23:28:01.138 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.139 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.139 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.139 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.139 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.139 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135511 11.3 11:c6db985a:::benchmark_data_smithi110_433746_object29631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T23:28:01.139 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.154862+0000", 2023-09-23T23:28:01.139 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 110.886343583, 2023-09-23T23:28:01.139 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39988463499999999, 2023-09-23T23:28:01.139 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.140 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.140 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:01.140 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:01.140 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:01.140 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135511 2023-09-23T23:28:01.140 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.140 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.140 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.140 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.140 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.154862+0000", 2023-09-23T23:28:01.140 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.141 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.141 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.141 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.141 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.154862+0000", 2023-09-23T23:28:01.141 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.141 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.141 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.141 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.141 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.154861+0000", 2023-09-23T23:28:01.141 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:01.142 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.142 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.142 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.142 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.154868+0000", 2023-09-23T23:28:01.142 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8730000000000001e-06 2023-09-23T23:28:01.142 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.142 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.142 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.142 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.154883+0000", 2023-09-23T23:28:01.142 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.509e-05 2023-09-23T23:28:01.142 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.143 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.143 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.143 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.154888+0000", 2023-09-23T23:28:01.143 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.519e-06 2023-09-23T23:28:01.143 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.143 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.143 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.143 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.154929+0000", 2023-09-23T23:28:01.143 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1297000000000001e-05 2023-09-23T23:28:01.143 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.144 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.144 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:01.144 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.154936+0000", 2023-09-23T23:28:01.144 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2450000000000003e-06 2023-09-23T23:28:01.144 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.144 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.144 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.144 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.544514+0000", 2023-09-23T23:28:01.144 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38957808199999999 2023-09-23T23:28:01.144 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.145 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.145 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.145 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.544623+0000", 2023-09-23T23:28:01.145 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108979 2023-09-23T23:28:01.145 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.145 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.145 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:28:01.145 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.544742+0000", 2023-09-23T23:28:01.145 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000119171 2023-09-23T23:28:01.145 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.145 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.146 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:01.146 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.548709+0000", 2023-09-23T23:28:01.146 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039672129999999998 2023-09-23T23:28:01.146 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.146 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.146 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.146 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.551968+0000", 2023-09-23T23:28:01.146 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003258927 2023-09-23T23:28:01.146 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.147 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.147 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.147 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.554678+0000", 2023-09-23T23:28:01.147 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0027096709999999999 2023-09-23T23:28:01.147 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.147 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.147 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.147 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.554713+0000", 2023-09-23T23:28:01.147 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5111000000000003e-05 2023-09-23T23:28:01.148 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.148 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.148 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.148 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.554747+0000", 2023-09-23T23:28:01.148 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3816999999999998e-05 2023-09-23T23:28:01.148 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.148 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.148 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.148 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.148 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.149 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:155336 11.13 11:c801a8d9:::benchmark_data_smithi110_433746_object49456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-09-23T23:28:01.149 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:44.333086+0000", 2023-09-23T23:28:01.149 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 76.708119327999995, 2023-09-23T23:28:01.149 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.092269076000000005, 2023-09-23T23:28:01.149 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.149 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.149 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:01.149 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:01.149 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:01.149 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 155336 2023-09-23T23:28:01.149 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.149 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.149 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.150 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.150 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.333086+0000", 2023-09-23T23:28:01.150 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.150 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.150 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.150 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.150 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.333086+0000", 2023-09-23T23:28:01.150 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.150 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.150 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.150 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.150 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.333085+0000", 2023-09-23T23:28:01.151 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:01.151 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.151 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.151 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.151 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.333093+0000", 2023-09-23T23:28:01.151 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7640000000000005e-06 2023-09-23T23:28:01.151 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.151 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.151 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.151 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.333109+0000", 2023-09-23T23:28:01.151 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6969999999999998e-05 2023-09-23T23:28:01.151 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.151 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.152 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.152 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.333117+0000", 2023-09-23T23:28:01.152 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.024e-06 2023-09-23T23:28:01.152 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.152 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.152 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.152 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.333172+0000", 2023-09-23T23:28:01.152 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5964000000000002e-05 2023-09-23T23:28:01.152 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.152 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.152 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.152 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.333313+0000", 2023-09-23T23:28:01.152 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000140314 2023-09-23T23:28:01.153 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.153 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.153 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:28:01.153 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.333730+0000", 2023-09-23T23:28:01.153 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00041700899999999998 2023-09-23T23:28:01.153 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.153 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.153 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.153 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.337715+0000", 2023-09-23T23:28:01.153 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039847540000000001 2023-09-23T23:28:01.153 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.153 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.154 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:01.154 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.339188+0000", 2023-09-23T23:28:01.154 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001473584 2023-09-23T23:28:01.154 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.154 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.154 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.154 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.425259+0000", 2023-09-23T23:28:01.154 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.086071022999999997 2023-09-23T23:28:01.154 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.154 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.154 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.154 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.425312+0000", 2023-09-23T23:28:01.154 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3279000000000003e-05 2023-09-23T23:28:01.155 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.155 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.155 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.155 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.425355+0000", 2023-09-23T23:28:01.155 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2959e-05 2023-09-23T23:28:01.155 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.155 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.155 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.155 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.155 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.155 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:155340 11.3 11:c79ae8c9:::benchmark_data_smithi110_433746_object49460:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-09-23T23:28:01.155 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:44.340071+0000", 2023-09-23T23:28:01.156 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 76.701134659999994, 2023-09-23T23:28:01.156 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.085400523000000006, 2023-09-23T23:28:01.156 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.156 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.156 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:01.156 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:01.156 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:01.156 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 155340 2023-09-23T23:28:01.156 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.156 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.156 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.156 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.157 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.340071+0000", 2023-09-23T23:28:01.157 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.157 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.157 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.157 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.157 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.340071+0000", 2023-09-23T23:28:01.157 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.157 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.157 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.157 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.157 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.340070+0000", 2023-09-23T23:28:01.157 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:01.157 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.158 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.158 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.158 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.340077+0000", 2023-09-23T23:28:01.158 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2609999999999996e-06 2023-09-23T23:28:01.158 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.158 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.158 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.158 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.340094+0000", 2023-09-23T23:28:01.158 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7132000000000001e-05 2023-09-23T23:28:01.158 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.158 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.158 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.159 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.340101+0000", 2023-09-23T23:28:01.159 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5209999999999997e-06 2023-09-23T23:28:01.159 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.159 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.159 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.159 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.340159+0000", 2023-09-23T23:28:01.159 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7330000000000002e-05 2023-09-23T23:28:01.159 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.159 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.159 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.159 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.340295+0000", 2023-09-23T23:28:01.159 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000135711 2023-09-23T23:28:01.159 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.160 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.160 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:28:01.160 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.340725+0000", 2023-09-23T23:28:01.160 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00043076699999999998 2023-09-23T23:28:01.160 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.160 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.160 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:01.160 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.342866+0000", 2023-09-23T23:28:01.160 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021405450000000002 2023-09-23T23:28:01.160 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.160 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.161 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.161 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.343059+0000", 2023-09-23T23:28:01.161 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019279799999999999 2023-09-23T23:28:01.161 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.161 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.161 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.161 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.425395+0000", 2023-09-23T23:28:01.161 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.082335997999999994 2023-09-23T23:28:01.161 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.161 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.161 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.161 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.425439+0000", 2023-09-23T23:28:01.161 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4109000000000001e-05 2023-09-23T23:28:01.162 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.162 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.162 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.162 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.425472+0000", 2023-09-23T23:28:01.162 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2784000000000003e-05 2023-09-23T23:28:01.162 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.162 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.162 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.162 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.162 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.162 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:155673 11.3 11:c0bdb8ad:::benchmark_data_smithi110_433746_object49793:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-09-23T23:28:01.162 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:44.786795+0000", 2023-09-23T23:28:01.163 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 76.254411073, 2023-09-23T23:28:01.163 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.057279675000000002, 2023-09-23T23:28:01.163 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.163 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.163 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:01.163 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:01.163 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:01.163 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 155673 2023-09-23T23:28:01.163 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.163 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.163 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.163 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.164 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.786795+0000", 2023-09-23T23:28:01.164 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.164 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.164 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.164 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.164 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.786795+0000", 2023-09-23T23:28:01.164 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.164 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.164 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.164 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.164 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.786794+0000", 2023-09-23T23:28:01.164 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:01.164 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.165 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.165 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.165 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.786798+0000", 2023-09-23T23:28:01.165 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3579999999999996e-06 2023-09-23T23:28:01.165 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.165 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.165 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.165 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.786810+0000", 2023-09-23T23:28:01.165 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1308000000000001e-05 2023-09-23T23:28:01.165 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.165 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.165 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.165 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.786817+0000", 2023-09-23T23:28:01.166 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9450000000000004e-06 2023-09-23T23:28:01.166 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.166 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.166 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.166 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.786891+0000", 2023-09-23T23:28:01.166 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4537999999999997e-05 2023-09-23T23:28:01.166 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.166 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.166 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.166 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.787014+0000", 2023-09-23T23:28:01.166 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000123313 2023-09-23T23:28:01.166 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.167 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.167 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:28:01.167 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.787282+0000", 2023-09-23T23:28:01.167 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000267345 2023-09-23T23:28:01.167 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.167 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.167 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:01.167 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.789789+0000", 2023-09-23T23:28:01.167 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025074950000000002 2023-09-23T23:28:01.167 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.167 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.167 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.167 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.816759+0000", 2023-09-23T23:28:01.168 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.026969559000000001 2023-09-23T23:28:01.168 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.168 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.168 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.168 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.844020+0000", 2023-09-23T23:28:01.168 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.027261128999999999 2023-09-23T23:28:01.168 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.168 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.168 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.168 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.844051+0000", 2023-09-23T23:28:01.168 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1078000000000003e-05 2023-09-23T23:28:01.168 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.168 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.169 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.169 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.844074+0000", 2023-09-23T23:28:01.169 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3286e-05 2023-09-23T23:28:01.169 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.169 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.169 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.169 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.169 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.169 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:155726 11.13 11:cdba31c2:::benchmark_data_smithi110_433746_object49846:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-09-23T23:28:01.169 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:44.882803+0000", 2023-09-23T23:28:01.169 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 76.158402422999998, 2023-09-23T23:28:01.169 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.15491022900000001, 2023-09-23T23:28:01.170 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.170 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.170 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:01.170 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:01.170 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:01.170 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 155726 2023-09-23T23:28:01.170 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.170 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.170 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.170 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.170 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.882803+0000", 2023-09-23T23:28:01.170 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.171 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.171 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.171 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.171 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.882803+0000", 2023-09-23T23:28:01.171 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.171 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.171 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.171 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.171 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.882802+0000", 2023-09-23T23:28:01.171 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:01.171 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.171 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.171 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.172 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.882809+0000", 2023-09-23T23:28:01.172 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8469999999999998e-06 2023-09-23T23:28:01.172 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.172 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.172 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.172 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.882820+0000", 2023-09-23T23:28:01.172 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1195e-05 2023-09-23T23:28:01.172 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.172 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.172 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.172 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.882826+0000", 2023-09-23T23:28:01.172 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7049999999999996e-06 2023-09-23T23:28:01.173 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.173 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.173 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.173 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.882871+0000", 2023-09-23T23:28:01.173 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5711000000000003e-05 2023-09-23T23:28:01.173 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.173 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.173 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.173 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.883015+0000", 2023-09-23T23:28:01.173 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014404199999999999 2023-09-23T23:28:01.173 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.173 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.173 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:28:01.174 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.883432+0000", 2023-09-23T23:28:01.174 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000416363 2023-09-23T23:28:01.174 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.174 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.174 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:01.174 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.885543+0000", 2023-09-23T23:28:01.174 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021111189999999998 2023-09-23T23:28:01.174 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.174 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.174 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.174 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:45.032527+0000", 2023-09-23T23:28:01.174 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.14698404400000001 2023-09-23T23:28:01.175 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.175 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.175 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:01.175 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:45.037625+0000", 2023-09-23T23:28:01.175 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050977690000000003 2023-09-23T23:28:01.175 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.175 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.175 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.175 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:45.037673+0000", 2023-09-23T23:28:01.175 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8040000000000001e-05 2023-09-23T23:28:01.175 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.175 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.175 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.176 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:45.037713+0000", 2023-09-23T23:28:01.176 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0723999999999998e-05 2023-09-23T23:28:01.176 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.176 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.176 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.176 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.176 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.176 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35904.0:192774 11.10 e438/428 11:096b4ebc:::benchmark_data_smithi110_433746_object86894:head v 438'9129, mlcod=438'9127)", 2023-09-23T23:28:01.176 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:44.715587+0000", 2023-09-23T23:28:01.176 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 16.32561828, 2023-09-23T23:28:01.176 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19084601900000001, 2023-09-23T23:28:01.176 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.177 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.177 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.177 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.177 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.177 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.715587+0000", 2023-09-23T23:28:01.177 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.177 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.177 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.177 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.177 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.715587+0000", 2023-09-23T23:28:01.177 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.177 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.177 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.178 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.178 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.715587+0000", 2023-09-23T23:28:01.178 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:01.178 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.178 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.178 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.178 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.715592+0000", 2023-09-23T23:28:01.178 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5840000000000002e-06 2023-09-23T23:28:01.178 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.178 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.178 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.178 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.715601+0000", 2023-09-23T23:28:01.179 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4789999999999996e-06 2023-09-23T23:28:01.179 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.179 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.179 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.179 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.715605+0000", 2023-09-23T23:28:01.179 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3309999999999997e-06 2023-09-23T23:28:01.179 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.179 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.179 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.179 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.715638+0000", 2023-09-23T23:28:01.179 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2509999999999999e-05 2023-09-23T23:28:01.179 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.179 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.180 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.180 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.715650+0000", 2023-09-23T23:28:01.180 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2601e-05 2023-09-23T23:28:01.180 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.180 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.180 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.180 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.906399+0000", 2023-09-23T23:28:01.180 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19074882900000001 2023-09-23T23:28:01.180 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.180 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.180 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.180 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.906433+0000", 2023-09-23T23:28:01.181 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4289000000000003e-05 2023-09-23T23:28:01.181 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.181 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.181 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.181 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.181 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.181 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(11.15 438/410 [PushOp(11:af35ca6b:::benchmark_data_smithi110_433746_object95814:head, version: 362'5984, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(11:af35ca6b:::benchmark_data_smithi110_433746_object95814:head@362'5984, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:28:01.181 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:44.717022+0000", 2023-09-23T23:28:01.181 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 16.324183678000001, 2023-09-23T23:28:01.181 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.191706239, 2023-09-23T23:28:01.181 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.181 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:28:01.182 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.182 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.182 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.182 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.717022+0000", 2023-09-23T23:28:01.182 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.182 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.182 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.182 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.182 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.717022+0000", 2023-09-23T23:28:01.182 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.182 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.182 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.183 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.183 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.717021+0000", 2023-09-23T23:28:01.183 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:01.183 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.183 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.183 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.183 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.717078+0000", 2023-09-23T23:28:01.183 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6514999999999997e-05 2023-09-23T23:28:01.183 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.183 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.183 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.183 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.906291+0000", 2023-09-23T23:28:01.183 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.18921305699999999 2023-09-23T23:28:01.184 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.184 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.184 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.184 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.906315+0000", 2023-09-23T23:28:01.184 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4219e-05 2023-09-23T23:28:01.184 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.184 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.184 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.184 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.906397+0000", 2023-09-23T23:28:01.184 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1390999999999994e-05 2023-09-23T23:28:01.184 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.184 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.185 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.185 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.906406+0000", 2023-09-23T23:28:01.185 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7399999999999999e-06 2023-09-23T23:28:01.185 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.185 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.185 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.185 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.908728+0000", 2023-09-23T23:28:01.185 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023218169999999999 2023-09-23T23:28:01.185 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.185 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.185 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.185 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.185 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.186 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35904.0:192785 11.3 e438/425 11:c28d0cdc:::benchmark_data_smithi110_433746_object86905:head v 438'9218, mlcod=438'9216)", 2023-09-23T23:28:01.186 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:44.725903+0000", 2023-09-23T23:28:01.186 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 16.315302701, 2023-09-23T23:28:01.186 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.18477083, 2023-09-23T23:28:01.186 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:01.186 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:01.186 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:01.186 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.186 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:01.186 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.725903+0000", 2023-09-23T23:28:01.186 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.186 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.186 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.187 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:01.187 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.725903+0000", 2023-09-23T23:28:01.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:01.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.187 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:01.187 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.725902+0000", 2023-09-23T23:28:01.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:28:01.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.187 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:01.187 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.725910+0000", 2023-09-23T23:28:01.188 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3259999999999996e-06 2023-09-23T23:28:01.188 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.188 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:01.188 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.906291+0000", 2023-09-23T23:28:01.188 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.18038058700000001 2023-09-23T23:28:01.188 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.188 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:01.188 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.906305+0000", 2023-09-23T23:28:01.188 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3801999999999999e-05 2023-09-23T23:28:01.188 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.188 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.189 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:01.189 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.906359+0000", 2023-09-23T23:28:01.189 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3758999999999998e-05 2023-09-23T23:28:01.189 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.189 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.189 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:01.189 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.906384+0000", 2023-09-23T23:28:01.189 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5837e-05 2023-09-23T23:28:01.189 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.189 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.189 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:01.189 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.910646+0000", 2023-09-23T23:28:01.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0042615719999999999 2023-09-23T23:28:01.190 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:01.190 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:01.190 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:01.190 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:44.910674+0000", 2023-09-23T23:28:01.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7804999999999999e-05 2023-09-23T23:28:01.190 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.190 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.190 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.190 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:01.190 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:01.190 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:28:01.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:01.192 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:01.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:01.293 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:01.355 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:01.452 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:01.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:01 smithi110 ceph-mon[212939]: pgmap v1251: 129 pgs: 1 peering, 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 120 active+clean; 690 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 2781/33162 objects misplaced (8.386%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T23:28:01.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:01 smithi110 ceph-mon[212939]: 11.11 scrub starts 2023-09-23T23:28:01.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:01 smithi110 ceph-mon[212939]: 1.f scrub starts 2023-09-23T23:28:01.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:01 smithi110 ceph-mon[212939]: failed to encode map e448 with expected crc 2023-09-23T23:28:01.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:01 smithi110 ceph-mon[212939]: failed to encode map e448 with expected crc 2023-09-23T23:28:01.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:01 smithi110 ceph-mon[212939]: osdmap e448: 8 total, 8 up, 7 in 2023-09-23T23:28:01.461 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:01 smithi110 ceph-mon[221102]: pgmap v1251: 129 pgs: 1 peering, 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 120 active+clean; 690 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 2781/33162 objects misplaced (8.386%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T23:28:01.461 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:01 smithi110 ceph-mon[221102]: 11.11 scrub starts 2023-09-23T23:28:01.461 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:01 smithi110 ceph-mon[221102]: 1.f scrub starts 2023-09-23T23:28:01.461 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:01 smithi110 ceph-mon[221102]: failed to encode map e448 with expected crc 2023-09-23T23:28:01.461 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:01 smithi110 ceph-mon[221102]: failed to encode map e448 with expected crc 2023-09-23T23:28:01.461 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:01 smithi110 ceph-mon[221102]: osdmap e448: 8 total, 8 up, 7 in 2023-09-23T23:28:01.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:01.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:01.563 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:01.613 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:01 smithi138 ceph-mon[193606]: pgmap v1251: 129 pgs: 1 peering, 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 120 active+clean; 690 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 2781/33162 objects misplaced (8.386%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T23:28:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:01 smithi138 ceph-mon[193606]: 11.11 scrub starts 2023-09-23T23:28:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:01 smithi138 ceph-mon[193606]: 1.f scrub starts 2023-09-23T23:28:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:01 smithi138 ceph-mon[193606]: failed to encode map e448 with expected crc 2023-09-23T23:28:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:01 smithi138 ceph-mon[193606]: failed to encode map e448 with expected crc 2023-09-23T23:28:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:01 smithi138 ceph-mon[193606]: osdmap e448: 8 total, 8 up, 7 in 2023-09-23T23:28:01.725 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:01.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:01.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:01.785 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:01.829 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:01.863 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:02.099 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:28:02.106 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:02.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:02.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:02.212 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:02.238 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:02.288 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:02.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:02.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:02.393 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:02.428 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:02 smithi138 ceph-mon[193606]: 11.12 scrub starts 2023-09-23T23:28:02.428 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:02 smithi138 ceph-mon[193606]: osdmap e449: 8 total, 8 up, 7 in 2023-09-23T23:28:02.428 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:02 smithi138 ceph-mon[193606]: failed to encode map e449 with expected crc 2023-09-23T23:28:02.428 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:02 smithi138 ceph-mon[193606]: failed to encode map e449 with expected crc 2023-09-23T23:28:02.428 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:02 smithi138 ceph-mon[193606]: failed to encode map e449 with expected crc 2023-09-23T23:28:02.428 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:02 smithi138 ceph-mon[193606]: 1.1a scrub starts 2023-09-23T23:28:02.432 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:02.495 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:02.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:02.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:02.537 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:28:02.592 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:28:02.601 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:02.616 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:28:02.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:02 smithi110 ceph-mon[212939]: 11.12 scrub starts 2023-09-23T23:28:02.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:02 smithi110 ceph-mon[212939]: osdmap e449: 8 total, 8 up, 7 in 2023-09-23T23:28:02.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:02 smithi110 ceph-mon[212939]: failed to encode map e449 with expected crc 2023-09-23T23:28:02.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:02 smithi110 ceph-mon[212939]: failed to encode map e449 with expected crc 2023-09-23T23:28:02.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:02 smithi110 ceph-mon[212939]: failed to encode map e449 with expected crc 2023-09-23T23:28:02.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:02 smithi110 ceph-mon[212939]: 1.1a scrub starts 2023-09-23T23:28:02.622 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:02 smithi110 ceph-mon[221102]: 11.12 scrub starts 2023-09-23T23:28:02.622 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:02 smithi110 ceph-mon[221102]: osdmap e449: 8 total, 8 up, 7 in 2023-09-23T23:28:02.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:02 smithi110 ceph-mon[221102]: failed to encode map e449 with expected crc 2023-09-23T23:28:02.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:02 smithi110 ceph-mon[221102]: failed to encode map e449 with expected crc 2023-09-23T23:28:02.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:02 smithi110 ceph-mon[221102]: failed to encode map e449 with expected crc 2023-09-23T23:28:02.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:02 smithi110 ceph-mon[221102]: 1.1a scrub starts 2023-09-23T23:28:02.683 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:03.319 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:28:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:03 smithi138 ceph-mon[193606]: pgmap v1254: 129 pgs: 1 peering, 5 active+remapped+backfill_wait, 123 active+clean; 627 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 1864/30138 objects misplaced (6.185%); 20 MiB/s, 322 objects/s recovering 2023-09-23T23:28:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:03 smithi138 ceph-mon[193606]: 11.2 scrub starts 2023-09-23T23:28:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:03 smithi138 ceph-mon[193606]: 1.11 scrub starts 2023-09-23T23:28:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:03 smithi138 ceph-mon[193606]: 11.2 scrub ok 2023-09-23T23:28:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:03 smithi138 ceph-mon[193606]: failed to encode map e450 with expected crc 2023-09-23T23:28:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:03 smithi138 ceph-mon[193606]: osdmap e450: 8 total, 8 up, 7 in 2023-09-23T23:28:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:03 smithi138 ceph-mon[193606]: failed to encode map e450 with expected crc 2023-09-23T23:28:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:03 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:03 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:03 smithi138 ceph-mon[193606]: 11.16 scrub starts 2023-09-23T23:28:03.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[212939]: pgmap v1254: 129 pgs: 1 peering, 5 active+remapped+backfill_wait, 123 active+clean; 627 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 1864/30138 objects misplaced (6.185%); 20 MiB/s, 322 objects/s recovering 2023-09-23T23:28:03.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[212939]: 11.2 scrub starts 2023-09-23T23:28:03.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[212939]: 1.11 scrub starts 2023-09-23T23:28:03.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[212939]: 11.2 scrub ok 2023-09-23T23:28:03.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[212939]: failed to encode map e450 with expected crc 2023-09-23T23:28:03.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[212939]: osdmap e450: 8 total, 8 up, 7 in 2023-09-23T23:28:03.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[212939]: failed to encode map e450 with expected crc 2023-09-23T23:28:03.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:03.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:03.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[212939]: 11.16 scrub starts 2023-09-23T23:28:03.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[221102]: pgmap v1254: 129 pgs: 1 peering, 5 active+remapped+backfill_wait, 123 active+clean; 627 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 1864/30138 objects misplaced (6.185%); 20 MiB/s, 322 objects/s recovering 2023-09-23T23:28:03.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[221102]: 11.2 scrub starts 2023-09-23T23:28:03.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[221102]: 1.11 scrub starts 2023-09-23T23:28:03.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[221102]: 11.2 scrub ok 2023-09-23T23:28:03.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[221102]: failed to encode map e450 with expected crc 2023-09-23T23:28:03.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[221102]: osdmap e450: 8 total, 8 up, 7 in 2023-09-23T23:28:03.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[221102]: failed to encode map e450 with expected crc 2023-09-23T23:28:03.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:03.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:03.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:03 smithi110 ceph-mon[221102]: 11.16 scrub starts 2023-09-23T23:28:04.314 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:28:04.315 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:28:04.315 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-09-23T23:28:04.315 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:28:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:04 smithi138 ceph-mon[193606]: from='client.36117 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:28:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:04 smithi138 ceph-mon[193606]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:04 smithi138 ceph-mon[193606]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:04 smithi138 ceph-mon[193606]: osdmap e451: 8 total, 8 up, 7 in 2023-09-23T23:28:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:04 smithi138 ceph-mon[193606]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:04 smithi138 ceph-mon[193606]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:04 smithi138 ceph-mon[193606]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:04 smithi138 ceph-mon[193606]: 1.e scrub starts 2023-09-23T23:28:04.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:04 smithi138 ceph-mon[193606]: 1.e scrub ok 2023-09-23T23:28:04.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:04 smithi138 ceph-mon[193606]: 11.16 scrub starts 2023-09-23T23:28:04.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:04 smithi138 ceph-mon[193606]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[212939]: from='client.36117 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:28:04.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[212939]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[212939]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[212939]: osdmap e451: 8 total, 8 up, 7 in 2023-09-23T23:28:04.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[212939]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[212939]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[212939]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[212939]: 1.e scrub starts 2023-09-23T23:28:04.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[212939]: 1.e scrub ok 2023-09-23T23:28:04.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[212939]: 11.16 scrub starts 2023-09-23T23:28:04.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[212939]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[221102]: from='client.36117 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:28:04.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[221102]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[221102]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[221102]: osdmap e451: 8 total, 8 up, 7 in 2023-09-23T23:28:04.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[221102]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[221102]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[221102]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[221102]: 1.e scrub starts 2023-09-23T23:28:04.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[221102]: 1.e scrub ok 2023-09-23T23:28:04.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[221102]: 11.16 scrub starts 2023-09-23T23:28:04.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:04 smithi110 ceph-mon[221102]: failed to encode map e451 with expected crc 2023-09-23T23:28:04.869 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:04.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:04.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:04.974 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:05.003 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1255,"stamp":"2023-09-23T23:28:02.331314+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":657188941,"num_objects":10046,"num_object_clones":0,"num_object_copies":30138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1864,"num_objects_unfound":0,"num_objects_dirty":10046,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":178548,"num_write_kb":5311354,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65415,"num_bytes_recovered":4176187467,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":162954,"ondisk_log_size":162954,"up":387,"acting":387,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":417,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7780184,"kb_used_data":3390880,"kb_used_omap":29,"kb_used_meta":4389218,"kb_avail":648435880,"statfs":{"total":671965249536,"available":663998341120,"internally_reserved":0,"allocated":3472261120,"data_stored":3459873344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":4494559955},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,0,0,2,1,7,2],"upper_bound":2048},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-129171456,"num_objects":-1971,"num_object_clones":0,"num_object_copies":-5913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1722,"num_objects_unfound":0,"num_objects_dirty":-1971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1971,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1289,"num_bytes_recovered":84475904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.039256"},"pg_stats":[{"pgid":"11.12","version":"448'9627","reported_seq":13677,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-09-23T23:28:00.796285+0000","last_change":"2023-09-23T23:27:57.267959+0000","last_active":"2023-09-23T23:28:00.796285+0000","last_peered":"2023-09-23T23:28:00.796285+0000","last_clean":"2023-09-23T23:28:00.796285+0000","last_became_active":"2023-09-23T23:26:42.095788+0000","last_became_peered":"2023-09-23T23:26:42.095788+0000","last_unstale":"2023-09-23T23:28:00.796285+0000","last_undegraded":"2023-09-23T23:28:00.796285+0000","last_fullsized":"2023-09-23T23:28:00.796285+0000","mapping_epoch":404,"log_start":"355'800","ondisk_log_start":"355'800","created":365,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":8827,"log_dups_size":0,"ondisk_log_size":8827,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22708224,"num_objects":347,"num_object_clones":0,"num_object_copies":1041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6381,"num_write_kb":215264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1954,"num_bytes_recovered":128057344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":632,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.299134+0000","last_change":"2023-09-23T23:26:50.557273+0000","last_active":"2023-09-23T23:27:57.299134+0000","last_peered":"2023-09-23T23:27:57.299134+0000","last_clean":"2023-09-23T23:27:57.299134+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:27:57.299134+0000","last_undegraded":"2023-09-23T23:27:57.299134+0000","last_fullsized":"2023-09-23T23:27:57.299134+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.058755+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.058755+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023428590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":481,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.347329+0000","last_change":"2023-09-23T23:22:38.439629+0000","last_active":"2023-09-23T23:27:58.347329+0000","last_peered":"2023-09-23T23:27:58.347329+0000","last_clean":"2023-09-23T23:27:58.347329+0000","last_became_active":"2023-09-23T23:22:38.439285+0000","last_became_peered":"2023-09-23T23:22:38.439285+0000","last_unstale":"2023-09-23T23:27:58.347329+0000","last_undegraded":"2023-09-23T23:27:58.347329+0000","last_fullsized":"2023-09-23T23:27:58.347329+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:34.431221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":415,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.283459+0000","last_change":"2023-09-23T23:23:43.511912+0000","last_active":"2023-09-23T23:27:57.283459+0000","last_peered":"2023-09-23T23:27:57.283459+0000","last_clean":"2023-09-23T23:27:57.283459+0000","last_became_active":"2023-09-23T23:23:43.511574+0000","last_became_peered":"2023-09-23T23:23:43.511574+0000","last_unstale":"2023-09-23T23:27:57.283459+0000","last_undegraded":"2023-09-23T23:27:57.283459+0000","last_fullsized":"2023-09-23T23:27:57.283459+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:16.286926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"448'9541","reported_seq":16290,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-09-23T23:28:00.841942+0000","last_change":"2023-09-23T23:28:00.282718+0000","last_active":"2023-09-23T23:28:00.841942+0000","last_peered":"2023-09-23T23:28:00.841942+0000","last_clean":"2023-09-23T23:28:00.841942+0000","last_became_active":"2023-09-23T23:27:34.078923+0000","last_became_peered":"2023-09-23T23:27:34.078923+0000","last_unstale":"2023-09-23T23:28:00.841942+0000","last_undegraded":"2023-09-23T23:28:00.841942+0000","last_fullsized":"2023-09-23T23:28:00.841942+0000","mapping_epoch":434,"log_start":"355'1400","ondisk_log_start":"355'1400","created":365,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":8141,"log_dups_size":0,"ondisk_log_size":8141,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18382848,"num_objects":281,"num_object_clones":0,"num_object_copies":843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6339,"num_write_kb":211808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2413,"num_bytes_recovered":158138368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":535,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.299314+0000","last_change":"2023-09-23T23:23:50.459634+0000","last_active":"2023-09-23T23:27:57.299314+0000","last_peered":"2023-09-23T23:27:57.299314+0000","last_clean":"2023-09-23T23:27:57.299314+0000","last_became_active":"2023-09-23T23:23:45.453958+0000","last_became_peered":"2023-09-23T23:23:45.453958+0000","last_unstale":"2023-09-23T23:27:57.299314+0000","last_undegraded":"2023-09-23T23:27:57.299314+0000","last_fullsized":"2023-09-23T23:27:57.299314+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:56.563201+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":157,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.303668+0000","last_change":"2023-09-23T23:26:11.528846+0000","last_active":"2023-09-23T23:27:58.303668+0000","last_peered":"2023-09-23T23:27:58.303668+0000","last_clean":"2023-09-23T23:27:58.303668+0000","last_became_active":"2023-09-23T23:26:11.528529+0000","last_became_peered":"2023-09-23T23:26:11.528529+0000","last_unstale":"2023-09-23T23:27:58.303668+0000","last_undegraded":"2023-09-23T23:27:58.303668+0000","last_fullsized":"2023-09-23T23:27:58.303668+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:35.094583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":466,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.304487+0000","last_change":"2023-09-23T23:26:48.558967+0000","last_active":"2023-09-23T23:27:59.304487+0000","last_peered":"2023-09-23T23:27:59.304487+0000","last_clean":"2023-09-23T23:27:59.304487+0000","last_became_active":"2023-09-23T23:26:48.555271+0000","last_became_peered":"2023-09-23T23:26:48.555271+0000","last_unstale":"2023-09-23T23:27:59.304487+0000","last_undegraded":"2023-09-23T23:27:59.304487+0000","last_fullsized":"2023-09-23T23:27:59.304487+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:50:57.509734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"448'9434","reported_seq":22945,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-09-23T23:28:00.539356+0000","last_change":"2023-09-23T23:27:24.606995+0000","last_active":"2023-09-23T23:28:00.539356+0000","last_peered":"2023-09-23T23:28:00.539356+0000","last_clean":"2023-09-23T23:28:00.539356+0000","last_became_active":"2023-09-23T23:27:24.589817+0000","last_became_peered":"2023-09-23T23:27:24.589817+0000","last_unstale":"2023-09-23T23:28:00.539356+0000","last_undegraded":"2023-09-23T23:28:00.539356+0000","last_fullsized":"2023-09-23T23:28:00.539356+0000","mapping_epoch":428,"log_start":"356'2537","ondisk_log_start":"356'2537","created":365,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"366'6410","last_scrub_stamp":"2023-09-23T23:25:21.346991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:21.346991+0000","objects_scrubbed":3231,"log_size":6897,"log_dups_size":0,"ondisk_log_size":6897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:37.469662+0000","scrub_duration":0.84688343099999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17825792,"num_objects":272,"num_object_clones":0,"num_object_copies":816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3039,"num_write_kb":2560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3371,"num_bytes_recovered":208666624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":737,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.347204+0000","last_change":"2023-09-23T23:27:23.551379+0000","last_active":"2023-09-23T23:27:58.347204+0000","last_peered":"2023-09-23T23:27:58.347204+0000","last_clean":"2023-09-23T23:27:58.347204+0000","last_became_active":"2023-09-23T23:27:23.551046+0000","last_became_peered":"2023-09-23T23:27:23.551046+0000","last_unstale":"2023-09-23T23:27:58.347204+0000","last_undegraded":"2023-09-23T23:27:58.347204+0000","last_fullsized":"2023-09-23T23:27:58.347204+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.842272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.842272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.023100959000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":443,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.286708+0000","last_change":"2023-09-23T23:27:20.554007+0000","last_active":"2023-09-23T23:27:57.286708+0000","last_peered":"2023-09-23T23:27:57.286708+0000","last_clean":"2023-09-23T23:27:57.286708+0000","last_became_active":"2023-09-23T23:27:20.553716+0000","last_became_peered":"2023-09-23T23:27:20.553716+0000","last_unstale":"2023-09-23T23:27:57.286708+0000","last_undegraded":"2023-09-23T23:27:57.286708+0000","last_fullsized":"2023-09-23T23:27:57.286708+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:38.911862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":380,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.303923+0000","last_change":"2023-09-23T23:25:31.735427+0000","last_active":"2023-09-23T23:27:58.303923+0000","last_peered":"2023-09-23T23:27:58.303923+0000","last_clean":"2023-09-23T23:27:58.303923+0000","last_became_active":"2023-09-23T23:25:31.734582+0000","last_became_peered":"2023-09-23T23:25:31.734582+0000","last_unstale":"2023-09-23T23:27:58.303923+0000","last_undegraded":"2023-09-23T23:27:58.303923+0000","last_fullsized":"2023-09-23T23:27:58.303923+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:39.075149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":765,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.283390+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:27:56.283390+0000","last_peered":"2023-09-23T23:27:56.283390+0000","last_clean":"2023-09-23T23:27:56.283390+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:27:56.283390+0000","last_undegraded":"2023-09-23T23:27:56.283390+0000","last_fullsized":"2023-09-23T23:27:56.283390+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":401,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.303368+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:27:59.303368+0000","last_peered":"2023-09-23T23:27:59.303368+0000","last_clean":"2023-09-23T23:27:59.303368+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:27:59.303368+0000","last_undegraded":"2023-09-23T23:27:59.303368+0000","last_fullsized":"2023-09-23T23:27:59.303368+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":434,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.286546+0000","last_change":"2023-09-23T23:26:11.528354+0000","last_active":"2023-09-23T23:27:57.286546+0000","last_peered":"2023-09-23T23:27:57.286546+0000","last_clean":"2023-09-23T23:27:57.286546+0000","last_became_active":"2023-09-23T23:26:11.528076+0000","last_became_peered":"2023-09-23T23:26:11.528076+0000","last_unstale":"2023-09-23T23:27:57.286546+0000","last_undegraded":"2023-09-23T23:27:57.286546+0000","last_fullsized":"2023-09-23T23:27:57.286546+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:16.100396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":720,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.309056+0000","last_change":"2023-09-23T23:26:14.521792+0000","last_active":"2023-09-23T23:27:59.309056+0000","last_peered":"2023-09-23T23:27:59.309056+0000","last_clean":"2023-09-23T23:27:59.309056+0000","last_became_active":"2023-09-23T23:26:14.521438+0000","last_became_peered":"2023-09-23T23:26:14.521438+0000","last_unstale":"2023-09-23T23:27:59.309056+0000","last_undegraded":"2023-09-23T23:27:59.309056+0000","last_fullsized":"2023-09-23T23:27:59.309056+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:09.103712+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":77,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.299446+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:27:57.299446+0000","last_peered":"2023-09-23T23:27:57.299446+0000","last_clean":"2023-09-23T23:27:57.299446+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:27:57.299446+0000","last_undegraded":"2023-09-23T23:27:57.299446+0000","last_fullsized":"2023-09-23T23:27:57.299446+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":395,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.287363+0000","last_change":"2023-09-23T23:26:48.538059+0000","last_active":"2023-09-23T23:27:57.287363+0000","last_peered":"2023-09-23T23:27:57.287363+0000","last_clean":"2023-09-23T23:27:57.287363+0000","last_became_active":"2023-09-23T23:26:48.537735+0000","last_became_peered":"2023-09-23T23:26:48.537735+0000","last_unstale":"2023-09-23T23:27:57.287363+0000","last_undegraded":"2023-09-23T23:27:57.287363+0000","last_fullsized":"2023-09-23T23:27:57.287363+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:32.878591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":830,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.299469+0000","last_change":"2023-09-23T23:26:51.563894+0000","last_active":"2023-09-23T23:27:57.299469+0000","last_peered":"2023-09-23T23:27:57.299469+0000","last_clean":"2023-09-23T23:27:57.299469+0000","last_became_active":"2023-09-23T23:26:51.563536+0000","last_became_peered":"2023-09-23T23:26:51.563536+0000","last_unstale":"2023-09-23T23:27:57.299469+0000","last_undegraded":"2023-09-23T23:27:57.299469+0000","last_fullsized":"2023-09-23T23:27:57.299469+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:54.095054+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:54.095054+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":50,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.283047+0000","last_change":"2023-09-23T23:26:48.669857+0000","last_active":"2023-09-23T23:27:56.283047+0000","last_peered":"2023-09-23T23:27:56.283047+0000","last_clean":"2023-09-23T23:27:56.283047+0000","last_became_active":"2023-09-23T23:26:48.538246+0000","last_became_peered":"2023-09-23T23:26:48.538246+0000","last_unstale":"2023-09-23T23:27:56.283047+0000","last_undegraded":"2023-09-23T23:27:56.283047+0000","last_fullsized":"2023-09-23T23:27:56.283047+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:19.039472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":33,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.302327+0000","last_change":"2023-09-23T23:27:20.551305+0000","last_active":"2023-09-23T23:27:58.302327+0000","last_peered":"2023-09-23T23:27:58.302327+0000","last_clean":"2023-09-23T23:27:58.302327+0000","last_became_active":"2023-09-23T23:27:20.551017+0000","last_became_peered":"2023-09-23T23:27:20.551017+0000","last_unstale":"2023-09-23T23:27:58.302327+0000","last_undegraded":"2023-09-23T23:27:58.302327+0000","last_fullsized":"2023-09-23T23:27:58.302327+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:27.026015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":521,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.304770+0000","last_change":"2023-09-23T23:24:02.268079+0000","last_active":"2023-09-23T23:27:58.304770+0000","last_peered":"2023-09-23T23:27:58.304770+0000","last_clean":"2023-09-23T23:27:58.304770+0000","last_became_active":"2023-09-23T23:23:45.463106+0000","last_became_peered":"2023-09-23T23:23:45.463106+0000","last_unstale":"2023-09-23T23:27:58.304770+0000","last_undegraded":"2023-09-23T23:27:58.304770+0000","last_fullsized":"2023-09-23T23:27:58.304770+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.128813+0000","scrub_duration":0.0018535310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":78,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.348792+0000","last_change":"2023-09-23T23:26:11.534727+0000","last_active":"2023-09-23T23:27:58.348792+0000","last_peered":"2023-09-23T23:27:58.348792+0000","last_clean":"2023-09-23T23:27:58.348792+0000","last_became_active":"2023-09-23T23:26:11.513694+0000","last_became_peered":"2023-09-23T23:26:11.513694+0000","last_unstale":"2023-09-23T23:27:58.348792+0000","last_undegraded":"2023-09-23T23:27:58.348792+0000","last_fullsized":"2023-09-23T23:27:58.348792+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:04.562837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":435,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.349060+0000","last_change":"2023-09-23T23:25:31.718910+0000","last_active":"2023-09-23T23:27:58.349060+0000","last_peered":"2023-09-23T23:27:58.349060+0000","last_clean":"2023-09-23T23:27:58.349060+0000","last_became_active":"2023-09-23T23:25:31.718567+0000","last_became_peered":"2023-09-23T23:25:31.718567+0000","last_unstale":"2023-09-23T23:27:58.349060+0000","last_undegraded":"2023-09-23T23:27:58.349060+0000","last_fullsized":"2023-09-23T23:27:58.349060+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:21.447944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":545,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.298873+0000","last_change":"2023-09-23T23:26:34.550496+0000","last_active":"2023-09-23T23:27:57.298873+0000","last_peered":"2023-09-23T23:27:57.298873+0000","last_clean":"2023-09-23T23:27:57.298873+0000","last_became_active":"2023-09-23T23:26:34.550151+0000","last_became_peered":"2023-09-23T23:26:34.550151+0000","last_unstale":"2023-09-23T23:27:57.298873+0000","last_undegraded":"2023-09-23T23:27:57.298873+0000","last_fullsized":"2023-09-23T23:27:57.298873+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:53.828311+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:53.828311+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":377,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.285732+0000","last_change":"2023-09-23T23:26:11.569228+0000","last_active":"2023-09-23T23:27:56.285732+0000","last_peered":"2023-09-23T23:27:56.285732+0000","last_clean":"2023-09-23T23:27:56.285732+0000","last_became_active":"2023-09-23T23:26:11.513159+0000","last_became_peered":"2023-09-23T23:26:11.513159+0000","last_unstale":"2023-09-23T23:27:56.285732+0000","last_undegraded":"2023-09-23T23:27:56.285732+0000","last_fullsized":"2023-09-23T23:27:56.285732+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:40.698257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":442,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.287608+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:27:57.287608+0000","last_peered":"2023-09-23T23:27:57.287608+0000","last_clean":"2023-09-23T23:27:57.287608+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:27:57.287608+0000","last_undegraded":"2023-09-23T23:27:57.287608+0000","last_fullsized":"2023-09-23T23:27:57.287608+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":784,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.282335+0000","last_change":"2023-09-23T23:26:34.551458+0000","last_active":"2023-09-23T23:27:56.282335+0000","last_peered":"2023-09-23T23:27:56.282335+0000","last_clean":"2023-09-23T23:27:56.282335+0000","last_became_active":"2023-09-23T23:26:34.551067+0000","last_became_peered":"2023-09-23T23:26:34.551067+0000","last_unstale":"2023-09-23T23:27:56.282335+0000","last_undegraded":"2023-09-23T23:27:56.282335+0000","last_fullsized":"2023-09-23T23:27:56.282335+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:05.788656+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":44,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.282752+0000","last_change":"2023-09-23T23:26:48.563723+0000","last_active":"2023-09-23T23:27:56.282752+0000","last_peered":"2023-09-23T23:27:56.282752+0000","last_clean":"2023-09-23T23:27:56.282752+0000","last_became_active":"2023-09-23T23:26:48.563340+0000","last_became_peered":"2023-09-23T23:26:48.563340+0000","last_unstale":"2023-09-23T23:27:56.282752+0000","last_undegraded":"2023-09-23T23:27:56.282752+0000","last_fullsized":"2023-09-23T23:27:56.282752+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:05.974131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":331,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.283346+0000","last_change":"2023-09-23T23:26:32.040316+0000","last_active":"2023-09-23T23:27:56.283346+0000","last_peered":"2023-09-23T23:27:56.283346+0000","last_clean":"2023-09-23T23:27:56.283346+0000","last_became_active":"2023-09-23T23:26:32.039758+0000","last_became_peered":"2023-09-23T23:26:32.039758+0000","last_unstale":"2023-09-23T23:27:56.283346+0000","last_undegraded":"2023-09-23T23:27:56.283346+0000","last_fullsized":"2023-09-23T23:27:56.283346+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:03.930331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.19","version":"446'11247","reported_seq":16363,"reported_epoch":446,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:58.665674+0000","last_change":"2023-09-23T23:27:21.715633+0000","last_active":"2023-09-23T23:27:58.665674+0000","last_peered":"2023-09-23T23:27:58.665674+0000","last_clean":"2023-09-23T23:27:19.516646+0000","last_became_active":"2023-09-23T23:27:21.687051+0000","last_became_peered":"2023-09-23T23:27:21.687051+0000","last_unstale":"2023-09-23T23:27:58.665674+0000","last_undegraded":"2023-09-23T23:27:58.665674+0000","last_fullsized":"2023-09-23T23:27:58.665674+0000","mapping_epoch":425,"log_start":"368'6829","ondisk_log_start":"368'6829","created":365,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":1949,"log_size":4418,"log_dups_size":0,"ondisk_log_size":4418,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15630336,"num_objects":239,"num_object_clones":0,"num_object_copies":717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":239,"num_objects_unfound":0,"num_objects_dirty":239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6431,"num_write_kb":213408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2401,"num_bytes_recovered":156762112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,7],"avail_no_missing":["3","0","6","7"],"object_location_counts":[{"shards":"0,3,7","objects":239}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":610,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.303300+0000","last_change":"2023-09-23T23:26:14.527752+0000","last_active":"2023-09-23T23:27:58.303300+0000","last_peered":"2023-09-23T23:27:58.303300+0000","last_clean":"2023-09-23T23:27:58.303300+0000","last_became_active":"2023-09-23T23:26:14.525188+0000","last_became_peered":"2023-09-23T23:26:14.525188+0000","last_unstale":"2023-09-23T23:27:58.303300+0000","last_undegraded":"2023-09-23T23:27:58.303300+0000","last_fullsized":"2023-09-23T23:27:58.303300+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:10.617744+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":299,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.282587+0000","last_change":"2023-09-23T23:25:31.725776+0000","last_active":"2023-09-23T23:27:56.282587+0000","last_peered":"2023-09-23T23:27:56.282587+0000","last_clean":"2023-09-23T23:27:56.282587+0000","last_became_active":"2023-09-23T23:25:31.725429+0000","last_became_peered":"2023-09-23T23:25:31.725429+0000","last_unstale":"2023-09-23T23:27:56.282587+0000","last_undegraded":"2023-09-23T23:27:56.282587+0000","last_fullsized":"2023-09-23T23:27:56.282587+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:07.289251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":437,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.304631+0000","last_change":"2023-09-23T23:26:48.559268+0000","last_active":"2023-09-23T23:27:59.304631+0000","last_peered":"2023-09-23T23:27:59.304631+0000","last_clean":"2023-09-23T23:27:59.304631+0000","last_became_active":"2023-09-23T23:26:48.558563+0000","last_became_peered":"2023-09-23T23:26:48.558563+0000","last_unstale":"2023-09-23T23:27:59.304631+0000","last_undegraded":"2023-09-23T23:27:59.304631+0000","last_fullsized":"2023-09-23T23:27:59.304631+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:58.779288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"446'9482","reported_seq":15301,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.653008+0000","last_change":"2023-09-23T23:27:30.609575+0000","last_active":"2023-09-23T23:27:58.653008+0000","last_peered":"2023-09-23T23:27:58.653008+0000","last_clean":"2023-09-23T23:27:58.653008+0000","last_became_active":"2023-09-23T23:27:30.579515+0000","last_became_peered":"2023-09-23T23:27:30.579515+0000","last_unstale":"2023-09-23T23:27:58.653008+0000","last_undegraded":"2023-09-23T23:27:58.653008+0000","last_fullsized":"2023-09-23T23:27:58.653008+0000","mapping_epoch":432,"log_start":"358'3911","ondisk_log_start":"358'3911","created":365,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5571,"log_dups_size":0,"ondisk_log_size":5571,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15499264,"num_objects":237,"num_object_clones":0,"num_object_copies":711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6309,"num_write_kb":209440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2724,"num_bytes_recovered":178520064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":789,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.299156+0000","last_change":"2023-09-23T23:27:23.549144+0000","last_active":"2023-09-23T23:27:57.299156+0000","last_peered":"2023-09-23T23:27:57.299156+0000","last_clean":"2023-09-23T23:27:57.299156+0000","last_became_active":"2023-09-23T23:27:23.548774+0000","last_became_peered":"2023-09-23T23:27:23.548774+0000","last_unstale":"2023-09-23T23:27:57.299156+0000","last_undegraded":"2023-09-23T23:27:57.299156+0000","last_fullsized":"2023-09-23T23:27:57.299156+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:18.211722+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":129,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.302698+0000","last_change":"2023-09-23T23:26:32.139091+0000","last_active":"2023-09-23T23:27:59.302698+0000","last_peered":"2023-09-23T23:27:59.302698+0000","last_clean":"2023-09-23T23:27:59.302698+0000","last_became_active":"2023-09-23T23:26:32.053033+0000","last_became_peered":"2023-09-23T23:26:32.053033+0000","last_unstale":"2023-09-23T23:27:59.302698+0000","last_undegraded":"2023-09-23T23:27:59.302698+0000","last_fullsized":"2023-09-23T23:27:59.302698+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:32.625818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.288221+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:27:57.288221+0000","last_peered":"2023-09-23T23:27:57.288221+0000","last_clean":"2023-09-23T23:27:57.288221+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:27:57.288221+0000","last_undegraded":"2023-09-23T23:27:57.288221+0000","last_fullsized":"2023-09-23T23:27:57.288221+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"447'9409","reported_seq":14201,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.601629+0000","last_change":"2023-09-23T23:27:57.334959+0000","last_active":"2023-09-23T23:27:59.601629+0000","last_peered":"2023-09-23T23:27:59.601629+0000","last_clean":"2023-09-23T23:27:59.601629+0000","last_became_active":"2023-09-23T23:27:57.299658+0000","last_became_peered":"2023-09-23T23:27:57.299658+0000","last_unstale":"2023-09-23T23:27:59.601629+0000","last_undegraded":"2023-09-23T23:27:59.601629+0000","last_fullsized":"2023-09-23T23:27:59.601629+0000","mapping_epoch":444,"log_start":"358'3634","ondisk_log_start":"358'3634","created":349,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"367'6563","last_scrub_stamp":"2023-09-23T23:25:25.026782+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:25.026782+0000","objects_scrubbed":3175,"log_size":5775,"log_dups_size":0,"ondisk_log_size":5775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:52.733728+0000","scrub_duration":0.41131605999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21299200,"num_objects":325,"num_object_clones":0,"num_object_copies":975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2850,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2248,"num_bytes_recovered":147324928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":772,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.283100+0000","last_change":"2023-09-23T23:26:51.563868+0000","last_active":"2023-09-23T23:27:56.283100+0000","last_peered":"2023-09-23T23:27:56.283100+0000","last_clean":"2023-09-23T23:27:56.283100+0000","last_became_active":"2023-09-23T23:26:51.563510+0000","last_became_peered":"2023-09-23T23:26:51.563510+0000","last_unstale":"2023-09-23T23:27:56.283100+0000","last_undegraded":"2023-09-23T23:27:56.283100+0000","last_fullsized":"2023-09-23T23:27:56.283100+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:34.244822+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":48,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.283701+0000","last_change":"2023-09-23T23:26:48.673221+0000","last_active":"2023-09-23T23:27:56.283701+0000","last_peered":"2023-09-23T23:27:56.283701+0000","last_clean":"2023-09-23T23:27:56.283701+0000","last_became_active":"2023-09-23T23:26:48.555920+0000","last_became_peered":"2023-09-23T23:26:48.555920+0000","last_unstale":"2023-09-23T23:27:56.283701+0000","last_undegraded":"2023-09-23T23:27:56.283701+0000","last_fullsized":"2023-09-23T23:27:56.283701+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:26.586824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":67,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.299046+0000","last_change":"2023-09-23T23:26:48.525190+0000","last_active":"2023-09-23T23:27:57.299046+0000","last_peered":"2023-09-23T23:27:57.299046+0000","last_clean":"2023-09-23T23:27:57.299046+0000","last_became_active":"2023-09-23T23:26:48.524874+0000","last_became_peered":"2023-09-23T23:26:48.524874+0000","last_unstale":"2023-09-23T23:27:57.299046+0000","last_undegraded":"2023-09-23T23:27:57.299046+0000","last_fullsized":"2023-09-23T23:27:57.299046+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:43.114769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"448'9554","reported_seq":10564,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-09-23T23:28:00.566212+0000","last_change":"2023-09-23T23:27:58.326500+0000","last_active":"2023-09-23T23:28:00.566212+0000","last_peered":"2023-09-23T23:28:00.566212+0000","last_clean":"2023-09-23T23:28:00.566212+0000","last_became_active":"2023-09-23T23:27:58.306249+0000","last_became_peered":"2023-09-23T23:27:58.306249+0000","last_unstale":"2023-09-23T23:28:00.566212+0000","last_undegraded":"2023-09-23T23:28:00.566212+0000","last_fullsized":"2023-09-23T23:28:00.566212+0000","mapping_epoch":445,"log_start":"356'2665","ondisk_log_start":"356'2665","created":349,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"366'6545","last_scrub_stamp":"2023-09-23T23:25:22.335514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:22.335514+0000","objects_scrubbed":3321,"log_size":6889,"log_dups_size":0,"ondisk_log_size":6889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:10.290711+0000","scrub_duration":0.75924003299999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20447232,"num_objects":312,"num_object_clones":0,"num_object_copies":936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3013,"num_write_kb":128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":395,"num_bytes_recovered":25886720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":833,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.287327+0000","last_change":"2023-09-23T23:26:14.530414+0000","last_active":"2023-09-23T23:27:57.287327+0000","last_peered":"2023-09-23T23:27:57.287327+0000","last_clean":"2023-09-23T23:27:57.287327+0000","last_became_active":"2023-09-23T23:26:14.530001+0000","last_became_peered":"2023-09-23T23:26:14.530001+0000","last_unstale":"2023-09-23T23:27:57.287327+0000","last_undegraded":"2023-09-23T23:27:57.287327+0000","last_fullsized":"2023-09-23T23:27:57.287327+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:37.443998+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":48,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.283289+0000","last_change":"2023-09-23T23:26:48.547146+0000","last_active":"2023-09-23T23:27:56.283289+0000","last_peered":"2023-09-23T23:27:56.283289+0000","last_clean":"2023-09-23T23:27:56.283289+0000","last_became_active":"2023-09-23T23:26:48.537682+0000","last_became_peered":"2023-09-23T23:26:48.537682+0000","last_unstale":"2023-09-23T23:27:56.283289+0000","last_undegraded":"2023-09-23T23:27:56.283289+0000","last_fullsized":"2023-09-23T23:27:56.283289+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:10.115563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":472,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.347224+0000","last_change":"2023-09-23T23:26:01.833889+0000","last_active":"2023-09-23T23:27:58.347224+0000","last_peered":"2023-09-23T23:27:58.347224+0000","last_clean":"2023-09-23T23:27:58.347224+0000","last_became_active":"2023-09-23T23:26:01.833626+0000","last_became_peered":"2023-09-23T23:26:01.833626+0000","last_unstale":"2023-09-23T23:27:58.347224+0000","last_undegraded":"2023-09-23T23:27:58.347224+0000","last_fullsized":"2023-09-23T23:27:58.347224+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:25.659671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"445'9448","reported_seq":18164,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.985065+0000","last_change":"2023-09-23T23:27:57.809298+0000","last_active":"2023-09-23T23:27:57.985065+0000","last_peered":"2023-09-23T23:27:57.985065+0000","last_clean":"2023-09-23T23:27:57.985065+0000","last_became_active":"2023-09-23T23:27:42.114973+0000","last_became_peered":"2023-09-23T23:27:42.114973+0000","last_unstale":"2023-09-23T23:27:57.985065+0000","last_undegraded":"2023-09-23T23:27:57.985065+0000","last_fullsized":"2023-09-23T23:27:57.985065+0000","mapping_epoch":436,"log_start":"357'3576","ondisk_log_start":"357'3576","created":349,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5872,"log_dups_size":0,"ondisk_log_size":5872,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23887872,"num_objects":364,"num_object_clones":0,"num_object_copies":1092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6254,"num_write_kb":211808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3490,"num_bytes_recovered":228720640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":768,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.299545+0000","last_change":"2023-09-23T23:27:30.256071+0000","last_active":"2023-09-23T23:27:57.299545+0000","last_peered":"2023-09-23T23:27:57.299545+0000","last_clean":"2023-09-23T23:27:57.299545+0000","last_became_active":"2023-09-23T23:26:14.521164+0000","last_became_peered":"2023-09-23T23:26:14.521164+0000","last_unstale":"2023-09-23T23:27:57.299545+0000","last_undegraded":"2023-09-23T23:27:57.299545+0000","last_fullsized":"2023-09-23T23:27:57.299545+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.042580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.042580+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":351,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.283267+0000","last_change":"2023-09-23T23:26:32.043926+0000","last_active":"2023-09-23T23:27:56.283267+0000","last_peered":"2023-09-23T23:27:56.283267+0000","last_clean":"2023-09-23T23:27:56.283267+0000","last_became_active":"2023-09-23T23:26:32.039282+0000","last_became_peered":"2023-09-23T23:26:32.039282+0000","last_unstale":"2023-09-23T23:27:56.283267+0000","last_undegraded":"2023-09-23T23:27:56.283267+0000","last_fullsized":"2023-09-23T23:27:56.283267+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:35.345086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":436,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.302487+0000","last_change":"2023-09-23T23:26:48.536996+0000","last_active":"2023-09-23T23:27:58.302487+0000","last_peered":"2023-09-23T23:27:58.302487+0000","last_clean":"2023-09-23T23:27:58.302487+0000","last_became_active":"2023-09-23T23:26:48.536638+0000","last_became_peered":"2023-09-23T23:26:48.536638+0000","last_unstale":"2023-09-23T23:27:58.302487+0000","last_undegraded":"2023-09-23T23:27:58.302487+0000","last_fullsized":"2023-09-23T23:27:58.302487+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:36.943318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.4","version":"445'9637","reported_seq":16480,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.069403+0000","last_change":"2023-09-23T23:27:47.203269+0000","last_active":"2023-09-23T23:27:58.069403+0000","last_peered":"2023-09-23T23:27:58.069403+0000","last_clean":"2023-09-23T23:27:58.069403+0000","last_became_active":"2023-09-23T23:27:47.156093+0000","last_became_peered":"2023-09-23T23:27:47.156093+0000","last_unstale":"2023-09-23T23:27:58.069403+0000","last_undegraded":"2023-09-23T23:27:58.069403+0000","last_fullsized":"2023-09-23T23:27:58.069403+0000","mapping_epoch":439,"log_start":"358'3830","ondisk_log_start":"358'3830","created":349,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5807,"log_dups_size":0,"ondisk_log_size":5807,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19169280,"num_objects":292,"num_object_clones":0,"num_object_copies":876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6390,"num_write_kb":213856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3348,"num_bytes_recovered":133038080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":501,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.730345+0000","last_change":"2023-09-23T23:27:58.730321+0000","last_active":"2023-09-23T23:27:58.730345+0000","last_peered":"2023-09-23T23:27:58.730345+0000","last_clean":"2023-09-23T23:27:58.730345+0000","last_became_active":"2023-09-23T23:26:51.588385+0000","last_became_peered":"2023-09-23T23:26:51.588385+0000","last_unstale":"2023-09-23T23:27:58.730345+0000","last_undegraded":"2023-09-23T23:27:58.730345+0000","last_fullsized":"2023-09-23T23:27:58.730345+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:04.832687+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:04.832687+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":273,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.287473+0000","last_change":"2023-09-23T23:25:31.732607+0000","last_active":"2023-09-23T23:27:57.287473+0000","last_peered":"2023-09-23T23:27:57.287473+0000","last_clean":"2023-09-23T23:27:57.287473+0000","last_became_active":"2023-09-23T23:25:31.732099+0000","last_became_peered":"2023-09-23T23:25:31.732099+0000","last_unstale":"2023-09-23T23:27:57.287473+0000","last_undegraded":"2023-09-23T23:27:57.287473+0000","last_fullsized":"2023-09-23T23:27:57.287473+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:23.639936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":423,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.303360+0000","last_change":"2023-09-23T23:22:38.438390+0000","last_active":"2023-09-23T23:27:59.303360+0000","last_peered":"2023-09-23T23:27:59.303360+0000","last_clean":"2023-09-23T23:27:59.303360+0000","last_became_active":"2023-09-23T23:22:38.438065+0000","last_became_peered":"2023-09-23T23:22:38.438065+0000","last_unstale":"2023-09-23T23:27:59.303360+0000","last_undegraded":"2023-09-23T23:27:59.303360+0000","last_fullsized":"2023-09-23T23:27:59.303360+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:51.254227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"445'9484","reported_seq":17762,"reported_epoch":445,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:58.047003+0000","last_change":"2023-09-23T23:27:21.744029+0000","last_active":"2023-09-23T23:27:58.047003+0000","last_peered":"2023-09-23T23:27:58.047003+0000","last_clean":"2023-09-23T23:27:19.524907+0000","last_became_active":"2023-09-23T23:27:21.716446+0000","last_became_peered":"2023-09-23T23:27:21.716446+0000","last_unstale":"2023-09-23T23:27:58.047003+0000","last_undegraded":"2023-09-23T23:27:58.047003+0000","last_fullsized":"2023-09-23T23:27:58.047003+0000","mapping_epoch":425,"log_start":"357'3593","ondisk_log_start":"357'3593","created":349,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"367'6627","last_scrub_stamp":"2023-09-23T23:25:24.842424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:24.842424+0000","objects_scrubbed":3200,"log_size":5891,"log_dups_size":0,"ondisk_log_size":5891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:58.481655+0000","scrub_duration":0.68031426699999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21889024,"num_objects":334,"num_object_clones":0,"num_object_copies":1002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":334,"num_objects_unfound":0,"num_objects_dirty":334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2866,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3980,"num_bytes_recovered":260833280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,5,2],"avail_no_missing":["7","2","3","5","6"],"object_location_counts":[{"shards":"2,5,7","objects":334}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":780,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.282633+0000","last_change":"2023-09-23T23:23:45.830384+0000","last_active":"2023-09-23T23:27:56.282633+0000","last_peered":"2023-09-23T23:27:56.282633+0000","last_clean":"2023-09-23T23:27:56.282633+0000","last_became_active":"2023-09-23T23:23:36.025724+0000","last_became_peered":"2023-09-23T23:23:36.025724+0000","last_unstale":"2023-09-23T23:27:56.282633+0000","last_undegraded":"2023-09-23T23:27:56.282633+0000","last_fullsized":"2023-09-23T23:27:56.282633+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:34.345528+0000","scrub_duration":0.001659259,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":413,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.283020+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:27:57.283020+0000","last_peered":"2023-09-23T23:27:57.283020+0000","last_clean":"2023-09-23T23:27:57.283020+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:27:57.283020+0000","last_undegraded":"2023-09-23T23:27:57.283020+0000","last_fullsized":"2023-09-23T23:27:57.283020+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":415,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.348903+0000","last_change":"2023-09-23T23:25:31.731172+0000","last_active":"2023-09-23T23:27:58.348903+0000","last_peered":"2023-09-23T23:27:58.348903+0000","last_clean":"2023-09-23T23:27:58.348903+0000","last_became_active":"2023-09-23T23:25:31.730903+0000","last_became_peered":"2023-09-23T23:25:31.730903+0000","last_unstale":"2023-09-23T23:27:58.348903+0000","last_undegraded":"2023-09-23T23:27:58.348903+0000","last_fullsized":"2023-09-23T23:27:58.348903+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:35.442945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"448'9697","reported_seq":13014,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-09-23T23:28:00.888195+0000","last_change":"2023-09-23T23:27:58.302652+0000","last_active":"2023-09-23T23:28:00.888195+0000","last_peered":"2023-09-23T23:28:00.888195+0000","last_clean":"2023-09-23T23:28:00.888195+0000","last_became_active":"2023-09-23T23:27:10.888274+0000","last_became_peered":"2023-09-23T23:27:10.888274+0000","last_unstale":"2023-09-23T23:28:00.888195+0000","last_undegraded":"2023-09-23T23:28:00.888195+0000","last_fullsized":"2023-09-23T23:28:00.888195+0000","mapping_epoch":421,"log_start":"355'800","ondisk_log_start":"355'800","created":349,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":8897,"log_dups_size":0,"ondisk_log_size":8897,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17727488,"num_objects":270,"num_object_clones":0,"num_object_copies":810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6450,"num_write_kb":215072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1584,"num_bytes_recovered":103809024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":727,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.309213+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:27:59.309213+0000","last_peered":"2023-09-23T23:27:59.309213+0000","last_clean":"2023-09-23T23:27:59.309213+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:27:59.309213+0000","last_undegraded":"2023-09-23T23:27:59.309213+0000","last_fullsized":"2023-09-23T23:27:59.309213+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":273,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.347292+0000","last_change":"2023-09-23T23:27:20.560074+0000","last_active":"2023-09-23T23:27:58.347292+0000","last_peered":"2023-09-23T23:27:58.347292+0000","last_clean":"2023-09-23T23:27:58.347292+0000","last_became_active":"2023-09-23T23:27:20.559764+0000","last_became_peered":"2023-09-23T23:27:20.559764+0000","last_unstale":"2023-09-23T23:27:58.347292+0000","last_undegraded":"2023-09-23T23:27:58.347292+0000","last_fullsized":"2023-09-23T23:27:58.347292+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:23.675227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":68,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.348645+0000","last_change":"2023-09-23T23:26:11.516475+0000","last_active":"2023-09-23T23:27:58.348645+0000","last_peered":"2023-09-23T23:27:58.348645+0000","last_clean":"2023-09-23T23:27:58.348645+0000","last_became_active":"2023-09-23T23:26:11.516198+0000","last_became_peered":"2023-09-23T23:26:11.516198+0000","last_unstale":"2023-09-23T23:27:58.348645+0000","last_undegraded":"2023-09-23T23:27:58.348645+0000","last_fullsized":"2023-09-23T23:27:58.348645+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:56.875103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"446'9459","reported_seq":12745,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.728940+0000","last_change":"2023-09-23T23:27:01.819921+0000","last_active":"2023-09-23T23:27:58.728940+0000","last_peered":"2023-09-23T23:27:58.728940+0000","last_clean":"2023-09-23T23:27:58.728940+0000","last_became_active":"2023-09-23T23:27:01.796624+0000","last_became_peered":"2023-09-23T23:27:01.796624+0000","last_unstale":"2023-09-23T23:27:58.728940+0000","last_undegraded":"2023-09-23T23:27:58.728940+0000","last_fullsized":"2023-09-23T23:27:58.728940+0000","mapping_epoch":417,"log_start":"358'3678","ondisk_log_start":"358'3678","created":349,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5781,"log_dups_size":0,"ondisk_log_size":5781,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22183936,"num_objects":338,"num_object_clones":0,"num_object_copies":1014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6256,"num_write_kb":211040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1570,"num_bytes_recovered":102891520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":602,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.303059+0000","last_change":"2023-09-23T23:26:34.548211+0000","last_active":"2023-09-23T23:27:58.303059+0000","last_peered":"2023-09-23T23:27:58.303059+0000","last_clean":"2023-09-23T23:27:58.303059+0000","last_became_active":"2023-09-23T23:26:34.547815+0000","last_became_peered":"2023-09-23T23:26:34.547815+0000","last_unstale":"2023-09-23T23:27:58.303059+0000","last_undegraded":"2023-09-23T23:27:58.303059+0000","last_fullsized":"2023-09-23T23:27:58.303059+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:37.675417+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":68,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.347111+0000","last_change":"2023-09-23T23:26:11.512815+0000","last_active":"2023-09-23T23:27:58.347111+0000","last_peered":"2023-09-23T23:27:58.347111+0000","last_clean":"2023-09-23T23:27:58.347111+0000","last_became_active":"2023-09-23T23:26:11.512466+0000","last_became_peered":"2023-09-23T23:26:11.512466+0000","last_unstale":"2023-09-23T23:27:58.347111+0000","last_undegraded":"2023-09-23T23:27:58.347111+0000","last_fullsized":"2023-09-23T23:27:58.347111+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:10.324158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":445,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.349185+0000","last_change":"2023-09-23T23:27:20.558731+0000","last_active":"2023-09-23T23:27:58.349185+0000","last_peered":"2023-09-23T23:27:58.349185+0000","last_clean":"2023-09-23T23:27:58.349185+0000","last_became_active":"2023-09-23T23:27:20.558394+0000","last_became_peered":"2023-09-23T23:27:20.558394+0000","last_unstale":"2023-09-23T23:27:58.349185+0000","last_undegraded":"2023-09-23T23:27:58.349185+0000","last_fullsized":"2023-09-23T23:27:58.349185+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:25:00.408551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"447'9394","reported_seq":9588,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.612640+0000","last_change":"2023-09-23T23:25:20.332216+0000","last_active":"2023-09-23T23:27:59.612640+0000","last_peered":"2023-09-23T23:27:59.612640+0000","last_clean":"2023-09-23T23:27:59.612640+0000","last_became_active":"2023-09-23T23:25:18.988253+0000","last_became_peered":"2023-09-23T23:25:18.988253+0000","last_unstale":"2023-09-23T23:27:59.612640+0000","last_undegraded":"2023-09-23T23:27:59.612640+0000","last_fullsized":"2023-09-23T23:27:59.612640+0000","mapping_epoch":349,"log_start":"358'3585","ondisk_log_start":"358'3585","created":349,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"366'6383","last_scrub_stamp":"2023-09-23T23:25:20.331998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:20.331998+0000","objects_scrubbed":3194,"log_size":5809,"log_dups_size":0,"ondisk_log_size":5809,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:37.230596+0000","scrub_duration":0.84482357799999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21102592,"num_objects":322,"num_object_clones":0,"num_object_copies":966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3022,"num_write_kb":4800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":769,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.308765+0000","last_change":"2023-09-23T23:25:34.500038+0000","last_active":"2023-09-23T23:27:59.308765+0000","last_peered":"2023-09-23T23:27:59.308765+0000","last_clean":"2023-09-23T23:27:59.308765+0000","last_became_active":"2023-09-23T23:25:34.499705+0000","last_became_peered":"2023-09-23T23:25:34.499705+0000","last_unstale":"2023-09-23T23:27:59.308765+0000","last_undegraded":"2023-09-23T23:27:59.308765+0000","last_fullsized":"2023-09-23T23:27:59.308765+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:43.096696+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":83,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.282254+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:27:57.282254+0000","last_peered":"2023-09-23T23:27:57.282254+0000","last_clean":"2023-09-23T23:27:57.282254+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:27:57.282254+0000","last_undegraded":"2023-09-23T23:27:57.282254+0000","last_fullsized":"2023-09-23T23:27:57.282254+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":368,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.303622+0000","last_change":"2023-09-23T23:22:39.489325+0000","last_active":"2023-09-23T23:27:58.303622+0000","last_peered":"2023-09-23T23:27:58.303622+0000","last_clean":"2023-09-23T23:27:58.303622+0000","last_became_active":"2023-09-23T23:22:39.489048+0000","last_became_peered":"2023-09-23T23:22:39.489048+0000","last_unstale":"2023-09-23T23:27:58.303622+0000","last_undegraded":"2023-09-23T23:27:58.303622+0000","last_fullsized":"2023-09-23T23:27:58.303622+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:29.984948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"448'12744","reported_seq":15119,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-09-23T23:28:00.865148+0000","last_change":"2023-09-23T23:27:53.301849+0000","last_active":"2023-09-23T23:28:00.865148+0000","last_peered":"2023-09-23T23:28:00.865148+0000","last_clean":"2023-09-23T23:28:00.865148+0000","last_became_active":"2023-09-23T23:27:53.251393+0000","last_became_peered":"2023-09-23T23:27:53.251393+0000","last_unstale":"2023-09-23T23:28:00.865148+0000","last_undegraded":"2023-09-23T23:28:00.865148+0000","last_fullsized":"2023-09-23T23:28:00.865148+0000","mapping_epoch":441,"log_start":"362'5758","ondisk_log_start":"362'5758","created":349,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":6986,"log_dups_size":0,"ondisk_log_size":6986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:42:29.466833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40566808,"num_objects":620,"num_object_clones":0,"num_object_copies":1860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":620,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12744,"num_write_kb":427585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1118,"num_bytes_recovered":73203736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":744,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.288144+0000","last_change":"2023-09-23T23:26:51.561734+0000","last_active":"2023-09-23T23:27:57.288144+0000","last_peered":"2023-09-23T23:27:57.288144+0000","last_clean":"2023-09-23T23:27:57.288144+0000","last_became_active":"2023-09-23T23:26:51.561394+0000","last_became_peered":"2023-09-23T23:26:51.561394+0000","last_unstale":"2023-09-23T23:27:57.288144+0000","last_undegraded":"2023-09-23T23:27:57.288144+0000","last_fullsized":"2023-09-23T23:27:57.288144+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.063071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.063071+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":148,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.304047+0000","last_change":"2023-09-23T23:23:36.048889+0000","last_active":"2023-09-23T23:27:58.304047+0000","last_peered":"2023-09-23T23:27:58.304047+0000","last_clean":"2023-09-23T23:27:58.304047+0000","last_became_active":"2023-09-23T23:23:36.048572+0000","last_became_peered":"2023-09-23T23:23:36.048572+0000","last_unstale":"2023-09-23T23:27:58.304047+0000","last_undegraded":"2023-09-23T23:27:58.304047+0000","last_fullsized":"2023-09-23T23:27:58.304047+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:55.581146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":461,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.299403+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:27:57.299403+0000","last_peered":"2023-09-23T23:27:57.299403+0000","last_clean":"2023-09-23T23:27:57.299403+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:27:57.299403+0000","last_undegraded":"2023-09-23T23:27:57.299403+0000","last_fullsized":"2023-09-23T23:27:57.299403+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.302409+0000","last_change":"2023-09-23T23:27:20.549904+0000","last_active":"2023-09-23T23:27:58.302409+0000","last_peered":"2023-09-23T23:27:58.302409+0000","last_clean":"2023-09-23T23:27:58.302409+0000","last_became_active":"2023-09-23T23:27:20.549645+0000","last_became_peered":"2023-09-23T23:27:20.549645+0000","last_unstale":"2023-09-23T23:27:58.302409+0000","last_undegraded":"2023-09-23T23:27:58.302409+0000","last_fullsized":"2023-09-23T23:27:58.302409+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:09.850640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"446'12657","reported_seq":27184,"reported_epoch":446,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:58.766565+0000","last_change":"2023-09-23T23:27:21.765359+0000","last_active":"2023-09-23T23:27:58.766565+0000","last_peered":"2023-09-23T23:27:58.766565+0000","last_clean":"2023-09-23T23:27:19.522554+0000","last_became_active":"2023-09-23T23:27:21.717182+0000","last_became_peered":"2023-09-23T23:27:21.717182+0000","last_unstale":"2023-09-23T23:27:58.766565+0000","last_undegraded":"2023-09-23T23:27:58.766565+0000","last_fullsized":"2023-09-23T23:27:58.766565+0000","mapping_epoch":425,"log_start":"384'8236","ondisk_log_start":"384'8236","created":349,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4421,"log_dups_size":0,"ondisk_log_size":4421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:29.188830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41484288,"num_objects":633,"num_object_clones":0,"num_object_copies":1899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":633,"num_objects_unfound":0,"num_objects_dirty":633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12657,"num_write_kb":425280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7186,"num_bytes_recovered":470941696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,0,4],"avail_no_missing":["3","0","4","6","7"],"object_location_counts":[{"shards":"0,3,4","objects":633}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":594,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.303639+0000","last_change":"2023-09-23T23:23:49.320969+0000","last_active":"2023-09-23T23:27:59.303639+0000","last_peered":"2023-09-23T23:27:59.303639+0000","last_clean":"2023-09-23T23:27:59.303639+0000","last_became_active":"2023-09-23T23:22:17.001911+0000","last_became_peered":"2023-09-23T23:22:17.001911+0000","last_unstale":"2023-09-23T23:27:59.303639+0000","last_undegraded":"2023-09-23T23:27:59.303639+0000","last_fullsized":"2023-09-23T23:27:59.303639+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.320823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.320823+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:07.000984+0000","scrub_duration":0.0018140630000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":148,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.302639+0000","last_change":"2023-09-23T23:25:31.731396+0000","last_active":"2023-09-23T23:27:58.302639+0000","last_peered":"2023-09-23T23:27:58.302639+0000","last_clean":"2023-09-23T23:27:58.302639+0000","last_became_active":"2023-09-23T23:25:31.730931+0000","last_became_peered":"2023-09-23T23:25:31.730931+0000","last_unstale":"2023-09-23T23:27:58.302639+0000","last_undegraded":"2023-09-23T23:27:58.302639+0000","last_fullsized":"2023-09-23T23:27:58.302639+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:48.002628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":426,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.348993+0000","last_change":"2023-09-23T23:27:20.556916+0000","last_active":"2023-09-23T23:27:58.348993+0000","last_peered":"2023-09-23T23:27:58.348993+0000","last_clean":"2023-09-23T23:27:58.348993+0000","last_became_active":"2023-09-23T23:27:20.556651+0000","last_became_peered":"2023-09-23T23:27:20.556651+0000","last_unstale":"2023-09-23T23:27:58.348993+0000","last_undegraded":"2023-09-23T23:27:58.348993+0000","last_fullsized":"2023-09-23T23:27:58.348993+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:06.012543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":33,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.303598+0000","last_change":"2023-09-23T23:27:20.551221+0000","last_active":"2023-09-23T23:27:58.303598+0000","last_peered":"2023-09-23T23:27:58.303598+0000","last_clean":"2023-09-23T23:27:58.303598+0000","last_became_active":"2023-09-23T23:27:20.550900+0000","last_became_peered":"2023-09-23T23:27:20.550900+0000","last_unstale":"2023-09-23T23:27:58.303598+0000","last_undegraded":"2023-09-23T23:27:58.303598+0000","last_fullsized":"2023-09-23T23:27:58.303598+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:23.012149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"446'9335","reported_seq":16734,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.588271+0000","last_change":"2023-09-23T23:27:25.617962+0000","last_active":"2023-09-23T23:27:58.588271+0000","last_peered":"2023-09-23T23:27:58.588271+0000","last_clean":"2023-09-23T23:27:58.588271+0000","last_became_active":"2023-09-23T23:27:25.598086+0000","last_became_peered":"2023-09-23T23:27:25.598086+0000","last_unstale":"2023-09-23T23:27:58.588271+0000","last_undegraded":"2023-09-23T23:27:58.588271+0000","last_fullsized":"2023-09-23T23:27:58.588271+0000","mapping_epoch":429,"log_start":"357'3360","ondisk_log_start":"357'3360","created":349,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":0,"last_scrub":"366'6384","last_scrub_stamp":"2023-09-23T23:25:20.132415+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:20.132415+0000","objects_scrubbed":3102,"log_size":5975,"log_dups_size":0,"ondisk_log_size":5975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:22.673284+0000","scrub_duration":0.99622401299999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20840448,"num_objects":318,"num_object_clones":0,"num_object_copies":954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2968,"num_write_kb":5888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3543,"num_bytes_recovered":232194048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":807,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.282476+0000","last_change":"2023-09-23T23:26:51.562598+0000","last_active":"2023-09-23T23:27:56.282476+0000","last_peered":"2023-09-23T23:27:56.282476+0000","last_clean":"2023-09-23T23:27:56.282476+0000","last_became_active":"2023-09-23T23:26:51.562167+0000","last_became_peered":"2023-09-23T23:26:51.562167+0000","last_unstale":"2023-09-23T23:27:56.282476+0000","last_undegraded":"2023-09-23T23:27:56.282476+0000","last_fullsized":"2023-09-23T23:27:56.282476+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:17.310285+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":182,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.308880+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:27:59.308880+0000","last_peered":"2023-09-23T23:27:59.308880+0000","last_clean":"2023-09-23T23:27:59.308880+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:27:59.308880+0000","last_undegraded":"2023-09-23T23:27:59.308880+0000","last_fullsized":"2023-09-23T23:27:59.308880+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":148,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.303165+0000","last_change":"2023-09-23T23:25:31.828908+0000","last_active":"2023-09-23T23:27:58.303165+0000","last_peered":"2023-09-23T23:27:58.303165+0000","last_clean":"2023-09-23T23:27:58.303165+0000","last_became_active":"2023-09-23T23:25:31.828582+0000","last_became_peered":"2023-09-23T23:25:31.828582+0000","last_unstale":"2023-09-23T23:27:58.303165+0000","last_undegraded":"2023-09-23T23:27:58.303165+0000","last_fullsized":"2023-09-23T23:27:58.303165+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:01.466593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":33,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.303997+0000","last_change":"2023-09-23T23:27:20.551574+0000","last_active":"2023-09-23T23:27:58.303997+0000","last_peered":"2023-09-23T23:27:58.303997+0000","last_clean":"2023-09-23T23:27:58.303997+0000","last_became_active":"2023-09-23T23:27:20.550820+0000","last_became_peered":"2023-09-23T23:27:20.550820+0000","last_unstale":"2023-09-23T23:27:58.303997+0000","last_undegraded":"2023-09-23T23:27:58.303997+0000","last_fullsized":"2023-09-23T23:27:58.303997+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:09.274552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"446'12456","reported_seq":14323,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.769759+0000","last_change":"2023-09-23T23:27:56.321233+0000","last_active":"2023-09-23T23:27:58.769759+0000","last_peered":"2023-09-23T23:27:58.769759+0000","last_clean":"2023-09-23T23:27:58.769759+0000","last_became_active":"2023-09-23T23:27:56.288002+0000","last_became_peered":"2023-09-23T23:27:56.288002+0000","last_unstale":"2023-09-23T23:27:58.769759+0000","last_undegraded":"2023-09-23T23:27:58.769759+0000","last_fullsized":"2023-09-23T23:27:58.769759+0000","mapping_epoch":443,"log_start":"384'8041","ondisk_log_start":"384'8041","created":349,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4415,"log_dups_size":0,"ondisk_log_size":4415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:02:54.188140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43384832,"num_objects":662,"num_object_clones":0,"num_object_copies":1986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12456,"num_write_kb":419776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":864,"num_bytes_recovered":56623104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":459,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.308737+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:27:59.308737+0000","last_peered":"2023-09-23T23:27:59.308737+0000","last_clean":"2023-09-23T23:27:59.308737+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:27:59.308737+0000","last_undegraded":"2023-09-23T23:27:59.308737+0000","last_fullsized":"2023-09-23T23:27:59.308737+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":765,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.282517+0000","last_change":"2023-09-23T23:26:51.564817+0000","last_active":"2023-09-23T23:27:56.282517+0000","last_peered":"2023-09-23T23:27:56.282517+0000","last_clean":"2023-09-23T23:27:56.282517+0000","last_became_active":"2023-09-23T23:26:51.564511+0000","last_became_peered":"2023-09-23T23:26:51.564511+0000","last_unstale":"2023-09-23T23:27:56.282517+0000","last_undegraded":"2023-09-23T23:27:56.282517+0000","last_fullsized":"2023-09-23T23:27:56.282517+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:02.415212+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":395,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.283412+0000","last_change":"2023-09-23T23:26:32.037619+0000","last_active":"2023-09-23T23:27:57.283412+0000","last_peered":"2023-09-23T23:27:57.283412+0000","last_clean":"2023-09-23T23:27:57.283412+0000","last_became_active":"2023-09-23T23:26:32.037045+0000","last_became_peered":"2023-09-23T23:26:32.037045+0000","last_unstale":"2023-09-23T23:27:57.283412+0000","last_undegraded":"2023-09-23T23:27:57.283412+0000","last_fullsized":"2023-09-23T23:27:57.283412+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:47.657527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":68,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.348728+0000","last_change":"2023-09-23T23:26:11.525845+0000","last_active":"2023-09-23T23:27:58.348728+0000","last_peered":"2023-09-23T23:27:58.348728+0000","last_clean":"2023-09-23T23:27:58.348728+0000","last_became_active":"2023-09-23T23:26:11.525507+0000","last_became_peered":"2023-09-23T23:26:11.525507+0000","last_unstale":"2023-09-23T23:27:58.348728+0000","last_undegraded":"2023-09-23T23:27:58.348728+0000","last_fullsized":"2023-09-23T23:27:58.348728+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:35.462874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"445'9475","reported_seq":15693,"reported_epoch":446,"state":"peering","last_fresh":"2023-09-23T23:27:58.299098+0000","last_change":"2023-09-23T23:27:58.299202+0000","last_active":"2023-09-23T23:27:58.299098+0000","last_peered":"2023-09-23T23:27:58.097496+0000","last_clean":"2023-09-23T23:27:58.097496+0000","last_became_active":"2023-09-23T23:27:21.688910+0000","last_became_peered":"2023-09-23T23:27:21.688910+0000","last_unstale":"2023-09-23T23:27:58.299098+0000","last_undegraded":"2023-09-23T23:27:58.299098+0000","last_fullsized":"2023-09-23T23:27:58.299098+0000","mapping_epoch":446,"log_start":"357'3616","ondisk_log_start":"357'3616","created":349,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5859,"log_dups_size":0,"ondisk_log_size":5859,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27099136,"num_objects":413,"num_object_clones":0,"num_object_copies":1239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6239,"num_write_kb":212896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3031,"num_bytes_recovered":198639616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1,7],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":610,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.303151+0000","last_change":"2023-09-23T23:26:34.550276+0000","last_active":"2023-09-23T23:27:58.303151+0000","last_peered":"2023-09-23T23:27:58.303151+0000","last_clean":"2023-09-23T23:27:58.303151+0000","last_became_active":"2023-09-23T23:26:34.549923+0000","last_became_peered":"2023-09-23T23:26:34.549923+0000","last_unstale":"2023-09-23T23:27:58.303151+0000","last_undegraded":"2023-09-23T23:27:58.303151+0000","last_fullsized":"2023-09-23T23:27:58.303151+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:22.636211+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":372,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.282281+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:27:56.282281+0000","last_peered":"2023-09-23T23:27:56.282281+0000","last_clean":"2023-09-23T23:27:56.282281+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:27:56.282281+0000","last_undegraded":"2023-09-23T23:27:56.282281+0000","last_fullsized":"2023-09-23T23:27:56.282281+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":450,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.303552+0000","last_change":"2023-09-23T23:26:11.528036+0000","last_active":"2023-09-23T23:27:59.303552+0000","last_peered":"2023-09-23T23:27:59.303552+0000","last_clean":"2023-09-23T23:27:59.303552+0000","last_became_active":"2023-09-23T23:26:11.527688+0000","last_became_peered":"2023-09-23T23:26:11.527688+0000","last_unstale":"2023-09-23T23:27:59.303552+0000","last_undegraded":"2023-09-23T23:27:59.303552+0000","last_fullsized":"2023-09-23T23:27:59.303552+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:49.411897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":478,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-09-23T23:28:00.313817+0000","last_change":"2023-09-23T23:22:38.438448+0000","last_active":"2023-09-23T23:28:00.313817+0000","last_peered":"2023-09-23T23:28:00.313817+0000","last_clean":"2023-09-23T23:28:00.313817+0000","last_became_active":"2023-09-23T23:22:38.437495+0000","last_became_peered":"2023-09-23T23:22:38.437495+0000","last_unstale":"2023-09-23T23:28:00.313817+0000","last_undegraded":"2023-09-23T23:28:00.313817+0000","last_fullsized":"2023-09-23T23:28:00.313817+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:49.587056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"447'12321","reported_seq":16996,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.612667+0000","last_change":"2023-09-23T23:27:29.668660+0000","last_active":"2023-09-23T23:27:59.612667+0000","last_peered":"2023-09-23T23:27:59.612667+0000","last_clean":"2023-09-23T23:27:59.612667+0000","last_became_active":"2023-09-23T23:27:29.641291+0000","last_became_peered":"2023-09-23T23:27:29.641291+0000","last_unstale":"2023-09-23T23:27:59.612667+0000","last_undegraded":"2023-09-23T23:27:59.612667+0000","last_fullsized":"2023-09-23T23:27:59.612667+0000","mapping_epoch":431,"log_start":"381'7783","ondisk_log_start":"381'7783","created":349,"last_epoch_clean":432,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4538,"log_dups_size":0,"ondisk_log_size":4538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:47.739467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39649280,"num_objects":605,"num_object_clones":0,"num_object_copies":1815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12321,"num_write_kb":413632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2267,"num_bytes_recovered":148570112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":474,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.282577+0000","last_change":"2023-09-23T23:26:04.560829+0000","last_active":"2023-09-23T23:27:57.282577+0000","last_peered":"2023-09-23T23:27:57.282577+0000","last_clean":"2023-09-23T23:27:57.282577+0000","last_became_active":"2023-09-23T23:26:04.560484+0000","last_became_peered":"2023-09-23T23:26:04.560484+0000","last_unstale":"2023-09-23T23:27:57.282577+0000","last_undegraded":"2023-09-23T23:27:57.282577+0000","last_fullsized":"2023-09-23T23:27:57.282577+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.048718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.048718+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":296,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.283157+0000","last_change":"2023-09-23T23:23:35.037351+0000","last_active":"2023-09-23T23:27:56.283157+0000","last_peered":"2023-09-23T23:27:56.283157+0000","last_clean":"2023-09-23T23:27:56.283157+0000","last_became_active":"2023-09-23T23:23:35.036841+0000","last_became_peered":"2023-09-23T23:23:35.036841+0000","last_unstale":"2023-09-23T23:27:56.283157+0000","last_undegraded":"2023-09-23T23:27:56.283157+0000","last_fullsized":"2023-09-23T23:27:56.283157+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:20.618292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":148,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.304258+0000","last_change":"2023-09-23T23:26:32.037099+0000","last_active":"2023-09-23T23:27:58.304258+0000","last_peered":"2023-09-23T23:27:58.304258+0000","last_clean":"2023-09-23T23:27:58.304258+0000","last_became_active":"2023-09-23T23:26:32.036681+0000","last_became_peered":"2023-09-23T23:26:32.036681+0000","last_unstale":"2023-09-23T23:27:58.304258+0000","last_undegraded":"2023-09-23T23:27:58.304258+0000","last_fullsized":"2023-09-23T23:27:58.304258+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:48.926576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":179,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.299372+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:27:57.299372+0000","last_peered":"2023-09-23T23:27:57.299372+0000","last_clean":"2023-09-23T23:27:57.299372+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:27:57.299372+0000","last_undegraded":"2023-09-23T23:27:57.299372+0000","last_fullsized":"2023-09-23T23:27:57.299372+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"445'12713","reported_seq":12817,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.115728+0000","last_change":"2023-09-23T23:23:43.505852+0000","last_active":"2023-09-23T23:27:58.115728+0000","last_peered":"2023-09-23T23:27:58.115728+0000","last_clean":"2023-09-23T23:27:58.115728+0000","last_became_active":"2023-09-23T23:23:43.505473+0000","last_became_peered":"2023-09-23T23:23:43.505473+0000","last_unstale":"2023-09-23T23:27:58.115728+0000","last_undegraded":"2023-09-23T23:27:58.115728+0000","last_fullsized":"2023-09-23T23:27:58.115728+0000","mapping_epoch":349,"log_start":"381'8066","ondisk_log_start":"381'8066","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4647,"log_dups_size":0,"ondisk_log_size":4647,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:31.385610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42532864,"num_objects":649,"num_object_clones":0,"num_object_copies":1947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":649,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12713,"num_write_kb":427584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":758,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.348745+0000","last_change":"2023-09-23T23:26:14.518921+0000","last_active":"2023-09-23T23:27:58.348745+0000","last_peered":"2023-09-23T23:27:58.348745+0000","last_clean":"2023-09-23T23:27:58.348745+0000","last_became_active":"2023-09-23T23:26:14.518558+0000","last_became_peered":"2023-09-23T23:26:14.518558+0000","last_unstale":"2023-09-23T23:27:58.348745+0000","last_undegraded":"2023-09-23T23:27:58.348745+0000","last_fullsized":"2023-09-23T23:27:58.348745+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:14.360597+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":439,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.288467+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:27:57.288467+0000","last_peered":"2023-09-23T23:27:57.288467+0000","last_clean":"2023-09-23T23:27:57.288467+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:27:57.288467+0000","last_undegraded":"2023-09-23T23:27:57.288467+0000","last_fullsized":"2023-09-23T23:27:57.288467+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":33,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.304659+0000","last_change":"2023-09-23T23:27:20.562744+0000","last_active":"2023-09-23T23:27:58.304659+0000","last_peered":"2023-09-23T23:27:58.304659+0000","last_clean":"2023-09-23T23:27:58.304659+0000","last_became_active":"2023-09-23T23:27:20.562383+0000","last_became_peered":"2023-09-23T23:27:20.562383+0000","last_unstale":"2023-09-23T23:27:58.304659+0000","last_undegraded":"2023-09-23T23:27:58.304659+0000","last_fullsized":"2023-09-23T23:27:58.304659+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:01.085133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":83,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.282216+0000","last_change":"2023-09-23T23:26:48.562966+0000","last_active":"2023-09-23T23:27:57.282216+0000","last_peered":"2023-09-23T23:27:57.282216+0000","last_clean":"2023-09-23T23:27:57.282216+0000","last_became_active":"2023-09-23T23:26:48.562630+0000","last_became_peered":"2023-09-23T23:26:48.562630+0000","last_unstale":"2023-09-23T23:27:57.282216+0000","last_undegraded":"2023-09-23T23:27:57.282216+0000","last_fullsized":"2023-09-23T23:27:57.282216+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:40.163647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"447'12616","reported_seq":21186,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.591105+0000","last_change":"2023-09-23T23:26:33.547524+0000","last_active":"2023-09-23T23:27:59.591105+0000","last_peered":"2023-09-23T23:27:59.591105+0000","last_clean":"2023-09-23T23:27:59.591105+0000","last_became_active":"2023-09-23T23:26:33.521810+0000","last_became_peered":"2023-09-23T23:26:33.521810+0000","last_unstale":"2023-09-23T23:27:59.591105+0000","last_undegraded":"2023-09-23T23:27:59.591105+0000","last_fullsized":"2023-09-23T23:27:59.591105+0000","mapping_epoch":400,"log_start":"379'7705","ondisk_log_start":"379'7705","created":349,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4911,"log_dups_size":0,"ondisk_log_size":4911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:49.164061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37879808,"num_objects":578,"num_object_clones":0,"num_object_copies":1734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12616,"num_write_kb":422208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4222,"num_bytes_recovered":276692992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":829,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.287741+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:27:57.287741+0000","last_peered":"2023-09-23T23:27:57.287741+0000","last_clean":"2023-09-23T23:27:57.287741+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:27:57.287741+0000","last_undegraded":"2023-09-23T23:27:57.287741+0000","last_fullsized":"2023-09-23T23:27:57.287741+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":44,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.283921+0000","last_change":"2023-09-23T23:26:48.544465+0000","last_active":"2023-09-23T23:27:56.283921+0000","last_peered":"2023-09-23T23:27:56.283921+0000","last_clean":"2023-09-23T23:27:56.283921+0000","last_became_active":"2023-09-23T23:26:48.544166+0000","last_became_peered":"2023-09-23T23:26:48.544166+0000","last_unstale":"2023-09-23T23:27:56.283921+0000","last_undegraded":"2023-09-23T23:27:56.283921+0000","last_fullsized":"2023-09-23T23:27:56.283921+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:38.844346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":433,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-09-23T23:28:00.313734+0000","last_change":"2023-09-23T23:22:38.482705+0000","last_active":"2023-09-23T23:28:00.313734+0000","last_peered":"2023-09-23T23:28:00.313734+0000","last_clean":"2023-09-23T23:28:00.313734+0000","last_became_active":"2023-09-23T23:22:38.438239+0000","last_became_peered":"2023-09-23T23:22:38.438239+0000","last_unstale":"2023-09-23T23:28:00.313734+0000","last_undegraded":"2023-09-23T23:28:00.313734+0000","last_fullsized":"2023-09-23T23:28:00.313734+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:52.006349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":68,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.347141+0000","last_change":"2023-09-23T23:26:11.522604+0000","last_active":"2023-09-23T23:27:58.347141+0000","last_peered":"2023-09-23T23:27:58.347141+0000","last_clean":"2023-09-23T23:27:58.347141+0000","last_became_active":"2023-09-23T23:26:11.522350+0000","last_became_peered":"2023-09-23T23:26:11.522350+0000","last_unstale":"2023-09-23T23:27:58.347141+0000","last_undegraded":"2023-09-23T23:27:58.347141+0000","last_fullsized":"2023-09-23T23:27:58.347141+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:20.164202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"445'9549","reported_seq":18815,"reported_epoch":445,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:58.069090+0000","last_change":"2023-09-23T23:27:21.744033+0000","last_active":"2023-09-23T23:27:58.069090+0000","last_peered":"2023-09-23T23:27:58.069090+0000","last_clean":"2023-09-23T23:27:19.485531+0000","last_became_active":"2023-09-23T23:27:21.716360+0000","last_became_peered":"2023-09-23T23:27:21.716360+0000","last_unstale":"2023-09-23T23:27:58.069090+0000","last_undegraded":"2023-09-23T23:27:58.069090+0000","last_fullsized":"2023-09-23T23:27:58.069090+0000","mapping_epoch":425,"log_start":"358'3740","ondisk_log_start":"358'3740","created":365,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5809,"log_dups_size":0,"ondisk_log_size":5809,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19595264,"num_objects":299,"num_object_clones":0,"num_object_copies":897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":299,"num_objects_unfound":0,"num_objects_dirty":299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6337,"num_write_kb":212352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4546,"num_bytes_recovered":297926656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,5,2],"avail_no_missing":["7","2","3","5","6"],"object_location_counts":[{"shards":"2,5,7","objects":299}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":828,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.287223+0000","last_change":"2023-09-23T23:25:34.505042+0000","last_active":"2023-09-23T23:27:57.287223+0000","last_peered":"2023-09-23T23:27:57.287223+0000","last_clean":"2023-09-23T23:27:57.287223+0000","last_became_active":"2023-09-23T23:25:34.504687+0000","last_became_peered":"2023-09-23T23:25:34.504687+0000","last_unstale":"2023-09-23T23:27:57.287223+0000","last_undegraded":"2023-09-23T23:27:57.287223+0000","last_fullsized":"2023-09-23T23:27:57.287223+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:06.369262+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":82,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.303518+0000","last_change":"2023-09-23T23:25:48.082121+0000","last_active":"2023-09-23T23:27:59.303518+0000","last_peered":"2023-09-23T23:27:59.303518+0000","last_clean":"2023-09-23T23:27:59.303518+0000","last_became_active":"2023-09-23T23:25:48.081746+0000","last_became_peered":"2023-09-23T23:25:48.081746+0000","last_unstale":"2023-09-23T23:27:59.303518+0000","last_undegraded":"2023-09-23T23:27:59.303518+0000","last_fullsized":"2023-09-23T23:27:59.303518+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:00.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":191,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.309603+0000","last_change":"2023-09-23T23:22:38.439602+0000","last_active":"2023-09-23T23:27:59.309603+0000","last_peered":"2023-09-23T23:27:59.309603+0000","last_clean":"2023-09-23T23:27:59.309603+0000","last_became_active":"2023-09-23T23:22:38.439261+0000","last_became_peered":"2023-09-23T23:22:38.439261+0000","last_unstale":"2023-09-23T23:27:59.309603+0000","last_undegraded":"2023-09-23T23:27:59.309603+0000","last_fullsized":"2023-09-23T23:27:59.309603+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:03.293816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"445'9601","reported_seq":16579,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.997982+0000","last_change":"2023-09-23T23:26:52.609382+0000","last_active":"2023-09-23T23:27:57.997982+0000","last_peered":"2023-09-23T23:27:57.997982+0000","last_clean":"2023-09-23T23:27:57.997982+0000","last_became_active":"2023-09-23T23:26:52.578253+0000","last_became_peered":"2023-09-23T23:26:52.578253+0000","last_unstale":"2023-09-23T23:27:57.997982+0000","last_undegraded":"2023-09-23T23:27:57.997982+0000","last_fullsized":"2023-09-23T23:27:57.997982+0000","mapping_epoch":414,"log_start":"358'3824","ondisk_log_start":"358'3824","created":365,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":5,"last_scrub":"366'6668","last_scrub_stamp":"2023-09-23T23:25:23.925139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:23.925139+0000","objects_scrubbed":3290,"log_size":5777,"log_dups_size":0,"ondisk_log_size":5777,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:07.350349+0000","scrub_duration":0.63215223600000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22806528,"num_objects":348,"num_object_clones":0,"num_object_copies":1044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2942,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3348,"num_bytes_recovered":209125376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":573,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.288709+0000","last_change":"2023-09-23T23:26:14.522158+0000","last_active":"2023-09-23T23:27:57.288709+0000","last_peered":"2023-09-23T23:27:57.288709+0000","last_clean":"2023-09-23T23:27:57.288709+0000","last_became_active":"2023-09-23T23:26:14.521785+0000","last_became_peered":"2023-09-23T23:26:14.521785+0000","last_unstale":"2023-09-23T23:27:57.288709+0000","last_undegraded":"2023-09-23T23:27:57.288709+0000","last_fullsized":"2023-09-23T23:27:57.288709+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:39.445601+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":188,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.348991+0000","last_change":"2023-09-23T23:22:38.452968+0000","last_active":"2023-09-23T23:27:58.348991+0000","last_peered":"2023-09-23T23:27:58.348991+0000","last_clean":"2023-09-23T23:27:58.348991+0000","last_became_active":"2023-09-23T23:22:38.437986+0000","last_became_peered":"2023-09-23T23:22:38.437986+0000","last_unstale":"2023-09-23T23:27:58.348991+0000","last_undegraded":"2023-09-23T23:27:58.348991+0000","last_fullsized":"2023-09-23T23:27:58.348991+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:10.333271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":78,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.347537+0000","last_change":"2023-09-23T23:27:20.558454+0000","last_active":"2023-09-23T23:27:58.347537+0000","last_peered":"2023-09-23T23:27:58.347537+0000","last_clean":"2023-09-23T23:27:58.347537+0000","last_became_active":"2023-09-23T23:27:20.558204+0000","last_became_peered":"2023-09-23T23:27:20.558204+0000","last_unstale":"2023-09-23T23:27:58.347537+0000","last_undegraded":"2023-09-23T23:27:58.347537+0000","last_fullsized":"2023-09-23T23:27:58.347537+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:15.079990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"445'9511","reported_seq":18158,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.034686+0000","last_change":"2023-09-23T23:27:47.198834+0000","last_active":"2023-09-23T23:27:58.034686+0000","last_peered":"2023-09-23T23:27:58.034686+0000","last_clean":"2023-09-23T23:27:58.034686+0000","last_became_active":"2023-09-23T23:27:47.157657+0000","last_became_peered":"2023-09-23T23:27:47.157657+0000","last_unstale":"2023-09-23T23:27:58.034686+0000","last_undegraded":"2023-09-23T23:27:58.034686+0000","last_fullsized":"2023-09-23T23:27:58.034686+0000","mapping_epoch":439,"log_start":"357'3388","ondisk_log_start":"357'3388","created":365,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":6123,"log_dups_size":0,"ondisk_log_size":6123,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20152320,"num_objects":308,"num_object_clones":0,"num_object_copies":924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6318,"num_write_kb":212000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3631,"num_bytes_recovered":237961216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":188,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.348890+0000","last_change":"2023-09-23T23:25:31.740467+0000","last_active":"2023-09-23T23:27:58.348890+0000","last_peered":"2023-09-23T23:27:58.348890+0000","last_clean":"2023-09-23T23:27:58.348890+0000","last_became_active":"2023-09-23T23:25:31.729046+0000","last_became_peered":"2023-09-23T23:25:31.729046+0000","last_unstale":"2023-09-23T23:27:58.348890+0000","last_undegraded":"2023-09-23T23:27:58.348890+0000","last_fullsized":"2023-09-23T23:27:58.348890+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:31.222969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":68,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.347176+0000","last_change":"2023-09-23T23:26:11.527453+0000","last_active":"2023-09-23T23:27:58.347176+0000","last_peered":"2023-09-23T23:27:58.347176+0000","last_clean":"2023-09-23T23:27:58.347176+0000","last_became_active":"2023-09-23T23:26:11.527208+0000","last_became_peered":"2023-09-23T23:26:11.527208+0000","last_unstale":"2023-09-23T23:27:58.347176+0000","last_undegraded":"2023-09-23T23:27:58.347176+0000","last_fullsized":"2023-09-23T23:27:58.347176+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:15.083611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"448'9548","reported_seq":10366,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-09-23T23:28:00.531779+0000","last_change":"2023-09-23T23:28:00.343585+0000","last_active":"2023-09-23T23:28:00.531779+0000","last_peered":"2023-09-23T23:28:00.531779+0000","last_clean":"2023-09-23T23:28:00.531779+0000","last_became_active":"2023-09-23T23:28:00.320769+0000","last_became_peered":"2023-09-23T23:28:00.320769+0000","last_unstale":"2023-09-23T23:28:00.531779+0000","last_undegraded":"2023-09-23T23:28:00.531779+0000","last_fullsized":"2023-09-23T23:28:00.531779+0000","mapping_epoch":447,"log_start":"356'2663","ondisk_log_start":"356'2663","created":365,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":6885,"log_dups_size":0,"ondisk_log_size":6885,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19398656,"num_objects":296,"num_object_clones":0,"num_object_copies":888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6330,"num_write_kb":212032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":336,"num_bytes_recovered":22020096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":711,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:27:56.283011+0000","last_change":"2023-09-23T23:25:34.510468+0000","last_active":"2023-09-23T23:27:56.283011+0000","last_peered":"2023-09-23T23:27:56.283011+0000","last_clean":"2023-09-23T23:27:56.283011+0000","last_became_active":"2023-09-23T23:25:34.510106+0000","last_became_peered":"2023-09-23T23:25:34.510106+0000","last_unstale":"2023-09-23T23:27:56.283011+0000","last_undegraded":"2023-09-23T23:27:56.283011+0000","last_fullsized":"2023-09-23T23:27:56.283011+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:42.157293+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":463,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.288630+0000","last_change":"2023-09-23T23:25:31.746383+0000","last_active":"2023-09-23T23:27:57.288630+0000","last_peered":"2023-09-23T23:27:57.288630+0000","last_clean":"2023-09-23T23:27:57.288630+0000","last_became_active":"2023-09-23T23:25:31.745929+0000","last_became_peered":"2023-09-23T23:25:31.745929+0000","last_unstale":"2023-09-23T23:27:57.288630+0000","last_undegraded":"2023-09-23T23:27:57.288630+0000","last_fullsized":"2023-09-23T23:27:57.288630+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:21.696449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":457,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T23:27:57.287499+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:27:57.287499+0000","last_peered":"2023-09-23T23:27:57.287499+0000","last_clean":"2023-09-23T23:27:57.287499+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:27:57.287499+0000","last_undegraded":"2023-09-23T23:27:57.287499+0000","last_fullsized":"2023-09-23T23:27:57.287499+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"443'9405","reported_seq":13186,"reported_epoch":443,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:27:55.638077+0000","last_change":"2023-09-23T23:27:21.773301+0000","last_active":"2023-09-23T23:27:55.638077+0000","last_peered":"2023-09-23T23:27:55.638077+0000","last_clean":"2023-09-23T23:27:19.482304+0000","last_became_active":"2023-09-23T23:27:21.727527+0000","last_became_peered":"2023-09-23T23:27:21.727527+0000","last_unstale":"2023-09-23T23:27:55.638077+0000","last_undegraded":"2023-09-23T23:27:55.638077+0000","last_fullsized":"2023-09-23T23:27:55.638077+0000","mapping_epoch":425,"log_start":"356'2418","ondisk_log_start":"356'2418","created":365,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":6987,"log_dups_size":0,"ondisk_log_size":6987,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23527424,"num_objects":359,"num_object_clones":0,"num_object_copies":1077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":359,"num_objects_unfound":0,"num_objects_dirty":359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6225,"num_write_kb":210688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1807,"num_bytes_recovered":118423552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[0,5,4],"avail_no_missing":["0","4","5","7"],"object_location_counts":[{"shards":"0,4,5","objects":359}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":820,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.303391+0000","last_change":"2023-09-23T23:26:14.527390+0000","last_active":"2023-09-23T23:27:58.303391+0000","last_peered":"2023-09-23T23:27:58.303391+0000","last_clean":"2023-09-23T23:27:58.303391+0000","last_became_active":"2023-09-23T23:26:14.527008+0000","last_became_peered":"2023-09-23T23:26:14.527008+0000","last_unstale":"2023-09-23T23:27:58.303391+0000","last_undegraded":"2023-09-23T23:27:58.303391+0000","last_fullsized":"2023-09-23T23:27:58.303391+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:36.460827+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":452,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-09-23T23:27:59.303287+0000","last_change":"2023-09-23T23:20:15.786279+0000","last_active":"2023-09-23T23:27:59.303287+0000","last_peered":"2023-09-23T23:27:59.303287+0000","last_clean":"2023-09-23T23:27:59.303287+0000","last_became_active":"2023-09-23T23:20:15.782731+0000","last_became_peered":"2023-09-23T23:20:15.782731+0000","last_unstale":"2023-09-23T23:27:59.303287+0000","last_undegraded":"2023-09-23T23:27:59.303287+0000","last_fullsized":"2023-09-23T23:27:59.303287+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:58.011396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":492,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T23:27:58.346983+0000","last_change":"2023-09-23T23:25:31.827217+0000","last_active":"2023-09-23T23:27:58.346983+0000","last_peered":"2023-09-23T23:27:58.346983+0000","last_clean":"2023-09-23T23:27:58.346983+0000","last_became_active":"2023-09-23T23:25:31.826915+0000","last_became_peered":"2023-09-23T23:25:31.826915+0000","last_unstale":"2023-09-23T23:27:58.346983+0000","last_undegraded":"2023-09-23T23:27:58.346983+0000","last_fullsized":"2023-09-23T23:27:58.346983+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:44.656199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":656670744,"num_objects":10021,"num_object_clones":0,"num_object_copies":30063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1864,"num_objects_unfound":0,"num_objects_dirty":10021,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":178466,"num_write_kb":5311105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65377,"num_bytes_recovered":4174970904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4315430912,"data_stored":4315414624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":157908,"ondisk_log_size":157908,"up":78,"acting":78,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":6162,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":512693,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":21,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1210401,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782582,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1320712,"kb_used_data":1136000,"kb_used_omap":8,"kb_used_meta":184695,"kb_avail":92424440,"statfs":{"total":95995035648,"available":94642626560,"internally_reserved":0,"allocated":1163264000,"data_stored":1161980110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":189127749},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":3.8119999999999998,"5min":3.024,"15min":1.998},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":30.672000000000001,"5min":85.721000000000004,"15min":85.721000000000004},"last":0.245},{"interface":"front","average":{"1min":4.8780000000000001,"5min":3.3580000000000001,"15min":1.8500000000000001},"min":{"1min":0.26600000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":30.504000000000001,"5min":85.965000000000003,"15min":85.965000000000003},"last":0.96099999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.7609999999999999,"15min":1.2789999999999999},"min":{"1min":0.27900000000000003,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":10.141,"5min":60.847000000000001,"15min":60.847000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.835,"15min":1.591},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":2.9449999999999998,"5min":59.512999999999998,"15min":59.512999999999998},"last":0.55400000000000005}]},{"osd":2,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":2.2759999999999998,"15min":1.98},"min":{"1min":0.27200000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.4269999999999996,"5min":55.811999999999998,"15min":55.811999999999998},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.3400000000000001,"5min":1.9039999999999999,"15min":1.429},"min":{"1min":0.28199999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.532,"5min":55.847999999999999,"15min":55.847999999999999},"last":0.91000000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":1.482,"15min":1.4870000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.008,"5min":9.9960000000000004,"15min":9.9960000000000004},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.272,"5min":1.498,"15min":1.3620000000000001},"min":{"1min":0.318,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.5949999999999998,"5min":9.9659999999999993,"15min":9.9659999999999993},"last":0.78100000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.72599999999999998,"15min":0.81399999999999995},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.2919999999999998,"5min":6.8380000000000001,"15min":6.8380000000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.80600000000000005,"15min":0.89400000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.68200000000000005,"5min":5.242,"15min":5.242},"last":0.36699999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.90000000000000002,"15min":1.024},"min":{"1min":0.307,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.71899999999999997,"5min":5.2110000000000003,"15min":5.2110000000000003},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.78700000000000003,"15min":0.80300000000000005},"min":{"1min":0.29899999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.8319999999999999,"5min":6.0149999999999997,"15min":6.0149999999999997},"last":0.38}]},{"osd":6,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.369,"15min":0.95899999999999996},"min":{"1min":0.33900000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.7989999999999999,"5min":62.213000000000001,"15min":62.213000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":1.526,"15min":1.234},"min":{"1min":0.34200000000000003,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.9299999999999999,"5min":62.805999999999997,"15min":62.805999999999997},"last":0.44600000000000001}]}]},{"osd":6,"up_from":0,"seq":197568496059,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":112,"seq":481036337400,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1054352,"kb_used_data":356808,"kb_used_omap":4,"kb_used_meta":697531,"kb_avail":92690800,"statfs":{"total":95995035648,"available":94915379200,"internally_reserved":0,"allocated":365371392,"data_stored":363264102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":714272654},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:27:59 2023","interfaces":[{"interface":"back","average":{"1min":15.005000000000001,"5min":5.6239999999999997,"15min":3.7549999999999999},"min":{"1min":0.25600000000000001,"5min":0.189,"15min":0.16800000000000001},"max":{"1min":130.768,"5min":130.768,"15min":130.768},"last":0.51100000000000001},{"interface":"front","average":{"1min":14.644,"5min":4.7809999999999997,"15min":3.6419999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":134.929,"5min":134.929,"15min":134.929},"last":0.42699999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:27:45 2023","interfaces":[{"interface":"back","average":{"1min":3.8620000000000001,"5min":6.0890000000000004,"15min":4.5659999999999998},"min":{"1min":0.373,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":18.123999999999999,"5min":209.84200000000001,"15min":209.84200000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":4.0369999999999999,"5min":6.0750000000000002,"15min":3.895},"min":{"1min":0.24199999999999999,"5min":0.23899999999999999,"15min":0.218},"max":{"1min":17.997,"5min":209.11099999999999,"15min":209.11099999999999},"last":0.252}]},{"osd":3,"last update":"Sat Sep 23 23:27:40 2023","interfaces":[{"interface":"back","average":{"1min":4.3920000000000003,"5min":3.383,"15min":3.8660000000000001},"min":{"1min":0.23799999999999999,"5min":0.22600000000000001,"15min":0.20799999999999999},"max":{"1min":33.210000000000001,"5min":157.90899999999999,"15min":157.90899999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":3.8570000000000002,"5min":3.2810000000000001,"15min":3.7090000000000001},"min":{"1min":0.26500000000000001,"5min":0.245,"15min":0.245},"max":{"1min":25.626999999999999,"5min":156.27000000000001,"15min":156.27000000000001},"last":0.32700000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7889999999999999,"5min":1.6619999999999999,"15min":1.597},"min":{"1min":0.318,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":4.3520000000000003,"5min":9.3780000000000001,"15min":18.474},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.7470000000000001,"15min":1.724},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.23,"5min":9.7460000000000004,"15min":25.710999999999999},"last":0.72399999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.159,"15min":3.0659999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.9939999999999998,"5min":13.064,"15min":250.33000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.776,"5min":1.944,"15min":3.2120000000000002},"min":{"1min":0.377,"5min":0.34899999999999998,"15min":0.30199999999999999},"max":{"1min":4.3109999999999999,"5min":10.92,"15min":250.21199999999999},"last":0.439}]},{"osd":6,"last update":"Sat Sep 23 23:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":2.4929999999999999,"15min":2.4550000000000001},"min":{"1min":0.33700000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.0640000000000001,"5min":74.087999999999994,"15min":74.087999999999994},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.835,"5min":2.843,"15min":2.3700000000000001},"min":{"1min":0.435,"5min":0.33100000000000002,"15min":0.32800000000000001},"max":{"1min":4.383,"5min":73.974999999999994,"15min":73.974999999999994},"last":0.47199999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:27:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5389999999999999,"5min":3.2429999999999999,"15min":4.1580000000000004},"min":{"1min":0.33000000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.6600000000000001,"5min":78.308000000000007,"15min":78.308000000000007},"last":0.753},{"interface":"front","average":{"1min":2.5880000000000001,"5min":3.0619999999999998,"15min":3.7330000000000001},"min":{"1min":0.45900000000000002,"5min":0.371,"15min":0.371},"max":{"1min":8.9930000000000003,"5min":77.204999999999998,"15min":77.204999999999998},"last":0.53600000000000003}]}]},{"osd":0,"up_from":91,"seq":390842024224,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1472420,"kb_used_data":416156,"kb_used_omap":5,"kb_used_meta":1056250,"kb_avail":92272732,"statfs":{"total":95995035648,"available":94487277568,"internally_reserved":0,"allocated":426143744,"data_stored":423960801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1081600634},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,0,0,2,1,7,2],"upper_bound":2048},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:27:18 2023","interfaces":[{"interface":"back","average":{"1min":3.29,"5min":4.29,"15min":3.9260000000000002},"min":{"1min":0.17799999999999999,"5min":0.17499999999999999,"15min":0.16900000000000001},"max":{"1min":15.144,"5min":139.30000000000001,"15min":155.91999999999999},"last":0.27500000000000002},{"interface":"front","average":{"1min":3.488,"5min":1.998,"15min":2.742},"min":{"1min":0.22700000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":10.423,"5min":10.423,"15min":153.28899999999999},"last":2.2570000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:27:40 2023","interfaces":[{"interface":"back","average":{"1min":6.4240000000000004,"5min":7.3380000000000001,"15min":4.9690000000000003},"min":{"1min":0.22,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":80.736000000000004,"5min":171.15100000000001,"15min":171.15100000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":7.1420000000000003,"5min":6.9139999999999997,"15min":5.2640000000000002},"min":{"1min":0.26400000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":85.375,"5min":173.04900000000001,"15min":173.04900000000001},"last":2.23}]},{"osd":3,"last update":"Sat Sep 23 23:27:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1019999999999999,"5min":4.6459999999999999,"15min":3.8399999999999999},"min":{"1min":0.32900000000000001,"5min":0.25600000000000001,"15min":0.23200000000000001},"max":{"1min":13.112,"5min":139.34899999999999,"15min":139.34899999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":3.2480000000000002,"5min":4.5229999999999997,"15min":3.4209999999999998},"min":{"1min":0.311,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":10.237,"5min":138.739,"15min":138.739},"last":2.2029999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:27:47 2023","interfaces":[{"interface":"back","average":{"1min":1.786,"5min":3.3809999999999998,"15min":2.6059999999999999},"min":{"1min":0.32200000000000001,"5min":0.29099999999999998,"15min":0.27000000000000002},"max":{"1min":9.2040000000000006,"5min":138.81299999999999,"15min":138.81299999999999},"last":2.1349999999999998},{"interface":"front","average":{"1min":1.55,"5min":5.415,"15min":3.2839999999999998},"min":{"1min":0.33200000000000002,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":7.0099999999999998,"5min":171.251,"15min":171.251},"last":0.317}]},{"osd":5,"last update":"Sat Sep 23 23:27:47 2023","interfaces":[{"interface":"back","average":{"1min":1.7889999999999999,"5min":5.5490000000000004,"15min":2.9649999999999999},"min":{"1min":0.34499999999999997,"5min":0.32100000000000001,"15min":0.29099999999999998},"max":{"1min":6.9699999999999998,"5min":171.30699999999999,"15min":171.30699999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.964,"5min":3.6080000000000001,"15min":2.2509999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":11.307,"5min":139.40799999999999,"15min":139.40799999999999},"last":0.49299999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:27:47 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":5.4320000000000004,"15min":3.161},"min":{"1min":0.371,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.0419999999999998,"5min":171.33600000000001,"15min":171.33600000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.948,"5min":6.1689999999999996,"15min":3.4049999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.33,"5min":184.13499999999999,"15min":184.13499999999999},"last":0.53400000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:27:40 2023","interfaces":[{"interface":"back","average":{"1min":2.375,"5min":3.7650000000000001,"15min":2.149},"min":{"1min":0.378,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":9.1739999999999995,"5min":138.77699999999999,"15min":138.77699999999999},"last":2.173},{"interface":"front","average":{"1min":2.29,"5min":3.6720000000000002,"15min":1.8120000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":11.27,"5min":139.37899999999999,"15min":139.37899999999999},"last":0.56799999999999995}]}]},{"osd":2,"up_from":199,"seq":854698492045,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":875668,"kb_used_data":355916,"kb_used_omap":7,"kb_used_meta":519736,"kb_avail":92869484,"statfs":{"total":95995035648,"available":95098351616,"internally_reserved":0,"allocated":364457984,"data_stored":362296203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":532209745},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":4.5220000000000002,"5min":4.6079999999999997,"15min":5.0019999999999998},"min":{"1min":0.63200000000000001,"5min":0.17699999999999999,"15min":0.16700000000000001},"max":{"1min":10.641,"5min":95.587999999999994,"15min":119.277},"last":0.52800000000000002},{"interface":"front","average":{"1min":4.7290000000000001,"5min":4.3849999999999998,"15min":5.1150000000000002},"min":{"1min":0.27400000000000002,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":18.055,"5min":95.796000000000006,"15min":119.56100000000001},"last":0.255}]},{"osd":1,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":5.1719999999999997,"5min":5.5960000000000001,"15min":7.4160000000000004},"min":{"1min":0.72899999999999998,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":13.441000000000001,"5min":157.98699999999999,"15min":157.98699999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":4.6319999999999997,"5min":5.6029999999999998,"15min":7.1150000000000002},"min":{"1min":0.247,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":14.271000000000001,"5min":158.03800000000001,"15min":158.03800000000001},"last":0.56000000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":7.3120000000000003,"5min":4.4080000000000004,"15min":5.266},"min":{"1min":0.26200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":46.807000000000002,"5min":74.867999999999995,"15min":119.366},"last":0.433},{"interface":"front","average":{"1min":6.8339999999999996,"5min":4.2999999999999998,"15min":4.5609999999999999},"min":{"1min":0.35999999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":47.706000000000003,"5min":75.418000000000006,"15min":75.418000000000006},"last":0.32800000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":3.165,"5min":1.9910000000000001,"15min":3.984},"min":{"1min":0.38300000000000001,"5min":0.28000000000000003,"15min":0.25},"max":{"1min":10.157,"5min":23.391999999999999,"15min":332.98099999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":2.7919999999999998,"5min":1.776,"15min":3.4340000000000002},"min":{"1min":0.30399999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.6839999999999993,"5min":21.076000000000001,"15min":332.96199999999999},"last":0.64100000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":3.3330000000000002,"5min":1.984,"15min":2.3730000000000002},"min":{"1min":0.35899999999999999,"5min":0.318,"15min":0.27500000000000002},"max":{"1min":10.192,"5min":23.359999999999999,"15min":119.919},"last":0.495},{"interface":"front","average":{"1min":2.8639999999999999,"5min":1.8600000000000001,"15min":2.323},"min":{"1min":0.35599999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.7110000000000003,"5min":21.105,"15min":119.306},"last":0.47799999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":3.206,"5min":2.0350000000000001,"15min":4.2329999999999997},"min":{"1min":0.40699999999999997,"5min":0.32000000000000001,"15min":0.31},"max":{"1min":10.218,"5min":23.425000000000001,"15min":349.32799999999997},"last":0.623},{"interface":"front","average":{"1min":2.8460000000000001,"5min":1.823,"15min":3.7149999999999999},"min":{"1min":0.42099999999999999,"5min":0.36699999999999999,"15min":0.34999999999999998},"max":{"1min":8.7300000000000004,"5min":21.353999999999999,"15min":349.17700000000002},"last":0.504}]},{"osd":7,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5419999999999998,"5min":1.905,"15min":1.419},"min":{"1min":0.45300000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.8079999999999998,"5min":20.350000000000001,"15min":20.350000000000001},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.7149999999999999,"5min":2.0169999999999999,"15min":1.516},"min":{"1min":0.50900000000000001,"5min":0.38,"15min":0.38},"max":{"1min":7.8579999999999997,"5min":20.321000000000002,"15min":20.321000000000002},"last":0.59299999999999997}]}]},{"osd":3,"up_from":148,"seq":635655160014,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1209600,"kb_used_data":582136,"kb_used_omap":2,"kb_used_meta":627453,"kb_avail":92535552,"statfs":{"total":95995035648,"available":94756405248,"internally_reserved":0,"allocated":596107264,"data_stored":594011273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":642511925},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:27:32 2023","interfaces":[{"interface":"back","average":{"1min":3.9369999999999998,"5min":4.4020000000000001,"15min":3.3399999999999999},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.158},"max":{"1min":40.901000000000003,"5min":103.866,"15min":103.866},"last":2.9790000000000001},{"interface":"front","average":{"1min":4.8689999999999998,"5min":4.2389999999999999,"15min":3.0339999999999998},"min":{"1min":0.191,"5min":0.191,"15min":0.186},"max":{"1min":46.204000000000001,"5min":101.884,"15min":101.884},"last":0.245}]},{"osd":1,"last update":"Sat Sep 23 23:27:32 2023","interfaces":[{"interface":"back","average":{"1min":3.5379999999999998,"5min":2.6829999999999998,"15min":2.851},"min":{"1min":0.218,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":26.959,"5min":38.094000000000001,"15min":76.322999999999993},"last":3.6680000000000001},{"interface":"front","average":{"1min":5.0220000000000002,"5min":2.8530000000000002,"15min":2.7330000000000001},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.20499999999999999},"max":{"1min":27.321999999999999,"5min":42.030999999999999,"15min":76.444000000000003},"last":0.38500000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:27:40 2023","interfaces":[{"interface":"back","average":{"1min":5.8639999999999999,"5min":3.96,"15min":3.8959999999999999},"min":{"1min":0.26000000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":66.364999999999995,"5min":66.364999999999995,"15min":101.279},"last":0.219},{"interface":"front","average":{"1min":6.3479999999999999,"5min":3.77,"15min":3.04},"min":{"1min":0.29299999999999998,"5min":0.214,"15min":0.214},"max":{"1min":66.450000000000003,"5min":66.450000000000003,"15min":101.294},"last":0.41299999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:27:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":1.1699999999999999,"15min":1.4339999999999999},"min":{"1min":0.27500000000000002,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":6.8890000000000002,"5min":7.2610000000000001,"15min":14.754},"last":0.502},{"interface":"front","average":{"1min":1.5029999999999999,"5min":1.482,"15min":1.4770000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.2800000000000002,"5min":15.619,"15min":15.619},"last":0.44}]},{"osd":5,"last update":"Sat Sep 23 23:27:32 2023","interfaces":[{"interface":"back","average":{"1min":1.788,"5min":1.573,"15min":1.6819999999999999},"min":{"1min":0.29599999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.2619999999999996,"5min":15.762,"15min":75.733000000000004},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.6990000000000001,"5min":1.679,"15min":1.9730000000000001},"min":{"1min":0.318,"5min":0.316,"15min":0.28399999999999997},"max":{"1min":6.9880000000000004,"5min":13.864000000000001,"15min":75.686999999999998},"last":0.52800000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:27:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6120000000000001,"5min":2.1480000000000001,"15min":1.7130000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.3019999999999996,"5min":53.057000000000002,"15min":53.057000000000002},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.6419999999999999,"5min":2.2029999999999998,"15min":1.9279999999999999},"min":{"1min":0.40000000000000002,"5min":0.35299999999999998,"15min":0.33300000000000002},"max":{"1min":7.2320000000000002,"5min":53.088000000000001,"15min":53.088000000000001},"last":0.57099999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:27:40 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":1.3149999999999999,"15min":1.0129999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.2590000000000003,"5min":8.2590000000000003,"15min":8.2590000000000003},"last":0.54300000000000004},{"interface":"front","average":{"1min":2.331,"5min":1.7509999999999999,"15min":1.405},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":13.41,"5min":15.795,"15min":15.795},"last":1.504}]}]},{"osd":4,"up_from":30,"seq":128849019332,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":978644,"kb_used_data":275980,"kb_used_omap":0,"kb_used_meta":702656,"kb_avail":92766508,"statfs":{"total":95995035648,"available":94992904192,"internally_reserved":0,"allocated":282603520,"data_stored":281315317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":719519744},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:27:55 2023","interfaces":[{"interface":"back","average":{"1min":3.8149999999999999,"5min":3.8330000000000002,"15min":4.7480000000000002},"min":{"1min":0.23599999999999999,"5min":0.219,"15min":0.218},"max":{"1min":40.710000000000001,"5min":92.141999999999996,"15min":199.64400000000001},"last":8.4030000000000005},{"interface":"front","average":{"1min":3.496,"5min":3.8140000000000001,"15min":4.8460000000000001},"min":{"1min":0.27100000000000002,"5min":0.216,"15min":0.216},"max":{"1min":38.078000000000003,"5min":92.284000000000006,"15min":195.649},"last":0.27100000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:27:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":1.585,"15min":3.8650000000000002},"min":{"1min":0.24299999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.8700000000000001,"5min":40.643000000000001,"15min":161.923},"last":0.378},{"interface":"front","average":{"1min":2.2650000000000001,"5min":1.901,"15min":4.0259999999999998},"min":{"1min":0.307,"5min":0.27500000000000002,"15min":0.26200000000000001},"max":{"1min":10.657999999999999,"5min":35.362000000000002,"15min":161.39699999999999},"last":2.2269999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:27:39 2023","interfaces":[{"interface":"back","average":{"1min":3.4390000000000001,"5min":4.4100000000000001,"15min":3.302},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.26300000000000001},"max":{"1min":29.797000000000001,"5min":97.962000000000003,"15min":97.962000000000003},"last":2.5790000000000002},{"interface":"front","average":{"1min":2.8940000000000001,"5min":4.2919999999999998,"15min":3.008},"min":{"1min":0.39300000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":25.446999999999999,"5min":98.137,"15min":98.137},"last":5.298}]},{"osd":3,"last update":"Sat Sep 23 23:27:35 2023","interfaces":[{"interface":"back","average":{"1min":4.1859999999999999,"5min":6.5609999999999999,"15min":5.6760000000000002},"min":{"1min":0.33500000000000002,"5min":0.27200000000000002,"15min":0.26300000000000001},"max":{"1min":50.755000000000003,"5min":176.74100000000001,"15min":303.41500000000002},"last":3.7429999999999999},{"interface":"front","average":{"1min":3.3660000000000001,"5min":6.1959999999999997,"15min":5.7530000000000001},"min":{"1min":0.39400000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":50.738999999999997,"5min":176.87799999999999,"15min":303.428},"last":5.4740000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:27:09 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.65000000000000002,"15min":1.4379999999999999},"min":{"1min":0.29499999999999998,"5min":0.26100000000000001,"15min":0.23699999999999999},"max":{"1min":4.266,"5min":4.266,"15min":160.79499999999999},"last":0.441},{"interface":"front","average":{"1min":0.879,"5min":0.746,"15min":1.6080000000000001},"min":{"1min":0.34999999999999998,"5min":0.249,"15min":0.249},"max":{"1min":4.4359999999999999,"5min":5.0789999999999997,"15min":160.61099999999999},"last":0.30099999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:27:35 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":1.55,"15min":1.169},"min":{"1min":0.29799999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.097,"5min":90.346999999999994,"15min":90.346999999999994},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":1.6220000000000001,"15min":1.3100000000000001},"min":{"1min":0.38300000000000001,"5min":0.31900000000000001,"15min":0.30599999999999999},"max":{"1min":1.0980000000000001,"5min":90.733000000000004,"15min":90.733000000000004},"last":0.41099999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:27:39 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":1.538,"15min":0.89000000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.8109999999999999,"5min":61.923999999999999,"15min":61.923999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.64900000000000002,"5min":1.5509999999999999,"15min":1.052},"min":{"1min":0.40100000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.655,"5min":61.990000000000002,"15min":61.990000000000002},"last":0.46300000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757695,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":868788,"kb_used_data":267884,"kb_used_omap":0,"kb_used_meta":600896,"kb_avail":92876364,"statfs":{"total":95995035648,"available":95105396736,"internally_reserved":0,"allocated":274313216,"data_stored":273045538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":615317504},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:27:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":2.1520000000000001,"15min":3.7829999999999999},"min":{"1min":0.26200000000000001,"5min":0.222,"15min":0.19500000000000001},"max":{"1min":19.952000000000002,"5min":63.741999999999997,"15min":113.869},"last":0.27200000000000002},{"interface":"front","average":{"1min":1.4219999999999999,"5min":3.3250000000000002,"15min":4.2169999999999996},"min":{"1min":0.26600000000000001,"5min":0.22900000000000001,"15min":0.22700000000000001},"max":{"1min":5.7320000000000002,"5min":138.55199999999999,"15min":138.55199999999999},"last":0.36799999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:27:02 2023","interfaces":[{"interface":"back","average":{"1min":9.375,"5min":3.3940000000000001,"15min":3.9710000000000001},"min":{"1min":0.25900000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":69.816999999999993,"5min":69.816999999999993,"15min":193.584},"last":0.29199999999999998},{"interface":"front","average":{"1min":10.48,"5min":3.8639999999999999,"15min":4.2549999999999999},"min":{"1min":0.375,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":70.268000000000001,"5min":70.268000000000001,"15min":194.03100000000001},"last":0.315}]},{"osd":2,"last update":"Sat Sep 23 23:27:35 2023","interfaces":[{"interface":"back","average":{"1min":2.0750000000000002,"5min":3.952,"15min":2.645},"min":{"1min":0.314,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.5640000000000001,"5min":144.91200000000001,"15min":144.91200000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.776,"5min":3.806,"15min":2.5619999999999998},"min":{"1min":0.28799999999999998,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":5.8410000000000002,"5min":145.15799999999999,"15min":145.15799999999999},"last":2.5569999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:27:29 2023","interfaces":[{"interface":"back","average":{"1min":4.4880000000000004,"5min":3.008,"15min":4.468},"min":{"1min":0.32700000000000001,"5min":0.315,"15min":0.27400000000000002},"max":{"1min":62.612000000000002,"5min":62.612000000000002,"15min":159.53299999999999},"last":1.2170000000000001},{"interface":"front","average":{"1min":4.3970000000000002,"5min":3.1000000000000001,"15min":4.6980000000000004},"min":{"1min":0.318,"5min":0.318,"15min":0.312},"max":{"1min":62.639000000000003,"5min":62.639000000000003,"15min":159.54900000000001},"last":1.21}]},{"osd":4,"last update":"Sat Sep 23 23:27:10 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.71699999999999997,"15min":0.98299999999999998},"min":{"1min":0.26700000000000002,"5min":0.251,"15min":0.23999999999999999},"max":{"1min":1.1919999999999999,"5min":5.3739999999999997,"15min":10.012},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.60899999999999999,"15min":0.76300000000000001},"min":{"1min":0.30299999999999999,"5min":0.245,"15min":0.245},"max":{"1min":3.2530000000000001,"5min":3.2530000000000001,"15min":11.923999999999999},"last":0.22900000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:27:10 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.4770000000000001,"15min":2.2250000000000001},"min":{"1min":0.33000000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.2749999999999999,"5min":68.456999999999994,"15min":310.68799999999999},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":1.452,"15min":2.4209999999999998},"min":{"1min":0.34799999999999998,"5min":0.26000000000000001,"15min":0.25900000000000001},"max":{"1min":1.2050000000000001,"5min":66.721999999999994,"15min":311.45400000000001},"last":0.34300000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:27:35 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":2.2160000000000002,"15min":3.5030000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.0859999999999999,"5min":66.372,"15min":66.372},"last":179.28800000000001},{"interface":"front","average":{"1min":0.66400000000000003,"5min":2.262,"15min":3.6440000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.758,"5min":66.558999999999997,"15min":66.558999999999997},"last":179.428}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":398721024,"data_stored":398721024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":367792128,"data_stored":367788056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":340135936,"data_stored":340131864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":588709888,"data_stored":588709888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":262017024,"data_stored":262012952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":260636672,"data_stored":260636672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":953487360,"data_stored":953483288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1143930880,"data_stored":1143930880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:28:05.011 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:05.012 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:28:05.049 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:05.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:05.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:05.085 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:05.154 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:05.180 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:05.225 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:05.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:05.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:05.332 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:05.346 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:05 smithi138 ceph-mon[193606]: pgmap v1257: 129 pgs: 2 peering, 3 active+remapped+backfill_wait, 124 active+clean; 588 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 406 op/s; 1231/28284 objects misplaced (4.352%); 23 MiB/s, 371 objects/s recovering 2023-09-23T23:28:05.346 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:05 smithi138 ceph-mon[193606]: failed to encode map e452 with expected crc 2023-09-23T23:28:05.346 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:05 smithi138 ceph-mon[193606]: failed to encode map e452 with expected crc 2023-09-23T23:28:05.346 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:05 smithi138 ceph-mon[193606]: osdmap e452: 8 total, 8 up, 7 in 2023-09-23T23:28:05.346 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:05 smithi138 ceph-mon[193606]: failed to encode map e452 with expected crc 2023-09-23T23:28:05.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:05 smithi138 ceph-mon[193606]: failed to encode map e452 with expected crc 2023-09-23T23:28:05.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:05 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:28:05.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:05 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:28:05.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:05 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:05.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:05 smithi138 ceph-mon[193606]: osdmap e453: 8 total, 8 up, 7 in 2023-09-23T23:28:05.361 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:05.420 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:28:05.471 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:05.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:05.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:05.556 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[221102]: pgmap v1257: 129 pgs: 2 peering, 3 active+remapped+backfill_wait, 124 active+clean; 588 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 406 op/s; 1231/28284 objects misplaced (4.352%); 23 MiB/s, 371 objects/s recovering 2023-09-23T23:28:05.557 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[221102]: failed to encode map e452 with expected crc 2023-09-23T23:28:05.557 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[221102]: failed to encode map e452 with expected crc 2023-09-23T23:28:05.557 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[221102]: osdmap e452: 8 total, 8 up, 7 in 2023-09-23T23:28:05.557 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[221102]: failed to encode map e452 with expected crc 2023-09-23T23:28:05.557 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[221102]: failed to encode map e452 with expected crc 2023-09-23T23:28:05.557 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:28:05.557 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:28:05.557 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:05.557 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[221102]: osdmap e453: 8 total, 8 up, 7 in 2023-09-23T23:28:05.558 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[212939]: pgmap v1257: 129 pgs: 2 peering, 3 active+remapped+backfill_wait, 124 active+clean; 588 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 406 op/s; 1231/28284 objects misplaced (4.352%); 23 MiB/s, 371 objects/s recovering 2023-09-23T23:28:05.558 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[212939]: failed to encode map e452 with expected crc 2023-09-23T23:28:05.558 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[212939]: failed to encode map e452 with expected crc 2023-09-23T23:28:05.558 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[212939]: osdmap e452: 8 total, 8 up, 7 in 2023-09-23T23:28:05.558 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[212939]: failed to encode map e452 with expected crc 2023-09-23T23:28:05.559 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[212939]: failed to encode map e452 with expected crc 2023-09-23T23:28:05.559 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:28:05.559 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:28:05.559 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:05.559 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:05 smithi110 ceph-mon[212939]: osdmap e453: 8 total, 8 up, 7 in 2023-09-23T23:28:05.577 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:05.675 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:05.809 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:05.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:05.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:05.927 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:06.021 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:06.133 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:06.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:06.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:06.246 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:06.274 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:06.326 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:06.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:06.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:06.379 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:28:06.406 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[212939]: failed to encode map e453 with expected crc 2023-09-23T23:28:06.406 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[212939]: failed to encode map e453 with expected crc 2023-09-23T23:28:06.406 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[212939]: failed to encode map e453 with expected crc 2023-09-23T23:28:06.406 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[212939]: 1.1a scrub starts 2023-09-23T23:28:06.407 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[212939]: 1.1a scrub ok 2023-09-23T23:28:06.407 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[212939]: 11.16 scrub starts 2023-09-23T23:28:06.407 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[212939]: 11.16 scrub ok 2023-09-23T23:28:06.407 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2205028841' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:06.407 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:06.407 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[221102]: failed to encode map e453 with expected crc 2023-09-23T23:28:06.407 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[221102]: failed to encode map e453 with expected crc 2023-09-23T23:28:06.407 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[221102]: failed to encode map e453 with expected crc 2023-09-23T23:28:06.408 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[221102]: 1.1a scrub starts 2023-09-23T23:28:06.408 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[221102]: 1.1a scrub ok 2023-09-23T23:28:06.408 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[221102]: 11.16 scrub starts 2023-09-23T23:28:06.408 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[221102]: 11.16 scrub ok 2023-09-23T23:28:06.408 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2205028841' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:06.408 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:06 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:06.434 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:06.465 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:06.600 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:06 smithi138 ceph-mon[193606]: failed to encode map e453 with expected crc 2023-09-23T23:28:06.600 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:06 smithi138 ceph-mon[193606]: failed to encode map e453 with expected crc 2023-09-23T23:28:06.601 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:06 smithi138 ceph-mon[193606]: failed to encode map e453 with expected crc 2023-09-23T23:28:06.601 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:06 smithi138 ceph-mon[193606]: 1.1a scrub starts 2023-09-23T23:28:06.601 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:06 smithi138 ceph-mon[193606]: 1.1a scrub ok 2023-09-23T23:28:06.601 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:06 smithi138 ceph-mon[193606]: 11.16 scrub starts 2023-09-23T23:28:06.601 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:06 smithi138 ceph-mon[193606]: 11.16 scrub ok 2023-09-23T23:28:06.601 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:06 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2205028841' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:06.601 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:06 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:06.623 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:06.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:06.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:06.731 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:06.762 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:06.817 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:06.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:06.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:06.922 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:06.966 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:07.075 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:07.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:07.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:07.139 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:28:07.139 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:28:07.139 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:28:07.139 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:28:07.139 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:28:07.179 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:07.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:28:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:28:06] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-09-23T23:28:07.206 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:07 smithi138 ceph-mon[193606]: pgmap v1260: 129 pgs: 1 active+remapped+backfilling, 2 peering, 126 active+clean; 486 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 180/23391 objects misplaced (0.770%); 23 MiB/s, 364 objects/s recovering 2023-09-23T23:28:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:07 smithi138 ceph-mon[193606]: 1.6 scrub starts 2023-09-23T23:28:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:07 smithi138 ceph-mon[193606]: 1.6 scrub ok 2023-09-23T23:28:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:07 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:28:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:07 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:28:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:07 smithi138 ceph-mon[193606]: 1.18 scrub starts 2023-09-23T23:28:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:07 smithi138 ceph-mon[193606]: 1.18 scrub ok 2023-09-23T23:28:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:07 smithi138 ceph-mon[193606]: failed to encode map e454 with expected crc 2023-09-23T23:28:07.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:07 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:28:07.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:07 smithi138 ceph-mon[193606]: osdmap e454: 8 total, 8 up, 7 in 2023-09-23T23:28:07.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:07 smithi138 ceph-mon[193606]: failed to encode map e454 with expected crc 2023-09-23T23:28:07.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:07 smithi138 ceph-mon[193606]: 11.15 scrub starts 2023-09-23T23:28:07.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:07 smithi138 ceph-mon[193606]: 11.15 scrub ok 2023-09-23T23:28:07.468 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:07.475 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[212939]: pgmap v1260: 129 pgs: 1 active+remapped+backfilling, 2 peering, 126 active+clean; 486 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 180/23391 objects misplaced (0.770%); 23 MiB/s, 364 objects/s recovering 2023-09-23T23:28:07.475 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[212939]: 1.6 scrub starts 2023-09-23T23:28:07.475 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[212939]: 1.6 scrub ok 2023-09-23T23:28:07.476 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:28:07.476 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:28:07.476 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[212939]: 1.18 scrub starts 2023-09-23T23:28:07.476 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[212939]: 1.18 scrub ok 2023-09-23T23:28:07.476 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[212939]: failed to encode map e454 with expected crc 2023-09-23T23:28:07.476 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:28:07.476 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[212939]: osdmap e454: 8 total, 8 up, 7 in 2023-09-23T23:28:07.476 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[212939]: failed to encode map e454 with expected crc 2023-09-23T23:28:07.476 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[212939]: 11.15 scrub starts 2023-09-23T23:28:07.476 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[212939]: 11.15 scrub ok 2023-09-23T23:28:07.476 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[221102]: pgmap v1260: 129 pgs: 1 active+remapped+backfilling, 2 peering, 126 active+clean; 486 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 180/23391 objects misplaced (0.770%); 23 MiB/s, 364 objects/s recovering 2023-09-23T23:28:07.477 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[221102]: 1.6 scrub starts 2023-09-23T23:28:07.477 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[221102]: 1.6 scrub ok 2023-09-23T23:28:07.477 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:28:07.477 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:28:07.477 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[221102]: 1.18 scrub starts 2023-09-23T23:28:07.477 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[221102]: 1.18 scrub ok 2023-09-23T23:28:07.477 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[221102]: failed to encode map e454 with expected crc 2023-09-23T23:28:07.477 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:28:07.477 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[221102]: osdmap e454: 8 total, 8 up, 7 in 2023-09-23T23:28:07.477 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[221102]: failed to encode map e454 with expected crc 2023-09-23T23:28:07.477 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[221102]: 11.15 scrub starts 2023-09-23T23:28:07.478 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:07 smithi110 ceph-mon[221102]: 11.15 scrub ok 2023-09-23T23:28:07.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:07.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:07.579 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:07.607 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:07.665 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:07.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:07.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:07.769 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:07.859 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:08.012 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:28:08.051 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:08.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:08.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:08.121 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:28:08.161 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:08.168 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:28:08.217 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:28:08.261 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:08.275 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:28:08.325 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:28:08.359 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:08.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:08.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:08.382 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:28:08.432 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:28:08.459 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:28:08.463 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:08.492 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:28:08.494 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:28:08.495 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:28:08.495 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:28:08.495 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:28:08.495 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:28:08.495 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:28:08.495 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:28:08.495 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:28:08.499 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:08.551 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:28:08.563 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:08.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:08.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:08.673 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:08.691 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:08 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2615860888' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:08.691 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:08 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2615860888' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:08.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:08 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2615860888' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:08.702 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:08.819 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:08.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:08.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:08.924 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:08.950 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:08.993 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:08.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:08.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:09.096 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:09.124 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:09.169 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:09.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:09.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:09.272 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:09.307 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:09.370 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:09.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:09.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:09.475 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:09.553 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:09.591 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:28:09.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:09 smithi138 ceph-mon[193606]: pgmap v1262: 129 pgs: 1 active+remapped+backfilling, 2 peering, 126 active+clean; 463 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 180/22293 objects misplaced (0.807%) 2023-09-23T23:28:09.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:09 smithi138 ceph-mon[193606]: 11.17 scrub starts 2023-09-23T23:28:09.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:09 smithi138 ceph-mon[193606]: 11.17 scrub ok 2023-09-23T23:28:09.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:09 smithi110 ceph-mon[221102]: pgmap v1262: 129 pgs: 1 active+remapped+backfilling, 2 peering, 126 active+clean; 463 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 180/22293 objects misplaced (0.807%) 2023-09-23T23:28:09.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:09 smithi110 ceph-mon[221102]: 11.17 scrub starts 2023-09-23T23:28:09.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:09 smithi110 ceph-mon[221102]: 11.17 scrub ok 2023-09-23T23:28:09.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:09 smithi110 ceph-mon[212939]: pgmap v1262: 129 pgs: 1 active+remapped+backfilling, 2 peering, 126 active+clean; 463 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 180/22293 objects misplaced (0.807%) 2023-09-23T23:28:09.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:09 smithi110 ceph-mon[212939]: 11.17 scrub starts 2023-09-23T23:28:09.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:09 smithi110 ceph-mon[212939]: 11.17 scrub ok 2023-09-23T23:28:09.710 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:09.711 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:09.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:09.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:09.822 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:09.850 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:09.906 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:09.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:09.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:10.019 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:10.053 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:10.104 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:10.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:10.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:10.209 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:10.264 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:10.390 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:10 smithi138 ceph-mon[193606]: 1.17 deep-scrub starts 2023-09-23T23:28:10.391 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:10 smithi138 ceph-mon[193606]: 1.17 deep-scrub ok 2023-09-23T23:28:10.391 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:10 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T23:28:10.391 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:10 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T23:28:10.391 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:10 smithi138 ceph-mon[193606]: 11.4 scrub starts 2023-09-23T23:28:10.391 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:10 smithi138 ceph-mon[193606]: 11.4 scrub ok 2023-09-23T23:28:10.433 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:10.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:10.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:10.503 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:28:10.503 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:28:10.540 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:10.572 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:10.623 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:10.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:10.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:10.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:10 smithi110 ceph-mon[212939]: 1.17 deep-scrub starts 2023-09-23T23:28:10.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:10 smithi110 ceph-mon[212939]: 1.17 deep-scrub ok 2023-09-23T23:28:10.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:10 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T23:28:10.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:10 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T23:28:10.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:10 smithi110 ceph-mon[212939]: 11.4 scrub starts 2023-09-23T23:28:10.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:10 smithi110 ceph-mon[212939]: 11.4 scrub ok 2023-09-23T23:28:10.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:10 smithi110 ceph-mon[221102]: 1.17 deep-scrub starts 2023-09-23T23:28:10.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:10 smithi110 ceph-mon[221102]: 1.17 deep-scrub ok 2023-09-23T23:28:10.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:10 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T23:28:10.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:10 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T23:28:10.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:10 smithi110 ceph-mon[221102]: 11.4 scrub starts 2023-09-23T23:28:10.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:10 smithi110 ceph-mon[221102]: 11.4 scrub ok 2023-09-23T23:28:10.727 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:10.754 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:10.797 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:10.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:10.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:10.900 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:10.928 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:10.971 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:10.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:10.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:11.075 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:11.105 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:11.259 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:28:11.270 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:11.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:11.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:11.377 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:11.390 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:11 smithi138 ceph-mon[193606]: pgmap v1263: 129 pgs: 129 active+clean; 349 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail 2023-09-23T23:28:11.390 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:11 smithi138 ceph-mon[193606]: 1.f scrub starts 2023-09-23T23:28:11.390 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:11 smithi138 ceph-mon[193606]: 1.f scrub ok 2023-09-23T23:28:11.390 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:11 smithi138 ceph-mon[193606]: failed to encode map e455 with expected crc 2023-09-23T23:28:11.390 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:11 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T23:28:11.390 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:11 smithi138 ceph-mon[193606]: osdmap e455: 8 total, 8 up, 7 in 2023-09-23T23:28:11.390 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:11 smithi138 ceph-mon[193606]: failed to encode map e455 with expected crc 2023-09-23T23:28:11.407 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:11.408 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:11.454 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:11.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:11.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:11.560 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:11.586 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:11.596 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:28:11.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:11 smithi110 ceph-mon[212939]: pgmap v1263: 129 pgs: 129 active+clean; 349 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail 2023-09-23T23:28:11.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:11 smithi110 ceph-mon[212939]: 1.f scrub starts 2023-09-23T23:28:11.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:11 smithi110 ceph-mon[212939]: 1.f scrub ok 2023-09-23T23:28:11.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:11 smithi110 ceph-mon[212939]: failed to encode map e455 with expected crc 2023-09-23T23:28:11.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:11 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T23:28:11.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:11 smithi110 ceph-mon[212939]: osdmap e455: 8 total, 8 up, 7 in 2023-09-23T23:28:11.622 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:11 smithi110 ceph-mon[212939]: failed to encode map e455 with expected crc 2023-09-23T23:28:11.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:11 smithi110 ceph-mon[221102]: pgmap v1263: 129 pgs: 129 active+clean; 349 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail 2023-09-23T23:28:11.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:11 smithi110 ceph-mon[221102]: 1.f scrub starts 2023-09-23T23:28:11.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:11 smithi110 ceph-mon[221102]: 1.f scrub ok 2023-09-23T23:28:11.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:11 smithi110 ceph-mon[221102]: failed to encode map e455 with expected crc 2023-09-23T23:28:11.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:11 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T23:28:11.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:11 smithi110 ceph-mon[221102]: osdmap e455: 8 total, 8 up, 7 in 2023-09-23T23:28:11.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:11 smithi110 ceph-mon[221102]: failed to encode map e455 with expected crc 2023-09-23T23:28:11.628 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:11.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:11.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:11.734 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:11.760 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:11.801 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:11.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:11.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:11.906 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:11.932 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:11.973 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:11.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:11.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:12.077 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:12.104 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:12.147 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:12.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:12.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:12.255 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:12.290 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:12.338 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:12.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:12.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:12.356 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:28:12.358 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:28:12.398 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:12.442 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:12.473 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:12.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:12 smithi138 ceph-mon[193606]: 11.11 deep-scrub starts 2023-09-23T23:28:12.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:12 smithi138 ceph-mon[193606]: 1.11 scrub starts 2023-09-23T23:28:12.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:12 smithi138 ceph-mon[193606]: osdmap e456: 8 total, 8 up, 7 in 2023-09-23T23:28:12.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:12 smithi138 ceph-mon[193606]: failed to encode map e456 with expected crc 2023-09-23T23:28:12.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:12 smithi138 ceph-mon[193606]: failed to encode map e456 with expected crc 2023-09-23T23:28:12.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:12 smithi138 ceph-mon[193606]: 11.11 deep-scrub ok 2023-09-23T23:28:12.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:12 smithi138 ceph-mon[193606]: failed to encode map e456 with expected crc 2023-09-23T23:28:12.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:12 smithi138 ceph-mon[193606]: failed to encode map e456 with expected crc 2023-09-23T23:28:12.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:12 smithi138 ceph-mon[193606]: pgmap v1266: 129 pgs: 129 active+clean; 264 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:28:12.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:12 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T23:28:12.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:12 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T23:28:12.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:12 smithi138 ceph-mon[193606]: 11.13 scrub starts 2023-09-23T23:28:12.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:12 smithi138 ceph-mon[193606]: 11.13 scrub ok 2023-09-23T23:28:12.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:12 smithi138 ceph-mon[193606]: from='client.36126 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:28:12.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[221102]: 11.11 deep-scrub starts 2023-09-23T23:28:12.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[221102]: 1.11 scrub starts 2023-09-23T23:28:12.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[221102]: osdmap e456: 8 total, 8 up, 7 in 2023-09-23T23:28:12.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[221102]: failed to encode map e456 with expected crc 2023-09-23T23:28:12.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[221102]: failed to encode map e456 with expected crc 2023-09-23T23:28:12.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[221102]: 11.11 deep-scrub ok 2023-09-23T23:28:12.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[221102]: failed to encode map e456 with expected crc 2023-09-23T23:28:12.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[221102]: failed to encode map e456 with expected crc 2023-09-23T23:28:12.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[221102]: pgmap v1266: 129 pgs: 129 active+clean; 264 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:28:12.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T23:28:12.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T23:28:12.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[221102]: 11.13 scrub starts 2023-09-23T23:28:12.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[221102]: 11.13 scrub ok 2023-09-23T23:28:12.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[221102]: from='client.36126 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:28:12.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[212939]: 11.11 deep-scrub starts 2023-09-23T23:28:12.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[212939]: 1.11 scrub starts 2023-09-23T23:28:12.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[212939]: osdmap e456: 8 total, 8 up, 7 in 2023-09-23T23:28:12.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[212939]: failed to encode map e456 with expected crc 2023-09-23T23:28:12.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[212939]: failed to encode map e456 with expected crc 2023-09-23T23:28:12.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[212939]: 11.11 deep-scrub ok 2023-09-23T23:28:12.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[212939]: failed to encode map e456 with expected crc 2023-09-23T23:28:12.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[212939]: failed to encode map e456 with expected crc 2023-09-23T23:28:12.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[212939]: pgmap v1266: 129 pgs: 129 active+clean; 264 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:28:12.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T23:28:12.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T23:28:12.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[212939]: 11.13 scrub starts 2023-09-23T23:28:12.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[212939]: 11.13 scrub ok 2023-09-23T23:28:12.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:12 smithi110 ceph-mon[212939]: from='client.36126 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:28:12.737 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:12.903 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:12.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:12.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:13.008 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:28:13.055 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:13 smithi138 ceph-mon[193606]: 1.7 scrub starts 2023-09-23T23:28:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:13 smithi138 ceph-mon[193606]: 1.7 scrub ok 2023-09-23T23:28:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:13 smithi138 ceph-mon[193606]: failed to encode map e457 with expected crc 2023-09-23T23:28:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T23:28:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:13 smithi138 ceph-mon[193606]: osdmap e457: 8 total, 8 up, 7 in 2023-09-23T23:28:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:13 smithi138 ceph-mon[193606]: failed to encode map e457 with expected crc 2023-09-23T23:28:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:13 smithi138 ceph-mon[193606]: 11.12 starting backfill to osd.7 from (355'800,403'8095] MIN to 454'9813 2023-09-23T23:28:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:28:13.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:13 smithi138 ceph-mon[193606]: 11.12 scrub starts 2023-09-23T23:28:13.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:13 smithi138 ceph-mon[193606]: 11.12 scrub ok 2023-09-23T23:28:13.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[212939]: 1.7 scrub starts 2023-09-23T23:28:13.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[212939]: 1.7 scrub ok 2023-09-23T23:28:13.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[212939]: failed to encode map e457 with expected crc 2023-09-23T23:28:13.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T23:28:13.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[212939]: osdmap e457: 8 total, 8 up, 7 in 2023-09-23T23:28:13.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[212939]: failed to encode map e457 with expected crc 2023-09-23T23:28:13.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[212939]: 11.12 starting backfill to osd.7 from (355'800,403'8095] MIN to 454'9813 2023-09-23T23:28:13.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:13.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:28:13.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[212939]: 11.12 scrub starts 2023-09-23T23:28:13.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[212939]: 11.12 scrub ok 2023-09-23T23:28:13.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[221102]: 1.7 scrub starts 2023-09-23T23:28:13.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[221102]: 1.7 scrub ok 2023-09-23T23:28:13.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[221102]: failed to encode map e457 with expected crc 2023-09-23T23:28:13.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T23:28:13.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[221102]: osdmap e457: 8 total, 8 up, 7 in 2023-09-23T23:28:13.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[221102]: failed to encode map e457 with expected crc 2023-09-23T23:28:13.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[221102]: 11.12 starting backfill to osd.7 from (355'800,403'8095] MIN to 454'9813 2023-09-23T23:28:13.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:13.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:28:13.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[221102]: 11.12 scrub starts 2023-09-23T23:28:13.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:13 smithi110 ceph-mon[221102]: 11.12 scrub ok 2023-09-23T23:28:13.966 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:28:13.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:13.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:28:14.048 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1266,"stamp":"2023-09-23T23:28:11.718733+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":276791435,"num_objects":4240,"num_object_clones":0,"num_object_copies":12720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4240,"num_whiteouts":0,"num_read":24,"num_read_kb":43,"num_write":158655,"num_write_kb":4461452,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59374,"num_bytes_recovered":3779941027,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":162915,"ondisk_log_size":162915,"up":387,"acting":387,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":408,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6420348,"kb_used_data":1969476,"kb_used_omap":29,"kb_used_meta":4450530,"kb_avail":649795716,"statfs":{"total":671965249536,"available":665390813184,"internally_reserved":0,"allocated":2016743424,"data_stored":2004253854,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":4557343443},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,1,0,0,0,4],"upper_bound":512},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-232993625,"num_objects":-3557,"num_object_clones":0,"num_object_copies":-10671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-180,"num_objects_unfound":0,"num_objects_dirty":-3557,"num_whiteouts":0,"num_read":-11,"num_read_kb":-20,"num_write":-22137,"num_write_kb":-849847,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-7499,"num_bytes_recovered":-491757676,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001716"},"pg_stats":[{"pgid":"11.12","version":"453'9699","reported_seq":13755,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.921145+0000","last_change":"2023-09-23T23:28:01.327504+0000","last_active":"2023-09-23T23:28:05.921145+0000","last_peered":"2023-09-23T23:28:05.921145+0000","last_clean":"2023-09-23T23:28:05.921145+0000","last_became_active":"2023-09-23T23:26:42.095788+0000","last_became_peered":"2023-09-23T23:26:42.095788+0000","last_unstale":"2023-09-23T23:28:05.921145+0000","last_undegraded":"2023-09-23T23:28:05.921145+0000","last_fullsized":"2023-09-23T23:28:05.921145+0000","mapping_epoch":404,"log_start":"355'800","ondisk_log_start":"355'800","created":365,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":8899,"log_dups_size":0,"ondisk_log_size":8899,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17989632,"num_objects":275,"num_object_clones":0,"num_object_copies":825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6453,"num_write_kb":215264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1954,"num_bytes_recovered":128057344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":650,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.356894+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:28:06.356894+0000","last_peered":"2023-09-23T23:28:06.356894+0000","last_clean":"2023-09-23T23:28:06.356894+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:28:06.356894+0000","last_undegraded":"2023-09-23T23:28:06.356894+0000","last_fullsized":"2023-09-23T23:28:06.356894+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":488,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342926+0000","last_change":"2023-09-23T23:22:38.439629+0000","last_active":"2023-09-23T23:28:05.342926+0000","last_peered":"2023-09-23T23:28:05.342926+0000","last_clean":"2023-09-23T23:28:05.342926+0000","last_became_active":"2023-09-23T23:22:38.439285+0000","last_became_peered":"2023-09-23T23:22:38.439285+0000","last_unstale":"2023-09-23T23:28:05.342926+0000","last_undegraded":"2023-09-23T23:28:05.342926+0000","last_fullsized":"2023-09-23T23:28:05.342926+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:34.431221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":423,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342412+0000","last_change":"2023-09-23T23:23:43.511912+0000","last_active":"2023-09-23T23:28:05.342412+0000","last_peered":"2023-09-23T23:28:05.342412+0000","last_clean":"2023-09-23T23:28:05.342412+0000","last_became_active":"2023-09-23T23:23:43.511574+0000","last_became_peered":"2023-09-23T23:23:43.511574+0000","last_unstale":"2023-09-23T23:28:05.342412+0000","last_undegraded":"2023-09-23T23:28:05.342412+0000","last_fullsized":"2023-09-23T23:28:05.342412+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:16.286926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"455'9731","reported_seq":16487,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T23:28:10.542359+0000","last_change":"2023-09-23T23:28:00.282718+0000","last_active":"2023-09-23T23:28:10.542359+0000","last_peered":"2023-09-23T23:28:10.542359+0000","last_clean":"2023-09-23T23:28:10.542359+0000","last_became_active":"2023-09-23T23:27:34.078923+0000","last_became_peered":"2023-09-23T23:27:34.078923+0000","last_unstale":"2023-09-23T23:28:10.542359+0000","last_undegraded":"2023-09-23T23:28:10.542359+0000","last_fullsized":"2023-09-23T23:28:10.542359+0000","mapping_epoch":434,"log_start":"355'1400","ondisk_log_start":"355'1400","created":365,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":8331,"log_dups_size":0,"ondisk_log_size":8331,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5931008,"num_objects":91,"num_object_clones":0,"num_object_copies":273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":91,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6529,"num_write_kb":211808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2413,"num_bytes_recovered":158138368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":543,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.384956+0000","last_change":"2023-09-23T23:23:50.459634+0000","last_active":"2023-09-23T23:28:05.384956+0000","last_peered":"2023-09-23T23:28:05.384956+0000","last_clean":"2023-09-23T23:28:05.384956+0000","last_became_active":"2023-09-23T23:23:45.453958+0000","last_became_peered":"2023-09-23T23:23:45.453958+0000","last_unstale":"2023-09-23T23:28:05.384956+0000","last_undegraded":"2023-09-23T23:28:05.384956+0000","last_fullsized":"2023-09-23T23:28:05.384956+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:56.563201+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":164,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.344823+0000","last_change":"2023-09-23T23:26:11.528846+0000","last_active":"2023-09-23T23:28:05.344823+0000","last_peered":"2023-09-23T23:28:05.344823+0000","last_clean":"2023-09-23T23:28:05.344823+0000","last_became_active":"2023-09-23T23:26:11.528529+0000","last_became_peered":"2023-09-23T23:26:11.528529+0000","last_unstale":"2023-09-23T23:28:05.344823+0000","last_undegraded":"2023-09-23T23:28:05.344823+0000","last_fullsized":"2023-09-23T23:28:05.344823+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:35.094583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":473,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.372358+0000","last_change":"2023-09-23T23:26:48.558967+0000","last_active":"2023-09-23T23:28:06.372358+0000","last_peered":"2023-09-23T23:28:06.372358+0000","last_clean":"2023-09-23T23:28:06.372358+0000","last_became_active":"2023-09-23T23:26:48.555271+0000","last_became_peered":"2023-09-23T23:26:48.555271+0000","last_unstale":"2023-09-23T23:28:06.372358+0000","last_undegraded":"2023-09-23T23:28:06.372358+0000","last_fullsized":"2023-09-23T23:28:06.372358+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:50:57.509734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"455'9617","reported_seq":23135,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T23:28:10.555249+0000","last_change":"2023-09-23T23:27:24.606995+0000","last_active":"2023-09-23T23:28:10.555249+0000","last_peered":"2023-09-23T23:28:10.555249+0000","last_clean":"2023-09-23T23:28:10.555249+0000","last_became_active":"2023-09-23T23:27:24.589817+0000","last_became_peered":"2023-09-23T23:27:24.589817+0000","last_unstale":"2023-09-23T23:28:10.555249+0000","last_undegraded":"2023-09-23T23:28:10.555249+0000","last_fullsized":"2023-09-23T23:28:10.555249+0000","mapping_epoch":428,"log_start":"356'2538","ondisk_log_start":"356'2538","created":365,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"366'6410","last_scrub_stamp":"2023-09-23T23:25:21.346991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:21.346991+0000","objects_scrubbed":3231,"log_size":7079,"log_dups_size":0,"ondisk_log_size":7079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:37.469662+0000","scrub_duration":0.84688343099999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5832704,"num_objects":89,"num_object_clones":0,"num_object_copies":267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3222,"num_write_kb":2560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3371,"num_bytes_recovered":208666624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":757,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.634892+0000","last_change":"2023-09-23T23:28:05.634824+0000","last_active":"2023-09-23T23:28:05.634892+0000","last_peered":"2023-09-23T23:28:05.634892+0000","last_clean":"2023-09-23T23:28:05.634892+0000","last_became_active":"2023-09-23T23:27:23.551046+0000","last_became_peered":"2023-09-23T23:27:23.551046+0000","last_unstale":"2023-09-23T23:28:05.634892+0000","last_undegraded":"2023-09-23T23:28:05.634892+0000","last_fullsized":"2023-09-23T23:28:05.634892+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:59.062239+0000","scrub_duration":0.0034158959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":451,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342654+0000","last_change":"2023-09-23T23:27:20.554007+0000","last_active":"2023-09-23T23:28:05.342654+0000","last_peered":"2023-09-23T23:28:05.342654+0000","last_clean":"2023-09-23T23:28:05.342654+0000","last_became_active":"2023-09-23T23:27:20.553716+0000","last_became_peered":"2023-09-23T23:27:20.553716+0000","last_unstale":"2023-09-23T23:28:05.342654+0000","last_undegraded":"2023-09-23T23:28:05.342654+0000","last_fullsized":"2023-09-23T23:28:05.342654+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:38.911862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":387,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.345735+0000","last_change":"2023-09-23T23:25:31.735427+0000","last_active":"2023-09-23T23:28:05.345735+0000","last_peered":"2023-09-23T23:28:05.345735+0000","last_clean":"2023-09-23T23:28:05.345735+0000","last_became_active":"2023-09-23T23:25:31.734582+0000","last_became_peered":"2023-09-23T23:25:31.734582+0000","last_unstale":"2023-09-23T23:28:05.345735+0000","last_undegraded":"2023-09-23T23:28:05.345735+0000","last_fullsized":"2023-09-23T23:28:05.345735+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:39.075149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":774,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342871+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:28:05.342871+0000","last_peered":"2023-09-23T23:28:05.342871+0000","last_clean":"2023-09-23T23:28:05.342871+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:28:05.342871+0000","last_undegraded":"2023-09-23T23:28:05.342871+0000","last_fullsized":"2023-09-23T23:28:05.342871+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":408,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.372521+0000","last_change":"2023-09-23T23:10:59.513146+0000","last_active":"2023-09-23T23:28:06.372521+0000","last_peered":"2023-09-23T23:28:06.372521+0000","last_clean":"2023-09-23T23:28:06.372521+0000","last_became_active":"2023-09-23T23:10:59.512703+0000","last_became_peered":"2023-09-23T23:10:59.512703+0000","last_unstale":"2023-09-23T23:28:06.372521+0000","last_undegraded":"2023-09-23T23:28:06.372521+0000","last_fullsized":"2023-09-23T23:28:06.372521+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:17.444783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":442,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342547+0000","last_change":"2023-09-23T23:26:11.528354+0000","last_active":"2023-09-23T23:28:05.342547+0000","last_peered":"2023-09-23T23:28:05.342547+0000","last_clean":"2023-09-23T23:28:05.342547+0000","last_became_active":"2023-09-23T23:26:11.528076+0000","last_became_peered":"2023-09-23T23:26:11.528076+0000","last_unstale":"2023-09-23T23:28:05.342547+0000","last_undegraded":"2023-09-23T23:28:05.342547+0000","last_fullsized":"2023-09-23T23:28:05.342547+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:16.100396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":727,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.360335+0000","last_change":"2023-09-23T23:26:14.521792+0000","last_active":"2023-09-23T23:28:06.360335+0000","last_peered":"2023-09-23T23:28:06.360335+0000","last_clean":"2023-09-23T23:28:06.360335+0000","last_became_active":"2023-09-23T23:26:14.521438+0000","last_became_peered":"2023-09-23T23:26:14.521438+0000","last_unstale":"2023-09-23T23:28:06.360335+0000","last_undegraded":"2023-09-23T23:28:06.360335+0000","last_fullsized":"2023-09-23T23:28:06.360335+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:09.103712+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":85,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.385029+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:28:05.385029+0000","last_peered":"2023-09-23T23:28:05.385029+0000","last_clean":"2023-09-23T23:28:05.385029+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:28:05.385029+0000","last_undegraded":"2023-09-23T23:28:05.385029+0000","last_fullsized":"2023-09-23T23:28:05.385029+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":403,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342259+0000","last_change":"2023-09-23T23:26:48.538059+0000","last_active":"2023-09-23T23:28:05.342259+0000","last_peered":"2023-09-23T23:28:05.342259+0000","last_clean":"2023-09-23T23:28:05.342259+0000","last_became_active":"2023-09-23T23:26:48.537735+0000","last_became_peered":"2023-09-23T23:26:48.537735+0000","last_unstale":"2023-09-23T23:28:05.342259+0000","last_undegraded":"2023-09-23T23:28:05.342259+0000","last_fullsized":"2023-09-23T23:28:05.342259+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:32.878591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":838,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.385214+0000","last_change":"2023-09-23T23:26:51.563894+0000","last_active":"2023-09-23T23:28:05.385214+0000","last_peered":"2023-09-23T23:28:05.385214+0000","last_clean":"2023-09-23T23:28:05.385214+0000","last_became_active":"2023-09-23T23:26:51.563536+0000","last_became_peered":"2023-09-23T23:26:51.563536+0000","last_unstale":"2023-09-23T23:28:05.385214+0000","last_undegraded":"2023-09-23T23:28:05.385214+0000","last_fullsized":"2023-09-23T23:28:05.385214+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:54.095054+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:54.095054+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":59,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342312+0000","last_change":"2023-09-23T23:26:48.669857+0000","last_active":"2023-09-23T23:28:05.342312+0000","last_peered":"2023-09-23T23:28:05.342312+0000","last_clean":"2023-09-23T23:28:05.342312+0000","last_became_active":"2023-09-23T23:26:48.538246+0000","last_became_peered":"2023-09-23T23:26:48.538246+0000","last_unstale":"2023-09-23T23:28:05.342312+0000","last_undegraded":"2023-09-23T23:28:05.342312+0000","last_fullsized":"2023-09-23T23:28:05.342312+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:19.039472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":40,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.345055+0000","last_change":"2023-09-23T23:27:20.551305+0000","last_active":"2023-09-23T23:28:05.345055+0000","last_peered":"2023-09-23T23:28:05.345055+0000","last_clean":"2023-09-23T23:28:05.345055+0000","last_became_active":"2023-09-23T23:27:20.551017+0000","last_became_peered":"2023-09-23T23:27:20.551017+0000","last_unstale":"2023-09-23T23:28:05.345055+0000","last_undegraded":"2023-09-23T23:28:05.345055+0000","last_fullsized":"2023-09-23T23:28:05.345055+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:27.026015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":528,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.345661+0000","last_change":"2023-09-23T23:24:02.268079+0000","last_active":"2023-09-23T23:28:05.345661+0000","last_peered":"2023-09-23T23:28:05.345661+0000","last_clean":"2023-09-23T23:28:05.345661+0000","last_became_active":"2023-09-23T23:23:45.463106+0000","last_became_peered":"2023-09-23T23:23:45.463106+0000","last_unstale":"2023-09-23T23:28:05.345661+0000","last_undegraded":"2023-09-23T23:28:05.345661+0000","last_fullsized":"2023-09-23T23:28:05.345661+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.128813+0000","scrub_duration":0.0018535310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":85,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.343790+0000","last_change":"2023-09-23T23:26:11.534727+0000","last_active":"2023-09-23T23:28:05.343790+0000","last_peered":"2023-09-23T23:28:05.343790+0000","last_clean":"2023-09-23T23:28:05.343790+0000","last_became_active":"2023-09-23T23:26:11.513694+0000","last_became_peered":"2023-09-23T23:26:11.513694+0000","last_unstale":"2023-09-23T23:28:05.343790+0000","last_undegraded":"2023-09-23T23:28:05.343790+0000","last_fullsized":"2023-09-23T23:28:05.343790+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:04.562837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":442,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.343699+0000","last_change":"2023-09-23T23:25:31.718910+0000","last_active":"2023-09-23T23:28:05.343699+0000","last_peered":"2023-09-23T23:28:05.343699+0000","last_clean":"2023-09-23T23:28:05.343699+0000","last_became_active":"2023-09-23T23:25:31.718567+0000","last_became_peered":"2023-09-23T23:25:31.718567+0000","last_unstale":"2023-09-23T23:28:05.343699+0000","last_undegraded":"2023-09-23T23:28:05.343699+0000","last_fullsized":"2023-09-23T23:28:05.343699+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:21.447944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":556,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.384668+0000","last_change":"2023-09-23T23:28:02.287254+0000","last_active":"2023-09-23T23:28:05.384668+0000","last_peered":"2023-09-23T23:28:05.384668+0000","last_clean":"2023-09-23T23:28:05.384668+0000","last_became_active":"2023-09-23T23:26:34.550151+0000","last_became_peered":"2023-09-23T23:26:34.550151+0000","last_unstale":"2023-09-23T23:28:05.384668+0000","last_undegraded":"2023-09-23T23:28:05.384668+0000","last_fullsized":"2023-09-23T23:28:05.384668+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:53.828311+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:53.828311+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":386,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342906+0000","last_change":"2023-09-23T23:26:11.569228+0000","last_active":"2023-09-23T23:28:05.342906+0000","last_peered":"2023-09-23T23:28:05.342906+0000","last_clean":"2023-09-23T23:28:05.342906+0000","last_became_active":"2023-09-23T23:26:11.513159+0000","last_became_peered":"2023-09-23T23:26:11.513159+0000","last_unstale":"2023-09-23T23:28:05.342906+0000","last_undegraded":"2023-09-23T23:28:05.342906+0000","last_fullsized":"2023-09-23T23:28:05.342906+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:40.698257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":450,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342143+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:28:05.342143+0000","last_peered":"2023-09-23T23:28:05.342143+0000","last_clean":"2023-09-23T23:28:05.342143+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:28:05.342143+0000","last_undegraded":"2023-09-23T23:28:05.342143+0000","last_fullsized":"2023-09-23T23:28:05.342143+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":793,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.341564+0000","last_change":"2023-09-23T23:26:34.551458+0000","last_active":"2023-09-23T23:28:05.341564+0000","last_peered":"2023-09-23T23:28:05.341564+0000","last_clean":"2023-09-23T23:28:05.341564+0000","last_became_active":"2023-09-23T23:26:34.551067+0000","last_became_peered":"2023-09-23T23:26:34.551067+0000","last_unstale":"2023-09-23T23:28:05.341564+0000","last_undegraded":"2023-09-23T23:28:05.341564+0000","last_fullsized":"2023-09-23T23:28:05.341564+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:05.788656+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342049+0000","last_change":"2023-09-23T23:26:48.563723+0000","last_active":"2023-09-23T23:28:05.342049+0000","last_peered":"2023-09-23T23:28:05.342049+0000","last_clean":"2023-09-23T23:28:05.342049+0000","last_became_active":"2023-09-23T23:26:48.563340+0000","last_became_peered":"2023-09-23T23:26:48.563340+0000","last_unstale":"2023-09-23T23:28:05.342049+0000","last_undegraded":"2023-09-23T23:28:05.342049+0000","last_fullsized":"2023-09-23T23:28:05.342049+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:05.974131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":340,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342790+0000","last_change":"2023-09-23T23:26:32.040316+0000","last_active":"2023-09-23T23:28:05.342790+0000","last_peered":"2023-09-23T23:28:05.342790+0000","last_clean":"2023-09-23T23:28:05.342790+0000","last_became_active":"2023-09-23T23:26:32.039758+0000","last_became_peered":"2023-09-23T23:26:32.039758+0000","last_unstale":"2023-09-23T23:28:05.342790+0000","last_undegraded":"2023-09-23T23:28:05.342790+0000","last_fullsized":"2023-09-23T23:28:05.342790+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:03.930331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.19","version":"454'11428","reported_seq":17061,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:08.756866+0000","last_change":"2023-09-23T23:28:04.367177+0000","last_active":"2023-09-23T23:28:08.756866+0000","last_peered":"2023-09-23T23:28:08.756866+0000","last_clean":"2023-09-23T23:28:08.756866+0000","last_became_active":"2023-09-23T23:28:04.349181+0000","last_became_peered":"2023-09-23T23:28:04.349181+0000","last_unstale":"2023-09-23T23:28:08.756866+0000","last_undegraded":"2023-09-23T23:28:08.756866+0000","last_fullsized":"2023-09-23T23:28:08.756866+0000","mapping_epoch":451,"log_start":"368'6929","ondisk_log_start":"368'6929","created":365,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":1949,"log_size":4499,"log_dups_size":0,"ondisk_log_size":4499,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3768320,"num_objects":58,"num_object_clones":0,"num_object_copies":174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6612,"num_write_kb":213408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2649,"num_bytes_recovered":173015040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":617,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.344282+0000","last_change":"2023-09-23T23:26:14.527752+0000","last_active":"2023-09-23T23:28:05.344282+0000","last_peered":"2023-09-23T23:28:05.344282+0000","last_clean":"2023-09-23T23:28:05.344282+0000","last_became_active":"2023-09-23T23:26:14.525188+0000","last_became_peered":"2023-09-23T23:26:14.525188+0000","last_unstale":"2023-09-23T23:28:05.344282+0000","last_undegraded":"2023-09-23T23:28:05.344282+0000","last_fullsized":"2023-09-23T23:28:05.344282+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:10.617744+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":308,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342945+0000","last_change":"2023-09-23T23:25:31.725776+0000","last_active":"2023-09-23T23:28:05.342945+0000","last_peered":"2023-09-23T23:28:05.342945+0000","last_clean":"2023-09-23T23:28:05.342945+0000","last_became_active":"2023-09-23T23:25:31.725429+0000","last_became_peered":"2023-09-23T23:25:31.725429+0000","last_unstale":"2023-09-23T23:28:05.342945+0000","last_undegraded":"2023-09-23T23:28:05.342945+0000","last_fullsized":"2023-09-23T23:28:05.342945+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:07.289251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":444,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.372463+0000","last_change":"2023-09-23T23:26:48.559268+0000","last_active":"2023-09-23T23:28:06.372463+0000","last_peered":"2023-09-23T23:28:06.372463+0000","last_clean":"2023-09-23T23:28:06.372463+0000","last_became_active":"2023-09-23T23:26:48.558563+0000","last_became_peered":"2023-09-23T23:26:48.558563+0000","last_unstale":"2023-09-23T23:28:06.372463+0000","last_undegraded":"2023-09-23T23:28:06.372463+0000","last_fullsized":"2023-09-23T23:28:06.372463+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:58.779288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"454'9670","reported_seq":15497,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:08.722786+0000","last_change":"2023-09-23T23:27:30.609575+0000","last_active":"2023-09-23T23:28:08.722786+0000","last_peered":"2023-09-23T23:28:08.722786+0000","last_clean":"2023-09-23T23:28:08.722786+0000","last_became_active":"2023-09-23T23:27:30.579515+0000","last_became_peered":"2023-09-23T23:27:30.579515+0000","last_unstale":"2023-09-23T23:28:08.722786+0000","last_undegraded":"2023-09-23T23:28:08.722786+0000","last_fullsized":"2023-09-23T23:28:08.722786+0000","mapping_epoch":432,"log_start":"359'4265","ondisk_log_start":"359'4265","created":365,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5405,"log_dups_size":0,"ondisk_log_size":5405,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3178496,"num_objects":49,"num_object_clones":0,"num_object_copies":147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":49,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6497,"num_write_kb":209440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2724,"num_bytes_recovered":178520064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":797,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.385154+0000","last_change":"2023-09-23T23:27:23.549144+0000","last_active":"2023-09-23T23:28:05.385154+0000","last_peered":"2023-09-23T23:28:05.385154+0000","last_clean":"2023-09-23T23:28:05.385154+0000","last_became_active":"2023-09-23T23:27:23.548774+0000","last_became_peered":"2023-09-23T23:27:23.548774+0000","last_unstale":"2023-09-23T23:28:05.385154+0000","last_undegraded":"2023-09-23T23:28:05.385154+0000","last_fullsized":"2023-09-23T23:28:05.385154+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:18.211722+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":136,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.370490+0000","last_change":"2023-09-23T23:26:32.139091+0000","last_active":"2023-09-23T23:28:06.370490+0000","last_peered":"2023-09-23T23:28:06.370490+0000","last_clean":"2023-09-23T23:28:06.370490+0000","last_became_active":"2023-09-23T23:26:32.053033+0000","last_became_peered":"2023-09-23T23:26:32.053033+0000","last_unstale":"2023-09-23T23:28:06.370490+0000","last_undegraded":"2023-09-23T23:28:06.370490+0000","last_fullsized":"2023-09-23T23:28:06.370490+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:32.625818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":91,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342137+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:28:05.342137+0000","last_peered":"2023-09-23T23:28:05.342137+0000","last_clean":"2023-09-23T23:28:05.342137+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:28:05.342137+0000","last_undegraded":"2023-09-23T23:28:05.342137+0000","last_fullsized":"2023-09-23T23:28:05.342137+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"454'9622","reported_seq":14421,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:09.778349+0000","last_change":"2023-09-23T23:27:57.334959+0000","last_active":"2023-09-23T23:28:09.778349+0000","last_peered":"2023-09-23T23:28:09.778349+0000","last_clean":"2023-09-23T23:28:09.778349+0000","last_became_active":"2023-09-23T23:27:57.299658+0000","last_became_peered":"2023-09-23T23:27:57.299658+0000","last_unstale":"2023-09-23T23:28:09.778349+0000","last_undegraded":"2023-09-23T23:28:09.778349+0000","last_fullsized":"2023-09-23T23:28:09.778349+0000","mapping_epoch":444,"log_start":"358'3971","ondisk_log_start":"358'3971","created":349,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"367'6563","last_scrub_stamp":"2023-09-23T23:25:25.026782+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:25.026782+0000","objects_scrubbed":3175,"log_size":5651,"log_dups_size":0,"ondisk_log_size":5651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:52.733728+0000","scrub_duration":0.41131605999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7340032,"num_objects":112,"num_object_clones":0,"num_object_copies":336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3063,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2248,"num_bytes_recovered":147324928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":781,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342388+0000","last_change":"2023-09-23T23:26:51.563868+0000","last_active":"2023-09-23T23:28:05.342388+0000","last_peered":"2023-09-23T23:28:05.342388+0000","last_clean":"2023-09-23T23:28:05.342388+0000","last_became_active":"2023-09-23T23:26:51.563510+0000","last_became_peered":"2023-09-23T23:26:51.563510+0000","last_unstale":"2023-09-23T23:28:05.342388+0000","last_undegraded":"2023-09-23T23:28:05.342388+0000","last_fullsized":"2023-09-23T23:28:05.342388+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:34.244822+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":57,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.344229+0000","last_change":"2023-09-23T23:26:48.673221+0000","last_active":"2023-09-23T23:28:05.344229+0000","last_peered":"2023-09-23T23:28:05.344229+0000","last_clean":"2023-09-23T23:28:05.344229+0000","last_became_active":"2023-09-23T23:26:48.555920+0000","last_became_peered":"2023-09-23T23:26:48.555920+0000","last_unstale":"2023-09-23T23:28:05.344229+0000","last_undegraded":"2023-09-23T23:28:05.344229+0000","last_fullsized":"2023-09-23T23:28:05.344229+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:26.586824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":75,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.384778+0000","last_change":"2023-09-23T23:26:48.525190+0000","last_active":"2023-09-23T23:28:05.384778+0000","last_peered":"2023-09-23T23:28:05.384778+0000","last_clean":"2023-09-23T23:28:05.384778+0000","last_became_active":"2023-09-23T23:26:48.524874+0000","last_became_peered":"2023-09-23T23:26:48.524874+0000","last_unstale":"2023-09-23T23:28:05.384778+0000","last_undegraded":"2023-09-23T23:28:05.384778+0000","last_fullsized":"2023-09-23T23:28:05.384778+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:43.114769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"455'9763","reported_seq":10781,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T23:28:10.502426+0000","last_change":"2023-09-23T23:27:58.326500+0000","last_active":"2023-09-23T23:28:10.502426+0000","last_peered":"2023-09-23T23:28:10.502426+0000","last_clean":"2023-09-23T23:28:10.502426+0000","last_became_active":"2023-09-23T23:27:58.306249+0000","last_became_peered":"2023-09-23T23:27:58.306249+0000","last_unstale":"2023-09-23T23:28:10.502426+0000","last_undegraded":"2023-09-23T23:28:10.502426+0000","last_fullsized":"2023-09-23T23:28:10.502426+0000","mapping_epoch":445,"log_start":"356'2729","ondisk_log_start":"356'2729","created":349,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"366'6545","last_scrub_stamp":"2023-09-23T23:25:22.335514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:22.335514+0000","objects_scrubbed":3321,"log_size":7034,"log_dups_size":0,"ondisk_log_size":7034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:10.290711+0000","scrub_duration":0.75924003299999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6750208,"num_objects":103,"num_object_clones":0,"num_object_copies":309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3222,"num_write_kb":128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":395,"num_bytes_recovered":25886720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":841,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.341924+0000","last_change":"2023-09-23T23:26:14.530414+0000","last_active":"2023-09-23T23:28:05.341924+0000","last_peered":"2023-09-23T23:28:05.341924+0000","last_clean":"2023-09-23T23:28:05.341924+0000","last_became_active":"2023-09-23T23:26:14.530001+0000","last_became_peered":"2023-09-23T23:26:14.530001+0000","last_unstale":"2023-09-23T23:28:05.341924+0000","last_undegraded":"2023-09-23T23:28:05.341924+0000","last_fullsized":"2023-09-23T23:28:05.341924+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:37.443998+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":57,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342620+0000","last_change":"2023-09-23T23:26:48.547146+0000","last_active":"2023-09-23T23:28:05.342620+0000","last_peered":"2023-09-23T23:28:05.342620+0000","last_clean":"2023-09-23T23:28:05.342620+0000","last_became_active":"2023-09-23T23:26:48.537682+0000","last_became_peered":"2023-09-23T23:26:48.537682+0000","last_unstale":"2023-09-23T23:28:05.342620+0000","last_undegraded":"2023-09-23T23:28:05.342620+0000","last_fullsized":"2023-09-23T23:28:05.342620+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:10.115563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":479,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342432+0000","last_change":"2023-09-23T23:26:01.833889+0000","last_active":"2023-09-23T23:28:05.342432+0000","last_peered":"2023-09-23T23:28:05.342432+0000","last_clean":"2023-09-23T23:28:05.342432+0000","last_became_active":"2023-09-23T23:26:01.833626+0000","last_became_peered":"2023-09-23T23:26:01.833626+0000","last_unstale":"2023-09-23T23:28:05.342432+0000","last_undegraded":"2023-09-23T23:28:05.342432+0000","last_fullsized":"2023-09-23T23:28:05.342432+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:25.659671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"454'9609","reported_seq":18334,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:08.049355+0000","last_change":"2023-09-23T23:27:57.809298+0000","last_active":"2023-09-23T23:28:08.049355+0000","last_peered":"2023-09-23T23:28:08.049355+0000","last_clean":"2023-09-23T23:28:08.049355+0000","last_became_active":"2023-09-23T23:27:42.114973+0000","last_became_peered":"2023-09-23T23:27:42.114973+0000","last_unstale":"2023-09-23T23:28:08.049355+0000","last_undegraded":"2023-09-23T23:28:08.049355+0000","last_fullsized":"2023-09-23T23:28:08.049355+0000","mapping_epoch":436,"log_start":"357'3599","ondisk_log_start":"357'3599","created":349,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":6010,"log_dups_size":0,"ondisk_log_size":6010,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13336576,"num_objects":203,"num_object_clones":0,"num_object_copies":609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6415,"num_write_kb":211808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3490,"num_bytes_recovered":228720640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":779,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.385216+0000","last_change":"2023-09-23T23:28:00.268186+0000","last_active":"2023-09-23T23:28:05.385216+0000","last_peered":"2023-09-23T23:28:05.385216+0000","last_clean":"2023-09-23T23:28:05.385216+0000","last_became_active":"2023-09-23T23:26:14.521164+0000","last_became_peered":"2023-09-23T23:26:14.521164+0000","last_unstale":"2023-09-23T23:28:05.385216+0000","last_undegraded":"2023-09-23T23:28:05.385216+0000","last_fullsized":"2023-09-23T23:28:05.385216+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.042580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.042580+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":360,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342613+0000","last_change":"2023-09-23T23:26:32.043926+0000","last_active":"2023-09-23T23:28:05.342613+0000","last_peered":"2023-09-23T23:28:05.342613+0000","last_clean":"2023-09-23T23:28:05.342613+0000","last_became_active":"2023-09-23T23:26:32.039282+0000","last_became_peered":"2023-09-23T23:26:32.039282+0000","last_unstale":"2023-09-23T23:28:05.342613+0000","last_undegraded":"2023-09-23T23:28:05.342613+0000","last_fullsized":"2023-09-23T23:28:05.342613+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:35.345086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":443,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.345198+0000","last_change":"2023-09-23T23:26:48.536996+0000","last_active":"2023-09-23T23:28:05.345198+0000","last_peered":"2023-09-23T23:28:05.345198+0000","last_clean":"2023-09-23T23:28:05.345198+0000","last_became_active":"2023-09-23T23:26:48.536638+0000","last_became_peered":"2023-09-23T23:26:48.536638+0000","last_unstale":"2023-09-23T23:28:05.345198+0000","last_undegraded":"2023-09-23T23:28:05.345198+0000","last_fullsized":"2023-09-23T23:28:05.345198+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:36.943318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.4","version":"454'9787","reported_seq":16639,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:08.031257+0000","last_change":"2023-09-23T23:27:47.203269+0000","last_active":"2023-09-23T23:28:08.031257+0000","last_peered":"2023-09-23T23:28:08.031257+0000","last_clean":"2023-09-23T23:28:08.031257+0000","last_became_active":"2023-09-23T23:27:47.156093+0000","last_became_peered":"2023-09-23T23:27:47.156093+0000","last_unstale":"2023-09-23T23:28:08.031257+0000","last_undegraded":"2023-09-23T23:28:08.031257+0000","last_fullsized":"2023-09-23T23:28:08.031257+0000","mapping_epoch":439,"log_start":"358'3830","ondisk_log_start":"358'3830","created":349,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5957,"log_dups_size":0,"ondisk_log_size":5957,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9338880,"num_objects":142,"num_object_clones":0,"num_object_copies":426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6540,"num_write_kb":213856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3348,"num_bytes_recovered":133038080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":518,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.343704+0000","last_change":"2023-09-23T23:28:03.629593+0000","last_active":"2023-09-23T23:28:05.343704+0000","last_peered":"2023-09-23T23:28:05.343704+0000","last_clean":"2023-09-23T23:28:05.343704+0000","last_became_active":"2023-09-23T23:26:51.588385+0000","last_became_peered":"2023-09-23T23:26:51.588385+0000","last_unstale":"2023-09-23T23:28:05.343704+0000","last_undegraded":"2023-09-23T23:28:05.343704+0000","last_fullsized":"2023-09-23T23:28:05.343704+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:20.818366+0000","scrub_duration":0.0050135120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":282,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.377808+0000","last_change":"2023-09-23T23:25:31.732607+0000","last_active":"2023-09-23T23:28:06.377808+0000","last_peered":"2023-09-23T23:28:06.377808+0000","last_clean":"2023-09-23T23:28:06.377808+0000","last_became_active":"2023-09-23T23:25:31.732099+0000","last_became_peered":"2023-09-23T23:25:31.732099+0000","last_unstale":"2023-09-23T23:28:06.377808+0000","last_undegraded":"2023-09-23T23:28:06.377808+0000","last_fullsized":"2023-09-23T23:28:06.377808+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:23.639936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":430,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.371149+0000","last_change":"2023-09-23T23:22:38.438390+0000","last_active":"2023-09-23T23:28:06.371149+0000","last_peered":"2023-09-23T23:28:06.371149+0000","last_clean":"2023-09-23T23:28:06.371149+0000","last_became_active":"2023-09-23T23:22:38.438065+0000","last_became_peered":"2023-09-23T23:22:38.438065+0000","last_unstale":"2023-09-23T23:28:06.371149+0000","last_undegraded":"2023-09-23T23:28:06.371149+0000","last_fullsized":"2023-09-23T23:28:06.371149+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:51.254227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"454'9647","reported_seq":18572,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:07.962916+0000","last_change":"2023-09-23T23:28:02.398397+0000","last_active":"2023-09-23T23:28:07.962916+0000","last_peered":"2023-09-23T23:28:07.962916+0000","last_clean":"2023-09-23T23:28:07.962916+0000","last_became_active":"2023-09-23T23:28:02.349384+0000","last_became_peered":"2023-09-23T23:28:02.349384+0000","last_unstale":"2023-09-23T23:28:07.962916+0000","last_undegraded":"2023-09-23T23:28:07.962916+0000","last_fullsized":"2023-09-23T23:28:07.962916+0000","mapping_epoch":449,"log_start":"358'3614","ondisk_log_start":"358'3614","created":349,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"367'6627","last_scrub_stamp":"2023-09-23T23:25:24.842424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:24.842424+0000","objects_scrubbed":3200,"log_size":6033,"log_dups_size":0,"ondisk_log_size":6033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:12.434525+0000","scrub_duration":0.68031426699999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11206656,"num_objects":171,"num_object_clones":0,"num_object_copies":513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3029,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4292,"num_bytes_recovered":281280512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":789,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342995+0000","last_change":"2023-09-23T23:23:45.830384+0000","last_active":"2023-09-23T23:28:05.342995+0000","last_peered":"2023-09-23T23:28:05.342995+0000","last_clean":"2023-09-23T23:28:05.342995+0000","last_became_active":"2023-09-23T23:23:36.025724+0000","last_became_peered":"2023-09-23T23:23:36.025724+0000","last_unstale":"2023-09-23T23:28:05.342995+0000","last_undegraded":"2023-09-23T23:28:05.342995+0000","last_fullsized":"2023-09-23T23:28:05.342995+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:34.345528+0000","scrub_duration":0.001659259,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":421,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342617+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:28:05.342617+0000","last_peered":"2023-09-23T23:28:05.342617+0000","last_clean":"2023-09-23T23:28:05.342617+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:28:05.342617+0000","last_undegraded":"2023-09-23T23:28:05.342617+0000","last_fullsized":"2023-09-23T23:28:05.342617+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":422,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.343868+0000","last_change":"2023-09-23T23:25:31.731172+0000","last_active":"2023-09-23T23:28:05.343868+0000","last_peered":"2023-09-23T23:28:05.343868+0000","last_clean":"2023-09-23T23:28:05.343868+0000","last_became_active":"2023-09-23T23:25:31.730903+0000","last_became_peered":"2023-09-23T23:25:31.730903+0000","last_unstale":"2023-09-23T23:28:05.343868+0000","last_undegraded":"2023-09-23T23:28:05.343868+0000","last_fullsized":"2023-09-23T23:28:05.343868+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:35.442945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"455'9905","reported_seq":13245,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T23:28:10.626126+0000","last_change":"2023-09-23T23:28:02.325439+0000","last_active":"2023-09-23T23:28:10.626126+0000","last_peered":"2023-09-23T23:28:10.626126+0000","last_clean":"2023-09-23T23:28:10.626126+0000","last_became_active":"2023-09-23T23:27:10.888274+0000","last_became_peered":"2023-09-23T23:27:10.888274+0000","last_unstale":"2023-09-23T23:28:10.626126+0000","last_undegraded":"2023-09-23T23:28:10.626126+0000","last_fullsized":"2023-09-23T23:28:10.626126+0000","mapping_epoch":421,"log_start":"355'800","ondisk_log_start":"355'800","created":349,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"449'9718","last_scrub_stamp":"2023-09-23T23:28:02.325102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:28:02.325102+0000","objects_scrubbed":292,"log_size":9105,"log_dups_size":0,"ondisk_log_size":9105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:11.669332+0000","scrub_duration":0.056247442000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6881280,"num_objects":105,"num_object_clones":0,"num_object_copies":315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":187,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":734,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.359877+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:28:06.359877+0000","last_peered":"2023-09-23T23:28:06.359877+0000","last_clean":"2023-09-23T23:28:06.359877+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:28:06.359877+0000","last_undegraded":"2023-09-23T23:28:06.359877+0000","last_fullsized":"2023-09-23T23:28:06.359877+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":280,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342844+0000","last_change":"2023-09-23T23:27:20.560074+0000","last_active":"2023-09-23T23:28:05.342844+0000","last_peered":"2023-09-23T23:28:05.342844+0000","last_clean":"2023-09-23T23:28:05.342844+0000","last_became_active":"2023-09-23T23:27:20.559764+0000","last_became_peered":"2023-09-23T23:27:20.559764+0000","last_unstale":"2023-09-23T23:28:05.342844+0000","last_undegraded":"2023-09-23T23:28:05.342844+0000","last_fullsized":"2023-09-23T23:28:05.342844+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:23.675227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.343727+0000","last_change":"2023-09-23T23:26:11.516475+0000","last_active":"2023-09-23T23:28:05.343727+0000","last_peered":"2023-09-23T23:28:05.343727+0000","last_clean":"2023-09-23T23:28:05.343727+0000","last_became_active":"2023-09-23T23:26:11.516198+0000","last_became_peered":"2023-09-23T23:26:11.516198+0000","last_unstale":"2023-09-23T23:28:05.343727+0000","last_undegraded":"2023-09-23T23:28:05.343727+0000","last_fullsized":"2023-09-23T23:28:05.343727+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:56.875103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"454'9652","reported_seq":12946,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:08.578764+0000","last_change":"2023-09-23T23:27:01.819921+0000","last_active":"2023-09-23T23:28:08.578764+0000","last_peered":"2023-09-23T23:28:08.578764+0000","last_clean":"2023-09-23T23:28:08.578764+0000","last_became_active":"2023-09-23T23:27:01.796624+0000","last_became_peered":"2023-09-23T23:27:01.796624+0000","last_unstale":"2023-09-23T23:28:08.578764+0000","last_undegraded":"2023-09-23T23:28:08.578764+0000","last_fullsized":"2023-09-23T23:28:08.578764+0000","mapping_epoch":417,"log_start":"358'4039","ondisk_log_start":"358'4039","created":349,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5613,"log_dups_size":0,"ondisk_log_size":5613,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9535488,"num_objects":145,"num_object_clones":0,"num_object_copies":435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6449,"num_write_kb":211040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1570,"num_bytes_recovered":102891520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":609,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.344112+0000","last_change":"2023-09-23T23:26:34.548211+0000","last_active":"2023-09-23T23:28:05.344112+0000","last_peered":"2023-09-23T23:28:05.344112+0000","last_clean":"2023-09-23T23:28:05.344112+0000","last_became_active":"2023-09-23T23:26:34.547815+0000","last_became_peered":"2023-09-23T23:26:34.547815+0000","last_unstale":"2023-09-23T23:28:05.344112+0000","last_undegraded":"2023-09-23T23:28:05.344112+0000","last_fullsized":"2023-09-23T23:28:05.344112+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:37.675417+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":75,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342227+0000","last_change":"2023-09-23T23:26:11.512815+0000","last_active":"2023-09-23T23:28:05.342227+0000","last_peered":"2023-09-23T23:28:05.342227+0000","last_clean":"2023-09-23T23:28:05.342227+0000","last_became_active":"2023-09-23T23:26:11.512466+0000","last_became_peered":"2023-09-23T23:26:11.512466+0000","last_unstale":"2023-09-23T23:28:05.342227+0000","last_undegraded":"2023-09-23T23:28:05.342227+0000","last_fullsized":"2023-09-23T23:28:05.342227+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:10.324158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":452,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.344309+0000","last_change":"2023-09-23T23:27:20.558731+0000","last_active":"2023-09-23T23:28:05.344309+0000","last_peered":"2023-09-23T23:28:05.344309+0000","last_clean":"2023-09-23T23:28:05.344309+0000","last_became_active":"2023-09-23T23:27:20.558394+0000","last_became_peered":"2023-09-23T23:27:20.558394+0000","last_unstale":"2023-09-23T23:28:05.344309+0000","last_undegraded":"2023-09-23T23:28:05.344309+0000","last_fullsized":"2023-09-23T23:28:05.344309+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:25:00.408551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"454'9600","reported_seq":9802,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:09.768890+0000","last_change":"2023-09-23T23:25:20.332216+0000","last_active":"2023-09-23T23:28:09.768890+0000","last_peered":"2023-09-23T23:28:09.768890+0000","last_clean":"2023-09-23T23:28:09.768890+0000","last_became_active":"2023-09-23T23:25:18.988253+0000","last_became_peered":"2023-09-23T23:25:18.988253+0000","last_unstale":"2023-09-23T23:28:09.768890+0000","last_undegraded":"2023-09-23T23:28:09.768890+0000","last_fullsized":"2023-09-23T23:28:09.768890+0000","mapping_epoch":349,"log_start":"358'3977","ondisk_log_start":"358'3977","created":349,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"366'6383","last_scrub_stamp":"2023-09-23T23:25:20.331998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:20.331998+0000","objects_scrubbed":3194,"log_size":5623,"log_dups_size":0,"ondisk_log_size":5623,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:37.230596+0000","scrub_duration":0.84482357799999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7602176,"num_objects":116,"num_object_clones":0,"num_object_copies":348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3228,"num_write_kb":4800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":776,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.359900+0000","last_change":"2023-09-23T23:25:34.500038+0000","last_active":"2023-09-23T23:28:06.359900+0000","last_peered":"2023-09-23T23:28:06.359900+0000","last_clean":"2023-09-23T23:28:06.359900+0000","last_became_active":"2023-09-23T23:25:34.499705+0000","last_became_peered":"2023-09-23T23:25:34.499705+0000","last_unstale":"2023-09-23T23:28:06.359900+0000","last_undegraded":"2023-09-23T23:28:06.359900+0000","last_fullsized":"2023-09-23T23:28:06.359900+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:43.096696+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":91,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.341513+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:28:05.341513+0000","last_peered":"2023-09-23T23:28:05.341513+0000","last_clean":"2023-09-23T23:28:05.341513+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:28:05.341513+0000","last_undegraded":"2023-09-23T23:28:05.341513+0000","last_fullsized":"2023-09-23T23:28:05.341513+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":375,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.345481+0000","last_change":"2023-09-23T23:22:39.489325+0000","last_active":"2023-09-23T23:28:05.345481+0000","last_peered":"2023-09-23T23:28:05.345481+0000","last_clean":"2023-09-23T23:28:05.345481+0000","last_became_active":"2023-09-23T23:22:39.489048+0000","last_became_peered":"2023-09-23T23:22:39.489048+0000","last_unstale":"2023-09-23T23:28:05.345481+0000","last_undegraded":"2023-09-23T23:28:05.345481+0000","last_fullsized":"2023-09-23T23:28:05.345481+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:29.984948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"455'13141","reported_seq":15523,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T23:28:10.607110+0000","last_change":"2023-09-23T23:27:53.301849+0000","last_active":"2023-09-23T23:28:10.607110+0000","last_peered":"2023-09-23T23:28:10.607110+0000","last_clean":"2023-09-23T23:28:10.607110+0000","last_became_active":"2023-09-23T23:27:53.251393+0000","last_became_peered":"2023-09-23T23:27:53.251393+0000","last_unstale":"2023-09-23T23:28:10.607110+0000","last_undegraded":"2023-09-23T23:28:10.607110+0000","last_fullsized":"2023-09-23T23:28:10.607110+0000","mapping_epoch":441,"log_start":"362'5758","ondisk_log_start":"362'5758","created":349,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":7383,"log_dups_size":0,"ondisk_log_size":7383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:42:29.466833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14549016,"num_objects":223,"num_object_clones":0,"num_object_copies":669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":223,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13141,"num_write_kb":427585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1118,"num_bytes_recovered":73203736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":752,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342043+0000","last_change":"2023-09-23T23:26:51.561734+0000","last_active":"2023-09-23T23:28:05.342043+0000","last_peered":"2023-09-23T23:28:05.342043+0000","last_clean":"2023-09-23T23:28:05.342043+0000","last_became_active":"2023-09-23T23:26:51.561394+0000","last_became_peered":"2023-09-23T23:26:51.561394+0000","last_unstale":"2023-09-23T23:28:05.342043+0000","last_undegraded":"2023-09-23T23:28:05.342043+0000","last_fullsized":"2023-09-23T23:28:05.342043+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.063071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.063071+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":155,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.344668+0000","last_change":"2023-09-23T23:23:36.048889+0000","last_active":"2023-09-23T23:28:05.344668+0000","last_peered":"2023-09-23T23:28:05.344668+0000","last_clean":"2023-09-23T23:28:05.344668+0000","last_became_active":"2023-09-23T23:23:36.048572+0000","last_became_peered":"2023-09-23T23:23:36.048572+0000","last_unstale":"2023-09-23T23:28:05.344668+0000","last_undegraded":"2023-09-23T23:28:05.344668+0000","last_fullsized":"2023-09-23T23:28:05.344668+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:55.581146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":469,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.385208+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:28:05.385208+0000","last_peered":"2023-09-23T23:28:05.385208+0000","last_clean":"2023-09-23T23:28:05.385208+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:28:05.385208+0000","last_undegraded":"2023-09-23T23:28:05.385208+0000","last_fullsized":"2023-09-23T23:28:05.385208+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":40,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.345126+0000","last_change":"2023-09-23T23:27:20.549904+0000","last_active":"2023-09-23T23:28:05.345126+0000","last_peered":"2023-09-23T23:28:05.345126+0000","last_clean":"2023-09-23T23:28:05.345126+0000","last_became_active":"2023-09-23T23:27:20.549645+0000","last_became_peered":"2023-09-23T23:27:20.549645+0000","last_unstale":"2023-09-23T23:28:05.345126+0000","last_undegraded":"2023-09-23T23:28:05.345126+0000","last_fullsized":"2023-09-23T23:28:05.345126+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:09.850640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"454'13034","reported_seq":28511,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:08.742830+0000","last_change":"2023-09-23T23:28:05.376238+0000","last_active":"2023-09-23T23:28:08.742830+0000","last_peered":"2023-09-23T23:28:08.742830+0000","last_clean":"2023-09-23T23:28:08.742830+0000","last_became_active":"2023-09-23T23:28:05.346629+0000","last_became_peered":"2023-09-23T23:28:05.346629+0000","last_unstale":"2023-09-23T23:28:08.742830+0000","last_undegraded":"2023-09-23T23:28:08.742830+0000","last_fullsized":"2023-09-23T23:28:08.742830+0000","mapping_epoch":452,"log_start":"387'8536","ondisk_log_start":"387'8536","created":349,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4498,"log_dups_size":0,"ondisk_log_size":4498,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:44:22.330856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16777216,"num_objects":256,"num_object_clones":0,"num_object_copies":768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13034,"num_write_kb":425280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7650,"num_bytes_recovered":501350400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":601,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.370929+0000","last_change":"2023-09-23T23:23:49.320969+0000","last_active":"2023-09-23T23:28:06.370929+0000","last_peered":"2023-09-23T23:28:06.370929+0000","last_clean":"2023-09-23T23:28:06.370929+0000","last_became_active":"2023-09-23T23:22:17.001911+0000","last_became_peered":"2023-09-23T23:22:17.001911+0000","last_unstale":"2023-09-23T23:28:06.370929+0000","last_undegraded":"2023-09-23T23:28:06.370929+0000","last_fullsized":"2023-09-23T23:28:06.370929+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.320823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.320823+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:07.000984+0000","scrub_duration":0.0018140630000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":155,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.345322+0000","last_change":"2023-09-23T23:25:31.731396+0000","last_active":"2023-09-23T23:28:05.345322+0000","last_peered":"2023-09-23T23:28:05.345322+0000","last_clean":"2023-09-23T23:28:05.345322+0000","last_became_active":"2023-09-23T23:25:31.730931+0000","last_became_peered":"2023-09-23T23:25:31.730931+0000","last_unstale":"2023-09-23T23:28:05.345322+0000","last_undegraded":"2023-09-23T23:28:05.345322+0000","last_fullsized":"2023-09-23T23:28:05.345322+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:48.002628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":433,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.343430+0000","last_change":"2023-09-23T23:27:20.556916+0000","last_active":"2023-09-23T23:28:05.343430+0000","last_peered":"2023-09-23T23:28:05.343430+0000","last_clean":"2023-09-23T23:28:05.343430+0000","last_became_active":"2023-09-23T23:27:20.556651+0000","last_became_peered":"2023-09-23T23:27:20.556651+0000","last_unstale":"2023-09-23T23:28:05.343430+0000","last_undegraded":"2023-09-23T23:28:05.343430+0000","last_fullsized":"2023-09-23T23:28:05.343430+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:06.012543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":40,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.344256+0000","last_change":"2023-09-23T23:27:20.551221+0000","last_active":"2023-09-23T23:28:05.344256+0000","last_peered":"2023-09-23T23:28:05.344256+0000","last_clean":"2023-09-23T23:28:05.344256+0000","last_became_active":"2023-09-23T23:27:20.550900+0000","last_became_peered":"2023-09-23T23:27:20.550900+0000","last_unstale":"2023-09-23T23:28:05.344256+0000","last_undegraded":"2023-09-23T23:28:05.344256+0000","last_fullsized":"2023-09-23T23:28:05.344256+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:23.012149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"454'9515","reported_seq":16922,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:08.577149+0000","last_change":"2023-09-23T23:27:25.617962+0000","last_active":"2023-09-23T23:28:08.577149+0000","last_peered":"2023-09-23T23:28:08.577149+0000","last_clean":"2023-09-23T23:28:08.577149+0000","last_became_active":"2023-09-23T23:27:25.598086+0000","last_became_peered":"2023-09-23T23:27:25.598086+0000","last_unstale":"2023-09-23T23:28:08.577149+0000","last_undegraded":"2023-09-23T23:28:08.577149+0000","last_fullsized":"2023-09-23T23:28:08.577149+0000","mapping_epoch":429,"log_start":"358'3723","ondisk_log_start":"358'3723","created":349,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":0,"last_scrub":"366'6384","last_scrub_stamp":"2023-09-23T23:25:20.132415+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:20.132415+0000","objects_scrubbed":3102,"log_size":5792,"log_dups_size":0,"ondisk_log_size":5792,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:22.673284+0000","scrub_duration":0.99622401299999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9043968,"num_objects":138,"num_object_clones":0,"num_object_copies":414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3148,"num_write_kb":5888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3543,"num_bytes_recovered":232194048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":816,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.341783+0000","last_change":"2023-09-23T23:26:51.562598+0000","last_active":"2023-09-23T23:28:05.341783+0000","last_peered":"2023-09-23T23:28:05.341783+0000","last_clean":"2023-09-23T23:28:05.341783+0000","last_became_active":"2023-09-23T23:26:51.562167+0000","last_became_peered":"2023-09-23T23:26:51.562167+0000","last_unstale":"2023-09-23T23:28:05.341783+0000","last_undegraded":"2023-09-23T23:28:05.341783+0000","last_fullsized":"2023-09-23T23:28:05.341783+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:17.310285+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":189,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.359974+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:28:06.359974+0000","last_peered":"2023-09-23T23:28:06.359974+0000","last_clean":"2023-09-23T23:28:06.359974+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:28:06.359974+0000","last_undegraded":"2023-09-23T23:28:06.359974+0000","last_fullsized":"2023-09-23T23:28:06.359974+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":155,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.343975+0000","last_change":"2023-09-23T23:25:31.828908+0000","last_active":"2023-09-23T23:28:05.343975+0000","last_peered":"2023-09-23T23:28:05.343975+0000","last_clean":"2023-09-23T23:28:05.343975+0000","last_became_active":"2023-09-23T23:25:31.828582+0000","last_became_peered":"2023-09-23T23:25:31.828582+0000","last_unstale":"2023-09-23T23:28:05.343975+0000","last_undegraded":"2023-09-23T23:28:05.343975+0000","last_fullsized":"2023-09-23T23:28:05.343975+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:01.466593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":40,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.345801+0000","last_change":"2023-09-23T23:27:20.551574+0000","last_active":"2023-09-23T23:28:05.345801+0000","last_peered":"2023-09-23T23:28:05.345801+0000","last_clean":"2023-09-23T23:28:05.345801+0000","last_became_active":"2023-09-23T23:27:20.550820+0000","last_became_peered":"2023-09-23T23:27:20.550820+0000","last_unstale":"2023-09-23T23:28:05.345801+0000","last_undegraded":"2023-09-23T23:28:05.345801+0000","last_fullsized":"2023-09-23T23:28:05.345801+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:09.274552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"454'12825","reported_seq":14700,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:08.768816+0000","last_change":"2023-09-23T23:27:56.321233+0000","last_active":"2023-09-23T23:28:08.768816+0000","last_peered":"2023-09-23T23:28:08.768816+0000","last_clean":"2023-09-23T23:28:08.768816+0000","last_became_active":"2023-09-23T23:27:56.288002+0000","last_became_peered":"2023-09-23T23:27:56.288002+0000","last_unstale":"2023-09-23T23:28:08.768816+0000","last_undegraded":"2023-09-23T23:28:08.768816+0000","last_fullsized":"2023-09-23T23:28:08.768816+0000","mapping_epoch":443,"log_start":"386'8341","ondisk_log_start":"386'8341","created":349,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4484,"log_dups_size":0,"ondisk_log_size":4484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:02:54.188140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19202048,"num_objects":293,"num_object_clones":0,"num_object_copies":879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12825,"num_write_kb":419776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":864,"num_bytes_recovered":56623104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":466,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.359968+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:28:06.359968+0000","last_peered":"2023-09-23T23:28:06.359968+0000","last_clean":"2023-09-23T23:28:06.359968+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:28:06.359968+0000","last_undegraded":"2023-09-23T23:28:06.359968+0000","last_fullsized":"2023-09-23T23:28:06.359968+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":774,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.341766+0000","last_change":"2023-09-23T23:26:51.564817+0000","last_active":"2023-09-23T23:28:05.341766+0000","last_peered":"2023-09-23T23:28:05.341766+0000","last_clean":"2023-09-23T23:28:05.341766+0000","last_became_active":"2023-09-23T23:26:51.564511+0000","last_became_peered":"2023-09-23T23:26:51.564511+0000","last_unstale":"2023-09-23T23:28:05.341766+0000","last_undegraded":"2023-09-23T23:28:05.341766+0000","last_fullsized":"2023-09-23T23:28:05.341766+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:02.415212+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":403,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.341696+0000","last_change":"2023-09-23T23:26:32.037619+0000","last_active":"2023-09-23T23:28:05.341696+0000","last_peered":"2023-09-23T23:28:05.341696+0000","last_clean":"2023-09-23T23:28:05.341696+0000","last_became_active":"2023-09-23T23:26:32.037045+0000","last_became_peered":"2023-09-23T23:26:32.037045+0000","last_unstale":"2023-09-23T23:28:05.341696+0000","last_undegraded":"2023-09-23T23:28:05.341696+0000","last_fullsized":"2023-09-23T23:28:05.341696+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:47.657527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":75,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.343818+0000","last_change":"2023-09-23T23:26:11.525845+0000","last_active":"2023-09-23T23:28:05.343818+0000","last_peered":"2023-09-23T23:28:05.343818+0000","last_clean":"2023-09-23T23:28:05.343818+0000","last_became_active":"2023-09-23T23:26:11.525507+0000","last_became_peered":"2023-09-23T23:26:11.525507+0000","last_unstale":"2023-09-23T23:28:05.343818+0000","last_undegraded":"2023-09-23T23:28:05.343818+0000","last_fullsized":"2023-09-23T23:28:05.343818+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:35.462874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"454'9687","reported_seq":15921,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:08.768690+0000","last_change":"2023-09-23T23:27:59.326239+0000","last_active":"2023-09-23T23:28:08.768690+0000","last_peered":"2023-09-23T23:28:08.768690+0000","last_clean":"2023-09-23T23:28:08.768690+0000","last_became_active":"2023-09-23T23:27:59.304134+0000","last_became_peered":"2023-09-23T23:27:59.304134+0000","last_unstale":"2023-09-23T23:28:08.768690+0000","last_undegraded":"2023-09-23T23:28:08.768690+0000","last_fullsized":"2023-09-23T23:28:08.768690+0000","mapping_epoch":446,"log_start":"358'4046","ondisk_log_start":"358'4046","created":349,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5641,"log_dups_size":0,"ondisk_log_size":5641,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13205504,"num_objects":201,"num_object_clones":0,"num_object_copies":603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6451,"num_write_kb":212896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3031,"num_bytes_recovered":198639616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":617,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.344859+0000","last_change":"2023-09-23T23:26:34.550276+0000","last_active":"2023-09-23T23:28:05.344859+0000","last_peered":"2023-09-23T23:28:05.344859+0000","last_clean":"2023-09-23T23:28:05.344859+0000","last_became_active":"2023-09-23T23:26:34.549923+0000","last_became_peered":"2023-09-23T23:26:34.549923+0000","last_unstale":"2023-09-23T23:28:05.344859+0000","last_undegraded":"2023-09-23T23:28:05.344859+0000","last_fullsized":"2023-09-23T23:28:05.344859+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:22.636211+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":381,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.341558+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:28:05.341558+0000","last_peered":"2023-09-23T23:28:05.341558+0000","last_clean":"2023-09-23T23:28:05.341558+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:28:05.341558+0000","last_undegraded":"2023-09-23T23:28:05.341558+0000","last_fullsized":"2023-09-23T23:28:05.341558+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":457,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.370942+0000","last_change":"2023-09-23T23:26:11.528036+0000","last_active":"2023-09-23T23:28:06.370942+0000","last_peered":"2023-09-23T23:28:06.370942+0000","last_clean":"2023-09-23T23:28:06.370942+0000","last_became_active":"2023-09-23T23:26:11.527688+0000","last_became_peered":"2023-09-23T23:26:11.527688+0000","last_unstale":"2023-09-23T23:28:06.370942+0000","last_undegraded":"2023-09-23T23:28:06.370942+0000","last_fullsized":"2023-09-23T23:28:06.370942+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:49.411897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":485,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.372526+0000","last_change":"2023-09-23T23:22:38.438448+0000","last_active":"2023-09-23T23:28:06.372526+0000","last_peered":"2023-09-23T23:28:06.372526+0000","last_clean":"2023-09-23T23:28:06.372526+0000","last_became_active":"2023-09-23T23:22:38.437495+0000","last_became_peered":"2023-09-23T23:22:38.437495+0000","last_unstale":"2023-09-23T23:28:06.372526+0000","last_undegraded":"2023-09-23T23:28:06.372526+0000","last_fullsized":"2023-09-23T23:28:06.372526+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:49.587056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"454'12699","reported_seq":17381,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:09.760242+0000","last_change":"2023-09-23T23:27:29.668660+0000","last_active":"2023-09-23T23:28:09.760242+0000","last_peered":"2023-09-23T23:28:09.760242+0000","last_clean":"2023-09-23T23:28:09.760242+0000","last_became_active":"2023-09-23T23:27:29.641291+0000","last_became_peered":"2023-09-23T23:27:29.641291+0000","last_unstale":"2023-09-23T23:28:09.760242+0000","last_undegraded":"2023-09-23T23:28:09.760242+0000","last_fullsized":"2023-09-23T23:28:09.760242+0000","mapping_epoch":431,"log_start":"386'8183","ondisk_log_start":"386'8183","created":349,"last_epoch_clean":432,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4516,"log_dups_size":0,"ondisk_log_size":4516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:47.739467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14876672,"num_objects":227,"num_object_clones":0,"num_object_copies":681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12699,"num_write_kb":413632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2267,"num_bytes_recovered":148570112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":494,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.843008+0000","last_change":"2023-09-23T23:28:05.843008+0000","last_active":"2023-09-23T23:28:05.843008+0000","last_peered":"2023-09-23T23:28:05.843008+0000","last_clean":"2023-09-23T23:28:05.843008+0000","last_became_active":"2023-09-23T23:26:04.560484+0000","last_became_peered":"2023-09-23T23:26:04.560484+0000","last_unstale":"2023-09-23T23:28:05.843008+0000","last_undegraded":"2023-09-23T23:28:05.843008+0000","last_fullsized":"2023-09-23T23:28:05.843008+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.842881+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.842881+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:37.322347+0000","scrub_duration":0.0047245100000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":305,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342457+0000","last_change":"2023-09-23T23:23:35.037351+0000","last_active":"2023-09-23T23:28:05.342457+0000","last_peered":"2023-09-23T23:28:05.342457+0000","last_clean":"2023-09-23T23:28:05.342457+0000","last_became_active":"2023-09-23T23:23:35.036841+0000","last_became_peered":"2023-09-23T23:23:35.036841+0000","last_unstale":"2023-09-23T23:28:05.342457+0000","last_undegraded":"2023-09-23T23:28:05.342457+0000","last_fullsized":"2023-09-23T23:28:05.342457+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:20.618292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":155,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.344784+0000","last_change":"2023-09-23T23:26:32.037099+0000","last_active":"2023-09-23T23:28:05.344784+0000","last_peered":"2023-09-23T23:28:05.344784+0000","last_clean":"2023-09-23T23:28:05.344784+0000","last_became_active":"2023-09-23T23:26:32.036681+0000","last_became_peered":"2023-09-23T23:26:32.036681+0000","last_unstale":"2023-09-23T23:28:05.344784+0000","last_undegraded":"2023-09-23T23:28:05.344784+0000","last_fullsized":"2023-09-23T23:28:05.344784+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:48.926576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":187,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.385047+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:28:05.385047+0000","last_peered":"2023-09-23T23:28:05.385047+0000","last_clean":"2023-09-23T23:28:05.385047+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:28:05.385047+0000","last_undegraded":"2023-09-23T23:28:05.385047+0000","last_fullsized":"2023-09-23T23:28:05.385047+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"454'13067","reported_seq":13180,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:08.092050+0000","last_change":"2023-09-23T23:23:43.505852+0000","last_active":"2023-09-23T23:28:08.092050+0000","last_peered":"2023-09-23T23:28:08.092050+0000","last_clean":"2023-09-23T23:28:08.092050+0000","last_became_active":"2023-09-23T23:23:43.505473+0000","last_became_peered":"2023-09-23T23:23:43.505473+0000","last_unstale":"2023-09-23T23:28:08.092050+0000","last_undegraded":"2023-09-23T23:28:08.092050+0000","last_fullsized":"2023-09-23T23:28:08.092050+0000","mapping_epoch":349,"log_start":"386'8366","ondisk_log_start":"386'8366","created":349,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":4701,"log_dups_size":0,"ondisk_log_size":4701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:31.385610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19333120,"num_objects":295,"num_object_clones":0,"num_object_copies":885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13067,"num_write_kb":427584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":765,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.343899+0000","last_change":"2023-09-23T23:26:14.518921+0000","last_active":"2023-09-23T23:28:05.343899+0000","last_peered":"2023-09-23T23:28:05.343899+0000","last_clean":"2023-09-23T23:28:05.343899+0000","last_became_active":"2023-09-23T23:26:14.518558+0000","last_became_peered":"2023-09-23T23:26:14.518558+0000","last_unstale":"2023-09-23T23:28:05.343899+0000","last_undegraded":"2023-09-23T23:28:05.343899+0000","last_fullsized":"2023-09-23T23:28:05.343899+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:14.360597+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":447,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342290+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:28:05.342290+0000","last_peered":"2023-09-23T23:28:05.342290+0000","last_clean":"2023-09-23T23:28:05.342290+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:28:05.342290+0000","last_undegraded":"2023-09-23T23:28:05.342290+0000","last_fullsized":"2023-09-23T23:28:05.342290+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":40,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.345463+0000","last_change":"2023-09-23T23:27:20.562744+0000","last_active":"2023-09-23T23:28:05.345463+0000","last_peered":"2023-09-23T23:28:05.345463+0000","last_clean":"2023-09-23T23:28:05.345463+0000","last_became_active":"2023-09-23T23:27:20.562383+0000","last_became_peered":"2023-09-23T23:27:20.562383+0000","last_unstale":"2023-09-23T23:28:05.345463+0000","last_undegraded":"2023-09-23T23:28:05.345463+0000","last_fullsized":"2023-09-23T23:28:05.345463+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:01.085133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":92,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.341482+0000","last_change":"2023-09-23T23:26:48.562966+0000","last_active":"2023-09-23T23:28:05.341482+0000","last_peered":"2023-09-23T23:28:05.341482+0000","last_clean":"2023-09-23T23:28:05.341482+0000","last_became_active":"2023-09-23T23:26:48.562630+0000","last_became_peered":"2023-09-23T23:26:48.562630+0000","last_unstale":"2023-09-23T23:28:05.341482+0000","last_undegraded":"2023-09-23T23:28:05.341482+0000","last_fullsized":"2023-09-23T23:28:05.341482+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:40.163647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"454'12971","reported_seq":21548,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:09.659640+0000","last_change":"2023-09-23T23:26:33.547524+0000","last_active":"2023-09-23T23:28:09.659640+0000","last_peered":"2023-09-23T23:28:09.659640+0000","last_clean":"2023-09-23T23:28:09.659640+0000","last_became_active":"2023-09-23T23:26:33.521810+0000","last_became_peered":"2023-09-23T23:26:33.521810+0000","last_unstale":"2023-09-23T23:28:09.659640+0000","last_undegraded":"2023-09-23T23:28:09.659640+0000","last_fullsized":"2023-09-23T23:28:09.659640+0000","mapping_epoch":400,"log_start":"381'7905","ondisk_log_start":"381'7905","created":349,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5066,"log_dups_size":0,"ondisk_log_size":5066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:49.164061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14614528,"num_objects":223,"num_object_clones":0,"num_object_copies":669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12971,"num_write_kb":422208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4222,"num_bytes_recovered":276692992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":837,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342305+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:28:05.342305+0000","last_peered":"2023-09-23T23:28:05.342305+0000","last_clean":"2023-09-23T23:28:05.342305+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:28:05.342305+0000","last_undegraded":"2023-09-23T23:28:05.342305+0000","last_fullsized":"2023-09-23T23:28:05.342305+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":53,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.344434+0000","last_change":"2023-09-23T23:26:48.544465+0000","last_active":"2023-09-23T23:28:05.344434+0000","last_peered":"2023-09-23T23:28:05.344434+0000","last_clean":"2023-09-23T23:28:05.344434+0000","last_became_active":"2023-09-23T23:26:48.544166+0000","last_became_peered":"2023-09-23T23:26:48.544166+0000","last_unstale":"2023-09-23T23:28:05.344434+0000","last_undegraded":"2023-09-23T23:28:05.344434+0000","last_fullsized":"2023-09-23T23:28:05.344434+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:38.844346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":440,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.372368+0000","last_change":"2023-09-23T23:22:38.482705+0000","last_active":"2023-09-23T23:28:06.372368+0000","last_peered":"2023-09-23T23:28:06.372368+0000","last_clean":"2023-09-23T23:28:06.372368+0000","last_became_active":"2023-09-23T23:22:38.438239+0000","last_became_peered":"2023-09-23T23:22:38.438239+0000","last_unstale":"2023-09-23T23:28:06.372368+0000","last_undegraded":"2023-09-23T23:28:06.372368+0000","last_fullsized":"2023-09-23T23:28:06.372368+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:52.006349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":75,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342301+0000","last_change":"2023-09-23T23:26:11.522604+0000","last_active":"2023-09-23T23:28:05.342301+0000","last_peered":"2023-09-23T23:28:05.342301+0000","last_clean":"2023-09-23T23:28:05.342301+0000","last_became_active":"2023-09-23T23:26:11.522350+0000","last_became_peered":"2023-09-23T23:26:11.522350+0000","last_unstale":"2023-09-23T23:28:05.342301+0000","last_undegraded":"2023-09-23T23:28:05.342301+0000","last_fullsized":"2023-09-23T23:28:05.342301+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:20.164202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"454'9718","reported_seq":19597,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:08.109886+0000","last_change":"2023-09-23T23:28:03.383467+0000","last_active":"2023-09-23T23:28:08.109886+0000","last_peered":"2023-09-23T23:28:08.109886+0000","last_clean":"2023-09-23T23:28:08.109886+0000","last_became_active":"2023-09-23T23:28:03.344095+0000","last_became_peered":"2023-09-23T23:28:03.344095+0000","last_unstale":"2023-09-23T23:28:08.109886+0000","last_undegraded":"2023-09-23T23:28:08.109886+0000","last_fullsized":"2023-09-23T23:28:08.109886+0000","mapping_epoch":450,"log_start":"358'3787","ondisk_log_start":"358'3787","created":365,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:23:43.052955+0000","objects_scrubbed":0,"log_size":5931,"log_dups_size":0,"ondisk_log_size":5931,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8519680,"num_objects":130,"num_object_clones":0,"num_object_copies":390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6506,"num_write_kb":212352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4841,"num_bytes_recovered":317259776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":836,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.341832+0000","last_change":"2023-09-23T23:25:34.505042+0000","last_active":"2023-09-23T23:28:05.341832+0000","last_peered":"2023-09-23T23:28:05.341832+0000","last_clean":"2023-09-23T23:28:05.341832+0000","last_became_active":"2023-09-23T23:25:34.504687+0000","last_became_peered":"2023-09-23T23:25:34.504687+0000","last_unstale":"2023-09-23T23:28:05.341832+0000","last_undegraded":"2023-09-23T23:28:05.341832+0000","last_fullsized":"2023-09-23T23:28:05.341832+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:06.369262+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":89,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.374293+0000","last_change":"2023-09-23T23:25:48.082121+0000","last_active":"2023-09-23T23:28:06.374293+0000","last_peered":"2023-09-23T23:28:06.374293+0000","last_clean":"2023-09-23T23:28:06.374293+0000","last_became_active":"2023-09-23T23:25:48.081746+0000","last_became_peered":"2023-09-23T23:25:48.081746+0000","last_unstale":"2023-09-23T23:28:06.374293+0000","last_undegraded":"2023-09-23T23:28:06.374293+0000","last_fullsized":"2023-09-23T23:28:06.374293+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:00.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":198,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.360457+0000","last_change":"2023-09-23T23:22:38.439602+0000","last_active":"2023-09-23T23:28:06.360457+0000","last_peered":"2023-09-23T23:28:06.360457+0000","last_clean":"2023-09-23T23:28:06.360457+0000","last_became_active":"2023-09-23T23:22:38.439261+0000","last_became_peered":"2023-09-23T23:22:38.439261+0000","last_unstale":"2023-09-23T23:28:06.360457+0000","last_undegraded":"2023-09-23T23:28:06.360457+0000","last_fullsized":"2023-09-23T23:28:06.360457+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:03.293816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"454'9789","reported_seq":16776,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:08.122803+0000","last_change":"2023-09-23T23:26:52.609382+0000","last_active":"2023-09-23T23:28:08.122803+0000","last_peered":"2023-09-23T23:28:08.122803+0000","last_clean":"2023-09-23T23:28:08.122803+0000","last_became_active":"2023-09-23T23:26:52.578253+0000","last_became_peered":"2023-09-23T23:26:52.578253+0000","last_unstale":"2023-09-23T23:28:08.122803+0000","last_undegraded":"2023-09-23T23:28:08.122803+0000","last_fullsized":"2023-09-23T23:28:08.122803+0000","mapping_epoch":414,"log_start":"358'3901","ondisk_log_start":"358'3901","created":365,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":5,"last_scrub":"366'6668","last_scrub_stamp":"2023-09-23T23:25:23.925139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:25:23.925139+0000","objects_scrubbed":3290,"log_size":5888,"log_dups_size":0,"ondisk_log_size":5888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:07.350349+0000","scrub_duration":0.63215223600000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10485760,"num_objects":160,"num_object_clones":0,"num_object_copies":480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3130,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3348,"num_bytes_recovered":209125376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":581,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342414+0000","last_change":"2023-09-23T23:26:14.522158+0000","last_active":"2023-09-23T23:28:05.342414+0000","last_peered":"2023-09-23T23:28:05.342414+0000","last_clean":"2023-09-23T23:28:05.342414+0000","last_became_active":"2023-09-23T23:26:14.521785+0000","last_became_peered":"2023-09-23T23:26:14.521785+0000","last_unstale":"2023-09-23T23:28:05.342414+0000","last_undegraded":"2023-09-23T23:28:05.342414+0000","last_fullsized":"2023-09-23T23:28:05.342414+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:39.445601+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":195,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.343975+0000","last_change":"2023-09-23T23:22:38.452968+0000","last_active":"2023-09-23T23:28:05.343975+0000","last_peered":"2023-09-23T23:28:05.343975+0000","last_clean":"2023-09-23T23:28:05.343975+0000","last_became_active":"2023-09-23T23:22:38.437986+0000","last_became_peered":"2023-09-23T23:22:38.437986+0000","last_unstale":"2023-09-23T23:28:05.343975+0000","last_undegraded":"2023-09-23T23:28:05.343975+0000","last_fullsized":"2023-09-23T23:28:05.343975+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:10.333271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.343255+0000","last_change":"2023-09-23T23:27:20.558454+0000","last_active":"2023-09-23T23:28:05.343255+0000","last_peered":"2023-09-23T23:28:05.343255+0000","last_clean":"2023-09-23T23:28:05.343255+0000","last_became_active":"2023-09-23T23:27:20.558204+0000","last_became_peered":"2023-09-23T23:27:20.558204+0000","last_unstale":"2023-09-23T23:28:05.343255+0000","last_undegraded":"2023-09-23T23:28:05.343255+0000","last_fullsized":"2023-09-23T23:28:05.343255+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:15.079990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"454'9665","reported_seq":18339,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:08.008589+0000","last_change":"2023-09-23T23:28:06.718148+0000","last_active":"2023-09-23T23:28:08.008589+0000","last_peered":"2023-09-23T23:28:08.008589+0000","last_clean":"2023-09-23T23:28:08.008589+0000","last_became_active":"2023-09-23T23:27:47.157657+0000","last_became_peered":"2023-09-23T23:27:47.157657+0000","last_unstale":"2023-09-23T23:28:08.008589+0000","last_undegraded":"2023-09-23T23:28:08.008589+0000","last_fullsized":"2023-09-23T23:28:08.008589+0000","mapping_epoch":439,"log_start":"358'3698","ondisk_log_start":"358'3698","created":365,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":5,"last_scrub":"454'9636","last_scrub_stamp":"2023-09-23T23:28:06.718041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.718041+0000","objects_scrubbed":218,"log_size":5967,"log_dups_size":0,"ondisk_log_size":5967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:35.672541+0000","scrub_duration":0.048163746,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12386304,"num_objects":189,"num_object_clones":0,"num_object_copies":567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":29,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":195,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.344125+0000","last_change":"2023-09-23T23:25:31.740467+0000","last_active":"2023-09-23T23:28:05.344125+0000","last_peered":"2023-09-23T23:28:05.344125+0000","last_clean":"2023-09-23T23:28:05.344125+0000","last_became_active":"2023-09-23T23:25:31.729046+0000","last_became_peered":"2023-09-23T23:25:31.729046+0000","last_unstale":"2023-09-23T23:28:05.344125+0000","last_undegraded":"2023-09-23T23:28:05.344125+0000","last_fullsized":"2023-09-23T23:28:05.344125+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:31.222969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":75,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342627+0000","last_change":"2023-09-23T23:26:11.527453+0000","last_active":"2023-09-23T23:28:05.342627+0000","last_peered":"2023-09-23T23:28:05.342627+0000","last_clean":"2023-09-23T23:28:05.342627+0000","last_became_active":"2023-09-23T23:26:11.527208+0000","last_became_peered":"2023-09-23T23:26:11.527208+0000","last_unstale":"2023-09-23T23:28:05.342627+0000","last_undegraded":"2023-09-23T23:28:05.342627+0000","last_fullsized":"2023-09-23T23:28:05.342627+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:15.083611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"455'9754","reported_seq":10597,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T23:28:10.575310+0000","last_change":"2023-09-23T23:28:05.990870+0000","last_active":"2023-09-23T23:28:10.575310+0000","last_peered":"2023-09-23T23:28:10.575310+0000","last_clean":"2023-09-23T23:28:10.575310+0000","last_became_active":"2023-09-23T23:28:00.320769+0000","last_became_peered":"2023-09-23T23:28:00.320769+0000","last_unstale":"2023-09-23T23:28:10.575310+0000","last_undegraded":"2023-09-23T23:28:10.575310+0000","last_fullsized":"2023-09-23T23:28:10.575310+0000","mapping_epoch":447,"log_start":"356'2736","ondisk_log_start":"356'2736","created":365,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"453'9644","last_scrub_stamp":"2023-09-23T23:28:05.990723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.990723+0000","objects_scrubbed":201,"log_size":7018,"log_dups_size":0,"ondisk_log_size":7018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:42.957390+0000","scrub_duration":0.076994301000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5898240,"num_objects":90,"num_object_clones":0,"num_object_copies":270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":111,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":720,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342286+0000","last_change":"2023-09-23T23:25:34.510468+0000","last_active":"2023-09-23T23:28:05.342286+0000","last_peered":"2023-09-23T23:28:05.342286+0000","last_clean":"2023-09-23T23:28:05.342286+0000","last_became_active":"2023-09-23T23:25:34.510106+0000","last_became_peered":"2023-09-23T23:25:34.510106+0000","last_unstale":"2023-09-23T23:28:05.342286+0000","last_undegraded":"2023-09-23T23:28:05.342286+0000","last_fullsized":"2023-09-23T23:28:05.342286+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:42.157293+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":471,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342506+0000","last_change":"2023-09-23T23:25:31.746383+0000","last_active":"2023-09-23T23:28:05.342506+0000","last_peered":"2023-09-23T23:28:05.342506+0000","last_clean":"2023-09-23T23:28:05.342506+0000","last_became_active":"2023-09-23T23:25:31.745929+0000","last_became_peered":"2023-09-23T23:25:31.745929+0000","last_unstale":"2023-09-23T23:28:05.342506+0000","last_undegraded":"2023-09-23T23:28:05.342506+0000","last_fullsized":"2023-09-23T23:28:05.342506+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:21.696449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":465,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.341975+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:28:05.341975+0000","last_peered":"2023-09-23T23:28:05.341975+0000","last_clean":"2023-09-23T23:28:05.341975+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:28:05.341975+0000","last_undegraded":"2023-09-23T23:28:05.341975+0000","last_fullsized":"2023-09-23T23:28:05.341975+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"454'9629","reported_seq":14065,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:09.778455+0000","last_change":"2023-09-23T23:28:09.021583+0000","last_active":"2023-09-23T23:28:09.778455+0000","last_peered":"2023-09-23T23:28:09.778455+0000","last_clean":"2023-09-23T23:28:09.778455+0000","last_became_active":"2023-09-23T23:28:01.331384+0000","last_became_peered":"2023-09-23T23:28:01.331384+0000","last_unstale":"2023-09-23T23:28:09.778455+0000","last_undegraded":"2023-09-23T23:28:09.778455+0000","last_fullsized":"2023-09-23T23:28:09.778455+0000","mapping_epoch":448,"log_start":"358'3884","ondisk_log_start":"358'3884","created":365,"last_epoch_clean":449,"parent":"0.0","parent_split_bits":5,"last_scrub":"454'9604","last_scrub_stamp":"2023-09-23T23:28:09.021395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:23:43.052955+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.021395+0000","objects_scrubbed":156,"log_size":5745,"log_dups_size":0,"ondisk_log_size":5745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:58.470079+0000","scrub_duration":0.022494910999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8585216,"num_objects":131,"num_object_clones":0,"num_object_copies":393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":25,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":827,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.345173+0000","last_change":"2023-09-23T23:26:14.527390+0000","last_active":"2023-09-23T23:28:05.345173+0000","last_peered":"2023-09-23T23:28:05.345173+0000","last_clean":"2023-09-23T23:28:05.345173+0000","last_became_active":"2023-09-23T23:26:14.527008+0000","last_became_peered":"2023-09-23T23:26:14.527008+0000","last_unstale":"2023-09-23T23:28:05.345173+0000","last_undegraded":"2023-09-23T23:28:05.345173+0000","last_fullsized":"2023-09-23T23:28:05.345173+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:36.460827+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":459,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:28:06.372446+0000","last_change":"2023-09-23T23:20:15.786279+0000","last_active":"2023-09-23T23:28:06.372446+0000","last_peered":"2023-09-23T23:28:06.372446+0000","last_clean":"2023-09-23T23:28:06.372446+0000","last_became_active":"2023-09-23T23:20:15.782731+0000","last_became_peered":"2023-09-23T23:20:15.782731+0000","last_unstale":"2023-09-23T23:28:06.372446+0000","last_undegraded":"2023-09-23T23:28:06.372446+0000","last_fullsized":"2023-09-23T23:28:06.372446+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:58.011396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":499,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:28:05.342010+0000","last_change":"2023-09-23T23:25:31.827217+0000","last_active":"2023-09-23T23:28:05.342010+0000","last_peered":"2023-09-23T23:28:05.342010+0000","last_clean":"2023-09-23T23:28:05.342010+0000","last_became_active":"2023-09-23T23:25:31.826915+0000","last_became_peered":"2023-09-23T23:25:31.826915+0000","last_unstale":"2023-09-23T23:28:05.342010+0000","last_undegraded":"2023-09-23T23:28:05.342010+0000","last_fullsized":"2023-09-23T23:28:05.342010+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:44.656199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":276168728,"num_objects":4215,"num_object_clones":0,"num_object_copies":12645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4215,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":158583,"num_write_kb":4461313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59338,"num_bytes_recovered":3779199000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2608476160,"data_stored":2608463944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":157869,"ondisk_log_size":157869,"up":78,"acting":78,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":6162,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":617203,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":735865,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782584,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1037080,"kb_used_data":858512,"kb_used_omap":8,"kb_used_meta":178551,"kb_avail":92708072,"statfs":{"total":95995035648,"available":94933065728,"internally_reserved":0,"allocated":879116288,"data_stored":877823567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":182836293},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":3.8119999999999998,"5min":3.024,"15min":1.998},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":30.672000000000001,"5min":85.721000000000004,"15min":85.721000000000004},"last":0.35199999999999998},{"interface":"front","average":{"1min":4.8780000000000001,"5min":3.3580000000000001,"15min":1.8500000000000001},"min":{"1min":0.26600000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":30.504000000000001,"5min":85.965000000000003,"15min":85.965000000000003},"last":0.253}]},{"osd":1,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.7609999999999999,"15min":1.2789999999999999},"min":{"1min":0.27900000000000003,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":10.141,"5min":60.847000000000001,"15min":60.847000000000001},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.835,"15min":1.591},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":2.9449999999999998,"5min":59.512999999999998,"15min":59.512999999999998},"last":0.45100000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":2.2759999999999998,"15min":1.98},"min":{"1min":0.27200000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.4269999999999996,"5min":55.811999999999998,"15min":55.811999999999998},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.3400000000000001,"5min":1.9039999999999999,"15min":1.429},"min":{"1min":0.28199999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.532,"5min":55.847999999999999,"15min":55.847999999999999},"last":4.3789999999999996}]},{"osd":3,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":1.482,"15min":1.4870000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.008,"5min":9.9960000000000004,"15min":9.9960000000000004},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.272,"5min":1.498,"15min":1.3620000000000001},"min":{"1min":0.318,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.5949999999999998,"5min":9.9659999999999993,"15min":9.9659999999999993},"last":0.41899999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.72599999999999998,"15min":0.81399999999999995},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.2919999999999998,"5min":6.8380000000000001,"15min":6.8380000000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.80600000000000005,"15min":0.89400000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.68200000000000005,"5min":5.242,"15min":5.242},"last":0.33200000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.90000000000000002,"15min":1.024},"min":{"1min":0.307,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.71899999999999997,"5min":5.2110000000000003,"15min":5.2110000000000003},"last":0.495},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.78700000000000003,"15min":0.80300000000000005},"min":{"1min":0.29899999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.8319999999999999,"5min":6.0149999999999997,"15min":6.0149999999999997},"last":0.61299999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.369,"15min":0.95899999999999996},"min":{"1min":0.33900000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.7989999999999999,"5min":62.213000000000001,"15min":62.213000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.57699999999999996,"5min":1.526,"15min":1.234},"min":{"1min":0.34200000000000003,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.9299999999999999,"5min":62.805999999999997,"15min":62.805999999999997},"last":0.50800000000000001}]}]},{"osd":6,"up_from":0,"seq":197568496061,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":112,"seq":481036337402,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":927656,"kb_used_data":203872,"kb_used_omap":4,"kb_used_meta":723771,"kb_avail":92817496,"statfs":{"total":95995035648,"available":95045115904,"internally_reserved":0,"allocated":208764928,"data_stored":206633886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":741142414},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:27:59 2023","interfaces":[{"interface":"back","average":{"1min":15.005000000000001,"5min":5.6239999999999997,"15min":3.7549999999999999},"min":{"1min":0.25600000000000001,"5min":0.189,"15min":0.16800000000000001},"max":{"1min":130.768,"5min":130.768,"15min":130.768},"last":0.25600000000000001},{"interface":"front","average":{"1min":14.644,"5min":4.7809999999999997,"15min":3.6419999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":134.929,"5min":134.929,"15min":134.929},"last":0.24399999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:27:45 2023","interfaces":[{"interface":"back","average":{"1min":3.8620000000000001,"5min":6.0890000000000004,"15min":4.5659999999999998},"min":{"1min":0.373,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":18.123999999999999,"5min":209.84200000000001,"15min":209.84200000000001},"last":231.67400000000001},{"interface":"front","average":{"1min":4.0369999999999999,"5min":6.0750000000000002,"15min":3.895},"min":{"1min":0.24199999999999999,"5min":0.23899999999999999,"15min":0.218},"max":{"1min":17.997,"5min":209.11099999999999,"15min":209.11099999999999},"last":232.21000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:27:40 2023","interfaces":[{"interface":"back","average":{"1min":4.3920000000000003,"5min":3.383,"15min":3.8660000000000001},"min":{"1min":0.23799999999999999,"5min":0.22600000000000001,"15min":0.20799999999999999},"max":{"1min":33.210000000000001,"5min":157.90899999999999,"15min":157.90899999999999},"last":231.27199999999999},{"interface":"front","average":{"1min":3.8570000000000002,"5min":3.2810000000000001,"15min":3.7090000000000001},"min":{"1min":0.26500000000000001,"5min":0.245,"15min":0.245},"max":{"1min":25.626999999999999,"5min":156.27000000000001,"15min":156.27000000000001},"last":231.43799999999999}]},{"osd":4,"last update":"Sat Sep 23 23:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7889999999999999,"5min":1.6619999999999999,"15min":1.597},"min":{"1min":0.318,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":4.3520000000000003,"5min":9.3780000000000001,"15min":18.474},"last":0.498},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.7470000000000001,"15min":1.724},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.23,"5min":9.7460000000000004,"15min":25.710999999999999},"last":0.57999999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.159,"15min":3.0659999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.9939999999999998,"5min":13.064,"15min":250.33000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.776,"5min":1.944,"15min":3.2120000000000002},"min":{"1min":0.377,"5min":0.34899999999999998,"15min":0.30199999999999999},"max":{"1min":4.3109999999999999,"5min":10.92,"15min":250.21199999999999},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":2.4929999999999999,"15min":2.4550000000000001},"min":{"1min":0.33700000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.0640000000000001,"5min":74.087999999999994,"15min":74.087999999999994},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.835,"5min":2.843,"15min":2.3700000000000001},"min":{"1min":0.435,"5min":0.33100000000000002,"15min":0.32800000000000001},"max":{"1min":4.383,"5min":73.974999999999994,"15min":73.974999999999994},"last":0.52900000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:27:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5389999999999999,"5min":3.2429999999999999,"15min":4.1580000000000004},"min":{"1min":0.33000000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.6600000000000001,"5min":78.308000000000007,"15min":78.308000000000007},"last":0.55600000000000005},{"interface":"front","average":{"1min":2.5880000000000001,"5min":3.0619999999999998,"15min":3.7330000000000001},"min":{"1min":0.45900000000000002,"5min":0.371,"15min":0.371},"max":{"1min":8.9930000000000003,"5min":77.204999999999998,"15min":77.204999999999998},"last":0.56699999999999995}]}]},{"osd":0,"up_from":91,"seq":390842024226,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1252852,"kb_used_data":188332,"kb_used_omap":5,"kb_used_meta":1064442,"kb_avail":92492300,"statfs":{"total":95995035648,"available":94712115200,"internally_reserved":0,"allocated":192851968,"data_stored":190645513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1089989242},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:27:18 2023","interfaces":[{"interface":"back","average":{"1min":3.29,"5min":4.29,"15min":3.9260000000000002},"min":{"1min":0.17799999999999999,"5min":0.17499999999999999,"15min":0.16900000000000001},"max":{"1min":15.144,"5min":139.30000000000001,"15min":155.91999999999999},"last":1.4279999999999999},{"interface":"front","average":{"1min":3.488,"5min":1.998,"15min":2.742},"min":{"1min":0.22700000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":10.423,"5min":10.423,"15min":153.28899999999999},"last":1.6499999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:27:40 2023","interfaces":[{"interface":"back","average":{"1min":6.4240000000000004,"5min":7.3380000000000001,"15min":4.9690000000000003},"min":{"1min":0.22,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":80.736000000000004,"5min":171.15100000000001,"15min":171.15100000000001},"last":0.96599999999999997},{"interface":"front","average":{"1min":7.1420000000000003,"5min":6.9139999999999997,"15min":5.2640000000000002},"min":{"1min":0.26400000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":85.375,"5min":173.04900000000001,"15min":173.04900000000001},"last":1.778}]},{"osd":3,"last update":"Sat Sep 23 23:27:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1019999999999999,"5min":4.6459999999999999,"15min":3.8399999999999999},"min":{"1min":0.32900000000000001,"5min":0.25600000000000001,"15min":0.23200000000000001},"max":{"1min":13.112,"5min":139.34899999999999,"15min":139.34899999999999},"last":0.253},{"interface":"front","average":{"1min":3.2480000000000002,"5min":4.5229999999999997,"15min":3.4209999999999998},"min":{"1min":0.311,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":10.237,"5min":138.739,"15min":138.739},"last":4.5510000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:27:47 2023","interfaces":[{"interface":"back","average":{"1min":1.786,"5min":3.3809999999999998,"15min":2.6059999999999999},"min":{"1min":0.32200000000000001,"5min":0.29099999999999998,"15min":0.27000000000000002},"max":{"1min":9.2040000000000006,"5min":138.81299999999999,"15min":138.81299999999999},"last":1.746},{"interface":"front","average":{"1min":1.55,"5min":5.415,"15min":3.2839999999999998},"min":{"1min":0.33200000000000002,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":7.0099999999999998,"5min":171.251,"15min":171.251},"last":0.33900000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:27:47 2023","interfaces":[{"interface":"back","average":{"1min":1.7889999999999999,"5min":5.5490000000000004,"15min":2.9649999999999999},"min":{"1min":0.34499999999999997,"5min":0.32100000000000001,"15min":0.29099999999999998},"max":{"1min":6.9699999999999998,"5min":171.30699999999999,"15min":171.30699999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.964,"5min":3.6080000000000001,"15min":2.2509999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":11.307,"5min":139.40799999999999,"15min":139.40799999999999},"last":0.40300000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:27:47 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":5.4320000000000004,"15min":3.161},"min":{"1min":0.371,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.0419999999999998,"5min":171.33600000000001,"15min":171.33600000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.948,"5min":6.1689999999999996,"15min":3.4049999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.33,"5min":184.13499999999999,"15min":184.13499999999999},"last":0.46600000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:27:40 2023","interfaces":[{"interface":"back","average":{"1min":2.375,"5min":3.7650000000000001,"15min":2.149},"min":{"1min":0.378,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":9.1739999999999995,"5min":138.77699999999999,"15min":138.77699999999999},"last":1.8089999999999999},{"interface":"front","average":{"1min":2.29,"5min":3.6720000000000002,"15min":1.8120000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":11.27,"5min":139.37899999999999,"15min":139.37899999999999},"last":0.441}]}]},{"osd":2,"up_from":199,"seq":854698492047,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":644696,"kb_used_data":116624,"kb_used_omap":7,"kb_used_meta":527928,"kb_avail":93100456,"statfs":{"total":95995035648,"available":95334866944,"internally_reserved":0,"allocated":119422976,"data_stored":117249078,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":540598353},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":4.5220000000000002,"5min":4.6079999999999997,"15min":5.0019999999999998},"min":{"1min":0.63200000000000001,"5min":0.17699999999999999,"15min":0.16700000000000001},"max":{"1min":10.641,"5min":95.587999999999994,"15min":119.277},"last":1.591},{"interface":"front","average":{"1min":4.7290000000000001,"5min":4.3849999999999998,"15min":5.1150000000000002},"min":{"1min":0.27400000000000002,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":18.055,"5min":95.796000000000006,"15min":119.56100000000001},"last":3.0299999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":5.1719999999999997,"5min":5.5960000000000001,"15min":7.4160000000000004},"min":{"1min":0.72899999999999998,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":13.441000000000001,"5min":157.98699999999999,"15min":157.98699999999999},"last":1.784},{"interface":"front","average":{"1min":4.6319999999999997,"5min":5.6029999999999998,"15min":7.1150000000000002},"min":{"1min":0.247,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":14.271000000000001,"5min":158.03800000000001,"15min":158.03800000000001},"last":1.7350000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":7.3120000000000003,"5min":4.4080000000000004,"15min":5.266},"min":{"1min":0.26200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":46.807000000000002,"5min":74.867999999999995,"15min":119.366},"last":1.6699999999999999},{"interface":"front","average":{"1min":6.8339999999999996,"5min":4.2999999999999998,"15min":4.5609999999999999},"min":{"1min":0.35999999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":47.706000000000003,"5min":75.418000000000006,"15min":75.418000000000006},"last":1.5569999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":3.165,"5min":1.9910000000000001,"15min":3.984},"min":{"1min":0.38300000000000001,"5min":0.28000000000000003,"15min":0.25},"max":{"1min":10.157,"5min":23.391999999999999,"15min":332.98099999999999},"last":2.9020000000000001},{"interface":"front","average":{"1min":2.7919999999999998,"5min":1.776,"15min":3.4340000000000002},"min":{"1min":0.30399999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.6839999999999993,"5min":21.076000000000001,"15min":332.96199999999999},"last":1.6259999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":3.3330000000000002,"5min":1.984,"15min":2.3730000000000002},"min":{"1min":0.35899999999999999,"5min":0.318,"15min":0.27500000000000002},"max":{"1min":10.192,"5min":23.359999999999999,"15min":119.919},"last":2.9399999999999999},{"interface":"front","average":{"1min":2.8639999999999999,"5min":1.8600000000000001,"15min":2.323},"min":{"1min":0.35599999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.7110000000000003,"5min":21.105,"15min":119.306},"last":1.6479999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":3.206,"5min":2.0350000000000001,"15min":4.2329999999999997},"min":{"1min":0.40699999999999997,"5min":0.32000000000000001,"15min":0.31},"max":{"1min":10.218,"5min":23.425000000000001,"15min":349.32799999999997},"last":2.9700000000000002},{"interface":"front","average":{"1min":2.8460000000000001,"5min":1.823,"15min":3.7149999999999999},"min":{"1min":0.42099999999999999,"5min":0.36699999999999999,"15min":0.34999999999999998},"max":{"1min":8.7300000000000004,"5min":21.353999999999999,"15min":349.17700000000002},"last":1.694}]},{"osd":7,"last update":"Sat Sep 23 23:27:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5419999999999998,"5min":1.905,"15min":1.419},"min":{"1min":0.45300000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.8079999999999998,"5min":20.350000000000001,"15min":20.350000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":2.7149999999999999,"5min":2.0169999999999999,"15min":1.516},"min":{"1min":0.50900000000000001,"5min":0.38,"15min":0.38},"max":{"1min":7.8579999999999997,"5min":20.321000000000002,"15min":20.321000000000002},"last":0.48499999999999999}]}]},{"osd":3,"up_from":148,"seq":635655160016,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1020628,"kb_used_data":384908,"kb_used_omap":2,"kb_used_meta":635645,"kb_avail":92724524,"statfs":{"total":95995035648,"available":94949912576,"internally_reserved":0,"allocated":394145792,"data_stored":392029835,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":650900533},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,4],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:27:32 2023","interfaces":[{"interface":"back","average":{"1min":3.9369999999999998,"5min":4.4020000000000001,"15min":3.3399999999999999},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.158},"max":{"1min":40.901000000000003,"5min":103.866,"15min":103.866},"last":0.53500000000000003},{"interface":"front","average":{"1min":4.8689999999999998,"5min":4.2389999999999999,"15min":3.0339999999999998},"min":{"1min":0.191,"5min":0.191,"15min":0.186},"max":{"1min":46.204000000000001,"5min":101.884,"15min":101.884},"last":0.376}]},{"osd":1,"last update":"Sat Sep 23 23:27:32 2023","interfaces":[{"interface":"back","average":{"1min":3.5379999999999998,"5min":2.6829999999999998,"15min":2.851},"min":{"1min":0.218,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":26.959,"5min":38.094000000000001,"15min":76.322999999999993},"last":0.40400000000000003},{"interface":"front","average":{"1min":5.0220000000000002,"5min":2.8530000000000002,"15min":2.7330000000000001},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.20499999999999999},"max":{"1min":27.321999999999999,"5min":42.030999999999999,"15min":76.444000000000003},"last":0.57299999999999995}]},{"osd":2,"last update":"Sat Sep 23 23:27:40 2023","interfaces":[{"interface":"back","average":{"1min":5.8639999999999999,"5min":3.96,"15min":3.8959999999999999},"min":{"1min":0.26000000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":66.364999999999995,"5min":66.364999999999995,"15min":101.279},"last":3.8769999999999998},{"interface":"front","average":{"1min":6.3479999999999999,"5min":3.77,"15min":3.04},"min":{"1min":0.29299999999999998,"5min":0.214,"15min":0.214},"max":{"1min":66.450000000000003,"5min":66.450000000000003,"15min":101.294},"last":5.1920000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:27:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":1.1699999999999999,"15min":1.4339999999999999},"min":{"1min":0.27500000000000002,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":6.8890000000000002,"5min":7.2610000000000001,"15min":14.754},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.5029999999999999,"5min":1.482,"15min":1.4770000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.2800000000000002,"5min":15.619,"15min":15.619},"last":0.47599999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:27:32 2023","interfaces":[{"interface":"back","average":{"1min":1.788,"5min":1.573,"15min":1.6819999999999999},"min":{"1min":0.29599999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.2619999999999996,"5min":15.762,"15min":75.733000000000004},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.6990000000000001,"5min":1.679,"15min":1.9730000000000001},"min":{"1min":0.318,"5min":0.316,"15min":0.28399999999999997},"max":{"1min":6.9880000000000004,"5min":13.864000000000001,"15min":75.686999999999998},"last":0.48899999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:27:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6120000000000001,"5min":2.1480000000000001,"15min":1.7130000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.3019999999999996,"5min":53.057000000000002,"15min":53.057000000000002},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.6419999999999999,"5min":2.2029999999999998,"15min":1.9279999999999999},"min":{"1min":0.40000000000000002,"5min":0.35299999999999998,"15min":0.33300000000000002},"max":{"1min":7.2320000000000002,"5min":53.088000000000001,"15min":53.088000000000001},"last":0.622}]},{"osd":7,"last update":"Sat Sep 23 23:27:40 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":1.3149999999999999,"15min":1.0129999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.2590000000000003,"5min":8.2590000000000003,"15min":8.2590000000000003},"last":0.61599999999999999},{"interface":"front","average":{"1min":2.331,"5min":1.7509999999999999,"15min":1.405},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":13.41,"5min":15.795,"15min":15.795},"last":0.44700000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019334,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":798488,"kb_used_data":89552,"kb_used_omap":0,"kb_used_meta":708928,"kb_avail":92946664,"statfs":{"total":95995035648,"available":95177383936,"internally_reserved":0,"allocated":91701248,"data_stored":90408260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":725942272},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:27:55 2023","interfaces":[{"interface":"back","average":{"1min":3.8149999999999999,"5min":3.8330000000000002,"15min":4.7480000000000002},"min":{"1min":0.23599999999999999,"5min":0.219,"15min":0.218},"max":{"1min":40.710000000000001,"5min":92.141999999999996,"15min":199.64400000000001},"last":1.657},{"interface":"front","average":{"1min":3.496,"5min":3.8140000000000001,"15min":4.8460000000000001},"min":{"1min":0.27100000000000002,"5min":0.216,"15min":0.216},"max":{"1min":38.078000000000003,"5min":92.284000000000006,"15min":195.649},"last":0.79300000000000004}]},{"osd":1,"last update":"Sat Sep 23 23:27:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":1.585,"15min":3.8650000000000002},"min":{"1min":0.24299999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.8700000000000001,"5min":40.643000000000001,"15min":161.923},"last":0.38500000000000001},{"interface":"front","average":{"1min":2.2650000000000001,"5min":1.901,"15min":4.0259999999999998},"min":{"1min":0.307,"5min":0.27500000000000002,"15min":0.26200000000000001},"max":{"1min":10.657999999999999,"5min":35.362000000000002,"15min":161.39699999999999},"last":2.1699999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:27:39 2023","interfaces":[{"interface":"back","average":{"1min":3.4390000000000001,"5min":4.4100000000000001,"15min":3.302},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.26300000000000001},"max":{"1min":29.797000000000001,"5min":97.962000000000003,"15min":97.962000000000003},"last":0.34499999999999997},{"interface":"front","average":{"1min":2.8940000000000001,"5min":4.2919999999999998,"15min":3.008},"min":{"1min":0.39300000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":25.446999999999999,"5min":98.137,"15min":98.137},"last":0.92000000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:27:35 2023","interfaces":[{"interface":"back","average":{"1min":4.1859999999999999,"5min":6.5609999999999999,"15min":5.6760000000000002},"min":{"1min":0.33500000000000002,"5min":0.27200000000000002,"15min":0.26300000000000001},"max":{"1min":50.755000000000003,"5min":176.74100000000001,"15min":303.41500000000002},"last":0.80300000000000005},{"interface":"front","average":{"1min":3.3660000000000001,"5min":6.1959999999999997,"15min":5.7530000000000001},"min":{"1min":0.39400000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":50.738999999999997,"5min":176.87799999999999,"15min":303.428},"last":0.28999999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:27:09 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.65000000000000002,"15min":1.4379999999999999},"min":{"1min":0.29499999999999998,"5min":0.26100000000000001,"15min":0.23699999999999999},"max":{"1min":4.266,"5min":4.266,"15min":160.79499999999999},"last":161.86500000000001},{"interface":"front","average":{"1min":0.879,"5min":0.746,"15min":1.6080000000000001},"min":{"1min":0.34999999999999998,"5min":0.249,"15min":0.249},"max":{"1min":4.4359999999999999,"5min":5.0789999999999997,"15min":160.61099999999999},"last":159.24199999999999}]},{"osd":6,"last update":"Sat Sep 23 23:27:35 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":1.55,"15min":1.169},"min":{"1min":0.29799999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.097,"5min":90.346999999999994,"15min":90.346999999999994},"last":0.372},{"interface":"front","average":{"1min":0.56100000000000005,"5min":1.6220000000000001,"15min":1.3100000000000001},"min":{"1min":0.38300000000000001,"5min":0.31900000000000001,"15min":0.30599999999999999},"max":{"1min":1.0980000000000001,"5min":90.733000000000004,"15min":90.733000000000004},"last":0.35599999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:27:39 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":1.538,"15min":0.89000000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.8109999999999999,"5min":61.923999999999999,"15min":61.923999999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.64900000000000002,"5min":1.5509999999999999,"15min":1.052},"min":{"1min":0.40100000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.655,"5min":61.990000000000002,"15min":61.990000000000002},"last":0.33500000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757697,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":738948,"kb_used_data":127676,"kb_used_omap":0,"kb_used_meta":611264,"kb_avail":93006204,"statfs":{"total":95995035648,"available":95238352896,"internally_reserved":0,"allocated":130740224,"data_stored":129463715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":625934336},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:27:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":2.1520000000000001,"15min":3.7829999999999999},"min":{"1min":0.26200000000000001,"5min":0.222,"15min":0.19500000000000001},"max":{"1min":19.952000000000002,"5min":63.741999999999997,"15min":113.869},"last":0.29999999999999999},{"interface":"front","average":{"1min":1.4219999999999999,"5min":3.3250000000000002,"15min":4.2169999999999996},"min":{"1min":0.26600000000000001,"5min":0.22900000000000001,"15min":0.22700000000000001},"max":{"1min":5.7320000000000002,"5min":138.55199999999999,"15min":138.55199999999999},"last":0.22700000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:28:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":3.5979999999999999,"15min":3.996},"min":{"1min":0.252,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":12.702,"5min":69.816999999999993,"15min":193.584},"last":0.25},{"interface":"front","average":{"1min":1.5069999999999999,"5min":4.0380000000000003,"15min":4.2699999999999996},"min":{"1min":0.28999999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.4560000000000004,"5min":70.268000000000001,"15min":194.03100000000001},"last":0.33000000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:27:35 2023","interfaces":[{"interface":"back","average":{"1min":2.0750000000000002,"5min":3.952,"15min":2.645},"min":{"1min":0.314,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.5640000000000001,"5min":144.91200000000001,"15min":144.91200000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.776,"5min":3.806,"15min":2.5619999999999998},"min":{"1min":0.28799999999999998,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":5.8410000000000002,"5min":145.15799999999999,"15min":145.15799999999999},"last":0.26500000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:27:29 2023","interfaces":[{"interface":"back","average":{"1min":4.4880000000000004,"5min":3.008,"15min":4.468},"min":{"1min":0.32700000000000001,"5min":0.315,"15min":0.27400000000000002},"max":{"1min":62.612000000000002,"5min":62.612000000000002,"15min":159.53299999999999},"last":0.748},{"interface":"front","average":{"1min":4.3970000000000002,"5min":3.1000000000000001,"15min":4.6980000000000004},"min":{"1min":0.318,"5min":0.318,"15min":0.312},"max":{"1min":62.639000000000003,"5min":62.639000000000003,"15min":159.54900000000001},"last":0.75600000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:27:10 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.71699999999999997,"15min":0.98299999999999998},"min":{"1min":0.26700000000000002,"5min":0.251,"15min":0.23999999999999999},"max":{"1min":1.1919999999999999,"5min":5.3739999999999997,"15min":10.012},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.60899999999999999,"15min":0.76300000000000001},"min":{"1min":0.30299999999999999,"5min":0.245,"15min":0.245},"max":{"1min":3.2530000000000001,"5min":3.2530000000000001,"15min":11.923999999999999},"last":0.33800000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:27:10 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.4770000000000001,"15min":2.2250000000000001},"min":{"1min":0.33000000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.2749999999999999,"5min":68.456999999999994,"15min":310.68799999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":1.452,"15min":2.4209999999999998},"min":{"1min":0.34799999999999998,"5min":0.26000000000000001,"15min":0.25900000000000001},"max":{"1min":1.2050000000000001,"5min":66.721999999999994,"15min":311.45400000000001},"last":0.317}]},{"osd":7,"last update":"Sat Sep 23 23:27:35 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":2.2160000000000002,"15min":3.5030000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.0859999999999999,"5min":66.372,"15min":66.372},"last":31.041},{"interface":"front","average":{"1min":0.66400000000000003,"5min":2.262,"15min":3.6440000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.758,"5min":66.558999999999997,"15min":66.558999999999997},"last":30.856999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":169869312,"data_stored":169869312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":177999872,"data_stored":177995800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":98897920,"data_stored":98893848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":355467264,"data_stored":355467264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":86577152,"data_stored":86573080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":114229248,"data_stored":114229248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":748879872,"data_stored":748879872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":856555520,"data_stored":856555520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:28:14.056 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T23:28:14.056 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:28:14.056 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T23:28:14.056 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.168135 2023-09-23T23:28:14.057 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 0.1681348044973131 2023-09-23T23:28:14.071 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:14.098 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:14.141 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:14.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:14.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:14.248 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:14.273 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:14.315 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:14.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:14.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:14.417 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:14 smithi138 ceph-mon[193606]: osdmap e458: 8 total, 8 up, 7 in 2023-09-23T23:28:14.417 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:14 smithi138 ceph-mon[193606]: failed to encode map e458 with expected crc 2023-09-23T23:28:14.417 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:14 smithi138 ceph-mon[193606]: failed to encode map e458 with expected crc 2023-09-23T23:28:14.417 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:14 smithi138 ceph-mon[193606]: failed to encode map e458 with expected crc 2023-09-23T23:28:14.417 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:14 smithi138 ceph-mon[193606]: failed to encode map e458 with expected crc 2023-09-23T23:28:14.417 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:14 smithi138 ceph-mon[193606]: pgmap v1269: 129 pgs: 129 active+clean; 231 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:28:14.417 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:14 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T23:28:14.417 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:14 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T23:28:14.417 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:14 smithi138 ceph-mon[193606]: 11.5 scrub starts 2023-09-23T23:28:14.418 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:14 smithi138 ceph-mon[193606]: 11.5 scrub ok 2023-09-23T23:28:14.421 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:14.447 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:14.492 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:14.492 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[221102]: osdmap e458: 8 total, 8 up, 7 in 2023-09-23T23:28:14.493 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[221102]: failed to encode map e458 with expected crc 2023-09-23T23:28:14.493 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[221102]: failed to encode map e458 with expected crc 2023-09-23T23:28:14.493 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[221102]: failed to encode map e458 with expected crc 2023-09-23T23:28:14.493 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[221102]: failed to encode map e458 with expected crc 2023-09-23T23:28:14.493 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[221102]: pgmap v1269: 129 pgs: 129 active+clean; 231 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:28:14.493 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T23:28:14.493 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T23:28:14.493 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[221102]: 11.5 scrub starts 2023-09-23T23:28:14.493 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[221102]: 11.5 scrub ok 2023-09-23T23:28:14.493 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[212939]: osdmap e458: 8 total, 8 up, 7 in 2023-09-23T23:28:14.494 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[212939]: failed to encode map e458 with expected crc 2023-09-23T23:28:14.494 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[212939]: failed to encode map e458 with expected crc 2023-09-23T23:28:14.494 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[212939]: failed to encode map e458 with expected crc 2023-09-23T23:28:14.494 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[212939]: failed to encode map e458 with expected crc 2023-09-23T23:28:14.494 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[212939]: pgmap v1269: 129 pgs: 129 active+clean; 231 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:28:14.494 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T23:28:14.494 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T23:28:14.494 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[212939]: 11.5 scrub starts 2023-09-23T23:28:14.494 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:14 smithi110 ceph-mon[212939]: 11.5 scrub ok 2023-09-23T23:28:14.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:14.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:14.598 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:14.625 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:15.090 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:15.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:15.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:15.198 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:15.223 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:15.266 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:15.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:15.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:15.373 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:15.399 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:15.424 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:28:15.445 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:15.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:15.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:15.551 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:15.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[212939]: 11.1 scrub starts 2023-09-23T23:28:15.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[212939]: 11.1 scrub ok 2023-09-23T23:28:15.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[212939]: 1.11 deep-scrub starts 2023-09-23T23:28:15.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[212939]: 1.11 deep-scrub ok 2023-09-23T23:28:15.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[212939]: failed to encode map e459 with expected crc 2023-09-23T23:28:15.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[212939]: failed to encode map e459 with expected crc 2023-09-23T23:28:15.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[212939]: failed to encode map e459 with expected crc 2023-09-23T23:28:15.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T23:28:15.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[212939]: osdmap e459: 8 total, 8 up, 7 in 2023-09-23T23:28:15.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[212939]: 11.13 starting backfill to osd.0 from (0'0,0'0] MAX to 455'9788 2023-09-23T23:28:15.578 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[212939]: 11.13 starting backfill to osd.4 from (0'0,0'0] MAX to 455'9788 2023-09-23T23:28:15.578 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[212939]: failed to encode map e459 with expected crc 2023-09-23T23:28:15.578 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/510746866' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:28:15.578 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[221102]: 11.1 scrub starts 2023-09-23T23:28:15.578 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[221102]: 11.1 scrub ok 2023-09-23T23:28:15.578 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[221102]: 1.11 deep-scrub starts 2023-09-23T23:28:15.578 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[221102]: 1.11 deep-scrub ok 2023-09-23T23:28:15.578 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[221102]: failed to encode map e459 with expected crc 2023-09-23T23:28:15.578 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[221102]: failed to encode map e459 with expected crc 2023-09-23T23:28:15.578 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[221102]: failed to encode map e459 with expected crc 2023-09-23T23:28:15.579 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T23:28:15.579 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[221102]: osdmap e459: 8 total, 8 up, 7 in 2023-09-23T23:28:15.579 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[221102]: 11.13 starting backfill to osd.0 from (0'0,0'0] MAX to 455'9788 2023-09-23T23:28:15.579 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[221102]: 11.13 starting backfill to osd.4 from (0'0,0'0] MAX to 455'9788 2023-09-23T23:28:15.579 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[221102]: failed to encode map e459 with expected crc 2023-09-23T23:28:15.579 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:15 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/510746866' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:28:15.579 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:15 smithi138 ceph-mon[193606]: 11.1 scrub starts 2023-09-23T23:28:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:15 smithi138 ceph-mon[193606]: 11.1 scrub ok 2023-09-23T23:28:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:15 smithi138 ceph-mon[193606]: 1.11 deep-scrub starts 2023-09-23T23:28:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:15 smithi138 ceph-mon[193606]: 1.11 deep-scrub ok 2023-09-23T23:28:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:15 smithi138 ceph-mon[193606]: failed to encode map e459 with expected crc 2023-09-23T23:28:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:15 smithi138 ceph-mon[193606]: failed to encode map e459 with expected crc 2023-09-23T23:28:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:15 smithi138 ceph-mon[193606]: failed to encode map e459 with expected crc 2023-09-23T23:28:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:15 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T23:28:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:15 smithi138 ceph-mon[193606]: osdmap e459: 8 total, 8 up, 7 in 2023-09-23T23:28:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:15 smithi138 ceph-mon[193606]: 11.13 starting backfill to osd.0 from (0'0,0'0] MAX to 455'9788 2023-09-23T23:28:15.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:15 smithi138 ceph-mon[193606]: 11.13 starting backfill to osd.4 from (0'0,0'0] MAX to 455'9788 2023-09-23T23:28:15.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:15 smithi138 ceph-mon[193606]: failed to encode map e459 with expected crc 2023-09-23T23:28:15.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:15 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/510746866' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:28:16.048 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:16.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:16.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:16.152 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:16.184 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:16.233 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:16.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:16.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:16.340 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:16.369 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:16.414 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:16.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:16.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:16.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[212939]: failed to encode map e459 with expected crc 2023-09-23T23:28:16.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:28:16.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[212939]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[212939]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/510746866' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:28:16.426 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[212939]: osdmap e460: 8 total, 8 up, 7 in 2023-09-23T23:28:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:16 smithi138 ceph-mon[193606]: failed to encode map e459 with expected crc 2023-09-23T23:28:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:16 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:28:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:16 smithi138 ceph-mon[193606]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:16 smithi138 ceph-mon[193606]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:16 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/510746866' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:28:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:16 smithi138 ceph-mon[193606]: osdmap e460: 8 total, 8 up, 7 in 2023-09-23T23:28:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:16 smithi138 ceph-mon[193606]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:16 smithi138 ceph-mon[193606]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:16 smithi138 ceph-mon[193606]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:16 smithi138 ceph-mon[193606]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:16 smithi138 ceph-mon[193606]: pgmap v1272: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 178 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 66/8586 objects misplaced (0.769%) 2023-09-23T23:28:16.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:16 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T23:28:16.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:16 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T23:28:16.520 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:16.550 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:16.606 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:16.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:16.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[221102]: failed to encode map e459 with expected crc 2023-09-23T23:28:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:28:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[221102]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[221102]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/510746866' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:28:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[221102]: osdmap e460: 8 total, 8 up, 7 in 2023-09-23T23:28:16.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[221102]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[221102]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[221102]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[221102]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[221102]: pgmap v1272: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 178 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 66/8586 objects misplaced (0.769%) 2023-09-23T23:28:16.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T23:28:16.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T23:28:16.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[212939]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[212939]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[212939]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[212939]: failed to encode map e460 with expected crc 2023-09-23T23:28:16.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[212939]: pgmap v1272: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 178 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 66/8586 objects misplaced (0.769%) 2023-09-23T23:28:16.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T23:28:16.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:16 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T23:28:16.713 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:16.745 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:16.799 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:16.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:16.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:16.906 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:16.934 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:16.959 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:28:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:28:16] "GET /metrics HTTP/1.1" 200 36533 "" "Prometheus/2.43.0" 2023-09-23T23:28:16.982 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:16.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:16.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:17.022 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:17.064 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:28:17.086 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:17.099 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:28:17.113 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:17.128 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:28:17.153 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:28:17.156 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:17.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:17.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:17.178 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:28:17.201 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:28:17.222 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:28:17.226 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:28:17.226 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:28:17.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:17.226 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:209863 11.15 11.99386895 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e461)", 2023-09-23T23:28:17.226 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:17.192412+0000", 2023-09-23T23:28:17.226 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.032606142999999997, 2023-09-23T23:28:17.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.032628111000000001, 2023-09-23T23:28:17.226 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:17.226 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "delayed", 2023-09-23T23:28:17.227 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:17.227 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:28:17.227 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:17.227 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 209863 2023-09-23T23:28:17.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:17.227 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:17.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:17.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:17.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.192412+0000", 2023-09-23T23:28:17.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:17.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:17.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:17.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:17.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.192412+0000", 2023-09-23T23:28:17.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:17.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:17.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:17.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:17.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.192411+0000", 2023-09-23T23:28:17.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:28:17.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:17.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:17.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:17.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.192416+0000", 2023-09-23T23:28:17.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8230000000000002e-06 2023-09-23T23:28:17.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:17.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:17.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:17.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.192427+0000", 2023-09-23T23:28:17.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0625000000000001e-05 2023-09-23T23:28:17.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:17.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:17.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:17.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.192432+0000", 2023-09-23T23:28:17.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4709999999999998e-06 2023-09-23T23:28:17.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:17.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:17.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:17.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.192467+0000", 2023-09-23T23:28:17.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4582999999999997e-05 2023-09-23T23:28:17.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:17.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:17.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:17.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.192472+0000", 2023-09-23T23:28:17.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3469999999999998e-06 2023-09-23T23:28:17.230 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:17.230 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:17.230 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:17.230 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:17.230 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-09-23T23:28:17.230 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 1 2023-09-23T23:28:17.230 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:28:17.246 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:28:17.251 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:28:17.251 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:28:17.252 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:28:17.252 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:28:17.252 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:28:17.252 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:28:17.252 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:28:17.252 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:28:17.262 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:17.288 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:17.330 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:17.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:17.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:17.436 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:28:17.469 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:17.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:17 smithi138 ceph-mon[193606]: 11.18 scrub starts 2023-09-23T23:28:17.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:17 smithi138 ceph-mon[193606]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:17 smithi138 ceph-mon[193606]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:17 smithi138 ceph-mon[193606]: 11.14 starting backfill to osd.4 from (0'0,0'0] MAX to 457'9857 2023-09-23T23:28:17.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:17 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T23:28:17.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:17 smithi138 ceph-mon[193606]: osdmap e461: 8 total, 8 up, 7 in 2023-09-23T23:28:17.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:17 smithi138 ceph-mon[193606]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:17 smithi138 ceph-mon[193606]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:17 smithi138 ceph-mon[193606]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:17 smithi138 ceph-mon[193606]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:17 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4063852380' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:17.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[212939]: 11.18 scrub starts 2023-09-23T23:28:17.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[212939]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[212939]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[212939]: 11.14 starting backfill to osd.4 from (0'0,0'0] MAX to 457'9857 2023-09-23T23:28:17.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T23:28:17.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[212939]: osdmap e461: 8 total, 8 up, 7 in 2023-09-23T23:28:17.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[212939]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[212939]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[212939]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[212939]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4063852380' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:17.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[221102]: 11.18 scrub starts 2023-09-23T23:28:17.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[221102]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[221102]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[221102]: 11.14 starting backfill to osd.4 from (0'0,0'0] MAX to 457'9857 2023-09-23T23:28:17.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T23:28:17.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[221102]: osdmap e461: 8 total, 8 up, 7 in 2023-09-23T23:28:17.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[221102]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[221102]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[221102]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[221102]: failed to encode map e461 with expected crc 2023-09-23T23:28:17.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:17 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4063852380' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:18.267 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:28:18.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:18.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:28:18.371 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:18.401 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:18.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:18 smithi138 ceph-mon[193606]: failed to encode map e462 with expected crc 2023-09-23T23:28:18.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:18 smithi138 ceph-mon[193606]: failed to encode map e462 with expected crc 2023-09-23T23:28:18.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:18 smithi138 ceph-mon[193606]: osdmap e462: 8 total, 8 up, 7 in 2023-09-23T23:28:18.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:18 smithi138 ceph-mon[193606]: failed to encode map e462 with expected crc 2023-09-23T23:28:18.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:18 smithi138 ceph-mon[193606]: pgmap v1275: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 172 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 66/8319 objects misplaced (0.793%) 2023-09-23T23:28:18.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:18 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T23:28:18.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:18 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T23:28:18.708 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:18 smithi110 ceph-mon[212939]: failed to encode map e462 with expected crc 2023-09-23T23:28:18.708 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:18 smithi110 ceph-mon[212939]: failed to encode map e462 with expected crc 2023-09-23T23:28:18.708 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:18 smithi110 ceph-mon[212939]: osdmap e462: 8 total, 8 up, 7 in 2023-09-23T23:28:18.708 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:18 smithi110 ceph-mon[212939]: failed to encode map e462 with expected crc 2023-09-23T23:28:18.708 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:18 smithi110 ceph-mon[212939]: pgmap v1275: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 172 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 66/8319 objects misplaced (0.793%) 2023-09-23T23:28:18.708 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:18 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T23:28:18.708 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:18 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T23:28:18.709 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:18 smithi110 ceph-mon[221102]: failed to encode map e462 with expected crc 2023-09-23T23:28:18.709 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:18 smithi110 ceph-mon[221102]: failed to encode map e462 with expected crc 2023-09-23T23:28:18.709 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:18 smithi110 ceph-mon[221102]: osdmap e462: 8 total, 8 up, 7 in 2023-09-23T23:28:18.709 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:18 smithi110 ceph-mon[221102]: failed to encode map e462 with expected crc 2023-09-23T23:28:18.709 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:18 smithi110 ceph-mon[221102]: pgmap v1275: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 172 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 66/8319 objects misplaced (0.793%) 2023-09-23T23:28:18.709 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:18 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T23:28:18.709 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:18 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T23:28:18.736 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:28:18.768 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:18.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:18.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:18.878 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:28:18.905 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:19.631 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:28:19.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:19.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:28:19.674 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:28:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:19 smithi138 ceph-mon[193606]: failed to encode map e463 with expected crc 2023-09-23T23:28:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:19 smithi138 ceph-mon[193606]: failed to encode map e463 with expected crc 2023-09-23T23:28:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:19 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T23:28:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:19 smithi138 ceph-mon[193606]: osdmap e463: 8 total, 8 up, 7 in 2023-09-23T23:28:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:19 smithi138 ceph-mon[193606]: failed to encode map e463 with expected crc 2023-09-23T23:28:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:19 smithi138 ceph-mon[193606]: 11.15 starting backfill to osd.1 from (0'0,0'0] MAX to 459'9769 2023-09-23T23:28:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:19 smithi138 ceph-mon[193606]: 11.15 starting backfill to osd.2 from (0'0,0'0] MAX to 459'9769 2023-09-23T23:28:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:19 smithi138 ceph-mon[193606]: 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 459'9769 2023-09-23T23:28:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:19 smithi138 ceph-mon[193606]: failed to encode map e464 with expected crc 2023-09-23T23:28:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:19 smithi138 ceph-mon[193606]: osdmap e464: 8 total, 8 up, 7 in 2023-09-23T23:28:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:19 smithi138 ceph-mon[193606]: failed to encode map e464 with expected crc 2023-09-23T23:28:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:19 smithi138 ceph-mon[193606]: failed to encode map e464 with expected crc 2023-09-23T23:28:19.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:19 smithi138 ceph-mon[193606]: failed to encode map e464 with expected crc 2023-09-23T23:28:19.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:19 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:19.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[212939]: failed to encode map e463 with expected crc 2023-09-23T23:28:19.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[212939]: failed to encode map e463 with expected crc 2023-09-23T23:28:19.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T23:28:19.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[212939]: osdmap e463: 8 total, 8 up, 7 in 2023-09-23T23:28:19.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[212939]: failed to encode map e463 with expected crc 2023-09-23T23:28:19.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[212939]: 11.15 starting backfill to osd.1 from (0'0,0'0] MAX to 459'9769 2023-09-23T23:28:19.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[212939]: 11.15 starting backfill to osd.2 from (0'0,0'0] MAX to 459'9769 2023-09-23T23:28:19.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[212939]: 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 459'9769 2023-09-23T23:28:19.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[212939]: failed to encode map e464 with expected crc 2023-09-23T23:28:19.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[212939]: osdmap e464: 8 total, 8 up, 7 in 2023-09-23T23:28:19.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[212939]: failed to encode map e464 with expected crc 2023-09-23T23:28:19.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[212939]: failed to encode map e464 with expected crc 2023-09-23T23:28:19.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[212939]: failed to encode map e464 with expected crc 2023-09-23T23:28:19.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:19.715 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[221102]: failed to encode map e463 with expected crc 2023-09-23T23:28:19.715 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[221102]: failed to encode map e463 with expected crc 2023-09-23T23:28:19.715 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T23:28:19.715 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[221102]: osdmap e463: 8 total, 8 up, 7 in 2023-09-23T23:28:19.715 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[221102]: failed to encode map e463 with expected crc 2023-09-23T23:28:19.715 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[221102]: 11.15 starting backfill to osd.1 from (0'0,0'0] MAX to 459'9769 2023-09-23T23:28:19.715 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[221102]: 11.15 starting backfill to osd.2 from (0'0,0'0] MAX to 459'9769 2023-09-23T23:28:19.715 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[221102]: 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 459'9769 2023-09-23T23:28:19.715 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[221102]: failed to encode map e464 with expected crc 2023-09-23T23:28:19.716 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[221102]: osdmap e464: 8 total, 8 up, 7 in 2023-09-23T23:28:19.716 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[221102]: failed to encode map e464 with expected crc 2023-09-23T23:28:19.716 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[221102]: failed to encode map e464 with expected crc 2023-09-23T23:28:19.716 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[221102]: failed to encode map e464 with expected crc 2023-09-23T23:28:19.716 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:19 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:19.738 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:19.766 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:19.818 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:19.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:19.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:19.924 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:19.951 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:20.001 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:20.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:20.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:20.108 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:20.139 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:20.186 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:20.188 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:20.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:20.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:20.294 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:20.321 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:20.368 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:20.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:20.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:20.420 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:28:20.476 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:20.504 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:20.550 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:20.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:20.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:20.658 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:20.749 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[212939]: osdmap e465: 8 total, 8 up, 7 in 2023-09-23T23:28:20.749 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[212939]: failed to encode map e465 with expected crc 2023-09-23T23:28:20.749 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[212939]: failed to encode map e465 with expected crc 2023-09-23T23:28:20.749 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[212939]: failed to encode map e465 with expected crc 2023-09-23T23:28:20.749 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[212939]: 11.16 starting backfill to osd.3 from (0'0,0'0] MAX to 461'9778 2023-09-23T23:28:20.749 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[212939]: 11.16 starting backfill to osd.5 from (0'0,0'0] MAX to 461'9778 2023-09-23T23:28:20.749 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[212939]: 11.16 starting backfill to osd.7 from (0'0,0'0] MAX to 461'9778 2023-09-23T23:28:20.749 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[212939]: pgmap v1279: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 124 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 165/6000 objects misplaced (2.750%) 2023-09-23T23:28:20.750 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T23:28:20.750 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T23:28:20.750 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[212939]: failed to encode map e465 with expected crc 2023-09-23T23:28:20.750 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[221102]: osdmap e465: 8 total, 8 up, 7 in 2023-09-23T23:28:20.750 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[221102]: failed to encode map e465 with expected crc 2023-09-23T23:28:20.750 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[221102]: failed to encode map e465 with expected crc 2023-09-23T23:28:20.750 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[221102]: failed to encode map e465 with expected crc 2023-09-23T23:28:20.750 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[221102]: 11.16 starting backfill to osd.3 from (0'0,0'0] MAX to 461'9778 2023-09-23T23:28:20.750 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[221102]: 11.16 starting backfill to osd.5 from (0'0,0'0] MAX to 461'9778 2023-09-23T23:28:20.751 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[221102]: 11.16 starting backfill to osd.7 from (0'0,0'0] MAX to 461'9778 2023-09-23T23:28:20.751 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[221102]: pgmap v1279: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 124 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 165/6000 objects misplaced (2.750%) 2023-09-23T23:28:20.751 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T23:28:20.751 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T23:28:20.751 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:20 smithi110 ceph-mon[221102]: failed to encode map e465 with expected crc 2023-09-23T23:28:20.751 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:20.832 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:20.876 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:20.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:20.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:20 smithi138 ceph-mon[193606]: osdmap e465: 8 total, 8 up, 7 in 2023-09-23T23:28:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:20 smithi138 ceph-mon[193606]: failed to encode map e465 with expected crc 2023-09-23T23:28:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:20 smithi138 ceph-mon[193606]: failed to encode map e465 with expected crc 2023-09-23T23:28:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:20 smithi138 ceph-mon[193606]: failed to encode map e465 with expected crc 2023-09-23T23:28:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:20 smithi138 ceph-mon[193606]: 11.16 starting backfill to osd.3 from (0'0,0'0] MAX to 461'9778 2023-09-23T23:28:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:20 smithi138 ceph-mon[193606]: 11.16 starting backfill to osd.5 from (0'0,0'0] MAX to 461'9778 2023-09-23T23:28:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:20 smithi138 ceph-mon[193606]: 11.16 starting backfill to osd.7 from (0'0,0'0] MAX to 461'9778 2023-09-23T23:28:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:20 smithi138 ceph-mon[193606]: pgmap v1279: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 124 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 165/6000 objects misplaced (2.750%) 2023-09-23T23:28:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:20 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T23:28:20.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:20 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T23:28:20.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:20 smithi138 ceph-mon[193606]: failed to encode map e465 with expected crc 2023-09-23T23:28:20.981 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:21.009 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:21.587 INFO:teuthology.orchestra.run.smithi110.stderr:set osd.0 primary-affinity to 0.168135 (8110182) 2023-09-23T23:28:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:21 smithi138 ceph-mon[193606]: failed to encode map e466 with expected crc 2023-09-23T23:28:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:21 smithi138 ceph-mon[193606]: failed to encode map e466 with expected crc 2023-09-23T23:28:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:21 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T23:28:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:21 smithi138 ceph-mon[193606]: osdmap e466: 8 total, 8 up, 7 in 2023-09-23T23:28:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:21 smithi138 ceph-mon[193606]: failed to encode map e466 with expected crc 2023-09-23T23:28:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:21 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/421625210' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.1681348044973131}]: dispatch 2023-09-23T23:28:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:21 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.1681348044973131}]: dispatch 2023-09-23T23:28:21.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:21 smithi110 ceph-mon[212939]: failed to encode map e466 with expected crc 2023-09-23T23:28:21.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:21 smithi110 ceph-mon[212939]: failed to encode map e466 with expected crc 2023-09-23T23:28:21.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:21 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T23:28:21.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:21 smithi110 ceph-mon[212939]: osdmap e466: 8 total, 8 up, 7 in 2023-09-23T23:28:21.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:21 smithi110 ceph-mon[212939]: failed to encode map e466 with expected crc 2023-09-23T23:28:21.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:21 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/421625210' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.1681348044973131}]: dispatch 2023-09-23T23:28:21.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:21 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.1681348044973131}]: dispatch 2023-09-23T23:28:21.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:21 smithi110 ceph-mon[221102]: failed to encode map e466 with expected crc 2023-09-23T23:28:21.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:21 smithi110 ceph-mon[221102]: failed to encode map e466 with expected crc 2023-09-23T23:28:21.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:21 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T23:28:21.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:21 smithi110 ceph-mon[221102]: osdmap e466: 8 total, 8 up, 7 in 2023-09-23T23:28:21.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:21 smithi110 ceph-mon[221102]: failed to encode map e466 with expected crc 2023-09-23T23:28:21.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:21 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/421625210' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.1681348044973131}]: dispatch 2023-09-23T23:28:21.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:21 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.1681348044973131}]: dispatch 2023-09-23T23:28:22.018 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:22.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:22.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:22.123 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:22.150 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:22.191 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:22.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:22.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:22.299 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:22.326 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:22.368 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:22.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:22.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:22.473 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:22.498 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:22.542 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:22.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:22.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:22.649 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:22.688 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:22 smithi138 ceph-mon[193606]: failed to encode map e467 with expected crc 2023-09-23T23:28:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:22 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.1681348044973131}]': finished 2023-09-23T23:28:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:22 smithi138 ceph-mon[193606]: osdmap e467: 8 total, 8 up, 7 in 2023-09-23T23:28:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:22 smithi138 ceph-mon[193606]: failed to encode map e467 with expected crc 2023-09-23T23:28:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:22 smithi138 ceph-mon[193606]: failed to encode map e467 with expected crc 2023-09-23T23:28:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:22 smithi138 ceph-mon[193606]: pgmap v1282: 129 pgs: 129 active+clean; 88 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:22 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T23:28:22.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:22 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T23:28:22.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:22 smithi110 ceph-mon[212939]: failed to encode map e467 with expected crc 2023-09-23T23:28:22.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:22 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.1681348044973131}]': finished 2023-09-23T23:28:22.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:22 smithi110 ceph-mon[212939]: osdmap e467: 8 total, 8 up, 7 in 2023-09-23T23:28:22.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:22 smithi110 ceph-mon[212939]: failed to encode map e467 with expected crc 2023-09-23T23:28:22.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:22 smithi110 ceph-mon[212939]: failed to encode map e467 with expected crc 2023-09-23T23:28:22.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:22 smithi110 ceph-mon[212939]: pgmap v1282: 129 pgs: 129 active+clean; 88 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:22.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:22 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T23:28:22.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:22 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T23:28:22.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:22 smithi110 ceph-mon[221102]: failed to encode map e467 with expected crc 2023-09-23T23:28:22.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:22 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.1681348044973131}]': finished 2023-09-23T23:28:22.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:22 smithi110 ceph-mon[221102]: osdmap e467: 8 total, 8 up, 7 in 2023-09-23T23:28:22.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:22 smithi110 ceph-mon[221102]: failed to encode map e467 with expected crc 2023-09-23T23:28:22.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:22 smithi110 ceph-mon[221102]: failed to encode map e467 with expected crc 2023-09-23T23:28:22.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:22 smithi110 ceph-mon[221102]: pgmap v1282: 129 pgs: 129 active+clean; 88 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:22.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:22 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T23:28:22.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:22 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T23:28:23.072 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:23.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:23.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:23.181 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:23.233 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:23 smithi138 ceph-mon[193606]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:23 smithi138 ceph-mon[193606]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:23 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T23:28:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:23 smithi138 ceph-mon[193606]: osdmap e468: 8 total, 8 up, 7 in 2023-09-23T23:28:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:23 smithi138 ceph-mon[193606]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:23 smithi138 ceph-mon[193606]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:23 smithi138 ceph-mon[193606]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:23 smithi138 ceph-mon[193606]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:23 smithi138 ceph-mon[193606]: 11.17 starting backfill to osd.7 from (355'562,405'8006] MIN to 465'9728 2023-09-23T23:28:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:23 smithi138 ceph-mon[193606]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:23 smithi138 ceph-mon[193606]: osdmap e469: 8 total, 8 up, 7 in 2023-09-23T23:28:23.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[221102]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[221102]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T23:28:23.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[221102]: osdmap e468: 8 total, 8 up, 7 in 2023-09-23T23:28:23.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[221102]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[221102]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[221102]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[221102]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[221102]: 11.17 starting backfill to osd.7 from (355'562,405'8006] MIN to 465'9728 2023-09-23T23:28:23.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[221102]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[221102]: osdmap e469: 8 total, 8 up, 7 in 2023-09-23T23:28:23.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[212939]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[212939]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T23:28:23.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[212939]: osdmap e468: 8 total, 8 up, 7 in 2023-09-23T23:28:23.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[212939]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[212939]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[212939]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[212939]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[212939]: 11.17 starting backfill to osd.7 from (355'562,405'8006] MIN to 465'9728 2023-09-23T23:28:23.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[212939]: failed to encode map e468 with expected crc 2023-09-23T23:28:23.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:23 smithi110 ceph-mon[212939]: osdmap e469: 8 total, 8 up, 7 in 2023-09-23T23:28:24.044 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:24.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:24.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:24.095 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:24.148 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:24.189 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:24.233 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:24.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:24.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:24.339 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:24.370 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:24.567 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:24.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:24.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:24.608 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:28:24.671 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:24.710 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:24.796 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:28:24.796 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:28:24.796 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:28:24.796 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:28:24.797 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:28:24.883 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:24 smithi110 ceph-mon[221102]: failed to encode map e469 with expected crc 2023-09-23T23:28:24.883 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:24 smithi110 ceph-mon[221102]: failed to encode map e469 with expected crc 2023-09-23T23:28:24.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:24 smithi110 ceph-mon[221102]: failed to encode map e469 with expected crc 2023-09-23T23:28:24.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:24 smithi110 ceph-mon[221102]: pgmap v1285: 129 pgs: 129 active+clean; 70 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 217 op/s; 784 KiB/s, 12 objects/s recovering 2023-09-23T23:28:24.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:24 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-09-23T23:28:24.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:24 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-09-23T23:28:24.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:24 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3192278961' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:24.884 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:24 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:24.884 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:24 smithi110 ceph-mon[212939]: failed to encode map e469 with expected crc 2023-09-23T23:28:24.884 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:24 smithi110 ceph-mon[212939]: failed to encode map e469 with expected crc 2023-09-23T23:28:24.885 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:24 smithi110 ceph-mon[212939]: failed to encode map e469 with expected crc 2023-09-23T23:28:24.885 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:24 smithi110 ceph-mon[212939]: pgmap v1285: 129 pgs: 129 active+clean; 70 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 217 op/s; 784 KiB/s, 12 objects/s recovering 2023-09-23T23:28:24.885 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:24 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-09-23T23:28:24.885 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:24 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-09-23T23:28:24.885 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:24 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3192278961' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:24.885 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:24 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:24 smithi138 ceph-mon[193606]: failed to encode map e469 with expected crc 2023-09-23T23:28:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:24 smithi138 ceph-mon[193606]: failed to encode map e469 with expected crc 2023-09-23T23:28:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:24 smithi138 ceph-mon[193606]: failed to encode map e469 with expected crc 2023-09-23T23:28:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:24 smithi138 ceph-mon[193606]: pgmap v1285: 129 pgs: 129 active+clean; 70 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 217 op/s; 784 KiB/s, 12 objects/s recovering 2023-09-23T23:28:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:24 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-09-23T23:28:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:24 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-09-23T23:28:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:24 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3192278961' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:24.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:24 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:25.531 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:25.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:25.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:25.635 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:25.663 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:25 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:28:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:25 smithi138 ceph-mon[193606]: failed to encode map e470 with expected crc 2023-09-23T23:28:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:25 smithi138 ceph-mon[193606]: failed to encode map e470 with expected crc 2023-09-23T23:28:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:25 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]': finished 2023-09-23T23:28:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:25 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:28:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:25 smithi138 ceph-mon[193606]: osdmap e470: 8 total, 8 up, 7 in 2023-09-23T23:28:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:25 smithi138 ceph-mon[193606]: failed to encode map e470 with expected crc 2023-09-23T23:28:25.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:25 smithi138 ceph-mon[193606]: 11.18 starting backfill to osd.2 from (0'0,0'0] MAX to 467'9793 2023-09-23T23:28:25.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:25 smithi138 ceph-mon[193606]: 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 467'9793 2023-09-23T23:28:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:28:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[212939]: failed to encode map e470 with expected crc 2023-09-23T23:28:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[212939]: failed to encode map e470 with expected crc 2023-09-23T23:28:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]': finished 2023-09-23T23:28:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:28:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[212939]: osdmap e470: 8 total, 8 up, 7 in 2023-09-23T23:28:25.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[212939]: failed to encode map e470 with expected crc 2023-09-23T23:28:25.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[212939]: 11.18 starting backfill to osd.2 from (0'0,0'0] MAX to 467'9793 2023-09-23T23:28:25.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[212939]: 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 467'9793 2023-09-23T23:28:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:28:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[221102]: failed to encode map e470 with expected crc 2023-09-23T23:28:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[221102]: failed to encode map e470 with expected crc 2023-09-23T23:28:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]': finished 2023-09-23T23:28:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:28:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[221102]: osdmap e470: 8 total, 8 up, 7 in 2023-09-23T23:28:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[221102]: failed to encode map e470 with expected crc 2023-09-23T23:28:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[221102]: 11.18 starting backfill to osd.2 from (0'0,0'0] MAX to 467'9793 2023-09-23T23:28:25.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:25 smithi110 ceph-mon[221102]: 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 467'9793 2023-09-23T23:28:26.389 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:26.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:26.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:26.413 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:28:26.494 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:26.523 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:26.574 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:26.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:26.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:26.679 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:28:26.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:26 smithi110 ceph-mon[221102]: osdmap e471: 8 total, 8 up, 7 in 2023-09-23T23:28:26.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:26 smithi110 ceph-mon[221102]: failed to encode map e471 with expected crc 2023-09-23T23:28:26.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:26 smithi110 ceph-mon[221102]: failed to encode map e471 with expected crc 2023-09-23T23:28:26.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:26 smithi110 ceph-mon[221102]: failed to encode map e471 with expected crc 2023-09-23T23:28:26.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:26 smithi110 ceph-mon[221102]: failed to encode map e471 with expected crc 2023-09-23T23:28:26.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:26 smithi110 ceph-mon[221102]: pgmap v1288: 129 pgs: 1 remapped+peering, 128 active+clean; 23 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 720 KiB/s, 11 objects/s recovering 2023-09-23T23:28:26.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:26 smithi110 ceph-mon[212939]: osdmap e471: 8 total, 8 up, 7 in 2023-09-23T23:28:26.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:26 smithi110 ceph-mon[212939]: failed to encode map e471 with expected crc 2023-09-23T23:28:26.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:26 smithi110 ceph-mon[212939]: failed to encode map e471 with expected crc 2023-09-23T23:28:26.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:26 smithi110 ceph-mon[212939]: failed to encode map e471 with expected crc 2023-09-23T23:28:26.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:26 smithi110 ceph-mon[212939]: failed to encode map e471 with expected crc 2023-09-23T23:28:26.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:26 smithi110 ceph-mon[212939]: pgmap v1288: 129 pgs: 1 remapped+peering, 128 active+clean; 23 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 720 KiB/s, 11 objects/s recovering 2023-09-23T23:28:26.706 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Removed 105877 objects 2023-09-23T23:28:26.706 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Clean up completed and total clean up time :184.519 2023-09-23T23:28:26.713 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:26.715 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_6 2023-09-23T23:28:26.715 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_6 unique_pool_6 --yes-i-really-really-mean-it 2023-09-23T23:28:26.759 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:28:26.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:26.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:28:26.863 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:26.895 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:26 smithi138 ceph-mon[193606]: osdmap e471: 8 total, 8 up, 7 in 2023-09-23T23:28:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:26 smithi138 ceph-mon[193606]: failed to encode map e471 with expected crc 2023-09-23T23:28:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:26 smithi138 ceph-mon[193606]: failed to encode map e471 with expected crc 2023-09-23T23:28:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:26 smithi138 ceph-mon[193606]: failed to encode map e471 with expected crc 2023-09-23T23:28:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:26 smithi138 ceph-mon[193606]: failed to encode map e471 with expected crc 2023-09-23T23:28:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:26 smithi138 ceph-mon[193606]: pgmap v1288: 129 pgs: 1 remapped+peering, 128 active+clean; 23 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 720 KiB/s, 11 objects/s recovering 2023-09-23T23:28:26.950 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:26.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:26.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:26.979 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:27.003 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:28:27.029 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:28:27.052 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:28:27.054 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:27.072 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:28:27.086 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:27.092 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:28:27.113 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:28:27.132 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:28:27.135 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:27.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:27.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:27.155 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:28:27.156 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:28:27.156 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:28:27.156 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:28:27.156 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:28:27.157 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:28:27.157 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:28:27.157 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:28:27.157 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:28:27.182 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:28:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:28:26] "GET /metrics HTTP/1.1" 200 36533 "" "Prometheus/2.43.0" 2023-09-23T23:28:27.243 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:27.271 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:27.313 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:27.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:27.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:27.420 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:27.448 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:27.523 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:28:27.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:27 smithi110 ceph-mon[212939]: failed to encode map e472 with expected crc 2023-09-23T23:28:27.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:27 smithi110 ceph-mon[212939]: failed to encode map e472 with expected crc 2023-09-23T23:28:27.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:27 smithi110 ceph-mon[212939]: osdmap e472: 8 total, 8 up, 7 in 2023-09-23T23:28:27.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:27 smithi110 ceph-mon[212939]: failed to encode map e472 with expected crc 2023-09-23T23:28:27.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:27 smithi110 ceph-mon[212939]: 11.19 starting backfill to osd.2 from (0'0,0'0] MAX to 468'11558 2023-09-23T23:28:27.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:27 smithi110 ceph-mon[212939]: 11.19 starting backfill to osd.7 from (0'0,0'0] MAX to 468'11558 2023-09-23T23:28:27.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:27 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3246772833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:27.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:27 smithi110 ceph-mon[221102]: failed to encode map e472 with expected crc 2023-09-23T23:28:27.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:27 smithi110 ceph-mon[221102]: failed to encode map e472 with expected crc 2023-09-23T23:28:27.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:27 smithi110 ceph-mon[221102]: osdmap e472: 8 total, 8 up, 7 in 2023-09-23T23:28:27.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:27 smithi110 ceph-mon[221102]: failed to encode map e472 with expected crc 2023-09-23T23:28:27.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:27 smithi110 ceph-mon[221102]: 11.19 starting backfill to osd.2 from (0'0,0'0] MAX to 468'11558 2023-09-23T23:28:27.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:27 smithi110 ceph-mon[221102]: 11.19 starting backfill to osd.7 from (0'0,0'0] MAX to 468'11558 2023-09-23T23:28:27.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:27 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3246772833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:27.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:27 smithi138 ceph-mon[193606]: failed to encode map e472 with expected crc 2023-09-23T23:28:27.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:27 smithi138 ceph-mon[193606]: failed to encode map e472 with expected crc 2023-09-23T23:28:27.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:27 smithi138 ceph-mon[193606]: osdmap e472: 8 total, 8 up, 7 in 2023-09-23T23:28:27.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:27 smithi138 ceph-mon[193606]: failed to encode map e472 with expected crc 2023-09-23T23:28:27.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:27 smithi138 ceph-mon[193606]: 11.19 starting backfill to osd.2 from (0'0,0'0] MAX to 468'11558 2023-09-23T23:28:27.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:27 smithi138 ceph-mon[193606]: 11.19 starting backfill to osd.7 from (0'0,0'0] MAX to 468'11558 2023-09-23T23:28:27.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:27 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3246772833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:28.044 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:28.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:28.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:28.098 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 7, 2, 1] out_osds: [6] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:28:28.098 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:28:28.147 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:28.173 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:28.554 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:28.641 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:28.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:28.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:28.692 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:28.746 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:28.773 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:28.890 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:28 smithi110 ceph-mon[212939]: failed to encode map e473 with expected crc 2023-09-23T23:28:28.890 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:28 smithi110 ceph-mon[212939]: failed to encode map e473 with expected crc 2023-09-23T23:28:28.890 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:28 smithi110 ceph-mon[212939]: osdmap e473: 8 total, 8 up, 7 in 2023-09-23T23:28:28.890 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:28 smithi110 ceph-mon[212939]: pgmap v1291: 129 pgs: 1 peering, 1 remapped+peering, 127 active+clean; 13 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 257 op/s; 720 KiB/s, 10 objects/s recovering 2023-09-23T23:28:28.890 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:28.890 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:28:28.890 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:28 smithi110 ceph-mon[221102]: failed to encode map e473 with expected crc 2023-09-23T23:28:28.890 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:28 smithi110 ceph-mon[221102]: failed to encode map e473 with expected crc 2023-09-23T23:28:28.891 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:28 smithi110 ceph-mon[221102]: osdmap e473: 8 total, 8 up, 7 in 2023-09-23T23:28:28.891 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:28 smithi110 ceph-mon[221102]: pgmap v1291: 129 pgs: 1 peering, 1 remapped+peering, 127 active+clean; 13 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 257 op/s; 720 KiB/s, 10 objects/s recovering 2023-09-23T23:28:28.891 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:28.891 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:28:28.928 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:28.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:28.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:28 smithi138 ceph-mon[193606]: failed to encode map e473 with expected crc 2023-09-23T23:28:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:28 smithi138 ceph-mon[193606]: failed to encode map e473 with expected crc 2023-09-23T23:28:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:28 smithi138 ceph-mon[193606]: osdmap e473: 8 total, 8 up, 7 in 2023-09-23T23:28:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:28 smithi138 ceph-mon[193606]: pgmap v1291: 129 pgs: 1 peering, 1 remapped+peering, 127 active+clean; 13 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 257 op/s; 720 KiB/s, 10 objects/s recovering 2023-09-23T23:28:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:28:29.032 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:29.063 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:29.116 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:29.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:29.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:29.223 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:29.254 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:29.294 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:29.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:29.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:29.399 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:29.429 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:29.824 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:29.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:29.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:29.927 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:29 smithi138 ceph-mon[193606]: failed to encode map e473 with expected crc 2023-09-23T23:28:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:29 smithi138 ceph-mon[193606]: failed to encode map e473 with expected crc 2023-09-23T23:28:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:29 smithi138 ceph-mon[193606]: failed to encode map e474 with expected crc 2023-09-23T23:28:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:29 smithi138 ceph-mon[193606]: failed to encode map e474 with expected crc 2023-09-23T23:28:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:29 smithi138 ceph-mon[193606]: osdmap e474: 8 total, 8 up, 7 in 2023-09-23T23:28:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:29 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:29 smithi138 ceph-mon[193606]: failed to encode map e474 with expected crc 2023-09-23T23:28:29.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:29 smithi138 ceph-mon[193606]: failed to encode map e474 with expected crc 2023-09-23T23:28:29.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:29 smithi110 ceph-mon[212939]: failed to encode map e473 with expected crc 2023-09-23T23:28:29.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:29 smithi110 ceph-mon[212939]: failed to encode map e473 with expected crc 2023-09-23T23:28:29.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:29 smithi110 ceph-mon[212939]: failed to encode map e474 with expected crc 2023-09-23T23:28:29.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:29 smithi110 ceph-mon[212939]: failed to encode map e474 with expected crc 2023-09-23T23:28:29.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:29 smithi110 ceph-mon[212939]: osdmap e474: 8 total, 8 up, 7 in 2023-09-23T23:28:29.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:29 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:29.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:29 smithi110 ceph-mon[212939]: failed to encode map e474 with expected crc 2023-09-23T23:28:29.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:29 smithi110 ceph-mon[212939]: failed to encode map e474 with expected crc 2023-09-23T23:28:29.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:29 smithi110 ceph-mon[221102]: failed to encode map e473 with expected crc 2023-09-23T23:28:29.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:29 smithi110 ceph-mon[221102]: failed to encode map e473 with expected crc 2023-09-23T23:28:29.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:29 smithi110 ceph-mon[221102]: failed to encode map e474 with expected crc 2023-09-23T23:28:29.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:29 smithi110 ceph-mon[221102]: failed to encode map e474 with expected crc 2023-09-23T23:28:29.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:29 smithi110 ceph-mon[221102]: osdmap e474: 8 total, 8 up, 7 in 2023-09-23T23:28:29.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:29 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:29.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:29 smithi110 ceph-mon[221102]: failed to encode map e474 with expected crc 2023-09-23T23:28:29.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:29 smithi110 ceph-mon[221102]: failed to encode map e474 with expected crc 2023-09-23T23:28:29.955 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:30.022 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:28:30.022 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:28:30.150 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:30.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:30.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:30.254 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:30.283 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:30.330 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:30.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:30.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:30.437 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:30.464 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:30.582 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:28:30.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:30 smithi138 ceph-mon[193606]: pgmap v1293: 129 pgs: 1 peering, 1 remapped+peering, 127 active+clean; 7.8 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 72 op/s; 77 KiB/s, 0 objects/s recovering 2023-09-23T23:28:30.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:30 smithi110 ceph-mon[212939]: pgmap v1293: 129 pgs: 1 peering, 1 remapped+peering, 127 active+clean; 7.8 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 72 op/s; 77 KiB/s, 0 objects/s recovering 2023-09-23T23:28:30.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:30 smithi110 ceph-mon[221102]: pgmap v1293: 129 pgs: 1 peering, 1 remapped+peering, 127 active+clean; 7.8 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 72 op/s; 77 KiB/s, 0 objects/s recovering 2023-09-23T23:28:31.133 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:28:31.205 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:31.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:31.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:31.233 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:31.314 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:31.340 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:31.386 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:31.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:31.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:31.491 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:31.493 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:28:31.519 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:31.594 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:31.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:31.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:31.698 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:31.904 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:31.950 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:31.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:31.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:32.055 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:32.086 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:32.129 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:32.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:32.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:32.233 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:32.264 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:32.306 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:32.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:32.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:32.409 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:32.439 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:32.485 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:32.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:32.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:32.589 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:32.617 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:32.752 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:32.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:32.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:32.818 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_6' removed 2023-09-23T23:28:32.855 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:28:32.857 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:32.886 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:32.890 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:32 smithi110 ceph-mon[221102]: pgmap v1294: 129 pgs: 129 active+clean; 7.8 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 47 op/s; 53 KiB/s, 0 objects/s recovering 2023-09-23T23:28:32.890 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:32 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1395799854' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:28:32.890 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:32 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:28:32.890 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:32 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/477331608' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:28:32.892 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:32 smithi110 ceph-mon[212939]: pgmap v1294: 129 pgs: 129 active+clean; 7.8 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 47 op/s; 53 KiB/s, 0 objects/s recovering 2023-09-23T23:28:32.892 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:32 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1395799854' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:28:32.892 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:32 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:28:32.892 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:32 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/477331608' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:28:32.932 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:32.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:32.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:33.035 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:28:33.062 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:33.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:32 smithi138 ceph-mon[193606]: pgmap v1294: 129 pgs: 129 active+clean; 7.8 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 47 op/s; 53 KiB/s, 0 objects/s recovering 2023-09-23T23:28:33.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:32 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1395799854' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:28:33.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:32 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:28:33.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:32 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/477331608' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:28:33.409 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:28:33.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:33.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:28:33.512 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:33.539 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:33.824 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:28:33.824 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:28:33.848 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:33 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:28:33.848 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:33 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:28:33.848 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:33 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/477331608' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:28:33.848 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:33 smithi138 ceph-mon[193606]: osdmap e475: 8 total, 8 up, 7 in 2023-09-23T23:28:33.996 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:34.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:34.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:34.016 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T23:28:34.016 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T23:28:34.016 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_7 2023-09-23T23:28:34.017 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_7 16 2023-09-23T23:28:34.083 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:33 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:28:34.083 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:33 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:28:34.083 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:33 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/477331608' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:28:34.083 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:33 smithi110 ceph-mon[212939]: osdmap e475: 8 total, 8 up, 7 in 2023-09-23T23:28:34.084 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:33 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:28:34.084 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:33 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:28:34.084 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:33 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/477331608' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:28:34.084 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:33 smithi110 ceph-mon[221102]: osdmap e475: 8 total, 8 up, 7 in 2023-09-23T23:28:34.116 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:34.143 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:34.192 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:28:34.199 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:34.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:34.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:34.214 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:28:34.233 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:28:34.253 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:28:34.272 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:28:34.299 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:28:34.303 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:34.321 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:28:34.334 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:34.344 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:28:34.346 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:28:34.346 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:28:34.346 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:28:34.346 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:28:34.346 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:28:34.346 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:28:34.346 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:28:34.346 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:28:34.382 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:34.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:34.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:34.486 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:34.518 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:34.525 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T23:28:34.563 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:34.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:34.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:34.668 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:34.695 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:34.916 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:28:34.936 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:34.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:34 smithi110 ceph-mon[221102]: failed to encode map e475 with expected crc 2023-09-23T23:28:34.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:34 smithi110 ceph-mon[221102]: failed to encode map e475 with expected crc 2023-09-23T23:28:34.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:34 smithi110 ceph-mon[221102]: pgmap v1296: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:34.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:34 smithi110 ceph-mon[221102]: failed to encode map e475 with expected crc 2023-09-23T23:28:34.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:34 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1840863143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:34.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:34 smithi110 ceph-mon[212939]: failed to encode map e475 with expected crc 2023-09-23T23:28:34.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:34 smithi110 ceph-mon[212939]: failed to encode map e475 with expected crc 2023-09-23T23:28:34.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:34 smithi110 ceph-mon[212939]: pgmap v1296: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:34.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:34 smithi110 ceph-mon[212939]: failed to encode map e475 with expected crc 2023-09-23T23:28:34.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:34 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1840863143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:34.953 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:34.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:34.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:35.058 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:35.085 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:34 smithi138 ceph-mon[193606]: failed to encode map e475 with expected crc 2023-09-23T23:28:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:34 smithi138 ceph-mon[193606]: failed to encode map e475 with expected crc 2023-09-23T23:28:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:34 smithi138 ceph-mon[193606]: pgmap v1296: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:34 smithi138 ceph-mon[193606]: failed to encode map e475 with expected crc 2023-09-23T23:28:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:34 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1840863143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:35.181 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:35.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:35.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:35.284 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:35.312 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:35.510 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:35.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:35.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:35.613 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:35.642 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:35.684 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:35.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:35.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:35.787 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:35.814 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:35.908 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:35.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:35.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:35.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:35 smithi110 ceph-mon[212939]: failed to encode map e475 with expected crc 2023-09-23T23:28:35.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:35 smithi110 ceph-mon[221102]: failed to encode map e475 with expected crc 2023-09-23T23:28:36.013 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:36.040 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:36.124 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:35 smithi138 ceph-mon[193606]: failed to encode map e475 with expected crc 2023-09-23T23:28:36.183 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:28:36.208 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:36.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:36.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:36.313 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:36.341 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:28:36.341 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:28:36.342 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:28:36.342 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:28:36.342 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:28:36.343 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:36.441 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:36.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:36.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:36.545 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:36.550 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:36.574 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:36.618 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:36.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:36.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:36.721 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:36.748 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:36.959 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:28:37.012 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T23:28:37.018 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:37.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:37.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:37.033 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_7' created 2023-09-23T23:28:37.058 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:28:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:28:36] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.43.0" 2023-09-23T23:28:37.058 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:36 smithi110 ceph-mon[212939]: pgmap v1297: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:37.058 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:36 smithi110 ceph-mon[221102]: pgmap v1297: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:37.124 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:37.149 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:37.193 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:37.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:37.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:37.301 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:28:37.329 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:37.372 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:28:37.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:36 smithi138 ceph-mon[193606]: pgmap v1297: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:37.978 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:28:37.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:37.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:28:38.050 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_7 rados --yes-i-really-mean-it || true 2023-09-23T23:28:38.082 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:38.112 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:38.125 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:38.157 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:38.158 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:37 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3527704430' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T23:28:38.158 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:37 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T23:28:38.158 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:37 smithi138 ceph-mon[193606]: failed to encode map e476 with expected crc 2023-09-23T23:28:38.158 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:37 smithi138 ceph-mon[193606]: failed to encode map e476 with expected crc 2023-09-23T23:28:38.158 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:37 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-09-23T23:28:38.159 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:37 smithi138 ceph-mon[193606]: osdmap e476: 8 total, 8 up, 7 in 2023-09-23T23:28:38.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:38.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:38.260 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:38.287 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:38.406 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:38.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:38.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:38.414 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:37 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3527704430' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T23:28:38.415 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:37 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T23:28:38.415 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:37 smithi110 ceph-mon[212939]: failed to encode map e476 with expected crc 2023-09-23T23:28:38.415 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:37 smithi110 ceph-mon[212939]: failed to encode map e476 with expected crc 2023-09-23T23:28:38.415 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:37 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-09-23T23:28:38.415 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:37 smithi110 ceph-mon[212939]: osdmap e476: 8 total, 8 up, 7 in 2023-09-23T23:28:38.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:37 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3527704430' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T23:28:38.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:37 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T23:28:38.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:37 smithi110 ceph-mon[221102]: failed to encode map e476 with expected crc 2023-09-23T23:28:38.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:37 smithi110 ceph-mon[221102]: failed to encode map e476 with expected crc 2023-09-23T23:28:38.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:37 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-09-23T23:28:38.416 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:37 smithi110 ceph-mon[221102]: osdmap e476: 8 total, 8 up, 7 in 2023-09-23T23:28:38.511 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:28:38.542 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:38.746 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:28:38.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:38.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:28:38.838 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:28:38.838 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:28:38.839 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:28:38.839 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:28:38.839 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.839 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:178614 11.3 11:c2d62249:::benchmark_data_smithi110_433746_object72734:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:28:38.839 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:19.570755+0000", 2023-09-23T23:28:38.839 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 79.263932065000006, 2023-09-23T23:28:38.839 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1817553470000002, 2023-09-23T23:28:38.839 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.839 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:38.839 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.839 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.839 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.839 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178614 2023-09-23T23:28:38.840 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.840 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.840 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.840 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570755+0000", 2023-09-23T23:28:38.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.840 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.840 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.840 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570755+0000", 2023-09-23T23:28:38.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.840 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.841 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.841 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570754+0000", 2023-09-23T23:28:38.841 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:28:38.841 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.841 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.841 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.841 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570759+0000", 2023-09-23T23:28:38.841 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7110000000000003e-06 2023-09-23T23:28:38.841 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.841 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.841 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.841 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570771+0000", 2023-09-23T23:28:38.841 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1965e-05 2023-09-23T23:28:38.842 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.842 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.842 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.842 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570776+0000", 2023-09-23T23:28:38.842 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0719999999999999e-06 2023-09-23T23:28:38.842 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.842 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.842 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.842 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570820+0000", 2023-09-23T23:28:38.842 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3640999999999998e-05 2023-09-23T23:28:38.842 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.842 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570826+0000", 2023-09-23T23:28:38.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.088e-06 2023-09-23T23:28:38.843 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:20.540024+0000", 2023-09-23T23:28:38.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96919833200000005 2023-09-23T23:28:38.843 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:20.540036+0000", 2023-09-23T23:28:38.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1731e-05 2023-09-23T23:28:38.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.844 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.844 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.844 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.740668+0000", 2023-09-23T23:28:38.844 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.20063242 2023-09-23T23:28:38.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.844 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.844 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:38.844 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.740808+0000", 2023-09-23T23:28:38.844 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000139813 2023-09-23T23:28:38.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.844 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.844 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-09-23T23:28:38.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.741123+0000", 2023-09-23T23:28:38.845 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000314968 2023-09-23T23:28:38.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:38.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.744153+0000", 2023-09-23T23:28:38.845 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003029538 2023-09-23T23:28:38.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.744306+0000", 2023-09-23T23:28:38.845 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000153614 2023-09-23T23:28:38.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.750752+0000", 2023-09-23T23:28:38.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006445175 2023-09-23T23:28:38.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752382+0000", 2023-09-23T23:28:38.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0016305670000000001 2023-09-23T23:28:38.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:38.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752422+0000", 2023-09-23T23:28:38.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0157999999999997e-05 2023-09-23T23:28:38.847 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.847 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.847 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752511+0000", 2023-09-23T23:28:38.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8374000000000005e-05 2023-09-23T23:28:38.847 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.847 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.847 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.847 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.847 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:178615 11.3 11:c151980f:::benchmark_data_smithi110_433746_object72735:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:28:38.847 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:19.573793+0000", 2023-09-23T23:28:38.848 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 79.260894723000007, 2023-09-23T23:28:38.848 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1788738529999998, 2023-09-23T23:28:38.848 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.848 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:38.848 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.848 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.848 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.848 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178615 2023-09-23T23:28:38.848 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.848 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.848 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.848 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.848 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573793+0000", 2023-09-23T23:28:38.848 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.849 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.849 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.849 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573793+0000", 2023-09-23T23:28:38.849 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.849 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.849 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.849 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573792+0000", 2023-09-23T23:28:38.849 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:28:38.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.849 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.849 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.849 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573797+0000", 2023-09-23T23:28:38.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7960000000000003e-06 2023-09-23T23:28:38.850 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.850 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.850 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.850 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573809+0000", 2023-09-23T23:28:38.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2575000000000001e-05 2023-09-23T23:28:38.850 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.850 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.850 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.850 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573816+0000", 2023-09-23T23:28:38.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4670000000000001e-06 2023-09-23T23:28:38.850 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.851 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573907+0000", 2023-09-23T23:28:38.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1496000000000006e-05 2023-09-23T23:28:38.851 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.851 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573930+0000", 2023-09-23T23:28:38.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2660999999999999e-05 2023-09-23T23:28:38.851 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.851 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:20.540059+0000", 2023-09-23T23:28:38.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96612945299999997 2023-09-23T23:28:38.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.852 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:20.540067+0000", 2023-09-23T23:28:38.852 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0919999999999998e-06 2023-09-23T23:28:38.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.852 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.742434+0000", 2023-09-23T23:28:38.852 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.202367089 2023-09-23T23:28:38.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:38.853 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.742547+0000", 2023-09-23T23:28:38.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112941 2023-09-23T23:28:38.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.853 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-09-23T23:28:38.853 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.742774+0000", 2023-09-23T23:28:38.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00022651400000000001 2023-09-23T23:28:38.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.853 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:38.853 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.744161+0000", 2023-09-23T23:28:38.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001387325 2023-09-23T23:28:38.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.854 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.744473+0000", 2023-09-23T23:28:38.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00031187499999999998 2023-09-23T23:28:38.854 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.854 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.750782+0000", 2023-09-23T23:28:38.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0063084719999999999 2023-09-23T23:28:38.854 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.854 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.855 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752547+0000", 2023-09-23T23:28:38.855 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0017649460000000001 2023-09-23T23:28:38.855 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.855 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.855 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:38.855 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752585+0000", 2023-09-23T23:28:38.855 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8121999999999999e-05 2023-09-23T23:28:38.855 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.855 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.855 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.855 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752667+0000", 2023-09-23T23:28:38.855 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1868999999999998e-05 2023-09-23T23:28:38.855 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.856 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.856 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.856 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.856 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.856 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:190414 11.5 11:a226ca2d:::benchmark_data_smithi110_433746_object84534:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e436)", 2023-09-23T23:28:38.856 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:41.139906+0000", 2023-09-23T23:28:38.856 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 57.694781401, 2023-09-23T23:28:38.856 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.992750672, 2023-09-23T23:28:38.856 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.856 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:28:38.856 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.856 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.856 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.857 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 190414 2023-09-23T23:28:38.857 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.857 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139906+0000", 2023-09-23T23:28:38.857 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.857 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139906+0000", 2023-09-23T23:28:38.857 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.857 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.858 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.858 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139905+0000", 2023-09-23T23:28:38.858 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:28:38.858 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.858 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.858 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.858 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139909+0000", 2023-09-23T23:28:38.858 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2320000000000003e-06 2023-09-23T23:28:38.858 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.858 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.858 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.858 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139918+0000", 2023-09-23T23:28:38.859 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8289999999999997e-06 2023-09-23T23:28:38.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.859 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139925+0000", 2023-09-23T23:28:38.859 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5200000000000003e-06 2023-09-23T23:28:38.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.859 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.141308+0000", 2023-09-23T23:28:38.859 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001383263 2023-09-23T23:28:38.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op must wait for map", 2023-09-23T23:28:38.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.141315+0000", 2023-09-23T23:28:38.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.182e-06 2023-09-23T23:28:38.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.143026+0000", 2023-09-23T23:28:38.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001711064 2023-09-23T23:28:38.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.143034+0000", 2023-09-23T23:28:38.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7079999999999997e-06 2023-09-23T23:28:38.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.861 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.861 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.861 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.132576+0000", 2023-09-23T23:28:38.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98954197099999996 2023-09-23T23:28:38.861 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.861 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.861 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.861 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.132657+0000", 2023-09-23T23:28:38.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0940999999999996e-05 2023-09-23T23:28:38.861 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.861 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.861 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.861 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.862 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.862 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:190429 11.5 11:a76d3bc6:::benchmark_data_smithi110_433746_object84549:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e436)", 2023-09-23T23:28:38.862 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:41.139932+0000", 2023-09-23T23:28:38.862 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 57.694755020000002, 2023-09-23T23:28:38.862 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0172525750000001, 2023-09-23T23:28:38.862 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.862 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:38.862 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.862 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.862 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.862 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 190429 2023-09-23T23:28:38.862 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.863 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.863 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.863 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.863 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139932+0000", 2023-09-23T23:28:38.863 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.863 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.863 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.863 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.863 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139932+0000", 2023-09-23T23:28:38.863 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.863 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.863 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.863 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.863 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139932+0000", 2023-09-23T23:28:38.864 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:38.864 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.864 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.864 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.864 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139934+0000", 2023-09-23T23:28:38.864 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.187e-06 2023-09-23T23:28:38.864 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.864 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.864 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.864 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139939+0000", 2023-09-23T23:28:38.864 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1340000000000003e-06 2023-09-23T23:28:38.864 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.864 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.864 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.865 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139943+0000", 2023-09-23T23:28:38.865 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4709999999999999e-06 2023-09-23T23:28:38.865 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.865 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.865 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.141339+0000", 2023-09-23T23:28:38.865 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001396481 2023-09-23T23:28:38.865 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.865 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:28:38.865 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.141345+0000", 2023-09-23T23:28:38.865 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8799999999999996e-06 2023-09-23T23:28:38.865 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.866 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.866 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.866 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.143057+0000", 2023-09-23T23:28:38.866 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0017121560000000001 2023-09-23T23:28:38.866 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.866 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.866 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.866 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.143062+0000", 2023-09-23T23:28:38.866 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.04e-06 2023-09-23T23:28:38.866 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.866 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.866 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.866 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.132701+0000", 2023-09-23T23:28:38.867 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98963869500000001 2023-09-23T23:28:38.867 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.867 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:38.867 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.132847+0000", 2023-09-23T23:28:38.867 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014609000000000001 2023-09-23T23:28:38.867 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.867 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:28:38.867 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.133176+0000", 2023-09-23T23:28:38.867 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00032873100000000002 2023-09-23T23:28:38.867 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.868 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:38.868 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.144793+0000", 2023-09-23T23:28:38.868 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011617647 2023-09-23T23:28:38.868 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.868 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.868 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.868 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157040+0000", 2023-09-23T23:28:38.868 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012246194 2023-09-23T23:28:38.868 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.868 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.868 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.868 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157117+0000", 2023-09-23T23:28:38.868 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7812999999999995e-05 2023-09-23T23:28:38.868 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.869 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.869 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:38.869 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157161+0000", 2023-09-23T23:28:38.869 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3009000000000001e-05 2023-09-23T23:28:38.869 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.869 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.869 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.869 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157185+0000", 2023-09-23T23:28:38.869 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4426e-05 2023-09-23T23:28:38.869 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.869 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.869 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.869 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.869 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.870 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:190458 11.5 11:a19c863d:::benchmark_data_smithi110_433746_object84578:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e436)", 2023-09-23T23:28:38.870 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:41.162298+0000", 2023-09-23T23:28:38.870 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 57.672389756000001, 2023-09-23T23:28:38.870 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99508530799999995, 2023-09-23T23:28:38.870 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.870 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:38.870 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.870 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.870 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.870 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 190458 2023-09-23T23:28:38.870 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.871 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.871 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.871 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.162298+0000", 2023-09-23T23:28:38.871 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.871 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.871 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.871 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.162298+0000", 2023-09-23T23:28:38.871 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.871 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.871 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.871 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.162296+0000", 2023-09-23T23:28:38.872 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:38.872 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.872 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.872 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.872 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.162302+0000", 2023-09-23T23:28:38.872 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9839999999999998e-06 2023-09-23T23:28:38.872 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.872 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.872 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.872 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.162313+0000", 2023-09-23T23:28:38.872 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1090000000000001e-05 2023-09-23T23:28:38.872 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.872 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.872 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.873 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.162320+0000", 2023-09-23T23:28:38.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2720000000000003e-06 2023-09-23T23:28:38.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.873 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.873 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.162357+0000", 2023-09-23T23:28:38.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7657999999999998e-05 2023-09-23T23:28:38.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.873 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.873 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.162363+0000", 2023-09-23T23:28:38.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3340000000000001e-06 2023-09-23T23:28:38.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.139003+0000", 2023-09-23T23:28:38.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97664008999999996 2023-09-23T23:28:38.874 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:38.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.139170+0000", 2023-09-23T23:28:38.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016747600000000001 2023-09-23T23:28:38.874 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:28:38.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.139427+0000", 2023-09-23T23:28:38.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025685299999999997 2023-09-23T23:28:38.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.875 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.875 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:38.875 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.144803+0000", 2023-09-23T23:28:38.875 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005376151 2023-09-23T23:28:38.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.875 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.875 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.875 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157064+0000", 2023-09-23T23:28:38.875 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012260801999999999 2023-09-23T23:28:38.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.875 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.875 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.876 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157330+0000", 2023-09-23T23:28:38.876 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00026610499999999999 2023-09-23T23:28:38.876 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.876 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.876 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:38.876 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157360+0000", 2023-09-23T23:28:38.876 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9739000000000001e-05 2023-09-23T23:28:38.876 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.876 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.876 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.876 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157383+0000", 2023-09-23T23:28:38.876 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3096999999999999e-05 2023-09-23T23:28:38.876 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.877 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.877 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.877 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.877 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.877 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:190470 11.5 11:a6b68272:::benchmark_data_smithi110_433746_object84590:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e436)", 2023-09-23T23:28:38.877 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:41.178750+0000", 2023-09-23T23:28:38.877 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 57.655937645999998, 2023-09-23T23:28:38.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97894028, 2023-09-23T23:28:38.877 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.877 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:38.877 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.877 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.877 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.877 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 190470 2023-09-23T23:28:38.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.878 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.878 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.878 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.178750+0000", 2023-09-23T23:28:38.878 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.878 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.878 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.178750+0000", 2023-09-23T23:28:38.878 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.879 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.879 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.879 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.178749+0000", 2023-09-23T23:28:38.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:28:38.879 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.879 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.879 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.879 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.178753+0000", 2023-09-23T23:28:38.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7550000000000002e-06 2023-09-23T23:28:38.879 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.879 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.879 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.879 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.178764+0000", 2023-09-23T23:28:38.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.099e-05 2023-09-23T23:28:38.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.880 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.880 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.178769+0000", 2023-09-23T23:28:38.880 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8849999999999998e-06 2023-09-23T23:28:38.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.880 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.880 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.178817+0000", 2023-09-23T23:28:38.880 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7546999999999997e-05 2023-09-23T23:28:38.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.880 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.880 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.178825+0000", 2023-09-23T23:28:38.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4190000000000002e-06 2023-09-23T23:28:38.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.140561+0000", 2023-09-23T23:28:38.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96173630200000004 2023-09-23T23:28:38.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:38.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.140642+0000", 2023-09-23T23:28:38.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0253999999999994e-05 2023-09-23T23:28:38.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:28:38.882 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.140813+0000", 2023-09-23T23:28:38.882 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017172199999999999 2023-09-23T23:28:38.882 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.882 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.882 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:38.882 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.147281+0000", 2023-09-23T23:28:38.882 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006467268 2023-09-23T23:28:38.882 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.882 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.882 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.882 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157221+0000", 2023-09-23T23:28:38.882 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0099399499999999995 2023-09-23T23:28:38.883 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.883 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.883 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157616+0000", 2023-09-23T23:28:38.883 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00039537300000000002 2023-09-23T23:28:38.883 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.883 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.883 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:38.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157654+0000", 2023-09-23T23:28:38.883 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7916000000000001e-05 2023-09-23T23:28:38.883 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.883 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.883 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157690+0000", 2023-09-23T23:28:38.884 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6043e-05 2023-09-23T23:28:38.884 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.884 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.884 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.884 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.884 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.884 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:190473 11.5 11:a024e1d7:::benchmark_data_smithi110_433746_object84593:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e436)", 2023-09-23T23:28:38.884 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:41.178803+0000", 2023-09-23T23:28:38.884 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 57.655883961999997, 2023-09-23T23:28:38.884 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97901457300000005, 2023-09-23T23:28:38.884 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.884 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:38.884 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.885 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.885 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.885 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 190473 2023-09-23T23:28:38.885 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.885 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.885 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.885 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.885 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.178803+0000", 2023-09-23T23:28:38.885 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.885 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.885 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.885 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.885 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.178803+0000", 2023-09-23T23:28:38.885 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.886 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.886 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.886 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.886 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.178803+0000", 2023-09-23T23:28:38.886 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:38.886 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.886 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.886 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.886 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.178805+0000", 2023-09-23T23:28:38.886 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4320000000000002e-06 2023-09-23T23:28:38.886 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.886 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.887 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.887 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.178811+0000", 2023-09-23T23:28:38.887 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8270000000000003e-06 2023-09-23T23:28:38.887 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.887 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.887 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.887 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.178816+0000", 2023-09-23T23:28:38.887 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3109999999999999e-06 2023-09-23T23:28:38.887 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.887 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.887 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.887 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.178839+0000", 2023-09-23T23:28:38.887 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3167e-05 2023-09-23T23:28:38.888 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.888 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.888 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.888 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.178842+0000", 2023-09-23T23:28:38.888 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.971e-06 2023-09-23T23:28:38.888 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.888 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.888 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.888 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.141816+0000", 2023-09-23T23:28:38.888 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96297473 2023-09-23T23:28:38.888 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.888 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.888 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:38.889 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.141934+0000", 2023-09-23T23:28:38.889 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117942 2023-09-23T23:28:38.889 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.889 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.889 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:28:38.889 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.142151+0000", 2023-09-23T23:28:38.889 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021667600000000001 2023-09-23T23:28:38.889 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.889 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.889 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:38.889 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.147288+0000", 2023-09-23T23:28:38.889 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051365810000000003 2023-09-23T23:28:38.889 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.890 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.890 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.890 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157271+0000", 2023-09-23T23:28:38.890 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0099831009999999994 2023-09-23T23:28:38.890 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.890 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.890 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.890 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157740+0000", 2023-09-23T23:28:38.890 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00046958200000000002 2023-09-23T23:28:38.890 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.890 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.890 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:38.891 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157781+0000", 2023-09-23T23:28:38.891 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0166000000000001e-05 2023-09-23T23:28:38.891 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.891 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.891 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.891 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157818+0000", 2023-09-23T23:28:38.891 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7457999999999999e-05 2023-09-23T23:28:38.891 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.891 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.891 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.891 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.891 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.891 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:193804 11.4 11:22521c06:::benchmark_data_smithi110_433746_object87924:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e439)", 2023-09-23T23:28:38.892 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:46.203923+0000", 2023-09-23T23:28:38.892 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 52.630764337999999, 2023-09-23T23:28:38.892 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99955791500000002, 2023-09-23T23:28:38.892 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.892 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:38.892 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.892 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.892 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.892 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 193804 2023-09-23T23:28:38.892 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.892 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.892 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.893 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.893 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.203923+0000", 2023-09-23T23:28:38.893 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.893 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.893 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.893 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.893 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.203923+0000", 2023-09-23T23:28:38.893 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.893 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.893 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.893 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.893 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.203922+0000", 2023-09-23T23:28:38.893 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:38.894 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.894 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.894 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.894 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.203927+0000", 2023-09-23T23:28:38.894 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2159999999999997e-06 2023-09-23T23:28:38.894 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.894 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.894 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.894 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.203939+0000", 2023-09-23T23:28:38.894 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2476999999999999e-05 2023-09-23T23:28:38.894 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.894 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.895 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.895 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.203959+0000", 2023-09-23T23:28:38.895 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9153000000000001e-05 2023-09-23T23:28:38.895 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.895 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.895 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.895 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.204022+0000", 2023-09-23T23:28:38.895 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3688000000000004e-05 2023-09-23T23:28:38.895 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.895 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.895 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.895 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.204029+0000", 2023-09-23T23:28:38.895 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2880000000000004e-06 2023-09-23T23:28:38.896 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.896 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.896 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.896 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.184533+0000", 2023-09-23T23:28:38.896 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98050325999999999 2023-09-23T23:28:38.896 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.896 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.896 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:38.896 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.184693+0000", 2023-09-23T23:28:38.896 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000160085 2023-09-23T23:28:38.896 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.896 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.896 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:28:38.897 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.185150+0000", 2023-09-23T23:28:38.897 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00045691599999999997 2023-09-23T23:28:38.897 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.897 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.897 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:38.897 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.193202+0000", 2023-09-23T23:28:38.897 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0080520060000000004 2023-09-23T23:28:38.897 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.897 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.897 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.897 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.203358+0000", 2023-09-23T23:28:38.897 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010156725 2023-09-23T23:28:38.897 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.898 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.898 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.898 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.203402+0000", 2023-09-23T23:28:38.898 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3198000000000002e-05 2023-09-23T23:28:38.898 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.898 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.898 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:38.898 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.203445+0000", 2023-09-23T23:28:38.898 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3229999999999998e-05 2023-09-23T23:28:38.898 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.898 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.898 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.898 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.203481+0000", 2023-09-23T23:28:38.899 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6053000000000002e-05 2023-09-23T23:28:38.899 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.899 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.899 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.899 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.899 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.899 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:193862 11.4 11:22bbc54c:::benchmark_data_smithi110_433746_object87982:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e439)", 2023-09-23T23:28:38.899 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:46.266176+0000", 2023-09-23T23:28:38.899 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 52.568511242, 2023-09-23T23:28:38.899 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.93741761800000001, 2023-09-23T23:28:38.899 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.899 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:38.900 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.900 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.900 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.900 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 193862 2023-09-23T23:28:38.900 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.900 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.900 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.900 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.900 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.266176+0000", 2023-09-23T23:28:38.900 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.900 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.900 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.900 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.901 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.266176+0000", 2023-09-23T23:28:38.901 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.901 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.901 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.901 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.901 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.266176+0000", 2023-09-23T23:28:38.901 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:38.901 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.901 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.901 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.901 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.266179+0000", 2023-09-23T23:28:38.901 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5659999999999998e-06 2023-09-23T23:28:38.901 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.902 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.902 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.902 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.266187+0000", 2023-09-23T23:28:38.902 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7449999999999995e-06 2023-09-23T23:28:38.902 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.902 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.902 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.902 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.266190+0000", 2023-09-23T23:28:38.902 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4599999999999999e-06 2023-09-23T23:28:38.902 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.902 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.902 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.903 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.266270+0000", 2023-09-23T23:28:38.903 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9639000000000001e-05 2023-09-23T23:28:38.903 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.903 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.903 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.903 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.266280+0000", 2023-09-23T23:28:38.903 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0215e-05 2023-09-23T23:28:38.903 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.903 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.903 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.903 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.187319+0000", 2023-09-23T23:28:38.903 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92103824199999995 2023-09-23T23:28:38.903 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.904 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.904 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:38.904 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.187420+0000", 2023-09-23T23:28:38.904 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101814 2023-09-23T23:28:38.904 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.904 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.904 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:28:38.904 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.187691+0000", 2023-09-23T23:28:38.904 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00027023900000000003 2023-09-23T23:28:38.904 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.904 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.904 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:38.904 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.193207+0000", 2023-09-23T23:28:38.904 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0055168270000000002 2023-09-23T23:28:38.905 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.905 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.905 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.905 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.203517+0000", 2023-09-23T23:28:38.905 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010309308999999999 2023-09-23T23:28:38.905 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.905 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.905 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.905 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.203548+0000", 2023-09-23T23:28:38.905 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1735e-05 2023-09-23T23:28:38.905 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.905 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.905 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:38.905 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.203573+0000", 2023-09-23T23:28:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4831999999999999e-05 2023-09-23T23:28:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.203594+0000", 2023-09-23T23:28:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0452e-05 2023-09-23T23:28:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:193876 11.4 11:26b04f3b:::benchmark_data_smithi110_433746_object87996:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e439)", 2023-09-23T23:28:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:46.282157+0000", 2023-09-23T23:28:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 52.552530681999997, 2023-09-23T23:28:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92152330199999999, 2023-09-23T23:28:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 193876 2023-09-23T23:28:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.282157+0000", 2023-09-23T23:28:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.282157+0000", 2023-09-23T23:28:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.282156+0000", 2023-09-23T23:28:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:28:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.909 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.909 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.282160+0000", 2023-09-23T23:28:38.909 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3760000000000001e-06 2023-09-23T23:28:38.909 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.909 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.909 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.909 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.282172+0000", 2023-09-23T23:28:38.909 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2064e-05 2023-09-23T23:28:38.909 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.909 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.909 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.909 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.282177+0000", 2023-09-23T23:28:38.909 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4129999999999999e-06 2023-09-23T23:28:38.910 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.910 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.910 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.910 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.282237+0000", 2023-09-23T23:28:38.910 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0226999999999998e-05 2023-09-23T23:28:38.910 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.910 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.910 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.910 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.282248+0000", 2023-09-23T23:28:38.910 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0752e-05 2023-09-23T23:28:38.910 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.910 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.911 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.911 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.189133+0000", 2023-09-23T23:28:38.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.90688548499999999 2023-09-23T23:28:38.911 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.911 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.911 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:38.911 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.189215+0000", 2023-09-23T23:28:38.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1882e-05 2023-09-23T23:28:38.911 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.911 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.911 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:28:38.911 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.189404+0000", 2023-09-23T23:28:38.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018919799999999999 2023-09-23T23:28:38.912 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.912 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.912 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:38.912 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.196002+0000", 2023-09-23T23:28:38.912 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0065979439999999997 2023-09-23T23:28:38.912 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.912 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.912 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.912 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.203621+0000", 2023-09-23T23:28:38.912 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0076185280000000003 2023-09-23T23:28:38.912 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.912 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.912 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.912 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.203638+0000", 2023-09-23T23:28:38.913 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7337e-05 2023-09-23T23:28:38.913 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.913 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.913 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:38.913 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.203661+0000", 2023-09-23T23:28:38.913 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2901e-05 2023-09-23T23:28:38.913 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.913 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.913 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.913 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.203680+0000", 2023-09-23T23:28:38.913 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9106999999999998e-05 2023-09-23T23:28:38.913 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.913 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.913 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.914 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.914 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.914 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:193879 11.4 11:27d7ec5e:::benchmark_data_smithi110_433746_object87999:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e439)", 2023-09-23T23:28:38.914 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:46.286828+0000", 2023-09-23T23:28:38.914 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 52.547859266000003, 2023-09-23T23:28:38.914 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92219974299999996, 2023-09-23T23:28:38.914 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.914 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:38.914 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.914 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.914 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.914 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 193879 2023-09-23T23:28:38.915 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.915 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.915 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.915 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.915 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.286828+0000", 2023-09-23T23:28:38.915 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.915 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.915 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.915 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.915 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.286828+0000", 2023-09-23T23:28:38.915 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.915 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.915 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.916 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.916 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.286827+0000", 2023-09-23T23:28:38.916 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:38.916 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.916 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.916 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.916 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.286838+0000", 2023-09-23T23:28:38.916 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.111e-05 2023-09-23T23:28:38.916 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.916 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.916 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.916 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.286857+0000", 2023-09-23T23:28:38.917 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9313999999999998e-05 2023-09-23T23:28:38.917 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.917 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.917 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.917 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.286865+0000", 2023-09-23T23:28:38.917 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7870000000000002e-06 2023-09-23T23:28:38.917 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.917 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.917 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.917 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.286924+0000", 2023-09-23T23:28:38.917 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8578000000000001e-05 2023-09-23T23:28:38.917 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.917 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.918 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.918 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.286933+0000", 2023-09-23T23:28:38.918 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7180000000000008e-06 2023-09-23T23:28:38.918 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.918 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.918 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.918 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.190575+0000", 2023-09-23T23:28:38.918 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.90364217099999999 2023-09-23T23:28:38.918 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.918 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.918 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:38.918 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.190639+0000", 2023-09-23T23:28:38.919 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3349000000000006e-05 2023-09-23T23:28:38.919 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.919 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.919 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:28:38.919 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.190829+0000", 2023-09-23T23:28:38.919 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000190099 2023-09-23T23:28:38.919 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.919 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.919 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:38.919 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.196008+0000", 2023-09-23T23:28:38.919 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051788219999999996 2023-09-23T23:28:38.919 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.919 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.203807+0000", 2023-09-23T23:28:38.920 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0077989319999999997 2023-09-23T23:28:38.920 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.920 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.208927+0000", 2023-09-23T23:28:38.920 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051206730000000001 2023-09-23T23:28:38.920 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.920 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:38.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.208984+0000", 2023-09-23T23:28:38.920 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6631e-05 2023-09-23T23:28:38.921 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.921 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.921 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.921 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.209028+0000", 2023-09-23T23:28:38.921 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3914e-05 2023-09-23T23:28:38.921 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.921 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.921 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.921 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.921 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.921 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:193896 11.4 11:205ad59e:::benchmark_data_smithi110_433746_object88016:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e439)", 2023-09-23T23:28:38.921 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:46.303146+0000", 2023-09-23T23:28:38.921 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 52.531541734999998, 2023-09-23T23:28:38.922 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.90603593400000004, 2023-09-23T23:28:38.922 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.922 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:38.922 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.922 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.922 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.922 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 193896 2023-09-23T23:28:38.922 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.922 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.922 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.922 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.922 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.303146+0000", 2023-09-23T23:28:38.923 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.923 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.923 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.923 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.923 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.303146+0000", 2023-09-23T23:28:38.923 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.923 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.923 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.923 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.923 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.303144+0000", 2023-09-23T23:28:38.923 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:28:38.923 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.924 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.924 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.924 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.303152+0000", 2023-09-23T23:28:38.924 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1130000000000001e-06 2023-09-23T23:28:38.924 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.924 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.924 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.924 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.303171+0000", 2023-09-23T23:28:38.924 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9289000000000001e-05 2023-09-23T23:28:38.924 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.924 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.924 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.924 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.303179+0000", 2023-09-23T23:28:38.924 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6809999999999998e-06 2023-09-23T23:28:38.925 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.925 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.925 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.925 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.303244+0000", 2023-09-23T23:28:38.925 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5406000000000004e-05 2023-09-23T23:28:38.925 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.925 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.925 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.925 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:46.303256+0000", 2023-09-23T23:28:38.925 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1793e-05 2023-09-23T23:28:38.925 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.925 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.926 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.926 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.191871+0000", 2023-09-23T23:28:38.926 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88861533100000001 2023-09-23T23:28:38.926 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.926 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.926 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:38.926 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.191947+0000", 2023-09-23T23:28:38.926 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5715000000000003e-05 2023-09-23T23:28:38.926 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.926 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.926 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:28:38.926 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.192153+0000", 2023-09-23T23:28:38.927 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020531100000000001 2023-09-23T23:28:38.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.927 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:38.927 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.198826+0000", 2023-09-23T23:28:38.927 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066733959999999998 2023-09-23T23:28:38.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.927 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.927 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.203866+0000", 2023-09-23T23:28:38.927 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050400820000000004 2023-09-23T23:28:38.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.928 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.928 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.209087+0000", 2023-09-23T23:28:38.928 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0052209240000000001 2023-09-23T23:28:38.928 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.928 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.928 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:38.928 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.209144+0000", 2023-09-23T23:28:38.928 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7371999999999999e-05 2023-09-23T23:28:38.928 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.928 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.928 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.928 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:47.209182+0000", 2023-09-23T23:28:38.928 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7293999999999999e-05 2023-09-23T23:28:38.928 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.929 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.929 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.929 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.929 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.929 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:202719 11.3 11:c1450c8c:::benchmark_data_smithi110_433746_object96839:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-09-23T23:28:38.929 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:28:01.471412+0000", 2023-09-23T23:28:38.929 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 37.363275055999999, 2023-09-23T23:28:38.929 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92726834999999996, 2023-09-23T23:28:38.929 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.929 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:38.929 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.929 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.929 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.930 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 202719 2023-09-23T23:28:38.930 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.930 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.930 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.930 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.930 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:01.471412+0000", 2023-09-23T23:28:38.930 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.930 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.930 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.930 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.930 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:01.471412+0000", 2023-09-23T23:28:38.930 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.931 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.931 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.931 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.931 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:01.471411+0000", 2023-09-23T23:28:38.931 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:38.931 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.931 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.931 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.931 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:01.471417+0000", 2023-09-23T23:28:38.931 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1310000000000003e-06 2023-09-23T23:28:38.931 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.931 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.931 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.931 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:01.471429+0000", 2023-09-23T23:28:38.932 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2622999999999999e-05 2023-09-23T23:28:38.932 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.932 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.932 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.932 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:01.471436+0000", 2023-09-23T23:28:38.932 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4910000000000002e-06 2023-09-23T23:28:38.932 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.932 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.932 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.932 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:01.471477+0000", 2023-09-23T23:28:38.932 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1084000000000002e-05 2023-09-23T23:28:38.932 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.932 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.933 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.933 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:01.471483+0000", 2023-09-23T23:28:38.933 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.84e-06 2023-09-23T23:28:38.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.933 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.933 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.933 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.371119+0000", 2023-09-23T23:28:38.933 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.89963585700000004 2023-09-23T23:28:38.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.933 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.933 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:38.933 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.371272+0000", 2023-09-23T23:28:38.933 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015289599999999999 2023-09-23T23:28:38.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.934 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.934 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,5", 2023-09-23T23:28:38.934 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.371647+0000", 2023-09-23T23:28:38.934 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00037551400000000002 2023-09-23T23:28:38.934 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.934 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.934 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:38.934 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.380418+0000", 2023-09-23T23:28:38.934 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0087710070000000008 2023-09-23T23:28:38.934 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.934 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.934 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.935 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.398497+0000", 2023-09-23T23:28:38.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018078543999999998 2023-09-23T23:28:38.935 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.935 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.935 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.935 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.398580+0000", 2023-09-23T23:28:38.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3020000000000001e-05 2023-09-23T23:28:38.935 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.935 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.935 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:38.935 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.398637+0000", 2023-09-23T23:28:38.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7021999999999997e-05 2023-09-23T23:28:38.935 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.936 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.936 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.398681+0000", 2023-09-23T23:28:38.936 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3961000000000001e-05 2023-09-23T23:28:38.936 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.936 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.936 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.936 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:203242 11.13 11:cd9783f2:::benchmark_data_smithi110_433746_object97362:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-09-23T23:28:38.936 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:28:02.344823+0000", 2023-09-23T23:28:38.936 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 36.489864193999999, 2023-09-23T23:28:38.936 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0388343680000001, 2023-09-23T23:28:38.936 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.937 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:38.937 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.937 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.937 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.937 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 203242 2023-09-23T23:28:38.937 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.937 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344823+0000", 2023-09-23T23:28:38.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.937 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.938 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344823+0000", 2023-09-23T23:28:38.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.938 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344822+0000", 2023-09-23T23:28:38.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:38.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.938 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344826+0000", 2023-09-23T23:28:38.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4530000000000004e-06 2023-09-23T23:28:38.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.939 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.939 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344838+0000", 2023-09-23T23:28:38.939 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.17e-05 2023-09-23T23:28:38.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.939 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.939 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344844+0000", 2023-09-23T23:28:38.939 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7139999999999998e-06 2023-09-23T23:28:38.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.939 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344927+0000", 2023-09-23T23:28:38.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2708999999999997e-05 2023-09-23T23:28:38.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344939+0000", 2023-09-23T23:28:38.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2213e-05 2023-09-23T23:28:38.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.362662+0000", 2023-09-23T23:28:38.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0177228979999999 2023-09-23T23:28:38.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:38.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.362790+0000", 2023-09-23T23:28:38.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012857299999999999 2023-09-23T23:28:38.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.941 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,5", 2023-09-23T23:28:38.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.363083+0000", 2023-09-23T23:28:38.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00029265399999999999 2023-09-23T23:28:38.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.941 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:38.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.369254+0000", 2023-09-23T23:28:38.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0061712609999999999 2023-09-23T23:28:38.942 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.942 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.942 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.942 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383552+0000", 2023-09-23T23:28:38.942 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014297667 2023-09-23T23:28:38.942 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.942 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.942 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.942 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383603+0000", 2023-09-23T23:28:38.942 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1678000000000002e-05 2023-09-23T23:28:38.942 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.942 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.943 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:38.943 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383632+0000", 2023-09-23T23:28:38.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8711e-05 2023-09-23T23:28:38.943 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.943 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.943 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383658+0000", 2023-09-23T23:28:38.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5386000000000001e-05 2023-09-23T23:28:38.943 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.943 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.943 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.943 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.943 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:203250 11.13 11:c8573015:::benchmark_data_smithi110_433746_object97370:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e450)", 2023-09-23T23:28:38.944 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:28:02.375673+0000", 2023-09-23T23:28:38.944 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 36.459014766000003, 2023-09-23T23:28:38.944 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0081006640000001, 2023-09-23T23:28:38.944 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.944 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:38.944 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.944 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.944 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.944 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 203250 2023-09-23T23:28:38.944 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.944 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.944 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.944 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375673+0000", 2023-09-23T23:28:38.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.945 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375673+0000", 2023-09-23T23:28:38.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.945 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375671+0000", 2023-09-23T23:28:38.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:38.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.946 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.946 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375678+0000", 2023-09-23T23:28:38.946 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2099999999999996e-06 2023-09-23T23:28:38.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.946 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.946 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375696+0000", 2023-09-23T23:28:38.946 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7385e-05 2023-09-23T23:28:38.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.946 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.947 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375703+0000", 2023-09-23T23:28:38.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5739999999999998e-06 2023-09-23T23:28:38.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.947 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.400000+0000", 2023-09-23T23:28:38.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.024297062000000001 2023-09-23T23:28:38.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.947 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.400008+0000", 2023-09-23T23:28:38.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7389999999999999e-06 2023-09-23T23:28:38.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.364392+0000", 2023-09-23T23:28:38.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96438375899999995 2023-09-23T23:28:38.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:38.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.364510+0000", 2023-09-23T23:28:38.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117889 2023-09-23T23:28:38.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,5", 2023-09-23T23:28:38.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.364768+0000", 2023-09-23T23:28:38.949 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025844600000000001 2023-09-23T23:28:38.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:38.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.369259+0000", 2023-09-23T23:28:38.949 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0044911650000000001 2023-09-23T23:28:38.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383576+0000", 2023-09-23T23:28:38.949 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014316297 2023-09-23T23:28:38.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.950 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.950 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383688+0000", 2023-09-23T23:28:38.950 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112046 2023-09-23T23:28:38.950 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.950 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.950 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:38.950 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383735+0000", 2023-09-23T23:28:38.950 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7889000000000002e-05 2023-09-23T23:28:38.950 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.950 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.950 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.951 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383773+0000", 2023-09-23T23:28:38.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7795e-05 2023-09-23T23:28:38.951 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.951 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.951 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.951 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.951 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.951 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:203269 11.13 11:cfbb3a7b:::benchmark_data_smithi110_433746_object97389:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e450)", 2023-09-23T23:28:38.951 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:28:02.413168+0000", 2023-09-23T23:28:38.951 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 36.421519545999999, 2023-09-23T23:28:38.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97080936799999995, 2023-09-23T23:28:38.952 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.952 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:38.952 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.952 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.952 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.952 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 203269 2023-09-23T23:28:38.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.952 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.952 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.952 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.952 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.413168+0000", 2023-09-23T23:28:38.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.413168+0000", 2023-09-23T23:28:38.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.953 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.413167+0000", 2023-09-23T23:28:38.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:38.953 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.413173+0000", 2023-09-23T23:28:38.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7850000000000003e-06 2023-09-23T23:28:38.954 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.954 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.954 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.954 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.413189+0000", 2023-09-23T23:28:38.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6121e-05 2023-09-23T23:28:38.954 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.954 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.954 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.954 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.413196+0000", 2023-09-23T23:28:38.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0729999999999999e-06 2023-09-23T23:28:38.954 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.413252+0000", 2023-09-23T23:28:38.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5353999999999999e-05 2023-09-23T23:28:38.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.413270+0000", 2023-09-23T23:28:38.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8624999999999999e-05 2023-09-23T23:28:38.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.956 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.365947+0000", 2023-09-23T23:28:38.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.952676517 2023-09-23T23:28:38.956 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.956 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:28:38.956 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.366009+0000", 2023-09-23T23:28:38.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2507000000000003e-05 2023-09-23T23:28:38.956 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.956 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,5", 2023-09-23T23:28:38.956 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.366154+0000", 2023-09-23T23:28:38.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000144747 2023-09-23T23:28:38.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:28:38.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.369263+0000", 2023-09-23T23:28:38.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031086809999999999 2023-09-23T23:28:38.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383827+0000", 2023-09-23T23:28:38.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014563876999999999 2023-09-23T23:28:38.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:38.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383917+0000", 2023-09-23T23:28:38.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0508999999999997e-05 2023-09-23T23:28:38.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:28:38.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383952+0000", 2023-09-23T23:28:38.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5092999999999998e-05 2023-09-23T23:28:38.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383977+0000", 2023-09-23T23:28:38.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4803999999999998e-05 2023-09-23T23:28:38.959 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.959 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.959 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.959 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.959 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:210997 11.17 11.246f3f77 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-09-23T23:28:38.959 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:28:20.654671+0000", 2023-09-23T23:28:38.959 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.180016553000002, 2023-09-23T23:28:38.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.93506263499999998, 2023-09-23T23:28:38.959 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.959 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:28:38.959 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.960 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.960 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.960 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 210997 2023-09-23T23:28:38.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.960 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.654671+0000", 2023-09-23T23:28:38.960 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.654671+0000", 2023-09-23T23:28:38.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.654670+0000", 2023-09-23T23:28:38.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:28:38.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.654675+0000", 2023-09-23T23:28:38.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8770000000000001e-06 2023-09-23T23:28:38.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.654688+0000", 2023-09-23T23:28:38.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2964e-05 2023-09-23T23:28:38.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.654693+0000", 2023-09-23T23:28:38.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6949999999999997e-06 2023-09-23T23:28:38.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.654745+0000", 2023-09-23T23:28:38.963 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1273000000000002e-05 2023-09-23T23:28:38.963 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.963 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.963 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.654755+0000", 2023-09-23T23:28:38.963 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9180000000000006e-06 2023-09-23T23:28:38.963 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.963 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.963 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:21.589717+0000", 2023-09-23T23:28:38.963 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.93496274499999998 2023-09-23T23:28:38.964 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.964 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.964 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.964 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:21.589733+0000", 2023-09-23T23:28:38.964 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6086000000000002e-05 2023-09-23T23:28:38.964 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.964 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.964 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.964 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.964 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.964 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:211002 11.17 11.d642d537 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-09-23T23:28:38.964 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:28:20.654727+0000", 2023-09-23T23:28:38.964 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.179960261000002, 2023-09-23T23:28:38.965 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.93504338399999998, 2023-09-23T23:28:38.965 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.965 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:28:38.965 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.965 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.965 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.965 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 211002 2023-09-23T23:28:38.965 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.965 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.965 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.965 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.965 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.654727+0000", 2023-09-23T23:28:38.965 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.966 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.966 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.966 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.654727+0000", 2023-09-23T23:28:38.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.966 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.966 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.966 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.654727+0000", 2023-09-23T23:28:38.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:38.966 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.966 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.654729+0000", 2023-09-23T23:28:38.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1830000000000001e-06 2023-09-23T23:28:38.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.654735+0000", 2023-09-23T23:28:38.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6010000000000002e-06 2023-09-23T23:28:38.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.654738+0000", 2023-09-23T23:28:38.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7390000000000001e-06 2023-09-23T23:28:38.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.968 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.654775+0000", 2023-09-23T23:28:38.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6165999999999999e-05 2023-09-23T23:28:38.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.968 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.654779+0000", 2023-09-23T23:28:38.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3640000000000001e-06 2023-09-23T23:28:38.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.968 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:21.589761+0000", 2023-09-23T23:28:38.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.93498203099999999 2023-09-23T23:28:38.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:21.589770+0000", 2023-09-23T23:28:38.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5990000000000008e-06 2023-09-23T23:28:38.969 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.969 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.969 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.969 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:211006 11.17 11.1ed13677 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-09-23T23:28:38.969 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:28:20.659027+0000", 2023-09-23T23:28:38.970 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.175660531999998, 2023-09-23T23:28:38.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.93076329000000002, 2023-09-23T23:28:38.970 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.970 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:28:38.970 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.970 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.970 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.970 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 211006 2023-09-23T23:28:38.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.970 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.659027+0000", 2023-09-23T23:28:38.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.659027+0000", 2023-09-23T23:28:38.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.659026+0000", 2023-09-23T23:28:38.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:28:38.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.659030+0000", 2023-09-23T23:28:38.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2780000000000004e-06 2023-09-23T23:28:38.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.659042+0000", 2023-09-23T23:28:38.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1333e-05 2023-09-23T23:28:38.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.659047+0000", 2023-09-23T23:28:38.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1050000000000003e-06 2023-09-23T23:28:38.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.659102+0000", 2023-09-23T23:28:38.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4962e-05 2023-09-23T23:28:38.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.659113+0000", 2023-09-23T23:28:38.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1412e-05 2023-09-23T23:28:38.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:21.589785+0000", 2023-09-23T23:28:38.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.93067166199999996 2023-09-23T23:28:38.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.974 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.974 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:21.589790+0000", 2023-09-23T23:28:38.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3029999999999999e-06 2023-09-23T23:28:38.974 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.974 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.974 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.974 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:211014 11.17 11.2313537 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-09-23T23:28:38.975 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:28:20.677245+0000", 2023-09-23T23:28:38.975 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.157442378999999, 2023-09-23T23:28:38.975 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91257571900000001, 2023-09-23T23:28:38.975 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:28:38.975 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:28:38.975 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:28:38.975 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:28:38.975 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:38.975 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 211014 2023-09-23T23:28:38.975 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.975 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:28:38.975 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.975 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:28:38.976 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.677245+0000", 2023-09-23T23:28:38.976 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.976 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.976 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.976 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:28:38.976 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.677245+0000", 2023-09-23T23:28:38.976 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:28:38.976 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.976 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.976 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:28:38.976 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.677244+0000", 2023-09-23T23:28:38.976 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:38.976 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.976 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:28:38.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.677249+0000", 2023-09-23T23:28:38.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.028e-06 2023-09-23T23:28:38.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:28:38.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.677261+0000", 2023-09-23T23:28:38.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2451e-05 2023-09-23T23:28:38.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:28:38.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.677267+0000", 2023-09-23T23:28:38.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.682e-06 2023-09-23T23:28:38.978 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.978 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.677319+0000", 2023-09-23T23:28:38.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2478000000000001e-05 2023-09-23T23:28:38.978 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:28:38.978 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:20.677332+0000", 2023-09-23T23:28:38.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2498e-05 2023-09-23T23:28:38.978 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:28:38.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:21.589812+0000", 2023-09-23T23:28:38.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91247999099999999 2023-09-23T23:28:38.979 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:28:38.979 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:28:38.979 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:28:38.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:21.589821+0000", 2023-09-23T23:28:38.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8820000000000007e-06 2023-09-23T23:28:38.979 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.979 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.979 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.979 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:28:38.979 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:28:38.979 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:28:38.980 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:39.007 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:39.041 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:28:39.055 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:39.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:39.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:39.158 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:39.184 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:39 smithi138 ceph-mon[193606]: pgmap v1299: 119 pgs: 16 unknown, 103 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:39 smithi138 ceph-mon[193606]: failed to encode map e477 with expected crc 2023-09-23T23:28:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:39 smithi138 ceph-mon[193606]: osdmap e477: 8 total, 8 up, 7 in 2023-09-23T23:28:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:39 smithi138 ceph-mon[193606]: failed to encode map e477 with expected crc 2023-09-23T23:28:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:39 smithi138 ceph-mon[193606]: failed to encode map e477 with expected crc 2023-09-23T23:28:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:39 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/751230505' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:39 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:39.227 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:39.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:39.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:39.330 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:39.354 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:39 smithi110 ceph-mon[212939]: pgmap v1299: 119 pgs: 16 unknown, 103 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:39.354 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:39 smithi110 ceph-mon[212939]: failed to encode map e477 with expected crc 2023-09-23T23:28:39.354 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:39 smithi110 ceph-mon[212939]: osdmap e477: 8 total, 8 up, 7 in 2023-09-23T23:28:39.354 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:39 smithi110 ceph-mon[212939]: failed to encode map e477 with expected crc 2023-09-23T23:28:39.355 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:39 smithi110 ceph-mon[212939]: failed to encode map e477 with expected crc 2023-09-23T23:28:39.355 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:39 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/751230505' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:39.355 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:39 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:39.355 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:39 smithi110 ceph-mon[221102]: pgmap v1299: 119 pgs: 16 unknown, 103 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:39.355 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:39 smithi110 ceph-mon[221102]: failed to encode map e477 with expected crc 2023-09-23T23:28:39.355 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:39 smithi110 ceph-mon[221102]: osdmap e477: 8 total, 8 up, 7 in 2023-09-23T23:28:39.355 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:39 smithi110 ceph-mon[221102]: failed to encode map e477 with expected crc 2023-09-23T23:28:39.355 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:39 smithi110 ceph-mon[221102]: failed to encode map e477 with expected crc 2023-09-23T23:28:39.355 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:39 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/751230505' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:39.355 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:39 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:28:39.356 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:39.550 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:39.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:39.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:39.556 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:28:39.653 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:39.684 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:39.728 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:39.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:39.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:39.837 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:39.862 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:39.905 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:39.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:39.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:40.011 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:28:40.018 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:40.045 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:40.169 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:40.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:40 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:28:40.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:40 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:28:40.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:40 smithi138 ceph-mon[193606]: failed to encode map e478 with expected crc 2023-09-23T23:28:40.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:40 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:28:40.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:40 smithi138 ceph-mon[193606]: osdmap e478: 8 total, 8 up, 7 in 2023-09-23T23:28:40.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:40 smithi138 ceph-mon[193606]: failed to encode map e478 with expected crc 2023-09-23T23:28:40.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:40 smithi138 ceph-mon[193606]: failed to encode map e478 with expected crc 2023-09-23T23:28:40.253 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:28:40.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:40.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:28:40.285 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:40 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:28:40.285 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:40 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:28:40.285 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:40 smithi110 ceph-mon[212939]: failed to encode map e478 with expected crc 2023-09-23T23:28:40.286 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:40 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:28:40.286 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:40 smithi110 ceph-mon[212939]: osdmap e478: 8 total, 8 up, 7 in 2023-09-23T23:28:40.286 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:40 smithi110 ceph-mon[212939]: failed to encode map e478 with expected crc 2023-09-23T23:28:40.286 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:40 smithi110 ceph-mon[212939]: failed to encode map e478 with expected crc 2023-09-23T23:28:40.286 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:40 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:28:40.286 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:40 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:28:40.286 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:40 smithi110 ceph-mon[221102]: failed to encode map e478 with expected crc 2023-09-23T23:28:40.286 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:40 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:28:40.287 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:40 smithi110 ceph-mon[221102]: osdmap e478: 8 total, 8 up, 7 in 2023-09-23T23:28:40.287 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:40 smithi110 ceph-mon[221102]: failed to encode map e478 with expected crc 2023-09-23T23:28:40.287 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:40 smithi110 ceph-mon[221102]: failed to encode map e478 with expected crc 2023-09-23T23:28:40.357 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:40.384 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:40.425 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:40.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:40.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:40.531 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:40.558 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:40.599 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:40.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:40.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:40.708 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:40.733 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:40.774 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:40.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:40.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:40.882 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:28:40.911 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:41.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:41 smithi110 ceph-mon[212939]: failed to encode map e478 with expected crc 2023-09-23T23:28:41.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:41 smithi110 ceph-mon[212939]: pgmap v1302: 119 pgs: 6 unknown, 113 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:41.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:41 smithi110 ceph-mon[221102]: failed to encode map e478 with expected crc 2023-09-23T23:28:41.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:41 smithi110 ceph-mon[221102]: pgmap v1302: 119 pgs: 6 unknown, 113 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:41 smithi138 ceph-mon[193606]: failed to encode map e478 with expected crc 2023-09-23T23:28:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:41 smithi138 ceph-mon[193606]: pgmap v1302: 119 pgs: 6 unknown, 113 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:41.907 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:28:41.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:41.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:28:42.010 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:28:42.012 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:42.038 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:28:42.040 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:42 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1692408055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:42.045 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:42.065 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:28:42.086 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:28:42.087 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:28:42.107 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:28:42.130 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:28:42.153 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:28:42.174 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:28:42.176 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:28:42.176 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:28:42.176 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:28:42.176 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:28:42.176 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:28:42.176 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:28:42.176 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:28:42.176 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:28:42.337 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:42 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1692408055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:42.339 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:42.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:42.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:42.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:42 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1692408055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:42.444 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:42.474 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:42.520 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:42.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:42.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:42.613 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:28:42.613 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:28:42.625 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:42.657 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:42.707 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:42.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:42.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:42.814 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:42.843 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:43.049 INFO:teuthology.orchestra.run.smithi110.stderr:enabled application 'rados' on pool 'unique_pool_7' 2023-09-23T23:28:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:43 smithi138 ceph-mon[193606]: pgmap v1303: 119 pgs: 119 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:43 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4283502093' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:28:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:43 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:28:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:28:43.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:43 smithi110 ceph-mon[212939]: pgmap v1303: 119 pgs: 119 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:43.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:43 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4283502093' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:28:43.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:43 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:28:43.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:43.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:28:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:43 smithi110 ceph-mon[221102]: pgmap v1303: 119 pgs: 119 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:43.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:43 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4283502093' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:28:43.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:43 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:28:43.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:43.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:28:43.894 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:28:44.144 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:44.144 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[212939]: failed to encode map e479 with expected crc 2023-09-23T23:28:44.144 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:28:44.144 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[212939]: osdmap e479: 8 total, 8 up, 7 in 2023-09-23T23:28:44.144 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.2", "id": [7, 2]}]: dispatch 2023-09-23T23:28:44.144 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.f", "id": [5, 2]}]: dispatch 2023-09-23T23:28:44.144 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]: dispatch 2023-09-23T23:28:44.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.2", "id": [7, 2]}]: dispatch 2023-09-23T23:28:44.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T23:28:44.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-09-23T23:28:44.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.f", "id": [5, 2]}]: dispatch 2023-09-23T23:28:44.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]: dispatch 2023-09-23T23:28:44.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T23:28:44.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-09-23T23:28:44.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:44.145 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[221102]: failed to encode map e479 with expected crc 2023-09-23T23:28:44.145 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:28:44.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[221102]: osdmap e479: 8 total, 8 up, 7 in 2023-09-23T23:28:44.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.2", "id": [7, 2]}]: dispatch 2023-09-23T23:28:44.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.f", "id": [5, 2]}]: dispatch 2023-09-23T23:28:44.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]: dispatch 2023-09-23T23:28:44.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.2", "id": [7, 2]}]: dispatch 2023-09-23T23:28:44.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T23:28:44.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-09-23T23:28:44.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.f", "id": [5, 2]}]: dispatch 2023-09-23T23:28:44.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]: dispatch 2023-09-23T23:28:44.147 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T23:28:44.147 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-09-23T23:28:44.147 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:44 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:44.180 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:44.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:44.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:44.288 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:44.317 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:44 smithi138 ceph-mon[193606]: failed to encode map e479 with expected crc 2023-09-23T23:28:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:44 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:28:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:44 smithi138 ceph-mon[193606]: osdmap e479: 8 total, 8 up, 7 in 2023-09-23T23:28:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:44 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.2", "id": [7, 2]}]: dispatch 2023-09-23T23:28:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:44 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.f", "id": [5, 2]}]: dispatch 2023-09-23T23:28:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:44 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]: dispatch 2023-09-23T23:28:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:44 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.2", "id": [7, 2]}]: dispatch 2023-09-23T23:28:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:44 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T23:28:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:44 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-09-23T23:28:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:44 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.f", "id": [5, 2]}]: dispatch 2023-09-23T23:28:44.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:44 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]: dispatch 2023-09-23T23:28:44.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:44 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T23:28:44.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:44 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-09-23T23:28:44.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:44 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:44.463 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:44.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:44.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:44.567 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:44.583 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:44.601 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:44.650 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:44.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:44.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:44.755 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:28:44.782 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:44.824 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:28:44.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:44.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:28:44.928 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:44.954 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:45.136 DEBUG:teuthology.orchestra.run.smithi110:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_7 bench 90 write' 2023-09-23T23:28:45.136 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T23:28:45.137 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:28:45.139 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T23:28:45.139 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T23:28:45.202 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:45.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:45.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:45.206 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:hints = 1 2023-09-23T23:28:45.206 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T23:28:45.206 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object prefix: benchmark_data_smithi110_491193 2023-09-23T23:28:45.207 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:28:45.207 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 0 0 0 0 0 0 - 0 2023-09-23T23:28:45.307 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:45.338 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:45 smithi138 ceph-mon[193606]: failed to encode map e479 with expected crc 2023-09-23T23:28:45.338 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:45 smithi138 ceph-mon[193606]: pgmap v1305: 119 pgs: 119 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:45.338 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:45 smithi138 ceph-mon[193606]: failed to encode map e480 with expected crc 2023-09-23T23:28:45.338 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:45 smithi138 ceph-mon[193606]: failed to encode map e480 with expected crc 2023-09-23T23:28:45.338 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:45 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.2", "id": [7, 2]}]': finished 2023-09-23T23:28:45.338 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:45 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.f", "id": [5, 2]}]': finished 2023-09-23T23:28:45.338 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:45 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2023-09-23T23:28:45.338 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:45 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-09-23T23:28:45.339 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:45 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-09-23T23:28:45.339 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:45 smithi138 ceph-mon[193606]: osdmap e480: 8 total, 8 up, 7 in 2023-09-23T23:28:45.341 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:45.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[221102]: failed to encode map e479 with expected crc 2023-09-23T23:28:45.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[221102]: pgmap v1305: 119 pgs: 119 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:45.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[221102]: failed to encode map e480 with expected crc 2023-09-23T23:28:45.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[221102]: failed to encode map e480 with expected crc 2023-09-23T23:28:45.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.2", "id": [7, 2]}]': finished 2023-09-23T23:28:45.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.f", "id": [5, 2]}]': finished 2023-09-23T23:28:45.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2023-09-23T23:28:45.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-09-23T23:28:45.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-09-23T23:28:45.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[221102]: osdmap e480: 8 total, 8 up, 7 in 2023-09-23T23:28:45.354 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[212939]: failed to encode map e479 with expected crc 2023-09-23T23:28:45.354 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[212939]: pgmap v1305: 119 pgs: 119 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:45.354 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[212939]: failed to encode map e480 with expected crc 2023-09-23T23:28:45.354 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[212939]: failed to encode map e480 with expected crc 2023-09-23T23:28:45.354 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.2", "id": [7, 2]}]': finished 2023-09-23T23:28:45.355 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.f", "id": [5, 2]}]': finished 2023-09-23T23:28:45.355 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2023-09-23T23:28:45.355 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-09-23T23:28:45.355 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-09-23T23:28:45.355 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:45 smithi110 ceph-mon[212939]: osdmap e480: 8 total, 8 up, 7 in 2023-09-23T23:28:45.395 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:45.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:45.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:45.501 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:45.532 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:45.595 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:45.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:45.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:45.701 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:45.736 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:45.800 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:45.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:45.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:45.905 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:45.952 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:46.170 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:28:46.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:46 smithi110 ceph-mon[212939]: failed to encode map e481 with expected crc 2023-09-23T23:28:46.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:46 smithi110 ceph-mon[212939]: failed to encode map e481 with expected crc 2023-09-23T23:28:46.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:46 smithi110 ceph-mon[212939]: osdmap e481: 8 total, 8 up, 7 in 2023-09-23T23:28:46.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:46 smithi110 ceph-mon[212939]: failed to encode map e481 with expected crc 2023-09-23T23:28:46.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:46 smithi110 ceph-mon[212939]: failed to encode map e481 with expected crc 2023-09-23T23:28:46.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:46 smithi110 ceph-mon[221102]: failed to encode map e481 with expected crc 2023-09-23T23:28:46.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:46 smithi110 ceph-mon[221102]: failed to encode map e481 with expected crc 2023-09-23T23:28:46.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:46 smithi110 ceph-mon[221102]: osdmap e481: 8 total, 8 up, 7 in 2023-09-23T23:28:46.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:46 smithi110 ceph-mon[221102]: failed to encode map e481 with expected crc 2023-09-23T23:28:46.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:46 smithi110 ceph-mon[221102]: failed to encode map e481 with expected crc 2023-09-23T23:28:46.207 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 1 16 2798 2782 173.779 173.875 0.00407354 0.00572735 2023-09-23T23:28:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:46 smithi138 ceph-mon[193606]: failed to encode map e481 with expected crc 2023-09-23T23:28:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:46 smithi138 ceph-mon[193606]: failed to encode map e481 with expected crc 2023-09-23T23:28:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:46 smithi138 ceph-mon[193606]: osdmap e481: 8 total, 8 up, 7 in 2023-09-23T23:28:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:46 smithi138 ceph-mon[193606]: failed to encode map e481 with expected crc 2023-09-23T23:28:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:46 smithi138 ceph-mon[193606]: failed to encode map e481 with expected crc 2023-09-23T23:28:46.974 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:46.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:46.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:47.057 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:28:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:28:46] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-09-23T23:28:47.084 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:47.117 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:47.173 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:47.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:47.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:47.207 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 2 16 4714 4698 146.769 119.75 0.00787006 0.00679505 2023-09-23T23:28:47.281 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:47.349 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:47 smithi110 ceph-mon[212939]: pgmap v1308: 119 pgs: 1 remapped+peering, 118 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:47.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:47 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:28:47.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:47 smithi110 ceph-mon[212939]: failed to encode map e482 with expected crc 2023-09-23T23:28:47.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:47 smithi110 ceph-mon[212939]: osdmap e482: 8 total, 8 up, 7 in 2023-09-23T23:28:47.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:47 smithi110 ceph-mon[212939]: failed to encode map e482 with expected crc 2023-09-23T23:28:47.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:47 smithi110 ceph-mon[212939]: failed to encode map e482 with expected crc 2023-09-23T23:28:47.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:47 smithi110 ceph-mon[221102]: pgmap v1308: 119 pgs: 1 remapped+peering, 118 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:47.350 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:47 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:28:47.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:47 smithi110 ceph-mon[221102]: failed to encode map e482 with expected crc 2023-09-23T23:28:47.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:47 smithi110 ceph-mon[221102]: osdmap e482: 8 total, 8 up, 7 in 2023-09-23T23:28:47.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:47 smithi110 ceph-mon[221102]: failed to encode map e482 with expected crc 2023-09-23T23:28:47.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:47 smithi110 ceph-mon[221102]: failed to encode map e482 with expected crc 2023-09-23T23:28:47.366 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:47.406 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:47.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:47 smithi138 ceph-mon[193606]: pgmap v1308: 119 pgs: 1 remapped+peering, 118 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T23:28:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:47 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:28:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:47 smithi138 ceph-mon[193606]: failed to encode map e482 with expected crc 2023-09-23T23:28:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:47 smithi138 ceph-mon[193606]: osdmap e482: 8 total, 8 up, 7 in 2023-09-23T23:28:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:47 smithi138 ceph-mon[193606]: failed to encode map e482 with expected crc 2023-09-23T23:28:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:47 smithi138 ceph-mon[193606]: failed to encode map e482 with expected crc 2023-09-23T23:28:47.535 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:47.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:47.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:47.640 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:47.671 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:47.704 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:47.725 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:47.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:47.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:47.835 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:28:47.846 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:28:47.846 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:28:47.920 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:48.097 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:48 smithi110 ceph-mon[212939]: osdmap e483: 8 total, 8 up, 7 in 2023-09-23T23:28:48.097 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:48 smithi110 ceph-mon[212939]: failed to encode map e483 with expected crc 2023-09-23T23:28:48.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:48 smithi110 ceph-mon[212939]: failed to encode map e483 with expected crc 2023-09-23T23:28:48.100 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:48 smithi110 ceph-mon[221102]: osdmap e483: 8 total, 8 up, 7 in 2023-09-23T23:28:48.100 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:48 smithi110 ceph-mon[221102]: failed to encode map e483 with expected crc 2023-09-23T23:28:48.100 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:48 smithi110 ceph-mon[221102]: failed to encode map e483 with expected crc 2023-09-23T23:28:48.100 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:48 smithi110 ceph-mon[221102]: failed to encode map e483 with expected crc 2023-09-23T23:28:48.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:48 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/959826668' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:28:48.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:48 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:28:48.122 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:28:48.172 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:28:48.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:48.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:28:48.210 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 3 16 6899 6883 143.222 136.562 0.00303853 0.00697239 2023-09-23T23:28:48.276 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:48.316 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:48.377 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:48.380 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:48 smithi138 ceph-mon[193606]: osdmap e483: 8 total, 8 up, 7 in 2023-09-23T23:28:48.380 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:48 smithi138 ceph-mon[193606]: failed to encode map e483 with expected crc 2023-09-23T23:28:48.380 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:48 smithi138 ceph-mon[193606]: failed to encode map e483 with expected crc 2023-09-23T23:28:48.380 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:48 smithi138 ceph-mon[193606]: failed to encode map e483 with expected crc 2023-09-23T23:28:48.380 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:48 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/959826668' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:28:48.380 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:48 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:28:48.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:48.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:48.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:48 smithi110 ceph-mon[212939]: failed to encode map e483 with expected crc 2023-09-23T23:28:48.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:48 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/959826668' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:28:48.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:48 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:28:48.482 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:48.523 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:48.590 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:48.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:48.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:48.697 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:48.723 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:48.765 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:48.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:48.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:48.873 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:48.932 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:49.096 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:49 smithi138 ceph-mon[193606]: failed to encode map e483 with expected crc 2023-09-23T23:28:49.096 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:49 smithi138 ceph-mon[193606]: pgmap v1311: 119 pgs: 1 remapped+peering, 118 active+clean; 33 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 8.1 MiB/s wr, 129 op/s 2023-09-23T23:28:49.096 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:49 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:28:49.097 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:49 smithi138 ceph-mon[193606]: failed to encode map e484 with expected crc 2023-09-23T23:28:49.097 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:49 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:28:49.097 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:49 smithi138 ceph-mon[193606]: osdmap e484: 8 total, 8 up, 7 in 2023-09-23T23:28:49.097 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:49 smithi138 ceph-mon[193606]: failed to encode map e484 with expected crc 2023-09-23T23:28:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:49 smithi110 ceph-mon[212939]: failed to encode map e483 with expected crc 2023-09-23T23:28:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:49 smithi110 ceph-mon[212939]: pgmap v1311: 119 pgs: 1 remapped+peering, 118 active+clean; 33 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 8.1 MiB/s wr, 129 op/s 2023-09-23T23:28:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:49 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:28:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:49 smithi110 ceph-mon[212939]: failed to encode map e484 with expected crc 2023-09-23T23:28:49.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:49 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:28:49.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:49 smithi110 ceph-mon[212939]: osdmap e484: 8 total, 8 up, 7 in 2023-09-23T23:28:49.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:49 smithi110 ceph-mon[212939]: failed to encode map e484 with expected crc 2023-09-23T23:28:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:49 smithi110 ceph-mon[221102]: failed to encode map e483 with expected crc 2023-09-23T23:28:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:49 smithi110 ceph-mon[221102]: pgmap v1311: 119 pgs: 1 remapped+peering, 118 active+clean; 33 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 8.1 MiB/s wr, 129 op/s 2023-09-23T23:28:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:49 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:28:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:49 smithi110 ceph-mon[221102]: failed to encode map e484 with expected crc 2023-09-23T23:28:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:49 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:28:49.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:49 smithi110 ceph-mon[221102]: osdmap e484: 8 total, 8 up, 7 in 2023-09-23T23:28:49.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:49 smithi110 ceph-mon[221102]: failed to encode map e484 with expected crc 2023-09-23T23:28:49.210 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 4 16 9452 9436 147.29 159.562 0.00668776 0.00677841 2023-09-23T23:28:49.673 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:49.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:49.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:49.780 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:49.815 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:49.877 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:49.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:49.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:49.984 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:50.017 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:50.079 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:50.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:50.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:50.184 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:50.210 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 5 16 11901 11885 148.441 153.062 0.00386673 0.00672754 2023-09-23T23:28:50.337 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:50.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:50 smithi138 ceph-mon[193606]: failed to encode map e484 with expected crc 2023-09-23T23:28:50.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:50 smithi110 ceph-mon[212939]: failed to encode map e484 with expected crc 2023-09-23T23:28:50.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:50 smithi110 ceph-mon[221102]: failed to encode map e484 with expected crc 2023-09-23T23:28:50.537 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:50.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:50.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:50.642 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:50.668 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:51.065 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:51.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:51.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:51.169 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:51.210 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 6 16 13985 13969 145.409 130.25 0.00656077 0.00686781 2023-09-23T23:28:51.216 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:51.344 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:51 smithi110 ceph-mon[212939]: pgmap v1313: 119 pgs: 1 remapped+peering, 118 active+clean; 352 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T23:28:51.344 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:51 smithi110 ceph-mon[221102]: pgmap v1313: 119 pgs: 1 remapped+peering, 118 active+clean; 352 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T23:28:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:51 smithi138 ceph-mon[193606]: pgmap v1313: 119 pgs: 1 remapped+peering, 118 active+clean; 352 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T23:28:51.825 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:51.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:51.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:51.932 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:51.966 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:52.025 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:52.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:52.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:52.133 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:52.164 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:52.170 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:52 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T23:28:52.210 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 7 16 15705 15689 139.995 107.5 0.0062463 0.0071357 2023-09-23T23:28:52.227 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:52.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:52.233 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:52 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T23:28:52.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:52.334 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:28:52.435 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:52 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T23:28:52.441 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:52.926 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:28:53.034 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:28:53.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:53.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:28:53.149 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:53.178 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:53 smithi110 ceph-mon[212939]: pgmap v1314: 119 pgs: 119 active+clean; 616 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-09-23T23:28:53.211 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 8 16 17835 17819 139.135 133.125 0.009642 0.0071795 2023-09-23T23:28:53.238 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:53.399 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:53.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:53.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:53.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:53 smithi138 ceph-mon[193606]: pgmap v1314: 119 pgs: 119 active+clean; 616 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-09-23T23:28:53.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:53 smithi110 ceph-mon[221102]: pgmap v1314: 119 pgs: 119 active+clean; 616 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-09-23T23:28:53.505 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:53.539 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:53.953 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:53.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:53.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:54.060 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:54.091 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:54.146 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:54.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:54.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:54.212 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 9 16 19556 19540 135.608 107.562 0.00167891 0.00736239 2023-09-23T23:28:54.256 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:54.279 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:54.288 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:54.341 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:54.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:54.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:54.447 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:54.477 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:54.528 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:54.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:54.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:54.635 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:54.664 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:54 smithi138 ceph-mon[193606]: pgmap v1315: 119 pgs: 119 active+clean; 700 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T23:28:54.666 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:54.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:54 smithi110 ceph-mon[212939]: pgmap v1315: 119 pgs: 119 active+clean; 700 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T23:28:54.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:54 smithi110 ceph-mon[221102]: pgmap v1315: 119 pgs: 119 active+clean; 700 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T23:28:54.718 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:54.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:54.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:54.824 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:28:54.853 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:54.907 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:28:54.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:54.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:28:54.922 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:28:55.012 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:55.019 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:28:55.042 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:55.069 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:28:55.092 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:55.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:55.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:55.123 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:28:55.180 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:28:55.198 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:55.214 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:28:55.215 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 10 16 21109 21093 131.754 97.0625 0.00977368 0.0075801 2023-09-23T23:28:55.216 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:28:55.240 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:28:55.266 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:28:55.291 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:55.306 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:28:55.342 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:28:55.348 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:28:55.348 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:28:55.348 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:28:55.348 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:28:55.348 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:28:55.348 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:28:55.348 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:28:55.348 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:28:55.387 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:55.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:55.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:55.493 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:28:55.529 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:55.577 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:28:55.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:55.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:28:55.682 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:55.717 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:55.777 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:55.782 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:55 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2814397299' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:55.782 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:55 smithi138 ceph-mon[193606]: from='client.46241 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:28:55.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:55.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:55.884 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:28:55.934 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:55 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2814397299' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:55.934 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:55 smithi110 ceph-mon[221102]: from='client.46241 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:28:55.935 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:55 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2814397299' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:28:55.935 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:55 smithi110 ceph-mon[212939]: from='client.46241 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:28:55.938 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:56.215 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 11 16 22644 22628 128.471 95.9375 0.180872 0.00775348 2023-09-23T23:28:56.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:56 smithi138 ceph-mon[193606]: pgmap v1316: 119 pgs: 119 active+clean; 1.1 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T23:28:56.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:28:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:28:56] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-09-23T23:28:56.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:56 smithi110 ceph-mon[212939]: pgmap v1316: 119 pgs: 119 active+clean; 1.1 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T23:28:56.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:56 smithi110 ceph-mon[221102]: pgmap v1316: 119 pgs: 119 active+clean; 1.1 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T23:28:57.215 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 12 16 23881 23865 124.208 77.3125 0.00534042 0.00804592 2023-09-23T23:28:57.407 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:28:57.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:57.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:28:57.516 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:28:57.547 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:57.568 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1315,"stamp":"2023-09-23T23:28:53.729550+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":733616032,"num_objects":11209,"num_object_clones":0,"num_object_copies":33627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11246,"num_write_kb":715869,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":660968,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":16231,"ondisk_log_size":16231,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6822328,"kb_used_data":2321280,"kb_used_omap":29,"kb_used_meta":4500962,"kb_avail":649393736,"statfs":{"total":671965249536,"available":664979185664,"internally_reserved":0,"allocated":2376990720,"data_stored":2364072943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":4608985811},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":699072512,"num_objects":10667,"num_object_clones":0,"num_object_copies":32001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10667,"num_write_kb":682688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.670285"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":680,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.098111+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:28:47.098111+0000","last_peered":"2023-09-23T23:28:47.098111+0000","last_clean":"2023-09-23T23:28:47.098111+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:28:47.098111+0000","last_undegraded":"2023-09-23T23:28:47.098111+0000","last_fullsized":"2023-09-23T23:28:47.098111+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":518,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.094646+0000","last_change":"2023-09-23T23:22:38.439629+0000","last_active":"2023-09-23T23:28:47.094646+0000","last_peered":"2023-09-23T23:28:47.094646+0000","last_clean":"2023-09-23T23:28:47.094646+0000","last_became_active":"2023-09-23T23:22:38.439285+0000","last_became_peered":"2023-09-23T23:22:38.439285+0000","last_unstale":"2023-09-23T23:28:47.094646+0000","last_undegraded":"2023-09-23T23:28:47.094646+0000","last_fullsized":"2023-09-23T23:28:47.094646+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:34.431221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":454,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.088941+0000","last_change":"2023-09-23T23:23:43.511912+0000","last_active":"2023-09-23T23:28:47.088941+0000","last_peered":"2023-09-23T23:28:47.088941+0000","last_clean":"2023-09-23T23:28:47.088941+0000","last_became_active":"2023-09-23T23:23:43.511574+0000","last_became_peered":"2023-09-23T23:23:43.511574+0000","last_unstale":"2023-09-23T23:28:47.088941+0000","last_undegraded":"2023-09-23T23:28:47.088941+0000","last_fullsized":"2023-09-23T23:28:47.088941+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:16.286926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":573,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.098825+0000","last_change":"2023-09-23T23:23:50.459634+0000","last_active":"2023-09-23T23:28:47.098825+0000","last_peered":"2023-09-23T23:28:47.098825+0000","last_clean":"2023-09-23T23:28:47.098825+0000","last_became_active":"2023-09-23T23:23:45.453958+0000","last_became_peered":"2023-09-23T23:23:45.453958+0000","last_unstale":"2023-09-23T23:28:47.098825+0000","last_undegraded":"2023-09-23T23:28:47.098825+0000","last_fullsized":"2023-09-23T23:28:47.098825+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:56.563201+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":194,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.091111+0000","last_change":"2023-09-23T23:26:11.528846+0000","last_active":"2023-09-23T23:28:47.091111+0000","last_peered":"2023-09-23T23:28:47.091111+0000","last_clean":"2023-09-23T23:28:47.091111+0000","last_became_active":"2023-09-23T23:26:11.528529+0000","last_became_peered":"2023-09-23T23:26:11.528529+0000","last_unstale":"2023-09-23T23:28:47.091111+0000","last_undegraded":"2023-09-23T23:28:47.091111+0000","last_fullsized":"2023-09-23T23:28:47.091111+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:35.094583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":449,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089853+0000","last_change":"2023-09-23T23:28:22.604718+0000","last_active":"2023-09-23T23:28:47.089853+0000","last_peered":"2023-09-23T23:28:47.089853+0000","last_clean":"2023-09-23T23:28:47.089853+0000","last_became_active":"2023-09-23T23:28:22.604498+0000","last_became_peered":"2023-09-23T23:28:22.604498+0000","last_unstale":"2023-09-23T23:28:47.089853+0000","last_undegraded":"2023-09-23T23:28:47.089853+0000","last_fullsized":"2023-09-23T23:28:47.089853+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:30.472412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":787,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.094565+0000","last_change":"2023-09-23T23:28:05.634824+0000","last_active":"2023-09-23T23:28:47.094565+0000","last_peered":"2023-09-23T23:28:47.094565+0000","last_clean":"2023-09-23T23:28:47.094565+0000","last_became_active":"2023-09-23T23:27:23.551046+0000","last_became_peered":"2023-09-23T23:27:23.551046+0000","last_unstale":"2023-09-23T23:28:47.094565+0000","last_undegraded":"2023-09-23T23:28:47.094565+0000","last_fullsized":"2023-09-23T23:28:47.094565+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:59.062239+0000","scrub_duration":0.0034158959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":481,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.088735+0000","last_change":"2023-09-23T23:27:20.554007+0000","last_active":"2023-09-23T23:28:47.088735+0000","last_peered":"2023-09-23T23:28:47.088735+0000","last_clean":"2023-09-23T23:28:47.088735+0000","last_became_active":"2023-09-23T23:27:20.553716+0000","last_became_peered":"2023-09-23T23:27:20.553716+0000","last_unstale":"2023-09-23T23:28:47.088735+0000","last_undegraded":"2023-09-23T23:28:47.088735+0000","last_fullsized":"2023-09-23T23:28:47.088735+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:38.911862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":417,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.093566+0000","last_change":"2023-09-23T23:25:31.735427+0000","last_active":"2023-09-23T23:28:47.093566+0000","last_peered":"2023-09-23T23:28:47.093566+0000","last_clean":"2023-09-23T23:28:47.093566+0000","last_became_active":"2023-09-23T23:25:31.734582+0000","last_became_peered":"2023-09-23T23:25:31.734582+0000","last_unstale":"2023-09-23T23:28:47.093566+0000","last_undegraded":"2023-09-23T23:28:47.093566+0000","last_fullsized":"2023-09-23T23:28:47.093566+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:39.075149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":804,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089926+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:28:47.089926+0000","last_peered":"2023-09-23T23:28:47.089926+0000","last_clean":"2023-09-23T23:28:47.089926+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:28:47.089926+0000","last_undegraded":"2023-09-23T23:28:47.089926+0000","last_fullsized":"2023-09-23T23:28:47.089926+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":116,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.099221+0000","last_change":"2023-09-23T23:28:22.602481+0000","last_active":"2023-09-23T23:28:47.099221+0000","last_peered":"2023-09-23T23:28:47.099221+0000","last_clean":"2023-09-23T23:28:47.099221+0000","last_became_active":"2023-09-23T23:28:22.602282+0000","last_became_peered":"2023-09-23T23:28:22.602282+0000","last_unstale":"2023-09-23T23:28:47.099221+0000","last_undegraded":"2023-09-23T23:28:47.099221+0000","last_fullsized":"2023-09-23T23:28:47.099221+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:39.906003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":472,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.088646+0000","last_change":"2023-09-23T23:26:11.528354+0000","last_active":"2023-09-23T23:28:47.088646+0000","last_peered":"2023-09-23T23:28:47.088646+0000","last_clean":"2023-09-23T23:28:47.088646+0000","last_became_active":"2023-09-23T23:26:11.528076+0000","last_became_peered":"2023-09-23T23:26:11.528076+0000","last_unstale":"2023-09-23T23:28:47.088646+0000","last_undegraded":"2023-09-23T23:28:47.088646+0000","last_fullsized":"2023-09-23T23:28:47.088646+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:16.100396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":757,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.093014+0000","last_change":"2023-09-23T23:26:14.521792+0000","last_active":"2023-09-23T23:28:47.093014+0000","last_peered":"2023-09-23T23:28:47.093014+0000","last_clean":"2023-09-23T23:28:47.093014+0000","last_became_active":"2023-09-23T23:26:14.521438+0000","last_became_peered":"2023-09-23T23:26:14.521438+0000","last_unstale":"2023-09-23T23:28:47.093014+0000","last_undegraded":"2023-09-23T23:28:47.093014+0000","last_fullsized":"2023-09-23T23:28:47.093014+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:09.103712+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":115,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.099073+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:28:47.099073+0000","last_peered":"2023-09-23T23:28:47.099073+0000","last_clean":"2023-09-23T23:28:47.099073+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:28:47.099073+0000","last_undegraded":"2023-09-23T23:28:47.099073+0000","last_fullsized":"2023-09-23T23:28:47.099073+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":433,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.090573+0000","last_change":"2023-09-23T23:26:48.538059+0000","last_active":"2023-09-23T23:28:47.090573+0000","last_peered":"2023-09-23T23:28:47.090573+0000","last_clean":"2023-09-23T23:28:47.090573+0000","last_became_active":"2023-09-23T23:26:48.537735+0000","last_became_peered":"2023-09-23T23:26:48.537735+0000","last_unstale":"2023-09-23T23:28:47.090573+0000","last_undegraded":"2023-09-23T23:28:47.090573+0000","last_fullsized":"2023-09-23T23:28:47.090573+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:32.878591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":878,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.099076+0000","last_change":"2023-09-23T23:28:09.363053+0000","last_active":"2023-09-23T23:28:47.099076+0000","last_peered":"2023-09-23T23:28:47.099076+0000","last_clean":"2023-09-23T23:28:47.099076+0000","last_became_active":"2023-09-23T23:26:51.563536+0000","last_became_peered":"2023-09-23T23:26:51.563536+0000","last_unstale":"2023-09-23T23:28:47.099076+0000","last_undegraded":"2023-09-23T23:28:47.099076+0000","last_fullsized":"2023-09-23T23:28:47.099076+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:29.922798+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":89,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089423+0000","last_change":"2023-09-23T23:26:48.669857+0000","last_active":"2023-09-23T23:28:47.089423+0000","last_peered":"2023-09-23T23:28:47.089423+0000","last_clean":"2023-09-23T23:28:47.089423+0000","last_became_active":"2023-09-23T23:26:48.538246+0000","last_became_peered":"2023-09-23T23:26:48.538246+0000","last_unstale":"2023-09-23T23:28:47.089423+0000","last_undegraded":"2023-09-23T23:28:47.089423+0000","last_fullsized":"2023-09-23T23:28:47.089423+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:19.039472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":70,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.091248+0000","last_change":"2023-09-23T23:27:20.551305+0000","last_active":"2023-09-23T23:28:47.091248+0000","last_peered":"2023-09-23T23:28:47.091248+0000","last_clean":"2023-09-23T23:28:47.091248+0000","last_became_active":"2023-09-23T23:27:20.551017+0000","last_became_peered":"2023-09-23T23:27:20.551017+0000","last_unstale":"2023-09-23T23:28:47.091248+0000","last_undegraded":"2023-09-23T23:28:47.091248+0000","last_fullsized":"2023-09-23T23:28:47.091248+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:27.026015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":559,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.093330+0000","last_change":"2023-09-23T23:24:02.268079+0000","last_active":"2023-09-23T23:28:47.093330+0000","last_peered":"2023-09-23T23:28:47.093330+0000","last_clean":"2023-09-23T23:28:47.093330+0000","last_became_active":"2023-09-23T23:23:45.463106+0000","last_became_peered":"2023-09-23T23:23:45.463106+0000","last_unstale":"2023-09-23T23:28:47.093330+0000","last_undegraded":"2023-09-23T23:28:47.093330+0000","last_fullsized":"2023-09-23T23:28:47.093330+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.128813+0000","scrub_duration":0.0018535310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":115,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.096460+0000","last_change":"2023-09-23T23:26:11.534727+0000","last_active":"2023-09-23T23:28:47.096460+0000","last_peered":"2023-09-23T23:28:47.096460+0000","last_clean":"2023-09-23T23:28:47.096460+0000","last_became_active":"2023-09-23T23:26:11.513694+0000","last_became_peered":"2023-09-23T23:26:11.513694+0000","last_unstale":"2023-09-23T23:28:47.096460+0000","last_undegraded":"2023-09-23T23:28:47.096460+0000","last_fullsized":"2023-09-23T23:28:47.096460+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:04.562837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":472,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.096368+0000","last_change":"2023-09-23T23:25:31.718910+0000","last_active":"2023-09-23T23:28:47.096368+0000","last_peered":"2023-09-23T23:28:47.096368+0000","last_clean":"2023-09-23T23:28:47.096368+0000","last_became_active":"2023-09-23T23:25:31.718567+0000","last_became_peered":"2023-09-23T23:25:31.718567+0000","last_unstale":"2023-09-23T23:28:47.096368+0000","last_undegraded":"2023-09-23T23:28:47.096368+0000","last_fullsized":"2023-09-23T23:28:47.096368+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:21.447944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":599,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.098667+0000","last_change":"2023-09-23T23:28:14.378358+0000","last_active":"2023-09-23T23:28:47.098667+0000","last_peered":"2023-09-23T23:28:47.098667+0000","last_clean":"2023-09-23T23:28:47.098667+0000","last_became_active":"2023-09-23T23:26:34.550151+0000","last_became_peered":"2023-09-23T23:26:34.550151+0000","last_unstale":"2023-09-23T23:28:47.098667+0000","last_undegraded":"2023-09-23T23:28:47.098667+0000","last_fullsized":"2023-09-23T23:28:47.098667+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:47.794690+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":416,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089693+0000","last_change":"2023-09-23T23:26:11.569228+0000","last_active":"2023-09-23T23:28:47.089693+0000","last_peered":"2023-09-23T23:28:47.089693+0000","last_clean":"2023-09-23T23:28:47.089693+0000","last_became_active":"2023-09-23T23:26:11.513159+0000","last_became_peered":"2023-09-23T23:26:11.513159+0000","last_unstale":"2023-09-23T23:28:47.089693+0000","last_undegraded":"2023-09-23T23:28:47.089693+0000","last_fullsized":"2023-09-23T23:28:47.089693+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:40.698257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":480,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089778+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:28:47.089778+0000","last_peered":"2023-09-23T23:28:47.089778+0000","last_clean":"2023-09-23T23:28:47.089778+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:28:47.089778+0000","last_undegraded":"2023-09-23T23:28:47.089778+0000","last_fullsized":"2023-09-23T23:28:47.089778+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":823,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.088831+0000","last_change":"2023-09-23T23:26:34.551458+0000","last_active":"2023-09-23T23:28:47.088831+0000","last_peered":"2023-09-23T23:28:47.088831+0000","last_clean":"2023-09-23T23:28:47.088831+0000","last_became_active":"2023-09-23T23:26:34.551067+0000","last_became_peered":"2023-09-23T23:26:34.551067+0000","last_unstale":"2023-09-23T23:28:47.088831+0000","last_undegraded":"2023-09-23T23:28:47.088831+0000","last_fullsized":"2023-09-23T23:28:47.088831+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:05.788656+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":83,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089240+0000","last_change":"2023-09-23T23:26:48.563723+0000","last_active":"2023-09-23T23:28:47.089240+0000","last_peered":"2023-09-23T23:28:47.089240+0000","last_clean":"2023-09-23T23:28:47.089240+0000","last_became_active":"2023-09-23T23:26:48.563340+0000","last_became_peered":"2023-09-23T23:26:48.563340+0000","last_unstale":"2023-09-23T23:28:47.089240+0000","last_undegraded":"2023-09-23T23:28:47.089240+0000","last_fullsized":"2023-09-23T23:28:47.089240+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:05.974131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":370,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089833+0000","last_change":"2023-09-23T23:26:32.040316+0000","last_active":"2023-09-23T23:28:47.089833+0000","last_peered":"2023-09-23T23:28:47.089833+0000","last_clean":"2023-09-23T23:28:47.089833+0000","last_became_active":"2023-09-23T23:26:32.039758+0000","last_became_peered":"2023-09-23T23:26:32.039758+0000","last_unstale":"2023-09-23T23:28:47.089833+0000","last_undegraded":"2023-09-23T23:28:47.089833+0000","last_fullsized":"2023-09-23T23:28:47.089833+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:03.930331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":647,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.091620+0000","last_change":"2023-09-23T23:26:14.527752+0000","last_active":"2023-09-23T23:28:47.091620+0000","last_peered":"2023-09-23T23:28:47.091620+0000","last_clean":"2023-09-23T23:28:47.091620+0000","last_became_active":"2023-09-23T23:26:14.525188+0000","last_became_peered":"2023-09-23T23:26:14.525188+0000","last_unstale":"2023-09-23T23:28:47.091620+0000","last_undegraded":"2023-09-23T23:28:47.091620+0000","last_fullsized":"2023-09-23T23:28:47.091620+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:10.617744+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":338,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089001+0000","last_change":"2023-09-23T23:25:31.725776+0000","last_active":"2023-09-23T23:28:47.089001+0000","last_peered":"2023-09-23T23:28:47.089001+0000","last_clean":"2023-09-23T23:28:47.089001+0000","last_became_active":"2023-09-23T23:25:31.725429+0000","last_became_peered":"2023-09-23T23:25:31.725429+0000","last_unstale":"2023-09-23T23:28:47.089001+0000","last_undegraded":"2023-09-23T23:28:47.089001+0000","last_fullsized":"2023-09-23T23:28:47.089001+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:07.289251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":419,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089119+0000","last_change":"2023-09-23T23:28:22.603836+0000","last_active":"2023-09-23T23:28:47.089119+0000","last_peered":"2023-09-23T23:28:47.089119+0000","last_clean":"2023-09-23T23:28:47.089119+0000","last_became_active":"2023-09-23T23:28:22.603490+0000","last_became_peered":"2023-09-23T23:28:22.603490+0000","last_unstale":"2023-09-23T23:28:47.089119+0000","last_undegraded":"2023-09-23T23:28:47.089119+0000","last_fullsized":"2023-09-23T23:28:47.089119+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:00.869241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":827,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.098606+0000","last_change":"2023-09-23T23:27:23.549144+0000","last_active":"2023-09-23T23:28:47.098606+0000","last_peered":"2023-09-23T23:28:47.098606+0000","last_clean":"2023-09-23T23:28:47.098606+0000","last_became_active":"2023-09-23T23:27:23.548774+0000","last_became_peered":"2023-09-23T23:27:23.548774+0000","last_unstale":"2023-09-23T23:28:47.098606+0000","last_undegraded":"2023-09-23T23:28:47.098606+0000","last_fullsized":"2023-09-23T23:28:47.098606+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:18.211722+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":165,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.085389+0000","last_change":"2023-09-23T23:26:32.139091+0000","last_active":"2023-09-23T23:28:47.085389+0000","last_peered":"2023-09-23T23:28:47.085389+0000","last_clean":"2023-09-23T23:28:47.085389+0000","last_became_active":"2023-09-23T23:26:32.053033+0000","last_became_peered":"2023-09-23T23:26:32.053033+0000","last_unstale":"2023-09-23T23:28:47.085389+0000","last_undegraded":"2023-09-23T23:28:47.085389+0000","last_fullsized":"2023-09-23T23:28:47.085389+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:32.625818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":122,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.088985+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:28:47.088985+0000","last_peered":"2023-09-23T23:28:47.088985+0000","last_clean":"2023-09-23T23:28:47.088985+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:28:47.088985+0000","last_undegraded":"2023-09-23T23:28:47.088985+0000","last_fullsized":"2023-09-23T23:28:47.088985+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"484'536","reported_seq":552,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:48.853226+0000","last_change":"2023-09-23T23:28:38.062488+0000","last_active":"2023-09-23T23:28:48.853226+0000","last_peered":"2023-09-23T23:28:48.853226+0000","last_clean":"2023-09-23T23:28:48.853226+0000","last_became_active":"2023-09-23T23:28:38.062098+0000","last_became_peered":"2023-09-23T23:28:38.062098+0000","last_unstale":"2023-09-23T23:28:48.853226+0000","last_undegraded":"2023-09-23T23:28:48.853226+0000","last_fullsized":"2023-09-23T23:28:48.853226+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":536,"log_dups_size":0,"ondisk_log_size":536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:55.507817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35127296,"num_objects":536,"num_object_clones":0,"num_object_copies":1608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":536,"num_write_kb":34304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":811,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089517+0000","last_change":"2023-09-23T23:26:51.563868+0000","last_active":"2023-09-23T23:28:47.089517+0000","last_peered":"2023-09-23T23:28:47.089517+0000","last_clean":"2023-09-23T23:28:47.089517+0000","last_became_active":"2023-09-23T23:26:51.563510+0000","last_became_peered":"2023-09-23T23:26:51.563510+0000","last_unstale":"2023-09-23T23:28:47.089517+0000","last_undegraded":"2023-09-23T23:28:47.089517+0000","last_fullsized":"2023-09-23T23:28:47.089517+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:34.244822+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":87,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.090041+0000","last_change":"2023-09-23T23:26:48.673221+0000","last_active":"2023-09-23T23:28:47.090041+0000","last_peered":"2023-09-23T23:28:47.090041+0000","last_clean":"2023-09-23T23:28:47.090041+0000","last_became_active":"2023-09-23T23:26:48.555920+0000","last_became_peered":"2023-09-23T23:26:48.555920+0000","last_unstale":"2023-09-23T23:28:47.090041+0000","last_undegraded":"2023-09-23T23:28:47.090041+0000","last_fullsized":"2023-09-23T23:28:47.090041+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:26.586824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":105,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.098061+0000","last_change":"2023-09-23T23:26:48.525190+0000","last_active":"2023-09-23T23:28:47.098061+0000","last_peered":"2023-09-23T23:28:47.098061+0000","last_clean":"2023-09-23T23:28:47.098061+0000","last_became_active":"2023-09-23T23:26:48.524874+0000","last_became_peered":"2023-09-23T23:26:48.524874+0000","last_unstale":"2023-09-23T23:28:47.098061+0000","last_undegraded":"2023-09-23T23:28:47.098061+0000","last_fullsized":"2023-09-23T23:28:47.098061+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:43.114769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"484'515","reported_seq":532,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:48.813308+0000","last_change":"2023-09-23T23:28:38.062602+0000","last_active":"2023-09-23T23:28:48.813308+0000","last_peered":"2023-09-23T23:28:48.813308+0000","last_clean":"2023-09-23T23:28:48.813308+0000","last_became_active":"2023-09-23T23:28:38.062327+0000","last_became_peered":"2023-09-23T23:28:38.062327+0000","last_unstale":"2023-09-23T23:28:48.813308+0000","last_undegraded":"2023-09-23T23:28:48.813308+0000","last_fullsized":"2023-09-23T23:28:48.813308+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":515,"log_dups_size":0,"ondisk_log_size":515,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:30.017731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33751040,"num_objects":515,"num_object_clones":0,"num_object_copies":1545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":515,"num_write_kb":32960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":871,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089417+0000","last_change":"2023-09-23T23:26:14.530414+0000","last_active":"2023-09-23T23:28:47.089417+0000","last_peered":"2023-09-23T23:28:47.089417+0000","last_clean":"2023-09-23T23:28:47.089417+0000","last_became_active":"2023-09-23T23:26:14.530001+0000","last_became_peered":"2023-09-23T23:26:14.530001+0000","last_unstale":"2023-09-23T23:28:47.089417+0000","last_undegraded":"2023-09-23T23:28:47.089417+0000","last_fullsized":"2023-09-23T23:28:47.089417+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:37.443998+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":87,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.090056+0000","last_change":"2023-09-23T23:26:48.547146+0000","last_active":"2023-09-23T23:28:47.090056+0000","last_peered":"2023-09-23T23:28:47.090056+0000","last_clean":"2023-09-23T23:28:47.090056+0000","last_became_active":"2023-09-23T23:26:48.537682+0000","last_became_peered":"2023-09-23T23:26:48.537682+0000","last_unstale":"2023-09-23T23:28:47.090056+0000","last_undegraded":"2023-09-23T23:28:47.090056+0000","last_fullsized":"2023-09-23T23:28:47.090056+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:10.115563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":509,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.094211+0000","last_change":"2023-09-23T23:26:01.833889+0000","last_active":"2023-09-23T23:28:47.094211+0000","last_peered":"2023-09-23T23:28:47.094211+0000","last_clean":"2023-09-23T23:28:47.094211+0000","last_became_active":"2023-09-23T23:26:01.833626+0000","last_became_peered":"2023-09-23T23:26:01.833626+0000","last_unstale":"2023-09-23T23:28:47.094211+0000","last_undegraded":"2023-09-23T23:28:47.094211+0000","last_fullsized":"2023-09-23T23:28:47.094211+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:25.659671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"484'530","reported_seq":546,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:48.846774+0000","last_change":"2023-09-23T23:28:38.061823+0000","last_active":"2023-09-23T23:28:48.846774+0000","last_peered":"2023-09-23T23:28:48.846774+0000","last_clean":"2023-09-23T23:28:48.846774+0000","last_became_active":"2023-09-23T23:28:38.061549+0000","last_became_peered":"2023-09-23T23:28:38.061549+0000","last_unstale":"2023-09-23T23:28:48.846774+0000","last_undegraded":"2023-09-23T23:28:48.846774+0000","last_fullsized":"2023-09-23T23:28:48.846774+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":530,"log_dups_size":0,"ondisk_log_size":530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:45.310904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34734080,"num_objects":530,"num_object_clones":0,"num_object_copies":1590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":530,"num_write_kb":33920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":752,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:48.121260+0000","last_change":"2023-09-23T23:28:48.116156+0000","last_active":"2023-09-23T23:28:48.121260+0000","last_peered":"2023-09-23T23:28:48.121260+0000","last_clean":"2023-09-23T23:28:48.121260+0000","last_became_active":"2023-09-23T23:28:48.115821+0000","last_became_peered":"2023-09-23T23:28:48.115821+0000","last_unstale":"2023-09-23T23:28:48.121260+0000","last_undegraded":"2023-09-23T23:28:48.121260+0000","last_fullsized":"2023-09-23T23:28:48.121260+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:10.347099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:10.347099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051741199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":390,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089621+0000","last_change":"2023-09-23T23:26:32.043926+0000","last_active":"2023-09-23T23:28:47.089621+0000","last_peered":"2023-09-23T23:28:47.089621+0000","last_clean":"2023-09-23T23:28:47.089621+0000","last_became_active":"2023-09-23T23:26:32.039282+0000","last_became_peered":"2023-09-23T23:26:32.039282+0000","last_unstale":"2023-09-23T23:28:47.089621+0000","last_undegraded":"2023-09-23T23:28:47.089621+0000","last_fullsized":"2023-09-23T23:28:47.089621+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:35.345086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":473,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.091331+0000","last_change":"2023-09-23T23:26:48.536996+0000","last_active":"2023-09-23T23:28:47.091331+0000","last_peered":"2023-09-23T23:28:47.091331+0000","last_clean":"2023-09-23T23:28:47.091331+0000","last_became_active":"2023-09-23T23:26:48.536638+0000","last_became_peered":"2023-09-23T23:26:48.536638+0000","last_unstale":"2023-09-23T23:28:47.091331+0000","last_undegraded":"2023-09-23T23:28:47.091331+0000","last_fullsized":"2023-09-23T23:28:47.091331+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:36.943318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.3","version":"484'1132","reported_seq":1148,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:53.132511+0000","last_change":"2023-09-23T23:28:38.104163+0000","last_active":"2023-09-23T23:28:53.132511+0000","last_peered":"2023-09-23T23:28:53.132511+0000","last_clean":"2023-09-23T23:28:53.132511+0000","last_became_active":"2023-09-23T23:28:38.103810+0000","last_became_peered":"2023-09-23T23:28:38.103810+0000","last_unstale":"2023-09-23T23:28:53.132511+0000","last_undegraded":"2023-09-23T23:28:53.132511+0000","last_fullsized":"2023-09-23T23:28:53.132511+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":1132,"log_dups_size":0,"ondisk_log_size":1132,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:41.275532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74186752,"num_objects":1132,"num_object_clones":0,"num_object_copies":3396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1132,"num_write_kb":72448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":548,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.096585+0000","last_change":"2023-09-23T23:28:03.629593+0000","last_active":"2023-09-23T23:28:47.096585+0000","last_peered":"2023-09-23T23:28:47.096585+0000","last_clean":"2023-09-23T23:28:47.096585+0000","last_became_active":"2023-09-23T23:26:51.588385+0000","last_became_peered":"2023-09-23T23:26:51.588385+0000","last_unstale":"2023-09-23T23:28:47.096585+0000","last_undegraded":"2023-09-23T23:28:47.096585+0000","last_fullsized":"2023-09-23T23:28:47.096585+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:20.818366+0000","scrub_duration":0.0050135120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":312,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089671+0000","last_change":"2023-09-23T23:25:31.732607+0000","last_active":"2023-09-23T23:28:47.089671+0000","last_peered":"2023-09-23T23:28:47.089671+0000","last_clean":"2023-09-23T23:28:47.089671+0000","last_became_active":"2023-09-23T23:25:31.732099+0000","last_became_peered":"2023-09-23T23:25:31.732099+0000","last_unstale":"2023-09-23T23:28:47.089671+0000","last_undegraded":"2023-09-23T23:28:47.089671+0000","last_fullsized":"2023-09-23T23:28:47.089671+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:23.639936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":468,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.086000+0000","last_change":"2023-09-23T23:28:45.066443+0000","last_active":"2023-09-23T23:28:47.086000+0000","last_peered":"2023-09-23T23:28:47.086000+0000","last_clean":"2023-09-23T23:28:47.086000+0000","last_became_active":"2023-09-23T23:28:45.066154+0000","last_became_peered":"2023-09-23T23:28:45.066154+0000","last_unstale":"2023-09-23T23:28:47.086000+0000","last_undegraded":"2023-09-23T23:28:47.086000+0000","last_fullsized":"2023-09-23T23:28:47.086000+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:51.651425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"484'521","reported_seq":538,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:48.803366+0000","last_change":"2023-09-23T23:28:38.061949+0000","last_active":"2023-09-23T23:28:48.803366+0000","last_peered":"2023-09-23T23:28:48.803366+0000","last_clean":"2023-09-23T23:28:48.803366+0000","last_became_active":"2023-09-23T23:28:38.061632+0000","last_became_peered":"2023-09-23T23:28:38.061632+0000","last_unstale":"2023-09-23T23:28:48.803366+0000","last_undegraded":"2023-09-23T23:28:48.803366+0000","last_fullsized":"2023-09-23T23:28:48.803366+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":521,"log_dups_size":0,"ondisk_log_size":521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:53.029897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34144256,"num_objects":521,"num_object_clones":0,"num_object_copies":1563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":521,"num_write_kb":33344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":819,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089092+0000","last_change":"2023-09-23T23:23:45.830384+0000","last_active":"2023-09-23T23:28:47.089092+0000","last_peered":"2023-09-23T23:28:47.089092+0000","last_clean":"2023-09-23T23:28:47.089092+0000","last_became_active":"2023-09-23T23:23:36.025724+0000","last_became_peered":"2023-09-23T23:23:36.025724+0000","last_unstale":"2023-09-23T23:28:47.089092+0000","last_undegraded":"2023-09-23T23:28:47.089092+0000","last_fullsized":"2023-09-23T23:28:47.089092+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:34.345528+0000","scrub_duration":0.001659259,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":451,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089257+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:28:47.089257+0000","last_peered":"2023-09-23T23:28:47.089257+0000","last_clean":"2023-09-23T23:28:47.089257+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:28:47.089257+0000","last_undegraded":"2023-09-23T23:28:47.089257+0000","last_fullsized":"2023-09-23T23:28:47.089257+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":452,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.096544+0000","last_change":"2023-09-23T23:25:31.731172+0000","last_active":"2023-09-23T23:28:47.096544+0000","last_peered":"2023-09-23T23:28:47.096544+0000","last_clean":"2023-09-23T23:28:47.096544+0000","last_became_active":"2023-09-23T23:25:31.730903+0000","last_became_peered":"2023-09-23T23:25:31.730903+0000","last_unstale":"2023-09-23T23:28:47.096544+0000","last_undegraded":"2023-09-23T23:28:47.096544+0000","last_fullsized":"2023-09-23T23:28:47.096544+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:35.442945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"484'872","reported_seq":888,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:50.972676+0000","last_change":"2023-09-23T23:28:38.032946+0000","last_active":"2023-09-23T23:28:50.972676+0000","last_peered":"2023-09-23T23:28:50.972676+0000","last_clean":"2023-09-23T23:28:50.972676+0000","last_became_active":"2023-09-23T23:28:38.032671+0000","last_became_peered":"2023-09-23T23:28:38.032671+0000","last_unstale":"2023-09-23T23:28:50.972676+0000","last_undegraded":"2023-09-23T23:28:50.972676+0000","last_fullsized":"2023-09-23T23:28:50.972676+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":872,"log_dups_size":0,"ondisk_log_size":872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:11:49.143996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57147392,"num_objects":872,"num_object_clones":0,"num_object_copies":2616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":872,"num_write_kb":55808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":763,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.087833+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:28:47.087833+0000","last_peered":"2023-09-23T23:28:47.087833+0000","last_clean":"2023-09-23T23:28:47.087833+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:28:47.087833+0000","last_undegraded":"2023-09-23T23:28:47.087833+0000","last_fullsized":"2023-09-23T23:28:47.087833+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":310,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.094658+0000","last_change":"2023-09-23T23:27:20.560074+0000","last_active":"2023-09-23T23:28:47.094658+0000","last_peered":"2023-09-23T23:28:47.094658+0000","last_clean":"2023-09-23T23:28:47.094658+0000","last_became_active":"2023-09-23T23:27:20.559764+0000","last_became_peered":"2023-09-23T23:27:20.559764+0000","last_unstale":"2023-09-23T23:28:47.094658+0000","last_undegraded":"2023-09-23T23:28:47.094658+0000","last_fullsized":"2023-09-23T23:28:47.094658+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:23.675227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":105,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.096566+0000","last_change":"2023-09-23T23:26:11.516475+0000","last_active":"2023-09-23T23:28:47.096566+0000","last_peered":"2023-09-23T23:28:47.096566+0000","last_clean":"2023-09-23T23:28:47.096566+0000","last_became_active":"2023-09-23T23:26:11.516198+0000","last_became_peered":"2023-09-23T23:26:11.516198+0000","last_unstale":"2023-09-23T23:28:47.096566+0000","last_undegraded":"2023-09-23T23:28:47.096566+0000","last_fullsized":"2023-09-23T23:28:47.096566+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:56.875103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"484'668","reported_seq":685,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:49.782861+0000","last_change":"2023-09-23T23:28:38.105484+0000","last_active":"2023-09-23T23:28:49.782861+0000","last_peered":"2023-09-23T23:28:49.782861+0000","last_clean":"2023-09-23T23:28:49.782861+0000","last_became_active":"2023-09-23T23:28:38.105230+0000","last_became_peered":"2023-09-23T23:28:38.105230+0000","last_unstale":"2023-09-23T23:28:49.782861+0000","last_undegraded":"2023-09-23T23:28:49.782861+0000","last_fullsized":"2023-09-23T23:28:49.782861+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":668,"log_dups_size":0,"ondisk_log_size":668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:40.364698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43778048,"num_objects":668,"num_object_clones":0,"num_object_copies":2004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":668,"num_write_kb":42752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":639,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.091516+0000","last_change":"2023-09-23T23:26:34.548211+0000","last_active":"2023-09-23T23:28:47.091516+0000","last_peered":"2023-09-23T23:28:47.091516+0000","last_clean":"2023-09-23T23:28:47.091516+0000","last_became_active":"2023-09-23T23:26:34.547815+0000","last_became_peered":"2023-09-23T23:26:34.547815+0000","last_unstale":"2023-09-23T23:28:47.091516+0000","last_undegraded":"2023-09-23T23:28:47.091516+0000","last_fullsized":"2023-09-23T23:28:47.091516+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:37.675417+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":105,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.093999+0000","last_change":"2023-09-23T23:26:11.512815+0000","last_active":"2023-09-23T23:28:47.093999+0000","last_peered":"2023-09-23T23:28:47.093999+0000","last_clean":"2023-09-23T23:28:47.093999+0000","last_became_active":"2023-09-23T23:26:11.512466+0000","last_became_peered":"2023-09-23T23:26:11.512466+0000","last_unstale":"2023-09-23T23:28:47.093999+0000","last_undegraded":"2023-09-23T23:28:47.093999+0000","last_fullsized":"2023-09-23T23:28:47.093999+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:10.324158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":482,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.097274+0000","last_change":"2023-09-23T23:27:20.558731+0000","last_active":"2023-09-23T23:28:47.097274+0000","last_peered":"2023-09-23T23:28:47.097274+0000","last_clean":"2023-09-23T23:28:47.097274+0000","last_became_active":"2023-09-23T23:27:20.558394+0000","last_became_peered":"2023-09-23T23:27:20.558394+0000","last_unstale":"2023-09-23T23:28:47.097274+0000","last_undegraded":"2023-09-23T23:28:47.097274+0000","last_fullsized":"2023-09-23T23:28:47.097274+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:25:00.408551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"484'643","reported_seq":660,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:49.778924+0000","last_change":"2023-09-23T23:28:38.062569+0000","last_active":"2023-09-23T23:28:49.778924+0000","last_peered":"2023-09-23T23:28:49.778924+0000","last_clean":"2023-09-23T23:28:49.778924+0000","last_became_active":"2023-09-23T23:28:38.062269+0000","last_became_peered":"2023-09-23T23:28:38.062269+0000","last_unstale":"2023-09-23T23:28:49.778924+0000","last_undegraded":"2023-09-23T23:28:49.778924+0000","last_fullsized":"2023-09-23T23:28:49.778924+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":643,"log_dups_size":0,"ondisk_log_size":643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:41.750221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42139648,"num_objects":643,"num_object_clones":0,"num_object_copies":1929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":643,"num_write_kb":41152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":805,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.092016+0000","last_change":"2023-09-23T23:25:34.500038+0000","last_active":"2023-09-23T23:28:47.092016+0000","last_peered":"2023-09-23T23:28:47.092016+0000","last_clean":"2023-09-23T23:28:47.092016+0000","last_became_active":"2023-09-23T23:25:34.499705+0000","last_became_peered":"2023-09-23T23:25:34.499705+0000","last_unstale":"2023-09-23T23:28:47.092016+0000","last_undegraded":"2023-09-23T23:28:47.092016+0000","last_fullsized":"2023-09-23T23:28:47.092016+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:43.096696+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":122,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.088514+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:28:47.088514+0000","last_peered":"2023-09-23T23:28:47.088514+0000","last_clean":"2023-09-23T23:28:47.088514+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:28:47.088514+0000","last_undegraded":"2023-09-23T23:28:47.088514+0000","last_fullsized":"2023-09-23T23:28:47.088514+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":405,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.093439+0000","last_change":"2023-09-23T23:22:39.489325+0000","last_active":"2023-09-23T23:28:47.093439+0000","last_peered":"2023-09-23T23:28:47.093439+0000","last_clean":"2023-09-23T23:28:47.093439+0000","last_became_active":"2023-09-23T23:22:39.489048+0000","last_became_peered":"2023-09-23T23:22:39.489048+0000","last_unstale":"2023-09-23T23:28:47.093439+0000","last_undegraded":"2023-09-23T23:28:47.093439+0000","last_fullsized":"2023-09-23T23:28:47.093439+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:29.984948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"484'877","reported_seq":894,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:50.926573+0000","last_change":"2023-09-23T23:28:38.107589+0000","last_active":"2023-09-23T23:28:50.926573+0000","last_peered":"2023-09-23T23:28:50.926573+0000","last_clean":"2023-09-23T23:28:50.926573+0000","last_became_active":"2023-09-23T23:28:38.107296+0000","last_became_peered":"2023-09-23T23:28:38.107296+0000","last_unstale":"2023-09-23T23:28:50.926573+0000","last_undegraded":"2023-09-23T23:28:50.926573+0000","last_fullsized":"2023-09-23T23:28:50.926573+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":877,"log_dups_size":0,"ondisk_log_size":877,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:12.352223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57475072,"num_objects":877,"num_object_clones":0,"num_object_copies":2631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":877,"num_write_kb":56128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":795,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.088898+0000","last_change":"2023-09-23T23:28:11.839674+0000","last_active":"2023-09-23T23:28:47.088898+0000","last_peered":"2023-09-23T23:28:47.088898+0000","last_clean":"2023-09-23T23:28:47.088898+0000","last_became_active":"2023-09-23T23:26:51.561394+0000","last_became_peered":"2023-09-23T23:26:51.561394+0000","last_unstale":"2023-09-23T23:28:47.088898+0000","last_undegraded":"2023-09-23T23:28:47.088898+0000","last_fullsized":"2023-09-23T23:28:47.088898+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:41.710212+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":185,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.092140+0000","last_change":"2023-09-23T23:23:36.048889+0000","last_active":"2023-09-23T23:28:47.092140+0000","last_peered":"2023-09-23T23:28:47.092140+0000","last_clean":"2023-09-23T23:28:47.092140+0000","last_became_active":"2023-09-23T23:23:36.048572+0000","last_became_peered":"2023-09-23T23:23:36.048572+0000","last_unstale":"2023-09-23T23:28:47.092140+0000","last_undegraded":"2023-09-23T23:28:47.092140+0000","last_fullsized":"2023-09-23T23:28:47.092140+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:55.581146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":499,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.099084+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:28:47.099084+0000","last_peered":"2023-09-23T23:28:47.099084+0000","last_clean":"2023-09-23T23:28:47.099084+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:28:47.099084+0000","last_undegraded":"2023-09-23T23:28:47.099084+0000","last_fullsized":"2023-09-23T23:28:47.099084+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":70,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.091247+0000","last_change":"2023-09-23T23:27:20.549904+0000","last_active":"2023-09-23T23:28:47.091247+0000","last_peered":"2023-09-23T23:28:47.091247+0000","last_clean":"2023-09-23T23:28:47.091247+0000","last_became_active":"2023-09-23T23:27:20.549645+0000","last_became_peered":"2023-09-23T23:27:20.549645+0000","last_unstale":"2023-09-23T23:28:47.091247+0000","last_undegraded":"2023-09-23T23:28:47.091247+0000","last_fullsized":"2023-09-23T23:28:47.091247+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:09.850640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"484'862","reported_seq":879,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:50.922524+0000","last_change":"2023-09-23T23:28:38.107697+0000","last_active":"2023-09-23T23:28:50.922524+0000","last_peered":"2023-09-23T23:28:50.922524+0000","last_clean":"2023-09-23T23:28:50.922524+0000","last_became_active":"2023-09-23T23:28:38.107379+0000","last_became_peered":"2023-09-23T23:28:38.107379+0000","last_unstale":"2023-09-23T23:28:50.922524+0000","last_undegraded":"2023-09-23T23:28:50.922524+0000","last_fullsized":"2023-09-23T23:28:50.922524+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":862,"log_dups_size":0,"ondisk_log_size":862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:12.478832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56492032,"num_objects":862,"num_object_clones":0,"num_object_copies":2586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":862,"num_write_kb":55168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":630,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.085448+0000","last_change":"2023-09-23T23:23:49.320969+0000","last_active":"2023-09-23T23:28:47.085448+0000","last_peered":"2023-09-23T23:28:47.085448+0000","last_clean":"2023-09-23T23:28:47.085448+0000","last_became_active":"2023-09-23T23:22:17.001911+0000","last_became_peered":"2023-09-23T23:22:17.001911+0000","last_unstale":"2023-09-23T23:28:47.085448+0000","last_undegraded":"2023-09-23T23:28:47.085448+0000","last_fullsized":"2023-09-23T23:28:47.085448+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.320823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.320823+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:07.000984+0000","scrub_duration":0.0018140630000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":185,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.091397+0000","last_change":"2023-09-23T23:25:31.731396+0000","last_active":"2023-09-23T23:28:47.091397+0000","last_peered":"2023-09-23T23:28:47.091397+0000","last_clean":"2023-09-23T23:28:47.091397+0000","last_became_active":"2023-09-23T23:25:31.730931+0000","last_became_peered":"2023-09-23T23:25:31.730931+0000","last_unstale":"2023-09-23T23:28:47.091397+0000","last_undegraded":"2023-09-23T23:28:47.091397+0000","last_fullsized":"2023-09-23T23:28:47.091397+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:48.002628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":463,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.096806+0000","last_change":"2023-09-23T23:27:20.556916+0000","last_active":"2023-09-23T23:28:47.096806+0000","last_peered":"2023-09-23T23:28:47.096806+0000","last_clean":"2023-09-23T23:28:47.096806+0000","last_became_active":"2023-09-23T23:27:20.556651+0000","last_became_peered":"2023-09-23T23:27:20.556651+0000","last_unstale":"2023-09-23T23:28:47.096806+0000","last_undegraded":"2023-09-23T23:28:47.096806+0000","last_fullsized":"2023-09-23T23:28:47.096806+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:06.012543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":70,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.091784+0000","last_change":"2023-09-23T23:27:20.551221+0000","last_active":"2023-09-23T23:28:47.091784+0000","last_peered":"2023-09-23T23:28:47.091784+0000","last_clean":"2023-09-23T23:28:47.091784+0000","last_became_active":"2023-09-23T23:27:20.550900+0000","last_became_peered":"2023-09-23T23:27:20.550900+0000","last_unstale":"2023-09-23T23:28:47.091784+0000","last_undegraded":"2023-09-23T23:28:47.091784+0000","last_fullsized":"2023-09-23T23:28:47.091784+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:23.012149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"484'554","reported_seq":570,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:48.850192+0000","last_change":"2023-09-23T23:28:38.032025+0000","last_active":"2023-09-23T23:28:48.850192+0000","last_peered":"2023-09-23T23:28:48.850192+0000","last_clean":"2023-09-23T23:28:48.850192+0000","last_became_active":"2023-09-23T23:28:38.031791+0000","last_became_peered":"2023-09-23T23:28:38.031791+0000","last_unstale":"2023-09-23T23:28:48.850192+0000","last_undegraded":"2023-09-23T23:28:48.850192+0000","last_fullsized":"2023-09-23T23:28:48.850192+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":554,"log_dups_size":0,"ondisk_log_size":554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:44.488888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36306944,"num_objects":554,"num_object_clones":0,"num_object_copies":1662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":554,"num_write_kb":35456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":784,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:48.114777+0000","last_change":"2023-09-23T23:28:48.113032+0000","last_active":"2023-09-23T23:28:48.114777+0000","last_peered":"2023-09-23T23:28:48.114777+0000","last_clean":"2023-09-23T23:28:48.114777+0000","last_became_active":"2023-09-23T23:28:48.112666+0000","last_became_peered":"2023-09-23T23:28:48.112666+0000","last_unstale":"2023-09-23T23:28:48.114777+0000","last_undegraded":"2023-09-23T23:28:48.114777+0000","last_fullsized":"2023-09-23T23:28:48.114777+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:59:59.838039+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":219,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.091974+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:28:47.091974+0000","last_peered":"2023-09-23T23:28:47.091974+0000","last_clean":"2023-09-23T23:28:47.091974+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:28:47.091974+0000","last_undegraded":"2023-09-23T23:28:47.091974+0000","last_fullsized":"2023-09-23T23:28:47.091974+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":186,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.091234+0000","last_change":"2023-09-23T23:25:31.828908+0000","last_active":"2023-09-23T23:28:47.091234+0000","last_peered":"2023-09-23T23:28:47.091234+0000","last_clean":"2023-09-23T23:28:47.091234+0000","last_became_active":"2023-09-23T23:25:31.828582+0000","last_became_peered":"2023-09-23T23:25:31.828582+0000","last_unstale":"2023-09-23T23:28:47.091234+0000","last_undegraded":"2023-09-23T23:28:47.091234+0000","last_fullsized":"2023-09-23T23:28:47.091234+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:01.466593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":70,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.093647+0000","last_change":"2023-09-23T23:27:20.551574+0000","last_active":"2023-09-23T23:28:47.093647+0000","last_peered":"2023-09-23T23:28:47.093647+0000","last_clean":"2023-09-23T23:28:47.093647+0000","last_became_active":"2023-09-23T23:27:20.550820+0000","last_became_peered":"2023-09-23T23:27:20.550820+0000","last_unstale":"2023-09-23T23:28:47.093647+0000","last_undegraded":"2023-09-23T23:28:47.093647+0000","last_fullsized":"2023-09-23T23:28:47.093647+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:09.274552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"484'1093","reported_seq":1109,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:53.144500+0000","last_change":"2023-09-23T23:28:38.065394+0000","last_active":"2023-09-23T23:28:53.144500+0000","last_peered":"2023-09-23T23:28:53.144500+0000","last_clean":"2023-09-23T23:28:53.144500+0000","last_became_active":"2023-09-23T23:28:38.065160+0000","last_became_peered":"2023-09-23T23:28:38.065160+0000","last_unstale":"2023-09-23T23:28:53.144500+0000","last_undegraded":"2023-09-23T23:28:53.144500+0000","last_fullsized":"2023-09-23T23:28:53.144500+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":1093,"log_dups_size":0,"ondisk_log_size":1093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:10:39.685945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71630848,"num_objects":1093,"num_object_clones":0,"num_object_copies":3279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1093,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1093,"num_write_kb":69952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":495,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.092633+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:28:47.092633+0000","last_peered":"2023-09-23T23:28:47.092633+0000","last_clean":"2023-09-23T23:28:47.092633+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:28:47.092633+0000","last_undegraded":"2023-09-23T23:28:47.092633+0000","last_fullsized":"2023-09-23T23:28:47.092633+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":804,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089008+0000","last_change":"2023-09-23T23:26:51.564817+0000","last_active":"2023-09-23T23:28:47.089008+0000","last_peered":"2023-09-23T23:28:47.089008+0000","last_clean":"2023-09-23T23:28:47.089008+0000","last_became_active":"2023-09-23T23:26:51.564511+0000","last_became_peered":"2023-09-23T23:26:51.564511+0000","last_unstale":"2023-09-23T23:28:47.089008+0000","last_undegraded":"2023-09-23T23:28:47.089008+0000","last_fullsized":"2023-09-23T23:28:47.089008+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:02.415212+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":433,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.088999+0000","last_change":"2023-09-23T23:26:32.037619+0000","last_active":"2023-09-23T23:28:47.088999+0000","last_peered":"2023-09-23T23:28:47.088999+0000","last_clean":"2023-09-23T23:28:47.088999+0000","last_became_active":"2023-09-23T23:26:32.037045+0000","last_became_peered":"2023-09-23T23:26:32.037045+0000","last_unstale":"2023-09-23T23:28:47.088999+0000","last_undegraded":"2023-09-23T23:28:47.088999+0000","last_fullsized":"2023-09-23T23:28:47.088999+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:47.657527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":105,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.096651+0000","last_change":"2023-09-23T23:26:11.525845+0000","last_active":"2023-09-23T23:28:47.096651+0000","last_peered":"2023-09-23T23:28:47.096651+0000","last_clean":"2023-09-23T23:28:47.096651+0000","last_became_active":"2023-09-23T23:26:11.525507+0000","last_became_peered":"2023-09-23T23:26:11.525507+0000","last_unstale":"2023-09-23T23:28:47.096651+0000","last_undegraded":"2023-09-23T23:28:47.096651+0000","last_fullsized":"2023-09-23T23:28:47.096651+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:35.462874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.e","version":"484'556","reported_seq":572,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:48.850862+0000","last_change":"2023-09-23T23:28:38.104180+0000","last_active":"2023-09-23T23:28:48.850862+0000","last_peered":"2023-09-23T23:28:48.850862+0000","last_clean":"2023-09-23T23:28:48.850862+0000","last_became_active":"2023-09-23T23:28:38.103844+0000","last_became_peered":"2023-09-23T23:28:38.103844+0000","last_unstale":"2023-09-23T23:28:48.850862+0000","last_undegraded":"2023-09-23T23:28:48.850862+0000","last_fullsized":"2023-09-23T23:28:48.850862+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":556,"log_dups_size":0,"ondisk_log_size":556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:48.427805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36438016,"num_objects":556,"num_object_clones":0,"num_object_copies":1668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":556,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":556,"num_write_kb":35584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":647,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.091575+0000","last_change":"2023-09-23T23:26:34.550276+0000","last_active":"2023-09-23T23:28:47.091575+0000","last_peered":"2023-09-23T23:28:47.091575+0000","last_clean":"2023-09-23T23:28:47.091575+0000","last_became_active":"2023-09-23T23:26:34.549923+0000","last_became_peered":"2023-09-23T23:26:34.549923+0000","last_unstale":"2023-09-23T23:28:47.091575+0000","last_undegraded":"2023-09-23T23:28:47.091575+0000","last_fullsized":"2023-09-23T23:28:47.091575+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:22.636211+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":411,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.088800+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:28:47.088800+0000","last_peered":"2023-09-23T23:28:47.088800+0000","last_clean":"2023-09-23T23:28:47.088800+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:28:47.088800+0000","last_undegraded":"2023-09-23T23:28:47.088800+0000","last_fullsized":"2023-09-23T23:28:47.088800+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":410,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.093801+0000","last_change":"2023-09-23T23:28:22.601436+0000","last_active":"2023-09-23T23:28:47.093801+0000","last_peered":"2023-09-23T23:28:47.093801+0000","last_clean":"2023-09-23T23:28:47.093801+0000","last_became_active":"2023-09-23T23:28:22.601154+0000","last_became_peered":"2023-09-23T23:28:22.601154+0000","last_unstale":"2023-09-23T23:28:47.093801+0000","last_undegraded":"2023-09-23T23:28:47.093801+0000","last_fullsized":"2023-09-23T23:28:47.093801+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:03.296778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":514,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.087272+0000","last_change":"2023-09-23T23:22:38.438448+0000","last_active":"2023-09-23T23:28:47.087272+0000","last_peered":"2023-09-23T23:28:47.087272+0000","last_clean":"2023-09-23T23:28:47.087272+0000","last_became_active":"2023-09-23T23:22:38.437495+0000","last_became_peered":"2023-09-23T23:22:38.437495+0000","last_unstale":"2023-09-23T23:28:47.087272+0000","last_undegraded":"2023-09-23T23:28:47.087272+0000","last_fullsized":"2023-09-23T23:28:47.087272+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:49.587056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"484'494","reported_seq":511,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:48.817884+0000","last_change":"2023-09-23T23:28:38.032628+0000","last_active":"2023-09-23T23:28:48.817884+0000","last_peered":"2023-09-23T23:28:48.817884+0000","last_clean":"2023-09-23T23:28:48.817884+0000","last_became_active":"2023-09-23T23:28:38.032274+0000","last_became_peered":"2023-09-23T23:28:38.032274+0000","last_unstale":"2023-09-23T23:28:48.817884+0000","last_undegraded":"2023-09-23T23:28:48.817884+0000","last_fullsized":"2023-09-23T23:28:48.817884+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":494,"log_dups_size":0,"ondisk_log_size":494,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:32:23.382504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32374784,"num_objects":494,"num_object_clones":0,"num_object_copies":1482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":494,"num_write_kb":31616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":525,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.088897+0000","last_change":"2023-09-23T23:28:05.843008+0000","last_active":"2023-09-23T23:28:47.088897+0000","last_peered":"2023-09-23T23:28:47.088897+0000","last_clean":"2023-09-23T23:28:47.088897+0000","last_became_active":"2023-09-23T23:26:04.560484+0000","last_became_peered":"2023-09-23T23:26:04.560484+0000","last_unstale":"2023-09-23T23:28:47.088897+0000","last_undegraded":"2023-09-23T23:28:47.088897+0000","last_fullsized":"2023-09-23T23:28:47.088897+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.842881+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.842881+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:37.322347+0000","scrub_duration":0.0047245100000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":335,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089567+0000","last_change":"2023-09-23T23:23:35.037351+0000","last_active":"2023-09-23T23:28:47.089567+0000","last_peered":"2023-09-23T23:28:47.089567+0000","last_clean":"2023-09-23T23:28:47.089567+0000","last_became_active":"2023-09-23T23:23:35.036841+0000","last_became_peered":"2023-09-23T23:23:35.036841+0000","last_unstale":"2023-09-23T23:28:47.089567+0000","last_undegraded":"2023-09-23T23:28:47.089567+0000","last_fullsized":"2023-09-23T23:28:47.089567+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:20.618292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":185,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.092216+0000","last_change":"2023-09-23T23:26:32.037099+0000","last_active":"2023-09-23T23:28:47.092216+0000","last_peered":"2023-09-23T23:28:47.092216+0000","last_clean":"2023-09-23T23:28:47.092216+0000","last_became_active":"2023-09-23T23:26:32.036681+0000","last_became_peered":"2023-09-23T23:26:32.036681+0000","last_unstale":"2023-09-23T23:28:47.092216+0000","last_undegraded":"2023-09-23T23:28:47.092216+0000","last_fullsized":"2023-09-23T23:28:47.092216+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:48.926576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":217,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.098897+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:28:47.098897+0000","last_peered":"2023-09-23T23:28:47.098897+0000","last_clean":"2023-09-23T23:28:47.098897+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:28:47.098897+0000","last_undegraded":"2023-09-23T23:28:47.098897+0000","last_fullsized":"2023-09-23T23:28:47.098897+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"484'510","reported_seq":527,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:48.801244+0000","last_change":"2023-09-23T23:28:38.104037+0000","last_active":"2023-09-23T23:28:48.801244+0000","last_peered":"2023-09-23T23:28:48.801244+0000","last_clean":"2023-09-23T23:28:48.801244+0000","last_became_active":"2023-09-23T23:28:38.103739+0000","last_became_peered":"2023-09-23T23:28:38.103739+0000","last_unstale":"2023-09-23T23:28:48.801244+0000","last_undegraded":"2023-09-23T23:28:48.801244+0000","last_fullsized":"2023-09-23T23:28:48.801244+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":510,"log_dups_size":0,"ondisk_log_size":510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:02:06.842073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33423360,"num_objects":510,"num_object_clones":0,"num_object_copies":1530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":510,"num_write_kb":32640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":795,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.096738+0000","last_change":"2023-09-23T23:26:14.518921+0000","last_active":"2023-09-23T23:28:47.096738+0000","last_peered":"2023-09-23T23:28:47.096738+0000","last_clean":"2023-09-23T23:28:47.096738+0000","last_became_active":"2023-09-23T23:26:14.518558+0000","last_became_peered":"2023-09-23T23:26:14.518558+0000","last_unstale":"2023-09-23T23:28:47.096738+0000","last_undegraded":"2023-09-23T23:28:47.096738+0000","last_fullsized":"2023-09-23T23:28:47.096738+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:14.360597+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":478,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089870+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:28:47.089870+0000","last_peered":"2023-09-23T23:28:47.089870+0000","last_clean":"2023-09-23T23:28:47.089870+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:28:47.089870+0000","last_undegraded":"2023-09-23T23:28:47.089870+0000","last_fullsized":"2023-09-23T23:28:47.089870+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":70,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.093193+0000","last_change":"2023-09-23T23:27:20.562744+0000","last_active":"2023-09-23T23:28:47.093193+0000","last_peered":"2023-09-23T23:28:47.093193+0000","last_clean":"2023-09-23T23:28:47.093193+0000","last_became_active":"2023-09-23T23:27:20.562383+0000","last_became_peered":"2023-09-23T23:27:20.562383+0000","last_unstale":"2023-09-23T23:28:47.093193+0000","last_undegraded":"2023-09-23T23:28:47.093193+0000","last_fullsized":"2023-09-23T23:28:47.093193+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:01.085133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.087726+0000","last_change":"2023-09-23T23:28:45.061717+0000","last_active":"2023-09-23T23:28:47.087726+0000","last_peered":"2023-09-23T23:28:47.087726+0000","last_clean":"2023-09-23T23:28:47.087726+0000","last_became_active":"2023-09-23T23:28:45.061404+0000","last_became_peered":"2023-09-23T23:28:45.061404+0000","last_unstale":"2023-09-23T23:28:47.087726+0000","last_undegraded":"2023-09-23T23:28:47.087726+0000","last_fullsized":"2023-09-23T23:28:47.087726+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:46.614114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.9","version":"484'822","reported_seq":839,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:50.951976+0000","last_change":"2023-09-23T23:28:38.034022+0000","last_active":"2023-09-23T23:28:50.951976+0000","last_peered":"2023-09-23T23:28:50.951976+0000","last_clean":"2023-09-23T23:28:50.951976+0000","last_became_active":"2023-09-23T23:28:38.033769+0000","last_became_peered":"2023-09-23T23:28:38.033769+0000","last_unstale":"2023-09-23T23:28:50.951976+0000","last_undegraded":"2023-09-23T23:28:50.951976+0000","last_fullsized":"2023-09-23T23:28:50.951976+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":822,"log_dups_size":0,"ondisk_log_size":822,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:23.894936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53870592,"num_objects":822,"num_object_clones":0,"num_object_copies":2466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":822,"num_write_kb":52608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":867,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089821+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:28:47.089821+0000","last_peered":"2023-09-23T23:28:47.089821+0000","last_clean":"2023-09-23T23:28:47.089821+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:28:47.089821+0000","last_undegraded":"2023-09-23T23:28:47.089821+0000","last_fullsized":"2023-09-23T23:28:47.089821+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":83,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.090057+0000","last_change":"2023-09-23T23:26:48.544465+0000","last_active":"2023-09-23T23:28:47.090057+0000","last_peered":"2023-09-23T23:28:47.090057+0000","last_clean":"2023-09-23T23:28:47.090057+0000","last_became_active":"2023-09-23T23:26:48.544166+0000","last_became_peered":"2023-09-23T23:26:48.544166+0000","last_unstale":"2023-09-23T23:28:47.090057+0000","last_undegraded":"2023-09-23T23:28:47.090057+0000","last_fullsized":"2023-09-23T23:28:47.090057+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:38.844346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":469,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.087195+0000","last_change":"2023-09-23T23:22:38.482705+0000","last_active":"2023-09-23T23:28:47.087195+0000","last_peered":"2023-09-23T23:28:47.087195+0000","last_clean":"2023-09-23T23:28:47.087195+0000","last_became_active":"2023-09-23T23:22:38.438239+0000","last_became_peered":"2023-09-23T23:22:38.438239+0000","last_unstale":"2023-09-23T23:28:47.087195+0000","last_undegraded":"2023-09-23T23:28:47.087195+0000","last_fullsized":"2023-09-23T23:28:47.087195+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:52.006349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":105,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.094081+0000","last_change":"2023-09-23T23:26:11.522604+0000","last_active":"2023-09-23T23:28:47.094081+0000","last_peered":"2023-09-23T23:28:47.094081+0000","last_clean":"2023-09-23T23:28:47.094081+0000","last_became_active":"2023-09-23T23:26:11.522350+0000","last_became_peered":"2023-09-23T23:26:11.522350+0000","last_unstale":"2023-09-23T23:28:47.094081+0000","last_undegraded":"2023-09-23T23:28:47.094081+0000","last_fullsized":"2023-09-23T23:28:47.094081+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:20.164202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":866,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089073+0000","last_change":"2023-09-23T23:25:34.505042+0000","last_active":"2023-09-23T23:28:47.089073+0000","last_peered":"2023-09-23T23:28:47.089073+0000","last_clean":"2023-09-23T23:28:47.089073+0000","last_became_active":"2023-09-23T23:25:34.504687+0000","last_became_peered":"2023-09-23T23:25:34.504687+0000","last_unstale":"2023-09-23T23:28:47.089073+0000","last_undegraded":"2023-09-23T23:28:47.089073+0000","last_fullsized":"2023-09-23T23:28:47.089073+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:06.369262+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":154,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.096282+0000","last_change":"2023-09-23T23:28:22.603176+0000","last_active":"2023-09-23T23:28:47.096282+0000","last_peered":"2023-09-23T23:28:47.096282+0000","last_clean":"2023-09-23T23:28:47.096282+0000","last_became_active":"2023-09-23T23:28:22.602824+0000","last_became_peered":"2023-09-23T23:28:22.602824+0000","last_unstale":"2023-09-23T23:28:47.096282+0000","last_undegraded":"2023-09-23T23:28:47.096282+0000","last_fullsized":"2023-09-23T23:28:47.096282+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:35.934235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":236,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.093262+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:28:47.093262+0000","last_peered":"2023-09-23T23:28:47.093262+0000","last_clean":"2023-09-23T23:28:47.093262+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:28:47.093262+0000","last_undegraded":"2023-09-23T23:28:47.093262+0000","last_fullsized":"2023-09-23T23:28:47.093262+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":611,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089369+0000","last_change":"2023-09-23T23:26:14.522158+0000","last_active":"2023-09-23T23:28:47.089369+0000","last_peered":"2023-09-23T23:28:47.089369+0000","last_clean":"2023-09-23T23:28:47.089369+0000","last_became_active":"2023-09-23T23:26:14.521785+0000","last_became_peered":"2023-09-23T23:26:14.521785+0000","last_unstale":"2023-09-23T23:28:47.089369+0000","last_undegraded":"2023-09-23T23:28:47.089369+0000","last_fullsized":"2023-09-23T23:28:47.089369+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:39.445601+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":225,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.096613+0000","last_change":"2023-09-23T23:22:38.452968+0000","last_active":"2023-09-23T23:28:47.096613+0000","last_peered":"2023-09-23T23:28:47.096613+0000","last_clean":"2023-09-23T23:28:47.096613+0000","last_became_active":"2023-09-23T23:22:38.437986+0000","last_became_peered":"2023-09-23T23:22:38.437986+0000","last_unstale":"2023-09-23T23:28:47.096613+0000","last_undegraded":"2023-09-23T23:28:47.096613+0000","last_fullsized":"2023-09-23T23:28:47.096613+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:10.333271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.094185+0000","last_change":"2023-09-23T23:27:20.558454+0000","last_active":"2023-09-23T23:28:47.094185+0000","last_peered":"2023-09-23T23:28:47.094185+0000","last_clean":"2023-09-23T23:28:47.094185+0000","last_became_active":"2023-09-23T23:27:20.558204+0000","last_became_peered":"2023-09-23T23:27:20.558204+0000","last_unstale":"2023-09-23T23:28:47.094185+0000","last_undegraded":"2023-09-23T23:28:47.094185+0000","last_fullsized":"2023-09-23T23:28:47.094185+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:15.079990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":225,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.096829+0000","last_change":"2023-09-23T23:25:31.740467+0000","last_active":"2023-09-23T23:28:47.096829+0000","last_peered":"2023-09-23T23:28:47.096829+0000","last_clean":"2023-09-23T23:28:47.096829+0000","last_became_active":"2023-09-23T23:25:31.729046+0000","last_became_peered":"2023-09-23T23:25:31.729046+0000","last_unstale":"2023-09-23T23:28:47.096829+0000","last_undegraded":"2023-09-23T23:28:47.096829+0000","last_fullsized":"2023-09-23T23:28:47.096829+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:31.222969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":105,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.094517+0000","last_change":"2023-09-23T23:26:11.527453+0000","last_active":"2023-09-23T23:28:47.094517+0000","last_peered":"2023-09-23T23:28:47.094517+0000","last_clean":"2023-09-23T23:28:47.094517+0000","last_became_active":"2023-09-23T23:26:11.527208+0000","last_became_peered":"2023-09-23T23:26:11.527208+0000","last_unstale":"2023-09-23T23:28:47.094517+0000","last_undegraded":"2023-09-23T23:28:47.094517+0000","last_fullsized":"2023-09-23T23:28:47.094517+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:15.083611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":750,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089556+0000","last_change":"2023-09-23T23:25:34.510468+0000","last_active":"2023-09-23T23:28:47.089556+0000","last_peered":"2023-09-23T23:28:47.089556+0000","last_clean":"2023-09-23T23:28:47.089556+0000","last_became_active":"2023-09-23T23:25:34.510106+0000","last_became_peered":"2023-09-23T23:25:34.510106+0000","last_unstale":"2023-09-23T23:28:47.089556+0000","last_undegraded":"2023-09-23T23:28:47.089556+0000","last_fullsized":"2023-09-23T23:28:47.089556+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:42.157293+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":501,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089285+0000","last_change":"2023-09-23T23:25:31.746383+0000","last_active":"2023-09-23T23:28:47.089285+0000","last_peered":"2023-09-23T23:28:47.089285+0000","last_clean":"2023-09-23T23:28:47.089285+0000","last_became_active":"2023-09-23T23:25:31.745929+0000","last_became_peered":"2023-09-23T23:25:31.745929+0000","last_unstale":"2023-09-23T23:28:47.089285+0000","last_undegraded":"2023-09-23T23:28:47.089285+0000","last_fullsized":"2023-09-23T23:28:47.089285+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:21.696449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":495,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.089409+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:28:47.089409+0000","last_peered":"2023-09-23T23:28:47.089409+0000","last_clean":"2023-09-23T23:28:47.089409+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:28:47.089409+0000","last_undegraded":"2023-09-23T23:28:47.089409+0000","last_fullsized":"2023-09-23T23:28:47.089409+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":858,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:28:48.110681+0000","last_change":"2023-09-23T23:26:14.527390+0000","last_active":"2023-09-23T23:28:48.110681+0000","last_peered":"2023-09-23T23:28:48.110681+0000","last_clean":"2023-09-23T23:28:48.110681+0000","last_became_active":"2023-09-23T23:26:14.527008+0000","last_became_peered":"2023-09-23T23:26:14.527008+0000","last_unstale":"2023-09-23T23:28:48.110681+0000","last_undegraded":"2023-09-23T23:28:48.110681+0000","last_fullsized":"2023-09-23T23:28:48.110681+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:36.460827+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":294,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.093534+0000","last_change":"2023-09-23T23:28:22.600921+0000","last_active":"2023-09-23T23:28:47.093534+0000","last_peered":"2023-09-23T23:28:47.093534+0000","last_clean":"2023-09-23T23:28:47.093534+0000","last_became_active":"2023-09-23T23:28:22.600565+0000","last_became_peered":"2023-09-23T23:28:22.600565+0000","last_unstale":"2023-09-23T23:28:47.093534+0000","last_undegraded":"2023-09-23T23:28:47.093534+0000","last_fullsized":"2023-09-23T23:28:47.093534+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:07.793563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":529,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:28:47.093995+0000","last_change":"2023-09-23T23:25:31.827217+0000","last_active":"2023-09-23T23:28:47.093995+0000","last_peered":"2023-09-23T23:28:47.093995+0000","last_clean":"2023-09-23T23:28:47.093995+0000","last_became_active":"2023-09-23T23:25:31.826915+0000","last_became_peered":"2023-09-23T23:25:31.826915+0000","last_unstale":"2023-09-23T23:28:47.093995+0000","last_undegraded":"2023-09-23T23:28:47.093995+0000","last_fullsized":"2023-09-23T23:28:47.093995+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:44.656199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":733020160,"num_objects":11185,"num_object_clones":0,"num_object_copies":33555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11185,"num_write_kb":715840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2382299136,"data_stored":2382299136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":11185,"ondisk_log_size":11185,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":5608,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782593,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":990804,"kb_used_data":777868,"kb_used_omap":8,"kb_used_meta":212919,"kb_avail":92754348,"statfs":{"total":95995035648,"available":94980452352,"internally_reserved":0,"allocated":796536832,"data_stored":795228766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":218029125},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:28:43 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":2.8759999999999999,"15min":1.9490000000000001},"min":{"1min":0.245,"5min":0.216,"15min":0.216},"max":{"1min":4.7089999999999996,"5min":85.721000000000004,"15min":85.721000000000004},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.67400000000000004,"5min":3.2730000000000001,"15min":1.821},"min":{"1min":0.253,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":2.7149999999999999,"5min":85.965000000000003,"15min":85.965000000000003},"last":0.46899999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:28:43 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":1.6910000000000001,"15min":1.256},"min":{"1min":0.26800000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.6629999999999998,"5min":60.847000000000001,"15min":60.847000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.65300000000000002,"5min":1.6719999999999999,"15min":1.536},"min":{"1min":0.375,"5min":0.246,"15min":0.246},"max":{"1min":2.7490000000000001,"5min":59.512999999999998,"15min":59.512999999999998},"last":0.39500000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:28:43 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":2.0409999999999999,"15min":1.9019999999999999},"min":{"1min":0.30499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.6019999999999999,"5min":55.811999999999998,"15min":55.811999999999998},"last":0.83799999999999997},{"interface":"front","average":{"1min":1.282,"5min":1.9219999999999999,"15min":1.4350000000000001},"min":{"1min":0.312,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.6779999999999999,"5min":55.847999999999999,"15min":55.847999999999999},"last":0.36699999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:28:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.4019999999999999,"15min":1.46},"min":{"1min":0.35399999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.7119999999999997,"5min":9.9960000000000004,"15min":9.9960000000000004},"last":1.911},{"interface":"front","average":{"1min":0.78800000000000003,"5min":1.397,"15min":1.3280000000000001},"min":{"1min":0.40000000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.9660000000000002,"5min":9.9659999999999993,"15min":9.9659999999999993},"last":0.77100000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:28:43 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.65700000000000003,"15min":0.79100000000000004},"min":{"1min":0.318,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.028,"5min":6.8380000000000001,"15min":6.8380000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.746,"15min":0.874},"min":{"1min":0.32400000000000001,"5min":0.245,"15min":0.245},"max":{"1min":3.2970000000000002,"5min":5.242,"15min":5.242},"last":0.32600000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:28:43 2023","interfaces":[{"interface":"back","average":{"1min":11.726000000000001,"5min":3.028,"15min":1.734},"min":{"1min":0.36199999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":210.83600000000001,"5min":210.83600000000001,"15min":210.83600000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":11.618,"5min":2.9489999999999998,"15min":1.5229999999999999},"min":{"1min":0.36399999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":207.04599999999999,"5min":207.04599999999999,"15min":207.04599999999999},"last":0.49099999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:28:43 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.407,"15min":0.97199999999999998},"min":{"1min":0.36899999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.758,"5min":62.213000000000001,"15min":62.213000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.76600000000000001,"5min":1.462,"15min":1.2130000000000001},"min":{"1min":0.36099999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.7309999999999999,"5min":62.805999999999997,"15min":62.805999999999997},"last":0.748}]}]},{"osd":6,"up_from":0,"seq":197568496069,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":112,"seq":481036337410,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":959868,"kb_used_data":248372,"kb_used_omap":4,"kb_used_meta":711483,"kb_avail":92785284,"statfs":{"total":95995035648,"available":95012130816,"internally_reserved":0,"allocated":254332928,"data_stored":252106308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":728559502},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:27:59 2023","interfaces":[{"interface":"back","average":{"1min":15.005000000000001,"5min":5.6239999999999997,"15min":3.7549999999999999},"min":{"1min":0.25600000000000001,"5min":0.189,"15min":0.16800000000000001},"max":{"1min":130.768,"5min":130.768,"15min":130.768},"last":0.94299999999999995},{"interface":"front","average":{"1min":14.644,"5min":4.7809999999999997,"15min":3.6419999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":134.929,"5min":134.929,"15min":134.929},"last":0.73199999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:28:47 2023","interfaces":[{"interface":"back","average":{"1min":17.056999999999999,"5min":8.5760000000000005,"15min":5.2709999999999999},"min":{"1min":0.255,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":231.67400000000001,"5min":231.67400000000001,"15min":231.67400000000001},"last":1.198},{"interface":"front","average":{"1min":17.753,"5min":8.5719999999999992,"15min":4.8280000000000003},"min":{"1min":0.221,"5min":0.221,"15min":0.218},"max":{"1min":232.21000000000001,"5min":232.21000000000001,"15min":232.21000000000001},"last":0.58399999999999996}]},{"osd":3,"last update":"Sat Sep 23 23:28:44 2023","interfaces":[{"interface":"back","average":{"1min":15.568,"5min":6.3250000000000002,"15min":4.8630000000000004},"min":{"1min":0.25700000000000001,"5min":0.23200000000000001,"15min":0.20799999999999999},"max":{"1min":231.27199999999999,"5min":231.27199999999999,"15min":231.27199999999999},"last":0.76100000000000001},{"interface":"front","average":{"1min":15.065,"5min":6.1050000000000004,"15min":4.6749999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":231.43799999999999,"5min":231.43799999999999,"15min":231.43799999999999},"last":0.67200000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7889999999999999,"5min":1.6619999999999999,"15min":1.597},"min":{"1min":0.318,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":4.3520000000000003,"5min":9.3780000000000001,"15min":18.474},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.7470000000000001,"15min":1.724},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.23,"5min":9.7460000000000004,"15min":25.710999999999999},"last":0.91800000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.159,"15min":3.0659999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.9939999999999998,"5min":13.064,"15min":250.33000000000001},"last":0.629},{"interface":"front","average":{"1min":1.776,"5min":1.944,"15min":3.2120000000000002},"min":{"1min":0.377,"5min":0.34899999999999998,"15min":0.30199999999999999},"max":{"1min":4.3109999999999999,"5min":10.92,"15min":250.21199999999999},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":2.4929999999999999,"15min":2.4550000000000001},"min":{"1min":0.33700000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.0640000000000001,"5min":74.087999999999994,"15min":74.087999999999994},"last":0.69999999999999996},{"interface":"front","average":{"1min":1.835,"5min":2.843,"15min":2.3700000000000001},"min":{"1min":0.435,"5min":0.33100000000000002,"15min":0.32800000000000001},"max":{"1min":4.383,"5min":73.974999999999994,"15min":73.974999999999994},"last":0.52200000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:28:47 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":2.504,"15min":3.9119999999999999},"min":{"1min":0.39100000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":6.492,"5min":78.308000000000007,"15min":78.308000000000007},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.79100000000000004,"5min":2.4060000000000001,"15min":3.5139999999999998},"min":{"1min":0.45700000000000002,"5min":0.371,"15min":0.371},"max":{"1min":2.8109999999999999,"5min":77.204999999999998,"15min":77.204999999999998},"last":0.49299999999999999}]}]},{"osd":0,"up_from":91,"seq":390842024234,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1365312,"kb_used_data":313144,"kb_used_omap":5,"kb_used_meta":1052154,"kb_avail":92379840,"statfs":{"total":95995035648,"available":94596956160,"internally_reserved":0,"allocated":320659456,"data_stored":318364374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1077406330},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:28:19 2023","interfaces":[{"interface":"back","average":{"1min":7.3769999999999998,"5min":5.5780000000000003,"15min":4.3419999999999996},"min":{"1min":0.20200000000000001,"5min":0.17499999999999999,"15min":0.16900000000000001},"max":{"1min":80.774000000000001,"5min":139.30000000000001,"15min":155.91999999999999},"last":0.91400000000000003},{"interface":"front","average":{"1min":8.1189999999999998,"5min":3.4060000000000001,"15min":3.1859999999999999},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":85.534999999999997,"5min":85.534999999999997,"15min":153.28899999999999},"last":7.0990000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":5.3419999999999996,"15min":4.8120000000000003},"min":{"1min":0.25900000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.0910000000000002,"5min":139.471,"15min":171.15100000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.26,"5min":5.1299999999999999,"15min":5.069},"min":{"1min":0.24399999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":5.0129999999999999,"5min":139.04400000000001,"15min":173.04900000000001},"last":7.2699999999999996}]},{"osd":3,"last update":"Sat Sep 23 23:28:19 2023","interfaces":[{"interface":"back","average":{"1min":1.944,"5min":4.8600000000000003,"15min":3.9260000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.23200000000000001},"max":{"1min":8.5660000000000007,"5min":139.34899999999999,"15min":139.34899999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.8999999999999999,"5min":4.6669999999999998,"15min":3.5150000000000001},"min":{"1min":0.27400000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.2300000000000004,"5min":138.739,"15min":138.739},"last":6.6600000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:27:47 2023","interfaces":[{"interface":"back","average":{"1min":1.786,"5min":3.3809999999999998,"15min":2.6059999999999999},"min":{"1min":0.32200000000000001,"5min":0.29099999999999998,"15min":0.27000000000000002},"max":{"1min":9.2040000000000006,"5min":138.81299999999999,"15min":138.81299999999999},"last":6.7069999999999999},{"interface":"front","average":{"1min":1.55,"5min":5.415,"15min":3.2839999999999998},"min":{"1min":0.33200000000000002,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":7.0099999999999998,"5min":171.251,"15min":171.251},"last":0.54200000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:27:47 2023","interfaces":[{"interface":"back","average":{"1min":1.7889999999999999,"5min":5.5490000000000004,"15min":2.9649999999999999},"min":{"1min":0.34499999999999997,"5min":0.32100000000000001,"15min":0.29099999999999998},"max":{"1min":6.9699999999999998,"5min":171.30699999999999,"15min":171.30699999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.964,"5min":3.6080000000000001,"15min":2.2509999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":11.307,"5min":139.40799999999999,"15min":139.40799999999999},"last":0.42399999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:27:47 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":5.4320000000000004,"15min":3.161},"min":{"1min":0.371,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.0419999999999998,"5min":171.33600000000001,"15min":171.33600000000001},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.948,"5min":6.1689999999999996,"15min":3.4049999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.33,"5min":184.13499999999999,"15min":184.13499999999999},"last":0.47999999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:28:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":3.7090000000000001,"15min":2.1299999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.3630000000000004,"5min":138.77699999999999,"15min":138.77699999999999},"last":6.6050000000000004},{"interface":"front","average":{"1min":0.73199999999999998,"5min":3.6419999999999999,"15min":1.802},"min":{"1min":0.40600000000000003,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.302,"5min":139.37899999999999,"15min":139.37899999999999},"last":0.45000000000000001}]}]},{"osd":2,"up_from":199,"seq":854698492055,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1028016,"kb_used_data":486120,"kb_used_omap":7,"kb_used_meta":541880,"kb_avail":92717136,"statfs":{"total":95995035648,"available":94942347264,"internally_reserved":0,"allocated":497786880,"data_stored":495514070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":554885201},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":3.5579999999999998,"15min":4.6900000000000004},"min":{"1min":0.215,"5min":0.182,"15min":0.16700000000000001},"max":{"1min":2.6680000000000001,"5min":76.942999999999998,"15min":119.277},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.876,"5min":3.5659999999999998,"15min":4.7699999999999996},"min":{"1min":0.214,"5min":0.192,"15min":0.18099999999999999},"max":{"1min":5.3529999999999998,"5min":76.564999999999998,"15min":119.56100000000001},"last":1.8999999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:28:46 2023","interfaces":[{"interface":"back","average":{"1min":1.79,"5min":4.274,"15min":6.5460000000000003},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.20699999999999999},"max":{"1min":15.718999999999999,"5min":103.65900000000001,"15min":157.98699999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":2.8050000000000002,"5min":4.2709999999999999,"15min":6.3840000000000003},"min":{"1min":0.29699999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":17.783999999999999,"5min":100.182,"15min":158.03800000000001},"last":1.0389999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:28:46 2023","interfaces":[{"interface":"back","average":{"1min":6.9930000000000003,"5min":5.5739999999999998,"15min":5.2080000000000002},"min":{"1min":0.26500000000000001,"5min":0.191,"15min":0.191},"max":{"1min":103.227,"5min":103.227,"15min":119.366},"last":1.272},{"interface":"front","average":{"1min":7.3840000000000003,"5min":5.3259999999999996,"15min":4.5869999999999997},"min":{"1min":0.32800000000000001,"5min":0.25700000000000001,"15min":0.23499999999999999},"max":{"1min":102.693,"5min":102.693,"15min":102.693},"last":1.25}]},{"osd":4,"last update":"Sat Sep 23 23:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":1.9099999999999999,"15min":3.903},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.25},"max":{"1min":2.9020000000000001,"5min":23.391999999999999,"15min":332.98099999999999},"last":1.7929999999999999},{"interface":"front","average":{"1min":0.72599999999999998,"5min":1.8069999999999999,"15min":3.3450000000000002},"min":{"1min":0.31,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.7320000000000002,"5min":21.076000000000001,"15min":332.96199999999999},"last":1.238}]},{"osd":5,"last update":"Sat Sep 23 23:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":1.8979999999999999,"15min":2.2919999999999998},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.9399999999999999,"5min":23.359999999999999,"15min":119.919},"last":1.831},{"interface":"front","average":{"1min":0.89800000000000002,"5min":1.877,"15min":2.242},"min":{"1min":0.34699999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.3490000000000002,"5min":21.105,"15min":119.306},"last":0.70399999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.9279999999999999,"15min":4.1429999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.31},"max":{"1min":2.9700000000000002,"5min":23.425000000000001,"15min":349.32799999999997},"last":1.869},{"interface":"front","average":{"1min":1.127,"5min":1.921,"15min":3.6509999999999998},"min":{"1min":0.374,"5min":0.372,"15min":0.34999999999999998},"max":{"1min":4,"5min":21.353999999999999,"15min":349.17700000000002},"last":1.3420000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1.859,"15min":1.403},"min":{"1min":0.372,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.629,"5min":20.350000000000001,"15min":20.350000000000001},"last":1.1020000000000001},{"interface":"front","average":{"1min":0.95199999999999996,"5min":1.954,"15min":1.4950000000000001},"min":{"1min":0.42799999999999999,"5min":0.38,"15min":0.38},"max":{"1min":2.6920000000000002,"5min":20.321000000000002,"15min":20.321000000000002},"last":1.077}]}]},{"osd":3,"up_from":148,"seq":635655160024,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":923752,"kb_used_data":275872,"kb_used_omap":2,"kb_used_meta":647869,"kb_avail":92821400,"statfs":{"total":95995035648,"available":95049113600,"internally_reserved":0,"allocated":282492928,"data_stored":280273844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":663417909},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:28:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5219999999999998,"5min":3.452,"15min":3.48},"min":{"1min":0.218,"5min":0.17699999999999999,"15min":0.158},"max":{"1min":17.456,"5min":62.914000000000001,"15min":103.866},"last":1.722},{"interface":"front","average":{"1min":1.9490000000000001,"5min":3.2909999999999999,"15min":3.1429999999999998},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":13.436,"5min":63.276000000000003,"15min":101.884},"last":1.177}]},{"osd":1,"last update":"Sat Sep 23 23:28:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2589999999999999,"5min":2.6360000000000001,"15min":2.964},"min":{"1min":0.253,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":16.111999999999998,"5min":38.094000000000001,"15min":76.322999999999993},"last":0.82699999999999996},{"interface":"front","average":{"1min":2.0390000000000001,"5min":2.9380000000000002,"15min":2.8439999999999999},"min":{"1min":0.24099999999999999,"5min":0.21199999999999999,"15min":0.20499999999999999},"max":{"1min":15.779,"5min":42.030999999999999,"15min":76.444000000000003},"last":1.6850000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:28:41 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":3.714,"15min":3.7749999999999999},"min":{"1min":0.219,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":13.135,"5min":66.364999999999995,"15min":101.279},"last":0.85199999999999998},{"interface":"front","average":{"1min":1.575,"5min":3.5939999999999999,"15min":3.032},"min":{"1min":0.28999999999999998,"5min":0.214,"15min":0.214},"max":{"1min":9.4550000000000001,"5min":66.450000000000003,"15min":101.294},"last":0.88700000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:28:34 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":1.3169999999999999,"15min":1.492},"min":{"1min":0.33700000000000002,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":8.3059999999999992,"5min":8.3059999999999992,"15min":14.754},"last":0.83699999999999997},{"interface":"front","average":{"1min":1.6499999999999999,"5min":1.5329999999999999,"15min":1.5269999999999999},"min":{"1min":0.30599999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.314,"5min":15.619,"15min":15.619},"last":0.41999999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:28:34 2023","interfaces":[{"interface":"back","average":{"1min":1.7030000000000001,"5min":1.71,"15min":1.7549999999999999},"min":{"1min":0.29599999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":13.353999999999999,"5min":15.762,"15min":75.733000000000004},"last":0.44800000000000001},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.7410000000000001,"15min":2.0579999999999998},"min":{"1min":0.42599999999999999,"5min":0.316,"15min":0.28399999999999997},"max":{"1min":13.183,"5min":13.864000000000001,"15min":75.686999999999998},"last":0.746}]},{"osd":6,"last update":"Sat Sep 23 23:28:34 2023","interfaces":[{"interface":"back","average":{"1min":1.7030000000000001,"5min":2.2040000000000002,"15min":1.784},"min":{"1min":0.35199999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":13.380000000000001,"5min":53.057000000000002,"15min":53.057000000000002},"last":0.47099999999999997},{"interface":"front","average":{"1min":2.0939999999999999,"5min":2.2759999999999998,"15min":1.9890000000000001},"min":{"1min":0.437,"5min":0.35299999999999998,"15min":0.33300000000000002},"max":{"1min":13.215999999999999,"5min":53.088000000000001,"15min":53.088000000000001},"last":0.79400000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:28:41 2023","interfaces":[{"interface":"back","average":{"1min":4.9850000000000003,"5min":2.1389999999999998,"15min":1.2869999999999999},"min":{"1min":0.38100000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":88.344999999999999,"5min":88.344999999999999,"15min":88.344999999999999},"last":0.86499999999999999},{"interface":"front","average":{"1min":4.681,"5min":2.4399999999999999,"15min":1.635},"min":{"1min":0.38500000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":87.855000000000004,"5min":87.855000000000004,"15min":87.855000000000004},"last":1.2070000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019342,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":877968,"kb_used_data":157384,"kb_used_omap":0,"kb_used_meta":720576,"kb_avail":92867184,"statfs":{"total":95995035648,"available":95095996416,"internally_reserved":0,"allocated":161161216,"data_stored":159856955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":737869824},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:27:55 2023","interfaces":[{"interface":"back","average":{"1min":3.8149999999999999,"5min":3.8330000000000002,"15min":4.7480000000000002},"min":{"1min":0.23599999999999999,"5min":0.219,"15min":0.218},"max":{"1min":40.710000000000001,"5min":92.141999999999996,"15min":199.64400000000001},"last":0.22700000000000001},{"interface":"front","average":{"1min":3.496,"5min":3.8140000000000001,"15min":4.8460000000000001},"min":{"1min":0.27100000000000002,"5min":0.216,"15min":0.216},"max":{"1min":38.078000000000003,"5min":92.284000000000006,"15min":195.649},"last":0.50600000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:28:21 2023","interfaces":[{"interface":"back","average":{"1min":2.9630000000000001,"5min":2.0409999999999999,"15min":3.9399999999999999},"min":{"1min":0.30299999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":36.595999999999997,"5min":40.643000000000001,"15min":161.923},"last":0.38300000000000001},{"interface":"front","average":{"1min":3.4510000000000001,"5min":2.4860000000000002,"15min":4.1509999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":36.497,"5min":36.497,"15min":161.39699999999999},"last":0.56999999999999995}]},{"osd":2,"last update":"Sat Sep 23 23:28:39 2023","interfaces":[{"interface":"back","average":{"1min":8.4849999999999994,"5min":4.7720000000000002,"15min":3.6320000000000001},"min":{"1min":0.34499999999999997,"5min":0.27900000000000003,"15min":0.26300000000000001},"max":{"1min":130.506,"5min":130.506,"15min":130.506},"last":0.72899999999999998},{"interface":"front","average":{"1min":8.4979999999999993,"5min":4.7880000000000003,"15min":3.3969999999999998},"min":{"1min":0.434,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":130.209,"5min":130.209,"15min":130.209},"last":0.41099999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:28:38 2023","interfaces":[{"interface":"back","average":{"1min":13.73,"5min":8.4469999999999992,"15min":6.5529999999999999},"min":{"1min":0.35499999999999998,"5min":0.27200000000000002,"15min":0.26300000000000001},"max":{"1min":130.227,"5min":176.74100000000001,"15min":303.41500000000002},"last":0.42199999999999999},{"interface":"front","average":{"1min":13.834,"5min":8.1120000000000001,"15min":6.6139999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":131.03999999999999,"5min":176.87799999999999,"15min":303.428},"last":1.3280000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:28:10 2023","interfaces":[{"interface":"back","average":{"1min":9.7720000000000002,"5min":2.5270000000000001,"15min":2.004},"min":{"1min":0.27200000000000002,"5min":0.26700000000000002,"15min":0.23699999999999999},"max":{"1min":161.86500000000001,"5min":161.86500000000001,"15min":161.86500000000001},"last":0.26400000000000001},{"interface":"front","average":{"1min":9.5429999999999993,"5min":2.552,"15min":2.1400000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.249},"max":{"1min":159.24199999999999,"5min":159.24199999999999,"15min":160.61099999999999},"last":0.32800000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:28:38 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.73099999999999998,"15min":1.177},"min":{"1min":0.33500000000000002,"5min":0.29799999999999999,"15min":0.26000000000000001},"max":{"1min":2.3029999999999999,"5min":5.3399999999999999,"15min":90.346999999999994},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.82099999999999995,"5min":0.81499999999999995,"15min":1.3180000000000001},"min":{"1min":0.35599999999999998,"5min":0.33200000000000002,"15min":0.30599999999999999},"max":{"1min":2.4780000000000002,"5min":4.298,"15min":90.733000000000004},"last":0.34999999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:28:39 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":1.589,"15min":0.90700000000000003},"min":{"1min":0.34599999999999997,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.3849999999999998,"5min":61.923999999999999,"15min":61.923999999999999},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.80700000000000005,"5min":1.5509999999999999,"15min":1.052},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.5630000000000002,"5min":61.990000000000002,"15min":61.990000000000002},"last":0.53900000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757705,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":676608,"kb_used_data":62520,"kb_used_omap":0,"kb_used_meta":614080,"kb_avail":93068544,"statfs":{"total":95995035648,"available":95302189056,"internally_reserved":0,"allocated":64020480,"data_stored":62728626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":628817920},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:27:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":2.1520000000000001,"15min":3.7829999999999999},"min":{"1min":0.26200000000000001,"5min":0.222,"15min":0.19500000000000001},"max":{"1min":19.952000000000002,"5min":63.741999999999997,"15min":113.869},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.4219999999999999,"5min":3.3250000000000002,"15min":4.2169999999999996},"min":{"1min":0.26600000000000001,"5min":0.22900000000000001,"15min":0.22700000000000001},"max":{"1min":5.7320000000000002,"5min":138.55199999999999,"15min":138.55199999999999},"last":0.42899999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:28:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":3.5979999999999999,"15min":3.996},"min":{"1min":0.252,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":12.702,"5min":69.816999999999993,"15min":193.584},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.5069999999999999,"5min":4.0380000000000003,"15min":4.2699999999999996},"min":{"1min":0.28999999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.4560000000000004,"5min":70.268000000000001,"15min":194.03100000000001},"last":0.41799999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:28:39 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":4.3760000000000003,"15min":2.7509999999999999},"min":{"1min":0.32700000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":46.177,"5min":144.91200000000001,"15min":144.91200000000001},"last":0.443},{"interface":"front","average":{"1min":3.1899999999999999,"5min":4.2770000000000001,"15min":2.6589999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.25800000000000001},"max":{"1min":45.863999999999997,"5min":145.15799999999999,"15min":145.15799999999999},"last":0.45800000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:28:31 2023","interfaces":[{"interface":"back","average":{"1min":3.1400000000000001,"5min":3.4359999999999999,"15min":4.6230000000000002},"min":{"1min":0.36299999999999999,"5min":0.315,"15min":0.27400000000000002},"max":{"1min":43.497,"5min":62.612000000000002,"15min":159.53299999999999},"last":112.71899999999999},{"interface":"front","average":{"1min":3.1309999999999998,"5min":3.5150000000000001,"15min":4.8390000000000004},"min":{"1min":0.32700000000000001,"5min":0.318,"15min":0.312},"max":{"1min":43.515000000000001,"5min":62.639000000000003,"15min":159.54900000000001},"last":112.70099999999999}]},{"osd":4,"last update":"Sat Sep 23 23:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.71699999999999997,"15min":0.93799999999999994},"min":{"1min":0.25700000000000001,"5min":0.251,"15min":0.23999999999999999},"max":{"1min":1.929,"5min":5.3739999999999997,"15min":10.012},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.622,"15min":0.73499999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.6230000000000002,"5min":3.2530000000000001,"15min":11.923999999999999},"last":0.51400000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":1.488,"15min":2.1880000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.26900000000000002},"max":{"1min":2.6890000000000001,"5min":68.456999999999994,"15min":310.68799999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.438,"5min":1.4399999999999999,"15min":2.379},"min":{"1min":0.316,"5min":0.312,"15min":0.25900000000000001},"max":{"1min":0.58399999999999996,"5min":66.721999999999994,"15min":311.45400000000001},"last":0.495}]},{"osd":7,"last update":"Sat Sep 23 23:28:39 2023","interfaces":[{"interface":"back","average":{"1min":10.975,"5min":3.5819999999999999,"15min":3.9580000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":179.28800000000001,"5min":179.28800000000001,"15min":179.28800000000001},"last":0.57599999999999996},{"interface":"front","average":{"1min":11.010999999999999,"5min":3.597,"15min":4.0890000000000004},"min":{"1min":0.40100000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":179.428,"5min":179.428,"15min":179.428},"last":0.59599999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":320536576,"data_stored":320536576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":302841856,"data_stored":302841856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":551223296,"data_stored":551223296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":306970624,"data_stored":306970624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":175702016,"data_stored":175702016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":72286208,"data_stored":72286208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":652738560,"data_stored":652738560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:28:57.576 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T23:28:57.576 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 61 2023-09-23T23:28:57.598 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:28:57.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:57.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:28:57.705 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:57.811 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:58.215 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 13 16 25462 25446 122.255 98.8125 0.00521695 0.00817307 2023-09-23T23:28:58.224 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:58.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:58.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:58.329 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:28:58.345 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:58.440 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:58.611 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:28:58.639 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:28:58.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:58.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:28:58.746 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:28:58.769 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:28:58.769 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:28:58.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:28:58.770 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:28:58.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.770 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156651 10.1 10:8a72b73b:::benchmark_data_smithi110_366450_object55443:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:28:58.770 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:49.044180+0000", 2023-09-23T23:28:58.770 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 429.718211387, 2023-09-23T23:28:58.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.112146391, 2023-09-23T23:28:58.770 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.770 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:28:58.770 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.770 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:28:58.770 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:28:58.771 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156651 2023-09-23T23:28:58.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.771 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044180+0000", 2023-09-23T23:28:58.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044180+0000", 2023-09-23T23:28:58.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.772 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.772 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044179+0000", 2023-09-23T23:28:58.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:28:58.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.772 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.772 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044185+0000", 2023-09-23T23:28:58.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9699999999999996e-06 2023-09-23T23:28:58.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.772 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044197+0000", 2023-09-23T23:28:58.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2187e-05 2023-09-23T23:28:58.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044205+0000", 2023-09-23T23:28:58.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.678e-06 2023-09-23T23:28:58.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044265+0000", 2023-09-23T23:28:58.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9889000000000002e-05 2023-09-23T23:28:58.774 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op must wait for map", 2023-09-23T23:28:58.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044271+0000", 2023-09-23T23:28:58.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.455e-06 2023-09-23T23:28:58.774 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.052559+0000", 2023-09-23T23:28:58.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0082876549999999997 2023-09-23T23:28:58.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.052569+0000", 2023-09-23T23:28:58.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0039e-05 2023-09-23T23:28:58.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032597+0000", 2023-09-23T23:28:58.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98002873899999998 2023-09-23T23:28:58.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032607+0000", 2023-09-23T23:28:58.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4720000000000001e-06 2023-09-23T23:28:58.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.154569+0000", 2023-09-23T23:28:58.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.121962066 2023-09-23T23:28:58.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.156326+0000", 2023-09-23T23:28:58.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017571030000000001 2023-09-23T23:28:58.777 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.777 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.777 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.777 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.777 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156668 10.1 10:8e70998a:::benchmark_data_smithi110_366450_object55460:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:28:58.777 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:49.064619+0000", 2023-09-23T23:28:58.777 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 429.69777174900003, 2023-09-23T23:28:58.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1088757770000002, 2023-09-23T23:28:58.777 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.777 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.777 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.778 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:28:58.778 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:28:58.778 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156668 2023-09-23T23:28:58.778 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.778 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.778 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.778 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064619+0000", 2023-09-23T23:28:58.778 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.778 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.778 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064619+0000", 2023-09-23T23:28:58.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064619+0000", 2023-09-23T23:28:58.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:28:58.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064623+0000", 2023-09-23T23:28:58.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4089999999999996e-06 2023-09-23T23:28:58.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.780 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064632+0000", 2023-09-23T23:28:58.780 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5830000000000007e-06 2023-09-23T23:28:58.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.780 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064637+0000", 2023-09-23T23:28:58.780 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0760000000000002e-06 2023-09-23T23:28:58.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.066069+0000", 2023-09-23T23:28:58.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014321799999999999 2023-09-23T23:28:58.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.066077+0000", 2023-09-23T23:28:58.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8390000000000007e-06 2023-09-23T23:28:58.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032631+0000", 2023-09-23T23:28:58.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96655426600000005 2023-09-23T23:28:58.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.782 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032638+0000", 2023-09-23T23:28:58.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7669999999999998e-06 2023-09-23T23:28:58.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.782 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.156763+0000", 2023-09-23T23:28:58.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.124125493 2023-09-23T23:28:58.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.157388+0000", 2023-09-23T23:28:58.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00062452500000000002 2023-09-23T23:28:58.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:28:58.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.157840+0000", 2023-09-23T23:28:58.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045213799999999999 2023-09-23T23:28:58.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.165314+0000", 2023-09-23T23:28:58.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074745469999999998 2023-09-23T23:28:58.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.165348+0000", 2023-09-23T23:28:58.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3593999999999997e-05 2023-09-23T23:28:58.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173335+0000", 2023-09-23T23:28:58.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079866030000000001 2023-09-23T23:28:58.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173385+0000", 2023-09-23T23:28:58.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0932e-05 2023-09-23T23:28:58.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173495+0000", 2023-09-23T23:28:58.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109603 2023-09-23T23:28:58.785 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.785 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.785 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.786 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156686 10.1 10:87df1f6d:::benchmark_data_smithi110_366450_object55478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:28:58.786 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:49.084621+0000", 2023-09-23T23:28:58.786 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 429.67777040800001, 2023-09-23T23:28:58.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0890410259999999, 2023-09-23T23:28:58.786 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.786 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.786 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.786 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:28:58.786 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:28:58.787 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156686 2023-09-23T23:28:58.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.787 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084621+0000", 2023-09-23T23:28:58.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084621+0000", 2023-09-23T23:28:58.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084620+0000", 2023-09-23T23:28:58.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:58.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084623+0000", 2023-09-23T23:28:58.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5040000000000001e-06 2023-09-23T23:28:58.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084634+0000", 2023-09-23T23:28:58.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1405e-05 2023-09-23T23:28:58.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084639+0000", 2023-09-23T23:28:58.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.729e-06 2023-09-23T23:28:58.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084684+0000", 2023-09-23T23:28:58.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5045e-05 2023-09-23T23:28:58.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084690+0000", 2023-09-23T23:28:58.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5650000000000001e-06 2023-09-23T23:28:58.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032662+0000", 2023-09-23T23:28:58.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94797212099999995 2023-09-23T23:28:58.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032668+0000", 2023-09-23T23:28:58.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6080000000000001e-06 2023-09-23T23:28:58.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.791 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.162444+0000", 2023-09-23T23:28:58.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1297754170000001 2023-09-23T23:28:58.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.791 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.162591+0000", 2023-09-23T23:28:58.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014782799999999999 2023-09-23T23:28:58.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:28:58.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.162685+0000", 2023-09-23T23:28:58.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3898000000000001e-05 2023-09-23T23:28:58.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.170322+0000", 2023-09-23T23:28:58.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007636866 2023-09-23T23:28:58.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.170366+0000", 2023-09-23T23:28:58.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3380000000000001e-05 2023-09-23T23:28:58.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173508+0000", 2023-09-23T23:28:58.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031422640000000001 2023-09-23T23:28:58.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173562+0000", 2023-09-23T23:28:58.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4169000000000003e-05 2023-09-23T23:28:58.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173662+0000", 2023-09-23T23:28:58.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9615000000000001e-05 2023-09-23T23:28:58.794 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.794 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.794 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.794 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156687 10.1 10:8bf04f95:::benchmark_data_smithi110_366450_object55479:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:28:58.795 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:49.084650+0000", 2023-09-23T23:28:58.795 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 429.67774123200002, 2023-09-23T23:28:58.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.089277176, 2023-09-23T23:28:58.795 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.795 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.795 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.795 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:28:58.795 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:28:58.795 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156687 2023-09-23T23:28:58.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.795 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084650+0000", 2023-09-23T23:28:58.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084650+0000", 2023-09-23T23:28:58.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084649+0000", 2023-09-23T23:28:58.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:58.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084652+0000", 2023-09-23T23:28:58.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2670000000000001e-06 2023-09-23T23:28:58.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084658+0000", 2023-09-23T23:28:58.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1279999999999996e-06 2023-09-23T23:28:58.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084662+0000", 2023-09-23T23:28:58.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1289999999999998e-06 2023-09-23T23:28:58.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084716+0000", 2023-09-23T23:28:58.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4110000000000002e-05 2023-09-23T23:28:58.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084725+0000", 2023-09-23T23:28:58.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6989999999999992e-06 2023-09-23T23:28:58.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032691+0000", 2023-09-23T23:28:58.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94796597400000004 2023-09-23T23:28:58.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032698+0000", 2023-09-23T23:28:58.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7479999999999996e-06 2023-09-23T23:28:58.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.801 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.801 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.165371+0000", 2023-09-23T23:28:58.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1326732019999999 2023-09-23T23:28:58.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.801 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.801 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.165505+0000", 2023-09-23T23:28:58.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013406900000000001 2023-09-23T23:28:58.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.802 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:28:58.802 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.165655+0000", 2023-09-23T23:28:58.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015065 2023-09-23T23:28:58.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.802 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.802 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173673+0000", 2023-09-23T23:28:58.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080170699999999994 2023-09-23T23:28:58.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173736+0000", 2023-09-23T23:28:58.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3116999999999999e-05 2023-09-23T23:28:58.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173781+0000", 2023-09-23T23:28:58.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5778000000000001e-05 2023-09-23T23:28:58.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173829+0000", 2023-09-23T23:28:58.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7182999999999999e-05 2023-09-23T23:28:58.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173927+0000", 2023-09-23T23:28:58.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8424000000000005e-05 2023-09-23T23:28:58.805 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.805 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.805 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.805 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135599 11.e 11:71e23346:::benchmark_data_smithi110_433746_object29719:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:28:58.805 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505704+0000", 2023-09-23T23:28:58.805 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 167.25668727300001, 2023-09-23T23:28:58.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.837012734, 2023-09-23T23:28:58.805 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.806 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.806 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.806 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:28:58.806 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:58.806 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135599 2023-09-23T23:28:58.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.806 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505704+0000", 2023-09-23T23:28:58.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505704+0000", 2023-09-23T23:28:58.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505702+0000", 2023-09-23T23:28:58.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:58.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505710+0000", 2023-09-23T23:28:58.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.038e-06 2023-09-23T23:28:58.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505724+0000", 2023-09-23T23:28:58.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3684e-05 2023-09-23T23:28:58.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505733+0000", 2023-09-23T23:28:58.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1309999999999995e-06 2023-09-23T23:28:58.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505925+0000", 2023-09-23T23:28:58.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000192282 2023-09-23T23:28:58.810 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.810 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op must wait for map", 2023-09-23T23:28:58.810 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505933+0000", 2023-09-23T23:28:58.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3089999999999998e-06 2023-09-23T23:28:58.810 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.810 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.810 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513411+0000", 2023-09-23T23:28:58.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074784489999999999 2023-09-23T23:28:58.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513421+0000", 2023-09-23T23:28:58.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0108e-05 2023-09-23T23:28:58.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.304735+0000", 2023-09-23T23:28:58.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.791314093 2023-09-23T23:28:58.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.306968+0000", 2023-09-23T23:28:58.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022327390000000001 2023-09-23T23:28:58.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:28:58.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.307182+0000", 2023-09-23T23:28:58.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000214291 2023-09-23T23:28:58.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.322749+0000", 2023-09-23T23:28:58.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015566695 2023-09-23T23:28:58.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.322789+0000", 2023-09-23T23:28:58.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9808999999999998e-05 2023-09-23T23:28:58.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334988+0000", 2023-09-23T23:28:58.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012198595 2023-09-23T23:28:58.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342545+0000", 2023-09-23T23:28:58.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075570389999999998 2023-09-23T23:28:58.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342599+0000", 2023-09-23T23:28:58.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4347e-05 2023-09-23T23:28:58.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342717+0000", 2023-09-23T23:28:58.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117578 2023-09-23T23:28:58.815 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.815 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.816 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.816 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135605 11.e 11:7071ef4a:::benchmark_data_smithi110_433746_object29725:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:28:58.816 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505749+0000", 2023-09-23T23:28:58.816 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 167.256642236, 2023-09-23T23:28:58.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.837123356, 2023-09-23T23:28:58.816 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.816 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.816 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.817 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:28:58.817 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:58.817 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135605 2023-09-23T23:28:58.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.817 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505749+0000", 2023-09-23T23:28:58.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505749+0000", 2023-09-23T23:28:58.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505748+0000", 2023-09-23T23:28:58.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:58.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.819 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505751+0000", 2023-09-23T23:28:58.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.056e-06 2023-09-23T23:28:58.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.819 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.819 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505760+0000", 2023-09-23T23:28:58.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3750000000000003e-06 2023-09-23T23:28:58.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.819 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505766+0000", 2023-09-23T23:28:58.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8760000000000002e-06 2023-09-23T23:28:58.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505982+0000", 2023-09-23T23:28:58.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021631100000000001 2023-09-23T23:28:58.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:28:58.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505988+0000", 2023-09-23T23:28:58.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5779999999999998e-06 2023-09-23T23:28:58.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513443+0000", 2023-09-23T23:28:58.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007455377 2023-09-23T23:28:58.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513450+0000", 2023-09-23T23:28:58.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1459999999999999e-06 2023-09-23T23:28:58.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.322816+0000", 2023-09-23T23:28:58.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.809365865 2023-09-23T23:28:58.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.322945+0000", 2023-09-23T23:28:58.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012866099999999999 2023-09-23T23:28:58.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:28:58.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.323097+0000", 2023-09-23T23:28:58.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015215899999999999 2023-09-23T23:28:58.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334919+0000", 2023-09-23T23:28:58.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011822687 2023-09-23T23:28:58.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334954+0000", 2023-09-23T23:28:58.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4987999999999998e-05 2023-09-23T23:28:58.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335050+0000", 2023-09-23T23:28:58.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5222000000000004e-05 2023-09-23T23:28:58.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342725+0000", 2023-09-23T23:28:58.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076749360000000003 2023-09-23T23:28:58.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342767+0000", 2023-09-23T23:28:58.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2929000000000002e-05 2023-09-23T23:28:58.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342872+0000", 2023-09-23T23:28:58.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104709 2023-09-23T23:28:58.826 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.826 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.826 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.826 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135609 11.e 11:73c3a869:::benchmark_data_smithi110_433746_object29729:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:28:58.827 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505771+0000", 2023-09-23T23:28:58.827 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 167.256619734, 2023-09-23T23:28:58.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8373706329999999, 2023-09-23T23:28:58.827 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.827 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.827 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.827 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:28:58.827 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:58.827 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135609 2023-09-23T23:28:58.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.827 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505771+0000", 2023-09-23T23:28:58.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505771+0000", 2023-09-23T23:28:58.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505771+0000", 2023-09-23T23:28:58.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:58.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505773+0000", 2023-09-23T23:28:58.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4109999999999999e-06 2023-09-23T23:28:58.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505779+0000", 2023-09-23T23:28:58.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9529999999999996e-06 2023-09-23T23:28:58.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505784+0000", 2023-09-23T23:28:58.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1279999999999999e-06 2023-09-23T23:28:58.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506007+0000", 2023-09-23T23:28:58.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022278700000000001 2023-09-23T23:28:58.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:28:58.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506012+0000", 2023-09-23T23:28:58.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0390000000000004e-06 2023-09-23T23:28:58.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513470+0000", 2023-09-23T23:28:58.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074576879999999996 2023-09-23T23:28:58.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513476+0000", 2023-09-23T23:28:58.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9059999999999999e-06 2023-09-23T23:28:58.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.325254+0000", 2023-09-23T23:28:58.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8117784690000001 2023-09-23T23:28:58.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.325381+0000", 2023-09-23T23:28:58.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012669499999999999 2023-09-23T23:28:58.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:28:58.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.325510+0000", 2023-09-23T23:28:58.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012926999999999999 2023-09-23T23:28:58.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335017+0000", 2023-09-23T23:28:58.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0095070620000000002 2023-09-23T23:28:58.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335119+0000", 2023-09-23T23:28:58.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102045 2023-09-23T23:28:58.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342880+0000", 2023-09-23T23:28:58.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077602000000000001 2023-09-23T23:28:58.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342995+0000", 2023-09-23T23:28:58.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000115253 2023-09-23T23:28:58.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343036+0000", 2023-09-23T23:28:58.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0632e-05 2023-09-23T23:28:58.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343142+0000", 2023-09-23T23:28:58.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010644 2023-09-23T23:28:58.837 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.837 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.837 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.837 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135623 11.e 11:7e8ebd90:::benchmark_data_smithi110_433746_object29743:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:28:58.837 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505789+0000", 2023-09-23T23:28:58.837 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 167.256602434, 2023-09-23T23:28:58.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.843964774, 2023-09-23T23:28:58.837 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.837 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.838 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.838 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:28:58.838 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:58.838 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135623 2023-09-23T23:28:58.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.838 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505789+0000", 2023-09-23T23:28:58.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505789+0000", 2023-09-23T23:28:58.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505788+0000", 2023-09-23T23:28:58.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:58.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505791+0000", 2023-09-23T23:28:58.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.339e-06 2023-09-23T23:28:58.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505797+0000", 2023-09-23T23:28:58.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.207e-06 2023-09-23T23:28:58.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.841 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.841 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505802+0000", 2023-09-23T23:28:58.841 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9969999999999998e-06 2023-09-23T23:28:58.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.841 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.841 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506033+0000", 2023-09-23T23:28:58.841 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000230824 2023-09-23T23:28:58.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.842 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:28:58.842 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506037+0000", 2023-09-23T23:28:58.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8690000000000003e-06 2023-09-23T23:28:58.842 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.842 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.842 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513487+0000", 2023-09-23T23:28:58.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074491310000000003 2023-09-23T23:28:58.842 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513490+0000", 2023-09-23T23:28:58.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4649999999999999e-06 2023-09-23T23:28:58.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.327901+0000", 2023-09-23T23:28:58.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.814410818 2023-09-23T23:28:58.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.328011+0000", 2023-09-23T23:28:58.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109596 2023-09-23T23:28:58.844 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:28:58.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.328129+0000", 2023-09-23T23:28:58.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118751 2023-09-23T23:28:58.844 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.845 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335081+0000", 2023-09-23T23:28:58.845 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0069518549999999998 2023-09-23T23:28:58.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.845 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342886+0000", 2023-09-23T23:28:58.845 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078050599999999999 2023-09-23T23:28:58.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.845 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343224+0000", 2023-09-23T23:28:58.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033781399999999997 2023-09-23T23:28:58.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.846 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349601+0000", 2023-09-23T23:28:58.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063765899999999997 2023-09-23T23:28:58.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.846 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349647+0000", 2023-09-23T23:28:58.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6164e-05 2023-09-23T23:28:58.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.847 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.847 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349753+0000", 2023-09-23T23:28:58.847 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000106669 2023-09-23T23:28:58.847 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.847 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.847 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.848 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135624 11.e 11:79f5e3a3:::benchmark_data_smithi110_433746_object29744:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:28:58.848 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505805+0000", 2023-09-23T23:28:58.848 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 167.25658560900001, 2023-09-23T23:28:58.848 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.844120325, 2023-09-23T23:28:58.848 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.848 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.848 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.848 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:28:58.848 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:58.848 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135624 2023-09-23T23:28:58.848 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.849 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.849 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.849 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.849 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505805+0000", 2023-09-23T23:28:58.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.849 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.849 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.849 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505805+0000", 2023-09-23T23:28:58.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505805+0000", 2023-09-23T23:28:58.850 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:58.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505807+0000", 2023-09-23T23:28:58.850 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2120000000000002e-06 2023-09-23T23:28:58.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.851 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.851 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505813+0000", 2023-09-23T23:28:58.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.57e-06 2023-09-23T23:28:58.851 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.851 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.851 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505817+0000", 2023-09-23T23:28:58.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2980000000000002e-06 2023-09-23T23:28:58.851 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.852 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.852 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.852 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506057+0000", 2023-09-23T23:28:58.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000239836 2023-09-23T23:28:58.852 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.852 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.852 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:28:58.852 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506062+0000", 2023-09-23T23:28:58.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7870000000000001e-06 2023-09-23T23:28:58.852 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.852 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.853 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.853 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513504+0000", 2023-09-23T23:28:58.853 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074420959999999996 2023-09-23T23:28:58.853 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.853 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.853 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513508+0000", 2023-09-23T23:28:58.853 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9339999999999999e-06 2023-09-23T23:28:58.853 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.330221+0000", 2023-09-23T23:28:58.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.816713196 2023-09-23T23:28:58.854 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.854 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.330346+0000", 2023-09-23T23:28:58.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000125406 2023-09-23T23:28:58.854 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.854 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:28:58.855 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.330485+0000", 2023-09-23T23:28:58.855 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013825799999999999 2023-09-23T23:28:58.855 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.855 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.855 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.855 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342893+0000", 2023-09-23T23:28:58.855 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012408130999999999 2023-09-23T23:28:58.855 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.855 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.855 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.856 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342951+0000", 2023-09-23T23:28:58.856 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8354999999999999e-05 2023-09-23T23:28:58.856 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.856 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.856 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.856 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343297+0000", 2023-09-23T23:28:58.856 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034624800000000001 2023-09-23T23:28:58.856 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.856 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.856 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.856 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349790+0000", 2023-09-23T23:28:58.857 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0064925399999999998 2023-09-23T23:28:58.857 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.857 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.857 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.857 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349831+0000", 2023-09-23T23:28:58.857 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1069999999999998e-05 2023-09-23T23:28:58.857 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.857 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.857 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.857 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349926+0000", 2023-09-23T23:28:58.858 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4718999999999999e-05 2023-09-23T23:28:58.858 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.858 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.858 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.858 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.858 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135642 11.e 11:7847da74:::benchmark_data_smithi110_433746_object29762:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:28:58.858 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505821+0000", 2023-09-23T23:28:58.858 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 167.25657012299999, 2023-09-23T23:28:58.858 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.844417151, 2023-09-23T23:28:58.858 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.859 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.859 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.859 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:28:58.859 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:58.859 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135642 2023-09-23T23:28:58.859 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.859 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.859 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505821+0000", 2023-09-23T23:28:58.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505821+0000", 2023-09-23T23:28:58.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505821+0000", 2023-09-23T23:28:58.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:58.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505823+0000", 2023-09-23T23:28:58.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4949999999999998e-06 2023-09-23T23:28:58.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505829+0000", 2023-09-23T23:28:58.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.383e-06 2023-09-23T23:28:58.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505833+0000", 2023-09-23T23:28:58.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8770000000000001e-06 2023-09-23T23:28:58.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506081+0000", 2023-09-23T23:28:58.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024778400000000003 2023-09-23T23:28:58.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:28:58.863 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506086+0000", 2023-09-23T23:28:58.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0170000000000004e-06 2023-09-23T23:28:58.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.863 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513526+0000", 2023-09-23T23:28:58.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074400029999999997 2023-09-23T23:28:58.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.864 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.864 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.864 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513533+0000", 2023-09-23T23:28:58.864 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5189999999999998e-06 2023-09-23T23:28:58.864 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.864 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.864 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.864 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.332515+0000", 2023-09-23T23:28:58.864 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.818982726 2023-09-23T23:28:58.864 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.865 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.332698+0000", 2023-09-23T23:28:58.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000182046 2023-09-23T23:28:58.865 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:28:58.865 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.332836+0000", 2023-09-23T23:28:58.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013802000000000001 2023-09-23T23:28:58.865 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.866 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.866 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.866 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342905+0000", 2023-09-23T23:28:58.866 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010069121 2023-09-23T23:28:58.866 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.866 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.866 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.866 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343190+0000", 2023-09-23T23:28:58.866 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028497700000000001 2023-09-23T23:28:58.866 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.867 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.867 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.867 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343385+0000", 2023-09-23T23:28:58.867 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019561000000000001 2023-09-23T23:28:58.867 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.867 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.867 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.867 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350092+0000", 2023-09-23T23:28:58.867 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067069870000000002 2023-09-23T23:28:58.867 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.868 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350141+0000", 2023-09-23T23:28:58.868 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8365000000000002e-05 2023-09-23T23:28:58.868 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.868 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350238+0000", 2023-09-23T23:28:58.868 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7510999999999995e-05 2023-09-23T23:28:58.868 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.869 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.869 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.869 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.869 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135646 11.e 11:7bf5000e:::benchmark_data_smithi110_433746_object29766:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:28:58.869 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505837+0000", 2023-09-23T23:28:58.869 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 167.25655365899999, 2023-09-23T23:28:58.869 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.844569372, 2023-09-23T23:28:58.869 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.869 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.870 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.870 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:28:58.870 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:58.870 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135646 2023-09-23T23:28:58.870 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.870 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.870 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.870 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.870 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505837+0000", 2023-09-23T23:28:58.870 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.871 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.871 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505837+0000", 2023-09-23T23:28:58.871 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.871 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.871 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505837+0000", 2023-09-23T23:28:58.871 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:58.872 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.872 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.872 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.872 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505839+0000", 2023-09-23T23:28:58.872 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3530000000000002e-06 2023-09-23T23:28:58.872 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.872 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.872 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.872 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505846+0000", 2023-09-23T23:28:58.872 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1800000000000001e-06 2023-09-23T23:28:58.872 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.873 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.873 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505850+0000", 2023-09-23T23:28:58.873 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5700000000000003e-06 2023-09-23T23:28:58.873 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.873 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.873 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506104+0000", 2023-09-23T23:28:58.873 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025424300000000003 2023-09-23T23:28:58.873 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.874 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.874 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:28:58.874 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506109+0000", 2023-09-23T23:28:58.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8749999999999999e-06 2023-09-23T23:28:58.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.874 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.874 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.874 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513566+0000", 2023-09-23T23:28:58.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074570510000000001 2023-09-23T23:28:58.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.875 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.875 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.875 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513573+0000", 2023-09-23T23:28:58.875 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9879999999999998e-06 2023-09-23T23:28:58.875 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.875 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.875 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.875 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335148+0000", 2023-09-23T23:28:58.875 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.82157426 2023-09-23T23:28:58.875 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335293+0000", 2023-09-23T23:28:58.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014563600000000001 2023-09-23T23:28:58.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:28:58.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335527+0000", 2023-09-23T23:28:58.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023380100000000001 2023-09-23T23:28:58.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.877 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342911+0000", 2023-09-23T23:28:58.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007384191 2023-09-23T23:28:58.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.877 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343259+0000", 2023-09-23T23:28:58.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034814699999999999 2023-09-23T23:28:58.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.878 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343422+0000", 2023-09-23T23:28:58.878 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016240100000000001 2023-09-23T23:28:58.878 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.878 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350272+0000", 2023-09-23T23:28:58.878 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068501539999999998 2023-09-23T23:28:58.878 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350313+0000", 2023-09-23T23:28:58.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1044000000000002e-05 2023-09-23T23:28:58.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.879 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.879 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350407+0000", 2023-09-23T23:28:58.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3838000000000005e-05 2023-09-23T23:28:58.879 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.879 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.879 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.880 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135676 11.e 11:7b57d3a0:::benchmark_data_smithi110_433746_object29796:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:28:58.880 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505854+0000", 2023-09-23T23:28:58.880 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 167.256537049, 2023-09-23T23:28:58.880 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.844725422, 2023-09-23T23:28:58.880 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.880 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.880 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.880 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:28:58.880 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:58.881 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135676 2023-09-23T23:28:58.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.881 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.881 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.881 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505854+0000", 2023-09-23T23:28:58.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.881 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.881 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.882 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505854+0000", 2023-09-23T23:28:58.882 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.882 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.882 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505854+0000", 2023-09-23T23:28:58.882 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:58.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.882 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.882 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505856+0000", 2023-09-23T23:28:58.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3700000000000002e-06 2023-09-23T23:28:58.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505862+0000", 2023-09-23T23:28:58.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2809999999999997e-06 2023-09-23T23:28:58.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505867+0000", 2023-09-23T23:28:58.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.933e-06 2023-09-23T23:28:58.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.884 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.884 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.884 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506128+0000", 2023-09-23T23:28:58.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000260938 2023-09-23T23:28:58.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.884 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.884 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:28:58.884 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506132+0000", 2023-09-23T23:28:58.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3409999999999999e-06 2023-09-23T23:28:58.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.885 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513595+0000", 2023-09-23T23:28:58.885 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074631480000000002 2023-09-23T23:28:58.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.885 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513602+0000", 2023-09-23T23:28:58.885 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9140000000000002e-06 2023-09-23T23:28:58.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337712+0000", 2023-09-23T23:28:58.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8241103679999999 2023-09-23T23:28:58.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337841+0000", 2023-09-23T23:28:58.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012925499999999999 2023-09-23T23:28:58.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:28:58.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.338028+0000", 2023-09-23T23:28:58.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018702899999999999 2023-09-23T23:28:58.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342960+0000", 2023-09-23T23:28:58.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0049315160000000004 2023-09-23T23:28:58.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343333+0000", 2023-09-23T23:28:58.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00037339399999999998 2023-09-23T23:28:58.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343461+0000", 2023-09-23T23:28:58.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012787399999999999 2023-09-23T23:28:58.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350439+0000", 2023-09-23T23:28:58.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0069780379999999998 2023-09-23T23:28:58.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350480+0000", 2023-09-23T23:28:58.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.057e-05 2023-09-23T23:28:58.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350579+0000", 2023-09-23T23:28:58.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9774999999999999e-05 2023-09-23T23:28:58.890 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.890 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.890 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.890 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135677 11.e 11:7ce6155e:::benchmark_data_smithi110_433746_object29797:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:28:58.890 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505871+0000", 2023-09-23T23:28:58.890 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 167.25651976399999, 2023-09-23T23:28:58.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8476306259999999, 2023-09-23T23:28:58.891 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.891 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.891 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.891 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:28:58.891 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:58.891 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135677 2023-09-23T23:28:58.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.891 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505871+0000", 2023-09-23T23:28:58.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505871+0000", 2023-09-23T23:28:58.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505871+0000", 2023-09-23T23:28:58.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:58.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505873+0000", 2023-09-23T23:28:58.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.43e-06 2023-09-23T23:28:58.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505879+0000", 2023-09-23T23:28:58.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8250000000000001e-06 2023-09-23T23:28:58.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505884+0000", 2023-09-23T23:28:58.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.955e-06 2023-09-23T23:28:58.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.895 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506151+0000", 2023-09-23T23:28:58.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000266591 2023-09-23T23:28:58.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:28:58.895 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506156+0000", 2023-09-23T23:28:58.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8400000000000002e-06 2023-09-23T23:28:58.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513622+0000", 2023-09-23T23:28:58.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007466744 2023-09-23T23:28:58.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513629+0000", 2023-09-23T23:28:58.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4400000000000002e-06 2023-09-23T23:28:58.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.340120+0000", 2023-09-23T23:28:58.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.826490739 2023-09-23T23:28:58.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.340250+0000", 2023-09-23T23:28:58.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000130448 2023-09-23T23:28:58.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:28:58.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.340609+0000", 2023-09-23T23:28:58.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035861699999999999 2023-09-23T23:28:58.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.898 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.348233+0000", 2023-09-23T23:28:58.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007624123 2023-09-23T23:28:58.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.898 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.348294+0000", 2023-09-23T23:28:58.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1487000000000003e-05 2023-09-23T23:28:58.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.348306+0000", 2023-09-23T23:28:58.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1941e-05 2023-09-23T23:28:58.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.353328+0000", 2023-09-23T23:28:58.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0050216280000000002 2023-09-23T23:28:58.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.353383+0000", 2023-09-23T23:28:58.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5682e-05 2023-09-23T23:28:58.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.353502+0000", 2023-09-23T23:28:58.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118492 2023-09-23T23:28:58.901 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.901 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.901 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.901 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178600 11.6 11:659f4d42:::benchmark_data_smithi110_433746_object72720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2023-09-23T23:28:58.901 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.523923+0000", 2023-09-23T23:28:58.901 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 99.238467829000001, 2023-09-23T23:28:58.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2361555310000001, 2023-09-23T23:28:58.901 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.901 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:28:58.902 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.902 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:28:58.902 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:58.902 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178600 2023-09-23T23:28:58.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.902 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.902 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.902 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523923+0000", 2023-09-23T23:28:58.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523923+0000", 2023-09-23T23:28:58.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523922+0000", 2023-09-23T23:28:58.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:58.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523929+0000", 2023-09-23T23:28:58.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8800000000000002e-06 2023-09-23T23:28:58.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523940+0000", 2023-09-23T23:28:58.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.149e-05 2023-09-23T23:28:58.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.905 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.905 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523948+0000", 2023-09-23T23:28:58.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7970000000000001e-06 2023-09-23T23:28:58.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.905 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.905 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.531901+0000", 2023-09-23T23:28:58.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079527399999999998 2023-09-23T23:28:58.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.532034+0000", 2023-09-23T23:28:58.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013345499999999999 2023-09-23T23:28:58.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:28:58.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.532514+0000", 2023-09-23T23:28:58.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047994200000000002 2023-09-23T23:28:58.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.537104+0000", 2023-09-23T23:28:58.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045898019999999996 2023-09-23T23:28:58.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.538159+0000", 2023-09-23T23:28:58.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001055437 2023-09-23T23:28:58.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.908 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.547877+0000", 2023-09-23T23:28:58.908 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0097177860000000008 2023-09-23T23:28:58.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.908 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.547890+0000", 2023-09-23T23:28:58.908 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2656e-05 2023-09-23T23:28:58.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.909 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.909 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.909 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542710+0000", 2023-09-23T23:28:58.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99482022199999998 2023-09-23T23:28:58.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.909 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.909 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.909 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542720+0000", 2023-09-23T23:28:58.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9029999999999999e-06 2023-09-23T23:28:58.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760000+0000", 2023-09-23T23:28:58.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.217280103 2023-09-23T23:28:58.910 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760079+0000", 2023-09-23T23:28:58.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8777999999999997e-05 2023-09-23T23:28:58.910 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.911 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.911 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.911 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.911 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178608 11.6 11:6177c8c0:::benchmark_data_smithi110_433746_object72728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2023-09-23T23:28:58.911 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.545706+0000", 2023-09-23T23:28:58.911 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 99.216685401000007, 2023-09-23T23:28:58.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2481279590000001, 2023-09-23T23:28:58.911 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.911 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.911 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.912 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:28:58.912 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:58.912 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178608 2023-09-23T23:28:58.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.912 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.912 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545706+0000", 2023-09-23T23:28:58.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.913 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545706+0000", 2023-09-23T23:28:58.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.913 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545705+0000", 2023-09-23T23:28:58.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:28:58.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545709+0000", 2023-09-23T23:28:58.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6140000000000002e-06 2023-09-23T23:28:58.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545721+0000", 2023-09-23T23:28:58.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2345e-05 2023-09-23T23:28:58.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545730+0000", 2023-09-23T23:28:58.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9579999999999996e-06 2023-09-23T23:28:58.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.570191+0000", 2023-09-23T23:28:58.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024460784999999999 2023-09-23T23:28:58.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.570202+0000", 2023-09-23T23:28:58.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1192999999999999e-05 2023-09-23T23:28:58.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542744+0000", 2023-09-23T23:28:58.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.972541879 2023-09-23T23:28:58.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542752+0000", 2023-09-23T23:28:58.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1599999999999998e-06 2023-09-23T23:28:58.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760100+0000", 2023-09-23T23:28:58.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2173479519999999 2023-09-23T23:28:58.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760221+0000", 2023-09-23T23:28:58.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000120155 2023-09-23T23:28:58.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.918 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T23:28:58.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760707+0000", 2023-09-23T23:28:58.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00048629800000000003 2023-09-23T23:28:58.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.918 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.774649+0000", 2023-09-23T23:28:58.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013942049 2023-09-23T23:28:58.919 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.919 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.919 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783406+0000", 2023-09-23T23:28:58.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0087574420000000007 2023-09-23T23:28:58.919 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.919 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.919 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783684+0000", 2023-09-23T23:28:58.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027739099999999999 2023-09-23T23:28:58.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.920 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.920 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793677+0000", 2023-09-23T23:28:58.920 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0099932059999999993 2023-09-23T23:28:58.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.920 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.920 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793723+0000", 2023-09-23T23:28:58.920 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5883000000000001e-05 2023-09-23T23:28:58.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793834+0000", 2023-09-23T23:28:58.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110819 2023-09-23T23:28:58.921 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.921 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.921 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.921 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178617 11.16 11:6d8a65a1:::benchmark_data_smithi110_433746_object72737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:28:58.922 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.584811+0000", 2023-09-23T23:28:58.922 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 99.177580000999995, 2023-09-23T23:28:58.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1985552099999999, 2023-09-23T23:28:58.922 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.922 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.922 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.922 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:28:58.922 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:58.922 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178617 2023-09-23T23:28:58.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.923 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584811+0000", 2023-09-23T23:28:58.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584811+0000", 2023-09-23T23:28:58.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584809+0000", 2023-09-23T23:28:58.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:58.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584817+0000", 2023-09-23T23:28:58.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3339999999999996e-06 2023-09-23T23:28:58.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584831+0000", 2023-09-23T23:28:58.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4355000000000001e-05 2023-09-23T23:28:58.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584840+0000", 2023-09-23T23:28:58.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3410000000000003e-06 2023-09-23T23:28:58.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591289+0000", 2023-09-23T23:28:58.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0064494030000000003 2023-09-23T23:28:58.926 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591300+0000", 2023-09-23T23:28:58.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0597e-05 2023-09-23T23:28:58.926 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.927 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.927 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542631+0000", 2023-09-23T23:28:58.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.951331449 2023-09-23T23:28:58.927 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.927 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.927 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.927 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542644+0000", 2023-09-23T23:28:58.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.288e-05 2023-09-23T23:28:58.927 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.927 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.927 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.928 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.756708+0000", 2023-09-23T23:28:58.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.21406408 2023-09-23T23:28:58.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.928 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.928 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.756818+0000", 2023-09-23T23:28:58.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110073 2023-09-23T23:28:58.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.928 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T23:28:58.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.757269+0000", 2023-09-23T23:28:58.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045095900000000002 2023-09-23T23:28:58.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764656+0000", 2023-09-23T23:28:58.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007387195 2023-09-23T23:28:58.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764667+0000", 2023-09-23T23:28:58.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0365e-05 2023-09-23T23:28:58.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.774688+0000", 2023-09-23T23:28:58.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010021577 2023-09-23T23:28:58.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783206+0000", 2023-09-23T23:28:58.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0085183289999999998 2023-09-23T23:28:58.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783262+0000", 2023-09-23T23:28:58.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5275999999999997e-05 2023-09-23T23:28:58.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783366+0000", 2023-09-23T23:28:58.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010458099999999999 2023-09-23T23:28:58.932 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.932 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.932 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.932 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178622 11.6 11:6344b54e:::benchmark_data_smithi110_433746_object72742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:28:58.932 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.591902+0000", 2023-09-23T23:28:58.932 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 99.170489021999998, 2023-09-23T23:28:58.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2021504429999998, 2023-09-23T23:28:58.933 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.933 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.933 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.933 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:28:58.933 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:58.933 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178622 2023-09-23T23:28:58.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.933 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.934 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591902+0000", 2023-09-23T23:28:58.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.934 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.934 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.934 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.934 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591902+0000", 2023-09-23T23:28:58.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.934 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.934 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.934 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.935 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591901+0000", 2023-09-23T23:28:58.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:58.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.935 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591906+0000", 2023-09-23T23:28:58.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8289999999999996e-06 2023-09-23T23:28:58.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.936 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591921+0000", 2023-09-23T23:28:58.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4149e-05 2023-09-23T23:28:58.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.936 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.936 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591928+0000", 2023-09-23T23:28:58.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0999999999999998e-06 2023-09-23T23:28:58.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.936 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.592603+0000", 2023-09-23T23:28:58.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00067509500000000001 2023-09-23T23:28:58.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.592613+0000", 2023-09-23T23:28:58.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0671e-05 2023-09-23T23:28:58.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542776+0000", 2023-09-23T23:28:58.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95016243199999995 2023-09-23T23:28:58.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542784+0000", 2023-09-23T23:28:58.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5580000000000001e-06 2023-09-23T23:28:58.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764299+0000", 2023-09-23T23:28:58.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2215145999999999 2023-09-23T23:28:58.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764438+0000", 2023-09-23T23:28:58.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000138536 2023-09-23T23:28:58.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T23:28:58.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.765798+0000", 2023-09-23T23:28:58.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001360874 2023-09-23T23:28:58.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.774753+0000", 2023-09-23T23:28:58.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0089545679999999996 2023-09-23T23:28:58.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783698+0000", 2023-09-23T23:28:58.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0089451630000000008 2023-09-23T23:28:58.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.941 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.941 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.785057+0000", 2023-09-23T23:28:58.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001358784 2023-09-23T23:28:58.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.941 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.941 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793921+0000", 2023-09-23T23:28:58.942 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0088636380000000001 2023-09-23T23:28:58.942 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.942 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.942 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793959+0000", 2023-09-23T23:28:58.942 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8328999999999998e-05 2023-09-23T23:28:58.942 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.942 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.942 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.794052+0000", 2023-09-23T23:28:58.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3539e-05 2023-09-23T23:28:58.943 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.943 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.943 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.943 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178633 11.16 11:6db526fe:::benchmark_data_smithi110_433746_object72753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:28:58.943 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.633733+0000", 2023-09-23T23:28:58.943 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 99.128657687, 2023-09-23T23:28:58.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.160140325, 2023-09-23T23:28:58.944 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.944 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.944 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.944 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:28:58.944 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:58.944 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178633 2023-09-23T23:28:58.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.944 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633733+0000", 2023-09-23T23:28:58.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633733+0000", 2023-09-23T23:28:58.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633732+0000", 2023-09-23T23:28:58.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:28:58.946 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.946 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.946 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633739+0000", 2023-09-23T23:28:58.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2899999999999997e-06 2023-09-23T23:28:58.946 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.946 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.946 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633756+0000", 2023-09-23T23:28:58.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6368999999999999e-05 2023-09-23T23:28:58.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633763+0000", 2023-09-23T23:28:58.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8930000000000005e-06 2023-09-23T23:28:58.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633828+0000", 2023-09-23T23:28:58.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4092999999999998e-05 2023-09-23T23:28:58.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633843+0000", 2023-09-23T23:28:58.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5512e-05 2023-09-23T23:28:58.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542670+0000", 2023-09-23T23:28:58.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.90882660500000001 2023-09-23T23:28:58.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542679+0000", 2023-09-23T23:28:58.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0000000000000002e-06 2023-09-23T23:28:58.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760612+0000", 2023-09-23T23:28:58.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.217933355 2023-09-23T23:28:58.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760705+0000", 2023-09-23T23:28:58.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3078999999999994e-05 2023-09-23T23:28:58.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T23:28:58.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.761116+0000", 2023-09-23T23:28:58.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00041080000000000001 2023-09-23T23:28:58.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764673+0000", 2023-09-23T23:28:58.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035569239999999999 2023-09-23T23:28:58.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.767908+0000", 2023-09-23T23:28:58.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032356749999999999 2023-09-23T23:28:58.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783437+0000", 2023-09-23T23:28:58.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015528767000000001 2023-09-23T23:28:58.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793721+0000", 2023-09-23T23:28:58.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010284188 2023-09-23T23:28:58.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793766+0000", 2023-09-23T23:28:58.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4631e-05 2023-09-23T23:28:58.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.954 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793874+0000", 2023-09-23T23:28:58.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010767100000000001 2023-09-23T23:28:58.954 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.954 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.954 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.955 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178629 11.7 11:e3eb7c4c:::benchmark_data_smithi110_433746_object72749:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e425)", 2023-09-23T23:28:58.955 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:20.543856+0000", 2023-09-23T23:28:58.955 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 98.218535293000002, 2023-09-23T23:28:58.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2512707000000001, 2023-09-23T23:28:58.955 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.955 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.955 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.955 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:28:58.955 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:58.956 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178629 2023-09-23T23:28:58.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.956 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543856+0000", 2023-09-23T23:28:58.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543856+0000", 2023-09-23T23:28:58.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543855+0000", 2023-09-23T23:28:58.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:28:58.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543857+0000", 2023-09-23T23:28:58.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.99e-06 2023-09-23T23:28:58.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543862+0000", 2023-09-23T23:28:58.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.899e-06 2023-09-23T23:28:58.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543867+0000", 2023-09-23T23:28:58.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3710000000000002e-06 2023-09-23T23:28:58.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.544825+0000", 2023-09-23T23:28:58.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000957864 2023-09-23T23:28:58.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.544832+0000", 2023-09-23T23:28:58.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3649999999999998e-06 2023-09-23T23:28:58.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.780766+0000", 2023-09-23T23:28:58.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2359338689999999 2023-09-23T23:28:58.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.780888+0000", 2023-09-23T23:28:58.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000121833 2023-09-23T23:28:58.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:28:58.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.781338+0000", 2023-09-23T23:28:58.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045035600000000001 2023-09-23T23:28:58.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.784204+0000", 2023-09-23T23:28:58.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002865884 2023-09-23T23:28:58.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.785473+0000", 2023-09-23T23:28:58.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001268838 2023-09-23T23:28:58.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.788590+0000", 2023-09-23T23:28:58.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031171839999999998 2023-09-23T23:28:58.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.794983+0000", 2023-09-23T23:28:58.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063934339999999999 2023-09-23T23:28:58.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.795024+0000", 2023-09-23T23:28:58.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0828000000000003e-05 2023-09-23T23:28:58.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.795126+0000", 2023-09-23T23:28:58.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102277 2023-09-23T23:28:58.964 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.964 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.964 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.964 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211753 11.19 11:9cef2dd2:::benchmark_data_smithi110_433746_object105873:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T23:28:58.964 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:24.737706+0000", 2023-09-23T23:28:58.965 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 34.024684874999998, 2023-09-23T23:28:58.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.967897013, 2023-09-23T23:28:58.965 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:28:58.965 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:28:58.965 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:28:58.965 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:28:58.965 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:28:58.965 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211753 2023-09-23T23:28:58.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.965 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:28:58.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:28:58.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737706+0000", 2023-09-23T23:28:58.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:28:58.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737706+0000", 2023-09-23T23:28:58.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:28:58.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:28:58.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737704+0000", 2023-09-23T23:28:58.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:28:58.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:28:58.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737713+0000", 2023-09-23T23:28:58.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3939999999999996e-06 2023-09-23T23:28:58.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:28:58.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737727+0000", 2023-09-23T23:28:58.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3893999999999999e-05 2023-09-23T23:28:58.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:28:58.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737736+0000", 2023-09-23T23:28:58.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0780000000000002e-06 2023-09-23T23:28:58.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737803+0000", 2023-09-23T23:28:58.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6959999999999996e-05 2023-09-23T23:28:58.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737815+0000", 2023-09-23T23:28:58.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2377e-05 2023-09-23T23:28:58.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:25.604895+0000", 2023-09-23T23:28:58.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86708047300000002 2023-09-23T23:28:58.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:28:58.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:25.604906+0000", 2023-09-23T23:28:58.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0903999999999999e-05 2023-09-23T23:28:58.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:28:58.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.701468+0000", 2023-09-23T23:28:58.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.096561801 2023-09-23T23:28:58.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:28:58.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.701626+0000", 2023-09-23T23:28:58.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015756900000000001 2023-09-23T23:28:58.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,3,7", 2023-09-23T23:28:58.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.701760+0000", 2023-09-23T23:28:58.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013463700000000001 2023-09-23T23:28:58.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.704681+0000", 2023-09-23T23:28:58.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029204080000000002 2023-09-23T23:28:58.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:28:58.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.704693+0000", 2023-09-23T23:28:58.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2427e-05 2023-09-23T23:28:58.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.704736+0000", 2023-09-23T23:28:58.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2911000000000003e-05 2023-09-23T23:28:58.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.704783+0000", 2023-09-23T23:28:58.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6931999999999997e-05 2023-09-23T23:28:58.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:28:58.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.705399+0000", 2023-09-23T23:28:58.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000615889 2023-09-23T23:28:58.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:28:58.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.705458+0000", 2023-09-23T23:28:58.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8731999999999999e-05 2023-09-23T23:28:58.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:28:58.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:28:58.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:28:58.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.705603+0000", 2023-09-23T23:28:58.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014554600000000001 2023-09-23T23:28:58.975 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.975 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.975 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.975 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:28:58.975 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:28:58.976 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:28:58.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:58.977 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:58.977 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:28:58.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:28:59.078 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:59.188 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:28:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:59 smithi138 ceph-mon[193606]: pgmap v1317: 119 pgs: 119 active+clean; 1.2 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-09-23T23:28:59.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:59 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:59.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:59 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:28:59.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:28:59 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:59.215 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 14 16 27032 27016 120.531 98.125 0.00610329 0.00829097 2023-09-23T23:28:59.415 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:59 smithi110 ceph-mon[212939]: pgmap v1317: 119 pgs: 119 active+clean; 1.2 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-09-23T23:28:59.415 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:59 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:59.416 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:59 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:28:59.416 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:28:59 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:59.416 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:59 smithi110 ceph-mon[221102]: pgmap v1317: 119 pgs: 119 active+clean; 1.2 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-09-23T23:28:59.416 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:59 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:59.416 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:59 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:28:59.416 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:28:59 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:28:59.745 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:28:59.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:28:59.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:28:59.854 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:28:59.922 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:00.215 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 15 16 28681 28665 119.366 103.062 0.00656546 0.00837101 2023-09-23T23:29:00.388 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:29:01.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:01 smithi110 ceph-mon[212939]: pgmap v1318: 119 pgs: 119 active+clean; 1.5 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-09-23T23:29:01.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:01 smithi110 ceph-mon[221102]: pgmap v1318: 119 pgs: 119 active+clean; 1.5 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-09-23T23:29:01.215 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 16 16 30102 30086 117.457 88.8125 0.00203128 0.00844673 2023-09-23T23:29:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:01 smithi138 ceph-mon[193606]: pgmap v1318: 119 pgs: 119 active+clean; 1.5 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-09-23T23:29:01.457 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:29:01.597 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:01.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:01.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:01.702 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:29:01.806 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:01.898 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:01.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:01.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:02.005 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:02.064 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:02.216 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 17 16 31597 31581 116.044 93.4375 0.0120323 0.00861298 2023-09-23T23:29:02.452 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:02.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:02.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:02.557 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:02.628 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:02.828 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:02.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:02.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:02.921 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:02.935 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:29:02.965 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:02.968 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:03.015 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:29:03.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:03.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:29:03.032 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:03 smithi110 ceph-mon[221102]: pgmap v1319: 119 pgs: 119 active+clean; 1.7 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T23:29:03.032 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:03 smithi110 ceph-mon[212939]: pgmap v1319: 119 pgs: 119 active+clean; 1.7 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T23:29:03.123 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:29:03.191 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:03.227 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 18 16 32981 32965 114.337 86.5 0.0126387 0.00873938 2023-09-23T23:29:03.362 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:03.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:03.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:03.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:03 smithi138 ceph-mon[193606]: pgmap v1319: 119 pgs: 119 active+clean; 1.7 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T23:29:03.479 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:03.544 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:03.811 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:03.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:03.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:03.918 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:29:03.951 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:04.003 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:29:04.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:04.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:29:04.051 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:29:04.085 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:04 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/826768800' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:04.085 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:04 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:04.086 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:04 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/826768800' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:04.086 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:04 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:04.111 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:04.140 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:04.226 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 19 16 34252 34236 112.501 79.4375 0.0150166 0.00888281 2023-09-23T23:29:04.323 INFO:teuthology.orchestra.run.smithi110.stderr:Error EINVAL: specified pg_num 61 < pg_num_max 32 2023-09-23T23:29:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:04 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/826768800' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:04 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:05.226 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:29:05.225935+0000 min lat: 0.00115016 max lat: 0.357929 avg lat: 0.0089533 2023-09-23T23:29:05.227 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:29:05.227 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 20 16 35609 35593 111.117 84.8125 0.00241052 0.0089533 2023-09-23T23:29:05.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:05 smithi138 ceph-mon[193606]: pgmap v1320: 119 pgs: 119 active+clean; 1.7 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T23:29:05.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:05 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:29:05.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:05 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:29:05.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:05 smithi138 ceph-mon[193606]: osdmap e485: 8 total, 8 up, 7 in 2023-09-23T23:29:05.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:05 smithi138 ceph-mon[193606]: failed to encode map e485 with expected crc 2023-09-23T23:29:05.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:05 smithi138 ceph-mon[193606]: failed to encode map e485 with expected crc 2023-09-23T23:29:05.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:05 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1112647299' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-23T23:29:05.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:05 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:29:05.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:05 smithi110 ceph-mon[212939]: pgmap v1320: 119 pgs: 119 active+clean; 1.7 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T23:29:05.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:05 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:29:05.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:05 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:29:05.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:05 smithi110 ceph-mon[212939]: osdmap e485: 8 total, 8 up, 7 in 2023-09-23T23:29:05.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:05 smithi110 ceph-mon[212939]: failed to encode map e485 with expected crc 2023-09-23T23:29:05.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:05 smithi110 ceph-mon[212939]: failed to encode map e485 with expected crc 2023-09-23T23:29:05.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:05 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1112647299' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-23T23:29:05.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:05 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:29:05.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:05 smithi110 ceph-mon[221102]: pgmap v1320: 119 pgs: 119 active+clean; 1.7 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T23:29:05.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:05 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:29:05.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:05 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:29:05.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:05 smithi110 ceph-mon[221102]: osdmap e485: 8 total, 8 up, 7 in 2023-09-23T23:29:05.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:05 smithi110 ceph-mon[221102]: failed to encode map e485 with expected crc 2023-09-23T23:29:05.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:05 smithi110 ceph-mon[221102]: failed to encode map e485 with expected crc 2023-09-23T23:29:05.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:05 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1112647299' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-23T23:29:05.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:05 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:29:06.227 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 21 16 36759 36743 109.25 71.875 0.00287928 0.00913932 2023-09-23T23:29:06.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:06 smithi138 ceph-mon[193606]: failed to encode map e485 with expected crc 2023-09-23T23:29:06.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:06 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:06.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:06 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:06.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:06 smithi110 ceph-mon[221102]: failed to encode map e485 with expected crc 2023-09-23T23:29:06.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:06 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:06.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:06 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:06.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:06 smithi110 ceph-mon[212939]: failed to encode map e485 with expected crc 2023-09-23T23:29:06.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:06 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:06.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:06 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:06.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:29:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:29:06] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.43.0" 2023-09-23T23:29:07.192 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:07.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:07.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:07.227 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 22 16 37982 37966 107.759 76.4375 0.0118238 0.00927513 2023-09-23T23:29:07.298 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:07.342 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:07.402 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:07.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:07.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:07.448 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:07 smithi110 ceph-mon[212939]: pgmap v1322: 119 pgs: 119 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T23:29:07.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:07 smithi110 ceph-mon[221102]: pgmap v1322: 119 pgs: 119 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T23:29:07.514 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:29:07.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:07 smithi138 ceph-mon[193606]: pgmap v1322: 119 pgs: 119 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T23:29:07.559 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:07.630 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T23:29:07.766 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:29:07.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:07.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:29:07.873 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:29:07.912 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:08.080 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:29:08.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:08.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:29:08.088 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:29:08.088 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:29:08.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.088 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:38984 12.6 12:62e041a5:::benchmark_data_smithi110_491193_object38983:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e485)", 2023-09-23T23:29:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:29:08.069017+0000", 2023-09-23T23:29:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0092580379999999997, 2023-09-23T23:29:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0092982789999999996, 2023-09-23T23:29:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:29:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:29:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:29:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 38984 2023-09-23T23:29:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:08.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.069017+0000", 2023-09-23T23:29:08.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:08.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:08.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.069017+0000", 2023-09-23T23:29:08.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:08.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.069015+0000", 2023-09-23T23:29:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-09-23T23:29:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.069045+0000", 2023-09-23T23:29:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.021e-05 2023-09-23T23:29:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.069065+0000", 2023-09-23T23:29:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9377000000000001e-05 2023-09-23T23:29:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.069075+0000", 2023-09-23T23:29:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0242e-05 2023-09-23T23:29:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.069144+0000", 2023-09-23T23:29:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8553000000000006e-05 2023-09-23T23:29:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.069327+0000", 2023-09-23T23:29:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000183764 2023-09-23T23:29:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:29:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.069552+0000", 2023-09-23T23:29:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022486500000000001 2023-09-23T23:29:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.073908+0000", 2023-09-23T23:29:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.004355701 2023-09-23T23:29:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:38988 12.6 12:6f918111:::benchmark_data_smithi110_491193_object38987:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e485)", 2023-09-23T23:29:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:29:08.069260+0000", 2023-09-23T23:29:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0090150950000000007, 2023-09-23T23:29:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0091463139999999991, 2023-09-23T23:29:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:29:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:29:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:29:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 38988 2023-09-23T23:29:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.069260+0000", 2023-09-23T23:29:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.069260+0000", 2023-09-23T23:29:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.069259+0000", 2023-09-23T23:29:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:29:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.069287+0000", 2023-09-23T23:29:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7608000000000001e-05 2023-09-23T23:29:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.069299+0000", 2023-09-23T23:29:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1714e-05 2023-09-23T23:29:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.069308+0000", 2023-09-23T23:29:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6569999999999992e-06 2023-09-23T23:29:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.071144+0000", 2023-09-23T23:29:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018353429999999999 2023-09-23T23:29:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.071292+0000", 2023-09-23T23:29:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014821600000000001 2023-09-23T23:29:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:29:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.071494+0000", 2023-09-23T23:29:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020198899999999999 2023-09-23T23:29:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.073914+0000", 2023-09-23T23:29:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002419895 2023-09-23T23:29:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:38993 12.1 e485/476 12:8e2585db:::benchmark_data_smithi110_491193_object38992:head v 485'2496, mlcod=485'2493)", 2023-09-23T23:29:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:29:08.073337+0000", 2023-09-23T23:29:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0049386760000000004, 2023-09-23T23:29:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051471160000000002, 2023-09-23T23:29:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:29:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.073337+0000", 2023-09-23T23:29:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.073337+0000", 2023-09-23T23:29:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.073336+0000", 2023-09-23T23:29:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.073370+0000", 2023-09-23T23:29:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3278999999999998e-05 2023-09-23T23:29:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.073379+0000", 2023-09-23T23:29:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2e-06 2023-09-23T23:29:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.073384+0000", 2023-09-23T23:29:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2800000000000003e-06 2023-09-23T23:29:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.077834+0000", 2023-09-23T23:29:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044502500000000002 2023-09-23T23:29:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.077862+0000", 2023-09-23T23:29:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7617999999999999e-05 2023-09-23T23:29:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:38992 12.4 e485/476 12:2d8253ae:::benchmark_data_smithi110_491193_object38991:head v 485'2343, mlcod=485'2341)", 2023-09-23T23:29:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:29:08.078080+0000", 2023-09-23T23:29:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00019489900000000001, 2023-09-23T23:29:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047054699999999998, 2023-09-23T23:29:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:29:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.078080+0000", 2023-09-23T23:29:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.078080+0000", 2023-09-23T23:29:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.078079+0000", 2023-09-23T23:29:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:29:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.078107+0000", 2023-09-23T23:29:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7492999999999999e-05 2023-09-23T23:29:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.078120+0000", 2023-09-23T23:29:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3661e-05 2023-09-23T23:29:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.078129+0000", 2023-09-23T23:29:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8030000000000003e-06 2023-09-23T23:29:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.078199+0000", 2023-09-23T23:29:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0166000000000005e-05 2023-09-23T23:29:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:29:08.078220+0000", 2023-09-23T23:29:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0485000000000001e-05 2023-09-23T23:29:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-09-23T23:29:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 4 2023-09-23T23:29:08.109 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:29:08.185 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:08.222 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:08.227 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 23 16 39188 39172 106.352 75.375 0.0162639 0.0093965 2023-09-23T23:29:08.284 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:08.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:08.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:08.325 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:29:08.362 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:29:08.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:08 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:08.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:08 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:08.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:08 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1599254632' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:08.389 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:29:08.402 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:29:08.442 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:29:08.464 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:08.482 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:29:08.522 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:29:08.563 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:08.566 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:29:08.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:08.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:08.598 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:29:08.608 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:29:08.608 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:29:08.608 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:29:08.608 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:29:08.608 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:29:08.609 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:29:08.609 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:29:08.609 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:29:08.635 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:08 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:08.635 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:08 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:08.635 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:08 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1599254632' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:08.679 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:29:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:08 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:08 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:08 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1599254632' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:08.706 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:08.751 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:29:08.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:08.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:29:08.854 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:08.880 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:08.943 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:08.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:08.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:09.049 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:09.122 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:09.227 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 24 16 40241 40225 104.662 65.8125 0.0157917 0.0095486 2023-09-23T23:29:09.441 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:09.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:09.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:09.521 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:29:09.549 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:09.591 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:09.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:09 smithi138 ceph-mon[193606]: pgmap v1323: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T23:29:09.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:09 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:09.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:09 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:09.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:09 smithi110 ceph-mon[221102]: pgmap v1323: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T23:29:09.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:09 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:09.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:09 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:09.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:09 smithi110 ceph-mon[212939]: pgmap v1323: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T23:29:09.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:09 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:09.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:09 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:09.863 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:29:10.152 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:10.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:10.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:10.227 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 25 16 41302 41286 103.128 66.3125 0.013855 0.00968919 2023-09-23T23:29:10.266 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:10.298 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:10.302 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:10.358 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:10.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:10.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:10.465 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:10.522 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:10.665 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:10.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:10.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:10.773 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:10.852 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:11.231 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 26 16 42541 42525 102.129 77.4375 0.0101016 0.00978607 2023-09-23T23:29:11.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:11 smithi138 ceph-mon[193606]: pgmap v1324: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-09-23T23:29:11.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:11 smithi110 ceph-mon[212939]: pgmap v1324: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-09-23T23:29:11.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:11 smithi110 ceph-mon[221102]: pgmap v1324: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-09-23T23:29:12.148 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:29:12.232 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 27 16 43709 43693 101.044 73 0.00838793 0.0098908 2023-09-23T23:29:12.242 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:12.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:12.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:12.345 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:29:12.357 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:12.458 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:12.631 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 7, 2, 1] out_osds: [6] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:29:12.632 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:29:12.632 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T23:29:12.632 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T23:29:12.632 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T23:29:12.632 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T23:29:12.736 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:12.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:12.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:12.852 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:29:12.953 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:13.042 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:29:13.233 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 28 16 44743 44727 99.743 64.625 0.0114692 0.0100202 2023-09-23T23:29:13.269 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:13.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:13.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:13.379 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:29:13.417 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:13.483 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:29:13.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:13.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:29:13.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:13 smithi138 ceph-mon[193606]: pgmap v1325: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T23:29:13.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:29:13.587 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:29:13.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:13 smithi110 ceph-mon[212939]: pgmap v1325: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T23:29:13.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:29:13.611 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:13 smithi110 ceph-mon[221102]: pgmap v1325: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T23:29:13.611 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:29:13.613 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:14.061 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:29:14.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:14.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:29:14.172 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:14.184 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:14.233 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 29 16 45810 45794 98.604 66.6875 0.0275479 0.0101343 2023-09-23T23:29:14.253 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:14.413 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:14.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:14.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:14.517 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:29:14.598 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:14.629 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:29:14.668 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:14 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2722377295' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:29:14.668 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:14 smithi110 ceph-mon[212939]: pgmap v1326: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T23:29:14.669 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:14 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2722377295' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:29:14.669 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:14 smithi110 ceph-mon[221102]: pgmap v1326: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T23:29:14.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:14 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2722377295' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:29:14.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:14 smithi138 ceph-mon[193606]: pgmap v1326: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T23:29:15.233 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 30 16 46972 46956 97.7384 72.625 0.0037178 0.0102233 2023-09-23T23:29:15.822 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:29:15.823 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:29:15.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:15 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:29:15.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:15 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2722377295' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:29:15.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:15 smithi138 ceph-mon[193606]: osdmap e486: 8 total, 8 up, 7 in 2023-09-23T23:29:15.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:15 smithi138 ceph-mon[193606]: failed to encode map e486 with expected crc 2023-09-23T23:29:15.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:15 smithi138 ceph-mon[193606]: failed to encode map e486 with expected crc 2023-09-23T23:29:15.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:15 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:29:15.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:15 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2722377295' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:29:15.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:15 smithi110 ceph-mon[221102]: osdmap e486: 8 total, 8 up, 7 in 2023-09-23T23:29:15.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:15 smithi110 ceph-mon[221102]: failed to encode map e486 with expected crc 2023-09-23T23:29:15.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:15 smithi110 ceph-mon[221102]: failed to encode map e486 with expected crc 2023-09-23T23:29:15.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:15 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:29:15.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:15 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2722377295' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:29:15.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:15 smithi110 ceph-mon[212939]: osdmap e486: 8 total, 8 up, 7 in 2023-09-23T23:29:15.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:15 smithi110 ceph-mon[212939]: failed to encode map e486 with expected crc 2023-09-23T23:29:15.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:15 smithi110 ceph-mon[212939]: failed to encode map e486 with expected crc 2023-09-23T23:29:16.108 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:29:16.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:16.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:29:16.221 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:29:16.233 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 31 16 47892 47876 96.4413 57.5 0.00814823 0.010364 2023-09-23T23:29:16.254 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:16.298 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:29:16.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:16.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:29:16.401 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:29:16.429 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:16.475 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:29:16.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:16.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:29:16.579 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:16.612 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:16.660 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:16.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:16.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:16.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:16 smithi110 ceph-mon[212939]: 1.f scrub starts 2023-09-23T23:29:16.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:16 smithi110 ceph-mon[212939]: 1.f scrub ok 2023-09-23T23:29:16.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:16 smithi110 ceph-mon[212939]: pgmap v1328: 119 pgs: 119 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T23:29:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:16 smithi110 ceph-mon[221102]: 1.f scrub starts 2023-09-23T23:29:16.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:16 smithi110 ceph-mon[221102]: 1.f scrub ok 2023-09-23T23:29:16.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:16 smithi110 ceph-mon[221102]: pgmap v1328: 119 pgs: 119 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T23:29:16.763 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:16.811 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:16 smithi138 ceph-mon[193606]: 1.f scrub starts 2023-09-23T23:29:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:16 smithi138 ceph-mon[193606]: 1.f scrub ok 2023-09-23T23:29:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:16 smithi138 ceph-mon[193606]: pgmap v1328: 119 pgs: 119 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T23:29:17.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:29:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:29:16] "GET /metrics HTTP/1.1" 200 36520 "" "Prometheus/2.43.0" 2023-09-23T23:29:17.235 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 32 16 48580 48564 94.7652 43 0.00860959 0.0105463 2023-09-23T23:29:17.328 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:17.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:17.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:17.417 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:29:17.437 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:17.465 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:17.702 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:17.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:17.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:17.806 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:29:17.843 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:17.934 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:29:17.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:17.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:29:18.038 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:18.116 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:18.137 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:29:18.236 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 33 16 49668 49652 93.9546 68 0.00951988 0.0106364 2023-09-23T23:29:18.297 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:29:18.297 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "" 2023-09-23T23:29:18.297 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:29:18.314 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:29:18.315 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:18.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:18.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:18.342 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:29:18.400 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:29:18.432 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:18.459 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:29:18.503 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:29:18.503 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:18.525 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:18.533 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:29:18.578 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:29:18.719 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:18.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:18.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:18.821 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:29:18.852 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:18.859 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:29:18.869 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:29:18.869 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:29:18.869 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:29:18.869 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:29:18.870 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:29:18.870 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:29:18.870 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:29:18.870 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:29:18.908 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:29:18.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:18.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:29:19.014 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:29:19.111 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:18 smithi110 ceph-mon[212939]: pgmap v1329: 119 pgs: 119 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T23:29:19.111 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/976453043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:19.111 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:18 smithi110 ceph-mon[221102]: pgmap v1329: 119 pgs: 119 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T23:29:19.111 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/976453043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:19.115 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:19.187 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:18 smithi138 ceph-mon[193606]: pgmap v1329: 119 pgs: 119 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T23:29:19.187 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/976453043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:19.236 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 34 16 50445 50429 92.6204 48.5625 0.00881522 0.0107907 2023-09-23T23:29:20.236 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 35 16 51586 51570 92.0119 71.3125 0.0114805 0.0108633 2023-09-23T23:29:20.994 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:21.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:21.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:21.002 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:20 smithi110 ceph-mon[212939]: pgmap v1330: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T23:29:21.002 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:20 smithi110 ceph-mon[221102]: pgmap v1330: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T23:29:21.103 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:29:21.130 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:20 smithi138 ceph-mon[193606]: pgmap v1330: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T23:29:21.133 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:21.186 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:29:21.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:21.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:29:21.236 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 36 16 52514 52498 91.0674 58 0.0120187 0.0109751 2023-09-23T23:29:21.294 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:21.317 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:21.366 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:21.483 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:21.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:21.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:21.590 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:21.621 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:21.674 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:21.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:21.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:21.780 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:29:21.881 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:21.947 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:29:21.947 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:29:21.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:29:21.947 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:29:21.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.947 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135622 11.c 11:38fc4d47:::benchmark_data_smithi110_433746_object29742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:29:21.947 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.568262+0000", 2023-09-23T23:29:21.947 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 191.37243159900001, 2023-09-23T23:29:21.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7526951390000001, 2023-09-23T23:29:21.947 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:21.947 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:21.948 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:21.948 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:21.948 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:21.948 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135622 2023-09-23T23:29:21.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.948 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:21.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:21.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568262+0000", 2023-09-23T23:29:21.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:21.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:21.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568262+0000", 2023-09-23T23:29:21.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:21.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:21.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568260+0000", 2023-09-23T23:29:21.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:29:21.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:21.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568267+0000", 2023-09-23T23:29:21.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5010000000000001e-06 2023-09-23T23:29:21.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:21.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568277+0000", 2023-09-23T23:29:21.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0165e-05 2023-09-23T23:29:21.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:21.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568284+0000", 2023-09-23T23:29:21.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5689999999999999e-06 2023-09-23T23:29:21.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568341+0000", 2023-09-23T23:29:21.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6894000000000001e-05 2023-09-23T23:29:21.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:21.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568349+0000", 2023-09-23T23:29:21.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3669999999999999e-06 2023-09-23T23:29:21.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505217+0000", 2023-09-23T23:29:21.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93686795 2023-09-23T23:29:21.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:21.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505229+0000", 2023-09-23T23:29:21.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2274e-05 2023-09-23T23:29:21.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.305074+0000", 2023-09-23T23:29:21.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.799845516 2023-09-23T23:29:21.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:21.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.305205+0000", 2023-09-23T23:29:21.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013038200000000001 2023-09-23T23:29:21.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:29:21.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.305293+0000", 2023-09-23T23:29:21.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8187000000000001e-05 2023-09-23T23:29:21.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:21.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.308879+0000", 2023-09-23T23:29:21.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035864999999999998 2023-09-23T23:29:21.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.954 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:21.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.308902+0000", 2023-09-23T23:29:21.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2161999999999999e-05 2023-09-23T23:29:21.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.954 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:21.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320708+0000", 2023-09-23T23:29:21.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011806535999999999 2023-09-23T23:29:21.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:21.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320814+0000", 2023-09-23T23:29:21.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105544 2023-09-23T23:29:21.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:21.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320848+0000", 2023-09-23T23:29:21.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4674e-05 2023-09-23T23:29:21.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:21.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320957+0000", 2023-09-23T23:29:21.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000108328 2023-09-23T23:29:21.956 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:21.956 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:21.956 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:21.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.956 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135633 11.c 11:3e1f1508:::benchmark_data_smithi110_433746_object29753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:29:21.956 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.573921+0000", 2023-09-23T23:29:21.956 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 191.36677208, 2023-09-23T23:29:21.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7472477729999998, 2023-09-23T23:29:21.956 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:21.957 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:21.957 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:21.957 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:21.957 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:21.957 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135633 2023-09-23T23:29:21.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.957 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:21.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:21.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573921+0000", 2023-09-23T23:29:21.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:21.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:21.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573921+0000", 2023-09-23T23:29:21.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:21.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:21.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573919+0000", 2023-09-23T23:29:21.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:29:21.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:21.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573926+0000", 2023-09-23T23:29:21.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2060000000000004e-06 2023-09-23T23:29:21.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:21.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573941+0000", 2023-09-23T23:29:21.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5325999999999999e-05 2023-09-23T23:29:21.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:21.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573949+0000", 2023-09-23T23:29:21.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1780000000000003e-06 2023-09-23T23:29:21.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.574200+0000", 2023-09-23T23:29:21.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025123000000000002 2023-09-23T23:29:21.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:21.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.574213+0000", 2023-09-23T23:29:21.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2498e-05 2023-09-23T23:29:21.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505252+0000", 2023-09-23T23:29:21.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93103919199999996 2023-09-23T23:29:21.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:21.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505260+0000", 2023-09-23T23:29:21.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2320000000000001e-06 2023-09-23T23:29:21.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.308916+0000", 2023-09-23T23:29:21.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.803655368 2023-09-23T23:29:21.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:21.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.309001+0000", 2023-09-23T23:29:21.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5617000000000004e-05 2023-09-23T23:29:21.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:29:21.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.309074+0000", 2023-09-23T23:29:21.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2853000000000002e-05 2023-09-23T23:29:21.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:21.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320731+0000", 2023-09-23T23:29:21.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011657351999999999 2023-09-23T23:29:21.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:21.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320771+0000", 2023-09-23T23:29:21.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9177000000000001e-05 2023-09-23T23:29:21.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:21.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320796+0000", 2023-09-23T23:29:21.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5031000000000001e-05 2023-09-23T23:29:21.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:21.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321024+0000", 2023-09-23T23:29:21.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000228156 2023-09-23T23:29:21.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:21.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321066+0000", 2023-09-23T23:29:21.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2126999999999998e-05 2023-09-23T23:29:21.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:21.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321169+0000", 2023-09-23T23:29:21.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010292 2023-09-23T23:29:21.965 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:21.965 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:21.965 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:21.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.965 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135638 11.c 11:32116872:::benchmark_data_smithi110_433746_object29758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:29:21.965 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.579560+0000", 2023-09-23T23:29:21.965 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 191.36113366500001, 2023-09-23T23:29:21.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7417568370000001, 2023-09-23T23:29:21.966 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:21.966 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:21.966 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:21.966 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:21.966 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:21.966 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135638 2023-09-23T23:29:21.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.966 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:21.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:21.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579560+0000", 2023-09-23T23:29:21.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:21.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:21.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579560+0000", 2023-09-23T23:29:21.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:21.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:21.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579558+0000", 2023-09-23T23:29:21.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:29:21.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:21.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579565+0000", 2023-09-23T23:29:21.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1459999999999999e-06 2023-09-23T23:29:21.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:21.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579578+0000", 2023-09-23T23:29:21.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3019e-05 2023-09-23T23:29:21.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:21.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579587+0000", 2023-09-23T23:29:21.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6999999999999997e-06 2023-09-23T23:29:21.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579670+0000", 2023-09-23T23:29:21.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2934000000000003e-05 2023-09-23T23:29:21.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:21.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579680+0000", 2023-09-23T23:29:21.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0742e-05 2023-09-23T23:29:21.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505282+0000", 2023-09-23T23:29:21.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.92560134699999996 2023-09-23T23:29:21.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:21.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505290+0000", 2023-09-23T23:29:21.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8380000000000002e-06 2023-09-23T23:29:21.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.312113+0000", 2023-09-23T23:29:21.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8068231020000001 2023-09-23T23:29:21.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:21.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.312237+0000", 2023-09-23T23:29:21.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012441899999999999 2023-09-23T23:29:21.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:29:21.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.312331+0000", 2023-09-23T23:29:21.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3719000000000002e-05 2023-09-23T23:29:21.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:21.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320776+0000", 2023-09-23T23:29:21.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084449720000000002 2023-09-23T23:29:21.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:21.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320988+0000", 2023-09-23T23:29:21.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021210700000000001 2023-09-23T23:29:21.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:21.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321006+0000", 2023-09-23T23:29:21.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8111999999999999e-05 2023-09-23T23:29:21.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:21.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321204+0000", 2023-09-23T23:29:21.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000197796 2023-09-23T23:29:21.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:21.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321233+0000", 2023-09-23T23:29:21.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9108999999999999e-05 2023-09-23T23:29:21.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:21.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321316+0000", 2023-09-23T23:29:21.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3517999999999996e-05 2023-09-23T23:29:21.974 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:21.974 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:21.974 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:21.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.974 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135640 11.c 11:3dfd0302:::benchmark_data_smithi110_433746_object29760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:29:21.974 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.585524+0000", 2023-09-23T23:29:21.974 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 191.35516869599999, 2023-09-23T23:29:21.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7460455509999999, 2023-09-23T23:29:21.975 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:21.975 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:21.975 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:21.975 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:21.975 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:21.975 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135640 2023-09-23T23:29:21.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.975 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:21.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:21.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585524+0000", 2023-09-23T23:29:21.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:21.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:21.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585524+0000", 2023-09-23T23:29:21.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:21.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:21.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585522+0000", 2023-09-23T23:29:21.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-09-23T23:29:21.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:21.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585530+0000", 2023-09-23T23:29:21.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0099999999999995e-06 2023-09-23T23:29:21.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:21.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585582+0000", 2023-09-23T23:29:21.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2074999999999999e-05 2023-09-23T23:29:21.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:21.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585593+0000", 2023-09-23T23:29:21.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1155000000000001e-05 2023-09-23T23:29:21.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585657+0000", 2023-09-23T23:29:21.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3563000000000001e-05 2023-09-23T23:29:21.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:21.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585669+0000", 2023-09-23T23:29:21.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2368000000000001e-05 2023-09-23T23:29:21.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505310+0000", 2023-09-23T23:29:21.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.91964102400000003 2023-09-23T23:29:21.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:21.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505318+0000", 2023-09-23T23:29:21.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7349999999999996e-06 2023-09-23T23:29:21.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.315785+0000", 2023-09-23T23:29:21.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.810466366 2023-09-23T23:29:21.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:21.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.315917+0000", 2023-09-23T23:29:21.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013256300000000001 2023-09-23T23:29:21.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:29:21.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.316041+0000", 2023-09-23T23:29:21.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012345100000000001 2023-09-23T23:29:21.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:21.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321343+0000", 2023-09-23T23:29:21.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053020849999999998 2023-09-23T23:29:21.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:21.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331351+0000", 2023-09-23T23:29:21.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010008193 2023-09-23T23:29:21.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:21.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331412+0000", 2023-09-23T23:29:21.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1023000000000002e-05 2023-09-23T23:29:21.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:21.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331438+0000", 2023-09-23T23:29:21.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6432999999999999e-05 2023-09-23T23:29:21.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:21.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331471+0000", 2023-09-23T23:29:21.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3127999999999999e-05 2023-09-23T23:29:21.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:21.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331570+0000", 2023-09-23T23:29:21.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8610999999999995e-05 2023-09-23T23:29:21.984 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:21.984 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:21.984 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:21.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.984 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135653 11.c 11:39fc45c5:::benchmark_data_smithi110_433746_object29773:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:29:21.984 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.612482+0000", 2023-09-23T23:29:21.984 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 191.328211678, 2023-09-23T23:29:21.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7224649520000002, 2023-09-23T23:29:21.984 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:21.984 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:21.985 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:21.985 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:21.985 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:21.985 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135653 2023-09-23T23:29:21.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.985 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:21.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:21.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612482+0000", 2023-09-23T23:29:21.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:21.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:21.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612482+0000", 2023-09-23T23:29:21.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:21.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:21.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612480+0000", 2023-09-23T23:29:21.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:29:21.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:21.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612487+0000", 2023-09-23T23:29:21.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2640000000000002e-06 2023-09-23T23:29:21.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:21.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612499+0000", 2023-09-23T23:29:21.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2116000000000001e-05 2023-09-23T23:29:21.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:21.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612509+0000", 2023-09-23T23:29:21.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6940000000000007e-06 2023-09-23T23:29:21.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612569+0000", 2023-09-23T23:29:21.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9738999999999998e-05 2023-09-23T23:29:21.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:21.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612579+0000", 2023-09-23T23:29:21.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0592e-05 2023-09-23T23:29:21.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505353+0000", 2023-09-23T23:29:21.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.89277419800000002 2023-09-23T23:29:21.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:21.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505365+0000", 2023-09-23T23:29:21.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1090999999999999e-05 2023-09-23T23:29:21.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321356+0000", 2023-09-23T23:29:21.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8159917220000001 2023-09-23T23:29:21.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:21.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321466+0000", 2023-09-23T23:29:21.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109347 2023-09-23T23:29:21.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:29:21.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321551+0000", 2023-09-23T23:29:21.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5669999999999998e-05 2023-09-23T23:29:21.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:21.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331357+0000", 2023-09-23T23:29:21.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00980549 2023-09-23T23:29:21.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:21.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331615+0000", 2023-09-23T23:29:21.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025800399999999999 2023-09-23T23:29:21.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:21.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331637+0000", 2023-09-23T23:29:21.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2036e-05 2023-09-23T23:29:21.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:21.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334781+0000", 2023-09-23T23:29:21.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031445589999999999 2023-09-23T23:29:21.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:21.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334836+0000", 2023-09-23T23:29:21.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4469999999999999e-05 2023-09-23T23:29:21.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:21.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334946+0000", 2023-09-23T23:29:21.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110579 2023-09-23T23:29:21.993 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:21.993 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:21.993 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:21.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.993 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135663 11.c 11:3bd42e62:::benchmark_data_smithi110_433746_object29783:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:29:21.993 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.646633+0000", 2023-09-23T23:29:21.993 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 191.294060527, 2023-09-23T23:29:21.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.691323304, 2023-09-23T23:29:21.993 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:21.994 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:21.994 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:21.994 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:21.994 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:21.994 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135663 2023-09-23T23:29:21.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.994 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:21.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:21.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646633+0000", 2023-09-23T23:29:21.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:21.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:21.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646633+0000", 2023-09-23T23:29:21.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:21.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:21.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646631+0000", 2023-09-23T23:29:21.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:29:21.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:21.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646638+0000", 2023-09-23T23:29:21.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9949999999999999e-06 2023-09-23T23:29:21.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:21.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646653+0000", 2023-09-23T23:29:21.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4892000000000001e-05 2023-09-23T23:29:21.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:21.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646661+0000", 2023-09-23T23:29:21.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9929999999999995e-06 2023-09-23T23:29:21.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646713+0000", 2023-09-23T23:29:21.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1544e-05 2023-09-23T23:29:21.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:21.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646725+0000", 2023-09-23T23:29:21.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.223e-05 2023-09-23T23:29:21.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505389+0000", 2023-09-23T23:29:21.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.858663756 2023-09-23T23:29:21.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:21.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505397+0000", 2023-09-23T23:29:21.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4670000000000004e-06 2023-09-23T23:29:21.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:21.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.324831+0000", 2023-09-23T23:29:21.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.819433461 2023-09-23T23:29:21.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:21.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.324942+0000", 2023-09-23T23:29:21.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011171900000000001 2023-09-23T23:29:21.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:29:21.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.325072+0000", 2023-09-23T23:29:21.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012980000000000001 2023-09-23T23:29:21.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:21.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:21.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331656+0000", 2023-09-23T23:29:22.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0065834639999999998 2023-09-23T23:29:22.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.332662+0000", 2023-09-23T23:29:22.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010060049999999999 2023-09-23T23:29:22.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:22.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.333742+0000", 2023-09-23T23:29:22.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010798170000000001 2023-09-23T23:29:22.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337799+0000", 2023-09-23T23:29:22.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040573579999999996 2023-09-23T23:29:22.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:22.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337850+0000", 2023-09-23T23:29:22.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0818e-05 2023-09-23T23:29:22.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:22.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337956+0000", 2023-09-23T23:29:22.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010621 2023-09-23T23:29:22.002 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.002 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:22.002 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.002 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135680 11.c 11:34dfa950:::benchmark_data_smithi110_433746_object29800:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:29:22.002 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.699865+0000", 2023-09-23T23:29:22.002 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 191.24082810300001, 2023-09-23T23:29:22.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6382614580000001, 2023-09-23T23:29:22.002 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:22.002 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:22.003 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:22.003 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:22.003 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:22.003 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135680 2023-09-23T23:29:22.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.003 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:22.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:22.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699865+0000", 2023-09-23T23:29:22.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:22.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699865+0000", 2023-09-23T23:29:22.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:22.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699864+0000", 2023-09-23T23:29:22.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:29:22.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:22.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699870+0000", 2023-09-23T23:29:22.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9329999999999998e-06 2023-09-23T23:29:22.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:22.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699886+0000", 2023-09-23T23:29:22.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6402e-05 2023-09-23T23:29:22.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:22.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699896+0000", 2023-09-23T23:29:22.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3889999999999994e-06 2023-09-23T23:29:22.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699953+0000", 2023-09-23T23:29:22.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7763999999999998e-05 2023-09-23T23:29:22.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:22.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699968+0000", 2023-09-23T23:29:22.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4749999999999999e-05 2023-09-23T23:29:22.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505806+0000", 2023-09-23T23:29:22.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.80583805100000006 2023-09-23T23:29:22.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:22.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505816+0000", 2023-09-23T23:29:22.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9320000000000008e-06 2023-09-23T23:29:22.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.328139+0000", 2023-09-23T23:29:22.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8223231129999999 2023-09-23T23:29:22.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:22.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.328247+0000", 2023-09-23T23:29:22.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010777600000000001 2023-09-23T23:29:22.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:29:22.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.328367+0000", 2023-09-23T23:29:22.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012031000000000001 2023-09-23T23:29:22.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331675+0000", 2023-09-23T23:29:22.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033076669999999998 2023-09-23T23:29:22.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.332701+0000", 2023-09-23T23:29:22.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010263959999999999 2023-09-23T23:29:22.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:22.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.333784+0000", 2023-09-23T23:29:22.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001082911 2023-09-23T23:29:22.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337990+0000", 2023-09-23T23:29:22.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042062970000000003 2023-09-23T23:29:22.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:22.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.338026+0000", 2023-09-23T23:29:22.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5335e-05 2023-09-23T23:29:22.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:22.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.338127+0000", 2023-09-23T23:29:22.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000100717 2023-09-23T23:29:22.011 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.011 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:22.011 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.011 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178625 11.7 11.bda50607 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:29:22.011 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.600905+0000", 2023-09-23T23:29:22.011 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 122.33978817400001, 2023-09-23T23:29:22.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93901017899999994, 2023-09-23T23:29:22.011 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:22.011 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:29:22.011 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:22.012 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:22.012 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:22.012 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178625 2023-09-23T23:29:22.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.012 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:22.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:22.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.600905+0000", 2023-09-23T23:29:22.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:22.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.600905+0000", 2023-09-23T23:29:22.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:22.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.600903+0000", 2023-09-23T23:29:22.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:29:22.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:22.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.600910+0000", 2023-09-23T23:29:22.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1080000000000004e-06 2023-09-23T23:29:22.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:22.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.600922+0000", 2023-09-23T23:29:22.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.169e-05 2023-09-23T23:29:22.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:22.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.600929+0000", 2023-09-23T23:29:22.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.798e-06 2023-09-23T23:29:22.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.600968+0000", 2023-09-23T23:29:22.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9560000000000001e-05 2023-09-23T23:29:22.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:22.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.600978+0000", 2023-09-23T23:29:22.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5189999999999998e-06 2023-09-23T23:29:22.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.539904+0000", 2023-09-23T23:29:22.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93892621499999995 2023-09-23T23:29:22.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:22.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.539915+0000", 2023-09-23T23:29:22.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1126e-05 2023-09-23T23:29:22.016 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.016 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:22.016 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.016 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:202131 11.17 11:efc6c41a:::benchmark_data_smithi110_433746_object96251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T23:29:22.017 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:00.397599+0000", 2023-09-23T23:29:22.017 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.543093872, 2023-09-23T23:29:22.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98048151299999997, 2023-09-23T23:29:22.017 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:22.017 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:22.017 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:22.017 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:22.017 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:22.017 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 202131 2023-09-23T23:29:22.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.017 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:22.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:22.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397599+0000", 2023-09-23T23:29:22.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:22.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397599+0000", 2023-09-23T23:29:22.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:22.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397599+0000", 2023-09-23T23:29:22.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:22.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:22.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397603+0000", 2023-09-23T23:29:22.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5870000000000003e-06 2023-09-23T23:29:22.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:22.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397615+0000", 2023-09-23T23:29:22.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1591e-05 2023-09-23T23:29:22.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:22.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397622+0000", 2023-09-23T23:29:22.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5009999999999998e-06 2023-09-23T23:29:22.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397741+0000", 2023-09-23T23:29:22.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118649 2023-09-23T23:29:22.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:22.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397749+0000", 2023-09-23T23:29:22.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4950000000000008e-06 2023-09-23T23:29:22.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.364031+0000", 2023-09-23T23:29:22.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96628133299999996 2023-09-23T23:29:22.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:22.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.364148+0000", 2023-09-23T23:29:22.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011695499999999999 2023-09-23T23:29:22.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:29:22.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.364473+0000", 2023-09-23T23:29:22.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032511900000000002 2023-09-23T23:29:22.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:22.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.370881+0000", 2023-09-23T23:29:22.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0064080700000000001 2023-09-23T23:29:22.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.377963+0000", 2023-09-23T23:29:22.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070824929999999996 2023-09-23T23:29:22.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378001+0000", 2023-09-23T23:29:22.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7595999999999997e-05 2023-09-23T23:29:22.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:22.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378043+0000", 2023-09-23T23:29:22.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2283999999999997e-05 2023-09-23T23:29:22.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:22.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378081+0000", 2023-09-23T23:29:22.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7478999999999998e-05 2023-09-23T23:29:22.024 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.024 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:22.024 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.024 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:202151 11.17 11:eead75a8:::benchmark_data_smithi110_433746_object96271:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T23:29:22.024 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:00.411787+0000", 2023-09-23T23:29:22.025 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.528905829999999, 2023-09-23T23:29:22.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96643706600000001, 2023-09-23T23:29:22.025 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:22.026 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:22.026 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:22.026 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:22.026 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:22.026 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 202151 2023-09-23T23:29:22.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.026 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:22.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:22.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411787+0000", 2023-09-23T23:29:22.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:22.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411787+0000", 2023-09-23T23:29:22.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:22.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411786+0000", 2023-09-23T23:29:22.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:29:22.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:22.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411792+0000", 2023-09-23T23:29:22.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3799999999999999e-06 2023-09-23T23:29:22.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:22.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411810+0000", 2023-09-23T23:29:22.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7751999999999999e-05 2023-09-23T23:29:22.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:22.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411822+0000", 2023-09-23T23:29:22.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.201e-05 2023-09-23T23:29:22.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411868+0000", 2023-09-23T23:29:22.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5695000000000002e-05 2023-09-23T23:29:22.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:22.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411876+0000", 2023-09-23T23:29:22.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6209999999999993e-06 2023-09-23T23:29:22.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.368736+0000", 2023-09-23T23:29:22.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95685995199999996 2023-09-23T23:29:22.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:22.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.368844+0000", 2023-09-23T23:29:22.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107253 2023-09-23T23:29:22.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:29:22.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.369149+0000", 2023-09-23T23:29:22.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030546099999999998 2023-09-23T23:29:22.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:22.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.377918+0000", 2023-09-23T23:29:22.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0087685660000000002 2023-09-23T23:29:22.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378118+0000", 2023-09-23T23:29:22.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020053900000000001 2023-09-23T23:29:22.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378148+0000", 2023-09-23T23:29:22.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9896999999999999e-05 2023-09-23T23:29:22.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:22.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378190+0000", 2023-09-23T23:29:22.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1541000000000001e-05 2023-09-23T23:29:22.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:22.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378224+0000", 2023-09-23T23:29:22.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4879999999999998e-05 2023-09-23T23:29:22.033 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.033 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:22.033 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.033 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:202156 11.17 11:ef634d51:::benchmark_data_smithi110_433746_object96276:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T23:29:22.033 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:00.415723+0000", 2023-09-23T23:29:22.033 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.524970174999993, 2023-09-23T23:29:22.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96665629500000005, 2023-09-23T23:29:22.033 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:22.034 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:22.034 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:22.034 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:22.034 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:22.034 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 202156 2023-09-23T23:29:22.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.034 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:22.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:22.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415723+0000", 2023-09-23T23:29:22.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:22.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415723+0000", 2023-09-23T23:29:22.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:22.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415723+0000", 2023-09-23T23:29:22.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:22.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:22.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415725+0000", 2023-09-23T23:29:22.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8040000000000002e-06 2023-09-23T23:29:22.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:22.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415732+0000", 2023-09-23T23:29:22.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6549999999999998e-06 2023-09-23T23:29:22.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:22.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415738+0000", 2023-09-23T23:29:22.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5940000000000001e-06 2023-09-23T23:29:22.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415772+0000", 2023-09-23T23:29:22.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3912000000000003e-05 2023-09-23T23:29:22.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:22.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415778+0000", 2023-09-23T23:29:22.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9939999999999997e-06 2023-09-23T23:29:22.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.370907+0000", 2023-09-23T23:29:22.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95512952699999998 2023-09-23T23:29:22.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:22.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.371032+0000", 2023-09-23T23:29:22.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012525899999999999 2023-09-23T23:29:22.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:29:22.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.371307+0000", 2023-09-23T23:29:22.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027482299999999999 2023-09-23T23:29:22.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378260+0000", 2023-09-23T23:29:22.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0069533099999999999 2023-09-23T23:29:22.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378290+0000", 2023-09-23T23:29:22.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9379000000000001e-05 2023-09-23T23:29:22.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:22.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.382296+0000", 2023-09-23T23:29:22.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040064779999999999 2023-09-23T23:29:22.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:22.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.382337+0000", 2023-09-23T23:29:22.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1029000000000003e-05 2023-09-23T23:29:22.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:22.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.382379+0000", 2023-09-23T23:29:22.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1980999999999997e-05 2023-09-23T23:29:22.041 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.041 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:22.041 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.041 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:210997 11.17 11:eefcf624:::benchmark_data_smithi110_433746_object105117:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:29:22.041 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686483+0000", 2023-09-23T23:29:22.041 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.254210315000002, 2023-09-23T23:29:22.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0169343049999999, 2023-09-23T23:29:22.042 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:22.042 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:22.042 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:22.042 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:22.042 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:22.042 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 210997 2023-09-23T23:29:22.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.042 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:22.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:22.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686483+0000", 2023-09-23T23:29:22.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:22.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686483+0000", 2023-09-23T23:29:22.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:22.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686481+0000", 2023-09-23T23:29:22.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:29:22.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:22.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686489+0000", 2023-09-23T23:29:22.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.439e-06 2023-09-23T23:29:22.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:22.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686502+0000", 2023-09-23T23:29:22.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2376e-05 2023-09-23T23:29:22.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:22.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686523+0000", 2023-09-23T23:29:22.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0860999999999999e-05 2023-09-23T23:29:22.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687605+0000", 2023-09-23T23:29:22.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001082698 2023-09-23T23:29:22.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:22.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687618+0000", 2023-09-23T23:29:22.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2848e-05 2023-09-23T23:29:22.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.694585+0000", 2023-09-23T23:29:22.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.006967221 2023-09-23T23:29:22.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:22.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.694712+0000", 2023-09-23T23:29:22.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012615400000000001 2023-09-23T23:29:22.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:29:22.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.695012+0000", 2023-09-23T23:29:22.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030036499999999997 2023-09-23T23:29:22.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:22.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.698738+0000", 2023-09-23T23:29:22.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003726203 2023-09-23T23:29:22.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.698996+0000", 2023-09-23T23:29:22.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025763799999999998 2023-09-23T23:29:22.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.699037+0000", 2023-09-23T23:29:22.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1517000000000002e-05 2023-09-23T23:29:22.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703242+0000", 2023-09-23T23:29:22.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042050509999999996 2023-09-23T23:29:22.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:22.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703295+0000", 2023-09-23T23:29:22.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2997000000000001e-05 2023-09-23T23:29:22.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:22.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703417+0000", 2023-09-23T23:29:22.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012184900000000001 2023-09-23T23:29:22.049 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.049 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:22.049 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.050 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211002 11.17 11:ecab426b:::benchmark_data_smithi110_433746_object105122:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:29:22.050 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686530+0000", 2023-09-23T23:29:22.050 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.254162688999997, 2023-09-23T23:29:22.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0172692640000001, 2023-09-23T23:29:22.050 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:22.050 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:22.050 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:22.050 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:22.050 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:22.050 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211002 2023-09-23T23:29:22.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.051 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:22.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:22.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686530+0000", 2023-09-23T23:29:22.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:22.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686530+0000", 2023-09-23T23:29:22.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:22.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686530+0000", 2023-09-23T23:29:22.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:22.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:22.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686534+0000", 2023-09-23T23:29:22.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9709999999999998e-06 2023-09-23T23:29:22.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:22.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686553+0000", 2023-09-23T23:29:22.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9241000000000001e-05 2023-09-23T23:29:22.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:22.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686560+0000", 2023-09-23T23:29:22.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9260000000000003e-06 2023-09-23T23:29:22.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687641+0000", 2023-09-23T23:29:22.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001081226 2023-09-23T23:29:22.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:22.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687650+0000", 2023-09-23T23:29:22.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.833e-06 2023-09-23T23:29:22.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.696593+0000", 2023-09-23T23:29:22.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0089432190000001 2023-09-23T23:29:22.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:22.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.696740+0000", 2023-09-23T23:29:22.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014711800000000001 2023-09-23T23:29:22.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:29:22.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.697010+0000", 2023-09-23T23:29:22.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026912600000000001 2023-09-23T23:29:22.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:22.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.700956+0000", 2023-09-23T23:29:22.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039468569999999998 2023-09-23T23:29:22.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703189+0000", 2023-09-23T23:29:22.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022322599999999998 2023-09-23T23:29:22.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703471+0000", 2023-09-23T23:29:22.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028231600000000001 2023-09-23T23:29:22.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703641+0000", 2023-09-23T23:29:22.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000170404 2023-09-23T23:29:22.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:22.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703696+0000", 2023-09-23T23:29:22.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4475999999999998e-05 2023-09-23T23:29:22.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:22.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703800+0000", 2023-09-23T23:29:22.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103943 2023-09-23T23:29:22.058 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.058 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:22.058 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.058 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211006 11.17 11:ee6c8b78:::benchmark_data_smithi110_433746_object105126:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:29:22.058 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686565+0000", 2023-09-23T23:29:22.058 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.254128168000001, 2023-09-23T23:29:22.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0175006170000001, 2023-09-23T23:29:22.058 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:22.059 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:22.059 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:22.059 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:22.059 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:22.059 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211006 2023-09-23T23:29:22.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.059 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:22.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:22.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686565+0000", 2023-09-23T23:29:22.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:22.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686565+0000", 2023-09-23T23:29:22.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:22.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686565+0000", 2023-09-23T23:29:22.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:22.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:22.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686568+0000", 2023-09-23T23:29:22.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2600000000000001e-06 2023-09-23T23:29:22.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.061 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:22.061 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686574+0000", 2023-09-23T23:29:22.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0789999999999997e-06 2023-09-23T23:29:22.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.061 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:22.061 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686579+0000", 2023-09-23T23:29:22.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3279999999999996e-06 2023-09-23T23:29:22.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687673+0000", 2023-09-23T23:29:22.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001093966 2023-09-23T23:29:22.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:22.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687681+0000", 2023-09-23T23:29:22.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9889999999999992e-06 2023-09-23T23:29:22.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.699069+0000", 2023-09-23T23:29:22.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.011387475 2023-09-23T23:29:22.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:22.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.699178+0000", 2023-09-23T23:29:22.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109238 2023-09-23T23:29:22.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:29:22.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.699384+0000", 2023-09-23T23:29:22.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020630299999999999 2023-09-23T23:29:22.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:22.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703201+0000", 2023-09-23T23:29:22.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038163939999999999 2023-09-23T23:29:22.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703505+0000", 2023-09-23T23:29:22.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030447999999999999 2023-09-23T23:29:22.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.065 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.065 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703847+0000", 2023-09-23T23:29:22.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034161899999999998 2023-09-23T23:29:22.065 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.065 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.065 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.065 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703911+0000", 2023-09-23T23:29:22.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4521000000000001e-05 2023-09-23T23:29:22.065 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.065 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.065 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:22.065 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703957+0000", 2023-09-23T23:29:22.066 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5840999999999997e-05 2023-09-23T23:29:22.066 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.066 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.066 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:22.066 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704066+0000", 2023-09-23T23:29:22.066 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000108546 2023-09-23T23:29:22.066 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.066 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:22.066 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.066 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.066 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.067 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211014 11.17 11:ecac8c40:::benchmark_data_smithi110_433746_object105134:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:29:22.067 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686583+0000", 2023-09-23T23:29:22.067 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.254109827000001, 2023-09-23T23:29:22.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0177152949999999, 2023-09-23T23:29:22.067 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:22.067 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:22.067 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:22.067 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:22.067 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:22.067 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211014 2023-09-23T23:29:22.067 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.067 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:22.068 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.068 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:22.068 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686583+0000", 2023-09-23T23:29:22.068 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.068 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.068 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.068 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:22.068 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686583+0000", 2023-09-23T23:29:22.068 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.068 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.068 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.068 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:22.068 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686583+0000", 2023-09-23T23:29:22.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:22.069 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.069 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.069 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:22.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686586+0000", 2023-09-23T23:29:22.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6769999999999999e-06 2023-09-23T23:29:22.069 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.069 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.069 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:22.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686592+0000", 2023-09-23T23:29:22.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2480000000000001e-06 2023-09-23T23:29:22.069 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.070 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.070 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:22.070 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686598+0000", 2023-09-23T23:29:22.070 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4500000000000001e-06 2023-09-23T23:29:22.070 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.070 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.070 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.070 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687703+0000", 2023-09-23T23:29:22.070 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001105113 2023-09-23T23:29:22.070 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.070 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.070 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:22.071 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687711+0000", 2023-09-23T23:29:22.071 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9410000000000007e-06 2023-09-23T23:29:22.071 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.071 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.071 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.071 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.700984+0000", 2023-09-23T23:29:22.071 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0132722620000001 2023-09-23T23:29:22.071 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.071 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.071 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:22.071 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.701130+0000", 2023-09-23T23:29:22.071 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000146804 2023-09-23T23:29:22.072 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.072 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.072 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:29:22.072 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.701398+0000", 2023-09-23T23:29:22.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026770100000000002 2023-09-23T23:29:22.072 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.072 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.072 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:22.072 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703434+0000", 2023-09-23T23:29:22.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020357940000000001 2023-09-23T23:29:22.072 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.072 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.072 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703879+0000", 2023-09-23T23:29:22.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00044474299999999999 2023-09-23T23:29:22.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704106+0000", 2023-09-23T23:29:22.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022766 2023-09-23T23:29:22.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704144+0000", 2023-09-23T23:29:22.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7836000000000002e-05 2023-09-23T23:29:22.074 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.074 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.074 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:22.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704191+0000", 2023-09-23T23:29:22.074 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6582999999999998e-05 2023-09-23T23:29:22.074 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.074 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.074 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:22.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704299+0000", 2023-09-23T23:29:22.074 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107909 2023-09-23T23:29:22.074 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.074 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:22.075 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.075 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.075 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.075 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211043 11.17 11:edfeec9c:::benchmark_data_smithi110_433746_object105163:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:29:22.075 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686612+0000", 2023-09-23T23:29:22.075 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.254080932999997, 2023-09-23T23:29:22.075 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0257805470000001, 2023-09-23T23:29:22.075 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:22.075 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:22.075 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:22.075 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:22.075 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:22.076 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211043 2023-09-23T23:29:22.076 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.076 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:22.076 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.076 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:22.076 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686612+0000", 2023-09-23T23:29:22.076 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.076 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.076 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.076 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:22.076 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686612+0000", 2023-09-23T23:29:22.076 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.076 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.077 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.077 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:22.077 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686612+0000", 2023-09-23T23:29:22.077 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:22.077 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.077 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.077 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:22.077 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686615+0000", 2023-09-23T23:29:22.077 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3289999999999998e-06 2023-09-23T23:29:22.077 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.077 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.077 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:22.078 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686622+0000", 2023-09-23T23:29:22.078 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1439999999999997e-06 2023-09-23T23:29:22.078 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.078 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.078 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:22.078 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686628+0000", 2023-09-23T23:29:22.078 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9560000000000002e-06 2023-09-23T23:29:22.078 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.078 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.078 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.078 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687733+0000", 2023-09-23T23:29:22.078 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011048640000000001 2023-09-23T23:29:22.079 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.079 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.079 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:22.079 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687741+0000", 2023-09-23T23:29:22.079 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9340000000000006e-06 2023-09-23T23:29:22.079 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.079 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.079 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.079 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704331+0000", 2023-09-23T23:29:22.079 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0165899519999999 2023-09-23T23:29:22.079 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.079 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:22.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704469+0000", 2023-09-23T23:29:22.080 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013822899999999999 2023-09-23T23:29:22.080 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:29:22.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704810+0000", 2023-09-23T23:29:22.080 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000341169 2023-09-23T23:29:22.080 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:22.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712010+0000", 2023-09-23T23:29:22.080 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071992189999999998 2023-09-23T23:29:22.081 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.081 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.081 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.081 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712083+0000", 2023-09-23T23:29:22.081 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3090999999999995e-05 2023-09-23T23:29:22.081 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.081 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.081 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.081 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712140+0000", 2023-09-23T23:29:22.081 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7030000000000001e-05 2023-09-23T23:29:22.081 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.081 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.082 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.082 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712203+0000", 2023-09-23T23:29:22.082 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3297000000000001e-05 2023-09-23T23:29:22.082 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.082 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.082 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:22.082 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712257+0000", 2023-09-23T23:29:22.082 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4211e-05 2023-09-23T23:29:22.082 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.082 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.082 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:22.083 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712393+0000", 2023-09-23T23:29:22.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013563 2023-09-23T23:29:22.083 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.083 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:22.083 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.083 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.083 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211063 11.17 11:ec1e6191:::benchmark_data_smithi110_433746_object105183:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:29:22.083 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686633+0000", 2023-09-23T23:29:22.083 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.254060496999998, 2023-09-23T23:29:22.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.026136234, 2023-09-23T23:29:22.083 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:22.084 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:22.084 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:22.084 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:22.084 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:22.084 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211063 2023-09-23T23:29:22.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.084 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:22.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.084 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:22.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686633+0000", 2023-09-23T23:29:22.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:22.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686633+0000", 2023-09-23T23:29:22.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:22.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686632+0000", 2023-09-23T23:29:22.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:22.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:22.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686635+0000", 2023-09-23T23:29:22.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9110000000000002e-06 2023-09-23T23:29:22.086 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.086 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.086 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:22.086 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686643+0000", 2023-09-23T23:29:22.086 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7309999999999993e-06 2023-09-23T23:29:22.086 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.086 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.086 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:22.086 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686648+0000", 2023-09-23T23:29:22.086 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1580000000000004e-06 2023-09-23T23:29:22.086 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.086 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.087 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.087 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687762+0000", 2023-09-23T23:29:22.087 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011139660000000001 2023-09-23T23:29:22.087 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.087 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.087 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:22.087 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687770+0000", 2023-09-23T23:29:22.087 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9820000000000008e-06 2023-09-23T23:29:22.087 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.087 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.087 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.087 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.706433+0000", 2023-09-23T23:29:22.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.018663052 2023-09-23T23:29:22.088 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.088 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:22.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.706571+0000", 2023-09-23T23:29:22.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013809200000000001 2023-09-23T23:29:22.088 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.088 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:29:22.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.706778+0000", 2023-09-23T23:29:22.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000206739 2023-09-23T23:29:22.088 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.089 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:22.089 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712153+0000", 2023-09-23T23:29:22.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053747880000000001 2023-09-23T23:29:22.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.089 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.089 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712460+0000", 2023-09-23T23:29:22.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030737499999999998 2023-09-23T23:29:22.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712524+0000", 2023-09-23T23:29:22.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4078000000000006e-05 2023-09-23T23:29:22.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712603+0000", 2023-09-23T23:29:22.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8625e-05 2023-09-23T23:29:22.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:22.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712666+0000", 2023-09-23T23:29:22.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2713999999999996e-05 2023-09-23T23:29:22.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:22.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712769+0000", 2023-09-23T23:29:22.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010334 2023-09-23T23:29:22.091 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.091 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:22.091 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.091 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211074 11.17 11:eb325c31:::benchmark_data_smithi110_433746_object105194:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:29:22.092 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686653+0000", 2023-09-23T23:29:22.092 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.254040568000001, 2023-09-23T23:29:22.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.02894591, 2023-09-23T23:29:22.092 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:22.092 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:22.092 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:22.092 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:22.092 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:22.092 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211074 2023-09-23T23:29:22.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.092 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:22.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:22.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686653+0000", 2023-09-23T23:29:22.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:22.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686653+0000", 2023-09-23T23:29:22.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:22.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686652+0000", 2023-09-23T23:29:22.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:22.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:22.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686655+0000", 2023-09-23T23:29:22.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6469999999999998e-06 2023-09-23T23:29:22.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:22.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686661+0000", 2023-09-23T23:29:22.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8969999999999996e-06 2023-09-23T23:29:22.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:22.095 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686666+0000", 2023-09-23T23:29:22.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9930000000000003e-06 2023-09-23T23:29:22.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.095 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687792+0000", 2023-09-23T23:29:22.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011261330000000001 2023-09-23T23:29:22.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:22.095 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687800+0000", 2023-09-23T23:29:22.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9389999999999997e-06 2023-09-23T23:29:22.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.096 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.708139+0000", 2023-09-23T23:29:22.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0203391749999999 2023-09-23T23:29:22.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.096 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:22.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.708257+0000", 2023-09-23T23:29:22.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117959 2023-09-23T23:29:22.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:29:22.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.708494+0000", 2023-09-23T23:29:22.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023732399999999999 2023-09-23T23:29:22.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:22.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712159+0000", 2023-09-23T23:29:22.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036648150000000001 2023-09-23T23:29:22.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715288+0000", 2023-09-23T23:29:22.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031290469999999998 2023-09-23T23:29:22.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715392+0000", 2023-09-23T23:29:22.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104362 2023-09-23T23:29:22.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715427+0000", 2023-09-23T23:29:22.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4581e-05 2023-09-23T23:29:22.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:22.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715475+0000", 2023-09-23T23:29:22.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7766999999999999e-05 2023-09-23T23:29:22.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:22.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715599+0000", 2023-09-23T23:29:22.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012368000000000001 2023-09-23T23:29:22.099 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.099 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:22.101 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.101 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211090 11.17 11:ee95de61:::benchmark_data_smithi110_433746_object105210:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:29:22.101 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686670+0000", 2023-09-23T23:29:22.101 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.254023173, 2023-09-23T23:29:22.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.029135624, 2023-09-23T23:29:22.101 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:22.102 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:22.102 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:22.102 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:22.102 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:22.102 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211090 2023-09-23T23:29:22.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.102 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:22.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:22.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686670+0000", 2023-09-23T23:29:22.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.103 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:22.103 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686670+0000", 2023-09-23T23:29:22.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.103 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:22.103 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686670+0000", 2023-09-23T23:29:22.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:22.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.103 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:22.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686672+0000", 2023-09-23T23:29:22.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1749999999999999e-06 2023-09-23T23:29:22.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:22.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686678+0000", 2023-09-23T23:29:22.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3589999999999996e-06 2023-09-23T23:29:22.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:22.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686683+0000", 2023-09-23T23:29:22.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1139999999999997e-06 2023-09-23T23:29:22.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687821+0000", 2023-09-23T23:29:22.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001137723 2023-09-23T23:29:22.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:22.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687829+0000", 2023-09-23T23:29:22.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6809999999999998e-06 2023-09-23T23:29:22.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.710062+0000", 2023-09-23T23:29:22.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0222334470000001 2023-09-23T23:29:22.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:22.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.710215+0000", 2023-09-23T23:29:22.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015298800000000001 2023-09-23T23:29:22.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:29:22.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.710532+0000", 2023-09-23T23:29:22.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00031638499999999997 2023-09-23T23:29:22.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:22.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712541+0000", 2023-09-23T23:29:22.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020091390000000001 2023-09-23T23:29:22.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715339+0000", 2023-09-23T23:29:22.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002798367 2023-09-23T23:29:22.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.108 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715645+0000", 2023-09-23T23:29:22.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030564500000000001 2023-09-23T23:29:22.108 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.108 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715685+0000", 2023-09-23T23:29:22.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0182000000000003e-05 2023-09-23T23:29:22.108 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.108 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:22.109 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715717+0000", 2023-09-23T23:29:22.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1649000000000001e-05 2023-09-23T23:29:22.109 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.109 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:22.109 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715806+0000", 2023-09-23T23:29:22.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9040000000000001e-05 2023-09-23T23:29:22.109 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.109 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:22.109 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.109 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.109 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211101 11.17 11:eed1b340:::benchmark_data_smithi110_433746_object105221:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:29:22.110 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686688+0000", 2023-09-23T23:29:22.110 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.254005585000002, 2023-09-23T23:29:22.110 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.034036953, 2023-09-23T23:29:22.110 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:22.110 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:22.110 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:22.110 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:22.110 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:22.110 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211101 2023-09-23T23:29:22.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.110 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:22.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:22.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686688+0000", 2023-09-23T23:29:22.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:22.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686688+0000", 2023-09-23T23:29:22.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:22.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:22.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686687+0000", 2023-09-23T23:29:22.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:22.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:22.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686690+0000", 2023-09-23T23:29:22.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4930000000000001e-06 2023-09-23T23:29:22.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:22.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686696+0000", 2023-09-23T23:29:22.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2400000000000004e-06 2023-09-23T23:29:22.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:22.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686701+0000", 2023-09-23T23:29:22.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2020000000000003e-06 2023-09-23T23:29:22.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687851+0000", 2023-09-23T23:29:22.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011494809999999999 2023-09-23T23:29:22.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:22.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687859+0000", 2023-09-23T23:29:22.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8779999999999998e-06 2023-09-23T23:29:22.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:22.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712809+0000", 2023-09-23T23:29:22.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0249501940000001 2023-09-23T23:29:22.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:22.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712967+0000", 2023-09-23T23:29:22.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015830900000000001 2023-09-23T23:29:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:29:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.713350+0000", 2023-09-23T23:29:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00038338199999999998 2023-09-23T23:29:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715863+0000", 2023-09-23T23:29:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025129060000000001 2023-09-23T23:29:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.116 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.116 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715903+0000", 2023-09-23T23:29:22.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9675000000000002e-05 2023-09-23T23:29:22.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.116 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:22.116 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715938+0000", 2023-09-23T23:29:22.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4501000000000001e-05 2023-09-23T23:29:22.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.116 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:22.116 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.720575+0000", 2023-09-23T23:29:22.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046374110000000001 2023-09-23T23:29:22.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:22.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.720618+0000", 2023-09-23T23:29:22.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3411000000000002e-05 2023-09-23T23:29:22.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:22.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:22.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:22.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.720725+0000", 2023-09-23T23:29:22.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000106209 2023-09-23T23:29:22.117 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.117 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:22.118 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.118 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:22.118 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:22.118 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:29:22.157 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:22.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:22.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:22.236 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 37 16 53491 53475 90.2568 61.0625 0.0195411 0.0110742 2023-09-23T23:29:22.268 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:29:22.301 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:22.362 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:29:22.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:22.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:29:22.470 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:29:22.537 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:22.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:22 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:22.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:22 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:22.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:22 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:29:22.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:22 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:29:22.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:22 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:22.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:22 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:22.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:22 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:22.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:22 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:29:22.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:22 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:29:22.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:22 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:22.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:22 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:22 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:22 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:29:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:22 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:29:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:22 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:23.018 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:29:23.236 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 38 16 54368 54352 89.3245 54.8125 0.00282232 0.0111818 2023-09-23T23:29:23.412 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:23 smithi110 ceph-mon[212939]: pgmap v1331: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T23:29:23.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:23 smithi110 ceph-mon[221102]: pgmap v1331: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T23:29:23.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:23 smithi138 ceph-mon[193606]: pgmap v1331: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T23:29:23.798 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:29:23.800 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:29:23.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:23.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:29:23.910 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:23.941 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:23.993 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:23.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:23.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:24.100 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:29:24.201 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 7, 2, 1] out_osds: [6] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:29:24.201 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:29:24.202 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_7 2023-09-23T23:29:24.202 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:29:24.206 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:24.236 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 39 16 55396 55380 88.6819 64.25 0.0235506 0.0112695 2023-09-23T23:29:24.359 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:24.375 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:24.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:24.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:24.486 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:24.609 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:24.657 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:24.864 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:24 smithi110 ceph-mon[212939]: pgmap v1332: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T23:29:24.864 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:24 smithi110 ceph-mon[221102]: pgmap v1332: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T23:29:24.865 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:24.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:24.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:24.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:24 smithi138 ceph-mon[193606]: pgmap v1332: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T23:29:24.980 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:29:25.011 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:25.073 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:29:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:25.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:29:25.180 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:29:25.211 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:25.236 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:29:25.235992+0000 min lat: 0.00115016 max lat: 0.357929 avg lat: 0.011373 2023-09-23T23:29:25.237 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:29:25.237 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 40 16 56308 56292 87.8902 57 0.00456759 0.011373 2023-09-23T23:29:25.270 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:29:25.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:25.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:29:25.378 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:25.451 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:25.682 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:29:25.807 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:25.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:25.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:25.813 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:25 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1018570553' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:25.813 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:25 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:25.813 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:25 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1018570553' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:25.813 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:25 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:25.914 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:25 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1018570553' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:25.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:25 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:25.994 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:26.237 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 41 16 57155 57139 87.038 52.9375 0.00280815 0.0114499 2023-09-23T23:29:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:26 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:29:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:26 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:29:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:26 smithi138 ceph-mon[193606]: failed to encode map e487 with expected crc 2023-09-23T23:29:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:26 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:29:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:26 smithi138 ceph-mon[193606]: osdmap e487: 8 total, 8 up, 7 in 2023-09-23T23:29:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:26 smithi138 ceph-mon[193606]: failed to encode map e487 with expected crc 2023-09-23T23:29:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:26 smithi138 ceph-mon[193606]: failed to encode map e487 with expected crc 2023-09-23T23:29:26.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:26 smithi138 ceph-mon[193606]: pgmap v1334: 119 pgs: 119 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 977 op/s 2023-09-23T23:29:26.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:29:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:29:26] "GET /metrics HTTP/1.1" 200 36520 "" "Prometheus/2.43.0" 2023-09-23T23:29:26.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:26 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:29:26.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:26 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:29:26.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:26 smithi110 ceph-mon[212939]: failed to encode map e487 with expected crc 2023-09-23T23:29:26.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:26 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:29:26.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:26 smithi110 ceph-mon[212939]: osdmap e487: 8 total, 8 up, 7 in 2023-09-23T23:29:26.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:26 smithi110 ceph-mon[212939]: failed to encode map e487 with expected crc 2023-09-23T23:29:26.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:26 smithi110 ceph-mon[212939]: failed to encode map e487 with expected crc 2023-09-23T23:29:26.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:26 smithi110 ceph-mon[212939]: pgmap v1334: 119 pgs: 119 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 977 op/s 2023-09-23T23:29:26.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:26 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:29:26.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:26 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:29:26.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:26 smithi110 ceph-mon[221102]: failed to encode map e487 with expected crc 2023-09-23T23:29:26.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:26 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:29:26.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:26 smithi110 ceph-mon[221102]: osdmap e487: 8 total, 8 up, 7 in 2023-09-23T23:29:26.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:26 smithi110 ceph-mon[221102]: failed to encode map e487 with expected crc 2023-09-23T23:29:26.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:26 smithi110 ceph-mon[221102]: failed to encode map e487 with expected crc 2023-09-23T23:29:26.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:26 smithi110 ceph-mon[221102]: pgmap v1334: 119 pgs: 119 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 977 op/s 2023-09-23T23:29:27.237 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 42 16 58120 58104 86.402 60.3125 0.0027796 0.0115575 2023-09-23T23:29:27.891 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:27.911 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:27.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:27.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:28.018 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:28.048 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:28.101 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:28.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:28.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:28.206 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:28.234 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:28.237 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 43 16 59057 59041 85.7541 58.5625 0.0262325 0.011656 2023-09-23T23:29:28.289 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:28.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:28.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:28.393 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:28.492 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:28.604 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:29:28.644 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:29:28.701 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:29:28.734 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:29:28.768 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:29:28.800 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:29:28.848 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:29:28.875 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:29:28.876 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:29:28.898 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:29:28.916 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:29:28.916 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:29:28.916 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:29:28.916 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:29:28.916 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:29:28.916 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:29:28.916 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:29:28.916 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:29:29.237 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 44 16 60103 60087 85.2911 65.375 0.00363129 0.011718 2023-09-23T23:29:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:29 smithi138 ceph-mon[193606]: pgmap v1335: 119 pgs: 119 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2023-09-23T23:29:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:29 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:29 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:29:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:29 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2834075033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:29 smithi110 ceph-mon[212939]: pgmap v1335: 119 pgs: 119 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2023-09-23T23:29:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:29 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:29 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:29:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:29 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2834075033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:29.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:29 smithi110 ceph-mon[221102]: pgmap v1335: 119 pgs: 119 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2023-09-23T23:29:29.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:29 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:29.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:29 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:29:29.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:29 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2834075033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:29.889 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:29:30.098 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:30.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:30.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:30.209 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:29:30.237 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 45 16 61075 61059 84.7459 60.75 0.00869955 0.0117936 2023-09-23T23:29:30.278 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:30.708 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:29:30.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:30.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:29:30.813 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:29:30.845 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:30.901 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:29:30.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:30.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:29:31.008 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:29:31.037 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:31.085 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:29:31.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:31.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:29:31.191 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:31.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:31 smithi138 ceph-mon[193606]: pgmap v1336: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 954 op/s 2023-09-23T23:29:31.238 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 46 16 62199 62183 84.4294 70.25 0.0174157 0.0118391 2023-09-23T23:29:31.261 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:31 smithi110 ceph-mon[212939]: pgmap v1336: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 954 op/s 2023-09-23T23:29:31.266 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:31 smithi110 ceph-mon[221102]: pgmap v1336: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 954 op/s 2023-09-23T23:29:31.269 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:31.617 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:31.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:31.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:31.692 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:29:31.724 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:31.839 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:31.904 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:32.007 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:32.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:32.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:32.130 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:29:32.168 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:29:32.194 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:29:32.205 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:32.238 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 47 16 63128 63112 83.8687 58.0625 0.00572053 0.0119187 2023-09-23T23:29:32.306 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:29:32.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:32.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:29:32.420 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:32.500 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:32.845 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:29:33.238 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 48 16 63880 63864 83.1009 47 0.0181491 0.012028 2023-09-23T23:29:33.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:33 smithi138 ceph-mon[193606]: pgmap v1337: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 960 op/s 2023-09-23T23:29:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:33 smithi138 ceph-mon[193606]: from='client.46283 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:29:33.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:33 smithi110 ceph-mon[212939]: pgmap v1337: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 960 op/s 2023-09-23T23:29:33.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:33 smithi110 ceph-mon[212939]: from='client.46283 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:29:33.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:33 smithi110 ceph-mon[221102]: pgmap v1337: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 960 op/s 2023-09-23T23:29:33.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:33 smithi110 ceph-mon[221102]: from='client.46283 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:29:33.539 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:33.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:33.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:33.646 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:29:33.707 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:34.027 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1337,"stamp":"2023-09-23T23:29:31.749285+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3951761312,"num_objects":60314,"num_object_clones":0,"num_object_copies":180942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":60314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":60351,"num_write_kb":3858589,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":660968,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":65336,"ondisk_log_size":65336,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":359,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16287096,"kb_used_data":11550592,"kb_used_omap":29,"kb_used_meta":4736418,"kb_avail":639928968,"statfs":{"total":671965249536,"available":655287263232,"internally_reserved":0,"allocated":11827806208,"data_stored":11814873653,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":4850092755},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":629997568,"num_objects":9613,"num_object_clones":0,"num_object_copies":28839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9613,"num_write_kb":615232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003898"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":683,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.602975+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:29:14.602975+0000","last_peered":"2023-09-23T23:29:14.602975+0000","last_clean":"2023-09-23T23:29:14.602975+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:29:14.602975+0000","last_undegraded":"2023-09-23T23:29:14.602975+0000","last_fullsized":"2023-09-23T23:29:14.602975+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":521,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.600902+0000","last_change":"2023-09-23T23:22:38.439629+0000","last_active":"2023-09-23T23:29:14.600902+0000","last_peered":"2023-09-23T23:29:14.600902+0000","last_clean":"2023-09-23T23:29:14.600902+0000","last_became_active":"2023-09-23T23:22:38.439285+0000","last_became_peered":"2023-09-23T23:22:38.439285+0000","last_unstale":"2023-09-23T23:29:14.600902+0000","last_undegraded":"2023-09-23T23:29:14.600902+0000","last_fullsized":"2023-09-23T23:29:14.600902+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:34.431221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":457,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.626378+0000","last_change":"2023-09-23T23:23:43.511912+0000","last_active":"2023-09-23T23:29:14.626378+0000","last_peered":"2023-09-23T23:29:14.626378+0000","last_clean":"2023-09-23T23:29:14.626378+0000","last_became_active":"2023-09-23T23:23:43.511574+0000","last_became_peered":"2023-09-23T23:23:43.511574+0000","last_unstale":"2023-09-23T23:29:14.626378+0000","last_undegraded":"2023-09-23T23:29:14.626378+0000","last_fullsized":"2023-09-23T23:29:14.626378+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:16.286926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":576,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.602761+0000","last_change":"2023-09-23T23:23:50.459634+0000","last_active":"2023-09-23T23:29:14.602761+0000","last_peered":"2023-09-23T23:29:14.602761+0000","last_clean":"2023-09-23T23:29:14.602761+0000","last_became_active":"2023-09-23T23:23:45.453958+0000","last_became_peered":"2023-09-23T23:23:45.453958+0000","last_unstale":"2023-09-23T23:29:14.602761+0000","last_undegraded":"2023-09-23T23:29:14.602761+0000","last_fullsized":"2023-09-23T23:29:14.602761+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:56.563201+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":197,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.608897+0000","last_change":"2023-09-23T23:26:11.528846+0000","last_active":"2023-09-23T23:29:14.608897+0000","last_peered":"2023-09-23T23:29:14.608897+0000","last_clean":"2023-09-23T23:29:14.608897+0000","last_became_active":"2023-09-23T23:26:11.528529+0000","last_became_peered":"2023-09-23T23:26:11.528529+0000","last_unstale":"2023-09-23T23:29:14.608897+0000","last_undegraded":"2023-09-23T23:29:14.608897+0000","last_fullsized":"2023-09-23T23:29:14.608897+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:35.094583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":452,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.605906+0000","last_change":"2023-09-23T23:28:22.604718+0000","last_active":"2023-09-23T23:29:14.605906+0000","last_peered":"2023-09-23T23:29:14.605906+0000","last_clean":"2023-09-23T23:29:14.605906+0000","last_became_active":"2023-09-23T23:28:22.604498+0000","last_became_peered":"2023-09-23T23:28:22.604498+0000","last_unstale":"2023-09-23T23:29:14.605906+0000","last_undegraded":"2023-09-23T23:29:14.605906+0000","last_fullsized":"2023-09-23T23:29:14.605906+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:30.472412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":790,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.600726+0000","last_change":"2023-09-23T23:28:05.634824+0000","last_active":"2023-09-23T23:29:14.600726+0000","last_peered":"2023-09-23T23:29:14.600726+0000","last_clean":"2023-09-23T23:29:14.600726+0000","last_became_active":"2023-09-23T23:27:23.551046+0000","last_became_peered":"2023-09-23T23:27:23.551046+0000","last_unstale":"2023-09-23T23:29:14.600726+0000","last_undegraded":"2023-09-23T23:29:14.600726+0000","last_fullsized":"2023-09-23T23:29:14.600726+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:59.062239+0000","scrub_duration":0.0034158959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":484,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.622320+0000","last_change":"2023-09-23T23:27:20.554007+0000","last_active":"2023-09-23T23:29:14.622320+0000","last_peered":"2023-09-23T23:29:14.622320+0000","last_clean":"2023-09-23T23:29:14.622320+0000","last_became_active":"2023-09-23T23:27:20.553716+0000","last_became_peered":"2023-09-23T23:27:20.553716+0000","last_unstale":"2023-09-23T23:29:14.622320+0000","last_undegraded":"2023-09-23T23:29:14.622320+0000","last_fullsized":"2023-09-23T23:29:14.622320+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:38.911862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":420,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.609659+0000","last_change":"2023-09-23T23:25:31.735427+0000","last_active":"2023-09-23T23:29:14.609659+0000","last_peered":"2023-09-23T23:29:14.609659+0000","last_clean":"2023-09-23T23:29:14.609659+0000","last_became_active":"2023-09-23T23:25:31.734582+0000","last_became_peered":"2023-09-23T23:25:31.734582+0000","last_unstale":"2023-09-23T23:29:14.609659+0000","last_undegraded":"2023-09-23T23:29:14.609659+0000","last_fullsized":"2023-09-23T23:29:14.609659+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:39.075149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":807,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.606344+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:29:14.606344+0000","last_peered":"2023-09-23T23:29:14.606344+0000","last_clean":"2023-09-23T23:29:14.606344+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:29:14.606344+0000","last_undegraded":"2023-09-23T23:29:14.606344+0000","last_fullsized":"2023-09-23T23:29:14.606344+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":119,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.603509+0000","last_change":"2023-09-23T23:28:22.602481+0000","last_active":"2023-09-23T23:29:14.603509+0000","last_peered":"2023-09-23T23:29:14.603509+0000","last_clean":"2023-09-23T23:29:14.603509+0000","last_became_active":"2023-09-23T23:28:22.602282+0000","last_became_peered":"2023-09-23T23:28:22.602282+0000","last_unstale":"2023-09-23T23:29:14.603509+0000","last_undegraded":"2023-09-23T23:29:14.603509+0000","last_fullsized":"2023-09-23T23:29:14.603509+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:39.906003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":475,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.622241+0000","last_change":"2023-09-23T23:26:11.528354+0000","last_active":"2023-09-23T23:29:14.622241+0000","last_peered":"2023-09-23T23:29:14.622241+0000","last_clean":"2023-09-23T23:29:14.622241+0000","last_became_active":"2023-09-23T23:26:11.528076+0000","last_became_peered":"2023-09-23T23:26:11.528076+0000","last_unstale":"2023-09-23T23:29:14.622241+0000","last_undegraded":"2023-09-23T23:29:14.622241+0000","last_fullsized":"2023-09-23T23:29:14.622241+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:16.100396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":760,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.611969+0000","last_change":"2023-09-23T23:26:14.521792+0000","last_active":"2023-09-23T23:29:14.611969+0000","last_peered":"2023-09-23T23:29:14.611969+0000","last_clean":"2023-09-23T23:29:14.611969+0000","last_became_active":"2023-09-23T23:26:14.521438+0000","last_became_peered":"2023-09-23T23:26:14.521438+0000","last_unstale":"2023-09-23T23:29:14.611969+0000","last_undegraded":"2023-09-23T23:29:14.611969+0000","last_fullsized":"2023-09-23T23:29:14.611969+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:09.103712+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":118,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.603271+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:29:14.603271+0000","last_peered":"2023-09-23T23:29:14.603271+0000","last_clean":"2023-09-23T23:29:14.603271+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:29:14.603271+0000","last_undegraded":"2023-09-23T23:29:14.603271+0000","last_fullsized":"2023-09-23T23:29:14.603271+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":436,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.625376+0000","last_change":"2023-09-23T23:26:48.538059+0000","last_active":"2023-09-23T23:29:14.625376+0000","last_peered":"2023-09-23T23:29:14.625376+0000","last_clean":"2023-09-23T23:29:14.625376+0000","last_became_active":"2023-09-23T23:26:48.537735+0000","last_became_peered":"2023-09-23T23:26:48.537735+0000","last_unstale":"2023-09-23T23:29:14.625376+0000","last_undegraded":"2023-09-23T23:29:14.625376+0000","last_fullsized":"2023-09-23T23:29:14.625376+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:32.878591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":881,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.603270+0000","last_change":"2023-09-23T23:28:09.363053+0000","last_active":"2023-09-23T23:29:14.603270+0000","last_peered":"2023-09-23T23:29:14.603270+0000","last_clean":"2023-09-23T23:29:14.603270+0000","last_became_active":"2023-09-23T23:26:51.563536+0000","last_became_peered":"2023-09-23T23:26:51.563536+0000","last_unstale":"2023-09-23T23:29:14.603270+0000","last_undegraded":"2023-09-23T23:29:14.603270+0000","last_fullsized":"2023-09-23T23:29:14.603270+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:29.922798+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":92,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.605913+0000","last_change":"2023-09-23T23:26:48.669857+0000","last_active":"2023-09-23T23:29:14.605913+0000","last_peered":"2023-09-23T23:29:14.605913+0000","last_clean":"2023-09-23T23:29:14.605913+0000","last_became_active":"2023-09-23T23:26:48.538246+0000","last_became_peered":"2023-09-23T23:26:48.538246+0000","last_unstale":"2023-09-23T23:29:14.605913+0000","last_undegraded":"2023-09-23T23:29:14.605913+0000","last_fullsized":"2023-09-23T23:29:14.605913+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:19.039472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":73,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.609094+0000","last_change":"2023-09-23T23:27:20.551305+0000","last_active":"2023-09-23T23:29:14.609094+0000","last_peered":"2023-09-23T23:29:14.609094+0000","last_clean":"2023-09-23T23:29:14.609094+0000","last_became_active":"2023-09-23T23:27:20.551017+0000","last_became_peered":"2023-09-23T23:27:20.551017+0000","last_unstale":"2023-09-23T23:29:14.609094+0000","last_undegraded":"2023-09-23T23:29:14.609094+0000","last_fullsized":"2023-09-23T23:29:14.609094+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:27.026015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":562,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.610817+0000","last_change":"2023-09-23T23:24:02.268079+0000","last_active":"2023-09-23T23:29:14.610817+0000","last_peered":"2023-09-23T23:29:14.610817+0000","last_clean":"2023-09-23T23:29:14.610817+0000","last_became_active":"2023-09-23T23:23:45.463106+0000","last_became_peered":"2023-09-23T23:23:45.463106+0000","last_unstale":"2023-09-23T23:29:14.610817+0000","last_undegraded":"2023-09-23T23:29:14.610817+0000","last_fullsized":"2023-09-23T23:29:14.610817+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.128813+0000","scrub_duration":0.0018535310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":118,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.601505+0000","last_change":"2023-09-23T23:26:11.534727+0000","last_active":"2023-09-23T23:29:14.601505+0000","last_peered":"2023-09-23T23:29:14.601505+0000","last_clean":"2023-09-23T23:29:14.601505+0000","last_became_active":"2023-09-23T23:26:11.513694+0000","last_became_peered":"2023-09-23T23:26:11.513694+0000","last_unstale":"2023-09-23T23:29:14.601505+0000","last_undegraded":"2023-09-23T23:29:14.601505+0000","last_fullsized":"2023-09-23T23:29:14.601505+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:04.562837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":475,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.600946+0000","last_change":"2023-09-23T23:25:31.718910+0000","last_active":"2023-09-23T23:29:14.600946+0000","last_peered":"2023-09-23T23:29:14.600946+0000","last_clean":"2023-09-23T23:29:14.600946+0000","last_became_active":"2023-09-23T23:25:31.718567+0000","last_became_peered":"2023-09-23T23:25:31.718567+0000","last_unstale":"2023-09-23T23:29:14.600946+0000","last_undegraded":"2023-09-23T23:29:14.600946+0000","last_fullsized":"2023-09-23T23:29:14.600946+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:21.447944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":602,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.602312+0000","last_change":"2023-09-23T23:28:14.378358+0000","last_active":"2023-09-23T23:29:14.602312+0000","last_peered":"2023-09-23T23:29:14.602312+0000","last_clean":"2023-09-23T23:29:14.602312+0000","last_became_active":"2023-09-23T23:26:34.550151+0000","last_became_peered":"2023-09-23T23:26:34.550151+0000","last_unstale":"2023-09-23T23:29:14.602312+0000","last_undegraded":"2023-09-23T23:29:14.602312+0000","last_fullsized":"2023-09-23T23:29:14.602312+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:47.794690+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":419,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.605865+0000","last_change":"2023-09-23T23:26:11.569228+0000","last_active":"2023-09-23T23:29:14.605865+0000","last_peered":"2023-09-23T23:29:14.605865+0000","last_clean":"2023-09-23T23:29:14.605865+0000","last_became_active":"2023-09-23T23:26:11.513159+0000","last_became_peered":"2023-09-23T23:26:11.513159+0000","last_unstale":"2023-09-23T23:29:14.605865+0000","last_undegraded":"2023-09-23T23:29:14.605865+0000","last_fullsized":"2023-09-23T23:29:14.605865+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:40.698257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":483,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.626852+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:29:14.626852+0000","last_peered":"2023-09-23T23:29:14.626852+0000","last_clean":"2023-09-23T23:29:14.626852+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:29:14.626852+0000","last_undegraded":"2023-09-23T23:29:14.626852+0000","last_fullsized":"2023-09-23T23:29:14.626852+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":826,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.606040+0000","last_change":"2023-09-23T23:26:34.551458+0000","last_active":"2023-09-23T23:29:14.606040+0000","last_peered":"2023-09-23T23:29:14.606040+0000","last_clean":"2023-09-23T23:29:14.606040+0000","last_became_active":"2023-09-23T23:26:34.551067+0000","last_became_peered":"2023-09-23T23:26:34.551067+0000","last_unstale":"2023-09-23T23:29:14.606040+0000","last_undegraded":"2023-09-23T23:29:14.606040+0000","last_fullsized":"2023-09-23T23:29:14.606040+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:05.788656+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":86,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.605689+0000","last_change":"2023-09-23T23:26:48.563723+0000","last_active":"2023-09-23T23:29:14.605689+0000","last_peered":"2023-09-23T23:29:14.605689+0000","last_clean":"2023-09-23T23:29:14.605689+0000","last_became_active":"2023-09-23T23:26:48.563340+0000","last_became_peered":"2023-09-23T23:26:48.563340+0000","last_unstale":"2023-09-23T23:29:14.605689+0000","last_undegraded":"2023-09-23T23:29:14.605689+0000","last_fullsized":"2023-09-23T23:29:14.605689+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:05.974131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":373,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.606353+0000","last_change":"2023-09-23T23:26:32.040316+0000","last_active":"2023-09-23T23:29:14.606353+0000","last_peered":"2023-09-23T23:29:14.606353+0000","last_clean":"2023-09-23T23:29:14.606353+0000","last_became_active":"2023-09-23T23:26:32.039758+0000","last_became_peered":"2023-09-23T23:26:32.039758+0000","last_unstale":"2023-09-23T23:29:14.606353+0000","last_undegraded":"2023-09-23T23:29:14.606353+0000","last_fullsized":"2023-09-23T23:29:14.606353+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:03.930331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":650,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.609635+0000","last_change":"2023-09-23T23:26:14.527752+0000","last_active":"2023-09-23T23:29:14.609635+0000","last_peered":"2023-09-23T23:29:14.609635+0000","last_clean":"2023-09-23T23:29:14.609635+0000","last_became_active":"2023-09-23T23:26:14.525188+0000","last_became_peered":"2023-09-23T23:26:14.525188+0000","last_unstale":"2023-09-23T23:29:14.609635+0000","last_undegraded":"2023-09-23T23:29:14.609635+0000","last_fullsized":"2023-09-23T23:29:14.609635+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:10.617744+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":341,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.605319+0000","last_change":"2023-09-23T23:25:31.725776+0000","last_active":"2023-09-23T23:29:14.605319+0000","last_peered":"2023-09-23T23:29:14.605319+0000","last_clean":"2023-09-23T23:29:14.605319+0000","last_became_active":"2023-09-23T23:25:31.725429+0000","last_became_peered":"2023-09-23T23:25:31.725429+0000","last_unstale":"2023-09-23T23:29:14.605319+0000","last_undegraded":"2023-09-23T23:29:14.605319+0000","last_fullsized":"2023-09-23T23:29:14.605319+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:07.289251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":422,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.623380+0000","last_change":"2023-09-23T23:28:22.603836+0000","last_active":"2023-09-23T23:29:14.623380+0000","last_peered":"2023-09-23T23:29:14.623380+0000","last_clean":"2023-09-23T23:29:14.623380+0000","last_became_active":"2023-09-23T23:28:22.603490+0000","last_became_peered":"2023-09-23T23:28:22.603490+0000","last_unstale":"2023-09-23T23:29:14.623380+0000","last_undegraded":"2023-09-23T23:29:14.623380+0000","last_fullsized":"2023-09-23T23:29:14.623380+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:00.869241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":830,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.602619+0000","last_change":"2023-09-23T23:27:23.549144+0000","last_active":"2023-09-23T23:29:14.602619+0000","last_peered":"2023-09-23T23:29:14.602619+0000","last_clean":"2023-09-23T23:29:14.602619+0000","last_became_active":"2023-09-23T23:27:23.548774+0000","last_became_peered":"2023-09-23T23:27:23.548774+0000","last_unstale":"2023-09-23T23:29:14.602619+0000","last_undegraded":"2023-09-23T23:29:14.602619+0000","last_fullsized":"2023-09-23T23:29:14.602619+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:18.211722+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":168,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.611362+0000","last_change":"2023-09-23T23:26:32.139091+0000","last_active":"2023-09-23T23:29:14.611362+0000","last_peered":"2023-09-23T23:29:14.611362+0000","last_clean":"2023-09-23T23:29:14.611362+0000","last_became_active":"2023-09-23T23:26:32.053033+0000","last_became_peered":"2023-09-23T23:26:32.053033+0000","last_unstale":"2023-09-23T23:29:14.611362+0000","last_undegraded":"2023-09-23T23:29:14.611362+0000","last_fullsized":"2023-09-23T23:29:14.611362+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:32.625818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":125,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.623667+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:29:14.623667+0000","last_peered":"2023-09-23T23:29:14.623667+0000","last_clean":"2023-09-23T23:29:14.623667+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:29:14.623667+0000","last_undegraded":"2023-09-23T23:29:14.623667+0000","last_fullsized":"2023-09-23T23:29:14.623667+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"487'3795","reported_seq":3814,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:28.857252+0000","last_change":"2023-09-23T23:28:38.062488+0000","last_active":"2023-09-23T23:29:28.857252+0000","last_peered":"2023-09-23T23:29:28.857252+0000","last_clean":"2023-09-23T23:29:28.857252+0000","last_became_active":"2023-09-23T23:28:38.062098+0000","last_became_peered":"2023-09-23T23:28:38.062098+0000","last_unstale":"2023-09-23T23:29:28.857252+0000","last_undegraded":"2023-09-23T23:29:28.857252+0000","last_fullsized":"2023-09-23T23:29:28.857252+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":3795,"log_dups_size":0,"ondisk_log_size":3795,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:55.507817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248709120,"num_objects":3795,"num_object_clones":0,"num_object_copies":11385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3795,"num_write_kb":242880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":814,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.606000+0000","last_change":"2023-09-23T23:26:51.563868+0000","last_active":"2023-09-23T23:29:14.606000+0000","last_peered":"2023-09-23T23:29:14.606000+0000","last_clean":"2023-09-23T23:29:14.606000+0000","last_became_active":"2023-09-23T23:26:51.563510+0000","last_became_peered":"2023-09-23T23:26:51.563510+0000","last_unstale":"2023-09-23T23:29:14.606000+0000","last_undegraded":"2023-09-23T23:29:14.606000+0000","last_fullsized":"2023-09-23T23:29:14.606000+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:34.244822+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":90,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.606176+0000","last_change":"2023-09-23T23:26:48.673221+0000","last_active":"2023-09-23T23:29:14.606176+0000","last_peered":"2023-09-23T23:29:14.606176+0000","last_clean":"2023-09-23T23:29:14.606176+0000","last_became_active":"2023-09-23T23:26:48.555920+0000","last_became_peered":"2023-09-23T23:26:48.555920+0000","last_unstale":"2023-09-23T23:29:14.606176+0000","last_undegraded":"2023-09-23T23:29:14.606176+0000","last_fullsized":"2023-09-23T23:29:14.606176+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:26.586824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":108,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.602935+0000","last_change":"2023-09-23T23:26:48.525190+0000","last_active":"2023-09-23T23:29:14.602935+0000","last_peered":"2023-09-23T23:29:14.602935+0000","last_clean":"2023-09-23T23:29:14.602935+0000","last_became_active":"2023-09-23T23:26:48.524874+0000","last_became_peered":"2023-09-23T23:26:48.524874+0000","last_unstale":"2023-09-23T23:29:14.602935+0000","last_undegraded":"2023-09-23T23:29:14.602935+0000","last_fullsized":"2023-09-23T23:29:14.602935+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:43.114769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"487'3795","reported_seq":3815,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:28.877361+0000","last_change":"2023-09-23T23:28:38.062602+0000","last_active":"2023-09-23T23:29:28.877361+0000","last_peered":"2023-09-23T23:29:28.877361+0000","last_clean":"2023-09-23T23:29:28.877361+0000","last_became_active":"2023-09-23T23:28:38.062327+0000","last_became_peered":"2023-09-23T23:28:38.062327+0000","last_unstale":"2023-09-23T23:29:28.877361+0000","last_undegraded":"2023-09-23T23:29:28.877361+0000","last_fullsized":"2023-09-23T23:29:28.877361+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":3795,"log_dups_size":0,"ondisk_log_size":3795,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:30.017731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248709120,"num_objects":3795,"num_object_clones":0,"num_object_copies":11385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3795,"num_write_kb":242880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":875,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.626544+0000","last_change":"2023-09-23T23:26:14.530414+0000","last_active":"2023-09-23T23:29:14.626544+0000","last_peered":"2023-09-23T23:29:14.626544+0000","last_clean":"2023-09-23T23:29:14.626544+0000","last_became_active":"2023-09-23T23:26:14.530001+0000","last_became_peered":"2023-09-23T23:26:14.530001+0000","last_unstale":"2023-09-23T23:29:14.626544+0000","last_undegraded":"2023-09-23T23:29:14.626544+0000","last_fullsized":"2023-09-23T23:29:14.626544+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:37.443998+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":90,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.606262+0000","last_change":"2023-09-23T23:26:48.547146+0000","last_active":"2023-09-23T23:29:14.606262+0000","last_peered":"2023-09-23T23:29:14.606262+0000","last_clean":"2023-09-23T23:29:14.606262+0000","last_became_active":"2023-09-23T23:26:48.537682+0000","last_became_peered":"2023-09-23T23:26:48.537682+0000","last_unstale":"2023-09-23T23:29:14.606262+0000","last_undegraded":"2023-09-23T23:29:14.606262+0000","last_fullsized":"2023-09-23T23:29:14.606262+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:10.115563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":512,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.600587+0000","last_change":"2023-09-23T23:26:01.833889+0000","last_active":"2023-09-23T23:29:14.600587+0000","last_peered":"2023-09-23T23:29:14.600587+0000","last_clean":"2023-09-23T23:29:14.600587+0000","last_became_active":"2023-09-23T23:26:01.833626+0000","last_became_peered":"2023-09-23T23:26:01.833626+0000","last_unstale":"2023-09-23T23:29:14.600587+0000","last_undegraded":"2023-09-23T23:29:14.600587+0000","last_fullsized":"2023-09-23T23:29:14.600587+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:25.659671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"487'3770","reported_seq":3789,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:28.822084+0000","last_change":"2023-09-23T23:28:38.061823+0000","last_active":"2023-09-23T23:29:28.822084+0000","last_peered":"2023-09-23T23:29:28.822084+0000","last_clean":"2023-09-23T23:29:28.822084+0000","last_became_active":"2023-09-23T23:28:38.061549+0000","last_became_peered":"2023-09-23T23:28:38.061549+0000","last_unstale":"2023-09-23T23:29:28.822084+0000","last_undegraded":"2023-09-23T23:29:28.822084+0000","last_fullsized":"2023-09-23T23:29:28.822084+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":3770,"log_dups_size":0,"ondisk_log_size":3770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:45.310904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247070720,"num_objects":3770,"num_object_clones":0,"num_object_copies":11310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3770,"num_write_kb":241280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":767,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:15.716627+0000","last_change":"2023-09-23T23:29:15.716627+0000","last_active":"2023-09-23T23:29:15.716627+0000","last_peered":"2023-09-23T23:29:15.716627+0000","last_clean":"2023-09-23T23:29:15.716627+0000","last_became_active":"2023-09-23T23:28:48.115821+0000","last_became_peered":"2023-09-23T23:28:48.115821+0000","last_unstale":"2023-09-23T23:29:15.716627+0000","last_undegraded":"2023-09-23T23:29:15.716627+0000","last_fullsized":"2023-09-23T23:29:15.716627+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:23.504701+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":393,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.606148+0000","last_change":"2023-09-23T23:26:32.043926+0000","last_active":"2023-09-23T23:29:14.606148+0000","last_peered":"2023-09-23T23:29:14.606148+0000","last_clean":"2023-09-23T23:29:14.606148+0000","last_became_active":"2023-09-23T23:26:32.039282+0000","last_became_peered":"2023-09-23T23:26:32.039282+0000","last_unstale":"2023-09-23T23:29:14.606148+0000","last_undegraded":"2023-09-23T23:29:14.606148+0000","last_fullsized":"2023-09-23T23:29:14.606148+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:35.345086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":476,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.609258+0000","last_change":"2023-09-23T23:26:48.536996+0000","last_active":"2023-09-23T23:29:14.609258+0000","last_peered":"2023-09-23T23:29:14.609258+0000","last_clean":"2023-09-23T23:29:14.609258+0000","last_became_active":"2023-09-23T23:26:48.536638+0000","last_became_peered":"2023-09-23T23:26:48.536638+0000","last_unstale":"2023-09-23T23:29:14.609258+0000","last_undegraded":"2023-09-23T23:29:14.609258+0000","last_fullsized":"2023-09-23T23:29:14.609258+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:36.943318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.3","version":"487'3789","reported_seq":3808,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:28.148023+0000","last_change":"2023-09-23T23:28:38.104163+0000","last_active":"2023-09-23T23:29:28.148023+0000","last_peered":"2023-09-23T23:29:28.148023+0000","last_clean":"2023-09-23T23:29:28.148023+0000","last_became_active":"2023-09-23T23:28:38.103810+0000","last_became_peered":"2023-09-23T23:28:38.103810+0000","last_unstale":"2023-09-23T23:29:28.148023+0000","last_undegraded":"2023-09-23T23:29:28.148023+0000","last_fullsized":"2023-09-23T23:29:28.148023+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":3789,"log_dups_size":0,"ondisk_log_size":3789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:41.275532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248315904,"num_objects":3789,"num_object_clones":0,"num_object_copies":11367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3789,"num_write_kb":242496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":551,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.602102+0000","last_change":"2023-09-23T23:28:03.629593+0000","last_active":"2023-09-23T23:29:14.602102+0000","last_peered":"2023-09-23T23:29:14.602102+0000","last_clean":"2023-09-23T23:29:14.602102+0000","last_became_active":"2023-09-23T23:26:51.588385+0000","last_became_peered":"2023-09-23T23:26:51.588385+0000","last_unstale":"2023-09-23T23:29:14.602102+0000","last_undegraded":"2023-09-23T23:29:14.602102+0000","last_fullsized":"2023-09-23T23:29:14.602102+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:20.818366+0000","scrub_duration":0.0050135120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":315,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.626748+0000","last_change":"2023-09-23T23:25:31.732607+0000","last_active":"2023-09-23T23:29:14.626748+0000","last_peered":"2023-09-23T23:29:14.626748+0000","last_clean":"2023-09-23T23:29:14.626748+0000","last_became_active":"2023-09-23T23:25:31.732099+0000","last_became_peered":"2023-09-23T23:25:31.732099+0000","last_unstale":"2023-09-23T23:29:14.626748+0000","last_undegraded":"2023-09-23T23:29:14.626748+0000","last_fullsized":"2023-09-23T23:29:14.626748+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:23.639936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":472,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.615743+0000","last_change":"2023-09-23T23:28:45.066443+0000","last_active":"2023-09-23T23:29:14.615743+0000","last_peered":"2023-09-23T23:29:14.615743+0000","last_clean":"2023-09-23T23:29:14.615743+0000","last_became_active":"2023-09-23T23:28:45.066154+0000","last_became_peered":"2023-09-23T23:28:45.066154+0000","last_unstale":"2023-09-23T23:29:14.615743+0000","last_undegraded":"2023-09-23T23:29:14.615743+0000","last_fullsized":"2023-09-23T23:29:14.615743+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:51.651425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"487'3621","reported_seq":3641,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:28.893607+0000","last_change":"2023-09-23T23:28:38.061949+0000","last_active":"2023-09-23T23:29:28.893607+0000","last_peered":"2023-09-23T23:29:28.893607+0000","last_clean":"2023-09-23T23:29:28.893607+0000","last_became_active":"2023-09-23T23:28:38.061632+0000","last_became_peered":"2023-09-23T23:28:38.061632+0000","last_unstale":"2023-09-23T23:29:28.893607+0000","last_undegraded":"2023-09-23T23:29:28.893607+0000","last_fullsized":"2023-09-23T23:29:28.893607+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":3621,"log_dups_size":0,"ondisk_log_size":3621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:53.029897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237305856,"num_objects":3621,"num_object_clones":0,"num_object_copies":10863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3621,"num_write_kb":231744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":822,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.605331+0000","last_change":"2023-09-23T23:23:45.830384+0000","last_active":"2023-09-23T23:29:14.605331+0000","last_peered":"2023-09-23T23:29:14.605331+0000","last_clean":"2023-09-23T23:29:14.605331+0000","last_became_active":"2023-09-23T23:23:36.025724+0000","last_became_peered":"2023-09-23T23:23:36.025724+0000","last_unstale":"2023-09-23T23:29:14.605331+0000","last_undegraded":"2023-09-23T23:29:14.605331+0000","last_fullsized":"2023-09-23T23:29:14.605331+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:34.345528+0000","scrub_duration":0.001659259,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":455,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.631844+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:29:14.631844+0000","last_peered":"2023-09-23T23:29:14.631844+0000","last_clean":"2023-09-23T23:29:14.631844+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:29:14.631844+0000","last_undegraded":"2023-09-23T23:29:14.631844+0000","last_fullsized":"2023-09-23T23:29:14.631844+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":455,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.601696+0000","last_change":"2023-09-23T23:25:31.731172+0000","last_active":"2023-09-23T23:29:14.601696+0000","last_peered":"2023-09-23T23:29:14.601696+0000","last_clean":"2023-09-23T23:29:14.601696+0000","last_became_active":"2023-09-23T23:25:31.730903+0000","last_became_peered":"2023-09-23T23:25:31.730903+0000","last_unstale":"2023-09-23T23:29:14.601696+0000","last_undegraded":"2023-09-23T23:29:14.601696+0000","last_fullsized":"2023-09-23T23:29:14.601696+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:35.442945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"487'3942","reported_seq":3962,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:30.985570+0000","last_change":"2023-09-23T23:28:38.032946+0000","last_active":"2023-09-23T23:29:30.985570+0000","last_peered":"2023-09-23T23:29:30.985570+0000","last_clean":"2023-09-23T23:29:30.985570+0000","last_became_active":"2023-09-23T23:28:38.032671+0000","last_became_peered":"2023-09-23T23:28:38.032671+0000","last_unstale":"2023-09-23T23:29:30.985570+0000","last_undegraded":"2023-09-23T23:29:30.985570+0000","last_fullsized":"2023-09-23T23:29:30.985570+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":3942,"log_dups_size":0,"ondisk_log_size":3942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:11:49.143996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258342912,"num_objects":3942,"num_object_clones":0,"num_object_copies":11826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3942,"num_write_kb":252288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":767,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.612785+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:29:14.612785+0000","last_peered":"2023-09-23T23:29:14.612785+0000","last_clean":"2023-09-23T23:29:14.612785+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:29:14.612785+0000","last_undegraded":"2023-09-23T23:29:14.612785+0000","last_fullsized":"2023-09-23T23:29:14.612785+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":313,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.600851+0000","last_change":"2023-09-23T23:27:20.560074+0000","last_active":"2023-09-23T23:29:14.600851+0000","last_peered":"2023-09-23T23:29:14.600851+0000","last_clean":"2023-09-23T23:29:14.600851+0000","last_became_active":"2023-09-23T23:27:20.559764+0000","last_became_peered":"2023-09-23T23:27:20.559764+0000","last_unstale":"2023-09-23T23:29:14.600851+0000","last_undegraded":"2023-09-23T23:29:14.600851+0000","last_fullsized":"2023-09-23T23:29:14.600851+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:23.675227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":108,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.601796+0000","last_change":"2023-09-23T23:26:11.516475+0000","last_active":"2023-09-23T23:29:14.601796+0000","last_peered":"2023-09-23T23:29:14.601796+0000","last_clean":"2023-09-23T23:29:14.601796+0000","last_became_active":"2023-09-23T23:26:11.516198+0000","last_became_peered":"2023-09-23T23:26:11.516198+0000","last_unstale":"2023-09-23T23:29:14.601796+0000","last_undegraded":"2023-09-23T23:29:14.601796+0000","last_fullsized":"2023-09-23T23:29:14.601796+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:56.875103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"487'3660","reported_seq":3680,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:29.759637+0000","last_change":"2023-09-23T23:28:38.105484+0000","last_active":"2023-09-23T23:29:29.759637+0000","last_peered":"2023-09-23T23:29:29.759637+0000","last_clean":"2023-09-23T23:29:29.759637+0000","last_became_active":"2023-09-23T23:28:38.105230+0000","last_became_peered":"2023-09-23T23:28:38.105230+0000","last_unstale":"2023-09-23T23:29:29.759637+0000","last_undegraded":"2023-09-23T23:29:29.759637+0000","last_fullsized":"2023-09-23T23:29:29.759637+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":3660,"log_dups_size":0,"ondisk_log_size":3660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:40.364698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239861760,"num_objects":3660,"num_object_clones":0,"num_object_copies":10980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3660,"num_write_kb":234240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":643,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:25.701812+0000","last_change":"2023-09-23T23:26:34.548211+0000","last_active":"2023-09-23T23:29:25.701812+0000","last_peered":"2023-09-23T23:29:25.701812+0000","last_clean":"2023-09-23T23:29:25.701812+0000","last_became_active":"2023-09-23T23:26:34.547815+0000","last_became_peered":"2023-09-23T23:26:34.547815+0000","last_unstale":"2023-09-23T23:29:25.701812+0000","last_undegraded":"2023-09-23T23:29:25.701812+0000","last_fullsized":"2023-09-23T23:29:25.701812+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:37.675417+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":108,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.600498+0000","last_change":"2023-09-23T23:26:11.512815+0000","last_active":"2023-09-23T23:29:14.600498+0000","last_peered":"2023-09-23T23:29:14.600498+0000","last_clean":"2023-09-23T23:29:14.600498+0000","last_became_active":"2023-09-23T23:26:11.512466+0000","last_became_peered":"2023-09-23T23:26:11.512466+0000","last_unstale":"2023-09-23T23:29:14.600498+0000","last_undegraded":"2023-09-23T23:29:14.600498+0000","last_fullsized":"2023-09-23T23:29:14.600498+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:10.324158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":485,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.602268+0000","last_change":"2023-09-23T23:27:20.558731+0000","last_active":"2023-09-23T23:29:14.602268+0000","last_peered":"2023-09-23T23:29:14.602268+0000","last_clean":"2023-09-23T23:29:14.602268+0000","last_became_active":"2023-09-23T23:27:20.558394+0000","last_became_peered":"2023-09-23T23:27:20.558394+0000","last_unstale":"2023-09-23T23:29:14.602268+0000","last_undegraded":"2023-09-23T23:29:14.602268+0000","last_fullsized":"2023-09-23T23:29:14.602268+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:25:00.408551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"487'3777","reported_seq":3797,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:29.792379+0000","last_change":"2023-09-23T23:28:38.062569+0000","last_active":"2023-09-23T23:29:29.792379+0000","last_peered":"2023-09-23T23:29:29.792379+0000","last_clean":"2023-09-23T23:29:29.792379+0000","last_became_active":"2023-09-23T23:28:38.062269+0000","last_became_peered":"2023-09-23T23:28:38.062269+0000","last_unstale":"2023-09-23T23:29:29.792379+0000","last_undegraded":"2023-09-23T23:29:29.792379+0000","last_fullsized":"2023-09-23T23:29:29.792379+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":3777,"log_dups_size":0,"ondisk_log_size":3777,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:41.750221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247529472,"num_objects":3777,"num_object_clones":0,"num_object_copies":11331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3777,"num_write_kb":241728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":808,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.613052+0000","last_change":"2023-09-23T23:25:34.500038+0000","last_active":"2023-09-23T23:29:14.613052+0000","last_peered":"2023-09-23T23:29:14.613052+0000","last_clean":"2023-09-23T23:29:14.613052+0000","last_became_active":"2023-09-23T23:25:34.499705+0000","last_became_peered":"2023-09-23T23:25:34.499705+0000","last_unstale":"2023-09-23T23:29:14.613052+0000","last_undegraded":"2023-09-23T23:29:14.613052+0000","last_fullsized":"2023-09-23T23:29:14.613052+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:43.096696+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":125,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.622042+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:29:14.622042+0000","last_peered":"2023-09-23T23:29:14.622042+0000","last_clean":"2023-09-23T23:29:14.622042+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:29:14.622042+0000","last_undegraded":"2023-09-23T23:29:14.622042+0000","last_fullsized":"2023-09-23T23:29:14.622042+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":409,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:24.806232+0000","last_change":"2023-09-23T23:22:39.489325+0000","last_active":"2023-09-23T23:29:24.806232+0000","last_peered":"2023-09-23T23:29:24.806232+0000","last_clean":"2023-09-23T23:29:24.806232+0000","last_became_active":"2023-09-23T23:22:39.489048+0000","last_became_peered":"2023-09-23T23:22:39.489048+0000","last_unstale":"2023-09-23T23:29:24.806232+0000","last_undegraded":"2023-09-23T23:29:24.806232+0000","last_fullsized":"2023-09-23T23:29:24.806232+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:29.984948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"487'3913","reported_seq":3931,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:31.030208+0000","last_change":"2023-09-23T23:28:38.107589+0000","last_active":"2023-09-23T23:29:31.030208+0000","last_peered":"2023-09-23T23:29:31.030208+0000","last_clean":"2023-09-23T23:29:31.030208+0000","last_became_active":"2023-09-23T23:28:38.107296+0000","last_became_peered":"2023-09-23T23:28:38.107296+0000","last_unstale":"2023-09-23T23:29:31.030208+0000","last_undegraded":"2023-09-23T23:29:31.030208+0000","last_fullsized":"2023-09-23T23:29:31.030208+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":3913,"log_dups_size":0,"ondisk_log_size":3913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:12.352223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256442368,"num_objects":3913,"num_object_clones":0,"num_object_copies":11739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3913,"num_write_kb":250432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":798,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.623589+0000","last_change":"2023-09-23T23:28:11.839674+0000","last_active":"2023-09-23T23:29:14.623589+0000","last_peered":"2023-09-23T23:29:14.623589+0000","last_clean":"2023-09-23T23:29:14.623589+0000","last_became_active":"2023-09-23T23:26:51.561394+0000","last_became_peered":"2023-09-23T23:26:51.561394+0000","last_unstale":"2023-09-23T23:29:14.623589+0000","last_undegraded":"2023-09-23T23:29:14.623589+0000","last_fullsized":"2023-09-23T23:29:14.623589+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:41.710212+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":189,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.610152+0000","last_change":"2023-09-23T23:23:36.048889+0000","last_active":"2023-09-23T23:29:14.610152+0000","last_peered":"2023-09-23T23:29:14.610152+0000","last_clean":"2023-09-23T23:29:14.610152+0000","last_became_active":"2023-09-23T23:23:36.048572+0000","last_became_peered":"2023-09-23T23:23:36.048572+0000","last_unstale":"2023-09-23T23:29:14.610152+0000","last_undegraded":"2023-09-23T23:29:14.610152+0000","last_fullsized":"2023-09-23T23:29:14.610152+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:55.581146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":502,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.603246+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:29:14.603246+0000","last_peered":"2023-09-23T23:29:14.603246+0000","last_clean":"2023-09-23T23:29:14.603246+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:29:14.603246+0000","last_undegraded":"2023-09-23T23:29:14.603246+0000","last_fullsized":"2023-09-23T23:29:14.603246+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":74,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:25.701671+0000","last_change":"2023-09-23T23:27:20.549904+0000","last_active":"2023-09-23T23:29:25.701671+0000","last_peered":"2023-09-23T23:29:25.701671+0000","last_clean":"2023-09-23T23:29:25.701671+0000","last_became_active":"2023-09-23T23:27:20.549645+0000","last_became_peered":"2023-09-23T23:27:20.549645+0000","last_unstale":"2023-09-23T23:29:25.701671+0000","last_undegraded":"2023-09-23T23:29:25.701671+0000","last_fullsized":"2023-09-23T23:29:25.701671+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:09.850640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"487'3863","reported_seq":3883,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:30.991291+0000","last_change":"2023-09-23T23:28:38.107697+0000","last_active":"2023-09-23T23:29:30.991291+0000","last_peered":"2023-09-23T23:29:30.991291+0000","last_clean":"2023-09-23T23:29:30.991291+0000","last_became_active":"2023-09-23T23:28:38.107379+0000","last_became_peered":"2023-09-23T23:28:38.107379+0000","last_unstale":"2023-09-23T23:29:30.991291+0000","last_undegraded":"2023-09-23T23:29:30.991291+0000","last_fullsized":"2023-09-23T23:29:30.991291+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":3863,"log_dups_size":0,"ondisk_log_size":3863,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:12.478832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253165568,"num_objects":3863,"num_object_clones":0,"num_object_copies":11589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3863,"num_write_kb":247232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":633,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.611605+0000","last_change":"2023-09-23T23:23:49.320969+0000","last_active":"2023-09-23T23:29:14.611605+0000","last_peered":"2023-09-23T23:29:14.611605+0000","last_clean":"2023-09-23T23:29:14.611605+0000","last_became_active":"2023-09-23T23:22:17.001911+0000","last_became_peered":"2023-09-23T23:22:17.001911+0000","last_unstale":"2023-09-23T23:29:14.611605+0000","last_undegraded":"2023-09-23T23:29:14.611605+0000","last_fullsized":"2023-09-23T23:29:14.611605+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.320823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.320823+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:07.000984+0000","scrub_duration":0.0018140630000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":189,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.609348+0000","last_change":"2023-09-23T23:25:31.731396+0000","last_active":"2023-09-23T23:29:14.609348+0000","last_peered":"2023-09-23T23:29:14.609348+0000","last_clean":"2023-09-23T23:29:14.609348+0000","last_became_active":"2023-09-23T23:25:31.730931+0000","last_became_peered":"2023-09-23T23:25:31.730931+0000","last_unstale":"2023-09-23T23:29:14.609348+0000","last_undegraded":"2023-09-23T23:29:14.609348+0000","last_fullsized":"2023-09-23T23:29:14.609348+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:48.002628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":466,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.600905+0000","last_change":"2023-09-23T23:27:20.556916+0000","last_active":"2023-09-23T23:29:14.600905+0000","last_peered":"2023-09-23T23:29:14.600905+0000","last_clean":"2023-09-23T23:29:14.600905+0000","last_became_active":"2023-09-23T23:27:20.556651+0000","last_became_peered":"2023-09-23T23:27:20.556651+0000","last_unstale":"2023-09-23T23:29:14.600905+0000","last_undegraded":"2023-09-23T23:29:14.600905+0000","last_fullsized":"2023-09-23T23:29:14.600905+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:06.012543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":74,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:25.702400+0000","last_change":"2023-09-23T23:27:20.551221+0000","last_active":"2023-09-23T23:29:25.702400+0000","last_peered":"2023-09-23T23:29:25.702400+0000","last_clean":"2023-09-23T23:29:25.702400+0000","last_became_active":"2023-09-23T23:27:20.550900+0000","last_became_peered":"2023-09-23T23:27:20.550900+0000","last_unstale":"2023-09-23T23:29:25.702400+0000","last_undegraded":"2023-09-23T23:29:25.702400+0000","last_fullsized":"2023-09-23T23:29:25.702400+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:23.012149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"487'3740","reported_seq":3759,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:28.845134+0000","last_change":"2023-09-23T23:28:38.032025+0000","last_active":"2023-09-23T23:29:28.845134+0000","last_peered":"2023-09-23T23:29:28.845134+0000","last_clean":"2023-09-23T23:29:28.845134+0000","last_became_active":"2023-09-23T23:28:38.031791+0000","last_became_peered":"2023-09-23T23:28:38.031791+0000","last_unstale":"2023-09-23T23:29:28.845134+0000","last_undegraded":"2023-09-23T23:29:28.845134+0000","last_fullsized":"2023-09-23T23:29:28.845134+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":3740,"log_dups_size":0,"ondisk_log_size":3740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:44.488888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245104640,"num_objects":3740,"num_object_clones":0,"num_object_copies":11220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3740,"num_write_kb":239360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":787,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.612949+0000","last_change":"2023-09-23T23:28:48.113032+0000","last_active":"2023-09-23T23:29:14.612949+0000","last_peered":"2023-09-23T23:29:14.612949+0000","last_clean":"2023-09-23T23:29:14.612949+0000","last_became_active":"2023-09-23T23:28:48.112666+0000","last_became_peered":"2023-09-23T23:28:48.112666+0000","last_unstale":"2023-09-23T23:29:14.612949+0000","last_undegraded":"2023-09-23T23:29:14.612949+0000","last_fullsized":"2023-09-23T23:29:14.612949+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:59:59.838039+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":222,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.611971+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:29:14.611971+0000","last_peered":"2023-09-23T23:29:14.611971+0000","last_clean":"2023-09-23T23:29:14.611971+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:29:14.611971+0000","last_undegraded":"2023-09-23T23:29:14.611971+0000","last_fullsized":"2023-09-23T23:29:14.611971+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":189,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.608984+0000","last_change":"2023-09-23T23:25:31.828908+0000","last_active":"2023-09-23T23:29:14.608984+0000","last_peered":"2023-09-23T23:29:14.608984+0000","last_clean":"2023-09-23T23:29:14.608984+0000","last_became_active":"2023-09-23T23:25:31.828582+0000","last_became_peered":"2023-09-23T23:25:31.828582+0000","last_unstale":"2023-09-23T23:29:14.608984+0000","last_undegraded":"2023-09-23T23:29:14.608984+0000","last_fullsized":"2023-09-23T23:29:14.608984+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:01.466593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":73,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.609723+0000","last_change":"2023-09-23T23:27:20.551574+0000","last_active":"2023-09-23T23:29:14.609723+0000","last_peered":"2023-09-23T23:29:14.609723+0000","last_clean":"2023-09-23T23:29:14.609723+0000","last_became_active":"2023-09-23T23:27:20.550820+0000","last_became_peered":"2023-09-23T23:27:20.550820+0000","last_unstale":"2023-09-23T23:29:14.609723+0000","last_undegraded":"2023-09-23T23:29:14.609723+0000","last_fullsized":"2023-09-23T23:29:14.609723+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:09.274552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"487'3647","reported_seq":3666,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:28.154085+0000","last_change":"2023-09-23T23:28:38.065394+0000","last_active":"2023-09-23T23:29:28.154085+0000","last_peered":"2023-09-23T23:29:28.154085+0000","last_clean":"2023-09-23T23:29:28.154085+0000","last_became_active":"2023-09-23T23:28:38.065160+0000","last_became_peered":"2023-09-23T23:28:38.065160+0000","last_unstale":"2023-09-23T23:29:28.154085+0000","last_undegraded":"2023-09-23T23:29:28.154085+0000","last_fullsized":"2023-09-23T23:29:28.154085+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":3647,"log_dups_size":0,"ondisk_log_size":3647,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:10:39.685945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239009792,"num_objects":3647,"num_object_clones":0,"num_object_copies":10941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3647,"num_write_kb":233408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":498,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.616290+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:29:14.616290+0000","last_peered":"2023-09-23T23:29:14.616290+0000","last_clean":"2023-09-23T23:29:14.616290+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:29:14.616290+0000","last_undegraded":"2023-09-23T23:29:14.616290+0000","last_fullsized":"2023-09-23T23:29:14.616290+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":807,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.606231+0000","last_change":"2023-09-23T23:26:51.564817+0000","last_active":"2023-09-23T23:29:14.606231+0000","last_peered":"2023-09-23T23:29:14.606231+0000","last_clean":"2023-09-23T23:29:14.606231+0000","last_became_active":"2023-09-23T23:26:51.564511+0000","last_became_peered":"2023-09-23T23:26:51.564511+0000","last_unstale":"2023-09-23T23:29:14.606231+0000","last_undegraded":"2023-09-23T23:29:14.606231+0000","last_fullsized":"2023-09-23T23:29:14.606231+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:02.415212+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":436,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.623231+0000","last_change":"2023-09-23T23:26:32.037619+0000","last_active":"2023-09-23T23:29:14.623231+0000","last_peered":"2023-09-23T23:29:14.623231+0000","last_clean":"2023-09-23T23:29:14.623231+0000","last_became_active":"2023-09-23T23:26:32.037045+0000","last_became_peered":"2023-09-23T23:26:32.037045+0000","last_unstale":"2023-09-23T23:29:14.623231+0000","last_undegraded":"2023-09-23T23:29:14.623231+0000","last_fullsized":"2023-09-23T23:29:14.623231+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:47.657527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":108,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.601802+0000","last_change":"2023-09-23T23:26:11.525845+0000","last_active":"2023-09-23T23:29:14.601802+0000","last_peered":"2023-09-23T23:29:14.601802+0000","last_clean":"2023-09-23T23:29:14.601802+0000","last_became_active":"2023-09-23T23:26:11.525507+0000","last_became_peered":"2023-09-23T23:26:11.525507+0000","last_unstale":"2023-09-23T23:29:14.601802+0000","last_undegraded":"2023-09-23T23:29:14.601802+0000","last_fullsized":"2023-09-23T23:29:14.601802+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:35.462874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.e","version":"487'3780","reported_seq":3799,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:28.835322+0000","last_change":"2023-09-23T23:28:38.104180+0000","last_active":"2023-09-23T23:29:28.835322+0000","last_peered":"2023-09-23T23:29:28.835322+0000","last_clean":"2023-09-23T23:29:28.835322+0000","last_became_active":"2023-09-23T23:28:38.103844+0000","last_became_peered":"2023-09-23T23:28:38.103844+0000","last_unstale":"2023-09-23T23:29:28.835322+0000","last_undegraded":"2023-09-23T23:29:28.835322+0000","last_fullsized":"2023-09-23T23:29:28.835322+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":3780,"log_dups_size":0,"ondisk_log_size":3780,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:48.427805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247726080,"num_objects":3780,"num_object_clones":0,"num_object_copies":11340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3780,"num_write_kb":241920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":651,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.609529+0000","last_change":"2023-09-23T23:26:34.550276+0000","last_active":"2023-09-23T23:29:14.609529+0000","last_peered":"2023-09-23T23:29:14.609529+0000","last_clean":"2023-09-23T23:29:14.609529+0000","last_became_active":"2023-09-23T23:26:34.549923+0000","last_became_peered":"2023-09-23T23:26:34.549923+0000","last_unstale":"2023-09-23T23:29:14.609529+0000","last_undegraded":"2023-09-23T23:29:14.609529+0000","last_fullsized":"2023-09-23T23:29:14.609529+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:22.636211+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":414,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.605921+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:29:14.605921+0000","last_peered":"2023-09-23T23:29:14.605921+0000","last_clean":"2023-09-23T23:29:14.605921+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:29:14.605921+0000","last_undegraded":"2023-09-23T23:29:14.605921+0000","last_fullsized":"2023-09-23T23:29:14.605921+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":413,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.600422+0000","last_change":"2023-09-23T23:28:22.601436+0000","last_active":"2023-09-23T23:29:14.600422+0000","last_peered":"2023-09-23T23:29:14.600422+0000","last_clean":"2023-09-23T23:29:14.600422+0000","last_became_active":"2023-09-23T23:28:22.601154+0000","last_became_peered":"2023-09-23T23:28:22.601154+0000","last_unstale":"2023-09-23T23:29:14.600422+0000","last_undegraded":"2023-09-23T23:29:14.600422+0000","last_fullsized":"2023-09-23T23:29:14.600422+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:03.296778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":517,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.617476+0000","last_change":"2023-09-23T23:22:38.438448+0000","last_active":"2023-09-23T23:29:14.617476+0000","last_peered":"2023-09-23T23:29:14.617476+0000","last_clean":"2023-09-23T23:29:14.617476+0000","last_became_active":"2023-09-23T23:22:38.437495+0000","last_became_peered":"2023-09-23T23:22:38.437495+0000","last_unstale":"2023-09-23T23:29:14.617476+0000","last_undegraded":"2023-09-23T23:29:14.617476+0000","last_fullsized":"2023-09-23T23:29:14.617476+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:49.587056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"487'3690","reported_seq":3709,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:28.891102+0000","last_change":"2023-09-23T23:28:38.032628+0000","last_active":"2023-09-23T23:29:28.891102+0000","last_peered":"2023-09-23T23:29:28.891102+0000","last_clean":"2023-09-23T23:29:28.891102+0000","last_became_active":"2023-09-23T23:28:38.032274+0000","last_became_peered":"2023-09-23T23:28:38.032274+0000","last_unstale":"2023-09-23T23:29:28.891102+0000","last_undegraded":"2023-09-23T23:29:28.891102+0000","last_fullsized":"2023-09-23T23:29:28.891102+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":3690,"log_dups_size":0,"ondisk_log_size":3690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:32:23.382504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241827840,"num_objects":3690,"num_object_clones":0,"num_object_copies":11070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3690,"num_write_kb":236160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":528,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.623204+0000","last_change":"2023-09-23T23:28:05.843008+0000","last_active":"2023-09-23T23:29:14.623204+0000","last_peered":"2023-09-23T23:29:14.623204+0000","last_clean":"2023-09-23T23:29:14.623204+0000","last_became_active":"2023-09-23T23:26:04.560484+0000","last_became_peered":"2023-09-23T23:26:04.560484+0000","last_unstale":"2023-09-23T23:29:14.623204+0000","last_undegraded":"2023-09-23T23:29:14.623204+0000","last_fullsized":"2023-09-23T23:29:14.623204+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.842881+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.842881+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:37.322347+0000","scrub_duration":0.0047245100000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":338,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.606061+0000","last_change":"2023-09-23T23:23:35.037351+0000","last_active":"2023-09-23T23:29:14.606061+0000","last_peered":"2023-09-23T23:29:14.606061+0000","last_clean":"2023-09-23T23:29:14.606061+0000","last_became_active":"2023-09-23T23:23:35.036841+0000","last_became_peered":"2023-09-23T23:23:35.036841+0000","last_unstale":"2023-09-23T23:29:14.606061+0000","last_undegraded":"2023-09-23T23:29:14.606061+0000","last_fullsized":"2023-09-23T23:29:14.606061+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:20.618292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":189,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.610231+0000","last_change":"2023-09-23T23:26:32.037099+0000","last_active":"2023-09-23T23:29:14.610231+0000","last_peered":"2023-09-23T23:29:14.610231+0000","last_clean":"2023-09-23T23:29:14.610231+0000","last_became_active":"2023-09-23T23:26:32.036681+0000","last_became_peered":"2023-09-23T23:26:32.036681+0000","last_unstale":"2023-09-23T23:29:14.610231+0000","last_undegraded":"2023-09-23T23:29:14.610231+0000","last_fullsized":"2023-09-23T23:29:14.610231+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:48.926576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":220,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.602827+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:29:14.602827+0000","last_peered":"2023-09-23T23:29:14.602827+0000","last_clean":"2023-09-23T23:29:14.602827+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:29:14.602827+0000","last_undegraded":"2023-09-23T23:29:14.602827+0000","last_fullsized":"2023-09-23T23:29:14.602827+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"487'3717","reported_seq":3737,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:28.878108+0000","last_change":"2023-09-23T23:28:38.104037+0000","last_active":"2023-09-23T23:29:28.878108+0000","last_peered":"2023-09-23T23:29:28.878108+0000","last_clean":"2023-09-23T23:29:28.878108+0000","last_became_active":"2023-09-23T23:28:38.103739+0000","last_became_peered":"2023-09-23T23:28:38.103739+0000","last_unstale":"2023-09-23T23:29:28.878108+0000","last_undegraded":"2023-09-23T23:29:28.878108+0000","last_fullsized":"2023-09-23T23:29:28.878108+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":3717,"log_dups_size":0,"ondisk_log_size":3717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:02:06.842073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243597312,"num_objects":3717,"num_object_clones":0,"num_object_copies":11151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3717,"num_write_kb":237888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":798,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.601871+0000","last_change":"2023-09-23T23:26:14.518921+0000","last_active":"2023-09-23T23:29:14.601871+0000","last_peered":"2023-09-23T23:29:14.601871+0000","last_clean":"2023-09-23T23:29:14.601871+0000","last_became_active":"2023-09-23T23:26:14.518558+0000","last_became_peered":"2023-09-23T23:26:14.518558+0000","last_unstale":"2023-09-23T23:29:14.601871+0000","last_undegraded":"2023-09-23T23:29:14.601871+0000","last_fullsized":"2023-09-23T23:29:14.601871+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:14.360597+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":481,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.623873+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:29:14.623873+0000","last_peered":"2023-09-23T23:29:14.623873+0000","last_clean":"2023-09-23T23:29:14.623873+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:29:14.623873+0000","last_undegraded":"2023-09-23T23:29:14.623873+0000","last_fullsized":"2023-09-23T23:29:14.623873+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":73,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.610692+0000","last_change":"2023-09-23T23:27:20.562744+0000","last_active":"2023-09-23T23:29:14.610692+0000","last_peered":"2023-09-23T23:29:14.610692+0000","last_clean":"2023-09-23T23:29:14.610692+0000","last_became_active":"2023-09-23T23:27:20.562383+0000","last_became_peered":"2023-09-23T23:27:20.562383+0000","last_unstale":"2023-09-23T23:29:14.610692+0000","last_undegraded":"2023-09-23T23:29:14.610692+0000","last_fullsized":"2023-09-23T23:29:14.610692+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:01.085133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":17,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.612509+0000","last_change":"2023-09-23T23:28:45.061717+0000","last_active":"2023-09-23T23:29:14.612509+0000","last_peered":"2023-09-23T23:29:14.612509+0000","last_clean":"2023-09-23T23:29:14.612509+0000","last_became_active":"2023-09-23T23:28:45.061404+0000","last_became_peered":"2023-09-23T23:28:45.061404+0000","last_unstale":"2023-09-23T23:29:14.612509+0000","last_undegraded":"2023-09-23T23:29:14.612509+0000","last_fullsized":"2023-09-23T23:29:14.612509+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:46.614114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.9","version":"487'3791","reported_seq":3810,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-09-23T23:29:31.029649+0000","last_change":"2023-09-23T23:28:38.034022+0000","last_active":"2023-09-23T23:29:31.029649+0000","last_peered":"2023-09-23T23:29:31.029649+0000","last_clean":"2023-09-23T23:29:31.029649+0000","last_became_active":"2023-09-23T23:28:38.033769+0000","last_became_peered":"2023-09-23T23:28:38.033769+0000","last_unstale":"2023-09-23T23:29:31.029649+0000","last_undegraded":"2023-09-23T23:29:31.029649+0000","last_fullsized":"2023-09-23T23:29:31.029649+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":3791,"log_dups_size":0,"ondisk_log_size":3791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:23.894936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248446976,"num_objects":3791,"num_object_clones":0,"num_object_copies":11373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3791,"num_write_kb":242624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":871,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:23.908276+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:29:23.908276+0000","last_peered":"2023-09-23T23:29:23.908276+0000","last_clean":"2023-09-23T23:29:23.908276+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:29:23.908276+0000","last_undegraded":"2023-09-23T23:29:23.908276+0000","last_fullsized":"2023-09-23T23:29:23.908276+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":86,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.606025+0000","last_change":"2023-09-23T23:26:48.544465+0000","last_active":"2023-09-23T23:29:14.606025+0000","last_peered":"2023-09-23T23:29:14.606025+0000","last_clean":"2023-09-23T23:29:14.606025+0000","last_became_active":"2023-09-23T23:26:48.544166+0000","last_became_peered":"2023-09-23T23:26:48.544166+0000","last_unstale":"2023-09-23T23:29:14.606025+0000","last_undegraded":"2023-09-23T23:29:14.606025+0000","last_fullsized":"2023-09-23T23:29:14.606025+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:38.844346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":472,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.617298+0000","last_change":"2023-09-23T23:22:38.482705+0000","last_active":"2023-09-23T23:29:14.617298+0000","last_peered":"2023-09-23T23:29:14.617298+0000","last_clean":"2023-09-23T23:29:14.617298+0000","last_became_active":"2023-09-23T23:22:38.438239+0000","last_became_peered":"2023-09-23T23:22:38.438239+0000","last_unstale":"2023-09-23T23:29:14.617298+0000","last_undegraded":"2023-09-23T23:29:14.617298+0000","last_fullsized":"2023-09-23T23:29:14.617298+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:52.006349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":108,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.600511+0000","last_change":"2023-09-23T23:26:11.522604+0000","last_active":"2023-09-23T23:29:14.600511+0000","last_peered":"2023-09-23T23:29:14.600511+0000","last_clean":"2023-09-23T23:29:14.600511+0000","last_became_active":"2023-09-23T23:26:11.522350+0000","last_became_peered":"2023-09-23T23:26:11.522350+0000","last_unstale":"2023-09-23T23:29:14.600511+0000","last_undegraded":"2023-09-23T23:29:14.600511+0000","last_fullsized":"2023-09-23T23:29:14.600511+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:20.164202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":869,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.625315+0000","last_change":"2023-09-23T23:25:34.505042+0000","last_active":"2023-09-23T23:29:14.625315+0000","last_peered":"2023-09-23T23:29:14.625315+0000","last_clean":"2023-09-23T23:29:14.625315+0000","last_became_active":"2023-09-23T23:25:34.504687+0000","last_became_peered":"2023-09-23T23:25:34.504687+0000","last_unstale":"2023-09-23T23:29:14.625315+0000","last_undegraded":"2023-09-23T23:29:14.625315+0000","last_fullsized":"2023-09-23T23:29:14.625315+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:06.369262+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":157,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.600877+0000","last_change":"2023-09-23T23:28:22.603176+0000","last_active":"2023-09-23T23:29:14.600877+0000","last_peered":"2023-09-23T23:29:14.600877+0000","last_clean":"2023-09-23T23:29:14.600877+0000","last_became_active":"2023-09-23T23:28:22.602824+0000","last_became_peered":"2023-09-23T23:28:22.602824+0000","last_unstale":"2023-09-23T23:29:14.600877+0000","last_undegraded":"2023-09-23T23:29:14.600877+0000","last_fullsized":"2023-09-23T23:29:14.600877+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:35.934235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":239,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.612241+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:29:14.612241+0000","last_peered":"2023-09-23T23:29:14.612241+0000","last_clean":"2023-09-23T23:29:14.612241+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:29:14.612241+0000","last_undegraded":"2023-09-23T23:29:14.612241+0000","last_fullsized":"2023-09-23T23:29:14.612241+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":614,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.623665+0000","last_change":"2023-09-23T23:26:14.522158+0000","last_active":"2023-09-23T23:29:14.623665+0000","last_peered":"2023-09-23T23:29:14.623665+0000","last_clean":"2023-09-23T23:29:14.623665+0000","last_became_active":"2023-09-23T23:26:14.521785+0000","last_became_peered":"2023-09-23T23:26:14.521785+0000","last_unstale":"2023-09-23T23:29:14.623665+0000","last_undegraded":"2023-09-23T23:29:14.623665+0000","last_fullsized":"2023-09-23T23:29:14.623665+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:39.445601+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":228,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.601701+0000","last_change":"2023-09-23T23:22:38.452968+0000","last_active":"2023-09-23T23:29:14.601701+0000","last_peered":"2023-09-23T23:29:14.601701+0000","last_clean":"2023-09-23T23:29:14.601701+0000","last_became_active":"2023-09-23T23:22:38.437986+0000","last_became_peered":"2023-09-23T23:22:38.437986+0000","last_unstale":"2023-09-23T23:29:14.601701+0000","last_undegraded":"2023-09-23T23:29:14.601701+0000","last_fullsized":"2023-09-23T23:29:14.601701+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:10.333271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":118,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.600744+0000","last_change":"2023-09-23T23:27:20.558454+0000","last_active":"2023-09-23T23:29:14.600744+0000","last_peered":"2023-09-23T23:29:14.600744+0000","last_clean":"2023-09-23T23:29:14.600744+0000","last_became_active":"2023-09-23T23:27:20.558204+0000","last_became_peered":"2023-09-23T23:27:20.558204+0000","last_unstale":"2023-09-23T23:29:14.600744+0000","last_undegraded":"2023-09-23T23:29:14.600744+0000","last_fullsized":"2023-09-23T23:29:14.600744+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:15.079990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":228,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.601900+0000","last_change":"2023-09-23T23:25:31.740467+0000","last_active":"2023-09-23T23:29:14.601900+0000","last_peered":"2023-09-23T23:29:14.601900+0000","last_clean":"2023-09-23T23:29:14.601900+0000","last_became_active":"2023-09-23T23:25:31.729046+0000","last_became_peered":"2023-09-23T23:25:31.729046+0000","last_unstale":"2023-09-23T23:29:14.601900+0000","last_undegraded":"2023-09-23T23:29:14.601900+0000","last_fullsized":"2023-09-23T23:29:14.601900+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:31.222969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":108,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.600642+0000","last_change":"2023-09-23T23:26:11.527453+0000","last_active":"2023-09-23T23:29:14.600642+0000","last_peered":"2023-09-23T23:29:14.600642+0000","last_clean":"2023-09-23T23:29:14.600642+0000","last_became_active":"2023-09-23T23:26:11.527208+0000","last_became_peered":"2023-09-23T23:26:11.527208+0000","last_unstale":"2023-09-23T23:29:14.600642+0000","last_undegraded":"2023-09-23T23:29:14.600642+0000","last_fullsized":"2023-09-23T23:29:14.600642+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:15.083611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":753,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.606213+0000","last_change":"2023-09-23T23:25:34.510468+0000","last_active":"2023-09-23T23:29:14.606213+0000","last_peered":"2023-09-23T23:29:14.606213+0000","last_clean":"2023-09-23T23:29:14.606213+0000","last_became_active":"2023-09-23T23:25:34.510106+0000","last_became_peered":"2023-09-23T23:25:34.510106+0000","last_unstale":"2023-09-23T23:29:14.606213+0000","last_undegraded":"2023-09-23T23:29:14.606213+0000","last_fullsized":"2023-09-23T23:29:14.606213+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:42.157293+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":505,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.623524+0000","last_change":"2023-09-23T23:25:31.746383+0000","last_active":"2023-09-23T23:29:14.623524+0000","last_peered":"2023-09-23T23:29:14.623524+0000","last_clean":"2023-09-23T23:29:14.623524+0000","last_became_active":"2023-09-23T23:25:31.745929+0000","last_became_peered":"2023-09-23T23:25:31.745929+0000","last_unstale":"2023-09-23T23:29:14.623524+0000","last_undegraded":"2023-09-23T23:29:14.623524+0000","last_fullsized":"2023-09-23T23:29:14.623524+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:21.696449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":498,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.625448+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:29:14.625448+0000","last_peered":"2023-09-23T23:29:14.625448+0000","last_clean":"2023-09-23T23:29:14.625448+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:29:14.625448+0000","last_undegraded":"2023-09-23T23:29:14.625448+0000","last_fullsized":"2023-09-23T23:29:14.625448+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":861,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.609225+0000","last_change":"2023-09-23T23:26:14.527390+0000","last_active":"2023-09-23T23:29:14.609225+0000","last_peered":"2023-09-23T23:29:14.609225+0000","last_clean":"2023-09-23T23:29:14.609225+0000","last_became_active":"2023-09-23T23:26:14.527008+0000","last_became_peered":"2023-09-23T23:26:14.527008+0000","last_unstale":"2023-09-23T23:29:14.609225+0000","last_undegraded":"2023-09-23T23:29:14.609225+0000","last_fullsized":"2023-09-23T23:29:14.609225+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:36.460827+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":297,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.612587+0000","last_change":"2023-09-23T23:28:22.600921+0000","last_active":"2023-09-23T23:29:14.612587+0000","last_peered":"2023-09-23T23:29:14.612587+0000","last_clean":"2023-09-23T23:29:14.612587+0000","last_became_active":"2023-09-23T23:28:22.600565+0000","last_became_peered":"2023-09-23T23:28:22.600565+0000","last_unstale":"2023-09-23T23:29:14.612587+0000","last_undegraded":"2023-09-23T23:29:14.612587+0000","last_fullsized":"2023-09-23T23:29:14.612587+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:07.793563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":532,"reported_epoch":486,"state":"active+clean","last_fresh":"2023-09-23T23:29:14.600528+0000","last_change":"2023-09-23T23:25:31.827217+0000","last_active":"2023-09-23T23:29:14.600528+0000","last_peered":"2023-09-23T23:29:14.600528+0000","last_clean":"2023-09-23T23:29:14.600528+0000","last_became_active":"2023-09-23T23:25:31.826915+0000","last_became_peered":"2023-09-23T23:25:31.826915+0000","last_unstale":"2023-09-23T23:29:14.600528+0000","last_undegraded":"2023-09-23T23:29:14.600528+0000","last_fullsized":"2023-09-23T23:29:14.600528+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:44.656199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":3951165440,"num_objects":60290,"num_object_clones":0,"num_object_copies":180870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":60290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":60290,"num_write_kb":3858560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11880497152,"data_stored":11880497152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":60290,"ondisk_log_size":60290,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":5608,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782600,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2367788,"kb_used_data":2139492,"kb_used_omap":8,"kb_used_meta":228279,"kb_avail":91377364,"statfs":{"total":95995035648,"available":93570420736,"internally_reserved":0,"allocated":2190839808,"data_stored":2189531733,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":233757765},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:28:43 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":2.8759999999999999,"15min":1.9490000000000001},"min":{"1min":0.245,"5min":0.216,"15min":0.216},"max":{"1min":4.7089999999999996,"5min":85.721000000000004,"15min":85.721000000000004},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.67400000000000004,"5min":3.2730000000000001,"15min":1.821},"min":{"1min":0.253,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":2.7149999999999999,"5min":85.965000000000003,"15min":85.965000000000003},"last":0.32400000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:28:43 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":1.6910000000000001,"15min":1.256},"min":{"1min":0.26800000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.6629999999999998,"5min":60.847000000000001,"15min":60.847000000000001},"last":0.23200000000000001},{"interface":"front","average":{"1min":0.65300000000000002,"5min":1.6719999999999999,"15min":1.536},"min":{"1min":0.375,"5min":0.246,"15min":0.246},"max":{"1min":2.7490000000000001,"5min":59.512999999999998,"15min":59.512999999999998},"last":8.8209999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:28:43 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":2.0409999999999999,"15min":1.9019999999999999},"min":{"1min":0.30499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.6019999999999999,"5min":55.811999999999998,"15min":55.811999999999998},"last":13.500999999999999},{"interface":"front","average":{"1min":1.282,"5min":1.9219999999999999,"15min":1.4350000000000001},"min":{"1min":0.312,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.6779999999999999,"5min":55.847999999999999,"15min":55.847999999999999},"last":0.36199999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:28:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.4019999999999999,"15min":1.46},"min":{"1min":0.35399999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.7119999999999997,"5min":9.9960000000000004,"15min":9.9960000000000004},"last":3.2890000000000001},{"interface":"front","average":{"1min":0.78800000000000003,"5min":1.397,"15min":1.3280000000000001},"min":{"1min":0.40000000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.9660000000000002,"5min":9.9659999999999993,"15min":9.9659999999999993},"last":2.1880000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:28:43 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.65700000000000003,"15min":0.79100000000000004},"min":{"1min":0.318,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.028,"5min":6.8380000000000001,"15min":6.8380000000000001},"last":0.307},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.746,"15min":0.874},"min":{"1min":0.32400000000000001,"5min":0.245,"15min":0.245},"max":{"1min":3.2970000000000002,"5min":5.242,"15min":5.242},"last":0.41399999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:28:43 2023","interfaces":[{"interface":"back","average":{"1min":11.726000000000001,"5min":3.028,"15min":1.734},"min":{"1min":0.36199999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":210.83600000000001,"5min":210.83600000000001,"15min":210.83600000000001},"last":0.374},{"interface":"front","average":{"1min":11.618,"5min":2.9489999999999998,"15min":1.5229999999999999},"min":{"1min":0.36399999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":207.04599999999999,"5min":207.04599999999999,"15min":207.04599999999999},"last":0.39900000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:28:43 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.407,"15min":0.97199999999999998},"min":{"1min":0.36899999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.758,"5min":62.213000000000001,"15min":62.213000000000001},"last":0.437},{"interface":"front","average":{"1min":0.76600000000000001,"5min":1.462,"15min":1.2130000000000001},"min":{"1min":0.36099999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.7309999999999999,"5min":62.805999999999997,"15min":62.805999999999997},"last":0.35099999999999998}]}]},{"osd":6,"up_from":0,"seq":197568496077,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":112,"seq":481036337418,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2412376,"kb_used_data":1660688,"kb_used_omap":4,"kb_used_meta":751675,"kb_avail":91332776,"statfs":{"total":95995035648,"available":93524762624,"internally_reserved":0,"allocated":1700544512,"data_stored":1698314205,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":769716110},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:28:59 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":4.9640000000000004,"15min":3.806},"min":{"1min":0.23100000000000001,"5min":0.192,"15min":0.16800000000000001},"max":{"1min":6.5090000000000003,"5min":130.768,"15min":130.768},"last":8.2050000000000001},{"interface":"front","average":{"1min":1.03,"5min":4.0789999999999997,"15min":3.6890000000000001},"min":{"1min":0.24399999999999999,"5min":0.191,"15min":0.191},"max":{"1min":5.7789999999999999,"5min":134.929,"15min":134.929},"last":0.29899999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:28:47 2023","interfaces":[{"interface":"back","average":{"1min":17.056999999999999,"5min":8.5760000000000005,"15min":5.2709999999999999},"min":{"1min":0.255,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":231.67400000000001,"5min":231.67400000000001,"15min":231.67400000000001},"last":8.5269999999999992},{"interface":"front","average":{"1min":17.753,"5min":8.5719999999999992,"15min":4.8280000000000003},"min":{"1min":0.221,"5min":0.221,"15min":0.218},"max":{"1min":232.21000000000001,"5min":232.21000000000001,"15min":232.21000000000001},"last":0.55300000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:28:44 2023","interfaces":[{"interface":"back","average":{"1min":15.568,"5min":6.3250000000000002,"15min":4.8630000000000004},"min":{"1min":0.25700000000000001,"5min":0.23200000000000001,"15min":0.20799999999999999},"max":{"1min":231.27199999999999,"5min":231.27199999999999,"15min":231.27199999999999},"last":4.2560000000000002},{"interface":"front","average":{"1min":15.065,"5min":6.1050000000000004,"15min":4.6749999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":231.43799999999999,"5min":231.43799999999999,"15min":231.43799999999999},"last":4.218}]},{"osd":4,"last update":"Sat Sep 23 23:28:59 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":1.593,"15min":1.6080000000000001},"min":{"1min":0.312,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":2.7490000000000001,"5min":9.3780000000000001,"15min":18.474},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.81999999999999995,"5min":1.6950000000000001,"15min":1.7410000000000001},"min":{"1min":0.29699999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.4720000000000004,"5min":9.7460000000000004,"15min":25.710999999999999},"last":8.2550000000000008}]},{"osd":5,"last update":"Sat Sep 23 23:28:59 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":1.1719999999999999,"15min":3.0819999999999999},"min":{"1min":0.34799999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.895,"5min":13.064,"15min":250.33000000000001},"last":0.375},{"interface":"front","average":{"1min":0.81200000000000006,"5min":1.786,"15min":3.2250000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.30199999999999999},"max":{"1min":2.7869999999999999,"5min":9.3450000000000006,"15min":250.21199999999999},"last":0.61099999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:28:59 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":2.5230000000000001,"15min":2.48},"min":{"1min":0.38300000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.0090000000000003,"5min":74.087999999999994,"15min":74.087999999999994},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.83899999999999997,"5min":2.7120000000000002,"15min":2.383},"min":{"1min":0.37,"5min":0.33100000000000002,"15min":0.32800000000000001},"max":{"1min":2.8340000000000001,"5min":73.974999999999994,"15min":73.974999999999994},"last":0.64300000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:28:47 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":2.504,"15min":3.9119999999999999},"min":{"1min":0.39100000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":6.492,"5min":78.308000000000007,"15min":78.308000000000007},"last":8.282},{"interface":"front","average":{"1min":0.79100000000000004,"5min":2.4060000000000001,"15min":3.5139999999999998},"min":{"1min":0.45700000000000002,"5min":0.371,"15min":0.371},"max":{"1min":2.8109999999999999,"5min":77.204999999999998,"15min":77.204999999999998},"last":0.71599999999999997}]}]},{"osd":0,"up_from":91,"seq":390842024242,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2556956,"kb_used_data":1473044,"kb_used_omap":5,"kb_used_meta":1083898,"kb_avail":91188196,"statfs":{"total":95995035648,"available":93376712704,"internally_reserved":0,"allocated":1508397056,"data_stored":1506098287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1109912186},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:29:22 2023","interfaces":[{"interface":"back","average":{"1min":9.8620000000000001,"5min":7.351,"15min":4.9710000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16900000000000001},"max":{"1min":98.099000000000004,"5min":139.30000000000001,"15min":155.91999999999999},"last":22.629000000000001},{"interface":"front","average":{"1min":10.385,"5min":5.2069999999999999,"15min":3.8460000000000001},"min":{"1min":0.22700000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":97.320999999999998,"5min":97.320999999999998,"15min":153.28899999999999},"last":21.771999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":5.3419999999999996,"15min":4.8120000000000003},"min":{"1min":0.25900000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.0910000000000002,"5min":139.471,"15min":171.15100000000001},"last":0.26600000000000001},{"interface":"front","average":{"1min":1.26,"5min":5.1299999999999999,"15min":5.069},"min":{"1min":0.24399999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":5.0129999999999999,"5min":139.04400000000001,"15min":173.04900000000001},"last":0.28100000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:29:22 2023","interfaces":[{"interface":"back","average":{"1min":4.0119999999999996,"5min":5.4649999999999999,"15min":4.133},"min":{"1min":0.26500000000000001,"5min":0.253,"15min":0.23200000000000001},"max":{"1min":66.332999999999998,"5min":139.34899999999999,"15min":139.34899999999999},"last":0.32300000000000001},{"interface":"front","average":{"1min":4.8239999999999998,"5min":5.4000000000000004,"15min":3.8069999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":66.364000000000004,"5min":138.739,"15min":138.739},"last":0.33700000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:28:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":3.476,"15min":2.6429999999999998},"min":{"1min":0.35599999999999998,"5min":0.30599999999999999,"15min":0.27000000000000002},"max":{"1min":6.7069999999999999,"5min":138.81299999999999,"15min":138.81299999999999},"last":0.505},{"interface":"front","average":{"1min":0.73199999999999998,"5min":3.5790000000000002,"15min":3.2970000000000002},"min":{"1min":0.312,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":3.214,"5min":139.19800000000001,"15min":171.251},"last":0.73299999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:28:51 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":3.6579999999999999,"15min":2.9769999999999999},"min":{"1min":0.32600000000000001,"5min":0.32200000000000001,"15min":0.29099999999999998},"max":{"1min":3.2799999999999998,"5min":139.119,"15min":171.30699999999999},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.53700000000000003,"5min":3.4700000000000002,"15min":2.25},"min":{"1min":0.35999999999999999,"5min":0.253,"15min":0.253},"max":{"1min":0.82899999999999996,"5min":139.40799999999999,"15min":139.40799999999999},"last":0.51300000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:28:51 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":3.601,"15min":3.173},"min":{"1min":0.38100000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.2530000000000001,"5min":139.16399999999999,"15min":171.33600000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.57499999999999996,"5min":6.04,"15min":3.4039999999999999},"min":{"1min":0.44700000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.86899999999999999,"5min":184.13499999999999,"15min":184.13499999999999},"last":0.68999999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:28:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":3.7090000000000001,"15min":2.1299999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.3630000000000004,"5min":138.77699999999999,"15min":138.77699999999999},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.73199999999999998,"5min":3.6419999999999999,"15min":1.802},"min":{"1min":0.40600000000000003,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.302,"5min":139.37899999999999,"15min":139.37899999999999},"last":0.745}]}]},{"osd":2,"up_from":199,"seq":854698492063,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3047376,"kb_used_data":2473992,"kb_used_omap":7,"kb_used_meta":573368,"kb_avail":90697776,"statfs":{"total":95995035648,"available":92874522624,"internally_reserved":0,"allocated":2533367808,"data_stored":2531087633,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":587128913},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":3.5579999999999998,"15min":4.6900000000000004},"min":{"1min":0.215,"5min":0.182,"15min":0.16700000000000001},"max":{"1min":2.6680000000000001,"5min":76.942999999999998,"15min":119.277},"last":4.1829999999999998},{"interface":"front","average":{"1min":0.876,"5min":3.5659999999999998,"15min":4.7699999999999996},"min":{"1min":0.214,"5min":0.192,"15min":0.18099999999999999},"max":{"1min":5.3529999999999998,"5min":76.564999999999998,"15min":119.56100000000001},"last":0.66100000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:28:46 2023","interfaces":[{"interface":"back","average":{"1min":1.79,"5min":4.274,"15min":6.5460000000000003},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.20699999999999999},"max":{"1min":15.718999999999999,"5min":103.65900000000001,"15min":157.98699999999999},"last":3.9470000000000001},{"interface":"front","average":{"1min":2.8050000000000002,"5min":4.2709999999999999,"15min":6.3840000000000003},"min":{"1min":0.29699999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":17.783999999999999,"5min":100.182,"15min":158.03800000000001},"last":4.2839999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:28:46 2023","interfaces":[{"interface":"back","average":{"1min":6.9930000000000003,"5min":5.5739999999999998,"15min":5.2080000000000002},"min":{"1min":0.26500000000000001,"5min":0.191,"15min":0.191},"max":{"1min":103.227,"5min":103.227,"15min":119.366},"last":3.2709999999999999},{"interface":"front","average":{"1min":7.3840000000000003,"5min":5.3259999999999996,"15min":4.5869999999999997},"min":{"1min":0.32800000000000001,"5min":0.25700000000000001,"15min":0.23499999999999999},"max":{"1min":102.693,"5min":102.693,"15min":102.693},"last":4.2140000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":1.9099999999999999,"15min":3.903},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.25},"max":{"1min":2.9020000000000001,"5min":23.391999999999999,"15min":332.98099999999999},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.72599999999999998,"5min":1.8069999999999999,"15min":3.3450000000000002},"min":{"1min":0.31,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.7320000000000002,"5min":21.076000000000001,"15min":332.96199999999999},"last":3.3399999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":1.8979999999999999,"15min":2.2919999999999998},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.9399999999999999,"5min":23.359999999999999,"15min":119.919},"last":0.746},{"interface":"front","average":{"1min":0.89800000000000002,"5min":1.877,"15min":2.242},"min":{"1min":0.34699999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.3490000000000002,"5min":21.105,"15min":119.306},"last":3.3029999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.9279999999999999,"15min":4.1429999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.31},"max":{"1min":2.9700000000000002,"5min":23.425000000000001,"15min":349.32799999999997},"last":0.77000000000000002},{"interface":"front","average":{"1min":1.127,"5min":1.921,"15min":3.6509999999999998},"min":{"1min":0.374,"5min":0.372,"15min":0.34999999999999998},"max":{"1min":4,"5min":21.353999999999999,"15min":349.17700000000002},"last":3.3679999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1.859,"15min":1.403},"min":{"1min":0.372,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.629,"5min":20.350000000000001,"15min":20.350000000000001},"last":1.5449999999999999},{"interface":"front","average":{"1min":0.95199999999999996,"5min":1.954,"15min":1.4950000000000001},"min":{"1min":0.42799999999999999,"5min":0.38,"15min":0.38},"max":{"1min":2.6920000000000002,"5min":20.321000000000002,"15min":20.321000000000002},"last":1.5700000000000001}]}]},{"osd":3,"up_from":148,"seq":635655160032,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2870536,"kb_used_data":2132416,"kb_used_omap":2,"kb_used_meta":738109,"kb_avail":90874616,"statfs":{"total":95995035648,"available":93055606784,"internally_reserved":0,"allocated":2183593984,"data_stored":2181374888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":755823669},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:28:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5219999999999998,"5min":3.452,"15min":3.48},"min":{"1min":0.218,"5min":0.17699999999999999,"15min":0.158},"max":{"1min":17.456,"5min":62.914000000000001,"15min":103.866},"last":0.23599999999999999},{"interface":"front","average":{"1min":1.9490000000000001,"5min":3.2909999999999999,"15min":3.1429999999999998},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":13.436,"5min":63.276000000000003,"15min":101.884},"last":0.28100000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:28:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2589999999999999,"5min":2.6360000000000001,"15min":2.964},"min":{"1min":0.253,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":16.111999999999998,"5min":38.094000000000001,"15min":76.322999999999993},"last":0.253},{"interface":"front","average":{"1min":2.0390000000000001,"5min":2.9380000000000002,"15min":2.8439999999999999},"min":{"1min":0.24099999999999999,"5min":0.21199999999999999,"15min":0.20499999999999999},"max":{"1min":15.779,"5min":42.030999999999999,"15min":76.444000000000003},"last":0.308}]},{"osd":2,"last update":"Sat Sep 23 23:28:41 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":3.714,"15min":3.7749999999999999},"min":{"1min":0.219,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":13.135,"5min":66.364999999999995,"15min":101.279},"last":84.724999999999994},{"interface":"front","average":{"1min":1.575,"5min":3.5939999999999999,"15min":3.032},"min":{"1min":0.28999999999999998,"5min":0.214,"15min":0.214},"max":{"1min":9.4550000000000001,"5min":66.450000000000003,"15min":101.294},"last":84.567999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:28:34 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":1.3169999999999999,"15min":1.492},"min":{"1min":0.33700000000000002,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":8.3059999999999992,"5min":8.3059999999999992,"15min":14.754},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.6499999999999999,"5min":1.5329999999999999,"15min":1.5269999999999999},"min":{"1min":0.30599999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.314,"5min":15.619,"15min":15.619},"last":0.66600000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:28:34 2023","interfaces":[{"interface":"back","average":{"1min":1.7030000000000001,"5min":1.71,"15min":1.7549999999999999},"min":{"1min":0.29599999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":13.353999999999999,"5min":15.762,"15min":75.733000000000004},"last":0.69299999999999995},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.7410000000000001,"15min":2.0579999999999998},"min":{"1min":0.42599999999999999,"5min":0.316,"15min":0.28399999999999997},"max":{"1min":13.183,"5min":13.864000000000001,"15min":75.686999999999998},"last":0.622}]},{"osd":6,"last update":"Sat Sep 23 23:28:34 2023","interfaces":[{"interface":"back","average":{"1min":1.7030000000000001,"5min":2.2040000000000002,"15min":1.784},"min":{"1min":0.35199999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":13.380000000000001,"5min":53.057000000000002,"15min":53.057000000000002},"last":0.58699999999999997},{"interface":"front","average":{"1min":2.0939999999999999,"5min":2.2759999999999998,"15min":1.9890000000000001},"min":{"1min":0.437,"5min":0.35299999999999998,"15min":0.33300000000000002},"max":{"1min":13.215999999999999,"5min":53.088000000000001,"15min":53.088000000000001},"last":0.65000000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:28:41 2023","interfaces":[{"interface":"back","average":{"1min":4.9850000000000003,"5min":2.1389999999999998,"15min":1.2869999999999999},"min":{"1min":0.38100000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":88.344999999999999,"5min":88.344999999999999,"15min":88.344999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":4.681,"5min":2.4399999999999999,"15min":1.635},"min":{"1min":0.38500000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":87.855000000000004,"5min":87.855000000000004,"15min":87.855000000000004},"last":0.63600000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019350,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1925480,"kb_used_data":1193056,"kb_used_omap":0,"kb_used_meta":732416,"kb_avail":91819672,"statfs":{"total":95995035648,"available":94023344128,"internally_reserved":0,"allocated":1221689344,"data_stored":1220385074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":749993984},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:28:56 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":2.359,"15min":4.7469999999999999},"min":{"1min":0.22700000000000001,"5min":0.219,"15min":0.218},"max":{"1min":1.657,"5min":56.279000000000003,"15min":199.64400000000001},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.80900000000000005,"5min":2.355,"15min":4.8689999999999998},"min":{"1min":0.30599999999999999,"5min":0.224,"15min":0.216},"max":{"1min":5.4710000000000001,"5min":56.386000000000003,"15min":195.649},"last":0.83099999999999996}]},{"osd":1,"last update":"Sat Sep 23 23:29:21 2023","interfaces":[{"interface":"back","average":{"1min":3.778,"5min":2.6259999999999999,"15min":4.1559999999999997},"min":{"1min":0.251,"5min":0.24299999999999999,"15min":0.23100000000000001},"max":{"1min":63.966000000000001,"5min":63.966000000000001,"15min":161.923},"last":1.6950000000000001},{"interface":"front","average":{"1min":3.7400000000000002,"5min":2.8839999999999999,"15min":4.3620000000000001},"min":{"1min":0.27800000000000002,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":63.738,"5min":63.738,"15min":161.39699999999999},"last":0.47299999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:28:39 2023","interfaces":[{"interface":"back","average":{"1min":8.4849999999999994,"5min":4.7720000000000002,"15min":3.6320000000000001},"min":{"1min":0.34499999999999997,"5min":0.27900000000000003,"15min":0.26300000000000001},"max":{"1min":130.506,"5min":130.506,"15min":130.506},"last":1.8200000000000001},{"interface":"front","average":{"1min":8.4979999999999993,"5min":4.7880000000000003,"15min":3.3969999999999998},"min":{"1min":0.434,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":130.209,"5min":130.209,"15min":130.209},"last":0.91000000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:28:38 2023","interfaces":[{"interface":"back","average":{"1min":13.73,"5min":8.4469999999999992,"15min":6.5529999999999999},"min":{"1min":0.35499999999999998,"5min":0.27200000000000002,"15min":0.26300000000000001},"max":{"1min":130.227,"5min":176.74100000000001,"15min":303.41500000000002},"last":0.48599999999999999},{"interface":"front","average":{"1min":13.834,"5min":8.1120000000000001,"15min":6.6139999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":131.03999999999999,"5min":176.87799999999999,"15min":303.428},"last":3.226}]},{"osd":5,"last update":"Sat Sep 23 23:29:14 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":2.5070000000000001,"15min":1.9930000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.23699999999999999},"max":{"1min":0.749,"5min":161.86500000000001,"15min":161.86500000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.50800000000000001,"5min":2.5249999999999999,"15min":2.1259999999999999},"min":{"1min":0.31,"5min":0.28399999999999997,"15min":0.249},"max":{"1min":1.6839999999999999,"5min":159.24199999999999,"15min":160.61099999999999},"last":0.30199999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:28:38 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.73099999999999998,"15min":1.177},"min":{"1min":0.33500000000000002,"5min":0.29799999999999999,"15min":0.26000000000000001},"max":{"1min":2.3029999999999999,"5min":5.3399999999999999,"15min":90.346999999999994},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.82099999999999995,"5min":0.81499999999999995,"15min":1.3180000000000001},"min":{"1min":0.35599999999999998,"5min":0.33200000000000002,"15min":0.30599999999999999},"max":{"1min":2.4780000000000002,"5min":4.298,"15min":90.733000000000004},"last":0.41199999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:28:39 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":1.589,"15min":0.90700000000000003},"min":{"1min":0.34599999999999997,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.3849999999999998,"5min":61.923999999999999,"15min":61.923999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.80700000000000005,"5min":1.5509999999999999,"15min":1.052},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.5630000000000002,"5min":61.990000000000002,"15min":61.990000000000002},"last":0.45300000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757713,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1106584,"kb_used_data":477904,"kb_used_omap":0,"kb_used_meta":628672,"kb_avail":92638568,"statfs":{"total":95995035648,"available":94861893632,"internally_reserved":0,"allocated":489373696,"data_stored":488081833,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":643760128},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:29:01 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":2.0409999999999999,"15min":3.7850000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.19500000000000001},"max":{"1min":1.726,"5min":63.741999999999997,"15min":113.869},"last":3.3599999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.9450000000000001,"15min":4.2229999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.3330000000000002,"5min":63.259,"15min":138.55199999999999},"last":1.6439999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:29:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":3.9359999999999999,"15min":4.1120000000000001},"min":{"1min":0.25,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":28.379999999999999,"5min":69.816999999999993,"15min":193.584},"last":3.4809999999999999},{"interface":"front","average":{"1min":2.4409999999999998,"5min":4.3730000000000002,"15min":4.3970000000000002},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.25900000000000001},"max":{"1min":28.417000000000002,"5min":70.268000000000001,"15min":194.03100000000001},"last":1.484}]},{"osd":2,"last update":"Sat Sep 23 23:28:39 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":4.3760000000000003,"15min":2.7509999999999999},"min":{"1min":0.32700000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":46.177,"5min":144.91200000000001,"15min":144.91200000000001},"last":1.431},{"interface":"front","average":{"1min":3.1899999999999999,"5min":4.2770000000000001,"15min":2.6589999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.25800000000000001},"max":{"1min":45.863999999999997,"5min":145.15799999999999,"15min":145.15799999999999},"last":2.153}]},{"osd":3,"last update":"Sat Sep 23 23:28:31 2023","interfaces":[{"interface":"back","average":{"1min":3.1400000000000001,"5min":3.4359999999999999,"15min":4.6230000000000002},"min":{"1min":0.36299999999999999,"5min":0.315,"15min":0.27400000000000002},"max":{"1min":43.497,"5min":62.612000000000002,"15min":159.53299999999999},"last":5.9160000000000004},{"interface":"front","average":{"1min":3.1309999999999998,"5min":3.5150000000000001,"15min":4.8390000000000004},"min":{"1min":0.32700000000000001,"5min":0.318,"15min":0.312},"max":{"1min":43.515000000000001,"5min":62.639000000000003,"15min":159.54900000000001},"last":5.9379999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:29:14 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.72399999999999998,"15min":0.93100000000000005},"min":{"1min":0.26200000000000001,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":0.65700000000000003,"5min":5.3739999999999997,"15min":10.012},"last":1.395},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.61899999999999999,"15min":0.72999999999999998},"min":{"1min":0.29099999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.628,"5min":3.2530000000000001,"15min":11.923999999999999},"last":1.5049999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:29:14 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":1.4510000000000001,"15min":2.1829999999999998},"min":{"1min":0.34699999999999998,"5min":0.28599999999999998,"15min":0.26900000000000002},"max":{"1min":0.63,"5min":68.456999999999994,"15min":310.68799999999999},"last":1.546},{"interface":"front","average":{"1min":0.498,"5min":1.4390000000000001,"15min":2.3700000000000001},"min":{"1min":0.35499999999999998,"5min":0.315,"15min":0.25900000000000001},"max":{"1min":0.83299999999999996,"5min":66.721999999999994,"15min":311.45400000000001},"last":1.6719999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:28:39 2023","interfaces":[{"interface":"back","average":{"1min":10.975,"5min":3.5819999999999999,"15min":3.9580000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":179.28800000000001,"5min":179.28800000000001,"15min":179.28800000000001},"last":1.6060000000000001},{"interface":"front","average":{"1min":11.010999999999999,"5min":3.597,"15min":4.0890000000000004},"min":{"1min":0.40100000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":179.428,"5min":179.428,"15min":179.428},"last":1.7050000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1517289472,"data_stored":1517289472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1723531264,"data_stored":1723531264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2537553920,"data_stored":2537553920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2206924800,"data_stored":2206924800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1230110720,"data_stored":1230110720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":487194624,"data_stored":487194624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2177892352,"data_stored":2177892352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:29:34.035 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T23:29:34.036 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_7 pg_num 26 2023-09-23T23:29:34.240 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 49 16 64770 64754 82.5395 55.625 0.0107165 0.0121092 2023-09-23T23:29:34.271 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:34.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:34.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:34.384 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:34.425 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:34.467 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:34.467 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:34.597 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:34.603 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:34 smithi110 ceph-mon[212939]: pgmap v1338: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s 2023-09-23T23:29:34.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:34.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:34.708 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:34.806 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:34.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:34 smithi138 ceph-mon[193606]: pgmap v1338: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s 2023-09-23T23:29:34.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:34 smithi110 ceph-mon[221102]: pgmap v1338: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s 2023-09-23T23:29:35.205 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:35.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:35.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:35.239 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 50 16 65633 65617 81.9676 53.9375 0.0148356 0.012194 2023-09-23T23:29:35.316 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:29:35.347 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:35.407 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:29:35.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:35.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:29:35.513 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:35.543 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:35.597 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:35.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:35.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:35.609 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:35 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2925684338' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:29:35.609 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:35 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:29:35.609 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:35 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2925684338' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:29:35.609 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:35 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:29:35.646 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:29:35.705 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:35.779 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:35.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:35 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2925684338' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:29:35.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:35 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:29:36.239 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 51 16 66534 66518 81.4648 56.3125 0.00312144 0.0122515 2023-09-23T23:29:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:36 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:29:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:36 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:29:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:36 smithi138 ceph-mon[193606]: osdmap e488: 8 total, 8 up, 7 in 2023-09-23T23:29:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:36 smithi138 ceph-mon[193606]: failed to encode map e488 with expected crc 2023-09-23T23:29:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:36 smithi138 ceph-mon[193606]: failed to encode map e488 with expected crc 2023-09-23T23:29:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:36 smithi138 ceph-mon[193606]: failed to encode map e488 with expected crc 2023-09-23T23:29:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:36 smithi138 ceph-mon[193606]: failed to encode map e488 with expected crc 2023-09-23T23:29:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:36 smithi138 ceph-mon[193606]: pgmap v1340: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 59 MiB/s wr, 937 op/s 2023-09-23T23:29:36.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:29:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:29:36] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.43.0" 2023-09-23T23:29:36.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:36 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:29:36.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:36 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:29:36.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:36 smithi110 ceph-mon[212939]: osdmap e488: 8 total, 8 up, 7 in 2023-09-23T23:29:36.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:36 smithi110 ceph-mon[212939]: failed to encode map e488 with expected crc 2023-09-23T23:29:36.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:36 smithi110 ceph-mon[212939]: failed to encode map e488 with expected crc 2023-09-23T23:29:36.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:36 smithi110 ceph-mon[212939]: failed to encode map e488 with expected crc 2023-09-23T23:29:36.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:36 smithi110 ceph-mon[212939]: failed to encode map e488 with expected crc 2023-09-23T23:29:36.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:36 smithi110 ceph-mon[212939]: pgmap v1340: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 59 MiB/s wr, 937 op/s 2023-09-23T23:29:36.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:36 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:29:36.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:36 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:29:36.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:36 smithi110 ceph-mon[221102]: osdmap e488: 8 total, 8 up, 7 in 2023-09-23T23:29:36.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:36 smithi110 ceph-mon[221102]: failed to encode map e488 with expected crc 2023-09-23T23:29:36.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:36 smithi110 ceph-mon[221102]: failed to encode map e488 with expected crc 2023-09-23T23:29:36.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:36 smithi110 ceph-mon[221102]: failed to encode map e488 with expected crc 2023-09-23T23:29:36.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:36 smithi110 ceph-mon[221102]: failed to encode map e488 with expected crc 2023-09-23T23:29:36.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:36 smithi110 ceph-mon[221102]: pgmap v1340: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 59 MiB/s wr, 937 op/s 2023-09-23T23:29:37.239 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 52 16 67270 67254 80.7831 46 0.0145657 0.0123747 2023-09-23T23:29:37.269 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:37.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:37.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:37.375 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:29:37.400 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:37.441 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:29:37.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:37.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:29:37.550 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:37.643 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:37.989 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:38.127 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:38.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:38.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:38.239 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 53 16 68103 68087 80.2414 52.0625 0.011422 0.0124562 2023-09-23T23:29:38.244 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:38.339 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:38.436 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:38.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:38.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:38.499 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:29:38.543 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:29:38.555 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:29:38.575 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:38.598 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:29:38.629 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:29:38.630 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:29:38.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:38.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:29:38.667 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:29:38.700 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:29:38.727 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:29:38.736 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:29:38.760 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:29:38.768 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:38.768 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:29:38.768 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:29:38.768 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:29:38.769 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:29:38.769 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:29:38.769 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:29:38.769 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:29:38.769 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:29:38.824 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:29:38.829 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:38 smithi138 ceph-mon[193606]: pgmap v1341: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 59 MiB/s wr, 939 op/s 2023-09-23T23:29:38.829 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:38 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2442535223' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:38.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:38.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:29:38.931 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:38.962 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:39.012 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:39.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:39.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:39.029 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:29:39.030 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:29:39.030 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:29:39.030 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:29:39.030 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:29:39.118 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:39.195 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:38 smithi110 ceph-mon[212939]: pgmap v1341: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 59 MiB/s wr, 939 op/s 2023-09-23T23:29:39.195 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:38 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2442535223' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:38 smithi110 ceph-mon[221102]: pgmap v1341: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 59 MiB/s wr, 939 op/s 2023-09-23T23:29:39.196 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:38 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2442535223' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:39.201 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:39.243 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 54 16 69032 69016 79.8259 58.0625 0.00765841 0.0125212 2023-09-23T23:29:39.950 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:29:40.244 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 55 16 69845 69829 79.2969 50.8125 0.00916742 0.0126049 2023-09-23T23:29:40.862 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:40.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:40.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:40.971 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:29:40.996 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:41.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:40 smithi138 ceph-mon[193606]: pgmap v1342: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 912 op/s 2023-09-23T23:29:41.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:40 smithi110 ceph-mon[212939]: pgmap v1342: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 912 op/s 2023-09-23T23:29:41.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:40 smithi110 ceph-mon[221102]: pgmap v1342: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 912 op/s 2023-09-23T23:29:41.207 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:29:41.245 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 56 16 70648 70632 78.7765 50.1875 0.0110001 0.0126892 2023-09-23T23:29:41.446 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:29:41.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:41.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:29:41.558 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:29:41.637 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:41.778 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:41.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:41.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:41.807 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:41.893 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:29:41.924 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:41.983 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:29:41.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:41.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:29:42.092 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:29:42.193 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:42.245 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 57 16 71456 71440 78.2807 50.5 0.0108844 0.0127692 2023-09-23T23:29:42.284 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:29:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:42.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:29:42.393 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:42.395 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:42.428 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:42.482 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:42.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:42.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:42.592 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:29:42.653 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:42.781 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:29:42.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:42.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:29:42.849 INFO:teuthology.orchestra.run.smithi110.stderr:set pool 12 pg_num to 26 2023-09-23T23:29:42.887 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:29:42.913 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:42.955 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:29:42.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:42.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:29:43.059 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:29:43.131 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:42 smithi110 ceph-mon[212939]: pgmap v1343: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 879 op/s 2023-09-23T23:29:43.131 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:42 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2283834610' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T23:29:43.131 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:42 smithi110 ceph-mon[221102]: pgmap v1343: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 879 op/s 2023-09-23T23:29:43.131 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:42 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2283834610' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T23:29:43.136 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:43.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:42 smithi138 ceph-mon[193606]: pgmap v1343: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 879 op/s 2023-09-23T23:29:43.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:42 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2283834610' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T23:29:43.245 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 58 16 72165 72149 77.6953 44.3125 0.0035958 0.0128654 2023-09-23T23:29:43.249 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:29:44.114 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[221102]: failed to encode map e489 with expected crc 2023-09-23T23:29:44.114 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2283834610' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2023-09-23T23:29:44.114 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[221102]: osdmap e489: 8 total, 8 up, 7 in 2023-09-23T23:29:44.114 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:44.115 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:29:44.115 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[221102]: failed to encode map e489 with expected crc 2023-09-23T23:29:44.115 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[221102]: failed to encode map e489 with expected crc 2023-09-23T23:29:44.115 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T23:29:44.115 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T23:29:44.121 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[212939]: failed to encode map e489 with expected crc 2023-09-23T23:29:44.121 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2283834610' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2023-09-23T23:29:44.121 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[212939]: osdmap e489: 8 total, 8 up, 7 in 2023-09-23T23:29:44.121 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:44.121 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:29:44.121 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[212939]: failed to encode map e489 with expected crc 2023-09-23T23:29:44.121 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[212939]: failed to encode map e489 with expected crc 2023-09-23T23:29:44.121 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T23:29:44.122 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T23:29:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:43 smithi138 ceph-mon[193606]: failed to encode map e489 with expected crc 2023-09-23T23:29:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:43 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2283834610' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2023-09-23T23:29:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:43 smithi138 ceph-mon[193606]: osdmap e489: 8 total, 8 up, 7 in 2023-09-23T23:29:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:29:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:43 smithi138 ceph-mon[193606]: failed to encode map e489 with expected crc 2023-09-23T23:29:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:43 smithi138 ceph-mon[193606]: failed to encode map e489 with expected crc 2023-09-23T23:29:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T23:29:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T23:29:44.245 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 59 16 73064 73048 77.3308 56.1875 0.0104215 0.0129242 2023-09-23T23:29:44.311 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:44.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:44.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:44.426 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:29:44.459 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:44.520 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:29:44.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:44.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:29:44.629 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:29:44.831 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:44.946 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:44.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:44.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:45.050 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:45.077 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:45.119 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:45.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:45.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:45.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:44 smithi138 ceph-mon[193606]: pgmap v1345: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 45 MiB/s wr, 718 op/s 2023-09-23T23:29:45.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:44 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T23:29:45.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:44 smithi138 ceph-mon[193606]: osdmap e490: 8 total, 8 up, 7 in 2023-09-23T23:29:45.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:44 smithi110 ceph-mon[212939]: pgmap v1345: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 45 MiB/s wr, 718 op/s 2023-09-23T23:29:45.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:44 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T23:29:45.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:44 smithi110 ceph-mon[212939]: osdmap e490: 8 total, 8 up, 7 in 2023-09-23T23:29:45.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:44 smithi110 ceph-mon[221102]: pgmap v1345: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 45 MiB/s wr, 718 op/s 2023-09-23T23:29:45.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:44 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T23:29:45.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:44 smithi110 ceph-mon[221102]: osdmap e490: 8 total, 8 up, 7 in 2023-09-23T23:29:45.225 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:45.245 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:29:45.244616+0000 min lat: 0.00115016 max lat: 0.35927 avg lat: 0.0129754 2023-09-23T23:29:45.245 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:29:45.245 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 60 16 73422 73406 76.4153 22.375 0.0068249 0.0129754 2023-09-23T23:29:45.251 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:45.292 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:45.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:45.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:45.397 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:29:45.422 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:45.463 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:29:45.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:45.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:29:45.568 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:29:45.600 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:45.720 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:45.790 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:45.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:45.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:45.895 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:45 smithi110 ceph-mon[212939]: failed to encode map e490 with expected crc 2023-09-23T23:29:45.895 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:45 smithi110 ceph-mon[212939]: failed to encode map e490 with expected crc 2023-09-23T23:29:45.895 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:45 smithi110 ceph-mon[212939]: osdmap e491: 8 total, 8 up, 7 in 2023-09-23T23:29:45.901 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:29:45.990 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:46.162 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:46.165 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:45 smithi110 ceph-mon[221102]: failed to encode map e490 with expected crc 2023-09-23T23:29:46.165 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:45 smithi110 ceph-mon[221102]: failed to encode map e490 with expected crc 2023-09-23T23:29:46.166 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:45 smithi110 ceph-mon[221102]: osdmap e491: 8 total, 8 up, 7 in 2023-09-23T23:29:46.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:46.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:46.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:45 smithi138 ceph-mon[193606]: failed to encode map e490 with expected crc 2023-09-23T23:29:46.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:45 smithi138 ceph-mon[193606]: failed to encode map e490 with expected crc 2023-09-23T23:29:46.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:45 smithi138 ceph-mon[193606]: osdmap e491: 8 total, 8 up, 7 in 2023-09-23T23:29:46.245 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 61 16 73990 73974 75.7449 35.5 0.008266 0.0131987 2023-09-23T23:29:46.268 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:46.366 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:46.645 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:29:46.645 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:29:46.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:29:46.645 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:29:46.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.645 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178607 11.a 11:5596dbed:::benchmark_data_smithi110_433746_object72727:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2023-09-23T23:29:46.645 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.541947+0000", 2023-09-23T23:29:46.646 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 147.026926731, 2023-09-23T23:29:46.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2167161870000001, 2023-09-23T23:29:46.646 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.646 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.646 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.646 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.646 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.646 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178607 2023-09-23T23:29:46.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.646 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.541947+0000", 2023-09-23T23:29:46.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.541947+0000", 2023-09-23T23:29:46.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.541946+0000", 2023-09-23T23:29:46.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:29:46.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.541954+0000", 2023-09-23T23:29:46.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2770000000000005e-06 2023-09-23T23:29:46.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.541967+0000", 2023-09-23T23:29:46.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2661e-05 2023-09-23T23:29:46.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.541976+0000", 2023-09-23T23:29:46.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0010000000000007e-06 2023-09-23T23:29:46.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.552351+0000", 2023-09-23T23:29:46.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01037542 2023-09-23T23:29:46.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.650 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.552364+0000", 2023-09-23T23:29:46.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2567e-05 2023-09-23T23:29:46.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.753353+0000", 2023-09-23T23:29:46.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.200989404 2023-09-23T23:29:46.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.753480+0000", 2023-09-23T23:29:46.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000126816 2023-09-23T23:29:46.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.652 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T23:29:46.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.753585+0000", 2023-09-23T23:29:46.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010534100000000001 2023-09-23T23:29:46.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.652 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.756243+0000", 2023-09-23T23:29:46.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002658306 2023-09-23T23:29:46.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.756306+0000", 2023-09-23T23:29:46.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2409999999999994e-05 2023-09-23T23:29:46.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.758212+0000", 2023-09-23T23:29:46.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019060380000000001 2023-09-23T23:29:46.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.758457+0000", 2023-09-23T23:29:46.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024545699999999998 2023-09-23T23:29:46.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.758514+0000", 2023-09-23T23:29:46.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6405e-05 2023-09-23T23:29:46.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.758663+0000", 2023-09-23T23:29:46.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014964500000000001 2023-09-23T23:29:46.655 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.655 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.655 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.656 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178611 11.9 11:94c60d7f:::benchmark_data_smithi110_433746_object72731:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:29:46.656 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.550185+0000", 2023-09-23T23:29:46.656 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 147.01868924999999, 2023-09-23T23:29:46.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.188218649, 2023-09-23T23:29:46.656 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.656 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.656 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.656 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.656 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.656 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178611 2023-09-23T23:29:46.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.657 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.550185+0000", 2023-09-23T23:29:46.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.550185+0000", 2023-09-23T23:29:46.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.550183+0000", 2023-09-23T23:29:46.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:29:46.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.550190+0000", 2023-09-23T23:29:46.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1590000000000004e-06 2023-09-23T23:29:46.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.550203+0000", 2023-09-23T23:29:46.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2585999999999999e-05 2023-09-23T23:29:46.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.550211+0000", 2023-09-23T23:29:46.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2029999999999992e-06 2023-09-23T23:29:46.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.550246+0000", 2023-09-23T23:29:46.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4647999999999998e-05 2023-09-23T23:29:46.662 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.550252+0000", 2023-09-23T23:29:46.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6819999999999997e-06 2023-09-23T23:29:46.662 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.547412+0000", 2023-09-23T23:29:46.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99715919500000005 2023-09-23T23:29:46.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.663 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.547424+0000", 2023-09-23T23:29:46.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2401999999999999e-05 2023-09-23T23:29:46.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.664 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.664 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.714868+0000", 2023-09-23T23:29:46.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1674435679999999 2023-09-23T23:29:46.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.715028+0000", 2023-09-23T23:29:46.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000159965 2023-09-23T23:29:46.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T23:29:46.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.715499+0000", 2023-09-23T23:29:46.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047178699999999999 2023-09-23T23:29:46.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.723954+0000", 2023-09-23T23:29:46.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084545130000000003 2023-09-23T23:29:46.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.723974+0000", 2023-09-23T23:29:46.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0497e-05 2023-09-23T23:29:46.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.667 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.729040+0000", 2023-09-23T23:29:46.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0050653909999999998 2023-09-23T23:29:46.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.667 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738198+0000", 2023-09-23T23:29:46.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.009158055 2023-09-23T23:29:46.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738264+0000", 2023-09-23T23:29:46.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6534999999999999e-05 2023-09-23T23:29:46.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738403+0000", 2023-09-23T23:29:46.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013902900000000001 2023-09-23T23:29:46.668 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.668 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.668 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.669 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178616 11.19 11:9db36ee3:::benchmark_data_smithi110_433746_object72736:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:29:46.669 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.573665+0000", 2023-09-23T23:29:46.669 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 146.995208496, 2023-09-23T23:29:46.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1649632130000001, 2023-09-23T23:29:46.669 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.669 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.669 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.669 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.669 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.669 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178616 2023-09-23T23:29:46.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.670 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.670 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.573665+0000", 2023-09-23T23:29:46.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.670 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.573665+0000", 2023-09-23T23:29:46.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.573664+0000", 2023-09-23T23:29:46.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:29:46.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.671 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.573672+0000", 2023-09-23T23:29:46.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1580000000000005e-06 2023-09-23T23:29:46.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.671 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.672 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.573687+0000", 2023-09-23T23:29:46.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5104e-05 2023-09-23T23:29:46.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.672 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.573697+0000", 2023-09-23T23:29:46.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8220000000000002e-06 2023-09-23T23:29:46.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.672 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.579724+0000", 2023-09-23T23:29:46.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060272930000000004 2023-09-23T23:29:46.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.673 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.579737+0000", 2023-09-23T23:29:46.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2323999999999999e-05 2023-09-23T23:29:46.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.673 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.547950+0000", 2023-09-23T23:29:46.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96821353799999998 2023-09-23T23:29:46.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.673 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.547961+0000", 2023-09-23T23:29:46.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0730999999999999e-05 2023-09-23T23:29:46.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.711546+0000", 2023-09-23T23:29:46.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1635849789999999 2023-09-23T23:29:46.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.711691+0000", 2023-09-23T23:29:46.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000144735 2023-09-23T23:29:46.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T23:29:46.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.712153+0000", 2023-09-23T23:29:46.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00046240000000000002 2023-09-23T23:29:46.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.722189+0000", 2023-09-23T23:29:46.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010035717 2023-09-23T23:29:46.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.676 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.723969+0000", 2023-09-23T23:29:46.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017799890000000001 2023-09-23T23:29:46.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.676 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.676 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.729015+0000", 2023-09-23T23:29:46.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.005045931 2023-09-23T23:29:46.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.676 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.676 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738451+0000", 2023-09-23T23:29:46.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0094367670000000004 2023-09-23T23:29:46.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738498+0000", 2023-09-23T23:29:46.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6147999999999999e-05 2023-09-23T23:29:46.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738629+0000", 2023-09-23T23:29:46.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013102200000000001 2023-09-23T23:29:46.677 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.677 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.677 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.678 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178620 11.a 11:55a13280:::benchmark_data_smithi110_433746_object72740:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:29:46.678 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.584802+0000", 2023-09-23T23:29:46.678 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 146.98407176399999, 2023-09-23T23:29:46.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1742229339999999, 2023-09-23T23:29:46.678 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.678 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.678 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.678 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.678 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.678 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178620 2023-09-23T23:29:46.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.679 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584802+0000", 2023-09-23T23:29:46.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584802+0000", 2023-09-23T23:29:46.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584800+0000", 2023-09-23T23:29:46.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:29:46.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584809+0000", 2023-09-23T23:29:46.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5450000000000003e-06 2023-09-23T23:29:46.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584823+0000", 2023-09-23T23:29:46.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4465e-05 2023-09-23T23:29:46.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584833+0000", 2023-09-23T23:29:46.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6390000000000004e-06 2023-09-23T23:29:46.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584885+0000", 2023-09-23T23:29:46.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2537999999999997e-05 2023-09-23T23:29:46.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584897+0000", 2023-09-23T23:29:46.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.146e-05 2023-09-23T23:29:46.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.682 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.756327+0000", 2023-09-23T23:29:46.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1714304689999999 2023-09-23T23:29:46.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.682 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.756417+0000", 2023-09-23T23:29:46.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9118999999999998e-05 2023-09-23T23:29:46.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T23:29:46.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.756556+0000", 2023-09-23T23:29:46.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000139445 2023-09-23T23:29:46.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.758731+0000", 2023-09-23T23:29:46.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021755099999999999 2023-09-23T23:29:46.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.684 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.684 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.758789+0000", 2023-09-23T23:29:46.684 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7952999999999999e-05 2023-09-23T23:29:46.684 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.684 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.684 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.758801+0000", 2023-09-23T23:29:46.684 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1612e-05 2023-09-23T23:29:46.684 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.684 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.684 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.758834+0000", 2023-09-23T23:29:46.684 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2651000000000003e-05 2023-09-23T23:29:46.685 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.685 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.685 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.758890+0000", 2023-09-23T23:29:46.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6666999999999998e-05 2023-09-23T23:29:46.685 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.685 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.685 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.759025+0000", 2023-09-23T23:29:46.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000134759 2023-09-23T23:29:46.685 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.686 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.686 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.686 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178621 11.a 11:5068fc2a:::benchmark_data_smithi110_433746_object72741:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:29:46.686 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.589725+0000", 2023-09-23T23:29:46.686 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 146.97914862499999, 2023-09-23T23:29:46.686 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.172631457, 2023-09-23T23:29:46.687 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.687 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.687 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.687 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.687 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.687 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178621 2023-09-23T23:29:46.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.687 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.589725+0000", 2023-09-23T23:29:46.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.688 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.688 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.688 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.589725+0000", 2023-09-23T23:29:46.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.688 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.688 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.688 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.688 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.589724+0000", 2023-09-23T23:29:46.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:29:46.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.689 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.589732+0000", 2023-09-23T23:29:46.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9950000000000005e-06 2023-09-23T23:29:46.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.689 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.589750+0000", 2023-09-23T23:29:46.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8697000000000001e-05 2023-09-23T23:29:46.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.690 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.690 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.589759+0000", 2023-09-23T23:29:46.690 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.49e-06 2023-09-23T23:29:46.690 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.690 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.690 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.589833+0000", 2023-09-23T23:29:46.690 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3787000000000004e-05 2023-09-23T23:29:46.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.691 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.691 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.589843+0000", 2023-09-23T23:29:46.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0839000000000001e-05 2023-09-23T23:29:46.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.691 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.691 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.759061+0000", 2023-09-23T23:29:46.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1692171920000001 2023-09-23T23:29:46.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.692 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.759171+0000", 2023-09-23T23:29:46.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110257 2023-09-23T23:29:46.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.692 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.692 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T23:29:46.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.759310+0000", 2023-09-23T23:29:46.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000139188 2023-09-23T23:29:46.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.693 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.693 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.693 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.761367+0000", 2023-09-23T23:29:46.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020571040000000001 2023-09-23T23:29:46.693 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.693 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.693 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.693 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.761854+0000", 2023-09-23T23:29:46.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000486417 2023-09-23T23:29:46.694 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.694 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.694 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.762081+0000", 2023-09-23T23:29:46.694 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000227209 2023-09-23T23:29:46.694 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.694 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.694 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.762183+0000", 2023-09-23T23:29:46.695 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102137 2023-09-23T23:29:46.695 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.695 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.695 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.695 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.762247+0000", 2023-09-23T23:29:46.695 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3835000000000001e-05 2023-09-23T23:29:46.695 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.695 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.696 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.696 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.762357+0000", 2023-09-23T23:29:46.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010998299999999999 2023-09-23T23:29:46.696 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.696 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.696 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.696 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.696 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178627 11.9 11:97e1ceb8:::benchmark_data_smithi110_433746_object72747:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:29:46.697 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.601902+0000", 2023-09-23T23:29:46.697 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 146.96697218899999, 2023-09-23T23:29:46.697 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.136940085, 2023-09-23T23:29:46.697 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.697 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.697 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.697 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.697 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.697 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178627 2023-09-23T23:29:46.698 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.698 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.698 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.601902+0000", 2023-09-23T23:29:46.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.698 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.698 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.601902+0000", 2023-09-23T23:29:46.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.698 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.699 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.699 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.601900+0000", 2023-09-23T23:29:46.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:29:46.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.699 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.699 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.601907+0000", 2023-09-23T23:29:46.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3669999999999999e-06 2023-09-23T23:29:46.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.699 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.601925+0000", 2023-09-23T23:29:46.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7229000000000001e-05 2023-09-23T23:29:46.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.601934+0000", 2023-09-23T23:29:46.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2199999999999998e-06 2023-09-23T23:29:46.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.602025+0000", 2023-09-23T23:29:46.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1458000000000004e-05 2023-09-23T23:29:46.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.602036+0000", 2023-09-23T23:29:46.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.061e-05 2023-09-23T23:29:46.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.547987+0000", 2023-09-23T23:29:46.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94595114300000005 2023-09-23T23:29:46.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.547994+0000", 2023-09-23T23:29:46.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4789999999999999e-06 2023-09-23T23:29:46.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.717738+0000", 2023-09-23T23:29:46.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1697435760000001 2023-09-23T23:29:46.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.717879+0000", 2023-09-23T23:29:46.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014050799999999999 2023-09-23T23:29:46.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T23:29:46.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.718216+0000", 2023-09-23T23:29:46.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033793099999999997 2023-09-23T23:29:46.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.723979+0000", 2023-09-23T23:29:46.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0057627909999999997 2023-09-23T23:29:46.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.704 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.704 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.725186+0000", 2023-09-23T23:29:46.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001206372 2023-09-23T23:29:46.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.704 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.704 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.729100+0000", 2023-09-23T23:29:46.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039139609999999997 2023-09-23T23:29:46.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.705 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.705 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738674+0000", 2023-09-23T23:29:46.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0095747220000000008 2023-09-23T23:29:46.705 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.705 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.705 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.705 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738719+0000", 2023-09-23T23:29:46.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4359e-05 2023-09-23T23:29:46.705 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.705 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.705 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.705 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738842+0000", 2023-09-23T23:29:46.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000123132 2023-09-23T23:29:46.706 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.706 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.706 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.706 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.706 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.706 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:209479 11.15 11:ac769207:::benchmark_data_smithi110_433746_object103599:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e462)", 2023-09-23T23:29:46.706 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:17.437317+0000", 2023-09-23T23:29:46.706 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 89.131557061999999, 2023-09-23T23:29:46.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.122424946, 2023-09-23T23:29:46.706 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.706 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.707 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.707 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.707 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.707 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 209479 2023-09-23T23:29:46.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.707 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437317+0000", 2023-09-23T23:29:46.707 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437317+0000", 2023-09-23T23:29:46.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437317+0000", 2023-09-23T23:29:46.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:46.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437319+0000", 2023-09-23T23:29:46.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1639999999999999e-06 2023-09-23T23:29:46.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437325+0000", 2023-09-23T23:29:46.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9819999999999996e-06 2023-09-23T23:29:46.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437330+0000", 2023-09-23T23:29:46.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.429e-06 2023-09-23T23:29:46.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437459+0000", 2023-09-23T23:29:46.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012924900000000001 2023-09-23T23:29:46.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:29:46.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437464+0000", 2023-09-23T23:29:46.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1270000000000002e-06 2023-09-23T23:29:46.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.439604+0000", 2023-09-23T23:29:46.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021392490000000002 2023-09-23T23:29:46.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.439612+0000", 2023-09-23T23:29:46.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7929999999999998e-06 2023-09-23T23:29:46.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.553119+0000", 2023-09-23T23:29:46.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1135072939999999 2023-09-23T23:29:46.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.553216+0000", 2023-09-23T23:29:46.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7405999999999995e-05 2023-09-23T23:29:46.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4,5,7", 2023-09-23T23:29:46.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.553382+0000", 2023-09-23T23:29:46.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016600999999999999 2023-09-23T23:29:46.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.554827+0000", 2023-09-23T23:29:46.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001444433 2023-09-23T23:29:46.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.713 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.713 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.558941+0000", 2023-09-23T23:29:46.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041140300000000003 2023-09-23T23:29:46.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.713 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.713 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.559406+0000", 2023-09-23T23:29:46.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000465091 2023-09-23T23:29:46.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.713 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.713 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.559459+0000", 2023-09-23T23:29:46.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3652000000000001e-05 2023-09-23T23:29:46.714 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.714 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.714 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.559508+0000", 2023-09-23T23:29:46.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8890000000000001e-05 2023-09-23T23:29:46.714 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.714 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.714 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.559561+0000", 2023-09-23T23:29:46.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2577000000000002e-05 2023-09-23T23:29:46.714 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.559609+0000", 2023-09-23T23:29:46.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8115999999999998e-05 2023-09-23T23:29:46.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.559742+0000", 2023-09-23T23:29:46.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000132803 2023-09-23T23:29:46.715 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.715 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.715 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.716 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:209496 11.15 11:a8d5039c:::benchmark_data_smithi110_433746_object103616:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e462)", 2023-09-23T23:29:46.716 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:17.437341+0000", 2023-09-23T23:29:46.716 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 89.131533031000004, 2023-09-23T23:29:46.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.132685699, 2023-09-23T23:29:46.716 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.716 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.716 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.716 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.716 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.716 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 209496 2023-09-23T23:29:46.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.717 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437341+0000", 2023-09-23T23:29:46.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437341+0000", 2023-09-23T23:29:46.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437341+0000", 2023-09-23T23:29:46.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:46.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437343+0000", 2023-09-23T23:29:46.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.571e-06 2023-09-23T23:29:46.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437349+0000", 2023-09-23T23:29:46.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1430000000000003e-06 2023-09-23T23:29:46.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437354+0000", 2023-09-23T23:29:46.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1070000000000004e-06 2023-09-23T23:29:46.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437522+0000", 2023-09-23T23:29:46.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016808900000000001 2023-09-23T23:29:46.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.720 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:29:46.720 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437528+0000", 2023-09-23T23:29:46.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3839999999999996e-06 2023-09-23T23:29:46.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.720 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.720 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.439637+0000", 2023-09-23T23:29:46.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021089580000000002 2023-09-23T23:29:46.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.720 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.439644+0000", 2023-09-23T23:29:46.721 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.951e-06 2023-09-23T23:29:46.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.555110+0000", 2023-09-23T23:29:46.721 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1154658470000001 2023-09-23T23:29:46.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.722 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.722 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.555213+0000", 2023-09-23T23:29:46.722 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103118 2023-09-23T23:29:46.722 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.722 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.722 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4,5,7", 2023-09-23T23:29:46.722 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.555421+0000", 2023-09-23T23:29:46.722 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020822200000000001 2023-09-23T23:29:46.722 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.723 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.723 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.723 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.559469+0000", 2023-09-23T23:29:46.723 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040478229999999999 2023-09-23T23:29:46.723 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.723 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.723 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.723 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.559793+0000", 2023-09-23T23:29:46.723 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032399399999999997 2023-09-23T23:29:46.723 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.723 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.723 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.724 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.559827+0000", 2023-09-23T23:29:46.724 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4588999999999997e-05 2023-09-23T23:29:46.724 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.724 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.724 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.724 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.559857+0000", 2023-09-23T23:29:46.724 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9791999999999999e-05 2023-09-23T23:29:46.724 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.568990+0000", 2023-09-23T23:29:46.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0091327979999999993 2023-09-23T23:29:46.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.569832+0000", 2023-09-23T23:29:46.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000842036 2023-09-23T23:29:46.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.569886+0000", 2023-09-23T23:29:46.726 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4017000000000001e-05 2023-09-23T23:29:46.726 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.570027+0000", 2023-09-23T23:29:46.726 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014065400000000001 2023-09-23T23:29:46.726 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.727 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.727 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.727 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.727 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.727 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:209517 11.15 11:a909e427:::benchmark_data_smithi110_433746_object103637:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e462)", 2023-09-23T23:29:46.727 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:17.437358+0000", 2023-09-23T23:29:46.727 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 89.131515635, 2023-09-23T23:29:46.727 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1330309620000001, 2023-09-23T23:29:46.727 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.727 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.727 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.728 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.728 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.728 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 209517 2023-09-23T23:29:46.728 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.728 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.728 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.728 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.728 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437358+0000", 2023-09-23T23:29:46.728 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.728 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437358+0000", 2023-09-23T23:29:46.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437358+0000", 2023-09-23T23:29:46.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:46.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437360+0000", 2023-09-23T23:29:46.730 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.232e-06 2023-09-23T23:29:46.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437366+0000", 2023-09-23T23:29:46.730 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7479999999999999e-06 2023-09-23T23:29:46.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437370+0000", 2023-09-23T23:29:46.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.578e-06 2023-09-23T23:29:46.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437564+0000", 2023-09-23T23:29:46.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019359600000000001 2023-09-23T23:29:46.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.732 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:29:46.732 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437567+0000", 2023-09-23T23:29:46.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7049999999999999e-06 2023-09-23T23:29:46.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.439669+0000", 2023-09-23T23:29:46.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021019609999999998 2023-09-23T23:29:46.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.439675+0000", 2023-09-23T23:29:46.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6749999999999996e-06 2023-09-23T23:29:46.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.734 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.556809+0000", 2023-09-23T23:29:46.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.117133462 2023-09-23T23:29:46.734 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.734 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.556926+0000", 2023-09-23T23:29:46.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117459 2023-09-23T23:29:46.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4,5,7", 2023-09-23T23:29:46.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.557170+0000", 2023-09-23T23:29:46.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024389799999999999 2023-09-23T23:29:46.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.559475+0000", 2023-09-23T23:29:46.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023050169999999999 2023-09-23T23:29:46.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.559883+0000", 2023-09-23T23:29:46.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00040789899999999998 2023-09-23T23:29:46.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.569716+0000", 2023-09-23T23:29:46.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0098327369999999994 2023-09-23T23:29:46.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.569754+0000", 2023-09-23T23:29:46.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8318999999999997e-05 2023-09-23T23:29:46.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.737 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.737 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.569797+0000", 2023-09-23T23:29:46.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3288000000000003e-05 2023-09-23T23:29:46.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.737 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.737 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.570209+0000", 2023-09-23T23:29:46.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00041179500000000001 2023-09-23T23:29:46.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.737 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.737 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.570251+0000", 2023-09-23T23:29:46.738 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1309e-05 2023-09-23T23:29:46.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.570389+0000", 2023-09-23T23:29:46.738 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000138703 2023-09-23T23:29:46.738 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.738 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.738 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.738 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:209550 11.15 11:ae180d47:::benchmark_data_smithi110_433746_object103670:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e462)", 2023-09-23T23:29:46.739 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:17.437375+0000", 2023-09-23T23:29:46.739 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 89.131499374000001, 2023-09-23T23:29:46.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1469645319999999, 2023-09-23T23:29:46.739 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.739 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.739 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.739 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.739 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.739 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 209550 2023-09-23T23:29:46.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.739 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.740 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437375+0000", 2023-09-23T23:29:46.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.740 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.740 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.740 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437375+0000", 2023-09-23T23:29:46.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.740 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.740 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.740 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437374+0000", 2023-09-23T23:29:46.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:46.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.740 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.741 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.741 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437376+0000", 2023-09-23T23:29:46.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2699999999999999e-06 2023-09-23T23:29:46.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.741 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.741 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437382+0000", 2023-09-23T23:29:46.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6799999999999998e-06 2023-09-23T23:29:46.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.741 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.741 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437389+0000", 2023-09-23T23:29:46.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3890000000000002e-06 2023-09-23T23:29:46.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437590+0000", 2023-09-23T23:29:46.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020106 2023-09-23T23:29:46.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:29:46.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437594+0000", 2023-09-23T23:29:46.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8280000000000002e-06 2023-09-23T23:29:46.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.439704+0000", 2023-09-23T23:29:46.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021095129999999999 2023-09-23T23:29:46.743 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.439711+0000", 2023-09-23T23:29:46.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8539999999999999e-06 2023-09-23T23:29:46.743 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.559910+0000", 2023-09-23T23:29:46.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1201990630000001 2023-09-23T23:29:46.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.559993+0000", 2023-09-23T23:29:46.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3642999999999999e-05 2023-09-23T23:29:46.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4,5,7", 2023-09-23T23:29:46.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.560219+0000", 2023-09-23T23:29:46.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022577400000000001 2023-09-23T23:29:46.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.570069+0000", 2023-09-23T23:29:46.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0098502250000000006 2023-09-23T23:29:46.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.570104+0000", 2023-09-23T23:29:46.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4514000000000002e-05 2023-09-23T23:29:46.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.570140+0000", 2023-09-23T23:29:46.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6094999999999999e-05 2023-09-23T23:29:46.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.746 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.746 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.570175+0000", 2023-09-23T23:29:46.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4436e-05 2023-09-23T23:29:46.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.746 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.746 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.575605+0000", 2023-09-23T23:29:46.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054305330000000004 2023-09-23T23:29:46.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.746 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584203+0000", 2023-09-23T23:29:46.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0085978870000000002 2023-09-23T23:29:46.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584238+0000", 2023-09-23T23:29:46.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4703999999999998e-05 2023-09-23T23:29:46.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584339+0000", 2023-09-23T23:29:46.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000101438 2023-09-23T23:29:46.747 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.747 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.748 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.748 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.748 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:209562 11.15 11:a8ccb1ec:::benchmark_data_smithi110_433746_object103682:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e462)", 2023-09-23T23:29:46.748 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:17.437394+0000", 2023-09-23T23:29:46.748 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 89.131480052000001, 2023-09-23T23:29:46.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1473578520000001, 2023-09-23T23:29:46.748 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.748 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.748 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.748 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.748 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.748 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 209562 2023-09-23T23:29:46.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.749 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.749 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437394+0000", 2023-09-23T23:29:46.749 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.749 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437394+0000", 2023-09-23T23:29:46.749 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.750 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437393+0000", 2023-09-23T23:29:46.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:46.750 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.750 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.750 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437396+0000", 2023-09-23T23:29:46.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.385e-06 2023-09-23T23:29:46.750 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.750 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.750 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437402+0000", 2023-09-23T23:29:46.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7780000000000004e-06 2023-09-23T23:29:46.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.751 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437406+0000", 2023-09-23T23:29:46.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7419999999999997e-06 2023-09-23T23:29:46.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.751 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437632+0000", 2023-09-23T23:29:46.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022588799999999999 2023-09-23T23:29:46.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.752 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:29:46.752 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437636+0000", 2023-09-23T23:29:46.752 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2799999999999999e-06 2023-09-23T23:29:46.752 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.752 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.752 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.752 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.439734+0000", 2023-09-23T23:29:46.752 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020979750000000002 2023-09-23T23:29:46.752 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.752 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.752 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.752 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.439740+0000", 2023-09-23T23:29:46.753 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7190000000000004e-06 2023-09-23T23:29:46.753 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.753 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.753 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.753 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.562232+0000", 2023-09-23T23:29:46.753 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.122491599 2023-09-23T23:29:46.753 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.753 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.753 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.753 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.562368+0000", 2023-09-23T23:29:46.753 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013639699999999999 2023-09-23T23:29:46.753 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.754 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4,5,7", 2023-09-23T23:29:46.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.562675+0000", 2023-09-23T23:29:46.754 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030702099999999999 2023-09-23T23:29:46.754 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.754 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.570435+0000", 2023-09-23T23:29:46.754 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077596669999999996 2023-09-23T23:29:46.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.755 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.755 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.755 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.575571+0000", 2023-09-23T23:29:46.755 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051357959999999998 2023-09-23T23:29:46.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.755 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.755 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.755 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.575662+0000", 2023-09-23T23:29:46.755 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0878000000000006e-05 2023-09-23T23:29:46.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.756 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.756 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.756 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.575731+0000", 2023-09-23T23:29:46.756 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9114000000000003e-05 2023-09-23T23:29:46.756 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.756 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.756 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.756 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584351+0000", 2023-09-23T23:29:46.756 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086200430000000008 2023-09-23T23:29:46.756 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.757 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.757 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.757 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584572+0000", 2023-09-23T23:29:46.757 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022153299999999999 2023-09-23T23:29:46.757 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.757 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.757 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.757 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584620+0000", 2023-09-23T23:29:46.757 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7398999999999998e-05 2023-09-23T23:29:46.757 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.758 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.758 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.758 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584752+0000", 2023-09-23T23:29:46.758 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013205500000000001 2023-09-23T23:29:46.758 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.758 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.758 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.758 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.758 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.759 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:209589 11.15 11:ab416532:::benchmark_data_smithi110_433746_object103709:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e462)", 2023-09-23T23:29:46.759 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:17.437409+0000", 2023-09-23T23:29:46.759 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 89.131464937999993, 2023-09-23T23:29:46.759 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.147711031, 2023-09-23T23:29:46.759 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.759 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.759 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.759 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.759 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.759 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 209589 2023-09-23T23:29:46.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.760 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.760 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437409+0000", 2023-09-23T23:29:46.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.760 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437409+0000", 2023-09-23T23:29:46.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437409+0000", 2023-09-23T23:29:46.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:46.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437411+0000", 2023-09-23T23:29:46.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.396e-06 2023-09-23T23:29:46.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437417+0000", 2023-09-23T23:29:46.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4310000000000002e-06 2023-09-23T23:29:46.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437421+0000", 2023-09-23T23:29:46.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6530000000000002e-06 2023-09-23T23:29:46.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437652+0000", 2023-09-23T23:29:46.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000230774 2023-09-23T23:29:46.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:29:46.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437655+0000", 2023-09-23T23:29:46.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5639999999999999e-06 2023-09-23T23:29:46.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.439850+0000", 2023-09-23T23:29:46.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021952479999999999 2023-09-23T23:29:46.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.439857+0000", 2023-09-23T23:29:46.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5329999999999997e-06 2023-09-23T23:29:46.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.564595+0000", 2023-09-23T23:29:46.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1247379719999999 2023-09-23T23:29:46.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.766 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.566629+0000", 2023-09-23T23:29:46.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020338499999999998 2023-09-23T23:29:46.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4,5,7", 2023-09-23T23:29:46.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.567006+0000", 2023-09-23T23:29:46.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000376556 2023-09-23T23:29:46.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.575634+0000", 2023-09-23T23:29:46.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086283269999999999 2023-09-23T23:29:46.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.768 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.575696+0000", 2023-09-23T23:29:46.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1939000000000004e-05 2023-09-23T23:29:46.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.768 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.768 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.768 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.575765+0000", 2023-09-23T23:29:46.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9531999999999998e-05 2023-09-23T23:29:46.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.768 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.768 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.768 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.575837+0000", 2023-09-23T23:29:46.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1348999999999997e-05 2023-09-23T23:29:46.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.769 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.769 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.769 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584358+0000", 2023-09-23T23:29:46.769 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0085213330000000007 2023-09-23T23:29:46.769 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.769 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.769 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.769 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584926+0000", 2023-09-23T23:29:46.769 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00056737400000000005 2023-09-23T23:29:46.769 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.769 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.769 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.769 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584971+0000", 2023-09-23T23:29:46.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4962000000000001e-05 2023-09-23T23:29:46.770 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.770 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.770 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.585120+0000", 2023-09-23T23:29:46.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014951499999999999 2023-09-23T23:29:46.770 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.770 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.770 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.770 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.771 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:209661 11.15 11:a878b7bb:::benchmark_data_smithi110_433746_object103781:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e462)", 2023-09-23T23:29:46.771 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:17.437455+0000", 2023-09-23T23:29:46.771 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 89.131418619000002, 2023-09-23T23:29:46.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.147860487, 2023-09-23T23:29:46.771 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.771 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.771 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.771 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.771 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.771 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 209661 2023-09-23T23:29:46.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.771 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.772 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437455+0000", 2023-09-23T23:29:46.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.772 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.772 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437455+0000", 2023-09-23T23:29:46.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.772 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.772 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437455+0000", 2023-09-23T23:29:46.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:46.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437458+0000", 2023-09-23T23:29:46.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9270000000000001e-06 2023-09-23T23:29:46.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437465+0000", 2023-09-23T23:29:46.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0469999999999996e-06 2023-09-23T23:29:46.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437469+0000", 2023-09-23T23:29:46.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.002e-06 2023-09-23T23:29:46.774 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437677+0000", 2023-09-23T23:29:46.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000208333 2023-09-23T23:29:46.774 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:29:46.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437681+0000", 2023-09-23T23:29:46.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6619999999999998e-06 2023-09-23T23:29:46.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.440166+0000", 2023-09-23T23:29:46.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024850509999999998 2023-09-23T23:29:46.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.440174+0000", 2023-09-23T23:29:46.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.568e-06 2023-09-23T23:29:46.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.570481+0000", 2023-09-23T23:29:46.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.13030648 2023-09-23T23:29:46.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.570623+0000", 2023-09-23T23:29:46.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014196599999999999 2023-09-23T23:29:46.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4,5,7", 2023-09-23T23:29:46.777 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.570995+0000", 2023-09-23T23:29:46.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00037248099999999997 2023-09-23T23:29:46.777 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.777 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.777 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.575803+0000", 2023-09-23T23:29:46.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048074299999999997 2023-09-23T23:29:46.777 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.777 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.777 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.575875+0000", 2023-09-23T23:29:46.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2516999999999997e-05 2023-09-23T23:29:46.777 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.778 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.778 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584368+0000", 2023-09-23T23:29:46.778 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084922650000000006 2023-09-23T23:29:46.778 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.778 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.778 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584526+0000", 2023-09-23T23:29:46.778 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015885699999999999 2023-09-23T23:29:46.778 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.778 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584795+0000", 2023-09-23T23:29:46.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026868600000000002 2023-09-23T23:29:46.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.585202+0000", 2023-09-23T23:29:46.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00040736400000000001 2023-09-23T23:29:46.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.585231+0000", 2023-09-23T23:29:46.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8844999999999999e-05 2023-09-23T23:29:46.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.780 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.585316+0000", 2023-09-23T23:29:46.780 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4536999999999994e-05 2023-09-23T23:29:46.780 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.780 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.780 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.780 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:209678 11.15 11:aa913f1d:::benchmark_data_smithi110_433746_object103798:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e462)", 2023-09-23T23:29:46.780 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:17.437472+0000", 2023-09-23T23:29:46.780 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 89.131402213000001, 2023-09-23T23:29:46.780 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.148038774, 2023-09-23T23:29:46.780 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.781 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.781 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.781 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.781 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.781 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 209678 2023-09-23T23:29:46.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.781 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437472+0000", 2023-09-23T23:29:46.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.782 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437472+0000", 2023-09-23T23:29:46.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.782 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437471+0000", 2023-09-23T23:29:46.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:46.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.782 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437474+0000", 2023-09-23T23:29:46.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2220000000000001e-06 2023-09-23T23:29:46.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437478+0000", 2023-09-23T23:29:46.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7180000000000004e-06 2023-09-23T23:29:46.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437482+0000", 2023-09-23T23:29:46.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.89e-06 2023-09-23T23:29:46.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437699+0000", 2023-09-23T23:29:46.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000216405 2023-09-23T23:29:46.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:29:46.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437701+0000", 2023-09-23T23:29:46.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.571e-06 2023-09-23T23:29:46.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.440200+0000", 2023-09-23T23:29:46.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024991710000000001 2023-09-23T23:29:46.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.440208+0000", 2023-09-23T23:29:46.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1810000000000004e-06 2023-09-23T23:29:46.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.573053+0000", 2023-09-23T23:29:46.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1328459099999999 2023-09-23T23:29:46.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.573204+0000", 2023-09-23T23:29:46.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000150617 2023-09-23T23:29:46.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4,5,7", 2023-09-23T23:29:46.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.573476+0000", 2023-09-23T23:29:46.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027195099999999999 2023-09-23T23:29:46.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584374+0000", 2023-09-23T23:29:46.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01089793 2023-09-23T23:29:46.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584425+0000", 2023-09-23T23:29:46.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1347000000000002e-05 2023-09-23T23:29:46.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584481+0000", 2023-09-23T23:29:46.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5479000000000002e-05 2023-09-23T23:29:46.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584895+0000", 2023-09-23T23:29:46.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00041428199999999997 2023-09-23T23:29:46.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.585146+0000", 2023-09-23T23:29:46.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025079900000000001 2023-09-23T23:29:46.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.585359+0000", 2023-09-23T23:29:46.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000213563 2023-09-23T23:29:46.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.585388+0000", 2023-09-23T23:29:46.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8178e-05 2023-09-23T23:29:46.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.585510+0000", 2023-09-23T23:29:46.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000122884 2023-09-23T23:29:46.790 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.790 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.790 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.790 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:209713 11.15 11:ae795b5a:::benchmark_data_smithi110_433746_object103833:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e462)", 2023-09-23T23:29:46.790 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:17.437486+0000", 2023-09-23T23:29:46.790 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 89.131388114000003, 2023-09-23T23:29:46.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1572487439999999, 2023-09-23T23:29:46.790 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.790 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.790 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.791 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.791 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.791 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 209713 2023-09-23T23:29:46.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.791 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.791 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437486+0000", 2023-09-23T23:29:46.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.791 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437486+0000", 2023-09-23T23:29:46.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437485+0000", 2023-09-23T23:29:46.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:46.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437488+0000", 2023-09-23T23:29:46.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0329999999999998e-06 2023-09-23T23:29:46.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437493+0000", 2023-09-23T23:29:46.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7259999999999999e-06 2023-09-23T23:29:46.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437498+0000", 2023-09-23T23:29:46.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8999999999999997e-06 2023-09-23T23:29:46.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437766+0000", 2023-09-23T23:29:46.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000268309 2023-09-23T23:29:46.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:29:46.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437771+0000", 2023-09-23T23:29:46.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.189e-06 2023-09-23T23:29:46.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.440232+0000", 2023-09-23T23:29:46.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024616939999999999 2023-09-23T23:29:46.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.440239+0000", 2023-09-23T23:29:46.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0169999999999999e-06 2023-09-23T23:29:46.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.575901+0000", 2023-09-23T23:29:46.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1356617959999999 2023-09-23T23:29:46.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.576053+0000", 2023-09-23T23:29:46.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000151435 2023-09-23T23:29:46.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4,5,7", 2023-09-23T23:29:46.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.576368+0000", 2023-09-23T23:29:46.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00031577599999999998 2023-09-23T23:29:46.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584380+0000", 2023-09-23T23:29:46.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080117820000000003 2023-09-23T23:29:46.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584829+0000", 2023-09-23T23:29:46.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00044914899999999998 2023-09-23T23:29:46.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584861+0000", 2023-09-23T23:29:46.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1479000000000001e-05 2023-09-23T23:29:46.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.585549+0000", 2023-09-23T23:29:46.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00068784100000000002 2023-09-23T23:29:46.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.592229+0000", 2023-09-23T23:29:46.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0066805850000000002 2023-09-23T23:29:46.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.594549+0000", 2023-09-23T23:29:46.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023202449999999999 2023-09-23T23:29:46.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.594610+0000", 2023-09-23T23:29:46.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0061e-05 2023-09-23T23:29:46.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.801 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.801 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.594735+0000", 2023-09-23T23:29:46.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012505099999999999 2023-09-23T23:29:46.801 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.801 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.801 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.801 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:209717 11.15 11:a94abe76:::benchmark_data_smithi110_433746_object103837:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e462)", 2023-09-23T23:29:46.801 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:17.437518+0000", 2023-09-23T23:29:46.802 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 89.131356074999999, 2023-09-23T23:29:46.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1574193429999999, 2023-09-23T23:29:46.802 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.802 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.802 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.802 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.802 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.802 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 209717 2023-09-23T23:29:46.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.803 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437518+0000", 2023-09-23T23:29:46.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437518+0000", 2023-09-23T23:29:46.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437517+0000", 2023-09-23T23:29:46.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:46.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437523+0000", 2023-09-23T23:29:46.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4480000000000002e-06 2023-09-23T23:29:46.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437530+0000", 2023-09-23T23:29:46.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7800000000000003e-06 2023-09-23T23:29:46.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437550+0000", 2023-09-23T23:29:46.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9911e-05 2023-09-23T23:29:46.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437799+0000", 2023-09-23T23:29:46.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024983999999999998 2023-09-23T23:29:46.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:29:46.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437803+0000", 2023-09-23T23:29:46.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1099999999999999e-06 2023-09-23T23:29:46.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.440264+0000", 2023-09-23T23:29:46.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024614049999999998 2023-09-23T23:29:46.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.440271+0000", 2023-09-23T23:29:46.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1350000000000003e-06 2023-09-23T23:29:46.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.578025+0000", 2023-09-23T23:29:46.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1377535700000001 2023-09-23T23:29:46.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.578152+0000", 2023-09-23T23:29:46.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012759499999999999 2023-09-23T23:29:46.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4,5,7", 2023-09-23T23:29:46.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.578419+0000", 2023-09-23T23:29:46.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026705700000000001 2023-09-23T23:29:46.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584386+0000", 2023-09-23T23:29:46.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059666989999999998 2023-09-23T23:29:46.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.585166+0000", 2023-09-23T23:29:46.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00077997899999999998 2023-09-23T23:29:46.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.585184+0000", 2023-09-23T23:29:46.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8482e-05 2023-09-23T23:29:46.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.810 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.585575+0000", 2023-09-23T23:29:46.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039077300000000002 2023-09-23T23:29:46.810 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.810 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.810 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.592271+0000", 2023-09-23T23:29:46.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0066960980000000002 2023-09-23T23:29:46.810 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.810 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.810 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.594762+0000", 2023-09-23T23:29:46.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002490964 2023-09-23T23:29:46.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.594802+0000", 2023-09-23T23:29:46.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9486000000000001e-05 2023-09-23T23:29:46.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.594937+0000", 2023-09-23T23:29:46.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013543299999999999 2023-09-23T23:29:46.811 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.811 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.811 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.812 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:209756 11.15 11:af197cbb:::benchmark_data_smithi110_433746_object103876:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e462)", 2023-09-23T23:29:46.812 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:17.437564+0000", 2023-09-23T23:29:46.812 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 89.131309548999994, 2023-09-23T23:29:46.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.157573183, 2023-09-23T23:29:46.812 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.812 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.812 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.812 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.812 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.812 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 209756 2023-09-23T23:29:46.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.813 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437564+0000", 2023-09-23T23:29:46.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437564+0000", 2023-09-23T23:29:46.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437564+0000", 2023-09-23T23:29:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437567+0000", 2023-09-23T23:29:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.954e-06 2023-09-23T23:29:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437574+0000", 2023-09-23T23:29:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.588e-06 2023-09-23T23:29:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437580+0000", 2023-09-23T23:29:46.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5010000000000004e-06 2023-09-23T23:29:46.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437850+0000", 2023-09-23T23:29:46.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027017399999999998 2023-09-23T23:29:46.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:29:46.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437855+0000", 2023-09-23T23:29:46.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1359999999999996e-06 2023-09-23T23:29:46.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.440296+0000", 2023-09-23T23:29:46.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024406760000000001 2023-09-23T23:29:46.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.440303+0000", 2023-09-23T23:29:46.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1269999999999997e-06 2023-09-23T23:29:46.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.580302+0000", 2023-09-23T23:29:46.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.139998933 2023-09-23T23:29:46.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.580420+0000", 2023-09-23T23:29:46.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118158 2023-09-23T23:29:46.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4,5,7", 2023-09-23T23:29:46.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.580700+0000", 2023-09-23T23:29:46.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027986200000000003 2023-09-23T23:29:46.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584434+0000", 2023-09-23T23:29:46.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037337070000000002 2023-09-23T23:29:46.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.585342+0000", 2023-09-23T23:29:46.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00090846000000000004 2023-09-23T23:29:46.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.819 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.819 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.592313+0000", 2023-09-23T23:29:46.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0069706179999999996 2023-09-23T23:29:46.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.819 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.819 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.592388+0000", 2023-09-23T23:29:46.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5578e-05 2023-09-23T23:29:46.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.819 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.819 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.592425+0000", 2023-09-23T23:29:46.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6488e-05 2023-09-23T23:29:46.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.594975+0000", 2023-09-23T23:29:46.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002550241 2023-09-23T23:29:46.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.595014+0000", 2023-09-23T23:29:46.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9307999999999997e-05 2023-09-23T23:29:46.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.595138+0000", 2023-09-23T23:29:46.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012318400000000001 2023-09-23T23:29:46.821 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.821 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.821 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.821 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:209825 11.15 11:ada69f9e:::benchmark_data_smithi110_433746_object103945:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e462)", 2023-09-23T23:29:46.821 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:17.437584+0000", 2023-09-23T23:29:46.821 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 89.131289735999999, 2023-09-23T23:29:46.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1577566560000001, 2023-09-23T23:29:46.821 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.821 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.822 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.822 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.822 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.822 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 209825 2023-09-23T23:29:46.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.822 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437584+0000", 2023-09-23T23:29:46.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437584+0000", 2023-09-23T23:29:46.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437584+0000", 2023-09-23T23:29:46.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:46.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437586+0000", 2023-09-23T23:29:46.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.153e-06 2023-09-23T23:29:46.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437591+0000", 2023-09-23T23:29:46.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3730000000000001e-06 2023-09-23T23:29:46.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437597+0000", 2023-09-23T23:29:46.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5600000000000001e-06 2023-09-23T23:29:46.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.440328+0000", 2023-09-23T23:29:46.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027309980000000001 2023-09-23T23:29:46.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.440335+0000", 2023-09-23T23:29:46.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2320000000000004e-06 2023-09-23T23:29:46.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.582295+0000", 2023-09-23T23:29:46.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1419602259999999 2023-09-23T23:29:46.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.582397+0000", 2023-09-23T23:29:46.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010146500000000001 2023-09-23T23:29:46.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4,5,7", 2023-09-23T23:29:46.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.582625+0000", 2023-09-23T23:29:46.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000228051 2023-09-23T23:29:46.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.584440+0000", 2023-09-23T23:29:46.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001814738 2023-09-23T23:29:46.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.592350+0000", 2023-09-23T23:29:46.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079106620000000006 2023-09-23T23:29:46.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.592463+0000", 2023-09-23T23:29:46.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000112453 2023-09-23T23:29:46.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.592499+0000", 2023-09-23T23:29:46.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.595e-05 2023-09-23T23:29:46.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.592542+0000", 2023-09-23T23:29:46.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2837000000000003e-05 2023-09-23T23:29:46.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.595180+0000", 2023-09-23T23:29:46.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026383420000000001 2023-09-23T23:29:46.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.595218+0000", 2023-09-23T23:29:46.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7727e-05 2023-09-23T23:29:46.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.595341+0000", 2023-09-23T23:29:46.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012324100000000001 2023-09-23T23:29:46.830 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.831 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.831 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.831 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:209863 11.15 11:a9161c99:::benchmark_data_smithi110_433746_object103983:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e462)", 2023-09-23T23:29:46.831 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:17.437765+0000", 2023-09-23T23:29:46.831 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 89.131109158000001, 2023-09-23T23:29:46.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1637293, 2023-09-23T23:29:46.831 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.832 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.832 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.832 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.832 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.832 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 209863 2023-09-23T23:29:46.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.832 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437765+0000", 2023-09-23T23:29:46.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437765+0000", 2023-09-23T23:29:46.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437764+0000", 2023-09-23T23:29:46.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:46.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437768+0000", 2023-09-23T23:29:46.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6100000000000002e-06 2023-09-23T23:29:46.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437777+0000", 2023-09-23T23:29:46.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0440000000000001e-06 2023-09-23T23:29:46.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437784+0000", 2023-09-23T23:29:46.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7669999999999998e-06 2023-09-23T23:29:46.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.440360+0000", 2023-09-23T23:29:46.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00257624 2023-09-23T23:29:46.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.440366+0000", 2023-09-23T23:29:46.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3659999999999997e-06 2023-09-23T23:29:46.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.585589+0000", 2023-09-23T23:29:46.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.145222792 2023-09-23T23:29:46.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.585710+0000", 2023-09-23T23:29:46.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012105299999999999 2023-09-23T23:29:46.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4,5,7", 2023-09-23T23:29:46.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.586085+0000", 2023-09-23T23:29:46.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00037486000000000003 2023-09-23T23:29:46.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.594472+0000", 2023-09-23T23:29:46.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083873200000000002 2023-09-23T23:29:46.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.595380+0000", 2023-09-23T23:29:46.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0009077 2023-09-23T23:29:46.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.600402+0000", 2023-09-23T23:29:46.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0050220610000000004 2023-09-23T23:29:46.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.600411+0000", 2023-09-23T23:29:46.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9299999999999992e-06 2023-09-23T23:29:46.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.600455+0000", 2023-09-23T23:29:46.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3668999999999998e-05 2023-09-23T23:29:46.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.601286+0000", 2023-09-23T23:29:46.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00083098199999999997 2023-09-23T23:29:46.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.601338+0000", 2023-09-23T23:29:46.841 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2710000000000002e-05 2023-09-23T23:29:46.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.841 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.841 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.601494+0000", 2023-09-23T23:29:46.841 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015572200000000001 2023-09-23T23:29:46.841 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.841 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.841 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.841 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:209891 11.15 11:acb7b847:::benchmark_data_smithi110_433746_object104011:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e462)", 2023-09-23T23:29:46.841 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:17.437797+0000", 2023-09-23T23:29:46.842 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 89.131076655000001, 2023-09-23T23:29:46.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1680687199999999, 2023-09-23T23:29:46.842 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:29:46.842 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:46.842 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:29:46.842 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:29:46.842 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:46.842 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 209891 2023-09-23T23:29:46.842 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.842 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:29:46.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.842 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:29:46.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437797+0000", 2023-09-23T23:29:46.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:29:46.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437797+0000", 2023-09-23T23:29:46.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:29:46.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:29:46.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437797+0000", 2023-09-23T23:29:46.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:46.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:29:46.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437799+0000", 2023-09-23T23:29:46.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1090000000000001e-06 2023-09-23T23:29:46.844 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:29:46.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437805+0000", 2023-09-23T23:29:46.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8499999999999999e-06 2023-09-23T23:29:46.844 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:29:46.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.437810+0000", 2023-09-23T23:29:46.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9799999999999998e-06 2023-09-23T23:29:46.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.845 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.440693+0000", 2023-09-23T23:29:46.845 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028831540000000002 2023-09-23T23:29:46.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:29:46.845 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:17.440703+0000", 2023-09-23T23:29:46.845 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7319999999999993e-06 2023-09-23T23:29:46.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:29:46.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.587858+0000", 2023-09-23T23:29:46.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1471554180000001 2023-09-23T23:29:46.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.846 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:29:46.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.587981+0000", 2023-09-23T23:29:46.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012235699999999999 2023-09-23T23:29:46.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.846 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4,5,7", 2023-09-23T23:29:46.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.588245+0000", 2023-09-23T23:29:46.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026415599999999998 2023-09-23T23:29:46.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.847 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.847 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.594509+0000", 2023-09-23T23:29:46.847 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0062639419999999998 2023-09-23T23:29:46.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:29:46.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.600461+0000", 2023-09-23T23:29:46.848 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059522739999999996 2023-09-23T23:29:46.848 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.848 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.849 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.849 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.600486+0000", 2023-09-23T23:29:46.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4700000000000001e-05 2023-09-23T23:29:46.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.849 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.849 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.849 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.600554+0000", 2023-09-23T23:29:46.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8745999999999995e-05 2023-09-23T23:29:46.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.849 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.849 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.849 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.601554+0000", 2023-09-23T23:29:46.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00099984700000000011 2023-09-23T23:29:46.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:46.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.605675+0000", 2023-09-23T23:29:46.850 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041212130000000003 2023-09-23T23:29:46.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:29:46.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.605726+0000", 2023-09-23T23:29:46.850 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0395000000000001e-05 2023-09-23T23:29:46.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:29:46.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:29:46.851 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:29:46.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:18.605866+0000", 2023-09-23T23:29:46.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014020300000000001 2023-09-23T23:29:46.851 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.851 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.851 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.851 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:29:46.851 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:29:46.851 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:29:46.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:46.853 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:46.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:46.901 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:29:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:29:46] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.43.0" 2023-09-23T23:29:46.902 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:46 smithi110 ceph-mon[221102]: failed to encode map e491 with expected crc 2023-09-23T23:29:46.902 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:46 smithi110 ceph-mon[221102]: failed to encode map e491 with expected crc 2023-09-23T23:29:46.902 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:46 smithi110 ceph-mon[221102]: failed to encode map e491 with expected crc 2023-09-23T23:29:46.902 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:46 smithi110 ceph-mon[221102]: failed to encode map e491 with expected crc 2023-09-23T23:29:46.902 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:46 smithi110 ceph-mon[221102]: failed to encode map e491 with expected crc 2023-09-23T23:29:46.903 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:46 smithi110 ceph-mon[221102]: pgmap v1348: 129 pgs: 2 peering, 10 unknown, 117 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 757 op/s 2023-09-23T23:29:46.953 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:29:46.979 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:47.021 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:29:47.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:47.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:29:47.064 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:29:47.127 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:29:47.152 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:46 smithi138 ceph-mon[193606]: failed to encode map e491 with expected crc 2023-09-23T23:29:47.152 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:46 smithi138 ceph-mon[193606]: failed to encode map e491 with expected crc 2023-09-23T23:29:47.152 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:46 smithi138 ceph-mon[193606]: failed to encode map e491 with expected crc 2023-09-23T23:29:47.152 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:46 smithi138 ceph-mon[193606]: failed to encode map e491 with expected crc 2023-09-23T23:29:47.152 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:46 smithi138 ceph-mon[193606]: failed to encode map e491 with expected crc 2023-09-23T23:29:47.152 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:46 smithi138 ceph-mon[193606]: pgmap v1348: 129 pgs: 2 peering, 10 unknown, 117 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 757 op/s 2023-09-23T23:29:47.152 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:46 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3606516460' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:47.153 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:46 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:47.153 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:47.199 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:29:47.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:46 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3606516460' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:47.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:46 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:47.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:46 smithi110 ceph-mon[212939]: failed to encode map e491 with expected crc 2023-09-23T23:29:47.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:46 smithi110 ceph-mon[212939]: failed to encode map e491 with expected crc 2023-09-23T23:29:47.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:46 smithi110 ceph-mon[212939]: failed to encode map e491 with expected crc 2023-09-23T23:29:47.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:46 smithi110 ceph-mon[212939]: failed to encode map e491 with expected crc 2023-09-23T23:29:47.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:46 smithi110 ceph-mon[212939]: failed to encode map e491 with expected crc 2023-09-23T23:29:47.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:46 smithi110 ceph-mon[212939]: pgmap v1348: 129 pgs: 2 peering, 10 unknown, 117 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 757 op/s 2023-09-23T23:29:47.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:46 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3606516460' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:47.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:46 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:29:47.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:47.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:29:47.245 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 62 16 74868 74852 75.4083 54.875 0.00887801 0.0132411 2023-09-23T23:29:47.305 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:29:47.334 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:47.411 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:29:47.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:47.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:29:47.518 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:47.566 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:47.913 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:47 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:29:47.914 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:47 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:29:47.914 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:47 smithi110 ceph-mon[221102]: osdmap e492: 8 total, 8 up, 7 in 2023-09-23T23:29:47.914 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:47 smithi110 ceph-mon[221102]: failed to encode map e492 with expected crc 2023-09-23T23:29:47.914 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:47 smithi110 ceph-mon[221102]: failed to encode map e492 with expected crc 2023-09-23T23:29:47.914 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:47 smithi110 ceph-mon[221102]: failed to encode map e492 with expected crc 2023-09-23T23:29:48.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:47 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:29:48.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:47 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:29:48.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:47 smithi138 ceph-mon[193606]: osdmap e492: 8 total, 8 up, 7 in 2023-09-23T23:29:48.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:47 smithi138 ceph-mon[193606]: failed to encode map e492 with expected crc 2023-09-23T23:29:48.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:47 smithi138 ceph-mon[193606]: failed to encode map e492 with expected crc 2023-09-23T23:29:48.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:47 smithi138 ceph-mon[193606]: failed to encode map e492 with expected crc 2023-09-23T23:29:48.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:47 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:29:48.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:47 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:29:48.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:47 smithi110 ceph-mon[212939]: osdmap e492: 8 total, 8 up, 7 in 2023-09-23T23:29:48.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:47 smithi110 ceph-mon[212939]: failed to encode map e492 with expected crc 2023-09-23T23:29:48.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:47 smithi110 ceph-mon[212939]: failed to encode map e492 with expected crc 2023-09-23T23:29:48.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:47 smithi110 ceph-mon[212939]: failed to encode map e492 with expected crc 2023-09-23T23:29:48.245 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 63 16 76005 75989 75.3393 71.0625 0.00215474 0.0132633 2023-09-23T23:29:48.685 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:48.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:48.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:48.792 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:48.863 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:48.926 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:29:49.116 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:48 smithi110 ceph-mon[221102]: failed to encode map e492 with expected crc 2023-09-23T23:29:49.116 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:48 smithi110 ceph-mon[221102]: pgmap v1350: 129 pgs: 2 peering, 8 unknown, 119 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 791 op/s 2023-09-23T23:29:49.117 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:48 smithi110 ceph-mon[212939]: failed to encode map e492 with expected crc 2023-09-23T23:29:49.117 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:48 smithi110 ceph-mon[212939]: pgmap v1350: 129 pgs: 2 peering, 8 unknown, 119 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 791 op/s 2023-09-23T23:29:49.139 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:49.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:49.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:48 smithi138 ceph-mon[193606]: failed to encode map e492 with expected crc 2023-09-23T23:29:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:48 smithi138 ceph-mon[193606]: pgmap v1350: 129 pgs: 2 peering, 8 unknown, 119 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49 MiB/s wr, 791 op/s 2023-09-23T23:29:49.247 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 64 16 77076 77060 75.208 66.9375 0.0124245 0.013292 2023-09-23T23:29:49.252 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:49.320 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:49.338 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 7, 2, 1] out_osds: [6] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:29:49.338 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:29:49.339 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-09-23T23:29:49.339 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd in 6 2023-09-23T23:29:49.396 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:29:49.451 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:49.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:49.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:49.496 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:29:49.533 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:29:49.564 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:49.587 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:29:49.629 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:29:49.655 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:49.664 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:29:49.704 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:29:49.742 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:29:49.770 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:49.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:49.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:49.780 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:29:49.786 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:29:49.786 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:29:49.786 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:29:49.786 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:29:49.787 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:29:49.787 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:29:49.787 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:29:49.787 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:29:49.881 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:29:49.927 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:49.984 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:29:49.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:49.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:29:50.089 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:50.145 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:49 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1299872501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:50.146 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:49 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1299872501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:50.152 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:50.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:49 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1299872501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:50.247 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 65 16 78183 78167 75.1138 69.1875 0.00819177 0.0133084 2023-09-23T23:29:50.263 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:50.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:50.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:50.372 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:29:50.442 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:50.528 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:29:50.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:50.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:29:50.634 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:29:50.710 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:50.919 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:29:50.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:50.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:29:51.011 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:51.030 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:29:51.053 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:29:51.091 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:50 smithi110 ceph-mon[212939]: pgmap v1351: 129 pgs: 2 peering, 2 unknown, 125 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s; 32 KiB/s, 0 objects/s recovering 2023-09-23T23:29:51.097 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:50 smithi110 ceph-mon[221102]: pgmap v1351: 129 pgs: 2 peering, 2 unknown, 125 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s; 32 KiB/s, 0 objects/s recovering 2023-09-23T23:29:51.103 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:51.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:50 smithi138 ceph-mon[193606]: pgmap v1351: 129 pgs: 2 peering, 2 unknown, 125 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s; 32 KiB/s, 0 objects/s recovering 2023-09-23T23:29:51.247 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 66 16 79408 79392 75.1357 76.5625 0.00858071 0.0133043 2023-09-23T23:29:51.351 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:29:51.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:51.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:29:51.459 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:29:51.486 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:51.528 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:29:51.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:51.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:29:51.632 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:51.719 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:51.862 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:51.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:51.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:51.972 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:29:51.974 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:51 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T23:29:51.974 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:51 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T23:29:51.974 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:51 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T23:29:51.974 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:51 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T23:29:52.004 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:52.121 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:29:52.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:52.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:29:52.233 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:29:52.247 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 67 16 80282 80266 74.8296 54.625 0.00451695 0.0133421 2023-09-23T23:29:52.262 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:51 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T23:29:52.262 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:51 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T23:29:52.265 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:52.367 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:29:52.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:52.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:29:52.472 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:52.499 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:52.587 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:29:52.588 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:29:52.703 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:52.742 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:52.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:52.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:52.852 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:29:52.881 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:52.925 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:29:52.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:52.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:29:52.931 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:29:53.028 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:29:53.048 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:52 smithi110 ceph-mon[212939]: pgmap v1352: 129 pgs: 129 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 787 op/s; 27 KiB/s, 0 objects/s recovering 2023-09-23T23:29:53.048 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:52 smithi110 ceph-mon[212939]: failed to encode map e493 with expected crc 2023-09-23T23:29:53.049 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:52 smithi110 ceph-mon[212939]: failed to encode map e493 with expected crc 2023-09-23T23:29:53.049 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:52 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T23:29:53.049 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:52 smithi110 ceph-mon[212939]: osdmap e493: 8 total, 8 up, 7 in 2023-09-23T23:29:53.049 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:52 smithi110 ceph-mon[221102]: pgmap v1352: 129 pgs: 129 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 787 op/s; 27 KiB/s, 0 objects/s recovering 2023-09-23T23:29:53.049 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:52 smithi110 ceph-mon[221102]: failed to encode map e493 with expected crc 2023-09-23T23:29:53.049 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:52 smithi110 ceph-mon[221102]: failed to encode map e493 with expected crc 2023-09-23T23:29:53.049 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:52 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T23:29:53.049 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:52 smithi110 ceph-mon[221102]: osdmap e493: 8 total, 8 up, 7 in 2023-09-23T23:29:53.056 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:53.099 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:29:53.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:53.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:29:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:52 smithi138 ceph-mon[193606]: pgmap v1352: 129 pgs: 129 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 787 op/s; 27 KiB/s, 0 objects/s recovering 2023-09-23T23:29:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:52 smithi138 ceph-mon[193606]: failed to encode map e493 with expected crc 2023-09-23T23:29:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:52 smithi138 ceph-mon[193606]: failed to encode map e493 with expected crc 2023-09-23T23:29:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:52 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T23:29:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:52 smithi138 ceph-mon[193606]: osdmap e493: 8 total, 8 up, 7 in 2023-09-23T23:29:53.203 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:53.234 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:53.247 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 68 16 80391 80375 73.8299 6.8125 0.0034511 0.0133288 2023-09-23T23:29:53.400 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:29:53.432 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:53.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:53.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:53.535 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:29:53.561 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:53.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:53 smithi110 conmon[212914]: 2023-09-23T23:29:53.465+0000 7f29aa464700 -1 mon.a@0(leader).osd e494 definitely_dead 0 2023-09-23T23:29:53.739 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:29:53.924 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:53.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:53.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:54.004 INFO:teuthology.orchestra.run.smithi110.stderr:marked in osd.6. 2023-09-23T23:29:54.027 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:29:54.088 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:54.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[212939]: failed to encode map e494 with expected crc 2023-09-23T23:29:54.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[212939]: failed to encode map e494 with expected crc 2023-09-23T23:29:54.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[212939]: failed to encode map e494 with expected crc 2023-09-23T23:29:54.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[212939]: osdmap e494: 8 total, 8 up, 7 in 2023-09-23T23:29:54.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[212939]: failed to encode map e494 with expected crc 2023-09-23T23:29:54.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3299250471' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T23:29:54.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T23:29:54.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T23:29:54.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T23:29:54.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[221102]: failed to encode map e494 with expected crc 2023-09-23T23:29:54.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[221102]: failed to encode map e494 with expected crc 2023-09-23T23:29:54.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[221102]: failed to encode map e494 with expected crc 2023-09-23T23:29:54.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[221102]: osdmap e494: 8 total, 8 up, 7 in 2023-09-23T23:29:54.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[221102]: failed to encode map e494 with expected crc 2023-09-23T23:29:54.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3299250471' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T23:29:54.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T23:29:54.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T23:29:54.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:53 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T23:29:54.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:53 smithi138 ceph-mon[193606]: failed to encode map e494 with expected crc 2023-09-23T23:29:54.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:53 smithi138 ceph-mon[193606]: failed to encode map e494 with expected crc 2023-09-23T23:29:54.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:53 smithi138 ceph-mon[193606]: failed to encode map e494 with expected crc 2023-09-23T23:29:54.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:53 smithi138 ceph-mon[193606]: osdmap e494: 8 total, 8 up, 7 in 2023-09-23T23:29:54.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:53 smithi138 ceph-mon[193606]: failed to encode map e494 with expected crc 2023-09-23T23:29:54.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:53 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3299250471' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T23:29:54.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:53 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T23:29:54.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:53 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T23:29:54.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:53 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T23:29:54.202 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:29:54.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:54.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:29:54.247 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 69 16 80430 80414 72.7957 2.4375 0.0022957 0.0137197 2023-09-23T23:29:54.308 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:29:54.337 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:54.381 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:29:54.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:54.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:29:54.485 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:54.512 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:54.910 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:54.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:54.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:54.956 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:29:54.976 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-09-23T23:29:55.016 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:55.044 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:55.097 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:55 smithi138 ceph-mon[193606]: pgmap v1355: 129 pgs: 129 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 712 op/s; 28 KiB/s, 0 objects/s recovering 2023-09-23T23:29:55.097 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:55 smithi138 ceph-mon[193606]: failed to encode map e494 with expected crc 2023-09-23T23:29:55.097 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:55 smithi138 ceph-mon[193606]: failed to encode map e495 with expected crc 2023-09-23T23:29:55.098 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:55 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T23:29:55.098 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:55 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T23:29:55.098 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:55 smithi138 ceph-mon[193606]: osdmap e495: 8 total, 8 up, 8 in 2023-09-23T23:29:55.098 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:55 smithi138 ceph-mon[193606]: failed to encode map e495 with expected crc 2023-09-23T23:29:55.098 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:55 smithi138 ceph-mon[193606]: failed to encode map e495 with expected crc 2023-09-23T23:29:55.098 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:55 smithi138 ceph-mon[193606]: failed to encode map e495 with expected crc 2023-09-23T23:29:55.098 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:55 smithi138 ceph-mon[193606]: failed to encode map e495 with expected crc 2023-09-23T23:29:55.098 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:55 smithi138 ceph-mon[193606]: osdmap e496: 8 total, 8 up, 8 in 2023-09-23T23:29:55.141 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:55.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:55.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:55.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[212939]: pgmap v1355: 129 pgs: 129 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 712 op/s; 28 KiB/s, 0 objects/s recovering 2023-09-23T23:29:55.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[212939]: failed to encode map e494 with expected crc 2023-09-23T23:29:55.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[212939]: failed to encode map e495 with expected crc 2023-09-23T23:29:55.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T23:29:55.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T23:29:55.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[212939]: osdmap e495: 8 total, 8 up, 8 in 2023-09-23T23:29:55.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[212939]: failed to encode map e495 with expected crc 2023-09-23T23:29:55.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[212939]: failed to encode map e495 with expected crc 2023-09-23T23:29:55.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[212939]: failed to encode map e495 with expected crc 2023-09-23T23:29:55.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[212939]: failed to encode map e495 with expected crc 2023-09-23T23:29:55.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[212939]: osdmap e496: 8 total, 8 up, 8 in 2023-09-23T23:29:55.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[221102]: pgmap v1355: 129 pgs: 129 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 712 op/s; 28 KiB/s, 0 objects/s recovering 2023-09-23T23:29:55.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[221102]: failed to encode map e494 with expected crc 2023-09-23T23:29:55.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[221102]: failed to encode map e495 with expected crc 2023-09-23T23:29:55.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T23:29:55.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T23:29:55.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[221102]: osdmap e495: 8 total, 8 up, 8 in 2023-09-23T23:29:55.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[221102]: failed to encode map e495 with expected crc 2023-09-23T23:29:55.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[221102]: failed to encode map e495 with expected crc 2023-09-23T23:29:55.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[221102]: failed to encode map e495 with expected crc 2023-09-23T23:29:55.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[221102]: failed to encode map e495 with expected crc 2023-09-23T23:29:55.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:55 smithi110 ceph-mon[221102]: osdmap e496: 8 total, 8 up, 8 in 2023-09-23T23:29:55.244 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:55.247 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 70 16 80430 80414 71.7563 0 - 0.0137197 2023-09-23T23:29:55.276 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:55.331 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:55.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:55.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:55.337 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:29:55.337 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:29:55.434 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:55.462 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:55.555 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:55.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:55.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:55.659 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:55.688 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:55.736 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:55.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:55.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:55.840 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:29:55.868 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:55.913 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:29:55.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:55.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:29:55.989 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:29:56.017 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:56.043 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:56.086 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:56.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:56.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:56.134 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:29:56.189 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:29:56.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[212939]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[212939]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[212939]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[212939]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[212939]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[212939]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[212939]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1907534362' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:29:56.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:29:56.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1907534362' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:29:56.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[212939]: osdmap e497: 8 total, 8 up, 8 in 2023-09-23T23:29:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[221102]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[221102]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[221102]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[221102]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[221102]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[221102]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[221102]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1907534362' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:29:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:29:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1907534362' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:29:56.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:56 smithi110 ceph-mon[221102]: osdmap e497: 8 total, 8 up, 8 in 2023-09-23T23:29:56.247 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 71 16 80441 80425 70.7558 0.34375 0.0224465 0.0138465 2023-09-23T23:29:56.258 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:56.307 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:56.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:56.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:56.410 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:56 smithi138 ceph-mon[193606]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:56 smithi138 ceph-mon[193606]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:56 smithi138 ceph-mon[193606]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:56 smithi138 ceph-mon[193606]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:56 smithi138 ceph-mon[193606]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:56 smithi138 ceph-mon[193606]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:56 smithi138 ceph-mon[193606]: failed to encode map e496 with expected crc 2023-09-23T23:29:56.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:56 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1907534362' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:29:56.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:56 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:29:56.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:56 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1907534362' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:29:56.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:56 smithi138 ceph-mon[193606]: osdmap e497: 8 total, 8 up, 8 in 2023-09-23T23:29:56.483 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:56.678 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:29:57.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:57 smithi138 ceph-mon[193606]: pgmap v1358: 129 pgs: 5 peering, 124 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 41 MiB/s wr, 662 op/s 2023-09-23T23:29:57.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:57 smithi138 ceph-mon[193606]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:57 smithi138 ceph-mon[193606]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:57 smithi138 ceph-mon[193606]: 12.b starting backfill to osd.6 from (0'0,0'0] MAX to 493'5017 2023-09-23T23:29:57.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:57 smithi138 ceph-mon[193606]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:57 smithi138 ceph-mon[193606]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:57 smithi138 ceph-mon[193606]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:57 smithi138 ceph-mon[193606]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:57 smithi138 ceph-mon[193606]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[221102]: pgmap v1358: 129 pgs: 5 peering, 124 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 41 MiB/s wr, 662 op/s 2023-09-23T23:29:57.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[221102]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[221102]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[221102]: 12.b starting backfill to osd.6 from (0'0,0'0] MAX to 493'5017 2023-09-23T23:29:57.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[221102]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[221102]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[221102]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[221102]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[221102]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[212939]: pgmap v1358: 129 pgs: 5 peering, 124 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 41 MiB/s wr, 662 op/s 2023-09-23T23:29:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[212939]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[212939]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[212939]: 12.b starting backfill to osd.6 from (0'0,0'0] MAX to 493'5017 2023-09-23T23:29:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[212939]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[212939]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[212939]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[212939]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:57 smithi110 ceph-mon[212939]: failed to encode map e497 with expected crc 2023-09-23T23:29:57.205 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:29:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:29:56] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.43.0" 2023-09-23T23:29:57.247 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 72 16 81111 81095 70.3548 41.875 0.00250971 0.0142026 2023-09-23T23:29:57.574 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:57.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:57.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:57.681 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:29:57.713 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:57.758 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:29:57.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:57.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:29:57.862 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:29:57.922 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:58.046 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:58 smithi138 ceph-mon[193606]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.046 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:58 smithi138 ceph-mon[193606]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:58 smithi138 ceph-mon[193606]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:58 smithi138 ceph-mon[193606]: osdmap e498: 8 total, 8 up, 8 in 2023-09-23T23:29:58.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:58 smithi138 ceph-mon[193606]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:58 smithi138 ceph-mon[193606]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:58 smithi138 ceph-mon[193606]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:58 smithi138 ceph-mon[193606]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:58 smithi138 ceph-mon[193606]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:58 smithi138 ceph-mon[193606]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:58 smithi138 ceph-mon[193606]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:58 smithi138 ceph-mon[193606]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:58 smithi138 ceph-mon[193606]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.048 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:29:58.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[221102]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[221102]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[221102]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[221102]: osdmap e498: 8 total, 8 up, 8 in 2023-09-23T23:29:58.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[221102]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[221102]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[221102]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[221102]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[221102]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[221102]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[221102]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[221102]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[221102]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:29:58.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[212939]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[212939]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[212939]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[212939]: osdmap e498: 8 total, 8 up, 8 in 2023-09-23T23:29:58.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[212939]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[212939]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[212939]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[212939]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[212939]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[212939]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[212939]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[212939]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[212939]: failed to encode map e498 with expected crc 2023-09-23T23:29:58.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:29:58.248 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 73 16 81528 81512 69.7483 26.0625 0.00666309 0.014298 2023-09-23T23:29:58.248 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:29:58.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:58.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:29:58.352 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:29:58.392 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:58.656 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:29:58.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:58.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:29:58.766 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:29:58.834 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:29:58.840 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:58.875 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:29:58.914 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:29:58.950 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:29:58.958 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:29:58.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:58.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:29:59.003 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:29:59.043 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:29:59.075 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:29:59.078 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:29:59.112 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:29:59.119 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:29:59.119 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:29:59.119 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:29:59.120 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:29:59.120 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:29:59.120 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:29:59.120 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:29:59.120 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:29:59.125 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[212939]: pgmap v1361: 129 pgs: 1 active+recovering+undersized+remapped, 3 activating+undersized+degraded+remapped, 2 activating+undersized+remapped, 1 remapped+peering, 5 peering, 117 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 5.0 KiB/s rd, 38 MiB/s wr, 613 op/s; 9983/241008 objects degraded (4.142%); 4682/241008 objects misplaced (1.943%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:29:59.125 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:29:59.125 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[212939]: Health check failed: Degraded data redundancy: 9983/241008 objects degraded (4.142%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:29:59.125 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[212939]: failed to encode map e499 with expected crc 2023-09-23T23:29:59.125 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[212939]: failed to encode map e499 with expected crc 2023-09-23T23:29:59.125 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[212939]: osdmap e499: 8 total, 8 up, 8 in 2023-09-23T23:29:59.125 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[212939]: failed to encode map e499 with expected crc 2023-09-23T23:29:59.125 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[212939]: failed to encode map e499 with expected crc 2023-09-23T23:29:59.125 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[212939]: failed to encode map e499 with expected crc 2023-09-23T23:29:59.126 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:59.126 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1636359252' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:59.126 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[221102]: pgmap v1361: 129 pgs: 1 active+recovering+undersized+remapped, 3 activating+undersized+degraded+remapped, 2 activating+undersized+remapped, 1 remapped+peering, 5 peering, 117 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 5.0 KiB/s rd, 38 MiB/s wr, 613 op/s; 9983/241008 objects degraded (4.142%); 4682/241008 objects misplaced (1.943%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:29:59.126 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:29:59.126 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[221102]: Health check failed: Degraded data redundancy: 9983/241008 objects degraded (4.142%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:29:59.126 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[221102]: failed to encode map e499 with expected crc 2023-09-23T23:29:59.127 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[221102]: failed to encode map e499 with expected crc 2023-09-23T23:29:59.127 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[221102]: osdmap e499: 8 total, 8 up, 8 in 2023-09-23T23:29:59.127 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[221102]: failed to encode map e499 with expected crc 2023-09-23T23:29:59.127 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[221102]: failed to encode map e499 with expected crc 2023-09-23T23:29:59.127 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[221102]: failed to encode map e499 with expected crc 2023-09-23T23:29:59.127 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:59.127 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:29:59 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1636359252' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:59.130 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:59.254 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 74 16 82394 82378 69.5322 54.125 0.0180739 0.0143763 2023-09-23T23:29:59.307 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:29:59.307 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:29:59.307 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:29:59.308 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:29:59.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.308 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135599 11.e 11.62cc478e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:29:59.308 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.552343+0000", 2023-09-23T23:29:59.308 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 228.75092617600001, 2023-09-23T23:29:59.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95499033600000005, 2023-09-23T23:29:59.308 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.308 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:29:59.308 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.308 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:29:59.308 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:59.308 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135599 2023-09-23T23:29:59.309 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.309 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.309 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.309 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.309 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552343+0000", 2023-09-23T23:29:59.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.309 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.309 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.309 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.309 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552343+0000", 2023-09-23T23:29:59.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.310 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.310 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552342+0000", 2023-09-23T23:29:59.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:29:59.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.310 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.310 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552347+0000", 2023-09-23T23:29:59.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5850000000000001e-06 2023-09-23T23:29:59.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552358+0000", 2023-09-23T23:29:59.311 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0998999999999999e-05 2023-09-23T23:29:59.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552365+0000", 2023-09-23T23:29:59.311 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8630000000000002e-06 2023-09-23T23:29:59.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552426+0000", 2023-09-23T23:29:59.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0708000000000002e-05 2023-09-23T23:29:59.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.312 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552433+0000", 2023-09-23T23:29:59.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8599999999999993e-06 2023-09-23T23:29:59.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.312 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507318+0000", 2023-09-23T23:29:59.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95488429500000005 2023-09-23T23:29:59.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.313 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.313 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507334+0000", 2023-09-23T23:29:59.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5894000000000001e-05 2023-09-23T23:29:59.313 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.313 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.313 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.313 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135605 11.e 11.52f78e0e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:29:59.313 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.552409+0000", 2023-09-23T23:29:59.313 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 228.75086023700001, 2023-09-23T23:29:59.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95496278599999995, 2023-09-23T23:29:59.314 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.314 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:29:59.314 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.314 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:29:59.314 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:59.314 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135605 2023-09-23T23:29:59.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.314 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552409+0000", 2023-09-23T23:29:59.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.315 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.315 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552409+0000", 2023-09-23T23:29:59.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.315 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.315 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552409+0000", 2023-09-23T23:29:59.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:59.315 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.316 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.316 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552412+0000", 2023-09-23T23:29:59.316 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7199999999999998e-06 2023-09-23T23:29:59.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.316 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.316 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552417+0000", 2023-09-23T23:29:59.316 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.023e-06 2023-09-23T23:29:59.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552419+0000", 2023-09-23T23:29:59.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7319999999999998e-06 2023-09-23T23:29:59.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.317 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552464+0000", 2023-09-23T23:29:59.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5096000000000003e-05 2023-09-23T23:29:59.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.317 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552472+0000", 2023-09-23T23:29:59.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5390000000000002e-06 2023-09-23T23:29:59.318 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.318 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.318 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.318 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507362+0000", 2023-09-23T23:29:59.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95488972999999999 2023-09-23T23:29:59.318 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.318 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.318 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.318 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507372+0000", 2023-09-23T23:29:59.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0360000000000001e-05 2023-09-23T23:29:59.318 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.318 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.319 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.319 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135609 11.e 11.9615c3ce (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:29:59.319 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.555799+0000", 2023-09-23T23:29:59.319 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 228.74747060199999, 2023-09-23T23:29:59.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95161954599999998, 2023-09-23T23:29:59.319 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.319 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:29:59.319 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.319 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:29:59.320 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:59.320 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135609 2023-09-23T23:29:59.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.320 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.555799+0000", 2023-09-23T23:29:59.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.555799+0000", 2023-09-23T23:29:59.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.555798+0000", 2023-09-23T23:29:59.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:29:59.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.555801+0000", 2023-09-23T23:29:59.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4709999999999999e-06 2023-09-23T23:29:59.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.555809+0000", 2023-09-23T23:29:59.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6760000000000007e-06 2023-09-23T23:29:59.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.555814+0000", 2023-09-23T23:29:59.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3560000000000003e-06 2023-09-23T23:29:59.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.323 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.555850+0000", 2023-09-23T23:29:59.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5987e-05 2023-09-23T23:29:59.323 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.323 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.323 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.323 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.555858+0000", 2023-09-23T23:29:59.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4279999999999996e-06 2023-09-23T23:29:59.323 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.323 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.323 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.323 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507405+0000", 2023-09-23T23:29:59.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.951547437 2023-09-23T23:29:59.323 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.324 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.324 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.324 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507418+0000", 2023-09-23T23:29:59.324 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3052e-05 2023-09-23T23:29:59.324 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.324 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.324 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.324 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.324 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.324 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135623 11.e 11.9bd717e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:29:59.324 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.567262+0000", 2023-09-23T23:29:59.324 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 228.73600761200001, 2023-09-23T23:29:59.325 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.940189689, 2023-09-23T23:29:59.325 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.325 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:29:59.325 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.325 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:29:59.325 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:59.325 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135623 2023-09-23T23:29:59.325 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.325 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.325 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.325 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.325 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567262+0000", 2023-09-23T23:29:59.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.326 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567262+0000", 2023-09-23T23:29:59.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.326 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567261+0000", 2023-09-23T23:29:59.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:29:59.326 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567265+0000", 2023-09-23T23:29:59.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.69e-06 2023-09-23T23:29:59.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567276+0000", 2023-09-23T23:29:59.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1178999999999999e-05 2023-09-23T23:29:59.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567281+0000", 2023-09-23T23:29:59.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9710000000000003e-06 2023-09-23T23:29:59.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567329+0000", 2023-09-23T23:29:59.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7754999999999999e-05 2023-09-23T23:29:59.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567335+0000", 2023-09-23T23:29:59.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5119999999999999e-06 2023-09-23T23:29:59.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507444+0000", 2023-09-23T23:29:59.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94010914000000001 2023-09-23T23:29:59.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.331 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.331 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507452+0000", 2023-09-23T23:29:59.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7479999999999993e-06 2023-09-23T23:29:59.331 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.331 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.331 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.331 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135624 11.e 11.c5c7af9e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:29:59.331 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.567292+0000", 2023-09-23T23:29:59.331 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 228.73597736100001, 2023-09-23T23:29:59.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94020217299999997, 2023-09-23T23:29:59.332 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.332 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:29:59.332 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.332 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:29:59.332 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:59.332 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135624 2023-09-23T23:29:59.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.332 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567292+0000", 2023-09-23T23:29:59.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567292+0000", 2023-09-23T23:29:59.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567292+0000", 2023-09-23T23:29:59.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:59.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567294+0000", 2023-09-23T23:29:59.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4279999999999999e-06 2023-09-23T23:29:59.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567307+0000", 2023-09-23T23:29:59.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2911999999999999e-05 2023-09-23T23:29:59.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567312+0000", 2023-09-23T23:29:59.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.527e-06 2023-09-23T23:29:59.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567373+0000", 2023-09-23T23:29:59.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1474000000000001e-05 2023-09-23T23:29:59.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567377+0000", 2023-09-23T23:29:59.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5650000000000002e-06 2023-09-23T23:29:59.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507471+0000", 2023-09-23T23:29:59.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94009431600000004 2023-09-23T23:29:59.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507494+0000", 2023-09-23T23:29:59.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3319000000000002e-05 2023-09-23T23:29:59.336 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.336 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.336 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.337 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135642 11.e 11.2e5be21e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:29:59.337 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.588167+0000", 2023-09-23T23:29:59.337 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 228.71510201699999, 2023-09-23T23:29:59.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91935951400000004, 2023-09-23T23:29:59.337 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.337 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:29:59.337 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.337 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:29:59.337 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:59.337 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135642 2023-09-23T23:29:59.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.337 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.588167+0000", 2023-09-23T23:29:59.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.588167+0000", 2023-09-23T23:29:59.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.588166+0000", 2023-09-23T23:29:59.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:29:59.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.588172+0000", 2023-09-23T23:29:59.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9109999999999998e-06 2023-09-23T23:29:59.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.588190+0000", 2023-09-23T23:29:59.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7731999999999999e-05 2023-09-23T23:29:59.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.340 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.588197+0000", 2023-09-23T23:29:59.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1459999999999999e-06 2023-09-23T23:29:59.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.340 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.588252+0000", 2023-09-23T23:29:59.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5120000000000001e-05 2023-09-23T23:29:59.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.588262+0000", 2023-09-23T23:29:59.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0597e-05 2023-09-23T23:29:59.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507517+0000", 2023-09-23T23:29:59.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.919254448 2023-09-23T23:29:59.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507527+0000", 2023-09-23T23:29:59.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0097e-05 2023-09-23T23:29:59.342 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.342 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.342 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.342 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135646 11.e 11.7000afde (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:29:59.342 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.590806+0000", 2023-09-23T23:29:59.342 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 228.71246370399999, 2023-09-23T23:29:59.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.916768315, 2023-09-23T23:29:59.342 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.342 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:29:59.343 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.343 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:29:59.343 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:59.343 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135646 2023-09-23T23:29:59.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.343 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.590806+0000", 2023-09-23T23:29:59.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.590806+0000", 2023-09-23T23:29:59.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.590805+0000", 2023-09-23T23:29:59.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:59.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.590809+0000", 2023-09-23T23:29:59.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2549999999999999e-06 2023-09-23T23:29:59.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.590820+0000", 2023-09-23T23:29:59.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0553999999999999e-05 2023-09-23T23:29:59.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.590824+0000", 2023-09-23T23:29:59.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5569999999999997e-06 2023-09-23T23:29:59.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.590861+0000", 2023-09-23T23:29:59.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6853000000000001e-05 2023-09-23T23:29:59.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.590866+0000", 2023-09-23T23:29:59.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3669999999999999e-06 2023-09-23T23:29:59.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507562+0000", 2023-09-23T23:29:59.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91669663300000004 2023-09-23T23:29:59.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507574+0000", 2023-09-23T23:29:59.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1783999999999999e-05 2023-09-23T23:29:59.347 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.347 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.347 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.347 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135676 11.e 11.5cbeade (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:29:59.347 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.688594+0000", 2023-09-23T23:29:59.348 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 228.614675613, 2023-09-23T23:29:59.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.81901241999999996, 2023-09-23T23:29:59.348 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.348 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:29:59.348 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.348 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:29:59.348 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:59.348 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135676 2023-09-23T23:29:59.348 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.348 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.348 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.348 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688594+0000", 2023-09-23T23:29:59.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688594+0000", 2023-09-23T23:29:59.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688593+0000", 2023-09-23T23:29:59.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:29:59.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.350 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.350 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688602+0000", 2023-09-23T23:29:59.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9069999999999996e-06 2023-09-23T23:29:59.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.350 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.350 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688617+0000", 2023-09-23T23:29:59.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5798e-05 2023-09-23T23:29:59.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.350 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688641+0000", 2023-09-23T23:29:59.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3781000000000002e-05 2023-09-23T23:29:59.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688682+0000", 2023-09-23T23:29:59.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1247e-05 2023-09-23T23:29:59.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688693+0000", 2023-09-23T23:29:59.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0159999999999999e-05 2023-09-23T23:29:59.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507599+0000", 2023-09-23T23:29:59.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.81890648200000005 2023-09-23T23:29:59.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507606+0000", 2023-09-23T23:29:59.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2289999999999997e-06 2023-09-23T23:29:59.352 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.352 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.352 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.353 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135677 11.e 11.7aa8673e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:29:59.353 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.688904+0000", 2023-09-23T23:29:59.353 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 228.61436568299999, 2023-09-23T23:29:59.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.81872850799999997, 2023-09-23T23:29:59.353 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.353 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:29:59.353 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.353 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:29:59.353 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:59.353 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135677 2023-09-23T23:29:59.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.354 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688904+0000", 2023-09-23T23:29:59.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688904+0000", 2023-09-23T23:29:59.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688903+0000", 2023-09-23T23:29:59.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:29:59.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688906+0000", 2023-09-23T23:29:59.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.526e-06 2023-09-23T23:29:59.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688912+0000", 2023-09-23T23:29:59.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3500000000000002e-06 2023-09-23T23:29:59.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688915+0000", 2023-09-23T23:29:59.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.202e-06 2023-09-23T23:29:59.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688948+0000", 2023-09-23T23:29:59.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2647000000000001e-05 2023-09-23T23:29:59.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688954+0000", 2023-09-23T23:29:59.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8279999999999999e-06 2023-09-23T23:29:59.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507625+0000", 2023-09-23T23:29:59.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.81867117700000003 2023-09-23T23:29:59.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.358 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507632+0000", 2023-09-23T23:29:59.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1779999999999997e-06 2023-09-23T23:29:59.358 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.358 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.358 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.358 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.358 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:154629 11.17 11:ed0449d2:::benchmark_data_smithi110_433746_object48749:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-09-23T23:29:59.358 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:43.186486+0000", 2023-09-23T23:29:59.358 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 196.11678342900001, 2023-09-23T23:29:59.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0259201550000001, 2023-09-23T23:29:59.358 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.359 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:59.359 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.359 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:29:59.359 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:59.359 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154629 2023-09-23T23:29:59.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.359 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.359 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.359 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.359 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186486+0000", 2023-09-23T23:29:59.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186486+0000", 2023-09-23T23:29:59.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186485+0000", 2023-09-23T23:29:59.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:29:59.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186490+0000", 2023-09-23T23:29:59.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5589999999999999e-06 2023-09-23T23:29:59.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186498+0000", 2023-09-23T23:29:59.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4509999999999992e-06 2023-09-23T23:29:59.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186504+0000", 2023-09-23T23:29:59.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3369999999999996e-06 2023-09-23T23:29:59.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186667+0000", 2023-09-23T23:29:59.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016259099999999999 2023-09-23T23:29:59.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186678+0000", 2023-09-23T23:29:59.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1362999999999999e-05 2023-09-23T23:29:59.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.183982+0000", 2023-09-23T23:29:59.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99730399700000005 2023-09-23T23:29:59.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:29:59.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.184074+0000", 2023-09-23T23:29:59.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2168000000000003e-05 2023-09-23T23:29:59.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:29:59.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.184404+0000", 2023-09-23T23:29:59.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00032955400000000002 2023-09-23T23:29:59.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:29:59.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.191347+0000", 2023-09-23T23:29:59.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069433059999999998 2023-09-23T23:29:59.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212265+0000", 2023-09-23T23:29:59.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020918125999999999 2023-09-23T23:29:59.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.365 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212315+0000", 2023-09-23T23:29:59.365 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9078e-05 2023-09-23T23:29:59.365 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.365 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:29:59.365 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212362+0000", 2023-09-23T23:29:59.365 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7893999999999999e-05 2023-09-23T23:29:59.365 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.365 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.365 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212406+0000", 2023-09-23T23:29:59.365 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3766000000000001e-05 2023-09-23T23:29:59.366 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.366 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.366 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.366 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.366 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.366 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:154670 11.17 11:ef11bd65:::benchmark_data_smithi110_433746_object48790:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-09-23T23:29:59.366 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:43.271391+0000", 2023-09-23T23:29:59.366 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 196.03187830900001, 2023-09-23T23:29:59.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94129660699999995, 2023-09-23T23:29:59.366 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.366 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:59.366 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.367 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:29:59.367 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:59.367 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154670 2023-09-23T23:29:59.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.367 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.271391+0000", 2023-09-23T23:29:59.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.271391+0000", 2023-09-23T23:29:59.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.271390+0000", 2023-09-23T23:29:59.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:29:59.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.271396+0000", 2023-09-23T23:29:59.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2990000000000002e-06 2023-09-23T23:29:59.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.271407+0000", 2023-09-23T23:29:59.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1076e-05 2023-09-23T23:29:59.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.271413+0000", 2023-09-23T23:29:59.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.164e-06 2023-09-23T23:29:59.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.271456+0000", 2023-09-23T23:29:59.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3331999999999998e-05 2023-09-23T23:29:59.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.370 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.271464+0000", 2023-09-23T23:29:59.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3329999999999997e-06 2023-09-23T23:29:59.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.370 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.186310+0000", 2023-09-23T23:29:59.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91484549599999998 2023-09-23T23:29:59.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:29:59.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.186435+0000", 2023-09-23T23:29:59.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012552599999999999 2023-09-23T23:29:59.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:29:59.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.186732+0000", 2023-09-23T23:29:59.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00029633900000000001 2023-09-23T23:29:59.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:29:59.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.195220+0000", 2023-09-23T23:29:59.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0084882880000000001 2023-09-23T23:29:59.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212466+0000", 2023-09-23T23:29:59.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017246396000000001 2023-09-23T23:29:59.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212598+0000", 2023-09-23T23:29:59.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013127200000000001 2023-09-23T23:29:59.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:29:59.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212642+0000", 2023-09-23T23:29:59.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4345000000000003e-05 2023-09-23T23:29:59.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212688+0000", 2023-09-23T23:29:59.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5893000000000003e-05 2023-09-23T23:29:59.373 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.373 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.373 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.374 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:154683 11.17 11:eb2dae53:::benchmark_data_smithi110_433746_object48803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-09-23T23:29:59.374 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:43.299814+0000", 2023-09-23T23:29:59.374 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 196.00345587300001, 2023-09-23T23:29:59.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91302723399999997, 2023-09-23T23:29:59.374 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.374 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:59.374 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.374 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:29:59.374 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:59.374 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154683 2023-09-23T23:29:59.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.375 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.299814+0000", 2023-09-23T23:29:59.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.299814+0000", 2023-09-23T23:29:59.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.299812+0000", 2023-09-23T23:29:59.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:29:59.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.299819+0000", 2023-09-23T23:29:59.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2369999999999997e-06 2023-09-23T23:29:59.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.299832+0000", 2023-09-23T23:29:59.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3730999999999999e-05 2023-09-23T23:29:59.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.299838+0000", 2023-09-23T23:29:59.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1850000000000003e-06 2023-09-23T23:29:59.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.299882+0000", 2023-09-23T23:29:59.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4895000000000003e-05 2023-09-23T23:29:59.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.299890+0000", 2023-09-23T23:29:59.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0229999999999992e-06 2023-09-23T23:29:59.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.188239+0000", 2023-09-23T23:29:59.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88834824000000001 2023-09-23T23:29:59.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:29:59.379 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.188329+0000", 2023-09-23T23:29:59.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0667000000000005e-05 2023-09-23T23:29:59.379 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.379 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:29:59.379 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.188539+0000", 2023-09-23T23:29:59.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000210125 2023-09-23T23:29:59.379 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.379 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:29:59.379 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.195226+0000", 2023-09-23T23:29:59.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066862850000000001 2023-09-23T23:29:59.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212498+0000", 2023-09-23T23:29:59.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017272032999999999 2023-09-23T23:29:59.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212765+0000", 2023-09-23T23:29:59.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000267283 2023-09-23T23:29:59.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:29:59.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212808+0000", 2023-09-23T23:29:59.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3245999999999999e-05 2023-09-23T23:29:59.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212841+0000", 2023-09-23T23:29:59.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2526e-05 2023-09-23T23:29:59.381 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.381 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.381 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.382 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:154729 11.17 11:eda0dbdd:::benchmark_data_smithi110_433746_object48849:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-09-23T23:29:59.382 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:43.371059+0000", 2023-09-23T23:29:59.382 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 195.93220999900001, 2023-09-23T23:29:59.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.84193335300000005, 2023-09-23T23:29:59.382 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.382 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:59.382 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.382 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:29:59.382 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:59.382 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154729 2023-09-23T23:29:59.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.382 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.371059+0000", 2023-09-23T23:29:59.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.371059+0000", 2023-09-23T23:29:59.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.371058+0000", 2023-09-23T23:29:59.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:29:59.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.384 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.371067+0000", 2023-09-23T23:29:59.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1120000000000002e-06 2023-09-23T23:29:59.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.384 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.371090+0000", 2023-09-23T23:29:59.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2761000000000001e-05 2023-09-23T23:29:59.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.371098+0000", 2023-09-23T23:29:59.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5799999999999992e-06 2023-09-23T23:29:59.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.371168+0000", 2023-09-23T23:29:59.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9419999999999996e-05 2023-09-23T23:29:59.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.386 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.386 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.371181+0000", 2023-09-23T23:29:59.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3069e-05 2023-09-23T23:29:59.386 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.386 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.386 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.189791+0000", 2023-09-23T23:29:59.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.81860996200000002 2023-09-23T23:29:59.386 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.386 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:29:59.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.189886+0000", 2023-09-23T23:29:59.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4759999999999994e-05 2023-09-23T23:29:59.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:29:59.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.190063+0000", 2023-09-23T23:29:59.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017765500000000001 2023-09-23T23:29:59.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:29:59.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.198958+0000", 2023-09-23T23:29:59.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088949460000000008 2023-09-23T23:29:59.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212729+0000", 2023-09-23T23:29:59.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013770469 2023-09-23T23:29:59.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212906+0000", 2023-09-23T23:29:59.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000177858 2023-09-23T23:29:59.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:29:59.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212958+0000", 2023-09-23T23:29:59.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1838000000000001e-05 2023-09-23T23:29:59.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212993+0000", 2023-09-23T23:29:59.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4505999999999998e-05 2023-09-23T23:29:59.389 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.389 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.389 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.390 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:154736 11.17 11:eac942b3:::benchmark_data_smithi110_433746_object48856:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-09-23T23:29:59.390 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:43.386205+0000", 2023-09-23T23:29:59.390 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 195.91706418499999, 2023-09-23T23:29:59.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.82686456699999999, 2023-09-23T23:29:59.390 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.390 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:59.390 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.390 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:29:59.390 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:59.390 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154736 2023-09-23T23:29:59.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.390 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.386205+0000", 2023-09-23T23:29:59.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.386205+0000", 2023-09-23T23:29:59.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.386204+0000", 2023-09-23T23:29:59.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:29:59.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.386211+0000", 2023-09-23T23:29:59.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4570000000000002e-06 2023-09-23T23:29:59.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.386227+0000", 2023-09-23T23:29:59.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6379e-05 2023-09-23T23:29:59.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.393 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.393 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.386234+0000", 2023-09-23T23:29:59.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0219999999999999e-06 2023-09-23T23:29:59.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.393 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.393 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.386291+0000", 2023-09-23T23:29:59.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6607999999999997e-05 2023-09-23T23:29:59.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.393 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.386301+0000", 2023-09-23T23:29:59.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0390999999999999e-05 2023-09-23T23:29:59.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.191429+0000", 2023-09-23T23:29:59.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.80512771000000005 2023-09-23T23:29:59.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:29:59.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.191512+0000", 2023-09-23T23:29:59.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3331000000000006e-05 2023-09-23T23:29:59.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:29:59.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.191721+0000", 2023-09-23T23:29:59.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000208612 2023-09-23T23:29:59.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:29:59.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.198970+0000", 2023-09-23T23:29:59.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0072496160000000004 2023-09-23T23:29:59.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212877+0000", 2023-09-23T23:29:59.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013906814999999999 2023-09-23T23:29:59.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213025+0000", 2023-09-23T23:29:59.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014817300000000001 2023-09-23T23:29:59.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:29:59.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213048+0000", 2023-09-23T23:29:59.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2549e-05 2023-09-23T23:29:59.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.397 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.397 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213070+0000", 2023-09-23T23:29:59.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2075000000000001e-05 2023-09-23T23:29:59.397 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.397 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.397 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.397 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:154754 11.17 11:eabc5fa3:::benchmark_data_smithi110_433746_object48874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-09-23T23:29:59.397 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:43.412873+0000", 2023-09-23T23:29:59.398 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 195.890396831, 2023-09-23T23:29:59.398 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.80035215800000004, 2023-09-23T23:29:59.398 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.398 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:59.398 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.398 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:29:59.398 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:59.398 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154754 2023-09-23T23:29:59.398 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.398 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.398 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.412873+0000", 2023-09-23T23:29:59.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.412873+0000", 2023-09-23T23:29:59.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.412872+0000", 2023-09-23T23:29:59.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:29:59.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.412877+0000", 2023-09-23T23:29:59.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3979999999999998e-06 2023-09-23T23:29:59.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.412895+0000", 2023-09-23T23:29:59.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8400999999999999e-05 2023-09-23T23:29:59.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.412903+0000", 2023-09-23T23:29:59.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.469e-06 2023-09-23T23:29:59.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.412954+0000", 2023-09-23T23:29:59.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0924000000000002e-05 2023-09-23T23:29:59.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.412963+0000", 2023-09-23T23:29:59.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5389999999999996e-06 2023-09-23T23:29:59.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.193311+0000", 2023-09-23T23:29:59.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.78034795400000001 2023-09-23T23:29:59.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:29:59.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.193430+0000", 2023-09-23T23:29:59.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118532 2023-09-23T23:29:59.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:29:59.403 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.193713+0000", 2023-09-23T23:29:59.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00028307700000000001 2023-09-23T23:29:59.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.403 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:29:59.403 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.202618+0000", 2023-09-23T23:29:59.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0089055019999999992 2023-09-23T23:29:59.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.404 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213107+0000", 2023-09-23T23:29:59.404 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010489002000000001 2023-09-23T23:29:59.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.404 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213158+0000", 2023-09-23T23:29:59.404 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0319999999999999e-05 2023-09-23T23:29:59.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:29:59.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213195+0000", 2023-09-23T23:29:59.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6899e-05 2023-09-23T23:29:59.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213225+0000", 2023-09-23T23:29:59.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0266000000000001e-05 2023-09-23T23:29:59.405 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.405 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.405 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.406 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:154821 11.17 11:ec5921e4:::benchmark_data_smithi110_433746_object48941:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-09-23T23:29:59.406 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:43.505849+0000", 2023-09-23T23:29:59.406 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 195.79742085699999, 2023-09-23T23:29:59.406 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.70750497800000001, 2023-09-23T23:29:59.406 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.406 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:59.406 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.406 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:29:59.406 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:59.406 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154821 2023-09-23T23:29:59.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.407 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.505849+0000", 2023-09-23T23:29:59.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.505849+0000", 2023-09-23T23:29:59.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.505848+0000", 2023-09-23T23:29:59.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:29:59.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.505854+0000", 2023-09-23T23:29:59.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1879999999999999e-06 2023-09-23T23:29:59.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.505865+0000", 2023-09-23T23:29:59.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1515000000000001e-05 2023-09-23T23:29:59.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.505873+0000", 2023-09-23T23:29:59.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6240000000000002e-06 2023-09-23T23:29:59.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.505925+0000", 2023-09-23T23:29:59.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2392999999999998e-05 2023-09-23T23:29:59.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.410 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.505932+0000", 2023-09-23T23:29:59.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4980000000000003e-06 2023-09-23T23:29:59.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.410 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.195272+0000", 2023-09-23T23:29:59.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.68934017000000003 2023-09-23T23:29:59.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:29:59.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.195344+0000", 2023-09-23T23:29:59.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2340000000000002e-05 2023-09-23T23:29:59.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.411 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:29:59.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.195571+0000", 2023-09-23T23:29:59.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00022642100000000001 2023-09-23T23:29:59.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.411 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:29:59.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.202628+0000", 2023-09-23T23:29:59.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0070578070000000001 2023-09-23T23:29:59.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.412 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.412 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213256+0000", 2023-09-23T23:29:59.412 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010627663000000001 2023-09-23T23:29:59.412 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.412 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.412 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213289+0000", 2023-09-23T23:29:59.412 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2437000000000001e-05 2023-09-23T23:29:59.412 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.412 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:29:59.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213327+0000", 2023-09-23T23:29:59.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7960999999999998e-05 2023-09-23T23:29:59.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213354+0000", 2023-09-23T23:29:59.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7080000000000002e-05 2023-09-23T23:29:59.413 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.413 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.413 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.413 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:154862 11.17 11:eb194d98:::benchmark_data_smithi110_433746_object48982:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-09-23T23:29:59.414 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:43.560853+0000", 2023-09-23T23:29:59.414 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 195.742416131, 2023-09-23T23:29:59.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.65272713599999999, 2023-09-23T23:29:59.414 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.414 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:59.414 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.414 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:29:59.414 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:29:59.414 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154862 2023-09-23T23:29:59.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.414 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.560853+0000", 2023-09-23T23:29:59.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.560853+0000", 2023-09-23T23:29:59.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.560852+0000", 2023-09-23T23:29:59.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:29:59.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.416 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.560859+0000", 2023-09-23T23:29:59.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8210000000000004e-06 2023-09-23T23:29:59.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.416 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.560871+0000", 2023-09-23T23:29:59.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.207e-05 2023-09-23T23:29:59.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.560878+0000", 2023-09-23T23:29:59.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1840000000000002e-06 2023-09-23T23:29:59.417 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.560931+0000", 2023-09-23T23:29:59.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3090000000000002e-05 2023-09-23T23:29:59.417 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.560940+0000", 2023-09-23T23:29:59.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.38e-06 2023-09-23T23:29:59.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.197385+0000", 2023-09-23T23:29:59.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.63644482400000002 2023-09-23T23:29:59.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:29:59.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.197491+0000", 2023-09-23T23:29:59.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105368 2023-09-23T23:29:59.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:29:59.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.197692+0000", 2023-09-23T23:29:59.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000201447 2023-09-23T23:29:59.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:29:59.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.209644+0000", 2023-09-23T23:29:59.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011951463000000001 2023-09-23T23:29:59.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.420 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213416+0000", 2023-09-23T23:29:59.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037722430000000002 2023-09-23T23:29:59.420 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.420 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213490+0000", 2023-09-23T23:29:59.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4005999999999996e-05 2023-09-23T23:29:59.420 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:29:59.420 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213542+0000", 2023-09-23T23:29:59.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2327999999999998e-05 2023-09-23T23:29:59.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.421 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.421 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213580+0000", 2023-09-23T23:29:59.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8333e-05 2023-09-23T23:29:59.421 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.421 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.421 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.421 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:73408 12.10 12:0ed56545:::benchmark_data_smithi110_491193_object73407:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T23:29:59.421 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:29:44.726507+0000", 2023-09-23T23:29:59.422 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 14.576762137999999, 2023-09-23T23:29:59.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95402578900000001, 2023-09-23T23:29:59.422 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.422 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:29:59.422 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.422 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:29:59.422 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:29:59.422 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 73408 2023-09-23T23:29:59.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.422 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726507+0000", 2023-09-23T23:29:59.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726507+0000", 2023-09-23T23:29:59.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726505+0000", 2023-09-23T23:29:59.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:29:59.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726548+0000", 2023-09-23T23:29:59.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2936999999999999e-05 2023-09-23T23:29:59.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726565+0000", 2023-09-23T23:29:59.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6705e-05 2023-09-23T23:29:59.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726572+0000", 2023-09-23T23:29:59.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9500000000000004e-06 2023-09-23T23:29:59.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.751276+0000", 2023-09-23T23:29:59.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.024704454000000001 2023-09-23T23:29:59.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.751287+0000", 2023-09-23T23:29:59.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0243e-05 2023-09-23T23:29:59.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.661352+0000", 2023-09-23T23:29:59.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91006557200000004 2023-09-23T23:29:59.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-09-23T23:29:59.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.661964+0000", 2023-09-23T23:29:59.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00061125199999999995 2023-09-23T23:29:59.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.427 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.427 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.680469+0000", 2023-09-23T23:29:59.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018505864 2023-09-23T23:29:59.427 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.427 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.427 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.680533+0000", 2023-09-23T23:29:59.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3732999999999994e-05 2023-09-23T23:29:59.427 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.427 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.427 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.428 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.428 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:73413 12.10 12:081290b7:::benchmark_data_smithi110_491193_object73412:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T23:29:59.428 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:29:44.907437+0000", 2023-09-23T23:29:59.428 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 14.395832971000001, 2023-09-23T23:29:59.428 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.773219714, 2023-09-23T23:29:59.428 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.428 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:59.428 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.428 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:29:59.428 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:29:59.428 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 73413 2023-09-23T23:29:59.428 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.429 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.907437+0000", 2023-09-23T23:29:59.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.907437+0000", 2023-09-23T23:29:59.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.430 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.430 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.907434+0000", 2023-09-23T23:29:59.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999971 2023-09-23T23:29:59.430 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.430 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.430 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.430 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.907500+0000", 2023-09-23T23:29:59.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5956000000000003e-05 2023-09-23T23:29:59.430 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.430 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.430 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.430 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.907540+0000", 2023-09-23T23:29:59.431 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9972000000000003e-05 2023-09-23T23:29:59.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.431 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.431 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.431 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.907554+0000", 2023-09-23T23:29:59.431 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.454e-05 2023-09-23T23:29:59.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.431 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.431 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.431 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.907652+0000", 2023-09-23T23:29:59.431 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7478000000000004e-05 2023-09-23T23:29:59.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.907667+0000", 2023-09-23T23:29:59.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.509e-05 2023-09-23T23:29:59.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.662209+0000", 2023-09-23T23:29:59.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.75454222999999998 2023-09-23T23:29:59.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:29:59.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.662307+0000", 2023-09-23T23:29:59.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7966000000000004e-05 2023-09-23T23:29:59.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.433 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-09-23T23:29:59.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.662419+0000", 2023-09-23T23:29:59.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112414 2023-09-23T23:29:59.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.433 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:29:59.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.663547+0000", 2023-09-23T23:29:59.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001127489 2023-09-23T23:29:59.434 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.664151+0000", 2023-09-23T23:29:59.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000604431 2023-09-23T23:29:59.434 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.680564+0000", 2023-09-23T23:29:59.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016412699999999999 2023-09-23T23:29:59.434 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.435 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:29:59.435 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.680598+0000", 2023-09-23T23:29:59.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4001999999999997e-05 2023-09-23T23:29:59.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.435 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.435 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.680656+0000", 2023-09-23T23:29:59.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8335999999999998e-05 2023-09-23T23:29:59.435 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.435 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.435 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.436 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.436 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:73422 12.0 12:0273ea89:::benchmark_data_smithi110_491193_object73421:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T23:29:59.436 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:29:44.939207+0000", 2023-09-23T23:29:59.436 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 14.364062245, 2023-09-23T23:29:59.436 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.70304230300000004, 2023-09-23T23:29:59.436 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:29:59.436 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:29:59.436 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:29:59.436 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:29:59.436 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:29:59.436 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 73422 2023-09-23T23:29:59.436 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.437 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:29:59.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:29:59.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.939207+0000", 2023-09-23T23:29:59.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:29:59.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.939207+0000", 2023-09-23T23:29:59.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:29:59.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:29:59.438 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.939206+0000", 2023-09-23T23:29:59.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:29:59.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.438 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.438 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:29:59.438 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.939265+0000", 2023-09-23T23:29:59.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9126000000000003e-05 2023-09-23T23:29:59.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.438 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.438 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:29:59.438 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.939274+0000", 2023-09-23T23:29:59.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0759999999999992e-06 2023-09-23T23:29:59.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:29:59.439 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.939280+0000", 2023-09-23T23:29:59.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4280000000000001e-06 2023-09-23T23:29:59.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.439 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.939318+0000", 2023-09-23T23:29:59.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8093000000000003e-05 2023-09-23T23:29:59.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:29:59.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.939326+0000", 2023-09-23T23:29:59.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3889999999999999e-06 2023-09-23T23:29:59.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:29:59.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.634664+0000", 2023-09-23T23:29:59.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.69533816500000001 2023-09-23T23:29:59.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:29:59.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.634772+0000", 2023-09-23T23:29:59.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108284 2023-09-23T23:29:59.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-09-23T23:29:59.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.634874+0000", 2023-09-23T23:29:59.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000102043 2023-09-23T23:29:59.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.636816+0000", 2023-09-23T23:29:59.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001941938 2023-09-23T23:29:59.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:29:59.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.638744+0000", 2023-09-23T23:29:59.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001928213 2023-09-23T23:29:59.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:29:59.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.642166+0000", 2023-09-23T23:29:59.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003421267 2023-09-23T23:29:59.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:29:59.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.642212+0000", 2023-09-23T23:29:59.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6459999999999999e-05 2023-09-23T23:29:59.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:29:59.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:29:59.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:29:59.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.642250+0000", 2023-09-23T23:29:59.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7496999999999997e-05 2023-09-23T23:29:59.443 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.443 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.443 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.443 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:29:59.444 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:29:59.444 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:29:59.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:59 smithi138 ceph-mon[193606]: pgmap v1361: 129 pgs: 1 active+recovering+undersized+remapped, 3 activating+undersized+degraded+remapped, 2 activating+undersized+remapped, 1 remapped+peering, 5 peering, 117 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 5.0 KiB/s rd, 38 MiB/s wr, 613 op/s; 9983/241008 objects degraded (4.142%); 4682/241008 objects misplaced (1.943%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:29:59.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:59 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:29:59.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:59 smithi138 ceph-mon[193606]: Health check failed: Degraded data redundancy: 9983/241008 objects degraded (4.142%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T23:29:59.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:59 smithi138 ceph-mon[193606]: failed to encode map e499 with expected crc 2023-09-23T23:29:59.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:59 smithi138 ceph-mon[193606]: failed to encode map e499 with expected crc 2023-09-23T23:29:59.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:59 smithi138 ceph-mon[193606]: osdmap e499: 8 total, 8 up, 8 in 2023-09-23T23:29:59.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:59 smithi138 ceph-mon[193606]: failed to encode map e499 with expected crc 2023-09-23T23:29:59.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:59 smithi138 ceph-mon[193606]: failed to encode map e499 with expected crc 2023-09-23T23:29:59.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:59 smithi138 ceph-mon[193606]: failed to encode map e499 with expected crc 2023-09-23T23:29:59.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:59 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:29:59.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:29:59 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1636359252' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:29:59.646 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:29:59.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:59.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:29:59.755 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:29:59.845 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:29:59.974 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:29:59.977 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 7, 2, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:29:59.978 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:29:59.978 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T23:29:59.978 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T23:29:59.978 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T23:29:59.978 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T23:29:59.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:29:59.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:30:00.083 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:30:00.148 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:00 smithi110 ceph-mon[212939]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Reduced data availability: 1 pg peering; Degraded data redundancy: 9983/241008 objects degraded (4.142%), 3 pgs degraded 2023-09-23T23:30:00.149 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:00 smithi110 ceph-mon[212939]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T23:30:00.149 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:00 smithi110 ceph-mon[212939]: [WRN] PG_AVAILABILITY: Reduced data availability: 1 pg peering 2023-09-23T23:30:00.149 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:00 smithi110 ceph-mon[212939]: pg 1.6 is stuck peering for 3m, current state remapped+peering, last acting [0,3] 2023-09-23T23:30:00.149 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:00 smithi110 ceph-mon[212939]: [WRN] PG_DEGRADED: Degraded data redundancy: 9983/241008 objects degraded (4.142%), 3 pgs degraded 2023-09-23T23:30:00.149 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:00 smithi110 ceph-mon[212939]: pg 12.9 is activating+undersized+degraded+remapped, acting [2,3] 2023-09-23T23:30:00.149 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:00 smithi110 ceph-mon[212939]: pg 12.f is activating+undersized+degraded+remapped, acting [2,5] 2023-09-23T23:30:00.149 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:00 smithi110 ceph-mon[212939]: pg 12.19 is activating+undersized+degraded+remapped, acting [2,3] 2023-09-23T23:30:00.150 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:00 smithi110 ceph-mon[221102]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Reduced data availability: 1 pg peering; Degraded data redundancy: 9983/241008 objects degraded (4.142%), 3 pgs degraded 2023-09-23T23:30:00.150 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:00 smithi110 ceph-mon[221102]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T23:30:00.150 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:00 smithi110 ceph-mon[221102]: [WRN] PG_AVAILABILITY: Reduced data availability: 1 pg peering 2023-09-23T23:30:00.150 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:00 smithi110 ceph-mon[221102]: pg 1.6 is stuck peering for 3m, current state remapped+peering, last acting [0,3] 2023-09-23T23:30:00.150 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:00 smithi110 ceph-mon[221102]: [WRN] PG_DEGRADED: Degraded data redundancy: 9983/241008 objects degraded (4.142%), 3 pgs degraded 2023-09-23T23:30:00.150 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:00 smithi110 ceph-mon[221102]: pg 12.9 is activating+undersized+degraded+remapped, acting [2,3] 2023-09-23T23:30:00.150 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:00 smithi110 ceph-mon[221102]: pg 12.f is activating+undersized+degraded+remapped, acting [2,5] 2023-09-23T23:30:00.150 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:00 smithi110 ceph-mon[221102]: pg 12.19 is activating+undersized+degraded+remapped, acting [2,3] 2023-09-23T23:30:00.151 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:00.196 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:30:00.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:00.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:30:00.253 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 75 16 83126 83110 69.2152 45.75 0.0842925 0.0144334 2023-09-23T23:30:00.256 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:30:00.300 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:30:00.326 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:00 smithi138 ceph-mon[193606]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Reduced data availability: 1 pg peering; Degraded data redundancy: 9983/241008 objects degraded (4.142%), 3 pgs degraded 2023-09-23T23:30:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:00 smithi138 ceph-mon[193606]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T23:30:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:00 smithi138 ceph-mon[193606]: [WRN] PG_AVAILABILITY: Reduced data availability: 1 pg peering 2023-09-23T23:30:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:00 smithi138 ceph-mon[193606]: pg 1.6 is stuck peering for 3m, current state remapped+peering, last acting [0,3] 2023-09-23T23:30:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:00 smithi138 ceph-mon[193606]: [WRN] PG_DEGRADED: Degraded data redundancy: 9983/241008 objects degraded (4.142%), 3 pgs degraded 2023-09-23T23:30:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:00 smithi138 ceph-mon[193606]: pg 12.9 is activating+undersized+degraded+remapped, acting [2,3] 2023-09-23T23:30:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:00 smithi138 ceph-mon[193606]: pg 12.f is activating+undersized+degraded+remapped, acting [2,5] 2023-09-23T23:30:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:00 smithi138 ceph-mon[193606]: pg 12.19 is activating+undersized+degraded+remapped, acting [2,3] 2023-09-23T23:30:00.772 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:30:00.857 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:30:00.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:00.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:30:00.967 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:30:01.064 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:01.171 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:30:01.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:01 smithi110 ceph-mon[221102]: pgmap v1363: 129 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 activating, 3 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 5 peering, 113 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 4.2 KiB/s rd, 12 MiB/s wr, 202 op/s; 9984/242802 objects degraded (4.112%); 12091/242802 objects misplaced (4.980%); 5.4 MiB/s, 88 objects/s recovering 2023-09-23T23:30:01.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:01 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T23:30:01.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:01 smithi110 ceph-mon[212939]: pgmap v1363: 129 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 activating, 3 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 5 peering, 113 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 4.2 KiB/s rd, 12 MiB/s wr, 202 op/s; 9984/242802 objects degraded (4.112%); 12091/242802 objects misplaced (4.980%); 5.4 MiB/s, 88 objects/s recovering 2023-09-23T23:30:01.207 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:01 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T23:30:01.237 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:30:01.255 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 76 16 83782 83766 68.8425 41 0.0164355 0.0145208 2023-09-23T23:30:01.274 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:30:01.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:01.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:30:01.308 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:30:01.392 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:30:01.431 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:01 smithi138 ceph-mon[193606]: pgmap v1363: 129 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 activating, 3 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 5 peering, 113 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 4.2 KiB/s rd, 12 MiB/s wr, 202 op/s; 9984/242802 objects degraded (4.112%); 12091/242802 objects misplaced (4.980%); 5.4 MiB/s, 88 objects/s recovering 2023-09-23T23:30:01.431 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:01 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T23:30:01.439 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:01.615 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:30:01.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:01.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:30:01.720 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:30:01.789 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:02.255 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 77 16 84652 84636 68.6546 54.375 0.00717806 0.0145607 2023-09-23T23:30:03.040 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:30:03.259 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 78 16 85520 85504 68.4666 54.25 0.0138298 0.0146009 2023-09-23T23:30:03.403 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:03 smithi138 ceph-mon[193606]: pgmap v1364: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 activating, 113 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 3.3 KiB/s rd, 28 MiB/s wr, 455 op/s; 22587/248103 objects degraded (9.104%); 16809/248103 objects misplaced (6.775%); 6.2 MiB/s, 100 objects/s recovering 2023-09-23T23:30:03.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:03 smithi110 ceph-mon[212939]: pgmap v1364: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 activating, 113 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 3.3 KiB/s rd, 28 MiB/s wr, 455 op/s; 22587/248103 objects degraded (9.104%); 16809/248103 objects misplaced (6.775%); 6.2 MiB/s, 100 objects/s recovering 2023-09-23T23:30:03.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:03 smithi110 ceph-mon[221102]: pgmap v1364: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 activating, 113 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 3.3 KiB/s rd, 28 MiB/s wr, 455 op/s; 22587/248103 objects degraded (9.104%); 16809/248103 objects misplaced (6.775%); 6.2 MiB/s, 100 objects/s recovering 2023-09-23T23:30:03.693 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:30:03.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:03.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:30:03.802 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:03.870 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:03.962 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:03.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:03.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:03.986 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:30:03.986 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:30:03.986 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.987 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46229.0:86043 12.f e499/496 12:fb708a2e:::benchmark_data_smithi110_491193_object86042:head v 499'5380, mlcod=0'0)", 2023-09-23T23:30:03.987 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:30:03.979091+0000", 2023-09-23T23:30:03.987 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0045261340000000002, 2023-09-23T23:30:03.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0045658649999999997, 2023-09-23T23:30:03.987 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:03.987 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:30:03.987 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:03.987 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.987 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:03.987 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:03.979091+0000", 2023-09-23T23:30:03.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:03.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:03.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.988 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:03.988 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:03.979091+0000", 2023-09-23T23:30:03.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:03.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:03.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.988 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:03.988 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:03.979090+0000", 2023-09-23T23:30:03.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:30:03.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:03.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.989 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:03.989 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:03.979200+0000", 2023-09-23T23:30:03.989 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109831 2023-09-23T23:30:03.989 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:03.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.989 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:03.989 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:03.979216+0000", 2023-09-23T23:30:03.989 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5900999999999999e-05 2023-09-23T23:30:03.989 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:03.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.990 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:03.990 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:03.979224+0000", 2023-09-23T23:30:03.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9289999999999998e-06 2023-09-23T23:30:03.990 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:03.990 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.990 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:03.990 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:03.979274+0000", 2023-09-23T23:30:03.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9657999999999998e-05 2023-09-23T23:30:03.990 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:03.990 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.990 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:03.991 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:03.979301+0000", 2023-09-23T23:30:03.991 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7597999999999999e-05 2023-09-23T23:30:03.991 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:03.991 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:03.991 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:03.991 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:03.991 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.991 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46229.0:86048 12.11 e499/496 12:8ecad874:::benchmark_data_smithi110_491193_object86047:head v 499'5051, mlcod=0'0)", 2023-09-23T23:30:03.991 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:30:03.982321+0000", 2023-09-23T23:30:03.991 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.001296193, 2023-09-23T23:30:03.991 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014045419999999999, 2023-09-23T23:30:03.991 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:03.992 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:30:03.992 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:03.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:03.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:03.982321+0000", 2023-09-23T23:30:03.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:03.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:03.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:03.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:03.982321+0000", 2023-09-23T23:30:03.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:03.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:03.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:03.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:03.982320+0000", 2023-09-23T23:30:03.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:30:03.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:03.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:03.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:03.982359+0000", 2023-09-23T23:30:03.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8872000000000003e-05 2023-09-23T23:30:03.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:03.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.994 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:03.994 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:03.982373+0000", 2023-09-23T23:30:03.994 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4100999999999999e-05 2023-09-23T23:30:03.994 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:03.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.994 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:03.994 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:03.982380+0000", 2023-09-23T23:30:03.994 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4980000000000003e-06 2023-09-23T23:30:03.994 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:03.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.994 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:03.994 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:03.982442+0000", 2023-09-23T23:30:03.995 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2106000000000005e-05 2023-09-23T23:30:03.995 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:03.995 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:03.995 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:03.995 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:03.982465+0000", 2023-09-23T23:30:03.995 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3195999999999999e-05 2023-09-23T23:30:03.995 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:03.995 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:03.995 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:03.995 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:03.995 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-09-23T23:30:03.996 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 2 2023-09-23T23:30:03.996 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:30:04.068 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:04.120 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:04.253 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:04.257 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:04 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 22587/248103 objects degraded (9.104%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:04.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:04.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:04.260 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 79 16 86313 86297 68.2266 49.5625 0.0184548 0.0146521 2023-09-23T23:30:04.359 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:30:04.408 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:04.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:04 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 22587/248103 objects degraded (9.104%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:04.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:04 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 22587/248103 objects degraded (9.104%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:04.493 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:30:04.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:04.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:30:04.601 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:30:04.629 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:04.683 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:30:04.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:04.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:30:04.788 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:30:04.865 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:05.033 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:30:05.050 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:30:05.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:05.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:30:05.148 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:05 smithi110 ceph-mon[212939]: pgmap v1365: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 26 MiB/s wr, 415 op/s; 22587/249663 objects degraded (9.047%); 16809/249663 objects misplaced (6.733%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T23:30:05.149 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:05 smithi110 ceph-mon[221102]: pgmap v1365: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 26 MiB/s wr, 415 op/s; 22587/249663 objects degraded (9.047%); 16809/249663 objects misplaced (6.733%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T23:30:05.179 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:30:05.262 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:30:05.260989+0000 min lat: 0.00115016 max lat: 2.27392 avg lat: 0.0147096 2023-09-23T23:30:05.262 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:30:05.262 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 80 16 87059 87043 67.9556 46.625 0.00958818 0.0147096 2023-09-23T23:30:05.264 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:05.414 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:05 smithi138 ceph-mon[193606]: pgmap v1365: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 26 MiB/s wr, 415 op/s; 22587/249663 objects degraded (9.047%); 16809/249663 objects misplaced (6.733%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T23:30:05.436 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:30:05.606 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:30:05.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:05.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:30:05.673 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:30:05.673 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "" 2023-09-23T23:30:05.674 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:30:05.714 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:05.771 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:05.852 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:05.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:05.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:05.961 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:05.971 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:30:06.013 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:06.079 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:06 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3270912054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:30:06.079 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:06 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:30:06.092 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:06.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:06.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:06.103 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:30:06.196 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:30:06.262 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 81 16 87827 87811 67.7093 48 0.0168695 0.0147632 2023-09-23T23:30:06.277 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:06 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3270912054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:30:06.277 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:06 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:30:06.277 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:06 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3270912054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:30:06.277 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:06 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:30:06.285 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:07.071 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:30:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:30:06] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.43.0" 2023-09-23T23:30:07.262 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 82 16 88584 88568 67.4607 47.3125 0.00595581 0.0148095 2023-09-23T23:30:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:07 smithi138 ceph-mon[193606]: pgmap v1366: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 37 MiB/s wr, 588 op/s; 22587/256437 objects degraded (8.808%); 16471/256437 objects misplaced (6.423%); 7.4 MiB/s, 120 objects/s recovering 2023-09-23T23:30:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:07 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:30:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:07 smithi138 ceph-mon[193606]: failed to encode map e500 with expected crc 2023-09-23T23:30:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:07 smithi138 ceph-mon[193606]: failed to encode map e500 with expected crc 2023-09-23T23:30:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:07 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:30:07.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:07 smithi138 ceph-mon[193606]: osdmap e500: 8 total, 8 up, 8 in 2023-09-23T23:30:07.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:07 smithi138 ceph-mon[193606]: failed to encode map e500 with expected crc 2023-09-23T23:30:07.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:07 smithi138 ceph-mon[193606]: failed to encode map e500 with expected crc 2023-09-23T23:30:07.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:07 smithi138 ceph-mon[193606]: failed to encode map e500 with expected crc 2023-09-23T23:30:07.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:07 smithi138 ceph-mon[193606]: failed to encode map e500 with expected crc 2023-09-23T23:30:07.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[221102]: pgmap v1366: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 37 MiB/s wr, 588 op/s; 22587/256437 objects degraded (8.808%); 16471/256437 objects misplaced (6.423%); 7.4 MiB/s, 120 objects/s recovering 2023-09-23T23:30:07.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:30:07.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[221102]: failed to encode map e500 with expected crc 2023-09-23T23:30:07.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[221102]: failed to encode map e500 with expected crc 2023-09-23T23:30:07.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:30:07.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[221102]: osdmap e500: 8 total, 8 up, 8 in 2023-09-23T23:30:07.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[221102]: failed to encode map e500 with expected crc 2023-09-23T23:30:07.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[221102]: failed to encode map e500 with expected crc 2023-09-23T23:30:07.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[221102]: failed to encode map e500 with expected crc 2023-09-23T23:30:07.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[221102]: failed to encode map e500 with expected crc 2023-09-23T23:30:07.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[212939]: pgmap v1366: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 37 MiB/s wr, 588 op/s; 22587/256437 objects degraded (8.808%); 16471/256437 objects misplaced (6.423%); 7.4 MiB/s, 120 objects/s recovering 2023-09-23T23:30:07.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:30:07.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[212939]: failed to encode map e500 with expected crc 2023-09-23T23:30:07.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[212939]: failed to encode map e500 with expected crc 2023-09-23T23:30:07.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:30:07.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[212939]: osdmap e500: 8 total, 8 up, 8 in 2023-09-23T23:30:07.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[212939]: failed to encode map e500 with expected crc 2023-09-23T23:30:07.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[212939]: failed to encode map e500 with expected crc 2023-09-23T23:30:07.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[212939]: failed to encode map e500 with expected crc 2023-09-23T23:30:07.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:07 smithi110 ceph-mon[212939]: failed to encode map e500 with expected crc 2023-09-23T23:30:08.081 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:30:08.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:08.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:30:08.185 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:08.247 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:08.262 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 83 16 89341 89325 67.218 47.3125 0.0191744 0.014872 2023-09-23T23:30:08.456 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:08.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:08.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:08.561 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:30:08.578 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:30:08.579 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:30:08.579 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:30:08.579 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:30:08.579 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:30:08.607 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:08.834 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:30:08.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:08.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:30:08.939 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:30:08.968 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:09.029 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:30:09.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:09.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:30:09.135 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:30:09.181 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:09.257 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:30:09.262 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 84 16 90137 90121 67.0101 49.75 0.00844117 0.0149182 2023-09-23T23:30:09.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:09.262 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:09 smithi138 ceph-mon[193606]: pgmap v1368: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 648 op/s; 22587/259923 objects degraded (8.690%); 16061/259923 objects misplaced (6.179%); 8.0 MiB/s, 129 objects/s recovering 2023-09-23T23:30:09.262 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:09 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 22587/259923 objects degraded (8.690%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:09.263 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:09 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:09.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:30:09.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:09 smithi110 ceph-mon[212939]: pgmap v1368: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 648 op/s; 22587/259923 objects degraded (8.690%); 16061/259923 objects misplaced (6.179%); 8.0 MiB/s, 129 objects/s recovering 2023-09-23T23:30:09.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:09 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 22587/259923 objects degraded (8.690%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:09.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:09 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:09.359 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:09 smithi110 ceph-mon[221102]: pgmap v1368: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 648 op/s; 22587/259923 objects degraded (8.690%); 16061/259923 objects misplaced (6.179%); 8.0 MiB/s, 129 objects/s recovering 2023-09-23T23:30:09.359 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:09 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 22587/259923 objects degraded (8.690%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:09.360 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:09 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:09.364 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:30:09.389 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:09.430 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:30:09.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:09.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:30:09.533 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:30:09.621 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:09.803 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:30:09.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:09.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:30:09.912 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:30:09.952 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:30:09.967 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:10.091 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:30:10.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:10.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:30:10.206 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:30:10.239 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:30:10.262 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 85 16 90851 90835 66.7469 44.625 0.0115069 0.0149766 2023-09-23T23:30:10.301 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:30:10.308 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:10.348 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:30:10.381 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:30:10.419 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:30:10.429 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:30:10.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:10.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:30:10.471 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:30:10.517 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:30:10.534 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:30:10.536 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:30:10.561 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:30:10.590 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:30:10.590 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:30:10.590 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:30:10.590 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:30:10.590 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:30:10.590 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:30:10.590 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:30:10.590 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:30:10.608 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:10.827 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:30:10.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:10.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:30:10.933 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:30:11.024 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:11.085 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:11 smithi138 ceph-mon[193606]: pgmap v1369: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 724 op/s; 22587/264531 objects degraded (8.539%); 15646/264531 objects misplaced (5.915%); 10 MiB/s, 164 objects/s recovering 2023-09-23T23:30:11.085 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:11 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3705721904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:30:11.189 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:30:11.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:11.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:30:11.264 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 86 16 91730 91714 66.608 54.9375 0.00590802 0.015009 2023-09-23T23:30:11.301 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:30:11.369 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:11 smithi110 ceph-mon[212939]: pgmap v1369: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 724 op/s; 22587/264531 objects degraded (8.539%); 15646/264531 objects misplaced (5.915%); 10 MiB/s, 164 objects/s recovering 2023-09-23T23:30:11.369 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:11 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3705721904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:30:11.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:11 smithi110 ceph-mon[221102]: pgmap v1369: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 724 op/s; 22587/264531 objects degraded (8.539%); 15646/264531 objects misplaced (5.915%); 10 MiB/s, 164 objects/s recovering 2023-09-23T23:30:11.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:11 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3705721904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:30:11.381 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:30:11.385 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:11.508 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:30:11.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:11.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:30:11.616 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:11.658 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:11.703 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:11.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:11.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:11.807 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:30:11.833 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:11.884 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:30:11.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:11.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:30:11.988 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:30:12.029 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:12.249 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:30:12.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:12.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:30:12.267 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 87 16 92459 92443 66.3636 45.5625 0.0104517 0.015063 2023-09-23T23:30:12.354 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:30:12.449 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:12.592 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:30:12.606 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:30:12.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:12.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:30:12.716 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:12.759 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:12.864 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:12.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:12.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:12.936 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:30:12.936 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:30:12.969 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:30:13.013 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:13.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:13 smithi110 ceph-mon[212939]: pgmap v1370: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s; 22587/271416 objects degraded (8.322%); 15182/271416 objects misplaced (5.594%); 11 MiB/s, 169 objects/s recovering 2023-09-23T23:30:13.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:13.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:30:13.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:13 smithi110 ceph-mon[221102]: pgmap v1370: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s; 22587/271416 objects degraded (8.322%); 15182/271416 objects misplaced (5.594%); 11 MiB/s, 169 objects/s recovering 2023-09-23T23:30:13.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:13.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:30:13.268 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 88 16 93334 93318 66.2307 54.6875 0.0210067 0.0150938 2023-09-23T23:30:13.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:13 smithi138 ceph-mon[193606]: pgmap v1370: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s; 22587/271416 objects degraded (8.322%); 15182/271416 objects misplaced (5.594%); 11 MiB/s, 169 objects/s recovering 2023-09-23T23:30:13.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:13.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:30:13.857 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:30:13.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:13.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:30:13.962 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:30:14.040 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:14.150 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:30:14.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:14.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:30:14.260 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:30:14.268 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 89 16 94193 94177 66.0898 53.6875 0.0201715 0.0151257 2023-09-23T23:30:14.295 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 7, 2, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:30:14.295 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:30:14.296 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T23:30:14.296 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:30:14.320 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:14.349 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:14 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 22587/271416 objects degraded (8.322%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:14.349 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:14 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 22587/271416 objects degraded (8.322%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:14.377 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:14 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 22587/271416 objects degraded (8.322%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:14.378 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:30:14.381 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:30:14.427 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:30:14.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:14.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:30:14.540 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:30:14.679 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:14.875 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:30:14.988 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:30:15.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:15.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:30:15.112 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:30:15.152 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:15.230 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:30:15.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:15.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:30:15.268 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total time run: 90.0209 2023-09-23T23:30:15.268 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total writes made: 94872 2023-09-23T23:30:15.268 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Write size: 65536 2023-09-23T23:30:15.269 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object size: 65536 2023-09-23T23:30:15.269 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Bandwidth (MB/sec): 65.868 2023-09-23T23:30:15.269 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Bandwidth: 31.3726 2023-09-23T23:30:15.269 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max bandwidth (MB/sec): 173.875 2023-09-23T23:30:15.269 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min bandwidth (MB/sec): 0 2023-09-23T23:30:15.269 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average IOPS: 1053 2023-09-23T23:30:15.269 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev IOPS: 501.973 2023-09-23T23:30:15.269 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max IOPS: 2782 2023-09-23T23:30:15.269 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min IOPS: 0 2023-09-23T23:30:15.269 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average Latency(s): 0.0151773 2023-09-23T23:30:15.269 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Latency(s): 0.0440521 2023-09-23T23:30:15.269 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max latency(s): 2.27392 2023-09-23T23:30:15.270 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min latency(s): 0.00115016 2023-09-23T23:30:15.275 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T23:30:15.289 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:30:15.334 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:30:15.370 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:15 smithi138 ceph-mon[193606]: pgmap v1371: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s; 22587/272958 objects degraded (8.275%); 15182/272958 objects misplaced (5.562%); 11 MiB/s, 169 objects/s recovering 2023-09-23T23:30:15.373 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:15.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:15 smithi110 ceph-mon[212939]: pgmap v1371: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s; 22587/272958 objects degraded (8.275%); 15182/272958 objects misplaced (5.562%); 11 MiB/s, 169 objects/s recovering 2023-09-23T23:30:15.423 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:15 smithi110 ceph-mon[221102]: pgmap v1371: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s; 22587/272958 objects degraded (8.275%); 15182/272958 objects misplaced (5.562%); 11 MiB/s, 169 objects/s recovering 2023-09-23T23:30:15.434 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:30:15.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:15.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:30:15.539 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:15.582 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:15.652 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:15.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:15.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:15.748 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:30:15.756 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:30:15.782 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:15.824 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:30:15.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:15.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:30:15.927 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:30:16.032 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:16.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:16 smithi110 ceph-mon[212939]: 12.2 scrub starts 2023-09-23T23:30:16.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:16 smithi110 ceph-mon[212939]: osdmap e501: 8 total, 8 up, 8 in 2023-09-23T23:30:16.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:16 smithi110 ceph-mon[221102]: 12.2 scrub starts 2023-09-23T23:30:16.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:16 smithi110 ceph-mon[221102]: osdmap e501: 8 total, 8 up, 8 in 2023-09-23T23:30:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:16 smithi138 ceph-mon[193606]: 12.2 scrub starts 2023-09-23T23:30:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:16 smithi138 ceph-mon[193606]: osdmap e501: 8 total, 8 up, 8 in 2023-09-23T23:30:16.658 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:30:16.840 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:30:16.840 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:30:17.126 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[212939]: failed to encode map e501 with expected crc 2023-09-23T23:30:17.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[212939]: pgmap v1373: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 116 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s; 20115/280203 objects degraded (7.179%); 17077/280203 objects misplaced (6.095%); 11 MiB/s, 178 objects/s recovering 2023-09-23T23:30:17.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[212939]: 12.12 scrub starts 2023-09-23T23:30:17.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2466713580' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:30:17.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:30:17.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:30:17.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:30:17.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[212939]: osdmap e502: 8 total, 8 up, 8 in 2023-09-23T23:30:17.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[212939]: failed to encode map e502 with expected crc 2023-09-23T23:30:17.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[212939]: failed to encode map e502 with expected crc 2023-09-23T23:30:17.128 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:30:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:30:16] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.43.0" 2023-09-23T23:30:17.128 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:17 smithi138 ceph-mon[193606]: failed to encode map e501 with expected crc 2023-09-23T23:30:17.128 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:17 smithi138 ceph-mon[193606]: pgmap v1373: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 116 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s; 20115/280203 objects degraded (7.179%); 17077/280203 objects misplaced (6.095%); 11 MiB/s, 178 objects/s recovering 2023-09-23T23:30:17.128 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:17 smithi138 ceph-mon[193606]: 12.12 scrub starts 2023-09-23T23:30:17.129 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:17 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2466713580' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:30:17.129 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:17 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:30:17.129 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:17 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:30:17.129 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:17 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:30:17.129 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:17 smithi138 ceph-mon[193606]: osdmap e502: 8 total, 8 up, 8 in 2023-09-23T23:30:17.129 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:17 smithi138 ceph-mon[193606]: failed to encode map e502 with expected crc 2023-09-23T23:30:17.129 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:17 smithi138 ceph-mon[193606]: failed to encode map e502 with expected crc 2023-09-23T23:30:17.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[221102]: failed to encode map e501 with expected crc 2023-09-23T23:30:17.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[221102]: pgmap v1373: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 116 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s; 20115/280203 objects degraded (7.179%); 17077/280203 objects misplaced (6.095%); 11 MiB/s, 178 objects/s recovering 2023-09-23T23:30:17.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[221102]: 12.12 scrub starts 2023-09-23T23:30:17.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2466713580' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:30:17.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:30:17.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:30:17.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:30:17.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[221102]: osdmap e502: 8 total, 8 up, 8 in 2023-09-23T23:30:17.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[221102]: failed to encode map e502 with expected crc 2023-09-23T23:30:17.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:17 smithi110 ceph-mon[221102]: failed to encode map e502 with expected crc 2023-09-23T23:30:17.753 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:30:18.167 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:30:18.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:18.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:30:18.253 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:30:18.278 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:30:18.354 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:18 smithi138 ceph-mon[193606]: failed to encode map e502 with expected crc 2023-09-23T23:30:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:18 smithi110 ceph-mon[221102]: failed to encode map e502 with expected crc 2023-09-23T23:30:18.459 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:18 smithi110 ceph-mon[212939]: failed to encode map e502 with expected crc 2023-09-23T23:30:19.317 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:30:19.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:19.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:30:19.430 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:19.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:19 smithi138 ceph-mon[193606]: pgmap v1375: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 116 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 784 op/s; 20115/282534 objects degraded (7.119%); 16546/282534 objects misplaced (5.856%); 10 MiB/s, 165 objects/s recovering 2023-09-23T23:30:19.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:19 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 20115/282534 objects degraded (7.119%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:19 smithi110 ceph-mon[221102]: pgmap v1375: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 116 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 784 op/s; 20115/282534 objects degraded (7.119%); 16546/282534 objects misplaced (5.856%); 10 MiB/s, 165 objects/s recovering 2023-09-23T23:30:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:19 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 20115/282534 objects degraded (7.119%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:19.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:19 smithi110 ceph-mon[212939]: pgmap v1375: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 116 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 784 op/s; 20115/282534 objects degraded (7.119%); 16546/282534 objects misplaced (5.856%); 10 MiB/s, 165 objects/s recovering 2023-09-23T23:30:19.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:19 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 20115/282534 objects degraded (7.119%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:19.470 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:19.554 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:19.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:19.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:19.664 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:30:19.695 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:19.756 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:30:19.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:19.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:30:19.860 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:30:19.918 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:20.037 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:30:20.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:20.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:30:20.101 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:30:20.101 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:30:20.147 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:30:20.174 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:20.937 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:30:20.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:20.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:30:21.016 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:30:21.041 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:30:21.137 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:21.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:21 smithi110 ceph-mon[221102]: pgmap v1376: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 34 MiB/s wr, 685 op/s; 20115/281016 objects degraded (7.158%); 16232/281016 objects misplaced (5.776%); 10 MiB/s, 163 objects/s recovering 2023-09-23T23:30:21.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:21 smithi110 ceph-mon[212939]: pgmap v1376: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 34 MiB/s wr, 685 op/s; 20115/281016 objects degraded (7.158%); 16232/281016 objects misplaced (5.776%); 10 MiB/s, 163 objects/s recovering 2023-09-23T23:30:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:21 smithi138 ceph-mon[193606]: pgmap v1376: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 34 MiB/s wr, 685 op/s; 20115/281016 objects degraded (7.158%); 16232/281016 objects misplaced (5.776%); 10 MiB/s, 163 objects/s recovering 2023-09-23T23:30:21.500 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:30:21.950 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:30:21.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:21.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:30:22.062 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:30:22.125 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:22.268 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:30:22.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:22.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:30:22.384 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:30:22.414 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:22.478 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:30:22.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:22.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:30:22.583 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:22.614 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:22.760 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:30:22.801 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:30:22.834 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:30:22.861 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:22.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:22.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:22.904 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:30:22.945 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:30:22.968 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:30:22.986 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:30:23.031 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:30:23.067 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:30:23.081 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:23.082 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:30:23.082 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:30:23.082 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:30:23.082 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:30:23.082 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:30:23.082 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:30:23.082 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:30:23.082 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:30:23.201 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:30:23.233 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:30:23.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:23 smithi138 ceph-mon[193606]: pgmap v1377: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 31 MiB/s wr, 894 op/s; 20115/274962 objects degraded (7.316%); 15315/274962 objects misplaced (5.570%); 15 MiB/s, 242 objects/s recovering 2023-09-23T23:30:23.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:23 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:30:23.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:23 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4191385623' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:30:23.445 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:23 smithi110 ceph-mon[212939]: pgmap v1377: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 31 MiB/s wr, 894 op/s; 20115/274962 objects degraded (7.316%); 15315/274962 objects misplaced (5.570%); 15 MiB/s, 242 objects/s recovering 2023-09-23T23:30:23.445 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:23 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:30:23.445 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:23 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4191385623' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:30:23.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:23 smithi110 ceph-mon[221102]: pgmap v1377: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 31 MiB/s wr, 894 op/s; 20115/274962 objects degraded (7.316%); 15315/274962 objects misplaced (5.570%); 15 MiB/s, 242 objects/s recovering 2023-09-23T23:30:23.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:23 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:30:23.446 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:23 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4191385623' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:30:23.474 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:30:23.474 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:30:23.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:30:23.474 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:30:23.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.475 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:92240 10.0 10:0ba3daac:::benchmark_data_smithi110_366450_object92239:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:30:23.475 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:25.873807+0000", 2023-09-23T23:30:23.475 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 597.59468630100002, 2023-09-23T23:30:23.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0193927490000001, 2023-09-23T23:30:23.475 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.475 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:23.475 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:30:23.475 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:30:23.475 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:23.475 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92240 2023-09-23T23:30:23.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.475 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.476 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.476 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873807+0000", 2023-09-23T23:30:23.476 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.476 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.476 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.476 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.476 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873807+0000", 2023-09-23T23:30:23.476 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.476 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.476 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.476 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.476 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873806+0000", 2023-09-23T23:30:23.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:30:23.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873834+0000", 2023-09-23T23:30:23.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8551000000000002e-05 2023-09-23T23:30:23.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873844+0000", 2023-09-23T23:30:23.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0769999999999997e-06 2023-09-23T23:30:23.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873849+0000", 2023-09-23T23:30:23.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3639999999999998e-06 2023-09-23T23:30:23.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873908+0000", 2023-09-23T23:30:23.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8962000000000002e-05 2023-09-23T23:30:23.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.873916+0000", 2023-09-23T23:30:23.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8210000000000002e-06 2023-09-23T23:30:23.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:26.840320+0000", 2023-09-23T23:30:23.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96640448599999995 2023-09-23T23:30:23.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:26.840332+0000", 2023-09-23T23:30:23.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1807000000000001e-05 2023-09-23T23:30:23.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.862906+0000", 2023-09-23T23:30:23.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0225740700000001 2023-09-23T23:30:23.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.863009+0000", 2023-09-23T23:30:23.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103318 2023-09-23T23:30:23.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:30:23.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.863135+0000", 2023-09-23T23:30:23.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000125955 2023-09-23T23:30:23.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:30:23.481 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.872367+0000", 2023-09-23T23:30:23.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0092317479999999997 2023-09-23T23:30:23.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.481 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:23.481 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.892992+0000", 2023-09-23T23:30:23.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020624462 2023-09-23T23:30:23.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.481 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:23.481 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893090+0000", 2023-09-23T23:30:23.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8231000000000002e-05 2023-09-23T23:30:23.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.482 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.482 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:30:23.482 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893126+0000", 2023-09-23T23:30:23.482 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6010000000000003e-05 2023-09-23T23:30:23.482 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.482 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.482 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.482 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893200+0000", 2023-09-23T23:30:23.482 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4126000000000001e-05 2023-09-23T23:30:23.482 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.482 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.482 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.482 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.483 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:92247 10.0 10:0876996b:::benchmark_data_smithi110_366450_object92246:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:30:23.483 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:25.877726+0000", 2023-09-23T23:30:23.483 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 597.59076748699999, 2023-09-23T23:30:23.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.01571897, 2023-09-23T23:30:23.483 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.483 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:23.483 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:30:23.483 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:30:23.483 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:23.483 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92247 2023-09-23T23:30:23.483 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.483 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.484 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877726+0000", 2023-09-23T23:30:23.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.484 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877726+0000", 2023-09-23T23:30:23.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.484 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877726+0000", 2023-09-23T23:30:23.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:30:23.485 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.485 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877760+0000", 2023-09-23T23:30:23.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.413e-05 2023-09-23T23:30:23.485 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.485 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877767+0000", 2023-09-23T23:30:23.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7910000000000005e-06 2023-09-23T23:30:23.485 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877773+0000", 2023-09-23T23:30:23.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3190000000000002e-06 2023-09-23T23:30:23.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.486 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877855+0000", 2023-09-23T23:30:23.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.242e-05 2023-09-23T23:30:23.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.486 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.877864+0000", 2023-09-23T23:30:23.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8929999999999994e-06 2023-09-23T23:30:23.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.487 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:26.840358+0000", 2023-09-23T23:30:23.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96249378399999996 2023-09-23T23:30:23.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.487 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:26.840365+0000", 2023-09-23T23:30:23.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0960000000000003e-06 2023-09-23T23:30:23.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.865238+0000", 2023-09-23T23:30:23.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0248726539999999 2023-09-23T23:30:23.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.865348+0000", 2023-09-23T23:30:23.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110773 2023-09-23T23:30:23.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:30:23.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.865452+0000", 2023-09-23T23:30:23.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103787 2023-09-23T23:30:23.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:30:23.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.872373+0000", 2023-09-23T23:30:23.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069208259999999997 2023-09-23T23:30:23.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:23.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893034+0000", 2023-09-23T23:30:23.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020660586000000002 2023-09-23T23:30:23.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:23.490 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893351+0000", 2023-09-23T23:30:23.490 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00031712999999999999 2023-09-23T23:30:23.490 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:30:23.490 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893383+0000", 2023-09-23T23:30:23.490 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2360000000000002e-05 2023-09-23T23:30:23.490 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.490 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893445+0000", 2023-09-23T23:30:23.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1867999999999998e-05 2023-09-23T23:30:23.491 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.491 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.491 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.491 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:92255 10.0 10:047e84c9:::benchmark_data_smithi110_366450_object92254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T23:30:23.491 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:20:25.889232+0000", 2023-09-23T23:30:23.491 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 597.57926192800005, 2023-09-23T23:30:23.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0043332720000002, 2023-09-23T23:30:23.491 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.491 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:23.491 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:30:23.492 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:30:23.492 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:23.492 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92255 2023-09-23T23:30:23.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.492 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889232+0000", 2023-09-23T23:30:23.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889232+0000", 2023-09-23T23:30:23.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889230+0000", 2023-09-23T23:30:23.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:30:23.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889267+0000", 2023-09-23T23:30:23.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7274000000000003e-05 2023-09-23T23:30:23.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889274+0000", 2023-09-23T23:30:23.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1899999999999998e-06 2023-09-23T23:30:23.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889279+0000", 2023-09-23T23:30:23.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9640000000000002e-06 2023-09-23T23:30:23.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889353+0000", 2023-09-23T23:30:23.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3460000000000005e-05 2023-09-23T23:30:23.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:25.889362+0000", 2023-09-23T23:30:23.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2520000000000005e-06 2023-09-23T23:30:23.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:26.840390+0000", 2023-09-23T23:30:23.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95102806500000003 2023-09-23T23:30:23.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:26.840398+0000", 2023-09-23T23:30:23.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2840000000000006e-06 2023-09-23T23:30:23.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.867281+0000", 2023-09-23T23:30:23.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.026882686 2023-09-23T23:30:23.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.867395+0000", 2023-09-23T23:30:23.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000113608 2023-09-23T23:30:23.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:30:23.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.867496+0000", 2023-09-23T23:30:23.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101588 2023-09-23T23:30:23.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:30:23.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.876926+0000", 2023-09-23T23:30:23.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0094297870000000002 2023-09-23T23:30:23.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:23.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893056+0000", 2023-09-23T23:30:23.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016130228999999999 2023-09-23T23:30:23.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:23.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893478+0000", 2023-09-23T23:30:23.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00042173999999999998 2023-09-23T23:30:23.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:30:23.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893511+0000", 2023-09-23T23:30:23.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3319999999999999e-05 2023-09-23T23:30:23.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:20:27.893565+0000", 2023-09-23T23:30:23.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3349e-05 2023-09-23T23:30:23.499 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.499 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.499 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.499 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:156641 10.0 10:00af6c9f:::benchmark_data_smithi110_366450_object55433:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T23:30:23.499 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:49.023609+0000", 2023-09-23T23:30:23.499 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 514.44488432200001, 2023-09-23T23:30:23.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0481008119999999, 2023-09-23T23:30:23.499 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.499 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:30:23.500 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:30:23.500 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:30:23.500 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:23.500 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 156641 2023-09-23T23:30:23.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.500 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.023609+0000", 2023-09-23T23:30:23.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.023609+0000", 2023-09-23T23:30:23.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.023609+0000", 2023-09-23T23:30:23.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:30:23.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.023611+0000", 2023-09-23T23:30:23.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0980000000000001e-06 2023-09-23T23:30:23.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.023616+0000", 2023-09-23T23:30:23.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4310000000000002e-06 2023-09-23T23:30:23.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.023620+0000", 2023-09-23T23:30:23.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8460000000000001e-06 2023-09-23T23:30:23.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.026676+0000", 2023-09-23T23:30:23.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030557929999999998 2023-09-23T23:30:23.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.026794+0000", 2023-09-23T23:30:23.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011783200000000001 2023-09-23T23:30:23.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:30:23.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.026924+0000", 2023-09-23T23:30:23.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012984199999999999 2023-09-23T23:30:23.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:30:23.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.030162+0000", 2023-09-23T23:30:23.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003238239 2023-09-23T23:30:23.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.039529+0000", 2023-09-23T23:30:23.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0093669070000000007 2023-09-23T23:30:23.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.039554+0000", 2023-09-23T23:30:23.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5494999999999999e-05 2023-09-23T23:30:23.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.036786+0000", 2023-09-23T23:30:23.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99723147700000003 2023-09-23T23:30:23.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.036802+0000", 2023-09-23T23:30:23.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6588000000000001e-05 2023-09-23T23:30:23.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071642+0000", 2023-09-23T23:30:23.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0348391619999999 2023-09-23T23:30:23.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071710+0000", 2023-09-23T23:30:23.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8406999999999997e-05 2023-09-23T23:30:23.506 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.506 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.506 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.506 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:156645 10.0 10:09d04cd1:::benchmark_data_smithi110_366450_object55437:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T23:30:23.507 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:49.033190+0000", 2023-09-23T23:30:23.507 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 514.43530326099994, 2023-09-23T23:30:23.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0386241300000001, 2023-09-23T23:30:23.507 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.507 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:30:23.507 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:30:23.507 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:30:23.507 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:23.507 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 156645 2023-09-23T23:30:23.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.507 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033190+0000", 2023-09-23T23:30:23.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033190+0000", 2023-09-23T23:30:23.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033189+0000", 2023-09-23T23:30:23.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:30:23.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033195+0000", 2023-09-23T23:30:23.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5199999999999997e-06 2023-09-23T23:30:23.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033206+0000", 2023-09-23T23:30:23.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1522000000000001e-05 2023-09-23T23:30:23.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033213+0000", 2023-09-23T23:30:23.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5540000000000003e-06 2023-09-23T23:30:23.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.510 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.510 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033436+0000", 2023-09-23T23:30:23.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00022373599999999999 2023-09-23T23:30:23.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.510 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.510 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033545+0000", 2023-09-23T23:30:23.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108587 2023-09-23T23:30:23.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:30:23.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.033657+0000", 2023-09-23T23:30:23.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112044 2023-09-23T23:30:23.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.039584+0000", 2023-09-23T23:30:23.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0059267770000000003 2023-09-23T23:30:23.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.039591+0000", 2023-09-23T23:30:23.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6930000000000007e-06 2023-09-23T23:30:23.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.512 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.512 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.036888+0000", 2023-09-23T23:30:23.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99729603 2023-09-23T23:30:23.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.512 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.512 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.036897+0000", 2023-09-23T23:30:23.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3370000000000006e-06 2023-09-23T23:30:23.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.512 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.513 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.513 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071737+0000", 2023-09-23T23:30:23.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0348404870000001 2023-09-23T23:30:23.513 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.513 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.513 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071814+0000", 2023-09-23T23:30:23.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6959999999999995e-05 2023-09-23T23:30:23.513 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.513 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.513 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.513 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.513 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:156648 10.b 10:d34329b9:::benchmark_data_smithi110_366450_object55440:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T23:30:23.514 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:49.042527+0000", 2023-09-23T23:30:23.514 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 514.42596613600006, 2023-09-23T23:30:23.514 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0423278210000002, 2023-09-23T23:30:23.514 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.514 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:23.514 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:30:23.514 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:30:23.514 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:23.514 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 156648 2023-09-23T23:30:23.514 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.514 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.514 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.515 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.515 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.042527+0000", 2023-09-23T23:30:23.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.515 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.515 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.515 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.515 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.042527+0000", 2023-09-23T23:30:23.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.515 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.515 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.515 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.515 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.042526+0000", 2023-09-23T23:30:23.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:30:23.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.042530+0000", 2023-09-23T23:30:23.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3000000000000002e-06 2023-09-23T23:30:23.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.042538+0000", 2023-09-23T23:30:23.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1200000000000002e-06 2023-09-23T23:30:23.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.042543+0000", 2023-09-23T23:30:23.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8069999999999999e-06 2023-09-23T23:30:23.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.050117+0000", 2023-09-23T23:30:23.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0075745989999999996 2023-09-23T23:30:23.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.050123+0000", 2023-09-23T23:30:23.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.942e-06 2023-09-23T23:30:23.518 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.518 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.518 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.518 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.038821+0000", 2023-09-23T23:30:23.518 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98869784299999997 2023-09-23T23:30:23.518 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.518 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.518 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.518 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.038828+0000", 2023-09-23T23:30:23.518 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1910000000000003e-06 2023-09-23T23:30:23.518 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.518 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.519 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.519 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.067513+0000", 2023-09-23T23:30:23.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0286844479999999 2023-09-23T23:30:23.519 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.519 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.519 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.519 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.067622+0000", 2023-09-23T23:30:23.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109442 2023-09-23T23:30:23.519 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.519 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.519 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:30:23.519 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.068137+0000", 2023-09-23T23:30:23.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00051531799999999996 2023-09-23T23:30:23.520 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.520 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:30:23.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.083787+0000", 2023-09-23T23:30:23.520 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015649676000000001 2023-09-23T23:30:23.520 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.520 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:23.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.084665+0000", 2023-09-23T23:30:23.520 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000877664 2023-09-23T23:30:23.520 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.521 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:23.521 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.084720+0000", 2023-09-23T23:30:23.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5492999999999999e-05 2023-09-23T23:30:23.521 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.521 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.521 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:30:23.521 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.084768+0000", 2023-09-23T23:30:23.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7715e-05 2023-09-23T23:30:23.521 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.521 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.521 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.521 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.084855+0000", 2023-09-23T23:30:23.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7238999999999996e-05 2023-09-23T23:30:23.522 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.522 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.522 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.522 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.522 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.522 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:156656 10.0 10:03429159:::benchmark_data_smithi110_366450_object55448:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:30:23.522 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:49.049592+0000", 2023-09-23T23:30:23.522 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 514.41890107500001, 2023-09-23T23:30:23.522 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.030975679, 2023-09-23T23:30:23.522 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.522 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:23.522 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:30:23.523 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:30:23.523 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:23.523 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 156656 2023-09-23T23:30:23.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.523 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.049592+0000", 2023-09-23T23:30:23.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.049592+0000", 2023-09-23T23:30:23.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.524 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.049591+0000", 2023-09-23T23:30:23.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:30:23.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.524 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.049597+0000", 2023-09-23T23:30:23.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.331e-06 2023-09-23T23:30:23.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.049609+0000", 2023-09-23T23:30:23.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2109e-05 2023-09-23T23:30:23.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.049614+0000", 2023-09-23T23:30:23.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.13e-06 2023-09-23T23:30:23.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.050086+0000", 2023-09-23T23:30:23.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00047189900000000002 2023-09-23T23:30:23.526 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.526 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.526 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.526 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.050095+0000", 2023-09-23T23:30:23.526 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4069999999999993e-06 2023-09-23T23:30:23.526 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.526 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.526 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.526 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.037453+0000", 2023-09-23T23:30:23.526 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98735800600000001 2023-09-23T23:30:23.526 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.526 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.527 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.527 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.037468+0000", 2023-09-23T23:30:23.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4931e-05 2023-09-23T23:30:23.527 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.527 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.527 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.527 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071833+0000", 2023-09-23T23:30:23.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0343654170000001 2023-09-23T23:30:23.527 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.527 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.527 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.527 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071917+0000", 2023-09-23T23:30:23.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4394000000000005e-05 2023-09-23T23:30:23.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:30:23.528 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.072419+0000", 2023-09-23T23:30:23.528 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00050144499999999999 2023-09-23T23:30:23.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:30:23.528 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.079021+0000", 2023-09-23T23:30:23.528 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066016110000000003 2023-09-23T23:30:23.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:23.529 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.079064+0000", 2023-09-23T23:30:23.529 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3686999999999997e-05 2023-09-23T23:30:23.529 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.529 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.529 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:23.529 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.080466+0000", 2023-09-23T23:30:23.529 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014013280000000001 2023-09-23T23:30:23.529 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.529 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.529 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:30:23.529 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.080504+0000", 2023-09-23T23:30:23.529 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8420000000000001e-05 2023-09-23T23:30:23.530 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.530 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.530 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.530 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.080568+0000", 2023-09-23T23:30:23.530 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4061999999999998e-05 2023-09-23T23:30:23.530 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.530 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.530 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.530 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.530 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.530 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:156674 10.b 10:d8537923:::benchmark_data_smithi110_366450_object55466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:30:23.530 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:49.070250+0000", 2023-09-23T23:30:23.531 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 514.39824305800005, 2023-09-23T23:30:23.531 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0147600799999998, 2023-09-23T23:30:23.531 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.531 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:23.531 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:30:23.531 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:30:23.531 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:23.531 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 156674 2023-09-23T23:30:23.531 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.531 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.070250+0000", 2023-09-23T23:30:23.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.532 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.532 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.070250+0000", 2023-09-23T23:30:23.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.532 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.532 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.070250+0000", 2023-09-23T23:30:23.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:30:23.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.070254+0000", 2023-09-23T23:30:23.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5349999999999998e-06 2023-09-23T23:30:23.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.070265+0000", 2023-09-23T23:30:23.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0645000000000001e-05 2023-09-23T23:30:23.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.070271+0000", 2023-09-23T23:30:23.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1720000000000004e-06 2023-09-23T23:30:23.534 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.534 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.534 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.070342+0000", 2023-09-23T23:30:23.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0698999999999995e-05 2023-09-23T23:30:23.534 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.534 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.534 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.070350+0000", 2023-09-23T23:30:23.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4679999999999992e-06 2023-09-23T23:30:23.534 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.038859+0000", 2023-09-23T23:30:23.535 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96850860999999999 2023-09-23T23:30:23.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.038867+0000", 2023-09-23T23:30:23.535 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4319999999999999e-06 2023-09-23T23:30:23.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071622+0000", 2023-09-23T23:30:23.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.032754452 2023-09-23T23:30:23.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071765+0000", 2023-09-23T23:30:23.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014328400000000001 2023-09-23T23:30:23.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:30:23.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.071909+0000", 2023-09-23T23:30:23.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014438999999999999 2023-09-23T23:30:23.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:30:23.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.083795+0000", 2023-09-23T23:30:23.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01188547 2023-09-23T23:30:23.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:23.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.084693+0000", 2023-09-23T23:30:23.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00089860500000000002 2023-09-23T23:30:23.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:23.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.084890+0000", 2023-09-23T23:30:23.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019686399999999999 2023-09-23T23:30:23.538 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.538 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.538 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:30:23.538 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.084929+0000", 2023-09-23T23:30:23.538 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8413e-05 2023-09-23T23:30:23.538 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.538 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.538 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.538 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085010+0000", 2023-09-23T23:30:23.538 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1774999999999995e-05 2023-09-23T23:30:23.538 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.539 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.539 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.539 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.539 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:156675 10.b 10:d38caa2d:::benchmark_data_smithi110_366450_object55467:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:30:23.539 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:49.071507+0000", 2023-09-23T23:30:23.539 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 514.39698622799995, 2023-09-23T23:30:23.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0138194789999999, 2023-09-23T23:30:23.539 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.539 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:23.539 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:30:23.539 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:30:23.539 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:23.540 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 156675 2023-09-23T23:30:23.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.540 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.540 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.540 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.071507+0000", 2023-09-23T23:30:23.540 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.540 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.540 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.071507+0000", 2023-09-23T23:30:23.540 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.071507+0000", 2023-09-23T23:30:23.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:30:23.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.071510+0000", 2023-09-23T23:30:23.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.343e-06 2023-09-23T23:30:23.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.071518+0000", 2023-09-23T23:30:23.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2379999999999997e-06 2023-09-23T23:30:23.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.071522+0000", 2023-09-23T23:30:23.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6789999999999998e-06 2023-09-23T23:30:23.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.071556+0000", 2023-09-23T23:30:23.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3973999999999997e-05 2023-09-23T23:30:23.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.071561+0000", 2023-09-23T23:30:23.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8400000000000002e-06 2023-09-23T23:30:23.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.038901+0000", 2023-09-23T23:30:23.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96734012999999996 2023-09-23T23:30:23.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.038906+0000", 2023-09-23T23:30:23.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1560000000000003e-06 2023-09-23T23:30:23.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.074645+0000", 2023-09-23T23:30:23.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0357391819999999 2023-09-23T23:30:23.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.074755+0000", 2023-09-23T23:30:23.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109843 2023-09-23T23:30:23.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:30:23.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.074883+0000", 2023-09-23T23:30:23.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012784600000000001 2023-09-23T23:30:23.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:30:23.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.083844+0000", 2023-09-23T23:30:23.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0089609440000000002 2023-09-23T23:30:23.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:23.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085122+0000", 2023-09-23T23:30:23.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012780440000000001 2023-09-23T23:30:23.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:23.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085193+0000", 2023-09-23T23:30:23.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.114e-05 2023-09-23T23:30:23.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:30:23.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085236+0000", 2023-09-23T23:30:23.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3235000000000002e-05 2023-09-23T23:30:23.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085327+0000", 2023-09-23T23:30:23.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0464000000000007e-05 2023-09-23T23:30:23.547 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.547 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.547 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.547 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35529.0:156680 10.b 10:dfe97b19:::benchmark_data_smithi110_366450_object55472:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:30:23.547 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:21:49.075711+0000", 2023-09-23T23:30:23.547 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 514.39278211099997, 2023-09-23T23:30:23.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.009800915, 2023-09-23T23:30:23.547 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.547 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:23.548 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:30:23.548 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35529", 2023-09-23T23:30:23.548 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:23.548 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 156680 2023-09-23T23:30:23.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.548 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.548 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.548 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.075711+0000", 2023-09-23T23:30:23.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.548 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.075711+0000", 2023-09-23T23:30:23.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.549 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.075710+0000", 2023-09-23T23:30:23.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:30:23.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.549 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.075715+0000", 2023-09-23T23:30:23.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8489999999999997e-06 2023-09-23T23:30:23.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.075725+0000", 2023-09-23T23:30:23.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3479999999999993e-06 2023-09-23T23:30:23.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.075736+0000", 2023-09-23T23:30:23.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1620999999999999e-05 2023-09-23T23:30:23.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.075800+0000", 2023-09-23T23:30:23.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3490000000000004e-05 2023-09-23T23:30:23.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:49.075811+0000", 2023-09-23T23:30:23.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0876000000000001e-05 2023-09-23T23:30:23.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.038937+0000", 2023-09-23T23:30:23.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96312656399999996 2023-09-23T23:30:23.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:30:23.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:50.038945+0000", 2023-09-23T23:30:23.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6040000000000004e-06 2023-09-23T23:30:23.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.077861+0000", 2023-09-23T23:30:23.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0389166460000001 2023-09-23T23:30:23.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.078000+0000", 2023-09-23T23:30:23.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000139025 2023-09-23T23:30:23.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-09-23T23:30:23.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.078146+0000", 2023-09-23T23:30:23.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014589299999999999 2023-09-23T23:30:23.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:30:23.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.083849+0000", 2023-09-23T23:30:23.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0057028219999999998 2023-09-23T23:30:23.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.554 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:23.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085164+0000", 2023-09-23T23:30:23.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0013147160000000001 2023-09-23T23:30:23.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.554 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:23.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085392+0000", 2023-09-23T23:30:23.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000228641 2023-09-23T23:30:23.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:30:23.555 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085430+0000", 2023-09-23T23:30:23.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7920000000000003e-05 2023-09-23T23:30:23.555 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.555 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:21:51.085512+0000", 2023-09-23T23:30:23.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1898999999999996e-05 2023-09-23T23:30:23.555 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.555 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.555 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.555 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.556 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46229.0:86622 12.1 e499/496 12:864d83a0:::benchmark_data_smithi110_491193_object86621:head v 499'5054, mlcod=0'0)", 2023-09-23T23:30:23.556 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:30:04.586368+0000", 2023-09-23T23:30:23.556 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.882125208000001, 2023-09-23T23:30:23.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.085107490999999993, 2023-09-23T23:30:23.556 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.556 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:23.556 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.556 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.586368+0000", 2023-09-23T23:30:23.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.586368+0000", 2023-09-23T23:30:23.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.586367+0000", 2023-09-23T23:30:23.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:30:23.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.586465+0000", 2023-09-23T23:30:23.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7980999999999996e-05 2023-09-23T23:30:23.558 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.558 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.558 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.558 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.586478+0000", 2023-09-23T23:30:23.558 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2146e-05 2023-09-23T23:30:23.558 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.558 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.558 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.558 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.586483+0000", 2023-09-23T23:30:23.558 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8000000000000004e-06 2023-09-23T23:30:23.558 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.558 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.558 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.559 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.586531+0000", 2023-09-23T23:30:23.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7206999999999997e-05 2023-09-23T23:30:23.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.559 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.559 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.559 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.586554+0000", 2023-09-23T23:30:23.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3434999999999998e-05 2023-09-23T23:30:23.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.559 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.559 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:30:23.559 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.593509+0000", 2023-09-23T23:30:23.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069550239999999998 2023-09-23T23:30:23.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.671476+0000", 2023-09-23T23:30:23.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.077966683999999994 2023-09-23T23:30:23.560 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.560 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.560 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.560 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46229.0:86615 12.f e499/496 12:f0cdb038:::benchmark_data_smithi110_491193_object86614:head v 499'5412, mlcod=0'0)", 2023-09-23T23:30:23.560 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:30:04.587940+0000", 2023-09-23T23:30:23.560 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.880553905999999, 2023-09-23T23:30:23.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.089253805000000005, 2023-09-23T23:30:23.560 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.561 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:23.561 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.587940+0000", 2023-09-23T23:30:23.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.587940+0000", 2023-09-23T23:30:23.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.587939+0000", 2023-09-23T23:30:23.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:30:23.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.587976+0000", 2023-09-23T23:30:23.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7697999999999997e-05 2023-09-23T23:30:23.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.587989+0000", 2023-09-23T23:30:23.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3149000000000001e-05 2023-09-23T23:30:23.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.587996+0000", 2023-09-23T23:30:23.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7229999999999998e-06 2023-09-23T23:30:23.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.588040+0000", 2023-09-23T23:30:23.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4297e-05 2023-09-23T23:30:23.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.588063+0000", 2023-09-23T23:30:23.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2750000000000001e-05 2023-09-23T23:30:23.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.564 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:30:23.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.677148+0000", 2023-09-23T23:30:23.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.089084333000000002 2023-09-23T23:30:23.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.564 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.677193+0000", 2023-09-23T23:30:23.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5782999999999999e-05 2023-09-23T23:30:23.564 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.564 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.565 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.565 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46229.0:86617 12.9 e499/496 12:953b6425:::benchmark_data_smithi110_491193_object86616:head v 499'4904, mlcod=0'0)", 2023-09-23T23:30:23.565 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:30:04.588013+0000", 2023-09-23T23:30:23.565 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.880479945000001, 2023-09-23T23:30:23.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.085818220000000001, 2023-09-23T23:30:23.565 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.565 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:23.565 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.565 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.565 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.588013+0000", 2023-09-23T23:30:23.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.588013+0000", 2023-09-23T23:30:23.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.588013+0000", 2023-09-23T23:30:23.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:30:23.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.588045+0000", 2023-09-23T23:30:23.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1387999999999998e-05 2023-09-23T23:30:23.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.588052+0000", 2023-09-23T23:30:23.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4220000000000003e-06 2023-09-23T23:30:23.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.588057+0000", 2023-09-23T23:30:23.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5750000000000002e-06 2023-09-23T23:30:23.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589593+0000", 2023-09-23T23:30:23.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001536733 2023-09-23T23:30:23.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589617+0000", 2023-09-23T23:30:23.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3209000000000001e-05 2023-09-23T23:30:23.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:30:23.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.673796+0000", 2023-09-23T23:30:23.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.084179778999999996 2023-09-23T23:30:23.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.673832+0000", 2023-09-23T23:30:23.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5373000000000002e-05 2023-09-23T23:30:23.569 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.569 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.569 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.570 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46229.0:86626 12.f e499/496 12:f5ce5be4:::benchmark_data_smithi110_491193_object86625:head v 499'5413, mlcod=0'0)", 2023-09-23T23:30:23.570 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:30:04.588441+0000", 2023-09-23T23:30:23.570 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.880052035999999, 2023-09-23T23:30:23.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.088777095, 2023-09-23T23:30:23.570 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.570 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:23.570 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.588441+0000", 2023-09-23T23:30:23.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.588441+0000", 2023-09-23T23:30:23.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.588441+0000", 2023-09-23T23:30:23.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:30:23.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.588501+0000", 2023-09-23T23:30:23.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9979000000000003e-05 2023-09-23T23:30:23.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.588509+0000", 2023-09-23T23:30:23.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2500000000000006e-06 2023-09-23T23:30:23.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.588513+0000", 2023-09-23T23:30:23.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7780000000000001e-06 2023-09-23T23:30:23.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.670763+0000", 2023-09-23T23:30:23.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.082249668999999997 2023-09-23T23:30:23.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.670782+0000", 2023-09-23T23:30:23.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9698999999999999e-05 2023-09-23T23:30:23.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:30:23.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.677198+0000", 2023-09-23T23:30:23.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064155410000000003 2023-09-23T23:30:23.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.677218+0000", 2023-09-23T23:30:23.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0704999999999999e-05 2023-09-23T23:30:23.574 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.574 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.575 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.575 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46229.0:86627 12.1 e499/496 12:8287b5de:::benchmark_data_smithi110_491193_object86626:head v 499'5055, mlcod=0'0)", 2023-09-23T23:30:23.575 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:30:04.588953+0000", 2023-09-23T23:30:23.575 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.8795401, 2023-09-23T23:30:23.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.084917565, 2023-09-23T23:30:23.575 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.575 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:23.575 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.588953+0000", 2023-09-23T23:30:23.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.588953+0000", 2023-09-23T23:30:23.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.588953+0000", 2023-09-23T23:30:23.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:30:23.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589015+0000", 2023-09-23T23:30:23.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2619000000000005e-05 2023-09-23T23:30:23.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589023+0000", 2023-09-23T23:30:23.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0930000000000003e-06 2023-09-23T23:30:23.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589028+0000", 2023-09-23T23:30:23.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4839999999999998e-06 2023-09-23T23:30:23.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589873+0000", 2023-09-23T23:30:23.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00084538500000000004 2023-09-23T23:30:23.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589892+0000", 2023-09-23T23:30:23.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9038e-05 2023-09-23T23:30:23.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.579 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.579 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:30:23.579 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.673836+0000", 2023-09-23T23:30:23.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.083943852999999999 2023-09-23T23:30:23.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.579 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.579 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.579 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.673871+0000", 2023-09-23T23:30:23.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4733000000000001e-05 2023-09-23T23:30:23.579 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.579 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.580 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.580 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46229.0:86629 12.5 e499/496 12:a7a7ff25:::benchmark_data_smithi110_491193_object86628:head v 499'5136, mlcod=0'0)", 2023-09-23T23:30:23.580 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:30:04.589033+0000", 2023-09-23T23:30:23.580 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.879460430999998, 2023-09-23T23:30:23.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.077358825000000006, 2023-09-23T23:30:23.580 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.580 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:23.580 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589033+0000", 2023-09-23T23:30:23.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589033+0000", 2023-09-23T23:30:23.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589033+0000", 2023-09-23T23:30:23.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:30:23.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.582 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589069+0000", 2023-09-23T23:30:23.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6514999999999999e-05 2023-09-23T23:30:23.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.582 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589077+0000", 2023-09-23T23:30:23.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4739999999999999e-06 2023-09-23T23:30:23.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.582 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589080+0000", 2023-09-23T23:30:23.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6370000000000001e-06 2023-09-23T23:30:23.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.591564+0000", 2023-09-23T23:30:23.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024831979999999998 2023-09-23T23:30:23.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.591587+0000", 2023-09-23T23:30:23.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3730000000000001e-05 2023-09-23T23:30:23.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:30:23.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.666353+0000", 2023-09-23T23:30:23.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.074765503999999997 2023-09-23T23:30:23.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.666392+0000", 2023-09-23T23:30:23.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9088999999999998e-05 2023-09-23T23:30:23.584 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.584 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.584 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.584 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46229.0:86623 12.14 e499/496 12:2e4823d4:::benchmark_data_smithi110_491193_object86622:head v 499'4892, mlcod=0'0)", 2023-09-23T23:30:23.584 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:30:04.589093+0000", 2023-09-23T23:30:23.585 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.879400872000001, 2023-09-23T23:30:23.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.077247788999999997, 2023-09-23T23:30:23.585 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.585 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:23.585 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589093+0000", 2023-09-23T23:30:23.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589093+0000", 2023-09-23T23:30:23.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589092+0000", 2023-09-23T23:30:23.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967296 2023-09-23T23:30:23.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589139+0000", 2023-09-23T23:30:23.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6715999999999998e-05 2023-09-23T23:30:23.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589145+0000", 2023-09-23T23:30:23.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1249999999999998e-06 2023-09-23T23:30:23.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589149+0000", 2023-09-23T23:30:23.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5810000000000001e-06 2023-09-23T23:30:23.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589203+0000", 2023-09-23T23:30:23.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4135000000000003e-05 2023-09-23T23:30:23.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.589227+0000", 2023-09-23T23:30:23.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3989e-05 2023-09-23T23:30:23.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:30:23.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.593554+0000", 2023-09-23T23:30:23.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043268780000000001 2023-09-23T23:30:23.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.666340+0000", 2023-09-23T23:30:23.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.072786587999999999 2023-09-23T23:30:23.589 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.589 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.589 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.589 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46229.0:86625 12.19 e499/496 12:9f332bf7:::benchmark_data_smithi110_491193_object86624:head v 499'4907, mlcod=0'0)", 2023-09-23T23:30:23.589 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:30:04.594159+0000", 2023-09-23T23:30:23.589 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.874334066999999, 2023-09-23T23:30:23.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.079735552000000001, 2023-09-23T23:30:23.590 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.590 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:23.590 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.594159+0000", 2023-09-23T23:30:23.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.594159+0000", 2023-09-23T23:30:23.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.594158+0000", 2023-09-23T23:30:23.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:30:23.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.594191+0000", 2023-09-23T23:30:23.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2431999999999997e-05 2023-09-23T23:30:23.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.666296+0000", 2023-09-23T23:30:23.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.072105388000000006 2023-09-23T23:30:23.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.666311+0000", 2023-09-23T23:30:23.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5375000000000001e-05 2023-09-23T23:30:23.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.666355+0000", 2023-09-23T23:30:23.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3303000000000002e-05 2023-09-23T23:30:23.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.666387+0000", 2023-09-23T23:30:23.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1875999999999998e-05 2023-09-23T23:30:23.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:30:23.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.673875+0000", 2023-09-23T23:30:23.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0074887 2023-09-23T23:30:23.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:04.673895+0000", 2023-09-23T23:30:23.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9612000000000001e-05 2023-09-23T23:30:23.594 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.594 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.594 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.595 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46229.0:88653 12.f e500/496 12:fa1cfd88:::benchmark_data_smithi110_491193_object88652:head v 500'5546, mlcod=0'0)", 2023-09-23T23:30:23.595 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:30:07.474506+0000", 2023-09-23T23:30:23.595 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 15.993987262999999, 2023-09-23T23:30:23.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.075000174000000003, 2023-09-23T23:30:23.595 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.595 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:23.595 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:07.474506+0000", 2023-09-23T23:30:23.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:07.474506+0000", 2023-09-23T23:30:23.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:07.474505+0000", 2023-09-23T23:30:23.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:30:23.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:07.474574+0000", 2023-09-23T23:30:23.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8901999999999999e-05 2023-09-23T23:30:23.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:07.516775+0000", 2023-09-23T23:30:23.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.042201477000000001 2023-09-23T23:30:23.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:07.516785+0000", 2023-09-23T23:30:23.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8079999999999999e-06 2023-09-23T23:30:23.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:07.516838+0000", 2023-09-23T23:30:23.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2598e-05 2023-09-23T23:30:23.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:07.516868+0000", 2023-09-23T23:30:23.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0292000000000001e-05 2023-09-23T23:30:23.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:30:23.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:07.549466+0000", 2023-09-23T23:30:23.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.032598275000000003 2023-09-23T23:30:23.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:07.549506+0000", 2023-09-23T23:30:23.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9941999999999998e-05 2023-09-23T23:30:23.599 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.599 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.599 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.599 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(12.0 500/496 [PushOp(12:031fe061:::benchmark_data_smithi110_491193_object49307:head, version: 486'3146, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:031fe061:::benchmark_data_smithi110_491193_object49307:head@486'3146, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:30:23.599 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:30:09.289318+0000", 2023-09-23T23:30:23.599 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 14.179175364000001, 2023-09-23T23:30:23.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.15992067300000001, 2023-09-23T23:30:23.600 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:30:23.600 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:30:23.600 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:30:23.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:30:23.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:09.289318+0000", 2023-09-23T23:30:23.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:30:23.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:09.289318+0000", 2023-09-23T23:30:23.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:30:23.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:30:23.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:09.289317+0000", 2023-09-23T23:30:23.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:30:23.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:30:23.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:09.289367+0000", 2023-09-23T23:30:23.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0658999999999997e-05 2023-09-23T23:30:23.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:30:23.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:09.289395+0000", 2023-09-23T23:30:23.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7535999999999999e-05 2023-09-23T23:30:23.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:30:23.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:09.289412+0000", 2023-09-23T23:30:23.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6566999999999999e-05 2023-09-23T23:30:23.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:30:23.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:09.289550+0000", 2023-09-23T23:30:23.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013794 2023-09-23T23:30:23.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:30:23.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:09.289562+0000", 2023-09-23T23:30:23.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2439999999999999e-05 2023-09-23T23:30:23.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:30:23.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:30:23.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:30:23.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:30:09.449239+0000", 2023-09-23T23:30:23.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.159676764 2023-09-23T23:30:23.603 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.603 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.603 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.603 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:30:23.603 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:30:23.603 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:30:23.788 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:30:23.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:23.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:30:23.892 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:30:24.005 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:24.056 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:30:24.122 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:30:24.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:24.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:30:24.233 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:24.262 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:24.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:24 smithi138 ceph-mon[193606]: from='client.36294 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:30:24.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:24 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 20115/274962 objects degraded (7.316%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:24.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:24 smithi110 ceph-mon[212939]: from='client.36294 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:30:24.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:24 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 20115/274962 objects degraded (7.316%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:24.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:24 smithi110 ceph-mon[221102]: from='client.36294 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:30:24.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:24 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 20115/274962 objects degraded (7.316%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:24.579 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:24.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:24.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:24.669 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:30:24.683 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:30:24.718 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:24.782 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:30:24.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:24.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:30:24.894 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:30:24.976 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:25.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:25 smithi138 ceph-mon[193606]: pgmap v1378: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 125 B/s rd, 11 MiB/s wr, 641 op/s; 20115/273525 objects degraded (7.354%); 15315/273525 objects misplaced (5.599%); 8.5 MiB/s, 136 objects/s recovering 2023-09-23T23:30:25.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:25 smithi110 ceph-mon[212939]: pgmap v1378: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 125 B/s rd, 11 MiB/s wr, 641 op/s; 20115/273525 objects degraded (7.354%); 15315/273525 objects misplaced (5.599%); 8.5 MiB/s, 136 objects/s recovering 2023-09-23T23:30:25.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:25 smithi110 ceph-mon[221102]: pgmap v1378: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 125 B/s rd, 11 MiB/s wr, 641 op/s; 20115/273525 objects degraded (7.354%); 15315/273525 objects misplaced (5.599%); 8.5 MiB/s, 136 objects/s recovering 2023-09-23T23:30:25.757 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:30:25.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:25.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:30:25.869 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:30:26.015 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:26.296 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:30:26.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:26.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:30:26.328 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1377,"stamp":"2023-09-23T23:30:21.768508+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6005607457,"num_objects":91654,"num_object_clones":0,"num_object_copies":274962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":20115,"num_objects_misplaced":15315,"num_objects_unfound":0,"num_objects_dirty":91654,"num_whiteouts":0,"num_read":12,"num_read_kb":21,"num_write":98182,"num_write_kb":6071893,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4276,"num_bytes_recovered":278119912,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":146525,"ondisk_log_size":146525,"up":387,"acting":377,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":405,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":23763588,"kb_used_data":18192900,"kb_used_omap":29,"kb_used_meta":5570466,"kb_avail":726197628,"statfs":{"total":767960285184,"available":743626371072,"internally_reserved":0,"allocated":18629529600,"data_stored":18614858690,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5704157907},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":43712536,"num_objects":668,"num_object_clones":0,"num_object_copies":2004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2472,"num_objects_misplaced":133,"num_objects_unfound":0,"num_objects_dirty":668,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7154,"num_write_kb":250241,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1939,"num_bytes_recovered":127074304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002367"},"pg_stats":[{"pgid":"12.15","version":"502'5445","reported_seq":5491,"reported_epoch":502,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:30:19.782668+0000","last_change":"2023-09-23T23:29:56.386966+0000","last_active":"2023-09-23T23:30:19.782668+0000","last_peered":"2023-09-23T23:30:19.782668+0000","last_clean":"2023-09-23T23:29:52.317573+0000","last_became_active":"2023-09-23T23:29:56.373989+0000","last_became_peered":"2023-09-23T23:29:56.373989+0000","last_unstale":"2023-09-23T23:30:19.782668+0000","last_undegraded":"2023-09-23T23:29:56.182909+0000","last_fullsized":"2023-09-23T23:29:56.153881+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5445,"log_dups_size":0,"ondisk_log_size":5445,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191758336,"num_objects":2926,"num_object_clones":0,"num_object_copies":8778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2564,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3098,"num_write_kb":192768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[1,2],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2","objects":362},{"shards":"1,2,3","objects":2478}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":698,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.653840+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:30:15.653840+0000","last_peered":"2023-09-23T23:30:15.653840+0000","last_clean":"2023-09-23T23:30:15.653840+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:30:15.653840+0000","last_undegraded":"2023-09-23T23:30:15.653840+0000","last_fullsized":"2023-09-23T23:30:15.653840+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":536,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.647360+0000","last_change":"2023-09-23T23:22:38.439629+0000","last_active":"2023-09-23T23:30:15.647360+0000","last_peered":"2023-09-23T23:30:15.647360+0000","last_clean":"2023-09-23T23:30:15.647360+0000","last_became_active":"2023-09-23T23:22:38.439285+0000","last_became_peered":"2023-09-23T23:22:38.439285+0000","last_unstale":"2023-09-23T23:30:15.647360+0000","last_undegraded":"2023-09-23T23:30:15.647360+0000","last_fullsized":"2023-09-23T23:30:15.647360+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:34.431221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":472,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650171+0000","last_change":"2023-09-23T23:23:43.511912+0000","last_active":"2023-09-23T23:30:15.650171+0000","last_peered":"2023-09-23T23:30:15.650171+0000","last_clean":"2023-09-23T23:30:15.650171+0000","last_became_active":"2023-09-23T23:23:43.511574+0000","last_became_peered":"2023-09-23T23:23:43.511574+0000","last_unstale":"2023-09-23T23:30:15.650171+0000","last_undegraded":"2023-09-23T23:30:15.650171+0000","last_fullsized":"2023-09-23T23:30:15.650171+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:16.286926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.16","version":"502'5277","reported_seq":5322,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:19.880552+0000","last_change":"2023-09-23T23:29:45.671198+0000","last_active":"2023-09-23T23:30:19.880552+0000","last_peered":"2023-09-23T23:30:19.880552+0000","last_clean":"2023-09-23T23:30:19.880552+0000","last_became_active":"2023-09-23T23:29:45.669030+0000","last_became_peered":"2023-09-23T23:29:45.669030+0000","last_unstale":"2023-09-23T23:30:19.880552+0000","last_undegraded":"2023-09-23T23:30:19.880552+0000","last_fullsized":"2023-09-23T23:30:19.880552+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5277,"log_dups_size":0,"ondisk_log_size":5277,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185040896,"num_objects":2824,"num_object_clones":0,"num_object_copies":8472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3060,"num_write_kb":188256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":591,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.654037+0000","last_change":"2023-09-23T23:23:50.459634+0000","last_active":"2023-09-23T23:30:15.654037+0000","last_peered":"2023-09-23T23:30:15.654037+0000","last_clean":"2023-09-23T23:30:15.654037+0000","last_became_active":"2023-09-23T23:23:45.453958+0000","last_became_peered":"2023-09-23T23:23:45.453958+0000","last_unstale":"2023-09-23T23:30:15.654037+0000","last_undegraded":"2023-09-23T23:30:15.654037+0000","last_fullsized":"2023-09-23T23:30:15.654037+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:56.563201+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":213,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.672496+0000","last_change":"2023-09-23T23:26:11.528846+0000","last_active":"2023-09-23T23:30:15.672496+0000","last_peered":"2023-09-23T23:30:15.672496+0000","last_clean":"2023-09-23T23:30:15.672496+0000","last_became_active":"2023-09-23T23:26:11.528529+0000","last_became_peered":"2023-09-23T23:26:11.528529+0000","last_unstale":"2023-09-23T23:30:15.672496+0000","last_undegraded":"2023-09-23T23:30:15.672496+0000","last_fullsized":"2023-09-23T23:30:15.672496+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:35.094583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":467,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651614+0000","last_change":"2023-09-23T23:28:22.604718+0000","last_active":"2023-09-23T23:30:15.651614+0000","last_peered":"2023-09-23T23:30:15.651614+0000","last_clean":"2023-09-23T23:30:15.651614+0000","last_became_active":"2023-09-23T23:28:22.604498+0000","last_became_peered":"2023-09-23T23:28:22.604498+0000","last_unstale":"2023-09-23T23:30:15.651614+0000","last_undegraded":"2023-09-23T23:30:15.651614+0000","last_fullsized":"2023-09-23T23:30:15.651614+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:30.472412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.17","version":"502'5302","reported_seq":5347,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:19.884042+0000","last_change":"2023-09-23T23:29:45.685688+0000","last_active":"2023-09-23T23:30:19.884042+0000","last_peered":"2023-09-23T23:30:19.884042+0000","last_clean":"2023-09-23T23:30:19.884042+0000","last_became_active":"2023-09-23T23:29:45.681573+0000","last_became_peered":"2023-09-23T23:29:45.681573+0000","last_unstale":"2023-09-23T23:30:19.884042+0000","last_undegraded":"2023-09-23T23:30:19.884042+0000","last_fullsized":"2023-09-23T23:30:19.884042+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5302,"log_dups_size":0,"ondisk_log_size":5302,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186023936,"num_objects":2839,"num_object_clones":0,"num_object_copies":8517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3017,"num_write_kb":187360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":805,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.647272+0000","last_change":"2023-09-23T23:28:05.634824+0000","last_active":"2023-09-23T23:30:15.647272+0000","last_peered":"2023-09-23T23:30:15.647272+0000","last_clean":"2023-09-23T23:30:15.647272+0000","last_became_active":"2023-09-23T23:27:23.551046+0000","last_became_peered":"2023-09-23T23:27:23.551046+0000","last_unstale":"2023-09-23T23:30:15.647272+0000","last_undegraded":"2023-09-23T23:30:15.647272+0000","last_fullsized":"2023-09-23T23:30:15.647272+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:59.062239+0000","scrub_duration":0.0034158959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":500,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:16.658439+0000","last_change":"2023-09-23T23:27:20.554007+0000","last_active":"2023-09-23T23:30:16.658439+0000","last_peered":"2023-09-23T23:30:16.658439+0000","last_clean":"2023-09-23T23:30:16.658439+0000","last_became_active":"2023-09-23T23:27:20.553716+0000","last_became_peered":"2023-09-23T23:27:20.553716+0000","last_unstale":"2023-09-23T23:30:16.658439+0000","last_undegraded":"2023-09-23T23:30:16.658439+0000","last_fullsized":"2023-09-23T23:30:16.658439+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:38.911862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":16,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651592+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:30:15.651592+0000","last_peered":"2023-09-23T23:30:15.651592+0000","last_clean":"2023-09-23T23:30:15.651592+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:30:15.651592+0000","last_undegraded":"2023-09-23T23:30:15.651592+0000","last_fullsized":"2023-09-23T23:30:15.651592+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.18","version":"502'5275","reported_seq":5320,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:18.816109+0000","last_change":"2023-09-23T23:29:45.694505+0000","last_active":"2023-09-23T23:30:18.816109+0000","last_peered":"2023-09-23T23:30:18.816109+0000","last_clean":"2023-09-23T23:30:18.816109+0000","last_became_active":"2023-09-23T23:29:45.692613+0000","last_became_peered":"2023-09-23T23:29:45.692613+0000","last_unstale":"2023-09-23T23:30:18.816109+0000","last_undegraded":"2023-09-23T23:30:18.816109+0000","last_fullsized":"2023-09-23T23:30:18.816109+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5275,"log_dups_size":0,"ondisk_log_size":5275,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185139200,"num_objects":2825,"num_object_clones":0,"num_object_copies":8475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3005,"num_write_kb":186560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":822,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650544+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:30:15.650544+0000","last_peered":"2023-09-23T23:30:15.650544+0000","last_clean":"2023-09-23T23:30:15.650544+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:30:15.650544+0000","last_undegraded":"2023-09-23T23:30:15.650544+0000","last_fullsized":"2023-09-23T23:30:15.650544+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":134,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.654715+0000","last_change":"2023-09-23T23:28:22.602481+0000","last_active":"2023-09-23T23:30:15.654715+0000","last_peered":"2023-09-23T23:30:15.654715+0000","last_clean":"2023-09-23T23:30:15.654715+0000","last_became_active":"2023-09-23T23:28:22.602282+0000","last_became_peered":"2023-09-23T23:28:22.602282+0000","last_unstale":"2023-09-23T23:30:15.654715+0000","last_undegraded":"2023-09-23T23:30:15.654715+0000","last_fullsized":"2023-09-23T23:30:15.654715+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:39.906003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":500,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650971+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:30:15.650971+0000","last_peered":"2023-09-23T23:30:15.650971+0000","last_clean":"2023-09-23T23:30:15.650971+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:30:15.650971+0000","last_undegraded":"2023-09-23T23:30:15.650971+0000","last_fullsized":"2023-09-23T23:30:15.650971+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.19","version":"502'5266","reported_seq":5318,"reported_epoch":502,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:30:21.071096+0000","last_change":"2023-09-23T23:29:56.185457+0000","last_active":"2023-09-23T23:30:21.071096+0000","last_peered":"2023-09-23T23:30:21.071096+0000","last_clean":"2023-09-23T23:29:52.968255+0000","last_became_active":"2023-09-23T23:29:56.180638+0000","last_became_peered":"2023-09-23T23:29:56.180638+0000","last_unstale":"2023-09-23T23:30:21.071096+0000","last_undegraded":"2023-09-23T23:29:56.000553+0000","last_fullsized":"2023-09-23T23:29:55.978433+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5266,"log_dups_size":0,"ondisk_log_size":5266,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182091776,"num_objects":2779,"num_object_clones":0,"num_object_copies":8337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2494,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3031,"num_write_kb":185888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,3],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2,3","objects":2368},{"shards":"2,3","objects":285}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":775,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651990+0000","last_change":"2023-09-23T23:26:14.521792+0000","last_active":"2023-09-23T23:30:15.651990+0000","last_peered":"2023-09-23T23:30:15.651990+0000","last_clean":"2023-09-23T23:30:15.651990+0000","last_became_active":"2023-09-23T23:26:14.521438+0000","last_became_peered":"2023-09-23T23:26:14.521438+0000","last_unstale":"2023-09-23T23:30:15.651990+0000","last_undegraded":"2023-09-23T23:30:15.651990+0000","last_fullsized":"2023-09-23T23:30:15.651990+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:09.103712+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":133,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.654301+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:30:15.654301+0000","last_peered":"2023-09-23T23:30:15.654301+0000","last_clean":"2023-09-23T23:30:15.654301+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:30:15.654301+0000","last_undegraded":"2023-09-23T23:30:15.654301+0000","last_fullsized":"2023-09-23T23:30:15.654301+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":461,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.653786+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:30:15.653786+0000","last_peered":"2023-09-23T23:30:15.653786+0000","last_clean":"2023-09-23T23:30:15.653786+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:30:15.653786+0000","last_undegraded":"2023-09-23T23:30:15.653786+0000","last_fullsized":"2023-09-23T23:30:15.653786+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":916,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.654633+0000","last_change":"2023-09-23T23:29:58.260674+0000","last_active":"2023-09-23T23:30:15.654633+0000","last_peered":"2023-09-23T23:30:15.654633+0000","last_clean":"2023-09-23T23:30:15.654633+0000","last_became_active":"2023-09-23T23:29:58.260318+0000","last_became_peered":"2023-09-23T23:29:58.260318+0000","last_unstale":"2023-09-23T23:30:15.654633+0000","last_undegraded":"2023-09-23T23:30:15.654633+0000","last_fullsized":"2023-09-23T23:30:15.654633+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:08.810582+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":107,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.649962+0000","last_change":"2023-09-23T23:26:48.669857+0000","last_active":"2023-09-23T23:30:15.649962+0000","last_peered":"2023-09-23T23:30:15.649962+0000","last_clean":"2023-09-23T23:30:15.649962+0000","last_became_active":"2023-09-23T23:26:48.538246+0000","last_became_peered":"2023-09-23T23:26:48.538246+0000","last_unstale":"2023-09-23T23:30:15.649962+0000","last_undegraded":"2023-09-23T23:30:15.649962+0000","last_fullsized":"2023-09-23T23:30:15.649962+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:19.039472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":98,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.670002+0000","last_change":"2023-09-23T23:29:54.996097+0000","last_active":"2023-09-23T23:30:15.670002+0000","last_peered":"2023-09-23T23:30:15.670002+0000","last_clean":"2023-09-23T23:30:15.670002+0000","last_became_active":"2023-09-23T23:29:54.995756+0000","last_became_peered":"2023-09-23T23:29:54.995756+0000","last_unstale":"2023-09-23T23:30:15.670002+0000","last_undegraded":"2023-09-23T23:30:15.670002+0000","last_fullsized":"2023-09-23T23:30:15.670002+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:15.856474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":577,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.669617+0000","last_change":"2023-09-23T23:24:02.268079+0000","last_active":"2023-09-23T23:30:15.669617+0000","last_peered":"2023-09-23T23:30:15.669617+0000","last_clean":"2023-09-23T23:30:15.669617+0000","last_became_active":"2023-09-23T23:23:45.463106+0000","last_became_peered":"2023-09-23T23:23:45.463106+0000","last_unstale":"2023-09-23T23:30:15.669617+0000","last_undegraded":"2023-09-23T23:30:15.669617+0000","last_fullsized":"2023-09-23T23:30:15.669617+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.128813+0000","scrub_duration":0.0018535310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":153,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.647274+0000","last_change":"2023-09-23T23:29:55.012082+0000","last_active":"2023-09-23T23:30:15.647274+0000","last_peered":"2023-09-23T23:30:15.647274+0000","last_clean":"2023-09-23T23:30:15.647274+0000","last_became_active":"2023-09-23T23:29:55.000202+0000","last_became_peered":"2023-09-23T23:29:55.000202+0000","last_unstale":"2023-09-23T23:30:15.647274+0000","last_undegraded":"2023-09-23T23:30:15.647274+0000","last_fullsized":"2023-09-23T23:30:15.647274+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:28.669994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":500,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.648038+0000","last_change":"2023-09-23T23:29:55.004242+0000","last_active":"2023-09-23T23:30:15.648038+0000","last_peered":"2023-09-23T23:30:15.648038+0000","last_clean":"2023-09-23T23:30:15.648038+0000","last_became_active":"2023-09-23T23:29:55.003934+0000","last_became_peered":"2023-09-23T23:29:55.003934+0000","last_unstale":"2023-09-23T23:30:15.648038+0000","last_undegraded":"2023-09-23T23:30:15.648038+0000","last_fullsized":"2023-09-23T23:30:15.648038+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:09.283771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":617,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.653886+0000","last_change":"2023-09-23T23:28:14.378358+0000","last_active":"2023-09-23T23:30:15.653886+0000","last_peered":"2023-09-23T23:30:15.653886+0000","last_clean":"2023-09-23T23:30:15.653886+0000","last_became_active":"2023-09-23T23:26:34.550151+0000","last_became_peered":"2023-09-23T23:26:34.550151+0000","last_unstale":"2023-09-23T23:30:15.653886+0000","last_undegraded":"2023-09-23T23:30:15.653886+0000","last_fullsized":"2023-09-23T23:30:15.653886+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:47.794690+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":434,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650790+0000","last_change":"2023-09-23T23:26:11.569228+0000","last_active":"2023-09-23T23:30:15.650790+0000","last_peered":"2023-09-23T23:30:15.650790+0000","last_clean":"2023-09-23T23:30:15.650790+0000","last_became_active":"2023-09-23T23:26:11.513159+0000","last_became_peered":"2023-09-23T23:26:11.513159+0000","last_unstale":"2023-09-23T23:30:15.650790+0000","last_undegraded":"2023-09-23T23:30:15.650790+0000","last_fullsized":"2023-09-23T23:30:15.650790+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:40.698257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":499,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651708+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:30:15.651708+0000","last_peered":"2023-09-23T23:30:15.651708+0000","last_clean":"2023-09-23T23:30:15.651708+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:30:15.651708+0000","last_undegraded":"2023-09-23T23:30:15.651708+0000","last_fullsized":"2023-09-23T23:30:15.651708+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":861,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651061+0000","last_change":"2023-09-23T23:29:58.259420+0000","last_active":"2023-09-23T23:30:15.651061+0000","last_peered":"2023-09-23T23:30:15.651061+0000","last_clean":"2023-09-23T23:30:15.651061+0000","last_became_active":"2023-09-23T23:29:58.258879+0000","last_became_peered":"2023-09-23T23:29:58.258879+0000","last_unstale":"2023-09-23T23:30:15.651061+0000","last_undegraded":"2023-09-23T23:30:15.651061+0000","last_fullsized":"2023-09-23T23:30:15.651061+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:14.307363+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":101,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651918+0000","last_change":"2023-09-23T23:26:48.563723+0000","last_active":"2023-09-23T23:30:15.651918+0000","last_peered":"2023-09-23T23:30:15.651918+0000","last_clean":"2023-09-23T23:30:15.651918+0000","last_became_active":"2023-09-23T23:26:48.563340+0000","last_became_peered":"2023-09-23T23:26:48.563340+0000","last_unstale":"2023-09-23T23:30:15.651918+0000","last_undegraded":"2023-09-23T23:30:15.651918+0000","last_fullsized":"2023-09-23T23:30:15.651918+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:05.974131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":388,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650463+0000","last_change":"2023-09-23T23:26:32.040316+0000","last_active":"2023-09-23T23:30:15.650463+0000","last_peered":"2023-09-23T23:30:15.650463+0000","last_clean":"2023-09-23T23:30:15.650463+0000","last_became_active":"2023-09-23T23:26:32.039758+0000","last_became_peered":"2023-09-23T23:26:32.039758+0000","last_unstale":"2023-09-23T23:30:15.650463+0000","last_undegraded":"2023-09-23T23:30:15.650463+0000","last_fullsized":"2023-09-23T23:30:15.650463+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:03.930331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":666,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.670559+0000","last_change":"2023-09-23T23:26:14.527752+0000","last_active":"2023-09-23T23:30:15.670559+0000","last_peered":"2023-09-23T23:30:15.670559+0000","last_clean":"2023-09-23T23:30:15.670559+0000","last_became_active":"2023-09-23T23:26:14.525188+0000","last_became_peered":"2023-09-23T23:26:14.525188+0000","last_unstale":"2023-09-23T23:30:15.670559+0000","last_undegraded":"2023-09-23T23:30:15.670559+0000","last_fullsized":"2023-09-23T23:30:15.670559+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:10.617744+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":366,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651302+0000","last_change":"2023-09-23T23:29:54.999839+0000","last_active":"2023-09-23T23:30:15.651302+0000","last_peered":"2023-09-23T23:30:15.651302+0000","last_clean":"2023-09-23T23:30:15.651302+0000","last_became_active":"2023-09-23T23:29:54.999530+0000","last_became_peered":"2023-09-23T23:29:54.999530+0000","last_unstale":"2023-09-23T23:30:15.651302+0000","last_undegraded":"2023-09-23T23:30:15.651302+0000","last_fullsized":"2023-09-23T23:30:15.651302+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:43.757346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":437,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650499+0000","last_change":"2023-09-23T23:28:22.603836+0000","last_active":"2023-09-23T23:30:15.650499+0000","last_peered":"2023-09-23T23:30:15.650499+0000","last_clean":"2023-09-23T23:30:15.650499+0000","last_became_active":"2023-09-23T23:28:22.603490+0000","last_became_peered":"2023-09-23T23:28:22.603490+0000","last_unstale":"2023-09-23T23:30:15.650499+0000","last_undegraded":"2023-09-23T23:30:15.650499+0000","last_fullsized":"2023-09-23T23:30:15.650499+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:00.869241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":865,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.654082+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:30:15.654082+0000","last_peered":"2023-09-23T23:30:15.654082+0000","last_clean":"2023-09-23T23:30:15.654082+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:30:15.654082+0000","last_undegraded":"2023-09-23T23:30:15.654082+0000","last_fullsized":"2023-09-23T23:30:15.654082+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":26,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650461+0000","last_change":"2023-09-23T23:29:55.020386+0000","last_active":"2023-09-23T23:30:15.650461+0000","last_peered":"2023-09-23T23:30:15.650461+0000","last_clean":"2023-09-23T23:30:15.650461+0000","last_became_active":"2023-09-23T23:29:54.991929+0000","last_became_peered":"2023-09-23T23:29:54.991929+0000","last_unstale":"2023-09-23T23:30:15.650461+0000","last_undegraded":"2023-09-23T23:30:15.650461+0000","last_fullsized":"2023-09-23T23:30:15.650461+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:25.161647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":140,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650856+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:30:15.650856+0000","last_peered":"2023-09-23T23:30:15.650856+0000","last_clean":"2023-09-23T23:30:15.650856+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:30:15.650856+0000","last_undegraded":"2023-09-23T23:30:15.650856+0000","last_fullsized":"2023-09-23T23:30:15.650856+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"502'5382","reported_seq":10559,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:18.865001+0000","last_change":"2023-09-23T23:30:16.655105+0000","last_active":"2023-09-23T23:30:18.865001+0000","last_peered":"2023-09-23T23:30:18.865001+0000","last_clean":"2023-09-23T23:30:18.865001+0000","last_became_active":"2023-09-23T23:30:16.652602+0000","last_became_peered":"2023-09-23T23:30:16.652602+0000","last_unstale":"2023-09-23T23:30:18.865001+0000","last_undegraded":"2023-09-23T23:30:18.865001+0000","last_fullsized":"2023-09-23T23:30:18.865001+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5382,"log_dups_size":0,"ondisk_log_size":5382,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189005824,"num_objects":2884,"num_object_clones":0,"num_object_copies":8652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3064,"num_write_kb":190336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2556,"num_bytes_recovered":167510016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":829,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650053+0000","last_change":"2023-09-23T23:26:51.563868+0000","last_active":"2023-09-23T23:30:15.650053+0000","last_peered":"2023-09-23T23:30:15.650053+0000","last_clean":"2023-09-23T23:30:15.650053+0000","last_became_active":"2023-09-23T23:26:51.563510+0000","last_became_peered":"2023-09-23T23:26:51.563510+0000","last_unstale":"2023-09-23T23:30:15.650053+0000","last_undegraded":"2023-09-23T23:30:15.650053+0000","last_fullsized":"2023-09-23T23:30:15.650053+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:34.244822+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":105,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651781+0000","last_change":"2023-09-23T23:26:48.673221+0000","last_active":"2023-09-23T23:30:15.651781+0000","last_peered":"2023-09-23T23:30:15.651781+0000","last_clean":"2023-09-23T23:30:15.651781+0000","last_became_active":"2023-09-23T23:26:48.555920+0000","last_became_peered":"2023-09-23T23:26:48.555920+0000","last_unstale":"2023-09-23T23:30:15.651781+0000","last_undegraded":"2023-09-23T23:30:15.651781+0000","last_fullsized":"2023-09-23T23:30:15.651781+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:26.586824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":133,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.653776+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:30:15.653776+0000","last_peered":"2023-09-23T23:30:15.653776+0000","last_clean":"2023-09-23T23:30:15.653776+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:30:15.653776+0000","last_undegraded":"2023-09-23T23:30:15.653776+0000","last_fullsized":"2023-09-23T23:30:15.653776+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"502'5407","reported_seq":5457,"reported_epoch":502,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T23:30:19.800426+0000","last_change":"2023-09-23T23:29:56.405351+0000","last_active":"2023-09-23T23:30:19.800426+0000","last_peered":"2023-09-23T23:30:19.800426+0000","last_clean":"2023-09-23T23:29:52.292686+0000","last_became_active":"2023-09-23T23:29:56.397141+0000","last_became_peered":"2023-09-23T23:29:56.397141+0000","last_unstale":"2023-09-23T23:30:19.800426+0000","last_undegraded":"2023-09-23T23:30:19.800426+0000","last_fullsized":"2023-09-23T23:30:19.800426+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5407,"log_dups_size":0,"ondisk_log_size":5407,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189726720,"num_objects":2895,"num_object_clones":0,"num_object_copies":8685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2496,"num_objects_unfound":0,"num_objects_dirty":2895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3081,"num_write_kb":191232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[1,3,7],"avail_no_missing":["1","3","7"],"object_location_counts":[{"shards":"1,3,7","objects":399}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":898,"reported_epoch":501,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:30:15.651278+0000","last_change":"2023-09-23T23:29:56.080936+0000","last_active":"2023-09-23T23:30:15.651278+0000","last_peered":"2023-09-23T23:30:15.651278+0000","last_clean":"2023-09-23T23:29:53.882798+0000","last_became_active":"2023-09-23T23:29:56.080613+0000","last_became_peered":"2023-09-23T23:29:56.080613+0000","last_unstale":"2023-09-23T23:30:15.651278+0000","last_undegraded":"2023-09-23T23:29:55.974614+0000","last_fullsized":"2023-09-23T23:29:55.974165+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:14.089938+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3,4","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":105,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651851+0000","last_change":"2023-09-23T23:26:48.547146+0000","last_active":"2023-09-23T23:30:15.651851+0000","last_peered":"2023-09-23T23:30:15.651851+0000","last_clean":"2023-09-23T23:30:15.651851+0000","last_became_active":"2023-09-23T23:26:48.537682+0000","last_became_peered":"2023-09-23T23:26:48.537682+0000","last_unstale":"2023-09-23T23:30:15.651851+0000","last_undegraded":"2023-09-23T23:30:15.651851+0000","last_fullsized":"2023-09-23T23:30:15.651851+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:10.115563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":527,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.647203+0000","last_change":"2023-09-23T23:26:01.833889+0000","last_active":"2023-09-23T23:30:15.647203+0000","last_peered":"2023-09-23T23:30:15.647203+0000","last_clean":"2023-09-23T23:30:15.647203+0000","last_became_active":"2023-09-23T23:26:01.833626+0000","last_became_peered":"2023-09-23T23:26:01.833626+0000","last_unstale":"2023-09-23T23:30:15.647203+0000","last_undegraded":"2023-09-23T23:30:15.647203+0000","last_fullsized":"2023-09-23T23:30:15.647203+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:25.659671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"502'5387","reported_seq":5435,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:18.851152+0000","last_change":"2023-09-23T23:30:15.111505+0000","last_active":"2023-09-23T23:30:18.851152+0000","last_peered":"2023-09-23T23:30:18.851152+0000","last_clean":"2023-09-23T23:30:18.851152+0000","last_became_active":"2023-09-23T23:29:45.617669+0000","last_became_peered":"2023-09-23T23:29:45.617669+0000","last_unstale":"2023-09-23T23:30:18.851152+0000","last_undegraded":"2023-09-23T23:30:18.851152+0000","last_fullsized":"2023-09-23T23:30:18.851152+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5387,"log_dups_size":0,"ondisk_log_size":5387,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190251008,"num_objects":2903,"num_object_clones":0,"num_object_copies":8709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3063,"num_write_kb":190912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":792,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651667+0000","last_change":"2023-09-23T23:29:58.262103+0000","last_active":"2023-09-23T23:30:15.651667+0000","last_peered":"2023-09-23T23:30:15.651667+0000","last_clean":"2023-09-23T23:30:15.651667+0000","last_became_active":"2023-09-23T23:29:58.261689+0000","last_became_peered":"2023-09-23T23:29:58.261689+0000","last_unstale":"2023-09-23T23:30:15.651667+0000","last_undegraded":"2023-09-23T23:30:15.651667+0000","last_fullsized":"2023-09-23T23:30:15.651667+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:44.945353+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":422,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650239+0000","last_change":"2023-09-23T23:29:55.006616+0000","last_active":"2023-09-23T23:30:15.650239+0000","last_peered":"2023-09-23T23:30:15.650239+0000","last_clean":"2023-09-23T23:30:15.650239+0000","last_became_active":"2023-09-23T23:29:55.000411+0000","last_became_peered":"2023-09-23T23:29:55.000411+0000","last_unstale":"2023-09-23T23:30:15.650239+0000","last_undegraded":"2023-09-23T23:30:15.650239+0000","last_fullsized":"2023-09-23T23:30:15.650239+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:22.629830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":16,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650651+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:30:15.650651+0000","last_peered":"2023-09-23T23:30:15.650651+0000","last_clean":"2023-09-23T23:30:15.650651+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:30:15.650651+0000","last_undegraded":"2023-09-23T23:30:15.650651+0000","last_fullsized":"2023-09-23T23:30:15.650651+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"502'5417","reported_seq":5462,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:18.049650+0000","last_change":"2023-09-23T23:29:45.642422+0000","last_active":"2023-09-23T23:30:18.049650+0000","last_peered":"2023-09-23T23:30:18.049650+0000","last_clean":"2023-09-23T23:30:18.049650+0000","last_became_active":"2023-09-23T23:29:45.637986+0000","last_became_peered":"2023-09-23T23:29:45.637986+0000","last_unstale":"2023-09-23T23:30:18.049650+0000","last_undegraded":"2023-09-23T23:30:18.049650+0000","last_fullsized":"2023-09-23T23:30:18.049650+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5417,"log_dups_size":0,"ondisk_log_size":5417,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193363968,"num_objects":2950,"num_object_clones":0,"num_object_copies":8850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3084,"num_write_kb":193120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":566,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.648297+0000","last_change":"2023-09-23T23:28:03.629593+0000","last_active":"2023-09-23T23:30:15.648297+0000","last_peered":"2023-09-23T23:30:15.648297+0000","last_clean":"2023-09-23T23:30:15.648297+0000","last_became_active":"2023-09-23T23:26:51.588385+0000","last_became_peered":"2023-09-23T23:26:51.588385+0000","last_unstale":"2023-09-23T23:30:15.648297+0000","last_undegraded":"2023-09-23T23:30:15.648297+0000","last_fullsized":"2023-09-23T23:30:15.648297+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:20.818366+0000","scrub_duration":0.0050135120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":339,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651577+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:30:15.651577+0000","last_peered":"2023-09-23T23:30:15.651577+0000","last_clean":"2023-09-23T23:30:15.651577+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:30:15.651577+0000","last_undegraded":"2023-09-23T23:30:15.651577+0000","last_fullsized":"2023-09-23T23:30:15.651577+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":487,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.648292+0000","last_change":"2023-09-23T23:28:45.066443+0000","last_active":"2023-09-23T23:30:15.648292+0000","last_peered":"2023-09-23T23:30:15.648292+0000","last_clean":"2023-09-23T23:30:15.648292+0000","last_became_active":"2023-09-23T23:28:45.066154+0000","last_became_peered":"2023-09-23T23:28:45.066154+0000","last_unstale":"2023-09-23T23:30:15.648292+0000","last_undegraded":"2023-09-23T23:30:15.648292+0000","last_fullsized":"2023-09-23T23:30:15.648292+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:51.651425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"502'5208","reported_seq":5260,"reported_epoch":502,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:30:19.819790+0000","last_change":"2023-09-23T23:29:56.383042+0000","last_active":"2023-09-23T23:30:19.819790+0000","last_peered":"2023-09-23T23:30:19.819790+0000","last_clean":"2023-09-23T23:29:52.324859+0000","last_became_active":"2023-09-23T23:29:56.375422+0000","last_became_peered":"2023-09-23T23:29:56.375422+0000","last_unstale":"2023-09-23T23:30:19.819790+0000","last_undegraded":"2023-09-23T23:29:56.180438+0000","last_fullsized":"2023-09-23T23:29:56.153796+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5208,"log_dups_size":0,"ondisk_log_size":5208,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183173120,"num_objects":2795,"num_object_clones":0,"num_object_copies":8385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2420,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2977,"num_write_kb":184704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[1,3],"avail_no_missing":["1","3","7"],"object_location_counts":[{"shards":"1,3","objects":375},{"shards":"1,3,7","objects":2329}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":837,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651394+0000","last_change":"2023-09-23T23:23:45.830384+0000","last_active":"2023-09-23T23:30:15.651394+0000","last_peered":"2023-09-23T23:30:15.651394+0000","last_clean":"2023-09-23T23:30:15.651394+0000","last_became_active":"2023-09-23T23:23:36.025724+0000","last_became_peered":"2023-09-23T23:23:36.025724+0000","last_unstale":"2023-09-23T23:30:15.651394+0000","last_undegraded":"2023-09-23T23:30:15.651394+0000","last_fullsized":"2023-09-23T23:30:15.651394+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:34.345528+0000","scrub_duration":0.001659259,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":470,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651497+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:30:15.651497+0000","last_peered":"2023-09-23T23:30:15.651497+0000","last_clean":"2023-09-23T23:30:15.651497+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:30:15.651497+0000","last_undegraded":"2023-09-23T23:30:15.651497+0000","last_fullsized":"2023-09-23T23:30:15.651497+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":480,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.647354+0000","last_change":"2023-09-23T23:29:55.001192+0000","last_active":"2023-09-23T23:30:15.647354+0000","last_peered":"2023-09-23T23:30:15.647354+0000","last_clean":"2023-09-23T23:30:15.647354+0000","last_became_active":"2023-09-23T23:29:55.000904+0000","last_became_peered":"2023-09-23T23:29:55.000904+0000","last_unstale":"2023-09-23T23:30:15.647354+0000","last_undegraded":"2023-09-23T23:30:15.647354+0000","last_fullsized":"2023-09-23T23:30:15.647354+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:29.169209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"502'5504","reported_seq":5550,"reported_epoch":502,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:30:19.813668+0000","last_change":"2023-09-23T23:29:56.386674+0000","last_active":"2023-09-23T23:30:19.813668+0000","last_peered":"2023-09-23T23:30:19.813668+0000","last_clean":"2023-09-23T23:29:52.296665+0000","last_became_active":"2023-09-23T23:29:56.375733+0000","last_became_peered":"2023-09-23T23:29:56.375733+0000","last_unstale":"2023-09-23T23:30:19.813668+0000","last_undegraded":"2023-09-23T23:29:56.189915+0000","last_fullsized":"2023-09-23T23:29:55.977445+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5504,"log_dups_size":0,"ondisk_log_size":5504,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193396736,"num_objects":2951,"num_object_clones":0,"num_object_copies":8853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3157,"num_write_kb":195456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[1,2],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2","objects":390},{"shards":"1,2,3","objects":2458}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":782,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.646254+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:30:15.646254+0000","last_peered":"2023-09-23T23:30:15.646254+0000","last_clean":"2023-09-23T23:30:15.646254+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:30:15.646254+0000","last_undegraded":"2023-09-23T23:30:15.646254+0000","last_fullsized":"2023-09-23T23:30:15.646254+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":16,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650576+0000","last_change":"2023-09-23T23:29:54.998914+0000","last_active":"2023-09-23T23:30:15.650576+0000","last_peered":"2023-09-23T23:30:15.650576+0000","last_clean":"2023-09-23T23:30:15.650576+0000","last_became_active":"2023-09-23T23:29:54.998620+0000","last_became_peered":"2023-09-23T23:29:54.998620+0000","last_unstale":"2023-09-23T23:30:15.650576+0000","last_undegraded":"2023-09-23T23:30:15.650576+0000","last_fullsized":"2023-09-23T23:30:15.650576+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:32.702628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":123,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.647483+0000","last_change":"2023-09-23T23:26:11.516475+0000","last_active":"2023-09-23T23:30:15.647483+0000","last_peered":"2023-09-23T23:30:15.647483+0000","last_clean":"2023-09-23T23:30:15.647483+0000","last_became_active":"2023-09-23T23:26:11.516198+0000","last_became_peered":"2023-09-23T23:26:11.516198+0000","last_unstale":"2023-09-23T23:30:15.647483+0000","last_undegraded":"2023-09-23T23:30:15.647483+0000","last_fullsized":"2023-09-23T23:30:15.647483+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:56.875103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"502'5231","reported_seq":5276,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:19.828283+0000","last_change":"2023-09-23T23:29:45.635904+0000","last_active":"2023-09-23T23:30:19.828283+0000","last_peered":"2023-09-23T23:30:19.828283+0000","last_clean":"2023-09-23T23:30:19.828283+0000","last_became_active":"2023-09-23T23:29:45.633590+0000","last_became_peered":"2023-09-23T23:29:45.633590+0000","last_unstale":"2023-09-23T23:30:19.828283+0000","last_undegraded":"2023-09-23T23:30:19.828283+0000","last_fullsized":"2023-09-23T23:30:19.828283+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5231,"log_dups_size":0,"ondisk_log_size":5231,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181370880,"num_objects":2767,"num_object_clones":0,"num_object_copies":8301,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2767,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3013,"num_write_kb":184992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":658,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.670367+0000","last_change":"2023-09-23T23:26:34.548211+0000","last_active":"2023-09-23T23:30:15.670367+0000","last_peered":"2023-09-23T23:30:15.670367+0000","last_clean":"2023-09-23T23:30:15.670367+0000","last_became_active":"2023-09-23T23:26:34.547815+0000","last_became_peered":"2023-09-23T23:26:34.547815+0000","last_unstale":"2023-09-23T23:30:15.670367+0000","last_undegraded":"2023-09-23T23:30:15.670367+0000","last_fullsized":"2023-09-23T23:30:15.670367+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:37.675417+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":123,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.647077+0000","last_change":"2023-09-23T23:26:11.512815+0000","last_active":"2023-09-23T23:30:15.647077+0000","last_peered":"2023-09-23T23:30:15.647077+0000","last_clean":"2023-09-23T23:30:15.647077+0000","last_became_active":"2023-09-23T23:26:11.512466+0000","last_became_peered":"2023-09-23T23:26:11.512466+0000","last_unstale":"2023-09-23T23:30:15.647077+0000","last_undegraded":"2023-09-23T23:30:15.647077+0000","last_fullsized":"2023-09-23T23:30:15.647077+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:10.324158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":510,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.648442+0000","last_change":"2023-09-23T23:29:55.001597+0000","last_active":"2023-09-23T23:30:15.648442+0000","last_peered":"2023-09-23T23:30:15.648442+0000","last_clean":"2023-09-23T23:30:15.648442+0000","last_became_active":"2023-09-23T23:29:55.001314+0000","last_became_peered":"2023-09-23T23:29:55.001314+0000","last_unstale":"2023-09-23T23:30:15.648442+0000","last_undegraded":"2023-09-23T23:30:15.648442+0000","last_fullsized":"2023-09-23T23:30:15.648442+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:00.024016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"502'5344","reported_seq":5389,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:19.886894+0000","last_change":"2023-09-23T23:29:45.649770+0000","last_active":"2023-09-23T23:30:19.886894+0000","last_peered":"2023-09-23T23:30:19.886894+0000","last_clean":"2023-09-23T23:30:19.886894+0000","last_became_active":"2023-09-23T23:29:45.646295+0000","last_became_peered":"2023-09-23T23:29:45.646295+0000","last_unstale":"2023-09-23T23:30:19.886894+0000","last_undegraded":"2023-09-23T23:30:19.886894+0000","last_fullsized":"2023-09-23T23:30:19.886894+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5344,"log_dups_size":0,"ondisk_log_size":5344,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187465728,"num_objects":2860,"num_object_clones":0,"num_object_copies":8580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3058,"num_write_kb":189408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":841,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.641360+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:30:15.641360+0000","last_peered":"2023-09-23T23:30:15.641360+0000","last_clean":"2023-09-23T23:30:15.641360+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:30:15.641360+0000","last_undegraded":"2023-09-23T23:30:15.641360+0000","last_fullsized":"2023-09-23T23:30:15.641360+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":140,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650741+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:30:15.650741+0000","last_peered":"2023-09-23T23:30:15.650741+0000","last_clean":"2023-09-23T23:30:15.650741+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:30:15.650741+0000","last_undegraded":"2023-09-23T23:30:15.650741+0000","last_fullsized":"2023-09-23T23:30:15.650741+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":424,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.673858+0000","last_change":"2023-09-23T23:22:39.489325+0000","last_active":"2023-09-23T23:30:15.673858+0000","last_peered":"2023-09-23T23:30:15.673858+0000","last_clean":"2023-09-23T23:30:15.673858+0000","last_became_active":"2023-09-23T23:22:39.489048+0000","last_became_peered":"2023-09-23T23:22:39.489048+0000","last_unstale":"2023-09-23T23:30:15.673858+0000","last_undegraded":"2023-09-23T23:30:15.673858+0000","last_fullsized":"2023-09-23T23:30:15.673858+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:29.984948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"502'6255","reported_seq":6290,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:21.071836+0000","last_change":"2023-09-23T23:28:38.107589+0000","last_active":"2023-09-23T23:30:21.071836+0000","last_peered":"2023-09-23T23:30:21.071836+0000","last_clean":"2023-09-23T23:30:21.071836+0000","last_became_active":"2023-09-23T23:28:38.107296+0000","last_became_peered":"2023-09-23T23:28:38.107296+0000","last_unstale":"2023-09-23T23:30:21.071836+0000","last_undegraded":"2023-09-23T23:30:21.071836+0000","last_fullsized":"2023-09-23T23:30:21.071836+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":6255,"log_dups_size":0,"ondisk_log_size":6255,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:12.352223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371654656,"num_objects":5671,"num_object_clones":0,"num_object_copies":17013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6255,"num_write_kb":381632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":823,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651389+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:30:15.651389+0000","last_peered":"2023-09-23T23:30:15.651389+0000","last_clean":"2023-09-23T23:30:15.651389+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:30:15.651389+0000","last_undegraded":"2023-09-23T23:30:15.651389+0000","last_fullsized":"2023-09-23T23:30:15.651389+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":204,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.670751+0000","last_change":"2023-09-23T23:23:36.048889+0000","last_active":"2023-09-23T23:30:15.670751+0000","last_peered":"2023-09-23T23:30:15.670751+0000","last_clean":"2023-09-23T23:30:15.670751+0000","last_became_active":"2023-09-23T23:23:36.048572+0000","last_became_peered":"2023-09-23T23:23:36.048572+0000","last_unstale":"2023-09-23T23:30:15.670751+0000","last_undegraded":"2023-09-23T23:30:15.670751+0000","last_fullsized":"2023-09-23T23:30:15.670751+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:55.581146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":517,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.654446+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:30:15.654446+0000","last_peered":"2023-09-23T23:30:15.654446+0000","last_clean":"2023-09-23T23:30:15.654446+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:30:15.654446+0000","last_undegraded":"2023-09-23T23:30:15.654446+0000","last_fullsized":"2023-09-23T23:30:15.654446+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":89,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.670067+0000","last_change":"2023-09-23T23:27:20.549904+0000","last_active":"2023-09-23T23:30:15.670067+0000","last_peered":"2023-09-23T23:30:15.670067+0000","last_clean":"2023-09-23T23:30:15.670067+0000","last_became_active":"2023-09-23T23:27:20.549645+0000","last_became_peered":"2023-09-23T23:27:20.549645+0000","last_unstale":"2023-09-23T23:30:15.670067+0000","last_undegraded":"2023-09-23T23:30:15.670067+0000","last_fullsized":"2023-09-23T23:30:15.670067+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:09.850640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"502'6250","reported_seq":6285,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:21.049376+0000","last_change":"2023-09-23T23:28:38.107697+0000","last_active":"2023-09-23T23:30:21.049376+0000","last_peered":"2023-09-23T23:30:21.049376+0000","last_clean":"2023-09-23T23:30:21.049376+0000","last_became_active":"2023-09-23T23:28:38.107379+0000","last_became_peered":"2023-09-23T23:28:38.107379+0000","last_unstale":"2023-09-23T23:30:21.049376+0000","last_undegraded":"2023-09-23T23:30:21.049376+0000","last_fullsized":"2023-09-23T23:30:21.049376+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":6250,"log_dups_size":0,"ondisk_log_size":6250,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:12.478832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373293056,"num_objects":5696,"num_object_clones":0,"num_object_copies":17088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6250,"num_write_kb":382272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":649,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.647468+0000","last_change":"2023-09-23T23:23:49.320969+0000","last_active":"2023-09-23T23:30:15.647468+0000","last_peered":"2023-09-23T23:30:15.647468+0000","last_clean":"2023-09-23T23:30:15.647468+0000","last_became_active":"2023-09-23T23:22:17.001911+0000","last_became_peered":"2023-09-23T23:22:17.001911+0000","last_unstale":"2023-09-23T23:30:15.647468+0000","last_undegraded":"2023-09-23T23:30:15.647468+0000","last_fullsized":"2023-09-23T23:30:15.647468+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.320823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.320823+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:07.000984+0000","scrub_duration":0.0018140630000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":213,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.670236+0000","last_change":"2023-09-23T23:29:54.992796+0000","last_active":"2023-09-23T23:30:15.670236+0000","last_peered":"2023-09-23T23:30:15.670236+0000","last_clean":"2023-09-23T23:30:15.670236+0000","last_became_active":"2023-09-23T23:29:54.992394+0000","last_became_peered":"2023-09-23T23:29:54.992394+0000","last_unstale":"2023-09-23T23:30:15.670236+0000","last_undegraded":"2023-09-23T23:30:15.670236+0000","last_fullsized":"2023-09-23T23:30:15.670236+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:01.353635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":491,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.647998+0000","last_change":"2023-09-23T23:29:55.001578+0000","last_active":"2023-09-23T23:30:15.647998+0000","last_peered":"2023-09-23T23:30:15.647998+0000","last_clean":"2023-09-23T23:30:15.647998+0000","last_became_active":"2023-09-23T23:29:55.001264+0000","last_became_peered":"2023-09-23T23:29:55.001264+0000","last_unstale":"2023-09-23T23:30:15.647998+0000","last_undegraded":"2023-09-23T23:30:15.647998+0000","last_fullsized":"2023-09-23T23:30:15.647998+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:16.411769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":89,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.669280+0000","last_change":"2023-09-23T23:27:20.551221+0000","last_active":"2023-09-23T23:30:15.669280+0000","last_peered":"2023-09-23T23:30:15.669280+0000","last_clean":"2023-09-23T23:30:15.669280+0000","last_became_active":"2023-09-23T23:27:20.550900+0000","last_became_peered":"2023-09-23T23:27:20.550900+0000","last_unstale":"2023-09-23T23:30:15.669280+0000","last_undegraded":"2023-09-23T23:30:15.669280+0000","last_fullsized":"2023-09-23T23:30:15.669280+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:23.012149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"502'6212","reported_seq":6250,"reported_epoch":502,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:30:21.047577+0000","last_change":"2023-09-23T23:29:56.269322+0000","last_active":"2023-09-23T23:30:21.047577+0000","last_peered":"2023-09-23T23:30:21.047577+0000","last_clean":"2023-09-23T23:29:52.322507+0000","last_became_active":"2023-09-23T23:29:56.254145+0000","last_became_peered":"2023-09-23T23:29:56.254145+0000","last_unstale":"2023-09-23T23:30:21.047577+0000","last_undegraded":"2023-09-23T23:29:56.011776+0000","last_fullsized":"2023-09-23T23:29:55.980461+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":6212,"log_dups_size":0,"ondisk_log_size":6212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:08:45.509316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371064832,"num_objects":5662,"num_object_clones":0,"num_object_copies":16986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5036,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6212,"num_write_kb":379968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[2,5],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":4761},{"shards":"2,5","objects":626}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":802,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.641219+0000","last_change":"2023-09-23T23:28:48.113032+0000","last_active":"2023-09-23T23:30:15.641219+0000","last_peered":"2023-09-23T23:30:15.641219+0000","last_clean":"2023-09-23T23:30:15.641219+0000","last_became_active":"2023-09-23T23:28:48.112666+0000","last_became_peered":"2023-09-23T23:28:48.112666+0000","last_unstale":"2023-09-23T23:30:15.641219+0000","last_undegraded":"2023-09-23T23:30:15.641219+0000","last_fullsized":"2023-09-23T23:30:15.641219+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:59:59.838039+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":237,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.643441+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:30:15.643441+0000","last_peered":"2023-09-23T23:30:15.643441+0000","last_clean":"2023-09-23T23:30:15.643441+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:30:15.643441+0000","last_undegraded":"2023-09-23T23:30:15.643441+0000","last_fullsized":"2023-09-23T23:30:15.643441+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":213,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.669174+0000","last_change":"2023-09-23T23:29:54.992719+0000","last_active":"2023-09-23T23:30:15.669174+0000","last_peered":"2023-09-23T23:30:15.669174+0000","last_clean":"2023-09-23T23:30:15.669174+0000","last_became_active":"2023-09-23T23:29:54.992412+0000","last_became_peered":"2023-09-23T23:29:54.992412+0000","last_unstale":"2023-09-23T23:30:15.669174+0000","last_undegraded":"2023-09-23T23:30:15.669174+0000","last_fullsized":"2023-09-23T23:30:15.669174+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:46.541397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":89,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.674087+0000","last_change":"2023-09-23T23:27:20.551574+0000","last_active":"2023-09-23T23:30:15.674087+0000","last_peered":"2023-09-23T23:30:15.674087+0000","last_clean":"2023-09-23T23:30:15.674087+0000","last_became_active":"2023-09-23T23:27:20.550820+0000","last_became_peered":"2023-09-23T23:27:20.550820+0000","last_unstale":"2023-09-23T23:30:15.674087+0000","last_undegraded":"2023-09-23T23:30:15.674087+0000","last_fullsized":"2023-09-23T23:30:15.674087+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:09.274552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"502'6064","reported_seq":6099,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:18.160498+0000","last_change":"2023-09-23T23:28:38.065394+0000","last_active":"2023-09-23T23:30:18.160498+0000","last_peered":"2023-09-23T23:30:18.160498+0000","last_clean":"2023-09-23T23:30:18.160498+0000","last_became_active":"2023-09-23T23:28:38.065160+0000","last_became_peered":"2023-09-23T23:28:38.065160+0000","last_unstale":"2023-09-23T23:30:18.160498+0000","last_undegraded":"2023-09-23T23:30:18.160498+0000","last_fullsized":"2023-09-23T23:30:18.160498+0000","mapping_epoch":476,"log_start":"484'800","ondisk_log_start":"484'800","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5264,"log_dups_size":0,"ondisk_log_size":5264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:10:39.685945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378208280,"num_objects":5772,"num_object_clones":0,"num_object_copies":17316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5772,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6064,"num_write_kb":378689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":514,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.641063+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:30:15.641063+0000","last_peered":"2023-09-23T23:30:15.641063+0000","last_clean":"2023-09-23T23:30:15.641063+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:30:15.641063+0000","last_undegraded":"2023-09-23T23:30:15.641063+0000","last_fullsized":"2023-09-23T23:30:15.641063+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":842,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651316+0000","last_change":"2023-09-23T23:29:58.259227+0000","last_active":"2023-09-23T23:30:15.651316+0000","last_peered":"2023-09-23T23:30:15.651316+0000","last_clean":"2023-09-23T23:30:15.651316+0000","last_became_active":"2023-09-23T23:29:58.258366+0000","last_became_peered":"2023-09-23T23:29:58.258366+0000","last_unstale":"2023-09-23T23:30:15.651316+0000","last_undegraded":"2023-09-23T23:30:15.651316+0000","last_fullsized":"2023-09-23T23:30:15.651316+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:57.581676+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":461,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650773+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:30:15.650773+0000","last_peered":"2023-09-23T23:30:15.650773+0000","last_clean":"2023-09-23T23:30:15.650773+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:30:15.650773+0000","last_undegraded":"2023-09-23T23:30:15.650773+0000","last_fullsized":"2023-09-23T23:30:15.650773+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":16,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651171+0000","last_change":"2023-09-23T23:29:54.998861+0000","last_active":"2023-09-23T23:30:15.651171+0000","last_peered":"2023-09-23T23:30:15.651171+0000","last_clean":"2023-09-23T23:30:15.651171+0000","last_became_active":"2023-09-23T23:29:54.998550+0000","last_became_peered":"2023-09-23T23:29:54.998550+0000","last_unstale":"2023-09-23T23:30:15.651171+0000","last_undegraded":"2023-09-23T23:30:15.651171+0000","last_fullsized":"2023-09-23T23:30:15.651171+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:26.515992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"502'6076","reported_seq":6110,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:18.832578+0000","last_change":"2023-09-23T23:28:38.104180+0000","last_active":"2023-09-23T23:30:18.832578+0000","last_peered":"2023-09-23T23:30:18.832578+0000","last_clean":"2023-09-23T23:30:18.832578+0000","last_became_active":"2023-09-23T23:28:38.103844+0000","last_became_peered":"2023-09-23T23:28:38.103844+0000","last_unstale":"2023-09-23T23:30:18.832578+0000","last_undegraded":"2023-09-23T23:30:18.832578+0000","last_fullsized":"2023-09-23T23:30:18.832578+0000","mapping_epoch":476,"log_start":"484'600","ondisk_log_start":"484'600","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5476,"log_dups_size":0,"ondisk_log_size":5476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:48.427805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376569856,"num_objects":5746,"num_object_clones":0,"num_object_copies":17238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6076,"num_write_kb":378304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":683,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.670438+0000","last_change":"2023-09-23T23:29:58.254396+0000","last_active":"2023-09-23T23:30:15.670438+0000","last_peered":"2023-09-23T23:30:15.670438+0000","last_clean":"2023-09-23T23:30:15.670438+0000","last_became_active":"2023-09-23T23:29:58.254127+0000","last_became_peered":"2023-09-23T23:29:58.254127+0000","last_unstale":"2023-09-23T23:30:15.670438+0000","last_undegraded":"2023-09-23T23:30:15.670438+0000","last_fullsized":"2023-09-23T23:30:15.670438+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:48.863625+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":429,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650981+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:30:15.650981+0000","last_peered":"2023-09-23T23:30:15.650981+0000","last_clean":"2023-09-23T23:30:15.650981+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:30:15.650981+0000","last_undegraded":"2023-09-23T23:30:15.650981+0000","last_fullsized":"2023-09-23T23:30:15.650981+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":428,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.646980+0000","last_change":"2023-09-23T23:28:22.601436+0000","last_active":"2023-09-23T23:30:15.646980+0000","last_peered":"2023-09-23T23:30:15.646980+0000","last_clean":"2023-09-23T23:30:15.646980+0000","last_became_active":"2023-09-23T23:28:22.601154+0000","last_became_peered":"2023-09-23T23:28:22.601154+0000","last_unstale":"2023-09-23T23:30:15.646980+0000","last_undegraded":"2023-09-23T23:30:15.646980+0000","last_fullsized":"2023-09-23T23:30:15.646980+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:03.296778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":533,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.653010+0000","last_change":"2023-09-23T23:22:38.438448+0000","last_active":"2023-09-23T23:30:15.653010+0000","last_peered":"2023-09-23T23:30:15.653010+0000","last_clean":"2023-09-23T23:30:15.653010+0000","last_became_active":"2023-09-23T23:22:38.437495+0000","last_became_peered":"2023-09-23T23:22:38.437495+0000","last_unstale":"2023-09-23T23:30:15.653010+0000","last_undegraded":"2023-09-23T23:30:15.653010+0000","last_fullsized":"2023-09-23T23:30:15.653010+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:49.587056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"502'6093","reported_seq":6137,"reported_epoch":502,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:30:18.891806+0000","last_change":"2023-09-23T23:29:56.148242+0000","last_active":"2023-09-23T23:30:18.891806+0000","last_peered":"2023-09-23T23:30:18.891806+0000","last_clean":"2023-09-23T23:29:53.882998+0000","last_became_active":"2023-09-23T23:29:56.142793+0000","last_became_peered":"2023-09-23T23:29:56.142793+0000","last_unstale":"2023-09-23T23:30:18.891806+0000","last_undegraded":"2023-09-23T23:30:18.891806+0000","last_fullsized":"2023-09-23T23:30:18.891806+0000","mapping_epoch":496,"log_start":"484'1011","ondisk_log_start":"484'1011","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5082,"log_dups_size":0,"ondisk_log_size":5082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:10.851062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378470400,"num_objects":5775,"num_object_clones":0,"num_object_copies":17325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5775,"num_objects_unfound":0,"num_objects_dirty":5775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6093,"num_write_kb":379776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":5775}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":427,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651055+0000","last_change":"2023-09-23T23:29:58.254981+0000","last_active":"2023-09-23T23:30:15.651055+0000","last_peered":"2023-09-23T23:30:15.651055+0000","last_clean":"2023-09-23T23:30:15.651055+0000","last_became_active":"2023-09-23T23:29:58.254665+0000","last_became_peered":"2023-09-23T23:29:58.254665+0000","last_unstale":"2023-09-23T23:30:15.651055+0000","last_undegraded":"2023-09-23T23:30:15.651055+0000","last_fullsized":"2023-09-23T23:30:15.651055+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.842881+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.842881+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":353,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650139+0000","last_change":"2023-09-23T23:23:35.037351+0000","last_active":"2023-09-23T23:30:15.650139+0000","last_peered":"2023-09-23T23:30:15.650139+0000","last_clean":"2023-09-23T23:30:15.650139+0000","last_became_active":"2023-09-23T23:23:35.036841+0000","last_became_peered":"2023-09-23T23:23:35.036841+0000","last_unstale":"2023-09-23T23:30:15.650139+0000","last_undegraded":"2023-09-23T23:30:15.650139+0000","last_fullsized":"2023-09-23T23:30:15.650139+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:20.618292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":204,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.671811+0000","last_change":"2023-09-23T23:26:32.037099+0000","last_active":"2023-09-23T23:30:15.671811+0000","last_peered":"2023-09-23T23:30:15.671811+0000","last_clean":"2023-09-23T23:30:15.671811+0000","last_became_active":"2023-09-23T23:26:32.036681+0000","last_became_peered":"2023-09-23T23:26:32.036681+0000","last_unstale":"2023-09-23T23:30:15.671811+0000","last_undegraded":"2023-09-23T23:30:15.671811+0000","last_fullsized":"2023-09-23T23:30:15.671811+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:48.926576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":235,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.654180+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:30:15.654180+0000","last_peered":"2023-09-23T23:30:15.654180+0000","last_clean":"2023-09-23T23:30:15.654180+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:30:15.654180+0000","last_undegraded":"2023-09-23T23:30:15.654180+0000","last_fullsized":"2023-09-23T23:30:15.654180+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"502'5268","reported_seq":5313,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:18.917719+0000","last_change":"2023-09-23T23:29:45.645107+0000","last_active":"2023-09-23T23:30:18.917719+0000","last_peered":"2023-09-23T23:30:18.917719+0000","last_clean":"2023-09-23T23:30:18.917719+0000","last_became_active":"2023-09-23T23:29:45.640211+0000","last_became_peered":"2023-09-23T23:29:45.640211+0000","last_unstale":"2023-09-23T23:30:18.917719+0000","last_undegraded":"2023-09-23T23:30:18.917719+0000","last_fullsized":"2023-09-23T23:30:18.917719+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5268,"log_dups_size":0,"ondisk_log_size":5268,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187039744,"num_objects":2854,"num_object_clones":0,"num_object_copies":8562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2998,"num_write_kb":187264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":833,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.647611+0000","last_change":"2023-09-23T23:29:58.258327+0000","last_active":"2023-09-23T23:30:15.647611+0000","last_peered":"2023-09-23T23:30:15.647611+0000","last_clean":"2023-09-23T23:30:15.647611+0000","last_became_active":"2023-09-23T23:29:58.257930+0000","last_became_peered":"2023-09-23T23:29:58.257930+0000","last_unstale":"2023-09-23T23:30:15.647611+0000","last_undegraded":"2023-09-23T23:30:15.647611+0000","last_fullsized":"2023-09-23T23:30:15.647611+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:00.144157+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":496,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651073+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:30:15.651073+0000","last_peered":"2023-09-23T23:30:15.651073+0000","last_clean":"2023-09-23T23:30:15.651073+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:30:15.651073+0000","last_undegraded":"2023-09-23T23:30:15.651073+0000","last_fullsized":"2023-09-23T23:30:15.651073+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":98,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.669531+0000","last_change":"2023-09-23T23:29:54.999590+0000","last_active":"2023-09-23T23:30:15.669531+0000","last_peered":"2023-09-23T23:30:15.669531+0000","last_clean":"2023-09-23T23:30:15.669531+0000","last_became_active":"2023-09-23T23:29:54.999209+0000","last_became_peered":"2023-09-23T23:29:54.999209+0000","last_unstale":"2023-09-23T23:30:15.669531+0000","last_undegraded":"2023-09-23T23:30:15.669531+0000","last_fullsized":"2023-09-23T23:30:15.669531+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:32.208030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":32,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.646032+0000","last_change":"2023-09-23T23:28:45.061717+0000","last_active":"2023-09-23T23:30:15.646032+0000","last_peered":"2023-09-23T23:30:15.646032+0000","last_clean":"2023-09-23T23:30:15.646032+0000","last_became_active":"2023-09-23T23:28:45.061404+0000","last_became_peered":"2023-09-23T23:28:45.061404+0000","last_unstale":"2023-09-23T23:30:15.646032+0000","last_undegraded":"2023-09-23T23:30:15.646032+0000","last_fullsized":"2023-09-23T23:30:15.646032+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:46.614114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.9","version":"502'5287","reported_seq":5339,"reported_epoch":502,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:30:21.072762+0000","last_change":"2023-09-23T23:29:56.178126+0000","last_active":"2023-09-23T23:30:21.072762+0000","last_peered":"2023-09-23T23:30:21.072762+0000","last_clean":"2023-09-23T23:29:52.968270+0000","last_became_active":"2023-09-23T23:29:56.173510+0000","last_became_peered":"2023-09-23T23:29:56.173510+0000","last_unstale":"2023-09-23T23:30:21.072762+0000","last_undegraded":"2023-09-23T23:29:56.005404+0000","last_fullsized":"2023-09-23T23:29:55.986389+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5287,"log_dups_size":0,"ondisk_log_size":5287,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183861248,"num_objects":2805,"num_object_clones":0,"num_object_copies":8415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2453,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3051,"num_write_kb":187424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,3],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2,3","objects":2330},{"shards":"2,3","objects":352}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":886,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651938+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:30:15.651938+0000","last_peered":"2023-09-23T23:30:15.651938+0000","last_clean":"2023-09-23T23:30:15.651938+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:30:15.651938+0000","last_undegraded":"2023-09-23T23:30:15.651938+0000","last_fullsized":"2023-09-23T23:30:15.651938+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.651467+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:30:15.651467+0000","last_peered":"2023-09-23T23:30:15.651467+0000","last_clean":"2023-09-23T23:30:15.651467+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:30:15.651467+0000","last_undegraded":"2023-09-23T23:30:15.651467+0000","last_fullsized":"2023-09-23T23:30:15.651467+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":488,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.648881+0000","last_change":"2023-09-23T23:22:38.482705+0000","last_active":"2023-09-23T23:30:15.648881+0000","last_peered":"2023-09-23T23:30:15.648881+0000","last_clean":"2023-09-23T23:30:15.648881+0000","last_became_active":"2023-09-23T23:22:38.438239+0000","last_became_peered":"2023-09-23T23:22:38.438239+0000","last_unstale":"2023-09-23T23:30:15.648881+0000","last_undegraded":"2023-09-23T23:30:15.648881+0000","last_fullsized":"2023-09-23T23:30:15.648881+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:52.006349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":133,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.647088+0000","last_change":"2023-09-23T23:29:55.000912+0000","last_active":"2023-09-23T23:30:15.647088+0000","last_peered":"2023-09-23T23:30:15.647088+0000","last_clean":"2023-09-23T23:30:15.647088+0000","last_became_active":"2023-09-23T23:29:54.999122+0000","last_became_peered":"2023-09-23T23:29:54.999122+0000","last_unstale":"2023-09-23T23:30:15.647088+0000","last_undegraded":"2023-09-23T23:30:15.647088+0000","last_fullsized":"2023-09-23T23:30:15.647088+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:10.962943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.14","version":"502'5230","reported_seq":5990,"reported_epoch":502,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T23:30:19.877872+0000","last_change":"2023-09-23T23:30:14.702655+0000","last_active":"2023-09-23T23:30:19.877872+0000","last_peered":"2023-09-23T23:30:19.877872+0000","last_clean":"2023-09-23T23:29:52.333229+0000","last_became_active":"2023-09-23T23:29:56.254489+0000","last_became_peered":"2023-09-23T23:29:56.254489+0000","last_unstale":"2023-09-23T23:30:19.877872+0000","last_undegraded":"2023-09-23T23:30:19.877872+0000","last_fullsized":"2023-09-23T23:29:55.977321+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5230,"log_dups_size":0,"ondisk_log_size":5230,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184221696,"num_objects":2811,"num_object_clones":0,"num_object_copies":8433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2118,"num_objects_unfound":0,"num_objects_dirty":2811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2999,"num_write_kb":185920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":355,"num_bytes_recovered":23265280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[1,3],"avail_no_missing":["1","3","7"],"object_location_counts":[{"shards":"1,3","objects":693},{"shards":"1,3,7","objects":2118}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":902,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.653585+0000","last_change":"2023-09-23T23:29:58.264914+0000","last_active":"2023-09-23T23:30:15.653585+0000","last_peered":"2023-09-23T23:30:15.653585+0000","last_clean":"2023-09-23T23:30:15.653585+0000","last_became_active":"2023-09-23T23:29:58.264356+0000","last_became_peered":"2023-09-23T23:29:58.264356+0000","last_unstale":"2023-09-23T23:30:15.653585+0000","last_undegraded":"2023-09-23T23:30:15.653585+0000","last_fullsized":"2023-09-23T23:30:15.653585+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:15:15.828500+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":172,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.647956+0000","last_change":"2023-09-23T23:28:22.603176+0000","last_active":"2023-09-23T23:30:15.647956+0000","last_peered":"2023-09-23T23:30:15.647956+0000","last_clean":"2023-09-23T23:30:15.647956+0000","last_became_active":"2023-09-23T23:28:22.602824+0000","last_became_peered":"2023-09-23T23:28:22.602824+0000","last_unstale":"2023-09-23T23:30:15.647956+0000","last_undegraded":"2023-09-23T23:30:15.647956+0000","last_fullsized":"2023-09-23T23:30:15.647956+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:35.934235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":255,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:16.646127+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:30:16.646127+0000","last_peered":"2023-09-23T23:30:16.646127+0000","last_clean":"2023-09-23T23:30:16.646127+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:30:16.646127+0000","last_undegraded":"2023-09-23T23:30:16.646127+0000","last_fullsized":"2023-09-23T23:30:16.646127+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.13","version":"502'5439","reported_seq":5483,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:18.124053+0000","last_change":"2023-09-23T23:29:45.701227+0000","last_active":"2023-09-23T23:30:18.124053+0000","last_peered":"2023-09-23T23:30:18.124053+0000","last_clean":"2023-09-23T23:30:18.124053+0000","last_became_active":"2023-09-23T23:29:45.699719+0000","last_became_peered":"2023-09-23T23:29:45.699719+0000","last_unstale":"2023-09-23T23:30:18.124053+0000","last_undegraded":"2023-09-23T23:30:18.124053+0000","last_fullsized":"2023-09-23T23:30:18.124053+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5439,"log_dups_size":0,"ondisk_log_size":5439,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196116480,"num_objects":2993,"num_object_clones":0,"num_object_copies":8979,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2993,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3107,"num_write_kb":195168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":647,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650687+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:30:15.650687+0000","last_peered":"2023-09-23T23:30:15.650687+0000","last_clean":"2023-09-23T23:30:15.650687+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:30:15.650687+0000","last_undegraded":"2023-09-23T23:30:15.650687+0000","last_fullsized":"2023-09-23T23:30:15.650687+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":243,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.647445+0000","last_change":"2023-09-23T23:22:38.452968+0000","last_active":"2023-09-23T23:30:15.647445+0000","last_peered":"2023-09-23T23:30:15.647445+0000","last_clean":"2023-09-23T23:30:15.647445+0000","last_became_active":"2023-09-23T23:22:38.437986+0000","last_became_peered":"2023-09-23T23:22:38.437986+0000","last_unstale":"2023-09-23T23:30:15.647445+0000","last_undegraded":"2023-09-23T23:30:15.647445+0000","last_fullsized":"2023-09-23T23:30:15.647445+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:10.333271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":133,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.647504+0000","last_change":"2023-09-23T23:27:20.558454+0000","last_active":"2023-09-23T23:30:15.647504+0000","last_peered":"2023-09-23T23:30:15.647504+0000","last_clean":"2023-09-23T23:30:15.647504+0000","last_became_active":"2023-09-23T23:27:20.558204+0000","last_became_peered":"2023-09-23T23:27:20.558204+0000","last_unstale":"2023-09-23T23:30:15.647504+0000","last_undegraded":"2023-09-23T23:30:15.647504+0000","last_fullsized":"2023-09-23T23:30:15.647504+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:15.079990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.12","version":"502'5407","reported_seq":5454,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:18.554305+0000","last_change":"2023-09-23T23:30:16.114907+0000","last_active":"2023-09-23T23:30:18.554305+0000","last_peered":"2023-09-23T23:30:18.554305+0000","last_clean":"2023-09-23T23:30:18.554305+0000","last_became_active":"2023-09-23T23:29:45.684008+0000","last_became_peered":"2023-09-23T23:29:45.684008+0000","last_unstale":"2023-09-23T23:30:18.554305+0000","last_undegraded":"2023-09-23T23:30:18.554305+0000","last_fullsized":"2023-09-23T23:30:18.554305+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5407,"log_dups_size":0,"ondisk_log_size":5407,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191430656,"num_objects":2921,"num_object_clones":0,"num_object_copies":8763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3083,"num_write_kb":192128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":257,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.647687+0000","last_change":"2023-09-23T23:29:55.022469+0000","last_active":"2023-09-23T23:30:15.647687+0000","last_peered":"2023-09-23T23:30:15.647687+0000","last_clean":"2023-09-23T23:30:15.647687+0000","last_became_active":"2023-09-23T23:29:55.003082+0000","last_became_peered":"2023-09-23T23:29:55.003082+0000","last_unstale":"2023-09-23T23:30:15.647687+0000","last_undegraded":"2023-09-23T23:30:15.647687+0000","last_fullsized":"2023-09-23T23:30:15.647687+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:15.791643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":123,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.647184+0000","last_change":"2023-09-23T23:26:11.527453+0000","last_active":"2023-09-23T23:30:15.647184+0000","last_peered":"2023-09-23T23:30:15.647184+0000","last_clean":"2023-09-23T23:30:15.647184+0000","last_became_active":"2023-09-23T23:26:11.527208+0000","last_became_peered":"2023-09-23T23:26:11.527208+0000","last_unstale":"2023-09-23T23:30:15.647184+0000","last_undegraded":"2023-09-23T23:30:15.647184+0000","last_fullsized":"2023-09-23T23:30:15.647184+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:15.083611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.11","version":"502'5455","reported_seq":5505,"reported_epoch":502,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:30:19.760413+0000","last_change":"2023-09-23T23:29:56.435212+0000","last_active":"2023-09-23T23:30:19.760413+0000","last_peered":"2023-09-23T23:30:19.760413+0000","last_clean":"2023-09-23T23:29:52.255669+0000","last_became_active":"2023-09-23T23:29:56.398423+0000","last_became_peered":"2023-09-23T23:29:56.398423+0000","last_unstale":"2023-09-23T23:30:19.760413+0000","last_undegraded":"2023-09-23T23:29:56.385600+0000","last_fullsized":"2023-09-23T23:29:55.976159+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5455,"log_dups_size":0,"ondisk_log_size":5455,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190775296,"num_objects":2911,"num_object_clones":0,"num_object_copies":8733,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2586,"num_objects_misplaced":2586,"num_objects_unfound":0,"num_objects_dirty":2911,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3129,"num_write_kb":193280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[1,3],"avail_no_missing":["1","3","7"],"object_location_counts":[{"shards":"1,3","objects":325},{"shards":"1,3,7","objects":2477}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":788,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650731+0000","last_change":"2023-09-23T23:29:58.252042+0000","last_active":"2023-09-23T23:30:15.650731+0000","last_peered":"2023-09-23T23:30:15.650731+0000","last_clean":"2023-09-23T23:30:15.650731+0000","last_became_active":"2023-09-23T23:29:58.251749+0000","last_became_peered":"2023-09-23T23:29:58.251749+0000","last_unstale":"2023-09-23T23:30:15.650731+0000","last_undegraded":"2023-09-23T23:30:15.650731+0000","last_fullsized":"2023-09-23T23:30:15.650731+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:39.324116+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":529,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.650597+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:30:15.650597+0000","last_peered":"2023-09-23T23:30:15.650597+0000","last_clean":"2023-09-23T23:30:15.650597+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:30:15.650597+0000","last_undegraded":"2023-09-23T23:30:15.650597+0000","last_fullsized":"2023-09-23T23:30:15.650597+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":514,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:30:16.659097+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:30:16.659097+0000","last_peered":"2023-09-23T23:30:16.659097+0000","last_clean":"2023-09-23T23:30:16.659097+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:30:16.659097+0000","last_undegraded":"2023-09-23T23:30:16.659097+0000","last_fullsized":"2023-09-23T23:30:16.659097+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.10","version":"502'5409","reported_seq":9427,"reported_epoch":502,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T23:30:21.082720+0000","last_change":"2023-09-23T23:29:55.031806+0000","last_active":"2023-09-23T23:30:21.082720+0000","last_peered":"2023-09-23T23:30:21.082720+0000","last_clean":"2023-09-23T23:29:51.952236+0000","last_became_active":"2023-09-23T23:29:54.117013+0000","last_became_peered":"2023-09-23T23:29:54.117013+0000","last_unstale":"2023-09-23T23:30:21.082720+0000","last_undegraded":"2023-09-23T23:30:21.082720+0000","last_fullsized":"2023-09-23T23:29:53.969643+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5409,"log_dups_size":0,"ondisk_log_size":5409,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184483840,"num_objects":2815,"num_object_clones":0,"num_object_copies":8445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2340,"num_objects_unfound":0,"num_objects_dirty":2815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3091,"num_write_kb":188992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1321,"num_bytes_recovered":86573056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[2,7],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,4,7","objects":1170},{"shards":"2,7","objects":1645}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":876,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.673964+0000","last_change":"2023-09-23T23:26:14.527390+0000","last_active":"2023-09-23T23:30:15.673964+0000","last_peered":"2023-09-23T23:30:15.673964+0000","last_clean":"2023-09-23T23:30:15.673964+0000","last_became_active":"2023-09-23T23:26:14.527008+0000","last_became_peered":"2023-09-23T23:26:14.527008+0000","last_unstale":"2023-09-23T23:30:15.673964+0000","last_undegraded":"2023-09-23T23:30:15.673964+0000","last_fullsized":"2023-09-23T23:30:15.673964+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:36.460827+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":312,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.666926+0000","last_change":"2023-09-23T23:28:22.600921+0000","last_active":"2023-09-23T23:30:15.666926+0000","last_peered":"2023-09-23T23:30:15.666926+0000","last_clean":"2023-09-23T23:30:15.666926+0000","last_became_active":"2023-09-23T23:28:22.600565+0000","last_became_peered":"2023-09-23T23:28:22.600565+0000","last_unstale":"2023-09-23T23:30:15.666926+0000","last_undegraded":"2023-09-23T23:30:15.666926+0000","last_fullsized":"2023-09-23T23:30:15.666926+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:07.793563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":557,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:30:15.646973+0000","last_change":"2023-09-23T23:29:55.002051+0000","last_active":"2023-09-23T23:30:15.646973+0000","last_peered":"2023-09-23T23:30:15.646973+0000","last_clean":"2023-09-23T23:30:15.646973+0000","last_became_active":"2023-09-23T23:29:55.001732+0000","last_became_peered":"2023-09-23T23:29:55.001732+0000","last_unstale":"2023-09-23T23:30:15.646973+0000","last_undegraded":"2023-09-23T23:30:15.646973+0000","last_fullsized":"2023-09-23T23:30:15.646973+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:28:01.353263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":26,"stat_sum":{"num_bytes":6004998168,"num_objects":91630,"num_object_clones":0,"num_object_copies":274890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":20114,"num_objects_misplaced":15315,"num_objects_unfound":0,"num_objects_dirty":91630,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":98116,"num_write_kb":6071809,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4233,"num_bytes_recovered":277413888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18504101888,"data_stored":18504089672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141479,"ondisk_log_size":141479,"up":78,"acting":69,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":10412,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":603785,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":695612,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":92,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782610,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3504652,"kb_used_data":3242116,"kb_used_omap":8,"kb_used_meta":262519,"kb_avail":90240500,"statfs":{"total":95995035648,"available":92406272000,"internally_reserved":0,"allocated":3319926784,"data_stored":3318581185,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":268819525},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:29:44 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":2.9289999999999998,"15min":1.9670000000000001},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":21.756,"5min":85.721000000000004,"15min":85.721000000000004},"last":0.255},{"interface":"front","average":{"1min":2.2090000000000001,"5min":3.496,"15min":1.8959999999999999},"min":{"1min":0.25700000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":21.803000000000001,"5min":85.965000000000003,"15min":85.965000000000003},"last":0.38700000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:29:44 2023","interfaces":[{"interface":"back","average":{"1min":2.5110000000000001,"5min":1.9850000000000001,"15min":1.3540000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.22900000000000001},"max":{"1min":34.204000000000001,"5min":60.847000000000001,"15min":60.847000000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":3.1200000000000001,"5min":2.0019999999999998,"15min":1.6459999999999999},"min":{"1min":0.30499999999999999,"5min":0.246,"15min":0.246},"max":{"1min":34.396000000000001,"5min":59.512999999999998,"15min":59.512999999999998},"last":0.46999999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:29:44 2023","interfaces":[{"interface":"back","average":{"1min":10.509,"5min":3.7770000000000001,"15min":2.4809999999999999},"min":{"1min":0.40300000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":155.32300000000001,"5min":155.32300000000001,"15min":155.32300000000001},"last":1.73},{"interface":"front","average":{"1min":9.5719999999999992,"5min":3.5979999999999999,"15min":1.994},"min":{"1min":0.36199999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":155.37,"5min":155.37,"15min":155.37},"last":0.51100000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:29:44 2023","interfaces":[{"interface":"back","average":{"1min":6.7859999999999996,"5min":2.4609999999999999,"15min":1.8129999999999999},"min":{"1min":0.36399999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":81.915000000000006,"5min":81.915000000000006,"15min":81.915000000000006},"last":0.46100000000000002},{"interface":"front","average":{"1min":6.3529999999999998,"5min":2.4079999999999999,"15min":1.665},"min":{"1min":0.316,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":81.346000000000004,"5min":81.346000000000004,"15min":81.346000000000004},"last":1.821}]},{"osd":4,"last update":"Sat Sep 23 23:29:44 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.56799999999999995,"15min":0.76100000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":1.1140000000000001,"5min":4.2270000000000003,"15min":6.8380000000000001},"last":0.317},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.65700000000000003,"15min":0.84399999999999997},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.2,"5min":5.242,"15min":5.242},"last":0.57499999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:29:44 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":2.9350000000000001,"15min":1.7030000000000001},"min":{"1min":0.27500000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":2.1499999999999999,"5min":210.83600000000001,"15min":210.83600000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.434,"5min":2.8730000000000002,"15min":1.498},"min":{"1min":0.28699999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.94799999999999995,"5min":207.04599999999999,"15min":207.04599999999999},"last":0.41399999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:29:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":1.371,"15min":0.95999999999999996},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":2.6480000000000001,"5min":62.213000000000001,"15min":62.213000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":1.353,"15min":1.177},"min":{"1min":0.35099999999999998,"5min":0.34100000000000003,"15min":0.33300000000000002},"max":{"1min":1.276,"5min":62.805999999999997,"15min":62.805999999999997},"last":0.43099999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496087,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1062132,"kb_used_data":464300,"kb_used_omap":0,"kb_used_meta":597824,"kb_avail":92683020,"statfs":{"total":95995035648,"available":94907412480,"internally_reserved":0,"allocated":475443200,"data_stored":474121198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612171776},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:30:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":7.0519999999999996,"15min":7.1219999999999999},"min":{"1min":0.246,"5min":0.23000000000000001,"15min":0.19900000000000001},"max":{"1min":6.7539999999999996,"5min":135.31999999999999,"15min":161.64599999999999},"last":0.253},{"interface":"front","average":{"1min":2.2530000000000001,"5min":7.1390000000000002,"15min":6.9299999999999997},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.22900000000000001},"max":{"1min":13.919,"5min":135.34299999999999,"15min":161.38499999999999},"last":0.42699999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:30:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":1.6739999999999999,"15min":3.2559999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":14.028,"5min":41.189999999999998,"15min":210.869},"last":3.3900000000000001},{"interface":"front","average":{"1min":3.508,"5min":2.0950000000000002,"15min":3.4550000000000001},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":14.269,"5min":41.140000000000001,"15min":210.89699999999999},"last":2.8370000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:29:52 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":3.4180000000000001,"15min":3.113},"min":{"1min":0.36899999999999999,"5min":0.28299999999999997,"15min":0.27800000000000002},"max":{"1min":5.8899999999999997,"5min":69.587000000000003,"15min":127.44499999999999},"last":0.374},{"interface":"front","average":{"1min":1.0720000000000001,"5min":3.431,"15min":3.0310000000000001},"min":{"1min":0.29999999999999999,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":3.4510000000000001,"5min":69.417000000000002,"15min":125.175},"last":3.8450000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:29:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":4.9909999999999997,"15min":3.7879999999999998},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.3360000000000003,"5min":128.28700000000001,"15min":128.28700000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":1.175,"5min":5.9589999999999996,"15min":4.085},"min":{"1min":0.33400000000000002,"5min":0.32700000000000001,"15min":0.24199999999999999},"max":{"1min":3.0790000000000002,"5min":128.03299999999999,"15min":128.03299999999999},"last":2.9670000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.60999999999999999,"15min":1.2809999999999999},"min":{"1min":0.27300000000000002,"5min":0.23699999999999999,"15min":0.22900000000000001},"max":{"1min":8.1999999999999993,"5min":8.1999999999999993,"15min":166.05199999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.628,"15min":1.579},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.26200000000000001},"max":{"1min":3.0960000000000001,"5min":3.0960000000000001,"15min":166.631},"last":0.60399999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:30:16 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":0.79300000000000004,"15min":1.1679999999999999},"min":{"1min":0.32200000000000001,"5min":0.28699999999999998,"15min":0.27900000000000003},"max":{"1min":8.2750000000000004,"5min":8.2750000000000004,"15min":91.899000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.155,"5min":0.69499999999999995,"15min":1.2370000000000001},"min":{"1min":0.373,"5min":0.26900000000000002,"15min":0.249},"max":{"1min":5.7350000000000003,"5min":5.7350000000000003,"15min":94.174000000000007},"last":0.627}]},{"osd":7,"last update":"Sat Sep 23 23:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.72799999999999998,"15min":0.84799999999999998},"min":{"1min":0.32500000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.4369999999999998,"5min":7.3890000000000002,"15min":7.3890000000000002},"last":0.58799999999999997},{"interface":"front","average":{"1min":1.3380000000000001,"5min":0.81399999999999995,"15min":0.84199999999999997},"min":{"1min":0.376,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.0389999999999997,"5min":5.0389999999999997,"15min":5.0389999999999997},"last":0.45200000000000001}]}]},{"osd":1,"up_from":112,"seq":481036337428,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3440420,"kb_used_data":2630492,"kb_used_omap":4,"kb_used_meta":809915,"kb_avail":90304732,"statfs":{"total":95995035648,"available":92472045568,"internally_reserved":0,"allocated":2693623808,"data_stored":2691318651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":829353870},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:30:03 2023","interfaces":[{"interface":"back","average":{"1min":2.1240000000000001,"5min":5.1239999999999997,"15min":3.8530000000000002},"min":{"1min":0.16700000000000001,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":9.4640000000000004,"5min":130.768,"15min":130.768},"last":2.391},{"interface":"front","average":{"1min":1.8029999999999999,"5min":4.2699999999999996,"15min":3.6659999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":13.372999999999999,"5min":134.929,"15min":134.929},"last":1.9199999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:29:48 2023","interfaces":[{"interface":"back","average":{"1min":12.356999999999999,"5min":8.8819999999999997,"15min":5.6619999999999999},"min":{"1min":0.222,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":226.744,"5min":231.67400000000001,"15min":231.67400000000001},"last":2.3660000000000001},{"interface":"front","average":{"1min":11.144,"5min":8.7119999999999997,"15min":5.3200000000000003},"min":{"1min":0.33800000000000002,"5min":0.221,"15min":0.218},"max":{"1min":226.59,"5min":232.21000000000001,"15min":232.21000000000001},"last":1.056}]},{"osd":3,"last update":"Sat Sep 23 23:29:46 2023","interfaces":[{"interface":"back","average":{"1min":12.022,"5min":7.2300000000000004,"15min":5.5369999999999999},"min":{"1min":0.309,"5min":0.23200000000000001,"15min":0.20799999999999999},"max":{"1min":234.404,"5min":234.404,"15min":234.404},"last":2.0019999999999998},{"interface":"front","average":{"1min":12.365,"5min":7.0869999999999997,"15min":5.4029999999999996},"min":{"1min":0.38100000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":236.33699999999999,"5min":236.33699999999999,"15min":236.33699999999999},"last":2.0219999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.6020000000000001,"15min":1.6080000000000001},"min":{"1min":0.33300000000000002,"5min":0.29099999999999998,"15min":0.28699999999999998},"max":{"1min":4.8019999999999996,"5min":9.3780000000000001,"15min":18.474},"last":1.173},{"interface":"front","average":{"1min":1.8129999999999999,"5min":1.7,"15min":1.764},"min":{"1min":0.39400000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.2550000000000008,"5min":9.7460000000000004,"15min":25.710999999999999},"last":2.282}]},{"osd":5,"last update":"Sat Sep 23 23:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.2030000000000001,"15min":3.0070000000000001},"min":{"1min":0.34699999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.9029999999999996,"5min":13.064,"15min":250.33000000000001},"last":1.9510000000000001},{"interface":"front","average":{"1min":1.2529999999999999,"5min":1.7669999999999999,"15min":3.2280000000000002},"min":{"1min":0.36199999999999999,"5min":0.34499999999999997,"15min":0.30199999999999999},"max":{"1min":4.8380000000000001,"5min":9.3450000000000006,"15min":250.21199999999999},"last":1.1259999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":1.5,"15min":2.4089999999999998},"min":{"1min":0.371,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.1360000000000001,"5min":13.119,"15min":74.087999999999994},"last":1.9810000000000001},{"interface":"front","average":{"1min":1.351,"5min":1.7709999999999999,"15min":2.3809999999999998},"min":{"1min":0.38200000000000001,"5min":0.37,"15min":0.32800000000000001},"max":{"1min":4.9109999999999996,"5min":9.4030000000000005,"15min":73.974999999999994},"last":1.1499999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:29:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":1.8819999999999999,"15min":3.7040000000000002},"min":{"1min":0.371,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.282,"5min":9.7929999999999993,"15min":78.308000000000007},"last":2.3380000000000001},{"interface":"front","average":{"1min":1.196,"5min":1.831,"15min":3.323},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.883,"5min":12.327,"15min":77.204999999999998},"last":1.0980000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024252,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3418472,"kb_used_data":2298336,"kb_used_omap":5,"kb_used_meta":1119994,"kb_avail":90326680,"statfs":{"total":95995035648,"available":92494520320,"internally_reserved":0,"allocated":2353496064,"data_stored":2351123152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1146874490},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:29:22 2023","interfaces":[{"interface":"back","average":{"1min":9.8620000000000001,"5min":7.351,"15min":4.9710000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16900000000000001},"max":{"1min":98.099000000000004,"5min":139.30000000000001,"15min":155.91999999999999},"last":0.219},{"interface":"front","average":{"1min":10.385,"5min":5.2069999999999999,"15min":3.8460000000000001},"min":{"1min":0.22700000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":97.320999999999998,"5min":97.320999999999998,"15min":153.28899999999999},"last":0.247}]},{"osd":2,"last update":"Sat Sep 23 23:29:47 2023","interfaces":[{"interface":"back","average":{"1min":5.4649999999999999,"5min":5.8029999999999999,"15min":4.9669999999999996},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":66.420000000000002,"5min":139.471,"15min":171.15100000000001},"last":0.28199999999999997},{"interface":"front","average":{"1min":6.6539999999999999,"5min":5.907,"15min":5.234},"min":{"1min":0.24299999999999999,"5min":0.215,"15min":0.17599999999999999},"max":{"1min":68.918000000000006,"5min":139.04400000000001,"15min":173.04900000000001},"last":0.33800000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:29:22 2023","interfaces":[{"interface":"back","average":{"1min":4.0119999999999996,"5min":5.4649999999999999,"15min":4.133},"min":{"1min":0.26500000000000001,"5min":0.253,"15min":0.23200000000000001},"max":{"1min":66.332999999999998,"5min":139.34899999999999,"15min":139.34899999999999},"last":0.311},{"interface":"front","average":{"1min":4.8239999999999998,"5min":5.4000000000000004,"15min":3.8069999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":66.364000000000004,"5min":138.739,"15min":138.739},"last":0.28999999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:29:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":3.536,"15min":2.6309999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.27000000000000002},"max":{"1min":6.4969999999999999,"5min":138.81299999999999,"15min":138.81299999999999},"last":108.01900000000001},{"interface":"front","average":{"1min":1.2889999999999999,"5min":3.5790000000000002,"15min":3.3220000000000001},"min":{"1min":0.32400000000000001,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":5.8319999999999999,"5min":139.19800000000001,"15min":171.251},"last":107.898}]},{"osd":5,"last update":"Sat Sep 23 23:29:55 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":3.6760000000000002,"15min":3.004},"min":{"1min":0.36699999999999999,"5min":0.32200000000000001,"15min":0.308},"max":{"1min":5.8710000000000004,"5min":139.119,"15min":171.30699999999999},"last":0.371},{"interface":"front","average":{"1min":1.0860000000000001,"5min":3.5249999999999999,"15min":2.2130000000000001},"min":{"1min":0.313,"5min":0.253,"15min":0.253},"max":{"1min":3.2509999999999999,"5min":139.40799999999999,"15min":139.40799999999999},"last":0.38200000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:29:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":3.6120000000000001,"15min":3.1800000000000002},"min":{"1min":0.40699999999999997,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":5.9109999999999996,"5min":139.16399999999999,"15min":171.33600000000001},"last":108.67},{"interface":"front","average":{"1min":1.1100000000000001,"5min":6.0709999999999997,"15min":3.355},"min":{"1min":0.435,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.3210000000000002,"5min":184.13499999999999,"15min":184.13499999999999},"last":108.804}]},{"osd":7,"last update":"Sat Sep 23 23:29:47 2023","interfaces":[{"interface":"back","average":{"1min":1.7310000000000001,"5min":3.7869999999999999,"15min":2.1560000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":6.6050000000000004,"5min":138.77699999999999,"15min":138.77699999999999},"last":111.71299999999999},{"interface":"front","average":{"1min":1.1930000000000001,"5min":3.7040000000000002,"15min":1.8220000000000001},"min":{"1min":0.38800000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.2909999999999999,"5min":139.37899999999999,"15min":139.37899999999999},"last":111.108}]}]},{"osd":2,"up_from":199,"seq":854698492073,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4285992,"kb_used_data":3658464,"kb_used_omap":7,"kb_used_meta":627512,"kb_avail":89459160,"statfs":{"total":95995035648,"available":91606179840,"internally_reserved":0,"allocated":3746267136,"data_stored":3743901446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":642572369},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:29:49 2023","interfaces":[{"interface":"back","average":{"1min":4.9770000000000003,"5min":4.2220000000000004,"15min":4.6529999999999996},"min":{"1min":0.28599999999999998,"5min":0.182,"15min":0.16700000000000001},"max":{"1min":58.802,"5min":76.942999999999998,"15min":119.277},"last":0.216},{"interface":"front","average":{"1min":5.085,"5min":4.3289999999999997,"15min":4.7060000000000004},"min":{"1min":0.223,"5min":0.192,"15min":0.18099999999999999},"max":{"1min":58.115000000000002,"5min":76.564999999999998,"15min":119.56100000000001},"last":0.23000000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:29:49 2023","interfaces":[{"interface":"back","average":{"1min":3.0369999999999999,"5min":4.7240000000000002,"15min":5.7599999999999998},"min":{"1min":0.247,"5min":0.23300000000000001,"15min":0.20699999999999999},"max":{"1min":16.129000000000001,"5min":103.65900000000001,"15min":157.98699999999999},"last":0.27900000000000003},{"interface":"front","average":{"1min":2.0430000000000001,"5min":4.4260000000000002,"15min":5.6020000000000003},"min":{"1min":0.26100000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.2249999999999996,"5min":100.182,"15min":158.03800000000001},"last":0.311}]},{"osd":3,"last update":"Sat Sep 23 23:29:49 2023","interfaces":[{"interface":"back","average":{"1min":2.2839999999999998,"5min":5.8719999999999999,"15min":4.8360000000000003},"min":{"1min":0.29199999999999998,"5min":0.191,"15min":0.191},"max":{"1min":4.9039999999999999,"5min":103.227,"15min":119.366},"last":0.436},{"interface":"front","average":{"1min":1.6599999999999999,"5min":5.3840000000000003,"15min":4.2300000000000004},"min":{"1min":0.30499999999999999,"5min":0.25700000000000001,"15min":0.23499999999999999},"max":{"1min":4.6699999999999999,"5min":102.693,"15min":102.693},"last":0.42499999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:29:49 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":2.0110000000000001,"15min":3.8820000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.25},"max":{"1min":5.2489999999999997,"5min":23.391999999999999,"15min":332.98099999999999},"last":11.586},{"interface":"front","average":{"1min":1.79,"5min":2.0230000000000001,"15min":3.327},"min":{"1min":0.33900000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.3410000000000002,"5min":21.076000000000001,"15min":332.96199999999999},"last":11.621}]},{"osd":5,"last update":"Sat Sep 23 23:29:49 2023","interfaces":[{"interface":"back","average":{"1min":1.732,"5min":2.0009999999999999,"15min":2.274},"min":{"1min":0.40999999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.2850000000000001,"5min":23.359999999999999,"15min":119.919},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.841,"5min":2.0939999999999999,"15min":2.2250000000000001},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.3700000000000001,"5min":21.105,"15min":119.306},"last":0.48699999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:29:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7230000000000001,"5min":2.0270000000000001,"15min":4.1139999999999999},"min":{"1min":0.41499999999999998,"5min":0.315,"15min":0.31},"max":{"1min":5.3129999999999997,"5min":23.425000000000001,"15min":349.32799999999997},"last":14.795999999999999},{"interface":"front","average":{"1min":1.9239999999999999,"5min":2.149,"15min":3.6389999999999998},"min":{"1min":0.41099999999999998,"5min":0.372,"15min":0.34999999999999998},"max":{"1min":3.4489999999999998,"5min":21.353999999999999,"15min":349.17700000000002},"last":12.496}]},{"osd":7,"last update":"Sat Sep 23 23:29:49 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.8120000000000001,"15min":1.3879999999999999},"min":{"1min":0.36599999999999999,"5min":0.36399999999999999,"15min":0.36299999999999999},"max":{"1min":2.3879999999999999,"5min":20.350000000000001,"15min":20.350000000000001},"last":15.734999999999999},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.891,"15min":1.474},"min":{"1min":0.39400000000000002,"5min":0.38,"15min":0.38},"max":{"1min":2.3599999999999999,"5min":20.321000000000002,"15min":20.321000000000002},"last":15.709}]}]},{"osd":3,"up_from":148,"seq":635655160042,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4088792,"kb_used_data":3300624,"kb_used_omap":2,"kb_used_meta":788157,"kb_avail":89656360,"statfs":{"total":95995035648,"available":91808112640,"internally_reserved":0,"allocated":3379838976,"data_stored":3377542463,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":807072821},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:29:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":3.395,"15min":3.5099999999999998},"min":{"1min":0.22600000000000001,"5min":0.17699999999999999,"15min":0.158},"max":{"1min":3.1899999999999999,"5min":62.914000000000001,"15min":103.866},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.96699999999999997,"5min":3.3199999999999998,"15min":3.161},"min":{"1min":0.245,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":3.4159999999999999,"5min":63.276000000000003,"15min":101.884},"last":2.149}]},{"osd":1,"last update":"Sat Sep 23 23:29:36 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":2.7469999999999999,"15min":2.7669999999999999},"min":{"1min":0.20000000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":6.5410000000000004,"5min":38.094000000000001,"15min":76.322999999999993},"last":1.0369999999999999},{"interface":"front","average":{"1min":1.5740000000000001,"5min":3.0489999999999999,"15min":2.621},"min":{"1min":0.23799999999999999,"5min":0.21199999999999999,"15min":0.20499999999999999},"max":{"1min":7.835,"5min":42.030999999999999,"15min":76.444000000000003},"last":0.46400000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:29:43 2023","interfaces":[{"interface":"back","average":{"1min":6.9059999999999997,"5min":4.7720000000000002,"15min":3.9849999999999999},"min":{"1min":0.33200000000000002,"5min":0.219,"15min":0.19800000000000001},"max":{"1min":84.724999999999994,"5min":84.724999999999994,"15min":101.279},"last":0.27600000000000002},{"interface":"front","average":{"1min":6.9260000000000002,"5min":4.6130000000000004,"15min":3.3809999999999998},"min":{"1min":0.32400000000000001,"5min":0.214,"15min":0.214},"max":{"1min":84.567999999999998,"5min":84.567999999999998,"15min":101.294},"last":0.48999999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:29:36 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.3640000000000001,"15min":1.5109999999999999},"min":{"1min":0.314,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":5.2249999999999996,"5min":8.3059999999999992,"15min":14.754},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.53,"15min":1.544},"min":{"1min":0.35999999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.0579999999999998,"5min":15.619,"15min":15.619},"last":3.7389999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":1.6379999999999999,"15min":1.7569999999999999},"min":{"1min":0.34899999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.583,"5min":15.762,"15min":75.733000000000004},"last":3.7040000000000002},{"interface":"front","average":{"1min":1.823,"5min":1.8320000000000001,"15min":2.093},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28399999999999997},"max":{"1min":5.8600000000000003,"5min":13.864000000000001,"15min":75.686999999999998},"last":0.34899999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:29:36 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":2.1560000000000001,"15min":1.7869999999999999},"min":{"1min":0.39800000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.1560000000000001,"5min":53.057000000000002,"15min":53.057000000000002},"last":3.8079999999999998},{"interface":"front","average":{"1min":1.8919999999999999,"5min":2.3969999999999998,"15min":2.0310000000000001},"min":{"1min":0.36099999999999999,"5min":0.35299999999999998,"15min":0.33300000000000002},"max":{"1min":5.8170000000000002,"5min":53.088000000000001,"15min":53.088000000000001},"last":0.39700000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:29:43 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":2.1909999999999998,"15min":1.3049999999999999},"min":{"1min":0.38600000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.1369999999999996,"5min":88.344999999999999,"15min":88.344999999999999},"last":146.803},{"interface":"front","average":{"1min":1.534,"5min":2.5,"15min":1.655},"min":{"1min":0.42299999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":9.4990000000000006,"5min":87.855000000000004,"15min":87.855000000000004},"last":146.422}]}]},{"osd":4,"up_from":30,"seq":128849019360,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2561156,"kb_used_data":1820604,"kb_used_omap":0,"kb_used_meta":740544,"kb_avail":91183996,"statfs":{"total":95995035648,"available":93372411904,"internally_reserved":0,"allocated":1864298496,"data_stored":1862960774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":758317056},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:29:59 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":1.752,"15min":4.7140000000000004},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":2.7120000000000002,"5min":40.710000000000001,"15min":199.64400000000001},"last":0.26900000000000002},{"interface":"front","average":{"1min":1.26,"5min":1.8680000000000001,"15min":4.8799999999999999},"min":{"1min":0.24199999999999999,"5min":0.224,"15min":0.216},"max":{"1min":9.3070000000000004,"5min":38.078000000000003,"15min":195.649},"last":0.251}]},{"osd":1,"last update":"Sat Sep 23 23:29:21 2023","interfaces":[{"interface":"back","average":{"1min":3.778,"5min":2.6259999999999999,"15min":4.1559999999999997},"min":{"1min":0.251,"5min":0.24299999999999999,"15min":0.23100000000000001},"max":{"1min":63.966000000000001,"5min":63.966000000000001,"15min":161.923},"last":36.814999999999998},{"interface":"front","average":{"1min":3.7400000000000002,"5min":2.8839999999999999,"15min":4.3620000000000001},"min":{"1min":0.27800000000000002,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":63.738,"5min":63.738,"15min":161.39699999999999},"last":34.497}]},{"osd":2,"last update":"Sat Sep 23 23:29:40 2023","interfaces":[{"interface":"back","average":{"1min":6.5540000000000003,"5min":5.8879999999999999,"15min":3.8340000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.26300000000000001},"max":{"1min":57.651000000000003,"5min":130.506,"15min":130.506},"last":0.317},{"interface":"front","average":{"1min":6.1399999999999997,"5min":5.8109999999999999,"15min":3.629},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":57.167000000000002,"5min":130.209,"15min":130.209},"last":0.44800000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:29:40 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":8.3160000000000007,"15min":6.3479999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.26300000000000001},"max":{"1min":3.1850000000000001,"5min":176.74100000000001,"15min":303.41500000000002},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.2350000000000001,"5min":7.9950000000000001,"15min":6.3979999999999997},"min":{"1min":0.36699999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.9489999999999998,"5min":176.87799999999999,"15min":303.428},"last":0.29199999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:30:15 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":2.5430000000000001,"15min":1.9650000000000001},"min":{"1min":0.26200000000000001,"5min":0.26000000000000001,"15min":0.23699999999999999},"max":{"1min":4.6719999999999997,"5min":161.86500000000001,"15min":161.86500000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.65900000000000003,"5min":2.492,"15min":2.0960000000000001},"min":{"1min":0.29799999999999999,"5min":0.28399999999999997,"15min":0.249},"max":{"1min":2.411,"5min":159.24199999999999,"15min":160.61099999999999},"last":0.32700000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.69499999999999995,"15min":1.163},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.26000000000000001},"max":{"1min":3.496,"5min":5.3399999999999999,"15min":90.346999999999994},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.621,"5min":0.82299999999999995,"15min":1.2869999999999999},"min":{"1min":0.34999999999999998,"5min":0.33200000000000002,"15min":0.30599999999999999},"max":{"1min":1.96,"5min":4.298,"15min":90.733000000000004},"last":0.40699999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":1.6080000000000001,"15min":0.91400000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.5600000000000001,"5min":61.923999999999999,"15min":61.923999999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.76100000000000001,"5min":1.5429999999999999,"15min":1.0489999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.0549999999999997,"5min":61.990000000000002,"15min":61.990000000000002},"last":0.42199999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757723,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1401972,"kb_used_data":777964,"kb_used_omap":0,"kb_used_meta":624000,"kb_avail":92343180,"statfs":{"total":95995035648,"available":94559416320,"internally_reserved":0,"allocated":796635136,"data_stored":795309821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":638976000},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:30:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":2.226,"15min":3.8769999999999998},"min":{"1min":0.23499999999999999,"5min":0.219,"15min":0.19500000000000001},"max":{"1min":10.801,"5min":63.741999999999997,"15min":113.869},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.3939999999999999,"5min":1.9530000000000001,"15min":4.2160000000000002},"min":{"1min":0.27700000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.0419999999999998,"5min":63.259,"15min":138.55199999999999},"last":0.23699999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:30:07 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":3.2360000000000002,"15min":4.1299999999999999},"min":{"1min":0.26800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.1360000000000001,"5min":69.816999999999993,"15min":193.584},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.79,"5min":3.5339999999999998,"15min":4.423},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.25900000000000001},"max":{"1min":7.1479999999999997,"5min":70.268000000000001,"15min":194.03100000000001},"last":0.249}]},{"osd":2,"last update":"Sat Sep 23 23:29:39 2023","interfaces":[{"interface":"back","average":{"1min":3.7010000000000001,"5min":4.3659999999999997,"15min":2.9009999999999998},"min":{"1min":0.27200000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":41.296999999999997,"5min":144.91200000000001,"15min":144.91200000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":4.2690000000000001,"5min":4.335,"15min":2.8290000000000002},"min":{"1min":0.40300000000000002,"5min":0.26500000000000001,"15min":0.25800000000000001},"max":{"1min":41.241,"5min":145.15799999999999,"15min":145.15799999999999},"last":0.46800000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:29:31 2023","interfaces":[{"interface":"back","average":{"1min":10.994999999999999,"5min":4.8959999999999999,"15min":5.3140000000000001},"min":{"1min":0.46700000000000003,"5min":0.32700000000000001,"15min":0.27400000000000002},"max":{"1min":112.71899999999999,"5min":112.71899999999999,"15min":159.53299999999999},"last":0.32000000000000001},{"interface":"front","average":{"1min":10.984999999999999,"5min":4.9240000000000004,"15min":5.5220000000000002},"min":{"1min":0.41599999999999998,"5min":0.318,"15min":0.312},"max":{"1min":112.70099999999999,"5min":112.70099999999999,"15min":159.54900000000001},"last":0.38800000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:29:14 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.72399999999999998,"15min":0.93100000000000005},"min":{"1min":0.26200000000000001,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":0.65700000000000003,"5min":5.3739999999999997,"15min":10.012},"last":19.962},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.61899999999999999,"15min":0.72999999999999998},"min":{"1min":0.29099999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.628,"5min":3.2530000000000001,"15min":11.923999999999999},"last":20.829999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:29:14 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":1.4510000000000001,"15min":2.1829999999999998},"min":{"1min":0.34699999999999998,"5min":0.28599999999999998,"15min":0.26900000000000002},"max":{"1min":0.63,"5min":68.456999999999994,"15min":310.68799999999999},"last":0.439},{"interface":"front","average":{"1min":0.498,"5min":1.4390000000000001,"15min":2.3700000000000001},"min":{"1min":0.35499999999999998,"5min":0.315,"15min":0.25900000000000001},"max":{"1min":0.83299999999999996,"5min":66.721999999999994,"15min":311.45400000000001},"last":0.42899999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:29:39 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":2.8700000000000001,"15min":3.7210000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":1.6060000000000001,"5min":179.28800000000001,"15min":179.28800000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":2.8439999999999999,"15min":3.8380000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.7050000000000001,"5min":179.428,"15min":179.428},"last":0.48699999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2337472512,"data_stored":2337472512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2674135040,"data_stored":2674130968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3713011712,"data_stored":3713007640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3357802496,"data_stored":3357802496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1851195392,"data_stored":1851195392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":787546112,"data_stored":787546112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":471990272,"data_stored":471990272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3310948352,"data_stored":3310944280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:30:26.337 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 61 2023-09-23T23:30:26.413 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:30:26.461 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:26.524 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:30:26.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:26.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:30:26.618 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:30:26.629 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:30:26.721 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:30:26.725 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:26.860 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:30:26.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:26.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:30:26.980 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:30:27.013 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:30:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:30:26] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.43.0" 2023-09-23T23:30:27.089 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:27.269 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:27 smithi110 ceph-mon[221102]: pgmap v1379: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 9.3 MiB/s wr, 726 op/s; 20115/267381 objects degraded (7.523%); 14752/267381 objects misplaced (5.517%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T23:30:27.270 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:27 smithi110 ceph-mon[212939]: pgmap v1379: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 9.3 MiB/s wr, 726 op/s; 20115/267381 objects degraded (7.523%); 14752/267381 objects misplaced (5.517%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T23:30:27.326 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:30:27.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:27.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:30:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:27 smithi138 ceph-mon[193606]: pgmap v1379: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 9.3 MiB/s wr, 726 op/s; 20115/267381 objects degraded (7.523%); 14752/267381 objects misplaced (5.517%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T23:30:27.443 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:30:27.545 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:27.800 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:30:27.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:27.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:30:27.911 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:30:27.946 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:28.009 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:30:28.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:28.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:30:28.116 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:30:28.199 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:28.389 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:30:28.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:28.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:28.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:28.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:30:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:28.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:30:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:30:29.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:29 smithi138 ceph-mon[193606]: pgmap v1380: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 92 B/s rd, 3.3 MiB/s wr, 646 op/s; 20115/264435 objects degraded (7.607%); 14206/264435 objects misplaced (5.372%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T23:30:29.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:29 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 20115/264435 objects degraded (7.607%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:29.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:29 smithi110 ceph-mon[221102]: pgmap v1380: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 92 B/s rd, 3.3 MiB/s wr, 646 op/s; 20115/264435 objects degraded (7.607%); 14206/264435 objects misplaced (5.372%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T23:30:29.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:29 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 20115/264435 objects degraded (7.607%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:29.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:29 smithi110 ceph-mon[212939]: pgmap v1380: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 92 B/s rd, 3.3 MiB/s wr, 646 op/s; 20115/264435 objects degraded (7.607%); 14206/264435 objects misplaced (5.372%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T23:30:29.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:29 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 20115/264435 objects degraded (7.607%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:30.119 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:30:30.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:30.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:30:30.223 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:30:30.251 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:30.318 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:30:30.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:30.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:30:30.424 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:30:30.460 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:30.517 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:30:30.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:30.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:30:30.624 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:30:30.711 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:30.720 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:30:30.869 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:30:30.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:30.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:30:30.966 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:30:30.973 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:30:31.002 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:31.053 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:30:31.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:31.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:30:31.164 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:30:31.195 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:31.254 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:30:31.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:31.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:30:31.360 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:30:31.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:31 smithi110 ceph-mon[212939]: pgmap v1381: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 3.0 MiB/s wr, 725 op/s; 20115/259920 objects degraded (7.739%); 13984/259920 objects misplaced (5.380%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T23:30:31.423 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:31 smithi110 ceph-mon[221102]: pgmap v1381: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 3.0 MiB/s wr, 725 op/s; 20115/259920 objects degraded (7.739%); 13984/259920 objects misplaced (5.380%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T23:30:31.446 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:31.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:31 smithi138 ceph-mon[193606]: pgmap v1381: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 3.0 MiB/s wr, 725 op/s; 20115/259920 objects degraded (7.739%); 13984/259920 objects misplaced (5.380%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T23:30:32.133 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:30:32.133 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:30:32.133 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.133 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:107059 12.f 12:fb94107b:::benchmark_data_smithi110_491193_object12184:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T23:30:32.133 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:30:32.100663+0000", 2023-09-23T23:30:32.134 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.024683366000000002, 2023-09-23T23:30:32.134 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024715096999999998, 2023-09-23T23:30:32.134 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:32.134 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:30:32.134 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:32.134 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:30:32.134 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:30:32.134 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 107059 2023-09-23T23:30:32.134 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.134 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:32.134 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.135 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:32.135 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100663+0000", 2023-09-23T23:30:32.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.135 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.135 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.135 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:32.135 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100663+0000", 2023-09-23T23:30:32.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.135 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.135 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.135 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:32.135 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100661+0000", 2023-09-23T23:30:32.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:30:32.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.136 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.136 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:32.136 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100671+0000", 2023-09-23T23:30:32.136 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0358e-05 2023-09-23T23:30:32.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.136 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.136 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:32.136 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100688+0000", 2023-09-23T23:30:32.136 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7329999999999998e-05 2023-09-23T23:30:32.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.136 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.136 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:32.137 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100699+0000", 2023-09-23T23:30:32.137 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.102e-05 2023-09-23T23:30:32.137 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.137 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:32.137 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100778+0000", 2023-09-23T23:30:32.137 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8683000000000005e-05 2023-09-23T23:30:32.137 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.137 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:32.137 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100985+0000", 2023-09-23T23:30:32.137 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020654700000000001 2023-09-23T23:30:32.137 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.138 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:30:32.138 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.101162+0000", 2023-09-23T23:30:32.138 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017781399999999999 2023-09-23T23:30:32.138 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.138 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.138 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:32.138 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.113422+0000", 2023-09-23T23:30:32.138 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012259404999999999 2023-09-23T23:30:32.138 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.138 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.138 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:32.138 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.114165+0000", 2023-09-23T23:30:32.139 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00074330499999999999 2023-09-23T23:30:32.139 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.139 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:32.139 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.139 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.139 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.139 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:107061 12.9 12:901af6a7:::benchmark_data_smithi110_491193_object12186:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T23:30:32.139 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:30:32.100721+0000", 2023-09-23T23:30:32.139 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.024624703000000001, 2023-09-23T23:30:32.139 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024748661000000002, 2023-09-23T23:30:32.139 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:32.139 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:30:32.139 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:32.140 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:30:32.140 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:30:32.140 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 107061 2023-09-23T23:30:32.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.140 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:32.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.140 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:32.140 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100721+0000", 2023-09-23T23:30:32.140 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.140 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:32.140 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100721+0000", 2023-09-23T23:30:32.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.141 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.141 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.141 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:32.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100721+0000", 2023-09-23T23:30:32.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:30:32.141 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.141 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.141 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:32.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100724+0000", 2023-09-23T23:30:32.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7929999999999999e-06 2023-09-23T23:30:32.141 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.141 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.142 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:32.142 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100733+0000", 2023-09-23T23:30:32.142 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.191e-06 2023-09-23T23:30:32.142 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.142 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.142 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:32.142 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100739+0000", 2023-09-23T23:30:32.142 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2060000000000004e-06 2023-09-23T23:30:32.142 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.142 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.142 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:32.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100865+0000", 2023-09-23T23:30:32.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012625599999999999 2023-09-23T23:30:32.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:32.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.101002+0000", 2023-09-23T23:30:32.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013682899999999999 2023-09-23T23:30:32.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:30:32.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.101135+0000", 2023-09-23T23:30:32.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013296299999999999 2023-09-23T23:30:32.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.144 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:32.144 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.106875+0000", 2023-09-23T23:30:32.144 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0057398939999999997 2023-09-23T23:30:32.144 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.144 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:32.144 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.144 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.144 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:107064 12.c 12:3593c8e1:::benchmark_data_smithi110_491193_object12189:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T23:30:32.145 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:30:32.100756+0000", 2023-09-23T23:30:32.145 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.024590078000000001, 2023-09-23T23:30:32.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024811691, 2023-09-23T23:30:32.145 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:32.145 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:30:32.145 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:32.145 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:30:32.145 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:30:32.145 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 107064 2023-09-23T23:30:32.145 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.146 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:32.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.146 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:32.146 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100756+0000", 2023-09-23T23:30:32.146 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.146 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.146 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:32.146 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100756+0000", 2023-09-23T23:30:32.146 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.146 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:32.147 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100755+0000", 2023-09-23T23:30:32.147 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:30:32.147 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.147 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:32.147 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100759+0000", 2023-09-23T23:30:32.147 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1970000000000001e-06 2023-09-23T23:30:32.147 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.147 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:32.147 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100766+0000", 2023-09-23T23:30:32.147 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0960000000000003e-06 2023-09-23T23:30:32.148 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.148 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:32.148 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100772+0000", 2023-09-23T23:30:32.148 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9039999999999997e-06 2023-09-23T23:30:32.148 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.148 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:32.148 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.101768+0000", 2023-09-23T23:30:32.154 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000996067 2023-09-23T23:30:32.154 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.154 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.154 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:32.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.101873+0000", 2023-09-23T23:30:32.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105791 2023-09-23T23:30:32.155 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.155 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:32.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.101974+0000", 2023-09-23T23:30:32.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000100889 2023-09-23T23:30:32.155 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.155 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:32.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.107109+0000", 2023-09-23T23:30:32.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051349409999999996 2023-09-23T23:30:32.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.156 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.156 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:32.156 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111111+0000", 2023-09-23T23:30:32.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040019110000000004 2023-09-23T23:30:32.156 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.156 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:32.156 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.156 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.156 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:107065 12.9 12:9298ac0f:::benchmark_data_smithi110_491193_object12190:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T23:30:32.157 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:30:32.100786+0000", 2023-09-23T23:30:32.157 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0245604, 2023-09-23T23:30:32.157 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024869685999999998, 2023-09-23T23:30:32.157 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:32.157 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:30:32.157 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:32.157 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:30:32.157 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:30:32.157 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 107065 2023-09-23T23:30:32.157 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.157 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:32.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:32.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100786+0000", 2023-09-23T23:30:32.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.158 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:32.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100786+0000", 2023-09-23T23:30:32.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.158 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:32.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100785+0000", 2023-09-23T23:30:32.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:30:32.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:32.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100788+0000", 2023-09-23T23:30:32.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0299999999999998e-06 2023-09-23T23:30:32.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:32.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100795+0000", 2023-09-23T23:30:32.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7510000000000002e-06 2023-09-23T23:30:32.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.160 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:32.160 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100800+0000", 2023-09-23T23:30:32.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4240000000000001e-06 2023-09-23T23:30:32.160 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.160 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.160 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:32.160 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.102979+0000", 2023-09-23T23:30:32.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021783689999999999 2023-09-23T23:30:32.160 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.160 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.160 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:32.160 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.103117+0000", 2023-09-23T23:30:32.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000138718 2023-09-23T23:30:32.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:30:32.161 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.103248+0000", 2023-09-23T23:30:32.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000130767 2023-09-23T23:30:32.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:32.161 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.107198+0000", 2023-09-23T23:30:32.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039495709999999998 2023-09-23T23:30:32.161 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.161 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:32.162 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.162 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:107066 12.9 12:96a34d52:::benchmark_data_smithi110_491193_object12191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T23:30:32.162 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:30:32.100806+0000", 2023-09-23T23:30:32.162 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.024540388, 2023-09-23T23:30:32.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024927035, 2023-09-23T23:30:32.162 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:32.162 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:30:32.162 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:32.162 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:30:32.163 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:30:32.163 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 107066 2023-09-23T23:30:32.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.163 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:32.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:32.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100806+0000", 2023-09-23T23:30:32.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:32.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100806+0000", 2023-09-23T23:30:32.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:32.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100805+0000", 2023-09-23T23:30:32.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:30:32.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:32.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100808+0000", 2023-09-23T23:30:32.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.835e-06 2023-09-23T23:30:32.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:32.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100814+0000", 2023-09-23T23:30:32.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4189999999999999e-06 2023-09-23T23:30:32.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:32.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.100820+0000", 2023-09-23T23:30:32.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4110000000000004e-06 2023-09-23T23:30:32.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:32.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.104935+0000", 2023-09-23T23:30:32.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041151850000000004 2023-09-23T23:30:32.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:32.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.105074+0000", 2023-09-23T23:30:32.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0001395 2023-09-23T23:30:32.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:30:32.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.105214+0000", 2023-09-23T23:30:32.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013940600000000001 2023-09-23T23:30:32.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:32.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.107203+0000", 2023-09-23T23:30:32.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019886970000000002 2023-09-23T23:30:32.167 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.167 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:32.167 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.167 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:107068 12.f 12:f76755a3:::benchmark_data_smithi110_491193_object12193:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T23:30:32.167 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:30:32.106908+0000", 2023-09-23T23:30:32.167 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.01843763, 2023-09-23T23:30:32.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018908034000000001, 2023-09-23T23:30:32.167 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:32.168 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:30:32.168 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:32.168 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:30:32.168 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:30:32.168 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 107068 2023-09-23T23:30:32.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.168 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:32.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:32.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.106908+0000", 2023-09-23T23:30:32.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:32.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.106908+0000", 2023-09-23T23:30:32.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:32.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.106907+0000", 2023-09-23T23:30:32.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:30:32.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:32.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.106914+0000", 2023-09-23T23:30:32.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3640000000000001e-06 2023-09-23T23:30:32.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:32.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.106926+0000", 2023-09-23T23:30:32.170 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1874e-05 2023-09-23T23:30:32.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:32.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.106934+0000", 2023-09-23T23:30:32.170 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8120000000000008e-06 2023-09-23T23:30:32.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.109569+0000", 2023-09-23T23:30:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026354109999999998 2023-09-23T23:30:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.109744+0000", 2023-09-23T23:30:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017476500000000001 2023-09-23T23:30:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:30:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.109918+0000", 2023-09-23T23:30:32.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017439300000000001 2023-09-23T23:30:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:107070 12.c 12:357b919d:::benchmark_data_smithi110_491193_object12195:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T23:30:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:30:32.110932+0000", 2023-09-23T23:30:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.014413719, 2023-09-23T23:30:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014956798, 2023-09-23T23:30:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:30:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:32.172 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:30:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:30:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 107070 2023-09-23T23:30:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.110932+0000", 2023-09-23T23:30:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:32.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.110932+0000", 2023-09-23T23:30:32.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:32.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.110930+0000", 2023-09-23T23:30:32.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:30:32.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:32.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.110938+0000", 2023-09-23T23:30:32.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1680000000000004e-06 2023-09-23T23:30:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.110951+0000", 2023-09-23T23:30:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2700999999999999e-05 2023-09-23T23:30:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.110959+0000", 2023-09-23T23:30:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4290000000000001e-06 2023-09-23T23:30:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:32.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111140+0000", 2023-09-23T23:30:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018056499999999999 2023-09-23T23:30:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111262+0000", 2023-09-23T23:30:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000121923 2023-09-23T23:30:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111390+0000", 2023-09-23T23:30:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000128317 2023-09-23T23:30:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.176 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:107073 12.9 12:960504f4:::benchmark_data_smithi110_491193_object12198:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T23:30:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:30:32.111065+0000", 2023-09-23T23:30:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.014281297, 2023-09-23T23:30:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014894640000000001, 2023-09-23T23:30:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:30:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:30:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:30:32.177 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 107073 2023-09-23T23:30:32.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.178 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:32.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:32.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111065+0000", 2023-09-23T23:30:32.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:32.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111065+0000", 2023-09-23T23:30:32.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:32.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111064+0000", 2023-09-23T23:30:32.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:30:32.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:32.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111068+0000", 2023-09-23T23:30:32.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3440000000000003e-06 2023-09-23T23:30:32.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:32.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111079+0000", 2023-09-23T23:30:32.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0257e-05 2023-09-23T23:30:32.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:32.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111085+0000", 2023-09-23T23:30:32.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8199999999999999e-06 2023-09-23T23:30:32.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:32.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111150+0000", 2023-09-23T23:30:32.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4709000000000007e-05 2023-09-23T23:30:32.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:32.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111275+0000", 2023-09-23T23:30:32.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012516400000000001 2023-09-23T23:30:32.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:30:32.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111388+0000", 2023-09-23T23:30:32.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000112979 2023-09-23T23:30:32.181 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.181 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:32.181 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.182 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:107069 12.d e502/476 12:b4b102e9:::benchmark_data_smithi110_491193_object12194:head v 502'6669, mlcod=502'6667)", 2023-09-23T23:30:32.182 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:30:32.111468+0000", 2023-09-23T23:30:32.182 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.013878016999999999, 2023-09-23T23:30:32.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014560099, 2023-09-23T23:30:32.182 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:32.182 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:30:32.182 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:32.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:32.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111468+0000", 2023-09-23T23:30:32.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:32.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111468+0000", 2023-09-23T23:30:32.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:32.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111467+0000", 2023-09-23T23:30:32.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:30:32.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:32.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111475+0000", 2023-09-23T23:30:32.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4070000000000004e-06 2023-09-23T23:30:32.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:32.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111482+0000", 2023-09-23T23:30:32.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5809999999999999e-06 2023-09-23T23:30:32.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:32.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111488+0000", 2023-09-23T23:30:32.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9689999999999999e-06 2023-09-23T23:30:32.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:32.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111533+0000", 2023-09-23T23:30:32.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4722000000000003e-05 2023-09-23T23:30:32.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:32.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.111576+0000", 2023-09-23T23:30:32.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2781999999999998e-05 2023-09-23T23:30:32.185 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.185 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:32.185 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.186 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:107072 12.18 e502/490 12:1b40bc3a:::benchmark_data_smithi110_491193_object12197:head v 502'5591, mlcod=502'5589)", 2023-09-23T23:30:32.186 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:30:32.113657+0000", 2023-09-23T23:30:32.186 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.01168926, 2023-09-23T23:30:32.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012432159999999999, 2023-09-23T23:30:32.186 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:32.186 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:30:32.186 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:32.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:32.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.113657+0000", 2023-09-23T23:30:32.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:32.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.113657+0000", 2023-09-23T23:30:32.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:32.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.113656+0000", 2023-09-23T23:30:32.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:30:32.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:32.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.113663+0000", 2023-09-23T23:30:32.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7429999999999996e-06 2023-09-23T23:30:32.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:32.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.113673+0000", 2023-09-23T23:30:32.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0529e-05 2023-09-23T23:30:32.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:32.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.113680+0000", 2023-09-23T23:30:32.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2870000000000001e-06 2023-09-23T23:30:32.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:32.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.116508+0000", 2023-09-23T23:30:32.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00282847 2023-09-23T23:30:32.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:32.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.116530+0000", 2023-09-23T23:30:32.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1814000000000002e-05 2023-09-23T23:30:32.189 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.189 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:32.189 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.190 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.46229.0:107061 12.9 e502/496)", 2023-09-23T23:30:32.190 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:30:32.114857+0000", 2023-09-23T23:30:32.190 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.010489274, 2023-09-23T23:30:32.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011291252999999999, 2023-09-23T23:30:32.190 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:32.190 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:30:32.190 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:32.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:32.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.114857+0000", 2023-09-23T23:30:32.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:32.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.114857+0000", 2023-09-23T23:30:32.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:32.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.114856+0000", 2023-09-23T23:30:32.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:30:32.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:32.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.114866+0000", 2023-09-23T23:30:32.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0264e-05 2023-09-23T23:30:32.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:32.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.114880+0000", 2023-09-23T23:30:32.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3179999999999999e-05 2023-09-23T23:30:32.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:32.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.114887+0000", 2023-09-23T23:30:32.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0180000000000004e-06 2023-09-23T23:30:32.192 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.192 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:32.193 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.193 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:107071 12.7 e502/490 12:e14bfb35:::benchmark_data_smithi110_491193_object12196:head v 502'5617, mlcod=502'5615)", 2023-09-23T23:30:32.193 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:30:32.121706+0000", 2023-09-23T23:30:32.193 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0036398939999999999, 2023-09-23T23:30:32.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044924500000000003, 2023-09-23T23:30:32.193 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:32.193 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:30:32.193 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:32.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:32.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.121706+0000", 2023-09-23T23:30:32.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:32.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.121706+0000", 2023-09-23T23:30:32.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:32.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.121705+0000", 2023-09-23T23:30:32.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:30:32.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:32.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.121714+0000", 2023-09-23T23:30:32.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3000000000000007e-06 2023-09-23T23:30:32.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:32.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.121732+0000", 2023-09-23T23:30:32.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7618e-05 2023-09-23T23:30:32.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:32.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.121741+0000", 2023-09-23T23:30:32.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3780000000000007e-06 2023-09-23T23:30:32.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:32.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.121821+0000", 2023-09-23T23:30:32.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9412000000000005e-05 2023-09-23T23:30:32.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:32.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.121844+0000", 2023-09-23T23:30:32.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.368e-05 2023-09-23T23:30:32.196 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.196 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:32.196 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.197 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:107074 12.f 12:f67fa451:::benchmark_data_smithi110_491193_object12199:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T23:30:32.197 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:30:32.125156+0000", 2023-09-23T23:30:32.197 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.000189789, 2023-09-23T23:30:32.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001108422, 2023-09-23T23:30:32.197 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:32.197 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:30:32.197 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:32.197 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:30:32.197 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:30:32.197 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 107074 2023-09-23T23:30:32.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.198 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:32.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:32.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.125156+0000", 2023-09-23T23:30:32.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:32.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.125156+0000", 2023-09-23T23:30:32.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:32.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:32.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.125154+0000", 2023-09-23T23:30:32.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:30:32.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:32.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.125164+0000", 2023-09-23T23:30:32.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4190000000000008e-06 2023-09-23T23:30:32.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:32.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.125181+0000", 2023-09-23T23:30:32.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7292e-05 2023-09-23T23:30:32.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:32.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.125190+0000", 2023-09-23T23:30:32.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2210000000000003e-06 2023-09-23T23:30:32.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:32.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.125243+0000", 2023-09-23T23:30:32.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2620000000000001e-05 2023-09-23T23:30:32.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:32.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.125383+0000", 2023-09-23T23:30:32.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014057100000000001 2023-09-23T23:30:32.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:32.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:32.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:30:32.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:30:32.125490+0000", 2023-09-23T23:30:32.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000106964 2023-09-23T23:30:32.201 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.201 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:32.201 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.201 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:32.201 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-09-23T23:30:32.202 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 13 2023-09-23T23:30:32.202 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:30:32.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:32.203 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:30:32.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:30:32.304 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:30:32.373 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:32.467 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:30:32.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:32 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2135871369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:30:32.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:32 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:30:32.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:32 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2135871369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:30:32.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:32 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:30:32.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:32 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2135871369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:30:32.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:32 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:30:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:33 smithi138 ceph-mon[193606]: pgmap v1382: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 432 KiB/s wr, 787 op/s; 20115/253149 objects degraded (7.946%); 13144/253149 objects misplaced (5.192%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T23:30:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:33 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:30:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:33 smithi138 ceph-mon[193606]: failed to encode map e503 with expected crc 2023-09-23T23:30:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:33 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:30:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:33 smithi138 ceph-mon[193606]: osdmap e503: 8 total, 8 up, 8 in 2023-09-23T23:30:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:33 smithi138 ceph-mon[193606]: failed to encode map e503 with expected crc 2023-09-23T23:30:33.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:33 smithi110 ceph-mon[212939]: pgmap v1382: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 432 KiB/s wr, 787 op/s; 20115/253149 objects degraded (7.946%); 13144/253149 objects misplaced (5.192%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T23:30:33.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:33 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:30:33.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:33 smithi110 ceph-mon[212939]: failed to encode map e503 with expected crc 2023-09-23T23:30:33.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:33 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:30:33.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:33 smithi110 ceph-mon[212939]: osdmap e503: 8 total, 8 up, 8 in 2023-09-23T23:30:33.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:33 smithi110 ceph-mon[212939]: failed to encode map e503 with expected crc 2023-09-23T23:30:33.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:33 smithi110 ceph-mon[221102]: pgmap v1382: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 432 KiB/s wr, 787 op/s; 20115/253149 objects degraded (7.946%); 13144/253149 objects misplaced (5.192%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T23:30:33.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:33 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:30:33.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:33 smithi110 ceph-mon[221102]: failed to encode map e503 with expected crc 2023-09-23T23:30:33.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:33 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:30:33.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:33 smithi110 ceph-mon[221102]: osdmap e503: 8 total, 8 up, 8 in 2023-09-23T23:30:33.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:33 smithi110 ceph-mon[221102]: failed to encode map e503 with expected crc 2023-09-23T23:30:34.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:34 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 20115/253149 objects degraded (7.946%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:34.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:34 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 20115/253149 objects degraded (7.946%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:34 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 20115/253149 objects degraded (7.946%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:34.951 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:30:34.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:34.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:30:35.057 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:30:35.144 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:35.366 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:30:35.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:35.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:30:35.479 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:30:35.542 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:35 smithi138 ceph-mon[193606]: pgmap v1384: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 20115/251730 objects degraded (7.991%); 13144/251730 objects misplaced (5.221%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T23:30:35.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:35 smithi110 ceph-mon[212939]: pgmap v1384: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 20115/251730 objects degraded (7.991%); 13144/251730 objects misplaced (5.221%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T23:30:35.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:35 smithi110 ceph-mon[221102]: pgmap v1384: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 20115/251730 objects degraded (7.991%); 13144/251730 objects misplaced (5.221%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T23:30:36.341 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:30:36.467 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:30:36.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:36.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:30:36.578 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:30:36.624 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:36.700 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:30:36.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:36.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:30:36.807 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:30:36.848 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:37.136 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:30:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:30:36] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.43.0" 2023-09-23T23:30:37.343 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:30:37.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:37.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:30:37.438 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:37 smithi110 ceph-mon[212939]: pgmap v1385: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 20115/245292 objects degraded (8.200%); 12350/245292 objects misplaced (5.035%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T23:30:37.439 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:37 smithi110 ceph-mon[221102]: pgmap v1385: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 20115/245292 objects degraded (8.200%); 12350/245292 objects misplaced (5.035%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T23:30:37.450 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:30:37.527 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:37.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:37 smithi138 ceph-mon[193606]: pgmap v1385: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 20115/245292 objects degraded (8.200%); 12350/245292 objects misplaced (5.035%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T23:30:37.703 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:30:37.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:37.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:30:37.808 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:30:37.848 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:37.913 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:30:37.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:37.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:30:38.019 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:38.047 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:38.110 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:30:38.116 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:38.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:38.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:38.226 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:30:38.262 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:38.320 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:30:38.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:38.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:30:38.425 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:30:38.527 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:38.668 INFO:teuthology.orchestra.run.smithi110.stderr:Error EINVAL: specified pgp_num 61 > pg_num 31 2023-09-23T23:30:39.172 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:30:39.524 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:39 smithi110 ceph-mon[212939]: pgmap v1386: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 20115/241974 objects degraded (8.313%); 11794/241974 objects misplaced (4.874%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T23:30:39.524 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:39 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 20115/241974 objects degraded (8.313%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:39.524 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:39 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1087484985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:30:39.524 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:39 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3326802190' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "61"}]: dispatch 2023-09-23T23:30:39.524 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:39 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "61"}]: dispatch 2023-09-23T23:30:39.524 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:39 smithi110 ceph-mon[221102]: pgmap v1386: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 20115/241974 objects degraded (8.313%); 11794/241974 objects misplaced (4.874%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T23:30:39.524 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:39 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 20115/241974 objects degraded (8.313%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:39.524 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:39 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1087484985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:30:39.525 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:39 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3326802190' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "61"}]: dispatch 2023-09-23T23:30:39.525 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:39 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "61"}]: dispatch 2023-09-23T23:30:39.527 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:30:39.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:39.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:30:39.639 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:39.673 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:39.746 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:39.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:39.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:39.761 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:39 smithi138 ceph-mon[193606]: pgmap v1386: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 20115/241974 objects degraded (8.313%); 11794/241974 objects misplaced (4.874%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T23:30:39.761 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:39 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 20115/241974 objects degraded (8.313%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:39.761 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:39 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1087484985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:30:39.762 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:39 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3326802190' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "61"}]: dispatch 2023-09-23T23:30:39.762 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:39 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "61"}]: dispatch 2023-09-23T23:30:39.861 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:30:39.952 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:40.097 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:30:40.097 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:30:40.097 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:30:40.097 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:30:40.097 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:30:40.097 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:30:40.098 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:30:40.098 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:30:40.098 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:30:40.098 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:30:40.098 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:30:40.098 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:30:40.098 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:30:40.098 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:30:40.099 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:30:40.099 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:30:40.933 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:40 smithi110 ceph-mon[212939]: pgmap v1387: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 20115/237549 objects degraded (8.468%); 11556/237549 objects misplaced (4.865%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T23:30:40.933 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:40 smithi110 ceph-mon[221102]: pgmap v1387: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 20115/237549 objects degraded (8.468%); 11556/237549 objects misplaced (4.865%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T23:30:40.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:40 smithi138 ceph-mon[193606]: pgmap v1387: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 20115/237549 objects degraded (8.468%); 11556/237549 objects misplaced (4.865%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T23:30:41.694 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T23:30:42.242 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:30:42.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:42.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:30:42.348 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:30:42.446 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:42.615 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:30:42.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:42.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:30:42.722 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:30:42.753 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:42.765 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:30:42.807 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:30:42.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:42.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:30:42.866 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:42 smithi110 ceph-mon[212939]: pgmap v1388: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 20115/230763 objects degraded (8.717%); 10401/230763 objects misplaced (4.507%); 11 MiB/s, 168 objects/s recovering 2023-09-23T23:30:42.866 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:42 smithi110 ceph-mon[221102]: pgmap v1388: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 20115/230763 objects degraded (8.717%); 10401/230763 objects misplaced (4.507%); 11 MiB/s, 168 objects/s recovering 2023-09-23T23:30:42.915 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:30:42.944 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:43.008 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:30:43.013 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:42 smithi138 ceph-mon[193606]: pgmap v1388: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 20115/230763 objects degraded (8.717%); 10401/230763 objects misplaced (4.507%); 11 MiB/s, 168 objects/s recovering 2023-09-23T23:30:43.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:43.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:30:43.114 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:30:43.153 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:43.317 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:30:43.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:43.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:30:43.421 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:30:43.447 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:43.490 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:30:43.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:43.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:30:43.580 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:30:43.580 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:30:43.597 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:30:43.632 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:43.836 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:30:43.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:43.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:30:43.940 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:30:44.035 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:43 smithi138 ceph-mon[193606]: failed to encode map e504 with expected crc 2023-09-23T23:30:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:43 smithi138 ceph-mon[193606]: osdmap e504: 8 total, 8 up, 8 in 2023-09-23T23:30:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:43 smithi138 ceph-mon[193606]: failed to encode map e504 with expected crc 2023-09-23T23:30:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:43 smithi138 ceph-mon[193606]: failed to encode map e504 with expected crc 2023-09-23T23:30:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:43 smithi138 ceph-mon[193606]: failed to encode map e504 with expected crc 2023-09-23T23:30:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:30:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:44.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:44.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:43 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 20115/230763 objects degraded (8.717%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:44.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[212939]: failed to encode map e504 with expected crc 2023-09-23T23:30:44.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[212939]: osdmap e504: 8 total, 8 up, 8 in 2023-09-23T23:30:44.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[212939]: failed to encode map e504 with expected crc 2023-09-23T23:30:44.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[212939]: failed to encode map e504 with expected crc 2023-09-23T23:30:44.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[212939]: failed to encode map e504 with expected crc 2023-09-23T23:30:44.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:30:44.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:44.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:44.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 20115/230763 objects degraded (8.717%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[221102]: failed to encode map e504 with expected crc 2023-09-23T23:30:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[221102]: osdmap e504: 8 total, 8 up, 8 in 2023-09-23T23:30:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[221102]: failed to encode map e504 with expected crc 2023-09-23T23:30:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[221102]: failed to encode map e504 with expected crc 2023-09-23T23:30:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[221102]: failed to encode map e504 with expected crc 2023-09-23T23:30:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:30:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:44.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:43 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 20115/230763 objects degraded (8.717%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:44.712 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:30:45.092 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:30:45.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:45.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:44 smithi110 ceph-mon[221102]: pgmap v1390: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 20115/229395 objects degraded (8.769%); 10401/229395 objects misplaced (4.534%); 11 MiB/s, 168 objects/s recovering 2023-09-23T23:30:45.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:44 smithi110 ceph-mon[221102]: failed to encode map e505 with expected crc 2023-09-23T23:30:45.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:44 smithi110 ceph-mon[221102]: failed to encode map e505 with expected crc 2023-09-23T23:30:45.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:44 smithi110 ceph-mon[221102]: osdmap e505: 8 total, 8 up, 8 in 2023-09-23T23:30:45.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:44 smithi110 ceph-mon[221102]: failed to encode map e505 with expected crc 2023-09-23T23:30:45.101 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:44 smithi110 ceph-mon[221102]: failed to encode map e505 with expected crc 2023-09-23T23:30:45.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:30:45.102 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:44 smithi110 ceph-mon[212939]: pgmap v1390: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 20115/229395 objects degraded (8.769%); 10401/229395 objects misplaced (4.534%); 11 MiB/s, 168 objects/s recovering 2023-09-23T23:30:45.102 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:44 smithi110 ceph-mon[212939]: failed to encode map e505 with expected crc 2023-09-23T23:30:45.103 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:44 smithi110 ceph-mon[212939]: failed to encode map e505 with expected crc 2023-09-23T23:30:45.103 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:44 smithi110 ceph-mon[212939]: osdmap e505: 8 total, 8 up, 8 in 2023-09-23T23:30:45.103 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:44 smithi110 ceph-mon[212939]: failed to encode map e505 with expected crc 2023-09-23T23:30:45.103 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:44 smithi110 ceph-mon[212939]: failed to encode map e505 with expected crc 2023-09-23T23:30:45.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:44 smithi138 ceph-mon[193606]: pgmap v1390: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 117 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 20115/229395 objects degraded (8.769%); 10401/229395 objects misplaced (4.534%); 11 MiB/s, 168 objects/s recovering 2023-09-23T23:30:45.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:44 smithi138 ceph-mon[193606]: failed to encode map e505 with expected crc 2023-09-23T23:30:45.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:44 smithi138 ceph-mon[193606]: failed to encode map e505 with expected crc 2023-09-23T23:30:45.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:44 smithi138 ceph-mon[193606]: osdmap e505: 8 total, 8 up, 8 in 2023-09-23T23:30:45.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:44 smithi138 ceph-mon[193606]: failed to encode map e505 with expected crc 2023-09-23T23:30:45.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:44 smithi138 ceph-mon[193606]: failed to encode map e505 with expected crc 2023-09-23T23:30:45.203 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:45.245 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:45.300 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:45.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:45.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:45.406 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:30:45.472 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:46.144 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:30:46.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:46.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:30:46.251 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:30:46.289 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:46.348 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:30:46.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:46.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:30:46.454 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:30:46.528 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:46.695 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 7, 2, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:30:46.696 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T23:30:46.707 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:30:46.724 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:30:46.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:46.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:30:46.828 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:30:46.863 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:46.927 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:30:46.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:46.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:30:46.950 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:30:47.033 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:30:47.057 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:46 smithi110 ceph-mon[212939]: pgmap v1392: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+degraded, 117 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 18296/223845 objects degraded (8.174%); 11517/223845 objects misplaced (5.145%); 11 MiB/s, 169 objects/s recovering 2023-09-23T23:30:47.057 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:46 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:30:47.057 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:46 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:30:47.057 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:46 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:47.058 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:30:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:30:46] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.43.0" 2023-09-23T23:30:47.058 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:46 smithi110 ceph-mon[221102]: pgmap v1392: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+degraded, 117 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 18296/223845 objects degraded (8.174%); 11517/223845 objects misplaced (5.145%); 11 MiB/s, 169 objects/s recovering 2023-09-23T23:30:47.058 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:46 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:30:47.058 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:46 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:30:47.058 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:46 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:47.066 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:47.119 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:30:47.124 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:46 smithi138 ceph-mon[193606]: pgmap v1392: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+degraded, 117 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 18296/223845 objects degraded (8.174%); 11517/223845 objects misplaced (5.145%); 11 MiB/s, 169 objects/s recovering 2023-09-23T23:30:47.124 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:46 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:30:47.124 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:46 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:30:47.124 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:46 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:30:47.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:47.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:30:47.226 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:30:47.263 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:47.324 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:30:47.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:47.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:30:47.430 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:30:47.459 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:47.514 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:30:47.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:47.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:30:47.620 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:47.654 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:47.715 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:47.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:47.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:47.825 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:30:47.912 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:30:47.912 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:47.975 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:47 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1477051657' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:30:47.975 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:47 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1477051657' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:30:48.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:47 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1477051657' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:30:48.908 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:48 smithi110 ceph-mon[212939]: pgmap v1393: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 118 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 18296/221283 objects degraded (8.268%); 11357/221283 objects misplaced (5.132%); 11 MiB/s, 179 objects/s recovering 2023-09-23T23:30:48.908 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:48 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:30:48.908 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:48 smithi110 ceph-mon[212939]: failed to encode map e506 with expected crc 2023-09-23T23:30:48.908 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:48 smithi110 ceph-mon[212939]: failed to encode map e506 with expected crc 2023-09-23T23:30:48.909 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:48 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1477051657' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:30:48.909 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:48 smithi110 ceph-mon[212939]: osdmap e506: 8 total, 8 up, 8 in 2023-09-23T23:30:48.909 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:48 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 18296/221283 objects degraded (8.268%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:48 smithi138 ceph-mon[193606]: pgmap v1393: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 118 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 18296/221283 objects degraded (8.268%); 11357/221283 objects misplaced (5.132%); 11 MiB/s, 179 objects/s recovering 2023-09-23T23:30:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:48 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:30:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:48 smithi138 ceph-mon[193606]: failed to encode map e506 with expected crc 2023-09-23T23:30:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:48 smithi138 ceph-mon[193606]: failed to encode map e506 with expected crc 2023-09-23T23:30:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:48 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1477051657' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:30:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:48 smithi138 ceph-mon[193606]: osdmap e506: 8 total, 8 up, 8 in 2023-09-23T23:30:49.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:48 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 18296/221283 objects degraded (8.268%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:49.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:48 smithi110 ceph-mon[221102]: pgmap v1393: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 118 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 18296/221283 objects degraded (8.268%); 11357/221283 objects misplaced (5.132%); 11 MiB/s, 179 objects/s recovering 2023-09-23T23:30:49.207 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:48 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:30:49.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:48 smithi110 ceph-mon[221102]: failed to encode map e506 with expected crc 2023-09-23T23:30:49.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:48 smithi110 ceph-mon[221102]: failed to encode map e506 with expected crc 2023-09-23T23:30:49.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:48 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1477051657' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:30:49.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:48 smithi110 ceph-mon[221102]: osdmap e506: 8 total, 8 up, 8 in 2023-09-23T23:30:49.208 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:48 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 18296/221283 objects degraded (8.268%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:49.918 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:49 smithi110 ceph-mon[212939]: 12.2 scrub starts 2023-09-23T23:30:49.919 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:49 smithi110 ceph-mon[221102]: 12.2 scrub starts 2023-09-23T23:30:50.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:49 smithi138 ceph-mon[193606]: 12.2 scrub starts 2023-09-23T23:30:50.232 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:30:50.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:50.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:30:50.339 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:30:50.375 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:50.701 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:30:50.725 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:30:50.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:50.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:30:50.811 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:30:50.850 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:30:50.926 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:50 smithi110 ceph-mon[212939]: pgmap v1395: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 18295/215988 objects degraded (8.470%); 11120/215988 objects misplaced (5.148%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:30:50.952 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:51.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:50 smithi138 ceph-mon[193606]: pgmap v1395: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 18295/215988 objects degraded (8.470%); 11120/215988 objects misplaced (5.148%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:30:51.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:50 smithi110 ceph-mon[221102]: pgmap v1395: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 18295/215988 objects degraded (8.470%); 11120/215988 objects misplaced (5.148%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:30:51.265 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:30:51.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:51.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:30:51.378 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:30:51.463 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:51.666 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:30:51.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:51.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:30:51.775 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:30:51.907 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:51.931 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:51 smithi110 ceph-mon[221102]: 12.3 scrub starts 2023-09-23T23:30:51.931 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:51 smithi110 ceph-mon[221102]: 12.12 scrub starts 2023-09-23T23:30:51.933 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:51 smithi110 ceph-mon[212939]: 12.3 scrub starts 2023-09-23T23:30:51.933 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:51 smithi110 ceph-mon[212939]: 12.12 scrub starts 2023-09-23T23:30:51.978 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:30:51.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:51.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:30:52.085 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:30:52.136 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:52.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:51 smithi138 ceph-mon[193606]: 12.3 scrub starts 2023-09-23T23:30:52.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:51 smithi138 ceph-mon[193606]: 12.12 scrub starts 2023-09-23T23:30:52.476 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:30:52.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:52.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:30:52.591 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:30:52.675 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:52.746 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:30:53.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:52 smithi138 ceph-mon[193606]: pgmap v1396: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 18295/209727 objects degraded (8.723%); 10715/209727 objects misplaced (5.109%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T23:30:53.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:52 smithi138 ceph-mon[193606]: 12.0 scrub starts 2023-09-23T23:30:53.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:52 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4096739018' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:30:53.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:52 smithi110 ceph-mon[212939]: pgmap v1396: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 18295/209727 objects degraded (8.723%); 10715/209727 objects misplaced (5.109%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T23:30:53.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:52 smithi110 ceph-mon[212939]: 12.0 scrub starts 2023-09-23T23:30:53.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:52 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4096739018' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:30:53.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:52 smithi110 ceph-mon[221102]: pgmap v1396: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 18295/209727 objects degraded (8.723%); 10715/209727 objects misplaced (5.109%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T23:30:53.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:52 smithi110 ceph-mon[221102]: 12.0 scrub starts 2023-09-23T23:30:53.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:52 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4096739018' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:30:54.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:53 smithi110 ceph-mon[212939]: 12.2 scrub starts 2023-09-23T23:30:54.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:53 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 18295/209727 objects degraded (8.723%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:54.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:53 smithi110 ceph-mon[221102]: 12.2 scrub starts 2023-09-23T23:30:54.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:53 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 18295/209727 objects degraded (8.723%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:54.281 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:30:54.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:54.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:30:54.336 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:30:54.337 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:30:54.337 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:30:54.337 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:30:54.337 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:30:54.337 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:30:54.337 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:30:54.337 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:30:54.337 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:30:54.338 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:30:54.338 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:30:54.338 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:30:54.338 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:30:54.338 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:30:54.338 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:30:54.338 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:30:54.393 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:30:54.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:53 smithi138 ceph-mon[193606]: 12.2 scrub starts 2023-09-23T23:30:54.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:53 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 18295/209727 objects degraded (8.723%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:30:54.499 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:54.697 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:30:54.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:54.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:30:54.814 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:30:54.841 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:54.882 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:30:54.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:54.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:30:54.988 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:30:55.020 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:55.087 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:30:55.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:55.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:30:55.092 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:54 smithi138 ceph-mon[193606]: pgmap v1397: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 18295/208362 objects degraded (8.780%); 10715/208362 objects misplaced (5.142%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T23:30:55.093 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:54 smithi138 ceph-mon[193606]: 12.13 scrub starts 2023-09-23T23:30:55.093 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:54 smithi138 ceph-mon[193606]: 12.12 scrub starts 2023-09-23T23:30:55.193 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:55.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:54 smithi110 ceph-mon[212939]: pgmap v1397: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 18295/208362 objects degraded (8.780%); 10715/208362 objects misplaced (5.142%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T23:30:55.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:54 smithi110 ceph-mon[212939]: 12.13 scrub starts 2023-09-23T23:30:55.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:54 smithi110 ceph-mon[212939]: 12.12 scrub starts 2023-09-23T23:30:55.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:54 smithi110 ceph-mon[221102]: pgmap v1397: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 18295/208362 objects degraded (8.780%); 10715/208362 objects misplaced (5.142%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T23:30:55.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:54 smithi110 ceph-mon[221102]: 12.13 scrub starts 2023-09-23T23:30:55.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:54 smithi110 ceph-mon[221102]: 12.12 scrub starts 2023-09-23T23:30:55.219 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:55.285 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:55.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:55.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:55.391 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:55.431 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:55.494 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:55.494 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:30:55.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:55.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:55.605 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:30:55.721 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:56.084 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:30:56.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:56.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:30:56.164 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:30:56.164 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:30:56.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:30:56.164 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:30:56.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.164 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156658 10.3 10:c8345f3f:::benchmark_data_smithi110_366450_object55450:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e301)", 2023-09-23T23:30:56.164 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:50.036433+0000", 2023-09-23T23:30:56.165 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 546.12227584300001, 2023-09-23T23:30:56.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2122256870000001, 2023-09-23T23:30:56.165 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.165 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.165 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.165 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.165 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.165 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156658 2023-09-23T23:30:56.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.165 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036433+0000", 2023-09-23T23:30:56.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036433+0000", 2023-09-23T23:30:56.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036432+0000", 2023-09-23T23:30:56.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:30:56.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036438+0000", 2023-09-23T23:30:56.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2260000000000002e-06 2023-09-23T23:30:56.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036447+0000", 2023-09-23T23:30:56.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2599999999999994e-06 2023-09-23T23:30:56.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036454+0000", 2023-09-23T23:30:56.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2180000000000002e-06 2023-09-23T23:30:56.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038082+0000", 2023-09-23T23:30:56.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016271840000000001 2023-09-23T23:30:56.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:30:56.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038092+0000", 2023-09-23T23:30:56.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0312999999999999e-05 2023-09-23T23:30:56.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.132541+0000", 2023-09-23T23:30:56.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.094449464 2023-09-23T23:30:56.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.135895+0000", 2023-09-23T23:30:56.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033539080000000001 2023-09-23T23:30:56.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.880055+0000", 2023-09-23T23:30:56.170 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.744159365 2023-09-23T23:30:56.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.883944+0000", 2023-09-23T23:30:56.170 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038893600000000001 2023-09-23T23:30:56.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.224607+0000", 2023-09-23T23:30:56.170 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.34066281199999998 2023-09-23T23:30:56.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.227336+0000", 2023-09-23T23:30:56.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002729553 2023-09-23T23:30:56.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.232121+0000", 2023-09-23T23:30:56.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047848830000000002 2023-09-23T23:30:56.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.172 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.238887+0000", 2023-09-23T23:30:56.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.006765971 2023-09-23T23:30:56.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.172 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.240149+0000", 2023-09-23T23:30:56.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001261543 2023-09-23T23:30:56.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.172 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.240604+0000", 2023-09-23T23:30:56.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045544900000000002 2023-09-23T23:30:56.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.242329+0000", 2023-09-23T23:30:56.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001724864 2023-09-23T23:30:56.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.248659+0000", 2023-09-23T23:30:56.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063300179999999998 2023-09-23T23:30:56.173 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.173 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.174 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.174 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156665 10.3 10:ce3a78fd:::benchmark_data_smithi110_366450_object55457:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e301)", 2023-09-23T23:30:56.174 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:50.036463+0000", 2023-09-23T23:30:56.174 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 546.12224577500001, 2023-09-23T23:30:56.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2684505270000002, 2023-09-23T23:30:56.174 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.174 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.174 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.174 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.174 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.175 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156665 2023-09-23T23:30:56.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.175 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036463+0000", 2023-09-23T23:30:56.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036463+0000", 2023-09-23T23:30:56.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.176 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036463+0000", 2023-09-23T23:30:56.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:30:56.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.176 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036465+0000", 2023-09-23T23:30:56.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4990000000000001e-06 2023-09-23T23:30:56.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.176 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036471+0000", 2023-09-23T23:30:56.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8850000000000004e-06 2023-09-23T23:30:56.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036477+0000", 2023-09-23T23:30:56.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.857e-06 2023-09-23T23:30:56.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038117+0000", 2023-09-23T23:30:56.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016396080000000001 2023-09-23T23:30:56.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:30:56.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038124+0000", 2023-09-23T23:30:56.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8190000000000003e-06 2023-09-23T23:30:56.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.887950+0000", 2023-09-23T23:30:56.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8498266559999998 2023-09-23T23:30:56.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.891672+0000", 2023-09-23T23:30:56.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037213459999999999 2023-09-23T23:30:56.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.909743+0000", 2023-09-23T23:30:56.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018071329000000001 2023-09-23T23:30:56.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.913176+0000", 2023-09-23T23:30:56.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034328200000000001 2023-09-23T23:30:56.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.256612+0000", 2023-09-23T23:30:56.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.343436199 2023-09-23T23:30:56.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.259680+0000", 2023-09-23T23:30:56.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030675870000000001 2023-09-23T23:30:56.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.265203+0000", 2023-09-23T23:30:56.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055235350000000004 2023-09-23T23:30:56.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.293752+0000", 2023-09-23T23:30:56.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028548720999999999 2023-09-23T23:30:56.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.295411+0000", 2023-09-23T23:30:56.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001659326 2023-09-23T23:30:56.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.296790+0000", 2023-09-23T23:30:56.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013783339999999999 2023-09-23T23:30:56.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.298340+0000", 2023-09-23T23:30:56.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015504679999999999 2023-09-23T23:30:56.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.304914+0000", 2023-09-23T23:30:56.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00657399 2023-09-23T23:30:56.183 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.183 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.183 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.183 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156677 10.3 10:c754b68c:::benchmark_data_smithi110_366450_object55469:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e301)", 2023-09-23T23:30:56.183 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:50.036483+0000", 2023-09-23T23:30:56.183 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 546.12222592199998, 2023-09-23T23:30:56.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2794486950000001, 2023-09-23T23:30:56.184 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.184 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.184 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.184 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.184 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.184 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156677 2023-09-23T23:30:56.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.184 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036483+0000", 2023-09-23T23:30:56.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036483+0000", 2023-09-23T23:30:56.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036483+0000", 2023-09-23T23:30:56.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:30:56.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036485+0000", 2023-09-23T23:30:56.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1509999999999998e-06 2023-09-23T23:30:56.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036491+0000", 2023-09-23T23:30:56.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1029999999999998e-06 2023-09-23T23:30:56.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036497+0000", 2023-09-23T23:30:56.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6699999999999999e-06 2023-09-23T23:30:56.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038146+0000", 2023-09-23T23:30:56.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001649614 2023-09-23T23:30:56.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:30:56.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038153+0000", 2023-09-23T23:30:56.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7120000000000003e-06 2023-09-23T23:30:56.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.892021+0000", 2023-09-23T23:30:56.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8538682949999998 2023-09-23T23:30:56.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.895469+0000", 2023-09-23T23:30:56.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034472249999999999 2023-09-23T23:30:56.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.925797+0000", 2023-09-23T23:30:56.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.030328755999999998 2023-09-23T23:30:56.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.929104+0000", 2023-09-23T23:30:56.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033061599999999998 2023-09-23T23:30:56.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.279422+0000", 2023-09-23T23:30:56.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.35031821000000002 2023-09-23T23:30:56.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.282430+0000", 2023-09-23T23:30:56.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030082590000000001 2023-09-23T23:30:56.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.287816+0000", 2023-09-23T23:30:56.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053855860000000004 2023-09-23T23:30:56.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.305266+0000", 2023-09-23T23:30:56.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017450195000000002 2023-09-23T23:30:56.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.306434+0000", 2023-09-23T23:30:56.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011678719999999999 2023-09-23T23:30:56.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.307444+0000", 2023-09-23T23:30:56.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010101159999999999 2023-09-23T23:30:56.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.309131+0000", 2023-09-23T23:30:56.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001687207 2023-09-23T23:30:56.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.315932+0000", 2023-09-23T23:30:56.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068009699999999999 2023-09-23T23:30:56.192 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.192 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.192 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.193 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156689 10.3 10:c3cc188a:::benchmark_data_smithi110_366450_object55481:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e301)", 2023-09-23T23:30:56.193 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:50.036502+0000", 2023-09-23T23:30:56.193 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 546.12220754500004, 2023-09-23T23:30:56.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.470265768, 2023-09-23T23:30:56.193 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.193 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.193 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.193 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.193 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.193 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156689 2023-09-23T23:30:56.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.194 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036502+0000", 2023-09-23T23:30:56.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036502+0000", 2023-09-23T23:30:56.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036501+0000", 2023-09-23T23:30:56.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:30:56.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036503+0000", 2023-09-23T23:30:56.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1349999999999999e-06 2023-09-23T23:30:56.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036509+0000", 2023-09-23T23:30:56.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9970000000000004e-06 2023-09-23T23:30:56.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.036515+0000", 2023-09-23T23:30:56.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7819999999999999e-06 2023-09-23T23:30:56.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038348+0000", 2023-09-23T23:30:56.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018323930000000001 2023-09-23T23:30:56.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:30:56.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.038356+0000", 2023-09-23T23:30:56.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5450000000000003e-06 2023-09-23T23:30:56.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.930347+0000", 2023-09-23T23:30:56.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.891991215 2023-09-23T23:30:56.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.933759+0000", 2023-09-23T23:30:56.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034111269999999999 2023-09-23T23:30:56.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.956094+0000", 2023-09-23T23:30:56.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022335738000000001 2023-09-23T23:30:56.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.960338+0000", 2023-09-23T23:30:56.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042442030000000002 2023-09-23T23:30:56.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.464888+0000", 2023-09-23T23:30:56.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.50454982800000003 2023-09-23T23:30:56.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.467869+0000", 2023-09-23T23:30:56.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029807459999999998 2023-09-23T23:30:56.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.473841+0000", 2023-09-23T23:30:56.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059724180000000002 2023-09-23T23:30:56.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.496377+0000", 2023-09-23T23:30:56.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022535192999999999 2023-09-23T23:30:56.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.497644+0000", 2023-09-23T23:30:56.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00126756 2023-09-23T23:30:56.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.498689+0000", 2023-09-23T23:30:56.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010452650000000001 2023-09-23T23:30:56.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.500547+0000", 2023-09-23T23:30:56.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018575410000000001 2023-09-23T23:30:56.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.506767+0000", 2023-09-23T23:30:56.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0062204340000000004 2023-09-23T23:30:56.202 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.202 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.202 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.202 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156699 10.3 10:c7e78881:::benchmark_data_smithi110_366450_object55491:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:30:56.202 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.085948+0000", 2023-09-23T23:30:56.203 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 545.07276109600002, 2023-09-23T23:30:56.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.459542565, 2023-09-23T23:30:56.203 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.203 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.203 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.203 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.203 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.203 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156699 2023-09-23T23:30:56.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.203 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085948+0000", 2023-09-23T23:30:56.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.204 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085948+0000", 2023-09-23T23:30:56.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.204 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085946+0000", 2023-09-23T23:30:56.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:30:56.204 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085954+0000", 2023-09-23T23:30:56.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8690000000000004e-06 2023-09-23T23:30:56.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085969+0000", 2023-09-23T23:30:56.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4579999999999999e-05 2023-09-23T23:30:56.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.085979+0000", 2023-09-23T23:30:56.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6730000000000004e-06 2023-09-23T23:30:56.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.934188+0000", 2023-09-23T23:30:56.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.848209207 2023-09-23T23:30:56.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.938192+0000", 2023-09-23T23:30:56.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040045879999999999 2023-09-23T23:30:56.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.207 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.207 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.207 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.974891+0000", 2023-09-23T23:30:56.207 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036698716999999999 2023-09-23T23:30:56.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.207 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.207 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.207 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.978909+0000", 2023-09-23T23:30:56.207 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040178669999999996 2023-09-23T23:30:56.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.207 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.208 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.515734+0000", 2023-09-23T23:30:56.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.53682527300000005 2023-09-23T23:30:56.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.208 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.208 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.518894+0000", 2023-09-23T23:30:56.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031600869999999998 2023-09-23T23:30:56.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.208 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.208 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.524144+0000", 2023-09-23T23:30:56.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052494250000000003 2023-09-23T23:30:56.209 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.209 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.209 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.535043+0000", 2023-09-23T23:30:56.209 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010899067 2023-09-23T23:30:56.209 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.209 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.209 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.535410+0000", 2023-09-23T23:30:56.209 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036688299999999998 2023-09-23T23:30:56.209 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.210 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.210 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.536594+0000", 2023-09-23T23:30:56.210 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001184095 2023-09-23T23:30:56.210 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.210 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.210 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.210 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.538475+0000", 2023-09-23T23:30:56.210 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018812639999999999 2023-09-23T23:30:56.210 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.210 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.210 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.210 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.545491+0000", 2023-09-23T23:30:56.211 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070156009999999998 2023-09-23T23:30:56.211 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.211 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.211 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.211 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.211 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.211 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156707 10.3 10:ca34f172:::benchmark_data_smithi110_366450_object55499:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:30:56.211 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.101757+0000", 2023-09-23T23:30:56.211 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 545.056952658, 2023-09-23T23:30:56.211 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5204141440000001, 2023-09-23T23:30:56.211 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.211 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.212 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.212 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.212 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.212 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156707 2023-09-23T23:30:56.212 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.212 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.212 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101757+0000", 2023-09-23T23:30:56.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.212 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101757+0000", 2023-09-23T23:30:56.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.213 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101754+0000", 2023-09-23T23:30:56.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-09-23T23:30:56.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.213 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101763+0000", 2023-09-23T23:30:56.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1020000000000003e-06 2023-09-23T23:30:56.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101778+0000", 2023-09-23T23:30:56.214 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4268e-05 2023-09-23T23:30:56.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.101787+0000", 2023-09-23T23:30:56.214 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7580000000000004e-06 2023-09-23T23:30:56.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.215 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.939584+0000", 2023-09-23T23:30:56.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8377970509999999 2023-09-23T23:30:56.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.215 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.943370+0000", 2023-09-23T23:30:56.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037859840000000001 2023-09-23T23:30:56.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.215 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.991414+0000", 2023-09-23T23:30:56.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048043296999999999 2023-09-23T23:30:56.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.994689+0000", 2023-09-23T23:30:56.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032749989999999998 2023-09-23T23:30:56.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.572368+0000", 2023-09-23T23:30:56.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.57767970499999999 2023-09-23T23:30:56.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.217 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.575376+0000", 2023-09-23T23:30:56.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030079059999999999 2023-09-23T23:30:56.217 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.217 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.217 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.580798+0000", 2023-09-23T23:30:56.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054212740000000002 2023-09-23T23:30:56.217 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.217 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.217 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.612099+0000", 2023-09-23T23:30:56.218 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031301739000000002 2023-09-23T23:30:56.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.218 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.218 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.612481+0000", 2023-09-23T23:30:56.218 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00038158799999999998 2023-09-23T23:30:56.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.218 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.218 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.613567+0000", 2023-09-23T23:30:56.218 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010858490000000001 2023-09-23T23:30:56.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.615343+0000", 2023-09-23T23:30:56.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017767620000000001 2023-09-23T23:30:56.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.622171+0000", 2023-09-23T23:30:56.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068272410000000004 2023-09-23T23:30:56.219 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.219 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.219 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.220 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPush(10.3 302/301 [PushOp(10:c8345f3f:::benchmark_data_smithi110_366450_object55450:head, version: 272'3485, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:c8345f3f:::benchmark_data_smithi110_366450_object55450:head@272'3485, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:30:56.220 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.136769+0000", 2023-09-23T23:30:56.220 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 545.02194029700001, 2023-09-23T23:30:56.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7428203010000001, 2023-09-23T23:30:56.220 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.220 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:30:56.220 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.220 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.220 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136769+0000", 2023-09-23T23:30:56.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136769+0000", 2023-09-23T23:30:56.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136765+0000", 2023-09-23T23:30:56.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999962 2023-09-23T23:30:56.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136850+0000", 2023-09-23T23:30:56.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5081999999999996e-05 2023-09-23T23:30:56.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136868+0000", 2023-09-23T23:30:56.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8357000000000001e-05 2023-09-23T23:30:56.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.136884+0000", 2023-09-23T23:30:56.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5732000000000001e-05 2023-09-23T23:30:56.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.867402+0000", 2023-09-23T23:30:56.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7305173149999999 2023-09-23T23:30:56.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.867705+0000", 2023-09-23T23:30:56.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030383999999999999 2023-09-23T23:30:56.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.879589+0000", 2023-09-23T23:30:56.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011883874000000001 2023-09-23T23:30:56.224 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.224 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.224 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.224 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156724 10.3 10:ca0db88d:::benchmark_data_smithi110_366450_object55516:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:30:56.224 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.159299+0000", 2023-09-23T23:30:56.224 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 544.99940984099999, 2023-09-23T23:30:56.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.473675901, 2023-09-23T23:30:56.225 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.225 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.225 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.225 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.225 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.225 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156724 2023-09-23T23:30:56.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.225 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159299+0000", 2023-09-23T23:30:56.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159299+0000", 2023-09-23T23:30:56.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159297+0000", 2023-09-23T23:30:56.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:30:56.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159310+0000", 2023-09-23T23:30:56.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2174000000000001e-05 2023-09-23T23:30:56.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159324+0000", 2023-09-23T23:30:56.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4511999999999999e-05 2023-09-23T23:30:56.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.159334+0000", 2023-09-23T23:30:56.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7480000000000005e-06 2023-09-23T23:30:56.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.996054+0000", 2023-09-23T23:30:56.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8367205470000001 2023-09-23T23:30:56.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.999367+0000", 2023-09-23T23:30:56.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033122659999999999 2023-09-23T23:30:56.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.019886+0000", 2023-09-23T23:30:56.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02051973 2023-09-23T23:30:56.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.023203+0000", 2023-09-23T23:30:56.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003316865 2023-09-23T23:30:56.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.595994+0000", 2023-09-23T23:30:56.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.57279082400000003 2023-09-23T23:30:56.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.598978+0000", 2023-09-23T23:30:56.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002984062 2023-09-23T23:30:56.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.604389+0000", 2023-09-23T23:30:56.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054107959999999998 2023-09-23T23:30:56.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.622507+0000", 2023-09-23T23:30:56.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018117985 2023-09-23T23:30:56.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.623486+0000", 2023-09-23T23:30:56.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0009786860000000001 2023-09-23T23:30:56.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.624625+0000", 2023-09-23T23:30:56.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011391330000000001 2023-09-23T23:30:56.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.626256+0000", 2023-09-23T23:30:56.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016308449999999999 2023-09-23T23:30:56.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.632975+0000", 2023-09-23T23:30:56.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.006719727 2023-09-23T23:30:56.233 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.233 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.233 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.233 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156745 10.3 10:c3a25812:::benchmark_data_smithi110_366450_object55537:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:30:56.233 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.191485+0000", 2023-09-23T23:30:56.233 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 544.96722424100005, 2023-09-23T23:30:56.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4810129569999999, 2023-09-23T23:30:56.233 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.233 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.234 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.234 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.234 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.234 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156745 2023-09-23T23:30:56.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.234 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191485+0000", 2023-09-23T23:30:56.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191485+0000", 2023-09-23T23:30:56.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.235 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191483+0000", 2023-09-23T23:30:56.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:30:56.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.235 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191491+0000", 2023-09-23T23:30:56.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1079999999999993e-06 2023-09-23T23:30:56.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191512+0000", 2023-09-23T23:30:56.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0267000000000001e-05 2023-09-23T23:30:56.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.191521+0000", 2023-09-23T23:30:56.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2480000000000002e-06 2023-09-23T23:30:56.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:52.999765+0000", 2023-09-23T23:30:56.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8082440449999999 2023-09-23T23:30:56.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.003124+0000", 2023-09-23T23:30:56.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033588789999999999 2023-09-23T23:30:56.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.036625+0000", 2023-09-23T23:30:56.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.03350144 2023-09-23T23:30:56.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.040838+0000", 2023-09-23T23:30:56.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042131260000000002 2023-09-23T23:30:56.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.641997+0000", 2023-09-23T23:30:56.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.60115844900000004 2023-09-23T23:30:56.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.645336+0000", 2023-09-23T23:30:56.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003339482 2023-09-23T23:30:56.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.651259+0000", 2023-09-23T23:30:56.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059230419999999999 2023-09-23T23:30:56.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.661663+0000", 2023-09-23T23:30:56.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010403302 2023-09-23T23:30:56.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.662676+0000", 2023-09-23T23:30:56.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010129189999999999 2023-09-23T23:30:56.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.663859+0000", 2023-09-23T23:30:56.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011834149999999999 2023-09-23T23:30:56.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.241 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.665740+0000", 2023-09-23T23:30:56.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001880545 2023-09-23T23:30:56.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.241 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.672498+0000", 2023-09-23T23:30:56.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067584250000000002 2023-09-23T23:30:56.241 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.242 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.242 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.242 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156758 10.3 10:c6bd9813:::benchmark_data_smithi110_366450_object55550:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:30:56.242 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.205665+0000", 2023-09-23T23:30:56.242 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 544.95304427999997, 2023-09-23T23:30:56.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6487948870000002, 2023-09-23T23:30:56.242 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.242 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.242 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.242 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.243 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.243 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156758 2023-09-23T23:30:56.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.243 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205665+0000", 2023-09-23T23:30:56.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205665+0000", 2023-09-23T23:30:56.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205664+0000", 2023-09-23T23:30:56.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:30:56.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205670+0000", 2023-09-23T23:30:56.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3929999999999996e-06 2023-09-23T23:30:56.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205683+0000", 2023-09-23T23:30:56.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3288999999999999e-05 2023-09-23T23:30:56.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.205690+0000", 2023-09-23T23:30:56.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1210000000000001e-06 2023-09-23T23:30:56.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.004368+0000", 2023-09-23T23:30:56.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.798677251 2023-09-23T23:30:56.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.007617+0000", 2023-09-23T23:30:56.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032493769999999999 2023-09-23T23:30:56.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.056183+0000", 2023-09-23T23:30:56.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048565482 2023-09-23T23:30:56.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.060370+0000", 2023-09-23T23:30:56.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041874099999999999 2023-09-23T23:30:56.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.811482+0000", 2023-09-23T23:30:56.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.751111588 2023-09-23T23:30:56.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.814815+0000", 2023-09-23T23:30:56.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033331239999999998 2023-09-23T23:30:56.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.819734+0000", 2023-09-23T23:30:56.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0049189960000000001 2023-09-23T23:30:56.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.844633+0000", 2023-09-23T23:30:56.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024899787999999999 2023-09-23T23:30:56.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.845534+0000", 2023-09-23T23:30:56.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00090030399999999999 2023-09-23T23:30:56.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.846942+0000", 2023-09-23T23:30:56.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001408573 2023-09-23T23:30:56.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.848519+0000", 2023-09-23T23:30:56.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001576521 2023-09-23T23:30:56.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.854460+0000", 2023-09-23T23:30:56.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059410060000000004 2023-09-23T23:30:56.250 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.250 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.250 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.251 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156771 10.3 10:c6f4a9de:::benchmark_data_smithi110_366450_object55563:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:30:56.251 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.219050+0000", 2023-09-23T23:30:56.251 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 544.93965940299995, 2023-09-23T23:30:56.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6446795810000001, 2023-09-23T23:30:56.251 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.251 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.251 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.251 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.251 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.251 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156771 2023-09-23T23:30:56.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.252 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219050+0000", 2023-09-23T23:30:56.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219050+0000", 2023-09-23T23:30:56.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219048+0000", 2023-09-23T23:30:56.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:30:56.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219054+0000", 2023-09-23T23:30:56.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8950000000000003e-06 2023-09-23T23:30:56.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219068+0000", 2023-09-23T23:30:56.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4149e-05 2023-09-23T23:30:56.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.254 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.219076+0000", 2023-09-23T23:30:56.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8960000000000003e-06 2023-09-23T23:30:56.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.254 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.070783+0000", 2023-09-23T23:30:56.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8517066980000001 2023-09-23T23:30:56.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.074340+0000", 2023-09-23T23:30:56.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003557185 2023-09-23T23:30:56.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.095850+0000", 2023-09-23T23:30:56.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021510344000000001 2023-09-23T23:30:56.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.099086+0000", 2023-09-23T23:30:56.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032357319999999998 2023-09-23T23:30:56.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.831720+0000", 2023-09-23T23:30:56.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.732633639 2023-09-23T23:30:56.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.834388+0000", 2023-09-23T23:30:56.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026684980000000001 2023-09-23T23:30:56.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.839278+0000", 2023-09-23T23:30:56.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048894430000000003 2023-09-23T23:30:56.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.846624+0000", 2023-09-23T23:30:56.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0073457380000000001 2023-09-23T23:30:56.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.854781+0000", 2023-09-23T23:30:56.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081573540000000003 2023-09-23T23:30:56.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.855781+0000", 2023-09-23T23:30:56.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00099972499999999992 2023-09-23T23:30:56.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.857550+0000", 2023-09-23T23:30:56.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001769135 2023-09-23T23:30:56.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.863729+0000", 2023-09-23T23:30:56.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061796489999999997 2023-09-23T23:30:56.259 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.259 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.259 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.259 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156792 10.3 10:c9793c14:::benchmark_data_smithi110_366450_object55584:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:30:56.260 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.244861+0000", 2023-09-23T23:30:56.260 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 544.91384801200002, 2023-09-23T23:30:56.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.66926905, 2023-09-23T23:30:56.260 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.260 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.260 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.260 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.260 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.260 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156792 2023-09-23T23:30:56.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.260 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244861+0000", 2023-09-23T23:30:56.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244861+0000", 2023-09-23T23:30:56.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244859+0000", 2023-09-23T23:30:56.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:30:56.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244868+0000", 2023-09-23T23:30:56.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.551e-06 2023-09-23T23:30:56.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244881+0000", 2023-09-23T23:30:56.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3215e-05 2023-09-23T23:30:56.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.244888+0000", 2023-09-23T23:30:56.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.199e-06 2023-09-23T23:30:56.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.074685+0000", 2023-09-23T23:30:56.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.829796778 2023-09-23T23:30:56.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.078783+0000", 2023-09-23T23:30:56.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040977440000000004 2023-09-23T23:30:56.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.112127+0000", 2023-09-23T23:30:56.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.033344602000000001 2023-09-23T23:30:56.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.115665+0000", 2023-09-23T23:30:56.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035371859999999999 2023-09-23T23:30:56.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.871651+0000", 2023-09-23T23:30:56.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.75598617700000004 2023-09-23T23:30:56.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.874503+0000", 2023-09-23T23:30:56.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028525780000000001 2023-09-23T23:30:56.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.879361+0000", 2023-09-23T23:30:56.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048573160000000004 2023-09-23T23:30:56.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.905198+0000", 2023-09-23T23:30:56.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.025837373 2023-09-23T23:30:56.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.905568+0000", 2023-09-23T23:30:56.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000370021 2023-09-23T23:30:56.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.906504+0000", 2023-09-23T23:30:56.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00093603000000000002 2023-09-23T23:30:56.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.907999+0000", 2023-09-23T23:30:56.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001495213 2023-09-23T23:30:56.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.914130+0000", 2023-09-23T23:30:56.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061308609999999996 2023-09-23T23:30:56.268 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.268 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.268 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.268 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156798 10.3 10:c087de90:::benchmark_data_smithi110_366450_object55590:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:30:56.268 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.251881+0000", 2023-09-23T23:30:56.268 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 544.90682855900002, 2023-09-23T23:30:56.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6721864360000001, 2023-09-23T23:30:56.269 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.269 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.269 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.269 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.269 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.269 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156798 2023-09-23T23:30:56.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.269 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251881+0000", 2023-09-23T23:30:56.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251881+0000", 2023-09-23T23:30:56.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251879+0000", 2023-09-23T23:30:56.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:30:56.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251886+0000", 2023-09-23T23:30:56.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4769999999999997e-06 2023-09-23T23:30:56.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251899+0000", 2023-09-23T23:30:56.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2914999999999999e-05 2023-09-23T23:30:56.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.251908+0000", 2023-09-23T23:30:56.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8259999999999999e-06 2023-09-23T23:30:56.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.079210+0000", 2023-09-23T23:30:56.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8273024840000001 2023-09-23T23:30:56.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.082825+0000", 2023-09-23T23:30:56.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036146450000000001 2023-09-23T23:30:56.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.129346+0000", 2023-09-23T23:30:56.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.046521376000000003 2023-09-23T23:30:56.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.132533+0000", 2023-09-23T23:30:56.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031862040000000002 2023-09-23T23:30:56.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.892035+0000", 2023-09-23T23:30:56.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.75950217900000005 2023-09-23T23:30:56.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.894859+0000", 2023-09-23T23:30:56.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002824287 2023-09-23T23:30:56.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.899648+0000", 2023-09-23T23:30:56.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047886220000000002 2023-09-23T23:30:56.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.915092+0000", 2023-09-23T23:30:56.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015443738 2023-09-23T23:30:56.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.915395+0000", 2023-09-23T23:30:56.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000303252 2023-09-23T23:30:56.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.916288+0000", 2023-09-23T23:30:56.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00089328499999999996 2023-09-23T23:30:56.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.917880+0000", 2023-09-23T23:30:56.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001592336 2023-09-23T23:30:56.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.924067+0000", 2023-09-23T23:30:56.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061867060000000001 2023-09-23T23:30:56.276 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.277 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.277 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.277 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156815 10.3 10:c745291f:::benchmark_data_smithi110_366450_object55607:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:30:56.277 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.282940+0000", 2023-09-23T23:30:56.277 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 544.87576906300001, 2023-09-23T23:30:56.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7155916270000002, 2023-09-23T23:30:56.277 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.277 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.277 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.277 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.278 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.278 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156815 2023-09-23T23:30:56.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.278 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282940+0000", 2023-09-23T23:30:56.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282940+0000", 2023-09-23T23:30:56.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282938+0000", 2023-09-23T23:30:56.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:30:56.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282946+0000", 2023-09-23T23:30:56.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6690000000000006e-06 2023-09-23T23:30:56.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282960+0000", 2023-09-23T23:30:56.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4154e-05 2023-09-23T23:30:56.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.282969+0000", 2023-09-23T23:30:56.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4290000000000001e-06 2023-09-23T23:30:56.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.132962+0000", 2023-09-23T23:30:56.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.849993564 2023-09-23T23:30:56.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.136239+0000", 2023-09-23T23:30:56.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032766829999999999 2023-09-23T23:30:56.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.153725+0000", 2023-09-23T23:30:56.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017485941000000001 2023-09-23T23:30:56.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.157066+0000", 2023-09-23T23:30:56.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003341283 2023-09-23T23:30:56.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.940774+0000", 2023-09-23T23:30:56.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.78370831600000002 2023-09-23T23:30:56.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.943618+0000", 2023-09-23T23:30:56.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002843632 2023-09-23T23:30:56.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.948942+0000", 2023-09-23T23:30:56.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053240989999999997 2023-09-23T23:30:56.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.984632+0000", 2023-09-23T23:30:56.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.035689569999999997 2023-09-23T23:30:56.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.990058+0000", 2023-09-23T23:30:56.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054259690000000001 2023-09-23T23:30:56.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.991453+0000", 2023-09-23T23:30:56.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013957710000000001 2023-09-23T23:30:56.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.992873+0000", 2023-09-23T23:30:56.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014191939999999999 2023-09-23T23:30:56.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.998532+0000", 2023-09-23T23:30:56.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056591080000000004 2023-09-23T23:30:56.285 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.285 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.285 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.286 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156870 10.3 10:cbcf9498:::benchmark_data_smithi110_366450_object55662:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:30:56.286 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.374775+0000", 2023-09-23T23:30:56.286 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 544.78393435099997, 2023-09-23T23:30:56.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6969683209999999, 2023-09-23T23:30:56.286 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.286 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.286 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.286 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.286 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.287 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156870 2023-09-23T23:30:56.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.287 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374775+0000", 2023-09-23T23:30:56.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374775+0000", 2023-09-23T23:30:56.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374773+0000", 2023-09-23T23:30:56.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:30:56.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374781+0000", 2023-09-23T23:30:56.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6690000000000006e-06 2023-09-23T23:30:56.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374798+0000", 2023-09-23T23:30:56.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7326e-05 2023-09-23T23:30:56.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.374807+0000", 2023-09-23T23:30:56.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8920000000000006e-06 2023-09-23T23:30:56.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.137607+0000", 2023-09-23T23:30:56.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.762799918 2023-09-23T23:30:56.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.140841+0000", 2023-09-23T23:30:56.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032347109999999999 2023-09-23T23:30:56.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.170460+0000", 2023-09-23T23:30:56.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029618499999999999 2023-09-23T23:30:56.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.173817+0000", 2023-09-23T23:30:56.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033569440000000002 2023-09-23T23:30:56.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.015338+0000", 2023-09-23T23:30:56.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.841521509 2023-09-23T23:30:56.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.017988+0000", 2023-09-23T23:30:56.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026499359999999999 2023-09-23T23:30:56.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.022695+0000", 2023-09-23T23:30:56.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047064819999999997 2023-09-23T23:30:56.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.042997+0000", 2023-09-23T23:30:56.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020301784999999999 2023-09-23T23:30:56.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.062327+0000", 2023-09-23T23:30:56.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019330791 2023-09-23T23:30:56.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.064213+0000", 2023-09-23T23:30:56.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018860210000000001 2023-09-23T23:30:56.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.065676+0000", 2023-09-23T23:30:56.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001462968 2023-09-23T23:30:56.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.071743+0000", 2023-09-23T23:30:56.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060667660000000003 2023-09-23T23:30:56.294 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.294 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.294 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.295 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156880 10.3 10:ca35dff2:::benchmark_data_smithi110_366450_object55672:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:30:56.295 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.396748+0000", 2023-09-23T23:30:56.295 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 544.76196106199995, 2023-09-23T23:30:56.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7345815240000002, 2023-09-23T23:30:56.295 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.295 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.295 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.295 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.295 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.295 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156880 2023-09-23T23:30:56.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.296 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396748+0000", 2023-09-23T23:30:56.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396748+0000", 2023-09-23T23:30:56.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396747+0000", 2023-09-23T23:30:56.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:30:56.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396753+0000", 2023-09-23T23:30:56.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3829999999999997e-06 2023-09-23T23:30:56.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396765+0000", 2023-09-23T23:30:56.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1868999999999999e-05 2023-09-23T23:30:56.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.396771+0000", 2023-09-23T23:30:56.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2299999999999996e-06 2023-09-23T23:30:56.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.174176+0000", 2023-09-23T23:30:56.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.777405116 2023-09-23T23:30:56.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.177553+0000", 2023-09-23T23:30:56.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033766130000000001 2023-09-23T23:30:56.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.196534+0000", 2023-09-23T23:30:56.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018980770000000001 2023-09-23T23:30:56.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.200001+0000", 2023-09-23T23:30:56.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003467159 2023-09-23T23:30:56.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.084438+0000", 2023-09-23T23:30:56.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.88443721799999997 2023-09-23T23:30:56.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.087726+0000", 2023-09-23T23:30:56.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003287805 2023-09-23T23:30:56.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.093532+0000", 2023-09-23T23:30:56.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058060129999999996 2023-09-23T23:30:56.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.120102+0000", 2023-09-23T23:30:56.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.026570277999999999 2023-09-23T23:30:56.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.121067+0000", 2023-09-23T23:30:56.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00096454899999999998 2023-09-23T23:30:56.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.122429+0000", 2023-09-23T23:30:56.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001362399 2023-09-23T23:30:56.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.124160+0000", 2023-09-23T23:30:56.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001731291 2023-09-23T23:30:56.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.131330+0000", 2023-09-23T23:30:56.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071693939999999999 2023-09-23T23:30:56.303 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.303 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.303 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.303 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:56.305 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156887 10.3 10:c3285ada:::benchmark_data_smithi110_366450_object55679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-09-23T23:30:56.305 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:51.419195+0000", 2023-09-23T23:30:56.305 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 544.73951464300001, 2023-09-23T23:30:56.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.750874418, 2023-09-23T23:30:56.305 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.305 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.305 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.305 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.305 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.306 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156887 2023-09-23T23:30:56.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.306 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419195+0000", 2023-09-23T23:30:56.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419195+0000", 2023-09-23T23:30:56.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419193+0000", 2023-09-23T23:30:56.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:30:56.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419199+0000", 2023-09-23T23:30:56.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1539999999999999e-06 2023-09-23T23:30:56.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419211+0000", 2023-09-23T23:30:56.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1651000000000001e-05 2023-09-23T23:30:56.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.419219+0000", 2023-09-23T23:30:56.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0220000000000004e-06 2023-09-23T23:30:56.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.178884+0000", 2023-09-23T23:30:56.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7596647829999998 2023-09-23T23:30:56.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.182782+0000", 2023-09-23T23:30:56.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038980529999999998 2023-09-23T23:30:56.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.212229+0000", 2023-09-23T23:30:56.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029447465999999999 2023-09-23T23:30:56.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:53.215590+0000", 2023-09-23T23:30:56.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033608779999999999 2023-09-23T23:30:56.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.141066+0000", 2023-09-23T23:30:56.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.92547559800000001 2023-09-23T23:30:56.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.144556+0000", 2023-09-23T23:30:56.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034902330000000001 2023-09-23T23:30:56.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.149839+0000", 2023-09-23T23:30:56.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052831730000000004 2023-09-23T23:30:56.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.160290+0000", 2023-09-23T23:30:56.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010451251999999999 2023-09-23T23:30:56.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.160640+0000", 2023-09-23T23:30:56.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034921699999999997 2023-09-23T23:30:56.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.161546+0000", 2023-09-23T23:30:56.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00090665099999999996 2023-09-23T23:30:56.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.163299+0000", 2023-09-23T23:30:56.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017525590000000001 2023-09-23T23:30:56.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:54.170069+0000", 2023-09-23T23:30:56.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067702309999999998 2023-09-23T23:30:56.313 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.313 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.313 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.313 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:162491 10.3 10:c27b7a62:::benchmark_data_smithi110_366450_object61283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T23:30:56.314 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:22:12.428302+0000", 2023-09-23T23:30:56.314 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 523.73040736200005, 2023-09-23T23:30:56.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.190000068, 2023-09-23T23:30:56.314 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.314 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.314 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.314 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.314 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.314 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 162491 2023-09-23T23:30:56.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.314 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428302+0000", 2023-09-23T23:30:56.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428302+0000", 2023-09-23T23:30:56.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428301+0000", 2023-09-23T23:30:56.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:30:56.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428306+0000", 2023-09-23T23:30:56.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3000000000000001e-06 2023-09-23T23:30:56.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428315+0000", 2023-09-23T23:30:56.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2450000000000004e-06 2023-09-23T23:30:56.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.428321+0000", 2023-09-23T23:30:56.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0780000000000001e-06 2023-09-23T23:30:56.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.484840+0000", 2023-09-23T23:30:56.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.056518688999999997 2023-09-23T23:30:56.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.486346+0000", 2023-09-23T23:30:56.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015063489999999999 2023-09-23T23:30:56.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.928898+0000", 2023-09-23T23:30:56.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.44255191999999999 2023-09-23T23:30:56.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:12.932431+0000", 2023-09-23T23:30:56.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035331630000000002 2023-09-23T23:30:56.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.575704+0000", 2023-09-23T23:30:56.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.64327284799999995 2023-09-23T23:30:56.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.578243+0000", 2023-09-23T23:30:56.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025391559999999999 2023-09-23T23:30:56.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.582742+0000", 2023-09-23T23:30:56.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044984200000000004 2023-09-23T23:30:56.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.610331+0000", 2023-09-23T23:30:56.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.027589460999999999 2023-09-23T23:30:56.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.610601+0000", 2023-09-23T23:30:56.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026943799999999999 2023-09-23T23:30:56.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.611382+0000", 2023-09-23T23:30:56.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00078106699999999998 2023-09-23T23:30:56.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.612748+0000", 2023-09-23T23:30:56.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013659690000000001 2023-09-23T23:30:56.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:13.618302+0000", 2023-09-23T23:30:56.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055542489999999998 2023-09-23T23:30:56.322 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.322 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.322 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.322 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:163210 10.3 10:cceb657e:::benchmark_data_smithi110_366450_object62002:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-09-23T23:30:56.322 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:22:15.221166+0000", 2023-09-23T23:30:56.325 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 520.93754287299998, 2023-09-23T23:30:56.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.360785763, 2023-09-23T23:30:56.325 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.325 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.325 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.325 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.325 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.325 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 163210 2023-09-23T23:30:56.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.326 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221166+0000", 2023-09-23T23:30:56.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221166+0000", 2023-09-23T23:30:56.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.327 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.327 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221165+0000", 2023-09-23T23:30:56.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:30:56.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.327 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.327 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221170+0000", 2023-09-23T23:30:56.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3000000000000003e-06 2023-09-23T23:30:56.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.327 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221180+0000", 2023-09-23T23:30:56.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0513000000000001e-05 2023-09-23T23:30:56.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.221186+0000", 2023-09-23T23:30:56.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3789999999999997e-06 2023-09-23T23:30:56.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.236027+0000", 2023-09-23T23:30:56.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014841425 2023-09-23T23:30:56.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.329 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.237325+0000", 2023-09-23T23:30:56.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012978829999999999 2023-09-23T23:30:56.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.329 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.914366+0000", 2023-09-23T23:30:56.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.677041059 2023-09-23T23:30:56.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:15.917400+0000", 2023-09-23T23:30:56.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003033507 2023-09-23T23:30:56.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.556394+0000", 2023-09-23T23:30:56.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.63899485600000006 2023-09-23T23:30:56.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.558747+0000", 2023-09-23T23:30:56.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023522899999999999 2023-09-23T23:30:56.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.563392+0000", 2023-09-23T23:30:56.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046450199999999997 2023-09-23T23:30:56.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.572010+0000", 2023-09-23T23:30:56.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086180040000000003 2023-09-23T23:30:56.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.572357+0000", 2023-09-23T23:30:56.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034713000000000002 2023-09-23T23:30:56.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.573395+0000", 2023-09-23T23:30:56.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010380890000000001 2023-09-23T23:30:56.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.575115+0000", 2023-09-23T23:30:56.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017195960000000001 2023-09-23T23:30:56.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:16.581952+0000", 2023-09-23T23:30:56.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068375789999999999 2023-09-23T23:30:56.333 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.333 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.333 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.334 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:168464 10.3 10:c5beec16:::benchmark_data_smithi110_366450_object67256:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T23:30:56.334 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:22:31.505622+0000", 2023-09-23T23:30:56.334 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 504.65308719299998, 2023-09-23T23:30:56.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1552177969999999, 2023-09-23T23:30:56.334 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:30:56.334 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:30:56.334 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:30:56.334 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:30:56.334 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:30:56.334 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 168464 2023-09-23T23:30:56.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.334 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:30:56.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.335 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:30:56.335 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505622+0000", 2023-09-23T23:30:56.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.335 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:30:56.335 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505622+0000", 2023-09-23T23:30:56.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:30:56.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.335 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:30:56.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505621+0000", 2023-09-23T23:30:56.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:30:56.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:30:56.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505627+0000", 2023-09-23T23:30:56.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8950000000000003e-06 2023-09-23T23:30:56.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:30:56.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505646+0000", 2023-09-23T23:30:56.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9667e-05 2023-09-23T23:30:56.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:30:56.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.505657+0000", 2023-09-23T23:30:56.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1141000000000001e-05 2023-09-23T23:30:56.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.560164+0000", 2023-09-23T23:30:56.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.054506720000000002 2023-09-23T23:30:56.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:30:56.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:31.561430+0000", 2023-09-23T23:30:56.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001266267 2023-09-23T23:30:56.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.028996+0000", 2023-09-23T23:30:56.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.46756556300000002 2023-09-23T23:30:56.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:30:56.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.031880+0000", 2023-09-23T23:30:56.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028842189999999999 2023-09-23T23:30:56.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:30:56.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.517916+0000", 2023-09-23T23:30:56.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.486035885 2023-09-23T23:30:56.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:30:56.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.519830+0000", 2023-09-23T23:30:56.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019141340000000001 2023-09-23T23:30:56.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:30:56.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.523467+0000", 2023-09-23T23:30:56.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036371060000000002 2023-09-23T23:30:56.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.530246+0000", 2023-09-23T23:30:56.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067788550000000003 2023-09-23T23:30:56.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:30:56.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.532994+0000", 2023-09-23T23:30:56.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027481290000000002 2023-09-23T23:30:56.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:30:56.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.654294+0000", 2023-09-23T23:30:56.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.121299324 2023-09-23T23:30:56.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:30:56.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.655550+0000", 2023-09-23T23:30:56.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012558789999999999 2023-09-23T23:30:56.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:30:56.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:30:56.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:30:56.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:22:32.660840+0000", 2023-09-23T23:30:56.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052902979999999997 2023-09-23T23:30:56.342 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.342 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.342 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.342 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:30:56.342 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:30:56.342 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:30:56.343 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:55 smithi110 ceph-mon[221102]: 12.3 scrub starts 2023-09-23T23:30:56.343 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:55 smithi110 ceph-mon[221102]: 12.0 scrub starts 2023-09-23T23:30:56.343 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:55 smithi110 ceph-mon[212939]: 12.3 scrub starts 2023-09-23T23:30:56.343 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:55 smithi110 ceph-mon[212939]: 12.0 scrub starts 2023-09-23T23:30:56.344 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:55 smithi138 ceph-mon[193606]: 12.3 scrub starts 2023-09-23T23:30:56.344 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:55 smithi138 ceph-mon[193606]: 12.0 scrub starts 2023-09-23T23:30:56.344 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:56.407 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:56.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:56.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:56.514 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:30:56.600 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:56.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:30:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:30:56] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.43.0" 2023-09-23T23:30:57.029 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T23:30:57.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:56 smithi138 ceph-mon[193606]: pgmap v1398: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 18295/201687 objects degraded (9.071%); 10053/201687 objects misplaced (4.984%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T23:30:57.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:56 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/7138098' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:30:57.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:56 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:30:57.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:56 smithi110 ceph-mon[221102]: pgmap v1398: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 18295/201687 objects degraded (9.071%); 10053/201687 objects misplaced (4.984%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T23:30:57.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:56 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/7138098' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:30:57.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:56 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:30:57.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:56 smithi110 ceph-mon[212939]: pgmap v1398: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 18295/201687 objects degraded (9.071%); 10053/201687 objects misplaced (4.984%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T23:30:57.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:56 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/7138098' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:30:57.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:56 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:30:57.535 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:30:58.297 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:57 smithi110 ceph-mon[221102]: failed to encode map e507 with expected crc 2023-09-23T23:30:58.297 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:57 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:30:58.297 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:57 smithi110 ceph-mon[221102]: osdmap e507: 8 total, 8 up, 8 in 2023-09-23T23:30:58.297 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:57 smithi110 ceph-mon[221102]: 12.2 scrub starts 2023-09-23T23:30:58.302 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:57 smithi110 ceph-mon[212939]: failed to encode map e507 with expected crc 2023-09-23T23:30:58.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:57 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:30:58.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:57 smithi110 ceph-mon[212939]: osdmap e507: 8 total, 8 up, 8 in 2023-09-23T23:30:58.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:57 smithi110 ceph-mon[212939]: 12.2 scrub starts 2023-09-23T23:30:58.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:57 smithi138 ceph-mon[193606]: failed to encode map e507 with expected crc 2023-09-23T23:30:58.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:57 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:30:58.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:57 smithi138 ceph-mon[193606]: osdmap e507: 8 total, 8 up, 8 in 2023-09-23T23:30:58.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:57 smithi138 ceph-mon[193606]: 12.2 scrub starts 2023-09-23T23:30:59.190 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:30:59.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:59.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:30:59.294 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:30:59.340 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:58 smithi138 ceph-mon[193606]: pgmap v1400: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 18295/198681 objects degraded (9.208%); 10053/198681 objects misplaced (5.060%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T23:30:59.340 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:58 smithi138 ceph-mon[193606]: 12.13 scrub starts 2023-09-23T23:30:59.340 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:30:59.340 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:30:58 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 18295/198681 objects degraded (9.208%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T23:30:59.344 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:59.405 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:30:59.405 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:58 smithi110 ceph-mon[221102]: pgmap v1400: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 18295/198681 objects degraded (9.208%); 10053/198681 objects misplaced (5.060%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T23:30:59.406 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:58 smithi110 ceph-mon[221102]: 12.13 scrub starts 2023-09-23T23:30:59.406 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:30:59.406 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:30:58 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 18295/198681 objects degraded (9.208%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T23:30:59.406 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:58 smithi110 ceph-mon[212939]: pgmap v1400: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 18295/198681 objects degraded (9.208%); 10053/198681 objects misplaced (5.060%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T23:30:59.407 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:58 smithi110 ceph-mon[212939]: 12.13 scrub starts 2023-09-23T23:30:59.407 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:30:59.407 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:30:58 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 18295/198681 objects degraded (9.208%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T23:30:59.408 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:30:59.409 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T23:30:59.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:59.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:30:59.516 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:30:59.576 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:59.667 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:30:59.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:59.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:30:59.774 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:30:59.810 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:30:59.840 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:30:59.882 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:30:59.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:30:59.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:30:59.931 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:30:59.996 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:31:00.077 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:00.442 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:31:00.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:00.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:31:00.567 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:00.619 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:00.688 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:00.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:00.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:00.742 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:00.792 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:31:00.883 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:01.029 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:31:01.265 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:31:01.268 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:01 smithi110 ceph-mon[212939]: pgmap v1401: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 18295/194079 objects degraded (9.427%); 9827/194079 objects misplaced (5.063%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T23:31:01.268 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:01 smithi110 ceph-mon[212939]: 12.12 scrub starts 2023-09-23T23:31:01.268 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:01 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1136561447' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:01.268 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:01 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:01.268 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:01 smithi110 ceph-mon[221102]: pgmap v1401: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 18295/194079 objects degraded (9.427%); 9827/194079 objects misplaced (5.063%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T23:31:01.268 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:01 smithi110 ceph-mon[221102]: 12.12 scrub starts 2023-09-23T23:31:01.269 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:01 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1136561447' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:01.269 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:01 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:01.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:01.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:31:01.370 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:31:01.398 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:01 smithi138 ceph-mon[193606]: pgmap v1401: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 18295/194079 objects degraded (9.427%); 9827/194079 objects misplaced (5.063%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T23:31:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:01 smithi138 ceph-mon[193606]: 12.12 scrub starts 2023-09-23T23:31:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:01 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1136561447' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:01 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:02.294 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:31:02.295 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:02 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:31:02.295 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:02 smithi110 ceph-mon[212939]: failed to encode map e508 with expected crc 2023-09-23T23:31:02.295 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:02 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:31:02.295 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:02 smithi110 ceph-mon[212939]: osdmap e508: 8 total, 8 up, 8 in 2023-09-23T23:31:02.295 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:02 smithi110 ceph-mon[212939]: failed to encode map e508 with expected crc 2023-09-23T23:31:02.295 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:02 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:31:02.295 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:02 smithi110 ceph-mon[221102]: failed to encode map e508 with expected crc 2023-09-23T23:31:02.296 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:02 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:31:02.296 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:02 smithi110 ceph-mon[221102]: osdmap e508: 8 total, 8 up, 8 in 2023-09-23T23:31:02.296 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:02 smithi110 ceph-mon[221102]: failed to encode map e508 with expected crc 2023-09-23T23:31:02.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:02.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:31:02.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:02 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:31:02.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:02 smithi138 ceph-mon[193606]: failed to encode map e508 with expected crc 2023-09-23T23:31:02.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:02 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:31:02.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:02 smithi138 ceph-mon[193606]: osdmap e508: 8 total, 8 up, 8 in 2023-09-23T23:31:02.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:02 smithi138 ceph-mon[193606]: failed to encode map e508 with expected crc 2023-09-23T23:31:02.402 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:02.456 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:02.550 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:02.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:02.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:02.600 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:31:02.601 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:31:02.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:31:02.601 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:129097 12.d 12:b3760388:::benchmark_data_smithi110_491193_object34222:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e508)", 2023-09-23T23:31:02.601 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:31:02.533362+0000", 2023-09-23T23:31:02.601 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.066046066, 2023-09-23T23:31:02.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.066088333999999999, 2023-09-23T23:31:02.601 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:31:02.601 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:31:02.601 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:31:02.602 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:31:02.602 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:31:02.602 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129097 2023-09-23T23:31:02.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:31:02.602 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:31:02.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:31:02.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:31:02.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:02.533362+0000", 2023-09-23T23:31:02.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:31:02.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:31:02.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:31:02.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:31:02.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:02.533362+0000", 2023-09-23T23:31:02.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:31:02.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:31:02.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:31:02.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:31:02.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:02.533361+0000", 2023-09-23T23:31:02.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:31:02.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:31:02.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:31:02.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:31:02.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:02.533371+0000", 2023-09-23T23:31:02.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0468999999999999e-05 2023-09-23T23:31:02.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:31:02.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:31:02.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:31:02.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:02.533385+0000", 2023-09-23T23:31:02.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3806e-05 2023-09-23T23:31:02.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:31:02.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:31:02.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:31:02.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:02.533391+0000", 2023-09-23T23:31:02.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1449999999999996e-06 2023-09-23T23:31:02.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:31:02.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:31:02.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:31:02.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:02.533435+0000", 2023-09-23T23:31:02.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4196000000000002e-05 2023-09-23T23:31:02.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:31:02.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:31:02.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:31:02.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:02.533536+0000", 2023-09-23T23:31:02.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000100535 2023-09-23T23:31:02.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:31:02.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:31:02.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:31:02.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:02.533620+0000", 2023-09-23T23:31:02.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4631999999999999e-05 2023-09-23T23:31:02.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:31:02.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:31:02.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:31:02.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:02.535963+0000", 2023-09-23T23:31:02.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023430690000000001 2023-09-23T23:31:02.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:31:02.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:31:02.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:02.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:02.537126+0000", 2023-09-23T23:31:02.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001162106 2023-09-23T23:31:02.607 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:31:02.607 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:31:02.607 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:31:02.607 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:31:02.607 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-09-23T23:31:02.607 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 1 2023-09-23T23:31:02.607 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:31:02.654 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:31:02.702 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:03.223 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:31:03.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:03.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:31:03.342 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:03.393 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:03 smithi138 ceph-mon[193606]: pgmap v1403: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 18295/187713 objects degraded (9.746%); 9403/187713 objects misplaced (5.009%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T23:31:03.396 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:03.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:03 smithi110 ceph-mon[221102]: pgmap v1403: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 18295/187713 objects degraded (9.746%); 9403/187713 objects misplaced (5.009%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T23:31:03.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:03 smithi110 ceph-mon[212939]: pgmap v1403: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 18295/187713 objects degraded (9.746%); 9403/187713 objects misplaced (5.009%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T23:31:03.464 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:03.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:03.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:03.532 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T23:31:03.570 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:31:03.619 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:03.697 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:31:03.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:03.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:31:03.809 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:31:03.912 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:04.032 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:31:04.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:04 smithi110 ceph-mon[212939]: 12.3 scrub starts 2023-09-23T23:31:04.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:04 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 18295/187713 objects degraded (9.746%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:04.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:04 smithi110 ceph-mon[221102]: 12.3 scrub starts 2023-09-23T23:31:04.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:04 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 18295/187713 objects degraded (9.746%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:04.297 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:04 smithi138 ceph-mon[193606]: 12.3 scrub starts 2023-09-23T23:31:04.297 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:04 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 18295/187713 objects degraded (9.746%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:04.466 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:31:04.466 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:31:04.466 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:31:04.466 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:31:04.466 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:31:04.469 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:31:04.491 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:31:04.590 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:31:04.590 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:31:04.590 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:31:04.590 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:31:04.590 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:31:04.591 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:31:04.591 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:31:04.591 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:31:04.591 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:31:04.622 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:31:04.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:04.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:31:04.731 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:04.785 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:04.861 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:04.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:04.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:04.965 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:31:05.036 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:05.052 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:05 smithi138 ceph-mon[193606]: pgmap v1404: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 18295/186417 objects degraded (9.814%); 9403/186417 objects misplaced (5.044%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T23:31:05.053 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:05 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3519118514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:31:05.185 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:31:05.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:05.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:31:05.242 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:31:05.290 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:05.311 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:05 smithi110 ceph-mon[212939]: pgmap v1404: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 18295/186417 objects degraded (9.814%); 9403/186417 objects misplaced (5.044%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T23:31:05.311 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:05 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3519118514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:31:05.314 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:05 smithi110 ceph-mon[221102]: pgmap v1404: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 18295/186417 objects degraded (9.814%); 9403/186417 objects misplaced (5.044%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T23:31:05.314 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:05 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3519118514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:31:05.360 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:05.458 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:05.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:05.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:05.565 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:31:05.685 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:05.857 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:31:05.858 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:31:05.987 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:31:05.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:05.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:31:06.096 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:06.139 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:06 smithi138 ceph-mon[193606]: 12.13 scrub starts 2023-09-23T23:31:06.140 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:06 smithi138 ceph-mon[193606]: 12.0 scrub starts 2023-09-23T23:31:06.143 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:06.227 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:06.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:06.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:06.336 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:31:06.406 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:06 smithi110 ceph-mon[212939]: 12.13 scrub starts 2023-09-23T23:31:06.407 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:06 smithi110 ceph-mon[212939]: 12.0 scrub starts 2023-09-23T23:31:06.408 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:06 smithi110 ceph-mon[221102]: 12.13 scrub starts 2023-09-23T23:31:06.408 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:06 smithi110 ceph-mon[221102]: 12.0 scrub starts 2023-09-23T23:31:06.412 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:06.446 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:06.504 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:31:06.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:06.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:31:06.609 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:06.655 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:06.888 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:06.893 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T23:31:06.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:06.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:06.997 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:07.041 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:07.080 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:31:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:31:06] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.43.0" 2023-09-23T23:31:07.087 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T23:31:07.130 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:07.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:07.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:07.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:07 smithi138 ceph-mon[193606]: pgmap v1405: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 18169/179988 objects degraded (10.095%); 8833/179988 objects misplaced (4.908%); 5.4 MiB/s, 101 objects/s recovering 2023-09-23T23:31:07.195 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:07 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3372249455' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:31:07.242 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:31:07.316 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:07.451 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:31:07.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:07 smithi110 ceph-mon[221102]: pgmap v1405: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 18169/179988 objects degraded (10.095%); 8833/179988 objects misplaced (4.908%); 5.4 MiB/s, 101 objects/s recovering 2023-09-23T23:31:07.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:07 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3372249455' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:31:07.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:07 smithi110 ceph-mon[212939]: pgmap v1405: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 18169/179988 objects degraded (10.095%); 8833/179988 objects misplaced (4.908%); 5.4 MiB/s, 101 objects/s recovering 2023-09-23T23:31:07.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:07 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3372249455' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:31:07.991 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:31:08.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:08 smithi110 ceph-mon[212939]: 12.3 scrub starts 2023-09-23T23:31:08.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:08 smithi110 ceph-mon[212939]: failed to encode map e509 with expected crc 2023-09-23T23:31:08.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:08 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3372249455' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:31:08.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:08 smithi110 ceph-mon[212939]: osdmap e509: 8 total, 8 up, 8 in 2023-09-23T23:31:08.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:08 smithi110 ceph-mon[212939]: failed to encode map e509 with expected crc 2023-09-23T23:31:08.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:08 smithi110 ceph-mon[212939]: failed to encode map e509 with expected crc 2023-09-23T23:31:08.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:08 smithi110 ceph-mon[212939]: 12.2 scrub starts 2023-09-23T23:31:08.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:08 smithi110 ceph-mon[221102]: 12.3 scrub starts 2023-09-23T23:31:08.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:08 smithi110 ceph-mon[221102]: failed to encode map e509 with expected crc 2023-09-23T23:31:08.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:08 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3372249455' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:31:08.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:08 smithi110 ceph-mon[221102]: osdmap e509: 8 total, 8 up, 8 in 2023-09-23T23:31:08.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:08 smithi110 ceph-mon[221102]: failed to encode map e509 with expected crc 2023-09-23T23:31:08.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:08 smithi110 ceph-mon[221102]: failed to encode map e509 with expected crc 2023-09-23T23:31:08.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:08 smithi110 ceph-mon[221102]: 12.2 scrub starts 2023-09-23T23:31:08.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:08 smithi138 ceph-mon[193606]: 12.3 scrub starts 2023-09-23T23:31:08.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:08 smithi138 ceph-mon[193606]: failed to encode map e509 with expected crc 2023-09-23T23:31:08.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:08 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3372249455' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:31:08.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:08 smithi138 ceph-mon[193606]: osdmap e509: 8 total, 8 up, 8 in 2023-09-23T23:31:08.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:08 smithi138 ceph-mon[193606]: failed to encode map e509 with expected crc 2023-09-23T23:31:08.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:08 smithi138 ceph-mon[193606]: failed to encode map e509 with expected crc 2023-09-23T23:31:08.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:08 smithi138 ceph-mon[193606]: 12.2 scrub starts 2023-09-23T23:31:09.129 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:31:09.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:09.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:31:09.241 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:31:09.266 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T23:31:09.293 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:31:09.293 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:31:09.313 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:09.315 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:09 smithi138 ceph-mon[193606]: pgmap v1407: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 18169/176829 objects degraded (10.275%); 8833/176829 objects misplaced (4.995%); 6.0 MiB/s, 111 objects/s recovering 2023-09-23T23:31:09.315 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:09 smithi138 ceph-mon[193606]: 12.12 scrub starts 2023-09-23T23:31:09.315 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:09 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 18169/176829 objects degraded (10.275%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:09.315 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:09 smithi138 ceph-mon[193606]: osdmap e510: 8 total, 8 up, 8 in 2023-09-23T23:31:09.315 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:09 smithi138 ceph-mon[193606]: failed to encode map e510 with expected crc 2023-09-23T23:31:09.399 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:09.400 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:09 smithi110 ceph-mon[221102]: pgmap v1407: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 18169/176829 objects degraded (10.275%); 8833/176829 objects misplaced (4.995%); 6.0 MiB/s, 111 objects/s recovering 2023-09-23T23:31:09.400 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:09 smithi110 ceph-mon[221102]: 12.12 scrub starts 2023-09-23T23:31:09.400 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:09 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 18169/176829 objects degraded (10.275%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:09.400 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:09 smithi110 ceph-mon[221102]: osdmap e510: 8 total, 8 up, 8 in 2023-09-23T23:31:09.400 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:09 smithi110 ceph-mon[221102]: failed to encode map e510 with expected crc 2023-09-23T23:31:09.400 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:09 smithi110 ceph-mon[212939]: pgmap v1407: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 18169/176829 objects degraded (10.275%); 8833/176829 objects misplaced (4.995%); 6.0 MiB/s, 111 objects/s recovering 2023-09-23T23:31:09.400 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:09 smithi110 ceph-mon[212939]: 12.12 scrub starts 2023-09-23T23:31:09.400 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:09 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 18169/176829 objects degraded (10.275%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:09.400 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:09 smithi110 ceph-mon[212939]: osdmap e510: 8 total, 8 up, 8 in 2023-09-23T23:31:09.401 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:09 smithi110 ceph-mon[212939]: failed to encode map e510 with expected crc 2023-09-23T23:31:09.409 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:31:09.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:09.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:31:09.521 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:09.555 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:09.601 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:09.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:09.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:09.705 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:09.731 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:09.773 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:09.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:09.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:09.877 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:31:09.904 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:09.945 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:31:09.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:09.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:31:10.048 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:31:10.120 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:10 smithi110 ceph-mon[212939]: failed to encode map e510 with expected crc 2023-09-23T23:31:10.120 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:10 smithi110 ceph-mon[212939]: failed to encode map e510 with expected crc 2023-09-23T23:31:10.120 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:10 smithi110 ceph-mon[212939]: 12.13 scrub starts 2023-09-23T23:31:10.120 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:10 smithi110 ceph-mon[212939]: osdmap e511: 8 total, 8 up, 8 in 2023-09-23T23:31:10.120 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:10 smithi110 ceph-mon[212939]: failed to encode map e511 with expected crc 2023-09-23T23:31:10.120 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:10 smithi110 ceph-mon[221102]: failed to encode map e510 with expected crc 2023-09-23T23:31:10.120 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:10 smithi110 ceph-mon[221102]: failed to encode map e510 with expected crc 2023-09-23T23:31:10.120 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:10 smithi110 ceph-mon[221102]: 12.13 scrub starts 2023-09-23T23:31:10.121 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:10 smithi110 ceph-mon[221102]: osdmap e511: 8 total, 8 up, 8 in 2023-09-23T23:31:10.121 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:10 smithi110 ceph-mon[221102]: failed to encode map e511 with expected crc 2023-09-23T23:31:10.145 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:10.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:10 smithi138 ceph-mon[193606]: failed to encode map e510 with expected crc 2023-09-23T23:31:10.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:10 smithi138 ceph-mon[193606]: failed to encode map e510 with expected crc 2023-09-23T23:31:10.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:10 smithi138 ceph-mon[193606]: 12.13 scrub starts 2023-09-23T23:31:10.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:10 smithi138 ceph-mon[193606]: osdmap e511: 8 total, 8 up, 8 in 2023-09-23T23:31:10.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:10 smithi138 ceph-mon[193606]: failed to encode map e511 with expected crc 2023-09-23T23:31:10.309 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:31:10.377 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:31:10.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:10.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:31:10.486 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:31:10.526 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:10.598 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:31:10.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:10.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:31:10.709 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:10.735 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:10.776 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:10.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:10.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:10.884 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:10.909 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:10.952 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:10.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:10.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:11.058 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:31:11.100 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:11.160 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:31:11.176 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:31:11.185 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:11 smithi138 ceph-mon[193606]: failed to encode map e511 with expected crc 2023-09-23T23:31:11.185 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:11 smithi138 ceph-mon[193606]: failed to encode map e511 with expected crc 2023-09-23T23:31:11.185 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:11 smithi138 ceph-mon[193606]: failed to encode map e511 with expected crc 2023-09-23T23:31:11.185 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:11 smithi138 ceph-mon[193606]: 12.10 deep-scrub starts 2023-09-23T23:31:11.185 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:11 smithi138 ceph-mon[193606]: pgmap v1410: 129 pgs: 1 creating+peering, 1 peering, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 17694/171501 objects degraded (10.317%); 8600/171501 objects misplaced (5.015%); 3.6 MiB/s, 157 objects/s recovering 2023-09-23T23:31:11.185 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:11 smithi138 ceph-mon[193606]: failed to encode map e511 with expected crc 2023-09-23T23:31:11.185 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:11 smithi138 ceph-mon[193606]: 12.3 scrub starts 2023-09-23T23:31:11.185 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:11 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2945684217' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:31:11.185 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:11 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:31:11.186 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:11 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:31:11.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:11.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:31:11.287 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:11.338 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:11.400 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:11.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:11.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:11.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[212939]: failed to encode map e511 with expected crc 2023-09-23T23:31:11.451 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[212939]: failed to encode map e511 with expected crc 2023-09-23T23:31:11.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[212939]: failed to encode map e511 with expected crc 2023-09-23T23:31:11.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[212939]: 12.10 deep-scrub starts 2023-09-23T23:31:11.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[212939]: pgmap v1410: 129 pgs: 1 creating+peering, 1 peering, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 17694/171501 objects degraded (10.317%); 8600/171501 objects misplaced (5.015%); 3.6 MiB/s, 157 objects/s recovering 2023-09-23T23:31:11.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[212939]: failed to encode map e511 with expected crc 2023-09-23T23:31:11.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[212939]: 12.3 scrub starts 2023-09-23T23:31:11.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2945684217' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:31:11.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:31:11.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:31:11.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[221102]: failed to encode map e511 with expected crc 2023-09-23T23:31:11.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[221102]: failed to encode map e511 with expected crc 2023-09-23T23:31:11.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[221102]: failed to encode map e511 with expected crc 2023-09-23T23:31:11.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[221102]: 12.10 deep-scrub starts 2023-09-23T23:31:11.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[221102]: pgmap v1410: 129 pgs: 1 creating+peering, 1 peering, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 17694/171501 objects degraded (10.317%); 8600/171501 objects misplaced (5.015%); 3.6 MiB/s, 157 objects/s recovering 2023-09-23T23:31:11.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[221102]: failed to encode map e511 with expected crc 2023-09-23T23:31:11.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[221102]: 12.3 scrub starts 2023-09-23T23:31:11.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2945684217' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:31:11.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:31:11.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:11 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:31:11.510 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:11.554 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:11.622 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:11.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:11.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:11.734 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:11.760 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:11.801 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:11.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:11.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:11.908 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:31:11.992 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:12.288 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:31:12.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:12.298 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:12 smithi110 ceph-mon[212939]: 12.13 scrub starts 2023-09-23T23:31:12.298 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:12 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:31:12.298 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:12 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:31:12.298 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:12 smithi110 ceph-mon[212939]: osdmap e512: 8 total, 8 up, 8 in 2023-09-23T23:31:12.298 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:12 smithi110 ceph-mon[212939]: failed to encode map e512 with expected crc 2023-09-23T23:31:12.298 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:12 smithi110 ceph-mon[212939]: failed to encode map e512 with expected crc 2023-09-23T23:31:12.298 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:12 smithi110 ceph-mon[212939]: failed to encode map e512 with expected crc 2023-09-23T23:31:12.299 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:12 smithi110 ceph-mon[221102]: 12.13 scrub starts 2023-09-23T23:31:12.299 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:12 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:31:12.299 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:12 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:31:12.299 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:12 smithi110 ceph-mon[221102]: osdmap e512: 8 total, 8 up, 8 in 2023-09-23T23:31:12.299 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:12 smithi110 ceph-mon[221102]: failed to encode map e512 with expected crc 2023-09-23T23:31:12.299 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:12 smithi110 ceph-mon[221102]: failed to encode map e512 with expected crc 2023-09-23T23:31:12.299 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:12 smithi110 ceph-mon[221102]: failed to encode map e512 with expected crc 2023-09-23T23:31:12.299 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:12.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:31:12.401 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:31:12.434 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:12.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:12 smithi138 ceph-mon[193606]: 12.13 scrub starts 2023-09-23T23:31:12.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:12 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:31:12.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:12 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:31:12.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:12 smithi138 ceph-mon[193606]: osdmap e512: 8 total, 8 up, 8 in 2023-09-23T23:31:12.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:12 smithi138 ceph-mon[193606]: failed to encode map e512 with expected crc 2023-09-23T23:31:12.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:12 smithi138 ceph-mon[193606]: failed to encode map e512 with expected crc 2023-09-23T23:31:12.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:12 smithi138 ceph-mon[193606]: failed to encode map e512 with expected crc 2023-09-23T23:31:12.497 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:12.686 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:31:12.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:12.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:31:12.793 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:12.829 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:12.894 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:12.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:12.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:13.001 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:13.035 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:13.101 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:13.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:13.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:13.186 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:31:13.212 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:31:13.239 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:31:13.266 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:13 smithi110 ceph-mon[212939]: pgmap v1412: 129 pgs: 1 active+recovery_wait+remapped, 1 creating+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 16438/166548 objects degraded (9.870%); 9696/166548 objects misplaced (5.822%); 2.2 MiB/s, 134 objects/s recovering 2023-09-23T23:31:13.267 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:31:13.267 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:31:13.267 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:13 smithi110 ceph-mon[221102]: pgmap v1412: 129 pgs: 1 active+recovery_wait+remapped, 1 creating+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 16438/166548 objects degraded (9.870%); 9696/166548 objects misplaced (5.822%); 2.2 MiB/s, 134 objects/s recovering 2023-09-23T23:31:13.267 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:31:13.267 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:31:13.315 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:31:13.320 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:13.372 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:31:13.402 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:31:13.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:13 smithi138 ceph-mon[193606]: pgmap v1412: 129 pgs: 1 active+recovery_wait+remapped, 1 creating+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 16438/166548 objects degraded (9.870%); 9696/166548 objects misplaced (5.822%); 2.2 MiB/s, 134 objects/s recovering 2023-09-23T23:31:13.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:31:13.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:31:13.472 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:31:13.505 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:31:13.546 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:31:13.552 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:31:13.552 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:31:13.552 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:31:13.552 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:31:13.552 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:31:13.552 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:31:13.552 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:31:13.552 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:31:13.934 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:31:13.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:13.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:31:14.048 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:14.104 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:14.160 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:14.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:14.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:14.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:14 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3125436468' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:31:14.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:14 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 16438/166548 objects degraded (9.870%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:14.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:14 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3125436468' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:31:14.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:14 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 16438/166548 objects degraded (9.870%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:14.266 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:31:14.294 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:14.339 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:31:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:14 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3125436468' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:31:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:14 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 16438/166548 objects degraded (9.870%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:15.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:15 smithi138 ceph-mon[193606]: pgmap v1413: 129 pgs: 1 active+recovery_wait+remapped, 1 creating+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 16438/165432 objects degraded (9.936%); 9696/165432 objects misplaced (5.861%); 1.7 MiB/s, 105 objects/s recovering 2023-09-23T23:31:15.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:15 smithi110 ceph-mon[212939]: pgmap v1413: 129 pgs: 1 active+recovery_wait+remapped, 1 creating+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 16438/165432 objects degraded (9.936%); 9696/165432 objects misplaced (5.861%); 1.7 MiB/s, 105 objects/s recovering 2023-09-23T23:31:15.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:15 smithi110 ceph-mon[221102]: pgmap v1413: 129 pgs: 1 active+recovery_wait+remapped, 1 creating+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 16438/165432 objects degraded (9.936%); 9696/165432 objects misplaced (5.861%); 1.7 MiB/s, 105 objects/s recovering 2023-09-23T23:31:15.821 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:31:15.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:15.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:31:15.932 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:15.985 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:16.049 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:16.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:16.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:16.153 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:31:16.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:16 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:31:16.212 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:16 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:31:16.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:16 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:31:16.755 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:31:16.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:16.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:31:16.863 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:31:16.954 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:31:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:31:16] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.43.0" 2023-09-23T23:31:16.988 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:17.158 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:31:17.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:17.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:31:17.264 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:17.300 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:17.368 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:17.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:17.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:17.393 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:17 smithi110 ceph-mon[212939]: pgmap v1414: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 16438/159168 objects degraded (10.327%); 9493/159168 objects misplaced (5.964%); 1.4 MiB/s, 88 objects/s recovering 2023-09-23T23:31:17.393 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:17 smithi110 ceph-mon[221102]: pgmap v1414: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 16438/159168 objects degraded (10.327%); 9493/159168 objects misplaced (5.964%); 1.4 MiB/s, 88 objects/s recovering 2023-09-23T23:31:17.405 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:17.477 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:17.511 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:17 smithi138 ceph-mon[193606]: pgmap v1414: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 16438/159168 objects degraded (10.327%); 9493/159168 objects misplaced (5.964%); 1.4 MiB/s, 88 objects/s recovering 2023-09-23T23:31:17.514 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:17.571 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:17.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:17.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:17.677 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:17.712 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:17.770 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:17.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:17.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:17.787 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:31:17.787 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:31:17.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.787 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:139972 12.c e512/476 12:39129a64:::benchmark_data_smithi110_491193_object45097:head v 512'8770, mlcod=512'8768)", 2023-09-23T23:31:17.787 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:31:17.771696+0000", 2023-09-23T23:31:17.787 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0030432520000000002, 2023-09-23T23:31:17.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030736840000000001, 2023-09-23T23:31:17.787 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:17.788 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:31:17.788 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:17.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:17.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:17.771696+0000", 2023-09-23T23:31:17.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:17.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:17.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:17.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:17.771696+0000", 2023-09-23T23:31:17.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:17.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:17.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:17.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:17.771695+0000", 2023-09-23T23:31:17.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:31:17.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:17.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:17.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:17.771702+0000", 2023-09-23T23:31:17.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.413e-06 2023-09-23T23:31:17.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:17.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:17.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:17.771717+0000", 2023-09-23T23:31:17.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4211e-05 2023-09-23T23:31:17.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:17.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:17.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:17.771725+0000", 2023-09-23T23:31:17.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0669999999999992e-06 2023-09-23T23:31:17.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:17.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:17.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:17.771786+0000", 2023-09-23T23:31:17.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1711000000000006e-05 2023-09-23T23:31:17.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:17.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:17.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:17.771805+0000", 2023-09-23T23:31:17.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8368000000000001e-05 2023-09-23T23:31:17.791 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:17.791 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:17.791 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:17.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:17.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.791 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:139976 12.8 e512/490 12:1799dcef:::benchmark_data_smithi110_491193_object45101:head v 512'6587, mlcod=512'6585)", 2023-09-23T23:31:17.791 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:31:17.772633+0000", 2023-09-23T23:31:17.791 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0021061880000000002, 2023-09-23T23:31:17.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022038209999999999, 2023-09-23T23:31:17.791 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:17.791 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:31:17.792 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:17.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:17.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:17.772633+0000", 2023-09-23T23:31:17.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:17.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:17.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:17.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:17.772633+0000", 2023-09-23T23:31:17.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:17.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:17.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:17.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:17.772632+0000", 2023-09-23T23:31:17.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:31:17.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:17.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:17.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:17.772639+0000", 2023-09-23T23:31:17.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0530000000000001e-06 2023-09-23T23:31:17.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:17.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:17.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:17.772653+0000", 2023-09-23T23:31:17.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3664e-05 2023-09-23T23:31:17.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:17.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:17.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:17.772660+0000", 2023-09-23T23:31:17.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4030000000000001e-06 2023-09-23T23:31:17.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:17.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:17.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:17.772719+0000", 2023-09-23T23:31:17.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9079000000000001e-05 2023-09-23T23:31:17.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:17.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:17.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:17.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:17.772740+0000", 2023-09-23T23:31:17.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0837999999999999e-05 2023-09-23T23:31:17.794 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:17.795 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:17.795 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:17.795 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:17.795 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-09-23T23:31:17.795 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 2 2023-09-23T23:31:17.795 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:31:17.878 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:17.910 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:17.978 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:17.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:17.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:18.085 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:31:18.164 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:18.271 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T23:31:18.366 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:31:18.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:18.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:31:18.376 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2166995965' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:31:18.376 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:18 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:31:18.377 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2166995965' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:31:18.377 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:18 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:31:18.477 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:18.508 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2166995965' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:31:18.508 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:18 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:31:18.510 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:18.563 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:18.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:18.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:18.646 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:31:18.672 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:18.722 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:18.781 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:18.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:18.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:18.888 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:31:18.978 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:19.246 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:19 smithi110 ceph-mon[221102]: pgmap v1415: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 16438/155904 objects degraded (10.544%); 9125/155904 objects misplaced (5.853%); 4.1 MiB/s, 64 objects/s recovering 2023-09-23T23:31:19.247 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:19 smithi110 ceph-mon[221102]: failed to encode map e513 with expected crc 2023-09-23T23:31:19.247 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:19 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:31:19.247 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:19 smithi110 ceph-mon[221102]: osdmap e513: 8 total, 8 up, 8 in 2023-09-23T23:31:19.247 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:19 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 16438/155904 objects degraded (10.544%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:19.247 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:19 smithi110 ceph-mon[212939]: pgmap v1415: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 16438/155904 objects degraded (10.544%); 9125/155904 objects misplaced (5.853%); 4.1 MiB/s, 64 objects/s recovering 2023-09-23T23:31:19.247 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:19 smithi110 ceph-mon[212939]: failed to encode map e513 with expected crc 2023-09-23T23:31:19.247 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:19 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:31:19.247 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:19 smithi110 ceph-mon[212939]: osdmap e513: 8 total, 8 up, 8 in 2023-09-23T23:31:19.247 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:19 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 16438/155904 objects degraded (10.544%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:19 smithi138 ceph-mon[193606]: pgmap v1415: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 16438/155904 objects degraded (10.544%); 9125/155904 objects misplaced (5.853%); 4.1 MiB/s, 64 objects/s recovering 2023-09-23T23:31:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:19 smithi138 ceph-mon[193606]: failed to encode map e513 with expected crc 2023-09-23T23:31:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:19 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:31:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:19 smithi138 ceph-mon[193606]: osdmap e513: 8 total, 8 up, 8 in 2023-09-23T23:31:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:19 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 16438/155904 objects degraded (10.544%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:19.761 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:31:19.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:19.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:31:19.874 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:31:19.901 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:20.765 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:31:20.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:20.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:31:20.870 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:31:20.875 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:31:20.943 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:21.051 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T23:31:21.170 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:31:21.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:21.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:31:21.288 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:21.320 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:21.328 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:21.373 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:21.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:21.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:21.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:21 smithi110 ceph-mon[221102]: pgmap v1417: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 16438/148875 objects degraded (11.041%); 8657/148875 objects misplaced (5.815%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T23:31:21.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:21 smithi110 ceph-mon[212939]: pgmap v1417: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 16438/148875 objects degraded (11.041%); 8657/148875 objects misplaced (5.815%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T23:31:21.482 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:31:21.573 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:21 smithi138 ceph-mon[193606]: pgmap v1417: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 16438/148875 objects degraded (11.041%); 8657/148875 objects misplaced (5.815%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T23:31:21.799 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:31:21.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:21.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:31:21.907 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:31:21.973 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:22.161 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:31:22.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:22.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:31:22.275 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:31:22.371 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:22.511 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:31:22.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:22.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:31:22.619 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:31:22.657 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:22.720 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:31:22.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:22.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:31:22.828 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:31:22.855 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:22.899 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:31:22.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:22.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:31:23.002 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:31:23.062 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:23.285 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:31:23.369 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:31:23.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:23.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:31:23.481 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:31:23.539 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:23 smithi110 ceph-mon[212939]: pgmap v1418: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 16438/145875 objects degraded (11.269%); 8657/145875 objects misplaced (5.935%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T23:31:23.540 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:23 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/887769564' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:23.540 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:23 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:23.540 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:23 smithi110 ceph-mon[221102]: pgmap v1418: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 16438/145875 objects degraded (11.269%); 8657/145875 objects misplaced (5.935%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T23:31:23.540 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:23 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/887769564' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:23.540 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:23 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:23.563 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:23.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:23 smithi138 ceph-mon[193606]: pgmap v1418: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 16438/145875 objects degraded (11.269%); 8657/145875 objects misplaced (5.935%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T23:31:23.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:23 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/887769564' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:23.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:23 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:24.271 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:24.275 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:24.451 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:31:24.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:24.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:31:24.548 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:24 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:31:24.549 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:24 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:31:24.549 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:24 smithi110 ceph-mon[212939]: osdmap e514: 8 total, 8 up, 8 in 2023-09-23T23:31:24.549 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:24 smithi110 ceph-mon[212939]: failed to encode map e514 with expected crc 2023-09-23T23:31:24.549 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:24 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 16438/145875 objects degraded (11.269%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:24.549 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:24 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:31:24.549 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:24 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:31:24.549 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:24 smithi110 ceph-mon[221102]: osdmap e514: 8 total, 8 up, 8 in 2023-09-23T23:31:24.549 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:24 smithi110 ceph-mon[221102]: failed to encode map e514 with expected crc 2023-09-23T23:31:24.549 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:24 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 16438/145875 objects degraded (11.269%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:24.558 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:24.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:24 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:31:24.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:24 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:31:24.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:24 smithi138 ceph-mon[193606]: osdmap e514: 8 total, 8 up, 8 in 2023-09-23T23:31:24.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:24 smithi138 ceph-mon[193606]: failed to encode map e514 with expected crc 2023-09-23T23:31:24.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:24 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 16438/145875 objects degraded (11.269%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:24.601 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:24.675 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:24.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:24.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:24.782 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:24.820 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:24.897 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:24.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:24.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:25.005 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:25.036 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:25.103 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:25.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:25.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:25.212 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:31:25.314 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:25.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:25 smithi110 ceph-mon[212939]: pgmap v1420: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 16438/144597 objects degraded (11.368%); 8657/144597 objects misplaced (5.987%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:31:25.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:25 smithi110 ceph-mon[221102]: pgmap v1420: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 16438/144597 objects degraded (11.368%); 8657/144597 objects misplaced (5.987%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:31:25.490 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:31:25.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:25.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:31:25.539 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:31:25.579 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:31:25.601 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:25.627 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:31:25.638 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:25 smithi138 ceph-mon[193606]: pgmap v1420: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 16438/144597 objects degraded (11.368%); 8657/144597 objects misplaced (5.987%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:31:25.646 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:25.666 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:31:25.699 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:31:25.705 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:25.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:25.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:25.736 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:31:25.777 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:31:25.814 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:25.817 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:31:25.828 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:31:25.828 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:31:25.829 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:31:25.829 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:31:25.829 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:31:25.829 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:31:25.829 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:31:25.829 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:31:25.852 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:25.912 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:25.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:25.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:26.019 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:26.062 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:26.131 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:26.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:26.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:26.240 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:31:26.314 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:26.586 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:31:26.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:26 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/586940971' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:31:26.611 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:26 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/586940971' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:31:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:26 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/586940971' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:31:26.953 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:31:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:31:26] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.43.0" 2023-09-23T23:31:27.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:27 smithi110 ceph-mon[212939]: pgmap v1421: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 16432/136209 objects degraded (12.064%); 7972/136209 objects misplaced (5.853%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T23:31:27.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:27 smithi110 ceph-mon[221102]: pgmap v1421: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 16432/136209 objects degraded (12.064%); 7972/136209 objects misplaced (5.853%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T23:31:27.575 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:31:27.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:27.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:31:27.682 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:31:27.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:27 smithi138 ceph-mon[193606]: pgmap v1421: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 16432/136209 objects degraded (12.064%); 7972/136209 objects misplaced (5.853%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T23:31:27.709 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:28.151 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:31:28.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:28.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:31:28.260 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:31:28.298 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:28.592 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:31:28.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:28.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:31:28.702 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:31:28.834 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:29.054 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:31:29.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:29.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:31:29.140 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:29 smithi110 ceph-mon[212939]: pgmap v1422: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 16432/135888 objects degraded (12.092%); 7972/135888 objects misplaced (5.867%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T23:31:29.140 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:29 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:31:29.141 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:29 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:31:29.141 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:29 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 16432/135888 objects degraded (12.092%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:29.143 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:29 smithi110 ceph-mon[221102]: pgmap v1422: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 16432/135888 objects degraded (12.092%); 7972/135888 objects misplaced (5.867%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T23:31:29.143 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:29 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:31:29.144 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:29 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:31:29.144 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:29 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 16432/135888 objects degraded (12.092%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:29.168 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:31:29.216 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:29.276 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:29.416 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:31:29.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:29.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:31:29.432 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:31:29.432 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:31:29.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:29 smithi138 ceph-mon[193606]: pgmap v1422: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 16432/135888 objects degraded (12.092%); 7972/135888 objects misplaced (5.867%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T23:31:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:29 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:31:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:29 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:31:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:29 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 16432/135888 objects degraded (12.092%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:29.531 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:29.564 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:29.625 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:29.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:29.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:29.731 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:29.776 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:29.858 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:29.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:29.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:29.969 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:30.023 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:30.048 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T23:31:30.097 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:30.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:30.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:30.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:30 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1106545712' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:31:30.216 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:31:30.260 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:30.272 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:30 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1106545712' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:31:30.273 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:30 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1106545712' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:31:30.371 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:31:30.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:30.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:31:30.481 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:30.526 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:30.568 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:30.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:30.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:30.673 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:31:30.700 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:30.756 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:31:31.353 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:31:31.356 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:31 smithi110 ceph-mon[212939]: pgmap v1423: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 15962/127866 objects degraded (12.483%); 7734/127866 objects misplaced (6.049%); 3.1 MiB/s, 97 objects/s recovering 2023-09-23T23:31:31.356 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:31 smithi110 ceph-mon[212939]: failed to encode map e515 with expected crc 2023-09-23T23:31:31.356 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:31 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1106545712' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:31:31.356 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:31 smithi110 ceph-mon[212939]: osdmap e515: 8 total, 8 up, 8 in 2023-09-23T23:31:31.356 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:31 smithi110 ceph-mon[212939]: failed to encode map e515 with expected crc 2023-09-23T23:31:31.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:31.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:31:31.357 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:31 smithi110 ceph-mon[221102]: pgmap v1423: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 15962/127866 objects degraded (12.483%); 7734/127866 objects misplaced (6.049%); 3.1 MiB/s, 97 objects/s recovering 2023-09-23T23:31:31.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:31 smithi110 ceph-mon[221102]: failed to encode map e515 with expected crc 2023-09-23T23:31:31.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:31 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1106545712' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:31:31.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:31 smithi110 ceph-mon[221102]: osdmap e515: 8 total, 8 up, 8 in 2023-09-23T23:31:31.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:31 smithi110 ceph-mon[221102]: failed to encode map e515 with expected crc 2023-09-23T23:31:31.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:31 smithi138 ceph-mon[193606]: pgmap v1423: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 15962/127866 objects degraded (12.483%); 7734/127866 objects misplaced (6.049%); 3.1 MiB/s, 97 objects/s recovering 2023-09-23T23:31:31.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:31 smithi138 ceph-mon[193606]: failed to encode map e515 with expected crc 2023-09-23T23:31:31.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:31 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1106545712' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:31:31.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:31 smithi138 ceph-mon[193606]: osdmap e515: 8 total, 8 up, 8 in 2023-09-23T23:31:31.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:31 smithi138 ceph-mon[193606]: failed to encode map e515 with expected crc 2023-09-23T23:31:31.458 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:31:31.531 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:32.081 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:31:32.764 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:31:32.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:32.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:31:32.859 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T23:31:32.887 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:31:33.013 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:33.049 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:33.291 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:31:33.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:33.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:31:33.305 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:33 smithi110 ceph-mon[212939]: pgmap v1425: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 15962/124815 objects degraded (12.789%); 7734/124815 objects misplaced (6.196%); 3.6 MiB/s, 114 objects/s recovering 2023-09-23T23:31:33.307 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:33 smithi110 ceph-mon[221102]: pgmap v1425: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 15962/124815 objects degraded (12.789%); 7734/124815 objects misplaced (6.196%); 3.6 MiB/s, 114 objects/s recovering 2023-09-23T23:31:33.403 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:31:33.429 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:33 smithi138 ceph-mon[193606]: pgmap v1425: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 15962/124815 objects degraded (12.789%); 7734/124815 objects misplaced (6.196%); 3.6 MiB/s, 114 objects/s recovering 2023-09-23T23:31:33.431 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:33.506 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:31:33.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:33.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:31:33.555 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:33.611 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:33.645 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:33.724 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:33.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:33.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:33.831 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:31:33.917 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:34.059 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:31:34.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:34.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:31:34.170 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:31:34.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:34 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 15962/124815 objects degraded (12.789%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:34.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:34 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4102778813' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:31:34.203 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:34.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:34 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 15962/124815 objects degraded (12.789%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:34.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:34 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4102778813' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:31:34.278 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:31:34.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:34.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:31:34.384 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:34.407 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:34 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 15962/124815 objects degraded (12.789%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:34.407 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:34 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4102778813' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:31:34.409 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:34.458 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:34.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:34.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:34.568 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:34.609 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:34.688 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:34.699 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:31:34.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:34.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:34.803 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:31:34.841 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:34.915 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:31:34.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:34.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:31:35.027 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:35.067 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:35.143 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:35.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:35.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:35.248 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:35.291 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:35.342 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:35.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:35 smithi138 ceph-mon[193606]: pgmap v1426: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 15962/123507 objects degraded (12.924%); 7734/123507 objects misplaced (6.262%); 3.1 MiB/s, 97 objects/s recovering 2023-09-23T23:31:35.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:35 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:31:35.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:35 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4102778813' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:31:35.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:35 smithi138 ceph-mon[193606]: osdmap e516: 8 total, 8 up, 8 in 2023-09-23T23:31:35.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:35 smithi138 ceph-mon[193606]: failed to encode map e516 with expected crc 2023-09-23T23:31:35.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:35.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:35.449 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:31:35.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:35 smithi110 ceph-mon[212939]: pgmap v1426: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 15962/123507 objects degraded (12.924%); 7734/123507 objects misplaced (6.262%); 3.1 MiB/s, 97 objects/s recovering 2023-09-23T23:31:35.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:35 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:31:35.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:35 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4102778813' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:31:35.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:35 smithi110 ceph-mon[212939]: osdmap e516: 8 total, 8 up, 8 in 2023-09-23T23:31:35.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:35 smithi110 ceph-mon[212939]: failed to encode map e516 with expected crc 2023-09-23T23:31:35.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:35 smithi110 ceph-mon[221102]: pgmap v1426: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 15962/123507 objects degraded (12.924%); 7734/123507 objects misplaced (6.262%); 3.1 MiB/s, 97 objects/s recovering 2023-09-23T23:31:35.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:35 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:31:35.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:35 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4102778813' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:31:35.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:35 smithi110 ceph-mon[221102]: osdmap e516: 8 total, 8 up, 8 in 2023-09-23T23:31:35.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:35 smithi110 ceph-mon[221102]: failed to encode map e516 with expected crc 2023-09-23T23:31:35.524 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:35.737 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:35.930 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:31:35.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:35.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:31:36.049 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:36.094 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:36.147 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:36.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:36.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:36.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:36 smithi110 ceph-mon[221102]: failed to encode map e516 with expected crc 2023-09-23T23:31:36.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:36 smithi110 ceph-mon[221102]: 12.10 scrub starts 2023-09-23T23:31:36.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:36 smithi110 ceph-mon[221102]: 12.3 scrub starts 2023-09-23T23:31:36.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:36 smithi110 ceph-mon[212939]: failed to encode map e516 with expected crc 2023-09-23T23:31:36.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:36 smithi110 ceph-mon[212939]: 12.10 scrub starts 2023-09-23T23:31:36.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:36 smithi110 ceph-mon[212939]: 12.3 scrub starts 2023-09-23T23:31:36.252 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:31:36.325 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:36.420 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:31:36.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:36.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:31:36.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:36 smithi138 ceph-mon[193606]: failed to encode map e516 with expected crc 2023-09-23T23:31:36.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:36 smithi138 ceph-mon[193606]: 12.10 scrub starts 2023-09-23T23:31:36.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:36 smithi138 ceph-mon[193606]: 12.3 scrub starts 2023-09-23T23:31:36.524 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:31:36.566 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:36.618 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:31:36.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:36.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:31:36.722 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:36.748 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:36.791 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:36.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:36.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:36.897 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:31:36.923 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:31:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:31:36] "GET /metrics HTTP/1.1" 200 36571 "" "Prometheus/2.43.0" 2023-09-23T23:31:36.926 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:37.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:37 smithi138 ceph-mon[193606]: pgmap v1428: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 896 op/s; 14609/114375 objects degraded (12.773%); 8241/114375 objects misplaced (7.205%); 16 KiB/s, 137 objects/s recovering 2023-09-23T23:31:37.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:37 smithi138 ceph-mon[193606]: 12.2 scrub starts 2023-09-23T23:31:37.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:37 smithi138 ceph-mon[193606]: osdmap e517: 8 total, 8 up, 8 in 2023-09-23T23:31:37.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:37 smithi138 ceph-mon[193606]: failed to encode map e517 with expected crc 2023-09-23T23:31:37.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:37 smithi138 ceph-mon[193606]: failed to encode map e517 with expected crc 2023-09-23T23:31:37.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:37 smithi138 ceph-mon[193606]: failed to encode map e517 with expected crc 2023-09-23T23:31:37.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:37 smithi138 ceph-mon[193606]: failed to encode map e517 with expected crc 2023-09-23T23:31:37.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:37 smithi138 ceph-mon[193606]: failed to encode map e517 with expected crc 2023-09-23T23:31:37.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:37 smithi110 ceph-mon[212939]: pgmap v1428: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 896 op/s; 14609/114375 objects degraded (12.773%); 8241/114375 objects misplaced (7.205%); 16 KiB/s, 137 objects/s recovering 2023-09-23T23:31:37.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:37 smithi110 ceph-mon[212939]: 12.2 scrub starts 2023-09-23T23:31:37.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:37 smithi110 ceph-mon[212939]: osdmap e517: 8 total, 8 up, 8 in 2023-09-23T23:31:37.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:37 smithi110 ceph-mon[212939]: failed to encode map e517 with expected crc 2023-09-23T23:31:37.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:37 smithi110 ceph-mon[212939]: failed to encode map e517 with expected crc 2023-09-23T23:31:37.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:37 smithi110 ceph-mon[212939]: failed to encode map e517 with expected crc 2023-09-23T23:31:37.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:37 smithi110 ceph-mon[212939]: failed to encode map e517 with expected crc 2023-09-23T23:31:37.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:37 smithi110 ceph-mon[212939]: failed to encode map e517 with expected crc 2023-09-23T23:31:37.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:37 smithi110 ceph-mon[221102]: pgmap v1428: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 896 op/s; 14609/114375 objects degraded (12.773%); 8241/114375 objects misplaced (7.205%); 16 KiB/s, 137 objects/s recovering 2023-09-23T23:31:37.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:37 smithi110 ceph-mon[221102]: 12.2 scrub starts 2023-09-23T23:31:37.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:37 smithi110 ceph-mon[221102]: osdmap e517: 8 total, 8 up, 8 in 2023-09-23T23:31:37.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:37 smithi110 ceph-mon[221102]: failed to encode map e517 with expected crc 2023-09-23T23:31:37.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:37 smithi110 ceph-mon[221102]: failed to encode map e517 with expected crc 2023-09-23T23:31:37.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:37 smithi110 ceph-mon[221102]: failed to encode map e517 with expected crc 2023-09-23T23:31:37.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:37 smithi110 ceph-mon[221102]: failed to encode map e517 with expected crc 2023-09-23T23:31:37.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:37 smithi110 ceph-mon[221102]: failed to encode map e517 with expected crc 2023-09-23T23:31:37.389 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:31:37.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:37.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:31:37.504 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:31:37.588 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:37.781 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:31:37.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:37.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:31:37.893 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:37.941 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:38.010 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:38.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:38.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:38.117 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:31:38.129 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:38 smithi110 ceph-mon[212939]: 12.10 scrub starts 2023-09-23T23:31:38.129 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:38 smithi110 ceph-mon[212939]: failed to encode map e518 with expected crc 2023-09-23T23:31:38.129 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:38 smithi110 ceph-mon[212939]: osdmap e518: 8 total, 8 up, 8 in 2023-09-23T23:31:38.129 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:38 smithi110 ceph-mon[212939]: failed to encode map e518 with expected crc 2023-09-23T23:31:38.129 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:38 smithi110 ceph-mon[212939]: failed to encode map e518 with expected crc 2023-09-23T23:31:38.130 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:38 smithi110 ceph-mon[212939]: failed to encode map e518 with expected crc 2023-09-23T23:31:38.130 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:38 smithi110 ceph-mon[221102]: 12.10 scrub starts 2023-09-23T23:31:38.130 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:38 smithi110 ceph-mon[221102]: failed to encode map e518 with expected crc 2023-09-23T23:31:38.130 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:38 smithi110 ceph-mon[221102]: osdmap e518: 8 total, 8 up, 8 in 2023-09-23T23:31:38.130 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:38 smithi110 ceph-mon[221102]: failed to encode map e518 with expected crc 2023-09-23T23:31:38.130 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:38 smithi110 ceph-mon[221102]: failed to encode map e518 with expected crc 2023-09-23T23:31:38.130 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:38 smithi110 ceph-mon[221102]: failed to encode map e518 with expected crc 2023-09-23T23:31:38.160 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:31:38.177 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:31:38.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:38 smithi138 ceph-mon[193606]: 12.10 scrub starts 2023-09-23T23:31:38.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:38 smithi138 ceph-mon[193606]: failed to encode map e518 with expected crc 2023-09-23T23:31:38.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:38 smithi138 ceph-mon[193606]: osdmap e518: 8 total, 8 up, 8 in 2023-09-23T23:31:38.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:38 smithi138 ceph-mon[193606]: failed to encode map e518 with expected crc 2023-09-23T23:31:38.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:38 smithi138 ceph-mon[193606]: failed to encode map e518 with expected crc 2023-09-23T23:31:38.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:38 smithi138 ceph-mon[193606]: failed to encode map e518 with expected crc 2023-09-23T23:31:38.200 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:31:38.218 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:38.264 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:31:38.308 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:31:38.357 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:31:38.387 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:31:38.423 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:31:38.458 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:31:38.489 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:31:38.489 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:31:38.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:31:38.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:31:38.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:31:38.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:31:38.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:31:38.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:31:39.069 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:31:39.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:39.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:31:39.175 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:31:39.260 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:39 smithi138 ceph-mon[193606]: pgmap v1431: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 14609/114036 objects degraded (12.811%); 8241/114036 objects misplaced (7.227%); 21 KiB/s, 104 objects/s recovering 2023-09-23T23:31:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:39 smithi138 ceph-mon[193606]: 12.13 scrub starts 2023-09-23T23:31:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:39 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1351711830' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:31:39.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:39 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 14609/114036 objects degraded (12.811%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:39.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:39 smithi110 ceph-mon[221102]: pgmap v1431: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 14609/114036 objects degraded (12.811%); 8241/114036 objects misplaced (7.227%); 21 KiB/s, 104 objects/s recovering 2023-09-23T23:31:39.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:39 smithi110 ceph-mon[221102]: 12.13 scrub starts 2023-09-23T23:31:39.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:39 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1351711830' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:31:39.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:39 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 14609/114036 objects degraded (12.811%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:39.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:39 smithi110 ceph-mon[212939]: pgmap v1431: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 14609/114036 objects degraded (12.811%); 8241/114036 objects misplaced (7.227%); 21 KiB/s, 104 objects/s recovering 2023-09-23T23:31:39.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:39 smithi110 ceph-mon[212939]: 12.13 scrub starts 2023-09-23T23:31:39.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:39 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1351711830' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:31:39.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:39 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 14609/114036 objects degraded (12.811%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:40.315 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:31:40.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:40.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:31:40.407 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:31:40.407 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:31:40.407 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:31:40.408 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:31:40.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.408 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156651 10.1 10:8a72b73b:::benchmark_data_smithi110_366450_object55443:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:31:40.408 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:49.044180+0000", 2023-09-23T23:31:40.408 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 591.35674070699997, 2023-09-23T23:31:40.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.112146391, 2023-09-23T23:31:40.408 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.408 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:31:40.408 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.408 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:31:40.408 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:31:40.408 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156651 2023-09-23T23:31:40.409 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.409 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.409 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.409 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.409 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044180+0000", 2023-09-23T23:31:40.409 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.409 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.409 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.409 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.409 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044180+0000", 2023-09-23T23:31:40.409 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.410 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.410 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.410 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.410 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044179+0000", 2023-09-23T23:31:40.410 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:31:40.410 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.410 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.410 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.410 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044185+0000", 2023-09-23T23:31:40.410 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9699999999999996e-06 2023-09-23T23:31:40.410 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.410 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.411 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044197+0000", 2023-09-23T23:31:40.411 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2187e-05 2023-09-23T23:31:40.411 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.411 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.411 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044205+0000", 2023-09-23T23:31:40.411 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.678e-06 2023-09-23T23:31:40.411 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.411 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.412 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044265+0000", 2023-09-23T23:31:40.412 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9889000000000002e-05 2023-09-23T23:31:40.412 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.412 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.412 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op must wait for map", 2023-09-23T23:31:40.412 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.044271+0000", 2023-09-23T23:31:40.412 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.455e-06 2023-09-23T23:31:40.412 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.412 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.412 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.412 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.052559+0000", 2023-09-23T23:31:40.413 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0082876549999999997 2023-09-23T23:31:40.413 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.413 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.413 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.413 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.052569+0000", 2023-09-23T23:31:40.413 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0039e-05 2023-09-23T23:31:40.413 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.413 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.413 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.413 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032597+0000", 2023-09-23T23:31:40.413 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98002873899999998 2023-09-23T23:31:40.413 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.414 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.414 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.414 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032607+0000", 2023-09-23T23:31:40.414 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4720000000000001e-06 2023-09-23T23:31:40.414 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.414 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.414 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.414 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.154569+0000", 2023-09-23T23:31:40.414 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.121962066 2023-09-23T23:31:40.414 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.414 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.414 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.415 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.156326+0000", 2023-09-23T23:31:40.415 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017571030000000001 2023-09-23T23:31:40.415 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.415 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.415 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.415 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.415 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.415 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156668 10.1 10:8e70998a:::benchmark_data_smithi110_366450_object55460:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:31:40.415 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:49.064619+0000", 2023-09-23T23:31:40.415 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 591.336301069, 2023-09-23T23:31:40.415 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1088757770000002, 2023-09-23T23:31:40.415 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.415 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.416 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.416 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:31:40.416 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:31:40.416 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156668 2023-09-23T23:31:40.416 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.416 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.416 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.416 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.416 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064619+0000", 2023-09-23T23:31:40.416 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.416 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.416 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.417 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.417 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064619+0000", 2023-09-23T23:31:40.417 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.417 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.417 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.417 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.417 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064619+0000", 2023-09-23T23:31:40.417 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:31:40.417 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.417 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.417 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.417 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064623+0000", 2023-09-23T23:31:40.417 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4089999999999996e-06 2023-09-23T23:31:40.418 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.418 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.418 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.418 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064632+0000", 2023-09-23T23:31:40.418 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5830000000000007e-06 2023-09-23T23:31:40.418 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.418 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.418 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.418 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.064637+0000", 2023-09-23T23:31:40.418 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0760000000000002e-06 2023-09-23T23:31:40.418 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.418 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.418 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.419 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.066069+0000", 2023-09-23T23:31:40.419 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014321799999999999 2023-09-23T23:31:40.419 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.419 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.419 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.419 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.066077+0000", 2023-09-23T23:31:40.419 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8390000000000007e-06 2023-09-23T23:31:40.419 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.419 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.419 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.419 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032631+0000", 2023-09-23T23:31:40.419 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96655426600000005 2023-09-23T23:31:40.420 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.420 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.420 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.420 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032638+0000", 2023-09-23T23:31:40.420 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7669999999999998e-06 2023-09-23T23:31:40.420 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.420 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.420 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.420 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.156763+0000", 2023-09-23T23:31:40.420 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.124125493 2023-09-23T23:31:40.420 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.420 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.420 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.421 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.157388+0000", 2023-09-23T23:31:40.421 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00062452500000000002 2023-09-23T23:31:40.421 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.421 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.421 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:31:40.421 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.157840+0000", 2023-09-23T23:31:40.421 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045213799999999999 2023-09-23T23:31:40.421 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.421 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.421 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.421 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.165314+0000", 2023-09-23T23:31:40.421 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074745469999999998 2023-09-23T23:31:40.422 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.422 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.422 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.422 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.165348+0000", 2023-09-23T23:31:40.422 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3593999999999997e-05 2023-09-23T23:31:40.422 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.422 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.422 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.422 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173335+0000", 2023-09-23T23:31:40.422 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079866030000000001 2023-09-23T23:31:40.422 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.422 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.423 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.423 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173385+0000", 2023-09-23T23:31:40.423 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0932e-05 2023-09-23T23:31:40.423 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.423 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.423 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.423 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173495+0000", 2023-09-23T23:31:40.423 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109603 2023-09-23T23:31:40.423 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.423 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.423 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.423 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.423 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.424 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156686 10.1 10:87df1f6d:::benchmark_data_smithi110_366450_object55478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:31:40.424 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:49.084621+0000", 2023-09-23T23:31:40.424 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 591.31629972799999, 2023-09-23T23:31:40.424 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0890410259999999, 2023-09-23T23:31:40.424 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.424 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.424 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.424 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:31:40.424 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:31:40.424 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156686 2023-09-23T23:31:40.424 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.424 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.425 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.425 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.425 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084621+0000", 2023-09-23T23:31:40.425 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.425 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.425 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.425 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.425 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084621+0000", 2023-09-23T23:31:40.425 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.425 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.425 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.425 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.426 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084620+0000", 2023-09-23T23:31:40.426 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:31:40.426 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.426 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.426 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.426 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084623+0000", 2023-09-23T23:31:40.426 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5040000000000001e-06 2023-09-23T23:31:40.426 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.426 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.426 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.426 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084634+0000", 2023-09-23T23:31:40.426 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1405e-05 2023-09-23T23:31:40.426 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.427 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.427 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.427 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084639+0000", 2023-09-23T23:31:40.427 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.729e-06 2023-09-23T23:31:40.427 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.427 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.427 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.427 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084684+0000", 2023-09-23T23:31:40.427 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5045e-05 2023-09-23T23:31:40.427 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.427 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.427 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.428 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084690+0000", 2023-09-23T23:31:40.428 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5650000000000001e-06 2023-09-23T23:31:40.428 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.428 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.428 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.428 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032662+0000", 2023-09-23T23:31:40.428 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94797212099999995 2023-09-23T23:31:40.428 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.428 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.428 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.428 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032668+0000", 2023-09-23T23:31:40.428 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6080000000000001e-06 2023-09-23T23:31:40.429 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.429 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.429 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.429 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.162444+0000", 2023-09-23T23:31:40.429 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1297754170000001 2023-09-23T23:31:40.429 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.429 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.429 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.429 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.162591+0000", 2023-09-23T23:31:40.429 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014782799999999999 2023-09-23T23:31:40.429 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.430 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.430 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:31:40.430 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.162685+0000", 2023-09-23T23:31:40.430 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3898000000000001e-05 2023-09-23T23:31:40.430 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.430 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.430 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.430 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.170322+0000", 2023-09-23T23:31:40.430 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007636866 2023-09-23T23:31:40.430 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.430 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.430 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.431 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.170366+0000", 2023-09-23T23:31:40.431 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3380000000000001e-05 2023-09-23T23:31:40.431 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.431 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.431 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.431 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173508+0000", 2023-09-23T23:31:40.431 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031422640000000001 2023-09-23T23:31:40.431 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.431 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.431 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.431 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173562+0000", 2023-09-23T23:31:40.431 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4169000000000003e-05 2023-09-23T23:31:40.431 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.431 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.432 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.432 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173662+0000", 2023-09-23T23:31:40.432 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9615000000000001e-05 2023-09-23T23:31:40.432 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.432 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.432 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.432 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.432 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.432 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35529.0:156687 10.1 10:8bf04f95:::benchmark_data_smithi110_366450_object55479:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T23:31:40.432 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:21:49.084650+0000", 2023-09-23T23:31:40.432 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 591.31627055199999, 2023-09-23T23:31:40.432 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.089277176, 2023-09-23T23:31:40.433 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.433 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.433 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.433 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35529", 2023-09-23T23:31:40.433 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/3649735323", 2023-09-23T23:31:40.433 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156687 2023-09-23T23:31:40.433 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.433 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.433 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.433 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.433 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084650+0000", 2023-09-23T23:31:40.433 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.433 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.434 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.434 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.434 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084650+0000", 2023-09-23T23:31:40.434 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.434 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.434 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.434 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.434 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084649+0000", 2023-09-23T23:31:40.434 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:31:40.434 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.434 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.434 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.435 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084652+0000", 2023-09-23T23:31:40.435 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2670000000000001e-06 2023-09-23T23:31:40.435 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.435 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.435 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.435 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084658+0000", 2023-09-23T23:31:40.435 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1279999999999996e-06 2023-09-23T23:31:40.435 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.435 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.435 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.435 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084662+0000", 2023-09-23T23:31:40.435 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1289999999999998e-06 2023-09-23T23:31:40.436 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.436 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.436 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084716+0000", 2023-09-23T23:31:40.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4110000000000002e-05 2023-09-23T23:31:40.436 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.436 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.436 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:49.084725+0000", 2023-09-23T23:31:40.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6989999999999992e-06 2023-09-23T23:31:40.436 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.437 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032691+0000", 2023-09-23T23:31:40.437 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94796597400000004 2023-09-23T23:31:40.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.437 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:50.032698+0000", 2023-09-23T23:31:40.437 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7479999999999996e-06 2023-09-23T23:31:40.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.437 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.165371+0000", 2023-09-23T23:31:40.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1326732019999999 2023-09-23T23:31:40.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.438 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.165505+0000", 2023-09-23T23:31:40.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013406900000000001 2023-09-23T23:31:40.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.438 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:31:40.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.165655+0000", 2023-09-23T23:31:40.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015065 2023-09-23T23:31:40.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.439 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.439 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.439 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173673+0000", 2023-09-23T23:31:40.439 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080170699999999994 2023-09-23T23:31:40.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.439 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.439 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.439 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173736+0000", 2023-09-23T23:31:40.439 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3116999999999999e-05 2023-09-23T23:31:40.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.439 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.439 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.440 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173781+0000", 2023-09-23T23:31:40.440 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5778000000000001e-05 2023-09-23T23:31:40.440 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.440 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.440 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.440 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173829+0000", 2023-09-23T23:31:40.440 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7182999999999999e-05 2023-09-23T23:31:40.440 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.440 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.440 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.440 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:21:51.173927+0000", 2023-09-23T23:31:40.440 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8424000000000005e-05 2023-09-23T23:31:40.441 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.441 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.441 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.441 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.441 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135599 11.e 11:71e23346:::benchmark_data_smithi110_433746_object29719:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:31:40.441 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505704+0000", 2023-09-23T23:31:40.441 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 328.89521659299999, 2023-09-23T23:31:40.441 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.837012734, 2023-09-23T23:31:40.441 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.441 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.441 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.442 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:31:40.442 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:31:40.442 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135599 2023-09-23T23:31:40.442 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.442 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.442 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.442 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.442 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505704+0000", 2023-09-23T23:31:40.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.442 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.442 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.442 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.443 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505704+0000", 2023-09-23T23:31:40.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.443 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505702+0000", 2023-09-23T23:31:40.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:31:40.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.443 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505710+0000", 2023-09-23T23:31:40.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.038e-06 2023-09-23T23:31:40.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.444 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505724+0000", 2023-09-23T23:31:40.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3684e-05 2023-09-23T23:31:40.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.444 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505733+0000", 2023-09-23T23:31:40.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1309999999999995e-06 2023-09-23T23:31:40.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.444 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.445 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505925+0000", 2023-09-23T23:31:40.445 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000192282 2023-09-23T23:31:40.445 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.445 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.445 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op must wait for map", 2023-09-23T23:31:40.445 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505933+0000", 2023-09-23T23:31:40.445 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3089999999999998e-06 2023-09-23T23:31:40.445 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.445 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.445 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.445 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513411+0000", 2023-09-23T23:31:40.445 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074784489999999999 2023-09-23T23:31:40.445 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.446 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.446 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513421+0000", 2023-09-23T23:31:40.446 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0108e-05 2023-09-23T23:31:40.446 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.446 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.446 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.304735+0000", 2023-09-23T23:31:40.446 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.791314093 2023-09-23T23:31:40.446 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.446 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.447 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.306968+0000", 2023-09-23T23:31:40.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022327390000000001 2023-09-23T23:31:40.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.447 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.447 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:31:40.447 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.307182+0000", 2023-09-23T23:31:40.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000214291 2023-09-23T23:31:40.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.447 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.447 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.447 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.322749+0000", 2023-09-23T23:31:40.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015566695 2023-09-23T23:31:40.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.322789+0000", 2023-09-23T23:31:40.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9808999999999998e-05 2023-09-23T23:31:40.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334988+0000", 2023-09-23T23:31:40.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012198595 2023-09-23T23:31:40.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342545+0000", 2023-09-23T23:31:40.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075570389999999998 2023-09-23T23:31:40.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342599+0000", 2023-09-23T23:31:40.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4347e-05 2023-09-23T23:31:40.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342717+0000", 2023-09-23T23:31:40.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117578 2023-09-23T23:31:40.450 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.450 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.450 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.450 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135605 11.e 11:7071ef4a:::benchmark_data_smithi110_433746_object29725:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:31:40.450 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505749+0000", 2023-09-23T23:31:40.450 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 328.89517155599998, 2023-09-23T23:31:40.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.837123356, 2023-09-23T23:31:40.450 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.451 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.451 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.451 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:31:40.451 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:31:40.451 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135605 2023-09-23T23:31:40.451 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.451 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.451 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.451 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.451 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505749+0000", 2023-09-23T23:31:40.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.451 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505749+0000", 2023-09-23T23:31:40.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505748+0000", 2023-09-23T23:31:40.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:31:40.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505751+0000", 2023-09-23T23:31:40.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.056e-06 2023-09-23T23:31:40.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505760+0000", 2023-09-23T23:31:40.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3750000000000003e-06 2023-09-23T23:31:40.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505766+0000", 2023-09-23T23:31:40.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8760000000000002e-06 2023-09-23T23:31:40.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505982+0000", 2023-09-23T23:31:40.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021631100000000001 2023-09-23T23:31:40.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:31:40.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505988+0000", 2023-09-23T23:31:40.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5779999999999998e-06 2023-09-23T23:31:40.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513443+0000", 2023-09-23T23:31:40.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007455377 2023-09-23T23:31:40.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513450+0000", 2023-09-23T23:31:40.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1459999999999999e-06 2023-09-23T23:31:40.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.322816+0000", 2023-09-23T23:31:40.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.809365865 2023-09-23T23:31:40.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.456 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.322945+0000", 2023-09-23T23:31:40.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012866099999999999 2023-09-23T23:31:40.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.456 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:31:40.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.323097+0000", 2023-09-23T23:31:40.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015215899999999999 2023-09-23T23:31:40.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.457 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.457 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334919+0000", 2023-09-23T23:31:40.457 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011822687 2023-09-23T23:31:40.457 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.457 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.457 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.457 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334954+0000", 2023-09-23T23:31:40.457 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4987999999999998e-05 2023-09-23T23:31:40.457 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.457 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.457 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.457 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335050+0000", 2023-09-23T23:31:40.458 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5222000000000004e-05 2023-09-23T23:31:40.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.458 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.458 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.458 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342725+0000", 2023-09-23T23:31:40.458 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076749360000000003 2023-09-23T23:31:40.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.458 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.458 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.458 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342767+0000", 2023-09-23T23:31:40.458 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2929000000000002e-05 2023-09-23T23:31:40.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.459 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342872+0000", 2023-09-23T23:31:40.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104709 2023-09-23T23:31:40.459 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.459 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.459 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.459 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.459 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135609 11.e 11:73c3a869:::benchmark_data_smithi110_433746_object29729:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:31:40.459 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505771+0000", 2023-09-23T23:31:40.459 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 328.895149054, 2023-09-23T23:31:40.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8373706329999999, 2023-09-23T23:31:40.460 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.460 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.460 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.460 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:31:40.460 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:31:40.460 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135609 2023-09-23T23:31:40.460 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.460 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.460 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.460 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.460 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505771+0000", 2023-09-23T23:31:40.460 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.460 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.461 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505771+0000", 2023-09-23T23:31:40.461 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.461 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.461 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505771+0000", 2023-09-23T23:31:40.461 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:31:40.461 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505773+0000", 2023-09-23T23:31:40.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4109999999999999e-06 2023-09-23T23:31:40.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505779+0000", 2023-09-23T23:31:40.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9529999999999996e-06 2023-09-23T23:31:40.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505784+0000", 2023-09-23T23:31:40.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1279999999999999e-06 2023-09-23T23:31:40.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.463 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.463 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506007+0000", 2023-09-23T23:31:40.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022278700000000001 2023-09-23T23:31:40.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.463 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:31:40.463 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506012+0000", 2023-09-23T23:31:40.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0390000000000004e-06 2023-09-23T23:31:40.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513470+0000", 2023-09-23T23:31:40.464 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074576879999999996 2023-09-23T23:31:40.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513476+0000", 2023-09-23T23:31:40.464 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9059999999999999e-06 2023-09-23T23:31:40.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.325254+0000", 2023-09-23T23:31:40.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8117784690000001 2023-09-23T23:31:40.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.465 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.465 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.325381+0000", 2023-09-23T23:31:40.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012669499999999999 2023-09-23T23:31:40.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.465 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:31:40.465 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.325510+0000", 2023-09-23T23:31:40.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012926999999999999 2023-09-23T23:31:40.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.466 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.466 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335017+0000", 2023-09-23T23:31:40.466 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0095070620000000002 2023-09-23T23:31:40.466 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.466 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.466 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.466 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335119+0000", 2023-09-23T23:31:40.466 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102045 2023-09-23T23:31:40.466 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.466 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.466 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.466 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342880+0000", 2023-09-23T23:31:40.467 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077602000000000001 2023-09-23T23:31:40.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.467 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.467 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342995+0000", 2023-09-23T23:31:40.467 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000115253 2023-09-23T23:31:40.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.467 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.467 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343036+0000", 2023-09-23T23:31:40.467 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0632e-05 2023-09-23T23:31:40.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.468 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.468 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343142+0000", 2023-09-23T23:31:40.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010644 2023-09-23T23:31:40.468 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.468 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.468 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.468 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.468 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135623 11.e 11:7e8ebd90:::benchmark_data_smithi110_433746_object29743:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:31:40.468 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505789+0000", 2023-09-23T23:31:40.468 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 328.89513175399998, 2023-09-23T23:31:40.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.843964774, 2023-09-23T23:31:40.469 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.469 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.469 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.469 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:31:40.469 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:31:40.469 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135623 2023-09-23T23:31:40.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.469 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.469 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.469 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.469 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505789+0000", 2023-09-23T23:31:40.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505789+0000", 2023-09-23T23:31:40.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505788+0000", 2023-09-23T23:31:40.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:31:40.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505791+0000", 2023-09-23T23:31:40.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.339e-06 2023-09-23T23:31:40.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505797+0000", 2023-09-23T23:31:40.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.207e-06 2023-09-23T23:31:40.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505802+0000", 2023-09-23T23:31:40.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9969999999999998e-06 2023-09-23T23:31:40.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506033+0000", 2023-09-23T23:31:40.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000230824 2023-09-23T23:31:40.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:31:40.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506037+0000", 2023-09-23T23:31:40.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8690000000000003e-06 2023-09-23T23:31:40.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513487+0000", 2023-09-23T23:31:40.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074491310000000003 2023-09-23T23:31:40.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513490+0000", 2023-09-23T23:31:40.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4649999999999999e-06 2023-09-23T23:31:40.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.327901+0000", 2023-09-23T23:31:40.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.814410818 2023-09-23T23:31:40.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.328011+0000", 2023-09-23T23:31:40.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109596 2023-09-23T23:31:40.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:31:40.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.328129+0000", 2023-09-23T23:31:40.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118751 2023-09-23T23:31:40.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335081+0000", 2023-09-23T23:31:40.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0069518549999999998 2023-09-23T23:31:40.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342886+0000", 2023-09-23T23:31:40.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078050599999999999 2023-09-23T23:31:40.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343224+0000", 2023-09-23T23:31:40.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033781399999999997 2023-09-23T23:31:40.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349601+0000", 2023-09-23T23:31:40.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063765899999999997 2023-09-23T23:31:40.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349647+0000", 2023-09-23T23:31:40.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6164e-05 2023-09-23T23:31:40.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.477 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.477 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349753+0000", 2023-09-23T23:31:40.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000106669 2023-09-23T23:31:40.477 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.477 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.477 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.477 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.477 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135624 11.e 11:79f5e3a3:::benchmark_data_smithi110_433746_object29744:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:31:40.477 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505805+0000", 2023-09-23T23:31:40.477 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 328.89511492899999, 2023-09-23T23:31:40.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.844120325, 2023-09-23T23:31:40.478 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.478 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.478 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.478 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:31:40.478 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:31:40.478 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135624 2023-09-23T23:31:40.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.478 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.478 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.478 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505805+0000", 2023-09-23T23:31:40.478 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505805+0000", 2023-09-23T23:31:40.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505805+0000", 2023-09-23T23:31:40.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:31:40.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505807+0000", 2023-09-23T23:31:40.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2120000000000002e-06 2023-09-23T23:31:40.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505813+0000", 2023-09-23T23:31:40.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.57e-06 2023-09-23T23:31:40.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505817+0000", 2023-09-23T23:31:40.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2980000000000002e-06 2023-09-23T23:31:40.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506057+0000", 2023-09-23T23:31:40.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000239836 2023-09-23T23:31:40.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:31:40.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506062+0000", 2023-09-23T23:31:40.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7870000000000001e-06 2023-09-23T23:31:40.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513504+0000", 2023-09-23T23:31:40.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074420959999999996 2023-09-23T23:31:40.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513508+0000", 2023-09-23T23:31:40.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9339999999999999e-06 2023-09-23T23:31:40.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.330221+0000", 2023-09-23T23:31:40.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.816713196 2023-09-23T23:31:40.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.330346+0000", 2023-09-23T23:31:40.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000125406 2023-09-23T23:31:40.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:31:40.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.330485+0000", 2023-09-23T23:31:40.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013825799999999999 2023-09-23T23:31:40.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342893+0000", 2023-09-23T23:31:40.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012408130999999999 2023-09-23T23:31:40.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.484 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.484 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342951+0000", 2023-09-23T23:31:40.484 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8354999999999999e-05 2023-09-23T23:31:40.484 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.484 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.484 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343297+0000", 2023-09-23T23:31:40.484 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034624800000000001 2023-09-23T23:31:40.484 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.484 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.484 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349790+0000", 2023-09-23T23:31:40.485 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0064925399999999998 2023-09-23T23:31:40.485 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.485 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.485 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.485 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349831+0000", 2023-09-23T23:31:40.485 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1069999999999998e-05 2023-09-23T23:31:40.485 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.485 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.485 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.485 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349926+0000", 2023-09-23T23:31:40.485 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4718999999999999e-05 2023-09-23T23:31:40.486 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.486 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.486 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.486 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.486 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135642 11.e 11:7847da74:::benchmark_data_smithi110_433746_object29762:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:31:40.486 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505821+0000", 2023-09-23T23:31:40.486 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 328.89509944299999, 2023-09-23T23:31:40.486 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.844417151, 2023-09-23T23:31:40.486 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.486 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.486 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.486 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:31:40.487 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:31:40.487 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135642 2023-09-23T23:31:40.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.487 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505821+0000", 2023-09-23T23:31:40.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505821+0000", 2023-09-23T23:31:40.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505821+0000", 2023-09-23T23:31:40.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:31:40.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505823+0000", 2023-09-23T23:31:40.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4949999999999998e-06 2023-09-23T23:31:40.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.489 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.489 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505829+0000", 2023-09-23T23:31:40.489 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.383e-06 2023-09-23T23:31:40.489 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.489 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.489 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505833+0000", 2023-09-23T23:31:40.489 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8770000000000001e-06 2023-09-23T23:31:40.489 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.489 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.489 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506081+0000", 2023-09-23T23:31:40.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024778400000000003 2023-09-23T23:31:40.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.490 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.490 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:31:40.490 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506086+0000", 2023-09-23T23:31:40.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0170000000000004e-06 2023-09-23T23:31:40.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.490 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.490 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.490 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513526+0000", 2023-09-23T23:31:40.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074400029999999997 2023-09-23T23:31:40.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513533+0000", 2023-09-23T23:31:40.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5189999999999998e-06 2023-09-23T23:31:40.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.332515+0000", 2023-09-23T23:31:40.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.818982726 2023-09-23T23:31:40.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.492 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.332698+0000", 2023-09-23T23:31:40.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000182046 2023-09-23T23:31:40.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.492 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:31:40.492 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.332836+0000", 2023-09-23T23:31:40.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013802000000000001 2023-09-23T23:31:40.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.492 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.492 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342905+0000", 2023-09-23T23:31:40.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010069121 2023-09-23T23:31:40.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.493 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.493 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343190+0000", 2023-09-23T23:31:40.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028497700000000001 2023-09-23T23:31:40.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.493 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.493 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343385+0000", 2023-09-23T23:31:40.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019561000000000001 2023-09-23T23:31:40.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.494 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.494 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350092+0000", 2023-09-23T23:31:40.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067069870000000002 2023-09-23T23:31:40.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.494 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.494 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350141+0000", 2023-09-23T23:31:40.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8365000000000002e-05 2023-09-23T23:31:40.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.494 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.494 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350238+0000", 2023-09-23T23:31:40.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7510999999999995e-05 2023-09-23T23:31:40.495 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.495 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.495 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.495 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.495 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135646 11.e 11:7bf5000e:::benchmark_data_smithi110_433746_object29766:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:31:40.495 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505837+0000", 2023-09-23T23:31:40.495 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 328.89508297899999, 2023-09-23T23:31:40.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.844569372, 2023-09-23T23:31:40.495 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.495 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.496 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.496 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:31:40.496 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:31:40.496 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135646 2023-09-23T23:31:40.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.496 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505837+0000", 2023-09-23T23:31:40.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505837+0000", 2023-09-23T23:31:40.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505837+0000", 2023-09-23T23:31:40.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:31:40.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505839+0000", 2023-09-23T23:31:40.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3530000000000002e-06 2023-09-23T23:31:40.498 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.498 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.498 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.498 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505846+0000", 2023-09-23T23:31:40.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1800000000000001e-06 2023-09-23T23:31:40.498 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.498 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.498 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.498 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505850+0000", 2023-09-23T23:31:40.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5700000000000003e-06 2023-09-23T23:31:40.498 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.498 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.498 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.499 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506104+0000", 2023-09-23T23:31:40.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025424300000000003 2023-09-23T23:31:40.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:31:40.499 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506109+0000", 2023-09-23T23:31:40.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8749999999999999e-06 2023-09-23T23:31:40.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.499 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513566+0000", 2023-09-23T23:31:40.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074570510000000001 2023-09-23T23:31:40.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513573+0000", 2023-09-23T23:31:40.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9879999999999998e-06 2023-09-23T23:31:40.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335148+0000", 2023-09-23T23:31:40.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.82157426 2023-09-23T23:31:40.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335293+0000", 2023-09-23T23:31:40.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014563600000000001 2023-09-23T23:31:40.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:31:40.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335527+0000", 2023-09-23T23:31:40.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023380100000000001 2023-09-23T23:31:40.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342911+0000", 2023-09-23T23:31:40.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007384191 2023-09-23T23:31:40.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.502 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.502 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343259+0000", 2023-09-23T23:31:40.502 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034814699999999999 2023-09-23T23:31:40.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.502 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.502 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343422+0000", 2023-09-23T23:31:40.502 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016240100000000001 2023-09-23T23:31:40.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.503 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.503 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350272+0000", 2023-09-23T23:31:40.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068501539999999998 2023-09-23T23:31:40.503 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.503 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.503 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350313+0000", 2023-09-23T23:31:40.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1044000000000002e-05 2023-09-23T23:31:40.503 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.503 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.503 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350407+0000", 2023-09-23T23:31:40.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3838000000000005e-05 2023-09-23T23:31:40.504 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.504 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.504 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.504 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.504 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.504 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135676 11.e 11:7b57d3a0:::benchmark_data_smithi110_433746_object29796:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:31:40.504 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505854+0000", 2023-09-23T23:31:40.504 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 328.89506636900001, 2023-09-23T23:31:40.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.844725422, 2023-09-23T23:31:40.504 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.504 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.504 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.505 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:31:40.505 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:31:40.505 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135676 2023-09-23T23:31:40.505 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.505 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.505 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.505 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.505 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505854+0000", 2023-09-23T23:31:40.505 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.505 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.505 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.505 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.506 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505854+0000", 2023-09-23T23:31:40.506 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.506 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.506 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.506 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.506 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505854+0000", 2023-09-23T23:31:40.506 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:31:40.506 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.506 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.506 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.506 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505856+0000", 2023-09-23T23:31:40.506 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3700000000000002e-06 2023-09-23T23:31:40.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.507 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.507 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505862+0000", 2023-09-23T23:31:40.507 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2809999999999997e-06 2023-09-23T23:31:40.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.507 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.507 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505867+0000", 2023-09-23T23:31:40.507 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.933e-06 2023-09-23T23:31:40.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.507 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506128+0000", 2023-09-23T23:31:40.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000260938 2023-09-23T23:31:40.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:31:40.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506132+0000", 2023-09-23T23:31:40.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3409999999999999e-06 2023-09-23T23:31:40.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513595+0000", 2023-09-23T23:31:40.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074631480000000002 2023-09-23T23:31:40.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.509 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.509 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513602+0000", 2023-09-23T23:31:40.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9140000000000002e-06 2023-09-23T23:31:40.509 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.509 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.509 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337712+0000", 2023-09-23T23:31:40.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8241103679999999 2023-09-23T23:31:40.509 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.509 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.509 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337841+0000", 2023-09-23T23:31:40.510 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012925499999999999 2023-09-23T23:31:40.510 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.510 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.510 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:31:40.510 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.338028+0000", 2023-09-23T23:31:40.510 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018702899999999999 2023-09-23T23:31:40.510 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.510 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.510 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.510 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342960+0000", 2023-09-23T23:31:40.510 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0049315160000000004 2023-09-23T23:31:40.510 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.511 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.511 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.511 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343333+0000", 2023-09-23T23:31:40.511 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00037339399999999998 2023-09-23T23:31:40.511 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.511 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.511 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.511 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343461+0000", 2023-09-23T23:31:40.511 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012787399999999999 2023-09-23T23:31:40.511 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.511 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.511 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.511 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350439+0000", 2023-09-23T23:31:40.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0069780379999999998 2023-09-23T23:31:40.512 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.512 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.512 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.512 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350480+0000", 2023-09-23T23:31:40.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.057e-05 2023-09-23T23:31:40.512 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.512 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.512 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.512 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350579+0000", 2023-09-23T23:31:40.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9774999999999999e-05 2023-09-23T23:31:40.512 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.513 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.513 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.513 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.513 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.513 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135677 11.e 11:7ce6155e:::benchmark_data_smithi110_433746_object29797:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:31:40.513 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505871+0000", 2023-09-23T23:31:40.513 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 328.89504908399999, 2023-09-23T23:31:40.513 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8476306259999999, 2023-09-23T23:31:40.513 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.513 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.513 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.513 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:31:40.513 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:31:40.514 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135677 2023-09-23T23:31:40.514 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.514 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.514 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.514 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.514 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505871+0000", 2023-09-23T23:31:40.514 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.514 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.514 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.514 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.514 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505871+0000", 2023-09-23T23:31:40.514 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.514 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.515 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.515 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.515 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505871+0000", 2023-09-23T23:31:40.515 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:31:40.515 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.515 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.515 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.515 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505873+0000", 2023-09-23T23:31:40.515 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.43e-06 2023-09-23T23:31:40.515 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.515 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.515 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.516 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505879+0000", 2023-09-23T23:31:40.516 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8250000000000001e-06 2023-09-23T23:31:40.516 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.516 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.516 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.516 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505884+0000", 2023-09-23T23:31:40.516 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.955e-06 2023-09-23T23:31:40.516 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.516 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.516 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.516 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506151+0000", 2023-09-23T23:31:40.516 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000266591 2023-09-23T23:31:40.517 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.517 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.517 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:31:40.517 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506156+0000", 2023-09-23T23:31:40.517 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8400000000000002e-06 2023-09-23T23:31:40.517 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.517 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.517 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.517 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513622+0000", 2023-09-23T23:31:40.517 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007466744 2023-09-23T23:31:40.517 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.517 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.518 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.518 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513629+0000", 2023-09-23T23:31:40.518 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4400000000000002e-06 2023-09-23T23:31:40.518 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.518 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.518 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.518 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.340120+0000", 2023-09-23T23:31:40.518 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.826490739 2023-09-23T23:31:40.518 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.518 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.518 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.518 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.340250+0000", 2023-09-23T23:31:40.519 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000130448 2023-09-23T23:31:40.519 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.519 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.519 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:31:40.519 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.340609+0000", 2023-09-23T23:31:40.519 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035861699999999999 2023-09-23T23:31:40.519 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.519 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.519 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.519 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.348233+0000", 2023-09-23T23:31:40.519 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007624123 2023-09-23T23:31:40.519 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.519 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.520 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.520 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.348294+0000", 2023-09-23T23:31:40.520 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1487000000000003e-05 2023-09-23T23:31:40.520 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.520 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.520 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.520 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.348306+0000", 2023-09-23T23:31:40.520 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1941e-05 2023-09-23T23:31:40.520 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.520 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.520 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.520 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.353328+0000", 2023-09-23T23:31:40.521 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0050216280000000002 2023-09-23T23:31:40.521 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.521 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.521 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.521 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.353383+0000", 2023-09-23T23:31:40.521 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5682e-05 2023-09-23T23:31:40.521 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.521 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.521 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.521 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.353502+0000", 2023-09-23T23:31:40.521 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118492 2023-09-23T23:31:40.521 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.522 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.522 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.522 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.522 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.522 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178600 11.6 11:659f4d42:::benchmark_data_smithi110_433746_object72720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2023-09-23T23:31:40.522 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.523923+0000", 2023-09-23T23:31:40.522 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 260.87699714899998, 2023-09-23T23:31:40.522 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2361555310000001, 2023-09-23T23:31:40.522 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.522 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:31:40.522 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.522 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:31:40.523 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:31:40.523 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178600 2023-09-23T23:31:40.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.523 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.523 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.523 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523923+0000", 2023-09-23T23:31:40.523 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.523 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.523 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523923+0000", 2023-09-23T23:31:40.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.524 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523922+0000", 2023-09-23T23:31:40.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:31:40.524 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523929+0000", 2023-09-23T23:31:40.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8800000000000002e-06 2023-09-23T23:31:40.524 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.525 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.525 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523940+0000", 2023-09-23T23:31:40.525 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.149e-05 2023-09-23T23:31:40.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.525 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.525 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523948+0000", 2023-09-23T23:31:40.525 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7970000000000001e-06 2023-09-23T23:31:40.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.525 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.525 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.531901+0000", 2023-09-23T23:31:40.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079527399999999998 2023-09-23T23:31:40.526 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.526 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.526 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.526 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.532034+0000", 2023-09-23T23:31:40.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013345499999999999 2023-09-23T23:31:40.526 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.526 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.526 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:31:40.526 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.532514+0000", 2023-09-23T23:31:40.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047994200000000002 2023-09-23T23:31:40.526 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.527 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.527 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.527 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.537104+0000", 2023-09-23T23:31:40.527 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045898019999999996 2023-09-23T23:31:40.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.527 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.527 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.527 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.538159+0000", 2023-09-23T23:31:40.527 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001055437 2023-09-23T23:31:40.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.527 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.527 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.527 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.547877+0000", 2023-09-23T23:31:40.528 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0097177860000000008 2023-09-23T23:31:40.528 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.528 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.528 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.547890+0000", 2023-09-23T23:31:40.528 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2656e-05 2023-09-23T23:31:40.528 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.528 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.528 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542710+0000", 2023-09-23T23:31:40.528 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99482022199999998 2023-09-23T23:31:40.528 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.529 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542720+0000", 2023-09-23T23:31:40.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9029999999999999e-06 2023-09-23T23:31:40.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.529 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760000+0000", 2023-09-23T23:31:40.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.217280103 2023-09-23T23:31:40.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.529 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760079+0000", 2023-09-23T23:31:40.530 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8777999999999997e-05 2023-09-23T23:31:40.530 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.530 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.530 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.530 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.530 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.530 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178608 11.6 11:6177c8c0:::benchmark_data_smithi110_433746_object72728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2023-09-23T23:31:40.530 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.545706+0000", 2023-09-23T23:31:40.530 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 260.85521472099998, 2023-09-23T23:31:40.530 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2481279590000001, 2023-09-23T23:31:40.530 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.530 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.531 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.531 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:31:40.531 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:31:40.531 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178608 2023-09-23T23:31:40.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.531 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545706+0000", 2023-09-23T23:31:40.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545706+0000", 2023-09-23T23:31:40.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545705+0000", 2023-09-23T23:31:40.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:31:40.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545709+0000", 2023-09-23T23:31:40.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6140000000000002e-06 2023-09-23T23:31:40.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.533 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.533 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545721+0000", 2023-09-23T23:31:40.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2345e-05 2023-09-23T23:31:40.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.533 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.533 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545730+0000", 2023-09-23T23:31:40.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9579999999999996e-06 2023-09-23T23:31:40.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.534 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.570191+0000", 2023-09-23T23:31:40.534 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024460784999999999 2023-09-23T23:31:40.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.534 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.570202+0000", 2023-09-23T23:31:40.534 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1192999999999999e-05 2023-09-23T23:31:40.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.534 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542744+0000", 2023-09-23T23:31:40.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.972541879 2023-09-23T23:31:40.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542752+0000", 2023-09-23T23:31:40.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1599999999999998e-06 2023-09-23T23:31:40.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760100+0000", 2023-09-23T23:31:40.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2173479519999999 2023-09-23T23:31:40.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760221+0000", 2023-09-23T23:31:40.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000120155 2023-09-23T23:31:40.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T23:31:40.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760707+0000", 2023-09-23T23:31:40.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00048629800000000003 2023-09-23T23:31:40.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.774649+0000", 2023-09-23T23:31:40.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013942049 2023-09-23T23:31:40.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783406+0000", 2023-09-23T23:31:40.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0087574420000000007 2023-09-23T23:31:40.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783684+0000", 2023-09-23T23:31:40.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027739099999999999 2023-09-23T23:31:40.538 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.538 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.538 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793677+0000", 2023-09-23T23:31:40.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0099932059999999993 2023-09-23T23:31:40.538 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.538 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.538 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793723+0000", 2023-09-23T23:31:40.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5883000000000001e-05 2023-09-23T23:31:40.539 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.539 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.539 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793834+0000", 2023-09-23T23:31:40.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110819 2023-09-23T23:31:40.539 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.539 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.539 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.539 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.540 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178617 11.16 11:6d8a65a1:::benchmark_data_smithi110_433746_object72737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:31:40.540 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.584811+0000", 2023-09-23T23:31:40.540 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 260.816109321, 2023-09-23T23:31:40.540 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1985552099999999, 2023-09-23T23:31:40.540 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.540 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.540 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.540 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:31:40.540 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:31:40.540 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178617 2023-09-23T23:31:40.540 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.540 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.541 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.541 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584811+0000", 2023-09-23T23:31:40.541 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.541 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.541 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584811+0000", 2023-09-23T23:31:40.541 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.541 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584809+0000", 2023-09-23T23:31:40.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:31:40.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584817+0000", 2023-09-23T23:31:40.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3339999999999996e-06 2023-09-23T23:31:40.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584831+0000", 2023-09-23T23:31:40.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4355000000000001e-05 2023-09-23T23:31:40.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584840+0000", 2023-09-23T23:31:40.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3410000000000003e-06 2023-09-23T23:31:40.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591289+0000", 2023-09-23T23:31:40.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0064494030000000003 2023-09-23T23:31:40.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591300+0000", 2023-09-23T23:31:40.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0597e-05 2023-09-23T23:31:40.544 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.544 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.544 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542631+0000", 2023-09-23T23:31:40.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.951331449 2023-09-23T23:31:40.544 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.544 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.544 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542644+0000", 2023-09-23T23:31:40.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.288e-05 2023-09-23T23:31:40.545 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.545 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.756708+0000", 2023-09-23T23:31:40.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.21406408 2023-09-23T23:31:40.545 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.545 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.756818+0000", 2023-09-23T23:31:40.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110073 2023-09-23T23:31:40.545 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T23:31:40.546 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.757269+0000", 2023-09-23T23:31:40.546 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045095900000000002 2023-09-23T23:31:40.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.546 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.546 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.546 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764656+0000", 2023-09-23T23:31:40.546 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007387195 2023-09-23T23:31:40.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.546 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.546 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.546 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764667+0000", 2023-09-23T23:31:40.546 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0365e-05 2023-09-23T23:31:40.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.547 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.547 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.774688+0000", 2023-09-23T23:31:40.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010021577 2023-09-23T23:31:40.547 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.547 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.547 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783206+0000", 2023-09-23T23:31:40.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0085183289999999998 2023-09-23T23:31:40.547 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.547 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.548 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783262+0000", 2023-09-23T23:31:40.548 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5275999999999997e-05 2023-09-23T23:31:40.548 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.548 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.548 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.548 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783366+0000", 2023-09-23T23:31:40.548 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010458099999999999 2023-09-23T23:31:40.548 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.548 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.548 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.548 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.548 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.549 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178622 11.6 11:6344b54e:::benchmark_data_smithi110_433746_object72742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:31:40.549 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.591902+0000", 2023-09-23T23:31:40.549 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 260.809018342, 2023-09-23T23:31:40.549 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2021504429999998, 2023-09-23T23:31:40.549 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.549 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.549 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.549 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:31:40.549 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:31:40.549 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178622 2023-09-23T23:31:40.549 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.549 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.549 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.550 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591902+0000", 2023-09-23T23:31:40.550 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.550 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.550 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591902+0000", 2023-09-23T23:31:40.550 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.550 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.550 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591901+0000", 2023-09-23T23:31:40.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:31:40.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591906+0000", 2023-09-23T23:31:40.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8289999999999996e-06 2023-09-23T23:31:40.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591921+0000", 2023-09-23T23:31:40.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4149e-05 2023-09-23T23:31:40.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591928+0000", 2023-09-23T23:31:40.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0999999999999998e-06 2023-09-23T23:31:40.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.592603+0000", 2023-09-23T23:31:40.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00067509500000000001 2023-09-23T23:31:40.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.592613+0000", 2023-09-23T23:31:40.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0671e-05 2023-09-23T23:31:40.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.553 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.553 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542776+0000", 2023-09-23T23:31:40.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95016243199999995 2023-09-23T23:31:40.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.553 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.553 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542784+0000", 2023-09-23T23:31:40.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5580000000000001e-06 2023-09-23T23:31:40.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.554 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.554 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764299+0000", 2023-09-23T23:31:40.554 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2215145999999999 2023-09-23T23:31:40.554 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.554 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.554 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.554 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764438+0000", 2023-09-23T23:31:40.554 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000138536 2023-09-23T23:31:40.554 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.554 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.554 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T23:31:40.554 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.765798+0000", 2023-09-23T23:31:40.555 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001360874 2023-09-23T23:31:40.555 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.555 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.555 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.774753+0000", 2023-09-23T23:31:40.555 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0089545679999999996 2023-09-23T23:31:40.555 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.555 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.555 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783698+0000", 2023-09-23T23:31:40.555 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0089451630000000008 2023-09-23T23:31:40.555 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.556 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.785057+0000", 2023-09-23T23:31:40.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001358784 2023-09-23T23:31:40.556 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.556 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793921+0000", 2023-09-23T23:31:40.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0088636380000000001 2023-09-23T23:31:40.556 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.556 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793959+0000", 2023-09-23T23:31:40.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8328999999999998e-05 2023-09-23T23:31:40.557 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.557 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.557 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.557 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.794052+0000", 2023-09-23T23:31:40.557 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3539e-05 2023-09-23T23:31:40.557 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.557 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.557 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.557 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.557 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.557 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178633 11.16 11:6db526fe:::benchmark_data_smithi110_433746_object72753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:31:40.557 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.633733+0000", 2023-09-23T23:31:40.557 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 260.76718700700002, 2023-09-23T23:31:40.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.160140325, 2023-09-23T23:31:40.558 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.558 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.558 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.558 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:31:40.558 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:31:40.558 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178633 2023-09-23T23:31:40.558 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.558 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.558 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.558 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633733+0000", 2023-09-23T23:31:40.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633733+0000", 2023-09-23T23:31:40.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633732+0000", 2023-09-23T23:31:40.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:31:40.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.560 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.560 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633739+0000", 2023-09-23T23:31:40.560 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2899999999999997e-06 2023-09-23T23:31:40.560 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.560 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.560 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.560 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633756+0000", 2023-09-23T23:31:40.560 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6368999999999999e-05 2023-09-23T23:31:40.560 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.560 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.560 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.560 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633763+0000", 2023-09-23T23:31:40.560 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8930000000000005e-06 2023-09-23T23:31:40.561 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.561 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.561 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.561 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633828+0000", 2023-09-23T23:31:40.561 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4092999999999998e-05 2023-09-23T23:31:40.561 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.561 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.561 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.561 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633843+0000", 2023-09-23T23:31:40.561 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5512e-05 2023-09-23T23:31:40.561 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.561 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.561 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542670+0000", 2023-09-23T23:31:40.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.90882660500000001 2023-09-23T23:31:40.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.562 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542679+0000", 2023-09-23T23:31:40.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0000000000000002e-06 2023-09-23T23:31:40.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.562 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760612+0000", 2023-09-23T23:31:40.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.217933355 2023-09-23T23:31:40.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760705+0000", 2023-09-23T23:31:40.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3078999999999994e-05 2023-09-23T23:31:40.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T23:31:40.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.761116+0000", 2023-09-23T23:31:40.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00041080000000000001 2023-09-23T23:31:40.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764673+0000", 2023-09-23T23:31:40.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035569239999999999 2023-09-23T23:31:40.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.767908+0000", 2023-09-23T23:31:40.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032356749999999999 2023-09-23T23:31:40.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783437+0000", 2023-09-23T23:31:40.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015528767000000001 2023-09-23T23:31:40.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.565 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.565 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793721+0000", 2023-09-23T23:31:40.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010284188 2023-09-23T23:31:40.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.565 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.565 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793766+0000", 2023-09-23T23:31:40.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4631e-05 2023-09-23T23:31:40.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.566 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.566 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793874+0000", 2023-09-23T23:31:40.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010767100000000001 2023-09-23T23:31:40.566 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.566 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.566 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.566 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.566 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.566 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178629 11.7 11:e3eb7c4c:::benchmark_data_smithi110_433746_object72749:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e425)", 2023-09-23T23:31:40.566 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:20.543856+0000", 2023-09-23T23:31:40.566 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 259.85706461299998, 2023-09-23T23:31:40.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2512707000000001, 2023-09-23T23:31:40.566 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.567 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.567 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.567 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:31:40.567 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:31:40.567 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178629 2023-09-23T23:31:40.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.567 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.567 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.567 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543856+0000", 2023-09-23T23:31:40.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543856+0000", 2023-09-23T23:31:40.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543855+0000", 2023-09-23T23:31:40.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:31:40.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543857+0000", 2023-09-23T23:31:40.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.99e-06 2023-09-23T23:31:40.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543862+0000", 2023-09-23T23:31:40.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.899e-06 2023-09-23T23:31:40.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543867+0000", 2023-09-23T23:31:40.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3710000000000002e-06 2023-09-23T23:31:40.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.570 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.544825+0000", 2023-09-23T23:31:40.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000957864 2023-09-23T23:31:40.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.570 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.544832+0000", 2023-09-23T23:31:40.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3649999999999998e-06 2023-09-23T23:31:40.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.570 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.780766+0000", 2023-09-23T23:31:40.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2359338689999999 2023-09-23T23:31:40.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.780888+0000", 2023-09-23T23:31:40.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000121833 2023-09-23T23:31:40.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:31:40.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.781338+0000", 2023-09-23T23:31:40.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045035600000000001 2023-09-23T23:31:40.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.784204+0000", 2023-09-23T23:31:40.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002865884 2023-09-23T23:31:40.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.785473+0000", 2023-09-23T23:31:40.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001268838 2023-09-23T23:31:40.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.788590+0000", 2023-09-23T23:31:40.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031171839999999998 2023-09-23T23:31:40.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.794983+0000", 2023-09-23T23:31:40.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063934339999999999 2023-09-23T23:31:40.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.795024+0000", 2023-09-23T23:31:40.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0828000000000003e-05 2023-09-23T23:31:40.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.795126+0000", 2023-09-23T23:31:40.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102277 2023-09-23T23:31:40.574 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.574 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.574 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.574 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211753 11.19 11:9cef2dd2:::benchmark_data_smithi110_433746_object105873:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T23:31:40.574 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:24.737706+0000", 2023-09-23T23:31:40.574 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 195.66321419499999, 2023-09-23T23:31:40.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.967897013, 2023-09-23T23:31:40.574 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:40.575 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:31:40.575 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:40.575 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:31:40.575 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:31:40.575 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211753 2023-09-23T23:31:40.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.575 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:40.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.575 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:40.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737706+0000", 2023-09-23T23:31:40.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.576 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:40.576 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737706+0000", 2023-09-23T23:31:40.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:40.576 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.576 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:40.576 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737704+0000", 2023-09-23T23:31:40.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:31:40.576 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.576 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:40.576 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737713+0000", 2023-09-23T23:31:40.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3939999999999996e-06 2023-09-23T23:31:40.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:40.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737727+0000", 2023-09-23T23:31:40.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3893999999999999e-05 2023-09-23T23:31:40.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:40.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737736+0000", 2023-09-23T23:31:40.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0780000000000002e-06 2023-09-23T23:31:40.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737803+0000", 2023-09-23T23:31:40.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6959999999999996e-05 2023-09-23T23:31:40.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737815+0000", 2023-09-23T23:31:40.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2377e-05 2023-09-23T23:31:40.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:25.604895+0000", 2023-09-23T23:31:40.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86708047300000002 2023-09-23T23:31:40.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:31:40.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:25.604906+0000", 2023-09-23T23:31:40.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0903999999999999e-05 2023-09-23T23:31:40.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:40.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.701468+0000", 2023-09-23T23:31:40.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.096561801 2023-09-23T23:31:40.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:31:40.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.701626+0000", 2023-09-23T23:31:40.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015756900000000001 2023-09-23T23:31:40.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,3,7", 2023-09-23T23:31:40.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.701760+0000", 2023-09-23T23:31:40.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013463700000000001 2023-09-23T23:31:40.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.704681+0000", 2023-09-23T23:31:40.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029204080000000002 2023-09-23T23:31:40.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:31:40.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.704693+0000", 2023-09-23T23:31:40.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2427e-05 2023-09-23T23:31:40.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.704736+0000", 2023-09-23T23:31:40.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2911000000000003e-05 2023-09-23T23:31:40.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.704783+0000", 2023-09-23T23:31:40.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6931999999999997e-05 2023-09-23T23:31:40.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:31:40.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.705399+0000", 2023-09-23T23:31:40.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000615889 2023-09-23T23:31:40.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:31:40.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.705458+0000", 2023-09-23T23:31:40.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8731999999999999e-05 2023-09-23T23:31:40.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:40.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:40.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:31:40.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.705603+0000", 2023-09-23T23:31:40.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014554600000000001 2023-09-23T23:31:40.583 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.583 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.583 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.583 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:40.583 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:40.583 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:31:40.584 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:31:40.618 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:40.701 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:31:40.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:40.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:31:40.779 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:40.814 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:31:40.931 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:41.135 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:41 smithi110 ceph-mon[212939]: pgmap v1432: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 968 op/s; 14514/106074 objects degraded (13.683%); 8000/106074 objects misplaced (7.542%); 21 KiB/s, 120 objects/s recovering 2023-09-23T23:31:41.135 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:41 smithi110 ceph-mon[212939]: 12.12 scrub starts 2023-09-23T23:31:41.135 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:41 smithi110 ceph-mon[221102]: pgmap v1432: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 968 op/s; 14514/106074 objects degraded (13.683%); 8000/106074 objects misplaced (7.542%); 21 KiB/s, 120 objects/s recovering 2023-09-23T23:31:41.135 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:41 smithi110 ceph-mon[221102]: 12.12 scrub starts 2023-09-23T23:31:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:41 smithi138 ceph-mon[193606]: pgmap v1432: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 968 op/s; 14514/106074 objects degraded (13.683%); 8000/106074 objects misplaced (7.542%); 21 KiB/s, 120 objects/s recovering 2023-09-23T23:31:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:41 smithi138 ceph-mon[193606]: 12.12 scrub starts 2023-09-23T23:31:41.793 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:31:41.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:41.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:31:41.909 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:31:41.938 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:31:41.987 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:42.125 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:31:42.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:42.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:31:42.208 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T23:31:42.234 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:31:42.266 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:42.291 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:31:42.321 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:31:42.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:42.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:31:42.330 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:42 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2611999363' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:31:42.330 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:42 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2611999363' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:31:42.430 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:42.462 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:42 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2611999363' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:31:42.467 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:42.536 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:42.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:42.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:42.645 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:31:42.737 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:43.312 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:31:43.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:43.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:31:43.422 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:43 smithi138 ceph-mon[193606]: pgmap v1433: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 14514/103230 objects degraded (14.060%); 7591/103230 objects misplaced (7.353%); 3.6 MiB/s, 71 objects/s recovering 2023-09-23T23:31:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:43 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2611999363' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:31:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:43 smithi138 ceph-mon[193606]: osdmap e519: 8 total, 8 up, 8 in 2023-09-23T23:31:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:43 smithi138 ceph-mon[193606]: failed to encode map e519 with expected crc 2023-09-23T23:31:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:43 smithi138 ceph-mon[193606]: failed to encode map e519 with expected crc 2023-09-23T23:31:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:31:43.462 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:43.531 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:43.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:43.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:43.638 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:43.676 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:43.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:43 smithi110 ceph-mon[212939]: pgmap v1433: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 14514/103230 objects degraded (14.060%); 7591/103230 objects misplaced (7.353%); 3.6 MiB/s, 71 objects/s recovering 2023-09-23T23:31:43.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:43 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2611999363' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:31:43.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:43 smithi110 ceph-mon[212939]: osdmap e519: 8 total, 8 up, 8 in 2023-09-23T23:31:43.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:43 smithi110 ceph-mon[212939]: failed to encode map e519 with expected crc 2023-09-23T23:31:43.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:43 smithi110 ceph-mon[212939]: failed to encode map e519 with expected crc 2023-09-23T23:31:43.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:31:43.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:43 smithi110 ceph-mon[221102]: pgmap v1433: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 14514/103230 objects degraded (14.060%); 7591/103230 objects misplaced (7.353%); 3.6 MiB/s, 71 objects/s recovering 2023-09-23T23:31:43.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:43 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2611999363' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:31:43.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:43 smithi110 ceph-mon[221102]: osdmap e519: 8 total, 8 up, 8 in 2023-09-23T23:31:43.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:43 smithi110 ceph-mon[221102]: failed to encode map e519 with expected crc 2023-09-23T23:31:43.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:43 smithi110 ceph-mon[221102]: failed to encode map e519 with expected crc 2023-09-23T23:31:43.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:31:43.748 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:43.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:43.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:43.862 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:43.888 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:43.931 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:43.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:43.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:44.036 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:44.061 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:44.103 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:44.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:44.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:44.208 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:44.263 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:44.329 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:44.333 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:44 smithi138 ceph-mon[193606]: 12.3 scrub starts 2023-09-23T23:31:44.333 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:44 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 14514/103230 objects degraded (14.060%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:44.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:44.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:44.435 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:31:44.491 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:44 smithi110 ceph-mon[221102]: 12.3 scrub starts 2023-09-23T23:31:44.491 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:44 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 14514/103230 objects degraded (14.060%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:44.491 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:44 smithi110 ceph-mon[212939]: 12.3 scrub starts 2023-09-23T23:31:44.491 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:44 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 14514/103230 objects degraded (14.060%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:44.499 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:44.657 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T23:31:44.797 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:31:44.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:44.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:31:44.894 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:44.912 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:44.946 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:44.948 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:45.000 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:45.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:45.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:45.106 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:45.139 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:45.202 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:45.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:45.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:45.310 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:31:45.353 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:45.431 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:31:45.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:45.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:31:45.543 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:31:45.557 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:45 smithi110 ceph-mon[221102]: pgmap v1435: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 14514/101925 objects degraded (14.240%); 7591/101925 objects misplaced (7.448%); 3.3 MiB/s, 65 objects/s recovering 2023-09-23T23:31:45.557 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:45 smithi110 ceph-mon[221102]: 12.0 scrub starts 2023-09-23T23:31:45.557 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:45 smithi110 ceph-mon[212939]: pgmap v1435: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 14514/101925 objects degraded (14.240%); 7591/101925 objects misplaced (7.448%); 3.3 MiB/s, 65 objects/s recovering 2023-09-23T23:31:45.557 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:45 smithi110 ceph-mon[212939]: 12.0 scrub starts 2023-09-23T23:31:45.570 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:45 smithi138 ceph-mon[193606]: pgmap v1435: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 14514/101925 objects degraded (14.240%); 7591/101925 objects misplaced (7.448%); 3.3 MiB/s, 65 objects/s recovering 2023-09-23T23:31:45.570 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:45 smithi138 ceph-mon[193606]: 12.0 scrub starts 2023-09-23T23:31:45.579 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:45.641 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:31:45.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:45.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:31:45.751 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:31:45.863 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:46.262 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:31:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:46 smithi110 ceph-mon[212939]: 12.10 scrub starts 2023-09-23T23:31:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:46 smithi110 ceph-mon[212939]: 12.13 scrub starts 2023-09-23T23:31:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:46 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1017420500' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:46 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:46.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:46 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:31:46.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:46 smithi110 ceph-mon[221102]: 12.10 scrub starts 2023-09-23T23:31:46.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:46 smithi110 ceph-mon[221102]: 12.13 scrub starts 2023-09-23T23:31:46.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:46 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1017420500' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:46.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:46 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:46.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:46 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:31:46.516 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:31:46.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:46.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:31:46.628 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:31:46.691 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:46.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:46 smithi138 ceph-mon[193606]: 12.10 scrub starts 2023-09-23T23:31:46.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:46 smithi138 ceph-mon[193606]: 12.13 scrub starts 2023-09-23T23:31:46.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:46 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1017420500' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:46.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:46 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:31:46.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:46 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:31:47.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:31:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:31:46] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.43.0" 2023-09-23T23:31:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:47 smithi138 ceph-mon[193606]: pgmap v1436: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 14453/92880 objects degraded (15.561%); 7053/92880 objects misplaced (7.594%); 5.2 MiB/s, 101 objects/s recovering 2023-09-23T23:31:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:47 smithi138 ceph-mon[193606]: 12.10 scrub starts 2023-09-23T23:31:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:47 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:31:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:47 smithi138 ceph-mon[193606]: failed to encode map e520 with expected crc 2023-09-23T23:31:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:47 smithi138 ceph-mon[193606]: failed to encode map e520 with expected crc 2023-09-23T23:31:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:47 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:31:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:47 smithi138 ceph-mon[193606]: osdmap e520: 8 total, 8 up, 8 in 2023-09-23T23:31:47.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:47 smithi110 ceph-mon[212939]: pgmap v1436: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 14453/92880 objects degraded (15.561%); 7053/92880 objects misplaced (7.594%); 5.2 MiB/s, 101 objects/s recovering 2023-09-23T23:31:47.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:47 smithi110 ceph-mon[212939]: 12.10 scrub starts 2023-09-23T23:31:47.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:47 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:31:47.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:47 smithi110 ceph-mon[212939]: failed to encode map e520 with expected crc 2023-09-23T23:31:47.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:47 smithi110 ceph-mon[212939]: failed to encode map e520 with expected crc 2023-09-23T23:31:47.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:47 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:31:47.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:47 smithi110 ceph-mon[212939]: osdmap e520: 8 total, 8 up, 8 in 2023-09-23T23:31:47.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:47 smithi110 ceph-mon[221102]: pgmap v1436: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 14453/92880 objects degraded (15.561%); 7053/92880 objects misplaced (7.594%); 5.2 MiB/s, 101 objects/s recovering 2023-09-23T23:31:47.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:47 smithi110 ceph-mon[221102]: 12.10 scrub starts 2023-09-23T23:31:47.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:47 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:31:47.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:47 smithi110 ceph-mon[221102]: failed to encode map e520 with expected crc 2023-09-23T23:31:47.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:47 smithi110 ceph-mon[221102]: failed to encode map e520 with expected crc 2023-09-23T23:31:47.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:47 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:31:47.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:47 smithi110 ceph-mon[221102]: osdmap e520: 8 total, 8 up, 8 in 2023-09-23T23:31:49.013 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:49.136 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:31:49.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:49.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:31:49.259 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:31:49.273 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:49 smithi110 ceph-mon[221102]: pgmap v1438: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 14453/92556 objects degraded (15.615%); 7053/92556 objects misplaced (7.620%); 5.7 MiB/s, 98 objects/s recovering 2023-09-23T23:31:49.274 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:49 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 14453/92556 objects degraded (15.615%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:49.362 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:49.626 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:49 smithi110 ceph-mon[212939]: pgmap v1438: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 14453/92556 objects degraded (15.615%); 7053/92556 objects misplaced (7.620%); 5.7 MiB/s, 98 objects/s recovering 2023-09-23T23:31:49.626 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:49 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 14453/92556 objects degraded (15.615%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:49.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:49 smithi138 ceph-mon[193606]: pgmap v1438: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 14453/92556 objects degraded (15.615%); 7053/92556 objects misplaced (7.620%); 5.7 MiB/s, 98 objects/s recovering 2023-09-23T23:31:49.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:49 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 14453/92556 objects degraded (15.615%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:50.041 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:31:50.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:50.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:31:50.149 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:31:50.224 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:50.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:50 smithi110 ceph-mon[212939]: 12.3 scrub starts 2023-09-23T23:31:50.299 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:50 smithi110 ceph-mon[212939]: 12.2 scrub starts 2023-09-23T23:31:50.299 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:50 smithi110 ceph-mon[221102]: 12.3 scrub starts 2023-09-23T23:31:50.300 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:50 smithi110 ceph-mon[221102]: 12.2 scrub starts 2023-09-23T23:31:50.477 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:31:50.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:50.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:31:50.584 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:31:50.632 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:50 smithi138 ceph-mon[193606]: 12.3 scrub starts 2023-09-23T23:31:50.632 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:50 smithi138 ceph-mon[193606]: 12.2 scrub starts 2023-09-23T23:31:50.643 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:50.736 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:31:50.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:50.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:31:50.847 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:50.900 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:50.966 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:50.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:50.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:51.071 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:31:51.073 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:31:51.184 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:31:51.185 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:31:51.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.185 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:163759 12.11 12.27d2c8f1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-09-23T23:31:51.185 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:31:51.178578+0000", 2023-09-23T23:31:51.185 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.000240196, 2023-09-23T23:31:51.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026657100000000001, 2023-09-23T23:31:51.185 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:51.185 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:31:51.185 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:51.185 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:31:51.186 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:31:51.186 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 163759 2023-09-23T23:31:51.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.186 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:51.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:51.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178578+0000", 2023-09-23T23:31:51.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:51.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:51.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178578+0000", 2023-09-23T23:31:51.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:51.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:51.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178576+0000", 2023-09-23T23:31:51.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:31:51.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:51.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178587+0000", 2023-09-23T23:31:51.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1011e-05 2023-09-23T23:31:51.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:51.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178611+0000", 2023-09-23T23:31:51.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3926000000000001e-05 2023-09-23T23:31:51.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:51.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178621+0000", 2023-09-23T23:31:51.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0061000000000001e-05 2023-09-23T23:31:51.188 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:51.188 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:51.189 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:51.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.189 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:163761 12.16 12.ee1f73d6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-09-23T23:31:51.189 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:31:51.178638+0000", 2023-09-23T23:31:51.189 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00018044299999999999, 2023-09-23T23:31:51.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000282687, 2023-09-23T23:31:51.189 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:51.189 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:31:51.189 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:51.189 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:31:51.189 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:31:51.190 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 163761 2023-09-23T23:31:51.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.190 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:51.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:51.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178638+0000", 2023-09-23T23:31:51.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:51.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:51.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178638+0000", 2023-09-23T23:31:51.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:51.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:51.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178638+0000", 2023-09-23T23:31:51.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:31:51.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:51.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178641+0000", 2023-09-23T23:31:51.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.943e-06 2023-09-23T23:31:51.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:51.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178646+0000", 2023-09-23T23:31:51.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6869999999999999e-06 2023-09-23T23:31:51.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:51.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178651+0000", 2023-09-23T23:31:51.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.865e-06 2023-09-23T23:31:51.192 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:51.192 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:51.192 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:51.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.193 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:163765 12.7 12:e76d16df:::benchmark_data_smithi110_491193_object68890:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-09-23T23:31:51.193 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:31:51.178662+0000", 2023-09-23T23:31:51.193 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00015609700000000001, 2023-09-23T23:31:51.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030432400000000002, 2023-09-23T23:31:51.193 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:51.193 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:31:51.193 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:31:51.193 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:31:51.193 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:31:51.193 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 163765 2023-09-23T23:31:51.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.193 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:51.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:51.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178662+0000", 2023-09-23T23:31:51.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:51.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:51.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178662+0000", 2023-09-23T23:31:51.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:51.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:51.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178662+0000", 2023-09-23T23:31:51.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:31:51.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:51.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178665+0000", 2023-09-23T23:31:51.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.661e-06 2023-09-23T23:31:51.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:51.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178670+0000", 2023-09-23T23:31:51.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0889999999999999e-06 2023-09-23T23:31:51.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:51.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178674+0000", 2023-09-23T23:31:51.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4630000000000003e-06 2023-09-23T23:31:51.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:31:51.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178848+0000", 2023-09-23T23:31:51.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017354499999999999 2023-09-23T23:31:51.196 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:51.196 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:51.196 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:51.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.197 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGRecoveryDeleteReply(12.5 e520,496 12:a77d9c6b:::benchmark_data_smithi110_491193_object15349:head,503'5871)", 2023-09-23T23:31:51.197 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:31:51.178700+0000", 2023-09-23T23:31:51.197 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00011863500000000001, 2023-09-23T23:31:51.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033513500000000002, 2023-09-23T23:31:51.197 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:51.197 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:31:51.197 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:51.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:51.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178700+0000", 2023-09-23T23:31:51.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:51.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:51.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178700+0000", 2023-09-23T23:31:51.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:51.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:51.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178699+0000", 2023-09-23T23:31:51.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:31:51.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:51.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178708+0000", 2023-09-23T23:31:51.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3040000000000005e-06 2023-09-23T23:31:51.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:51.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178716+0000", 2023-09-23T23:31:51.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7329999999999992e-06 2023-09-23T23:31:51.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:51.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178722+0000", 2023-09-23T23:31:51.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8510000000000004e-06 2023-09-23T23:31:51.199 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:51.199 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:51.200 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:51.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.200 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:163764 12.d e520/476)", 2023-09-23T23:31:51.200 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:31:51.178743+0000", 2023-09-23T23:31:51.200 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 7.5975000000000004e-05, 2023-09-23T23:31:51.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033746200000000002, 2023-09-23T23:31:51.200 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:31:51.200 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:31:51.200 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:31:51.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:31:51.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178743+0000", 2023-09-23T23:31:51.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:51.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:31:51.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178743+0000", 2023-09-23T23:31:51.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:31:51.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:31:51.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178742+0000", 2023-09-23T23:31:51.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:31:51.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:31:51.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178750+0000", 2023-09-23T23:31:51.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3710000000000002e-06 2023-09-23T23:31:51.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:31:51.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178759+0000", 2023-09-23T23:31:51.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2269999999999999e-06 2023-09-23T23:31:51.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:31:51.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:31:51.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:31:51.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:31:51.178763+0000", 2023-09-23T23:31:51.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4519999999999999e-06 2023-09-23T23:31:51.203 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:51.203 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:31:51.203 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:51.203 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:31:51.203 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-09-23T23:31:51.203 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 5 2023-09-23T23:31:51.203 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:31:51.228 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:31:51.309 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:31:51.310 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:51.363 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:31:51.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:51 smithi110 ceph-mon[221102]: pgmap v1439: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 14453/84975 objects degraded (17.009%); 6843/84975 objects misplaced (8.053%); 2.5 MiB/s, 47 objects/s recovering 2023-09-23T23:31:51.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:51 smithi110 ceph-mon[221102]: 12.10 deep-scrub starts 2023-09-23T23:31:51.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:51 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:31:51.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:51 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:31:51.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:51 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3531235059' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:31:51.381 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:51 smithi110 ceph-mon[212939]: pgmap v1439: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 14453/84975 objects degraded (17.009%); 6843/84975 objects misplaced (8.053%); 2.5 MiB/s, 47 objects/s recovering 2023-09-23T23:31:51.381 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:51 smithi110 ceph-mon[212939]: 12.10 deep-scrub starts 2023-09-23T23:31:51.381 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:51 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:31:51.381 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:51 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:31:51.382 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:51 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3531235059' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:31:51.412 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:31:51.473 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:31:51.518 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:31:51.560 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:31:51.594 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:31:51.595 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:31:51.596 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:31:51.596 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:31:51.596 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:31:51.596 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:31:51.596 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:31:51.596 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:31:51.596 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:31:51.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:51 smithi138 ceph-mon[193606]: pgmap v1439: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 14453/84975 objects degraded (17.009%); 6843/84975 objects misplaced (8.053%); 2.5 MiB/s, 47 objects/s recovering 2023-09-23T23:31:51.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:51 smithi138 ceph-mon[193606]: 12.10 deep-scrub starts 2023-09-23T23:31:51.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:51 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:31:51.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:51 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:31:51.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:51 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3531235059' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:31:52.404 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:31:52.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:52.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:31:52.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:52 smithi110 ceph-mon[221102]: 12.10 deep-scrub starts 2023-09-23T23:31:52.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:52 smithi110 ceph-mon[212939]: 12.10 deep-scrub starts 2023-09-23T23:31:52.513 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:31:52.605 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:52.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:52 smithi138 ceph-mon[193606]: 12.10 deep-scrub starts 2023-09-23T23:31:53.514 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:31:53.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:53.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:31:53.616 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:53 smithi110 ceph-mon[221102]: pgmap v1440: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 14099/82011 objects degraded (17.192%); 6843/82011 objects misplaced (8.344%); 2.0 MiB/s, 75 objects/s recovering 2023-09-23T23:31:53.617 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:53 smithi110 ceph-mon[221102]: 12.12 deep-scrub starts 2023-09-23T23:31:53.618 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:53 smithi110 ceph-mon[212939]: pgmap v1440: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 14099/82011 objects degraded (17.192%); 6843/82011 objects misplaced (8.344%); 2.0 MiB/s, 75 objects/s recovering 2023-09-23T23:31:53.618 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:53 smithi110 ceph-mon[212939]: 12.12 deep-scrub starts 2023-09-23T23:31:53.625 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:53.654 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:53 smithi138 ceph-mon[193606]: pgmap v1440: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 14099/82011 objects degraded (17.192%); 6843/82011 objects misplaced (8.344%); 2.0 MiB/s, 75 objects/s recovering 2023-09-23T23:31:53.655 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:53 smithi138 ceph-mon[193606]: 12.12 deep-scrub starts 2023-09-23T23:31:53.658 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:53.721 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:53.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:53.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:53.761 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:31:53.827 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:31:53.857 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:53.912 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:31:53.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:53.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:31:54.019 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:54.056 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:54.105 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:54.110 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:54.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:54.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:54.216 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:31:54.322 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:54.474 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:31:54.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:54.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:31:54.579 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:54 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 14099/82011 objects degraded (17.192%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:54.579 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:54 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2825085847' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:31:54.580 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:54 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:31:54.583 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:54 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 14099/82011 objects degraded (17.192%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:54.583 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:54 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2825085847' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:31:54.583 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:54 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:31:54.587 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:54.629 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:54 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 14099/82011 objects degraded (17.192%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:54.629 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:54 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2825085847' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:31:54.629 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:54 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:31:54.632 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:54.697 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:54.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:54.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:54.725 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T23:31:54.812 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:54.847 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:54.901 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:54.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:54.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:55.006 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:55.040 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:55.098 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:55.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:55.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:55.205 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:31:55.277 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:55.369 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:55 smithi110 ceph-mon[212939]: pgmap v1441: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 14099/80646 objects degraded (17.483%); 6843/80646 objects misplaced (8.485%); 2.0 MiB/s, 72 objects/s recovering 2023-09-23T23:31:55.369 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:55 smithi110 ceph-mon[212939]: 12.10 scrub starts 2023-09-23T23:31:55.369 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:55 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:31:55.370 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:55 smithi110 ceph-mon[212939]: osdmap e521: 8 total, 8 up, 8 in 2023-09-23T23:31:55.370 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:55 smithi110 ceph-mon[212939]: failed to encode map e521 with expected crc 2023-09-23T23:31:55.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:55 smithi138 ceph-mon[193606]: pgmap v1441: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 14099/80646 objects degraded (17.483%); 6843/80646 objects misplaced (8.485%); 2.0 MiB/s, 72 objects/s recovering 2023-09-23T23:31:55.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:55 smithi138 ceph-mon[193606]: 12.10 scrub starts 2023-09-23T23:31:55.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:55 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:31:55.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:55 smithi138 ceph-mon[193606]: osdmap e521: 8 total, 8 up, 8 in 2023-09-23T23:31:55.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:55 smithi138 ceph-mon[193606]: failed to encode map e521 with expected crc 2023-09-23T23:31:55.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:55 smithi110 ceph-mon[221102]: pgmap v1441: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 14099/80646 objects degraded (17.483%); 6843/80646 objects misplaced (8.485%); 2.0 MiB/s, 72 objects/s recovering 2023-09-23T23:31:55.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:55 smithi110 ceph-mon[221102]: 12.10 scrub starts 2023-09-23T23:31:55.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:55 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:31:55.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:55 smithi110 ceph-mon[221102]: osdmap e521: 8 total, 8 up, 8 in 2023-09-23T23:31:55.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:55 smithi110 ceph-mon[221102]: failed to encode map e521 with expected crc 2023-09-23T23:31:55.896 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:31:56.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:56 smithi138 ceph-mon[193606]: failed to encode map e521 with expected crc 2023-09-23T23:31:56.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:56 smithi138 ceph-mon[193606]: failed to encode map e521 with expected crc 2023-09-23T23:31:56.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:56 smithi138 ceph-mon[193606]: 12.13 scrub starts 2023-09-23T23:31:56.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:56 smithi110 ceph-mon[212939]: failed to encode map e521 with expected crc 2023-09-23T23:31:56.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:56 smithi110 ceph-mon[212939]: failed to encode map e521 with expected crc 2023-09-23T23:31:56.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:56 smithi110 ceph-mon[212939]: 12.13 scrub starts 2023-09-23T23:31:56.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:56 smithi110 ceph-mon[221102]: failed to encode map e521 with expected crc 2023-09-23T23:31:56.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:56 smithi110 ceph-mon[221102]: failed to encode map e521 with expected crc 2023-09-23T23:31:56.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:56 smithi110 ceph-mon[221102]: 12.13 scrub starts 2023-09-23T23:31:56.817 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:31:56.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:56.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:31:56.929 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:31:56.994 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:57.134 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:31:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:31:56] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.43.0" 2023-09-23T23:31:57.226 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T23:31:57.240 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:57.269 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:31:57.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:57.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:31:57.376 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:57.406 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:57.412 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:31:57.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:57 smithi110 ceph-mon[212939]: pgmap v1443: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 13638/71298 objects degraded (19.128%); 6598/71298 objects misplaced (9.254%); 0 B/s, 85 objects/s recovering 2023-09-23T23:31:57.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:57 smithi110 ceph-mon[212939]: 12.0 scrub starts 2023-09-23T23:31:57.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:57 smithi110 ceph-mon[221102]: pgmap v1443: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 13638/71298 objects degraded (19.128%); 6598/71298 objects misplaced (9.254%); 0 B/s, 85 objects/s recovering 2023-09-23T23:31:57.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:57 smithi110 ceph-mon[221102]: 12.0 scrub starts 2023-09-23T23:31:57.460 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:57.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:57.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:57.568 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:57.616 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:57.689 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:57.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:57 smithi138 ceph-mon[193606]: pgmap v1443: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 13638/71298 objects degraded (19.128%); 6598/71298 objects misplaced (9.254%); 0 B/s, 85 objects/s recovering 2023-09-23T23:31:57.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:57 smithi138 ceph-mon[193606]: 12.0 scrub starts 2023-09-23T23:31:57.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:57.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:57.800 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:31:57.833 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:57.893 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:31:57.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:57.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:31:57.999 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:31:58.129 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:58.332 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:31:58.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:58.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:31:58.447 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:31:58.482 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:58.551 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:31:58.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:58.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:31:58.662 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:31:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:58 smithi138 ceph-mon[193606]: 12.10 scrub starts 2023-09-23T23:31:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:58 smithi138 ceph-mon[193606]: 12.3 scrub starts 2023-09-23T23:31:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:58 smithi138 ceph-mon[193606]: 12.2 scrub starts 2023-09-23T23:31:58.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:31:58.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:31:58.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:58 smithi110 ceph-mon[212939]: 12.10 scrub starts 2023-09-23T23:31:58.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:58 smithi110 ceph-mon[212939]: 12.3 scrub starts 2023-09-23T23:31:58.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:58 smithi110 ceph-mon[212939]: 12.2 scrub starts 2023-09-23T23:31:58.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:31:58.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:31:58.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:58 smithi110 ceph-mon[221102]: 12.10 scrub starts 2023-09-23T23:31:58.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:58 smithi110 ceph-mon[221102]: 12.3 scrub starts 2023-09-23T23:31:58.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:58 smithi110 ceph-mon[221102]: 12.2 scrub starts 2023-09-23T23:31:58.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:31:58.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:31:58.723 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:58.792 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:31:58.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:58.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:31:58.899 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:31:58.944 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:59.021 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:31:59.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:59.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:31:59.135 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:31:59.162 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:59.206 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:31:59.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:31:59.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:31:59.312 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:31:59.400 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:59 smithi138 ceph-mon[193606]: pgmap v1444: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 13638/70974 objects degraded (19.215%); 6598/70974 objects misplaced (9.296%); 0 B/s, 81 objects/s recovering 2023-09-23T23:31:59.400 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:59 smithi138 ceph-mon[193606]: 12.13 deep-scrub starts 2023-09-23T23:31:59.400 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:59 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1947363175' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:31:59.400 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:59 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:31:59.400 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:31:59 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 13638/70974 objects degraded (19.215%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:59.415 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:31:59.424 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:31:59.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:59 smithi110 ceph-mon[212939]: pgmap v1444: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 13638/70974 objects degraded (19.215%); 6598/70974 objects misplaced (9.296%); 0 B/s, 81 objects/s recovering 2023-09-23T23:31:59.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:59 smithi110 ceph-mon[212939]: 12.13 deep-scrub starts 2023-09-23T23:31:59.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:59 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1947363175' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:31:59.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:59 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:31:59.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:31:59 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 13638/70974 objects degraded (19.215%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:31:59.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:59 smithi110 ceph-mon[221102]: pgmap v1444: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 13638/70974 objects degraded (19.215%); 6598/70974 objects misplaced (9.296%); 0 B/s, 81 objects/s recovering 2023-09-23T23:31:59.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:59 smithi110 ceph-mon[221102]: 12.13 deep-scrub starts 2023-09-23T23:31:59.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:59 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1947363175' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:31:59.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:59 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:31:59.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:31:59 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 13638/70974 objects degraded (19.215%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:00.187 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:32:00.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:00.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:32:00.293 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:32:00.320 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:00.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:00 smithi110 ceph-mon[212939]: 12.12 scrub starts 2023-09-23T23:32:00.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:00 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:32:00.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:00 smithi110 ceph-mon[212939]: failed to encode map e522 with expected crc 2023-09-23T23:32:00.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:00 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:32:00.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:00 smithi110 ceph-mon[212939]: osdmap e522: 8 total, 8 up, 8 in 2023-09-23T23:32:00.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:00 smithi110 ceph-mon[221102]: 12.12 scrub starts 2023-09-23T23:32:00.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:00 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:32:00.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:00 smithi110 ceph-mon[221102]: failed to encode map e522 with expected crc 2023-09-23T23:32:00.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:00 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:32:00.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:00 smithi110 ceph-mon[221102]: osdmap e522: 8 total, 8 up, 8 in 2023-09-23T23:32:00.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:00 smithi138 ceph-mon[193606]: 12.12 scrub starts 2023-09-23T23:32:00.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:00 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:32:00.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:00 smithi138 ceph-mon[193606]: failed to encode map e522 with expected crc 2023-09-23T23:32:00.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:00 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:32:00.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:00 smithi138 ceph-mon[193606]: osdmap e522: 8 total, 8 up, 8 in 2023-09-23T23:32:01.020 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:32:01.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:01.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:32:01.130 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:32:01.205 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:01.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:01 smithi110 ceph-mon[212939]: pgmap v1446: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 13638/63231 objects degraded (21.569%); 6358/63231 objects misplaced (10.055%); 0 B/s, 57 objects/s recovering 2023-09-23T23:32:01.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:01 smithi110 ceph-mon[221102]: pgmap v1446: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 13638/63231 objects degraded (21.569%); 6358/63231 objects misplaced (10.055%); 0 B/s, 57 objects/s recovering 2023-09-23T23:32:01.617 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:01.677 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:32:01.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:01.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:32:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:01 smithi138 ceph-mon[193606]: pgmap v1446: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 13638/63231 objects degraded (21.569%); 6358/63231 objects misplaced (10.055%); 0 B/s, 57 objects/s recovering 2023-09-23T23:32:01.789 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:32:01.865 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:02.045 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:32:02.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:02.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:32:02.153 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:02.193 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:02.229 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:32:02.229 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:32:02.242 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:32:02.267 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:02.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:02.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:02.293 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:32:02.361 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:32:02.371 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:32:02.404 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:02.406 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:32:02.439 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:32:02.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:02 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2517265345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:32:02.457 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:32:02.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:02.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:32:02.472 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:32:02.514 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:32:02.546 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:32:02.548 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:32:02.548 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:32:02.549 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:32:02.549 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:32:02.549 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:32:02.549 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:32:02.549 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:32:02.549 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:32:02.564 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:32:02.581 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:02 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2517265345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:32:02.581 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:02 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2517265345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:32:02.676 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:03.619 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:32:03.693 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:03 smithi110 ceph-mon[212939]: pgmap v1447: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 13010/60285 objects degraded (21.581%); 6358/60285 objects misplaced (10.547%); 0 B/s, 135 objects/s recovering 2023-09-23T23:32:03.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:03 smithi138 ceph-mon[193606]: pgmap v1447: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 13010/60285 objects degraded (21.581%); 6358/60285 objects misplaced (10.547%); 0 B/s, 135 objects/s recovering 2023-09-23T23:32:03.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:03 smithi110 ceph-mon[221102]: pgmap v1447: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 13010/60285 objects degraded (21.581%); 6358/60285 objects misplaced (10.547%); 0 B/s, 135 objects/s recovering 2023-09-23T23:32:03.997 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:32:04.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:04.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:32:04.104 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:32:04.151 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:04.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:04 smithi110 ceph-mon[221102]: failed to encode map e523 with expected crc 2023-09-23T23:32:04.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:04 smithi110 ceph-mon[221102]: osdmap e523: 8 total, 8 up, 8 in 2023-09-23T23:32:04.423 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:04 smithi110 ceph-mon[221102]: failed to encode map e523 with expected crc 2023-09-23T23:32:04.423 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:04 smithi110 ceph-mon[221102]: failed to encode map e523 with expected crc 2023-09-23T23:32:04.423 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:04 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 13010/60285 objects degraded (21.581%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:04.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:04 smithi138 ceph-mon[193606]: failed to encode map e523 with expected crc 2023-09-23T23:32:04.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:04 smithi138 ceph-mon[193606]: osdmap e523: 8 total, 8 up, 8 in 2023-09-23T23:32:04.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:04 smithi138 ceph-mon[193606]: failed to encode map e523 with expected crc 2023-09-23T23:32:04.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:04 smithi138 ceph-mon[193606]: failed to encode map e523 with expected crc 2023-09-23T23:32:04.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:04 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 13010/60285 objects degraded (21.581%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:04.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:04 smithi110 ceph-mon[212939]: failed to encode map e523 with expected crc 2023-09-23T23:32:04.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:04 smithi110 ceph-mon[212939]: osdmap e523: 8 total, 8 up, 8 in 2023-09-23T23:32:04.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:04 smithi110 ceph-mon[212939]: failed to encode map e523 with expected crc 2023-09-23T23:32:04.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:04 smithi110 ceph-mon[212939]: failed to encode map e523 with expected crc 2023-09-23T23:32:04.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:04 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 13010/60285 objects degraded (21.581%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:05.273 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:32:05.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:05.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:32:05.323 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:32:05.382 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:05.415 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:05.484 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:05.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:05.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:05.572 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:05 smithi110 ceph-mon[212939]: pgmap v1449: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 13010/58800 objects degraded (22.126%); 6358/58800 objects misplaced (10.813%); 0 B/s, 78 objects/s recovering 2023-09-23T23:32:05.572 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:05 smithi110 ceph-mon[212939]: failed to encode map e524 with expected crc 2023-09-23T23:32:05.572 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:05 smithi110 ceph-mon[212939]: failed to encode map e524 with expected crc 2023-09-23T23:32:05.572 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:05 smithi110 ceph-mon[212939]: osdmap e524: 8 total, 8 up, 8 in 2023-09-23T23:32:05.572 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:05 smithi110 ceph-mon[212939]: failed to encode map e524 with expected crc 2023-09-23T23:32:05.573 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:05 smithi110 ceph-mon[212939]: failed to encode map e524 with expected crc 2023-09-23T23:32:05.573 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:05 smithi110 ceph-mon[221102]: pgmap v1449: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 13010/58800 objects degraded (22.126%); 6358/58800 objects misplaced (10.813%); 0 B/s, 78 objects/s recovering 2023-09-23T23:32:05.573 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:05 smithi110 ceph-mon[221102]: failed to encode map e524 with expected crc 2023-09-23T23:32:05.573 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:05 smithi110 ceph-mon[221102]: failed to encode map e524 with expected crc 2023-09-23T23:32:05.573 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:05 smithi110 ceph-mon[221102]: osdmap e524: 8 total, 8 up, 8 in 2023-09-23T23:32:05.573 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:05 smithi110 ceph-mon[221102]: failed to encode map e524 with expected crc 2023-09-23T23:32:05.573 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:05 smithi110 ceph-mon[221102]: failed to encode map e524 with expected crc 2023-09-23T23:32:05.590 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:32:05.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:05 smithi138 ceph-mon[193606]: pgmap v1449: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 13010/58800 objects degraded (22.126%); 6358/58800 objects misplaced (10.813%); 0 B/s, 78 objects/s recovering 2023-09-23T23:32:05.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:05 smithi138 ceph-mon[193606]: failed to encode map e524 with expected crc 2023-09-23T23:32:05.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:05 smithi138 ceph-mon[193606]: failed to encode map e524 with expected crc 2023-09-23T23:32:05.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:05 smithi138 ceph-mon[193606]: osdmap e524: 8 total, 8 up, 8 in 2023-09-23T23:32:05.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:05 smithi138 ceph-mon[193606]: failed to encode map e524 with expected crc 2023-09-23T23:32:05.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:05 smithi138 ceph-mon[193606]: failed to encode map e524 with expected crc 2023-09-23T23:32:05.721 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:05.881 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:32:05.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:05.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:32:05.997 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:32:06.027 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:06.029 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:06.094 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:32:06.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:06.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:32:06.202 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:32:06.242 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:06.304 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:32:06.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:06.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:32:06.419 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:32:06.452 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:06.509 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:32:06.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:06.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:32:06.581 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:06.617 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:32:06.707 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:06.937 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:32:06.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:06.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:32:06.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:32:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:32:06] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-09-23T23:32:07.047 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:32:07.146 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:07.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:07 smithi110 ceph-mon[212939]: pgmap v1451: 129 pgs: 1 active+recovery_wait+remapped, 1 creating+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 12546/49545 objects degraded (25.322%); 6141/49545 objects misplaced (12.395%); 1.1 MiB/s, 169 objects/s recovering 2023-09-23T23:32:07.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:07 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:07.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:07 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:07.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:07 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:32:07.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:07 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2944696492' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:32:07.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:07 smithi110 ceph-mon[221102]: pgmap v1451: 129 pgs: 1 active+recovery_wait+remapped, 1 creating+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 12546/49545 objects degraded (25.322%); 6141/49545 objects misplaced (12.395%); 1.1 MiB/s, 169 objects/s recovering 2023-09-23T23:32:07.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:07 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:07.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:07 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:07.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:07 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:32:07.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:07 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2944696492' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:32:07.457 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T23:32:07.687 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:32:07.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:07 smithi138 ceph-mon[193606]: pgmap v1451: 129 pgs: 1 active+recovery_wait+remapped, 1 creating+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 12546/49545 objects degraded (25.322%); 6141/49545 objects misplaced (12.395%); 1.1 MiB/s, 169 objects/s recovering 2023-09-23T23:32:07.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:07 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:07.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:07 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:07.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:07 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:32:07.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:07 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2944696492' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:32:08.746 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:32:08.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:08.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:32:08.753 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:08 smithi110 ceph-mon[212939]: failed to encode map e525 with expected crc 2023-09-23T23:32:08.753 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:08 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2944696492' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:32:08.754 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:08 smithi110 ceph-mon[212939]: osdmap e525: 8 total, 8 up, 8 in 2023-09-23T23:32:08.754 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:08 smithi110 ceph-mon[212939]: failed to encode map e525 with expected crc 2023-09-23T23:32:08.754 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:08 smithi110 ceph-mon[212939]: failed to encode map e525 with expected crc 2023-09-23T23:32:08.754 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:08 smithi110 ceph-mon[212939]: failed to encode map e525 with expected crc 2023-09-23T23:32:08.759 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:08 smithi110 ceph-mon[221102]: failed to encode map e525 with expected crc 2023-09-23T23:32:08.759 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:08 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2944696492' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:32:08.759 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:08 smithi110 ceph-mon[221102]: osdmap e525: 8 total, 8 up, 8 in 2023-09-23T23:32:08.759 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:08 smithi110 ceph-mon[221102]: failed to encode map e525 with expected crc 2023-09-23T23:32:08.760 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:08 smithi110 ceph-mon[221102]: failed to encode map e525 with expected crc 2023-09-23T23:32:08.760 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:08 smithi110 ceph-mon[221102]: failed to encode map e525 with expected crc 2023-09-23T23:32:08.854 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:32:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:08 smithi138 ceph-mon[193606]: failed to encode map e525 with expected crc 2023-09-23T23:32:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:08 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2944696492' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:32:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:08 smithi138 ceph-mon[193606]: osdmap e525: 8 total, 8 up, 8 in 2023-09-23T23:32:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:08 smithi138 ceph-mon[193606]: failed to encode map e525 with expected crc 2023-09-23T23:32:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:08 smithi138 ceph-mon[193606]: failed to encode map e525 with expected crc 2023-09-23T23:32:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:08 smithi138 ceph-mon[193606]: failed to encode map e525 with expected crc 2023-09-23T23:32:08.970 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:09.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:09 smithi110 ceph-mon[212939]: pgmap v1453: 129 pgs: 1 active+recovery_wait+remapped, 1 creating+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 12546/49545 objects degraded (25.322%); 6141/49545 objects misplaced (12.395%); 1.1 MiB/s, 77 objects/s recovering 2023-09-23T23:32:09.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:09 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 12546/49545 objects degraded (25.322%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:09.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:09 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:32:09.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:09 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:32:09.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:09 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:09.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:09 smithi110 ceph-mon[221102]: pgmap v1453: 129 pgs: 1 active+recovery_wait+remapped, 1 creating+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 12546/49545 objects degraded (25.322%); 6141/49545 objects misplaced (12.395%); 1.1 MiB/s, 77 objects/s recovering 2023-09-23T23:32:09.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:09 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 12546/49545 objects degraded (25.322%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:09.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:09 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:32:09.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:09 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:32:09.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:09 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:09.824 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T23:32:09.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:09 smithi138 ceph-mon[193606]: pgmap v1453: 129 pgs: 1 active+recovery_wait+remapped, 1 creating+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 12546/49545 objects degraded (25.322%); 6141/49545 objects misplaced (12.395%); 1.1 MiB/s, 77 objects/s recovering 2023-09-23T23:32:09.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:09 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 12546/49545 objects degraded (25.322%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:09.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:09 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:32:09.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:09 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:32:09.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:09 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:10.022 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:32:10.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:10.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:32:10.129 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:32:10.170 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:10.221 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:32:10.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:10.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:32:10.326 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:32:10.352 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:10.434 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:32:10.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:10.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:32:10.543 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:32:10.575 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:10.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:10 smithi138 ceph-mon[193606]: pgmap v1454: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 880 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 12099/42297 objects degraded (28.605%); 5931/42297 objects misplaced (14.022%); 1.1 MiB/s, 141 objects/s recovering 2023-09-23T23:32:10.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:10 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:32:10.788 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:32:10.797 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:10 smithi110 ceph-mon[221102]: pgmap v1454: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 880 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 12099/42297 objects degraded (28.605%); 5931/42297 objects misplaced (14.022%); 1.1 MiB/s, 141 objects/s recovering 2023-09-23T23:32:10.797 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:10 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:32:10.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:10.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:32:10.799 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:10 smithi110 ceph-mon[212939]: pgmap v1454: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 880 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 12099/42297 objects degraded (28.605%); 5931/42297 objects misplaced (14.022%); 1.1 MiB/s, 141 objects/s recovering 2023-09-23T23:32:10.799 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:10 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:32:10.900 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:10.936 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:10.938 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:10.986 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:10.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:10.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:11.093 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:32:11.210 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:11.349 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:32:11.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:11.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:32:11.406 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:11.459 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:32:11.579 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:11.703 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:32:11.922 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:32:11.928 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:11 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2012855467' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:32:11.928 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:11 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:32:11.928 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:11 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2012855467' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:32:11.929 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:11 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:32:11.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:11.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:32:12.030 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:32:12.123 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:11 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2012855467' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:32:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:11 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:32:12.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:12 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:32:12.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:12 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:32:12.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:12 smithi110 ceph-mon[212939]: osdmap e526: 8 total, 8 up, 8 in 2023-09-23T23:32:12.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:12 smithi110 ceph-mon[212939]: failed to encode map e526 with expected crc 2023-09-23T23:32:12.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:12 smithi110 ceph-mon[212939]: failed to encode map e526 with expected crc 2023-09-23T23:32:12.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:12 smithi110 ceph-mon[212939]: failed to encode map e526 with expected crc 2023-09-23T23:32:12.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:12 smithi110 ceph-mon[212939]: pgmap v1456: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 830 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 12099/39900 objects degraded (30.323%); 5931/39900 objects misplaced (14.865%); 941 KiB/s, 78 objects/s recovering 2023-09-23T23:32:12.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:12 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:32:12.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:12 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:32:12.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:12 smithi110 ceph-mon[221102]: osdmap e526: 8 total, 8 up, 8 in 2023-09-23T23:32:12.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:12 smithi110 ceph-mon[221102]: failed to encode map e526 with expected crc 2023-09-23T23:32:12.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:12 smithi110 ceph-mon[221102]: failed to encode map e526 with expected crc 2023-09-23T23:32:12.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:12 smithi110 ceph-mon[221102]: failed to encode map e526 with expected crc 2023-09-23T23:32:12.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:12 smithi110 ceph-mon[221102]: pgmap v1456: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 830 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 12099/39900 objects degraded (30.323%); 5931/39900 objects misplaced (14.865%); 941 KiB/s, 78 objects/s recovering 2023-09-23T23:32:13.175 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:32:13.176 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:32:13.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:32:13.176 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:32:13.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.176 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135622 11.c 11:38fc4d47:::benchmark_data_smithi110_433746_object29742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:32:13.176 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.568262+0000", 2023-09-23T23:32:13.176 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 361.33990124899998, 2023-09-23T23:32:13.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7526951390000001, 2023-09-23T23:32:13.176 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.176 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.176 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.177 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.177 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.177 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135622 2023-09-23T23:32:13.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.177 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568262+0000", 2023-09-23T23:32:13.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568262+0000", 2023-09-23T23:32:13.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568260+0000", 2023-09-23T23:32:13.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:32:13.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568267+0000", 2023-09-23T23:32:13.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5010000000000001e-06 2023-09-23T23:32:13.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568277+0000", 2023-09-23T23:32:13.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0165e-05 2023-09-23T23:32:13.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568284+0000", 2023-09-23T23:32:13.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5689999999999999e-06 2023-09-23T23:32:13.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568341+0000", 2023-09-23T23:32:13.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6894000000000001e-05 2023-09-23T23:32:13.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.568349+0000", 2023-09-23T23:32:13.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3669999999999999e-06 2023-09-23T23:32:13.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505217+0000", 2023-09-23T23:32:13.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93686795 2023-09-23T23:32:13.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505229+0000", 2023-09-23T23:32:13.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2274e-05 2023-09-23T23:32:13.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.305074+0000", 2023-09-23T23:32:13.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.799845516 2023-09-23T23:32:13.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.305205+0000", 2023-09-23T23:32:13.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013038200000000001 2023-09-23T23:32:13.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:32:13.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.305293+0000", 2023-09-23T23:32:13.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8187000000000001e-05 2023-09-23T23:32:13.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.308879+0000", 2023-09-23T23:32:13.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035864999999999998 2023-09-23T23:32:13.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.308902+0000", 2023-09-23T23:32:13.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2161999999999999e-05 2023-09-23T23:32:13.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320708+0000", 2023-09-23T23:32:13.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011806535999999999 2023-09-23T23:32:13.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320814+0000", 2023-09-23T23:32:13.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105544 2023-09-23T23:32:13.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320848+0000", 2023-09-23T23:32:13.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4674e-05 2023-09-23T23:32:13.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320957+0000", 2023-09-23T23:32:13.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000108328 2023-09-23T23:32:13.185 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.185 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.185 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.185 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135633 11.c 11:3e1f1508:::benchmark_data_smithi110_433746_object29753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:32:13.185 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.573921+0000", 2023-09-23T23:32:13.185 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 361.33424172999997, 2023-09-23T23:32:13.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7472477729999998, 2023-09-23T23:32:13.186 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.186 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.186 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.186 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.186 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.186 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135633 2023-09-23T23:32:13.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.186 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573921+0000", 2023-09-23T23:32:13.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573921+0000", 2023-09-23T23:32:13.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573919+0000", 2023-09-23T23:32:13.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:32:13.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573926+0000", 2023-09-23T23:32:13.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2060000000000004e-06 2023-09-23T23:32:13.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573941+0000", 2023-09-23T23:32:13.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5325999999999999e-05 2023-09-23T23:32:13.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.573949+0000", 2023-09-23T23:32:13.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1780000000000003e-06 2023-09-23T23:32:13.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.574200+0000", 2023-09-23T23:32:13.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025123000000000002 2023-09-23T23:32:13.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.574213+0000", 2023-09-23T23:32:13.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2498e-05 2023-09-23T23:32:13.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505252+0000", 2023-09-23T23:32:13.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93103919199999996 2023-09-23T23:32:13.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505260+0000", 2023-09-23T23:32:13.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2320000000000001e-06 2023-09-23T23:32:13.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.308916+0000", 2023-09-23T23:32:13.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.803655368 2023-09-23T23:32:13.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.309001+0000", 2023-09-23T23:32:13.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5617000000000004e-05 2023-09-23T23:32:13.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:32:13.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.309074+0000", 2023-09-23T23:32:13.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2853000000000002e-05 2023-09-23T23:32:13.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320731+0000", 2023-09-23T23:32:13.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011657351999999999 2023-09-23T23:32:13.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320771+0000", 2023-09-23T23:32:13.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9177000000000001e-05 2023-09-23T23:32:13.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320796+0000", 2023-09-23T23:32:13.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5031000000000001e-05 2023-09-23T23:32:13.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321024+0000", 2023-09-23T23:32:13.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000228156 2023-09-23T23:32:13.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321066+0000", 2023-09-23T23:32:13.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2126999999999998e-05 2023-09-23T23:32:13.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321169+0000", 2023-09-23T23:32:13.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010292 2023-09-23T23:32:13.194 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.194 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.194 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.194 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135638 11.c 11:32116872:::benchmark_data_smithi110_433746_object29758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:32:13.195 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.579560+0000", 2023-09-23T23:32:13.195 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 361.32860331500001, 2023-09-23T23:32:13.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7417568370000001, 2023-09-23T23:32:13.195 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.195 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.195 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.195 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.195 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.195 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135638 2023-09-23T23:32:13.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.195 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579560+0000", 2023-09-23T23:32:13.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579560+0000", 2023-09-23T23:32:13.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579558+0000", 2023-09-23T23:32:13.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:32:13.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579565+0000", 2023-09-23T23:32:13.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1459999999999999e-06 2023-09-23T23:32:13.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579578+0000", 2023-09-23T23:32:13.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3019e-05 2023-09-23T23:32:13.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579587+0000", 2023-09-23T23:32:13.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6999999999999997e-06 2023-09-23T23:32:13.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579670+0000", 2023-09-23T23:32:13.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2934000000000003e-05 2023-09-23T23:32:13.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.579680+0000", 2023-09-23T23:32:13.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0742e-05 2023-09-23T23:32:13.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505282+0000", 2023-09-23T23:32:13.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.92560134699999996 2023-09-23T23:32:13.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505290+0000", 2023-09-23T23:32:13.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8380000000000002e-06 2023-09-23T23:32:13.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.312113+0000", 2023-09-23T23:32:13.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8068231020000001 2023-09-23T23:32:13.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.312237+0000", 2023-09-23T23:32:13.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012441899999999999 2023-09-23T23:32:13.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:32:13.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.312331+0000", 2023-09-23T23:32:13.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3719000000000002e-05 2023-09-23T23:32:13.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320776+0000", 2023-09-23T23:32:13.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084449720000000002 2023-09-23T23:32:13.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.320988+0000", 2023-09-23T23:32:13.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021210700000000001 2023-09-23T23:32:13.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321006+0000", 2023-09-23T23:32:13.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8111999999999999e-05 2023-09-23T23:32:13.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321204+0000", 2023-09-23T23:32:13.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000197796 2023-09-23T23:32:13.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.203 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321233+0000", 2023-09-23T23:32:13.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9108999999999999e-05 2023-09-23T23:32:13.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.203 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321316+0000", 2023-09-23T23:32:13.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3517999999999996e-05 2023-09-23T23:32:13.203 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.203 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.203 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.204 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135640 11.c 11:3dfd0302:::benchmark_data_smithi110_433746_object29760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:32:13.204 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.585524+0000", 2023-09-23T23:32:13.204 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 361.32263834600002, 2023-09-23T23:32:13.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7460455509999999, 2023-09-23T23:32:13.204 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.204 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.204 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.204 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.204 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.204 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135640 2023-09-23T23:32:13.204 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.205 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585524+0000", 2023-09-23T23:32:13.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585524+0000", 2023-09-23T23:32:13.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585522+0000", 2023-09-23T23:32:13.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-09-23T23:32:13.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585530+0000", 2023-09-23T23:32:13.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0099999999999995e-06 2023-09-23T23:32:13.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585582+0000", 2023-09-23T23:32:13.207 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2074999999999999e-05 2023-09-23T23:32:13.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.207 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.207 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.207 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585593+0000", 2023-09-23T23:32:13.207 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1155000000000001e-05 2023-09-23T23:32:13.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.207 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.207 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.207 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585657+0000", 2023-09-23T23:32:13.207 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3563000000000001e-05 2023-09-23T23:32:13.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.208 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.208 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.585669+0000", 2023-09-23T23:32:13.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2368000000000001e-05 2023-09-23T23:32:13.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.208 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.208 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505310+0000", 2023-09-23T23:32:13.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.91964102400000003 2023-09-23T23:32:13.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.208 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.209 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505318+0000", 2023-09-23T23:32:13.209 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7349999999999996e-06 2023-09-23T23:32:13.209 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.209 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.209 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.315785+0000", 2023-09-23T23:32:13.209 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.810466366 2023-09-23T23:32:13.209 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.209 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.209 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.315917+0000", 2023-09-23T23:32:13.209 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013256300000000001 2023-09-23T23:32:13.210 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.210 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.210 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:32:13.210 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.316041+0000", 2023-09-23T23:32:13.210 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012345100000000001 2023-09-23T23:32:13.210 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.210 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.210 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.210 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321343+0000", 2023-09-23T23:32:13.210 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053020849999999998 2023-09-23T23:32:13.210 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.210 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.211 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.211 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331351+0000", 2023-09-23T23:32:13.211 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010008193 2023-09-23T23:32:13.211 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.211 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.211 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.211 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331412+0000", 2023-09-23T23:32:13.211 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1023000000000002e-05 2023-09-23T23:32:13.211 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.211 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.211 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.211 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331438+0000", 2023-09-23T23:32:13.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6432999999999999e-05 2023-09-23T23:32:13.212 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.212 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331471+0000", 2023-09-23T23:32:13.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3127999999999999e-05 2023-09-23T23:32:13.212 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.212 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331570+0000", 2023-09-23T23:32:13.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8610999999999995e-05 2023-09-23T23:32:13.212 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.213 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.213 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.213 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135653 11.c 11:39fc45c5:::benchmark_data_smithi110_433746_object29773:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:32:13.213 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.612482+0000", 2023-09-23T23:32:13.213 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 361.295681328, 2023-09-23T23:32:13.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7224649520000002, 2023-09-23T23:32:13.213 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.213 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.213 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.213 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.213 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.214 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135653 2023-09-23T23:32:13.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.214 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612482+0000", 2023-09-23T23:32:13.214 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612482+0000", 2023-09-23T23:32:13.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.215 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612480+0000", 2023-09-23T23:32:13.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:32:13.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.215 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612487+0000", 2023-09-23T23:32:13.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2640000000000002e-06 2023-09-23T23:32:13.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612499+0000", 2023-09-23T23:32:13.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2116000000000001e-05 2023-09-23T23:32:13.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612509+0000", 2023-09-23T23:32:13.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6940000000000007e-06 2023-09-23T23:32:13.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612569+0000", 2023-09-23T23:32:13.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9738999999999998e-05 2023-09-23T23:32:13.217 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.217 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.217 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.612579+0000", 2023-09-23T23:32:13.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0592e-05 2023-09-23T23:32:13.217 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.217 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.217 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505353+0000", 2023-09-23T23:32:13.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.89277419800000002 2023-09-23T23:32:13.217 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.218 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.218 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505365+0000", 2023-09-23T23:32:13.218 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1090999999999999e-05 2023-09-23T23:32:13.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.218 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.218 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321356+0000", 2023-09-23T23:32:13.218 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8159917220000001 2023-09-23T23:32:13.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321466+0000", 2023-09-23T23:32:13.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109347 2023-09-23T23:32:13.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:32:13.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.321551+0000", 2023-09-23T23:32:13.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5669999999999998e-05 2023-09-23T23:32:13.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331357+0000", 2023-09-23T23:32:13.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00980549 2023-09-23T23:32:13.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.220 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.220 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331615+0000", 2023-09-23T23:32:13.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025800399999999999 2023-09-23T23:32:13.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.220 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.220 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331637+0000", 2023-09-23T23:32:13.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2036e-05 2023-09-23T23:32:13.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334781+0000", 2023-09-23T23:32:13.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031445589999999999 2023-09-23T23:32:13.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334836+0000", 2023-09-23T23:32:13.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4469999999999999e-05 2023-09-23T23:32:13.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334946+0000", 2023-09-23T23:32:13.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110579 2023-09-23T23:32:13.222 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.222 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.222 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.222 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135663 11.c 11:3bd42e62:::benchmark_data_smithi110_433746_object29783:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:32:13.222 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.646633+0000", 2023-09-23T23:32:13.222 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 361.261530177, 2023-09-23T23:32:13.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.691323304, 2023-09-23T23:32:13.222 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.222 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.223 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.223 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.223 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.223 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135663 2023-09-23T23:32:13.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.223 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646633+0000", 2023-09-23T23:32:13.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646633+0000", 2023-09-23T23:32:13.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646631+0000", 2023-09-23T23:32:13.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:32:13.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646638+0000", 2023-09-23T23:32:13.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9949999999999999e-06 2023-09-23T23:32:13.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646653+0000", 2023-09-23T23:32:13.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4892000000000001e-05 2023-09-23T23:32:13.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646661+0000", 2023-09-23T23:32:13.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9929999999999995e-06 2023-09-23T23:32:13.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646713+0000", 2023-09-23T23:32:13.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1544e-05 2023-09-23T23:32:13.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.646725+0000", 2023-09-23T23:32:13.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.223e-05 2023-09-23T23:32:13.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505389+0000", 2023-09-23T23:32:13.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.858663756 2023-09-23T23:32:13.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505397+0000", 2023-09-23T23:32:13.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4670000000000004e-06 2023-09-23T23:32:13.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.324831+0000", 2023-09-23T23:32:13.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.819433461 2023-09-23T23:32:13.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.324942+0000", 2023-09-23T23:32:13.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011171900000000001 2023-09-23T23:32:13.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:32:13.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.325072+0000", 2023-09-23T23:32:13.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012980000000000001 2023-09-23T23:32:13.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331656+0000", 2023-09-23T23:32:13.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0065834639999999998 2023-09-23T23:32:13.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.332662+0000", 2023-09-23T23:32:13.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010060049999999999 2023-09-23T23:32:13.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.333742+0000", 2023-09-23T23:32:13.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010798170000000001 2023-09-23T23:32:13.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337799+0000", 2023-09-23T23:32:13.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040573579999999996 2023-09-23T23:32:13.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337850+0000", 2023-09-23T23:32:13.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0818e-05 2023-09-23T23:32:13.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337956+0000", 2023-09-23T23:32:13.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010621 2023-09-23T23:32:13.231 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.231 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.231 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.232 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135680 11.c 11:34dfa950:::benchmark_data_smithi110_433746_object29800:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:32:13.232 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:10.699865+0000", 2023-09-23T23:32:13.232 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 361.20829775300001, 2023-09-23T23:32:13.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6382614580000001, 2023-09-23T23:32:13.232 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.232 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.232 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.232 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.232 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.232 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135680 2023-09-23T23:32:13.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.232 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699865+0000", 2023-09-23T23:32:13.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699865+0000", 2023-09-23T23:32:13.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699864+0000", 2023-09-23T23:32:13.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:32:13.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699870+0000", 2023-09-23T23:32:13.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9329999999999998e-06 2023-09-23T23:32:13.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699886+0000", 2023-09-23T23:32:13.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6402e-05 2023-09-23T23:32:13.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.235 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699896+0000", 2023-09-23T23:32:13.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3889999999999994e-06 2023-09-23T23:32:13.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.235 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699953+0000", 2023-09-23T23:32:13.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7763999999999998e-05 2023-09-23T23:32:13.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.235 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:10.699968+0000", 2023-09-23T23:32:13.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4749999999999999e-05 2023-09-23T23:32:13.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505806+0000", 2023-09-23T23:32:13.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.80583805100000006 2023-09-23T23:32:13.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505816+0000", 2023-09-23T23:32:13.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9320000000000008e-06 2023-09-23T23:32:13.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.328139+0000", 2023-09-23T23:32:13.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8223231129999999 2023-09-23T23:32:13.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.328247+0000", 2023-09-23T23:32:13.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010777600000000001 2023-09-23T23:32:13.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T23:32:13.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.328367+0000", 2023-09-23T23:32:13.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012031000000000001 2023-09-23T23:32:13.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.331675+0000", 2023-09-23T23:32:13.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033076669999999998 2023-09-23T23:32:13.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.332701+0000", 2023-09-23T23:32:13.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010263959999999999 2023-09-23T23:32:13.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.333784+0000", 2023-09-23T23:32:13.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001082911 2023-09-23T23:32:13.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337990+0000", 2023-09-23T23:32:13.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042062970000000003 2023-09-23T23:32:13.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.338026+0000", 2023-09-23T23:32:13.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5335e-05 2023-09-23T23:32:13.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.338127+0000", 2023-09-23T23:32:13.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000100717 2023-09-23T23:32:13.240 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.240 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.240 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.240 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178625 11.7 11.bda50607 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:32:13.240 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.600905+0000", 2023-09-23T23:32:13.241 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 292.30725782399998, 2023-09-23T23:32:13.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93901017899999994, 2023-09-23T23:32:13.241 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.241 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:32:13.241 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.241 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.241 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.241 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178625 2023-09-23T23:32:13.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.241 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.241 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.600905+0000", 2023-09-23T23:32:13.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.600905+0000", 2023-09-23T23:32:13.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.600903+0000", 2023-09-23T23:32:13.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:32:13.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.600910+0000", 2023-09-23T23:32:13.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1080000000000004e-06 2023-09-23T23:32:13.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.600922+0000", 2023-09-23T23:32:13.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.169e-05 2023-09-23T23:32:13.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.600929+0000", 2023-09-23T23:32:13.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.798e-06 2023-09-23T23:32:13.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.600968+0000", 2023-09-23T23:32:13.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9560000000000001e-05 2023-09-23T23:32:13.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.600978+0000", 2023-09-23T23:32:13.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5189999999999998e-06 2023-09-23T23:32:13.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.539904+0000", 2023-09-23T23:32:13.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93892621499999995 2023-09-23T23:32:13.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.539915+0000", 2023-09-23T23:32:13.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1126e-05 2023-09-23T23:32:13.245 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.245 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.245 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.246 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:202131 11.17 11:efc6c41a:::benchmark_data_smithi110_433746_object96251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T23:32:13.246 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:00.397599+0000", 2023-09-23T23:32:13.246 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 251.51056352200001, 2023-09-23T23:32:13.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98048151299999997, 2023-09-23T23:32:13.246 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.246 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.246 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.246 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.246 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.246 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 202131 2023-09-23T23:32:13.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.247 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397599+0000", 2023-09-23T23:32:13.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397599+0000", 2023-09-23T23:32:13.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397599+0000", 2023-09-23T23:32:13.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:32:13.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397603+0000", 2023-09-23T23:32:13.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5870000000000003e-06 2023-09-23T23:32:13.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397615+0000", 2023-09-23T23:32:13.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1591e-05 2023-09-23T23:32:13.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397622+0000", 2023-09-23T23:32:13.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5009999999999998e-06 2023-09-23T23:32:13.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397741+0000", 2023-09-23T23:32:13.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118649 2023-09-23T23:32:13.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397749+0000", 2023-09-23T23:32:13.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4950000000000008e-06 2023-09-23T23:32:13.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.364031+0000", 2023-09-23T23:32:13.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96628133299999996 2023-09-23T23:32:13.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.364148+0000", 2023-09-23T23:32:13.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011695499999999999 2023-09-23T23:32:13.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:32:13.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.364473+0000", 2023-09-23T23:32:13.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032511900000000002 2023-09-23T23:32:13.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.370881+0000", 2023-09-23T23:32:13.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0064080700000000001 2023-09-23T23:32:13.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.377963+0000", 2023-09-23T23:32:13.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070824929999999996 2023-09-23T23:32:13.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378001+0000", 2023-09-23T23:32:13.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7595999999999997e-05 2023-09-23T23:32:13.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378043+0000", 2023-09-23T23:32:13.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2283999999999997e-05 2023-09-23T23:32:13.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378081+0000", 2023-09-23T23:32:13.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7478999999999998e-05 2023-09-23T23:32:13.253 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.253 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.253 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.254 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:202151 11.17 11:eead75a8:::benchmark_data_smithi110_433746_object96271:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T23:32:13.254 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:00.411787+0000", 2023-09-23T23:32:13.254 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 251.49637548000001, 2023-09-23T23:32:13.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96643706600000001, 2023-09-23T23:32:13.254 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.254 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.254 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.254 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.254 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.254 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 202151 2023-09-23T23:32:13.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.254 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411787+0000", 2023-09-23T23:32:13.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411787+0000", 2023-09-23T23:32:13.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411786+0000", 2023-09-23T23:32:13.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:32:13.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411792+0000", 2023-09-23T23:32:13.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3799999999999999e-06 2023-09-23T23:32:13.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411810+0000", 2023-09-23T23:32:13.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7751999999999999e-05 2023-09-23T23:32:13.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411822+0000", 2023-09-23T23:32:13.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.201e-05 2023-09-23T23:32:13.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411868+0000", 2023-09-23T23:32:13.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5695000000000002e-05 2023-09-23T23:32:13.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411876+0000", 2023-09-23T23:32:13.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6209999999999993e-06 2023-09-23T23:32:13.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.368736+0000", 2023-09-23T23:32:13.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95685995199999996 2023-09-23T23:32:13.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.368844+0000", 2023-09-23T23:32:13.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107253 2023-09-23T23:32:13.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:32:13.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.369149+0000", 2023-09-23T23:32:13.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030546099999999998 2023-09-23T23:32:13.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.377918+0000", 2023-09-23T23:32:13.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0087685660000000002 2023-09-23T23:32:13.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378118+0000", 2023-09-23T23:32:13.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020053900000000001 2023-09-23T23:32:13.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378148+0000", 2023-09-23T23:32:13.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9896999999999999e-05 2023-09-23T23:32:13.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378190+0000", 2023-09-23T23:32:13.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1541000000000001e-05 2023-09-23T23:32:13.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378224+0000", 2023-09-23T23:32:13.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4879999999999998e-05 2023-09-23T23:32:13.261 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.261 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.261 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.261 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:202156 11.17 11:ef634d51:::benchmark_data_smithi110_433746_object96276:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T23:32:13.262 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:00.415723+0000", 2023-09-23T23:32:13.262 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 251.49243982499999, 2023-09-23T23:32:13.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96665629500000005, 2023-09-23T23:32:13.262 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.262 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.262 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.262 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.262 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.262 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 202156 2023-09-23T23:32:13.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.262 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415723+0000", 2023-09-23T23:32:13.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415723+0000", 2023-09-23T23:32:13.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415723+0000", 2023-09-23T23:32:13.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:32:13.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415725+0000", 2023-09-23T23:32:13.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8040000000000002e-06 2023-09-23T23:32:13.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415732+0000", 2023-09-23T23:32:13.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6549999999999998e-06 2023-09-23T23:32:13.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415738+0000", 2023-09-23T23:32:13.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5940000000000001e-06 2023-09-23T23:32:13.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415772+0000", 2023-09-23T23:32:13.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3912000000000003e-05 2023-09-23T23:32:13.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415778+0000", 2023-09-23T23:32:13.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9939999999999997e-06 2023-09-23T23:32:13.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.370907+0000", 2023-09-23T23:32:13.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95512952699999998 2023-09-23T23:32:13.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.371032+0000", 2023-09-23T23:32:13.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012525899999999999 2023-09-23T23:32:13.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:32:13.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.371307+0000", 2023-09-23T23:32:13.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027482299999999999 2023-09-23T23:32:13.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378260+0000", 2023-09-23T23:32:13.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0069533099999999999 2023-09-23T23:32:13.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378290+0000", 2023-09-23T23:32:13.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9379000000000001e-05 2023-09-23T23:32:13.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.382296+0000", 2023-09-23T23:32:13.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040064779999999999 2023-09-23T23:32:13.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.382337+0000", 2023-09-23T23:32:13.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1029000000000003e-05 2023-09-23T23:32:13.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.382379+0000", 2023-09-23T23:32:13.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1980999999999997e-05 2023-09-23T23:32:13.269 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.269 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.269 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.270 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:210997 11.17 11:eefcf624:::benchmark_data_smithi110_433746_object105117:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:32:13.270 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686483+0000", 2023-09-23T23:32:13.270 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 230.22167996499999, 2023-09-23T23:32:13.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0169343049999999, 2023-09-23T23:32:13.270 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.270 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.270 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.270 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.270 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.270 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 210997 2023-09-23T23:32:13.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.270 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686483+0000", 2023-09-23T23:32:13.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686483+0000", 2023-09-23T23:32:13.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686481+0000", 2023-09-23T23:32:13.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:32:13.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686489+0000", 2023-09-23T23:32:13.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.439e-06 2023-09-23T23:32:13.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686502+0000", 2023-09-23T23:32:13.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2376e-05 2023-09-23T23:32:13.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686523+0000", 2023-09-23T23:32:13.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0860999999999999e-05 2023-09-23T23:32:13.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687605+0000", 2023-09-23T23:32:13.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001082698 2023-09-23T23:32:13.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687618+0000", 2023-09-23T23:32:13.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2848e-05 2023-09-23T23:32:13.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.694585+0000", 2023-09-23T23:32:13.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.006967221 2023-09-23T23:32:13.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.694712+0000", 2023-09-23T23:32:13.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012615400000000001 2023-09-23T23:32:13.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:32:13.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.695012+0000", 2023-09-23T23:32:13.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030036499999999997 2023-09-23T23:32:13.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.698738+0000", 2023-09-23T23:32:13.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003726203 2023-09-23T23:32:13.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.698996+0000", 2023-09-23T23:32:13.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025763799999999998 2023-09-23T23:32:13.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.699037+0000", 2023-09-23T23:32:13.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1517000000000002e-05 2023-09-23T23:32:13.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703242+0000", 2023-09-23T23:32:13.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042050509999999996 2023-09-23T23:32:13.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.277 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.277 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703295+0000", 2023-09-23T23:32:13.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2997000000000001e-05 2023-09-23T23:32:13.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.277 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.277 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703417+0000", 2023-09-23T23:32:13.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012184900000000001 2023-09-23T23:32:13.277 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.277 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.277 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.278 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211002 11.17 11:ecab426b:::benchmark_data_smithi110_433746_object105122:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:32:13.278 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686530+0000", 2023-09-23T23:32:13.278 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 230.221632339, 2023-09-23T23:32:13.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0172692640000001, 2023-09-23T23:32:13.278 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.278 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.278 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.278 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.278 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.278 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211002 2023-09-23T23:32:13.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.278 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686530+0000", 2023-09-23T23:32:13.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686530+0000", 2023-09-23T23:32:13.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686530+0000", 2023-09-23T23:32:13.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:32:13.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686534+0000", 2023-09-23T23:32:13.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9709999999999998e-06 2023-09-23T23:32:13.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686553+0000", 2023-09-23T23:32:13.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9241000000000001e-05 2023-09-23T23:32:13.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686560+0000", 2023-09-23T23:32:13.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9260000000000003e-06 2023-09-23T23:32:13.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687641+0000", 2023-09-23T23:32:13.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001081226 2023-09-23T23:32:13.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687650+0000", 2023-09-23T23:32:13.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.833e-06 2023-09-23T23:32:13.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.696593+0000", 2023-09-23T23:32:13.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0089432190000001 2023-09-23T23:32:13.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.696740+0000", 2023-09-23T23:32:13.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014711800000000001 2023-09-23T23:32:13.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:32:13.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.697010+0000", 2023-09-23T23:32:13.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026912600000000001 2023-09-23T23:32:13.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.700956+0000", 2023-09-23T23:32:13.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039468569999999998 2023-09-23T23:32:13.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703189+0000", 2023-09-23T23:32:13.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022322599999999998 2023-09-23T23:32:13.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703471+0000", 2023-09-23T23:32:13.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028231600000000001 2023-09-23T23:32:13.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703641+0000", 2023-09-23T23:32:13.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000170404 2023-09-23T23:32:13.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703696+0000", 2023-09-23T23:32:13.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4475999999999998e-05 2023-09-23T23:32:13.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703800+0000", 2023-09-23T23:32:13.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103943 2023-09-23T23:32:13.285 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.285 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.285 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.285 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211006 11.17 11:ee6c8b78:::benchmark_data_smithi110_433746_object105126:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:32:13.286 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686565+0000", 2023-09-23T23:32:13.286 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 230.22159781799999, 2023-09-23T23:32:13.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0175006170000001, 2023-09-23T23:32:13.286 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.286 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.286 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.286 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.286 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.286 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211006 2023-09-23T23:32:13.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.286 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686565+0000", 2023-09-23T23:32:13.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686565+0000", 2023-09-23T23:32:13.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686565+0000", 2023-09-23T23:32:13.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:32:13.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686568+0000", 2023-09-23T23:32:13.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2600000000000001e-06 2023-09-23T23:32:13.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686574+0000", 2023-09-23T23:32:13.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0789999999999997e-06 2023-09-23T23:32:13.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686579+0000", 2023-09-23T23:32:13.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3279999999999996e-06 2023-09-23T23:32:13.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687673+0000", 2023-09-23T23:32:13.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001093966 2023-09-23T23:32:13.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687681+0000", 2023-09-23T23:32:13.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9889999999999992e-06 2023-09-23T23:32:13.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.699069+0000", 2023-09-23T23:32:13.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.011387475 2023-09-23T23:32:13.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.699178+0000", 2023-09-23T23:32:13.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109238 2023-09-23T23:32:13.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:32:13.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.699384+0000", 2023-09-23T23:32:13.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020630299999999999 2023-09-23T23:32:13.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703201+0000", 2023-09-23T23:32:13.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038163939999999999 2023-09-23T23:32:13.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703505+0000", 2023-09-23T23:32:13.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030447999999999999 2023-09-23T23:32:13.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703847+0000", 2023-09-23T23:32:13.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034161899999999998 2023-09-23T23:32:13.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703911+0000", 2023-09-23T23:32:13.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4521000000000001e-05 2023-09-23T23:32:13.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703957+0000", 2023-09-23T23:32:13.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5840999999999997e-05 2023-09-23T23:32:13.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704066+0000", 2023-09-23T23:32:13.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000108546 2023-09-23T23:32:13.293 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.293 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.293 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.293 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211014 11.17 11:ecac8c40:::benchmark_data_smithi110_433746_object105134:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:32:13.294 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686583+0000", 2023-09-23T23:32:13.294 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 230.22157947700001, 2023-09-23T23:32:13.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0177152949999999, 2023-09-23T23:32:13.294 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.294 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.294 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.294 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.294 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.294 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211014 2023-09-23T23:32:13.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.294 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686583+0000", 2023-09-23T23:32:13.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686583+0000", 2023-09-23T23:32:13.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686583+0000", 2023-09-23T23:32:13.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:32:13.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686586+0000", 2023-09-23T23:32:13.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6769999999999999e-06 2023-09-23T23:32:13.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686592+0000", 2023-09-23T23:32:13.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2480000000000001e-06 2023-09-23T23:32:13.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686598+0000", 2023-09-23T23:32:13.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4500000000000001e-06 2023-09-23T23:32:13.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687703+0000", 2023-09-23T23:32:13.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001105113 2023-09-23T23:32:13.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687711+0000", 2023-09-23T23:32:13.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9410000000000007e-06 2023-09-23T23:32:13.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.700984+0000", 2023-09-23T23:32:13.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0132722620000001 2023-09-23T23:32:13.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.701130+0000", 2023-09-23T23:32:13.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000146804 2023-09-23T23:32:13.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:32:13.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.701398+0000", 2023-09-23T23:32:13.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026770100000000002 2023-09-23T23:32:13.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703434+0000", 2023-09-23T23:32:13.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020357940000000001 2023-09-23T23:32:13.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703879+0000", 2023-09-23T23:32:13.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00044474299999999999 2023-09-23T23:32:13.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704106+0000", 2023-09-23T23:32:13.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022766 2023-09-23T23:32:13.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704144+0000", 2023-09-23T23:32:13.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7836000000000002e-05 2023-09-23T23:32:13.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704191+0000", 2023-09-23T23:32:13.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6582999999999998e-05 2023-09-23T23:32:13.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704299+0000", 2023-09-23T23:32:13.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107909 2023-09-23T23:32:13.301 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.301 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.301 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.301 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211043 11.17 11:edfeec9c:::benchmark_data_smithi110_433746_object105163:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:32:13.302 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686612+0000", 2023-09-23T23:32:13.302 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 230.22155058300001, 2023-09-23T23:32:13.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0257805470000001, 2023-09-23T23:32:13.302 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.302 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.302 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.302 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.302 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.302 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211043 2023-09-23T23:32:13.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.302 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686612+0000", 2023-09-23T23:32:13.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686612+0000", 2023-09-23T23:32:13.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686612+0000", 2023-09-23T23:32:13.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:32:13.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686615+0000", 2023-09-23T23:32:13.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3289999999999998e-06 2023-09-23T23:32:13.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686622+0000", 2023-09-23T23:32:13.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1439999999999997e-06 2023-09-23T23:32:13.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686628+0000", 2023-09-23T23:32:13.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9560000000000002e-06 2023-09-23T23:32:13.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687733+0000", 2023-09-23T23:32:13.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011048640000000001 2023-09-23T23:32:13.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687741+0000", 2023-09-23T23:32:13.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9340000000000006e-06 2023-09-23T23:32:13.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704331+0000", 2023-09-23T23:32:13.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0165899519999999 2023-09-23T23:32:13.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704469+0000", 2023-09-23T23:32:13.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013822899999999999 2023-09-23T23:32:13.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:32:13.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704810+0000", 2023-09-23T23:32:13.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000341169 2023-09-23T23:32:13.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712010+0000", 2023-09-23T23:32:13.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071992189999999998 2023-09-23T23:32:13.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712083+0000", 2023-09-23T23:32:13.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3090999999999995e-05 2023-09-23T23:32:13.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712140+0000", 2023-09-23T23:32:13.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7030000000000001e-05 2023-09-23T23:32:13.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712203+0000", 2023-09-23T23:32:13.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3297000000000001e-05 2023-09-23T23:32:13.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712257+0000", 2023-09-23T23:32:13.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4211e-05 2023-09-23T23:32:13.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712393+0000", 2023-09-23T23:32:13.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013563 2023-09-23T23:32:13.340 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.340 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.340 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.341 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211063 11.17 11:ec1e6191:::benchmark_data_smithi110_433746_object105183:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:32:13.341 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686633+0000", 2023-09-23T23:32:13.341 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 230.22153014700001, 2023-09-23T23:32:13.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.026136234, 2023-09-23T23:32:13.341 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.341 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.341 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.341 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.341 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.341 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211063 2023-09-23T23:32:13.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.342 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686633+0000", 2023-09-23T23:32:13.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686633+0000", 2023-09-23T23:32:13.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686632+0000", 2023-09-23T23:32:13.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:32:13.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686635+0000", 2023-09-23T23:32:13.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9110000000000002e-06 2023-09-23T23:32:13.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686643+0000", 2023-09-23T23:32:13.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7309999999999993e-06 2023-09-23T23:32:13.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686648+0000", 2023-09-23T23:32:13.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1580000000000004e-06 2023-09-23T23:32:13.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687762+0000", 2023-09-23T23:32:13.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011139660000000001 2023-09-23T23:32:13.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687770+0000", 2023-09-23T23:32:13.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9820000000000008e-06 2023-09-23T23:32:13.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.706433+0000", 2023-09-23T23:32:13.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.018663052 2023-09-23T23:32:13.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.706571+0000", 2023-09-23T23:32:13.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013809200000000001 2023-09-23T23:32:13.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:32:13.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.706778+0000", 2023-09-23T23:32:13.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000206739 2023-09-23T23:32:13.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712153+0000", 2023-09-23T23:32:13.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053747880000000001 2023-09-23T23:32:13.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712460+0000", 2023-09-23T23:32:13.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030737499999999998 2023-09-23T23:32:13.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712524+0000", 2023-09-23T23:32:13.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4078000000000006e-05 2023-09-23T23:32:13.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712603+0000", 2023-09-23T23:32:13.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8625e-05 2023-09-23T23:32:13.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712666+0000", 2023-09-23T23:32:13.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2713999999999996e-05 2023-09-23T23:32:13.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712769+0000", 2023-09-23T23:32:13.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010334 2023-09-23T23:32:13.348 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.349 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.349 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.349 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211074 11.17 11:eb325c31:::benchmark_data_smithi110_433746_object105194:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:32:13.349 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686653+0000", 2023-09-23T23:32:13.349 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 230.22151021799999, 2023-09-23T23:32:13.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.02894591, 2023-09-23T23:32:13.349 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.349 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.349 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.349 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.350 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.350 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211074 2023-09-23T23:32:13.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.350 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686653+0000", 2023-09-23T23:32:13.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686653+0000", 2023-09-23T23:32:13.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686652+0000", 2023-09-23T23:32:13.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:32:13.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686655+0000", 2023-09-23T23:32:13.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6469999999999998e-06 2023-09-23T23:32:13.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686661+0000", 2023-09-23T23:32:13.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8969999999999996e-06 2023-09-23T23:32:13.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686666+0000", 2023-09-23T23:32:13.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9930000000000003e-06 2023-09-23T23:32:13.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687792+0000", 2023-09-23T23:32:13.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011261330000000001 2023-09-23T23:32:13.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687800+0000", 2023-09-23T23:32:13.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9389999999999997e-06 2023-09-23T23:32:13.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.708139+0000", 2023-09-23T23:32:13.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0203391749999999 2023-09-23T23:32:13.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.708257+0000", 2023-09-23T23:32:13.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117959 2023-09-23T23:32:13.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:32:13.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.708494+0000", 2023-09-23T23:32:13.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023732399999999999 2023-09-23T23:32:13.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712159+0000", 2023-09-23T23:32:13.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036648150000000001 2023-09-23T23:32:13.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715288+0000", 2023-09-23T23:32:13.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031290469999999998 2023-09-23T23:32:13.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715392+0000", 2023-09-23T23:32:13.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104362 2023-09-23T23:32:13.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715427+0000", 2023-09-23T23:32:13.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4581e-05 2023-09-23T23:32:13.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715475+0000", 2023-09-23T23:32:13.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7766999999999999e-05 2023-09-23T23:32:13.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715599+0000", 2023-09-23T23:32:13.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012368000000000001 2023-09-23T23:32:13.357 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.357 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.357 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.357 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211090 11.17 11:ee95de61:::benchmark_data_smithi110_433746_object105210:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:32:13.357 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686670+0000", 2023-09-23T23:32:13.357 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 230.22149282300001, 2023-09-23T23:32:13.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.029135624, 2023-09-23T23:32:13.358 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.358 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.358 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.358 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.358 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.358 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211090 2023-09-23T23:32:13.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.358 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686670+0000", 2023-09-23T23:32:13.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686670+0000", 2023-09-23T23:32:13.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686670+0000", 2023-09-23T23:32:13.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:32:13.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686672+0000", 2023-09-23T23:32:13.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1749999999999999e-06 2023-09-23T23:32:13.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686678+0000", 2023-09-23T23:32:13.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3589999999999996e-06 2023-09-23T23:32:13.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686683+0000", 2023-09-23T23:32:13.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1139999999999997e-06 2023-09-23T23:32:13.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687821+0000", 2023-09-23T23:32:13.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001137723 2023-09-23T23:32:13.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687829+0000", 2023-09-23T23:32:13.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6809999999999998e-06 2023-09-23T23:32:13.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.710062+0000", 2023-09-23T23:32:13.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0222334470000001 2023-09-23T23:32:13.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.710215+0000", 2023-09-23T23:32:13.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015298800000000001 2023-09-23T23:32:13.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:32:13.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.710532+0000", 2023-09-23T23:32:13.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00031638499999999997 2023-09-23T23:32:13.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712541+0000", 2023-09-23T23:32:13.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020091390000000001 2023-09-23T23:32:13.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715339+0000", 2023-09-23T23:32:13.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002798367 2023-09-23T23:32:13.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715645+0000", 2023-09-23T23:32:13.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030564500000000001 2023-09-23T23:32:13.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715685+0000", 2023-09-23T23:32:13.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0182000000000003e-05 2023-09-23T23:32:13.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715717+0000", 2023-09-23T23:32:13.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1649000000000001e-05 2023-09-23T23:32:13.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715806+0000", 2023-09-23T23:32:13.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9040000000000001e-05 2023-09-23T23:32:13.365 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.365 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.365 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.365 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211101 11.17 11:eed1b340:::benchmark_data_smithi110_433746_object105221:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:32:13.365 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686688+0000", 2023-09-23T23:32:13.366 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 230.22147523500001, 2023-09-23T23:32:13.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.034036953, 2023-09-23T23:32:13.366 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:32:13.366 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:13.366 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:32:13.366 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:32:13.366 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:13.366 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211101 2023-09-23T23:32:13.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.366 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:32:13.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:32:13.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686688+0000", 2023-09-23T23:32:13.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:32:13.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686688+0000", 2023-09-23T23:32:13.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:32:13.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:32:13.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686687+0000", 2023-09-23T23:32:13.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:32:13.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:32:13.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686690+0000", 2023-09-23T23:32:13.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4930000000000001e-06 2023-09-23T23:32:13.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:32:13.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686696+0000", 2023-09-23T23:32:13.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2400000000000004e-06 2023-09-23T23:32:13.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:32:13.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686701+0000", 2023-09-23T23:32:13.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2020000000000003e-06 2023-09-23T23:32:13.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687851+0000", 2023-09-23T23:32:13.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011494809999999999 2023-09-23T23:32:13.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:32:13.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687859+0000", 2023-09-23T23:32:13.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8779999999999998e-06 2023-09-23T23:32:13.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:32:13.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712809+0000", 2023-09-23T23:32:13.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0249501940000001 2023-09-23T23:32:13.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:32:13.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712967+0000", 2023-09-23T23:32:13.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015830900000000001 2023-09-23T23:32:13.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:32:13.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.713350+0000", 2023-09-23T23:32:13.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00038338199999999998 2023-09-23T23:32:13.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715863+0000", 2023-09-23T23:32:13.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025129060000000001 2023-09-23T23:32:13.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715903+0000", 2023-09-23T23:32:13.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9675000000000002e-05 2023-09-23T23:32:13.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:13.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715938+0000", 2023-09-23T23:32:13.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4501000000000001e-05 2023-09-23T23:32:13.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:32:13.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.720575+0000", 2023-09-23T23:32:13.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046374110000000001 2023-09-23T23:32:13.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:32:13.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.720618+0000", 2023-09-23T23:32:13.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3411000000000002e-05 2023-09-23T23:32:13.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:32:13.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:32:13.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:32:13.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.720725+0000", 2023-09-23T23:32:13.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000106209 2023-09-23T23:32:13.373 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.373 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.373 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.373 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:32:13.373 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:32:13.373 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:32:13.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:12 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:32:13.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:12 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:32:13.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:12 smithi138 ceph-mon[193606]: osdmap e526: 8 total, 8 up, 8 in 2023-09-23T23:32:13.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:12 smithi138 ceph-mon[193606]: failed to encode map e526 with expected crc 2023-09-23T23:32:13.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:12 smithi138 ceph-mon[193606]: failed to encode map e526 with expected crc 2023-09-23T23:32:13.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:12 smithi138 ceph-mon[193606]: failed to encode map e526 with expected crc 2023-09-23T23:32:13.374 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:12 smithi138 ceph-mon[193606]: pgmap v1456: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 830 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 12099/39900 objects degraded (30.323%); 5931/39900 objects misplaced (14.865%); 941 KiB/s, 78 objects/s recovering 2023-09-23T23:32:13.810 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:32:13.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:13.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:32:13.921 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:13.966 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:14.027 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:14.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:14.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:14.112 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:14 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:14.112 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:14 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:32:14.112 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:14 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 12099/39900 objects degraded (30.323%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:14.112 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:14 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:14.112 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:14 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:32:14.112 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:14 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 12099/39900 objects degraded (30.323%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:14.135 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:14.167 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:14.242 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:14.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:14.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:14.249 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:14 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:14.249 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:14 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:32:14.249 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:14 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 12099/39900 objects degraded (30.323%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:14.349 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:32:14.384 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:14.428 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:32:14.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:14.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:32:14.534 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:14.570 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:14.631 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:14.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:14.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:14.741 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:32:14.784 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:14.858 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:32:14.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:14.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:32:14.972 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:32:15.007 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:15.060 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:32:15.076 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:32:15.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:15.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:32:15.184 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:32:15.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:15 smithi138 ceph-mon[193606]: pgmap v1457: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 801 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 12099/38517 objects degraded (31.412%); 5931/38517 objects misplaced (15.398%); 0 B/s, 55 objects/s recovering 2023-09-23T23:32:15.262 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:15 smithi110 ceph-mon[212939]: pgmap v1457: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 801 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 12099/38517 objects degraded (31.412%); 5931/38517 objects misplaced (15.398%); 0 B/s, 55 objects/s recovering 2023-09-23T23:32:15.262 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:15 smithi110 ceph-mon[221102]: pgmap v1457: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 801 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 12099/38517 objects degraded (31.412%); 5931/38517 objects misplaced (15.398%); 0 B/s, 55 objects/s recovering 2023-09-23T23:32:15.294 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:15.715 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:32:15.758 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:32:15.802 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:32:15.833 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:32:15.853 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:32:15.874 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:32:15.894 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:32:16.000 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:32:16.032 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:32:16.032 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:32:16.032 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:32:16.032 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:32:16.032 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:32:16.033 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:32:16.033 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:32:16.033 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:32:16.033 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:32:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:16 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2859242089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:32:16.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:16 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2859242089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:32:16.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:16 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2859242089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:32:16.939 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:32:16.941 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:32:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:32:16] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.43.0" 2023-09-23T23:32:16.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:16.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:32:17.043 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:32:17.083 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:17.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:17 smithi110 ceph-mon[212939]: pgmap v1458: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 614 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 11707/29541 objects degraded (39.630%); 5710/29541 objects misplaced (19.329%); 0 B/s, 99 objects/s recovering 2023-09-23T23:32:17.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:17 smithi110 ceph-mon[221102]: pgmap v1458: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 614 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 11707/29541 objects degraded (39.630%); 5710/29541 objects misplaced (19.329%); 0 B/s, 99 objects/s recovering 2023-09-23T23:32:17.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:17 smithi138 ceph-mon[193606]: pgmap v1458: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 614 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 11707/29541 objects degraded (39.630%); 5710/29541 objects misplaced (19.329%); 0 B/s, 99 objects/s recovering 2023-09-23T23:32:17.489 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:32:17.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:17.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:32:17.601 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:32:17.707 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:17.718 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:17.913 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:32:17.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:17.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:32:18.031 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:32:18.038 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:18.142 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:18.278 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:32:18.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:18.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:32:18.390 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:32:18.425 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:18.499 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:32:18.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:18.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:32:18.605 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:32:18.638 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:18.725 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:32:18.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:18.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:32:18.831 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:32:18.892 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:19.077 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:19 smithi138 ceph-mon[193606]: pgmap v1459: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 609 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 11707/29256 objects degraded (40.016%); 5710/29256 objects misplaced (19.517%); 0 B/s, 83 objects/s recovering 2023-09-23T23:32:19.077 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:19 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2962078317' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:32:19.077 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:19 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:32:19.078 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:19 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 11707/29256 objects degraded (40.016%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:19.096 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T23:32:19.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:19 smithi110 ceph-mon[212939]: pgmap v1459: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 609 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 11707/29256 objects degraded (40.016%); 5710/29256 objects misplaced (19.517%); 0 B/s, 83 objects/s recovering 2023-09-23T23:32:19.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:19 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2962078317' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:32:19.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:19 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:32:19.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:19 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 11707/29256 objects degraded (40.016%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:19.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:19 smithi110 ceph-mon[221102]: pgmap v1459: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 609 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 11707/29256 objects degraded (40.016%); 5710/29256 objects misplaced (19.517%); 0 B/s, 83 objects/s recovering 2023-09-23T23:32:19.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:19 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2962078317' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:32:19.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:19 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:32:19.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:19 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 11707/29256 objects degraded (40.016%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:19.611 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:32:19.748 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:32:19.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:19.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:32:19.863 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:32:19.924 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:20 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:32:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:20 smithi138 ceph-mon[193606]: osdmap e527: 8 total, 8 up, 8 in 2023-09-23T23:32:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:20 smithi138 ceph-mon[193606]: failed to encode map e527 with expected crc 2023-09-23T23:32:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:20 smithi138 ceph-mon[193606]: failed to encode map e527 with expected crc 2023-09-23T23:32:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:20 smithi138 ceph-mon[193606]: failed to encode map e527 with expected crc 2023-09-23T23:32:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:20 smithi138 ceph-mon[193606]: failed to encode map e527 with expected crc 2023-09-23T23:32:20.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:20 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:32:20.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:20 smithi110 ceph-mon[212939]: osdmap e527: 8 total, 8 up, 8 in 2023-09-23T23:32:20.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:20 smithi110 ceph-mon[212939]: failed to encode map e527 with expected crc 2023-09-23T23:32:20.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:20 smithi110 ceph-mon[212939]: failed to encode map e527 with expected crc 2023-09-23T23:32:20.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:20 smithi110 ceph-mon[212939]: failed to encode map e527 with expected crc 2023-09-23T23:32:20.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:20 smithi110 ceph-mon[212939]: failed to encode map e527 with expected crc 2023-09-23T23:32:20.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:20 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:32:20.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:20 smithi110 ceph-mon[221102]: osdmap e527: 8 total, 8 up, 8 in 2023-09-23T23:32:20.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:20 smithi110 ceph-mon[221102]: failed to encode map e527 with expected crc 2023-09-23T23:32:20.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:20 smithi110 ceph-mon[221102]: failed to encode map e527 with expected crc 2023-09-23T23:32:20.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:20 smithi110 ceph-mon[221102]: failed to encode map e527 with expected crc 2023-09-23T23:32:20.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:20 smithi110 ceph-mon[221102]: failed to encode map e527 with expected crc 2023-09-23T23:32:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:21 smithi138 ceph-mon[193606]: pgmap v1461: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 441 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 11186/21213 objects degraded (52.732%); 5478/21213 objects misplaced (25.824%); 0 B/s, 111 objects/s recovering 2023-09-23T23:32:21.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:21 smithi110 ceph-mon[221102]: pgmap v1461: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 441 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 11186/21213 objects degraded (52.732%); 5478/21213 objects misplaced (25.824%); 0 B/s, 111 objects/s recovering 2023-09-23T23:32:21.462 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:21 smithi110 ceph-mon[212939]: pgmap v1461: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 441 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 11186/21213 objects degraded (52.732%); 5478/21213 objects misplaced (25.824%); 0 B/s, 111 objects/s recovering 2023-09-23T23:32:21.714 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:32:21.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:21.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:32:21.834 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:32:21.879 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:21.939 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:32:21.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:21.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:32:22.043 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T23:32:22.043 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:32:22.046 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:32:22.080 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:22.142 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:32:22.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:22.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:32:22.257 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:32:22.403 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:22.588 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:22.630 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:32:22.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:22.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:32:22.692 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:32:22.693 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:32:22.744 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:22.773 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:22.830 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:22.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:22.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:22.938 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:32:22.975 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:23.037 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:32:23.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:23.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:32:23.146 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:32:23.224 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:23.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:23 smithi138 ceph-mon[193606]: pgmap v1462: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 386 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 11186/18576 objects degraded (60.217%); 5478/18576 objects misplaced (29.490%); 0 B/s, 91 objects/s recovering 2023-09-23T23:32:23.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:23 smithi110 ceph-mon[212939]: pgmap v1462: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 386 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 11186/18576 objects degraded (60.217%); 5478/18576 objects misplaced (29.490%); 0 B/s, 91 objects/s recovering 2023-09-23T23:32:23.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:23 smithi110 ceph-mon[221102]: pgmap v1462: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 386 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 11186/18576 objects degraded (60.217%); 5478/18576 objects misplaced (29.490%); 0 B/s, 91 objects/s recovering 2023-09-23T23:32:24.133 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:32:24.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:24.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:32:24.168 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:32:24.244 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:32:24.275 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:24.292 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:32:24.331 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:32:24.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:24.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:32:24.397 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:24 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1935135881' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:32:24.397 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:24 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:32:24.397 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:24 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 11186/18576 objects degraded (60.217%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:24.397 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:24 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1935135881' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:32:24.398 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:24 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:32:24.398 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:24 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 11186/18576 objects degraded (60.217%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:24.436 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:32:24.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:24 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1935135881' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:32:24.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:24 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:32:24.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:24 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 11186/18576 objects degraded (60.217%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:24.543 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:25.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:25 smithi138 ceph-mon[193606]: pgmap v1463: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 358 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 11186/17226 objects degraded (64.937%); 5478/17226 objects misplaced (31.801%); 0 B/s, 91 objects/s recovering 2023-09-23T23:32:25.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:25 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:32:25.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:25 smithi138 ceph-mon[193606]: failed to encode map e528 with expected crc 2023-09-23T23:32:25.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:25 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:32:25.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:25 smithi138 ceph-mon[193606]: osdmap e528: 8 total, 8 up, 8 in 2023-09-23T23:32:25.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:25 smithi138 ceph-mon[193606]: failed to encode map e528 with expected crc 2023-09-23T23:32:25.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:25 smithi110 ceph-mon[221102]: pgmap v1463: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 358 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 11186/17226 objects degraded (64.937%); 5478/17226 objects misplaced (31.801%); 0 B/s, 91 objects/s recovering 2023-09-23T23:32:25.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:25 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:32:25.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:25 smithi110 ceph-mon[221102]: failed to encode map e528 with expected crc 2023-09-23T23:32:25.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:25 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:32:25.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:25 smithi110 ceph-mon[221102]: osdmap e528: 8 total, 8 up, 8 in 2023-09-23T23:32:25.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:25 smithi110 ceph-mon[221102]: failed to encode map e528 with expected crc 2023-09-23T23:32:25.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:25 smithi110 ceph-mon[212939]: pgmap v1463: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 358 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 11186/17226 objects degraded (64.937%); 5478/17226 objects misplaced (31.801%); 0 B/s, 91 objects/s recovering 2023-09-23T23:32:25.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:25 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:32:25.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:25 smithi110 ceph-mon[212939]: failed to encode map e528 with expected crc 2023-09-23T23:32:25.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:25 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:32:25.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:25 smithi110 ceph-mon[212939]: osdmap e528: 8 total, 8 up, 8 in 2023-09-23T23:32:25.450 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:25 smithi110 ceph-mon[212939]: failed to encode map e528 with expected crc 2023-09-23T23:32:26.158 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:32:26.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:26.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:32:26.266 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:32:26.290 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:26.395 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:26.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:26 smithi138 ceph-mon[193606]: 12.10 scrub starts 2023-09-23T23:32:26.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:26 smithi138 ceph-mon[193606]: 12.2 scrub starts 2023-09-23T23:32:26.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:26 smithi138 ceph-mon[193606]: 12.3 scrub starts 2023-09-23T23:32:26.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:26 smithi110 ceph-mon[212939]: 12.10 scrub starts 2023-09-23T23:32:26.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:26 smithi110 ceph-mon[212939]: 12.2 scrub starts 2023-09-23T23:32:26.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:26 smithi110 ceph-mon[212939]: 12.3 scrub starts 2023-09-23T23:32:26.455 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:26.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:26 smithi110 ceph-mon[221102]: 12.10 scrub starts 2023-09-23T23:32:26.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:26 smithi110 ceph-mon[221102]: 12.2 scrub starts 2023-09-23T23:32:26.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:26 smithi110 ceph-mon[221102]: 12.3 scrub starts 2023-09-23T23:32:26.510 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:32:26.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:26.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:32:26.616 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:32:26.715 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:26.806 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:32:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:32:26] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.43.0" 2023-09-23T23:32:27.448 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:27 smithi138 ceph-mon[193606]: pgmap v1465: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 164 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 10686/7923 objects degraded (134.873%); 5248/7923 objects misplaced (66.238%); 0 B/s, 127 objects/s recovering 2023-09-23T23:32:27.448 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:27 smithi138 ceph-mon[193606]: 12.10 scrub starts 2023-09-23T23:32:27.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:27 smithi110 ceph-mon[212939]: pgmap v1465: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 164 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 10686/7923 objects degraded (134.873%); 5248/7923 objects misplaced (66.238%); 0 B/s, 127 objects/s recovering 2023-09-23T23:32:27.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:27 smithi110 ceph-mon[212939]: 12.10 scrub starts 2023-09-23T23:32:27.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:27 smithi110 ceph-mon[221102]: pgmap v1465: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 164 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 10686/7923 objects degraded (134.873%); 5248/7923 objects misplaced (66.238%); 0 B/s, 127 objects/s recovering 2023-09-23T23:32:27.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:27 smithi110 ceph-mon[221102]: 12.10 scrub starts 2023-09-23T23:32:27.930 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Removed 94872 objects 2023-09-23T23:32:27.930 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Clean up completed and total clean up time :132.654 2023-09-23T23:32:27.993 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:32:27.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:27.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:32:28.097 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:32:28.129 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:28.202 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:32:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:28 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1260485363' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:32:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:32:28.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:28 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1260485363' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:32:28.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:32:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:28 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1260485363' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:32:28.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:32:28.713 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:32:28.714 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:32:28.714 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:32:28.714 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:32:28.714 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:32:28.714 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:32:28.714 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:32:28.714 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:32:28.714 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:32:28.715 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:32:28.715 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:32:28.715 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:32:28.715 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:32:28.715 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:32:28.715 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:32:28.715 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:32:29.237 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:32:29.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:29.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:32:29.341 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:32:29.370 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:29 smithi138 ceph-mon[193606]: pgmap v1466: 129 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 157 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 10686/7575 objects degraded (141.069%); 5248/7575 objects misplaced (69.281%); 0 B/s, 116 objects/s recovering 2023-09-23T23:32:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:29 smithi138 ceph-mon[193606]: failed to encode map e529 with expected crc 2023-09-23T23:32:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:29 smithi138 ceph-mon[193606]: osdmap e529: 8 total, 8 up, 8 in 2023-09-23T23:32:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:29 smithi138 ceph-mon[193606]: failed to encode map e529 with expected crc 2023-09-23T23:32:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:29 smithi138 ceph-mon[193606]: failed to encode map e529 with expected crc 2023-09-23T23:32:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:29 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 10686/7575 objects degraded (141.069%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:29 smithi110 ceph-mon[212939]: pgmap v1466: 129 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 157 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 10686/7575 objects degraded (141.069%); 5248/7575 objects misplaced (69.281%); 0 B/s, 116 objects/s recovering 2023-09-23T23:32:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:29 smithi110 ceph-mon[212939]: failed to encode map e529 with expected crc 2023-09-23T23:32:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:29 smithi110 ceph-mon[212939]: osdmap e529: 8 total, 8 up, 8 in 2023-09-23T23:32:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:29 smithi110 ceph-mon[212939]: failed to encode map e529 with expected crc 2023-09-23T23:32:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:29 smithi110 ceph-mon[212939]: failed to encode map e529 with expected crc 2023-09-23T23:32:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:29 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 10686/7575 objects degraded (141.069%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:29.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:29 smithi110 ceph-mon[221102]: pgmap v1466: 129 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 157 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 10686/7575 objects degraded (141.069%); 5248/7575 objects misplaced (69.281%); 0 B/s, 116 objects/s recovering 2023-09-23T23:32:29.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:29 smithi110 ceph-mon[221102]: failed to encode map e529 with expected crc 2023-09-23T23:32:29.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:29 smithi110 ceph-mon[221102]: osdmap e529: 8 total, 8 up, 8 in 2023-09-23T23:32:29.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:29 smithi110 ceph-mon[221102]: failed to encode map e529 with expected crc 2023-09-23T23:32:29.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:29 smithi110 ceph-mon[221102]: failed to encode map e529 with expected crc 2023-09-23T23:32:29.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:29 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 10686/7575 objects degraded (141.069%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:30.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:30 smithi138 ceph-mon[193606]: 12.12 scrub starts 2023-09-23T23:32:30.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:30 smithi138 ceph-mon[193606]: 12.12 scrub ok 2023-09-23T23:32:30.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:30 smithi138 ceph-mon[193606]: failed to encode map e530 with expected crc 2023-09-23T23:32:30.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:30 smithi138 ceph-mon[193606]: failed to encode map e530 with expected crc 2023-09-23T23:32:30.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:30 smithi138 ceph-mon[193606]: osdmap e530: 8 total, 8 up, 8 in 2023-09-23T23:32:30.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:30 smithi138 ceph-mon[193606]: failed to encode map e530 with expected crc 2023-09-23T23:32:30.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:30 smithi138 ceph-mon[193606]: failed to encode map e530 with expected crc 2023-09-23T23:32:30.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:30 smithi138 ceph-mon[193606]: failed to encode map e530 with expected crc 2023-09-23T23:32:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:30 smithi110 ceph-mon[212939]: 12.12 scrub starts 2023-09-23T23:32:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:30 smithi110 ceph-mon[212939]: 12.12 scrub ok 2023-09-23T23:32:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:30 smithi110 ceph-mon[212939]: failed to encode map e530 with expected crc 2023-09-23T23:32:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:30 smithi110 ceph-mon[212939]: failed to encode map e530 with expected crc 2023-09-23T23:32:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:30 smithi110 ceph-mon[212939]: osdmap e530: 8 total, 8 up, 8 in 2023-09-23T23:32:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:30 smithi110 ceph-mon[212939]: failed to encode map e530 with expected crc 2023-09-23T23:32:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:30 smithi110 ceph-mon[212939]: failed to encode map e530 with expected crc 2023-09-23T23:32:30.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:30 smithi110 ceph-mon[212939]: failed to encode map e530 with expected crc 2023-09-23T23:32:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:30 smithi110 ceph-mon[221102]: 12.12 scrub starts 2023-09-23T23:32:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:30 smithi110 ceph-mon[221102]: 12.12 scrub ok 2023-09-23T23:32:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:30 smithi110 ceph-mon[221102]: failed to encode map e530 with expected crc 2023-09-23T23:32:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:30 smithi110 ceph-mon[221102]: failed to encode map e530 with expected crc 2023-09-23T23:32:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:30 smithi110 ceph-mon[221102]: osdmap e530: 8 total, 8 up, 8 in 2023-09-23T23:32:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:30 smithi110 ceph-mon[221102]: failed to encode map e530 with expected crc 2023-09-23T23:32:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:30 smithi110 ceph-mon[221102]: failed to encode map e530 with expected crc 2023-09-23T23:32:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:30 smithi110 ceph-mon[221102]: failed to encode map e530 with expected crc 2023-09-23T23:32:30.889 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_7 2023-09-23T23:32:30.889 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_7 unique_pool_7 --yes-i-really-really-mean-it 2023-09-23T23:32:31.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:31 smithi138 ceph-mon[193606]: pgmap v1469: 129 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 36 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 10186/1788 objects degraded (569.687%); 5082/1788 objects misplaced (284.228%); 0 B/s, 166 objects/s recovering 2023-09-23T23:32:31.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:31 smithi110 ceph-mon[212939]: pgmap v1469: 129 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 36 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 10186/1788 objects degraded (569.687%); 5082/1788 objects misplaced (284.228%); 0 B/s, 166 objects/s recovering 2023-09-23T23:32:31.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:31 smithi110 ceph-mon[221102]: pgmap v1469: 129 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 36 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 10186/1788 objects degraded (569.687%); 5082/1788 objects misplaced (284.228%); 0 B/s, 166 objects/s recovering 2023-09-23T23:32:32.245 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:32:32.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:32.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:32:32.349 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:32:32.374 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:32 smithi138 ceph-mon[193606]: 12.13 scrub starts 2023-09-23T23:32:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:32 smithi138 ceph-mon[193606]: 12.13 scrub ok 2023-09-23T23:32:32.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:32 smithi110 ceph-mon[212939]: 12.13 scrub starts 2023-09-23T23:32:32.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:32 smithi110 ceph-mon[212939]: 12.13 scrub ok 2023-09-23T23:32:32.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:32 smithi110 ceph-mon[221102]: 12.13 scrub starts 2023-09-23T23:32:32.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:32 smithi110 ceph-mon[221102]: 12.13 scrub ok 2023-09-23T23:32:32.762 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:32.803 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:32:32.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:32.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:32:32.842 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:32.908 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:32:32.934 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:32.978 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:32:32.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:32.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:32:33.085 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:33.110 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:33.156 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:33.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:33.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:33.191 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:33 smithi138 ceph-mon[193606]: pgmap v1470: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 17 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 9226/840 objects degraded (1098.333%); 5082/840 objects misplaced (605.000%); 0 B/s, 254 objects/s recovering 2023-09-23T23:32:33.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:33 smithi138 ceph-mon[193606]: 12.10 scrub starts 2023-09-23T23:32:33.261 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:32:33.294 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:33.334 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:32:33.334 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:32:33.334 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:32:33.334 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:32:33.334 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:32:33.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:33 smithi110 ceph-mon[212939]: pgmap v1470: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 17 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 9226/840 objects degraded (1098.333%); 5082/840 objects misplaced (605.000%); 0 B/s, 254 objects/s recovering 2023-09-23T23:32:33.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:33 smithi110 ceph-mon[212939]: 12.10 scrub starts 2023-09-23T23:32:33.359 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:33 smithi110 ceph-mon[221102]: pgmap v1470: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 17 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 9226/840 objects degraded (1098.333%); 5082/840 objects misplaced (605.000%); 0 B/s, 254 objects/s recovering 2023-09-23T23:32:33.359 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:33 smithi110 ceph-mon[221102]: 12.10 scrub starts 2023-09-23T23:32:33.731 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:32:33.733 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:32:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:34 smithi138 ceph-mon[193606]: 12.8 scrub starts 2023-09-23T23:32:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:34 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 9226/840 objects degraded (1098.333%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:34.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:34 smithi110 ceph-mon[221102]: 12.8 scrub starts 2023-09-23T23:32:34.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:34 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 9226/840 objects degraded (1098.333%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:34 smithi110 ceph-mon[212939]: 12.8 scrub starts 2023-09-23T23:32:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:34 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 9226/840 objects degraded (1098.333%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:34.712 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:32:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:35 smithi138 ceph-mon[193606]: 12.18 scrub starts 2023-09-23T23:32:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:35 smithi138 ceph-mon[193606]: from='client.46454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:32:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:35 smithi138 ceph-mon[193606]: pgmap v1471: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 16 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 9226/807 objects degraded (1143.247%); 5082/807 objects misplaced (629.740%); 0 B/s, 182 objects/s recovering 2023-09-23T23:32:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:35 smithi138 ceph-mon[193606]: 12.0 scrub starts 2023-09-23T23:32:35.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:35 smithi138 ceph-mon[193606]: osdmap e531: 8 total, 8 up, 8 in 2023-09-23T23:32:35.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:35 smithi110 ceph-mon[221102]: 12.18 scrub starts 2023-09-23T23:32:35.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:35 smithi110 ceph-mon[221102]: from='client.46454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:32:35.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:35 smithi110 ceph-mon[221102]: pgmap v1471: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 16 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 9226/807 objects degraded (1143.247%); 5082/807 objects misplaced (629.740%); 0 B/s, 182 objects/s recovering 2023-09-23T23:32:35.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:35 smithi110 ceph-mon[221102]: 12.0 scrub starts 2023-09-23T23:32:35.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:35 smithi110 ceph-mon[221102]: osdmap e531: 8 total, 8 up, 8 in 2023-09-23T23:32:35.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:35 smithi110 ceph-mon[212939]: 12.18 scrub starts 2023-09-23T23:32:35.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:35 smithi110 ceph-mon[212939]: from='client.46454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:32:35.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:35 smithi110 ceph-mon[212939]: pgmap v1471: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 16 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 9226/807 objects degraded (1143.247%); 5082/807 objects misplaced (629.740%); 0 B/s, 182 objects/s recovering 2023-09-23T23:32:35.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:35 smithi110 ceph-mon[212939]: 12.0 scrub starts 2023-09-23T23:32:35.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:35 smithi110 ceph-mon[212939]: osdmap e531: 8 total, 8 up, 8 in 2023-09-23T23:32:35.900 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:32:35.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:35.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:32:36.004 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:32:36.081 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:36.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:36 smithi138 ceph-mon[193606]: failed to encode map e531 with expected crc 2023-09-23T23:32:36.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:36 smithi138 ceph-mon[193606]: failed to encode map e531 with expected crc 2023-09-23T23:32:36.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:36 smithi138 ceph-mon[193606]: 12.2 scrub starts 2023-09-23T23:32:36.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:36 smithi138 ceph-mon[193606]: osdmap e532: 8 total, 8 up, 8 in 2023-09-23T23:32:36.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:36 smithi110 ceph-mon[212939]: failed to encode map e531 with expected crc 2023-09-23T23:32:36.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:36 smithi110 ceph-mon[212939]: failed to encode map e531 with expected crc 2023-09-23T23:32:36.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:36 smithi110 ceph-mon[212939]: 12.2 scrub starts 2023-09-23T23:32:36.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:36 smithi110 ceph-mon[212939]: osdmap e532: 8 total, 8 up, 8 in 2023-09-23T23:32:36.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:36 smithi110 ceph-mon[221102]: failed to encode map e531 with expected crc 2023-09-23T23:32:36.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:36 smithi110 ceph-mon[221102]: failed to encode map e531 with expected crc 2023-09-23T23:32:36.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:36 smithi110 ceph-mon[221102]: 12.2 scrub starts 2023-09-23T23:32:36.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:36 smithi110 ceph-mon[221102]: osdmap e532: 8 total, 8 up, 8 in 2023-09-23T23:32:36.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:32:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:32:36] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.43.0" 2023-09-23T23:32:37.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:37 smithi138 ceph-mon[193606]: failed to encode map e532 with expected crc 2023-09-23T23:32:37.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:37 smithi138 ceph-mon[193606]: failed to encode map e532 with expected crc 2023-09-23T23:32:37.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:37 smithi138 ceph-mon[193606]: failed to encode map e532 with expected crc 2023-09-23T23:32:37.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:37 smithi138 ceph-mon[193606]: pgmap v1474: 129 pgs: 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 122 active+clean; 14 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 9146/717 objects degraded (1275.593%); 5082/717 objects misplaced (708.787%); 0 B/s, 230 objects/s recovering 2023-09-23T23:32:37.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:37 smithi138 ceph-mon[193606]: 12.3 scrub starts 2023-09-23T23:32:37.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:37 smithi110 ceph-mon[212939]: failed to encode map e532 with expected crc 2023-09-23T23:32:37.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:37 smithi110 ceph-mon[212939]: failed to encode map e532 with expected crc 2023-09-23T23:32:37.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:37 smithi110 ceph-mon[212939]: failed to encode map e532 with expected crc 2023-09-23T23:32:37.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:37 smithi110 ceph-mon[212939]: pgmap v1474: 129 pgs: 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 122 active+clean; 14 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 9146/717 objects degraded (1275.593%); 5082/717 objects misplaced (708.787%); 0 B/s, 230 objects/s recovering 2023-09-23T23:32:37.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:37 smithi110 ceph-mon[212939]: 12.3 scrub starts 2023-09-23T23:32:37.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:37 smithi110 ceph-mon[221102]: failed to encode map e532 with expected crc 2023-09-23T23:32:37.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:37 smithi110 ceph-mon[221102]: failed to encode map e532 with expected crc 2023-09-23T23:32:37.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:37 smithi110 ceph-mon[221102]: failed to encode map e532 with expected crc 2023-09-23T23:32:37.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:37 smithi110 ceph-mon[221102]: pgmap v1474: 129 pgs: 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 122 active+clean; 14 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 9146/717 objects degraded (1275.593%); 5082/717 objects misplaced (708.787%); 0 B/s, 230 objects/s recovering 2023-09-23T23:32:37.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:37 smithi110 ceph-mon[221102]: 12.3 scrub starts 2023-09-23T23:32:38.291 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1470,"stamp":"2023-09-23T23:32:31.845594+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":17452041,"num_objects":280,"num_object_clones":0,"num_object_copies":840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9226,"num_objects_misplaced":5082,"num_objects_unfound":0,"num_objects_dirty":280,"num_whiteouts":0,"num_read":12,"num_read_kb":21,"num_write":189812,"num_write_kb":6071893,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18454,"num_bytes_recovered":750306792,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":189274,"ondisk_log_size":189274,"up":387,"acting":384,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":401,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7230400,"kb_used_data":1340672,"kb_used_omap":29,"kb_used_meta":5889634,"kb_avail":742730816,"statfs":{"total":767960285184,"available":760556355584,"internally_reserved":0,"allocated":1372848128,"data_stored":1357135236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":6030985939},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-297992216,"num_objects":-4549,"num_object_clones":0,"num_object_copies":-13647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1960,"num_objects_misplaced":-396,"num_objects_unfound":0,"num_objects_dirty":-4549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4805,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1960,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.714392"},"pg_stats":[{"pgid":"12.15","version":"528'8384","reported_seq":10380,"reported_epoch":530,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:32:30.434645+0000","last_change":"2023-09-23T23:32:28.067470+0000","last_active":"2023-09-23T23:32:30.434645+0000","last_peered":"2023-09-23T23:32:30.434645+0000","last_clean":"2023-09-23T23:29:52.317573+0000","last_became_active":"2023-09-23T23:29:56.373989+0000","last_became_peered":"2023-09-23T23:29:56.373989+0000","last_unstale":"2023-09-23T23:32:30.434645+0000","last_undegraded":"2023-09-23T23:29:56.182909+0000","last_fullsized":"2023-09-23T23:29:56.153881+0000","mapping_epoch":496,"log_start":"484'1541","ondisk_log_start":"484'1541","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":6843,"log_dups_size":0,"ondisk_log_size":6843,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1604,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6037,"num_write_kb":192768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":960,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[1,2],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2","objects":-1604}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":725,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.161788+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:32:24.161788+0000","last_peered":"2023-09-23T23:32:24.161788+0000","last_clean":"2023-09-23T23:32:24.161788+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:32:24.161788+0000","last_undegraded":"2023-09-23T23:32:24.161788+0000","last_fullsized":"2023-09-23T23:32:24.161788+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":563,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.149790+0000","last_change":"2023-09-23T23:22:38.439629+0000","last_active":"2023-09-23T23:32:24.149790+0000","last_peered":"2023-09-23T23:32:24.149790+0000","last_clean":"2023-09-23T23:32:24.149790+0000","last_became_active":"2023-09-23T23:22:38.439285+0000","last_became_peered":"2023-09-23T23:22:38.439285+0000","last_unstale":"2023-09-23T23:32:24.149790+0000","last_undegraded":"2023-09-23T23:32:24.149790+0000","last_fullsized":"2023-09-23T23:32:24.149790+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:34.431221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":501,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.162310+0000","last_change":"2023-09-23T23:23:43.511912+0000","last_active":"2023-09-23T23:32:28.162310+0000","last_peered":"2023-09-23T23:32:28.162310+0000","last_clean":"2023-09-23T23:32:28.162310+0000","last_became_active":"2023-09-23T23:23:43.511574+0000","last_became_peered":"2023-09-23T23:23:43.511574+0000","last_unstale":"2023-09-23T23:32:28.162310+0000","last_undegraded":"2023-09-23T23:32:28.162310+0000","last_fullsized":"2023-09-23T23:32:28.162310+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:16.286926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.16","version":"528'8178","reported_seq":8252,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.163005+0000","last_change":"2023-09-23T23:29:45.671198+0000","last_active":"2023-09-23T23:32:28.163005+0000","last_peered":"2023-09-23T23:32:28.163005+0000","last_clean":"2023-09-23T23:32:28.163005+0000","last_became_active":"2023-09-23T23:29:45.669030+0000","last_became_peered":"2023-09-23T23:29:45.669030+0000","last_unstale":"2023-09-23T23:32:28.163005+0000","last_undegraded":"2023-09-23T23:32:28.163005+0000","last_fullsized":"2023-09-23T23:32:28.163005+0000","mapping_epoch":490,"log_start":"484'1478","ondisk_log_start":"484'1478","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":6700,"log_dups_size":0,"ondisk_log_size":6700,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5961,"num_write_kb":188256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":618,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.162059+0000","last_change":"2023-09-23T23:23:50.459634+0000","last_active":"2023-09-23T23:32:24.162059+0000","last_peered":"2023-09-23T23:32:24.162059+0000","last_clean":"2023-09-23T23:32:24.162059+0000","last_became_active":"2023-09-23T23:23:45.453958+0000","last_became_peered":"2023-09-23T23:23:45.453958+0000","last_unstale":"2023-09-23T23:32:24.162059+0000","last_undegraded":"2023-09-23T23:32:24.162059+0000","last_fullsized":"2023-09-23T23:32:24.162059+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:56.563201+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":242,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.164313+0000","last_change":"2023-09-23T23:26:11.528846+0000","last_active":"2023-09-23T23:32:28.164313+0000","last_peered":"2023-09-23T23:32:28.164313+0000","last_clean":"2023-09-23T23:32:28.164313+0000","last_became_active":"2023-09-23T23:26:11.528529+0000","last_became_peered":"2023-09-23T23:26:11.528529+0000","last_unstale":"2023-09-23T23:32:28.164313+0000","last_undegraded":"2023-09-23T23:32:28.164313+0000","last_fullsized":"2023-09-23T23:32:28.164313+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:35.094583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":494,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.145376+0000","last_change":"2023-09-23T23:28:22.604718+0000","last_active":"2023-09-23T23:32:24.145376+0000","last_peered":"2023-09-23T23:32:24.145376+0000","last_clean":"2023-09-23T23:32:24.145376+0000","last_became_active":"2023-09-23T23:28:22.604498+0000","last_became_peered":"2023-09-23T23:28:22.604498+0000","last_unstale":"2023-09-23T23:32:24.145376+0000","last_undegraded":"2023-09-23T23:32:24.145376+0000","last_fullsized":"2023-09-23T23:32:24.145376+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:30.472412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.17","version":"528'8148","reported_seq":8222,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.164768+0000","last_change":"2023-09-23T23:29:45.685688+0000","last_active":"2023-09-23T23:32:28.164768+0000","last_peered":"2023-09-23T23:32:28.164768+0000","last_clean":"2023-09-23T23:32:28.164768+0000","last_became_active":"2023-09-23T23:29:45.681573+0000","last_became_peered":"2023-09-23T23:29:45.681573+0000","last_unstale":"2023-09-23T23:32:28.164768+0000","last_undegraded":"2023-09-23T23:32:28.164768+0000","last_fullsized":"2023-09-23T23:32:28.164768+0000","mapping_epoch":490,"log_start":"484'1233","ondisk_log_start":"484'1233","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":6915,"log_dups_size":0,"ondisk_log_size":6915,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5863,"num_write_kb":187360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":832,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.149742+0000","last_change":"2023-09-23T23:28:05.634824+0000","last_active":"2023-09-23T23:32:24.149742+0000","last_peered":"2023-09-23T23:32:24.149742+0000","last_clean":"2023-09-23T23:32:24.149742+0000","last_became_active":"2023-09-23T23:27:23.551046+0000","last_became_peered":"2023-09-23T23:27:23.551046+0000","last_unstale":"2023-09-23T23:32:24.149742+0000","last_undegraded":"2023-09-23T23:32:24.149742+0000","last_fullsized":"2023-09-23T23:32:24.149742+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:59.062239+0000","scrub_duration":0.0034158959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":528,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.163947+0000","last_change":"2023-09-23T23:27:20.554007+0000","last_active":"2023-09-23T23:32:28.163947+0000","last_peered":"2023-09-23T23:32:28.163947+0000","last_clean":"2023-09-23T23:32:28.163947+0000","last_became_active":"2023-09-23T23:27:20.553716+0000","last_became_peered":"2023-09-23T23:27:20.553716+0000","last_unstale":"2023-09-23T23:32:28.163947+0000","last_undegraded":"2023-09-23T23:32:28.163947+0000","last_fullsized":"2023-09-23T23:32:28.163947+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:38.911862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":43,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.130555+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:32:24.130555+0000","last_peered":"2023-09-23T23:32:24.130555+0000","last_clean":"2023-09-23T23:32:24.130555+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:32:24.130555+0000","last_undegraded":"2023-09-23T23:32:24.130555+0000","last_fullsized":"2023-09-23T23:32:24.130555+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.18","version":"528'8124","reported_seq":8198,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.163785+0000","last_change":"2023-09-23T23:29:45.694505+0000","last_active":"2023-09-23T23:32:28.163785+0000","last_peered":"2023-09-23T23:32:28.163785+0000","last_clean":"2023-09-23T23:32:28.163785+0000","last_became_active":"2023-09-23T23:29:45.692613+0000","last_became_peered":"2023-09-23T23:29:45.692613+0000","last_unstale":"2023-09-23T23:32:28.163785+0000","last_undegraded":"2023-09-23T23:32:28.163785+0000","last_fullsized":"2023-09-23T23:32:28.163785+0000","mapping_epoch":490,"log_start":"484'1529","ondisk_log_start":"484'1529","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":6595,"log_dups_size":0,"ondisk_log_size":6595,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5854,"num_write_kb":186560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":849,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.145284+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:32:24.145284+0000","last_peered":"2023-09-23T23:32:24.145284+0000","last_clean":"2023-09-23T23:32:24.145284+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:32:24.145284+0000","last_undegraded":"2023-09-23T23:32:24.145284+0000","last_fullsized":"2023-09-23T23:32:24.145284+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":161,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.164019+0000","last_change":"2023-09-23T23:28:22.602481+0000","last_active":"2023-09-23T23:32:24.164019+0000","last_peered":"2023-09-23T23:32:24.164019+0000","last_clean":"2023-09-23T23:32:24.164019+0000","last_became_active":"2023-09-23T23:28:22.602282+0000","last_became_peered":"2023-09-23T23:28:22.602282+0000","last_unstale":"2023-09-23T23:32:24.164019+0000","last_undegraded":"2023-09-23T23:32:24.164019+0000","last_fullsized":"2023-09-23T23:32:24.164019+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:39.906003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":529,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.163867+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:32:28.163867+0000","last_peered":"2023-09-23T23:32:28.163867+0000","last_clean":"2023-09-23T23:32:28.163867+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:32:28.163867+0000","last_undegraded":"2023-09-23T23:32:28.163867+0000","last_fullsized":"2023-09-23T23:32:28.163867+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.19","version":"528'8063","reported_seq":13143,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T23:32:29.188032+0000","last_change":"2023-09-23T23:32:29.188032+0000","last_active":"2023-09-23T23:32:29.188032+0000","last_peered":"2023-09-23T23:32:29.188032+0000","last_clean":"2023-09-23T23:32:29.188032+0000","last_became_active":"2023-09-23T23:32:29.168946+0000","last_became_peered":"2023-09-23T23:32:29.168946+0000","last_unstale":"2023-09-23T23:32:29.188032+0000","last_undegraded":"2023-09-23T23:32:29.188032+0000","last_fullsized":"2023-09-23T23:32:29.188032+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":8063,"log_dups_size":0,"ondisk_log_size":8063,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5828,"num_write_kb":185888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2494,"num_bytes_recovered":7143424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":804,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.157440+0000","last_change":"2023-09-23T23:26:14.521792+0000","last_active":"2023-09-23T23:32:28.157440+0000","last_peered":"2023-09-23T23:32:28.157440+0000","last_clean":"2023-09-23T23:32:28.157440+0000","last_became_active":"2023-09-23T23:26:14.521438+0000","last_became_peered":"2023-09-23T23:26:14.521438+0000","last_unstale":"2023-09-23T23:32:28.157440+0000","last_undegraded":"2023-09-23T23:32:28.157440+0000","last_fullsized":"2023-09-23T23:32:28.157440+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:09.103712+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":160,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.162596+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:32:24.162596+0000","last_peered":"2023-09-23T23:32:24.162596+0000","last_clean":"2023-09-23T23:32:24.162596+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:32:24.162596+0000","last_undegraded":"2023-09-23T23:32:24.162596+0000","last_fullsized":"2023-09-23T23:32:24.162596+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":490,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.163984+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:32:28.163984+0000","last_peered":"2023-09-23T23:32:28.163984+0000","last_clean":"2023-09-23T23:32:28.163984+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:32:28.163984+0000","last_undegraded":"2023-09-23T23:32:28.163984+0000","last_fullsized":"2023-09-23T23:32:28.163984+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":943,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.163811+0000","last_change":"2023-09-23T23:29:58.260674+0000","last_active":"2023-09-23T23:32:24.163811+0000","last_peered":"2023-09-23T23:32:24.163811+0000","last_clean":"2023-09-23T23:32:24.163811+0000","last_became_active":"2023-09-23T23:29:58.260318+0000","last_became_peered":"2023-09-23T23:29:58.260318+0000","last_unstale":"2023-09-23T23:32:24.163811+0000","last_undegraded":"2023-09-23T23:32:24.163811+0000","last_fullsized":"2023-09-23T23:32:24.163811+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:08.810582+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":134,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.144897+0000","last_change":"2023-09-23T23:26:48.669857+0000","last_active":"2023-09-23T23:32:24.144897+0000","last_peered":"2023-09-23T23:32:24.144897+0000","last_clean":"2023-09-23T23:32:24.144897+0000","last_became_active":"2023-09-23T23:26:48.538246+0000","last_became_peered":"2023-09-23T23:26:48.538246+0000","last_unstale":"2023-09-23T23:32:24.144897+0000","last_undegraded":"2023-09-23T23:32:24.144897+0000","last_fullsized":"2023-09-23T23:32:24.144897+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:19.039472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":127,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.161883+0000","last_change":"2023-09-23T23:29:54.996097+0000","last_active":"2023-09-23T23:32:28.161883+0000","last_peered":"2023-09-23T23:32:28.161883+0000","last_clean":"2023-09-23T23:32:28.161883+0000","last_became_active":"2023-09-23T23:29:54.995756+0000","last_became_peered":"2023-09-23T23:29:54.995756+0000","last_unstale":"2023-09-23T23:32:28.161883+0000","last_undegraded":"2023-09-23T23:32:28.161883+0000","last_fullsized":"2023-09-23T23:32:28.161883+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:15.856474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":606,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.163365+0000","last_change":"2023-09-23T23:24:02.268079+0000","last_active":"2023-09-23T23:32:28.163365+0000","last_peered":"2023-09-23T23:32:28.163365+0000","last_clean":"2023-09-23T23:32:28.163365+0000","last_became_active":"2023-09-23T23:23:45.463106+0000","last_became_peered":"2023-09-23T23:23:45.463106+0000","last_unstale":"2023-09-23T23:32:28.163365+0000","last_undegraded":"2023-09-23T23:32:28.163365+0000","last_fullsized":"2023-09-23T23:32:28.163365+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.128813+0000","scrub_duration":0.0018535310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":180,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.149352+0000","last_change":"2023-09-23T23:29:55.012082+0000","last_active":"2023-09-23T23:32:24.149352+0000","last_peered":"2023-09-23T23:32:24.149352+0000","last_clean":"2023-09-23T23:32:24.149352+0000","last_became_active":"2023-09-23T23:29:55.000202+0000","last_became_peered":"2023-09-23T23:29:55.000202+0000","last_unstale":"2023-09-23T23:32:24.149352+0000","last_undegraded":"2023-09-23T23:32:24.149352+0000","last_fullsized":"2023-09-23T23:32:24.149352+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:28.669994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":527,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.147295+0000","last_change":"2023-09-23T23:29:55.004242+0000","last_active":"2023-09-23T23:32:24.147295+0000","last_peered":"2023-09-23T23:32:24.147295+0000","last_clean":"2023-09-23T23:32:24.147295+0000","last_became_active":"2023-09-23T23:29:55.003934+0000","last_became_peered":"2023-09-23T23:29:55.003934+0000","last_unstale":"2023-09-23T23:32:24.147295+0000","last_undegraded":"2023-09-23T23:32:24.147295+0000","last_fullsized":"2023-09-23T23:32:24.147295+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:09.283771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":644,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.162118+0000","last_change":"2023-09-23T23:28:14.378358+0000","last_active":"2023-09-23T23:32:24.162118+0000","last_peered":"2023-09-23T23:32:24.162118+0000","last_clean":"2023-09-23T23:32:24.162118+0000","last_became_active":"2023-09-23T23:26:34.550151+0000","last_became_peered":"2023-09-23T23:26:34.550151+0000","last_unstale":"2023-09-23T23:32:24.162118+0000","last_undegraded":"2023-09-23T23:32:24.162118+0000","last_fullsized":"2023-09-23T23:32:24.162118+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:47.794690+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":461,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.145269+0000","last_change":"2023-09-23T23:26:11.569228+0000","last_active":"2023-09-23T23:32:24.145269+0000","last_peered":"2023-09-23T23:32:24.145269+0000","last_clean":"2023-09-23T23:32:24.145269+0000","last_became_active":"2023-09-23T23:26:11.513159+0000","last_became_peered":"2023-09-23T23:26:11.513159+0000","last_unstale":"2023-09-23T23:32:24.145269+0000","last_undegraded":"2023-09-23T23:32:24.145269+0000","last_fullsized":"2023-09-23T23:32:24.145269+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:40.698257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":528,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.163157+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:32:28.163157+0000","last_peered":"2023-09-23T23:32:28.163157+0000","last_clean":"2023-09-23T23:32:28.163157+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:32:28.163157+0000","last_undegraded":"2023-09-23T23:32:28.163157+0000","last_fullsized":"2023-09-23T23:32:28.163157+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":888,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.144331+0000","last_change":"2023-09-23T23:29:58.259420+0000","last_active":"2023-09-23T23:32:24.144331+0000","last_peered":"2023-09-23T23:32:24.144331+0000","last_clean":"2023-09-23T23:32:24.144331+0000","last_became_active":"2023-09-23T23:29:58.258879+0000","last_became_peered":"2023-09-23T23:29:58.258879+0000","last_unstale":"2023-09-23T23:32:24.144331+0000","last_undegraded":"2023-09-23T23:32:24.144331+0000","last_fullsized":"2023-09-23T23:32:24.144331+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:14.307363+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":128,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.144813+0000","last_change":"2023-09-23T23:26:48.563723+0000","last_active":"2023-09-23T23:32:24.144813+0000","last_peered":"2023-09-23T23:32:24.144813+0000","last_clean":"2023-09-23T23:32:24.144813+0000","last_became_active":"2023-09-23T23:26:48.563340+0000","last_became_peered":"2023-09-23T23:26:48.563340+0000","last_unstale":"2023-09-23T23:32:24.144813+0000","last_undegraded":"2023-09-23T23:32:24.144813+0000","last_fullsized":"2023-09-23T23:32:24.144813+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:05.974131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":415,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.145215+0000","last_change":"2023-09-23T23:26:32.040316+0000","last_active":"2023-09-23T23:32:24.145215+0000","last_peered":"2023-09-23T23:32:24.145215+0000","last_clean":"2023-09-23T23:32:24.145215+0000","last_became_active":"2023-09-23T23:26:32.039758+0000","last_became_peered":"2023-09-23T23:26:32.039758+0000","last_unstale":"2023-09-23T23:32:24.145215+0000","last_undegraded":"2023-09-23T23:32:24.145215+0000","last_fullsized":"2023-09-23T23:32:24.145215+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:03.930331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":695,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.162447+0000","last_change":"2023-09-23T23:26:14.527752+0000","last_active":"2023-09-23T23:32:28.162447+0000","last_peered":"2023-09-23T23:32:28.162447+0000","last_clean":"2023-09-23T23:32:28.162447+0000","last_became_active":"2023-09-23T23:26:14.525188+0000","last_became_peered":"2023-09-23T23:26:14.525188+0000","last_unstale":"2023-09-23T23:32:28.162447+0000","last_undegraded":"2023-09-23T23:32:28.162447+0000","last_fullsized":"2023-09-23T23:32:28.162447+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:10.617744+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":393,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.144464+0000","last_change":"2023-09-23T23:29:54.999839+0000","last_active":"2023-09-23T23:32:24.144464+0000","last_peered":"2023-09-23T23:32:24.144464+0000","last_clean":"2023-09-23T23:32:24.144464+0000","last_became_active":"2023-09-23T23:29:54.999530+0000","last_became_peered":"2023-09-23T23:29:54.999530+0000","last_unstale":"2023-09-23T23:32:24.144464+0000","last_undegraded":"2023-09-23T23:32:24.144464+0000","last_fullsized":"2023-09-23T23:32:24.144464+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:43.757346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":466,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.162478+0000","last_change":"2023-09-23T23:28:22.603836+0000","last_active":"2023-09-23T23:32:28.162478+0000","last_peered":"2023-09-23T23:32:28.162478+0000","last_clean":"2023-09-23T23:32:28.162478+0000","last_became_active":"2023-09-23T23:28:22.603490+0000","last_became_peered":"2023-09-23T23:28:22.603490+0000","last_unstale":"2023-09-23T23:32:28.162478+0000","last_undegraded":"2023-09-23T23:32:28.162478+0000","last_fullsized":"2023-09-23T23:32:28.162478+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:00.869241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":892,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.161915+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:32:24.161915+0000","last_peered":"2023-09-23T23:32:24.161915+0000","last_clean":"2023-09-23T23:32:24.161915+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:32:24.161915+0000","last_undegraded":"2023-09-23T23:32:24.161915+0000","last_fullsized":"2023-09-23T23:32:24.161915+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":53,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.128704+0000","last_change":"2023-09-23T23:29:55.020386+0000","last_active":"2023-09-23T23:32:24.128704+0000","last_peered":"2023-09-23T23:32:24.128704+0000","last_clean":"2023-09-23T23:32:24.128704+0000","last_became_active":"2023-09-23T23:29:54.991929+0000","last_became_peered":"2023-09-23T23:29:54.991929+0000","last_unstale":"2023-09-23T23:32:24.128704+0000","last_undegraded":"2023-09-23T23:32:24.128704+0000","last_fullsized":"2023-09-23T23:32:24.128704+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:25.161647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":169,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.162728+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:32:28.162728+0000","last_peered":"2023-09-23T23:32:28.162728+0000","last_clean":"2023-09-23T23:32:28.162728+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:32:28.162728+0000","last_undegraded":"2023-09-23T23:32:28.162728+0000","last_fullsized":"2023-09-23T23:32:28.162728+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"528'8285","reported_seq":13504,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:27.908332+0000","last_change":"2023-09-23T23:31:56.136846+0000","last_active":"2023-09-23T23:32:27.908332+0000","last_peered":"2023-09-23T23:32:27.908332+0000","last_clean":"2023-09-23T23:32:27.908332+0000","last_became_active":"2023-09-23T23:30:16.652602+0000","last_became_peered":"2023-09-23T23:30:16.652602+0000","last_unstale":"2023-09-23T23:32:27.908332+0000","last_undegraded":"2023-09-23T23:32:27.908332+0000","last_fullsized":"2023-09-23T23:32:27.908332+0000","mapping_epoch":501,"log_start":"482'207","ondisk_log_start":"482'207","created":476,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":8078,"log_dups_size":0,"ondisk_log_size":8078,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5967,"num_write_kb":190336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2556,"num_bytes_recovered":167510016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":856,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.144992+0000","last_change":"2023-09-23T23:26:51.563868+0000","last_active":"2023-09-23T23:32:24.144992+0000","last_peered":"2023-09-23T23:32:24.144992+0000","last_clean":"2023-09-23T23:32:24.144992+0000","last_became_active":"2023-09-23T23:26:51.563510+0000","last_became_peered":"2023-09-23T23:26:51.563510+0000","last_unstale":"2023-09-23T23:32:24.144992+0000","last_undegraded":"2023-09-23T23:32:24.144992+0000","last_fullsized":"2023-09-23T23:32:24.144992+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:34.244822+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":132,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.145047+0000","last_change":"2023-09-23T23:26:48.673221+0000","last_active":"2023-09-23T23:32:24.145047+0000","last_peered":"2023-09-23T23:32:24.145047+0000","last_clean":"2023-09-23T23:32:24.145047+0000","last_became_active":"2023-09-23T23:26:48.555920+0000","last_became_peered":"2023-09-23T23:26:48.555920+0000","last_unstale":"2023-09-23T23:32:24.145047+0000","last_undegraded":"2023-09-23T23:32:24.145047+0000","last_fullsized":"2023-09-23T23:32:24.145047+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:26.586824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":160,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.161712+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:32:24.161712+0000","last_peered":"2023-09-23T23:32:24.161712+0000","last_clean":"2023-09-23T23:32:24.161712+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:32:24.161712+0000","last_undegraded":"2023-09-23T23:32:24.161712+0000","last_fullsized":"2023-09-23T23:32:24.161712+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"528'8264","reported_seq":8343,"reported_epoch":529,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T23:32:28.163381+0000","last_change":"2023-09-23T23:29:56.405351+0000","last_active":"2023-09-23T23:32:28.163381+0000","last_peered":"2023-09-23T23:32:28.163381+0000","last_clean":"2023-09-23T23:29:52.292686+0000","last_became_active":"2023-09-23T23:29:56.397141+0000","last_became_peered":"2023-09-23T23:29:56.397141+0000","last_unstale":"2023-09-23T23:32:28.163381+0000","last_undegraded":"2023-09-23T23:32:28.163381+0000","last_fullsized":"2023-09-23T23:32:28.163381+0000","mapping_epoch":496,"log_start":"484'1285","ondisk_log_start":"484'1285","created":476,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":6979,"log_dups_size":0,"ondisk_log_size":6979,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2490368,"num_objects":38,"num_object_clones":0,"num_object_copies":114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2496,"num_objects_unfound":0,"num_objects_dirty":38,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5938,"num_write_kb":191232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[1,3,7],"avail_no_missing":["1","3","7"],"object_location_counts":[{"shards":"1,3,7","objects":-2458}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":940,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.162977+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:32:28.162977+0000","last_peered":"2023-09-23T23:32:28.162977+0000","last_clean":"2023-09-23T23:32:28.162977+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:32:28.162977+0000","last_undegraded":"2023-09-23T23:32:28.162977+0000","last_fullsized":"2023-09-23T23:32:28.162977+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":132,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.145524+0000","last_change":"2023-09-23T23:26:48.547146+0000","last_active":"2023-09-23T23:32:24.145524+0000","last_peered":"2023-09-23T23:32:24.145524+0000","last_clean":"2023-09-23T23:32:24.145524+0000","last_became_active":"2023-09-23T23:26:48.537682+0000","last_became_peered":"2023-09-23T23:26:48.537682+0000","last_unstale":"2023-09-23T23:32:24.145524+0000","last_undegraded":"2023-09-23T23:32:24.145524+0000","last_fullsized":"2023-09-23T23:32:24.145524+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:10.115563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":554,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.149349+0000","last_change":"2023-09-23T23:26:01.833889+0000","last_active":"2023-09-23T23:32:24.149349+0000","last_peered":"2023-09-23T23:32:24.149349+0000","last_clean":"2023-09-23T23:32:24.149349+0000","last_became_active":"2023-09-23T23:26:01.833626+0000","last_became_peered":"2023-09-23T23:26:01.833626+0000","last_unstale":"2023-09-23T23:32:24.149349+0000","last_undegraded":"2023-09-23T23:32:24.149349+0000","last_fullsized":"2023-09-23T23:32:24.149349+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:25.659671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"528'8320","reported_seq":8419,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:27.917708+0000","last_change":"2023-09-23T23:32:25.144912+0000","last_active":"2023-09-23T23:32:27.917708+0000","last_peered":"2023-09-23T23:32:27.917708+0000","last_clean":"2023-09-23T23:32:27.917708+0000","last_became_active":"2023-09-23T23:29:45.617669+0000","last_became_peered":"2023-09-23T23:29:45.617669+0000","last_unstale":"2023-09-23T23:32:27.917708+0000","last_undegraded":"2023-09-23T23:32:27.917708+0000","last_fullsized":"2023-09-23T23:32:27.917708+0000","mapping_epoch":476,"log_start":"483'292","ondisk_log_start":"483'292","created":476,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":8028,"log_dups_size":0,"ondisk_log_size":8028,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5996,"num_write_kb":190912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":819,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.130566+0000","last_change":"2023-09-23T23:29:58.262103+0000","last_active":"2023-09-23T23:32:24.130566+0000","last_peered":"2023-09-23T23:32:24.130566+0000","last_clean":"2023-09-23T23:32:24.130566+0000","last_became_active":"2023-09-23T23:29:58.261689+0000","last_became_peered":"2023-09-23T23:29:58.261689+0000","last_unstale":"2023-09-23T23:32:24.130566+0000","last_undegraded":"2023-09-23T23:32:24.130566+0000","last_fullsized":"2023-09-23T23:32:24.130566+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:44.945353+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":449,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.145085+0000","last_change":"2023-09-23T23:29:55.006616+0000","last_active":"2023-09-23T23:32:24.145085+0000","last_peered":"2023-09-23T23:32:24.145085+0000","last_clean":"2023-09-23T23:32:24.145085+0000","last_became_active":"2023-09-23T23:29:55.000411+0000","last_became_peered":"2023-09-23T23:29:55.000411+0000","last_unstale":"2023-09-23T23:32:24.145085+0000","last_undegraded":"2023-09-23T23:32:24.145085+0000","last_fullsized":"2023-09-23T23:32:24.145085+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:22.629830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":43,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.128770+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:32:24.128770+0000","last_peered":"2023-09-23T23:32:24.128770+0000","last_clean":"2023-09-23T23:32:24.128770+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:32:24.128770+0000","last_undegraded":"2023-09-23T23:32:24.128770+0000","last_fullsized":"2023-09-23T23:32:24.128770+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"528'8378","reported_seq":8480,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:27.870152+0000","last_change":"2023-09-23T23:32:25.160238+0000","last_active":"2023-09-23T23:32:27.870152+0000","last_peered":"2023-09-23T23:32:27.870152+0000","last_clean":"2023-09-23T23:32:27.870152+0000","last_became_active":"2023-09-23T23:29:45.637986+0000","last_became_peered":"2023-09-23T23:29:45.637986+0000","last_unstale":"2023-09-23T23:32:27.870152+0000","last_undegraded":"2023-09-23T23:32:27.870152+0000","last_fullsized":"2023-09-23T23:32:27.870152+0000","mapping_epoch":476,"log_start":"484'1592","ondisk_log_start":"484'1592","created":476,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":6786,"log_dups_size":0,"ondisk_log_size":6786,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6045,"num_write_kb":193120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":593,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.149455+0000","last_change":"2023-09-23T23:28:03.629593+0000","last_active":"2023-09-23T23:32:24.149455+0000","last_peered":"2023-09-23T23:32:24.149455+0000","last_clean":"2023-09-23T23:32:24.149455+0000","last_became_active":"2023-09-23T23:26:51.588385+0000","last_became_peered":"2023-09-23T23:26:51.588385+0000","last_unstale":"2023-09-23T23:32:24.149455+0000","last_undegraded":"2023-09-23T23:32:24.149455+0000","last_fullsized":"2023-09-23T23:32:24.149455+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:20.818366+0000","scrub_duration":0.0050135120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":368,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.163093+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:32:28.163093+0000","last_peered":"2023-09-23T23:32:28.163093+0000","last_clean":"2023-09-23T23:32:28.163093+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:32:28.163093+0000","last_undegraded":"2023-09-23T23:32:28.163093+0000","last_fullsized":"2023-09-23T23:32:28.163093+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":516,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.709390+0000","last_change":"2023-09-23T23:28:45.066443+0000","last_active":"2023-09-23T23:32:28.709390+0000","last_peered":"2023-09-23T23:32:28.709390+0000","last_clean":"2023-09-23T23:32:28.709390+0000","last_became_active":"2023-09-23T23:28:45.066154+0000","last_became_peered":"2023-09-23T23:28:45.066154+0000","last_unstale":"2023-09-23T23:32:28.709390+0000","last_undegraded":"2023-09-23T23:32:28.709390+0000","last_fullsized":"2023-09-23T23:32:28.709390+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:51.651425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"528'7984","reported_seq":12916,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:27.874018+0000","last_change":"2023-09-23T23:31:09.842831+0000","last_active":"2023-09-23T23:32:27.874018+0000","last_peered":"2023-09-23T23:32:27.874018+0000","last_clean":"2023-09-23T23:32:27.874018+0000","last_became_active":"2023-09-23T23:31:09.815902+0000","last_became_peered":"2023-09-23T23:31:09.815902+0000","last_unstale":"2023-09-23T23:32:27.874018+0000","last_undegraded":"2023-09-23T23:32:27.874018+0000","last_fullsized":"2023-09-23T23:32:27.874018+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":7984,"log_dups_size":0,"ondisk_log_size":7984,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1245184,"num_objects":19,"num_object_clones":0,"num_object_copies":57,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5753,"num_write_kb":184704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2421,"num_bytes_recovered":119341056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":864,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.144561+0000","last_change":"2023-09-23T23:23:45.830384+0000","last_active":"2023-09-23T23:32:24.144561+0000","last_peered":"2023-09-23T23:32:24.144561+0000","last_clean":"2023-09-23T23:32:24.144561+0000","last_became_active":"2023-09-23T23:23:36.025724+0000","last_became_peered":"2023-09-23T23:23:36.025724+0000","last_unstale":"2023-09-23T23:32:24.144561+0000","last_undegraded":"2023-09-23T23:32:24.144561+0000","last_fullsized":"2023-09-23T23:32:24.144561+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:34.345528+0000","scrub_duration":0.001659259,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":499,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.163910+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:32:28.163910+0000","last_peered":"2023-09-23T23:32:28.163910+0000","last_clean":"2023-09-23T23:32:28.163910+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:32:28.163910+0000","last_undegraded":"2023-09-23T23:32:28.163910+0000","last_fullsized":"2023-09-23T23:32:28.163910+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":507,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.149082+0000","last_change":"2023-09-23T23:29:55.001192+0000","last_active":"2023-09-23T23:32:24.149082+0000","last_peered":"2023-09-23T23:32:24.149082+0000","last_clean":"2023-09-23T23:32:24.149082+0000","last_became_active":"2023-09-23T23:29:55.000904+0000","last_became_peered":"2023-09-23T23:29:55.000904+0000","last_unstale":"2023-09-23T23:32:24.149082+0000","last_undegraded":"2023-09-23T23:32:24.149082+0000","last_fullsized":"2023-09-23T23:32:24.149082+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:29.169209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"528'8442","reported_seq":13649,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:27.928352+0000","last_change":"2023-09-23T23:32:04.487636+0000","last_active":"2023-09-23T23:32:27.928352+0000","last_peered":"2023-09-23T23:32:27.928352+0000","last_clean":"2023-09-23T23:32:27.928352+0000","last_became_active":"2023-09-23T23:32:04.448366+0000","last_became_peered":"2023-09-23T23:32:04.448366+0000","last_unstale":"2023-09-23T23:32:27.928352+0000","last_undegraded":"2023-09-23T23:32:27.928352+0000","last_fullsized":"2023-09-23T23:32:27.928352+0000","mapping_epoch":523,"log_start":"484'496","ondisk_log_start":"484'496","created":476,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":7946,"log_dups_size":0,"ondisk_log_size":7946,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":851968,"num_objects":13,"num_object_clones":0,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6095,"num_write_kb":195456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2561,"num_bytes_recovered":47644672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":811,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.157403+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:32:28.157403+0000","last_peered":"2023-09-23T23:32:28.157403+0000","last_clean":"2023-09-23T23:32:28.157403+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:32:28.157403+0000","last_undegraded":"2023-09-23T23:32:28.157403+0000","last_fullsized":"2023-09-23T23:32:28.157403+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":43,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.128753+0000","last_change":"2023-09-23T23:29:54.998914+0000","last_active":"2023-09-23T23:32:24.128753+0000","last_peered":"2023-09-23T23:32:24.128753+0000","last_clean":"2023-09-23T23:32:24.128753+0000","last_became_active":"2023-09-23T23:29:54.998620+0000","last_became_peered":"2023-09-23T23:29:54.998620+0000","last_unstale":"2023-09-23T23:32:24.128753+0000","last_undegraded":"2023-09-23T23:32:24.128753+0000","last_fullsized":"2023-09-23T23:32:24.128753+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:32.702628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":150,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.147305+0000","last_change":"2023-09-23T23:26:11.516475+0000","last_active":"2023-09-23T23:32:24.147305+0000","last_peered":"2023-09-23T23:32:24.147305+0000","last_clean":"2023-09-23T23:32:24.147305+0000","last_became_active":"2023-09-23T23:26:11.516198+0000","last_became_peered":"2023-09-23T23:26:11.516198+0000","last_unstale":"2023-09-23T23:32:24.147305+0000","last_undegraded":"2023-09-23T23:32:24.147305+0000","last_fullsized":"2023-09-23T23:32:24.147305+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:56.875103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"528'7921","reported_seq":7995,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.163094+0000","last_change":"2023-09-23T23:29:45.635904+0000","last_active":"2023-09-23T23:32:28.163094+0000","last_peered":"2023-09-23T23:32:28.163094+0000","last_clean":"2023-09-23T23:32:28.163094+0000","last_became_active":"2023-09-23T23:29:45.633590+0000","last_became_peered":"2023-09-23T23:29:45.633590+0000","last_unstale":"2023-09-23T23:32:28.163094+0000","last_undegraded":"2023-09-23T23:32:28.163094+0000","last_fullsized":"2023-09-23T23:32:28.163094+0000","mapping_epoch":476,"log_start":"484'738","ondisk_log_start":"484'738","created":476,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":7183,"log_dups_size":0,"ondisk_log_size":7183,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5079040,"num_objects":77,"num_object_clones":0,"num_object_copies":231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":77,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5703,"num_write_kb":184992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":687,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.162269+0000","last_change":"2023-09-23T23:26:34.548211+0000","last_active":"2023-09-23T23:32:28.162269+0000","last_peered":"2023-09-23T23:32:28.162269+0000","last_clean":"2023-09-23T23:32:28.162269+0000","last_became_active":"2023-09-23T23:26:34.547815+0000","last_became_peered":"2023-09-23T23:26:34.547815+0000","last_unstale":"2023-09-23T23:32:28.162269+0000","last_undegraded":"2023-09-23T23:32:28.162269+0000","last_fullsized":"2023-09-23T23:32:28.162269+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:37.675417+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":150,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.147021+0000","last_change":"2023-09-23T23:26:11.512815+0000","last_active":"2023-09-23T23:32:24.147021+0000","last_peered":"2023-09-23T23:32:24.147021+0000","last_clean":"2023-09-23T23:32:24.147021+0000","last_became_active":"2023-09-23T23:26:11.512466+0000","last_became_peered":"2023-09-23T23:26:11.512466+0000","last_unstale":"2023-09-23T23:32:24.147021+0000","last_undegraded":"2023-09-23T23:32:24.147021+0000","last_fullsized":"2023-09-23T23:32:24.147021+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:10.324158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":537,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.149609+0000","last_change":"2023-09-23T23:29:55.001597+0000","last_active":"2023-09-23T23:32:24.149609+0000","last_peered":"2023-09-23T23:32:24.149609+0000","last_clean":"2023-09-23T23:32:24.149609+0000","last_became_active":"2023-09-23T23:29:55.001314+0000","last_became_peered":"2023-09-23T23:29:55.001314+0000","last_unstale":"2023-09-23T23:32:24.149609+0000","last_undegraded":"2023-09-23T23:32:24.149609+0000","last_fullsized":"2023-09-23T23:32:24.149609+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:00.024016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"528'8197","reported_seq":8271,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.164553+0000","last_change":"2023-09-23T23:29:45.649770+0000","last_active":"2023-09-23T23:32:28.164553+0000","last_peered":"2023-09-23T23:32:28.164553+0000","last_clean":"2023-09-23T23:32:28.164553+0000","last_became_active":"2023-09-23T23:29:45.646295+0000","last_became_peered":"2023-09-23T23:29:45.646295+0000","last_unstale":"2023-09-23T23:32:28.164553+0000","last_undegraded":"2023-09-23T23:32:28.164553+0000","last_fullsized":"2023-09-23T23:32:28.164553+0000","mapping_epoch":476,"log_start":"484'1259","ondisk_log_start":"484'1259","created":476,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":6938,"log_dups_size":0,"ondisk_log_size":6938,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":491520,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5911,"num_write_kb":189408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":870,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.157233+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:32:28.157233+0000","last_peered":"2023-09-23T23:32:28.157233+0000","last_clean":"2023-09-23T23:32:28.157233+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:32:28.157233+0000","last_undegraded":"2023-09-23T23:32:28.157233+0000","last_fullsized":"2023-09-23T23:32:28.157233+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":169,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.163635+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:32:28.163635+0000","last_peered":"2023-09-23T23:32:28.163635+0000","last_clean":"2023-09-23T23:32:28.163635+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:32:28.163635+0000","last_undegraded":"2023-09-23T23:32:28.163635+0000","last_fullsized":"2023-09-23T23:32:28.163635+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":453,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.164635+0000","last_change":"2023-09-23T23:22:39.489325+0000","last_active":"2023-09-23T23:32:28.164635+0000","last_peered":"2023-09-23T23:32:28.164635+0000","last_clean":"2023-09-23T23:32:28.164635+0000","last_became_active":"2023-09-23T23:22:39.489048+0000","last_became_peered":"2023-09-23T23:22:39.489048+0000","last_unstale":"2023-09-23T23:32:28.164635+0000","last_undegraded":"2023-09-23T23:32:28.164635+0000","last_fullsized":"2023-09-23T23:32:28.164635+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:29.984948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"528'11926","reported_seq":11990,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.157151+0000","last_change":"2023-09-23T23:28:38.107589+0000","last_active":"2023-09-23T23:32:28.157151+0000","last_peered":"2023-09-23T23:32:28.157151+0000","last_clean":"2023-09-23T23:32:28.157151+0000","last_became_active":"2023-09-23T23:28:38.107296+0000","last_became_peered":"2023-09-23T23:28:38.107296+0000","last_unstale":"2023-09-23T23:32:28.157151+0000","last_undegraded":"2023-09-23T23:32:28.157151+0000","last_fullsized":"2023-09-23T23:32:28.157151+0000","mapping_epoch":476,"log_start":"499'5200","ondisk_log_start":"499'5200","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":6726,"log_dups_size":0,"ondisk_log_size":6726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:12.352223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11926,"num_write_kb":381632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":850,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.130397+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:32:24.130397+0000","last_peered":"2023-09-23T23:32:24.130397+0000","last_clean":"2023-09-23T23:32:24.130397+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:32:24.130397+0000","last_undegraded":"2023-09-23T23:32:24.130397+0000","last_fullsized":"2023-09-23T23:32:24.130397+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":233,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.162756+0000","last_change":"2023-09-23T23:23:36.048889+0000","last_active":"2023-09-23T23:32:28.162756+0000","last_peered":"2023-09-23T23:32:28.162756+0000","last_clean":"2023-09-23T23:32:28.162756+0000","last_became_active":"2023-09-23T23:23:36.048572+0000","last_became_peered":"2023-09-23T23:23:36.048572+0000","last_unstale":"2023-09-23T23:32:28.162756+0000","last_undegraded":"2023-09-23T23:32:28.162756+0000","last_fullsized":"2023-09-23T23:32:28.162756+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:55.581146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":544,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.162431+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:32:24.162431+0000","last_peered":"2023-09-23T23:32:24.162431+0000","last_clean":"2023-09-23T23:32:24.162431+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:32:24.162431+0000","last_undegraded":"2023-09-23T23:32:24.162431+0000","last_fullsized":"2023-09-23T23:32:24.162431+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":118,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.161960+0000","last_change":"2023-09-23T23:27:20.549904+0000","last_active":"2023-09-23T23:32:28.161960+0000","last_peered":"2023-09-23T23:32:28.161960+0000","last_clean":"2023-09-23T23:32:28.161960+0000","last_became_active":"2023-09-23T23:27:20.549645+0000","last_became_peered":"2023-09-23T23:27:20.549645+0000","last_unstale":"2023-09-23T23:32:28.161960+0000","last_undegraded":"2023-09-23T23:32:28.161960+0000","last_fullsized":"2023-09-23T23:32:28.161960+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:09.850640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"528'11946","reported_seq":12010,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.157529+0000","last_change":"2023-09-23T23:28:38.107697+0000","last_active":"2023-09-23T23:32:28.157529+0000","last_peered":"2023-09-23T23:32:28.157529+0000","last_clean":"2023-09-23T23:32:28.157529+0000","last_became_active":"2023-09-23T23:28:38.107379+0000","last_became_peered":"2023-09-23T23:28:38.107379+0000","last_unstale":"2023-09-23T23:32:28.157529+0000","last_undegraded":"2023-09-23T23:32:28.157529+0000","last_fullsized":"2023-09-23T23:32:28.157529+0000","mapping_epoch":476,"log_start":"499'5200","ondisk_log_start":"499'5200","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":6746,"log_dups_size":0,"ondisk_log_size":6746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:12.478832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11946,"num_write_kb":382272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":678,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.709206+0000","last_change":"2023-09-23T23:23:49.320969+0000","last_active":"2023-09-23T23:32:28.709206+0000","last_peered":"2023-09-23T23:32:28.709206+0000","last_clean":"2023-09-23T23:32:28.709206+0000","last_became_active":"2023-09-23T23:22:17.001911+0000","last_became_peered":"2023-09-23T23:22:17.001911+0000","last_unstale":"2023-09-23T23:32:28.709206+0000","last_undegraded":"2023-09-23T23:32:28.709206+0000","last_fullsized":"2023-09-23T23:32:28.709206+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.320823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.320823+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:07.000984+0000","scrub_duration":0.0018140630000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":242,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.162138+0000","last_change":"2023-09-23T23:29:54.992796+0000","last_active":"2023-09-23T23:32:28.162138+0000","last_peered":"2023-09-23T23:32:28.162138+0000","last_clean":"2023-09-23T23:32:28.162138+0000","last_became_active":"2023-09-23T23:29:54.992394+0000","last_became_peered":"2023-09-23T23:29:54.992394+0000","last_unstale":"2023-09-23T23:32:28.162138+0000","last_undegraded":"2023-09-23T23:32:28.162138+0000","last_fullsized":"2023-09-23T23:32:28.162138+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:01.353635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":518,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.147227+0000","last_change":"2023-09-23T23:29:55.001578+0000","last_active":"2023-09-23T23:32:24.147227+0000","last_peered":"2023-09-23T23:32:24.147227+0000","last_clean":"2023-09-23T23:32:24.147227+0000","last_became_active":"2023-09-23T23:29:55.001264+0000","last_became_peered":"2023-09-23T23:29:55.001264+0000","last_unstale":"2023-09-23T23:32:24.147227+0000","last_undegraded":"2023-09-23T23:32:24.147227+0000","last_fullsized":"2023-09-23T23:32:24.147227+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:16.411769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":118,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.162564+0000","last_change":"2023-09-23T23:27:20.551221+0000","last_active":"2023-09-23T23:32:28.162564+0000","last_peered":"2023-09-23T23:32:28.162564+0000","last_clean":"2023-09-23T23:32:28.162564+0000","last_became_active":"2023-09-23T23:27:20.550900+0000","last_became_peered":"2023-09-23T23:27:20.550900+0000","last_unstale":"2023-09-23T23:32:28.162564+0000","last_undegraded":"2023-09-23T23:32:28.162564+0000","last_fullsized":"2023-09-23T23:32:28.162564+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:23.012149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"528'11874","reported_seq":11941,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:32:28.158430+0000","last_change":"2023-09-23T23:29:56.269322+0000","last_active":"2023-09-23T23:32:28.158430+0000","last_peered":"2023-09-23T23:32:28.158430+0000","last_clean":"2023-09-23T23:29:52.322507+0000","last_became_active":"2023-09-23T23:29:56.254145+0000","last_became_peered":"2023-09-23T23:29:56.254145+0000","last_unstale":"2023-09-23T23:32:28.158430+0000","last_undegraded":"2023-09-23T23:29:56.011776+0000","last_fullsized":"2023-09-23T23:29:55.980461+0000","mapping_epoch":496,"log_start":"499'5200","ondisk_log_start":"499'5200","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":6674,"log_dups_size":0,"ondisk_log_size":6674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:08:45.509316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5036,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11874,"num_write_kb":379968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[2,5],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,5","objects":-5036}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":831,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.157061+0000","last_change":"2023-09-23T23:28:48.113032+0000","last_active":"2023-09-23T23:32:28.157061+0000","last_peered":"2023-09-23T23:32:28.157061+0000","last_clean":"2023-09-23T23:32:28.157061+0000","last_became_active":"2023-09-23T23:28:48.112666+0000","last_became_peered":"2023-09-23T23:28:48.112666+0000","last_unstale":"2023-09-23T23:32:28.157061+0000","last_undegraded":"2023-09-23T23:32:28.157061+0000","last_fullsized":"2023-09-23T23:32:28.157061+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:59:59.838039+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":266,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.158131+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:32:28.158131+0000","last_peered":"2023-09-23T23:32:28.158131+0000","last_clean":"2023-09-23T23:32:28.158131+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:32:28.158131+0000","last_undegraded":"2023-09-23T23:32:28.158131+0000","last_fullsized":"2023-09-23T23:32:28.158131+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":242,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.161921+0000","last_change":"2023-09-23T23:29:54.992719+0000","last_active":"2023-09-23T23:32:28.161921+0000","last_peered":"2023-09-23T23:32:28.161921+0000","last_clean":"2023-09-23T23:32:28.161921+0000","last_became_active":"2023-09-23T23:29:54.992412+0000","last_became_peered":"2023-09-23T23:29:54.992412+0000","last_unstale":"2023-09-23T23:32:28.161921+0000","last_undegraded":"2023-09-23T23:32:28.161921+0000","last_fullsized":"2023-09-23T23:32:28.161921+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:46.541397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":118,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.164888+0000","last_change":"2023-09-23T23:27:20.551574+0000","last_active":"2023-09-23T23:32:28.164888+0000","last_peered":"2023-09-23T23:32:28.164888+0000","last_clean":"2023-09-23T23:32:28.164888+0000","last_became_active":"2023-09-23T23:27:20.550820+0000","last_became_peered":"2023-09-23T23:27:20.550820+0000","last_unstale":"2023-09-23T23:32:28.164888+0000","last_undegraded":"2023-09-23T23:32:28.164888+0000","last_fullsized":"2023-09-23T23:32:28.164888+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:09.274552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"528'11836","reported_seq":11898,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:27.933145+0000","last_change":"2023-09-23T23:28:38.065394+0000","last_active":"2023-09-23T23:32:27.933145+0000","last_peered":"2023-09-23T23:32:27.933145+0000","last_clean":"2023-09-23T23:32:27.933145+0000","last_became_active":"2023-09-23T23:28:38.065160+0000","last_became_peered":"2023-09-23T23:28:38.065160+0000","last_unstale":"2023-09-23T23:32:27.933145+0000","last_undegraded":"2023-09-23T23:32:27.933145+0000","last_fullsized":"2023-09-23T23:32:27.933145+0000","mapping_epoch":476,"log_start":"502'6500","ondisk_log_start":"502'6500","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5336,"log_dups_size":0,"ondisk_log_size":5336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:10:39.685945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11836,"num_write_kb":378689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":543,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.157341+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:32:28.157341+0000","last_peered":"2023-09-23T23:32:28.157341+0000","last_clean":"2023-09-23T23:32:28.157341+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:32:28.157341+0000","last_undegraded":"2023-09-23T23:32:28.157341+0000","last_fullsized":"2023-09-23T23:32:28.157341+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":869,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.144493+0000","last_change":"2023-09-23T23:29:58.259227+0000","last_active":"2023-09-23T23:32:24.144493+0000","last_peered":"2023-09-23T23:32:24.144493+0000","last_clean":"2023-09-23T23:32:24.144493+0000","last_became_active":"2023-09-23T23:29:58.258366+0000","last_became_peered":"2023-09-23T23:29:58.258366+0000","last_unstale":"2023-09-23T23:32:24.144493+0000","last_undegraded":"2023-09-23T23:32:24.144493+0000","last_fullsized":"2023-09-23T23:32:24.144493+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:57.581676+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":490,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.162743+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:32:28.162743+0000","last_peered":"2023-09-23T23:32:28.162743+0000","last_clean":"2023-09-23T23:32:28.162743+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:32:28.162743+0000","last_undegraded":"2023-09-23T23:32:28.162743+0000","last_fullsized":"2023-09-23T23:32:28.162743+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":43,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.130229+0000","last_change":"2023-09-23T23:29:54.998861+0000","last_active":"2023-09-23T23:32:24.130229+0000","last_peered":"2023-09-23T23:32:24.130229+0000","last_clean":"2023-09-23T23:32:24.130229+0000","last_became_active":"2023-09-23T23:29:54.998550+0000","last_became_peered":"2023-09-23T23:29:54.998550+0000","last_unstale":"2023-09-23T23:32:24.130229+0000","last_undegraded":"2023-09-23T23:32:24.130229+0000","last_fullsized":"2023-09-23T23:32:24.130229+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:26.515992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"528'11822","reported_seq":11883,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:27.917552+0000","last_change":"2023-09-23T23:28:38.104180+0000","last_active":"2023-09-23T23:32:27.917552+0000","last_peered":"2023-09-23T23:32:27.917552+0000","last_clean":"2023-09-23T23:32:27.917552+0000","last_became_active":"2023-09-23T23:28:38.103844+0000","last_became_peered":"2023-09-23T23:28:38.103844+0000","last_unstale":"2023-09-23T23:32:27.917552+0000","last_undegraded":"2023-09-23T23:32:27.917552+0000","last_fullsized":"2023-09-23T23:32:27.917552+0000","mapping_epoch":476,"log_start":"500'5900","ondisk_log_start":"500'5900","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5922,"log_dups_size":0,"ondisk_log_size":5922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:48.427805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11822,"num_write_kb":378304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":712,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.162344+0000","last_change":"2023-09-23T23:29:58.254396+0000","last_active":"2023-09-23T23:32:28.162344+0000","last_peered":"2023-09-23T23:32:28.162344+0000","last_clean":"2023-09-23T23:32:28.162344+0000","last_became_active":"2023-09-23T23:29:58.254127+0000","last_became_peered":"2023-09-23T23:29:58.254127+0000","last_unstale":"2023-09-23T23:32:28.162344+0000","last_undegraded":"2023-09-23T23:32:28.162344+0000","last_fullsized":"2023-09-23T23:32:28.162344+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:48.863625+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":456,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.144320+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:32:24.144320+0000","last_peered":"2023-09-23T23:32:24.144320+0000","last_clean":"2023-09-23T23:32:24.144320+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:32:24.144320+0000","last_undegraded":"2023-09-23T23:32:24.144320+0000","last_fullsized":"2023-09-23T23:32:24.144320+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":455,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.146888+0000","last_change":"2023-09-23T23:28:22.601436+0000","last_active":"2023-09-23T23:32:24.146888+0000","last_peered":"2023-09-23T23:32:24.146888+0000","last_clean":"2023-09-23T23:32:24.146888+0000","last_became_active":"2023-09-23T23:28:22.601154+0000","last_became_peered":"2023-09-23T23:28:22.601154+0000","last_unstale":"2023-09-23T23:32:24.146888+0000","last_undegraded":"2023-09-23T23:32:24.146888+0000","last_fullsized":"2023-09-23T23:32:24.146888+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:03.296778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":562,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.709642+0000","last_change":"2023-09-23T23:22:38.438448+0000","last_active":"2023-09-23T23:32:28.709642+0000","last_peered":"2023-09-23T23:32:28.709642+0000","last_clean":"2023-09-23T23:32:28.709642+0000","last_became_active":"2023-09-23T23:22:38.437495+0000","last_became_peered":"2023-09-23T23:22:38.437495+0000","last_unstale":"2023-09-23T23:32:28.709642+0000","last_undegraded":"2023-09-23T23:32:28.709642+0000","last_fullsized":"2023-09-23T23:32:28.709642+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:49.587056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"528'11868","reported_seq":11942,"reported_epoch":529,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:32:28.163772+0000","last_change":"2023-09-23T23:29:56.148242+0000","last_active":"2023-09-23T23:32:28.163772+0000","last_peered":"2023-09-23T23:32:28.163772+0000","last_clean":"2023-09-23T23:29:53.882998+0000","last_became_active":"2023-09-23T23:29:56.142793+0000","last_became_peered":"2023-09-23T23:29:56.142793+0000","last_unstale":"2023-09-23T23:32:28.163772+0000","last_undegraded":"2023-09-23T23:32:28.163772+0000","last_fullsized":"2023-09-23T23:32:28.163772+0000","mapping_epoch":496,"log_start":"503'6711","ondisk_log_start":"503'6711","created":476,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":5157,"log_dups_size":0,"ondisk_log_size":5157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:10.851062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11868,"num_write_kb":379776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,4],"avail_no_missing":["3","2","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":454,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.130383+0000","last_change":"2023-09-23T23:29:58.254981+0000","last_active":"2023-09-23T23:32:24.130383+0000","last_peered":"2023-09-23T23:32:24.130383+0000","last_clean":"2023-09-23T23:32:24.130383+0000","last_became_active":"2023-09-23T23:29:58.254665+0000","last_became_peered":"2023-09-23T23:29:58.254665+0000","last_unstale":"2023-09-23T23:32:24.130383+0000","last_undegraded":"2023-09-23T23:32:24.130383+0000","last_fullsized":"2023-09-23T23:32:24.130383+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.842881+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.842881+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":380,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.145040+0000","last_change":"2023-09-23T23:23:35.037351+0000","last_active":"2023-09-23T23:32:24.145040+0000","last_peered":"2023-09-23T23:32:24.145040+0000","last_clean":"2023-09-23T23:32:24.145040+0000","last_became_active":"2023-09-23T23:23:35.036841+0000","last_became_peered":"2023-09-23T23:23:35.036841+0000","last_unstale":"2023-09-23T23:32:24.145040+0000","last_undegraded":"2023-09-23T23:32:24.145040+0000","last_fullsized":"2023-09-23T23:32:24.145040+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:20.618292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":233,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.162862+0000","last_change":"2023-09-23T23:26:32.037099+0000","last_active":"2023-09-23T23:32:28.162862+0000","last_peered":"2023-09-23T23:32:28.162862+0000","last_clean":"2023-09-23T23:32:28.162862+0000","last_became_active":"2023-09-23T23:26:32.036681+0000","last_became_peered":"2023-09-23T23:26:32.036681+0000","last_unstale":"2023-09-23T23:32:28.162862+0000","last_undegraded":"2023-09-23T23:32:28.162862+0000","last_fullsized":"2023-09-23T23:32:28.162862+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:48.926576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":262,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.162178+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:32:24.162178+0000","last_peered":"2023-09-23T23:32:24.162178+0000","last_clean":"2023-09-23T23:32:24.162178+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:32:24.162178+0000","last_undegraded":"2023-09-23T23:32:24.162178+0000","last_fullsized":"2023-09-23T23:32:24.162178+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"528'8098","reported_seq":8172,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.163534+0000","last_change":"2023-09-23T23:29:45.645107+0000","last_active":"2023-09-23T23:32:28.163534+0000","last_peered":"2023-09-23T23:32:28.163534+0000","last_clean":"2023-09-23T23:32:28.163534+0000","last_became_active":"2023-09-23T23:29:45.640211+0000","last_became_peered":"2023-09-23T23:29:45.640211+0000","last_unstale":"2023-09-23T23:32:28.163534+0000","last_undegraded":"2023-09-23T23:32:28.163534+0000","last_fullsized":"2023-09-23T23:32:28.163534+0000","mapping_epoch":476,"log_start":"484'1499","ondisk_log_start":"484'1499","created":476,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":6599,"log_dups_size":0,"ondisk_log_size":6599,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1572864,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5828,"num_write_kb":187264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":860,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.147423+0000","last_change":"2023-09-23T23:29:58.258327+0000","last_active":"2023-09-23T23:32:24.147423+0000","last_peered":"2023-09-23T23:32:24.147423+0000","last_clean":"2023-09-23T23:32:24.147423+0000","last_became_active":"2023-09-23T23:29:58.257930+0000","last_became_peered":"2023-09-23T23:29:58.257930+0000","last_unstale":"2023-09-23T23:32:24.147423+0000","last_undegraded":"2023-09-23T23:32:24.147423+0000","last_fullsized":"2023-09-23T23:32:24.147423+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:00.144157+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":525,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.162858+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:32:28.162858+0000","last_peered":"2023-09-23T23:32:28.162858+0000","last_clean":"2023-09-23T23:32:28.162858+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:32:28.162858+0000","last_undegraded":"2023-09-23T23:32:28.162858+0000","last_fullsized":"2023-09-23T23:32:28.162858+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":127,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.163283+0000","last_change":"2023-09-23T23:29:54.999590+0000","last_active":"2023-09-23T23:32:28.163283+0000","last_peered":"2023-09-23T23:32:28.163283+0000","last_clean":"2023-09-23T23:32:28.163283+0000","last_became_active":"2023-09-23T23:29:54.999209+0000","last_became_peered":"2023-09-23T23:29:54.999209+0000","last_unstale":"2023-09-23T23:32:28.163283+0000","last_undegraded":"2023-09-23T23:32:28.163283+0000","last_fullsized":"2023-09-23T23:32:28.163283+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:32.208030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":61,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.157789+0000","last_change":"2023-09-23T23:28:45.061717+0000","last_active":"2023-09-23T23:32:28.157789+0000","last_peered":"2023-09-23T23:32:28.157789+0000","last_clean":"2023-09-23T23:32:28.157789+0000","last_became_active":"2023-09-23T23:28:45.061404+0000","last_became_peered":"2023-09-23T23:28:45.061404+0000","last_unstale":"2023-09-23T23:32:28.157789+0000","last_undegraded":"2023-09-23T23:32:28.157789+0000","last_fullsized":"2023-09-23T23:32:28.157789+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:46.614114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.9","version":"528'8074","reported_seq":13071,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.158053+0000","last_change":"2023-09-23T23:31:37.117056+0000","last_active":"2023-09-23T23:32:28.158053+0000","last_peered":"2023-09-23T23:32:28.158053+0000","last_clean":"2023-09-23T23:32:28.158053+0000","last_became_active":"2023-09-23T23:31:37.105545+0000","last_became_peered":"2023-09-23T23:31:37.105545+0000","last_unstale":"2023-09-23T23:32:28.158053+0000","last_undegraded":"2023-09-23T23:32:28.158053+0000","last_fullsized":"2023-09-23T23:32:28.158053+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":476,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":8074,"log_dups_size":0,"ondisk_log_size":8074,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1212416,"num_objects":18,"num_object_clones":0,"num_object_copies":54,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5838,"num_write_kb":187424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2453,"num_bytes_recovered":82313216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":915,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.163274+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:32:28.163274+0000","last_peered":"2023-09-23T23:32:28.163274+0000","last_clean":"2023-09-23T23:32:28.163274+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:32:28.163274+0000","last_undegraded":"2023-09-23T23:32:28.163274+0000","last_fullsized":"2023-09-23T23:32:28.163274+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":43,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.130480+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:32:24.130480+0000","last_peered":"2023-09-23T23:32:24.130480+0000","last_clean":"2023-09-23T23:32:24.130480+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:32:24.130480+0000","last_undegraded":"2023-09-23T23:32:24.130480+0000","last_fullsized":"2023-09-23T23:32:24.130480+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":517,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.709472+0000","last_change":"2023-09-23T23:22:38.482705+0000","last_active":"2023-09-23T23:32:28.709472+0000","last_peered":"2023-09-23T23:32:28.709472+0000","last_clean":"2023-09-23T23:32:28.709472+0000","last_became_active":"2023-09-23T23:22:38.438239+0000","last_became_peered":"2023-09-23T23:22:38.438239+0000","last_unstale":"2023-09-23T23:32:28.709472+0000","last_undegraded":"2023-09-23T23:32:28.709472+0000","last_fullsized":"2023-09-23T23:32:28.709472+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:52.006349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":160,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.149204+0000","last_change":"2023-09-23T23:29:55.000912+0000","last_active":"2023-09-23T23:32:24.149204+0000","last_peered":"2023-09-23T23:32:24.149204+0000","last_clean":"2023-09-23T23:32:24.149204+0000","last_became_active":"2023-09-23T23:29:54.999122+0000","last_became_peered":"2023-09-23T23:29:54.999122+0000","last_unstale":"2023-09-23T23:32:24.149204+0000","last_undegraded":"2023-09-23T23:32:24.149204+0000","last_fullsized":"2023-09-23T23:32:24.149204+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:10.962943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.14","version":"528'8060","reported_seq":13100,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:27.883605+0000","last_change":"2023-09-23T23:31:35.256501+0000","last_active":"2023-09-23T23:32:27.883605+0000","last_peered":"2023-09-23T23:32:27.883605+0000","last_clean":"2023-09-23T23:32:27.883605+0000","last_became_active":"2023-09-23T23:30:43.903692+0000","last_became_peered":"2023-09-23T23:30:43.903692+0000","last_unstale":"2023-09-23T23:32:27.883605+0000","last_undegraded":"2023-09-23T23:32:27.883605+0000","last_fullsized":"2023-09-23T23:32:27.883605+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":8060,"log_dups_size":0,"ondisk_log_size":8060,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5829,"num_write_kb":185920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2474,"num_bytes_recovered":162070528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":930,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.163892+0000","last_change":"2023-09-23T23:29:58.264914+0000","last_active":"2023-09-23T23:32:28.163892+0000","last_peered":"2023-09-23T23:32:28.163892+0000","last_clean":"2023-09-23T23:32:28.163892+0000","last_became_active":"2023-09-23T23:29:58.264356+0000","last_became_peered":"2023-09-23T23:29:58.264356+0000","last_unstale":"2023-09-23T23:32:28.163892+0000","last_undegraded":"2023-09-23T23:32:28.163892+0000","last_fullsized":"2023-09-23T23:32:28.163892+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:15:15.828500+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":199,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.147219+0000","last_change":"2023-09-23T23:28:22.603176+0000","last_active":"2023-09-23T23:32:24.147219+0000","last_peered":"2023-09-23T23:32:24.147219+0000","last_clean":"2023-09-23T23:32:24.147219+0000","last_became_active":"2023-09-23T23:28:22.602824+0000","last_became_peered":"2023-09-23T23:28:22.602824+0000","last_unstale":"2023-09-23T23:32:24.147219+0000","last_undegraded":"2023-09-23T23:32:24.147219+0000","last_fullsized":"2023-09-23T23:32:24.147219+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:35.934235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":283,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.157737+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:32:28.157737+0000","last_peered":"2023-09-23T23:32:28.157737+0000","last_clean":"2023-09-23T23:32:28.157737+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:32:28.157737+0000","last_undegraded":"2023-09-23T23:32:28.157737+0000","last_fullsized":"2023-09-23T23:32:28.157737+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.13","version":"528'8421","reported_seq":8519,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:27.921517+0000","last_change":"2023-09-23T23:31:58.122871+0000","last_active":"2023-09-23T23:32:27.921517+0000","last_peered":"2023-09-23T23:32:27.921517+0000","last_clean":"2023-09-23T23:32:27.921517+0000","last_became_active":"2023-09-23T23:29:45.699719+0000","last_became_peered":"2023-09-23T23:29:45.699719+0000","last_unstale":"2023-09-23T23:32:27.921517+0000","last_undegraded":"2023-09-23T23:32:27.921517+0000","last_fullsized":"2023-09-23T23:32:27.921517+0000","mapping_epoch":490,"log_start":"484'1625","ondisk_log_start":"484'1625","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":6796,"log_dups_size":0,"ondisk_log_size":6796,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":688128,"num_objects":11,"num_object_clones":0,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6089,"num_write_kb":195168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":676,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.162596+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:32:28.162596+0000","last_peered":"2023-09-23T23:32:28.162596+0000","last_clean":"2023-09-23T23:32:28.162596+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:32:28.162596+0000","last_undegraded":"2023-09-23T23:32:28.162596+0000","last_fullsized":"2023-09-23T23:32:28.162596+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":270,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.149176+0000","last_change":"2023-09-23T23:22:38.452968+0000","last_active":"2023-09-23T23:32:24.149176+0000","last_peered":"2023-09-23T23:32:24.149176+0000","last_clean":"2023-09-23T23:32:24.149176+0000","last_became_active":"2023-09-23T23:22:38.437986+0000","last_became_peered":"2023-09-23T23:22:38.437986+0000","last_unstale":"2023-09-23T23:32:24.149176+0000","last_undegraded":"2023-09-23T23:32:24.149176+0000","last_fullsized":"2023-09-23T23:32:24.149176+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:10.333271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":160,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.146996+0000","last_change":"2023-09-23T23:27:20.558454+0000","last_active":"2023-09-23T23:32:24.146996+0000","last_peered":"2023-09-23T23:32:24.146996+0000","last_clean":"2023-09-23T23:32:24.146996+0000","last_became_active":"2023-09-23T23:27:20.558204+0000","last_became_peered":"2023-09-23T23:27:20.558204+0000","last_unstale":"2023-09-23T23:32:24.146996+0000","last_undegraded":"2023-09-23T23:32:24.146996+0000","last_fullsized":"2023-09-23T23:32:24.146996+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:15.079990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.12","version":"528'8298","reported_seq":8393,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:27.896336+0000","last_change":"2023-09-23T23:31:59.101031+0000","last_active":"2023-09-23T23:32:27.896336+0000","last_peered":"2023-09-23T23:32:27.896336+0000","last_clean":"2023-09-23T23:32:27.896336+0000","last_became_active":"2023-09-23T23:29:45.684008+0000","last_became_peered":"2023-09-23T23:29:45.684008+0000","last_unstale":"2023-09-23T23:32:27.896336+0000","last_undegraded":"2023-09-23T23:32:27.896336+0000","last_fullsized":"2023-09-23T23:32:27.896336+0000","mapping_epoch":490,"log_start":"481'116","ondisk_log_start":"481'116","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":8182,"log_dups_size":0,"ondisk_log_size":8182,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1966080,"num_objects":30,"num_object_clones":0,"num_object_copies":90,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5974,"num_write_kb":192128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":284,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.149271+0000","last_change":"2023-09-23T23:29:55.022469+0000","last_active":"2023-09-23T23:32:24.149271+0000","last_peered":"2023-09-23T23:32:24.149271+0000","last_clean":"2023-09-23T23:32:24.149271+0000","last_became_active":"2023-09-23T23:29:55.003082+0000","last_became_peered":"2023-09-23T23:29:55.003082+0000","last_unstale":"2023-09-23T23:32:24.149271+0000","last_undegraded":"2023-09-23T23:32:24.149271+0000","last_fullsized":"2023-09-23T23:32:24.149271+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:15.791643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":150,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.149652+0000","last_change":"2023-09-23T23:26:11.527453+0000","last_active":"2023-09-23T23:32:24.149652+0000","last_peered":"2023-09-23T23:32:24.149652+0000","last_clean":"2023-09-23T23:32:24.149652+0000","last_became_active":"2023-09-23T23:26:11.527208+0000","last_became_peered":"2023-09-23T23:26:11.527208+0000","last_unstale":"2023-09-23T23:32:24.149652+0000","last_undegraded":"2023-09-23T23:32:24.149652+0000","last_fullsized":"2023-09-23T23:32:24.149652+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:15.083611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.11","version":"528'8404","reported_seq":8483,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:32:28.162286+0000","last_change":"2023-09-23T23:29:56.435212+0000","last_active":"2023-09-23T23:32:28.162286+0000","last_peered":"2023-09-23T23:32:28.162286+0000","last_clean":"2023-09-23T23:29:52.255669+0000","last_became_active":"2023-09-23T23:29:56.398423+0000","last_became_peered":"2023-09-23T23:29:56.398423+0000","last_unstale":"2023-09-23T23:32:28.162286+0000","last_undegraded":"2023-09-23T23:29:56.385600+0000","last_fullsized":"2023-09-23T23:29:55.976159+0000","mapping_epoch":496,"log_start":"484'1691","ondisk_log_start":"484'1691","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":6713,"log_dups_size":0,"ondisk_log_size":6713,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2586,"num_objects_misplaced":2586,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6078,"num_write_kb":193280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[1,3],"avail_no_missing":["1","3","7"],"object_location_counts":[{"shards":"1,3","objects":-2586}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":815,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.144775+0000","last_change":"2023-09-23T23:29:58.252042+0000","last_active":"2023-09-23T23:32:24.144775+0000","last_peered":"2023-09-23T23:32:24.144775+0000","last_clean":"2023-09-23T23:32:24.144775+0000","last_became_active":"2023-09-23T23:29:58.251749+0000","last_became_peered":"2023-09-23T23:29:58.251749+0000","last_unstale":"2023-09-23T23:32:24.144775+0000","last_undegraded":"2023-09-23T23:32:24.144775+0000","last_fullsized":"2023-09-23T23:32:24.144775+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:39.324116+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":558,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.162523+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:32:28.162523+0000","last_peered":"2023-09-23T23:32:28.162523+0000","last_clean":"2023-09-23T23:32:28.162523+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:32:28.162523+0000","last_undegraded":"2023-09-23T23:32:28.162523+0000","last_fullsized":"2023-09-23T23:32:28.162523+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":542,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.164088+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:32:28.164088+0000","last_peered":"2023-09-23T23:32:28.164088+0000","last_clean":"2023-09-23T23:32:28.164088+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:32:28.164088+0000","last_undegraded":"2023-09-23T23:32:28.164088+0000","last_fullsized":"2023-09-23T23:32:28.164088+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.10","version":"528'8205","reported_seq":15776,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.708913+0000","last_change":"2023-09-23T23:32:25.990935+0000","last_active":"2023-09-23T23:32:28.708913+0000","last_peered":"2023-09-23T23:32:28.708913+0000","last_clean":"2023-09-23T23:32:28.708913+0000","last_became_active":"2023-09-23T23:30:43.874289+0000","last_became_peered":"2023-09-23T23:30:43.874289+0000","last_unstale":"2023-09-23T23:32:28.708913+0000","last_undegraded":"2023-09-23T23:32:28.708913+0000","last_fullsized":"2023-09-23T23:32:28.708913+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:28:37.015484+0000","last_clean_scrub_stamp":"2023-09-23T23:28:37.015484+0000","objects_scrubbed":0,"log_size":8205,"log_dups_size":0,"ondisk_log_size":8205,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1245184,"num_objects":19,"num_object_clones":0,"num_object_copies":57,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5887,"num_write_kb":188992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2491,"num_bytes_recovered":163250176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":905,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.165422+0000","last_change":"2023-09-23T23:26:14.527390+0000","last_active":"2023-09-23T23:32:28.165422+0000","last_peered":"2023-09-23T23:32:28.165422+0000","last_clean":"2023-09-23T23:32:28.165422+0000","last_became_active":"2023-09-23T23:26:14.527008+0000","last_became_peered":"2023-09-23T23:26:14.527008+0000","last_unstale":"2023-09-23T23:32:28.165422+0000","last_undegraded":"2023-09-23T23:32:28.165422+0000","last_fullsized":"2023-09-23T23:32:28.165422+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:36.460827+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":341,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T23:32:28.158177+0000","last_change":"2023-09-23T23:28:22.600921+0000","last_active":"2023-09-23T23:32:28.158177+0000","last_peered":"2023-09-23T23:32:28.158177+0000","last_clean":"2023-09-23T23:32:28.158177+0000","last_became_active":"2023-09-23T23:28:22.600565+0000","last_became_peered":"2023-09-23T23:28:22.600565+0000","last_unstale":"2023-09-23T23:32:28.158177+0000","last_undegraded":"2023-09-23T23:32:28.158177+0000","last_fullsized":"2023-09-23T23:32:28.158177+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:07.793563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":584,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T23:32:24.146722+0000","last_change":"2023-09-23T23:29:55.002051+0000","last_active":"2023-09-23T23:32:24.146722+0000","last_peered":"2023-09-23T23:32:24.146722+0000","last_clean":"2023-09-23T23:32:24.146722+0000","last_became_active":"2023-09-23T23:29:55.001732+0000","last_became_peered":"2023-09-23T23:29:55.001732+0000","last_unstale":"2023-09-23T23:32:24.146722+0000","last_undegraded":"2023-09-23T23:32:24.146722+0000","last_fullsized":"2023-09-23T23:32:24.146722+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:28:01.353263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":26,"stat_sum":{"num_bytes":16842752,"num_objects":256,"num_object_clones":0,"num_object_copies":768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9226,"num_objects_misplaced":5082,"num_objects_unfound":0,"num_objects_dirty":256,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":189746,"num_write_kb":6071809,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18410,"num_bytes_recovered":749273088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1321861120,"data_stored":1321861120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":184228,"ondisk_log_size":184228,"up":78,"acting":75,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":10412,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":603785,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1023292,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782636,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":516608,"kb_used_data":230968,"kb_used_omap":8,"kb_used_meta":285623,"kb_avail":93228544,"statfs":{"total":95995035648,"available":95466029056,"internally_reserved":0,"allocated":236511232,"data_stored":235069313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":292478021},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:31:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9350000000000001,"5min":2.7149999999999999,"15min":2.2530000000000001},"min":{"1min":0.224,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":36.956000000000003,"5min":44.094999999999999,"15min":85.721000000000004},"last":0.98999999999999999},{"interface":"front","average":{"1min":2.782,"5min":2.8889999999999998,"15min":2.1949999999999998},"min":{"1min":0.27900000000000003,"5min":0.253,"15min":0.19800000000000001},"max":{"1min":37.247999999999998,"5min":44.335999999999999,"15min":85.965000000000003},"last":1.048}]},{"osd":1,"last update":"Sat Sep 23 23:31:48 2023","interfaces":[{"interface":"back","average":{"1min":7.2539999999999996,"5min":5.2149999999999999,"15min":2.633},"min":{"1min":0.29099999999999998,"5min":0.23200000000000001,"15min":0.22900000000000001},"max":{"1min":92.061000000000007,"5min":198.58000000000001,"15min":198.58000000000001},"last":3.7509999999999999},{"interface":"front","average":{"1min":7.3380000000000001,"5min":5.1790000000000003,"15min":2.8650000000000002},"min":{"1min":0.28799999999999998,"5min":0.246,"15min":0.246},"max":{"1min":92.075000000000003,"5min":198.71100000000001,"15min":198.71100000000001},"last":5.7190000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:31:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3120000000000003,"5min":6.3470000000000004,"15min":3.52},"min":{"1min":0.33500000000000002,"5min":0.27200000000000002,"15min":0.26000000000000001},"max":{"1min":27.498000000000001,"5min":203.58000000000001,"15min":203.58000000000001},"last":3.407},{"interface":"front","average":{"1min":3.6800000000000002,"5min":5.8659999999999997,"15min":2.9769999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":27.574000000000002,"5min":198.69,"15min":198.69},"last":3.2559999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:31:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1320000000000001,"5min":6.2489999999999997,"15min":3.0910000000000002},"min":{"1min":0.30499999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":31.391999999999999,"5min":279.26499999999999,"15min":279.26499999999999},"last":3.9489999999999998},{"interface":"front","average":{"1min":11.055,"5min":7.5060000000000002,"15min":3.4340000000000002},"min":{"1min":0.32000000000000001,"5min":0.316,"15min":0.29599999999999999},"max":{"1min":150.608,"5min":279.29899999999998,"15min":279.29899999999998},"last":1.8200000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:31:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":0.78600000000000003,"15min":0.81000000000000005},"min":{"1min":0.26100000000000001,"5min":0.246,"15min":0.246},"max":{"1min":5.4660000000000002,"5min":5.4660000000000002,"15min":6.8380000000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.67800000000000005,"15min":0.84199999999999997},"min":{"1min":0.28100000000000003,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.234,"5min":7.9189999999999996,"15min":7.9189999999999996},"last":0.46700000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:31:48 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":2.9399999999999999,"15min":1.6850000000000001},"min":{"1min":0.29199999999999998,"5min":0.27500000000000002,"15min":0.26500000000000001},"max":{"1min":2.3050000000000002,"5min":210.83600000000001,"15min":210.83600000000001},"last":1.768},{"interface":"front","average":{"1min":0.63500000000000001,"5min":2.8340000000000001,"15min":1.484},"min":{"1min":0.316,"5min":0.28699999999999998,"15min":0.28199999999999997},"max":{"1min":2.448,"5min":207.04599999999999,"15min":207.04599999999999},"last":0.39600000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:31:48 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.77900000000000003,"15min":0.97399999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.9510000000000001,"5min":4.758,"15min":62.213000000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.4399999999999999,"5min":0.95299999999999996,"15min":1.2230000000000001},"min":{"1min":0.39400000000000002,"5min":0.34200000000000003,"15min":0.33300000000000002},"max":{"1min":7.0350000000000001,"5min":7.0350000000000001,"15min":62.805999999999997},"last":0.42299999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496113,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":654060,"kb_used_data":5732,"kb_used_omap":0,"kb_used_meta":648320,"kb_avail":93091092,"statfs":{"total":95995035648,"available":95325278208,"internally_reserved":0,"allocated":5869568,"data_stored":4447174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":663879680},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:32:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3620000000000001,"5min":3.081,"15min":6.6680000000000001},"min":{"1min":0.251,"5min":0.223,"15min":0.20399999999999999},"max":{"1min":8.8119999999999994,"5min":135.31999999999999,"15min":161.64599999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.97599999999999998,"5min":3.339,"15min":6.4199999999999999},"min":{"1min":0.23999999999999999,"5min":0.23000000000000001,"15min":0.22900000000000001},"max":{"1min":3.2269999999999999,"5min":135.34299999999999,"15min":161.38499999999999},"last":0.41699999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:32:08 2023","interfaces":[{"interface":"back","average":{"1min":4.7359999999999998,"5min":3.9409999999999998,"15min":3.6749999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.245},"max":{"1min":65.891000000000005,"5min":89.605000000000004,"15min":210.869},"last":0.49199999999999999},{"interface":"front","average":{"1min":4.3170000000000002,"5min":4.1980000000000004,"15min":3.7989999999999999},"min":{"1min":0.27200000000000002,"5min":0.249,"15min":0.23300000000000001},"max":{"1min":66.087000000000003,"5min":91.025999999999996,"15min":210.89699999999999},"last":0.379}]},{"osd":2,"last update":"Sat Sep 23 23:31:56 2023","interfaces":[{"interface":"back","average":{"1min":7.0339999999999998,"5min":4.319,"15min":3.8820000000000001},"min":{"1min":0.29599999999999999,"5min":0.28399999999999997,"15min":0.27800000000000002},"max":{"1min":97.408000000000001,"5min":97.408000000000001,"15min":127.44499999999999},"last":0.70599999999999996},{"interface":"front","average":{"1min":12.733000000000001,"5min":5.5,"15min":4.2569999999999997},"min":{"1min":0.375,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":125.539,"5min":125.539,"15min":125.539},"last":0.54500000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:31:41 2023","interfaces":[{"interface":"back","average":{"1min":7.6459999999999999,"5min":2.9969999999999999,"15min":3.3279999999999998},"min":{"1min":0.33700000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":138.87299999999999,"5min":138.87299999999999,"15min":138.87299999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":7.5250000000000004,"5min":3.1699999999999999,"15min":3.6070000000000002},"min":{"1min":0.36299999999999999,"5min":0.32700000000000001,"15min":0.24199999999999999},"max":{"1min":138.976,"5min":138.976,"15min":138.976},"last":0.69599999999999995}]},{"osd":4,"last update":"Sat Sep 23 23:32:20 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.70699999999999996,"15min":1.286},"min":{"1min":0.247,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":2.0299999999999998,"5min":8.1999999999999993,"15min":166.05199999999999},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.79000000000000004,"15min":1.619},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27300000000000002},"max":{"1min":2.9980000000000002,"5min":7.915,"15min":166.631},"last":0.59299999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:32:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.92900000000000005,"15min":1.1890000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.27900000000000003},"max":{"1min":5.915,"5min":11.473000000000001,"15min":91.899000000000001},"last":0.73999999999999999},{"interface":"front","average":{"1min":1.4330000000000001,"5min":0.91200000000000003,"15min":1.302},"min":{"1min":0.35699999999999998,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":5.3490000000000002,"5min":7.5670000000000002,"15min":94.174000000000007},"last":0.68300000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:31:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":0.93799999999999994,"15min":0.91600000000000004},"min":{"1min":0.34499999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.944,"5min":12.208,"15min":12.208},"last":0.67000000000000004},{"interface":"front","average":{"1min":1.597,"5min":1.1399999999999999,"15min":0.93899999999999995},"min":{"1min":0.38500000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":11.515000000000001,"5min":11.515000000000001,"15min":11.515000000000001},"last":0.76800000000000002}]}]},{"osd":1,"up_from":112,"seq":481036337454,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1155824,"kb_used_data":323176,"kb_used_omap":4,"kb_used_meta":832635,"kb_avail":92589328,"statfs":{"total":95995035648,"available":94811471872,"internally_reserved":0,"allocated":330932224,"data_stored":328464274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":852619150},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:32:11 2023","interfaces":[{"interface":"back","average":{"1min":2.907,"5min":5.2539999999999996,"15min":4.04},"min":{"1min":0.17799999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":15.483000000000001,"5min":130.768,"15min":130.768},"last":0.24399999999999999},{"interface":"front","average":{"1min":2.5179999999999998,"5min":4.4169999999999998,"15min":3.5619999999999998},"min":{"1min":0.218,"5min":0.19,"15min":0.19},"max":{"1min":15.699,"5min":134.929,"15min":134.929},"last":0.31}]},{"osd":2,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":18.552,"5min":11.241,"15min":6.3259999999999996},"min":{"1min":0.214,"5min":0.214,"15min":0.20100000000000001},"max":{"1min":182.554,"5min":231.67400000000001,"15min":231.67400000000001},"last":0.29599999999999999},{"interface":"front","average":{"1min":19.277000000000001,"5min":11.140000000000001,"15min":6.3369999999999997},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":180.09100000000001,"5min":232.21000000000001,"15min":232.21000000000001},"last":0.318}]},{"osd":3,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":3.6469999999999998,"5min":7.7830000000000004,"15min":5.3460000000000001},"min":{"1min":0.23899999999999999,"5min":0.23799999999999999,"15min":0.20799999999999999},"max":{"1min":14.57,"5min":234.404,"15min":234.404},"last":0.27100000000000002},{"interface":"front","average":{"1min":3.8849999999999998,"5min":7.5510000000000002,"15min":5.1619999999999999},"min":{"1min":0.30299999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":15.74,"5min":236.33699999999999,"15min":236.33699999999999},"last":0.34599999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:32:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5840000000000001,"5min":1.724,"15min":1.655},"min":{"1min":0.34399999999999997,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":12.042999999999999,"5min":12.042999999999999,"15min":18.474},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.1240000000000001,"5min":1.6890000000000001,"15min":1.6819999999999999},"min":{"1min":0.315,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.3239999999999998,"5min":18.108000000000001,"15min":18.108000000000001},"last":0.35799999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:32:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9770000000000001,"5min":1.484,"15min":2.9399999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.0190000000000001,"5min":9.0229999999999997,"15min":250.33000000000001},"last":0.371},{"interface":"front","average":{"1min":2.6909999999999998,"5min":1.7370000000000001,"15min":3.2679999999999998},"min":{"1min":0.38,"5min":0.34499999999999997,"15min":0.308},"max":{"1min":12.007999999999999,"5min":12.007999999999999,"15min":250.21199999999999},"last":0.41899999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:32:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2080000000000002,"5min":1.5469999999999999,"15min":2.3250000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":5.601,"5min":7.4470000000000001,"15min":74.087999999999994},"last":0.38400000000000001},{"interface":"front","average":{"1min":2.5790000000000002,"5min":1.8160000000000001,"15min":2.4159999999999999},"min":{"1min":0.40799999999999997,"5min":0.37,"15min":0.32800000000000001},"max":{"1min":11.927,"5min":11.927,"15min":73.974999999999994},"last":0.497}]},{"osd":7,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":1.6719999999999999,"15min":3.3820000000000001},"min":{"1min":0.39000000000000001,"5min":0.33000000000000002,"15min":0.32900000000000001},"max":{"1min":6.2380000000000004,"5min":11.385,"15min":78.308000000000007},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.2589999999999999,"5min":1.907,"15min":3.1110000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.4829999999999997,"5min":8.9930000000000003,"15min":77.204999999999998},"last":0.47099999999999997}]}]},{"osd":0,"up_from":91,"seq":390842024278,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1181104,"kb_used_data":6952,"kb_used_omap":5,"kb_used_meta":1174138,"kb_avail":92564048,"statfs":{"total":95995035648,"available":94785585152,"internally_reserved":0,"allocated":7118848,"data_stored":4586406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1202317946},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:32:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2629999999999999,"5min":11.031000000000001,"15min":6.1470000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.17499999999999999},"max":{"1min":14.115,"5min":213.45699999999999,"15min":213.45699999999999},"last":0.20699999999999999},{"interface":"front","average":{"1min":2.4510000000000001,"5min":11.238,"15min":5.181},"min":{"1min":0.22500000000000001,"5min":0.186,"15min":0.17599999999999999},"max":{"1min":14.478,"5min":213.04900000000001,"15min":213.04900000000001},"last":0.35299999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:31:49 2023","interfaces":[{"interface":"back","average":{"1min":15.987,"5min":6.5490000000000004,"15min":5.8869999999999996},"min":{"1min":0.17699999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":220.779,"5min":220.779,"15min":220.779},"last":0.33500000000000002},{"interface":"front","average":{"1min":18.346,"5min":7.2809999999999997,"15min":6.0990000000000002},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17599999999999999},"max":{"1min":218.74700000000001,"5min":218.74700000000001,"15min":218.74700000000001},"last":0.28299999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:32:24 2023","interfaces":[{"interface":"back","average":{"1min":8.1720000000000006,"5min":7.9409999999999998,"15min":5.2629999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":114.381,"5min":220.732,"15min":220.732},"last":0.29599999999999999},{"interface":"front","average":{"1min":9.0609999999999999,"5min":8.4740000000000002,"15min":5.2809999999999997},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":114.30500000000001,"5min":218.78200000000001,"15min":218.78200000000001},"last":0.313}]},{"osd":4,"last update":"Sat Sep 23 23:32:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2360000000000002,"5min":2.8530000000000002,"15min":3.121},"min":{"1min":0.33500000000000002,"5min":0.28899999999999998,"15min":0.27000000000000002},"max":{"1min":14.676,"5min":108.01900000000001,"15min":138.81299999999999},"last":0.436},{"interface":"front","average":{"1min":2.5249999999999999,"5min":2.7370000000000001,"15min":3.7770000000000001},"min":{"1min":0.316,"5min":0.312,"15min":0.28799999999999998},"max":{"1min":9.9039999999999999,"5min":107.898,"15min":171.251},"last":0.39700000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:32:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":1.583,"15min":3.0800000000000001},"min":{"1min":0.35099999999999998,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":9.8230000000000004,"5min":9.8230000000000004,"15min":171.30699999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.6759999999999999,"5min":1.4119999999999999,"15min":2.2069999999999999},"min":{"1min":0.34100000000000003,"5min":0.253,"15min":0.253},"max":{"1min":8.9909999999999997,"5min":11.307,"15min":139.40799999999999},"last":0.47899999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:32:01 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":2.6859999999999999,"15min":3.6040000000000001},"min":{"1min":0.372,"5min":0.371,"15min":0.29499999999999998},"max":{"1min":9.8729999999999993,"5min":108.67,"15min":171.33600000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.1019999999999999,"5min":2.6120000000000001,"15min":3.71},"min":{"1min":0.39700000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.8320000000000007,"5min":108.804,"15min":184.13499999999999},"last":0.44800000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:31:49 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":3.0590000000000002,"15min":2.6520000000000001},"min":{"1min":0.38700000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.452,"5min":111.71299999999999,"15min":138.77699999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":2.5110000000000001,"5min":2.8079999999999998,"15min":2.3599999999999999},"min":{"1min":0.36199999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.9670000000000005,"5min":111.108,"15min":139.37899999999999},"last":0.42599999999999999}]}]},{"osd":2,"up_from":199,"seq":854698492099,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":839284,"kb_used_data":153516,"kb_used_omap":7,"kb_used_meta":685752,"kb_avail":92905868,"statfs":{"total":95995035648,"available":95135608832,"internally_reserved":0,"allocated":157200384,"data_stored":154671851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":702210129},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":2.8039999999999998,"5min":3.2400000000000002,"15min":4.3049999999999997},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.16700000000000001},"max":{"1min":11.550000000000001,"5min":58.802,"15min":119.277},"last":1.758},{"interface":"front","average":{"1min":1.9370000000000001,"5min":3,"15min":4.1870000000000003},"min":{"1min":0.25900000000000001,"5min":0.214,"15min":0.18099999999999999},"max":{"1min":6.9649999999999999,"5min":58.115000000000002,"15min":119.56100000000001},"last":0.44800000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":4.6870000000000003,"5min":4.2910000000000004,"15min":4.5460000000000003},"min":{"1min":0.40799999999999997,"5min":0.23300000000000001,"15min":0.20699999999999999},"max":{"1min":16.541,"5min":82.876000000000005,"15min":157.98699999999999},"last":2.9159999999999999},{"interface":"front","average":{"1min":4.6120000000000001,"5min":4.4459999999999997,"15min":4.617},"min":{"1min":0.246,"5min":0.24099999999999999,"15min":0.22600000000000001},"max":{"1min":16.305,"5min":82.909000000000006,"15min":158.03800000000001},"last":1.659}]},{"osd":3,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":2.8090000000000002,"5min":4.4169999999999998,"15min":4.1529999999999996},"min":{"1min":0.28100000000000003,"5min":0.24299999999999999,"15min":0.191},"max":{"1min":7.851,"5min":103.227,"15min":119.366},"last":1.5529999999999999},{"interface":"front","average":{"1min":2.423,"5min":4.2649999999999997,"15min":3.6589999999999998},"min":{"1min":0.30499999999999999,"5min":0.26500000000000001,"15min":0.23499999999999999},"max":{"1min":7.6239999999999997,"5min":102.693,"15min":102.693},"last":1.621}]},{"osd":4,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":2.056,"15min":3.9249999999999998},"min":{"1min":0.41099999999999998,"5min":0.27700000000000002,"15min":0.25},"max":{"1min":6.0869999999999997,"5min":11.586,"15min":332.98099999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":2.6659999999999999,"5min":2.0369999999999999,"15min":3.3769999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":7.7629999999999999,"5min":11.621,"15min":332.96199999999999},"last":1.4670000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.9770000000000001,"15min":2.2770000000000001},"min":{"1min":0.38400000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.1449999999999996,"5min":10.192,"15min":119.919},"last":0.52500000000000002},{"interface":"front","average":{"1min":2.472,"5min":1.8779999999999999,"15min":2.1970000000000001},"min":{"1min":0.34200000000000003,"5min":0.33900000000000002,"15min":0.33000000000000002},"max":{"1min":7.7910000000000004,"5min":8.7110000000000003,"15min":119.306},"last":1.528}]},{"osd":6,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1499999999999999,"5min":2.2869999999999999,"15min":4.2039999999999997},"min":{"1min":0.40799999999999997,"5min":0.315,"15min":0.31},"max":{"1min":6.117,"5min":14.795999999999999,"15min":349.32799999999997},"last":0.501},{"interface":"front","average":{"1min":3.0739999999999998,"5min":2.2450000000000001,"15min":3.714},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":7.7249999999999996,"5min":12.496,"15min":349.17700000000002},"last":1.506}]},{"osd":7,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":2.0990000000000002,"15min":1.6359999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":6.7640000000000002,"5min":15.734999999999999,"15min":20.350000000000001},"last":1.4139999999999999},{"interface":"front","average":{"1min":2.0880000000000001,"5min":2.0590000000000002,"15min":1.6839999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.6340000000000003,"5min":15.709,"15min":20.321000000000002},"last":1.4430000000000001}]}]},{"osd":3,"up_from":148,"seq":635655160068,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1133336,"kb_used_data":286928,"kb_used_omap":2,"kb_used_meta":846397,"kb_avail":92611816,"statfs":{"total":95995035648,"available":94834499584,"internally_reserved":0,"allocated":293814272,"data_stored":291358439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":866710581},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:31:39 2023","interfaces":[{"interface":"back","average":{"1min":1.9790000000000001,"5min":2.6659999999999999,"15min":2.923},"min":{"1min":0.23999999999999999,"5min":0.17699999999999999,"15min":0.158},"max":{"1min":6.5259999999999998,"5min":40.901000000000003,"15min":103.866},"last":1.022},{"interface":"front","average":{"1min":3.0449999999999999,"5min":3.581,"15min":2.996},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":9.984,"5min":46.204000000000001,"15min":101.884},"last":3.9009999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:31:39 2023","interfaces":[{"interface":"back","average":{"1min":3.5880000000000001,"5min":3.7650000000000001,"15min":2.8919999999999999},"min":{"1min":0.224,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":11.976000000000001,"5min":85.242999999999995,"15min":85.242999999999995},"last":4.1859999999999999},{"interface":"front","average":{"1min":2.3759999999999999,"5min":4.0640000000000001,"15min":2.7130000000000001},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.159000000000001,"5min":85.302000000000007,"15min":85.302000000000007},"last":4.0140000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:31:50 2023","interfaces":[{"interface":"back","average":{"1min":13.750999999999999,"5min":6.625,"15min":4.7140000000000004},"min":{"1min":0.27400000000000002,"5min":0.219,"15min":0.19800000000000001},"max":{"1min":73.742000000000004,"5min":84.724999999999994,"15min":101.279},"last":6.9059999999999997},{"interface":"front","average":{"1min":13.571,"5min":6.6059999999999999,"15min":4.367},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.214},"max":{"1min":76.525999999999996,"5min":84.567999999999998,"15min":101.294},"last":9.5440000000000005}]},{"osd":4,"last update":"Sat Sep 23 23:31:39 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":1.3640000000000001,"15min":1.3859999999999999},"min":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":4.2590000000000003,"5min":8.5869999999999997,"15min":14.754},"last":1.194},{"interface":"front","average":{"1min":3.1080000000000001,"5min":2.1459999999999999,"15min":1.7989999999999999},"min":{"1min":0.40500000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":10.029999999999999,"5min":24.782,"15min":24.782},"last":3.5680000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:31:39 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":2.1080000000000001,"15min":1.7310000000000001},"min":{"1min":0.39800000000000002,"5min":0.29599999999999999,"15min":0.28000000000000003},"max":{"1min":10.081,"5min":24.565000000000001,"15min":24.565000000000001},"last":3.528},{"interface":"front","average":{"1min":2.4119999999999999,"5min":2.0990000000000002,"15min":1.905},"min":{"1min":0.32600000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":11.760999999999999,"5min":22.472999999999999,"15min":22.472999999999999},"last":3.8439999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:31:39 2023","interfaces":[{"interface":"back","average":{"1min":2.996,"5min":2.2069999999999999,"15min":2.0270000000000001},"min":{"1min":0.46200000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.055999999999999,"5min":24.658000000000001,"15min":53.057000000000002},"last":3.6059999999999999},{"interface":"front","average":{"1min":2.5,"5min":1.907,"15min":2.0899999999999999},"min":{"1min":0.38100000000000001,"5min":0.36099999999999999,"15min":0.35299999999999998},"max":{"1min":11.717000000000001,"5min":13.215999999999999,"15min":53.088000000000001},"last":3.8140000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:31:50 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":3.9420000000000002,"15min":1.954},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":5.548,"5min":146.803,"15min":146.803},"last":4.0389999999999997},{"interface":"front","average":{"1min":3.0470000000000002,"5min":4.7880000000000003,"15min":2.5169999999999999},"min":{"1min":0.46200000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":11.428000000000001,"5min":146.422,"15min":146.422},"last":3.6379999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019386,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1097340,"kb_used_data":327732,"kb_used_omap":0,"kb_used_meta":769600,"kb_avail":92647812,"statfs":{"total":95995035648,"available":94871359488,"internally_reserved":0,"allocated":335597568,"data_stored":334159454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":788070400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:32:03 2023","interfaces":[{"interface":"back","average":{"1min":5.4489999999999998,"5min":3.0939999999999999,"15min":4.7430000000000003},"min":{"1min":0.26200000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":63.143999999999998,"5min":67.707999999999998,"15min":199.64400000000001},"last":1.1830000000000001},{"interface":"front","average":{"1min":5.3029999999999999,"5min":3.2320000000000002,"15min":4.9429999999999996},"min":{"1min":0.26700000000000002,"5min":0.24199999999999999,"15min":0.216},"max":{"1min":62.954999999999998,"5min":67.635000000000005,"15min":195.649},"last":0.29099999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:32:25 2023","interfaces":[{"interface":"back","average":{"1min":1.9510000000000001,"5min":5.7110000000000003,"15min":3.6269999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.23100000000000001},"max":{"1min":4.6139999999999999,"5min":198.27199999999999,"15min":198.27199999999999},"last":1.2110000000000001},{"interface":"front","average":{"1min":1.1160000000000001,"5min":5.508,"15min":3.7040000000000002},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.6989999999999998,"5min":196.47,"15min":196.47},"last":0.39000000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:31:42 2023","interfaces":[{"interface":"back","average":{"1min":11.99,"5min":9.5609999999999999,"15min":5.3179999999999996},"min":{"1min":0.29499999999999998,"5min":0.27100000000000002,"15min":0.26300000000000001},"max":{"1min":110.313,"5min":198.178,"15min":198.178},"last":0.26700000000000002},{"interface":"front","average":{"1min":10.316000000000001,"5min":9.0459999999999994,"15min":5.1219999999999999},"min":{"1min":0.34999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":108.077,"5min":201.56,"15min":201.56},"last":1.3340000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:31:42 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":8.3290000000000006,"15min":7.2290000000000001},"min":{"1min":0.34799999999999998,"5min":0.31900000000000001,"15min":0.26300000000000001},"max":{"1min":5.6239999999999997,"5min":277.459,"15min":303.41500000000002},"last":1.3959999999999999},{"interface":"front","average":{"1min":1.4710000000000001,"5min":7.9359999999999999,"15min":7.1950000000000003},"min":{"1min":0.30399999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.7469999999999999,"5min":277.20800000000003,"15min":303.428},"last":0.42299999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:32:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":2.6469999999999998,"15min":2.004},"min":{"1min":0.26800000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.2750000000000004,"5min":161.86500000000001,"15min":161.86500000000001},"last":1.1399999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":2.4390000000000001,"15min":2.0270000000000001},"min":{"1min":0.35799999999999998,"5min":0.28399999999999997,"15min":0.249},"max":{"1min":2.3809999999999998,"5min":159.24199999999999,"15min":160.61099999999999},"last":0.314}]},{"osd":6,"last update":"Sat Sep 23 23:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.78500000000000003,"15min":1.1000000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.26000000000000001},"max":{"1min":6.1299999999999999,"5min":7.1130000000000004,"15min":90.346999999999994},"last":0.29899999999999999},{"interface":"front","average":{"1min":1.0389999999999999,"5min":0.88,"15min":1.3029999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.4429999999999996,"5min":9.8079999999999998,"15min":90.733000000000004},"last":1.2669999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:31:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.2130000000000001,"15min":1.1000000000000001},"min":{"1min":0.38100000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.9279999999999999,"5min":24.533999999999999,"15min":61.923999999999999},"last":1.2390000000000001},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.071,"15min":1.151},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":2.4129999999999998,"5min":27.771999999999998,"15min":61.990000000000002},"last":0.33100000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757749,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":652844,"kb_used_data":5668,"kb_used_omap":0,"kb_used_meta":647168,"kb_avail":93092308,"statfs":{"total":95995035648,"available":95326523392,"internally_reserved":0,"allocated":5804032,"data_stored":4378325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":662700032},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:32:05 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":1.506,"15min":3.5750000000000002},"min":{"1min":0.223,"5min":0.219,"15min":0.19500000000000001},"max":{"1min":3.0609999999999999,"5min":19.952000000000002,"15min":113.869},"last":0.26900000000000002},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.244,"15min":3.8879999999999999},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.6829999999999998,"5min":5.7320000000000002,"15min":138.55199999999999},"last":0.30499999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:32:10 2023","interfaces":[{"interface":"back","average":{"1min":3.0979999999999999,"5min":1.95,"15min":2.8730000000000002},"min":{"1min":0.29699999999999999,"5min":0.25,"15min":0.23100000000000001},"max":{"1min":19.347000000000001,"5min":28.379999999999999,"15min":91.518000000000001},"last":4.2919999999999998},{"interface":"front","average":{"1min":3.472,"5min":2.1040000000000001,"15min":3.1280000000000001},"min":{"1min":0.28799999999999998,"5min":0.249,"15min":0.249},"max":{"1min":19.268000000000001,"5min":28.417000000000002,"15min":87.718000000000004},"last":3.6219999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:31:43 2023","interfaces":[{"interface":"back","average":{"1min":2.948,"5min":2.7639999999999998,"15min":3.0409999999999999},"min":{"1min":0.35399999999999998,"5min":0.27200000000000002,"15min":0.26900000000000002},"max":{"1min":9.2690000000000001,"5min":46.177,"15min":144.91200000000001},"last":1.9410000000000001},{"interface":"front","average":{"1min":1.9370000000000001,"5min":2.718,"15min":2.8879999999999999},"min":{"1min":0.38700000000000001,"5min":0.26500000000000001,"15min":0.25800000000000001},"max":{"1min":7.0069999999999997,"5min":45.863999999999997,"15min":145.15799999999999},"last":7.468}]},{"osd":3,"last update":"Sat Sep 23 23:31:34 2023","interfaces":[{"interface":"back","average":{"1min":1.756,"5min":4.3630000000000004,"15min":4.883},"min":{"1min":0.34899999999999998,"5min":0.32000000000000001,"15min":0.27400000000000002},"max":{"1min":5.5380000000000003,"5min":112.71899999999999,"15min":159.53299999999999},"last":1.1459999999999999},{"interface":"front","average":{"1min":1.8149999999999999,"5min":4.4260000000000002,"15min":5.0970000000000004},"min":{"1min":0.39800000000000002,"5min":0.318,"15min":0.312},"max":{"1min":5.5529999999999999,"5min":112.70099999999999,"15min":159.54900000000001},"last":1.161}]},{"osd":4,"last update":"Sat Sep 23 23:32:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.032,"15min":0.99099999999999999},"min":{"1min":0.25700000000000001,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":6.1310000000000002,"5min":19.962,"15min":19.962},"last":1.663},{"interface":"front","average":{"1min":0.83999999999999997,"5min":0.89900000000000002,"15min":0.80200000000000005},"min":{"1min":0.30599999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.4080000000000004,"5min":20.829999999999998,"15min":20.829999999999998},"last":0.34699999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:32:22 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.76400000000000001,"15min":2.1739999999999999},"min":{"1min":0.33100000000000002,"5min":0.27600000000000002,"15min":0.26900000000000002},"max":{"1min":2.044,"5min":5.1299999999999999,"15min":310.68799999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.149,"5min":0.93799999999999994,"15min":2.3969999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.25900000000000001},"max":{"1min":3.3410000000000002,"5min":8.9979999999999993,"15min":311.45400000000001},"last":1.7509999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:31:43 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":2.9809999999999999,"15min":3.3490000000000002},"min":{"1min":0.35899999999999999,"5min":0.317,"15min":0.317},"max":{"1min":3.411,"5min":179.28800000000001,"15min":179.28800000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.0249999999999999,"5min":3.1099999999999999,"15min":3.4809999999999999},"min":{"1min":0.438,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":3.4980000000000002,"5min":179.428,"15min":179.428},"last":0.51300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":324206592,"data_stored":324206592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":150142976,"data_stored":150142976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":286785536,"data_stored":286785536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":330039296,"data_stored":330039296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":230686720,"data_stored":230686720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:32:38.300 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:32:38.640 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:32:38.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:38.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:32:38.744 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:32:38.770 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:38.774 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:32:38.812 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:32:38.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:38.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:32:38.897 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:38.921 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:32:38.932 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:38.961 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:39.061 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:32:39.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:39.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:32:39.166 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:39.182 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:39 smithi138 ceph-mon[193606]: pgmap v1475: 129 pgs: 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 122 active+clean; 14 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 9146/717 objects degraded (1275.593%); 5082/717 objects misplaced (708.787%); 0 B/s, 129 objects/s recovering 2023-09-23T23:32:39.182 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:39 smithi138 ceph-mon[193606]: 12.3 scrub starts 2023-09-23T23:32:39.182 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:39 smithi138 ceph-mon[193606]: 12.0 scrub starts 2023-09-23T23:32:39.183 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:39 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 9146/717 objects degraded (1275.593%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:39.192 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:39.234 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:39.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:39.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:39.340 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:32:39.365 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:39.407 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:32:39.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:39.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:32:39.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:39 smithi110 ceph-mon[221102]: pgmap v1475: 129 pgs: 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 122 active+clean; 14 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 9146/717 objects degraded (1275.593%); 5082/717 objects misplaced (708.787%); 0 B/s, 129 objects/s recovering 2023-09-23T23:32:39.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:39 smithi110 ceph-mon[221102]: 12.3 scrub starts 2023-09-23T23:32:39.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:39 smithi110 ceph-mon[221102]: 12.0 scrub starts 2023-09-23T23:32:39.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:39 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 9146/717 objects degraded (1275.593%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:39.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:39 smithi110 ceph-mon[212939]: pgmap v1475: 129 pgs: 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 122 active+clean; 14 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 9146/717 objects degraded (1275.593%); 5082/717 objects misplaced (708.787%); 0 B/s, 129 objects/s recovering 2023-09-23T23:32:39.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:39 smithi110 ceph-mon[212939]: 12.3 scrub starts 2023-09-23T23:32:39.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:39 smithi110 ceph-mon[212939]: 12.0 scrub starts 2023-09-23T23:32:39.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:39 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 9146/717 objects degraded (1275.593%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:32:39.513 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:32:39.555 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:39.611 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:32:39.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:39.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:32:39.716 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:32:39.748 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:39.900 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:32:39.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:39.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:32:40.005 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:40.031 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:40.074 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:40.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:40.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:40.181 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:32:40.193 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_7' removed 2023-09-23T23:32:40.198 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:32:40.213 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:40.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:40 smithi138 ceph-mon[193606]: 12.3 scrub starts 2023-09-23T23:32:40.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:40 smithi138 ceph-mon[193606]: 12.10 scrub starts 2023-09-23T23:32:40.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:40 smithi138 ceph-mon[193606]: 12.2 scrub starts 2023-09-23T23:32:40.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:40 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3357453380' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:32:40.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:40 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1333248540' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:32:40.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:40 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:32:40.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:40 smithi110 ceph-mon[212939]: 12.3 scrub starts 2023-09-23T23:32:40.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:40 smithi110 ceph-mon[212939]: 12.10 scrub starts 2023-09-23T23:32:40.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:40 smithi110 ceph-mon[212939]: 12.2 scrub starts 2023-09-23T23:32:40.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:40 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3357453380' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:32:40.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:40 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1333248540' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:32:40.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:40 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:32:40.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:40 smithi110 ceph-mon[221102]: 12.3 scrub starts 2023-09-23T23:32:40.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:40 smithi110 ceph-mon[221102]: 12.10 scrub starts 2023-09-23T23:32:40.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:40 smithi110 ceph-mon[221102]: 12.2 scrub starts 2023-09-23T23:32:40.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:40 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3357453380' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:32:40.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:40 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1333248540' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:32:40.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:40 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:32:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:41 smithi138 ceph-mon[193606]: pgmap v1476: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 125 active+clean; 14 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 6961/717 objects degraded (970.851%); 5082/717 objects misplaced (708.787%); 0 B/s, 283 objects/s recovering 2023-09-23T23:32:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:41 smithi138 ceph-mon[193606]: 12.10 scrub starts 2023-09-23T23:32:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:41 smithi138 ceph-mon[193606]: 12.0 scrub starts 2023-09-23T23:32:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:41 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:32:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:41 smithi138 ceph-mon[193606]: failed to encode map e533 with expected crc 2023-09-23T23:32:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:41 smithi138 ceph-mon[193606]: failed to encode map e533 with expected crc 2023-09-23T23:32:41.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:41 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3357453380' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:32:41.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:41 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:32:41.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:41 smithi138 ceph-mon[193606]: osdmap e533: 8 total, 8 up, 8 in 2023-09-23T23:32:41.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[212939]: pgmap v1476: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 125 active+clean; 14 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 6961/717 objects degraded (970.851%); 5082/717 objects misplaced (708.787%); 0 B/s, 283 objects/s recovering 2023-09-23T23:32:41.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[212939]: 12.10 scrub starts 2023-09-23T23:32:41.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[212939]: 12.0 scrub starts 2023-09-23T23:32:41.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:32:41.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[212939]: failed to encode map e533 with expected crc 2023-09-23T23:32:41.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[212939]: failed to encode map e533 with expected crc 2023-09-23T23:32:41.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3357453380' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:32:41.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:32:41.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[212939]: osdmap e533: 8 total, 8 up, 8 in 2023-09-23T23:32:41.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[221102]: pgmap v1476: 129 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 125 active+clean; 14 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 6961/717 objects degraded (970.851%); 5082/717 objects misplaced (708.787%); 0 B/s, 283 objects/s recovering 2023-09-23T23:32:41.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[221102]: 12.10 scrub starts 2023-09-23T23:32:41.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[221102]: 12.0 scrub starts 2023-09-23T23:32:41.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:32:41.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[221102]: failed to encode map e533 with expected crc 2023-09-23T23:32:41.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[221102]: failed to encode map e533 with expected crc 2023-09-23T23:32:41.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3357453380' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:32:41.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:32:41.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:41 smithi110 ceph-mon[221102]: osdmap e533: 8 total, 8 up, 8 in 2023-09-23T23:32:42.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:42 smithi138 ceph-mon[193606]: failed to encode map e533 with expected crc 2023-09-23T23:32:42.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:42 smithi138 ceph-mon[193606]: failed to encode map e533 with expected crc 2023-09-23T23:32:42.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:42 smithi138 ceph-mon[193606]: 1.6 scrub starts 2023-09-23T23:32:42.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:42 smithi138 ceph-mon[193606]: 1.6 scrub ok 2023-09-23T23:32:42.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:42 smithi110 ceph-mon[212939]: failed to encode map e533 with expected crc 2023-09-23T23:32:42.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:42 smithi110 ceph-mon[212939]: failed to encode map e533 with expected crc 2023-09-23T23:32:42.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:42 smithi110 ceph-mon[212939]: 1.6 scrub starts 2023-09-23T23:32:42.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:42 smithi110 ceph-mon[212939]: 1.6 scrub ok 2023-09-23T23:32:42.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:42 smithi110 ceph-mon[221102]: failed to encode map e533 with expected crc 2023-09-23T23:32:42.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:42 smithi110 ceph-mon[221102]: failed to encode map e533 with expected crc 2023-09-23T23:32:42.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:42 smithi110 ceph-mon[221102]: 1.6 scrub starts 2023-09-23T23:32:42.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:42 smithi110 ceph-mon[221102]: 1.6 scrub ok 2023-09-23T23:32:42.543 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:32:42.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:42.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:32:42.647 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:32:42.676 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:43.020 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:32:43.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:43.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:32:43.128 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:32:43.156 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:43.279 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:43 smithi110 ceph-mon[212939]: pgmap v1478: 103 pgs: 103 active+clean; 595 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:43.279 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:43 smithi110 ceph-mon[212939]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6961/717 objects degraded (970.851%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T23:32:43.279 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:43 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:32:43.279 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:43.279 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:32:43.279 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:43 smithi110 ceph-mon[221102]: pgmap v1478: 103 pgs: 103 active+clean; 595 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:43.279 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:43 smithi110 ceph-mon[221102]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6961/717 objects degraded (970.851%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T23:32:43.279 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:43 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:32:43.280 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:43.280 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:32:43.489 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T23:32:43.489 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T23:32:43.490 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_8 2023-09-23T23:32:43.490 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_8 16 2023-09-23T23:32:43.552 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:32:43.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:43.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:32:43.594 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:43.656 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:32:43.686 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:43 smithi138 ceph-mon[193606]: pgmap v1478: 103 pgs: 103 active+clean; 595 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:43.686 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:43 smithi138 ceph-mon[193606]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6961/717 objects degraded (970.851%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T23:32:43.686 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:43 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:32:43.686 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:43.686 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:32:43.688 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:43.738 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:32:43.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:43.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:32:43.761 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:32:43.763 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:32:43.845 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:32:43.874 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:43.918 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:32:43.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:43.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:32:44.025 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:32:44.054 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:44.249 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:32:44.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:44.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:32:44.353 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:32:44.379 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:44.673 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:32:44.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:44.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:32:44.777 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:32:44.809 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:44.858 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:32:44.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:44.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:32:44.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:44 smithi138 ceph-mon[193606]: from='client.36414 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:32:44.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:44 smithi138 ceph-mon[193606]: pgmap v1479: 103 pgs: 103 active+clean; 595 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:44.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:44 smithi110 ceph-mon[212939]: from='client.36414 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:32:44.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:44 smithi110 ceph-mon[212939]: pgmap v1479: 103 pgs: 103 active+clean; 595 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:44.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:44 smithi110 ceph-mon[221102]: from='client.36414 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:32:44.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:44 smithi110 ceph-mon[221102]: pgmap v1479: 103 pgs: 103 active+clean; 595 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:44.968 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:44.996 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:45.027 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:32:45.041 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:45.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:45.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:45.048 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:32:45.070 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:32:45.092 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:32:45.111 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:32:45.129 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:32:45.146 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:32:45.148 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:32:45.166 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:32:45.167 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:32:45.167 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:32:45.167 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:32:45.167 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:32:45.167 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:32:45.167 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:32:45.168 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:32:45.168 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:32:45.172 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:45.213 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:32:45.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:45.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:32:45.319 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:32:45.344 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:45.386 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:32:45.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:45.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:32:45.439 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:32:45.494 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:32:45.521 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:45.696 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1478,"stamp":"2023-09-23T23:32:41.848130+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":609289,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":66,"num_write_kb":84,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":1033704,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5046,"ondisk_log_size":5046,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":400,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7197692,"kb_used_data":1295548,"kb_used_omap":29,"kb_used_meta":5902050,"kb_avail":742763524,"statfs":{"total":767960285184,"available":760589848576,"internally_reserved":0,"allocated":1326641152,"data_stored":1310814487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":6043699923},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":15,"apply_latency_ms":15,"commit_latency_ns":15000000,"apply_latency_ns":15000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-14188544,"num_objects":-215,"num_object_clones":0,"num_object_copies":-645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-9226,"num_objects_misplaced":-5082,"num_objects_unfound":0,"num_objects_dirty":-215,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-177683,"num_write_kb":-5684513,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-18410,"num_bytes_recovered":-749273088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.173250"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":728,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.252717+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:32:35.252717+0000","last_peered":"2023-09-23T23:32:35.252717+0000","last_clean":"2023-09-23T23:32:35.252717+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:32:35.252717+0000","last_undegraded":"2023-09-23T23:32:35.252717+0000","last_fullsized":"2023-09-23T23:32:35.252717+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":566,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.002194+0000","last_change":"2023-09-23T23:22:38.439629+0000","last_active":"2023-09-23T23:32:35.002194+0000","last_peered":"2023-09-23T23:32:35.002194+0000","last_clean":"2023-09-23T23:32:35.002194+0000","last_became_active":"2023-09-23T23:22:38.439285+0000","last_became_peered":"2023-09-23T23:22:38.439285+0000","last_unstale":"2023-09-23T23:32:35.002194+0000","last_undegraded":"2023-09-23T23:32:35.002194+0000","last_fullsized":"2023-09-23T23:32:35.002194+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:34.431221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":503,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.681913+0000","last_change":"2023-09-23T23:23:43.511912+0000","last_active":"2023-09-23T23:32:34.681913+0000","last_peered":"2023-09-23T23:32:34.681913+0000","last_clean":"2023-09-23T23:32:34.681913+0000","last_became_active":"2023-09-23T23:23:43.511574+0000","last_became_peered":"2023-09-23T23:23:43.511574+0000","last_unstale":"2023-09-23T23:32:34.681913+0000","last_undegraded":"2023-09-23T23:32:34.681913+0000","last_fullsized":"2023-09-23T23:32:34.681913+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:16.286926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":621,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.253120+0000","last_change":"2023-09-23T23:23:50.459634+0000","last_active":"2023-09-23T23:32:35.253120+0000","last_peered":"2023-09-23T23:32:35.253120+0000","last_clean":"2023-09-23T23:32:35.253120+0000","last_became_active":"2023-09-23T23:23:45.453958+0000","last_became_peered":"2023-09-23T23:23:45.453958+0000","last_unstale":"2023-09-23T23:32:35.253120+0000","last_undegraded":"2023-09-23T23:32:35.253120+0000","last_fullsized":"2023-09-23T23:32:35.253120+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:56.563201+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":244,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.761897+0000","last_change":"2023-09-23T23:26:11.528846+0000","last_active":"2023-09-23T23:32:34.761897+0000","last_peered":"2023-09-23T23:32:34.761897+0000","last_clean":"2023-09-23T23:32:34.761897+0000","last_became_active":"2023-09-23T23:26:11.528529+0000","last_became_peered":"2023-09-23T23:26:11.528529+0000","last_unstale":"2023-09-23T23:32:34.761897+0000","last_undegraded":"2023-09-23T23:32:34.761897+0000","last_fullsized":"2023-09-23T23:32:34.761897+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:35.094583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":497,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.679950+0000","last_change":"2023-09-23T23:28:22.604718+0000","last_active":"2023-09-23T23:32:34.679950+0000","last_peered":"2023-09-23T23:32:34.679950+0000","last_clean":"2023-09-23T23:32:34.679950+0000","last_became_active":"2023-09-23T23:28:22.604498+0000","last_became_peered":"2023-09-23T23:28:22.604498+0000","last_unstale":"2023-09-23T23:32:34.679950+0000","last_undegraded":"2023-09-23T23:32:34.679950+0000","last_fullsized":"2023-09-23T23:32:34.679950+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:30.472412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":835,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.002415+0000","last_change":"2023-09-23T23:28:05.634824+0000","last_active":"2023-09-23T23:32:35.002415+0000","last_peered":"2023-09-23T23:32:35.002415+0000","last_clean":"2023-09-23T23:32:35.002415+0000","last_became_active":"2023-09-23T23:27:23.551046+0000","last_became_peered":"2023-09-23T23:27:23.551046+0000","last_unstale":"2023-09-23T23:32:35.002415+0000","last_undegraded":"2023-09-23T23:32:35.002415+0000","last_fullsized":"2023-09-23T23:32:35.002415+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:59.062239+0000","scrub_duration":0.0034158959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":530,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.682260+0000","last_change":"2023-09-23T23:27:20.554007+0000","last_active":"2023-09-23T23:32:34.682260+0000","last_peered":"2023-09-23T23:32:34.682260+0000","last_clean":"2023-09-23T23:32:34.682260+0000","last_became_active":"2023-09-23T23:27:20.553716+0000","last_became_peered":"2023-09-23T23:27:20.553716+0000","last_unstale":"2023-09-23T23:32:34.682260+0000","last_undegraded":"2023-09-23T23:32:34.682260+0000","last_fullsized":"2023-09-23T23:32:34.682260+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:38.911862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":46,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.679162+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:32:34.679162+0000","last_peered":"2023-09-23T23:32:34.679162+0000","last_clean":"2023-09-23T23:32:34.679162+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:32:34.679162+0000","last_undegraded":"2023-09-23T23:32:34.679162+0000","last_fullsized":"2023-09-23T23:32:34.679162+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":852,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.680646+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:32:34.680646+0000","last_peered":"2023-09-23T23:32:34.680646+0000","last_clean":"2023-09-23T23:32:34.680646+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:32:34.680646+0000","last_undegraded":"2023-09-23T23:32:34.680646+0000","last_fullsized":"2023-09-23T23:32:34.680646+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":164,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.253420+0000","last_change":"2023-09-23T23:28:22.602481+0000","last_active":"2023-09-23T23:32:35.253420+0000","last_peered":"2023-09-23T23:32:35.253420+0000","last_clean":"2023-09-23T23:32:35.253420+0000","last_became_active":"2023-09-23T23:28:22.602282+0000","last_became_peered":"2023-09-23T23:28:22.602282+0000","last_unstale":"2023-09-23T23:32:35.253420+0000","last_undegraded":"2023-09-23T23:32:35.253420+0000","last_fullsized":"2023-09-23T23:32:35.253420+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:39.906003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":531,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.682176+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:32:34.682176+0000","last_peered":"2023-09-23T23:32:34.682176+0000","last_clean":"2023-09-23T23:32:34.682176+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:32:34.682176+0000","last_undegraded":"2023-09-23T23:32:34.682176+0000","last_fullsized":"2023-09-23T23:32:34.682176+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":806,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.682163+0000","last_change":"2023-09-23T23:26:14.521792+0000","last_active":"2023-09-23T23:32:34.682163+0000","last_peered":"2023-09-23T23:32:34.682163+0000","last_clean":"2023-09-23T23:32:34.682163+0000","last_became_active":"2023-09-23T23:26:14.521438+0000","last_became_peered":"2023-09-23T23:26:14.521438+0000","last_unstale":"2023-09-23T23:32:34.682163+0000","last_undegraded":"2023-09-23T23:32:34.682163+0000","last_fullsized":"2023-09-23T23:32:34.682163+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:09.103712+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":163,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.253240+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:32:35.253240+0000","last_peered":"2023-09-23T23:32:35.253240+0000","last_clean":"2023-09-23T23:32:35.253240+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:32:35.253240+0000","last_undegraded":"2023-09-23T23:32:35.253240+0000","last_fullsized":"2023-09-23T23:32:35.253240+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":492,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.682709+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:32:34.682709+0000","last_peered":"2023-09-23T23:32:34.682709+0000","last_clean":"2023-09-23T23:32:34.682709+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:32:34.682709+0000","last_undegraded":"2023-09-23T23:32:34.682709+0000","last_fullsized":"2023-09-23T23:32:34.682709+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":946,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.253296+0000","last_change":"2023-09-23T23:29:58.260674+0000","last_active":"2023-09-23T23:32:35.253296+0000","last_peered":"2023-09-23T23:32:35.253296+0000","last_clean":"2023-09-23T23:32:35.253296+0000","last_became_active":"2023-09-23T23:29:58.260318+0000","last_became_peered":"2023-09-23T23:29:58.260318+0000","last_unstale":"2023-09-23T23:32:35.253296+0000","last_undegraded":"2023-09-23T23:32:35.253296+0000","last_fullsized":"2023-09-23T23:32:35.253296+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:08.810582+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":137,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.680376+0000","last_change":"2023-09-23T23:26:48.669857+0000","last_active":"2023-09-23T23:32:34.680376+0000","last_peered":"2023-09-23T23:32:34.680376+0000","last_clean":"2023-09-23T23:32:34.680376+0000","last_became_active":"2023-09-23T23:26:48.538246+0000","last_became_peered":"2023-09-23T23:26:48.538246+0000","last_unstale":"2023-09-23T23:32:34.680376+0000","last_undegraded":"2023-09-23T23:32:34.680376+0000","last_fullsized":"2023-09-23T23:32:34.680376+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:19.039472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":129,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.678708+0000","last_change":"2023-09-23T23:29:54.996097+0000","last_active":"2023-09-23T23:32:34.678708+0000","last_peered":"2023-09-23T23:32:34.678708+0000","last_clean":"2023-09-23T23:32:34.678708+0000","last_became_active":"2023-09-23T23:29:54.995756+0000","last_became_peered":"2023-09-23T23:29:54.995756+0000","last_unstale":"2023-09-23T23:32:34.678708+0000","last_undegraded":"2023-09-23T23:32:34.678708+0000","last_fullsized":"2023-09-23T23:32:34.678708+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:15.856474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":608,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.761719+0000","last_change":"2023-09-23T23:24:02.268079+0000","last_active":"2023-09-23T23:32:34.761719+0000","last_peered":"2023-09-23T23:32:34.761719+0000","last_clean":"2023-09-23T23:32:34.761719+0000","last_became_active":"2023-09-23T23:23:45.463106+0000","last_became_peered":"2023-09-23T23:23:45.463106+0000","last_unstale":"2023-09-23T23:32:34.761719+0000","last_undegraded":"2023-09-23T23:32:34.761719+0000","last_fullsized":"2023-09-23T23:32:34.761719+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.128813+0000","scrub_duration":0.0018535310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":183,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.002185+0000","last_change":"2023-09-23T23:29:55.012082+0000","last_active":"2023-09-23T23:32:35.002185+0000","last_peered":"2023-09-23T23:32:35.002185+0000","last_clean":"2023-09-23T23:32:35.002185+0000","last_became_active":"2023-09-23T23:29:55.000202+0000","last_became_peered":"2023-09-23T23:29:55.000202+0000","last_unstale":"2023-09-23T23:32:35.002185+0000","last_undegraded":"2023-09-23T23:32:35.002185+0000","last_fullsized":"2023-09-23T23:32:35.002185+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:28.669994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":530,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.002414+0000","last_change":"2023-09-23T23:29:55.004242+0000","last_active":"2023-09-23T23:32:35.002414+0000","last_peered":"2023-09-23T23:32:35.002414+0000","last_clean":"2023-09-23T23:32:35.002414+0000","last_became_active":"2023-09-23T23:29:55.003934+0000","last_became_peered":"2023-09-23T23:29:55.003934+0000","last_unstale":"2023-09-23T23:32:35.002414+0000","last_undegraded":"2023-09-23T23:32:35.002414+0000","last_fullsized":"2023-09-23T23:32:35.002414+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:09.283771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":647,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.252842+0000","last_change":"2023-09-23T23:28:14.378358+0000","last_active":"2023-09-23T23:32:35.252842+0000","last_peered":"2023-09-23T23:32:35.252842+0000","last_clean":"2023-09-23T23:32:35.252842+0000","last_became_active":"2023-09-23T23:26:34.550151+0000","last_became_peered":"2023-09-23T23:26:34.550151+0000","last_unstale":"2023-09-23T23:32:35.252842+0000","last_undegraded":"2023-09-23T23:32:35.252842+0000","last_fullsized":"2023-09-23T23:32:35.252842+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:47.794690+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":464,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.679286+0000","last_change":"2023-09-23T23:26:11.569228+0000","last_active":"2023-09-23T23:32:34.679286+0000","last_peered":"2023-09-23T23:32:34.679286+0000","last_clean":"2023-09-23T23:32:34.679286+0000","last_became_active":"2023-09-23T23:26:11.513159+0000","last_became_peered":"2023-09-23T23:26:11.513159+0000","last_unstale":"2023-09-23T23:32:34.679286+0000","last_undegraded":"2023-09-23T23:32:34.679286+0000","last_fullsized":"2023-09-23T23:32:34.679286+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:40.698257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":530,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.682554+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:32:34.682554+0000","last_peered":"2023-09-23T23:32:34.682554+0000","last_clean":"2023-09-23T23:32:34.682554+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:32:34.682554+0000","last_undegraded":"2023-09-23T23:32:34.682554+0000","last_fullsized":"2023-09-23T23:32:34.682554+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":891,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.678789+0000","last_change":"2023-09-23T23:29:58.259420+0000","last_active":"2023-09-23T23:32:34.678789+0000","last_peered":"2023-09-23T23:32:34.678789+0000","last_clean":"2023-09-23T23:32:34.678789+0000","last_became_active":"2023-09-23T23:29:58.258879+0000","last_became_peered":"2023-09-23T23:29:58.258879+0000","last_unstale":"2023-09-23T23:32:34.678789+0000","last_undegraded":"2023-09-23T23:32:34.678789+0000","last_fullsized":"2023-09-23T23:32:34.678789+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:14.307363+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":131,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.680110+0000","last_change":"2023-09-23T23:26:48.563723+0000","last_active":"2023-09-23T23:32:34.680110+0000","last_peered":"2023-09-23T23:32:34.680110+0000","last_clean":"2023-09-23T23:32:34.680110+0000","last_became_active":"2023-09-23T23:26:48.563340+0000","last_became_peered":"2023-09-23T23:26:48.563340+0000","last_unstale":"2023-09-23T23:32:34.680110+0000","last_undegraded":"2023-09-23T23:32:34.680110+0000","last_fullsized":"2023-09-23T23:32:34.680110+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:05.974131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":418,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.680638+0000","last_change":"2023-09-23T23:26:32.040316+0000","last_active":"2023-09-23T23:32:34.680638+0000","last_peered":"2023-09-23T23:32:34.680638+0000","last_clean":"2023-09-23T23:32:34.680638+0000","last_became_active":"2023-09-23T23:26:32.039758+0000","last_became_peered":"2023-09-23T23:26:32.039758+0000","last_unstale":"2023-09-23T23:32:34.680638+0000","last_undegraded":"2023-09-23T23:32:34.680638+0000","last_fullsized":"2023-09-23T23:32:34.680638+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:03.930331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":697,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.678946+0000","last_change":"2023-09-23T23:26:14.527752+0000","last_active":"2023-09-23T23:32:34.678946+0000","last_peered":"2023-09-23T23:32:34.678946+0000","last_clean":"2023-09-23T23:32:34.678946+0000","last_became_active":"2023-09-23T23:26:14.525188+0000","last_became_peered":"2023-09-23T23:26:14.525188+0000","last_unstale":"2023-09-23T23:32:34.678946+0000","last_undegraded":"2023-09-23T23:32:34.678946+0000","last_fullsized":"2023-09-23T23:32:34.678946+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:10.617744+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":396,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.678990+0000","last_change":"2023-09-23T23:29:54.999839+0000","last_active":"2023-09-23T23:32:34.678990+0000","last_peered":"2023-09-23T23:32:34.678990+0000","last_clean":"2023-09-23T23:32:34.678990+0000","last_became_active":"2023-09-23T23:29:54.999530+0000","last_became_peered":"2023-09-23T23:29:54.999530+0000","last_unstale":"2023-09-23T23:32:34.678990+0000","last_undegraded":"2023-09-23T23:32:34.678990+0000","last_fullsized":"2023-09-23T23:32:34.678990+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:43.757346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":468,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.681564+0000","last_change":"2023-09-23T23:28:22.603836+0000","last_active":"2023-09-23T23:32:34.681564+0000","last_peered":"2023-09-23T23:32:34.681564+0000","last_clean":"2023-09-23T23:32:34.681564+0000","last_became_active":"2023-09-23T23:28:22.603490+0000","last_became_peered":"2023-09-23T23:28:22.603490+0000","last_unstale":"2023-09-23T23:32:34.681564+0000","last_undegraded":"2023-09-23T23:32:34.681564+0000","last_fullsized":"2023-09-23T23:32:34.681564+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:00.869241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":895,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.253030+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:32:35.253030+0000","last_peered":"2023-09-23T23:32:35.253030+0000","last_clean":"2023-09-23T23:32:35.253030+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:32:35.253030+0000","last_undegraded":"2023-09-23T23:32:35.253030+0000","last_fullsized":"2023-09-23T23:32:35.253030+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":56,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.678540+0000","last_change":"2023-09-23T23:29:55.020386+0000","last_active":"2023-09-23T23:32:34.678540+0000","last_peered":"2023-09-23T23:32:34.678540+0000","last_clean":"2023-09-23T23:32:34.678540+0000","last_became_active":"2023-09-23T23:29:54.991929+0000","last_became_peered":"2023-09-23T23:29:54.991929+0000","last_unstale":"2023-09-23T23:32:34.678540+0000","last_undegraded":"2023-09-23T23:32:34.678540+0000","last_fullsized":"2023-09-23T23:32:34.678540+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:25.161647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":171,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.681843+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:32:34.681843+0000","last_peered":"2023-09-23T23:32:34.681843+0000","last_clean":"2023-09-23T23:32:34.681843+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:32:34.681843+0000","last_undegraded":"2023-09-23T23:32:34.681843+0000","last_fullsized":"2023-09-23T23:32:34.681843+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":859,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.680468+0000","last_change":"2023-09-23T23:26:51.563868+0000","last_active":"2023-09-23T23:32:34.680468+0000","last_peered":"2023-09-23T23:32:34.680468+0000","last_clean":"2023-09-23T23:32:34.680468+0000","last_became_active":"2023-09-23T23:26:51.563510+0000","last_became_peered":"2023-09-23T23:26:51.563510+0000","last_unstale":"2023-09-23T23:32:34.680468+0000","last_undegraded":"2023-09-23T23:32:34.680468+0000","last_fullsized":"2023-09-23T23:32:34.680468+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:34.244822+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":135,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.679476+0000","last_change":"2023-09-23T23:26:48.673221+0000","last_active":"2023-09-23T23:32:34.679476+0000","last_peered":"2023-09-23T23:32:34.679476+0000","last_clean":"2023-09-23T23:32:34.679476+0000","last_became_active":"2023-09-23T23:26:48.555920+0000","last_became_peered":"2023-09-23T23:26:48.555920+0000","last_unstale":"2023-09-23T23:32:34.679476+0000","last_undegraded":"2023-09-23T23:32:34.679476+0000","last_fullsized":"2023-09-23T23:32:34.679476+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:26.586824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":163,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.252325+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:32:35.252325+0000","last_peered":"2023-09-23T23:32:35.252325+0000","last_clean":"2023-09-23T23:32:35.252325+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:32:35.252325+0000","last_undegraded":"2023-09-23T23:32:35.252325+0000","last_fullsized":"2023-09-23T23:32:35.252325+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":942,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.682112+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:32:34.682112+0000","last_peered":"2023-09-23T23:32:34.682112+0000","last_clean":"2023-09-23T23:32:34.682112+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:32:34.682112+0000","last_undegraded":"2023-09-23T23:32:34.682112+0000","last_fullsized":"2023-09-23T23:32:34.682112+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":135,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.680232+0000","last_change":"2023-09-23T23:26:48.547146+0000","last_active":"2023-09-23T23:32:34.680232+0000","last_peered":"2023-09-23T23:32:34.680232+0000","last_clean":"2023-09-23T23:32:34.680232+0000","last_became_active":"2023-09-23T23:26:48.537682+0000","last_became_peered":"2023-09-23T23:26:48.537682+0000","last_unstale":"2023-09-23T23:32:34.680232+0000","last_undegraded":"2023-09-23T23:32:34.680232+0000","last_fullsized":"2023-09-23T23:32:34.680232+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:10.115563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":557,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.002002+0000","last_change":"2023-09-23T23:26:01.833889+0000","last_active":"2023-09-23T23:32:35.002002+0000","last_peered":"2023-09-23T23:32:35.002002+0000","last_clean":"2023-09-23T23:32:35.002002+0000","last_became_active":"2023-09-23T23:26:01.833626+0000","last_became_peered":"2023-09-23T23:26:01.833626+0000","last_unstale":"2023-09-23T23:32:35.002002+0000","last_undegraded":"2023-09-23T23:32:35.002002+0000","last_fullsized":"2023-09-23T23:32:35.002002+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:25.659671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":822,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.679205+0000","last_change":"2023-09-23T23:29:58.262103+0000","last_active":"2023-09-23T23:32:34.679205+0000","last_peered":"2023-09-23T23:32:34.679205+0000","last_clean":"2023-09-23T23:32:34.679205+0000","last_became_active":"2023-09-23T23:29:58.261689+0000","last_became_peered":"2023-09-23T23:29:58.261689+0000","last_unstale":"2023-09-23T23:32:34.679205+0000","last_undegraded":"2023-09-23T23:32:34.679205+0000","last_fullsized":"2023-09-23T23:32:34.679205+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:44.945353+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":452,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.680550+0000","last_change":"2023-09-23T23:29:55.006616+0000","last_active":"2023-09-23T23:32:34.680550+0000","last_peered":"2023-09-23T23:32:34.680550+0000","last_clean":"2023-09-23T23:32:34.680550+0000","last_became_active":"2023-09-23T23:29:55.000411+0000","last_became_peered":"2023-09-23T23:29:55.000411+0000","last_unstale":"2023-09-23T23:32:34.680550+0000","last_undegraded":"2023-09-23T23:32:34.680550+0000","last_fullsized":"2023-09-23T23:32:34.680550+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:22.629830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":46,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.678673+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:32:34.678673+0000","last_peered":"2023-09-23T23:32:34.678673+0000","last_clean":"2023-09-23T23:32:34.678673+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:32:34.678673+0000","last_undegraded":"2023-09-23T23:32:34.678673+0000","last_fullsized":"2023-09-23T23:32:34.678673+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":596,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.002205+0000","last_change":"2023-09-23T23:28:03.629593+0000","last_active":"2023-09-23T23:32:35.002205+0000","last_peered":"2023-09-23T23:32:35.002205+0000","last_clean":"2023-09-23T23:32:35.002205+0000","last_became_active":"2023-09-23T23:26:51.588385+0000","last_became_peered":"2023-09-23T23:26:51.588385+0000","last_unstale":"2023-09-23T23:32:35.002205+0000","last_undegraded":"2023-09-23T23:32:35.002205+0000","last_fullsized":"2023-09-23T23:32:35.002205+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:20.818366+0000","scrub_duration":0.0050135120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":371,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T23:32:36.016534+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:32:36.016534+0000","last_peered":"2023-09-23T23:32:36.016534+0000","last_clean":"2023-09-23T23:32:36.016534+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:32:36.016534+0000","last_undegraded":"2023-09-23T23:32:36.016534+0000","last_fullsized":"2023-09-23T23:32:36.016534+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":519,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.908653+0000","last_change":"2023-09-23T23:28:45.066443+0000","last_active":"2023-09-23T23:32:35.908653+0000","last_peered":"2023-09-23T23:32:35.908653+0000","last_clean":"2023-09-23T23:32:35.908653+0000","last_became_active":"2023-09-23T23:28:45.066154+0000","last_became_peered":"2023-09-23T23:28:45.066154+0000","last_unstale":"2023-09-23T23:32:35.908653+0000","last_undegraded":"2023-09-23T23:32:35.908653+0000","last_fullsized":"2023-09-23T23:32:35.908653+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:51.651425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":867,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.679141+0000","last_change":"2023-09-23T23:23:45.830384+0000","last_active":"2023-09-23T23:32:34.679141+0000","last_peered":"2023-09-23T23:32:34.679141+0000","last_clean":"2023-09-23T23:32:34.679141+0000","last_became_active":"2023-09-23T23:23:36.025724+0000","last_became_peered":"2023-09-23T23:23:36.025724+0000","last_unstale":"2023-09-23T23:32:34.679141+0000","last_undegraded":"2023-09-23T23:32:34.679141+0000","last_fullsized":"2023-09-23T23:32:34.679141+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:34.345528+0000","scrub_duration":0.001659259,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":501,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.682225+0000","last_change":"2023-09-23T23:10:59.511652+0000","last_active":"2023-09-23T23:32:34.682225+0000","last_peered":"2023-09-23T23:32:34.682225+0000","last_clean":"2023-09-23T23:32:34.682225+0000","last_became_active":"2023-09-23T23:10:59.511163+0000","last_became_peered":"2023-09-23T23:10:59.511163+0000","last_unstale":"2023-09-23T23:32:34.682225+0000","last_undegraded":"2023-09-23T23:32:34.682225+0000","last_fullsized":"2023-09-23T23:32:34.682225+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:21.652930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":510,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.002259+0000","last_change":"2023-09-23T23:29:55.001192+0000","last_active":"2023-09-23T23:32:35.002259+0000","last_peered":"2023-09-23T23:32:35.002259+0000","last_clean":"2023-09-23T23:32:35.002259+0000","last_became_active":"2023-09-23T23:29:55.000904+0000","last_became_peered":"2023-09-23T23:29:55.000904+0000","last_unstale":"2023-09-23T23:32:35.002259+0000","last_undegraded":"2023-09-23T23:32:35.002259+0000","last_fullsized":"2023-09-23T23:32:35.002259+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:29.169209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":813,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.681270+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:32:34.681270+0000","last_peered":"2023-09-23T23:32:34.681270+0000","last_clean":"2023-09-23T23:32:34.681270+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:32:34.681270+0000","last_undegraded":"2023-09-23T23:32:34.681270+0000","last_fullsized":"2023-09-23T23:32:34.681270+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":46,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.678655+0000","last_change":"2023-09-23T23:29:54.998914+0000","last_active":"2023-09-23T23:32:34.678655+0000","last_peered":"2023-09-23T23:32:34.678655+0000","last_clean":"2023-09-23T23:32:34.678655+0000","last_became_active":"2023-09-23T23:29:54.998620+0000","last_became_peered":"2023-09-23T23:29:54.998620+0000","last_unstale":"2023-09-23T23:32:34.678655+0000","last_undegraded":"2023-09-23T23:32:34.678655+0000","last_fullsized":"2023-09-23T23:32:34.678655+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:32.702628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":153,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.002179+0000","last_change":"2023-09-23T23:26:11.516475+0000","last_active":"2023-09-23T23:32:35.002179+0000","last_peered":"2023-09-23T23:32:35.002179+0000","last_clean":"2023-09-23T23:32:35.002179+0000","last_became_active":"2023-09-23T23:26:11.516198+0000","last_became_peered":"2023-09-23T23:26:11.516198+0000","last_unstale":"2023-09-23T23:32:35.002179+0000","last_undegraded":"2023-09-23T23:32:35.002179+0000","last_fullsized":"2023-09-23T23:32:35.002179+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:56.875103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":689,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.678828+0000","last_change":"2023-09-23T23:26:34.548211+0000","last_active":"2023-09-23T23:32:34.678828+0000","last_peered":"2023-09-23T23:32:34.678828+0000","last_clean":"2023-09-23T23:32:34.678828+0000","last_became_active":"2023-09-23T23:26:34.547815+0000","last_became_peered":"2023-09-23T23:26:34.547815+0000","last_unstale":"2023-09-23T23:32:34.678828+0000","last_undegraded":"2023-09-23T23:32:34.678828+0000","last_fullsized":"2023-09-23T23:32:34.678828+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:37.675417+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":153,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.001853+0000","last_change":"2023-09-23T23:26:11.512815+0000","last_active":"2023-09-23T23:32:35.001853+0000","last_peered":"2023-09-23T23:32:35.001853+0000","last_clean":"2023-09-23T23:32:35.001853+0000","last_became_active":"2023-09-23T23:26:11.512466+0000","last_became_peered":"2023-09-23T23:26:11.512466+0000","last_unstale":"2023-09-23T23:32:35.001853+0000","last_undegraded":"2023-09-23T23:32:35.001853+0000","last_fullsized":"2023-09-23T23:32:35.001853+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:10.324158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":540,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.002366+0000","last_change":"2023-09-23T23:29:55.001597+0000","last_active":"2023-09-23T23:32:35.002366+0000","last_peered":"2023-09-23T23:32:35.002366+0000","last_clean":"2023-09-23T23:32:35.002366+0000","last_became_active":"2023-09-23T23:29:55.001314+0000","last_became_peered":"2023-09-23T23:29:55.001314+0000","last_unstale":"2023-09-23T23:32:35.002366+0000","last_undegraded":"2023-09-23T23:32:35.002366+0000","last_fullsized":"2023-09-23T23:32:35.002366+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:00.024016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":872,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.681945+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:32:34.681945+0000","last_peered":"2023-09-23T23:32:34.681945+0000","last_clean":"2023-09-23T23:32:34.681945+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:32:34.681945+0000","last_undegraded":"2023-09-23T23:32:34.681945+0000","last_fullsized":"2023-09-23T23:32:34.681945+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":172,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.681196+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:32:34.681196+0000","last_peered":"2023-09-23T23:32:34.681196+0000","last_clean":"2023-09-23T23:32:34.681196+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:32:34.681196+0000","last_undegraded":"2023-09-23T23:32:34.681196+0000","last_fullsized":"2023-09-23T23:32:34.681196+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":455,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.761774+0000","last_change":"2023-09-23T23:22:39.489325+0000","last_active":"2023-09-23T23:32:34.761774+0000","last_peered":"2023-09-23T23:32:34.761774+0000","last_clean":"2023-09-23T23:32:34.761774+0000","last_became_active":"2023-09-23T23:22:39.489048+0000","last_became_peered":"2023-09-23T23:22:39.489048+0000","last_unstale":"2023-09-23T23:32:34.761774+0000","last_undegraded":"2023-09-23T23:32:34.761774+0000","last_fullsized":"2023-09-23T23:32:34.761774+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:29.984948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":853,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.678956+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:32:34.678956+0000","last_peered":"2023-09-23T23:32:34.678956+0000","last_clean":"2023-09-23T23:32:34.678956+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:32:34.678956+0000","last_undegraded":"2023-09-23T23:32:34.678956+0000","last_fullsized":"2023-09-23T23:32:34.678956+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":235,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.761985+0000","last_change":"2023-09-23T23:23:36.048889+0000","last_active":"2023-09-23T23:32:34.761985+0000","last_peered":"2023-09-23T23:32:34.761985+0000","last_clean":"2023-09-23T23:32:34.761985+0000","last_became_active":"2023-09-23T23:23:36.048572+0000","last_became_peered":"2023-09-23T23:23:36.048572+0000","last_unstale":"2023-09-23T23:32:34.761985+0000","last_undegraded":"2023-09-23T23:32:34.761985+0000","last_fullsized":"2023-09-23T23:32:34.761985+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:55.581146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":547,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.253283+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:32:35.253283+0000","last_peered":"2023-09-23T23:32:35.253283+0000","last_clean":"2023-09-23T23:32:35.253283+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:32:35.253283+0000","last_undegraded":"2023-09-23T23:32:35.253283+0000","last_fullsized":"2023-09-23T23:32:35.253283+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":120,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.678727+0000","last_change":"2023-09-23T23:27:20.549904+0000","last_active":"2023-09-23T23:32:34.678727+0000","last_peered":"2023-09-23T23:32:34.678727+0000","last_clean":"2023-09-23T23:32:34.678727+0000","last_became_active":"2023-09-23T23:27:20.549645+0000","last_became_peered":"2023-09-23T23:27:20.549645+0000","last_unstale":"2023-09-23T23:32:34.678727+0000","last_undegraded":"2023-09-23T23:32:34.678727+0000","last_fullsized":"2023-09-23T23:32:34.678727+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:09.850640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":681,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.908378+0000","last_change":"2023-09-23T23:23:49.320969+0000","last_active":"2023-09-23T23:32:35.908378+0000","last_peered":"2023-09-23T23:32:35.908378+0000","last_clean":"2023-09-23T23:32:35.908378+0000","last_became_active":"2023-09-23T23:22:17.001911+0000","last_became_peered":"2023-09-23T23:22:17.001911+0000","last_unstale":"2023-09-23T23:32:35.908378+0000","last_undegraded":"2023-09-23T23:32:35.908378+0000","last_fullsized":"2023-09-23T23:32:35.908378+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:49.320823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:49.320823+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:07.000984+0000","scrub_duration":0.0018140630000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":244,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.678828+0000","last_change":"2023-09-23T23:29:54.992796+0000","last_active":"2023-09-23T23:32:34.678828+0000","last_peered":"2023-09-23T23:32:34.678828+0000","last_clean":"2023-09-23T23:32:34.678828+0000","last_became_active":"2023-09-23T23:29:54.992394+0000","last_became_peered":"2023-09-23T23:29:54.992394+0000","last_unstale":"2023-09-23T23:32:34.678828+0000","last_undegraded":"2023-09-23T23:32:34.678828+0000","last_fullsized":"2023-09-23T23:32:34.678828+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:01.353635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":521,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.002357+0000","last_change":"2023-09-23T23:29:55.001578+0000","last_active":"2023-09-23T23:32:35.002357+0000","last_peered":"2023-09-23T23:32:35.002357+0000","last_clean":"2023-09-23T23:32:35.002357+0000","last_became_active":"2023-09-23T23:29:55.001264+0000","last_became_peered":"2023-09-23T23:29:55.001264+0000","last_unstale":"2023-09-23T23:32:35.002357+0000","last_undegraded":"2023-09-23T23:32:35.002357+0000","last_fullsized":"2023-09-23T23:32:35.002357+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:16.411769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":120,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.679031+0000","last_change":"2023-09-23T23:27:20.551221+0000","last_active":"2023-09-23T23:32:34.679031+0000","last_peered":"2023-09-23T23:32:34.679031+0000","last_clean":"2023-09-23T23:32:34.679031+0000","last_became_active":"2023-09-23T23:27:20.550900+0000","last_became_peered":"2023-09-23T23:27:20.550900+0000","last_unstale":"2023-09-23T23:32:34.679031+0000","last_undegraded":"2023-09-23T23:32:34.679031+0000","last_fullsized":"2023-09-23T23:32:34.679031+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:23.012149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":833,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.681775+0000","last_change":"2023-09-23T23:28:48.113032+0000","last_active":"2023-09-23T23:32:34.681775+0000","last_peered":"2023-09-23T23:32:34.681775+0000","last_clean":"2023-09-23T23:32:34.681775+0000","last_became_active":"2023-09-23T23:28:48.112666+0000","last_became_peered":"2023-09-23T23:28:48.112666+0000","last_unstale":"2023-09-23T23:32:34.681775+0000","last_undegraded":"2023-09-23T23:32:34.681775+0000","last_fullsized":"2023-09-23T23:32:34.681775+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:59:59.838039+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":268,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.681416+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:32:34.681416+0000","last_peered":"2023-09-23T23:32:34.681416+0000","last_clean":"2023-09-23T23:32:34.681416+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:32:34.681416+0000","last_undegraded":"2023-09-23T23:32:34.681416+0000","last_fullsized":"2023-09-23T23:32:34.681416+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":245,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.682702+0000","last_change":"2023-09-23T23:29:54.992719+0000","last_active":"2023-09-23T23:32:35.682702+0000","last_peered":"2023-09-23T23:32:35.682702+0000","last_clean":"2023-09-23T23:32:35.682702+0000","last_became_active":"2023-09-23T23:29:54.992412+0000","last_became_peered":"2023-09-23T23:29:54.992412+0000","last_unstale":"2023-09-23T23:32:35.682702+0000","last_undegraded":"2023-09-23T23:32:35.682702+0000","last_fullsized":"2023-09-23T23:32:35.682702+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:46.541397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":120,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.761967+0000","last_change":"2023-09-23T23:27:20.551574+0000","last_active":"2023-09-23T23:32:34.761967+0000","last_peered":"2023-09-23T23:32:34.761967+0000","last_clean":"2023-09-23T23:32:34.761967+0000","last_became_active":"2023-09-23T23:27:20.550820+0000","last_became_peered":"2023-09-23T23:27:20.550820+0000","last_unstale":"2023-09-23T23:32:34.761967+0000","last_undegraded":"2023-09-23T23:32:34.761967+0000","last_fullsized":"2023-09-23T23:32:34.761967+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:09.274552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":545,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.681563+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:32:34.681563+0000","last_peered":"2023-09-23T23:32:34.681563+0000","last_clean":"2023-09-23T23:32:34.681563+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:32:34.681563+0000","last_undegraded":"2023-09-23T23:32:34.681563+0000","last_fullsized":"2023-09-23T23:32:34.681563+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":872,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.678961+0000","last_change":"2023-09-23T23:29:58.259227+0000","last_active":"2023-09-23T23:32:34.678961+0000","last_peered":"2023-09-23T23:32:34.678961+0000","last_clean":"2023-09-23T23:32:34.678961+0000","last_became_active":"2023-09-23T23:29:58.258366+0000","last_became_peered":"2023-09-23T23:29:58.258366+0000","last_unstale":"2023-09-23T23:32:34.678961+0000","last_undegraded":"2023-09-23T23:32:34.678961+0000","last_fullsized":"2023-09-23T23:32:34.678961+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:57.581676+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":492,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.682318+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:32:34.682318+0000","last_peered":"2023-09-23T23:32:34.682318+0000","last_clean":"2023-09-23T23:32:34.682318+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:32:34.682318+0000","last_undegraded":"2023-09-23T23:32:34.682318+0000","last_fullsized":"2023-09-23T23:32:34.682318+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":46,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.679317+0000","last_change":"2023-09-23T23:29:54.998861+0000","last_active":"2023-09-23T23:32:34.679317+0000","last_peered":"2023-09-23T23:32:34.679317+0000","last_clean":"2023-09-23T23:32:34.679317+0000","last_became_active":"2023-09-23T23:29:54.998550+0000","last_became_peered":"2023-09-23T23:29:54.998550+0000","last_unstale":"2023-09-23T23:32:34.679317+0000","last_undegraded":"2023-09-23T23:32:34.679317+0000","last_fullsized":"2023-09-23T23:32:34.679317+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:26.515992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":714,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.678875+0000","last_change":"2023-09-23T23:29:58.254396+0000","last_active":"2023-09-23T23:32:34.678875+0000","last_peered":"2023-09-23T23:32:34.678875+0000","last_clean":"2023-09-23T23:32:34.678875+0000","last_became_active":"2023-09-23T23:29:58.254127+0000","last_became_peered":"2023-09-23T23:29:58.254127+0000","last_unstale":"2023-09-23T23:32:34.678875+0000","last_undegraded":"2023-09-23T23:32:34.678875+0000","last_fullsized":"2023-09-23T23:32:34.678875+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:48.863625+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":459,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.678735+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:32:34.678735+0000","last_peered":"2023-09-23T23:32:34.678735+0000","last_clean":"2023-09-23T23:32:34.678735+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:32:34.678735+0000","last_undegraded":"2023-09-23T23:32:34.678735+0000","last_fullsized":"2023-09-23T23:32:34.678735+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":458,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.001776+0000","last_change":"2023-09-23T23:28:22.601436+0000","last_active":"2023-09-23T23:32:35.001776+0000","last_peered":"2023-09-23T23:32:35.001776+0000","last_clean":"2023-09-23T23:32:35.001776+0000","last_became_active":"2023-09-23T23:28:22.601154+0000","last_became_peered":"2023-09-23T23:28:22.601154+0000","last_unstale":"2023-09-23T23:32:35.001776+0000","last_undegraded":"2023-09-23T23:32:35.001776+0000","last_fullsized":"2023-09-23T23:32:35.001776+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:03.296778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":565,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.909227+0000","last_change":"2023-09-23T23:22:38.438448+0000","last_active":"2023-09-23T23:32:35.909227+0000","last_peered":"2023-09-23T23:32:35.909227+0000","last_clean":"2023-09-23T23:32:35.909227+0000","last_became_active":"2023-09-23T23:22:38.437495+0000","last_became_peered":"2023-09-23T23:22:38.437495+0000","last_unstale":"2023-09-23T23:32:35.909227+0000","last_undegraded":"2023-09-23T23:32:35.909227+0000","last_fullsized":"2023-09-23T23:32:35.909227+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:49.587056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":457,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.679423+0000","last_change":"2023-09-23T23:29:58.254981+0000","last_active":"2023-09-23T23:32:34.679423+0000","last_peered":"2023-09-23T23:32:34.679423+0000","last_clean":"2023-09-23T23:32:34.679423+0000","last_became_active":"2023-09-23T23:29:58.254665+0000","last_became_peered":"2023-09-23T23:29:58.254665+0000","last_unstale":"2023-09-23T23:32:34.679423+0000","last_undegraded":"2023-09-23T23:32:34.679423+0000","last_fullsized":"2023-09-23T23:32:34.679423+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.842881+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.842881+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":383,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.680535+0000","last_change":"2023-09-23T23:23:35.037351+0000","last_active":"2023-09-23T23:32:34.680535+0000","last_peered":"2023-09-23T23:32:34.680535+0000","last_clean":"2023-09-23T23:32:34.680535+0000","last_became_active":"2023-09-23T23:23:35.036841+0000","last_became_peered":"2023-09-23T23:23:35.036841+0000","last_unstale":"2023-09-23T23:32:34.680535+0000","last_undegraded":"2023-09-23T23:32:34.680535+0000","last_fullsized":"2023-09-23T23:32:34.680535+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:20.618292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":235,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.762089+0000","last_change":"2023-09-23T23:26:32.037099+0000","last_active":"2023-09-23T23:32:34.762089+0000","last_peered":"2023-09-23T23:32:34.762089+0000","last_clean":"2023-09-23T23:32:34.762089+0000","last_became_active":"2023-09-23T23:26:32.036681+0000","last_became_peered":"2023-09-23T23:26:32.036681+0000","last_unstale":"2023-09-23T23:32:34.762089+0000","last_undegraded":"2023-09-23T23:32:34.762089+0000","last_fullsized":"2023-09-23T23:32:34.762089+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:48.926576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":265,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.253187+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:32:35.253187+0000","last_peered":"2023-09-23T23:32:35.253187+0000","last_clean":"2023-09-23T23:32:35.253187+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:32:35.253187+0000","last_undegraded":"2023-09-23T23:32:35.253187+0000","last_fullsized":"2023-09-23T23:32:35.253187+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":863,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.002332+0000","last_change":"2023-09-23T23:29:58.258327+0000","last_active":"2023-09-23T23:32:35.002332+0000","last_peered":"2023-09-23T23:32:35.002332+0000","last_clean":"2023-09-23T23:32:35.002332+0000","last_became_active":"2023-09-23T23:29:58.257930+0000","last_became_peered":"2023-09-23T23:29:58.257930+0000","last_unstale":"2023-09-23T23:32:35.002332+0000","last_undegraded":"2023-09-23T23:32:35.002332+0000","last_fullsized":"2023-09-23T23:32:35.002332+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:00.144157+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":528,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T23:32:36.016337+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:32:36.016337+0000","last_peered":"2023-09-23T23:32:36.016337+0000","last_clean":"2023-09-23T23:32:36.016337+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:32:36.016337+0000","last_undegraded":"2023-09-23T23:32:36.016337+0000","last_fullsized":"2023-09-23T23:32:36.016337+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":129,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.761655+0000","last_change":"2023-09-23T23:29:54.999590+0000","last_active":"2023-09-23T23:32:34.761655+0000","last_peered":"2023-09-23T23:32:34.761655+0000","last_clean":"2023-09-23T23:32:34.761655+0000","last_became_active":"2023-09-23T23:29:54.999209+0000","last_became_peered":"2023-09-23T23:29:54.999209+0000","last_unstale":"2023-09-23T23:32:34.761655+0000","last_undegraded":"2023-09-23T23:32:34.761655+0000","last_fullsized":"2023-09-23T23:32:34.761655+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:32.208030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":63,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.681065+0000","last_change":"2023-09-23T23:28:45.061717+0000","last_active":"2023-09-23T23:32:34.681065+0000","last_peered":"2023-09-23T23:32:34.681065+0000","last_clean":"2023-09-23T23:32:34.681065+0000","last_became_active":"2023-09-23T23:28:45.061404+0000","last_became_peered":"2023-09-23T23:28:45.061404+0000","last_unstale":"2023-09-23T23:32:34.681065+0000","last_undegraded":"2023-09-23T23:32:34.681065+0000","last_fullsized":"2023-09-23T23:32:34.681065+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:46.614114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":917,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.682656+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:32:34.682656+0000","last_peered":"2023-09-23T23:32:34.682656+0000","last_clean":"2023-09-23T23:32:34.682656+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:32:34.682656+0000","last_undegraded":"2023-09-23T23:32:34.682656+0000","last_fullsized":"2023-09-23T23:32:34.682656+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":46,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.679063+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:32:34.679063+0000","last_peered":"2023-09-23T23:32:34.679063+0000","last_clean":"2023-09-23T23:32:34.679063+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:32:34.679063+0000","last_undegraded":"2023-09-23T23:32:34.679063+0000","last_fullsized":"2023-09-23T23:32:34.679063+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":520,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.909011+0000","last_change":"2023-09-23T23:22:38.482705+0000","last_active":"2023-09-23T23:32:35.909011+0000","last_peered":"2023-09-23T23:32:35.909011+0000","last_clean":"2023-09-23T23:32:35.909011+0000","last_became_active":"2023-09-23T23:22:38.438239+0000","last_became_peered":"2023-09-23T23:22:38.438239+0000","last_unstale":"2023-09-23T23:32:35.909011+0000","last_undegraded":"2023-09-23T23:32:35.909011+0000","last_fullsized":"2023-09-23T23:32:35.909011+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:52.006349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":163,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.001917+0000","last_change":"2023-09-23T23:29:55.000912+0000","last_active":"2023-09-23T23:32:35.001917+0000","last_peered":"2023-09-23T23:32:35.001917+0000","last_clean":"2023-09-23T23:32:35.001917+0000","last_became_active":"2023-09-23T23:29:54.999122+0000","last_became_peered":"2023-09-23T23:29:54.999122+0000","last_unstale":"2023-09-23T23:32:35.001917+0000","last_undegraded":"2023-09-23T23:32:35.001917+0000","last_fullsized":"2023-09-23T23:32:35.001917+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:10.962943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":933,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T23:32:36.016858+0000","last_change":"2023-09-23T23:29:58.264914+0000","last_active":"2023-09-23T23:32:36.016858+0000","last_peered":"2023-09-23T23:32:36.016858+0000","last_clean":"2023-09-23T23:32:36.016858+0000","last_became_active":"2023-09-23T23:29:58.264356+0000","last_became_peered":"2023-09-23T23:29:58.264356+0000","last_unstale":"2023-09-23T23:32:36.016858+0000","last_undegraded":"2023-09-23T23:32:36.016858+0000","last_fullsized":"2023-09-23T23:32:36.016858+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:15:15.828500+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":202,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.002258+0000","last_change":"2023-09-23T23:28:22.603176+0000","last_active":"2023-09-23T23:32:35.002258+0000","last_peered":"2023-09-23T23:32:35.002258+0000","last_clean":"2023-09-23T23:32:35.002258+0000","last_became_active":"2023-09-23T23:28:22.602824+0000","last_became_peered":"2023-09-23T23:28:22.602824+0000","last_unstale":"2023-09-23T23:32:35.002258+0000","last_undegraded":"2023-09-23T23:32:35.002258+0000","last_fullsized":"2023-09-23T23:32:35.002258+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:35.934235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":285,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.682428+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:32:34.682428+0000","last_peered":"2023-09-23T23:32:34.682428+0000","last_clean":"2023-09-23T23:32:34.682428+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:32:34.682428+0000","last_undegraded":"2023-09-23T23:32:34.682428+0000","last_fullsized":"2023-09-23T23:32:34.682428+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":678,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.681719+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:32:34.681719+0000","last_peered":"2023-09-23T23:32:34.681719+0000","last_clean":"2023-09-23T23:32:34.681719+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:32:34.681719+0000","last_undegraded":"2023-09-23T23:32:34.681719+0000","last_fullsized":"2023-09-23T23:32:34.681719+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":273,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.002350+0000","last_change":"2023-09-23T23:22:38.452968+0000","last_active":"2023-09-23T23:32:35.002350+0000","last_peered":"2023-09-23T23:32:35.002350+0000","last_clean":"2023-09-23T23:32:35.002350+0000","last_became_active":"2023-09-23T23:22:38.437986+0000","last_became_peered":"2023-09-23T23:22:38.437986+0000","last_unstale":"2023-09-23T23:32:35.002350+0000","last_undegraded":"2023-09-23T23:32:35.002350+0000","last_fullsized":"2023-09-23T23:32:35.002350+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:10.333271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":163,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.001963+0000","last_change":"2023-09-23T23:27:20.558454+0000","last_active":"2023-09-23T23:32:35.001963+0000","last_peered":"2023-09-23T23:32:35.001963+0000","last_clean":"2023-09-23T23:32:35.001963+0000","last_became_active":"2023-09-23T23:27:20.558204+0000","last_became_peered":"2023-09-23T23:27:20.558204+0000","last_unstale":"2023-09-23T23:32:35.001963+0000","last_undegraded":"2023-09-23T23:32:35.001963+0000","last_fullsized":"2023-09-23T23:32:35.001963+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:15.079990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":287,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.002430+0000","last_change":"2023-09-23T23:29:55.022469+0000","last_active":"2023-09-23T23:32:35.002430+0000","last_peered":"2023-09-23T23:32:35.002430+0000","last_clean":"2023-09-23T23:32:35.002430+0000","last_became_active":"2023-09-23T23:29:55.003082+0000","last_became_peered":"2023-09-23T23:29:55.003082+0000","last_unstale":"2023-09-23T23:32:35.002430+0000","last_undegraded":"2023-09-23T23:32:35.002430+0000","last_fullsized":"2023-09-23T23:32:35.002430+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:15.791643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":153,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.002034+0000","last_change":"2023-09-23T23:26:11.527453+0000","last_active":"2023-09-23T23:32:35.002034+0000","last_peered":"2023-09-23T23:32:35.002034+0000","last_clean":"2023-09-23T23:32:35.002034+0000","last_became_active":"2023-09-23T23:26:11.527208+0000","last_became_peered":"2023-09-23T23:26:11.527208+0000","last_unstale":"2023-09-23T23:32:35.002034+0000","last_undegraded":"2023-09-23T23:32:35.002034+0000","last_fullsized":"2023-09-23T23:32:35.002034+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:15.083611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":818,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.679621+0000","last_change":"2023-09-23T23:29:58.252042+0000","last_active":"2023-09-23T23:32:34.679621+0000","last_peered":"2023-09-23T23:32:34.679621+0000","last_clean":"2023-09-23T23:32:34.679621+0000","last_became_active":"2023-09-23T23:29:58.251749+0000","last_became_peered":"2023-09-23T23:29:58.251749+0000","last_unstale":"2023-09-23T23:32:34.679621+0000","last_undegraded":"2023-09-23T23:32:34.679621+0000","last_fullsized":"2023-09-23T23:32:34.679621+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:39.324116+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":560,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.681643+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:32:34.681643+0000","last_peered":"2023-09-23T23:32:34.681643+0000","last_clean":"2023-09-23T23:32:34.681643+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:32:34.681643+0000","last_undegraded":"2023-09-23T23:32:34.681643+0000","last_fullsized":"2023-09-23T23:32:34.681643+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":544,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.682786+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:32:34.682786+0000","last_peered":"2023-09-23T23:32:34.682786+0000","last_clean":"2023-09-23T23:32:34.682786+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:32:34.682786+0000","last_undegraded":"2023-09-23T23:32:34.682786+0000","last_fullsized":"2023-09-23T23:32:34.682786+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":907,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.761485+0000","last_change":"2023-09-23T23:26:14.527390+0000","last_active":"2023-09-23T23:32:34.761485+0000","last_peered":"2023-09-23T23:32:34.761485+0000","last_clean":"2023-09-23T23:32:34.761485+0000","last_became_active":"2023-09-23T23:26:14.527008+0000","last_became_peered":"2023-09-23T23:26:14.527008+0000","last_unstale":"2023-09-23T23:32:34.761485+0000","last_undegraded":"2023-09-23T23:32:34.761485+0000","last_fullsized":"2023-09-23T23:32:34.761485+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:36.460827+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":344,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:34.715056+0000","last_change":"2023-09-23T23:28:22.600921+0000","last_active":"2023-09-23T23:32:34.715056+0000","last_peered":"2023-09-23T23:32:34.715056+0000","last_clean":"2023-09-23T23:32:34.715056+0000","last_became_active":"2023-09-23T23:28:22.600565+0000","last_became_peered":"2023-09-23T23:28:22.600565+0000","last_unstale":"2023-09-23T23:32:34.715056+0000","last_undegraded":"2023-09-23T23:32:34.715056+0000","last_fullsized":"2023-09-23T23:32:34.715056+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:07.793563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":587,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T23:32:35.001757+0000","last_change":"2023-09-23T23:29:55.002051+0000","last_active":"2023-09-23T23:32:35.001757+0000","last_peered":"2023-09-23T23:32:35.001757+0000","last_clean":"2023-09-23T23:32:35.001757+0000","last_became_active":"2023-09-23T23:29:55.001732+0000","last_became_peered":"2023-09-23T23:29:55.001732+0000","last_unstale":"2023-09-23T23:32:35.001757+0000","last_undegraded":"2023-09-23T23:32:35.001757+0000","last_fullsized":"2023-09-23T23:32:35.001757+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:28:01.353263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":10412,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":603785,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1023292,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782638,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":494384,"kb_used_data":208552,"kb_used_omap":8,"kb_used_meta":285815,"kb_avail":93250768,"statfs":{"total":95995035648,"available":95488786432,"internally_reserved":0,"allocated":213557248,"data_stored":212099487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":292674629},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:31:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9350000000000001,"5min":2.7149999999999999,"15min":2.2530000000000001},"min":{"1min":0.224,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":36.956000000000003,"5min":44.094999999999999,"15min":85.721000000000004},"last":0.47999999999999998},{"interface":"front","average":{"1min":2.782,"5min":2.8889999999999998,"15min":2.1949999999999998},"min":{"1min":0.27900000000000003,"5min":0.253,"15min":0.19800000000000001},"max":{"1min":37.247999999999998,"5min":44.335999999999999,"15min":85.965000000000003},"last":0.32000000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:31:48 2023","interfaces":[{"interface":"back","average":{"1min":7.2539999999999996,"5min":5.2149999999999999,"15min":2.633},"min":{"1min":0.29099999999999998,"5min":0.23200000000000001,"15min":0.22900000000000001},"max":{"1min":92.061000000000007,"5min":198.58000000000001,"15min":198.58000000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":7.3380000000000001,"5min":5.1790000000000003,"15min":2.8650000000000002},"min":{"1min":0.28799999999999998,"5min":0.246,"15min":0.246},"max":{"1min":92.075000000000003,"5min":198.71100000000001,"15min":198.71100000000001},"last":0.35299999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:31:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3120000000000003,"5min":6.3470000000000004,"15min":3.52},"min":{"1min":0.33500000000000002,"5min":0.27200000000000002,"15min":0.26000000000000001},"max":{"1min":27.498000000000001,"5min":203.58000000000001,"15min":203.58000000000001},"last":0.26900000000000002},{"interface":"front","average":{"1min":3.6800000000000002,"5min":5.8659999999999997,"15min":2.9769999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":27.574000000000002,"5min":198.69,"15min":198.69},"last":0.33500000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:31:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1320000000000001,"5min":6.2489999999999997,"15min":3.0910000000000002},"min":{"1min":0.30499999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":31.391999999999999,"5min":279.26499999999999,"15min":279.26499999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":11.055,"5min":7.5060000000000002,"15min":3.4340000000000002},"min":{"1min":0.32000000000000001,"5min":0.316,"15min":0.29599999999999999},"max":{"1min":150.608,"5min":279.29899999999998,"15min":279.29899999999998},"last":0.40100000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:31:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":0.78600000000000003,"15min":0.81000000000000005},"min":{"1min":0.26100000000000001,"5min":0.246,"15min":0.246},"max":{"1min":5.4660000000000002,"5min":5.4660000000000002,"15min":6.8380000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.67800000000000005,"15min":0.84199999999999997},"min":{"1min":0.28100000000000003,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.234,"5min":7.9189999999999996,"15min":7.9189999999999996},"last":0.42199999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:31:48 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":2.9399999999999999,"15min":1.6850000000000001},"min":{"1min":0.29199999999999998,"5min":0.27500000000000002,"15min":0.26500000000000001},"max":{"1min":2.3050000000000002,"5min":210.83600000000001,"15min":210.83600000000001},"last":0.375},{"interface":"front","average":{"1min":0.63500000000000001,"5min":2.8340000000000001,"15min":1.484},"min":{"1min":0.316,"5min":0.28699999999999998,"15min":0.28199999999999997},"max":{"1min":2.448,"5min":207.04599999999999,"15min":207.04599999999999},"last":0.434}]},{"osd":6,"last update":"Sat Sep 23 23:31:48 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.77900000000000003,"15min":0.97399999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.9510000000000001,"5min":4.758,"15min":62.213000000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.4399999999999999,"5min":0.95299999999999996,"15min":1.2230000000000001},"min":{"1min":0.39400000000000002,"5min":0.34200000000000003,"15min":0.33300000000000002},"max":{"1min":7.0350000000000001,"5min":7.0350000000000001,"15min":62.805999999999997},"last":0.5}]}]},{"osd":6,"up_from":46,"seq":197568496115,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":666204,"kb_used_data":5780,"kb_used_omap":0,"kb_used_meta":660416,"kb_avail":93078948,"statfs":{"total":95995035648,"available":95312842752,"internally_reserved":0,"allocated":5918720,"data_stored":4480484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":676265984},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:32:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3620000000000001,"5min":3.081,"15min":6.6680000000000001},"min":{"1min":0.251,"5min":0.223,"15min":0.20399999999999999},"max":{"1min":8.8119999999999994,"5min":135.31999999999999,"15min":161.64599999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.97599999999999998,"5min":3.339,"15min":6.4199999999999999},"min":{"1min":0.23999999999999999,"5min":0.23000000000000001,"15min":0.22900000000000001},"max":{"1min":3.2269999999999999,"5min":135.34299999999999,"15min":161.38499999999999},"last":0.32500000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:32:08 2023","interfaces":[{"interface":"back","average":{"1min":4.7359999999999998,"5min":3.9409999999999998,"15min":3.6749999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.245},"max":{"1min":65.891000000000005,"5min":89.605000000000004,"15min":210.869},"last":0.309},{"interface":"front","average":{"1min":4.3170000000000002,"5min":4.1980000000000004,"15min":3.7989999999999999},"min":{"1min":0.27200000000000002,"5min":0.249,"15min":0.23300000000000001},"max":{"1min":66.087000000000003,"5min":91.025999999999996,"15min":210.89699999999999},"last":0.441}]},{"osd":2,"last update":"Sat Sep 23 23:31:56 2023","interfaces":[{"interface":"back","average":{"1min":7.0339999999999998,"5min":4.319,"15min":3.8820000000000001},"min":{"1min":0.29599999999999999,"5min":0.28399999999999997,"15min":0.27800000000000002},"max":{"1min":97.408000000000001,"5min":97.408000000000001,"15min":127.44499999999999},"last":0.36199999999999999},{"interface":"front","average":{"1min":12.733000000000001,"5min":5.5,"15min":4.2569999999999997},"min":{"1min":0.375,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":125.539,"5min":125.539,"15min":125.539},"last":0.45600000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:31:41 2023","interfaces":[{"interface":"back","average":{"1min":7.6459999999999999,"5min":2.9969999999999999,"15min":3.3279999999999998},"min":{"1min":0.33700000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":138.87299999999999,"5min":138.87299999999999,"15min":138.87299999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":7.5250000000000004,"5min":3.1699999999999999,"15min":3.6070000000000002},"min":{"1min":0.36299999999999999,"5min":0.32700000000000001,"15min":0.24199999999999999},"max":{"1min":138.976,"5min":138.976,"15min":138.976},"last":0.371}]},{"osd":4,"last update":"Sat Sep 23 23:32:20 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.70699999999999996,"15min":1.286},"min":{"1min":0.247,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":2.0299999999999998,"5min":8.1999999999999993,"15min":166.05199999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.79000000000000004,"15min":1.619},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27300000000000002},"max":{"1min":2.9980000000000002,"5min":7.915,"15min":166.631},"last":0.34599999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:32:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.92900000000000005,"15min":1.1890000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.27900000000000003},"max":{"1min":5.915,"5min":11.473000000000001,"15min":91.899000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.4330000000000001,"5min":0.91200000000000003,"15min":1.302},"min":{"1min":0.35699999999999998,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":5.3490000000000002,"5min":7.5670000000000002,"15min":94.174000000000007},"last":0.38}]},{"osd":7,"last update":"Sat Sep 23 23:31:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":0.93799999999999994,"15min":0.91600000000000004},"min":{"1min":0.34499999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.944,"5min":12.208,"15min":12.208},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.597,"5min":1.1399999999999999,"15min":0.93899999999999995},"min":{"1min":0.38500000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":11.515000000000001,"5min":11.515000000000001,"15min":11.515000000000001},"last":0.41799999999999998}]}]},{"osd":1,"up_from":112,"seq":481036337456,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1155852,"kb_used_data":323204,"kb_used_omap":4,"kb_used_meta":832635,"kb_avail":92589300,"statfs":{"total":95995035648,"available":94811443200,"internally_reserved":0,"allocated":330960896,"data_stored":328481347,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":852619150},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:32:11 2023","interfaces":[{"interface":"back","average":{"1min":2.907,"5min":5.2539999999999996,"15min":4.04},"min":{"1min":0.17799999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":15.483000000000001,"5min":130.768,"15min":130.768},"last":0.24099999999999999},{"interface":"front","average":{"1min":2.5179999999999998,"5min":4.4169999999999998,"15min":3.5619999999999998},"min":{"1min":0.218,"5min":0.19,"15min":0.19},"max":{"1min":15.699,"5min":134.929,"15min":134.929},"last":0.40799999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":18.552,"5min":11.241,"15min":6.3259999999999996},"min":{"1min":0.214,"5min":0.214,"15min":0.20100000000000001},"max":{"1min":182.554,"5min":231.67400000000001,"15min":231.67400000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":19.277000000000001,"5min":11.140000000000001,"15min":6.3369999999999997},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":180.09100000000001,"5min":232.21000000000001,"15min":232.21000000000001},"last":0.28399999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":3.6469999999999998,"5min":7.7830000000000004,"15min":5.3460000000000001},"min":{"1min":0.23899999999999999,"5min":0.23799999999999999,"15min":0.20799999999999999},"max":{"1min":14.57,"5min":234.404,"15min":234.404},"last":0.47699999999999998},{"interface":"front","average":{"1min":3.8849999999999998,"5min":7.5510000000000002,"15min":5.1619999999999999},"min":{"1min":0.30299999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":15.74,"5min":236.33699999999999,"15min":236.33699999999999},"last":0.373}]},{"osd":4,"last update":"Sat Sep 23 23:32:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5840000000000001,"5min":1.724,"15min":1.655},"min":{"1min":0.34399999999999997,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":12.042999999999999,"5min":12.042999999999999,"15min":18.474},"last":0.33800000000000002},{"interface":"front","average":{"1min":2.1240000000000001,"5min":1.6890000000000001,"15min":1.6819999999999999},"min":{"1min":0.315,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.3239999999999998,"5min":18.108000000000001,"15min":18.108000000000001},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:32:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9770000000000001,"5min":1.484,"15min":2.9399999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.0190000000000001,"5min":9.0229999999999997,"15min":250.33000000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.6909999999999998,"5min":1.7370000000000001,"15min":3.2679999999999998},"min":{"1min":0.38,"5min":0.34499999999999997,"15min":0.308},"max":{"1min":12.007999999999999,"5min":12.007999999999999,"15min":250.21199999999999},"last":0.42999999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:32:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2080000000000002,"5min":1.5469999999999999,"15min":2.3250000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":5.601,"5min":7.4470000000000001,"15min":74.087999999999994},"last":0.53900000000000003},{"interface":"front","average":{"1min":2.5790000000000002,"5min":1.8160000000000001,"15min":2.4159999999999999},"min":{"1min":0.40799999999999997,"5min":0.37,"15min":0.32800000000000001},"max":{"1min":11.927,"5min":11.927,"15min":73.974999999999994},"last":0.38900000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":1.6719999999999999,"15min":3.3820000000000001},"min":{"1min":0.39000000000000001,"5min":0.33000000000000002,"15min":0.32900000000000001},"max":{"1min":6.2380000000000004,"5min":11.385,"15min":78.308000000000007},"last":0.42299999999999999},{"interface":"front","average":{"1min":2.2589999999999999,"5min":1.907,"15min":3.1110000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.4829999999999997,"5min":8.9930000000000003,"15min":77.204999999999998},"last":0.46300000000000002}]}]},{"osd":0,"up_from":91,"seq":390842024280,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1181132,"kb_used_data":6980,"kb_used_omap":5,"kb_used_meta":1174138,"kb_avail":92564020,"statfs":{"total":95995035648,"available":94785556480,"internally_reserved":0,"allocated":7147520,"data_stored":4603458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1202317946},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":15,"apply_latency_ms":15,"commit_latency_ns":15000000,"apply_latency_ns":15000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:32:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2629999999999999,"5min":11.031000000000001,"15min":6.1470000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.17499999999999999},"max":{"1min":14.115,"5min":213.45699999999999,"15min":213.45699999999999},"last":0.27200000000000002},{"interface":"front","average":{"1min":2.4510000000000001,"5min":11.238,"15min":5.181},"min":{"1min":0.22500000000000001,"5min":0.186,"15min":0.17599999999999999},"max":{"1min":14.478,"5min":213.04900000000001,"15min":213.04900000000001},"last":0.28799999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:31:49 2023","interfaces":[{"interface":"back","average":{"1min":15.987,"5min":6.5490000000000004,"15min":5.8869999999999996},"min":{"1min":0.17699999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":220.779,"5min":220.779,"15min":220.779},"last":0.22800000000000001},{"interface":"front","average":{"1min":18.346,"5min":7.2809999999999997,"15min":6.0990000000000002},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17599999999999999},"max":{"1min":218.74700000000001,"5min":218.74700000000001,"15min":218.74700000000001},"last":0.318}]},{"osd":3,"last update":"Sat Sep 23 23:32:24 2023","interfaces":[{"interface":"back","average":{"1min":8.1720000000000006,"5min":7.9409999999999998,"15min":5.2629999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":114.381,"5min":220.732,"15min":220.732},"last":0.32800000000000001},{"interface":"front","average":{"1min":9.0609999999999999,"5min":8.4740000000000002,"15min":5.2809999999999997},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":114.30500000000001,"5min":218.78200000000001,"15min":218.78200000000001},"last":0.33900000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:32:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2360000000000002,"5min":2.8530000000000002,"15min":3.121},"min":{"1min":0.33500000000000002,"5min":0.28899999999999998,"15min":0.27000000000000002},"max":{"1min":14.676,"5min":108.01900000000001,"15min":138.81299999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":2.5249999999999999,"5min":2.7370000000000001,"15min":3.7770000000000001},"min":{"1min":0.316,"5min":0.312,"15min":0.28799999999999998},"max":{"1min":9.9039999999999999,"5min":107.898,"15min":171.251},"last":0.41499999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:32:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":1.583,"15min":3.0800000000000001},"min":{"1min":0.35099999999999998,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":9.8230000000000004,"5min":9.8230000000000004,"15min":171.30699999999999},"last":0.442},{"interface":"front","average":{"1min":1.6759999999999999,"5min":1.4119999999999999,"15min":2.2069999999999999},"min":{"1min":0.34100000000000003,"5min":0.253,"15min":0.253},"max":{"1min":8.9909999999999997,"5min":11.307,"15min":139.40799999999999},"last":0.40300000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:32:01 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":2.6859999999999999,"15min":3.6040000000000001},"min":{"1min":0.372,"5min":0.371,"15min":0.29499999999999998},"max":{"1min":9.8729999999999993,"5min":108.67,"15min":171.33600000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":2.1019999999999999,"5min":2.6120000000000001,"15min":3.71},"min":{"1min":0.39700000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.8320000000000007,"5min":108.804,"15min":184.13499999999999},"last":0.36699999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:31:49 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":3.0590000000000002,"15min":2.6520000000000001},"min":{"1min":0.38700000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.452,"5min":111.71299999999999,"15min":138.77699999999999},"last":0.377},{"interface":"front","average":{"1min":2.5110000000000001,"5min":2.8079999999999998,"15min":2.3599999999999999},"min":{"1min":0.36199999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.9670000000000005,"5min":111.108,"15min":139.37899999999999},"last":0.45500000000000002}]}]},{"osd":2,"up_from":199,"seq":854698492101,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":839312,"kb_used_data":153544,"kb_used_omap":7,"kb_used_meta":685752,"kb_avail":92905840,"statfs":{"total":95995035648,"available":95135580160,"internally_reserved":0,"allocated":157229056,"data_stored":154688903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":702210129},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":2.8039999999999998,"5min":3.2400000000000002,"15min":4.3049999999999997},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.16700000000000001},"max":{"1min":11.550000000000001,"5min":58.802,"15min":119.277},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.9370000000000001,"5min":3,"15min":4.1870000000000003},"min":{"1min":0.25900000000000001,"5min":0.214,"15min":0.18099999999999999},"max":{"1min":6.9649999999999999,"5min":58.115000000000002,"15min":119.56100000000001},"last":0.28999999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":4.6870000000000003,"5min":4.2910000000000004,"15min":4.5460000000000003},"min":{"1min":0.40799999999999997,"5min":0.23300000000000001,"15min":0.20699999999999999},"max":{"1min":16.541,"5min":82.876000000000005,"15min":157.98699999999999},"last":0.20999999999999999},{"interface":"front","average":{"1min":4.6120000000000001,"5min":4.4459999999999997,"15min":4.617},"min":{"1min":0.246,"5min":0.24099999999999999,"15min":0.22600000000000001},"max":{"1min":16.305,"5min":82.909000000000006,"15min":158.03800000000001},"last":0.27000000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":2.8090000000000002,"5min":4.4169999999999998,"15min":4.1529999999999996},"min":{"1min":0.28100000000000003,"5min":0.24299999999999999,"15min":0.191},"max":{"1min":7.851,"5min":103.227,"15min":119.366},"last":0.33300000000000002},{"interface":"front","average":{"1min":2.423,"5min":4.2649999999999997,"15min":3.6589999999999998},"min":{"1min":0.30499999999999999,"5min":0.26500000000000001,"15min":0.23499999999999999},"max":{"1min":7.6239999999999997,"5min":102.693,"15min":102.693},"last":0.35299999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":2.056,"15min":3.9249999999999998},"min":{"1min":0.41099999999999998,"5min":0.27700000000000002,"15min":0.25},"max":{"1min":6.0869999999999997,"5min":11.586,"15min":332.98099999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.6659999999999999,"5min":2.0369999999999999,"15min":3.3769999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":7.7629999999999999,"5min":11.621,"15min":332.96199999999999},"last":0.437}]},{"osd":5,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.9770000000000001,"15min":2.2770000000000001},"min":{"1min":0.38400000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.1449999999999996,"5min":10.192,"15min":119.919},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.472,"5min":1.8779999999999999,"15min":2.1970000000000001},"min":{"1min":0.34200000000000003,"5min":0.33900000000000002,"15min":0.33000000000000002},"max":{"1min":7.7910000000000004,"5min":8.7110000000000003,"15min":119.306},"last":0.41099999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1499999999999999,"5min":2.2869999999999999,"15min":4.2039999999999997},"min":{"1min":0.40799999999999997,"5min":0.315,"15min":0.31},"max":{"1min":6.117,"5min":14.795999999999999,"15min":349.32799999999997},"last":0.38200000000000001},{"interface":"front","average":{"1min":3.0739999999999998,"5min":2.2450000000000001,"15min":3.714},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":7.7249999999999996,"5min":12.496,"15min":349.17700000000002},"last":0.38900000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:31:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":2.0990000000000002,"15min":1.6359999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":6.7640000000000002,"5min":15.734999999999999,"15min":20.350000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":2.0880000000000001,"5min":2.0590000000000002,"15min":1.6839999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.6340000000000003,"5min":15.709,"15min":20.321000000000002},"last":0.45100000000000001}]}]},{"osd":3,"up_from":148,"seq":635655160070,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1110400,"kb_used_data":263992,"kb_used_omap":2,"kb_used_meta":846397,"kb_avail":92634752,"statfs":{"total":95995035648,"available":94857986048,"internally_reserved":0,"allocated":270327808,"data_stored":267856409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":866710581},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:31:39 2023","interfaces":[{"interface":"back","average":{"1min":1.9790000000000001,"5min":2.6659999999999999,"15min":2.923},"min":{"1min":0.23999999999999999,"5min":0.17699999999999999,"15min":0.158},"max":{"1min":6.5259999999999998,"5min":40.901000000000003,"15min":103.866},"last":0.252},{"interface":"front","average":{"1min":3.0449999999999999,"5min":3.581,"15min":2.996},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":9.984,"5min":46.204000000000001,"15min":101.884},"last":0.28799999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:31:39 2023","interfaces":[{"interface":"back","average":{"1min":3.5880000000000001,"5min":3.7650000000000001,"15min":2.8919999999999999},"min":{"1min":0.224,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":11.976000000000001,"5min":85.242999999999995,"15min":85.242999999999995},"last":0.27400000000000002},{"interface":"front","average":{"1min":2.3759999999999999,"5min":4.0640000000000001,"15min":2.7130000000000001},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.159000000000001,"5min":85.302000000000007,"15min":85.302000000000007},"last":0.30099999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:31:50 2023","interfaces":[{"interface":"back","average":{"1min":13.750999999999999,"5min":6.625,"15min":4.7140000000000004},"min":{"1min":0.27400000000000002,"5min":0.219,"15min":0.19800000000000001},"max":{"1min":73.742000000000004,"5min":84.724999999999994,"15min":101.279},"last":0.33600000000000002},{"interface":"front","average":{"1min":13.571,"5min":6.6059999999999999,"15min":4.367},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.214},"max":{"1min":76.525999999999996,"5min":84.567999999999998,"15min":101.294},"last":0.35799999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:31:39 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":1.3640000000000001,"15min":1.3859999999999999},"min":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":4.2590000000000003,"5min":8.5869999999999997,"15min":14.754},"last":0.433},{"interface":"front","average":{"1min":3.1080000000000001,"5min":2.1459999999999999,"15min":1.7989999999999999},"min":{"1min":0.40500000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":10.029999999999999,"5min":24.782,"15min":24.782},"last":0.42499999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:31:39 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":2.1080000000000001,"15min":1.7310000000000001},"min":{"1min":0.39800000000000002,"5min":0.29599999999999999,"15min":0.28000000000000003},"max":{"1min":10.081,"5min":24.565000000000001,"15min":24.565000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.4119999999999999,"5min":2.0990000000000002,"15min":1.905},"min":{"1min":0.32600000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":11.760999999999999,"5min":22.472999999999999,"15min":22.472999999999999},"last":0.50600000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:31:39 2023","interfaces":[{"interface":"back","average":{"1min":2.996,"5min":2.2069999999999999,"15min":2.0270000000000001},"min":{"1min":0.46200000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.055999999999999,"5min":24.658000000000001,"15min":53.057000000000002},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.5,"5min":1.907,"15min":2.0899999999999999},"min":{"1min":0.38100000000000001,"5min":0.36099999999999999,"15min":0.35299999999999998},"max":{"1min":11.717000000000001,"5min":13.215999999999999,"15min":53.088000000000001},"last":0.47599999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:31:50 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":3.9420000000000002,"15min":1.954},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":5.548,"5min":146.803,"15min":146.803},"last":0.49199999999999999},{"interface":"front","average":{"1min":3.0470000000000002,"5min":4.7880000000000003,"15min":2.5169999999999999},"min":{"1min":0.46200000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":11.428000000000001,"5min":146.422,"15min":146.422},"last":0.56000000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019388,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1097452,"kb_used_data":327780,"kb_used_omap":0,"kb_used_meta":769664,"kb_avail":92647700,"statfs":{"total":95995035648,"available":94871244800,"internally_reserved":0,"allocated":335646720,"data_stored":334192764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":788135936},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:32:03 2023","interfaces":[{"interface":"back","average":{"1min":5.4489999999999998,"5min":3.0939999999999999,"15min":4.7430000000000003},"min":{"1min":0.26200000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":63.143999999999998,"5min":67.707999999999998,"15min":199.64400000000001},"last":0.434},{"interface":"front","average":{"1min":5.3029999999999999,"5min":3.2320000000000002,"15min":4.9429999999999996},"min":{"1min":0.26700000000000002,"5min":0.24199999999999999,"15min":0.216},"max":{"1min":62.954999999999998,"5min":67.635000000000005,"15min":195.649},"last":0.32200000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:32:25 2023","interfaces":[{"interface":"back","average":{"1min":1.9510000000000001,"5min":5.7110000000000003,"15min":3.6269999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.23100000000000001},"max":{"1min":4.6139999999999999,"5min":198.27199999999999,"15min":198.27199999999999},"last":0.28199999999999997},{"interface":"front","average":{"1min":1.1160000000000001,"5min":5.508,"15min":3.7040000000000002},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.6989999999999998,"5min":196.47,"15min":196.47},"last":0.41899999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:31:42 2023","interfaces":[{"interface":"back","average":{"1min":11.99,"5min":9.5609999999999999,"15min":5.3179999999999996},"min":{"1min":0.29499999999999998,"5min":0.27100000000000002,"15min":0.26300000000000001},"max":{"1min":110.313,"5min":198.178,"15min":198.178},"last":0.47999999999999998},{"interface":"front","average":{"1min":10.316000000000001,"5min":9.0459999999999994,"15min":5.1219999999999999},"min":{"1min":0.34999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":108.077,"5min":201.56,"15min":201.56},"last":0.53300000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:31:42 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":8.3290000000000006,"15min":7.2290000000000001},"min":{"1min":0.34799999999999998,"5min":0.31900000000000001,"15min":0.26300000000000001},"max":{"1min":5.6239999999999997,"5min":277.459,"15min":303.41500000000002},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.4710000000000001,"5min":7.9359999999999999,"15min":7.1950000000000003},"min":{"1min":0.30399999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.7469999999999999,"5min":277.20800000000003,"15min":303.428},"last":0.46600000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:32:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":2.6469999999999998,"15min":2.004},"min":{"1min":0.26800000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.2750000000000004,"5min":161.86500000000001,"15min":161.86500000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":2.4390000000000001,"15min":2.0270000000000001},"min":{"1min":0.35799999999999998,"5min":0.28399999999999997,"15min":0.249},"max":{"1min":2.3809999999999998,"5min":159.24199999999999,"15min":160.61099999999999},"last":0.377}]},{"osd":6,"last update":"Sat Sep 23 23:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.78500000000000003,"15min":1.1000000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.26000000000000001},"max":{"1min":6.1299999999999999,"5min":7.1130000000000004,"15min":90.346999999999994},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.0389999999999999,"5min":0.88,"15min":1.3029999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.4429999999999996,"5min":9.8079999999999998,"15min":90.733000000000004},"last":0.40899999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:31:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.2130000000000001,"15min":1.1000000000000001},"min":{"1min":0.38100000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.9279999999999999,"5min":24.533999999999999,"15min":61.923999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.071,"15min":1.151},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":2.4129999999999998,"5min":27.771999999999998,"15min":61.990000000000002},"last":0.5}]}]},{"osd":5,"up_from":38,"seq":163208757751,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":652956,"kb_used_data":5716,"kb_used_omap":0,"kb_used_meta":647232,"kb_avail":93092196,"statfs":{"total":95995035648,"available":95326408704,"internally_reserved":0,"allocated":5853184,"data_stored":4411635,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":662765568},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:32:05 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":1.506,"15min":3.5750000000000002},"min":{"1min":0.223,"5min":0.219,"15min":0.19500000000000001},"max":{"1min":3.0609999999999999,"5min":19.952000000000002,"15min":113.869},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.244,"15min":3.8879999999999999},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.6829999999999998,"5min":5.7320000000000002,"15min":138.55199999999999},"last":0.34599999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:32:10 2023","interfaces":[{"interface":"back","average":{"1min":3.0979999999999999,"5min":1.95,"15min":2.8730000000000002},"min":{"1min":0.29699999999999999,"5min":0.25,"15min":0.23100000000000001},"max":{"1min":19.347000000000001,"5min":28.379999999999999,"15min":91.518000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":3.472,"5min":2.1040000000000001,"15min":3.1280000000000001},"min":{"1min":0.28799999999999998,"5min":0.249,"15min":0.249},"max":{"1min":19.268000000000001,"5min":28.417000000000002,"15min":87.718000000000004},"last":0.30599999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:31:43 2023","interfaces":[{"interface":"back","average":{"1min":2.948,"5min":2.7639999999999998,"15min":3.0409999999999999},"min":{"1min":0.35399999999999998,"5min":0.27200000000000002,"15min":0.26900000000000002},"max":{"1min":9.2690000000000001,"5min":46.177,"15min":144.91200000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.9370000000000001,"5min":2.718,"15min":2.8879999999999999},"min":{"1min":0.38700000000000001,"5min":0.26500000000000001,"15min":0.25800000000000001},"max":{"1min":7.0069999999999997,"5min":45.863999999999997,"15min":145.15799999999999},"last":0.49199999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:32:36 2023","interfaces":[{"interface":"back","average":{"1min":1.49,"5min":3.7629999999999999,"15min":3.7240000000000002},"min":{"1min":0.37,"5min":0.32000000000000001,"15min":0.27400000000000002},"max":{"1min":3.4809999999999999,"5min":112.71899999999999,"15min":159.53299999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.675,"5min":3.8809999999999998,"15min":3.9390000000000001},"min":{"1min":0.38900000000000001,"5min":0.32700000000000001,"15min":0.312},"max":{"1min":3.4900000000000002,"5min":112.70099999999999,"15min":159.54900000000001},"last":0.47799999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:32:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.032,"15min":0.99099999999999999},"min":{"1min":0.25700000000000001,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":6.1310000000000002,"5min":19.962,"15min":19.962},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.83999999999999997,"5min":0.89900000000000002,"15min":0.80200000000000005},"min":{"1min":0.30599999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.4080000000000004,"5min":20.829999999999998,"15min":20.829999999999998},"last":0.53100000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:32:22 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.76400000000000001,"15min":2.1739999999999999},"min":{"1min":0.33100000000000002,"5min":0.27600000000000002,"15min":0.26900000000000002},"max":{"1min":2.044,"5min":5.1299999999999999,"15min":310.68799999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.149,"5min":0.93799999999999994,"15min":2.3969999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.25900000000000001},"max":{"1min":3.3410000000000002,"5min":8.9979999999999993,"15min":311.45400000000001},"last":0.432}]},{"osd":7,"last update":"Sat Sep 23 23:31:43 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":2.9809999999999999,"15min":3.3490000000000002},"min":{"1min":0.35899999999999999,"5min":0.317,"15min":0.317},"max":{"1min":3.411,"5min":179.28800000000001,"15min":179.28800000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.0249999999999999,"5min":3.1099999999999999,"15min":3.4809999999999999},"min":{"1min":0.438,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":3.4980000000000002,"5min":179.428,"15min":179.428},"last":0.50900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:32:45.703 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T23:32:45.703 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:32:45.703 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T23:32:45.704 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T23:32:45.704 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T23:32:45.704 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T23:32:45.731 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:32:45.733 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:45 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/455169819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:32:45.733 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:45 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/455169819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:32:45.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:45.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:32:45.838 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:32:45.865 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:45.909 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:32:45.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:45.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:32:45.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:45 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/455169819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:32:46.016 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:46.043 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:46.084 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:46.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:46.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:46.191 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:32:46.218 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:46.255 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:32:46.255 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:32:46.643 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:32:46.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:46.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:32:46.686 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:46.747 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:32:46.774 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:46.815 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:32:46.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:46.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:32:46.922 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:32:46.931 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:46.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:46 smithi138 ceph-mon[193606]: pgmap v1480: 103 pgs: 103 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:46.946 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:46 smithi110 ceph-mon[212939]: pgmap v1480: 103 pgs: 103 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:46.946 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:32:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:32:46] "GET /metrics HTTP/1.1" 200 34117 "" "Prometheus/2.43.0" 2023-09-23T23:32:46.947 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:46 smithi110 ceph-mon[221102]: pgmap v1480: 103 pgs: 103 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:46.959 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:46.962 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:32:46.962 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:47.061 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:32:47.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:47.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:32:47.164 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:32:47.194 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:47.236 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:32:47.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:47.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:32:47.306 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:32:47.340 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:47.367 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:47.469 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:47.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:47.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:47.572 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:32:47.601 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:47.650 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:32:47.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:47.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:32:47.754 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:32:47.781 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:48.705 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:32:48.841 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:32:48.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:48.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:32:48.944 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:32:48.975 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:49.019 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:32:49.020 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:32:49.020 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:32:49.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:49.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:32:49.046 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:48 smithi138 ceph-mon[193606]: pgmap v1481: 103 pgs: 103 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:49.046 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:48 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:49.124 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:49.152 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:48 smithi110 ceph-mon[212939]: pgmap v1481: 103 pgs: 103 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:49.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:48 smithi110 ceph-mon[221102]: pgmap v1481: 103 pgs: 103 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:48 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:49.246 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:49.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:49.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:49.350 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:32:49.375 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:49.417 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:32:49.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:49.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:32:49.511 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:49.521 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:32:49.548 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:49.716 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:32:49.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:49.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:32:49.771 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:32:49.820 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:32:49.854 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:49.911 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:32:49.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:49.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:32:49.945 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:32:49.945 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "" 2023-09-23T23:32:49.945 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:32:50.015 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:32:50.045 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:50.111 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:32:50.923 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_8' created 2023-09-23T23:32:50.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:50 smithi138 ceph-mon[193606]: pgmap v1482: 103 pgs: 103 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:50.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:50 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1404486600' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-09-23T23:32:50.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:50 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-09-23T23:32:50.945 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:32:50.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:50.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:32:51.050 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:32:51.087 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:51.133 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:32:51.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:51.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:32:51.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:50 smithi110 ceph-mon[221102]: pgmap v1482: 103 pgs: 103 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:51.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:50 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1404486600' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-09-23T23:32:51.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:50 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-09-23T23:32:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:50 smithi110 ceph-mon[212939]: pgmap v1482: 103 pgs: 103 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:50 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1404486600' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-09-23T23:32:51.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:50 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-09-23T23:32:51.238 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:32:51.266 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:51.310 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:32:51.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:51.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:32:51.414 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:32:51.442 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:51.628 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:32:51.628 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:32:51.642 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:32:51.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:51.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:32:51.745 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:32:51.772 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:51.924 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:32:52.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:51 smithi138 ceph-mon[193606]: failed to encode map e534 with expected crc 2023-09-23T23:32:52.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:51 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-09-23T23:32:52.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:51 smithi138 ceph-mon[193606]: osdmap e534: 8 total, 8 up, 8 in 2023-09-23T23:32:52.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:51 smithi138 ceph-mon[193606]: failed to encode map e534 with expected crc 2023-09-23T23:32:52.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:51 smithi138 ceph-mon[193606]: failed to encode map e534 with expected crc 2023-09-23T23:32:52.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:51 smithi138 ceph-mon[193606]: failed to encode map e534 with expected crc 2023-09-23T23:32:52.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:51 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1210170035' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:32:52.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:51 smithi110 ceph-mon[212939]: failed to encode map e534 with expected crc 2023-09-23T23:32:52.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:51 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-09-23T23:32:52.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:51 smithi110 ceph-mon[212939]: osdmap e534: 8 total, 8 up, 8 in 2023-09-23T23:32:52.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:51 smithi110 ceph-mon[212939]: failed to encode map e534 with expected crc 2023-09-23T23:32:52.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:51 smithi110 ceph-mon[212939]: failed to encode map e534 with expected crc 2023-09-23T23:32:52.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:51 smithi110 ceph-mon[212939]: failed to encode map e534 with expected crc 2023-09-23T23:32:52.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:51 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1210170035' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:32:52.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:51 smithi110 ceph-mon[221102]: failed to encode map e534 with expected crc 2023-09-23T23:32:52.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:51 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-09-23T23:32:52.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:51 smithi110 ceph-mon[221102]: osdmap e534: 8 total, 8 up, 8 in 2023-09-23T23:32:52.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:51 smithi110 ceph-mon[221102]: failed to encode map e534 with expected crc 2023-09-23T23:32:52.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:51 smithi110 ceph-mon[221102]: failed to encode map e534 with expected crc 2023-09-23T23:32:52.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:51 smithi110 ceph-mon[221102]: failed to encode map e534 with expected crc 2023-09-23T23:32:52.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:51 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1210170035' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:32:52.456 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:32:52.668 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_8 rados --yes-i-really-mean-it || true 2023-09-23T23:32:52.688 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:32:52.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:52.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:32:52.792 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:32:52.800 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:32:52.820 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:52.867 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:32:52.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:52.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:32:52.971 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:32:53.000 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:53.047 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:32:53.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:53.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:32:53.115 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:32:53.137 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:32:53.151 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:32:53.158 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:53.158 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:32:53.179 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:52 smithi110 ceph-mon[212939]: pgmap v1484: 119 pgs: 16 unknown, 103 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:53.179 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:52 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:32:53.179 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:52 smithi110 ceph-mon[212939]: failed to encode map e535 with expected crc 2023-09-23T23:32:53.180 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:52 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1210170035' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:32:53.180 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:52 smithi110 ceph-mon[212939]: osdmap e535: 8 total, 8 up, 8 in 2023-09-23T23:32:53.180 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:52 smithi110 ceph-mon[212939]: failed to encode map e535 with expected crc 2023-09-23T23:32:53.180 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:52 smithi110 ceph-mon[212939]: failed to encode map e535 with expected crc 2023-09-23T23:32:53.180 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:52 smithi110 ceph-mon[212939]: failed to encode map e535 with expected crc 2023-09-23T23:32:53.180 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:52 smithi110 ceph-mon[221102]: pgmap v1484: 119 pgs: 16 unknown, 103 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:53.180 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:52 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:32:53.180 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:52 smithi110 ceph-mon[221102]: failed to encode map e535 with expected crc 2023-09-23T23:32:53.181 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:52 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1210170035' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:32:53.181 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:52 smithi110 ceph-mon[221102]: osdmap e535: 8 total, 8 up, 8 in 2023-09-23T23:32:53.181 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:52 smithi110 ceph-mon[221102]: failed to encode map e535 with expected crc 2023-09-23T23:32:53.181 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:52 smithi110 ceph-mon[221102]: failed to encode map e535 with expected crc 2023-09-23T23:32:53.181 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:52 smithi110 ceph-mon[221102]: failed to encode map e535 with expected crc 2023-09-23T23:32:53.181 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:32:53.183 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:52 smithi138 ceph-mon[193606]: pgmap v1484: 119 pgs: 16 unknown, 103 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:52 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:32:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:52 smithi138 ceph-mon[193606]: failed to encode map e535 with expected crc 2023-09-23T23:32:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:52 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1210170035' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:32:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:52 smithi138 ceph-mon[193606]: osdmap e535: 8 total, 8 up, 8 in 2023-09-23T23:32:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:52 smithi138 ceph-mon[193606]: failed to encode map e535 with expected crc 2023-09-23T23:32:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:52 smithi138 ceph-mon[193606]: failed to encode map e535 with expected crc 2023-09-23T23:32:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:52 smithi138 ceph-mon[193606]: failed to encode map e535 with expected crc 2023-09-23T23:32:53.206 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:32:53.227 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:32:53.233 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:32:53.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:53.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:32:53.247 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:32:53.265 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:32:53.267 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:32:53.267 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:32:53.267 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:32:53.267 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:32:53.267 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:32:53.267 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:32:53.267 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:32:53.267 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:32:53.337 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:32:53.365 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:53.511 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:32:53.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:53.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:32:53.615 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:32:53.643 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:54.088 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:32:54.090 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:53 smithi110 ceph-mon[212939]: osdmap e536: 8 total, 8 up, 8 in 2023-09-23T23:32:54.090 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:53 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2268291054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:32:54.090 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:53 smithi110 ceph-mon[212939]: failed to encode map e536 with expected crc 2023-09-23T23:32:54.090 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:53 smithi110 ceph-mon[212939]: failed to encode map e536 with expected crc 2023-09-23T23:32:54.090 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:53 smithi110 ceph-mon[221102]: osdmap e536: 8 total, 8 up, 8 in 2023-09-23T23:32:54.090 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:53 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2268291054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:32:54.090 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:53 smithi110 ceph-mon[221102]: failed to encode map e536 with expected crc 2023-09-23T23:32:54.091 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:53 smithi110 ceph-mon[221102]: failed to encode map e536 with expected crc 2023-09-23T23:32:54.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:54.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:32:54.134 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:53 smithi138 ceph-mon[193606]: osdmap e536: 8 total, 8 up, 8 in 2023-09-23T23:32:54.134 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:53 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2268291054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:32:54.134 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:53 smithi138 ceph-mon[193606]: failed to encode map e536 with expected crc 2023-09-23T23:32:54.134 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:53 smithi138 ceph-mon[193606]: failed to encode map e536 with expected crc 2023-09-23T23:32:54.192 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:32:54.221 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:54.317 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:32:54.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:54.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:32:54.375 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:32:54.375 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:32:54.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:32:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:32:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135599 11.e 11.62cc478e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:32:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.552343+0000", 2023-09-23T23:32:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 403.81940944600001, 2023-09-23T23:32:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95499033600000005, 2023-09-23T23:32:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:32:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:32:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135599 2023-09-23T23:32:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552343+0000", 2023-09-23T23:32:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552343+0000", 2023-09-23T23:32:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552342+0000", 2023-09-23T23:32:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:32:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552347+0000", 2023-09-23T23:32:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5850000000000001e-06 2023-09-23T23:32:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552358+0000", 2023-09-23T23:32:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0998999999999999e-05 2023-09-23T23:32:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552365+0000", 2023-09-23T23:32:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8630000000000002e-06 2023-09-23T23:32:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552426+0000", 2023-09-23T23:32:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0708000000000002e-05 2023-09-23T23:32:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552433+0000", 2023-09-23T23:32:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8599999999999993e-06 2023-09-23T23:32:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507318+0000", 2023-09-23T23:32:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95488429500000005 2023-09-23T23:32:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507334+0000", 2023-09-23T23:32:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5894000000000001e-05 2023-09-23T23:32:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135605 11.e 11.52f78e0e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:32:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.552409+0000", 2023-09-23T23:32:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 403.81934350699999, 2023-09-23T23:32:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95496278599999995, 2023-09-23T23:32:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:32:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:32:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135605 2023-09-23T23:32:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552409+0000", 2023-09-23T23:32:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552409+0000", 2023-09-23T23:32:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552409+0000", 2023-09-23T23:32:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:32:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552412+0000", 2023-09-23T23:32:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7199999999999998e-06 2023-09-23T23:32:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552417+0000", 2023-09-23T23:32:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.023e-06 2023-09-23T23:32:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552419+0000", 2023-09-23T23:32:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7319999999999998e-06 2023-09-23T23:32:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552464+0000", 2023-09-23T23:32:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5096000000000003e-05 2023-09-23T23:32:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552472+0000", 2023-09-23T23:32:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5390000000000002e-06 2023-09-23T23:32:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507362+0000", 2023-09-23T23:32:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95488972999999999 2023-09-23T23:32:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507372+0000", 2023-09-23T23:32:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0360000000000001e-05 2023-09-23T23:32:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135609 11.e 11.9615c3ce (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:32:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.555799+0000", 2023-09-23T23:32:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 403.81595387200002, 2023-09-23T23:32:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95161954599999998, 2023-09-23T23:32:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:32:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:32:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135609 2023-09-23T23:32:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.555799+0000", 2023-09-23T23:32:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.555799+0000", 2023-09-23T23:32:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.555798+0000", 2023-09-23T23:32:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:32:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.555801+0000", 2023-09-23T23:32:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4709999999999999e-06 2023-09-23T23:32:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.555809+0000", 2023-09-23T23:32:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6760000000000007e-06 2023-09-23T23:32:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.555814+0000", 2023-09-23T23:32:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3560000000000003e-06 2023-09-23T23:32:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.555850+0000", 2023-09-23T23:32:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5987e-05 2023-09-23T23:32:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.555858+0000", 2023-09-23T23:32:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4279999999999996e-06 2023-09-23T23:32:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507405+0000", 2023-09-23T23:32:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.951547437 2023-09-23T23:32:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507418+0000", 2023-09-23T23:32:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3052e-05 2023-09-23T23:32:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135623 11.e 11.9bd717e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:32:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.567262+0000", 2023-09-23T23:32:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 403.80449088199998, 2023-09-23T23:32:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.940189689, 2023-09-23T23:32:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:32:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:32:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135623 2023-09-23T23:32:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567262+0000", 2023-09-23T23:32:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567262+0000", 2023-09-23T23:32:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567261+0000", 2023-09-23T23:32:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:32:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567265+0000", 2023-09-23T23:32:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.69e-06 2023-09-23T23:32:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567276+0000", 2023-09-23T23:32:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1178999999999999e-05 2023-09-23T23:32:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567281+0000", 2023-09-23T23:32:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9710000000000003e-06 2023-09-23T23:32:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567329+0000", 2023-09-23T23:32:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7754999999999999e-05 2023-09-23T23:32:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567335+0000", 2023-09-23T23:32:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5119999999999999e-06 2023-09-23T23:32:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507444+0000", 2023-09-23T23:32:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94010914000000001 2023-09-23T23:32:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507452+0000", 2023-09-23T23:32:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7479999999999993e-06 2023-09-23T23:32:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135624 11.e 11.c5c7af9e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:32:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.567292+0000", 2023-09-23T23:32:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 403.80446063099998, 2023-09-23T23:32:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94020217299999997, 2023-09-23T23:32:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:32:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:32:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135624 2023-09-23T23:32:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567292+0000", 2023-09-23T23:32:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567292+0000", 2023-09-23T23:32:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567292+0000", 2023-09-23T23:32:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:32:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567294+0000", 2023-09-23T23:32:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4279999999999999e-06 2023-09-23T23:32:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567307+0000", 2023-09-23T23:32:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2911999999999999e-05 2023-09-23T23:32:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567312+0000", 2023-09-23T23:32:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.527e-06 2023-09-23T23:32:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567373+0000", 2023-09-23T23:32:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1474000000000001e-05 2023-09-23T23:32:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.567377+0000", 2023-09-23T23:32:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5650000000000002e-06 2023-09-23T23:32:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507471+0000", 2023-09-23T23:32:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94009431600000004 2023-09-23T23:32:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507494+0000", 2023-09-23T23:32:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3319000000000002e-05 2023-09-23T23:32:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135642 11.e 11.2e5be21e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:32:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.588167+0000", 2023-09-23T23:32:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 403.78358528699999, 2023-09-23T23:32:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91935951400000004, 2023-09-23T23:32:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:32:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:32:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135642 2023-09-23T23:32:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.588167+0000", 2023-09-23T23:32:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.588167+0000", 2023-09-23T23:32:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.588166+0000", 2023-09-23T23:32:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:32:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.588172+0000", 2023-09-23T23:32:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9109999999999998e-06 2023-09-23T23:32:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.588190+0000", 2023-09-23T23:32:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7731999999999999e-05 2023-09-23T23:32:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.588197+0000", 2023-09-23T23:32:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1459999999999999e-06 2023-09-23T23:32:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.588252+0000", 2023-09-23T23:32:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5120000000000001e-05 2023-09-23T23:32:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.588262+0000", 2023-09-23T23:32:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0597e-05 2023-09-23T23:32:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507517+0000", 2023-09-23T23:32:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.919254448 2023-09-23T23:32:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507527+0000", 2023-09-23T23:32:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0097e-05 2023-09-23T23:32:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135646 11.e 11.7000afde (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:32:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.590806+0000", 2023-09-23T23:32:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 403.78094697400002, 2023-09-23T23:32:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.916768315, 2023-09-23T23:32:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:32:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:32:54.421 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:54.421 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135646 2023-09-23T23:32:54.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.421 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.421 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.590806+0000", 2023-09-23T23:32:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.590806+0000", 2023-09-23T23:32:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.590805+0000", 2023-09-23T23:32:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:32:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.590809+0000", 2023-09-23T23:32:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2549999999999999e-06 2023-09-23T23:32:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.590820+0000", 2023-09-23T23:32:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0553999999999999e-05 2023-09-23T23:32:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.590824+0000", 2023-09-23T23:32:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5569999999999997e-06 2023-09-23T23:32:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.590861+0000", 2023-09-23T23:32:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6853000000000001e-05 2023-09-23T23:32:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.590866+0000", 2023-09-23T23:32:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3669999999999999e-06 2023-09-23T23:32:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507562+0000", 2023-09-23T23:32:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91669663300000004 2023-09-23T23:32:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507574+0000", 2023-09-23T23:32:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1783999999999999e-05 2023-09-23T23:32:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135676 11.e 11.5cbeade (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:32:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.688594+0000", 2023-09-23T23:32:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 403.68315888299998, 2023-09-23T23:32:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.81901241999999996, 2023-09-23T23:32:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:32:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:32:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135676 2023-09-23T23:32:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688594+0000", 2023-09-23T23:32:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688594+0000", 2023-09-23T23:32:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688593+0000", 2023-09-23T23:32:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:32:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688602+0000", 2023-09-23T23:32:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9069999999999996e-06 2023-09-23T23:32:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688617+0000", 2023-09-23T23:32:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5798e-05 2023-09-23T23:32:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688641+0000", 2023-09-23T23:32:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3781000000000002e-05 2023-09-23T23:32:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688682+0000", 2023-09-23T23:32:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1247e-05 2023-09-23T23:32:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688693+0000", 2023-09-23T23:32:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0159999999999999e-05 2023-09-23T23:32:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507599+0000", 2023-09-23T23:32:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.81890648200000005 2023-09-23T23:32:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507606+0000", 2023-09-23T23:32:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2289999999999997e-06 2023-09-23T23:32:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135677 11.e 11.7aa8673e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:32:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.688904+0000", 2023-09-23T23:32:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 403.68284895300002, 2023-09-23T23:32:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.81872850799999997, 2023-09-23T23:32:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:32:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:32:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135677 2023-09-23T23:32:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688904+0000", 2023-09-23T23:32:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688904+0000", 2023-09-23T23:32:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688903+0000", 2023-09-23T23:32:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:32:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688906+0000", 2023-09-23T23:32:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.526e-06 2023-09-23T23:32:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688912+0000", 2023-09-23T23:32:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3500000000000002e-06 2023-09-23T23:32:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688915+0000", 2023-09-23T23:32:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.202e-06 2023-09-23T23:32:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688948+0000", 2023-09-23T23:32:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2647000000000001e-05 2023-09-23T23:32:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.688954+0000", 2023-09-23T23:32:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8279999999999999e-06 2023-09-23T23:32:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507625+0000", 2023-09-23T23:32:54.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.81867117700000003 2023-09-23T23:32:54.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507632+0000", 2023-09-23T23:32:54.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1779999999999997e-06 2023-09-23T23:32:54.446 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.446 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.446 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:154629 11.17 11:ed0449d2:::benchmark_data_smithi110_433746_object48749:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-09-23T23:32:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:43.186486+0000", 2023-09-23T23:32:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 371.18526669900001, 2023-09-23T23:32:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0259201550000001, 2023-09-23T23:32:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:32:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154629 2023-09-23T23:32:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186486+0000", 2023-09-23T23:32:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186486+0000", 2023-09-23T23:32:54.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186485+0000", 2023-09-23T23:32:54.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:32:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186490+0000", 2023-09-23T23:32:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5589999999999999e-06 2023-09-23T23:32:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186498+0000", 2023-09-23T23:32:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4509999999999992e-06 2023-09-23T23:32:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186504+0000", 2023-09-23T23:32:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3369999999999996e-06 2023-09-23T23:32:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186667+0000", 2023-09-23T23:32:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016259099999999999 2023-09-23T23:32:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186678+0000", 2023-09-23T23:32:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1362999999999999e-05 2023-09-23T23:32:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.183982+0000", 2023-09-23T23:32:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99730399700000005 2023-09-23T23:32:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:32:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.184074+0000", 2023-09-23T23:32:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2168000000000003e-05 2023-09-23T23:32:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:32:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.184404+0000", 2023-09-23T23:32:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00032955400000000002 2023-09-23T23:32:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:32:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.191347+0000", 2023-09-23T23:32:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069433059999999998 2023-09-23T23:32:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212265+0000", 2023-09-23T23:32:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020918125999999999 2023-09-23T23:32:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212315+0000", 2023-09-23T23:32:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9078e-05 2023-09-23T23:32:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:32:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212362+0000", 2023-09-23T23:32:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7893999999999999e-05 2023-09-23T23:32:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212406+0000", 2023-09-23T23:32:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3766000000000001e-05 2023-09-23T23:32:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:154670 11.17 11:ef11bd65:::benchmark_data_smithi110_433746_object48790:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-09-23T23:32:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:43.271391+0000", 2023-09-23T23:32:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 371.10036157899998, 2023-09-23T23:32:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94129660699999995, 2023-09-23T23:32:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:32:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154670 2023-09-23T23:32:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.271391+0000", 2023-09-23T23:32:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.271391+0000", 2023-09-23T23:32:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.271390+0000", 2023-09-23T23:32:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:32:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.271396+0000", 2023-09-23T23:32:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2990000000000002e-06 2023-09-23T23:32:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.271407+0000", 2023-09-23T23:32:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1076e-05 2023-09-23T23:32:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.271413+0000", 2023-09-23T23:32:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.164e-06 2023-09-23T23:32:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.271456+0000", 2023-09-23T23:32:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3331999999999998e-05 2023-09-23T23:32:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.271464+0000", 2023-09-23T23:32:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3329999999999997e-06 2023-09-23T23:32:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.186310+0000", 2023-09-23T23:32:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91484549599999998 2023-09-23T23:32:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:32:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.186435+0000", 2023-09-23T23:32:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012552599999999999 2023-09-23T23:32:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:32:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.186732+0000", 2023-09-23T23:32:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00029633900000000001 2023-09-23T23:32:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:32:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.195220+0000", 2023-09-23T23:32:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0084882880000000001 2023-09-23T23:32:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212466+0000", 2023-09-23T23:32:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017246396000000001 2023-09-23T23:32:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212598+0000", 2023-09-23T23:32:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013127200000000001 2023-09-23T23:32:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:32:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212642+0000", 2023-09-23T23:32:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4345000000000003e-05 2023-09-23T23:32:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212688+0000", 2023-09-23T23:32:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5893000000000003e-05 2023-09-23T23:32:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:154683 11.17 11:eb2dae53:::benchmark_data_smithi110_433746_object48803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-09-23T23:32:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:43.299814+0000", 2023-09-23T23:32:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 371.07193914300001, 2023-09-23T23:32:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91302723399999997, 2023-09-23T23:32:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:32:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154683 2023-09-23T23:32:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.299814+0000", 2023-09-23T23:32:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.299814+0000", 2023-09-23T23:32:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.299812+0000", 2023-09-23T23:32:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:32:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.299819+0000", 2023-09-23T23:32:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2369999999999997e-06 2023-09-23T23:32:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.299832+0000", 2023-09-23T23:32:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3730999999999999e-05 2023-09-23T23:32:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.299838+0000", 2023-09-23T23:32:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1850000000000003e-06 2023-09-23T23:32:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.299882+0000", 2023-09-23T23:32:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4895000000000003e-05 2023-09-23T23:32:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.299890+0000", 2023-09-23T23:32:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0229999999999992e-06 2023-09-23T23:32:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.188239+0000", 2023-09-23T23:32:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88834824000000001 2023-09-23T23:32:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:32:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.188329+0000", 2023-09-23T23:32:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0667000000000005e-05 2023-09-23T23:32:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:32:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.188539+0000", 2023-09-23T23:32:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000210125 2023-09-23T23:32:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:32:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.195226+0000", 2023-09-23T23:32:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066862850000000001 2023-09-23T23:32:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212498+0000", 2023-09-23T23:32:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017272032999999999 2023-09-23T23:32:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212765+0000", 2023-09-23T23:32:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000267283 2023-09-23T23:32:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:32:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212808+0000", 2023-09-23T23:32:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3245999999999999e-05 2023-09-23T23:32:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212841+0000", 2023-09-23T23:32:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2526e-05 2023-09-23T23:32:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:154729 11.17 11:eda0dbdd:::benchmark_data_smithi110_433746_object48849:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-09-23T23:32:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:43.371059+0000", 2023-09-23T23:32:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 371.00069326900001, 2023-09-23T23:32:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.84193335300000005, 2023-09-23T23:32:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:32:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154729 2023-09-23T23:32:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.371059+0000", 2023-09-23T23:32:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.371059+0000", 2023-09-23T23:32:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.371058+0000", 2023-09-23T23:32:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:32:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.371067+0000", 2023-09-23T23:32:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1120000000000002e-06 2023-09-23T23:32:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.371090+0000", 2023-09-23T23:32:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2761000000000001e-05 2023-09-23T23:32:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.371098+0000", 2023-09-23T23:32:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5799999999999992e-06 2023-09-23T23:32:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.371168+0000", 2023-09-23T23:32:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9419999999999996e-05 2023-09-23T23:32:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.371181+0000", 2023-09-23T23:32:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3069e-05 2023-09-23T23:32:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.189791+0000", 2023-09-23T23:32:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.81860996200000002 2023-09-23T23:32:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:32:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.189886+0000", 2023-09-23T23:32:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4759999999999994e-05 2023-09-23T23:32:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:32:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.190063+0000", 2023-09-23T23:32:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017765500000000001 2023-09-23T23:32:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:32:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.198958+0000", 2023-09-23T23:32:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088949460000000008 2023-09-23T23:32:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212729+0000", 2023-09-23T23:32:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013770469 2023-09-23T23:32:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212906+0000", 2023-09-23T23:32:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000177858 2023-09-23T23:32:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:32:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212958+0000", 2023-09-23T23:32:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1838000000000001e-05 2023-09-23T23:32:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212993+0000", 2023-09-23T23:32:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4505999999999998e-05 2023-09-23T23:32:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:154736 11.17 11:eac942b3:::benchmark_data_smithi110_433746_object48856:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-09-23T23:32:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:43.386205+0000", 2023-09-23T23:32:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 370.98554745500002, 2023-09-23T23:32:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.82686456699999999, 2023-09-23T23:32:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:32:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154736 2023-09-23T23:32:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.386205+0000", 2023-09-23T23:32:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.386205+0000", 2023-09-23T23:32:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.386204+0000", 2023-09-23T23:32:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:32:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.386211+0000", 2023-09-23T23:32:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4570000000000002e-06 2023-09-23T23:32:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.386227+0000", 2023-09-23T23:32:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6379e-05 2023-09-23T23:32:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.386234+0000", 2023-09-23T23:32:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0219999999999999e-06 2023-09-23T23:32:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.386291+0000", 2023-09-23T23:32:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6607999999999997e-05 2023-09-23T23:32:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.386301+0000", 2023-09-23T23:32:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0390999999999999e-05 2023-09-23T23:32:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.191429+0000", 2023-09-23T23:32:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.80512771000000005 2023-09-23T23:32:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:32:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.191512+0000", 2023-09-23T23:32:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3331000000000006e-05 2023-09-23T23:32:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:32:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.191721+0000", 2023-09-23T23:32:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000208612 2023-09-23T23:32:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:32:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.198970+0000", 2023-09-23T23:32:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0072496160000000004 2023-09-23T23:32:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212877+0000", 2023-09-23T23:32:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013906814999999999 2023-09-23T23:32:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213025+0000", 2023-09-23T23:32:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014817300000000001 2023-09-23T23:32:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:32:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213048+0000", 2023-09-23T23:32:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2549e-05 2023-09-23T23:32:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213070+0000", 2023-09-23T23:32:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2075000000000001e-05 2023-09-23T23:32:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:154754 11.17 11:eabc5fa3:::benchmark_data_smithi110_433746_object48874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-09-23T23:32:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:43.412873+0000", 2023-09-23T23:32:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 370.95888010099998, 2023-09-23T23:32:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.80035215800000004, 2023-09-23T23:32:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:32:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154754 2023-09-23T23:32:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.412873+0000", 2023-09-23T23:32:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.412873+0000", 2023-09-23T23:32:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.412872+0000", 2023-09-23T23:32:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:32:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.412877+0000", 2023-09-23T23:32:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3979999999999998e-06 2023-09-23T23:32:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.412895+0000", 2023-09-23T23:32:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8400999999999999e-05 2023-09-23T23:32:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.412903+0000", 2023-09-23T23:32:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.469e-06 2023-09-23T23:32:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.412954+0000", 2023-09-23T23:32:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0924000000000002e-05 2023-09-23T23:32:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.412963+0000", 2023-09-23T23:32:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5389999999999996e-06 2023-09-23T23:32:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.193311+0000", 2023-09-23T23:32:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.78034795400000001 2023-09-23T23:32:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:32:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.193430+0000", 2023-09-23T23:32:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118532 2023-09-23T23:32:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:32:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.193713+0000", 2023-09-23T23:32:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00028307700000000001 2023-09-23T23:32:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:32:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.202618+0000", 2023-09-23T23:32:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0089055019999999992 2023-09-23T23:32:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213107+0000", 2023-09-23T23:32:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010489002000000001 2023-09-23T23:32:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213158+0000", 2023-09-23T23:32:54.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0319999999999999e-05 2023-09-23T23:32:54.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:32:54.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213195+0000", 2023-09-23T23:32:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6899e-05 2023-09-23T23:32:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213225+0000", 2023-09-23T23:32:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0266000000000001e-05 2023-09-23T23:32:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:154821 11.17 11:ec5921e4:::benchmark_data_smithi110_433746_object48941:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-09-23T23:32:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:43.505849+0000", 2023-09-23T23:32:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 370.86590412700002, 2023-09-23T23:32:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.70750497800000001, 2023-09-23T23:32:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:32:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154821 2023-09-23T23:32:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.505849+0000", 2023-09-23T23:32:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.505849+0000", 2023-09-23T23:32:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.505848+0000", 2023-09-23T23:32:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:32:54.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.512 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.512 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.505854+0000", 2023-09-23T23:32:54.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1879999999999999e-06 2023-09-23T23:32:54.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.512 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.512 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.505865+0000", 2023-09-23T23:32:54.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1515000000000001e-05 2023-09-23T23:32:54.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.512 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.513 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.505873+0000", 2023-09-23T23:32:54.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6240000000000002e-06 2023-09-23T23:32:54.513 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.513 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.513 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.505925+0000", 2023-09-23T23:32:54.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2392999999999998e-05 2023-09-23T23:32:54.513 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.513 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.513 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.505932+0000", 2023-09-23T23:32:54.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4980000000000003e-06 2023-09-23T23:32:54.514 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.514 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.514 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.514 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.195272+0000", 2023-09-23T23:32:54.514 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.68934017000000003 2023-09-23T23:32:54.514 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.514 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.514 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:32:54.515 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.195344+0000", 2023-09-23T23:32:54.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2340000000000002e-05 2023-09-23T23:32:54.515 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.515 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.515 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:32:54.515 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.195571+0000", 2023-09-23T23:32:54.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00022642100000000001 2023-09-23T23:32:54.515 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.515 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.515 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:32:54.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.202628+0000", 2023-09-23T23:32:54.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0070578070000000001 2023-09-23T23:32:54.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213256+0000", 2023-09-23T23:32:54.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010627663000000001 2023-09-23T23:32:54.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213289+0000", 2023-09-23T23:32:54.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2437000000000001e-05 2023-09-23T23:32:54.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:32:54.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213327+0000", 2023-09-23T23:32:54.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7960999999999998e-05 2023-09-23T23:32:54.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213354+0000", 2023-09-23T23:32:54.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7080000000000002e-05 2023-09-23T23:32:54.517 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.518 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.518 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.518 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.518 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.518 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:154862 11.17 11:eb194d98:::benchmark_data_smithi110_433746_object48982:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-09-23T23:32:54.518 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:43.560853+0000", 2023-09-23T23:32:54.518 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 370.81089940099997, 2023-09-23T23:32:54.518 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.65272713599999999, 2023-09-23T23:32:54.518 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.518 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:54.518 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.519 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:32:54.519 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:32:54.519 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154862 2023-09-23T23:32:54.519 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.519 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.519 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.519 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.519 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.560853+0000", 2023-09-23T23:32:54.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.519 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.519 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.519 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.519 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.560853+0000", 2023-09-23T23:32:54.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.520 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.520 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.560852+0000", 2023-09-23T23:32:54.520 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:32:54.520 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.520 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.560859+0000", 2023-09-23T23:32:54.520 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8210000000000004e-06 2023-09-23T23:32:54.520 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.520 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.521 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.560871+0000", 2023-09-23T23:32:54.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.207e-05 2023-09-23T23:32:54.521 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.521 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.521 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.521 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.560878+0000", 2023-09-23T23:32:54.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1840000000000002e-06 2023-09-23T23:32:54.521 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.521 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.521 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.521 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.560931+0000", 2023-09-23T23:32:54.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3090000000000002e-05 2023-09-23T23:32:54.521 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.522 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.522 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.522 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.560940+0000", 2023-09-23T23:32:54.522 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.38e-06 2023-09-23T23:32:54.522 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.522 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.522 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.522 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.197385+0000", 2023-09-23T23:32:54.522 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.63644482400000002 2023-09-23T23:32:54.522 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.522 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.522 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:32:54.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.197491+0000", 2023-09-23T23:32:54.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105368 2023-09-23T23:32:54.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:32:54.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.197692+0000", 2023-09-23T23:32:54.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000201447 2023-09-23T23:32:54.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:32:54.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.209644+0000", 2023-09-23T23:32:54.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011951463000000001 2023-09-23T23:32:54.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.524 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213416+0000", 2023-09-23T23:32:54.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037722430000000002 2023-09-23T23:32:54.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.524 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213490+0000", 2023-09-23T23:32:54.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4005999999999996e-05 2023-09-23T23:32:54.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:32:54.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213542+0000", 2023-09-23T23:32:54.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2327999999999998e-05 2023-09-23T23:32:54.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.213580+0000", 2023-09-23T23:32:54.526 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8333e-05 2023-09-23T23:32:54.526 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.526 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.526 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.526 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.526 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.526 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:73408 12.10 12:0ed56545:::benchmark_data_smithi110_491193_object73407:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T23:32:54.527 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:29:44.726507+0000", 2023-09-23T23:32:54.527 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 189.64524540799999, 2023-09-23T23:32:54.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95402578900000001, 2023-09-23T23:32:54.527 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.527 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:32:54.527 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.527 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:32:54.527 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:32:54.527 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 73408 2023-09-23T23:32:54.527 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.527 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.527 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.528 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726507+0000", 2023-09-23T23:32:54.528 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.528 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726507+0000", 2023-09-23T23:32:54.528 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.529 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726505+0000", 2023-09-23T23:32:54.529 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:32:54.529 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.529 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.529 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.530 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726548+0000", 2023-09-23T23:32:54.530 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2936999999999999e-05 2023-09-23T23:32:54.530 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.530 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.530 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.530 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726565+0000", 2023-09-23T23:32:54.530 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6705e-05 2023-09-23T23:32:54.530 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.530 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.530 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726572+0000", 2023-09-23T23:32:54.531 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9500000000000004e-06 2023-09-23T23:32:54.531 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.751276+0000", 2023-09-23T23:32:54.531 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.024704454000000001 2023-09-23T23:32:54.531 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.751287+0000", 2023-09-23T23:32:54.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0243e-05 2023-09-23T23:32:54.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.532 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.532 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.661352+0000", 2023-09-23T23:32:54.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91006557200000004 2023-09-23T23:32:54.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.532 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-09-23T23:32:54.532 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.661964+0000", 2023-09-23T23:32:54.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00061125199999999995 2023-09-23T23:32:54.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.680469+0000", 2023-09-23T23:32:54.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018505864 2023-09-23T23:32:54.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.680533+0000", 2023-09-23T23:32:54.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3732999999999994e-05 2023-09-23T23:32:54.534 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.534 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.534 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.534 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.534 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:73413 12.10 12:081290b7:::benchmark_data_smithi110_491193_object73412:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T23:32:54.534 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:29:44.907437+0000", 2023-09-23T23:32:54.534 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 189.46431624100001, 2023-09-23T23:32:54.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.773219714, 2023-09-23T23:32:54.535 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.535 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:54.535 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.535 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:32:54.535 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:32:54.535 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 73413 2023-09-23T23:32:54.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.535 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.907437+0000", 2023-09-23T23:32:54.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.907437+0000", 2023-09-23T23:32:54.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.907434+0000", 2023-09-23T23:32:54.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999971 2023-09-23T23:32:54.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.907500+0000", 2023-09-23T23:32:54.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5956000000000003e-05 2023-09-23T23:32:54.538 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.538 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.538 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.538 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.907540+0000", 2023-09-23T23:32:54.538 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9972000000000003e-05 2023-09-23T23:32:54.538 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.538 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.538 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.538 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.907554+0000", 2023-09-23T23:32:54.538 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.454e-05 2023-09-23T23:32:54.538 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.539 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.539 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.907652+0000", 2023-09-23T23:32:54.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7478000000000004e-05 2023-09-23T23:32:54.539 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.539 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.539 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.907667+0000", 2023-09-23T23:32:54.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.509e-05 2023-09-23T23:32:54.539 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.539 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.539 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.662209+0000", 2023-09-23T23:32:54.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.75454222999999998 2023-09-23T23:32:54.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.540 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:32:54.540 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.662307+0000", 2023-09-23T23:32:54.540 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7966000000000004e-05 2023-09-23T23:32:54.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.540 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-09-23T23:32:54.540 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.662419+0000", 2023-09-23T23:32:54.540 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112414 2023-09-23T23:32:54.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:32:54.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.663547+0000", 2023-09-23T23:32:54.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001127489 2023-09-23T23:32:54.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.664151+0000", 2023-09-23T23:32:54.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000604431 2023-09-23T23:32:54.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.680564+0000", 2023-09-23T23:32:54.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016412699999999999 2023-09-23T23:32:54.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:32:54.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.680598+0000", 2023-09-23T23:32:54.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4001999999999997e-05 2023-09-23T23:32:54.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.680656+0000", 2023-09-23T23:32:54.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8335999999999998e-05 2023-09-23T23:32:54.543 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.543 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.543 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.543 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:73422 12.0 12:0273ea89:::benchmark_data_smithi110_491193_object73421:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T23:32:54.543 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:29:44.939207+0000", 2023-09-23T23:32:54.543 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 189.43254551499999, 2023-09-23T23:32:54.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.70304230300000004, 2023-09-23T23:32:54.545 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:32:54.545 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:32:54.545 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:32:54.545 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:32:54.545 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:32:54.545 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 73422 2023-09-23T23:32:54.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.545 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:32:54.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:32:54.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.939207+0000", 2023-09-23T23:32:54.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:32:54.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.939207+0000", 2023-09-23T23:32:54.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:32:54.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.547 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:32:54.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.939206+0000", 2023-09-23T23:32:54.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:32:54.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.547 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:32:54.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.939265+0000", 2023-09-23T23:32:54.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9126000000000003e-05 2023-09-23T23:32:54.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.547 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:32:54.548 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.939274+0000", 2023-09-23T23:32:54.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0759999999999992e-06 2023-09-23T23:32:54.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.548 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:32:54.548 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.939280+0000", 2023-09-23T23:32:54.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4280000000000001e-06 2023-09-23T23:32:54.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.548 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.939318+0000", 2023-09-23T23:32:54.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8093000000000003e-05 2023-09-23T23:32:54.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.549 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:32:54.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.939326+0000", 2023-09-23T23:32:54.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3889999999999999e-06 2023-09-23T23:32:54.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.549 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:32:54.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.634664+0000", 2023-09-23T23:32:54.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.69533816500000001 2023-09-23T23:32:54.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:32:54.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.634772+0000", 2023-09-23T23:32:54.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108284 2023-09-23T23:32:54.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-09-23T23:32:54.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.634874+0000", 2023-09-23T23:32:54.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000102043 2023-09-23T23:32:54.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.636816+0000", 2023-09-23T23:32:54.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001941938 2023-09-23T23:32:54.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:32:54.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.638744+0000", 2023-09-23T23:32:54.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001928213 2023-09-23T23:32:54.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:32:54.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.642166+0000", 2023-09-23T23:32:54.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003421267 2023-09-23T23:32:54.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:32:54.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.642212+0000", 2023-09-23T23:32:54.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6459999999999999e-05 2023-09-23T23:32:54.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:32:54.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:32:54.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:32:54.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.642250+0000", 2023-09-23T23:32:54.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7496999999999997e-05 2023-09-23T23:32:54.553 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.553 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.553 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.553 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:32:54.553 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:32:54.553 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:32:54.554 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:32:54.581 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:54.626 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:32:54.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:54.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:32:54.729 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:54.757 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:54.774 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:32:54.798 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:54.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:54.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:54.902 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:32:54.929 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:54.998 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:54 smithi110 ceph-mon[212939]: failed to encode map e536 with expected crc 2023-09-23T23:32:54.998 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:54 smithi110 ceph-mon[212939]: pgmap v1487: 119 pgs: 13 unknown, 106 active+clean; 582 KiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:54.999 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:54 smithi110 ceph-mon[221102]: failed to encode map e536 with expected crc 2023-09-23T23:32:54.999 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:54 smithi110 ceph-mon[221102]: pgmap v1487: 119 pgs: 13 unknown, 106 active+clean; 582 KiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:55.043 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:32:55.046 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:54 smithi138 ceph-mon[193606]: failed to encode map e536 with expected crc 2023-09-23T23:32:55.046 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:54 smithi138 ceph-mon[193606]: pgmap v1487: 119 pgs: 13 unknown, 106 active+clean; 582 KiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-09-23T23:32:55.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:55.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:32:55.104 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:32:55.148 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:55.175 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:55.188 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:55.218 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:55.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:55.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:55.326 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:32:55.354 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:55.400 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:32:55.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:55.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:32:55.455 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:32:55.504 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:32:55.530 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:55.603 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:32:55.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:55.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:32:55.706 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:32:55.733 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:55.737 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 7, 2, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:32:55.737 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:32:55.737 INFO:tasks.thrashosds.thrasher:Reweighting osd 0 to 0.48909520664291783 2023-09-23T23:32:55.737 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 0.48909520664291783 2023-09-23T23:32:55.777 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:32:55.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:55.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:32:55.882 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:55.911 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:55.933 INFO:teuthology.orchestra.run.smithi110.stderr:enabled application 'rados' on pool 'unique_pool_8' 2023-09-23T23:32:55.957 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:55.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:55.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:56.061 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:32:56.085 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:55 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/716051018' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:32:56.087 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:56.130 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:32:56.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:56.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:32:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:55 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/716051018' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:32:56.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:55 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/716051018' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:32:56.203 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:32:56.233 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:32:56.274 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:56.670 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:32:56.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:56.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:32:56.775 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:32:56.805 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:56.841 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:56.906 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:32:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:32:56] "GET /metrics HTTP/1.1" 200 34117 "" "Prometheus/2.43.0" 2023-09-23T23:32:56.953 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:32:56.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:56.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:32:56.993 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:32:56.993 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:32:56.993 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-09-23T23:32:56.993 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:32:57.018 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:56 smithi138 ceph-mon[193606]: pgmap v1488: 119 pgs: 119 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:32:57.018 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:56 smithi138 ceph-mon[193606]: failed to encode map e537 with expected crc 2023-09-23T23:32:57.018 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:56 smithi138 ceph-mon[193606]: failed to encode map e537 with expected crc 2023-09-23T23:32:57.018 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:56 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/716051018' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:32:57.018 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:56 smithi138 ceph-mon[193606]: osdmap e537: 8 total, 8 up, 8 in 2023-09-23T23:32:57.057 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:32:57.069 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:32:57.086 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:57.158 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:56 smithi110 ceph-mon[221102]: pgmap v1488: 119 pgs: 119 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:32:57.158 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:56 smithi110 ceph-mon[221102]: failed to encode map e537 with expected crc 2023-09-23T23:32:57.158 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:56 smithi110 ceph-mon[221102]: failed to encode map e537 with expected crc 2023-09-23T23:32:57.158 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:56 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/716051018' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:32:57.158 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:56 smithi110 ceph-mon[221102]: osdmap e537: 8 total, 8 up, 8 in 2023-09-23T23:32:57.158 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:56 smithi110 ceph-mon[212939]: pgmap v1488: 119 pgs: 119 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:32:57.158 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:56 smithi110 ceph-mon[212939]: failed to encode map e537 with expected crc 2023-09-23T23:32:57.159 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:56 smithi110 ceph-mon[212939]: failed to encode map e537 with expected crc 2023-09-23T23:32:57.159 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:56 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/716051018' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:32:57.159 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:56 smithi110 ceph-mon[212939]: osdmap e537: 8 total, 8 up, 8 in 2023-09-23T23:32:57.230 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:32:57.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:57.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:32:57.334 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:32:57.360 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:57.403 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:32:57.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:57.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:32:57.507 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:32:57.537 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:57.771 DEBUG:teuthology.orchestra.run.smithi110:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_8 bench 90 write' 2023-09-23T23:32:57.774 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T23:32:57.774 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T23:32:57.781 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:32:57.836 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:hints = 1 2023-09-23T23:32:57.836 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T23:32:57.836 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object prefix: benchmark_data_smithi110_541928 2023-09-23T23:32:57.837 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:32:57.837 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 0 0 0 0 0 0 - 0 2023-09-23T23:32:57.946 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:32:58.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:57 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3217009486' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:32:58.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:57 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3217009486' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:32:58.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:57 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3217009486' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:32:58.255 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:32:58.836 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 1 16 2515 2499 156.171 156.188 0.00145935 0.00577478 2023-09-23T23:32:58.948 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:32:58.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:58.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:32:59.052 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:32:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:58 smithi138 ceph-mon[193606]: pgmap v1490: 119 pgs: 119 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:32:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:58 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:32:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:58 smithi138 ceph-mon[193606]: failed to encode map e538 with expected crc 2023-09-23T23:32:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:58 smithi138 ceph-mon[193606]: failed to encode map e538 with expected crc 2023-09-23T23:32:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:58 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3217009486' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:32:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:58 smithi138 ceph-mon[193606]: osdmap e538: 8 total, 8 up, 8 in 2023-09-23T23:32:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:32:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:32:59.111 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:59.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:58 smithi110 ceph-mon[221102]: pgmap v1490: 119 pgs: 119 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:32:59.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:58 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:32:59.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:58 smithi110 ceph-mon[221102]: failed to encode map e538 with expected crc 2023-09-23T23:32:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:58 smithi110 ceph-mon[221102]: failed to encode map e538 with expected crc 2023-09-23T23:32:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:58 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3217009486' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:32:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:58 smithi110 ceph-mon[221102]: osdmap e538: 8 total, 8 up, 8 in 2023-09-23T23:32:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:59.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:32:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:32:59.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:58 smithi110 ceph-mon[212939]: pgmap v1490: 119 pgs: 119 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:32:59.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:58 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:32:59.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:58 smithi110 ceph-mon[212939]: failed to encode map e538 with expected crc 2023-09-23T23:32:59.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:58 smithi110 ceph-mon[212939]: failed to encode map e538 with expected crc 2023-09-23T23:32:59.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:58 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3217009486' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:32:59.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:58 smithi110 ceph-mon[212939]: osdmap e538: 8 total, 8 up, 8 in 2023-09-23T23:32:59.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:32:59.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:32:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:32:59.204 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:32:59.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:59.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:32:59.308 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:32:59.368 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:59.470 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:32:59.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:59.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:32:59.575 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:32:59.639 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:32:59.782 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:32:59.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:32:59.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:32:59.836 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 2 16 5715 5699 178.079 200 0.00710863 0.00560525 2023-09-23T23:32:59.888 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:32:59.939 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:00.028 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:33:00.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:00.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:33:00.134 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:00.185 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:33:00.186 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:00.222 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:33:00.265 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:33:00.301 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:00.302 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:33:00.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:00.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:00.341 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:33:00.375 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:33:00.408 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:00.411 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:33:00.451 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:33:00.456 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:00.457 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:33:00.457 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:33:00.457 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:33:00.457 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:33:00.457 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:33:00.458 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:33:00.458 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:33:00.458 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:33:00.646 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:00.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:00.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:00.752 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:00.801 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:00.836 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 3 16 8806 8790 183.11 193.188 0.00403044 0.00545526 2023-09-23T23:33:00.876 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:00.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:00.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:00.959 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.0 to 0.489095 (7d35) 2023-09-23T23:33:00.982 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:01.028 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:01.069 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:01.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:00 smithi110 ceph-mon[212939]: pgmap v1492: 119 pgs: 119 active+clean; 167 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 24 MiB/s wr, 384 op/s 2023-09-23T23:33:01.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:00 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4191148933' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:01.071 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:00 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3379862735' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.48909520664291783}]: dispatch 2023-09-23T23:33:01.072 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:00 smithi110 ceph-mon[221102]: pgmap v1492: 119 pgs: 119 active+clean; 167 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 24 MiB/s wr, 384 op/s 2023-09-23T23:33:01.072 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:00 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4191148933' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:01.072 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:00 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3379862735' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.48909520664291783}]: dispatch 2023-09-23T23:33:01.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:01.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:01.173 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:01.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:00 smithi138 ceph-mon[193606]: pgmap v1492: 119 pgs: 119 active+clean; 167 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 24 MiB/s wr, 384 op/s 2023-09-23T23:33:01.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:00 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4191148933' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:01.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:00 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3379862735' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.48909520664291783}]: dispatch 2023-09-23T23:33:01.199 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:01.241 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:01.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:01.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:01.344 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:01.370 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:01.464 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:33:01.464 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:33:01.468 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:33:01.837 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 4 16 9208 9192 143.613 25.125 0.00181033 0.00547345 2023-09-23T23:33:02.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:01 smithi138 ceph-mon[193606]: failed to encode map e539 with expected crc 2023-09-23T23:33:02.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:01 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3379862735' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.48909520664291783}]': finished 2023-09-23T23:33:02.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:01 smithi138 ceph-mon[193606]: osdmap e539: 8 total, 8 up, 8 in 2023-09-23T23:33:02.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:01 smithi138 ceph-mon[193606]: failed to encode map e539 with expected crc 2023-09-23T23:33:02.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:01 smithi138 ceph-mon[193606]: failed to encode map e539 with expected crc 2023-09-23T23:33:02.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:01 smithi138 ceph-mon[193606]: failed to encode map e539 with expected crc 2023-09-23T23:33:02.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:01 smithi138 ceph-mon[193606]: osdmap e540: 8 total, 8 up, 8 in 2023-09-23T23:33:02.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:01 smithi110 ceph-mon[221102]: failed to encode map e539 with expected crc 2023-09-23T23:33:02.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:01 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3379862735' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.48909520664291783}]': finished 2023-09-23T23:33:02.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:01 smithi110 ceph-mon[221102]: osdmap e539: 8 total, 8 up, 8 in 2023-09-23T23:33:02.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:01 smithi110 ceph-mon[221102]: failed to encode map e539 with expected crc 2023-09-23T23:33:02.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:01 smithi110 ceph-mon[221102]: failed to encode map e539 with expected crc 2023-09-23T23:33:02.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:01 smithi110 ceph-mon[221102]: failed to encode map e539 with expected crc 2023-09-23T23:33:02.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:01 smithi110 ceph-mon[221102]: osdmap e540: 8 total, 8 up, 8 in 2023-09-23T23:33:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:01 smithi110 ceph-mon[212939]: failed to encode map e539 with expected crc 2023-09-23T23:33:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:01 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3379862735' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.48909520664291783}]': finished 2023-09-23T23:33:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:01 smithi110 ceph-mon[212939]: osdmap e539: 8 total, 8 up, 8 in 2023-09-23T23:33:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:01 smithi110 ceph-mon[212939]: failed to encode map e539 with expected crc 2023-09-23T23:33:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:01 smithi110 ceph-mon[212939]: failed to encode map e539 with expected crc 2023-09-23T23:33:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:01 smithi110 ceph-mon[212939]: failed to encode map e539 with expected crc 2023-09-23T23:33:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:01 smithi110 ceph-mon[212939]: osdmap e540: 8 total, 8 up, 8 in 2023-09-23T23:33:02.519 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:02.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:02.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:02.623 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:02.650 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:02.698 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:02.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:02.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:02.728 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:02.800 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:33:02.803 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:02.830 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:02.836 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 5 16 9208 9192 114.89 0 - 0.00547345 2023-09-23T23:33:02.871 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:02.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:02.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:02.975 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:03.046 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:02 smithi138 ceph-mon[193606]: pgmap v1494: 119 pgs: 119 active+clean; 197 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s 2023-09-23T23:33:03.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:02 smithi138 ceph-mon[193606]: failed to encode map e540 with expected crc 2023-09-23T23:33:03.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:02 smithi138 ceph-mon[193606]: failed to encode map e540 with expected crc 2023-09-23T23:33:03.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:02 smithi138 ceph-mon[193606]: failed to encode map e540 with expected crc 2023-09-23T23:33:03.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:02 smithi138 ceph-mon[193606]: failed to encode map e540 with expected crc 2023-09-23T23:33:03.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:02 smithi138 ceph-mon[193606]: failed to encode map e540 with expected crc 2023-09-23T23:33:03.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:02 smithi138 ceph-mon[193606]: osdmap e541: 8 total, 8 up, 8 in 2023-09-23T23:33:03.057 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:03.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:02 smithi110 ceph-mon[212939]: pgmap v1494: 119 pgs: 119 active+clean; 197 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s 2023-09-23T23:33:03.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:02 smithi110 ceph-mon[212939]: failed to encode map e540 with expected crc 2023-09-23T23:33:03.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:02 smithi110 ceph-mon[212939]: failed to encode map e540 with expected crc 2023-09-23T23:33:03.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:02 smithi110 ceph-mon[212939]: failed to encode map e540 with expected crc 2023-09-23T23:33:03.128 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:02 smithi110 ceph-mon[212939]: failed to encode map e540 with expected crc 2023-09-23T23:33:03.128 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:02 smithi110 ceph-mon[212939]: failed to encode map e540 with expected crc 2023-09-23T23:33:03.128 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:02 smithi110 ceph-mon[212939]: osdmap e541: 8 total, 8 up, 8 in 2023-09-23T23:33:03.128 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:02 smithi110 ceph-mon[221102]: pgmap v1494: 119 pgs: 119 active+clean; 197 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s 2023-09-23T23:33:03.128 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:02 smithi110 ceph-mon[221102]: failed to encode map e540 with expected crc 2023-09-23T23:33:03.129 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:02 smithi110 ceph-mon[221102]: failed to encode map e540 with expected crc 2023-09-23T23:33:03.129 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:02 smithi110 ceph-mon[221102]: failed to encode map e540 with expected crc 2023-09-23T23:33:03.129 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:02 smithi110 ceph-mon[221102]: failed to encode map e540 with expected crc 2023-09-23T23:33:03.129 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:02 smithi110 ceph-mon[221102]: failed to encode map e540 with expected crc 2023-09-23T23:33:03.129 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:02 smithi110 ceph-mon[221102]: osdmap e541: 8 total, 8 up, 8 in 2023-09-23T23:33:03.161 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:03.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:03.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:03.279 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:33:03.341 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:03.402 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:33:03.417 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:33:03.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:03.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:33:03.523 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:03.551 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:03.665 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:03.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:03.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:03.773 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:03.829 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:03.836 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 6 16 10630 10614 110.553 44.4375 0.00994959 0.00903584 2023-09-23T23:33:03.980 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:03.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:03.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:04.087 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:33:04.146 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:04.201 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:03 smithi138 ceph-mon[193606]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.201 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:03 smithi138 ceph-mon[193606]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.201 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:03 smithi138 ceph-mon[193606]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.201 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:03 smithi138 ceph-mon[193606]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.201 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:03 smithi138 ceph-mon[193606]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.201 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:03 smithi138 ceph-mon[193606]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.201 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:03 smithi138 ceph-mon[193606]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.201 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:03 smithi138 ceph-mon[193606]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.201 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:03 smithi138 ceph-mon[193606]: osdmap e542: 8 total, 8 up, 8 in 2023-09-23T23:33:04.201 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:03 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:04.236 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:33:04.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:04.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:33:04.334 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[212939]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.334 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[212939]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.334 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[212939]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.335 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[212939]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.335 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[212939]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.335 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[212939]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.335 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[212939]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.335 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[212939]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.335 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[212939]: osdmap e542: 8 total, 8 up, 8 in 2023-09-23T23:33:04.335 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:04.335 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[221102]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.336 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[221102]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.336 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[221102]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.336 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[221102]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.336 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[221102]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.336 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[221102]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.336 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[221102]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.336 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[221102]: failed to encode map e541 with expected crc 2023-09-23T23:33:04.336 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[221102]: osdmap e542: 8 total, 8 up, 8 in 2023-09-23T23:33:04.336 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:03 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:04.346 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:04.399 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:04.482 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:04.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:04.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:04.588 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:04.602 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:33:04.632 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:04.729 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:04.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:04.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:04.836 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:04.851 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 7 15 13141 13126 117.08 157 0.0080417 0.00853397 2023-09-23T23:33:04.898 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:04.989 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:04.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:04.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:05.093 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:33:05.150 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:05.200 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:05.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[212939]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[212939]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[212939]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[212939]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[212939]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[212939]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[212939]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[212939]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[212939]: pgmap v1498: 119 pgs: 1 active+undersized+remapped, 118 active+clean; 299 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 33 MiB/s wr, 526 op/s; 5 B/s, 0 objects/s recovering 2023-09-23T23:33:05.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[212939]: osdmap e543: 8 total, 8 up, 8 in 2023-09-23T23:33:05.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[221102]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[221102]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[221102]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[221102]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[221102]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[221102]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[221102]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[221102]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[221102]: pgmap v1498: 119 pgs: 1 active+undersized+remapped, 118 active+clean; 299 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 33 MiB/s wr, 526 op/s; 5 B/s, 0 objects/s recovering 2023-09-23T23:33:05.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:04 smithi110 ceph-mon[221102]: osdmap e543: 8 total, 8 up, 8 in 2023-09-23T23:33:05.241 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:33:05.244 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:04 smithi138 ceph-mon[193606]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.245 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:04 smithi138 ceph-mon[193606]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.245 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:04 smithi138 ceph-mon[193606]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.245 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:04 smithi138 ceph-mon[193606]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.245 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:04 smithi138 ceph-mon[193606]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.245 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:04 smithi138 ceph-mon[193606]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.245 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:04 smithi138 ceph-mon[193606]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.245 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:04 smithi138 ceph-mon[193606]: failed to encode map e542 with expected crc 2023-09-23T23:33:05.245 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:04 smithi138 ceph-mon[193606]: pgmap v1498: 119 pgs: 1 active+undersized+remapped, 118 active+clean; 299 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 33 MiB/s wr, 526 op/s; 5 B/s, 0 objects/s recovering 2023-09-23T23:33:05.245 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:04 smithi138 ceph-mon[193606]: osdmap e543: 8 total, 8 up, 8 in 2023-09-23T23:33:05.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:05.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:33:05.347 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:33:05.383 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:05.469 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:33:05.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:05.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:33:05.574 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:05.611 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:05.842 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:05.843 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 8 16 15591 15575 121.571 153.062 0.00971188 0.00821963 2023-09-23T23:33:05.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:05.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:05.947 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:33:05.993 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:33:06.015 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:06.105 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:33:06.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:06.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:33:06.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:05 smithi138 ceph-mon[193606]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:05 smithi138 ceph-mon[193606]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:05 smithi138 ceph-mon[193606]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:05 smithi138 ceph-mon[193606]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:05 smithi138 ceph-mon[193606]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:05 smithi138 ceph-mon[193606]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:05 smithi138 ceph-mon[193606]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:05 smithi138 ceph-mon[193606]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:05 smithi138 ceph-mon[193606]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:05 smithi138 ceph-mon[193606]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:05 smithi138 ceph-mon[193606]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:05 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2455301864' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:33:06.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:05 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:33:06.215 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:06.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[212939]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[212939]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[212939]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[212939]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[212939]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.254 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[212939]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[212939]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[212939]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[212939]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[212939]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[212939]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2455301864' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:33:06.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:33:06.255 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[221102]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.255 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[221102]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.255 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[221102]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.256 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[221102]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.256 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[221102]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.256 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[221102]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.256 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[221102]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.256 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[221102]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.256 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[221102]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.256 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[221102]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.256 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[221102]: failed to encode map e543 with expected crc 2023-09-23T23:33:06.256 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2455301864' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:33:06.256 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:05 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:33:06.259 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:06.697 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:33:06.697 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:33:06.843 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 9 16 17805 17789 123.436 138.375 0.00508467 0.00809526 2023-09-23T23:33:06.981 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:06 smithi138 ceph-mon[193606]: pgmap v1500: 119 pgs: 1 active+recovering+undersized+remapped, 5 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 111 active+clean; 704 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s; 608/33825 objects degraded (1.797%); 461/33825 objects misplaced (1.363%); 2.5 MiB/s, 42 objects/s recovering 2023-09-23T23:33:06.981 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:06 smithi138 ceph-mon[193606]: Health check failed: Degraded data redundancy: 608/33825 objects degraded (1.797%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:33:07.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:33:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:33:06] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.43.0" 2023-09-23T23:33:07.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[221102]: pgmap v1500: 119 pgs: 1 active+recovering+undersized+remapped, 5 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 111 active+clean; 704 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s; 608/33825 objects degraded (1.797%); 461/33825 objects misplaced (1.363%); 2.5 MiB/s, 42 objects/s recovering 2023-09-23T23:33:07.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[221102]: Health check failed: Degraded data redundancy: 608/33825 objects degraded (1.797%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:33:07.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:33:07.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[221102]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[221102]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[221102]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:33:07.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[221102]: osdmap e544: 8 total, 8 up, 8 in 2023-09-23T23:33:07.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[221102]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[221102]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[221102]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[221102]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[221102]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[221102]: 1.1 scrub starts 2023-09-23T23:33:07.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[212939]: pgmap v1500: 119 pgs: 1 active+recovering+undersized+remapped, 5 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 111 active+clean; 704 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s; 608/33825 objects degraded (1.797%); 461/33825 objects misplaced (1.363%); 2.5 MiB/s, 42 objects/s recovering 2023-09-23T23:33:07.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[212939]: Health check failed: Degraded data redundancy: 608/33825 objects degraded (1.797%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:33:07.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:33:07.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[212939]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[212939]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[212939]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:33:07.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[212939]: osdmap e544: 8 total, 8 up, 8 in 2023-09-23T23:33:07.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[212939]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[212939]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[212939]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[212939]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[212939]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:06 smithi110 ceph-mon[212939]: 1.1 scrub starts 2023-09-23T23:33:07.394 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:07.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:07.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:06 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:33:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:06 smithi138 ceph-mon[193606]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:06 smithi138 ceph-mon[193606]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:06 smithi138 ceph-mon[193606]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:06 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:33:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:06 smithi138 ceph-mon[193606]: osdmap e544: 8 total, 8 up, 8 in 2023-09-23T23:33:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:06 smithi138 ceph-mon[193606]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:06 smithi138 ceph-mon[193606]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:06 smithi138 ceph-mon[193606]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:06 smithi138 ceph-mon[193606]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:06 smithi138 ceph-mon[193606]: failed to encode map e544 with expected crc 2023-09-23T23:33:07.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:06 smithi138 ceph-mon[193606]: 1.1 scrub starts 2023-09-23T23:33:07.497 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:33:07.527 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:07.629 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:33:07.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:07.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:33:07.678 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 7, 2, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:33:07.678 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:33:07.678 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_8 2023-09-23T23:33:07.679 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:33:07.737 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:33:07.798 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:07.844 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 10 16 19367 19351 120.847 97.625 0.0189039 0.00826486 2023-09-23T23:33:07.879 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:33:07.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:07.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:33:07.989 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:33:08.048 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:08.131 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:33:08.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:08.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:33:08.237 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:33:08.265 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:08.307 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:33:08.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:08.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:33:08.411 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:08.435 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:08.445 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:08.497 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:08.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:08.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:08.549 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:33:08.569 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:33:08.588 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:33:08.601 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:08.609 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:33:08.629 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:33:08.630 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:08.653 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:33:08.681 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:33:08.682 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:08.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:08.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:08.708 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:33:08.709 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:33:08.709 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:33:08.709 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:33:08.709 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:33:08.709 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:33:08.709 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:33:08.709 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:33:08.709 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:33:08.787 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:33:08.816 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:08.844 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 11 16 20091 20075 113.979 45.25 0.00180841 0.00820555 2023-09-23T23:33:08.862 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:33:08.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:08.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:33:08.966 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:33:08.992 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:09.245 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:33:09.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:09.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:33:09.257 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:08 smithi138 ceph-mon[193606]: pgmap v1502: 119 pgs: 1 active+recovering+undersized+remapped, 5 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 111 active+clean; 704 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 608/33825 objects degraded (1.797%); 461/33825 objects misplaced (1.363%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T23:33:09.257 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:08 smithi138 ceph-mon[193606]: failed to encode map e545 with expected crc 2023-09-23T23:33:09.257 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:08 smithi138 ceph-mon[193606]: osdmap e545: 8 total, 8 up, 8 in 2023-09-23T23:33:09.257 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:08 smithi138 ceph-mon[193606]: failed to encode map e545 with expected crc 2023-09-23T23:33:09.257 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:08 smithi138 ceph-mon[193606]: failed to encode map e545 with expected crc 2023-09-23T23:33:09.257 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:08 smithi138 ceph-mon[193606]: failed to encode map e545 with expected crc 2023-09-23T23:33:09.257 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:08 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/8449438' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:09.258 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:33:09.358 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:09.383 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:08 smithi110 ceph-mon[212939]: pgmap v1502: 119 pgs: 1 active+recovering+undersized+remapped, 5 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 111 active+clean; 704 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 608/33825 objects degraded (1.797%); 461/33825 objects misplaced (1.363%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T23:33:09.383 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:08 smithi110 ceph-mon[212939]: failed to encode map e545 with expected crc 2023-09-23T23:33:09.383 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:08 smithi110 ceph-mon[212939]: osdmap e545: 8 total, 8 up, 8 in 2023-09-23T23:33:09.383 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:08 smithi110 ceph-mon[212939]: failed to encode map e545 with expected crc 2023-09-23T23:33:09.383 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:08 smithi110 ceph-mon[212939]: failed to encode map e545 with expected crc 2023-09-23T23:33:09.383 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:08 smithi110 ceph-mon[212939]: failed to encode map e545 with expected crc 2023-09-23T23:33:09.383 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:08 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/8449438' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:09.384 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:08 smithi110 ceph-mon[221102]: pgmap v1502: 119 pgs: 1 active+recovering+undersized+remapped, 5 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 111 active+clean; 704 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 608/33825 objects degraded (1.797%); 461/33825 objects misplaced (1.363%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T23:33:09.384 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:08 smithi110 ceph-mon[221102]: failed to encode map e545 with expected crc 2023-09-23T23:33:09.384 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:08 smithi110 ceph-mon[221102]: osdmap e545: 8 total, 8 up, 8 in 2023-09-23T23:33:09.384 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:08 smithi110 ceph-mon[221102]: failed to encode map e545 with expected crc 2023-09-23T23:33:09.384 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:08 smithi110 ceph-mon[221102]: failed to encode map e545 with expected crc 2023-09-23T23:33:09.384 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:08 smithi110 ceph-mon[221102]: failed to encode map e545 with expected crc 2023-09-23T23:33:09.384 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:08 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/8449438' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:09.385 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:09.844 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 12 16 21385 21369 111.221 80.875 0.00482523 0.00898514 2023-09-23T23:33:09.911 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:33:10.061 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:33:10.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:10 smithi110 ceph-mon[212939]: failed to encode map e546 with expected crc 2023-09-23T23:33:10.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:10 smithi110 ceph-mon[212939]: failed to encode map e546 with expected crc 2023-09-23T23:33:10.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:10 smithi110 ceph-mon[212939]: osdmap e546: 8 total, 8 up, 8 in 2023-09-23T23:33:10.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:10 smithi110 ceph-mon[212939]: failed to encode map e546 with expected crc 2023-09-23T23:33:10.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:10 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:33:10.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:10 smithi110 ceph-mon[212939]: failed to encode map e546 with expected crc 2023-09-23T23:33:10.093 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:10 smithi110 ceph-mon[212939]: 1.1 scrub starts 2023-09-23T23:33:10.093 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:10 smithi110 ceph-mon[212939]: 1.1 scrub ok 2023-09-23T23:33:10.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:10 smithi110 ceph-mon[221102]: failed to encode map e546 with expected crc 2023-09-23T23:33:10.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:10 smithi110 ceph-mon[221102]: failed to encode map e546 with expected crc 2023-09-23T23:33:10.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:10 smithi110 ceph-mon[221102]: osdmap e546: 8 total, 8 up, 8 in 2023-09-23T23:33:10.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:10 smithi110 ceph-mon[221102]: failed to encode map e546 with expected crc 2023-09-23T23:33:10.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:10 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:33:10.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:10 smithi110 ceph-mon[221102]: failed to encode map e546 with expected crc 2023-09-23T23:33:10.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:10 smithi110 ceph-mon[221102]: 1.1 scrub starts 2023-09-23T23:33:10.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:10 smithi110 ceph-mon[221102]: 1.1 scrub ok 2023-09-23T23:33:10.272 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:10 smithi138 ceph-mon[193606]: failed to encode map e546 with expected crc 2023-09-23T23:33:10.272 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:10 smithi138 ceph-mon[193606]: failed to encode map e546 with expected crc 2023-09-23T23:33:10.272 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:10 smithi138 ceph-mon[193606]: osdmap e546: 8 total, 8 up, 8 in 2023-09-23T23:33:10.272 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:10 smithi138 ceph-mon[193606]: failed to encode map e546 with expected crc 2023-09-23T23:33:10.272 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:10 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:33:10.273 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:10 smithi138 ceph-mon[193606]: failed to encode map e546 with expected crc 2023-09-23T23:33:10.273 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:10 smithi138 ceph-mon[193606]: 1.1 scrub starts 2023-09-23T23:33:10.273 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:10 smithi138 ceph-mon[193606]: 1.1 scrub ok 2023-09-23T23:33:10.384 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:10.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:10.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:10.489 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:33:10.518 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:10.563 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:33:10.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:10.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:33:10.615 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:10.667 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:10.706 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:10.805 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:10.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:10.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:10.844 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 13 16 21693 21677 104.151 19.25 0.0016771 0.00905191 2023-09-23T23:33:10.909 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:10.935 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:11.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:11 smithi138 ceph-mon[193606]: pgmap v1505: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 117 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s; 1/59862 objects degraded (0.002%); 134/59862 objects misplaced (0.224%); 11 MiB/s, 180 objects/s recovering 2023-09-23T23:33:11.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:11 smithi138 ceph-mon[193606]: failed to encode map e547 with expected crc 2023-09-23T23:33:11.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:11 smithi138 ceph-mon[193606]: failed to encode map e547 with expected crc 2023-09-23T23:33:11.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:11 smithi138 ceph-mon[193606]: osdmap e547: 8 total, 8 up, 8 in 2023-09-23T23:33:11.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:11 smithi138 ceph-mon[193606]: failed to encode map e547 with expected crc 2023-09-23T23:33:11.274 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:11 smithi110 ceph-mon[212939]: pgmap v1505: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 117 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s; 1/59862 objects degraded (0.002%); 134/59862 objects misplaced (0.224%); 11 MiB/s, 180 objects/s recovering 2023-09-23T23:33:11.274 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:11 smithi110 ceph-mon[212939]: failed to encode map e547 with expected crc 2023-09-23T23:33:11.274 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:11 smithi110 ceph-mon[212939]: failed to encode map e547 with expected crc 2023-09-23T23:33:11.274 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:11 smithi110 ceph-mon[212939]: osdmap e547: 8 total, 8 up, 8 in 2023-09-23T23:33:11.274 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:11 smithi110 ceph-mon[212939]: failed to encode map e547 with expected crc 2023-09-23T23:33:11.274 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:11 smithi110 ceph-mon[221102]: pgmap v1505: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 117 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s; 1/59862 objects degraded (0.002%); 134/59862 objects misplaced (0.224%); 11 MiB/s, 180 objects/s recovering 2023-09-23T23:33:11.275 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:11 smithi110 ceph-mon[221102]: failed to encode map e547 with expected crc 2023-09-23T23:33:11.275 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:11 smithi110 ceph-mon[221102]: failed to encode map e547 with expected crc 2023-09-23T23:33:11.275 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:11 smithi110 ceph-mon[221102]: osdmap e547: 8 total, 8 up, 8 in 2023-09-23T23:33:11.275 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:11 smithi110 ceph-mon[221102]: failed to encode map e547 with expected crc 2023-09-23T23:33:11.284 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:33:11.287 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:33:11.487 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:11.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:11.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:11.595 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:33:11.683 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:11.800 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:33:11.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:11.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:33:11.844 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 14 16 23026 23010 102.662 83.3125 0.00664453 0.00973492 2023-09-23T23:33:11.905 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:33:11.972 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:12.082 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:12 smithi138 ceph-mon[193606]: failed to encode map e548 with expected crc 2023-09-23T23:33:12.082 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:12 smithi138 ceph-mon[193606]: osdmap e548: 8 total, 8 up, 8 in 2023-09-23T23:33:12.083 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:12 smithi138 ceph-mon[193606]: failed to encode map e548 with expected crc 2023-09-23T23:33:12.083 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:12 smithi138 ceph-mon[193606]: failed to encode map e548 with expected crc 2023-09-23T23:33:12.083 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:12 smithi138 ceph-mon[193606]: failed to encode map e548 with expected crc 2023-09-23T23:33:12.083 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:12 smithi138 ceph-mon[193606]: from='client.36480 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:33:12.225 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:33:12.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:12.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:33:12.268 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:33:12.268 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:33:12.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.268 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:23587 13.4 13:26f3f0ba:::benchmark_data_smithi110_541928_object23586:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e548)", 2023-09-23T23:33:12.268 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:12.143824+0000", 2023-09-23T23:33:12.268 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.122933628, 2023-09-23T23:33:12.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.122961467, 2023-09-23T23:33:12.269 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:33:12.269 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:33:12.269 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:33:12.269 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:33:12.269 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:33:12.269 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 23587 2023-09-23T23:33:12.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.269 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:33:12.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:33:12.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.143824+0000", 2023-09-23T23:33:12.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:33:12.270 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.270 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:33:12.270 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.143824+0000", 2023-09-23T23:33:12.270 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:33:12.270 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.270 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:33:12.270 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.143823+0000", 2023-09-23T23:33:12.270 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:33:12.270 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.271 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.271 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:33:12.271 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.143869+0000", 2023-09-23T23:33:12.271 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6477999999999998e-05 2023-09-23T23:33:12.271 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.271 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.271 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:33:12.271 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.143884+0000", 2023-09-23T23:33:12.271 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.519e-05 2023-09-23T23:33:12.271 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.271 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.271 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:33:12.271 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.143892+0000", 2023-09-23T23:33:12.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.34e-06 2023-09-23T23:33:12.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.272 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.272 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:33:12.272 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.143953+0000", 2023-09-23T23:33:12.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1005000000000003e-05 2023-09-23T23:33:12.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.272 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.272 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:33:12.272 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.144107+0000", 2023-09-23T23:33:12.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015481199999999999 2023-09-23T23:33:12.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.273 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:33:12.273 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.144279+0000", 2023-09-23T23:33:12.273 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017117099999999999 2023-09-23T23:33:12.273 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.273 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:33:12.273 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.145407+0000", 2023-09-23T23:33:12.273 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001127945 2023-09-23T23:33:12.273 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.273 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:12.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.145522+0000", 2023-09-23T23:33:12.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000115927 2023-09-23T23:33:12.274 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:33:12.274 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:33:12.274 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:33:12.274 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.274 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.274 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:23560 13.4 13:2fad76e2:::benchmark_data_smithi110_541928_object23559:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e548)", 2023-09-23T23:33:12.274 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:12.122469+0000", 2023-09-23T23:33:12.274 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.144289109, 2023-09-23T23:33:12.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.14437566800000001, 2023-09-23T23:33:12.274 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:33:12.275 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:33:12.275 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:33:12.275 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:33:12.275 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:33:12.275 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 23560 2023-09-23T23:33:12.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.275 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:33:12.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.275 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:33:12.275 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.122469+0000", 2023-09-23T23:33:12.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:33:12.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:33:12.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.122469+0000", 2023-09-23T23:33:12.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:33:12.276 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:33:12.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.122459+0000", 2023-09-23T23:33:12.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999905 2023-09-23T23:33:12.276 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:33:12.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.122581+0000", 2023-09-23T23:33:12.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012170799999999999 2023-09-23T23:33:12.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.277 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.277 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:33:12.277 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.122595+0000", 2023-09-23T23:33:12.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3628000000000001e-05 2023-09-23T23:33:12.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.277 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.277 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:33:12.277 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.122601+0000", 2023-09-23T23:33:12.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1449999999999996e-06 2023-09-23T23:33:12.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.278 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:33:12.278 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.123357+0000", 2023-09-23T23:33:12.278 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00075660899999999999 2023-09-23T23:33:12.278 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.278 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:33:12.278 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.123506+0000", 2023-09-23T23:33:12.278 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014906899999999999 2023-09-23T23:33:12.278 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.278 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:33:12.278 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.123670+0000", 2023-09-23T23:33:12.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016375499999999999 2023-09-23T23:33:12.279 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:12.279 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.124932+0000", 2023-09-23T23:33:12.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001261561 2023-09-23T23:33:12.279 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:33:12.279 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.128166+0000", 2023-09-23T23:33:12.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003234439 2023-09-23T23:33:12.279 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:33:12.280 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:33:12.280 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:33:12.280 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.280 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.280 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:23573 13.4 13:2d42b517:::benchmark_data_smithi110_541928_object23572:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e548)", 2023-09-23T23:33:12.280 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:12.130570+0000", 2023-09-23T23:33:12.280 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.136187432, 2023-09-23T23:33:12.280 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.13632392300000001, 2023-09-23T23:33:12.280 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:33:12.280 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:33:12.280 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:33:12.280 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:33:12.280 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:33:12.281 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 23573 2023-09-23T23:33:12.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.281 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:33:12.281 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.281 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:33:12.281 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.130570+0000", 2023-09-23T23:33:12.281 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:33:12.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.281 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.281 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:33:12.281 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.130570+0000", 2023-09-23T23:33:12.281 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:33:12.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.282 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:33:12.282 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.130569+0000", 2023-09-23T23:33:12.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:33:12.282 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.282 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:33:12.282 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.130600+0000", 2023-09-23T23:33:12.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0809999999999998e-05 2023-09-23T23:33:12.282 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.282 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:33:12.283 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.130617+0000", 2023-09-23T23:33:12.283 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7805e-05 2023-09-23T23:33:12.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.283 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.283 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:33:12.283 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.130624+0000", 2023-09-23T23:33:12.283 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8669999999999996e-06 2023-09-23T23:33:12.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.283 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.283 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:33:12.283 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.130685+0000", 2023-09-23T23:33:12.283 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1253000000000005e-05 2023-09-23T23:33:12.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:33:12.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.130818+0000", 2023-09-23T23:33:12.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013223099999999999 2023-09-23T23:33:12.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:33:12.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.130966+0000", 2023-09-23T23:33:12.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014837800000000001 2023-09-23T23:33:12.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:33:12.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.132261+0000", 2023-09-23T23:33:12.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012948269999999999 2023-09-23T23:33:12.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:12.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.132315+0000", 2023-09-23T23:33:12.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4218999999999997e-05 2023-09-23T23:33:12.285 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:33:12.285 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:33:12.285 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:33:12.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.286 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:23542 13.4 13:224c4a9f:::benchmark_data_smithi110_541928_object23541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e548)", 2023-09-23T23:33:12.286 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:12.111766+0000", 2023-09-23T23:33:12.286 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.154991459, 2023-09-23T23:33:12.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.15517715000000001, 2023-09-23T23:33:12.286 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:33:12.286 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:33:12.286 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:33:12.286 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:33:12.286 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:33:12.286 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 23542 2023-09-23T23:33:12.286 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.286 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:33:12.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:33:12.287 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.111766+0000", 2023-09-23T23:33:12.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:33:12.287 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:33:12.287 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.111766+0000", 2023-09-23T23:33:12.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:33:12.287 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:33:12.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.111766+0000", 2023-09-23T23:33:12.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:33:12.288 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.288 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.288 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:33:12.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.111796+0000", 2023-09-23T23:33:12.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0932000000000001e-05 2023-09-23T23:33:12.288 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.288 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.288 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:33:12.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.111806+0000", 2023-09-23T23:33:12.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8460000000000003e-06 2023-09-23T23:33:12.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.289 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:33:12.289 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.111811+0000", 2023-09-23T23:33:12.289 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7489999999999998e-06 2023-09-23T23:33:12.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.289 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:33:12.289 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.113254+0000", 2023-09-23T23:33:12.289 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00144294 2023-09-23T23:33:12.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.290 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:33:12.290 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.113361+0000", 2023-09-23T23:33:12.290 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106733 2023-09-23T23:33:12.290 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.290 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.290 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:33:12.290 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.113480+0000", 2023-09-23T23:33:12.290 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118818 2023-09-23T23:33:12.290 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.290 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.290 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:33:12.290 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.117356+0000", 2023-09-23T23:33:12.290 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0038763410000000002 2023-09-23T23:33:12.291 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:33:12.291 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:33:12.291 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:12.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:12.117386+0000", 2023-09-23T23:33:12.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9641e-05 2023-09-23T23:33:12.291 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:33:12.291 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:33:12.291 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:33:12.291 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:33:12.291 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-09-23T23:33:12.291 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 4 2023-09-23T23:33:12.291 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:33:12.330 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:33:12.408 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:12.450 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:33:12.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:12 smithi110 ceph-mon[212939]: failed to encode map e548 with expected crc 2023-09-23T23:33:12.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:12 smithi110 ceph-mon[212939]: osdmap e548: 8 total, 8 up, 8 in 2023-09-23T23:33:12.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:12 smithi110 ceph-mon[212939]: failed to encode map e548 with expected crc 2023-09-23T23:33:12.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:12 smithi110 ceph-mon[212939]: failed to encode map e548 with expected crc 2023-09-23T23:33:12.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:12 smithi110 ceph-mon[212939]: failed to encode map e548 with expected crc 2023-09-23T23:33:12.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:12 smithi110 ceph-mon[212939]: from='client.36480 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:33:12.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:12 smithi110 ceph-mon[221102]: failed to encode map e548 with expected crc 2023-09-23T23:33:12.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:12 smithi110 ceph-mon[221102]: osdmap e548: 8 total, 8 up, 8 in 2023-09-23T23:33:12.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:12 smithi110 ceph-mon[221102]: failed to encode map e548 with expected crc 2023-09-23T23:33:12.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:12 smithi110 ceph-mon[221102]: failed to encode map e548 with expected crc 2023-09-23T23:33:12.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:12 smithi110 ceph-mon[221102]: failed to encode map e548 with expected crc 2023-09-23T23:33:12.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:12 smithi110 ceph-mon[221102]: from='client.36480 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:33:12.613 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:33:12.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:12.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:33:12.719 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:12.743 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1507,"stamp":"2023-09-23T23:33:11.005422+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1336350624,"num_objects":20406,"num_object_clones":0,"num_object_copies":61218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":134,"num_objects_unfound":0,"num_objects_dirty":20406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20443,"num_write_kb":1304477,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1149,"num_bytes_recovered":72887020,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":25428,"ondisk_log_size":25428,"up":357,"acting":356,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":366,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10832972,"kb_used_data":4833612,"kb_used_omap":29,"kb_used_meta":5999266,"kb_avail":739128244,"statfs":{"total":767960285184,"available":756867321856,"internally_reserved":0,"allocated":4949618688,"data_stored":4933430997,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":6143249107},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":598409216,"num_objects":9131,"num_object_clones":0,"num_object_copies":27393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-607,"num_objects_misplaced":-327,"num_objects_unfound":0,"num_objects_dirty":9131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9131,"num_write_kb":584384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":934,"num_bytes_recovered":61210624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.026123"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":741,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.989414+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:33:05.989414+0000","last_peered":"2023-09-23T23:33:05.989414+0000","last_clean":"2023-09-23T23:33:05.989414+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:33:05.989414+0000","last_undegraded":"2023-09-23T23:33:05.989414+0000","last_fullsized":"2023-09-23T23:33:05.989414+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":579,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.991406+0000","last_change":"2023-09-23T23:22:38.439629+0000","last_active":"2023-09-23T23:33:05.991406+0000","last_peered":"2023-09-23T23:33:05.991406+0000","last_clean":"2023-09-23T23:33:05.991406+0000","last_became_active":"2023-09-23T23:22:38.439285+0000","last_became_peered":"2023-09-23T23:22:38.439285+0000","last_unstale":"2023-09-23T23:33:05.991406+0000","last_undegraded":"2023-09-23T23:33:05.991406+0000","last_fullsized":"2023-09-23T23:33:05.991406+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:34.431221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":517,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.003100+0000","last_change":"2023-09-23T23:23:43.511912+0000","last_active":"2023-09-23T23:33:08.003100+0000","last_peered":"2023-09-23T23:33:08.003100+0000","last_clean":"2023-09-23T23:33:08.003100+0000","last_became_active":"2023-09-23T23:23:43.511574+0000","last_became_peered":"2023-09-23T23:23:43.511574+0000","last_unstale":"2023-09-23T23:33:08.003100+0000","last_undegraded":"2023-09-23T23:33:08.003100+0000","last_fullsized":"2023-09-23T23:33:08.003100+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:16.286926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":654,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.989699+0000","last_change":"2023-09-23T23:33:04.705919+0000","last_active":"2023-09-23T23:33:05.989699+0000","last_peered":"2023-09-23T23:33:05.989699+0000","last_clean":"2023-09-23T23:33:05.989699+0000","last_became_active":"2023-09-23T23:33:04.705569+0000","last_became_peered":"2023-09-23T23:33:04.705569+0000","last_unstale":"2023-09-23T23:33:05.989699+0000","last_undegraded":"2023-09-23T23:33:05.989699+0000","last_fullsized":"2023-09-23T23:33:05.989699+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:56.250925+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":259,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.018503+0000","last_change":"2023-09-23T23:26:11.528846+0000","last_active":"2023-09-23T23:33:09.018503+0000","last_peered":"2023-09-23T23:33:09.018503+0000","last_clean":"2023-09-23T23:33:09.018503+0000","last_became_active":"2023-09-23T23:26:11.528529+0000","last_became_peered":"2023-09-23T23:26:11.528529+0000","last_unstale":"2023-09-23T23:33:09.018503+0000","last_undegraded":"2023-09-23T23:33:09.018503+0000","last_fullsized":"2023-09-23T23:33:09.018503+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:35.094583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":520,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.997681+0000","last_change":"2023-09-23T23:33:01.953319+0000","last_active":"2023-09-23T23:33:05.997681+0000","last_peered":"2023-09-23T23:33:05.997681+0000","last_clean":"2023-09-23T23:33:05.997681+0000","last_became_active":"2023-09-23T23:33:01.952918+0000","last_became_peered":"2023-09-23T23:33:01.952918+0000","last_unstale":"2023-09-23T23:33:05.997681+0000","last_undegraded":"2023-09-23T23:33:05.997681+0000","last_fullsized":"2023-09-23T23:33:05.997681+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:59.313898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":848,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.991100+0000","last_change":"2023-09-23T23:28:05.634824+0000","last_active":"2023-09-23T23:33:05.991100+0000","last_peered":"2023-09-23T23:33:05.991100+0000","last_clean":"2023-09-23T23:33:05.991100+0000","last_became_active":"2023-09-23T23:27:23.551046+0000","last_became_peered":"2023-09-23T23:27:23.551046+0000","last_unstale":"2023-09-23T23:33:05.991100+0000","last_undegraded":"2023-09-23T23:33:05.991100+0000","last_fullsized":"2023-09-23T23:33:05.991100+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:59.062239+0000","scrub_duration":0.0034158959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":554,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.001187+0000","last_change":"2023-09-23T23:33:01.954991+0000","last_active":"2023-09-23T23:33:08.001187+0000","last_peered":"2023-09-23T23:33:08.001187+0000","last_clean":"2023-09-23T23:33:08.001187+0000","last_became_active":"2023-09-23T23:33:01.954686+0000","last_became_peered":"2023-09-23T23:33:01.954686+0000","last_unstale":"2023-09-23T23:33:08.001187+0000","last_undegraded":"2023-09-23T23:33:08.001187+0000","last_fullsized":"2023-09-23T23:33:08.001187+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:44:20.221144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":59,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.999553+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:33:05.999553+0000","last_peered":"2023-09-23T23:33:05.999553+0000","last_clean":"2023-09-23T23:33:05.999553+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:33:05.999553+0000","last_undegraded":"2023-09-23T23:33:05.999553+0000","last_fullsized":"2023-09-23T23:33:05.999553+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":865,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.993747+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:33:05.993747+0000","last_peered":"2023-09-23T23:33:05.993747+0000","last_clean":"2023-09-23T23:33:05.993747+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:33:05.993747+0000","last_undegraded":"2023-09-23T23:33:05.993747+0000","last_fullsized":"2023-09-23T23:33:05.993747+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":187,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.990077+0000","last_change":"2023-09-23T23:33:01.952256+0000","last_active":"2023-09-23T23:33:05.990077+0000","last_peered":"2023-09-23T23:33:05.990077+0000","last_clean":"2023-09-23T23:33:05.990077+0000","last_became_active":"2023-09-23T23:33:01.951916+0000","last_became_peered":"2023-09-23T23:33:01.951916+0000","last_unstale":"2023-09-23T23:33:05.990077+0000","last_undegraded":"2023-09-23T23:33:05.990077+0000","last_fullsized":"2023-09-23T23:33:05.990077+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:15.646713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":545,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.001118+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:33:08.001118+0000","last_peered":"2023-09-23T23:33:08.001118+0000","last_clean":"2023-09-23T23:33:08.001118+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:33:08.001118+0000","last_undegraded":"2023-09-23T23:33:08.001118+0000","last_fullsized":"2023-09-23T23:33:08.001118+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":821,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.013889+0000","last_change":"2023-09-23T23:26:14.521792+0000","last_active":"2023-09-23T23:33:09.013889+0000","last_peered":"2023-09-23T23:33:09.013889+0000","last_clean":"2023-09-23T23:33:09.013889+0000","last_became_active":"2023-09-23T23:26:14.521438+0000","last_became_peered":"2023-09-23T23:26:14.521438+0000","last_unstale":"2023-09-23T23:33:09.013889+0000","last_undegraded":"2023-09-23T23:33:09.013889+0000","last_fullsized":"2023-09-23T23:33:09.013889+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:09.103712+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":176,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.990373+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:33:05.990373+0000","last_peered":"2023-09-23T23:33:05.990373+0000","last_clean":"2023-09-23T23:33:05.990373+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:33:05.990373+0000","last_undegraded":"2023-09-23T23:33:05.990373+0000","last_fullsized":"2023-09-23T23:33:05.990373+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":506,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.002274+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:33:08.002274+0000","last_peered":"2023-09-23T23:33:08.002274+0000","last_clean":"2023-09-23T23:33:08.002274+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:33:08.002274+0000","last_undegraded":"2023-09-23T23:33:08.002274+0000","last_fullsized":"2023-09-23T23:33:08.002274+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":979,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.989911+0000","last_change":"2023-09-23T23:33:04.699680+0000","last_active":"2023-09-23T23:33:05.989911+0000","last_peered":"2023-09-23T23:33:05.989911+0000","last_clean":"2023-09-23T23:33:05.989911+0000","last_became_active":"2023-09-23T23:33:04.699261+0000","last_became_peered":"2023-09-23T23:33:04.699261+0000","last_unstale":"2023-09-23T23:33:05.989911+0000","last_undegraded":"2023-09-23T23:33:05.989911+0000","last_fullsized":"2023-09-23T23:33:05.989911+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:59.599804+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":150,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.993239+0000","last_change":"2023-09-23T23:26:48.669857+0000","last_active":"2023-09-23T23:33:05.993239+0000","last_peered":"2023-09-23T23:33:05.993239+0000","last_clean":"2023-09-23T23:33:05.993239+0000","last_became_active":"2023-09-23T23:26:48.538246+0000","last_became_peered":"2023-09-23T23:26:48.538246+0000","last_unstale":"2023-09-23T23:33:05.993239+0000","last_undegraded":"2023-09-23T23:33:05.993239+0000","last_fullsized":"2023-09-23T23:33:05.993239+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:19.039472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":144,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.018567+0000","last_change":"2023-09-23T23:29:54.996097+0000","last_active":"2023-09-23T23:33:09.018567+0000","last_peered":"2023-09-23T23:33:09.018567+0000","last_clean":"2023-09-23T23:33:09.018567+0000","last_became_active":"2023-09-23T23:29:54.995756+0000","last_became_peered":"2023-09-23T23:29:54.995756+0000","last_unstale":"2023-09-23T23:33:09.018567+0000","last_undegraded":"2023-09-23T23:33:09.018567+0000","last_fullsized":"2023-09-23T23:33:09.018567+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:15.856474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":623,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.019632+0000","last_change":"2023-09-23T23:24:02.268079+0000","last_active":"2023-09-23T23:33:09.019632+0000","last_peered":"2023-09-23T23:33:09.019632+0000","last_clean":"2023-09-23T23:33:09.019632+0000","last_became_active":"2023-09-23T23:23:45.463106+0000","last_became_peered":"2023-09-23T23:23:45.463106+0000","last_unstale":"2023-09-23T23:33:09.019632+0000","last_undegraded":"2023-09-23T23:33:09.019632+0000","last_fullsized":"2023-09-23T23:33:09.019632+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.128813+0000","scrub_duration":0.0018535310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":196,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.991104+0000","last_change":"2023-09-23T23:29:55.012082+0000","last_active":"2023-09-23T23:33:05.991104+0000","last_peered":"2023-09-23T23:33:05.991104+0000","last_clean":"2023-09-23T23:33:05.991104+0000","last_became_active":"2023-09-23T23:29:55.000202+0000","last_became_peered":"2023-09-23T23:29:55.000202+0000","last_unstale":"2023-09-23T23:33:05.991104+0000","last_undegraded":"2023-09-23T23:33:05.991104+0000","last_fullsized":"2023-09-23T23:33:05.991104+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:28.669994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":543,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.988952+0000","last_change":"2023-09-23T23:29:55.004242+0000","last_active":"2023-09-23T23:33:05.988952+0000","last_peered":"2023-09-23T23:33:05.988952+0000","last_clean":"2023-09-23T23:33:05.988952+0000","last_became_active":"2023-09-23T23:29:55.003934+0000","last_became_peered":"2023-09-23T23:29:55.003934+0000","last_unstale":"2023-09-23T23:33:05.988952+0000","last_undegraded":"2023-09-23T23:33:05.988952+0000","last_fullsized":"2023-09-23T23:33:05.988952+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:09.283771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":680,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.992315+0000","last_change":"2023-09-23T23:33:04.701609+0000","last_active":"2023-09-23T23:33:05.992315+0000","last_peered":"2023-09-23T23:33:05.992315+0000","last_clean":"2023-09-23T23:33:05.992315+0000","last_became_active":"2023-09-23T23:33:04.701281+0000","last_became_peered":"2023-09-23T23:33:04.701281+0000","last_unstale":"2023-09-23T23:33:05.992315+0000","last_undegraded":"2023-09-23T23:33:05.992315+0000","last_fullsized":"2023-09-23T23:33:05.992315+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:31.726920+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":477,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.993672+0000","last_change":"2023-09-23T23:26:11.569228+0000","last_active":"2023-09-23T23:33:05.993672+0000","last_peered":"2023-09-23T23:33:05.993672+0000","last_clean":"2023-09-23T23:33:05.993672+0000","last_became_active":"2023-09-23T23:26:11.513159+0000","last_became_peered":"2023-09-23T23:26:11.513159+0000","last_unstale":"2023-09-23T23:33:05.993672+0000","last_undegraded":"2023-09-23T23:33:05.993672+0000","last_fullsized":"2023-09-23T23:33:05.993672+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:40.698257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":544,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.001704+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:33:08.001704+0000","last_peered":"2023-09-23T23:33:08.001704+0000","last_clean":"2023-09-23T23:33:08.001704+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:33:08.001704+0000","last_undegraded":"2023-09-23T23:33:08.001704+0000","last_fullsized":"2023-09-23T23:33:08.001704+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":904,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.992944+0000","last_change":"2023-09-23T23:29:58.259420+0000","last_active":"2023-09-23T23:33:05.992944+0000","last_peered":"2023-09-23T23:33:05.992944+0000","last_clean":"2023-09-23T23:33:05.992944+0000","last_became_active":"2023-09-23T23:29:58.258879+0000","last_became_peered":"2023-09-23T23:29:58.258879+0000","last_unstale":"2023-09-23T23:33:05.992944+0000","last_undegraded":"2023-09-23T23:33:05.992944+0000","last_fullsized":"2023-09-23T23:33:05.992944+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:14.307363+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":154,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.993444+0000","last_change":"2023-09-23T23:33:01.947164+0000","last_active":"2023-09-23T23:33:05.993444+0000","last_peered":"2023-09-23T23:33:05.993444+0000","last_clean":"2023-09-23T23:33:05.993444+0000","last_became_active":"2023-09-23T23:33:01.946821+0000","last_became_peered":"2023-09-23T23:33:01.946821+0000","last_unstale":"2023-09-23T23:33:05.993444+0000","last_undegraded":"2023-09-23T23:33:05.993444+0000","last_fullsized":"2023-09-23T23:33:05.993444+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:59.172357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":431,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.993667+0000","last_change":"2023-09-23T23:26:32.040316+0000","last_active":"2023-09-23T23:33:05.993667+0000","last_peered":"2023-09-23T23:33:05.993667+0000","last_clean":"2023-09-23T23:33:05.993667+0000","last_became_active":"2023-09-23T23:26:32.039758+0000","last_became_peered":"2023-09-23T23:26:32.039758+0000","last_unstale":"2023-09-23T23:33:05.993667+0000","last_undegraded":"2023-09-23T23:33:05.993667+0000","last_fullsized":"2023-09-23T23:33:05.993667+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:03.930331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":712,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.019030+0000","last_change":"2023-09-23T23:26:14.527752+0000","last_active":"2023-09-23T23:33:09.019030+0000","last_peered":"2023-09-23T23:33:09.019030+0000","last_clean":"2023-09-23T23:33:09.019030+0000","last_became_active":"2023-09-23T23:26:14.525188+0000","last_became_peered":"2023-09-23T23:26:14.525188+0000","last_unstale":"2023-09-23T23:33:09.019030+0000","last_undegraded":"2023-09-23T23:33:09.019030+0000","last_fullsized":"2023-09-23T23:33:09.019030+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:10.617744+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":409,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.994788+0000","last_change":"2023-09-23T23:29:54.999839+0000","last_active":"2023-09-23T23:33:05.994788+0000","last_peered":"2023-09-23T23:33:05.994788+0000","last_clean":"2023-09-23T23:33:05.994788+0000","last_became_active":"2023-09-23T23:29:54.999530+0000","last_became_peered":"2023-09-23T23:29:54.999530+0000","last_unstale":"2023-09-23T23:33:05.994788+0000","last_undegraded":"2023-09-23T23:33:05.994788+0000","last_fullsized":"2023-09-23T23:33:05.994788+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:43.757346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":491,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.004168+0000","last_change":"2023-09-23T23:33:01.951268+0000","last_active":"2023-09-23T23:33:08.004168+0000","last_peered":"2023-09-23T23:33:08.004168+0000","last_clean":"2023-09-23T23:33:08.004168+0000","last_became_active":"2023-09-23T23:33:01.950984+0000","last_became_peered":"2023-09-23T23:33:01.950984+0000","last_unstale":"2023-09-23T23:33:08.004168+0000","last_undegraded":"2023-09-23T23:33:08.004168+0000","last_fullsized":"2023-09-23T23:33:08.004168+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:53.187008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":908,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.989667+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:33:05.989667+0000","last_peered":"2023-09-23T23:33:05.989667+0000","last_clean":"2023-09-23T23:33:05.989667+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:33:05.989667+0000","last_undegraded":"2023-09-23T23:33:05.989667+0000","last_fullsized":"2023-09-23T23:33:05.989667+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":89,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.997493+0000","last_change":"2023-09-23T23:33:01.960045+0000","last_active":"2023-09-23T23:33:05.997493+0000","last_peered":"2023-09-23T23:33:05.997493+0000","last_clean":"2023-09-23T23:33:05.997493+0000","last_became_active":"2023-09-23T23:33:01.951262+0000","last_became_peered":"2023-09-23T23:33:01.951262+0000","last_unstale":"2023-09-23T23:33:05.997493+0000","last_undegraded":"2023-09-23T23:33:05.997493+0000","last_fullsized":"2023-09-23T23:33:05.997493+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:05.742976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":185,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.004071+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:33:08.004071+0000","last_peered":"2023-09-23T23:33:08.004071+0000","last_clean":"2023-09-23T23:33:08.004071+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:33:08.004071+0000","last_undegraded":"2023-09-23T23:33:08.004071+0000","last_fullsized":"2023-09-23T23:33:08.004071+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"545'1233","reported_seq":1252,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.235638+0000","last_change":"2023-09-23T23:32:51.929504+0000","last_active":"2023-09-23T23:33:08.235638+0000","last_peered":"2023-09-23T23:33:08.235638+0000","last_clean":"2023-09-23T23:33:08.235638+0000","last_became_active":"2023-09-23T23:32:51.929246+0000","last_became_peered":"2023-09-23T23:32:51.929246+0000","last_unstale":"2023-09-23T23:33:08.235638+0000","last_undegraded":"2023-09-23T23:33:08.235638+0000","last_fullsized":"2023-09-23T23:33:08.235638+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":1233,"log_dups_size":0,"ondisk_log_size":1233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:59.358999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80805888,"num_objects":1233,"num_object_clones":0,"num_object_copies":3699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1233,"num_write_kb":78912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":872,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.993322+0000","last_change":"2023-09-23T23:26:51.563868+0000","last_active":"2023-09-23T23:33:05.993322+0000","last_peered":"2023-09-23T23:33:05.993322+0000","last_clean":"2023-09-23T23:33:05.993322+0000","last_became_active":"2023-09-23T23:26:51.563510+0000","last_became_peered":"2023-09-23T23:26:51.563510+0000","last_unstale":"2023-09-23T23:33:05.993322+0000","last_undegraded":"2023-09-23T23:33:05.993322+0000","last_fullsized":"2023-09-23T23:33:05.993322+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:34.244822+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":162,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.997708+0000","last_change":"2023-09-23T23:33:01.958261+0000","last_active":"2023-09-23T23:33:05.997708+0000","last_peered":"2023-09-23T23:33:05.997708+0000","last_clean":"2023-09-23T23:33:05.997708+0000","last_became_active":"2023-09-23T23:33:01.948504+0000","last_became_peered":"2023-09-23T23:33:01.948504+0000","last_unstale":"2023-09-23T23:33:05.997708+0000","last_undegraded":"2023-09-23T23:33:05.997708+0000","last_fullsized":"2023-09-23T23:33:05.997708+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:19.975041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":176,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.989331+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:33:05.989331+0000","last_peered":"2023-09-23T23:33:05.989331+0000","last_clean":"2023-09-23T23:33:05.989331+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:33:05.989331+0000","last_undegraded":"2023-09-23T23:33:05.989331+0000","last_fullsized":"2023-09-23T23:33:05.989331+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"545'1240","reported_seq":1259,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.206089+0000","last_change":"2023-09-23T23:32:51.925959+0000","last_active":"2023-09-23T23:33:08.206089+0000","last_peered":"2023-09-23T23:33:08.206089+0000","last_clean":"2023-09-23T23:33:08.206089+0000","last_became_active":"2023-09-23T23:32:51.925673+0000","last_became_peered":"2023-09-23T23:32:51.925673+0000","last_unstale":"2023-09-23T23:33:08.206089+0000","last_undegraded":"2023-09-23T23:33:08.206089+0000","last_fullsized":"2023-09-23T23:33:08.206089+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":1240,"log_dups_size":0,"ondisk_log_size":1240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:18:15.809611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81264640,"num_objects":1240,"num_object_clones":0,"num_object_copies":3720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1240,"num_write_kb":79360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":956,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.001410+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:33:08.001410+0000","last_peered":"2023-09-23T23:33:08.001410+0000","last_clean":"2023-09-23T23:33:08.001410+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:33:08.001410+0000","last_undegraded":"2023-09-23T23:33:08.001410+0000","last_fullsized":"2023-09-23T23:33:08.001410+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":148,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.997945+0000","last_change":"2023-09-23T23:26:48.547146+0000","last_active":"2023-09-23T23:33:05.997945+0000","last_peered":"2023-09-23T23:33:05.997945+0000","last_clean":"2023-09-23T23:33:05.997945+0000","last_became_active":"2023-09-23T23:26:48.537682+0000","last_became_peered":"2023-09-23T23:26:48.537682+0000","last_unstale":"2023-09-23T23:33:05.997945+0000","last_undegraded":"2023-09-23T23:33:05.997945+0000","last_fullsized":"2023-09-23T23:33:05.997945+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:10.115563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":570,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.991527+0000","last_change":"2023-09-23T23:26:01.833889+0000","last_active":"2023-09-23T23:33:05.991527+0000","last_peered":"2023-09-23T23:33:05.991527+0000","last_clean":"2023-09-23T23:33:05.991527+0000","last_became_active":"2023-09-23T23:26:01.833626+0000","last_became_peered":"2023-09-23T23:26:01.833626+0000","last_unstale":"2023-09-23T23:33:05.991527+0000","last_undegraded":"2023-09-23T23:33:05.991527+0000","last_fullsized":"2023-09-23T23:33:05.991527+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:25.659671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"544'1290","reported_seq":2571,"reported_epoch":545,"state":"peering","last_fresh":"2023-09-23T23:33:08.003299+0000","last_change":"2023-09-23T23:33:08.003848+0000","last_active":"2023-09-23T23:33:08.003260+0000","last_peered":"2023-09-23T23:33:08.000408+0000","last_clean":"2023-09-23T23:33:00.943034+0000","last_became_active":"2023-09-23T23:33:02.968701+0000","last_became_peered":"2023-09-23T23:33:02.968701+0000","last_unstale":"2023-09-23T23:33:08.003299+0000","last_undegraded":"2023-09-23T23:33:08.003299+0000","last_fullsized":"2023-09-23T23:33:08.003299+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":1290,"log_dups_size":0,"ondisk_log_size":1290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:08.897838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84541440,"num_objects":1290,"num_object_clones":0,"num_object_copies":3870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1290,"num_write_kb":82560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":624,"num_bytes_recovered":40894464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,3,5],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":835,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.999635+0000","last_change":"2023-09-23T23:29:58.262103+0000","last_active":"2023-09-23T23:33:05.999635+0000","last_peered":"2023-09-23T23:33:05.999635+0000","last_clean":"2023-09-23T23:33:05.999635+0000","last_became_active":"2023-09-23T23:29:58.261689+0000","last_became_peered":"2023-09-23T23:29:58.261689+0000","last_unstale":"2023-09-23T23:33:05.999635+0000","last_undegraded":"2023-09-23T23:33:05.999635+0000","last_fullsized":"2023-09-23T23:33:05.999635+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:44.945353+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":465,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.993481+0000","last_change":"2023-09-23T23:29:55.006616+0000","last_active":"2023-09-23T23:33:05.993481+0000","last_peered":"2023-09-23T23:33:05.993481+0000","last_clean":"2023-09-23T23:33:05.993481+0000","last_became_active":"2023-09-23T23:29:55.000411+0000","last_became_peered":"2023-09-23T23:29:55.000411+0000","last_unstale":"2023-09-23T23:33:05.993481+0000","last_undegraded":"2023-09-23T23:33:05.993481+0000","last_fullsized":"2023-09-23T23:33:05.993481+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:22.629830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":59,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.998972+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:33:05.998972+0000","last_peered":"2023-09-23T23:33:05.998972+0000","last_clean":"2023-09-23T23:33:05.998972+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:33:05.998972+0000","last_undegraded":"2023-09-23T23:33:05.998972+0000","last_fullsized":"2023-09-23T23:33:05.998972+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"546'1257","reported_seq":1278,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.330786+0000","last_change":"2023-09-23T23:32:51.925955+0000","last_active":"2023-09-23T23:33:09.330786+0000","last_peered":"2023-09-23T23:33:09.330786+0000","last_clean":"2023-09-23T23:33:09.330786+0000","last_became_active":"2023-09-23T23:32:51.925649+0000","last_became_peered":"2023-09-23T23:32:51.925649+0000","last_unstale":"2023-09-23T23:33:09.330786+0000","last_undegraded":"2023-09-23T23:33:09.330786+0000","last_fullsized":"2023-09-23T23:33:09.330786+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":1257,"log_dups_size":0,"ondisk_log_size":1257,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:59.420467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82378752,"num_objects":1257,"num_object_clones":0,"num_object_copies":3771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1257,"num_write_kb":80448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":609,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.991547+0000","last_change":"2023-09-23T23:28:03.629593+0000","last_active":"2023-09-23T23:33:05.991547+0000","last_peered":"2023-09-23T23:33:05.991547+0000","last_clean":"2023-09-23T23:33:05.991547+0000","last_became_active":"2023-09-23T23:26:51.588385+0000","last_became_peered":"2023-09-23T23:26:51.588385+0000","last_unstale":"2023-09-23T23:33:05.991547+0000","last_undegraded":"2023-09-23T23:33:05.991547+0000","last_fullsized":"2023-09-23T23:33:05.991547+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:20.818366+0000","scrub_duration":0.0050135120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":385,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.001571+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:33:08.001571+0000","last_peered":"2023-09-23T23:33:08.001571+0000","last_clean":"2023-09-23T23:33:08.001571+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:33:08.001571+0000","last_undegraded":"2023-09-23T23:33:08.001571+0000","last_fullsized":"2023-09-23T23:33:08.001571+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":533,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.018220+0000","last_change":"2023-09-23T23:28:45.066443+0000","last_active":"2023-09-23T23:33:09.018220+0000","last_peered":"2023-09-23T23:33:09.018220+0000","last_clean":"2023-09-23T23:33:09.018220+0000","last_became_active":"2023-09-23T23:28:45.066154+0000","last_became_peered":"2023-09-23T23:28:45.066154+0000","last_unstale":"2023-09-23T23:33:09.018220+0000","last_undegraded":"2023-09-23T23:33:09.018220+0000","last_fullsized":"2023-09-23T23:33:09.018220+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:51.651425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"545'1299","reported_seq":2274,"reported_epoch":545,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:33:08.917031+0000","last_change":"2023-09-23T23:33:06.994798+0000","last_active":"2023-09-23T23:33:08.917031+0000","last_peered":"2023-09-23T23:33:08.917031+0000","last_clean":"2023-09-23T23:33:00.942436+0000","last_became_active":"2023-09-23T23:33:02.970045+0000","last_became_peered":"2023-09-23T23:33:02.970045+0000","last_unstale":"2023-09-23T23:33:08.917031+0000","last_undegraded":"2023-09-23T23:33:02.951389+0000","last_fullsized":"2023-09-23T23:33:02.948535+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":1299,"log_dups_size":0,"ondisk_log_size":1299,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:32.781968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85131264,"num_objects":1299,"num_object_clones":0,"num_object_copies":3897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":134,"num_objects_unfound":0,"num_objects_dirty":1299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1299,"num_write_kb":83136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":473,"num_bytes_recovered":30998528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5],"avail_no_missing":["4","0","5"],"object_location_counts":[{"shards":"0,4,5","objects":134},{"shards":"4,5","objects":1164}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":880,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.994866+0000","last_change":"2023-09-23T23:23:45.830384+0000","last_active":"2023-09-23T23:33:05.994866+0000","last_peered":"2023-09-23T23:33:05.994866+0000","last_clean":"2023-09-23T23:33:05.994866+0000","last_became_active":"2023-09-23T23:23:36.025724+0000","last_became_peered":"2023-09-23T23:23:36.025724+0000","last_unstale":"2023-09-23T23:33:05.994866+0000","last_undegraded":"2023-09-23T23:33:05.994866+0000","last_fullsized":"2023-09-23T23:33:05.994866+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:34.345528+0000","scrub_duration":0.001659259,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":528,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.002042+0000","last_change":"2023-09-23T23:33:01.966036+0000","last_active":"2023-09-23T23:33:08.002042+0000","last_peered":"2023-09-23T23:33:08.002042+0000","last_clean":"2023-09-23T23:33:08.002042+0000","last_became_active":"2023-09-23T23:33:01.955027+0000","last_became_peered":"2023-09-23T23:33:01.955027+0000","last_unstale":"2023-09-23T23:33:08.002042+0000","last_undegraded":"2023-09-23T23:33:08.002042+0000","last_fullsized":"2023-09-23T23:33:08.002042+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:32.631422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":523,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.991163+0000","last_change":"2023-09-23T23:29:55.001192+0000","last_active":"2023-09-23T23:33:05.991163+0000","last_peered":"2023-09-23T23:33:05.991163+0000","last_clean":"2023-09-23T23:33:05.991163+0000","last_became_active":"2023-09-23T23:29:55.000904+0000","last_became_peered":"2023-09-23T23:29:55.000904+0000","last_unstale":"2023-09-23T23:33:05.991163+0000","last_undegraded":"2023-09-23T23:33:05.991163+0000","last_fullsized":"2023-09-23T23:33:05.991163+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:29.169209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"545'1280","reported_seq":1299,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.220070+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:33:08.220070+0000","last_peered":"2023-09-23T23:33:08.220070+0000","last_clean":"2023-09-23T23:33:08.220070+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:33:08.220070+0000","last_undegraded":"2023-09-23T23:33:08.220070+0000","last_fullsized":"2023-09-23T23:33:08.220070+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":1280,"log_dups_size":0,"ondisk_log_size":1280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83886080,"num_objects":1280,"num_object_clones":0,"num_object_copies":3840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1280,"num_write_kb":81920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":827,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:07.992410+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:33:07.992410+0000","last_peered":"2023-09-23T23:33:07.992410+0000","last_clean":"2023-09-23T23:33:07.992410+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:33:07.992410+0000","last_undegraded":"2023-09-23T23:33:07.992410+0000","last_fullsized":"2023-09-23T23:33:07.992410+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":59,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.998899+0000","last_change":"2023-09-23T23:29:54.998914+0000","last_active":"2023-09-23T23:33:05.998899+0000","last_peered":"2023-09-23T23:33:05.998899+0000","last_clean":"2023-09-23T23:33:05.998899+0000","last_became_active":"2023-09-23T23:29:54.998620+0000","last_became_peered":"2023-09-23T23:29:54.998620+0000","last_unstale":"2023-09-23T23:33:05.998899+0000","last_undegraded":"2023-09-23T23:33:05.998899+0000","last_fullsized":"2023-09-23T23:33:05.998899+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:32.702628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":166,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.991705+0000","last_change":"2023-09-23T23:26:11.516475+0000","last_active":"2023-09-23T23:33:05.991705+0000","last_peered":"2023-09-23T23:33:05.991705+0000","last_clean":"2023-09-23T23:33:05.991705+0000","last_became_active":"2023-09-23T23:26:11.516198+0000","last_became_peered":"2023-09-23T23:26:11.516198+0000","last_unstale":"2023-09-23T23:33:05.991705+0000","last_undegraded":"2023-09-23T23:33:05.991705+0000","last_fullsized":"2023-09-23T23:33:05.991705+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:56.875103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"545'1320","reported_seq":1339,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.217722+0000","last_change":"2023-09-23T23:32:51.927456+0000","last_active":"2023-09-23T23:33:08.217722+0000","last_peered":"2023-09-23T23:33:08.217722+0000","last_clean":"2023-09-23T23:33:08.217722+0000","last_became_active":"2023-09-23T23:32:51.926961+0000","last_became_peered":"2023-09-23T23:32:51.926961+0000","last_unstale":"2023-09-23T23:33:08.217722+0000","last_undegraded":"2023-09-23T23:33:08.217722+0000","last_fullsized":"2023-09-23T23:33:08.217722+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":1320,"log_dups_size":0,"ondisk_log_size":1320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:09.809921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86507520,"num_objects":1320,"num_object_clones":0,"num_object_copies":3960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1320,"num_write_kb":84480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":721,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.018847+0000","last_change":"2023-09-23T23:33:04.697442+0000","last_active":"2023-09-23T23:33:09.018847+0000","last_peered":"2023-09-23T23:33:09.018847+0000","last_clean":"2023-09-23T23:33:09.018847+0000","last_became_active":"2023-09-23T23:33:04.697062+0000","last_became_peered":"2023-09-23T23:33:04.697062+0000","last_unstale":"2023-09-23T23:33:09.018847+0000","last_undegraded":"2023-09-23T23:33:09.018847+0000","last_fullsized":"2023-09-23T23:33:09.018847+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:50.295822+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":166,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.991354+0000","last_change":"2023-09-23T23:26:11.512815+0000","last_active":"2023-09-23T23:33:05.991354+0000","last_peered":"2023-09-23T23:33:05.991354+0000","last_clean":"2023-09-23T23:33:05.991354+0000","last_became_active":"2023-09-23T23:26:11.512466+0000","last_became_peered":"2023-09-23T23:26:11.512466+0000","last_unstale":"2023-09-23T23:33:05.991354+0000","last_undegraded":"2023-09-23T23:33:05.991354+0000","last_fullsized":"2023-09-23T23:33:05.991354+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:10.324158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":553,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.991789+0000","last_change":"2023-09-23T23:29:55.001597+0000","last_active":"2023-09-23T23:33:05.991789+0000","last_peered":"2023-09-23T23:33:05.991789+0000","last_clean":"2023-09-23T23:33:05.991789+0000","last_became_active":"2023-09-23T23:29:55.001314+0000","last_became_peered":"2023-09-23T23:29:55.001314+0000","last_unstale":"2023-09-23T23:33:05.991789+0000","last_undegraded":"2023-09-23T23:33:05.991789+0000","last_fullsized":"2023-09-23T23:33:05.991789+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:00.024016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"545'1304","reported_seq":1323,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.217374+0000","last_change":"2023-09-23T23:32:51.927504+0000","last_active":"2023-09-23T23:33:08.217374+0000","last_peered":"2023-09-23T23:33:08.217374+0000","last_clean":"2023-09-23T23:33:08.217374+0000","last_became_active":"2023-09-23T23:32:51.927190+0000","last_became_peered":"2023-09-23T23:32:51.927190+0000","last_unstale":"2023-09-23T23:33:08.217374+0000","last_undegraded":"2023-09-23T23:33:08.217374+0000","last_fullsized":"2023-09-23T23:33:08.217374+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":1304,"log_dups_size":0,"ondisk_log_size":1304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:04.208118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85458944,"num_objects":1304,"num_object_clones":0,"num_object_copies":3912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1304,"num_write_kb":83456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":886,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:07.995880+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:33:07.995880+0000","last_peered":"2023-09-23T23:33:07.995880+0000","last_clean":"2023-09-23T23:33:07.995880+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:33:07.995880+0000","last_undegraded":"2023-09-23T23:33:07.995880+0000","last_fullsized":"2023-09-23T23:33:07.995880+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":186,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.000999+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:33:08.000999+0000","last_peered":"2023-09-23T23:33:08.000999+0000","last_clean":"2023-09-23T23:33:08.000999+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:33:08.000999+0000","last_undegraded":"2023-09-23T23:33:08.000999+0000","last_fullsized":"2023-09-23T23:33:08.000999+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":470,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.019858+0000","last_change":"2023-09-23T23:22:39.489325+0000","last_active":"2023-09-23T23:33:09.019858+0000","last_peered":"2023-09-23T23:33:09.019858+0000","last_clean":"2023-09-23T23:33:09.019858+0000","last_became_active":"2023-09-23T23:22:39.489048+0000","last_became_peered":"2023-09-23T23:22:39.489048+0000","last_unstale":"2023-09-23T23:33:09.019858+0000","last_undegraded":"2023-09-23T23:33:09.019858+0000","last_fullsized":"2023-09-23T23:33:09.019858+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:29.984948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"546'1205","reported_seq":1226,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.287768+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:33:09.287768+0000","last_peered":"2023-09-23T23:33:09.287768+0000","last_clean":"2023-09-23T23:33:09.287768+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:33:09.287768+0000","last_undegraded":"2023-09-23T23:33:09.287768+0000","last_fullsized":"2023-09-23T23:33:09.287768+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":1205,"log_dups_size":0,"ondisk_log_size":1205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78970880,"num_objects":1205,"num_object_clones":0,"num_object_copies":3615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1205,"num_write_kb":77120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":866,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.999269+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:33:05.999269+0000","last_peered":"2023-09-23T23:33:05.999269+0000","last_clean":"2023-09-23T23:33:05.999269+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:33:05.999269+0000","last_undegraded":"2023-09-23T23:33:05.999269+0000","last_fullsized":"2023-09-23T23:33:05.999269+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":251,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.019549+0000","last_change":"2023-09-23T23:23:36.048889+0000","last_active":"2023-09-23T23:33:09.019549+0000","last_peered":"2023-09-23T23:33:09.019549+0000","last_clean":"2023-09-23T23:33:09.019549+0000","last_became_active":"2023-09-23T23:23:36.048572+0000","last_became_peered":"2023-09-23T23:23:36.048572+0000","last_unstale":"2023-09-23T23:33:09.019549+0000","last_undegraded":"2023-09-23T23:33:09.019549+0000","last_fullsized":"2023-09-23T23:33:09.019549+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:55.581146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":560,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.990814+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:33:05.990814+0000","last_peered":"2023-09-23T23:33:05.990814+0000","last_clean":"2023-09-23T23:33:05.990814+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:33:05.990814+0000","last_undegraded":"2023-09-23T23:33:05.990814+0000","last_fullsized":"2023-09-23T23:33:05.990814+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":135,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.018646+0000","last_change":"2023-09-23T23:27:20.549904+0000","last_active":"2023-09-23T23:33:09.018646+0000","last_peered":"2023-09-23T23:33:09.018646+0000","last_clean":"2023-09-23T23:33:09.018646+0000","last_became_active":"2023-09-23T23:27:20.549645+0000","last_became_peered":"2023-09-23T23:27:20.549645+0000","last_unstale":"2023-09-23T23:33:09.018646+0000","last_undegraded":"2023-09-23T23:33:09.018646+0000","last_fullsized":"2023-09-23T23:33:09.018646+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:09.850640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"545'1298","reported_seq":1317,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.215413+0000","last_change":"2023-09-23T23:32:51.928373+0000","last_active":"2023-09-23T23:33:08.215413+0000","last_peered":"2023-09-23T23:33:08.215413+0000","last_clean":"2023-09-23T23:33:08.215413+0000","last_became_active":"2023-09-23T23:32:51.928093+0000","last_became_peered":"2023-09-23T23:32:51.928093+0000","last_unstale":"2023-09-23T23:33:08.215413+0000","last_undegraded":"2023-09-23T23:33:08.215413+0000","last_fullsized":"2023-09-23T23:33:08.215413+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":1298,"log_dups_size":0,"ondisk_log_size":1298,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:44.690480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85065728,"num_objects":1298,"num_object_clones":0,"num_object_copies":3894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1298,"num_write_kb":83072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":603,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.592846+0000","last_change":"2023-09-23T23:33:09.592953+0000","last_active":"2023-09-23T23:33:09.592846+0000","last_peered":"2023-09-23T23:33:09.592846+0000","last_clean":"2023-09-23T23:33:09.592846+0000","last_became_active":"2023-09-23T23:33:04.703345+0000","last_became_peered":"2023-09-23T23:33:04.703345+0000","last_unstale":"2023-09-23T23:33:09.592846+0000","last_undegraded":"2023-09-23T23:33:09.592846+0000","last_fullsized":"2023-09-23T23:33:09.592846+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:40.463955+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":260,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.018873+0000","last_change":"2023-09-23T23:29:54.992796+0000","last_active":"2023-09-23T23:33:09.018873+0000","last_peered":"2023-09-23T23:33:09.018873+0000","last_clean":"2023-09-23T23:33:09.018873+0000","last_became_active":"2023-09-23T23:29:54.992394+0000","last_became_peered":"2023-09-23T23:29:54.992394+0000","last_unstale":"2023-09-23T23:33:09.018873+0000","last_undegraded":"2023-09-23T23:33:09.018873+0000","last_fullsized":"2023-09-23T23:33:09.018873+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:01.353635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":534,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.988889+0000","last_change":"2023-09-23T23:29:55.001578+0000","last_active":"2023-09-23T23:33:05.988889+0000","last_peered":"2023-09-23T23:33:05.988889+0000","last_clean":"2023-09-23T23:33:05.988889+0000","last_became_active":"2023-09-23T23:29:55.001264+0000","last_became_peered":"2023-09-23T23:29:55.001264+0000","last_unstale":"2023-09-23T23:33:05.988889+0000","last_undegraded":"2023-09-23T23:33:05.988889+0000","last_fullsized":"2023-09-23T23:33:05.988889+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:16.411769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":135,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.019360+0000","last_change":"2023-09-23T23:27:20.551221+0000","last_active":"2023-09-23T23:33:09.019360+0000","last_peered":"2023-09-23T23:33:09.019360+0000","last_clean":"2023-09-23T23:33:09.019360+0000","last_became_active":"2023-09-23T23:27:20.550900+0000","last_became_peered":"2023-09-23T23:27:20.550900+0000","last_unstale":"2023-09-23T23:33:09.019360+0000","last_undegraded":"2023-09-23T23:33:09.019360+0000","last_fullsized":"2023-09-23T23:33:09.019360+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:23.012149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.e","version":"545'1194","reported_seq":1213,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.223222+0000","last_change":"2023-09-23T23:32:51.927183+0000","last_active":"2023-09-23T23:33:08.223222+0000","last_peered":"2023-09-23T23:33:08.223222+0000","last_clean":"2023-09-23T23:33:08.223222+0000","last_became_active":"2023-09-23T23:32:51.926883+0000","last_became_peered":"2023-09-23T23:32:51.926883+0000","last_unstale":"2023-09-23T23:33:08.223222+0000","last_undegraded":"2023-09-23T23:33:08.223222+0000","last_fullsized":"2023-09-23T23:33:08.223222+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":1194,"log_dups_size":0,"ondisk_log_size":1194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:05.556061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78249984,"num_objects":1194,"num_object_clones":0,"num_object_copies":3582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1194,"num_write_kb":76416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":848,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:07.992652+0000","last_change":"2023-09-23T23:28:48.113032+0000","last_active":"2023-09-23T23:33:07.992652+0000","last_peered":"2023-09-23T23:33:07.992652+0000","last_clean":"2023-09-23T23:33:07.992652+0000","last_became_active":"2023-09-23T23:28:48.112666+0000","last_became_peered":"2023-09-23T23:28:48.112666+0000","last_unstale":"2023-09-23T23:33:07.992652+0000","last_undegraded":"2023-09-23T23:33:07.992652+0000","last_fullsized":"2023-09-23T23:33:07.992652+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:59:59.838039+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":283,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:07.992054+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:33:07.992054+0000","last_peered":"2023-09-23T23:33:07.992054+0000","last_clean":"2023-09-23T23:33:07.992054+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:33:07.992054+0000","last_undegraded":"2023-09-23T23:33:07.992054+0000","last_fullsized":"2023-09-23T23:33:07.992054+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":260,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.018655+0000","last_change":"2023-09-23T23:29:54.992719+0000","last_active":"2023-09-23T23:33:09.018655+0000","last_peered":"2023-09-23T23:33:09.018655+0000","last_clean":"2023-09-23T23:33:09.018655+0000","last_became_active":"2023-09-23T23:29:54.992412+0000","last_became_peered":"2023-09-23T23:29:54.992412+0000","last_unstale":"2023-09-23T23:33:09.018655+0000","last_undegraded":"2023-09-23T23:33:09.018655+0000","last_fullsized":"2023-09-23T23:33:09.018655+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:46.541397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":144,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.020014+0000","last_change":"2023-09-23T23:33:01.953232+0000","last_active":"2023-09-23T23:33:09.020014+0000","last_peered":"2023-09-23T23:33:09.020014+0000","last_clean":"2023-09-23T23:33:09.020014+0000","last_became_active":"2023-09-23T23:33:01.952957+0000","last_became_peered":"2023-09-23T23:33:01.952957+0000","last_unstale":"2023-09-23T23:33:09.020014+0000","last_undegraded":"2023-09-23T23:33:09.020014+0000","last_fullsized":"2023-09-23T23:33:09.020014+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:32.835910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"547'1359","reported_seq":1381,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T23:33:10.155470+0000","last_change":"2023-09-23T23:32:51.927139+0000","last_active":"2023-09-23T23:33:10.155470+0000","last_peered":"2023-09-23T23:33:10.155470+0000","last_clean":"2023-09-23T23:33:10.155470+0000","last_became_active":"2023-09-23T23:32:51.926947+0000","last_became_peered":"2023-09-23T23:32:51.926947+0000","last_unstale":"2023-09-23T23:33:10.155470+0000","last_undegraded":"2023-09-23T23:33:10.155470+0000","last_fullsized":"2023-09-23T23:33:10.155470+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":1359,"log_dups_size":0,"ondisk_log_size":1359,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:29.860611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89063424,"num_objects":1359,"num_object_clones":0,"num_object_copies":4077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1359,"num_write_kb":86976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":559,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:07.992437+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:33:07.992437+0000","last_peered":"2023-09-23T23:33:07.992437+0000","last_clean":"2023-09-23T23:33:07.992437+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:33:07.992437+0000","last_undegraded":"2023-09-23T23:33:07.992437+0000","last_fullsized":"2023-09-23T23:33:07.992437+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":905,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.993144+0000","last_change":"2023-09-23T23:33:04.704192+0000","last_active":"2023-09-23T23:33:05.993144+0000","last_peered":"2023-09-23T23:33:05.993144+0000","last_clean":"2023-09-23T23:33:05.993144+0000","last_became_active":"2023-09-23T23:33:04.703763+0000","last_became_peered":"2023-09-23T23:33:04.703763+0000","last_unstale":"2023-09-23T23:33:05.993144+0000","last_undegraded":"2023-09-23T23:33:05.993144+0000","last_fullsized":"2023-09-23T23:33:05.993144+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:45.012595+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":506,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.002057+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:33:08.002057+0000","last_peered":"2023-09-23T23:33:08.002057+0000","last_clean":"2023-09-23T23:33:08.002057+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:33:08.002057+0000","last_undegraded":"2023-09-23T23:33:08.002057+0000","last_fullsized":"2023-09-23T23:33:08.002057+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":59,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.998805+0000","last_change":"2023-09-23T23:29:54.998861+0000","last_active":"2023-09-23T23:33:05.998805+0000","last_peered":"2023-09-23T23:33:05.998805+0000","last_clean":"2023-09-23T23:33:05.998805+0000","last_became_active":"2023-09-23T23:29:54.998550+0000","last_became_peered":"2023-09-23T23:29:54.998550+0000","last_unstale":"2023-09-23T23:33:05.998805+0000","last_undegraded":"2023-09-23T23:33:05.998805+0000","last_fullsized":"2023-09-23T23:33:05.998805+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:26.515992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"545'1230","reported_seq":1249,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.213346+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:33:08.213346+0000","last_peered":"2023-09-23T23:33:08.213346+0000","last_clean":"2023-09-23T23:33:08.213346+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:33:08.213346+0000","last_undegraded":"2023-09-23T23:33:08.213346+0000","last_fullsized":"2023-09-23T23:33:08.213346+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":1230,"log_dups_size":0,"ondisk_log_size":1230,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80609280,"num_objects":1230,"num_object_clones":0,"num_object_copies":3690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1230,"num_write_kb":78720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":730,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T23:33:10.042848+0000","last_change":"2023-09-23T23:29:58.254396+0000","last_active":"2023-09-23T23:33:10.042848+0000","last_peered":"2023-09-23T23:33:10.042848+0000","last_clean":"2023-09-23T23:33:10.042848+0000","last_became_active":"2023-09-23T23:29:58.254127+0000","last_became_peered":"2023-09-23T23:29:58.254127+0000","last_unstale":"2023-09-23T23:33:10.042848+0000","last_undegraded":"2023-09-23T23:33:10.042848+0000","last_fullsized":"2023-09-23T23:33:10.042848+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:48.863625+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":472,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.992861+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:33:05.992861+0000","last_peered":"2023-09-23T23:33:05.992861+0000","last_clean":"2023-09-23T23:33:05.992861+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:33:05.992861+0000","last_undegraded":"2023-09-23T23:33:05.992861+0000","last_fullsized":"2023-09-23T23:33:05.992861+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":471,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.991066+0000","last_change":"2023-09-23T23:28:22.601436+0000","last_active":"2023-09-23T23:33:05.991066+0000","last_peered":"2023-09-23T23:33:05.991066+0000","last_clean":"2023-09-23T23:33:05.991066+0000","last_became_active":"2023-09-23T23:28:22.601154+0000","last_became_peered":"2023-09-23T23:28:22.601154+0000","last_unstale":"2023-09-23T23:33:05.991066+0000","last_undegraded":"2023-09-23T23:33:05.991066+0000","last_fullsized":"2023-09-23T23:33:05.991066+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:03.296778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":359,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.019673+0000","last_change":"2023-09-23T23:33:01.953646+0000","last_active":"2023-09-23T23:33:09.019673+0000","last_peered":"2023-09-23T23:33:09.019673+0000","last_clean":"2023-09-23T23:33:09.019673+0000","last_became_active":"2023-09-23T23:33:01.952920+0000","last_became_peered":"2023-09-23T23:33:01.952920+0000","last_unstale":"2023-09-23T23:33:09.019673+0000","last_undegraded":"2023-09-23T23:33:09.019673+0000","last_fullsized":"2023-09-23T23:33:09.019673+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:56.358346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"546'1320","reported_seq":1341,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.704131+0000","last_change":"2023-09-23T23:32:51.927217+0000","last_active":"2023-09-23T23:33:09.704131+0000","last_peered":"2023-09-23T23:33:09.704131+0000","last_clean":"2023-09-23T23:33:09.704131+0000","last_became_active":"2023-09-23T23:32:51.927016+0000","last_became_peered":"2023-09-23T23:32:51.927016+0000","last_unstale":"2023-09-23T23:33:09.704131+0000","last_undegraded":"2023-09-23T23:33:09.704131+0000","last_fullsized":"2023-09-23T23:33:09.704131+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":1320,"log_dups_size":0,"ondisk_log_size":1320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:12.763576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86507520,"num_objects":1320,"num_object_clones":0,"num_object_copies":3960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1320,"num_write_kb":84480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":500,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.998924+0000","last_change":"2023-09-23T23:33:04.705269+0000","last_active":"2023-09-23T23:33:05.998924+0000","last_peered":"2023-09-23T23:33:05.998924+0000","last_clean":"2023-09-23T23:33:05.998924+0000","last_became_active":"2023-09-23T23:33:04.704933+0000","last_became_peered":"2023-09-23T23:33:04.704933+0000","last_unstale":"2023-09-23T23:33:05.998924+0000","last_undegraded":"2023-09-23T23:33:05.998924+0000","last_fullsized":"2023-09-23T23:33:05.998924+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:34.679767+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":406,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.993397+0000","last_change":"2023-09-23T23:33:01.953282+0000","last_active":"2023-09-23T23:33:05.993397+0000","last_peered":"2023-09-23T23:33:05.993397+0000","last_clean":"2023-09-23T23:33:05.993397+0000","last_became_active":"2023-09-23T23:33:01.952964+0000","last_became_peered":"2023-09-23T23:33:01.952964+0000","last_unstale":"2023-09-23T23:33:05.993397+0000","last_undegraded":"2023-09-23T23:33:05.993397+0000","last_fullsized":"2023-09-23T23:33:05.993397+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:23.201663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":251,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.019624+0000","last_change":"2023-09-23T23:26:32.037099+0000","last_active":"2023-09-23T23:33:09.019624+0000","last_peered":"2023-09-23T23:33:09.019624+0000","last_clean":"2023-09-23T23:33:09.019624+0000","last_became_active":"2023-09-23T23:26:32.036681+0000","last_became_peered":"2023-09-23T23:26:32.036681+0000","last_unstale":"2023-09-23T23:33:09.019624+0000","last_undegraded":"2023-09-23T23:33:09.019624+0000","last_fullsized":"2023-09-23T23:33:09.019624+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:48.926576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":278,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.989779+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:33:05.989779+0000","last_peered":"2023-09-23T23:33:05.989779+0000","last_clean":"2023-09-23T23:33:05.989779+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:33:05.989779+0000","last_undegraded":"2023-09-23T23:33:05.989779+0000","last_fullsized":"2023-09-23T23:33:05.989779+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"545'1282","reported_seq":1301,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.209952+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:33:08.209952+0000","last_peered":"2023-09-23T23:33:08.209952+0000","last_clean":"2023-09-23T23:33:08.209952+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:33:08.209952+0000","last_undegraded":"2023-09-23T23:33:08.209952+0000","last_fullsized":"2023-09-23T23:33:08.209952+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":1282,"log_dups_size":0,"ondisk_log_size":1282,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84017152,"num_objects":1282,"num_object_clones":0,"num_object_copies":3846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1282,"num_write_kb":82048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":876,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.991833+0000","last_change":"2023-09-23T23:29:58.258327+0000","last_active":"2023-09-23T23:33:05.991833+0000","last_peered":"2023-09-23T23:33:05.991833+0000","last_clean":"2023-09-23T23:33:05.991833+0000","last_became_active":"2023-09-23T23:29:58.257930+0000","last_became_peered":"2023-09-23T23:29:58.257930+0000","last_unstale":"2023-09-23T23:33:05.991833+0000","last_undegraded":"2023-09-23T23:33:05.991833+0000","last_fullsized":"2023-09-23T23:33:05.991833+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:00.144157+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":542,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.004230+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:33:08.004230+0000","last_peered":"2023-09-23T23:33:08.004230+0000","last_clean":"2023-09-23T23:33:08.004230+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:33:08.004230+0000","last_undegraded":"2023-09-23T23:33:08.004230+0000","last_fullsized":"2023-09-23T23:33:08.004230+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":144,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.019553+0000","last_change":"2023-09-23T23:29:54.999590+0000","last_active":"2023-09-23T23:33:09.019553+0000","last_peered":"2023-09-23T23:33:09.019553+0000","last_clean":"2023-09-23T23:33:09.019553+0000","last_became_active":"2023-09-23T23:29:54.999209+0000","last_became_peered":"2023-09-23T23:29:54.999209+0000","last_unstale":"2023-09-23T23:33:09.019553+0000","last_undegraded":"2023-09-23T23:33:09.019553+0000","last_fullsized":"2023-09-23T23:33:09.019553+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:32.208030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":87,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:07.992238+0000","last_change":"2023-09-23T23:33:01.954301+0000","last_active":"2023-09-23T23:33:07.992238+0000","last_peered":"2023-09-23T23:33:07.992238+0000","last_clean":"2023-09-23T23:33:07.992238+0000","last_became_active":"2023-09-23T23:33:01.953960+0000","last_became_peered":"2023-09-23T23:33:01.953960+0000","last_unstale":"2023-09-23T23:33:07.992238+0000","last_undegraded":"2023-09-23T23:33:07.992238+0000","last_fullsized":"2023-09-23T23:33:07.992238+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:56.310561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"546'1271","reported_seq":1292,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.675516+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:33:09.675516+0000","last_peered":"2023-09-23T23:33:09.675516+0000","last_clean":"2023-09-23T23:33:09.675516+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:33:09.675516+0000","last_undegraded":"2023-09-23T23:33:09.675516+0000","last_fullsized":"2023-09-23T23:33:09.675516+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":1271,"log_dups_size":0,"ondisk_log_size":1271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83296256,"num_objects":1271,"num_object_clones":0,"num_object_copies":3813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1271,"num_write_kb":81344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":931,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.001882+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:33:08.001882+0000","last_peered":"2023-09-23T23:33:08.001882+0000","last_clean":"2023-09-23T23:33:08.001882+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:33:08.001882+0000","last_undegraded":"2023-09-23T23:33:08.001882+0000","last_fullsized":"2023-09-23T23:33:08.001882+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":59,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.999406+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:33:05.999406+0000","last_peered":"2023-09-23T23:33:05.999406+0000","last_clean":"2023-09-23T23:33:05.999406+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:33:05.999406+0000","last_undegraded":"2023-09-23T23:33:05.999406+0000","last_fullsized":"2023-09-23T23:33:05.999406+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":314,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.019717+0000","last_change":"2023-09-23T23:33:01.960385+0000","last_active":"2023-09-23T23:33:09.019717+0000","last_peered":"2023-09-23T23:33:09.019717+0000","last_clean":"2023-09-23T23:33:09.019717+0000","last_became_active":"2023-09-23T23:33:01.953339+0000","last_became_peered":"2023-09-23T23:33:01.953339+0000","last_unstale":"2023-09-23T23:33:09.019717+0000","last_undegraded":"2023-09-23T23:33:09.019717+0000","last_fullsized":"2023-09-23T23:33:09.019717+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:29.776228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":176,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.991443+0000","last_change":"2023-09-23T23:29:55.000912+0000","last_active":"2023-09-23T23:33:05.991443+0000","last_peered":"2023-09-23T23:33:05.991443+0000","last_clean":"2023-09-23T23:33:05.991443+0000","last_became_active":"2023-09-23T23:29:54.999122+0000","last_became_peered":"2023-09-23T23:29:54.999122+0000","last_unstale":"2023-09-23T23:33:05.991443+0000","last_undegraded":"2023-09-23T23:33:05.991443+0000","last_fullsized":"2023-09-23T23:33:05.991443+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:10.962943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":964,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.002127+0000","last_change":"2023-09-23T23:33:04.703982+0000","last_active":"2023-09-23T23:33:08.002127+0000","last_peered":"2023-09-23T23:33:08.002127+0000","last_clean":"2023-09-23T23:33:08.002127+0000","last_became_active":"2023-09-23T23:33:04.703619+0000","last_became_peered":"2023-09-23T23:33:04.703619+0000","last_unstale":"2023-09-23T23:33:08.002127+0000","last_undegraded":"2023-09-23T23:33:08.002127+0000","last_fullsized":"2023-09-23T23:33:08.002127+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:15.891154+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":225,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.988824+0000","last_change":"2023-09-23T23:33:01.954946+0000","last_active":"2023-09-23T23:33:05.988824+0000","last_peered":"2023-09-23T23:33:05.988824+0000","last_clean":"2023-09-23T23:33:05.988824+0000","last_became_active":"2023-09-23T23:33:01.954580+0000","last_became_peered":"2023-09-23T23:33:01.954580+0000","last_unstale":"2023-09-23T23:33:05.988824+0000","last_undegraded":"2023-09-23T23:33:05.988824+0000","last_fullsized":"2023-09-23T23:33:05.988824+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:21.793612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":300,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:07.996764+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:33:07.996764+0000","last_peered":"2023-09-23T23:33:07.996764+0000","last_clean":"2023-09-23T23:33:07.996764+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:33:07.996764+0000","last_undegraded":"2023-09-23T23:33:07.996764+0000","last_fullsized":"2023-09-23T23:33:07.996764+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":692,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.004369+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:33:08.004369+0000","last_peered":"2023-09-23T23:33:08.004369+0000","last_clean":"2023-09-23T23:33:08.004369+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:33:08.004369+0000","last_undegraded":"2023-09-23T23:33:08.004369+0000","last_fullsized":"2023-09-23T23:33:08.004369+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":286,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.991219+0000","last_change":"2023-09-23T23:22:38.452968+0000","last_active":"2023-09-23T23:33:05.991219+0000","last_peered":"2023-09-23T23:33:05.991219+0000","last_clean":"2023-09-23T23:33:05.991219+0000","last_became_active":"2023-09-23T23:22:38.437986+0000","last_became_peered":"2023-09-23T23:22:38.437986+0000","last_unstale":"2023-09-23T23:33:05.991219+0000","last_undegraded":"2023-09-23T23:33:05.991219+0000","last_fullsized":"2023-09-23T23:33:05.991219+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:10.333271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":176,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.988613+0000","last_change":"2023-09-23T23:27:20.558454+0000","last_active":"2023-09-23T23:33:05.988613+0000","last_peered":"2023-09-23T23:33:05.988613+0000","last_clean":"2023-09-23T23:33:05.988613+0000","last_became_active":"2023-09-23T23:27:20.558204+0000","last_became_peered":"2023-09-23T23:27:20.558204+0000","last_unstale":"2023-09-23T23:33:05.988613+0000","last_undegraded":"2023-09-23T23:33:05.988613+0000","last_fullsized":"2023-09-23T23:33:05.988613+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:15.079990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":300,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.991901+0000","last_change":"2023-09-23T23:29:55.022469+0000","last_active":"2023-09-23T23:33:05.991901+0000","last_peered":"2023-09-23T23:33:05.991901+0000","last_clean":"2023-09-23T23:33:05.991901+0000","last_became_active":"2023-09-23T23:29:55.003082+0000","last_became_peered":"2023-09-23T23:29:55.003082+0000","last_unstale":"2023-09-23T23:33:05.991901+0000","last_undegraded":"2023-09-23T23:33:05.991901+0000","last_fullsized":"2023-09-23T23:33:05.991901+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:15.791643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":166,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.991172+0000","last_change":"2023-09-23T23:26:11.527453+0000","last_active":"2023-09-23T23:33:05.991172+0000","last_peered":"2023-09-23T23:33:05.991172+0000","last_clean":"2023-09-23T23:33:05.991172+0000","last_became_active":"2023-09-23T23:26:11.527208+0000","last_became_peered":"2023-09-23T23:26:11.527208+0000","last_unstale":"2023-09-23T23:33:05.991172+0000","last_undegraded":"2023-09-23T23:33:05.991172+0000","last_fullsized":"2023-09-23T23:33:05.991172+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:15.083611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":831,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.994018+0000","last_change":"2023-09-23T23:29:58.252042+0000","last_active":"2023-09-23T23:33:05.994018+0000","last_peered":"2023-09-23T23:33:05.994018+0000","last_clean":"2023-09-23T23:33:05.994018+0000","last_became_active":"2023-09-23T23:29:58.251749+0000","last_became_peered":"2023-09-23T23:29:58.251749+0000","last_unstale":"2023-09-23T23:33:05.994018+0000","last_undegraded":"2023-09-23T23:33:05.994018+0000","last_fullsized":"2023-09-23T23:33:05.994018+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:39.324116+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":575,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.004260+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:33:08.004260+0000","last_peered":"2023-09-23T23:33:08.004260+0000","last_clean":"2023-09-23T23:33:08.004260+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:33:08.004260+0000","last_undegraded":"2023-09-23T23:33:08.004260+0000","last_fullsized":"2023-09-23T23:33:08.004260+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":559,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:08.002405+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:33:08.002405+0000","last_peered":"2023-09-23T23:33:08.002405+0000","last_clean":"2023-09-23T23:33:08.002405+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:33:08.002405+0000","last_undegraded":"2023-09-23T23:33:08.002405+0000","last_fullsized":"2023-09-23T23:33:08.002405+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":922,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:33:09.019872+0000","last_change":"2023-09-23T23:26:14.527390+0000","last_active":"2023-09-23T23:33:09.019872+0000","last_peered":"2023-09-23T23:33:09.019872+0000","last_clean":"2023-09-23T23:33:09.019872+0000","last_became_active":"2023-09-23T23:26:14.527008+0000","last_became_peered":"2023-09-23T23:26:14.527008+0000","last_unstale":"2023-09-23T23:33:09.019872+0000","last_undegraded":"2023-09-23T23:33:09.019872+0000","last_fullsized":"2023-09-23T23:33:09.019872+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:36.460827+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":373,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T23:33:07.995783+0000","last_change":"2023-09-23T23:33:01.962896+0000","last_active":"2023-09-23T23:33:07.995783+0000","last_peered":"2023-09-23T23:33:07.995783+0000","last_clean":"2023-09-23T23:33:07.995783+0000","last_became_active":"2023-09-23T23:33:01.951853+0000","last_became_peered":"2023-09-23T23:33:01.951853+0000","last_unstale":"2023-09-23T23:33:07.995783+0000","last_undegraded":"2023-09-23T23:33:07.995783+0000","last_fullsized":"2023-09-23T23:33:07.995783+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:33.748007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":600,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T23:33:05.988338+0000","last_change":"2023-09-23T23:29:55.002051+0000","last_active":"2023-09-23T23:33:05.988338+0000","last_peered":"2023-09-23T23:33:05.988338+0000","last_clean":"2023-09-23T23:33:05.988338+0000","last_became_active":"2023-09-23T23:29:55.001732+0000","last_became_peered":"2023-09-23T23:29:55.001732+0000","last_unstale":"2023-09-23T23:33:05.988338+0000","last_undegraded":"2023-09-23T23:33:05.988338+0000","last_fullsized":"2023-09-23T23:33:05.988338+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:28:01.353263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":1335754752,"num_objects":20382,"num_object_clones":0,"num_object_copies":61146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":134,"num_objects_unfound":0,"num_objects_dirty":20382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20382,"num_write_kb":1304448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1097,"num_bytes_recovered":71892992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4112515072,"data_stored":4112515072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":20382,"ondisk_log_size":20382,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":10988,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782644,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":808296,"kb_used_data":509152,"kb_used_omap":8,"kb_used_meta":299127,"kb_avail":92936856,"statfs":{"total":95995035648,"available":95167340544,"internally_reserved":0,"allocated":521371648,"data_stored":519890010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":306306117},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:32:50 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":2.2989999999999999,"15min":2.2690000000000001},"min":{"1min":0.29999999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":17.140000000000001,"5min":44.094999999999999,"15min":85.721000000000004},"last":1.774},{"interface":"front","average":{"1min":1.954,"5min":2.3039999999999998,"15min":2.2519999999999998},"min":{"1min":0.32000000000000001,"5min":0.253,"15min":0.19800000000000001},"max":{"1min":11.069000000000001,"5min":44.335999999999999,"15min":85.965000000000003},"last":0.501}]},{"osd":1,"last update":"Sat Sep 23 23:32:50 2023","interfaces":[{"interface":"back","average":{"1min":2.5379999999999998,"5min":5.3979999999999997,"15min":2.7330000000000001},"min":{"1min":0.27600000000000002,"5min":0.23200000000000001,"15min":0.22900000000000001},"max":{"1min":10.054,"5min":198.58000000000001,"15min":198.58000000000001},"last":0.64900000000000002},{"interface":"front","average":{"1min":1.9530000000000001,"5min":5.3899999999999997,"15min":2.8980000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.246},"max":{"1min":7.5449999999999999,"5min":198.71100000000001,"15min":198.71100000000001},"last":0.54400000000000004}]},{"osd":2,"last update":"Sat Sep 23 23:32:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":6.5049999999999999,"15min":3.5379999999999998},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":8.1440000000000001,"5min":203.58000000000001,"15min":203.58000000000001},"last":0.59599999999999997},{"interface":"front","average":{"1min":2.2280000000000002,"5min":6.0439999999999996,"15min":3.0459999999999998},"min":{"1min":0.30599999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.0719999999999992,"5min":198.69,"15min":198.69},"last":0.75900000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:32:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":6.3159999999999998,"15min":3.0960000000000001},"min":{"1min":0.32500000000000001,"5min":0.30499999999999999,"15min":0.26700000000000002},"max":{"1min":6.1139999999999999,"5min":279.26499999999999,"15min":279.26499999999999},"last":0.93600000000000005},{"interface":"front","average":{"1min":1.819,"5min":7.6150000000000002,"15min":3.4689999999999999},"min":{"1min":0.373,"5min":0.316,"15min":0.29599999999999999},"max":{"1min":5.8540000000000001,"5min":279.29899999999998,"15min":279.29899999999998},"last":1.917}]},{"osd":4,"last update":"Sat Sep 23 23:32:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":0.88700000000000001,"15min":0.82399999999999995},"min":{"1min":0.25700000000000001,"5min":0.246,"15min":0.246},"max":{"1min":9.9819999999999993,"5min":9.9819999999999993,"15min":9.9819999999999993},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.77700000000000002,"15min":0.83999999999999997},"min":{"1min":0.35799999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.3360000000000003,"5min":7.9189999999999996,"15min":7.9189999999999996},"last":1.6399999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:32:50 2023","interfaces":[{"interface":"back","average":{"1min":1.274,"5min":3.1059999999999999,"15min":1.698},"min":{"1min":0.36599999999999999,"5min":0.27500000000000002,"15min":0.26500000000000001},"max":{"1min":6.4050000000000002,"5min":210.83600000000001,"15min":210.83600000000001},"last":0.998},{"interface":"front","average":{"1min":0.50900000000000001,"5min":2.7949999999999999,"15min":1.464},"min":{"1min":0.36199999999999999,"5min":0.28699999999999998,"15min":0.28199999999999997},"max":{"1min":1.2609999999999999,"5min":207.04599999999999,"15min":207.04599999999999},"last":2.9079999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:32:50 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.76800000000000002,"15min":0.96399999999999997},"min":{"1min":0.34699999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.577,"5min":4.758,"15min":62.213000000000001},"last":0.81100000000000005},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.0589999999999999,"15min":1.2250000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.33300000000000002},"max":{"1min":9.9239999999999995,"5min":9.9239999999999995,"15min":62.805999999999997},"last":0.48799999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496121,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1065700,"kb_used_data":407708,"kb_used_omap":0,"kb_used_meta":657984,"kb_avail":92679452,"statfs":{"total":95995035648,"available":94903758848,"internally_reserved":0,"allocated":417492992,"data_stored":416023267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":673775616},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:32:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3620000000000001,"5min":3.081,"15min":6.6680000000000001},"min":{"1min":0.251,"5min":0.223,"15min":0.20399999999999999},"max":{"1min":8.8119999999999994,"5min":135.31999999999999,"15min":161.64599999999999},"last":0.436},{"interface":"front","average":{"1min":0.97599999999999998,"5min":3.339,"15min":6.4199999999999999},"min":{"1min":0.23999999999999999,"5min":0.23000000000000001,"15min":0.22900000000000001},"max":{"1min":3.2269999999999999,"5min":135.34299999999999,"15min":161.38499999999999},"last":0.34899999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:32:08 2023","interfaces":[{"interface":"back","average":{"1min":4.7359999999999998,"5min":3.9409999999999998,"15min":3.6749999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.245},"max":{"1min":65.891000000000005,"5min":89.605000000000004,"15min":210.869},"last":0.318},{"interface":"front","average":{"1min":4.3170000000000002,"5min":4.1980000000000004,"15min":3.7989999999999999},"min":{"1min":0.27200000000000002,"5min":0.249,"15min":0.23300000000000001},"max":{"1min":66.087000000000003,"5min":91.025999999999996,"15min":210.89699999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:33:00 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":3.806,"15min":3.847},"min":{"1min":0.36199999999999999,"5min":0.28399999999999997,"15min":0.27800000000000002},"max":{"1min":3.2639999999999998,"5min":97.408000000000001,"15min":127.44499999999999},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.94799999999999995,"5min":4.8029999999999999,"15min":4.2359999999999998},"min":{"1min":0.35699999999999998,"5min":0.29999999999999999,"15min":0.26500000000000001},"max":{"1min":3.0870000000000002,"5min":125.539,"15min":125.539},"last":0.84699999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:32:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":2.5960000000000001,"15min":3.1030000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.2430000000000003,"5min":138.87299999999999,"15min":138.87299999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":1.2889999999999999,"5min":2.698,"15min":3.4260000000000002},"min":{"1min":0.371,"5min":0.32700000000000001,"15min":0.28100000000000003},"max":{"1min":4.8360000000000003,"5min":138.976,"15min":138.976},"last":0.879}]},{"osd":4,"last update":"Sat Sep 23 23:32:20 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.70699999999999996,"15min":1.286},"min":{"1min":0.247,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":2.0299999999999998,"5min":8.1999999999999993,"15min":166.05199999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.79000000000000004,"15min":1.619},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27300000000000002},"max":{"1min":2.9980000000000002,"5min":7.915,"15min":166.631},"last":0.52900000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:32:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.92900000000000005,"15min":1.1890000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.27900000000000003},"max":{"1min":5.915,"5min":11.473000000000001,"15min":91.899000000000001},"last":0.496},{"interface":"front","average":{"1min":1.4330000000000001,"5min":0.91200000000000003,"15min":1.302},"min":{"1min":0.35699999999999998,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":5.3490000000000002,"5min":7.5670000000000002,"15min":94.174000000000007},"last":0.56499999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:33:00 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.94099999999999995,"15min":0.89800000000000002},"min":{"1min":0.378,"5min":0.32500000000000001,"15min":0.29599999999999999},"max":{"1min":1.9219999999999999,"5min":12.208,"15min":12.208},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.57099999999999995,"5min":1.1459999999999999,"15min":0.92000000000000004},"min":{"1min":0.35499999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.393,"5min":11.515000000000001,"15min":11.515000000000001},"last":0.64200000000000002}]}]},{"osd":1,"up_from":112,"seq":481036337462,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1467088,"kb_used_data":642632,"kb_used_omap":4,"kb_used_meta":824443,"kb_avail":92278064,"statfs":{"total":95995035648,"available":94492737536,"internally_reserved":0,"allocated":658055168,"data_stored":655497455,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":844230542},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:32:11 2023","interfaces":[{"interface":"back","average":{"1min":2.907,"5min":5.2539999999999996,"15min":4.04},"min":{"1min":0.17799999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":15.483000000000001,"5min":130.768,"15min":130.768},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.5179999999999998,"5min":4.4169999999999998,"15min":3.5619999999999998},"min":{"1min":0.218,"5min":0.19,"15min":0.19},"max":{"1min":15.699,"5min":134.929,"15min":134.929},"last":0.40200000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8180000000000001,"5min":10.832000000000001,"15min":6.0149999999999997},"min":{"1min":0.26100000000000001,"5min":0.214,"15min":0.20100000000000001},"max":{"1min":9.6329999999999991,"5min":231.67400000000001,"15min":231.67400000000001},"last":0.437},{"interface":"front","average":{"1min":3.3599999999999999,"5min":11.005000000000001,"15min":6.3109999999999999},"min":{"1min":0.23499999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":24.594999999999999,"5min":232.21000000000001,"15min":232.21000000000001},"last":0.54100000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":7.2629999999999999,"15min":4.9939999999999998},"min":{"1min":0.25900000000000001,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":9.7949999999999999,"5min":234.404,"15min":234.404},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.833,"5min":7.1459999999999999,"15min":4.9169999999999998},"min":{"1min":0.33500000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":14.522,"5min":236.33699999999999,"15min":236.33699999999999},"last":0.36499999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:32:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5840000000000001,"5min":1.724,"15min":1.655},"min":{"1min":0.34399999999999997,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":12.042999999999999,"5min":12.042999999999999,"15min":18.474},"last":0.41399999999999998},{"interface":"front","average":{"1min":2.1240000000000001,"5min":1.6890000000000001,"15min":1.6819999999999999},"min":{"1min":0.315,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.3239999999999998,"5min":18.108000000000001,"15min":18.108000000000001},"last":0.38700000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:32:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9770000000000001,"5min":1.484,"15min":2.9399999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.0190000000000001,"5min":9.0229999999999997,"15min":250.33000000000001},"last":0.497},{"interface":"front","average":{"1min":2.6909999999999998,"5min":1.7370000000000001,"15min":3.2679999999999998},"min":{"1min":0.38,"5min":0.34499999999999997,"15min":0.308},"max":{"1min":12.007999999999999,"5min":12.007999999999999,"15min":250.21199999999999},"last":0.47099999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:32:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2080000000000002,"5min":1.5469999999999999,"15min":2.3250000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":5.601,"5min":7.4470000000000001,"15min":74.087999999999994},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.5790000000000002,"5min":1.8160000000000001,"15min":2.4159999999999999},"min":{"1min":0.40799999999999997,"5min":0.37,"15min":0.32800000000000001},"max":{"1min":11.927,"5min":11.927,"15min":73.974999999999994},"last":0.56899999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":1.599,"15min":3.153},"min":{"1min":0.42299999999999999,"5min":0.37,"15min":0.32900000000000001},"max":{"1min":9.5570000000000004,"5min":11.385,"15min":78.308000000000007},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.865,"5min":1.7629999999999999,"15min":2.964},"min":{"1min":0.435,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":11.977,"5min":11.977,"15min":77.204999999999998},"last":0.51100000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024286,"num_pgs":16,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1339632,"kb_used_data":169576,"kb_used_omap":5,"kb_used_meta":1170042,"kb_avail":92405520,"statfs":{"total":95995035648,"available":94623252480,"internally_reserved":0,"allocated":173645824,"data_stored":171059913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1198123642},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:32:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2629999999999999,"5min":11.031000000000001,"15min":6.1470000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.17499999999999999},"max":{"1min":14.115,"5min":213.45699999999999,"15min":213.45699999999999},"last":0.625},{"interface":"front","average":{"1min":2.4510000000000001,"5min":11.238,"15min":5.181},"min":{"1min":0.22500000000000001,"5min":0.186,"15min":0.17599999999999999},"max":{"1min":14.478,"5min":213.04900000000001,"15min":213.04900000000001},"last":0.56599999999999995}]},{"osd":2,"last update":"Sat Sep 23 23:32:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1459999999999999,"5min":5.694,"15min":5.8209999999999997},"min":{"1min":0.214,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":14.970000000000001,"5min":220.779,"15min":220.779},"last":0.76300000000000001},{"interface":"front","average":{"1min":1.8919999999999999,"5min":6.2309999999999999,"15min":5.9459999999999997},"min":{"1min":0.251,"5min":0.19800000000000001,"15min":0.17599999999999999},"max":{"1min":15.140000000000001,"5min":218.74700000000001,"15min":218.74700000000001},"last":3.516}]},{"osd":3,"last update":"Sat Sep 23 23:32:24 2023","interfaces":[{"interface":"back","average":{"1min":8.1720000000000006,"5min":7.9409999999999998,"15min":5.2629999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":114.381,"5min":220.732,"15min":220.732},"last":0.436},{"interface":"front","average":{"1min":9.0609999999999999,"5min":8.4740000000000002,"15min":5.2809999999999997},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":114.30500000000001,"5min":218.78200000000001,"15min":218.78200000000001},"last":0.46300000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":2.7120000000000002,"15min":3.0859999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.27000000000000002},"max":{"1min":9.3140000000000001,"5min":108.01900000000001,"15min":138.81299999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.86799999999999999,"5min":2.601,"15min":3.6240000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.1539999999999999,"5min":107.898,"15min":171.251},"last":0.93500000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":1.399,"15min":2.8969999999999998},"min":{"1min":0.33500000000000002,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":6.1870000000000003,"5min":9.8230000000000004,"15min":171.30699999999999},"last":0.97199999999999998},{"interface":"front","average":{"1min":0.72599999999999998,"5min":1.165,"15min":2.1480000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":2.1459999999999999,"5min":9.9000000000000004,"15min":139.40799999999999},"last":0.70299999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":2.5510000000000002,"15min":3.452},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.29499999999999998},"max":{"1min":6.2320000000000002,"5min":108.67,"15min":171.33600000000001},"last":1.004},{"interface":"front","average":{"1min":0.748,"5min":2.3719999999999999,"15min":3.6379999999999999},"min":{"1min":0.35199999999999998,"5min":0.34899999999999998,"15min":0.27100000000000002},"max":{"1min":2.1720000000000002,"5min":108.804,"15min":184.13499999999999},"last":0.72799999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:32:51 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":2.8260000000000001,"15min":2.6440000000000001},"min":{"1min":0.35499999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.0150000000000006,"5min":111.71299999999999,"15min":138.77699999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.98699999999999999,"5min":2.5470000000000002,"15min":2.367},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":3.347,"5min":111.108,"15min":139.37899999999999},"last":0.67000000000000004}]}]},{"osd":2,"up_from":199,"seq":854698492107,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1588800,"kb_used_data":839032,"kb_used_omap":7,"kb_used_meta":749752,"kb_avail":92156352,"statfs":{"total":95995035648,"available":94368104448,"internally_reserved":0,"allocated":859168768,"data_stored":856565224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":767746129},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":2.798,"15min":4.0899999999999999},"min":{"1min":0.218,"5min":0.21299999999999999,"15min":0.16700000000000001},"max":{"1min":11.353,"5min":58.802,"15min":119.277},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.7549999999999999,"5min":2.4049999999999998,"15min":3.8999999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.18099999999999999},"max":{"1min":7.9180000000000001,"5min":58.115000000000002,"15min":119.56100000000001},"last":0.69699999999999995}]},{"osd":1,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.6970000000000001,"5min":3.5960000000000001,"15min":3.6699999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":7.6479999999999997,"5min":82.876000000000005,"15min":157.98699999999999},"last":0.24099999999999999},{"interface":"front","average":{"1min":1.71,"5min":3.8610000000000002,"15min":3.8130000000000002},"min":{"1min":0.27000000000000002,"5min":0.24099999999999999,"15min":0.22600000000000001},"max":{"1min":11.278,"5min":82.909000000000006,"15min":158.03800000000001},"last":0.71099999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.954,"5min":3.3450000000000002,"15min":3.7589999999999999},"min":{"1min":0.311,"5min":0.24299999999999999,"15min":0.191},"max":{"1min":10.451000000000001,"5min":103.227,"15min":119.366},"last":0.27300000000000002},{"interface":"front","average":{"1min":2.2999999999999998,"5min":3.359,"15min":3.3450000000000002},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":11.324999999999999,"5min":102.693,"15min":102.693},"last":0.67300000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.6359999999999999,"15min":3.863},"min":{"1min":0.33700000000000002,"5min":0.27700000000000002,"15min":0.25},"max":{"1min":4.5739999999999998,"5min":11.586,"15min":332.98099999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":1.3819999999999999,"5min":1.7549999999999999,"15min":3.3319999999999999},"min":{"1min":0.42899999999999999,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":6.1189999999999998,"5min":11.621,"15min":332.96199999999999},"last":0.35599999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":1.5309999999999999,"15min":2.2170000000000001},"min":{"1min":0.34300000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.6399999999999997,"5min":6.2990000000000004,"15min":119.919},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.242,"5min":1.5529999999999999,"15min":2.1400000000000001},"min":{"1min":0.41099999999999998,"5min":0.33900000000000002,"15min":0.33000000000000002},"max":{"1min":3.827,"5min":7.7910000000000004,"15min":119.306},"last":0.46700000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.8640000000000001,"15min":4.1319999999999997},"min":{"1min":0.38200000000000001,"5min":0.315,"15min":0.31},"max":{"1min":4.6150000000000002,"5min":14.795999999999999,"15min":349.32799999999997},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.339,"5min":1.9430000000000001,"15min":3.6629999999999998},"min":{"1min":0.38900000000000001,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":3.8079999999999998,"5min":12.496,"15min":349.17700000000002},"last":0.99099999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.9590000000000001,"5min":1.9830000000000001,"15min":1.6879999999999999},"min":{"1min":0.39600000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":11.252000000000001,"5min":15.734999999999999,"15min":20.350000000000001},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.984,"5min":1.9119999999999999,"15min":1.7310000000000001},"min":{"1min":0.41299999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":11.304,"5min":15.709,"15min":20.321000000000002},"last":0.57799999999999996}]}]},{"osd":3,"up_from":148,"seq":635655160076,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1390832,"kb_used_data":537576,"kb_used_omap":2,"kb_used_meta":853245,"kb_avail":92354320,"statfs":{"total":95995035648,"available":94570823680,"internally_reserved":0,"allocated":550477824,"data_stored":547940063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":873722933},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:32:40 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":2.1139999999999999,"15min":2.8410000000000002},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.158},"max":{"1min":4.782,"5min":22.271999999999998,"15min":103.866},"last":0.379},{"interface":"front","average":{"1min":2.8929999999999998,"5min":3.1859999999999999,"15min":3.0630000000000002},"min":{"1min":0.26400000000000001,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":22.754000000000001,"5min":24.683,"15min":101.884},"last":0.45900000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:32:40 2023","interfaces":[{"interface":"back","average":{"1min":8.782,"5min":4.8140000000000001,"15min":3.1960000000000002},"min":{"1min":0.27400000000000002,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":56.119,"5min":85.242999999999995,"15min":85.242999999999995},"last":0.48899999999999999},{"interface":"front","average":{"1min":6.4029999999999996,"5min":4.3399999999999999,"15min":2.8769999999999998},"min":{"1min":0.224,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":53.962000000000003,"5min":85.302000000000007,"15min":85.302000000000007},"last":0.439}]},{"osd":2,"last update":"Sat Sep 23 23:32:52 2023","interfaces":[{"interface":"back","average":{"1min":6.7460000000000004,"5min":6.8010000000000002,"15min":4.9139999999999997},"min":{"1min":0.32900000000000001,"5min":0.219,"15min":0.19800000000000001},"max":{"1min":56.226999999999997,"5min":84.724999999999994,"15min":101.279},"last":0.52200000000000002},{"interface":"front","average":{"1min":6.2850000000000001,"5min":6.5940000000000003,"15min":4.673},"min":{"1min":0.34899999999999998,"5min":0.25700000000000001,"15min":0.214},"max":{"1min":54.280999999999999,"5min":84.567999999999998,"15min":101.294},"last":0.41099999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:32:40 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":1.403,"15min":1.375},"min":{"1min":0.32900000000000001,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":5.8630000000000004,"5min":8.5869999999999997,"15min":14.754},"last":1.429},{"interface":"front","average":{"1min":2.0339999999999998,"5min":2.2519999999999998,"15min":1.8700000000000001},"min":{"1min":0.375,"5min":0.30599999999999999,"15min":0.26800000000000002},"max":{"1min":11.494999999999999,"5min":24.782,"15min":24.782},"last":0.88600000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:32:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":2.1640000000000001,"15min":1.796},"min":{"1min":0.40300000000000002,"5min":0.29599999999999999,"15min":0.28000000000000003},"max":{"1min":11.470000000000001,"5min":24.565000000000001,"15min":24.565000000000001},"last":0.58799999999999997},{"interface":"front","average":{"1min":2.3210000000000002,"5min":2.2229999999999999,"15min":1.9159999999999999},"min":{"1min":0.39500000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":11.702,"5min":22.472999999999999,"15min":22.472999999999999},"last":0.73599999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:32:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1659999999999999,"5min":2.3180000000000001,"15min":2.1040000000000001},"min":{"1min":0.45300000000000001,"5min":0.35199999999999998,"15min":0.31900000000000001},"max":{"1min":11.523,"5min":24.658000000000001,"15min":53.057000000000002},"last":0.69999999999999996},{"interface":"front","average":{"1min":2.7989999999999999,"5min":2.1379999999999999,"15min":2.1339999999999999},"min":{"1min":0.40899999999999997,"5min":0.36099999999999999,"15min":0.35299999999999998},"max":{"1min":18.956,"5min":18.956,"15min":53.088000000000001},"last":0.80800000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:32:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":3.73,"15min":1.9690000000000001},"min":{"1min":0.41199999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.0389999999999997,"5min":146.803,"15min":146.803},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.1699999999999999,"5min":4.556,"15min":2.5129999999999999},"min":{"1min":0.39200000000000002,"5min":0.38500000000000001,"15min":0.36699999999999999},"max":{"1min":3.6379999999999999,"5min":146.422,"15min":146.422},"last":0.84199999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019394,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1786544,"kb_used_data":1002984,"kb_used_omap":0,"kb_used_meta":783552,"kb_avail":91958608,"statfs":{"total":95995035648,"available":94165614592,"internally_reserved":0,"allocated":1027055616,"data_stored":1025573713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":802357248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:33:03 2023","interfaces":[{"interface":"back","average":{"1min":4.8780000000000001,"5min":3.3069999999999999,"15min":4.8869999999999996},"min":{"1min":0.23499999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":84.066999999999993,"5min":84.066999999999993,"15min":199.64400000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":4.6449999999999996,"5min":3.4620000000000002,"15min":5.1020000000000003},"min":{"1min":0.25600000000000001,"5min":0.24199999999999999,"15min":0.216},"max":{"1min":83.930000000000007,"5min":83.930000000000007,"15min":195.649},"last":0.26700000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:32:25 2023","interfaces":[{"interface":"back","average":{"1min":1.9510000000000001,"5min":5.7110000000000003,"15min":3.6269999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.23100000000000001},"max":{"1min":4.6139999999999999,"5min":198.27199999999999,"15min":198.27199999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.1160000000000001,"5min":5.508,"15min":3.7040000000000002},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.6989999999999998,"5min":196.47,"15min":196.47},"last":0.34300000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:32:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1930000000000001,"5min":9.1120000000000001,"15min":5.1630000000000003},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.9180000000000001,"5min":198.178,"15min":198.178},"last":57.499000000000002},{"interface":"front","average":{"1min":1.833,"5min":8.8339999999999996,"15min":5.0659999999999998},"min":{"1min":0.316,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.999000000000001,"5min":201.56,"15min":201.56},"last":57.374000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:32:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":7.7839999999999998,"15min":5.7779999999999996},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26300000000000001},"max":{"1min":7.6689999999999996,"5min":277.459,"15min":277.459},"last":53.256},{"interface":"front","average":{"1min":1.105,"5min":7.484,"15min":5.7039999999999997},"min":{"1min":0.32800000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.6870000000000003,"5min":277.20800000000003,"15min":277.20800000000003},"last":53.435000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:32:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":2.6469999999999998,"15min":2.004},"min":{"1min":0.26800000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.2750000000000004,"5min":161.86500000000001,"15min":161.86500000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.56799999999999995,"5min":2.4390000000000001,"15min":2.0270000000000001},"min":{"1min":0.35799999999999998,"5min":0.28399999999999997,"15min":0.249},"max":{"1min":2.3809999999999998,"5min":159.24199999999999,"15min":160.61099999999999},"last":0.30299999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:32:45 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.78800000000000003,"15min":1.0780000000000001},"min":{"1min":0.29899999999999999,"5min":0.27900000000000003,"15min":0.26000000000000001},"max":{"1min":2.9209999999999998,"5min":7.1130000000000004,"15min":90.346999999999994},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.876,"5min":0.94299999999999995,"15min":1.272},"min":{"1min":0.34200000000000003,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.1390000000000002,"5min":9.8079999999999998,"15min":90.733000000000004},"last":1.8480000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:32:45 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.2709999999999999,"15min":1.125},"min":{"1min":0.35799999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.0299999999999998,"5min":24.533999999999999,"15min":61.923999999999999},"last":1.177},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.0569999999999999,"15min":1.137},"min":{"1min":0.33100000000000002,"5min":0.316,"15min":0.316},"max":{"1min":2.8639999999999999,"5min":27.771999999999998,"15min":61.990000000000002},"last":0.42099999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757757,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1386080,"kb_used_data":724952,"kb_used_omap":0,"kb_used_meta":661120,"kb_avail":92359072,"statfs":{"total":95995035648,"available":94575689728,"internally_reserved":0,"allocated":742350848,"data_stored":740881352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":676986880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:33:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":1.4279999999999999,"15min":3.508},"min":{"1min":0.25900000000000001,"5min":0.219,"15min":0.19500000000000001},"max":{"1min":21.109000000000002,"5min":21.109000000000002,"15min":113.869},"last":0.316},{"interface":"front","average":{"1min":2.1459999999999999,"5min":1.3879999999999999,"15min":3.9209999999999998},"min":{"1min":0.30299999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":20.876999999999999,"5min":20.876999999999999,"15min":138.55199999999999},"last":0.98799999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:32:10 2023","interfaces":[{"interface":"back","average":{"1min":3.0979999999999999,"5min":1.95,"15min":2.8730000000000002},"min":{"1min":0.29699999999999999,"5min":0.25,"15min":0.23100000000000001},"max":{"1min":19.347000000000001,"5min":28.379999999999999,"15min":91.518000000000001},"last":1.5629999999999999},{"interface":"front","average":{"1min":3.472,"5min":2.1040000000000001,"15min":3.1280000000000001},"min":{"1min":0.28799999999999998,"5min":0.249,"15min":0.249},"max":{"1min":19.268000000000001,"5min":28.417000000000002,"15min":87.718000000000004},"last":1.536}]},{"osd":2,"last update":"Sat Sep 23 23:32:43 2023","interfaces":[{"interface":"back","average":{"1min":1.464,"5min":2.6419999999999999,"15min":3.0419999999999998},"min":{"1min":0.34799999999999998,"5min":0.27200000000000002,"15min":0.26900000000000002},"max":{"1min":3.6070000000000002,"5min":46.177,"15min":144.91200000000001},"last":1.591},{"interface":"front","average":{"1min":1.7649999999999999,"5min":2.7160000000000002,"15min":2.891},"min":{"1min":0.34200000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.468,"5min":45.863999999999997,"15min":145.15799999999999},"last":0.38300000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:32:36 2023","interfaces":[{"interface":"back","average":{"1min":1.49,"5min":3.7629999999999999,"15min":3.7240000000000002},"min":{"1min":0.37,"5min":0.32000000000000001,"15min":0.27400000000000002},"max":{"1min":3.4809999999999999,"5min":112.71899999999999,"15min":159.53299999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.675,"5min":3.8809999999999998,"15min":3.9390000000000001},"min":{"1min":0.38900000000000001,"5min":0.32700000000000001,"15min":0.312},"max":{"1min":3.4900000000000002,"5min":112.70099999999999,"15min":159.54900000000001},"last":1.097}]},{"osd":4,"last update":"Sat Sep 23 23:32:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.032,"15min":0.99099999999999999},"min":{"1min":0.25700000000000001,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":6.1310000000000002,"5min":19.962,"15min":19.962},"last":1.6160000000000001},{"interface":"front","average":{"1min":0.83999999999999997,"5min":0.89900000000000002,"15min":0.80200000000000005},"min":{"1min":0.30599999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.4080000000000004,"5min":20.829999999999998,"15min":20.829999999999998},"last":1.5149999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:32:22 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.76400000000000001,"15min":2.1739999999999999},"min":{"1min":0.33100000000000002,"5min":0.27600000000000002,"15min":0.26900000000000002},"max":{"1min":2.044,"5min":5.1299999999999999,"15min":310.68799999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":1.149,"5min":0.93799999999999994,"15min":2.3969999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.25900000000000001},"max":{"1min":3.3410000000000002,"5min":8.9979999999999993,"15min":311.45400000000001},"last":0.88900000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:32:43 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":3.008,"15min":3.1240000000000001},"min":{"1min":0.34599999999999997,"5min":0.317,"15min":0.317},"max":{"1min":2.6880000000000002,"5min":179.28800000000001,"15min":179.28800000000001},"last":3.137},{"interface":"front","average":{"1min":0.82499999999999996,"5min":3.1429999999999998,"15min":3.2469999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.2109999999999999,"5min":179.428,"15min":179.428},"last":0.93799999999999994}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":166592512,"data_stored":166592512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":447873024,"data_stored":447873024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":767688704,"data_stored":767688704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":424607744,"data_stored":424607744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":746323968,"data_stored":746323968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":736362496,"data_stored":736362496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":411435008,"data_stored":411435008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":411631616,"data_stored":411631616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:33:12.751 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_8 pgp_num 16 2023-09-23T23:33:12.798 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:12.845 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 15 16 24638 24622 102.533 100.75 0.00584553 0.00974799 2023-09-23T23:33:12.955 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:12.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:12.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:13.027 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:13 smithi110 ceph-mon[212939]: pgmap v1508: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 117 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s; 1/61218 objects degraded (0.002%); 134/61218 objects misplaced (0.219%); 15 MiB/s, 233 objects/s recovering 2023-09-23T23:33:13.027 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:13 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1/61218 objects degraded (0.002%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:33:13.059 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:33:13.122 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:13.276 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:33:13.291 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:13 smithi110 ceph-mon[221102]: pgmap v1508: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 117 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s; 1/61218 objects degraded (0.002%); 134/61218 objects misplaced (0.219%); 15 MiB/s, 233 objects/s recovering 2023-09-23T23:33:13.291 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:13 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1/61218 objects degraded (0.002%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:33:13.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:13.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:33:13.393 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:33:13.424 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:13.434 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:13 smithi138 ceph-mon[193606]: pgmap v1508: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 117 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s; 1/61218 objects degraded (0.002%); 134/61218 objects misplaced (0.219%); 15 MiB/s, 233 objects/s recovering 2023-09-23T23:33:13.435 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:13 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1/61218 objects degraded (0.002%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:33:13.440 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:13.527 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:13.784 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:33:13.793 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:33:13.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:13.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:33:13.845 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 16 16 26298 26282 102.609 103.75 0.00759326 0.00973834 2023-09-23T23:33:13.895 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:13.949 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:14.031 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:14 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:14.032 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:14 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:33:14.032 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:14 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/201608653' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:33:14.032 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:14 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:14.033 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:14 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:14.034 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:14 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:33:14.034 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:14 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/201608653' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:33:14.034 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:14 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:14.283 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:14.286 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:14 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:14.286 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:14 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:33:14.286 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:14 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/201608653' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:33:14.286 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:14 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:14.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:14.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:14.388 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:14.452 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:14.601 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:14.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:14.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:14.698 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:33:14.707 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:14.724 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:33:14.795 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:14.845 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 17 16 28044 28028 102.989 109.125 0.00766893 0.00970353 2023-09-23T23:33:14.950 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:14.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:14.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:15.031 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:15 smithi138 ceph-mon[193606]: pgmap v1509: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 117 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s; 1/64065 objects degraded (0.002%); 134/64065 objects misplaced (0.209%); 10 MiB/s, 159 objects/s recovering 2023-09-23T23:33:15.031 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:15 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:33:15.031 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:15 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/201608653' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:33:15.061 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:15.097 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:15.344 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:15 smithi138 ceph-mon[193606]: osdmap e549: 8 total, 8 up, 8 in 2023-09-23T23:33:15.345 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:15 smithi138 ceph-mon[193606]: failed to encode map e549 with expected crc 2023-09-23T23:33:15.444 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:15 smithi110 ceph-mon[221102]: pgmap v1509: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 117 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s; 1/64065 objects degraded (0.002%); 134/64065 objects misplaced (0.209%); 10 MiB/s, 159 objects/s recovering 2023-09-23T23:33:15.445 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:15 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:33:15.445 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:15 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/201608653' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:33:15.445 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:15 smithi110 ceph-mon[221102]: osdmap e549: 8 total, 8 up, 8 in 2023-09-23T23:33:15.445 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:15 smithi110 ceph-mon[221102]: failed to encode map e549 with expected crc 2023-09-23T23:33:15.445 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:15 smithi110 ceph-mon[212939]: pgmap v1509: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 117 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s; 1/64065 objects degraded (0.002%); 134/64065 objects misplaced (0.209%); 10 MiB/s, 159 objects/s recovering 2023-09-23T23:33:15.445 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:15 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:33:15.445 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:15 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/201608653' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:33:15.445 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:15 smithi110 ceph-mon[212939]: osdmap e549: 8 total, 8 up, 8 in 2023-09-23T23:33:15.445 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:15 smithi110 ceph-mon[212939]: failed to encode map e549 with expected crc 2023-09-23T23:33:15.845 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 18 16 29530 29514 102.427 92.875 0.00182065 0.00975392 2023-09-23T23:33:16.347 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:16 smithi110 ceph-mon[212939]: failed to encode map e549 with expected crc 2023-09-23T23:33:16.347 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:16 smithi110 ceph-mon[212939]: failed to encode map e549 with expected crc 2023-09-23T23:33:16.347 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:16 smithi110 ceph-mon[212939]: failed to encode map e549 with expected crc 2023-09-23T23:33:16.347 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:16 smithi110 ceph-mon[212939]: failed to encode map e549 with expected crc 2023-09-23T23:33:16.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:16 smithi110 ceph-mon[221102]: failed to encode map e549 with expected crc 2023-09-23T23:33:16.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:16 smithi110 ceph-mon[221102]: failed to encode map e549 with expected crc 2023-09-23T23:33:16.348 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:16 smithi110 ceph-mon[221102]: failed to encode map e549 with expected crc 2023-09-23T23:33:16.348 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:16 smithi110 ceph-mon[221102]: failed to encode map e549 with expected crc 2023-09-23T23:33:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:16 smithi138 ceph-mon[193606]: failed to encode map e549 with expected crc 2023-09-23T23:33:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:16 smithi138 ceph-mon[193606]: failed to encode map e549 with expected crc 2023-09-23T23:33:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:16 smithi138 ceph-mon[193606]: failed to encode map e549 with expected crc 2023-09-23T23:33:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:16 smithi138 ceph-mon[193606]: failed to encode map e549 with expected crc 2023-09-23T23:33:16.822 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:16.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:16.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:16.845 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 19 16 31055 31039 102.052 95.3125 0.0058077 0.00979377 2023-09-23T23:33:16.932 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:17.001 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:17.179 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:17 smithi110 ceph-mon[212939]: pgmap v1511: 119 pgs: 119 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:33:17.179 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:17 smithi110 ceph-mon[212939]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/64065 objects degraded (0.002%), 1 pg degraded) 2023-09-23T23:33:17.179 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:17 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:33:17.179 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:33:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:33:16] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2023-09-23T23:33:17.179 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:17 smithi110 ceph-mon[221102]: pgmap v1511: 119 pgs: 119 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:33:17.179 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:17 smithi110 ceph-mon[221102]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/64065 objects degraded (0.002%), 1 pg degraded) 2023-09-23T23:33:17.179 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:17 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:33:17.253 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:17.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:17.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:17.359 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:33:17.417 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:17 smithi138 ceph-mon[193606]: pgmap v1511: 119 pgs: 119 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:33:17.417 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:17 smithi138 ceph-mon[193606]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/64065 objects degraded (0.002%), 1 pg degraded) 2023-09-23T23:33:17.417 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:17 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:33:17.429 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:17.659 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:33:17.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:17.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:33:17.764 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:33:17.832 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:17.846 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:33:17.844944+0000 min lat: 0.00107054 max lat: 2.03043 avg lat: 0.00976634 2023-09-23T23:33:17.846 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:33:17.846 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 20 16 32772 32756 102.315 107.312 0.0107578 0.00976634 2023-09-23T23:33:18.032 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:33:18.069 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:33:18.075 INFO:teuthology.orchestra.run.smithi110.stderr:set pool 13 pgp_num to 16 2023-09-23T23:33:18.099 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:33:18.133 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:33:18.133 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2316019596' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T23:33:18.133 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/424694687' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:18.166 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:33:18.209 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:33:18.245 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:33:18.282 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:33:18.285 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:33:18.285 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:33:18.285 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:33:18.285 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:33:18.285 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:33:18.285 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:33:18.285 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:33:18.286 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:33:18.306 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:33:18.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:18.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:33:18.328 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2316019596' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T23:33:18.328 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/424694687' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:18.328 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2316019596' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T23:33:18.329 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/424694687' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:18.418 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:33:18.487 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:18.583 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:33:18.583 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:33:18.617 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:33:18.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:18.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:33:18.725 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:33:18.771 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:18.846 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 21 16 34356 34340 102.156 99 0.00738722 0.0097832 2023-09-23T23:33:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:19 smithi138 ceph-mon[193606]: pgmap v1512: 119 pgs: 119 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s; 1.1 MiB/s, 17 objects/s recovering 2023-09-23T23:33:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:19 smithi138 ceph-mon[193606]: failed to encode map e550 with expected crc 2023-09-23T23:33:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:19 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2316019596' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T23:33:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:19 smithi138 ceph-mon[193606]: osdmap e550: 8 total, 8 up, 8 in 2023-09-23T23:33:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:19 smithi138 ceph-mon[193606]: failed to encode map e550 with expected crc 2023-09-23T23:33:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:19 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:19 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:19.210 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:33:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:19 smithi110 ceph-mon[221102]: pgmap v1512: 119 pgs: 119 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s; 1.1 MiB/s, 17 objects/s recovering 2023-09-23T23:33:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:19 smithi110 ceph-mon[221102]: failed to encode map e550 with expected crc 2023-09-23T23:33:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:19 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2316019596' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T23:33:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:19 smithi110 ceph-mon[221102]: osdmap e550: 8 total, 8 up, 8 in 2023-09-23T23:33:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:19 smithi110 ceph-mon[221102]: failed to encode map e550 with expected crc 2023-09-23T23:33:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:19 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:19 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:19.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:19 smithi110 ceph-mon[212939]: pgmap v1512: 119 pgs: 119 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s; 1.1 MiB/s, 17 objects/s recovering 2023-09-23T23:33:19.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:19 smithi110 ceph-mon[212939]: failed to encode map e550 with expected crc 2023-09-23T23:33:19.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:19 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2316019596' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T23:33:19.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:19 smithi110 ceph-mon[212939]: osdmap e550: 8 total, 8 up, 8 in 2023-09-23T23:33:19.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:19 smithi110 ceph-mon[212939]: failed to encode map e550 with expected crc 2023-09-23T23:33:19.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:19 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:19.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:19 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:19.846 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 22 16 35232 35216 100.002 54.75 0.00228888 0.00998342 2023-09-23T23:33:20.198 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:33:20.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:20.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:33:20.216 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:33:20.308 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:33:20.396 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:20.411 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:20 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:20 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:20.591 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:33:20.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:20.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:33:20.639 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:20 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:20.639 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:20 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:20.640 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:20 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:20.640 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:20 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:20.701 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:20.746 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:20.846 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 23 16 36605 36589 99.3849 85.8125 0.00919165 0.0100555 2023-09-23T23:33:20.867 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:20.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:20.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:20.973 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:20.990 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:33:21.068 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:21.189 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:21.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:21.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:21.293 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:21.320 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:21.363 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:21.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:21.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:21.466 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:33:21.550 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:21.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:21 smithi138 ceph-mon[193606]: pgmap v1514: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 1.1 MiB/s, 16 objects/s recovering 2023-09-23T23:33:21.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:21 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:21.697 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:21 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:21.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:21 smithi110 ceph-mon[221102]: pgmap v1514: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 1.1 MiB/s, 16 objects/s recovering 2023-09-23T23:33:21.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:21 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:21.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:21 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:21 smithi110 ceph-mon[212939]: pgmap v1514: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 1.1 MiB/s, 16 objects/s recovering 2023-09-23T23:33:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:21 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:21.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:21 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:21.765 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:33:21.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:21.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:33:21.846 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 24 16 37838 37822 98.4548 77.0625 0.00824997 0.0101526 2023-09-23T23:33:21.871 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:33:21.941 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:22.049 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:33:22.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:22.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:33:22.156 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:22.186 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:22.233 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:22.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:22.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:22.309 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:33:22.337 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:22.379 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:22.427 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:22.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:22.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:22.531 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:33:22.604 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:22.813 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:22.847 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 25 16 39047 39031 97.5391 75.5625 0.00573637 0.0102476 2023-09-23T23:33:22.943 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:33:22.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:22.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:33:23.049 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:23.089 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:23.163 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:23.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:23.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:23.267 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:23.310 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:23.417 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:23 smithi110 ceph-mon[212939]: pgmap v1515: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 1.1 MiB/s, 16 objects/s recovering 2023-09-23T23:33:23.417 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:23 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1266837875' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:33:23.433 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:33:23.500 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:23.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:23.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:23.519 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:23 smithi138 ceph-mon[193606]: pgmap v1515: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 1.1 MiB/s, 16 objects/s recovering 2023-09-23T23:33:23.519 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:23 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1266837875' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:33:23.605 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:23.641 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:23.703 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:23.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:23 smithi110 ceph-mon[221102]: pgmap v1515: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 1.1 MiB/s, 16 objects/s recovering 2023-09-23T23:33:23.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:23 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1266837875' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:33:23.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:23.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:23.807 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:33:23.846 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 26 16 40549 40533 97.3979 93.875 0.00702647 0.0102623 2023-09-23T23:33:23.858 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:23.934 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:33:23.934 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:33:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:24 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:33:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:24 smithi138 ceph-mon[193606]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:24 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1266837875' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:33:24.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:24 smithi138 ceph-mon[193606]: osdmap e551: 8 total, 8 up, 8 in 2023-09-23T23:33:24.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:24 smithi138 ceph-mon[193606]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:24 smithi138 ceph-mon[193606]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:24 smithi138 ceph-mon[193606]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:24 smithi138 ceph-mon[193606]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:24 smithi138 ceph-mon[193606]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:24 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:24.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:33:24.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[212939]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1266837875' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:33:24.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[212939]: osdmap e551: 8 total, 8 up, 8 in 2023-09-23T23:33:24.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[212939]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[212939]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[212939]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[212939]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[212939]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:24.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:33:24.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[221102]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1266837875' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:33:24.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[221102]: osdmap e551: 8 total, 8 up, 8 in 2023-09-23T23:33:24.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[221102]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[221102]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[221102]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[221102]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[221102]: failed to encode map e551 with expected crc 2023-09-23T23:33:24.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:24 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:24.835 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:33:24.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:24.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:33:24.846 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 27 16 41320 41304 95.5756 48.1875 0.0122343 0.0104579 2023-09-23T23:33:24.941 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:24.983 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:25.048 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 7, 2, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:33:25.048 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:33:25.048 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T23:33:25.049 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:33:25.093 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:25.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:25.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:25.208 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:33:25.254 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:25.420 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:25 smithi138 ceph-mon[193606]: pgmap v1517: 119 pgs: 119 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T23:33:25.643 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:33:25.664 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:25.675 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:33:25.701 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:33:25.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:25 smithi110 ceph-mon[212939]: pgmap v1517: 119 pgs: 119 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T23:33:25.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:25 smithi110 ceph-mon[221102]: pgmap v1517: 119 pgs: 119 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T23:33:25.709 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:33:25.740 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:33:25.744 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:33:25.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:25.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:33:25.776 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:33:25.814 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:33:25.848 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 28 16 42654 42638 95.1352 83.375 0.00782022 0.0105059 2023-09-23T23:33:25.854 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:25.859 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:33:25.904 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:33:25.909 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:33:25.909 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:33:25.909 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:33:25.910 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:33:25.910 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:33:25.910 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:33:25.910 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:33:25.910 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:33:25.936 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:26.562 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:26.564 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:26 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2362954922' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:26.564 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:26 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2362954922' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:26.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:26.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:26.666 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:26 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2362954922' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:26.734 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:26.831 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:26.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:26.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:26.848 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 29 16 43914 43898 94.5701 78.75 0.011359 0.0105685 2023-09-23T23:33:26.936 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:33:26.953 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:33:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:33:26] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2023-09-23T23:33:27.005 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:27.088 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:33:27.098 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:33:27.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:27.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:33:27.207 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:33:27.233 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:27.315 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:33:27.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:27.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:33:27.426 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:27.491 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:27.576 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:27.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:27.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:27.582 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:27 smithi138 ceph-mon[193606]: pgmap v1518: 119 pgs: 119 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T23:33:27.682 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:27 smithi110 ceph-mon[212939]: pgmap v1518: 119 pgs: 119 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T23:33:27.682 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:27 smithi110 ceph-mon[221102]: pgmap v1518: 119 pgs: 119 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T23:33:27.683 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:27.747 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:27.837 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:27.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:27.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:27.848 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 30 16 45205 45189 94.1073 80.6875 0.0116761 0.0106215 2023-09-23T23:33:27.913 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:27.941 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:27.993 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:28.107 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:28.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:28.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:28.213 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:33:28.239 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:28.271 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:28.383 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:33:28.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:28.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:33:28.493 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:33:28.564 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:28.671 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:33:28.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:28.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:33:28.706 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:33:28.708 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:33:28.777 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:28.824 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:28.848 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 31 16 46627 46611 93.9383 88.875 0.01127 0.0106401 2023-09-23T23:33:29.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:29 smithi138 ceph-mon[193606]: pgmap v1519: 119 pgs: 119 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-09-23T23:33:29.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:29 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:29.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:29 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:33:29.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:29 smithi110 ceph-mon[212939]: pgmap v1519: 119 pgs: 119 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-09-23T23:33:29.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:29 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:29.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:29 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:33:29.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:29 smithi110 ceph-mon[221102]: pgmap v1519: 119 pgs: 119 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-09-23T23:33:29.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:29 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:29.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:29 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:33:29.679 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:33:29.847 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:29.848 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 32 16 47415 47399 92.5421 49.25 0.00746073 0.0107873 2023-09-23T23:33:29.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:29.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:29.951 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:33:30.014 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:30.026 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:30 smithi110 ceph-mon[221102]: from='client.46571 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:33:30.340 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:30 smithi138 ceph-mon[193606]: from='client.46571 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:33:30.341 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:33:30.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:30.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:33:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:30 smithi110 ceph-mon[212939]: from='client.46571 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:33:30.456 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:30.548 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:30.578 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1519,"stamp":"2023-09-23T23:33:27.860253+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2668631968,"num_objects":40735,"num_object_clones":0,"num_object_copies":122205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":40772,"num_write_kb":2605533,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1284,"num_bytes_recovered":81734380,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45757,"ondisk_log_size":45757,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":366,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14557532,"kb_used_data":8468124,"kb_used_omap":29,"kb_used_meta":6089314,"kb_avail":735403684,"statfs":{"total":767960285184,"available":753053372416,"internally_reserved":0,"allocated":8671358976,"data_stored":8655117354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":6235458259},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":922288128,"num_objects":14073,"num_object_clones":0,"num_object_copies":42219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14073,"num_write_kb":900672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.807427"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":747,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.058044+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:33:18.058044+0000","last_peered":"2023-09-23T23:33:18.058044+0000","last_clean":"2023-09-23T23:33:18.058044+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:33:18.058044+0000","last_undegraded":"2023-09-23T23:33:18.058044+0000","last_fullsized":"2023-09-23T23:33:18.058044+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":585,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.062634+0000","last_change":"2023-09-23T23:22:38.439629+0000","last_active":"2023-09-23T23:33:18.062634+0000","last_peered":"2023-09-23T23:33:18.062634+0000","last_clean":"2023-09-23T23:33:18.062634+0000","last_became_active":"2023-09-23T23:22:38.439285+0000","last_became_peered":"2023-09-23T23:22:38.439285+0000","last_unstale":"2023-09-23T23:33:18.062634+0000","last_undegraded":"2023-09-23T23:33:18.062634+0000","last_fullsized":"2023-09-23T23:33:18.062634+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:34.431221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":522,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.062434+0000","last_change":"2023-09-23T23:23:43.511912+0000","last_active":"2023-09-23T23:33:18.062434+0000","last_peered":"2023-09-23T23:33:18.062434+0000","last_clean":"2023-09-23T23:33:18.062434+0000","last_became_active":"2023-09-23T23:23:43.511574+0000","last_became_peered":"2023-09-23T23:23:43.511574+0000","last_unstale":"2023-09-23T23:33:18.062434+0000","last_undegraded":"2023-09-23T23:33:18.062434+0000","last_fullsized":"2023-09-23T23:33:18.062434+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:16.286926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":660,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.060601+0000","last_change":"2023-09-23T23:33:04.705919+0000","last_active":"2023-09-23T23:33:18.060601+0000","last_peered":"2023-09-23T23:33:18.060601+0000","last_clean":"2023-09-23T23:33:18.060601+0000","last_became_active":"2023-09-23T23:33:04.705569+0000","last_became_peered":"2023-09-23T23:33:04.705569+0000","last_unstale":"2023-09-23T23:33:18.060601+0000","last_undegraded":"2023-09-23T23:33:18.060601+0000","last_fullsized":"2023-09-23T23:33:18.060601+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:56.250925+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":263,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.064631+0000","last_change":"2023-09-23T23:26:11.528846+0000","last_active":"2023-09-23T23:33:18.064631+0000","last_peered":"2023-09-23T23:33:18.064631+0000","last_clean":"2023-09-23T23:33:18.064631+0000","last_became_active":"2023-09-23T23:26:11.528529+0000","last_became_peered":"2023-09-23T23:26:11.528529+0000","last_unstale":"2023-09-23T23:33:18.064631+0000","last_undegraded":"2023-09-23T23:33:18.064631+0000","last_fullsized":"2023-09-23T23:33:18.064631+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:35.094583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":525,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T23:33:14.690241+0000","last_change":"2023-09-23T23:33:01.953319+0000","last_active":"2023-09-23T23:33:14.690241+0000","last_peered":"2023-09-23T23:33:14.690241+0000","last_clean":"2023-09-23T23:33:14.690241+0000","last_became_active":"2023-09-23T23:33:01.952918+0000","last_became_peered":"2023-09-23T23:33:01.952918+0000","last_unstale":"2023-09-23T23:33:14.690241+0000","last_undegraded":"2023-09-23T23:33:14.690241+0000","last_fullsized":"2023-09-23T23:33:14.690241+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:59.313898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":854,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.062528+0000","last_change":"2023-09-23T23:28:05.634824+0000","last_active":"2023-09-23T23:33:18.062528+0000","last_peered":"2023-09-23T23:33:18.062528+0000","last_clean":"2023-09-23T23:33:18.062528+0000","last_became_active":"2023-09-23T23:27:23.551046+0000","last_became_peered":"2023-09-23T23:27:23.551046+0000","last_unstale":"2023-09-23T23:33:18.062528+0000","last_undegraded":"2023-09-23T23:33:18.062528+0000","last_fullsized":"2023-09-23T23:33:18.062528+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:59.062239+0000","scrub_duration":0.0034158959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":559,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.060767+0000","last_change":"2023-09-23T23:33:01.954991+0000","last_active":"2023-09-23T23:33:18.060767+0000","last_peered":"2023-09-23T23:33:18.060767+0000","last_clean":"2023-09-23T23:33:18.060767+0000","last_became_active":"2023-09-23T23:33:01.954686+0000","last_became_peered":"2023-09-23T23:33:01.954686+0000","last_unstale":"2023-09-23T23:33:18.060767+0000","last_undegraded":"2023-09-23T23:33:18.060767+0000","last_fullsized":"2023-09-23T23:33:18.060767+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:44:20.221144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":65,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.069687+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:33:18.069687+0000","last_peered":"2023-09-23T23:33:18.069687+0000","last_clean":"2023-09-23T23:33:18.069687+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:33:18.069687+0000","last_undegraded":"2023-09-23T23:33:18.069687+0000","last_fullsized":"2023-09-23T23:33:18.069687+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":870,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T23:33:14.690413+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:33:14.690413+0000","last_peered":"2023-09-23T23:33:14.690413+0000","last_clean":"2023-09-23T23:33:14.690413+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:33:14.690413+0000","last_undegraded":"2023-09-23T23:33:14.690413+0000","last_fullsized":"2023-09-23T23:33:14.690413+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":193,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.062251+0000","last_change":"2023-09-23T23:33:01.952256+0000","last_active":"2023-09-23T23:33:18.062251+0000","last_peered":"2023-09-23T23:33:18.062251+0000","last_clean":"2023-09-23T23:33:18.062251+0000","last_became_active":"2023-09-23T23:33:01.951916+0000","last_became_peered":"2023-09-23T23:33:01.951916+0000","last_unstale":"2023-09-23T23:33:18.062251+0000","last_undegraded":"2023-09-23T23:33:18.062251+0000","last_fullsized":"2023-09-23T23:33:18.062251+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:15.646713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":550,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.060685+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:33:18.060685+0000","last_peered":"2023-09-23T23:33:18.060685+0000","last_clean":"2023-09-23T23:33:18.060685+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:33:18.060685+0000","last_undegraded":"2023-09-23T23:33:18.060685+0000","last_fullsized":"2023-09-23T23:33:18.060685+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":826,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.063092+0000","last_change":"2023-09-23T23:26:14.521792+0000","last_active":"2023-09-23T23:33:18.063092+0000","last_peered":"2023-09-23T23:33:18.063092+0000","last_clean":"2023-09-23T23:33:18.063092+0000","last_became_active":"2023-09-23T23:26:14.521438+0000","last_became_peered":"2023-09-23T23:26:14.521438+0000","last_unstale":"2023-09-23T23:33:18.063092+0000","last_undegraded":"2023-09-23T23:33:18.063092+0000","last_fullsized":"2023-09-23T23:33:18.063092+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:09.103712+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":182,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.062102+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:33:18.062102+0000","last_peered":"2023-09-23T23:33:18.062102+0000","last_clean":"2023-09-23T23:33:18.062102+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:33:18.062102+0000","last_undegraded":"2023-09-23T23:33:18.062102+0000","last_fullsized":"2023-09-23T23:33:18.062102+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":511,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.061178+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:33:18.061178+0000","last_peered":"2023-09-23T23:33:18.061178+0000","last_clean":"2023-09-23T23:33:18.061178+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:33:18.061178+0000","last_undegraded":"2023-09-23T23:33:18.061178+0000","last_fullsized":"2023-09-23T23:33:18.061178+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":985,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.062075+0000","last_change":"2023-09-23T23:33:04.699680+0000","last_active":"2023-09-23T23:33:18.062075+0000","last_peered":"2023-09-23T23:33:18.062075+0000","last_clean":"2023-09-23T23:33:18.062075+0000","last_became_active":"2023-09-23T23:33:04.699261+0000","last_became_peered":"2023-09-23T23:33:04.699261+0000","last_unstale":"2023-09-23T23:33:18.062075+0000","last_undegraded":"2023-09-23T23:33:18.062075+0000","last_fullsized":"2023-09-23T23:33:18.062075+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:59.599804+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":155,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T23:33:14.690073+0000","last_change":"2023-09-23T23:26:48.669857+0000","last_active":"2023-09-23T23:33:14.690073+0000","last_peered":"2023-09-23T23:33:14.690073+0000","last_clean":"2023-09-23T23:33:14.690073+0000","last_became_active":"2023-09-23T23:26:48.538246+0000","last_became_peered":"2023-09-23T23:26:48.538246+0000","last_unstale":"2023-09-23T23:33:14.690073+0000","last_undegraded":"2023-09-23T23:33:14.690073+0000","last_fullsized":"2023-09-23T23:33:14.690073+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:19.039472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":148,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.065451+0000","last_change":"2023-09-23T23:29:54.996097+0000","last_active":"2023-09-23T23:33:18.065451+0000","last_peered":"2023-09-23T23:33:18.065451+0000","last_clean":"2023-09-23T23:33:18.065451+0000","last_became_active":"2023-09-23T23:29:54.995756+0000","last_became_peered":"2023-09-23T23:29:54.995756+0000","last_unstale":"2023-09-23T23:33:18.065451+0000","last_undegraded":"2023-09-23T23:33:18.065451+0000","last_fullsized":"2023-09-23T23:33:18.065451+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:15.856474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":627,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.064822+0000","last_change":"2023-09-23T23:24:02.268079+0000","last_active":"2023-09-23T23:33:18.064822+0000","last_peered":"2023-09-23T23:33:18.064822+0000","last_clean":"2023-09-23T23:33:18.064822+0000","last_became_active":"2023-09-23T23:23:45.463106+0000","last_became_peered":"2023-09-23T23:23:45.463106+0000","last_unstale":"2023-09-23T23:33:18.064822+0000","last_undegraded":"2023-09-23T23:33:18.064822+0000","last_fullsized":"2023-09-23T23:33:18.064822+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.128813+0000","scrub_duration":0.0018535310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":202,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.062845+0000","last_change":"2023-09-23T23:29:55.012082+0000","last_active":"2023-09-23T23:33:18.062845+0000","last_peered":"2023-09-23T23:33:18.062845+0000","last_clean":"2023-09-23T23:33:18.062845+0000","last_became_active":"2023-09-23T23:29:55.000202+0000","last_became_peered":"2023-09-23T23:29:55.000202+0000","last_unstale":"2023-09-23T23:33:18.062845+0000","last_undegraded":"2023-09-23T23:33:18.062845+0000","last_fullsized":"2023-09-23T23:33:18.062845+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:28.669994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":549,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.064459+0000","last_change":"2023-09-23T23:29:55.004242+0000","last_active":"2023-09-23T23:33:18.064459+0000","last_peered":"2023-09-23T23:33:18.064459+0000","last_clean":"2023-09-23T23:33:18.064459+0000","last_became_active":"2023-09-23T23:29:55.003934+0000","last_became_peered":"2023-09-23T23:29:55.003934+0000","last_unstale":"2023-09-23T23:33:18.064459+0000","last_undegraded":"2023-09-23T23:33:18.064459+0000","last_fullsized":"2023-09-23T23:33:18.064459+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:09.283771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":686,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.058776+0000","last_change":"2023-09-23T23:33:04.701609+0000","last_active":"2023-09-23T23:33:18.058776+0000","last_peered":"2023-09-23T23:33:18.058776+0000","last_clean":"2023-09-23T23:33:18.058776+0000","last_became_active":"2023-09-23T23:33:04.701281+0000","last_became_peered":"2023-09-23T23:33:04.701281+0000","last_unstale":"2023-09-23T23:33:18.058776+0000","last_undegraded":"2023-09-23T23:33:18.058776+0000","last_fullsized":"2023-09-23T23:33:18.058776+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:31.726920+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":482,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T23:33:14.689482+0000","last_change":"2023-09-23T23:26:11.569228+0000","last_active":"2023-09-23T23:33:14.689482+0000","last_peered":"2023-09-23T23:33:14.689482+0000","last_clean":"2023-09-23T23:33:14.689482+0000","last_became_active":"2023-09-23T23:26:11.513159+0000","last_became_peered":"2023-09-23T23:26:11.513159+0000","last_unstale":"2023-09-23T23:33:14.689482+0000","last_undegraded":"2023-09-23T23:33:14.689482+0000","last_fullsized":"2023-09-23T23:33:14.689482+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:40.698257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":549,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.061557+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:33:18.061557+0000","last_peered":"2023-09-23T23:33:18.061557+0000","last_clean":"2023-09-23T23:33:18.061557+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:33:18.061557+0000","last_undegraded":"2023-09-23T23:33:18.061557+0000","last_fullsized":"2023-09-23T23:33:18.061557+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":909,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T23:33:14.689007+0000","last_change":"2023-09-23T23:29:58.259420+0000","last_active":"2023-09-23T23:33:14.689007+0000","last_peered":"2023-09-23T23:33:14.689007+0000","last_clean":"2023-09-23T23:33:14.689007+0000","last_became_active":"2023-09-23T23:29:58.258879+0000","last_became_peered":"2023-09-23T23:29:58.258879+0000","last_unstale":"2023-09-23T23:33:14.689007+0000","last_undegraded":"2023-09-23T23:33:14.689007+0000","last_fullsized":"2023-09-23T23:33:14.689007+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:14.307363+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":159,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T23:33:14.689452+0000","last_change":"2023-09-23T23:33:01.947164+0000","last_active":"2023-09-23T23:33:14.689452+0000","last_peered":"2023-09-23T23:33:14.689452+0000","last_clean":"2023-09-23T23:33:14.689452+0000","last_became_active":"2023-09-23T23:33:01.946821+0000","last_became_peered":"2023-09-23T23:33:01.946821+0000","last_unstale":"2023-09-23T23:33:14.689452+0000","last_undegraded":"2023-09-23T23:33:14.689452+0000","last_fullsized":"2023-09-23T23:33:14.689452+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:59.172357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":436,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T23:33:14.690363+0000","last_change":"2023-09-23T23:26:32.040316+0000","last_active":"2023-09-23T23:33:14.690363+0000","last_peered":"2023-09-23T23:33:14.690363+0000","last_clean":"2023-09-23T23:33:14.690363+0000","last_became_active":"2023-09-23T23:26:32.039758+0000","last_became_peered":"2023-09-23T23:26:32.039758+0000","last_unstale":"2023-09-23T23:33:14.690363+0000","last_undegraded":"2023-09-23T23:33:14.690363+0000","last_fullsized":"2023-09-23T23:33:14.690363+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:03.930331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":716,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.065211+0000","last_change":"2023-09-23T23:26:14.527752+0000","last_active":"2023-09-23T23:33:18.065211+0000","last_peered":"2023-09-23T23:33:18.065211+0000","last_clean":"2023-09-23T23:33:18.065211+0000","last_became_active":"2023-09-23T23:26:14.525188+0000","last_became_peered":"2023-09-23T23:26:14.525188+0000","last_unstale":"2023-09-23T23:33:18.065211+0000","last_undegraded":"2023-09-23T23:33:18.065211+0000","last_fullsized":"2023-09-23T23:33:18.065211+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:10.617744+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":414,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T23:33:14.689078+0000","last_change":"2023-09-23T23:29:54.999839+0000","last_active":"2023-09-23T23:33:14.689078+0000","last_peered":"2023-09-23T23:33:14.689078+0000","last_clean":"2023-09-23T23:33:14.689078+0000","last_became_active":"2023-09-23T23:29:54.999530+0000","last_became_peered":"2023-09-23T23:29:54.999530+0000","last_unstale":"2023-09-23T23:33:14.689078+0000","last_undegraded":"2023-09-23T23:33:14.689078+0000","last_fullsized":"2023-09-23T23:33:14.689078+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:43.757346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":497,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.061155+0000","last_change":"2023-09-23T23:33:01.951268+0000","last_active":"2023-09-23T23:33:18.061155+0000","last_peered":"2023-09-23T23:33:18.061155+0000","last_clean":"2023-09-23T23:33:18.061155+0000","last_became_active":"2023-09-23T23:33:01.950984+0000","last_became_peered":"2023-09-23T23:33:01.950984+0000","last_unstale":"2023-09-23T23:33:18.061155+0000","last_undegraded":"2023-09-23T23:33:18.061155+0000","last_fullsized":"2023-09-23T23:33:18.061155+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:53.187008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":914,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.058127+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:33:18.058127+0000","last_peered":"2023-09-23T23:33:18.058127+0000","last_clean":"2023-09-23T23:33:18.058127+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:33:18.058127+0000","last_undegraded":"2023-09-23T23:33:18.058127+0000","last_fullsized":"2023-09-23T23:33:18.058127+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":95,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.070796+0000","last_change":"2023-09-23T23:33:01.960045+0000","last_active":"2023-09-23T23:33:18.070796+0000","last_peered":"2023-09-23T23:33:18.070796+0000","last_clean":"2023-09-23T23:33:18.070796+0000","last_became_active":"2023-09-23T23:33:01.951262+0000","last_became_peered":"2023-09-23T23:33:01.951262+0000","last_unstale":"2023-09-23T23:33:18.070796+0000","last_undegraded":"2023-09-23T23:33:18.070796+0000","last_fullsized":"2023-09-23T23:33:18.070796+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:05.742976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":191,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:19.364604+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:33:19.364604+0000","last_peered":"2023-09-23T23:33:19.364604+0000","last_clean":"2023-09-23T23:33:19.364604+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:33:19.364604+0000","last_undegraded":"2023-09-23T23:33:19.364604+0000","last_fullsized":"2023-09-23T23:33:19.364604+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"551'2485","reported_seq":2508,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-09-23T23:33:23.954010+0000","last_change":"2023-09-23T23:32:51.929504+0000","last_active":"2023-09-23T23:33:23.954010+0000","last_peered":"2023-09-23T23:33:23.954010+0000","last_clean":"2023-09-23T23:33:23.954010+0000","last_became_active":"2023-09-23T23:32:51.929246+0000","last_became_peered":"2023-09-23T23:32:51.929246+0000","last_unstale":"2023-09-23T23:33:23.954010+0000","last_undegraded":"2023-09-23T23:33:23.954010+0000","last_fullsized":"2023-09-23T23:33:23.954010+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":2485,"log_dups_size":0,"ondisk_log_size":2485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:59.358999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162856960,"num_objects":2485,"num_object_clones":0,"num_object_copies":7455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2485,"num_write_kb":159040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":877,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T23:33:14.690063+0000","last_change":"2023-09-23T23:26:51.563868+0000","last_active":"2023-09-23T23:33:14.690063+0000","last_peered":"2023-09-23T23:33:14.690063+0000","last_clean":"2023-09-23T23:33:14.690063+0000","last_became_active":"2023-09-23T23:26:51.563510+0000","last_became_peered":"2023-09-23T23:26:51.563510+0000","last_unstale":"2023-09-23T23:33:14.690063+0000","last_undegraded":"2023-09-23T23:33:14.690063+0000","last_fullsized":"2023-09-23T23:33:14.690063+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:34.244822+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":167,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T23:33:14.690599+0000","last_change":"2023-09-23T23:33:01.958261+0000","last_active":"2023-09-23T23:33:14.690599+0000","last_peered":"2023-09-23T23:33:14.690599+0000","last_clean":"2023-09-23T23:33:14.690599+0000","last_became_active":"2023-09-23T23:33:01.948504+0000","last_became_peered":"2023-09-23T23:33:01.948504+0000","last_unstale":"2023-09-23T23:33:14.690599+0000","last_undegraded":"2023-09-23T23:33:14.690599+0000","last_fullsized":"2023-09-23T23:33:14.690599+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:19.975041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":182,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.058049+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:33:18.058049+0000","last_peered":"2023-09-23T23:33:18.058049+0000","last_clean":"2023-09-23T23:33:18.058049+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:33:18.058049+0000","last_undegraded":"2023-09-23T23:33:18.058049+0000","last_fullsized":"2023-09-23T23:33:18.058049+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"550'2456","reported_seq":2480,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:23.224528+0000","last_change":"2023-09-23T23:32:51.925959+0000","last_active":"2023-09-23T23:33:23.224528+0000","last_peered":"2023-09-23T23:33:23.224528+0000","last_clean":"2023-09-23T23:33:23.224528+0000","last_became_active":"2023-09-23T23:32:51.925673+0000","last_became_peered":"2023-09-23T23:32:51.925673+0000","last_unstale":"2023-09-23T23:33:23.224528+0000","last_undegraded":"2023-09-23T23:33:23.224528+0000","last_fullsized":"2023-09-23T23:33:23.224528+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":2456,"log_dups_size":0,"ondisk_log_size":2456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:18:15.809611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160956416,"num_objects":2456,"num_object_clones":0,"num_object_copies":7368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2456,"num_write_kb":157184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":961,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.061272+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:33:18.061272+0000","last_peered":"2023-09-23T23:33:18.061272+0000","last_clean":"2023-09-23T23:33:18.061272+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:33:18.061272+0000","last_undegraded":"2023-09-23T23:33:18.061272+0000","last_fullsized":"2023-09-23T23:33:18.061272+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":153,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T23:33:14.690371+0000","last_change":"2023-09-23T23:26:48.547146+0000","last_active":"2023-09-23T23:33:14.690371+0000","last_peered":"2023-09-23T23:33:14.690371+0000","last_clean":"2023-09-23T23:33:14.690371+0000","last_became_active":"2023-09-23T23:26:48.537682+0000","last_became_peered":"2023-09-23T23:26:48.537682+0000","last_unstale":"2023-09-23T23:33:14.690371+0000","last_undegraded":"2023-09-23T23:33:14.690371+0000","last_fullsized":"2023-09-23T23:33:14.690371+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:10.115563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":576,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.062737+0000","last_change":"2023-09-23T23:26:01.833889+0000","last_active":"2023-09-23T23:33:18.062737+0000","last_peered":"2023-09-23T23:33:18.062737+0000","last_clean":"2023-09-23T23:33:18.062737+0000","last_became_active":"2023-09-23T23:26:01.833626+0000","last_became_peered":"2023-09-23T23:26:01.833626+0000","last_unstale":"2023-09-23T23:33:18.062737+0000","last_undegraded":"2023-09-23T23:33:18.062737+0000","last_fullsized":"2023-09-23T23:33:18.062737+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:25.659671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"551'2581","reported_seq":3876,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-09-23T23:33:23.881997+0000","last_change":"2023-09-23T23:33:09.007899+0000","last_active":"2023-09-23T23:33:23.881997+0000","last_peered":"2023-09-23T23:33:23.881997+0000","last_clean":"2023-09-23T23:33:23.881997+0000","last_became_active":"2023-09-23T23:33:09.005001+0000","last_became_peered":"2023-09-23T23:33:09.005001+0000","last_unstale":"2023-09-23T23:33:23.881997+0000","last_undegraded":"2023-09-23T23:33:23.881997+0000","last_fullsized":"2023-09-23T23:33:23.881997+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":2581,"log_dups_size":0,"ondisk_log_size":2581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:28.483677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169148416,"num_objects":2581,"num_object_clones":0,"num_object_copies":7743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2581,"num_write_kb":165184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":624,"num_bytes_recovered":40894464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":841,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.069777+0000","last_change":"2023-09-23T23:29:58.262103+0000","last_active":"2023-09-23T23:33:18.069777+0000","last_peered":"2023-09-23T23:33:18.069777+0000","last_clean":"2023-09-23T23:33:18.069777+0000","last_became_active":"2023-09-23T23:29:58.261689+0000","last_became_peered":"2023-09-23T23:29:58.261689+0000","last_unstale":"2023-09-23T23:33:18.069777+0000","last_undegraded":"2023-09-23T23:33:18.069777+0000","last_fullsized":"2023-09-23T23:33:18.069777+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:44.945353+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":470,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T23:33:14.690234+0000","last_change":"2023-09-23T23:29:55.006616+0000","last_active":"2023-09-23T23:33:14.690234+0000","last_peered":"2023-09-23T23:33:14.690234+0000","last_clean":"2023-09-23T23:33:14.690234+0000","last_became_active":"2023-09-23T23:29:55.000411+0000","last_became_peered":"2023-09-23T23:29:55.000411+0000","last_unstale":"2023-09-23T23:33:14.690234+0000","last_undegraded":"2023-09-23T23:33:14.690234+0000","last_fullsized":"2023-09-23T23:33:14.690234+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:22.629830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":65,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.069273+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:33:18.069273+0000","last_peered":"2023-09-23T23:33:18.069273+0000","last_clean":"2023-09-23T23:33:18.069273+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:33:18.069273+0000","last_undegraded":"2023-09-23T23:33:18.069273+0000","last_fullsized":"2023-09-23T23:33:18.069273+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"551'2562","reported_seq":2588,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-09-23T23:33:24.355680+0000","last_change":"2023-09-23T23:32:51.925955+0000","last_active":"2023-09-23T23:33:24.355680+0000","last_peered":"2023-09-23T23:33:24.355680+0000","last_clean":"2023-09-23T23:33:24.355680+0000","last_became_active":"2023-09-23T23:32:51.925649+0000","last_became_peered":"2023-09-23T23:32:51.925649+0000","last_unstale":"2023-09-23T23:33:24.355680+0000","last_undegraded":"2023-09-23T23:33:24.355680+0000","last_fullsized":"2023-09-23T23:33:24.355680+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":2562,"log_dups_size":0,"ondisk_log_size":2562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:59.420467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167903232,"num_objects":2562,"num_object_clones":0,"num_object_copies":7686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2562,"num_write_kb":163968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":615,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.063077+0000","last_change":"2023-09-23T23:28:03.629593+0000","last_active":"2023-09-23T23:33:18.063077+0000","last_peered":"2023-09-23T23:33:18.063077+0000","last_clean":"2023-09-23T23:33:18.063077+0000","last_became_active":"2023-09-23T23:26:51.588385+0000","last_became_peered":"2023-09-23T23:26:51.588385+0000","last_unstale":"2023-09-23T23:33:18.063077+0000","last_undegraded":"2023-09-23T23:33:18.063077+0000","last_fullsized":"2023-09-23T23:33:18.063077+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:20.818366+0000","scrub_duration":0.0050135120000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":390,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.061415+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:33:18.061415+0000","last_peered":"2023-09-23T23:33:18.061415+0000","last_clean":"2023-09-23T23:33:18.061415+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:33:18.061415+0000","last_undegraded":"2023-09-23T23:33:18.061415+0000","last_fullsized":"2023-09-23T23:33:18.061415+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":538,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-09-23T23:33:23.447508+0000","last_change":"2023-09-23T23:28:45.066443+0000","last_active":"2023-09-23T23:33:23.447508+0000","last_peered":"2023-09-23T23:33:23.447508+0000","last_clean":"2023-09-23T23:33:23.447508+0000","last_became_active":"2023-09-23T23:28:45.066154+0000","last_became_peered":"2023-09-23T23:28:45.066154+0000","last_unstale":"2023-09-23T23:33:23.447508+0000","last_undegraded":"2023-09-23T23:33:23.447508+0000","last_fullsized":"2023-09-23T23:33:23.447508+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:51.651425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"551'2608","reported_seq":3869,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-09-23T23:33:23.907085+0000","last_change":"2023-09-23T23:33:11.179081+0000","last_active":"2023-09-23T23:33:23.907085+0000","last_peered":"2023-09-23T23:33:23.907085+0000","last_clean":"2023-09-23T23:33:23.907085+0000","last_became_active":"2023-09-23T23:33:11.176554+0000","last_became_peered":"2023-09-23T23:33:11.176554+0000","last_unstale":"2023-09-23T23:33:23.907085+0000","last_undegraded":"2023-09-23T23:33:23.907085+0000","last_fullsized":"2023-09-23T23:33:23.907085+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":2608,"log_dups_size":0,"ondisk_log_size":2608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:02.235652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170917888,"num_objects":2608,"num_object_clones":0,"num_object_copies":7824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2608,"num_write_kb":166912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":608,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":885,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T23:33:14.689180+0000","last_change":"2023-09-23T23:23:45.830384+0000","last_active":"2023-09-23T23:33:14.689180+0000","last_peered":"2023-09-23T23:33:14.689180+0000","last_clean":"2023-09-23T23:33:14.689180+0000","last_became_active":"2023-09-23T23:23:36.025724+0000","last_became_peered":"2023-09-23T23:23:36.025724+0000","last_unstale":"2023-09-23T23:33:14.689180+0000","last_undegraded":"2023-09-23T23:33:14.689180+0000","last_fullsized":"2023-09-23T23:33:14.689180+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:34.345528+0000","scrub_duration":0.001659259,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":533,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.061719+0000","last_change":"2023-09-23T23:33:01.966036+0000","last_active":"2023-09-23T23:33:18.061719+0000","last_peered":"2023-09-23T23:33:18.061719+0000","last_clean":"2023-09-23T23:33:18.061719+0000","last_became_active":"2023-09-23T23:33:01.955027+0000","last_became_peered":"2023-09-23T23:33:01.955027+0000","last_unstale":"2023-09-23T23:33:18.061719+0000","last_undegraded":"2023-09-23T23:33:18.061719+0000","last_fullsized":"2023-09-23T23:33:18.061719+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:32.631422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":529,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.062960+0000","last_change":"2023-09-23T23:29:55.001192+0000","last_active":"2023-09-23T23:33:18.062960+0000","last_peered":"2023-09-23T23:33:18.062960+0000","last_clean":"2023-09-23T23:33:18.062960+0000","last_became_active":"2023-09-23T23:29:55.000904+0000","last_became_peered":"2023-09-23T23:29:55.000904+0000","last_unstale":"2023-09-23T23:33:18.062960+0000","last_undegraded":"2023-09-23T23:33:18.062960+0000","last_fullsized":"2023-09-23T23:33:18.062960+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:29.169209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"551'2588","reported_seq":2611,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-09-23T23:33:23.951228+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:33:23.951228+0000","last_peered":"2023-09-23T23:33:23.951228+0000","last_clean":"2023-09-23T23:33:23.951228+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:33:23.951228+0000","last_undegraded":"2023-09-23T23:33:23.951228+0000","last_fullsized":"2023-09-23T23:33:23.951228+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":2588,"log_dups_size":0,"ondisk_log_size":2588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169607168,"num_objects":2588,"num_object_clones":0,"num_object_copies":7764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2588,"num_write_kb":165632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":833,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:19.722010+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:33:19.722010+0000","last_peered":"2023-09-23T23:33:19.722010+0000","last_clean":"2023-09-23T23:33:19.722010+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:33:19.722010+0000","last_undegraded":"2023-09-23T23:33:19.722010+0000","last_fullsized":"2023-09-23T23:33:19.722010+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":65,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.069199+0000","last_change":"2023-09-23T23:29:54.998914+0000","last_active":"2023-09-23T23:33:18.069199+0000","last_peered":"2023-09-23T23:33:18.069199+0000","last_clean":"2023-09-23T23:33:18.069199+0000","last_became_active":"2023-09-23T23:29:54.998620+0000","last_became_peered":"2023-09-23T23:29:54.998620+0000","last_unstale":"2023-09-23T23:33:18.069199+0000","last_undegraded":"2023-09-23T23:33:18.069199+0000","last_fullsized":"2023-09-23T23:33:18.069199+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:32.702628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":172,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.064777+0000","last_change":"2023-09-23T23:26:11.516475+0000","last_active":"2023-09-23T23:33:18.064777+0000","last_peered":"2023-09-23T23:33:18.064777+0000","last_clean":"2023-09-23T23:33:18.064777+0000","last_became_active":"2023-09-23T23:26:11.516198+0000","last_became_peered":"2023-09-23T23:26:11.516198+0000","last_unstale":"2023-09-23T23:33:18.064777+0000","last_undegraded":"2023-09-23T23:33:18.064777+0000","last_fullsized":"2023-09-23T23:33:18.064777+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:56.875103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"550'2541","reported_seq":2564,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:23.219614+0000","last_change":"2023-09-23T23:32:51.927456+0000","last_active":"2023-09-23T23:33:23.219614+0000","last_peered":"2023-09-23T23:33:23.219614+0000","last_clean":"2023-09-23T23:33:23.219614+0000","last_became_active":"2023-09-23T23:32:51.926961+0000","last_became_peered":"2023-09-23T23:32:51.926961+0000","last_unstale":"2023-09-23T23:33:23.219614+0000","last_undegraded":"2023-09-23T23:33:23.219614+0000","last_fullsized":"2023-09-23T23:33:23.219614+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":2541,"log_dups_size":0,"ondisk_log_size":2541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:09.809921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166526976,"num_objects":2541,"num_object_clones":0,"num_object_copies":7623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2541,"num_write_kb":162624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":725,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.064985+0000","last_change":"2023-09-23T23:33:04.697442+0000","last_active":"2023-09-23T23:33:18.064985+0000","last_peered":"2023-09-23T23:33:18.064985+0000","last_clean":"2023-09-23T23:33:18.064985+0000","last_became_active":"2023-09-23T23:33:04.697062+0000","last_became_peered":"2023-09-23T23:33:04.697062+0000","last_unstale":"2023-09-23T23:33:18.064985+0000","last_undegraded":"2023-09-23T23:33:18.064985+0000","last_fullsized":"2023-09-23T23:33:18.064985+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:50.295822+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":172,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.062589+0000","last_change":"2023-09-23T23:26:11.512815+0000","last_active":"2023-09-23T23:33:18.062589+0000","last_peered":"2023-09-23T23:33:18.062589+0000","last_clean":"2023-09-23T23:33:18.062589+0000","last_became_active":"2023-09-23T23:26:11.512466+0000","last_became_peered":"2023-09-23T23:26:11.512466+0000","last_unstale":"2023-09-23T23:33:18.062589+0000","last_undegraded":"2023-09-23T23:33:18.062589+0000","last_fullsized":"2023-09-23T23:33:18.062589+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:10.324158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":559,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.063281+0000","last_change":"2023-09-23T23:29:55.001597+0000","last_active":"2023-09-23T23:33:18.063281+0000","last_peered":"2023-09-23T23:33:18.063281+0000","last_clean":"2023-09-23T23:33:18.063281+0000","last_became_active":"2023-09-23T23:29:55.001314+0000","last_became_peered":"2023-09-23T23:29:55.001314+0000","last_unstale":"2023-09-23T23:33:18.063281+0000","last_undegraded":"2023-09-23T23:33:18.063281+0000","last_fullsized":"2023-09-23T23:33:18.063281+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:00.024016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"551'2558","reported_seq":2583,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-09-23T23:33:23.902385+0000","last_change":"2023-09-23T23:32:51.927504+0000","last_active":"2023-09-23T23:33:23.902385+0000","last_peered":"2023-09-23T23:33:23.902385+0000","last_clean":"2023-09-23T23:33:23.902385+0000","last_became_active":"2023-09-23T23:32:51.927190+0000","last_became_peered":"2023-09-23T23:32:51.927190+0000","last_unstale":"2023-09-23T23:33:23.902385+0000","last_undegraded":"2023-09-23T23:33:23.902385+0000","last_fullsized":"2023-09-23T23:33:23.902385+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":2558,"log_dups_size":0,"ondisk_log_size":2558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:04.208118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167641088,"num_objects":2558,"num_object_clones":0,"num_object_copies":7674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2558,"num_write_kb":163712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":891,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.063184+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:33:18.063184+0000","last_peered":"2023-09-23T23:33:18.063184+0000","last_clean":"2023-09-23T23:33:18.063184+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:33:18.063184+0000","last_undegraded":"2023-09-23T23:33:18.063184+0000","last_fullsized":"2023-09-23T23:33:18.063184+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":191,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.060526+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:33:18.060526+0000","last_peered":"2023-09-23T23:33:18.060526+0000","last_clean":"2023-09-23T23:33:18.060526+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:33:18.060526+0000","last_undegraded":"2023-09-23T23:33:18.060526+0000","last_fullsized":"2023-09-23T23:33:18.060526+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":474,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.065327+0000","last_change":"2023-09-23T23:22:39.489325+0000","last_active":"2023-09-23T23:33:18.065327+0000","last_peered":"2023-09-23T23:33:18.065327+0000","last_clean":"2023-09-23T23:33:18.065327+0000","last_became_active":"2023-09-23T23:22:39.489048+0000","last_became_peered":"2023-09-23T23:22:39.489048+0000","last_unstale":"2023-09-23T23:33:18.065327+0000","last_undegraded":"2023-09-23T23:33:18.065327+0000","last_fullsized":"2023-09-23T23:33:18.065327+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:29.984948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"551'2436","reported_seq":2462,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-09-23T23:33:24.326635+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:33:24.326635+0000","last_peered":"2023-09-23T23:33:24.326635+0000","last_clean":"2023-09-23T23:33:24.326635+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:33:24.326635+0000","last_undegraded":"2023-09-23T23:33:24.326635+0000","last_fullsized":"2023-09-23T23:33:24.326635+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":2436,"log_dups_size":0,"ondisk_log_size":2436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159645696,"num_objects":2436,"num_object_clones":0,"num_object_copies":7308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2436,"num_write_kb":155904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":872,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.069413+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:33:18.069413+0000","last_peered":"2023-09-23T23:33:18.069413+0000","last_clean":"2023-09-23T23:33:18.069413+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:33:18.069413+0000","last_undegraded":"2023-09-23T23:33:18.069413+0000","last_fullsized":"2023-09-23T23:33:18.069413+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":255,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.065940+0000","last_change":"2023-09-23T23:23:36.048889+0000","last_active":"2023-09-23T23:33:18.065940+0000","last_peered":"2023-09-23T23:33:18.065940+0000","last_clean":"2023-09-23T23:33:18.065940+0000","last_became_active":"2023-09-23T23:23:36.048572+0000","last_became_peered":"2023-09-23T23:23:36.048572+0000","last_unstale":"2023-09-23T23:33:18.065940+0000","last_undegraded":"2023-09-23T23:33:18.065940+0000","last_fullsized":"2023-09-23T23:33:18.065940+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:55.581146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":566,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.060687+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:33:18.060687+0000","last_peered":"2023-09-23T23:33:18.060687+0000","last_clean":"2023-09-23T23:33:18.060687+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:33:18.060687+0000","last_undegraded":"2023-09-23T23:33:18.060687+0000","last_fullsized":"2023-09-23T23:33:18.060687+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":139,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.065470+0000","last_change":"2023-09-23T23:27:20.549904+0000","last_active":"2023-09-23T23:33:18.065470+0000","last_peered":"2023-09-23T23:33:18.065470+0000","last_clean":"2023-09-23T23:33:18.065470+0000","last_became_active":"2023-09-23T23:27:20.549645+0000","last_became_peered":"2023-09-23T23:27:20.549645+0000","last_unstale":"2023-09-23T23:33:18.065470+0000","last_undegraded":"2023-09-23T23:33:18.065470+0000","last_fullsized":"2023-09-23T23:33:18.065470+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:09.850640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"551'2624","reported_seq":2649,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-09-23T23:33:23.919516+0000","last_change":"2023-09-23T23:32:51.928373+0000","last_active":"2023-09-23T23:33:23.919516+0000","last_peered":"2023-09-23T23:33:23.919516+0000","last_clean":"2023-09-23T23:33:23.919516+0000","last_became_active":"2023-09-23T23:32:51.928093+0000","last_became_peered":"2023-09-23T23:32:51.928093+0000","last_unstale":"2023-09-23T23:33:23.919516+0000","last_undegraded":"2023-09-23T23:33:23.919516+0000","last_fullsized":"2023-09-23T23:33:23.919516+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":2624,"log_dups_size":0,"ondisk_log_size":2624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:44.690480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171966464,"num_objects":2624,"num_object_clones":0,"num_object_copies":7872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2624,"num_write_kb":167936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":608,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.063118+0000","last_change":"2023-09-23T23:33:09.592953+0000","last_active":"2023-09-23T23:33:18.063118+0000","last_peered":"2023-09-23T23:33:18.063118+0000","last_clean":"2023-09-23T23:33:18.063118+0000","last_became_active":"2023-09-23T23:33:04.703345+0000","last_became_peered":"2023-09-23T23:33:04.703345+0000","last_unstale":"2023-09-23T23:33:18.063118+0000","last_undegraded":"2023-09-23T23:33:18.063118+0000","last_fullsized":"2023-09-23T23:33:18.063118+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:40.463955+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":264,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.065643+0000","last_change":"2023-09-23T23:29:54.992796+0000","last_active":"2023-09-23T23:33:18.065643+0000","last_peered":"2023-09-23T23:33:18.065643+0000","last_clean":"2023-09-23T23:33:18.065643+0000","last_became_active":"2023-09-23T23:29:54.992394+0000","last_became_peered":"2023-09-23T23:29:54.992394+0000","last_unstale":"2023-09-23T23:33:18.065643+0000","last_undegraded":"2023-09-23T23:33:18.065643+0000","last_fullsized":"2023-09-23T23:33:18.065643+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:01.353635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":540,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.064370+0000","last_change":"2023-09-23T23:29:55.001578+0000","last_active":"2023-09-23T23:33:18.064370+0000","last_peered":"2023-09-23T23:33:18.064370+0000","last_clean":"2023-09-23T23:33:18.064370+0000","last_became_active":"2023-09-23T23:29:55.001264+0000","last_became_peered":"2023-09-23T23:29:55.001264+0000","last_unstale":"2023-09-23T23:33:18.064370+0000","last_undegraded":"2023-09-23T23:33:18.064370+0000","last_fullsized":"2023-09-23T23:33:18.064370+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:16.411769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":139,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.065845+0000","last_change":"2023-09-23T23:27:20.551221+0000","last_active":"2023-09-23T23:33:18.065845+0000","last_peered":"2023-09-23T23:33:18.065845+0000","last_clean":"2023-09-23T23:33:18.065845+0000","last_became_active":"2023-09-23T23:27:20.550900+0000","last_became_peered":"2023-09-23T23:27:20.550900+0000","last_unstale":"2023-09-23T23:33:18.065845+0000","last_undegraded":"2023-09-23T23:33:18.065845+0000","last_fullsized":"2023-09-23T23:33:18.065845+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:23.012149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.e","version":"550'2415","reported_seq":2439,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:23.223575+0000","last_change":"2023-09-23T23:32:51.927183+0000","last_active":"2023-09-23T23:33:23.223575+0000","last_peered":"2023-09-23T23:33:23.223575+0000","last_clean":"2023-09-23T23:33:23.223575+0000","last_became_active":"2023-09-23T23:32:51.926883+0000","last_became_peered":"2023-09-23T23:32:51.926883+0000","last_unstale":"2023-09-23T23:33:23.223575+0000","last_undegraded":"2023-09-23T23:33:23.223575+0000","last_fullsized":"2023-09-23T23:33:23.223575+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":2415,"log_dups_size":0,"ondisk_log_size":2415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:05.556061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158269440,"num_objects":2415,"num_object_clones":0,"num_object_copies":7245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2415,"num_write_kb":154560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":853,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.063099+0000","last_change":"2023-09-23T23:28:48.113032+0000","last_active":"2023-09-23T23:33:18.063099+0000","last_peered":"2023-09-23T23:33:18.063099+0000","last_clean":"2023-09-23T23:33:18.063099+0000","last_became_active":"2023-09-23T23:28:48.112666+0000","last_became_peered":"2023-09-23T23:28:48.112666+0000","last_unstale":"2023-09-23T23:33:18.063099+0000","last_undegraded":"2023-09-23T23:33:18.063099+0000","last_fullsized":"2023-09-23T23:33:18.063099+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:59:59.838039+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":288,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.062745+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:33:18.062745+0000","last_peered":"2023-09-23T23:33:18.062745+0000","last_clean":"2023-09-23T23:33:18.062745+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:33:18.062745+0000","last_undegraded":"2023-09-23T23:33:18.062745+0000","last_fullsized":"2023-09-23T23:33:18.062745+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":264,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.064600+0000","last_change":"2023-09-23T23:29:54.992719+0000","last_active":"2023-09-23T23:33:18.064600+0000","last_peered":"2023-09-23T23:33:18.064600+0000","last_clean":"2023-09-23T23:33:18.064600+0000","last_became_active":"2023-09-23T23:29:54.992412+0000","last_became_peered":"2023-09-23T23:29:54.992412+0000","last_unstale":"2023-09-23T23:33:18.064600+0000","last_undegraded":"2023-09-23T23:33:18.064600+0000","last_fullsized":"2023-09-23T23:33:18.064600+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:46.541397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":148,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.065414+0000","last_change":"2023-09-23T23:33:01.953232+0000","last_active":"2023-09-23T23:33:18.065414+0000","last_peered":"2023-09-23T23:33:18.065414+0000","last_clean":"2023-09-23T23:33:18.065414+0000","last_became_active":"2023-09-23T23:33:01.952957+0000","last_became_peered":"2023-09-23T23:33:01.952957+0000","last_unstale":"2023-09-23T23:33:18.065414+0000","last_undegraded":"2023-09-23T23:33:18.065414+0000","last_fullsized":"2023-09-23T23:33:18.065414+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:32.835910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"551'2699","reported_seq":2725,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-09-23T23:33:25.440371+0000","last_change":"2023-09-23T23:32:51.927139+0000","last_active":"2023-09-23T23:33:25.440371+0000","last_peered":"2023-09-23T23:33:25.440371+0000","last_clean":"2023-09-23T23:33:25.440371+0000","last_became_active":"2023-09-23T23:32:51.926947+0000","last_became_peered":"2023-09-23T23:32:51.926947+0000","last_unstale":"2023-09-23T23:33:25.440371+0000","last_undegraded":"2023-09-23T23:33:25.440371+0000","last_fullsized":"2023-09-23T23:33:25.440371+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":2699,"log_dups_size":0,"ondisk_log_size":2699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:29.860611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176881664,"num_objects":2699,"num_object_clones":0,"num_object_copies":8097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2699,"num_write_kb":172736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":564,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.063752+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:33:18.063752+0000","last_peered":"2023-09-23T23:33:18.063752+0000","last_clean":"2023-09-23T23:33:18.063752+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:33:18.063752+0000","last_undegraded":"2023-09-23T23:33:18.063752+0000","last_fullsized":"2023-09-23T23:33:18.063752+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":910,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T23:33:14.689196+0000","last_change":"2023-09-23T23:33:04.704192+0000","last_active":"2023-09-23T23:33:14.689196+0000","last_peered":"2023-09-23T23:33:14.689196+0000","last_clean":"2023-09-23T23:33:14.689196+0000","last_became_active":"2023-09-23T23:33:04.703763+0000","last_became_peered":"2023-09-23T23:33:04.703763+0000","last_unstale":"2023-09-23T23:33:14.689196+0000","last_undegraded":"2023-09-23T23:33:14.689196+0000","last_fullsized":"2023-09-23T23:33:14.689196+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:45.012595+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":511,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.061611+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:33:18.061611+0000","last_peered":"2023-09-23T23:33:18.061611+0000","last_clean":"2023-09-23T23:33:18.061611+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:33:18.061611+0000","last_undegraded":"2023-09-23T23:33:18.061611+0000","last_fullsized":"2023-09-23T23:33:18.061611+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":65,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.067811+0000","last_change":"2023-09-23T23:29:54.998861+0000","last_active":"2023-09-23T23:33:18.067811+0000","last_peered":"2023-09-23T23:33:18.067811+0000","last_clean":"2023-09-23T23:33:18.067811+0000","last_became_active":"2023-09-23T23:29:54.998550+0000","last_became_peered":"2023-09-23T23:29:54.998550+0000","last_unstale":"2023-09-23T23:33:18.067811+0000","last_undegraded":"2023-09-23T23:33:18.067811+0000","last_fullsized":"2023-09-23T23:33:18.067811+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:26.515992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"551'2471","reported_seq":2496,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-09-23T23:33:23.943117+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:33:23.943117+0000","last_peered":"2023-09-23T23:33:23.943117+0000","last_clean":"2023-09-23T23:33:23.943117+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:33:23.943117+0000","last_undegraded":"2023-09-23T23:33:23.943117+0000","last_fullsized":"2023-09-23T23:33:23.943117+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":2471,"log_dups_size":0,"ondisk_log_size":2471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161939456,"num_objects":2471,"num_object_clones":0,"num_object_copies":7413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2471,"num_write_kb":158144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":734,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.065074+0000","last_change":"2023-09-23T23:29:58.254396+0000","last_active":"2023-09-23T23:33:18.065074+0000","last_peered":"2023-09-23T23:33:18.065074+0000","last_clean":"2023-09-23T23:33:18.065074+0000","last_became_active":"2023-09-23T23:29:58.254127+0000","last_became_peered":"2023-09-23T23:29:58.254127+0000","last_unstale":"2023-09-23T23:33:18.065074+0000","last_undegraded":"2023-09-23T23:33:18.065074+0000","last_fullsized":"2023-09-23T23:33:18.065074+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:48.863625+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":477,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T23:33:14.688930+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:33:14.688930+0000","last_peered":"2023-09-23T23:33:14.688930+0000","last_clean":"2023-09-23T23:33:14.688930+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:33:14.688930+0000","last_undegraded":"2023-09-23T23:33:14.688930+0000","last_fullsized":"2023-09-23T23:33:14.688930+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":477,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.062402+0000","last_change":"2023-09-23T23:28:22.601436+0000","last_active":"2023-09-23T23:33:18.062402+0000","last_peered":"2023-09-23T23:33:18.062402+0000","last_clean":"2023-09-23T23:33:18.062402+0000","last_became_active":"2023-09-23T23:28:22.601154+0000","last_became_peered":"2023-09-23T23:28:22.601154+0000","last_unstale":"2023-09-23T23:33:18.062402+0000","last_undegraded":"2023-09-23T23:33:18.062402+0000","last_fullsized":"2023-09-23T23:33:18.062402+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:03.296778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":363,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.064865+0000","last_change":"2023-09-23T23:33:01.953646+0000","last_active":"2023-09-23T23:33:18.064865+0000","last_peered":"2023-09-23T23:33:18.064865+0000","last_clean":"2023-09-23T23:33:18.064865+0000","last_became_active":"2023-09-23T23:33:01.952920+0000","last_became_peered":"2023-09-23T23:33:01.952920+0000","last_unstale":"2023-09-23T23:33:18.064865+0000","last_undegraded":"2023-09-23T23:33:18.064865+0000","last_fullsized":"2023-09-23T23:33:18.064865+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:56.358346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"551'2603","reported_seq":2629,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-09-23T23:33:24.636509+0000","last_change":"2023-09-23T23:32:51.927217+0000","last_active":"2023-09-23T23:33:24.636509+0000","last_peered":"2023-09-23T23:33:24.636509+0000","last_clean":"2023-09-23T23:33:24.636509+0000","last_became_active":"2023-09-23T23:32:51.927016+0000","last_became_peered":"2023-09-23T23:32:51.927016+0000","last_unstale":"2023-09-23T23:33:24.636509+0000","last_undegraded":"2023-09-23T23:33:24.636509+0000","last_fullsized":"2023-09-23T23:33:24.636509+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":2603,"log_dups_size":0,"ondisk_log_size":2603,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:12.763576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170590208,"num_objects":2603,"num_object_clones":0,"num_object_copies":7809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2603,"num_write_kb":166592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":506,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.067404+0000","last_change":"2023-09-23T23:33:04.705269+0000","last_active":"2023-09-23T23:33:18.067404+0000","last_peered":"2023-09-23T23:33:18.067404+0000","last_clean":"2023-09-23T23:33:18.067404+0000","last_became_active":"2023-09-23T23:33:04.704933+0000","last_became_peered":"2023-09-23T23:33:04.704933+0000","last_unstale":"2023-09-23T23:33:18.067404+0000","last_undegraded":"2023-09-23T23:33:18.067404+0000","last_fullsized":"2023-09-23T23:33:18.067404+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:34.679767+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":411,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T23:33:14.690456+0000","last_change":"2023-09-23T23:33:01.953282+0000","last_active":"2023-09-23T23:33:14.690456+0000","last_peered":"2023-09-23T23:33:14.690456+0000","last_clean":"2023-09-23T23:33:14.690456+0000","last_became_active":"2023-09-23T23:33:01.952964+0000","last_became_peered":"2023-09-23T23:33:01.952964+0000","last_unstale":"2023-09-23T23:33:14.690456+0000","last_undegraded":"2023-09-23T23:33:14.690456+0000","last_fullsized":"2023-09-23T23:33:14.690456+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:23.201663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":255,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.065987+0000","last_change":"2023-09-23T23:26:32.037099+0000","last_active":"2023-09-23T23:33:18.065987+0000","last_peered":"2023-09-23T23:33:18.065987+0000","last_clean":"2023-09-23T23:33:18.065987+0000","last_became_active":"2023-09-23T23:26:32.036681+0000","last_became_peered":"2023-09-23T23:26:32.036681+0000","last_unstale":"2023-09-23T23:33:18.065987+0000","last_undegraded":"2023-09-23T23:33:18.065987+0000","last_fullsized":"2023-09-23T23:33:18.065987+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:48.926576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":284,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.060748+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:33:18.060748+0000","last_peered":"2023-09-23T23:33:18.060748+0000","last_clean":"2023-09-23T23:33:18.060748+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:33:18.060748+0000","last_undegraded":"2023-09-23T23:33:18.060748+0000","last_fullsized":"2023-09-23T23:33:18.060748+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"551'2550","reported_seq":2575,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-09-23T23:33:23.928769+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:33:23.928769+0000","last_peered":"2023-09-23T23:33:23.928769+0000","last_clean":"2023-09-23T23:33:23.928769+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:33:23.928769+0000","last_undegraded":"2023-09-23T23:33:23.928769+0000","last_fullsized":"2023-09-23T23:33:23.928769+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":2550,"log_dups_size":0,"ondisk_log_size":2550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167116800,"num_objects":2550,"num_object_clones":0,"num_object_copies":7650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2550,"num_write_kb":163200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":882,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.064911+0000","last_change":"2023-09-23T23:29:58.258327+0000","last_active":"2023-09-23T23:33:18.064911+0000","last_peered":"2023-09-23T23:33:18.064911+0000","last_clean":"2023-09-23T23:33:18.064911+0000","last_became_active":"2023-09-23T23:29:58.257930+0000","last_became_peered":"2023-09-23T23:29:58.257930+0000","last_unstale":"2023-09-23T23:33:18.064911+0000","last_undegraded":"2023-09-23T23:33:18.064911+0000","last_fullsized":"2023-09-23T23:33:18.064911+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:00.144157+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":547,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.061442+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:33:18.061442+0000","last_peered":"2023-09-23T23:33:18.061442+0000","last_clean":"2023-09-23T23:33:18.061442+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:33:18.061442+0000","last_undegraded":"2023-09-23T23:33:18.061442+0000","last_fullsized":"2023-09-23T23:33:18.061442+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":148,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.064775+0000","last_change":"2023-09-23T23:29:54.999590+0000","last_active":"2023-09-23T23:33:18.064775+0000","last_peered":"2023-09-23T23:33:18.064775+0000","last_clean":"2023-09-23T23:33:18.064775+0000","last_became_active":"2023-09-23T23:29:54.999209+0000","last_became_peered":"2023-09-23T23:29:54.999209+0000","last_unstale":"2023-09-23T23:33:18.064775+0000","last_undegraded":"2023-09-23T23:33:18.064775+0000","last_fullsized":"2023-09-23T23:33:18.064775+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:32.208030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":92,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.062766+0000","last_change":"2023-09-23T23:33:01.954301+0000","last_active":"2023-09-23T23:33:18.062766+0000","last_peered":"2023-09-23T23:33:18.062766+0000","last_clean":"2023-09-23T23:33:18.062766+0000","last_became_active":"2023-09-23T23:33:01.953960+0000","last_became_peered":"2023-09-23T23:33:01.953960+0000","last_unstale":"2023-09-23T23:33:18.062766+0000","last_undegraded":"2023-09-23T23:33:18.062766+0000","last_fullsized":"2023-09-23T23:33:18.062766+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:56.310561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"551'2534","reported_seq":2560,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-09-23T23:33:24.623739+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:33:24.623739+0000","last_peered":"2023-09-23T23:33:24.623739+0000","last_clean":"2023-09-23T23:33:24.623739+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:33:24.623739+0000","last_undegraded":"2023-09-23T23:33:24.623739+0000","last_fullsized":"2023-09-23T23:33:24.623739+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":2534,"log_dups_size":0,"ondisk_log_size":2534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166068224,"num_objects":2534,"num_object_clones":0,"num_object_copies":7602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2534,"num_write_kb":162176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":936,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.061698+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:33:18.061698+0000","last_peered":"2023-09-23T23:33:18.061698+0000","last_clean":"2023-09-23T23:33:18.061698+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:33:18.061698+0000","last_undegraded":"2023-09-23T23:33:18.061698+0000","last_fullsized":"2023-09-23T23:33:18.061698+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":65,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.069544+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:33:18.069544+0000","last_peered":"2023-09-23T23:33:18.069544+0000","last_clean":"2023-09-23T23:33:18.069544+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:33:18.069544+0000","last_undegraded":"2023-09-23T23:33:18.069544+0000","last_fullsized":"2023-09-23T23:33:18.069544+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":318,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.064915+0000","last_change":"2023-09-23T23:33:01.960385+0000","last_active":"2023-09-23T23:33:18.064915+0000","last_peered":"2023-09-23T23:33:18.064915+0000","last_clean":"2023-09-23T23:33:18.064915+0000","last_became_active":"2023-09-23T23:33:01.953339+0000","last_became_peered":"2023-09-23T23:33:01.953339+0000","last_unstale":"2023-09-23T23:33:18.064915+0000","last_undegraded":"2023-09-23T23:33:18.064915+0000","last_fullsized":"2023-09-23T23:33:18.064915+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:29.776228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":182,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.062660+0000","last_change":"2023-09-23T23:29:55.000912+0000","last_active":"2023-09-23T23:33:18.062660+0000","last_peered":"2023-09-23T23:33:18.062660+0000","last_clean":"2023-09-23T23:33:18.062660+0000","last_became_active":"2023-09-23T23:29:54.999122+0000","last_became_peered":"2023-09-23T23:29:54.999122+0000","last_unstale":"2023-09-23T23:33:18.062660+0000","last_undegraded":"2023-09-23T23:33:18.062660+0000","last_fullsized":"2023-09-23T23:33:18.062660+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:10.962943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":969,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.061042+0000","last_change":"2023-09-23T23:33:04.703982+0000","last_active":"2023-09-23T23:33:18.061042+0000","last_peered":"2023-09-23T23:33:18.061042+0000","last_clean":"2023-09-23T23:33:18.061042+0000","last_became_active":"2023-09-23T23:33:04.703619+0000","last_became_peered":"2023-09-23T23:33:04.703619+0000","last_unstale":"2023-09-23T23:33:18.061042+0000","last_undegraded":"2023-09-23T23:33:18.061042+0000","last_fullsized":"2023-09-23T23:33:18.061042+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:15.891154+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":231,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.064272+0000","last_change":"2023-09-23T23:33:01.954946+0000","last_active":"2023-09-23T23:33:18.064272+0000","last_peered":"2023-09-23T23:33:18.064272+0000","last_clean":"2023-09-23T23:33:18.064272+0000","last_became_active":"2023-09-23T23:33:01.954580+0000","last_became_peered":"2023-09-23T23:33:01.954580+0000","last_unstale":"2023-09-23T23:33:18.064272+0000","last_undegraded":"2023-09-23T23:33:18.064272+0000","last_fullsized":"2023-09-23T23:33:18.064272+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:21.793612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":305,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.063381+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:33:18.063381+0000","last_peered":"2023-09-23T23:33:18.063381+0000","last_clean":"2023-09-23T23:33:18.063381+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:33:18.063381+0000","last_undegraded":"2023-09-23T23:33:18.063381+0000","last_fullsized":"2023-09-23T23:33:18.063381+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":697,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.061386+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:33:18.061386+0000","last_peered":"2023-09-23T23:33:18.061386+0000","last_clean":"2023-09-23T23:33:18.061386+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:33:18.061386+0000","last_undegraded":"2023-09-23T23:33:18.061386+0000","last_fullsized":"2023-09-23T23:33:18.061386+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":292,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.062936+0000","last_change":"2023-09-23T23:22:38.452968+0000","last_active":"2023-09-23T23:33:18.062936+0000","last_peered":"2023-09-23T23:33:18.062936+0000","last_clean":"2023-09-23T23:33:18.062936+0000","last_became_active":"2023-09-23T23:22:38.437986+0000","last_became_peered":"2023-09-23T23:22:38.437986+0000","last_unstale":"2023-09-23T23:33:18.062936+0000","last_undegraded":"2023-09-23T23:33:18.062936+0000","last_fullsized":"2023-09-23T23:33:18.062936+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:10.333271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":182,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.063979+0000","last_change":"2023-09-23T23:27:20.558454+0000","last_active":"2023-09-23T23:33:18.063979+0000","last_peered":"2023-09-23T23:33:18.063979+0000","last_clean":"2023-09-23T23:33:18.063979+0000","last_became_active":"2023-09-23T23:27:20.558204+0000","last_became_peered":"2023-09-23T23:27:20.558204+0000","last_unstale":"2023-09-23T23:33:18.063979+0000","last_undegraded":"2023-09-23T23:33:18.063979+0000","last_fullsized":"2023-09-23T23:33:18.063979+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:15.079990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":306,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.065006+0000","last_change":"2023-09-23T23:29:55.022469+0000","last_active":"2023-09-23T23:33:18.065006+0000","last_peered":"2023-09-23T23:33:18.065006+0000","last_clean":"2023-09-23T23:33:18.065006+0000","last_became_active":"2023-09-23T23:29:55.003082+0000","last_became_peered":"2023-09-23T23:29:55.003082+0000","last_unstale":"2023-09-23T23:33:18.065006+0000","last_undegraded":"2023-09-23T23:33:18.065006+0000","last_fullsized":"2023-09-23T23:33:18.065006+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:15.791643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":172,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.062450+0000","last_change":"2023-09-23T23:26:11.527453+0000","last_active":"2023-09-23T23:33:18.062450+0000","last_peered":"2023-09-23T23:33:18.062450+0000","last_clean":"2023-09-23T23:33:18.062450+0000","last_became_active":"2023-09-23T23:26:11.527208+0000","last_became_peered":"2023-09-23T23:26:11.527208+0000","last_unstale":"2023-09-23T23:33:18.062450+0000","last_undegraded":"2023-09-23T23:33:18.062450+0000","last_fullsized":"2023-09-23T23:33:18.062450+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:15.083611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":836,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T23:33:14.690043+0000","last_change":"2023-09-23T23:29:58.252042+0000","last_active":"2023-09-23T23:33:14.690043+0000","last_peered":"2023-09-23T23:33:14.690043+0000","last_clean":"2023-09-23T23:33:14.690043+0000","last_became_active":"2023-09-23T23:29:58.251749+0000","last_became_peered":"2023-09-23T23:29:58.251749+0000","last_unstale":"2023-09-23T23:33:14.690043+0000","last_undegraded":"2023-09-23T23:33:14.690043+0000","last_fullsized":"2023-09-23T23:33:14.690043+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:39.324116+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":580,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.061303+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:33:18.061303+0000","last_peered":"2023-09-23T23:33:18.061303+0000","last_clean":"2023-09-23T23:33:18.061303+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:33:18.061303+0000","last_undegraded":"2023-09-23T23:33:18.061303+0000","last_fullsized":"2023-09-23T23:33:18.061303+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":564,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.061265+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:33:18.061265+0000","last_peered":"2023-09-23T23:33:18.061265+0000","last_clean":"2023-09-23T23:33:18.061265+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:33:18.061265+0000","last_undegraded":"2023-09-23T23:33:18.061265+0000","last_fullsized":"2023-09-23T23:33:18.061265+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":926,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.064878+0000","last_change":"2023-09-23T23:26:14.527390+0000","last_active":"2023-09-23T23:33:18.064878+0000","last_peered":"2023-09-23T23:33:18.064878+0000","last_clean":"2023-09-23T23:33:18.064878+0000","last_became_active":"2023-09-23T23:26:14.527008+0000","last_became_peered":"2023-09-23T23:26:14.527008+0000","last_unstale":"2023-09-23T23:33:18.064878+0000","last_undegraded":"2023-09-23T23:33:18.064878+0000","last_fullsized":"2023-09-23T23:33:18.064878+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:36.460827+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":378,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.063586+0000","last_change":"2023-09-23T23:33:01.962896+0000","last_active":"2023-09-23T23:33:18.063586+0000","last_peered":"2023-09-23T23:33:18.063586+0000","last_clean":"2023-09-23T23:33:18.063586+0000","last_became_active":"2023-09-23T23:33:01.951853+0000","last_became_peered":"2023-09-23T23:33:01.951853+0000","last_unstale":"2023-09-23T23:33:18.063586+0000","last_undegraded":"2023-09-23T23:33:18.063586+0000","last_fullsized":"2023-09-23T23:33:18.063586+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:33.748007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":606,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T23:33:18.062106+0000","last_change":"2023-09-23T23:29:55.002051+0000","last_active":"2023-09-23T23:33:18.062106+0000","last_peered":"2023-09-23T23:33:18.062106+0000","last_clean":"2023-09-23T23:33:18.062106+0000","last_became_active":"2023-09-23T23:29:55.001732+0000","last_became_peered":"2023-09-23T23:29:55.001732+0000","last_unstale":"2023-09-23T23:33:18.062106+0000","last_undegraded":"2023-09-23T23:33:18.062106+0000","last_fullsized":"2023-09-23T23:33:18.062106+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:28:01.353263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":2668036096,"num_objects":40711,"num_object_clones":0,"num_object_copies":122133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":40711,"num_write_kb":2605504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1232,"num_bytes_recovered":80740352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8061124608,"data_stored":8061124608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":40711,"ondisk_log_size":40711,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":10988,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782647,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1184416,"kb_used_data":874968,"kb_used_omap":8,"kb_used_meta":309431,"kb_avail":92560736,"statfs":{"total":95995035648,"available":94782193664,"internally_reserved":0,"allocated":895967232,"data_stored":894477442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":316857413},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:32:50 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":2.2989999999999999,"15min":2.2690000000000001},"min":{"1min":0.29999999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":17.140000000000001,"5min":44.094999999999999,"15min":85.721000000000004},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.954,"5min":2.3039999999999998,"15min":2.2519999999999998},"min":{"1min":0.32000000000000001,"5min":0.253,"15min":0.19800000000000001},"max":{"1min":11.069000000000001,"5min":44.335999999999999,"15min":85.965000000000003},"last":0.439}]},{"osd":1,"last update":"Sat Sep 23 23:32:50 2023","interfaces":[{"interface":"back","average":{"1min":2.5379999999999998,"5min":5.3979999999999997,"15min":2.7330000000000001},"min":{"1min":0.27600000000000002,"5min":0.23200000000000001,"15min":0.22900000000000001},"max":{"1min":10.054,"5min":198.58000000000001,"15min":198.58000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.9530000000000001,"5min":5.3899999999999997,"15min":2.8980000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.246},"max":{"1min":7.5449999999999999,"5min":198.71100000000001,"15min":198.71100000000001},"last":0.46100000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:32:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":6.5049999999999999,"15min":3.5379999999999998},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":8.1440000000000001,"5min":203.58000000000001,"15min":203.58000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":2.2280000000000002,"5min":6.0439999999999996,"15min":3.0459999999999998},"min":{"1min":0.30599999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.0719999999999992,"5min":198.69,"15min":198.69},"last":0.46899999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:32:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":6.3159999999999998,"15min":3.0960000000000001},"min":{"1min":0.32500000000000001,"5min":0.30499999999999999,"15min":0.26700000000000002},"max":{"1min":6.1139999999999999,"5min":279.26499999999999,"15min":279.26499999999999},"last":54.499000000000002},{"interface":"front","average":{"1min":1.819,"5min":7.6150000000000002,"15min":3.4689999999999999},"min":{"1min":0.373,"5min":0.316,"15min":0.29599999999999999},"max":{"1min":5.8540000000000001,"5min":279.29899999999998,"15min":279.29899999999998},"last":54.531999999999996}]},{"osd":4,"last update":"Sat Sep 23 23:32:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":0.88700000000000001,"15min":0.82399999999999995},"min":{"1min":0.25700000000000001,"5min":0.246,"15min":0.246},"max":{"1min":9.9819999999999993,"5min":9.9819999999999993,"15min":9.9819999999999993},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.77700000000000002,"15min":0.83999999999999997},"min":{"1min":0.35799999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.3360000000000003,"5min":7.9189999999999996,"15min":7.9189999999999996},"last":0.40000000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:32:50 2023","interfaces":[{"interface":"back","average":{"1min":1.274,"5min":3.1059999999999999,"15min":1.698},"min":{"1min":0.36599999999999999,"5min":0.27500000000000002,"15min":0.26500000000000001},"max":{"1min":6.4050000000000002,"5min":210.83600000000001,"15min":210.83600000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.50900000000000001,"5min":2.7949999999999999,"15min":1.464},"min":{"1min":0.36199999999999999,"5min":0.28699999999999998,"15min":0.28199999999999997},"max":{"1min":1.2609999999999999,"5min":207.04599999999999,"15min":207.04599999999999},"last":0.38500000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:32:50 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.76800000000000002,"15min":0.96399999999999997},"min":{"1min":0.34699999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.577,"5min":4.758,"15min":62.213000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.0589999999999999,"15min":1.2250000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.33300000000000002},"max":{"1min":9.9239999999999995,"5min":9.9239999999999995,"15min":62.805999999999997},"last":0.45000000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496124,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1485016,"kb_used_data":816528,"kb_used_omap":0,"kb_used_meta":668480,"kb_avail":92260136,"statfs":{"total":95995035648,"available":94474379264,"internally_reserved":0,"allocated":836124672,"data_stored":834650862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":684523520},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:33:09 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":2.6389999999999998,"15min":5.6840000000000002},"min":{"1min":0.29899999999999999,"5min":0.223,"15min":0.20399999999999999},"max":{"1min":3.5470000000000002,"5min":135.31999999999999,"15min":135.31999999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.82999999999999996,"5min":2.9169999999999998,"15min":5.4950000000000001},"min":{"1min":0.26800000000000002,"5min":0.23000000000000001,"15min":0.22900000000000001},"max":{"1min":3.3359999999999999,"5min":135.34299999999999,"15min":135.34299999999999},"last":0.53700000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:33:09 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":3.4340000000000002,"15min":2.4780000000000002},"min":{"1min":0.25800000000000001,"5min":0.251,"15min":0.245},"max":{"1min":2.9620000000000002,"5min":89.605000000000004,"15min":89.605000000000004},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.76000000000000001,"5min":3.6970000000000001,"15min":2.6080000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":2.5489999999999999,"5min":91.025999999999996,"15min":91.025999999999996},"last":0.57799999999999996}]},{"osd":2,"last update":"Sat Sep 23 23:33:00 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":3.806,"15min":3.847},"min":{"1min":0.36199999999999999,"5min":0.28399999999999997,"15min":0.27800000000000002},"max":{"1min":3.2639999999999998,"5min":97.408000000000001,"15min":127.44499999999999},"last":60.055999999999997},{"interface":"front","average":{"1min":0.94799999999999995,"5min":4.8029999999999999,"15min":4.2359999999999998},"min":{"1min":0.35699999999999998,"5min":0.29999999999999999,"15min":0.26500000000000001},"max":{"1min":3.0870000000000002,"5min":125.539,"15min":125.539},"last":60.409999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:32:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":2.5960000000000001,"15min":3.1030000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.2430000000000003,"5min":138.87299999999999,"15min":138.87299999999999},"last":58.133000000000003},{"interface":"front","average":{"1min":1.2889999999999999,"5min":2.698,"15min":3.4260000000000002},"min":{"1min":0.371,"5min":0.32700000000000001,"15min":0.28100000000000003},"max":{"1min":4.8360000000000003,"5min":138.976,"15min":138.976},"last":58.200000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:33:21 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":0.81999999999999995,"15min":1.302},"min":{"1min":0.26800000000000002,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":7.101,"5min":8.1999999999999993,"15min":166.05199999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.80700000000000005,"15min":1.591},"min":{"1min":0.33600000000000002,"5min":0.27900000000000003,"15min":0.27300000000000002},"max":{"1min":2.5019999999999998,"5min":7.915,"15min":166.631},"last":0.45100000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.93200000000000005,"15min":1.1599999999999999},"min":{"1min":0.30599999999999999,"5min":0.29099999999999998,"15min":0.27900000000000003},"max":{"1min":1.5529999999999999,"5min":11.473000000000001,"15min":91.899000000000001},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.95199999999999996,"15min":1.28},"min":{"1min":0.318,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":2.3969999999999998,"5min":7.5670000000000002,"15min":94.174000000000007},"last":0.50600000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:33:00 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.94099999999999995,"15min":0.89800000000000002},"min":{"1min":0.378,"5min":0.32500000000000001,"15min":0.29599999999999999},"max":{"1min":1.9219999999999999,"5min":12.208,"15min":12.208},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.57099999999999995,"5min":1.1459999999999999,"15min":0.92000000000000004},"min":{"1min":0.35499999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.393,"5min":11.515000000000001,"15min":11.515000000000001},"last":0.60699999999999998}]}]},{"osd":1,"up_from":112,"seq":481036337465,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1842820,"kb_used_data":992124,"kb_used_omap":4,"kb_used_meta":850683,"kb_avail":91902332,"statfs":{"total":95995035648,"available":94107987968,"internally_reserved":0,"allocated":1015934976,"data_stored":1013366222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":871100302},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:33:12 2023","interfaces":[{"interface":"back","average":{"1min":3.181,"5min":2.8889999999999998,"15min":3.9420000000000002},"min":{"1min":0.22500000000000001,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":47.195999999999998,"5min":47.195999999999998,"15min":130.768},"last":0.34899999999999998},{"interface":"front","average":{"1min":3.1200000000000001,"5min":2.1120000000000001,"15min":3.3100000000000001},"min":{"1min":0.214,"5min":0.19,"15min":0.19},"max":{"1min":46.582000000000001,"5min":46.582000000000001,"15min":134.929},"last":0.31900000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8180000000000001,"5min":10.832000000000001,"15min":6.0149999999999997},"min":{"1min":0.26100000000000001,"5min":0.214,"15min":0.20100000000000001},"max":{"1min":9.6329999999999991,"5min":231.67400000000001,"15min":231.67400000000001},"last":0.497},{"interface":"front","average":{"1min":3.3599999999999999,"5min":11.005000000000001,"15min":6.3109999999999999},"min":{"1min":0.23499999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":24.594999999999999,"5min":232.21000000000001,"15min":232.21000000000001},"last":0.79400000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":7.2629999999999999,"15min":4.9939999999999998},"min":{"1min":0.25900000000000001,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":9.7949999999999999,"5min":234.404,"15min":234.404},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.833,"5min":7.1459999999999999,"15min":4.9169999999999998},"min":{"1min":0.33500000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":14.522,"5min":236.33699999999999,"15min":236.33699999999999},"last":0.98699999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:33:12 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":1.5529999999999999,"15min":1.5369999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":8.0850000000000009,"5min":12.042999999999999,"15min":18.474},"last":0.94499999999999995},{"interface":"front","average":{"1min":0.94899999999999995,"5min":1.698,"15min":1.5389999999999999},"min":{"1min":0.35799999999999998,"5min":0.29699999999999999,"15min":0.28299999999999997},"max":{"1min":9.5069999999999997,"5min":18.108000000000001,"15min":18.108000000000001},"last":0.56299999999999994}]},{"osd":5,"last update":"Sat Sep 23 23:33:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.4590000000000001,"15min":2.7210000000000001},"min":{"1min":0.371,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.8859999999999992,"5min":9.0229999999999997,"15min":250.33000000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.94599999999999995,"5min":1.571,"15min":3.141},"min":{"1min":0.38300000000000001,"5min":0.34499999999999997,"15min":0.308},"max":{"1min":8.1140000000000008,"5min":12.007999999999999,"15min":250.21199999999999},"last":0.40600000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:33:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1339999999999999,"5min":1.5389999999999999,"15min":2.101},"min":{"1min":0.38400000000000001,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":8.9269999999999996,"5min":8.9269999999999996,"15min":74.087999999999994},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.6439999999999999,"15min":2.2949999999999999},"min":{"1min":0.38,"5min":0.37,"15min":0.32800000000000001},"max":{"1min":8.1400000000000006,"5min":11.927,"15min":73.974999999999994},"last":0.503}]},{"osd":7,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":1.599,"15min":3.153},"min":{"1min":0.42299999999999999,"5min":0.37,"15min":0.32900000000000001},"max":{"1min":9.5570000000000004,"5min":11.385,"15min":78.308000000000007},"last":2.456},{"interface":"front","average":{"1min":1.865,"5min":1.7629999999999999,"15min":2.964},"min":{"1min":0.435,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":11.977,"5min":11.977,"15min":77.204999999999998},"last":0.44800000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024289,"num_pgs":16,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1367504,"kb_used_data":193352,"kb_used_omap":5,"kb_used_meta":1174138,"kb_avail":92377648,"statfs":{"total":95995035648,"available":94594711552,"internally_reserved":0,"allocated":197992448,"data_stored":195406525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1202317946},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:32:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2629999999999999,"5min":11.031000000000001,"15min":6.1470000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.17499999999999999},"max":{"1min":14.115,"5min":213.45699999999999,"15min":213.45699999999999},"last":0.71799999999999997},{"interface":"front","average":{"1min":2.4510000000000001,"5min":11.238,"15min":5.181},"min":{"1min":0.22500000000000001,"5min":0.186,"15min":0.17599999999999999},"max":{"1min":14.478,"5min":213.04900000000001,"15min":213.04900000000001},"last":1.2689999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:32:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1459999999999999,"5min":5.694,"15min":5.8209999999999997},"min":{"1min":0.214,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":14.970000000000001,"5min":220.779,"15min":220.779},"last":0.251},{"interface":"front","average":{"1min":1.8919999999999999,"5min":6.2309999999999999,"15min":5.9459999999999997},"min":{"1min":0.251,"5min":0.19800000000000001,"15min":0.17599999999999999},"max":{"1min":15.140000000000001,"5min":218.74700000000001,"15min":218.74700000000001},"last":1.4490000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:32:24 2023","interfaces":[{"interface":"back","average":{"1min":8.1720000000000006,"5min":7.9409999999999998,"15min":5.2629999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":114.381,"5min":220.732,"15min":220.732},"last":2.0369999999999999},{"interface":"front","average":{"1min":9.0609999999999999,"5min":8.4740000000000002,"15min":5.2809999999999997},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":114.30500000000001,"5min":218.78200000000001,"15min":218.78200000000001},"last":0.45400000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":2.7120000000000002,"15min":3.0859999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.27000000000000002},"max":{"1min":9.3140000000000001,"5min":108.01900000000001,"15min":138.81299999999999},"last":1.8720000000000001},{"interface":"front","average":{"1min":0.86799999999999999,"5min":2.601,"15min":3.6240000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.1539999999999999,"5min":107.898,"15min":171.251},"last":1.8999999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":1.399,"15min":2.8969999999999998},"min":{"1min":0.33500000000000002,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":6.1870000000000003,"5min":9.8230000000000004,"15min":171.30699999999999},"last":2.3519999999999999},{"interface":"front","average":{"1min":0.72599999999999998,"5min":1.165,"15min":2.1480000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":2.1459999999999999,"5min":9.9000000000000004,"15min":139.40799999999999},"last":2.0979999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":2.5510000000000002,"15min":3.452},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.29499999999999998},"max":{"1min":6.2320000000000002,"5min":108.67,"15min":171.33600000000001},"last":1.621},{"interface":"front","average":{"1min":0.748,"5min":2.3719999999999999,"15min":3.6379999999999999},"min":{"1min":0.35199999999999998,"5min":0.34899999999999998,"15min":0.27100000000000002},"max":{"1min":2.1720000000000002,"5min":108.804,"15min":184.13499999999999},"last":2.0720000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:32:51 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":2.8260000000000001,"15min":2.6440000000000001},"min":{"1min":0.35499999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.0150000000000006,"5min":111.71299999999999,"15min":138.77699999999999},"last":1.8320000000000001},{"interface":"front","average":{"1min":0.98699999999999999,"5min":2.5470000000000002,"15min":2.367},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":3.347,"5min":111.108,"15min":139.37899999999999},"last":1.103}]}]},{"osd":2,"up_from":199,"seq":854698492110,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2291380,"kb_used_data":1537516,"kb_used_omap":7,"kb_used_meta":753848,"kb_avail":91453772,"statfs":{"total":95995035648,"available":93648662528,"internally_reserved":0,"allocated":1574416384,"data_stored":1571801799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":771940433},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":2.798,"15min":4.0899999999999999},"min":{"1min":0.218,"5min":0.21299999999999999,"15min":0.16700000000000001},"max":{"1min":11.353,"5min":58.802,"15min":119.277},"last":1.5880000000000001},{"interface":"front","average":{"1min":1.7549999999999999,"5min":2.4049999999999998,"15min":3.8999999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.18099999999999999},"max":{"1min":7.9180000000000001,"5min":58.115000000000002,"15min":119.56100000000001},"last":4.0590000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.6970000000000001,"5min":3.5960000000000001,"15min":3.6699999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":7.6479999999999997,"5min":82.876000000000005,"15min":157.98699999999999},"last":2.661},{"interface":"front","average":{"1min":1.71,"5min":3.8610000000000002,"15min":3.8130000000000002},"min":{"1min":0.27000000000000002,"5min":0.24099999999999999,"15min":0.22600000000000001},"max":{"1min":11.278,"5min":82.909000000000006,"15min":158.03800000000001},"last":2.6749999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.954,"5min":3.3450000000000002,"15min":3.7589999999999999},"min":{"1min":0.311,"5min":0.24299999999999999,"15min":0.191},"max":{"1min":10.451000000000001,"5min":103.227,"15min":119.366},"last":0.251},{"interface":"front","average":{"1min":2.2999999999999998,"5min":3.359,"15min":3.3450000000000002},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":11.324999999999999,"5min":102.693,"15min":102.693},"last":0.75600000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.6359999999999999,"15min":3.863},"min":{"1min":0.33700000000000002,"5min":0.27700000000000002,"15min":0.25},"max":{"1min":4.5739999999999998,"5min":11.586,"15min":332.98099999999999},"last":3.3119999999999998},{"interface":"front","average":{"1min":1.3819999999999999,"5min":1.7549999999999999,"15min":3.3319999999999999},"min":{"1min":0.42899999999999999,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":6.1189999999999998,"5min":11.621,"15min":332.96199999999999},"last":0.52800000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":1.5309999999999999,"15min":2.2170000000000001},"min":{"1min":0.34300000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.6399999999999997,"5min":6.2990000000000004,"15min":119.919},"last":3.3809999999999998},{"interface":"front","average":{"1min":1.242,"5min":1.5529999999999999,"15min":2.1400000000000001},"min":{"1min":0.41099999999999998,"5min":0.33900000000000002,"15min":0.33000000000000002},"max":{"1min":3.827,"5min":7.7910000000000004,"15min":119.306},"last":0.49299999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.8640000000000001,"15min":4.1319999999999997},"min":{"1min":0.38200000000000001,"5min":0.315,"15min":0.31},"max":{"1min":4.6150000000000002,"5min":14.795999999999999,"15min":349.32799999999997},"last":3.3530000000000002},{"interface":"front","average":{"1min":1.339,"5min":1.9430000000000001,"15min":3.6629999999999998},"min":{"1min":0.38900000000000001,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":3.8079999999999998,"5min":12.496,"15min":349.17700000000002},"last":0.82499999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.9590000000000001,"5min":1.9830000000000001,"15min":1.6879999999999999},"min":{"1min":0.39600000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":11.252000000000001,"5min":15.734999999999999,"15min":20.350000000000001},"last":9.6400000000000006},{"interface":"front","average":{"1min":1.984,"5min":1.9119999999999999,"15min":1.7310000000000001},"min":{"1min":0.41299999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":11.304,"5min":15.709,"15min":20.321000000000002},"last":0.85299999999999998}]}]},{"osd":3,"up_from":148,"seq":635655160079,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1771692,"kb_used_data":906148,"kb_used_omap":2,"kb_used_meta":865533,"kb_avail":91973460,"statfs":{"total":95995035648,"available":94180823040,"internally_reserved":0,"allocated":927895552,"data_stored":925346358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":886305845},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:32:40 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":2.1139999999999999,"15min":2.8410000000000002},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.158},"max":{"1min":4.782,"5min":22.271999999999998,"15min":103.866},"last":0.70199999999999996},{"interface":"front","average":{"1min":2.8929999999999998,"5min":3.1859999999999999,"15min":3.0630000000000002},"min":{"1min":0.26400000000000001,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":22.754000000000001,"5min":24.683,"15min":101.884},"last":0.59699999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:32:40 2023","interfaces":[{"interface":"back","average":{"1min":8.782,"5min":4.8140000000000001,"15min":3.1960000000000002},"min":{"1min":0.27400000000000002,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":56.119,"5min":85.242999999999995,"15min":85.242999999999995},"last":0.71699999999999997},{"interface":"front","average":{"1min":6.4029999999999996,"5min":4.3399999999999999,"15min":2.8769999999999998},"min":{"1min":0.224,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":53.962000000000003,"5min":85.302000000000007,"15min":85.302000000000007},"last":0.65300000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:32:52 2023","interfaces":[{"interface":"back","average":{"1min":6.7460000000000004,"5min":6.8010000000000002,"15min":4.9139999999999997},"min":{"1min":0.32900000000000001,"5min":0.219,"15min":0.19800000000000001},"max":{"1min":56.226999999999997,"5min":84.724999999999994,"15min":101.279},"last":0.63200000000000001},{"interface":"front","average":{"1min":6.2850000000000001,"5min":6.5940000000000003,"15min":4.673},"min":{"1min":0.34899999999999998,"5min":0.25700000000000001,"15min":0.214},"max":{"1min":54.280999999999999,"5min":84.567999999999998,"15min":101.294},"last":0.76200000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:32:40 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":1.403,"15min":1.375},"min":{"1min":0.32900000000000001,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":5.8630000000000004,"5min":8.5869999999999997,"15min":14.754},"last":0.74199999999999999},{"interface":"front","average":{"1min":2.0339999999999998,"5min":2.2519999999999998,"15min":1.8700000000000001},"min":{"1min":0.375,"5min":0.30599999999999999,"15min":0.26800000000000002},"max":{"1min":11.494999999999999,"5min":24.782,"15min":24.782},"last":0.73099999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:32:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":2.1640000000000001,"15min":1.796},"min":{"1min":0.40300000000000002,"5min":0.29599999999999999,"15min":0.28000000000000003},"max":{"1min":11.470000000000001,"5min":24.565000000000001,"15min":24.565000000000001},"last":206.22900000000001},{"interface":"front","average":{"1min":2.3210000000000002,"5min":2.2229999999999999,"15min":1.9159999999999999},"min":{"1min":0.39500000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":11.702,"5min":22.472999999999999,"15min":22.472999999999999},"last":205.90799999999999}]},{"osd":6,"last update":"Sat Sep 23 23:32:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1659999999999999,"5min":2.3180000000000001,"15min":2.1040000000000001},"min":{"1min":0.45300000000000001,"5min":0.35199999999999998,"15min":0.31900000000000001},"max":{"1min":11.523,"5min":24.658000000000001,"15min":53.057000000000002},"last":0.69299999999999995},{"interface":"front","average":{"1min":2.7989999999999999,"5min":2.1379999999999999,"15min":2.1339999999999999},"min":{"1min":0.40899999999999997,"5min":0.36099999999999999,"15min":0.35299999999999998},"max":{"1min":18.956,"5min":18.956,"15min":53.088000000000001},"last":0.75}]},{"osd":7,"last update":"Sat Sep 23 23:32:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":3.73,"15min":1.9690000000000001},"min":{"1min":0.41199999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.0389999999999997,"5min":146.803,"15min":146.803},"last":192.69},{"interface":"front","average":{"1min":1.1699999999999999,"5min":4.556,"15min":2.5129999999999999},"min":{"1min":0.39200000000000002,"5min":0.38500000000000001,"15min":0.36699999999999999},"max":{"1min":3.6379999999999999,"5min":146.422,"15min":146.422},"last":192.65600000000001}]}]},{"osd":4,"up_from":30,"seq":128849019397,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2488348,"kb_used_data":1693588,"kb_used_omap":0,"kb_used_meta":794752,"kb_avail":91256804,"statfs":{"total":95995035648,"available":93446967296,"internally_reserved":0,"allocated":1734234112,"data_stored":1732748127,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":813826048},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:33:03 2023","interfaces":[{"interface":"back","average":{"1min":4.8780000000000001,"5min":3.3069999999999999,"15min":4.8869999999999996},"min":{"1min":0.23499999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":84.066999999999993,"5min":84.066999999999993,"15min":199.64400000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":4.6449999999999996,"5min":3.4620000000000002,"15min":5.1020000000000003},"min":{"1min":0.25600000000000001,"5min":0.24199999999999999,"15min":0.216},"max":{"1min":83.930000000000007,"5min":83.930000000000007,"15min":195.649},"last":0.41599999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:32:25 2023","interfaces":[{"interface":"back","average":{"1min":1.9510000000000001,"5min":5.7110000000000003,"15min":3.6269999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.23100000000000001},"max":{"1min":4.6139999999999999,"5min":198.27199999999999,"15min":198.27199999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.1160000000000001,"5min":5.508,"15min":3.7040000000000002},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.6989999999999998,"5min":196.47,"15min":196.47},"last":0.35299999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:32:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1930000000000001,"5min":9.1120000000000001,"15min":5.1630000000000003},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.9180000000000001,"5min":198.178,"15min":198.178},"last":9.7430000000000003},{"interface":"front","average":{"1min":1.833,"5min":8.8339999999999996,"15min":5.0659999999999998},"min":{"1min":0.316,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.999000000000001,"5min":201.56,"15min":201.56},"last":0.50800000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:32:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":7.7839999999999998,"15min":5.7779999999999996},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26300000000000001},"max":{"1min":7.6689999999999996,"5min":277.459,"15min":277.459},"last":9.7910000000000004},{"interface":"front","average":{"1min":1.105,"5min":7.484,"15min":5.7039999999999997},"min":{"1min":0.32800000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.6870000000000003,"5min":277.20800000000003,"15min":277.20800000000003},"last":0.39400000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:33:19 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":0.84599999999999997,"15min":1.47},"min":{"1min":0.308,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.7750000000000004,"5min":7.2460000000000004,"15min":161.86500000000001},"last":2.9239999999999999},{"interface":"front","average":{"1min":1.099,"5min":0.75,"15min":1.5209999999999999},"min":{"1min":0.30299999999999999,"5min":0.28899999999999998,"15min":0.249},"max":{"1min":6.0899999999999999,"5min":6.0899999999999999,"15min":159.24199999999999},"last":0.439}]},{"osd":6,"last update":"Sat Sep 23 23:32:45 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.78800000000000003,"15min":1.0780000000000001},"min":{"1min":0.29899999999999999,"5min":0.27900000000000003,"15min":0.26000000000000001},"max":{"1min":2.9209999999999998,"5min":7.1130000000000004,"15min":90.346999999999994},"last":9.8209999999999997},{"interface":"front","average":{"1min":0.876,"5min":0.94299999999999995,"15min":1.272},"min":{"1min":0.34200000000000003,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.1390000000000002,"5min":9.8079999999999998,"15min":90.733000000000004},"last":0.54100000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:32:45 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.2709999999999999,"15min":1.125},"min":{"1min":0.35799999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.0299999999999998,"5min":24.533999999999999,"15min":61.923999999999999},"last":1.2569999999999999},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.0569999999999999,"15min":1.137},"min":{"1min":0.33100000000000002,"5min":0.316,"15min":0.316},"max":{"1min":2.8639999999999999,"5min":27.771999999999998,"15min":61.990000000000002},"last":1.0509999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757760,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2126356,"kb_used_data":1453900,"kb_used_omap":0,"kb_used_meta":672448,"kb_avail":91618796,"statfs":{"total":95995035648,"available":93817647104,"internally_reserved":0,"allocated":1488793600,"data_stored":1487320019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":688586752},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:33:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":1.4279999999999999,"15min":3.508},"min":{"1min":0.25900000000000001,"5min":0.219,"15min":0.19500000000000001},"max":{"1min":21.109000000000002,"5min":21.109000000000002,"15min":113.869},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.1459999999999999,"5min":1.3879999999999999,"15min":3.9209999999999998},"min":{"1min":0.30299999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":20.876999999999999,"5min":20.876999999999999,"15min":138.55199999999999},"last":2.1110000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:33:15 2023","interfaces":[{"interface":"back","average":{"1min":2.605,"5min":2.1760000000000002,"15min":2.665},"min":{"1min":0.25700000000000001,"5min":0.25,"15min":0.23100000000000001},"max":{"1min":29.007999999999999,"5min":29.007999999999999,"15min":69.816999999999993},"last":2.1309999999999998},{"interface":"front","average":{"1min":2.7080000000000002,"5min":2.3439999999999999,"15min":2.976},"min":{"1min":0.30599999999999999,"5min":0.249,"15min":0.249},"max":{"1min":29.053999999999998,"5min":29.053999999999998,"15min":70.268000000000001},"last":2.0939999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:32:43 2023","interfaces":[{"interface":"back","average":{"1min":1.464,"5min":2.6419999999999999,"15min":3.0419999999999998},"min":{"1min":0.34799999999999998,"5min":0.27200000000000002,"15min":0.26900000000000002},"max":{"1min":3.6070000000000002,"5min":46.177,"15min":144.91200000000001},"last":3.7719999999999998},{"interface":"front","average":{"1min":1.7649999999999999,"5min":2.7160000000000002,"15min":2.891},"min":{"1min":0.34200000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.468,"5min":45.863999999999997,"15min":145.15799999999999},"last":0.373}]},{"osd":3,"last update":"Sat Sep 23 23:32:36 2023","interfaces":[{"interface":"back","average":{"1min":1.49,"5min":3.7629999999999999,"15min":3.7240000000000002},"min":{"1min":0.37,"5min":0.32000000000000001,"15min":0.27400000000000002},"max":{"1min":3.4809999999999999,"5min":112.71899999999999,"15min":159.53299999999999},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.675,"5min":3.8809999999999998,"15min":3.9390000000000001},"min":{"1min":0.38900000000000001,"5min":0.32700000000000001,"15min":0.312},"max":{"1min":3.4900000000000002,"5min":112.70099999999999,"15min":159.54900000000001},"last":2.1560000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:32:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.032,"15min":0.99099999999999999},"min":{"1min":0.25700000000000001,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":6.1310000000000002,"5min":19.962,"15min":19.962},"last":5.0529999999999999},{"interface":"front","average":{"1min":0.83999999999999997,"5min":0.89900000000000002,"15min":0.80200000000000005},"min":{"1min":0.30599999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.4080000000000004,"5min":20.829999999999998,"15min":20.829999999999998},"last":5.3639999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:32:22 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.76400000000000001,"15min":2.1739999999999999},"min":{"1min":0.33100000000000002,"5min":0.27600000000000002,"15min":0.26900000000000002},"max":{"1min":2.044,"5min":5.1299999999999999,"15min":310.68799999999999},"last":1.117},{"interface":"front","average":{"1min":1.149,"5min":0.93799999999999994,"15min":2.3969999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.25900000000000001},"max":{"1min":3.3410000000000002,"5min":8.9979999999999993,"15min":311.45400000000001},"last":2.1949999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:32:43 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":3.008,"15min":3.1240000000000001},"min":{"1min":0.34599999999999997,"5min":0.317,"15min":0.317},"max":{"1min":2.6880000000000002,"5min":179.28800000000001,"15min":179.28800000000001},"last":2.226},{"interface":"front","average":{"1min":0.82499999999999996,"5min":3.1429999999999998,"15min":3.2469999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.2109999999999999,"5min":179.428,"15min":179.428},"last":4.9989999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":192675840,"data_stored":192675840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":874905600,"data_stored":874905600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1507983360,"data_stored":1507983360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":838795264,"data_stored":838795264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1510146048,"data_stored":1510146048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1496252416,"data_stored":1496252416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":830013440,"data_stored":830013440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":810352640,"data_stored":810352640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:33:30.586 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T23:33:30.586 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 71 2023-09-23T23:33:30.642 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:30.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:30.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:30.749 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:30.830 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:30.848 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 33 16 48572 48556 91.9291 72.3125 0.0087116 0.0108709 2023-09-23T23:33:30.929 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:30.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:30.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:31.037 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:31.075 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:31.401 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:31 smithi110 ceph-mon[221102]: pgmap v1520: 119 pgs: 119 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T23:33:31.401 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:31 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:31.402 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:31 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:31.402 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:31 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:33:31.402 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:31 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:33:31.402 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:31 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:31.469 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:31.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:31.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:31.540 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:31.575 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:31.642 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:31.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:31 smithi138 ceph-mon[193606]: pgmap v1520: 119 pgs: 119 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T23:33:31.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:31 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:31.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:31 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:31.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:31 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:33:31.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:31 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:33:31.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:31 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:31.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:31 smithi110 ceph-mon[212939]: pgmap v1520: 119 pgs: 119 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T23:33:31.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:31 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:31.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:31 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:31.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:31 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:33:31.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:31 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:33:31.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:31 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:31.736 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:31.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:31.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:31.811 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:31.822 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:33:31.823 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:33:31.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.823 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:49967 13.0 e551/534 13:06c5ed91:::benchmark_data_smithi110_541928_object49966:head v 551'3121, mlcod=551'3119)", 2023-09-23T23:33:31.823 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:31.818046+0000", 2023-09-23T23:33:31.823 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0028154930000000001, 2023-09-23T23:33:31.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028471960000000002, 2023-09-23T23:33:31.823 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:31.823 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:33:31.823 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:31.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:31.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.818046+0000", 2023-09-23T23:33:31.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:31.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:31.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.818046+0000", 2023-09-23T23:33:31.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:31.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:31.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.818046+0000", 2023-09-23T23:33:31.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:33:31.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:31.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.818110+0000", 2023-09-23T23:33:31.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4469999999999998e-05 2023-09-23T23:33:31.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:31.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.818121+0000", 2023-09-23T23:33:31.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1011e-05 2023-09-23T23:33:31.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.818129+0000", 2023-09-23T23:33:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4050000000000003e-06 2023-09-23T23:33:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.818177+0000", 2023-09-23T23:33:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8056999999999997e-05 2023-09-23T23:33:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.818197+0000", 2023-09-23T23:33:31.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.056e-05 2023-09-23T23:33:31.827 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:31.827 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:31.827 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:31.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.827 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:49968 13.1 e551/534 13:828faa47:::benchmark_data_smithi110_541928_object49967:head v 551'3066, mlcod=551'3062)", 2023-09-23T23:33:31.827 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:31.819014+0000", 2023-09-23T23:33:31.827 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0018484, 2023-09-23T23:33:31.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019353479999999999, 2023-09-23T23:33:31.827 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:31.827 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:33:31.827 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:31.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:31.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.819014+0000", 2023-09-23T23:33:31.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:31.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:31.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.819014+0000", 2023-09-23T23:33:31.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:31.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:31.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.819013+0000", 2023-09-23T23:33:31.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:33:31.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:31.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.819074+0000", 2023-09-23T23:33:31.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0724999999999999e-05 2023-09-23T23:33:31.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:31.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.819086+0000", 2023-09-23T23:33:31.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1977999999999999e-05 2023-09-23T23:33:31.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:31.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.819092+0000", 2023-09-23T23:33:31.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8549999999999996e-06 2023-09-23T23:33:31.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:31.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.819155+0000", 2023-09-23T23:33:31.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2599000000000002e-05 2023-09-23T23:33:31.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:31.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.819175+0000", 2023-09-23T23:33:31.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0468000000000001e-05 2023-09-23T23:33:31.831 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:31.831 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:31.831 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:31.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.831 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:49960 13.f e551/534 13:fcbe5ece:::benchmark_data_smithi110_541928_object49959:head v 551'3010, mlcod=551'3008)", 2023-09-23T23:33:31.831 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:31.819102+0000", 2023-09-23T23:33:31.831 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0017602029999999999, 2023-09-23T23:33:31.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001885956, 2023-09-23T23:33:31.831 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:31.832 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:33:31.832 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:31.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:31.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.819102+0000", 2023-09-23T23:33:31.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:31.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:31.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.819102+0000", 2023-09-23T23:33:31.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:31.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:31.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.819101+0000", 2023-09-23T23:33:31.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:33:31.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:31.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.819130+0000", 2023-09-23T23:33:31.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8691e-05 2023-09-23T23:33:31.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:31.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.819135+0000", 2023-09-23T23:33:31.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.164e-06 2023-09-23T23:33:31.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:31.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.819141+0000", 2023-09-23T23:33:31.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.609e-06 2023-09-23T23:33:31.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:31.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.819180+0000", 2023-09-23T23:33:31.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9391999999999998e-05 2023-09-23T23:33:31.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:31.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.819201+0000", 2023-09-23T23:33:31.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0341000000000001e-05 2023-09-23T23:33:31.835 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:31.835 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:31.835 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:31.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.835 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:49970 13.a 13:55632bb5:::benchmark_data_smithi110_541928_object49969:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T23:33:31.835 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:31.816486+0000", 2023-09-23T23:33:31.836 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0043757520000000001, 2023-09-23T23:33:31.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045572449999999997, 2023-09-23T23:33:31.836 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:31.836 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:33:31.836 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:33:31.836 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:33:31.836 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:33:31.836 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 49970 2023-09-23T23:33:31.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.836 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:31.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:31.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.816486+0000", 2023-09-23T23:33:31.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:31.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:31.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.816486+0000", 2023-09-23T23:33:31.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:31.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:31.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.816484+0000", 2023-09-23T23:33:31.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:33:31.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:31.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.816510+0000", 2023-09-23T23:33:31.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5409e-05 2023-09-23T23:33:31.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:31.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.816529+0000", 2023-09-23T23:33:31.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9351000000000002e-05 2023-09-23T23:33:31.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:31.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.816554+0000", 2023-09-23T23:33:31.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4561000000000001e-05 2023-09-23T23:33:31.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:31.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.816635+0000", 2023-09-23T23:33:31.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0984999999999998e-05 2023-09-23T23:33:31.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:31.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.816800+0000", 2023-09-23T23:33:31.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000165506 2023-09-23T23:33:31.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:31.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:31.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:33:31.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:31.816962+0000", 2023-09-23T23:33:31.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016161300000000001 2023-09-23T23:33:31.840 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:31.840 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:31.840 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:31.840 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:31.840 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-09-23T23:33:31.840 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 4 2023-09-23T23:33:31.840 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:33:31.848 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:31.848 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 34 16 50011 49995 91.8703 89.9375 0.0148265 0.0108793 2023-09-23T23:33:31.897 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:32.848 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 35 16 51330 51314 91.6006 82.4375 0.00274821 0.0108978 2023-09-23T23:33:33.433 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:33:33.532 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:33 smithi110 ceph-mon[212939]: pgmap v1521: 119 pgs: 119 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T23:33:33.532 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:33 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2733998671' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:33:33.532 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:33 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:33:33.532 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:33 smithi110 ceph-mon[221102]: pgmap v1521: 119 pgs: 119 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T23:33:33.533 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:33 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2733998671' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:33:33.533 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:33 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:33:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:33 smithi138 ceph-mon[193606]: pgmap v1521: 119 pgs: 119 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T23:33:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:33 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2733998671' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:33:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:33 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:33:33.848 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 36 16 52484 52468 91.0595 72.125 0.00593746 0.0109413 2023-09-23T23:33:34.424 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:33:34.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:34 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:33:34.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:34 smithi138 ceph-mon[193606]: failed to encode map e552 with expected crc 2023-09-23T23:33:34.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:34 smithi138 ceph-mon[193606]: failed to encode map e552 with expected crc 2023-09-23T23:33:34.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:34 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:33:34.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:34 smithi138 ceph-mon[193606]: osdmap e552: 8 total, 8 up, 8 in 2023-09-23T23:33:34.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:34 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:33:34.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:34 smithi110 ceph-mon[212939]: failed to encode map e552 with expected crc 2023-09-23T23:33:34.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:34 smithi110 ceph-mon[212939]: failed to encode map e552 with expected crc 2023-09-23T23:33:34.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:34 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:33:34.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:34 smithi110 ceph-mon[212939]: osdmap e552: 8 total, 8 up, 8 in 2023-09-23T23:33:34.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:34 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:33:34.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:34 smithi110 ceph-mon[221102]: failed to encode map e552 with expected crc 2023-09-23T23:33:34.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:34 smithi110 ceph-mon[221102]: failed to encode map e552 with expected crc 2023-09-23T23:33:34.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:34 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:33:34.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:34 smithi110 ceph-mon[221102]: osdmap e552: 8 total, 8 up, 8 in 2023-09-23T23:33:34.849 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 37 16 53795 53779 90.8128 81.9375 0.0128886 0.0110069 2023-09-23T23:33:35.458 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:35.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:35.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:35.513 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:35 smithi110 ceph-mon[212939]: pgmap v1523: 119 pgs: 119 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T23:33:35.513 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:35 smithi110 ceph-mon[221102]: pgmap v1523: 119 pgs: 119 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T23:33:35.564 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:35.639 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:35.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:35 smithi138 ceph-mon[193606]: pgmap v1523: 119 pgs: 119 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T23:33:35.725 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:35.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:35.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:35.832 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:33:35.849 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 38 16 54921 54905 90.2745 70.375 0.0106958 0.0110713 2023-09-23T23:33:35.907 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:35.998 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:33:36.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:36.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:33:36.020 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:33:36.052 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:33:36.088 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:33:36.105 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:36.124 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:33:36.135 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:36.166 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:33:36.167 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:36.216 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:33:36.254 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:33:36.284 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:36.298 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:33:36.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:36.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:36.303 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:33:36.303 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:33:36.303 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:33:36.303 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:33:36.303 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:33:36.304 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:33:36.304 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:33:36.304 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:33:36.399 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:33:36.445 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:36 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2718864353' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:36.457 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:36.550 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:33:36.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:36.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:33:36.590 INFO:teuthology.orchestra.run.smithi110.stderr:Error EINVAL: specified pg_num 71 < pg_num_max 32 2023-09-23T23:33:36.662 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:33:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:36 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2718864353' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:36.718 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:36 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2718864353' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:36.725 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:36.830 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:33:36.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:36.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:33:36.849 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 39 16 56165 56149 89.9533 77.75 0.00910506 0.0111115 2023-09-23T23:33:36.938 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:37.008 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:37.088 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:37.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:37.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:37.199 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:33:37.203 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:33:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:33:36] "GET /metrics HTTP/1.1" 200 36536 "" "Prometheus/2.43.0" 2023-09-23T23:33:37.248 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:37.277 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:33:37.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:37 smithi110 ceph-mon[212939]: pgmap v1524: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T23:33:37.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:37 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2338990403' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-23T23:33:37.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:37 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-23T23:33:37.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:37 smithi110 ceph-mon[221102]: pgmap v1524: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T23:33:37.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:37 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2338990403' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-23T23:33:37.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:37 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-23T23:33:37.849 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:33:37.848286+0000 min lat: 0.00107054 max lat: 2.03043 avg lat: 0.0111595 2023-09-23T23:33:37.849 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:33:37.849 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 40 16 57361 57345 89.5731 74.75 0.00776207 0.0111595 2023-09-23T23:33:37.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:37 smithi138 ceph-mon[193606]: pgmap v1524: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T23:33:37.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:37 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2338990403' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-23T23:33:37.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:37 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-23T23:33:38.086 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:33:38.092 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T23:33:38.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:38.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:33:38.194 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:38.235 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:38.581 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:38.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:38.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:38.686 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:38.714 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:38.755 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:38.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:38.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:38.849 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 41 16 58531 58515 89.1718 73.125 0.00321443 0.0111658 2023-09-23T23:33:38.859 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:33:38.886 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:38.956 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:33:38.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:38.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:33:39.062 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:39.102 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:39.282 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:39.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:39.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:39.321 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:39.391 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:33:39.441 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:39.576 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:33:39.579 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:39 smithi110 ceph-mon[212939]: pgmap v1525: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T23:33:39.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:39.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:33:39.581 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:39 smithi110 ceph-mon[221102]: pgmap v1525: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T23:33:39.682 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:39.742 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:39.849 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 42 16 59845 59829 89.0039 82.125 0.00556631 0.011231 2023-09-23T23:33:39.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:39 smithi138 ceph-mon[193606]: pgmap v1525: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T23:33:40.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:40.038 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:40.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:40.056 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:33:40.101 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:33:40.101 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:33:40.139 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:40.186 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:40.276 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:40.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:40.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:40.381 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:40.431 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:40.849 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 43 16 61025 61009 88.6491 73.75 0.0135954 0.011276 2023-09-23T23:33:41.421 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:41.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:41.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:41.526 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:41.572 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:41.602 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:33:41.666 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:41.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:41.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:41.695 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:41 smithi110 ceph-mon[221102]: pgmap v1526: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T23:33:41.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:41 smithi110 ceph-mon[212939]: pgmap v1526: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T23:33:41.770 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:41.776 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:41.827 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:41 smithi138 ceph-mon[193606]: pgmap v1526: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T23:33:41.830 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:41.849 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 44 16 62187 62171 88.2849 72.625 0.00660671 0.0113202 2023-09-23T23:33:41.918 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:41.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:41.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:42.028 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:33:42.071 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:42.216 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:33:42.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:42.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:33:42.324 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:42.384 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:42.594 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:42.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:42.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:42.706 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:42.744 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:42.849 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 45 16 63445 63429 88.0701 78.625 0.00453436 0.0113499 2023-09-23T23:33:43.093 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 7, 2, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:33:43.093 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:33:43.093 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 3, 4, 5, 7, 2, 1, 6] 2023-09-23T23:33:43.093 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd out 4 2023-09-23T23:33:43.255 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:43.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:43.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:43.354 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:43.361 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:43.458 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:43.522 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:33:43.553 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:43.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:43.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:43.659 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:43.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:43 smithi110 ceph-mon[212939]: pgmap v1527: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T23:33:43.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:43 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1198804246' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:33:43.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:43 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:33:43.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:33:43.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:43 smithi110 ceph-mon[221102]: pgmap v1527: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T23:33:43.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:43 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1198804246' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:33:43.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:43 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:33:43.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:33:43.740 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:43.850 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 46 16 64682 64666 87.8361 77.3125 0.00465375 0.0113795 2023-09-23T23:33:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:43 smithi138 ceph-mon[193606]: pgmap v1527: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T23:33:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:43 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1198804246' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:33:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:43 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:33:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:33:44.771 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:33:44.772 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:33:44.772 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[221102]: osdmap e553: 8 total, 8 up, 8 in 2023-09-23T23:33:44.772 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[221102]: failed to encode map e553 with expected crc 2023-09-23T23:33:44.772 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[221102]: failed to encode map e553 with expected crc 2023-09-23T23:33:44.772 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[221102]: failed to encode map e553 with expected crc 2023-09-23T23:33:44.772 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[221102]: failed to encode map e553 with expected crc 2023-09-23T23:33:44.772 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-09-23T23:33:44.772 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T23:33:44.772 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-09-23T23:33:44.773 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T23:33:44.773 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:33:44.773 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:33:44.773 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[212939]: osdmap e553: 8 total, 8 up, 8 in 2023-09-23T23:33:44.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[212939]: failed to encode map e553 with expected crc 2023-09-23T23:33:44.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[212939]: failed to encode map e553 with expected crc 2023-09-23T23:33:44.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[212939]: failed to encode map e553 with expected crc 2023-09-23T23:33:44.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[212939]: failed to encode map e553 with expected crc 2023-09-23T23:33:44.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-09-23T23:33:44.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T23:33:44.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-09-23T23:33:44.774 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:44 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T23:33:44.849 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 47 16 65687 65671 87.3036 62.8125 0.00812119 0.0114493 2023-09-23T23:33:44.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:44 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:33:44.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:44 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:33:44.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:44 smithi138 ceph-mon[193606]: osdmap e553: 8 total, 8 up, 8 in 2023-09-23T23:33:44.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:44 smithi138 ceph-mon[193606]: failed to encode map e553 with expected crc 2023-09-23T23:33:44.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:44 smithi138 ceph-mon[193606]: failed to encode map e553 with expected crc 2023-09-23T23:33:44.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:44 smithi138 ceph-mon[193606]: failed to encode map e553 with expected crc 2023-09-23T23:33:44.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:44 smithi138 ceph-mon[193606]: failed to encode map e553 with expected crc 2023-09-23T23:33:44.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:44 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-09-23T23:33:44.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:44 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T23:33:44.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:44 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-09-23T23:33:44.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:44 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2023-09-23T23:33:45.024 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:45.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:45.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:45.131 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:45.158 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:45.456 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:45.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:45.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:45.563 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:33:45.621 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:45.725 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:33:45.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:45.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:33:45.767 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:45.837 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:45.849 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 48 16 66663 66647 86.7557 61 0.00888316 0.0115207 2023-09-23T23:33:45.886 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:33:45.887 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:45 smithi110 ceph-mon[221102]: pgmap v1529: 119 pgs: 119 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-09-23T23:33:45.888 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:45 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-09-23T23:33:45.888 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:45 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2023-09-23T23:33:45.888 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:45 smithi110 ceph-mon[221102]: osdmap e554: 8 total, 8 up, 8 in 2023-09-23T23:33:45.888 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:45 smithi110 ceph-mon[221102]: failed to encode map e554 with expected crc 2023-09-23T23:33:45.888 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:45 smithi110 ceph-mon[221102]: failed to encode map e554 with expected crc 2023-09-23T23:33:45.888 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:45 smithi110 ceph-mon[221102]: failed to encode map e554 with expected crc 2023-09-23T23:33:45.892 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:45 smithi110 ceph-mon[212939]: pgmap v1529: 119 pgs: 119 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-09-23T23:33:45.892 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:45 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-09-23T23:33:45.893 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:45 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2023-09-23T23:33:45.893 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:45 smithi110 ceph-mon[212939]: osdmap e554: 8 total, 8 up, 8 in 2023-09-23T23:33:45.893 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:45 smithi110 ceph-mon[212939]: failed to encode map e554 with expected crc 2023-09-23T23:33:45.893 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:45 smithi110 ceph-mon[212939]: failed to encode map e554 with expected crc 2023-09-23T23:33:45.893 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:45 smithi110 ceph-mon[212939]: failed to encode map e554 with expected crc 2023-09-23T23:33:45.894 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:45.926 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:33:45.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:45 smithi138 ceph-mon[193606]: pgmap v1529: 119 pgs: 119 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-09-23T23:33:45.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:45 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-09-23T23:33:45.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:45 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2023-09-23T23:33:45.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:45 smithi138 ceph-mon[193606]: osdmap e554: 8 total, 8 up, 8 in 2023-09-23T23:33:45.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:45 smithi138 ceph-mon[193606]: failed to encode map e554 with expected crc 2023-09-23T23:33:45.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:45 smithi138 ceph-mon[193606]: failed to encode map e554 with expected crc 2023-09-23T23:33:45.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:45 smithi138 ceph-mon[193606]: failed to encode map e554 with expected crc 2023-09-23T23:33:45.959 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:33:45.994 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:33:46.012 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:46.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:46.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:46.030 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:33:46.072 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:33:46.116 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:33:46.117 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:33:46.153 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:33:46.165 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:33:46.165 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:33:46.165 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:33:46.165 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:33:46.165 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:33:46.166 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:33:46.166 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:33:46.166 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:33:46.175 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:46.275 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:33:46.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:46.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:33:46.389 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:46.392 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:33:46.392 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:33:46.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:33:46.392 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:33:46.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.393 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:175248 12.c 12:307f7e7c:::benchmark_data_smithi110_491193_object80373:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2023-09-23T23:33:46.393 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.412665+0000", 2023-09-23T23:33:46.393 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 98.974258078000005, 2023-09-23T23:33:46.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.29629335899999998, 2023-09-23T23:33:46.393 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.393 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:33:46.393 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:33:46.393 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:33:46.393 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:33:46.393 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175248 2023-09-23T23:33:46.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.394 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.412665+0000", 2023-09-23T23:33:46.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.412665+0000", 2023-09-23T23:33:46.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.412663+0000", 2023-09-23T23:33:46.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:33:46.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.412669+0000", 2023-09-23T23:33:46.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2099999999999998e-06 2023-09-23T23:33:46.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.412681+0000", 2023-09-23T23:33:46.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1521e-05 2023-09-23T23:33:46.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.412688+0000", 2023-09-23T23:33:46.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2219999999999996e-06 2023-09-23T23:33:46.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.414735+0000", 2023-09-23T23:33:46.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020465000000000001 2023-09-23T23:33:46.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.397 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.414851+0000", 2023-09-23T23:33:46.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000116075 2023-09-23T23:33:46.397 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.397 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.397 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:33:46.397 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.414953+0000", 2023-09-23T23:33:46.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102536 2023-09-23T23:33:46.397 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.397 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.397 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:33:46.397 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.418389+0000", 2023-09-23T23:33:46.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034361230000000001 2023-09-23T23:33:46.397 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.398 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.398 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.398 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.424325+0000", 2023-09-23T23:33:46.398 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.005935671 2023-09-23T23:33:46.398 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.398 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.398 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.398 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708871+0000", 2023-09-23T23:33:46.398 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28454617199999999 2023-09-23T23:33:46.398 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.398 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.398 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:33:46.398 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708921+0000", 2023-09-23T23:33:46.399 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9780000000000001e-05 2023-09-23T23:33:46.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.399 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.399 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708958+0000", 2023-09-23T23:33:46.399 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7088000000000001e-05 2023-09-23T23:33:46.399 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.399 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.399 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.399 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:175252 12.c 12:31048612:::benchmark_data_smithi110_491193_object80377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2023-09-23T23:33:46.399 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.420253+0000", 2023-09-23T23:33:46.400 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 98.966669901000003, 2023-09-23T23:33:46.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28881690999999998, 2023-09-23T23:33:46.400 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.400 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:33:46.400 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:33:46.400 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:33:46.400 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:33:46.400 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175252 2023-09-23T23:33:46.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.400 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.400 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.420253+0000", 2023-09-23T23:33:46.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.420253+0000", 2023-09-23T23:33:46.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.420252+0000", 2023-09-23T23:33:46.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:33:46.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.420257+0000", 2023-09-23T23:33:46.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3079999999999998e-06 2023-09-23T23:33:46.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.420266+0000", 2023-09-23T23:33:46.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4320000000000005e-06 2023-09-23T23:33:46.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.420273+0000", 2023-09-23T23:33:46.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3849999999999999e-06 2023-09-23T23:33:46.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.420499+0000", 2023-09-23T23:33:46.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000226515 2023-09-23T23:33:46.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.420656+0000", 2023-09-23T23:33:46.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015663299999999999 2023-09-23T23:33:46.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.404 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.404 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:33:46.404 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.420785+0000", 2023-09-23T23:33:46.404 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012860599999999999 2023-09-23T23:33:46.404 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.404 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.404 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:33:46.404 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433627+0000", 2023-09-23T23:33:46.404 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012842851000000001 2023-09-23T23:33:46.404 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.404 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.404 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.404 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.437040+0000", 2023-09-23T23:33:46.405 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034124540000000001 2023-09-23T23:33:46.405 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.405 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.405 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.405 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708990+0000", 2023-09-23T23:33:46.405 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.27195066400000001 2023-09-23T23:33:46.405 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.405 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.405 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:33:46.405 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.709023+0000", 2023-09-23T23:33:46.405 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2455e-05 2023-09-23T23:33:46.405 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.405 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.406 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.406 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.709070+0000", 2023-09-23T23:33:46.406 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6922999999999998e-05 2023-09-23T23:33:46.406 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.406 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.406 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.406 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.406 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.406 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:175256 12.c 12:370f9430:::benchmark_data_smithi110_491193_object80381:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2023-09-23T23:33:46.406 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.421218+0000", 2023-09-23T23:33:46.406 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 98.965704591999994, 2023-09-23T23:33:46.406 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28794408100000002, 2023-09-23T23:33:46.406 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.407 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:33:46.407 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:33:46.407 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:33:46.407 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:33:46.407 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175256 2023-09-23T23:33:46.407 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.407 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.407 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.407 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.407 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.421218+0000", 2023-09-23T23:33:46.407 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.407 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.408 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.408 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.421218+0000", 2023-09-23T23:33:46.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.408 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.408 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.408 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.421217+0000", 2023-09-23T23:33:46.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:33:46.408 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.408 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.408 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.421223+0000", 2023-09-23T23:33:46.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4870000000000002e-06 2023-09-23T23:33:46.409 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.409 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.409 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.409 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.421234+0000", 2023-09-23T23:33:46.409 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1583000000000001e-05 2023-09-23T23:33:46.409 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.409 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.409 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.409 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.421242+0000", 2023-09-23T23:33:46.409 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5939999999999996e-06 2023-09-23T23:33:46.409 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.409 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.409 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.410 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.424374+0000", 2023-09-23T23:33:46.410 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031317889999999998 2023-09-23T23:33:46.410 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.410 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.410 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.410 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.424503+0000", 2023-09-23T23:33:46.410 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000129553 2023-09-23T23:33:46.410 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.410 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.410 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:33:46.410 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.424650+0000", 2023-09-23T23:33:46.410 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000146971 2023-09-23T23:33:46.411 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.411 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:33:46.411 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436976+0000", 2023-09-23T23:33:46.411 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012326044 2023-09-23T23:33:46.411 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.411 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.411 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.441345+0000", 2023-09-23T23:33:46.411 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043688800000000003 2023-09-23T23:33:46.411 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.411 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.412 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.709100+0000", 2023-09-23T23:33:46.412 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.26775439299999998 2023-09-23T23:33:46.412 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.412 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.412 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:33:46.412 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.709132+0000", 2023-09-23T23:33:46.412 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2806999999999999e-05 2023-09-23T23:33:46.412 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.412 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.412 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.412 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.709162+0000", 2023-09-23T23:33:46.412 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9899999999999998e-05 2023-09-23T23:33:46.412 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.413 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.413 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.413 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.413 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.413 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:175257 12.c 12:3b546749:::benchmark_data_smithi110_491193_object80382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2023-09-23T23:33:46.413 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.421310+0000", 2023-09-23T23:33:46.413 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 98.965612816999993, 2023-09-23T23:33:46.413 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28794258499999997, 2023-09-23T23:33:46.413 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.413 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:33:46.413 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:33:46.413 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:33:46.413 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:33:46.414 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175257 2023-09-23T23:33:46.414 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.414 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.414 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.414 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.414 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.421310+0000", 2023-09-23T23:33:46.414 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.414 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.414 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.414 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.414 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.421310+0000", 2023-09-23T23:33:46.414 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.415 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.415 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.415 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.415 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.421309+0000", 2023-09-23T23:33:46.415 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:33:46.415 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.415 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.415 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.415 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.421313+0000", 2023-09-23T23:33:46.415 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.138e-06 2023-09-23T23:33:46.415 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.415 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.415 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.416 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.421322+0000", 2023-09-23T23:33:46.416 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8000000000000004e-06 2023-09-23T23:33:46.416 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.416 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.416 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.416 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.421328+0000", 2023-09-23T23:33:46.416 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5879999999999997e-06 2023-09-23T23:33:46.416 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.416 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.416 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.416 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428117+0000", 2023-09-23T23:33:46.416 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067891139999999997 2023-09-23T23:33:46.416 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.416 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.417 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.417 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428225+0000", 2023-09-23T23:33:46.417 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000108516 2023-09-23T23:33:46.417 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.417 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.417 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:33:46.417 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428307+0000", 2023-09-23T23:33:46.417 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1785000000000004e-05 2023-09-23T23:33:46.417 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.417 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.417 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:33:46.417 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436983+0000", 2023-09-23T23:33:46.417 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086761430000000007 2023-09-23T23:33:46.418 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.418 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.418 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.418 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.441380+0000", 2023-09-23T23:33:46.418 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043963739999999998 2023-09-23T23:33:46.418 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.418 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.418 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.418 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.709191+0000", 2023-09-23T23:33:46.418 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.26781119199999998 2023-09-23T23:33:46.418 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.418 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.418 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:33:46.419 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.709223+0000", 2023-09-23T23:33:46.419 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1934000000000003e-05 2023-09-23T23:33:46.419 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.419 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.419 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.419 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.709253+0000", 2023-09-23T23:33:46.419 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.974e-05 2023-09-23T23:33:46.419 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.419 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.419 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.419 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.419 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.419 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:175259 12.c 12:3a3a4253:::benchmark_data_smithi110_491193_object80384:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2023-09-23T23:33:46.420 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.423074+0000", 2023-09-23T23:33:46.420 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 98.963848670000004, 2023-09-23T23:33:46.420 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.30677402500000001, 2023-09-23T23:33:46.420 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.420 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:33:46.420 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:33:46.420 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:33:46.420 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:33:46.420 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175259 2023-09-23T23:33:46.420 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.420 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.420 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.420 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.421 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.423074+0000", 2023-09-23T23:33:46.421 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.421 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.421 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.421 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.421 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.423074+0000", 2023-09-23T23:33:46.421 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.421 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.421 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.421 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.421 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.423073+0000", 2023-09-23T23:33:46.421 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:33:46.421 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.422 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.422 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.422 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.423079+0000", 2023-09-23T23:33:46.422 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5300000000000004e-06 2023-09-23T23:33:46.422 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.422 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.422 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.422 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.423093+0000", 2023-09-23T23:33:46.422 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4314999999999999e-05 2023-09-23T23:33:46.422 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.422 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.422 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.423 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.423100+0000", 2023-09-23T23:33:46.423 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2840000000000001e-06 2023-09-23T23:33:46.423 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.423 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.423 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.423 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433657+0000", 2023-09-23T23:33:46.423 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010556528000000001 2023-09-23T23:33:46.423 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.423 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.423 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.423 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433752+0000", 2023-09-23T23:33:46.423 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5221000000000002e-05 2023-09-23T23:33:46.423 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.424 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.424 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:33:46.424 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433841+0000", 2023-09-23T23:33:46.424 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8955999999999993e-05 2023-09-23T23:33:46.424 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.424 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.424 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:33:46.424 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.437065+0000", 2023-09-23T23:33:46.424 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003224219 2023-09-23T23:33:46.424 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.424 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.424 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.424 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.441479+0000", 2023-09-23T23:33:46.425 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.004414177 2023-09-23T23:33:46.425 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.425 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.425 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.425 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.729759+0000", 2023-09-23T23:33:46.425 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28828009199999999 2023-09-23T23:33:46.425 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.425 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.425 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:33:46.425 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.729805+0000", 2023-09-23T23:33:46.425 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5633999999999997e-05 2023-09-23T23:33:46.425 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.425 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.426 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.426 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.729848+0000", 2023-09-23T23:33:46.426 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3220999999999998e-05 2023-09-23T23:33:46.426 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.426 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.426 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.426 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.426 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.426 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:175272 12.c 12:339207bf:::benchmark_data_smithi110_491193_object80397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2023-09-23T23:33:46.426 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.442607+0000", 2023-09-23T23:33:46.426 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 98.944315981000003, 2023-09-23T23:33:46.426 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28735201900000001, 2023-09-23T23:33:46.427 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.427 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:33:46.427 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:33:46.427 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:33:46.427 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:33:46.427 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175272 2023-09-23T23:33:46.427 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.427 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.427 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.427 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.427 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.442607+0000", 2023-09-23T23:33:46.427 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.427 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.428 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.428 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.428 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.442607+0000", 2023-09-23T23:33:46.428 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.428 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.428 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.428 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.428 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.442606+0000", 2023-09-23T23:33:46.428 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:33:46.428 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.428 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.428 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.428 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.442612+0000", 2023-09-23T23:33:46.428 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2260000000000002e-06 2023-09-23T23:33:46.429 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.429 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.429 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.429 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.442623+0000", 2023-09-23T23:33:46.429 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1623999999999999e-05 2023-09-23T23:33:46.429 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.429 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.429 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.429 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.442628+0000", 2023-09-23T23:33:46.429 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.916e-06 2023-09-23T23:33:46.429 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.429 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.429 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.430 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.442677+0000", 2023-09-23T23:33:46.430 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8599e-05 2023-09-23T23:33:46.430 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.430 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.430 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.430 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.442807+0000", 2023-09-23T23:33:46.430 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000129887 2023-09-23T23:33:46.430 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.430 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.430 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:33:46.430 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.442940+0000", 2023-09-23T23:33:46.430 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013285900000000001 2023-09-23T23:33:46.431 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.431 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.431 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:33:46.431 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.446023+0000", 2023-09-23T23:33:46.431 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030831019999999999 2023-09-23T23:33:46.431 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.431 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.431 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.431 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.446724+0000", 2023-09-23T23:33:46.431 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00070103699999999995 2023-09-23T23:33:46.431 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.431 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.431 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.432 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.729889+0000", 2023-09-23T23:33:46.432 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28316529699999998 2023-09-23T23:33:46.432 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.432 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.432 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:33:46.432 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.729928+0000", 2023-09-23T23:33:46.432 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8432000000000001e-05 2023-09-23T23:33:46.432 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.432 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.432 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.432 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.729959+0000", 2023-09-23T23:33:46.432 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1291999999999998e-05 2023-09-23T23:33:46.432 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.433 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.433 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.433 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.433 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.433 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:175281 12.c 12:3c0f49cd:::benchmark_data_smithi110_491193_object80406:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2023-09-23T23:33:46.433 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.467104+0000", 2023-09-23T23:33:46.433 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 98.919818579999998, 2023-09-23T23:33:46.433 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.26297548100000001, 2023-09-23T23:33:46.433 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.433 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:33:46.433 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:33:46.433 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:33:46.434 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:33:46.434 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175281 2023-09-23T23:33:46.434 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.434 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.434 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.434 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.434 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.467104+0000", 2023-09-23T23:33:46.434 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.434 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.434 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.434 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.434 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.467104+0000", 2023-09-23T23:33:46.434 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.435 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.435 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.435 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.435 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.467103+0000", 2023-09-23T23:33:46.435 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:33:46.435 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.435 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.435 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.435 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.467109+0000", 2023-09-23T23:33:46.435 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8919999999999997e-06 2023-09-23T23:33:46.435 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.435 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.435 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.436 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.467123+0000", 2023-09-23T23:33:46.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4491000000000001e-05 2023-09-23T23:33:46.436 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.436 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.436 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.467128+0000", 2023-09-23T23:33:46.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7090000000000001e-06 2023-09-23T23:33:46.436 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.436 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.436 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.467208+0000", 2023-09-23T23:33:46.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9699999999999999e-05 2023-09-23T23:33:46.436 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.437 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.467370+0000", 2023-09-23T23:33:46.437 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000162795 2023-09-23T23:33:46.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:33:46.437 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.467528+0000", 2023-09-23T23:33:46.437 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015793099999999999 2023-09-23T23:33:46.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.471218+0000", 2023-09-23T23:33:46.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036897919999999999 2023-09-23T23:33:46.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.438 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:33:46.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.471334+0000", 2023-09-23T23:33:46.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011618899999999999 2023-09-23T23:33:46.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.438 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.730001+0000", 2023-09-23T23:33:46.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.25866644900000002 2023-09-23T23:33:46.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.439 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.439 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:33:46.439 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.730043+0000", 2023-09-23T23:33:46.439 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2586999999999997e-05 2023-09-23T23:33:46.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.439 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.439 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.439 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.730080+0000", 2023-09-23T23:33:46.439 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6448000000000001e-05 2023-09-23T23:33:46.439 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.439 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.439 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.440 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.440 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:176321 12.19 12:98530c5b:::benchmark_data_smithi110_491193_object81446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T23:33:46.440 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:09.099805+0000", 2023-09-23T23:33:46.440 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 97.287117590999998, 2023-09-23T23:33:46.440 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.26881945099999999, 2023-09-23T23:33:46.440 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.440 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:33:46.440 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:33:46.440 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:33:46.440 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:33:46.440 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 176321 2023-09-23T23:33:46.440 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.440 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.441 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.441 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.099805+0000", 2023-09-23T23:33:46.441 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.441 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.441 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.441 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.099805+0000", 2023-09-23T23:33:46.441 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.441 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.441 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.442 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.099805+0000", 2023-09-23T23:33:46.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:33:46.442 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.442 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.442 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.442 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.099808+0000", 2023-09-23T23:33:46.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7810000000000001e-06 2023-09-23T23:33:46.442 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.442 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.442 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.442 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.099813+0000", 2023-09-23T23:33:46.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6099999999999997e-06 2023-09-23T23:33:46.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.443 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.099818+0000", 2023-09-23T23:33:46.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4789999999999998e-06 2023-09-23T23:33:46.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.443 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.103758+0000", 2023-09-23T23:33:46.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003940018 2023-09-23T23:33:46.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.103877+0000", 2023-09-23T23:33:46.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000119753 2023-09-23T23:33:46.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.444 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:33:46.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.104333+0000", 2023-09-23T23:33:46.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045564200000000003 2023-09-23T23:33:46.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.444 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.112833+0000", 2023-09-23T23:33:46.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0085003379999999996 2023-09-23T23:33:46.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.445 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.445 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:33:46.445 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.112855+0000", 2023-09-23T23:33:46.445 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1500000000000001e-05 2023-09-23T23:33:46.445 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.445 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.445 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.445 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.368191+0000", 2023-09-23T23:33:46.445 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.25533642400000001 2023-09-23T23:33:46.445 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.445 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.445 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:33:46.445 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.368240+0000", 2023-09-23T23:33:46.446 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8585999999999999e-05 2023-09-23T23:33:46.446 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.446 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.446 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.368625+0000", 2023-09-23T23:33:46.446 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000384736 2023-09-23T23:33:46.446 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.446 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.446 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.446 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.446 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:176324 12.f 12:f7bca133:::benchmark_data_smithi110_491193_object81449:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T23:33:46.447 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:09.099827+0000", 2023-09-23T23:33:46.447 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 97.287095937000004, 2023-09-23T23:33:46.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.26855114899999999, 2023-09-23T23:33:46.447 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.447 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:33:46.447 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:33:46.447 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:33:46.447 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:33:46.447 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 176324 2023-09-23T23:33:46.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.447 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.447 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.447 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.099827+0000", 2023-09-23T23:33:46.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.099827+0000", 2023-09-23T23:33:46.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.099827+0000", 2023-09-23T23:33:46.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:33:46.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.099829+0000", 2023-09-23T23:33:46.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0159999999999998e-06 2023-09-23T23:33:46.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.099834+0000", 2023-09-23T23:33:46.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9450000000000001e-06 2023-09-23T23:33:46.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.099838+0000", 2023-09-23T23:33:46.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0450000000000002e-06 2023-09-23T23:33:46.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.450 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.103410+0000", 2023-09-23T23:33:46.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035726949999999999 2023-09-23T23:33:46.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.450 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.103534+0000", 2023-09-23T23:33:46.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000123449 2023-09-23T23:33:46.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.451 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.451 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:33:46.451 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.103677+0000", 2023-09-23T23:33:46.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014354299999999999 2023-09-23T23:33:46.451 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.451 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.451 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.451 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.112615+0000", 2023-09-23T23:33:46.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0089377429999999997 2023-09-23T23:33:46.451 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.451 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.451 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:33:46.451 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.114025+0000", 2023-09-23T23:33:46.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014102139999999999 2023-09-23T23:33:46.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.366874+0000", 2023-09-23T23:33:46.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.252848668 2023-09-23T23:33:46.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:33:46.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.366916+0000", 2023-09-23T23:33:46.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2150999999999997e-05 2023-09-23T23:33:46.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.368378+0000", 2023-09-23T23:33:46.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014619990000000001 2023-09-23T23:33:46.453 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.453 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.453 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.453 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:176314 12.8 e525/490 12:10581712:::benchmark_data_smithi110_491193_object81439:head v 525'7698, mlcod=525'7696)", 2023-09-23T23:33:46.453 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:09.110488+0000", 2023-09-23T23:33:46.453 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 97.276435057, 2023-09-23T23:33:46.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.25630956100000002, 2023-09-23T23:33:46.453 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.454 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:33:46.454 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.110488+0000", 2023-09-23T23:33:46.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.110488+0000", 2023-09-23T23:33:46.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.110487+0000", 2023-09-23T23:33:46.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:33:46.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.110494+0000", 2023-09-23T23:33:46.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4110000000000001e-06 2023-09-23T23:33:46.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.110501+0000", 2023-09-23T23:33:46.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7400000000000004e-06 2023-09-23T23:33:46.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.456 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.110508+0000", 2023-09-23T23:33:46.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4740000000000002e-06 2023-09-23T23:33:46.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.456 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.110570+0000", 2023-09-23T23:33:46.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1904999999999998e-05 2023-09-23T23:33:46.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.457 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.457 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.457 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.110589+0000", 2023-09-23T23:33:46.457 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.942e-05 2023-09-23T23:33:46.457 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.457 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.457 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:33:46.457 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.117462+0000", 2023-09-23T23:33:46.457 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068724490000000001 2023-09-23T23:33:46.457 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.457 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.457 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.457 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.366797+0000", 2023-09-23T23:33:46.458 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.249335786 2023-09-23T23:33:46.458 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.458 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.458 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.458 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.458 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:176333 12.0 e525/501 12:02ab3fb1:::benchmark_data_smithi110_491193_object81458:head v 525'7877, mlcod=525'7875)", 2023-09-23T23:33:46.458 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:09.116246+0000", 2023-09-23T23:33:46.458 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 97.270676828000006, 2023-09-23T23:33:46.458 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.25945619599999997, 2023-09-23T23:33:46.458 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.458 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:33:46.458 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.459 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.116246+0000", 2023-09-23T23:33:46.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.459 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.459 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.116246+0000", 2023-09-23T23:33:46.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.459 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.459 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.116245+0000", 2023-09-23T23:33:46.460 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:33:46.460 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.460 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.460 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.460 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.116252+0000", 2023-09-23T23:33:46.460 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8229999999999997e-06 2023-09-23T23:33:46.460 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.460 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.460 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.460 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.116259+0000", 2023-09-23T23:33:46.460 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9369999999999998e-06 2023-09-23T23:33:46.460 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.460 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.461 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.116263+0000", 2023-09-23T23:33:46.461 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1529999999999999e-06 2023-09-23T23:33:46.461 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.461 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.117424+0000", 2023-09-23T23:33:46.461 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011606629999999999 2023-09-23T23:33:46.461 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.461 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.117443+0000", 2023-09-23T23:33:46.461 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9398999999999998e-05 2023-09-23T23:33:46.461 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:33:46.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.375668+0000", 2023-09-23T23:33:46.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.25822476799999999 2023-09-23T23:33:46.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.375702+0000", 2023-09-23T23:33:46.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4112999999999997e-05 2023-09-23T23:33:46.462 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.462 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.462 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.463 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.46229.0:176321 12.19 e525/496 ondisk, result = 0)", 2023-09-23T23:33:46.463 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:09.117588+0000", 2023-09-23T23:33:46.463 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 97.269334486999995, 2023-09-23T23:33:46.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.25104239499999997, 2023-09-23T23:33:46.463 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.463 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:33:46.463 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.463 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.463 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.117588+0000", 2023-09-23T23:33:46.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.117588+0000", 2023-09-23T23:33:46.464 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.117588+0000", 2023-09-23T23:33:46.464 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:33:46.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.465 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.117593+0000", 2023-09-23T23:33:46.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8069999999999999e-06 2023-09-23T23:33:46.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.465 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.465 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.366717+0000", 2023-09-23T23:33:46.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.24912448800000001 2023-09-23T23:33:46.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.465 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.465 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.366745+0000", 2023-09-23T23:33:46.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7826000000000001e-05 2023-09-23T23:33:46.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.466 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.466 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.466 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.368174+0000", 2023-09-23T23:33:46.466 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001428885 2023-09-23T23:33:46.466 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.466 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.466 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.466 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.368183+0000", 2023-09-23T23:33:46.466 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.679e-06 2023-09-23T23:33:46.466 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.466 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.466 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.467 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.368631+0000", 2023-09-23T23:33:46.467 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00044733000000000001 2023-09-23T23:33:46.467 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.467 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.467 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.467 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGRecoveryDeleteReply(12.19 e525,496 12:9c2c67bb:::benchmark_data_smithi110_491193_object14449:head,503'5586)", 2023-09-23T23:33:46.467 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:09.118314+0000", 2023-09-23T23:33:46.467 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 97.268608869999994, 2023-09-23T23:33:46.467 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.24982463799999999, 2023-09-23T23:33:46.467 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.467 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:33:46.468 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.468 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.468 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.118314+0000", 2023-09-23T23:33:46.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.468 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.468 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.468 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.118314+0000", 2023-09-23T23:33:46.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.468 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.469 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.469 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.118313+0000", 2023-09-23T23:33:46.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:33:46.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.469 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.469 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.469 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.118324+0000", 2023-09-23T23:33:46.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0694e-05 2023-09-23T23:33:46.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.469 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.469 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.469 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.366723+0000", 2023-09-23T23:33:46.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.24839940799999999 2023-09-23T23:33:46.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.366755+0000", 2023-09-23T23:33:46.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1996000000000003e-05 2023-09-23T23:33:46.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.366827+0000", 2023-09-23T23:33:46.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2062000000000002e-05 2023-09-23T23:33:46.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:09.368139+0000", 2023-09-23T23:33:46.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013111889999999999 2023-09-23T23:33:46.471 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.471 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.471 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.471 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:29502 13.a 13:568f7efd:::benchmark_data_smithi110_541928_object29501:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e549)", 2023-09-23T23:33:46.471 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:15.826202+0000", 2023-09-23T23:33:46.471 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 30.560720513, 2023-09-23T23:33:46.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.26484817700000002, 2023-09-23T23:33:46.471 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.471 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:33:46.472 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:33:46.472 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:33:46.472 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:33:46.472 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 29502 2023-09-23T23:33:46.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.472 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.826202+0000", 2023-09-23T23:33:46.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.826202+0000", 2023-09-23T23:33:46.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.826201+0000", 2023-09-23T23:33:46.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:33:46.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.826230+0000", 2023-09-23T23:33:46.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8612e-05 2023-09-23T23:33:46.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.826238+0000", 2023-09-23T23:33:46.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6890000000000004e-06 2023-09-23T23:33:46.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.826243+0000", 2023-09-23T23:33:46.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2039999999999996e-06 2023-09-23T23:33:46.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.826293+0000", 2023-09-23T23:33:46.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9906e-05 2023-09-23T23:33:46.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.826435+0000", 2023-09-23T23:33:46.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014220500000000001 2023-09-23T23:33:46.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:33:46.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.826578+0000", 2023-09-23T23:33:46.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014273700000000001 2023-09-23T23:33:46.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:33:46.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.827893+0000", 2023-09-23T23:33:46.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001315423 2023-09-23T23:33:46.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.828007+0000", 2023-09-23T23:33:46.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000114126 2023-09-23T23:33:46.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.090968+0000", 2023-09-23T23:33:46.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.26296108800000001 2023-09-23T23:33:46.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.477 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.477 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:33:46.477 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.091010+0000", 2023-09-23T23:33:46.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1535999999999997e-05 2023-09-23T23:33:46.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.477 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.477 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.477 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.091051+0000", 2023-09-23T23:33:46.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0503000000000002e-05 2023-09-23T23:33:46.477 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.477 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.478 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.478 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:29513 13.8 13:19b99eea:::benchmark_data_smithi110_541928_object29512:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e549)", 2023-09-23T23:33:46.478 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:15.831352+0000", 2023-09-23T23:33:46.478 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 30.555570916000001, 2023-09-23T23:33:46.478 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.26185462500000001, 2023-09-23T23:33:46.478 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.478 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:33:46.478 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:33:46.478 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:33:46.478 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:33:46.479 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 29513 2023-09-23T23:33:46.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.479 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.831352+0000", 2023-09-23T23:33:46.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.831352+0000", 2023-09-23T23:33:46.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.831351+0000", 2023-09-23T23:33:46.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:33:46.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.831370+0000", 2023-09-23T23:33:46.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9281000000000001e-05 2023-09-23T23:33:46.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.831379+0000", 2023-09-23T23:33:46.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1400000000000006e-06 2023-09-23T23:33:46.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.831385+0000", 2023-09-23T23:33:46.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8560000000000003e-06 2023-09-23T23:33:46.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.831421+0000", 2023-09-23T23:33:46.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5703e-05 2023-09-23T23:33:46.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.831560+0000", 2023-09-23T23:33:46.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000138608 2023-09-23T23:33:46.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:33:46.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.831692+0000", 2023-09-23T23:33:46.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013254 2023-09-23T23:33:46.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.090918+0000", 2023-09-23T23:33:46.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.25922558899999998 2023-09-23T23:33:46.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:33:46.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.093104+0000", 2023-09-23T23:33:46.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021865539999999998 2023-09-23T23:33:46.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.093133+0000", 2023-09-23T23:33:46.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8549000000000001e-05 2023-09-23T23:33:46.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:33:46.484 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.093168+0000", 2023-09-23T23:33:46.484 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5083000000000003e-05 2023-09-23T23:33:46.484 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.484 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.484 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.093207+0000", 2023-09-23T23:33:46.484 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8729000000000001e-05 2023-09-23T23:33:46.484 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.484 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.484 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.484 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.484 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:29510 13.0 e549/534 13:087ce450:::benchmark_data_smithi110_541928_object29509:head v 549'1821, mlcod=549'1819)", 2023-09-23T23:33:46.485 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:15.831420+0000", 2023-09-23T23:33:46.485 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 30.555502995000001, 2023-09-23T23:33:46.485 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.26167215199999999, 2023-09-23T23:33:46.485 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.485 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:33:46.485 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.485 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.485 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.485 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.831420+0000", 2023-09-23T23:33:46.485 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.485 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.485 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.485 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.486 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.831420+0000", 2023-09-23T23:33:46.486 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.486 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.486 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.486 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.831419+0000", 2023-09-23T23:33:46.486 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:33:46.486 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.486 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.486 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.831481+0000", 2023-09-23T23:33:46.486 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1914000000000004e-05 2023-09-23T23:33:46.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.831489+0000", 2023-09-23T23:33:46.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1149999999999994e-06 2023-09-23T23:33:46.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.831493+0000", 2023-09-23T23:33:46.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.929e-06 2023-09-23T23:33:46.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.831525+0000", 2023-09-23T23:33:46.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1656999999999999e-05 2023-09-23T23:33:46.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.831555+0000", 2023-09-23T23:33:46.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0103e-05 2023-09-23T23:33:46.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:33:46.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.093058+0000", 2023-09-23T23:33:46.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.26150272899999999 2023-09-23T23:33:46.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.489 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.489 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.093092+0000", 2023-09-23T23:33:46.489 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4347000000000002e-05 2023-09-23T23:33:46.489 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.489 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.489 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.489 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.489 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:29512 13.0 e549/534 13:0bc9f7ab:::benchmark_data_smithi110_541928_object29511:head v 549'1822, mlcod=549'1819)", 2023-09-23T23:33:46.489 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:15.832127+0000", 2023-09-23T23:33:46.489 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 30.554795340999998, 2023-09-23T23:33:46.489 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.26198722200000002, 2023-09-23T23:33:46.490 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.490 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:33:46.490 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.490 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.490 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.490 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.832127+0000", 2023-09-23T23:33:46.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.490 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.490 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.490 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.832127+0000", 2023-09-23T23:33:46.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.832127+0000", 2023-09-23T23:33:46.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:33:46.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.832200+0000", 2023-09-23T23:33:46.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2681000000000004e-05 2023-09-23T23:33:46.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.832210+0000", 2023-09-23T23:33:46.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0057e-05 2023-09-23T23:33:46.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.492 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.492 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.832215+0000", 2023-09-23T23:33:46.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9269999999999996e-06 2023-09-23T23:33:46.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.492 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.492 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.832613+0000", 2023-09-23T23:33:46.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039860699999999999 2023-09-23T23:33:46.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.493 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.493 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.832642+0000", 2023-09-23T23:33:46.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9136999999999999e-05 2023-09-23T23:33:46.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.493 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:33:46.493 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.094080+0000", 2023-09-23T23:33:46.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.26143806000000003 2023-09-23T23:33:46.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.493 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.493 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.094115+0000", 2023-09-23T23:33:46.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4326000000000003e-05 2023-09-23T23:33:46.494 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.494 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.494 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.494 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.46514.0:29513 13.8 e549/534 ondisk, result = 0)", 2023-09-23T23:33:46.494 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:15.833651+0000", 2023-09-23T23:33:46.494 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 30.553271408000001, 2023-09-23T23:33:46.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.25726930799999997, 2023-09-23T23:33:46.494 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.494 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:33:46.494 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.495 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.833651+0000", 2023-09-23T23:33:46.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.495 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.495 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.833651+0000", 2023-09-23T23:33:46.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.495 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.495 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.833651+0000", 2023-09-23T23:33:46.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:33:46.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.833655+0000", 2023-09-23T23:33:46.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1520000000000002e-06 2023-09-23T23:33:46.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.090807+0000", 2023-09-23T23:33:46.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.25715234799999998 2023-09-23T23:33:46.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.090819+0000", 2023-09-23T23:33:46.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1977000000000001e-05 2023-09-23T23:33:46.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.090899+0000", 2023-09-23T23:33:46.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9820000000000005e-05 2023-09-23T23:33:46.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.090911+0000", 2023-09-23T23:33:46.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1379000000000001e-05 2023-09-23T23:33:46.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.498 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.498 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.498 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.090921+0000", 2023-09-23T23:33:46.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0169999999999999e-05 2023-09-23T23:33:46.498 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.498 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.498 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.498 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.498 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.498 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.46514.0:29502 13.a e549/534 ondisk, result = 0)", 2023-09-23T23:33:46.498 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:15.834129+0000", 2023-09-23T23:33:46.498 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 30.552793352999998, 2023-09-23T23:33:46.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.25692784499999999, 2023-09-23T23:33:46.499 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.499 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:33:46.499 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.499 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.834129+0000", 2023-09-23T23:33:46.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.499 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.834129+0000", 2023-09-23T23:33:46.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.834129+0000", 2023-09-23T23:33:46.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:33:46.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.834135+0000", 2023-09-23T23:33:46.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2480000000000001e-06 2023-09-23T23:33:46.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.090813+0000", 2023-09-23T23:33:46.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.25667825799999999 2023-09-23T23:33:46.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.090821+0000", 2023-09-23T23:33:46.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2030000000000003e-06 2023-09-23T23:33:46.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.090955+0000", 2023-09-23T23:33:46.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013477500000000001 2023-09-23T23:33:46.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.502 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.502 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.090963+0000", 2023-09-23T23:33:46.502 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4590000000000001e-06 2023-09-23T23:33:46.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.502 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.502 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.091057+0000", 2023-09-23T23:33:46.502 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4420999999999996e-05 2023-09-23T23:33:46.502 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.502 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.502 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.503 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:29522 13.a 13:554e77af:::benchmark_data_smithi110_541928_object29521:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e549)", 2023-09-23T23:33:46.503 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:15.834322+0000", 2023-09-23T23:33:46.503 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 30.552601213999999, 2023-09-23T23:33:46.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.25958237000000001, 2023-09-23T23:33:46.503 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:33:46.503 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:33:46.503 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:33:46.503 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:33:46.503 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:33:46.503 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 29522 2023-09-23T23:33:46.503 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.503 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:33:46.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.504 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:33:46.504 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.834322+0000", 2023-09-23T23:33:46.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.504 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.504 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.504 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:33:46.504 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.834322+0000", 2023-09-23T23:33:46.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:33:46.504 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.504 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.504 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:33:46.504 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.834320+0000", 2023-09-23T23:33:46.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:33:46.505 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.505 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.505 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:33:46.505 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:15.834348+0000", 2023-09-23T23:33:46.505 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7931000000000001e-05 2023-09-23T23:33:46.505 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.505 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.505 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:33:46.505 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.090803+0000", 2023-09-23T23:33:46.505 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.25645467700000002 2023-09-23T23:33:46.505 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.505 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.505 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:33:46.506 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.090820+0000", 2023-09-23T23:33:46.506 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7367000000000002e-05 2023-09-23T23:33:46.506 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.506 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.506 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:33:46.506 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.091079+0000", 2023-09-23T23:33:46.506 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025873999999999998 2023-09-23T23:33:46.506 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.506 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.506 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:33:46.506 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.091222+0000", 2023-09-23T23:33:46.506 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000143072 2023-09-23T23:33:46.506 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.507 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:33:46.507 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.091368+0000", 2023-09-23T23:33:46.507 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014547599999999999 2023-09-23T23:33:46.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.507 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:33:46.507 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.092823+0000", 2023-09-23T23:33:46.507 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001455458 2023-09-23T23:33:46.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.507 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.507 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.093284+0000", 2023-09-23T23:33:46.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00046147400000000002 2023-09-23T23:33:46.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:33:46.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.093827+0000", 2023-09-23T23:33:46.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00054232399999999995 2023-09-23T23:33:46.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:33:46.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.093870+0000", 2023-09-23T23:33:46.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3433999999999998e-05 2023-09-23T23:33:46.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:33:46.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:33:46.509 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:33:46.509 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:16.093904+0000", 2023-09-23T23:33:46.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3800000000000002e-05 2023-09-23T23:33:46.509 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.509 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.509 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.509 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:33:46.509 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:33:46.509 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:33:46.576 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:46.672 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:46.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:46.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:46.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[212939]: failed to encode map e554 with expected crc 2023-09-23T23:33:46.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[212939]: failed to encode map e554 with expected crc 2023-09-23T23:33:46.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[212939]: failed to encode map e554 with expected crc 2023-09-23T23:33:46.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[212939]: failed to encode map e555 with expected crc 2023-09-23T23:33:46.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[212939]: osdmap e555: 8 total, 8 up, 8 in 2023-09-23T23:33:46.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[212939]: failed to encode map e555 with expected crc 2023-09-23T23:33:46.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[212939]: failed to encode map e555 with expected crc 2023-09-23T23:33:46.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[212939]: failed to encode map e555 with expected crc 2023-09-23T23:33:46.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[212939]: failed to encode map e555 with expected crc 2023-09-23T23:33:46.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/143587664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:46.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[221102]: failed to encode map e554 with expected crc 2023-09-23T23:33:46.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[221102]: failed to encode map e554 with expected crc 2023-09-23T23:33:46.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[221102]: failed to encode map e554 with expected crc 2023-09-23T23:33:46.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[221102]: failed to encode map e555 with expected crc 2023-09-23T23:33:46.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[221102]: osdmap e555: 8 total, 8 up, 8 in 2023-09-23T23:33:46.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[221102]: failed to encode map e555 with expected crc 2023-09-23T23:33:46.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[221102]: failed to encode map e555 with expected crc 2023-09-23T23:33:46.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[221102]: failed to encode map e555 with expected crc 2023-09-23T23:33:46.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[221102]: failed to encode map e555 with expected crc 2023-09-23T23:33:46.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:46 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/143587664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:46.785 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:46.797 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:33:46.834 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:46 smithi138 ceph-mon[193606]: failed to encode map e554 with expected crc 2023-09-23T23:33:46.835 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:46 smithi138 ceph-mon[193606]: failed to encode map e554 with expected crc 2023-09-23T23:33:46.835 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:46 smithi138 ceph-mon[193606]: failed to encode map e554 with expected crc 2023-09-23T23:33:46.835 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:46 smithi138 ceph-mon[193606]: failed to encode map e555 with expected crc 2023-09-23T23:33:46.835 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:46 smithi138 ceph-mon[193606]: osdmap e555: 8 total, 8 up, 8 in 2023-09-23T23:33:46.835 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:46 smithi138 ceph-mon[193606]: failed to encode map e555 with expected crc 2023-09-23T23:33:46.835 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:46 smithi138 ceph-mon[193606]: failed to encode map e555 with expected crc 2023-09-23T23:33:46.835 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:46 smithi138 ceph-mon[193606]: failed to encode map e555 with expected crc 2023-09-23T23:33:46.835 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:46 smithi138 ceph-mon[193606]: failed to encode map e555 with expected crc 2023-09-23T23:33:46.835 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:46 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/143587664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:46.844 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:46.850 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 49 16 67857 67841 86.508 74.625 0.00948765 0.0115538 2023-09-23T23:33:46.912 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:46.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:46.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:47.019 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:33:47.045 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:47.087 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:33:47.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:47.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:33:47.194 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:47.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:33:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:33:46] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.43.0" 2023-09-23T23:33:47.243 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:47.852 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 50 16 68832 68816 85.9925 60.9375 0.0107034 0.0116241 2023-09-23T23:33:47.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:47 smithi138 ceph-mon[193606]: failed to encode map e555 with expected crc 2023-09-23T23:33:47.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:47 smithi138 ceph-mon[193606]: failed to encode map e555 with expected crc 2023-09-23T23:33:47.944 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:47 smithi138 ceph-mon[193606]: pgmap v1532: 119 pgs: 119 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T23:33:47.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:47 smithi110 ceph-mon[212939]: failed to encode map e555 with expected crc 2023-09-23T23:33:47.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:47 smithi110 ceph-mon[212939]: failed to encode map e555 with expected crc 2023-09-23T23:33:47.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:47 smithi110 ceph-mon[212939]: pgmap v1532: 119 pgs: 119 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T23:33:47.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:47 smithi110 ceph-mon[221102]: failed to encode map e555 with expected crc 2023-09-23T23:33:47.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:47 smithi110 ceph-mon[221102]: failed to encode map e555 with expected crc 2023-09-23T23:33:47.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:47 smithi110 ceph-mon[221102]: pgmap v1532: 119 pgs: 119 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T23:33:48.063 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:33:48.581 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:48.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:48.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:48.685 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:33:48.726 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:33:48.736 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:48.792 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:48.852 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 51 16 69817 69801 85.5135 61.5625 0.0138004 0.0116881 2023-09-23T23:33:49.027 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:33:49.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:49.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:33:49.134 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:49.180 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:49.462 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:49.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:49.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:49.569 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:49.643 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:49 smithi138 ceph-mon[193606]: pgmap v1533: 119 pgs: 119 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 997 op/s 2023-09-23T23:33:49.652 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:49.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:49 smithi110 ceph-mon[221102]: pgmap v1533: 119 pgs: 119 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 997 op/s 2023-09-23T23:33:49.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:49 smithi110 ceph-mon[212939]: pgmap v1533: 119 pgs: 119 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 997 op/s 2023-09-23T23:33:49.755 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:49.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:49.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:49.852 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 52 16 70884 70868 85.1513 66.6875 0.00993462 0.0117268 2023-09-23T23:33:49.864 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:49.892 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:49.935 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:49.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:49.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:50.039 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:33:50.084 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:49 smithi110 conmon[212914]: 2023-09-23T23:33:49.751+0000 7f29aa464700 -1 mon.a@0(leader).osd e555 definitely_dead 0 2023-09-23T23:33:50.089 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:50.383 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:33:50.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:50.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:33:50.417 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:33:50.490 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:50.562 INFO:teuthology.orchestra.run.smithi110.stderr:marked out osd.4. 2023-09-23T23:33:50.574 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:50.588 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:50 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/338090488' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T23:33:50.588 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:50 smithi110 ceph-mon[212939]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T23:33:50.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:50 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/338090488' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T23:33:50.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:50 smithi110 ceph-mon[221102]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T23:33:50.827 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:50.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:50.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:50.854 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 53 16 71540 71524 84.3183 41 0.00413154 0.0117888 2023-09-23T23:33:50.936 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:33:50.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:50 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/338090488' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T23:33:50.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:50 smithi138 ceph-mon[193606]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T23:33:50.971 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:51.492 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:33:51.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:51.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:33:51.546 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:33:51.596 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:33:51.638 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:51.661 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:51.689 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:33:51.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:51.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:33:51.792 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:33:51.816 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:51 smithi138 ceph-mon[193606]: pgmap v1534: 119 pgs: 119 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T23:33:51.816 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:51 smithi138 ceph-mon[193606]: failed to encode map e556 with expected crc 2023-09-23T23:33:51.816 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:51 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/338090488' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T23:33:51.817 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:51 smithi138 ceph-mon[193606]: osdmap e556: 8 total, 8 up, 7 in 2023-09-23T23:33:51.817 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:51 smithi138 ceph-mon[193606]: failed to encode map e556 with expected crc 2023-09-23T23:33:51.817 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:51 smithi138 ceph-mon[193606]: failed to encode map e556 with expected crc 2023-09-23T23:33:51.817 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:51 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1139667817' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:33:51.818 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:51.853 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 54 16 71540 71524 82.7572 0 - 0.0117888 2023-09-23T23:33:51.860 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:33:51.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:51.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:33:51.864 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:51 smithi110 ceph-mon[212939]: pgmap v1534: 119 pgs: 119 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T23:33:51.864 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:51 smithi110 ceph-mon[212939]: failed to encode map e556 with expected crc 2023-09-23T23:33:51.864 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:51 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/338090488' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T23:33:51.864 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:51 smithi110 ceph-mon[212939]: osdmap e556: 8 total, 8 up, 7 in 2023-09-23T23:33:51.864 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:51 smithi110 ceph-mon[212939]: failed to encode map e556 with expected crc 2023-09-23T23:33:51.864 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:51 smithi110 ceph-mon[212939]: failed to encode map e556 with expected crc 2023-09-23T23:33:51.865 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:51 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1139667817' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:33:51.865 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:51 smithi110 ceph-mon[221102]: pgmap v1534: 119 pgs: 119 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T23:33:51.865 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:51 smithi110 ceph-mon[221102]: failed to encode map e556 with expected crc 2023-09-23T23:33:51.865 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:51 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/338090488' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T23:33:51.865 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:51 smithi110 ceph-mon[221102]: osdmap e556: 8 total, 8 up, 7 in 2023-09-23T23:33:51.865 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:51 smithi110 ceph-mon[221102]: failed to encode map e556 with expected crc 2023-09-23T23:33:51.865 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:51 smithi110 ceph-mon[221102]: failed to encode map e556 with expected crc 2023-09-23T23:33:51.865 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:51 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1139667817' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:33:51.964 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:33:51.992 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:52.543 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:33:52.557 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: failed to encode map e556 with expected crc 2023-09-23T23:33:52.558 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: failed to encode map e556 with expected crc 2023-09-23T23:33:52.558 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:33:52.558 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:33:52.558 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.558 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.558 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1139667817' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:33:52.558 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: osdmap e557: 8 total, 8 up, 7 in 2023-09-23T23:33:52.558 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.558 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.559 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.559 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.559 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.559 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.559 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.559 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.559 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.559 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.559 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[212939]: osdmap e558: 8 total, 8 up, 7 in 2023-09-23T23:33:52.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:52.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:33:52.667 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:52.774 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:52.853 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 55 16 71551 71535 81.2654 0.34375 2.31233 0.0120432 2023-09-23T23:33:52.917 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:52.921 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: failed to encode map e556 with expected crc 2023-09-23T23:33:52.921 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: failed to encode map e556 with expected crc 2023-09-23T23:33:52.921 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:33:52.921 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:33:52.921 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.921 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.921 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1139667817' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:33:52.921 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: osdmap e557: 8 total, 8 up, 7 in 2023-09-23T23:33:52.922 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.922 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.922 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.922 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.922 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.922 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.922 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.922 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.922 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.922 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.922 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:52 smithi138 ceph-mon[193606]: osdmap e558: 8 total, 8 up, 7 in 2023-09-23T23:33:52.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:52.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:52.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: failed to encode map e556 with expected crc 2023-09-23T23:33:52.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: failed to encode map e556 with expected crc 2023-09-23T23:33:52.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:33:52.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:33:52.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1139667817' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:33:52.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: osdmap e557: 8 total, 8 up, 7 in 2023-09-23T23:33:52.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: failed to encode map e557 with expected crc 2023-09-23T23:33:52.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:52 smithi110 ceph-mon[221102]: osdmap e558: 8 total, 8 up, 7 in 2023-09-23T23:33:53.024 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:53.074 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:53.399 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:53.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:53.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:53.510 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:53.581 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:53.669 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:53 smithi110 ceph-mon[212939]: pgmap v1537: 119 pgs: 1 peering, 118 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 53 MiB/s wr, 849 op/s 2023-09-23T23:33:53.669 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:53 smithi110 ceph-mon[212939]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.669 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:53 smithi110 ceph-mon[212939]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.669 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:53 smithi110 ceph-mon[212939]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.669 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:53 smithi110 ceph-mon[212939]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.669 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:53 smithi110 ceph-mon[212939]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.669 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:53 smithi110 ceph-mon[212939]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.670 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:53 smithi110 ceph-mon[221102]: pgmap v1537: 119 pgs: 1 peering, 118 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 53 MiB/s wr, 849 op/s 2023-09-23T23:33:53.670 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:53 smithi110 ceph-mon[221102]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.670 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:53 smithi110 ceph-mon[221102]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.670 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:53 smithi110 ceph-mon[221102]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.670 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:53 smithi110 ceph-mon[221102]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.670 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:53 smithi110 ceph-mon[221102]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.670 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:53 smithi110 ceph-mon[221102]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.712 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:53.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:53.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:53.819 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:53.853 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 56 16 72082 72066 80.4069 33.1875 0.0148265 0.0124306 2023-09-23T23:33:53.878 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:53 smithi138 ceph-mon[193606]: pgmap v1537: 119 pgs: 1 peering, 118 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 53 MiB/s wr, 849 op/s 2023-09-23T23:33:53.878 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:53 smithi138 ceph-mon[193606]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.878 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:53 smithi138 ceph-mon[193606]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.879 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:53 smithi138 ceph-mon[193606]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.879 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:53 smithi138 ceph-mon[193606]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.879 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:53 smithi138 ceph-mon[193606]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.879 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:53 smithi138 ceph-mon[193606]: failed to encode map e558 with expected crc 2023-09-23T23:33:53.885 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:54.013 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:54.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:54.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:54.024 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:33:54.054 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:33:54.093 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:33:54.120 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:33:54.125 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:33:54.152 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:33:54.163 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:54.180 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:33:54.238 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:33:54.249 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:33:54.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:54.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:33:54.270 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:33:54.275 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:33:54.275 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:33:54.275 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:33:54.275 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:33:54.276 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:33:54.276 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:33:54.276 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:33:54.276 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:33:54.356 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:33:54.432 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:54.571 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:33:54.578 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[212939]: osdmap e559: 8 total, 8 up, 7 in 2023-09-23T23:33:54.578 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[212939]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.578 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[212939]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.578 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[212939]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.578 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[212939]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.579 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[212939]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.579 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[212939]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.579 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[212939]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.579 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[212939]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.579 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[212939]: pgmap v1540: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 1 peering, 113 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 5.2 KiB/s rd, 12 MiB/s wr, 203 op/s; 4547/211602 objects degraded (2.149%); 4357/211602 objects misplaced (2.059%); 298 KiB/s, 2 keys/s, 5 objects/s recovering 2023-09-23T23:33:54.579 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1387661111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:54.579 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[212939]: osdmap e560: 8 total, 8 up, 7 in 2023-09-23T23:33:54.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:54.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:33:54.635 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:33:54.685 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:33:54.774 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:54.853 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 57 16 72741 72725 79.7189 41.1875 0.0249771 0.0125365 2023-09-23T23:33:54.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:54 smithi138 ceph-mon[193606]: osdmap e559: 8 total, 8 up, 7 in 2023-09-23T23:33:54.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:54 smithi138 ceph-mon[193606]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:54 smithi138 ceph-mon[193606]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:54 smithi138 ceph-mon[193606]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:54 smithi138 ceph-mon[193606]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:54 smithi138 ceph-mon[193606]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:54 smithi138 ceph-mon[193606]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:54 smithi138 ceph-mon[193606]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:54 smithi138 ceph-mon[193606]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:54 smithi138 ceph-mon[193606]: pgmap v1540: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 1 peering, 113 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 5.2 KiB/s rd, 12 MiB/s wr, 203 op/s; 4547/211602 objects degraded (2.149%); 4357/211602 objects misplaced (2.059%); 298 KiB/s, 2 keys/s, 5 objects/s recovering 2023-09-23T23:33:54.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:54 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1387661111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:54.946 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:54 smithi138 ceph-mon[193606]: osdmap e560: 8 total, 8 up, 7 in 2023-09-23T23:33:54.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[221102]: osdmap e559: 8 total, 8 up, 7 in 2023-09-23T23:33:54.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[221102]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[221102]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[221102]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[221102]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[221102]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[221102]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[221102]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[221102]: failed to encode map e559 with expected crc 2023-09-23T23:33:54.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[221102]: pgmap v1540: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 1 peering, 113 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 5.2 KiB/s rd, 12 MiB/s wr, 203 op/s; 4547/211602 objects degraded (2.149%); 4357/211602 objects misplaced (2.059%); 298 KiB/s, 2 keys/s, 5 objects/s recovering 2023-09-23T23:33:54.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1387661111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:33:54.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:54 smithi110 ceph-mon[221102]: osdmap e560: 8 total, 8 up, 7 in 2023-09-23T23:33:54.955 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:33:54.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:54.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:33:55.067 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:33:55.097 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:55.232 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:33:55.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:55.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:33:55.346 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:33:55.464 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:55.533 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:33:55.533 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:33:55.599 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:33:55.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:55.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:33:55.710 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:55.806 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:55.836 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:55 smithi110 ceph-mon[212939]: failed to encode map e560 with expected crc 2023-09-23T23:33:55.836 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:55 smithi110 ceph-mon[212939]: failed to encode map e560 with expected crc 2023-09-23T23:33:55.837 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:55 smithi110 ceph-mon[212939]: failed to encode map e560 with expected crc 2023-09-23T23:33:55.837 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:55 smithi110 ceph-mon[212939]: failed to encode map e560 with expected crc 2023-09-23T23:33:55.837 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:55 smithi110 ceph-mon[212939]: failed to encode map e560 with expected crc 2023-09-23T23:33:55.837 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:55 smithi110 ceph-mon[212939]: Health check failed: Degraded data redundancy: 4547/211602 objects degraded (2.149%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:33:55.837 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:55 smithi110 ceph-mon[221102]: failed to encode map e560 with expected crc 2023-09-23T23:33:55.837 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:55 smithi110 ceph-mon[221102]: failed to encode map e560 with expected crc 2023-09-23T23:33:55.837 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:55 smithi110 ceph-mon[221102]: failed to encode map e560 with expected crc 2023-09-23T23:33:55.837 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:55 smithi110 ceph-mon[221102]: failed to encode map e560 with expected crc 2023-09-23T23:33:55.838 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:55 smithi110 ceph-mon[221102]: failed to encode map e560 with expected crc 2023-09-23T23:33:55.838 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:55 smithi110 ceph-mon[221102]: Health check failed: Degraded data redundancy: 4547/211602 objects degraded (2.149%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:33:55.853 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 58 16 73210 73194 78.8499 29.3125 0.00751437 0.0126749 2023-09-23T23:33:55.933 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:55.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:55.939 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:55 smithi138 ceph-mon[193606]: failed to encode map e560 with expected crc 2023-09-23T23:33:55.939 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:55 smithi138 ceph-mon[193606]: failed to encode map e560 with expected crc 2023-09-23T23:33:55.939 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:55 smithi138 ceph-mon[193606]: failed to encode map e560 with expected crc 2023-09-23T23:33:55.939 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:55 smithi138 ceph-mon[193606]: failed to encode map e560 with expected crc 2023-09-23T23:33:55.939 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:55 smithi138 ceph-mon[193606]: failed to encode map e560 with expected crc 2023-09-23T23:33:55.940 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:55 smithi138 ceph-mon[193606]: Health check failed: Degraded data redundancy: 4547/211602 objects degraded (2.149%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:33:55.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:55.942 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:56.041 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:33:56.108 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:56.201 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:33:56.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:56.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:33:56.305 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:56.336 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:56.507 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:56.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:56.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:56.580 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 5, 7, 2, 1, 6] out_osds: [4] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:33:56.581 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:33:56.581 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T23:33:56.581 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.000000 2023-09-23T23:33:56.581 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 0 2023-09-23T23:33:56.612 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:56.669 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:56.764 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:56.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:56.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:56.772 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:56 smithi110 ceph-mon[221102]: pgmap v1542: 119 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 4 peering, 103 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 4.8 KiB/s rd, 32 MiB/s wr, 516 op/s; 31488/216918 objects degraded (14.516%); 8736/216918 objects misplaced (4.027%); 2.2 MiB/s, 2 keys/s, 37 objects/s recovering 2023-09-23T23:33:56.772 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:56 smithi110 ceph-mon[212939]: pgmap v1542: 119 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 4 peering, 103 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 4.8 KiB/s rd, 32 MiB/s wr, 516 op/s; 31488/216918 objects degraded (14.516%); 8736/216918 objects misplaced (4.027%); 2.2 MiB/s, 2 keys/s, 37 objects/s recovering 2023-09-23T23:33:56.856 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 59 16 73725 73709 78.0551 32.1875 0.0152919 0.0128048 2023-09-23T23:33:56.873 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:33:56.912 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:56.913 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:56 smithi138 ceph-mon[193606]: pgmap v1542: 119 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 4 peering, 103 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 4.8 KiB/s rd, 32 MiB/s wr, 516 op/s; 31488/216918 objects degraded (14.516%); 8736/216918 objects misplaced (4.027%); 2.2 MiB/s, 2 keys/s, 37 objects/s recovering 2023-09-23T23:33:57.072 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:33:57.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:57.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:33:57.142 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T23:33:57.177 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:33:57.203 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:33:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:33:56] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.43.0" 2023-09-23T23:33:57.225 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:57.717 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:33:57.856 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:33:57.855719+0000 min lat: 0.00107054 max lat: 2.51963 avg lat: 0.0129165 2023-09-23T23:33:57.857 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:33:57.857 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 60 16 74331 74315 77.3855 37.875 0.0144319 0.0129165 2023-09-23T23:33:57.869 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:33:57.973 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:33:57.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:57.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:33:58.085 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:33:58.163 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:58.166 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:58.489 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:33:58.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:58.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:33:58.599 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:33:58.685 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:58.857 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 61 16 74994 74978 76.7962 41.4375 0.0295661 0.0130142 2023-09-23T23:33:58.874 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:33:59.007 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:33:59.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:59.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:33:59.111 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:33:59.153 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:59.333 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:59 smithi110 ceph-mon[221102]: pgmap v1543: 119 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 3 peering, 104 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 3.5 KiB/s rd, 23 MiB/s wr, 374 op/s; 31488/216918 objects degraded (14.516%); 8736/216918 objects misplaced (4.027%); 1.6 MiB/s, 1 keys/s, 27 objects/s recovering 2023-09-23T23:33:59.333 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:59 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:59.333 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:33:59 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:33:59.334 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:59 smithi110 ceph-mon[212939]: pgmap v1543: 119 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 3 peering, 104 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 3.5 KiB/s rd, 23 MiB/s wr, 374 op/s; 31488/216918 objects degraded (14.516%); 8736/216918 objects misplaced (4.027%); 1.6 MiB/s, 1 keys/s, 27 objects/s recovering 2023-09-23T23:33:59.334 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:59 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:59.334 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:33:59 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:33:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:59 smithi138 ceph-mon[193606]: pgmap v1543: 119 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 3 peering, 104 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 3.5 KiB/s rd, 23 MiB/s wr, 374 op/s; 31488/216918 objects degraded (14.516%); 8736/216918 objects misplaced (4.027%); 1.6 MiB/s, 1 keys/s, 27 objects/s recovering 2023-09-23T23:33:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:59 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:33:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:33:59 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:33:59.502 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:33:59.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:59.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:33:59.606 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:33:59.635 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:33:59.857 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 62 16 75357 75341 75.9237 22.6875 0.0261369 0.0131648 2023-09-23T23:33:59.864 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:33:59.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:33:59.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:33:59.977 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:34:00.034 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:00.063 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:34:00.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:00 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3132338191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:34:00.250 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:34:00.250 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:34:00.334 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:00 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3132338191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:34:00.335 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:00 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3132338191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:34:00.433 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:34:00.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:00.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:34:00.540 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:34:00.588 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:00.637 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:34:00.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:00.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:34:00.742 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:00.770 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:00.857 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 63 16 75906 75890 75.2632 34.3125 0.286346 0.0132648 2023-09-23T23:34:01.040 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:01 smithi110 ceph-mon[221102]: pgmap v1544: 119 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 2.9 KiB/s rd, 40 MiB/s wr, 642 op/s; 31488/224340 objects degraded (14.036%); 8561/224340 objects misplaced (3.816%); 2.8 MiB/s, 1 keys/s, 46 objects/s recovering 2023-09-23T23:34:01.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:01 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 31488/224340 objects degraded (14.036%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:01.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:01 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:34:01.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:01 smithi110 ceph-mon[221102]: failed to encode map e561 with expected crc 2023-09-23T23:34:01.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:01 smithi110 ceph-mon[221102]: failed to encode map e561 with expected crc 2023-09-23T23:34:01.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:01 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3132338191' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:34:01.041 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:01 smithi110 ceph-mon[221102]: osdmap e561: 8 total, 8 up, 7 in 2023-09-23T23:34:01.279 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:01.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:01.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:01.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:01 smithi110 ceph-mon[212939]: pgmap v1544: 119 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 2.9 KiB/s rd, 40 MiB/s wr, 642 op/s; 31488/224340 objects degraded (14.036%); 8561/224340 objects misplaced (3.816%); 2.8 MiB/s, 1 keys/s, 46 objects/s recovering 2023-09-23T23:34:01.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:01 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 31488/224340 objects degraded (14.036%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:01.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:01 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:34:01.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:01 smithi110 ceph-mon[212939]: failed to encode map e561 with expected crc 2023-09-23T23:34:01.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:01 smithi110 ceph-mon[212939]: failed to encode map e561 with expected crc 2023-09-23T23:34:01.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:01 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3132338191' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:34:01.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:01 smithi110 ceph-mon[212939]: osdmap e561: 8 total, 8 up, 7 in 2023-09-23T23:34:01.388 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:01.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:01 smithi138 ceph-mon[193606]: pgmap v1544: 119 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 2.9 KiB/s rd, 40 MiB/s wr, 642 op/s; 31488/224340 objects degraded (14.036%); 8561/224340 objects misplaced (3.816%); 2.8 MiB/s, 1 keys/s, 46 objects/s recovering 2023-09-23T23:34:01.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:01 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 31488/224340 objects degraded (14.036%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:01.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:01 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:34:01.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:01 smithi138 ceph-mon[193606]: failed to encode map e561 with expected crc 2023-09-23T23:34:01.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:01 smithi138 ceph-mon[193606]: failed to encode map e561 with expected crc 2023-09-23T23:34:01.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:01 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3132338191' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:34:01.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:01 smithi138 ceph-mon[193606]: osdmap e561: 8 total, 8 up, 7 in 2023-09-23T23:34:01.463 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:01.575 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:01.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:01.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:01.610 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T23:34:01.696 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:01.758 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:01.857 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 64 16 76549 76533 74.7152 40.1875 0.0220237 0.0133779 2023-09-23T23:34:01.901 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:01.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:01.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:02.024 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:02.144 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:02.326 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:02.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:02.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:02.375 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:34:02.446 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:02.546 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:02.694 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:02.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:02.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:02.808 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:34:02.857 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 65 16 77211 77195 74.2024 41.375 0.0280184 0.0134712 2023-09-23T23:34:02.877 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:02.931 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:34:02.969 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:34:03.004 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:34:03.038 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:34:03.079 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:34:03.097 INFO:teuthology.orchestra.run.smithi110.stderr:set osd.1 primary-affinity to 0 (802) 2023-09-23T23:34:03.129 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:34:03.147 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:03 smithi138 ceph-mon[193606]: pgmap v1546: 119 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 36 MiB/s wr, 572 op/s; 31488/225348 objects degraded (13.973%); 8353/225348 objects misplaced (3.707%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:34:03.147 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:03 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/637763694' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-23T23:34:03.147 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:03 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/598575775' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:34:03.168 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:34:03.203 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:34:03.206 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:34:03.206 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:34:03.206 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:34:03.206 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:34:03.206 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:34:03.206 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:34:03.206 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:34:03.206 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:34:03.265 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:34:03.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:03.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:34:03.370 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:34:03.400 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:03 smithi110 ceph-mon[212939]: pgmap v1546: 119 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 36 MiB/s wr, 572 op/s; 31488/225348 objects degraded (13.973%); 8353/225348 objects misplaced (3.707%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:34:03.400 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:03 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/637763694' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-23T23:34:03.401 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:03 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/598575775' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:34:03.401 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:03 smithi110 ceph-mon[221102]: pgmap v1546: 119 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 36 MiB/s wr, 572 op/s; 31488/225348 objects degraded (13.973%); 8353/225348 objects misplaced (3.707%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:34:03.401 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:03 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/637763694' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-23T23:34:03.401 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:03 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/598575775' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:34:03.406 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:03.710 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:34:03.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:03.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:34:03.820 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:34:03.841 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:34:03.857 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 66 16 77545 77529 73.3945 20.875 0.0162569 0.013552 2023-09-23T23:34:03.889 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:03.958 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:34:03.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:03.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:34:04.070 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:04.085 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:04 smithi110 ceph-mon[212939]: failed to encode map e562 with expected crc 2023-09-23T23:34:04.085 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:04 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/637763694' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-09-23T23:34:04.085 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:04 smithi110 ceph-mon[212939]: osdmap e562: 8 total, 8 up, 7 in 2023-09-23T23:34:04.085 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:04 smithi110 ceph-mon[212939]: failed to encode map e562 with expected crc 2023-09-23T23:34:04.085 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:04 smithi110 ceph-mon[212939]: osdmap e563: 8 total, 8 up, 7 in 2023-09-23T23:34:04.085 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:04 smithi110 ceph-mon[212939]: failed to encode map e563 with expected crc 2023-09-23T23:34:04.085 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:04 smithi110 ceph-mon[212939]: failed to encode map e563 with expected crc 2023-09-23T23:34:04.085 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:04 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:04.103 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:04.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:04 smithi138 ceph-mon[193606]: failed to encode map e562 with expected crc 2023-09-23T23:34:04.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:04 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/637763694' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-09-23T23:34:04.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:04 smithi138 ceph-mon[193606]: osdmap e562: 8 total, 8 up, 7 in 2023-09-23T23:34:04.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:04 smithi138 ceph-mon[193606]: failed to encode map e562 with expected crc 2023-09-23T23:34:04.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:04 smithi138 ceph-mon[193606]: osdmap e563: 8 total, 8 up, 7 in 2023-09-23T23:34:04.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:04 smithi138 ceph-mon[193606]: failed to encode map e563 with expected crc 2023-09-23T23:34:04.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:04 smithi138 ceph-mon[193606]: failed to encode map e563 with expected crc 2023-09-23T23:34:04.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:04 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:04.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:04 smithi110 ceph-mon[221102]: failed to encode map e562 with expected crc 2023-09-23T23:34:04.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:04 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/637763694' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-09-23T23:34:04.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:04 smithi110 ceph-mon[221102]: osdmap e562: 8 total, 8 up, 7 in 2023-09-23T23:34:04.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:04 smithi110 ceph-mon[221102]: failed to encode map e562 with expected crc 2023-09-23T23:34:04.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:04 smithi110 ceph-mon[221102]: osdmap e563: 8 total, 8 up, 7 in 2023-09-23T23:34:04.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:04 smithi110 ceph-mon[221102]: failed to encode map e563 with expected crc 2023-09-23T23:34:04.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:04 smithi110 ceph-mon[221102]: failed to encode map e563 with expected crc 2023-09-23T23:34:04.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:04 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:04.857 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 67 16 77613 77597 72.3628 4.25 0.0122922 0.0135494 2023-09-23T23:34:05.438 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:34:05.438 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:34:05.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:34:05.438 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:34:05.438 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.438 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:178614 11.3 11:c2d62249:::benchmark_data_smithi110_433746_object72734:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:34:05.438 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:19.570755+0000", 2023-09-23T23:34:05.439 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 405.86109969099999, 2023-09-23T23:34:05.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1817553470000002, 2023-09-23T23:34:05.439 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.439 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.439 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.439 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:34:05.439 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:05.439 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178614 2023-09-23T23:34:05.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.439 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570755+0000", 2023-09-23T23:34:05.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570755+0000", 2023-09-23T23:34:05.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570754+0000", 2023-09-23T23:34:05.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:34:05.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570759+0000", 2023-09-23T23:34:05.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7110000000000003e-06 2023-09-23T23:34:05.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570771+0000", 2023-09-23T23:34:05.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1965e-05 2023-09-23T23:34:05.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570776+0000", 2023-09-23T23:34:05.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0719999999999999e-06 2023-09-23T23:34:05.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570820+0000", 2023-09-23T23:34:05.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3640999999999998e-05 2023-09-23T23:34:05.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570826+0000", 2023-09-23T23:34:05.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.088e-06 2023-09-23T23:34:05.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:20.540024+0000", 2023-09-23T23:34:05.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96919833200000005 2023-09-23T23:34:05.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:20.540036+0000", 2023-09-23T23:34:05.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1731e-05 2023-09-23T23:34:05.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.740668+0000", 2023-09-23T23:34:05.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.20063242 2023-09-23T23:34:05.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.740808+0000", 2023-09-23T23:34:05.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000139813 2023-09-23T23:34:05.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-09-23T23:34:05.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.741123+0000", 2023-09-23T23:34:05.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000314968 2023-09-23T23:34:05.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.744153+0000", 2023-09-23T23:34:05.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003029538 2023-09-23T23:34:05.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.744306+0000", 2023-09-23T23:34:05.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000153614 2023-09-23T23:34:05.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.750752+0000", 2023-09-23T23:34:05.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006445175 2023-09-23T23:34:05.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752382+0000", 2023-09-23T23:34:05.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0016305670000000001 2023-09-23T23:34:05.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752422+0000", 2023-09-23T23:34:05.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0157999999999997e-05 2023-09-23T23:34:05.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752511+0000", 2023-09-23T23:34:05.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8374000000000005e-05 2023-09-23T23:34:05.448 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.448 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.448 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.448 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:178615 11.3 11:c151980f:::benchmark_data_smithi110_433746_object72735:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:34:05.448 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:19.573793+0000", 2023-09-23T23:34:05.448 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 405.85806234900002, 2023-09-23T23:34:05.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1788738529999998, 2023-09-23T23:34:05.448 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.448 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.449 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.449 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:34:05.449 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:05.449 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178615 2023-09-23T23:34:05.449 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.449 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.449 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.449 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573793+0000", 2023-09-23T23:34:05.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573793+0000", 2023-09-23T23:34:05.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573792+0000", 2023-09-23T23:34:05.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:34:05.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573797+0000", 2023-09-23T23:34:05.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7960000000000003e-06 2023-09-23T23:34:05.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573809+0000", 2023-09-23T23:34:05.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2575000000000001e-05 2023-09-23T23:34:05.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573816+0000", 2023-09-23T23:34:05.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4670000000000001e-06 2023-09-23T23:34:05.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573907+0000", 2023-09-23T23:34:05.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1496000000000006e-05 2023-09-23T23:34:05.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573930+0000", 2023-09-23T23:34:05.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2660999999999999e-05 2023-09-23T23:34:05.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:20.540059+0000", 2023-09-23T23:34:05.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96612945299999997 2023-09-23T23:34:05.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.454 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:20.540067+0000", 2023-09-23T23:34:05.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0919999999999998e-06 2023-09-23T23:34:05.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.454 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.742434+0000", 2023-09-23T23:34:05.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.202367089 2023-09-23T23:34:05.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.455 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.742547+0000", 2023-09-23T23:34:05.455 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112941 2023-09-23T23:34:05.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-09-23T23:34:05.455 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.742774+0000", 2023-09-23T23:34:05.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00022651400000000001 2023-09-23T23:34:05.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.744161+0000", 2023-09-23T23:34:05.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001387325 2023-09-23T23:34:05.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.744473+0000", 2023-09-23T23:34:05.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00031187499999999998 2023-09-23T23:34:05.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.750782+0000", 2023-09-23T23:34:05.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0063084719999999999 2023-09-23T23:34:05.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.458 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752547+0000", 2023-09-23T23:34:05.458 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0017649460000000001 2023-09-23T23:34:05.458 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.458 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.458 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.458 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752585+0000", 2023-09-23T23:34:05.458 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8121999999999999e-05 2023-09-23T23:34:05.458 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.458 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.459 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.459 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752667+0000", 2023-09-23T23:34:05.459 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1868999999999998e-05 2023-09-23T23:34:05.459 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.459 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.459 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.459 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.459 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.459 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:190429 11.5 11:a76d3bc6:::benchmark_data_smithi110_433746_object84549:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e436)", 2023-09-23T23:34:05.459 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:41.139932+0000", 2023-09-23T23:34:05.460 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 384.29192264599999, 2023-09-23T23:34:05.460 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0172525750000001, 2023-09-23T23:34:05.460 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.460 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.460 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.460 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:34:05.460 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:05.460 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 190429 2023-09-23T23:34:05.460 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.460 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.460 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.460 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.461 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139932+0000", 2023-09-23T23:34:05.461 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.461 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.461 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.461 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.461 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139932+0000", 2023-09-23T23:34:05.461 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.461 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.461 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.461 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.461 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139932+0000", 2023-09-23T23:34:05.461 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:34:05.461 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.462 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.462 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.462 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139934+0000", 2023-09-23T23:34:05.462 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.187e-06 2023-09-23T23:34:05.462 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.462 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.462 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.462 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139939+0000", 2023-09-23T23:34:05.462 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1340000000000003e-06 2023-09-23T23:34:05.462 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.462 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.462 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139943+0000", 2023-09-23T23:34:05.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4709999999999999e-06 2023-09-23T23:34:05.463 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.463 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.141339+0000", 2023-09-23T23:34:05.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001396481 2023-09-23T23:34:05.463 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.463 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:34:05.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.141345+0000", 2023-09-23T23:34:05.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8799999999999996e-06 2023-09-23T23:34:05.463 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.464 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.464 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.143057+0000", 2023-09-23T23:34:05.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0017121560000000001 2023-09-23T23:34:05.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.464 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.464 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.143062+0000", 2023-09-23T23:34:05.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.04e-06 2023-09-23T23:34:05.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.464 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.464 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.132701+0000", 2023-09-23T23:34:05.465 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98963869500000001 2023-09-23T23:34:05.465 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.465 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.465 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.465 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.132847+0000", 2023-09-23T23:34:05.465 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014609000000000001 2023-09-23T23:34:05.465 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.465 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.465 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:34:05.465 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.133176+0000", 2023-09-23T23:34:05.465 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00032873100000000002 2023-09-23T23:34:05.466 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.466 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.466 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.466 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.144793+0000", 2023-09-23T23:34:05.466 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011617647 2023-09-23T23:34:05.466 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.466 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.466 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.466 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157040+0000", 2023-09-23T23:34:05.466 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012246194 2023-09-23T23:34:05.466 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.466 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.467 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.467 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157117+0000", 2023-09-23T23:34:05.467 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7812999999999995e-05 2023-09-23T23:34:05.467 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.467 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.467 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.467 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157161+0000", 2023-09-23T23:34:05.467 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3009000000000001e-05 2023-09-23T23:34:05.467 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.467 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.467 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.467 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157185+0000", 2023-09-23T23:34:05.467 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4426e-05 2023-09-23T23:34:05.468 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.468 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.468 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.468 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.468 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.468 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:203242 11.13 11:cd9783f2:::benchmark_data_smithi110_433746_object97362:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-09-23T23:34:05.468 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:28:02.344823+0000", 2023-09-23T23:34:05.468 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 363.08703181999999, 2023-09-23T23:34:05.468 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0388343680000001, 2023-09-23T23:34:05.468 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.468 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.468 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.468 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:34:05.469 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:05.469 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 203242 2023-09-23T23:34:05.469 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.469 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.469 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.469 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.469 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344823+0000", 2023-09-23T23:34:05.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.469 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.469 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.469 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.469 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344823+0000", 2023-09-23T23:34:05.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.470 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.470 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344822+0000", 2023-09-23T23:34:05.470 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:34:05.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.470 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.470 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344826+0000", 2023-09-23T23:34:05.470 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4530000000000004e-06 2023-09-23T23:34:05.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344838+0000", 2023-09-23T23:34:05.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.17e-05 2023-09-23T23:34:05.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344844+0000", 2023-09-23T23:34:05.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7139999999999998e-06 2023-09-23T23:34:05.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344927+0000", 2023-09-23T23:34:05.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2708999999999997e-05 2023-09-23T23:34:05.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344939+0000", 2023-09-23T23:34:05.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2213e-05 2023-09-23T23:34:05.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.362662+0000", 2023-09-23T23:34:05.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0177228979999999 2023-09-23T23:34:05.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.362790+0000", 2023-09-23T23:34:05.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012857299999999999 2023-09-23T23:34:05.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,5", 2023-09-23T23:34:05.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.363083+0000", 2023-09-23T23:34:05.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00029265399999999999 2023-09-23T23:34:05.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.369254+0000", 2023-09-23T23:34:05.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0061712609999999999 2023-09-23T23:34:05.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383552+0000", 2023-09-23T23:34:05.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014297667 2023-09-23T23:34:05.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383603+0000", 2023-09-23T23:34:05.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1678000000000002e-05 2023-09-23T23:34:05.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.475 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.475 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383632+0000", 2023-09-23T23:34:05.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8711e-05 2023-09-23T23:34:05.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.475 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.475 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383658+0000", 2023-09-23T23:34:05.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5386000000000001e-05 2023-09-23T23:34:05.475 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.475 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.475 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.476 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.476 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.476 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:203250 11.13 11:c8573015:::benchmark_data_smithi110_433746_object97370:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e450)", 2023-09-23T23:34:05.476 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:28:02.375673+0000", 2023-09-23T23:34:05.476 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 363.05618239199998, 2023-09-23T23:34:05.476 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0081006640000001, 2023-09-23T23:34:05.476 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.476 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.476 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.476 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:34:05.476 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:05.476 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 203250 2023-09-23T23:34:05.476 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.477 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375673+0000", 2023-09-23T23:34:05.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375673+0000", 2023-09-23T23:34:05.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375671+0000", 2023-09-23T23:34:05.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:34:05.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375678+0000", 2023-09-23T23:34:05.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2099999999999996e-06 2023-09-23T23:34:05.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375696+0000", 2023-09-23T23:34:05.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7385e-05 2023-09-23T23:34:05.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375703+0000", 2023-09-23T23:34:05.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5739999999999998e-06 2023-09-23T23:34:05.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.400000+0000", 2023-09-23T23:34:05.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.024297062000000001 2023-09-23T23:34:05.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.400008+0000", 2023-09-23T23:34:05.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7389999999999999e-06 2023-09-23T23:34:05.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.364392+0000", 2023-09-23T23:34:05.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96438375899999995 2023-09-23T23:34:05.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.481 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.481 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.364510+0000", 2023-09-23T23:34:05.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117889 2023-09-23T23:34:05.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.481 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,5", 2023-09-23T23:34:05.481 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.364768+0000", 2023-09-23T23:34:05.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025844600000000001 2023-09-23T23:34:05.482 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.482 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.482 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.482 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.369259+0000", 2023-09-23T23:34:05.482 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0044911650000000001 2023-09-23T23:34:05.482 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.482 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.482 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.482 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383576+0000", 2023-09-23T23:34:05.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014316297 2023-09-23T23:34:05.483 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383688+0000", 2023-09-23T23:34:05.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112046 2023-09-23T23:34:05.483 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383735+0000", 2023-09-23T23:34:05.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7889000000000002e-05 2023-09-23T23:34:05.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.484 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383773+0000", 2023-09-23T23:34:05.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7795e-05 2023-09-23T23:34:05.484 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.484 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.484 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.485 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71516 13.e 13:7de122bf:::benchmark_data_smithi110_541928_object71515:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e555)", 2023-09-23T23:34:05.485 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.528660+0000", 2023-09-23T23:34:05.485 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 14.903194545, 2023-09-23T23:34:05.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2167170459999999, 2023-09-23T23:34:05.485 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.485 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:34:05.485 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.485 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:34:05.485 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:34:05.485 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71516 2023-09-23T23:34:05.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.486 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.486 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528660+0000", 2023-09-23T23:34:05.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.486 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528660+0000", 2023-09-23T23:34:05.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.487 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528659+0000", 2023-09-23T23:34:05.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:34:05.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528700+0000", 2023-09-23T23:34:05.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0859999999999998e-05 2023-09-23T23:34:05.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528712+0000", 2023-09-23T23:34:05.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2262000000000001e-05 2023-09-23T23:34:05.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528719+0000", 2023-09-23T23:34:05.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3339999999999998e-06 2023-09-23T23:34:05.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531320+0000", 2023-09-23T23:34:05.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002601506 2023-09-23T23:34:05.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531460+0000", 2023-09-23T23:34:05.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013956799999999999 2023-09-23T23:34:05.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:34:05.490 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531617+0000", 2023-09-23T23:34:05.490 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015701199999999999 2023-09-23T23:34:05.490 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.490 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.534591+0000", 2023-09-23T23:34:05.490 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029740880000000002 2023-09-23T23:34:05.490 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.545743+0000", 2023-09-23T23:34:05.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011152331 2023-09-23T23:34:05.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.491 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.545752+0000", 2023-09-23T23:34:05.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0000000000000002e-06 2023-09-23T23:34:05.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552315+0000", 2023-09-23T23:34:05.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.006563179 2023-09-23T23:34:05.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552330+0000", 2023-09-23T23:34:05.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4734e-05 2023-09-23T23:34:05.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745274+0000", 2023-09-23T23:34:05.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1929441679999999 2023-09-23T23:34:05.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745377+0000", 2023-09-23T23:34:05.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103303 2023-09-23T23:34:05.493 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.493 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.493 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.494 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71528 13.e 13:790d8dd9:::benchmark_data_smithi110_541928_object71527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:34:05.494 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.572698+0000", 2023-09-23T23:34:05.494 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 14.859156776000001, 2023-09-23T23:34:05.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2677280570000002, 2023-09-23T23:34:05.494 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.494 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.494 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.494 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:34:05.494 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:34:05.494 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71528 2023-09-23T23:34:05.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.495 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572698+0000", 2023-09-23T23:34:05.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572698+0000", 2023-09-23T23:34:05.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572697+0000", 2023-09-23T23:34:05.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:34:05.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572737+0000", 2023-09-23T23:34:05.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0278000000000003e-05 2023-09-23T23:34:05.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572746+0000", 2023-09-23T23:34:05.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3340000000000008e-06 2023-09-23T23:34:05.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572753+0000", 2023-09-23T23:34:05.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3899999999999998e-06 2023-09-23T23:34:05.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572817+0000", 2023-09-23T23:34:05.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4576e-05 2023-09-23T23:34:05.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572827+0000", 2023-09-23T23:34:05.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.978e-06 2023-09-23T23:34:05.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552357+0000", 2023-09-23T23:34:05.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97953002600000005 2023-09-23T23:34:05.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552365+0000", 2023-09-23T23:34:05.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5390000000000002e-06 2023-09-23T23:34:05.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745405+0000", 2023-09-23T23:34:05.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1930404109999999 2023-09-23T23:34:05.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745543+0000", 2023-09-23T23:34:05.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013780300000000001 2023-09-23T23:34:05.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:34:05.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745737+0000", 2023-09-23T23:34:05.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019410400000000001 2023-09-23T23:34:05.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.759566+0000", 2023-09-23T23:34:05.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013828844999999999 2023-09-23T23:34:05.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.768156+0000", 2023-09-23T23:34:05.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085902789999999993 2023-09-23T23:34:05.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.836351+0000", 2023-09-23T23:34:05.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.068194406999999999 2023-09-23T23:34:05.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.836426+0000", 2023-09-23T23:34:05.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4889999999999996e-05 2023-09-23T23:34:05.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.840426+0000", 2023-09-23T23:34:05.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040005659999999997 2023-09-23T23:34:05.502 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.502 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.502 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.502 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71532 13.e 13:7da393ae:::benchmark_data_smithi110_541928_object71531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:34:05.502 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.575009+0000", 2023-09-23T23:34:05.502 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 14.856845689, 2023-09-23T23:34:05.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2696883859999999, 2023-09-23T23:34:05.503 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.503 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.503 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.503 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:34:05.503 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:34:05.503 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71532 2023-09-23T23:34:05.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.503 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575009+0000", 2023-09-23T23:34:05.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575009+0000", 2023-09-23T23:34:05.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575004+0000", 2023-09-23T23:34:05.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999948 2023-09-23T23:34:05.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575068+0000", 2023-09-23T23:34:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3359000000000001e-05 2023-09-23T23:34:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575081+0000", 2023-09-23T23:34:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3434999999999999e-05 2023-09-23T23:34:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575090+0000", 2023-09-23T23:34:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7490000000000004e-06 2023-09-23T23:34:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575154+0000", 2023-09-23T23:34:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4023999999999996e-05 2023-09-23T23:34:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575166+0000", 2023-09-23T23:34:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2033e-05 2023-09-23T23:34:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552388+0000", 2023-09-23T23:34:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97722166399999999 2023-09-23T23:34:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552395+0000", 2023-09-23T23:34:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3429999999999998e-06 2023-09-23T23:34:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.748633+0000", 2023-09-23T23:34:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1962377799999999 2023-09-23T23:34:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.748799+0000", 2023-09-23T23:34:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016630200000000001 2023-09-23T23:34:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:34:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.748992+0000", 2023-09-23T23:34:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019316 2023-09-23T23:34:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.759572+0000", 2023-09-23T23:34:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010580032 2023-09-23T23:34:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.768191+0000", 2023-09-23T23:34:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086191280000000002 2023-09-23T23:34:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.840481+0000", 2023-09-23T23:34:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.072289500000000007 2023-09-23T23:34:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.840542+0000", 2023-09-23T23:34:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1509999999999999e-05 2023-09-23T23:34:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.844698+0000", 2023-09-23T23:34:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041553880000000003 2023-09-23T23:34:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77337 13.d 13:ba15f28f:::benchmark_data_smithi110_541928_object77336:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:34:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.706918+0000", 2023-09-23T23:34:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.724936714, 2023-09-23T23:34:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1549711069999999, 2023-09-23T23:34:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:34:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:34:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:34:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77337 2023-09-23T23:34:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706918+0000", 2023-09-23T23:34:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706918+0000", 2023-09-23T23:34:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706917+0000", 2023-09-23T23:34:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:34:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706970+0000", 2023-09-23T23:34:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2435000000000002e-05 2023-09-23T23:34:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706986+0000", 2023-09-23T23:34:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6252999999999998e-05 2023-09-23T23:34:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706993+0000", 2023-09-23T23:34:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1439999999999997e-06 2023-09-23T23:34:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709103+0000", 2023-09-23T23:34:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021098419999999998 2023-09-23T23:34:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709112+0000", 2023-09-23T23:34:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3640000000000005e-06 2023-09-23T23:34:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.860680+0000", 2023-09-23T23:34:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1515676990000001 2023-09-23T23:34:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.861889+0000", 2023-09-23T23:34:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012094930000000001 2023-09-23T23:34:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77365 13.d 13:bd3fed09:::benchmark_data_smithi110_541928_object77364:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:34:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.707869+0000", 2023-09-23T23:34:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.7239864360000001, 2023-09-23T23:34:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1996441330000001, 2023-09-23T23:34:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:34:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:34:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77365 2023-09-23T23:34:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707869+0000", 2023-09-23T23:34:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707869+0000", 2023-09-23T23:34:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707868+0000", 2023-09-23T23:34:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:34:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707912+0000", 2023-09-23T23:34:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4286999999999998e-05 2023-09-23T23:34:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707922+0000", 2023-09-23T23:34:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0132000000000001e-05 2023-09-23T23:34:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707929+0000", 2023-09-23T23:34:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4690000000000002e-06 2023-09-23T23:34:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709143+0000", 2023-09-23T23:34:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012139539999999999 2023-09-23T23:34:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709148+0000", 2023-09-23T23:34:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8420000000000001e-06 2023-09-23T23:34:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.861913+0000", 2023-09-23T23:34:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.152764984 2023-09-23T23:34:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.862371+0000", 2023-09-23T23:34:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00045722599999999998 2023-09-23T23:34:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:34:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.862643+0000", 2023-09-23T23:34:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000271983 2023-09-23T23:34:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.872385+0000", 2023-09-23T23:34:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0097426379999999996 2023-09-23T23:34:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.905589+0000", 2023-09-23T23:34:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.033203780000000002 2023-09-23T23:34:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.905624+0000", 2023-09-23T23:34:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5058000000000002e-05 2023-09-23T23:34:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.905662+0000", 2023-09-23T23:34:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7523e-05 2023-09-23T23:34:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907513+0000", 2023-09-23T23:34:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0018510320000000001 2023-09-23T23:34:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77391 13.d 13:bfca221d:::benchmark_data_smithi110_541928_object77390:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:34:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.707935+0000", 2023-09-23T23:34:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.723919975, 2023-09-23T23:34:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2024361749999999, 2023-09-23T23:34:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:34:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:34:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77391 2023-09-23T23:34:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707935+0000", 2023-09-23T23:34:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707935+0000", 2023-09-23T23:34:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707935+0000", 2023-09-23T23:34:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:34:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707971+0000", 2023-09-23T23:34:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6470000000000001e-05 2023-09-23T23:34:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707977+0000", 2023-09-23T23:34:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1489999999999999e-06 2023-09-23T23:34:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707982+0000", 2023-09-23T23:34:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5349999999999998e-06 2023-09-23T23:34:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709170+0000", 2023-09-23T23:34:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011882080000000001 2023-09-23T23:34:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709176+0000", 2023-09-23T23:34:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.297e-06 2023-09-23T23:34:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.866177+0000", 2023-09-23T23:34:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1570007 2023-09-23T23:34:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.866317+0000", 2023-09-23T23:34:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013973499999999999 2023-09-23T23:34:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:34:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.866488+0000", 2023-09-23T23:34:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000171495 2023-09-23T23:34:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.875118+0000", 2023-09-23T23:34:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086296299999999992 2023-09-23T23:34:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.905607+0000", 2023-09-23T23:34:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.030489532999999999 2023-09-23T23:34:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907981+0000", 2023-09-23T23:34:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023734239999999998 2023-09-23T23:34:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.908040+0000", 2023-09-23T23:34:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9463000000000003e-05 2023-09-23T23:34:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910371+0000", 2023-09-23T23:34:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023309279999999999 2023-09-23T23:34:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77394 13.d 13:bd1f7538:::benchmark_data_smithi110_541928_object77393:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:34:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.707987+0000", 2023-09-23T23:34:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.7238682330000001, 2023-09-23T23:34:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.204224143, 2023-09-23T23:34:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:34:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:34:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77394 2023-09-23T23:34:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707987+0000", 2023-09-23T23:34:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707987+0000", 2023-09-23T23:34:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707986+0000", 2023-09-23T23:34:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:34:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708018+0000", 2023-09-23T23:34:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1340999999999997e-05 2023-09-23T23:34:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708024+0000", 2023-09-23T23:34:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9050000000000002e-06 2023-09-23T23:34:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708027+0000", 2023-09-23T23:34:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6550000000000002e-06 2023-09-23T23:34:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709199+0000", 2023-09-23T23:34:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011713400000000001 2023-09-23T23:34:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709204+0000", 2023-09-23T23:34:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4480000000000002e-06 2023-09-23T23:34:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.869263+0000", 2023-09-23T23:34:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1600593370000001 2023-09-23T23:34:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.869419+0000", 2023-09-23T23:34:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015545899999999999 2023-09-23T23:34:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:34:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.869589+0000", 2023-09-23T23:34:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000170225 2023-09-23T23:34:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889913+0000", 2023-09-23T23:34:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020323536 2023-09-23T23:34:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907553+0000", 2023-09-23T23:34:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017640769000000001 2023-09-23T23:34:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910571+0000", 2023-09-23T23:34:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030181119999999999 2023-09-23T23:34:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910620+0000", 2023-09-23T23:34:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8891000000000003e-05 2023-09-23T23:34:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912211+0000", 2023-09-23T23:34:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015906270000000001 2023-09-23T23:34:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77463 13.d 13:b8eb00c3:::benchmark_data_smithi110_541928_object77462:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:34:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.708031+0000", 2023-09-23T23:34:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.7238235319999999, 2023-09-23T23:34:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.206478288, 2023-09-23T23:34:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:34:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:34:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77463 2023-09-23T23:34:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708031+0000", 2023-09-23T23:34:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708031+0000", 2023-09-23T23:34:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708031+0000", 2023-09-23T23:34:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:34:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708086+0000", 2023-09-23T23:34:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4630999999999999e-05 2023-09-23T23:34:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708092+0000", 2023-09-23T23:34:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1800000000000001e-06 2023-09-23T23:34:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708096+0000", 2023-09-23T23:34:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.811e-06 2023-09-23T23:34:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709228+0000", 2023-09-23T23:34:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011318190000000001 2023-09-23T23:34:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709234+0000", 2023-09-23T23:34:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2979999999999997e-06 2023-09-23T23:34:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.872414+0000", 2023-09-23T23:34:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.163180543 2023-09-23T23:34:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.872546+0000", 2023-09-23T23:34:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013156499999999999 2023-09-23T23:34:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:34:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.872714+0000", 2023-09-23T23:34:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000167713 2023-09-23T23:34:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889920+0000", 2023-09-23T23:34:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017206255 2023-09-23T23:34:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907578+0000", 2023-09-23T23:34:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017658231 2023-09-23T23:34:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912462+0000", 2023-09-23T23:34:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048837949999999998 2023-09-23T23:34:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912506+0000", 2023-09-23T23:34:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4131999999999997e-05 2023-09-23T23:34:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.914510+0000", 2023-09-23T23:34:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002003632 2023-09-23T23:34:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77467 13.d 13:b265ce95:::benchmark_data_smithi110_541928_object77466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:34:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.708101+0000", 2023-09-23T23:34:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.723754118, 2023-09-23T23:34:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2086463730000001, 2023-09-23T23:34:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:34:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:34:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77467 2023-09-23T23:34:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708101+0000", 2023-09-23T23:34:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708101+0000", 2023-09-23T23:34:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708101+0000", 2023-09-23T23:34:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:34:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708157+0000", 2023-09-23T23:34:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6196999999999997e-05 2023-09-23T23:34:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708163+0000", 2023-09-23T23:34:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0939999999999996e-06 2023-09-23T23:34:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708168+0000", 2023-09-23T23:34:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1800000000000004e-06 2023-09-23T23:34:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709259+0000", 2023-09-23T23:34:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010911370000000001 2023-09-23T23:34:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709266+0000", 2023-09-23T23:34:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7870000000000004e-06 2023-09-23T23:34:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.875164+0000", 2023-09-23T23:34:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1658984800000001 2023-09-23T23:34:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.875272+0000", 2023-09-23T23:34:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010731299999999999 2023-09-23T23:34:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:34:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.875417+0000", 2023-09-23T23:34:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014540399999999999 2023-09-23T23:34:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889926+0000", 2023-09-23T23:34:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014509024000000001 2023-09-23T23:34:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907925+0000", 2023-09-23T23:34:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017998692 2023-09-23T23:34:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.914568+0000", 2023-09-23T23:34:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066434789999999999 2023-09-23T23:34:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.914630+0000", 2023-09-23T23:34:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1175999999999998e-05 2023-09-23T23:34:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.916747+0000", 2023-09-23T23:34:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021177470000000001 2023-09-23T23:34:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77471 13.d 13:bd1602ed:::benchmark_data_smithi110_541928_object77470:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:34:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.708173+0000", 2023-09-23T23:34:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.723682267, 2023-09-23T23:34:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2113011469999999, 2023-09-23T23:34:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:34:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:34:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77471 2023-09-23T23:34:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708173+0000", 2023-09-23T23:34:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708173+0000", 2023-09-23T23:34:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708172+0000", 2023-09-23T23:34:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:34:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708227+0000", 2023-09-23T23:34:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4215000000000002e-05 2023-09-23T23:34:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708234+0000", 2023-09-23T23:34:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0790000000000003e-06 2023-09-23T23:34:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708238+0000", 2023-09-23T23:34:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6450000000000004e-06 2023-09-23T23:34:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709564+0000", 2023-09-23T23:34:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0013253379999999999 2023-09-23T23:34:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709570+0000", 2023-09-23T23:34:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5570000000000001e-06 2023-09-23T23:34:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.877443+0000", 2023-09-23T23:34:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.167873216 2023-09-23T23:34:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.877572+0000", 2023-09-23T23:34:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012810999999999999 2023-09-23T23:34:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:34:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.877732+0000", 2023-09-23T23:34:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000160376 2023-09-23T23:34:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889933+0000", 2023-09-23T23:34:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012201033999999999 2023-09-23T23:34:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910445+0000", 2023-09-23T23:34:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020511715999999999 2023-09-23T23:34:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.916902+0000", 2023-09-23T23:34:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064569379999999997 2023-09-23T23:34:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.916958+0000", 2023-09-23T23:34:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6443000000000002e-05 2023-09-23T23:34:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.919474+0000", 2023-09-23T23:34:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002515804 2023-09-23T23:34:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77477 13.d 13:b03c7b50:::benchmark_data_smithi110_541928_object77476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:34:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.708243+0000", 2023-09-23T23:34:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.723612277, 2023-09-23T23:34:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.213803755, 2023-09-23T23:34:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:34:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:34:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77477 2023-09-23T23:34:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708243+0000", 2023-09-23T23:34:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708243+0000", 2023-09-23T23:34:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708242+0000", 2023-09-23T23:34:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:34:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708286+0000", 2023-09-23T23:34:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3625999999999999e-05 2023-09-23T23:34:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708292+0000", 2023-09-23T23:34:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0100000000000001e-06 2023-09-23T23:34:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708296+0000", 2023-09-23T23:34:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1860000000000002e-06 2023-09-23T23:34:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709595+0000", 2023-09-23T23:34:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012991089999999999 2023-09-23T23:34:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709602+0000", 2023-09-23T23:34:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6100000000000002e-06 2023-09-23T23:34:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.879894+0000", 2023-09-23T23:34:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.170292077 2023-09-23T23:34:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.879990+0000", 2023-09-23T23:34:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6212000000000007e-05 2023-09-23T23:34:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:34:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.880120+0000", 2023-09-23T23:34:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012987 2023-09-23T23:34:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889937+0000", 2023-09-23T23:34:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0098164270000000008 2023-09-23T23:34:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910473+0000", 2023-09-23T23:34:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020536688000000001 2023-09-23T23:34:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.919527+0000", 2023-09-23T23:34:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0090542190000000005 2023-09-23T23:34:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.919586+0000", 2023-09-23T23:34:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8749999999999998e-05 2023-09-23T23:34:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.922046+0000", 2023-09-23T23:34:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024603099999999998 2023-09-23T23:34:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77503 13.d 13:b2378296:::benchmark_data_smithi110_541928_object77502:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:34:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.781601+0000", 2023-09-23T23:34:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.650253948, 2023-09-23T23:34:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1428059340000001, 2023-09-23T23:34:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:34:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:34:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77503 2023-09-23T23:34:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781601+0000", 2023-09-23T23:34:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781601+0000", 2023-09-23T23:34:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781600+0000", 2023-09-23T23:34:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:34:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781628+0000", 2023-09-23T23:34:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8694999999999999e-05 2023-09-23T23:34:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781638+0000", 2023-09-23T23:34:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5959999999999993e-06 2023-09-23T23:34:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781643+0000", 2023-09-23T23:34:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4970000000000001e-06 2023-09-23T23:34:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781701+0000", 2023-09-23T23:34:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7757000000000003e-05 2023-09-23T23:34:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781711+0000", 2023-09-23T23:34:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4409999999999999e-06 2023-09-23T23:34:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.882289+0000", 2023-09-23T23:34:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1005785459999999 2023-09-23T23:34:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.882445+0000", 2023-09-23T23:34:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015557 2023-09-23T23:34:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:34:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.882635+0000", 2023-09-23T23:34:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000190087 2023-09-23T23:34:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.892019+0000", 2023-09-23T23:34:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0093840280000000009 2023-09-23T23:34:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910507+0000", 2023-09-23T23:34:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018488389000000001 2023-09-23T23:34:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.922095+0000", 2023-09-23T23:34:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011588144999999999 2023-09-23T23:34:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.922148+0000", 2023-09-23T23:34:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2661000000000003e-05 2023-09-23T23:34:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.924407+0000", 2023-09-23T23:34:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022589189999999999 2023-09-23T23:34:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77524 13.d 13:bc4b3c00:::benchmark_data_smithi110_541928_object77523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:34:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.815029+0000", 2023-09-23T23:34:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.616825757, 2023-09-23T23:34:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.112009437, 2023-09-23T23:34:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:34:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:34:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77524 2023-09-23T23:34:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815029+0000", 2023-09-23T23:34:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815029+0000", 2023-09-23T23:34:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815028+0000", 2023-09-23T23:34:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:34:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815060+0000", 2023-09-23T23:34:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2218999999999998e-05 2023-09-23T23:34:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815074+0000", 2023-09-23T23:34:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.43e-05 2023-09-23T23:34:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815081+0000", 2023-09-23T23:34:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0700000000000003e-06 2023-09-23T23:34:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815152+0000", 2023-09-23T23:34:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1408999999999993e-05 2023-09-23T23:34:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815162+0000", 2023-09-23T23:34:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9459999999999993e-06 2023-09-23T23:34:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.884789+0000", 2023-09-23T23:34:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.069627511 2023-09-23T23:34:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.884895+0000", 2023-09-23T23:34:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105283 2023-09-23T23:34:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:34:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.885012+0000", 2023-09-23T23:34:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011697900000000001 2023-09-23T23:34:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.892026+0000", 2023-09-23T23:34:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0070145779999999996 2023-09-23T23:34:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910539+0000", 2023-09-23T23:34:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018512857000000001 2023-09-23T23:34:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.924468+0000", 2023-09-23T23:34:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01392849 2023-09-23T23:34:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.924523+0000", 2023-09-23T23:34:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4948000000000003e-05 2023-09-23T23:34:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.927039+0000", 2023-09-23T23:34:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002516108 2023-09-23T23:34:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77547 13.d 13:bc3de976:::benchmark_data_smithi110_541928_object77546:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:34:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.859759+0000", 2023-09-23T23:34:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.5720955619999999, 2023-09-23T23:34:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0697625900000001, 2023-09-23T23:34:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:34:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:34:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77547 2023-09-23T23:34:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859759+0000", 2023-09-23T23:34:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859759+0000", 2023-09-23T23:34:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859758+0000", 2023-09-23T23:34:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:34:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859847+0000", 2023-09-23T23:34:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8799000000000001e-05 2023-09-23T23:34:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859861+0000", 2023-09-23T23:34:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4229e-05 2023-09-23T23:34:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859868+0000", 2023-09-23T23:34:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4949999999999996e-06 2023-09-23T23:34:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859928+0000", 2023-09-23T23:34:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0738e-05 2023-09-23T23:34:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859941+0000", 2023-09-23T23:34:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.24e-05 2023-09-23T23:34:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.887534+0000", 2023-09-23T23:34:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0275931309999999 2023-09-23T23:34:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.887728+0000", 2023-09-23T23:34:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000194165 2023-09-23T23:34:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:34:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.887881+0000", 2023-09-23T23:34:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000153263 2023-09-23T23:34:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.903469+0000", 2023-09-23T23:34:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015587215 2023-09-23T23:34:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912270+0000", 2023-09-23T23:34:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088015479999999993 2023-09-23T23:34:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.927101+0000", 2023-09-23T23:34:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014831195 2023-09-23T23:34:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.927169+0000", 2023-09-23T23:34:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8172999999999999e-05 2023-09-23T23:34:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.929522+0000", 2023-09-23T23:34:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023525769999999998 2023-09-23T23:34:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77576 13.d 13:b73edecb:::benchmark_data_smithi110_541928_object77575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:34:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.919341+0000", 2023-09-23T23:34:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.5125142220000001, 2023-09-23T23:34:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0128839199999999, 2023-09-23T23:34:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:34:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:34:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:34:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:34:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77576 2023-09-23T23:34:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:34:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:34:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919341+0000", 2023-09-23T23:34:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:34:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919341+0000", 2023-09-23T23:34:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:34:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:34:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919339+0000", 2023-09-23T23:34:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:34:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:34:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919384+0000", 2023-09-23T23:34:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5042e-05 2023-09-23T23:34:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:34:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919403+0000", 2023-09-23T23:34:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9136000000000001e-05 2023-09-23T23:34:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:34:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919412+0000", 2023-09-23T23:34:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4589999999999998e-06 2023-09-23T23:34:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.924505+0000", 2023-09-23T23:34:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050933469999999998 2023-09-23T23:34:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:34:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.924519+0000", 2023-09-23T23:34:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3852000000000001e-05 2023-09-23T23:34:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:34:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889955+0000", 2023-09-23T23:34:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96543663400000002 2023-09-23T23:34:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:34:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.890057+0000", 2023-09-23T23:34:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010134899999999999 2023-09-23T23:34:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:34:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.890180+0000", 2023-09-23T23:34:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012325699999999999 2023-09-23T23:34:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:34:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.903476+0000", 2023-09-23T23:34:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013296290000000001 2023-09-23T23:34:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912329+0000", 2023-09-23T23:34:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088524599999999995 2023-09-23T23:34:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.929574+0000", 2023-09-23T23:34:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017245415 2023-09-23T23:34:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:34:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.929630+0000", 2023-09-23T23:34:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6097999999999997e-05 2023-09-23T23:34:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:34:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:34:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:34:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.932225+0000", 2023-09-23T23:34:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002594332 2023-09-23T23:34:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:34:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:34:05.614 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:34:05.615 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[212939]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.615 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[212939]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.615 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[212939]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.615 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[212939]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[212939]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[212939]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[212939]: pgmap v1549: 119 pgs: 3 peering, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 35 MiB/s wr, 558 op/s; 31488/226980 objects degraded (13.873%); 8181/226980 objects misplaced (3.604%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T23:34:05.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[212939]: failed to encode map e564 with expected crc 2023-09-23T23:34:05.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[212939]: osdmap e564: 8 total, 8 up, 7 in 2023-09-23T23:34:05.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[212939]: failed to encode map e564 with expected crc 2023-09-23T23:34:05.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:34:05.617 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:05 smithi138 ceph-mon[193606]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.617 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:05 smithi138 ceph-mon[193606]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.617 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:05 smithi138 ceph-mon[193606]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.617 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:05 smithi138 ceph-mon[193606]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.617 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:05 smithi138 ceph-mon[193606]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.617 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:05 smithi138 ceph-mon[193606]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.618 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:05 smithi138 ceph-mon[193606]: pgmap v1549: 119 pgs: 3 peering, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 35 MiB/s wr, 558 op/s; 31488/226980 objects degraded (13.873%); 8181/226980 objects misplaced (3.604%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T23:34:05.618 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:05 smithi138 ceph-mon[193606]: failed to encode map e564 with expected crc 2023-09-23T23:34:05.618 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:05 smithi138 ceph-mon[193606]: osdmap e564: 8 total, 8 up, 7 in 2023-09-23T23:34:05.618 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:05 smithi138 ceph-mon[193606]: failed to encode map e564 with expected crc 2023-09-23T23:34:05.618 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:05 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:34:05.618 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[221102]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.618 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[221102]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.618 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[221102]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.618 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[221102]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.619 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[221102]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.619 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[221102]: failed to encode map e563 with expected crc 2023-09-23T23:34:05.619 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[221102]: pgmap v1549: 119 pgs: 3 peering, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 35 MiB/s wr, 558 op/s; 31488/226980 objects degraded (13.873%); 8181/226980 objects misplaced (3.604%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T23:34:05.619 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[221102]: failed to encode map e564 with expected crc 2023-09-23T23:34:05.619 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[221102]: osdmap e564: 8 total, 8 up, 7 in 2023-09-23T23:34:05.619 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[221102]: failed to encode map e564 with expected crc 2023-09-23T23:34:05.619 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:05 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:34:05.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:05.620 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:34:05.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:34:05.722 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:34:05.784 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:05.798 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:05.857 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 68 16 78209 78193 71.8465 37.25 0.0211558 0.0139112 2023-09-23T23:34:05.928 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:34:05.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:05.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:34:06.038 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:06.107 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:06.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:06 smithi138 ceph-mon[193606]: failed to encode map e564 with expected crc 2023-09-23T23:34:06.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:06 smithi138 ceph-mon[193606]: failed to encode map e564 with expected crc 2023-09-23T23:34:06.184 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:34:06.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:06.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:34:06.293 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:34:06.370 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:06.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:06 smithi110 ceph-mon[212939]: failed to encode map e564 with expected crc 2023-09-23T23:34:06.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:06 smithi110 ceph-mon[212939]: failed to encode map e564 with expected crc 2023-09-23T23:34:06.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:06 smithi110 ceph-mon[221102]: failed to encode map e564 with expected crc 2023-09-23T23:34:06.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:06 smithi110 ceph-mon[221102]: failed to encode map e564 with expected crc 2023-09-23T23:34:06.469 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:34:06.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:06.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:34:06.583 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:34:06.668 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:06.844 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:34:06.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:06.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:34:06.858 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 69 16 78865 78849 71.3987 41 0.0105979 0.0139995 2023-09-23T23:34:06.952 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:34:06.982 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:07.110 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:07 smithi110 ceph-mon[212939]: pgmap v1551: 119 pgs: 3 peering, 4 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.6 KiB/s rd, 29 MiB/s wr, 468 op/s; 22522/232491 objects degraded (9.687%); 16963/232491 objects misplaced (7.296%); 6.1 MiB/s, 96 objects/s recovering 2023-09-23T23:34:07.110 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:07 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 22522/232491 objects degraded (9.687%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:07.112 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:34:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:34:06] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.43.0" 2023-09-23T23:34:07.316 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:34:07.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:07.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:34:07.423 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:07.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:07 smithi138 ceph-mon[193606]: pgmap v1551: 119 pgs: 3 peering, 4 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.6 KiB/s rd, 29 MiB/s wr, 468 op/s; 22522/232491 objects degraded (9.687%); 16963/232491 objects misplaced (7.296%); 6.1 MiB/s, 96 objects/s recovering 2023-09-23T23:34:07.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:07 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 22522/232491 objects degraded (9.687%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:07.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:07 smithi110 ceph-mon[221102]: pgmap v1551: 119 pgs: 3 peering, 4 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.6 KiB/s rd, 29 MiB/s wr, 468 op/s; 22522/232491 objects degraded (9.687%); 16963/232491 objects misplaced (7.296%); 6.1 MiB/s, 96 objects/s recovering 2023-09-23T23:34:07.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:07 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 22522/232491 objects degraded (9.687%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:07.492 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:07.610 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:07.623 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:34:07.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:07.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:07.725 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:07.803 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:07.821 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:34:07.858 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 70 16 79464 79448 70.9136 37.4375 0.0244299 0.0140947 2023-09-23T23:34:07.939 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:07.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:07.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:08.047 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:34:08.245 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:08.353 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:34:08.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:08.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:34:08.406 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:08.479 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:34:08.544 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:08.629 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:08.791 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:34:08.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:08.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:34:08.858 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 71 16 80108 80092 70.4817 40.25 0.0168024 0.0141819 2023-09-23T23:34:08.905 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:08.998 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:09.141 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:09 smithi110 ceph-mon[212939]: pgmap v1552: 119 pgs: 3 peering, 4 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.5 KiB/s rd, 25 MiB/s wr, 399 op/s; 22522/232491 objects degraded (9.687%); 16963/232491 objects misplaced (7.296%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T23:34:09.173 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:34:09.189 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:09.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:09.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:09.305 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:09.371 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:09 smithi138 ceph-mon[193606]: pgmap v1552: 119 pgs: 3 peering, 4 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.5 KiB/s rd, 25 MiB/s wr, 399 op/s; 22522/232491 objects degraded (9.687%); 16963/232491 objects misplaced (7.296%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T23:34:09.371 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:09 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1330965491' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:34:09.371 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:09 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:34:09.372 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:09.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:09 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1330965491' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:34:09.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:09 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:34:09.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:09 smithi110 ceph-mon[221102]: pgmap v1552: 119 pgs: 3 peering, 4 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.5 KiB/s rd, 25 MiB/s wr, 399 op/s; 22522/232491 objects degraded (9.687%); 16963/232491 objects misplaced (7.296%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T23:34:09.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:09 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1330965491' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:34:09.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:09 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:34:09.498 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:09.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:09.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:09.610 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:34:09.670 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:09.859 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 72 16 80756 80740 70.0653 40.5 0.0250565 0.0142669 2023-09-23T23:34:10.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:10 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:34:10.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:10 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:34:10.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:10 smithi138 ceph-mon[193606]: osdmap e565: 8 total, 8 up, 7 in 2023-09-23T23:34:10.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:10 smithi138 ceph-mon[193606]: failed to encode map e565 with expected crc 2023-09-23T23:34:10.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:10 smithi138 ceph-mon[193606]: failed to encode map e565 with expected crc 2023-09-23T23:34:10.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:10 smithi138 ceph-mon[193606]: failed to encode map e565 with expected crc 2023-09-23T23:34:10.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:10 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:34:10.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:10 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:34:10.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:10 smithi110 ceph-mon[212939]: osdmap e565: 8 total, 8 up, 7 in 2023-09-23T23:34:10.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:10 smithi110 ceph-mon[212939]: failed to encode map e565 with expected crc 2023-09-23T23:34:10.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:10 smithi110 ceph-mon[212939]: failed to encode map e565 with expected crc 2023-09-23T23:34:10.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:10 smithi110 ceph-mon[212939]: failed to encode map e565 with expected crc 2023-09-23T23:34:10.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:10 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:34:10.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:10 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:34:10.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:10 smithi110 ceph-mon[221102]: osdmap e565: 8 total, 8 up, 7 in 2023-09-23T23:34:10.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:10 smithi110 ceph-mon[221102]: failed to encode map e565 with expected crc 2023-09-23T23:34:10.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:10 smithi110 ceph-mon[221102]: failed to encode map e565 with expected crc 2023-09-23T23:34:10.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:10 smithi110 ceph-mon[221102]: failed to encode map e565 with expected crc 2023-09-23T23:34:10.623 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 5, 7, 2, 1, 6] out_osds: [4] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:34:10.623 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:34:10.623 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T23:34:10.623 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.000000 2023-09-23T23:34:10.623 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 0 2023-09-23T23:34:10.858 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 73 16 81418 81402 69.6723 41.375 0.0198006 0.0143477 2023-09-23T23:34:11.147 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:11 smithi110 ceph-mon[212939]: failed to encode map e565 with expected crc 2023-09-23T23:34:11.148 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:11 smithi110 ceph-mon[212939]: pgmap v1554: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 3.4 KiB/s rd, 46 MiB/s wr, 742 op/s; 26593/240699 objects degraded (11.048%); 12183/240699 objects misplaced (5.062%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T23:34:11.148 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:11 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T23:34:11.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:11 smithi138 ceph-mon[193606]: failed to encode map e565 with expected crc 2023-09-23T23:34:11.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:11 smithi138 ceph-mon[193606]: pgmap v1554: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 3.4 KiB/s rd, 46 MiB/s wr, 742 op/s; 26593/240699 objects degraded (11.048%); 12183/240699 objects misplaced (5.062%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T23:34:11.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:11 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T23:34:11.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:11 smithi110 ceph-mon[221102]: failed to encode map e565 with expected crc 2023-09-23T23:34:11.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:11 smithi110 ceph-mon[221102]: pgmap v1554: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 3.4 KiB/s rd, 46 MiB/s wr, 742 op/s; 26593/240699 objects degraded (11.048%); 12183/240699 objects misplaced (5.062%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T23:34:11.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:11 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T23:34:11.565 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:34:11.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:11.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:34:11.682 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:34:11.738 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:11.858 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 74 16 82080 82064 69.29 41.375 0.0426285 0.0144266 2023-09-23T23:34:12.663 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:34:12.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:12.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:34:12.776 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:34:12.858 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 75 16 82679 82663 68.8653 37.4375 0.0215361 0.0145149 2023-09-23T23:34:12.861 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:12.951 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:13.011 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:34:13.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:13.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:34:13.055 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:34:13.092 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:34:13.124 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:34:13.149 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:34:13.186 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:34:13.199 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:13.229 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:34:13.296 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:34:13.337 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:34:13.383 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:34:13.391 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:34:13.392 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:34:13.392 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:34:13.392 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:34:13.392 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:34:13.392 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:34:13.392 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:34:13.392 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:34:13.411 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:34:13.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:13.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:13 smithi110 ceph-mon[212939]: pgmap v1555: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2.6 KiB/s rd, 36 MiB/s wr, 573 op/s; 26593/240699 objects degraded (11.048%); 12183/240699 objects misplaced (5.062%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T23:34:13.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:13 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/116126240' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:34:13.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:13.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:34:13.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:13 smithi110 ceph-mon[221102]: pgmap v1555: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2.6 KiB/s rd, 36 MiB/s wr, 573 op/s; 26593/240699 objects degraded (11.048%); 12183/240699 objects misplaced (5.062%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T23:34:13.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:13 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/116126240' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:34:13.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:13.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:34:13.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:34:13.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:13 smithi138 ceph-mon[193606]: pgmap v1555: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2.6 KiB/s rd, 36 MiB/s wr, 573 op/s; 26593/240699 objects degraded (11.048%); 12183/240699 objects misplaced (5.062%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T23:34:13.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:13 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/116126240' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:34:13.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:13.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:34:13.446 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:34:13.446 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:34:13.526 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:34:13.589 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:13.595 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:34:13.859 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 76 16 83320 83304 68.4864 40.0625 0.0270726 0.0145961 2023-09-23T23:34:13.952 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:34:13.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:13.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:34:14.060 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:14.163 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:14 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 26593/240699 objects degraded (11.048%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:14 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:14.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:14 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 26593/240699 objects degraded (11.048%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:14.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:14 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:14.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:14 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 26593/240699 objects degraded (11.048%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:14.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:14 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:14.859 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 77 16 83933 83917 68.0945 38.3125 0.0195385 0.0146789 2023-09-23T23:34:15.157 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:15 smithi110 ceph-mon[212939]: pgmap v1556: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 26 MiB/s wr, 420 op/s; 26593/243141 objects degraded (10.937%); 12005/243141 objects misplaced (4.937%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T23:34:15.160 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:15 smithi110 ceph-mon[221102]: pgmap v1556: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 26 MiB/s wr, 420 op/s; 26593/243141 objects degraded (10.937%); 12005/243141 objects misplaced (4.937%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T23:34:15.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:15 smithi138 ceph-mon[193606]: pgmap v1556: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 26 MiB/s wr, 420 op/s; 26593/243141 objects degraded (10.937%); 12005/243141 objects misplaced (4.937%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T23:34:15.859 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 78 16 84494 84478 67.6711 35.0625 0.0237015 0.0147627 2023-09-23T23:34:16.040 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:16.041 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:34:16.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:16.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:16.147 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:16.211 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:16.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:16.389 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:16.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:16.402 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:16.490 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:16.575 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:16.727 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:34:16.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:16.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:34:16.834 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:34:16.859 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 79 16 85062 85046 67.2639 35.5 0.0121423 0.0148624 2023-09-23T23:34:16.910 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:16.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:34:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:34:16] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.43.0" 2023-09-23T23:34:17.013 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:34:17.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:17.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:34:17.124 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:34:17.197 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:17.225 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:17.337 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:34:17.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:17.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:34:17.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:17 smithi138 ceph-mon[193606]: pgmap v1557: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 26593/250386 objects degraded (10.621%); 11682/250386 objects misplaced (4.666%); 7.6 MiB/s, 120 objects/s recovering 2023-09-23T23:34:17.452 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:34:17.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:17 smithi110 ceph-mon[212939]: pgmap v1557: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 26593/250386 objects degraded (10.621%); 11682/250386 objects misplaced (4.666%); 7.6 MiB/s, 120 objects/s recovering 2023-09-23T23:34:17.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:17 smithi110 ceph-mon[221102]: pgmap v1557: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 26593/250386 objects degraded (10.621%); 11682/250386 objects misplaced (4.666%); 7.6 MiB/s, 120 objects/s recovering 2023-09-23T23:34:17.534 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:17.693 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:34:17.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:17.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:34:17.812 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:17.840 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:17.859 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:34:17.858362+0000 min lat: 0.00107054 max lat: 2.51963 avg lat: 0.0149266 2023-09-23T23:34:17.859 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:34:17.859 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 80 16 85645 85629 66.8786 36.4375 0.00936134 0.0149266 2023-09-23T23:34:18.147 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:34:18.218 INFO:teuthology.orchestra.run.smithi110.stderr:set osd.7 primary-affinity to 0 (802) 2023-09-23T23:34:18.281 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:18.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:18.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:18.389 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:34:18.417 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:18.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/143871215' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-09-23T23:34:18.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:18 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-09-23T23:34:18.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/143871215' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-09-23T23:34:18.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:18 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-09-23T23:34:18.464 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:34:18.471 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/143871215' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-09-23T23:34:18.471 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:18 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-09-23T23:34:18.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:18.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:34:18.573 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:18.600 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:18.645 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:18.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:18.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:18.751 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:18.782 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:18.859 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 81 16 85888 85872 66.2405 15.1875 0.46321 0.0149955 2023-09-23T23:34:19.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[212939]: pgmap v1558: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 26593/250386 objects degraded (10.621%); 11682/250386 objects misplaced (4.666%); 7.6 MiB/s, 120 objects/s recovering 2023-09-23T23:34:19.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[212939]: failed to encode map e566 with expected crc 2023-09-23T23:34:19.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[212939]: failed to encode map e566 with expected crc 2023-09-23T23:34:19.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-09-23T23:34:19.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[212939]: osdmap e566: 8 total, 8 up, 7 in 2023-09-23T23:34:19.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 26593/250386 objects degraded (10.621%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:19.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[212939]: failed to encode map e567 with expected crc 2023-09-23T23:34:19.187 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[212939]: osdmap e567: 8 total, 8 up, 7 in 2023-09-23T23:34:19.188 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[212939]: failed to encode map e567 with expected crc 2023-09-23T23:34:19.188 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[212939]: failed to encode map e567 with expected crc 2023-09-23T23:34:19.188 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[212939]: failed to encode map e567 with expected crc 2023-09-23T23:34:19.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[221102]: pgmap v1558: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 26593/250386 objects degraded (10.621%); 11682/250386 objects misplaced (4.666%); 7.6 MiB/s, 120 objects/s recovering 2023-09-23T23:34:19.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[221102]: failed to encode map e566 with expected crc 2023-09-23T23:34:19.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[221102]: failed to encode map e566 with expected crc 2023-09-23T23:34:19.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-09-23T23:34:19.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[221102]: osdmap e566: 8 total, 8 up, 7 in 2023-09-23T23:34:19.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 26593/250386 objects degraded (10.621%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:19.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[221102]: failed to encode map e567 with expected crc 2023-09-23T23:34:19.188 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[221102]: osdmap e567: 8 total, 8 up, 7 in 2023-09-23T23:34:19.189 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[221102]: failed to encode map e567 with expected crc 2023-09-23T23:34:19.189 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[221102]: failed to encode map e567 with expected crc 2023-09-23T23:34:19.189 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:19 smithi110 ceph-mon[221102]: failed to encode map e567 with expected crc 2023-09-23T23:34:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:19 smithi138 ceph-mon[193606]: pgmap v1558: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 26593/250386 objects degraded (10.621%); 11682/250386 objects misplaced (4.666%); 7.6 MiB/s, 120 objects/s recovering 2023-09-23T23:34:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:19 smithi138 ceph-mon[193606]: failed to encode map e566 with expected crc 2023-09-23T23:34:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:19 smithi138 ceph-mon[193606]: failed to encode map e566 with expected crc 2023-09-23T23:34:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:19 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-09-23T23:34:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:19 smithi138 ceph-mon[193606]: osdmap e566: 8 total, 8 up, 7 in 2023-09-23T23:34:19.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:19 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 26593/250386 objects degraded (10.621%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:19.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:19 smithi138 ceph-mon[193606]: failed to encode map e567 with expected crc 2023-09-23T23:34:19.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:19 smithi138 ceph-mon[193606]: osdmap e567: 8 total, 8 up, 7 in 2023-09-23T23:34:19.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:19 smithi138 ceph-mon[193606]: failed to encode map e567 with expected crc 2023-09-23T23:34:19.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:19 smithi138 ceph-mon[193606]: failed to encode map e567 with expected crc 2023-09-23T23:34:19.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:19 smithi138 ceph-mon[193606]: failed to encode map e567 with expected crc 2023-09-23T23:34:19.532 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:19.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:19.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:19.637 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:34:19.664 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:19.711 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:34:19.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:19.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:34:19.826 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:19.859 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 82 16 85928 85912 65.4633 2.5 0.00931134 0.0151727 2023-09-23T23:34:19.874 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:19.931 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:19.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:19.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:20.036 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:34:20.069 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:20.077 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:20.352 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:20 smithi110 ceph-mon[212939]: failed to encode map e567 with expected crc 2023-09-23T23:34:20.352 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:20 smithi110 ceph-mon[212939]: osdmap e568: 8 total, 8 up, 7 in 2023-09-23T23:34:20.352 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:20 smithi110 ceph-mon[212939]: failed to encode map e568 with expected crc 2023-09-23T23:34:20.353 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:20 smithi110 ceph-mon[221102]: failed to encode map e567 with expected crc 2023-09-23T23:34:20.353 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:20 smithi110 ceph-mon[221102]: osdmap e568: 8 total, 8 up, 7 in 2023-09-23T23:34:20.353 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:20 smithi110 ceph-mon[221102]: failed to encode map e568 with expected crc 2023-09-23T23:34:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:20 smithi138 ceph-mon[193606]: failed to encode map e567 with expected crc 2023-09-23T23:34:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:20 smithi138 ceph-mon[193606]: osdmap e568: 8 total, 8 up, 7 in 2023-09-23T23:34:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:20 smithi138 ceph-mon[193606]: failed to encode map e568 with expected crc 2023-09-23T23:34:20.859 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 83 16 85980 85964 64.7139 3.25 0.00430598 0.0151722 2023-09-23T23:34:21.210 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:34:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:21 smithi138 ceph-mon[193606]: failed to encode map e568 with expected crc 2023-09-23T23:34:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:21 smithi138 ceph-mon[193606]: failed to encode map e568 with expected crc 2023-09-23T23:34:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:21 smithi138 ceph-mon[193606]: pgmap v1562: 119 pgs: 1 remapped+peering, 1 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 44 MiB/s wr, 708 op/s; 22046/255891 objects degraded (8.615%); 7647/255891 objects misplaced (2.988%); 10 MiB/s, 158 objects/s recovering 2023-09-23T23:34:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:21 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3256198782' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:34:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:21 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:34:21.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:21 smithi110 ceph-mon[212939]: failed to encode map e568 with expected crc 2023-09-23T23:34:21.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:21 smithi110 ceph-mon[212939]: failed to encode map e568 with expected crc 2023-09-23T23:34:21.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:21 smithi110 ceph-mon[212939]: pgmap v1562: 119 pgs: 1 remapped+peering, 1 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 44 MiB/s wr, 708 op/s; 22046/255891 objects degraded (8.615%); 7647/255891 objects misplaced (2.988%); 10 MiB/s, 158 objects/s recovering 2023-09-23T23:34:21.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:21 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3256198782' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:34:21.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:21 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:34:21.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:21 smithi110 ceph-mon[221102]: failed to encode map e568 with expected crc 2023-09-23T23:34:21.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:21 smithi110 ceph-mon[221102]: failed to encode map e568 with expected crc 2023-09-23T23:34:21.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:21 smithi110 ceph-mon[221102]: pgmap v1562: 119 pgs: 1 remapped+peering, 1 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 44 MiB/s wr, 708 op/s; 22046/255891 objects degraded (8.615%); 7647/255891 objects misplaced (2.988%); 10 MiB/s, 158 objects/s recovering 2023-09-23T23:34:21.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:21 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3256198782' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:34:21.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:21 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:34:21.859 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 84 16 86231 86215 64.1304 15.6875 0.547403 0.0155579 2023-09-23T23:34:21.893 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:34:21.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:21.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:34:21.998 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:22.028 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:22.076 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:22.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:22.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:22.183 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:34:22.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:22 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:34:22.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:22 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:34:22.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:22 smithi110 ceph-mon[212939]: osdmap e569: 8 total, 8 up, 7 in 2023-09-23T23:34:22.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:22 smithi110 ceph-mon[212939]: failed to encode map e569 with expected crc 2023-09-23T23:34:22.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:22 smithi110 ceph-mon[212939]: failed to encode map e569 with expected crc 2023-09-23T23:34:22.191 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:22 smithi110 ceph-mon[212939]: failed to encode map e569 with expected crc 2023-09-23T23:34:22.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:22 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:34:22.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:22 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:34:22.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:22 smithi110 ceph-mon[221102]: osdmap e569: 8 total, 8 up, 7 in 2023-09-23T23:34:22.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:22 smithi110 ceph-mon[221102]: failed to encode map e569 with expected crc 2023-09-23T23:34:22.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:22 smithi110 ceph-mon[221102]: failed to encode map e569 with expected crc 2023-09-23T23:34:22.192 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:22 smithi110 ceph-mon[221102]: failed to encode map e569 with expected crc 2023-09-23T23:34:22.214 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:22.269 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:34:22.276 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:22 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:34:22.276 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:22 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:34:22.276 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:22 smithi138 ceph-mon[193606]: osdmap e569: 8 total, 8 up, 7 in 2023-09-23T23:34:22.276 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:22 smithi138 ceph-mon[193606]: failed to encode map e569 with expected crc 2023-09-23T23:34:22.276 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:22 smithi138 ceph-mon[193606]: failed to encode map e569 with expected crc 2023-09-23T23:34:22.276 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:22 smithi138 ceph-mon[193606]: failed to encode map e569 with expected crc 2023-09-23T23:34:22.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:22.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:34:22.378 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:34:22.411 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:22.468 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:34:22.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:22.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:34:22.576 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:22.652 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:22.859 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 85 16 86485 86469 63.5627 15.875 0.00979495 0.0156895 2023-09-23T23:34:23.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:23 smithi110 ceph-mon[221102]: pgmap v1564: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 1 peering, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 5.2 KiB/s rd, 38 MiB/s wr, 618 op/s; 17511/257757 objects degraded (6.794%); 12004/257757 objects misplaced (4.657%); 13 MiB/s, 201 objects/s recovering 2023-09-23T23:34:23.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:23 smithi110 ceph-mon[212939]: pgmap v1564: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 1 peering, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 5.2 KiB/s rd, 38 MiB/s wr, 618 op/s; 17511/257757 objects degraded (6.794%); 12004/257757 objects misplaced (4.657%); 13 MiB/s, 201 objects/s recovering 2023-09-23T23:34:23.684 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:23.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:23 smithi138 ceph-mon[193606]: pgmap v1564: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 1 peering, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 5.2 KiB/s rd, 38 MiB/s wr, 618 op/s; 17511/257757 objects degraded (6.794%); 12004/257757 objects misplaced (4.657%); 13 MiB/s, 201 objects/s recovering 2023-09-23T23:34:23.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:23.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:23.797 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:34:23.856 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:23.859 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 86 16 86745 86729 63.0127 16.25 0.0064955 0.0158272 2023-09-23T23:34:24.060 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:34:24.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:24.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:34:24.166 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:24.198 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:24.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:24 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 17511/257757 objects degraded (6.794%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:24.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:24 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 17511/257757 objects degraded (6.794%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:24.529 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 5, 7, 2, 1, 6] out_osds: [4] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:34:24.529 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T23:34:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:24 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 17511/257757 objects degraded (6.794%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:24.860 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 87 16 87002 86986 62.4731 16.0625 0.00405278 0.0159621 2023-09-23T23:34:25.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:25 smithi110 ceph-mon[221102]: pgmap v1565: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 1 peering, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 3.7 KiB/s rd, 27 MiB/s wr, 434 op/s; 17511/257757 objects degraded (6.794%); 12004/257757 objects misplaced (4.657%); 9.0 MiB/s, 141 objects/s recovering 2023-09-23T23:34:25.459 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:25 smithi110 ceph-mon[212939]: pgmap v1565: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 1 peering, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 3.7 KiB/s rd, 27 MiB/s wr, 434 op/s; 17511/257757 objects degraded (6.794%); 12004/257757 objects misplaced (4.657%); 9.0 MiB/s, 141 objects/s recovering 2023-09-23T23:34:25.517 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:34:25.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:25.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:34:25.629 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:25.654 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:25 smithi138 ceph-mon[193606]: pgmap v1565: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 1 peering, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 3.7 KiB/s rd, 27 MiB/s wr, 434 op/s; 17511/257757 objects degraded (6.794%); 12004/257757 objects misplaced (4.657%); 9.0 MiB/s, 141 objects/s recovering 2023-09-23T23:34:25.657 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:25.701 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:34:25.709 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:25.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:25.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:25.787 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:34:25.788 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:34:25.807 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:25.814 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:34:25.868 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 88 16 87310 87294 61.976 19.25 0.0122165 0.0160808 2023-09-23T23:34:25.904 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:26.350 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:34:26.351 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:34:26.351 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:34:26.358 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:34:26.385 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:34:26.411 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:34:26.419 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:34:26.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:26.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:34:26.445 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:34:26.480 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:34:26.489 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:34:26.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:34:26.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:34:26.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:34:26.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:34:26.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:34:26.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:34:26.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:34:26.524 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:34:26.557 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:26.617 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:34:26.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:26.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:34:26.723 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:26.762 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:26.868 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 89 16 87540 87524 61.4413 14.375 0.138037 0.0162266 2023-09-23T23:34:27.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:34:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:34:26] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.43.0" 2023-09-23T23:34:27.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:27 smithi138 ceph-mon[193606]: pgmap v1566: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2.9 KiB/s rd, 25 MiB/s wr, 398 op/s; 25450/259896 objects degraded (9.792%); 11300/259896 objects misplaced (4.348%); 11 MiB/s, 179 objects/s recovering 2023-09-23T23:34:27.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:27 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2042555872' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:34:27.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:27 smithi110 ceph-mon[212939]: pgmap v1566: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2.9 KiB/s rd, 25 MiB/s wr, 398 op/s; 25450/259896 objects degraded (9.792%); 11300/259896 objects misplaced (4.348%); 11 MiB/s, 179 objects/s recovering 2023-09-23T23:34:27.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:27 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2042555872' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:34:27.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:27 smithi110 ceph-mon[221102]: pgmap v1566: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2.9 KiB/s rd, 25 MiB/s wr, 398 op/s; 25450/259896 objects degraded (9.792%); 11300/259896 objects misplaced (4.348%); 11 MiB/s, 179 objects/s recovering 2023-09-23T23:34:27.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:27 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2042555872' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:34:27.816 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:27.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:27.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:27.868 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 90 12 87777 87765 60.9261 15.0625 0.562889 0.0163653 2023-09-23T23:34:27.921 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:27.953 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:28.016 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:34:28.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:28.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:34:28.121 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:34:28.151 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:28.334 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:34:28.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:28.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:34:28.439 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:34:28.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:34:28.466 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:28.514 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:34:28.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:28.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:34:28.619 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:34:28.646 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:28.692 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:34:28.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:28.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:34:28.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:34:28.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:34:28.799 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:28.826 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:28.869 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total time run: 90.4825 2023-09-23T23:34:28.869 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total writes made: 87777 2023-09-23T23:34:28.869 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Write size: 65536 2023-09-23T23:34:28.869 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object size: 65536 2023-09-23T23:34:28.869 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Bandwidth (MB/sec): 60.6312 2023-09-23T23:34:28.869 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Bandwidth: 40.8283 2023-09-23T23:34:28.869 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max bandwidth (MB/sec): 200 2023-09-23T23:34:28.869 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min bandwidth (MB/sec): 0 2023-09-23T23:34:28.869 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average IOPS: 970 2023-09-23T23:34:28.870 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev IOPS: 653.261 2023-09-23T23:34:28.870 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max IOPS: 3200 2023-09-23T23:34:28.870 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min IOPS: 0 2023-09-23T23:34:28.870 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average Latency(s): 0.0164362 2023-09-23T23:34:28.870 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Latency(s): 0.0656713 2023-09-23T23:34:28.870 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max latency(s): 2.85597 2023-09-23T23:34:28.870 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min latency(s): 0.00107054 2023-09-23T23:34:28.874 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:28.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:28.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:28.924 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T23:34:28.980 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:29.027 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:29.096 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:29.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:29.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:29.208 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:29.246 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:29.312 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:29.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:29.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:29.424 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:34:29.441 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:34:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:29 smithi138 ceph-mon[193606]: pgmap v1567: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2.6 KiB/s rd, 16 MiB/s wr, 264 op/s; 25450/261000 objects degraded (9.751%); 11041/261000 objects misplaced (4.230%); 10 MiB/s, 165 objects/s recovering 2023-09-23T23:34:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:29 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 25450/261000 objects degraded (9.751%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:29.506 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:29 smithi110 ceph-mon[212939]: pgmap v1567: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2.6 KiB/s rd, 16 MiB/s wr, 264 op/s; 25450/261000 objects degraded (9.751%); 11041/261000 objects misplaced (4.230%); 10 MiB/s, 165 objects/s recovering 2023-09-23T23:34:29.507 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:29 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 25450/261000 objects degraded (9.751%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:29.511 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:29 smithi110 ceph-mon[221102]: pgmap v1567: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2.6 KiB/s rd, 16 MiB/s wr, 264 op/s; 25450/261000 objects degraded (9.751%); 11041/261000 objects misplaced (4.230%); 10 MiB/s, 165 objects/s recovering 2023-09-23T23:34:29.511 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:29 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 25450/261000 objects degraded (9.751%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:29.512 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:29.618 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:34:29.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:29.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:34:29.725 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:34:29.761 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:29.762 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:30.303 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:34:30.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:30.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:34:30.410 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:30.452 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:30.631 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:30.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:30.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:30.743 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:30.772 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:30.842 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:30.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:30.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:30.960 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:31.001 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:31.341 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:34:31.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:31.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:34:31.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:31 smithi110 ceph-mon[221102]: pgmap v1568: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2.1 KiB/s rd, 15 MiB/s wr, 252 op/s; 25444/262671 objects degraded (9.687%); 10265/262671 objects misplaced (3.908%); 12 MiB/s, 192 objects/s recovering 2023-09-23T23:34:31.347 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:31 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:34:31.348 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:31 smithi110 ceph-mon[212939]: pgmap v1568: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2.1 KiB/s rd, 15 MiB/s wr, 252 op/s; 25444/262671 objects degraded (9.687%); 10265/262671 objects misplaced (3.908%); 12 MiB/s, 192 objects/s recovering 2023-09-23T23:34:31.348 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:31 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:34:31.447 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:31.449 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:34:31.498 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:31.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:31 smithi138 ceph-mon[193606]: pgmap v1568: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2.1 KiB/s rd, 15 MiB/s wr, 252 op/s; 25444/262671 objects degraded (9.687%); 10265/262671 objects misplaced (3.908%); 12 MiB/s, 192 objects/s recovering 2023-09-23T23:34:31.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:31 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:34:31.705 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:34:31.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:31.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:34:31.810 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:31.845 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:31.884 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:32.064 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:34:32.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:32.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:34:32.169 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:34:32.212 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:32.329 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:34:32.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:32.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:34:32.435 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:32.475 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:32.652 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:32.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:32.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:32.760 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:34:32.807 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:33.511 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:33 smithi110 ceph-mon[221102]: pgmap v1569: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 95 B/s rd, 11 MiB/s wr, 207 op/s; 25444/262413 objects degraded (9.696%); 9975/262413 objects misplaced (3.801%); 12 MiB/s, 190 objects/s recovering 2023-09-23T23:34:33.512 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:33 smithi110 ceph-mon[212939]: pgmap v1569: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 95 B/s rd, 11 MiB/s wr, 207 op/s; 25444/262413 objects degraded (9.696%); 9975/262413 objects misplaced (3.801%); 12 MiB/s, 190 objects/s recovering 2023-09-23T23:34:33.516 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:34:33.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:33.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:34:33.629 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:34:33.662 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:33 smithi138 ceph-mon[193606]: pgmap v1569: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 95 B/s rd, 11 MiB/s wr, 207 op/s; 25444/262413 objects degraded (9.696%); 9975/262413 objects misplaced (3.801%); 12 MiB/s, 190 objects/s recovering 2023-09-23T23:34:33.671 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:33.750 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:34:33.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:33.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:34:33.855 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:34:33.884 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:33.964 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:34:33.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:33.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:34:34.068 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:34:34.128 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:34.239 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:34 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 25444/262413 objects degraded (9.696%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:34.239 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:34 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2782604558' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:34:34.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:34 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 25444/262413 objects degraded (9.696%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:34.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:34 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2782604558' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:34:34.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:34 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 25444/262413 objects degraded (9.696%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:34.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:34 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2782604558' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:34:34.729 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T23:34:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:35 smithi138 ceph-mon[193606]: pgmap v1570: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 85 B/s rd, 9.8 MiB/s wr, 184 op/s; 25444/262413 objects degraded (9.696%); 9975/262413 objects misplaced (3.801%); 11 MiB/s, 169 objects/s recovering 2023-09-23T23:34:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:35 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2782604558' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:34:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:35 smithi138 ceph-mon[193606]: osdmap e570: 8 total, 8 up, 7 in 2023-09-23T23:34:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:35 smithi138 ceph-mon[193606]: failed to encode map e570 with expected crc 2023-09-23T23:34:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:35 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:35 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:35.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:35 smithi110 ceph-mon[212939]: pgmap v1570: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 85 B/s rd, 9.8 MiB/s wr, 184 op/s; 25444/262413 objects degraded (9.696%); 9975/262413 objects misplaced (3.801%); 11 MiB/s, 169 objects/s recovering 2023-09-23T23:34:35.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:35 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2782604558' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:34:35.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:35 smithi110 ceph-mon[212939]: osdmap e570: 8 total, 8 up, 7 in 2023-09-23T23:34:35.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:35 smithi110 ceph-mon[212939]: failed to encode map e570 with expected crc 2023-09-23T23:34:35.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:35 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:35.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:35 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:35.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:35 smithi110 ceph-mon[221102]: pgmap v1570: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 85 B/s rd, 9.8 MiB/s wr, 184 op/s; 25444/262413 objects degraded (9.696%); 9975/262413 objects misplaced (3.801%); 11 MiB/s, 169 objects/s recovering 2023-09-23T23:34:35.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:35 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2782604558' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:34:35.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:35 smithi110 ceph-mon[221102]: osdmap e570: 8 total, 8 up, 7 in 2023-09-23T23:34:35.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:35 smithi110 ceph-mon[221102]: failed to encode map e570 with expected crc 2023-09-23T23:34:35.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:35 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:35.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:35 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:36 smithi138 ceph-mon[193606]: failed to encode map e570 with expected crc 2023-09-23T23:34:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:36 smithi138 ceph-mon[193606]: failed to encode map e571 with expected crc 2023-09-23T23:34:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:36 smithi138 ceph-mon[193606]: failed to encode map e571 with expected crc 2023-09-23T23:34:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:36 smithi138 ceph-mon[193606]: osdmap e571: 8 total, 8 up, 7 in 2023-09-23T23:34:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:36 smithi138 ceph-mon[193606]: failed to encode map e571 with expected crc 2023-09-23T23:34:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:36 smithi138 ceph-mon[193606]: failed to encode map e571 with expected crc 2023-09-23T23:34:36.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:36 smithi110 ceph-mon[212939]: failed to encode map e570 with expected crc 2023-09-23T23:34:36.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:36 smithi110 ceph-mon[212939]: failed to encode map e571 with expected crc 2023-09-23T23:34:36.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:36 smithi110 ceph-mon[212939]: failed to encode map e571 with expected crc 2023-09-23T23:34:36.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:36 smithi110 ceph-mon[212939]: osdmap e571: 8 total, 8 up, 7 in 2023-09-23T23:34:36.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:36 smithi110 ceph-mon[212939]: failed to encode map e571 with expected crc 2023-09-23T23:34:36.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:36 smithi110 ceph-mon[212939]: failed to encode map e571 with expected crc 2023-09-23T23:34:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:36 smithi110 ceph-mon[221102]: failed to encode map e570 with expected crc 2023-09-23T23:34:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:36 smithi110 ceph-mon[221102]: failed to encode map e571 with expected crc 2023-09-23T23:34:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:36 smithi110 ceph-mon[221102]: failed to encode map e571 with expected crc 2023-09-23T23:34:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:36 smithi110 ceph-mon[221102]: osdmap e571: 8 total, 8 up, 7 in 2023-09-23T23:34:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:36 smithi110 ceph-mon[221102]: failed to encode map e571 with expected crc 2023-09-23T23:34:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:36 smithi110 ceph-mon[221102]: failed to encode map e571 with expected crc 2023-09-23T23:34:36.994 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:34:37.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:37.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:34:37.060 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:34:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:34:36] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.43.0" 2023-09-23T23:34:37.061 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T23:34:37.064 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:37.102 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:34:37.129 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:37.174 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:34:37.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:37.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:34:37.281 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:37.308 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:37.354 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:37.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:37.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:37.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:37 smithi138 ceph-mon[193606]: failed to encode map e571 with expected crc 2023-09-23T23:34:37.486 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:37 smithi138 ceph-mon[193606]: failed to encode map e571 with expected crc 2023-09-23T23:34:37.486 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:37 smithi138 ceph-mon[193606]: failed to encode map e571 with expected crc 2023-09-23T23:34:37.486 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:37 smithi138 ceph-mon[193606]: pgmap v1573: 119 pgs: 4 peering, 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 6.3 MiB/s wr, 298 op/s; 25324/258639 objects degraded (9.791%); 9299/258639 objects misplaced (3.595%); 15 MiB/s, 233 objects/s recovering 2023-09-23T23:34:37.486 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:37 smithi138 ceph-mon[193606]: failed to encode map e572 with expected crc 2023-09-23T23:34:37.486 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:37 smithi138 ceph-mon[193606]: osdmap e572: 8 total, 8 up, 7 in 2023-09-23T23:34:37.487 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:37 smithi138 ceph-mon[193606]: failed to encode map e572 with expected crc 2023-09-23T23:34:37.487 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:37 smithi138 ceph-mon[193606]: failed to encode map e572 with expected crc 2023-09-23T23:34:37.487 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:37.488 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:37 smithi110 ceph-mon[212939]: failed to encode map e571 with expected crc 2023-09-23T23:34:37.488 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:37 smithi110 ceph-mon[212939]: failed to encode map e571 with expected crc 2023-09-23T23:34:37.488 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:37 smithi110 ceph-mon[212939]: failed to encode map e571 with expected crc 2023-09-23T23:34:37.488 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:37 smithi110 ceph-mon[212939]: pgmap v1573: 119 pgs: 4 peering, 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 6.3 MiB/s wr, 298 op/s; 25324/258639 objects degraded (9.791%); 9299/258639 objects misplaced (3.595%); 15 MiB/s, 233 objects/s recovering 2023-09-23T23:34:37.488 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:37 smithi110 ceph-mon[212939]: failed to encode map e572 with expected crc 2023-09-23T23:34:37.488 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:37 smithi110 ceph-mon[212939]: osdmap e572: 8 total, 8 up, 7 in 2023-09-23T23:34:37.488 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:37 smithi110 ceph-mon[212939]: failed to encode map e572 with expected crc 2023-09-23T23:34:37.488 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:37 smithi110 ceph-mon[212939]: failed to encode map e572 with expected crc 2023-09-23T23:34:37.489 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:37 smithi110 ceph-mon[221102]: failed to encode map e571 with expected crc 2023-09-23T23:34:37.489 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:37 smithi110 ceph-mon[221102]: failed to encode map e571 with expected crc 2023-09-23T23:34:37.489 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:37 smithi110 ceph-mon[221102]: failed to encode map e571 with expected crc 2023-09-23T23:34:37.489 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:37 smithi110 ceph-mon[221102]: pgmap v1573: 119 pgs: 4 peering, 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 6.3 MiB/s wr, 298 op/s; 25324/258639 objects degraded (9.791%); 9299/258639 objects misplaced (3.595%); 15 MiB/s, 233 objects/s recovering 2023-09-23T23:34:37.489 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:37 smithi110 ceph-mon[221102]: failed to encode map e572 with expected crc 2023-09-23T23:34:37.489 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:37 smithi110 ceph-mon[221102]: osdmap e572: 8 total, 8 up, 7 in 2023-09-23T23:34:37.489 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:37 smithi110 ceph-mon[221102]: failed to encode map e572 with expected crc 2023-09-23T23:34:37.489 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:37 smithi110 ceph-mon[221102]: failed to encode map e572 with expected crc 2023-09-23T23:34:37.518 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:37.571 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:37.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:37.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:37.677 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:37.714 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:37.774 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:37.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:37.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:37.882 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:34:37.918 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:37.956 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:34:37.956 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:34:37.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:34:37.956 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:34:37.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.956 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135599 11.e 11:71e23346:::benchmark_data_smithi110_433746_object29719:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:34:37.956 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505704+0000", 2023-09-23T23:34:37.956 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 506.44418944300003, 2023-09-23T23:34:37.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.837012734, 2023-09-23T23:34:37.956 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:37.957 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:37.957 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:37.957 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:34:37.957 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:37.957 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135599 2023-09-23T23:34:37.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.957 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:37.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:37.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505704+0000", 2023-09-23T23:34:37.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:37.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:37.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505704+0000", 2023-09-23T23:34:37.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:37.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:37.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505702+0000", 2023-09-23T23:34:37.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:34:37.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:37.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505710+0000", 2023-09-23T23:34:37.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.038e-06 2023-09-23T23:34:37.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:37.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505724+0000", 2023-09-23T23:34:37.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3684e-05 2023-09-23T23:34:37.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:37.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505733+0000", 2023-09-23T23:34:37.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1309999999999995e-06 2023-09-23T23:34:37.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:37.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505925+0000", 2023-09-23T23:34:37.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000192282 2023-09-23T23:34:37.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op must wait for map", 2023-09-23T23:34:37.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505933+0000", 2023-09-23T23:34:37.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3089999999999998e-06 2023-09-23T23:34:37.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:37.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513411+0000", 2023-09-23T23:34:37.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074784489999999999 2023-09-23T23:34:37.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:37.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513421+0000", 2023-09-23T23:34:37.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0108e-05 2023-09-23T23:34:37.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:37.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.304735+0000", 2023-09-23T23:34:37.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.791314093 2023-09-23T23:34:37.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:37.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.306968+0000", 2023-09-23T23:34:37.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022327390000000001 2023-09-23T23:34:37.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:34:37.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.307182+0000", 2023-09-23T23:34:37.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000214291 2023-09-23T23:34:37.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:37.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.322749+0000", 2023-09-23T23:34:37.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015566695 2023-09-23T23:34:37.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:37.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.322789+0000", 2023-09-23T23:34:37.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9808999999999998e-05 2023-09-23T23:34:37.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:37.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334988+0000", 2023-09-23T23:34:37.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012198595 2023-09-23T23:34:37.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:37.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342545+0000", 2023-09-23T23:34:37.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075570389999999998 2023-09-23T23:34:37.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:37.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342599+0000", 2023-09-23T23:34:37.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4347e-05 2023-09-23T23:34:37.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:37.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342717+0000", 2023-09-23T23:34:37.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117578 2023-09-23T23:34:37.965 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:37.965 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:37.965 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:37.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.965 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135605 11.e 11:7071ef4a:::benchmark_data_smithi110_433746_object29725:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:34:37.965 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505749+0000", 2023-09-23T23:34:37.965 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 506.44414440600002, 2023-09-23T23:34:37.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.837123356, 2023-09-23T23:34:37.965 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:37.965 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:37.965 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:37.965 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:34:37.965 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:37.966 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135605 2023-09-23T23:34:37.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.966 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:37.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:37.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505749+0000", 2023-09-23T23:34:37.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:37.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:37.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505749+0000", 2023-09-23T23:34:37.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:37.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:37.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505748+0000", 2023-09-23T23:34:37.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:34:37.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:37.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505751+0000", 2023-09-23T23:34:37.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.056e-06 2023-09-23T23:34:37.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:37.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505760+0000", 2023-09-23T23:34:37.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3750000000000003e-06 2023-09-23T23:34:37.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:37.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505766+0000", 2023-09-23T23:34:37.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8760000000000002e-06 2023-09-23T23:34:37.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:37.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505982+0000", 2023-09-23T23:34:37.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021631100000000001 2023-09-23T23:34:37.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:34:37.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505988+0000", 2023-09-23T23:34:37.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5779999999999998e-06 2023-09-23T23:34:37.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:37.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513443+0000", 2023-09-23T23:34:37.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007455377 2023-09-23T23:34:37.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:37.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513450+0000", 2023-09-23T23:34:37.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1459999999999999e-06 2023-09-23T23:34:37.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:37.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.322816+0000", 2023-09-23T23:34:37.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.809365865 2023-09-23T23:34:37.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:37.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.322945+0000", 2023-09-23T23:34:37.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012866099999999999 2023-09-23T23:34:37.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:34:37.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.323097+0000", 2023-09-23T23:34:37.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015215899999999999 2023-09-23T23:34:37.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:37.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334919+0000", 2023-09-23T23:34:37.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011822687 2023-09-23T23:34:37.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:37.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.334954+0000", 2023-09-23T23:34:37.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4987999999999998e-05 2023-09-23T23:34:37.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:37.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335050+0000", 2023-09-23T23:34:37.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5222000000000004e-05 2023-09-23T23:34:37.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:37.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342725+0000", 2023-09-23T23:34:37.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076749360000000003 2023-09-23T23:34:37.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:37.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342767+0000", 2023-09-23T23:34:37.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2929000000000002e-05 2023-09-23T23:34:37.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:37.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342872+0000", 2023-09-23T23:34:37.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104709 2023-09-23T23:34:37.973 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:37.973 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:37.973 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:37.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.973 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135609 11.e 11:73c3a869:::benchmark_data_smithi110_433746_object29729:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:34:37.974 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505771+0000", 2023-09-23T23:34:37.974 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 506.44412190399999, 2023-09-23T23:34:37.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8373706329999999, 2023-09-23T23:34:37.974 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:37.974 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:37.974 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:37.974 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:34:37.974 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:37.974 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135609 2023-09-23T23:34:37.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.974 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:37.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:37.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505771+0000", 2023-09-23T23:34:37.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:37.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:37.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505771+0000", 2023-09-23T23:34:37.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:37.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:37.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505771+0000", 2023-09-23T23:34:37.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:34:37.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:37.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505773+0000", 2023-09-23T23:34:37.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4109999999999999e-06 2023-09-23T23:34:37.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:37.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505779+0000", 2023-09-23T23:34:37.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9529999999999996e-06 2023-09-23T23:34:37.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:37.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505784+0000", 2023-09-23T23:34:37.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1279999999999999e-06 2023-09-23T23:34:37.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:37.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506007+0000", 2023-09-23T23:34:37.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022278700000000001 2023-09-23T23:34:37.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:34:37.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506012+0000", 2023-09-23T23:34:37.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0390000000000004e-06 2023-09-23T23:34:37.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:37.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513470+0000", 2023-09-23T23:34:37.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074576879999999996 2023-09-23T23:34:37.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:37.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513476+0000", 2023-09-23T23:34:37.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9059999999999999e-06 2023-09-23T23:34:37.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.325254+0000", 2023-09-23T23:34:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8117784690000001 2023-09-23T23:34:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.325381+0000", 2023-09-23T23:34:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012669499999999999 2023-09-23T23:34:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:34:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.325510+0000", 2023-09-23T23:34:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012926999999999999 2023-09-23T23:34:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335017+0000", 2023-09-23T23:34:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0095070620000000002 2023-09-23T23:34:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335119+0000", 2023-09-23T23:34:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102045 2023-09-23T23:34:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342880+0000", 2023-09-23T23:34:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077602000000000001 2023-09-23T23:34:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342995+0000", 2023-09-23T23:34:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000115253 2023-09-23T23:34:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343036+0000", 2023-09-23T23:34:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0632e-05 2023-09-23T23:34:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343142+0000", 2023-09-23T23:34:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010644 2023-09-23T23:34:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135623 11.e 11:7e8ebd90:::benchmark_data_smithi110_433746_object29743:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:34:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505789+0000", 2023-09-23T23:34:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 506.44410460400002, 2023-09-23T23:34:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.843964774, 2023-09-23T23:34:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:34:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135623 2023-09-23T23:34:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505789+0000", 2023-09-23T23:34:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505789+0000", 2023-09-23T23:34:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505788+0000", 2023-09-23T23:34:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:34:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505791+0000", 2023-09-23T23:34:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.339e-06 2023-09-23T23:34:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505797+0000", 2023-09-23T23:34:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.207e-06 2023-09-23T23:34:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505802+0000", 2023-09-23T23:34:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9969999999999998e-06 2023-09-23T23:34:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506033+0000", 2023-09-23T23:34:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000230824 2023-09-23T23:34:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:34:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506037+0000", 2023-09-23T23:34:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8690000000000003e-06 2023-09-23T23:34:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513487+0000", 2023-09-23T23:34:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074491310000000003 2023-09-23T23:34:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513490+0000", 2023-09-23T23:34:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4649999999999999e-06 2023-09-23T23:34:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.327901+0000", 2023-09-23T23:34:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.814410818 2023-09-23T23:34:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.328011+0000", 2023-09-23T23:34:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109596 2023-09-23T23:34:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:34:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.328129+0000", 2023-09-23T23:34:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118751 2023-09-23T23:34:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335081+0000", 2023-09-23T23:34:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0069518549999999998 2023-09-23T23:34:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342886+0000", 2023-09-23T23:34:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078050599999999999 2023-09-23T23:34:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343224+0000", 2023-09-23T23:34:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033781399999999997 2023-09-23T23:34:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349601+0000", 2023-09-23T23:34:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063765899999999997 2023-09-23T23:34:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349647+0000", 2023-09-23T23:34:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6164e-05 2023-09-23T23:34:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349753+0000", 2023-09-23T23:34:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000106669 2023-09-23T23:34:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135624 11.e 11:79f5e3a3:::benchmark_data_smithi110_433746_object29744:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:34:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505805+0000", 2023-09-23T23:34:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 506.44408777899997, 2023-09-23T23:34:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.844120325, 2023-09-23T23:34:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:34:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135624 2023-09-23T23:34:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505805+0000", 2023-09-23T23:34:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505805+0000", 2023-09-23T23:34:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505805+0000", 2023-09-23T23:34:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:34:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505807+0000", 2023-09-23T23:34:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2120000000000002e-06 2023-09-23T23:34:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505813+0000", 2023-09-23T23:34:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.57e-06 2023-09-23T23:34:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505817+0000", 2023-09-23T23:34:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2980000000000002e-06 2023-09-23T23:34:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506057+0000", 2023-09-23T23:34:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000239836 2023-09-23T23:34:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:34:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506062+0000", 2023-09-23T23:34:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7870000000000001e-06 2023-09-23T23:34:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513504+0000", 2023-09-23T23:34:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074420959999999996 2023-09-23T23:34:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:37.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513508+0000", 2023-09-23T23:34:37.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9339999999999999e-06 2023-09-23T23:34:37.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:37.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.330221+0000", 2023-09-23T23:34:37.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.816713196 2023-09-23T23:34:37.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:37.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.330346+0000", 2023-09-23T23:34:37.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000125406 2023-09-23T23:34:37.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:34:37.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.330485+0000", 2023-09-23T23:34:37.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013825799999999999 2023-09-23T23:34:37.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:37.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342893+0000", 2023-09-23T23:34:37.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012408130999999999 2023-09-23T23:34:37.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:37.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342951+0000", 2023-09-23T23:34:37.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8354999999999999e-05 2023-09-23T23:34:37.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:37.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343297+0000", 2023-09-23T23:34:37.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034624800000000001 2023-09-23T23:34:37.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:37.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349790+0000", 2023-09-23T23:34:37.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0064925399999999998 2023-09-23T23:34:37.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:37.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349831+0000", 2023-09-23T23:34:37.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1069999999999998e-05 2023-09-23T23:34:37.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:37.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:37.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:37.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.349926+0000", 2023-09-23T23:34:37.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4718999999999999e-05 2023-09-23T23:34:37.999 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:37.999 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:37.999 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.000 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135642 11.e 11:7847da74:::benchmark_data_smithi110_433746_object29762:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:34:38.000 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505821+0000", 2023-09-23T23:34:38.000 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 506.44407229299998, 2023-09-23T23:34:38.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.844417151, 2023-09-23T23:34:38.000 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:38.000 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:38.000 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:38.000 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:34:38.000 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:38.000 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135642 2023-09-23T23:34:38.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.001 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:38.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:38.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505821+0000", 2023-09-23T23:34:38.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:38.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505821+0000", 2023-09-23T23:34:38.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:38.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505821+0000", 2023-09-23T23:34:38.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:34:38.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:38.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505823+0000", 2023-09-23T23:34:38.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4949999999999998e-06 2023-09-23T23:34:38.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:38.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505829+0000", 2023-09-23T23:34:38.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.383e-06 2023-09-23T23:34:38.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:38.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505833+0000", 2023-09-23T23:34:38.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8770000000000001e-06 2023-09-23T23:34:38.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506081+0000", 2023-09-23T23:34:38.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024778400000000003 2023-09-23T23:34:38.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:34:38.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506086+0000", 2023-09-23T23:34:38.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0170000000000004e-06 2023-09-23T23:34:38.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513526+0000", 2023-09-23T23:34:38.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074400029999999997 2023-09-23T23:34:38.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:38.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513533+0000", 2023-09-23T23:34:38.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5189999999999998e-06 2023-09-23T23:34:38.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.332515+0000", 2023-09-23T23:34:38.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.818982726 2023-09-23T23:34:38.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:38.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.332698+0000", 2023-09-23T23:34:38.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000182046 2023-09-23T23:34:38.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:34:38.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.332836+0000", 2023-09-23T23:34:38.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013802000000000001 2023-09-23T23:34:38.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:38.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342905+0000", 2023-09-23T23:34:38.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010069121 2023-09-23T23:34:38.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343190+0000", 2023-09-23T23:34:38.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028497700000000001 2023-09-23T23:34:38.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343385+0000", 2023-09-23T23:34:38.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019561000000000001 2023-09-23T23:34:38.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350092+0000", 2023-09-23T23:34:38.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067069870000000002 2023-09-23T23:34:38.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:38.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350141+0000", 2023-09-23T23:34:38.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8365000000000002e-05 2023-09-23T23:34:38.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:38.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350238+0000", 2023-09-23T23:34:38.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7510999999999995e-05 2023-09-23T23:34:38.008 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.008 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:38.008 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.009 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135646 11.e 11:7bf5000e:::benchmark_data_smithi110_433746_object29766:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:34:38.009 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505837+0000", 2023-09-23T23:34:38.009 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 506.44405582899998, 2023-09-23T23:34:38.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.844569372, 2023-09-23T23:34:38.009 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:38.009 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:38.009 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:38.009 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:34:38.009 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:38.009 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135646 2023-09-23T23:34:38.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.009 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:38.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:38.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505837+0000", 2023-09-23T23:34:38.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:38.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505837+0000", 2023-09-23T23:34:38.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:38.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505837+0000", 2023-09-23T23:34:38.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:34:38.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:38.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505839+0000", 2023-09-23T23:34:38.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3530000000000002e-06 2023-09-23T23:34:38.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:38.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505846+0000", 2023-09-23T23:34:38.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1800000000000001e-06 2023-09-23T23:34:38.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:38.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505850+0000", 2023-09-23T23:34:38.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5700000000000003e-06 2023-09-23T23:34:38.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506104+0000", 2023-09-23T23:34:38.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025424300000000003 2023-09-23T23:34:38.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:34:38.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506109+0000", 2023-09-23T23:34:38.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8749999999999999e-06 2023-09-23T23:34:38.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513566+0000", 2023-09-23T23:34:38.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074570510000000001 2023-09-23T23:34:38.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:38.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513573+0000", 2023-09-23T23:34:38.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9879999999999998e-06 2023-09-23T23:34:38.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335148+0000", 2023-09-23T23:34:38.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.82157426 2023-09-23T23:34:38.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:38.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335293+0000", 2023-09-23T23:34:38.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014563600000000001 2023-09-23T23:34:38.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:34:38.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.335527+0000", 2023-09-23T23:34:38.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023380100000000001 2023-09-23T23:34:38.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:38.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342911+0000", 2023-09-23T23:34:38.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007384191 2023-09-23T23:34:38.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343259+0000", 2023-09-23T23:34:38.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034814699999999999 2023-09-23T23:34:38.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343422+0000", 2023-09-23T23:34:38.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016240100000000001 2023-09-23T23:34:38.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350272+0000", 2023-09-23T23:34:38.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068501539999999998 2023-09-23T23:34:38.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:38.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350313+0000", 2023-09-23T23:34:38.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1044000000000002e-05 2023-09-23T23:34:38.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:38.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350407+0000", 2023-09-23T23:34:38.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3838000000000005e-05 2023-09-23T23:34:38.017 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.017 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:38.017 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.017 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135676 11.e 11:7b57d3a0:::benchmark_data_smithi110_433746_object29796:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:34:38.017 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505854+0000", 2023-09-23T23:34:38.017 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 506.44403921899999, 2023-09-23T23:34:38.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.844725422, 2023-09-23T23:34:38.018 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:38.018 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:38.018 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:38.018 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:34:38.018 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:38.018 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135676 2023-09-23T23:34:38.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.018 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:38.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:38.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505854+0000", 2023-09-23T23:34:38.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:38.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505854+0000", 2023-09-23T23:34:38.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:38.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505854+0000", 2023-09-23T23:34:38.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:34:38.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:38.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505856+0000", 2023-09-23T23:34:38.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3700000000000002e-06 2023-09-23T23:34:38.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:38.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505862+0000", 2023-09-23T23:34:38.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2809999999999997e-06 2023-09-23T23:34:38.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:38.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505867+0000", 2023-09-23T23:34:38.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.933e-06 2023-09-23T23:34:38.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506128+0000", 2023-09-23T23:34:38.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000260938 2023-09-23T23:34:38.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:34:38.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506132+0000", 2023-09-23T23:34:38.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3409999999999999e-06 2023-09-23T23:34:38.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513595+0000", 2023-09-23T23:34:38.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074631480000000002 2023-09-23T23:34:38.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:38.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513602+0000", 2023-09-23T23:34:38.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9140000000000002e-06 2023-09-23T23:34:38.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337712+0000", 2023-09-23T23:34:38.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8241103679999999 2023-09-23T23:34:38.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:38.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.337841+0000", 2023-09-23T23:34:38.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012925499999999999 2023-09-23T23:34:38.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:34:38.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.338028+0000", 2023-09-23T23:34:38.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018702899999999999 2023-09-23T23:34:38.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:38.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.342960+0000", 2023-09-23T23:34:38.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0049315160000000004 2023-09-23T23:34:38.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343333+0000", 2023-09-23T23:34:38.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00037339399999999998 2023-09-23T23:34:38.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.343461+0000", 2023-09-23T23:34:38.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012787399999999999 2023-09-23T23:34:38.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350439+0000", 2023-09-23T23:34:38.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0069780379999999998 2023-09-23T23:34:38.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:38.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350480+0000", 2023-09-23T23:34:38.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.057e-05 2023-09-23T23:34:38.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:38.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.350579+0000", 2023-09-23T23:34:38.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9774999999999999e-05 2023-09-23T23:34:38.026 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.026 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:38.026 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.026 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:135677 11.e 11:7ce6155e:::benchmark_data_smithi110_433746_object29797:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-09-23T23:34:38.026 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:26:11.505871+0000", 2023-09-23T23:34:38.026 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 506.44402193399998, 2023-09-23T23:34:38.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8476306259999999, 2023-09-23T23:34:38.026 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:38.027 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:38.027 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:38.027 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:34:38.027 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:38.027 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 135677 2023-09-23T23:34:38.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.027 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:38.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:38.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505871+0000", 2023-09-23T23:34:38.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:38.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505871+0000", 2023-09-23T23:34:38.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:38.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505871+0000", 2023-09-23T23:34:38.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:34:38.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:38.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505873+0000", 2023-09-23T23:34:38.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.43e-06 2023-09-23T23:34:38.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:38.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505879+0000", 2023-09-23T23:34:38.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8250000000000001e-06 2023-09-23T23:34:38.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:38.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.505884+0000", 2023-09-23T23:34:38.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.955e-06 2023-09-23T23:34:38.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506151+0000", 2023-09-23T23:34:38.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000266591 2023-09-23T23:34:38.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:34:38.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.506156+0000", 2023-09-23T23:34:38.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8400000000000002e-06 2023-09-23T23:34:38.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513622+0000", 2023-09-23T23:34:38.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007466744 2023-09-23T23:34:38.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:38.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:11.513629+0000", 2023-09-23T23:34:38.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4400000000000002e-06 2023-09-23T23:34:38.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.340120+0000", 2023-09-23T23:34:38.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.826490739 2023-09-23T23:34:38.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:38.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.340250+0000", 2023-09-23T23:34:38.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000130448 2023-09-23T23:34:38.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T23:34:38.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.340609+0000", 2023-09-23T23:34:38.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035861699999999999 2023-09-23T23:34:38.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.348233+0000", 2023-09-23T23:34:38.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007624123 2023-09-23T23:34:38.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.348294+0000", 2023-09-23T23:34:38.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1487000000000003e-05 2023-09-23T23:34:38.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:38.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.348306+0000", 2023-09-23T23:34:38.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1941e-05 2023-09-23T23:34:38.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.353328+0000", 2023-09-23T23:34:38.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0050216280000000002 2023-09-23T23:34:38.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:38.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.353383+0000", 2023-09-23T23:34:38.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5682e-05 2023-09-23T23:34:38.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:38.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:26:13.353502+0000", 2023-09-23T23:34:38.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118492 2023-09-23T23:34:38.035 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.035 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:38.035 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.035 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178600 11.6 11:659f4d42:::benchmark_data_smithi110_433746_object72720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2023-09-23T23:34:38.035 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.523923+0000", 2023-09-23T23:34:38.035 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 438.42596999900002, 2023-09-23T23:34:38.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2361555310000001, 2023-09-23T23:34:38.036 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:38.036 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:34:38.036 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:38.036 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:34:38.036 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:38.036 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178600 2023-09-23T23:34:38.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.036 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:38.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:38.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523923+0000", 2023-09-23T23:34:38.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:38.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523923+0000", 2023-09-23T23:34:38.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:38.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523922+0000", 2023-09-23T23:34:38.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:34:38.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:38.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523929+0000", 2023-09-23T23:34:38.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8800000000000002e-06 2023-09-23T23:34:38.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:38.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523940+0000", 2023-09-23T23:34:38.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.149e-05 2023-09-23T23:34:38.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:38.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523948+0000", 2023-09-23T23:34:38.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7970000000000001e-06 2023-09-23T23:34:38.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.531901+0000", 2023-09-23T23:34:38.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079527399999999998 2023-09-23T23:34:38.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:38.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.532034+0000", 2023-09-23T23:34:38.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013345499999999999 2023-09-23T23:34:38.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:34:38.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.532514+0000", 2023-09-23T23:34:38.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047994200000000002 2023-09-23T23:34:38.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.537104+0000", 2023-09-23T23:34:38.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045898019999999996 2023-09-23T23:34:38.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:38.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.538159+0000", 2023-09-23T23:34:38.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001055437 2023-09-23T23:34:38.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.547877+0000", 2023-09-23T23:34:38.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0097177860000000008 2023-09-23T23:34:38.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:38.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.547890+0000", 2023-09-23T23:34:38.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2656e-05 2023-09-23T23:34:38.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542710+0000", 2023-09-23T23:34:38.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99482022199999998 2023-09-23T23:34:38.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:38.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542720+0000", 2023-09-23T23:34:38.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9029999999999999e-06 2023-09-23T23:34:38.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760000+0000", 2023-09-23T23:34:38.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.217280103 2023-09-23T23:34:38.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:38.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760079+0000", 2023-09-23T23:34:38.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8777999999999997e-05 2023-09-23T23:34:38.043 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.043 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:38.043 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.043 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178608 11.6 11:6177c8c0:::benchmark_data_smithi110_433746_object72728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2023-09-23T23:34:38.043 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.545706+0000", 2023-09-23T23:34:38.044 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 438.40418757100002, 2023-09-23T23:34:38.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2481279590000001, 2023-09-23T23:34:38.044 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:38.044 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:38.044 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:38.044 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:34:38.044 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:38.044 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178608 2023-09-23T23:34:38.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.044 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:38.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:38.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545706+0000", 2023-09-23T23:34:38.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:38.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545706+0000", 2023-09-23T23:34:38.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:38.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545705+0000", 2023-09-23T23:34:38.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:34:38.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:38.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545709+0000", 2023-09-23T23:34:38.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6140000000000002e-06 2023-09-23T23:34:38.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:38.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545721+0000", 2023-09-23T23:34:38.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2345e-05 2023-09-23T23:34:38.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:38.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545730+0000", 2023-09-23T23:34:38.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9579999999999996e-06 2023-09-23T23:34:38.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.570191+0000", 2023-09-23T23:34:38.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024460784999999999 2023-09-23T23:34:38.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:38.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.570202+0000", 2023-09-23T23:34:38.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1192999999999999e-05 2023-09-23T23:34:38.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542744+0000", 2023-09-23T23:34:38.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.972541879 2023-09-23T23:34:38.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:38.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542752+0000", 2023-09-23T23:34:38.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1599999999999998e-06 2023-09-23T23:34:38.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760100+0000", 2023-09-23T23:34:38.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2173479519999999 2023-09-23T23:34:38.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:38.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760221+0000", 2023-09-23T23:34:38.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000120155 2023-09-23T23:34:38.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T23:34:38.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760707+0000", 2023-09-23T23:34:38.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00048629800000000003 2023-09-23T23:34:38.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.774649+0000", 2023-09-23T23:34:38.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013942049 2023-09-23T23:34:38.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783406+0000", 2023-09-23T23:34:38.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0087574420000000007 2023-09-23T23:34:38.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:38.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783684+0000", 2023-09-23T23:34:38.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027739099999999999 2023-09-23T23:34:38.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793677+0000", 2023-09-23T23:34:38.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0099932059999999993 2023-09-23T23:34:38.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:38.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793723+0000", 2023-09-23T23:34:38.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5883000000000001e-05 2023-09-23T23:34:38.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:38.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793834+0000", 2023-09-23T23:34:38.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110819 2023-09-23T23:34:38.052 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.052 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:38.052 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.052 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178617 11.16 11:6d8a65a1:::benchmark_data_smithi110_433746_object72737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:34:38.052 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.584811+0000", 2023-09-23T23:34:38.052 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 438.36508217099998, 2023-09-23T23:34:38.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1985552099999999, 2023-09-23T23:34:38.053 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:38.053 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:38.053 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:38.053 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:34:38.053 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:38.053 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178617 2023-09-23T23:34:38.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.053 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:38.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:38.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584811+0000", 2023-09-23T23:34:38.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:38.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584811+0000", 2023-09-23T23:34:38.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:38.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584809+0000", 2023-09-23T23:34:38.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:34:38.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:38.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584817+0000", 2023-09-23T23:34:38.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3339999999999996e-06 2023-09-23T23:34:38.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:38.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584831+0000", 2023-09-23T23:34:38.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4355000000000001e-05 2023-09-23T23:34:38.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:38.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584840+0000", 2023-09-23T23:34:38.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3410000000000003e-06 2023-09-23T23:34:38.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591289+0000", 2023-09-23T23:34:38.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0064494030000000003 2023-09-23T23:34:38.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:38.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591300+0000", 2023-09-23T23:34:38.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0597e-05 2023-09-23T23:34:38.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542631+0000", 2023-09-23T23:34:38.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.951331449 2023-09-23T23:34:38.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:38.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542644+0000", 2023-09-23T23:34:38.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.288e-05 2023-09-23T23:34:38.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.756708+0000", 2023-09-23T23:34:38.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.21406408 2023-09-23T23:34:38.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:38.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.756818+0000", 2023-09-23T23:34:38.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110073 2023-09-23T23:34:38.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T23:34:38.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.757269+0000", 2023-09-23T23:34:38.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045095900000000002 2023-09-23T23:34:38.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764656+0000", 2023-09-23T23:34:38.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007387195 2023-09-23T23:34:38.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:38.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764667+0000", 2023-09-23T23:34:38.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0365e-05 2023-09-23T23:34:38.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.774688+0000", 2023-09-23T23:34:38.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010021577 2023-09-23T23:34:38.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783206+0000", 2023-09-23T23:34:38.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0085183289999999998 2023-09-23T23:34:38.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:38.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783262+0000", 2023-09-23T23:34:38.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5275999999999997e-05 2023-09-23T23:34:38.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:38.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783366+0000", 2023-09-23T23:34:38.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010458099999999999 2023-09-23T23:34:38.061 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.061 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:38.061 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.061 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178622 11.6 11:6344b54e:::benchmark_data_smithi110_433746_object72742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:34:38.061 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.591902+0000", 2023-09-23T23:34:38.061 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 438.35799119199999, 2023-09-23T23:34:38.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2021504429999998, 2023-09-23T23:34:38.061 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:38.061 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:38.061 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:38.061 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:34:38.062 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:38.062 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178622 2023-09-23T23:34:38.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.062 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:38.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:38.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591902+0000", 2023-09-23T23:34:38.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:38.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591902+0000", 2023-09-23T23:34:38.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:38.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591901+0000", 2023-09-23T23:34:38.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:34:38.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:38.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591906+0000", 2023-09-23T23:34:38.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8289999999999996e-06 2023-09-23T23:34:38.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:38.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591921+0000", 2023-09-23T23:34:38.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4149e-05 2023-09-23T23:34:38.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:38.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591928+0000", 2023-09-23T23:34:38.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0999999999999998e-06 2023-09-23T23:34:38.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.592603+0000", 2023-09-23T23:34:38.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00067509500000000001 2023-09-23T23:34:38.065 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.065 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.065 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:38.065 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.592613+0000", 2023-09-23T23:34:38.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0671e-05 2023-09-23T23:34:38.065 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.065 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.065 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.065 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542776+0000", 2023-09-23T23:34:38.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95016243199999995 2023-09-23T23:34:38.065 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.065 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.066 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:38.066 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542784+0000", 2023-09-23T23:34:38.066 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5580000000000001e-06 2023-09-23T23:34:38.066 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.066 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.066 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.066 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764299+0000", 2023-09-23T23:34:38.066 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2215145999999999 2023-09-23T23:34:38.066 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.066 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.066 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:38.066 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764438+0000", 2023-09-23T23:34:38.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000138536 2023-09-23T23:34:38.067 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.067 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.067 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T23:34:38.067 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.765798+0000", 2023-09-23T23:34:38.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001360874 2023-09-23T23:34:38.067 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.067 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.067 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.067 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.774753+0000", 2023-09-23T23:34:38.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0089545679999999996 2023-09-23T23:34:38.067 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.068 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.068 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:38.068 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783698+0000", 2023-09-23T23:34:38.068 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0089451630000000008 2023-09-23T23:34:38.068 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.068 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.068 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.068 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.785057+0000", 2023-09-23T23:34:38.068 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001358784 2023-09-23T23:34:38.068 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.068 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.068 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793921+0000", 2023-09-23T23:34:38.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0088636380000000001 2023-09-23T23:34:38.069 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.069 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.069 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:38.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793959+0000", 2023-09-23T23:34:38.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8328999999999998e-05 2023-09-23T23:34:38.069 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.069 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.069 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:38.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.794052+0000", 2023-09-23T23:34:38.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3539e-05 2023-09-23T23:34:38.069 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.070 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:38.070 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.070 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.070 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.070 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178633 11.16 11:6db526fe:::benchmark_data_smithi110_433746_object72753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:34:38.070 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.633733+0000", 2023-09-23T23:34:38.070 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 438.316159857, 2023-09-23T23:34:38.070 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.160140325, 2023-09-23T23:34:38.070 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:38.070 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:38.070 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:38.070 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:34:38.071 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:38.071 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178633 2023-09-23T23:34:38.071 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.071 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:38.071 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.071 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:38.071 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633733+0000", 2023-09-23T23:34:38.071 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.071 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.071 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.071 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:38.071 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633733+0000", 2023-09-23T23:34:38.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.072 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.072 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.072 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:38.072 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633732+0000", 2023-09-23T23:34:38.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:34:38.072 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.072 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.072 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:38.072 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633739+0000", 2023-09-23T23:34:38.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2899999999999997e-06 2023-09-23T23:34:38.072 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.072 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:38.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633756+0000", 2023-09-23T23:34:38.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6368999999999999e-05 2023-09-23T23:34:38.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:38.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633763+0000", 2023-09-23T23:34:38.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8930000000000005e-06 2023-09-23T23:34:38.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633828+0000", 2023-09-23T23:34:38.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4092999999999998e-05 2023-09-23T23:34:38.074 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.074 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.074 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:38.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633843+0000", 2023-09-23T23:34:38.074 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5512e-05 2023-09-23T23:34:38.074 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.074 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.074 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542670+0000", 2023-09-23T23:34:38.074 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.90882660500000001 2023-09-23T23:34:38.074 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.074 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.075 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:38.075 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542679+0000", 2023-09-23T23:34:38.075 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0000000000000002e-06 2023-09-23T23:34:38.075 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.075 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.075 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.075 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760612+0000", 2023-09-23T23:34:38.075 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.217933355 2023-09-23T23:34:38.075 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.075 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.075 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:38.075 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760705+0000", 2023-09-23T23:34:38.075 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3078999999999994e-05 2023-09-23T23:34:38.076 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.076 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.076 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T23:34:38.076 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.761116+0000", 2023-09-23T23:34:38.076 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00041080000000000001 2023-09-23T23:34:38.076 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.076 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.076 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:38.076 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764673+0000", 2023-09-23T23:34:38.076 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035569239999999999 2023-09-23T23:34:38.076 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.076 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.077 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.077 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.767908+0000", 2023-09-23T23:34:38.077 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032356749999999999 2023-09-23T23:34:38.077 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.077 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.077 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.077 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783437+0000", 2023-09-23T23:34:38.077 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015528767000000001 2023-09-23T23:34:38.077 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.077 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.077 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.077 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793721+0000", 2023-09-23T23:34:38.077 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010284188 2023-09-23T23:34:38.078 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.078 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.078 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:38.078 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793766+0000", 2023-09-23T23:34:38.078 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4631e-05 2023-09-23T23:34:38.078 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.078 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.078 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:38.078 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793874+0000", 2023-09-23T23:34:38.078 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010767100000000001 2023-09-23T23:34:38.078 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.078 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:38.078 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.079 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.079 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.079 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178629 11.7 11:e3eb7c4c:::benchmark_data_smithi110_433746_object72749:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e425)", 2023-09-23T23:34:38.079 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:20.543856+0000", 2023-09-23T23:34:38.079 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 437.40603746300002, 2023-09-23T23:34:38.079 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2512707000000001, 2023-09-23T23:34:38.079 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:38.079 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:38.079 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:38.079 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:34:38.079 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:38.079 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178629 2023-09-23T23:34:38.080 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.080 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:38.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:38.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543856+0000", 2023-09-23T23:34:38.080 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.080 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:38.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543856+0000", 2023-09-23T23:34:38.080 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.080 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.081 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:38.081 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543855+0000", 2023-09-23T23:34:38.081 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:34:38.081 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.081 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.081 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:38.081 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543857+0000", 2023-09-23T23:34:38.081 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.99e-06 2023-09-23T23:34:38.081 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.081 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.081 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:38.081 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543862+0000", 2023-09-23T23:34:38.082 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.899e-06 2023-09-23T23:34:38.082 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.082 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.082 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:38.082 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543867+0000", 2023-09-23T23:34:38.082 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3710000000000002e-06 2023-09-23T23:34:38.082 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.082 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.082 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.082 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.544825+0000", 2023-09-23T23:34:38.082 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000957864 2023-09-23T23:34:38.082 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.083 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:38.083 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.544832+0000", 2023-09-23T23:34:38.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3649999999999998e-06 2023-09-23T23:34:38.083 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.083 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.083 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.780766+0000", 2023-09-23T23:34:38.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2359338689999999 2023-09-23T23:34:38.083 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.083 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:38.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.780888+0000", 2023-09-23T23:34:38.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000121833 2023-09-23T23:34:38.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.084 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:34:38.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.781338+0000", 2023-09-23T23:34:38.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045035600000000001 2023-09-23T23:34:38.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.084 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.784204+0000", 2023-09-23T23:34:38.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002865884 2023-09-23T23:34:38.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.785473+0000", 2023-09-23T23:34:38.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001268838 2023-09-23T23:34:38.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:38.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.788590+0000", 2023-09-23T23:34:38.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031171839999999998 2023-09-23T23:34:38.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.086 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.086 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.794983+0000", 2023-09-23T23:34:38.086 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063934339999999999 2023-09-23T23:34:38.086 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.086 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.086 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:38.086 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.795024+0000", 2023-09-23T23:34:38.086 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0828000000000003e-05 2023-09-23T23:34:38.086 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.086 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.086 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:38.086 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.795126+0000", 2023-09-23T23:34:38.086 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102277 2023-09-23T23:34:38.087 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.087 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:38.087 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.087 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.087 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.087 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211753 11.19 11:9cef2dd2:::benchmark_data_smithi110_433746_object105873:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T23:34:38.087 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:24.737706+0000", 2023-09-23T23:34:38.087 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 373.21218704500001, 2023-09-23T23:34:38.087 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.967897013, 2023-09-23T23:34:38.087 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:38.087 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:38.087 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:38.087 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:34:38.088 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:34:38.088 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211753 2023-09-23T23:34:38.088 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.088 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:38.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.088 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:38.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737706+0000", 2023-09-23T23:34:38.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.088 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.088 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:38.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737706+0000", 2023-09-23T23:34:38.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.089 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:38.089 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737704+0000", 2023-09-23T23:34:38.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:34:38.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.089 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:38.089 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737713+0000", 2023-09-23T23:34:38.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3939999999999996e-06 2023-09-23T23:34:38.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:38.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737727+0000", 2023-09-23T23:34:38.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3893999999999999e-05 2023-09-23T23:34:38.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:38.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737736+0000", 2023-09-23T23:34:38.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0780000000000002e-06 2023-09-23T23:34:38.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737803+0000", 2023-09-23T23:34:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6959999999999996e-05 2023-09-23T23:34:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737815+0000", 2023-09-23T23:34:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2377e-05 2023-09-23T23:34:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:25.604895+0000", 2023-09-23T23:34:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86708047300000002 2023-09-23T23:34:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:34:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:25.604906+0000", 2023-09-23T23:34:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0903999999999999e-05 2023-09-23T23:34:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.701468+0000", 2023-09-23T23:34:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.096561801 2023-09-23T23:34:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.701626+0000", 2023-09-23T23:34:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015756900000000001 2023-09-23T23:34:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,3,7", 2023-09-23T23:34:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.701760+0000", 2023-09-23T23:34:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013463700000000001 2023-09-23T23:34:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.704681+0000", 2023-09-23T23:34:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029204080000000002 2023-09-23T23:34:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.704693+0000", 2023-09-23T23:34:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2427e-05 2023-09-23T23:34:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.704736+0000", 2023-09-23T23:34:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2911000000000003e-05 2023-09-23T23:34:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.704783+0000", 2023-09-23T23:34:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6931999999999997e-05 2023-09-23T23:34:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.705399+0000", 2023-09-23T23:34:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000615889 2023-09-23T23:34:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.705458+0000", 2023-09-23T23:34:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8731999999999999e-05 2023-09-23T23:34:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.705603+0000", 2023-09-23T23:34:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014554600000000001 2023-09-23T23:34:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:175261 12.10 12:0f9d8e44:::benchmark_data_smithi110_491193_object80386:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2023-09-23T23:34:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.428760+0000", 2023-09-23T23:34:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 150.52113329900001, 2023-09-23T23:34:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.279936254, 2023-09-23T23:34:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:34:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:34:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175261 2023-09-23T23:34:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428760+0000", 2023-09-23T23:34:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428760+0000", 2023-09-23T23:34:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428758+0000", 2023-09-23T23:34:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:34:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428765+0000", 2023-09-23T23:34:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8680000000000001e-06 2023-09-23T23:34:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428778+0000", 2023-09-23T23:34:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2687000000000001e-05 2023-09-23T23:34:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428785+0000", 2023-09-23T23:34:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1679999999999998e-06 2023-09-23T23:34:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428858+0000", 2023-09-23T23:34:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3231000000000004e-05 2023-09-23T23:34:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428968+0000", 2023-09-23T23:34:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010950699999999999 2023-09-23T23:34:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:34:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.429420+0000", 2023-09-23T23:34:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045183000000000002 2023-09-23T23:34:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.432669+0000", 2023-09-23T23:34:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032491830000000001 2023-09-23T23:34:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708584+0000", 2023-09-23T23:34:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.27591494900000002 2023-09-23T23:34:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708613+0000", 2023-09-23T23:34:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9178000000000001e-05 2023-09-23T23:34:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708659+0000", 2023-09-23T23:34:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6415000000000002e-05 2023-09-23T23:34:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708696+0000", 2023-09-23T23:34:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6832000000000002e-05 2023-09-23T23:34:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:175263 12.8 e524/490 12:11714bff:::benchmark_data_smithi110_491193_object80388:head v 524'7666, mlcod=524'7664)", 2023-09-23T23:34:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.433325+0000", 2023-09-23T23:34:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 150.51656789399999, 2023-09-23T23:34:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.282317341, 2023-09-23T23:34:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433325+0000", 2023-09-23T23:34:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433325+0000", 2023-09-23T23:34:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433325+0000", 2023-09-23T23:34:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:34:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433333+0000", 2023-09-23T23:34:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5420000000000005e-06 2023-09-23T23:34:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433344+0000", 2023-09-23T23:34:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1358e-05 2023-09-23T23:34:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433351+0000", 2023-09-23T23:34:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7209999999999997e-06 2023-09-23T23:34:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433384+0000", 2023-09-23T23:34:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3065000000000003e-05 2023-09-23T23:34:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433404+0000", 2023-09-23T23:34:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9762000000000002e-05 2023-09-23T23:34:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.715624+0000", 2023-09-23T23:34:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28222047 2023-09-23T23:34:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.715643+0000", 2023-09-23T23:34:38.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8238999999999999e-05 2023-09-23T23:34:38.108 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.108 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:38.109 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.109 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.109 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:175265 12.10 12:09b7804b:::benchmark_data_smithi110_491193_object80390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2023-09-23T23:34:38.109 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.434661+0000", 2023-09-23T23:34:38.109 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 150.51523189900001, 2023-09-23T23:34:38.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28935229200000001, 2023-09-23T23:34:38.109 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:38.109 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:34:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:34:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:34:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175265 2023-09-23T23:34:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.434661+0000", 2023-09-23T23:34:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.434661+0000", 2023-09-23T23:34:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.434660+0000", 2023-09-23T23:34:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:34:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.434667+0000", 2023-09-23T23:34:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6919999999999996e-06 2023-09-23T23:34:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708315+0000", 2023-09-23T23:34:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.27364855599999999 2023-09-23T23:34:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708328+0000", 2023-09-23T23:34:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2444e-05 2023-09-23T23:34:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.712241+0000", 2023-09-23T23:34:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039134829999999997 2023-09-23T23:34:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.712351+0000", 2023-09-23T23:34:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110096 2023-09-23T23:34:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:34:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.712806+0000", 2023-09-23T23:34:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000454604 2023-09-23T23:34:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.718758+0000", 2023-09-23T23:34:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059521080000000002 2023-09-23T23:34:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:34:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.718791+0000", 2023-09-23T23:34:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3618000000000002e-05 2023-09-23T23:34:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:34:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.723932+0000", 2023-09-23T23:34:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051407349999999996 2023-09-23T23:34:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.723976+0000", 2023-09-23T23:34:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3618000000000001e-05 2023-09-23T23:34:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.724014+0000", 2023-09-23T23:34:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7829e-05 2023-09-23T23:34:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:175259 12.c e524/476 12:3a3a4253:::benchmark_data_smithi110_491193_object80384:head v 524'10995, mlcod=524'10989)", 2023-09-23T23:34:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.436466+0000", 2023-09-23T23:34:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 150.513426856, 2023-09-23T23:34:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28907641299999998, 2023-09-23T23:34:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:34:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:34:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:34:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:34:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436466+0000", 2023-09-23T23:34:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:34:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436466+0000", 2023-09-23T23:34:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:34:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:34:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436466+0000", 2023-09-23T23:34:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:34:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:34:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436473+0000", 2023-09-23T23:34:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4479999999999997e-06 2023-09-23T23:34:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:34:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708281+0000", 2023-09-23T23:34:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.27180817299999999 2023-09-23T23:34:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:34:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708296+0000", 2023-09-23T23:34:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4875000000000001e-05 2023-09-23T23:34:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:34:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708345+0000", 2023-09-23T23:34:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9089999999999999e-05 2023-09-23T23:34:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:34:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708371+0000", 2023-09-23T23:34:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5809e-05 2023-09-23T23:34:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:34:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.725504+0000", 2023-09-23T23:34:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017133116 2023-09-23T23:34:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:34:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:34:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:34:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.725543+0000", 2023-09-23T23:34:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8574e-05 2023-09-23T23:34:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:34:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:34:38.121 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:34:38.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:38.122 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:34:38.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:34:38.223 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:38.260 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:38.272 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:34:38.331 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:38.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:38.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:38.438 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:38.476 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:38.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:38 smithi138 ceph-mon[193606]: failed to encode map e572 with expected crc 2023-09-23T23:34:38.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:38 smithi138 ceph-mon[193606]: 13.3 starting backfill to osd.0 from (0'0,0'0] MAX to 569'5646 2023-09-23T23:34:38.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:38 smithi138 ceph-mon[193606]: failed to encode map e572 with expected crc 2023-09-23T23:34:38.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:38 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4010081888' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:34:38.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:38 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:34:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:38 smithi110 ceph-mon[212939]: failed to encode map e572 with expected crc 2023-09-23T23:34:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:38 smithi110 ceph-mon[212939]: 13.3 starting backfill to osd.0 from (0'0,0'0] MAX to 569'5646 2023-09-23T23:34:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:38 smithi110 ceph-mon[212939]: failed to encode map e572 with expected crc 2023-09-23T23:34:38.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:38 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4010081888' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:34:38.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:38 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:34:38.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:38 smithi110 ceph-mon[221102]: failed to encode map e572 with expected crc 2023-09-23T23:34:38.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:38 smithi110 ceph-mon[221102]: 13.3 starting backfill to osd.0 from (0'0,0'0] MAX to 569'5646 2023-09-23T23:34:38.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:38 smithi110 ceph-mon[221102]: failed to encode map e572 with expected crc 2023-09-23T23:34:38.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:38 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4010081888' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:34:38.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:38 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:34:39.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:39 smithi138 ceph-mon[193606]: pgmap v1575: 119 pgs: 2 remapped+peering, 4 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 263 op/s; 20290/257667 objects degraded (7.875%); 9557/257667 objects misplaced (3.709%); 11 MiB/s, 177 objects/s recovering 2023-09-23T23:34:39.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:39 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:34:39.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:39 smithi138 ceph-mon[193606]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:39 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:34:39.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:39 smithi138 ceph-mon[193606]: osdmap e573: 8 total, 8 up, 7 in 2023-09-23T23:34:39.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:39 smithi138 ceph-mon[193606]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:39 smithi138 ceph-mon[193606]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:39 smithi138 ceph-mon[193606]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:39 smithi138 ceph-mon[193606]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:39 smithi138 ceph-mon[193606]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:39 smithi138 ceph-mon[193606]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:39 smithi138 ceph-mon[193606]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:39 smithi138 ceph-mon[193606]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:39 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 20290/257667 objects degraded (7.875%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:39.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[212939]: pgmap v1575: 119 pgs: 2 remapped+peering, 4 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 263 op/s; 20290/257667 objects degraded (7.875%); 9557/257667 objects misplaced (3.709%); 11 MiB/s, 177 objects/s recovering 2023-09-23T23:34:39.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:34:39.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[212939]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:34:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[212939]: osdmap e573: 8 total, 8 up, 7 in 2023-09-23T23:34:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[212939]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[212939]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[212939]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[212939]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[212939]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[212939]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[212939]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[212939]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 20290/257667 objects degraded (7.875%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[221102]: pgmap v1575: 119 pgs: 2 remapped+peering, 4 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 263 op/s; 20290/257667 objects degraded (7.875%); 9557/257667 objects misplaced (3.709%); 11 MiB/s, 177 objects/s recovering 2023-09-23T23:34:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:34:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[221102]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:34:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[221102]: osdmap e573: 8 total, 8 up, 7 in 2023-09-23T23:34:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[221102]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[221102]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[221102]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[221102]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[221102]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[221102]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[221102]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[221102]: failed to encode map e573 with expected crc 2023-09-23T23:34:39.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:39 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 20290/257667 objects degraded (7.875%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:39.887 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:34:39.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:39.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:34:39.929 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:39.967 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:34:39.996 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:34:40.024 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:40.088 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:34:40.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:40.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:34:40.197 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:40.236 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:40.413 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:40.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:40.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:40.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:40 smithi138 ceph-mon[193606]: failed to encode map e574 with expected crc 2023-09-23T23:34:40.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:40 smithi138 ceph-mon[193606]: osdmap e574: 8 total, 8 up, 7 in 2023-09-23T23:34:40.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:40 smithi138 ceph-mon[193606]: failed to encode map e574 with expected crc 2023-09-23T23:34:40.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:40 smithi138 ceph-mon[193606]: failed to encode map e574 with expected crc 2023-09-23T23:34:40.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:40 smithi138 ceph-mon[193606]: failed to encode map e574 with expected crc 2023-09-23T23:34:40.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:40 smithi138 ceph-mon[193606]: failed to encode map e574 with expected crc 2023-09-23T23:34:40.507 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:40 smithi110 ceph-mon[212939]: failed to encode map e574 with expected crc 2023-09-23T23:34:40.507 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:40 smithi110 ceph-mon[212939]: osdmap e574: 8 total, 8 up, 7 in 2023-09-23T23:34:40.507 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:40 smithi110 ceph-mon[212939]: failed to encode map e574 with expected crc 2023-09-23T23:34:40.507 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:40 smithi110 ceph-mon[212939]: failed to encode map e574 with expected crc 2023-09-23T23:34:40.508 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:40 smithi110 ceph-mon[212939]: failed to encode map e574 with expected crc 2023-09-23T23:34:40.508 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:40 smithi110 ceph-mon[212939]: failed to encode map e574 with expected crc 2023-09-23T23:34:40.508 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:40 smithi110 ceph-mon[221102]: failed to encode map e574 with expected crc 2023-09-23T23:34:40.508 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:40 smithi110 ceph-mon[221102]: osdmap e574: 8 total, 8 up, 7 in 2023-09-23T23:34:40.508 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:40 smithi110 ceph-mon[221102]: failed to encode map e574 with expected crc 2023-09-23T23:34:40.508 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:40 smithi110 ceph-mon[221102]: failed to encode map e574 with expected crc 2023-09-23T23:34:40.508 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:40 smithi110 ceph-mon[221102]: failed to encode map e574 with expected crc 2023-09-23T23:34:40.508 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:40 smithi110 ceph-mon[221102]: failed to encode map e574 with expected crc 2023-09-23T23:34:40.526 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:34:40.569 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:40.661 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:34:40.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:40.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:34:40.773 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:34:40.795 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:34:40.804 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:40.817 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:34:40.852 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:34:40.857 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:34:40.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:40.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:34:40.965 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:40.991 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:41.032 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:41.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:41.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:41.096 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:34:41.096 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:34:41.097 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:34:41.097 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:34:41.097 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:34:41.097 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:34:41.097 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:34:41.097 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:34:41.097 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:34:41.097 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:34:41.097 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:34:41.098 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:34:41.098 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:34:41.139 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:41.172 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:41.258 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:41 smithi110 ceph-mon[221102]: pgmap v1578: 119 pgs: 1 active+recovery_wait+degraded, 2 remapped+peering, 7 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 246 op/s; 23968/255558 objects degraded (9.379%); 6263/255558 objects misplaced (2.451%); 14 MiB/s, 225 objects/s recovering 2023-09-23T23:34:41.259 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:41 smithi110 ceph-mon[221102]: failed to encode map e574 with expected crc 2023-09-23T23:34:41.259 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:41 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1190303526' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:34:41.579 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:41 smithi110 ceph-mon[212939]: pgmap v1578: 119 pgs: 1 active+recovery_wait+degraded, 2 remapped+peering, 7 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 246 op/s; 23968/255558 objects degraded (9.379%); 6263/255558 objects misplaced (2.451%); 14 MiB/s, 225 objects/s recovering 2023-09-23T23:34:41.579 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:41 smithi110 ceph-mon[212939]: failed to encode map e574 with expected crc 2023-09-23T23:34:41.579 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:41 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1190303526' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:34:41.657 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:41.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:41.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:41.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:41 smithi138 ceph-mon[193606]: pgmap v1578: 119 pgs: 1 active+recovery_wait+degraded, 2 remapped+peering, 7 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 246 op/s; 23968/255558 objects degraded (9.379%); 6263/255558 objects misplaced (2.451%); 14 MiB/s, 225 objects/s recovering 2023-09-23T23:34:41.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:41 smithi138 ceph-mon[193606]: failed to encode map e574 with expected crc 2023-09-23T23:34:41.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:41 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1190303526' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:34:41.763 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:34:41.790 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:41.837 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:34:41.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:41.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:34:41.856 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:34:41.944 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:34:41.971 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:42.034 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:34:42.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:42.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:34:42.141 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:34:42.179 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:42.294 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:34:42.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:42.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:34:42.399 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:34:42.436 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:42.621 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:34:42.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:42.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:34:42.667 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:42.728 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:34:42.770 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:42.915 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:34:42.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:42.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:34:43.025 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:43.095 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:43.286 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:43.364 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:34:43.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:43 smithi110 ceph-mon[221102]: pgmap v1579: 119 pgs: 1 active+recovery_wait+degraded, 6 peering, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 28365/254556 objects degraded (11.143%); 7135/254556 objects misplaced (2.803%); 11 MiB/s, 179 objects/s recovering 2023-09-23T23:34:43.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:43.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:34:43.370 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:43 smithi110 ceph-mon[212939]: pgmap v1579: 119 pgs: 1 active+recovery_wait+degraded, 6 peering, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 28365/254556 objects degraded (11.143%); 7135/254556 objects misplaced (2.803%); 11 MiB/s, 179 objects/s recovering 2023-09-23T23:34:43.371 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:43.371 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:34:43.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:43.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:34:43.473 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:43.511 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:43 smithi138 ceph-mon[193606]: pgmap v1579: 119 pgs: 1 active+recovery_wait+degraded, 6 peering, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 28365/254556 objects degraded (11.143%); 7135/254556 objects misplaced (2.803%); 11 MiB/s, 179 objects/s recovering 2023-09-23T23:34:43.511 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:43.511 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:34:43.512 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:43.564 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:43.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:43.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:43.671 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:34:43.705 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:44.232 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:34:44.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:44.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:34:44.276 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T23:34:44.345 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:44.402 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:44.555 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:34:44.585 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:44 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3516014197' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:34:44.585 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:44 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 28365/254556 objects degraded (11.143%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:44.585 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:44 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3516014197' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:34:44.585 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:44 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 28365/254556 objects degraded (11.143%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:44.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:44 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3516014197' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:34:44.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:44 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 28365/254556 objects degraded (11.143%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:45.083 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:45.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:45.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:45.182 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T23:34:45.194 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:34:45.223 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:45.281 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:34:45.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:45.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:34:45.358 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:45 smithi110 ceph-mon[212939]: pgmap v1580: 119 pgs: 1 active+recovery_wait+degraded, 6 peering, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 144 op/s; 28365/254556 objects degraded (11.143%); 7135/254556 objects misplaced (2.803%); 6.9 MiB/s, 112 objects/s recovering 2023-09-23T23:34:45.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:45 smithi110 ceph-mon[212939]: failed to encode map e575 with expected crc 2023-09-23T23:34:45.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:45 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3516014197' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:34:45.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:45 smithi110 ceph-mon[212939]: osdmap e575: 8 total, 8 up, 7 in 2023-09-23T23:34:45.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:45 smithi110 ceph-mon[212939]: failed to encode map e575 with expected crc 2023-09-23T23:34:45.360 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:45 smithi110 ceph-mon[221102]: pgmap v1580: 119 pgs: 1 active+recovery_wait+degraded, 6 peering, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 144 op/s; 28365/254556 objects degraded (11.143%); 7135/254556 objects misplaced (2.803%); 6.9 MiB/s, 112 objects/s recovering 2023-09-23T23:34:45.360 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:45 smithi110 ceph-mon[221102]: failed to encode map e575 with expected crc 2023-09-23T23:34:45.360 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:45 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3516014197' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:34:45.360 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:45 smithi110 ceph-mon[221102]: osdmap e575: 8 total, 8 up, 7 in 2023-09-23T23:34:45.361 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:45 smithi110 ceph-mon[221102]: failed to encode map e575 with expected crc 2023-09-23T23:34:45.389 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:34:45.477 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:45.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:45 smithi138 ceph-mon[193606]: pgmap v1580: 119 pgs: 1 active+recovery_wait+degraded, 6 peering, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 144 op/s; 28365/254556 objects degraded (11.143%); 7135/254556 objects misplaced (2.803%); 6.9 MiB/s, 112 objects/s recovering 2023-09-23T23:34:45.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:45 smithi138 ceph-mon[193606]: failed to encode map e575 with expected crc 2023-09-23T23:34:45.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:45 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3516014197' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:34:45.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:45 smithi138 ceph-mon[193606]: osdmap e575: 8 total, 8 up, 7 in 2023-09-23T23:34:45.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:45 smithi138 ceph-mon[193606]: failed to encode map e575 with expected crc 2023-09-23T23:34:46.581 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:34:46.582 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:34:46.589 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:34:46.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:46.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:34:46.616 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:46.698 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:46.759 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:46.865 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:34:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:34:46] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.43.0" 2023-09-23T23:34:46.887 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:46.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:46.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:47.006 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:47.034 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:47.427 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:47.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:47.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:47.532 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:34:47.566 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:47 smithi138 ceph-mon[193606]: pgmap v1582: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 284 op/s; 28265/251148 objects degraded (11.254%); 6651/251148 objects misplaced (2.648%); 11 MiB/s, 181 objects/s recovering 2023-09-23T23:34:47.566 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:47 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:47.566 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:47 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:47.571 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:47.597 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:47 smithi110 ceph-mon[212939]: pgmap v1582: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 284 op/s; 28265/251148 objects degraded (11.254%); 6651/251148 objects misplaced (2.648%); 11 MiB/s, 181 objects/s recovering 2023-09-23T23:34:47.597 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:47.597 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:47.597 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:47 smithi110 ceph-mon[221102]: pgmap v1582: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 284 op/s; 28265/251148 objects degraded (11.254%); 6651/251148 objects misplaced (2.648%); 11 MiB/s, 181 objects/s recovering 2023-09-23T23:34:47.597 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:47 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:47.598 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:47 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:47.627 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:47.628 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:34:47.637 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:34:47.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:47.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:34:47.743 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:34:47.771 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:47.836 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:34:47.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:47.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:34:47.944 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:34:47.976 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:48.044 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:34:48.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:48.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:34:48.149 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:48.204 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:48.468 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:34:48.477 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:48 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:34:48.478 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:48 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:34:48.478 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:48.478 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:48 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3131016606' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:34:48.478 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:48 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:34:48.478 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:48 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:34:48.479 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:48 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:34:48.479 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:48 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:48.479 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:48 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3131016606' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:34:48.479 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:48 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:34:48.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:48 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:34:48.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:48 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:34:48.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:48 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:48.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:48 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3131016606' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:34:48.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:48 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:34:49.053 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:49.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:49.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:49.158 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:49.185 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:49.241 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:49.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:49.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:49.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:49 smithi110 ceph-mon[212939]: pgmap v1583: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 239 op/s; 28155/249915 objects degraded (11.266%); 6650/249915 objects misplaced (2.661%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T23:34:49.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:49 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:34:49.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:49 smithi110 ceph-mon[212939]: failed to encode map e576 with expected crc 2023-09-23T23:34:49.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:49 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:34:49.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:49 smithi110 ceph-mon[212939]: osdmap e576: 8 total, 8 up, 7 in 2023-09-23T23:34:49.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:49 smithi110 ceph-mon[212939]: failed to encode map e576 with expected crc 2023-09-23T23:34:49.282 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:49 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 28155/249915 objects degraded (11.266%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:49.282 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:49 smithi110 ceph-mon[221102]: pgmap v1583: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 239 op/s; 28155/249915 objects degraded (11.266%); 6650/249915 objects misplaced (2.661%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T23:34:49.282 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:49 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:34:49.283 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:49 smithi110 ceph-mon[221102]: failed to encode map e576 with expected crc 2023-09-23T23:34:49.283 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:49 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:34:49.283 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:49 smithi110 ceph-mon[221102]: osdmap e576: 8 total, 8 up, 7 in 2023-09-23T23:34:49.283 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:49 smithi110 ceph-mon[221102]: failed to encode map e576 with expected crc 2023-09-23T23:34:49.283 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:49 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 28155/249915 objects degraded (11.266%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:49.350 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:34:49.403 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:49 smithi138 ceph-mon[193606]: pgmap v1583: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 239 op/s; 28155/249915 objects degraded (11.266%); 6650/249915 objects misplaced (2.661%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T23:34:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:49 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:34:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:49 smithi138 ceph-mon[193606]: failed to encode map e576 with expected crc 2023-09-23T23:34:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:49 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:34:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:49 smithi138 ceph-mon[193606]: osdmap e576: 8 total, 8 up, 7 in 2023-09-23T23:34:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:49 smithi138 ceph-mon[193606]: failed to encode map e576 with expected crc 2023-09-23T23:34:49.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:49 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 28155/249915 objects degraded (11.266%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:49.726 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:34:49.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:49.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:34:49.831 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:34:49.875 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:50.188 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:34:50.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:50.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:34:50.292 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:34:50.336 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:50.492 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:34:50.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:50.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:34:50.557 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:50.604 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:50.637 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:34:50.645 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:50.681 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:34:50.710 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:34:50.740 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:34:50.771 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:34:50.802 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:34:50.815 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:50.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:50.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:50.831 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:34:50.863 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:34:50.865 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:34:50.865 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:34:50.865 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:34:50.865 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:34:50.865 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:34:50.865 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:34:50.865 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:34:50.865 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:34:50.922 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:50.962 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:51.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:51 smithi110 ceph-mon[212939]: pgmap v1585: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 28041/245952 objects degraded (11.401%); 6190/245952 objects misplaced (2.517%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T23:34:51.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:51 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2255135381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:34:51.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:51 smithi110 ceph-mon[221102]: pgmap v1585: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 28041/245952 objects degraded (11.401%); 6190/245952 objects misplaced (2.517%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T23:34:51.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:51 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2255135381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:34:51.618 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:34:51.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:51 smithi138 ceph-mon[193606]: pgmap v1585: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 28041/245952 objects degraded (11.401%); 6190/245952 objects misplaced (2.517%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T23:34:51.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:51 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2255135381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:34:51.763 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:51.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:51.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:51.876 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:34:51.906 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:51.956 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:34:51.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:51.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:34:52.062 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:52.089 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:52.309 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T23:34:52.337 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:52 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3679566496' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:34:52.338 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:52 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3679566496' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:34:52.484 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:34:52.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:52.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:34:52.592 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:34:52.617 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:52 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3679566496' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:34:52.621 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:52.665 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:34:52.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:52.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:34:52.771 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:34:52.798 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:52.852 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:34:52.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:52.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:34:52.962 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:34:52.997 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:53.070 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:34:53.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:53.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:34:53.115 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T23:34:53.179 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:53.237 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:53.303 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:53 smithi138 ceph-mon[193606]: pgmap v1586: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 27908/244662 objects degraded (11.407%); 6187/244662 objects misplaced (2.529%); 11 MiB/s, 175 objects/s recovering 2023-09-23T23:34:53.303 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:53 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3679566496' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:34:53.303 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:53 smithi138 ceph-mon[193606]: osdmap e577: 8 total, 8 up, 7 in 2023-09-23T23:34:53.303 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:53 smithi138 ceph-mon[193606]: failed to encode map e577 with expected crc 2023-09-23T23:34:53.354 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:34:53.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:53.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:53 smithi110 ceph-mon[212939]: pgmap v1586: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 27908/244662 objects degraded (11.407%); 6187/244662 objects misplaced (2.529%); 11 MiB/s, 175 objects/s recovering 2023-09-23T23:34:53.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:53 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3679566496' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:34:53.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:53 smithi110 ceph-mon[212939]: osdmap e577: 8 total, 8 up, 7 in 2023-09-23T23:34:53.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:53 smithi110 ceph-mon[212939]: failed to encode map e577 with expected crc 2023-09-23T23:34:53.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:53 smithi110 ceph-mon[221102]: pgmap v1586: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 27908/244662 objects degraded (11.407%); 6187/244662 objects misplaced (2.529%); 11 MiB/s, 175 objects/s recovering 2023-09-23T23:34:53.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:53 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3679566496' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:34:53.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:53 smithi110 ceph-mon[221102]: osdmap e577: 8 total, 8 up, 7 in 2023-09-23T23:34:53.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:53 smithi110 ceph-mon[221102]: failed to encode map e577 with expected crc 2023-09-23T23:34:53.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:34:53.467 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:53.481 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:53.512 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:53.602 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:34:53.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:53.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:34:53.623 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:34:53.623 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:34:53.623 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:34:53.623 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:34:53.623 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:34:53.708 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:53.750 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:53.885 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:53.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:53.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:53.990 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:34:54.017 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:54.060 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:34:54.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:54.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:34:54.166 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:34:54.195 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:54.205 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:34:54.238 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:34:54.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:54.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:34:54.348 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:54.378 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:54.631 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:54.650 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:54 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 27908/244662 objects degraded (11.407%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:54.651 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:54 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 27908/244662 objects degraded (11.407%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:54.663 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:34:54.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:54 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 27908/244662 objects degraded (11.407%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T23:34:55.164 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:34:55.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:55.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:34:55.268 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:34:55.295 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:55.347 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:34:55.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:55.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:34:55.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:55 smithi110 ceph-mon[212939]: pgmap v1588: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 270 op/s; 27908/244662 objects degraded (11.407%); 6187/244662 objects misplaced (2.529%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T23:34:55.435 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:55 smithi110 ceph-mon[221102]: pgmap v1588: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 270 op/s; 27908/244662 objects degraded (11.407%); 6187/244662 objects misplaced (2.529%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T23:34:55.455 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:34:55.488 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:55.548 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:34:55.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:55.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:34:55.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:55 smithi138 ceph-mon[193606]: pgmap v1588: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 270 op/s; 27908/244662 objects degraded (11.407%); 6187/244662 objects misplaced (2.529%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T23:34:55.615 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:55.655 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:55.708 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:55.924 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:55.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:55.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:56.032 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:56.063 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:56.318 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:34:56.348 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:56 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4278314756' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:34:56.348 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:56 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4278314756' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:34:56.486 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:34:56.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:56.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:34:56.591 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:56.621 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:56.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:56 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4278314756' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:34:56.954 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:34:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:34:56] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.43.0" 2023-09-23T23:34:57.296 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:34:57.307 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:57 smithi110 ceph-mon[221102]: pgmap v1589: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 27825/241230 objects degraded (11.535%); 5573/241230 objects misplaced (2.310%); 11 MiB/s, 175 objects/s recovering 2023-09-23T23:34:57.307 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:57 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:34:57.308 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:57 smithi110 ceph-mon[221102]: failed to encode map e578 with expected crc 2023-09-23T23:34:57.308 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:57 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4278314756' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:34:57.308 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:57 smithi110 ceph-mon[221102]: osdmap e578: 8 total, 8 up, 7 in 2023-09-23T23:34:57.308 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:57 smithi110 ceph-mon[221102]: failed to encode map e578 with expected crc 2023-09-23T23:34:57.308 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:57 smithi110 ceph-mon[221102]: failed to encode map e578 with expected crc 2023-09-23T23:34:57.308 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:57 smithi110 ceph-mon[221102]: failed to encode map e578 with expected crc 2023-09-23T23:34:57.355 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:34:57.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:57.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:34:57.465 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:34:57.490 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:57.535 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:34:57.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:57.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:34:57.642 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:57.673 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:57 smithi110 ceph-mon[212939]: pgmap v1589: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 27825/241230 objects degraded (11.535%); 5573/241230 objects misplaced (2.310%); 11 MiB/s, 175 objects/s recovering 2023-09-23T23:34:57.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:57 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:34:57.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:57 smithi110 ceph-mon[212939]: failed to encode map e578 with expected crc 2023-09-23T23:34:57.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:57 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4278314756' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:34:57.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:57 smithi110 ceph-mon[212939]: osdmap e578: 8 total, 8 up, 7 in 2023-09-23T23:34:57.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:57 smithi110 ceph-mon[212939]: failed to encode map e578 with expected crc 2023-09-23T23:34:57.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:57 smithi110 ceph-mon[212939]: failed to encode map e578 with expected crc 2023-09-23T23:34:57.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:57 smithi110 ceph-mon[212939]: failed to encode map e578 with expected crc 2023-09-23T23:34:57.678 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:57.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:57 smithi138 ceph-mon[193606]: pgmap v1589: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 361 op/s; 27825/241230 objects degraded (11.535%); 5573/241230 objects misplaced (2.310%); 11 MiB/s, 175 objects/s recovering 2023-09-23T23:34:57.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:57 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:34:57.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:57 smithi138 ceph-mon[193606]: failed to encode map e578 with expected crc 2023-09-23T23:34:57.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:57 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4278314756' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:34:57.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:57 smithi138 ceph-mon[193606]: osdmap e578: 8 total, 8 up, 7 in 2023-09-23T23:34:57.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:57 smithi138 ceph-mon[193606]: failed to encode map e578 with expected crc 2023-09-23T23:34:57.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:57 smithi138 ceph-mon[193606]: failed to encode map e578 with expected crc 2023-09-23T23:34:57.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:57 smithi138 ceph-mon[193606]: failed to encode map e578 with expected crc 2023-09-23T23:34:57.724 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:57.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:57.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:57.828 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:34:57.855 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:58.272 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:34:58.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:58.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:34:58.376 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:58.453 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:58.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:58 smithi110 ceph-mon[212939]: failed to encode map e579 with expected crc 2023-09-23T23:34:58.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:58 smithi110 ceph-mon[212939]: failed to encode map e579 with expected crc 2023-09-23T23:34:58.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:58 smithi110 ceph-mon[212939]: osdmap e579: 8 total, 8 up, 7 in 2023-09-23T23:34:58.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:58 smithi110 ceph-mon[212939]: failed to encode map e579 with expected crc 2023-09-23T23:34:58.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:58 smithi110 ceph-mon[212939]: failed to encode map e579 with expected crc 2023-09-23T23:34:58.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:58.674 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:34:58.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:58 smithi110 ceph-mon[221102]: failed to encode map e579 with expected crc 2023-09-23T23:34:58.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:58 smithi110 ceph-mon[221102]: failed to encode map e579 with expected crc 2023-09-23T23:34:58.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:58 smithi110 ceph-mon[221102]: osdmap e579: 8 total, 8 up, 7 in 2023-09-23T23:34:58.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:58 smithi110 ceph-mon[221102]: failed to encode map e579 with expected crc 2023-09-23T23:34:58.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:58 smithi110 ceph-mon[221102]: failed to encode map e579 with expected crc 2023-09-23T23:34:58.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:58.675 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:34:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:58 smithi138 ceph-mon[193606]: failed to encode map e579 with expected crc 2023-09-23T23:34:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:58 smithi138 ceph-mon[193606]: failed to encode map e579 with expected crc 2023-09-23T23:34:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:58 smithi138 ceph-mon[193606]: osdmap e579: 8 total, 8 up, 7 in 2023-09-23T23:34:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:58 smithi138 ceph-mon[193606]: failed to encode map e579 with expected crc 2023-09-23T23:34:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:58 smithi138 ceph-mon[193606]: failed to encode map e579 with expected crc 2023-09-23T23:34:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:34:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:34:58.719 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:34:58.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:58.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:34:58.823 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:34:58.850 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:58.897 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:34:58.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:58.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:34:59.002 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:34:59.029 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:59.073 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:34:59.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:59.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:34:59.180 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:34:59.214 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:59.321 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:59 smithi138 ceph-mon[193606]: pgmap v1592: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 243 op/s; 27739/240285 objects degraded (11.544%); 5536/240285 objects misplaced (2.304%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T23:34:59.321 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:59 smithi138 ceph-mon[193606]: osdmap e580: 8 total, 8 up, 7 in 2023-09-23T23:34:59.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:59 smithi138 ceph-mon[193606]: failed to encode map e580 with expected crc 2023-09-23T23:34:59.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:59 smithi138 ceph-mon[193606]: failed to encode map e580 with expected crc 2023-09-23T23:34:59.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:59 smithi138 ceph-mon[193606]: failed to encode map e580 with expected crc 2023-09-23T23:34:59.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:59 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 27739/240285 objects degraded (11.544%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:34:59.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:34:59 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1773014817' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:34:59.331 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:34:59.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:59.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:34:59.365 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:34:59.436 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:34:59.465 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:59.512 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:34:59.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:59.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:34:59.591 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:34:59.615 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:34:59.617 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:34:59.645 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:34:59.648 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:59 smithi110 ceph-mon[221102]: pgmap v1592: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 243 op/s; 27739/240285 objects degraded (11.544%); 5536/240285 objects misplaced (2.304%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T23:34:59.649 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:59 smithi110 ceph-mon[221102]: osdmap e580: 8 total, 8 up, 7 in 2023-09-23T23:34:59.649 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:59 smithi110 ceph-mon[221102]: failed to encode map e580 with expected crc 2023-09-23T23:34:59.649 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:59 smithi110 ceph-mon[221102]: failed to encode map e580 with expected crc 2023-09-23T23:34:59.649 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:59 smithi110 ceph-mon[221102]: failed to encode map e580 with expected crc 2023-09-23T23:34:59.649 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:59 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 27739/240285 objects degraded (11.544%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:34:59.649 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:34:59 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1773014817' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:34:59.649 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:59 smithi110 ceph-mon[212939]: pgmap v1592: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 109 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 243 op/s; 27739/240285 objects degraded (11.544%); 5536/240285 objects misplaced (2.304%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T23:34:59.649 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:59 smithi110 ceph-mon[212939]: osdmap e580: 8 total, 8 up, 7 in 2023-09-23T23:34:59.650 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:59 smithi110 ceph-mon[212939]: failed to encode map e580 with expected crc 2023-09-23T23:34:59.650 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:59 smithi110 ceph-mon[212939]: failed to encode map e580 with expected crc 2023-09-23T23:34:59.650 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:59 smithi110 ceph-mon[212939]: failed to encode map e580 with expected crc 2023-09-23T23:34:59.650 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:59 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 27739/240285 objects degraded (11.544%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:34:59.650 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:34:59 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1773014817' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:34:59.652 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:34:59.678 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:34:59.700 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:34:59.729 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:34:59.729 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T23:34:59.763 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:34:59.766 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:34:59.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:34:59.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:34:59.798 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:34:59.800 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:34:59.801 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:34:59.801 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:34:59.801 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:34:59.801 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:34:59.801 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:34:59.801 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:34:59.801 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:34:59.872 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:34:59.923 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:00.021 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:35:00.022 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:35:00.041 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:00.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:00.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:00.146 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:00.175 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:00.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:00 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/847375131' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:00.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:00 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1773014817' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:35:00.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:00 smithi110 ceph-mon[221102]: osdmap e581: 8 total, 8 up, 7 in 2023-09-23T23:35:00.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:00 smithi110 ceph-mon[221102]: failed to encode map e581 with expected crc 2023-09-23T23:35:00.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:00 smithi110 ceph-mon[221102]: failed to encode map e581 with expected crc 2023-09-23T23:35:00.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:00 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/847375131' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:00.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:00 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1773014817' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:35:00.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:00 smithi110 ceph-mon[212939]: osdmap e581: 8 total, 8 up, 7 in 2023-09-23T23:35:00.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:00 smithi110 ceph-mon[212939]: failed to encode map e581 with expected crc 2023-09-23T23:35:00.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:00 smithi110 ceph-mon[212939]: failed to encode map e581 with expected crc 2023-09-23T23:35:00.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:00 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/847375131' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:00.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:00 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1773014817' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:35:00.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:00 smithi138 ceph-mon[193606]: osdmap e581: 8 total, 8 up, 7 in 2023-09-23T23:35:00.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:00 smithi138 ceph-mon[193606]: failed to encode map e581 with expected crc 2023-09-23T23:35:00.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:00 smithi138 ceph-mon[193606]: failed to encode map e581 with expected crc 2023-09-23T23:35:00.852 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:00.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:00.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:00.956 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:00.988 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:01 smithi138 ceph-mon[193606]: failed to encode map e581 with expected crc 2023-09-23T23:35:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:01 smithi138 ceph-mon[193606]: pgmap v1595: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 236 op/s; 27681/238389 objects degraded (11.612%); 4897/238389 objects misplaced (2.054%); 13 MiB/s, 201 objects/s recovering 2023-09-23T23:35:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:01 smithi138 ceph-mon[193606]: failed to encode map e581 with expected crc 2023-09-23T23:35:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:01 smithi110 ceph-mon[212939]: failed to encode map e581 with expected crc 2023-09-23T23:35:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:01 smithi110 ceph-mon[212939]: pgmap v1595: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 236 op/s; 27681/238389 objects degraded (11.612%); 4897/238389 objects misplaced (2.054%); 13 MiB/s, 201 objects/s recovering 2023-09-23T23:35:01.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:01 smithi110 ceph-mon[212939]: failed to encode map e581 with expected crc 2023-09-23T23:35:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:01 smithi110 ceph-mon[221102]: failed to encode map e581 with expected crc 2023-09-23T23:35:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:01 smithi110 ceph-mon[221102]: pgmap v1595: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 236 op/s; 27681/238389 objects degraded (11.612%); 4897/238389 objects misplaced (2.054%); 13 MiB/s, 201 objects/s recovering 2023-09-23T23:35:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:01 smithi110 ceph-mon[221102]: failed to encode map e581 with expected crc 2023-09-23T23:35:01.740 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T23:35:01.783 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:35:01.791 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:01.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:01.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:01.886 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:01.895 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:01.935 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:02.003 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:02.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:02.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:02.108 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:02.141 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:02.208 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:02.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:02.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:02.312 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:02.345 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:02.462 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:02.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:02.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:02.568 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:02.605 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:02.658 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:02.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:02.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:02.761 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:35:02.762 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:02.788 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:02.829 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:02.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:02.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:02.935 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:02.961 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:03.008 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:03.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:03.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:03.099 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:03.115 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:03.138 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:03.174 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:03.284 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:03.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:03.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:03.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:03 smithi110 ceph-mon[221102]: pgmap v1596: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 27630/237744 objects degraded (11.622%); 4837/237744 objects misplaced (2.035%); 10 MiB/s, 164 objects/s recovering 2023-09-23T23:35:03.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:03 smithi110 ceph-mon[212939]: pgmap v1596: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 27630/237744 objects degraded (11.622%); 4837/237744 objects misplaced (2.035%); 10 MiB/s, 164 objects/s recovering 2023-09-23T23:35:03.390 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:03.421 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:03.644 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:03.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:03 smithi138 ceph-mon[193606]: pgmap v1596: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 27630/237744 objects degraded (11.622%); 4837/237744 objects misplaced (2.035%); 10 MiB/s, 164 objects/s recovering 2023-09-23T23:35:04.476 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:04.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:04.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:04.581 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:04.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:04 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 27630/237744 objects degraded (11.622%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:04.608 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:04.661 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:04.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:04.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:04.698 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:04 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 27630/237744 objects degraded (11.622%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:04.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:04 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 27630/237744 objects degraded (11.622%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:04.769 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:04.804 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:05.274 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:05.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:05.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:05.341 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:35:05.379 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:05.411 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:05.575 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:05 smithi110 ceph-mon[212939]: pgmap v1597: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 27630/237744 objects degraded (11.622%); 4837/237744 objects misplaced (2.035%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T23:35:05.575 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:05 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/297523083' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:35:05.575 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:05 smithi110 ceph-mon[221102]: pgmap v1597: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 27630/237744 objects degraded (11.622%); 4837/237744 objects misplaced (2.035%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T23:35:05.575 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:05 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/297523083' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:35:05.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:05 smithi138 ceph-mon[193606]: pgmap v1597: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 27630/237744 objects degraded (11.622%); 4837/237744 objects misplaced (2.035%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T23:35:05.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:05 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/297523083' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:35:06.150 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:06.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:06.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:06.260 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:06.287 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:06.337 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:06.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:06.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:06.375 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:35:06.376 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:35:06.409 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:06 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:35:06.410 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:06 smithi110 ceph-mon[212939]: failed to encode map e582 with expected crc 2023-09-23T23:35:06.410 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:06 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/297523083' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:35:06.410 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:06 smithi110 ceph-mon[212939]: osdmap e582: 8 total, 8 up, 7 in 2023-09-23T23:35:06.410 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:06 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:35:06.410 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:06 smithi110 ceph-mon[221102]: failed to encode map e582 with expected crc 2023-09-23T23:35:06.411 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:06 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/297523083' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:35:06.411 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:06 smithi110 ceph-mon[221102]: osdmap e582: 8 total, 8 up, 7 in 2023-09-23T23:35:06.444 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:06.472 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:06.521 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:06.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:06.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:06.630 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:06.663 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:06 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:35:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:06 smithi138 ceph-mon[193606]: failed to encode map e582 with expected crc 2023-09-23T23:35:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:06 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/297523083' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:35:06.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:06 smithi138 ceph-mon[193606]: osdmap e582: 8 total, 8 up, 7 in 2023-09-23T23:35:06.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:35:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:35:06] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.43.0" 2023-09-23T23:35:07.235 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:07.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:07.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:07.338 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:07.348 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T23:35:07.377 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:07.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:07 smithi110 ceph-mon[212939]: pgmap v1599: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 197 op/s; 27564/235785 objects degraded (11.690%); 4837/235785 objects misplaced (2.051%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:35:07.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:07 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2124649715' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:35:07.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:07 smithi110 ceph-mon[221102]: pgmap v1599: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 197 op/s; 27564/235785 objects degraded (11.690%); 4837/235785 objects misplaced (2.051%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:35:07.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:07 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2124649715' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:35:07.660 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:07.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:07.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:07.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:07 smithi138 ceph-mon[193606]: pgmap v1599: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 197 op/s; 27564/235785 objects degraded (11.690%); 4837/235785 objects misplaced (2.051%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:35:07.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:07 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2124649715' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:35:07.761 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:35:07.764 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:07.790 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:07.834 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:07.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:07.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:07.942 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:07.974 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:08.269 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:35:08.549 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:08.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:08 smithi110 ceph-mon[212939]: failed to encode map e583 with expected crc 2023-09-23T23:35:08.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:08 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2124649715' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:35:08.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:08 smithi110 ceph-mon[212939]: osdmap e583: 8 total, 8 up, 7 in 2023-09-23T23:35:08.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:08 smithi110 ceph-mon[212939]: failed to encode map e583 with expected crc 2023-09-23T23:35:08.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:08 smithi110 ceph-mon[212939]: failed to encode map e583 with expected crc 2023-09-23T23:35:08.557 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:08 smithi110 ceph-mon[212939]: failed to encode map e583 with expected crc 2023-09-23T23:35:08.557 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:08 smithi110 ceph-mon[212939]: failed to encode map e583 with expected crc 2023-09-23T23:35:08.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:08.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:08.558 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:08 smithi110 ceph-mon[221102]: failed to encode map e583 with expected crc 2023-09-23T23:35:08.558 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:08 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2124649715' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:35:08.558 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:08 smithi110 ceph-mon[221102]: osdmap e583: 8 total, 8 up, 7 in 2023-09-23T23:35:08.559 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:08 smithi110 ceph-mon[221102]: failed to encode map e583 with expected crc 2023-09-23T23:35:08.559 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:08 smithi110 ceph-mon[221102]: failed to encode map e583 with expected crc 2023-09-23T23:35:08.559 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:08 smithi110 ceph-mon[221102]: failed to encode map e583 with expected crc 2023-09-23T23:35:08.559 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:08 smithi110 ceph-mon[221102]: failed to encode map e583 with expected crc 2023-09-23T23:35:08.636 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T23:35:08.659 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:08 smithi138 ceph-mon[193606]: failed to encode map e583 with expected crc 2023-09-23T23:35:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:08 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2124649715' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:35:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:08 smithi138 ceph-mon[193606]: osdmap e583: 8 total, 8 up, 7 in 2023-09-23T23:35:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:08 smithi138 ceph-mon[193606]: failed to encode map e583 with expected crc 2023-09-23T23:35:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:08 smithi138 ceph-mon[193606]: failed to encode map e583 with expected crc 2023-09-23T23:35:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:08 smithi138 ceph-mon[193606]: failed to encode map e583 with expected crc 2023-09-23T23:35:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:08 smithi138 ceph-mon[193606]: failed to encode map e583 with expected crc 2023-09-23T23:35:08.729 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:08.805 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:08.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:08.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:08.831 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:08.854 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:35:08.868 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:08.880 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:35:08.912 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:08.923 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:35:08.938 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:08.959 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:35:08.979 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:35:08.988 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:08.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:08.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:09.000 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:35:09.021 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:35:09.050 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:35:09.051 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:35:09.051 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:35:09.052 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:35:09.052 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:35:09.052 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:35:09.052 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:35:09.052 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:35:09.052 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:35:09.095 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:09.141 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:09.350 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:09 smithi110 ceph-mon[212939]: pgmap v1601: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 135 op/s; 27510/235134 objects degraded (11.700%); 4776/235134 objects misplaced (2.031%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:35:09.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:09 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 27510/235134 objects degraded (11.700%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:09.351 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:09 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3539829204' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:09.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:09 smithi110 ceph-mon[221102]: pgmap v1601: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 135 op/s; 27510/235134 objects degraded (11.700%); 4776/235134 objects misplaced (2.031%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:35:09.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:09 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 27510/235134 objects degraded (11.700%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:09.351 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:09 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3539829204' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:09.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:09 smithi138 ceph-mon[193606]: pgmap v1601: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 135 op/s; 27510/235134 objects degraded (11.700%); 4776/235134 objects misplaced (2.031%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:35:09.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:09 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 27510/235134 objects degraded (11.700%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:09.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:09 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3539829204' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:09.958 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:09.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:09.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:10.063 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:10.093 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:10.358 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:10.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:10.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:10.464 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:10.491 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:11.302 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:11.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:11.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:11.405 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:11.409 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:11.445 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:11.565 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:11.567 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:11 smithi110 ceph-mon[212939]: pgmap v1602: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 177 op/s; 27447/233478 objects degraded (11.756%); 4776/233478 objects misplaced (2.046%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T23:35:11.567 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:11 smithi110 ceph-mon[221102]: pgmap v1602: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 177 op/s; 27447/233478 objects degraded (11.756%); 4776/233478 objects misplaced (2.046%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T23:35:11.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:11.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:11.670 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:11.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:11 smithi138 ceph-mon[193606]: pgmap v1602: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 177 op/s; 27447/233478 objects degraded (11.756%); 4776/233478 objects misplaced (2.046%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T23:35:11.698 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:11.741 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:11.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:11.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:11.847 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:11.874 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:11.921 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:11.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:11.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:12.017 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:35:12.029 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:12.121 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:12.204 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:12.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:12.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:12.309 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:12.338 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:12.371 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:35:12.397 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:12.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:12.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:12.404 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:12 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2788994590' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:35:12.404 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:12 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:35:12.404 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:12 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2788994590' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:35:12.405 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:12 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:35:12.504 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:12.530 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:12.579 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:12.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:12 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2788994590' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:35:12.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:12 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:35:12.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:12.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:12.645 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:35:12.645 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:35:12.645 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:35:12.645 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:35:12.645 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:35:12.687 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:12.721 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:12.830 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:12.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:12.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:12.935 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:12.966 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:13.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:13 smithi110 ceph-mon[212939]: pgmap v1603: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 27395/232308 objects degraded (11.793%); 4714/232308 objects misplaced (2.029%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:35:13.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:13 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:35:13.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:13 smithi110 ceph-mon[212939]: failed to encode map e584 with expected crc 2023-09-23T23:35:13.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:13 smithi110 ceph-mon[212939]: failed to encode map e584 with expected crc 2023-09-23T23:35:13.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:13 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:35:13.364 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:13 smithi110 ceph-mon[212939]: osdmap e584: 8 total, 8 up, 7 in 2023-09-23T23:35:13.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:13.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:35:13.640 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:13 smithi110 ceph-mon[221102]: pgmap v1603: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 27395/232308 objects degraded (11.793%); 4714/232308 objects misplaced (2.029%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:35:13.640 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:13 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:35:13.640 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:13 smithi110 ceph-mon[221102]: failed to encode map e584 with expected crc 2023-09-23T23:35:13.640 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:13 smithi110 ceph-mon[221102]: failed to encode map e584 with expected crc 2023-09-23T23:35:13.641 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:13 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:35:13.641 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:13 smithi110 ceph-mon[221102]: osdmap e584: 8 total, 8 up, 7 in 2023-09-23T23:35:13.641 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:13.641 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:35:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:13 smithi138 ceph-mon[193606]: pgmap v1603: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 27395/232308 objects degraded (11.793%); 4714/232308 objects misplaced (2.029%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:35:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:13 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:35:13.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:13 smithi138 ceph-mon[193606]: failed to encode map e584 with expected crc 2023-09-23T23:35:13.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:13 smithi138 ceph-mon[193606]: failed to encode map e584 with expected crc 2023-09-23T23:35:13.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:13 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:35:13.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:13 smithi138 ceph-mon[193606]: osdmap e584: 8 total, 8 up, 7 in 2023-09-23T23:35:13.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:13.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:35:13.804 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:35:13.830 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:13.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:13.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:13.932 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:13.936 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:13.977 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:14.054 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:14.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:14.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:14.157 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:14.190 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:14.236 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:14.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:14.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:14.343 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:14.382 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:14.430 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:14.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:14.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:14.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:14 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 27395/232308 objects degraded (11.793%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:14.534 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:14.564 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:14.611 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:14.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:14.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:14.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:14 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 27395/232308 objects degraded (11.793%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:14.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:14 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 27395/232308 objects degraded (11.793%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:14.718 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:14.751 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:14.803 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:14.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:14.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:14.909 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:14.940 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:14.990 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:14.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:14.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:15.098 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:15.126 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:15.179 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:15.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:15.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:15.262 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:15.287 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:15.332 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:15.375 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:15 smithi110 ceph-mon[212939]: pgmap v1605: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 144 op/s; 27395/232308 objects degraded (11.793%); 4714/232308 objects misplaced (2.029%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:35:15.375 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:15 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2564329020' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:35:15.375 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:15 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:35:15.375 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:15 smithi110 ceph-mon[221102]: pgmap v1605: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 144 op/s; 27395/232308 objects degraded (11.793%); 4714/232308 objects misplaced (2.029%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:35:15.376 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:15 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2564329020' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:35:15.376 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:15 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:35:15.376 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T23:35:15.409 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:15.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:15.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:15.514 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:15.540 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:15.585 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:15.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:15.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:15.644 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:35:15.691 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:15 smithi138 ceph-mon[193606]: pgmap v1605: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 144 op/s; 27395/232308 objects degraded (11.793%); 4714/232308 objects misplaced (2.029%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:35:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:15 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2564329020' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:35:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:15 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:35:15.720 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:16.130 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:16.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:16.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:16.223 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T23:35:16.236 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:16.315 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:16.383 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:16 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:35:16.383 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:16 smithi110 ceph-mon[212939]: osdmap e585: 8 total, 8 up, 7 in 2023-09-23T23:35:16.383 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:16 smithi110 ceph-mon[212939]: failed to encode map e585 with expected crc 2023-09-23T23:35:16.383 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:16 smithi110 ceph-mon[212939]: failed to encode map e585 with expected crc 2023-09-23T23:35:16.441 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:16.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:16.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:16.550 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:16.587 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:16.605 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:35:16.629 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:35:16.651 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:35:16.675 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:35:16.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:16 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:35:16.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:16 smithi138 ceph-mon[193606]: osdmap e585: 8 total, 8 up, 7 in 2023-09-23T23:35:16.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:16 smithi138 ceph-mon[193606]: failed to encode map e585 with expected crc 2023-09-23T23:35:16.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:16 smithi138 ceph-mon[193606]: failed to encode map e585 with expected crc 2023-09-23T23:35:16.699 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:35:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:16 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:35:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:16 smithi110 ceph-mon[221102]: osdmap e585: 8 total, 8 up, 7 in 2023-09-23T23:35:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:16 smithi110 ceph-mon[221102]: failed to encode map e585 with expected crc 2023-09-23T23:35:16.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:16 smithi110 ceph-mon[221102]: failed to encode map e585 with expected crc 2023-09-23T23:35:16.718 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:35:16.737 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:35:16.756 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:35:16.757 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:35:16.757 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:35:16.758 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:35:16.758 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:35:16.758 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:35:16.758 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:35:16.758 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:35:16.758 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:35:17.135 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:35:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:35:16] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2023-09-23T23:35:17.143 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:17.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:17.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:17.247 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:17.274 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:17.318 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:17.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:17.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:17.426 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:17.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:17 smithi110 ceph-mon[212939]: pgmap v1607: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 200 op/s; 27347/230310 objects degraded (11.874%); 4714/230310 objects misplaced (2.047%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:35:17.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:17 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2946232298' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:17.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:17 smithi110 ceph-mon[221102]: pgmap v1607: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 200 op/s; 27347/230310 objects degraded (11.874%); 4714/230310 objects misplaced (2.047%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:35:17.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:17 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2946232298' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:17.453 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:17.501 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:17.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:17.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:17.606 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:17.631 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:17 smithi138 ceph-mon[193606]: pgmap v1607: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 200 op/s; 27347/230310 objects degraded (11.874%); 4714/230310 objects misplaced (2.047%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:35:17.632 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:17 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2946232298' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:17.634 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:17.681 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:17.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:17.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:17.740 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:17.789 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:17.824 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:17.928 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:17.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:17.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:17.940 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:18.033 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:18.061 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:18.109 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:18.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:18.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:18.176 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:35:18.176 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:35:18.215 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:18.249 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:18.963 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:18.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:18.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:19.070 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:19.101 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:19.155 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:35:19.309 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:19.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:19.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:19.414 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:19 smithi138 ceph-mon[193606]: pgmap v1608: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 159 op/s; 27295/229653 objects degraded (11.885%); 4652/229653 objects misplaced (2.026%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T23:35:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:19 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 27295/229653 objects degraded (11.885%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:19.444 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:19.520 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:19 smithi110 ceph-mon[221102]: pgmap v1608: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 159 op/s; 27295/229653 objects degraded (11.885%); 4652/229653 objects misplaced (2.026%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T23:35:19.520 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:19 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 27295/229653 objects degraded (11.885%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:19.521 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:19 smithi110 ceph-mon[212939]: pgmap v1608: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 159 op/s; 27295/229653 objects degraded (11.885%); 4652/229653 objects misplaced (2.026%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T23:35:19.521 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:19 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 27295/229653 objects degraded (11.885%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:19.533 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:35:19.667 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:35:20.037 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:20.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:20.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:20.143 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:20.170 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:20.427 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:20.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:20.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:20.531 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:20.561 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:20.609 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:20.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:20.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:20.713 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:20.739 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:20.787 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:20.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:20.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:20.791 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:20.892 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:20.917 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:20.959 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:20.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:20.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:21.063 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:21.089 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:21.138 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:21.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:21.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:21.242 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:21.284 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:21.344 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:21.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:21.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:21.388 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:21 smithi110 ceph-mon[212939]: pgmap v1609: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 175 op/s; 27235/228090 objects degraded (11.940%); 4652/228090 objects misplaced (2.040%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:35:21.388 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:21 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2213730647' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:35:21.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:21 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:35:21.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:21 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3911933408' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:35:21.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:21 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:35:21.399 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:35:21.410 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T23:35:21.450 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:21.486 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:21.598 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:21.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:21.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:21 smithi138 ceph-mon[193606]: pgmap v1609: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 175 op/s; 27235/228090 objects degraded (11.940%); 4652/228090 objects misplaced (2.040%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:35:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:21 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2213730647' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:35:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:21 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:35:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:21 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3911933408' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:35:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:21 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:35:21.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:21 smithi110 ceph-mon[221102]: pgmap v1609: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 175 op/s; 27235/228090 objects degraded (11.940%); 4652/228090 objects misplaced (2.040%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:35:21.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:21 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2213730647' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:35:21.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:21 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:35:21.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:21 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3911933408' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:35:21.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:21 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:35:21.703 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:21.732 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:21.776 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:21.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:21.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:21.879 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:21.909 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:22.134 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:35:22.134 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:35:22.359 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:22.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:22.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:22.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:22 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:35:22.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:22 smithi138 ceph-mon[193606]: failed to encode map e586 with expected crc 2023-09-23T23:35:22.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:22 smithi138 ceph-mon[193606]: failed to encode map e586 with expected crc 2023-09-23T23:35:22.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:22 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:35:22.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:22 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:35:22.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:22 smithi138 ceph-mon[193606]: osdmap e586: 8 total, 8 up, 7 in 2023-09-23T23:35:22.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:22 smithi138 ceph-mon[193606]: failed to encode map e586 with expected crc 2023-09-23T23:35:22.463 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:22.491 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:22.533 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:22.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:22.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:22.636 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:22.663 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:22 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:35:22.664 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:22 smithi110 ceph-mon[212939]: failed to encode map e586 with expected crc 2023-09-23T23:35:22.664 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:22 smithi110 ceph-mon[212939]: failed to encode map e586 with expected crc 2023-09-23T23:35:22.664 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:22 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:35:22.664 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:22 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:35:22.664 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:22 smithi110 ceph-mon[212939]: osdmap e586: 8 total, 8 up, 7 in 2023-09-23T23:35:22.664 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:22 smithi110 ceph-mon[212939]: failed to encode map e586 with expected crc 2023-09-23T23:35:22.664 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:22 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:35:22.664 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:22 smithi110 ceph-mon[221102]: failed to encode map e586 with expected crc 2023-09-23T23:35:22.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:22 smithi110 ceph-mon[221102]: failed to encode map e586 with expected crc 2023-09-23T23:35:22.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:22 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:35:22.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:22 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:35:22.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:22 smithi110 ceph-mon[221102]: osdmap e586: 8 total, 8 up, 7 in 2023-09-23T23:35:22.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:22 smithi110 ceph-mon[221102]: failed to encode map e586 with expected crc 2023-09-23T23:35:22.667 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:22.834 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:22.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:22.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:22.905 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:35:22.940 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:22.982 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:23.040 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:23.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:23.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:23.115 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T23:35:23.115 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:35:23.154 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:23.209 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:23.351 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:35:23.373 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:23.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:23.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:23.437 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:35:23.460 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:35:23.481 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:23.483 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:35:23.512 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:35:23.515 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:23.536 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:35:23.559 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:35:23.581 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:35:23.589 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:23.605 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:35:23.607 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:35:23.607 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:35:23.607 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:35:23.607 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:35:23.607 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:35:23.607 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:35:23.607 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:35:23.607 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:35:23.622 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:23.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:23.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:23.633 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:23 smithi110 ceph-mon[212939]: pgmap v1611: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 197 op/s; 27235/227568 objects degraded (11.968%); 4652/227568 objects misplaced (2.044%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:35:23.633 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:23 smithi110 ceph-mon[221102]: pgmap v1611: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 197 op/s; 27235/227568 objects degraded (11.968%); 4652/227568 objects misplaced (2.044%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:35:23.649 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:23 smithi138 ceph-mon[193606]: pgmap v1611: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 197 op/s; 27235/227568 objects degraded (11.968%); 4652/227568 objects misplaced (2.044%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T23:35:23.727 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:23.757 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:23.801 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:23.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:23.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:23.905 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:23.942 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:24.559 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:24 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2908347283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:24.559 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:24 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 27235/227568 objects degraded (11.968%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:24.562 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:24 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2908347283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:24.562 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:24 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 27235/227568 objects degraded (11.968%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:24.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:24 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2908347283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:24.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:24 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 27235/227568 objects degraded (11.968%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:24.733 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:24.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:24.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:24.823 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:35:24.838 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:24.892 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:24.942 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:24.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:24.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:25.046 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:25.080 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:25.105 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:35:25.105 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:35:25.105 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:35:25.105 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:35:25.105 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:35:25.249 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:25.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:25.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:25.358 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:25.369 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:25.388 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:25.434 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:25.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:25.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:25.537 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:25.565 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:25.572 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:25 smithi110 ceph-mon[212939]: pgmap v1612: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 27181/226863 objects degraded (11.981%); 4587/226863 objects misplaced (2.022%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T23:35:25.573 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:25 smithi110 ceph-mon[221102]: pgmap v1612: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 27181/226863 objects degraded (11.981%); 4587/226863 objects misplaced (2.022%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T23:35:25.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:25 smithi138 ceph-mon[193606]: pgmap v1612: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 27181/226863 objects degraded (11.981%); 4587/226863 objects misplaced (2.022%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T23:35:25.820 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:25.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:25.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:25.901 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:35:25.925 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:25.959 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:26.113 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:35:26.127 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:35:26.230 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:26.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:26.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:26.293 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:35:26.334 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:26.365 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:26.423 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:26.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:26.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:26.527 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:26.568 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:26.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:35:26.620 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:26.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:26.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:26.724 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:26.755 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:26.815 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:26.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:26.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:26.919 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:26.942 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:35:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:35:26] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2023-09-23T23:35:26.945 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:27.318 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:27.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:27.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:27.393 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1613,"stamp":"2023-09-23T23:35:25.902812+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4914263285,"num_objects":75001,"num_object_clones":0,"num_object_copies":225003,"num_objects_missing_on_primary":3692,"num_objects_missing":3692,"num_objects_degraded":27114,"num_objects_misplaced":4587,"num_objects_unfound":0,"num_objects_dirty":75001,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":100656,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11893,"num_bytes_recovered":776729000,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94703,"ondisk_log_size":94703,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18241672,"kb_used_data":12657744,"kb_used_omap":29,"kb_used_meta":5583842,"kb_avail":637974392,"statfs":{"total":671965249536,"available":653285777408,"internally_reserved":0,"allocated":12961529856,"data_stored":12945606259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5717854931},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-115933184,"num_objects":-1769,"num_object_clones":0,"num_object_copies":-5307,"num_objects_missing_on_primary":-127,"num_objects_missing":-127,"num_objects_degraded":-233,"num_objects_misplaced":-127,"num_objects_unfound":0,"num_objects_dirty":-1769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1769,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":360,"num_bytes_recovered":23592960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002945"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":782,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.413018+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:35:15.413018+0000","last_peered":"2023-09-23T23:35:15.413018+0000","last_clean":"2023-09-23T23:35:15.413018+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:35:15.413018+0000","last_undegraded":"2023-09-23T23:35:15.413018+0000","last_fullsized":"2023-09-23T23:35:15.413018+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":340,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.367895+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:35:15.367895+0000","last_peered":"2023-09-23T23:35:15.367895+0000","last_clean":"2023-09-23T23:35:15.367895+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:35:15.367895+0000","last_undegraded":"2023-09-23T23:35:15.367895+0000","last_fullsized":"2023-09-23T23:35:15.367895+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":567,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.452941+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:35:15.452941+0000","last_peered":"2023-09-23T23:35:15.452941+0000","last_clean":"2023-09-23T23:35:15.452941+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:35:15.452941+0000","last_undegraded":"2023-09-23T23:35:15.452941+0000","last_fullsized":"2023-09-23T23:35:15.452941+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":735,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.413307+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:35:15.413307+0000","last_peered":"2023-09-23T23:35:15.413307+0000","last_clean":"2023-09-23T23:35:15.413307+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:35:15.413307+0000","last_undegraded":"2023-09-23T23:35:15.413307+0000","last_fullsized":"2023-09-23T23:35:15.413307+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":304,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417918+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:35:15.417918+0000","last_peered":"2023-09-23T23:35:15.417918+0000","last_clean":"2023-09-23T23:35:15.417918+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:35:15.417918+0000","last_undegraded":"2023-09-23T23:35:15.417918+0000","last_fullsized":"2023-09-23T23:35:15.417918+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":546,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.422841+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:35:15.422841+0000","last_peered":"2023-09-23T23:35:15.422841+0000","last_clean":"2023-09-23T23:35:15.422841+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:35:15.422841+0000","last_undegraded":"2023-09-23T23:35:15.422841+0000","last_fullsized":"2023-09-23T23:35:15.422841+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":747,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.426356+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:35:15.426356+0000","last_peered":"2023-09-23T23:35:15.426356+0000","last_clean":"2023-09-23T23:35:15.426356+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:35:15.426356+0000","last_undegraded":"2023-09-23T23:35:15.426356+0000","last_fullsized":"2023-09-23T23:35:15.426356+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":604,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417733+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:35:15.417733+0000","last_peered":"2023-09-23T23:35:15.417733+0000","last_clean":"2023-09-23T23:35:15.417733+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:35:15.417733+0000","last_undegraded":"2023-09-23T23:35:15.417733+0000","last_fullsized":"2023-09-23T23:35:15.417733+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":100,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.506121+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:35:15.506121+0000","last_peered":"2023-09-23T23:35:15.506121+0000","last_clean":"2023-09-23T23:35:15.506121+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:35:15.506121+0000","last_undegraded":"2023-09-23T23:35:15.506121+0000","last_fullsized":"2023-09-23T23:35:15.506121+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":906,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.426615+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:35:15.426615+0000","last_peered":"2023-09-23T23:35:15.426615+0000","last_clean":"2023-09-23T23:35:15.426615+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:35:15.426615+0000","last_undegraded":"2023-09-23T23:35:15.426615+0000","last_fullsized":"2023-09-23T23:35:15.426615+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":238,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.413657+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:35:15.413657+0000","last_peered":"2023-09-23T23:35:15.413657+0000","last_clean":"2023-09-23T23:35:15.413657+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:35:15.413657+0000","last_undegraded":"2023-09-23T23:35:15.413657+0000","last_fullsized":"2023-09-23T23:35:15.413657+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":586,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417704+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:35:15.417704+0000","last_peered":"2023-09-23T23:35:15.417704+0000","last_clean":"2023-09-23T23:35:15.417704+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:35:15.417704+0000","last_undegraded":"2023-09-23T23:35:15.417704+0000","last_fullsized":"2023-09-23T23:35:15.417704+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":878,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.367588+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:35:15.367588+0000","last_peered":"2023-09-23T23:35:15.367588+0000","last_clean":"2023-09-23T23:35:15.367588+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:35:15.367588+0000","last_undegraded":"2023-09-23T23:35:15.367588+0000","last_fullsized":"2023-09-23T23:35:15.367588+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":217,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.413714+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:35:15.413714+0000","last_peered":"2023-09-23T23:35:15.413714+0000","last_clean":"2023-09-23T23:35:15.413714+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:35:15.413714+0000","last_undegraded":"2023-09-23T23:35:15.413714+0000","last_fullsized":"2023-09-23T23:35:15.413714+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":547,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417819+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:35:15.417819+0000","last_peered":"2023-09-23T23:35:15.417819+0000","last_clean":"2023-09-23T23:35:15.417819+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:35:15.417819+0000","last_undegraded":"2023-09-23T23:35:15.417819+0000","last_fullsized":"2023-09-23T23:35:15.417819+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1040,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.413610+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:35:15.413610+0000","last_peered":"2023-09-23T23:35:15.413610+0000","last_clean":"2023-09-23T23:35:15.413610+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:35:15.413610+0000","last_undegraded":"2023-09-23T23:35:15.413610+0000","last_fullsized":"2023-09-23T23:35:15.413610+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":258,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.432397+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:35:15.432397+0000","last_peered":"2023-09-23T23:35:15.432397+0000","last_clean":"2023-09-23T23:35:15.432397+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:35:15.432397+0000","last_undegraded":"2023-09-23T23:35:15.432397+0000","last_fullsized":"2023-09-23T23:35:15.432397+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":119,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505730+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:35:15.505730+0000","last_peered":"2023-09-23T23:35:15.505730+0000","last_clean":"2023-09-23T23:35:15.505730+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:35:15.505730+0000","last_undegraded":"2023-09-23T23:35:15.505730+0000","last_fullsized":"2023-09-23T23:35:15.505730+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":436,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.423138+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:35:15.423138+0000","last_peered":"2023-09-23T23:35:15.423138+0000","last_clean":"2023-09-23T23:35:15.423138+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:35:15.423138+0000","last_undegraded":"2023-09-23T23:35:15.423138+0000","last_fullsized":"2023-09-23T23:35:15.423138+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":160,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505605+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:35:15.505605+0000","last_peered":"2023-09-23T23:35:15.505605+0000","last_clean":"2023-09-23T23:35:15.505605+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:35:15.505605+0000","last_undegraded":"2023-09-23T23:35:15.505605+0000","last_fullsized":"2023-09-23T23:35:15.505605+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":527,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505575+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:35:15.505575+0000","last_peered":"2023-09-23T23:35:15.505575+0000","last_clean":"2023-09-23T23:35:15.505575+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:35:15.505575+0000","last_undegraded":"2023-09-23T23:35:15.505575+0000","last_fullsized":"2023-09-23T23:35:15.505575+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":741,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.412958+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:35:15.412958+0000","last_peered":"2023-09-23T23:35:15.412958+0000","last_clean":"2023-09-23T23:35:15.412958+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:35:15.412958+0000","last_undegraded":"2023-09-23T23:35:15.412958+0000","last_fullsized":"2023-09-23T23:35:15.412958+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":132,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.365891+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:35:15.365891+0000","last_peered":"2023-09-23T23:35:15.365891+0000","last_clean":"2023-09-23T23:35:15.365891+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:35:15.365891+0000","last_undegraded":"2023-09-23T23:35:15.365891+0000","last_fullsized":"2023-09-23T23:35:15.365891+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":585,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.422831+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:35:15.422831+0000","last_peered":"2023-09-23T23:35:15.422831+0000","last_clean":"2023-09-23T23:35:15.422831+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:35:15.422831+0000","last_undegraded":"2023-09-23T23:35:15.422831+0000","last_fullsized":"2023-09-23T23:35:15.422831+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":871,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505256+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:35:15.505256+0000","last_peered":"2023-09-23T23:35:15.505256+0000","last_clean":"2023-09-23T23:35:15.505256+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:35:15.505256+0000","last_undegraded":"2023-09-23T23:35:15.505256+0000","last_fullsized":"2023-09-23T23:35:15.505256+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":180,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417996+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:35:15.417996+0000","last_peered":"2023-09-23T23:35:15.417996+0000","last_clean":"2023-09-23T23:35:15.417996+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:35:15.417996+0000","last_undegraded":"2023-09-23T23:35:15.417996+0000","last_fullsized":"2023-09-23T23:35:15.417996+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":307,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.438250+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:35:15.438250+0000","last_peered":"2023-09-23T23:35:15.438250+0000","last_clean":"2023-09-23T23:35:15.438250+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:35:15.438250+0000","last_undegraded":"2023-09-23T23:35:15.438250+0000","last_fullsized":"2023-09-23T23:35:15.438250+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":548,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.418722+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:35:15.418722+0000","last_peered":"2023-09-23T23:35:15.418722+0000","last_clean":"2023-09-23T23:35:15.418722+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:35:15.418722+0000","last_undegraded":"2023-09-23T23:35:15.418722+0000","last_fullsized":"2023-09-23T23:35:15.418722+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":454,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505375+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:35:15.505375+0000","last_peered":"2023-09-23T23:35:15.505375+0000","last_clean":"2023-09-23T23:35:15.505375+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:35:15.505375+0000","last_undegraded":"2023-09-23T23:35:15.505375+0000","last_fullsized":"2023-09-23T23:35:15.505375+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":541,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.423236+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:35:15.423236+0000","last_peered":"2023-09-23T23:35:15.423236+0000","last_clean":"2023-09-23T23:35:15.423236+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:35:15.423236+0000","last_undegraded":"2023-09-23T23:35:15.423236+0000","last_fullsized":"2023-09-23T23:35:15.423236+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":949,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.413207+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:35:15.413207+0000","last_peered":"2023-09-23T23:35:15.413207+0000","last_clean":"2023-09-23T23:35:15.413207+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:35:15.413207+0000","last_undegraded":"2023-09-23T23:35:15.413207+0000","last_fullsized":"2023-09-23T23:35:15.413207+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":150,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505459+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:35:15.505459+0000","last_peered":"2023-09-23T23:35:15.505459+0000","last_clean":"2023-09-23T23:35:15.505459+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:35:15.505459+0000","last_undegraded":"2023-09-23T23:35:15.505459+0000","last_fullsized":"2023-09-23T23:35:15.505459+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":226,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.423066+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:35:15.423066+0000","last_peered":"2023-09-23T23:35:15.423066+0000","last_clean":"2023-09-23T23:35:15.423066+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:35:15.423066+0000","last_undegraded":"2023-09-23T23:35:15.423066+0000","last_fullsized":"2023-09-23T23:35:15.423066+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"586'6235","reported_seq":6304,"reported_epoch":586,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:23.936068+0000","last_change":"2023-09-23T23:33:52.884769+0000","last_active":"2023-09-23T23:35:23.936068+0000","last_peered":"2023-09-23T23:35:23.936068+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:35:23.936068+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"542'800","ondisk_log_start":"542'800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5435,"log_dups_size":0,"ondisk_log_size":5435,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302317568,"num_objects":4613,"num_object_clones":0,"num_object_copies":13839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6235,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":3587},{"shards":"2,6","objects":215}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":919,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.366296+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:35:15.366296+0000","last_peered":"2023-09-23T23:35:15.366296+0000","last_clean":"2023-09-23T23:35:15.366296+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:35:15.366296+0000","last_undegraded":"2023-09-23T23:35:15.366296+0000","last_fullsized":"2023-09-23T23:35:15.366296+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":227,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.426772+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:35:15.426772+0000","last_peered":"2023-09-23T23:35:15.426772+0000","last_clean":"2023-09-23T23:35:15.426772+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:35:15.426772+0000","last_undegraded":"2023-09-23T23:35:15.426772+0000","last_fullsized":"2023-09-23T23:35:15.426772+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":217,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.412968+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:35:15.412968+0000","last_peered":"2023-09-23T23:35:15.412968+0000","last_clean":"2023-09-23T23:35:15.412968+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:35:15.412968+0000","last_undegraded":"2023-09-23T23:35:15.412968+0000","last_fullsized":"2023-09-23T23:35:15.412968+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"586'6325","reported_seq":6393,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:24.560123+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:35:24.560123+0000","last_peered":"2023-09-23T23:35:24.560123+0000","last_clean":"2023-09-23T23:35:24.560123+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:35:24.560123+0000","last_undegraded":"2023-09-23T23:35:24.560123+0000","last_fullsized":"2023-09-23T23:35:24.560123+0000","mapping_epoch":566,"log_start":"538'500","ondisk_log_start":"538'500","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5825,"log_dups_size":0,"ondisk_log_size":5825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308477952,"num_objects":4707,"num_object_clones":0,"num_object_copies":14121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6325,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":997,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417817+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:35:15.417817+0000","last_peered":"2023-09-23T23:35:15.417817+0000","last_clean":"2023-09-23T23:35:15.417817+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:35:15.417817+0000","last_undegraded":"2023-09-23T23:35:15.417817+0000","last_fullsized":"2023-09-23T23:35:15.417817+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":217,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.426712+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:35:15.426712+0000","last_peered":"2023-09-23T23:35:15.426712+0000","last_clean":"2023-09-23T23:35:15.426712+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:35:15.426712+0000","last_undegraded":"2023-09-23T23:35:15.426712+0000","last_fullsized":"2023-09-23T23:35:15.426712+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":436,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417629+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:35:15.417629+0000","last_peered":"2023-09-23T23:35:15.417629+0000","last_clean":"2023-09-23T23:35:15.417629+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:35:15.417629+0000","last_undegraded":"2023-09-23T23:35:15.417629+0000","last_fullsized":"2023-09-23T23:35:15.417629+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"586'6348","reported_seq":9099,"reported_epoch":586,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:35:21.695203+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:35:21.695203+0000","last_peered":"2023-09-23T23:35:21.695203+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:35:21.695203+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:35:21.695203+0000","mapping_epoch":571,"log_start":"543'1000","ondisk_log_start":"543'1000","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5348,"log_dups_size":0,"ondisk_log_size":5348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308019200,"num_objects":4700,"num_object_clones":0,"num_object_copies":14100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3805,"num_objects_misplaced":895,"num_objects_unfound":0,"num_objects_dirty":4700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6348,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1326,"num_bytes_recovered":86900736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":3683},{"shards":"3,5,7","objects":1017}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":896,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.506178+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:35:15.506178+0000","last_peered":"2023-09-23T23:35:15.506178+0000","last_clean":"2023-09-23T23:35:15.506178+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:35:15.506178+0000","last_undegraded":"2023-09-23T23:35:15.506178+0000","last_fullsized":"2023-09-23T23:35:15.506178+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":324,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.413538+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:35:15.413538+0000","last_peered":"2023-09-23T23:35:15.413538+0000","last_clean":"2023-09-23T23:35:15.413538+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:35:15.413538+0000","last_undegraded":"2023-09-23T23:35:15.413538+0000","last_fullsized":"2023-09-23T23:35:15.413538+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":100,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505879+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:35:15.505879+0000","last_peered":"2023-09-23T23:35:15.505879+0000","last_clean":"2023-09-23T23:35:15.505879+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:35:15.505879+0000","last_undegraded":"2023-09-23T23:35:15.505879+0000","last_fullsized":"2023-09-23T23:35:15.505879+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"586'6288","reported_seq":8343,"reported_epoch":586,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:21.714011+0000","last_change":"2023-09-23T23:34:35.750428+0000","last_active":"2023-09-23T23:35:21.714011+0000","last_peered":"2023-09-23T23:35:21.714011+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:35:21.714011+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"544'1000","ondisk_log_start":"544'1000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5288,"log_dups_size":0,"ondisk_log_size":5288,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314703872,"num_objects":4802,"num_object_clones":0,"num_object_copies":14406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6288,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":991,"num_bytes_recovered":64946176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":3544},{"shards":"3,5","objects":1258}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":492,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.423194+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:35:15.423194+0000","last_peered":"2023-09-23T23:35:15.423194+0000","last_clean":"2023-09-23T23:35:15.423194+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:35:15.423194+0000","last_undegraded":"2023-09-23T23:35:15.423194+0000","last_fullsized":"2023-09-23T23:35:15.423194+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":426,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.422758+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:35:15.422758+0000","last_peered":"2023-09-23T23:35:15.422758+0000","last_clean":"2023-09-23T23:35:15.422758+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:35:15.422758+0000","last_undegraded":"2023-09-23T23:35:15.422758+0000","last_fullsized":"2023-09-23T23:35:15.422758+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":581,"reported_epoch":584,"state":"active+clean","last_fresh":"2023-09-23T23:35:12.391039+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:35:12.391039+0000","last_peered":"2023-09-23T23:35:12.391039+0000","last_clean":"2023-09-23T23:35:12.391039+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:35:12.391039+0000","last_undegraded":"2023-09-23T23:35:12.391039+0000","last_fullsized":"2023-09-23T23:35:12.391039+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"586'6407","reported_seq":7723,"reported_epoch":586,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:23.905880+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:35:23.905880+0000","last_peered":"2023-09-23T23:35:23.905880+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:35:23.905880+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"542'700","ondisk_log_start":"542'700","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5707,"log_dups_size":0,"ondisk_log_size":5707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311623680,"num_objects":4755,"num_object_clones":0,"num_object_copies":14265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6407,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":3709},{"shards":"3,5","objects":-705},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":713,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417787+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:35:15.417787+0000","last_peered":"2023-09-23T23:35:15.417787+0000","last_clean":"2023-09-23T23:35:15.417787+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:35:15.417787+0000","last_undegraded":"2023-09-23T23:35:15.417787+0000","last_fullsized":"2023-09-23T23:35:15.417787+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":595,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.418673+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:35:15.418673+0000","last_peered":"2023-09-23T23:35:15.418673+0000","last_clean":"2023-09-23T23:35:15.418673+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:35:15.418673+0000","last_undegraded":"2023-09-23T23:35:15.418673+0000","last_fullsized":"2023-09-23T23:35:15.418673+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":506,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.422667+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:35:15.422667+0000","last_peered":"2023-09-23T23:35:15.422667+0000","last_clean":"2023-09-23T23:35:15.422667+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:35:15.422667+0000","last_undegraded":"2023-09-23T23:35:15.422667+0000","last_fullsized":"2023-09-23T23:35:15.422667+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"586'6327","reported_seq":6387,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:23.940821+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:35:23.940821+0000","last_peered":"2023-09-23T23:35:23.940821+0000","last_clean":"2023-09-23T23:35:23.940821+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:35:23.940821+0000","last_undegraded":"2023-09-23T23:35:23.940821+0000","last_fullsized":"2023-09-23T23:35:23.940821+0000","mapping_epoch":534,"log_start":"543'900","ondisk_log_start":"543'900","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5427,"log_dups_size":0,"ondisk_log_size":5427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311492608,"num_objects":4753,"num_object_clones":0,"num_object_copies":14259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6327,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":868,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.365959+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:35:15.365959+0000","last_peered":"2023-09-23T23:35:15.365959+0000","last_clean":"2023-09-23T23:35:15.365959+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:35:15.365959+0000","last_undegraded":"2023-09-23T23:35:15.365959+0000","last_fullsized":"2023-09-23T23:35:15.365959+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":110,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505797+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:35:15.505797+0000","last_peered":"2023-09-23T23:35:15.505797+0000","last_clean":"2023-09-23T23:35:15.505797+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:35:15.505797+0000","last_undegraded":"2023-09-23T23:35:15.505797+0000","last_fullsized":"2023-09-23T23:35:15.505797+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":451,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.366401+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:35:15.366401+0000","last_peered":"2023-09-23T23:35:15.366401+0000","last_clean":"2023-09-23T23:35:15.366401+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:35:15.366401+0000","last_undegraded":"2023-09-23T23:35:15.366401+0000","last_fullsized":"2023-09-23T23:35:15.366401+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"586'6405","reported_seq":8193,"reported_epoch":586,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:35:24.135951+0000","last_change":"2023-09-23T23:34:35.785270+0000","last_active":"2023-09-23T23:35:24.135951+0000","last_peered":"2023-09-23T23:35:24.135951+0000","last_clean":"2023-09-23T23:33:50.504296+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:35:24.135951+0000","last_undegraded":"2023-09-23T23:34:35.780981+0000","last_fullsized":"2023-09-23T23:35:24.135951+0000","mapping_epoch":567,"log_start":"543'1000","ondisk_log_start":"543'1000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5405,"log_dups_size":0,"ondisk_log_size":5405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307953664,"num_objects":4699,"num_object_clones":0,"num_object_copies":14097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3693,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6405,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":854,"num_bytes_recovered":55967744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":3693},{"shards":"5,6,7","objects":1006}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":594,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.413257+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:35:15.413257+0000","last_peered":"2023-09-23T23:35:15.413257+0000","last_clean":"2023-09-23T23:35:15.413257+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:35:15.413257+0000","last_undegraded":"2023-09-23T23:35:15.413257+0000","last_fullsized":"2023-09-23T23:35:15.413257+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":448,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.366748+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:35:15.366748+0000","last_peered":"2023-09-23T23:35:15.366748+0000","last_clean":"2023-09-23T23:35:15.366748+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:35:15.366748+0000","last_undegraded":"2023-09-23T23:35:15.366748+0000","last_fullsized":"2023-09-23T23:35:15.366748+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":537,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505831+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:35:15.505831+0000","last_peered":"2023-09-23T23:35:15.505831+0000","last_clean":"2023-09-23T23:35:15.505831+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:35:15.505831+0000","last_undegraded":"2023-09-23T23:35:15.505831+0000","last_fullsized":"2023-09-23T23:35:15.505831+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"586'6350","reported_seq":15373,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:23.913283+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:35:23.913283+0000","last_peered":"2023-09-23T23:35:23.913283+0000","last_clean":"2023-09-23T23:35:23.913283+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:35:23.913283+0000","last_undegraded":"2023-09-23T23:35:23.913283+0000","last_fullsized":"2023-09-23T23:35:23.913283+0000","mapping_epoch":579,"log_start":"541'600","ondisk_log_start":"541'600","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5750,"log_dups_size":0,"ondisk_log_size":5750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305790976,"num_objects":4666,"num_object_clones":0,"num_object_copies":13998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6350,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":927,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.367765+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:35:15.367765+0000","last_peered":"2023-09-23T23:35:15.367765+0000","last_clean":"2023-09-23T23:35:15.367765+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:35:15.367765+0000","last_undegraded":"2023-09-23T23:35:15.367765+0000","last_fullsized":"2023-09-23T23:35:15.367765+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":227,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417625+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:35:15.417625+0000","last_peered":"2023-09-23T23:35:15.417625+0000","last_clean":"2023-09-23T23:35:15.417625+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:35:15.417625+0000","last_undegraded":"2023-09-23T23:35:15.417625+0000","last_fullsized":"2023-09-23T23:35:15.417625+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":515,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.423142+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:35:15.423142+0000","last_peered":"2023-09-23T23:35:15.423142+0000","last_clean":"2023-09-23T23:35:15.423142+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:35:15.423142+0000","last_undegraded":"2023-09-23T23:35:15.423142+0000","last_fullsized":"2023-09-23T23:35:15.423142+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"586'6099","reported_seq":6161,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.450389+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:35:21.450389+0000","last_peered":"2023-09-23T23:35:21.450389+0000","last_clean":"2023-09-23T23:35:21.450389+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:35:21.450389+0000","last_undegraded":"2023-09-23T23:35:21.450389+0000","last_fullsized":"2023-09-23T23:35:21.450389+0000","mapping_epoch":534,"log_start":"543'800","ondisk_log_start":"543'800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5299,"log_dups_size":0,"ondisk_log_size":5299,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303366144,"num_objects":4629,"num_object_clones":0,"num_object_copies":13887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4629,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6099,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":907,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.506026+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:35:15.506026+0000","last_peered":"2023-09-23T23:35:15.506026+0000","last_clean":"2023-09-23T23:35:15.506026+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:35:15.506026+0000","last_undegraded":"2023-09-23T23:35:15.506026+0000","last_fullsized":"2023-09-23T23:35:15.506026+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":295,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.441550+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:35:15.441550+0000","last_peered":"2023-09-23T23:35:15.441550+0000","last_clean":"2023-09-23T23:35:15.441550+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:35:15.441550+0000","last_undegraded":"2023-09-23T23:35:15.441550+0000","last_fullsized":"2023-09-23T23:35:15.441550+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":601,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.413693+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:35:15.413693+0000","last_peered":"2023-09-23T23:35:15.413693+0000","last_clean":"2023-09-23T23:35:15.413693+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:35:15.413693+0000","last_undegraded":"2023-09-23T23:35:15.413693+0000","last_fullsized":"2023-09-23T23:35:15.413693+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":182,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.413060+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:35:15.413060+0000","last_peered":"2023-09-23T23:35:15.413060+0000","last_clean":"2023-09-23T23:35:15.413060+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:35:15.413060+0000","last_undegraded":"2023-09-23T23:35:15.413060+0000","last_fullsized":"2023-09-23T23:35:15.413060+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"586'6394","reported_seq":8109,"reported_epoch":586,"state":"active+recovering","last_fresh":"2023-09-23T23:35:21.731256+0000","last_change":"2023-09-23T23:34:35.815566+0000","last_active":"2023-09-23T23:35:21.731256+0000","last_peered":"2023-09-23T23:35:21.731256+0000","last_clean":"2023-09-23T23:33:50.519308+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:35:21.731256+0000","last_undegraded":"2023-09-23T23:35:21.731256+0000","last_fullsized":"2023-09-23T23:35:21.731256+0000","mapping_epoch":566,"log_start":"544'1122","ondisk_log_start":"544'1122","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5272,"log_dups_size":0,"ondisk_log_size":5272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312803352,"num_objects":4774,"num_object_clones":0,"num_object_copies":14322,"num_objects_missing_on_primary":3692,"num_objects_missing":3692,"num_objects_degraded":0,"num_objects_misplaced":3692,"num_objects_unfound":0,"num_objects_dirty":4774,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6394,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":810,"num_bytes_recovered":53084160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,3,4,7","objects":13},{"shards":"1,3,7","objects":1069},{"shards":"1,4,7","objects":3692}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":668,"reported_epoch":584,"state":"active+clean","last_fresh":"2023-09-23T23:35:12.390603+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:35:12.390603+0000","last_peered":"2023-09-23T23:35:12.390603+0000","last_clean":"2023-09-23T23:35:12.390603+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:35:12.390603+0000","last_undegraded":"2023-09-23T23:35:12.390603+0000","last_fullsized":"2023-09-23T23:35:12.390603+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":234,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505815+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:35:15.505815+0000","last_peered":"2023-09-23T23:35:15.505815+0000","last_clean":"2023-09-23T23:35:15.505815+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:35:15.505815+0000","last_undegraded":"2023-09-23T23:35:15.505815+0000","last_fullsized":"2023-09-23T23:35:15.505815+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":518,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505588+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:35:15.505588+0000","last_peered":"2023-09-23T23:35:15.505588+0000","last_clean":"2023-09-23T23:35:15.505588+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:35:15.505588+0000","last_undegraded":"2023-09-23T23:35:15.505588+0000","last_fullsized":"2023-09-23T23:35:15.505588+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":218,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.367642+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:35:15.367642+0000","last_peered":"2023-09-23T23:35:15.367642+0000","last_clean":"2023-09-23T23:35:15.367642+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:35:15.367642+0000","last_undegraded":"2023-09-23T23:35:15.367642+0000","last_fullsized":"2023-09-23T23:35:15.367642+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"586'6132","reported_seq":8171,"reported_epoch":586,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:23.955002+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:35:23.955002+0000","last_peered":"2023-09-23T23:35:23.955002+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:35:23.955002+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"538'400","ondisk_log_start":"538'400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5732,"log_dups_size":0,"ondisk_log_size":5732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304087040,"num_objects":4640,"num_object_clones":0,"num_object_copies":13920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6132,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":3392},{"shards":"5,7","objects":1248}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":887,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.367674+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:35:15.367674+0000","last_peered":"2023-09-23T23:35:15.367674+0000","last_clean":"2023-09-23T23:35:15.367674+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:35:15.367674+0000","last_undegraded":"2023-09-23T23:35:15.367674+0000","last_fullsized":"2023-09-23T23:35:15.367674+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":324,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.366677+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:35:15.366677+0000","last_peered":"2023-09-23T23:35:15.366677+0000","last_clean":"2023-09-23T23:35:15.366677+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:35:15.366677+0000","last_undegraded":"2023-09-23T23:35:15.366677+0000","last_fullsized":"2023-09-23T23:35:15.366677+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":233,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.366039+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:35:15.366039+0000","last_peered":"2023-09-23T23:35:15.366039+0000","last_clean":"2023-09-23T23:35:15.366039+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:35:15.366039+0000","last_undegraded":"2023-09-23T23:35:15.366039+0000","last_fullsized":"2023-09-23T23:35:15.366039+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":197,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.440318+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:35:15.440318+0000","last_peered":"2023-09-23T23:35:15.440318+0000","last_clean":"2023-09-23T23:35:15.440318+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:35:15.440318+0000","last_undegraded":"2023-09-23T23:35:15.440318+0000","last_fullsized":"2023-09-23T23:35:15.440318+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"586'6258","reported_seq":6328,"reported_epoch":586,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:35:24.129834+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:35:24.129834+0000","last_peered":"2023-09-23T23:35:24.129834+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:35:24.129834+0000","last_undegraded":"2023-09-23T23:35:24.129834+0000","last_fullsized":"2023-09-23T23:35:24.129834+0000","mapping_epoch":562,"log_start":"538'500","ondisk_log_start":"538'500","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5758,"log_dups_size":0,"ondisk_log_size":5758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304349184,"num_objects":4644,"num_object_clones":0,"num_object_copies":13932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6258,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":4644}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":600,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.365966+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:35:15.365966+0000","last_peered":"2023-09-23T23:35:15.365966+0000","last_clean":"2023-09-23T23:35:15.365966+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:35:15.365966+0000","last_undegraded":"2023-09-23T23:35:15.365966+0000","last_fullsized":"2023-09-23T23:35:15.365966+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":872,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505324+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:35:15.505324+0000","last_peered":"2023-09-23T23:35:15.505324+0000","last_clean":"2023-09-23T23:35:15.505324+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:35:15.505324+0000","last_undegraded":"2023-09-23T23:35:15.505324+0000","last_fullsized":"2023-09-23T23:35:15.505324+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":547,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417831+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:35:15.417831+0000","last_peered":"2023-09-23T23:35:15.417831+0000","last_clean":"2023-09-23T23:35:15.417831+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:35:15.417831+0000","last_undegraded":"2023-09-23T23:35:15.417831+0000","last_fullsized":"2023-09-23T23:35:15.417831+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":110,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505825+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:35:15.505825+0000","last_peered":"2023-09-23T23:35:15.505825+0000","last_clean":"2023-09-23T23:35:15.505825+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:35:15.505825+0000","last_undegraded":"2023-09-23T23:35:15.505825+0000","last_fullsized":"2023-09-23T23:35:15.505825+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"586'6160","reported_seq":6220,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:23.840271+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:35:23.840271+0000","last_peered":"2023-09-23T23:35:23.840271+0000","last_clean":"2023-09-23T23:35:23.840271+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:35:23.840271+0000","last_undegraded":"2023-09-23T23:35:23.840271+0000","last_fullsized":"2023-09-23T23:35:23.840271+0000","mapping_epoch":534,"log_start":"543'800","ondisk_log_start":"543'800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5360,"log_dups_size":0,"ondisk_log_size":5360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299892736,"num_objects":4576,"num_object_clones":0,"num_object_copies":13728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6160,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":698,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505486+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:35:15.505486+0000","last_peered":"2023-09-23T23:35:15.505486+0000","last_clean":"2023-09-23T23:35:15.505486+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:35:15.505486+0000","last_undegraded":"2023-09-23T23:35:15.505486+0000","last_fullsized":"2023-09-23T23:35:15.505486+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":513,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.426159+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:35:15.426159+0000","last_peered":"2023-09-23T23:35:15.426159+0000","last_clean":"2023-09-23T23:35:15.426159+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:35:15.426159+0000","last_undegraded":"2023-09-23T23:35:15.426159+0000","last_fullsized":"2023-09-23T23:35:15.426159+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":426,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417657+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:35:15.417657+0000","last_peered":"2023-09-23T23:35:15.417657+0000","last_clean":"2023-09-23T23:35:15.417657+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:35:15.417657+0000","last_undegraded":"2023-09-23T23:35:15.417657+0000","last_fullsized":"2023-09-23T23:35:15.417657+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":25,"reported_epoch":584,"state":"active+clean","last_fresh":"2023-09-23T23:35:12.391325+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:35:12.391325+0000","last_peered":"2023-09-23T23:35:12.391325+0000","last_clean":"2023-09-23T23:35:12.391325+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:35:12.391325+0000","last_undegraded":"2023-09-23T23:35:12.391325+0000","last_fullsized":"2023-09-23T23:35:12.391325+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"586'6297","reported_seq":9956,"reported_epoch":586,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:24.585317+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:35:24.585317+0000","last_peered":"2023-09-23T23:35:24.585317+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:35:24.585317+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"538'100","ondisk_log_start":"538'100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6197,"log_dups_size":0,"ondisk_log_size":6197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308084736,"num_objects":4701,"num_object_clones":0,"num_object_copies":14103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6297,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":1997},{"shards":"1,2,4","objects":2704}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":581,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505968+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:35:15.505968+0000","last_peered":"2023-09-23T23:35:15.505968+0000","last_clean":"2023-09-23T23:35:15.505968+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:35:15.505968+0000","last_undegraded":"2023-09-23T23:35:15.505968+0000","last_fullsized":"2023-09-23T23:35:15.505968+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":25,"reported_epoch":584,"state":"active+clean","last_fresh":"2023-09-23T23:35:12.390920+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:35:12.390920+0000","last_peered":"2023-09-23T23:35:12.390920+0000","last_clean":"2023-09-23T23:35:12.390920+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:35:12.390920+0000","last_undegraded":"2023-09-23T23:35:12.390920+0000","last_fullsized":"2023-09-23T23:35:12.390920+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":125,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.367665+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:35:15.367665+0000","last_peered":"2023-09-23T23:35:15.367665+0000","last_clean":"2023-09-23T23:35:15.367665+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:35:15.367665+0000","last_undegraded":"2023-09-23T23:35:15.367665+0000","last_fullsized":"2023-09-23T23:35:15.367665+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":319,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.413368+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:35:15.413368+0000","last_peered":"2023-09-23T23:35:15.413368+0000","last_clean":"2023-09-23T23:35:15.413368+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:35:15.413368+0000","last_undegraded":"2023-09-23T23:35:15.413368+0000","last_fullsized":"2023-09-23T23:35:15.413368+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"586'6312","reported_seq":6372,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:23.991730+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:35:23.991730+0000","last_peered":"2023-09-23T23:35:23.991730+0000","last_clean":"2023-09-23T23:35:23.991730+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:35:23.991730+0000","last_undegraded":"2023-09-23T23:35:23.991730+0000","last_fullsized":"2023-09-23T23:35:23.991730+0000","mapping_epoch":534,"log_start":"541'600","ondisk_log_start":"541'600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5712,"log_dups_size":0,"ondisk_log_size":5712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304873472,"num_objects":4652,"num_object_clones":0,"num_object_copies":13956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6312,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":766,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.366457+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:35:15.366457+0000","last_peered":"2023-09-23T23:35:15.366457+0000","last_clean":"2023-09-23T23:35:15.366457+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:35:15.366457+0000","last_undegraded":"2023-09-23T23:35:15.366457+0000","last_fullsized":"2023-09-23T23:35:15.366457+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":583,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.423118+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:35:15.423118+0000","last_peered":"2023-09-23T23:35:15.423118+0000","last_clean":"2023-09-23T23:35:15.423118+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:35:15.423118+0000","last_undegraded":"2023-09-23T23:35:15.423118+0000","last_fullsized":"2023-09-23T23:35:15.423118+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":190,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505878+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:35:15.505878+0000","last_peered":"2023-09-23T23:35:15.505878+0000","last_clean":"2023-09-23T23:35:15.505878+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:35:15.505878+0000","last_undegraded":"2023-09-23T23:35:15.505878+0000","last_fullsized":"2023-09-23T23:35:15.505878+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":137,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.365822+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:35:15.365822+0000","last_peered":"2023-09-23T23:35:15.365822+0000","last_clean":"2023-09-23T23:35:15.365822+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:35:15.365822+0000","last_undegraded":"2023-09-23T23:35:15.365822+0000","last_fullsized":"2023-09-23T23:35:15.365822+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"586'6242","reported_seq":6304,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:24.850185+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:35:24.850185+0000","last_peered":"2023-09-23T23:35:24.850185+0000","last_clean":"2023-09-23T23:35:24.850185+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:35:24.850185+0000","last_undegraded":"2023-09-23T23:35:24.850185+0000","last_fullsized":"2023-09-23T23:35:24.850185+0000","mapping_epoch":534,"log_start":"538'100","ondisk_log_start":"538'100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6142,"log_dups_size":0,"ondisk_log_size":6142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305790976,"num_objects":4666,"num_object_clones":0,"num_object_copies":13998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6242,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":972,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.422901+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:35:15.422901+0000","last_peered":"2023-09-23T23:35:15.422901+0000","last_clean":"2023-09-23T23:35:15.422901+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:35:15.422901+0000","last_undegraded":"2023-09-23T23:35:15.422901+0000","last_fullsized":"2023-09-23T23:35:15.422901+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":100,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.506079+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:35:15.506079+0000","last_peered":"2023-09-23T23:35:15.506079+0000","last_clean":"2023-09-23T23:35:15.506079+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:35:15.506079+0000","last_undegraded":"2023-09-23T23:35:15.506079+0000","last_fullsized":"2023-09-23T23:35:15.506079+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":29,"reported_epoch":584,"state":"active+clean","last_fresh":"2023-09-23T23:35:12.391381+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:35:12.391381+0000","last_peered":"2023-09-23T23:35:12.391381+0000","last_clean":"2023-09-23T23:35:12.391381+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:35:12.391381+0000","last_undegraded":"2023-09-23T23:35:12.391381+0000","last_fullsized":"2023-09-23T23:35:12.391381+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":160,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.413045+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:35:15.413045+0000","last_peered":"2023-09-23T23:35:15.413045+0000","last_clean":"2023-09-23T23:35:15.413045+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:35:15.413045+0000","last_undegraded":"2023-09-23T23:35:15.413045+0000","last_fullsized":"2023-09-23T23:35:15.413045+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1022,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417741+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:35:15.417741+0000","last_peered":"2023-09-23T23:35:15.417741+0000","last_clean":"2023-09-23T23:35:15.417741+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:35:15.417741+0000","last_undegraded":"2023-09-23T23:35:15.417741+0000","last_fullsized":"2023-09-23T23:35:15.417741+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":25,"reported_epoch":584,"state":"active+clean","last_fresh":"2023-09-23T23:35:12.391084+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:35:12.391084+0000","last_peered":"2023-09-23T23:35:12.391084+0000","last_clean":"2023-09-23T23:35:12.391084+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:35:12.391084+0000","last_undegraded":"2023-09-23T23:35:12.391084+0000","last_fullsized":"2023-09-23T23:35:12.391084+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":341,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:19.909163+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:35:19.909163+0000","last_peered":"2023-09-23T23:35:19.909163+0000","last_clean":"2023-09-23T23:35:19.909163+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:35:19.909163+0000","last_undegraded":"2023-09-23T23:35:19.909163+0000","last_fullsized":"2023-09-23T23:35:19.909163+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":733,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.423334+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:35:15.423334+0000","last_peered":"2023-09-23T23:35:15.423334+0000","last_clean":"2023-09-23T23:35:15.423334+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:35:15.423334+0000","last_undegraded":"2023-09-23T23:35:15.423334+0000","last_fullsized":"2023-09-23T23:35:15.423334+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":88,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505506+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:35:15.505506+0000","last_peered":"2023-09-23T23:35:15.505506+0000","last_clean":"2023-09-23T23:35:15.505506+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:35:15.505506+0000","last_undegraded":"2023-09-23T23:35:15.505506+0000","last_fullsized":"2023-09-23T23:35:15.505506+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":433,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.418612+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:35:15.418612+0000","last_peered":"2023-09-23T23:35:15.418612+0000","last_clean":"2023-09-23T23:35:15.418612+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:35:15.418612+0000","last_undegraded":"2023-09-23T23:35:15.418612+0000","last_fullsized":"2023-09-23T23:35:15.418612+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":284,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505747+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:35:15.505747+0000","last_peered":"2023-09-23T23:35:15.505747+0000","last_clean":"2023-09-23T23:35:15.505747+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:35:15.505747+0000","last_undegraded":"2023-09-23T23:35:15.505747+0000","last_fullsized":"2023-09-23T23:35:15.505747+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":272,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417915+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:35:15.417915+0000","last_peered":"2023-09-23T23:35:15.417915+0000","last_clean":"2023-09-23T23:35:15.417915+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:35:15.417915+0000","last_undegraded":"2023-09-23T23:35:15.417915+0000","last_fullsized":"2023-09-23T23:35:15.417915+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":798,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505623+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:35:15.505623+0000","last_peered":"2023-09-23T23:35:15.505623+0000","last_clean":"2023-09-23T23:35:15.505623+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:35:15.505623+0000","last_undegraded":"2023-09-23T23:35:15.505623+0000","last_fullsized":"2023-09-23T23:35:15.505623+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":616,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.423287+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:35:15.423287+0000","last_peered":"2023-09-23T23:35:15.423287+0000","last_clean":"2023-09-23T23:35:15.423287+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:35:15.423287+0000","last_undegraded":"2023-09-23T23:35:15.423287+0000","last_fullsized":"2023-09-23T23:35:15.423287+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":600,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417897+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:35:15.417897+0000","last_peered":"2023-09-23T23:35:15.417897+0000","last_clean":"2023-09-23T23:35:15.417897+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:35:15.417897+0000","last_undegraded":"2023-09-23T23:35:15.417897+0000","last_fullsized":"2023-09-23T23:35:15.417897+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":758,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.432319+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:35:15.432319+0000","last_peered":"2023-09-23T23:35:15.432319+0000","last_clean":"2023-09-23T23:35:15.432319+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:35:15.432319+0000","last_undegraded":"2023-09-23T23:35:15.432319+0000","last_fullsized":"2023-09-23T23:35:15.432319+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":429,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.367512+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:35:15.367512+0000","last_peered":"2023-09-23T23:35:15.367512+0000","last_clean":"2023-09-23T23:35:15.367512+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:35:15.367512+0000","last_undegraded":"2023-09-23T23:35:15.367512+0000","last_fullsized":"2023-09-23T23:35:15.367512+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":653,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.505229+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:35:15.505229+0000","last_peered":"2023-09-23T23:35:15.505229+0000","last_clean":"2023-09-23T23:35:15.505229+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:35:15.505229+0000","last_undegraded":"2023-09-23T23:35:15.505229+0000","last_fullsized":"2023-09-23T23:35:15.505229+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":4913627160,"num_objects":74977,"num_object_clones":0,"num_object_copies":224931,"num_objects_missing_on_primary":3692,"num_objects_missing":3692,"num_objects_degraded":27114,"num_objects_misplaced":4587,"num_objects_unfound":0,"num_objects_dirty":74977,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":100579,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11838,"num_bytes_recovered":775618560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15495213056,"data_stored":15495200840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89657,"ondisk_log_size":89657,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782671,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2126472,"kb_used_data":1768448,"kb_used_omap":8,"kb_used_meta":358007,"kb_avail":91618680,"statfs":{"total":95995035648,"available":93817528320,"internally_reserved":0,"allocated":1810890752,"data_stored":1809271132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":366599237},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.9339999999999999,"5min":2.3500000000000001,"15min":2.2549999999999999},"min":{"1min":0.26600000000000001,"5min":0.224,"15min":0.20899999999999999},"max":{"1min":11.173,"5min":44.094999999999999,"15min":85.721000000000004},"last":0.23100000000000001},{"interface":"front","average":{"1min":1.1519999999999999,"5min":2.2080000000000002,"15min":2.2669999999999999},"min":{"1min":0.28199999999999997,"5min":0.25600000000000001,"15min":0.19800000000000001},"max":{"1min":4.3109999999999999,"5min":44.335999999999999,"15min":85.965000000000003},"last":0.29199999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8819999999999999,"5min":6.5810000000000004,"15min":3.202},"min":{"1min":0.252,"5min":0.248,"15min":0.22900000000000001},"max":{"1min":9.0139999999999993,"5min":198.58000000000001,"15min":198.58000000000001},"last":0.26600000000000001},{"interface":"front","average":{"1min":1.179,"5min":6.3109999999999999,"15min":3.2610000000000001},"min":{"1min":0.254,"5min":0.247,"15min":0.246},"max":{"1min":4.1660000000000004,"5min":198.71100000000001,"15min":198.71100000000001},"last":0.39600000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":6.0019999999999998,"5min":5.7329999999999997,"15min":3.7810000000000001},"min":{"1min":0.26900000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":78.528999999999996,"5min":203.58000000000001,"15min":203.58000000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.8799999999999999,"5min":4.5350000000000001,"15min":3.1080000000000001},"min":{"1min":0.27900000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.5670000000000002,"5min":198.69,"15min":198.69},"last":0.46600000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":7.0250000000000004,"15min":3.6579999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":6.1210000000000004,"5min":279.26499999999999,"15min":279.26499999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":3.1579999999999999,"5min":8.7859999999999996,"15min":4.1630000000000003},"min":{"1min":0.40200000000000002,"5min":0.32000000000000001,"15min":0.29599999999999999},"max":{"1min":18.643999999999998,"5min":279.29899999999998,"15min":279.29899999999998},"last":0.38200000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":1.857,"15min":1.0940000000000001},"min":{"1min":0.35299999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":10.6,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.79600000000000004,"5min":1.4970000000000001,"15min":1.03},"min":{"1min":0.317,"5min":0.28100000000000003,"15min":0.23899999999999999},"max":{"1min":2.8570000000000002,"5min":31.68,"15min":31.68},"last":0.33300000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":1.1659999999999999,"15min":1.7290000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26500000000000001},"max":{"1min":12.273999999999999,"5min":12.273999999999999,"15min":210.83600000000001},"last":0.372},{"interface":"front","average":{"1min":2.141,"5min":1.0529999999999999,"15min":1.579},"min":{"1min":0.371,"5min":0.316,"15min":0.28199999999999997},"max":{"1min":18.117999999999999,"5min":18.117999999999999,"15min":207.04599999999999},"last":0.35799999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.96,"15min":1.3620000000000001},"min":{"1min":0.379,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.6440000000000001,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.42599999999999999},{"interface":"front","average":{"1min":2.2109999999999999,"5min":2.403,"15min":1.615},"min":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.33300000000000002},"max":{"1min":10.332000000000001,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.315}]}]},{"osd":6,"up_from":46,"seq":197568496148,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2337912,"kb_used_data":1645552,"kb_used_omap":0,"kb_used_meta":692352,"kb_avail":91407240,"statfs":{"total":95995035648,"available":93601013760,"internally_reserved":0,"allocated":1685045248,"data_stored":1683441621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":708968448},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:14 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.98799999999999999,"15min":4.7699999999999996},"min":{"1min":0.26500000000000001,"5min":0.223,"15min":0.20399999999999999},"max":{"1min":2.274,"5min":8.8119999999999994,"15min":135.31999999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.93000000000000005,"15min":4.617},"min":{"1min":0.26800000000000002,"5min":0.23999999999999999,"15min":0.22900000000000001},"max":{"1min":3.8239999999999998,"5min":8.3230000000000004,"15min":135.34299999999999},"last":0.35599999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:35:14 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":3.3690000000000002,"15min":2.4089999999999998},"min":{"1min":0.26000000000000001,"5min":0.251,"15min":0.245},"max":{"1min":6.9080000000000004,"5min":89.605000000000004,"15min":89.605000000000004},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.71599999999999997,"5min":3.1600000000000001,"15min":2.4849999999999999},"min":{"1min":0.33500000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":1.9159999999999999,"5min":91.025999999999996,"15min":91.025999999999996},"last":0.438}]},{"osd":2,"last update":"Sat Sep 23 23:35:03 2023","interfaces":[{"interface":"back","average":{"1min":9.4120000000000008,"5min":6.4420000000000002,"15min":4.6150000000000002},"min":{"1min":0.315,"5min":0.28399999999999997,"15min":0.27800000000000002},"max":{"1min":67.796999999999997,"5min":97.408000000000001,"15min":127.44499999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":8.6270000000000007,"5min":7.6139999999999999,"15min":5.0419999999999998},"min":{"1min":0.30599999999999999,"5min":0.29999999999999999,"15min":0.26500000000000001},"max":{"1min":67.893000000000001,"5min":125.539,"15min":125.539},"last":0.32400000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:34:48 2023","interfaces":[{"interface":"back","average":{"1min":8.5120000000000005,"5min":5.7119999999999997,"15min":3.9969999999999999},"min":{"1min":0.39400000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":69.433000000000007,"5min":138.87299999999999,"15min":138.87299999999999},"last":0.316},{"interface":"front","average":{"1min":8.0280000000000005,"5min":5.5289999999999999,"15min":4.2859999999999996},"min":{"1min":0.35199999999999998,"5min":0.33200000000000002,"15min":0.28799999999999998},"max":{"1min":69.463999999999999,"5min":138.976,"15min":138.976},"last":0.38700000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:34:24 2023","interfaces":[{"interface":"back","average":{"1min":1.2969999999999999,"5min":0.92500000000000004,"15min":1.2929999999999999},"min":{"1min":0.31,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":5.6050000000000004,"5min":8.1999999999999993,"15min":166.05199999999999},"last":0.373},{"interface":"front","average":{"1min":1.901,"5min":1.0780000000000001,"15min":1.5569999999999999},"min":{"1min":0.28299999999999997,"5min":0.27900000000000003,"15min":0.27300000000000002},"max":{"1min":8.3499999999999996,"5min":8.3499999999999996,"15min":166.631},"last":0.26000000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:34:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6679999999999999,"5min":1.1220000000000001,"15min":0.85799999999999998},"min":{"1min":0.34399999999999997,"5min":0.29099999999999998,"15min":0.27900000000000003},"max":{"1min":11.372,"5min":11.473000000000001,"15min":11.473000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.9830000000000001,"5min":1.2350000000000001,"15min":1.0149999999999999},"min":{"1min":0.38100000000000001,"5min":0.311,"15min":0.249},"max":{"1min":7.0129999999999999,"5min":7.5670000000000002,"15min":8.4090000000000007},"last":0.33600000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:35:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.5389999999999999,"15min":1.0580000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":2.3010000000000002,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.573,"15min":1.081},"min":{"1min":0.36199999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.9009999999999998,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.48299999999999998}]}]},{"osd":1,"up_from":112,"seq":481036337489,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2402152,"kb_used_data":1511264,"kb_used_omap":4,"kb_used_meta":890875,"kb_avail":91343000,"statfs":{"total":95995035648,"available":93535232000,"internally_reserved":0,"allocated":1547534336,"data_stored":1544777025,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":912256910},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":2.8220000000000001,"15min":3.5670000000000002},"min":{"1min":0.21199999999999999,"5min":0.17799999999999999,"15min":0.16700000000000001},"max":{"1min":15.983000000000001,"5min":47.195999999999998,"15min":130.768},"last":0.253},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.883,"15min":2.8650000000000002},"min":{"1min":0.20699999999999999,"5min":0.20000000000000001,"15min":0.19},"max":{"1min":4.9809999999999999,"5min":46.582000000000001,"15min":134.929},"last":0.27200000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:34:57 2023","interfaces":[{"interface":"back","average":{"1min":10.555,"5min":7.9610000000000003,"15min":6.7919999999999998},"min":{"1min":0.23000000000000001,"5min":0.214,"15min":0.20100000000000001},"max":{"1min":201.03999999999999,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.28699999999999998},{"interface":"front","average":{"1min":10.605,"5min":8.173,"15min":6.9109999999999996},"min":{"1min":0.24299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":202.12,"5min":202.12,"15min":232.21000000000001},"last":1.968}]},{"osd":3,"last update":"Sat Sep 23 23:34:57 2023","interfaces":[{"interface":"back","average":{"1min":9.6969999999999992,"5min":5.7679999999999998,"15min":5.1859999999999999},"min":{"1min":0.311,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":163.94300000000001,"5min":181.94399999999999,"15min":234.404},"last":0.30599999999999999},{"interface":"front","average":{"1min":9.9290000000000003,"5min":5.6550000000000002,"15min":5.0430000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.24399999999999999},"max":{"1min":163.98699999999999,"5min":181.98500000000001,"15min":236.33699999999999},"last":0.34599999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":1.5569999999999999,"15min":1.4339999999999999},"min":{"1min":0.32700000000000001,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":2.8210000000000002,"5min":12.042999999999999,"15min":12.042999999999999},"last":2.1059999999999999},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.5609999999999999,"15min":1.498},"min":{"1min":0.34499999999999997,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":2.7810000000000001,"5min":18.108000000000001,"15min":18.108000000000001},"last":0.54800000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":3.2879999999999998,"15min":1.8260000000000001},"min":{"1min":0.307,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.0460000000000003,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.91300000000000003,"5min":3.2890000000000001,"15min":2.141},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.7450000000000001,"5min":151.821,"15min":151.821},"last":2.044}]},{"osd":6,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.6850000000000001,"15min":1.954},"min":{"1min":0.34699999999999998,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":5.0960000000000001,"5min":8.9269999999999996,"15min":74.087999999999994},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.3300000000000001,"5min":1.885,"15min":2.157},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.32800000000000001},"max":{"1min":6.8920000000000003,"5min":11.927,"15min":73.974999999999994},"last":2.0760000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:34:57 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":2.1269999999999998,"15min":2.875},"min":{"1min":0.38700000000000001,"5min":0.37,"15min":0.32900000000000001},"max":{"1min":10.629,"5min":44.241,"15min":78.308000000000007},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.165,"5min":2.218,"15min":2.706},"min":{"1min":0.42899999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.9880000000000004,"5min":43.920999999999999,"15min":77.204999999999998},"last":2.1339999999999999}]}]},{"osd":0,"up_from":91,"seq":390842024312,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1601320,"kb_used_data":391072,"kb_used_omap":5,"kb_used_meta":1210234,"kb_avail":92143832,"statfs":{"total":95995035648,"available":94355283968,"internally_reserved":0,"allocated":400457728,"data_stored":397687067,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1239280250},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:34:29 2023","interfaces":[{"interface":"back","average":{"1min":16.536000000000001,"5min":11.071999999999999,"15min":6.7530000000000001},"min":{"1min":0.188,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":301.12700000000001,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":16.061,"5min":10.94,"15min":5.8929999999999998},"min":{"1min":0.217,"5min":0.19700000000000001,"15min":0.17599999999999999},"max":{"1min":297.72399999999999,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.39300000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:34:55 2023","interfaces":[{"interface":"back","average":{"1min":19.536999999999999,"5min":10.204000000000001,"15min":7.4470000000000001},"min":{"1min":0.23300000000000001,"5min":0.17699999999999999,"15min":0.17199999999999999},"max":{"1min":283.07400000000001,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.96399999999999997},{"interface":"front","average":{"1min":18.260000000000002,"5min":10.146000000000001,"15min":7.5259999999999998},"min":{"1min":0.26800000000000002,"5min":0.19800000000000001,"15min":0.17599999999999999},"max":{"1min":283.161,"5min":283.161,"15min":283.161},"last":0.89500000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:34:29 2023","interfaces":[{"interface":"back","average":{"1min":5.2089999999999996,"5min":8.5380000000000003,"15min":5.3230000000000004},"min":{"1min":0.219,"5min":0.217,"15min":0.217},"max":{"1min":78.960999999999999,"5min":220.732,"15min":220.732},"last":0.42899999999999999},{"interface":"front","average":{"1min":4.9020000000000001,"5min":8.8420000000000005,"15min":5.4299999999999997},"min":{"1min":0.20599999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":79.185000000000002,"5min":218.78200000000001,"15min":218.78200000000001},"last":0.70399999999999996}]},{"osd":4,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":2.5859999999999999,"15min":2.8660000000000001},"min":{"1min":0.33300000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.4500000000000002,"5min":108.01900000000001,"15min":138.81299999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.25,"5min":2.7490000000000001,"15min":3.4550000000000001},"min":{"1min":0.35499999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.3670000000000009,"5min":107.898,"15min":171.251},"last":0.51100000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":1.7310000000000001,"15min":2.8340000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32100000000000001},"max":{"1min":15.076000000000001,"5min":15.076000000000001,"15min":171.30699999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.5649999999999999,"5min":1.4790000000000001,"15min":2.0880000000000001},"min":{"1min":0.35499999999999998,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":6.5279999999999996,"5min":9.9000000000000004,"15min":139.40799999999999},"last":0.72999999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":2.6360000000000001,"15min":3.161},"min":{"1min":0.38,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":9.968,"5min":108.67,"15min":171.33600000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.915,"5min":2.6699999999999999,"15min":3.4209999999999998},"min":{"1min":0.38900000000000001,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":9.157,"5min":108.804,"15min":184.13499999999999},"last":0.68999999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:34:55 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":2.8660000000000001,"15min":2.6640000000000001},"min":{"1min":0.34300000000000003,"5min":0.34000000000000002,"15min":0.32900000000000001},"max":{"1min":4.923,"5min":111.71299999999999,"15min":138.77699999999999},"last":0.495},{"interface":"front","average":{"1min":2.4950000000000001,"5min":2.9689999999999999,"15min":2.5179999999999998},"min":{"1min":0.36499999999999999,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":9.8870000000000005,"5min":111.108,"15min":139.37899999999999},"last":0.75700000000000001}]}]},{"osd":2,"up_from":199,"seq":854698492134,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3797552,"kb_used_data":2985640,"kb_used_omap":7,"kb_used_meta":811896,"kb_avail":89947600,"statfs":{"total":95995035648,"available":92106342400,"internally_reserved":0,"allocated":3057295360,"data_stored":3054474504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":831381585},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":4.726,"5min":2.6949999999999998,"15min":3.9089999999999998},"min":{"1min":0.19500000000000001,"5min":0.193,"15min":0.16700000000000001},"max":{"1min":61.950000000000003,"5min":61.950000000000003,"15min":119.277},"last":1.089},{"interface":"front","average":{"1min":5.0199999999999996,"5min":2.504,"15min":3.726},"min":{"1min":0.27800000000000002,"5min":0.20399999999999999,"15min":0.18099999999999999},"max":{"1min":61.984999999999999,"5min":61.984999999999999,"15min":119.56100000000001},"last":1.6539999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":11.630000000000001,"5min":6.2729999999999997,"15min":4.5359999999999996},"min":{"1min":0.29299999999999998,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":106.765,"5min":106.765,"15min":157.98699999999999},"last":2.6890000000000001},{"interface":"front","average":{"1min":11.114000000000001,"5min":6.3659999999999997,"15min":4.6349999999999998},"min":{"1min":0.29599999999999999,"5min":0.222,"15min":0.222},"max":{"1min":106.732,"5min":106.732,"15min":158.03800000000001},"last":2.714}]},{"osd":3,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":11.803000000000001,"5min":6.0069999999999997,"15min":4.7309999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":106.70399999999999,"5min":106.70399999999999,"15min":119.366},"last":0.39500000000000002},{"interface":"front","average":{"1min":11.427,"5min":5.891,"15min":4.3280000000000003},"min":{"1min":0.35499999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":106.66800000000001,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.40200000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":5.0979999999999999,"5min":2.4510000000000001,"15min":2.3170000000000002},"min":{"1min":0.41399999999999998,"5min":0.33700000000000002,"15min":0.25},"max":{"1min":62.081000000000003,"5min":62.081000000000003,"15min":119.88200000000001},"last":1.7310000000000001},{"interface":"front","average":{"1min":4.9169999999999998,"5min":2.5369999999999999,"15min":1.835},"min":{"1min":0.39000000000000001,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":62.072000000000003,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.37}]},{"osd":5,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":5.3090000000000002,"5min":3.391,"15min":2.7400000000000002},"min":{"1min":0.38800000000000001,"5min":0.34300000000000003,"15min":0.26200000000000001},"max":{"1min":62.058,"5min":91.207999999999998,"15min":119.919},"last":1.6970000000000001},{"interface":"front","average":{"1min":4.8929999999999998,"5min":3.2469999999999999,"15min":2.6120000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":62.042999999999999,"5min":91.150999999999996,"15min":119.306},"last":0.33500000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":12.76,"5min":4.1360000000000001,"15min":3.0720000000000001},"min":{"1min":0.41599999999999998,"5min":0.36599999999999999,"15min":0.315},"max":{"1min":107.018,"5min":107.018,"15min":119.943},"last":1.758},{"interface":"front","average":{"1min":5.7709999999999999,"5min":2.798,"15min":2.1819999999999999},"min":{"1min":0.48199999999999998,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":62.493000000000002,"5min":62.493000000000002,"15min":62.493000000000002},"last":0.42699999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":4.8719999999999999,"5min":2.8879999999999999,"15min":1.958},"min":{"1min":0.41999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":62.603000000000002,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.46000000000000002},{"interface":"front","average":{"1min":11.744999999999999,"5min":4.0620000000000003,"15min":2.4060000000000001},"min":{"1min":0.48799999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":106.809,"5min":106.809,"15min":106.809},"last":0.438}]}]},{"osd":3,"up_from":148,"seq":635655160102,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2487700,"kb_used_data":1590156,"kb_used_omap":2,"kb_used_meta":897533,"kb_avail":91257452,"statfs":{"total":95995035648,"available":93447630848,"internally_reserved":0,"allocated":1628319744,"data_stored":1625571632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":919073845},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.835,"15min":2.7349999999999999},"min":{"1min":0.215,"5min":0.19400000000000001,"15min":0.158},"max":{"1min":17.213000000000001,"5min":22.271999999999998,"15min":103.866},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.95499999999999996,"5min":2.9750000000000001,"15min":2.8250000000000002},"min":{"1min":0.20799999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":4.5469999999999997,"5min":24.683,"15min":101.884},"last":0.33700000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":4.3789999999999996,"15min":2.9239999999999999},"min":{"1min":0.23300000000000001,"5min":0.214,"15min":0.19600000000000001},"max":{"1min":3.516,"5min":85.242999999999995,"15min":85.242999999999995},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.871,"5min":3.9910000000000001,"15min":2.7250000000000001},"min":{"1min":0.25800000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.2949999999999999,"5min":85.302000000000007,"15min":85.302000000000007},"last":0.26500000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:34:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7230000000000001,"5min":5.5739999999999998,"15min":4.8579999999999997},"min":{"1min":0.25,"5min":0.25,"15min":0.19800000000000001},"max":{"1min":7.1699999999999999,"5min":73.742000000000004,"15min":101.279},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.99,"5min":5.468,"15min":4.6529999999999996},"min":{"1min":0.36699999999999999,"5min":0.25700000000000001,"15min":0.214},"max":{"1min":11.156000000000001,"5min":76.525999999999996,"15min":101.294},"last":0.29299999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.74,"5min":1.3859999999999999,"15min":1.3049999999999999},"min":{"1min":0.34999999999999998,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":10.151999999999999,"5min":10.151999999999999,"15min":10.151999999999999},"last":0.376},{"interface":"front","average":{"1min":1.464,"5min":2.3479999999999999,"15min":1.8240000000000001},"min":{"1min":0.35399999999999998,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":7.681,"5min":24.782,"15min":24.782},"last":0.48599999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":7.2480000000000002,"15min":3.4140000000000001},"min":{"1min":0.39500000000000002,"5min":0.308,"15min":0.28000000000000003},"max":{"1min":6.5010000000000003,"5min":214.43700000000001,"15min":214.43700000000001},"last":0.443},{"interface":"front","average":{"1min":2.0779999999999998,"5min":7.0030000000000001,"15min":3.488},"min":{"1min":0.376,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":9.4160000000000004,"5min":214.46600000000001,"15min":214.46600000000001},"last":0.499}]},{"osd":6,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":3.1419999999999999,"15min":2.3050000000000002},"min":{"1min":0.39000000000000001,"5min":0.35899999999999999,"15min":0.31900000000000001},"max":{"1min":5.4169999999999998,"5min":58.932000000000002,"15min":58.932000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":2.9900000000000002,"5min":2.8969999999999998,"15min":2.359},"min":{"1min":0.41799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":16.759,"5min":59.186,"15min":59.186},"last":0.433}]},{"osd":7,"last update":"Sat Sep 23 23:34:55 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":6.609,"15min":3.2200000000000002},"min":{"1min":0.42699999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":11.191000000000001,"5min":192.69,"15min":192.69},"last":0.52700000000000002},{"interface":"front","average":{"1min":2.5779999999999998,"5min":7.4489999999999998,"15min":3.7269999999999999},"min":{"1min":0.47599999999999998,"5min":0.39200000000000002,"15min":0.36699999999999999},"max":{"1min":9.7449999999999992,"5min":192.65600000000001,"15min":192.65600000000001},"last":0.51700000000000002}]}]},{"osd":4,"up_from":0,"seq":128849019421,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757784,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3488564,"kb_used_data":2765612,"kb_used_omap":0,"kb_used_meta":722944,"kb_avail":90256588,"statfs":{"total":95995035648,"available":92422746112,"internally_reserved":0,"allocated":2831986688,"data_stored":2830383278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":740294656},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.3720000000000001,"15min":3.48},"min":{"1min":0.254,"5min":0.20899999999999999,"15min":0.19500000000000001},"max":{"1min":3.0089999999999999,"5min":21.109000000000002,"15min":113.869},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.6699999999999999,"15min":3.8650000000000002},"min":{"1min":0.25,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":5.1280000000000001,"5min":20.876999999999999,"15min":138.55199999999999},"last":0.76200000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.772,"15min":2.323},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.23100000000000001},"max":{"1min":3.629,"5min":29.007999999999999,"15min":69.816999999999993},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.9390000000000001,"15min":2.665},"min":{"1min":0.29999999999999999,"5min":0.249,"15min":0.249},"max":{"1min":4.4859999999999998,"5min":29.053999999999998,"15min":70.268000000000001},"last":0.747}]},{"osd":2,"last update":"Sat Sep 23 23:34:47 2023","interfaces":[{"interface":"back","average":{"1min":8.3900000000000006,"5min":5.3979999999999997,"15min":4.2009999999999996},"min":{"1min":0.32000000000000001,"5min":0.28299999999999997,"15min":0.26900000000000002},"max":{"1min":69.75,"5min":208.738,"15min":208.738},"last":0.69599999999999995},{"interface":"front","average":{"1min":8.1739999999999995,"5min":5.3280000000000003,"15min":4.0960000000000001},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":66.421000000000006,"5min":208.631,"15min":208.631},"last":0.71499999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:34:43 2023","interfaces":[{"interface":"back","average":{"1min":8.5440000000000005,"5min":3.1419999999999999,"15min":3.3100000000000001},"min":{"1min":0.35399999999999998,"5min":0.29299999999999998,"15min":0.27400000000000002},"max":{"1min":66.591999999999999,"5min":66.591999999999999,"15min":112.71899999999999},"last":0.78000000000000003},{"interface":"front","average":{"1min":9.8930000000000007,"5min":3.5640000000000001,"15min":3.5419999999999998},"min":{"1min":0.372,"5min":0.35399999999999998,"15min":0.318},"max":{"1min":66.692999999999998,"5min":66.692999999999998,"15min":112.70099999999999},"last":0.83699999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:34:27 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":1.359,"15min":0.96799999999999997},"min":{"1min":0.33300000000000002,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":9.2690000000000001,"5min":19.962,"15min":19.962},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.784,"5min":1.331,"15min":0.90200000000000002},"min":{"1min":0.31,"5min":0.27600000000000002,"15min":0.22900000000000001},"max":{"1min":10.057,"5min":20.829999999999998,"15min":20.829999999999998},"last":0.67400000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:34:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2130000000000001,"5min":1.1759999999999999,"15min":2.2389999999999999},"min":{"1min":0.33800000000000002,"5min":0.27600000000000002,"15min":0.26900000000000002},"max":{"1min":9.6679999999999993,"5min":9.6679999999999993,"15min":310.68799999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":2.6560000000000001,"5min":1.667,"15min":2.52},"min":{"1min":0.309,"5min":0.307,"15min":0.25900000000000001},"max":{"1min":15.619999999999999,"5min":15.619999999999999,"15min":311.45400000000001},"last":0.81699999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:34:47 2023","interfaces":[{"interface":"back","average":{"1min":2.5760000000000001,"5min":1.577,"15min":2.8639999999999999},"min":{"1min":0.34999999999999998,"5min":0.34599999999999997,"15min":0.317},"max":{"1min":15.497,"5min":16.408999999999999,"15min":179.28800000000001},"last":0.79000000000000004},{"interface":"front","average":{"1min":2.387,"5min":1.6890000000000001,"15min":2.956},"min":{"1min":0.36799999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":15.653,"5min":18.972000000000001,"15min":179.428},"last":0.73099999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":392232960,"data_stored":392232960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1539313664,"data_stored":1539309592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3049127936,"data_stored":3049127936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1617629184,"data_stored":1617625112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2595291136,"data_stored":2595291136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2824667136,"data_stored":2824667136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1677131776,"data_stored":1677131776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1799819264,"data_stored":1799815192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:35:27.401 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:35:27.429 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:27.517 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:27.571 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:27.649 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:27.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:27.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:27.658 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:27 smithi110 ceph-mon[221102]: pgmap v1613: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 27114/225003 objects degraded (12.051%); 4587/225003 objects misplaced (2.039%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T23:35:27.658 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:27 smithi110 ceph-mon[221102]: from='client.46745 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:35:27.658 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:27 smithi110 ceph-mon[212939]: pgmap v1613: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 27114/225003 objects degraded (12.051%); 4587/225003 objects misplaced (2.039%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T23:35:27.658 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:27 smithi110 ceph-mon[212939]: from='client.46745 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:35:27.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:27 smithi138 ceph-mon[193606]: pgmap v1613: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 27114/225003 objects degraded (12.051%); 4587/225003 objects misplaced (2.039%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T23:35:27.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:27 smithi138 ceph-mon[193606]: from='client.46745 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:35:27.758 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:27.797 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:27.950 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:27.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:27.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:27.990 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:35:27.990 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:35:27.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:35:27.991 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:35:27.991 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:27.991 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135599 11.e 11.62cc478e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:35:27.991 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.552343+0000", 2023-09-23T23:35:27.991 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 557.43403513600003, 2023-09-23T23:35:27.991 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95499033600000005, 2023-09-23T23:35:27.991 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:27.991 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:35:27.991 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:27.991 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:35:27.991 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:35:27.992 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135599 2023-09-23T23:35:27.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:27.992 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:27.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:27.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:27.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552343+0000", 2023-09-23T23:35:27.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:27.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:27.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:27.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:27.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552343+0000", 2023-09-23T23:35:27.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:27.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:27.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:27.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:27.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552342+0000", 2023-09-23T23:35:27.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:35:27.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:27.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:27.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:27.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552347+0000", 2023-09-23T23:35:27.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5850000000000001e-06 2023-09-23T23:35:27.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:27.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:27.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:27.994 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552358+0000", 2023-09-23T23:35:27.994 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0998999999999999e-05 2023-09-23T23:35:27.994 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:27.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:27.994 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:27.994 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552365+0000", 2023-09-23T23:35:27.994 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8630000000000002e-06 2023-09-23T23:35:27.994 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:27.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:27.994 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:27.994 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552426+0000", 2023-09-23T23:35:27.995 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0708000000000002e-05 2023-09-23T23:35:27.995 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:27.995 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:27.995 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:27.995 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552433+0000", 2023-09-23T23:35:27.995 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8599999999999993e-06 2023-09-23T23:35:27.995 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:27.995 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:27.995 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:27.995 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507318+0000", 2023-09-23T23:35:27.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95488429500000005 2023-09-23T23:35:27.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:27.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:27.996 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:27.996 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507334+0000", 2023-09-23T23:35:27.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5894000000000001e-05 2023-09-23T23:35:27.996 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:27.996 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:27.996 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:27.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:27.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:27.996 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:135605 11.e 11.52f78e0e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-09-23T23:35:27.997 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:10.552409+0000", 2023-09-23T23:35:27.997 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 557.43396919700001, 2023-09-23T23:35:27.997 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95496278599999995, 2023-09-23T23:35:27.997 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:27.997 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:35:27.997 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:27.997 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:35:27.997 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:35:27.997 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 135605 2023-09-23T23:35:27.997 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:27.997 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:27.997 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:27.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:27.998 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552409+0000", 2023-09-23T23:35:27.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:27.998 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:27.998 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:27.998 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:27.998 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552409+0000", 2023-09-23T23:35:27.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:27.998 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:27.998 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:27.998 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:27.999 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552409+0000", 2023-09-23T23:35:27.999 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:35:27.999 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:27.999 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:27.999 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:27.999 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552412+0000", 2023-09-23T23:35:28.000 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7199999999999998e-06 2023-09-23T23:35:28.000 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.000 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.000 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552417+0000", 2023-09-23T23:35:28.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.023e-06 2023-09-23T23:35:28.001 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.001 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552419+0000", 2023-09-23T23:35:28.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7319999999999998e-06 2023-09-23T23:35:28.002 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.002 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.002 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.002 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552464+0000", 2023-09-23T23:35:28.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5096000000000003e-05 2023-09-23T23:35:28.002 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.002 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.002 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.002 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:10.552472+0000", 2023-09-23T23:35:28.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5390000000000002e-06 2023-09-23T23:35:28.002 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.002 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.003 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.003 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507362+0000", 2023-09-23T23:35:28.003 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95488972999999999 2023-09-23T23:35:28.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.003 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.003 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.003 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:11.507372+0000", 2023-09-23T23:35:28.003 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0360000000000001e-05 2023-09-23T23:35:28.003 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.003 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.003 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.004 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.004 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:154629 11.17 11:ed0449d2:::benchmark_data_smithi110_433746_object48749:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-09-23T23:35:28.004 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:26:43.186486+0000", 2023-09-23T23:35:28.004 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 524.79989238899998, 2023-09-23T23:35:28.004 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0259201550000001, 2023-09-23T23:35:28.004 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.004 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:35:28.004 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.004 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:35:28.004 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:35:28.004 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 154629 2023-09-23T23:35:28.004 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.005 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.005 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186486+0000", 2023-09-23T23:35:28.005 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.005 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.005 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186486+0000", 2023-09-23T23:35:28.005 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.005 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186485+0000", 2023-09-23T23:35:28.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:35:28.006 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.006 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186490+0000", 2023-09-23T23:35:28.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5589999999999999e-06 2023-09-23T23:35:28.006 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.006 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186498+0000", 2023-09-23T23:35:28.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4509999999999992e-06 2023-09-23T23:35:28.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.007 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.007 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.007 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186504+0000", 2023-09-23T23:35:28.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3369999999999996e-06 2023-09-23T23:35:28.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.007 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.007 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.007 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186667+0000", 2023-09-23T23:35:28.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016259099999999999 2023-09-23T23:35:28.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.008 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:43.186678+0000", 2023-09-23T23:35:28.008 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1362999999999999e-05 2023-09-23T23:35:28.008 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.008 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.183982+0000", 2023-09-23T23:35:28.008 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99730399700000005 2023-09-23T23:35:28.008 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:35:28.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.184074+0000", 2023-09-23T23:35:28.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2168000000000003e-05 2023-09-23T23:35:28.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.009 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:35:28.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.184404+0000", 2023-09-23T23:35:28.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00032955400000000002 2023-09-23T23:35:28.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.009 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:35:28.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.191347+0000", 2023-09-23T23:35:28.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069433059999999998 2023-09-23T23:35:28.010 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.010 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.010 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212265+0000", 2023-09-23T23:35:28.010 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020918125999999999 2023-09-23T23:35:28.010 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.010 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.010 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212315+0000", 2023-09-23T23:35:28.010 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9078e-05 2023-09-23T23:35:28.010 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:35:28.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212362+0000", 2023-09-23T23:35:28.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7893999999999999e-05 2023-09-23T23:35:28.011 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:26:44.212406+0000", 2023-09-23T23:35:28.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3766000000000001e-05 2023-09-23T23:35:28.011 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.011 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.011 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.012 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:73408 12.10 12:0ed56545:::benchmark_data_smithi110_491193_object73407:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T23:35:28.012 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:29:44.726507+0000", 2023-09-23T23:35:28.012 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 343.25987109800002, 2023-09-23T23:35:28.012 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95402578900000001, 2023-09-23T23:35:28.012 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.012 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:35:28.012 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.012 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:35:28.012 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:35:28.012 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 73408 2023-09-23T23:35:28.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.013 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.013 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.013 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.013 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726507+0000", 2023-09-23T23:35:28.013 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.013 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.013 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.013 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726507+0000", 2023-09-23T23:35:28.013 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.013 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726505+0000", 2023-09-23T23:35:28.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:35:28.014 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726548+0000", 2023-09-23T23:35:28.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2936999999999999e-05 2023-09-23T23:35:28.014 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726565+0000", 2023-09-23T23:35:28.015 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6705e-05 2023-09-23T23:35:28.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.015 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.015 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726572+0000", 2023-09-23T23:35:28.015 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9500000000000004e-06 2023-09-23T23:35:28.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.015 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.015 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.751276+0000", 2023-09-23T23:35:28.015 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.024704454000000001 2023-09-23T23:35:28.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.751287+0000", 2023-09-23T23:35:28.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0243e-05 2023-09-23T23:35:28.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.661352+0000", 2023-09-23T23:35:28.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91006557200000004 2023-09-23T23:35:28.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-09-23T23:35:28.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.661964+0000", 2023-09-23T23:35:28.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00061125199999999995 2023-09-23T23:35:28.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.017 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.017 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.680469+0000", 2023-09-23T23:35:28.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018505864 2023-09-23T23:35:28.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.017 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.017 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.680533+0000", 2023-09-23T23:35:28.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3732999999999994e-05 2023-09-23T23:35:28.017 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.018 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.018 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.018 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.018 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.018 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21483 13.5 13:a8fdedd1:::benchmark_data_smithi110_541928_object21482:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:35:28.018 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.045464+0000", 2023-09-23T23:35:28.018 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 137.94091480399999, 2023-09-23T23:35:28.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.135403025, 2023-09-23T23:35:28.018 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.018 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:35:28.018 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.018 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:35:28.019 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:35:28.019 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21483 2023-09-23T23:35:28.019 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.019 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.019 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.019 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.019 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045464+0000", 2023-09-23T23:35:28.019 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.019 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.019 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.019 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.019 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045464+0000", 2023-09-23T23:35:28.019 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.020 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.020 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045462+0000", 2023-09-23T23:35:28.020 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:35:28.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.020 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.020 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045507+0000", 2023-09-23T23:35:28.020 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4555999999999998e-05 2023-09-23T23:35:28.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.020 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045524+0000", 2023-09-23T23:35:28.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7450999999999999e-05 2023-09-23T23:35:28.021 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.021 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045531+0000", 2023-09-23T23:35:28.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7329999999999997e-06 2023-09-23T23:35:28.021 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.021 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045729+0000", 2023-09-23T23:35:28.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000197634 2023-09-23T23:35:28.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.022 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.022 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045739+0000", 2023-09-23T23:35:28.022 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.825e-06 2023-09-23T23:35:28.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.022 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.022 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.177900+0000", 2023-09-23T23:35:28.022 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.132160896 2023-09-23T23:35:28.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.022 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:35:28.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.178003+0000", 2023-09-23T23:35:28.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103212 2023-09-23T23:35:28.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.023 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:35:28.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.178112+0000", 2023-09-23T23:35:28.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109002 2023-09-23T23:35:28.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.023 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:35:28.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.179965+0000", 2023-09-23T23:35:28.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001853234 2023-09-23T23:35:28.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.024 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.024 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180695+0000", 2023-09-23T23:35:28.024 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000729913 2023-09-23T23:35:28.024 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.024 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.024 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180800+0000", 2023-09-23T23:35:28.024 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000104758 2023-09-23T23:35:28.024 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.024 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:35:28.025 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180834+0000", 2023-09-23T23:35:28.025 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4038000000000002e-05 2023-09-23T23:35:28.025 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.025 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.025 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.025 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180867+0000", 2023-09-23T23:35:28.025 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2935000000000002e-05 2023-09-23T23:35:28.025 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.025 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.025 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.025 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.025 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.026 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21498 13.5 13:ac1dd01c:::benchmark_data_smithi110_541928_object21497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:35:28.026 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.050241+0000", 2023-09-23T23:35:28.026 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 137.93613700200001, 2023-09-23T23:35:28.026 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.130775804, 2023-09-23T23:35:28.026 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.026 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:35:28.026 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.026 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:35:28.026 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:35:28.026 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21498 2023-09-23T23:35:28.026 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.026 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.027 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050241+0000", 2023-09-23T23:35:28.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.027 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.027 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050241+0000", 2023-09-23T23:35:28.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.027 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.027 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.028 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050240+0000", 2023-09-23T23:35:28.028 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:35:28.028 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.028 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.028 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.028 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050290+0000", 2023-09-23T23:35:28.028 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0144e-05 2023-09-23T23:35:28.028 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.028 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.028 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.028 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050304+0000", 2023-09-23T23:35:28.028 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3708e-05 2023-09-23T23:35:28.029 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.029 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.029 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.029 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050310+0000", 2023-09-23T23:35:28.029 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9290000000000003e-06 2023-09-23T23:35:28.029 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.029 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.029 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.029 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050369+0000", 2023-09-23T23:35:28.029 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9196e-05 2023-09-23T23:35:28.029 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.029 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.030 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.030 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050379+0000", 2023-09-23T23:35:28.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9639999999999998e-06 2023-09-23T23:35:28.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.030 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.030 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.030 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.179168+0000", 2023-09-23T23:35:28.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.128789193 2023-09-23T23:35:28.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.030 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.030 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:35:28.030 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.179266+0000", 2023-09-23T23:35:28.031 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7250000000000006e-05 2023-09-23T23:35:28.031 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.031 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.031 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:35:28.031 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.179375+0000", 2023-09-23T23:35:28.031 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109688 2023-09-23T23:35:28.031 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.031 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.031 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.031 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180921+0000", 2023-09-23T23:35:28.031 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015460490000000001 2023-09-23T23:35:28.031 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.032 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.032 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:35:28.032 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180927+0000", 2023-09-23T23:35:28.032 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9109999999999998e-06 2023-09-23T23:35:28.032 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.032 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.032 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.032 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180956+0000", 2023-09-23T23:35:28.032 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8926000000000001e-05 2023-09-23T23:35:28.032 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.032 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.032 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:35:28.033 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180995+0000", 2023-09-23T23:35:28.033 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8621999999999997e-05 2023-09-23T23:35:28.033 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.033 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.033 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.033 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181017+0000", 2023-09-23T23:35:28.033 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.251e-05 2023-09-23T23:35:28.033 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.033 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.033 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.033 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.034 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.034 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21499 13.5 13:a6d31f08:::benchmark_data_smithi110_541928_object21498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:35:28.034 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.054699+0000", 2023-09-23T23:35:28.034 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 137.931679605, 2023-09-23T23:35:28.034 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1271894090000001, 2023-09-23T23:35:28.034 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.034 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:35:28.034 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.034 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:35:28.034 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:35:28.034 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21499 2023-09-23T23:35:28.035 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.035 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.035 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.035 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.035 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054699+0000", 2023-09-23T23:35:28.035 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.035 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.035 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.035 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.035 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054699+0000", 2023-09-23T23:35:28.035 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.035 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.035 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.036 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.036 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054697+0000", 2023-09-23T23:35:28.036 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:35:28.036 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.036 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.036 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.036 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054738+0000", 2023-09-23T23:35:28.036 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0728e-05 2023-09-23T23:35:28.036 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.036 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.036 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.037 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054757+0000", 2023-09-23T23:35:28.037 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9037000000000001e-05 2023-09-23T23:35:28.037 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.037 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.037 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.037 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054765+0000", 2023-09-23T23:35:28.037 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8010000000000004e-06 2023-09-23T23:35:28.037 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.037 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.037 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.037 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054821+0000", 2023-09-23T23:35:28.037 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5702999999999998e-05 2023-09-23T23:35:28.038 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.038 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.038 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.038 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054829+0000", 2023-09-23T23:35:28.038 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9579999999999996e-06 2023-09-23T23:35:28.038 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.038 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.038 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.038 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.179986+0000", 2023-09-23T23:35:28.038 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.125156388 2023-09-23T23:35:28.038 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.039 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.039 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:35:28.039 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180059+0000", 2023-09-23T23:35:28.039 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3510999999999995e-05 2023-09-23T23:35:28.039 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.039 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.039 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:35:28.039 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180157+0000", 2023-09-23T23:35:28.039 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8039000000000001e-05 2023-09-23T23:35:28.039 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.039 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.039 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:35:28.040 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181753+0000", 2023-09-23T23:35:28.040 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001595602 2023-09-23T23:35:28.040 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.040 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.040 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.040 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181813+0000", 2023-09-23T23:35:28.040 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9984e-05 2023-09-23T23:35:28.040 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.040 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.040 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.040 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181836+0000", 2023-09-23T23:35:28.040 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3373000000000001e-05 2023-09-23T23:35:28.040 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.041 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.041 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:35:28.041 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181865+0000", 2023-09-23T23:35:28.041 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8147e-05 2023-09-23T23:35:28.041 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.041 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.041 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.041 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181888+0000", 2023-09-23T23:35:28.041 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.37e-05 2023-09-23T23:35:28.041 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.041 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.041 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.042 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.042 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.042 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21528 13.5 13:a8de34c9:::benchmark_data_smithi110_541928_object21527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:35:28.042 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.066977+0000", 2023-09-23T23:35:28.042 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 137.919401349, 2023-09-23T23:35:28.042 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.116824818, 2023-09-23T23:35:28.042 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.042 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:35:28.042 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.042 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:35:28.042 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:35:28.042 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21528 2023-09-23T23:35:28.043 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.043 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.043 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.043 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.043 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.066977+0000", 2023-09-23T23:35:28.043 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.043 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.043 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.043 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.043 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.066977+0000", 2023-09-23T23:35:28.043 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.043 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.044 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.044 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.044 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.066976+0000", 2023-09-23T23:35:28.044 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:35:28.044 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.044 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.044 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.044 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067009+0000", 2023-09-23T23:35:28.044 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2963999999999998e-05 2023-09-23T23:35:28.044 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.044 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.044 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.045 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067017+0000", 2023-09-23T23:35:28.045 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8350000000000004e-06 2023-09-23T23:35:28.045 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.045 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.045 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.045 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067022+0000", 2023-09-23T23:35:28.045 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9989999999999999e-06 2023-09-23T23:35:28.045 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.045 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.045 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.045 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.068428+0000", 2023-09-23T23:35:28.045 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014058860000000001 2023-09-23T23:35:28.046 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.046 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.046 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.046 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.068438+0000", 2023-09-23T23:35:28.046 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0033e-05 2023-09-23T23:35:28.046 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.046 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.046 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.046 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181053+0000", 2023-09-23T23:35:28.046 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.112614311 2023-09-23T23:35:28.046 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.046 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.047 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:35:28.047 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181184+0000", 2023-09-23T23:35:28.047 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000131314 2023-09-23T23:35:28.047 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.047 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.047 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:35:28.047 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181273+0000", 2023-09-23T23:35:28.047 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.954e-05 2023-09-23T23:35:28.047 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.049 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.049 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.049 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183686+0000", 2023-09-23T23:35:28.049 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002412871 2023-09-23T23:35:28.049 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.049 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.049 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.049 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183729+0000", 2023-09-23T23:35:28.049 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2329000000000001e-05 2023-09-23T23:35:28.049 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.050 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:35:28.050 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183736+0000", 2023-09-23T23:35:28.050 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0360000000000001e-06 2023-09-23T23:35:28.050 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.050 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:35:28.050 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183767+0000", 2023-09-23T23:35:28.050 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1133000000000002e-05 2023-09-23T23:35:28.050 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.050 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.051 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183802+0000", 2023-09-23T23:35:28.051 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5176000000000003e-05 2023-09-23T23:35:28.051 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.051 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.051 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.051 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.051 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.051 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21532 13.5 13:a4c16192:::benchmark_data_smithi110_541928_object21531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:35:28.051 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.067036+0000", 2023-09-23T23:35:28.051 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 137.91934279899999, 2023-09-23T23:35:28.051 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1169511729999999, 2023-09-23T23:35:28.051 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.052 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:35:28.052 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.052 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:35:28.052 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:35:28.052 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21532 2023-09-23T23:35:28.052 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.052 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.052 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.052 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.052 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067036+0000", 2023-09-23T23:35:28.052 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.052 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.053 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.053 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.053 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067036+0000", 2023-09-23T23:35:28.053 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.053 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.053 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.053 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.053 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067035+0000", 2023-09-23T23:35:28.053 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:35:28.053 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.053 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.053 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.054 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067063+0000", 2023-09-23T23:35:28.054 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7453e-05 2023-09-23T23:35:28.054 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.054 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.054 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.054 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067070+0000", 2023-09-23T23:35:28.054 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4699999999999996e-06 2023-09-23T23:35:28.054 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.054 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.054 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.054 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067075+0000", 2023-09-23T23:35:28.054 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2730000000000002e-06 2023-09-23T23:35:28.055 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.055 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.055 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.055 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.068458+0000", 2023-09-23T23:35:28.055 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001382505 2023-09-23T23:35:28.055 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.055 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.055 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.055 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.068464+0000", 2023-09-23T23:35:28.055 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0909999999999998e-06 2023-09-23T23:35:28.055 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.055 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181930+0000", 2023-09-23T23:35:28.056 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.113466209 2023-09-23T23:35:28.056 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.056 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:35:28.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.182050+0000", 2023-09-23T23:35:28.056 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120068 2023-09-23T23:35:28.056 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.056 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:35:28.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.182218+0000", 2023-09-23T23:35:28.057 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016763300000000001 2023-09-23T23:35:28.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:35:28.057 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183812+0000", 2023-09-23T23:35:28.057 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001594179 2023-09-23T23:35:28.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.057 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183872+0000", 2023-09-23T23:35:28.057 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.033e-05 2023-09-23T23:35:28.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.058 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.058 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.058 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183920+0000", 2023-09-23T23:35:28.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7956e-05 2023-09-23T23:35:28.058 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.058 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.058 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:35:28.058 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183953+0000", 2023-09-23T23:35:28.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2855999999999998e-05 2023-09-23T23:35:28.058 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.058 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.058 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183987+0000", 2023-09-23T23:35:28.059 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3695000000000002e-05 2023-09-23T23:35:28.059 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.059 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.059 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.059 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21564 13.5 13:a601eb05:::benchmark_data_smithi110_541928_object21563:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:35:28.059 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.088589+0000", 2023-09-23T23:35:28.059 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 137.89778982000001, 2023-09-23T23:35:28.059 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0965361170000001, 2023-09-23T23:35:28.059 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.059 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:35:28.060 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.060 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:35:28.060 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:35:28.060 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21564 2023-09-23T23:35:28.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.060 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.060 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.060 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088589+0000", 2023-09-23T23:35:28.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.060 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.061 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088589+0000", 2023-09-23T23:35:28.061 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.061 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.061 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088588+0000", 2023-09-23T23:35:28.061 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:35:28.061 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.061 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088625+0000", 2023-09-23T23:35:28.062 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7500999999999999e-05 2023-09-23T23:35:28.062 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.062 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.062 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.062 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088639+0000", 2023-09-23T23:35:28.062 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3568e-05 2023-09-23T23:35:28.062 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.062 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.062 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.062 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088643+0000", 2023-09-23T23:35:28.062 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7219999999999999e-06 2023-09-23T23:35:28.062 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.063 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.063 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.063 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088698+0000", 2023-09-23T23:35:28.063 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4797999999999999e-05 2023-09-23T23:35:28.063 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.063 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.063 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.063 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088713+0000", 2023-09-23T23:35:28.063 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4809e-05 2023-09-23T23:35:28.063 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.063 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.063 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.182903+0000", 2023-09-23T23:35:28.064 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.094190306 2023-09-23T23:35:28.064 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.064 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.064 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:35:28.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.182976+0000", 2023-09-23T23:35:28.064 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3059000000000006e-05 2023-09-23T23:35:28.064 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.064 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.064 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:35:28.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183121+0000", 2023-09-23T23:35:28.064 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000144797 2023-09-23T23:35:28.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.065 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:35:28.065 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183999+0000", 2023-09-23T23:35:28.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00087746999999999999 2023-09-23T23:35:28.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.065 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.065 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185012+0000", 2023-09-23T23:35:28.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001013735 2023-09-23T23:35:28.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185045+0000", 2023-09-23T23:35:28.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2339000000000003e-05 2023-09-23T23:35:28.066 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.066 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:35:28.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185088+0000", 2023-09-23T23:35:28.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2854e-05 2023-09-23T23:35:28.066 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.066 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185125+0000", 2023-09-23T23:35:28.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.714e-05 2023-09-23T23:35:28.067 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.067 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.067 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.067 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.067 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.067 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21580 13.5 13:afa4f38d:::benchmark_data_smithi110_541928_object21579:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:35:28.067 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.097520+0000", 2023-09-23T23:35:28.067 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 137.88885885400001, 2023-09-23T23:35:28.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.089325219, 2023-09-23T23:35:28.068 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.068 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:35:28.068 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.068 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:35:28.068 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:35:28.068 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21580 2023-09-23T23:35:28.068 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.068 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.068 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097520+0000", 2023-09-23T23:35:28.069 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.069 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.069 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.069 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097520+0000", 2023-09-23T23:35:28.069 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.069 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.069 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.069 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097518+0000", 2023-09-23T23:35:28.069 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:35:28.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.070 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.070 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097552+0000", 2023-09-23T23:35:28.070 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3627999999999997e-05 2023-09-23T23:35:28.070 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.070 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.070 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097567+0000", 2023-09-23T23:35:28.070 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5477000000000001e-05 2023-09-23T23:35:28.070 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.071 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.071 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097574+0000", 2023-09-23T23:35:28.071 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7309999999999996e-06 2023-09-23T23:35:28.071 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.071 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.071 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.071 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097718+0000", 2023-09-23T23:35:28.071 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014400700000000001 2023-09-23T23:35:28.071 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.071 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.071 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.071 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097740+0000", 2023-09-23T23:35:28.072 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2238999999999998e-05 2023-09-23T23:35:28.072 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.072 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.072 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.072 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.184022+0000", 2023-09-23T23:35:28.072 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0862815990000001 2023-09-23T23:35:28.072 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.072 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.072 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:35:28.072 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.184170+0000", 2023-09-23T23:35:28.072 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000147754 2023-09-23T23:35:28.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.073 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.073 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:35:28.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.184340+0000", 2023-09-23T23:35:28.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017066499999999999 2023-09-23T23:35:28.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.073 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.073 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186756+0000", 2023-09-23T23:35:28.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024154049999999998 2023-09-23T23:35:28.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.073 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.074 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.074 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186789+0000", 2023-09-23T23:35:28.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3621000000000002e-05 2023-09-23T23:35:28.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.074 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.074 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:35:28.074 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186795+0000", 2023-09-23T23:35:28.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4809999999999997e-06 2023-09-23T23:35:28.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.074 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.074 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:35:28.074 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186822+0000", 2023-09-23T23:35:28.075 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7166e-05 2023-09-23T23:35:28.075 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.075 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.075 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.075 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186845+0000", 2023-09-23T23:35:28.075 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2716000000000001e-05 2023-09-23T23:35:28.075 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.075 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.075 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.075 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.075 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.075 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21590 13.5 13:a913081b:::benchmark_data_smithi110_541928_object21589:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:35:28.076 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.101884+0000", 2023-09-23T23:35:28.076 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 137.88449454600001, 2023-09-23T23:35:28.076 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0874988189999999, 2023-09-23T23:35:28.076 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.076 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:35:28.076 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.076 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:35:28.076 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:35:28.076 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21590 2023-09-23T23:35:28.076 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.076 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.076 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.076 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.077 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101884+0000", 2023-09-23T23:35:28.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.077 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.077 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.077 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.077 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101884+0000", 2023-09-23T23:35:28.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.077 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.077 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.077 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.077 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101883+0000", 2023-09-23T23:35:28.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:35:28.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.078 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101943+0000", 2023-09-23T23:35:28.078 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9757999999999999e-05 2023-09-23T23:35:28.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.078 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101952+0000", 2023-09-23T23:35:28.078 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6420000000000002e-06 2023-09-23T23:35:28.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101958+0000", 2023-09-23T23:35:28.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0900000000000001e-06 2023-09-23T23:35:28.079 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.079 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102140+0000", 2023-09-23T23:35:28.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018145100000000001 2023-09-23T23:35:28.079 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.079 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102153+0000", 2023-09-23T23:35:28.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2676e-05 2023-09-23T23:35:28.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.080 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.080 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185166+0000", 2023-09-23T23:35:28.080 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0830130069999999 2023-09-23T23:35:28.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.080 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:35:28.080 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185286+0000", 2023-09-23T23:35:28.080 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120227 2023-09-23T23:35:28.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.081 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:35:28.081 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185443+0000", 2023-09-23T23:35:28.081 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000156844 2023-09-23T23:35:28.081 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.081 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.081 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:35:28.081 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186851+0000", 2023-09-23T23:35:28.081 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014085199999999999 2023-09-23T23:35:28.081 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.081 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.081 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.081 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188371+0000", 2023-09-23T23:35:28.081 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015201780000000001 2023-09-23T23:35:28.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.082 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.082 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189290+0000", 2023-09-23T23:35:28.082 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00091839000000000005 2023-09-23T23:35:28.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.082 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:35:28.082 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189342+0000", 2023-09-23T23:35:28.082 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2110000000000001e-05 2023-09-23T23:35:28.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.083 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.083 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189383+0000", 2023-09-23T23:35:28.083 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0859999999999998e-05 2023-09-23T23:35:28.083 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.083 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.083 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.083 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.083 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.083 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21591 13.5 13:ae681958:::benchmark_data_smithi110_541928_object21590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:35:28.083 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.102344+0000", 2023-09-23T23:35:28.083 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 137.88403471500001, 2023-09-23T23:35:28.083 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0912746840000001, 2023-09-23T23:35:28.084 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.084 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:35:28.084 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.084 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:35:28.084 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:35:28.084 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21591 2023-09-23T23:35:28.084 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.084 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.084 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.084 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.084 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102344+0000", 2023-09-23T23:35:28.084 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.084 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.085 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.085 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.085 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102344+0000", 2023-09-23T23:35:28.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.085 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.085 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.085 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.085 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102343+0000", 2023-09-23T23:35:28.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:35:28.085 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.085 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.086 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102429+0000", 2023-09-23T23:35:28.086 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6637000000000004e-05 2023-09-23T23:35:28.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.086 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.086 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102441+0000", 2023-09-23T23:35:28.086 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1902999999999999e-05 2023-09-23T23:35:28.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.086 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.086 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102450+0000", 2023-09-23T23:35:28.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2320000000000001e-06 2023-09-23T23:35:28.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102504+0000", 2023-09-23T23:35:28.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3981999999999999e-05 2023-09-23T23:35:28.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102511+0000", 2023-09-23T23:35:28.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8399999999999995e-06 2023-09-23T23:35:28.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186108+0000", 2023-09-23T23:35:28.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.083596351 2023-09-23T23:35:28.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:35:28.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186201+0000", 2023-09-23T23:35:28.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3474000000000007e-05 2023-09-23T23:35:28.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:35:28.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186283+0000", 2023-09-23T23:35:28.089 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1625999999999994e-05 2023-09-23T23:35:28.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.089 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.089 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.089 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188420+0000", 2023-09-23T23:35:28.089 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021368749999999999 2023-09-23T23:35:28.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.089 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.089 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:35:28.089 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188429+0000", 2023-09-23T23:35:28.089 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5370000000000003e-06 2023-09-23T23:35:28.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.090 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.090 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.090 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193525+0000", 2023-09-23T23:35:28.090 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050954379999999999 2023-09-23T23:35:28.090 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.090 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.090 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:35:28.090 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193578+0000", 2023-09-23T23:35:28.090 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2927999999999999e-05 2023-09-23T23:35:28.090 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.090 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.090 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.091 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193618+0000", 2023-09-23T23:35:28.091 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0822000000000003e-05 2023-09-23T23:35:28.091 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.091 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.091 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.091 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.091 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.091 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21598 13.5 13:a172c780:::benchmark_data_smithi110_541928_object21597:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:35:28.091 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.106175+0000", 2023-09-23T23:35:28.091 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 137.88020352000001, 2023-09-23T23:35:28.091 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0875860749999999, 2023-09-23T23:35:28.091 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.092 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:35:28.092 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.092 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:35:28.092 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:35:28.092 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21598 2023-09-23T23:35:28.092 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.092 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.092 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.092 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.092 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106175+0000", 2023-09-23T23:35:28.092 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.092 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.093 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.093 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.093 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106175+0000", 2023-09-23T23:35:28.093 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.093 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.093 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.093 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.093 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106174+0000", 2023-09-23T23:35:28.093 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:35:28.093 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.093 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.093 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.093 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106283+0000", 2023-09-23T23:35:28.094 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109133 2023-09-23T23:35:28.094 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.094 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.094 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.094 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106294+0000", 2023-09-23T23:35:28.094 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0994e-05 2023-09-23T23:35:28.094 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.094 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.094 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.094 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106301+0000", 2023-09-23T23:35:28.094 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5089999999999999e-06 2023-09-23T23:35:28.094 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.095 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.095 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.095 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106360+0000", 2023-09-23T23:35:28.095 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8828e-05 2023-09-23T23:35:28.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.095 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.095 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.095 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106369+0000", 2023-09-23T23:35:28.095 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1880000000000008e-06 2023-09-23T23:35:28.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.095 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.095 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.095 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186882+0000", 2023-09-23T23:35:28.096 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0805130059999999 2023-09-23T23:35:28.096 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.096 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:35:28.096 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186960+0000", 2023-09-23T23:35:28.096 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7863999999999999e-05 2023-09-23T23:35:28.096 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.096 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:35:28.097 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.187041+0000", 2023-09-23T23:35:28.097 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1542e-05 2023-09-23T23:35:28.097 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.097 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.097 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:35:28.097 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188432+0000", 2023-09-23T23:35:28.097 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001390797 2023-09-23T23:35:28.097 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.097 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.097 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.097 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188460+0000", 2023-09-23T23:35:28.097 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7668e-05 2023-09-23T23:35:28.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.098 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.098 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.098 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193681+0000", 2023-09-23T23:35:28.098 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0052214999999999996 2023-09-23T23:35:28.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.098 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.098 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:35:28.098 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193725+0000", 2023-09-23T23:35:28.098 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3377000000000002e-05 2023-09-23T23:35:28.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.098 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.098 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.099 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193761+0000", 2023-09-23T23:35:28.099 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6464000000000002e-05 2023-09-23T23:35:28.099 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.099 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.099 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.099 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.099 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.099 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21614 13.5 13:ab7b6c08:::benchmark_data_smithi110_541928_object21613:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:35:28.099 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.115290+0000", 2023-09-23T23:35:28.099 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 137.871088415, 2023-09-23T23:35:28.099 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.078720337, 2023-09-23T23:35:28.099 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.100 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:35:28.100 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.100 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:35:28.100 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:35:28.100 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21614 2023-09-23T23:35:28.100 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.100 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.100 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.100 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.100 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115290+0000", 2023-09-23T23:35:28.100 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.100 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.100 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.101 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.101 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115290+0000", 2023-09-23T23:35:28.101 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.101 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.101 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.101 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.101 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115289+0000", 2023-09-23T23:35:28.101 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:35:28.101 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.101 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.101 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.102 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115353+0000", 2023-09-23T23:35:28.102 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4473000000000004e-05 2023-09-23T23:35:28.102 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.102 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.102 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.102 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115365+0000", 2023-09-23T23:35:28.102 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.164e-05 2023-09-23T23:35:28.102 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.102 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.102 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.102 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115371+0000", 2023-09-23T23:35:28.103 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1500000000000004e-06 2023-09-23T23:35:28.103 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.103 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.103 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.103 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115421+0000", 2023-09-23T23:35:28.103 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0117999999999997e-05 2023-09-23T23:35:28.103 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.103 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.103 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.103 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115431+0000", 2023-09-23T23:35:28.103 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4350000000000003e-06 2023-09-23T23:35:28.103 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.103 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.104 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.187560+0000", 2023-09-23T23:35:28.104 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0721295150000001 2023-09-23T23:35:28.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.104 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.104 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:35:28.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.187686+0000", 2023-09-23T23:35:28.104 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012613699999999999 2023-09-23T23:35:28.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.104 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.104 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:35:28.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.187797+0000", 2023-09-23T23:35:28.105 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110852 2023-09-23T23:35:28.105 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.105 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:35:28.105 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188463+0000", 2023-09-23T23:35:28.105 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00066563799999999997 2023-09-23T23:35:28.105 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.105 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.105 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189424+0000", 2023-09-23T23:35:28.105 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00096155399999999999 2023-09-23T23:35:28.105 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.106 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.106 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193937+0000", 2023-09-23T23:35:28.106 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00451308 2023-09-23T23:35:28.106 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.106 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.106 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:35:28.106 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193979+0000", 2023-09-23T23:35:28.106 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1501999999999997e-05 2023-09-23T23:35:28.106 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.106 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.106 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.106 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194010+0000", 2023-09-23T23:35:28.106 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1535999999999998e-05 2023-09-23T23:35:28.107 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.107 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.107 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.107 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.107 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.107 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21623 13.5 13:ac3554a5:::benchmark_data_smithi110_541928_object21622:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:35:28.107 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.123936+0000", 2023-09-23T23:35:28.107 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 137.862442375, 2023-09-23T23:35:28.107 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0702429680000001, 2023-09-23T23:35:28.107 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.107 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:35:28.107 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.108 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:35:28.108 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:35:28.108 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21623 2023-09-23T23:35:28.108 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.108 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.108 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.108 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.108 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.123936+0000", 2023-09-23T23:35:28.108 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.108 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.108 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.108 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.109 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.123936+0000", 2023-09-23T23:35:28.109 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.109 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.109 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.123935+0000", 2023-09-23T23:35:28.109 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:35:28.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.109 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.109 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.124042+0000", 2023-09-23T23:35:28.109 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010689200000000001 2023-09-23T23:35:28.110 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.110 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.110 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.110 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.124053+0000", 2023-09-23T23:35:28.110 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1221e-05 2023-09-23T23:35:28.110 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.110 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.110 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.110 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.124058+0000", 2023-09-23T23:35:28.110 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.074e-06 2023-09-23T23:35:28.110 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.110 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.111 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.124114+0000", 2023-09-23T23:35:28.111 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6026999999999997e-05 2023-09-23T23:35:28.111 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.111 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.124124+0000", 2023-09-23T23:35:28.111 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0032e-05 2023-09-23T23:35:28.111 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.111 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188483+0000", 2023-09-23T23:35:28.111 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.064359354 2023-09-23T23:35:28.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:35:28.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188589+0000", 2023-09-23T23:35:28.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105889 2023-09-23T23:35:28.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:35:28.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188737+0000", 2023-09-23T23:35:28.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014783099999999999 2023-09-23T23:35:28.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:35:28.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191026+0000", 2023-09-23T23:35:28.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022893409999999999 2023-09-23T23:35:28.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191847+0000", 2023-09-23T23:35:28.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00082065100000000004 2023-09-23T23:35:28.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194089+0000", 2023-09-23T23:35:28.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022423339999999999 2023-09-23T23:35:28.114 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.114 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:35:28.114 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194143+0000", 2023-09-23T23:35:28.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3446999999999998e-05 2023-09-23T23:35:28.114 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.114 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.114 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194179+0000", 2023-09-23T23:35:28.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6216e-05 2023-09-23T23:35:28.114 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.115 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.115 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.115 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.115 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.115 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21639 13.5 13:afb98446:::benchmark_data_smithi110_541928_object21638:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:35:28.115 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.135557+0000", 2023-09-23T23:35:28.115 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 137.850821177, 2023-09-23T23:35:28.115 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0589459779999999, 2023-09-23T23:35:28.115 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.115 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:35:28.115 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.115 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:35:28.115 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:35:28.116 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21639 2023-09-23T23:35:28.116 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.116 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.116 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.116 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.116 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135557+0000", 2023-09-23T23:35:28.116 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.116 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.116 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.116 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.116 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135557+0000", 2023-09-23T23:35:28.116 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.117 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.117 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.117 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.117 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135557+0000", 2023-09-23T23:35:28.117 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:35:28.117 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.117 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.117 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.117 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135622+0000", 2023-09-23T23:35:28.117 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4926999999999997e-05 2023-09-23T23:35:28.117 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.117 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.117 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.118 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135628+0000", 2023-09-23T23:35:28.118 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0299999999999999e-06 2023-09-23T23:35:28.118 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.118 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.118 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.118 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135631+0000", 2023-09-23T23:35:28.118 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.841e-06 2023-09-23T23:35:28.118 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.118 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.118 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.118 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135680+0000", 2023-09-23T23:35:28.118 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9818999999999999e-05 2023-09-23T23:35:28.119 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.119 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.119 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.119 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135691+0000", 2023-09-23T23:35:28.119 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0976e-05 2023-09-23T23:35:28.119 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.119 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.119 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.119 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189484+0000", 2023-09-23T23:35:28.119 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.053793092 2023-09-23T23:35:28.119 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.120 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.120 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:35:28.120 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189619+0000", 2023-09-23T23:35:28.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013436399999999999 2023-09-23T23:35:28.120 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.120 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.120 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:35:28.120 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189777+0000", 2023-09-23T23:35:28.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015826999999999999 2023-09-23T23:35:28.120 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.120 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.120 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:35:28.120 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191857+0000", 2023-09-23T23:35:28.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020803929999999998 2023-09-23T23:35:28.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191914+0000", 2023-09-23T23:35:28.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6075000000000001e-05 2023-09-23T23:35:28.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194427+0000", 2023-09-23T23:35:28.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002513684 2023-09-23T23:35:28.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.122 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:35:28.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194468+0000", 2023-09-23T23:35:28.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1183999999999997e-05 2023-09-23T23:35:28.122 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.122 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.122 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194503+0000", 2023-09-23T23:35:28.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4867999999999999e-05 2023-09-23T23:35:28.122 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.122 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.122 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.122 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.123 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.123 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21659 13.5 13:aad23106:::benchmark_data_smithi110_541928_object21658:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:35:28.123 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.150190+0000", 2023-09-23T23:35:28.123 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 137.83618881699999, 2023-09-23T23:35:28.123 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0444529309999999, 2023-09-23T23:35:28.123 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.123 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:35:28.123 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.123 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:35:28.123 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:35:28.123 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21659 2023-09-23T23:35:28.123 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.124 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.124 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.124 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.124 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150190+0000", 2023-09-23T23:35:28.124 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.124 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.124 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.124 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.124 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150190+0000", 2023-09-23T23:35:28.124 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.124 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.124 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.124 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.125 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150189+0000", 2023-09-23T23:35:28.125 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:35:28.125 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.125 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.125 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.125 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150260+0000", 2023-09-23T23:35:28.125 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1550000000000004e-05 2023-09-23T23:35:28.125 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.125 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.125 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.125 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150272+0000", 2023-09-23T23:35:28.125 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1328000000000001e-05 2023-09-23T23:35:28.126 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.126 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.126 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.126 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150276+0000", 2023-09-23T23:35:28.126 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6700000000000002e-06 2023-09-23T23:35:28.126 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.126 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.126 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.126 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150329+0000", 2023-09-23T23:35:28.126 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3118999999999997e-05 2023-09-23T23:35:28.126 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.126 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.126 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.127 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150339+0000", 2023-09-23T23:35:28.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4660000000000005e-06 2023-09-23T23:35:28.127 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.127 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.127 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.127 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.190332+0000", 2023-09-23T23:35:28.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.039992633 2023-09-23T23:35:28.127 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.127 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.127 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:35:28.127 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.190453+0000", 2023-09-23T23:35:28.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012097899999999999 2023-09-23T23:35:28.128 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.128 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:35:28.128 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.190560+0000", 2023-09-23T23:35:28.128 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107587 2023-09-23T23:35:28.128 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.128 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:35:28.128 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.192645+0000", 2023-09-23T23:35:28.128 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020851329999999999 2023-09-23T23:35:28.128 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.128 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.129 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193487+0000", 2023-09-23T23:35:28.129 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00084168199999999996 2023-09-23T23:35:28.129 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.129 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.129 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.129 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194573+0000", 2023-09-23T23:35:28.129 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001085978 2023-09-23T23:35:28.129 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.129 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.129 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:35:28.129 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194610+0000", 2023-09-23T23:35:28.129 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7512000000000003e-05 2023-09-23T23:35:28.129 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.130 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.130 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.130 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194643+0000", 2023-09-23T23:35:28.130 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2147000000000002e-05 2023-09-23T23:35:28.130 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.130 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.130 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.130 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.130 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.130 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21680 13.5 13:ac31eb73:::benchmark_data_smithi110_541928_object21679:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:35:28.130 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.170348+0000", 2023-09-23T23:35:28.130 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 137.81602999099999, 2023-09-23T23:35:28.130 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0244998839999999, 2023-09-23T23:35:28.131 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.131 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:35:28.131 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.131 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:35:28.131 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:35:28.131 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21680 2023-09-23T23:35:28.131 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.131 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.131 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.131 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.131 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170348+0000", 2023-09-23T23:35:28.131 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.131 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.132 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.132 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.132 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170348+0000", 2023-09-23T23:35:28.132 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.132 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.132 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.132 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170347+0000", 2023-09-23T23:35:28.132 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:35:28.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.132 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.132 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.132 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170403+0000", 2023-09-23T23:35:28.133 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6042000000000003e-05 2023-09-23T23:35:28.133 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.133 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.133 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170415+0000", 2023-09-23T23:35:28.133 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2238e-05 2023-09-23T23:35:28.133 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.133 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.133 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170423+0000", 2023-09-23T23:35:28.133 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1829999999999997e-06 2023-09-23T23:35:28.133 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.134 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.134 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170480+0000", 2023-09-23T23:35:28.134 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7476999999999999e-05 2023-09-23T23:35:28.134 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.134 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.134 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.134 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170488+0000", 2023-09-23T23:35:28.134 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2390000000000002e-06 2023-09-23T23:35:28.134 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.134 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.134 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.134 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191062+0000", 2023-09-23T23:35:28.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0205732489999999 2023-09-23T23:35:28.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:35:28.135 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191141+0000", 2023-09-23T23:35:28.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9435000000000001e-05 2023-09-23T23:35:28.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:35:28.135 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191244+0000", 2023-09-23T23:35:28.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010341299999999999 2023-09-23T23:35:28.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.136 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:35:28.136 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.192657+0000", 2023-09-23T23:35:28.136 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014124599999999999 2023-09-23T23:35:28.136 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.136 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.136 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.136 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193656+0000", 2023-09-23T23:35:28.136 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00099945200000000007 2023-09-23T23:35:28.136 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.136 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.136 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.136 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194770+0000", 2023-09-23T23:35:28.136 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001114163 2023-09-23T23:35:28.137 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.137 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.137 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:35:28.137 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194815+0000", 2023-09-23T23:35:28.137 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4583999999999999e-05 2023-09-23T23:35:28.137 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.137 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.137 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.137 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194848+0000", 2023-09-23T23:35:28.137 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3272000000000003e-05 2023-09-23T23:35:28.137 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.137 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.137 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.138 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.138 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.138 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21693 13.5 13:a03052c3:::benchmark_data_smithi110_541928_object21692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:35:28.138 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.193329+0000", 2023-09-23T23:35:28.138 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 137.79304985100001, 2023-09-23T23:35:28.138 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.001638582, 2023-09-23T23:35:28.138 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:35:28.138 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:35:28.138 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:35:28.138 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:35:28.138 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:35:28.138 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21693 2023-09-23T23:35:28.139 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.139 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:35:28.139 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.139 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:35:28.139 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193329+0000", 2023-09-23T23:35:28.139 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.139 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.139 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.139 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:35:28.139 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193329+0000", 2023-09-23T23:35:28.139 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:35:28.139 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.140 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.140 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:35:28.140 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193328+0000", 2023-09-23T23:35:28.140 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:35:28.140 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.140 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.140 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:35:28.140 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193375+0000", 2023-09-23T23:35:28.140 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7132000000000002e-05 2023-09-23T23:35:28.140 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.140 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.140 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:35:28.140 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193390+0000", 2023-09-23T23:35:28.141 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4841000000000001e-05 2023-09-23T23:35:28.141 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.141 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.141 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:35:28.141 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193396+0000", 2023-09-23T23:35:28.141 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1299999999999998e-06 2023-09-23T23:35:28.141 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.141 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.141 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.141 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193442+0000", 2023-09-23T23:35:28.141 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6621e-05 2023-09-23T23:35:28.141 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.141 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.142 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:35:28.142 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193450+0000", 2023-09-23T23:35:28.142 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1079999999999993e-06 2023-09-23T23:35:28.142 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.142 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.142 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:35:28.142 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191950+0000", 2023-09-23T23:35:28.142 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99849948600000005 2023-09-23T23:35:28.142 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.142 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.142 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:35:28.142 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.192054+0000", 2023-09-23T23:35:28.142 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103933 2023-09-23T23:35:28.143 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.143 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.143 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:35:28.143 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.192153+0000", 2023-09-23T23:35:28.143 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9110999999999993e-05 2023-09-23T23:35:28.143 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.143 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.143 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:35:28.143 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193631+0000", 2023-09-23T23:35:28.143 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014778619999999999 2023-09-23T23:35:28.143 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.143 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.143 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.144 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193884+0000", 2023-09-23T23:35:28.144 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025308299999999998 2023-09-23T23:35:28.144 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.144 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.144 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:35:28.144 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194896+0000", 2023-09-23T23:35:28.144 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010124159999999999 2023-09-23T23:35:28.144 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.144 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.144 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:35:28.144 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194935+0000", 2023-09-23T23:35:28.144 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8470000000000003e-05 2023-09-23T23:35:28.145 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:35:28.145 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:35:28.145 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:35:28.145 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194967+0000", 2023-09-23T23:35:28.145 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2385999999999998e-05 2023-09-23T23:35:28.145 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.145 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.145 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.145 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:35:28.145 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:35:28.145 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:35:28.146 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:28.199 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:28.264 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:28.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:28.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:28.369 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:28.397 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:28.441 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:28.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:28.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:28.545 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:28.573 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:28.792 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:35:28.804 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:28.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:28.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:28.857 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:28.909 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:28.935 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:28.985 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:28.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:28.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:29.037 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:35:29.061 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:29 smithi110 ceph-mon[212939]: pgmap v1614: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 178 op/s; 27065/224289 objects degraded (12.067%); 4520/224289 objects misplaced (2.015%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:35:29.061 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:29 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:29.061 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:29 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:35:29.061 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:29 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4111781458' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:35:29.061 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:29 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 27065/224289 objects degraded (12.067%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:29.062 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:29 smithi110 ceph-mon[221102]: pgmap v1614: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 178 op/s; 27065/224289 objects degraded (12.067%); 4520/224289 objects misplaced (2.015%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:35:29.062 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:29 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:29.062 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:29 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:35:29.062 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:29 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4111781458' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:35:29.062 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:29 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 27065/224289 objects degraded (12.067%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:29.089 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:29.117 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:29.168 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:29.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:29.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:29.203 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:35:29.275 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:29.299 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:29 smithi138 ceph-mon[193606]: pgmap v1614: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 178 op/s; 27065/224289 objects degraded (12.067%); 4520/224289 objects misplaced (2.015%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:35:29.299 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:29 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:29.299 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:29 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:35:29.299 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:29 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4111781458' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:35:29.299 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:29 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 27065/224289 objects degraded (12.067%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:29.302 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:29.343 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:29.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:29.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:29.447 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:29.476 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:29.519 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:29.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:29.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:29.622 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:29.662 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:30.070 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:35:30.090 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:35:30.112 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:35:30.140 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:35:30.157 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:30 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:35:30.157 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:30 smithi110 ceph-mon[212939]: failed to encode map e587 with expected crc 2023-09-23T23:35:30.157 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:30 smithi110 ceph-mon[212939]: failed to encode map e587 with expected crc 2023-09-23T23:35:30.157 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:30 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4111781458' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:35:30.157 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:30 smithi110 ceph-mon[212939]: osdmap e587: 8 total, 8 up, 7 in 2023-09-23T23:35:30.158 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:30 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:35:30.158 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:30 smithi110 ceph-mon[221102]: failed to encode map e587 with expected crc 2023-09-23T23:35:30.158 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:30 smithi110 ceph-mon[221102]: failed to encode map e587 with expected crc 2023-09-23T23:35:30.158 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:30 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4111781458' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:35:30.158 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:30 smithi110 ceph-mon[221102]: osdmap e587: 8 total, 8 up, 7 in 2023-09-23T23:35:30.167 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:30.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:30.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:30.177 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:35:30.272 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:30.297 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:30 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:35:30.297 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:30 smithi138 ceph-mon[193606]: failed to encode map e587 with expected crc 2023-09-23T23:35:30.297 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:30 smithi138 ceph-mon[193606]: failed to encode map e587 with expected crc 2023-09-23T23:35:30.298 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:30 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4111781458' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:35:30.298 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:30 smithi138 ceph-mon[193606]: osdmap e587: 8 total, 8 up, 7 in 2023-09-23T23:35:30.301 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:30.319 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:35:30.320 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:35:30.320 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:35:30.320 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:35:30.320 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:35:30.320 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:35:30.320 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:35:30.320 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:35:30.320 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:35:30.321 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:35:30.321 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:35:30.437 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:30.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:30.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:30.548 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:30.586 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:30.806 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:30.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:30.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:30.910 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:30.944 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:30.972 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:35:30.972 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:35:30.993 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:30.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:30.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:31.097 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:31.128 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:31.187 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:31.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:31.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:31.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:31 smithi110 ceph-mon[212939]: pgmap v1616: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 204 op/s; 26987/222345 objects degraded (12.137%); 4520/222345 objects misplaced (2.033%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:35:31.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:31 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/647869951' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:31.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:31 smithi110 ceph-mon[221102]: pgmap v1616: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 204 op/s; 26987/222345 objects degraded (12.137%); 4520/222345 objects misplaced (2.033%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:35:31.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:31 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/647869951' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:31.291 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:31.318 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:31 smithi138 ceph-mon[193606]: pgmap v1616: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 204 op/s; 26987/222345 objects degraded (12.137%); 4520/222345 objects misplaced (2.033%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:35:31.319 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:31 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/647869951' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:31.319 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:31.360 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:31.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:31.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:31.464 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:31.498 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:31.610 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:31.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:31.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:31.714 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:31.744 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:31.779 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:35:31.793 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:31.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:31.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:31.895 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:35:31.897 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:31.926 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:31.933 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:35:31.935 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:35:31.974 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:31.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:31.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:32.078 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:32.111 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:32.143 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:35:32.166 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:32.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:32.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:32.277 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:32.303 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:32.565 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:32.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:32.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:32.669 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:32.679 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1617,"stamp":"2023-09-23T23:35:31.904605+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4842632437,"num_objects":73908,"num_object_clones":0,"num_object_copies":221724,"num_objects_missing_on_primary":3625,"num_objects_missing":3625,"num_objects_degraded":26987,"num_objects_misplaced":4520,"num_objects_unfound":0,"num_objects_dirty":73908,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":101749,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12087,"num_bytes_recovered":789442984,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94796,"ondisk_log_size":94796,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18045316,"kb_used_data":12463308,"kb_used_omap":29,"kb_used_meta":5581858,"kb_avail":638170748,"statfs":{"total":671965249536,"available":653486845952,"internally_reserved":0,"allocated":12762427392,"data_stored":12746484480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5715823315},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,0,2,0,2,4,2,8],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-127664128,"num_objects":-1948,"num_object_clones":0,"num_object_copies":-5844,"num_objects_missing_on_primary":-132,"num_objects_missing":-132,"num_objects_degraded":-248,"num_objects_misplaced":-132,"num_objects_unfound":0,"num_objects_dirty":-1948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1948,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":381,"num_bytes_recovered":24969216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003028"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":783,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.415875+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:35:21.415875+0000","last_peered":"2023-09-23T23:35:21.415875+0000","last_clean":"2023-09-23T23:35:21.415875+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:35:21.415875+0000","last_undegraded":"2023-09-23T23:35:21.415875+0000","last_fullsized":"2023-09-23T23:35:21.415875+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":341,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.404977+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:35:21.404977+0000","last_peered":"2023-09-23T23:35:21.404977+0000","last_clean":"2023-09-23T23:35:21.404977+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:35:21.404977+0000","last_undegraded":"2023-09-23T23:35:21.404977+0000","last_fullsized":"2023-09-23T23:35:21.404977+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":567,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.452941+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:35:15.452941+0000","last_peered":"2023-09-23T23:35:15.452941+0000","last_clean":"2023-09-23T23:35:15.452941+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:35:15.452941+0000","last_undegraded":"2023-09-23T23:35:15.452941+0000","last_fullsized":"2023-09-23T23:35:15.452941+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":736,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.416124+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:35:21.416124+0000","last_peered":"2023-09-23T23:35:21.416124+0000","last_clean":"2023-09-23T23:35:21.416124+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:35:21.416124+0000","last_undegraded":"2023-09-23T23:35:21.416124+0000","last_fullsized":"2023-09-23T23:35:21.416124+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":304,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417918+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:35:15.417918+0000","last_peered":"2023-09-23T23:35:15.417918+0000","last_clean":"2023-09-23T23:35:15.417918+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:35:15.417918+0000","last_undegraded":"2023-09-23T23:35:15.417918+0000","last_fullsized":"2023-09-23T23:35:15.417918+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":546,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.422841+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:35:15.422841+0000","last_peered":"2023-09-23T23:35:15.422841+0000","last_clean":"2023-09-23T23:35:15.422841+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:35:15.422841+0000","last_undegraded":"2023-09-23T23:35:15.422841+0000","last_fullsized":"2023-09-23T23:35:15.422841+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":747,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.426356+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:35:15.426356+0000","last_peered":"2023-09-23T23:35:15.426356+0000","last_clean":"2023-09-23T23:35:15.426356+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:35:15.426356+0000","last_undegraded":"2023-09-23T23:35:15.426356+0000","last_fullsized":"2023-09-23T23:35:15.426356+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":604,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417733+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:35:15.417733+0000","last_peered":"2023-09-23T23:35:15.417733+0000","last_clean":"2023-09-23T23:35:15.417733+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:35:15.417733+0000","last_undegraded":"2023-09-23T23:35:15.417733+0000","last_fullsized":"2023-09-23T23:35:15.417733+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":101,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.400358+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:35:21.400358+0000","last_peered":"2023-09-23T23:35:21.400358+0000","last_clean":"2023-09-23T23:35:21.400358+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:35:21.400358+0000","last_undegraded":"2023-09-23T23:35:21.400358+0000","last_fullsized":"2023-09-23T23:35:21.400358+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":906,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.426615+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:35:15.426615+0000","last_peered":"2023-09-23T23:35:15.426615+0000","last_clean":"2023-09-23T23:35:15.426615+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:35:15.426615+0000","last_undegraded":"2023-09-23T23:35:15.426615+0000","last_fullsized":"2023-09-23T23:35:15.426615+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":239,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.420975+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:35:21.420975+0000","last_peered":"2023-09-23T23:35:21.420975+0000","last_clean":"2023-09-23T23:35:21.420975+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:35:21.420975+0000","last_undegraded":"2023-09-23T23:35:21.420975+0000","last_fullsized":"2023-09-23T23:35:21.420975+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":586,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417704+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:35:15.417704+0000","last_peered":"2023-09-23T23:35:15.417704+0000","last_clean":"2023-09-23T23:35:15.417704+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:35:15.417704+0000","last_undegraded":"2023-09-23T23:35:15.417704+0000","last_fullsized":"2023-09-23T23:35:15.417704+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":880,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.066146+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:35:29.066146+0000","last_peered":"2023-09-23T23:35:29.066146+0000","last_clean":"2023-09-23T23:35:29.066146+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:35:29.066146+0000","last_undegraded":"2023-09-23T23:35:29.066146+0000","last_fullsized":"2023-09-23T23:35:29.066146+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":218,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.420869+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:35:21.420869+0000","last_peered":"2023-09-23T23:35:21.420869+0000","last_clean":"2023-09-23T23:35:21.420869+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:35:21.420869+0000","last_undegraded":"2023-09-23T23:35:21.420869+0000","last_fullsized":"2023-09-23T23:35:21.420869+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":547,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417819+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:35:15.417819+0000","last_peered":"2023-09-23T23:35:15.417819+0000","last_clean":"2023-09-23T23:35:15.417819+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:35:15.417819+0000","last_undegraded":"2023-09-23T23:35:15.417819+0000","last_fullsized":"2023-09-23T23:35:15.417819+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1041,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.420958+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:35:21.420958+0000","last_peered":"2023-09-23T23:35:21.420958+0000","last_clean":"2023-09-23T23:35:21.420958+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:35:21.420958+0000","last_undegraded":"2023-09-23T23:35:21.420958+0000","last_fullsized":"2023-09-23T23:35:21.420958+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":258,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.432397+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:35:15.432397+0000","last_peered":"2023-09-23T23:35:15.432397+0000","last_clean":"2023-09-23T23:35:15.432397+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:35:15.432397+0000","last_undegraded":"2023-09-23T23:35:15.432397+0000","last_fullsized":"2023-09-23T23:35:15.432397+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":120,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.394655+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:35:21.394655+0000","last_peered":"2023-09-23T23:35:21.394655+0000","last_clean":"2023-09-23T23:35:21.394655+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:35:21.394655+0000","last_undegraded":"2023-09-23T23:35:21.394655+0000","last_fullsized":"2023-09-23T23:35:21.394655+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":436,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.423138+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:35:15.423138+0000","last_peered":"2023-09-23T23:35:15.423138+0000","last_clean":"2023-09-23T23:35:15.423138+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:35:15.423138+0000","last_undegraded":"2023-09-23T23:35:15.423138+0000","last_fullsized":"2023-09-23T23:35:15.423138+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":161,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.395313+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:35:21.395313+0000","last_peered":"2023-09-23T23:35:21.395313+0000","last_clean":"2023-09-23T23:35:21.395313+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:35:21.395313+0000","last_undegraded":"2023-09-23T23:35:21.395313+0000","last_fullsized":"2023-09-23T23:35:21.395313+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":528,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.395748+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:35:21.395748+0000","last_peered":"2023-09-23T23:35:21.395748+0000","last_clean":"2023-09-23T23:35:21.395748+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:35:21.395748+0000","last_undegraded":"2023-09-23T23:35:21.395748+0000","last_fullsized":"2023-09-23T23:35:21.395748+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":742,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.415834+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:35:21.415834+0000","last_peered":"2023-09-23T23:35:21.415834+0000","last_clean":"2023-09-23T23:35:21.415834+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:35:21.415834+0000","last_undegraded":"2023-09-23T23:35:21.415834+0000","last_fullsized":"2023-09-23T23:35:21.415834+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":133,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.400830+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:35:21.400830+0000","last_peered":"2023-09-23T23:35:21.400830+0000","last_clean":"2023-09-23T23:35:21.400830+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:35:21.400830+0000","last_undegraded":"2023-09-23T23:35:21.400830+0000","last_fullsized":"2023-09-23T23:35:21.400830+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":585,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.422831+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:35:15.422831+0000","last_peered":"2023-09-23T23:35:15.422831+0000","last_clean":"2023-09-23T23:35:15.422831+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:35:15.422831+0000","last_undegraded":"2023-09-23T23:35:15.422831+0000","last_fullsized":"2023-09-23T23:35:15.422831+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":872,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.394292+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:35:21.394292+0000","last_peered":"2023-09-23T23:35:21.394292+0000","last_clean":"2023-09-23T23:35:21.394292+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:35:21.394292+0000","last_undegraded":"2023-09-23T23:35:21.394292+0000","last_fullsized":"2023-09-23T23:35:21.394292+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":180,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417996+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:35:15.417996+0000","last_peered":"2023-09-23T23:35:15.417996+0000","last_clean":"2023-09-23T23:35:15.417996+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:35:15.417996+0000","last_undegraded":"2023-09-23T23:35:15.417996+0000","last_fullsized":"2023-09-23T23:35:15.417996+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":307,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.438250+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:35:15.438250+0000","last_peered":"2023-09-23T23:35:15.438250+0000","last_clean":"2023-09-23T23:35:15.438250+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:35:15.438250+0000","last_undegraded":"2023-09-23T23:35:15.438250+0000","last_fullsized":"2023-09-23T23:35:15.438250+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":548,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.418722+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:35:15.418722+0000","last_peered":"2023-09-23T23:35:15.418722+0000","last_clean":"2023-09-23T23:35:15.418722+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:35:15.418722+0000","last_undegraded":"2023-09-23T23:35:15.418722+0000","last_fullsized":"2023-09-23T23:35:15.418722+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":455,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.395196+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:35:21.395196+0000","last_peered":"2023-09-23T23:35:21.395196+0000","last_clean":"2023-09-23T23:35:21.395196+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:35:21.395196+0000","last_undegraded":"2023-09-23T23:35:21.395196+0000","last_fullsized":"2023-09-23T23:35:21.395196+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":541,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.423236+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:35:15.423236+0000","last_peered":"2023-09-23T23:35:15.423236+0000","last_clean":"2023-09-23T23:35:15.423236+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:35:15.423236+0000","last_undegraded":"2023-09-23T23:35:15.423236+0000","last_fullsized":"2023-09-23T23:35:15.423236+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":950,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.415990+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:35:21.415990+0000","last_peered":"2023-09-23T23:35:21.415990+0000","last_clean":"2023-09-23T23:35:21.415990+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:35:21.415990+0000","last_undegraded":"2023-09-23T23:35:21.415990+0000","last_fullsized":"2023-09-23T23:35:21.415990+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":151,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.394520+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:35:21.394520+0000","last_peered":"2023-09-23T23:35:21.394520+0000","last_clean":"2023-09-23T23:35:21.394520+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:35:21.394520+0000","last_undegraded":"2023-09-23T23:35:21.394520+0000","last_fullsized":"2023-09-23T23:35:21.394520+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":226,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.423066+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:35:15.423066+0000","last_peered":"2023-09-23T23:35:15.423066+0000","last_clean":"2023-09-23T23:35:15.423066+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:35:15.423066+0000","last_undegraded":"2023-09-23T23:35:15.423066+0000","last_fullsized":"2023-09-23T23:35:15.423066+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"586'6309","reported_seq":6379,"reported_epoch":586,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:28.910429+0000","last_change":"2023-09-23T23:33:52.884769+0000","last_active":"2023-09-23T23:35:28.910429+0000","last_peered":"2023-09-23T23:35:28.910429+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:35:28.910429+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"543'900","ondisk_log_start":"543'900","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5409,"log_dups_size":0,"ondisk_log_size":5409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297467904,"num_objects":4539,"num_object_clones":0,"num_object_copies":13617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6309,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":3513},{"shards":"2,6","objects":141}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":920,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.401137+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:35:21.401137+0000","last_peered":"2023-09-23T23:35:21.401137+0000","last_clean":"2023-09-23T23:35:21.401137+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:35:21.401137+0000","last_undegraded":"2023-09-23T23:35:21.401137+0000","last_fullsized":"2023-09-23T23:35:21.401137+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":227,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.426772+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:35:15.426772+0000","last_peered":"2023-09-23T23:35:15.426772+0000","last_clean":"2023-09-23T23:35:15.426772+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:35:15.426772+0000","last_undegraded":"2023-09-23T23:35:15.426772+0000","last_fullsized":"2023-09-23T23:35:15.426772+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":218,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.415826+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:35:21.415826+0000","last_peered":"2023-09-23T23:35:21.415826+0000","last_clean":"2023-09-23T23:35:21.415826+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:35:21.415826+0000","last_undegraded":"2023-09-23T23:35:21.415826+0000","last_fullsized":"2023-09-23T23:35:21.415826+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"587'6397","reported_seq":6466,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.771638+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:35:29.771638+0000","last_peered":"2023-09-23T23:35:29.771638+0000","last_clean":"2023-09-23T23:35:29.771638+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:35:29.771638+0000","last_undegraded":"2023-09-23T23:35:29.771638+0000","last_fullsized":"2023-09-23T23:35:29.771638+0000","mapping_epoch":566,"log_start":"538'500","ondisk_log_start":"538'500","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5897,"log_dups_size":0,"ondisk_log_size":5897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303759360,"num_objects":4635,"num_object_clones":0,"num_object_copies":13905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6397,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":997,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417817+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:35:15.417817+0000","last_peered":"2023-09-23T23:35:15.417817+0000","last_clean":"2023-09-23T23:35:15.417817+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:35:15.417817+0000","last_undegraded":"2023-09-23T23:35:15.417817+0000","last_fullsized":"2023-09-23T23:35:15.417817+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":217,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.426712+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:35:15.426712+0000","last_peered":"2023-09-23T23:35:15.426712+0000","last_clean":"2023-09-23T23:35:15.426712+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:35:15.426712+0000","last_undegraded":"2023-09-23T23:35:15.426712+0000","last_fullsized":"2023-09-23T23:35:15.426712+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":436,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417629+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:35:15.417629+0000","last_peered":"2023-09-23T23:35:15.417629+0000","last_clean":"2023-09-23T23:35:15.417629+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:35:15.417629+0000","last_undegraded":"2023-09-23T23:35:15.417629+0000","last_fullsized":"2023-09-23T23:35:15.417629+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"586'6397","reported_seq":9246,"reported_epoch":586,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:35:26.893600+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:35:26.893600+0000","last_peered":"2023-09-23T23:35:26.893600+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:35:26.893600+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:35:26.893600+0000","mapping_epoch":571,"log_start":"544'1100","ondisk_log_start":"544'1100","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5297,"log_dups_size":0,"ondisk_log_size":5297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304807936,"num_objects":4651,"num_object_clones":0,"num_object_copies":13953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3756,"num_objects_misplaced":895,"num_objects_unfound":0,"num_objects_dirty":4651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6397,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1375,"num_bytes_recovered":90112000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":3634},{"shards":"3,5,7","objects":1017}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":897,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.400410+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:35:21.400410+0000","last_peered":"2023-09-23T23:35:21.400410+0000","last_clean":"2023-09-23T23:35:21.400410+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:35:21.400410+0000","last_undegraded":"2023-09-23T23:35:21.400410+0000","last_fullsized":"2023-09-23T23:35:21.400410+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":325,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.416379+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:35:21.416379+0000","last_peered":"2023-09-23T23:35:21.416379+0000","last_clean":"2023-09-23T23:35:21.416379+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:35:21.416379+0000","last_undegraded":"2023-09-23T23:35:21.416379+0000","last_fullsized":"2023-09-23T23:35:21.416379+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":101,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.394892+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:35:21.394892+0000","last_peered":"2023-09-23T23:35:21.394892+0000","last_clean":"2023-09-23T23:35:21.394892+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:35:21.394892+0000","last_undegraded":"2023-09-23T23:35:21.394892+0000","last_fullsized":"2023-09-23T23:35:21.394892+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"586'6357","reported_seq":8412,"reported_epoch":586,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:26.781887+0000","last_change":"2023-09-23T23:34:35.750428+0000","last_active":"2023-09-23T23:35:26.781887+0000","last_peered":"2023-09-23T23:35:26.781887+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:35:26.781887+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"544'1000","ondisk_log_start":"544'1000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5357,"log_dups_size":0,"ondisk_log_size":5357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310181888,"num_objects":4733,"num_object_clones":0,"num_object_copies":14199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6357,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":991,"num_bytes_recovered":64946176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":3544},{"shards":"3,5","objects":1189}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":492,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.423194+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:35:15.423194+0000","last_peered":"2023-09-23T23:35:15.423194+0000","last_clean":"2023-09-23T23:35:15.423194+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:35:15.423194+0000","last_undegraded":"2023-09-23T23:35:15.423194+0000","last_fullsized":"2023-09-23T23:35:15.423194+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":426,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.422758+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:35:15.422758+0000","last_peered":"2023-09-23T23:35:15.422758+0000","last_clean":"2023-09-23T23:35:15.422758+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:35:15.422758+0000","last_undegraded":"2023-09-23T23:35:15.422758+0000","last_fullsized":"2023-09-23T23:35:15.422758+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":583,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.417945+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:35:21.417945+0000","last_peered":"2023-09-23T23:35:21.417945+0000","last_clean":"2023-09-23T23:35:21.417945+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:35:21.417945+0000","last_undegraded":"2023-09-23T23:35:21.417945+0000","last_fullsized":"2023-09-23T23:35:21.417945+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"586'6475","reported_seq":7792,"reported_epoch":586,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:28.962020+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:35:28.962020+0000","last_peered":"2023-09-23T23:35:28.962020+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:35:28.962020+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"542'800","ondisk_log_start":"542'800","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5675,"log_dups_size":0,"ondisk_log_size":5675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307167232,"num_objects":4687,"num_object_clones":0,"num_object_copies":14061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6475,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":3641},{"shards":"3,5","objects":-773},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":713,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417787+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:35:15.417787+0000","last_peered":"2023-09-23T23:35:15.417787+0000","last_clean":"2023-09-23T23:35:15.417787+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:35:15.417787+0000","last_undegraded":"2023-09-23T23:35:15.417787+0000","last_fullsized":"2023-09-23T23:35:15.417787+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":595,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.418673+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:35:15.418673+0000","last_peered":"2023-09-23T23:35:15.418673+0000","last_clean":"2023-09-23T23:35:15.418673+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:35:15.418673+0000","last_undegraded":"2023-09-23T23:35:15.418673+0000","last_fullsized":"2023-09-23T23:35:15.418673+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":506,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.422667+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:35:15.422667+0000","last_peered":"2023-09-23T23:35:15.422667+0000","last_clean":"2023-09-23T23:35:15.422667+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:35:15.422667+0000","last_undegraded":"2023-09-23T23:35:15.422667+0000","last_fullsized":"2023-09-23T23:35:15.422667+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"586'6400","reported_seq":6461,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.014855+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:35:29.014855+0000","last_peered":"2023-09-23T23:35:29.014855+0000","last_clean":"2023-09-23T23:35:29.014855+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:35:29.014855+0000","last_undegraded":"2023-09-23T23:35:29.014855+0000","last_fullsized":"2023-09-23T23:35:29.014855+0000","mapping_epoch":534,"log_start":"544'1000","ondisk_log_start":"544'1000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5400,"log_dups_size":0,"ondisk_log_size":5400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306708480,"num_objects":4680,"num_object_clones":0,"num_object_copies":14040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4680,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6400,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":870,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.062481+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:35:29.062481+0000","last_peered":"2023-09-23T23:35:29.062481+0000","last_clean":"2023-09-23T23:35:29.062481+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:35:29.062481+0000","last_undegraded":"2023-09-23T23:35:29.062481+0000","last_fullsized":"2023-09-23T23:35:29.062481+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":111,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.394816+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:35:21.394816+0000","last_peered":"2023-09-23T23:35:21.394816+0000","last_clean":"2023-09-23T23:35:21.394816+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:35:21.394816+0000","last_undegraded":"2023-09-23T23:35:21.394816+0000","last_fullsized":"2023-09-23T23:35:21.394816+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":452,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.401241+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:35:21.401241+0000","last_peered":"2023-09-23T23:35:21.401241+0000","last_clean":"2023-09-23T23:35:21.401241+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:35:21.401241+0000","last_undegraded":"2023-09-23T23:35:21.401241+0000","last_fullsized":"2023-09-23T23:35:21.401241+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"586'6484","reported_seq":8429,"reported_epoch":587,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:35:29.047932+0000","last_change":"2023-09-23T23:34:35.785270+0000","last_active":"2023-09-23T23:35:29.047932+0000","last_peered":"2023-09-23T23:35:29.047932+0000","last_clean":"2023-09-23T23:33:50.504296+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:35:29.047932+0000","last_undegraded":"2023-09-23T23:34:35.780981+0000","last_fullsized":"2023-09-23T23:35:29.047932+0000","mapping_epoch":567,"log_start":"544'1100","ondisk_log_start":"544'1100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5384,"log_dups_size":0,"ondisk_log_size":5384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302776320,"num_objects":4620,"num_object_clones":0,"num_object_copies":13860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6484,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":932,"num_bytes_recovered":61079552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":3615},{"shards":"5,6,7","objects":1005}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":595,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.416116+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:35:21.416116+0000","last_peered":"2023-09-23T23:35:21.416116+0000","last_clean":"2023-09-23T23:35:21.416116+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:35:21.416116+0000","last_undegraded":"2023-09-23T23:35:21.416116+0000","last_fullsized":"2023-09-23T23:35:21.416116+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":449,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.400831+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:35:21.400831+0000","last_peered":"2023-09-23T23:35:21.400831+0000","last_clean":"2023-09-23T23:35:21.400831+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:35:21.400831+0000","last_undegraded":"2023-09-23T23:35:21.400831+0000","last_fullsized":"2023-09-23T23:35:21.400831+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":538,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.395574+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:35:21.395574+0000","last_peered":"2023-09-23T23:35:21.395574+0000","last_clean":"2023-09-23T23:35:21.395574+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:35:21.395574+0000","last_undegraded":"2023-09-23T23:35:21.395574+0000","last_fullsized":"2023-09-23T23:35:21.395574+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"587'6429","reported_seq":15453,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.097011+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:35:29.097011+0000","last_peered":"2023-09-23T23:35:29.097011+0000","last_clean":"2023-09-23T23:35:29.097011+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:35:29.097011+0000","last_undegraded":"2023-09-23T23:35:29.097011+0000","last_fullsized":"2023-09-23T23:35:29.097011+0000","mapping_epoch":579,"log_start":"542'700","ondisk_log_start":"542'700","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5729,"log_dups_size":0,"ondisk_log_size":5729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300613632,"num_objects":4587,"num_object_clones":0,"num_object_copies":13761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6429,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":928,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.404843+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:35:21.404843+0000","last_peered":"2023-09-23T23:35:21.404843+0000","last_clean":"2023-09-23T23:35:21.404843+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:35:21.404843+0000","last_undegraded":"2023-09-23T23:35:21.404843+0000","last_fullsized":"2023-09-23T23:35:21.404843+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":227,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417625+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:35:15.417625+0000","last_peered":"2023-09-23T23:35:15.417625+0000","last_clean":"2023-09-23T23:35:15.417625+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:35:15.417625+0000","last_undegraded":"2023-09-23T23:35:15.417625+0000","last_fullsized":"2023-09-23T23:35:15.417625+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":515,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.423142+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:35:15.423142+0000","last_peered":"2023-09-23T23:35:15.423142+0000","last_clean":"2023-09-23T23:35:15.423142+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:35:15.423142+0000","last_undegraded":"2023-09-23T23:35:15.423142+0000","last_fullsized":"2023-09-23T23:35:15.423142+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"586'6151","reported_seq":6213,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:26.697750+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:35:26.697750+0000","last_peered":"2023-09-23T23:35:26.697750+0000","last_clean":"2023-09-23T23:35:26.697750+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:35:26.697750+0000","last_undegraded":"2023-09-23T23:35:26.697750+0000","last_fullsized":"2023-09-23T23:35:26.697750+0000","mapping_epoch":534,"log_start":"543'800","ondisk_log_start":"543'800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5351,"log_dups_size":0,"ondisk_log_size":5351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299958272,"num_objects":4577,"num_object_clones":0,"num_object_copies":13731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6151,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":908,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.396112+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:35:21.396112+0000","last_peered":"2023-09-23T23:35:21.396112+0000","last_clean":"2023-09-23T23:35:21.396112+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:35:21.396112+0000","last_undegraded":"2023-09-23T23:35:21.396112+0000","last_fullsized":"2023-09-23T23:35:21.396112+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":296,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.446448+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:35:21.446448+0000","last_peered":"2023-09-23T23:35:21.446448+0000","last_clean":"2023-09-23T23:35:21.446448+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:35:21.446448+0000","last_undegraded":"2023-09-23T23:35:21.446448+0000","last_fullsized":"2023-09-23T23:35:21.446448+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":602,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.416512+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:35:21.416512+0000","last_peered":"2023-09-23T23:35:21.416512+0000","last_clean":"2023-09-23T23:35:21.416512+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:35:21.416512+0000","last_undegraded":"2023-09-23T23:35:21.416512+0000","last_fullsized":"2023-09-23T23:35:21.416512+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":183,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.415847+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:35:21.415847+0000","last_peered":"2023-09-23T23:35:21.415847+0000","last_clean":"2023-09-23T23:35:21.415847+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:35:21.415847+0000","last_undegraded":"2023-09-23T23:35:21.415847+0000","last_fullsized":"2023-09-23T23:35:21.415847+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"586'6462","reported_seq":8311,"reported_epoch":586,"state":"active+recovering","last_fresh":"2023-09-23T23:35:26.937357+0000","last_change":"2023-09-23T23:34:35.815566+0000","last_active":"2023-09-23T23:35:26.937357+0000","last_peered":"2023-09-23T23:35:26.937357+0000","last_clean":"2023-09-23T23:33:50.519308+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:35:26.937357+0000","last_undegraded":"2023-09-23T23:35:26.937357+0000","last_fullsized":"2023-09-23T23:35:26.937357+0000","mapping_epoch":566,"log_start":"544'1122","ondisk_log_start":"544'1122","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5340,"log_dups_size":0,"ondisk_log_size":5340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308346904,"num_objects":4706,"num_object_clones":0,"num_object_copies":14118,"num_objects_missing_on_primary":3625,"num_objects_missing":3625,"num_objects_degraded":0,"num_objects_misplaced":3625,"num_objects_unfound":0,"num_objects_dirty":4706,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6462,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":877,"num_bytes_recovered":57475072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,3,4,7","objects":13},{"shards":"1,3,7","objects":1068},{"shards":"1,4,7","objects":3625}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":670,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.417261+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:35:21.417261+0000","last_peered":"2023-09-23T23:35:21.417261+0000","last_clean":"2023-09-23T23:35:21.417261+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:35:21.417261+0000","last_undegraded":"2023-09-23T23:35:21.417261+0000","last_fullsized":"2023-09-23T23:35:21.417261+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":235,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.394727+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:35:21.394727+0000","last_peered":"2023-09-23T23:35:21.394727+0000","last_clean":"2023-09-23T23:35:21.394727+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:35:21.394727+0000","last_undegraded":"2023-09-23T23:35:21.394727+0000","last_fullsized":"2023-09-23T23:35:21.394727+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":519,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.395856+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:35:21.395856+0000","last_peered":"2023-09-23T23:35:21.395856+0000","last_clean":"2023-09-23T23:35:21.395856+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:35:21.395856+0000","last_undegraded":"2023-09-23T23:35:21.395856+0000","last_fullsized":"2023-09-23T23:35:21.395856+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":219,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.405169+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:35:21.405169+0000","last_peered":"2023-09-23T23:35:21.405169+0000","last_clean":"2023-09-23T23:35:21.405169+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:35:21.405169+0000","last_undegraded":"2023-09-23T23:35:21.405169+0000","last_fullsized":"2023-09-23T23:35:21.405169+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"587'6192","reported_seq":8232,"reported_epoch":587,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:29.092607+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:35:29.092607+0000","last_peered":"2023-09-23T23:35:29.092607+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:35:29.092607+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"538'500","ondisk_log_start":"538'500","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5692,"log_dups_size":0,"ondisk_log_size":5692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300154880,"num_objects":4580,"num_object_clones":0,"num_object_copies":13740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6192,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":3392},{"shards":"5,7","objects":1188}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":888,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.404750+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:35:21.404750+0000","last_peered":"2023-09-23T23:35:21.404750+0000","last_clean":"2023-09-23T23:35:21.404750+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:35:21.404750+0000","last_undegraded":"2023-09-23T23:35:21.404750+0000","last_fullsized":"2023-09-23T23:35:21.404750+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":325,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.400756+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:35:21.400756+0000","last_peered":"2023-09-23T23:35:21.400756+0000","last_clean":"2023-09-23T23:35:21.400756+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:35:21.400756+0000","last_undegraded":"2023-09-23T23:35:21.400756+0000","last_fullsized":"2023-09-23T23:35:21.400756+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":234,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.400769+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:35:21.400769+0000","last_peered":"2023-09-23T23:35:21.400769+0000","last_clean":"2023-09-23T23:35:21.400769+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:35:21.400769+0000","last_undegraded":"2023-09-23T23:35:21.400769+0000","last_fullsized":"2023-09-23T23:35:21.400769+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":198,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.445348+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:35:21.445348+0000","last_peered":"2023-09-23T23:35:21.445348+0000","last_clean":"2023-09-23T23:35:21.445348+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:35:21.445348+0000","last_undegraded":"2023-09-23T23:35:21.445348+0000","last_fullsized":"2023-09-23T23:35:21.445348+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"587'6329","reported_seq":6400,"reported_epoch":587,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:35:29.034466+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:35:29.034466+0000","last_peered":"2023-09-23T23:35:29.034466+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:35:29.034466+0000","last_undegraded":"2023-09-23T23:35:29.034466+0000","last_fullsized":"2023-09-23T23:35:29.034466+0000","mapping_epoch":562,"log_start":"541'600","ondisk_log_start":"541'600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5729,"log_dups_size":0,"ondisk_log_size":5729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299696128,"num_objects":4573,"num_object_clones":0,"num_object_copies":13719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6329,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":4573}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":601,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.400879+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:35:21.400879+0000","last_peered":"2023-09-23T23:35:21.400879+0000","last_clean":"2023-09-23T23:35:21.400879+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:35:21.400879+0000","last_undegraded":"2023-09-23T23:35:21.400879+0000","last_fullsized":"2023-09-23T23:35:21.400879+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":873,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.394311+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:35:21.394311+0000","last_peered":"2023-09-23T23:35:21.394311+0000","last_clean":"2023-09-23T23:35:21.394311+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:35:21.394311+0000","last_undegraded":"2023-09-23T23:35:21.394311+0000","last_fullsized":"2023-09-23T23:35:21.394311+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":547,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417831+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:35:15.417831+0000","last_peered":"2023-09-23T23:35:15.417831+0000","last_clean":"2023-09-23T23:35:15.417831+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:35:15.417831+0000","last_undegraded":"2023-09-23T23:35:15.417831+0000","last_fullsized":"2023-09-23T23:35:15.417831+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":111,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.395479+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:35:21.395479+0000","last_peered":"2023-09-23T23:35:21.395479+0000","last_clean":"2023-09-23T23:35:21.395479+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:35:21.395479+0000","last_undegraded":"2023-09-23T23:35:21.395479+0000","last_fullsized":"2023-09-23T23:35:21.395479+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"587'6228","reported_seq":6289,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.045690+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:35:29.045690+0000","last_peered":"2023-09-23T23:35:29.045690+0000","last_clean":"2023-09-23T23:35:29.045690+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:35:29.045690+0000","last_undegraded":"2023-09-23T23:35:29.045690+0000","last_fullsized":"2023-09-23T23:35:29.045690+0000","mapping_epoch":534,"log_start":"543'800","ondisk_log_start":"543'800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5428,"log_dups_size":0,"ondisk_log_size":5428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295436288,"num_objects":4508,"num_object_clones":0,"num_object_copies":13524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6228,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":699,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.395049+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:35:21.395049+0000","last_peered":"2023-09-23T23:35:21.395049+0000","last_clean":"2023-09-23T23:35:21.395049+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:35:21.395049+0000","last_undegraded":"2023-09-23T23:35:21.395049+0000","last_fullsized":"2023-09-23T23:35:21.395049+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":513,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.426159+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:35:15.426159+0000","last_peered":"2023-09-23T23:35:15.426159+0000","last_clean":"2023-09-23T23:35:15.426159+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:35:15.426159+0000","last_undegraded":"2023-09-23T23:35:15.426159+0000","last_fullsized":"2023-09-23T23:35:15.426159+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":426,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417657+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:35:15.417657+0000","last_peered":"2023-09-23T23:35:15.417657+0000","last_clean":"2023-09-23T23:35:15.417657+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:35:15.417657+0000","last_undegraded":"2023-09-23T23:35:15.417657+0000","last_fullsized":"2023-09-23T23:35:15.417657+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":27,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.418937+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:35:21.418937+0000","last_peered":"2023-09-23T23:35:21.418937+0000","last_clean":"2023-09-23T23:35:21.418937+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:35:21.418937+0000","last_undegraded":"2023-09-23T23:35:21.418937+0000","last_fullsized":"2023-09-23T23:35:21.418937+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"587'6366","reported_seq":10026,"reported_epoch":587,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:29.730090+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:35:29.730090+0000","last_peered":"2023-09-23T23:35:29.730090+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:35:29.730090+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"538'200","ondisk_log_start":"538'200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6166,"log_dups_size":0,"ondisk_log_size":6166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303562752,"num_objects":4632,"num_object_clones":0,"num_object_copies":13896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6366,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":1928},{"shards":"1,2,4","objects":2704}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":582,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.395806+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:35:21.395806+0000","last_peered":"2023-09-23T23:35:21.395806+0000","last_clean":"2023-09-23T23:35:21.395806+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:35:21.395806+0000","last_undegraded":"2023-09-23T23:35:21.395806+0000","last_fullsized":"2023-09-23T23:35:21.395806+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":27,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.418509+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:35:21.418509+0000","last_peered":"2023-09-23T23:35:21.418509+0000","last_clean":"2023-09-23T23:35:21.418509+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:35:21.418509+0000","last_undegraded":"2023-09-23T23:35:21.418509+0000","last_fullsized":"2023-09-23T23:35:21.418509+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":126,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.405213+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:35:21.405213+0000","last_peered":"2023-09-23T23:35:21.405213+0000","last_clean":"2023-09-23T23:35:21.405213+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:35:21.405213+0000","last_undegraded":"2023-09-23T23:35:21.405213+0000","last_fullsized":"2023-09-23T23:35:21.405213+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":320,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.416223+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:35:21.416223+0000","last_peered":"2023-09-23T23:35:21.416223+0000","last_clean":"2023-09-23T23:35:21.416223+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:35:21.416223+0000","last_undegraded":"2023-09-23T23:35:21.416223+0000","last_fullsized":"2023-09-23T23:35:21.416223+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"587'6388","reported_seq":6449,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.034149+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:35:29.034149+0000","last_peered":"2023-09-23T23:35:29.034149+0000","last_clean":"2023-09-23T23:35:29.034149+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:35:29.034149+0000","last_undegraded":"2023-09-23T23:35:29.034149+0000","last_fullsized":"2023-09-23T23:35:29.034149+0000","mapping_epoch":534,"log_start":"542'700","ondisk_log_start":"542'700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5688,"log_dups_size":0,"ondisk_log_size":5688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299892736,"num_objects":4576,"num_object_clones":0,"num_object_copies":13728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6388,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":767,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.401292+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:35:21.401292+0000","last_peered":"2023-09-23T23:35:21.401292+0000","last_clean":"2023-09-23T23:35:21.401292+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:35:21.401292+0000","last_undegraded":"2023-09-23T23:35:21.401292+0000","last_fullsized":"2023-09-23T23:35:21.401292+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":583,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.423118+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:35:15.423118+0000","last_peered":"2023-09-23T23:35:15.423118+0000","last_clean":"2023-09-23T23:35:15.423118+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:35:15.423118+0000","last_undegraded":"2023-09-23T23:35:15.423118+0000","last_fullsized":"2023-09-23T23:35:15.423118+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":191,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.395647+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:35:21.395647+0000","last_peered":"2023-09-23T23:35:21.395647+0000","last_clean":"2023-09-23T23:35:21.395647+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:35:21.395647+0000","last_undegraded":"2023-09-23T23:35:21.395647+0000","last_fullsized":"2023-09-23T23:35:21.395647+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":138,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.400603+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:35:21.400603+0000","last_peered":"2023-09-23T23:35:21.400603+0000","last_clean":"2023-09-23T23:35:21.400603+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:35:21.400603+0000","last_undegraded":"2023-09-23T23:35:21.400603+0000","last_fullsized":"2023-09-23T23:35:21.400603+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"587'6308","reported_seq":6371,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.804444+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:35:29.804444+0000","last_peered":"2023-09-23T23:35:29.804444+0000","last_clean":"2023-09-23T23:35:29.804444+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:35:29.804444+0000","last_undegraded":"2023-09-23T23:35:29.804444+0000","last_fullsized":"2023-09-23T23:35:29.804444+0000","mapping_epoch":534,"log_start":"538'100","ondisk_log_start":"538'100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6208,"log_dups_size":0,"ondisk_log_size":6208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301465600,"num_objects":4600,"num_object_clones":0,"num_object_copies":13800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6308,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":972,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.422901+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:35:15.422901+0000","last_peered":"2023-09-23T23:35:15.422901+0000","last_clean":"2023-09-23T23:35:15.422901+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:35:15.422901+0000","last_undegraded":"2023-09-23T23:35:15.422901+0000","last_fullsized":"2023-09-23T23:35:15.422901+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":101,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.396295+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:35:21.396295+0000","last_peered":"2023-09-23T23:35:21.396295+0000","last_clean":"2023-09-23T23:35:21.396295+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:35:21.396295+0000","last_undegraded":"2023-09-23T23:35:21.396295+0000","last_fullsized":"2023-09-23T23:35:21.396295+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":31,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.419018+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:35:21.419018+0000","last_peered":"2023-09-23T23:35:21.419018+0000","last_clean":"2023-09-23T23:35:21.419018+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:35:21.419018+0000","last_undegraded":"2023-09-23T23:35:21.419018+0000","last_fullsized":"2023-09-23T23:35:21.419018+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":161,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.415926+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:35:21.415926+0000","last_peered":"2023-09-23T23:35:21.415926+0000","last_clean":"2023-09-23T23:35:21.415926+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:35:21.415926+0000","last_undegraded":"2023-09-23T23:35:21.415926+0000","last_fullsized":"2023-09-23T23:35:21.415926+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1022,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417741+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:35:15.417741+0000","last_peered":"2023-09-23T23:35:15.417741+0000","last_clean":"2023-09-23T23:35:15.417741+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:35:15.417741+0000","last_undegraded":"2023-09-23T23:35:15.417741+0000","last_fullsized":"2023-09-23T23:35:15.417741+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":27,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.418029+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:35:21.418029+0000","last_peered":"2023-09-23T23:35:21.418029+0000","last_clean":"2023-09-23T23:35:21.418029+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:35:21.418029+0000","last_undegraded":"2023-09-23T23:35:21.418029+0000","last_fullsized":"2023-09-23T23:35:21.418029+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":342,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.405257+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:35:21.405257+0000","last_peered":"2023-09-23T23:35:21.405257+0000","last_clean":"2023-09-23T23:35:21.405257+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:35:21.405257+0000","last_undegraded":"2023-09-23T23:35:21.405257+0000","last_fullsized":"2023-09-23T23:35:21.405257+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":733,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.423334+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:35:15.423334+0000","last_peered":"2023-09-23T23:35:15.423334+0000","last_clean":"2023-09-23T23:35:15.423334+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:35:15.423334+0000","last_undegraded":"2023-09-23T23:35:15.423334+0000","last_fullsized":"2023-09-23T23:35:15.423334+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":89,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.394408+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:35:21.394408+0000","last_peered":"2023-09-23T23:35:21.394408+0000","last_clean":"2023-09-23T23:35:21.394408+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:35:21.394408+0000","last_undegraded":"2023-09-23T23:35:21.394408+0000","last_fullsized":"2023-09-23T23:35:21.394408+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":433,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.418612+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:35:15.418612+0000","last_peered":"2023-09-23T23:35:15.418612+0000","last_clean":"2023-09-23T23:35:15.418612+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:35:15.418612+0000","last_undegraded":"2023-09-23T23:35:15.418612+0000","last_fullsized":"2023-09-23T23:35:15.418612+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":285,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.395338+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:35:21.395338+0000","last_peered":"2023-09-23T23:35:21.395338+0000","last_clean":"2023-09-23T23:35:21.395338+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:35:21.395338+0000","last_undegraded":"2023-09-23T23:35:21.395338+0000","last_fullsized":"2023-09-23T23:35:21.395338+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":272,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417915+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:35:15.417915+0000","last_peered":"2023-09-23T23:35:15.417915+0000","last_clean":"2023-09-23T23:35:15.417915+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:35:15.417915+0000","last_undegraded":"2023-09-23T23:35:15.417915+0000","last_fullsized":"2023-09-23T23:35:15.417915+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":799,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.395399+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:35:21.395399+0000","last_peered":"2023-09-23T23:35:21.395399+0000","last_clean":"2023-09-23T23:35:21.395399+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:35:21.395399+0000","last_undegraded":"2023-09-23T23:35:21.395399+0000","last_fullsized":"2023-09-23T23:35:21.395399+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":616,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.423287+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:35:15.423287+0000","last_peered":"2023-09-23T23:35:15.423287+0000","last_clean":"2023-09-23T23:35:15.423287+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:35:15.423287+0000","last_undegraded":"2023-09-23T23:35:15.423287+0000","last_fullsized":"2023-09-23T23:35:15.423287+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":600,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.417897+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:35:15.417897+0000","last_peered":"2023-09-23T23:35:15.417897+0000","last_clean":"2023-09-23T23:35:15.417897+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:35:15.417897+0000","last_undegraded":"2023-09-23T23:35:15.417897+0000","last_fullsized":"2023-09-23T23:35:15.417897+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":758,"reported_epoch":585,"state":"active+clean","last_fresh":"2023-09-23T23:35:15.432319+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:35:15.432319+0000","last_peered":"2023-09-23T23:35:15.432319+0000","last_clean":"2023-09-23T23:35:15.432319+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:35:15.432319+0000","last_undegraded":"2023-09-23T23:35:15.432319+0000","last_fullsized":"2023-09-23T23:35:15.432319+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":430,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.404976+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:35:21.404976+0000","last_peered":"2023-09-23T23:35:21.404976+0000","last_clean":"2023-09-23T23:35:21.404976+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:35:21.404976+0000","last_undegraded":"2023-09-23T23:35:21.404976+0000","last_fullsized":"2023-09-23T23:35:21.404976+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":654,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.394216+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:35:21.394216+0000","last_peered":"2023-09-23T23:35:21.394216+0000","last_clean":"2023-09-23T23:35:21.394216+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:35:21.394216+0000","last_undegraded":"2023-09-23T23:35:21.394216+0000","last_fullsized":"2023-09-23T23:35:21.394216+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":4841996312,"num_objects":73884,"num_object_clones":0,"num_object_copies":221652,"num_objects_missing_on_primary":3625,"num_objects_missing":3625,"num_objects_degraded":26987,"num_objects_misplaced":4520,"num_objects_unfound":0,"num_objects_dirty":73884,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":101672,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12032,"num_bytes_recovered":788332544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15283073024,"data_stored":15283060808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89750,"ondisk_log_size":89750,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782672,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2105224,"kb_used_data":1742336,"kb_used_omap":8,"kb_used_meta":362871,"kb_avail":91639928,"statfs":{"total":95995035648,"available":93839286272,"internally_reserved":0,"allocated":1784152064,"data_stored":1782532444,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":371579973},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.9339999999999999,"5min":2.3500000000000001,"15min":2.2549999999999999},"min":{"1min":0.26600000000000001,"5min":0.224,"15min":0.20899999999999999},"max":{"1min":11.173,"5min":44.094999999999999,"15min":85.721000000000004},"last":0.26900000000000002},{"interface":"front","average":{"1min":1.1519999999999999,"5min":2.2080000000000002,"15min":2.2669999999999999},"min":{"1min":0.28199999999999997,"5min":0.25600000000000001,"15min":0.19800000000000001},"max":{"1min":4.3109999999999999,"5min":44.335999999999999,"15min":85.965000000000003},"last":0.375}]},{"osd":1,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8819999999999999,"5min":6.5810000000000004,"15min":3.202},"min":{"1min":0.252,"5min":0.248,"15min":0.22900000000000001},"max":{"1min":9.0139999999999993,"5min":198.58000000000001,"15min":198.58000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.179,"5min":6.3109999999999999,"15min":3.2610000000000001},"min":{"1min":0.254,"5min":0.247,"15min":0.246},"max":{"1min":4.1660000000000004,"5min":198.71100000000001,"15min":198.71100000000001},"last":0.29699999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":6.0019999999999998,"5min":5.7329999999999997,"15min":3.7810000000000001},"min":{"1min":0.26900000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":78.528999999999996,"5min":203.58000000000001,"15min":203.58000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.8799999999999999,"5min":4.5350000000000001,"15min":3.1080000000000001},"min":{"1min":0.27900000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.5670000000000002,"5min":198.69,"15min":198.69},"last":0.314}]},{"osd":3,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":7.0250000000000004,"15min":3.6579999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":6.1210000000000004,"5min":279.26499999999999,"15min":279.26499999999999},"last":0.628},{"interface":"front","average":{"1min":3.1579999999999999,"5min":8.7859999999999996,"15min":4.1630000000000003},"min":{"1min":0.40200000000000002,"5min":0.32000000000000001,"15min":0.29599999999999999},"max":{"1min":18.643999999999998,"5min":279.29899999999998,"15min":279.29899999999998},"last":0.56499999999999995}]},{"osd":4,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":1.857,"15min":1.0940000000000001},"min":{"1min":0.35299999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":10.6,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.79600000000000004,"5min":1.4970000000000001,"15min":1.03},"min":{"1min":0.317,"5min":0.28100000000000003,"15min":0.23899999999999999},"max":{"1min":2.8570000000000002,"5min":31.68,"15min":31.68},"last":0.33000000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":1.1659999999999999,"15min":1.7290000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26500000000000001},"max":{"1min":12.273999999999999,"5min":12.273999999999999,"15min":210.83600000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":2.141,"5min":1.0529999999999999,"15min":1.579},"min":{"1min":0.371,"5min":0.316,"15min":0.28199999999999997},"max":{"1min":18.117999999999999,"5min":18.117999999999999,"15min":207.04599999999999},"last":0.59099999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.96,"15min":1.3620000000000001},"min":{"1min":0.379,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.6440000000000001,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.61299999999999999},{"interface":"front","average":{"1min":2.2109999999999999,"5min":2.403,"15min":1.615},"min":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.33300000000000002},"max":{"1min":10.332000000000001,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.60399999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496149,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2315640,"kb_used_data":1618736,"kb_used_omap":0,"kb_used_meta":696896,"kb_avail":91429512,"statfs":{"total":95995035648,"available":93623820288,"internally_reserved":0,"allocated":1657585664,"data_stored":1655982037,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":713621504},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,4],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:14 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.98799999999999999,"15min":4.7699999999999996},"min":{"1min":0.26500000000000001,"5min":0.223,"15min":0.20399999999999999},"max":{"1min":2.274,"5min":8.8119999999999994,"15min":135.31999999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.93000000000000005,"15min":4.617},"min":{"1min":0.26800000000000002,"5min":0.23999999999999999,"15min":0.22900000000000001},"max":{"1min":3.8239999999999998,"5min":8.3230000000000004,"15min":135.34299999999999},"last":0.54300000000000004}]},{"osd":1,"last update":"Sat Sep 23 23:35:14 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":3.3690000000000002,"15min":2.4089999999999998},"min":{"1min":0.26000000000000001,"5min":0.251,"15min":0.245},"max":{"1min":6.9080000000000004,"5min":89.605000000000004,"15min":89.605000000000004},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.71599999999999997,"5min":3.1600000000000001,"15min":2.4849999999999999},"min":{"1min":0.33500000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":1.9159999999999999,"5min":91.025999999999996,"15min":91.025999999999996},"last":0.51600000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:35:03 2023","interfaces":[{"interface":"back","average":{"1min":9.4120000000000008,"5min":6.4420000000000002,"15min":4.6150000000000002},"min":{"1min":0.315,"5min":0.28399999999999997,"15min":0.27800000000000002},"max":{"1min":67.796999999999997,"5min":97.408000000000001,"15min":127.44499999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":8.6270000000000007,"5min":7.6139999999999999,"15min":5.0419999999999998},"min":{"1min":0.30599999999999999,"5min":0.29999999999999999,"15min":0.26500000000000001},"max":{"1min":67.893000000000001,"5min":125.539,"15min":125.539},"last":0.67900000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:34:48 2023","interfaces":[{"interface":"back","average":{"1min":8.5120000000000005,"5min":5.7119999999999997,"15min":3.9969999999999999},"min":{"1min":0.39400000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":69.433000000000007,"5min":138.87299999999999,"15min":138.87299999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":8.0280000000000005,"5min":5.5289999999999999,"15min":4.2859999999999996},"min":{"1min":0.35199999999999998,"5min":0.33200000000000002,"15min":0.28799999999999998},"max":{"1min":69.463999999999999,"5min":138.976,"15min":138.976},"last":0.64200000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.82999999999999996,"15min":1.274},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":1.633,"5min":7.101,"15min":166.05199999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.55800000000000005,"5min":1.0289999999999999,"15min":1.4890000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.089,"5min":8.3499999999999996,"15min":166.631},"last":0.628}]},{"osd":5,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.96699999999999997,"15min":0.82399999999999995},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.1579999999999999,"5min":11.473000000000001,"15min":11.473000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.433,"5min":1.091,"15min":0.94099999999999995},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.249},"max":{"1min":0.76700000000000002,"5min":7.5670000000000002,"15min":8.4090000000000007},"last":0.76700000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:35:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.5389999999999999,"15min":1.0580000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":2.3010000000000002,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.573,"15min":1.081},"min":{"1min":0.36199999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.9009999999999998,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.61499999999999999}]}]},{"osd":1,"up_from":112,"seq":481036337490,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2380148,"kb_used_data":1489196,"kb_used_omap":4,"kb_used_meta":890875,"kb_avail":91365004,"statfs":{"total":95995035648,"available":93557764096,"internally_reserved":0,"allocated":1524936704,"data_stored":1522175530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":912256910},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":2.8220000000000001,"15min":3.5670000000000002},"min":{"1min":0.21199999999999999,"5min":0.17799999999999999,"15min":0.16700000000000001},"max":{"1min":15.983000000000001,"5min":47.195999999999998,"15min":130.768},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.883,"15min":2.8650000000000002},"min":{"1min":0.20699999999999999,"5min":0.20000000000000001,"15min":0.19},"max":{"1min":4.9809999999999999,"5min":46.582000000000001,"15min":134.929},"last":0.61399999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:34:57 2023","interfaces":[{"interface":"back","average":{"1min":10.555,"5min":7.9610000000000003,"15min":6.7919999999999998},"min":{"1min":0.23000000000000001,"5min":0.214,"15min":0.20100000000000001},"max":{"1min":201.03999999999999,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":10.605,"5min":8.173,"15min":6.9109999999999996},"min":{"1min":0.24299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":202.12,"5min":202.12,"15min":232.21000000000001},"last":0.72799999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:34:57 2023","interfaces":[{"interface":"back","average":{"1min":9.6969999999999992,"5min":5.7679999999999998,"15min":5.1859999999999999},"min":{"1min":0.311,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":163.94300000000001,"5min":181.94399999999999,"15min":234.404},"last":0.35599999999999998},{"interface":"front","average":{"1min":9.9290000000000003,"5min":5.6550000000000002,"15min":5.0430000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.24399999999999999},"max":{"1min":163.98699999999999,"5min":181.98500000000001,"15min":236.33699999999999},"last":0.81699999999999995}]},{"osd":4,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":1.5569999999999999,"15min":1.4339999999999999},"min":{"1min":0.32700000000000001,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":2.8210000000000002,"5min":12.042999999999999,"15min":12.042999999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.5609999999999999,"15min":1.498},"min":{"1min":0.34499999999999997,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":2.7810000000000001,"5min":18.108000000000001,"15min":18.108000000000001},"last":0.55400000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":3.2879999999999998,"15min":1.8260000000000001},"min":{"1min":0.307,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.0460000000000003,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.91300000000000003,"5min":3.2890000000000001,"15min":2.141},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.7450000000000001,"5min":151.821,"15min":151.821},"last":0.41299999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.6850000000000001,"15min":1.954},"min":{"1min":0.34699999999999998,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":5.0960000000000001,"5min":8.9269999999999996,"15min":74.087999999999994},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.3300000000000001,"5min":1.885,"15min":2.157},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.32800000000000001},"max":{"1min":6.8920000000000003,"5min":11.927,"15min":73.974999999999994},"last":0.45600000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:34:57 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":2.1269999999999998,"15min":2.875},"min":{"1min":0.38700000000000001,"5min":0.37,"15min":0.32900000000000001},"max":{"1min":10.629,"5min":44.241,"15min":78.308000000000007},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.165,"5min":2.218,"15min":2.706},"min":{"1min":0.42899999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.9880000000000004,"5min":43.920999999999999,"15min":77.204999999999998},"last":0.496}]}]},{"osd":0,"up_from":91,"seq":390842024314,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1584704,"kb_used_data":374456,"kb_used_omap":5,"kb_used_meta":1210234,"kb_avail":92160448,"statfs":{"total":95995035648,"available":94372298752,"internally_reserved":0,"allocated":383442944,"data_stored":380664557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1239280250},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:34:29 2023","interfaces":[{"interface":"back","average":{"1min":16.536000000000001,"5min":11.071999999999999,"15min":6.7530000000000001},"min":{"1min":0.188,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":301.12700000000001,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.25900000000000001},{"interface":"front","average":{"1min":16.061,"5min":10.94,"15min":5.8929999999999998},"min":{"1min":0.217,"5min":0.19700000000000001,"15min":0.17599999999999999},"max":{"1min":297.72399999999999,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.35199999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:34:55 2023","interfaces":[{"interface":"back","average":{"1min":19.536999999999999,"5min":10.204000000000001,"15min":7.4470000000000001},"min":{"1min":0.23300000000000001,"5min":0.17699999999999999,"15min":0.17199999999999999},"max":{"1min":283.07400000000001,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.29699999999999999},{"interface":"front","average":{"1min":18.260000000000002,"5min":10.146000000000001,"15min":7.5259999999999998},"min":{"1min":0.26800000000000002,"5min":0.19800000000000001,"15min":0.17599999999999999},"max":{"1min":283.161,"5min":283.161,"15min":283.161},"last":0.311}]},{"osd":3,"last update":"Sat Sep 23 23:34:29 2023","interfaces":[{"interface":"back","average":{"1min":5.2089999999999996,"5min":8.5380000000000003,"15min":5.3230000000000004},"min":{"1min":0.219,"5min":0.217,"15min":0.217},"max":{"1min":78.960999999999999,"5min":220.732,"15min":220.732},"last":0.32600000000000001},{"interface":"front","average":{"1min":4.9020000000000001,"5min":8.8420000000000005,"15min":5.4299999999999997},"min":{"1min":0.20599999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":79.185000000000002,"5min":218.78200000000001,"15min":218.78200000000001},"last":0.39000000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":2.5859999999999999,"15min":2.8660000000000001},"min":{"1min":0.33300000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.4500000000000002,"5min":108.01900000000001,"15min":138.81299999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.25,"5min":2.7490000000000001,"15min":3.4550000000000001},"min":{"1min":0.35499999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.3670000000000009,"5min":107.898,"15min":171.251},"last":0.46300000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":1.7310000000000001,"15min":2.8340000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32100000000000001},"max":{"1min":15.076000000000001,"5min":15.076000000000001,"15min":171.30699999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.5649999999999999,"5min":1.4790000000000001,"15min":2.0880000000000001},"min":{"1min":0.35499999999999998,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":6.5279999999999996,"5min":9.9000000000000004,"15min":139.40799999999999},"last":0.52600000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":2.6360000000000001,"15min":3.161},"min":{"1min":0.38,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":9.968,"5min":108.67,"15min":171.33600000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.915,"5min":2.6699999999999999,"15min":3.4209999999999998},"min":{"1min":0.38900000000000001,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":9.157,"5min":108.804,"15min":184.13499999999999},"last":0.55600000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:34:55 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":2.8660000000000001,"15min":2.6640000000000001},"min":{"1min":0.34300000000000003,"5min":0.34000000000000002,"15min":0.32900000000000001},"max":{"1min":4.923,"5min":111.71299999999999,"15min":138.77699999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":2.4950000000000001,"5min":2.9689999999999999,"15min":2.5179999999999998},"min":{"1min":0.36499999999999999,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":9.8870000000000005,"5min":111.108,"15min":139.37899999999999},"last":0.57099999999999995}]}]},{"osd":2,"up_from":199,"seq":854698492135,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3752380,"kb_used_data":2940468,"kb_used_omap":7,"kb_used_meta":811896,"kb_avail":89992772,"statfs":{"total":95995035648,"available":92152598528,"internally_reserved":0,"allocated":3011039232,"data_stored":3008214513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":831381585},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":4.726,"5min":2.6949999999999998,"15min":3.9089999999999998},"min":{"1min":0.19500000000000001,"5min":0.193,"15min":0.16700000000000001},"max":{"1min":61.950000000000003,"5min":61.950000000000003,"15min":119.277},"last":0.24399999999999999},{"interface":"front","average":{"1min":5.0199999999999996,"5min":2.504,"15min":3.726},"min":{"1min":0.27800000000000002,"5min":0.20399999999999999,"15min":0.18099999999999999},"max":{"1min":61.984999999999999,"5min":61.984999999999999,"15min":119.56100000000001},"last":0.26300000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":11.630000000000001,"5min":6.2729999999999997,"15min":4.5359999999999996},"min":{"1min":0.29299999999999998,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":106.765,"5min":106.765,"15min":157.98699999999999},"last":0.28100000000000003},{"interface":"front","average":{"1min":11.114000000000001,"5min":6.3659999999999997,"15min":4.6349999999999998},"min":{"1min":0.29599999999999999,"5min":0.222,"15min":0.222},"max":{"1min":106.732,"5min":106.732,"15min":158.03800000000001},"last":0.32600000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":11.803000000000001,"5min":6.0069999999999997,"15min":4.7309999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":106.70399999999999,"5min":106.70399999999999,"15min":119.366},"last":0.312},{"interface":"front","average":{"1min":11.427,"5min":5.891,"15min":4.3280000000000003},"min":{"1min":0.35499999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":106.66800000000001,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.29999999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":5.0979999999999999,"5min":2.4510000000000001,"15min":2.3170000000000002},"min":{"1min":0.41399999999999998,"5min":0.33700000000000002,"15min":0.25},"max":{"1min":62.081000000000003,"5min":62.081000000000003,"15min":119.88200000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":4.9169999999999998,"5min":2.5369999999999999,"15min":1.835},"min":{"1min":0.39000000000000001,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":62.072000000000003,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.39000000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":5.3090000000000002,"5min":3.391,"15min":2.7400000000000002},"min":{"1min":0.38800000000000001,"5min":0.34300000000000003,"15min":0.26200000000000001},"max":{"1min":62.058,"5min":91.207999999999998,"15min":119.919},"last":0.35599999999999998},{"interface":"front","average":{"1min":4.8929999999999998,"5min":3.2469999999999999,"15min":2.6120000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":62.042999999999999,"5min":91.150999999999996,"15min":119.306},"last":0.34699999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":12.76,"5min":4.1360000000000001,"15min":3.0720000000000001},"min":{"1min":0.41599999999999998,"5min":0.36599999999999999,"15min":0.315},"max":{"1min":107.018,"5min":107.018,"15min":119.943},"last":0.495},{"interface":"front","average":{"1min":5.7709999999999999,"5min":2.798,"15min":2.1819999999999999},"min":{"1min":0.48199999999999998,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":62.493000000000002,"5min":62.493000000000002,"15min":62.493000000000002},"last":0.371}]},{"osd":7,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":4.8719999999999999,"5min":2.8879999999999999,"15min":1.958},"min":{"1min":0.41999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":62.603000000000002,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.50900000000000001},{"interface":"front","average":{"1min":11.744999999999999,"5min":4.0620000000000003,"15min":2.4060000000000001},"min":{"1min":0.48799999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":106.809,"5min":106.809,"15min":106.809},"last":0.47099999999999997}]}]},{"osd":3,"up_from":148,"seq":635655160103,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2470304,"kb_used_data":1572760,"kb_used_omap":2,"kb_used_meta":897533,"kb_avail":91274848,"statfs":{"total":95995035648,"available":93465444352,"internally_reserved":0,"allocated":1610506240,"data_stored":1607754265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":919073845},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,1,0,2,0,2,8],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.835,"15min":2.7349999999999999},"min":{"1min":0.215,"5min":0.19400000000000001,"15min":0.158},"max":{"1min":17.213000000000001,"5min":22.271999999999998,"15min":103.866},"last":2.5110000000000001},{"interface":"front","average":{"1min":0.95499999999999996,"5min":2.9750000000000001,"15min":2.8250000000000002},"min":{"1min":0.20799999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":4.5469999999999997,"5min":24.683,"15min":101.884},"last":0.23400000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":4.3789999999999996,"15min":2.9239999999999999},"min":{"1min":0.23300000000000001,"5min":0.214,"15min":0.19600000000000001},"max":{"1min":3.516,"5min":85.242999999999995,"15min":85.242999999999995},"last":0.372},{"interface":"front","average":{"1min":0.871,"5min":3.9910000000000001,"15min":2.7250000000000001},"min":{"1min":0.25800000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.2949999999999999,"5min":85.302000000000007,"15min":85.302000000000007},"last":0.435}]},{"osd":2,"last update":"Sat Sep 23 23:34:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7230000000000001,"5min":5.5739999999999998,"15min":4.8579999999999997},"min":{"1min":0.25,"5min":0.25,"15min":0.19800000000000001},"max":{"1min":7.1699999999999999,"5min":73.742000000000004,"15min":101.279},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.99,"5min":5.468,"15min":4.6529999999999996},"min":{"1min":0.36699999999999999,"5min":0.25700000000000001,"15min":0.214},"max":{"1min":11.156000000000001,"5min":76.525999999999996,"15min":101.294},"last":0.57599999999999996}]},{"osd":4,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.74,"5min":1.3859999999999999,"15min":1.3049999999999999},"min":{"1min":0.34999999999999998,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":10.151999999999999,"5min":10.151999999999999,"15min":10.151999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.464,"5min":2.3479999999999999,"15min":1.8240000000000001},"min":{"1min":0.35399999999999998,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":7.681,"5min":24.782,"15min":24.782},"last":0.41199999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":7.2480000000000002,"15min":3.4140000000000001},"min":{"1min":0.39500000000000002,"5min":0.308,"15min":0.28000000000000003},"max":{"1min":6.5010000000000003,"5min":214.43700000000001,"15min":214.43700000000001},"last":2.5219999999999998},{"interface":"front","average":{"1min":2.0779999999999998,"5min":7.0030000000000001,"15min":3.488},"min":{"1min":0.376,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":9.4160000000000004,"5min":214.46600000000001,"15min":214.46600000000001},"last":0.48699999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":3.1419999999999999,"15min":2.3050000000000002},"min":{"1min":0.39000000000000001,"5min":0.35899999999999999,"15min":0.31900000000000001},"max":{"1min":5.4169999999999998,"5min":58.932000000000002,"15min":58.932000000000002},"last":2.6800000000000002},{"interface":"front","average":{"1min":2.9900000000000002,"5min":2.8969999999999998,"15min":2.359},"min":{"1min":0.41799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":16.759,"5min":59.186,"15min":59.186},"last":0.52000000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:34:55 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":6.609,"15min":3.2200000000000002},"min":{"1min":0.42699999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":11.191000000000001,"5min":192.69,"15min":192.69},"last":2.415},{"interface":"front","average":{"1min":2.5779999999999998,"5min":7.4489999999999998,"15min":3.7269999999999999},"min":{"1min":0.47599999999999998,"5min":0.39200000000000002,"15min":0.36699999999999999},"max":{"1min":9.7449999999999992,"5min":192.65600000000001,"15min":192.65600000000001},"last":0.46899999999999997}]}]},{"osd":4,"up_from":0,"seq":128849019422,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757785,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3436916,"kb_used_data":2725356,"kb_used_omap":0,"kb_used_meta":711552,"kb_avail":90308236,"statfs":{"total":95995035648,"available":92475633664,"internally_reserved":0,"allocated":2790764544,"data_stored":2789161134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":728629248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.3720000000000001,"15min":3.48},"min":{"1min":0.254,"5min":0.20899999999999999,"15min":0.19500000000000001},"max":{"1min":3.0089999999999999,"5min":21.109000000000002,"15min":113.869},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.6699999999999999,"15min":3.8650000000000002},"min":{"1min":0.25,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":5.1280000000000001,"5min":20.876999999999999,"15min":138.55199999999999},"last":0.29199999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.772,"15min":2.323},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.23100000000000001},"max":{"1min":3.629,"5min":29.007999999999999,"15min":69.816999999999993},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.9390000000000001,"15min":2.665},"min":{"1min":0.29999999999999999,"5min":0.249,"15min":0.249},"max":{"1min":4.4859999999999998,"5min":29.053999999999998,"15min":70.268000000000001},"last":0.39100000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:34:47 2023","interfaces":[{"interface":"back","average":{"1min":8.3900000000000006,"5min":5.3979999999999997,"15min":4.2009999999999996},"min":{"1min":0.32000000000000001,"5min":0.28299999999999997,"15min":0.26900000000000002},"max":{"1min":69.75,"5min":208.738,"15min":208.738},"last":145.93199999999999},{"interface":"front","average":{"1min":8.1739999999999995,"5min":5.3280000000000003,"15min":4.0960000000000001},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":66.421000000000006,"5min":208.631,"15min":208.631},"last":0.33100000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:34:43 2023","interfaces":[{"interface":"back","average":{"1min":8.5440000000000005,"5min":3.1419999999999999,"15min":3.3100000000000001},"min":{"1min":0.35399999999999998,"5min":0.29299999999999998,"15min":0.27400000000000002},"max":{"1min":66.591999999999999,"5min":66.591999999999999,"15min":112.71899999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":9.8930000000000007,"5min":3.5640000000000001,"15min":3.5419999999999998},"min":{"1min":0.372,"5min":0.35399999999999998,"15min":0.318},"max":{"1min":66.692999999999998,"5min":66.692999999999998,"15min":112.70099999999999},"last":0.41099999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.151,"15min":0.92000000000000004},"min":{"1min":0.34200000000000003,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":3.5499999999999998,"5min":9.2690000000000001,"15min":19.962},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.093,"15min":0.872},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.22900000000000001},"max":{"1min":2.4990000000000001,"5min":10.273999999999999,"15min":20.829999999999998},"last":0.36599999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":7.2519999999999998,"5min":2.3799999999999999,"15min":1.635},"min":{"1min":0.32400000000000001,"5min":0.27600000000000002,"15min":0.26900000000000002},"max":{"1min":148.36600000000001,"5min":148.36600000000001,"15min":148.36600000000001},"last":148.36600000000001},{"interface":"front","average":{"1min":7.7210000000000001,"5min":2.9420000000000002,"15min":1.869},"min":{"1min":0.34599999999999997,"5min":0.307,"15min":0.26000000000000001},"max":{"1min":148.42500000000001,"5min":148.42500000000001,"15min":148.42500000000001},"last":148.42500000000001}]},{"osd":7,"last update":"Sat Sep 23 23:34:47 2023","interfaces":[{"interface":"back","average":{"1min":2.5760000000000001,"5min":1.577,"15min":2.8639999999999999},"min":{"1min":0.34999999999999998,"5min":0.34599999999999997,"15min":0.317},"max":{"1min":15.497,"5min":16.408999999999999,"15min":179.28800000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":2.387,"5min":1.6890000000000001,"15min":2.956},"min":{"1min":0.36799999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":15.653,"5min":18.972000000000001,"15min":179.428},"last":0.38}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":374603776,"data_stored":374603776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1516507136,"data_stored":1516503064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3002073088,"data_stored":3002073088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1598164992,"data_stored":1598160920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2585460736,"data_stored":2585460736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2782789632,"data_stored":2782789632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1648361472,"data_stored":1648361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1775112192,"data_stored":1775108120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:35:32.687 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:35:32.698 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:32.778 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:32.783 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:32.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:32.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:32.885 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:32.924 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:32.982 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:32.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:32.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:33.085 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:33.116 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:33.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:33 smithi138 ceph-mon[193606]: pgmap v1617: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 194 op/s; 26987/221724 objects degraded (12.171%); 4520/221724 objects misplaced (2.039%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T23:35:33.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:33 smithi138 ceph-mon[193606]: from='client.46757 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:35:33.290 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:33.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:33.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:33.310 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:33 smithi110 ceph-mon[221102]: pgmap v1617: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 194 op/s; 26987/221724 objects degraded (12.171%); 4520/221724 objects misplaced (2.039%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T23:35:33.310 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:33 smithi110 ceph-mon[221102]: from='client.46757 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:35:33.310 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:33 smithi110 ceph-mon[212939]: pgmap v1617: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 194 op/s; 26987/221724 objects degraded (12.171%); 4520/221724 objects misplaced (2.039%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T23:35:33.310 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:33 smithi110 ceph-mon[212939]: from='client.46757 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:35:33.378 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:35:33.396 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:33.425 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:33.469 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:33.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:33.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:33.574 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:33.608 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:33.758 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:33.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:33.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:33.797 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:35:33.798 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:35:33.818 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:33.862 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:33.893 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:34.043 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:34.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:34.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:34.077 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:34 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 26987/221724 objects degraded (12.171%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:34.077 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:34 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 26987/221724 objects degraded (12.171%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:34.150 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:34.174 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:34 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 26987/221724 objects degraded (12.171%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:34.176 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:34.218 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:34.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:34.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:34.321 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:34.394 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:34.443 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:34.458 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:34.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:34.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:34.563 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:34.589 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:34.648 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:34.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:34.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:34.706 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:35:34.752 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:34.779 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:35.051 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:35 smithi110 ceph-mon[212939]: pgmap v1618: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 26911/220878 objects degraded (12.184%); 4454/220878 objects misplaced (2.016%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:35:35.051 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:35 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1595310978' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:35:35.051 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:35 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:35:35.061 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:35:35.109 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:35.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:35.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:35.126 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:35:35.214 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:35.242 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:35.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:35 smithi138 ceph-mon[193606]: pgmap v1618: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 26911/220878 objects degraded (12.184%); 4454/220878 objects misplaced (2.016%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:35:35.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:35 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1595310978' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:35:35.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:35 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:35:35.337 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:35.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:35.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:35.440 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:35 smithi110 ceph-mon[221102]: pgmap v1618: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 26911/220878 objects degraded (12.184%); 4454/220878 objects misplaced (2.016%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:35:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:35 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1595310978' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:35:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:35 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:35:35.470 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:35.582 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:35.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:35.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:35.686 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:35.716 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:35.931 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:35.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:35.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:36.035 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:36.064 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:36.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:36 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:35:36.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:36 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:35:36.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:36 smithi110 ceph-mon[212939]: osdmap e588: 8 total, 8 up, 7 in 2023-09-23T23:35:36.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:36 smithi110 ceph-mon[212939]: failed to encode map e588 with expected crc 2023-09-23T23:35:36.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:36 smithi110 ceph-mon[212939]: failed to encode map e588 with expected crc 2023-09-23T23:35:36.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:36 smithi110 ceph-mon[212939]: failed to encode map e588 with expected crc 2023-09-23T23:35:36.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:36 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:35:36.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:36 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:35:36.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:36 smithi110 ceph-mon[221102]: osdmap e588: 8 total, 8 up, 7 in 2023-09-23T23:35:36.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:36 smithi110 ceph-mon[221102]: failed to encode map e588 with expected crc 2023-09-23T23:35:36.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:36 smithi110 ceph-mon[221102]: failed to encode map e588 with expected crc 2023-09-23T23:35:36.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:36 smithi110 ceph-mon[221102]: failed to encode map e588 with expected crc 2023-09-23T23:35:36.315 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:36 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:35:36.315 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:36 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:35:36.315 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:36 smithi138 ceph-mon[193606]: osdmap e588: 8 total, 8 up, 7 in 2023-09-23T23:35:36.315 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:36 smithi138 ceph-mon[193606]: failed to encode map e588 with expected crc 2023-09-23T23:35:36.315 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:36 smithi138 ceph-mon[193606]: failed to encode map e588 with expected crc 2023-09-23T23:35:36.315 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:36 smithi138 ceph-mon[193606]: failed to encode map e588 with expected crc 2023-09-23T23:35:36.328 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:36.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:36.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:36.434 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:36.466 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:36.592 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:35:36.592 INFO:teuthology.orchestra.run.smithi110.stdout:{"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 26851/218760 objects degraded (12.274%), 7 pgs degraded, 5 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2014,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":588,"num_osds":8,"num_up_osds":8,"osd_up_since":1695511414,"num_in_osds":7,"osd_in_since":1695512030,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":110},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+recovering","count":1},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":119,"num_pools":5,"num_objects":72920,"data_bytes":4777882869,"bytes_used":18340384768,"bytes_avail":653624864768,"bytes_total":671965249536,"degraded_objects":26851,"degraded_total":218760,"degraded_ratio":0.12274181751691352,"misplaced_objects":4454,"misplaced_total":218760,"misplaced_ratio":0.020360212104589503,"recovering_objects_per_sec":34,"recovering_bytes_per_sec":2284909,"recovering_keys_per_sec":0,"num_objects_recovered":279,"num_bytes_recovered":18284544,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":230},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi110.fuglpi","status":"up:active","gid":24572}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.110:8443/","prometheus":"http://172.21.15.110:9283/"}},"servicemap":{"epoch":32,"modified":"2023-09-23T23:32:57.852217+0000","services":{}},"progress_events":{"833ef15d-07ae-4299-b4e4-2cf94e068147":{"message":"Upgrade to 18.2.0-460-gbb0daeff (10m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true},"c67c6d89-9cd0-495b-b1de-fa24b5d12727":{"message":"Global Recovery Event (80s)\n [=========================...] (remaining: 6s)","progress":0.92436975240707397,"add_to_ceph_s":true}}} 2023-09-23T23:35:36.624 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:36.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:36.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:36.728 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:36.761 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:36.825 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:36.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:36.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:36.864 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:35:36.864 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:35:36.928 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:36.961 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:37.059 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:35:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:35:36] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2023-09-23T23:35:37.062 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:37.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:37.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:37.169 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:37.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:37 smithi138 ceph-mon[193606]: pgmap v1620: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 230 op/s; 26851/218760 objects degraded (12.274%); 4454/218760 objects misplaced (2.036%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:35:37.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:37 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3013675493' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:35:37.197 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:37.240 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:37.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:37.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:37.343 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:37.378 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:37.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:37 smithi110 ceph-mon[212939]: pgmap v1620: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 230 op/s; 26851/218760 objects degraded (12.274%); 4454/218760 objects misplaced (2.036%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:35:37.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:37 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3013675493' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:35:37.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:37 smithi110 ceph-mon[221102]: pgmap v1620: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 230 op/s; 26851/218760 objects degraded (12.274%); 4454/218760 objects misplaced (2.036%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T23:35:37.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:37 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3013675493' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:35:37.625 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:35:37.625 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:35:37.632 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:35:37.643 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:37.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:37.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:37.749 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:37.787 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:37.852 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:37.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:37.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:37.894 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:35:37.957 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:35:37.959 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:37.983 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:35:37.993 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:38.016 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:35:38.017 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:35:38.052 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:35:38.078 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:38.081 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:35:38.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:38.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:38.108 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:35:38.118 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:38.137 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:35:38.172 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:35:38.174 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:35:38.174 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:35:38.174 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:35:38.174 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:35:38.174 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:35:38.174 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:35:38.174 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:35:38.175 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:35:38.184 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:38.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:38 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3452696428' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:38.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:38 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3452696428' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:38.213 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:38.279 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:38.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:38.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:38.387 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:38.413 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:38 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3452696428' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:38.433 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:38.492 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:38.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:38.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:38.597 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:38.625 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:38.669 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:38.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:38.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:38.773 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:38.798 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:38.846 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:38.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:38.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:38.950 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:38.983 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:39.137 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:39 smithi110 ceph-mon[212939]: pgmap v1621: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 236 op/s; 26775/217986 objects degraded (12.283%); 4390/217986 objects misplaced (2.014%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T23:35:39.137 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:39 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 26775/217986 objects degraded (12.283%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:39.138 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:39 smithi110 ceph-mon[221102]: pgmap v1621: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 236 op/s; 26775/217986 objects degraded (12.283%); 4390/217986 objects misplaced (2.014%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T23:35:39.138 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:39 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 26775/217986 objects degraded (12.283%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:39.288 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:39.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:39.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:39.395 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:39.528 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:39 smithi138 ceph-mon[193606]: pgmap v1621: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 236 op/s; 26775/217986 objects degraded (12.283%); 4390/217986 objects misplaced (2.014%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T23:35:39.529 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:39 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 26775/217986 objects degraded (12.283%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:39.577 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:39.652 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:39.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:39.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:39.756 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:39.758 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:35:39.758 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:35:39.758 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-09-23T23:35:39.759 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:35:39.790 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:39.907 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:39.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:39.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:39.955 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:40.014 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:40.042 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:40.085 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:40.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:40.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:40.189 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:40.214 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:40.258 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:40.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:40.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:40.361 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:40.391 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:40.618 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:40.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:40.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:40.675 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:35:40.704 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:35:40.706 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:35:40.722 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:40.751 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:40.794 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:40.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:40.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:40.902 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:40.932 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:40.994 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:41.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:41.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:41.063 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:35:41.110 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:41.141 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:41.310 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:35:41.326 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:41 smithi110 ceph-mon[221102]: pgmap v1622: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 26716/216186 objects degraded (12.358%); 4390/216186 objects misplaced (2.031%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:35:41.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:41 smithi110 ceph-mon[212939]: pgmap v1622: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 26716/216186 objects degraded (12.358%); 4390/216186 objects misplaced (2.031%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:35:41.436 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:41 smithi138 ceph-mon[193606]: pgmap v1622: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 26716/216186 objects degraded (12.358%); 4390/216186 objects misplaced (2.031%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:35:41.644 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:41.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:41.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:41.748 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:41.779 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:41.834 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:41.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:41.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:41.854 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1622,"stamp":"2023-09-23T23:35:39.906933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4721652981,"num_objects":72062,"num_object_clones":0,"num_object_copies":216186,"num_objects_missing_on_primary":3495,"num_objects_missing":3495,"num_objects_degraded":26716,"num_objects_misplaced":4390,"num_objects_unfound":0,"num_objects_dirty":72062,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":103595,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12488,"num_bytes_recovered":815722920,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94742,"ondisk_log_size":94742,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17811016,"kb_used_data":12182288,"kb_used_omap":29,"kb_used_meta":5628642,"kb_avail":638405048,"statfs":{"total":671965249536,"available":653726769152,"internally_reserved":0,"allocated":12474662912,"data_stored":12458670510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5763730131},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-134545408,"num_objects":-2053,"num_object_clones":0,"num_object_copies":-6159,"num_objects_missing_on_primary":-130,"num_objects_missing":-130,"num_objects_degraded":-271,"num_objects_misplaced":-130,"num_objects_unfound":0,"num_objects_dirty":-2053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2053,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":401,"num_bytes_recovered":26279936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002845"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":784,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.034358+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:35:29.034358+0000","last_peered":"2023-09-23T23:35:29.034358+0000","last_clean":"2023-09-23T23:35:29.034358+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:35:29.034358+0000","last_undegraded":"2023-09-23T23:35:29.034358+0000","last_fullsized":"2023-09-23T23:35:29.034358+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":341,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.404977+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:35:21.404977+0000","last_peered":"2023-09-23T23:35:21.404977+0000","last_clean":"2023-09-23T23:35:21.404977+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:35:21.404977+0000","last_undegraded":"2023-09-23T23:35:21.404977+0000","last_fullsized":"2023-09-23T23:35:21.404977+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":569,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.085688+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:35:29.085688+0000","last_peered":"2023-09-23T23:35:29.085688+0000","last_clean":"2023-09-23T23:35:29.085688+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:35:29.085688+0000","last_undegraded":"2023-09-23T23:35:29.085688+0000","last_fullsized":"2023-09-23T23:35:29.085688+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":737,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.034763+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:35:29.034763+0000","last_peered":"2023-09-23T23:35:29.034763+0000","last_clean":"2023-09-23T23:35:29.034763+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:35:29.034763+0000","last_undegraded":"2023-09-23T23:35:29.034763+0000","last_fullsized":"2023-09-23T23:35:29.034763+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":306,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057408+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:35:29.057408+0000","last_peered":"2023-09-23T23:35:29.057408+0000","last_clean":"2023-09-23T23:35:29.057408+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:35:29.057408+0000","last_undegraded":"2023-09-23T23:35:29.057408+0000","last_fullsized":"2023-09-23T23:35:29.057408+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":548,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056872+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:35:29.056872+0000","last_peered":"2023-09-23T23:35:29.056872+0000","last_clean":"2023-09-23T23:35:29.056872+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:35:29.056872+0000","last_undegraded":"2023-09-23T23:35:29.056872+0000","last_fullsized":"2023-09-23T23:35:29.056872+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":749,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.035699+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:35:29.035699+0000","last_peered":"2023-09-23T23:35:29.035699+0000","last_clean":"2023-09-23T23:35:29.035699+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:35:29.035699+0000","last_undegraded":"2023-09-23T23:35:29.035699+0000","last_fullsized":"2023-09-23T23:35:29.035699+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":606,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056119+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:35:29.056119+0000","last_peered":"2023-09-23T23:35:29.056119+0000","last_clean":"2023-09-23T23:35:29.056119+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:35:29.056119+0000","last_undegraded":"2023-09-23T23:35:29.056119+0000","last_fullsized":"2023-09-23T23:35:29.056119+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":102,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.045889+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:35:29.045889+0000","last_peered":"2023-09-23T23:35:29.045889+0000","last_clean":"2023-09-23T23:35:29.045889+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:35:29.045889+0000","last_undegraded":"2023-09-23T23:35:29.045889+0000","last_fullsized":"2023-09-23T23:35:29.045889+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":908,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.035887+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:35:29.035887+0000","last_peered":"2023-09-23T23:35:29.035887+0000","last_clean":"2023-09-23T23:35:29.035887+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:35:29.035887+0000","last_undegraded":"2023-09-23T23:35:29.035887+0000","last_fullsized":"2023-09-23T23:35:29.035887+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":240,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.037255+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:35:29.037255+0000","last_peered":"2023-09-23T23:35:29.037255+0000","last_clean":"2023-09-23T23:35:29.037255+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:35:29.037255+0000","last_undegraded":"2023-09-23T23:35:29.037255+0000","last_fullsized":"2023-09-23T23:35:29.037255+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":588,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056066+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:35:29.056066+0000","last_peered":"2023-09-23T23:35:29.056066+0000","last_clean":"2023-09-23T23:35:29.056066+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:35:29.056066+0000","last_undegraded":"2023-09-23T23:35:29.056066+0000","last_fullsized":"2023-09-23T23:35:29.056066+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":880,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.066146+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:35:29.066146+0000","last_peered":"2023-09-23T23:35:29.066146+0000","last_clean":"2023-09-23T23:35:29.066146+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:35:29.066146+0000","last_undegraded":"2023-09-23T23:35:29.066146+0000","last_fullsized":"2023-09-23T23:35:29.066146+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":219,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.037167+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:35:29.037167+0000","last_peered":"2023-09-23T23:35:29.037167+0000","last_clean":"2023-09-23T23:35:29.037167+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:35:29.037167+0000","last_undegraded":"2023-09-23T23:35:29.037167+0000","last_fullsized":"2023-09-23T23:35:29.037167+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":549,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057216+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:35:29.057216+0000","last_peered":"2023-09-23T23:35:29.057216+0000","last_clean":"2023-09-23T23:35:29.057216+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:35:29.057216+0000","last_undegraded":"2023-09-23T23:35:29.057216+0000","last_fullsized":"2023-09-23T23:35:29.057216+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1042,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.037161+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:35:29.037161+0000","last_peered":"2023-09-23T23:35:29.037161+0000","last_clean":"2023-09-23T23:35:29.037161+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:35:29.037161+0000","last_undegraded":"2023-09-23T23:35:29.037161+0000","last_fullsized":"2023-09-23T23:35:29.037161+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":260,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.067752+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:35:29.067752+0000","last_peered":"2023-09-23T23:35:29.067752+0000","last_clean":"2023-09-23T23:35:29.067752+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:35:29.067752+0000","last_undegraded":"2023-09-23T23:35:29.067752+0000","last_fullsized":"2023-09-23T23:35:29.067752+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":121,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.044970+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:35:29.044970+0000","last_peered":"2023-09-23T23:35:29.044970+0000","last_clean":"2023-09-23T23:35:29.044970+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:35:29.044970+0000","last_undegraded":"2023-09-23T23:35:29.044970+0000","last_fullsized":"2023-09-23T23:35:29.044970+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":438,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057004+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:35:29.057004+0000","last_peered":"2023-09-23T23:35:29.057004+0000","last_clean":"2023-09-23T23:35:29.057004+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:35:29.057004+0000","last_undegraded":"2023-09-23T23:35:29.057004+0000","last_fullsized":"2023-09-23T23:35:29.057004+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":162,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.045441+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:35:29.045441+0000","last_peered":"2023-09-23T23:35:29.045441+0000","last_clean":"2023-09-23T23:35:29.045441+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:35:29.045441+0000","last_undegraded":"2023-09-23T23:35:29.045441+0000","last_fullsized":"2023-09-23T23:35:29.045441+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":529,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.044931+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:35:29.044931+0000","last_peered":"2023-09-23T23:35:29.044931+0000","last_clean":"2023-09-23T23:35:29.044931+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:35:29.044931+0000","last_undegraded":"2023-09-23T23:35:29.044931+0000","last_fullsized":"2023-09-23T23:35:29.044931+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":743,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.034251+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:35:29.034251+0000","last_peered":"2023-09-23T23:35:29.034251+0000","last_clean":"2023-09-23T23:35:29.034251+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:35:29.034251+0000","last_undegraded":"2023-09-23T23:35:29.034251+0000","last_fullsized":"2023-09-23T23:35:29.034251+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":133,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.400830+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:35:21.400830+0000","last_peered":"2023-09-23T23:35:21.400830+0000","last_clean":"2023-09-23T23:35:21.400830+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:35:21.400830+0000","last_undegraded":"2023-09-23T23:35:21.400830+0000","last_fullsized":"2023-09-23T23:35:21.400830+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":587,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056465+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:35:29.056465+0000","last_peered":"2023-09-23T23:35:29.056465+0000","last_clean":"2023-09-23T23:35:29.056465+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:35:29.056465+0000","last_undegraded":"2023-09-23T23:35:29.056465+0000","last_fullsized":"2023-09-23T23:35:29.056465+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":873,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.044623+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:35:29.044623+0000","last_peered":"2023-09-23T23:35:29.044623+0000","last_clean":"2023-09-23T23:35:29.044623+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:35:29.044623+0000","last_undegraded":"2023-09-23T23:35:29.044623+0000","last_fullsized":"2023-09-23T23:35:29.044623+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":182,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056220+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:35:29.056220+0000","last_peered":"2023-09-23T23:35:29.056220+0000","last_clean":"2023-09-23T23:35:29.056220+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:35:29.056220+0000","last_undegraded":"2023-09-23T23:35:29.056220+0000","last_fullsized":"2023-09-23T23:35:29.056220+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":309,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.085529+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:35:29.085529+0000","last_peered":"2023-09-23T23:35:29.085529+0000","last_clean":"2023-09-23T23:35:29.085529+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:35:29.085529+0000","last_undegraded":"2023-09-23T23:35:29.085529+0000","last_fullsized":"2023-09-23T23:35:29.085529+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":550,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.060891+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:35:29.060891+0000","last_peered":"2023-09-23T23:35:29.060891+0000","last_clean":"2023-09-23T23:35:29.060891+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:35:29.060891+0000","last_undegraded":"2023-09-23T23:35:29.060891+0000","last_fullsized":"2023-09-23T23:35:29.060891+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":456,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.046212+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:35:29.046212+0000","last_peered":"2023-09-23T23:35:29.046212+0000","last_clean":"2023-09-23T23:35:29.046212+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:35:29.046212+0000","last_undegraded":"2023-09-23T23:35:29.046212+0000","last_fullsized":"2023-09-23T23:35:29.046212+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":544,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.055724+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:35:35.055724+0000","last_peered":"2023-09-23T23:35:35.055724+0000","last_clean":"2023-09-23T23:35:35.055724+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:35:35.055724+0000","last_undegraded":"2023-09-23T23:35:35.055724+0000","last_fullsized":"2023-09-23T23:35:35.055724+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":951,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.034482+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:35:29.034482+0000","last_peered":"2023-09-23T23:35:29.034482+0000","last_clean":"2023-09-23T23:35:29.034482+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:35:29.034482+0000","last_undegraded":"2023-09-23T23:35:29.034482+0000","last_fullsized":"2023-09-23T23:35:29.034482+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":152,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.044851+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:35:29.044851+0000","last_peered":"2023-09-23T23:35:29.044851+0000","last_clean":"2023-09-23T23:35:29.044851+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:35:29.044851+0000","last_undegraded":"2023-09-23T23:35:29.044851+0000","last_fullsized":"2023-09-23T23:35:29.044851+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":228,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056772+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:35:29.056772+0000","last_peered":"2023-09-23T23:35:29.056772+0000","last_clean":"2023-09-23T23:35:29.056772+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:35:29.056772+0000","last_undegraded":"2023-09-23T23:35:29.056772+0000","last_fullsized":"2023-09-23T23:35:29.056772+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"588'6428","reported_seq":6499,"reported_epoch":588,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:38.981103+0000","last_change":"2023-09-23T23:33:52.884769+0000","last_active":"2023-09-23T23:35:38.981103+0000","last_peered":"2023-09-23T23:35:38.981103+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:35:38.981103+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"543'1000","ondisk_log_start":"543'1000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5428,"log_dups_size":0,"ondisk_log_size":5428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289669120,"num_objects":4420,"num_object_clones":0,"num_object_copies":13260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6428,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":3394},{"shards":"2,6","objects":22}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":920,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.401137+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:35:21.401137+0000","last_peered":"2023-09-23T23:35:21.401137+0000","last_clean":"2023-09-23T23:35:21.401137+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:35:21.401137+0000","last_undegraded":"2023-09-23T23:35:21.401137+0000","last_fullsized":"2023-09-23T23:35:21.401137+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":229,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.035896+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:35:29.035896+0000","last_peered":"2023-09-23T23:35:29.035896+0000","last_clean":"2023-09-23T23:35:29.035896+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:35:29.035896+0000","last_undegraded":"2023-09-23T23:35:29.035896+0000","last_fullsized":"2023-09-23T23:35:29.035896+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":219,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.034280+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:35:29.034280+0000","last_peered":"2023-09-23T23:35:29.034280+0000","last_clean":"2023-09-23T23:35:29.034280+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:35:29.034280+0000","last_undegraded":"2023-09-23T23:35:29.034280+0000","last_fullsized":"2023-09-23T23:35:29.034280+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"587'6445","reported_seq":6514,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:34.880806+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:35:34.880806+0000","last_peered":"2023-09-23T23:35:34.880806+0000","last_clean":"2023-09-23T23:35:34.880806+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:35:34.880806+0000","last_undegraded":"2023-09-23T23:35:34.880806+0000","last_fullsized":"2023-09-23T23:35:34.880806+0000","mapping_epoch":566,"log_start":"538'500","ondisk_log_start":"538'500","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5945,"log_dups_size":0,"ondisk_log_size":5945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300613632,"num_objects":4587,"num_object_clones":0,"num_object_copies":13761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6445,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":999,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056184+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:35:29.056184+0000","last_peered":"2023-09-23T23:35:29.056184+0000","last_clean":"2023-09-23T23:35:29.056184+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:35:29.056184+0000","last_undegraded":"2023-09-23T23:35:29.056184+0000","last_fullsized":"2023-09-23T23:35:29.056184+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":219,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.036294+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:35:29.036294+0000","last_peered":"2023-09-23T23:35:29.036294+0000","last_clean":"2023-09-23T23:35:29.036294+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:35:29.036294+0000","last_undegraded":"2023-09-23T23:35:29.036294+0000","last_fullsized":"2023-09-23T23:35:29.036294+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":438,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056823+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:35:29.056823+0000","last_peered":"2023-09-23T23:35:29.056823+0000","last_clean":"2023-09-23T23:35:29.056823+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:35:29.056823+0000","last_undegraded":"2023-09-23T23:35:29.056823+0000","last_fullsized":"2023-09-23T23:35:29.056823+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"588'6549","reported_seq":9704,"reported_epoch":588,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:35:37.037911+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:35:37.037911+0000","last_peered":"2023-09-23T23:35:37.037911+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:35:37.037911+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:35:37.037911+0000","mapping_epoch":571,"log_start":"544'1200","ondisk_log_start":"544'1200","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5349,"log_dups_size":0,"ondisk_log_size":5349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294846464,"num_objects":4499,"num_object_clones":0,"num_object_copies":13497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3604,"num_objects_misplaced":895,"num_objects_unfound":0,"num_objects_dirty":4499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6549,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1527,"num_bytes_recovered":100073472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":3482},{"shards":"3,5,7","objects":1017}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":898,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.045968+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:35:29.045968+0000","last_peered":"2023-09-23T23:35:29.045968+0000","last_clean":"2023-09-23T23:35:29.045968+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:35:29.045968+0000","last_undegraded":"2023-09-23T23:35:29.045968+0000","last_fullsized":"2023-09-23T23:35:29.045968+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":326,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.036830+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:35:29.036830+0000","last_peered":"2023-09-23T23:35:29.036830+0000","last_clean":"2023-09-23T23:35:29.036830+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:35:29.036830+0000","last_undegraded":"2023-09-23T23:35:29.036830+0000","last_fullsized":"2023-09-23T23:35:29.036830+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":102,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.045197+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:35:29.045197+0000","last_peered":"2023-09-23T23:35:29.045197+0000","last_clean":"2023-09-23T23:35:29.045197+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:35:29.045197+0000","last_undegraded":"2023-09-23T23:35:29.045197+0000","last_fullsized":"2023-09-23T23:35:29.045197+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"588'6489","reported_seq":8546,"reported_epoch":588,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:36.850127+0000","last_change":"2023-09-23T23:34:35.750428+0000","last_active":"2023-09-23T23:35:36.850127+0000","last_peered":"2023-09-23T23:35:36.850127+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:35:36.850127+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"545'1200","ondisk_log_start":"545'1200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5289,"log_dups_size":0,"ondisk_log_size":5289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301531136,"num_objects":4601,"num_object_clones":0,"num_object_copies":13803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6489,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":991,"num_bytes_recovered":64946176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":3544},{"shards":"3,5","objects":1057}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":494,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057061+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:35:29.057061+0000","last_peered":"2023-09-23T23:35:29.057061+0000","last_clean":"2023-09-23T23:35:29.057061+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:35:29.057061+0000","last_undegraded":"2023-09-23T23:35:29.057061+0000","last_fullsized":"2023-09-23T23:35:29.057061+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":428,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056345+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:35:29.056345+0000","last_peered":"2023-09-23T23:35:29.056345+0000","last_clean":"2023-09-23T23:35:29.056345+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:35:29.056345+0000","last_undegraded":"2023-09-23T23:35:29.056345+0000","last_fullsized":"2023-09-23T23:35:29.056345+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":584,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.052768+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:35:29.052768+0000","last_peered":"2023-09-23T23:35:29.052768+0000","last_clean":"2023-09-23T23:35:29.052768+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:35:29.052768+0000","last_undegraded":"2023-09-23T23:35:29.052768+0000","last_fullsized":"2023-09-23T23:35:29.052768+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"588'6606","reported_seq":7923,"reported_epoch":588,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:39.204138+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:35:39.204138+0000","last_peered":"2023-09-23T23:35:39.204138+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:35:39.204138+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"543'900","ondisk_log_start":"543'900","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5706,"log_dups_size":0,"ondisk_log_size":5706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298582016,"num_objects":4556,"num_object_clones":0,"num_object_copies":13668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4556,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6606,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":3510},{"shards":"3,5","objects":-904},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":715,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057140+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:35:29.057140+0000","last_peered":"2023-09-23T23:35:29.057140+0000","last_clean":"2023-09-23T23:35:29.057140+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:35:29.057140+0000","last_undegraded":"2023-09-23T23:35:29.057140+0000","last_fullsized":"2023-09-23T23:35:29.057140+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":597,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.060843+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:35:29.060843+0000","last_peered":"2023-09-23T23:35:29.060843+0000","last_clean":"2023-09-23T23:35:29.060843+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:35:29.060843+0000","last_undegraded":"2023-09-23T23:35:29.060843+0000","last_fullsized":"2023-09-23T23:35:29.060843+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":508,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056265+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:35:29.056265+0000","last_peered":"2023-09-23T23:35:29.056265+0000","last_clean":"2023-09-23T23:35:29.056265+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:35:29.056265+0000","last_undegraded":"2023-09-23T23:35:29.056265+0000","last_fullsized":"2023-09-23T23:35:29.056265+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"588'6535","reported_seq":6597,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:39.136208+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:35:39.136208+0000","last_peered":"2023-09-23T23:35:39.136208+0000","last_clean":"2023-09-23T23:35:39.136208+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:35:39.136208+0000","last_undegraded":"2023-09-23T23:35:39.136208+0000","last_fullsized":"2023-09-23T23:35:39.136208+0000","mapping_epoch":534,"log_start":"544'1100","ondisk_log_start":"544'1100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5435,"log_dups_size":0,"ondisk_log_size":5435,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297861120,"num_objects":4545,"num_object_clones":0,"num_object_copies":13635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6535,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":870,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.062481+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:35:29.062481+0000","last_peered":"2023-09-23T23:35:29.062481+0000","last_clean":"2023-09-23T23:35:29.062481+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:35:29.062481+0000","last_undegraded":"2023-09-23T23:35:29.062481+0000","last_fullsized":"2023-09-23T23:35:29.062481+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":112,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.045119+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:35:29.045119+0000","last_peered":"2023-09-23T23:35:29.045119+0000","last_clean":"2023-09-23T23:35:29.045119+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:35:29.045119+0000","last_undegraded":"2023-09-23T23:35:29.045119+0000","last_fullsized":"2023-09-23T23:35:29.045119+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":452,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.401241+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:35:21.401241+0000","last_peered":"2023-09-23T23:35:21.401241+0000","last_clean":"2023-09-23T23:35:21.401241+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:35:21.401241+0000","last_undegraded":"2023-09-23T23:35:21.401241+0000","last_fullsized":"2023-09-23T23:35:21.401241+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"588'6603","reported_seq":8787,"reported_epoch":588,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:35:38.954143+0000","last_change":"2023-09-23T23:34:35.785270+0000","last_active":"2023-09-23T23:35:38.954143+0000","last_peered":"2023-09-23T23:35:38.954143+0000","last_clean":"2023-09-23T23:33:50.504296+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:35:38.954143+0000","last_undegraded":"2023-09-23T23:34:35.780981+0000","last_fullsized":"2023-09-23T23:35:38.954143+0000","mapping_epoch":567,"log_start":"544'1200","ondisk_log_start":"544'1200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5403,"log_dups_size":0,"ondisk_log_size":5403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294977536,"num_objects":4501,"num_object_clones":0,"num_object_copies":13503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6603,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1051,"num_bytes_recovered":68878336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":3496},{"shards":"5,6,7","objects":1005}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":596,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.034681+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:35:29.034681+0000","last_peered":"2023-09-23T23:35:29.034681+0000","last_clean":"2023-09-23T23:35:29.034681+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:35:29.034681+0000","last_undegraded":"2023-09-23T23:35:29.034681+0000","last_fullsized":"2023-09-23T23:35:29.034681+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":449,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.400831+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:35:21.400831+0000","last_peered":"2023-09-23T23:35:21.400831+0000","last_clean":"2023-09-23T23:35:21.400831+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:35:21.400831+0000","last_undegraded":"2023-09-23T23:35:21.400831+0000","last_fullsized":"2023-09-23T23:35:21.400831+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":539,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.044931+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:35:29.044931+0000","last_peered":"2023-09-23T23:35:29.044931+0000","last_clean":"2023-09-23T23:35:29.044931+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:35:29.044931+0000","last_undegraded":"2023-09-23T23:35:29.044931+0000","last_fullsized":"2023-09-23T23:35:29.044931+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"588'6570","reported_seq":15595,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:39.257973+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:35:39.257973+0000","last_peered":"2023-09-23T23:35:39.257973+0000","last_clean":"2023-09-23T23:35:39.257973+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:35:39.257973+0000","last_undegraded":"2023-09-23T23:35:39.257973+0000","last_fullsized":"2023-09-23T23:35:39.257973+0000","mapping_epoch":579,"log_start":"543'900","ondisk_log_start":"543'900","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5670,"log_dups_size":0,"ondisk_log_size":5670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291373056,"num_objects":4446,"num_object_clones":0,"num_object_copies":13338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6570,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":928,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.404843+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:35:21.404843+0000","last_peered":"2023-09-23T23:35:21.404843+0000","last_clean":"2023-09-23T23:35:21.404843+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:35:21.404843+0000","last_undegraded":"2023-09-23T23:35:21.404843+0000","last_fullsized":"2023-09-23T23:35:21.404843+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":229,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056015+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:35:29.056015+0000","last_peered":"2023-09-23T23:35:29.056015+0000","last_clean":"2023-09-23T23:35:29.056015+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:35:29.056015+0000","last_undegraded":"2023-09-23T23:35:29.056015+0000","last_fullsized":"2023-09-23T23:35:29.056015+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":517,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056942+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:35:29.056942+0000","last_peered":"2023-09-23T23:35:29.056942+0000","last_clean":"2023-09-23T23:35:29.056942+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:35:29.056942+0000","last_undegraded":"2023-09-23T23:35:29.056942+0000","last_fullsized":"2023-09-23T23:35:29.056942+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"588'6277","reported_seq":6341,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:36.997488+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:35:36.997488+0000","last_peered":"2023-09-23T23:35:36.997488+0000","last_clean":"2023-09-23T23:35:36.997488+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:35:36.997488+0000","last_undegraded":"2023-09-23T23:35:36.997488+0000","last_fullsized":"2023-09-23T23:35:36.997488+0000","mapping_epoch":534,"log_start":"544'1000","ondisk_log_start":"544'1000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5277,"log_dups_size":0,"ondisk_log_size":5277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291700736,"num_objects":4451,"num_object_clones":0,"num_object_copies":13353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6277,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":909,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.045726+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:35:29.045726+0000","last_peered":"2023-09-23T23:35:29.045726+0000","last_clean":"2023-09-23T23:35:29.045726+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:35:29.045726+0000","last_undegraded":"2023-09-23T23:35:29.045726+0000","last_fullsized":"2023-09-23T23:35:29.045726+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":297,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.081894+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:35:29.081894+0000","last_peered":"2023-09-23T23:35:29.081894+0000","last_clean":"2023-09-23T23:35:29.081894+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:35:29.081894+0000","last_undegraded":"2023-09-23T23:35:29.081894+0000","last_fullsized":"2023-09-23T23:35:29.081894+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":603,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.036992+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:35:29.036992+0000","last_peered":"2023-09-23T23:35:29.036992+0000","last_clean":"2023-09-23T23:35:29.036992+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:35:29.036992+0000","last_undegraded":"2023-09-23T23:35:29.036992+0000","last_fullsized":"2023-09-23T23:35:29.036992+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":184,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.034245+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:35:29.034245+0000","last_peered":"2023-09-23T23:35:29.034245+0000","last_clean":"2023-09-23T23:35:29.034245+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:35:29.034245+0000","last_undegraded":"2023-09-23T23:35:29.034245+0000","last_fullsized":"2023-09-23T23:35:29.034245+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"588'6592","reported_seq":8703,"reported_epoch":588,"state":"active+recovering","last_fresh":"2023-09-23T23:35:37.021471+0000","last_change":"2023-09-23T23:34:35.815566+0000","last_active":"2023-09-23T23:35:37.021471+0000","last_peered":"2023-09-23T23:35:37.021471+0000","last_clean":"2023-09-23T23:33:50.519308+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:35:37.021471+0000","last_undegraded":"2023-09-23T23:35:37.021471+0000","last_fullsized":"2023-09-23T23:35:37.021471+0000","mapping_epoch":566,"log_start":"546'1322","ondisk_log_start":"546'1322","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5270,"log_dups_size":0,"ondisk_log_size":5270,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299827224,"num_objects":4576,"num_object_clones":0,"num_object_copies":13728,"num_objects_missing_on_primary":3495,"num_objects_missing":3495,"num_objects_degraded":0,"num_objects_misplaced":3495,"num_objects_unfound":0,"num_objects_dirty":4576,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6592,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1007,"num_bytes_recovered":65994752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,3,4,7","objects":13},{"shards":"1,3,7","objects":1068},{"shards":"1,4,7","objects":3495}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":671,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.052247+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:35:29.052247+0000","last_peered":"2023-09-23T23:35:29.052247+0000","last_clean":"2023-09-23T23:35:29.052247+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:35:29.052247+0000","last_undegraded":"2023-09-23T23:35:29.052247+0000","last_fullsized":"2023-09-23T23:35:29.052247+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":236,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.045042+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:35:29.045042+0000","last_peered":"2023-09-23T23:35:29.045042+0000","last_clean":"2023-09-23T23:35:29.045042+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:35:29.045042+0000","last_undegraded":"2023-09-23T23:35:29.045042+0000","last_fullsized":"2023-09-23T23:35:29.045042+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":520,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.045009+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:35:29.045009+0000","last_peered":"2023-09-23T23:35:29.045009+0000","last_clean":"2023-09-23T23:35:29.045009+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:35:29.045009+0000","last_undegraded":"2023-09-23T23:35:29.045009+0000","last_fullsized":"2023-09-23T23:35:29.045009+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":219,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.405169+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:35:21.405169+0000","last_peered":"2023-09-23T23:35:21.405169+0000","last_clean":"2023-09-23T23:35:21.405169+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:35:21.405169+0000","last_undegraded":"2023-09-23T23:35:21.405169+0000","last_fullsized":"2023-09-23T23:35:21.405169+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"588'6324","reported_seq":8365,"reported_epoch":588,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:39.296855+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:35:39.296855+0000","last_peered":"2023-09-23T23:35:39.296855+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:35:39.296855+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"542'600","ondisk_log_start":"542'600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5724,"log_dups_size":0,"ondisk_log_size":5724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291504128,"num_objects":4448,"num_object_clones":0,"num_object_copies":13344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6324,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":3392},{"shards":"5,7","objects":1056}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":888,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.404750+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:35:21.404750+0000","last_peered":"2023-09-23T23:35:21.404750+0000","last_clean":"2023-09-23T23:35:21.404750+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:35:21.404750+0000","last_undegraded":"2023-09-23T23:35:21.404750+0000","last_fullsized":"2023-09-23T23:35:21.404750+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":325,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.400756+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:35:21.400756+0000","last_peered":"2023-09-23T23:35:21.400756+0000","last_clean":"2023-09-23T23:35:21.400756+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:35:21.400756+0000","last_undegraded":"2023-09-23T23:35:21.400756+0000","last_fullsized":"2023-09-23T23:35:21.400756+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":234,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.400769+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:35:21.400769+0000","last_peered":"2023-09-23T23:35:21.400769+0000","last_clean":"2023-09-23T23:35:21.400769+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:35:21.400769+0000","last_undegraded":"2023-09-23T23:35:21.400769+0000","last_fullsized":"2023-09-23T23:35:21.400769+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":199,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.080733+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:35:29.080733+0000","last_peered":"2023-09-23T23:35:29.080733+0000","last_clean":"2023-09-23T23:35:29.080733+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:35:29.080733+0000","last_undegraded":"2023-09-23T23:35:29.080733+0000","last_fullsized":"2023-09-23T23:35:29.080733+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"588'6462","reported_seq":6534,"reported_epoch":588,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:35:39.238004+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:35:39.238004+0000","last_peered":"2023-09-23T23:35:39.238004+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:35:39.238004+0000","last_undegraded":"2023-09-23T23:35:39.238004+0000","last_fullsized":"2023-09-23T23:35:39.238004+0000","mapping_epoch":562,"log_start":"543'800","ondisk_log_start":"543'800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5662,"log_dups_size":0,"ondisk_log_size":5662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290979840,"num_objects":4440,"num_object_clones":0,"num_object_copies":13320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6462,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":4440}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":601,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.400879+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:35:21.400879+0000","last_peered":"2023-09-23T23:35:21.400879+0000","last_clean":"2023-09-23T23:35:21.400879+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:35:21.400879+0000","last_undegraded":"2023-09-23T23:35:21.400879+0000","last_fullsized":"2023-09-23T23:35:21.400879+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":874,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.044669+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:35:29.044669+0000","last_peered":"2023-09-23T23:35:29.044669+0000","last_clean":"2023-09-23T23:35:29.044669+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:35:29.044669+0000","last_undegraded":"2023-09-23T23:35:29.044669+0000","last_fullsized":"2023-09-23T23:35:29.044669+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":549,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056088+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:35:29.056088+0000","last_peered":"2023-09-23T23:35:29.056088+0000","last_clean":"2023-09-23T23:35:29.056088+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:35:29.056088+0000","last_undegraded":"2023-09-23T23:35:29.056088+0000","last_fullsized":"2023-09-23T23:35:29.056088+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":112,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.045356+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:35:29.045356+0000","last_peered":"2023-09-23T23:35:29.045356+0000","last_clean":"2023-09-23T23:35:29.045356+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:35:29.045356+0000","last_undegraded":"2023-09-23T23:35:29.045356+0000","last_fullsized":"2023-09-23T23:35:29.045356+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"588'6347","reported_seq":6409,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:39.081683+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:35:39.081683+0000","last_peered":"2023-09-23T23:35:39.081683+0000","last_clean":"2023-09-23T23:35:39.081683+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:35:39.081683+0000","last_undegraded":"2023-09-23T23:35:39.081683+0000","last_fullsized":"2023-09-23T23:35:39.081683+0000","mapping_epoch":534,"log_start":"543'900","ondisk_log_start":"543'900","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5447,"log_dups_size":0,"ondisk_log_size":5447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287637504,"num_objects":4389,"num_object_clones":0,"num_object_copies":13167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6347,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":700,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.044851+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:35:29.044851+0000","last_peered":"2023-09-23T23:35:29.044851+0000","last_clean":"2023-09-23T23:35:29.044851+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:35:29.044851+0000","last_undegraded":"2023-09-23T23:35:29.044851+0000","last_fullsized":"2023-09-23T23:35:29.044851+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":515,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.035229+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:35:29.035229+0000","last_peered":"2023-09-23T23:35:29.035229+0000","last_clean":"2023-09-23T23:35:29.035229+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:35:29.035229+0000","last_undegraded":"2023-09-23T23:35:29.035229+0000","last_fullsized":"2023-09-23T23:35:29.035229+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":428,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056912+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:35:29.056912+0000","last_peered":"2023-09-23T23:35:29.056912+0000","last_clean":"2023-09-23T23:35:29.056912+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:35:29.056912+0000","last_undegraded":"2023-09-23T23:35:29.056912+0000","last_fullsized":"2023-09-23T23:35:29.056912+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":28,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.052862+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:35:29.052862+0000","last_peered":"2023-09-23T23:35:29.052862+0000","last_clean":"2023-09-23T23:35:29.052862+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:35:29.052862+0000","last_undegraded":"2023-09-23T23:35:29.052862+0000","last_fullsized":"2023-09-23T23:35:29.052862+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"587'6409","reported_seq":10069,"reported_epoch":587,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:34.671793+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:35:34.671793+0000","last_peered":"2023-09-23T23:35:34.671793+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:35:34.671793+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"538'200","ondisk_log_start":"538'200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6209,"log_dups_size":0,"ondisk_log_size":6209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300744704,"num_objects":4589,"num_object_clones":0,"num_object_copies":13767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6409,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":1885},{"shards":"1,2,4","objects":2704}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":583,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.045222+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:35:29.045222+0000","last_peered":"2023-09-23T23:35:29.045222+0000","last_clean":"2023-09-23T23:35:29.045222+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:35:29.045222+0000","last_undegraded":"2023-09-23T23:35:29.045222+0000","last_fullsized":"2023-09-23T23:35:29.045222+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.052522+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:35:29.052522+0000","last_peered":"2023-09-23T23:35:29.052522+0000","last_clean":"2023-09-23T23:35:29.052522+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:35:29.052522+0000","last_undegraded":"2023-09-23T23:35:29.052522+0000","last_fullsized":"2023-09-23T23:35:29.052522+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":126,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.405213+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:35:21.405213+0000","last_peered":"2023-09-23T23:35:21.405213+0000","last_clean":"2023-09-23T23:35:21.405213+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:35:21.405213+0000","last_undegraded":"2023-09-23T23:35:21.405213+0000","last_fullsized":"2023-09-23T23:35:21.405213+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":321,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.034881+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:35:29.034881+0000","last_peered":"2023-09-23T23:35:29.034881+0000","last_clean":"2023-09-23T23:35:29.034881+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:35:29.034881+0000","last_undegraded":"2023-09-23T23:35:29.034881+0000","last_fullsized":"2023-09-23T23:35:29.034881+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"588'6519","reported_seq":6581,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:39.233821+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:35:39.233821+0000","last_peered":"2023-09-23T23:35:39.233821+0000","last_clean":"2023-09-23T23:35:39.233821+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:35:39.233821+0000","last_undegraded":"2023-09-23T23:35:39.233821+0000","last_fullsized":"2023-09-23T23:35:39.233821+0000","mapping_epoch":534,"log_start":"542'800","ondisk_log_start":"542'800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5719,"log_dups_size":0,"ondisk_log_size":5719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291307520,"num_objects":4445,"num_object_clones":0,"num_object_copies":13335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6519,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":767,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.401292+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:35:21.401292+0000","last_peered":"2023-09-23T23:35:21.401292+0000","last_clean":"2023-09-23T23:35:21.401292+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:35:21.401292+0000","last_undegraded":"2023-09-23T23:35:21.401292+0000","last_fullsized":"2023-09-23T23:35:21.401292+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":585,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056943+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:35:29.056943+0000","last_peered":"2023-09-23T23:35:29.056943+0000","last_clean":"2023-09-23T23:35:29.056943+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:35:29.056943+0000","last_undegraded":"2023-09-23T23:35:29.056943+0000","last_fullsized":"2023-09-23T23:35:29.056943+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":192,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.045003+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:35:29.045003+0000","last_peered":"2023-09-23T23:35:29.045003+0000","last_clean":"2023-09-23T23:35:29.045003+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:35:29.045003+0000","last_undegraded":"2023-09-23T23:35:29.045003+0000","last_fullsized":"2023-09-23T23:35:29.045003+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":138,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.400603+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:35:21.400603+0000","last_peered":"2023-09-23T23:35:21.400603+0000","last_clean":"2023-09-23T23:35:21.400603+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:35:21.400603+0000","last_undegraded":"2023-09-23T23:35:21.400603+0000","last_fullsized":"2023-09-23T23:35:21.400603+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"587'6363","reported_seq":6426,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:34.858919+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:35:34.858919+0000","last_peered":"2023-09-23T23:35:34.858919+0000","last_clean":"2023-09-23T23:35:34.858919+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:35:34.858919+0000","last_undegraded":"2023-09-23T23:35:34.858919+0000","last_fullsized":"2023-09-23T23:35:34.858919+0000","mapping_epoch":534,"log_start":"538'200","ondisk_log_start":"538'200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6163,"log_dups_size":0,"ondisk_log_size":6163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297861120,"num_objects":4545,"num_object_clones":0,"num_object_copies":13635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6363,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":974,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056609+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:35:29.056609+0000","last_peered":"2023-09-23T23:35:29.056609+0000","last_clean":"2023-09-23T23:35:29.056609+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:35:29.056609+0000","last_undegraded":"2023-09-23T23:35:29.056609+0000","last_fullsized":"2023-09-23T23:35:29.056609+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":102,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.045809+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:35:29.045809+0000","last_peered":"2023-09-23T23:35:29.045809+0000","last_clean":"2023-09-23T23:35:29.045809+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:35:29.045809+0000","last_undegraded":"2023-09-23T23:35:29.045809+0000","last_fullsized":"2023-09-23T23:35:29.045809+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":32,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.052950+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:35:29.052950+0000","last_peered":"2023-09-23T23:35:29.052950+0000","last_clean":"2023-09-23T23:35:29.052950+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:35:29.052950+0000","last_undegraded":"2023-09-23T23:35:29.052950+0000","last_fullsized":"2023-09-23T23:35:29.052950+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":162,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.034320+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:35:29.034320+0000","last_peered":"2023-09-23T23:35:29.034320+0000","last_clean":"2023-09-23T23:35:29.034320+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:35:29.034320+0000","last_undegraded":"2023-09-23T23:35:29.034320+0000","last_fullsized":"2023-09-23T23:35:29.034320+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1024,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057056+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:35:29.057056+0000","last_peered":"2023-09-23T23:35:29.057056+0000","last_clean":"2023-09-23T23:35:29.057056+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:35:29.057056+0000","last_undegraded":"2023-09-23T23:35:29.057056+0000","last_fullsized":"2023-09-23T23:35:29.057056+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":28,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.052843+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:35:29.052843+0000","last_peered":"2023-09-23T23:35:29.052843+0000","last_clean":"2023-09-23T23:35:29.052843+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:35:29.052843+0000","last_undegraded":"2023-09-23T23:35:29.052843+0000","last_fullsized":"2023-09-23T23:35:29.052843+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":342,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.405257+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:35:21.405257+0000","last_peered":"2023-09-23T23:35:21.405257+0000","last_clean":"2023-09-23T23:35:21.405257+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:35:21.405257+0000","last_undegraded":"2023-09-23T23:35:21.405257+0000","last_fullsized":"2023-09-23T23:35:21.405257+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":735,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057194+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:35:29.057194+0000","last_peered":"2023-09-23T23:35:29.057194+0000","last_clean":"2023-09-23T23:35:29.057194+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:35:29.057194+0000","last_undegraded":"2023-09-23T23:35:29.057194+0000","last_fullsized":"2023-09-23T23:35:29.057194+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":90,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.045318+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:35:29.045318+0000","last_peered":"2023-09-23T23:35:29.045318+0000","last_clean":"2023-09-23T23:35:29.045318+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:35:29.045318+0000","last_undegraded":"2023-09-23T23:35:29.045318+0000","last_fullsized":"2023-09-23T23:35:29.045318+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":435,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.060786+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:35:29.060786+0000","last_peered":"2023-09-23T23:35:29.060786+0000","last_clean":"2023-09-23T23:35:29.060786+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:35:29.060786+0000","last_undegraded":"2023-09-23T23:35:29.060786+0000","last_fullsized":"2023-09-23T23:35:29.060786+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":286,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.045284+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:35:29.045284+0000","last_peered":"2023-09-23T23:35:29.045284+0000","last_clean":"2023-09-23T23:35:29.045284+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:35:29.045284+0000","last_undegraded":"2023-09-23T23:35:29.045284+0000","last_fullsized":"2023-09-23T23:35:29.045284+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":274,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056159+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:35:29.056159+0000","last_peered":"2023-09-23T23:35:29.056159+0000","last_clean":"2023-09-23T23:35:29.056159+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:35:29.056159+0000","last_undegraded":"2023-09-23T23:35:29.056159+0000","last_fullsized":"2023-09-23T23:35:29.056159+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":800,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.045483+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:35:29.045483+0000","last_peered":"2023-09-23T23:35:29.045483+0000","last_clean":"2023-09-23T23:35:29.045483+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:35:29.045483+0000","last_undegraded":"2023-09-23T23:35:29.045483+0000","last_fullsized":"2023-09-23T23:35:29.045483+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":618,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057149+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:35:29.057149+0000","last_peered":"2023-09-23T23:35:29.057149+0000","last_clean":"2023-09-23T23:35:29.057149+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:35:29.057149+0000","last_undegraded":"2023-09-23T23:35:29.057149+0000","last_fullsized":"2023-09-23T23:35:29.057149+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":602,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057333+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:35:29.057333+0000","last_peered":"2023-09-23T23:35:29.057333+0000","last_clean":"2023-09-23T23:35:29.057333+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:35:29.057333+0000","last_undegraded":"2023-09-23T23:35:29.057333+0000","last_fullsized":"2023-09-23T23:35:29.057333+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":760,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.067681+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:35:29.067681+0000","last_peered":"2023-09-23T23:35:29.067681+0000","last_clean":"2023-09-23T23:35:29.067681+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:35:29.067681+0000","last_undegraded":"2023-09-23T23:35:29.067681+0000","last_fullsized":"2023-09-23T23:35:29.067681+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":430,"reported_epoch":586,"state":"active+clean","last_fresh":"2023-09-23T23:35:21.404976+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:35:21.404976+0000","last_peered":"2023-09-23T23:35:21.404976+0000","last_clean":"2023-09-23T23:35:21.404976+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:35:21.404976+0000","last_undegraded":"2023-09-23T23:35:21.404976+0000","last_fullsized":"2023-09-23T23:35:21.404976+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":655,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.044573+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:35:29.044573+0000","last_peered":"2023-09-23T23:35:29.044573+0000","last_clean":"2023-09-23T23:35:29.044573+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:35:29.044573+0000","last_undegraded":"2023-09-23T23:35:29.044573+0000","last_fullsized":"2023-09-23T23:35:29.044573+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":4721016856,"num_objects":72038,"num_object_clones":0,"num_object_copies":216114,"num_objects_missing_on_primary":3495,"num_objects_missing":3495,"num_objects_degraded":26716,"num_objects_misplaced":4390,"num_objects_unfound":0,"num_objects_dirty":72038,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":103518,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12433,"num_bytes_recovered":814612480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14979051520,"data_stored":14979039304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89696,"ondisk_log_size":89696,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782674,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2062944,"kb_used_data":1694296,"kb_used_omap":8,"kb_used_meta":368631,"kb_avail":91682208,"statfs":{"total":95995035648,"available":93882580992,"internally_reserved":0,"allocated":1734959104,"data_stored":1733331758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":377478213},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.9339999999999999,"5min":2.3500000000000001,"15min":2.2549999999999999},"min":{"1min":0.26600000000000001,"5min":0.224,"15min":0.20899999999999999},"max":{"1min":11.173,"5min":44.094999999999999,"15min":85.721000000000004},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.1519999999999999,"5min":2.2080000000000002,"15min":2.2669999999999999},"min":{"1min":0.28199999999999997,"5min":0.25600000000000001,"15min":0.19800000000000001},"max":{"1min":4.3109999999999999,"5min":44.335999999999999,"15min":85.965000000000003},"last":0.41899999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8819999999999999,"5min":6.5810000000000004,"15min":3.202},"min":{"1min":0.252,"5min":0.248,"15min":0.22900000000000001},"max":{"1min":9.0139999999999993,"5min":198.58000000000001,"15min":198.58000000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.179,"5min":6.3109999999999999,"15min":3.2610000000000001},"min":{"1min":0.254,"5min":0.247,"15min":0.246},"max":{"1min":4.1660000000000004,"5min":198.71100000000001,"15min":198.71100000000001},"last":0.49199999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":6.0019999999999998,"5min":5.7329999999999997,"15min":3.7810000000000001},"min":{"1min":0.26900000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":78.528999999999996,"5min":203.58000000000001,"15min":203.58000000000001},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.8799999999999999,"5min":4.5350000000000001,"15min":3.1080000000000001},"min":{"1min":0.27900000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.5670000000000002,"5min":198.69,"15min":198.69},"last":0.442}]},{"osd":3,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":7.0250000000000004,"15min":3.6579999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":6.1210000000000004,"5min":279.26499999999999,"15min":279.26499999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":3.1579999999999999,"5min":8.7859999999999996,"15min":4.1630000000000003},"min":{"1min":0.40200000000000002,"5min":0.32000000000000001,"15min":0.29599999999999999},"max":{"1min":18.643999999999998,"5min":279.29899999999998,"15min":279.29899999999998},"last":0.45200000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":1.857,"15min":1.0940000000000001},"min":{"1min":0.35299999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":10.6,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.372},{"interface":"front","average":{"1min":0.79600000000000004,"5min":1.4970000000000001,"15min":1.03},"min":{"1min":0.317,"5min":0.28100000000000003,"15min":0.23899999999999999},"max":{"1min":2.8570000000000002,"5min":31.68,"15min":31.68},"last":0.50900000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":1.1659999999999999,"15min":1.7290000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26500000000000001},"max":{"1min":12.273999999999999,"5min":12.273999999999999,"15min":210.83600000000001},"last":60.463999999999999},{"interface":"front","average":{"1min":2.141,"5min":1.0529999999999999,"15min":1.579},"min":{"1min":0.371,"5min":0.316,"15min":0.28199999999999997},"max":{"1min":18.117999999999999,"5min":18.117999999999999,"15min":207.04599999999999},"last":63.006}]},{"osd":6,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.96,"15min":1.3620000000000001},"min":{"1min":0.379,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.6440000000000001,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.2109999999999999,"5min":2.403,"15min":1.615},"min":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.33300000000000002},"max":{"1min":10.332000000000001,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.47599999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496151,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2273104,"kb_used_data":1571336,"kb_used_omap":0,"kb_used_meta":701760,"kb_avail":91472048,"statfs":{"total":95995035648,"available":93667377152,"internally_reserved":0,"allocated":1609048064,"data_stored":1607436711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":718602240},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:14 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.98799999999999999,"15min":4.7699999999999996},"min":{"1min":0.26500000000000001,"5min":0.223,"15min":0.20399999999999999},"max":{"1min":2.274,"5min":8.8119999999999994,"15min":135.31999999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.93000000000000005,"15min":4.617},"min":{"1min":0.26800000000000002,"5min":0.23999999999999999,"15min":0.22900000000000001},"max":{"1min":3.8239999999999998,"5min":8.3230000000000004,"15min":135.34299999999999},"last":0.38400000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:35:14 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":3.3690000000000002,"15min":2.4089999999999998},"min":{"1min":0.26000000000000001,"5min":0.251,"15min":0.245},"max":{"1min":6.9080000000000004,"5min":89.605000000000004,"15min":89.605000000000004},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.71599999999999997,"5min":3.1600000000000001,"15min":2.4849999999999999},"min":{"1min":0.33500000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":1.9159999999999999,"5min":91.025999999999996,"15min":91.025999999999996},"last":0.46800000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:35:03 2023","interfaces":[{"interface":"back","average":{"1min":9.4120000000000008,"5min":6.4420000000000002,"15min":4.6150000000000002},"min":{"1min":0.315,"5min":0.28399999999999997,"15min":0.27800000000000002},"max":{"1min":67.796999999999997,"5min":97.408000000000001,"15min":127.44499999999999},"last":0.434},{"interface":"front","average":{"1min":8.6270000000000007,"5min":7.6139999999999999,"15min":5.0419999999999998},"min":{"1min":0.30599999999999999,"5min":0.29999999999999999,"15min":0.26500000000000001},"max":{"1min":67.893000000000001,"5min":125.539,"15min":125.539},"last":0.46000000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:34:48 2023","interfaces":[{"interface":"back","average":{"1min":8.5120000000000005,"5min":5.7119999999999997,"15min":3.9969999999999999},"min":{"1min":0.39400000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":69.433000000000007,"5min":138.87299999999999,"15min":138.87299999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":8.0280000000000005,"5min":5.5289999999999999,"15min":4.2859999999999996},"min":{"1min":0.35199999999999998,"5min":0.33200000000000002,"15min":0.28799999999999998},"max":{"1min":69.463999999999999,"5min":138.976,"15min":138.976},"last":0.36399999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.82999999999999996,"15min":1.274},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":1.633,"5min":7.101,"15min":166.05199999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.55800000000000005,"5min":1.0289999999999999,"15min":1.4890000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.089,"5min":8.3499999999999996,"15min":166.631},"last":0.496}]},{"osd":5,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.96699999999999997,"15min":0.82399999999999995},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.1579999999999999,"5min":11.473000000000001,"15min":11.473000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.433,"5min":1.091,"15min":0.94099999999999995},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.249},"max":{"1min":0.76700000000000002,"5min":7.5670000000000002,"15min":8.4090000000000007},"last":0.44900000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:35:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.5389999999999999,"15min":1.0580000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":2.3010000000000002,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.573,"15min":1.081},"min":{"1min":0.36199999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.9009999999999998,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.60299999999999998}]}]},{"osd":1,"up_from":112,"seq":481036337491,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2360772,"kb_used_data":1469884,"kb_used_omap":4,"kb_used_meta":890875,"kb_avail":91384380,"statfs":{"total":95995035648,"available":93577605120,"internally_reserved":0,"allocated":1505161216,"data_stored":1502392559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":912256910},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":2.8220000000000001,"15min":3.5670000000000002},"min":{"1min":0.21199999999999999,"5min":0.17799999999999999,"15min":0.16700000000000001},"max":{"1min":15.983000000000001,"5min":47.195999999999998,"15min":130.768},"last":2.7970000000000002},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.883,"15min":2.8650000000000002},"min":{"1min":0.20699999999999999,"5min":0.20000000000000001,"15min":0.19},"max":{"1min":4.9809999999999999,"5min":46.582000000000001,"15min":134.929},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:34:57 2023","interfaces":[{"interface":"back","average":{"1min":10.555,"5min":7.9610000000000003,"15min":6.7919999999999998},"min":{"1min":0.23000000000000001,"5min":0.214,"15min":0.20100000000000001},"max":{"1min":201.03999999999999,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.317},{"interface":"front","average":{"1min":10.605,"5min":8.173,"15min":6.9109999999999996},"min":{"1min":0.24299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":202.12,"5min":202.12,"15min":232.21000000000001},"last":0.29499999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:34:57 2023","interfaces":[{"interface":"back","average":{"1min":9.6969999999999992,"5min":5.7679999999999998,"15min":5.1859999999999999},"min":{"1min":0.311,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":163.94300000000001,"5min":181.94399999999999,"15min":234.404},"last":0.35599999999999998},{"interface":"front","average":{"1min":9.9290000000000003,"5min":5.6550000000000002,"15min":5.0430000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.24399999999999999},"max":{"1min":163.98699999999999,"5min":181.98500000000001,"15min":236.33699999999999},"last":0.52100000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":1.5569999999999999,"15min":1.4339999999999999},"min":{"1min":0.32700000000000001,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":2.8210000000000002,"5min":12.042999999999999,"15min":12.042999999999999},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.5609999999999999,"15min":1.498},"min":{"1min":0.34499999999999997,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":2.7810000000000001,"5min":18.108000000000001,"15min":18.108000000000001},"last":0.499}]},{"osd":5,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":3.2879999999999998,"15min":1.8260000000000001},"min":{"1min":0.307,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.0460000000000003,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.91300000000000003,"5min":3.2890000000000001,"15min":2.141},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.7450000000000001,"5min":151.821,"15min":151.821},"last":0.46600000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.6850000000000001,"15min":1.954},"min":{"1min":0.34699999999999998,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":5.0960000000000001,"5min":8.9269999999999996,"15min":74.087999999999994},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.3300000000000001,"5min":1.885,"15min":2.157},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.32800000000000001},"max":{"1min":6.8920000000000003,"5min":11.927,"15min":73.974999999999994},"last":0.57999999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:34:57 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":2.1269999999999998,"15min":2.875},"min":{"1min":0.38700000000000001,"5min":0.37,"15min":0.32900000000000001},"max":{"1min":10.629,"5min":44.241,"15min":78.308000000000007},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.165,"5min":2.218,"15min":2.706},"min":{"1min":0.42899999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.9880000000000004,"5min":43.920999999999999,"15min":77.204999999999998},"last":0.625}]}]},{"osd":0,"up_from":91,"seq":390842024315,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1576848,"kb_used_data":366600,"kb_used_omap":5,"kb_used_meta":1210234,"kb_avail":92168304,"statfs":{"total":95995035648,"available":94380343296,"internally_reserved":0,"allocated":375398400,"data_stored":372612530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1239280250},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:35:32 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":10.577,"15min":6.7670000000000003},"min":{"1min":0.20499999999999999,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":9.0310000000000006,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.754,"5min":10.407999999999999,"15min":5.9059999999999997},"min":{"1min":0.20200000000000001,"5min":0.19700000000000001,"15min":0.17599999999999999},"max":{"1min":5.4729999999999999,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.55100000000000005}]},{"osd":2,"last update":"Sat Sep 23 23:34:55 2023","interfaces":[{"interface":"back","average":{"1min":19.536999999999999,"5min":10.204000000000001,"15min":7.4470000000000001},"min":{"1min":0.23300000000000001,"5min":0.17699999999999999,"15min":0.17199999999999999},"max":{"1min":283.07400000000001,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.439},{"interface":"front","average":{"1min":18.260000000000002,"5min":10.146000000000001,"15min":7.5259999999999998},"min":{"1min":0.26800000000000002,"5min":0.19800000000000001,"15min":0.17599999999999999},"max":{"1min":283.161,"5min":283.161,"15min":283.161},"last":0.27000000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:35:32 2023","interfaces":[{"interface":"back","average":{"1min":2.7360000000000002,"5min":8.6509999999999998,"15min":5.4589999999999996},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":41.191000000000003,"5min":220.732,"15min":220.732},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.9390000000000001,"5min":9.0440000000000005,"15min":5.5570000000000004},"min":{"1min":0.28799999999999998,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":46.688000000000002,"5min":218.78200000000001,"15min":218.78200000000001},"last":0.30199999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":2.5859999999999999,"15min":2.8660000000000001},"min":{"1min":0.33300000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.4500000000000002,"5min":108.01900000000001,"15min":138.81299999999999},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.25,"5min":2.7490000000000001,"15min":3.4550000000000001},"min":{"1min":0.35499999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.3670000000000009,"5min":107.898,"15min":171.251},"last":0.33800000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":1.7310000000000001,"15min":2.8340000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32100000000000001},"max":{"1min":15.076000000000001,"5min":15.076000000000001,"15min":171.30699999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.5649999999999999,"5min":1.4790000000000001,"15min":2.0880000000000001},"min":{"1min":0.35499999999999998,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":6.5279999999999996,"5min":9.9000000000000004,"15min":139.40799999999999},"last":0.47199999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":2.6360000000000001,"15min":3.161},"min":{"1min":0.38,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":9.968,"5min":108.67,"15min":171.33600000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.915,"5min":2.6699999999999999,"15min":3.4209999999999998},"min":{"1min":0.38900000000000001,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":9.157,"5min":108.804,"15min":184.13499999999999},"last":0.60199999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:34:55 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":2.8660000000000001,"15min":2.6640000000000001},"min":{"1min":0.34300000000000003,"5min":0.34000000000000002,"15min":0.32900000000000001},"max":{"1min":4.923,"5min":111.71299999999999,"15min":138.77699999999999},"last":0.496},{"interface":"front","average":{"1min":2.4950000000000001,"5min":2.9689999999999999,"15min":2.5179999999999998},"min":{"1min":0.36499999999999999,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":9.8870000000000005,"5min":111.108,"15min":139.37899999999999},"last":0.57499999999999996}]}]},{"osd":2,"up_from":199,"seq":854698492136,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3737916,"kb_used_data":2903860,"kb_used_omap":7,"kb_used_meta":834040,"kb_avail":90007236,"statfs":{"total":95995035648,"available":92167409664,"internally_reserved":0,"allocated":2973552640,"data_stored":2970727921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":854057041},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":4.726,"5min":2.6949999999999998,"15min":3.9089999999999998},"min":{"1min":0.19500000000000001,"5min":0.193,"15min":0.16700000000000001},"max":{"1min":61.950000000000003,"5min":61.950000000000003,"15min":119.277},"last":0.22},{"interface":"front","average":{"1min":5.0199999999999996,"5min":2.504,"15min":3.726},"min":{"1min":0.27800000000000002,"5min":0.20399999999999999,"15min":0.18099999999999999},"max":{"1min":61.984999999999999,"5min":61.984999999999999,"15min":119.56100000000001},"last":0.26600000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":11.630000000000001,"5min":6.2729999999999997,"15min":4.5359999999999996},"min":{"1min":0.29299999999999998,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":106.765,"5min":106.765,"15min":157.98699999999999},"last":1.9830000000000001},{"interface":"front","average":{"1min":11.114000000000001,"5min":6.3659999999999997,"15min":4.6349999999999998},"min":{"1min":0.29599999999999999,"5min":0.222,"15min":0.222},"max":{"1min":106.732,"5min":106.732,"15min":158.03800000000001},"last":0.45000000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":11.803000000000001,"5min":6.0069999999999997,"15min":4.7309999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":106.70399999999999,"5min":106.70399999999999,"15min":119.366},"last":2.02},{"interface":"front","average":{"1min":11.427,"5min":5.891,"15min":4.3280000000000003},"min":{"1min":0.35499999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":106.66800000000001,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.47399999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":5.0979999999999999,"5min":2.4510000000000001,"15min":2.3170000000000002},"min":{"1min":0.41399999999999998,"5min":0.33700000000000002,"15min":0.25},"max":{"1min":62.081000000000003,"5min":62.081000000000003,"15min":119.88200000000001},"last":0.39600000000000002},{"interface":"front","average":{"1min":4.9169999999999998,"5min":2.5369999999999999,"15min":1.835},"min":{"1min":0.39000000000000001,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":62.072000000000003,"5min":62.072000000000003,"15min":62.072000000000003},"last":2.1419999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":5.3090000000000002,"5min":3.391,"15min":2.7400000000000002},"min":{"1min":0.38800000000000001,"5min":0.34300000000000003,"15min":0.26200000000000001},"max":{"1min":62.058,"5min":91.207999999999998,"15min":119.919},"last":0.33700000000000002},{"interface":"front","average":{"1min":4.8929999999999998,"5min":3.2469999999999999,"15min":2.6120000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":62.042999999999999,"5min":91.150999999999996,"15min":119.306},"last":2.1099999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":12.76,"5min":4.1360000000000001,"15min":3.0720000000000001},"min":{"1min":0.41599999999999998,"5min":0.36599999999999999,"15min":0.315},"max":{"1min":107.018,"5min":107.018,"15min":119.943},"last":0.36599999999999999},{"interface":"front","average":{"1min":5.7709999999999999,"5min":2.798,"15min":2.1819999999999999},"min":{"1min":0.48199999999999998,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":62.493000000000002,"5min":62.493000000000002,"15min":62.493000000000002},"last":2.073}]},{"osd":7,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":4.8719999999999999,"5min":2.8879999999999999,"15min":1.958},"min":{"1min":0.41999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":62.603000000000002,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.50700000000000001},{"interface":"front","average":{"1min":11.744999999999999,"5min":4.0620000000000003,"15min":2.4060000000000001},"min":{"1min":0.48799999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":106.809,"5min":106.809,"15min":106.809},"last":0.49099999999999999}]}]},{"osd":3,"up_from":148,"seq":635655160105,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2426492,"kb_used_data":1524852,"kb_used_omap":2,"kb_used_meta":901629,"kb_avail":91318660,"statfs":{"total":95995035648,"available":93510307840,"internally_reserved":0,"allocated":1561448448,"data_stored":1558685127,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":923268149},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.835,"15min":2.7349999999999999},"min":{"1min":0.215,"5min":0.19400000000000001,"15min":0.158},"max":{"1min":17.213000000000001,"5min":22.271999999999998,"15min":103.866},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.95499999999999996,"5min":2.9750000000000001,"15min":2.8250000000000002},"min":{"1min":0.20799999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":4.5469999999999997,"5min":24.683,"15min":101.884},"last":0.28799999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":4.3789999999999996,"15min":2.9239999999999999},"min":{"1min":0.23300000000000001,"5min":0.214,"15min":0.19600000000000001},"max":{"1min":3.516,"5min":85.242999999999995,"15min":85.242999999999995},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.871,"5min":3.9910000000000001,"15min":2.7250000000000001},"min":{"1min":0.25800000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.2949999999999999,"5min":85.302000000000007,"15min":85.302000000000007},"last":0.22800000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:34:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7230000000000001,"5min":5.5739999999999998,"15min":4.8579999999999997},"min":{"1min":0.25,"5min":0.25,"15min":0.19800000000000001},"max":{"1min":7.1699999999999999,"5min":73.742000000000004,"15min":101.279},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.99,"5min":5.468,"15min":4.6529999999999996},"min":{"1min":0.36699999999999999,"5min":0.25700000000000001,"15min":0.214},"max":{"1min":11.156000000000001,"5min":76.525999999999996,"15min":101.294},"last":0.30299999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.74,"5min":1.3859999999999999,"15min":1.3049999999999999},"min":{"1min":0.34999999999999998,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":10.151999999999999,"5min":10.151999999999999,"15min":10.151999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.464,"5min":2.3479999999999999,"15min":1.8240000000000001},"min":{"1min":0.35399999999999998,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":7.681,"5min":24.782,"15min":24.782},"last":0.63800000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":7.2480000000000002,"15min":3.4140000000000001},"min":{"1min":0.39500000000000002,"5min":0.308,"15min":0.28000000000000003},"max":{"1min":6.5010000000000003,"5min":214.43700000000001,"15min":214.43700000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":2.0779999999999998,"5min":7.0030000000000001,"15min":3.488},"min":{"1min":0.376,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":9.4160000000000004,"5min":214.46600000000001,"15min":214.46600000000001},"last":0.57699999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":3.1419999999999999,"15min":2.3050000000000002},"min":{"1min":0.39000000000000001,"5min":0.35899999999999999,"15min":0.31900000000000001},"max":{"1min":5.4169999999999998,"5min":58.932000000000002,"15min":58.932000000000002},"last":0.59499999999999997},{"interface":"front","average":{"1min":2.9900000000000002,"5min":2.8969999999999998,"15min":2.359},"min":{"1min":0.41799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":16.759,"5min":59.186,"15min":59.186},"last":0.625}]},{"osd":7,"last update":"Sat Sep 23 23:34:55 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":6.609,"15min":3.2200000000000002},"min":{"1min":0.42699999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":11.191000000000001,"5min":192.69,"15min":192.69},"last":0.46000000000000002},{"interface":"front","average":{"1min":2.5779999999999998,"5min":7.4489999999999998,"15min":3.7269999999999999},"min":{"1min":0.47599999999999998,"5min":0.39200000000000002,"15min":0.36699999999999999},"max":{"1min":9.7449999999999992,"5min":192.65600000000001,"15min":192.65600000000001},"last":0.61699999999999999}]}]},{"osd":4,"up_from":0,"seq":128849019424,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757787,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3372940,"kb_used_data":2651460,"kb_used_omap":0,"kb_used_meta":721472,"kb_avail":90372212,"statfs":{"total":95995035648,"available":92541145088,"internally_reserved":0,"allocated":2715095040,"data_stored":2713483904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":738787328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.3720000000000001,"15min":3.48},"min":{"1min":0.254,"5min":0.20899999999999999,"15min":0.19500000000000001},"max":{"1min":3.0089999999999999,"5min":21.109000000000002,"15min":113.869},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.6699999999999999,"15min":3.8650000000000002},"min":{"1min":0.25,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":5.1280000000000001,"5min":20.876999999999999,"15min":138.55199999999999},"last":0.34300000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.772,"15min":2.323},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.23100000000000001},"max":{"1min":3.629,"5min":29.007999999999999,"15min":69.816999999999993},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.9390000000000001,"15min":2.665},"min":{"1min":0.29999999999999999,"5min":0.249,"15min":0.249},"max":{"1min":4.4859999999999998,"5min":29.053999999999998,"15min":70.268000000000001},"last":0.29599999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:34:47 2023","interfaces":[{"interface":"back","average":{"1min":8.3900000000000006,"5min":5.3979999999999997,"15min":4.2009999999999996},"min":{"1min":0.32000000000000001,"5min":0.28299999999999997,"15min":0.26900000000000002},"max":{"1min":69.75,"5min":208.738,"15min":208.738},"last":0.50600000000000001},{"interface":"front","average":{"1min":8.1739999999999995,"5min":5.3280000000000003,"15min":4.0960000000000001},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":66.421000000000006,"5min":208.631,"15min":208.631},"last":0.54100000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:34:43 2023","interfaces":[{"interface":"back","average":{"1min":8.5440000000000005,"5min":3.1419999999999999,"15min":3.3100000000000001},"min":{"1min":0.35399999999999998,"5min":0.29299999999999998,"15min":0.27400000000000002},"max":{"1min":66.591999999999999,"5min":66.591999999999999,"15min":112.71899999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":9.8930000000000007,"5min":3.5640000000000001,"15min":3.5419999999999998},"min":{"1min":0.372,"5min":0.35399999999999998,"15min":0.318},"max":{"1min":66.692999999999998,"5min":66.692999999999998,"15min":112.70099999999999},"last":0.40799999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.151,"15min":0.92000000000000004},"min":{"1min":0.34200000000000003,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":3.5499999999999998,"5min":9.2690000000000001,"15min":19.962},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.093,"15min":0.872},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.22900000000000001},"max":{"1min":2.4990000000000001,"5min":10.273999999999999,"15min":20.829999999999998},"last":0.47299999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":7.2519999999999998,"5min":2.3799999999999999,"15min":1.635},"min":{"1min":0.32400000000000001,"5min":0.27600000000000002,"15min":0.26900000000000002},"max":{"1min":148.36600000000001,"5min":148.36600000000001,"15min":148.36600000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":7.7210000000000001,"5min":2.9420000000000002,"15min":1.869},"min":{"1min":0.34599999999999997,"5min":0.307,"15min":0.26000000000000001},"max":{"1min":148.42500000000001,"5min":148.42500000000001,"15min":148.42500000000001},"last":0.42899999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:34:47 2023","interfaces":[{"interface":"back","average":{"1min":2.5760000000000001,"5min":1.577,"15min":2.8639999999999999},"min":{"1min":0.34999999999999998,"5min":0.34599999999999997,"15min":0.317},"max":{"1min":15.497,"5min":16.408999999999999,"15min":179.28800000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.387,"5min":1.6890000000000001,"15min":2.956},"min":{"1min":0.36799999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":15.653,"5min":18.972000000000001,"15min":179.428},"last":0.443}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":366608384,"data_stored":366608384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1497108480,"data_stored":1497104408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2964979712,"data_stored":2964979712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1552879616,"data_stored":1552875544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2565799936,"data_stored":2565799936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2705195008,"data_stored":2705195008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1600323584,"data_stored":1600323584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1726156800,"data_stored":1726152728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:35:41.949 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:41.980 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:41.995 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:42.245 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:42 smithi138 ceph-mon[193606]: from='client.46778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:35:42.273 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:42.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:42.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:42.329 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:42 smithi110 ceph-mon[212939]: from='client.46778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:35:42.329 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:42 smithi110 ceph-mon[221102]: from='client.46778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:35:42.379 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:42.418 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:42.582 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:42.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:42.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:42.686 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:42.717 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:42.769 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:42.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:42.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:42.791 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:35:42.791 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:35:42.791 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:35:42.791 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:35:42.791 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:35:42.874 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:42.909 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:42.958 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:42.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:42.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:43.064 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:43.096 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:35:43.109 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:43.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:43 smithi110 ceph-mon[212939]: pgmap v1623: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 26716/215454 objects degraded (12.400%); 4390/215454 objects misplaced (2.038%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:35:43.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:43 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/26447933' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:35:43.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:43 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:35:43.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:35:43.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:43 smithi110 ceph-mon[221102]: pgmap v1623: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 26716/215454 objects degraded (12.400%); 4390/215454 objects misplaced (2.038%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:35:43.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:43 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/26447933' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:35:43.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:43 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:35:43.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:35:43.346 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:43.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:43.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:43 smithi138 ceph-mon[193606]: pgmap v1623: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 26716/215454 objects degraded (12.400%); 4390/215454 objects misplaced (2.038%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:35:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:43 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/26447933' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:35:43.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:43 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:35:43.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:35:43.450 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:43.482 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:43.709 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:35:43.800 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:43.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:43.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:43.905 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:43.935 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:43.984 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:43.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:43.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:44.090 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:44.091 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:44 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:35:44.091 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:44 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:35:44.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:44 smithi110 ceph-mon[212939]: osdmap e589: 8 total, 8 up, 7 in 2023-09-23T23:35:44.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:44 smithi110 ceph-mon[212939]: failed to encode map e589 with expected crc 2023-09-23T23:35:44.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:44 smithi110 ceph-mon[212939]: failed to encode map e589 with expected crc 2023-09-23T23:35:44.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:44 smithi110 ceph-mon[212939]: failed to encode map e589 with expected crc 2023-09-23T23:35:44.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:44 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 26716/215454 objects degraded (12.400%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:44.092 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:44 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:35:44.092 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:44 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:35:44.092 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:44 smithi110 ceph-mon[221102]: osdmap e589: 8 total, 8 up, 7 in 2023-09-23T23:35:44.092 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:44 smithi110 ceph-mon[221102]: failed to encode map e589 with expected crc 2023-09-23T23:35:44.092 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:44 smithi110 ceph-mon[221102]: failed to encode map e589 with expected crc 2023-09-23T23:35:44.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:44 smithi110 ceph-mon[221102]: failed to encode map e589 with expected crc 2023-09-23T23:35:44.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:44 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 26716/215454 objects degraded (12.400%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:44.126 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:44.135 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:35:44.181 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:35:44.185 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:44.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:44.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:44.206 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:35:44.231 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:35:44.260 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:35:44.289 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:44.293 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:35:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:44 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:35:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:44 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:35:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:44 smithi138 ceph-mon[193606]: osdmap e589: 8 total, 8 up, 7 in 2023-09-23T23:35:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:44 smithi138 ceph-mon[193606]: failed to encode map e589 with expected crc 2023-09-23T23:35:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:44 smithi138 ceph-mon[193606]: failed to encode map e589 with expected crc 2023-09-23T23:35:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:44 smithi138 ceph-mon[193606]: failed to encode map e589 with expected crc 2023-09-23T23:35:44.318 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:44 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 26716/215454 objects degraded (12.400%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:44.320 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:44.321 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:35:44.351 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:35:44.375 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:35:44.380 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:35:44.380 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:35:44.380 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:35:44.380 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:35:44.380 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:35:44.380 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:35:44.380 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:35:44.380 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:35:44.383 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:44.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:44.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:44.494 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:44.529 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:44.577 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:44.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:44.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:44.681 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:44.707 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:44.749 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:44.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:44.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:44.853 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:44.863 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:35:44.884 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:45.143 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:45.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:45.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:45.248 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:45.283 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:45.340 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:45.344 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:45 smithi138 ceph-mon[193606]: pgmap v1625: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 154 op/s; 26646/214665 objects degraded (12.413%); 4327/214665 objects misplaced (2.016%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:35:45.345 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:45 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2944443956' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:45.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:45.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:45.376 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:45.446 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:45.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:45 smithi110 ceph-mon[212939]: pgmap v1625: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 154 op/s; 26646/214665 objects degraded (12.413%); 4327/214665 objects misplaced (2.016%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:35:45.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:45 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2944443956' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:45.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:45 smithi110 ceph-mon[221102]: pgmap v1625: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 154 op/s; 26646/214665 objects degraded (12.413%); 4327/214665 objects misplaced (2.016%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:35:45.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:45 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2944443956' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:45.476 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:45.584 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:45.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:45.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:45.688 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:45.716 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:45.769 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:45.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:45.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:45.795 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:35:45.800 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:35:45.800 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:35:45.873 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:45.918 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:45.982 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:45.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:45.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:46.086 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:46.113 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:46.494 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:46.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:46.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:46.552 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:35:46.598 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:46.644 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:46.665 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:46.760 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:46.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:46.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:46.865 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:46.895 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:46.952 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:46.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:46.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:47.060 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:47.093 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:35:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:35:46] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2023-09-23T23:35:47.093 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:47 smithi110 ceph-mon[212939]: pgmap v1626: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 26560/211977 objects degraded (12.530%); 4327/211977 objects misplaced (2.041%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:35:47.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:47 smithi110 ceph-mon[221102]: pgmap v1626: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 26560/211977 objects degraded (12.530%); 4327/211977 objects misplaced (2.041%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:35:47.099 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:47.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:47 smithi138 ceph-mon[193606]: pgmap v1626: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 26560/211977 objects degraded (12.530%); 4327/211977 objects misplaced (2.041%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:35:47.217 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:35:47.223 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:47.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:47.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:47.333 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:47.409 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:47.447 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:35:47.458 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:35:47.488 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:47.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:47.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:47.592 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:47.620 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:48.304 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:48.305 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:48 smithi110 ceph-mon[212939]: from='client.56704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:35:48.305 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:48 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:35:48.306 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:48 smithi110 ceph-mon[221102]: from='client.56704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:35:48.306 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:48 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:35:48.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:48.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:48.408 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:48.438 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:48 smithi138 ceph-mon[193606]: from='client.56704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:35:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:48 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:35:48.501 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:48.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:48.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:48.509 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1626,"stamp":"2023-09-23T23:35:45.908787+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4629705973,"num_objects":70659,"num_object_clones":0,"num_object_copies":211977,"num_objects_missing_on_primary":3432,"num_objects_missing":3432,"num_objects_degraded":26560,"num_objects_misplaced":4327,"num_objects_unfound":0,"num_objects_dirty":70659,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":104998,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12708,"num_bytes_recovered":830140840,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95045,"ondisk_log_size":95045,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17526608,"kb_used_data":11911896,"kb_used_omap":29,"kb_used_meta":5614626,"kb_avail":638689456,"statfs":{"total":671965249536,"available":654018002944,"internally_reserved":0,"allocated":12197781504,"data_stored":12181758927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5749377747},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1,0,0,3,4,1,3,7],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-148176896,"num_objects":-2261,"num_object_clones":0,"num_object_copies":-6783,"num_objects_missing_on_primary":-127,"num_objects_missing":-127,"num_objects_degraded":-291,"num_objects_misplaced":-127,"num_objects_unfound":0,"num_objects_dirty":-2261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2261,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":420,"num_bytes_recovered":27525120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002904"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":785,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.067483+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:35:35.067483+0000","last_peered":"2023-09-23T23:35:35.067483+0000","last_clean":"2023-09-23T23:35:35.067483+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:35:35.067483+0000","last_undegraded":"2023-09-23T23:35:35.067483+0000","last_fullsized":"2023-09-23T23:35:35.067483+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":343,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.045937+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:35:35.045937+0000","last_peered":"2023-09-23T23:35:35.045937+0000","last_clean":"2023-09-23T23:35:35.045937+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:35:35.045937+0000","last_undegraded":"2023-09-23T23:35:35.045937+0000","last_fullsized":"2023-09-23T23:35:35.045937+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":569,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.085688+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:35:29.085688+0000","last_peered":"2023-09-23T23:35:29.085688+0000","last_clean":"2023-09-23T23:35:29.085688+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:35:29.085688+0000","last_undegraded":"2023-09-23T23:35:29.085688+0000","last_fullsized":"2023-09-23T23:35:29.085688+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":738,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.068707+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:35:35.068707+0000","last_peered":"2023-09-23T23:35:35.068707+0000","last_clean":"2023-09-23T23:35:35.068707+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:35:35.068707+0000","last_undegraded":"2023-09-23T23:35:35.068707+0000","last_fullsized":"2023-09-23T23:35:35.068707+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":306,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057408+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:35:29.057408+0000","last_peered":"2023-09-23T23:35:29.057408+0000","last_clean":"2023-09-23T23:35:29.057408+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:35:29.057408+0000","last_undegraded":"2023-09-23T23:35:29.057408+0000","last_fullsized":"2023-09-23T23:35:29.057408+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":548,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056872+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:35:29.056872+0000","last_peered":"2023-09-23T23:35:29.056872+0000","last_clean":"2023-09-23T23:35:29.056872+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:35:29.056872+0000","last_undegraded":"2023-09-23T23:35:29.056872+0000","last_fullsized":"2023-09-23T23:35:29.056872+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":750,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.057009+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:35:35.057009+0000","last_peered":"2023-09-23T23:35:35.057009+0000","last_clean":"2023-09-23T23:35:35.057009+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:35:35.057009+0000","last_undegraded":"2023-09-23T23:35:35.057009+0000","last_fullsized":"2023-09-23T23:35:35.057009+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":606,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056119+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:35:29.056119+0000","last_peered":"2023-09-23T23:35:29.056119+0000","last_clean":"2023-09-23T23:35:29.056119+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:35:29.056119+0000","last_undegraded":"2023-09-23T23:35:29.056119+0000","last_fullsized":"2023-09-23T23:35:29.056119+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":103,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.064716+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:35:35.064716+0000","last_peered":"2023-09-23T23:35:35.064716+0000","last_clean":"2023-09-23T23:35:35.064716+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:35:35.064716+0000","last_undegraded":"2023-09-23T23:35:35.064716+0000","last_fullsized":"2023-09-23T23:35:35.064716+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":909,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.057406+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:35:35.057406+0000","last_peered":"2023-09-23T23:35:35.057406+0000","last_clean":"2023-09-23T23:35:35.057406+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:35:35.057406+0000","last_undegraded":"2023-09-23T23:35:35.057406+0000","last_fullsized":"2023-09-23T23:35:35.057406+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":241,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.069088+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:35:35.069088+0000","last_peered":"2023-09-23T23:35:35.069088+0000","last_clean":"2023-09-23T23:35:35.069088+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:35:35.069088+0000","last_undegraded":"2023-09-23T23:35:35.069088+0000","last_fullsized":"2023-09-23T23:35:35.069088+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":588,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056066+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:35:29.056066+0000","last_peered":"2023-09-23T23:35:29.056066+0000","last_clean":"2023-09-23T23:35:29.056066+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:35:29.056066+0000","last_undegraded":"2023-09-23T23:35:29.056066+0000","last_fullsized":"2023-09-23T23:35:29.056066+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":882,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.046868+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:35:35.046868+0000","last_peered":"2023-09-23T23:35:35.046868+0000","last_clean":"2023-09-23T23:35:35.046868+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:35:35.046868+0000","last_undegraded":"2023-09-23T23:35:35.046868+0000","last_fullsized":"2023-09-23T23:35:35.046868+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":220,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.069184+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:35:35.069184+0000","last_peered":"2023-09-23T23:35:35.069184+0000","last_clean":"2023-09-23T23:35:35.069184+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:35:35.069184+0000","last_undegraded":"2023-09-23T23:35:35.069184+0000","last_fullsized":"2023-09-23T23:35:35.069184+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":549,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057216+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:35:29.057216+0000","last_peered":"2023-09-23T23:35:29.057216+0000","last_clean":"2023-09-23T23:35:29.057216+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:35:29.057216+0000","last_undegraded":"2023-09-23T23:35:29.057216+0000","last_fullsized":"2023-09-23T23:35:29.057216+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1043,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.069001+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:35:35.069001+0000","last_peered":"2023-09-23T23:35:35.069001+0000","last_clean":"2023-09-23T23:35:35.069001+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:35:35.069001+0000","last_undegraded":"2023-09-23T23:35:35.069001+0000","last_fullsized":"2023-09-23T23:35:35.069001+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":260,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.067752+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:35:29.067752+0000","last_peered":"2023-09-23T23:35:29.067752+0000","last_clean":"2023-09-23T23:35:29.067752+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:35:29.067752+0000","last_undegraded":"2023-09-23T23:35:29.067752+0000","last_fullsized":"2023-09-23T23:35:29.067752+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":122,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.060400+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:35:35.060400+0000","last_peered":"2023-09-23T23:35:35.060400+0000","last_clean":"2023-09-23T23:35:35.060400+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:35:35.060400+0000","last_undegraded":"2023-09-23T23:35:35.060400+0000","last_fullsized":"2023-09-23T23:35:35.060400+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":438,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057004+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:35:29.057004+0000","last_peered":"2023-09-23T23:35:29.057004+0000","last_clean":"2023-09-23T23:35:29.057004+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:35:29.057004+0000","last_undegraded":"2023-09-23T23:35:29.057004+0000","last_fullsized":"2023-09-23T23:35:29.057004+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":163,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.061574+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:35:35.061574+0000","last_peered":"2023-09-23T23:35:35.061574+0000","last_clean":"2023-09-23T23:35:35.061574+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:35:35.061574+0000","last_undegraded":"2023-09-23T23:35:35.061574+0000","last_fullsized":"2023-09-23T23:35:35.061574+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":530,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.060574+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:35:35.060574+0000","last_peered":"2023-09-23T23:35:35.060574+0000","last_clean":"2023-09-23T23:35:35.060574+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:35:35.060574+0000","last_undegraded":"2023-09-23T23:35:35.060574+0000","last_fullsized":"2023-09-23T23:35:35.060574+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":744,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.068191+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:35:35.068191+0000","last_peered":"2023-09-23T23:35:35.068191+0000","last_clean":"2023-09-23T23:35:35.068191+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:35:35.068191+0000","last_undegraded":"2023-09-23T23:35:35.068191+0000","last_fullsized":"2023-09-23T23:35:35.068191+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":135,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.044878+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:35:35.044878+0000","last_peered":"2023-09-23T23:35:35.044878+0000","last_clean":"2023-09-23T23:35:35.044878+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:35:35.044878+0000","last_undegraded":"2023-09-23T23:35:35.044878+0000","last_fullsized":"2023-09-23T23:35:35.044878+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":587,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056465+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:35:29.056465+0000","last_peered":"2023-09-23T23:35:29.056465+0000","last_clean":"2023-09-23T23:35:29.056465+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:35:29.056465+0000","last_undegraded":"2023-09-23T23:35:29.056465+0000","last_fullsized":"2023-09-23T23:35:29.056465+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":874,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.060115+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:35:35.060115+0000","last_peered":"2023-09-23T23:35:35.060115+0000","last_clean":"2023-09-23T23:35:35.060115+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:35:35.060115+0000","last_undegraded":"2023-09-23T23:35:35.060115+0000","last_fullsized":"2023-09-23T23:35:35.060115+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":182,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056220+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:35:29.056220+0000","last_peered":"2023-09-23T23:35:29.056220+0000","last_clean":"2023-09-23T23:35:29.056220+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:35:29.056220+0000","last_undegraded":"2023-09-23T23:35:29.056220+0000","last_fullsized":"2023-09-23T23:35:29.056220+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":309,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.085529+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:35:29.085529+0000","last_peered":"2023-09-23T23:35:29.085529+0000","last_clean":"2023-09-23T23:35:29.085529+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:35:29.085529+0000","last_undegraded":"2023-09-23T23:35:29.085529+0000","last_fullsized":"2023-09-23T23:35:29.085529+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":550,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.060891+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:35:29.060891+0000","last_peered":"2023-09-23T23:35:29.060891+0000","last_clean":"2023-09-23T23:35:29.060891+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:35:29.060891+0000","last_undegraded":"2023-09-23T23:35:29.060891+0000","last_fullsized":"2023-09-23T23:35:29.060891+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":457,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.060252+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:35:35.060252+0000","last_peered":"2023-09-23T23:35:35.060252+0000","last_clean":"2023-09-23T23:35:35.060252+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:35:35.060252+0000","last_undegraded":"2023-09-23T23:35:35.060252+0000","last_fullsized":"2023-09-23T23:35:35.060252+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":544,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.055724+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:35:35.055724+0000","last_peered":"2023-09-23T23:35:35.055724+0000","last_clean":"2023-09-23T23:35:35.055724+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:35:35.055724+0000","last_undegraded":"2023-09-23T23:35:35.055724+0000","last_fullsized":"2023-09-23T23:35:35.055724+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":952,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.068575+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:35:35.068575+0000","last_peered":"2023-09-23T23:35:35.068575+0000","last_clean":"2023-09-23T23:35:35.068575+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:35:35.068575+0000","last_undegraded":"2023-09-23T23:35:35.068575+0000","last_fullsized":"2023-09-23T23:35:35.068575+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":153,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.060312+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:35:35.060312+0000","last_peered":"2023-09-23T23:35:35.060312+0000","last_clean":"2023-09-23T23:35:35.060312+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:35:35.060312+0000","last_undegraded":"2023-09-23T23:35:35.060312+0000","last_fullsized":"2023-09-23T23:35:35.060312+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":228,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056772+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:35:29.056772+0000","last_peered":"2023-09-23T23:35:29.056772+0000","last_clean":"2023-09-23T23:35:29.056772+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:35:29.056772+0000","last_undegraded":"2023-09-23T23:35:29.056772+0000","last_fullsized":"2023-09-23T23:35:29.056772+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"589'6484","reported_seq":6556,"reported_epoch":589,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:43.784846+0000","last_change":"2023-09-23T23:33:52.884769+0000","last_active":"2023-09-23T23:35:43.784846+0000","last_peered":"2023-09-23T23:35:43.784846+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:35:43.784846+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"544'1100","ondisk_log_start":"544'1100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5384,"log_dups_size":0,"ondisk_log_size":5384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285999104,"num_objects":4364,"num_object_clones":0,"num_object_copies":13092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6484,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":3338},{"shards":"2,6","objects":-34}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":922,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.044465+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:35:35.044465+0000","last_peered":"2023-09-23T23:35:35.044465+0000","last_clean":"2023-09-23T23:35:35.044465+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:35:35.044465+0000","last_undegraded":"2023-09-23T23:35:35.044465+0000","last_fullsized":"2023-09-23T23:35:35.044465+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":230,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.057476+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:35:35.057476+0000","last_peered":"2023-09-23T23:35:35.057476+0000","last_clean":"2023-09-23T23:35:35.057476+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:35:35.057476+0000","last_undegraded":"2023-09-23T23:35:35.057476+0000","last_fullsized":"2023-09-23T23:35:35.057476+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":220,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.067401+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:35:35.067401+0000","last_peered":"2023-09-23T23:35:35.067401+0000","last_clean":"2023-09-23T23:35:35.067401+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:35:35.067401+0000","last_undegraded":"2023-09-23T23:35:35.067401+0000","last_fullsized":"2023-09-23T23:35:35.067401+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"589'6596","reported_seq":6667,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:44.667187+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:35:44.667187+0000","last_peered":"2023-09-23T23:35:44.667187+0000","last_clean":"2023-09-23T23:35:44.667187+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:35:44.667187+0000","last_undegraded":"2023-09-23T23:35:44.667187+0000","last_fullsized":"2023-09-23T23:35:44.667187+0000","mapping_epoch":566,"log_start":"538'500","ondisk_log_start":"538'500","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6096,"log_dups_size":0,"ondisk_log_size":6096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290717696,"num_objects":4436,"num_object_clones":0,"num_object_copies":13308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6596,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":999,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056184+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:35:29.056184+0000","last_peered":"2023-09-23T23:35:29.056184+0000","last_clean":"2023-09-23T23:35:29.056184+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:35:29.056184+0000","last_undegraded":"2023-09-23T23:35:29.056184+0000","last_fullsized":"2023-09-23T23:35:29.056184+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":220,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.057616+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:35:35.057616+0000","last_peered":"2023-09-23T23:35:35.057616+0000","last_clean":"2023-09-23T23:35:35.057616+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:35:35.057616+0000","last_undegraded":"2023-09-23T23:35:35.057616+0000","last_fullsized":"2023-09-23T23:35:35.057616+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":438,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056823+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:35:29.056823+0000","last_peered":"2023-09-23T23:35:29.056823+0000","last_clean":"2023-09-23T23:35:29.056823+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:35:29.056823+0000","last_undegraded":"2023-09-23T23:35:29.056823+0000","last_fullsized":"2023-09-23T23:35:29.056823+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"588'6619","reported_seq":9914,"reported_epoch":588,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:35:41.654829+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:35:41.654829+0000","last_peered":"2023-09-23T23:35:41.654829+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:35:41.654829+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:35:41.654829+0000","mapping_epoch":571,"log_start":"546'1300","ondisk_log_start":"546'1300","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5319,"log_dups_size":0,"ondisk_log_size":5319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290258944,"num_objects":4429,"num_object_clones":0,"num_object_copies":13287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3534,"num_objects_misplaced":895,"num_objects_unfound":0,"num_objects_dirty":4429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6619,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1597,"num_bytes_recovered":104660992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":3412},{"shards":"3,5,7","objects":1017}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":899,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.064796+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:35:35.064796+0000","last_peered":"2023-09-23T23:35:35.064796+0000","last_clean":"2023-09-23T23:35:35.064796+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:35:35.064796+0000","last_undegraded":"2023-09-23T23:35:35.064796+0000","last_fullsized":"2023-09-23T23:35:35.064796+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":327,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.068966+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:35:35.068966+0000","last_peered":"2023-09-23T23:35:35.068966+0000","last_clean":"2023-09-23T23:35:35.068966+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:35:35.068966+0000","last_undegraded":"2023-09-23T23:35:35.068966+0000","last_fullsized":"2023-09-23T23:35:35.068966+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":103,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.060494+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:35:35.060494+0000","last_peered":"2023-09-23T23:35:35.060494+0000","last_clean":"2023-09-23T23:35:35.060494+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:35:35.060494+0000","last_undegraded":"2023-09-23T23:35:35.060494+0000","last_fullsized":"2023-09-23T23:35:35.060494+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"588'6553","reported_seq":8610,"reported_epoch":588,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:42.072147+0000","last_change":"2023-09-23T23:34:35.750428+0000","last_active":"2023-09-23T23:35:42.072147+0000","last_peered":"2023-09-23T23:35:42.072147+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:35:42.072147+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"545'1200","ondisk_log_start":"545'1200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5353,"log_dups_size":0,"ondisk_log_size":5353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297336832,"num_objects":4537,"num_object_clones":0,"num_object_copies":13611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6553,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":991,"num_bytes_recovered":64946176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":3527},{"shards":"3,5","objects":993}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":494,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057061+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:35:29.057061+0000","last_peered":"2023-09-23T23:35:29.057061+0000","last_clean":"2023-09-23T23:35:29.057061+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:35:29.057061+0000","last_undegraded":"2023-09-23T23:35:29.057061+0000","last_fullsized":"2023-09-23T23:35:29.057061+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":428,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056345+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:35:29.056345+0000","last_peered":"2023-09-23T23:35:29.056345+0000","last_clean":"2023-09-23T23:35:29.056345+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:35:29.056345+0000","last_undegraded":"2023-09-23T23:35:29.056345+0000","last_fullsized":"2023-09-23T23:35:29.056345+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":585,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.069756+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:35:35.069756+0000","last_peered":"2023-09-23T23:35:35.069756+0000","last_clean":"2023-09-23T23:35:35.069756+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:35:35.069756+0000","last_undegraded":"2023-09-23T23:35:35.069756+0000","last_fullsized":"2023-09-23T23:35:35.069756+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"589'6687","reported_seq":8005,"reported_epoch":589,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:44.232343+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:35:44.232343+0000","last_peered":"2023-09-23T23:35:44.232343+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:35:44.232343+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"543'1000","ondisk_log_start":"543'1000","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5687,"log_dups_size":0,"ondisk_log_size":5687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293273600,"num_objects":4475,"num_object_clones":0,"num_object_copies":13425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6687,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":3429},{"shards":"3,5","objects":-985},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":715,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057140+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:35:29.057140+0000","last_peered":"2023-09-23T23:35:29.057140+0000","last_clean":"2023-09-23T23:35:29.057140+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:35:29.057140+0000","last_undegraded":"2023-09-23T23:35:29.057140+0000","last_fullsized":"2023-09-23T23:35:29.057140+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":598,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.058004+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:35:35.058004+0000","last_peered":"2023-09-23T23:35:35.058004+0000","last_clean":"2023-09-23T23:35:35.058004+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:35:35.058004+0000","last_undegraded":"2023-09-23T23:35:35.058004+0000","last_fullsized":"2023-09-23T23:35:35.058004+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":508,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056265+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:35:29.056265+0000","last_peered":"2023-09-23T23:35:29.056265+0000","last_clean":"2023-09-23T23:35:29.056265+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:35:29.056265+0000","last_undegraded":"2023-09-23T23:35:29.056265+0000","last_fullsized":"2023-09-23T23:35:29.056265+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"589'6618","reported_seq":6681,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.967992+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:35:43.967992+0000","last_peered":"2023-09-23T23:35:43.967992+0000","last_clean":"2023-09-23T23:35:43.967992+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:35:43.967992+0000","last_undegraded":"2023-09-23T23:35:43.967992+0000","last_fullsized":"2023-09-23T23:35:43.967992+0000","mapping_epoch":534,"log_start":"544'1200","ondisk_log_start":"544'1200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5418,"log_dups_size":0,"ondisk_log_size":5418,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292421632,"num_objects":4462,"num_object_clones":0,"num_object_copies":13386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6618,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":872,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.046692+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:35:35.046692+0000","last_peered":"2023-09-23T23:35:35.046692+0000","last_clean":"2023-09-23T23:35:35.046692+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:35:35.046692+0000","last_undegraded":"2023-09-23T23:35:35.046692+0000","last_fullsized":"2023-09-23T23:35:35.046692+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":113,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.060475+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:35:35.060475+0000","last_peered":"2023-09-23T23:35:35.060475+0000","last_clean":"2023-09-23T23:35:35.060475+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:35:35.060475+0000","last_undegraded":"2023-09-23T23:35:35.060475+0000","last_fullsized":"2023-09-23T23:35:35.060475+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":454,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.044664+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:35:35.044664+0000","last_peered":"2023-09-23T23:35:35.044664+0000","last_clean":"2023-09-23T23:35:35.044664+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:35:35.044664+0000","last_undegraded":"2023-09-23T23:35:35.044664+0000","last_fullsized":"2023-09-23T23:35:35.044664+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"589'6688","reported_seq":9045,"reported_epoch":589,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:35:44.142829+0000","last_change":"2023-09-23T23:34:35.785270+0000","last_active":"2023-09-23T23:35:44.142829+0000","last_peered":"2023-09-23T23:35:44.142829+0000","last_clean":"2023-09-23T23:33:50.504296+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:35:44.142829+0000","last_undegraded":"2023-09-23T23:34:35.780981+0000","last_fullsized":"2023-09-23T23:35:44.142829+0000","mapping_epoch":567,"log_start":"545'1300","ondisk_log_start":"545'1300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5388,"log_dups_size":0,"ondisk_log_size":5388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289406976,"num_objects":4416,"num_object_clones":0,"num_object_copies":13248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3410,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6688,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1137,"num_bytes_recovered":74514432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":3410},{"shards":"5,6,7","objects":1006}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":597,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.068658+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:35:35.068658+0000","last_peered":"2023-09-23T23:35:35.068658+0000","last_clean":"2023-09-23T23:35:35.068658+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:35:35.068658+0000","last_undegraded":"2023-09-23T23:35:35.068658+0000","last_fullsized":"2023-09-23T23:35:35.068658+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":451,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.044308+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:35:35.044308+0000","last_peered":"2023-09-23T23:35:35.044308+0000","last_clean":"2023-09-23T23:35:35.044308+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:35:35.044308+0000","last_undegraded":"2023-09-23T23:35:35.044308+0000","last_fullsized":"2023-09-23T23:35:35.044308+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":540,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.061274+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:35:35.061274+0000","last_peered":"2023-09-23T23:35:35.061274+0000","last_clean":"2023-09-23T23:35:35.061274+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:35:35.061274+0000","last_undegraded":"2023-09-23T23:35:35.061274+0000","last_fullsized":"2023-09-23T23:35:35.061274+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"589'6650","reported_seq":15676,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:44.294804+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:35:44.294804+0000","last_peered":"2023-09-23T23:35:44.294804+0000","last_clean":"2023-09-23T23:35:44.294804+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:35:44.294804+0000","last_undegraded":"2023-09-23T23:35:44.294804+0000","last_fullsized":"2023-09-23T23:35:44.294804+0000","mapping_epoch":579,"log_start":"543'900","ondisk_log_start":"543'900","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5750,"log_dups_size":0,"ondisk_log_size":5750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286130176,"num_objects":4366,"num_object_clones":0,"num_object_copies":13098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6650,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":930,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.045814+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:35:35.045814+0000","last_peered":"2023-09-23T23:35:35.045814+0000","last_clean":"2023-09-23T23:35:35.045814+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:35:35.045814+0000","last_undegraded":"2023-09-23T23:35:35.045814+0000","last_fullsized":"2023-09-23T23:35:35.045814+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":229,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056015+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:35:29.056015+0000","last_peered":"2023-09-23T23:35:29.056015+0000","last_clean":"2023-09-23T23:35:29.056015+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:35:29.056015+0000","last_undegraded":"2023-09-23T23:35:29.056015+0000","last_fullsized":"2023-09-23T23:35:29.056015+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":517,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056942+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:35:29.056942+0000","last_peered":"2023-09-23T23:35:29.056942+0000","last_clean":"2023-09-23T23:35:29.056942+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:35:29.056942+0000","last_undegraded":"2023-09-23T23:35:29.056942+0000","last_fullsized":"2023-09-23T23:35:29.056942+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"588'6343","reported_seq":6407,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:41.903801+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:35:41.903801+0000","last_peered":"2023-09-23T23:35:41.903801+0000","last_clean":"2023-09-23T23:35:41.903801+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:35:41.903801+0000","last_undegraded":"2023-09-23T23:35:41.903801+0000","last_fullsized":"2023-09-23T23:35:41.903801+0000","mapping_epoch":534,"log_start":"544'1000","ondisk_log_start":"544'1000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5343,"log_dups_size":0,"ondisk_log_size":5343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287375360,"num_objects":4385,"num_object_clones":0,"num_object_copies":13155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6343,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":910,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.061604+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:35:35.061604+0000","last_peered":"2023-09-23T23:35:35.061604+0000","last_clean":"2023-09-23T23:35:35.061604+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:35:35.061604+0000","last_undegraded":"2023-09-23T23:35:35.061604+0000","last_fullsized":"2023-09-23T23:35:35.061604+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":298,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.051029+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:35:35.051029+0000","last_peered":"2023-09-23T23:35:35.051029+0000","last_clean":"2023-09-23T23:35:35.051029+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:35:35.051029+0000","last_undegraded":"2023-09-23T23:35:35.051029+0000","last_fullsized":"2023-09-23T23:35:35.051029+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":604,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.069051+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:35:35.069051+0000","last_peered":"2023-09-23T23:35:35.069051+0000","last_clean":"2023-09-23T23:35:35.069051+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:35:35.069051+0000","last_undegraded":"2023-09-23T23:35:35.069051+0000","last_fullsized":"2023-09-23T23:35:35.069051+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":185,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.068440+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:35:35.068440+0000","last_peered":"2023-09-23T23:35:35.068440+0000","last_clean":"2023-09-23T23:35:35.068440+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:35:35.068440+0000","last_undegraded":"2023-09-23T23:35:35.068440+0000","last_fullsized":"2023-09-23T23:35:35.068440+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"588'6655","reported_seq":8893,"reported_epoch":588,"state":"active+recovering","last_fresh":"2023-09-23T23:35:42.055410+0000","last_change":"2023-09-23T23:34:35.815566+0000","last_active":"2023-09-23T23:35:42.055410+0000","last_peered":"2023-09-23T23:35:42.055410+0000","last_clean":"2023-09-23T23:33:50.519308+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:35:42.055410+0000","last_undegraded":"2023-09-23T23:35:42.055410+0000","last_fullsized":"2023-09-23T23:35:42.055410+0000","mapping_epoch":566,"log_start":"546'1322","ondisk_log_start":"546'1322","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5333,"log_dups_size":0,"ondisk_log_size":5333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295698456,"num_objects":4513,"num_object_clones":0,"num_object_copies":13539,"num_objects_missing_on_primary":3432,"num_objects_missing":3432,"num_objects_degraded":0,"num_objects_misplaced":3432,"num_objects_unfound":0,"num_objects_dirty":4513,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6655,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1071,"num_bytes_recovered":70189056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,3,4,7","objects":12},{"shards":"1,3,7","objects":1069},{"shards":"1,4,7","objects":3432}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":671,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.052247+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:35:29.052247+0000","last_peered":"2023-09-23T23:35:29.052247+0000","last_clean":"2023-09-23T23:35:29.052247+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:35:29.052247+0000","last_undegraded":"2023-09-23T23:35:29.052247+0000","last_fullsized":"2023-09-23T23:35:29.052247+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":237,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.060421+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:35:35.060421+0000","last_peered":"2023-09-23T23:35:35.060421+0000","last_clean":"2023-09-23T23:35:35.060421+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:35:35.060421+0000","last_undegraded":"2023-09-23T23:35:35.060421+0000","last_fullsized":"2023-09-23T23:35:35.060421+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":521,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.060652+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:35:35.060652+0000","last_peered":"2023-09-23T23:35:35.060652+0000","last_clean":"2023-09-23T23:35:35.060652+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:35:35.060652+0000","last_undegraded":"2023-09-23T23:35:35.060652+0000","last_fullsized":"2023-09-23T23:35:35.060652+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":221,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.046945+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:35:35.046945+0000","last_peered":"2023-09-23T23:35:35.046945+0000","last_clean":"2023-09-23T23:35:35.046945+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:35:35.046945+0000","last_undegraded":"2023-09-23T23:35:35.046945+0000","last_fullsized":"2023-09-23T23:35:35.046945+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"589'6377","reported_seq":8419,"reported_epoch":589,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:43.880636+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:35:43.880636+0000","last_peered":"2023-09-23T23:35:43.880636+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:35:43.880636+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"542'700","ondisk_log_start":"542'700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5677,"log_dups_size":0,"ondisk_log_size":5677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288030720,"num_objects":4395,"num_object_clones":0,"num_object_copies":13185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6377,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":3384},{"shards":"5,7","objects":1003}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":890,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.045741+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:35:35.045741+0000","last_peered":"2023-09-23T23:35:35.045741+0000","last_clean":"2023-09-23T23:35:35.045741+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:35:35.045741+0000","last_undegraded":"2023-09-23T23:35:35.045741+0000","last_fullsized":"2023-09-23T23:35:35.045741+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":327,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.044210+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:35:35.044210+0000","last_peered":"2023-09-23T23:35:35.044210+0000","last_clean":"2023-09-23T23:35:35.044210+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:35:35.044210+0000","last_undegraded":"2023-09-23T23:35:35.044210+0000","last_fullsized":"2023-09-23T23:35:35.044210+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":236,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.046771+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:35:35.046771+0000","last_peered":"2023-09-23T23:35:35.046771+0000","last_clean":"2023-09-23T23:35:35.046771+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:35:35.046771+0000","last_undegraded":"2023-09-23T23:35:35.046771+0000","last_fullsized":"2023-09-23T23:35:35.046771+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":200,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.051689+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:35:35.051689+0000","last_peered":"2023-09-23T23:35:35.051689+0000","last_clean":"2023-09-23T23:35:35.051689+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:35:35.051689+0000","last_undegraded":"2023-09-23T23:35:35.051689+0000","last_fullsized":"2023-09-23T23:35:35.051689+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"589'6536","reported_seq":6609,"reported_epoch":589,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:35:44.269611+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:35:44.269611+0000","last_peered":"2023-09-23T23:35:44.269611+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:35:44.269611+0000","last_undegraded":"2023-09-23T23:35:44.269611+0000","last_fullsized":"2023-09-23T23:35:44.269611+0000","mapping_epoch":562,"log_start":"543'800","ondisk_log_start":"543'800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5736,"log_dups_size":0,"ondisk_log_size":5736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286130176,"num_objects":4366,"num_object_clones":0,"num_object_copies":13098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6536,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":4366}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":603,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.044920+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:35:35.044920+0000","last_peered":"2023-09-23T23:35:35.044920+0000","last_clean":"2023-09-23T23:35:35.044920+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:35:35.044920+0000","last_undegraded":"2023-09-23T23:35:35.044920+0000","last_fullsized":"2023-09-23T23:35:35.044920+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":875,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.060158+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:35:35.060158+0000","last_peered":"2023-09-23T23:35:35.060158+0000","last_clean":"2023-09-23T23:35:35.060158+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:35:35.060158+0000","last_undegraded":"2023-09-23T23:35:35.060158+0000","last_fullsized":"2023-09-23T23:35:35.060158+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":549,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056088+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:35:29.056088+0000","last_peered":"2023-09-23T23:35:29.056088+0000","last_clean":"2023-09-23T23:35:29.056088+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:35:29.056088+0000","last_undegraded":"2023-09-23T23:35:29.056088+0000","last_fullsized":"2023-09-23T23:35:29.056088+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":113,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.061180+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:35:35.061180+0000","last_peered":"2023-09-23T23:35:35.061180+0000","last_clean":"2023-09-23T23:35:35.061180+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:35:35.061180+0000","last_undegraded":"2023-09-23T23:35:35.061180+0000","last_fullsized":"2023-09-23T23:35:35.061180+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"589'6430","reported_seq":6493,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.810673+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:35:43.810673+0000","last_peered":"2023-09-23T23:35:43.810673+0000","last_clean":"2023-09-23T23:35:43.810673+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:35:43.810673+0000","last_undegraded":"2023-09-23T23:35:43.810673+0000","last_fullsized":"2023-09-23T23:35:43.810673+0000","mapping_epoch":534,"log_start":"544'1000","ondisk_log_start":"544'1000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5430,"log_dups_size":0,"ondisk_log_size":5430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282198016,"num_objects":4306,"num_object_clones":0,"num_object_copies":12918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6430,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":701,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.060493+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:35:35.060493+0000","last_peered":"2023-09-23T23:35:35.060493+0000","last_clean":"2023-09-23T23:35:35.060493+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:35:35.060493+0000","last_undegraded":"2023-09-23T23:35:35.060493+0000","last_fullsized":"2023-09-23T23:35:35.060493+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":516,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.056755+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:35:35.056755+0000","last_peered":"2023-09-23T23:35:35.056755+0000","last_clean":"2023-09-23T23:35:35.056755+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:35:35.056755+0000","last_undegraded":"2023-09-23T23:35:35.056755+0000","last_fullsized":"2023-09-23T23:35:35.056755+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":428,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056912+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:35:29.056912+0000","last_peered":"2023-09-23T23:35:29.056912+0000","last_clean":"2023-09-23T23:35:29.056912+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:35:29.056912+0000","last_undegraded":"2023-09-23T23:35:29.056912+0000","last_fullsized":"2023-09-23T23:35:29.056912+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":28,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.052862+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:35:29.052862+0000","last_peered":"2023-09-23T23:35:29.052862+0000","last_clean":"2023-09-23T23:35:29.052862+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:35:29.052862+0000","last_undegraded":"2023-09-23T23:35:29.052862+0000","last_fullsized":"2023-09-23T23:35:29.052862+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"589'6570","reported_seq":10232,"reported_epoch":589,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:44.645720+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:35:44.645720+0000","last_peered":"2023-09-23T23:35:44.645720+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:35:44.645720+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"538'400","ondisk_log_start":"538'400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6170,"log_dups_size":0,"ondisk_log_size":6170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290193408,"num_objects":4428,"num_object_clones":0,"num_object_copies":13284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6570,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":1724},{"shards":"1,2,4","objects":2704}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":584,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.061506+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:35:35.061506+0000","last_peered":"2023-09-23T23:35:35.061506+0000","last_clean":"2023-09-23T23:35:35.061506+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:35:35.061506+0000","last_undegraded":"2023-09-23T23:35:35.061506+0000","last_fullsized":"2023-09-23T23:35:35.061506+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.052522+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:35:29.052522+0000","last_peered":"2023-09-23T23:35:29.052522+0000","last_clean":"2023-09-23T23:35:29.052522+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:35:29.052522+0000","last_undegraded":"2023-09-23T23:35:29.052522+0000","last_fullsized":"2023-09-23T23:35:29.052522+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":128,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.047021+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:35:35.047021+0000","last_peered":"2023-09-23T23:35:35.047021+0000","last_clean":"2023-09-23T23:35:35.047021+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:35:35.047021+0000","last_undegraded":"2023-09-23T23:35:35.047021+0000","last_fullsized":"2023-09-23T23:35:35.047021+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":322,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.068778+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:35:35.068778+0000","last_peered":"2023-09-23T23:35:35.068778+0000","last_clean":"2023-09-23T23:35:35.068778+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:35:35.068778+0000","last_undegraded":"2023-09-23T23:35:35.068778+0000","last_fullsized":"2023-09-23T23:35:35.068778+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"589'6600","reported_seq":6663,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:44.242630+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:35:44.242630+0000","last_peered":"2023-09-23T23:35:44.242630+0000","last_clean":"2023-09-23T23:35:44.242630+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:35:44.242630+0000","last_undegraded":"2023-09-23T23:35:44.242630+0000","last_fullsized":"2023-09-23T23:35:44.242630+0000","mapping_epoch":534,"log_start":"543'900","ondisk_log_start":"543'900","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5700,"log_dups_size":0,"ondisk_log_size":5700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285999104,"num_objects":4364,"num_object_clones":0,"num_object_copies":13092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6600,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":769,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.044740+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:35:35.044740+0000","last_peered":"2023-09-23T23:35:35.044740+0000","last_clean":"2023-09-23T23:35:35.044740+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:35:35.044740+0000","last_undegraded":"2023-09-23T23:35:35.044740+0000","last_fullsized":"2023-09-23T23:35:35.044740+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":585,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056943+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:35:29.056943+0000","last_peered":"2023-09-23T23:35:29.056943+0000","last_clean":"2023-09-23T23:35:29.056943+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:35:29.056943+0000","last_undegraded":"2023-09-23T23:35:29.056943+0000","last_fullsized":"2023-09-23T23:35:29.056943+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":193,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.061380+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:35:35.061380+0000","last_peered":"2023-09-23T23:35:35.061380+0000","last_clean":"2023-09-23T23:35:35.061380+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:35:35.061380+0000","last_undegraded":"2023-09-23T23:35:35.061380+0000","last_fullsized":"2023-09-23T23:35:35.061380+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":140,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.046271+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:35:35.046271+0000","last_peered":"2023-09-23T23:35:35.046271+0000","last_clean":"2023-09-23T23:35:35.046271+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:35:35.046271+0000","last_undegraded":"2023-09-23T23:35:35.046271+0000","last_fullsized":"2023-09-23T23:35:35.046271+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"589'6515","reported_seq":6580,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:44.425370+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:35:44.425370+0000","last_peered":"2023-09-23T23:35:44.425370+0000","last_clean":"2023-09-23T23:35:44.425370+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:35:44.425370+0000","last_undegraded":"2023-09-23T23:35:44.425370+0000","last_fullsized":"2023-09-23T23:35:44.425370+0000","mapping_epoch":534,"log_start":"538'300","ondisk_log_start":"538'300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6215,"log_dups_size":0,"ondisk_log_size":6215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287899648,"num_objects":4393,"num_object_clones":0,"num_object_copies":13179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6515,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":975,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.056741+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:35:35.056741+0000","last_peered":"2023-09-23T23:35:35.056741+0000","last_clean":"2023-09-23T23:35:35.056741+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:35:35.056741+0000","last_undegraded":"2023-09-23T23:35:35.056741+0000","last_fullsized":"2023-09-23T23:35:35.056741+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":103,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.061735+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:35:35.061735+0000","last_peered":"2023-09-23T23:35:35.061735+0000","last_clean":"2023-09-23T23:35:35.061735+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:35:35.061735+0000","last_undegraded":"2023-09-23T23:35:35.061735+0000","last_fullsized":"2023-09-23T23:35:35.061735+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":32,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.052950+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:35:29.052950+0000","last_peered":"2023-09-23T23:35:29.052950+0000","last_clean":"2023-09-23T23:35:29.052950+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:35:29.052950+0000","last_undegraded":"2023-09-23T23:35:29.052950+0000","last_fullsized":"2023-09-23T23:35:29.052950+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":163,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.068268+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:35:35.068268+0000","last_peered":"2023-09-23T23:35:35.068268+0000","last_clean":"2023-09-23T23:35:35.068268+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:35:35.068268+0000","last_undegraded":"2023-09-23T23:35:35.068268+0000","last_fullsized":"2023-09-23T23:35:35.068268+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1024,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057056+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:35:29.057056+0000","last_peered":"2023-09-23T23:35:29.057056+0000","last_clean":"2023-09-23T23:35:29.057056+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:35:29.057056+0000","last_undegraded":"2023-09-23T23:35:29.057056+0000","last_fullsized":"2023-09-23T23:35:29.057056+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":28,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.052843+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:35:29.052843+0000","last_peered":"2023-09-23T23:35:29.052843+0000","last_clean":"2023-09-23T23:35:29.052843+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:35:29.052843+0000","last_undegraded":"2023-09-23T23:35:29.052843+0000","last_fullsized":"2023-09-23T23:35:29.052843+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":344,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.047094+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:35:35.047094+0000","last_peered":"2023-09-23T23:35:35.047094+0000","last_clean":"2023-09-23T23:35:35.047094+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:35:35.047094+0000","last_undegraded":"2023-09-23T23:35:35.047094+0000","last_fullsized":"2023-09-23T23:35:35.047094+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":735,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057194+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:35:29.057194+0000","last_peered":"2023-09-23T23:35:29.057194+0000","last_clean":"2023-09-23T23:35:29.057194+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:35:29.057194+0000","last_undegraded":"2023-09-23T23:35:29.057194+0000","last_fullsized":"2023-09-23T23:35:29.057194+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":91,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.061392+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:35:35.061392+0000","last_peered":"2023-09-23T23:35:35.061392+0000","last_clean":"2023-09-23T23:35:35.061392+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:35:35.061392+0000","last_undegraded":"2023-09-23T23:35:35.061392+0000","last_fullsized":"2023-09-23T23:35:35.061392+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":435,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.060786+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:35:29.060786+0000","last_peered":"2023-09-23T23:35:29.060786+0000","last_clean":"2023-09-23T23:35:29.060786+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:35:29.060786+0000","last_undegraded":"2023-09-23T23:35:29.060786+0000","last_fullsized":"2023-09-23T23:35:29.060786+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":287,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.061035+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:35:35.061035+0000","last_peered":"2023-09-23T23:35:35.061035+0000","last_clean":"2023-09-23T23:35:35.061035+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:35:35.061035+0000","last_undegraded":"2023-09-23T23:35:35.061035+0000","last_fullsized":"2023-09-23T23:35:35.061035+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":274,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.056159+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:35:29.056159+0000","last_peered":"2023-09-23T23:35:29.056159+0000","last_clean":"2023-09-23T23:35:29.056159+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:35:29.056159+0000","last_undegraded":"2023-09-23T23:35:29.056159+0000","last_fullsized":"2023-09-23T23:35:29.056159+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":801,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.061654+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:35:35.061654+0000","last_peered":"2023-09-23T23:35:35.061654+0000","last_clean":"2023-09-23T23:35:35.061654+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:35:35.061654+0000","last_undegraded":"2023-09-23T23:35:35.061654+0000","last_fullsized":"2023-09-23T23:35:35.061654+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":618,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057149+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:35:29.057149+0000","last_peered":"2023-09-23T23:35:29.057149+0000","last_clean":"2023-09-23T23:35:29.057149+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:35:29.057149+0000","last_undegraded":"2023-09-23T23:35:29.057149+0000","last_fullsized":"2023-09-23T23:35:29.057149+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":602,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.057333+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:35:29.057333+0000","last_peered":"2023-09-23T23:35:29.057333+0000","last_clean":"2023-09-23T23:35:29.057333+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:35:29.057333+0000","last_undegraded":"2023-09-23T23:35:29.057333+0000","last_fullsized":"2023-09-23T23:35:29.057333+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":760,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-09-23T23:35:29.067681+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:35:29.067681+0000","last_peered":"2023-09-23T23:35:29.067681+0000","last_clean":"2023-09-23T23:35:29.067681+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:35:29.067681+0000","last_undegraded":"2023-09-23T23:35:29.067681+0000","last_fullsized":"2023-09-23T23:35:29.067681+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":432,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.046852+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:35:35.046852+0000","last_peered":"2023-09-23T23:35:35.046852+0000","last_clean":"2023-09-23T23:35:35.046852+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:35:35.046852+0000","last_undegraded":"2023-09-23T23:35:35.046852+0000","last_fullsized":"2023-09-23T23:35:35.046852+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":656,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-09-23T23:35:35.060063+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:35:35.060063+0000","last_peered":"2023-09-23T23:35:35.060063+0000","last_clean":"2023-09-23T23:35:35.060063+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:35:35.060063+0000","last_undegraded":"2023-09-23T23:35:35.060063+0000","last_fullsized":"2023-09-23T23:35:35.060063+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":4629069848,"num_objects":70635,"num_object_clones":0,"num_object_copies":211905,"num_objects_missing_on_primary":3432,"num_objects_missing":3432,"num_objects_degraded":26560,"num_objects_misplaced":4327,"num_objects_unfound":0,"num_objects_dirty":70635,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":104921,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12653,"num_bytes_recovered":829030400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14693117952,"data_stored":14693105736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89999,"ondisk_log_size":89999,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782675,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2034912,"kb_used_data":1665368,"kb_used_omap":8,"kb_used_meta":369527,"kb_avail":91710240,"statfs":{"total":95995035648,"available":93911285760,"internally_reserved":0,"allocated":1705336832,"data_stored":1703709486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":378395717},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.9339999999999999,"5min":2.3500000000000001,"15min":2.2549999999999999},"min":{"1min":0.26600000000000001,"5min":0.224,"15min":0.20899999999999999},"max":{"1min":11.173,"5min":44.094999999999999,"15min":85.721000000000004},"last":0.28699999999999998},{"interface":"front","average":{"1min":1.1519999999999999,"5min":2.2080000000000002,"15min":2.2669999999999999},"min":{"1min":0.28199999999999997,"5min":0.25600000000000001,"15min":0.19800000000000001},"max":{"1min":4.3109999999999999,"5min":44.335999999999999,"15min":85.965000000000003},"last":0.34000000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8819999999999999,"5min":6.5810000000000004,"15min":3.202},"min":{"1min":0.252,"5min":0.248,"15min":0.22900000000000001},"max":{"1min":9.0139999999999993,"5min":198.58000000000001,"15min":198.58000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.179,"5min":6.3109999999999999,"15min":3.2610000000000001},"min":{"1min":0.254,"5min":0.247,"15min":0.246},"max":{"1min":4.1660000000000004,"5min":198.71100000000001,"15min":198.71100000000001},"last":0.40500000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":6.0019999999999998,"5min":5.7329999999999997,"15min":3.7810000000000001},"min":{"1min":0.26900000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":78.528999999999996,"5min":203.58000000000001,"15min":203.58000000000001},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.8799999999999999,"5min":4.5350000000000001,"15min":3.1080000000000001},"min":{"1min":0.27900000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.5670000000000002,"5min":198.69,"15min":198.69},"last":0.30399999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":7.0250000000000004,"15min":3.6579999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":6.1210000000000004,"5min":279.26499999999999,"15min":279.26499999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":3.1579999999999999,"5min":8.7859999999999996,"15min":4.1630000000000003},"min":{"1min":0.40200000000000002,"5min":0.32000000000000001,"15min":0.29599999999999999},"max":{"1min":18.643999999999998,"5min":279.29899999999998,"15min":279.29899999999998},"last":0.38400000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":1.857,"15min":1.0940000000000001},"min":{"1min":0.35299999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":10.6,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.443},{"interface":"front","average":{"1min":0.79600000000000004,"5min":1.4970000000000001,"15min":1.03},"min":{"1min":0.317,"5min":0.28100000000000003,"15min":0.23899999999999999},"max":{"1min":2.8570000000000002,"5min":31.68,"15min":31.68},"last":0.45500000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":1.1659999999999999,"15min":1.7290000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26500000000000001},"max":{"1min":12.273999999999999,"5min":12.273999999999999,"15min":210.83600000000001},"last":0.495},{"interface":"front","average":{"1min":2.141,"5min":1.0529999999999999,"15min":1.579},"min":{"1min":0.371,"5min":0.316,"15min":0.28199999999999997},"max":{"1min":18.117999999999999,"5min":18.117999999999999,"15min":207.04599999999999},"last":0.35999999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.96,"15min":1.3620000000000001},"min":{"1min":0.379,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.6440000000000001,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.46800000000000003},{"interface":"front","average":{"1min":2.2109999999999999,"5min":2.403,"15min":1.615},"min":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.33300000000000002},"max":{"1min":10.332000000000001,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.48199999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496152,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2246620,"kb_used_data":1544340,"kb_used_omap":0,"kb_used_meta":702272,"kb_avail":91498532,"statfs":{"total":95995035648,"available":93694496768,"internally_reserved":0,"allocated":1581404160,"data_stored":1579788944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":719126528},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:14 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.98799999999999999,"15min":4.7699999999999996},"min":{"1min":0.26500000000000001,"5min":0.223,"15min":0.20399999999999999},"max":{"1min":2.274,"5min":8.8119999999999994,"15min":135.31999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.93000000000000005,"15min":4.617},"min":{"1min":0.26800000000000002,"5min":0.23999999999999999,"15min":0.22900000000000001},"max":{"1min":3.8239999999999998,"5min":8.3230000000000004,"15min":135.34299999999999},"last":0.52800000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:35:14 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":3.3690000000000002,"15min":2.4089999999999998},"min":{"1min":0.26000000000000001,"5min":0.251,"15min":0.245},"max":{"1min":6.9080000000000004,"5min":89.605000000000004,"15min":89.605000000000004},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.71599999999999997,"5min":3.1600000000000001,"15min":2.4849999999999999},"min":{"1min":0.33500000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":1.9159999999999999,"5min":91.025999999999996,"15min":91.025999999999996},"last":0.59999999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:35:03 2023","interfaces":[{"interface":"back","average":{"1min":9.4120000000000008,"5min":6.4420000000000002,"15min":4.6150000000000002},"min":{"1min":0.315,"5min":0.28399999999999997,"15min":0.27800000000000002},"max":{"1min":67.796999999999997,"5min":97.408000000000001,"15min":127.44499999999999},"last":0.65800000000000003},{"interface":"front","average":{"1min":8.6270000000000007,"5min":7.6139999999999999,"15min":5.0419999999999998},"min":{"1min":0.30599999999999999,"5min":0.29999999999999999,"15min":0.26500000000000001},"max":{"1min":67.893000000000001,"5min":125.539,"15min":125.539},"last":0.61499999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:34:48 2023","interfaces":[{"interface":"back","average":{"1min":8.5120000000000005,"5min":5.7119999999999997,"15min":3.9969999999999999},"min":{"1min":0.39400000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":69.433000000000007,"5min":138.87299999999999,"15min":138.87299999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":8.0280000000000005,"5min":5.5289999999999999,"15min":4.2859999999999996},"min":{"1min":0.35199999999999998,"5min":0.33200000000000002,"15min":0.28799999999999998},"max":{"1min":69.463999999999999,"5min":138.976,"15min":138.976},"last":0.55400000000000005}]},{"osd":4,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.82999999999999996,"15min":1.274},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":1.633,"5min":7.101,"15min":166.05199999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.55800000000000005,"5min":1.0289999999999999,"15min":1.4890000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.089,"5min":8.3499999999999996,"15min":166.631},"last":0.48499999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.96699999999999997,"15min":0.82399999999999995},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.1579999999999999,"5min":11.473000000000001,"15min":11.473000000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.433,"5min":1.091,"15min":0.94099999999999995},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.249},"max":{"1min":0.76700000000000002,"5min":7.5670000000000002,"15min":8.4090000000000007},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:35:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.5389999999999999,"15min":1.0580000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":2.3010000000000002,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.573,"15min":1.081},"min":{"1min":0.36199999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.9009999999999998,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.63100000000000001}]}]},{"osd":1,"up_from":112,"seq":481036337493,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2301332,"kb_used_data":1422732,"kb_used_omap":4,"kb_used_meta":878587,"kb_avail":91443820,"statfs":{"total":95995035648,"available":93638471680,"internally_reserved":0,"allocated":1456877568,"data_stored":1454101428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":899673998},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":2.8220000000000001,"15min":3.5670000000000002},"min":{"1min":0.21199999999999999,"5min":0.17799999999999999,"15min":0.16700000000000001},"max":{"1min":15.983000000000001,"5min":47.195999999999998,"15min":130.768},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.883,"15min":2.8650000000000002},"min":{"1min":0.20699999999999999,"5min":0.20000000000000001,"15min":0.19},"max":{"1min":4.9809999999999999,"5min":46.582000000000001,"15min":134.929},"last":0.28199999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:34:57 2023","interfaces":[{"interface":"back","average":{"1min":10.555,"5min":7.9610000000000003,"15min":6.7919999999999998},"min":{"1min":0.23000000000000001,"5min":0.214,"15min":0.20100000000000001},"max":{"1min":201.03999999999999,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":10.605,"5min":8.173,"15min":6.9109999999999996},"min":{"1min":0.24299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":202.12,"5min":202.12,"15min":232.21000000000001},"last":0.66300000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:34:57 2023","interfaces":[{"interface":"back","average":{"1min":9.6969999999999992,"5min":5.7679999999999998,"15min":5.1859999999999999},"min":{"1min":0.311,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":163.94300000000001,"5min":181.94399999999999,"15min":234.404},"last":0.432},{"interface":"front","average":{"1min":9.9290000000000003,"5min":5.6550000000000002,"15min":5.0430000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.24399999999999999},"max":{"1min":163.98699999999999,"5min":181.98500000000001,"15min":236.33699999999999},"last":0.34399999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":1.5569999999999999,"15min":1.4339999999999999},"min":{"1min":0.32700000000000001,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":2.8210000000000002,"5min":12.042999999999999,"15min":12.042999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.5609999999999999,"15min":1.498},"min":{"1min":0.34499999999999997,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":2.7810000000000001,"5min":18.108000000000001,"15min":18.108000000000001},"last":0.60699999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":3.2879999999999998,"15min":1.8260000000000001},"min":{"1min":0.307,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.0460000000000003,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.372},{"interface":"front","average":{"1min":0.91300000000000003,"5min":3.2890000000000001,"15min":2.141},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.7450000000000001,"5min":151.821,"15min":151.821},"last":0.58199999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.6850000000000001,"15min":1.954},"min":{"1min":0.34699999999999998,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":5.0960000000000001,"5min":8.9269999999999996,"15min":74.087999999999994},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.3300000000000001,"5min":1.885,"15min":2.157},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.32800000000000001},"max":{"1min":6.8920000000000003,"5min":11.927,"15min":73.974999999999994},"last":0.51800000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:34:57 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":2.1269999999999998,"15min":2.875},"min":{"1min":0.38700000000000001,"5min":0.37,"15min":0.32900000000000001},"max":{"1min":10.629,"5min":44.241,"15min":78.308000000000007},"last":0.64800000000000002},{"interface":"front","average":{"1min":1.165,"5min":2.218,"15min":2.706},"min":{"1min":0.42899999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.9880000000000004,"5min":43.920999999999999,"15min":77.204999999999998},"last":0.624}]}]},{"osd":0,"up_from":91,"seq":390842024316,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1571856,"kb_used_data":357512,"kb_used_omap":5,"kb_used_meta":1214330,"kb_avail":92173296,"statfs":{"total":95995035648,"available":94385455104,"internally_reserved":0,"allocated":366092288,"data_stored":363306418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1243474554},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:35:32 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":10.577,"15min":6.7670000000000003},"min":{"1min":0.20499999999999999,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":9.0310000000000006,"5min":301.12700000000001,"15min":301.12700000000001},"last":1.7709999999999999},{"interface":"front","average":{"1min":0.754,"5min":10.407999999999999,"15min":5.9059999999999997},"min":{"1min":0.20200000000000001,"5min":0.19700000000000001,"15min":0.17599999999999999},"max":{"1min":5.4729999999999999,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.91000000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:34:55 2023","interfaces":[{"interface":"back","average":{"1min":19.536999999999999,"5min":10.204000000000001,"15min":7.4470000000000001},"min":{"1min":0.23300000000000001,"5min":0.17699999999999999,"15min":0.17199999999999999},"max":{"1min":283.07400000000001,"5min":283.07400000000001,"15min":283.07400000000001},"last":1.8979999999999999},{"interface":"front","average":{"1min":18.260000000000002,"5min":10.146000000000001,"15min":7.5259999999999998},"min":{"1min":0.26800000000000002,"5min":0.19800000000000001,"15min":0.17599999999999999},"max":{"1min":283.161,"5min":283.161,"15min":283.161},"last":3.9740000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:35:32 2023","interfaces":[{"interface":"back","average":{"1min":2.7360000000000002,"5min":8.6509999999999998,"15min":5.4589999999999996},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":41.191000000000003,"5min":220.732,"15min":220.732},"last":4.1699999999999999},{"interface":"front","average":{"1min":2.9390000000000001,"5min":9.0440000000000005,"15min":5.5570000000000004},"min":{"1min":0.28799999999999998,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":46.688000000000002,"5min":218.78200000000001,"15min":218.78200000000001},"last":0.68000000000000005}]},{"osd":4,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":2.5859999999999999,"15min":2.8660000000000001},"min":{"1min":0.33300000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.4500000000000002,"5min":108.01900000000001,"15min":138.81299999999999},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.25,"5min":2.7490000000000001,"15min":3.4550000000000001},"min":{"1min":0.35499999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.3670000000000009,"5min":107.898,"15min":171.251},"last":4.7469999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":1.7310000000000001,"15min":2.8340000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32100000000000001},"max":{"1min":15.076000000000001,"5min":15.076000000000001,"15min":171.30699999999999},"last":4.6820000000000004},{"interface":"front","average":{"1min":1.5649999999999999,"5min":1.4790000000000001,"15min":2.0880000000000001},"min":{"1min":0.35499999999999998,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":6.5279999999999996,"5min":9.9000000000000004,"15min":139.40799999999999},"last":1.8149999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":2.6360000000000001,"15min":3.161},"min":{"1min":0.38,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":9.968,"5min":108.67,"15min":171.33600000000001},"last":4.718},{"interface":"front","average":{"1min":1.915,"5min":2.6699999999999999,"15min":3.4209999999999998},"min":{"1min":0.38900000000000001,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":9.157,"5min":108.804,"15min":184.13499999999999},"last":1.8700000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:34:55 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":2.8660000000000001,"15min":2.6640000000000001},"min":{"1min":0.34300000000000003,"5min":0.34000000000000002,"15min":0.32900000000000001},"max":{"1min":4.923,"5min":111.71299999999999,"15min":138.77699999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":2.4950000000000001,"5min":2.9689999999999999,"15min":2.5179999999999998},"min":{"1min":0.36499999999999999,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":9.8870000000000005,"5min":111.108,"15min":139.37899999999999},"last":1.839}]}]},{"osd":2,"up_from":199,"seq":854698492138,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3648220,"kb_used_data":2810068,"kb_used_omap":7,"kb_used_meta":838136,"kb_avail":90096932,"statfs":{"total":95995035648,"available":92259258368,"internally_reserved":0,"allocated":2877509632,"data_stored":2874669947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":858251345},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":4.726,"5min":2.6949999999999998,"15min":3.9089999999999998},"min":{"1min":0.19500000000000001,"5min":0.193,"15min":0.16700000000000001},"max":{"1min":61.950000000000003,"5min":61.950000000000003,"15min":119.277},"last":0.46000000000000002},{"interface":"front","average":{"1min":5.0199999999999996,"5min":2.504,"15min":3.726},"min":{"1min":0.27800000000000002,"5min":0.20399999999999999,"15min":0.18099999999999999},"max":{"1min":61.984999999999999,"5min":61.984999999999999,"15min":119.56100000000001},"last":1.155}]},{"osd":1,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":11.630000000000001,"5min":6.2729999999999997,"15min":4.5359999999999996},"min":{"1min":0.29299999999999998,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":106.765,"5min":106.765,"15min":157.98699999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":11.114000000000001,"5min":6.3659999999999997,"15min":4.6349999999999998},"min":{"1min":0.29599999999999999,"5min":0.222,"15min":0.222},"max":{"1min":106.732,"5min":106.732,"15min":158.03800000000001},"last":0.41899999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":11.803000000000001,"5min":6.0069999999999997,"15min":4.7309999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":106.70399999999999,"5min":106.70399999999999,"15min":119.366},"last":0.47899999999999998},{"interface":"front","average":{"1min":11.427,"5min":5.891,"15min":4.3280000000000003},"min":{"1min":0.35499999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":106.66800000000001,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.49399999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":5.0979999999999999,"5min":2.4510000000000001,"15min":2.3170000000000002},"min":{"1min":0.41399999999999998,"5min":0.33700000000000002,"15min":0.25},"max":{"1min":62.081000000000003,"5min":62.081000000000003,"15min":119.88200000000001},"last":0.68400000000000005},{"interface":"front","average":{"1min":4.9169999999999998,"5min":2.5369999999999999,"15min":1.835},"min":{"1min":0.39000000000000001,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":62.072000000000003,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.50900000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":5.3090000000000002,"5min":3.391,"15min":2.7400000000000002},"min":{"1min":0.38800000000000001,"5min":0.34300000000000003,"15min":0.26200000000000001},"max":{"1min":62.058,"5min":91.207999999999998,"15min":119.919},"last":0.71899999999999997},{"interface":"front","average":{"1min":4.8929999999999998,"5min":3.2469999999999999,"15min":2.6120000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":62.042999999999999,"5min":91.150999999999996,"15min":119.306},"last":0.441}]},{"osd":6,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":12.76,"5min":4.1360000000000001,"15min":3.0720000000000001},"min":{"1min":0.41599999999999998,"5min":0.36599999999999999,"15min":0.315},"max":{"1min":107.018,"5min":107.018,"15min":119.943},"last":0.78000000000000003},{"interface":"front","average":{"1min":5.7709999999999999,"5min":2.798,"15min":2.1819999999999999},"min":{"1min":0.48199999999999998,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":62.493000000000002,"5min":62.493000000000002,"15min":62.493000000000002},"last":0.75600000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:34:58 2023","interfaces":[{"interface":"back","average":{"1min":4.8719999999999999,"5min":2.8879999999999999,"15min":1.958},"min":{"1min":0.41999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":62.603000000000002,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.69999999999999996},{"interface":"front","average":{"1min":11.744999999999999,"5min":4.0620000000000003,"15min":2.4060000000000001},"min":{"1min":0.48799999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":106.809,"5min":106.809,"15min":106.809},"last":0.87}]}]},{"osd":3,"up_from":148,"seq":635655160106,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2391996,"kb_used_data":1502644,"kb_used_omap":2,"kb_used_meta":889341,"kb_avail":91353156,"statfs":{"total":95995035648,"available":93545631744,"internally_reserved":0,"allocated":1538707456,"data_stored":1535944135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":910685237},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,0,0,3,3,1,3,7],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.835,"15min":2.7349999999999999},"min":{"1min":0.215,"5min":0.19400000000000001,"15min":0.158},"max":{"1min":17.213000000000001,"5min":22.271999999999998,"15min":103.866},"last":0.216},{"interface":"front","average":{"1min":0.95499999999999996,"5min":2.9750000000000001,"15min":2.8250000000000002},"min":{"1min":0.20799999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":4.5469999999999997,"5min":24.683,"15min":101.884},"last":0.42199999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":4.3789999999999996,"15min":2.9239999999999999},"min":{"1min":0.23300000000000001,"5min":0.214,"15min":0.19600000000000001},"max":{"1min":3.516,"5min":85.242999999999995,"15min":85.242999999999995},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.871,"5min":3.9910000000000001,"15min":2.7250000000000001},"min":{"1min":0.25800000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.2949999999999999,"5min":85.302000000000007,"15min":85.302000000000007},"last":0.35099999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:34:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7230000000000001,"5min":5.5739999999999998,"15min":4.8579999999999997},"min":{"1min":0.25,"5min":0.25,"15min":0.19800000000000001},"max":{"1min":7.1699999999999999,"5min":73.742000000000004,"15min":101.279},"last":0.312},{"interface":"front","average":{"1min":1.99,"5min":5.468,"15min":4.6529999999999996},"min":{"1min":0.36699999999999999,"5min":0.25700000000000001,"15min":0.214},"max":{"1min":11.156000000000001,"5min":76.525999999999996,"15min":101.294},"last":0.38500000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.74,"5min":1.3859999999999999,"15min":1.3049999999999999},"min":{"1min":0.34999999999999998,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":10.151999999999999,"5min":10.151999999999999,"15min":10.151999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.464,"5min":2.3479999999999999,"15min":1.8240000000000001},"min":{"1min":0.35399999999999998,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":7.681,"5min":24.782,"15min":24.782},"last":0.63100000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":7.2480000000000002,"15min":3.4140000000000001},"min":{"1min":0.39500000000000002,"5min":0.308,"15min":0.28000000000000003},"max":{"1min":6.5010000000000003,"5min":214.43700000000001,"15min":214.43700000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":2.0779999999999998,"5min":7.0030000000000001,"15min":3.488},"min":{"1min":0.376,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":9.4160000000000004,"5min":214.46600000000001,"15min":214.46600000000001},"last":0.39600000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:34:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":3.1419999999999999,"15min":2.3050000000000002},"min":{"1min":0.39000000000000001,"5min":0.35899999999999999,"15min":0.31900000000000001},"max":{"1min":5.4169999999999998,"5min":58.932000000000002,"15min":58.932000000000002},"last":0.66900000000000004},{"interface":"front","average":{"1min":2.9900000000000002,"5min":2.8969999999999998,"15min":2.359},"min":{"1min":0.41799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":16.759,"5min":59.186,"15min":59.186},"last":0.36499999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:34:55 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":6.609,"15min":3.2200000000000002},"min":{"1min":0.42699999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":11.191000000000001,"5min":192.69,"15min":192.69},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.5779999999999998,"5min":7.4489999999999998,"15min":3.7269999999999999},"min":{"1min":0.47599999999999998,"5min":0.39200000000000002,"15min":0.36699999999999999},"max":{"1min":9.7449999999999992,"5min":192.65600000000001,"15min":192.65600000000001},"last":0.65000000000000002}]}]},{"osd":4,"up_from":0,"seq":128849019425,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757788,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3331672,"kb_used_data":2609232,"kb_used_omap":0,"kb_used_meta":722432,"kb_avail":90413480,"statfs":{"total":95995035648,"available":92583403520,"internally_reserved":0,"allocated":2671853568,"data_stored":2670238569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":739770368},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.3720000000000001,"15min":3.48},"min":{"1min":0.254,"5min":0.20899999999999999,"15min":0.19500000000000001},"max":{"1min":3.0089999999999999,"5min":21.109000000000002,"15min":113.869},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.6699999999999999,"15min":3.8650000000000002},"min":{"1min":0.25,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":5.1280000000000001,"5min":20.876999999999999,"15min":138.55199999999999},"last":0.44800000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.772,"15min":2.323},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.23100000000000001},"max":{"1min":3.629,"5min":29.007999999999999,"15min":69.816999999999993},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.9390000000000001,"15min":2.665},"min":{"1min":0.29999999999999999,"5min":0.249,"15min":0.249},"max":{"1min":4.4859999999999998,"5min":29.053999999999998,"15min":70.268000000000001},"last":0.46700000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:34:47 2023","interfaces":[{"interface":"back","average":{"1min":8.3900000000000006,"5min":5.3979999999999997,"15min":4.2009999999999996},"min":{"1min":0.32000000000000001,"5min":0.28299999999999997,"15min":0.26900000000000002},"max":{"1min":69.75,"5min":208.738,"15min":208.738},"last":0.51400000000000001},{"interface":"front","average":{"1min":8.1739999999999995,"5min":5.3280000000000003,"15min":4.0960000000000001},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":66.421000000000006,"5min":208.631,"15min":208.631},"last":0.34000000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:35:44 2023","interfaces":[{"interface":"back","average":{"1min":6.4889999999999999,"5min":4.1520000000000001,"15min":3.6589999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":122.071,"5min":122.071,"15min":122.071},"last":0.53800000000000003},{"interface":"front","average":{"1min":6.4669999999999996,"5min":4.4969999999999999,"15min":3.8359999999999999},"min":{"1min":0.35799999999999998,"5min":0.35399999999999998,"15min":0.318},"max":{"1min":122.09699999999999,"5min":122.09699999999999,"15min":122.09699999999999},"last":0.56299999999999994}]},{"osd":4,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.151,"15min":0.92000000000000004},"min":{"1min":0.34200000000000003,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":3.5499999999999998,"5min":9.2690000000000001,"15min":19.962},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.093,"15min":0.872},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.22900000000000001},"max":{"1min":2.4990000000000001,"5min":10.273999999999999,"15min":20.829999999999998},"last":0.377}]},{"osd":6,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":7.2519999999999998,"5min":2.3799999999999999,"15min":1.635},"min":{"1min":0.32400000000000001,"5min":0.27600000000000002,"15min":0.26900000000000002},"max":{"1min":148.36600000000001,"5min":148.36600000000001,"15min":148.36600000000001},"last":0.497},{"interface":"front","average":{"1min":7.7210000000000001,"5min":2.9420000000000002,"15min":1.869},"min":{"1min":0.34599999999999997,"5min":0.307,"15min":0.26000000000000001},"max":{"1min":148.42500000000001,"5min":148.42500000000001,"15min":148.42500000000001},"last":0.39500000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:34:47 2023","interfaces":[{"interface":"back","average":{"1min":2.5760000000000001,"5min":1.577,"15min":2.8639999999999999},"min":{"1min":0.34999999999999998,"5min":0.34599999999999997,"15min":0.317},"max":{"1min":15.497,"5min":16.408999999999999,"15min":179.28800000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.387,"5min":1.6890000000000001,"15min":2.956},"min":{"1min":0.36799999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":15.653,"5min":18.972000000000001,"15min":179.428},"last":0.52900000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":357302272,"data_stored":357302272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1448611840,"data_stored":1448607768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2868838400,"data_stored":2868838400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1530269696,"data_stored":1530265624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2555969536,"data_stored":2555969536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2662268928,"data_stored":2662268928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1573191680,"data_stored":1573191680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1696665600,"data_stored":1696661528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:35:48.516 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:35:48.539 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:48.609 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:48.689 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:48.845 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:48.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:48.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:48.951 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:48.979 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:49.077 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:49.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:49.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:49.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:49 smithi110 ceph-mon[212939]: pgmap v1627: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 229 op/s; 26485/211095 objects degraded (12.546%); 4263/211095 objects misplaced (2.019%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:35:49.098 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:49 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 26485/211095 objects degraded (12.546%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:49.098 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:49 smithi110 ceph-mon[221102]: pgmap v1627: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 229 op/s; 26485/211095 objects degraded (12.546%); 4263/211095 objects misplaced (2.019%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:35:49.098 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:49 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 26485/211095 objects degraded (12.546%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:49.183 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:49.224 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:49.338 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:49.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:49.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:49.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:49 smithi138 ceph-mon[193606]: pgmap v1627: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 229 op/s; 26485/211095 objects degraded (12.546%); 4263/211095 objects misplaced (2.019%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:35:49.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:49 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 26485/211095 objects degraded (12.546%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:49.446 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:49.486 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:49.538 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:49.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:49.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:49.642 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:49.669 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:49.714 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:49.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:49.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:49.825 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:49.855 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:50.108 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:35:50.135 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:50 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1582696636' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:35:50.136 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:50 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1582696636' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:35:50.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:50 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1582696636' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:35:50.787 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:50.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:50.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:50.865 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:50.890 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:50.924 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:50.975 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:50.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:50.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:51.082 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:51.136 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:51.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:51 smithi138 ceph-mon[193606]: pgmap v1628: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 26427/209502 objects degraded (12.614%); 4263/209502 objects misplaced (2.035%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:35:51.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:51 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:35:51.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:51 smithi138 ceph-mon[193606]: failed to encode map e590 with expected crc 2023-09-23T23:35:51.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:51 smithi138 ceph-mon[193606]: failed to encode map e590 with expected crc 2023-09-23T23:35:51.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:51 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1582696636' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:35:51.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:51 smithi138 ceph-mon[193606]: osdmap e590: 8 total, 8 up, 7 in 2023-09-23T23:35:51.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:51 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:51.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:51 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:51.248 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:51.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:51.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:51.326 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:51.352 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:51.405 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:51 smithi110 ceph-mon[212939]: pgmap v1628: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 26427/209502 objects degraded (12.614%); 4263/209502 objects misplaced (2.035%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:35:51.405 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:51 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:35:51.405 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:51 smithi110 ceph-mon[212939]: failed to encode map e590 with expected crc 2023-09-23T23:35:51.405 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:51 smithi110 ceph-mon[212939]: failed to encode map e590 with expected crc 2023-09-23T23:35:51.405 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:51 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1582696636' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:35:51.405 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:51 smithi110 ceph-mon[212939]: osdmap e590: 8 total, 8 up, 7 in 2023-09-23T23:35:51.405 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:51 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:51.405 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:51 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:51.406 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:51 smithi110 ceph-mon[221102]: pgmap v1628: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 26427/209502 objects degraded (12.614%); 4263/209502 objects misplaced (2.035%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:35:51.406 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:51 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:35:51.406 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:51 smithi110 ceph-mon[221102]: failed to encode map e590 with expected crc 2023-09-23T23:35:51.406 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:51 smithi110 ceph-mon[221102]: failed to encode map e590 with expected crc 2023-09-23T23:35:51.406 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:51 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1582696636' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:35:51.406 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:51 smithi110 ceph-mon[221102]: osdmap e590: 8 total, 8 up, 7 in 2023-09-23T23:35:51.406 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:51 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:51.406 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:51 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:51.414 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:51.462 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:35:51.462 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:35:51.677 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:51.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:51.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:51.781 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:51.812 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:51.862 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:51.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:51.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:51.969 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:51.998 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:52.044 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:52.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:52.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:52.150 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:52.197 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:52.758 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:52.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:52.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:52.816 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:35:52.864 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:52.894 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:35:52.916 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:53.181 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:53.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:53.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:53.287 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:53.313 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:53.356 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:53.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:53 smithi138 ceph-mon[193606]: pgmap v1630: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 213 op/s; 26427/209025 objects degraded (12.643%); 4263/209025 objects misplaced (2.039%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:35:53.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:53.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:53.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:53 smithi110 ceph-mon[212939]: pgmap v1630: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 213 op/s; 26427/209025 objects degraded (12.643%); 4263/209025 objects misplaced (2.039%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:35:53.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:53 smithi110 ceph-mon[221102]: pgmap v1630: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 213 op/s; 26427/209025 objects degraded (12.643%); 4263/209025 objects misplaced (2.039%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:35:53.462 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:35:53.488 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:53.532 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:35:53.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:53.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:35:53.638 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:53.664 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:53.706 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:53.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:53.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:53.716 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:35:53.716 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:35:53.716 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:35:53.716 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:35:53.717 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:35:53.717 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:35:53.717 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:35:53.717 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:35:53.717 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:35:53.717 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:35:53.717 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:35:53.717 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:35:53.717 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:35:53.717 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:35:53.718 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:35:53.718 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:35:53.812 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:35:53.844 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:54.110 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:54 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1643430855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:54.111 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:54 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 26427/209025 objects degraded (12.643%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:54.111 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:54 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1643430855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:54.111 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:54 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 26427/209025 objects degraded (12.643%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:54.343 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:35:54.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:54.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:35:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:54 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1643430855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:35:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:54 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 26427/209025 objects degraded (12.643%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:54.449 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:35:54.481 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:54.544 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:35:54.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:54.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:35:54.651 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:54.684 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:54.814 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:54.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:54.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:54.925 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:54.932 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:54.951 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:55.007 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:55.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:55.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:55.114 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:55.144 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:55.351 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:55.355 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:55 smithi110 ceph-mon[212939]: pgmap v1631: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 214 op/s; 26357/208239 objects degraded (12.657%); 4196/208239 objects misplaced (2.015%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:35:55.356 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:55 smithi110 ceph-mon[221102]: pgmap v1631: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 214 op/s; 26357/208239 objects degraded (12.657%); 4196/208239 objects misplaced (2.015%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:35:55.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:55.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:55.422 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:55.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:55 smithi138 ceph-mon[193606]: pgmap v1631: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 214 op/s; 26357/208239 objects degraded (12.657%); 4196/208239 objects misplaced (2.015%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:35:55.461 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:35:55.513 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:55.574 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:35:55.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:55.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:35:55.681 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:55.710 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:55.757 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:55.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:55.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:55.767 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:35:55.767 INFO:teuthology.orchestra.run.smithi110.stdout:{"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 26357/208239 objects degraded (12.657%), 7 pgs degraded, 5 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2033,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":590,"num_osds":8,"num_up_osds":8,"osd_up_since":1695511414,"num_in_osds":7,"osd_in_since":1695512030,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":110},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+recovering","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":119,"num_pools":5,"num_objects":69413,"data_bytes":4548048117,"bytes_used":17723469824,"bytes_avail":654241779712,"bytes_total":671965249536,"degraded_objects":26357,"degraded_total":208239,"degraded_ratio":0.12657091130864057,"misplaced_objects":4196,"misplaced_total":208239,"misplaced_ratio":0.020149923885535371,"recovering_objects_per_sec":41,"recovering_bytes_per_sec":2745168,"recovering_keys_per_sec":0,"num_objects_recovered":419,"num_bytes_recovered":27459584,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":214},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi110.fuglpi","status":"up:active","gid":24572}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.110:8443/","prometheus":"http://172.21.15.110:9283/"}},"servicemap":{"epoch":32,"modified":"2023-09-23T23:32:57.852217+0000","services":{}},"progress_events":{"833ef15d-07ae-4299-b4e4-2cf94e068147":{"message":"Upgrade to 18.2.0-460-gbb0daeff (10m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true},"c67c6d89-9cd0-495b-b1de-fa24b5d12727":{"message":"Global Recovery Event (95s)\n [=========================...] (remaining: 7s)","progress":0.92436975240707397,"add_to_ceph_s":true}}} 2023-09-23T23:35:55.862 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:55.904 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:56 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3574045914' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:35:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:56 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3574045914' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:35:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:56 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3574045914' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:35:56.828 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:35:57.041 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:35:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:35:56] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2023-09-23T23:35:57.066 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:57.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:57.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:57.072 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:35:57.072 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:35:57.172 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:57.198 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:57.246 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:57.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:57.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:57.312 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:57 smithi110 ceph-mon[212939]: pgmap v1632: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 26300/205875 objects degraded (12.775%); 4196/205875 objects misplaced (2.038%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T23:35:57.312 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:57 smithi110 ceph-mon[221102]: pgmap v1632: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 26300/205875 objects degraded (12.775%); 4196/205875 objects misplaced (2.038%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T23:35:57.353 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:57.391 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:57 smithi138 ceph-mon[193606]: pgmap v1632: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 26300/205875 objects degraded (12.775%); 4196/205875 objects misplaced (2.038%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T23:35:58.105 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:35:58.105 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:35:58.105 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:35:58.105 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:35:58.106 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:35:58.174 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:58.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:58.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:58.280 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:58.306 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:58.347 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:58.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:58.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:35:58.452 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:58.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:35:58.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:35:58.486 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:58.818 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:58.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:58.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:58.921 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:58.948 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:58.991 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:35:58.993 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:58.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:58.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:59.013 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:59.072 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:35:59.099 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:35:59.123 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:59 smithi110 ceph-mon[212939]: pgmap v1633: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 197 op/s; 26242/205182 objects degraded (12.790%); 4127/205182 objects misplaced (2.011%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:35:59.123 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:59 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 26242/205182 objects degraded (12.790%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:59.123 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:59 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:59.124 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:35:59 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:59.125 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:59.168 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:35:59.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:59.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:35:59.274 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:35:59.298 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:59 smithi138 ceph-mon[193606]: pgmap v1633: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 197 op/s; 26242/205182 objects degraded (12.790%); 4127/205182 objects misplaced (2.011%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:35:59.298 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:59 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 26242/205182 objects degraded (12.790%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:59.298 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:59 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:59.299 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:35:59 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:59.301 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:59.345 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:35:59.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:59.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:35:59.450 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:35:59.460 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:59 smithi110 ceph-mon[221102]: pgmap v1633: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 197 op/s; 26242/205182 objects degraded (12.790%); 4127/205182 objects misplaced (2.011%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:35:59.460 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:59 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 26242/205182 objects degraded (12.790%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:35:59.460 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:59 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:59.460 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:35:59 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:35:59.509 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:59.612 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:35:59.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:59.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:35:59.721 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:59.759 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:35:59.813 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:35:59.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:35:59.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:35:59.918 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:35:59.945 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:00.044 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:00.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:00.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:00.148 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:00.175 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:00 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:36:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:00 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:36:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:00 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:36:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:00 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1386847944' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:00 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:00 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:36:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:00 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:36:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:00 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:36:00.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:00 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1386847944' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:00.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:00 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:00.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:00 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:36:00.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:00 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:36:00.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:00 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:36:00.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:00 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1386847944' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:00.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:00 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:00.571 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:36:00.585 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:00.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:00.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:00.692 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:00.721 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:01.131 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:01.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:01.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:01.235 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:36:01.261 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:01.312 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:36:01.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:01.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:36:01.419 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:01.439 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:01 smithi110 ceph-mon[212939]: pgmap v1634: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 26164/203346 objects degraded (12.867%); 4127/203346 objects misplaced (2.030%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:36:01.439 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:01 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:36:01.439 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:01 smithi110 ceph-mon[212939]: failed to encode map e591 with expected crc 2023-09-23T23:36:01.439 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:01 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:36:01.439 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:01 smithi110 ceph-mon[212939]: osdmap e591: 8 total, 8 up, 7 in 2023-09-23T23:36:01.439 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:01 smithi110 ceph-mon[212939]: failed to encode map e591 with expected crc 2023-09-23T23:36:01.439 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:01 smithi110 ceph-mon[221102]: pgmap v1634: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 26164/203346 objects degraded (12.867%); 4127/203346 objects misplaced (2.030%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:36:01.439 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:01 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:36:01.439 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:01 smithi110 ceph-mon[221102]: failed to encode map e591 with expected crc 2023-09-23T23:36:01.440 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:01 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:36:01.440 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:01 smithi110 ceph-mon[221102]: osdmap e591: 8 total, 8 up, 7 in 2023-09-23T23:36:01.440 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:01 smithi110 ceph-mon[221102]: failed to encode map e591 with expected crc 2023-09-23T23:36:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:01 smithi138 ceph-mon[193606]: pgmap v1634: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 26164/203346 objects degraded (12.867%); 4127/203346 objects misplaced (2.030%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:36:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:01 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:36:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:01 smithi138 ceph-mon[193606]: failed to encode map e591 with expected crc 2023-09-23T23:36:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:01 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:36:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:01 smithi138 ceph-mon[193606]: osdmap e591: 8 total, 8 up, 7 in 2023-09-23T23:36:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:01 smithi138 ceph-mon[193606]: failed to encode map e591 with expected crc 2023-09-23T23:36:01.454 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:01.698 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:01.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:01.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:01.768 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:01.803 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:01.835 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:02.414 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:02 smithi110 ceph-mon[212939]: failed to encode map e591 with expected crc 2023-09-23T23:36:02.414 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:02 smithi110 ceph-mon[221102]: failed to encode map e591 with expected crc 2023-09-23T23:36:02.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:02 smithi138 ceph-mon[193606]: failed to encode map e591 with expected crc 2023-09-23T23:36:02.445 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:36:02.474 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:36:02.507 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:36:02.511 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:36:02.516 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:36:02.544 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:36:02.568 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:36:02.600 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:36:02.629 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:36:02.655 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:36:02.657 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:36:02.657 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:36:02.657 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:36:02.658 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:36:02.658 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:36:02.658 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:36:02.658 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:36:02.658 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:36:03.423 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:03 smithi110 ceph-mon[221102]: pgmap v1636: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 26164/202857 objects degraded (12.898%); 4127/202857 objects misplaced (2.034%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:36:03.423 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:03 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/143075527' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:03.423 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:03 smithi110 ceph-mon[221102]: from='client.36765 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:03.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:03 smithi110 ceph-mon[212939]: pgmap v1636: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 26164/202857 objects degraded (12.898%); 4127/202857 objects misplaced (2.034%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:36:03.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:03 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/143075527' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:03.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:03 smithi110 ceph-mon[212939]: from='client.36765 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:03 smithi138 ceph-mon[193606]: pgmap v1636: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 26164/202857 objects degraded (12.898%); 4127/202857 objects misplaced (2.034%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:36:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:03 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/143075527' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:03 smithi138 ceph-mon[193606]: from='client.36765 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:03.654 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:03.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:03.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:03.710 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:36:03.761 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:03.787 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:03.837 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:03.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:03.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:03.943 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:03.977 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:04.388 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:04 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 26164/202857 objects degraded (12.898%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:04.389 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:04 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 26164/202857 objects degraded (12.898%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:04.390 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1636,"stamp":"2023-09-23T23:36:01.913660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4430476533,"num_objects":67619,"num_object_clones":0,"num_object_copies":202857,"num_objects_missing_on_primary":3232,"num_objects_missing":3232,"num_objects_degraded":26164,"num_objects_misplaced":4127,"num_objects_unfound":0,"num_objects_dirty":67619,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":108038,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13304,"num_bytes_recovered":869200296,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94985,"ondisk_log_size":94985,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17024272,"kb_used_data":11409944,"kb_used_omap":29,"kb_used_meta":5614242,"kb_avail":639191792,"statfs":{"total":671965249536,"available":654532395008,"internally_reserved":0,"allocated":11683782656,"data_stored":11667714209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5748984531},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,3,1,1,1,1,5,1,9],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-134742016,"num_objects":-2056,"num_object_clones":0,"num_object_copies":-6168,"num_objects_missing_on_primary":-136,"num_objects_missing":-136,"num_objects_degraded":-263,"num_objects_misplaced":-136,"num_objects_unfound":0,"num_objects_dirty":-2056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2056,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":397,"num_bytes_recovered":26017792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002999"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":786,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.105739+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:35:43.105739+0000","last_peered":"2023-09-23T23:35:43.105739+0000","last_clean":"2023-09-23T23:35:43.105739+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:35:43.105739+0000","last_undegraded":"2023-09-23T23:35:43.105739+0000","last_fullsized":"2023-09-23T23:35:43.105739+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":344,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.083744+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:35:43.083744+0000","last_peered":"2023-09-23T23:35:43.083744+0000","last_clean":"2023-09-23T23:35:43.083744+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:35:43.083744+0000","last_undegraded":"2023-09-23T23:35:43.083744+0000","last_fullsized":"2023-09-23T23:35:43.083744+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":571,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.138124+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:35:43.138124+0000","last_peered":"2023-09-23T23:35:43.138124+0000","last_clean":"2023-09-23T23:35:43.138124+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:35:43.138124+0000","last_undegraded":"2023-09-23T23:35:43.138124+0000","last_fullsized":"2023-09-23T23:35:43.138124+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":739,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.106146+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:35:43.106146+0000","last_peered":"2023-09-23T23:35:43.106146+0000","last_clean":"2023-09-23T23:35:43.106146+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:35:43.106146+0000","last_undegraded":"2023-09-23T23:35:43.106146+0000","last_fullsized":"2023-09-23T23:35:43.106146+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":308,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.097181+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:35:43.097181+0000","last_peered":"2023-09-23T23:35:43.097181+0000","last_clean":"2023-09-23T23:35:43.097181+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:35:43.097181+0000","last_undegraded":"2023-09-23T23:35:43.097181+0000","last_fullsized":"2023-09-23T23:35:43.097181+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":550,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.097964+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:35:43.097964+0000","last_peered":"2023-09-23T23:35:43.097964+0000","last_clean":"2023-09-23T23:35:43.097964+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:35:43.097964+0000","last_undegraded":"2023-09-23T23:35:43.097964+0000","last_fullsized":"2023-09-23T23:35:43.097964+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":751,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.094747+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:35:43.094747+0000","last_peered":"2023-09-23T23:35:43.094747+0000","last_clean":"2023-09-23T23:35:43.094747+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:35:43.094747+0000","last_undegraded":"2023-09-23T23:35:43.094747+0000","last_fullsized":"2023-09-23T23:35:43.094747+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":608,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.096589+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:35:43.096589+0000","last_peered":"2023-09-23T23:35:43.096589+0000","last_clean":"2023-09-23T23:35:43.096589+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:35:43.096589+0000","last_undegraded":"2023-09-23T23:35:43.096589+0000","last_fullsized":"2023-09-23T23:35:43.096589+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":104,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.123809+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:35:43.123809+0000","last_peered":"2023-09-23T23:35:43.123809+0000","last_clean":"2023-09-23T23:35:43.123809+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:35:43.123809+0000","last_undegraded":"2023-09-23T23:35:43.123809+0000","last_fullsized":"2023-09-23T23:35:43.123809+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":910,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.095048+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:35:43.095048+0000","last_peered":"2023-09-23T23:35:43.095048+0000","last_clean":"2023-09-23T23:35:43.095048+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:35:43.095048+0000","last_undegraded":"2023-09-23T23:35:43.095048+0000","last_fullsized":"2023-09-23T23:35:43.095048+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":242,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.107209+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:35:43.107209+0000","last_peered":"2023-09-23T23:35:43.107209+0000","last_clean":"2023-09-23T23:35:43.107209+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:35:43.107209+0000","last_undegraded":"2023-09-23T23:35:43.107209+0000","last_fullsized":"2023-09-23T23:35:43.107209+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":590,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.096562+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:35:43.096562+0000","last_peered":"2023-09-23T23:35:43.096562+0000","last_clean":"2023-09-23T23:35:43.096562+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:35:43.096562+0000","last_undegraded":"2023-09-23T23:35:43.096562+0000","last_fullsized":"2023-09-23T23:35:43.096562+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":883,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.083380+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:35:43.083380+0000","last_peered":"2023-09-23T23:35:43.083380+0000","last_clean":"2023-09-23T23:35:43.083380+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:35:43.083380+0000","last_undegraded":"2023-09-23T23:35:43.083380+0000","last_fullsized":"2023-09-23T23:35:43.083380+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":221,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.106893+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:35:43.106893+0000","last_peered":"2023-09-23T23:35:43.106893+0000","last_clean":"2023-09-23T23:35:43.106893+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:35:43.106893+0000","last_undegraded":"2023-09-23T23:35:43.106893+0000","last_fullsized":"2023-09-23T23:35:43.106893+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":551,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.097051+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:35:43.097051+0000","last_peered":"2023-09-23T23:35:43.097051+0000","last_clean":"2023-09-23T23:35:43.097051+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:35:43.097051+0000","last_undegraded":"2023-09-23T23:35:43.097051+0000","last_fullsized":"2023-09-23T23:35:43.097051+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1044,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.107049+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:35:43.107049+0000","last_peered":"2023-09-23T23:35:43.107049+0000","last_clean":"2023-09-23T23:35:43.107049+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:35:43.107049+0000","last_undegraded":"2023-09-23T23:35:43.107049+0000","last_fullsized":"2023-09-23T23:35:43.107049+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":262,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.122750+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:35:43.122750+0000","last_peered":"2023-09-23T23:35:43.122750+0000","last_clean":"2023-09-23T23:35:43.122750+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:35:43.122750+0000","last_undegraded":"2023-09-23T23:35:43.122750+0000","last_fullsized":"2023-09-23T23:35:43.122750+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":123,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.117028+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:35:43.117028+0000","last_peered":"2023-09-23T23:35:43.117028+0000","last_clean":"2023-09-23T23:35:43.117028+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:35:43.117028+0000","last_undegraded":"2023-09-23T23:35:43.117028+0000","last_fullsized":"2023-09-23T23:35:43.117028+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":440,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.098096+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:35:43.098096+0000","last_peered":"2023-09-23T23:35:43.098096+0000","last_clean":"2023-09-23T23:35:43.098096+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:35:43.098096+0000","last_undegraded":"2023-09-23T23:35:43.098096+0000","last_fullsized":"2023-09-23T23:35:43.098096+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":164,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.116875+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:35:43.116875+0000","last_peered":"2023-09-23T23:35:43.116875+0000","last_clean":"2023-09-23T23:35:43.116875+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:35:43.116875+0000","last_undegraded":"2023-09-23T23:35:43.116875+0000","last_fullsized":"2023-09-23T23:35:43.116875+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":531,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.115742+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:35:43.115742+0000","last_peered":"2023-09-23T23:35:43.115742+0000","last_clean":"2023-09-23T23:35:43.115742+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:35:43.115742+0000","last_undegraded":"2023-09-23T23:35:43.115742+0000","last_fullsized":"2023-09-23T23:35:43.115742+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":745,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.105685+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:35:43.105685+0000","last_peered":"2023-09-23T23:35:43.105685+0000","last_clean":"2023-09-23T23:35:43.105685+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:35:43.105685+0000","last_undegraded":"2023-09-23T23:35:43.105685+0000","last_fullsized":"2023-09-23T23:35:43.105685+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":136,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.081395+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:35:43.081395+0000","last_peered":"2023-09-23T23:35:43.081395+0000","last_clean":"2023-09-23T23:35:43.081395+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:35:43.081395+0000","last_undegraded":"2023-09-23T23:35:43.081395+0000","last_fullsized":"2023-09-23T23:35:43.081395+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":589,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.097517+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:35:43.097517+0000","last_peered":"2023-09-23T23:35:43.097517+0000","last_clean":"2023-09-23T23:35:43.097517+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:35:43.097517+0000","last_undegraded":"2023-09-23T23:35:43.097517+0000","last_fullsized":"2023-09-23T23:35:43.097517+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":875,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.115415+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:35:43.115415+0000","last_peered":"2023-09-23T23:35:43.115415+0000","last_clean":"2023-09-23T23:35:43.115415+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:35:43.115415+0000","last_undegraded":"2023-09-23T23:35:43.115415+0000","last_fullsized":"2023-09-23T23:35:43.115415+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":184,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.101837+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:35:43.101837+0000","last_peered":"2023-09-23T23:35:43.101837+0000","last_clean":"2023-09-23T23:35:43.101837+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:35:43.101837+0000","last_undegraded":"2023-09-23T23:35:43.101837+0000","last_fullsized":"2023-09-23T23:35:43.101837+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":311,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.123380+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:35:43.123380+0000","last_peered":"2023-09-23T23:35:43.123380+0000","last_clean":"2023-09-23T23:35:43.123380+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:35:43.123380+0000","last_undegraded":"2023-09-23T23:35:43.123380+0000","last_fullsized":"2023-09-23T23:35:43.123380+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":552,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.098879+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:35:43.098879+0000","last_peered":"2023-09-23T23:35:43.098879+0000","last_clean":"2023-09-23T23:35:43.098879+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:35:43.098879+0000","last_undegraded":"2023-09-23T23:35:43.098879+0000","last_fullsized":"2023-09-23T23:35:43.098879+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":458,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.120255+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:35:43.120255+0000","last_peered":"2023-09-23T23:35:43.120255+0000","last_clean":"2023-09-23T23:35:43.120255+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:35:43.120255+0000","last_undegraded":"2023-09-23T23:35:43.120255+0000","last_fullsized":"2023-09-23T23:35:43.120255+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":546,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.098240+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:35:43.098240+0000","last_peered":"2023-09-23T23:35:43.098240+0000","last_clean":"2023-09-23T23:35:43.098240+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:35:43.098240+0000","last_undegraded":"2023-09-23T23:35:43.098240+0000","last_fullsized":"2023-09-23T23:35:43.098240+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":953,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.106117+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:35:43.106117+0000","last_peered":"2023-09-23T23:35:43.106117+0000","last_clean":"2023-09-23T23:35:43.106117+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:35:43.106117+0000","last_undegraded":"2023-09-23T23:35:43.106117+0000","last_fullsized":"2023-09-23T23:35:43.106117+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":154,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.115614+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:35:43.115614+0000","last_peered":"2023-09-23T23:35:43.115614+0000","last_clean":"2023-09-23T23:35:43.115614+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:35:43.115614+0000","last_undegraded":"2023-09-23T23:35:43.115614+0000","last_fullsized":"2023-09-23T23:35:43.115614+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":231,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:47.197851+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:35:47.197851+0000","last_peered":"2023-09-23T23:35:47.197851+0000","last_clean":"2023-09-23T23:35:47.197851+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:35:47.197851+0000","last_undegraded":"2023-09-23T23:35:47.197851+0000","last_fullsized":"2023-09-23T23:35:47.197851+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"590'6665","reported_seq":6738,"reported_epoch":590,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:58.883934+0000","last_change":"2023-09-23T23:33:52.884769+0000","last_active":"2023-09-23T23:35:58.883934+0000","last_peered":"2023-09-23T23:35:58.883934+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:35:58.883934+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"546'1300","ondisk_log_start":"546'1300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5365,"log_dups_size":0,"ondisk_log_size":5365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274137088,"num_objects":4183,"num_object_clones":0,"num_object_copies":12549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6665,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":3157},{"shards":"2,6","objects":-215}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":923,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.081837+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:35:43.081837+0000","last_peered":"2023-09-23T23:35:43.081837+0000","last_clean":"2023-09-23T23:35:43.081837+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:35:43.081837+0000","last_undegraded":"2023-09-23T23:35:43.081837+0000","last_fullsized":"2023-09-23T23:35:43.081837+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":231,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.096334+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:35:43.096334+0000","last_peered":"2023-09-23T23:35:43.096334+0000","last_clean":"2023-09-23T23:35:43.096334+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:35:43.096334+0000","last_undegraded":"2023-09-23T23:35:43.096334+0000","last_fullsized":"2023-09-23T23:35:43.096334+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":221,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.105680+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:35:43.105680+0000","last_peered":"2023-09-23T23:35:43.105680+0000","last_clean":"2023-09-23T23:35:43.105680+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:35:43.105680+0000","last_undegraded":"2023-09-23T23:35:43.105680+0000","last_fullsized":"2023-09-23T23:35:43.105680+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"590'6777","reported_seq":6849,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:59.701854+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:35:59.701854+0000","last_peered":"2023-09-23T23:35:59.701854+0000","last_clean":"2023-09-23T23:35:59.701854+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:35:59.701854+0000","last_undegraded":"2023-09-23T23:35:59.701854+0000","last_fullsized":"2023-09-23T23:35:59.701854+0000","mapping_epoch":566,"log_start":"541'600","ondisk_log_start":"541'600","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6177,"log_dups_size":0,"ondisk_log_size":6177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278855680,"num_objects":4255,"num_object_clones":0,"num_object_copies":12765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6777,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1002,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:47.159896+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:35:47.159896+0000","last_peered":"2023-09-23T23:35:47.159896+0000","last_clean":"2023-09-23T23:35:47.159896+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:35:47.159896+0000","last_undegraded":"2023-09-23T23:35:47.159896+0000","last_fullsized":"2023-09-23T23:35:47.159896+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":221,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.096036+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:35:43.096036+0000","last_peered":"2023-09-23T23:35:43.096036+0000","last_clean":"2023-09-23T23:35:43.096036+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:35:43.096036+0000","last_undegraded":"2023-09-23T23:35:43.096036+0000","last_fullsized":"2023-09-23T23:35:43.096036+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":440,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.096709+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:35:43.096709+0000","last_peered":"2023-09-23T23:35:43.096709+0000","last_clean":"2023-09-23T23:35:43.096709+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:35:43.096709+0000","last_undegraded":"2023-09-23T23:35:43.096709+0000","last_fullsized":"2023-09-23T23:35:43.096709+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"590'6822","reported_seq":10525,"reported_epoch":590,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:35:57.232050+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:35:57.232050+0000","last_peered":"2023-09-23T23:35:57.232050+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:35:57.232050+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:35:57.232050+0000","mapping_epoch":571,"log_start":"548'1500","ondisk_log_start":"548'1500","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5322,"log_dups_size":0,"ondisk_log_size":5322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276955136,"num_objects":4226,"num_object_clones":0,"num_object_copies":12678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3331,"num_objects_misplaced":895,"num_objects_unfound":0,"num_objects_dirty":4226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6822,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1800,"num_bytes_recovered":117964800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":3209},{"shards":"3,5,7","objects":1017}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":900,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.123885+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:35:43.123885+0000","last_peered":"2023-09-23T23:35:43.123885+0000","last_clean":"2023-09-23T23:35:43.123885+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:35:43.123885+0000","last_undegraded":"2023-09-23T23:35:43.123885+0000","last_fullsized":"2023-09-23T23:35:43.123885+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":328,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.106702+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:35:43.106702+0000","last_peered":"2023-09-23T23:35:43.106702+0000","last_clean":"2023-09-23T23:35:43.106702+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:35:43.106702+0000","last_undegraded":"2023-09-23T23:35:43.106702+0000","last_fullsized":"2023-09-23T23:35:43.106702+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":104,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.117233+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:35:43.117233+0000","last_peered":"2023-09-23T23:35:43.117233+0000","last_clean":"2023-09-23T23:35:43.117233+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:35:43.117233+0000","last_undegraded":"2023-09-23T23:35:43.117233+0000","last_fullsized":"2023-09-23T23:35:43.117233+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"590'6752","reported_seq":8811,"reported_epoch":590,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:57.208246+0000","last_change":"2023-09-23T23:34:35.750428+0000","last_active":"2023-09-23T23:35:57.208246+0000","last_peered":"2023-09-23T23:35:57.208246+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:35:57.208246+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"548'1400","ondisk_log_start":"548'1400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5352,"log_dups_size":0,"ondisk_log_size":5352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284295168,"num_objects":4338,"num_object_clones":0,"num_object_copies":13014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6752,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":991,"num_bytes_recovered":64946176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":3328},{"shards":"3,5","objects":794}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":496,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.098170+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:35:43.098170+0000","last_peered":"2023-09-23T23:35:43.098170+0000","last_clean":"2023-09-23T23:35:43.098170+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:35:43.098170+0000","last_undegraded":"2023-09-23T23:35:43.098170+0000","last_fullsized":"2023-09-23T23:35:43.098170+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":430,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.097405+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:35:43.097405+0000","last_peered":"2023-09-23T23:35:43.097405+0000","last_clean":"2023-09-23T23:35:43.097405+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:35:43.097405+0000","last_undegraded":"2023-09-23T23:35:43.097405+0000","last_fullsized":"2023-09-23T23:35:43.097405+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":588,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.155818+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:35:50.155818+0000","last_peered":"2023-09-23T23:35:50.155818+0000","last_clean":"2023-09-23T23:35:50.155818+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:35:50.155818+0000","last_undegraded":"2023-09-23T23:35:50.155818+0000","last_fullsized":"2023-09-23T23:35:50.155818+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"590'6894","reported_seq":8214,"reported_epoch":590,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:59.067606+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:35:59.067606+0000","last_peered":"2023-09-23T23:35:59.067606+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:35:59.067606+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"544'1200","ondisk_log_start":"544'1200","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5694,"log_dups_size":0,"ondisk_log_size":5694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279707648,"num_objects":4268,"num_object_clones":0,"num_object_copies":12804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6894,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":3222},{"shards":"3,5","objects":-1192},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":717,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.096984+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:35:43.096984+0000","last_peered":"2023-09-23T23:35:43.096984+0000","last_clean":"2023-09-23T23:35:43.096984+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:35:43.096984+0000","last_undegraded":"2023-09-23T23:35:43.096984+0000","last_fullsized":"2023-09-23T23:35:43.096984+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":600,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.098802+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:35:43.098802+0000","last_peered":"2023-09-23T23:35:43.098802+0000","last_clean":"2023-09-23T23:35:43.098802+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:35:43.098802+0000","last_undegraded":"2023-09-23T23:35:43.098802+0000","last_fullsized":"2023-09-23T23:35:43.098802+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":510,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.097337+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:35:43.097337+0000","last_peered":"2023-09-23T23:35:43.097337+0000","last_clean":"2023-09-23T23:35:43.097337+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:35:43.097337+0000","last_undegraded":"2023-09-23T23:35:43.097337+0000","last_fullsized":"2023-09-23T23:35:43.097337+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"590'6834","reported_seq":6898,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:58.966379+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:35:58.966379+0000","last_peered":"2023-09-23T23:35:58.966379+0000","last_clean":"2023-09-23T23:35:58.966379+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:35:58.966379+0000","last_undegraded":"2023-09-23T23:35:58.966379+0000","last_fullsized":"2023-09-23T23:35:58.966379+0000","mapping_epoch":534,"log_start":"548'1400","ondisk_log_start":"548'1400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5434,"log_dups_size":0,"ondisk_log_size":5434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278265856,"num_objects":4246,"num_object_clones":0,"num_object_copies":12738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6834,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":873,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.084007+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:35:43.084007+0000","last_peered":"2023-09-23T23:35:43.084007+0000","last_clean":"2023-09-23T23:35:43.084007+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:35:43.084007+0000","last_undegraded":"2023-09-23T23:35:43.084007+0000","last_fullsized":"2023-09-23T23:35:43.084007+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":114,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.117172+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:35:43.117172+0000","last_peered":"2023-09-23T23:35:43.117172+0000","last_clean":"2023-09-23T23:35:43.117172+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:35:43.117172+0000","last_undegraded":"2023-09-23T23:35:43.117172+0000","last_fullsized":"2023-09-23T23:35:43.117172+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":455,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.082002+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:35:43.082002+0000","last_peered":"2023-09-23T23:35:43.082002+0000","last_clean":"2023-09-23T23:35:43.082002+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:35:43.082002+0000","last_undegraded":"2023-09-23T23:35:43.082002+0000","last_fullsized":"2023-09-23T23:35:43.082002+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"590'6882","reported_seq":9626,"reported_epoch":590,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:35:59.014208+0000","last_change":"2023-09-23T23:34:35.785270+0000","last_active":"2023-09-23T23:35:59.014208+0000","last_peered":"2023-09-23T23:35:59.014208+0000","last_clean":"2023-09-23T23:33:50.504296+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:35:59.014208+0000","last_undegraded":"2023-09-23T23:34:35.780981+0000","last_fullsized":"2023-09-23T23:35:59.014208+0000","mapping_epoch":567,"log_start":"548'1500","ondisk_log_start":"548'1500","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5382,"log_dups_size":0,"ondisk_log_size":5382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276692992,"num_objects":4222,"num_object_clones":0,"num_object_copies":12666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3217,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6882,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1330,"num_bytes_recovered":87162880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":3217},{"shards":"5,6,7","objects":1005}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":598,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.106092+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:35:43.106092+0000","last_peered":"2023-09-23T23:35:43.106092+0000","last_clean":"2023-09-23T23:35:43.106092+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:35:43.106092+0000","last_undegraded":"2023-09-23T23:35:43.106092+0000","last_fullsized":"2023-09-23T23:35:43.106092+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":452,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.082603+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:35:43.082603+0000","last_peered":"2023-09-23T23:35:43.082603+0000","last_clean":"2023-09-23T23:35:43.082603+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:35:43.082603+0000","last_undegraded":"2023-09-23T23:35:43.082603+0000","last_fullsized":"2023-09-23T23:35:43.082603+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":541,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.116474+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:35:43.116474+0000","last_peered":"2023-09-23T23:35:43.116474+0000","last_clean":"2023-09-23T23:35:43.116474+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:35:43.116474+0000","last_undegraded":"2023-09-23T23:35:43.116474+0000","last_fullsized":"2023-09-23T23:35:43.116474+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"590'6818","reported_seq":15845,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:59.144461+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:35:59.144461+0000","last_peered":"2023-09-23T23:35:59.144461+0000","last_clean":"2023-09-23T23:35:59.144461+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:35:59.144461+0000","last_undegraded":"2023-09-23T23:35:59.144461+0000","last_fullsized":"2023-09-23T23:35:59.144461+0000","mapping_epoch":579,"log_start":"544'1100","ondisk_log_start":"544'1100","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5718,"log_dups_size":0,"ondisk_log_size":5718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275120128,"num_objects":4198,"num_object_clones":0,"num_object_copies":12594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4198,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6818,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":931,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.083602+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:35:43.083602+0000","last_peered":"2023-09-23T23:35:43.083602+0000","last_clean":"2023-09-23T23:35:43.083602+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:35:43.083602+0000","last_undegraded":"2023-09-23T23:35:43.083602+0000","last_fullsized":"2023-09-23T23:35:43.083602+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":231,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.096491+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:35:43.096491+0000","last_peered":"2023-09-23T23:35:43.096491+0000","last_clean":"2023-09-23T23:35:43.096491+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:35:43.096491+0000","last_undegraded":"2023-09-23T23:35:43.096491+0000","last_fullsized":"2023-09-23T23:35:43.096491+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":519,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.098479+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:35:43.098479+0000","last_peered":"2023-09-23T23:35:43.098479+0000","last_clean":"2023-09-23T23:35:43.098479+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:35:43.098479+0000","last_undegraded":"2023-09-23T23:35:43.098479+0000","last_fullsized":"2023-09-23T23:35:43.098479+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"590'6528","reported_seq":6594,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:56.817752+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:35:56.817752+0000","last_peered":"2023-09-23T23:35:56.817752+0000","last_clean":"2023-09-23T23:35:56.817752+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:35:56.817752+0000","last_undegraded":"2023-09-23T23:35:56.817752+0000","last_fullsized":"2023-09-23T23:35:56.817752+0000","mapping_epoch":534,"log_start":"546'1200","ondisk_log_start":"546'1200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5328,"log_dups_size":0,"ondisk_log_size":5328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275251200,"num_objects":4200,"num_object_clones":0,"num_object_copies":12600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6528,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":911,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.117203+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:35:43.117203+0000","last_peered":"2023-09-23T23:35:43.117203+0000","last_clean":"2023-09-23T23:35:43.117203+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:35:43.117203+0000","last_undegraded":"2023-09-23T23:35:43.117203+0000","last_fullsized":"2023-09-23T23:35:43.117203+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":299,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.089272+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:35:43.089272+0000","last_peered":"2023-09-23T23:35:43.089272+0000","last_clean":"2023-09-23T23:35:43.089272+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:35:43.089272+0000","last_undegraded":"2023-09-23T23:35:43.089272+0000","last_fullsized":"2023-09-23T23:35:43.089272+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":605,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.106777+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:35:43.106777+0000","last_peered":"2023-09-23T23:35:43.106777+0000","last_clean":"2023-09-23T23:35:43.106777+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:35:43.106777+0000","last_undegraded":"2023-09-23T23:35:43.106777+0000","last_fullsized":"2023-09-23T23:35:43.106777+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":186,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.105887+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:35:43.105887+0000","last_peered":"2023-09-23T23:35:43.105887+0000","last_clean":"2023-09-23T23:35:43.105887+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:35:43.105887+0000","last_undegraded":"2023-09-23T23:35:43.105887+0000","last_fullsized":"2023-09-23T23:35:43.105887+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"590'6855","reported_seq":9494,"reported_epoch":590,"state":"active+recovering","last_fresh":"2023-09-23T23:35:57.215190+0000","last_change":"2023-09-23T23:34:35.815566+0000","last_active":"2023-09-23T23:35:57.215190+0000","last_peered":"2023-09-23T23:35:57.215190+0000","last_clean":"2023-09-23T23:33:50.519308+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:35:57.215190+0000","last_undegraded":"2023-09-23T23:35:57.215190+0000","last_fullsized":"2023-09-23T23:35:57.215190+0000","mapping_epoch":566,"log_start":"548'1522","ondisk_log_start":"548'1522","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5333,"log_dups_size":0,"ondisk_log_size":5333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282591256,"num_objects":4313,"num_object_clones":0,"num_object_copies":12939,"num_objects_missing_on_primary":3232,"num_objects_missing":3232,"num_objects_degraded":0,"num_objects_misplaced":3232,"num_objects_unfound":0,"num_objects_dirty":4313,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6855,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1271,"num_bytes_recovered":83296256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,3,4,7","objects":12},{"shards":"1,3,7","objects":1069},{"shards":"1,4,7","objects":3232}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":674,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.154500+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:35:50.154500+0000","last_peered":"2023-09-23T23:35:50.154500+0000","last_clean":"2023-09-23T23:35:50.154500+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:35:50.154500+0000","last_undegraded":"2023-09-23T23:35:50.154500+0000","last_fullsized":"2023-09-23T23:35:50.154500+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":238,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.117090+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:35:43.117090+0000","last_peered":"2023-09-23T23:35:43.117090+0000","last_clean":"2023-09-23T23:35:43.117090+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:35:43.117090+0000","last_undegraded":"2023-09-23T23:35:43.117090+0000","last_fullsized":"2023-09-23T23:35:43.117090+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":522,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.115808+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:35:43.115808+0000","last_peered":"2023-09-23T23:35:43.115808+0000","last_clean":"2023-09-23T23:35:43.115808+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:35:43.115808+0000","last_undegraded":"2023-09-23T23:35:43.115808+0000","last_fullsized":"2023-09-23T23:35:43.115808+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":222,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.083455+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:35:43.083455+0000","last_peered":"2023-09-23T23:35:43.083455+0000","last_clean":"2023-09-23T23:35:43.083455+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:35:43.083455+0000","last_undegraded":"2023-09-23T23:35:43.083455+0000","last_fullsized":"2023-09-23T23:35:43.083455+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"590'6591","reported_seq":8634,"reported_epoch":590,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:59.036417+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:35:59.036417+0000","last_peered":"2023-09-23T23:35:59.036417+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:35:59.036417+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"543'900","ondisk_log_start":"543'900","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5691,"log_dups_size":0,"ondisk_log_size":5691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274006016,"num_objects":4181,"num_object_clones":0,"num_object_copies":12543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6591,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":3170},{"shards":"5,7","objects":789}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":891,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.083510+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:35:43.083510+0000","last_peered":"2023-09-23T23:35:43.083510+0000","last_clean":"2023-09-23T23:35:43.083510+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:35:43.083510+0000","last_undegraded":"2023-09-23T23:35:43.083510+0000","last_fullsized":"2023-09-23T23:35:43.083510+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":328,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.082518+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:35:43.082518+0000","last_peered":"2023-09-23T23:35:43.082518+0000","last_clean":"2023-09-23T23:35:43.082518+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:35:43.082518+0000","last_undegraded":"2023-09-23T23:35:43.082518+0000","last_fullsized":"2023-09-23T23:35:43.082518+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":237,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.084089+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:35:43.084089+0000","last_peered":"2023-09-23T23:35:43.084089+0000","last_clean":"2023-09-23T23:35:43.084089+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:35:43.084089+0000","last_undegraded":"2023-09-23T23:35:43.084089+0000","last_fullsized":"2023-09-23T23:35:43.084089+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":201,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.089147+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:35:43.089147+0000","last_peered":"2023-09-23T23:35:43.089147+0000","last_clean":"2023-09-23T23:35:43.089147+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:35:43.089147+0000","last_undegraded":"2023-09-23T23:35:43.089147+0000","last_fullsized":"2023-09-23T23:35:43.089147+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"590'6726","reported_seq":6800,"reported_epoch":590,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:35:58.869978+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:35:58.869978+0000","last_peered":"2023-09-23T23:35:58.869978+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:35:58.869978+0000","last_undegraded":"2023-09-23T23:35:58.869978+0000","last_fullsized":"2023-09-23T23:35:58.869978+0000","mapping_epoch":562,"log_start":"544'1000","ondisk_log_start":"544'1000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5726,"log_dups_size":0,"ondisk_log_size":5726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273678336,"num_objects":4176,"num_object_clones":0,"num_object_copies":12528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6726,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":4176}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":605,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.122909+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:35:50.122909+0000","last_peered":"2023-09-23T23:35:50.122909+0000","last_clean":"2023-09-23T23:35:50.122909+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:35:50.122909+0000","last_undegraded":"2023-09-23T23:35:50.122909+0000","last_fullsized":"2023-09-23T23:35:50.122909+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":876,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.115452+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:35:43.115452+0000","last_peered":"2023-09-23T23:35:43.115452+0000","last_clean":"2023-09-23T23:35:43.115452+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:35:43.115452+0000","last_undegraded":"2023-09-23T23:35:43.115452+0000","last_fullsized":"2023-09-23T23:35:43.115452+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":552,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.131141+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:35:50.131141+0000","last_peered":"2023-09-23T23:35:50.131141+0000","last_clean":"2023-09-23T23:35:50.131141+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:35:50.131141+0000","last_undegraded":"2023-09-23T23:35:50.131141+0000","last_fullsized":"2023-09-23T23:35:50.131141+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":114,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.116376+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:35:43.116376+0000","last_peered":"2023-09-23T23:35:43.116376+0000","last_clean":"2023-09-23T23:35:43.116376+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:35:43.116376+0000","last_undegraded":"2023-09-23T23:35:43.116376+0000","last_fullsized":"2023-09-23T23:35:43.116376+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"590'6609","reported_seq":6673,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:58.798508+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:35:58.798508+0000","last_peered":"2023-09-23T23:35:58.798508+0000","last_clean":"2023-09-23T23:35:58.798508+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:35:58.798508+0000","last_undegraded":"2023-09-23T23:35:58.798508+0000","last_fullsized":"2023-09-23T23:35:58.798508+0000","mapping_epoch":534,"log_start":"544'1200","ondisk_log_start":"544'1200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5409,"log_dups_size":0,"ondisk_log_size":5409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270467072,"num_objects":4127,"num_object_clones":0,"num_object_copies":12381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6609,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":702,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.115674+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:35:43.115674+0000","last_peered":"2023-09-23T23:35:43.115674+0000","last_clean":"2023-09-23T23:35:43.115674+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:35:43.115674+0000","last_undegraded":"2023-09-23T23:35:43.115674+0000","last_fullsized":"2023-09-23T23:35:43.115674+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":517,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.094447+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:35:43.094447+0000","last_peered":"2023-09-23T23:35:43.094447+0000","last_clean":"2023-09-23T23:35:43.094447+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:35:43.094447+0000","last_undegraded":"2023-09-23T23:35:43.094447+0000","last_fullsized":"2023-09-23T23:35:43.094447+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":431,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.130532+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:35:50.130532+0000","last_peered":"2023-09-23T23:35:50.130532+0000","last_clean":"2023-09-23T23:35:50.130532+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:35:50.130532+0000","last_undegraded":"2023-09-23T23:35:50.130532+0000","last_fullsized":"2023-09-23T23:35:50.130532+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":31,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.155119+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:35:50.155119+0000","last_peered":"2023-09-23T23:35:50.155119+0000","last_clean":"2023-09-23T23:35:50.155119+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:35:50.155119+0000","last_undegraded":"2023-09-23T23:35:50.155119+0000","last_fullsized":"2023-09-23T23:35:50.155119+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"590'6750","reported_seq":10413,"reported_epoch":590,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:35:59.748158+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:35:59.748158+0000","last_peered":"2023-09-23T23:35:59.748158+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:35:59.748158+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"541'600","ondisk_log_start":"541'600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6150,"log_dups_size":0,"ondisk_log_size":6150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278396928,"num_objects":4248,"num_object_clones":0,"num_object_copies":12744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6750,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":1544},{"shards":"1,2,4","objects":2704}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":585,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.116742+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:35:43.116742+0000","last_peered":"2023-09-23T23:35:43.116742+0000","last_clean":"2023-09-23T23:35:43.116742+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:35:43.116742+0000","last_undegraded":"2023-09-23T23:35:43.116742+0000","last_fullsized":"2023-09-23T23:35:43.116742+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":31,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.154845+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:35:50.154845+0000","last_peered":"2023-09-23T23:35:50.154845+0000","last_clean":"2023-09-23T23:35:50.154845+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:35:50.154845+0000","last_undegraded":"2023-09-23T23:35:50.154845+0000","last_fullsized":"2023-09-23T23:35:50.154845+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":129,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.083530+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:35:43.083530+0000","last_peered":"2023-09-23T23:35:43.083530+0000","last_clean":"2023-09-23T23:35:43.083530+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:35:43.083530+0000","last_undegraded":"2023-09-23T23:35:43.083530+0000","last_fullsized":"2023-09-23T23:35:43.083530+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":323,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.106216+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:35:43.106216+0000","last_peered":"2023-09-23T23:35:43.106216+0000","last_clean":"2023-09-23T23:35:43.106216+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:35:43.106216+0000","last_undegraded":"2023-09-23T23:35:43.106216+0000","last_fullsized":"2023-09-23T23:35:43.106216+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"590'6774","reported_seq":6838,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:58.612837+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:35:58.612837+0000","last_peered":"2023-09-23T23:35:58.612837+0000","last_clean":"2023-09-23T23:35:58.612837+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:35:58.612837+0000","last_undegraded":"2023-09-23T23:35:58.612837+0000","last_fullsized":"2023-09-23T23:35:58.612837+0000","mapping_epoch":534,"log_start":"544'1100","ondisk_log_start":"544'1100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5674,"log_dups_size":0,"ondisk_log_size":5674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274595840,"num_objects":4190,"num_object_clones":0,"num_object_copies":12570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6774,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":771,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.122812+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:35:50.122812+0000","last_peered":"2023-09-23T23:35:50.122812+0000","last_clean":"2023-09-23T23:35:50.122812+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:35:50.122812+0000","last_undegraded":"2023-09-23T23:35:50.122812+0000","last_fullsized":"2023-09-23T23:35:50.122812+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":587,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.098444+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:35:43.098444+0000","last_peered":"2023-09-23T23:35:43.098444+0000","last_clean":"2023-09-23T23:35:43.098444+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:35:43.098444+0000","last_undegraded":"2023-09-23T23:35:43.098444+0000","last_fullsized":"2023-09-23T23:35:43.098444+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":194,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.116540+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:35:43.116540+0000","last_peered":"2023-09-23T23:35:43.116540+0000","last_clean":"2023-09-23T23:35:43.116540+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:35:43.116540+0000","last_undegraded":"2023-09-23T23:35:43.116540+0000","last_fullsized":"2023-09-23T23:35:43.116540+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":141,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.084290+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:35:43.084290+0000","last_peered":"2023-09-23T23:35:43.084290+0000","last_clean":"2023-09-23T23:35:43.084290+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:35:43.084290+0000","last_undegraded":"2023-09-23T23:35:43.084290+0000","last_fullsized":"2023-09-23T23:35:43.084290+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"590'6684","reported_seq":6750,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:59.634873+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:35:59.634873+0000","last_peered":"2023-09-23T23:35:59.634873+0000","last_clean":"2023-09-23T23:35:59.634873+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:35:59.634873+0000","last_undegraded":"2023-09-23T23:35:59.634873+0000","last_fullsized":"2023-09-23T23:35:59.634873+0000","mapping_epoch":534,"log_start":"538'500","ondisk_log_start":"538'500","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6184,"log_dups_size":0,"ondisk_log_size":6184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276824064,"num_objects":4224,"num_object_clones":0,"num_object_copies":12672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6684,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":977,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.097733+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:35:43.097733+0000","last_peered":"2023-09-23T23:35:43.097733+0000","last_clean":"2023-09-23T23:35:43.097733+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:35:43.097733+0000","last_undegraded":"2023-09-23T23:35:43.097733+0000","last_fullsized":"2023-09-23T23:35:43.097733+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":104,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.123738+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:35:43.123738+0000","last_peered":"2023-09-23T23:35:43.123738+0000","last_clean":"2023-09-23T23:35:43.123738+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:35:43.123738+0000","last_undegraded":"2023-09-23T23:35:43.123738+0000","last_fullsized":"2023-09-23T23:35:43.123738+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":35,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.155193+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:35:50.155193+0000","last_peered":"2023-09-23T23:35:50.155193+0000","last_clean":"2023-09-23T23:35:50.155193+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:35:50.155193+0000","last_undegraded":"2023-09-23T23:35:50.155193+0000","last_fullsized":"2023-09-23T23:35:50.155193+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":164,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.105737+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:35:43.105737+0000","last_peered":"2023-09-23T23:35:43.105737+0000","last_clean":"2023-09-23T23:35:43.105737+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:35:43.105737+0000","last_undegraded":"2023-09-23T23:35:43.105737+0000","last_fullsized":"2023-09-23T23:35:43.105737+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1026,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.096912+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:35:43.096912+0000","last_peered":"2023-09-23T23:35:43.096912+0000","last_clean":"2023-09-23T23:35:43.096912+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:35:43.096912+0000","last_undegraded":"2023-09-23T23:35:43.096912+0000","last_fullsized":"2023-09-23T23:35:43.096912+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":31,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.155829+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:35:50.155829+0000","last_peered":"2023-09-23T23:35:50.155829+0000","last_clean":"2023-09-23T23:35:50.155829+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:35:50.155829+0000","last_undegraded":"2023-09-23T23:35:50.155829+0000","last_fullsized":"2023-09-23T23:35:50.155829+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":345,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.083604+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:35:43.083604+0000","last_peered":"2023-09-23T23:35:43.083604+0000","last_clean":"2023-09-23T23:35:43.083604+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:35:43.083604+0000","last_undegraded":"2023-09-23T23:35:43.083604+0000","last_fullsized":"2023-09-23T23:35:43.083604+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":738,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.132543+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:35:50.132543+0000","last_peered":"2023-09-23T23:35:50.132543+0000","last_clean":"2023-09-23T23:35:50.132543+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:35:50.132543+0000","last_undegraded":"2023-09-23T23:35:50.132543+0000","last_fullsized":"2023-09-23T23:35:50.132543+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":92,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.116715+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:35:43.116715+0000","last_peered":"2023-09-23T23:35:43.116715+0000","last_clean":"2023-09-23T23:35:43.116715+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:35:43.116715+0000","last_undegraded":"2023-09-23T23:35:43.116715+0000","last_fullsized":"2023-09-23T23:35:43.116715+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":437,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.098726+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:35:43.098726+0000","last_peered":"2023-09-23T23:35:43.098726+0000","last_clean":"2023-09-23T23:35:43.098726+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:35:43.098726+0000","last_undegraded":"2023-09-23T23:35:43.098726+0000","last_fullsized":"2023-09-23T23:35:43.098726+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":288,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.116229+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:35:43.116229+0000","last_peered":"2023-09-23T23:35:43.116229+0000","last_clean":"2023-09-23T23:35:43.116229+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:35:43.116229+0000","last_undegraded":"2023-09-23T23:35:43.116229+0000","last_fullsized":"2023-09-23T23:35:43.116229+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":276,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.101761+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:35:43.101761+0000","last_peered":"2023-09-23T23:35:43.101761+0000","last_clean":"2023-09-23T23:35:43.101761+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:35:43.101761+0000","last_undegraded":"2023-09-23T23:35:43.101761+0000","last_fullsized":"2023-09-23T23:35:43.101761+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":802,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.116951+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:35:43.116951+0000","last_peered":"2023-09-23T23:35:43.116951+0000","last_clean":"2023-09-23T23:35:43.116951+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:35:43.116951+0000","last_undegraded":"2023-09-23T23:35:43.116951+0000","last_fullsized":"2023-09-23T23:35:43.116951+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":620,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.098323+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:35:43.098323+0000","last_peered":"2023-09-23T23:35:43.098323+0000","last_clean":"2023-09-23T23:35:43.098323+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:35:43.098323+0000","last_undegraded":"2023-09-23T23:35:43.098323+0000","last_fullsized":"2023-09-23T23:35:43.098323+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":604,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.097172+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:35:43.097172+0000","last_peered":"2023-09-23T23:35:43.097172+0000","last_clean":"2023-09-23T23:35:43.097172+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:35:43.097172+0000","last_undegraded":"2023-09-23T23:35:43.097172+0000","last_fullsized":"2023-09-23T23:35:43.097172+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":762,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.102719+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:35:43.102719+0000","last_peered":"2023-09-23T23:35:43.102719+0000","last_clean":"2023-09-23T23:35:43.102719+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:35:43.102719+0000","last_undegraded":"2023-09-23T23:35:43.102719+0000","last_fullsized":"2023-09-23T23:35:43.102719+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":433,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.083098+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:35:43.083098+0000","last_peered":"2023-09-23T23:35:43.083098+0000","last_clean":"2023-09-23T23:35:43.083098+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:35:43.083098+0000","last_undegraded":"2023-09-23T23:35:43.083098+0000","last_fullsized":"2023-09-23T23:35:43.083098+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":657,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-09-23T23:35:43.115355+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:35:43.115355+0000","last_peered":"2023-09-23T23:35:43.115355+0000","last_clean":"2023-09-23T23:35:43.115355+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:35:43.115355+0000","last_undegraded":"2023-09-23T23:35:43.115355+0000","last_fullsized":"2023-09-23T23:35:43.115355+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":4429840408,"num_objects":67595,"num_object_clones":0,"num_object_copies":202785,"num_objects_missing_on_primary":3232,"num_objects_missing":3232,"num_objects_degraded":26164,"num_objects_misplaced":4127,"num_objects_unfound":0,"num_objects_dirty":67595,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":107961,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13249,"num_bytes_recovered":868089856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14153822208,"data_stored":14153809992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89939,"ondisk_log_size":89939,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782678,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1954680,"kb_used_data":1590960,"kb_used_omap":8,"kb_used_meta":363703,"kb_avail":91790472,"statfs":{"total":95995035648,"available":93993443328,"internally_reserved":0,"allocated":1629143040,"data_stored":1627507968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":372431941},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":1.5569999999999999,"15min":2.1800000000000002},"min":{"1min":0.23100000000000001,"5min":0.224,"15min":0.20899999999999999},"max":{"1min":0.61699999999999999,"5min":36.956000000000003,"15min":85.721000000000004},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.47399999999999998,"5min":1.522,"15min":2.2250000000000001},"min":{"1min":0.29199999999999998,"5min":0.27900000000000003,"15min":0.19800000000000001},"max":{"1min":1.4419999999999999,"5min":37.247999999999998,"15min":85.965000000000003},"last":1.4419999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":3.8900000000000001,"15min":3.169},"min":{"1min":0.26600000000000001,"5min":0.248,"15min":0.22900000000000001},"max":{"1min":3.3500000000000001,"5min":129.11199999999999,"15min":198.58000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.65600000000000003,"5min":3.665,"15min":3.206},"min":{"1min":0.29699999999999999,"5min":0.247,"15min":0.246},"max":{"1min":3.7290000000000001,"5min":129.02600000000001,"15min":198.71100000000001},"last":1.403}]},{"osd":2,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":2.871,"15min":3.7010000000000001},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.452,"5min":78.528999999999996,"15min":203.58000000000001},"last":1.4790000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":1.952,"15min":3.0649999999999999},"min":{"1min":0.30399999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.2529999999999999,"5min":27.574000000000002,"15min":198.69},"last":0.92100000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":3.3290000000000002,"15min":3.5950000000000002},"min":{"1min":0.31900000000000001,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":1.591,"5min":125.511,"15min":279.26499999999999},"last":1.0720000000000001},{"interface":"front","average":{"1min":0.67500000000000004,"5min":5.3090000000000002,"15min":4.1210000000000004},"min":{"1min":0.374,"5min":0.32000000000000001,"15min":0.29599999999999999},"max":{"1min":3.6240000000000001,"5min":150.608,"15min":279.29899999999998},"last":0.48099999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":1.694,"15min":1.0669999999999999},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":0.68100000000000005,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.363,"15min":1.0029999999999999},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.23899999999999999},"max":{"1min":1.6080000000000001,"5min":31.68,"15min":31.68},"last":1.6080000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":3.637,"5min":1.657,"15min":1.899},"min":{"1min":0.31900000000000001,"5min":0.28699999999999998,"15min":0.26500000000000001},"max":{"1min":60.463999999999999,"5min":60.463999999999999,"15min":210.83600000000001},"last":1.6399999999999999},{"interface":"front","average":{"1min":3.5990000000000002,"5min":1.6160000000000001,"15min":1.7649999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28199999999999997},"max":{"1min":63.006,"5min":63.006,"15min":207.04599999999999},"last":0.377}]},{"osd":6,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":1.879,"15min":1.3480000000000001},"min":{"1min":0.35199999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.4419999999999999,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.441},{"interface":"front","average":{"1min":0.59299999999999997,"5min":2.2330000000000001,"15min":1.581},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":2.3180000000000001,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.54900000000000004}]}]},{"osd":6,"up_from":46,"seq":197568496155,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2190376,"kb_used_data":1482720,"kb_used_omap":0,"kb_used_meta":707648,"kb_avail":91554776,"statfs":{"total":95995035648,"available":93752090624,"internally_reserved":0,"allocated":1518305280,"data_stored":1516686201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":724631552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:14 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.98799999999999999,"15min":4.7699999999999996},"min":{"1min":0.26500000000000001,"5min":0.223,"15min":0.20399999999999999},"max":{"1min":2.274,"5min":8.8119999999999994,"15min":135.31999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.93000000000000005,"15min":4.617},"min":{"1min":0.26800000000000002,"5min":0.23999999999999999,"15min":0.22900000000000001},"max":{"1min":3.8239999999999998,"5min":8.3230000000000004,"15min":135.34299999999999},"last":0.46200000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:35:14 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":3.3690000000000002,"15min":2.4089999999999998},"min":{"1min":0.26000000000000001,"5min":0.251,"15min":0.245},"max":{"1min":6.9080000000000004,"5min":89.605000000000004,"15min":89.605000000000004},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.71599999999999997,"5min":3.1600000000000001,"15min":2.4849999999999999},"min":{"1min":0.33500000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":1.9159999999999999,"5min":91.025999999999996,"15min":91.025999999999996},"last":0.376}]},{"osd":2,"last update":"Sat Sep 23 23:35:03 2023","interfaces":[{"interface":"back","average":{"1min":9.4120000000000008,"5min":6.4420000000000002,"15min":4.6150000000000002},"min":{"1min":0.315,"5min":0.28399999999999997,"15min":0.27800000000000002},"max":{"1min":67.796999999999997,"5min":97.408000000000001,"15min":127.44499999999999},"last":0.433},{"interface":"front","average":{"1min":8.6270000000000007,"5min":7.6139999999999999,"15min":5.0419999999999998},"min":{"1min":0.30599999999999999,"5min":0.29999999999999999,"15min":0.26500000000000001},"max":{"1min":67.893000000000001,"5min":125.539,"15min":125.539},"last":0.40999999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:35:53 2023","interfaces":[{"interface":"back","average":{"1min":7.923,"5min":7.0129999999999999,"15min":4.4459999999999997},"min":{"1min":0.316,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":73.813999999999993,"5min":138.87299999999999,"15min":138.87299999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":8.4450000000000003,"5min":6.9260000000000002,"15min":4.7350000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":73.852999999999994,"5min":138.976,"15min":138.976},"last":0.47099999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.82999999999999996,"15min":1.274},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":1.633,"5min":7.101,"15min":166.05199999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":1.0289999999999999,"15min":1.4890000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.089,"5min":8.3499999999999996,"15min":166.631},"last":0.38800000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.96699999999999997,"15min":0.82399999999999995},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.1579999999999999,"5min":11.473000000000001,"15min":11.473000000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.433,"5min":1.091,"15min":0.94099999999999995},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.249},"max":{"1min":0.76700000000000002,"5min":7.5670000000000002,"15min":8.4090000000000007},"last":0.28699999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:35:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.5389999999999999,"15min":1.0580000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":2.3010000000000002,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.314},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.573,"15min":1.081},"min":{"1min":0.36199999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.9009999999999998,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.40000000000000002}]}]},{"osd":1,"up_from":112,"seq":481036337496,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2244960,"kb_used_data":1362264,"kb_used_omap":4,"kb_used_meta":882683,"kb_avail":91500192,"statfs":{"total":95995035648,"available":93696196608,"internally_reserved":0,"allocated":1394958336,"data_stored":1392178333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":903868302},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":2.8220000000000001,"15min":3.5670000000000002},"min":{"1min":0.21199999999999999,"5min":0.17799999999999999,"15min":0.16700000000000001},"max":{"1min":15.983000000000001,"5min":47.195999999999998,"15min":130.768},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.883,"15min":2.8650000000000002},"min":{"1min":0.20699999999999999,"5min":0.20000000000000001,"15min":0.19},"max":{"1min":4.9809999999999999,"5min":46.582000000000001,"15min":134.929},"last":0.26000000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:34:57 2023","interfaces":[{"interface":"back","average":{"1min":10.555,"5min":7.9610000000000003,"15min":6.7919999999999998},"min":{"1min":0.23000000000000001,"5min":0.214,"15min":0.20100000000000001},"max":{"1min":201.03999999999999,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.34000000000000002},{"interface":"front","average":{"1min":10.605,"5min":8.173,"15min":6.9109999999999996},"min":{"1min":0.24299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":202.12,"5min":202.12,"15min":232.21000000000001},"last":0.29399999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:34:57 2023","interfaces":[{"interface":"back","average":{"1min":9.6969999999999992,"5min":5.7679999999999998,"15min":5.1859999999999999},"min":{"1min":0.311,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":163.94300000000001,"5min":181.94399999999999,"15min":234.404},"last":0.36399999999999999},{"interface":"front","average":{"1min":9.9290000000000003,"5min":5.6550000000000002,"15min":5.0430000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.24399999999999999},"max":{"1min":163.98699999999999,"5min":181.98500000000001,"15min":236.33699999999999},"last":0.32000000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":1.5569999999999999,"15min":1.4339999999999999},"min":{"1min":0.32700000000000001,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":2.8210000000000002,"5min":12.042999999999999,"15min":12.042999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.5609999999999999,"15min":1.498},"min":{"1min":0.34499999999999997,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":2.7810000000000001,"5min":18.108000000000001,"15min":18.108000000000001},"last":0.621}]},{"osd":5,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":3.2879999999999998,"15min":1.8260000000000001},"min":{"1min":0.307,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.0460000000000003,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.91300000000000003,"5min":3.2890000000000001,"15min":2.141},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.7450000000000001,"5min":151.821,"15min":151.821},"last":0.5}]},{"osd":6,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.6850000000000001,"15min":1.954},"min":{"1min":0.34699999999999998,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":5.0960000000000001,"5min":8.9269999999999996,"15min":74.087999999999994},"last":0.74099999999999999},{"interface":"front","average":{"1min":1.3300000000000001,"5min":1.885,"15min":2.157},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.32800000000000001},"max":{"1min":6.8920000000000003,"5min":11.927,"15min":73.974999999999994},"last":0.56999999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:34:57 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":2.1269999999999998,"15min":2.875},"min":{"1min":0.38700000000000001,"5min":0.37,"15min":0.32900000000000001},"max":{"1min":10.629,"5min":44.241,"15min":78.308000000000007},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.165,"5min":2.218,"15min":2.706},"min":{"1min":0.42899999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.9880000000000004,"5min":43.920999999999999,"15min":77.204999999999998},"last":0.65100000000000002}]}]},{"osd":0,"up_from":91,"seq":390842024320,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1541868,"kb_used_data":339812,"kb_used_omap":5,"kb_used_meta":1202042,"kb_avail":92203284,"statfs":{"total":95995035648,"available":94416162816,"internally_reserved":0,"allocated":347967488,"data_stored":345170272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1230891642},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:35:32 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":10.577,"15min":6.7670000000000003},"min":{"1min":0.20499999999999999,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":9.0310000000000006,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.754,"5min":10.407999999999999,"15min":5.9059999999999997},"min":{"1min":0.20200000000000001,"5min":0.19700000000000001,"15min":0.17599999999999999},"max":{"1min":5.4729999999999999,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.29099999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:35:57 2023","interfaces":[{"interface":"back","average":{"1min":3.2130000000000001,"5min":10.026999999999999,"15min":7.5819999999999999},"min":{"1min":0.26200000000000001,"5min":0.17699999999999999,"15min":0.17199999999999999},"max":{"1min":40.326999999999998,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.307},{"interface":"front","average":{"1min":3.1720000000000002,"5min":10.18,"15min":7.6520000000000001},"min":{"1min":0.22900000000000001,"5min":0.19800000000000001,"15min":0.17599999999999999},"max":{"1min":40.396000000000001,"5min":283.161,"15min":283.161},"last":0.32400000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:35:32 2023","interfaces":[{"interface":"back","average":{"1min":2.7360000000000002,"5min":8.6509999999999998,"15min":5.4589999999999996},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":41.191000000000003,"5min":220.732,"15min":220.732},"last":0.35899999999999999},{"interface":"front","average":{"1min":2.9390000000000001,"5min":9.0440000000000005,"15min":5.5570000000000004},"min":{"1min":0.28799999999999998,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":46.688000000000002,"5min":218.78200000000001,"15min":218.78200000000001},"last":0.55300000000000005}]},{"osd":4,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":2.5859999999999999,"15min":2.8660000000000001},"min":{"1min":0.33300000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.4500000000000002,"5min":108.01900000000001,"15min":138.81299999999999},"last":0.626},{"interface":"front","average":{"1min":1.25,"5min":2.7490000000000001,"15min":3.4550000000000001},"min":{"1min":0.35499999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.3670000000000009,"5min":107.898,"15min":171.251},"last":0.502}]},{"osd":5,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":1.7310000000000001,"15min":2.8340000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32100000000000001},"max":{"1min":15.076000000000001,"5min":15.076000000000001,"15min":171.30699999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.5649999999999999,"5min":1.4790000000000001,"15min":2.0880000000000001},"min":{"1min":0.35499999999999998,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":6.5279999999999996,"5min":9.9000000000000004,"15min":139.40799999999999},"last":0.69799999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":2.6360000000000001,"15min":3.161},"min":{"1min":0.38,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":9.968,"5min":108.67,"15min":171.33600000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.915,"5min":2.6699999999999999,"15min":3.4209999999999998},"min":{"1min":0.38900000000000001,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":9.157,"5min":108.804,"15min":184.13499999999999},"last":0.66800000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":1.373,"15min":2.6099999999999999},"min":{"1min":0.38400000000000001,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":0.751,"5min":9.452,"15min":138.77699999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.6379999999999999,"15min":2.5030000000000001},"min":{"1min":0.379,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":1.839,"5min":9.9670000000000005,"15min":139.37899999999999},"last":1.4159999999999999}]}]},{"osd":2,"up_from":199,"seq":854698492141,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3542888,"kb_used_data":2694880,"kb_used_omap":7,"kb_used_meta":847992,"kb_avail":90202264,"statfs":{"total":95995035648,"available":92367118336,"internally_reserved":0,"allocated":2759557120,"data_stored":2756713572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":868343889},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":2.2080000000000002,"15min":3.8599999999999999},"min":{"1min":0.186,"5min":0.186,"15min":0.16700000000000001},"max":{"1min":4.7590000000000003,"5min":61.950000000000003,"15min":119.277},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":2.153,"15min":3.6669999999999998},"min":{"1min":0.23999999999999999,"5min":0.20399999999999999,"15min":0.18099999999999999},"max":{"1min":2.5070000000000001,"5min":61.984999999999999,"15min":119.56100000000001},"last":0.35699999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.911,"5min":5.3010000000000002,"15min":4.5940000000000003},"min":{"1min":0.25,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":8.3640000000000008,"5min":106.765,"15min":157.98699999999999},"last":0.94399999999999995},{"interface":"front","average":{"1min":0.79000000000000004,"5min":4.8959999999999999,"15min":4.577},"min":{"1min":0.253,"5min":0.222,"15min":0.222},"max":{"1min":4.0940000000000003,"5min":106.732,"15min":158.03800000000001},"last":0.253}]},{"osd":3,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":5.7800000000000002,"15min":4.7770000000000001},"min":{"1min":0.255,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":8.3070000000000004,"5min":106.70399999999999,"15min":119.366},"last":3.3580000000000001},{"interface":"front","average":{"1min":0.65800000000000003,"5min":5.4169999999999998,"15min":4.2709999999999999},"min":{"1min":0.29299999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":3.375,"5min":106.66800000000001,"15min":106.66800000000001},"last":3.375}]},{"osd":4,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":2.036,"15min":2.262},"min":{"1min":0.36099999999999999,"5min":0.33700000000000002,"15min":0.25},"max":{"1min":2.6070000000000002,"5min":62.081000000000003,"15min":119.88200000000001},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.4590000000000001,"5min":2.3860000000000001,"15min":1.865},"min":{"1min":0.36899999999999999,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":8.343,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.99199999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":3.0299999999999998,"15min":2.6480000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.26200000000000001},"max":{"1min":2.548,"5min":91.207999999999998,"15min":119.919},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.4550000000000001,"5min":3.2749999999999999,"15min":2.6160000000000001},"min":{"1min":0.33500000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.3889999999999993,"5min":91.150999999999996,"15min":119.306},"last":1.0249999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":3.5750000000000002,"15min":3.0019999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":2.5819999999999999,"5min":107.018,"15min":119.943},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.486,"5min":2.6440000000000001,"15min":2.1600000000000001},"min":{"1min":0.371,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":8.4060000000000006,"5min":62.493000000000002,"15min":62.493000000000002},"last":1.054}]},{"osd":7,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":2.2010000000000001,"15min":1.9179999999999999},"min":{"1min":0.40000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.381,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.60099999999999998,"5min":3.464,"15min":2.3620000000000001},"min":{"1min":0.42699999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.417,"5min":106.809,"15min":106.809},"last":0.50800000000000001}]}]},{"osd":3,"up_from":148,"seq":635655160109,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2332056,"kb_used_data":1438608,"kb_used_omap":2,"kb_used_meta":893437,"kb_avail":91413096,"statfs":{"total":95995035648,"available":93607010304,"internally_reserved":0,"allocated":1473134592,"data_stored":1470359925,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":914879541},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,0,1,0,1,5,1,9],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7390000000000001,"5min":1.4059999999999999,"15min":2.79},"min":{"1min":0.19600000000000001,"5min":0.19400000000000001,"15min":0.158},"max":{"1min":15.603,"5min":17.213000000000001,"15min":103.866},"last":0.25900000000000001},{"interface":"front","average":{"1min":1.5880000000000001,"5min":1.877,"15min":2.9049999999999998},"min":{"1min":0.223,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":12.202,"5min":22.754000000000001,"15min":101.884},"last":0.44600000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":3.2200000000000002,"15min":2.9969999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19600000000000001},"max":{"1min":7.3319999999999999,"5min":56.119,"15min":85.242999999999995},"last":0.20499999999999999},{"interface":"front","average":{"1min":1.4910000000000001,"5min":2.427,"15min":2.7999999999999998},"min":{"1min":0.22800000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":11.228,"5min":53.962000000000003,"15min":85.302000000000007},"last":0.28499999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":4.7770000000000001,"15min":4.7859999999999996},"min":{"1min":0.25800000000000001,"5min":0.25,"15min":0.19800000000000001},"max":{"1min":2.1190000000000002,"5min":73.742000000000004,"15min":101.279},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.45000000000000001,"5min":4.6349999999999998,"15min":4.6130000000000004},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.214},"max":{"1min":0.86699999999999999,"5min":76.525999999999996,"15min":101.294},"last":0.32700000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.3959999999999999,"15min":1.339},"min":{"1min":0.28799999999999998,"5min":0.28499999999999998,"15min":0.26700000000000002},"max":{"1min":11.209,"5min":11.209,"15min":11.209},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.2829999999999999,"5min":1.905,"15min":1.8049999999999999},"min":{"1min":0.33500000000000002,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":7.3940000000000001,"5min":11.494999999999999,"15min":24.782},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4410000000000001,"5min":6.8159999999999998,"15min":3.4289999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":7.3120000000000003,"5min":214.43700000000001,"15min":214.43700000000001},"last":0.622},{"interface":"front","average":{"1min":1.2470000000000001,"5min":6.7569999999999997,"15min":3.4900000000000002},"min":{"1min":0.34699999999999998,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":7.2290000000000001,"5min":214.46600000000001,"15min":214.46600000000001},"last":0.47299999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":2.7090000000000001,"15min":2.3159999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":9.798,"5min":58.932000000000002,"15min":58.932000000000002},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.25,"5min":2.8660000000000001,"15min":2.3170000000000002},"min":{"1min":0.36499999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.202,"5min":59.186,"15min":59.186},"last":0.52100000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":4.7480000000000002,"15min":3.2069999999999999},"min":{"1min":0.38700000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.415,"5min":192.69,"15min":192.69},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.753,"5min":5.1289999999999996,"15min":3.6949999999999998},"min":{"1min":0.43099999999999999,"5min":0.39200000000000002,"15min":0.36699999999999999},"max":{"1min":2.6560000000000001,"5min":192.65600000000001,"15min":192.65600000000001},"last":0.59599999999999997}]}]},{"osd":4,"up_from":0,"seq":128849019428,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757791,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3217444,"kb_used_data":2500700,"kb_used_omap":0,"kb_used_meta":716736,"kb_avail":90527708,"statfs":{"total":95995035648,"available":92700372992,"internally_reserved":0,"allocated":2560716800,"data_stored":2559097938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":733937664},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.3720000000000001,"15min":3.48},"min":{"1min":0.254,"5min":0.20899999999999999,"15min":0.19500000000000001},"max":{"1min":3.0089999999999999,"5min":21.109000000000002,"15min":113.869},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.6699999999999999,"15min":3.8650000000000002},"min":{"1min":0.25,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":5.1280000000000001,"5min":20.876999999999999,"15min":138.55199999999999},"last":0.26500000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.772,"15min":2.323},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.23100000000000001},"max":{"1min":3.629,"5min":29.007999999999999,"15min":69.816999999999993},"last":0.44},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.9390000000000001,"15min":2.665},"min":{"1min":0.29999999999999999,"5min":0.249,"15min":0.249},"max":{"1min":4.4859999999999998,"5min":29.053999999999998,"15min":70.268000000000001},"last":0.35499999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:35:47 2023","interfaces":[{"interface":"back","average":{"1min":7.7469999999999999,"5min":6.5350000000000001,"15min":4.609},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.26900000000000002},"max":{"1min":145.93199999999999,"5min":208.738,"15min":208.738},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.0269999999999999,"5min":5.0499999999999998,"15min":4.1150000000000002},"min":{"1min":0.31,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":2.859,"5min":208.631,"15min":208.631},"last":0.39400000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:35:44 2023","interfaces":[{"interface":"back","average":{"1min":6.4889999999999999,"5min":4.1520000000000001,"15min":3.6589999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":122.071,"5min":122.071,"15min":122.071},"last":0.36599999999999999},{"interface":"front","average":{"1min":6.4669999999999996,"5min":4.4969999999999999,"15min":3.8359999999999999},"min":{"1min":0.35799999999999998,"5min":0.35399999999999998,"15min":0.318},"max":{"1min":122.09699999999999,"5min":122.09699999999999,"15min":122.09699999999999},"last":0.376}]},{"osd":4,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.151,"15min":0.92000000000000004},"min":{"1min":0.34200000000000003,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":3.5499999999999998,"5min":9.2690000000000001,"15min":19.962},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.093,"15min":0.872},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.22900000000000001},"max":{"1min":2.4990000000000001,"5min":10.273999999999999,"15min":20.829999999999998},"last":0.314}]},{"osd":6,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":7.2519999999999998,"5min":2.3799999999999999,"15min":1.635},"min":{"1min":0.32400000000000001,"5min":0.27600000000000002,"15min":0.26900000000000002},"max":{"1min":148.36600000000001,"5min":148.36600000000001,"15min":148.36600000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":7.7210000000000001,"5min":2.9420000000000002,"15min":1.869},"min":{"1min":0.34599999999999997,"5min":0.307,"15min":0.26000000000000001},"max":{"1min":148.42500000000001,"5min":148.42500000000001,"15min":148.42500000000001},"last":0.30399999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:35:47 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.3160000000000001,"15min":2.625},"min":{"1min":0.34699999999999998,"5min":0.34599999999999997,"15min":0.317},"max":{"1min":1.843,"5min":15.497,"15min":179.28800000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.3480000000000001,"15min":2.7050000000000001},"min":{"1min":0.36699999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.867,"5min":15.653,"15min":179.428},"last":0.42899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":339542016,"data_stored":339542016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1386811392,"data_stored":1386807320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2751332352,"data_stored":2751332352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1464537088,"data_stored":1464533016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2526478336,"data_stored":2526478336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2553020416,"data_stored":2553020416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1511456768,"data_stored":1511456768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1620643840,"data_stored":1620639768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:36:04.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:04.399 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:04.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:04 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 26164/202857 objects degraded (12.898%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:04.500 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:04.527 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:04.574 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:04.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:04.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:04.600 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:04.681 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:36:04.710 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:04.756 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:36:04.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:04.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:36:04.861 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:36:04.887 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:04.935 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:36:04.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:04.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:36:05.035 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:36:05.036 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:36:05.041 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:05.068 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:05.124 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:05.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:05.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:05.229 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:05.271 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:05.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:05 smithi138 ceph-mon[193606]: pgmap v1637: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 201 op/s; 26119/202188 objects degraded (12.918%); 4055/202188 objects misplaced (2.006%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:36:05.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:05 smithi110 ceph-mon[212939]: pgmap v1637: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 201 op/s; 26119/202188 objects degraded (12.918%); 4055/202188 objects misplaced (2.006%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:36:05.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:05 smithi110 ceph-mon[221102]: pgmap v1637: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 201 op/s; 26119/202188 objects degraded (12.918%); 4055/202188 objects misplaced (2.006%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:36:06.342 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:36:06.385 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:06.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:06.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:06.466 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:36:06.491 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:36:06.535 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:06.580 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:36:06.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:06.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:36:06.685 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:06.717 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:06.826 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:36:06.963 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:06.983 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:06.986 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:36:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:36:06] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.43.0" 2023-09-23T23:36:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:06.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:07.089 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:07.129 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:07.194 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:07.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:07.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:07.238 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:07 smithi110 ceph-mon[221102]: pgmap v1638: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 26060/199764 objects degraded (13.045%); 4055/199764 objects misplaced (2.030%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:36:07.239 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:07 smithi110 ceph-mon[212939]: pgmap v1638: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 26060/199764 objects degraded (13.045%); 4055/199764 objects misplaced (2.030%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:36:07.301 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:07.332 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:07.399 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:36:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:07 smithi138 ceph-mon[193606]: pgmap v1638: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 26060/199764 objects degraded (13.045%); 4055/199764 objects misplaced (2.030%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T23:36:07.605 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:07.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:07.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:07.710 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:07.739 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:08.129 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:08.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:08.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:08.160 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:36:08.233 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:36:08.267 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:08.325 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:36:08.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:08.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:36:08.429 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:08.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:08 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/668584809' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:36:08.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:08 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:36:08.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:08 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/668584809' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:36:08.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:08 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:36:08.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:08 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/668584809' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:36:08.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:08 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:36:08.457 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:08.508 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:08.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:08.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:08.612 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:08.644 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:08.808 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:08.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:08.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:08.911 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:08.945 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:09.155 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:09.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:09.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:09.198 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:09.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[212939]: pgmap v1639: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 25969/198861 objects degraded (13.059%); 3989/198861 objects misplaced (2.006%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:36:09.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:36:09.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:36:09.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[212939]: osdmap e592: 8 total, 8 up, 7 in 2023-09-23T23:36:09.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[212939]: failed to encode map e592 with expected crc 2023-09-23T23:36:09.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[212939]: failed to encode map e592 with expected crc 2023-09-23T23:36:09.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[212939]: failed to encode map e592 with expected crc 2023-09-23T23:36:09.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[212939]: failed to encode map e592 with expected crc 2023-09-23T23:36:09.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 25969/198861 objects degraded (13.059%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:09.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[221102]: pgmap v1639: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 25969/198861 objects degraded (13.059%); 3989/198861 objects misplaced (2.006%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:36:09.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:36:09.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:36:09.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[221102]: osdmap e592: 8 total, 8 up, 7 in 2023-09-23T23:36:09.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[221102]: failed to encode map e592 with expected crc 2023-09-23T23:36:09.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[221102]: failed to encode map e592 with expected crc 2023-09-23T23:36:09.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[221102]: failed to encode map e592 with expected crc 2023-09-23T23:36:09.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[221102]: failed to encode map e592 with expected crc 2023-09-23T23:36:09.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:09 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 25969/198861 objects degraded (13.059%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:09.242 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:36:09.262 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:09.273 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:36:09.303 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:36:09.311 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:09.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:09 smithi138 ceph-mon[193606]: pgmap v1639: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 25969/198861 objects degraded (13.059%); 3989/198861 objects misplaced (2.006%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:36:09.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:09 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:36:09.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:09 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:36:09.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:09 smithi138 ceph-mon[193606]: osdmap e592: 8 total, 8 up, 7 in 2023-09-23T23:36:09.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:09 smithi138 ceph-mon[193606]: failed to encode map e592 with expected crc 2023-09-23T23:36:09.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:09 smithi138 ceph-mon[193606]: failed to encode map e592 with expected crc 2023-09-23T23:36:09.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:09 smithi138 ceph-mon[193606]: failed to encode map e592 with expected crc 2023-09-23T23:36:09.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:09 smithi138 ceph-mon[193606]: failed to encode map e592 with expected crc 2023-09-23T23:36:09.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:09 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 25969/198861 objects degraded (13.059%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:09.332 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:36:09.359 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:36:09.373 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:09.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:09.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:09.383 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:36:09.402 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:36:09.432 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:36:09.433 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:36:09.433 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:36:09.434 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:36:09.434 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:36:09.434 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:36:09.434 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:36:09.434 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:36:09.434 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:36:09.477 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:09.518 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:09.534 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:36:09.534 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:36:09.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:09.534 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46514.0:110323 13.1 e592/557 13:8d92563c:::benchmark_data_smithi110_541928_object22543:head v 592'6779, mlcod=0'0)", 2023-09-23T23:36:09.534 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:36:09.532141+0000", 2023-09-23T23:36:09.534 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.00059461200000000001, 2023-09-23T23:36:09.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000617211, 2023-09-23T23:36:09.534 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:09.534 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:36:09.534 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:09.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:09.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:09.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:09.532141+0000", 2023-09-23T23:36:09.535 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:09.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:09.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:09.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:09.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:09.532141+0000", 2023-09-23T23:36:09.535 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:09.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:09.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:09.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:09.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:09.532140+0000", 2023-09-23T23:36:09.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:36:09.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:09.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:09.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:09.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:09.532147+0000", 2023-09-23T23:36:09.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5719999999999997e-06 2023-09-23T23:36:09.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:09.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:09.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:09.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:09.532160+0000", 2023-09-23T23:36:09.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2313999999999999e-05 2023-09-23T23:36:09.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:09.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:09.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:09.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:09.532165+0000", 2023-09-23T23:36:09.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6909999999999996e-06 2023-09-23T23:36:09.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:09.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:09.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:09.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:09.532212+0000", 2023-09-23T23:36:09.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7079000000000001e-05 2023-09-23T23:36:09.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:09.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:09.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:09.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:09.532241+0000", 2023-09-23T23:36:09.538 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9380999999999999e-05 2023-09-23T23:36:09.538 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:09.538 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:09.538 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:09.538 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:09.538 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-09-23T23:36:09.538 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 1 2023-09-23T23:36:09.538 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:36:10.156 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:10 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1671722902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:10.364 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T23:36:10.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:10 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1671722902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:10.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:10 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1671722902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:10.587 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:10.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:10.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:10.692 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:10.725 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:10.784 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:36:10.879 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:10.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:10.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:10.985 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:11.017 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:11.045 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:36:11.185 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:11.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:11.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:11.292 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:11.336 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:11.394 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:11.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:11.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:11.399 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:11 smithi110 ceph-mon[212939]: pgmap v1641: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 213 op/s; 25900/196860 objects degraded (13.157%); 3989/196860 objects misplaced (2.026%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:36:11.400 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:11 smithi110 ceph-mon[221102]: pgmap v1641: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 213 op/s; 25900/196860 objects degraded (13.157%); 3989/196860 objects misplaced (2.026%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:36:11.409 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:11 smithi138 ceph-mon[193606]: pgmap v1641: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 213 op/s; 25900/196860 objects degraded (13.157%); 3989/196860 objects misplaced (2.026%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:36:11.498 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:11.519 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:11.538 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:11.589 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:11.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:11.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:11.681 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:36:11.684 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:36:11.693 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:11.729 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:11.959 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:11.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:11.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:12.067 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:12.097 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:12.431 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:36:12.431 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:36:12.431 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:36:12.431 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:36:12.431 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:36:12.560 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:12.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:12.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:12.630 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:36:12.667 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:12.705 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:13.122 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1641,"stamp":"2023-09-23T23:36:09.915912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4299470069,"num_objects":65620,"num_object_clones":0,"num_object_copies":196860,"num_objects_missing_on_primary":3094,"num_objects_missing":3094,"num_objects_degraded":25900,"num_objects_misplaced":3989,"num_objects_unfound":0,"num_objects_dirty":65620,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":110037,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13707,"num_bytes_recovered":895611304,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94984,"ondisk_log_size":94984,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16754172,"kb_used_data":11123076,"kb_used_omap":29,"kb_used_meta":5630882,"kb_avail":639461892,"statfs":{"total":671965249536,"available":654808977408,"internally_reserved":0,"allocated":11390029824,"data_stored":11373926610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5766023891},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-131006464,"num_objects":-1999,"num_object_clones":0,"num_object_copies":-5997,"num_objects_missing_on_primary":-138,"num_objects_missing":-138,"num_objects_degraded":-264,"num_objects_misplaced":-138,"num_objects_unfound":0,"num_objects_dirty":-1999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1999,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":403,"num_bytes_recovered":26411008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.360369"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":788,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.558634+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:36:00.558634+0000","last_peered":"2023-09-23T23:36:00.558634+0000","last_clean":"2023-09-23T23:36:00.558634+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:36:00.558634+0000","last_undegraded":"2023-09-23T23:36:00.558634+0000","last_fullsized":"2023-09-23T23:36:00.558634+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":345,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.127430+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:35:50.127430+0000","last_peered":"2023-09-23T23:35:50.127430+0000","last_clean":"2023-09-23T23:35:50.127430+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:35:50.127430+0000","last_undegraded":"2023-09-23T23:35:50.127430+0000","last_fullsized":"2023-09-23T23:35:50.127430+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":573,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.586839+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:36:00.586839+0000","last_peered":"2023-09-23T23:36:00.586839+0000","last_clean":"2023-09-23T23:36:00.586839+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:36:00.586839+0000","last_undegraded":"2023-09-23T23:36:00.586839+0000","last_fullsized":"2023-09-23T23:36:00.586839+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":741,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.559201+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:36:00.559201+0000","last_peered":"2023-09-23T23:36:00.559201+0000","last_clean":"2023-09-23T23:36:00.559201+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:36:00.559201+0000","last_undegraded":"2023-09-23T23:36:00.559201+0000","last_fullsized":"2023-09-23T23:36:00.559201+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":309,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.131043+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:35:50.131043+0000","last_peered":"2023-09-23T23:35:50.131043+0000","last_clean":"2023-09-23T23:35:50.131043+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:35:50.131043+0000","last_undegraded":"2023-09-23T23:35:50.131043+0000","last_fullsized":"2023-09-23T23:35:50.131043+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":551,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.132062+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:35:50.132062+0000","last_peered":"2023-09-23T23:35:50.132062+0000","last_clean":"2023-09-23T23:35:50.132062+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:35:50.132062+0000","last_undegraded":"2023-09-23T23:35:50.132062+0000","last_fullsized":"2023-09-23T23:35:50.132062+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":753,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.560376+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:36:00.560376+0000","last_peered":"2023-09-23T23:36:00.560376+0000","last_clean":"2023-09-23T23:36:00.560376+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:36:00.560376+0000","last_undegraded":"2023-09-23T23:36:00.560376+0000","last_fullsized":"2023-09-23T23:36:00.560376+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":609,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.130427+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:35:50.130427+0000","last_peered":"2023-09-23T23:35:50.130427+0000","last_clean":"2023-09-23T23:35:50.130427+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:35:50.130427+0000","last_undegraded":"2023-09-23T23:35:50.130427+0000","last_fullsized":"2023-09-23T23:35:50.130427+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":106,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571745+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:36:00.571745+0000","last_peered":"2023-09-23T23:36:00.571745+0000","last_clean":"2023-09-23T23:36:00.571745+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:36:00.571745+0000","last_undegraded":"2023-09-23T23:36:00.571745+0000","last_fullsized":"2023-09-23T23:36:00.571745+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":912,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.560451+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:36:00.560451+0000","last_peered":"2023-09-23T23:36:00.560451+0000","last_clean":"2023-09-23T23:36:00.560451+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:36:00.560451+0000","last_undegraded":"2023-09-23T23:36:00.560451+0000","last_fullsized":"2023-09-23T23:36:00.560451+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":244,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.558693+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:36:00.558693+0000","last_peered":"2023-09-23T23:36:00.558693+0000","last_clean":"2023-09-23T23:36:00.558693+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:36:00.558693+0000","last_undegraded":"2023-09-23T23:36:00.558693+0000","last_fullsized":"2023-09-23T23:36:00.558693+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":591,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.130389+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:35:50.130389+0000","last_peered":"2023-09-23T23:35:50.130389+0000","last_clean":"2023-09-23T23:35:50.130389+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:35:50.130389+0000","last_undegraded":"2023-09-23T23:35:50.130389+0000","last_fullsized":"2023-09-23T23:35:50.130389+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":884,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.123069+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:35:50.123069+0000","last_peered":"2023-09-23T23:35:50.123069+0000","last_clean":"2023-09-23T23:35:50.123069+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:35:50.123069+0000","last_undegraded":"2023-09-23T23:35:50.123069+0000","last_fullsized":"2023-09-23T23:35:50.123069+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":223,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.561326+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:36:00.561326+0000","last_peered":"2023-09-23T23:36:00.561326+0000","last_clean":"2023-09-23T23:36:00.561326+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:36:00.561326+0000","last_undegraded":"2023-09-23T23:36:00.561326+0000","last_fullsized":"2023-09-23T23:36:00.561326+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":552,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.130845+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:35:50.130845+0000","last_peered":"2023-09-23T23:35:50.130845+0000","last_clean":"2023-09-23T23:35:50.130845+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:35:50.130845+0000","last_undegraded":"2023-09-23T23:35:50.130845+0000","last_fullsized":"2023-09-23T23:35:50.130845+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1046,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.558558+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:36:00.558558+0000","last_peered":"2023-09-23T23:36:00.558558+0000","last_clean":"2023-09-23T23:36:00.558558+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:36:00.558558+0000","last_undegraded":"2023-09-23T23:36:00.558558+0000","last_fullsized":"2023-09-23T23:36:00.558558+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":263,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.143349+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:35:50.143349+0000","last_peered":"2023-09-23T23:35:50.143349+0000","last_clean":"2023-09-23T23:35:50.143349+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:35:50.143349+0000","last_undegraded":"2023-09-23T23:35:50.143349+0000","last_fullsized":"2023-09-23T23:35:50.143349+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":125,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571483+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:36:00.571483+0000","last_peered":"2023-09-23T23:36:00.571483+0000","last_clean":"2023-09-23T23:36:00.571483+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:36:00.571483+0000","last_undegraded":"2023-09-23T23:36:00.571483+0000","last_fullsized":"2023-09-23T23:36:00.571483+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":441,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.132204+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:35:50.132204+0000","last_peered":"2023-09-23T23:35:50.132204+0000","last_clean":"2023-09-23T23:35:50.132204+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:35:50.132204+0000","last_undegraded":"2023-09-23T23:35:50.132204+0000","last_fullsized":"2023-09-23T23:35:50.132204+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":166,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.574119+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:36:00.574119+0000","last_peered":"2023-09-23T23:36:00.574119+0000","last_clean":"2023-09-23T23:36:00.574119+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:36:00.574119+0000","last_undegraded":"2023-09-23T23:36:00.574119+0000","last_fullsized":"2023-09-23T23:36:00.574119+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":533,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571109+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:36:00.571109+0000","last_peered":"2023-09-23T23:36:00.571109+0000","last_clean":"2023-09-23T23:36:00.571109+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:36:00.571109+0000","last_undegraded":"2023-09-23T23:36:00.571109+0000","last_fullsized":"2023-09-23T23:36:00.571109+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":747,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.559507+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:36:00.559507+0000","last_peered":"2023-09-23T23:36:00.559507+0000","last_clean":"2023-09-23T23:36:00.559507+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:36:00.559507+0000","last_undegraded":"2023-09-23T23:36:00.559507+0000","last_fullsized":"2023-09-23T23:36:00.559507+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":137,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.122693+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:35:50.122693+0000","last_peered":"2023-09-23T23:35:50.122693+0000","last_clean":"2023-09-23T23:35:50.122693+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:35:50.122693+0000","last_undegraded":"2023-09-23T23:35:50.122693+0000","last_fullsized":"2023-09-23T23:35:50.122693+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":590,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.131679+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:35:50.131679+0000","last_peered":"2023-09-23T23:35:50.131679+0000","last_clean":"2023-09-23T23:35:50.131679+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:35:50.131679+0000","last_undegraded":"2023-09-23T23:35:50.131679+0000","last_fullsized":"2023-09-23T23:35:50.131679+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":877,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.570858+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:36:00.570858+0000","last_peered":"2023-09-23T23:36:00.570858+0000","last_clean":"2023-09-23T23:36:00.570858+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:36:00.570858+0000","last_undegraded":"2023-09-23T23:36:00.570858+0000","last_fullsized":"2023-09-23T23:36:00.570858+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":185,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.131291+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:35:50.131291+0000","last_peered":"2023-09-23T23:35:50.131291+0000","last_clean":"2023-09-23T23:35:50.131291+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:35:50.131291+0000","last_undegraded":"2023-09-23T23:35:50.131291+0000","last_fullsized":"2023-09-23T23:35:50.131291+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":312,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.144477+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:35:50.144477+0000","last_peered":"2023-09-23T23:35:50.144477+0000","last_clean":"2023-09-23T23:35:50.144477+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:35:50.144477+0000","last_undegraded":"2023-09-23T23:35:50.144477+0000","last_fullsized":"2023-09-23T23:35:50.144477+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":553,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.134853+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:35:50.134853+0000","last_peered":"2023-09-23T23:35:50.134853+0000","last_clean":"2023-09-23T23:35:50.134853+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:35:50.134853+0000","last_undegraded":"2023-09-23T23:35:50.134853+0000","last_fullsized":"2023-09-23T23:35:50.134853+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":460,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.570976+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:36:00.570976+0000","last_peered":"2023-09-23T23:36:00.570976+0000","last_clean":"2023-09-23T23:36:00.570976+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:36:00.570976+0000","last_undegraded":"2023-09-23T23:36:00.570976+0000","last_fullsized":"2023-09-23T23:36:00.570976+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":547,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.132365+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:35:50.132365+0000","last_peered":"2023-09-23T23:35:50.132365+0000","last_clean":"2023-09-23T23:35:50.132365+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:35:50.132365+0000","last_undegraded":"2023-09-23T23:35:50.132365+0000","last_fullsized":"2023-09-23T23:35:50.132365+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":955,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.559101+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:36:00.559101+0000","last_peered":"2023-09-23T23:36:00.559101+0000","last_clean":"2023-09-23T23:36:00.559101+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:36:00.559101+0000","last_undegraded":"2023-09-23T23:36:00.559101+0000","last_fullsized":"2023-09-23T23:36:00.559101+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":156,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571030+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:36:00.571030+0000","last_peered":"2023-09-23T23:36:00.571030+0000","last_clean":"2023-09-23T23:36:00.571030+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:36:00.571030+0000","last_undegraded":"2023-09-23T23:36:00.571030+0000","last_fullsized":"2023-09-23T23:36:00.571030+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":232,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.132756+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:35:50.132756+0000","last_peered":"2023-09-23T23:35:50.132756+0000","last_clean":"2023-09-23T23:35:50.132756+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:35:50.132756+0000","last_undegraded":"2023-09-23T23:35:50.132756+0000","last_fullsized":"2023-09-23T23:35:50.132756+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"592'6777","reported_seq":6852,"reported_epoch":592,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:09.037399+0000","last_change":"2023-09-23T23:33:52.884769+0000","last_active":"2023-09-23T23:36:09.037399+0000","last_peered":"2023-09-23T23:36:09.037399+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:36:09.037399+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"548'1400","ondisk_log_start":"548'1400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5377,"log_dups_size":0,"ondisk_log_size":5377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266797056,"num_objects":4071,"num_object_clones":0,"num_object_copies":12213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6777,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":3045},{"shards":"2,6","objects":-327}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":924,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.122526+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:35:50.122526+0000","last_peered":"2023-09-23T23:35:50.122526+0000","last_clean":"2023-09-23T23:35:50.122526+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:35:50.122526+0000","last_undegraded":"2023-09-23T23:35:50.122526+0000","last_fullsized":"2023-09-23T23:35:50.122526+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":233,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.559301+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:36:00.559301+0000","last_peered":"2023-09-23T23:36:00.559301+0000","last_clean":"2023-09-23T23:36:00.559301+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:36:00.559301+0000","last_undegraded":"2023-09-23T23:36:00.559301+0000","last_fullsized":"2023-09-23T23:36:00.559301+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":223,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.558549+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:36:00.558549+0000","last_peered":"2023-09-23T23:36:00.558549+0000","last_clean":"2023-09-23T23:36:00.558549+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:36:00.558549+0000","last_undegraded":"2023-09-23T23:36:00.558549+0000","last_fullsized":"2023-09-23T23:36:00.558549+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"591'6841","reported_seq":6914,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:04.837268+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:36:04.837268+0000","last_peered":"2023-09-23T23:36:04.837268+0000","last_clean":"2023-09-23T23:36:04.837268+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:36:04.837268+0000","last_undegraded":"2023-09-23T23:36:04.837268+0000","last_fullsized":"2023-09-23T23:36:04.837268+0000","mapping_epoch":566,"log_start":"542'700","ondisk_log_start":"542'700","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6141,"log_dups_size":0,"ondisk_log_size":6141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274661376,"num_objects":4191,"num_object_clones":0,"num_object_copies":12573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6841,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1003,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.131394+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:35:50.131394+0000","last_peered":"2023-09-23T23:35:50.131394+0000","last_clean":"2023-09-23T23:35:50.131394+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:35:50.131394+0000","last_undegraded":"2023-09-23T23:35:50.131394+0000","last_fullsized":"2023-09-23T23:35:50.131394+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":223,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.559662+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:36:00.559662+0000","last_peered":"2023-09-23T23:36:00.559662+0000","last_clean":"2023-09-23T23:36:00.559662+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:36:00.559662+0000","last_undegraded":"2023-09-23T23:36:00.559662+0000","last_fullsized":"2023-09-23T23:36:00.559662+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":441,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.130450+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:35:50.130450+0000","last_peered":"2023-09-23T23:35:50.130450+0000","last_clean":"2023-09-23T23:35:50.130450+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:35:50.130450+0000","last_undegraded":"2023-09-23T23:35:50.130450+0000","last_fullsized":"2023-09-23T23:35:50.130450+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"591'6958","reported_seq":10935,"reported_epoch":591,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:36:07.262002+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:36:07.262002+0000","last_peered":"2023-09-23T23:36:07.262002+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:36:07.262002+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:36:07.262002+0000","mapping_epoch":571,"log_start":"548'1600","ondisk_log_start":"548'1600","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5358,"log_dups_size":0,"ondisk_log_size":5358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268042240,"num_objects":4090,"num_object_clones":0,"num_object_copies":12270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3195,"num_objects_misplaced":895,"num_objects_unfound":0,"num_objects_dirty":4090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6958,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1936,"num_bytes_recovered":126877696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":3073},{"shards":"3,5,7","objects":1017}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":902,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571804+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:36:00.571804+0000","last_peered":"2023-09-23T23:36:00.571804+0000","last_clean":"2023-09-23T23:36:00.571804+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:36:00.571804+0000","last_undegraded":"2023-09-23T23:36:00.571804+0000","last_fullsized":"2023-09-23T23:36:00.571804+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":330,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.560996+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:36:00.560996+0000","last_peered":"2023-09-23T23:36:00.560996+0000","last_clean":"2023-09-23T23:36:00.560996+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:36:00.560996+0000","last_undegraded":"2023-09-23T23:36:00.560996+0000","last_fullsized":"2023-09-23T23:36:00.560996+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":106,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571677+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:36:00.571677+0000","last_peered":"2023-09-23T23:36:00.571677+0000","last_clean":"2023-09-23T23:36:00.571677+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:36:00.571677+0000","last_undegraded":"2023-09-23T23:36:00.571677+0000","last_fullsized":"2023-09-23T23:36:00.571677+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"591'6884","reported_seq":8944,"reported_epoch":591,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:07.151393+0000","last_change":"2023-09-23T23:34:35.750428+0000","last_active":"2023-09-23T23:36:07.151393+0000","last_peered":"2023-09-23T23:36:07.151393+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:36:07.151393+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"548'1600","ondisk_log_start":"548'1600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5284,"log_dups_size":0,"ondisk_log_size":5284,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275644416,"num_objects":4206,"num_object_clones":0,"num_object_copies":12618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6884,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":991,"num_bytes_recovered":64946176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":3196},{"shards":"3,5","objects":662}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":497,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.132290+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:35:50.132290+0000","last_peered":"2023-09-23T23:35:50.132290+0000","last_clean":"2023-09-23T23:35:50.132290+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:35:50.132290+0000","last_undegraded":"2023-09-23T23:35:50.132290+0000","last_fullsized":"2023-09-23T23:35:50.132290+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":431,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.131560+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:35:50.131560+0000","last_peered":"2023-09-23T23:35:50.131560+0000","last_clean":"2023-09-23T23:35:50.131560+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:35:50.131560+0000","last_undegraded":"2023-09-23T23:35:50.131560+0000","last_fullsized":"2023-09-23T23:35:50.131560+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":588,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.155818+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:35:50.155818+0000","last_peered":"2023-09-23T23:35:50.155818+0000","last_clean":"2023-09-23T23:35:50.155818+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:35:50.155818+0000","last_undegraded":"2023-09-23T23:35:50.155818+0000","last_fullsized":"2023-09-23T23:35:50.155818+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"592'7048","reported_seq":8370,"reported_epoch":592,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:09.237423+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:36:09.237423+0000","last_peered":"2023-09-23T23:36:09.237423+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:36:09.237423+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"546'1300","ondisk_log_start":"546'1300","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5748,"log_dups_size":0,"ondisk_log_size":5748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269615104,"num_objects":4114,"num_object_clones":0,"num_object_copies":12342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7048,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":3068},{"shards":"3,5","objects":-1346},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":718,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.130773+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:35:50.130773+0000","last_peered":"2023-09-23T23:35:50.130773+0000","last_clean":"2023-09-23T23:35:50.130773+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:35:50.130773+0000","last_undegraded":"2023-09-23T23:35:50.130773+0000","last_fullsized":"2023-09-23T23:35:50.130773+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":601,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.134776+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:35:50.134776+0000","last_peered":"2023-09-23T23:35:50.134776+0000","last_clean":"2023-09-23T23:35:50.134776+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:35:50.134776+0000","last_undegraded":"2023-09-23T23:35:50.134776+0000","last_fullsized":"2023-09-23T23:35:50.134776+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":512,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.576721+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:36:00.576721+0000","last_peered":"2023-09-23T23:36:00.576721+0000","last_clean":"2023-09-23T23:36:00.576721+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:36:00.576721+0000","last_undegraded":"2023-09-23T23:36:00.576721+0000","last_fullsized":"2023-09-23T23:36:00.576721+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"592'6976","reported_seq":7042,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:09.109375+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:36:09.109375+0000","last_peered":"2023-09-23T23:36:09.109375+0000","last_clean":"2023-09-23T23:36:09.109375+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:36:09.109375+0000","last_undegraded":"2023-09-23T23:36:09.109375+0000","last_fullsized":"2023-09-23T23:36:09.109375+0000","mapping_epoch":534,"log_start":"548'1600","ondisk_log_start":"548'1600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5376,"log_dups_size":0,"ondisk_log_size":5376,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268959744,"num_objects":4104,"num_object_clones":0,"num_object_copies":12312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6976,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":874,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.122393+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:35:50.122393+0000","last_peered":"2023-09-23T23:35:50.122393+0000","last_clean":"2023-09-23T23:35:50.122393+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:35:50.122393+0000","last_undegraded":"2023-09-23T23:35:50.122393+0000","last_fullsized":"2023-09-23T23:35:50.122393+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":116,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571605+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:36:00.571605+0000","last_peered":"2023-09-23T23:36:00.571605+0000","last_clean":"2023-09-23T23:36:00.571605+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:36:00.571605+0000","last_undegraded":"2023-09-23T23:36:00.571605+0000","last_fullsized":"2023-09-23T23:36:00.571605+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":456,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.122731+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:35:50.122731+0000","last_peered":"2023-09-23T23:35:50.122731+0000","last_clean":"2023-09-23T23:35:50.122731+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:35:50.122731+0000","last_undegraded":"2023-09-23T23:35:50.122731+0000","last_fullsized":"2023-09-23T23:35:50.122731+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"592'7010","reported_seq":10012,"reported_epoch":592,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:36:09.133352+0000","last_change":"2023-09-23T23:34:35.785270+0000","last_active":"2023-09-23T23:36:09.133352+0000","last_peered":"2023-09-23T23:36:09.133352+0000","last_clean":"2023-09-23T23:33:50.504296+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:36:09.133352+0000","last_undegraded":"2023-09-23T23:34:35.780981+0000","last_fullsized":"2023-09-23T23:36:09.133352+0000","mapping_epoch":567,"log_start":"548'1600","ondisk_log_start":"548'1600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5410,"log_dups_size":0,"ondisk_log_size":5410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268304384,"num_objects":4094,"num_object_clones":0,"num_object_copies":12282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3089,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7010,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1458,"num_bytes_recovered":95551488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":3089},{"shards":"5,6,7","objects":1005}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":600,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.559147+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:36:00.559147+0000","last_peered":"2023-09-23T23:36:00.559147+0000","last_clean":"2023-09-23T23:36:00.559147+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:36:00.559147+0000","last_undegraded":"2023-09-23T23:36:00.559147+0000","last_fullsized":"2023-09-23T23:36:00.559147+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":454,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.565147+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:36:00.565147+0000","last_peered":"2023-09-23T23:36:00.565147+0000","last_clean":"2023-09-23T23:36:00.565147+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:36:00.565147+0000","last_undegraded":"2023-09-23T23:36:00.565147+0000","last_fullsized":"2023-09-23T23:36:00.565147+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":543,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571100+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:36:00.571100+0000","last_peered":"2023-09-23T23:36:00.571100+0000","last_clean":"2023-09-23T23:36:00.571100+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:36:00.571100+0000","last_undegraded":"2023-09-23T23:36:00.571100+0000","last_fullsized":"2023-09-23T23:36:00.571100+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"592'6955","reported_seq":15984,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:09.263390+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:36:09.263390+0000","last_peered":"2023-09-23T23:36:09.263390+0000","last_clean":"2023-09-23T23:36:09.263390+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:36:09.263390+0000","last_undegraded":"2023-09-23T23:36:09.263390+0000","last_fullsized":"2023-09-23T23:36:09.263390+0000","mapping_epoch":579,"log_start":"544'1200","ondisk_log_start":"544'1200","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5755,"log_dups_size":0,"ondisk_log_size":5755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266141696,"num_objects":4061,"num_object_clones":0,"num_object_copies":12183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6955,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":932,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.127282+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:35:50.127282+0000","last_peered":"2023-09-23T23:35:50.127282+0000","last_clean":"2023-09-23T23:35:50.127282+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:35:50.127282+0000","last_undegraded":"2023-09-23T23:35:50.127282+0000","last_fullsized":"2023-09-23T23:35:50.127282+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":232,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.130310+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:35:50.130310+0000","last_peered":"2023-09-23T23:35:50.130310+0000","last_clean":"2023-09-23T23:35:50.130310+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:35:50.130310+0000","last_undegraded":"2023-09-23T23:35:50.130310+0000","last_fullsized":"2023-09-23T23:35:50.130310+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":520,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.132975+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:35:50.132975+0000","last_peered":"2023-09-23T23:35:50.132975+0000","last_clean":"2023-09-23T23:35:50.132975+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:35:50.132975+0000","last_undegraded":"2023-09-23T23:35:50.132975+0000","last_fullsized":"2023-09-23T23:35:50.132975+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"591'6645","reported_seq":6712,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:06.887677+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:36:06.887677+0000","last_peered":"2023-09-23T23:36:06.887677+0000","last_clean":"2023-09-23T23:36:06.887677+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:36:06.887677+0000","last_undegraded":"2023-09-23T23:36:06.887677+0000","last_fullsized":"2023-09-23T23:36:06.887677+0000","mapping_epoch":534,"log_start":"548'1300","ondisk_log_start":"548'1300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5345,"log_dups_size":0,"ondisk_log_size":5345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267583488,"num_objects":4083,"num_object_clones":0,"num_object_copies":12249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4083,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6645,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":913,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571573+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:36:00.571573+0000","last_peered":"2023-09-23T23:36:00.571573+0000","last_clean":"2023-09-23T23:36:00.571573+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:36:00.571573+0000","last_undegraded":"2023-09-23T23:36:00.571573+0000","last_fullsized":"2023-09-23T23:36:00.571573+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":300,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.125879+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:35:50.125879+0000","last_peered":"2023-09-23T23:35:50.125879+0000","last_clean":"2023-09-23T23:35:50.125879+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:35:50.125879+0000","last_undegraded":"2023-09-23T23:35:50.125879+0000","last_fullsized":"2023-09-23T23:35:50.125879+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":607,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.561079+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:36:00.561079+0000","last_peered":"2023-09-23T23:36:00.561079+0000","last_clean":"2023-09-23T23:36:00.561079+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:36:00.561079+0000","last_undegraded":"2023-09-23T23:36:00.561079+0000","last_fullsized":"2023-09-23T23:36:00.561079+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":188,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.558854+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:36:00.558854+0000","last_peered":"2023-09-23T23:36:00.558854+0000","last_clean":"2023-09-23T23:36:00.558854+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:36:00.558854+0000","last_undegraded":"2023-09-23T23:36:00.558854+0000","last_fullsized":"2023-09-23T23:36:00.558854+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"591'6994","reported_seq":9911,"reported_epoch":591,"state":"active+recovering","last_fresh":"2023-09-23T23:36:07.333406+0000","last_change":"2023-09-23T23:34:35.815566+0000","last_active":"2023-09-23T23:36:07.333406+0000","last_peered":"2023-09-23T23:36:07.333406+0000","last_clean":"2023-09-23T23:33:50.519308+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:36:07.333406+0000","last_undegraded":"2023-09-23T23:36:07.333406+0000","last_fullsized":"2023-09-23T23:36:07.333406+0000","mapping_epoch":566,"log_start":"548'1722","ondisk_log_start":"548'1722","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5272,"log_dups_size":0,"ondisk_log_size":5272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273481752,"num_objects":4174,"num_object_clones":0,"num_object_copies":12522,"num_objects_missing_on_primary":3094,"num_objects_missing":3094,"num_objects_degraded":0,"num_objects_misplaced":3094,"num_objects_unfound":0,"num_objects_dirty":4174,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6994,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1410,"num_bytes_recovered":92405760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,3,4,7","objects":11},{"shards":"1,3,7","objects":1069},{"shards":"1,4,7","objects":3094}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":674,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.154500+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:35:50.154500+0000","last_peered":"2023-09-23T23:35:50.154500+0000","last_clean":"2023-09-23T23:35:50.154500+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:35:50.154500+0000","last_undegraded":"2023-09-23T23:35:50.154500+0000","last_fullsized":"2023-09-23T23:35:50.154500+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":240,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571623+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:36:00.571623+0000","last_peered":"2023-09-23T23:36:00.571623+0000","last_clean":"2023-09-23T23:36:00.571623+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:36:00.571623+0000","last_undegraded":"2023-09-23T23:36:00.571623+0000","last_fullsized":"2023-09-23T23:36:00.571623+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":524,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571140+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:36:00.571140+0000","last_peered":"2023-09-23T23:36:00.571140+0000","last_clean":"2023-09-23T23:36:00.571140+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:36:00.571140+0000","last_undegraded":"2023-09-23T23:36:00.571140+0000","last_fullsized":"2023-09-23T23:36:00.571140+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":223,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.123113+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:35:50.123113+0000","last_peered":"2023-09-23T23:35:50.123113+0000","last_clean":"2023-09-23T23:35:50.123113+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:35:50.123113+0000","last_undegraded":"2023-09-23T23:35:50.123113+0000","last_fullsized":"2023-09-23T23:35:50.123113+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"592'6743","reported_seq":8788,"reported_epoch":592,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:09.246347+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:36:09.246347+0000","last_peered":"2023-09-23T23:36:09.246347+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:36:09.246347+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"544'1000","ondisk_log_start":"544'1000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5743,"log_dups_size":0,"ondisk_log_size":5743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264044544,"num_objects":4029,"num_object_clones":0,"num_object_copies":12087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6743,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":3018},{"shards":"5,7","objects":637}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":892,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.127200+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:35:50.127200+0000","last_peered":"2023-09-23T23:35:50.127200+0000","last_clean":"2023-09-23T23:35:50.127200+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:35:50.127200+0000","last_undegraded":"2023-09-23T23:35:50.127200+0000","last_fullsized":"2023-09-23T23:35:50.127200+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":329,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.122310+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:35:50.122310+0000","last_peered":"2023-09-23T23:35:50.122310+0000","last_clean":"2023-09-23T23:35:50.122310+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:35:50.122310+0000","last_undegraded":"2023-09-23T23:35:50.122310+0000","last_fullsized":"2023-09-23T23:35:50.122310+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":239,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.562023+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:36:00.562023+0000","last_peered":"2023-09-23T23:36:00.562023+0000","last_clean":"2023-09-23T23:36:00.562023+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:36:00.562023+0000","last_undegraded":"2023-09-23T23:36:00.562023+0000","last_fullsized":"2023-09-23T23:36:00.562023+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":203,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.560338+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:36:00.560338+0000","last_peered":"2023-09-23T23:36:00.560338+0000","last_clean":"2023-09-23T23:36:00.560338+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:36:00.560338+0000","last_undegraded":"2023-09-23T23:36:00.560338+0000","last_fullsized":"2023-09-23T23:36:00.560338+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"592'6861","reported_seq":6937,"reported_epoch":592,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:36:09.296772+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:36:09.296772+0000","last_peered":"2023-09-23T23:36:09.296772+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:36:09.296772+0000","last_undegraded":"2023-09-23T23:36:09.296772+0000","last_fullsized":"2023-09-23T23:36:09.296772+0000","mapping_epoch":562,"log_start":"544'1200","ondisk_log_start":"544'1200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5661,"log_dups_size":0,"ondisk_log_size":5661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264830976,"num_objects":4041,"num_object_clones":0,"num_object_copies":12123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6861,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":4041}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":606,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:59.924073+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:35:59.924073+0000","last_peered":"2023-09-23T23:35:59.924073+0000","last_clean":"2023-09-23T23:35:59.924073+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:35:59.924073+0000","last_undegraded":"2023-09-23T23:35:59.924073+0000","last_fullsized":"2023-09-23T23:35:59.924073+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":878,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.570846+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:36:00.570846+0000","last_peered":"2023-09-23T23:36:00.570846+0000","last_clean":"2023-09-23T23:36:00.570846+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:36:00.570846+0000","last_undegraded":"2023-09-23T23:36:00.570846+0000","last_fullsized":"2023-09-23T23:36:00.570846+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":553,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:57.265522+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:35:57.265522+0000","last_peered":"2023-09-23T23:35:57.265522+0000","last_clean":"2023-09-23T23:35:57.265522+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:35:57.265522+0000","last_undegraded":"2023-09-23T23:35:57.265522+0000","last_fullsized":"2023-09-23T23:35:57.265522+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":116,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571329+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:36:00.571329+0000","last_peered":"2023-09-23T23:36:00.571329+0000","last_clean":"2023-09-23T23:36:00.571329+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:36:00.571329+0000","last_undegraded":"2023-09-23T23:36:00.571329+0000","last_fullsized":"2023-09-23T23:36:00.571329+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"592'6754","reported_seq":6820,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:09.102660+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:36:09.102660+0000","last_peered":"2023-09-23T23:36:09.102660+0000","last_clean":"2023-09-23T23:36:09.102660+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:36:09.102660+0000","last_undegraded":"2023-09-23T23:36:09.102660+0000","last_fullsized":"2023-09-23T23:36:09.102660+0000","mapping_epoch":534,"log_start":"546'1300","ondisk_log_start":"546'1300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5454,"log_dups_size":0,"ondisk_log_size":5454,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260964352,"num_objects":3982,"num_object_clones":0,"num_object_copies":11946,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3982,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6754,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":704,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571052+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:36:00.571052+0000","last_peered":"2023-09-23T23:36:00.571052+0000","last_clean":"2023-09-23T23:36:00.571052+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:36:00.571052+0000","last_undegraded":"2023-09-23T23:36:00.571052+0000","last_fullsized":"2023-09-23T23:36:00.571052+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":519,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.558647+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:36:00.558647+0000","last_peered":"2023-09-23T23:36:00.558647+0000","last_clean":"2023-09-23T23:36:00.558647+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:36:00.558647+0000","last_undegraded":"2023-09-23T23:36:00.558647+0000","last_fullsized":"2023-09-23T23:36:00.558647+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":432,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:57.265423+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:35:57.265423+0000","last_peered":"2023-09-23T23:35:57.265423+0000","last_clean":"2023-09-23T23:35:57.265423+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:35:57.265423+0000","last_undegraded":"2023-09-23T23:35:57.265423+0000","last_fullsized":"2023-09-23T23:35:57.265423+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":31,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.155119+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:35:50.155119+0000","last_peered":"2023-09-23T23:35:50.155119+0000","last_clean":"2023-09-23T23:35:50.155119+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:35:50.155119+0000","last_undegraded":"2023-09-23T23:35:50.155119+0000","last_fullsized":"2023-09-23T23:35:50.155119+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"591'6840","reported_seq":10504,"reported_epoch":591,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:04.274374+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:36:04.274374+0000","last_peered":"2023-09-23T23:36:04.274374+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:36:04.274374+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"542'700","ondisk_log_start":"542'700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6140,"log_dups_size":0,"ondisk_log_size":6140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272498688,"num_objects":4158,"num_object_clones":0,"num_object_copies":12474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6840,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":1454},{"shards":"1,2,4","objects":2704}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":587,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571516+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:36:00.571516+0000","last_peered":"2023-09-23T23:36:00.571516+0000","last_clean":"2023-09-23T23:36:00.571516+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:36:00.571516+0000","last_undegraded":"2023-09-23T23:36:00.571516+0000","last_fullsized":"2023-09-23T23:36:00.571516+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":31,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.154845+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:35:50.154845+0000","last_peered":"2023-09-23T23:35:50.154845+0000","last_clean":"2023-09-23T23:35:50.154845+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:35:50.154845+0000","last_undegraded":"2023-09-23T23:35:50.154845+0000","last_fullsized":"2023-09-23T23:35:50.154845+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":130,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.123159+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:35:50.123159+0000","last_peered":"2023-09-23T23:35:50.123159+0000","last_clean":"2023-09-23T23:35:50.123159+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:35:50.123159+0000","last_undegraded":"2023-09-23T23:35:50.123159+0000","last_fullsized":"2023-09-23T23:35:50.123159+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":325,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.559258+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:36:00.559258+0000","last_peered":"2023-09-23T23:36:00.559258+0000","last_clean":"2023-09-23T23:36:00.559258+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:36:00.559258+0000","last_undegraded":"2023-09-23T23:36:00.559258+0000","last_fullsized":"2023-09-23T23:36:00.559258+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"592'6919","reported_seq":6985,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:09.246959+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:36:09.246959+0000","last_peered":"2023-09-23T23:36:09.246959+0000","last_clean":"2023-09-23T23:36:09.246959+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:36:09.246959+0000","last_undegraded":"2023-09-23T23:36:09.246959+0000","last_fullsized":"2023-09-23T23:36:09.246959+0000","mapping_epoch":534,"log_start":"544'1200","ondisk_log_start":"544'1200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5719,"log_dups_size":0,"ondisk_log_size":5719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265093120,"num_objects":4045,"num_object_clones":0,"num_object_copies":12135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6919,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":772,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:59.924144+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:35:59.924144+0000","last_peered":"2023-09-23T23:35:59.924144+0000","last_clean":"2023-09-23T23:35:59.924144+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:35:59.924144+0000","last_undegraded":"2023-09-23T23:35:59.924144+0000","last_fullsized":"2023-09-23T23:35:59.924144+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":588,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.132897+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:35:50.132897+0000","last_peered":"2023-09-23T23:35:50.132897+0000","last_clean":"2023-09-23T23:35:50.132897+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:35:50.132897+0000","last_undegraded":"2023-09-23T23:35:50.132897+0000","last_fullsized":"2023-09-23T23:35:50.132897+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":196,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571402+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:36:00.571402+0000","last_peered":"2023-09-23T23:36:00.571402+0000","last_clean":"2023-09-23T23:36:00.571402+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:36:00.571402+0000","last_undegraded":"2023-09-23T23:36:00.571402+0000","last_fullsized":"2023-09-23T23:36:00.571402+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":142,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.122306+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:35:50.122306+0000","last_peered":"2023-09-23T23:35:50.122306+0000","last_clean":"2023-09-23T23:35:50.122306+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:35:50.122306+0000","last_undegraded":"2023-09-23T23:35:50.122306+0000","last_fullsized":"2023-09-23T23:35:50.122306+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"591'6755","reported_seq":6822,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:04.863202+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:36:04.863202+0000","last_peered":"2023-09-23T23:36:04.863202+0000","last_clean":"2023-09-23T23:36:04.863202+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:36:04.863202+0000","last_undegraded":"2023-09-23T23:36:04.863202+0000","last_fullsized":"2023-09-23T23:36:04.863202+0000","mapping_epoch":534,"log_start":"541'600","ondisk_log_start":"541'600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6155,"log_dups_size":0,"ondisk_log_size":6155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272171008,"num_objects":4153,"num_object_clones":0,"num_object_copies":12459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6755,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":978,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.131801+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:35:50.131801+0000","last_peered":"2023-09-23T23:35:50.131801+0000","last_clean":"2023-09-23T23:35:50.131801+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:35:50.131801+0000","last_undegraded":"2023-09-23T23:35:50.131801+0000","last_fullsized":"2023-09-23T23:35:50.131801+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":106,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571651+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:36:00.571651+0000","last_peered":"2023-09-23T23:36:00.571651+0000","last_clean":"2023-09-23T23:36:00.571651+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:36:00.571651+0000","last_undegraded":"2023-09-23T23:36:00.571651+0000","last_fullsized":"2023-09-23T23:36:00.571651+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":35,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.155193+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:35:50.155193+0000","last_peered":"2023-09-23T23:35:50.155193+0000","last_clean":"2023-09-23T23:35:50.155193+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:35:50.155193+0000","last_undegraded":"2023-09-23T23:35:50.155193+0000","last_fullsized":"2023-09-23T23:35:50.155193+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":166,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.559573+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:36:00.559573+0000","last_peered":"2023-09-23T23:36:00.559573+0000","last_clean":"2023-09-23T23:36:00.559573+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:36:00.559573+0000","last_undegraded":"2023-09-23T23:36:00.559573+0000","last_fullsized":"2023-09-23T23:36:00.559573+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1027,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.130694+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:35:50.130694+0000","last_peered":"2023-09-23T23:35:50.130694+0000","last_clean":"2023-09-23T23:35:50.130694+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:35:50.130694+0000","last_undegraded":"2023-09-23T23:35:50.130694+0000","last_fullsized":"2023-09-23T23:35:50.130694+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":31,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.155829+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:35:50.155829+0000","last_peered":"2023-09-23T23:35:50.155829+0000","last_clean":"2023-09-23T23:35:50.155829+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:35:50.155829+0000","last_undegraded":"2023-09-23T23:35:50.155829+0000","last_fullsized":"2023-09-23T23:35:50.155829+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":346,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.123186+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:35:50.123186+0000","last_peered":"2023-09-23T23:35:50.123186+0000","last_clean":"2023-09-23T23:35:50.123186+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:35:50.123186+0000","last_undegraded":"2023-09-23T23:35:50.123186+0000","last_fullsized":"2023-09-23T23:35:50.123186+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":739,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:52.236752+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:35:52.236752+0000","last_peered":"2023-09-23T23:35:52.236752+0000","last_clean":"2023-09-23T23:35:52.236752+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:35:52.236752+0000","last_undegraded":"2023-09-23T23:35:52.236752+0000","last_fullsized":"2023-09-23T23:35:52.236752+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":94,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571238+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:36:00.571238+0000","last_peered":"2023-09-23T23:36:00.571238+0000","last_clean":"2023-09-23T23:36:00.571238+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:36:00.571238+0000","last_undegraded":"2023-09-23T23:36:00.571238+0000","last_fullsized":"2023-09-23T23:36:00.571238+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":438,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.134688+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:35:50.134688+0000","last_peered":"2023-09-23T23:35:50.134688+0000","last_clean":"2023-09-23T23:35:50.134688+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:35:50.134688+0000","last_undegraded":"2023-09-23T23:35:50.134688+0000","last_fullsized":"2023-09-23T23:35:50.134688+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":290,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.571275+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:36:00.571275+0000","last_peered":"2023-09-23T23:36:00.571275+0000","last_clean":"2023-09-23T23:36:00.571275+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:36:00.571275+0000","last_undegraded":"2023-09-23T23:36:00.571275+0000","last_fullsized":"2023-09-23T23:36:00.571275+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":277,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.131217+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:35:50.131217+0000","last_peered":"2023-09-23T23:35:50.131217+0000","last_clean":"2023-09-23T23:35:50.131217+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:35:50.131217+0000","last_undegraded":"2023-09-23T23:35:50.131217+0000","last_fullsized":"2023-09-23T23:35:50.131217+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":804,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.574182+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:36:00.574182+0000","last_peered":"2023-09-23T23:36:00.574182+0000","last_clean":"2023-09-23T23:36:00.574182+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:36:00.574182+0000","last_undegraded":"2023-09-23T23:36:00.574182+0000","last_fullsized":"2023-09-23T23:36:00.574182+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":621,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.132452+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:35:50.132452+0000","last_peered":"2023-09-23T23:35:50.132452+0000","last_clean":"2023-09-23T23:35:50.132452+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:35:50.132452+0000","last_undegraded":"2023-09-23T23:35:50.132452+0000","last_fullsized":"2023-09-23T23:35:50.132452+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":605,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.130967+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:35:50.130967+0000","last_peered":"2023-09-23T23:35:50.130967+0000","last_clean":"2023-09-23T23:35:50.130967+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:35:50.130967+0000","last_undegraded":"2023-09-23T23:35:50.130967+0000","last_fullsized":"2023-09-23T23:35:50.130967+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":763,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.143275+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:35:50.143275+0000","last_peered":"2023-09-23T23:35:50.143275+0000","last_clean":"2023-09-23T23:35:50.143275+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:35:50.143275+0000","last_undegraded":"2023-09-23T23:35:50.143275+0000","last_fullsized":"2023-09-23T23:35:50.143275+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":434,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-09-23T23:35:50.122641+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:35:50.122641+0000","last_peered":"2023-09-23T23:35:50.122641+0000","last_clean":"2023-09-23T23:35:50.122641+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:35:50.122641+0000","last_undegraded":"2023-09-23T23:35:50.122641+0000","last_fullsized":"2023-09-23T23:35:50.122641+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":659,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-09-23T23:36:00.570787+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:36:00.570787+0000","last_peered":"2023-09-23T23:36:00.570787+0000","last_clean":"2023-09-23T23:36:00.570787+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:36:00.570787+0000","last_undegraded":"2023-09-23T23:36:00.570787+0000","last_fullsized":"2023-09-23T23:36:00.570787+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":4298833944,"num_objects":65596,"num_object_clones":0,"num_object_copies":196788,"num_objects_missing_on_primary":3094,"num_objects_missing":3094,"num_objects_degraded":25900,"num_objects_misplaced":3989,"num_objects_unfound":0,"num_objects_dirty":65596,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":109960,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13652,"num_bytes_recovered":894500864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13835776000,"data_stored":13835763784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89938,"ondisk_log_size":89938,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782680,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1907268,"kb_used_data":1537660,"kb_used_omap":8,"kb_used_meta":369463,"kb_avail":91837884,"statfs":{"total":95995035648,"available":94041993216,"internally_reserved":0,"allocated":1574563840,"data_stored":1572924905,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":378330181},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":1.5569999999999999,"15min":2.1800000000000002},"min":{"1min":0.23100000000000001,"5min":0.224,"15min":0.20899999999999999},"max":{"1min":0.61699999999999999,"5min":36.956000000000003,"15min":85.721000000000004},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.47399999999999998,"5min":1.522,"15min":2.2250000000000001},"min":{"1min":0.29199999999999998,"5min":0.27900000000000003,"15min":0.19800000000000001},"max":{"1min":1.4419999999999999,"5min":37.247999999999998,"15min":85.965000000000003},"last":0.36799999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":3.8900000000000001,"15min":3.169},"min":{"1min":0.26600000000000001,"5min":0.248,"15min":0.22900000000000001},"max":{"1min":3.3500000000000001,"5min":129.11199999999999,"15min":198.58000000000001},"last":0.245},{"interface":"front","average":{"1min":0.65600000000000003,"5min":3.665,"15min":3.206},"min":{"1min":0.29699999999999999,"5min":0.247,"15min":0.246},"max":{"1min":3.7290000000000001,"5min":129.02600000000001,"15min":198.71100000000001},"last":0.46600000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":2.871,"15min":3.7010000000000001},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.452,"5min":78.528999999999996,"15min":203.58000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.54200000000000004,"5min":1.952,"15min":3.0649999999999999},"min":{"1min":0.30399999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.2529999999999999,"5min":27.574000000000002,"15min":198.69},"last":3.4670000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":3.3290000000000002,"15min":3.5950000000000002},"min":{"1min":0.31900000000000001,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":1.591,"5min":125.511,"15min":279.26499999999999},"last":1.9219999999999999},{"interface":"front","average":{"1min":0.67500000000000004,"5min":5.3090000000000002,"15min":4.1210000000000004},"min":{"1min":0.374,"5min":0.32000000000000001,"15min":0.29599999999999999},"max":{"1min":3.6240000000000001,"5min":150.608,"15min":279.29899999999998},"last":2.4740000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":1.694,"15min":1.0669999999999999},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":0.68100000000000005,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.505},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.363,"15min":1.0029999999999999},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.23899999999999999},"max":{"1min":1.6080000000000001,"5min":31.68,"15min":31.68},"last":0.39800000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":3.637,"5min":1.657,"15min":1.899},"min":{"1min":0.31900000000000001,"5min":0.28699999999999998,"15min":0.26500000000000001},"max":{"1min":60.463999999999999,"5min":60.463999999999999,"15min":210.83600000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":3.5990000000000002,"5min":1.6160000000000001,"15min":1.7649999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28199999999999997},"max":{"1min":63.006,"5min":63.006,"15min":207.04599999999999},"last":0.45300000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":1.879,"15min":1.3480000000000001},"min":{"1min":0.35199999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.4419999999999999,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.59299999999999997,"5min":2.2330000000000001,"15min":1.581},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":2.3180000000000001,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.53300000000000003}]}]},{"osd":6,"up_from":46,"seq":197568496157,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2136384,"kb_used_data":1440120,"kb_used_omap":0,"kb_used_meta":696256,"kb_avail":91608768,"statfs":{"total":95995035648,"available":93807378432,"internally_reserved":0,"allocated":1474682880,"data_stored":1473056075,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":712966144},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:14 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.98799999999999999,"15min":4.7699999999999996},"min":{"1min":0.26500000000000001,"5min":0.223,"15min":0.20399999999999999},"max":{"1min":2.274,"5min":8.8119999999999994,"15min":135.31999999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.93000000000000005,"15min":4.617},"min":{"1min":0.26800000000000002,"5min":0.23999999999999999,"15min":0.22900000000000001},"max":{"1min":3.8239999999999998,"5min":8.3230000000000004,"15min":135.34299999999999},"last":0.24399999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:35:14 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":3.3690000000000002,"15min":2.4089999999999998},"min":{"1min":0.26000000000000001,"5min":0.251,"15min":0.245},"max":{"1min":6.9080000000000004,"5min":89.605000000000004,"15min":89.605000000000004},"last":1.929},{"interface":"front","average":{"1min":0.71599999999999997,"5min":3.1600000000000001,"15min":2.4849999999999999},"min":{"1min":0.33500000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":1.9159999999999999,"5min":91.025999999999996,"15min":91.025999999999996},"last":2.1269999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":4.9459999999999997,"15min":4.5620000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.861,"5min":97.408000000000001,"15min":127.44499999999999},"last":1.5109999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":6.0880000000000001,"15min":4.9740000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.2120000000000002,"5min":125.539,"15min":125.539},"last":0.46300000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:35:53 2023","interfaces":[{"interface":"back","average":{"1min":7.923,"5min":7.0129999999999999,"15min":4.4459999999999997},"min":{"1min":0.316,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":73.813999999999993,"5min":138.87299999999999,"15min":138.87299999999999},"last":1.633},{"interface":"front","average":{"1min":8.4450000000000003,"5min":6.9260000000000002,"15min":4.7350000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":73.852999999999994,"5min":138.976,"15min":138.976},"last":0.52300000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.82999999999999996,"15min":1.274},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":1.633,"5min":7.101,"15min":166.05199999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":1.0289999999999999,"15min":1.4890000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.089,"5min":8.3499999999999996,"15min":166.631},"last":0.374}]},{"osd":5,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.96699999999999997,"15min":0.82399999999999995},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.1579999999999999,"5min":11.473000000000001,"15min":11.473000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.433,"5min":1.091,"15min":0.94099999999999995},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.249},"max":{"1min":0.76700000000000002,"5min":7.5670000000000002,"15min":8.4090000000000007},"last":0.33800000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":1.294,"15min":1.0289999999999999},"min":{"1min":0.30499999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":0.70599999999999996,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.505},{"interface":"front","average":{"1min":0.72999999999999998,"5min":1.4019999999999999,"15min":1.0720000000000001},"min":{"1min":0.36099999999999999,"5min":0.35499999999999998,"15min":0.33500000000000002},"max":{"1min":2.665,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.49399999999999999}]}]},{"osd":1,"up_from":112,"seq":481036337497,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2224940,"kb_used_data":1338148,"kb_used_omap":4,"kb_used_meta":886779,"kb_avail":91520212,"statfs":{"total":95995035648,"available":93716697088,"internally_reserved":0,"allocated":1370263552,"data_stored":1367479686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":908062606},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":2.8220000000000001,"15min":3.5670000000000002},"min":{"1min":0.21199999999999999,"5min":0.17799999999999999,"15min":0.16700000000000001},"max":{"1min":15.983000000000001,"5min":47.195999999999998,"15min":130.768},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.883,"15min":2.8650000000000002},"min":{"1min":0.20699999999999999,"5min":0.20000000000000001,"15min":0.19},"max":{"1min":4.9809999999999999,"5min":46.582000000000001,"15min":134.929},"last":0.32800000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":7.1859999999999999,"15min":6.7910000000000004},"min":{"1min":0.26300000000000001,"5min":0.214,"15min":0.20100000000000001},"max":{"1min":1.8839999999999999,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.65800000000000003,"5min":7.6059999999999999,"15min":6.9160000000000004},"min":{"1min":0.27900000000000003,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":1.968,"5min":202.12,"15min":232.21000000000001},"last":0.45700000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":5.1989999999999998,"15min":5.1799999999999997},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.221},"max":{"1min":0.999,"5min":181.94399999999999,"15min":234.404},"last":0.503},{"interface":"front","average":{"1min":0.75900000000000001,"5min":5.29,"15min":5.0469999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":6.6829999999999998,"5min":181.98500000000001,"15min":236.33699999999999},"last":2.665}]},{"osd":4,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":1.5569999999999999,"15min":1.4339999999999999},"min":{"1min":0.32700000000000001,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":2.8210000000000002,"5min":12.042999999999999,"15min":12.042999999999999},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.5609999999999999,"15min":1.498},"min":{"1min":0.34499999999999997,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":2.7810000000000001,"5min":18.108000000000001,"15min":18.108000000000001},"last":0.63700000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":3.2879999999999998,"15min":1.8260000000000001},"min":{"1min":0.307,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.0460000000000003,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.91300000000000003,"5min":3.2890000000000001,"15min":2.141},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.7450000000000001,"5min":151.821,"15min":151.821},"last":0.76200000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.6850000000000001,"15min":1.954},"min":{"1min":0.34699999999999998,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":5.0960000000000001,"5min":8.9269999999999996,"15min":74.087999999999994},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.3300000000000001,"5min":1.885,"15min":2.157},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.32800000000000001},"max":{"1min":6.8920000000000003,"5min":11.927,"15min":73.974999999999994},"last":0.78300000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.7390000000000001,"15min":2.6099999999999999},"min":{"1min":0.50600000000000001,"5min":0.38700000000000001,"15min":0.32900000000000001},"max":{"1min":1.3069999999999999,"5min":44.241,"15min":78.308000000000007},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.8240000000000001,"15min":2.484},"min":{"1min":0.37,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":2.1339999999999999,"5min":43.920999999999999,"15min":77.204999999999998},"last":0.71899999999999997}]}]},{"osd":0,"up_from":91,"seq":390842024321,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1537848,"kb_used_data":335792,"kb_used_omap":5,"kb_used_meta":1202042,"kb_avail":92207304,"statfs":{"total":95995035648,"available":94420279296,"internally_reserved":0,"allocated":343851008,"data_stored":341049929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1230891642},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:35:32 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":10.577,"15min":6.7670000000000003},"min":{"1min":0.20499999999999999,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":9.0310000000000006,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.754,"5min":10.407999999999999,"15min":5.9059999999999997},"min":{"1min":0.20200000000000001,"5min":0.19700000000000001,"15min":0.17599999999999999},"max":{"1min":5.4729999999999999,"5min":297.72399999999999,"15min":297.72399999999999},"last":1.0249999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:35:57 2023","interfaces":[{"interface":"back","average":{"1min":3.2130000000000001,"5min":10.026999999999999,"15min":7.5819999999999999},"min":{"1min":0.26200000000000001,"5min":0.17699999999999999,"15min":0.17199999999999999},"max":{"1min":40.326999999999998,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.56999999999999995},{"interface":"front","average":{"1min":3.1720000000000002,"5min":10.18,"15min":7.6520000000000001},"min":{"1min":0.22900000000000001,"5min":0.19800000000000001,"15min":0.17599999999999999},"max":{"1min":40.396000000000001,"5min":283.161,"15min":283.161},"last":0.29799999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:35:32 2023","interfaces":[{"interface":"back","average":{"1min":2.7360000000000002,"5min":8.6509999999999998,"15min":5.4589999999999996},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":41.191000000000003,"5min":220.732,"15min":220.732},"last":0.443},{"interface":"front","average":{"1min":2.9390000000000001,"5min":9.0440000000000005,"15min":5.5570000000000004},"min":{"1min":0.28799999999999998,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":46.688000000000002,"5min":218.78200000000001,"15min":218.78200000000001},"last":0.52800000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":2.5859999999999999,"15min":2.8660000000000001},"min":{"1min":0.33300000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.4500000000000002,"5min":108.01900000000001,"15min":138.81299999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.25,"5min":2.7490000000000001,"15min":3.4550000000000001},"min":{"1min":0.35499999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.3670000000000009,"5min":107.898,"15min":171.251},"last":0.49399999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":1.7310000000000001,"15min":2.8340000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32100000000000001},"max":{"1min":15.076000000000001,"5min":15.076000000000001,"15min":171.30699999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.5649999999999999,"5min":1.4790000000000001,"15min":2.0880000000000001},"min":{"1min":0.35499999999999998,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":6.5279999999999996,"5min":9.9000000000000004,"15min":139.40799999999999},"last":0.48399999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":2.6360000000000001,"15min":3.161},"min":{"1min":0.38,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":9.968,"5min":108.67,"15min":171.33600000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.915,"5min":2.6699999999999999,"15min":3.4209999999999998},"min":{"1min":0.38900000000000001,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":9.157,"5min":108.804,"15min":184.13499999999999},"last":0.51500000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":1.373,"15min":2.6099999999999999},"min":{"1min":0.38400000000000001,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":0.751,"5min":9.452,"15min":138.77699999999999},"last":0.37},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.6379999999999999,"15min":2.5030000000000001},"min":{"1min":0.379,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":1.839,"5min":9.9670000000000005,"15min":139.37899999999999},"last":0.61799999999999999}]}]},{"osd":2,"up_from":199,"seq":854698492142,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3504500,"kb_used_data":2652396,"kb_used_omap":7,"kb_used_meta":852088,"kb_avail":90240652,"statfs":{"total":95995035648,"available":92406427648,"internally_reserved":0,"allocated":2716053504,"data_stored":2713206093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":872538193},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":2.2080000000000002,"15min":3.8599999999999999},"min":{"1min":0.186,"5min":0.186,"15min":0.16700000000000001},"max":{"1min":4.7590000000000003,"5min":61.950000000000003,"15min":119.277},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.61899999999999999,"5min":2.153,"15min":3.6669999999999998},"min":{"1min":0.23999999999999999,"5min":0.20399999999999999,"15min":0.18099999999999999},"max":{"1min":2.5070000000000001,"5min":61.984999999999999,"15min":119.56100000000001},"last":0.25900000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.911,"5min":5.3010000000000002,"15min":4.5940000000000003},"min":{"1min":0.25,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":8.3640000000000008,"5min":106.765,"15min":157.98699999999999},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.79000000000000004,"5min":4.8959999999999999,"15min":4.577},"min":{"1min":0.253,"5min":0.222,"15min":0.222},"max":{"1min":4.0940000000000003,"5min":106.732,"15min":158.03800000000001},"last":0.31}]},{"osd":3,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":5.7800000000000002,"15min":4.7770000000000001},"min":{"1min":0.255,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":8.3070000000000004,"5min":106.70399999999999,"15min":119.366},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.65800000000000003,"5min":5.4169999999999998,"15min":4.2709999999999999},"min":{"1min":0.29299999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":3.375,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.35799999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":2.036,"15min":2.262},"min":{"1min":0.36099999999999999,"5min":0.33700000000000002,"15min":0.25},"max":{"1min":2.6070000000000002,"5min":62.081000000000003,"15min":119.88200000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.4590000000000001,"5min":2.3860000000000001,"15min":1.865},"min":{"1min":0.36899999999999999,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":8.343,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.622}]},{"osd":5,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":3.0299999999999998,"15min":2.6480000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.26200000000000001},"max":{"1min":2.548,"5min":91.207999999999998,"15min":119.919},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.4550000000000001,"5min":3.2749999999999999,"15min":2.6160000000000001},"min":{"1min":0.33500000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.3889999999999993,"5min":91.150999999999996,"15min":119.306},"last":0.66900000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":3.5750000000000002,"15min":3.0019999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":2.5819999999999999,"5min":107.018,"15min":119.943},"last":0.504},{"interface":"front","average":{"1min":1.486,"5min":2.6440000000000001,"15min":2.1600000000000001},"min":{"1min":0.371,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":8.4060000000000006,"5min":62.493000000000002,"15min":62.493000000000002},"last":0.59199999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":2.2010000000000001,"15min":1.9179999999999999},"min":{"1min":0.40000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.381,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.60099999999999998,"5min":3.464,"15min":2.3620000000000001},"min":{"1min":0.42699999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.417,"5min":106.809,"15min":106.809},"last":0.69399999999999995}]}]},{"osd":3,"up_from":148,"seq":635655160111,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2295076,"kb_used_data":1397532,"kb_used_omap":2,"kb_used_meta":897533,"kb_avail":91450076,"statfs":{"total":95995035648,"available":93644877824,"internally_reserved":0,"allocated":1431072768,"data_stored":1428294238,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":919073845},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7390000000000001,"5min":1.4059999999999999,"15min":2.79},"min":{"1min":0.19600000000000001,"5min":0.19400000000000001,"15min":0.158},"max":{"1min":15.603,"5min":17.213000000000001,"15min":103.866},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.5880000000000001,"5min":1.877,"15min":2.9049999999999998},"min":{"1min":0.223,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":12.202,"5min":22.754000000000001,"15min":101.884},"last":0.32000000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":3.2200000000000002,"15min":2.9969999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19600000000000001},"max":{"1min":7.3319999999999999,"5min":56.119,"15min":85.242999999999995},"last":0.248},{"interface":"front","average":{"1min":1.4910000000000001,"5min":2.427,"15min":2.7999999999999998},"min":{"1min":0.22800000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":11.228,"5min":53.962000000000003,"15min":85.302000000000007},"last":0.54200000000000004}]},{"osd":2,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":4.7770000000000001,"15min":4.7859999999999996},"min":{"1min":0.25800000000000001,"5min":0.25,"15min":0.19800000000000001},"max":{"1min":2.1190000000000002,"5min":73.742000000000004,"15min":101.279},"last":1.9239999999999999},{"interface":"front","average":{"1min":0.45000000000000001,"5min":4.6349999999999998,"15min":4.6130000000000004},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.214},"max":{"1min":0.86699999999999999,"5min":76.525999999999996,"15min":101.294},"last":1.8879999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.3959999999999999,"15min":1.339},"min":{"1min":0.28799999999999998,"5min":0.28499999999999998,"15min":0.26700000000000002},"max":{"1min":11.209,"5min":11.209,"15min":11.209},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.2829999999999999,"5min":1.905,"15min":1.8049999999999999},"min":{"1min":0.33500000000000002,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":7.3940000000000001,"5min":11.494999999999999,"15min":24.782},"last":0.38200000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4410000000000001,"5min":6.8159999999999998,"15min":3.4289999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":7.3120000000000003,"5min":214.43700000000001,"15min":214.43700000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.2470000000000001,"5min":6.7569999999999997,"15min":3.4900000000000002},"min":{"1min":0.34699999999999998,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":7.2290000000000001,"5min":214.46600000000001,"15min":214.46600000000001},"last":0.39500000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":2.7090000000000001,"15min":2.3159999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":9.798,"5min":58.932000000000002,"15min":58.932000000000002},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.25,"5min":2.8660000000000001,"15min":2.3170000000000002},"min":{"1min":0.36499999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.202,"5min":59.186,"15min":59.186},"last":0.438}]},{"osd":7,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":4.7480000000000002,"15min":3.2069999999999999},"min":{"1min":0.38700000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.415,"5min":192.69,"15min":192.69},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.753,"5min":5.1289999999999996,"15min":3.6949999999999998},"min":{"1min":0.43099999999999999,"5min":0.39200000000000002,"15min":0.36699999999999999},"max":{"1min":2.6560000000000001,"5min":192.65600000000001,"15min":192.65600000000001},"last":1.2769999999999999}]}]},{"osd":4,"up_from":0,"seq":128849019430,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757793,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3148156,"kb_used_data":2421428,"kb_used_omap":0,"kb_used_meta":726720,"kb_avail":90596996,"statfs":{"total":95995035648,"available":92771323904,"internally_reserved":0,"allocated":2479542272,"data_stored":2477915684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":744161280},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.3720000000000001,"15min":3.48},"min":{"1min":0.254,"5min":0.20899999999999999,"15min":0.19500000000000001},"max":{"1min":3.0089999999999999,"5min":21.109000000000002,"15min":113.869},"last":0.252},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.6699999999999999,"15min":3.8650000000000002},"min":{"1min":0.25,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":5.1280000000000001,"5min":20.876999999999999,"15min":138.55199999999999},"last":0.314}]},{"osd":1,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.772,"15min":2.323},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.23100000000000001},"max":{"1min":3.629,"5min":29.007999999999999,"15min":69.816999999999993},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.9390000000000001,"15min":2.665},"min":{"1min":0.29999999999999999,"5min":0.249,"15min":0.249},"max":{"1min":4.4859999999999998,"5min":29.053999999999998,"15min":70.268000000000001},"last":0.46600000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:35:47 2023","interfaces":[{"interface":"back","average":{"1min":7.7469999999999999,"5min":6.5350000000000001,"15min":4.609},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.26900000000000002},"max":{"1min":145.93199999999999,"5min":208.738,"15min":208.738},"last":60.441000000000003},{"interface":"front","average":{"1min":1.0269999999999999,"5min":5.0499999999999998,"15min":4.1150000000000002},"min":{"1min":0.31,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":2.859,"5min":208.631,"15min":208.631},"last":60.520000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:35:44 2023","interfaces":[{"interface":"back","average":{"1min":6.4889999999999999,"5min":4.1520000000000001,"15min":3.6589999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":122.071,"5min":122.071,"15min":122.071},"last":0.38500000000000001},{"interface":"front","average":{"1min":6.4669999999999996,"5min":4.4969999999999999,"15min":3.8359999999999999},"min":{"1min":0.35799999999999998,"5min":0.35399999999999998,"15min":0.318},"max":{"1min":122.09699999999999,"5min":122.09699999999999,"15min":122.09699999999999},"last":0.35199999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.151,"15min":0.92000000000000004},"min":{"1min":0.34200000000000003,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":3.5499999999999998,"5min":9.2690000000000001,"15min":19.962},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.093,"15min":0.872},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.22900000000000001},"max":{"1min":2.4990000000000001,"5min":10.273999999999999,"15min":20.829999999999998},"last":0.44}]},{"osd":6,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":7.2519999999999998,"5min":2.3799999999999999,"15min":1.635},"min":{"1min":0.32400000000000001,"5min":0.27600000000000002,"15min":0.26900000000000002},"max":{"1min":148.36600000000001,"5min":148.36600000000001,"15min":148.36600000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":7.7210000000000001,"5min":2.9420000000000002,"15min":1.869},"min":{"1min":0.34599999999999997,"5min":0.307,"15min":0.26000000000000001},"max":{"1min":148.42500000000001,"5min":148.42500000000001,"15min":148.42500000000001},"last":0.378}]},{"osd":7,"last update":"Sat Sep 23 23:35:47 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.3160000000000001,"15min":2.625},"min":{"1min":0.34699999999999998,"5min":0.34599999999999997,"15min":0.317},"max":{"1min":1.843,"5min":15.497,"15min":179.28800000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.3480000000000001,"15min":2.7050000000000001},"min":{"1min":0.36699999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.867,"5min":15.653,"15min":179.428},"last":0.40600000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":335675392,"data_stored":335675392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1362563072,"data_stored":1362559000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2707030016,"data_stored":2707030016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1422004224,"data_stored":1422000152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2506817536,"data_stored":2506817536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2468544512,"data_stored":2468544512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1467940864,"data_stored":1467940864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1565200384,"data_stored":1565196312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:36:13.131 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:36:13.224 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T23:36:13.307 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:13.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:13.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:13.391 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:13 smithi110 ceph-mon[212939]: from='client.36789 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:13.391 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:13 smithi110 ceph-mon[212939]: pgmap v1642: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 25900/196179 objects degraded (13.202%); 3989/196179 objects misplaced (2.033%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:36:13.391 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:36:13.392 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:13 smithi110 ceph-mon[221102]: from='client.36789 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:13.392 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:13 smithi110 ceph-mon[221102]: pgmap v1642: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 25900/196179 objects degraded (13.202%); 3989/196179 objects misplaced (2.033%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:36:13.392 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:36:13.411 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:13.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:13 smithi138 ceph-mon[193606]: from='client.36789 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:13.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:13 smithi138 ceph-mon[193606]: pgmap v1642: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 25900/196179 objects degraded (13.202%); 3989/196179 objects misplaced (2.033%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:36:13.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:36:13.458 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:13.519 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:13.611 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:36:13.613 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:13.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:13.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:13.717 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:13.746 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:13.855 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:13.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:13.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:13.927 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:13.959 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:14.003 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:14.283 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:14 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 25900/196179 objects degraded (13.202%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:14.284 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:14 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 25900/196179 objects degraded (13.202%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:14.369 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:14.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:14.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:14 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 25900/196179 objects degraded (13.202%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:14.473 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:14.501 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:14.737 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:14.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:14.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:14.845 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:14.876 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:14.928 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:14.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:14.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:15.032 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:15.067 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:15.163 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:15 smithi138 ceph-mon[193606]: pgmap v1643: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 231 op/s; 25819/195234 objects degraded (13.225%); 3914/195234 objects misplaced (2.005%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:15.163 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:15 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1804662317' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:15.163 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:15 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:15.167 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:15.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:15.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:15.180 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:36:15.252 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:36:15.252 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:36:15.252 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:36:15.253 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:36:15.253 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.253 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:178614 11.3 11:c2d62249:::benchmark_data_smithi110_433746_object72734:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:36:15.253 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:19.570755+0000", 2023-09-23T23:36:15.253 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 535.67723558700004, 2023-09-23T23:36:15.253 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1817553470000002, 2023-09-23T23:36:15.253 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.253 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.253 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.253 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:36:15.253 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:15.253 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178614 2023-09-23T23:36:15.254 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.254 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.254 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.254 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.254 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570755+0000", 2023-09-23T23:36:15.254 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.254 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.254 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.254 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.254 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570755+0000", 2023-09-23T23:36:15.254 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.255 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.255 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.255 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.255 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570754+0000", 2023-09-23T23:36:15.255 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:36:15.255 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.255 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.255 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.255 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570759+0000", 2023-09-23T23:36:15.255 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7110000000000003e-06 2023-09-23T23:36:15.255 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.255 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.256 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.256 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570771+0000", 2023-09-23T23:36:15.256 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1965e-05 2023-09-23T23:36:15.256 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.256 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.256 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.256 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570776+0000", 2023-09-23T23:36:15.256 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0719999999999999e-06 2023-09-23T23:36:15.256 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.256 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.256 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.257 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570820+0000", 2023-09-23T23:36:15.257 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3640999999999998e-05 2023-09-23T23:36:15.257 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.257 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.257 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.257 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.570826+0000", 2023-09-23T23:36:15.257 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.088e-06 2023-09-23T23:36:15.257 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.257 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.257 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.257 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:20.540024+0000", 2023-09-23T23:36:15.257 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96919833200000005 2023-09-23T23:36:15.258 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.258 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.258 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.258 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:20.540036+0000", 2023-09-23T23:36:15.258 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1731e-05 2023-09-23T23:36:15.258 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.258 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.258 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.258 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.740668+0000", 2023-09-23T23:36:15.258 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.20063242 2023-09-23T23:36:15.258 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.258 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.259 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.259 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.740808+0000", 2023-09-23T23:36:15.259 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000139813 2023-09-23T23:36:15.259 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.259 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.259 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-09-23T23:36:15.259 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.741123+0000", 2023-09-23T23:36:15.259 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000314968 2023-09-23T23:36:15.259 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.259 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.259 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.259 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.744153+0000", 2023-09-23T23:36:15.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003029538 2023-09-23T23:36:15.260 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.260 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.260 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.260 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.744306+0000", 2023-09-23T23:36:15.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000153614 2023-09-23T23:36:15.260 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.260 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.260 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.260 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.750752+0000", 2023-09-23T23:36:15.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006445175 2023-09-23T23:36:15.260 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752382+0000", 2023-09-23T23:36:15.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0016305670000000001 2023-09-23T23:36:15.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752422+0000", 2023-09-23T23:36:15.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0157999999999997e-05 2023-09-23T23:36:15.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.262 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752511+0000", 2023-09-23T23:36:15.262 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8374000000000005e-05 2023-09-23T23:36:15.262 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.262 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.262 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.262 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.262 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:178615 11.3 11:c151980f:::benchmark_data_smithi110_433746_object72735:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:36:15.262 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:19.573793+0000", 2023-09-23T23:36:15.262 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 535.67419824499996, 2023-09-23T23:36:15.262 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1788738529999998, 2023-09-23T23:36:15.263 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.263 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.263 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.263 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:36:15.263 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:15.263 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178615 2023-09-23T23:36:15.263 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.263 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.263 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.263 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.263 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573793+0000", 2023-09-23T23:36:15.263 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.264 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.264 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573793+0000", 2023-09-23T23:36:15.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.264 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.264 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573792+0000", 2023-09-23T23:36:15.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:36:15.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.265 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.265 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573797+0000", 2023-09-23T23:36:15.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7960000000000003e-06 2023-09-23T23:36:15.265 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.265 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.265 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573809+0000", 2023-09-23T23:36:15.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2575000000000001e-05 2023-09-23T23:36:15.265 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.265 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.266 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573816+0000", 2023-09-23T23:36:15.266 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4670000000000001e-06 2023-09-23T23:36:15.266 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.266 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.266 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.266 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573907+0000", 2023-09-23T23:36:15.266 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1496000000000006e-05 2023-09-23T23:36:15.266 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.266 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.266 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:19.573930+0000", 2023-09-23T23:36:15.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2660999999999999e-05 2023-09-23T23:36:15.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.267 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:20.540059+0000", 2023-09-23T23:36:15.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96612945299999997 2023-09-23T23:36:15.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.267 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:20.540067+0000", 2023-09-23T23:36:15.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0919999999999998e-06 2023-09-23T23:36:15.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.268 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.742434+0000", 2023-09-23T23:36:15.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.202367089 2023-09-23T23:36:15.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.268 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.742547+0000", 2023-09-23T23:36:15.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112941 2023-09-23T23:36:15.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-09-23T23:36:15.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.742774+0000", 2023-09-23T23:36:15.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00022651400000000001 2023-09-23T23:36:15.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.744161+0000", 2023-09-23T23:36:15.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001387325 2023-09-23T23:36:15.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.270 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.270 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.744473+0000", 2023-09-23T23:36:15.270 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00031187499999999998 2023-09-23T23:36:15.270 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.270 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.270 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.750782+0000", 2023-09-23T23:36:15.270 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0063084719999999999 2023-09-23T23:36:15.270 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.270 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.270 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752547+0000", 2023-09-23T23:36:15.271 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0017649460000000001 2023-09-23T23:36:15.271 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.271 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.271 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.271 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752585+0000", 2023-09-23T23:36:15.271 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8121999999999999e-05 2023-09-23T23:36:15.271 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.271 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.271 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.271 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:21.752667+0000", 2023-09-23T23:36:15.271 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1868999999999998e-05 2023-09-23T23:36:15.272 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.272 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.272 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.272 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.272 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:190429 11.5 11:a76d3bc6:::benchmark_data_smithi110_433746_object84549:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e436)", 2023-09-23T23:36:15.272 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:27:41.139932+0000", 2023-09-23T23:36:15.272 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 514.10805854199998, 2023-09-23T23:36:15.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0172525750000001, 2023-09-23T23:36:15.272 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.272 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.272 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.272 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:36:15.273 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:15.273 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 190429 2023-09-23T23:36:15.273 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.273 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.273 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.273 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139932+0000", 2023-09-23T23:36:15.273 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.273 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.273 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.273 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139932+0000", 2023-09-23T23:36:15.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.274 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.274 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.274 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139932+0000", 2023-09-23T23:36:15.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:15.274 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.274 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.274 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139934+0000", 2023-09-23T23:36:15.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.187e-06 2023-09-23T23:36:15.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.275 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.275 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139939+0000", 2023-09-23T23:36:15.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1340000000000003e-06 2023-09-23T23:36:15.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.275 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.275 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.139943+0000", 2023-09-23T23:36:15.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4709999999999999e-06 2023-09-23T23:36:15.276 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.141339+0000", 2023-09-23T23:36:15.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001396481 2023-09-23T23:36:15.276 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:36:15.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.141345+0000", 2023-09-23T23:36:15.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8799999999999996e-06 2023-09-23T23:36:15.276 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.277 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.277 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.143057+0000", 2023-09-23T23:36:15.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0017121560000000001 2023-09-23T23:36:15.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.277 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.277 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.277 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:41.143062+0000", 2023-09-23T23:36:15.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.04e-06 2023-09-23T23:36:15.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.277 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.277 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.277 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.132701+0000", 2023-09-23T23:36:15.278 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98963869500000001 2023-09-23T23:36:15.278 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.278 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.278 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.132847+0000", 2023-09-23T23:36:15.278 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014609000000000001 2023-09-23T23:36:15.278 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.278 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:36:15.278 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.133176+0000", 2023-09-23T23:36:15.278 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00032873100000000002 2023-09-23T23:36:15.278 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.279 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.144793+0000", 2023-09-23T23:36:15.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011617647 2023-09-23T23:36:15.279 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.279 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157040+0000", 2023-09-23T23:36:15.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012246194 2023-09-23T23:36:15.279 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.280 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157117+0000", 2023-09-23T23:36:15.280 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7812999999999995e-05 2023-09-23T23:36:15.280 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.280 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.280 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.280 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157161+0000", 2023-09-23T23:36:15.280 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3009000000000001e-05 2023-09-23T23:36:15.280 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.280 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.280 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.280 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:27:42.157185+0000", 2023-09-23T23:36:15.281 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4426e-05 2023-09-23T23:36:15.281 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.281 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.281 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.281 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.281 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:203242 11.13 11:cd9783f2:::benchmark_data_smithi110_433746_object97362:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-09-23T23:36:15.281 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:28:02.344823+0000", 2023-09-23T23:36:15.281 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 492.90316771599998, 2023-09-23T23:36:15.281 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0388343680000001, 2023-09-23T23:36:15.281 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.281 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.282 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.282 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:36:15.282 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:15.282 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 203242 2023-09-23T23:36:15.282 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.282 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.282 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.282 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344823+0000", 2023-09-23T23:36:15.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.282 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.283 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.283 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344823+0000", 2023-09-23T23:36:15.283 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.283 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.283 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.283 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344822+0000", 2023-09-23T23:36:15.283 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:15.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.283 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.283 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344826+0000", 2023-09-23T23:36:15.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4530000000000004e-06 2023-09-23T23:36:15.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344838+0000", 2023-09-23T23:36:15.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.17e-05 2023-09-23T23:36:15.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344844+0000", 2023-09-23T23:36:15.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7139999999999998e-06 2023-09-23T23:36:15.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344927+0000", 2023-09-23T23:36:15.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2708999999999997e-05 2023-09-23T23:36:15.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.344939+0000", 2023-09-23T23:36:15.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2213e-05 2023-09-23T23:36:15.286 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.286 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.286 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.286 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.362662+0000", 2023-09-23T23:36:15.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0177228979999999 2023-09-23T23:36:15.286 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.286 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.286 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.286 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.362790+0000", 2023-09-23T23:36:15.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012857299999999999 2023-09-23T23:36:15.286 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,5", 2023-09-23T23:36:15.287 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.363083+0000", 2023-09-23T23:36:15.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00029265399999999999 2023-09-23T23:36:15.287 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.287 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.369254+0000", 2023-09-23T23:36:15.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0061712609999999999 2023-09-23T23:36:15.287 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383552+0000", 2023-09-23T23:36:15.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014297667 2023-09-23T23:36:15.288 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.288 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.288 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383603+0000", 2023-09-23T23:36:15.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1678000000000002e-05 2023-09-23T23:36:15.288 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.288 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.288 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383632+0000", 2023-09-23T23:36:15.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8711e-05 2023-09-23T23:36:15.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.289 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.289 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383658+0000", 2023-09-23T23:36:15.289 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5386000000000001e-05 2023-09-23T23:36:15.289 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.289 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.289 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.289 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35904.0:203250 11.13 11:c8573015:::benchmark_data_smithi110_433746_object97370:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e450)", 2023-09-23T23:36:15.290 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:28:02.375673+0000", 2023-09-23T23:36:15.290 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 492.87231828799997, 2023-09-23T23:36:15.290 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0081006640000001, 2023-09-23T23:36:15.290 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.290 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.290 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.290 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35904", 2023-09-23T23:36:15.290 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:15.290 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 203250 2023-09-23T23:36:15.290 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.290 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.290 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.291 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375673+0000", 2023-09-23T23:36:15.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.291 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.291 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.291 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375673+0000", 2023-09-23T23:36:15.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.291 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.291 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.292 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.292 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375671+0000", 2023-09-23T23:36:15.292 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:15.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.292 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.292 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.292 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375678+0000", 2023-09-23T23:36:15.292 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2099999999999996e-06 2023-09-23T23:36:15.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.292 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.292 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.292 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375696+0000", 2023-09-23T23:36:15.293 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7385e-05 2023-09-23T23:36:15.293 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.293 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.293 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.375703+0000", 2023-09-23T23:36:15.293 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5739999999999998e-06 2023-09-23T23:36:15.293 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.293 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.293 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.400000+0000", 2023-09-23T23:36:15.293 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.024297062000000001 2023-09-23T23:36:15.293 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.294 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:02.400008+0000", 2023-09-23T23:36:15.294 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7389999999999999e-06 2023-09-23T23:36:15.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.294 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.294 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.364392+0000", 2023-09-23T23:36:15.294 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96438375899999995 2023-09-23T23:36:15.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.294 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.295 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.364510+0000", 2023-09-23T23:36:15.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117889 2023-09-23T23:36:15.295 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.295 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,5", 2023-09-23T23:36:15.295 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.364768+0000", 2023-09-23T23:36:15.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025844600000000001 2023-09-23T23:36:15.295 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.295 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.295 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.369259+0000", 2023-09-23T23:36:15.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0044911650000000001 2023-09-23T23:36:15.295 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.296 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.296 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383576+0000", 2023-09-23T23:36:15.296 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014316297 2023-09-23T23:36:15.296 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.296 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.296 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383688+0000", 2023-09-23T23:36:15.296 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112046 2023-09-23T23:36:15.296 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.296 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.297 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383735+0000", 2023-09-23T23:36:15.297 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7889000000000002e-05 2023-09-23T23:36:15.297 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.297 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.297 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.297 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:28:03.383773+0000", 2023-09-23T23:36:15.297 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7795e-05 2023-09-23T23:36:15.297 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.297 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.297 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.297 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.298 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.298 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71516 13.e 13:7de122bf:::benchmark_data_smithi110_541928_object71515:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e555)", 2023-09-23T23:36:15.298 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.528660+0000", 2023-09-23T23:36:15.298 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 144.71933044100001, 2023-09-23T23:36:15.298 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2167170459999999, 2023-09-23T23:36:15.298 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.298 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:36:15.298 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.298 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:36:15.298 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:36:15.298 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71516 2023-09-23T23:36:15.298 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.299 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.299 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.299 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528660+0000", 2023-09-23T23:36:15.299 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.299 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.299 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528660+0000", 2023-09-23T23:36:15.299 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528659+0000", 2023-09-23T23:36:15.300 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:15.300 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.300 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528700+0000", 2023-09-23T23:36:15.300 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0859999999999998e-05 2023-09-23T23:36:15.300 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.300 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528712+0000", 2023-09-23T23:36:15.301 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2262000000000001e-05 2023-09-23T23:36:15.301 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.301 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.301 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.301 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528719+0000", 2023-09-23T23:36:15.301 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3339999999999998e-06 2023-09-23T23:36:15.301 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.301 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.301 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.301 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531320+0000", 2023-09-23T23:36:15.301 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002601506 2023-09-23T23:36:15.302 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.302 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.302 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.302 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531460+0000", 2023-09-23T23:36:15.302 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013956799999999999 2023-09-23T23:36:15.302 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.302 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.302 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:36:15.302 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531617+0000", 2023-09-23T23:36:15.302 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015701199999999999 2023-09-23T23:36:15.302 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.302 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.303 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.303 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.534591+0000", 2023-09-23T23:36:15.303 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029740880000000002 2023-09-23T23:36:15.303 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.303 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.303 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.303 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.545743+0000", 2023-09-23T23:36:15.303 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011152331 2023-09-23T23:36:15.303 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.303 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.303 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.303 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.545752+0000", 2023-09-23T23:36:15.304 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0000000000000002e-06 2023-09-23T23:36:15.304 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.304 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.304 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.304 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552315+0000", 2023-09-23T23:36:15.304 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.006563179 2023-09-23T23:36:15.304 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.304 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.304 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.304 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552330+0000", 2023-09-23T23:36:15.304 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4734e-05 2023-09-23T23:36:15.305 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.305 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.305 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.305 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745274+0000", 2023-09-23T23:36:15.305 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1929441679999999 2023-09-23T23:36:15.305 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.305 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.305 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.305 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745377+0000", 2023-09-23T23:36:15.305 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103303 2023-09-23T23:36:15.305 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.305 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.306 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.306 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.306 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.306 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71528 13.e 13:790d8dd9:::benchmark_data_smithi110_541928_object71527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:36:15.306 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.572698+0000", 2023-09-23T23:36:15.306 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 144.67529267200001, 2023-09-23T23:36:15.306 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2677280570000002, 2023-09-23T23:36:15.306 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.306 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.306 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.306 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:36:15.306 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:36:15.307 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71528 2023-09-23T23:36:15.307 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.307 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.307 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.307 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.307 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572698+0000", 2023-09-23T23:36:15.307 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.307 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.307 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.307 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.307 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572698+0000", 2023-09-23T23:36:15.307 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.308 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.308 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.308 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572697+0000", 2023-09-23T23:36:15.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:15.308 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.308 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.308 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572737+0000", 2023-09-23T23:36:15.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0278000000000003e-05 2023-09-23T23:36:15.308 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.308 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.309 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572746+0000", 2023-09-23T23:36:15.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3340000000000008e-06 2023-09-23T23:36:15.309 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.309 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.309 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.309 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572753+0000", 2023-09-23T23:36:15.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3899999999999998e-06 2023-09-23T23:36:15.309 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.309 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.309 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.309 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572817+0000", 2023-09-23T23:36:15.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4576e-05 2023-09-23T23:36:15.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.310 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.310 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572827+0000", 2023-09-23T23:36:15.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.978e-06 2023-09-23T23:36:15.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.310 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.310 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552357+0000", 2023-09-23T23:36:15.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97953002600000005 2023-09-23T23:36:15.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552365+0000", 2023-09-23T23:36:15.311 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5390000000000002e-06 2023-09-23T23:36:15.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745405+0000", 2023-09-23T23:36:15.311 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1930404109999999 2023-09-23T23:36:15.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745543+0000", 2023-09-23T23:36:15.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013780300000000001 2023-09-23T23:36:15.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.312 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:36:15.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745737+0000", 2023-09-23T23:36:15.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019410400000000001 2023-09-23T23:36:15.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.312 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.759566+0000", 2023-09-23T23:36:15.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013828844999999999 2023-09-23T23:36:15.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.313 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.313 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.768156+0000", 2023-09-23T23:36:15.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085902789999999993 2023-09-23T23:36:15.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.313 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.313 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.836351+0000", 2023-09-23T23:36:15.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.068194406999999999 2023-09-23T23:36:15.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.836426+0000", 2023-09-23T23:36:15.314 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4889999999999996e-05 2023-09-23T23:36:15.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.840426+0000", 2023-09-23T23:36:15.314 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040005659999999997 2023-09-23T23:36:15.314 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.314 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.314 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.315 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.315 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71532 13.e 13:7da393ae:::benchmark_data_smithi110_541928_object71531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:36:15.315 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.575009+0000", 2023-09-23T23:36:15.315 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 144.672981585, 2023-09-23T23:36:15.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2696883859999999, 2023-09-23T23:36:15.315 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.315 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.315 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.315 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:36:15.315 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:36:15.316 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71532 2023-09-23T23:36:15.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.316 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.316 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.316 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575009+0000", 2023-09-23T23:36:15.316 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.316 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.316 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575009+0000", 2023-09-23T23:36:15.316 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.317 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575004+0000", 2023-09-23T23:36:15.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999948 2023-09-23T23:36:15.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.317 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575068+0000", 2023-09-23T23:36:15.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3359000000000001e-05 2023-09-23T23:36:15.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.317 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.318 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575081+0000", 2023-09-23T23:36:15.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3434999999999999e-05 2023-09-23T23:36:15.318 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.318 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.318 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.318 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575090+0000", 2023-09-23T23:36:15.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7490000000000004e-06 2023-09-23T23:36:15.318 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.318 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.318 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.318 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575154+0000", 2023-09-23T23:36:15.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4023999999999996e-05 2023-09-23T23:36:15.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.319 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.319 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575166+0000", 2023-09-23T23:36:15.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2033e-05 2023-09-23T23:36:15.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.319 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.319 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552388+0000", 2023-09-23T23:36:15.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97722166399999999 2023-09-23T23:36:15.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552395+0000", 2023-09-23T23:36:15.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3429999999999998e-06 2023-09-23T23:36:15.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.748633+0000", 2023-09-23T23:36:15.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1962377799999999 2023-09-23T23:36:15.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.748799+0000", 2023-09-23T23:36:15.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016630200000000001 2023-09-23T23:36:15.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:36:15.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.748992+0000", 2023-09-23T23:36:15.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019316 2023-09-23T23:36:15.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.759572+0000", 2023-09-23T23:36:15.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010580032 2023-09-23T23:36:15.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.768191+0000", 2023-09-23T23:36:15.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086191280000000002 2023-09-23T23:36:15.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.840481+0000", 2023-09-23T23:36:15.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.072289500000000007 2023-09-23T23:36:15.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.323 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.323 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.323 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.840542+0000", 2023-09-23T23:36:15.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1509999999999999e-05 2023-09-23T23:36:15.323 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.323 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.323 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.323 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.844698+0000", 2023-09-23T23:36:15.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041553880000000003 2023-09-23T23:36:15.323 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.323 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.323 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.324 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.324 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.324 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77337 13.d 13:ba15f28f:::benchmark_data_smithi110_541928_object77336:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:36:15.324 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.706918+0000", 2023-09-23T23:36:15.324 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 131.54107260999999, 2023-09-23T23:36:15.324 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1549711069999999, 2023-09-23T23:36:15.324 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.324 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:36:15.324 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.324 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:36:15.324 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:36:15.324 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77337 2023-09-23T23:36:15.325 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.325 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.325 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.325 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.325 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706918+0000", 2023-09-23T23:36:15.325 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.325 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.325 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.325 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.325 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706918+0000", 2023-09-23T23:36:15.325 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.325 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706917+0000", 2023-09-23T23:36:15.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:36:15.326 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706970+0000", 2023-09-23T23:36:15.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2435000000000002e-05 2023-09-23T23:36:15.326 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706986+0000", 2023-09-23T23:36:15.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6252999999999998e-05 2023-09-23T23:36:15.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706993+0000", 2023-09-23T23:36:15.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1439999999999997e-06 2023-09-23T23:36:15.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709103+0000", 2023-09-23T23:36:15.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021098419999999998 2023-09-23T23:36:15.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709112+0000", 2023-09-23T23:36:15.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3640000000000005e-06 2023-09-23T23:36:15.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.860680+0000", 2023-09-23T23:36:15.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1515676990000001 2023-09-23T23:36:15.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.329 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.329 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.861889+0000", 2023-09-23T23:36:15.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012094930000000001 2023-09-23T23:36:15.329 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.329 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.329 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.329 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77365 13.d 13:bd3fed09:::benchmark_data_smithi110_541928_object77364:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:36:15.330 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.707869+0000", 2023-09-23T23:36:15.330 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 131.54012233200001, 2023-09-23T23:36:15.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1996441330000001, 2023-09-23T23:36:15.330 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.330 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.330 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.330 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:36:15.330 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:36:15.330 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77365 2023-09-23T23:36:15.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.331 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.331 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.331 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707869+0000", 2023-09-23T23:36:15.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.331 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.331 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707869+0000", 2023-09-23T23:36:15.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707868+0000", 2023-09-23T23:36:15.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:36:15.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707912+0000", 2023-09-23T23:36:15.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4286999999999998e-05 2023-09-23T23:36:15.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707922+0000", 2023-09-23T23:36:15.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0132000000000001e-05 2023-09-23T23:36:15.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707929+0000", 2023-09-23T23:36:15.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4690000000000002e-06 2023-09-23T23:36:15.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709143+0000", 2023-09-23T23:36:15.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012139539999999999 2023-09-23T23:36:15.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709148+0000", 2023-09-23T23:36:15.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8420000000000001e-06 2023-09-23T23:36:15.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.861913+0000", 2023-09-23T23:36:15.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.152764984 2023-09-23T23:36:15.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.862371+0000", 2023-09-23T23:36:15.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00045722599999999998 2023-09-23T23:36:15.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:36:15.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.862643+0000", 2023-09-23T23:36:15.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000271983 2023-09-23T23:36:15.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.872385+0000", 2023-09-23T23:36:15.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0097426379999999996 2023-09-23T23:36:15.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.905589+0000", 2023-09-23T23:36:15.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.033203780000000002 2023-09-23T23:36:15.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.905624+0000", 2023-09-23T23:36:15.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5058000000000002e-05 2023-09-23T23:36:15.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.905662+0000", 2023-09-23T23:36:15.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7523e-05 2023-09-23T23:36:15.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907513+0000", 2023-09-23T23:36:15.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0018510320000000001 2023-09-23T23:36:15.337 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.337 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.337 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.338 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77391 13.d 13:bfca221d:::benchmark_data_smithi110_541928_object77390:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:36:15.338 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.707935+0000", 2023-09-23T23:36:15.338 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 131.54005587099999, 2023-09-23T23:36:15.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2024361749999999, 2023-09-23T23:36:15.338 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.338 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.338 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.338 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:36:15.338 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:36:15.338 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77391 2023-09-23T23:36:15.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.339 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707935+0000", 2023-09-23T23:36:15.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707935+0000", 2023-09-23T23:36:15.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.340 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707935+0000", 2023-09-23T23:36:15.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:15.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.340 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707971+0000", 2023-09-23T23:36:15.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6470000000000001e-05 2023-09-23T23:36:15.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707977+0000", 2023-09-23T23:36:15.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1489999999999999e-06 2023-09-23T23:36:15.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707982+0000", 2023-09-23T23:36:15.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5349999999999998e-06 2023-09-23T23:36:15.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709170+0000", 2023-09-23T23:36:15.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011882080000000001 2023-09-23T23:36:15.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709176+0000", 2023-09-23T23:36:15.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.297e-06 2023-09-23T23:36:15.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.866177+0000", 2023-09-23T23:36:15.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1570007 2023-09-23T23:36:15.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.866317+0000", 2023-09-23T23:36:15.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013973499999999999 2023-09-23T23:36:15.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:36:15.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.866488+0000", 2023-09-23T23:36:15.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000171495 2023-09-23T23:36:15.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.875118+0000", 2023-09-23T23:36:15.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086296299999999992 2023-09-23T23:36:15.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.905607+0000", 2023-09-23T23:36:15.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.030489532999999999 2023-09-23T23:36:15.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907981+0000", 2023-09-23T23:36:15.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023734239999999998 2023-09-23T23:36:15.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.908040+0000", 2023-09-23T23:36:15.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9463000000000003e-05 2023-09-23T23:36:15.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910371+0000", 2023-09-23T23:36:15.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023309279999999999 2023-09-23T23:36:15.345 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.346 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.346 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.346 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77394 13.d 13:bd1f7538:::benchmark_data_smithi110_541928_object77393:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:36:15.346 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.707987+0000", 2023-09-23T23:36:15.346 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 131.54000412900001, 2023-09-23T23:36:15.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.204224143, 2023-09-23T23:36:15.346 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.346 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.346 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.347 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:36:15.347 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:36:15.347 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77394 2023-09-23T23:36:15.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.347 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707987+0000", 2023-09-23T23:36:15.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.348 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707987+0000", 2023-09-23T23:36:15.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.348 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.348 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.348 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707986+0000", 2023-09-23T23:36:15.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:15.348 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.348 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.348 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708018+0000", 2023-09-23T23:36:15.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1340999999999997e-05 2023-09-23T23:36:15.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708024+0000", 2023-09-23T23:36:15.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9050000000000002e-06 2023-09-23T23:36:15.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708027+0000", 2023-09-23T23:36:15.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6550000000000002e-06 2023-09-23T23:36:15.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.350 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.350 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709199+0000", 2023-09-23T23:36:15.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011713400000000001 2023-09-23T23:36:15.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.350 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.350 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709204+0000", 2023-09-23T23:36:15.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4480000000000002e-06 2023-09-23T23:36:15.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.869263+0000", 2023-09-23T23:36:15.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1600593370000001 2023-09-23T23:36:15.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.869419+0000", 2023-09-23T23:36:15.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015545899999999999 2023-09-23T23:36:15.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:36:15.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.869589+0000", 2023-09-23T23:36:15.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000170225 2023-09-23T23:36:15.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889913+0000", 2023-09-23T23:36:15.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020323536 2023-09-23T23:36:15.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907553+0000", 2023-09-23T23:36:15.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017640769000000001 2023-09-23T23:36:15.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910571+0000", 2023-09-23T23:36:15.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030181119999999999 2023-09-23T23:36:15.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910620+0000", 2023-09-23T23:36:15.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8891000000000003e-05 2023-09-23T23:36:15.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912211+0000", 2023-09-23T23:36:15.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015906270000000001 2023-09-23T23:36:15.354 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.354 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.354 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.354 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77463 13.d 13:b8eb00c3:::benchmark_data_smithi110_541928_object77462:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:36:15.354 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.708031+0000", 2023-09-23T23:36:15.354 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 131.539959428, 2023-09-23T23:36:15.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.206478288, 2023-09-23T23:36:15.355 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.355 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.355 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.355 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:36:15.355 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:36:15.355 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77463 2023-09-23T23:36:15.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.355 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708031+0000", 2023-09-23T23:36:15.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708031+0000", 2023-09-23T23:36:15.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708031+0000", 2023-09-23T23:36:15.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:15.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708086+0000", 2023-09-23T23:36:15.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4630999999999999e-05 2023-09-23T23:36:15.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708092+0000", 2023-09-23T23:36:15.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1800000000000001e-06 2023-09-23T23:36:15.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.358 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708096+0000", 2023-09-23T23:36:15.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.811e-06 2023-09-23T23:36:15.358 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.358 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.358 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709228+0000", 2023-09-23T23:36:15.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011318190000000001 2023-09-23T23:36:15.358 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.358 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.358 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709234+0000", 2023-09-23T23:36:15.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2979999999999997e-06 2023-09-23T23:36:15.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.359 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.359 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.359 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.872414+0000", 2023-09-23T23:36:15.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.163180543 2023-09-23T23:36:15.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.359 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.359 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.359 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.872546+0000", 2023-09-23T23:36:15.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013156499999999999 2023-09-23T23:36:15.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:36:15.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.872714+0000", 2023-09-23T23:36:15.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000167713 2023-09-23T23:36:15.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889920+0000", 2023-09-23T23:36:15.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017206255 2023-09-23T23:36:15.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907578+0000", 2023-09-23T23:36:15.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017658231 2023-09-23T23:36:15.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912462+0000", 2023-09-23T23:36:15.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048837949999999998 2023-09-23T23:36:15.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912506+0000", 2023-09-23T23:36:15.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4131999999999997e-05 2023-09-23T23:36:15.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.914510+0000", 2023-09-23T23:36:15.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002003632 2023-09-23T23:36:15.362 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.362 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.362 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.362 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77467 13.d 13:b265ce95:::benchmark_data_smithi110_541928_object77466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:36:15.362 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.708101+0000", 2023-09-23T23:36:15.362 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 131.53989001400001, 2023-09-23T23:36:15.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2086463730000001, 2023-09-23T23:36:15.363 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.363 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.363 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.363 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:36:15.363 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:36:15.363 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77467 2023-09-23T23:36:15.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.363 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708101+0000", 2023-09-23T23:36:15.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708101+0000", 2023-09-23T23:36:15.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708101+0000", 2023-09-23T23:36:15.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:15.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.365 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.365 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708157+0000", 2023-09-23T23:36:15.365 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6196999999999997e-05 2023-09-23T23:36:15.365 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.365 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.365 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708163+0000", 2023-09-23T23:36:15.365 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0939999999999996e-06 2023-09-23T23:36:15.365 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.365 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.365 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708168+0000", 2023-09-23T23:36:15.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1800000000000004e-06 2023-09-23T23:36:15.366 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.366 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.366 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.366 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709259+0000", 2023-09-23T23:36:15.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010911370000000001 2023-09-23T23:36:15.366 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.366 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.366 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.366 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709266+0000", 2023-09-23T23:36:15.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7870000000000004e-06 2023-09-23T23:36:15.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.875164+0000", 2023-09-23T23:36:15.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1658984800000001 2023-09-23T23:36:15.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.875272+0000", 2023-09-23T23:36:15.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010731299999999999 2023-09-23T23:36:15.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:36:15.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.875417+0000", 2023-09-23T23:36:15.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014540399999999999 2023-09-23T23:36:15.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889926+0000", 2023-09-23T23:36:15.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014509024000000001 2023-09-23T23:36:15.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907925+0000", 2023-09-23T23:36:15.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017998692 2023-09-23T23:36:15.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.914568+0000", 2023-09-23T23:36:15.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066434789999999999 2023-09-23T23:36:15.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.914630+0000", 2023-09-23T23:36:15.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1175999999999998e-05 2023-09-23T23:36:15.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.370 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.916747+0000", 2023-09-23T23:36:15.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021177470000000001 2023-09-23T23:36:15.370 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.370 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.370 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.370 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77471 13.d 13:bd1602ed:::benchmark_data_smithi110_541928_object77470:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:36:15.370 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.708173+0000", 2023-09-23T23:36:15.370 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 131.53981816300001, 2023-09-23T23:36:15.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2113011469999999, 2023-09-23T23:36:15.371 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.371 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.371 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.371 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:36:15.371 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:36:15.371 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77471 2023-09-23T23:36:15.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.371 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708173+0000", 2023-09-23T23:36:15.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708173+0000", 2023-09-23T23:36:15.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708172+0000", 2023-09-23T23:36:15.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:15.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708227+0000", 2023-09-23T23:36:15.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4215000000000002e-05 2023-09-23T23:36:15.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708234+0000", 2023-09-23T23:36:15.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0790000000000003e-06 2023-09-23T23:36:15.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708238+0000", 2023-09-23T23:36:15.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6450000000000004e-06 2023-09-23T23:36:15.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709564+0000", 2023-09-23T23:36:15.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0013253379999999999 2023-09-23T23:36:15.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709570+0000", 2023-09-23T23:36:15.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5570000000000001e-06 2023-09-23T23:36:15.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.877443+0000", 2023-09-23T23:36:15.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.167873216 2023-09-23T23:36:15.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.877572+0000", 2023-09-23T23:36:15.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012810999999999999 2023-09-23T23:36:15.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:36:15.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.877732+0000", 2023-09-23T23:36:15.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000160376 2023-09-23T23:36:15.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889933+0000", 2023-09-23T23:36:15.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012201033999999999 2023-09-23T23:36:15.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910445+0000", 2023-09-23T23:36:15.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020511715999999999 2023-09-23T23:36:15.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.916902+0000", 2023-09-23T23:36:15.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064569379999999997 2023-09-23T23:36:15.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.916958+0000", 2023-09-23T23:36:15.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6443000000000002e-05 2023-09-23T23:36:15.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.919474+0000", 2023-09-23T23:36:15.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002515804 2023-09-23T23:36:15.378 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.378 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.378 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.379 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77477 13.d 13:b03c7b50:::benchmark_data_smithi110_541928_object77476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:36:15.379 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.708243+0000", 2023-09-23T23:36:15.379 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 131.53974817299999, 2023-09-23T23:36:15.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.213803755, 2023-09-23T23:36:15.379 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.379 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.379 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.379 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:36:15.379 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:36:15.379 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77477 2023-09-23T23:36:15.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.380 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708243+0000", 2023-09-23T23:36:15.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708243+0000", 2023-09-23T23:36:15.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708242+0000", 2023-09-23T23:36:15.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:15.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708286+0000", 2023-09-23T23:36:15.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3625999999999999e-05 2023-09-23T23:36:15.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708292+0000", 2023-09-23T23:36:15.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0100000000000001e-06 2023-09-23T23:36:15.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708296+0000", 2023-09-23T23:36:15.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1860000000000002e-06 2023-09-23T23:36:15.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709595+0000", 2023-09-23T23:36:15.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012991089999999999 2023-09-23T23:36:15.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709602+0000", 2023-09-23T23:36:15.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6100000000000002e-06 2023-09-23T23:36:15.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.386 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.386 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.879894+0000", 2023-09-23T23:36:15.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.170292077 2023-09-23T23:36:15.386 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.386 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.386 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.879990+0000", 2023-09-23T23:36:15.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6212000000000007e-05 2023-09-23T23:36:15.386 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:36:15.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.880120+0000", 2023-09-23T23:36:15.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012987 2023-09-23T23:36:15.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889937+0000", 2023-09-23T23:36:15.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0098164270000000008 2023-09-23T23:36:15.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910473+0000", 2023-09-23T23:36:15.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020536688000000001 2023-09-23T23:36:15.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.919527+0000", 2023-09-23T23:36:15.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0090542190000000005 2023-09-23T23:36:15.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.919586+0000", 2023-09-23T23:36:15.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8749999999999998e-05 2023-09-23T23:36:15.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.922046+0000", 2023-09-23T23:36:15.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024603099999999998 2023-09-23T23:36:15.389 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.389 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.389 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.389 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77503 13.d 13:b2378296:::benchmark_data_smithi110_541928_object77502:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:36:15.389 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.781601+0000", 2023-09-23T23:36:15.390 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 131.46638984399999, 2023-09-23T23:36:15.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1428059340000001, 2023-09-23T23:36:15.390 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.390 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.390 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.390 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:36:15.390 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:36:15.390 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77503 2023-09-23T23:36:15.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.390 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781601+0000", 2023-09-23T23:36:15.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781601+0000", 2023-09-23T23:36:15.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781600+0000", 2023-09-23T23:36:15.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:15.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781628+0000", 2023-09-23T23:36:15.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8694999999999999e-05 2023-09-23T23:36:15.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781638+0000", 2023-09-23T23:36:15.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5959999999999993e-06 2023-09-23T23:36:15.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.393 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.393 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781643+0000", 2023-09-23T23:36:15.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4970000000000001e-06 2023-09-23T23:36:15.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.393 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.393 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781701+0000", 2023-09-23T23:36:15.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7757000000000003e-05 2023-09-23T23:36:15.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.393 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781711+0000", 2023-09-23T23:36:15.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4409999999999999e-06 2023-09-23T23:36:15.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.882289+0000", 2023-09-23T23:36:15.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1005785459999999 2023-09-23T23:36:15.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.882445+0000", 2023-09-23T23:36:15.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015557 2023-09-23T23:36:15.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:36:15.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.882635+0000", 2023-09-23T23:36:15.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000190087 2023-09-23T23:36:15.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.892019+0000", 2023-09-23T23:36:15.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0093840280000000009 2023-09-23T23:36:15.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910507+0000", 2023-09-23T23:36:15.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018488389000000001 2023-09-23T23:36:15.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.922095+0000", 2023-09-23T23:36:15.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011588144999999999 2023-09-23T23:36:15.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.922148+0000", 2023-09-23T23:36:15.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2661000000000003e-05 2023-09-23T23:36:15.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.397 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.397 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.924407+0000", 2023-09-23T23:36:15.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022589189999999999 2023-09-23T23:36:15.397 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.397 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.397 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.397 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77524 13.d 13:bc4b3c00:::benchmark_data_smithi110_541928_object77523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:36:15.398 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.815029+0000", 2023-09-23T23:36:15.398 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 131.43296165300001, 2023-09-23T23:36:15.398 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.112009437, 2023-09-23T23:36:15.398 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.398 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.398 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.398 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:36:15.398 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:36:15.398 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77524 2023-09-23T23:36:15.398 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.398 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815029+0000", 2023-09-23T23:36:15.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815029+0000", 2023-09-23T23:36:15.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815028+0000", 2023-09-23T23:36:15.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:15.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815060+0000", 2023-09-23T23:36:15.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2218999999999998e-05 2023-09-23T23:36:15.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815074+0000", 2023-09-23T23:36:15.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.43e-05 2023-09-23T23:36:15.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815081+0000", 2023-09-23T23:36:15.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0700000000000003e-06 2023-09-23T23:36:15.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815152+0000", 2023-09-23T23:36:15.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1408999999999993e-05 2023-09-23T23:36:15.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815162+0000", 2023-09-23T23:36:15.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9459999999999993e-06 2023-09-23T23:36:15.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.884789+0000", 2023-09-23T23:36:15.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.069627511 2023-09-23T23:36:15.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.403 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.403 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.884895+0000", 2023-09-23T23:36:15.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105283 2023-09-23T23:36:15.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.403 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:36:15.403 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.885012+0000", 2023-09-23T23:36:15.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011697900000000001 2023-09-23T23:36:15.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.404 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.892026+0000", 2023-09-23T23:36:15.404 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0070145779999999996 2023-09-23T23:36:15.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.404 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910539+0000", 2023-09-23T23:36:15.404 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018512857000000001 2023-09-23T23:36:15.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.924468+0000", 2023-09-23T23:36:15.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01392849 2023-09-23T23:36:15.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.924523+0000", 2023-09-23T23:36:15.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4948000000000003e-05 2023-09-23T23:36:15.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.927039+0000", 2023-09-23T23:36:15.406 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002516108 2023-09-23T23:36:15.406 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.406 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.406 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.406 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77547 13.d 13:bc3de976:::benchmark_data_smithi110_541928_object77546:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:36:15.406 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.859759+0000", 2023-09-23T23:36:15.406 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 131.38823145800001, 2023-09-23T23:36:15.406 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0697625900000001, 2023-09-23T23:36:15.406 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.406 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.407 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.407 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:36:15.407 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:36:15.407 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77547 2023-09-23T23:36:15.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.407 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859759+0000", 2023-09-23T23:36:15.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859759+0000", 2023-09-23T23:36:15.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859758+0000", 2023-09-23T23:36:15.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:15.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859847+0000", 2023-09-23T23:36:15.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8799000000000001e-05 2023-09-23T23:36:15.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859861+0000", 2023-09-23T23:36:15.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4229e-05 2023-09-23T23:36:15.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859868+0000", 2023-09-23T23:36:15.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4949999999999996e-06 2023-09-23T23:36:15.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.410 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859928+0000", 2023-09-23T23:36:15.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0738e-05 2023-09-23T23:36:15.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.410 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859941+0000", 2023-09-23T23:36:15.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.24e-05 2023-09-23T23:36:15.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.411 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.887534+0000", 2023-09-23T23:36:15.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0275931309999999 2023-09-23T23:36:15.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.411 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.887728+0000", 2023-09-23T23:36:15.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000194165 2023-09-23T23:36:15.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:36:15.412 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.887881+0000", 2023-09-23T23:36:15.412 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000153263 2023-09-23T23:36:15.412 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.412 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.412 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.903469+0000", 2023-09-23T23:36:15.412 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015587215 2023-09-23T23:36:15.412 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.412 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.412 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912270+0000", 2023-09-23T23:36:15.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088015479999999993 2023-09-23T23:36:15.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.927101+0000", 2023-09-23T23:36:15.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014831195 2023-09-23T23:36:15.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.927169+0000", 2023-09-23T23:36:15.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8172999999999999e-05 2023-09-23T23:36:15.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.929522+0000", 2023-09-23T23:36:15.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023525769999999998 2023-09-23T23:36:15.414 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.414 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.414 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.414 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77576 13.d 13:b73edecb:::benchmark_data_smithi110_541928_object77575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:36:15.414 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.919341+0000", 2023-09-23T23:36:15.415 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 131.32865011800001, 2023-09-23T23:36:15.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0128839199999999, 2023-09-23T23:36:15.415 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:36:15.415 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:15.415 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:36:15.415 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:36:15.415 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:36:15.415 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77576 2023-09-23T23:36:15.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.415 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:36:15.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:36:15.416 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919341+0000", 2023-09-23T23:36:15.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:36:15.416 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919341+0000", 2023-09-23T23:36:15.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:36:15.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:36:15.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919339+0000", 2023-09-23T23:36:15.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:36:15.417 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:36:15.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919384+0000", 2023-09-23T23:36:15.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5042e-05 2023-09-23T23:36:15.417 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:36:15.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919403+0000", 2023-09-23T23:36:15.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9136000000000001e-05 2023-09-23T23:36:15.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:36:15.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919412+0000", 2023-09-23T23:36:15.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4589999999999998e-06 2023-09-23T23:36:15.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.924505+0000", 2023-09-23T23:36:15.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050933469999999998 2023-09-23T23:36:15.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:36:15.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.924519+0000", 2023-09-23T23:36:15.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3852000000000001e-05 2023-09-23T23:36:15.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:36:15.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889955+0000", 2023-09-23T23:36:15.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96543663400000002 2023-09-23T23:36:15.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:36:15.420 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.890057+0000", 2023-09-23T23:36:15.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010134899999999999 2023-09-23T23:36:15.420 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:36:15.420 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.890180+0000", 2023-09-23T23:36:15.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012325699999999999 2023-09-23T23:36:15.420 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:36:15.421 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.903476+0000", 2023-09-23T23:36:15.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013296290000000001 2023-09-23T23:36:15.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.421 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.421 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912329+0000", 2023-09-23T23:36:15.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088524599999999995 2023-09-23T23:36:15.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.421 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:15.421 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.929574+0000", 2023-09-23T23:36:15.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017245415 2023-09-23T23:36:15.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:36:15.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.929630+0000", 2023-09-23T23:36:15.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6097999999999997e-05 2023-09-23T23:36:15.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:36:15.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:36:15.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:36:15.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.932225+0000", 2023-09-23T23:36:15.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002594332 2023-09-23T23:36:15.422 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.422 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.423 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.423 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:36:15.423 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:36:15.423 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:36:15.423 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:15.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:15 smithi110 ceph-mon[212939]: pgmap v1643: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 231 op/s; 25819/195234 objects degraded (13.225%); 3914/195234 objects misplaced (2.005%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:15.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:15 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1804662317' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:15.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:15 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:15.425 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:15 smithi110 ceph-mon[221102]: pgmap v1643: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 231 op/s; 25819/195234 objects degraded (13.225%); 3914/195234 objects misplaced (2.005%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:15.425 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:15 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1804662317' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:15.425 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:15 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:15.453 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:15.497 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:15.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:15.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:15.600 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:15.629 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:16.029 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:36:16.319 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:16 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:36:16.319 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:16 smithi110 ceph-mon[221102]: failed to encode map e593 with expected crc 2023-09-23T23:36:16.319 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:16 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:36:16.319 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:16 smithi110 ceph-mon[221102]: osdmap e593: 8 total, 8 up, 7 in 2023-09-23T23:36:16.319 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:16 smithi110 ceph-mon[221102]: failed to encode map e593 with expected crc 2023-09-23T23:36:16.320 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:16 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:36:16.320 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:16 smithi110 ceph-mon[212939]: failed to encode map e593 with expected crc 2023-09-23T23:36:16.320 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:16 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:36:16.320 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:16 smithi110 ceph-mon[212939]: osdmap e593: 8 total, 8 up, 7 in 2023-09-23T23:36:16.320 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:16 smithi110 ceph-mon[212939]: failed to encode map e593 with expected crc 2023-09-23T23:36:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:16 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:36:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:16 smithi138 ceph-mon[193606]: failed to encode map e593 with expected crc 2023-09-23T23:36:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:16 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:36:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:16 smithi138 ceph-mon[193606]: osdmap e593: 8 total, 8 up, 7 in 2023-09-23T23:36:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:16 smithi138 ceph-mon[193606]: failed to encode map e593 with expected crc 2023-09-23T23:36:16.587 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:16.590 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:36:16.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:16.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:16.623 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:36:16.654 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:36:16.668 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:36:16.692 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:16.701 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:36:16.718 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:16.727 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:36:16.753 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:36:16.767 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:16.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:16.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:16.786 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:36:16.788 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:36:16.788 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:36:16.788 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:36:16.788 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:36:16.788 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:36:16.788 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:36:16.788 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:36:16.789 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:36:16.872 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:36:16.899 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:16.942 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:36:16.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:16.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:36:16.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:36:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:36:16] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.43.0" 2023-09-23T23:36:17.048 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:17.075 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:17.122 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:17.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:17.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:17.227 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:17.263 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:17.348 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:36:17.348 INFO:teuthology.orchestra.run.smithi110.stdout:{"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 25754/192672 objects degraded (13.367%), 7 pgs degraded, 5 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2055,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":593,"num_osds":8,"num_up_osds":8,"osd_up_since":1695511414,"num_in_osds":7,"osd_in_since":1695512030,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":110},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+recovering","count":1},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":119,"num_pools":5,"num_objects":64224,"data_bytes":4207981813,"bytes_used":16922787840,"bytes_avail":655042461696,"bytes_total":671965249536,"degraded_objects":25754,"degraded_total":192672,"degraded_ratio":0.13366758013619001,"misplaced_objects":3914,"misplaced_total":192672,"misplaced_ratio":0.020314316558711177,"recovering_objects_per_sec":36,"recovering_bytes_per_sec":2366736,"recovering_keys_per_sec":0,"num_objects_recovered":289,"num_bytes_recovered":18939904,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":257},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi110.fuglpi","status":"up:active","gid":24572}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.110:8443/","prometheus":"http://172.21.15.110:9283/"}},"servicemap":{"epoch":32,"modified":"2023-09-23T23:32:57.852217+0000","services":{}},"progress_events":{"833ef15d-07ae-4299-b4e4-2cf94e068147":{"message":"Upgrade to 18.2.0-460-gbb0daeff (10m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true},"c67c6d89-9cd0-495b-b1de-fa24b5d12727":{"message":"Global Recovery Event (2m)\n [=========================...] (remaining: 9s)","progress":0.92436975240707397,"add_to_ceph_s":true}}} 2023-09-23T23:36:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:17 smithi138 ceph-mon[193606]: pgmap v1645: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 257 op/s; 25754/192672 objects degraded (13.367%); 3914/192672 objects misplaced (2.031%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:36:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:17 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1417389401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:17.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:17 smithi110 ceph-mon[221102]: pgmap v1645: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 257 op/s; 25754/192672 objects degraded (13.367%); 3914/192672 objects misplaced (2.031%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:36:17.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:17 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1417389401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:17.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:17 smithi110 ceph-mon[212939]: pgmap v1645: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 257 op/s; 25754/192672 objects degraded (13.367%); 3914/192672 objects misplaced (2.031%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T23:36:17.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:17 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1417389401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:17.564 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:18.175 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2852187389' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:36:18.225 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:18.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:18.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:18.330 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:18.378 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:18.440 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2852187389' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:36:18.441 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:36:18.441 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:36:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2852187389' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:36:18.551 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:18.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:18.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:18.656 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:18.684 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:18.711 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:36:18.729 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:18.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:18.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:18.835 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:36:18.863 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:18.907 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:36:18.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:18.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:36:19.013 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:19.041 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:19.404 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:19.406 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:19 smithi110 ceph-mon[212939]: pgmap v1646: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 25696/191886 objects degraded (13.391%); 3840/191886 objects misplaced (2.001%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:36:19.406 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:19 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 25696/191886 objects degraded (13.391%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:19.406 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:19 smithi110 ceph-mon[221102]: pgmap v1646: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 25696/191886 objects degraded (13.391%); 3840/191886 objects misplaced (2.001%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:36:19.406 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:19 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 25696/191886 objects degraded (13.391%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:19.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:19.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:19.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:19 smithi138 ceph-mon[193606]: pgmap v1646: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 25696/191886 objects degraded (13.391%); 3840/191886 objects misplaced (2.001%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:36:19.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:19 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 25696/191886 objects degraded (13.391%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:19.508 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:19.581 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:19.663 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:19.671 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:19.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:19.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:19.735 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:19.777 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:19.822 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:19.951 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:19.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:19.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:20.057 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:20.080 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:36:20.080 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:36:20.083 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:20.126 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:20.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:20.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:20.233 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:20.271 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:20.371 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:36:20.439 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:20.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:20.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:20.550 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:20.599 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:21 smithi138 ceph-mon[193606]: pgmap v1647: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 225 op/s; 25619/190101 objects degraded (13.477%); 3840/190101 objects misplaced (2.020%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:36:21.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:21 smithi110 ceph-mon[212939]: pgmap v1647: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 225 op/s; 25619/190101 objects degraded (13.477%); 3840/190101 objects misplaced (2.020%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:36:21.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:21 smithi110 ceph-mon[221102]: pgmap v1647: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 225 op/s; 25619/190101 objects degraded (13.477%); 3840/190101 objects misplaced (2.020%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:36:22.116 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:36:22.308 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:22.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:22.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:22.412 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:22.462 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:22.544 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:22.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:22.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:22.649 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:36:22.678 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:22.727 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:36:22.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:22.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:36:22.837 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:22.838 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:22.908 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:23.046 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:23.059 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:23.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:23.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:23.166 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:23.212 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:36:23.215 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:23.427 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:36:23.443 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:36:23.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:23 smithi110 ceph-mon[212939]: pgmap v1648: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 224 op/s; 25619/189438 objects degraded (13.524%); 3840/189438 objects misplaced (2.027%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:36:23.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:23 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2965970542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:36:23.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:23 smithi110 ceph-mon[221102]: pgmap v1648: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 224 op/s; 25619/189438 objects degraded (13.524%); 3840/189438 objects misplaced (2.027%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:36:23.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:23 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2965970542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:36:23.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:23 smithi138 ceph-mon[193606]: pgmap v1648: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 224 op/s; 25619/189438 objects degraded (13.524%); 3840/189438 objects misplaced (2.027%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:36:23.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:23 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2965970542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:36:23.839 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:23.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:23.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:23.945 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:23.981 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:24.378 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:36:24.379 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[212939]: failed to encode map e594 with expected crc 2023-09-23T23:36:24.379 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2965970542' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:36:24.379 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[212939]: osdmap e594: 8 total, 8 up, 7 in 2023-09-23T23:36:24.379 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[212939]: failed to encode map e594 with expected crc 2023-09-23T23:36:24.379 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[212939]: failed to encode map e594 with expected crc 2023-09-23T23:36:24.379 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[212939]: failed to encode map e594 with expected crc 2023-09-23T23:36:24.379 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[212939]: from='client.46865 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:24.379 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 25619/189438 objects degraded (13.524%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:24.379 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:36:24.380 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[221102]: failed to encode map e594 with expected crc 2023-09-23T23:36:24.380 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2965970542' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:36:24.380 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[221102]: osdmap e594: 8 total, 8 up, 7 in 2023-09-23T23:36:24.380 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[221102]: failed to encode map e594 with expected crc 2023-09-23T23:36:24.380 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[221102]: failed to encode map e594 with expected crc 2023-09-23T23:36:24.380 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[221102]: failed to encode map e594 with expected crc 2023-09-23T23:36:24.380 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[221102]: from='client.46865 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:24.380 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:24 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 25619/189438 objects degraded (13.524%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:24 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:36:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:24 smithi138 ceph-mon[193606]: failed to encode map e594 with expected crc 2023-09-23T23:36:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:24 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2965970542' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:36:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:24 smithi138 ceph-mon[193606]: osdmap e594: 8 total, 8 up, 7 in 2023-09-23T23:36:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:24 smithi138 ceph-mon[193606]: failed to encode map e594 with expected crc 2023-09-23T23:36:24.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:24 smithi138 ceph-mon[193606]: failed to encode map e594 with expected crc 2023-09-23T23:36:24.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:24 smithi138 ceph-mon[193606]: failed to encode map e594 with expected crc 2023-09-23T23:36:24.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:24 smithi138 ceph-mon[193606]: from='client.46865 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:24.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:24 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 25619/189438 objects degraded (13.524%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:25.086 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1649,"stamp":"2023-09-23T23:36:23.189311+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4116886773,"num_objects":62834,"num_object_clones":0,"num_object_copies":188502,"num_objects_missing_on_primary":2871,"num_objects_missing":2871,"num_objects_degraded":25541,"num_objects_misplaced":3766,"num_objects_unfound":0,"num_objects_dirty":62834,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":112823,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14289,"num_bytes_recovered":933753256,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94970,"ondisk_log_size":94970,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16336220,"kb_used_data":10658148,"kb_used_omap":29,"kb_used_meta":5677986,"kb_avail":639879844,"statfs":{"total":671965249536,"available":655236960256,"internally_reserved":0,"allocated":10913943552,"data_stored":10897779502,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5814258387},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-147062784,"num_objects":-2244,"num_object_clones":0,"num_object_copies":-6732,"num_objects_missing_on_primary":-148,"num_objects_missing":-148,"num_objects_degraded":-278,"num_objects_misplaced":-148,"num_objects_unfound":0,"num_objects_dirty":-2244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2244,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":427,"num_bytes_recovered":27983872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.272056"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":789,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.167810+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:36:08.167810+0000","last_peered":"2023-09-23T23:36:08.167810+0000","last_clean":"2023-09-23T23:36:08.167810+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:36:08.167810+0000","last_undegraded":"2023-09-23T23:36:08.167810+0000","last_fullsized":"2023-09-23T23:36:08.167810+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":347,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.151395+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:36:08.151395+0000","last_peered":"2023-09-23T23:36:08.151395+0000","last_clean":"2023-09-23T23:36:08.151395+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:36:08.151395+0000","last_undegraded":"2023-09-23T23:36:08.151395+0000","last_fullsized":"2023-09-23T23:36:08.151395+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":575,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.209846+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:36:08.209846+0000","last_peered":"2023-09-23T23:36:08.209846+0000","last_clean":"2023-09-23T23:36:08.209846+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:36:08.209846+0000","last_undegraded":"2023-09-23T23:36:08.209846+0000","last_fullsized":"2023-09-23T23:36:08.209846+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":742,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.168492+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:36:08.168492+0000","last_peered":"2023-09-23T23:36:08.168492+0000","last_clean":"2023-09-23T23:36:08.168492+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:36:08.168492+0000","last_undegraded":"2023-09-23T23:36:08.168492+0000","last_fullsized":"2023-09-23T23:36:08.168492+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":311,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169348+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:36:08.169348+0000","last_peered":"2023-09-23T23:36:08.169348+0000","last_clean":"2023-09-23T23:36:08.169348+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:36:08.169348+0000","last_undegraded":"2023-09-23T23:36:08.169348+0000","last_fullsized":"2023-09-23T23:36:08.169348+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":554,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.171436+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:36:15.171436+0000","last_peered":"2023-09-23T23:36:15.171436+0000","last_clean":"2023-09-23T23:36:15.171436+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:36:15.171436+0000","last_undegraded":"2023-09-23T23:36:15.171436+0000","last_fullsized":"2023-09-23T23:36:15.171436+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":754,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.168875+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:36:08.168875+0000","last_peered":"2023-09-23T23:36:08.168875+0000","last_clean":"2023-09-23T23:36:08.168875+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:36:08.168875+0000","last_undegraded":"2023-09-23T23:36:08.168875+0000","last_fullsized":"2023-09-23T23:36:08.168875+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":611,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169023+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:36:08.169023+0000","last_peered":"2023-09-23T23:36:08.169023+0000","last_clean":"2023-09-23T23:36:08.169023+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:36:08.169023+0000","last_undegraded":"2023-09-23T23:36:08.169023+0000","last_fullsized":"2023-09-23T23:36:08.169023+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":107,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169450+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:36:08.169450+0000","last_peered":"2023-09-23T23:36:08.169450+0000","last_clean":"2023-09-23T23:36:08.169450+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:36:08.169450+0000","last_undegraded":"2023-09-23T23:36:08.169450+0000","last_fullsized":"2023-09-23T23:36:08.169450+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":913,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.168379+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:36:08.168379+0000","last_peered":"2023-09-23T23:36:08.168379+0000","last_clean":"2023-09-23T23:36:08.168379+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:36:08.168379+0000","last_undegraded":"2023-09-23T23:36:08.168379+0000","last_fullsized":"2023-09-23T23:36:08.168379+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":245,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.170347+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:36:08.170347+0000","last_peered":"2023-09-23T23:36:08.170347+0000","last_clean":"2023-09-23T23:36:08.170347+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:36:08.170347+0000","last_undegraded":"2023-09-23T23:36:08.170347+0000","last_fullsized":"2023-09-23T23:36:08.170347+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":593,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.168981+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:36:08.168981+0000","last_peered":"2023-09-23T23:36:08.168981+0000","last_clean":"2023-09-23T23:36:08.168981+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:36:08.168981+0000","last_undegraded":"2023-09-23T23:36:08.168981+0000","last_fullsized":"2023-09-23T23:36:08.168981+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":886,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.148329+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:36:08.148329+0000","last_peered":"2023-09-23T23:36:08.148329+0000","last_clean":"2023-09-23T23:36:08.148329+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:36:08.148329+0000","last_undegraded":"2023-09-23T23:36:08.148329+0000","last_fullsized":"2023-09-23T23:36:08.148329+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":224,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.173262+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:36:08.173262+0000","last_peered":"2023-09-23T23:36:08.173262+0000","last_clean":"2023-09-23T23:36:08.173262+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:36:08.173262+0000","last_undegraded":"2023-09-23T23:36:08.173262+0000","last_fullsized":"2023-09-23T23:36:08.173262+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":554,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169247+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:36:08.169247+0000","last_peered":"2023-09-23T23:36:08.169247+0000","last_clean":"2023-09-23T23:36:08.169247+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:36:08.169247+0000","last_undegraded":"2023-09-23T23:36:08.169247+0000","last_fullsized":"2023-09-23T23:36:08.169247+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1047,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.170220+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:36:08.170220+0000","last_peered":"2023-09-23T23:36:08.170220+0000","last_clean":"2023-09-23T23:36:08.170220+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:36:08.170220+0000","last_undegraded":"2023-09-23T23:36:08.170220+0000","last_fullsized":"2023-09-23T23:36:08.170220+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":265,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.176190+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:36:08.176190+0000","last_peered":"2023-09-23T23:36:08.176190+0000","last_clean":"2023-09-23T23:36:08.176190+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:36:08.176190+0000","last_undegraded":"2023-09-23T23:36:08.176190+0000","last_fullsized":"2023-09-23T23:36:08.176190+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":126,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169681+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:36:08.169681+0000","last_peered":"2023-09-23T23:36:08.169681+0000","last_clean":"2023-09-23T23:36:08.169681+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:36:08.169681+0000","last_undegraded":"2023-09-23T23:36:08.169681+0000","last_fullsized":"2023-09-23T23:36:08.169681+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":443,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169482+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:36:08.169482+0000","last_peered":"2023-09-23T23:36:08.169482+0000","last_clean":"2023-09-23T23:36:08.169482+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:36:08.169482+0000","last_undegraded":"2023-09-23T23:36:08.169482+0000","last_fullsized":"2023-09-23T23:36:08.169482+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":167,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169161+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:36:08.169161+0000","last_peered":"2023-09-23T23:36:08.169161+0000","last_clean":"2023-09-23T23:36:08.169161+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:36:08.169161+0000","last_undegraded":"2023-09-23T23:36:08.169161+0000","last_fullsized":"2023-09-23T23:36:08.169161+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":534,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.167898+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:36:08.167898+0000","last_peered":"2023-09-23T23:36:08.167898+0000","last_clean":"2023-09-23T23:36:08.167898+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:36:08.167898+0000","last_undegraded":"2023-09-23T23:36:08.167898+0000","last_fullsized":"2023-09-23T23:36:08.167898+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":748,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.170889+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:36:08.170889+0000","last_peered":"2023-09-23T23:36:08.170889+0000","last_clean":"2023-09-23T23:36:08.170889+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:36:08.170889+0000","last_undegraded":"2023-09-23T23:36:08.170889+0000","last_fullsized":"2023-09-23T23:36:08.170889+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":139,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.147864+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:36:08.147864+0000","last_peered":"2023-09-23T23:36:08.147864+0000","last_clean":"2023-09-23T23:36:08.147864+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:36:08.147864+0000","last_undegraded":"2023-09-23T23:36:08.147864+0000","last_fullsized":"2023-09-23T23:36:08.147864+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":592,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.170132+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:36:08.170132+0000","last_peered":"2023-09-23T23:36:08.170132+0000","last_clean":"2023-09-23T23:36:08.170132+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:36:08.170132+0000","last_undegraded":"2023-09-23T23:36:08.170132+0000","last_fullsized":"2023-09-23T23:36:08.170132+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":878,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.167309+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:36:08.167309+0000","last_peered":"2023-09-23T23:36:08.167309+0000","last_clean":"2023-09-23T23:36:08.167309+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:36:08.167309+0000","last_undegraded":"2023-09-23T23:36:08.167309+0000","last_fullsized":"2023-09-23T23:36:08.167309+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":187,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.170354+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:36:08.170354+0000","last_peered":"2023-09-23T23:36:08.170354+0000","last_clean":"2023-09-23T23:36:08.170354+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:36:08.170354+0000","last_undegraded":"2023-09-23T23:36:08.170354+0000","last_fullsized":"2023-09-23T23:36:08.170354+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":314,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.198051+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:36:08.198051+0000","last_peered":"2023-09-23T23:36:08.198051+0000","last_clean":"2023-09-23T23:36:08.198051+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:36:08.198051+0000","last_undegraded":"2023-09-23T23:36:08.198051+0000","last_fullsized":"2023-09-23T23:36:08.198051+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":555,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.171893+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:36:08.171893+0000","last_peered":"2023-09-23T23:36:08.171893+0000","last_clean":"2023-09-23T23:36:08.171893+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:36:08.171893+0000","last_undegraded":"2023-09-23T23:36:08.171893+0000","last_fullsized":"2023-09-23T23:36:08.171893+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":461,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.170682+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:36:08.170682+0000","last_peered":"2023-09-23T23:36:08.170682+0000","last_clean":"2023-09-23T23:36:08.170682+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:36:08.170682+0000","last_undegraded":"2023-09-23T23:36:08.170682+0000","last_fullsized":"2023-09-23T23:36:08.170682+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":549,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169604+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:36:08.169604+0000","last_peered":"2023-09-23T23:36:08.169604+0000","last_clean":"2023-09-23T23:36:08.169604+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:36:08.169604+0000","last_undegraded":"2023-09-23T23:36:08.169604+0000","last_fullsized":"2023-09-23T23:36:08.169604+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":956,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.168253+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:36:08.168253+0000","last_peered":"2023-09-23T23:36:08.168253+0000","last_clean":"2023-09-23T23:36:08.168253+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:36:08.168253+0000","last_undegraded":"2023-09-23T23:36:08.168253+0000","last_fullsized":"2023-09-23T23:36:08.168253+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":157,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.167606+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:36:08.167606+0000","last_peered":"2023-09-23T23:36:08.167606+0000","last_clean":"2023-09-23T23:36:08.167606+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:36:08.167606+0000","last_undegraded":"2023-09-23T23:36:08.167606+0000","last_fullsized":"2023-09-23T23:36:08.167606+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":234,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169822+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:36:08.169822+0000","last_peered":"2023-09-23T23:36:08.169822+0000","last_clean":"2023-09-23T23:36:08.169822+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:36:08.169822+0000","last_undegraded":"2023-09-23T23:36:08.169822+0000","last_fullsized":"2023-09-23T23:36:08.169822+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"593'6905","reported_seq":6981,"reported_epoch":593,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:18.954271+0000","last_change":"2023-09-23T23:33:52.884769+0000","last_active":"2023-09-23T23:36:18.954271+0000","last_peered":"2023-09-23T23:36:18.954271+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:36:18.954271+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"548'1500","ondisk_log_start":"548'1500","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5405,"log_dups_size":0,"ondisk_log_size":5405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258408448,"num_objects":3943,"num_object_clones":0,"num_object_copies":11829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6905,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":2917},{"shards":"2,6","objects":-455}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":926,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.148194+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:36:08.148194+0000","last_peered":"2023-09-23T23:36:08.148194+0000","last_clean":"2023-09-23T23:36:08.148194+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:36:08.148194+0000","last_undegraded":"2023-09-23T23:36:08.148194+0000","last_fullsized":"2023-09-23T23:36:08.148194+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":234,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.167847+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:36:08.167847+0000","last_peered":"2023-09-23T23:36:08.167847+0000","last_clean":"2023-09-23T23:36:08.167847+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:36:08.167847+0000","last_undegraded":"2023-09-23T23:36:08.167847+0000","last_fullsized":"2023-09-23T23:36:08.167847+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":224,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.167727+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:36:08.167727+0000","last_peered":"2023-09-23T23:36:08.167727+0000","last_clean":"2023-09-23T23:36:08.167727+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:36:08.167727+0000","last_undegraded":"2023-09-23T23:36:08.167727+0000","last_fullsized":"2023-09-23T23:36:08.167727+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"593'7060","reported_seq":7136,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:19.906469+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:36:19.906469+0000","last_peered":"2023-09-23T23:36:19.906469+0000","last_clean":"2023-09-23T23:36:19.906469+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:36:19.906469+0000","last_undegraded":"2023-09-23T23:36:19.906469+0000","last_fullsized":"2023-09-23T23:36:19.906469+0000","mapping_epoch":566,"log_start":"543'900","ondisk_log_start":"543'900","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6160,"log_dups_size":0,"ondisk_log_size":6160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260308992,"num_objects":3972,"num_object_clones":0,"num_object_copies":11916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7060,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1005,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.168976+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:36:08.168976+0000","last_peered":"2023-09-23T23:36:08.168976+0000","last_clean":"2023-09-23T23:36:08.168976+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:36:08.168976+0000","last_undegraded":"2023-09-23T23:36:08.168976+0000","last_fullsized":"2023-09-23T23:36:08.168976+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":224,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.170022+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:36:08.170022+0000","last_peered":"2023-09-23T23:36:08.170022+0000","last_clean":"2023-09-23T23:36:08.170022+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:36:08.170022+0000","last_undegraded":"2023-09-23T23:36:08.170022+0000","last_fullsized":"2023-09-23T23:36:08.170022+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":444,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.168623+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:36:15.168623+0000","last_peered":"2023-09-23T23:36:15.168623+0000","last_clean":"2023-09-23T23:36:15.168623+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:36:15.168623+0000","last_undegraded":"2023-09-23T23:36:15.168623+0000","last_fullsized":"2023-09-23T23:36:15.168623+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"593'7175","reported_seq":11588,"reported_epoch":593,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:36:22.453667+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:36:22.453667+0000","last_peered":"2023-09-23T23:36:22.453667+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:36:22.453667+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:36:22.453667+0000","mapping_epoch":571,"log_start":"549'1900","ondisk_log_start":"549'1900","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5275,"log_dups_size":0,"ondisk_log_size":5275,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253820928,"num_objects":3873,"num_object_clones":0,"num_object_copies":11619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2978,"num_objects_misplaced":895,"num_objects_unfound":0,"num_objects_dirty":3873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7175,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2153,"num_bytes_recovered":141099008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":2856},{"shards":"3,5,7","objects":1017}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":903,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169530+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:36:08.169530+0000","last_peered":"2023-09-23T23:36:08.169530+0000","last_clean":"2023-09-23T23:36:08.169530+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:36:08.169530+0000","last_undegraded":"2023-09-23T23:36:08.169530+0000","last_fullsized":"2023-09-23T23:36:08.169530+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":331,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.172304+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:36:08.172304+0000","last_peered":"2023-09-23T23:36:08.172304+0000","last_clean":"2023-09-23T23:36:08.172304+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:36:08.172304+0000","last_undegraded":"2023-09-23T23:36:08.172304+0000","last_fullsized":"2023-09-23T23:36:08.172304+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":107,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169874+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:36:08.169874+0000","last_peered":"2023-09-23T23:36:08.169874+0000","last_clean":"2023-09-23T23:36:08.169874+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:36:08.169874+0000","last_undegraded":"2023-09-23T23:36:08.169874+0000","last_fullsized":"2023-09-23T23:36:08.169874+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"593'7117","reported_seq":9179,"reported_epoch":593,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:22.328723+0000","last_change":"2023-09-23T23:34:35.750428+0000","last_active":"2023-09-23T23:36:22.328723+0000","last_peered":"2023-09-23T23:36:22.328723+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:36:22.328723+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"549'1800","ondisk_log_start":"549'1800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5317,"log_dups_size":0,"ondisk_log_size":5317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260374528,"num_objects":3973,"num_object_clones":0,"num_object_copies":11919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7117,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":991,"num_bytes_recovered":64946176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":2963},{"shards":"3,5","objects":429}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":499,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169532+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:36:08.169532+0000","last_peered":"2023-09-23T23:36:08.169532+0000","last_clean":"2023-09-23T23:36:08.169532+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:36:08.169532+0000","last_undegraded":"2023-09-23T23:36:08.169532+0000","last_fullsized":"2023-09-23T23:36:08.169532+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":433,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.170042+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:36:08.170042+0000","last_peered":"2023-09-23T23:36:08.170042+0000","last_clean":"2023-09-23T23:36:08.170042+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:36:08.170042+0000","last_undegraded":"2023-09-23T23:36:08.170042+0000","last_fullsized":"2023-09-23T23:36:08.170042+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":590,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.177817+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:36:08.177817+0000","last_peered":"2023-09-23T23:36:08.177817+0000","last_clean":"2023-09-23T23:36:08.177817+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:36:08.177817+0000","last_undegraded":"2023-09-23T23:36:08.177817+0000","last_fullsized":"2023-09-23T23:36:08.177817+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"593'7174","reported_seq":8496,"reported_epoch":593,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:19.368999+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:36:19.368999+0000","last_peered":"2023-09-23T23:36:19.368999+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:36:19.368999+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"548'1500","ondisk_log_start":"548'1500","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5674,"log_dups_size":0,"ondisk_log_size":5674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261357568,"num_objects":3988,"num_object_clones":0,"num_object_copies":11964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7174,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":2942},{"shards":"3,5","objects":-1472},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":720,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169215+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:36:08.169215+0000","last_peered":"2023-09-23T23:36:08.169215+0000","last_clean":"2023-09-23T23:36:08.169215+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:36:08.169215+0000","last_undegraded":"2023-09-23T23:36:08.169215+0000","last_fullsized":"2023-09-23T23:36:08.169215+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":603,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.171770+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:36:08.171770+0000","last_peered":"2023-09-23T23:36:08.171770+0000","last_clean":"2023-09-23T23:36:08.171770+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:36:08.171770+0000","last_undegraded":"2023-09-23T23:36:08.171770+0000","last_fullsized":"2023-09-23T23:36:08.171770+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":514,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.170000+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:36:08.170000+0000","last_peered":"2023-09-23T23:36:08.170000+0000","last_clean":"2023-09-23T23:36:08.170000+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:36:08.170000+0000","last_undegraded":"2023-09-23T23:36:08.170000+0000","last_fullsized":"2023-09-23T23:36:08.170000+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"593'7138","reported_seq":7205,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:18.620184+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:36:18.620184+0000","last_peered":"2023-09-23T23:36:18.620184+0000","last_clean":"2023-09-23T23:36:18.620184+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:36:18.620184+0000","last_undegraded":"2023-09-23T23:36:18.620184+0000","last_fullsized":"2023-09-23T23:36:18.620184+0000","mapping_epoch":534,"log_start":"548'1700","ondisk_log_start":"548'1700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5438,"log_dups_size":0,"ondisk_log_size":5438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258342912,"num_objects":3942,"num_object_clones":0,"num_object_copies":11826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7138,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":876,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.147950+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:36:08.147950+0000","last_peered":"2023-09-23T23:36:08.147950+0000","last_clean":"2023-09-23T23:36:08.147950+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:36:08.147950+0000","last_undegraded":"2023-09-23T23:36:08.147950+0000","last_fullsized":"2023-09-23T23:36:08.147950+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":117,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169812+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:36:08.169812+0000","last_peered":"2023-09-23T23:36:08.169812+0000","last_clean":"2023-09-23T23:36:08.169812+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:36:08.169812+0000","last_undegraded":"2023-09-23T23:36:08.169812+0000","last_fullsized":"2023-09-23T23:36:08.169812+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":459,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:09.925529+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:36:09.925529+0000","last_peered":"2023-09-23T23:36:09.925529+0000","last_clean":"2023-09-23T23:36:09.925529+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:36:09.925529+0000","last_undegraded":"2023-09-23T23:36:09.925529+0000","last_fullsized":"2023-09-23T23:36:09.925529+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"593'7152","reported_seq":10439,"reported_epoch":593,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:36:19.147469+0000","last_change":"2023-09-23T23:34:35.785270+0000","last_active":"2023-09-23T23:36:19.147469+0000","last_peered":"2023-09-23T23:36:19.147469+0000","last_clean":"2023-09-23T23:33:50.504296+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:36:19.147469+0000","last_undegraded":"2023-09-23T23:34:35.780981+0000","last_fullsized":"2023-09-23T23:36:19.147469+0000","mapping_epoch":567,"log_start":"548'1700","ondisk_log_start":"548'1700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5452,"log_dups_size":0,"ondisk_log_size":5452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258998272,"num_objects":3952,"num_object_clones":0,"num_object_copies":11856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2947,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7152,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1600,"num_bytes_recovered":104857600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":2947},{"shards":"5,6,7","objects":1005}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":601,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.168398+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:36:08.168398+0000","last_peered":"2023-09-23T23:36:08.168398+0000","last_clean":"2023-09-23T23:36:08.168398+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:36:08.168398+0000","last_undegraded":"2023-09-23T23:36:08.168398+0000","last_fullsized":"2023-09-23T23:36:08.168398+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":456,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.148782+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:36:08.148782+0000","last_peered":"2023-09-23T23:36:08.148782+0000","last_clean":"2023-09-23T23:36:08.148782+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:36:08.148782+0000","last_undegraded":"2023-09-23T23:36:08.148782+0000","last_fullsized":"2023-09-23T23:36:08.148782+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":544,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.168605+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:36:08.168605+0000","last_peered":"2023-09-23T23:36:08.168605+0000","last_clean":"2023-09-23T23:36:08.168605+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:36:08.168605+0000","last_undegraded":"2023-09-23T23:36:08.168605+0000","last_fullsized":"2023-09-23T23:36:08.168605+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"593'7096","reported_seq":16126,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:19.295634+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:36:19.295634+0000","last_peered":"2023-09-23T23:36:19.295634+0000","last_clean":"2023-09-23T23:36:19.295634+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:36:19.295634+0000","last_undegraded":"2023-09-23T23:36:19.295634+0000","last_fullsized":"2023-09-23T23:36:19.295634+0000","mapping_epoch":579,"log_start":"548'1400","ondisk_log_start":"548'1400","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5696,"log_dups_size":0,"ondisk_log_size":5696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256901120,"num_objects":3920,"num_object_clones":0,"num_object_copies":11760,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3920,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7096,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":934,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.151285+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:36:08.151285+0000","last_peered":"2023-09-23T23:36:08.151285+0000","last_clean":"2023-09-23T23:36:08.151285+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:36:08.151285+0000","last_undegraded":"2023-09-23T23:36:08.151285+0000","last_fullsized":"2023-09-23T23:36:08.151285+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":234,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.168898+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:36:08.168898+0000","last_peered":"2023-09-23T23:36:08.168898+0000","last_clean":"2023-09-23T23:36:08.168898+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:36:08.168898+0000","last_undegraded":"2023-09-23T23:36:08.168898+0000","last_fullsized":"2023-09-23T23:36:08.168898+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":523,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.172017+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:36:15.172017+0000","last_peered":"2023-09-23T23:36:15.172017+0000","last_clean":"2023-09-23T23:36:15.172017+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:36:15.172017+0000","last_undegraded":"2023-09-23T23:36:15.172017+0000","last_fullsized":"2023-09-23T23:36:15.172017+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"593'6860","reported_seq":6929,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:22.432817+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:36:22.432817+0000","last_peered":"2023-09-23T23:36:22.432817+0000","last_clean":"2023-09-23T23:36:22.432817+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:36:22.432817+0000","last_undegraded":"2023-09-23T23:36:22.432817+0000","last_fullsized":"2023-09-23T23:36:22.432817+0000","mapping_epoch":534,"log_start":"548'1500","ondisk_log_start":"548'1500","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5360,"log_dups_size":0,"ondisk_log_size":5360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253493248,"num_objects":3868,"num_object_clones":0,"num_object_copies":11604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6860,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":914,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.167896+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:36:08.167896+0000","last_peered":"2023-09-23T23:36:08.167896+0000","last_clean":"2023-09-23T23:36:08.167896+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:36:08.167896+0000","last_undegraded":"2023-09-23T23:36:08.167896+0000","last_fullsized":"2023-09-23T23:36:08.167896+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":302,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.151007+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:36:08.151007+0000","last_peered":"2023-09-23T23:36:08.151007+0000","last_clean":"2023-09-23T23:36:08.151007+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:36:08.151007+0000","last_undegraded":"2023-09-23T23:36:08.151007+0000","last_fullsized":"2023-09-23T23:36:08.151007+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":608,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.172389+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:36:08.172389+0000","last_peered":"2023-09-23T23:36:08.172389+0000","last_clean":"2023-09-23T23:36:08.172389+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:36:08.172389+0000","last_undegraded":"2023-09-23T23:36:08.172389+0000","last_fullsized":"2023-09-23T23:36:08.172389+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":189,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.168037+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:36:08.168037+0000","last_peered":"2023-09-23T23:36:08.168037+0000","last_clean":"2023-09-23T23:36:08.168037+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:36:08.168037+0000","last_undegraded":"2023-09-23T23:36:08.168037+0000","last_fullsized":"2023-09-23T23:36:08.168037+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"593'7218","reported_seq":10583,"reported_epoch":593,"state":"active+recovering","last_fresh":"2023-09-23T23:36:22.465480+0000","last_change":"2023-09-23T23:34:35.815566+0000","last_active":"2023-09-23T23:36:22.465480+0000","last_peered":"2023-09-23T23:36:22.465480+0000","last_clean":"2023-09-23T23:33:50.519308+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:36:22.465480+0000","last_undegraded":"2023-09-23T23:36:22.465480+0000","last_fullsized":"2023-09-23T23:36:22.465480+0000","mapping_epoch":566,"log_start":"549'1922","ondisk_log_start":"549'1922","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5296,"log_dups_size":0,"ondisk_log_size":5296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258801688,"num_objects":3950,"num_object_clones":0,"num_object_copies":11850,"num_objects_missing_on_primary":2871,"num_objects_missing":2871,"num_objects_degraded":0,"num_objects_misplaced":2871,"num_objects_unfound":0,"num_objects_dirty":3950,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7218,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1633,"num_bytes_recovered":107020288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,3,4,7","objects":11},{"shards":"1,3,7","objects":1068},{"shards":"1,4,7","objects":2871}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":676,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.176386+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:36:08.176386+0000","last_peered":"2023-09-23T23:36:08.176386+0000","last_clean":"2023-09-23T23:36:08.176386+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:36:08.176386+0000","last_undegraded":"2023-09-23T23:36:08.176386+0000","last_fullsized":"2023-09-23T23:36:08.176386+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":241,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169745+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:36:08.169745+0000","last_peered":"2023-09-23T23:36:08.169745+0000","last_clean":"2023-09-23T23:36:08.169745+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:36:08.169745+0000","last_undegraded":"2023-09-23T23:36:08.169745+0000","last_fullsized":"2023-09-23T23:36:08.169745+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":525,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.167978+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:36:08.167978+0000","last_peered":"2023-09-23T23:36:08.167978+0000","last_clean":"2023-09-23T23:36:08.167978+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:36:08.167978+0000","last_undegraded":"2023-09-23T23:36:08.167978+0000","last_fullsized":"2023-09-23T23:36:08.167978+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":226,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:09.925497+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:36:09.925497+0000","last_peered":"2023-09-23T23:36:09.925497+0000","last_clean":"2023-09-23T23:36:09.925497+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:36:09.925497+0000","last_undegraded":"2023-09-23T23:36:09.925497+0000","last_fullsized":"2023-09-23T23:36:09.925497+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"593'6867","reported_seq":8913,"reported_epoch":593,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:19.366194+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:36:19.366194+0000","last_peered":"2023-09-23T23:36:19.366194+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:36:19.366194+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"546'1200","ondisk_log_start":"546'1200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5667,"log_dups_size":0,"ondisk_log_size":5667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255918080,"num_objects":3905,"num_object_clones":0,"num_object_copies":11715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6867,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":2894},{"shards":"5,7","objects":513}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":894,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.151233+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:36:08.151233+0000","last_peered":"2023-09-23T23:36:08.151233+0000","last_clean":"2023-09-23T23:36:08.151233+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:36:08.151233+0000","last_undegraded":"2023-09-23T23:36:08.151233+0000","last_fullsized":"2023-09-23T23:36:08.151233+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":331,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.148702+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:36:08.148702+0000","last_peered":"2023-09-23T23:36:08.148702+0000","last_clean":"2023-09-23T23:36:08.148702+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:36:08.148702+0000","last_undegraded":"2023-09-23T23:36:08.148702+0000","last_fullsized":"2023-09-23T23:36:08.148702+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":241,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.148032+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:36:08.148032+0000","last_peered":"2023-09-23T23:36:08.148032+0000","last_clean":"2023-09-23T23:36:08.148032+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:36:08.148032+0000","last_undegraded":"2023-09-23T23:36:08.148032+0000","last_fullsized":"2023-09-23T23:36:08.148032+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":205,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.149094+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:36:08.149094+0000","last_peered":"2023-09-23T23:36:08.149094+0000","last_clean":"2023-09-23T23:36:08.149094+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:36:08.149094+0000","last_undegraded":"2023-09-23T23:36:08.149094+0000","last_fullsized":"2023-09-23T23:36:08.149094+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"593'7007","reported_seq":7084,"reported_epoch":593,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:36:19.125269+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:36:19.125269+0000","last_peered":"2023-09-23T23:36:19.125269+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:36:19.125269+0000","last_undegraded":"2023-09-23T23:36:19.125269+0000","last_fullsized":"2023-09-23T23:36:19.125269+0000","mapping_epoch":562,"log_start":"546'1300","ondisk_log_start":"546'1300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5707,"log_dups_size":0,"ondisk_log_size":5707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255262720,"num_objects":3895,"num_object_clones":0,"num_object_copies":11685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7007,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":3895}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":608,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.147943+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:36:08.147943+0000","last_peered":"2023-09-23T23:36:08.147943+0000","last_clean":"2023-09-23T23:36:08.147943+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:36:08.147943+0000","last_undegraded":"2023-09-23T23:36:08.147943+0000","last_fullsized":"2023-09-23T23:36:08.147943+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":879,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.167384+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:36:08.167384+0000","last_peered":"2023-09-23T23:36:08.167384+0000","last_clean":"2023-09-23T23:36:08.167384+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:36:08.167384+0000","last_undegraded":"2023-09-23T23:36:08.167384+0000","last_fullsized":"2023-09-23T23:36:08.167384+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":555,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.170219+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:36:08.170219+0000","last_peered":"2023-09-23T23:36:08.170219+0000","last_clean":"2023-09-23T23:36:08.170219+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:36:08.170219+0000","last_undegraded":"2023-09-23T23:36:08.170219+0000","last_fullsized":"2023-09-23T23:36:08.170219+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":117,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.168512+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:36:08.168512+0000","last_peered":"2023-09-23T23:36:08.168512+0000","last_clean":"2023-09-23T23:36:08.168512+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:36:08.168512+0000","last_undegraded":"2023-09-23T23:36:08.168512+0000","last_fullsized":"2023-09-23T23:36:08.168512+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"593'6893","reported_seq":6960,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:18.663284+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:36:18.663284+0000","last_peered":"2023-09-23T23:36:18.663284+0000","last_clean":"2023-09-23T23:36:18.663284+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:36:18.663284+0000","last_undegraded":"2023-09-23T23:36:18.663284+0000","last_fullsized":"2023-09-23T23:36:18.663284+0000","mapping_epoch":534,"log_start":"548'1500","ondisk_log_start":"548'1500","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5393,"log_dups_size":0,"ondisk_log_size":5393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251854848,"num_objects":3843,"num_object_clones":0,"num_object_copies":11529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6893,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":705,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.167819+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:36:08.167819+0000","last_peered":"2023-09-23T23:36:08.167819+0000","last_clean":"2023-09-23T23:36:08.167819+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:36:08.167819+0000","last_undegraded":"2023-09-23T23:36:08.167819+0000","last_fullsized":"2023-09-23T23:36:08.167819+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":520,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169149+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:36:08.169149+0000","last_peered":"2023-09-23T23:36:08.169149+0000","last_clean":"2023-09-23T23:36:08.169149+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:36:08.169149+0000","last_undegraded":"2023-09-23T23:36:08.169149+0000","last_fullsized":"2023-09-23T23:36:08.169149+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":434,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169022+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:36:08.169022+0000","last_peered":"2023-09-23T23:36:08.169022+0000","last_clean":"2023-09-23T23:36:08.169022+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:36:08.169022+0000","last_undegraded":"2023-09-23T23:36:08.169022+0000","last_fullsized":"2023-09-23T23:36:08.169022+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":33,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.177291+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:36:08.177291+0000","last_peered":"2023-09-23T23:36:08.177291+0000","last_clean":"2023-09-23T23:36:08.177291+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:36:08.177291+0000","last_undegraded":"2023-09-23T23:36:08.177291+0000","last_fullsized":"2023-09-23T23:36:08.177291+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"593'7071","reported_seq":10737,"reported_epoch":593,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:19.911697+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:36:19.911697+0000","last_peered":"2023-09-23T23:36:19.911697+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:36:19.911697+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"543'900","ondisk_log_start":"543'900","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6171,"log_dups_size":0,"ondisk_log_size":6171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257359872,"num_objects":3927,"num_object_clones":0,"num_object_copies":11781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7071,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":1223},{"shards":"1,2,4","objects":2704}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":588,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.168879+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:36:08.168879+0000","last_peered":"2023-09-23T23:36:08.168879+0000","last_clean":"2023-09-23T23:36:08.168879+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:36:08.168879+0000","last_undegraded":"2023-09-23T23:36:08.168879+0000","last_fullsized":"2023-09-23T23:36:08.168879+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":33,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.177101+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:36:08.177101+0000","last_peered":"2023-09-23T23:36:08.177101+0000","last_clean":"2023-09-23T23:36:08.177101+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:36:08.177101+0000","last_undegraded":"2023-09-23T23:36:08.177101+0000","last_fullsized":"2023-09-23T23:36:08.177101+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":133,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:09.925509+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:36:09.925509+0000","last_peered":"2023-09-23T23:36:09.925509+0000","last_clean":"2023-09-23T23:36:09.925509+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:36:09.925509+0000","last_undegraded":"2023-09-23T23:36:09.925509+0000","last_fullsized":"2023-09-23T23:36:09.925509+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":326,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.168623+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:36:08.168623+0000","last_peered":"2023-09-23T23:36:08.168623+0000","last_clean":"2023-09-23T23:36:08.168623+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:36:08.168623+0000","last_undegraded":"2023-09-23T23:36:08.168623+0000","last_fullsized":"2023-09-23T23:36:08.168623+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"593'7044","reported_seq":7111,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:19.314600+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:36:19.314600+0000","last_peered":"2023-09-23T23:36:19.314600+0000","last_clean":"2023-09-23T23:36:19.314600+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:36:19.314600+0000","last_undegraded":"2023-09-23T23:36:19.314600+0000","last_fullsized":"2023-09-23T23:36:19.314600+0000","mapping_epoch":534,"log_start":"546'1300","ondisk_log_start":"546'1300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5744,"log_dups_size":0,"ondisk_log_size":5744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256901120,"num_objects":3920,"num_object_clones":0,"num_object_copies":11760,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3920,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7044,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":774,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.148457+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:36:08.148457+0000","last_peered":"2023-09-23T23:36:08.148457+0000","last_clean":"2023-09-23T23:36:08.148457+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:36:08.148457+0000","last_undegraded":"2023-09-23T23:36:08.148457+0000","last_fullsized":"2023-09-23T23:36:08.148457+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":590,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169901+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:36:08.169901+0000","last_peered":"2023-09-23T23:36:08.169901+0000","last_clean":"2023-09-23T23:36:08.169901+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:36:08.169901+0000","last_undegraded":"2023-09-23T23:36:08.169901+0000","last_fullsized":"2023-09-23T23:36:08.169901+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":197,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.168675+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:36:08.168675+0000","last_peered":"2023-09-23T23:36:08.168675+0000","last_clean":"2023-09-23T23:36:08.168675+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:36:08.168675+0000","last_undegraded":"2023-09-23T23:36:08.168675+0000","last_fullsized":"2023-09-23T23:36:08.168675+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":144,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.147808+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:36:08.147808+0000","last_peered":"2023-09-23T23:36:08.147808+0000","last_clean":"2023-09-23T23:36:08.147808+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:36:08.147808+0000","last_undegraded":"2023-09-23T23:36:08.147808+0000","last_fullsized":"2023-09-23T23:36:08.147808+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"593'6969","reported_seq":7038,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:19.923120+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:36:19.923120+0000","last_peered":"2023-09-23T23:36:19.923120+0000","last_clean":"2023-09-23T23:36:19.923120+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:36:19.923120+0000","last_undegraded":"2023-09-23T23:36:19.923120+0000","last_fullsized":"2023-09-23T23:36:19.923120+0000","mapping_epoch":534,"log_start":"543'800","ondisk_log_start":"543'800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6169,"log_dups_size":0,"ondisk_log_size":6169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258146304,"num_objects":3939,"num_object_clones":0,"num_object_copies":11817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6969,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":980,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.170246+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:36:08.170246+0000","last_peered":"2023-09-23T23:36:08.170246+0000","last_clean":"2023-09-23T23:36:08.170246+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:36:08.170246+0000","last_undegraded":"2023-09-23T23:36:08.170246+0000","last_fullsized":"2023-09-23T23:36:08.170246+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":107,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.168016+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:36:08.168016+0000","last_peered":"2023-09-23T23:36:08.168016+0000","last_clean":"2023-09-23T23:36:08.168016+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:36:08.168016+0000","last_undegraded":"2023-09-23T23:36:08.168016+0000","last_fullsized":"2023-09-23T23:36:08.168016+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":37,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.177373+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:36:08.177373+0000","last_peered":"2023-09-23T23:36:08.177373+0000","last_clean":"2023-09-23T23:36:08.177373+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:36:08.177373+0000","last_undegraded":"2023-09-23T23:36:08.177373+0000","last_fullsized":"2023-09-23T23:36:08.177373+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":167,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.170957+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:36:08.170957+0000","last_peered":"2023-09-23T23:36:08.170957+0000","last_clean":"2023-09-23T23:36:08.170957+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:36:08.170957+0000","last_undegraded":"2023-09-23T23:36:08.170957+0000","last_fullsized":"2023-09-23T23:36:08.170957+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1029,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169127+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:36:08.169127+0000","last_peered":"2023-09-23T23:36:08.169127+0000","last_clean":"2023-09-23T23:36:08.169127+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:36:08.169127+0000","last_undegraded":"2023-09-23T23:36:08.169127+0000","last_fullsized":"2023-09-23T23:36:08.169127+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":33,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.177866+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:36:08.177866+0000","last_peered":"2023-09-23T23:36:08.177866+0000","last_clean":"2023-09-23T23:36:08.177866+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:36:08.177866+0000","last_undegraded":"2023-09-23T23:36:08.177866+0000","last_fullsized":"2023-09-23T23:36:08.177866+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":348,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.148559+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:36:08.148559+0000","last_peered":"2023-09-23T23:36:08.148559+0000","last_clean":"2023-09-23T23:36:08.148559+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:36:08.148559+0000","last_undegraded":"2023-09-23T23:36:08.148559+0000","last_fullsized":"2023-09-23T23:36:08.148559+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":741,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169700+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:36:08.169700+0000","last_peered":"2023-09-23T23:36:08.169700+0000","last_clean":"2023-09-23T23:36:08.169700+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:36:08.169700+0000","last_undegraded":"2023-09-23T23:36:08.169700+0000","last_fullsized":"2023-09-23T23:36:08.169700+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":95,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.168995+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:36:08.168995+0000","last_peered":"2023-09-23T23:36:08.168995+0000","last_clean":"2023-09-23T23:36:08.168995+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:36:08.168995+0000","last_undegraded":"2023-09-23T23:36:08.168995+0000","last_fullsized":"2023-09-23T23:36:08.168995+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":441,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.172979+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:36:15.172979+0000","last_peered":"2023-09-23T23:36:15.172979+0000","last_clean":"2023-09-23T23:36:15.172979+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:36:15.172979+0000","last_undegraded":"2023-09-23T23:36:15.172979+0000","last_fullsized":"2023-09-23T23:36:15.172979+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":291,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.168368+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:36:08.168368+0000","last_peered":"2023-09-23T23:36:08.168368+0000","last_clean":"2023-09-23T23:36:08.168368+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:36:08.168368+0000","last_undegraded":"2023-09-23T23:36:08.168368+0000","last_fullsized":"2023-09-23T23:36:08.168368+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":280,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.169955+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:36:15.169955+0000","last_peered":"2023-09-23T23:36:15.169955+0000","last_clean":"2023-09-23T23:36:15.169955+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:36:15.169955+0000","last_undegraded":"2023-09-23T23:36:15.169955+0000","last_fullsized":"2023-09-23T23:36:15.169955+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":805,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169235+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:36:08.169235+0000","last_peered":"2023-09-23T23:36:08.169235+0000","last_clean":"2023-09-23T23:36:08.169235+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:36:08.169235+0000","last_undegraded":"2023-09-23T23:36:08.169235+0000","last_fullsized":"2023-09-23T23:36:08.169235+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":623,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169652+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:36:08.169652+0000","last_peered":"2023-09-23T23:36:08.169652+0000","last_clean":"2023-09-23T23:36:08.169652+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:36:08.169652+0000","last_undegraded":"2023-09-23T23:36:08.169652+0000","last_fullsized":"2023-09-23T23:36:08.169652+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":607,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.169322+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:36:08.169322+0000","last_peered":"2023-09-23T23:36:08.169322+0000","last_clean":"2023-09-23T23:36:08.169322+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:36:08.169322+0000","last_undegraded":"2023-09-23T23:36:08.169322+0000","last_fullsized":"2023-09-23T23:36:08.169322+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":765,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.175367+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:36:08.175367+0000","last_peered":"2023-09-23T23:36:08.175367+0000","last_clean":"2023-09-23T23:36:08.175367+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:36:08.175367+0000","last_undegraded":"2023-09-23T23:36:08.175367+0000","last_fullsized":"2023-09-23T23:36:08.175367+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":436,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.148532+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:36:08.148532+0000","last_peered":"2023-09-23T23:36:08.148532+0000","last_clean":"2023-09-23T23:36:08.148532+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:36:08.148532+0000","last_undegraded":"2023-09-23T23:36:08.148532+0000","last_fullsized":"2023-09-23T23:36:08.148532+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":660,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-09-23T23:36:08.167223+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:36:08.167223+0000","last_peered":"2023-09-23T23:36:08.167223+0000","last_clean":"2023-09-23T23:36:08.167223+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:36:08.167223+0000","last_undegraded":"2023-09-23T23:36:08.167223+0000","last_fullsized":"2023-09-23T23:36:08.167223+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":4116250648,"num_objects":62810,"num_object_clones":0,"num_object_copies":188430,"num_objects_missing_on_primary":2871,"num_objects_missing":2871,"num_objects_degraded":25541,"num_objects_misplaced":3766,"num_objects_unfound":0,"num_objects_dirty":62810,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":112746,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14234,"num_bytes_recovered":932642816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13332918272,"data_stored":13332906056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89924,"ondisk_log_size":89924,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782682,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1862044,"kb_used_data":1482644,"kb_used_omap":8,"kb_used_meta":379383,"kb_avail":91883108,"statfs":{"total":95995035648,"available":94088302592,"internally_reserved":0,"allocated":1518227456,"data_stored":1516580795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":388488261},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":1.5569999999999999,"15min":2.1800000000000002},"min":{"1min":0.23100000000000001,"5min":0.224,"15min":0.20899999999999999},"max":{"1min":0.61699999999999999,"5min":36.956000000000003,"15min":85.721000000000004},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.47399999999999998,"5min":1.522,"15min":2.2250000000000001},"min":{"1min":0.29199999999999998,"5min":0.27900000000000003,"15min":0.19800000000000001},"max":{"1min":1.4419999999999999,"5min":37.247999999999998,"15min":85.965000000000003},"last":0.311}]},{"osd":1,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":3.8900000000000001,"15min":3.169},"min":{"1min":0.26600000000000001,"5min":0.248,"15min":0.22900000000000001},"max":{"1min":3.3500000000000001,"5min":129.11199999999999,"15min":198.58000000000001},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.65600000000000003,"5min":3.665,"15min":3.206},"min":{"1min":0.29699999999999999,"5min":0.247,"15min":0.246},"max":{"1min":3.7290000000000001,"5min":129.02600000000001,"15min":198.71100000000001},"last":0.502}]},{"osd":2,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":2.871,"15min":3.7010000000000001},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.452,"5min":78.528999999999996,"15min":203.58000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.54200000000000004,"5min":1.952,"15min":3.0649999999999999},"min":{"1min":0.30399999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.2529999999999999,"5min":27.574000000000002,"15min":198.69},"last":0.39700000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":3.3290000000000002,"15min":3.5950000000000002},"min":{"1min":0.31900000000000001,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":1.591,"5min":125.511,"15min":279.26499999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.67500000000000004,"5min":5.3090000000000002,"15min":4.1210000000000004},"min":{"1min":0.374,"5min":0.32000000000000001,"15min":0.29599999999999999},"max":{"1min":3.6240000000000001,"5min":150.608,"15min":279.29899999999998},"last":0.439}]},{"osd":4,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":1.694,"15min":1.0669999999999999},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":0.68100000000000005,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.363,"15min":1.0029999999999999},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.23899999999999999},"max":{"1min":1.6080000000000001,"5min":31.68,"15min":31.68},"last":0.41699999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":3.637,"5min":1.657,"15min":1.899},"min":{"1min":0.31900000000000001,"5min":0.28699999999999998,"15min":0.26500000000000001},"max":{"1min":60.463999999999999,"5min":60.463999999999999,"15min":210.83600000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":3.5990000000000002,"5min":1.6160000000000001,"15min":1.7649999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28199999999999997},"max":{"1min":63.006,"5min":63.006,"15min":207.04599999999999},"last":0.42899999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":1.879,"15min":1.3480000000000001},"min":{"1min":0.35199999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.4419999999999999,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.59299999999999997,"5min":2.2330000000000001,"15min":1.581},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":2.3180000000000001,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.48799999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496159,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2096524,"kb_used_data":1395268,"kb_used_omap":0,"kb_used_meta":701248,"kb_avail":91648628,"statfs":{"total":95995035648,"available":93848195072,"internally_reserved":0,"allocated":1428754432,"data_stored":1427123764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":718077952},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.93500000000000005,"15min":4.6799999999999997},"min":{"1min":0.27700000000000002,"5min":0.251,"15min":0.20399999999999999},"max":{"1min":2.8450000000000002,"5min":8.8119999999999994,"15min":135.31999999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.73399999999999999,"15min":4.601},"min":{"1min":0.24399999999999999,"5min":0.23999999999999999,"15min":0.22900000000000001},"max":{"1min":1.8859999999999999,"5min":3.8239999999999998,"15min":135.34299999999999},"last":0.34300000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":1.7509999999999999,"15min":2.3759999999999999},"min":{"1min":0.253,"5min":0.251,"15min":0.245},"max":{"1min":1.944,"5min":65.891000000000005,"15min":89.605000000000004},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.64100000000000001,"5min":1.4390000000000001,"15min":2.4510000000000001},"min":{"1min":0.33400000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":2.1269999999999998,"5min":66.087000000000003,"15min":91.025999999999996},"last":0.29499999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":4.9459999999999997,"15min":4.5620000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.861,"5min":97.408000000000001,"15min":127.44499999999999},"last":2.1800000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":6.0880000000000001,"15min":4.9740000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.2120000000000002,"5min":125.539,"15min":125.539},"last":0.49099999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:35:53 2023","interfaces":[{"interface":"back","average":{"1min":7.923,"5min":7.0129999999999999,"15min":4.4459999999999997},"min":{"1min":0.316,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":73.813999999999993,"5min":138.87299999999999,"15min":138.87299999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":8.4450000000000003,"5min":6.9260000000000002,"15min":4.7350000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":73.852999999999994,"5min":138.976,"15min":138.976},"last":0.51700000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.82999999999999996,"15min":1.274},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":1.633,"5min":7.101,"15min":166.05199999999999},"last":0.379},{"interface":"front","average":{"1min":0.55800000000000005,"5min":1.0289999999999999,"15min":1.4890000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.089,"5min":8.3499999999999996,"15min":166.631},"last":0.42999999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.96699999999999997,"15min":0.82399999999999995},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.1579999999999999,"5min":11.473000000000001,"15min":11.473000000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.433,"5min":1.091,"15min":0.94099999999999995},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.249},"max":{"1min":0.76700000000000002,"5min":7.5670000000000002,"15min":8.4090000000000007},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":1.294,"15min":1.0289999999999999},"min":{"1min":0.30499999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":0.70599999999999996,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.72999999999999998,"5min":1.4019999999999999,"15min":1.0720000000000001},"min":{"1min":0.36099999999999999,"5min":0.35499999999999998,"15min":0.33500000000000002},"max":{"1min":2.665,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.5}]}]},{"osd":1,"up_from":112,"seq":481036337500,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2179464,"kb_used_data":1270528,"kb_used_omap":4,"kb_used_meta":908923,"kb_avail":91565688,"statfs":{"total":95995035648,"available":93763264512,"internally_reserved":0,"allocated":1301020672,"data_stored":1298225460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":930738062},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":1.881,"15min":3.4060000000000001},"min":{"1min":0.217,"5min":0.17799999999999999,"15min":0.16700000000000001},"max":{"1min":2.7970000000000002,"5min":47.195999999999998,"15min":130.768},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.40999999999999998,"5min":1.5469999999999999,"15min":2.726},"min":{"1min":0.24299999999999999,"5min":0.20499999999999999,"15min":0.19},"max":{"1min":1.016,"5min":46.582000000000001,"15min":134.929},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":7.1859999999999999,"15min":6.7910000000000004},"min":{"1min":0.26300000000000001,"5min":0.214,"15min":0.20100000000000001},"max":{"1min":1.8839999999999999,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.65800000000000003,"5min":7.6059999999999999,"15min":6.9160000000000004},"min":{"1min":0.27900000000000003,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":1.968,"5min":202.12,"15min":232.21000000000001},"last":0.33900000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":5.1989999999999998,"15min":5.1799999999999997},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.221},"max":{"1min":0.999,"5min":181.94399999999999,"15min":234.404},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.75900000000000001,"5min":5.29,"15min":5.0469999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":6.6829999999999998,"5min":181.98500000000001,"15min":236.33699999999999},"last":0.45500000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":1.23,"15min":1.423},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":2.1059999999999999,"5min":12.042999999999999,"15min":12.042999999999999},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.624,"5min":1.129,"15min":1.46},"min":{"1min":0.48099999999999998,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":1.2669999999999999,"5min":9.5069999999999997,"15min":18.108000000000001},"last":0.64700000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":2.9209999999999998,"15min":1.796},"min":{"1min":0.34300000000000003,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66600000000000004,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.68200000000000005,"5min":2.9940000000000002,"15min":2.113},"min":{"1min":0.40899999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.044,"5min":151.821,"15min":151.821},"last":0.69299999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.3600000000000001,"15min":1.7529999999999999},"min":{"1min":0.377,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":1.1040000000000001,"5min":8.9269999999999996,"15min":74.087999999999994},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.97699999999999998,"5min":1.585,"15min":1.968},"min":{"1min":0.42999999999999999,"5min":0.35699999999999998,"15min":0.33100000000000002},"max":{"1min":5.367,"5min":11.927,"15min":73.974999999999994},"last":0.78500000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.7390000000000001,"15min":2.6099999999999999},"min":{"1min":0.50600000000000001,"5min":0.38700000000000001,"15min":0.32900000000000001},"max":{"1min":1.3069999999999999,"5min":44.241,"15min":78.308000000000007},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.8240000000000001,"15min":2.484},"min":{"1min":0.37,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":2.1339999999999999,"5min":43.920999999999999,"15min":77.204999999999998},"last":0.72999999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024324,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1528912,"kb_used_data":322760,"kb_used_omap":5,"kb_used_meta":1206138,"kb_avail":92216240,"statfs":{"total":95995035648,"available":94429429760,"internally_reserved":0,"allocated":330506240,"data_stored":327697435,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1235085946},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:35:32 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":10.577,"15min":6.7670000000000003},"min":{"1min":0.20499999999999999,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":9.0310000000000006,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.754,"5min":10.407999999999999,"15min":5.9059999999999997},"min":{"1min":0.20200000000000001,"5min":0.19700000000000001,"15min":0.17599999999999999},"max":{"1min":5.4729999999999999,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.315}]},{"osd":2,"last update":"Sat Sep 23 23:35:57 2023","interfaces":[{"interface":"back","average":{"1min":3.2130000000000001,"5min":10.026999999999999,"15min":7.5819999999999999},"min":{"1min":0.26200000000000001,"5min":0.17699999999999999,"15min":0.17199999999999999},"max":{"1min":40.326999999999998,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":3.1720000000000002,"5min":10.18,"15min":7.6520000000000001},"min":{"1min":0.22900000000000001,"5min":0.19800000000000001,"15min":0.17599999999999999},"max":{"1min":40.396000000000001,"5min":283.161,"15min":283.161},"last":0.42699999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:35:32 2023","interfaces":[{"interface":"back","average":{"1min":2.7360000000000002,"5min":8.6509999999999998,"15min":5.4589999999999996},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":41.191000000000003,"5min":220.732,"15min":220.732},"last":0.82799999999999996},{"interface":"front","average":{"1min":2.9390000000000001,"5min":9.0440000000000005,"15min":5.5570000000000004},"min":{"1min":0.28799999999999998,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":46.688000000000002,"5min":218.78200000000001,"15min":218.78200000000001},"last":0.51500000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":1.1599999999999999,"15min":2.4769999999999999},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.70799999999999996,"5min":14.676,"15min":138.81299999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.80100000000000005,"5min":1.391,"15min":3.0830000000000002},"min":{"1min":0.33800000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.7469999999999999,"5min":9.9039999999999999,"15min":171.251},"last":0.47699999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":1.504,"15min":2.7930000000000001},"min":{"1min":0.35299999999999998,"5min":0.32700000000000001,"15min":0.32100000000000001},"max":{"1min":4.6820000000000004,"5min":15.076000000000001,"15min":171.30699999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.625,"5min":1.244,"15min":2.0819999999999999},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":1.8149999999999999,"5min":8.9909999999999997,"15min":139.40799999999999},"last":0.41199999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.325,"15min":3.097},"min":{"1min":0.38100000000000001,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":4.718,"5min":9.968,"15min":171.33600000000001},"last":0.442},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.335,"15min":3.3650000000000002},"min":{"1min":0.37,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":1.8700000000000001,"5min":9.8320000000000007,"15min":184.13499999999999},"last":0.45500000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":1.373,"15min":2.6099999999999999},"min":{"1min":0.38400000000000001,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":0.751,"5min":9.452,"15min":138.77699999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.6379999999999999,"15min":2.5030000000000001},"min":{"1min":0.379,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":1.839,"5min":9.9670000000000005,"15min":139.37899999999999},"last":0.49299999999999999}]}]},{"osd":2,"up_from":199,"seq":854698492145,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3358608,"kb_used_data":2514696,"kb_used_omap":7,"kb_used_meta":843896,"kb_avail":90386544,"statfs":{"total":95995035648,"available":92555821056,"internally_reserved":0,"allocated":2575048704,"data_stored":2572189947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":864149585},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":2.2080000000000002,"15min":3.8599999999999999},"min":{"1min":0.186,"5min":0.186,"15min":0.16700000000000001},"max":{"1min":4.7590000000000003,"5min":61.950000000000003,"15min":119.277},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.61899999999999999,"5min":2.153,"15min":3.6669999999999998},"min":{"1min":0.23999999999999999,"5min":0.20399999999999999,"15min":0.18099999999999999},"max":{"1min":2.5070000000000001,"5min":61.984999999999999,"15min":119.56100000000001},"last":0.19700000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.911,"5min":5.3010000000000002,"15min":4.5940000000000003},"min":{"1min":0.25,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":8.3640000000000008,"5min":106.765,"15min":157.98699999999999},"last":0.311},{"interface":"front","average":{"1min":0.79000000000000004,"5min":4.8959999999999999,"15min":4.577},"min":{"1min":0.253,"5min":0.222,"15min":0.222},"max":{"1min":4.0940000000000003,"5min":106.732,"15min":158.03800000000001},"last":0.378}]},{"osd":3,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":5.7800000000000002,"15min":4.7770000000000001},"min":{"1min":0.255,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":8.3070000000000004,"5min":106.70399999999999,"15min":119.366},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.65800000000000003,"5min":5.4169999999999998,"15min":4.2709999999999999},"min":{"1min":0.29299999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":3.375,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.56599999999999995}]},{"osd":4,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":2.036,"15min":2.262},"min":{"1min":0.36099999999999999,"5min":0.33700000000000002,"15min":0.25},"max":{"1min":2.6070000000000002,"5min":62.081000000000003,"15min":119.88200000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.4590000000000001,"5min":2.3860000000000001,"15min":1.865},"min":{"1min":0.36899999999999999,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":8.343,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.44500000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":3.0299999999999998,"15min":2.6480000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.26200000000000001},"max":{"1min":2.548,"5min":91.207999999999998,"15min":119.919},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.4550000000000001,"5min":3.2749999999999999,"15min":2.6160000000000001},"min":{"1min":0.33500000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.3889999999999993,"5min":91.150999999999996,"15min":119.306},"last":0.41599999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":3.5750000000000002,"15min":3.0019999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":2.5819999999999999,"5min":107.018,"15min":119.943},"last":1.04},{"interface":"front","average":{"1min":1.486,"5min":2.6440000000000001,"15min":2.1600000000000001},"min":{"1min":0.371,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":8.4060000000000006,"5min":62.493000000000002,"15min":62.493000000000002},"last":0.497}]},{"osd":7,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":2.2010000000000001,"15min":1.9179999999999999},"min":{"1min":0.40000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.381,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.60099999999999998,"5min":3.464,"15min":2.3620000000000001},"min":{"1min":0.42699999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.417,"5min":106.809,"15min":106.809},"last":0.51400000000000001}]}]},{"osd":3,"up_from":148,"seq":635655160114,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2235268,"kb_used_data":1329532,"kb_used_omap":2,"kb_used_meta":905725,"kb_avail":91509884,"statfs":{"total":95995035648,"available":93706121216,"internally_reserved":0,"allocated":1361440768,"data_stored":1358647272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":927462453},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7390000000000001,"5min":1.4059999999999999,"15min":2.79},"min":{"1min":0.19600000000000001,"5min":0.19400000000000001,"15min":0.158},"max":{"1min":15.603,"5min":17.213000000000001,"15min":103.866},"last":0.184},{"interface":"front","average":{"1min":1.5880000000000001,"5min":1.877,"15min":2.9049999999999998},"min":{"1min":0.223,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":12.202,"5min":22.754000000000001,"15min":101.884},"last":0.27100000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":3.2200000000000002,"15min":2.9969999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19600000000000001},"max":{"1min":7.3319999999999999,"5min":56.119,"15min":85.242999999999995},"last":0.20000000000000001},{"interface":"front","average":{"1min":1.4910000000000001,"5min":2.427,"15min":2.7999999999999998},"min":{"1min":0.22800000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":11.228,"5min":53.962000000000003,"15min":85.302000000000007},"last":0.254}]},{"osd":2,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":4.7770000000000001,"15min":4.7859999999999996},"min":{"1min":0.25800000000000001,"5min":0.25,"15min":0.19800000000000001},"max":{"1min":2.1190000000000002,"5min":73.742000000000004,"15min":101.279},"last":14.965},{"interface":"front","average":{"1min":0.45000000000000001,"5min":4.6349999999999998,"15min":4.6130000000000004},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.214},"max":{"1min":0.86699999999999999,"5min":76.525999999999996,"15min":101.294},"last":15.058999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.3959999999999999,"15min":1.339},"min":{"1min":0.28799999999999998,"5min":0.28499999999999998,"15min":0.26700000000000002},"max":{"1min":11.209,"5min":11.209,"15min":11.209},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.2829999999999999,"5min":1.905,"15min":1.8049999999999999},"min":{"1min":0.33500000000000002,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":7.3940000000000001,"5min":11.494999999999999,"15min":24.782},"last":0.46400000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4410000000000001,"5min":6.8159999999999998,"15min":3.4289999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":7.3120000000000003,"5min":214.43700000000001,"15min":214.43700000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.2470000000000001,"5min":6.7569999999999997,"15min":3.4900000000000002},"min":{"1min":0.34699999999999998,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":7.2290000000000001,"5min":214.46600000000001,"15min":214.46600000000001},"last":0.40999999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":2.7090000000000001,"15min":2.3159999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":9.798,"5min":58.932000000000002,"15min":58.932000000000002},"last":0.433},{"interface":"front","average":{"1min":1.25,"5min":2.8660000000000001,"15min":2.3170000000000002},"min":{"1min":0.36499999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.202,"5min":59.186,"15min":59.186},"last":0.442}]},{"osd":7,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":4.7480000000000002,"15min":3.2069999999999999},"min":{"1min":0.38700000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.415,"5min":192.69,"15min":192.69},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.753,"5min":5.1289999999999996,"15min":3.6949999999999998},"min":{"1min":0.43099999999999999,"5min":0.39200000000000002,"15min":0.36699999999999999},"max":{"1min":2.6560000000000001,"5min":192.65600000000001,"15min":192.65600000000001},"last":0.40100000000000002}]}]},{"osd":4,"up_from":0,"seq":128849019432,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757795,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3075400,"kb_used_data":2342720,"kb_used_omap":0,"kb_used_meta":732672,"kb_avail":90669752,"statfs":{"total":95995035648,"available":92845826048,"internally_reserved":0,"allocated":2398945280,"data_stored":2397314829,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":750256128},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:14 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":1.141,"15min":3.198},"min":{"1min":0.252,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":1.5640000000000001,"5min":21.109000000000002,"15min":113.869},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.38,"5min":1.3939999999999999,"15min":3.5710000000000002},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.22700000000000001},"max":{"1min":0.76200000000000001,"5min":20.876999999999999,"15min":138.55199999999999},"last":0.40799999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:35:16 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.772,"15min":2.323},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.23100000000000001},"max":{"1min":3.629,"5min":29.007999999999999,"15min":69.816999999999993},"last":1.9930000000000001},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.9390000000000001,"15min":2.665},"min":{"1min":0.29999999999999999,"5min":0.249,"15min":0.249},"max":{"1min":4.4859999999999998,"5min":29.053999999999998,"15min":70.268000000000001},"last":1.095}]},{"osd":2,"last update":"Sat Sep 23 23:35:47 2023","interfaces":[{"interface":"back","average":{"1min":7.7469999999999999,"5min":6.5350000000000001,"15min":4.609},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.26900000000000002},"max":{"1min":145.93199999999999,"5min":208.738,"15min":208.738},"last":2.3370000000000002},{"interface":"front","average":{"1min":1.0269999999999999,"5min":5.0499999999999998,"15min":4.1150000000000002},"min":{"1min":0.31,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":2.859,"5min":208.631,"15min":208.631},"last":2.2610000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:35:44 2023","interfaces":[{"interface":"back","average":{"1min":6.4889999999999999,"5min":4.1520000000000001,"15min":3.6589999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":122.071,"5min":122.071,"15min":122.071},"last":2.024},{"interface":"front","average":{"1min":6.4669999999999996,"5min":4.4969999999999999,"15min":3.8359999999999999},"min":{"1min":0.35799999999999998,"5min":0.35399999999999998,"15min":0.318},"max":{"1min":122.09699999999999,"5min":122.09699999999999,"15min":122.09699999999999},"last":3.7250000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.151,"15min":0.92000000000000004},"min":{"1min":0.34200000000000003,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":3.5499999999999998,"5min":9.2690000000000001,"15min":19.962},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.093,"15min":0.872},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.22900000000000001},"max":{"1min":2.4990000000000001,"5min":10.273999999999999,"15min":20.829999999999998},"last":0.34699999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":7.2519999999999998,"5min":2.3799999999999999,"15min":1.635},"min":{"1min":0.32400000000000001,"5min":0.27600000000000002,"15min":0.26900000000000002},"max":{"1min":148.36600000000001,"5min":148.36600000000001,"15min":148.36600000000001},"last":0.314},{"interface":"front","average":{"1min":7.7210000000000001,"5min":2.9420000000000002,"15min":1.869},"min":{"1min":0.34599999999999997,"5min":0.307,"15min":0.26000000000000001},"max":{"1min":148.42500000000001,"5min":148.42500000000001,"15min":148.42500000000001},"last":0.42999999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:35:47 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.3160000000000001,"15min":2.625},"min":{"1min":0.34699999999999998,"5min":0.34599999999999997,"15min":0.317},"max":{"1min":1.843,"5min":15.497,"15min":179.28800000000001},"last":0.496},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.3480000000000001,"15min":2.7050000000000001},"min":{"1min":0.36699999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.867,"5min":15.653,"15min":179.428},"last":0.44900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":322174976,"data_stored":322174976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1287983104,"data_stored":1287979032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2564489216,"data_stored":2564489216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1349652480,"data_stored":1349648408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2487156736,"data_stored":2487156736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2390032384,"data_stored":2390032384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1421410304,"data_stored":1421410304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1510019072,"data_stored":1510015000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:36:25.176 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:25.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:25.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:25.280 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:25.310 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:25 smithi110 ceph-mon[221102]: pgmap v1650: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 158 op/s; 25541/188502 objects degraded (13.549%); 3766/188502 objects misplaced (1.998%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:36:25.313 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:25 smithi110 ceph-mon[212939]: pgmap v1650: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 158 op/s; 25541/188502 objects degraded (13.549%); 3766/188502 objects misplaced (1.998%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:36:25.315 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:25.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:25 smithi138 ceph-mon[193606]: pgmap v1650: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 158 op/s; 25541/188502 objects degraded (13.549%); 3766/188502 objects misplaced (1.998%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:36:26.260 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:26.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:26.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:26.364 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:26.400 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:26.477 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:26.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:26.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:26.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:26.631 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:26.746 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:26.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:26.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:26.771 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:36:26.771 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:36:26.854 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:26.881 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:26.923 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:26.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:26.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:26.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:36:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:36:26] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.43.0" 2023-09-23T23:36:26.987 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:36:27.007 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:36:27.027 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:36:27.029 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:36:27.046 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:36:27.057 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:27.066 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:36:27.088 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:36:27.103 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:36:27.109 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:36:27.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:27.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:36:27.138 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:36:27.139 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:36:27.139 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:36:27.139 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:36:27.139 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:36:27.140 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:36:27.140 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:36:27.140 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:36:27.140 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:36:27.184 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:36:27.210 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:27.265 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:27.332 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:27.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:27.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:27.421 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:27 smithi110 ceph-mon[221102]: pgmap v1651: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 241 op/s; 25468/185418 objects degraded (13.735%); 3766/185418 objects misplaced (2.031%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:36:27.421 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:27 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3712752088' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:27.422 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:27 smithi110 ceph-mon[212939]: pgmap v1651: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 241 op/s; 25468/185418 objects degraded (13.735%); 3766/185418 objects misplaced (2.031%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:36:27.422 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:27 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3712752088' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:27.440 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:27.502 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:27.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:27 smithi138 ceph-mon[193606]: pgmap v1651: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 241 op/s; 25468/185418 objects degraded (13.735%); 3766/185418 objects misplaced (2.031%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:36:27.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:27 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3712752088' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:28.094 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:36:28.216 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:28.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:28.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:28.289 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:36:28.325 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:28.359 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:28.415 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:28.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:28.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:28.522 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:28.537 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:36:28.537 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:36:28.587 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:28.668 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:36:28.702 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:28.722 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:28.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:28.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:28.834 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:36:28.860 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:28.907 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:36:28.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:28.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:36:29.014 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:29.049 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:29.161 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:29 smithi138 ceph-mon[193606]: pgmap v1652: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 246 op/s; 25396/184488 objects degraded (13.766%); 3694/184488 objects misplaced (2.002%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:29 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 25396/184488 objects degraded (13.766%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:29.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:29 smithi110 ceph-mon[212939]: pgmap v1652: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 246 op/s; 25396/184488 objects degraded (13.766%); 3694/184488 objects misplaced (2.002%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:29.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:29 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 25396/184488 objects degraded (13.766%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:29.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:29 smithi110 ceph-mon[221102]: pgmap v1652: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 246 op/s; 25396/184488 objects degraded (13.766%); 3694/184488 objects misplaced (2.002%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:29.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:29 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 25396/184488 objects degraded (13.766%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:30.467 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:30.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:30.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:30.570 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:36:30.572 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:36:30.599 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:30.641 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:36:30.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:30.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:36:30.747 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:30.782 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:31.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:31 smithi110 ceph-mon[212939]: pgmap v1653: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 25296/182259 objects degraded (13.879%); 3694/182259 objects misplaced (2.027%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T23:36:31.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:31 smithi110 ceph-mon[221102]: pgmap v1653: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 25296/182259 objects degraded (13.879%); 3694/182259 objects misplaced (2.027%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T23:36:31.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:31 smithi138 ceph-mon[193606]: pgmap v1653: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 25296/182259 objects degraded (13.879%); 3694/182259 objects misplaced (2.027%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T23:36:31.748 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:31.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:31.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:31.851 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:36:31.877 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:31.925 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:36:31.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:31.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:36:32.031 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:32.070 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:32.276 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:32.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:32.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:32.293 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:36:32.323 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:32 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3994338032' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:32.324 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:32 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3994338032' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:32.385 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:32.462 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:32.526 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:32.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:32.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:32.543 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:32.567 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:36:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:36:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:36:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:36:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178600 11.6 11:659f4d42:::benchmark_data_smithi110_433746_object72720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2023-09-23T23:36:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.523923+0000", 2023-09-23T23:36:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 553.03819255999997, 2023-09-23T23:36:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2361555310000001, 2023-09-23T23:36:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:36:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178600 2023-09-23T23:36:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523923+0000", 2023-09-23T23:36:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523923+0000", 2023-09-23T23:36:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523922+0000", 2023-09-23T23:36:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523929+0000", 2023-09-23T23:36:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8800000000000002e-06 2023-09-23T23:36:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523940+0000", 2023-09-23T23:36:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.149e-05 2023-09-23T23:36:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.523948+0000", 2023-09-23T23:36:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7970000000000001e-06 2023-09-23T23:36:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.531901+0000", 2023-09-23T23:36:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079527399999999998 2023-09-23T23:36:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.532034+0000", 2023-09-23T23:36:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013345499999999999 2023-09-23T23:36:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:36:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.532514+0000", 2023-09-23T23:36:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047994200000000002 2023-09-23T23:36:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.537104+0000", 2023-09-23T23:36:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045898019999999996 2023-09-23T23:36:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.538159+0000", 2023-09-23T23:36:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001055437 2023-09-23T23:36:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.547877+0000", 2023-09-23T23:36:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0097177860000000008 2023-09-23T23:36:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.547890+0000", 2023-09-23T23:36:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2656e-05 2023-09-23T23:36:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542710+0000", 2023-09-23T23:36:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99482022199999998 2023-09-23T23:36:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542720+0000", 2023-09-23T23:36:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9029999999999999e-06 2023-09-23T23:36:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760000+0000", 2023-09-23T23:36:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.217280103 2023-09-23T23:36:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760079+0000", 2023-09-23T23:36:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8777999999999997e-05 2023-09-23T23:36:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178608 11.6 11:6177c8c0:::benchmark_data_smithi110_433746_object72728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2023-09-23T23:36:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.545706+0000", 2023-09-23T23:36:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 553.01641013200003, 2023-09-23T23:36:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2481279590000001, 2023-09-23T23:36:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178608 2023-09-23T23:36:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545706+0000", 2023-09-23T23:36:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545706+0000", 2023-09-23T23:36:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545705+0000", 2023-09-23T23:36:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:36:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545709+0000", 2023-09-23T23:36:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6140000000000002e-06 2023-09-23T23:36:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545721+0000", 2023-09-23T23:36:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2345e-05 2023-09-23T23:36:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.545730+0000", 2023-09-23T23:36:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9579999999999996e-06 2023-09-23T23:36:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.570191+0000", 2023-09-23T23:36:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024460784999999999 2023-09-23T23:36:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.570202+0000", 2023-09-23T23:36:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1192999999999999e-05 2023-09-23T23:36:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542744+0000", 2023-09-23T23:36:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.972541879 2023-09-23T23:36:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542752+0000", 2023-09-23T23:36:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1599999999999998e-06 2023-09-23T23:36:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760100+0000", 2023-09-23T23:36:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2173479519999999 2023-09-23T23:36:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760221+0000", 2023-09-23T23:36:32.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000120155 2023-09-23T23:36:32.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T23:36:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760707+0000", 2023-09-23T23:36:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00048629800000000003 2023-09-23T23:36:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.774649+0000", 2023-09-23T23:36:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013942049 2023-09-23T23:36:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783406+0000", 2023-09-23T23:36:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0087574420000000007 2023-09-23T23:36:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783684+0000", 2023-09-23T23:36:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027739099999999999 2023-09-23T23:36:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793677+0000", 2023-09-23T23:36:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0099932059999999993 2023-09-23T23:36:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793723+0000", 2023-09-23T23:36:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5883000000000001e-05 2023-09-23T23:36:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793834+0000", 2023-09-23T23:36:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110819 2023-09-23T23:36:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178617 11.16 11:6d8a65a1:::benchmark_data_smithi110_433746_object72737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:36:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.584811+0000", 2023-09-23T23:36:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 552.97730473199999, 2023-09-23T23:36:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1985552099999999, 2023-09-23T23:36:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178617 2023-09-23T23:36:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584811+0000", 2023-09-23T23:36:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584811+0000", 2023-09-23T23:36:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584809+0000", 2023-09-23T23:36:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584817+0000", 2023-09-23T23:36:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3339999999999996e-06 2023-09-23T23:36:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584831+0000", 2023-09-23T23:36:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4355000000000001e-05 2023-09-23T23:36:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.584840+0000", 2023-09-23T23:36:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3410000000000003e-06 2023-09-23T23:36:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591289+0000", 2023-09-23T23:36:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0064494030000000003 2023-09-23T23:36:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591300+0000", 2023-09-23T23:36:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0597e-05 2023-09-23T23:36:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542631+0000", 2023-09-23T23:36:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.951331449 2023-09-23T23:36:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542644+0000", 2023-09-23T23:36:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.288e-05 2023-09-23T23:36:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.756708+0000", 2023-09-23T23:36:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.21406408 2023-09-23T23:36:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.756818+0000", 2023-09-23T23:36:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110073 2023-09-23T23:36:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T23:36:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.757269+0000", 2023-09-23T23:36:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045095900000000002 2023-09-23T23:36:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764656+0000", 2023-09-23T23:36:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007387195 2023-09-23T23:36:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764667+0000", 2023-09-23T23:36:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0365e-05 2023-09-23T23:36:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.774688+0000", 2023-09-23T23:36:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010021577 2023-09-23T23:36:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783206+0000", 2023-09-23T23:36:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0085183289999999998 2023-09-23T23:36:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783262+0000", 2023-09-23T23:36:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5275999999999997e-05 2023-09-23T23:36:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783366+0000", 2023-09-23T23:36:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010458099999999999 2023-09-23T23:36:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178622 11.6 11:6344b54e:::benchmark_data_smithi110_433746_object72742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:36:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.591902+0000", 2023-09-23T23:36:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 552.97021375300005, 2023-09-23T23:36:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2021504429999998, 2023-09-23T23:36:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178622 2023-09-23T23:36:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591902+0000", 2023-09-23T23:36:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591902+0000", 2023-09-23T23:36:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591901+0000", 2023-09-23T23:36:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591906+0000", 2023-09-23T23:36:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8289999999999996e-06 2023-09-23T23:36:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591921+0000", 2023-09-23T23:36:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4149e-05 2023-09-23T23:36:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.591928+0000", 2023-09-23T23:36:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0999999999999998e-06 2023-09-23T23:36:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.592603+0000", 2023-09-23T23:36:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00067509500000000001 2023-09-23T23:36:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.592613+0000", 2023-09-23T23:36:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0671e-05 2023-09-23T23:36:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542776+0000", 2023-09-23T23:36:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95016243199999995 2023-09-23T23:36:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542784+0000", 2023-09-23T23:36:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5580000000000001e-06 2023-09-23T23:36:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764299+0000", 2023-09-23T23:36:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2215145999999999 2023-09-23T23:36:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764438+0000", 2023-09-23T23:36:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000138536 2023-09-23T23:36:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T23:36:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.765798+0000", 2023-09-23T23:36:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001360874 2023-09-23T23:36:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.774753+0000", 2023-09-23T23:36:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0089545679999999996 2023-09-23T23:36:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783698+0000", 2023-09-23T23:36:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0089451630000000008 2023-09-23T23:36:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.785057+0000", 2023-09-23T23:36:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001358784 2023-09-23T23:36:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793921+0000", 2023-09-23T23:36:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0088636380000000001 2023-09-23T23:36:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793959+0000", 2023-09-23T23:36:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8328999999999998e-05 2023-09-23T23:36:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.794052+0000", 2023-09-23T23:36:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3539e-05 2023-09-23T23:36:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178633 11.16 11:6db526fe:::benchmark_data_smithi110_433746_object72753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:36:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.633733+0000", 2023-09-23T23:36:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 552.92838241799996, 2023-09-23T23:36:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.160140325, 2023-09-23T23:36:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178633 2023-09-23T23:36:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633733+0000", 2023-09-23T23:36:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633733+0000", 2023-09-23T23:36:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633732+0000", 2023-09-23T23:36:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633739+0000", 2023-09-23T23:36:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2899999999999997e-06 2023-09-23T23:36:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633756+0000", 2023-09-23T23:36:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6368999999999999e-05 2023-09-23T23:36:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633763+0000", 2023-09-23T23:36:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8930000000000005e-06 2023-09-23T23:36:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633828+0000", 2023-09-23T23:36:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4092999999999998e-05 2023-09-23T23:36:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.633843+0000", 2023-09-23T23:36:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5512e-05 2023-09-23T23:36:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542670+0000", 2023-09-23T23:36:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.90882660500000001 2023-09-23T23:36:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.542679+0000", 2023-09-23T23:36:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0000000000000002e-06 2023-09-23T23:36:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760612+0000", 2023-09-23T23:36:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.217933355 2023-09-23T23:36:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.760705+0000", 2023-09-23T23:36:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3078999999999994e-05 2023-09-23T23:36:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T23:36:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.761116+0000", 2023-09-23T23:36:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00041080000000000001 2023-09-23T23:36:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.764673+0000", 2023-09-23T23:36:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035569239999999999 2023-09-23T23:36:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.767908+0000", 2023-09-23T23:36:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032356749999999999 2023-09-23T23:36:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.783437+0000", 2023-09-23T23:36:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015528767000000001 2023-09-23T23:36:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793721+0000", 2023-09-23T23:36:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010284188 2023-09-23T23:36:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793766+0000", 2023-09-23T23:36:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4631e-05 2023-09-23T23:36:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.793874+0000", 2023-09-23T23:36:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010767100000000001 2023-09-23T23:36:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178629 11.7 11:e3eb7c4c:::benchmark_data_smithi110_433746_object72749:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e425)", 2023-09-23T23:36:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:20.543856+0000", 2023-09-23T23:36:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 552.01826002400003, 2023-09-23T23:36:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2512707000000001, 2023-09-23T23:36:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178629 2023-09-23T23:36:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543856+0000", 2023-09-23T23:36:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543856+0000", 2023-09-23T23:36:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543855+0000", 2023-09-23T23:36:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543857+0000", 2023-09-23T23:36:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.99e-06 2023-09-23T23:36:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543862+0000", 2023-09-23T23:36:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.899e-06 2023-09-23T23:36:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.543867+0000", 2023-09-23T23:36:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3710000000000002e-06 2023-09-23T23:36:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.544825+0000", 2023-09-23T23:36:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000957864 2023-09-23T23:36:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.544832+0000", 2023-09-23T23:36:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3649999999999998e-06 2023-09-23T23:36:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.780766+0000", 2023-09-23T23:36:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2359338689999999 2023-09-23T23:36:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.780888+0000", 2023-09-23T23:36:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000121833 2023-09-23T23:36:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T23:36:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.781338+0000", 2023-09-23T23:36:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045035600000000001 2023-09-23T23:36:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.784204+0000", 2023-09-23T23:36:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002865884 2023-09-23T23:36:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.785473+0000", 2023-09-23T23:36:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001268838 2023-09-23T23:36:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.788590+0000", 2023-09-23T23:36:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031171839999999998 2023-09-23T23:36:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.794983+0000", 2023-09-23T23:36:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063934339999999999 2023-09-23T23:36:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.795024+0000", 2023-09-23T23:36:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0828000000000003e-05 2023-09-23T23:36:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.795126+0000", 2023-09-23T23:36:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102277 2023-09-23T23:36:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211753 11.19 11:9cef2dd2:::benchmark_data_smithi110_433746_object105873:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T23:36:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:24.737706+0000", 2023-09-23T23:36:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 487.82440960600002, 2023-09-23T23:36:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.967897013, 2023-09-23T23:36:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211753 2023-09-23T23:36:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737706+0000", 2023-09-23T23:36:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737706+0000", 2023-09-23T23:36:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737704+0000", 2023-09-23T23:36:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:36:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737713+0000", 2023-09-23T23:36:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3939999999999996e-06 2023-09-23T23:36:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737727+0000", 2023-09-23T23:36:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3893999999999999e-05 2023-09-23T23:36:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737736+0000", 2023-09-23T23:36:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0780000000000002e-06 2023-09-23T23:36:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737803+0000", 2023-09-23T23:36:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6959999999999996e-05 2023-09-23T23:36:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:24.737815+0000", 2023-09-23T23:36:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2377e-05 2023-09-23T23:36:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:25.604895+0000", 2023-09-23T23:36:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86708047300000002 2023-09-23T23:36:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:25.604906+0000", 2023-09-23T23:36:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0903999999999999e-05 2023-09-23T23:36:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.701468+0000", 2023-09-23T23:36:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.096561801 2023-09-23T23:36:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.701626+0000", 2023-09-23T23:36:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015756900000000001 2023-09-23T23:36:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,3,7", 2023-09-23T23:36:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.701760+0000", 2023-09-23T23:36:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013463700000000001 2023-09-23T23:36:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.704681+0000", 2023-09-23T23:36:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029204080000000002 2023-09-23T23:36:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.704693+0000", 2023-09-23T23:36:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2427e-05 2023-09-23T23:36:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.704736+0000", 2023-09-23T23:36:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2911000000000003e-05 2023-09-23T23:36:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.704783+0000", 2023-09-23T23:36:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6931999999999997e-05 2023-09-23T23:36:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.705399+0000", 2023-09-23T23:36:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000615889 2023-09-23T23:36:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.705458+0000", 2023-09-23T23:36:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8731999999999999e-05 2023-09-23T23:36:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:26.705603+0000", 2023-09-23T23:36:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014554600000000001 2023-09-23T23:36:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:175261 12.10 12:0f9d8e44:::benchmark_data_smithi110_491193_object80386:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2023-09-23T23:36:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.428760+0000", 2023-09-23T23:36:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 265.13335585999999, 2023-09-23T23:36:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.279936254, 2023-09-23T23:36:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:36:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:36:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175261 2023-09-23T23:36:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428760+0000", 2023-09-23T23:36:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428760+0000", 2023-09-23T23:36:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428758+0000", 2023-09-23T23:36:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428765+0000", 2023-09-23T23:36:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8680000000000001e-06 2023-09-23T23:36:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428778+0000", 2023-09-23T23:36:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2687000000000001e-05 2023-09-23T23:36:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428785+0000", 2023-09-23T23:36:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1679999999999998e-06 2023-09-23T23:36:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428858+0000", 2023-09-23T23:36:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3231000000000004e-05 2023-09-23T23:36:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428968+0000", 2023-09-23T23:36:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010950699999999999 2023-09-23T23:36:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:36:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.429420+0000", 2023-09-23T23:36:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045183000000000002 2023-09-23T23:36:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.432669+0000", 2023-09-23T23:36:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032491830000000001 2023-09-23T23:36:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.641 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708584+0000", 2023-09-23T23:36:32.641 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.27591494900000002 2023-09-23T23:36:32.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.641 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.641 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708613+0000", 2023-09-23T23:36:32.641 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9178000000000001e-05 2023-09-23T23:36:32.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.641 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.641 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708659+0000", 2023-09-23T23:36:32.641 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6415000000000002e-05 2023-09-23T23:36:32.642 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.642 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.642 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708696+0000", 2023-09-23T23:36:32.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6832000000000002e-05 2023-09-23T23:36:32.642 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.642 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.642 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.642 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.642 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:175263 12.8 e524/490 12:11714bff:::benchmark_data_smithi110_491193_object80388:head v 524'7666, mlcod=524'7664)", 2023-09-23T23:36:32.643 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.433325+0000", 2023-09-23T23:36:32.643 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 265.128790455, 2023-09-23T23:36:32.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.282317341, 2023-09-23T23:36:32.643 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.643 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.643 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433325+0000", 2023-09-23T23:36:32.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433325+0000", 2023-09-23T23:36:32.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433325+0000", 2023-09-23T23:36:32.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:36:32.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433333+0000", 2023-09-23T23:36:32.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5420000000000005e-06 2023-09-23T23:36:32.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433344+0000", 2023-09-23T23:36:32.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1358e-05 2023-09-23T23:36:32.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433351+0000", 2023-09-23T23:36:32.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7209999999999997e-06 2023-09-23T23:36:32.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433384+0000", 2023-09-23T23:36:32.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3065000000000003e-05 2023-09-23T23:36:32.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433404+0000", 2023-09-23T23:36:32.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9762000000000002e-05 2023-09-23T23:36:32.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.715624+0000", 2023-09-23T23:36:32.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28222047 2023-09-23T23:36:32.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.715643+0000", 2023-09-23T23:36:32.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8238999999999999e-05 2023-09-23T23:36:32.647 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.647 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.647 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.647 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:175265 12.10 12:09b7804b:::benchmark_data_smithi110_491193_object80390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2023-09-23T23:36:32.647 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.434661+0000", 2023-09-23T23:36:32.647 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 265.12745446000002, 2023-09-23T23:36:32.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28935229200000001, 2023-09-23T23:36:32.648 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.648 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.648 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:32.648 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:36:32.648 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:36:32.648 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175265 2023-09-23T23:36:32.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.648 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.434661+0000", 2023-09-23T23:36:32.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.434661+0000", 2023-09-23T23:36:32.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.434660+0000", 2023-09-23T23:36:32.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:32.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.650 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.434667+0000", 2023-09-23T23:36:32.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6919999999999996e-06 2023-09-23T23:36:32.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.650 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708315+0000", 2023-09-23T23:36:32.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.27364855599999999 2023-09-23T23:36:32.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708328+0000", 2023-09-23T23:36:32.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2444e-05 2023-09-23T23:36:32.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.712241+0000", 2023-09-23T23:36:32.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039134829999999997 2023-09-23T23:36:32.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.712351+0000", 2023-09-23T23:36:32.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110096 2023-09-23T23:36:32.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.652 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:36:32.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.712806+0000", 2023-09-23T23:36:32.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000454604 2023-09-23T23:36:32.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.652 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.718758+0000", 2023-09-23T23:36:32.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059521080000000002 2023-09-23T23:36:32.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.652 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:32.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.718791+0000", 2023-09-23T23:36:32.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3618000000000002e-05 2023-09-23T23:36:32.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:32.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.723932+0000", 2023-09-23T23:36:32.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051407349999999996 2023-09-23T23:36:32.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.723976+0000", 2023-09-23T23:36:32.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3618000000000001e-05 2023-09-23T23:36:32.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.724014+0000", 2023-09-23T23:36:32.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7829e-05 2023-09-23T23:36:32.654 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.654 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.654 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.654 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:175259 12.c e524/476 12:3a3a4253:::benchmark_data_smithi110_491193_object80384:head v 524'10995, mlcod=524'10989)", 2023-09-23T23:36:32.654 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.436466+0000", 2023-09-23T23:36:32.654 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 265.12564941699998, 2023-09-23T23:36:32.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28907641299999998, 2023-09-23T23:36:32.655 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.655 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.655 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436466+0000", 2023-09-23T23:36:32.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436466+0000", 2023-09-23T23:36:32.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.656 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.656 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436466+0000", 2023-09-23T23:36:32.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:32.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.656 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.656 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436473+0000", 2023-09-23T23:36:32.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4479999999999997e-06 2023-09-23T23:36:32.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708281+0000", 2023-09-23T23:36:32.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.27180817299999999 2023-09-23T23:36:32.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708296+0000", 2023-09-23T23:36:32.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4875000000000001e-05 2023-09-23T23:36:32.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708345+0000", 2023-09-23T23:36:32.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9089999999999999e-05 2023-09-23T23:36:32.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708371+0000", 2023-09-23T23:36:32.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5809e-05 2023-09-23T23:36:32.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.725504+0000", 2023-09-23T23:36:32.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017133116 2023-09-23T23:36:32.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.725543+0000", 2023-09-23T23:36:32.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8574e-05 2023-09-23T23:36:32.659 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.659 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.659 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.659 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:114586 13.5 e594/570 13:a727bcea:::benchmark_data_smithi110_541928_object26806:head v 594'7313, mlcod=538'122)", 2023-09-23T23:36:32.659 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:36:27.252750+0000", 2023-09-23T23:36:32.659 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 5.3093657939999996, 2023-09-23T23:36:32.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016055679999999999, 2023-09-23T23:36:32.660 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.660 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.660 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:27.252750+0000", 2023-09-23T23:36:32.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:27.252750+0000", 2023-09-23T23:36:32.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:27.252749+0000", 2023-09-23T23:36:32.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:32.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:27.252761+0000", 2023-09-23T23:36:32.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2148e-05 2023-09-23T23:36:32.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:27.252784+0000", 2023-09-23T23:36:32.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3541e-05 2023-09-23T23:36:32.662 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:27.252796+0000", 2023-09-23T23:36:32.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1083999999999999e-05 2023-09-23T23:36:32.662 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:27.252867+0000", 2023-09-23T23:36:32.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1849999999999998e-05 2023-09-23T23:36:32.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.663 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:27.252904+0000", 2023-09-23T23:36:32.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6270999999999999e-05 2023-09-23T23:36:32.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.663 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:27.268754+0000", 2023-09-23T23:36:32.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015850336999999999 2023-09-23T23:36:32.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.663 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.664 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:27.268806+0000", 2023-09-23T23:36:32.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1725999999999999e-05 2023-09-23T23:36:32.664 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.664 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.664 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.664 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:115193 13.5 e594/570 13:aaa429cb:::benchmark_data_smithi110_541928_object27413:head v 594'7348, mlcod=538'122)", 2023-09-23T23:36:32.664 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:36:29.365254+0000", 2023-09-23T23:36:32.664 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 3.1968621439999998, 2023-09-23T23:36:32.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013362206999999999, 2023-09-23T23:36:32.664 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.665 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.665 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:29.365254+0000", 2023-09-23T23:36:32.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:29.365254+0000", 2023-09-23T23:36:32.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:29.365253+0000", 2023-09-23T23:36:32.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:36:32.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:29.365259+0000", 2023-09-23T23:36:32.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6030000000000001e-06 2023-09-23T23:36:32.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:29.365272+0000", 2023-09-23T23:36:32.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3002e-05 2023-09-23T23:36:32.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.667 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:29.365279+0000", 2023-09-23T23:36:32.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1249999999999998e-06 2023-09-23T23:36:32.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.667 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:29.365316+0000", 2023-09-23T23:36:32.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7212000000000002e-05 2023-09-23T23:36:32.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:29.365340+0000", 2023-09-23T23:36:32.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.427e-05 2023-09-23T23:36:32.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:29.378573+0000", 2023-09-23T23:36:32.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013232436 2023-09-23T23:36:32.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:29.378616+0000", 2023-09-23T23:36:32.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3340999999999997e-05 2023-09-23T23:36:32.669 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.669 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.669 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.669 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:115371 13.5 e594/570 13:a53626fe:::benchmark_data_smithi110_541928_object27591:head v 594'7358, mlcod=538'122)", 2023-09-23T23:36:32.669 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:36:30.184714+0000", 2023-09-23T23:36:32.669 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 2.3774022759999998, 2023-09-23T23:36:32.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016346006999999999, 2023-09-23T23:36:32.669 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.669 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.669 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.670 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.184714+0000", 2023-09-23T23:36:32.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.670 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.184714+0000", 2023-09-23T23:36:32.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.184712+0000", 2023-09-23T23:36:32.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:36:32.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.671 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.184722+0000", 2023-09-23T23:36:32.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0569999999999999e-06 2023-09-23T23:36:32.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.671 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.184735+0000", 2023-09-23T23:36:32.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3475000000000001e-05 2023-09-23T23:36:32.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.672 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.184742+0000", 2023-09-23T23:36:32.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8510000000000001e-06 2023-09-23T23:36:32.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.672 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.184786+0000", 2023-09-23T23:36:32.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4243999999999999e-05 2023-09-23T23:36:32.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.184808+0000", 2023-09-23T23:36:32.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1721000000000001e-05 2023-09-23T23:36:32.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.673 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.201021+0000", 2023-09-23T23:36:32.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016212956000000001 2023-09-23T23:36:32.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.673 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.201060+0000", 2023-09-23T23:36:32.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8733000000000003e-05 2023-09-23T23:36:32.674 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.674 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.674 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.674 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:115372 13.5 e594/570 13:a089f335:::benchmark_data_smithi110_541928_object27592:head v 594'7359, mlcod=538'122)", 2023-09-23T23:36:32.674 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:36:30.187497+0000", 2023-09-23T23:36:32.674 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 2.374618769, 2023-09-23T23:36:32.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01357818, 2023-09-23T23:36:32.674 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.674 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.674 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.187497+0000", 2023-09-23T23:36:32.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.187497+0000", 2023-09-23T23:36:32.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.187496+0000", 2023-09-23T23:36:32.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:32.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.676 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.676 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.187506+0000", 2023-09-23T23:36:32.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0562e-05 2023-09-23T23:36:32.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.676 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.676 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.187520+0000", 2023-09-23T23:36:32.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4024e-05 2023-09-23T23:36:32.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.187529+0000", 2023-09-23T23:36:32.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3790000000000006e-06 2023-09-23T23:36:32.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.190139+0000", 2023-09-23T23:36:32.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026103749999999998 2023-09-23T23:36:32.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.678 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.190161+0000", 2023-09-23T23:36:32.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.145e-05 2023-09-23T23:36:32.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.678 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.201064+0000", 2023-09-23T23:36:32.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010903105 2023-09-23T23:36:32.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.678 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.201075+0000", 2023-09-23T23:36:32.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1572e-05 2023-09-23T23:36:32.679 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.679 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.679 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.679 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:115383 13.5 e594/570 13:a81359ee:::benchmark_data_smithi110_541928_object27603:head v 594'7363, mlcod=538'122)", 2023-09-23T23:36:32.679 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:36:30.331271+0000", 2023-09-23T23:36:32.679 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 2.2308451599999999, 2023-09-23T23:36:32.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01735855, 2023-09-23T23:36:32.679 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.679 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.679 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.331271+0000", 2023-09-23T23:36:32.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.331271+0000", 2023-09-23T23:36:32.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.331270+0000", 2023-09-23T23:36:32.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:36:32.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.331277+0000", 2023-09-23T23:36:32.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4170000000000003e-06 2023-09-23T23:36:32.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.331291+0000", 2023-09-23T23:36:32.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3645000000000001e-05 2023-09-23T23:36:32.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.682 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.331299+0000", 2023-09-23T23:36:32.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2700000000000004e-06 2023-09-23T23:36:32.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.682 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.336026+0000", 2023-09-23T23:36:32.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047264560000000004 2023-09-23T23:36:32.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.336057+0000", 2023-09-23T23:36:32.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1099000000000002e-05 2023-09-23T23:36:32.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.348596+0000", 2023-09-23T23:36:32.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012538873000000001 2023-09-23T23:36:32.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.684 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.348629+0000", 2023-09-23T23:36:32.684 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3661000000000002e-05 2023-09-23T23:36:32.684 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.684 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.684 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.684 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.684 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:115416 13.5 e594/570 13:a312f1d7:::benchmark_data_smithi110_541928_object27636:head v 594'7369, mlcod=538'122)", 2023-09-23T23:36:32.684 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:36:30.435447+0000", 2023-09-23T23:36:32.684 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 2.126669025, 2023-09-23T23:36:32.684 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022453852, 2023-09-23T23:36:32.684 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.684 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.685 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.685 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.685 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.435447+0000", 2023-09-23T23:36:32.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.685 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.685 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.685 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.435447+0000", 2023-09-23T23:36:32.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.685 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.685 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.686 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.686 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.435446+0000", 2023-09-23T23:36:32.686 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:36:32.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.686 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.686 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.435459+0000", 2023-09-23T23:36:32.686 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3087e-05 2023-09-23T23:36:32.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.686 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.686 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.435481+0000", 2023-09-23T23:36:32.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2535e-05 2023-09-23T23:36:32.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.435491+0000", 2023-09-23T23:36:32.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3549999999999994e-06 2023-09-23T23:36:32.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.435548+0000", 2023-09-23T23:36:32.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7173000000000003e-05 2023-09-23T23:36:32.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.688 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.688 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.435580+0000", 2023-09-23T23:36:32.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2150999999999998e-05 2023-09-23T23:36:32.688 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.688 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.688 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.688 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.457860+0000", 2023-09-23T23:36:32.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022280228999999999 2023-09-23T23:36:32.688 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.688 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.688 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.688 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.457901+0000", 2023-09-23T23:36:32.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0496e-05 2023-09-23T23:36:32.689 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.689 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.689 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.689 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:115417 13.5 e594/570 13:ab5cf083:::benchmark_data_smithi110_541928_object27637:head v 594'7370, mlcod=538'122)", 2023-09-23T23:36:32.689 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:36:30.437858+0000", 2023-09-23T23:36:32.689 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 2.124257896, 2023-09-23T23:36:32.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020060525999999999, 2023-09-23T23:36:32.689 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.689 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.689 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.690 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.690 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.437858+0000", 2023-09-23T23:36:32.690 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.690 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.690 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.690 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.437858+0000", 2023-09-23T23:36:32.690 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.690 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.690 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.691 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.437857+0000", 2023-09-23T23:36:32.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:36:32.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.691 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.691 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.437868+0000", 2023-09-23T23:36:32.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0516999999999999e-05 2023-09-23T23:36:32.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.691 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.691 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.437881+0000", 2023-09-23T23:36:32.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3079e-05 2023-09-23T23:36:32.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.692 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.692 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.437889+0000", 2023-09-23T23:36:32.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2800000000000003e-06 2023-09-23T23:36:32.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.692 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.692 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.441972+0000", 2023-09-23T23:36:32.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040828829999999998 2023-09-23T23:36:32.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.692 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.693 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.693 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.441993+0000", 2023-09-23T23:36:32.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1160999999999999e-05 2023-09-23T23:36:32.693 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.693 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.693 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.693 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.457904+0000", 2023-09-23T23:36:32.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015911213 2023-09-23T23:36:32.693 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.693 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.693 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.693 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.457918+0000", 2023-09-23T23:36:32.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4181e-05 2023-09-23T23:36:32.694 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.694 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.694 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.694 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.694 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:115421 13.5 e594/570 13:a928fa77:::benchmark_data_smithi110_541928_object27641:head v 594'7371, mlcod=538'122)", 2023-09-23T23:36:32.694 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:36:30.441887+0000", 2023-09-23T23:36:32.694 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 2.1202291440000001, 2023-09-23T23:36:32.694 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016051180000000002, 2023-09-23T23:36:32.694 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.694 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.695 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.695 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.695 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.695 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.441887+0000", 2023-09-23T23:36:32.695 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.695 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.695 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.695 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.695 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.441887+0000", 2023-09-23T23:36:32.695 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.695 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.695 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.695 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.696 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.441885+0000", 2023-09-23T23:36:32.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:32.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.696 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.696 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.696 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.441899+0000", 2023-09-23T23:36:32.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3509000000000001e-05 2023-09-23T23:36:32.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.696 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.696 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.696 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.441915+0000", 2023-09-23T23:36:32.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5644000000000001e-05 2023-09-23T23:36:32.697 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.697 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.697 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.441923+0000", 2023-09-23T23:36:32.697 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4009999999999997e-06 2023-09-23T23:36:32.697 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.697 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.697 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.447439+0000", 2023-09-23T23:36:32.697 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055160399999999998 2023-09-23T23:36:32.697 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.698 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.447474+0000", 2023-09-23T23:36:32.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5271999999999997e-05 2023-09-23T23:36:32.698 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.698 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.457920+0000", 2023-09-23T23:36:32.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010446165 2023-09-23T23:36:32.698 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.698 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.699 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:30.457938+0000", 2023-09-23T23:36:32.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7435000000000002e-05 2023-09-23T23:36:32.699 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.699 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.699 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.699 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:115814 13.5 e594/570 13:a8465c5e:::benchmark_data_smithi110_541928_object28034:head v 594'7392, mlcod=538'122)", 2023-09-23T23:36:32.699 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:36:32.221945+0000", 2023-09-23T23:36:32.699 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.34017032800000002, 2023-09-23T23:36:32.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014502744999999999, 2023-09-23T23:36:32.699 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:32.700 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:32.700 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:32.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:32.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.221945+0000", 2023-09-23T23:36:32.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:32.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.221945+0000", 2023-09-23T23:36:32.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:32.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:32.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.221945+0000", 2023-09-23T23:36:32.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:36:32.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:32.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.221955+0000", 2023-09-23T23:36:32.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0634e-05 2023-09-23T23:36:32.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:32.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.221971+0000", 2023-09-23T23:36:32.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5637999999999998e-05 2023-09-23T23:36:32.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:32.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.221982+0000", 2023-09-23T23:36:32.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0767e-05 2023-09-23T23:36:32.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:32.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.228187+0000", 2023-09-23T23:36:32.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0062051249999999997 2023-09-23T23:36:32.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:32.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.228217+0000", 2023-09-23T23:36:32.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0707000000000003e-05 2023-09-23T23:36:32.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:32.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.236360+0000", 2023-09-23T23:36:32.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081425860000000003 2023-09-23T23:36:32.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:32.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:32.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:32.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.236448+0000", 2023-09-23T23:36:32.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8179999999999999e-05 2023-09-23T23:36:32.704 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.704 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.704 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.704 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:32.704 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:32.704 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:36:32.707 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:32.708 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:32 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3994338032' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:32.709 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:36:32.711 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:36:32.737 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:32.793 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:32.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:32.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:32.899 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:32.930 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:32.989 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:32.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:32.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:33.095 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:33.123 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:33.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[221102]: pgmap v1654: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 263 op/s; 25296/181539 objects degraded (13.934%); 3694/181539 objects misplaced (2.035%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T23:36:33.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:36:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3994338032' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:36:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[221102]: osdmap e595: 8 total, 8 up, 7 in 2023-09-23T23:36:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[221102]: failed to encode map e595 with expected crc 2023-09-23T23:36:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[221102]: failed to encode map e595 with expected crc 2023-09-23T23:36:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[221102]: failed to encode map e595 with expected crc 2023-09-23T23:36:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[221102]: failed to encode map e595 with expected crc 2023-09-23T23:36:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[221102]: failed to encode map e595 with expected crc 2023-09-23T23:36:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[221102]: from='client.46880 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[212939]: pgmap v1654: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 263 op/s; 25296/181539 objects degraded (13.934%); 3694/181539 objects misplaced (2.035%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T23:36:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:36:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3994338032' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:36:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[212939]: osdmap e595: 8 total, 8 up, 7 in 2023-09-23T23:36:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[212939]: failed to encode map e595 with expected crc 2023-09-23T23:36:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[212939]: failed to encode map e595 with expected crc 2023-09-23T23:36:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[212939]: failed to encode map e595 with expected crc 2023-09-23T23:36:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[212939]: failed to encode map e595 with expected crc 2023-09-23T23:36:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[212939]: failed to encode map e595 with expected crc 2023-09-23T23:36:33.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:33 smithi110 ceph-mon[212939]: from='client.46880 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:33 smithi138 ceph-mon[193606]: pgmap v1654: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 263 op/s; 25296/181539 objects degraded (13.934%); 3694/181539 objects misplaced (2.035%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T23:36:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:33 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:36:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:33 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3994338032' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:36:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:33 smithi138 ceph-mon[193606]: osdmap e595: 8 total, 8 up, 7 in 2023-09-23T23:36:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:33 smithi138 ceph-mon[193606]: failed to encode map e595 with expected crc 2023-09-23T23:36:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:33 smithi138 ceph-mon[193606]: failed to encode map e595 with expected crc 2023-09-23T23:36:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:33 smithi138 ceph-mon[193606]: failed to encode map e595 with expected crc 2023-09-23T23:36:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:33 smithi138 ceph-mon[193606]: failed to encode map e595 with expected crc 2023-09-23T23:36:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:33 smithi138 ceph-mon[193606]: failed to encode map e595 with expected crc 2023-09-23T23:36:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:33 smithi138 ceph-mon[193606]: from='client.46880 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:34.202 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1655,"stamp":"2023-09-23T23:36:32.269297+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3964777717,"num_objects":60513,"num_object_clones":0,"num_object_copies":181539,"num_objects_missing_on_primary":2798,"num_objects_missing":2798,"num_objects_degraded":25296,"num_objects_misplaced":3694,"num_objects_unfound":0,"num_objects_dirty":60513,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":115144,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14608,"num_bytes_recovered":954659240,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95091,"ondisk_log_size":95091,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15922108,"kb_used_data":10249476,"kb_used_omap":29,"kb_used_meta":5672546,"kb_avail":640293956,"statfs":{"total":671965249536,"available":655661010944,"internally_reserved":0,"allocated":10495463424,"data_stored":10479261473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5808687827},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-152109056,"num_objects":-2321,"num_object_clones":0,"num_object_copies":-6963,"num_objects_missing_on_primary":-73,"num_objects_missing":-73,"num_objects_degraded":-245,"num_objects_misplaced":-72,"num_objects_unfound":0,"num_objects_dirty":-2321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2321,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":319,"num_bytes_recovered":20905984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.079986"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":790,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.181748+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:36:15.181748+0000","last_peered":"2023-09-23T23:36:15.181748+0000","last_clean":"2023-09-23T23:36:15.181748+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:36:15.181748+0000","last_undegraded":"2023-09-23T23:36:15.181748+0000","last_fullsized":"2023-09-23T23:36:15.181748+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":348,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.198284+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:36:15.198284+0000","last_peered":"2023-09-23T23:36:15.198284+0000","last_clean":"2023-09-23T23:36:15.198284+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:36:15.198284+0000","last_undegraded":"2023-09-23T23:36:15.198284+0000","last_fullsized":"2023-09-23T23:36:15.198284+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":576,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.178341+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:36:15.178341+0000","last_peered":"2023-09-23T23:36:15.178341+0000","last_clean":"2023-09-23T23:36:15.178341+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:36:15.178341+0000","last_undegraded":"2023-09-23T23:36:15.178341+0000","last_fullsized":"2023-09-23T23:36:15.178341+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":743,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.182656+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:36:15.182656+0000","last_peered":"2023-09-23T23:36:15.182656+0000","last_clean":"2023-09-23T23:36:15.182656+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:36:15.182656+0000","last_undegraded":"2023-09-23T23:36:15.182656+0000","last_fullsized":"2023-09-23T23:36:15.182656+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":312,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.169685+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:36:15.169685+0000","last_peered":"2023-09-23T23:36:15.169685+0000","last_clean":"2023-09-23T23:36:15.169685+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:36:15.169685+0000","last_undegraded":"2023-09-23T23:36:15.169685+0000","last_fullsized":"2023-09-23T23:36:15.169685+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":555,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:22.514522+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:36:22.514522+0000","last_peered":"2023-09-23T23:36:22.514522+0000","last_clean":"2023-09-23T23:36:22.514522+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:36:22.514522+0000","last_undegraded":"2023-09-23T23:36:22.514522+0000","last_fullsized":"2023-09-23T23:36:22.514522+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":755,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.172463+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:36:15.172463+0000","last_peered":"2023-09-23T23:36:15.172463+0000","last_clean":"2023-09-23T23:36:15.172463+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:36:15.172463+0000","last_undegraded":"2023-09-23T23:36:15.172463+0000","last_fullsized":"2023-09-23T23:36:15.172463+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":612,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.168959+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:36:15.168959+0000","last_peered":"2023-09-23T23:36:15.168959+0000","last_clean":"2023-09-23T23:36:15.168959+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:36:15.168959+0000","last_undegraded":"2023-09-23T23:36:15.168959+0000","last_fullsized":"2023-09-23T23:36:15.168959+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":108,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.191712+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:36:15.191712+0000","last_peered":"2023-09-23T23:36:15.191712+0000","last_clean":"2023-09-23T23:36:15.191712+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:36:15.191712+0000","last_undegraded":"2023-09-23T23:36:15.191712+0000","last_fullsized":"2023-09-23T23:36:15.191712+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":914,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.173305+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:36:15.173305+0000","last_peered":"2023-09-23T23:36:15.173305+0000","last_clean":"2023-09-23T23:36:15.173305+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:36:15.173305+0000","last_undegraded":"2023-09-23T23:36:15.173305+0000","last_fullsized":"2023-09-23T23:36:15.173305+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":246,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.183479+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:36:15.183479+0000","last_peered":"2023-09-23T23:36:15.183479+0000","last_clean":"2023-09-23T23:36:15.183479+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:36:15.183479+0000","last_undegraded":"2023-09-23T23:36:15.183479+0000","last_fullsized":"2023-09-23T23:36:15.183479+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":594,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.168885+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:36:15.168885+0000","last_peered":"2023-09-23T23:36:15.168885+0000","last_clean":"2023-09-23T23:36:15.168885+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:36:15.168885+0000","last_undegraded":"2023-09-23T23:36:15.168885+0000","last_fullsized":"2023-09-23T23:36:15.168885+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":887,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.197410+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:36:15.197410+0000","last_peered":"2023-09-23T23:36:15.197410+0000","last_clean":"2023-09-23T23:36:15.197410+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:36:15.197410+0000","last_undegraded":"2023-09-23T23:36:15.197410+0000","last_fullsized":"2023-09-23T23:36:15.197410+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":225,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.184220+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:36:15.184220+0000","last_peered":"2023-09-23T23:36:15.184220+0000","last_clean":"2023-09-23T23:36:15.184220+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:36:15.184220+0000","last_undegraded":"2023-09-23T23:36:15.184220+0000","last_fullsized":"2023-09-23T23:36:15.184220+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":555,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.169480+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:36:15.169480+0000","last_peered":"2023-09-23T23:36:15.169480+0000","last_clean":"2023-09-23T23:36:15.169480+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:36:15.169480+0000","last_undegraded":"2023-09-23T23:36:15.169480+0000","last_fullsized":"2023-09-23T23:36:15.169480+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1048,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.183354+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:36:15.183354+0000","last_peered":"2023-09-23T23:36:15.183354+0000","last_clean":"2023-09-23T23:36:15.183354+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:36:15.183354+0000","last_undegraded":"2023-09-23T23:36:15.183354+0000","last_fullsized":"2023-09-23T23:36:15.183354+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":266,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.172997+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:36:15.172997+0000","last_peered":"2023-09-23T23:36:15.172997+0000","last_clean":"2023-09-23T23:36:15.172997+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:36:15.172997+0000","last_undegraded":"2023-09-23T23:36:15.172997+0000","last_fullsized":"2023-09-23T23:36:15.172997+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":127,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.189734+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:36:15.189734+0000","last_peered":"2023-09-23T23:36:15.189734+0000","last_clean":"2023-09-23T23:36:15.189734+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:36:15.189734+0000","last_undegraded":"2023-09-23T23:36:15.189734+0000","last_fullsized":"2023-09-23T23:36:15.189734+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":445,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.186916+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:36:23.186916+0000","last_peered":"2023-09-23T23:36:23.186916+0000","last_clean":"2023-09-23T23:36:23.186916+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:36:23.186916+0000","last_undegraded":"2023-09-23T23:36:23.186916+0000","last_fullsized":"2023-09-23T23:36:23.186916+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":168,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.191052+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:36:15.191052+0000","last_peered":"2023-09-23T23:36:15.191052+0000","last_clean":"2023-09-23T23:36:15.191052+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:36:15.191052+0000","last_undegraded":"2023-09-23T23:36:15.191052+0000","last_fullsized":"2023-09-23T23:36:15.191052+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":535,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.190126+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:36:15.190126+0000","last_peered":"2023-09-23T23:36:15.190126+0000","last_clean":"2023-09-23T23:36:15.190126+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:36:15.190126+0000","last_undegraded":"2023-09-23T23:36:15.190126+0000","last_fullsized":"2023-09-23T23:36:15.190126+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":749,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.181692+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:36:15.181692+0000","last_peered":"2023-09-23T23:36:15.181692+0000","last_clean":"2023-09-23T23:36:15.181692+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:36:15.181692+0000","last_undegraded":"2023-09-23T23:36:15.181692+0000","last_fullsized":"2023-09-23T23:36:15.181692+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":140,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.197638+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:36:15.197638+0000","last_peered":"2023-09-23T23:36:15.197638+0000","last_clean":"2023-09-23T23:36:15.197638+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:36:15.197638+0000","last_undegraded":"2023-09-23T23:36:15.197638+0000","last_fullsized":"2023-09-23T23:36:15.197638+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":593,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.171234+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:36:15.171234+0000","last_peered":"2023-09-23T23:36:15.171234+0000","last_clean":"2023-09-23T23:36:15.171234+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:36:15.171234+0000","last_undegraded":"2023-09-23T23:36:15.171234+0000","last_fullsized":"2023-09-23T23:36:15.171234+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":879,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.189854+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:36:15.189854+0000","last_peered":"2023-09-23T23:36:15.189854+0000","last_clean":"2023-09-23T23:36:15.189854+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:36:15.189854+0000","last_undegraded":"2023-09-23T23:36:15.189854+0000","last_fullsized":"2023-09-23T23:36:15.189854+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":188,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.170020+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:36:15.170020+0000","last_peered":"2023-09-23T23:36:15.170020+0000","last_clean":"2023-09-23T23:36:15.170020+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:36:15.170020+0000","last_undegraded":"2023-09-23T23:36:15.170020+0000","last_fullsized":"2023-09-23T23:36:15.170020+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":315,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.178148+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:36:15.178148+0000","last_peered":"2023-09-23T23:36:15.178148+0000","last_clean":"2023-09-23T23:36:15.178148+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:36:15.178148+0000","last_undegraded":"2023-09-23T23:36:15.178148+0000","last_fullsized":"2023-09-23T23:36:15.178148+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":557,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.192221+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:36:23.192221+0000","last_peered":"2023-09-23T23:36:23.192221+0000","last_clean":"2023-09-23T23:36:23.192221+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:36:23.192221+0000","last_undegraded":"2023-09-23T23:36:23.192221+0000","last_fullsized":"2023-09-23T23:36:23.192221+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":462,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.190234+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:36:15.190234+0000","last_peered":"2023-09-23T23:36:15.190234+0000","last_clean":"2023-09-23T23:36:15.190234+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:36:15.190234+0000","last_undegraded":"2023-09-23T23:36:15.190234+0000","last_fullsized":"2023-09-23T23:36:15.190234+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":550,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.171638+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:36:15.171638+0000","last_peered":"2023-09-23T23:36:15.171638+0000","last_clean":"2023-09-23T23:36:15.171638+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:36:15.171638+0000","last_undegraded":"2023-09-23T23:36:15.171638+0000","last_fullsized":"2023-09-23T23:36:15.171638+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":957,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.182224+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:36:15.182224+0000","last_peered":"2023-09-23T23:36:15.182224+0000","last_clean":"2023-09-23T23:36:15.182224+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:36:15.182224+0000","last_undegraded":"2023-09-23T23:36:15.182224+0000","last_fullsized":"2023-09-23T23:36:15.182224+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":158,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.190145+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:36:15.190145+0000","last_peered":"2023-09-23T23:36:15.190145+0000","last_clean":"2023-09-23T23:36:15.190145+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:36:15.190145+0000","last_undegraded":"2023-09-23T23:36:15.190145+0000","last_fullsized":"2023-09-23T23:36:15.190145+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":235,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.171920+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:36:15.171920+0000","last_peered":"2023-09-23T23:36:15.171920+0000","last_clean":"2023-09-23T23:36:15.171920+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:36:15.171920+0000","last_undegraded":"2023-09-23T23:36:15.171920+0000","last_fullsized":"2023-09-23T23:36:15.171920+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"594'7078","reported_seq":7155,"reported_epoch":594,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:29.041732+0000","last_change":"2023-09-23T23:33:52.884769+0000","last_active":"2023-09-23T23:36:29.041732+0000","last_peered":"2023-09-23T23:36:29.041732+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:36:29.041732+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"549'1700","ondisk_log_start":"549'1700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5378,"log_dups_size":0,"ondisk_log_size":5378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247070720,"num_objects":3770,"num_object_clones":0,"num_object_copies":11310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7078,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":2744},{"shards":"2,6","objects":-628}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":927,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.196803+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:36:15.196803+0000","last_peered":"2023-09-23T23:36:15.196803+0000","last_clean":"2023-09-23T23:36:15.196803+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:36:15.196803+0000","last_undegraded":"2023-09-23T23:36:15.196803+0000","last_fullsized":"2023-09-23T23:36:15.196803+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":235,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.173048+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:36:15.173048+0000","last_peered":"2023-09-23T23:36:15.173048+0000","last_clean":"2023-09-23T23:36:15.173048+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:36:15.173048+0000","last_undegraded":"2023-09-23T23:36:15.173048+0000","last_fullsized":"2023-09-23T23:36:15.173048+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":225,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.181703+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:36:15.181703+0000","last_peered":"2023-09-23T23:36:15.181703+0000","last_clean":"2023-09-23T23:36:15.181703+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:36:15.181703+0000","last_undegraded":"2023-09-23T23:36:15.181703+0000","last_fullsized":"2023-09-23T23:36:15.181703+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"594'7222","reported_seq":7299,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:29.364908+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:36:29.364908+0000","last_peered":"2023-09-23T23:36:29.364908+0000","last_clean":"2023-09-23T23:36:29.364908+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:36:29.364908+0000","last_undegraded":"2023-09-23T23:36:29.364908+0000","last_fullsized":"2023-09-23T23:36:29.364908+0000","mapping_epoch":566,"log_start":"544'1000","ondisk_log_start":"544'1000","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6222,"log_dups_size":0,"ondisk_log_size":6222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249692160,"num_objects":3810,"num_object_clones":0,"num_object_copies":11430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7222,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1006,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.169895+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:36:15.169895+0000","last_peered":"2023-09-23T23:36:15.169895+0000","last_clean":"2023-09-23T23:36:15.169895+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:36:15.169895+0000","last_undegraded":"2023-09-23T23:36:15.169895+0000","last_fullsized":"2023-09-23T23:36:15.169895+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":225,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.172791+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:36:15.172791+0000","last_peered":"2023-09-23T23:36:15.172791+0000","last_clean":"2023-09-23T23:36:15.172791+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:36:15.172791+0000","last_undegraded":"2023-09-23T23:36:15.172791+0000","last_fullsized":"2023-09-23T23:36:15.172791+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":445,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:17.462877+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:36:17.462877+0000","last_peered":"2023-09-23T23:36:17.462877+0000","last_clean":"2023-09-23T23:36:17.462877+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:36:17.462877+0000","last_undegraded":"2023-09-23T23:36:17.462877+0000","last_fullsized":"2023-09-23T23:36:17.462877+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"594'7246","reported_seq":11804,"reported_epoch":594,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:36:27.515588+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:36:27.515588+0000","last_peered":"2023-09-23T23:36:27.515588+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:36:27.515588+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:36:27.515588+0000","mapping_epoch":571,"log_start":"549'1900","ondisk_log_start":"549'1900","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5346,"log_dups_size":0,"ondisk_log_size":5346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249167872,"num_objects":3802,"num_object_clones":0,"num_object_copies":11406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2906,"num_objects_misplaced":896,"num_objects_unfound":0,"num_objects_dirty":3802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7246,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2225,"num_bytes_recovered":145817600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":2784},{"shards":"3,5,7","objects":1018}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":904,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.191763+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:36:15.191763+0000","last_peered":"2023-09-23T23:36:15.191763+0000","last_clean":"2023-09-23T23:36:15.191763+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:36:15.191763+0000","last_undegraded":"2023-09-23T23:36:15.191763+0000","last_fullsized":"2023-09-23T23:36:15.191763+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":332,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.182923+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:36:15.182923+0000","last_peered":"2023-09-23T23:36:15.182923+0000","last_clean":"2023-09-23T23:36:15.182923+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:36:15.182923+0000","last_undegraded":"2023-09-23T23:36:15.182923+0000","last_fullsized":"2023-09-23T23:36:15.182923+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":108,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.189928+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:36:15.189928+0000","last_peered":"2023-09-23T23:36:15.189928+0000","last_clean":"2023-09-23T23:36:15.189928+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:36:15.189928+0000","last_undegraded":"2023-09-23T23:36:15.189928+0000","last_fullsized":"2023-09-23T23:36:15.189928+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"594'7194","reported_seq":9257,"reported_epoch":594,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:27.374202+0000","last_change":"2023-09-23T23:34:35.750428+0000","last_active":"2023-09-23T23:36:27.374202+0000","last_peered":"2023-09-23T23:36:27.374202+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:36:27.374202+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"549'1900","ondisk_log_start":"549'1900","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5294,"log_dups_size":0,"ondisk_log_size":5294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255328256,"num_objects":3896,"num_object_clones":0,"num_object_copies":11688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7194,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":991,"num_bytes_recovered":64946176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":2886},{"shards":"3,5","objects":352}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":500,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.171579+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:36:15.171579+0000","last_peered":"2023-09-23T23:36:15.171579+0000","last_clean":"2023-09-23T23:36:15.171579+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:36:15.171579+0000","last_undegraded":"2023-09-23T23:36:15.171579+0000","last_fullsized":"2023-09-23T23:36:15.171579+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":434,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.171117+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:36:15.171117+0000","last_peered":"2023-09-23T23:36:15.171117+0000","last_clean":"2023-09-23T23:36:15.171117+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:36:15.171117+0000","last_undegraded":"2023-09-23T23:36:15.171117+0000","last_fullsized":"2023-09-23T23:36:15.171117+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":591,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.204502+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:36:15.204502+0000","last_peered":"2023-09-23T23:36:15.204502+0000","last_clean":"2023-09-23T23:36:15.204502+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:36:15.204502+0000","last_undegraded":"2023-09-23T23:36:15.204502+0000","last_fullsized":"2023-09-23T23:36:15.204502+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"594'7349","reported_seq":8671,"reported_epoch":594,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:29.361952+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:36:29.361952+0000","last_peered":"2023-09-23T23:36:29.361952+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:36:29.361952+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"548'1600","ondisk_log_start":"548'1600","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5749,"log_dups_size":0,"ondisk_log_size":5749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249888768,"num_objects":3813,"num_object_clones":0,"num_object_copies":11439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7349,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":2767},{"shards":"3,5","objects":-1647},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":721,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.169404+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:36:15.169404+0000","last_peered":"2023-09-23T23:36:15.169404+0000","last_clean":"2023-09-23T23:36:15.169404+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:36:15.169404+0000","last_undegraded":"2023-09-23T23:36:15.169404+0000","last_fullsized":"2023-09-23T23:36:15.169404+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":604,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.173066+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:36:15.173066+0000","last_peered":"2023-09-23T23:36:15.173066+0000","last_clean":"2023-09-23T23:36:15.173066+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:36:15.173066+0000","last_undegraded":"2023-09-23T23:36:15.173066+0000","last_fullsized":"2023-09-23T23:36:15.173066+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":515,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.171078+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:36:15.171078+0000","last_peered":"2023-09-23T23:36:15.171078+0000","last_clean":"2023-09-23T23:36:15.171078+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:36:15.171078+0000","last_undegraded":"2023-09-23T23:36:15.171078+0000","last_fullsized":"2023-09-23T23:36:15.171078+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"594'7320","reported_seq":7388,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:29.003034+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:36:29.003034+0000","last_peered":"2023-09-23T23:36:29.003034+0000","last_clean":"2023-09-23T23:36:29.003034+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:36:29.003034+0000","last_undegraded":"2023-09-23T23:36:29.003034+0000","last_fullsized":"2023-09-23T23:36:29.003034+0000","mapping_epoch":534,"log_start":"549'1900","ondisk_log_start":"549'1900","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5420,"log_dups_size":0,"ondisk_log_size":5420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246415360,"num_objects":3760,"num_object_clones":0,"num_object_copies":11280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7320,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":877,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.198293+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:36:15.198293+0000","last_peered":"2023-09-23T23:36:15.198293+0000","last_clean":"2023-09-23T23:36:15.198293+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:36:15.198293+0000","last_undegraded":"2023-09-23T23:36:15.198293+0000","last_fullsized":"2023-09-23T23:36:15.198293+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":118,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.189865+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:36:15.189865+0000","last_peered":"2023-09-23T23:36:15.189865+0000","last_clean":"2023-09-23T23:36:15.189865+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:36:15.189865+0000","last_undegraded":"2023-09-23T23:36:15.189865+0000","last_fullsized":"2023-09-23T23:36:15.189865+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":460,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.196970+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:36:15.196970+0000","last_peered":"2023-09-23T23:36:15.196970+0000","last_clean":"2023-09-23T23:36:15.196970+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:36:15.196970+0000","last_undegraded":"2023-09-23T23:36:15.196970+0000","last_fullsized":"2023-09-23T23:36:15.196970+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"594'7325","reported_seq":10959,"reported_epoch":594,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:36:29.099522+0000","last_change":"2023-09-23T23:34:35.785270+0000","last_active":"2023-09-23T23:36:29.099522+0000","last_peered":"2023-09-23T23:36:29.099522+0000","last_clean":"2023-09-23T23:33:50.504296+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:36:29.099522+0000","last_undegraded":"2023-09-23T23:34:35.780981+0000","last_fullsized":"2023-09-23T23:36:29.099522+0000","mapping_epoch":567,"log_start":"549'1900","ondisk_log_start":"549'1900","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5425,"log_dups_size":0,"ondisk_log_size":5425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247660544,"num_objects":3779,"num_object_clones":0,"num_object_copies":11337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2774,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7325,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1773,"num_bytes_recovered":116195328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":2774},{"shards":"5,6,7","objects":1005}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":602,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.182574+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:36:15.182574+0000","last_peered":"2023-09-23T23:36:15.182574+0000","last_clean":"2023-09-23T23:36:15.182574+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:36:15.182574+0000","last_undegraded":"2023-09-23T23:36:15.182574+0000","last_fullsized":"2023-09-23T23:36:15.182574+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":457,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.196624+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:36:15.196624+0000","last_peered":"2023-09-23T23:36:15.196624+0000","last_clean":"2023-09-23T23:36:15.196624+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:36:15.196624+0000","last_undegraded":"2023-09-23T23:36:15.196624+0000","last_fullsized":"2023-09-23T23:36:15.196624+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":545,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.190295+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:36:15.190295+0000","last_peered":"2023-09-23T23:36:15.190295+0000","last_clean":"2023-09-23T23:36:15.190295+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:36:15.190295+0000","last_undegraded":"2023-09-23T23:36:15.190295+0000","last_fullsized":"2023-09-23T23:36:15.190295+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"594'7239","reported_seq":16270,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:29.356569+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:36:29.356569+0000","last_peered":"2023-09-23T23:36:29.356569+0000","last_clean":"2023-09-23T23:36:29.356569+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:36:29.356569+0000","last_undegraded":"2023-09-23T23:36:29.356569+0000","last_fullsized":"2023-09-23T23:36:29.356569+0000","mapping_epoch":579,"log_start":"548'1500","ondisk_log_start":"548'1500","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5739,"log_dups_size":0,"ondisk_log_size":5739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247529472,"num_objects":3777,"num_object_clones":0,"num_object_copies":11331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7239,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":935,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.198154+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:36:15.198154+0000","last_peered":"2023-09-23T23:36:15.198154+0000","last_clean":"2023-09-23T23:36:15.198154+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:36:15.198154+0000","last_undegraded":"2023-09-23T23:36:15.198154+0000","last_fullsized":"2023-09-23T23:36:15.198154+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":235,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.168755+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:36:15.168755+0000","last_peered":"2023-09-23T23:36:15.168755+0000","last_clean":"2023-09-23T23:36:15.168755+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:36:15.168755+0000","last_undegraded":"2023-09-23T23:36:15.168755+0000","last_fullsized":"2023-09-23T23:36:15.168755+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":524,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:22.514645+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:36:22.514645+0000","last_peered":"2023-09-23T23:36:22.514645+0000","last_clean":"2023-09-23T23:36:22.514645+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:36:22.514645+0000","last_undegraded":"2023-09-23T23:36:22.514645+0000","last_fullsized":"2023-09-23T23:36:22.514645+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"594'6948","reported_seq":7018,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:27.490994+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:36:27.490994+0000","last_peered":"2023-09-23T23:36:27.490994+0000","last_clean":"2023-09-23T23:36:27.490994+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:36:27.490994+0000","last_undegraded":"2023-09-23T23:36:27.490994+0000","last_fullsized":"2023-09-23T23:36:27.490994+0000","mapping_epoch":534,"log_start":"548'1600","ondisk_log_start":"548'1600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5348,"log_dups_size":0,"ondisk_log_size":5348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247726080,"num_objects":3780,"num_object_clones":0,"num_object_copies":11340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6948,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":915,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.190682+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:36:15.190682+0000","last_peered":"2023-09-23T23:36:15.190682+0000","last_clean":"2023-09-23T23:36:15.190682+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:36:15.190682+0000","last_undegraded":"2023-09-23T23:36:15.190682+0000","last_fullsized":"2023-09-23T23:36:15.190682+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":303,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.205425+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:36:15.205425+0000","last_peered":"2023-09-23T23:36:15.205425+0000","last_clean":"2023-09-23T23:36:15.205425+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:36:15.205425+0000","last_undegraded":"2023-09-23T23:36:15.205425+0000","last_fullsized":"2023-09-23T23:36:15.205425+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":609,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.183063+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:36:15.183063+0000","last_peered":"2023-09-23T23:36:15.183063+0000","last_clean":"2023-09-23T23:36:15.183063+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:36:15.183063+0000","last_undegraded":"2023-09-23T23:36:15.183063+0000","last_fullsized":"2023-09-23T23:36:15.183063+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":190,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.181960+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:36:15.181960+0000","last_peered":"2023-09-23T23:36:15.181960+0000","last_clean":"2023-09-23T23:36:15.181960+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:36:15.181960+0000","last_undegraded":"2023-09-23T23:36:15.181960+0000","last_fullsized":"2023-09-23T23:36:15.181960+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"594'7292","reported_seq":10806,"reported_epoch":594,"state":"active+recovering","last_fresh":"2023-09-23T23:36:27.491685+0000","last_change":"2023-09-23T23:34:35.815566+0000","last_active":"2023-09-23T23:36:27.491685+0000","last_peered":"2023-09-23T23:36:27.491685+0000","last_clean":"2023-09-23T23:33:50.519308+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:36:27.491685+0000","last_undegraded":"2023-09-23T23:36:27.491685+0000","last_fullsized":"2023-09-23T23:36:27.491685+0000","mapping_epoch":566,"log_start":"549'2022","ondisk_log_start":"549'2022","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5270,"log_dups_size":0,"ondisk_log_size":5270,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253952024,"num_objects":3876,"num_object_clones":0,"num_object_copies":11628,"num_objects_missing_on_primary":2798,"num_objects_missing":2798,"num_objects_degraded":0,"num_objects_misplaced":2798,"num_objects_unfound":0,"num_objects_dirty":3876,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7292,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1707,"num_bytes_recovered":111869952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,3,4,7","objects":10},{"shards":"1,3,7","objects":1068},{"shards":"1,4,7","objects":2798}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":677,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.204842+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:36:15.204842+0000","last_peered":"2023-09-23T23:36:15.204842+0000","last_clean":"2023-09-23T23:36:15.204842+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:36:15.204842+0000","last_undegraded":"2023-09-23T23:36:15.204842+0000","last_fullsized":"2023-09-23T23:36:15.204842+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":242,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.189798+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:36:15.189798+0000","last_peered":"2023-09-23T23:36:15.189798+0000","last_clean":"2023-09-23T23:36:15.189798+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:36:15.189798+0000","last_undegraded":"2023-09-23T23:36:15.189798+0000","last_fullsized":"2023-09-23T23:36:15.189798+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":526,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.190208+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:36:15.190208+0000","last_peered":"2023-09-23T23:36:15.190208+0000","last_clean":"2023-09-23T23:36:15.190208+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:36:15.190208+0000","last_undegraded":"2023-09-23T23:36:15.190208+0000","last_fullsized":"2023-09-23T23:36:15.190208+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":227,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.197417+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:36:15.197417+0000","last_peered":"2023-09-23T23:36:15.197417+0000","last_clean":"2023-09-23T23:36:15.197417+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:36:15.197417+0000","last_undegraded":"2023-09-23T23:36:15.197417+0000","last_fullsized":"2023-09-23T23:36:15.197417+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"594'7027","reported_seq":9074,"reported_epoch":594,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:29.334867+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:36:29.334867+0000","last_peered":"2023-09-23T23:36:29.334867+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:36:29.334867+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"548'1300","ondisk_log_start":"548'1300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5727,"log_dups_size":0,"ondisk_log_size":5727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245432320,"num_objects":3745,"num_object_clones":0,"num_object_copies":11235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7027,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":2734},{"shards":"5,7","objects":353}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":895,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.198064+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:36:15.198064+0000","last_peered":"2023-09-23T23:36:15.198064+0000","last_clean":"2023-09-23T23:36:15.198064+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:36:15.198064+0000","last_undegraded":"2023-09-23T23:36:15.198064+0000","last_fullsized":"2023-09-23T23:36:15.198064+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":332,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.196531+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:36:15.196531+0000","last_peered":"2023-09-23T23:36:15.196531+0000","last_clean":"2023-09-23T23:36:15.196531+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:36:15.196531+0000","last_undegraded":"2023-09-23T23:36:15.196531+0000","last_fullsized":"2023-09-23T23:36:15.196531+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":242,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.198316+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:36:15.198316+0000","last_peered":"2023-09-23T23:36:15.198316+0000","last_clean":"2023-09-23T23:36:15.198316+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:36:15.198316+0000","last_undegraded":"2023-09-23T23:36:15.198316+0000","last_fullsized":"2023-09-23T23:36:15.198316+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":206,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.204784+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:36:15.204784+0000","last_peered":"2023-09-23T23:36:15.204784+0000","last_clean":"2023-09-23T23:36:15.204784+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:36:15.204784+0000","last_undegraded":"2023-09-23T23:36:15.204784+0000","last_fullsized":"2023-09-23T23:36:15.204784+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"594'7172","reported_seq":7250,"reported_epoch":594,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:36:29.350787+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:36:29.350787+0000","last_peered":"2023-09-23T23:36:29.350787+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:36:29.350787+0000","last_undegraded":"2023-09-23T23:36:29.350787+0000","last_fullsized":"2023-09-23T23:36:29.350787+0000","mapping_epoch":562,"log_start":"548'1500","ondisk_log_start":"548'1500","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5672,"log_dups_size":0,"ondisk_log_size":5672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244449280,"num_objects":3730,"num_object_clones":0,"num_object_copies":11190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7172,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":3730}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":609,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.197701+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:36:15.197701+0000","last_peered":"2023-09-23T23:36:15.197701+0000","last_clean":"2023-09-23T23:36:15.197701+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:36:15.197701+0000","last_undegraded":"2023-09-23T23:36:15.197701+0000","last_fullsized":"2023-09-23T23:36:15.197701+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":880,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.189928+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:36:15.189928+0000","last_peered":"2023-09-23T23:36:15.189928+0000","last_clean":"2023-09-23T23:36:15.189928+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:36:15.189928+0000","last_undegraded":"2023-09-23T23:36:15.189928+0000","last_fullsized":"2023-09-23T23:36:15.189928+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":556,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.169886+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:36:15.169886+0000","last_peered":"2023-09-23T23:36:15.169886+0000","last_clean":"2023-09-23T23:36:15.169886+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:36:15.169886+0000","last_undegraded":"2023-09-23T23:36:15.169886+0000","last_fullsized":"2023-09-23T23:36:15.169886+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":118,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.189654+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:36:15.189654+0000","last_peered":"2023-09-23T23:36:15.189654+0000","last_clean":"2023-09-23T23:36:15.189654+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:36:15.189654+0000","last_undegraded":"2023-09-23T23:36:15.189654+0000","last_fullsized":"2023-09-23T23:36:15.189654+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"594'7056","reported_seq":7124,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:28.992341+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:36:28.992341+0000","last_peered":"2023-09-23T23:36:28.992341+0000","last_clean":"2023-09-23T23:36:28.992341+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:36:28.992341+0000","last_undegraded":"2023-09-23T23:36:28.992341+0000","last_fullsized":"2023-09-23T23:36:28.992341+0000","mapping_epoch":534,"log_start":"548'1600","ondisk_log_start":"548'1600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5456,"log_dups_size":0,"ondisk_log_size":5456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241172480,"num_objects":3680,"num_object_clones":0,"num_object_copies":11040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3680,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7056,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":706,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.190062+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:36:15.190062+0000","last_peered":"2023-09-23T23:36:15.190062+0000","last_clean":"2023-09-23T23:36:15.190062+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:36:15.190062+0000","last_undegraded":"2023-09-23T23:36:15.190062+0000","last_fullsized":"2023-09-23T23:36:15.190062+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":521,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.172208+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:36:15.172208+0000","last_peered":"2023-09-23T23:36:15.172208+0000","last_clean":"2023-09-23T23:36:15.172208+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:36:15.172208+0000","last_undegraded":"2023-09-23T23:36:15.172208+0000","last_fullsized":"2023-09-23T23:36:15.172208+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":435,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.169079+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:36:15.169079+0000","last_peered":"2023-09-23T23:36:15.169079+0000","last_clean":"2023-09-23T23:36:15.169079+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:36:15.169079+0000","last_undegraded":"2023-09-23T23:36:15.169079+0000","last_fullsized":"2023-09-23T23:36:15.169079+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":34,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.204087+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:36:15.204087+0000","last_peered":"2023-09-23T23:36:15.204087+0000","last_clean":"2023-09-23T23:36:15.204087+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:36:15.204087+0000","last_undegraded":"2023-09-23T23:36:15.204087+0000","last_fullsized":"2023-09-23T23:36:15.204087+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"594'7261","reported_seq":10928,"reported_epoch":594,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:29.693818+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:36:29.693818+0000","last_peered":"2023-09-23T23:36:29.693818+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:36:29.693818+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"544'1100","ondisk_log_start":"544'1100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6161,"log_dups_size":0,"ondisk_log_size":6161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244908032,"num_objects":3737,"num_object_clones":0,"num_object_copies":11211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7261,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":1033},{"shards":"1,2,4","objects":2704}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":589,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.190525+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:36:15.190525+0000","last_peered":"2023-09-23T23:36:15.190525+0000","last_clean":"2023-09-23T23:36:15.190525+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:36:15.190525+0000","last_undegraded":"2023-09-23T23:36:15.190525+0000","last_fullsized":"2023-09-23T23:36:15.190525+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":34,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.204893+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:36:15.204893+0000","last_peered":"2023-09-23T23:36:15.204893+0000","last_clean":"2023-09-23T23:36:15.204893+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:36:15.204893+0000","last_undegraded":"2023-09-23T23:36:15.204893+0000","last_fullsized":"2023-09-23T23:36:15.204893+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":134,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.197489+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:36:15.197489+0000","last_peered":"2023-09-23T23:36:15.197489+0000","last_clean":"2023-09-23T23:36:15.197489+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:36:15.197489+0000","last_undegraded":"2023-09-23T23:36:15.197489+0000","last_fullsized":"2023-09-23T23:36:15.197489+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":327,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.182776+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:36:15.182776+0000","last_peered":"2023-09-23T23:36:15.182776+0000","last_clean":"2023-09-23T23:36:15.182776+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:36:15.182776+0000","last_undegraded":"2023-09-23T23:36:15.182776+0000","last_fullsized":"2023-09-23T23:36:15.182776+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"594'7201","reported_seq":7269,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:29.354460+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:36:29.354460+0000","last_peered":"2023-09-23T23:36:29.354460+0000","last_clean":"2023-09-23T23:36:29.354460+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:36:29.354460+0000","last_undegraded":"2023-09-23T23:36:29.354460+0000","last_fullsized":"2023-09-23T23:36:29.354460+0000","mapping_epoch":534,"log_start":"548'1500","ondisk_log_start":"548'1500","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246611968,"num_objects":3763,"num_object_clones":0,"num_object_copies":11289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7201,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":775,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.197041+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:36:15.197041+0000","last_peered":"2023-09-23T23:36:15.197041+0000","last_clean":"2023-09-23T23:36:15.197041+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:36:15.197041+0000","last_undegraded":"2023-09-23T23:36:15.197041+0000","last_fullsized":"2023-09-23T23:36:15.197041+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":591,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.171998+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:36:15.171998+0000","last_peered":"2023-09-23T23:36:15.171998+0000","last_clean":"2023-09-23T23:36:15.171998+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:36:15.171998+0000","last_undegraded":"2023-09-23T23:36:15.171998+0000","last_fullsized":"2023-09-23T23:36:15.171998+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":198,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.190358+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:36:15.190358+0000","last_peered":"2023-09-23T23:36:15.190358+0000","last_clean":"2023-09-23T23:36:15.190358+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:36:15.190358+0000","last_undegraded":"2023-09-23T23:36:15.190358+0000","last_fullsized":"2023-09-23T23:36:15.190358+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":145,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.198187+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:36:15.198187+0000","last_peered":"2023-09-23T23:36:15.198187+0000","last_clean":"2023-09-23T23:36:15.198187+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:36:15.198187+0000","last_undegraded":"2023-09-23T23:36:15.198187+0000","last_fullsized":"2023-09-23T23:36:15.198187+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"594'7137","reported_seq":7207,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:29.562877+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:36:29.562877+0000","last_peered":"2023-09-23T23:36:29.562877+0000","last_clean":"2023-09-23T23:36:29.562877+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:36:29.562877+0000","last_undegraded":"2023-09-23T23:36:29.562877+0000","last_fullsized":"2023-09-23T23:36:29.562877+0000","mapping_epoch":534,"log_start":"544'1000","ondisk_log_start":"544'1000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6137,"log_dups_size":0,"ondisk_log_size":6137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247136256,"num_objects":3771,"num_object_clones":0,"num_object_copies":11313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7137,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":981,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.171313+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:36:15.171313+0000","last_peered":"2023-09-23T23:36:15.171313+0000","last_clean":"2023-09-23T23:36:15.171313+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:36:15.171313+0000","last_undegraded":"2023-09-23T23:36:15.171313+0000","last_fullsized":"2023-09-23T23:36:15.171313+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":108,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.190798+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:36:15.190798+0000","last_peered":"2023-09-23T23:36:15.190798+0000","last_clean":"2023-09-23T23:36:15.190798+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:36:15.190798+0000","last_undegraded":"2023-09-23T23:36:15.190798+0000","last_fullsized":"2023-09-23T23:36:15.190798+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":38,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.204166+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:36:15.204166+0000","last_peered":"2023-09-23T23:36:15.204166+0000","last_clean":"2023-09-23T23:36:15.204166+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:36:15.204166+0000","last_undegraded":"2023-09-23T23:36:15.204166+0000","last_fullsized":"2023-09-23T23:36:15.204166+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":168,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.181708+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:36:15.181708+0000","last_peered":"2023-09-23T23:36:15.181708+0000","last_clean":"2023-09-23T23:36:15.181708+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:36:15.181708+0000","last_undegraded":"2023-09-23T23:36:15.181708+0000","last_fullsized":"2023-09-23T23:36:15.181708+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1030,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.169320+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:36:15.169320+0000","last_peered":"2023-09-23T23:36:15.169320+0000","last_clean":"2023-09-23T23:36:15.169320+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:36:15.169320+0000","last_undegraded":"2023-09-23T23:36:15.169320+0000","last_fullsized":"2023-09-23T23:36:15.169320+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":34,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.204574+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:36:15.204574+0000","last_peered":"2023-09-23T23:36:15.204574+0000","last_clean":"2023-09-23T23:36:15.204574+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:36:15.204574+0000","last_undegraded":"2023-09-23T23:36:15.204574+0000","last_fullsized":"2023-09-23T23:36:15.204574+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":349,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.197512+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:36:15.197512+0000","last_peered":"2023-09-23T23:36:15.197512+0000","last_clean":"2023-09-23T23:36:15.197512+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:36:15.197512+0000","last_undegraded":"2023-09-23T23:36:15.197512+0000","last_fullsized":"2023-09-23T23:36:15.197512+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":742,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.171784+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:36:15.171784+0000","last_peered":"2023-09-23T23:36:15.171784+0000","last_clean":"2023-09-23T23:36:15.171784+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:36:15.171784+0000","last_undegraded":"2023-09-23T23:36:15.171784+0000","last_fullsized":"2023-09-23T23:36:15.171784+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":96,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.190904+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:36:15.190904+0000","last_peered":"2023-09-23T23:36:15.190904+0000","last_clean":"2023-09-23T23:36:15.190904+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:36:15.190904+0000","last_undegraded":"2023-09-23T23:36:15.190904+0000","last_fullsized":"2023-09-23T23:36:15.190904+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":442,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:22.484633+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:36:22.484633+0000","last_peered":"2023-09-23T23:36:22.484633+0000","last_clean":"2023-09-23T23:36:22.484633+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:36:22.484633+0000","last_undegraded":"2023-09-23T23:36:22.484633+0000","last_fullsized":"2023-09-23T23:36:22.484633+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":292,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.189520+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:36:15.189520+0000","last_peered":"2023-09-23T23:36:15.189520+0000","last_clean":"2023-09-23T23:36:15.189520+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:36:15.189520+0000","last_undegraded":"2023-09-23T23:36:15.189520+0000","last_fullsized":"2023-09-23T23:36:15.189520+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":281,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:22.484566+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:36:22.484566+0000","last_peered":"2023-09-23T23:36:22.484566+0000","last_clean":"2023-09-23T23:36:22.484566+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:36:22.484566+0000","last_undegraded":"2023-09-23T23:36:22.484566+0000","last_fullsized":"2023-09-23T23:36:22.484566+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":806,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.191123+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:36:15.191123+0000","last_peered":"2023-09-23T23:36:15.191123+0000","last_clean":"2023-09-23T23:36:15.191123+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:36:15.191123+0000","last_undegraded":"2023-09-23T23:36:15.191123+0000","last_fullsized":"2023-09-23T23:36:15.191123+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":624,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.171714+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:36:15.171714+0000","last_peered":"2023-09-23T23:36:15.171714+0000","last_clean":"2023-09-23T23:36:15.171714+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:36:15.171714+0000","last_undegraded":"2023-09-23T23:36:15.171714+0000","last_fullsized":"2023-09-23T23:36:15.171714+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":608,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.169607+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:36:15.169607+0000","last_peered":"2023-09-23T23:36:15.169607+0000","last_clean":"2023-09-23T23:36:15.169607+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:36:15.169607+0000","last_undegraded":"2023-09-23T23:36:15.169607+0000","last_fullsized":"2023-09-23T23:36:15.169607+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":766,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.172907+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:36:15.172907+0000","last_peered":"2023-09-23T23:36:15.172907+0000","last_clean":"2023-09-23T23:36:15.172907+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:36:15.172907+0000","last_undegraded":"2023-09-23T23:36:15.172907+0000","last_fullsized":"2023-09-23T23:36:15.172907+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":437,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.197550+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:36:15.197550+0000","last_peered":"2023-09-23T23:36:15.197550+0000","last_clean":"2023-09-23T23:36:15.197550+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:36:15.197550+0000","last_undegraded":"2023-09-23T23:36:15.197550+0000","last_fullsized":"2023-09-23T23:36:15.197550+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":661,"reported_epoch":593,"state":"active+clean","last_fresh":"2023-09-23T23:36:15.189785+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:36:15.189785+0000","last_peered":"2023-09-23T23:36:15.189785+0000","last_clean":"2023-09-23T23:36:15.189785+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:36:15.189785+0000","last_undegraded":"2023-09-23T23:36:15.189785+0000","last_fullsized":"2023-09-23T23:36:15.189785+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":3964141592,"num_objects":60489,"num_object_clones":0,"num_object_copies":181467,"num_objects_missing_on_primary":2798,"num_objects_missing":2798,"num_objects_degraded":25296,"num_objects_misplaced":3694,"num_objects_unfound":0,"num_objects_dirty":60489,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":115067,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14553,"num_bytes_recovered":953548800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12896579584,"data_stored":12896567368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90045,"ondisk_log_size":90045,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782684,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1787432,"kb_used_data":1418336,"kb_used_omap":8,"kb_used_meta":369079,"kb_avail":91957720,"statfs":{"total":95995035648,"available":94164705280,"internally_reserved":0,"allocated":1452376064,"data_stored":1450725540,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":377936965},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":1.5569999999999999,"15min":2.1800000000000002},"min":{"1min":0.23100000000000001,"5min":0.224,"15min":0.20899999999999999},"max":{"1min":0.61699999999999999,"5min":36.956000000000003,"15min":85.721000000000004},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.47399999999999998,"5min":1.522,"15min":2.2250000000000001},"min":{"1min":0.29199999999999998,"5min":0.27900000000000003,"15min":0.19800000000000001},"max":{"1min":1.4419999999999999,"5min":37.247999999999998,"15min":85.965000000000003},"last":0.26500000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":3.8900000000000001,"15min":3.169},"min":{"1min":0.26600000000000001,"5min":0.248,"15min":0.22900000000000001},"max":{"1min":3.3500000000000001,"5min":129.11199999999999,"15min":198.58000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.65600000000000003,"5min":3.665,"15min":3.206},"min":{"1min":0.29699999999999999,"5min":0.247,"15min":0.246},"max":{"1min":3.7290000000000001,"5min":129.02600000000001,"15min":198.71100000000001},"last":0.42299999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":2.871,"15min":3.7010000000000001},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.452,"5min":78.528999999999996,"15min":203.58000000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.54200000000000004,"5min":1.952,"15min":3.0649999999999999},"min":{"1min":0.30399999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.2529999999999999,"5min":27.574000000000002,"15min":198.69},"last":0.48699999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":3.3290000000000002,"15min":3.5950000000000002},"min":{"1min":0.31900000000000001,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":1.591,"5min":125.511,"15min":279.26499999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.67500000000000004,"5min":5.3090000000000002,"15min":4.1210000000000004},"min":{"1min":0.374,"5min":0.32000000000000001,"15min":0.29599999999999999},"max":{"1min":3.6240000000000001,"5min":150.608,"15min":279.29899999999998},"last":0.44400000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":1.694,"15min":1.0669999999999999},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":0.68100000000000005,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.363,"15min":1.0029999999999999},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.23899999999999999},"max":{"1min":1.6080000000000001,"5min":31.68,"15min":31.68},"last":0.32500000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":3.637,"5min":1.657,"15min":1.899},"min":{"1min":0.31900000000000001,"5min":0.28699999999999998,"15min":0.26500000000000001},"max":{"1min":60.463999999999999,"5min":60.463999999999999,"15min":210.83600000000001},"last":0.38100000000000001},{"interface":"front","average":{"1min":3.5990000000000002,"5min":1.6160000000000001,"15min":1.7649999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28199999999999997},"max":{"1min":63.006,"5min":63.006,"15min":207.04599999999999},"last":0.55000000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":1.879,"15min":1.3480000000000001},"min":{"1min":0.35199999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.4419999999999999,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.59299999999999997,"5min":2.2330000000000001,"15min":1.581},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":2.3180000000000001,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.53500000000000003}]}]},{"osd":6,"up_from":46,"seq":197568496161,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2046872,"kb_used_data":1340432,"kb_used_omap":0,"kb_used_meta":706432,"kb_avail":91698280,"statfs":{"total":95995035648,"available":93899038720,"internally_reserved":0,"allocated":1372602368,"data_stored":1370967837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":723386368},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.93500000000000005,"15min":4.6799999999999997},"min":{"1min":0.27700000000000002,"5min":0.251,"15min":0.20399999999999999},"max":{"1min":2.8450000000000002,"5min":8.8119999999999994,"15min":135.31999999999999},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.73399999999999999,"15min":4.601},"min":{"1min":0.24399999999999999,"5min":0.23999999999999999,"15min":0.22900000000000001},"max":{"1min":1.8859999999999999,"5min":3.8239999999999998,"15min":135.34299999999999},"last":0.38100000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":1.7509999999999999,"15min":2.3759999999999999},"min":{"1min":0.253,"5min":0.251,"15min":0.245},"max":{"1min":1.944,"5min":65.891000000000005,"15min":89.605000000000004},"last":0.308},{"interface":"front","average":{"1min":0.64100000000000001,"5min":1.4390000000000001,"15min":2.4510000000000001},"min":{"1min":0.33400000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":2.1269999999999998,"5min":66.087000000000003,"15min":91.025999999999996},"last":0.40600000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":4.9459999999999997,"15min":4.5620000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.861,"5min":97.408000000000001,"15min":127.44499999999999},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":6.0880000000000001,"15min":4.9740000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.2120000000000002,"5min":125.539,"15min":125.539},"last":0.33900000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:35:53 2023","interfaces":[{"interface":"back","average":{"1min":7.923,"5min":7.0129999999999999,"15min":4.4459999999999997},"min":{"1min":0.316,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":73.813999999999993,"5min":138.87299999999999,"15min":138.87299999999999},"last":1.5029999999999999},{"interface":"front","average":{"1min":8.4450000000000003,"5min":6.9260000000000002,"15min":4.7350000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":73.852999999999994,"5min":138.976,"15min":138.976},"last":0.51300000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.82999999999999996,"15min":1.274},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":1.633,"5min":7.101,"15min":166.05199999999999},"last":0.441},{"interface":"front","average":{"1min":0.55800000000000005,"5min":1.0289999999999999,"15min":1.4890000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.089,"5min":8.3499999999999996,"15min":166.631},"last":0.35599999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.96699999999999997,"15min":0.82399999999999995},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.1579999999999999,"5min":11.473000000000001,"15min":11.473000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.433,"5min":1.091,"15min":0.94099999999999995},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.249},"max":{"1min":0.76700000000000002,"5min":7.5670000000000002,"15min":8.4090000000000007},"last":0.39900000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":1.294,"15min":1.0289999999999999},"min":{"1min":0.30499999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":0.70599999999999996,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.497},{"interface":"front","average":{"1min":0.72999999999999998,"5min":1.4019999999999999,"15min":1.0720000000000001},"min":{"1min":0.36099999999999999,"5min":0.35499999999999998,"15min":0.33500000000000002},"max":{"1min":2.665,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.58999999999999997}]}]},{"osd":1,"up_from":112,"seq":481036337502,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2124696,"kb_used_data":1215760,"kb_used_omap":4,"kb_used_meta":908923,"kb_avail":91620456,"statfs":{"total":95995035648,"available":93819346944,"internally_reserved":0,"allocated":1244938240,"data_stored":1242135545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":930738062},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":1.881,"15min":3.4060000000000001},"min":{"1min":0.217,"5min":0.17799999999999999,"15min":0.16700000000000001},"max":{"1min":2.7970000000000002,"5min":47.195999999999998,"15min":130.768},"last":0.254},{"interface":"front","average":{"1min":0.40999999999999998,"5min":1.5469999999999999,"15min":2.726},"min":{"1min":0.24299999999999999,"5min":0.20499999999999999,"15min":0.19},"max":{"1min":1.016,"5min":46.582000000000001,"15min":134.929},"last":0.32400000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":7.1859999999999999,"15min":6.7910000000000004},"min":{"1min":0.26300000000000001,"5min":0.214,"15min":0.20100000000000001},"max":{"1min":1.8839999999999999,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.65800000000000003,"5min":7.6059999999999999,"15min":6.9160000000000004},"min":{"1min":0.27900000000000003,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":1.968,"5min":202.12,"15min":232.21000000000001},"last":0.373}]},{"osd":3,"last update":"Sat Sep 23 23:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":5.1989999999999998,"15min":5.1799999999999997},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.221},"max":{"1min":0.999,"5min":181.94399999999999,"15min":234.404},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.75900000000000001,"5min":5.29,"15min":5.0469999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":6.6829999999999998,"5min":181.98500000000001,"15min":236.33699999999999},"last":0.39900000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":1.23,"15min":1.423},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":2.1059999999999999,"5min":12.042999999999999,"15min":12.042999999999999},"last":0.434},{"interface":"front","average":{"1min":0.624,"5min":1.129,"15min":1.46},"min":{"1min":0.48099999999999998,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":1.2669999999999999,"5min":9.5069999999999997,"15min":18.108000000000001},"last":0.46300000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":2.9209999999999998,"15min":1.796},"min":{"1min":0.34300000000000003,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66600000000000004,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.68200000000000005,"5min":2.9940000000000002,"15min":2.113},"min":{"1min":0.40899999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.044,"5min":151.821,"15min":151.821},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.3600000000000001,"15min":1.7529999999999999},"min":{"1min":0.377,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":1.1040000000000001,"5min":8.9269999999999996,"15min":74.087999999999994},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.97699999999999998,"5min":1.585,"15min":1.968},"min":{"1min":0.42999999999999999,"5min":0.35699999999999998,"15min":0.33100000000000002},"max":{"1min":5.367,"5min":11.927,"15min":73.974999999999994},"last":0.40999999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.7390000000000001,"15min":2.6099999999999999},"min":{"1min":0.50600000000000001,"5min":0.38700000000000001,"15min":0.32900000000000001},"max":{"1min":1.3069999999999999,"5min":44.241,"15min":78.308000000000007},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.8240000000000001,"15min":2.484},"min":{"1min":0.37,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":2.1339999999999999,"5min":43.920999999999999,"15min":77.204999999999998},"last":0.96699999999999997}]}]},{"osd":0,"up_from":91,"seq":390842024326,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1518880,"kb_used_data":312728,"kb_used_omap":5,"kb_used_meta":1206138,"kb_avail":92226272,"statfs":{"total":95995035648,"available":94439702528,"internally_reserved":0,"allocated":320233472,"data_stored":317417184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1235085946},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:35:32 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":10.577,"15min":6.7670000000000003},"min":{"1min":0.20499999999999999,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":9.0310000000000006,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.754,"5min":10.407999999999999,"15min":5.9059999999999997},"min":{"1min":0.20200000000000001,"5min":0.19700000000000001,"15min":0.17599999999999999},"max":{"1min":5.4729999999999999,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.40500000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:35:57 2023","interfaces":[{"interface":"back","average":{"1min":3.2130000000000001,"5min":10.026999999999999,"15min":7.5819999999999999},"min":{"1min":0.26200000000000001,"5min":0.17699999999999999,"15min":0.17199999999999999},"max":{"1min":40.326999999999998,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":3.1720000000000002,"5min":10.18,"15min":7.6520000000000001},"min":{"1min":0.22900000000000001,"5min":0.19800000000000001,"15min":0.17599999999999999},"max":{"1min":40.396000000000001,"5min":283.161,"15min":283.161},"last":0.44}]},{"osd":3,"last update":"Sat Sep 23 23:35:32 2023","interfaces":[{"interface":"back","average":{"1min":2.7360000000000002,"5min":8.6509999999999998,"15min":5.4589999999999996},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":41.191000000000003,"5min":220.732,"15min":220.732},"last":0.59599999999999997},{"interface":"front","average":{"1min":2.9390000000000001,"5min":9.0440000000000005,"15min":5.5570000000000004},"min":{"1min":0.28799999999999998,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":46.688000000000002,"5min":218.78200000000001,"15min":218.78200000000001},"last":0.47599999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":1.1599999999999999,"15min":2.4769999999999999},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.70799999999999996,"5min":14.676,"15min":138.81299999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.80100000000000005,"5min":1.391,"15min":3.0830000000000002},"min":{"1min":0.33800000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.7469999999999999,"5min":9.9039999999999999,"15min":171.251},"last":0.61099999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":1.504,"15min":2.7930000000000001},"min":{"1min":0.35299999999999998,"5min":0.32700000000000001,"15min":0.32100000000000001},"max":{"1min":4.6820000000000004,"5min":15.076000000000001,"15min":171.30699999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.625,"5min":1.244,"15min":2.0819999999999999},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":1.8149999999999999,"5min":8.9909999999999997,"15min":139.40799999999999},"last":0.629}]},{"osd":6,"last update":"Sat Sep 23 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.325,"15min":3.097},"min":{"1min":0.38100000000000001,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":4.718,"5min":9.968,"15min":171.33600000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.335,"15min":3.3650000000000002},"min":{"1min":0.37,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":1.8700000000000001,"5min":9.8320000000000007,"15min":184.13499999999999},"last":0.65800000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":1.373,"15min":2.6099999999999999},"min":{"1min":0.38400000000000001,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":0.751,"5min":9.452,"15min":138.77699999999999},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.6379999999999999,"15min":2.5030000000000001},"min":{"1min":0.379,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":1.839,"5min":9.9670000000000005,"15min":139.37899999999999},"last":0.57399999999999995}]}]},{"osd":2,"up_from":199,"seq":854698492147,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3256988,"kb_used_data":2408980,"kb_used_omap":7,"kb_used_meta":847992,"kb_avail":90488164,"statfs":{"total":95995035648,"available":92659879936,"internally_reserved":0,"allocated":2466795520,"data_stored":2463932900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":868343889},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":2.2080000000000002,"15min":3.8599999999999999},"min":{"1min":0.186,"5min":0.186,"15min":0.16700000000000001},"max":{"1min":4.7590000000000003,"5min":61.950000000000003,"15min":119.277},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":2.153,"15min":3.6669999999999998},"min":{"1min":0.23999999999999999,"5min":0.20399999999999999,"15min":0.18099999999999999},"max":{"1min":2.5070000000000001,"5min":61.984999999999999,"15min":119.56100000000001},"last":0.26900000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.911,"5min":5.3010000000000002,"15min":4.5940000000000003},"min":{"1min":0.25,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":8.3640000000000008,"5min":106.765,"15min":157.98699999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.79000000000000004,"5min":4.8959999999999999,"15min":4.577},"min":{"1min":0.253,"5min":0.222,"15min":0.222},"max":{"1min":4.0940000000000003,"5min":106.732,"15min":158.03800000000001},"last":0.40500000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":5.7800000000000002,"15min":4.7770000000000001},"min":{"1min":0.255,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":8.3070000000000004,"5min":106.70399999999999,"15min":119.366},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.65800000000000003,"5min":5.4169999999999998,"15min":4.2709999999999999},"min":{"1min":0.29299999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":3.375,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.56499999999999995}]},{"osd":4,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":2.036,"15min":2.262},"min":{"1min":0.36099999999999999,"5min":0.33700000000000002,"15min":0.25},"max":{"1min":2.6070000000000002,"5min":62.081000000000003,"15min":119.88200000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.4590000000000001,"5min":2.3860000000000001,"15min":1.865},"min":{"1min":0.36899999999999999,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":8.343,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.44400000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":3.0299999999999998,"15min":2.6480000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.26200000000000001},"max":{"1min":2.548,"5min":91.207999999999998,"15min":119.919},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.4550000000000001,"5min":3.2749999999999999,"15min":2.6160000000000001},"min":{"1min":0.33500000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.3889999999999993,"5min":91.150999999999996,"15min":119.306},"last":0.48399999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":3.5750000000000002,"15min":3.0019999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":2.5819999999999999,"5min":107.018,"15min":119.943},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.486,"5min":2.6440000000000001,"15min":2.1600000000000001},"min":{"1min":0.371,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":8.4060000000000006,"5min":62.493000000000002,"15min":62.493000000000002},"last":0.52200000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":2.2010000000000001,"15min":1.9179999999999999},"min":{"1min":0.40000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.381,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.60099999999999998,"5min":3.464,"15min":2.3620000000000001},"min":{"1min":0.42699999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.417,"5min":106.809,"15min":106.809},"last":0.61399999999999999}]}]},{"osd":3,"up_from":148,"seq":635655160115,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2211796,"kb_used_data":1304396,"kb_used_omap":2,"kb_used_meta":907389,"kb_avail":91533356,"statfs":{"total":95995035648,"available":93730156544,"internally_reserved":0,"allocated":1335701504,"data_stored":1332900525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":929166389},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7390000000000001,"5min":1.4059999999999999,"15min":2.79},"min":{"1min":0.19600000000000001,"5min":0.19400000000000001,"15min":0.158},"max":{"1min":15.603,"5min":17.213000000000001,"15min":103.866},"last":0.25},{"interface":"front","average":{"1min":1.5880000000000001,"5min":1.877,"15min":2.9049999999999998},"min":{"1min":0.223,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":12.202,"5min":22.754000000000001,"15min":101.884},"last":3.4980000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":3.2200000000000002,"15min":2.9969999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19600000000000001},"max":{"1min":7.3319999999999999,"5min":56.119,"15min":85.242999999999995},"last":1.123},{"interface":"front","average":{"1min":1.4910000000000001,"5min":2.427,"15min":2.7999999999999998},"min":{"1min":0.22800000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":11.228,"5min":53.962000000000003,"15min":85.302000000000007},"last":0.312}]},{"osd":2,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":4.7770000000000001,"15min":4.7859999999999996},"min":{"1min":0.25800000000000001,"5min":0.25,"15min":0.19800000000000001},"max":{"1min":2.1190000000000002,"5min":73.742000000000004,"15min":101.279},"last":1.1599999999999999},{"interface":"front","average":{"1min":0.45000000000000001,"5min":4.6349999999999998,"15min":4.6130000000000004},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.214},"max":{"1min":0.86699999999999999,"5min":76.525999999999996,"15min":101.294},"last":0.35399999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.3959999999999999,"15min":1.339},"min":{"1min":0.28799999999999998,"5min":0.28499999999999998,"15min":0.26700000000000002},"max":{"1min":11.209,"5min":11.209,"15min":11.209},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.2829999999999999,"5min":1.905,"15min":1.8049999999999999},"min":{"1min":0.33500000000000002,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":7.3940000000000001,"5min":11.494999999999999,"15min":24.782},"last":0.42699999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4410000000000001,"5min":6.8159999999999998,"15min":3.4289999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":7.3120000000000003,"5min":214.43700000000001,"15min":214.43700000000001},"last":0.373},{"interface":"front","average":{"1min":1.2470000000000001,"5min":6.7569999999999997,"15min":3.4900000000000002},"min":{"1min":0.34699999999999998,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":7.2290000000000001,"5min":214.46600000000001,"15min":214.46600000000001},"last":1.196}]},{"osd":6,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":2.7090000000000001,"15min":2.3159999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":9.798,"5min":58.932000000000002,"15min":58.932000000000002},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.25,"5min":2.8660000000000001,"15min":2.3170000000000002},"min":{"1min":0.36499999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.202,"5min":59.186,"15min":59.186},"last":1.2270000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":4.7480000000000002,"15min":3.2069999999999999},"min":{"1min":0.38700000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.415,"5min":192.69,"15min":192.69},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.753,"5min":5.1289999999999996,"15min":3.6949999999999998},"min":{"1min":0.43099999999999999,"5min":0.39200000000000002,"15min":0.36699999999999999},"max":{"1min":2.6560000000000001,"5min":192.65600000000001,"15min":192.65600000000001},"last":0.47099999999999997}]}]},{"osd":4,"up_from":0,"seq":128849019434,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757797,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2975444,"kb_used_data":2248844,"kb_used_omap":0,"kb_used_meta":726592,"kb_avail":90769708,"statfs":{"total":95995035648,"available":92948180992,"internally_reserved":0,"allocated":2302816256,"data_stored":2301181942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":744030208},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:14 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":1.141,"15min":3.198},"min":{"1min":0.252,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":1.5640000000000001,"5min":21.109000000000002,"15min":113.869},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.38,"5min":1.3939999999999999,"15min":3.5710000000000002},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.22700000000000001},"max":{"1min":0.76200000000000001,"5min":20.876999999999999,"15min":138.55199999999999},"last":0.34200000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:36:20 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.6200000000000001,"15min":2.258},"min":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.23100000000000001},"max":{"1min":1.9930000000000001,"5min":29.007999999999999,"15min":69.816999999999993},"last":0.439},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.8,"15min":2.5840000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":2.8130000000000002,"5min":29.053999999999998,"15min":70.268000000000001},"last":0.46800000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:35:47 2023","interfaces":[{"interface":"back","average":{"1min":7.7469999999999999,"5min":6.5350000000000001,"15min":4.609},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.26900000000000002},"max":{"1min":145.93199999999999,"5min":208.738,"15min":208.738},"last":2.2040000000000002},{"interface":"front","average":{"1min":1.0269999999999999,"5min":5.0499999999999998,"15min":4.1150000000000002},"min":{"1min":0.31,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":2.859,"5min":208.631,"15min":208.631},"last":0.52400000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:35:44 2023","interfaces":[{"interface":"back","average":{"1min":6.4889999999999999,"5min":4.1520000000000001,"15min":3.6589999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":122.071,"5min":122.071,"15min":122.071},"last":0.45100000000000001},{"interface":"front","average":{"1min":6.4669999999999996,"5min":4.4969999999999999,"15min":3.8359999999999999},"min":{"1min":0.35799999999999998,"5min":0.35399999999999998,"15min":0.318},"max":{"1min":122.09699999999999,"5min":122.09699999999999,"15min":122.09699999999999},"last":0.42799999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.151,"15min":0.92000000000000004},"min":{"1min":0.34200000000000003,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":3.5499999999999998,"5min":9.2690000000000001,"15min":19.962},"last":0.505},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.093,"15min":0.872},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.22900000000000001},"max":{"1min":2.4990000000000001,"5min":10.273999999999999,"15min":20.829999999999998},"last":0.49099999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:35:28 2023","interfaces":[{"interface":"back","average":{"1min":7.2519999999999998,"5min":2.3799999999999999,"15min":1.635},"min":{"1min":0.32400000000000001,"5min":0.27600000000000002,"15min":0.26900000000000002},"max":{"1min":148.36600000000001,"5min":148.36600000000001,"15min":148.36600000000001},"last":0.41199999999999998},{"interface":"front","average":{"1min":7.7210000000000001,"5min":2.9420000000000002,"15min":1.869},"min":{"1min":0.34599999999999997,"5min":0.307,"15min":0.26000000000000001},"max":{"1min":148.42500000000001,"5min":148.42500000000001,"15min":148.42500000000001},"last":2.1549999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:35:47 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.3160000000000001,"15min":2.625},"min":{"1min":0.34699999999999998,"5min":0.34599999999999997,"15min":0.317},"max":{"1min":1.843,"5min":15.497,"15min":179.28800000000001},"last":2.29},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.3480000000000001,"15min":2.7050000000000001},"min":{"1min":0.36699999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.867,"5min":15.653,"15min":179.428},"last":0.47899999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":311951360,"data_stored":311951360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1233457152,"data_stored":1233453080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2457206784,"data_stored":2457206784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1324355584,"data_stored":1324351512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2467495936,"data_stored":2467495936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2293104640,"data_stored":2293104640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1364721664,"data_stored":1364721664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1444286464,"data_stored":1444282392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:36:34.212 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:36:34.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:34 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 25296/181539 objects degraded (13.934%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:34.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:34 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 25296/181539 objects degraded (13.934%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:34.483 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:34.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:34.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:34.589 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:36:34.613 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:34 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 25296/181539 objects degraded (13.934%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:34.616 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:34.635 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:36:34.666 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:36:34.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:34.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:36:34.772 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:34.797 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:34.847 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:34.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:34.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:34.955 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:34.991 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:35.041 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:35.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:35.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:35.147 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:35.178 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:35.453 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:35.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:35 smithi110 ceph-mon[212939]: pgmap v1656: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 25231/180624 objects degraded (13.969%); 3615/180624 objects misplaced (2.001%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T23:36:35.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:35 smithi110 ceph-mon[221102]: pgmap v1656: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 25231/180624 objects degraded (13.969%); 3615/180624 objects misplaced (2.001%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T23:36:35.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:35.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:35.488 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:35.558 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:35.590 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:35 smithi138 ceph-mon[193606]: pgmap v1656: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 25231/180624 objects degraded (13.969%); 3615/180624 objects misplaced (2.001%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T23:36:35.798 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:35.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:35.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:35.821 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:35.904 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:36:35.930 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:35.973 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:36:35.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:35.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:36:36.078 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:36.107 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:36.158 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:36.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:36.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:36.264 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:36.298 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:36.515 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:36:36.775 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:36:36.775 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:36:36.775 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:36:36.779 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:36:36.807 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:36:36.833 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:36:36.845 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:36:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:36:36] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.43.0" 2023-09-23T23:36:36.861 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:36.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:36.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:36.865 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:36:36.894 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:36:36.901 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:36:36.901 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:36:36.901 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:36:36.901 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:36:36.902 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:36:36.902 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:36:36.902 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:36:36.902 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:36:36.965 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:37.013 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:37.132 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:37.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:37.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:37.236 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:37.264 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:37.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:37 smithi110 ceph-mon[221102]: pgmap v1657: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 242 op/s; 25166/178146 objects degraded (14.127%); 3615/178146 objects misplaced (2.029%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:37.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:37 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2820195512' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:37.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:37 smithi110 ceph-mon[212939]: pgmap v1657: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 242 op/s; 25166/178146 objects degraded (14.127%); 3615/178146 objects misplaced (2.029%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:37.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:37 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2820195512' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:37.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:37 smithi138 ceph-mon[193606]: pgmap v1657: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 242 op/s; 25166/178146 objects degraded (14.127%); 3615/178146 objects misplaced (2.029%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:37.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:37 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2820195512' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:37.961 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:37.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:37.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:38.066 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:38.107 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:38.273 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:38.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:38.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:38.377 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:38.404 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:38.449 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:38.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:38.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:38.493 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:36:38.493 INFO:teuthology.orchestra.run.smithi110.stdout:{"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 25086/177246 objects degraded (14.153%), 7 pgs degraded, 5 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2076,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":595,"num_osds":8,"num_up_osds":8,"osd_up_since":1695511414,"num_in_osds":7,"osd_in_since":1695512030,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":110},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+recovering","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":119,"num_pools":5,"num_objects":59082,"data_bytes":3870995701,"bytes_used":16134660096,"bytes_avail":655830589440,"bytes_total":671965249536,"degraded_objects":25086,"degraded_total":177246,"degraded_ratio":0.14153210791780915,"misplaced_objects":3537,"misplaced_total":177246,"misplaced_ratio":0.019955316340002031,"recovering_objects_per_sec":46,"recovering_bytes_per_sec":3039671,"recovering_keys_per_sec":0,"num_objects_recovered":464,"num_bytes_recovered":30408704,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":241},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi110.fuglpi","status":"up:active","gid":24572}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.110:8443/","prometheus":"http://172.21.15.110:9283/"}},"servicemap":{"epoch":32,"modified":"2023-09-23T23:32:57.852217+0000","services":{}},"progress_events":{"833ef15d-07ae-4299-b4e4-2cf94e068147":{"message":"Upgrade to 18.2.0-460-gbb0daeff (10m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true},"c67c6d89-9cd0-495b-b1de-fa24b5d12727":{"message":"Global Recovery Event (2m)\n [=========================...] (remaining: 11s)","progress":0.92436975240707397,"add_to_ceph_s":true}}} 2023-09-23T23:36:38.555 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:36:38.581 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:38.626 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:36:38.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:38.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:36:38.731 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:38.757 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:38.802 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:38.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:38.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:38.909 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:38.943 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:39.250 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:39.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:39.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:39.354 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:39.361 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:39.397 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:36:39.398 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:36:39.410 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:39.513 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:39 smithi110 ceph-mon[221102]: pgmap v1658: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 241 op/s; 25086/177246 objects degraded (14.153%); 3537/177246 objects misplaced (1.996%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T23:36:39.513 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:39 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4173473845' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:36:39.513 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:39 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 25086/177246 objects degraded (14.153%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:39.514 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:39 smithi110 ceph-mon[212939]: pgmap v1658: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 241 op/s; 25086/177246 objects degraded (14.153%); 3537/177246 objects misplaced (1.996%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T23:36:39.514 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:39 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4173473845' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:36:39.514 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:39 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 25086/177246 objects degraded (14.153%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:39.616 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:39.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:39.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:39.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:39 smithi138 ceph-mon[193606]: pgmap v1658: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 241 op/s; 25086/177246 objects degraded (14.153%); 3537/177246 objects misplaced (1.996%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T23:36:39.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:39 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4173473845' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:36:39.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:39 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 25086/177246 objects degraded (14.153%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:39.721 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:36:39.746 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:39.779 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:36:39.779 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:36:39.790 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:36:39.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:39.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:36:39.895 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:39.935 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:39.967 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:36:40.416 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:40.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:40.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:40.523 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:36:40.549 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:40.607 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:36:40.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:40.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:36:40.712 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:40.745 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:41.121 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:41.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:41.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:41.155 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:36:41.226 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:36:41.227 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:41.252 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:41.295 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:36:41.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:41.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:36:41.400 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:41.444 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:41 smithi110 ceph-mon[221102]: pgmap v1659: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 25004/175281 objects degraded (14.265%); 3537/175281 objects misplaced (2.018%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:41.444 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:41 smithi110 ceph-mon[212939]: pgmap v1659: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 25004/175281 objects degraded (14.265%); 3537/175281 objects misplaced (2.018%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:41.449 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:41.514 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:41.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:41.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:41.525 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:41.621 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:41.645 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:41 smithi138 ceph-mon[193606]: pgmap v1659: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 25004/175281 objects degraded (14.265%); 3537/175281 objects misplaced (2.018%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:41.647 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:41.691 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:41.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:41.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:41.796 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:41.835 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:42.046 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:42.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:42.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:42.150 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:42.176 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:42.264 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:42 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/611250940' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:36:42.269 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:36:42.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:42 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/611250940' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:36:42.695 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:42 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/611250940' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:36:42.904 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:42.926 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:42.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:42.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:43.031 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:43.058 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:43.101 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:43.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:43.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:43.207 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:36:43.233 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:43.275 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:36:43.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:43.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:36:43.381 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:43.404 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:43 smithi138 ceph-mon[193606]: pgmap v1660: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 229 op/s; 25004/174639 objects degraded (14.318%); 3537/174639 objects misplaced (2.025%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:43.404 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:43 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:36:43.404 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:43 smithi138 ceph-mon[193606]: failed to encode map e596 with expected crc 2023-09-23T23:36:43.404 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:43 smithi138 ceph-mon[193606]: failed to encode map e596 with expected crc 2023-09-23T23:36:43.404 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:43 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/611250940' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:36:43.404 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:43 smithi138 ceph-mon[193606]: osdmap e596: 8 total, 8 up, 7 in 2023-09-23T23:36:43.405 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:36:43.406 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:43.450 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:43.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:43.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:43.555 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:43.586 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:43 smithi110 ceph-mon[212939]: pgmap v1660: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 229 op/s; 25004/174639 objects degraded (14.318%); 3537/174639 objects misplaced (2.025%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:43.586 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:43 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:36:43.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:43 smithi110 ceph-mon[212939]: failed to encode map e596 with expected crc 2023-09-23T23:36:43.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:43 smithi110 ceph-mon[212939]: failed to encode map e596 with expected crc 2023-09-23T23:36:43.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:43 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/611250940' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:36:43.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:43 smithi110 ceph-mon[212939]: osdmap e596: 8 total, 8 up, 7 in 2023-09-23T23:36:43.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:36:43.588 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:43 smithi110 ceph-mon[221102]: pgmap v1660: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 229 op/s; 25004/174639 objects degraded (14.318%); 3537/174639 objects misplaced (2.025%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:43.588 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:43 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:36:43.588 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:43 smithi110 ceph-mon[221102]: failed to encode map e596 with expected crc 2023-09-23T23:36:43.588 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:43 smithi110 ceph-mon[221102]: failed to encode map e596 with expected crc 2023-09-23T23:36:43.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:43 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/611250940' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:36:43.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:43 smithi110 ceph-mon[221102]: osdmap e596: 8 total, 8 up, 7 in 2023-09-23T23:36:43.589 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:36:43.590 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:43.692 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:43.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:43.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:43.798 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:43.824 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:44.371 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:44.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:44.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:44.476 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:44.508 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:44.563 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:44.570 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:44 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 25004/174639 objects degraded (14.318%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:44.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:44.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:44.640 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:44 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 25004/174639 objects degraded (14.318%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:44.640 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:44 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 25004/174639 objects degraded (14.318%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:44.672 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:44.720 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:44.916 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:36:44.918 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:36:45.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:45 smithi138 ceph-mon[193606]: pgmap v1662: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 24938/173820 objects degraded (14.347%); 3454/173820 objects misplaced (1.987%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T23:36:45.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:45 smithi110 ceph-mon[212939]: pgmap v1662: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 24938/173820 objects degraded (14.347%); 3454/173820 objects misplaced (1.987%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T23:36:45.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:45 smithi110 ceph-mon[221102]: pgmap v1662: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 24938/173820 objects degraded (14.347%); 3454/173820 objects misplaced (1.987%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T23:36:45.712 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:36:45.772 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:45.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:45.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:45.878 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:36:45.904 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:45.946 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:36:45.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:45.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:36:46.094 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:36:46.123 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:46.168 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:36:46.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:46.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:36:46.274 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:46.277 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:46 smithi110 ceph-mon[221102]: from='client.36855 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:46.300 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:46.346 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:46.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:46.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:46.366 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1662,"stamp":"2023-09-23T23:36:43.926681+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3796153589,"num_objects":57940,"num_object_clones":0,"num_object_copies":173820,"num_objects_missing_on_primary":2559,"num_objects_missing":2559,"num_objects_degraded":24938,"num_objects_misplaced":3454,"num_objects_unfound":0,"num_objects_dirty":57940,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":117717,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15201,"num_bytes_recovered":993522088,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94964,"ondisk_log_size":94964,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15563300,"kb_used_data":9838508,"kb_used_omap":29,"kb_used_meta":5724706,"kb_avail":640652764,"statfs":{"total":671965249536,"available":656028430336,"internally_reserved":0,"allocated":10074632192,"data_stored":10058392097,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5862099667},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-148635648,"num_objects":-2268,"num_object_clones":0,"num_object_copies":-6804,"num_objects_missing_on_primary":-161,"num_objects_missing":-161,"num_objects_degraded":-293,"num_objects_misplaced":-161,"num_objects_unfound":0,"num_objects_dirty":-2268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2268,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":454,"num_bytes_recovered":29753344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002998"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":791,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.211265+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:36:23.211265+0000","last_peered":"2023-09-23T23:36:23.211265+0000","last_clean":"2023-09-23T23:36:23.211265+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:36:23.211265+0000","last_undegraded":"2023-09-23T23:36:23.211265+0000","last_fullsized":"2023-09-23T23:36:23.211265+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":350,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.264754+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:36:32.264754+0000","last_peered":"2023-09-23T23:36:32.264754+0000","last_clean":"2023-09-23T23:36:32.264754+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:36:32.264754+0000","last_undegraded":"2023-09-23T23:36:32.264754+0000","last_fullsized":"2023-09-23T23:36:32.264754+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":578,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.300494+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:36:32.300494+0000","last_peered":"2023-09-23T23:36:32.300494+0000","last_clean":"2023-09-23T23:36:32.300494+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:36:32.300494+0000","last_undegraded":"2023-09-23T23:36:32.300494+0000","last_fullsized":"2023-09-23T23:36:32.300494+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":744,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.211870+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:36:23.211870+0000","last_peered":"2023-09-23T23:36:23.211870+0000","last_clean":"2023-09-23T23:36:23.211870+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:36:23.211870+0000","last_undegraded":"2023-09-23T23:36:23.211870+0000","last_fullsized":"2023-09-23T23:36:23.211870+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":315,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.540789+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:36:32.540789+0000","last_peered":"2023-09-23T23:36:32.540789+0000","last_clean":"2023-09-23T23:36:32.540789+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:36:32.540789+0000","last_undegraded":"2023-09-23T23:36:32.540789+0000","last_fullsized":"2023-09-23T23:36:32.540789+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":557,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.274984+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:36:32.274984+0000","last_peered":"2023-09-23T23:36:32.274984+0000","last_clean":"2023-09-23T23:36:32.274984+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:36:32.274984+0000","last_undegraded":"2023-09-23T23:36:32.274984+0000","last_fullsized":"2023-09-23T23:36:32.274984+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":757,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.271825+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:36:32.271825+0000","last_peered":"2023-09-23T23:36:32.271825+0000","last_clean":"2023-09-23T23:36:32.271825+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:36:32.271825+0000","last_undegraded":"2023-09-23T23:36:32.271825+0000","last_fullsized":"2023-09-23T23:36:32.271825+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":614,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.274789+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:36:32.274789+0000","last_peered":"2023-09-23T23:36:32.274789+0000","last_clean":"2023-09-23T23:36:32.274789+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:36:32.274789+0000","last_undegraded":"2023-09-23T23:36:32.274789+0000","last_fullsized":"2023-09-23T23:36:32.274789+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":109,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.218011+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:36:23.218011+0000","last_peered":"2023-09-23T23:36:23.218011+0000","last_clean":"2023-09-23T23:36:23.218011+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:36:23.218011+0000","last_undegraded":"2023-09-23T23:36:23.218011+0000","last_fullsized":"2023-09-23T23:36:23.218011+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":916,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.272143+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:36:32.272143+0000","last_peered":"2023-09-23T23:36:32.272143+0000","last_clean":"2023-09-23T23:36:32.272143+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:36:32.272143+0000","last_undegraded":"2023-09-23T23:36:32.272143+0000","last_fullsized":"2023-09-23T23:36:32.272143+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":247,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.211919+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:36:23.211919+0000","last_peered":"2023-09-23T23:36:23.211919+0000","last_clean":"2023-09-23T23:36:23.211919+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:36:23.211919+0000","last_undegraded":"2023-09-23T23:36:23.211919+0000","last_fullsized":"2023-09-23T23:36:23.211919+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":597,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:37.573594+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:36:37.573594+0000","last_peered":"2023-09-23T23:36:37.573594+0000","last_clean":"2023-09-23T23:36:37.573594+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:36:37.573594+0000","last_undegraded":"2023-09-23T23:36:37.573594+0000","last_fullsized":"2023-09-23T23:36:37.573594+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":888,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.225169+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:36:23.225169+0000","last_peered":"2023-09-23T23:36:23.225169+0000","last_clean":"2023-09-23T23:36:23.225169+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:36:23.225169+0000","last_undegraded":"2023-09-23T23:36:23.225169+0000","last_fullsized":"2023-09-23T23:36:23.225169+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":226,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.212686+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:36:23.212686+0000","last_peered":"2023-09-23T23:36:23.212686+0000","last_clean":"2023-09-23T23:36:23.212686+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:36:23.212686+0000","last_undegraded":"2023-09-23T23:36:23.212686+0000","last_fullsized":"2023-09-23T23:36:23.212686+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":558,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.279036+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:36:42.279036+0000","last_peered":"2023-09-23T23:36:42.279036+0000","last_clean":"2023-09-23T23:36:42.279036+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:36:42.279036+0000","last_undegraded":"2023-09-23T23:36:42.279036+0000","last_fullsized":"2023-09-23T23:36:42.279036+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1049,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.211698+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:36:23.211698+0000","last_peered":"2023-09-23T23:36:23.211698+0000","last_clean":"2023-09-23T23:36:23.211698+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:36:23.211698+0000","last_undegraded":"2023-09-23T23:36:23.211698+0000","last_fullsized":"2023-09-23T23:36:23.211698+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":269,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.291299+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:36:42.291299+0000","last_peered":"2023-09-23T23:36:42.291299+0000","last_clean":"2023-09-23T23:36:42.291299+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:36:42.291299+0000","last_undegraded":"2023-09-23T23:36:42.291299+0000","last_fullsized":"2023-09-23T23:36:42.291299+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":128,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.216895+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:36:23.216895+0000","last_peered":"2023-09-23T23:36:23.216895+0000","last_clean":"2023-09-23T23:36:23.216895+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:36:23.216895+0000","last_undegraded":"2023-09-23T23:36:23.216895+0000","last_fullsized":"2023-09-23T23:36:23.216895+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":447,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.275124+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:36:32.275124+0000","last_peered":"2023-09-23T23:36:32.275124+0000","last_clean":"2023-09-23T23:36:32.275124+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:36:32.275124+0000","last_undegraded":"2023-09-23T23:36:32.275124+0000","last_fullsized":"2023-09-23T23:36:32.275124+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":169,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.217035+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:36:23.217035+0000","last_peered":"2023-09-23T23:36:23.217035+0000","last_clean":"2023-09-23T23:36:23.217035+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:36:23.217035+0000","last_undegraded":"2023-09-23T23:36:23.217035+0000","last_fullsized":"2023-09-23T23:36:23.217035+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":536,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.217360+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:36:23.217360+0000","last_peered":"2023-09-23T23:36:23.217360+0000","last_clean":"2023-09-23T23:36:23.217360+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:36:23.217360+0000","last_undegraded":"2023-09-23T23:36:23.217360+0000","last_fullsized":"2023-09-23T23:36:23.217360+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":750,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.211204+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:36:23.211204+0000","last_peered":"2023-09-23T23:36:23.211204+0000","last_clean":"2023-09-23T23:36:23.211204+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:36:23.211204+0000","last_undegraded":"2023-09-23T23:36:23.211204+0000","last_fullsized":"2023-09-23T23:36:23.211204+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":142,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.263094+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:36:32.263094+0000","last_peered":"2023-09-23T23:36:32.263094+0000","last_clean":"2023-09-23T23:36:32.263094+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:36:32.263094+0000","last_undegraded":"2023-09-23T23:36:32.263094+0000","last_fullsized":"2023-09-23T23:36:32.263094+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":596,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.279630+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:36:42.279630+0000","last_peered":"2023-09-23T23:36:42.279630+0000","last_clean":"2023-09-23T23:36:42.279630+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:36:42.279630+0000","last_undegraded":"2023-09-23T23:36:42.279630+0000","last_fullsized":"2023-09-23T23:36:42.279630+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":880,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.216669+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:36:23.216669+0000","last_peered":"2023-09-23T23:36:23.216669+0000","last_clean":"2023-09-23T23:36:23.216669+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:36:23.216669+0000","last_undegraded":"2023-09-23T23:36:23.216669+0000","last_fullsized":"2023-09-23T23:36:23.216669+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":191,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.280782+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:36:42.280782+0000","last_peered":"2023-09-23T23:36:42.280782+0000","last_clean":"2023-09-23T23:36:42.280782+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:36:42.280782+0000","last_undegraded":"2023-09-23T23:36:42.280782+0000","last_fullsized":"2023-09-23T23:36:42.280782+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":318,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.292341+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:36:42.292341+0000","last_peered":"2023-09-23T23:36:42.292341+0000","last_clean":"2023-09-23T23:36:42.292341+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:36:42.292341+0000","last_undegraded":"2023-09-23T23:36:42.292341+0000","last_fullsized":"2023-09-23T23:36:42.292341+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":559,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.282150+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:36:32.282150+0000","last_peered":"2023-09-23T23:36:32.282150+0000","last_clean":"2023-09-23T23:36:32.282150+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:36:32.282150+0000","last_undegraded":"2023-09-23T23:36:32.282150+0000","last_fullsized":"2023-09-23T23:36:32.282150+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":463,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.216743+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:36:23.216743+0000","last_peered":"2023-09-23T23:36:23.216743+0000","last_clean":"2023-09-23T23:36:23.216743+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:36:23.216743+0000","last_undegraded":"2023-09-23T23:36:23.216743+0000","last_fullsized":"2023-09-23T23:36:23.216743+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":552,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.275289+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:36:32.275289+0000","last_peered":"2023-09-23T23:36:32.275289+0000","last_clean":"2023-09-23T23:36:32.275289+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:36:32.275289+0000","last_undegraded":"2023-09-23T23:36:32.275289+0000","last_fullsized":"2023-09-23T23:36:32.275289+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":958,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.211438+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:36:23.211438+0000","last_peered":"2023-09-23T23:36:23.211438+0000","last_clean":"2023-09-23T23:36:23.211438+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:36:23.211438+0000","last_undegraded":"2023-09-23T23:36:23.211438+0000","last_fullsized":"2023-09-23T23:36:23.211438+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":159,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.216885+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:36:23.216885+0000","last_peered":"2023-09-23T23:36:23.216885+0000","last_clean":"2023-09-23T23:36:23.216885+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:36:23.216885+0000","last_undegraded":"2023-09-23T23:36:23.216885+0000","last_fullsized":"2023-09-23T23:36:23.216885+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":237,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.275461+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:36:32.275461+0000","last_peered":"2023-09-23T23:36:32.275461+0000","last_clean":"2023-09-23T23:36:32.275461+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:36:32.275461+0000","last_undegraded":"2023-09-23T23:36:32.275461+0000","last_fullsized":"2023-09-23T23:36:32.275461+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"595'7201","reported_seq":7279,"reported_epoch":595,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:39.032375+0000","last_change":"2023-09-23T23:33:52.884769+0000","last_active":"2023-09-23T23:36:39.032375+0000","last_peered":"2023-09-23T23:36:39.032375+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:36:39.032375+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"549'1800","ondisk_log_start":"549'1800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5401,"log_dups_size":0,"ondisk_log_size":5401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239009792,"num_objects":3647,"num_object_clones":0,"num_object_copies":10941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7201,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":2621},{"shards":"2,6","objects":-751}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":929,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.263506+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:36:32.263506+0000","last_peered":"2023-09-23T23:36:32.263506+0000","last_clean":"2023-09-23T23:36:32.263506+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:36:32.263506+0000","last_undegraded":"2023-09-23T23:36:32.263506+0000","last_fullsized":"2023-09-23T23:36:32.263506+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":237,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.272115+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:36:32.272115+0000","last_peered":"2023-09-23T23:36:32.272115+0000","last_clean":"2023-09-23T23:36:32.272115+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:36:32.272115+0000","last_undegraded":"2023-09-23T23:36:32.272115+0000","last_fullsized":"2023-09-23T23:36:32.272115+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":226,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.211233+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:36:23.211233+0000","last_peered":"2023-09-23T23:36:23.211233+0000","last_clean":"2023-09-23T23:36:23.211233+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:36:23.211233+0000","last_undegraded":"2023-09-23T23:36:23.211233+0000","last_fullsized":"2023-09-23T23:36:23.211233+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"595'7354","reported_seq":7432,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:39.536523+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:36:39.536523+0000","last_peered":"2023-09-23T23:36:39.536523+0000","last_clean":"2023-09-23T23:36:39.536523+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:36:39.536523+0000","last_undegraded":"2023-09-23T23:36:39.536523+0000","last_fullsized":"2023-09-23T23:36:39.536523+0000","mapping_epoch":566,"log_start":"544'1200","ondisk_log_start":"544'1200","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6154,"log_dups_size":0,"ondisk_log_size":6154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241041408,"num_objects":3678,"num_object_clones":0,"num_object_copies":11034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7354,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1008,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.274883+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:36:32.274883+0000","last_peered":"2023-09-23T23:36:32.274883+0000","last_clean":"2023-09-23T23:36:32.274883+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:36:32.274883+0000","last_undegraded":"2023-09-23T23:36:32.274883+0000","last_fullsized":"2023-09-23T23:36:32.274883+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":227,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.272207+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:36:32.272207+0000","last_peered":"2023-09-23T23:36:32.272207+0000","last_clean":"2023-09-23T23:36:32.272207+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:36:32.272207+0000","last_undegraded":"2023-09-23T23:36:32.272207+0000","last_fullsized":"2023-09-23T23:36:32.272207+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":447,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.275713+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:36:32.275713+0000","last_peered":"2023-09-23T23:36:32.275713+0000","last_clean":"2023-09-23T23:36:32.275713+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:36:32.275713+0000","last_undegraded":"2023-09-23T23:36:32.275713+0000","last_fullsized":"2023-09-23T23:36:32.275713+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"596'7458","reported_seq":12440,"reported_epoch":596,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:36:42.424928+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:36:42.424928+0000","last_peered":"2023-09-23T23:36:42.424928+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:36:42.424928+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:36:42.424928+0000","mapping_epoch":571,"log_start":"549'2100","ondisk_log_start":"549'2100","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5358,"log_dups_size":0,"ondisk_log_size":5358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235274240,"num_objects":3590,"num_object_clones":0,"num_object_copies":10770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2695,"num_objects_misplaced":895,"num_objects_unfound":0,"num_objects_dirty":3590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7458,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2436,"num_bytes_recovered":159645696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":2573},{"shards":"3,5,7","objects":1017}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":905,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.218097+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:36:23.218097+0000","last_peered":"2023-09-23T23:36:23.218097+0000","last_clean":"2023-09-23T23:36:23.218097+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:36:23.218097+0000","last_undegraded":"2023-09-23T23:36:23.218097+0000","last_fullsized":"2023-09-23T23:36:23.218097+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":333,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.211819+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:36:23.211819+0000","last_peered":"2023-09-23T23:36:23.211819+0000","last_clean":"2023-09-23T23:36:23.211819+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:36:23.211819+0000","last_undegraded":"2023-09-23T23:36:23.211819+0000","last_fullsized":"2023-09-23T23:36:23.211819+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":109,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.217180+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:36:23.217180+0000","last_peered":"2023-09-23T23:36:23.217180+0000","last_clean":"2023-09-23T23:36:23.217180+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:36:23.217180+0000","last_undegraded":"2023-09-23T23:36:23.217180+0000","last_fullsized":"2023-09-23T23:36:23.217180+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"596'7430","reported_seq":9495,"reported_epoch":596,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:42.509965+0000","last_change":"2023-09-23T23:34:35.750428+0000","last_active":"2023-09-23T23:36:42.509965+0000","last_peered":"2023-09-23T23:36:42.509965+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:36:42.509965+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"550'2100","ondisk_log_start":"550'2100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5330,"log_dups_size":0,"ondisk_log_size":5330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239861760,"num_objects":3660,"num_object_clones":0,"num_object_copies":10980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7430,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":991,"num_bytes_recovered":64946176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":2650},{"shards":"3,5","objects":116}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":503,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:37.588840+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:36:37.588840+0000","last_peered":"2023-09-23T23:36:37.588840+0000","last_clean":"2023-09-23T23:36:37.588840+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:36:37.588840+0000","last_undegraded":"2023-09-23T23:36:37.588840+0000","last_fullsized":"2023-09-23T23:36:37.588840+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":436,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.275031+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:36:32.275031+0000","last_peered":"2023-09-23T23:36:32.275031+0000","last_clean":"2023-09-23T23:36:32.275031+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:36:32.275031+0000","last_undegraded":"2023-09-23T23:36:32.275031+0000","last_fullsized":"2023-09-23T23:36:32.275031+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":592,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.210522+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:36:23.210522+0000","last_peered":"2023-09-23T23:36:23.210522+0000","last_clean":"2023-09-23T23:36:23.210522+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:36:23.210522+0000","last_undegraded":"2023-09-23T23:36:23.210522+0000","last_fullsized":"2023-09-23T23:36:23.210522+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"595'7509","reported_seq":8832,"reported_epoch":595,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:39.320046+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:36:39.320046+0000","last_peered":"2023-09-23T23:36:39.320046+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:36:39.320046+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"549'1800","ondisk_log_start":"549'1800","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5709,"log_dups_size":0,"ondisk_log_size":5709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239403008,"num_objects":3653,"num_object_clones":0,"num_object_copies":10959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7509,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":2607},{"shards":"3,5","objects":-1807},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":724,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.540737+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:36:32.540737+0000","last_peered":"2023-09-23T23:36:32.540737+0000","last_clean":"2023-09-23T23:36:32.540737+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:36:32.540737+0000","last_undegraded":"2023-09-23T23:36:32.540737+0000","last_fullsized":"2023-09-23T23:36:32.540737+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":606,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.282063+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:36:32.282063+0000","last_peered":"2023-09-23T23:36:32.282063+0000","last_clean":"2023-09-23T23:36:32.282063+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:36:32.282063+0000","last_undegraded":"2023-09-23T23:36:32.282063+0000","last_fullsized":"2023-09-23T23:36:32.282063+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":517,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.274957+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:36:32.274957+0000","last_peered":"2023-09-23T23:36:32.274957+0000","last_clean":"2023-09-23T23:36:32.274957+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:36:32.274957+0000","last_undegraded":"2023-09-23T23:36:32.274957+0000","last_fullsized":"2023-09-23T23:36:32.274957+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"595'7461","reported_seq":7530,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:39.101233+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:36:39.101233+0000","last_peered":"2023-09-23T23:36:39.101233+0000","last_clean":"2023-09-23T23:36:39.101233+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:36:39.101233+0000","last_undegraded":"2023-09-23T23:36:39.101233+0000","last_fullsized":"2023-09-23T23:36:39.101233+0000","mapping_epoch":534,"log_start":"549'2100","ondisk_log_start":"549'2100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5361,"log_dups_size":0,"ondisk_log_size":5361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237174784,"num_objects":3619,"num_object_clones":0,"num_object_copies":10857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7461,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":878,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.225059+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:36:23.225059+0000","last_peered":"2023-09-23T23:36:23.225059+0000","last_clean":"2023-09-23T23:36:23.225059+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:36:23.225059+0000","last_undegraded":"2023-09-23T23:36:23.225059+0000","last_fullsized":"2023-09-23T23:36:23.225059+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":119,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.217057+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:36:23.217057+0000","last_peered":"2023-09-23T23:36:23.217057+0000","last_clean":"2023-09-23T23:36:23.217057+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:36:23.217057+0000","last_undegraded":"2023-09-23T23:36:23.217057+0000","last_fullsized":"2023-09-23T23:36:23.217057+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":461,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.224816+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:36:23.224816+0000","last_peered":"2023-09-23T23:36:23.224816+0000","last_clean":"2023-09-23T23:36:23.224816+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:36:23.224816+0000","last_undegraded":"2023-09-23T23:36:23.224816+0000","last_fullsized":"2023-09-23T23:36:23.224816+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"595'7472","reported_seq":11401,"reported_epoch":595,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:36:39.111086+0000","last_change":"2023-09-23T23:34:35.785270+0000","last_active":"2023-09-23T23:36:39.111086+0000","last_peered":"2023-09-23T23:36:39.111086+0000","last_clean":"2023-09-23T23:33:50.504296+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:36:39.111086+0000","last_undegraded":"2023-09-23T23:34:35.780981+0000","last_fullsized":"2023-09-23T23:36:39.111086+0000","mapping_epoch":567,"log_start":"549'2100","ondisk_log_start":"549'2100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5372,"log_dups_size":0,"ondisk_log_size":5372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238026752,"num_objects":3632,"num_object_clones":0,"num_object_copies":10896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2627,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7472,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1920,"num_bytes_recovered":125829120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":2627},{"shards":"5,6,7","objects":1005}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":603,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.211790+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:36:23.211790+0000","last_peered":"2023-09-23T23:36:23.211790+0000","last_clean":"2023-09-23T23:36:23.211790+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:36:23.211790+0000","last_undegraded":"2023-09-23T23:36:23.211790+0000","last_fullsized":"2023-09-23T23:36:23.211790+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":458,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.225432+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:36:23.225432+0000","last_peered":"2023-09-23T23:36:23.225432+0000","last_clean":"2023-09-23T23:36:23.225432+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:36:23.225432+0000","last_undegraded":"2023-09-23T23:36:23.225432+0000","last_fullsized":"2023-09-23T23:36:23.225432+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":546,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.216904+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:36:23.216904+0000","last_peered":"2023-09-23T23:36:23.216904+0000","last_clean":"2023-09-23T23:36:23.216904+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:36:23.216904+0000","last_undegraded":"2023-09-23T23:36:23.216904+0000","last_fullsized":"2023-09-23T23:36:23.216904+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"595'7390","reported_seq":16422,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:39.352402+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:36:39.352402+0000","last_peered":"2023-09-23T23:36:39.352402+0000","last_clean":"2023-09-23T23:36:39.352402+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:36:39.352402+0000","last_undegraded":"2023-09-23T23:36:39.352402+0000","last_fullsized":"2023-09-23T23:36:39.352402+0000","mapping_epoch":579,"log_start":"548'1700","ondisk_log_start":"548'1700","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5690,"log_dups_size":0,"ondisk_log_size":5690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237633536,"num_objects":3626,"num_object_clones":0,"num_object_copies":10878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7390,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":937,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.264607+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:36:32.264607+0000","last_peered":"2023-09-23T23:36:32.264607+0000","last_clean":"2023-09-23T23:36:32.264607+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:36:32.264607+0000","last_undegraded":"2023-09-23T23:36:32.264607+0000","last_fullsized":"2023-09-23T23:36:32.264607+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":237,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.274685+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:36:32.274685+0000","last_peered":"2023-09-23T23:36:32.274685+0000","last_clean":"2023-09-23T23:36:32.274685+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:36:32.274685+0000","last_undegraded":"2023-09-23T23:36:32.274685+0000","last_fullsized":"2023-09-23T23:36:32.274685+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":526,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.275644+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:36:32.275644+0000","last_peered":"2023-09-23T23:36:32.275644+0000","last_clean":"2023-09-23T23:36:32.275644+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:36:32.275644+0000","last_undegraded":"2023-09-23T23:36:32.275644+0000","last_fullsized":"2023-09-23T23:36:32.275644+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"596'7143","reported_seq":7215,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.527950+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:36:42.527950+0000","last_peered":"2023-09-23T23:36:42.527950+0000","last_clean":"2023-09-23T23:36:42.527950+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:36:42.527950+0000","last_undegraded":"2023-09-23T23:36:42.527950+0000","last_fullsized":"2023-09-23T23:36:42.527950+0000","mapping_epoch":534,"log_start":"549'1800","ondisk_log_start":"549'1800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5343,"log_dups_size":0,"ondisk_log_size":5343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234946560,"num_objects":3585,"num_object_clones":0,"num_object_copies":10755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7143,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":916,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.217339+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:36:23.217339+0000","last_peered":"2023-09-23T23:36:23.217339+0000","last_clean":"2023-09-23T23:36:23.217339+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:36:23.217339+0000","last_undegraded":"2023-09-23T23:36:23.217339+0000","last_fullsized":"2023-09-23T23:36:23.217339+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":304,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.219610+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:36:23.219610+0000","last_peered":"2023-09-23T23:36:23.219610+0000","last_clean":"2023-09-23T23:36:23.219610+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:36:23.219610+0000","last_undegraded":"2023-09-23T23:36:23.219610+0000","last_fullsized":"2023-09-23T23:36:23.219610+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":610,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.211963+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:36:23.211963+0000","last_peered":"2023-09-23T23:36:23.211963+0000","last_clean":"2023-09-23T23:36:23.211963+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:36:23.211963+0000","last_undegraded":"2023-09-23T23:36:23.211963+0000","last_fullsized":"2023-09-23T23:36:23.211963+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":191,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.211271+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:36:23.211271+0000","last_peered":"2023-09-23T23:36:23.211271+0000","last_clean":"2023-09-23T23:36:23.211271+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:36:23.211271+0000","last_undegraded":"2023-09-23T23:36:23.211271+0000","last_fullsized":"2023-09-23T23:36:23.211271+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"596'7527","reported_seq":11518,"reported_epoch":596,"state":"active+recovering","last_fresh":"2023-09-23T23:36:42.585834+0000","last_change":"2023-09-23T23:34:35.815566+0000","last_active":"2023-09-23T23:36:42.585834+0000","last_peered":"2023-09-23T23:36:42.585834+0000","last_clean":"2023-09-23T23:33:50.519308+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:36:42.585834+0000","last_undegraded":"2023-09-23T23:36:42.585834+0000","last_fullsized":"2023-09-23T23:36:42.585834+0000","mapping_epoch":566,"log_start":"550'2222","ondisk_log_start":"550'2222","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5305,"log_dups_size":0,"ondisk_log_size":5305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238551064,"num_objects":3641,"num_object_clones":0,"num_object_copies":10923,"num_objects_missing_on_primary":2559,"num_objects_missing":2559,"num_objects_degraded":0,"num_objects_misplaced":2559,"num_objects_unfound":0,"num_objects_dirty":3641,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7527,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1942,"num_bytes_recovered":127270912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,3,4,7","objects":14},{"shards":"1,3,7","objects":1068},{"shards":"1,4,7","objects":2559}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":678,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.209248+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:36:23.209248+0000","last_peered":"2023-09-23T23:36:23.209248+0000","last_clean":"2023-09-23T23:36:23.209248+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:36:23.209248+0000","last_undegraded":"2023-09-23T23:36:23.209248+0000","last_fullsized":"2023-09-23T23:36:23.209248+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":243,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.216976+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:36:23.216976+0000","last_peered":"2023-09-23T23:36:23.216976+0000","last_clean":"2023-09-23T23:36:23.216976+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:36:23.216976+0000","last_undegraded":"2023-09-23T23:36:23.216976+0000","last_fullsized":"2023-09-23T23:36:23.216976+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":527,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.217436+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:36:23.217436+0000","last_peered":"2023-09-23T23:36:23.217436+0000","last_clean":"2023-09-23T23:36:23.217436+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:36:23.217436+0000","last_undegraded":"2023-09-23T23:36:23.217436+0000","last_fullsized":"2023-09-23T23:36:23.217436+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":228,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.225215+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:36:23.225215+0000","last_peered":"2023-09-23T23:36:23.225215+0000","last_clean":"2023-09-23T23:36:23.225215+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:36:23.225215+0000","last_undegraded":"2023-09-23T23:36:23.225215+0000","last_fullsized":"2023-09-23T23:36:23.225215+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"595'7162","reported_seq":9210,"reported_epoch":595,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:39.267033+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:36:39.267033+0000","last_peered":"2023-09-23T23:36:39.267033+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:36:39.267033+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"548'1500","ondisk_log_start":"548'1500","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5662,"log_dups_size":0,"ondisk_log_size":5662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236584960,"num_objects":3610,"num_object_clones":0,"num_object_copies":10830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7162,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":2599},{"shards":"5,7","objects":218}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":896,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.225490+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:36:23.225490+0000","last_peered":"2023-09-23T23:36:23.225490+0000","last_clean":"2023-09-23T23:36:23.225490+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:36:23.225490+0000","last_undegraded":"2023-09-23T23:36:23.225490+0000","last_fullsized":"2023-09-23T23:36:23.225490+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":333,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.225386+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:36:23.225386+0000","last_peered":"2023-09-23T23:36:23.225386+0000","last_clean":"2023-09-23T23:36:23.225386+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:36:23.225386+0000","last_undegraded":"2023-09-23T23:36:23.225386+0000","last_fullsized":"2023-09-23T23:36:23.225386+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":243,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.224517+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:36:23.224517+0000","last_peered":"2023-09-23T23:36:23.224517+0000","last_clean":"2023-09-23T23:36:23.224517+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:36:23.224517+0000","last_undegraded":"2023-09-23T23:36:23.224517+0000","last_fullsized":"2023-09-23T23:36:23.224517+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":207,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.218448+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:36:23.218448+0000","last_peered":"2023-09-23T23:36:23.218448+0000","last_clean":"2023-09-23T23:36:23.218448+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:36:23.218448+0000","last_undegraded":"2023-09-23T23:36:23.218448+0000","last_fullsized":"2023-09-23T23:36:23.218448+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"595'7305","reported_seq":7384,"reported_epoch":595,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:36:39.050778+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:36:39.050778+0000","last_peered":"2023-09-23T23:36:39.050778+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:36:39.050778+0000","last_undegraded":"2023-09-23T23:36:39.050778+0000","last_fullsized":"2023-09-23T23:36:39.050778+0000","mapping_epoch":562,"log_start":"548'1600","ondisk_log_start":"548'1600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5705,"log_dups_size":0,"ondisk_log_size":5705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235732992,"num_objects":3597,"num_object_clones":0,"num_object_copies":10791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7305,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":3597}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":610,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.224784+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:36:23.224784+0000","last_peered":"2023-09-23T23:36:23.224784+0000","last_clean":"2023-09-23T23:36:23.224784+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:36:23.224784+0000","last_undegraded":"2023-09-23T23:36:23.224784+0000","last_fullsized":"2023-09-23T23:36:23.224784+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":881,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.216684+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:36:23.216684+0000","last_peered":"2023-09-23T23:36:23.216684+0000","last_clean":"2023-09-23T23:36:23.216684+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:36:23.216684+0000","last_undegraded":"2023-09-23T23:36:23.216684+0000","last_fullsized":"2023-09-23T23:36:23.216684+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":558,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.276379+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:36:32.276379+0000","last_peered":"2023-09-23T23:36:32.276379+0000","last_clean":"2023-09-23T23:36:32.276379+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:36:32.276379+0000","last_undegraded":"2023-09-23T23:36:32.276379+0000","last_fullsized":"2023-09-23T23:36:32.276379+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":119,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.217451+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:36:23.217451+0000","last_peered":"2023-09-23T23:36:23.217451+0000","last_clean":"2023-09-23T23:36:23.217451+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:36:23.217451+0000","last_undegraded":"2023-09-23T23:36:23.217451+0000","last_fullsized":"2023-09-23T23:36:23.217451+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"595'7194","reported_seq":7263,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:39.057522+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:36:39.057522+0000","last_peered":"2023-09-23T23:36:39.057522+0000","last_clean":"2023-09-23T23:36:39.057522+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:36:39.057522+0000","last_undegraded":"2023-09-23T23:36:39.057522+0000","last_fullsized":"2023-09-23T23:36:39.057522+0000","mapping_epoch":534,"log_start":"549'1800","ondisk_log_start":"549'1800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5394,"log_dups_size":0,"ondisk_log_size":5394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232128512,"num_objects":3542,"num_object_clones":0,"num_object_copies":10626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7194,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":707,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.217282+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:36:23.217282+0000","last_peered":"2023-09-23T23:36:23.217282+0000","last_clean":"2023-09-23T23:36:23.217282+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:36:23.217282+0000","last_undegraded":"2023-09-23T23:36:23.217282+0000","last_fullsized":"2023-09-23T23:36:23.217282+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":523,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.271534+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:36:32.271534+0000","last_peered":"2023-09-23T23:36:32.271534+0000","last_clean":"2023-09-23T23:36:32.271534+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:36:32.271534+0000","last_undegraded":"2023-09-23T23:36:32.271534+0000","last_fullsized":"2023-09-23T23:36:32.271534+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":437,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.275800+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:36:32.275800+0000","last_peered":"2023-09-23T23:36:32.275800+0000","last_clean":"2023-09-23T23:36:32.275800+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:36:32.275800+0000","last_undegraded":"2023-09-23T23:36:32.275800+0000","last_fullsized":"2023-09-23T23:36:32.275800+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":35,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.209902+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:36:23.209902+0000","last_peered":"2023-09-23T23:36:23.209902+0000","last_clean":"2023-09-23T23:36:23.209902+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:36:23.209902+0000","last_undegraded":"2023-09-23T23:36:23.209902+0000","last_fullsized":"2023-09-23T23:36:23.209902+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"595'7401","reported_seq":11069,"reported_epoch":595,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:39.839267+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:36:39.839267+0000","last_peered":"2023-09-23T23:36:39.839267+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:36:39.839267+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"544'1200","ondisk_log_start":"544'1200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6201,"log_dups_size":0,"ondisk_log_size":6201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235732992,"num_objects":3597,"num_object_clones":0,"num_object_copies":10791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7401,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":893},{"shards":"1,2,4","objects":2595}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":590,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.217556+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:36:23.217556+0000","last_peered":"2023-09-23T23:36:23.217556+0000","last_clean":"2023-09-23T23:36:23.217556+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:36:23.217556+0000","last_undegraded":"2023-09-23T23:36:23.217556+0000","last_fullsized":"2023-09-23T23:36:23.217556+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":36,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.308732+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:36:32.308732+0000","last_peered":"2023-09-23T23:36:32.308732+0000","last_clean":"2023-09-23T23:36:32.308732+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:36:32.308732+0000","last_undegraded":"2023-09-23T23:36:32.308732+0000","last_fullsized":"2023-09-23T23:36:32.308732+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":135,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.225259+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:36:23.225259+0000","last_peered":"2023-09-23T23:36:23.225259+0000","last_clean":"2023-09-23T23:36:23.225259+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:36:23.225259+0000","last_undegraded":"2023-09-23T23:36:23.225259+0000","last_fullsized":"2023-09-23T23:36:23.225259+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":328,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.212104+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:36:23.212104+0000","last_peered":"2023-09-23T23:36:23.212104+0000","last_clean":"2023-09-23T23:36:23.212104+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:36:23.212104+0000","last_undegraded":"2023-09-23T23:36:23.212104+0000","last_fullsized":"2023-09-23T23:36:23.212104+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"595'7343","reported_seq":7412,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:39.324167+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:36:39.324167+0000","last_peered":"2023-09-23T23:36:39.324167+0000","last_clean":"2023-09-23T23:36:39.324167+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:36:39.324167+0000","last_undegraded":"2023-09-23T23:36:39.324167+0000","last_fullsized":"2023-09-23T23:36:39.324167+0000","mapping_epoch":534,"log_start":"548'1600","ondisk_log_start":"548'1600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5743,"log_dups_size":0,"ondisk_log_size":5743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237305856,"num_objects":3621,"num_object_clones":0,"num_object_copies":10863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7343,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":776,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.224893+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:36:23.224893+0000","last_peered":"2023-09-23T23:36:23.224893+0000","last_clean":"2023-09-23T23:36:23.224893+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:36:23.224893+0000","last_undegraded":"2023-09-23T23:36:23.224893+0000","last_fullsized":"2023-09-23T23:36:23.224893+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":593,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.275574+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:36:32.275574+0000","last_peered":"2023-09-23T23:36:32.275574+0000","last_clean":"2023-09-23T23:36:32.275574+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:36:32.275574+0000","last_undegraded":"2023-09-23T23:36:32.275574+0000","last_fullsized":"2023-09-23T23:36:32.275574+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":199,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.217117+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:36:23.217117+0000","last_peered":"2023-09-23T23:36:23.217117+0000","last_clean":"2023-09-23T23:36:23.217117+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:36:23.217117+0000","last_undegraded":"2023-09-23T23:36:23.217117+0000","last_fullsized":"2023-09-23T23:36:23.217117+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":146,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.224419+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:36:23.224419+0000","last_peered":"2023-09-23T23:36:23.224419+0000","last_clean":"2023-09-23T23:36:23.224419+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:36:23.224419+0000","last_undegraded":"2023-09-23T23:36:23.224419+0000","last_fullsized":"2023-09-23T23:36:23.224419+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"595'7290","reported_seq":7361,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:39.872453+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:36:39.872453+0000","last_peered":"2023-09-23T23:36:39.872453+0000","last_clean":"2023-09-23T23:36:39.872453+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:36:39.872453+0000","last_undegraded":"2023-09-23T23:36:39.872453+0000","last_fullsized":"2023-09-23T23:36:39.872453+0000","mapping_epoch":534,"log_start":"544'1100","ondisk_log_start":"544'1100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6190,"log_dups_size":0,"ondisk_log_size":6190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237109248,"num_objects":3618,"num_object_clones":0,"num_object_copies":10854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7290,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":983,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.275265+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:36:32.275265+0000","last_peered":"2023-09-23T23:36:32.275265+0000","last_clean":"2023-09-23T23:36:32.275265+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:36:32.275265+0000","last_undegraded":"2023-09-23T23:36:32.275265+0000","last_fullsized":"2023-09-23T23:36:32.275265+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":109,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.217455+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:36:23.217455+0000","last_peered":"2023-09-23T23:36:23.217455+0000","last_clean":"2023-09-23T23:36:23.217455+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:36:23.217455+0000","last_undegraded":"2023-09-23T23:36:23.217455+0000","last_fullsized":"2023-09-23T23:36:23.217455+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":39,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.209983+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:36:23.209983+0000","last_peered":"2023-09-23T23:36:23.209983+0000","last_clean":"2023-09-23T23:36:23.209983+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:36:23.209983+0000","last_undegraded":"2023-09-23T23:36:23.209983+0000","last_fullsized":"2023-09-23T23:36:23.209983+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":169,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.211240+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:36:23.211240+0000","last_peered":"2023-09-23T23:36:23.211240+0000","last_clean":"2023-09-23T23:36:23.211240+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:36:23.211240+0000","last_undegraded":"2023-09-23T23:36:23.211240+0000","last_fullsized":"2023-09-23T23:36:23.211240+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1032,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.275924+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:36:32.275924+0000","last_peered":"2023-09-23T23:36:32.275924+0000","last_clean":"2023-09-23T23:36:32.275924+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:36:32.275924+0000","last_undegraded":"2023-09-23T23:36:32.275924+0000","last_fullsized":"2023-09-23T23:36:32.275924+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":35,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.210578+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:36:23.210578+0000","last_peered":"2023-09-23T23:36:23.210578+0000","last_clean":"2023-09-23T23:36:23.210578+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:36:23.210578+0000","last_undegraded":"2023-09-23T23:36:23.210578+0000","last_fullsized":"2023-09-23T23:36:23.210578+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":350,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.225300+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:36:23.225300+0000","last_peered":"2023-09-23T23:36:23.225300+0000","last_clean":"2023-09-23T23:36:23.225300+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:36:23.225300+0000","last_undegraded":"2023-09-23T23:36:23.225300+0000","last_fullsized":"2023-09-23T23:36:23.225300+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":744,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.275460+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:36:32.275460+0000","last_peered":"2023-09-23T23:36:32.275460+0000","last_clean":"2023-09-23T23:36:32.275460+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:36:32.275460+0000","last_undegraded":"2023-09-23T23:36:32.275460+0000","last_fullsized":"2023-09-23T23:36:32.275460+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":97,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.216947+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:36:23.216947+0000","last_peered":"2023-09-23T23:36:23.216947+0000","last_clean":"2023-09-23T23:36:23.216947+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:36:23.216947+0000","last_undegraded":"2023-09-23T23:36:23.216947+0000","last_fullsized":"2023-09-23T23:36:23.216947+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":444,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.281966+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:36:32.281966+0000","last_peered":"2023-09-23T23:36:32.281966+0000","last_clean":"2023-09-23T23:36:32.281966+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:36:32.281966+0000","last_undegraded":"2023-09-23T23:36:32.281966+0000","last_fullsized":"2023-09-23T23:36:32.281966+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":293,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.217305+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:36:23.217305+0000","last_peered":"2023-09-23T23:36:23.217305+0000","last_clean":"2023-09-23T23:36:23.217305+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:36:23.217305+0000","last_undegraded":"2023-09-23T23:36:23.217305+0000","last_fullsized":"2023-09-23T23:36:23.217305+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":283,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.276446+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:36:32.276446+0000","last_peered":"2023-09-23T23:36:32.276446+0000","last_clean":"2023-09-23T23:36:32.276446+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:36:32.276446+0000","last_undegraded":"2023-09-23T23:36:32.276446+0000","last_fullsized":"2023-09-23T23:36:32.276446+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":807,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.217118+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:36:23.217118+0000","last_peered":"2023-09-23T23:36:23.217118+0000","last_clean":"2023-09-23T23:36:23.217118+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:36:23.217118+0000","last_undegraded":"2023-09-23T23:36:23.217118+0000","last_fullsized":"2023-09-23T23:36:23.217118+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":626,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.275371+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:36:32.275371+0000","last_peered":"2023-09-23T23:36:32.275371+0000","last_clean":"2023-09-23T23:36:32.275371+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:36:32.275371+0000","last_undegraded":"2023-09-23T23:36:32.275371+0000","last_fullsized":"2023-09-23T23:36:32.275371+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":610,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.276175+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:36:32.276175+0000","last_peered":"2023-09-23T23:36:32.276175+0000","last_clean":"2023-09-23T23:36:32.276175+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:36:32.276175+0000","last_undegraded":"2023-09-23T23:36:32.276175+0000","last_fullsized":"2023-09-23T23:36:32.276175+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":769,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:37.573849+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:36:37.573849+0000","last_peered":"2023-09-23T23:36:37.573849+0000","last_clean":"2023-09-23T23:36:37.573849+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:36:37.573849+0000","last_undegraded":"2023-09-23T23:36:37.573849+0000","last_fullsized":"2023-09-23T23:36:37.573849+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":438,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.224991+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:36:23.224991+0000","last_peered":"2023-09-23T23:36:23.224991+0000","last_clean":"2023-09-23T23:36:23.224991+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:36:23.224991+0000","last_undegraded":"2023-09-23T23:36:23.224991+0000","last_fullsized":"2023-09-23T23:36:23.224991+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":662,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T23:36:23.216596+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:36:23.216596+0000","last_peered":"2023-09-23T23:36:23.216596+0000","last_clean":"2023-09-23T23:36:23.216596+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:36:23.216596+0000","last_undegraded":"2023-09-23T23:36:23.216596+0000","last_fullsized":"2023-09-23T23:36:23.216596+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":3795517464,"num_objects":57916,"num_object_clones":0,"num_object_copies":173748,"num_objects_missing_on_primary":2559,"num_objects_missing":2559,"num_objects_degraded":24938,"num_objects_misplaced":3454,"num_objects_unfound":0,"num_objects_dirty":57916,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":117640,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15146,"num_bytes_recovered":992411648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12455522304,"data_stored":12455510088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89918,"ondisk_log_size":89918,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782687,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1725248,"kb_used_data":1345336,"kb_used_omap":8,"kb_used_meta":379895,"kb_avail":92019904,"statfs":{"total":95995035648,"available":94228381696,"internally_reserved":0,"allocated":1377624064,"data_stored":1375965814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":389012549},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":1.5569999999999999,"15min":2.1800000000000002},"min":{"1min":0.23100000000000001,"5min":0.224,"15min":0.20899999999999999},"max":{"1min":0.61699999999999999,"5min":36.956000000000003,"15min":85.721000000000004},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.47399999999999998,"5min":1.522,"15min":2.2250000000000001},"min":{"1min":0.29199999999999998,"5min":0.27900000000000003,"15min":0.19800000000000001},"max":{"1min":1.4419999999999999,"5min":37.247999999999998,"15min":85.965000000000003},"last":0.42399999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":3.8900000000000001,"15min":3.169},"min":{"1min":0.26600000000000001,"5min":0.248,"15min":0.22900000000000001},"max":{"1min":3.3500000000000001,"5min":129.11199999999999,"15min":198.58000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.65600000000000003,"5min":3.665,"15min":3.206},"min":{"1min":0.29699999999999999,"5min":0.247,"15min":0.246},"max":{"1min":3.7290000000000001,"5min":129.02600000000001,"15min":198.71100000000001},"last":0.45500000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":2.871,"15min":3.7010000000000001},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.452,"5min":78.528999999999996,"15min":203.58000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.54200000000000004,"5min":1.952,"15min":3.0649999999999999},"min":{"1min":0.30399999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.2529999999999999,"5min":27.574000000000002,"15min":198.69},"last":0.41499999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":3.3290000000000002,"15min":3.5950000000000002},"min":{"1min":0.31900000000000001,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":1.591,"5min":125.511,"15min":279.26499999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.67500000000000004,"5min":5.3090000000000002,"15min":4.1210000000000004},"min":{"1min":0.374,"5min":0.32000000000000001,"15min":0.29599999999999999},"max":{"1min":3.6240000000000001,"5min":150.608,"15min":279.29899999999998},"last":0.438}]},{"osd":4,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":1.694,"15min":1.0669999999999999},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":0.68100000000000005,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.38},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.363,"15min":1.0029999999999999},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.23899999999999999},"max":{"1min":1.6080000000000001,"5min":31.68,"15min":31.68},"last":0.38900000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":3.637,"5min":1.657,"15min":1.899},"min":{"1min":0.31900000000000001,"5min":0.28699999999999998,"15min":0.26500000000000001},"max":{"1min":60.463999999999999,"5min":60.463999999999999,"15min":210.83600000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":3.5990000000000002,"5min":1.6160000000000001,"15min":1.7649999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28199999999999997},"max":{"1min":63.006,"5min":63.006,"15min":207.04599999999999},"last":0.45000000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":1.879,"15min":1.3480000000000001},"min":{"1min":0.35199999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.4419999999999999,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":2.2330000000000001,"15min":1.581},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":2.3180000000000001,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.30599999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496163,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2008740,"kb_used_data":1297244,"kb_used_omap":0,"kb_used_meta":711488,"kb_avail":91736412,"statfs":{"total":95995035648,"available":93938085888,"internally_reserved":0,"allocated":1328377856,"data_stored":1326739462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":728563712},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.93500000000000005,"15min":4.6799999999999997},"min":{"1min":0.27700000000000002,"5min":0.251,"15min":0.20399999999999999},"max":{"1min":2.8450000000000002,"5min":8.8119999999999994,"15min":135.31999999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.73399999999999999,"15min":4.601},"min":{"1min":0.24399999999999999,"5min":0.23999999999999999,"15min":0.22900000000000001},"max":{"1min":1.8859999999999999,"5min":3.8239999999999998,"15min":135.34299999999999},"last":0.315}]},{"osd":1,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":1.7509999999999999,"15min":2.3759999999999999},"min":{"1min":0.253,"5min":0.251,"15min":0.245},"max":{"1min":1.944,"5min":65.891000000000005,"15min":89.605000000000004},"last":0.379},{"interface":"front","average":{"1min":0.64100000000000001,"5min":1.4390000000000001,"15min":2.4510000000000001},"min":{"1min":0.33400000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":2.1269999999999998,"5min":66.087000000000003,"15min":91.025999999999996},"last":0.39400000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":4.9459999999999997,"15min":4.5620000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.861,"5min":97.408000000000001,"15min":127.44499999999999},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.55200000000000005,"5min":6.0880000000000001,"15min":4.9740000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.2120000000000002,"5min":125.539,"15min":125.539},"last":0.44600000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:35:53 2023","interfaces":[{"interface":"back","average":{"1min":7.923,"5min":7.0129999999999999,"15min":4.4459999999999997},"min":{"1min":0.316,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":73.813999999999993,"5min":138.87299999999999,"15min":138.87299999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":8.4450000000000003,"5min":6.9260000000000002,"15min":4.7350000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":73.852999999999994,"5min":138.976,"15min":138.976},"last":0.50700000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.76400000000000001,"15min":0.68300000000000005},"min":{"1min":0.27900000000000003,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":0.57799999999999996,"5min":7.101,"15min":8.1999999999999993},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.88900000000000001,"15min":0.77600000000000002},"min":{"1min":0.32500000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.54600000000000004,"5min":8.3499999999999996,"15min":8.3499999999999996},"last":0.49199999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.82899999999999996,"15min":0.80700000000000005},"min":{"1min":0.34300000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.266,"5min":11.372,"15min":11.473000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.98999999999999999,"15min":0.83599999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26900000000000002},"max":{"1min":0.57699999999999996,"5min":7.0129999999999999,"15min":7.5670000000000002},"last":0.42999999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":1.294,"15min":1.0289999999999999},"min":{"1min":0.30499999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":0.70599999999999996,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.72999999999999998,"5min":1.4019999999999999,"15min":1.0720000000000001},"min":{"1min":0.36099999999999999,"5min":0.35499999999999998,"15min":0.33500000000000002},"max":{"1min":2.665,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.57199999999999995}]}]},{"osd":1,"up_from":112,"seq":481036337504,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2067368,"kb_used_data":1170720,"kb_used_omap":4,"kb_used_meta":896635,"kb_avail":91677784,"statfs":{"total":95995035648,"available":93878050816,"internally_reserved":0,"allocated":1198817280,"data_stored":1196007102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":918155150},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":1.881,"15min":3.4060000000000001},"min":{"1min":0.217,"5min":0.17799999999999999,"15min":0.16700000000000001},"max":{"1min":2.7970000000000002,"5min":47.195999999999998,"15min":130.768},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.40999999999999998,"5min":1.5469999999999999,"15min":2.726},"min":{"1min":0.24299999999999999,"5min":0.20499999999999999,"15min":0.19},"max":{"1min":1.016,"5min":46.582000000000001,"15min":134.929},"last":0.221}]},{"osd":2,"last update":"Sat Sep 23 23:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":7.1859999999999999,"15min":6.7910000000000004},"min":{"1min":0.26300000000000001,"5min":0.214,"15min":0.20100000000000001},"max":{"1min":1.8839999999999999,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.65800000000000003,"5min":7.6059999999999999,"15min":6.9160000000000004},"min":{"1min":0.27900000000000003,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":1.968,"5min":202.12,"15min":232.21000000000001},"last":0.32000000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":5.1989999999999998,"15min":5.1799999999999997},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.221},"max":{"1min":0.999,"5min":181.94399999999999,"15min":234.404},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.75900000000000001,"5min":5.29,"15min":5.0469999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":6.6829999999999998,"5min":181.98500000000001,"15min":236.33699999999999},"last":0.26900000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":1.23,"15min":1.423},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":2.1059999999999999,"5min":12.042999999999999,"15min":12.042999999999999},"last":0.37},{"interface":"front","average":{"1min":0.624,"5min":1.129,"15min":1.46},"min":{"1min":0.48099999999999998,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":1.2669999999999999,"5min":9.5069999999999997,"15min":18.108000000000001},"last":0.44800000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":2.9209999999999998,"15min":1.796},"min":{"1min":0.34300000000000003,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66600000000000004,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.68200000000000005,"5min":2.9940000000000002,"15min":2.113},"min":{"1min":0.40899999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.044,"5min":151.821,"15min":151.821},"last":0.46100000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.3600000000000001,"15min":1.7529999999999999},"min":{"1min":0.377,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":1.1040000000000001,"5min":8.9269999999999996,"15min":74.087999999999994},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.97699999999999998,"5min":1.585,"15min":1.968},"min":{"1min":0.42999999999999999,"5min":0.35699999999999998,"15min":0.33100000000000002},"max":{"1min":5.367,"5min":11.927,"15min":73.974999999999994},"last":0.39700000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.7390000000000001,"15min":2.6099999999999999},"min":{"1min":0.50600000000000001,"5min":0.38700000000000001,"15min":0.32900000000000001},"max":{"1min":1.3069999999999999,"5min":44.241,"15min":78.308000000000007},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.8240000000000001,"15min":2.484},"min":{"1min":0.37,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":2.1339999999999999,"5min":43.920999999999999,"15min":77.204999999999998},"last":0.42699999999999999}]}]},{"osd":0,"up_from":91,"seq":390842024328,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1513964,"kb_used_data":303716,"kb_used_omap":5,"kb_used_meta":1210234,"kb_avail":92231188,"statfs":{"total":95995035648,"available":94444736512,"internally_reserved":0,"allocated":311005184,"data_stored":308185033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1239280250},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:36:36 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":4.391,"15min":6.6539999999999999},"min":{"1min":0.219,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":1.7709999999999999,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.43099999999999999,"5min":4.1289999999999996,"15min":5.7969999999999997},"min":{"1min":0.245,"5min":0.20200000000000001,"15min":0.17599999999999999},"max":{"1min":1.0249999999999999,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.25900000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:35:57 2023","interfaces":[{"interface":"back","average":{"1min":3.2130000000000001,"5min":10.026999999999999,"15min":7.5819999999999999},"min":{"1min":0.26200000000000001,"5min":0.17699999999999999,"15min":0.17199999999999999},"max":{"1min":40.326999999999998,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":3.1720000000000002,"5min":10.18,"15min":7.6520000000000001},"min":{"1min":0.22900000000000001,"5min":0.19800000000000001,"15min":0.17599999999999999},"max":{"1min":40.396000000000001,"5min":283.161,"15min":283.161},"last":0.35299999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:36:36 2023","interfaces":[{"interface":"back","average":{"1min":5.9699999999999998,"5min":5.1630000000000003,"15min":5.6079999999999997},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":95.655000000000001,"5min":114.381,"15min":220.732},"last":0.36599999999999999},{"interface":"front","average":{"1min":5.8300000000000001,"5min":5.2779999999999996,"15min":5.6669999999999998},"min":{"1min":0.26600000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":95.695999999999998,"5min":114.30500000000001,"15min":218.78200000000001},"last":0.318}]},{"osd":4,"last update":"Sat Sep 23 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":1.1599999999999999,"15min":2.4769999999999999},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.70799999999999996,"5min":14.676,"15min":138.81299999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.80100000000000005,"5min":1.391,"15min":3.0830000000000002},"min":{"1min":0.33800000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.7469999999999999,"5min":9.9039999999999999,"15min":171.251},"last":0.41499999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":1.504,"15min":2.7930000000000001},"min":{"1min":0.35299999999999998,"5min":0.32700000000000001,"15min":0.32100000000000001},"max":{"1min":4.6820000000000004,"5min":15.076000000000001,"15min":171.30699999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.625,"5min":1.244,"15min":2.0819999999999999},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":1.8149999999999999,"5min":8.9909999999999997,"15min":139.40799999999999},"last":0.55800000000000005}]},{"osd":6,"last update":"Sat Sep 23 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.325,"15min":3.097},"min":{"1min":0.38100000000000001,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":4.718,"5min":9.968,"15min":171.33600000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.335,"15min":3.3650000000000002},"min":{"1min":0.37,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":1.8700000000000001,"5min":9.8320000000000007,"15min":184.13499999999999},"last":0.59399999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":1.373,"15min":2.6099999999999999},"min":{"1min":0.38400000000000001,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":0.751,"5min":9.452,"15min":138.77699999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.6379999999999999,"15min":2.5030000000000001},"min":{"1min":0.379,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":1.839,"5min":9.9670000000000005,"15min":139.37899999999999},"last":0.623}]}]},{"osd":2,"up_from":199,"seq":854698492149,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3177132,"kb_used_data":2320932,"kb_used_omap":7,"kb_used_meta":856184,"kb_avail":90568020,"statfs":{"total":95995035648,"available":92741652480,"internally_reserved":0,"allocated":2376634368,"data_stored":2373764265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":876732497},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":2.2080000000000002,"15min":3.8599999999999999},"min":{"1min":0.186,"5min":0.186,"15min":0.16700000000000001},"max":{"1min":4.7590000000000003,"5min":61.950000000000003,"15min":119.277},"last":6.3940000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":2.153,"15min":3.6669999999999998},"min":{"1min":0.23999999999999999,"5min":0.20399999999999999,"15min":0.18099999999999999},"max":{"1min":2.5070000000000001,"5min":61.984999999999999,"15min":119.56100000000001},"last":0.24399999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.911,"5min":5.3010000000000002,"15min":4.5940000000000003},"min":{"1min":0.25,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":8.3640000000000008,"5min":106.765,"15min":157.98699999999999},"last":4.7060000000000004},{"interface":"front","average":{"1min":0.79000000000000004,"5min":4.8959999999999999,"15min":4.577},"min":{"1min":0.253,"5min":0.222,"15min":0.222},"max":{"1min":4.0940000000000003,"5min":106.732,"15min":158.03800000000001},"last":6.4340000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":5.7800000000000002,"15min":4.7770000000000001},"min":{"1min":0.255,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":8.3070000000000004,"5min":106.70399999999999,"15min":119.366},"last":0.433},{"interface":"front","average":{"1min":0.65800000000000003,"5min":5.4169999999999998,"15min":4.2709999999999999},"min":{"1min":0.29299999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":3.375,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.27200000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":2.036,"15min":2.262},"min":{"1min":0.36099999999999999,"5min":0.33700000000000002,"15min":0.25},"max":{"1min":2.6070000000000002,"5min":62.081000000000003,"15min":119.88200000000001},"last":0.498},{"interface":"front","average":{"1min":1.4590000000000001,"5min":2.3860000000000001,"15min":1.865},"min":{"1min":0.36899999999999999,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":8.343,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.54800000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":3.0299999999999998,"15min":2.6480000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.26200000000000001},"max":{"1min":2.548,"5min":91.207999999999998,"15min":119.919},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.4550000000000001,"5min":3.2749999999999999,"15min":2.6160000000000001},"min":{"1min":0.33500000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.3889999999999993,"5min":91.150999999999996,"15min":119.306},"last":0.51700000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":3.5750000000000002,"15min":3.0019999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":2.5819999999999999,"5min":107.018,"15min":119.943},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.486,"5min":2.6440000000000001,"15min":2.1600000000000001},"min":{"1min":0.371,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":8.4060000000000006,"5min":62.493000000000002,"15min":62.493000000000002},"last":0.47699999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":2.2010000000000001,"15min":1.9179999999999999},"min":{"1min":0.40000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.381,"5min":62.603000000000002,"15min":62.603000000000002},"last":1.831},{"interface":"front","average":{"1min":0.60099999999999998,"5min":3.464,"15min":2.3620000000000001},"min":{"1min":0.42699999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.417,"5min":106.809,"15min":106.809},"last":1.792}]}]},{"osd":3,"up_from":148,"seq":635655160118,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2166752,"kb_used_data":1233112,"kb_used_omap":2,"kb_used_meta":933629,"kb_avail":91578400,"statfs":{"total":95995035648,"available":93776281600,"internally_reserved":0,"allocated":1262706688,"data_stored":1259901846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":956036149},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7390000000000001,"5min":1.4059999999999999,"15min":2.79},"min":{"1min":0.19600000000000001,"5min":0.19400000000000001,"15min":0.158},"max":{"1min":15.603,"5min":17.213000000000001,"15min":103.866},"last":0.25900000000000001},{"interface":"front","average":{"1min":1.5880000000000001,"5min":1.877,"15min":2.9049999999999998},"min":{"1min":0.223,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":12.202,"5min":22.754000000000001,"15min":101.884},"last":0.28100000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":3.2200000000000002,"15min":2.9969999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19600000000000001},"max":{"1min":7.3319999999999999,"5min":56.119,"15min":85.242999999999995},"last":0.29699999999999999},{"interface":"front","average":{"1min":1.4910000000000001,"5min":2.427,"15min":2.7999999999999998},"min":{"1min":0.22800000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":11.228,"5min":53.962000000000003,"15min":85.302000000000007},"last":0.32800000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":4.7770000000000001,"15min":4.7859999999999996},"min":{"1min":0.25800000000000001,"5min":0.25,"15min":0.19800000000000001},"max":{"1min":2.1190000000000002,"5min":73.742000000000004,"15min":101.279},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.45000000000000001,"5min":4.6349999999999998,"15min":4.6130000000000004},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.214},"max":{"1min":0.86699999999999999,"5min":76.525999999999996,"15min":101.294},"last":0.45200000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.3959999999999999,"15min":1.339},"min":{"1min":0.28799999999999998,"5min":0.28499999999999998,"15min":0.26700000000000002},"max":{"1min":11.209,"5min":11.209,"15min":11.209},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.2829999999999999,"5min":1.905,"15min":1.8049999999999999},"min":{"1min":0.33500000000000002,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":7.3940000000000001,"5min":11.494999999999999,"15min":24.782},"last":0.35299999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4410000000000001,"5min":6.8159999999999998,"15min":3.4289999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":7.3120000000000003,"5min":214.43700000000001,"15min":214.43700000000001},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.2470000000000001,"5min":6.7569999999999997,"15min":3.4900000000000002},"min":{"1min":0.34699999999999998,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":7.2290000000000001,"5min":214.46600000000001,"15min":214.46600000000001},"last":0.54800000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:35:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":2.7090000000000001,"15min":2.3159999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":9.798,"5min":58.932000000000002,"15min":58.932000000000002},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.25,"5min":2.8660000000000001,"15min":2.3170000000000002},"min":{"1min":0.36499999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.202,"5min":59.186,"15min":59.186},"last":0.50600000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":4.7480000000000002,"15min":3.2069999999999999},"min":{"1min":0.38700000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.415,"5min":192.69,"15min":192.69},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.753,"5min":5.1289999999999996,"15min":3.6949999999999998},"min":{"1min":0.43099999999999999,"5min":0.39200000000000002,"15min":0.36699999999999999},"max":{"1min":2.6560000000000001,"5min":192.65600000000001,"15min":192.65600000000001},"last":0.495}]}]},{"osd":4,"up_from":0,"seq":128849019436,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757799,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2904096,"kb_used_data":2167448,"kb_used_omap":0,"kb_used_meta":736640,"kb_avail":90841056,"statfs":{"total":95995035648,"available":93021241344,"internally_reserved":0,"allocated":2219466752,"data_stored":2217828575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":754319360},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:14 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":1.141,"15min":3.198},"min":{"1min":0.252,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":1.5640000000000001,"5min":21.109000000000002,"15min":113.869},"last":0.317},{"interface":"front","average":{"1min":0.38,"5min":1.3939999999999999,"15min":3.5710000000000002},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.22700000000000001},"max":{"1min":0.76200000000000001,"5min":20.876999999999999,"15min":138.55199999999999},"last":0.33300000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:36:20 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.6200000000000001,"15min":2.258},"min":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.23100000000000001},"max":{"1min":1.9930000000000001,"5min":29.007999999999999,"15min":69.816999999999993},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.8,"15min":2.5840000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":2.8130000000000002,"5min":29.053999999999998,"15min":70.268000000000001},"last":0.878}]},{"osd":2,"last update":"Sat Sep 23 23:35:47 2023","interfaces":[{"interface":"back","average":{"1min":7.7469999999999999,"5min":6.5350000000000001,"15min":4.609},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.26900000000000002},"max":{"1min":145.93199999999999,"5min":208.738,"15min":208.738},"last":2.5310000000000001},{"interface":"front","average":{"1min":1.0269999999999999,"5min":5.0499999999999998,"15min":4.1150000000000002},"min":{"1min":0.31,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":2.859,"5min":208.631,"15min":208.631},"last":1.0189999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:35:44 2023","interfaces":[{"interface":"back","average":{"1min":6.4889999999999999,"5min":4.1520000000000001,"15min":3.6589999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":122.071,"5min":122.071,"15min":122.071},"last":0.45700000000000002},{"interface":"front","average":{"1min":6.4669999999999996,"5min":4.4969999999999999,"15min":3.8359999999999999},"min":{"1min":0.35799999999999998,"5min":0.35399999999999998,"15min":0.318},"max":{"1min":122.09699999999999,"5min":122.09699999999999,"15min":122.09699999999999},"last":0.47199999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.98699999999999999,"15min":0.90700000000000003},"min":{"1min":0.32900000000000001,"5min":0.25700000000000001,"15min":0.251},"max":{"1min":0.749,"5min":9.2690000000000001,"15min":19.962},"last":0.434},{"interface":"front","average":{"1min":0.502,"5min":1.0129999999999999,"15min":0.81599999999999995},"min":{"1min":0.314,"5min":0.29599999999999999,"15min":0.22900000000000001},"max":{"1min":1.595,"5min":10.273999999999999,"15min":20.829999999999998},"last":0.497}]},{"osd":6,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":2.29,"15min":1.5049999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.26900000000000002},"max":{"1min":1.619,"5min":148.36600000000001,"15min":148.36600000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.54000000000000004,"5min":2.798,"15min":1.7250000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.26000000000000001},"max":{"1min":2.1549999999999998,"5min":148.42500000000001,"15min":148.42500000000001},"last":0.38}]},{"osd":7,"last update":"Sat Sep 23 23:35:47 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.3160000000000001,"15min":2.625},"min":{"1min":0.34699999999999998,"5min":0.34599999999999997,"15min":0.317},"max":{"1min":1.843,"5min":15.497,"15min":179.28800000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.3480000000000001,"15min":2.7050000000000001},"min":{"1min":0.36699999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.867,"5min":15.653,"15min":179.428},"last":0.44600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":302186496,"data_stored":302186496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1187647488,"data_stored":1187643416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2367356928,"data_stored":2367356928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1252855808,"data_stored":1252851736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2447835136,"data_stored":2447835136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2208169984,"data_stored":2208169984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1319108608,"data_stored":1319108608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1370361856,"data_stored":1370357784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:36:46.389 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:36:46.415 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:36:46.439 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:36:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:46 smithi138 ceph-mon[193606]: from='client.36855 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:46.456 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:46.472 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:36:46.498 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:46.506 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:36:46.536 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:36:46.556 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:46 smithi110 ceph-mon[212939]: from='client.36855 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:46.569 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:36:46.590 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:36:46.592 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:36:46.592 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:36:46.592 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:36:46.592 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:36:46.592 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:36:46.592 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:36:46.592 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:36:46.592 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:36:46.766 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:36:46.766 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:36:46.767 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:36:46.767 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:36:46.767 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:36:46.874 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:46.877 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:36:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:36:46] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-09-23T23:36:46.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:46.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:46.979 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:47.006 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:47.048 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:47.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:47.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:47.153 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:36:47.185 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:47.231 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:36:47.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:47.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:36:47.277 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:47 smithi138 ceph-mon[193606]: pgmap v1663: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 24872/171537 objects degraded (14.499%); 3454/171537 objects misplaced (2.014%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T23:36:47.277 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:47 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2952958980' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:47.343 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:47.374 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:47.426 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:47.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:47.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:47.531 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:47.560 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:47 smithi110 ceph-mon[212939]: pgmap v1663: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 24872/171537 objects degraded (14.499%); 3454/171537 objects misplaced (2.014%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T23:36:47.560 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:47 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2952958980' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:47.560 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:47 smithi110 ceph-mon[221102]: pgmap v1663: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 24872/171537 objects degraded (14.499%); 3454/171537 objects misplaced (2.014%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T23:36:47.561 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:47 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2952958980' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:47.563 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:47.711 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:47.936 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:47.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:47.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:48.002 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:36:48.040 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:48.081 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:48.192 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:48.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:48.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:48.298 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:36:48.324 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:48.366 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:36:48.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:48.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:36:48.472 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:48.500 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:48.549 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:48.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:48.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:48.656 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:48.682 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:48.723 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:48.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:48.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:48.828 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:48.860 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:49.106 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:49.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:49.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:49.171 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:49.226 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:49.260 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:49.319 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:49.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:49.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:49.376 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:36:49.401 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:49 smithi110 ceph-mon[212939]: pgmap v1664: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 24801/170577 objects degraded (14.539%); 3373/170577 objects misplaced (1.977%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:49.402 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:49 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 24801/170577 objects degraded (14.539%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:49.402 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:49 smithi110 ceph-mon[221102]: pgmap v1664: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 24801/170577 objects degraded (14.539%); 3373/170577 objects misplaced (1.977%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:49.402 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:49 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 24801/170577 objects degraded (14.539%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:49.422 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:36:49.452 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:49.510 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:36:49.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:49.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:36:49.617 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:49.641 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:49 smithi138 ceph-mon[193606]: pgmap v1664: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 24801/170577 objects degraded (14.539%); 3373/170577 objects misplaced (1.977%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:49.641 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:49 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 24801/170577 objects degraded (14.539%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:49.643 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:49.685 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:49.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:49.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:49.742 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:36:49.790 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:36:49.825 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:50.288 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:36:50.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:50.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:36:50.294 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:36:50.394 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:36:50.421 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:50.471 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:36:50.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:50.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:36:50.578 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:50.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:50 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3137137427' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:50.608 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:50.642 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:50 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3137137427' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:50.643 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:50 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3137137427' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:36:50.660 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:50.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:50.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:50.767 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:50.794 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:50.841 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:50.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:50.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:50.949 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:50.976 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:51.079 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:51.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:51.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:51.184 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:51.188 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:51.222 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:51.344 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:51.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:51.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:51.451 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:51.477 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:51.520 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:51.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:51.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:51.553 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:51 smithi110 ceph-mon[212939]: pgmap v1665: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 218 op/s; 24737/168717 objects degraded (14.662%); 3373/168717 objects misplaced (1.999%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:36:51.554 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:51 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:36:51.554 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:51 smithi110 ceph-mon[212939]: failed to encode map e597 with expected crc 2023-09-23T23:36:51.554 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:51 smithi110 ceph-mon[212939]: failed to encode map e597 with expected crc 2023-09-23T23:36:51.554 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:51 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3137137427' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:36:51.554 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:51 smithi110 ceph-mon[212939]: osdmap e597: 8 total, 8 up, 7 in 2023-09-23T23:36:51.554 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:51 smithi110 ceph-mon[221102]: pgmap v1665: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 218 op/s; 24737/168717 objects degraded (14.662%); 3373/168717 objects misplaced (1.999%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:36:51.554 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:51 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:36:51.554 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:51 smithi110 ceph-mon[221102]: failed to encode map e597 with expected crc 2023-09-23T23:36:51.555 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:51 smithi110 ceph-mon[221102]: failed to encode map e597 with expected crc 2023-09-23T23:36:51.555 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:51 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3137137427' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:36:51.555 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:51 smithi110 ceph-mon[221102]: osdmap e597: 8 total, 8 up, 7 in 2023-09-23T23:36:51.627 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:51.652 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:51 smithi138 ceph-mon[193606]: pgmap v1665: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 218 op/s; 24737/168717 objects degraded (14.662%); 3373/168717 objects misplaced (1.999%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:36:51.652 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:51 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:36:51.652 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:51 smithi138 ceph-mon[193606]: failed to encode map e597 with expected crc 2023-09-23T23:36:51.652 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:51 smithi138 ceph-mon[193606]: failed to encode map e597 with expected crc 2023-09-23T23:36:51.652 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:51 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3137137427' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:36:51.652 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:51 smithi138 ceph-mon[193606]: osdmap e597: 8 total, 8 up, 7 in 2023-09-23T23:36:51.654 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:51.704 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:51.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:51.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:51.799 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:51.812 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:36:51.853 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:51.904 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:36:51.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:51.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:36:52.010 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:52.039 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:52.251 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:36:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:36:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:36:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:36:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:202131 11.17 11:efc6c41a:::benchmark_data_smithi110_433746_object96251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T23:36:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:00.397599+0000", 2023-09-23T23:36:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 531.48365888700005, 2023-09-23T23:36:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98048151299999997, 2023-09-23T23:36:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 202131 2023-09-23T23:36:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397599+0000", 2023-09-23T23:36:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397599+0000", 2023-09-23T23:36:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397599+0000", 2023-09-23T23:36:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397603+0000", 2023-09-23T23:36:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5870000000000003e-06 2023-09-23T23:36:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397615+0000", 2023-09-23T23:36:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1591e-05 2023-09-23T23:36:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397622+0000", 2023-09-23T23:36:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5009999999999998e-06 2023-09-23T23:36:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397741+0000", 2023-09-23T23:36:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118649 2023-09-23T23:36:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.397749+0000", 2023-09-23T23:36:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4950000000000008e-06 2023-09-23T23:36:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.364031+0000", 2023-09-23T23:36:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96628133299999996 2023-09-23T23:36:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.364148+0000", 2023-09-23T23:36:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011695499999999999 2023-09-23T23:36:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:36:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.364473+0000", 2023-09-23T23:36:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032511900000000002 2023-09-23T23:36:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.370881+0000", 2023-09-23T23:36:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0064080700000000001 2023-09-23T23:36:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.377963+0000", 2023-09-23T23:36:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070824929999999996 2023-09-23T23:36:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378001+0000", 2023-09-23T23:36:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7595999999999997e-05 2023-09-23T23:36:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378043+0000", 2023-09-23T23:36:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2283999999999997e-05 2023-09-23T23:36:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378081+0000", 2023-09-23T23:36:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7478999999999998e-05 2023-09-23T23:36:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:202151 11.17 11:eead75a8:::benchmark_data_smithi110_433746_object96271:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T23:36:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:00.411787+0000", 2023-09-23T23:36:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 531.46947084500005, 2023-09-23T23:36:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96643706600000001, 2023-09-23T23:36:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 202151 2023-09-23T23:36:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411787+0000", 2023-09-23T23:36:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411787+0000", 2023-09-23T23:36:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411786+0000", 2023-09-23T23:36:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411792+0000", 2023-09-23T23:36:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3799999999999999e-06 2023-09-23T23:36:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411810+0000", 2023-09-23T23:36:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7751999999999999e-05 2023-09-23T23:36:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411822+0000", 2023-09-23T23:36:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.201e-05 2023-09-23T23:36:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411868+0000", 2023-09-23T23:36:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5695000000000002e-05 2023-09-23T23:36:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.411876+0000", 2023-09-23T23:36:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6209999999999993e-06 2023-09-23T23:36:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.368736+0000", 2023-09-23T23:36:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95685995199999996 2023-09-23T23:36:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.368844+0000", 2023-09-23T23:36:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107253 2023-09-23T23:36:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:36:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.369149+0000", 2023-09-23T23:36:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030546099999999998 2023-09-23T23:36:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.377918+0000", 2023-09-23T23:36:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0087685660000000002 2023-09-23T23:36:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378118+0000", 2023-09-23T23:36:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020053900000000001 2023-09-23T23:36:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378148+0000", 2023-09-23T23:36:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9896999999999999e-05 2023-09-23T23:36:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378190+0000", 2023-09-23T23:36:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1541000000000001e-05 2023-09-23T23:36:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378224+0000", 2023-09-23T23:36:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4879999999999998e-05 2023-09-23T23:36:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:202156 11.17 11:ef634d51:::benchmark_data_smithi110_433746_object96276:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T23:36:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:00.415723+0000", 2023-09-23T23:36:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 531.46553518999997, 2023-09-23T23:36:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96665629500000005, 2023-09-23T23:36:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 202156 2023-09-23T23:36:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415723+0000", 2023-09-23T23:36:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415723+0000", 2023-09-23T23:36:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415723+0000", 2023-09-23T23:36:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415725+0000", 2023-09-23T23:36:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8040000000000002e-06 2023-09-23T23:36:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415732+0000", 2023-09-23T23:36:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6549999999999998e-06 2023-09-23T23:36:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415738+0000", 2023-09-23T23:36:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5940000000000001e-06 2023-09-23T23:36:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415772+0000", 2023-09-23T23:36:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3912000000000003e-05 2023-09-23T23:36:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:00.415778+0000", 2023-09-23T23:36:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9939999999999997e-06 2023-09-23T23:36:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.370907+0000", 2023-09-23T23:36:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95512952699999998 2023-09-23T23:36:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.371032+0000", 2023-09-23T23:36:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012525899999999999 2023-09-23T23:36:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:36:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.371307+0000", 2023-09-23T23:36:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027482299999999999 2023-09-23T23:36:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378260+0000", 2023-09-23T23:36:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0069533099999999999 2023-09-23T23:36:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.378290+0000", 2023-09-23T23:36:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9379000000000001e-05 2023-09-23T23:36:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.382296+0000", 2023-09-23T23:36:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040064779999999999 2023-09-23T23:36:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.382337+0000", 2023-09-23T23:36:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1029000000000003e-05 2023-09-23T23:36:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:01.382379+0000", 2023-09-23T23:36:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1980999999999997e-05 2023-09-23T23:36:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:210997 11.17 11:eefcf624:::benchmark_data_smithi110_433746_object105117:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:36:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686483+0000", 2023-09-23T23:36:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 510.19477533000003, 2023-09-23T23:36:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0169343049999999, 2023-09-23T23:36:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 210997 2023-09-23T23:36:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686483+0000", 2023-09-23T23:36:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686483+0000", 2023-09-23T23:36:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686481+0000", 2023-09-23T23:36:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:36:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686489+0000", 2023-09-23T23:36:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.439e-06 2023-09-23T23:36:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686502+0000", 2023-09-23T23:36:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2376e-05 2023-09-23T23:36:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686523+0000", 2023-09-23T23:36:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0860999999999999e-05 2023-09-23T23:36:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687605+0000", 2023-09-23T23:36:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001082698 2023-09-23T23:36:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687618+0000", 2023-09-23T23:36:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2848e-05 2023-09-23T23:36:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.694585+0000", 2023-09-23T23:36:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.006967221 2023-09-23T23:36:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.694712+0000", 2023-09-23T23:36:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012615400000000001 2023-09-23T23:36:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:36:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.695012+0000", 2023-09-23T23:36:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030036499999999997 2023-09-23T23:36:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.698738+0000", 2023-09-23T23:36:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003726203 2023-09-23T23:36:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.698996+0000", 2023-09-23T23:36:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025763799999999998 2023-09-23T23:36:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.699037+0000", 2023-09-23T23:36:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1517000000000002e-05 2023-09-23T23:36:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703242+0000", 2023-09-23T23:36:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042050509999999996 2023-09-23T23:36:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703295+0000", 2023-09-23T23:36:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2997000000000001e-05 2023-09-23T23:36:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703417+0000", 2023-09-23T23:36:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012184900000000001 2023-09-23T23:36:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211002 11.17 11:ecab426b:::benchmark_data_smithi110_433746_object105122:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:36:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686530+0000", 2023-09-23T23:36:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 510.194727704, 2023-09-23T23:36:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0172692640000001, 2023-09-23T23:36:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211002 2023-09-23T23:36:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686530+0000", 2023-09-23T23:36:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686530+0000", 2023-09-23T23:36:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686530+0000", 2023-09-23T23:36:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686534+0000", 2023-09-23T23:36:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9709999999999998e-06 2023-09-23T23:36:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686553+0000", 2023-09-23T23:36:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9241000000000001e-05 2023-09-23T23:36:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686560+0000", 2023-09-23T23:36:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9260000000000003e-06 2023-09-23T23:36:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687641+0000", 2023-09-23T23:36:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001081226 2023-09-23T23:36:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687650+0000", 2023-09-23T23:36:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.833e-06 2023-09-23T23:36:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.696593+0000", 2023-09-23T23:36:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0089432190000001 2023-09-23T23:36:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.696740+0000", 2023-09-23T23:36:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014711800000000001 2023-09-23T23:36:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:36:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.697010+0000", 2023-09-23T23:36:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026912600000000001 2023-09-23T23:36:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.700956+0000", 2023-09-23T23:36:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039468569999999998 2023-09-23T23:36:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703189+0000", 2023-09-23T23:36:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022322599999999998 2023-09-23T23:36:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703471+0000", 2023-09-23T23:36:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028231600000000001 2023-09-23T23:36:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703641+0000", 2023-09-23T23:36:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000170404 2023-09-23T23:36:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703696+0000", 2023-09-23T23:36:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4475999999999998e-05 2023-09-23T23:36:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703800+0000", 2023-09-23T23:36:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103943 2023-09-23T23:36:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211006 11.17 11:ee6c8b78:::benchmark_data_smithi110_433746_object105126:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:36:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686565+0000", 2023-09-23T23:36:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 510.19469318300003, 2023-09-23T23:36:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0175006170000001, 2023-09-23T23:36:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211006 2023-09-23T23:36:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686565+0000", 2023-09-23T23:36:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686565+0000", 2023-09-23T23:36:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686565+0000", 2023-09-23T23:36:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686568+0000", 2023-09-23T23:36:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2600000000000001e-06 2023-09-23T23:36:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686574+0000", 2023-09-23T23:36:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0789999999999997e-06 2023-09-23T23:36:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686579+0000", 2023-09-23T23:36:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3279999999999996e-06 2023-09-23T23:36:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687673+0000", 2023-09-23T23:36:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001093966 2023-09-23T23:36:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687681+0000", 2023-09-23T23:36:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9889999999999992e-06 2023-09-23T23:36:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.699069+0000", 2023-09-23T23:36:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.011387475 2023-09-23T23:36:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.699178+0000", 2023-09-23T23:36:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109238 2023-09-23T23:36:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:36:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.699384+0000", 2023-09-23T23:36:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020630299999999999 2023-09-23T23:36:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703201+0000", 2023-09-23T23:36:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038163939999999999 2023-09-23T23:36:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703505+0000", 2023-09-23T23:36:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030447999999999999 2023-09-23T23:36:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703847+0000", 2023-09-23T23:36:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034161899999999998 2023-09-23T23:36:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703911+0000", 2023-09-23T23:36:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4521000000000001e-05 2023-09-23T23:36:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703957+0000", 2023-09-23T23:36:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5840999999999997e-05 2023-09-23T23:36:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704066+0000", 2023-09-23T23:36:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000108546 2023-09-23T23:36:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211014 11.17 11:ecac8c40:::benchmark_data_smithi110_433746_object105134:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:36:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686583+0000", 2023-09-23T23:36:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 510.19467484199998, 2023-09-23T23:36:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0177152949999999, 2023-09-23T23:36:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211014 2023-09-23T23:36:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686583+0000", 2023-09-23T23:36:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686583+0000", 2023-09-23T23:36:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686583+0000", 2023-09-23T23:36:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686586+0000", 2023-09-23T23:36:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6769999999999999e-06 2023-09-23T23:36:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686592+0000", 2023-09-23T23:36:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2480000000000001e-06 2023-09-23T23:36:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686598+0000", 2023-09-23T23:36:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4500000000000001e-06 2023-09-23T23:36:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687703+0000", 2023-09-23T23:36:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001105113 2023-09-23T23:36:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687711+0000", 2023-09-23T23:36:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9410000000000007e-06 2023-09-23T23:36:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.700984+0000", 2023-09-23T23:36:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0132722620000001 2023-09-23T23:36:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.701130+0000", 2023-09-23T23:36:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000146804 2023-09-23T23:36:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:36:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.701398+0000", 2023-09-23T23:36:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026770100000000002 2023-09-23T23:36:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703434+0000", 2023-09-23T23:36:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020357940000000001 2023-09-23T23:36:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.703879+0000", 2023-09-23T23:36:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00044474299999999999 2023-09-23T23:36:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704106+0000", 2023-09-23T23:36:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022766 2023-09-23T23:36:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704144+0000", 2023-09-23T23:36:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7836000000000002e-05 2023-09-23T23:36:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704191+0000", 2023-09-23T23:36:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6582999999999998e-05 2023-09-23T23:36:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704299+0000", 2023-09-23T23:36:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107909 2023-09-23T23:36:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211043 11.17 11:edfeec9c:::benchmark_data_smithi110_433746_object105163:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:36:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686612+0000", 2023-09-23T23:36:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 510.19464594800002, 2023-09-23T23:36:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0257805470000001, 2023-09-23T23:36:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211043 2023-09-23T23:36:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686612+0000", 2023-09-23T23:36:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686612+0000", 2023-09-23T23:36:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686612+0000", 2023-09-23T23:36:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686615+0000", 2023-09-23T23:36:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3289999999999998e-06 2023-09-23T23:36:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686622+0000", 2023-09-23T23:36:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1439999999999997e-06 2023-09-23T23:36:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686628+0000", 2023-09-23T23:36:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9560000000000002e-06 2023-09-23T23:36:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687733+0000", 2023-09-23T23:36:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011048640000000001 2023-09-23T23:36:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687741+0000", 2023-09-23T23:36:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9340000000000006e-06 2023-09-23T23:36:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704331+0000", 2023-09-23T23:36:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0165899519999999 2023-09-23T23:36:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704469+0000", 2023-09-23T23:36:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013822899999999999 2023-09-23T23:36:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:36:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.704810+0000", 2023-09-23T23:36:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000341169 2023-09-23T23:36:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712010+0000", 2023-09-23T23:36:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071992189999999998 2023-09-23T23:36:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712083+0000", 2023-09-23T23:36:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3090999999999995e-05 2023-09-23T23:36:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712140+0000", 2023-09-23T23:36:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7030000000000001e-05 2023-09-23T23:36:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712203+0000", 2023-09-23T23:36:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3297000000000001e-05 2023-09-23T23:36:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712257+0000", 2023-09-23T23:36:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4211e-05 2023-09-23T23:36:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712393+0000", 2023-09-23T23:36:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013563 2023-09-23T23:36:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211063 11.17 11:ec1e6191:::benchmark_data_smithi110_433746_object105183:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:36:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686633+0000", 2023-09-23T23:36:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 510.19462551200002, 2023-09-23T23:36:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.026136234, 2023-09-23T23:36:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211063 2023-09-23T23:36:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686633+0000", 2023-09-23T23:36:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686633+0000", 2023-09-23T23:36:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686632+0000", 2023-09-23T23:36:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686635+0000", 2023-09-23T23:36:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9110000000000002e-06 2023-09-23T23:36:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686643+0000", 2023-09-23T23:36:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7309999999999993e-06 2023-09-23T23:36:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686648+0000", 2023-09-23T23:36:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1580000000000004e-06 2023-09-23T23:36:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687762+0000", 2023-09-23T23:36:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011139660000000001 2023-09-23T23:36:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687770+0000", 2023-09-23T23:36:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9820000000000008e-06 2023-09-23T23:36:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.706433+0000", 2023-09-23T23:36:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.018663052 2023-09-23T23:36:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.706571+0000", 2023-09-23T23:36:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013809200000000001 2023-09-23T23:36:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:36:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.706778+0000", 2023-09-23T23:36:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000206739 2023-09-23T23:36:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712153+0000", 2023-09-23T23:36:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053747880000000001 2023-09-23T23:36:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712460+0000", 2023-09-23T23:36:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030737499999999998 2023-09-23T23:36:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712524+0000", 2023-09-23T23:36:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4078000000000006e-05 2023-09-23T23:36:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712603+0000", 2023-09-23T23:36:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8625e-05 2023-09-23T23:36:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712666+0000", 2023-09-23T23:36:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2713999999999996e-05 2023-09-23T23:36:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712769+0000", 2023-09-23T23:36:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010334 2023-09-23T23:36:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211074 11.17 11:eb325c31:::benchmark_data_smithi110_433746_object105194:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:36:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686653+0000", 2023-09-23T23:36:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 510.194605583, 2023-09-23T23:36:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.02894591, 2023-09-23T23:36:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211074 2023-09-23T23:36:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686653+0000", 2023-09-23T23:36:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686653+0000", 2023-09-23T23:36:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686652+0000", 2023-09-23T23:36:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686655+0000", 2023-09-23T23:36:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6469999999999998e-06 2023-09-23T23:36:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686661+0000", 2023-09-23T23:36:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8969999999999996e-06 2023-09-23T23:36:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686666+0000", 2023-09-23T23:36:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9930000000000003e-06 2023-09-23T23:36:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687792+0000", 2023-09-23T23:36:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011261330000000001 2023-09-23T23:36:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687800+0000", 2023-09-23T23:36:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9389999999999997e-06 2023-09-23T23:36:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.708139+0000", 2023-09-23T23:36:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0203391749999999 2023-09-23T23:36:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.708257+0000", 2023-09-23T23:36:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117959 2023-09-23T23:36:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:36:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.708494+0000", 2023-09-23T23:36:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023732399999999999 2023-09-23T23:36:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712159+0000", 2023-09-23T23:36:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036648150000000001 2023-09-23T23:36:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715288+0000", 2023-09-23T23:36:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031290469999999998 2023-09-23T23:36:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715392+0000", 2023-09-23T23:36:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104362 2023-09-23T23:36:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715427+0000", 2023-09-23T23:36:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4581e-05 2023-09-23T23:36:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715475+0000", 2023-09-23T23:36:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7766999999999999e-05 2023-09-23T23:36:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715599+0000", 2023-09-23T23:36:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012368000000000001 2023-09-23T23:36:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211090 11.17 11:ee95de61:::benchmark_data_smithi110_433746_object105210:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:36:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686670+0000", 2023-09-23T23:36:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 510.19458818800001, 2023-09-23T23:36:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.029135624, 2023-09-23T23:36:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211090 2023-09-23T23:36:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686670+0000", 2023-09-23T23:36:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686670+0000", 2023-09-23T23:36:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686670+0000", 2023-09-23T23:36:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686672+0000", 2023-09-23T23:36:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1749999999999999e-06 2023-09-23T23:36:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686678+0000", 2023-09-23T23:36:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3589999999999996e-06 2023-09-23T23:36:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686683+0000", 2023-09-23T23:36:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1139999999999997e-06 2023-09-23T23:36:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687821+0000", 2023-09-23T23:36:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001137723 2023-09-23T23:36:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687829+0000", 2023-09-23T23:36:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6809999999999998e-06 2023-09-23T23:36:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.710062+0000", 2023-09-23T23:36:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0222334470000001 2023-09-23T23:36:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.710215+0000", 2023-09-23T23:36:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015298800000000001 2023-09-23T23:36:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:36:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.710532+0000", 2023-09-23T23:36:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00031638499999999997 2023-09-23T23:36:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712541+0000", 2023-09-23T23:36:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020091390000000001 2023-09-23T23:36:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715339+0000", 2023-09-23T23:36:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002798367 2023-09-23T23:36:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715645+0000", 2023-09-23T23:36:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030564500000000001 2023-09-23T23:36:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715685+0000", 2023-09-23T23:36:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0182000000000003e-05 2023-09-23T23:36:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715717+0000", 2023-09-23T23:36:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1649000000000001e-05 2023-09-23T23:36:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715806+0000", 2023-09-23T23:36:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9040000000000001e-05 2023-09-23T23:36:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:211101 11.17 11:eed1b340:::benchmark_data_smithi110_433746_object105221:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T23:36:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:28:21.686688+0000", 2023-09-23T23:36:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 510.19457060000002, 2023-09-23T23:36:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.034036953, 2023-09-23T23:36:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:36:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:36:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 211101 2023-09-23T23:36:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686688+0000", 2023-09-23T23:36:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686688+0000", 2023-09-23T23:36:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686687+0000", 2023-09-23T23:36:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686690+0000", 2023-09-23T23:36:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4930000000000001e-06 2023-09-23T23:36:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686696+0000", 2023-09-23T23:36:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2400000000000004e-06 2023-09-23T23:36:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.686701+0000", 2023-09-23T23:36:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2020000000000003e-06 2023-09-23T23:36:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687851+0000", 2023-09-23T23:36:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011494809999999999 2023-09-23T23:36:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:21.687859+0000", 2023-09-23T23:36:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8779999999999998e-06 2023-09-23T23:36:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712809+0000", 2023-09-23T23:36:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0249501940000001 2023-09-23T23:36:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.712967+0000", 2023-09-23T23:36:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015830900000000001 2023-09-23T23:36:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T23:36:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.713350+0000", 2023-09-23T23:36:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00038338199999999998 2023-09-23T23:36:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715863+0000", 2023-09-23T23:36:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025129060000000001 2023-09-23T23:36:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715903+0000", 2023-09-23T23:36:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9675000000000002e-05 2023-09-23T23:36:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:36:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.715938+0000", 2023-09-23T23:36:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4501000000000001e-05 2023-09-23T23:36:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:36:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.720575+0000", 2023-09-23T23:36:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046374110000000001 2023-09-23T23:36:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.720618+0000", 2023-09-23T23:36:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3411000000000002e-05 2023-09-23T23:36:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:28:22.720725+0000", 2023-09-23T23:36:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000106209 2023-09-23T23:36:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71508 13.d 13:b7936a93:::benchmark_data_smithi110_541928_object71507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:36:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.552274+0000", 2023-09-23T23:36:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 181.32898387899999, 2023-09-23T23:36:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1860481599999999, 2023-09-23T23:36:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:36:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:36:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:36:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:36:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71508 2023-09-23T23:36:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.552274+0000", 2023-09-23T23:36:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.552274+0000", 2023-09-23T23:36:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.552273+0000", 2023-09-23T23:36:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:36:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.552302+0000", 2023-09-23T23:36:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9275999999999999e-05 2023-09-23T23:36:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.552314+0000", 2023-09-23T23:36:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1657999999999999e-05 2023-09-23T23:36:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.552322+0000", 2023-09-23T23:36:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8289999999999997e-06 2023-09-23T23:36:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.556287+0000", 2023-09-23T23:36:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039647220000000004 2023-09-23T23:36:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.556297+0000", 2023-09-23T23:36:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8099999999999992e-06 2023-09-23T23:36:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.537459+0000", 2023-09-23T23:36:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98116200399999998 2023-09-23T23:36:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:36:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.537473+0000", 2023-09-23T23:36:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4430000000000001e-05 2023-09-23T23:36:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.737038+0000", 2023-09-23T23:36:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1995647469999999 2023-09-23T23:36:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.738322+0000", 2023-09-23T23:36:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012843780000000001 2023-09-23T23:36:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:110962 13.a e592/557 13:53402878:::benchmark_data_smithi110_541928_object23182:head v 592'6955, mlcod=549'1793)", 2023-09-23T23:36:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:36:11.857179+0000", 2023-09-23T23:36:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 40.024079041, 2023-09-23T23:36:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014861452000000001, 2023-09-23T23:36:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.857179+0000", 2023-09-23T23:36:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.857179+0000", 2023-09-23T23:36:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.857178+0000", 2023-09-23T23:36:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:36:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.857188+0000", 2023-09-23T23:36:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5589999999999994e-06 2023-09-23T23:36:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.857201+0000", 2023-09-23T23:36:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3369e-05 2023-09-23T23:36:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.857209+0000", 2023-09-23T23:36:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2430000000000005e-06 2023-09-23T23:36:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.867007+0000", 2023-09-23T23:36:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0097973169999999998 2023-09-23T23:36:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.867038+0000", 2023-09-23T23:36:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1025000000000002e-05 2023-09-23T23:36:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.872004+0000", 2023-09-23T23:36:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0049662450000000002 2023-09-23T23:36:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.872041+0000", 2023-09-23T23:36:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6831e-05 2023-09-23T23:36:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:110963 13.a e592/557 13:537dff46:::benchmark_data_smithi110_541928_object23183:head v 592'6956, mlcod=549'1793)", 2023-09-23T23:36:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:36:11.862614+0000", 2023-09-23T23:36:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 40.018644070000001, 2023-09-23T23:36:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01269668, 2023-09-23T23:36:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.862614+0000", 2023-09-23T23:36:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.862614+0000", 2023-09-23T23:36:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.862613+0000", 2023-09-23T23:36:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:36:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.862623+0000", 2023-09-23T23:36:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5610000000000004e-06 2023-09-23T23:36:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.862637+0000", 2023-09-23T23:36:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4523e-05 2023-09-23T23:36:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.862646+0000", 2023-09-23T23:36:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4929999999999998e-06 2023-09-23T23:36:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.869669+0000", 2023-09-23T23:36:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070232360000000004 2023-09-23T23:36:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.869700+0000", 2023-09-23T23:36:52.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1511999999999999e-05 2023-09-23T23:36:52.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.875259+0000", 2023-09-23T23:36:52.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055584620000000001 2023-09-23T23:36:52.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.875311+0000", 2023-09-23T23:36:52.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1935000000000003e-05 2023-09-23T23:36:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:110966 13.a e592/557 13:54388575:::benchmark_data_smithi110_541928_object23186:head v 592'6957, mlcod=549'1793)", 2023-09-23T23:36:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:36:11.864541+0000", 2023-09-23T23:36:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 40.016717286999999, 2023-09-23T23:36:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010914004999999999, 2023-09-23T23:36:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.864541+0000", 2023-09-23T23:36:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.864541+0000", 2023-09-23T23:36:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.864540+0000", 2023-09-23T23:36:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:36:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.864547+0000", 2023-09-23T23:36:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8410000000000002e-06 2023-09-23T23:36:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.864558+0000", 2023-09-23T23:36:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1226999999999999e-05 2023-09-23T23:36:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.864564+0000", 2023-09-23T23:36:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0009999999999998e-06 2023-09-23T23:36:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.872114+0000", 2023-09-23T23:36:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075494289999999999 2023-09-23T23:36:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.872143+0000", 2023-09-23T23:36:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9607999999999998e-05 2023-09-23T23:36:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.875421+0000", 2023-09-23T23:36:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032776279999999999 2023-09-23T23:36:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:11.875455+0000", 2023-09-23T23:36:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3912999999999999e-05 2023-09-23T23:36:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:115015 13.a e594/557 13:571a185d:::benchmark_data_smithi110_541928_object27235:head v 594'7242, mlcod=549'1793)", 2023-09-23T23:36:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:36:28.548271+0000", 2023-09-23T23:36:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 23.332986873999999, 2023-09-23T23:36:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010203698000000001, 2023-09-23T23:36:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:28.548271+0000", 2023-09-23T23:36:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:28.548271+0000", 2023-09-23T23:36:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:28.548270+0000", 2023-09-23T23:36:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:28.548282+0000", 2023-09-23T23:36:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1914e-05 2023-09-23T23:36:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:28.548304+0000", 2023-09-23T23:36:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2283e-05 2023-09-23T23:36:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:28.548315+0000", 2023-09-23T23:36:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0626e-05 2023-09-23T23:36:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:28.550613+0000", 2023-09-23T23:36:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002298502 2023-09-23T23:36:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:28.550650+0000", 2023-09-23T23:36:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7188999999999999e-05 2023-09-23T23:36:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:28.558408+0000", 2023-09-23T23:36:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077574590000000004 2023-09-23T23:36:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:28.558475+0000", 2023-09-23T23:36:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7113000000000006e-05 2023-09-23T23:36:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:115826 13.c e594/562 13:37221de2:::benchmark_data_smithi110_541928_object28046:head v 594'7213, mlcod=594'7211)", 2023-09-23T23:36:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:36:32.250985+0000", 2023-09-23T23:36:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 19.630273076000002, 2023-09-23T23:36:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.026274506, 2023-09-23T23:36:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.250985+0000", 2023-09-23T23:36:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.250985+0000", 2023-09-23T23:36:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.250984+0000", 2023-09-23T23:36:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.250996+0000", 2023-09-23T23:36:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2574e-05 2023-09-23T23:36:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.251013+0000", 2023-09-23T23:36:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7329999999999998e-05 2023-09-23T23:36:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.251024+0000", 2023-09-23T23:36:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0154e-05 2023-09-23T23:36:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.251162+0000", 2023-09-23T23:36:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013886000000000001 2023-09-23T23:36:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.251192+0000", 2023-09-23T23:36:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9119e-05 2023-09-23T23:36:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.277214+0000", 2023-09-23T23:36:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.026022927000000001 2023-09-23T23:36:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.277260+0000", 2023-09-23T23:36:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5132000000000001e-05 2023-09-23T23:36:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:115827 13.a e594/557 13:593b61e0:::benchmark_data_smithi110_541928_object28047:head v 594'7287, mlcod=549'1793)", 2023-09-23T23:36:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:36:32.256768+0000", 2023-09-23T23:36:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 19.62448994, 2023-09-23T23:36:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019991661000000001, 2023-09-23T23:36:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.256768+0000", 2023-09-23T23:36:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.256768+0000", 2023-09-23T23:36:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.256767+0000", 2023-09-23T23:36:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.256778+0000", 2023-09-23T23:36:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1102e-05 2023-09-23T23:36:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.256800+0000", 2023-09-23T23:36:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1597999999999999e-05 2023-09-23T23:36:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.256810+0000", 2023-09-23T23:36:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0428000000000001e-05 2023-09-23T23:36:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.256883+0000", 2023-09-23T23:36:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3328e-05 2023-09-23T23:36:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.256914+0000", 2023-09-23T23:36:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0651000000000002e-05 2023-09-23T23:36:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.276661+0000", 2023-09-23T23:36:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019746857999999999 2023-09-23T23:36:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.276760+0000", 2023-09-23T23:36:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8967000000000003e-05 2023-09-23T23:36:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:115874 13.0 e595/566 13:03c3e306:::benchmark_data_smithi110_541928_object28094:head v 595'7257, mlcod=595'7255)", 2023-09-23T23:36:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:36:32.410556+0000", 2023-09-23T23:36:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 19.470702276000001, 2023-09-23T23:36:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011111728, 2023-09-23T23:36:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:36:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:36:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:36:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:36:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.410556+0000", 2023-09-23T23:36:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:36:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.410556+0000", 2023-09-23T23:36:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:36:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:36:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.410554+0000", 2023-09-23T23:36:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:36:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:36:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.410566+0000", 2023-09-23T23:36:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1989e-05 2023-09-23T23:36:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:36:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.410589+0000", 2023-09-23T23:36:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2385999999999999e-05 2023-09-23T23:36:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:36:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.410599+0000", 2023-09-23T23:36:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6409999999999997e-06 2023-09-23T23:36:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:36:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.410671+0000", 2023-09-23T23:36:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2581000000000002e-05 2023-09-23T23:36:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:36:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.410696+0000", 2023-09-23T23:36:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4661e-05 2023-09-23T23:36:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:36:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.421603+0000", 2023-09-23T23:36:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010907296 2023-09-23T23:36:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:36:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:36:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:36:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:36:32.421668+0000", 2023-09-23T23:36:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4609000000000004e-05 2023-09-23T23:36:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.394 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.394 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:36:52.394 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:36:52.394 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:36:52.687 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:52.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:52.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:52.792 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:52.818 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:52.863 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:52.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:52.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:52.969 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:36:52.996 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:53.052 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:36:53.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:53.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:36:53.159 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:53.190 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:53.204 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:36:53.360 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:53.361 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:36:53.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:53.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:53.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:53 smithi110 ceph-mon[212939]: pgmap v1667: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 24737/168084 objects degraded (14.717%); 3373/168084 objects misplaced (2.007%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:53.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:53 smithi110 ceph-mon[221102]: pgmap v1667: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 24737/168084 objects degraded (14.717%); 3373/168084 objects misplaced (2.007%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:53.468 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:53.508 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:53.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:53 smithi138 ceph-mon[193606]: pgmap v1667: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 24737/168084 objects degraded (14.717%); 3373/168084 objects misplaced (2.007%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:54.385 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:54.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:54.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:54.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:54 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 24737/168084 objects degraded (14.717%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:54.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:54 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 24737/168084 objects degraded (14.717%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:54.496 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:54.517 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:54.544 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:54.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:54 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 24737/168084 objects degraded (14.717%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:54.720 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:54.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:54.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:54.783 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:36:54.793 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:36:54.825 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:54.855 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:54.872 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:54.956 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:36:54.986 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:36:55.015 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:36:55.039 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:36:55.211 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:36:55.211 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:36:55.211 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:36:55.211 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:36:55.211 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:36:55.211 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:36:55.211 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:36:55.212 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:36:55.212 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:36:55.212 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:36:55.212 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:36:55.212 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:36:55.440 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:55.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:55 smithi110 ceph-mon[221102]: pgmap v1668: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 218 op/s; 24677/167277 objects degraded (14.752%); 3287/167277 objects misplaced (1.965%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:36:55.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:55 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2713701130' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:55.443 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:55 smithi110 ceph-mon[212939]: pgmap v1668: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 218 op/s; 24677/167277 objects degraded (14.752%); 3287/167277 objects misplaced (1.965%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:36:55.443 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:55 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2713701130' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:55.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:55.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:55.545 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:55.582 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:55.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:55 smithi138 ceph-mon[193606]: pgmap v1668: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 218 op/s; 24677/167277 objects degraded (14.752%); 3287/167277 objects misplaced (1.965%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:36:55.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:55 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2713701130' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:36:56.241 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:56.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:56.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:56.345 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:56.374 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:56.444 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:56.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:56.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:56.494 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:56 smithi110 ceph-mon[212939]: from='client.56824 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:56.495 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1668,"stamp":"2023-09-23T23:36:53.929668+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3653219573,"num_objects":55759,"num_object_clones":0,"num_object_copies":167277,"num_objects_missing_on_primary":2392,"num_objects_missing":2392,"num_objects_degraded":24677,"num_objects_misplaced":3287,"num_objects_unfound":0,"num_objects_dirty":55759,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":119898,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15629,"num_bytes_recovered":1021571496,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94945,"ondisk_log_size":94945,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15191172,"kb_used_data":9490700,"kb_used_omap":29,"kb_used_meta":5700386,"kb_avail":641024892,"statfs":{"total":671965249536,"available":656409489408,"internally_reserved":0,"allocated":9718476800,"data_stored":9702205801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5837195987},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,4,1,0,1,0,1,4,6,5],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-142934016,"num_objects":-2181,"num_object_clones":0,"num_object_copies":-6543,"num_objects_missing_on_primary":-167,"num_objects_missing":-167,"num_objects_degraded":-261,"num_objects_misplaced":-167,"num_objects_unfound":0,"num_objects_dirty":-2181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2181,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":428,"num_bytes_recovered":28049408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002987"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":792,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.277762+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:36:32.277762+0000","last_peered":"2023-09-23T23:36:32.277762+0000","last_clean":"2023-09-23T23:36:32.277762+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:36:32.277762+0000","last_undegraded":"2023-09-23T23:36:32.277762+0000","last_fullsized":"2023-09-23T23:36:32.277762+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":351,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:34.928302+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:36:34.928302+0000","last_peered":"2023-09-23T23:36:34.928302+0000","last_clean":"2023-09-23T23:36:34.928302+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:36:34.928302+0000","last_undegraded":"2023-09-23T23:36:34.928302+0000","last_fullsized":"2023-09-23T23:36:34.928302+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":579,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.292432+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:36:42.292432+0000","last_peered":"2023-09-23T23:36:42.292432+0000","last_clean":"2023-09-23T23:36:42.292432+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:36:42.292432+0000","last_undegraded":"2023-09-23T23:36:42.292432+0000","last_fullsized":"2023-09-23T23:36:42.292432+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":745,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.277941+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:36:32.277941+0000","last_peered":"2023-09-23T23:36:32.277941+0000","last_clean":"2023-09-23T23:36:32.277941+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:36:32.277941+0000","last_undegraded":"2023-09-23T23:36:32.277941+0000","last_fullsized":"2023-09-23T23:36:32.277941+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":316,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.279239+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:36:42.279239+0000","last_peered":"2023-09-23T23:36:42.279239+0000","last_clean":"2023-09-23T23:36:42.279239+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:36:42.279239+0000","last_undegraded":"2023-09-23T23:36:42.279239+0000","last_fullsized":"2023-09-23T23:36:42.279239+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":558,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.280013+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:36:42.280013+0000","last_peered":"2023-09-23T23:36:42.280013+0000","last_clean":"2023-09-23T23:36:42.280013+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:36:42.280013+0000","last_undegraded":"2023-09-23T23:36:42.280013+0000","last_fullsized":"2023-09-23T23:36:42.280013+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":758,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.268510+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:36:42.268510+0000","last_peered":"2023-09-23T23:36:42.268510+0000","last_clean":"2023-09-23T23:36:42.268510+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:36:42.268510+0000","last_undegraded":"2023-09-23T23:36:42.268510+0000","last_fullsized":"2023-09-23T23:36:42.268510+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":615,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.278709+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:36:42.278709+0000","last_peered":"2023-09-23T23:36:42.278709+0000","last_clean":"2023-09-23T23:36:42.278709+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:36:42.278709+0000","last_undegraded":"2023-09-23T23:36:42.278709+0000","last_fullsized":"2023-09-23T23:36:42.278709+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":110,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.294412+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:36:32.294412+0000","last_peered":"2023-09-23T23:36:32.294412+0000","last_clean":"2023-09-23T23:36:32.294412+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:36:32.294412+0000","last_undegraded":"2023-09-23T23:36:32.294412+0000","last_fullsized":"2023-09-23T23:36:32.294412+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":917,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.267964+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:36:42.267964+0000","last_peered":"2023-09-23T23:36:42.267964+0000","last_clean":"2023-09-23T23:36:42.267964+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:36:42.267964+0000","last_undegraded":"2023-09-23T23:36:42.267964+0000","last_fullsized":"2023-09-23T23:36:42.267964+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":248,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.278227+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:36:32.278227+0000","last_peered":"2023-09-23T23:36:32.278227+0000","last_clean":"2023-09-23T23:36:32.278227+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:36:32.278227+0000","last_undegraded":"2023-09-23T23:36:32.278227+0000","last_fullsized":"2023-09-23T23:36:32.278227+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":598,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.278636+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:36:42.278636+0000","last_peered":"2023-09-23T23:36:42.278636+0000","last_clean":"2023-09-23T23:36:42.278636+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:36:42.278636+0000","last_undegraded":"2023-09-23T23:36:42.278636+0000","last_fullsized":"2023-09-23T23:36:42.278636+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":889,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.263019+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:36:32.263019+0000","last_peered":"2023-09-23T23:36:32.263019+0000","last_clean":"2023-09-23T23:36:32.263019+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:36:32.263019+0000","last_undegraded":"2023-09-23T23:36:32.263019+0000","last_fullsized":"2023-09-23T23:36:32.263019+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":227,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.278360+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:36:32.278360+0000","last_peered":"2023-09-23T23:36:32.278360+0000","last_clean":"2023-09-23T23:36:32.278360+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:36:32.278360+0000","last_undegraded":"2023-09-23T23:36:32.278360+0000","last_fullsized":"2023-09-23T23:36:32.278360+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":559,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.592814+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:36:42.592814+0000","last_peered":"2023-09-23T23:36:42.592814+0000","last_clean":"2023-09-23T23:36:42.592814+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:36:42.592814+0000","last_undegraded":"2023-09-23T23:36:42.592814+0000","last_fullsized":"2023-09-23T23:36:42.592814+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1050,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.278190+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:36:32.278190+0000","last_peered":"2023-09-23T23:36:32.278190+0000","last_clean":"2023-09-23T23:36:32.278190+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:36:32.278190+0000","last_undegraded":"2023-09-23T23:36:32.278190+0000","last_fullsized":"2023-09-23T23:36:32.278190+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":270,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.593037+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:36:42.593037+0000","last_peered":"2023-09-23T23:36:42.593037+0000","last_clean":"2023-09-23T23:36:42.593037+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:36:42.593037+0000","last_undegraded":"2023-09-23T23:36:42.593037+0000","last_fullsized":"2023-09-23T23:36:42.593037+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":129,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.290766+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:36:32.290766+0000","last_peered":"2023-09-23T23:36:32.290766+0000","last_clean":"2023-09-23T23:36:32.290766+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:36:32.290766+0000","last_undegraded":"2023-09-23T23:36:32.290766+0000","last_fullsized":"2023-09-23T23:36:32.290766+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":448,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.280154+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:36:42.280154+0000","last_peered":"2023-09-23T23:36:42.280154+0000","last_clean":"2023-09-23T23:36:42.280154+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:36:42.280154+0000","last_undegraded":"2023-09-23T23:36:42.280154+0000","last_fullsized":"2023-09-23T23:36:42.280154+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":170,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.290310+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:36:32.290310+0000","last_peered":"2023-09-23T23:36:32.290310+0000","last_clean":"2023-09-23T23:36:32.290310+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:36:32.290310+0000","last_undegraded":"2023-09-23T23:36:32.290310+0000","last_fullsized":"2023-09-23T23:36:32.290310+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":537,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.290051+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:36:32.290051+0000","last_peered":"2023-09-23T23:36:32.290051+0000","last_clean":"2023-09-23T23:36:32.290051+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:36:32.290051+0000","last_undegraded":"2023-09-23T23:36:32.290051+0000","last_fullsized":"2023-09-23T23:36:32.290051+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":751,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.277627+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:36:32.277627+0000","last_peered":"2023-09-23T23:36:32.277627+0000","last_clean":"2023-09-23T23:36:32.277627+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:36:32.277627+0000","last_undegraded":"2023-09-23T23:36:32.277627+0000","last_fullsized":"2023-09-23T23:36:32.277627+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":143,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:39.933463+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:36:39.933463+0000","last_peered":"2023-09-23T23:36:39.933463+0000","last_clean":"2023-09-23T23:36:39.933463+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:36:39.933463+0000","last_undegraded":"2023-09-23T23:36:39.933463+0000","last_fullsized":"2023-09-23T23:36:39.933463+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":597,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.592999+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:36:42.592999+0000","last_peered":"2023-09-23T23:36:42.592999+0000","last_clean":"2023-09-23T23:36:42.592999+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:36:42.592999+0000","last_undegraded":"2023-09-23T23:36:42.592999+0000","last_fullsized":"2023-09-23T23:36:42.592999+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":881,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.289793+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:36:32.289793+0000","last_peered":"2023-09-23T23:36:32.289793+0000","last_clean":"2023-09-23T23:36:32.289793+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:36:32.289793+0000","last_undegraded":"2023-09-23T23:36:32.289793+0000","last_fullsized":"2023-09-23T23:36:32.289793+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":192,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.592896+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:36:42.592896+0000","last_peered":"2023-09-23T23:36:42.592896+0000","last_clean":"2023-09-23T23:36:42.592896+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:36:42.592896+0000","last_undegraded":"2023-09-23T23:36:42.592896+0000","last_fullsized":"2023-09-23T23:36:42.592896+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":319,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.614180+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:36:42.614180+0000","last_peered":"2023-09-23T23:36:42.614180+0000","last_clean":"2023-09-23T23:36:42.614180+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:36:42.614180+0000","last_undegraded":"2023-09-23T23:36:42.614180+0000","last_fullsized":"2023-09-23T23:36:42.614180+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":560,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.280261+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:36:42.280261+0000","last_peered":"2023-09-23T23:36:42.280261+0000","last_clean":"2023-09-23T23:36:42.280261+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:36:42.280261+0000","last_undegraded":"2023-09-23T23:36:42.280261+0000","last_fullsized":"2023-09-23T23:36:42.280261+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":464,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.290517+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:36:32.290517+0000","last_peered":"2023-09-23T23:36:32.290517+0000","last_clean":"2023-09-23T23:36:32.290517+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:36:32.290517+0000","last_undegraded":"2023-09-23T23:36:32.290517+0000","last_fullsized":"2023-09-23T23:36:32.290517+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":553,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.280313+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:36:42.280313+0000","last_peered":"2023-09-23T23:36:42.280313+0000","last_clean":"2023-09-23T23:36:42.280313+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:36:42.280313+0000","last_undegraded":"2023-09-23T23:36:42.280313+0000","last_fullsized":"2023-09-23T23:36:42.280313+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":959,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.277860+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:36:32.277860+0000","last_peered":"2023-09-23T23:36:32.277860+0000","last_clean":"2023-09-23T23:36:32.277860+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:36:32.277860+0000","last_undegraded":"2023-09-23T23:36:32.277860+0000","last_fullsized":"2023-09-23T23:36:32.277860+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":160,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.290050+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:36:32.290050+0000","last_peered":"2023-09-23T23:36:32.290050+0000","last_clean":"2023-09-23T23:36:32.290050+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:36:32.290050+0000","last_undegraded":"2023-09-23T23:36:32.290050+0000","last_fullsized":"2023-09-23T23:36:32.290050+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":238,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.280354+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:36:42.280354+0000","last_peered":"2023-09-23T23:36:42.280354+0000","last_clean":"2023-09-23T23:36:42.280354+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:36:42.280354+0000","last_undegraded":"2023-09-23T23:36:42.280354+0000","last_fullsized":"2023-09-23T23:36:42.280354+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"596'7336","reported_seq":7415,"reported_epoch":596,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:48.924602+0000","last_change":"2023-09-23T23:33:52.884769+0000","last_active":"2023-09-23T23:36:48.924602+0000","last_peered":"2023-09-23T23:36:48.924602+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:36:48.924602+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"549'1900","ondisk_log_start":"549'1900","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5436,"log_dups_size":0,"ondisk_log_size":5436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230162432,"num_objects":3512,"num_object_clones":0,"num_object_copies":10536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7336,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":2486},{"shards":"2,6","objects":-886}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":930,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:34.928347+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:36:34.928347+0000","last_peered":"2023-09-23T23:36:34.928347+0000","last_clean":"2023-09-23T23:36:34.928347+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:36:34.928347+0000","last_undegraded":"2023-09-23T23:36:34.928347+0000","last_fullsized":"2023-09-23T23:36:34.928347+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":238,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.269199+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:36:42.269199+0000","last_peered":"2023-09-23T23:36:42.269199+0000","last_clean":"2023-09-23T23:36:42.269199+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:36:42.269199+0000","last_undegraded":"2023-09-23T23:36:42.269199+0000","last_fullsized":"2023-09-23T23:36:42.269199+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":227,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.277686+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:36:32.277686+0000","last_peered":"2023-09-23T23:36:32.277686+0000","last_clean":"2023-09-23T23:36:32.277686+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:36:32.277686+0000","last_undegraded":"2023-09-23T23:36:32.277686+0000","last_fullsized":"2023-09-23T23:36:32.277686+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"596'7499","reported_seq":7578,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:49.494657+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:36:49.494657+0000","last_peered":"2023-09-23T23:36:49.494657+0000","last_clean":"2023-09-23T23:36:49.494657+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:36:49.494657+0000","last_undegraded":"2023-09-23T23:36:49.494657+0000","last_fullsized":"2023-09-23T23:36:49.494657+0000","mapping_epoch":566,"log_start":"546'1300","ondisk_log_start":"546'1300","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6199,"log_dups_size":0,"ondisk_log_size":6199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231538688,"num_objects":3533,"num_object_clones":0,"num_object_copies":10599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7499,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1009,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.279346+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:36:42.279346+0000","last_peered":"2023-09-23T23:36:42.279346+0000","last_clean":"2023-09-23T23:36:42.279346+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:36:42.279346+0000","last_undegraded":"2023-09-23T23:36:42.279346+0000","last_fullsized":"2023-09-23T23:36:42.279346+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":228,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.268011+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:36:42.268011+0000","last_peered":"2023-09-23T23:36:42.268011+0000","last_clean":"2023-09-23T23:36:42.268011+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:36:42.268011+0000","last_undegraded":"2023-09-23T23:36:42.268011+0000","last_fullsized":"2023-09-23T23:36:42.268011+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":448,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.278657+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:36:42.278657+0000","last_peered":"2023-09-23T23:36:42.278657+0000","last_clean":"2023-09-23T23:36:42.278657+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:36:42.278657+0000","last_undegraded":"2023-09-23T23:36:42.278657+0000","last_fullsized":"2023-09-23T23:36:42.278657+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"597'7589","reported_seq":12834,"reported_epoch":597,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:36:52.545542+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:36:52.545542+0000","last_peered":"2023-09-23T23:36:52.545542+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:36:52.545542+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:36:52.545542+0000","mapping_epoch":571,"log_start":"550'2300","ondisk_log_start":"550'2300","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5289,"log_dups_size":0,"ondisk_log_size":5289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226689024,"num_objects":3459,"num_object_clones":0,"num_object_copies":10377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2564,"num_objects_misplaced":895,"num_objects_unfound":0,"num_objects_dirty":3459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7589,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2567,"num_bytes_recovered":168230912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":2442},{"shards":"3,5,7","objects":1017}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":906,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.294498+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:36:32.294498+0000","last_peered":"2023-09-23T23:36:32.294498+0000","last_clean":"2023-09-23T23:36:32.294498+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:36:32.294498+0000","last_undegraded":"2023-09-23T23:36:32.294498+0000","last_fullsized":"2023-09-23T23:36:32.294498+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":334,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.278069+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:36:32.278069+0000","last_peered":"2023-09-23T23:36:32.278069+0000","last_clean":"2023-09-23T23:36:32.278069+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:36:32.278069+0000","last_undegraded":"2023-09-23T23:36:32.278069+0000","last_fullsized":"2023-09-23T23:36:32.278069+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":110,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.290950+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:36:32.290950+0000","last_peered":"2023-09-23T23:36:32.290950+0000","last_clean":"2023-09-23T23:36:32.290950+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:36:32.290950+0000","last_undegraded":"2023-09-23T23:36:32.290950+0000","last_fullsized":"2023-09-23T23:36:32.290950+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"597'7579","reported_seq":9645,"reported_epoch":597,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:52.411982+0000","last_change":"2023-09-23T23:34:35.750428+0000","last_active":"2023-09-23T23:36:52.411982+0000","last_peered":"2023-09-23T23:36:52.411982+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:36:52.411982+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"550'2300","ondisk_log_start":"550'2300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5279,"log_dups_size":0,"ondisk_log_size":5279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230096896,"num_objects":3511,"num_object_clones":0,"num_object_copies":10533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7579,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":991,"num_bytes_recovered":64946176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":2501},{"shards":"3,5","objects":-33}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":504,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.280239+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:36:42.280239+0000","last_peered":"2023-09-23T23:36:42.280239+0000","last_clean":"2023-09-23T23:36:42.280239+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:36:42.280239+0000","last_undegraded":"2023-09-23T23:36:42.280239+0000","last_fullsized":"2023-09-23T23:36:42.280239+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":437,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.279494+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:36:42.279494+0000","last_peered":"2023-09-23T23:36:42.279494+0000","last_clean":"2023-09-23T23:36:42.279494+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:36:42.279494+0000","last_undegraded":"2023-09-23T23:36:42.279494+0000","last_fullsized":"2023-09-23T23:36:42.279494+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":594,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.279128+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:36:42.279128+0000","last_peered":"2023-09-23T23:36:42.279128+0000","last_clean":"2023-09-23T23:36:42.279128+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:36:42.279128+0000","last_undegraded":"2023-09-23T23:36:42.279128+0000","last_fullsized":"2023-09-23T23:36:42.279128+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"596'7657","reported_seq":8983,"reported_epoch":596,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:49.199387+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:36:49.199387+0000","last_peered":"2023-09-23T23:36:49.199387+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:36:49.199387+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"549'1900","ondisk_log_start":"549'1900","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5757,"log_dups_size":0,"ondisk_log_size":5757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229703680,"num_objects":3505,"num_object_clones":0,"num_object_copies":10515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7657,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":2459},{"shards":"3,5","objects":-1955},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":725,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.278961+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:36:42.278961+0000","last_peered":"2023-09-23T23:36:42.278961+0000","last_clean":"2023-09-23T23:36:42.278961+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:36:42.278961+0000","last_undegraded":"2023-09-23T23:36:42.278961+0000","last_fullsized":"2023-09-23T23:36:42.278961+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":607,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.280174+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:36:42.280174+0000","last_peered":"2023-09-23T23:36:42.280174+0000","last_clean":"2023-09-23T23:36:42.280174+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:36:42.280174+0000","last_undegraded":"2023-09-23T23:36:42.280174+0000","last_fullsized":"2023-09-23T23:36:42.280174+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":518,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.279421+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:36:42.279421+0000","last_peered":"2023-09-23T23:36:42.279421+0000","last_clean":"2023-09-23T23:36:42.279421+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:36:42.279421+0000","last_undegraded":"2023-09-23T23:36:42.279421+0000","last_fullsized":"2023-09-23T23:36:42.279421+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"596'7574","reported_seq":7644,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:49.074628+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:36:49.074628+0000","last_peered":"2023-09-23T23:36:49.074628+0000","last_clean":"2023-09-23T23:36:49.074628+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:36:49.074628+0000","last_undegraded":"2023-09-23T23:36:49.074628+0000","last_fullsized":"2023-09-23T23:36:49.074628+0000","mapping_epoch":534,"log_start":"550'2200","ondisk_log_start":"550'2200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5374,"log_dups_size":0,"ondisk_log_size":5374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229769216,"num_objects":3506,"num_object_clones":0,"num_object_copies":10518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7574,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":879,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.262877+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:36:32.262877+0000","last_peered":"2023-09-23T23:36:32.262877+0000","last_clean":"2023-09-23T23:36:32.262877+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:36:32.262877+0000","last_undegraded":"2023-09-23T23:36:32.262877+0000","last_fullsized":"2023-09-23T23:36:32.262877+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":120,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.290889+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:36:32.290889+0000","last_peered":"2023-09-23T23:36:32.290889+0000","last_clean":"2023-09-23T23:36:32.290889+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:36:32.290889+0000","last_undegraded":"2023-09-23T23:36:32.290889+0000","last_fullsized":"2023-09-23T23:36:32.290889+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":462,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.263651+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:36:32.263651+0000","last_peered":"2023-09-23T23:36:32.263651+0000","last_clean":"2023-09-23T23:36:32.263651+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:36:32.263651+0000","last_undegraded":"2023-09-23T23:36:32.263651+0000","last_fullsized":"2023-09-23T23:36:32.263651+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"596'7602","reported_seq":11792,"reported_epoch":596,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:36:49.137256+0000","last_change":"2023-09-23T23:34:35.785270+0000","last_active":"2023-09-23T23:36:49.137256+0000","last_peered":"2023-09-23T23:36:49.137256+0000","last_clean":"2023-09-23T23:33:50.504296+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:36:49.137256+0000","last_undegraded":"2023-09-23T23:34:35.780981+0000","last_fullsized":"2023-09-23T23:36:49.137256+0000","mapping_epoch":567,"log_start":"550'2200","ondisk_log_start":"550'2200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5402,"log_dups_size":0,"ondisk_log_size":5402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229507072,"num_objects":3502,"num_object_clones":0,"num_object_copies":10506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2497,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7602,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2050,"num_bytes_recovered":134348800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":2497},{"shards":"5,6,7","objects":1005}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":604,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.277924+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:36:32.277924+0000","last_peered":"2023-09-23T23:36:32.277924+0000","last_clean":"2023-09-23T23:36:32.277924+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:36:32.277924+0000","last_undegraded":"2023-09-23T23:36:32.277924+0000","last_fullsized":"2023-09-23T23:36:32.277924+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":459,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.263956+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:36:32.263956+0000","last_peered":"2023-09-23T23:36:32.263956+0000","last_clean":"2023-09-23T23:36:32.263956+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:36:32.263956+0000","last_undegraded":"2023-09-23T23:36:32.263956+0000","last_fullsized":"2023-09-23T23:36:32.263956+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":547,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.290158+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:36:32.290158+0000","last_peered":"2023-09-23T23:36:32.290158+0000","last_clean":"2023-09-23T23:36:32.290158+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:36:32.290158+0000","last_undegraded":"2023-09-23T23:36:32.290158+0000","last_fullsized":"2023-09-23T23:36:32.290158+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"596'7496","reported_seq":16529,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:49.347878+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:36:49.347878+0000","last_peered":"2023-09-23T23:36:49.347878+0000","last_clean":"2023-09-23T23:36:49.347878+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:36:49.347878+0000","last_undegraded":"2023-09-23T23:36:49.347878+0000","last_fullsized":"2023-09-23T23:36:49.347878+0000","mapping_epoch":579,"log_start":"549'1800","ondisk_log_start":"549'1800","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5696,"log_dups_size":0,"ondisk_log_size":5696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230686720,"num_objects":3520,"num_object_clones":0,"num_object_copies":10560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7496,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":938,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:39.933421+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:36:39.933421+0000","last_peered":"2023-09-23T23:36:39.933421+0000","last_clean":"2023-09-23T23:36:39.933421+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:36:39.933421+0000","last_undegraded":"2023-09-23T23:36:39.933421+0000","last_fullsized":"2023-09-23T23:36:39.933421+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":238,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.278503+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:36:42.278503+0000","last_peered":"2023-09-23T23:36:42.278503+0000","last_clean":"2023-09-23T23:36:42.278503+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:36:42.278503+0000","last_undegraded":"2023-09-23T23:36:42.278503+0000","last_fullsized":"2023-09-23T23:36:42.278503+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":527,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.280564+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:36:42.280564+0000","last_peered":"2023-09-23T23:36:42.280564+0000","last_clean":"2023-09-23T23:36:42.280564+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:36:42.280564+0000","last_undegraded":"2023-09-23T23:36:42.280564+0000","last_fullsized":"2023-09-23T23:36:42.280564+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"597'7285","reported_seq":7358,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:52.336380+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:36:52.336380+0000","last_peered":"2023-09-23T23:36:52.336380+0000","last_clean":"2023-09-23T23:36:52.336380+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:36:52.336380+0000","last_undegraded":"2023-09-23T23:36:52.336380+0000","last_fullsized":"2023-09-23T23:36:52.336380+0000","mapping_epoch":534,"log_start":"550'2000","ondisk_log_start":"550'2000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5285,"log_dups_size":0,"ondisk_log_size":5285,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225640448,"num_objects":3443,"num_object_clones":0,"num_object_copies":10329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7285,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":917,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.291114+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:36:32.291114+0000","last_peered":"2023-09-23T23:36:32.291114+0000","last_clean":"2023-09-23T23:36:32.291114+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:36:32.291114+0000","last_undegraded":"2023-09-23T23:36:32.291114+0000","last_fullsized":"2023-09-23T23:36:32.291114+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":306,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.330016+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:36:42.330016+0000","last_peered":"2023-09-23T23:36:42.330016+0000","last_clean":"2023-09-23T23:36:42.330016+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:36:42.330016+0000","last_undegraded":"2023-09-23T23:36:42.330016+0000","last_fullsized":"2023-09-23T23:36:42.330016+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":611,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.278259+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:36:32.278259+0000","last_peered":"2023-09-23T23:36:32.278259+0000","last_clean":"2023-09-23T23:36:32.278259+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:36:32.278259+0000","last_undegraded":"2023-09-23T23:36:32.278259+0000","last_fullsized":"2023-09-23T23:36:32.278259+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":192,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.277690+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:36:32.277690+0000","last_peered":"2023-09-23T23:36:32.277690+0000","last_clean":"2023-09-23T23:36:32.277690+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:36:32.277690+0000","last_undegraded":"2023-09-23T23:36:32.277690+0000","last_fullsized":"2023-09-23T23:36:32.277690+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"597'7694","reported_seq":12019,"reported_epoch":597,"state":"active+recovering","last_fresh":"2023-09-23T23:36:52.645060+0000","last_change":"2023-09-23T23:34:35.815566+0000","last_active":"2023-09-23T23:36:52.645060+0000","last_peered":"2023-09-23T23:36:52.645060+0000","last_clean":"2023-09-23T23:33:50.519308+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:36:52.645060+0000","last_undegraded":"2023-09-23T23:36:52.645060+0000","last_fullsized":"2023-09-23T23:36:52.645060+0000","mapping_epoch":566,"log_start":"550'2422","ondisk_log_start":"550'2422","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5272,"log_dups_size":0,"ondisk_log_size":5272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227606552,"num_objects":3474,"num_object_clones":0,"num_object_copies":10422,"num_objects_missing_on_primary":2392,"num_objects_missing":2392,"num_objects_degraded":0,"num_objects_misplaced":2392,"num_objects_unfound":0,"num_objects_dirty":3474,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7694,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2109,"num_bytes_recovered":138215424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,3,4,7","objects":14},{"shards":"1,3,7","objects":1068},{"shards":"1,4,7","objects":2392}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":681,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:45.961527+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:36:45.961527+0000","last_peered":"2023-09-23T23:36:45.961527+0000","last_clean":"2023-09-23T23:36:45.961527+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:36:45.961527+0000","last_undegraded":"2023-09-23T23:36:45.961527+0000","last_fullsized":"2023-09-23T23:36:45.961527+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":244,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.290825+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:36:32.290825+0000","last_peered":"2023-09-23T23:36:32.290825+0000","last_clean":"2023-09-23T23:36:32.290825+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:36:32.290825+0000","last_undegraded":"2023-09-23T23:36:32.290825+0000","last_fullsized":"2023-09-23T23:36:32.290825+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":528,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.290072+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:36:32.290072+0000","last_peered":"2023-09-23T23:36:32.290072+0000","last_clean":"2023-09-23T23:36:32.290072+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:36:32.290072+0000","last_undegraded":"2023-09-23T23:36:32.290072+0000","last_fullsized":"2023-09-23T23:36:32.290072+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":229,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.263098+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:36:32.263098+0000","last_peered":"2023-09-23T23:36:32.263098+0000","last_clean":"2023-09-23T23:36:32.263098+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:36:32.263098+0000","last_undegraded":"2023-09-23T23:36:32.263098+0000","last_fullsized":"2023-09-23T23:36:32.263098+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"596'7301","reported_seq":9350,"reported_epoch":596,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:49.341017+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:36:49.341017+0000","last_peered":"2023-09-23T23:36:49.341017+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:36:49.341017+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"548'1600","ondisk_log_start":"548'1600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227475456,"num_objects":3471,"num_object_clones":0,"num_object_copies":10413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7301,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":2460},{"shards":"5,7","objects":79}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":897,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.264513+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:36:32.264513+0000","last_peered":"2023-09-23T23:36:32.264513+0000","last_clean":"2023-09-23T23:36:32.264513+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:36:32.264513+0000","last_undegraded":"2023-09-23T23:36:32.264513+0000","last_fullsized":"2023-09-23T23:36:32.264513+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":334,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.263876+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:36:32.263876+0000","last_peered":"2023-09-23T23:36:32.263876+0000","last_clean":"2023-09-23T23:36:32.263876+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:36:32.263876+0000","last_undegraded":"2023-09-23T23:36:32.263876+0000","last_fullsized":"2023-09-23T23:36:32.263876+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":244,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.262958+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:36:32.262958+0000","last_peered":"2023-09-23T23:36:32.262958+0000","last_clean":"2023-09-23T23:36:32.262958+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:36:32.262958+0000","last_undegraded":"2023-09-23T23:36:32.262958+0000","last_fullsized":"2023-09-23T23:36:32.262958+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":209,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.330274+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:36:42.330274+0000","last_peered":"2023-09-23T23:36:42.330274+0000","last_clean":"2023-09-23T23:36:42.330274+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:36:42.330274+0000","last_undegraded":"2023-09-23T23:36:42.330274+0000","last_fullsized":"2023-09-23T23:36:42.330274+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"596'7443","reported_seq":7523,"reported_epoch":596,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:36:49.351681+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:36:49.351681+0000","last_peered":"2023-09-23T23:36:49.351681+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:36:49.351681+0000","last_undegraded":"2023-09-23T23:36:49.351681+0000","last_fullsized":"2023-09-23T23:36:49.351681+0000","mapping_epoch":562,"log_start":"548'1700","ondisk_log_start":"548'1700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5743,"log_dups_size":0,"ondisk_log_size":5743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226689024,"num_objects":3459,"num_object_clones":0,"num_object_copies":10377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7443,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":3459}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":611,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.263156+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:36:32.263156+0000","last_peered":"2023-09-23T23:36:32.263156+0000","last_clean":"2023-09-23T23:36:32.263156+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:36:32.263156+0000","last_undegraded":"2023-09-23T23:36:32.263156+0000","last_fullsized":"2023-09-23T23:36:32.263156+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":882,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.289815+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:36:32.289815+0000","last_peered":"2023-09-23T23:36:32.289815+0000","last_clean":"2023-09-23T23:36:32.289815+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:36:32.289815+0000","last_undegraded":"2023-09-23T23:36:32.289815+0000","last_fullsized":"2023-09-23T23:36:32.289815+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":559,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.280638+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:36:42.280638+0000","last_peered":"2023-09-23T23:36:42.280638+0000","last_clean":"2023-09-23T23:36:42.280638+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:36:42.280638+0000","last_undegraded":"2023-09-23T23:36:42.280638+0000","last_fullsized":"2023-09-23T23:36:42.280638+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":120,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.290476+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:36:32.290476+0000","last_peered":"2023-09-23T23:36:32.290476+0000","last_clean":"2023-09-23T23:36:32.290476+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:36:32.290476+0000","last_undegraded":"2023-09-23T23:36:32.290476+0000","last_fullsized":"2023-09-23T23:36:32.290476+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"596'7337","reported_seq":7407,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:49.083114+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:36:49.083114+0000","last_peered":"2023-09-23T23:36:49.083114+0000","last_clean":"2023-09-23T23:36:49.083114+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:36:49.083114+0000","last_undegraded":"2023-09-23T23:36:49.083114+0000","last_fullsized":"2023-09-23T23:36:49.083114+0000","mapping_epoch":534,"log_start":"549'1900","ondisk_log_start":"549'1900","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5437,"log_dups_size":0,"ondisk_log_size":5437,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222756864,"num_objects":3399,"num_object_clones":0,"num_object_copies":10197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7337,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":708,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.289973+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:36:32.289973+0000","last_peered":"2023-09-23T23:36:32.289973+0000","last_clean":"2023-09-23T23:36:32.289973+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:36:32.289973+0000","last_undegraded":"2023-09-23T23:36:32.289973+0000","last_fullsized":"2023-09-23T23:36:32.289973+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":524,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.267206+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:36:42.267206+0000","last_peered":"2023-09-23T23:36:42.267206+0000","last_clean":"2023-09-23T23:36:42.267206+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:36:42.267206+0000","last_undegraded":"2023-09-23T23:36:42.267206+0000","last_fullsized":"2023-09-23T23:36:42.267206+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":438,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.278740+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:36:42.278740+0000","last_peered":"2023-09-23T23:36:42.278740+0000","last_clean":"2023-09-23T23:36:42.278740+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:36:42.278740+0000","last_undegraded":"2023-09-23T23:36:42.278740+0000","last_fullsized":"2023-09-23T23:36:42.278740+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":38,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:45.961577+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:36:45.961577+0000","last_peered":"2023-09-23T23:36:45.961577+0000","last_clean":"2023-09-23T23:36:45.961577+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:36:45.961577+0000","last_undegraded":"2023-09-23T23:36:45.961577+0000","last_fullsized":"2023-09-23T23:36:45.961577+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"596'7534","reported_seq":11203,"reported_epoch":596,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:36:49.724150+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:36:49.724150+0000","last_peered":"2023-09-23T23:36:49.724150+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:36:49.724150+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"548'1400","ondisk_log_start":"548'1400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6134,"log_dups_size":0,"ondisk_log_size":6134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227016704,"num_objects":3464,"num_object_clones":0,"num_object_copies":10392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7534,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":760},{"shards":"1,2,4","objects":2462}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":591,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.290425+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:36:32.290425+0000","last_peered":"2023-09-23T23:36:32.290425+0000","last_clean":"2023-09-23T23:36:32.290425+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:36:32.290425+0000","last_undegraded":"2023-09-23T23:36:32.290425+0000","last_fullsized":"2023-09-23T23:36:32.290425+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":38,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.283862+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:36:42.283862+0000","last_peered":"2023-09-23T23:36:42.283862+0000","last_clean":"2023-09-23T23:36:42.283862+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:36:42.283862+0000","last_undegraded":"2023-09-23T23:36:42.283862+0000","last_fullsized":"2023-09-23T23:36:42.283862+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":136,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.263175+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:36:32.263175+0000","last_peered":"2023-09-23T23:36:32.263175+0000","last_clean":"2023-09-23T23:36:32.263175+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:36:32.263175+0000","last_undegraded":"2023-09-23T23:36:32.263175+0000","last_fullsized":"2023-09-23T23:36:32.263175+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":329,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.278031+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:36:32.278031+0000","last_peered":"2023-09-23T23:36:32.278031+0000","last_clean":"2023-09-23T23:36:32.278031+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:36:32.278031+0000","last_undegraded":"2023-09-23T23:36:32.278031+0000","last_fullsized":"2023-09-23T23:36:32.278031+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"596'7481","reported_seq":7551,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:49.146490+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:36:49.146490+0000","last_peered":"2023-09-23T23:36:49.146490+0000","last_clean":"2023-09-23T23:36:49.146490+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:36:49.146490+0000","last_undegraded":"2023-09-23T23:36:49.146490+0000","last_fullsized":"2023-09-23T23:36:49.146490+0000","mapping_epoch":534,"log_start":"549'1800","ondisk_log_start":"549'1800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5681,"log_dups_size":0,"ondisk_log_size":5681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228261888,"num_objects":3483,"num_object_clones":0,"num_object_copies":10449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7481,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":777,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.263729+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:36:32.263729+0000","last_peered":"2023-09-23T23:36:32.263729+0000","last_clean":"2023-09-23T23:36:32.263729+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:36:32.263729+0000","last_undegraded":"2023-09-23T23:36:32.263729+0000","last_fullsized":"2023-09-23T23:36:32.263729+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":594,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.280487+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:36:42.280487+0000","last_peered":"2023-09-23T23:36:42.280487+0000","last_clean":"2023-09-23T23:36:42.280487+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:36:42.280487+0000","last_undegraded":"2023-09-23T23:36:42.280487+0000","last_fullsized":"2023-09-23T23:36:42.280487+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":200,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.290230+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:36:32.290230+0000","last_peered":"2023-09-23T23:36:32.290230+0000","last_clean":"2023-09-23T23:36:32.290230+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:36:32.290230+0000","last_undegraded":"2023-09-23T23:36:32.290230+0000","last_fullsized":"2023-09-23T23:36:32.290230+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":148,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.356529+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:36:42.356529+0000","last_peered":"2023-09-23T23:36:42.356529+0000","last_clean":"2023-09-23T23:36:42.356529+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:36:42.356529+0000","last_undegraded":"2023-09-23T23:36:42.356529+0000","last_fullsized":"2023-09-23T23:36:42.356529+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"596'7414","reported_seq":7486,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:49.466458+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:36:49.466458+0000","last_peered":"2023-09-23T23:36:49.466458+0000","last_clean":"2023-09-23T23:36:49.466458+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:36:49.466458+0000","last_undegraded":"2023-09-23T23:36:49.466458+0000","last_fullsized":"2023-09-23T23:36:49.466458+0000","mapping_epoch":534,"log_start":"545'1200","ondisk_log_start":"545'1200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6214,"log_dups_size":0,"ondisk_log_size":6214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228982784,"num_objects":3494,"num_object_clones":0,"num_object_copies":10482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7414,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":984,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.279757+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:36:42.279757+0000","last_peered":"2023-09-23T23:36:42.279757+0000","last_clean":"2023-09-23T23:36:42.279757+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:36:42.279757+0000","last_undegraded":"2023-09-23T23:36:42.279757+0000","last_fullsized":"2023-09-23T23:36:42.279757+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":110,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.291245+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:36:32.291245+0000","last_peered":"2023-09-23T23:36:32.291245+0000","last_clean":"2023-09-23T23:36:32.291245+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:36:32.291245+0000","last_undegraded":"2023-09-23T23:36:32.291245+0000","last_fullsized":"2023-09-23T23:36:32.291245+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":42,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:45.961588+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:36:45.961588+0000","last_peered":"2023-09-23T23:36:45.961588+0000","last_clean":"2023-09-23T23:36:45.961588+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:36:45.961588+0000","last_undegraded":"2023-09-23T23:36:45.961588+0000","last_fullsized":"2023-09-23T23:36:45.961588+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":170,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.277681+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:36:32.277681+0000","last_peered":"2023-09-23T23:36:32.277681+0000","last_clean":"2023-09-23T23:36:32.277681+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:36:32.277681+0000","last_undegraded":"2023-09-23T23:36:32.277681+0000","last_fullsized":"2023-09-23T23:36:32.277681+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1033,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.278881+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:36:42.278881+0000","last_peered":"2023-09-23T23:36:42.278881+0000","last_clean":"2023-09-23T23:36:42.278881+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:36:42.278881+0000","last_undegraded":"2023-09-23T23:36:42.278881+0000","last_fullsized":"2023-09-23T23:36:42.278881+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":37,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.279215+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:36:42.279215+0000","last_peered":"2023-09-23T23:36:42.279215+0000","last_clean":"2023-09-23T23:36:42.279215+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:36:42.279215+0000","last_undegraded":"2023-09-23T23:36:42.279215+0000","last_fullsized":"2023-09-23T23:36:42.279215+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":351,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.263248+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:36:32.263248+0000","last_peered":"2023-09-23T23:36:32.263248+0000","last_clean":"2023-09-23T23:36:32.263248+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:36:32.263248+0000","last_undegraded":"2023-09-23T23:36:32.263248+0000","last_fullsized":"2023-09-23T23:36:32.263248+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":745,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.280471+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:36:42.280471+0000","last_peered":"2023-09-23T23:36:42.280471+0000","last_clean":"2023-09-23T23:36:42.280471+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:36:42.280471+0000","last_undegraded":"2023-09-23T23:36:42.280471+0000","last_fullsized":"2023-09-23T23:36:42.280471+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":98,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.290143+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:36:32.290143+0000","last_peered":"2023-09-23T23:36:32.290143+0000","last_clean":"2023-09-23T23:36:32.290143+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:36:32.290143+0000","last_undegraded":"2023-09-23T23:36:32.290143+0000","last_fullsized":"2023-09-23T23:36:32.290143+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":445,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.280090+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:36:42.280090+0000","last_peered":"2023-09-23T23:36:42.280090+0000","last_clean":"2023-09-23T23:36:42.280090+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:36:42.280090+0000","last_undegraded":"2023-09-23T23:36:42.280090+0000","last_fullsized":"2023-09-23T23:36:42.280090+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":294,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.290353+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:36:32.290353+0000","last_peered":"2023-09-23T23:36:32.290353+0000","last_clean":"2023-09-23T23:36:32.290353+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:36:32.290353+0000","last_undegraded":"2023-09-23T23:36:32.290353+0000","last_fullsized":"2023-09-23T23:36:32.290353+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":284,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.280712+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:36:42.280712+0000","last_peered":"2023-09-23T23:36:42.280712+0000","last_clean":"2023-09-23T23:36:42.280712+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:36:42.280712+0000","last_undegraded":"2023-09-23T23:36:42.280712+0000","last_fullsized":"2023-09-23T23:36:42.280712+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":808,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.290384+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:36:32.290384+0000","last_peered":"2023-09-23T23:36:32.290384+0000","last_clean":"2023-09-23T23:36:32.290384+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:36:32.290384+0000","last_undegraded":"2023-09-23T23:36:32.290384+0000","last_fullsized":"2023-09-23T23:36:32.290384+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":627,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.280393+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:36:42.280393+0000","last_peered":"2023-09-23T23:36:42.280393+0000","last_clean":"2023-09-23T23:36:42.280393+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:36:42.280393+0000","last_undegraded":"2023-09-23T23:36:42.280393+0000","last_fullsized":"2023-09-23T23:36:42.280393+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":611,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.279160+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:36:42.279160+0000","last_peered":"2023-09-23T23:36:42.279160+0000","last_clean":"2023-09-23T23:36:42.279160+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:36:42.279160+0000","last_undegraded":"2023-09-23T23:36:42.279160+0000","last_fullsized":"2023-09-23T23:36:42.279160+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":770,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.287661+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:36:42.287661+0000","last_peered":"2023-09-23T23:36:42.287661+0000","last_clean":"2023-09-23T23:36:42.287661+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:36:42.287661+0000","last_undegraded":"2023-09-23T23:36:42.287661+0000","last_fullsized":"2023-09-23T23:36:42.287661+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":440,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T23:36:42.357505+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:36:42.357505+0000","last_peered":"2023-09-23T23:36:42.357505+0000","last_clean":"2023-09-23T23:36:42.357505+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:36:42.357505+0000","last_undegraded":"2023-09-23T23:36:42.357505+0000","last_fullsized":"2023-09-23T23:36:42.357505+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":663,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T23:36:32.289731+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:36:32.289731+0000","last_peered":"2023-09-23T23:36:32.289731+0000","last_clean":"2023-09-23T23:36:32.289731+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:36:32.289731+0000","last_undegraded":"2023-09-23T23:36:32.289731+0000","last_fullsized":"2023-09-23T23:36:32.289731+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":3652583448,"num_objects":55735,"num_object_clones":0,"num_object_copies":167205,"num_objects_missing_on_primary":2392,"num_objects_missing":2392,"num_objects_degraded":24677,"num_objects_misplaced":3287,"num_objects_unfound":0,"num_objects_dirty":55735,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":119821,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15574,"num_bytes_recovered":1020461056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12081508352,"data_stored":12081496136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89899,"ondisk_log_size":89899,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782689,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1672652,"kb_used_data":1303364,"kb_used_omap":8,"kb_used_meta":369271,"kb_avail":92072500,"statfs":{"total":95995035648,"available":94282240000,"internally_reserved":0,"allocated":1334644736,"data_stored":1332982623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":378133573},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":1.5569999999999999,"15min":2.1800000000000002},"min":{"1min":0.23100000000000001,"5min":0.224,"15min":0.20899999999999999},"max":{"1min":0.61699999999999999,"5min":36.956000000000003,"15min":85.721000000000004},"last":0.443},{"interface":"front","average":{"1min":0.47399999999999998,"5min":1.522,"15min":2.2250000000000001},"min":{"1min":0.29199999999999998,"5min":0.27900000000000003,"15min":0.19800000000000001},"max":{"1min":1.4419999999999999,"5min":37.247999999999998,"15min":85.965000000000003},"last":0.38400000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":3.8900000000000001,"15min":3.169},"min":{"1min":0.26600000000000001,"5min":0.248,"15min":0.22900000000000001},"max":{"1min":3.3500000000000001,"5min":129.11199999999999,"15min":198.58000000000001},"last":0.375},{"interface":"front","average":{"1min":0.65600000000000003,"5min":3.665,"15min":3.206},"min":{"1min":0.29699999999999999,"5min":0.247,"15min":0.246},"max":{"1min":3.7290000000000001,"5min":129.02600000000001,"15min":198.71100000000001},"last":0.48099999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":2.871,"15min":3.7010000000000001},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.452,"5min":78.528999999999996,"15min":203.58000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.54200000000000004,"5min":1.952,"15min":3.0649999999999999},"min":{"1min":0.30399999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.2529999999999999,"5min":27.574000000000002,"15min":198.69},"last":0.50800000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":3.3290000000000002,"15min":3.5950000000000002},"min":{"1min":0.31900000000000001,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":1.591,"5min":125.511,"15min":279.26499999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.67500000000000004,"5min":5.3090000000000002,"15min":4.1210000000000004},"min":{"1min":0.374,"5min":0.32000000000000001,"15min":0.29599999999999999},"max":{"1min":3.6240000000000001,"5min":150.608,"15min":279.29899999999998},"last":0.32700000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":1.694,"15min":1.0669999999999999},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":0.68100000000000005,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.363,"15min":1.0029999999999999},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.23899999999999999},"max":{"1min":1.6080000000000001,"5min":31.68,"15min":31.68},"last":0.43099999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":3.637,"5min":1.657,"15min":1.899},"min":{"1min":0.31900000000000001,"5min":0.28699999999999998,"15min":0.26500000000000001},"max":{"1min":60.463999999999999,"5min":60.463999999999999,"15min":210.83600000000001},"last":0.31},{"interface":"front","average":{"1min":3.5990000000000002,"5min":1.6160000000000001,"15min":1.7649999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28199999999999997},"max":{"1min":63.006,"5min":63.006,"15min":207.04599999999999},"last":0.35299999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":1.879,"15min":1.3480000000000001},"min":{"1min":0.35199999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.4419999999999999,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.59299999999999997,"5min":2.2330000000000001,"15min":1.581},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":2.3180000000000001,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.40000000000000002}]}]},{"osd":6,"up_from":46,"seq":197568496165,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1966384,"kb_used_data":1253992,"kb_used_omap":0,"kb_used_meta":712384,"kb_avail":91778768,"statfs":{"total":95995035648,"available":93981458432,"internally_reserved":0,"allocated":1284087808,"data_stored":1282445551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":729481216},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.93500000000000005,"15min":4.6799999999999997},"min":{"1min":0.27700000000000002,"5min":0.251,"15min":0.20399999999999999},"max":{"1min":2.8450000000000002,"5min":8.8119999999999994,"15min":135.31999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.73399999999999999,"15min":4.601},"min":{"1min":0.24399999999999999,"5min":0.23999999999999999,"15min":0.22900000000000001},"max":{"1min":1.8859999999999999,"5min":3.8239999999999998,"15min":135.34299999999999},"last":1.653}]},{"osd":1,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":1.7509999999999999,"15min":2.3759999999999999},"min":{"1min":0.253,"5min":0.251,"15min":0.245},"max":{"1min":1.944,"5min":65.891000000000005,"15min":89.605000000000004},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.64100000000000001,"5min":1.4390000000000001,"15min":2.4510000000000001},"min":{"1min":0.33400000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":2.1269999999999998,"5min":66.087000000000003,"15min":91.025999999999996},"last":1.6970000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":4.9459999999999997,"15min":4.5620000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.861,"5min":97.408000000000001,"15min":127.44499999999999},"last":1.7270000000000001},{"interface":"front","average":{"1min":0.55200000000000005,"5min":6.0880000000000001,"15min":4.9740000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.2120000000000002,"5min":125.539,"15min":125.539},"last":0.318}]},{"osd":3,"last update":"Sat Sep 23 23:35:53 2023","interfaces":[{"interface":"back","average":{"1min":7.923,"5min":7.0129999999999999,"15min":4.4459999999999997},"min":{"1min":0.316,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":73.813999999999993,"5min":138.87299999999999,"15min":138.87299999999999},"last":1.831},{"interface":"front","average":{"1min":8.4450000000000003,"5min":6.9260000000000002,"15min":4.7350000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":73.852999999999994,"5min":138.976,"15min":138.976},"last":0.34399999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.76400000000000001,"15min":0.68300000000000005},"min":{"1min":0.27900000000000003,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":0.57799999999999996,"5min":7.101,"15min":8.1999999999999993},"last":1.76},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.88900000000000001,"15min":0.77600000000000002},"min":{"1min":0.32500000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.54600000000000004,"5min":8.3499999999999996,"15min":8.3499999999999996},"last":1.865}]},{"osd":5,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.82899999999999996,"15min":0.80700000000000005},"min":{"1min":0.34300000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.266,"5min":11.372,"15min":11.473000000000001},"last":1.847},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.98999999999999999,"15min":0.83599999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26900000000000002},"max":{"1min":0.57699999999999996,"5min":7.0129999999999999,"15min":7.5670000000000002},"last":1.8879999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":1.294,"15min":1.0289999999999999},"min":{"1min":0.30499999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":0.70599999999999996,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.38},{"interface":"front","average":{"1min":0.72999999999999998,"5min":1.4019999999999999,"15min":1.0720000000000001},"min":{"1min":0.36099999999999999,"5min":0.35499999999999998,"15min":0.33500000000000002},"max":{"1min":2.665,"5min":40.012999999999998,"15min":40.012999999999998},"last":1.792}]}]},{"osd":1,"up_from":112,"seq":481036337506,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2026484,"kb_used_data":1125740,"kb_used_omap":4,"kb_used_meta":900731,"kb_avail":91718668,"statfs":{"total":95995035648,"available":93919916032,"internally_reserved":0,"allocated":1152757760,"data_stored":1149943719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":922349454},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":1.881,"15min":3.4060000000000001},"min":{"1min":0.217,"5min":0.17799999999999999,"15min":0.16700000000000001},"max":{"1min":2.7970000000000002,"5min":47.195999999999998,"15min":130.768},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":1.5469999999999999,"15min":2.726},"min":{"1min":0.24299999999999999,"5min":0.20499999999999999,"15min":0.19},"max":{"1min":1.016,"5min":46.582000000000001,"15min":134.929},"last":0.24099999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":7.1859999999999999,"15min":6.7910000000000004},"min":{"1min":0.26300000000000001,"5min":0.214,"15min":0.20100000000000001},"max":{"1min":1.8839999999999999,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.65800000000000003,"5min":7.6059999999999999,"15min":6.9160000000000004},"min":{"1min":0.27900000000000003,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":1.968,"5min":202.12,"15min":232.21000000000001},"last":0.20899999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":5.1989999999999998,"15min":5.1799999999999997},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.221},"max":{"1min":0.999,"5min":181.94399999999999,"15min":234.404},"last":71.745000000000005},{"interface":"front","average":{"1min":0.75900000000000001,"5min":5.29,"15min":5.0469999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":6.6829999999999998,"5min":181.98500000000001,"15min":236.33699999999999},"last":71.786000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":1.23,"15min":1.423},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":2.1059999999999999,"5min":12.042999999999999,"15min":12.042999999999999},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.624,"5min":1.129,"15min":1.46},"min":{"1min":0.48099999999999998,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":1.2669999999999999,"5min":9.5069999999999997,"15min":18.108000000000001},"last":0.45900000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":2.9209999999999998,"15min":1.796},"min":{"1min":0.34300000000000003,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66600000000000004,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.68200000000000005,"5min":2.9940000000000002,"15min":2.113},"min":{"1min":0.40899999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.044,"5min":151.821,"15min":151.821},"last":0.372}]},{"osd":6,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.3600000000000001,"15min":1.7529999999999999},"min":{"1min":0.377,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":1.1040000000000001,"5min":8.9269999999999996,"15min":74.087999999999994},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.97699999999999998,"5min":1.585,"15min":1.968},"min":{"1min":0.42999999999999999,"5min":0.35699999999999998,"15min":0.33100000000000002},"max":{"1min":5.367,"5min":11.927,"15min":73.974999999999994},"last":0.34699999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.7390000000000001,"15min":2.6099999999999999},"min":{"1min":0.50600000000000001,"5min":0.38700000000000001,"15min":0.32900000000000001},"max":{"1min":1.3069999999999999,"5min":44.241,"15min":78.308000000000007},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.8240000000000001,"15min":2.484},"min":{"1min":0.37,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":2.1339999999999999,"5min":43.920999999999999,"15min":77.204999999999998},"last":0.41999999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024330,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1504696,"kb_used_data":294448,"kb_used_omap":5,"kb_used_meta":1210234,"kb_avail":92240456,"statfs":{"total":95995035648,"available":94454226944,"internally_reserved":0,"allocated":301514752,"data_stored":298690738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1239280250},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:36:36 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":4.391,"15min":6.6539999999999999},"min":{"1min":0.219,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":1.7709999999999999,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.246},{"interface":"front","average":{"1min":0.43099999999999999,"5min":4.1289999999999996,"15min":5.7969999999999997},"min":{"1min":0.245,"5min":0.20200000000000001,"15min":0.17599999999999999},"max":{"1min":1.0249999999999999,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.219}]},{"osd":2,"last update":"Sat Sep 23 23:35:57 2023","interfaces":[{"interface":"back","average":{"1min":3.2130000000000001,"5min":10.026999999999999,"15min":7.5819999999999999},"min":{"1min":0.26200000000000001,"5min":0.17699999999999999,"15min":0.17199999999999999},"max":{"1min":40.326999999999998,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":3.1720000000000002,"5min":10.18,"15min":7.6520000000000001},"min":{"1min":0.22900000000000001,"5min":0.19800000000000001,"15min":0.17599999999999999},"max":{"1min":40.396000000000001,"5min":283.161,"15min":283.161},"last":2.6419999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:36:36 2023","interfaces":[{"interface":"back","average":{"1min":5.9699999999999998,"5min":5.1630000000000003,"15min":5.6079999999999997},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":95.655000000000001,"5min":114.381,"15min":220.732},"last":72.144999999999996},{"interface":"front","average":{"1min":5.8300000000000001,"5min":5.2779999999999996,"15min":5.6669999999999998},"min":{"1min":0.26600000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":95.695999999999998,"5min":114.30500000000001,"15min":218.78200000000001},"last":72.188999999999993}]},{"osd":4,"last update":"Sat Sep 23 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":1.1599999999999999,"15min":2.4769999999999999},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.70799999999999996,"5min":14.676,"15min":138.81299999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.80100000000000005,"5min":1.391,"15min":3.0830000000000002},"min":{"1min":0.33800000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.7469999999999999,"5min":9.9039999999999999,"15min":171.251},"last":0.39000000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":1.504,"15min":2.7930000000000001},"min":{"1min":0.35299999999999998,"5min":0.32700000000000001,"15min":0.32100000000000001},"max":{"1min":4.6820000000000004,"5min":15.076000000000001,"15min":171.30699999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.625,"5min":1.244,"15min":2.0819999999999999},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":1.8149999999999999,"5min":8.9909999999999997,"15min":139.40799999999999},"last":0.33600000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.325,"15min":3.097},"min":{"1min":0.38100000000000001,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":4.718,"5min":9.968,"15min":171.33600000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.335,"15min":3.3650000000000002},"min":{"1min":0.37,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":1.8700000000000001,"5min":9.8320000000000007,"15min":184.13499999999999},"last":0.54800000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":1.373,"15min":2.6099999999999999},"min":{"1min":0.38400000000000001,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":0.751,"5min":9.452,"15min":138.77699999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.6379999999999999,"15min":2.5030000000000001},"min":{"1min":0.379,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":1.839,"5min":9.9670000000000005,"15min":139.37899999999999},"last":0.373}]}]},{"osd":2,"up_from":199,"seq":854698492151,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3078712,"kb_used_data":2234800,"kb_used_omap":7,"kb_used_meta":843896,"kb_avail":90666440,"statfs":{"total":95995035648,"available":92842434560,"internally_reserved":0,"allocated":2288435200,"data_stored":2285561234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":864149585},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":2.2080000000000002,"15min":3.8599999999999999},"min":{"1min":0.186,"5min":0.186,"15min":0.16700000000000001},"max":{"1min":4.7590000000000003,"5min":61.950000000000003,"15min":119.277},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.61899999999999999,"5min":2.153,"15min":3.6669999999999998},"min":{"1min":0.23999999999999999,"5min":0.20399999999999999,"15min":0.18099999999999999},"max":{"1min":2.5070000000000001,"5min":61.984999999999999,"15min":119.56100000000001},"last":0.42999999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.911,"5min":5.3010000000000002,"15min":4.5940000000000003},"min":{"1min":0.25,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":8.3640000000000008,"5min":106.765,"15min":157.98699999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.79000000000000004,"5min":4.8959999999999999,"15min":4.577},"min":{"1min":0.253,"5min":0.222,"15min":0.222},"max":{"1min":4.0940000000000003,"5min":106.732,"15min":158.03800000000001},"last":0.41299999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":5.7800000000000002,"15min":4.7770000000000001},"min":{"1min":0.255,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":8.3070000000000004,"5min":106.70399999999999,"15min":119.366},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.65800000000000003,"5min":5.4169999999999998,"15min":4.2709999999999999},"min":{"1min":0.29299999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":3.375,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.44900000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":2.036,"15min":2.262},"min":{"1min":0.36099999999999999,"5min":0.33700000000000002,"15min":0.25},"max":{"1min":2.6070000000000002,"5min":62.081000000000003,"15min":119.88200000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.4590000000000001,"5min":2.3860000000000001,"15min":1.865},"min":{"1min":0.36899999999999999,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":8.343,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.60099999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":3.0299999999999998,"15min":2.6480000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.26200000000000001},"max":{"1min":2.548,"5min":91.207999999999998,"15min":119.919},"last":0.64600000000000002},{"interface":"front","average":{"1min":1.4550000000000001,"5min":3.2749999999999999,"15min":2.6160000000000001},"min":{"1min":0.33500000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.3889999999999993,"5min":91.150999999999996,"15min":119.306},"last":0.52900000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":3.5750000000000002,"15min":3.0019999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":2.5819999999999999,"5min":107.018,"15min":119.943},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.486,"5min":2.6440000000000001,"15min":2.1600000000000001},"min":{"1min":0.371,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":8.4060000000000006,"5min":62.493000000000002,"15min":62.493000000000002},"last":0.56299999999999994}]},{"osd":7,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":2.2010000000000001,"15min":1.9179999999999999},"min":{"1min":0.40000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.381,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.505},{"interface":"front","average":{"1min":0.60099999999999998,"5min":3.464,"15min":2.3620000000000001},"min":{"1min":0.42699999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.417,"5min":106.809,"15min":106.809},"last":0.54800000000000004}]}]},{"osd":3,"up_from":148,"seq":635655160120,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2128312,"kb_used_data":1190576,"kb_used_omap":2,"kb_used_meta":937725,"kb_avail":91616840,"statfs":{"total":95995035648,"available":93815644160,"internally_reserved":0,"allocated":1219149824,"data_stored":1216337256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":960230453},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,1,0,1,2,6,5],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.1359999999999999,"15min":2.6019999999999999},"min":{"1min":0.184,"5min":0.184,"15min":0.17699999999999999},"max":{"1min":6.7699999999999996,"5min":17.213000000000001,"15min":103.866},"last":0.19500000000000001},{"interface":"front","average":{"1min":0.72899999999999998,"5min":1.4139999999999999,"15min":2.855},"min":{"1min":0.218,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":3.4980000000000002,"5min":22.754000000000001,"15min":101.884},"last":0.22600000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":2.5960000000000001,"15min":2.903},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":1.649,"5min":56.119,"15min":85.242999999999995},"last":0.246},{"interface":"front","average":{"1min":0.42299999999999999,"5min":2.036,"15min":2.7469999999999999},"min":{"1min":0.254,"5min":0.224,"15min":0.20899999999999999},"max":{"1min":1.6990000000000001,"5min":53.962000000000003,"15min":85.302000000000007},"last":0.29999999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":4.7770000000000001,"15min":4.7859999999999996},"min":{"1min":0.25800000000000001,"5min":0.25,"15min":0.19800000000000001},"max":{"1min":2.1190000000000002,"5min":73.742000000000004,"15min":101.279},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.45000000000000001,"5min":4.6349999999999998,"15min":4.6130000000000004},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.214},"max":{"1min":0.86699999999999999,"5min":76.525999999999996,"15min":101.294},"last":0.33700000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":0.68999999999999995,"5min":11.209,"15min":11.209},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.44500000000000001,"5min":1.3720000000000001,"15min":1.6719999999999999},"min":{"1min":0.33000000000000002,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":0.67900000000000005,"5min":11.494999999999999,"15min":24.782},"last":0.48599999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":6.3570000000000002,"15min":3.3220000000000001},"min":{"1min":0.33000000000000002,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":0.622,"5min":214.43700000000001,"15min":214.43700000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":6.3970000000000002,"15min":3.3969999999999998},"min":{"1min":0.36499999999999999,"5min":0.34699999999999998,"15min":0.30599999999999999},"max":{"1min":1.716,"5min":214.46600000000001,"15min":214.46600000000001},"last":0.53000000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":2.218,"15min":2.1739999999999999},"min":{"1min":0.35399999999999998,"5min":0.318,"15min":0.318},"max":{"1min":2.008,"5min":58.932000000000002,"15min":58.932000000000002},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.629,"5min":2.492,"15min":2.2210000000000001},"min":{"1min":0.36699999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.7450000000000001,"5min":59.186,"15min":59.186},"last":0.42999999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":4.7480000000000002,"15min":3.2069999999999999},"min":{"1min":0.38700000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.415,"5min":192.69,"15min":192.69},"last":0.502},{"interface":"front","average":{"1min":0.753,"5min":5.1289999999999996,"15min":3.6949999999999998},"min":{"1min":0.43099999999999999,"5min":0.39200000000000002,"15min":0.36699999999999999},"max":{"1min":2.6560000000000001,"5min":192.65600000000001,"15min":192.65600000000001},"last":0.51700000000000002}]}]},{"osd":4,"up_from":0,"seq":128849019438,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757801,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2813932,"kb_used_data":2087780,"kb_used_omap":0,"kb_used_meta":726144,"kb_avail":90931220,"statfs":{"total":95995035648,"available":93113569280,"internally_reserved":0,"allocated":2137886720,"data_stored":2136244680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":743571456},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:14 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":1.141,"15min":3.198},"min":{"1min":0.252,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":1.5640000000000001,"5min":21.109000000000002,"15min":113.869},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.38,"5min":1.3939999999999999,"15min":3.5710000000000002},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.22700000000000001},"max":{"1min":0.76200000000000001,"5min":20.876999999999999,"15min":138.55199999999999},"last":0.41799999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:36:20 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.6200000000000001,"15min":2.258},"min":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.23100000000000001},"max":{"1min":1.9930000000000001,"5min":29.007999999999999,"15min":69.816999999999993},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.8,"15min":2.5840000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":2.8130000000000002,"5min":29.053999999999998,"15min":70.268000000000001},"last":0.58499999999999996}]},{"osd":2,"last update":"Sat Sep 23 23:36:48 2023","interfaces":[{"interface":"back","average":{"1min":5.194,"5min":6.984,"15min":4.8159999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.26900000000000002},"max":{"1min":60.441000000000003,"5min":208.738,"15min":208.738},"last":21.533000000000001},{"interface":"front","average":{"1min":5.2839999999999998,"5min":5.7190000000000003,"15min":4.3259999999999996},"min":{"1min":0.29999999999999999,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":60.520000000000003,"5min":208.631,"15min":208.631},"last":20.835000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:36:48 2023","interfaces":[{"interface":"back","average":{"1min":4.1749999999999998,"5min":4.6360000000000001,"15min":3.8100000000000001},"min":{"1min":0.34999999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":37.878,"5min":122.071,"15min":122.071},"last":20.785},{"interface":"front","average":{"1min":4.2460000000000004,"5min":4.9829999999999997,"15min":3.984},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.318},"max":{"1min":37.853000000000002,"5min":122.09699999999999,"15min":122.09699999999999},"last":20.800999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.98699999999999999,"15min":0.90700000000000003},"min":{"1min":0.32900000000000001,"5min":0.25700000000000001,"15min":0.251},"max":{"1min":0.749,"5min":9.2690000000000001,"15min":19.962},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.502,"5min":1.0129999999999999,"15min":0.81599999999999995},"min":{"1min":0.314,"5min":0.29599999999999999,"15min":0.22900000000000001},"max":{"1min":1.595,"5min":10.273999999999999,"15min":20.829999999999998},"last":0.434}]},{"osd":6,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":2.29,"15min":1.5049999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.26900000000000002},"max":{"1min":1.619,"5min":148.36600000000001,"15min":148.36600000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":2.798,"15min":1.7250000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.26000000000000001},"max":{"1min":2.1549999999999998,"5min":148.42500000000001,"15min":148.42500000000001},"last":0.48099999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:36:48 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.2629999999999999,"15min":2.3879999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":2.29,"5min":15.497,"15min":179.28800000000001},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.64900000000000002,"5min":1.2729999999999999,"15min":2.46},"min":{"1min":0.36699999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.0630000000000002,"5min":15.653,"15min":179.428},"last":0.56000000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":293011456,"data_stored":293011456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1141903360,"data_stored":1141899288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2279800832,"data_stored":2279800832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1208881152,"data_stored":1208877080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2428174336,"data_stored":2428174336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2127101952,"data_stored":2127101952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1276313600,"data_stored":1276313600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1326321664,"data_stored":1326317592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:36:56.504 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:36:56.504 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:56 smithi110 ceph-mon[221102]: from='client.56824 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:56.550 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:56.628 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:56.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:56 smithi138 ceph-mon[193606]: from='client.56824 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:36:56.952 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:56.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:56.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:57.057 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:57.086 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:36:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:36:56] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-09-23T23:36:57.089 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:57.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:57 smithi110 ceph-mon[212939]: pgmap v1669: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 215 op/s; 24621/165066 objects degraded (14.916%); 3287/165066 objects misplaced (1.991%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:36:57.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:57 smithi110 ceph-mon[221102]: pgmap v1669: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 215 op/s; 24621/165066 objects degraded (14.916%); 3287/165066 objects misplaced (1.991%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:36:57.462 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:57.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:57.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:57.570 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:36:57.595 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:57 smithi138 ceph-mon[193606]: pgmap v1669: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 215 op/s; 24621/165066 objects degraded (14.916%); 3287/165066 objects misplaced (1.991%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:36:57.597 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:57.642 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:36:57.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:57.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:36:57.749 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:57.791 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:58.109 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:58.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:58.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:58.217 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:36:58.250 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:58.252 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:36:58.300 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:36:58.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:58.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:36:58.406 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:58.437 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:58.609 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:36:58.764 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:36:58.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:58.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:36:58.876 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:36:58.904 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:58.949 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:36:58.949 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:36:58.955 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:36:58.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:58.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:36:59.061 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:36:59.112 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:59.169 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:36:59.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:59.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:36:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:59 smithi138 ceph-mon[193606]: pgmap v1670: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 218 op/s; 24533/164034 objects degraded (14.956%); 3203/164034 objects misplaced (1.953%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:59 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:36:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:59 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:36:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:59 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 24533/164034 objects degraded (14.956%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:59 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3316112754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:36:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:36:59 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:36:59.274 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:36:59.301 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:59.347 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:36:59.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:36:59.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:36:59.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:59 smithi110 ceph-mon[212939]: pgmap v1670: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 218 op/s; 24533/164034 objects degraded (14.956%); 3203/164034 objects misplaced (1.953%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:59 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:36:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:59 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:36:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:59 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 24533/164034 objects degraded (14.956%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:59 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3316112754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:36:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:36:59 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:36:59.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:59 smithi110 ceph-mon[221102]: pgmap v1670: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 218 op/s; 24533/164034 objects degraded (14.956%); 3203/164034 objects misplaced (1.953%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:36:59.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:59 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:36:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:59 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:36:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:59 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 24533/164034 objects degraded (14.956%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:36:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:59 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3316112754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:36:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:36:59 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:36:59.454 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:36:59.483 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:36:59.777 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:37:00.093 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:00 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:37:00.093 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:00 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:00.094 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:00 smithi110 ceph-mon[212939]: failed to encode map e598 with expected crc 2023-09-23T23:37:00.094 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:00 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:37:00.094 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:00 smithi110 ceph-mon[212939]: osdmap e598: 8 total, 8 up, 7 in 2023-09-23T23:37:00.094 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:00 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:37:00.094 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:00 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:00.094 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:00 smithi110 ceph-mon[221102]: failed to encode map e598 with expected crc 2023-09-23T23:37:00.094 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:00 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:37:00.094 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:00 smithi110 ceph-mon[221102]: osdmap e598: 8 total, 8 up, 7 in 2023-09-23T23:37:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:00 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:37:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:00 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:00 smithi138 ceph-mon[193606]: failed to encode map e598 with expected crc 2023-09-23T23:37:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:00 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:37:00.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:00 smithi138 ceph-mon[193606]: osdmap e598: 8 total, 8 up, 7 in 2023-09-23T23:37:00.451 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:37:01.102 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:37:01.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:01.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:37:01.146 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:01.215 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:37:01.257 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:01.313 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:01.373 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:37:01.375 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:01 smithi110 ceph-mon[212939]: failed to encode map e598 with expected crc 2023-09-23T23:37:01.375 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:01 smithi110 ceph-mon[212939]: failed to encode map e598 with expected crc 2023-09-23T23:37:01.375 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:01 smithi110 ceph-mon[212939]: pgmap v1672: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 224 op/s; 24410/161580 objects degraded (15.107%); 3203/161580 objects misplaced (1.982%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T23:37:01.376 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:01 smithi110 ceph-mon[221102]: failed to encode map e598 with expected crc 2023-09-23T23:37:01.376 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:01 smithi110 ceph-mon[221102]: failed to encode map e598 with expected crc 2023-09-23T23:37:01.376 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:01 smithi110 ceph-mon[221102]: pgmap v1672: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 224 op/s; 24410/161580 objects degraded (15.107%); 3203/161580 objects misplaced (1.982%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T23:37:01.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:01.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:37:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:01 smithi138 ceph-mon[193606]: failed to encode map e598 with expected crc 2023-09-23T23:37:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:01 smithi138 ceph-mon[193606]: failed to encode map e598 with expected crc 2023-09-23T23:37:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:01 smithi138 ceph-mon[193606]: pgmap v1672: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 224 op/s; 24410/161580 objects degraded (15.107%); 3203/161580 objects misplaced (1.982%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T23:37:01.478 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:37:01.510 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:01.622 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:37:01.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:01.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:37:01.730 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:01.761 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:01.923 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:01.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:01.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:02.030 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:37:02.095 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:02.095 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:37:02.095 INFO:teuthology.orchestra.run.smithi110.stdout:{"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 24410/160707 objects degraded (15.189%), 7 pgs degraded, 5 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2099,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":598,"num_osds":8,"num_up_osds":8,"osd_up_since":1695511414,"num_in_osds":7,"osd_in_since":1695512030,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":110},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+recovering","count":1},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":119,"num_pools":5,"num_objects":53569,"data_bytes":3509695733,"bytes_used":15205281792,"bytes_avail":656759967744,"bytes_total":671965249536,"degraded_objects":24410,"degraded_total":160707,"degraded_ratio":0.15189133018474615,"misplaced_objects":3203,"misplaced_total":160707,"misplaced_ratio":0.019930681301996801,"recovering_objects_per_sec":49,"recovering_bytes_per_sec":3275789,"recovering_keys_per_sec":0,"num_objects_recovered":500,"num_bytes_recovered":32768000,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":245},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi110.fuglpi","status":"up:active","gid":24572}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.110:8443/","prometheus":"http://172.21.15.110:9283/"}},"servicemap":{"epoch":32,"modified":"2023-09-23T23:32:57.852217+0000","services":{}},"progress_events":{"833ef15d-07ae-4299-b4e4-2cf94e068147":{"message":"Upgrade to 18.2.0-460-gbb0daeff (10m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true},"c67c6d89-9cd0-495b-b1de-fa24b5d12727":{"message":"Global Recovery Event (2m)\n [=========================...] (remaining: 13s)","progress":0.92436975240707397,"add_to_ceph_s":true}}} 2023-09-23T23:37:03.053 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:37:03.256 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:37:03.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:03.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:37:03.366 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:03.394 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:03 smithi138 ceph-mon[193606]: pgmap v1673: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 245 op/s; 24410/160707 objects degraded (15.189%); 3203/160707 objects misplaced (1.993%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:37:03.394 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:03 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1860753117' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:37:03.397 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:03.417 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:03 smithi110 ceph-mon[212939]: pgmap v1673: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 245 op/s; 24410/160707 objects degraded (15.189%); 3203/160707 objects misplaced (1.993%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:37:03.417 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:03 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1860753117' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:37:03.425 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:03 smithi110 ceph-mon[221102]: pgmap v1673: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 245 op/s; 24410/160707 objects degraded (15.189%); 3203/160707 objects misplaced (1.993%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:37:03.425 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:03 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1860753117' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:37:03.461 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:03.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:03.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:03.567 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:03.594 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:03.787 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:37:03.788 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:37:04.094 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:04.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:04.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:04.205 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:04.236 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:04.303 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:04.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:04.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:04.416 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:37:04.441 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:04 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:04.441 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:04 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:04.441 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:04 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 24410/160707 objects degraded (15.189%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:04.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:04 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:04.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:04 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:04.442 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:04 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 24410/160707 objects degraded (15.189%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:04.444 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:04.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:04 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:04.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:04 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:04.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:04 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 24410/160707 objects degraded (15.189%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:04.747 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:37:04.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:04.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:37:04.853 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:37:04.880 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:04.931 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:37:04.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:04.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:37:05.036 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:37:05.062 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:05.105 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:37:05.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:05.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:37:05.150 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:05.153 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:05.210 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:05.235 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:37:05.250 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:05.272 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:37:05.312 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:37:05.343 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:37:05.365 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:37:05.401 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:37:05.421 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:37:05.448 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:05.452 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:37:05.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:05.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:05.454 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:37:05.454 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:37:05.454 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:37:05.454 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:37:05.454 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:37:05.454 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:37:05.454 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:37:05.454 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:37:05.554 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:05.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:05 smithi138 ceph-mon[193606]: pgmap v1674: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 255 op/s; 24315/159603 objects degraded (15.235%); 3116/159603 objects misplaced (1.952%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T23:37:05.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:05 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2732159339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:05.587 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:05.599 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:05 smithi110 ceph-mon[221102]: pgmap v1674: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 255 op/s; 24315/159603 objects degraded (15.235%); 3116/159603 objects misplaced (1.952%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T23:37:05.599 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:05 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2732159339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:05.603 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:05 smithi110 ceph-mon[212939]: pgmap v1674: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 255 op/s; 24315/159603 objects degraded (15.235%); 3116/159603 objects misplaced (1.952%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T23:37:05.603 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:05 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2732159339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:05.621 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:37:05.621 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:37:05.621 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:37:05.621 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:37:05.621 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:37:05.640 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:05.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:05.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:05.746 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:37:05.780 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:05.837 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:37:05.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:05.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:37:05.944 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:05.978 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:06.038 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:06.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:06.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:06.146 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:06.175 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:06.232 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:06.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:06.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:06.339 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:37:06.374 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:07.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:37:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:37:06] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2023-09-23T23:37:07.634 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:37:07.636 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:07 smithi110 ceph-mon[212939]: pgmap v1675: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 282 op/s; 24250/156588 objects degraded (15.486%); 3116/156588 objects misplaced (1.990%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T23:37:07.636 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:07 smithi110 ceph-mon[221102]: pgmap v1675: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 282 op/s; 24250/156588 objects degraded (15.486%); 3116/156588 objects misplaced (1.990%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T23:37:07.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:07.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:37:07.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:07 smithi138 ceph-mon[193606]: pgmap v1675: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 282 op/s; 24250/156588 objects degraded (15.486%); 3116/156588 objects misplaced (1.990%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T23:37:07.738 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:07.768 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:07.814 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:07.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:07.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:07.919 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:07.983 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:08.011 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:37:08.192 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 23:37:08 smithi138 conmon[172649]: logger=cleanup t=2023-09-23T23:37:08.001225402Z level=info msg="Completed cleanup jobs" duration=66.218278ms 2023-09-23T23:37:08.746 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:08.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:08.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:08.851 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:08.882 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:08.930 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:08.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:08.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:09.038 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:09.068 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:09.128 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:09.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:09.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:09.236 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:09.260 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:09.264 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:09.319 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:09.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:09.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:09.353 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:37:09.426 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:37:09.436 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:37:09.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:09 smithi138 ceph-mon[193606]: pgmap v1676: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 248 op/s; 24250/156588 objects degraded (15.486%); 3116/156588 objects misplaced (1.990%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T23:37:09.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:09 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 24250/156588 objects degraded (15.486%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:09.454 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:37:09.492 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:09 smithi110 ceph-mon[221102]: pgmap v1676: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 248 op/s; 24250/156588 objects degraded (15.486%); 3116/156588 objects misplaced (1.990%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T23:37:09.492 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:09 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 24250/156588 objects degraded (15.486%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:09.493 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:09 smithi110 ceph-mon[212939]: pgmap v1676: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 248 op/s; 24250/156588 objects degraded (15.486%); 3116/156588 objects misplaced (1.990%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T23:37:09.493 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:09 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 24250/156588 objects degraded (15.486%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:09.496 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:10.253 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:37:10.500 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:37:10.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:10.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:37:10.504 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:10 smithi110 ceph-mon[212939]: from='client.56851 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:37:10.504 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:10 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/114498514' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:10.504 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:10 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:10.505 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:10 smithi110 ceph-mon[221102]: from='client.56851 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:37:10.505 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:10 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/114498514' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:10.505 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:10 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:10.604 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:10.630 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:10 smithi138 ceph-mon[193606]: from='client.56851 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:37:10.630 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:10 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/114498514' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:10.630 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:10 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:10.632 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:10.689 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:10.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:10.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:10.794 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:10.820 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:10.862 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:10.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:10.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:10.968 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:37:10.997 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:11.054 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:37:11.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:11.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:37:11.160 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:37:11.204 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:11.249 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:11 smithi110 ceph-mon[221102]: pgmap v1677: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 300 op/s; 24038/152412 objects degraded (15.772%); 3027/152412 objects misplaced (1.986%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T23:37:11.250 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:11 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:11.250 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:11 smithi110 ceph-mon[221102]: failed to encode map e599 with expected crc 2023-09-23T23:37:11.250 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:11 smithi110 ceph-mon[221102]: failed to encode map e599 with expected crc 2023-09-23T23:37:11.250 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:11 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:37:11.250 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:11 smithi110 ceph-mon[221102]: osdmap e599: 8 total, 8 up, 7 in 2023-09-23T23:37:11.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:11 smithi138 ceph-mon[193606]: pgmap v1677: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 300 op/s; 24038/152412 objects degraded (15.772%); 3027/152412 objects misplaced (1.986%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T23:37:11.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:11 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:11.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:11 smithi138 ceph-mon[193606]: failed to encode map e599 with expected crc 2023-09-23T23:37:11.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:11 smithi138 ceph-mon[193606]: failed to encode map e599 with expected crc 2023-09-23T23:37:11.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:11 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:37:11.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:11 smithi138 ceph-mon[193606]: osdmap e599: 8 total, 8 up, 7 in 2023-09-23T23:37:11.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:11 smithi110 ceph-mon[212939]: pgmap v1677: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 300 op/s; 24038/152412 objects degraded (15.772%); 3027/152412 objects misplaced (1.986%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T23:37:11.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:11 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:11.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:11 smithi110 ceph-mon[212939]: failed to encode map e599 with expected crc 2023-09-23T23:37:11.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:11 smithi110 ceph-mon[212939]: failed to encode map e599 with expected crc 2023-09-23T23:37:11.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:11 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:37:11.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:11 smithi110 ceph-mon[212939]: osdmap e599: 8 total, 8 up, 7 in 2023-09-23T23:37:11.909 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:37:11.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:11.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:37:11.939 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1676,"stamp":"2023-09-23T23:37:07.935196+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3419714805,"num_objects":52196,"num_object_clones":0,"num_object_copies":156588,"num_objects_missing_on_primary":2221,"num_objects_missing":2221,"num_objects_degraded":24250,"num_objects_misplaced":3116,"num_objects_unfound":0,"num_objects_dirty":52196,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":123461,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16233,"num_bytes_recovered":1061155240,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95008,"ondisk_log_size":95008,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14636312,"kb_used_data":8928800,"kb_used_omap":29,"kb_used_meta":5707426,"kb_avail":641579752,"statfs":{"total":671965249536,"available":656977666048,"internally_reserved":0,"allocated":9143091200,"data_stored":9126770225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5844404947},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-162660352,"num_objects":-2482,"num_object_clones":0,"num_object_copies":-7446,"num_objects_missing_on_primary":-87,"num_objects_missing":-87,"num_objects_degraded":-283,"num_objects_misplaced":-87,"num_objects_unfound":0,"num_objects_dirty":-2482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2482,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":373,"num_bytes_recovered":24444928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004086"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":794,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.298258+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:36:50.298258+0000","last_peered":"2023-09-23T23:36:50.298258+0000","last_clean":"2023-09-23T23:36:50.298258+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:36:50.298258+0000","last_undegraded":"2023-09-23T23:36:50.298258+0000","last_fullsized":"2023-09-23T23:36:50.298258+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":353,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.301027+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:36:50.301027+0000","last_peered":"2023-09-23T23:36:50.301027+0000","last_clean":"2023-09-23T23:36:50.301027+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:36:50.301027+0000","last_undegraded":"2023-09-23T23:36:50.301027+0000","last_fullsized":"2023-09-23T23:36:50.301027+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":580,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.335865+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:36:50.335865+0000","last_peered":"2023-09-23T23:36:50.335865+0000","last_clean":"2023-09-23T23:36:50.335865+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:36:50.335865+0000","last_undegraded":"2023-09-23T23:36:50.335865+0000","last_fullsized":"2023-09-23T23:36:50.335865+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":747,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.298471+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:36:50.298471+0000","last_peered":"2023-09-23T23:36:50.298471+0000","last_clean":"2023-09-23T23:36:50.298471+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:36:50.298471+0000","last_undegraded":"2023-09-23T23:36:50.298471+0000","last_fullsized":"2023-09-23T23:36:50.298471+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":317,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.300078+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:36:50.300078+0000","last_peered":"2023-09-23T23:36:50.300078+0000","last_clean":"2023-09-23T23:36:50.300078+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:36:50.300078+0000","last_undegraded":"2023-09-23T23:36:50.300078+0000","last_fullsized":"2023-09-23T23:36:50.300078+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":559,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.305703+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:36:50.305703+0000","last_peered":"2023-09-23T23:36:50.305703+0000","last_clean":"2023-09-23T23:36:50.305703+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:36:50.305703+0000","last_undegraded":"2023-09-23T23:36:50.305703+0000","last_fullsized":"2023-09-23T23:36:50.305703+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":759,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.289768+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:36:50.289768+0000","last_peered":"2023-09-23T23:36:50.289768+0000","last_clean":"2023-09-23T23:36:50.289768+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:36:50.289768+0000","last_undegraded":"2023-09-23T23:36:50.289768+0000","last_fullsized":"2023-09-23T23:36:50.289768+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":616,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.299420+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:36:50.299420+0000","last_peered":"2023-09-23T23:36:50.299420+0000","last_clean":"2023-09-23T23:36:50.299420+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:36:50.299420+0000","last_undegraded":"2023-09-23T23:36:50.299420+0000","last_fullsized":"2023-09-23T23:36:50.299420+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":112,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.293512+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:36:50.293512+0000","last_peered":"2023-09-23T23:36:50.293512+0000","last_clean":"2023-09-23T23:36:50.293512+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:36:50.293512+0000","last_undegraded":"2023-09-23T23:36:50.293512+0000","last_fullsized":"2023-09-23T23:36:50.293512+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":918,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.293429+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:36:50.293429+0000","last_peered":"2023-09-23T23:36:50.293429+0000","last_clean":"2023-09-23T23:36:50.293429+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:36:50.293429+0000","last_undegraded":"2023-09-23T23:36:50.293429+0000","last_fullsized":"2023-09-23T23:36:50.293429+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":250,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.299281+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:36:50.299281+0000","last_peered":"2023-09-23T23:36:50.299281+0000","last_clean":"2023-09-23T23:36:50.299281+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:36:50.299281+0000","last_undegraded":"2023-09-23T23:36:50.299281+0000","last_fullsized":"2023-09-23T23:36:50.299281+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":599,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.298849+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:36:50.298849+0000","last_peered":"2023-09-23T23:36:50.298849+0000","last_clean":"2023-09-23T23:36:50.298849+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:36:50.298849+0000","last_undegraded":"2023-09-23T23:36:50.298849+0000","last_fullsized":"2023-09-23T23:36:50.298849+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":891,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.297157+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:36:50.297157+0000","last_peered":"2023-09-23T23:36:50.297157+0000","last_clean":"2023-09-23T23:36:50.297157+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:36:50.297157+0000","last_undegraded":"2023-09-23T23:36:50.297157+0000","last_fullsized":"2023-09-23T23:36:50.297157+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":229,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.299118+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:36:50.299118+0000","last_peered":"2023-09-23T23:36:50.299118+0000","last_clean":"2023-09-23T23:36:50.299118+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:36:50.299118+0000","last_undegraded":"2023-09-23T23:36:50.299118+0000","last_fullsized":"2023-09-23T23:36:50.299118+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":560,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.299874+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:36:50.299874+0000","last_peered":"2023-09-23T23:36:50.299874+0000","last_clean":"2023-09-23T23:36:50.299874+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:36:50.299874+0000","last_undegraded":"2023-09-23T23:36:50.299874+0000","last_fullsized":"2023-09-23T23:36:50.299874+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1052,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.299153+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:36:50.299153+0000","last_peered":"2023-09-23T23:36:50.299153+0000","last_clean":"2023-09-23T23:36:50.299153+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:36:50.299153+0000","last_undegraded":"2023-09-23T23:36:50.299153+0000","last_fullsized":"2023-09-23T23:36:50.299153+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":271,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.300032+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:36:50.300032+0000","last_peered":"2023-09-23T23:36:50.300032+0000","last_clean":"2023-09-23T23:36:50.300032+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:36:50.300032+0000","last_undegraded":"2023-09-23T23:36:50.300032+0000","last_fullsized":"2023-09-23T23:36:50.300032+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":131,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.290359+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:36:50.290359+0000","last_peered":"2023-09-23T23:36:50.290359+0000","last_clean":"2023-09-23T23:36:50.290359+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:36:50.290359+0000","last_undegraded":"2023-09-23T23:36:50.290359+0000","last_fullsized":"2023-09-23T23:36:50.290359+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":449,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.305846+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:36:50.305846+0000","last_peered":"2023-09-23T23:36:50.305846+0000","last_clean":"2023-09-23T23:36:50.305846+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:36:50.305846+0000","last_undegraded":"2023-09-23T23:36:50.305846+0000","last_fullsized":"2023-09-23T23:36:50.305846+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":172,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.290365+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:36:50.290365+0000","last_peered":"2023-09-23T23:36:50.290365+0000","last_clean":"2023-09-23T23:36:50.290365+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:36:50.290365+0000","last_undegraded":"2023-09-23T23:36:50.290365+0000","last_fullsized":"2023-09-23T23:36:50.290365+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":539,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.290383+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:36:50.290383+0000","last_peered":"2023-09-23T23:36:50.290383+0000","last_clean":"2023-09-23T23:36:50.290383+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:36:50.290383+0000","last_undegraded":"2023-09-23T23:36:50.290383+0000","last_fullsized":"2023-09-23T23:36:50.290383+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":753,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.298093+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:36:50.298093+0000","last_peered":"2023-09-23T23:36:50.298093+0000","last_clean":"2023-09-23T23:36:50.298093+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:36:50.298093+0000","last_undegraded":"2023-09-23T23:36:50.298093+0000","last_fullsized":"2023-09-23T23:36:50.298093+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":145,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.298176+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:36:50.298176+0000","last_peered":"2023-09-23T23:36:50.298176+0000","last_clean":"2023-09-23T23:36:50.298176+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:36:50.298176+0000","last_undegraded":"2023-09-23T23:36:50.298176+0000","last_fullsized":"2023-09-23T23:36:50.298176+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":598,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.305088+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:36:50.305088+0000","last_peered":"2023-09-23T23:36:50.305088+0000","last_clean":"2023-09-23T23:36:50.305088+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:36:50.305088+0000","last_undegraded":"2023-09-23T23:36:50.305088+0000","last_fullsized":"2023-09-23T23:36:50.305088+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":883,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.289732+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:36:50.289732+0000","last_peered":"2023-09-23T23:36:50.289732+0000","last_clean":"2023-09-23T23:36:50.289732+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:36:50.289732+0000","last_undegraded":"2023-09-23T23:36:50.289732+0000","last_fullsized":"2023-09-23T23:36:50.289732+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":193,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.306772+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:36:50.306772+0000","last_peered":"2023-09-23T23:36:50.306772+0000","last_clean":"2023-09-23T23:36:50.306772+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:36:50.306772+0000","last_undegraded":"2023-09-23T23:36:50.306772+0000","last_fullsized":"2023-09-23T23:36:50.306772+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":320,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.335758+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:36:50.335758+0000","last_peered":"2023-09-23T23:36:50.335758+0000","last_clean":"2023-09-23T23:36:50.335758+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:36:50.335758+0000","last_undegraded":"2023-09-23T23:36:50.335758+0000","last_fullsized":"2023-09-23T23:36:50.335758+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":561,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.304745+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:36:50.304745+0000","last_peered":"2023-09-23T23:36:50.304745+0000","last_clean":"2023-09-23T23:36:50.304745+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:36:50.304745+0000","last_undegraded":"2023-09-23T23:36:50.304745+0000","last_fullsized":"2023-09-23T23:36:50.304745+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":466,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.292001+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:36:50.292001+0000","last_peered":"2023-09-23T23:36:50.292001+0000","last_clean":"2023-09-23T23:36:50.292001+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:36:50.292001+0000","last_undegraded":"2023-09-23T23:36:50.292001+0000","last_fullsized":"2023-09-23T23:36:50.292001+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":554,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.305983+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:36:50.305983+0000","last_peered":"2023-09-23T23:36:50.305983+0000","last_clean":"2023-09-23T23:36:50.305983+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:36:50.305983+0000","last_undegraded":"2023-09-23T23:36:50.305983+0000","last_fullsized":"2023-09-23T23:36:50.305983+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":961,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.298846+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:36:50.298846+0000","last_peered":"2023-09-23T23:36:50.298846+0000","last_clean":"2023-09-23T23:36:50.298846+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:36:50.298846+0000","last_undegraded":"2023-09-23T23:36:50.298846+0000","last_fullsized":"2023-09-23T23:36:50.298846+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":162,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.290324+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:36:50.290324+0000","last_peered":"2023-09-23T23:36:50.290324+0000","last_clean":"2023-09-23T23:36:50.290324+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:36:50.290324+0000","last_undegraded":"2023-09-23T23:36:50.290324+0000","last_fullsized":"2023-09-23T23:36:50.290324+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":239,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.308779+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:36:50.308779+0000","last_peered":"2023-09-23T23:36:50.308779+0000","last_clean":"2023-09-23T23:36:50.308779+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:36:50.308779+0000","last_undegraded":"2023-09-23T23:36:50.308779+0000","last_fullsized":"2023-09-23T23:36:50.308779+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"598'7586","reported_seq":7667,"reported_epoch":598,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:04.101496+0000","last_change":"2023-09-23T23:33:52.884769+0000","last_active":"2023-09-23T23:37:04.101496+0000","last_peered":"2023-09-23T23:37:04.101496+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:37:04.101496+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"550'2200","ondisk_log_start":"550'2200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5386,"log_dups_size":0,"ondisk_log_size":5386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213778432,"num_objects":3262,"num_object_clones":0,"num_object_copies":9786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7586,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":2236},{"shards":"2,6","objects":-1136}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":932,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.297027+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:36:50.297027+0000","last_peered":"2023-09-23T23:36:50.297027+0000","last_clean":"2023-09-23T23:36:50.297027+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:36:50.297027+0000","last_undegraded":"2023-09-23T23:36:50.297027+0000","last_fullsized":"2023-09-23T23:36:50.297027+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":239,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.293658+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:36:50.293658+0000","last_peered":"2023-09-23T23:36:50.293658+0000","last_clean":"2023-09-23T23:36:50.293658+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:36:50.293658+0000","last_undegraded":"2023-09-23T23:36:50.293658+0000","last_fullsized":"2023-09-23T23:36:50.293658+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":229,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.298187+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:36:50.298187+0000","last_peered":"2023-09-23T23:36:50.298187+0000","last_clean":"2023-09-23T23:36:50.298187+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:36:50.298187+0000","last_undegraded":"2023-09-23T23:36:50.298187+0000","last_fullsized":"2023-09-23T23:36:50.298187+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"598'7742","reported_seq":7823,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:37:04.693262+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:37:04.693262+0000","last_peered":"2023-09-23T23:37:04.693262+0000","last_clean":"2023-09-23T23:37:04.693262+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:37:04.693262+0000","last_undegraded":"2023-09-23T23:37:04.693262+0000","last_fullsized":"2023-09-23T23:37:04.693262+0000","mapping_epoch":566,"log_start":"548'1600","ondisk_log_start":"548'1600","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6142,"log_dups_size":0,"ondisk_log_size":6142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215613440,"num_objects":3290,"num_object_clones":0,"num_object_copies":9870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7742,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1010,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.304837+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:36:50.304837+0000","last_peered":"2023-09-23T23:36:50.304837+0000","last_clean":"2023-09-23T23:36:50.304837+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:36:50.304837+0000","last_undegraded":"2023-09-23T23:36:50.304837+0000","last_fullsized":"2023-09-23T23:36:50.304837+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":229,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.290692+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:36:50.290692+0000","last_peered":"2023-09-23T23:36:50.290692+0000","last_clean":"2023-09-23T23:36:50.290692+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:36:50.290692+0000","last_undegraded":"2023-09-23T23:36:50.290692+0000","last_fullsized":"2023-09-23T23:36:50.290692+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":449,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.299420+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:36:50.299420+0000","last_peered":"2023-09-23T23:36:50.299420+0000","last_clean":"2023-09-23T23:36:50.299420+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:36:50.299420+0000","last_undegraded":"2023-09-23T23:36:50.299420+0000","last_fullsized":"2023-09-23T23:36:50.299420+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"598'7772","reported_seq":13384,"reported_epoch":598,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:37:02.958615+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:37:02.958615+0000","last_peered":"2023-09-23T23:37:02.958615+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:37:02.958615+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:37:02.958615+0000","mapping_epoch":571,"log_start":"550'2500","ondisk_log_start":"550'2500","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5272,"log_dups_size":0,"ondisk_log_size":5272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214695936,"num_objects":3276,"num_object_clones":0,"num_object_copies":9828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2381,"num_objects_misplaced":895,"num_objects_unfound":0,"num_objects_dirty":3276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7772,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2750,"num_bytes_recovered":180224000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":2259},{"shards":"3,5,7","objects":1017}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":908,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.296756+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:36:50.296756+0000","last_peered":"2023-09-23T23:36:50.296756+0000","last_clean":"2023-09-23T23:36:50.296756+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:36:50.296756+0000","last_undegraded":"2023-09-23T23:36:50.296756+0000","last_fullsized":"2023-09-23T23:36:50.296756+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":336,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.299923+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:36:50.299923+0000","last_peered":"2023-09-23T23:36:50.299923+0000","last_clean":"2023-09-23T23:36:50.299923+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:36:50.299923+0000","last_undegraded":"2023-09-23T23:36:50.299923+0000","last_fullsized":"2023-09-23T23:36:50.299923+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":112,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.290577+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:36:50.290577+0000","last_peered":"2023-09-23T23:36:50.290577+0000","last_clean":"2023-09-23T23:36:50.290577+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:36:50.290577+0000","last_undegraded":"2023-09-23T23:36:50.290577+0000","last_fullsized":"2023-09-23T23:36:50.290577+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"598'7749","reported_seq":9817,"reported_epoch":598,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:02.964961+0000","last_change":"2023-09-23T23:34:35.750428+0000","last_active":"2023-09-23T23:37:02.964961+0000","last_peered":"2023-09-23T23:37:02.964961+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:37:02.964961+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"550'2400","ondisk_log_start":"550'2400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5349,"log_dups_size":0,"ondisk_log_size":5349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218955776,"num_objects":3341,"num_object_clones":0,"num_object_copies":10023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7749,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":991,"num_bytes_recovered":64946176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":2331},{"shards":"3,5","objects":-203}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":505,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.305922+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:36:50.305922+0000","last_peered":"2023-09-23T23:36:50.305922+0000","last_clean":"2023-09-23T23:36:50.305922+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:36:50.305922+0000","last_undegraded":"2023-09-23T23:36:50.305922+0000","last_fullsized":"2023-09-23T23:36:50.305922+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":438,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.304979+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:36:50.304979+0000","last_peered":"2023-09-23T23:36:50.304979+0000","last_clean":"2023-09-23T23:36:50.304979+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:36:50.304979+0000","last_undegraded":"2023-09-23T23:36:50.304979+0000","last_fullsized":"2023-09-23T23:36:50.304979+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":595,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.321657+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:36:50.321657+0000","last_peered":"2023-09-23T23:36:50.321657+0000","last_clean":"2023-09-23T23:36:50.321657+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:36:50.321657+0000","last_undegraded":"2023-09-23T23:36:50.321657+0000","last_fullsized":"2023-09-23T23:36:50.321657+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"598'7881","reported_seq":9209,"reported_epoch":598,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:04.300932+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:37:04.300932+0000","last_peered":"2023-09-23T23:37:04.300932+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:37:04.300932+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"550'2200","ondisk_log_start":"550'2200","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5681,"log_dups_size":0,"ondisk_log_size":5681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215023616,"num_objects":3281,"num_object_clones":0,"num_object_copies":9843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7881,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":2235},{"shards":"3,5","objects":-2179},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":726,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.299795+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:36:50.299795+0000","last_peered":"2023-09-23T23:36:50.299795+0000","last_clean":"2023-09-23T23:36:50.299795+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:36:50.299795+0000","last_undegraded":"2023-09-23T23:36:50.299795+0000","last_fullsized":"2023-09-23T23:36:50.299795+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":608,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.304656+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:36:50.304656+0000","last_peered":"2023-09-23T23:36:50.304656+0000","last_clean":"2023-09-23T23:36:50.304656+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:36:50.304656+0000","last_undegraded":"2023-09-23T23:36:50.304656+0000","last_fullsized":"2023-09-23T23:36:50.304656+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":519,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.304909+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:36:50.304909+0000","last_peered":"2023-09-23T23:36:50.304909+0000","last_clean":"2023-09-23T23:36:50.304909+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:36:50.304909+0000","last_undegraded":"2023-09-23T23:36:50.304909+0000","last_fullsized":"2023-09-23T23:36:50.304909+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"598'7793","reported_seq":7865,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:37:03.922287+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:37:03.922287+0000","last_peered":"2023-09-23T23:37:03.922287+0000","last_clean":"2023-09-23T23:37:03.922287+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:37:03.922287+0000","last_undegraded":"2023-09-23T23:37:03.922287+0000","last_fullsized":"2023-09-23T23:37:03.922287+0000","mapping_epoch":534,"log_start":"550'2400","ondisk_log_start":"550'2400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5393,"log_dups_size":0,"ondisk_log_size":5393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215416832,"num_objects":3287,"num_object_clones":0,"num_object_copies":9861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7793,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":881,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.297157+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:36:50.297157+0000","last_peered":"2023-09-23T23:36:50.297157+0000","last_clean":"2023-09-23T23:36:50.297157+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:36:50.297157+0000","last_undegraded":"2023-09-23T23:36:50.297157+0000","last_fullsized":"2023-09-23T23:36:50.297157+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":122,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.290513+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:36:50.290513+0000","last_peered":"2023-09-23T23:36:50.290513+0000","last_clean":"2023-09-23T23:36:50.290513+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:36:50.290513+0000","last_undegraded":"2023-09-23T23:36:50.290513+0000","last_fullsized":"2023-09-23T23:36:50.290513+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":464,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.297238+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:36:50.297238+0000","last_peered":"2023-09-23T23:36:50.297238+0000","last_clean":"2023-09-23T23:36:50.297238+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:36:50.297238+0000","last_undegraded":"2023-09-23T23:36:50.297238+0000","last_fullsized":"2023-09-23T23:36:50.297238+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"598'7846","reported_seq":12526,"reported_epoch":598,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:37:03.994596+0000","last_change":"2023-09-23T23:34:35.785270+0000","last_active":"2023-09-23T23:37:03.994596+0000","last_peered":"2023-09-23T23:37:03.994596+0000","last_clean":"2023-09-23T23:33:50.504296+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:37:03.994596+0000","last_undegraded":"2023-09-23T23:34:35.780981+0000","last_fullsized":"2023-09-23T23:37:03.994596+0000","mapping_epoch":567,"log_start":"550'2400","ondisk_log_start":"550'2400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5446,"log_dups_size":0,"ondisk_log_size":5446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213516288,"num_objects":3258,"num_object_clones":0,"num_object_copies":9774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2253,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7846,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2294,"num_bytes_recovered":150339584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":2253},{"shards":"5,6,7","objects":1005}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":606,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.298411+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:36:50.298411+0000","last_peered":"2023-09-23T23:36:50.298411+0000","last_clean":"2023-09-23T23:36:50.298411+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:36:50.298411+0000","last_undegraded":"2023-09-23T23:36:50.298411+0000","last_fullsized":"2023-09-23T23:36:50.298411+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":461,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.298246+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:36:50.298246+0000","last_peered":"2023-09-23T23:36:50.298246+0000","last_clean":"2023-09-23T23:36:50.298246+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:36:50.298246+0000","last_undegraded":"2023-09-23T23:36:50.298246+0000","last_fullsized":"2023-09-23T23:36:50.298246+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":549,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.290291+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:36:50.290291+0000","last_peered":"2023-09-23T23:36:50.290291+0000","last_clean":"2023-09-23T23:36:50.290291+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:36:50.290291+0000","last_undegraded":"2023-09-23T23:36:50.290291+0000","last_fullsized":"2023-09-23T23:36:50.290291+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"598'7730","reported_seq":16765,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:37:04.315581+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:37:04.315581+0000","last_peered":"2023-09-23T23:37:04.315581+0000","last_clean":"2023-09-23T23:37:04.315581+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:37:04.315581+0000","last_undegraded":"2023-09-23T23:37:04.315581+0000","last_fullsized":"2023-09-23T23:37:04.315581+0000","mapping_epoch":579,"log_start":"549'2000","ondisk_log_start":"549'2000","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5730,"log_dups_size":0,"ondisk_log_size":5730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215351296,"num_objects":3286,"num_object_clones":0,"num_object_copies":9858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7730,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":940,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.300882+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:36:50.300882+0000","last_peered":"2023-09-23T23:36:50.300882+0000","last_clean":"2023-09-23T23:36:50.300882+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:36:50.300882+0000","last_undegraded":"2023-09-23T23:36:50.300882+0000","last_fullsized":"2023-09-23T23:36:50.300882+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":240,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:57.703942+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:36:57.703942+0000","last_peered":"2023-09-23T23:36:57.703942+0000","last_clean":"2023-09-23T23:36:57.703942+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:36:57.703942+0000","last_undegraded":"2023-09-23T23:36:57.703942+0000","last_fullsized":"2023-09-23T23:36:57.703942+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":528,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.309006+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:36:50.309006+0000","last_peered":"2023-09-23T23:36:50.309006+0000","last_clean":"2023-09-23T23:36:50.309006+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:36:50.309006+0000","last_undegraded":"2023-09-23T23:36:50.309006+0000","last_fullsized":"2023-09-23T23:36:50.309006+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"598'7468","reported_seq":7542,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:37:02.972029+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:37:02.972029+0000","last_peered":"2023-09-23T23:37:02.972029+0000","last_clean":"2023-09-23T23:37:02.972029+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:37:02.972029+0000","last_undegraded":"2023-09-23T23:37:02.972029+0000","last_fullsized":"2023-09-23T23:37:02.972029+0000","mapping_epoch":534,"log_start":"550'2200","ondisk_log_start":"550'2200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5268,"log_dups_size":0,"ondisk_log_size":5268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213647360,"num_objects":3260,"num_object_clones":0,"num_object_copies":9780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7468,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":919,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.291744+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:36:50.291744+0000","last_peered":"2023-09-23T23:36:50.291744+0000","last_clean":"2023-09-23T23:36:50.291744+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:36:50.291744+0000","last_undegraded":"2023-09-23T23:36:50.291744+0000","last_fullsized":"2023-09-23T23:36:50.291744+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":307,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.304569+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:36:50.304569+0000","last_peered":"2023-09-23T23:36:50.304569+0000","last_clean":"2023-09-23T23:36:50.304569+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:36:50.304569+0000","last_undegraded":"2023-09-23T23:36:50.304569+0000","last_fullsized":"2023-09-23T23:36:50.304569+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":613,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.300071+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:36:50.300071+0000","last_peered":"2023-09-23T23:36:50.300071+0000","last_clean":"2023-09-23T23:36:50.300071+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:36:50.300071+0000","last_undegraded":"2023-09-23T23:36:50.300071+0000","last_fullsized":"2023-09-23T23:36:50.300071+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":194,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.298695+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:36:50.298695+0000","last_peered":"2023-09-23T23:36:50.298695+0000","last_clean":"2023-09-23T23:36:50.298695+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:36:50.298695+0000","last_undegraded":"2023-09-23T23:36:50.298695+0000","last_fullsized":"2023-09-23T23:36:50.298695+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"598'7870","reported_seq":12544,"reported_epoch":598,"state":"active+recovering","last_fresh":"2023-09-23T23:37:02.700280+0000","last_change":"2023-09-23T23:34:35.815566+0000","last_active":"2023-09-23T23:37:02.700280+0000","last_peered":"2023-09-23T23:37:02.700280+0000","last_clean":"2023-09-23T23:33:50.519308+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:37:02.700280+0000","last_undegraded":"2023-09-23T23:37:02.700280+0000","last_fullsized":"2023-09-23T23:37:02.700280+0000","mapping_epoch":566,"log_start":"550'2522","ondisk_log_start":"550'2522","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5348,"log_dups_size":0,"ondisk_log_size":5348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216072216,"num_objects":3298,"num_object_clones":0,"num_object_copies":9894,"num_objects_missing_on_primary":2221,"num_objects_missing":2221,"num_objects_degraded":0,"num_objects_misplaced":2221,"num_objects_unfound":0,"num_objects_dirty":3298,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7870,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2286,"num_bytes_recovered":149815296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,3,4,7","objects":8},{"shards":"1,3,7","objects":1069},{"shards":"1,4,7","objects":2221}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":682,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.320494+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:36:50.320494+0000","last_peered":"2023-09-23T23:36:50.320494+0000","last_clean":"2023-09-23T23:36:50.320494+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:36:50.320494+0000","last_undegraded":"2023-09-23T23:36:50.320494+0000","last_fullsized":"2023-09-23T23:36:50.320494+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":246,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.290433+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:36:50.290433+0000","last_peered":"2023-09-23T23:36:50.290433+0000","last_clean":"2023-09-23T23:36:50.290433+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:36:50.290433+0000","last_undegraded":"2023-09-23T23:36:50.290433+0000","last_fullsized":"2023-09-23T23:36:50.290433+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":530,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.290469+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:36:50.290469+0000","last_peered":"2023-09-23T23:36:50.290469+0000","last_clean":"2023-09-23T23:36:50.290469+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:36:50.290469+0000","last_undegraded":"2023-09-23T23:36:50.290469+0000","last_fullsized":"2023-09-23T23:36:50.290469+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":231,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.297234+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:36:50.297234+0000","last_peered":"2023-09-23T23:36:50.297234+0000","last_clean":"2023-09-23T23:36:50.297234+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:36:50.297234+0000","last_undegraded":"2023-09-23T23:36:50.297234+0000","last_fullsized":"2023-09-23T23:36:50.297234+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"598'7549","reported_seq":9600,"reported_epoch":598,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:04.389623+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:37:04.389623+0000","last_peered":"2023-09-23T23:37:04.389623+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:37:04.389623+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"549'1800","ondisk_log_start":"549'1800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5749,"log_dups_size":0,"ondisk_log_size":5749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211222528,"num_objects":3223,"num_object_clones":0,"num_object_copies":9669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7549,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":2212},{"shards":"5,7","objects":-169}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":899,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.300798+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:36:50.300798+0000","last_peered":"2023-09-23T23:36:50.300798+0000","last_clean":"2023-09-23T23:36:50.300798+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:36:50.300798+0000","last_undegraded":"2023-09-23T23:36:50.300798+0000","last_fullsized":"2023-09-23T23:36:50.300798+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":337,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.782230+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:36:59.782230+0000","last_peered":"2023-09-23T23:36:59.782230+0000","last_clean":"2023-09-23T23:36:59.782230+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:36:59.782230+0000","last_undegraded":"2023-09-23T23:36:59.782230+0000","last_fullsized":"2023-09-23T23:36:59.782230+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":246,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.297236+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:36:50.297236+0000","last_peered":"2023-09-23T23:36:50.297236+0000","last_clean":"2023-09-23T23:36:50.297236+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:36:50.297236+0000","last_undegraded":"2023-09-23T23:36:50.297236+0000","last_fullsized":"2023-09-23T23:36:50.297236+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":210,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.307276+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:36:50.307276+0000","last_peered":"2023-09-23T23:36:50.307276+0000","last_clean":"2023-09-23T23:36:50.307276+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:36:50.307276+0000","last_undegraded":"2023-09-23T23:36:50.307276+0000","last_fullsized":"2023-09-23T23:36:50.307276+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"598'7705","reported_seq":7787,"reported_epoch":598,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:37:04.337636+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:37:04.337636+0000","last_peered":"2023-09-23T23:37:04.337636+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:37:04.337636+0000","last_undegraded":"2023-09-23T23:37:04.337636+0000","last_fullsized":"2023-09-23T23:37:04.337636+0000","mapping_epoch":562,"log_start":"549'2000","ondisk_log_start":"549'2000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5705,"log_dups_size":0,"ondisk_log_size":5705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209518592,"num_objects":3197,"num_object_clones":0,"num_object_copies":9591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7705,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":3197}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":613,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.298269+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:36:50.298269+0000","last_peered":"2023-09-23T23:36:50.298269+0000","last_clean":"2023-09-23T23:36:50.298269+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:36:50.298269+0000","last_undegraded":"2023-09-23T23:36:50.298269+0000","last_fullsized":"2023-09-23T23:36:50.298269+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":884,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.289802+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:36:50.289802+0000","last_peered":"2023-09-23T23:36:50.289802+0000","last_clean":"2023-09-23T23:36:50.289802+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:36:50.289802+0000","last_undegraded":"2023-09-23T23:36:50.289802+0000","last_fullsized":"2023-09-23T23:36:50.289802+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":560,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.306635+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:36:50.306635+0000","last_peered":"2023-09-23T23:36:50.306635+0000","last_clean":"2023-09-23T23:36:50.306635+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:36:50.306635+0000","last_undegraded":"2023-09-23T23:36:50.306635+0000","last_fullsized":"2023-09-23T23:36:50.306635+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":122,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.291164+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:36:50.291164+0000","last_peered":"2023-09-23T23:36:50.291164+0000","last_clean":"2023-09-23T23:36:50.291164+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:36:50.291164+0000","last_undegraded":"2023-09-23T23:36:50.291164+0000","last_fullsized":"2023-09-23T23:36:50.291164+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"598'7536","reported_seq":7608,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:37:03.950228+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:37:03.950228+0000","last_peered":"2023-09-23T23:37:03.950228+0000","last_clean":"2023-09-23T23:37:03.950228+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:37:03.950228+0000","last_undegraded":"2023-09-23T23:37:03.950228+0000","last_fullsized":"2023-09-23T23:37:03.950228+0000","mapping_epoch":534,"log_start":"550'2100","ondisk_log_start":"550'2100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5436,"log_dups_size":0,"ondisk_log_size":5436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209715200,"num_objects":3200,"num_object_clones":0,"num_object_copies":9600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7536,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":710,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.290177+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:36:50.290177+0000","last_peered":"2023-09-23T23:36:50.290177+0000","last_clean":"2023-09-23T23:36:50.290177+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:36:50.290177+0000","last_undegraded":"2023-09-23T23:36:50.290177+0000","last_fullsized":"2023-09-23T23:36:50.290177+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":525,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.289373+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:36:50.289373+0000","last_peered":"2023-09-23T23:36:50.289373+0000","last_clean":"2023-09-23T23:36:50.289373+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:36:50.289373+0000","last_undegraded":"2023-09-23T23:36:50.289373+0000","last_fullsized":"2023-09-23T23:36:50.289373+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":439,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.299579+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:36:50.299579+0000","last_peered":"2023-09-23T23:36:50.299579+0000","last_clean":"2023-09-23T23:36:50.299579+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:36:50.299579+0000","last_undegraded":"2023-09-23T23:36:50.299579+0000","last_fullsized":"2023-09-23T23:36:50.299579+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":39,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.321092+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:36:50.321092+0000","last_peered":"2023-09-23T23:36:50.321092+0000","last_clean":"2023-09-23T23:36:50.321092+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:36:50.321092+0000","last_undegraded":"2023-09-23T23:36:50.321092+0000","last_fullsized":"2023-09-23T23:36:50.321092+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"598'7799","reported_seq":11470,"reported_epoch":598,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:04.744662+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:37:04.744662+0000","last_peered":"2023-09-23T23:37:04.744662+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:37:04.744662+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"548'1600","ondisk_log_start":"548'1600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6199,"log_dups_size":0,"ondisk_log_size":6199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209649664,"num_objects":3199,"num_object_clones":0,"num_object_copies":9597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7799,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":495},{"shards":"1,2,4","objects":2197}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":593,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.293412+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:36:50.293412+0000","last_peered":"2023-09-23T23:36:50.293412+0000","last_clean":"2023-09-23T23:36:50.293412+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:36:50.293412+0000","last_undegraded":"2023-09-23T23:36:50.293412+0000","last_fullsized":"2023-09-23T23:36:50.293412+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":39,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.326029+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:36:50.326029+0000","last_peered":"2023-09-23T23:36:50.326029+0000","last_clean":"2023-09-23T23:36:50.326029+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:36:50.326029+0000","last_undegraded":"2023-09-23T23:36:50.326029+0000","last_fullsized":"2023-09-23T23:36:50.326029+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":138,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.297306+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:36:50.297306+0000","last_peered":"2023-09-23T23:36:50.297306+0000","last_clean":"2023-09-23T23:36:50.297306+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:36:50.297306+0000","last_undegraded":"2023-09-23T23:36:50.297306+0000","last_fullsized":"2023-09-23T23:36:50.297306+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":331,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.298568+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:36:50.298568+0000","last_peered":"2023-09-23T23:36:50.298568+0000","last_clean":"2023-09-23T23:36:50.298568+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:36:50.298568+0000","last_undegraded":"2023-09-23T23:36:50.298568+0000","last_fullsized":"2023-09-23T23:36:50.298568+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"598'7718","reported_seq":7790,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:37:04.366257+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:37:04.366257+0000","last_peered":"2023-09-23T23:37:04.366257+0000","last_clean":"2023-09-23T23:37:04.366257+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:37:04.366257+0000","last_undegraded":"2023-09-23T23:37:04.366257+0000","last_fullsized":"2023-09-23T23:37:04.366257+0000","mapping_epoch":534,"log_start":"549'2000","ondisk_log_start":"549'2000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5718,"log_dups_size":0,"ondisk_log_size":5718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212729856,"num_objects":3246,"num_object_clones":0,"num_object_copies":9738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7718,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":779,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.297329+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:36:50.297329+0000","last_peered":"2023-09-23T23:36:50.297329+0000","last_clean":"2023-09-23T23:36:50.297329+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:36:50.297329+0000","last_undegraded":"2023-09-23T23:36:50.297329+0000","last_fullsized":"2023-09-23T23:36:50.297329+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":596,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.793820+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:36:59.793820+0000","last_peered":"2023-09-23T23:36:59.793820+0000","last_clean":"2023-09-23T23:36:59.793820+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:36:59.793820+0000","last_undegraded":"2023-09-23T23:36:59.793820+0000","last_fullsized":"2023-09-23T23:36:59.793820+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":202,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.293220+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:36:50.293220+0000","last_peered":"2023-09-23T23:36:50.293220+0000","last_clean":"2023-09-23T23:36:50.293220+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:36:50.293220+0000","last_undegraded":"2023-09-23T23:36:50.293220+0000","last_fullsized":"2023-09-23T23:36:50.293220+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":150,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.297018+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:36:50.297018+0000","last_peered":"2023-09-23T23:36:50.297018+0000","last_clean":"2023-09-23T23:36:50.297018+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:36:50.297018+0000","last_undegraded":"2023-09-23T23:36:50.297018+0000","last_fullsized":"2023-09-23T23:36:50.297018+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"598'7640","reported_seq":7714,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:37:04.581613+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:37:04.581613+0000","last_peered":"2023-09-23T23:37:04.581613+0000","last_clean":"2023-09-23T23:37:04.581613+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:37:04.581613+0000","last_undegraded":"2023-09-23T23:37:04.581613+0000","last_fullsized":"2023-09-23T23:37:04.581613+0000","mapping_epoch":534,"log_start":"548'1500","ondisk_log_start":"548'1500","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6140,"log_dups_size":0,"ondisk_log_size":6140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214171648,"num_objects":3268,"num_object_clones":0,"num_object_copies":9804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7640,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":985,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.305203+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:36:50.305203+0000","last_peered":"2023-09-23T23:36:50.305203+0000","last_clean":"2023-09-23T23:36:50.305203+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:36:50.305203+0000","last_undegraded":"2023-09-23T23:36:50.305203+0000","last_fullsized":"2023-09-23T23:36:50.305203+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":112,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.291890+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:36:50.291890+0000","last_peered":"2023-09-23T23:36:50.291890+0000","last_clean":"2023-09-23T23:36:50.291890+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:36:50.291890+0000","last_undegraded":"2023-09-23T23:36:50.291890+0000","last_fullsized":"2023-09-23T23:36:50.291890+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":43,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.321176+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:36:50.321176+0000","last_peered":"2023-09-23T23:36:50.321176+0000","last_clean":"2023-09-23T23:36:50.321176+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:36:50.321176+0000","last_undegraded":"2023-09-23T23:36:50.321176+0000","last_fullsized":"2023-09-23T23:36:50.321176+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":172,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.298185+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:36:50.298185+0000","last_peered":"2023-09-23T23:36:50.298185+0000","last_clean":"2023-09-23T23:36:50.298185+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:36:50.298185+0000","last_undegraded":"2023-09-23T23:36:50.298185+0000","last_fullsized":"2023-09-23T23:36:50.298185+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1034,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.299684+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:36:50.299684+0000","last_peered":"2023-09-23T23:36:50.299684+0000","last_clean":"2023-09-23T23:36:50.299684+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:36:50.299684+0000","last_undegraded":"2023-09-23T23:36:50.299684+0000","last_fullsized":"2023-09-23T23:36:50.299684+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":39,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.825350+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:36:59.825350+0000","last_peered":"2023-09-23T23:36:59.825350+0000","last_clean":"2023-09-23T23:36:59.825350+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:36:59.825350+0000","last_undegraded":"2023-09-23T23:36:59.825350+0000","last_fullsized":"2023-09-23T23:36:59.825350+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":353,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.297385+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:36:50.297385+0000","last_peered":"2023-09-23T23:36:50.297385+0000","last_clean":"2023-09-23T23:36:50.297385+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:36:50.297385+0000","last_undegraded":"2023-09-23T23:36:50.297385+0000","last_fullsized":"2023-09-23T23:36:50.297385+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":746,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.306126+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:36:50.306126+0000","last_peered":"2023-09-23T23:36:50.306126+0000","last_clean":"2023-09-23T23:36:50.306126+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:36:50.306126+0000","last_undegraded":"2023-09-23T23:36:50.306126+0000","last_fullsized":"2023-09-23T23:36:50.306126+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":100,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.290104+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:36:50.290104+0000","last_peered":"2023-09-23T23:36:50.290104+0000","last_clean":"2023-09-23T23:36:50.290104+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:36:50.290104+0000","last_undegraded":"2023-09-23T23:36:50.290104+0000","last_fullsized":"2023-09-23T23:36:50.290104+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":446,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.304569+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:36:50.304569+0000","last_peered":"2023-09-23T23:36:50.304569+0000","last_clean":"2023-09-23T23:36:50.304569+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:36:50.304569+0000","last_undegraded":"2023-09-23T23:36:50.304569+0000","last_fullsized":"2023-09-23T23:36:50.304569+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":296,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.290059+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:36:50.290059+0000","last_peered":"2023-09-23T23:36:50.290059+0000","last_clean":"2023-09-23T23:36:50.290059+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:36:50.290059+0000","last_undegraded":"2023-09-23T23:36:50.290059+0000","last_fullsized":"2023-09-23T23:36:50.290059+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":285,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.306705+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:36:50.306705+0000","last_peered":"2023-09-23T23:36:50.306705+0000","last_clean":"2023-09-23T23:36:50.306705+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:36:50.306705+0000","last_undegraded":"2023-09-23T23:36:50.306705+0000","last_fullsized":"2023-09-23T23:36:50.306705+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":810,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.290461+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:36:50.290461+0000","last_peered":"2023-09-23T23:36:50.290461+0000","last_clean":"2023-09-23T23:36:50.290461+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:36:50.290461+0000","last_undegraded":"2023-09-23T23:36:50.290461+0000","last_fullsized":"2023-09-23T23:36:50.290461+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":629,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.794219+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:36:59.794219+0000","last_peered":"2023-09-23T23:36:59.794219+0000","last_clean":"2023-09-23T23:36:59.794219+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:36:59.794219+0000","last_undegraded":"2023-09-23T23:36:59.794219+0000","last_fullsized":"2023-09-23T23:36:59.794219+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":612,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.300003+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:36:50.300003+0000","last_peered":"2023-09-23T23:36:50.300003+0000","last_clean":"2023-09-23T23:36:50.300003+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:36:50.300003+0000","last_undegraded":"2023-09-23T23:36:50.300003+0000","last_fullsized":"2023-09-23T23:36:50.300003+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":771,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.299939+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:36:50.299939+0000","last_peered":"2023-09-23T23:36:50.299939+0000","last_clean":"2023-09-23T23:36:50.299939+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:36:50.299939+0000","last_undegraded":"2023-09-23T23:36:50.299939+0000","last_fullsized":"2023-09-23T23:36:50.299939+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":442,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.301043+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:36:50.301043+0000","last_peered":"2023-09-23T23:36:50.301043+0000","last_clean":"2023-09-23T23:36:50.301043+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:36:50.301043+0000","last_undegraded":"2023-09-23T23:36:50.301043+0000","last_fullsized":"2023-09-23T23:36:50.301043+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":665,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T23:36:50.289661+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:36:50.289661+0000","last_peered":"2023-09-23T23:36:50.289661+0000","last_clean":"2023-09-23T23:36:50.289661+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:36:50.289661+0000","last_undegraded":"2023-09-23T23:36:50.289661+0000","last_fullsized":"2023-09-23T23:36:50.289661+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":3419078680,"num_objects":52172,"num_object_clones":0,"num_object_copies":156516,"num_objects_missing_on_primary":2221,"num_objects_missing":2221,"num_objects_degraded":24250,"num_objects_misplaced":3116,"num_objects_unfound":0,"num_objects_dirty":52172,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":123384,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16178,"num_bytes_recovered":1060044800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11465338880,"data_stored":11465326664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89962,"ondisk_log_size":89962,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782691,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1627416,"kb_used_data":1247696,"kb_used_omap":8,"kb_used_meta":379703,"kb_avail":92117736,"statfs":{"total":95995035648,"available":94328561664,"internally_reserved":0,"allocated":1277640704,"data_stored":1275974728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":388815941},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":1.048,"15min":2.1080000000000001},"min":{"1min":0.25900000000000001,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":0.51300000000000001,"5min":17.140000000000001,"15min":85.721000000000004},"last":0.316},{"interface":"front","average":{"1min":0.47199999999999998,"5min":1.0600000000000001,"15min":2.1829999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":2.331,"5min":11.069000000000001,"15min":85.965000000000003},"last":0.21299999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":2.532,"15min":3.1309999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":1.5780000000000001,"5min":129.11199999999999,"15min":198.58000000000001},"last":0.379},{"interface":"front","average":{"1min":0.53200000000000003,"5min":2.3039999999999998,"15min":3.1440000000000001},"min":{"1min":0.31900000000000001,"5min":0.247,"15min":0.246},"max":{"1min":2.3690000000000002,"5min":129.02600000000001,"15min":198.71100000000001},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":3.3980000000000001,"5min":2.6880000000000002,"15min":3.8050000000000002},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":53.054000000000002,"5min":78.528999999999996,"15min":203.58000000000001},"last":0.28599999999999998},{"interface":"front","average":{"1min":3.5699999999999998,"5min":1.9299999999999999,"15min":3.2229999999999999},"min":{"1min":0.33400000000000002,"5min":0.27900000000000003,"15min":0.26200000000000001},"max":{"1min":53.023000000000003,"5min":53.023000000000003,"15min":198.69},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":8.6839999999999993,"5min":4.4390000000000001,"15min":4.0750000000000002},"min":{"1min":0.33300000000000002,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":100.40600000000001,"5min":125.511,"15min":279.26499999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":8.7420000000000009,"5min":4.8460000000000001,"15min":4.6180000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29599999999999999},"max":{"1min":100.42400000000001,"5min":125.48999999999999,"15min":279.29899999999998},"last":0.47999999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":1.546,"15min":1.0389999999999999},"min":{"1min":0.28999999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":0.58099999999999996,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":1.341,"15min":0.97599999999999998},"min":{"1min":0.32500000000000001,"5min":0.317,"15min":0.23899999999999999},"max":{"1min":2.3980000000000001,"5min":31.68,"15min":31.68},"last":0.39900000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.6220000000000001,"15min":1.863},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.26500000000000001},"max":{"1min":2.4510000000000001,"5min":60.463999999999999,"15min":210.83600000000001},"last":0.442},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.5840000000000001,"15min":1.742},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.5940000000000001,"5min":63.006,"15min":207.04599999999999},"last":0.40999999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.827,"15min":1.3320000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.29899999999999999},"max":{"1min":1.619,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":2.052,"15min":1.544},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.389,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.54700000000000004}]}]},{"osd":6,"up_from":46,"seq":197568496168,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1887816,"kb_used_data":1182016,"kb_used_omap":0,"kb_used_meta":705792,"kb_avail":91857336,"statfs":{"total":95995035648,"available":94061912064,"internally_reserved":0,"allocated":1210384384,"data_stored":1208734401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":722731008},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.93500000000000005,"15min":4.6799999999999997},"min":{"1min":0.27700000000000002,"5min":0.251,"15min":0.20399999999999999},"max":{"1min":2.8450000000000002,"5min":8.8119999999999994,"15min":135.31999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.73399999999999999,"15min":4.601},"min":{"1min":0.24399999999999999,"5min":0.23999999999999999,"15min":0.22900000000000001},"max":{"1min":1.8859999999999999,"5min":3.8239999999999998,"15min":135.34299999999999},"last":0.41899999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":1.7509999999999999,"15min":2.3759999999999999},"min":{"1min":0.253,"5min":0.251,"15min":0.245},"max":{"1min":1.944,"5min":65.891000000000005,"15min":89.605000000000004},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.64100000000000001,"5min":1.4390000000000001,"15min":2.4510000000000001},"min":{"1min":0.33400000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":2.1269999999999998,"5min":66.087000000000003,"15min":91.025999999999996},"last":0.40300000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":4.9459999999999997,"15min":4.5620000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.861,"5min":97.408000000000001,"15min":127.44499999999999},"last":0.437},{"interface":"front","average":{"1min":0.55200000000000005,"5min":6.0880000000000001,"15min":4.9740000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.2120000000000002,"5min":125.539,"15min":125.539},"last":0.39400000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:36:55 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":5.6589999999999998,"15min":4.3159999999999998},"min":{"1min":0.30299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.3610000000000002,"5min":87.283000000000001,"15min":138.87299999999999},"last":28},{"interface":"front","average":{"1min":0.47699999999999998,"5min":5.516,"15min":4.625},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0349999999999999,"5min":87.491,"15min":138.976},"last":28.084}]},{"osd":4,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.76400000000000001,"15min":0.68300000000000005},"min":{"1min":0.27900000000000003,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":0.57799999999999996,"5min":7.101,"15min":8.1999999999999993},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.88900000000000001,"15min":0.77600000000000002},"min":{"1min":0.32500000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.54600000000000004,"5min":8.3499999999999996,"15min":8.3499999999999996},"last":0.32300000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.82899999999999996,"15min":0.80700000000000005},"min":{"1min":0.34300000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.266,"5min":11.372,"15min":11.473000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.98999999999999999,"15min":0.83599999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26900000000000002},"max":{"1min":0.57699999999999996,"5min":7.0129999999999999,"15min":7.5670000000000002},"last":0.371}]},{"osd":7,"last update":"Sat Sep 23 23:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":1.294,"15min":1.0289999999999999},"min":{"1min":0.30499999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":0.70599999999999996,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.72999999999999998,"5min":1.4019999999999999,"15min":1.0720000000000001},"min":{"1min":0.36099999999999999,"5min":0.35499999999999998,"15min":0.33500000000000002},"max":{"1min":2.665,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.45600000000000002}]}]},{"osd":1,"up_from":112,"seq":481036337509,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1951948,"kb_used_data":1047108,"kb_used_omap":4,"kb_used_meta":904827,"kb_avail":91793204,"statfs":{"total":95995035648,"available":93996240896,"internally_reserved":0,"allocated":1072238592,"data_stored":1069416825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":926543758},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":1.881,"15min":3.4060000000000001},"min":{"1min":0.217,"5min":0.17799999999999999,"15min":0.16700000000000001},"max":{"1min":2.7970000000000002,"5min":47.195999999999998,"15min":130.768},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":1.5469999999999999,"15min":2.726},"min":{"1min":0.24299999999999999,"5min":0.20499999999999999,"15min":0.19},"max":{"1min":1.016,"5min":46.582000000000001,"15min":134.929},"last":0.89400000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":3.5659999999999998,"15min":6.7649999999999997},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":1.401,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.66900000000000004,"5min":3.8849999999999998,"15min":6.8339999999999996},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.23,"5min":202.12,"15min":232.21000000000001},"last":0.75900000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":6.883,"5min":5.8460000000000001,"15min":5.4379999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22600000000000001},"max":{"1min":71.745000000000005,"5min":181.94399999999999,"15min":234.404},"last":0.41299999999999998},{"interface":"front","average":{"1min":6.8639999999999999,"5min":5.8860000000000001,"15min":5.3899999999999997},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":71.786000000000001,"5min":181.98500000000001,"15min":236.33699999999999},"last":0.47899999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":1.23,"15min":1.423},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":2.1059999999999999,"5min":12.042999999999999,"15min":12.042999999999999},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.624,"5min":1.129,"15min":1.46},"min":{"1min":0.48099999999999998,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":1.2669999999999999,"5min":9.5069999999999997,"15min":18.108000000000001},"last":0.503}]},{"osd":5,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":2.9209999999999998,"15min":1.796},"min":{"1min":0.34300000000000003,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66600000000000004,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.68200000000000005,"5min":2.9940000000000002,"15min":2.113},"min":{"1min":0.40899999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.044,"5min":151.821,"15min":151.821},"last":0.73599999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.3600000000000001,"15min":1.7529999999999999},"min":{"1min":0.377,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":1.1040000000000001,"5min":8.9269999999999996,"15min":74.087999999999994},"last":1.103},{"interface":"front","average":{"1min":0.97699999999999998,"5min":1.585,"15min":1.968},"min":{"1min":0.42999999999999999,"5min":0.35699999999999998,"15min":0.33100000000000002},"max":{"1min":5.367,"5min":11.927,"15min":73.974999999999994},"last":0.71099999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":1.5449999999999999,"15min":2.359},"min":{"1min":0.40200000000000002,"5min":0.38700000000000001,"15min":0.32900000000000001},"max":{"1min":2.504,"5min":44.241,"15min":78.308000000000007},"last":0.621},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.556,"15min":2.2730000000000001},"min":{"1min":0.41299999999999998,"5min":0.37,"15min":0.32700000000000001},"max":{"1min":3.258,"5min":43.920999999999999,"15min":77.204999999999998},"last":1.1899999999999999}]}]},{"osd":0,"up_from":91,"seq":390842024333,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1477328,"kb_used_data":279368,"kb_used_omap":5,"kb_used_meta":1197946,"kb_avail":92267824,"statfs":{"total":95995035648,"available":94482251776,"internally_reserved":0,"allocated":286072832,"data_stored":283241092,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1226697338},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:36:36 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":4.391,"15min":6.6539999999999999},"min":{"1min":0.219,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":1.7709999999999999,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.99099999999999999},{"interface":"front","average":{"1min":0.43099999999999999,"5min":4.1289999999999996,"15min":5.7969999999999997},"min":{"1min":0.245,"5min":0.20200000000000001,"15min":0.17599999999999999},"max":{"1min":1.0249999999999999,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.41399999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":6.9610000000000003,"15min":7.3559999999999999},"min":{"1min":0.26200000000000001,"5min":0.214,"15min":0.17199999999999999},"max":{"1min":4.0899999999999999,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.96199999999999997},{"interface":"front","average":{"1min":0.84599999999999997,"5min":6.6799999999999997,"15min":7.3869999999999996},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.17599999999999999},"max":{"1min":3.468,"5min":283.161,"15min":283.161},"last":0.45800000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:36:36 2023","interfaces":[{"interface":"back","average":{"1min":5.9699999999999998,"5min":5.1630000000000003,"15min":5.6079999999999997},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":95.655000000000001,"5min":114.381,"15min":220.732},"last":1.022},{"interface":"front","average":{"1min":5.8300000000000001,"5min":5.2779999999999996,"15min":5.6669999999999998},"min":{"1min":0.26600000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":95.695999999999998,"5min":114.30500000000001,"15min":218.78200000000001},"last":0.79800000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":1.1599999999999999,"15min":2.4769999999999999},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.70799999999999996,"5min":14.676,"15min":138.81299999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.80100000000000005,"5min":1.391,"15min":3.0830000000000002},"min":{"1min":0.33800000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.7469999999999999,"5min":9.9039999999999999,"15min":171.251},"last":0.93999999999999995}]},{"osd":5,"last update":"Sat Sep 23 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":1.504,"15min":2.7930000000000001},"min":{"1min":0.35299999999999998,"5min":0.32700000000000001,"15min":0.32100000000000001},"max":{"1min":4.6820000000000004,"5min":15.076000000000001,"15min":171.30699999999999},"last":0.97399999999999998},{"interface":"front","average":{"1min":0.625,"5min":1.244,"15min":2.0819999999999999},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":1.8149999999999999,"5min":8.9909999999999997,"15min":139.40799999999999},"last":1.0429999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.325,"15min":3.097},"min":{"1min":0.38100000000000001,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":4.718,"5min":9.968,"15min":171.33600000000001},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.335,"15min":3.3650000000000002},"min":{"1min":0.37,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":1.8700000000000001,"5min":9.8320000000000007,"15min":184.13499999999999},"last":1.0629999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":1.089,"15min":2.569},"min":{"1min":0.35099999999999998,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":2.6749999999999998,"5min":8.9510000000000005,"15min":138.77699999999999},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.3260000000000001,"15min":2.508},"min":{"1min":0.373,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":4.2229999999999999,"5min":9.8870000000000005,"15min":139.37899999999999},"last":1.0820000000000001}]}]},{"osd":2,"up_from":199,"seq":854698492154,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2935056,"kb_used_data":2082952,"kb_used_omap":7,"kb_used_meta":852088,"kb_avail":90810096,"statfs":{"total":95995035648,"available":92989538304,"internally_reserved":0,"allocated":2132942848,"data_stored":2130061156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":872538193},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":2.2080000000000002,"15min":3.8599999999999999},"min":{"1min":0.186,"5min":0.186,"15min":0.16700000000000001},"max":{"1min":4.7590000000000003,"5min":61.950000000000003,"15min":119.277},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.61899999999999999,"5min":2.153,"15min":3.6669999999999998},"min":{"1min":0.23999999999999999,"5min":0.20399999999999999,"15min":0.18099999999999999},"max":{"1min":2.5070000000000001,"5min":61.984999999999999,"15min":119.56100000000001},"last":0.20200000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.911,"5min":5.3010000000000002,"15min":4.5940000000000003},"min":{"1min":0.25,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":8.3640000000000008,"5min":106.765,"15min":157.98699999999999},"last":0.192},{"interface":"front","average":{"1min":0.79000000000000004,"5min":4.8959999999999999,"15min":4.577},"min":{"1min":0.253,"5min":0.222,"15min":0.222},"max":{"1min":4.0940000000000003,"5min":106.732,"15min":158.03800000000001},"last":0.33300000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":5.7800000000000002,"15min":4.7770000000000001},"min":{"1min":0.255,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":8.3070000000000004,"5min":106.70399999999999,"15min":119.366},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.65800000000000003,"5min":5.4169999999999998,"15min":4.2709999999999999},"min":{"1min":0.29299999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":3.375,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.42699999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":2.036,"15min":2.262},"min":{"1min":0.36099999999999999,"5min":0.33700000000000002,"15min":0.25},"max":{"1min":2.6070000000000002,"5min":62.081000000000003,"15min":119.88200000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.4590000000000001,"5min":2.3860000000000001,"15min":1.865},"min":{"1min":0.36899999999999999,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":8.343,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.52200000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":3.0299999999999998,"15min":2.6480000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.26200000000000001},"max":{"1min":2.548,"5min":91.207999999999998,"15min":119.919},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.4550000000000001,"5min":3.2749999999999999,"15min":2.6160000000000001},"min":{"1min":0.33500000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.3889999999999993,"5min":91.150999999999996,"15min":119.306},"last":0.54400000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":3.5750000000000002,"15min":3.0019999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":2.5819999999999999,"5min":107.018,"15min":119.943},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.486,"5min":2.6440000000000001,"15min":2.1600000000000001},"min":{"1min":0.371,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":8.4060000000000006,"5min":62.493000000000002,"15min":62.493000000000002},"last":0.48499999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":2.2010000000000001,"15min":1.9179999999999999},"min":{"1min":0.40000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.381,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.60099999999999998,"5min":3.464,"15min":2.3620000000000001},"min":{"1min":0.42699999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.417,"5min":106.809,"15min":106.809},"last":0.56699999999999995}]}]},{"osd":3,"up_from":148,"seq":635655160122,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2063112,"kb_used_data":1137664,"kb_used_omap":2,"kb_used_meta":925437,"kb_avail":91682040,"statfs":{"total":95995035648,"available":93882408960,"internally_reserved":0,"allocated":1164967936,"data_stored":1162147885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":947647541},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.1359999999999999,"15min":2.6019999999999999},"min":{"1min":0.184,"5min":0.184,"15min":0.17699999999999999},"max":{"1min":6.7699999999999996,"5min":17.213000000000001,"15min":103.866},"last":0.308},{"interface":"front","average":{"1min":0.72899999999999998,"5min":1.4139999999999999,"15min":2.855},"min":{"1min":0.218,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":3.4980000000000002,"5min":22.754000000000001,"15min":101.884},"last":0.188}]},{"osd":1,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":2.5960000000000001,"15min":2.903},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":1.649,"5min":56.119,"15min":85.242999999999995},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.42299999999999999,"5min":2.036,"15min":2.7469999999999999},"min":{"1min":0.254,"5min":0.224,"15min":0.20899999999999999},"max":{"1min":1.6990000000000001,"5min":53.962000000000003,"15min":85.302000000000007},"last":0.20799999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:36:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":2.2989999999999999,"15min":4.5999999999999996},"min":{"1min":0.25800000000000001,"5min":0.25,"15min":0.19800000000000001},"max":{"1min":14.965,"5min":56.226999999999997,"15min":101.279},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.2210000000000001,"5min":2.165,"15min":4.4370000000000003},"min":{"1min":0.315,"5min":0.29299999999999998,"15min":0.214},"max":{"1min":15.058999999999999,"5min":54.280999999999999,"15min":101.294},"last":0.27800000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":0.68999999999999995,"5min":11.209,"15min":11.209},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.44500000000000001,"5min":1.3720000000000001,"15min":1.6719999999999999},"min":{"1min":0.33000000000000002,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":0.67900000000000005,"5min":11.494999999999999,"15min":24.782},"last":0.34000000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":6.3570000000000002,"15min":3.3220000000000001},"min":{"1min":0.33000000000000002,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":0.622,"5min":214.43700000000001,"15min":214.43700000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":6.3970000000000002,"15min":3.3969999999999998},"min":{"1min":0.36499999999999999,"5min":0.34699999999999998,"15min":0.30599999999999999},"max":{"1min":1.716,"5min":214.46600000000001,"15min":214.46600000000001},"last":0.35999999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":2.218,"15min":2.1739999999999999},"min":{"1min":0.35399999999999998,"5min":0.318,"15min":0.318},"max":{"1min":2.008,"5min":58.932000000000002,"15min":58.932000000000002},"last":0.378},{"interface":"front","average":{"1min":0.629,"5min":2.492,"15min":2.2210000000000001},"min":{"1min":0.36699999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.7450000000000001,"5min":59.186,"15min":59.186},"last":1.129}]},{"osd":7,"last update":"Sat Sep 23 23:36:56 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":4.5469999999999997,"15min":3.1829999999999998},"min":{"1min":0.42199999999999999,"5min":0.38700000000000001,"15min":0.32600000000000001},"max":{"1min":0.66700000000000004,"5min":192.69,"15min":192.69},"last":0.436},{"interface":"front","average":{"1min":0.54300000000000004,"5min":4.6280000000000001,"15min":3.649},"min":{"1min":0.40100000000000002,"5min":0.39200000000000002,"15min":0.36699999999999999},"max":{"1min":1.2769999999999999,"5min":192.65600000000001,"15min":192.65600000000001},"last":0.40699999999999997}]}]},{"osd":4,"up_from":0,"seq":128849019441,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757804,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2693636,"kb_used_data":1951996,"kb_used_omap":0,"kb_used_meta":741632,"kb_avail":91051516,"statfs":{"total":95995035648,"available":93236752384,"internally_reserved":0,"allocated":1998843904,"data_stored":1997194138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":759431168},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:14 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":1.141,"15min":3.198},"min":{"1min":0.252,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":1.5640000000000001,"5min":21.109000000000002,"15min":113.869},"last":0.496},{"interface":"front","average":{"1min":0.38,"5min":1.3939999999999999,"15min":3.5710000000000002},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.22700000000000001},"max":{"1min":0.76200000000000001,"5min":20.876999999999999,"15min":138.55199999999999},"last":0.47099999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:36:20 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.6200000000000001,"15min":2.258},"min":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.23100000000000001},"max":{"1min":1.9930000000000001,"5min":29.007999999999999,"15min":69.816999999999993},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.8,"15min":2.5840000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":2.8130000000000002,"5min":29.053999999999998,"15min":70.268000000000001},"last":0.52000000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:36:48 2023","interfaces":[{"interface":"back","average":{"1min":5.194,"5min":6.984,"15min":4.8159999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.26900000000000002},"max":{"1min":60.441000000000003,"5min":208.738,"15min":208.738},"last":0.45800000000000002},{"interface":"front","average":{"1min":5.2839999999999998,"5min":5.7190000000000003,"15min":4.3259999999999996},"min":{"1min":0.29999999999999999,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":60.520000000000003,"5min":208.631,"15min":208.631},"last":0.53400000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:36:48 2023","interfaces":[{"interface":"back","average":{"1min":4.1749999999999998,"5min":4.6360000000000001,"15min":3.8100000000000001},"min":{"1min":0.34999999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":37.878,"5min":122.071,"15min":122.071},"last":143.66300000000001},{"interface":"front","average":{"1min":4.2460000000000004,"5min":4.9829999999999997,"15min":3.984},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.318},"max":{"1min":37.853000000000002,"5min":122.09699999999999,"15min":122.09699999999999},"last":143.62200000000001}]},{"osd":4,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.98699999999999999,"15min":0.90700000000000003},"min":{"1min":0.32900000000000001,"5min":0.25700000000000001,"15min":0.251},"max":{"1min":0.749,"5min":9.2690000000000001,"15min":19.962},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.502,"5min":1.0129999999999999,"15min":0.81599999999999995},"min":{"1min":0.314,"5min":0.29599999999999999,"15min":0.22900000000000001},"max":{"1min":1.595,"5min":10.273999999999999,"15min":20.829999999999998},"last":0.33500000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":2.29,"15min":1.5049999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.26900000000000002},"max":{"1min":1.619,"5min":148.36600000000001,"15min":148.36600000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":2.798,"15min":1.7250000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.26000000000000001},"max":{"1min":2.1549999999999998,"5min":148.42500000000001,"15min":148.42500000000001},"last":0.378}]},{"osd":7,"last update":"Sat Sep 23 23:36:48 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.2629999999999999,"15min":2.3879999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":2.29,"5min":15.497,"15min":179.28800000000001},"last":0.439},{"interface":"front","average":{"1min":0.64900000000000002,"5min":1.2729999999999999,"15min":2.46},"min":{"1min":0.36699999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.0630000000000002,"5min":15.653,"15min":179.428},"last":0.50900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":276103168,"data_stored":276103168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1059786752,"data_stored":1059782680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2121531392,"data_stored":2121531392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1151930368,"data_stored":1151926296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2398683136,"data_stored":2398683136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1986199552,"data_stored":1986199552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1201930240,"data_stored":1201930240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1269174272,"data_stored":1269170200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:37:12.015 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:12.068 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:12.195 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:12.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:12.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:12.306 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:12.332 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:12.333 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:12.376 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:12.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:12.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:12.481 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:37:12.508 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:12.562 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:37:12.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:12.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:37:12.669 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:12.696 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:12.741 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:12.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:12.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:12.846 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:12.857 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:37:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:37:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:37:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:37:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178611 11.9 11:94c60d7f:::benchmark_data_smithi110_433746_object72731:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:37:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.550185+0000", 2023-09-23T23:37:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 593.30173220400002, 2023-09-23T23:37:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.188218649, 2023-09-23T23:37:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:12.857 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:37:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:37:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178611 2023-09-23T23:37:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.550185+0000", 2023-09-23T23:37:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.550185+0000", 2023-09-23T23:37:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.550183+0000", 2023-09-23T23:37:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:12.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.550190+0000", 2023-09-23T23:37:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1590000000000004e-06 2023-09-23T23:37:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.550203+0000", 2023-09-23T23:37:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2585999999999999e-05 2023-09-23T23:37:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.550211+0000", 2023-09-23T23:37:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2029999999999992e-06 2023-09-23T23:37:12.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.550246+0000", 2023-09-23T23:37:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4647999999999998e-05 2023-09-23T23:37:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.550252+0000", 2023-09-23T23:37:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6819999999999997e-06 2023-09-23T23:37:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.547412+0000", 2023-09-23T23:37:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99715919500000005 2023-09-23T23:37:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.547424+0000", 2023-09-23T23:37:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2401999999999999e-05 2023-09-23T23:37:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.714868+0000", 2023-09-23T23:37:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1674435679999999 2023-09-23T23:37:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.715028+0000", 2023-09-23T23:37:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000159965 2023-09-23T23:37:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T23:37:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.715499+0000", 2023-09-23T23:37:12.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047178699999999999 2023-09-23T23:37:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.723954+0000", 2023-09-23T23:37:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084545130000000003 2023-09-23T23:37:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.723974+0000", 2023-09-23T23:37:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0497e-05 2023-09-23T23:37:12.864 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.729040+0000", 2023-09-23T23:37:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0050653909999999998 2023-09-23T23:37:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738198+0000", 2023-09-23T23:37:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.009158055 2023-09-23T23:37:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738264+0000", 2023-09-23T23:37:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6534999999999999e-05 2023-09-23T23:37:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738403+0000", 2023-09-23T23:37:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013902900000000001 2023-09-23T23:37:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.866 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178616 11.19 11:9db36ee3:::benchmark_data_smithi110_433746_object72736:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:37:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.573665+0000", 2023-09-23T23:37:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 593.27825144999997, 2023-09-23T23:37:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1649632130000001, 2023-09-23T23:37:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:12.867 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:37:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:37:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178616 2023-09-23T23:37:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.573665+0000", 2023-09-23T23:37:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.573665+0000", 2023-09-23T23:37:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.573664+0000", 2023-09-23T23:37:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.573672+0000", 2023-09-23T23:37:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1580000000000005e-06 2023-09-23T23:37:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.870 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.870 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:12.870 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.573687+0000", 2023-09-23T23:37:12.870 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5104e-05 2023-09-23T23:37:12.870 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.573697+0000", 2023-09-23T23:37:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8220000000000002e-06 2023-09-23T23:37:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.579724+0000", 2023-09-23T23:37:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060272930000000004 2023-09-23T23:37:12.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.579737+0000", 2023-09-23T23:37:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2323999999999999e-05 2023-09-23T23:37:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.547950+0000", 2023-09-23T23:37:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96821353799999998 2023-09-23T23:37:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.872 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.547961+0000", 2023-09-23T23:37:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0730999999999999e-05 2023-09-23T23:37:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.711546+0000", 2023-09-23T23:37:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1635849789999999 2023-09-23T23:37:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:12.873 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.711691+0000", 2023-09-23T23:37:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000144735 2023-09-23T23:37:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T23:37:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.712153+0000", 2023-09-23T23:37:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00046240000000000002 2023-09-23T23:37:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.722189+0000", 2023-09-23T23:37:12.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010035717 2023-09-23T23:37:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.723969+0000", 2023-09-23T23:37:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017799890000000001 2023-09-23T23:37:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.729015+0000", 2023-09-23T23:37:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.005045931 2023-09-23T23:37:12.875 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738451+0000", 2023-09-23T23:37:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0094367670000000004 2023-09-23T23:37:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738498+0000", 2023-09-23T23:37:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6147999999999999e-05 2023-09-23T23:37:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738629+0000", 2023-09-23T23:37:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013102200000000001 2023-09-23T23:37:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35904.0:178627 11.9 11:97e1ceb8:::benchmark_data_smithi110_433746_object72747:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-09-23T23:37:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:27:19.601902+0000", 2023-09-23T23:37:12.877 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 593.25001514300004, 2023-09-23T23:37:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.136940085, 2023-09-23T23:37:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35904", 2023-09-23T23:37:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2647178046", 2023-09-23T23:37:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178627 2023-09-23T23:37:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.601902+0000", 2023-09-23T23:37:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.601902+0000", 2023-09-23T23:37:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.601900+0000", 2023-09-23T23:37:12.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:37:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.601907+0000", 2023-09-23T23:37:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3669999999999999e-06 2023-09-23T23:37:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.601925+0000", 2023-09-23T23:37:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7229000000000001e-05 2023-09-23T23:37:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.601934+0000", 2023-09-23T23:37:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2199999999999998e-06 2023-09-23T23:37:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.602025+0000", 2023-09-23T23:37:12.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1458000000000004e-05 2023-09-23T23:37:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:19.602036+0000", 2023-09-23T23:37:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.061e-05 2023-09-23T23:37:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.547987+0000", 2023-09-23T23:37:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94595114300000005 2023-09-23T23:37:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:20.547994+0000", 2023-09-23T23:37:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4789999999999999e-06 2023-09-23T23:37:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.717738+0000", 2023-09-23T23:37:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1697435760000001 2023-09-23T23:37:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.884 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.717879+0000", 2023-09-23T23:37:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014050799999999999 2023-09-23T23:37:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T23:37:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.718216+0000", 2023-09-23T23:37:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033793099999999997 2023-09-23T23:37:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.723979+0000", 2023-09-23T23:37:12.885 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0057627909999999997 2023-09-23T23:37:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.725186+0000", 2023-09-23T23:37:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001206372 2023-09-23T23:37:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.729100+0000", 2023-09-23T23:37:12.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039139609999999997 2023-09-23T23:37:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738674+0000", 2023-09-23T23:37:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0095747220000000008 2023-09-23T23:37:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738719+0000", 2023-09-23T23:37:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4359e-05 2023-09-23T23:37:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:27:21.738842+0000", 2023-09-23T23:37:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000123132 2023-09-23T23:37:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71524 13.2 13:443908cd:::benchmark_data_smithi110_541928_object71523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e555)", 2023-09-23T23:37:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.543571+0000", 2023-09-23T23:37:12.888 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 202.30834598000001, 2023-09-23T23:37:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.309790005, 2023-09-23T23:37:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:37:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71524 2023-09-23T23:37:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:12.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543571+0000", 2023-09-23T23:37:12.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:12.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543571+0000", 2023-09-23T23:37:12.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:12.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543570+0000", 2023-09-23T23:37:12.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:37:12.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:12.891 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543592+0000", 2023-09-23T23:37:12.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1195e-05 2023-09-23T23:37:12.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:12.891 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543598+0000", 2023-09-23T23:37:12.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8190000000000003e-06 2023-09-23T23:37:12.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:12.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543604+0000", 2023-09-23T23:37:12.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.451e-06 2023-09-23T23:37:12.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.544968+0000", 2023-09-23T23:37:12.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001364007 2023-09-23T23:37:12.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.544984+0000", 2023-09-23T23:37:12.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5699999999999999e-05 2023-09-23T23:37:12.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527908+0000", 2023-09-23T23:37:12.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98292410900000005 2023-09-23T23:37:12.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527920+0000", 2023-09-23T23:37:12.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2852e-05 2023-09-23T23:37:12.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.815907+0000", 2023-09-23T23:37:12.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.287986606 2023-09-23T23:37:12.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:12.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.816011+0000", 2023-09-23T23:37:12.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103791 2023-09-23T23:37:12.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:37:12.895 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.816116+0000", 2023-09-23T23:37:12.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104955 2023-09-23T23:37:12.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:12.895 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.825412+0000", 2023-09-23T23:37:12.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.009295757 2023-09-23T23:37:12.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.847154+0000", 2023-09-23T23:37:12.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021742053000000001 2023-09-23T23:37:12.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.850549+0000", 2023-09-23T23:37:12.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033955470000000001 2023-09-23T23:37:12.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:12.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.850599+0000", 2023-09-23T23:37:12.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9645000000000003e-05 2023-09-23T23:37:12.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:12.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.853361+0000", 2023-09-23T23:37:12.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027620710000000001 2023-09-23T23:37:12.897 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.897 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:12.897 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.898 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71514 13.3 13:cdfd1184:::benchmark_data_smithi110_541928_object71513:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:37:12.898 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.547244+0000", 2023-09-23T23:37:12.898 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 202.304672856, 2023-09-23T23:37:12.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4756996080000002, 2023-09-23T23:37:12.898 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:12.898 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:37:12.898 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:12.898 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:37:12.898 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:12.899 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71514 2023-09-23T23:37:12.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.899 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:12.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:12.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547244+0000", 2023-09-23T23:37:12.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:12.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547244+0000", 2023-09-23T23:37:12.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:12.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547242+0000", 2023-09-23T23:37:12.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:37:12.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:12.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547281+0000", 2023-09-23T23:37:12.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8605000000000001e-05 2023-09-23T23:37:12.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.901 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:12.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547302+0000", 2023-09-23T23:37:12.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0873000000000002e-05 2023-09-23T23:37:12.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.901 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:12.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547312+0000", 2023-09-23T23:37:12.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.859e-06 2023-09-23T23:37:12.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.902 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.902 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.548286+0000", 2023-09-23T23:37:12.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00097434599999999996 2023-09-23T23:37:12.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.902 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.902 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.548298+0000", 2023-09-23T23:37:12.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1881e-05 2023-09-23T23:37:12.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.902 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527773+0000", 2023-09-23T23:37:12.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97947559500000003 2023-09-23T23:37:12.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527787+0000", 2023-09-23T23:37:12.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3329999999999999e-05 2023-09-23T23:37:12.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.021377+0000", 2023-09-23T23:37:12.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.493589799 2023-09-23T23:37:12.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:12.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.022944+0000", 2023-09-23T23:37:12.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015670910000000001 2023-09-23T23:37:12.904 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.904 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:12.904 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.904 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71522 13.3 13:c49d694d:::benchmark_data_smithi110_541928_object71521:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:37:12.904 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.547319+0000", 2023-09-23T23:37:12.904 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 202.30459750899999, 2023-09-23T23:37:12.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5115680760000001, 2023-09-23T23:37:12.905 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:12.905 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:12.905 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:12.905 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:37:12.905 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:12.905 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71522 2023-09-23T23:37:12.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.905 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:12.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.905 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:12.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547319+0000", 2023-09-23T23:37:12.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:12.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547319+0000", 2023-09-23T23:37:12.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:12.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547319+0000", 2023-09-23T23:37:12.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:37:12.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:12.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547342+0000", 2023-09-23T23:37:12.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2857000000000002e-05 2023-09-23T23:37:12.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:12.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547348+0000", 2023-09-23T23:37:12.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8129999999999998e-06 2023-09-23T23:37:12.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:12.908 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547354+0000", 2023-09-23T23:37:12.908 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5090000000000001e-06 2023-09-23T23:37:12.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.908 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.548319+0000", 2023-09-23T23:37:12.908 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00096521499999999995 2023-09-23T23:37:12.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.909 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.548327+0000", 2023-09-23T23:37:12.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7079999999999997e-06 2023-09-23T23:37:12.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.909 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.909 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.909 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527817+0000", 2023-09-23T23:37:12.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97948980500000005 2023-09-23T23:37:12.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.909 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.909 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.909 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527825+0000", 2023-09-23T23:37:12.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3440000000000001e-06 2023-09-23T23:37:12.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.026234+0000", 2023-09-23T23:37:12.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4984093380000001 2023-09-23T23:37:12.910 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:12.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.027045+0000", 2023-09-23T23:37:12.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00081087800000000001 2023-09-23T23:37:12.910 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,7", 2023-09-23T23:37:12.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.027252+0000", 2023-09-23T23:37:12.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000206553 2023-09-23T23:37:12.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.911 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.911 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.038654+0000", 2023-09-23T23:37:12.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011401669 2023-09-23T23:37:12.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.911 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.911 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:12.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.046827+0000", 2023-09-23T23:37:12.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081732569999999997 2023-09-23T23:37:12.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.912 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.055749+0000", 2023-09-23T23:37:12.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0089217600000000008 2023-09-23T23:37:12.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:12.912 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.055791+0000", 2023-09-23T23:37:12.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2493999999999997e-05 2023-09-23T23:37:12.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:12.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.058887+0000", 2023-09-23T23:37:12.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030964320000000001 2023-09-23T23:37:12.913 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.913 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:12.913 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.913 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71530 13.2 13:4cd5e89c:::benchmark_data_smithi110_541928_object71529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:37:12.913 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.571622+0000", 2023-09-23T23:37:12.913 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 202.280294978, 2023-09-23T23:37:12.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2848813030000001, 2023-09-23T23:37:12.914 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:12.914 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:12.914 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:12.914 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:37:12.914 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:12.914 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71530 2023-09-23T23:37:12.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.914 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:12.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:12.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571622+0000", 2023-09-23T23:37:12.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:12.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571622+0000", 2023-09-23T23:37:12.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:12.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571620+0000", 2023-09-23T23:37:12.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:12.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:12.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571678+0000", 2023-09-23T23:37:12.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7633000000000002e-05 2023-09-23T23:37:12.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:12.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571696+0000", 2023-09-23T23:37:12.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8215000000000001e-05 2023-09-23T23:37:12.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:12.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571714+0000", 2023-09-23T23:37:12.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7839e-05 2023-09-23T23:37:12.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571774+0000", 2023-09-23T23:37:12.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0322999999999998e-05 2023-09-23T23:37:12.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571784+0000", 2023-09-23T23:37:12.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1719999999999996e-06 2023-09-23T23:37:12.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.918 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527947+0000", 2023-09-23T23:37:12.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95616350999999999 2023-09-23T23:37:12.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.918 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527957+0000", 2023-09-23T23:37:12.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9369999999999999e-06 2023-09-23T23:37:12.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.918 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.919 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.820763+0000", 2023-09-23T23:37:12.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.292805757 2023-09-23T23:37:12.919 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.919 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:12.919 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.820924+0000", 2023-09-23T23:37:12.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016086599999999999 2023-09-23T23:37:12.919 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.919 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:37:12.919 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.821115+0000", 2023-09-23T23:37:12.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000191137 2023-09-23T23:37:12.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.920 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:12.920 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.825434+0000", 2023-09-23T23:37:12.920 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043196479999999997 2023-09-23T23:37:12.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.920 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.920 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.847182+0000", 2023-09-23T23:37:12.920 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021747295999999999 2023-09-23T23:37:12.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.853421+0000", 2023-09-23T23:37:12.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.006239306 2023-09-23T23:37:12.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:12.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.853476+0000", 2023-09-23T23:37:12.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4929000000000002e-05 2023-09-23T23:37:12.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:12.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.856503+0000", 2023-09-23T23:37:12.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030272390000000001 2023-09-23T23:37:12.922 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.922 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:12.922 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.922 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71537 13.3 13:cd57dd3d:::benchmark_data_smithi110_541928_object71536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:37:12.922 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.580860+0000", 2023-09-23T23:37:12.922 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 202.27105719900001, 2023-09-23T23:37:12.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4840317629999999, 2023-09-23T23:37:12.923 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:12.923 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:12.923 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:12.923 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:37:12.923 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:12.923 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71537 2023-09-23T23:37:12.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.923 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:12.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:12.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580860+0000", 2023-09-23T23:37:12.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:12.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580860+0000", 2023-09-23T23:37:12.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:12.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580858+0000", 2023-09-23T23:37:12.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:37:12.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:12.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580883+0000", 2023-09-23T23:37:12.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5388000000000002e-05 2023-09-23T23:37:12.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:12.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580901+0000", 2023-09-23T23:37:12.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7496e-05 2023-09-23T23:37:12.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:12.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580911+0000", 2023-09-23T23:37:12.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0093e-05 2023-09-23T23:37:12.926 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580968+0000", 2023-09-23T23:37:12.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6971000000000001e-05 2023-09-23T23:37:12.926 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580979+0000", 2023-09-23T23:37:12.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1423999999999999e-05 2023-09-23T23:37:12.927 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.927 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.927 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.927 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527852+0000", 2023-09-23T23:37:12.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94687271200000001 2023-09-23T23:37:12.927 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.927 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.927 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.927 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527860+0000", 2023-09-23T23:37:12.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8159999999999994e-06 2023-09-23T23:37:12.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.928 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.928 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.029524+0000", 2023-09-23T23:37:12.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.501664385 2023-09-23T23:37:12.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.928 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:12.928 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.029705+0000", 2023-09-23T23:37:12.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000181423 2023-09-23T23:37:12.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,7", 2023-09-23T23:37:12.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.029886+0000", 2023-09-23T23:37:12.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018058199999999999 2023-09-23T23:37:12.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.038692+0000", 2023-09-23T23:37:12.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0088061979999999995 2023-09-23T23:37:12.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:12.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.046831+0000", 2023-09-23T23:37:12.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081387630000000002 2023-09-23T23:37:12.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.058949+0000", 2023-09-23T23:37:12.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012117909 2023-09-23T23:37:12.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:12.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.059022+0000", 2023-09-23T23:37:12.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2758999999999999e-05 2023-09-23T23:37:12.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:12.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.064891+0000", 2023-09-23T23:37:12.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058699099999999999 2023-09-23T23:37:12.931 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.931 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:12.931 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.931 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71540 13.2 13:4d0dc391:::benchmark_data_smithi110_541928_object71539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:37:12.931 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.586761+0000", 2023-09-23T23:37:12.932 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 202.26515618299999, 2023-09-23T23:37:12.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4196291790000002, 2023-09-23T23:37:12.932 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:12.932 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:12.932 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:12.932 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:37:12.932 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:12.932 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71540 2023-09-23T23:37:12.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.932 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:12.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.932 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:12.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586761+0000", 2023-09-23T23:37:12.933 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:12.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586761+0000", 2023-09-23T23:37:12.933 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:12.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586758+0000", 2023-09-23T23:37:12.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-09-23T23:37:12.934 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.934 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.934 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:12.934 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586795+0000", 2023-09-23T23:37:12.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6177999999999998e-05 2023-09-23T23:37:12.934 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.934 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.934 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:12.934 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586821+0000", 2023-09-23T23:37:12.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6081999999999999e-05 2023-09-23T23:37:12.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:12.935 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586834+0000", 2023-09-23T23:37:12.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3441999999999999e-05 2023-09-23T23:37:12.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.935 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586917+0000", 2023-09-23T23:37:12.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2551000000000003e-05 2023-09-23T23:37:12.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.936 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.936 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586938+0000", 2023-09-23T23:37:12.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1016e-05 2023-09-23T23:37:12.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.936 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.936 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527980+0000", 2023-09-23T23:37:12.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94104277599999997 2023-09-23T23:37:12.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.936 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527989+0000", 2023-09-23T23:37:12.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7439999999999996e-06 2023-09-23T23:37:12.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.823717+0000", 2023-09-23T23:37:12.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.295727691 2023-09-23T23:37:12.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:12.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.823855+0000", 2023-09-23T23:37:12.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013797700000000001 2023-09-23T23:37:12.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:37:12.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.823956+0000", 2023-09-23T23:37:12.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010142899999999999 2023-09-23T23:37:12.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:12.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.825554+0000", 2023-09-23T23:37:12.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015978839999999999 2023-09-23T23:37:12.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.849319+0000", 2023-09-23T23:37:12.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023764928000000001 2023-09-23T23:37:12.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.856559+0000", 2023-09-23T23:37:12.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0072398460000000003 2023-09-23T23:37:12.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:12.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.856611+0000", 2023-09-23T23:37:12.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1752999999999997e-05 2023-09-23T23:37:12.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:12.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.006390+0000", 2023-09-23T23:37:12.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14977916799999999 2023-09-23T23:37:12.940 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.941 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:12.941 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.941 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89738 13.3 13:cade2564:::benchmark_data_smithi110_541928_object1958:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:37:12.941 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:34.801877+0000", 2023-09-23T23:37:12.941 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 158.05003960100001, 2023-09-23T23:37:12.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5955823369999997, 2023-09-23T23:37:12.941 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:12.941 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:12.941 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:12.942 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:37:12.942 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:12.942 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89738 2023-09-23T23:37:12.942 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.942 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:12.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.942 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:12.942 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801877+0000", 2023-09-23T23:37:12.942 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.942 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.942 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:12.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801877+0000", 2023-09-23T23:37:12.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.943 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:12.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801876+0000", 2023-09-23T23:37:12.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:12.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.943 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:12.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801883+0000", 2023-09-23T23:37:12.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8959999999999997e-06 2023-09-23T23:37:12.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:12.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801895+0000", 2023-09-23T23:37:12.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2709e-05 2023-09-23T23:37:12.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:12.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801905+0000", 2023-09-23T23:37:12.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1589999999999999e-06 2023-09-23T23:37:12.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801984+0000", 2023-09-23T23:37:12.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9043999999999997e-05 2023-09-23T23:37:12.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801997+0000", 2023-09-23T23:37:12.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3294e-05 2023-09-23T23:37:12.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.946 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731046+0000", 2023-09-23T23:37:12.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.92904877500000005 2023-09-23T23:37:12.946 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.946 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.946 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731060+0000", 2023-09-23T23:37:12.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4042e-05 2023-09-23T23:37:12.946 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.946 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.946 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.371733+0000", 2023-09-23T23:37:12.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6406730409999999 2023-09-23T23:37:12.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:37:12.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.372202+0000", 2023-09-23T23:37:12.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000469529 2023-09-23T23:37:12.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.381823+0000", 2023-09-23T23:37:12.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0096204110000000006 2023-09-23T23:37:12.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:12.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.381932+0000", 2023-09-23T23:37:12.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010885 2023-09-23T23:37:12.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:37:12.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.382080+0000", 2023-09-23T23:37:12.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000148555 2023-09-23T23:37:12.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394009+0000", 2023-09-23T23:37:12.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011928887000000001 2023-09-23T23:37:12.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394057+0000", 2023-09-23T23:37:12.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8157e-05 2023-09-23T23:37:12.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:12.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394066+0000", 2023-09-23T23:37:12.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7320000000000004e-06 2023-09-23T23:37:12.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394132+0000", 2023-09-23T23:37:12.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5845000000000004e-05 2023-09-23T23:37:12.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:12.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394208+0000", 2023-09-23T23:37:12.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6352000000000004e-05 2023-09-23T23:37:12.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:12.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397460+0000", 2023-09-23T23:37:12.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032514520000000002 2023-09-23T23:37:12.951 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.951 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:12.951 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.951 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89748 13.3 13:c6535c6c:::benchmark_data_smithi110_541928_object1968:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:37:12.951 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:34.962737+0000", 2023-09-23T23:37:12.951 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 157.88918010200001, 2023-09-23T23:37:12.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.437414017, 2023-09-23T23:37:12.952 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:12.952 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:12.952 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:12.952 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:37:12.952 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:12.952 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89748 2023-09-23T23:37:12.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.952 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:12.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:12.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962737+0000", 2023-09-23T23:37:12.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:12.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962737+0000", 2023-09-23T23:37:12.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:12.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962735+0000", 2023-09-23T23:37:12.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:37:12.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.954 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:12.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962743+0000", 2023-09-23T23:37:12.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7700000000000001e-06 2023-09-23T23:37:12.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.954 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:12.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962756+0000", 2023-09-23T23:37:12.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3013999999999999e-05 2023-09-23T23:37:12.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:12.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962763+0000", 2023-09-23T23:37:12.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7260000000000002e-06 2023-09-23T23:37:12.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962867+0000", 2023-09-23T23:37:12.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103978 2023-09-23T23:37:12.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962878+0000", 2023-09-23T23:37:12.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0927999999999999e-05 2023-09-23T23:37:12.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731084+0000", 2023-09-23T23:37:12.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.76820582999999998 2023-09-23T23:37:12.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731093+0000", 2023-09-23T23:37:12.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6149999999999997e-06 2023-09-23T23:37:12.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.375727+0000", 2023-09-23T23:37:12.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6446342889999999 2023-09-23T23:37:12.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:37:12.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.376199+0000", 2023-09-23T23:37:12.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047155199999999997 2023-09-23T23:37:12.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.390857+0000", 2023-09-23T23:37:12.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014658200999999999 2023-09-23T23:37:12.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:12.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.390980+0000", 2023-09-23T23:37:12.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000122774 2023-09-23T23:37:12.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:37:12.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.391164+0000", 2023-09-23T23:37:12.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018492700000000001 2023-09-23T23:37:12.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397524+0000", 2023-09-23T23:37:12.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063597879999999999 2023-09-23T23:37:12.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:12.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397564+0000", 2023-09-23T23:37:12.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9746999999999997e-05 2023-09-23T23:37:12.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397611+0000", 2023-09-23T23:37:12.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7403e-05 2023-09-23T23:37:12.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397668+0000", 2023-09-23T23:37:12.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6178999999999999e-05 2023-09-23T23:37:12.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:12.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397732+0000", 2023-09-23T23:37:12.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4548999999999995e-05 2023-09-23T23:37:12.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:12.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.400151+0000", 2023-09-23T23:37:12.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024186429999999998 2023-09-23T23:37:12.961 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.962 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:12.962 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.962 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89756 13.3 13:cf60a43c:::benchmark_data_smithi110_541928_object1976:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:37:12.962 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.051526+0000", 2023-09-23T23:37:12.962 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 157.80039132900001, 2023-09-23T23:37:12.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4865002390000002, 2023-09-23T23:37:12.962 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:12.962 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:12.963 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:12.963 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:37:12.963 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:12.963 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89756 2023-09-23T23:37:12.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.963 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:12.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:12.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051526+0000", 2023-09-23T23:37:12.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:12.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051526+0000", 2023-09-23T23:37:12.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:12.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051524+0000", 2023-09-23T23:37:12.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:12.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:12.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051532+0000", 2023-09-23T23:37:12.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6310000000000003e-06 2023-09-23T23:37:12.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:12.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051559+0000", 2023-09-23T23:37:12.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7444999999999999e-05 2023-09-23T23:37:12.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:12.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051568+0000", 2023-09-23T23:37:12.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8310000000000007e-06 2023-09-23T23:37:12.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051616+0000", 2023-09-23T23:37:12.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8168999999999999e-05 2023-09-23T23:37:12.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051624+0000", 2023-09-23T23:37:12.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2290000000000003e-06 2023-09-23T23:37:12.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731118+0000", 2023-09-23T23:37:12.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.67949406400000001 2023-09-23T23:37:12.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731126+0000", 2023-09-23T23:37:12.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0949999999999996e-06 2023-09-23T23:37:12.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.400190+0000", 2023-09-23T23:37:12.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.669063559 2023-09-23T23:37:12.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:37:12.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.400555+0000", 2023-09-23T23:37:12.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036511099999999999 2023-09-23T23:37:12.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.416430+0000", 2023-09-23T23:37:12.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015875185999999999 2023-09-23T23:37:12.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:12.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.416532+0000", 2023-09-23T23:37:12.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000101456 2023-09-23T23:37:12.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:37:12.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.416657+0000", 2023-09-23T23:37:12.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012510200000000001 2023-09-23T23:37:12.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:12.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.434768+0000", 2023-09-23T23:37:12.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018111008000000001 2023-09-23T23:37:12.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.434835+0000", 2023-09-23T23:37:12.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7475e-05 2023-09-23T23:37:12.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.534925+0000", 2023-09-23T23:37:12.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.100090106 2023-09-23T23:37:12.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.534976+0000", 2023-09-23T23:37:12.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0603999999999999e-05 2023-09-23T23:37:12.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:12.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.535014+0000", 2023-09-23T23:37:12.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8379e-05 2023-09-23T23:37:12.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:12.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.538026+0000", 2023-09-23T23:37:12.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003011391 2023-09-23T23:37:12.972 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.972 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:12.972 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.973 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89769 13.3 13:c7ccd55e:::benchmark_data_smithi110_541928_object1989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:37:12.973 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.139666+0000", 2023-09-23T23:37:12.973 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 157.712250794, 2023-09-23T23:37:12.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4017686810000001, 2023-09-23T23:37:12.973 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:12.973 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:12.973 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:12.973 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:37:12.973 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:12.973 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89769 2023-09-23T23:37:12.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.974 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:12.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:12.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139666+0000", 2023-09-23T23:37:12.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:12.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139666+0000", 2023-09-23T23:37:12.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:12.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139665+0000", 2023-09-23T23:37:12.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:12.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:12.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139673+0000", 2023-09-23T23:37:12.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1049999999999995e-06 2023-09-23T23:37:12.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:12.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139689+0000", 2023-09-23T23:37:12.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6656000000000001e-05 2023-09-23T23:37:12.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:12.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139701+0000", 2023-09-23T23:37:12.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1127e-05 2023-09-23T23:37:12.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139754+0000", 2023-09-23T23:37:12.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3362000000000002e-05 2023-09-23T23:37:12.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139764+0000", 2023-09-23T23:37:12.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0474e-05 2023-09-23T23:37:12.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731150+0000", 2023-09-23T23:37:12.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.59138585200000005 2023-09-23T23:37:12.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.735349+0000", 2023-09-23T23:37:12.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041989519999999997 2023-09-23T23:37:12.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.400620+0000", 2023-09-23T23:37:12.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6652705239999999 2023-09-23T23:37:12.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:37:12.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.401065+0000", 2023-09-23T23:37:12.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00044538800000000001 2023-09-23T23:37:12.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.424058+0000", 2023-09-23T23:37:12.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022993085999999999 2023-09-23T23:37:12.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:12.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.424146+0000", 2023-09-23T23:37:12.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7412999999999998e-05 2023-09-23T23:37:12.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:37:12.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.424253+0000", 2023-09-23T23:37:12.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107246 2023-09-23T23:37:12.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:12.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.440042+0000", 2023-09-23T23:37:12.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015789306999999999 2023-09-23T23:37:12.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.443102+0000", 2023-09-23T23:37:12.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030595700000000002 2023-09-23T23:37:12.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.538201+0000", 2023-09-23T23:37:12.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.095099702999999994 2023-09-23T23:37:12.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.538249+0000", 2023-09-23T23:37:12.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7549000000000002e-05 2023-09-23T23:37:12.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:12.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.538328+0000", 2023-09-23T23:37:12.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8671000000000006e-05 2023-09-23T23:37:12.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:12.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.541435+0000", 2023-09-23T23:37:12.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031071649999999998 2023-09-23T23:37:12.983 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.983 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:12.983 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.983 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89790 13.3 13:ce3e2df6:::benchmark_data_smithi110_541928_object2010:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:37:12.984 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.273997+0000", 2023-09-23T23:37:12.984 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 157.577920205, 2023-09-23T23:37:12.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.270580657, 2023-09-23T23:37:12.984 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:12.984 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:12.984 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:12.984 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:37:12.984 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:12.984 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89790 2023-09-23T23:37:12.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.984 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:12.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:12.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.273997+0000", 2023-09-23T23:37:12.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:12.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.273997+0000", 2023-09-23T23:37:12.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:12.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.273995+0000", 2023-09-23T23:37:12.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:12.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:12.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274004+0000", 2023-09-23T23:37:12.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7940000000000008e-06 2023-09-23T23:37:12.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:12.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274017+0000", 2023-09-23T23:37:12.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3405e-05 2023-09-23T23:37:12.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:12.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274027+0000", 2023-09-23T23:37:12.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7200000000000001e-06 2023-09-23T23:37:12.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274086+0000", 2023-09-23T23:37:12.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.94e-05 2023-09-23T23:37:12.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274101+0000", 2023-09-23T23:37:12.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4109e-05 2023-09-23T23:37:12.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.735405+0000", 2023-09-23T23:37:12.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.46130481400000001 2023-09-23T23:37:12.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.735419+0000", 2023-09-23T23:37:12.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3217000000000001e-05 2023-09-23T23:37:12.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.401121+0000", 2023-09-23T23:37:12.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6657021570000001 2023-09-23T23:37:12.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:37:12.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.401570+0000", 2023-09-23T23:37:12.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000449343 2023-09-23T23:37:12.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.431522+0000", 2023-09-23T23:37:12.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029951951000000001 2023-09-23T23:37:12.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:12.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.431654+0000", 2023-09-23T23:37:12.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013153199999999999 2023-09-23T23:37:12.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:37:12.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.431813+0000", 2023-09-23T23:37:12.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000159937 2023-09-23T23:37:12.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:12.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.451910+0000", 2023-09-23T23:37:12.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02009613 2023-09-23T23:37:12.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.451987+0000", 2023-09-23T23:37:12.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7597999999999998e-05 2023-09-23T23:37:12.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.541568+0000", 2023-09-23T23:37:12.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.089580504000000005 2023-09-23T23:37:12.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:12.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.541608+0000", 2023-09-23T23:37:12.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0274000000000001e-05 2023-09-23T23:37:12.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:12.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.541675+0000", 2023-09-23T23:37:12.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6890000000000005e-05 2023-09-23T23:37:12.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:12.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.544577+0000", 2023-09-23T23:37:12.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029024509999999999 2023-09-23T23:37:12.995 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.995 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:12.995 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:12.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.995 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89805 13.3 13:c4e9783b:::benchmark_data_smithi110_541928_object2025:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e571)", 2023-09-23T23:37:12.995 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.955885+0000", 2023-09-23T23:37:12.995 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 156.89603201400001, 2023-09-23T23:37:12.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.592296927, 2023-09-23T23:37:12.996 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:12.996 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:12.996 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:12.996 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:37:12.996 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:12.996 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89805 2023-09-23T23:37:12.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.996 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:12.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:12.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955885+0000", 2023-09-23T23:37:12.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:12.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955885+0000", 2023-09-23T23:37:12.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:12.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:12.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955883+0000", 2023-09-23T23:37:12.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:12.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:12.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955891+0000", 2023-09-23T23:37:12.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0209999999999999e-06 2023-09-23T23:37:12.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:12.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955908+0000", 2023-09-23T23:37:12.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.643e-05 2023-09-23T23:37:12.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:12.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955917+0000", 2023-09-23T23:37:12.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2089999999999994e-06 2023-09-23T23:37:12.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:12.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955987+0000", 2023-09-23T23:37:12.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9494999999999998e-05 2023-09-23T23:37:12.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:12.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:12.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:12.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.956002+0000", 2023-09-23T23:37:13.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5223e-05 2023-09-23T23:37:13.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:13.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.401618+0000", 2023-09-23T23:37:13.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4456160360000001 2023-09-23T23:37:13.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:37:13.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.402009+0000", 2023-09-23T23:37:13.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039078399999999998 2023-09-23T23:37:13.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:13.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.440091+0000", 2023-09-23T23:37:13.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038082498999999999 2023-09-23T23:37:13.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:13.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.440216+0000", 2023-09-23T23:37:13.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012535399999999999 2023-09-23T23:37:13.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:37:13.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.440388+0000", 2023-09-23T23:37:13.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017125300000000001 2023-09-23T23:37:13.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:13.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.457767+0000", 2023-09-23T23:37:13.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017379131999999999 2023-09-23T23:37:13.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:13.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.460556+0000", 2023-09-23T23:37:13.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027891050000000001 2023-09-23T23:37:13.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:13.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.544650+0000", 2023-09-23T23:37:13.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.084093672999999994 2023-09-23T23:37:13.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:13.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.544694+0000", 2023-09-23T23:37:13.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4366000000000002e-05 2023-09-23T23:37:13.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:13.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.544759+0000", 2023-09-23T23:37:13.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5371999999999997e-05 2023-09-23T23:37:13.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:13.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.548182+0000", 2023-09-23T23:37:13.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034224870000000001 2023-09-23T23:37:13.004 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:13.004 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:13.004 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:13.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.005 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89823 13.3 13:c65a1b7a:::benchmark_data_smithi110_541928_object2043:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e571)", 2023-09-23T23:37:13.005 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.990158+0000", 2023-09-23T23:37:13.005 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 156.86175901999999, 2023-09-23T23:37:13.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5615935460000001, 2023-09-23T23:37:13.005 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:13.005 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:13.005 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:13.005 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:37:13.005 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:13.005 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89823 2023-09-23T23:37:13.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.005 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:13.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:13.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990158+0000", 2023-09-23T23:37:13.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:13.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:13.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990158+0000", 2023-09-23T23:37:13.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:13.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:13.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990157+0000", 2023-09-23T23:37:13.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:37:13.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:13.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990162+0000", 2023-09-23T23:37:13.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1359999999999996e-06 2023-09-23T23:37:13.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:13.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990174+0000", 2023-09-23T23:37:13.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1695000000000001e-05 2023-09-23T23:37:13.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:13.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990184+0000", 2023-09-23T23:37:13.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0465000000000001e-05 2023-09-23T23:37:13.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:13.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990231+0000", 2023-09-23T23:37:13.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6329000000000003e-05 2023-09-23T23:37:13.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:37:13.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990240+0000", 2023-09-23T23:37:13.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5310000000000007e-06 2023-09-23T23:37:13.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:13.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.402069+0000", 2023-09-23T23:37:13.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.411828474 2023-09-23T23:37:13.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:37:13.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.402423+0000", 2023-09-23T23:37:13.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035466000000000002 2023-09-23T23:37:13.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:13.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.448598+0000", 2023-09-23T23:37:13.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.046175040000000001 2023-09-23T23:37:13.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:13.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.448732+0000", 2023-09-23T23:37:13.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013384599999999999 2023-09-23T23:37:13.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:37:13.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.448935+0000", 2023-09-23T23:37:13.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020236700000000001 2023-09-23T23:37:13.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:13.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.460571+0000", 2023-09-23T23:37:13.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011636069000000001 2023-09-23T23:37:13.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:13.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.468058+0000", 2023-09-23T23:37:13.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074870570000000001 2023-09-23T23:37:13.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:13.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.548273+0000", 2023-09-23T23:37:13.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.080215195000000003 2023-09-23T23:37:13.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:13.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.548328+0000", 2023-09-23T23:37:13.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4493000000000002e-05 2023-09-23T23:37:13.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:13.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.548424+0000", 2023-09-23T23:37:13.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6016999999999999e-05 2023-09-23T23:37:13.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:13.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.551751+0000", 2023-09-23T23:37:13.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033278750000000001 2023-09-23T23:37:13.014 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:13.014 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:13.014 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:13.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.014 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:95149 13.d 13:bc7a1628:::benchmark_data_smithi110_541928_object7369:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T23:37:13.014 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:53.232568+0000", 2023-09-23T23:37:13.014 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 139.61934871099999, 2023-09-23T23:37:13.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4660194620000002, 2023-09-23T23:37:13.014 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:13.015 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:13.015 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:13.015 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:37:13.015 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:13.015 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95149 2023-09-23T23:37:13.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.015 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:13.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:13.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232568+0000", 2023-09-23T23:37:13.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:13.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:13.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232568+0000", 2023-09-23T23:37:13.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:13.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:13.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232567+0000", 2023-09-23T23:37:13.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:13.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:13.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232574+0000", 2023-09-23T23:37:13.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.824e-06 2023-09-23T23:37:13.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:13.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232588+0000", 2023-09-23T23:37:13.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3597e-05 2023-09-23T23:37:13.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:13.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232596+0000", 2023-09-23T23:37:13.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8120000000000008e-06 2023-09-23T23:37:13.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:13.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.737972+0000", 2023-09-23T23:37:13.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.50537641300000002 2023-09-23T23:37:13.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:37:13.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.741095+0000", 2023-09-23T23:37:13.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031224780000000001 2023-09-23T23:37:13.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:13.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.346073+0000", 2023-09-23T23:37:13.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.60497817499999995 2023-09-23T23:37:13.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:37:13.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.351441+0000", 2023-09-23T23:37:13.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053685449999999997 2023-09-23T23:37:13.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:13.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.606909+0000", 2023-09-23T23:37:13.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2554679260000001 2023-09-23T23:37:13.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:13.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.612108+0000", 2023-09-23T23:37:13.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051983919999999996 2023-09-23T23:37:13.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:37:13.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.632573+0000", 2023-09-23T23:37:13.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020465006000000001 2023-09-23T23:37:13.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:13.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.668063+0000", 2023-09-23T23:37:13.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.035490090000000002 2023-09-23T23:37:13.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:13.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.680799+0000", 2023-09-23T23:37:13.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012736032 2023-09-23T23:37:13.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:13.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.682749+0000", 2023-09-23T23:37:13.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019503509999999999 2023-09-23T23:37:13.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:13.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.685937+0000", 2023-09-23T23:37:13.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031873520000000001 2023-09-23T23:37:13.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:13.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.698588+0000", 2023-09-23T23:37:13.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012651051999999999 2023-09-23T23:37:13.022 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:13.023 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:13.023 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:13.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.023 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:95282 13.d 13:b9f8e077:::benchmark_data_smithi110_541928_object7502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T23:37:13.023 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:53.626451+0000", 2023-09-23T23:37:13.023 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 139.22546561600001, 2023-09-23T23:37:13.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5002010320000001, 2023-09-23T23:37:13.023 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:13.023 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:13.023 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:13.023 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:37:13.024 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:13.024 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95282 2023-09-23T23:37:13.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.024 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:13.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:13.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626451+0000", 2023-09-23T23:37:13.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:13.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:13.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626451+0000", 2023-09-23T23:37:13.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:13.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:13.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626449+0000", 2023-09-23T23:37:13.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:37:13.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:13.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626459+0000", 2023-09-23T23:37:13.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7790000000000007e-06 2023-09-23T23:37:13.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:13.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626473+0000", 2023-09-23T23:37:13.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3655000000000001e-05 2023-09-23T23:37:13.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:13.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626483+0000", 2023-09-23T23:37:13.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.013e-05 2023-09-23T23:37:13.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:13.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.973570+0000", 2023-09-23T23:37:13.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.34708731599999998 2023-09-23T23:37:13.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:37:13.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.978239+0000", 2023-09-23T23:37:13.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046685850000000003 2023-09-23T23:37:13.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:13.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.019769+0000", 2023-09-23T23:37:13.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041529772 2023-09-23T23:37:13.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:37:13.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.024313+0000", 2023-09-23T23:37:13.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045445479999999998 2023-09-23T23:37:13.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:13.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.064268+0000", 2023-09-23T23:37:13.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.039955124 2023-09-23T23:37:13.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:13.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.067970+0000", 2023-09-23T23:37:13.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00370164 2023-09-23T23:37:13.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:37:13.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.076658+0000", 2023-09-23T23:37:13.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086883770000000006 2023-09-23T23:37:13.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:13.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.104140+0000", 2023-09-23T23:37:13.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02748134 2023-09-23T23:37:13.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:13.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.113193+0000", 2023-09-23T23:37:13.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0090528350000000004 2023-09-23T23:37:13.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:13.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.114921+0000", 2023-09-23T23:37:13.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017289149999999999 2023-09-23T23:37:13.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:13.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.117081+0000", 2023-09-23T23:37:13.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021593519999999998 2023-09-23T23:37:13.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:13.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.126652+0000", 2023-09-23T23:37:13.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.009571474 2023-09-23T23:37:13.031 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:13.031 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:13.031 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:13.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.032 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:95302 13.d 13:b56e9a5b:::benchmark_data_smithi110_541928_object7522:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T23:37:13.032 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:53.699709+0000", 2023-09-23T23:37:13.032 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 139.15220745900001, 2023-09-23T23:37:13.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.504555552, 2023-09-23T23:37:13.032 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:13.032 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:13.032 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:13.032 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:37:13.032 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:13.032 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95302 2023-09-23T23:37:13.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.033 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:13.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:13.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699709+0000", 2023-09-23T23:37:13.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:13.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:13.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699709+0000", 2023-09-23T23:37:13.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:13.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:13.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699708+0000", 2023-09-23T23:37:13.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:37:13.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:13.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699717+0000", 2023-09-23T23:37:13.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0899999999999994e-06 2023-09-23T23:37:13.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:13.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699730+0000", 2023-09-23T23:37:13.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3392e-05 2023-09-23T23:37:13.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:13.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699740+0000", 2023-09-23T23:37:13.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5559999999999996e-06 2023-09-23T23:37:13.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:13.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.025484+0000", 2023-09-23T23:37:13.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.325744649 2023-09-23T23:37:13.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:37:13.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.030494+0000", 2023-09-23T23:37:13.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.005009365 2023-09-23T23:37:13.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:13.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.166809+0000", 2023-09-23T23:37:13.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.13631576300000001 2023-09-23T23:37:13.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:37:13.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.171655+0000", 2023-09-23T23:37:13.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048457680000000003 2023-09-23T23:37:13.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:13.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.138451+0000", 2023-09-23T23:37:13.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96679576300000003 2023-09-23T23:37:13.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:13.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.142480+0000", 2023-09-23T23:37:13.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040295679999999999 2023-09-23T23:37:13.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:37:13.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.150317+0000", 2023-09-23T23:37:13.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078367109999999997 2023-09-23T23:37:13.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:13.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.186750+0000", 2023-09-23T23:37:13.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036433140000000003 2023-09-23T23:37:13.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:13.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.188250+0000", 2023-09-23T23:37:13.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001500145 2023-09-23T23:37:13.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:13.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.188845+0000", 2023-09-23T23:37:13.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00059478099999999996 2023-09-23T23:37:13.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:13.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.191339+0000", 2023-09-23T23:37:13.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024934559999999998 2023-09-23T23:37:13.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:13.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.204265+0000", 2023-09-23T23:37:13.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012926272000000001 2023-09-23T23:37:13.040 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:13.040 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:13.040 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:13.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.040 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:95356 13.d 13:b7510188:::benchmark_data_smithi110_541928_object7576:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T23:37:13.041 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:53.853143+0000", 2023-09-23T23:37:13.041 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 138.998773704, 2023-09-23T23:37:13.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5531699410000002, 2023-09-23T23:37:13.041 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:37:13.041 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:13.041 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:37:13.041 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:37:13.041 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:13.041 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95356 2023-09-23T23:37:13.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.041 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:37:13.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:37:13.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853143+0000", 2023-09-23T23:37:13.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:13.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:37:13.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853143+0000", 2023-09-23T23:37:13.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:37:13.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:37:13.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853142+0000", 2023-09-23T23:37:13.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:13.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:37:13.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853150+0000", 2023-09-23T23:37:13.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8709999999999997e-06 2023-09-23T23:37:13.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:37:13.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853166+0000", 2023-09-23T23:37:13.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6568000000000001e-05 2023-09-23T23:37:13.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:37:13.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853176+0000", 2023-09-23T23:37:13.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9520000000000006e-06 2023-09-23T23:37:13.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:13.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.200709+0000", 2023-09-23T23:37:13.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.347532703 2023-09-23T23:37:13.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:37:13.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.205641+0000", 2023-09-23T23:37:13.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.004932199 2023-09-23T23:37:13.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:13.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.252766+0000", 2023-09-23T23:37:13.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047124528999999998 2023-09-23T23:37:13.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:37:13.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.257597+0000", 2023-09-23T23:37:13.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048309980000000004 2023-09-23T23:37:13.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:37:13.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.269273+0000", 2023-09-23T23:37:13.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.01167611 2023-09-23T23:37:13.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:37:13.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.276153+0000", 2023-09-23T23:37:13.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068806689999999998 2023-09-23T23:37:13.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:37:13.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.296160+0000", 2023-09-23T23:37:13.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020006954 2023-09-23T23:37:13.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:13.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.310175+0000", 2023-09-23T23:37:13.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014014235 2023-09-23T23:37:13.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:13.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.313239+0000", 2023-09-23T23:37:13.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030640490000000001 2023-09-23T23:37:13.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:37:13.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.383950+0000", 2023-09-23T23:37:13.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.070711100999999998 2023-09-23T23:37:13.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:37:13.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.388728+0000", 2023-09-23T23:37:13.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047779540000000001 2023-09-23T23:37:13.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:37:13.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:37:13.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:37:13.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.406313+0000", 2023-09-23T23:37:13.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017585478000000002 2023-09-23T23:37:13.049 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:13.049 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:13.049 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:13.049 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:37:13.049 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:37:13.050 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:37:13.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:13.051 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:13.051 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:13.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:13.152 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:13.190 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:13.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:13 smithi110 ceph-mon[221102]: pgmap v1679: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 307 op/s; 24038/151464 objects degraded (15.870%); 3027/151464 objects misplaced (1.998%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T23:37:13.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:13.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:37:13.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:13 smithi110 ceph-mon[212939]: pgmap v1679: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 307 op/s; 24038/151464 objects degraded (15.870%); 3027/151464 objects misplaced (1.998%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T23:37:13.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:13.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:37:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:13 smithi138 ceph-mon[193606]: pgmap v1679: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 307 op/s; 24038/151464 objects degraded (15.870%); 3027/151464 objects misplaced (1.998%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T23:37:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:37:13.795 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:37:13.824 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:13.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:13.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:13.930 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:13.956 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:13.998 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:14.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:14.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:14.106 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:37:14.142 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:14.210 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:37:14.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:14.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:37:14.316 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:37:14.387 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:14.491 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:37:14.518 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:14 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 24038/151464 objects degraded (15.870%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:14.658 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:37:14.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:14.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:37:14.665 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:14 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 24038/151464 objects degraded (15.870%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:14.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:14 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 24038/151464 objects degraded (15.870%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:14.710 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:14.765 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:14.830 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:14.886 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:37:14.948 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:37:14.998 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:15.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:15.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:15.108 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:15.134 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:15.149 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:37:15.175 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:37:15.211 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:37:15.248 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:37:15.257 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:15.259 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:15 smithi110 ceph-mon[212939]: pgmap v1680: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 309 op/s; 23948/150318 objects degraded (15.932%); 2927/150318 objects misplaced (1.947%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T23:37:15.259 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:15 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:15.259 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:15 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:15.259 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:15 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3082524278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:15.259 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:15 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:37:15.259 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:15 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:37:15.259 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:15 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:15.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:15.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:15.281 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:37:15.310 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:37:15.339 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:37:15.361 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:15.370 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:37:15.372 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:37:15.372 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:37:15.372 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:37:15.372 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:37:15.372 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:37:15.372 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:37:15.373 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:37:15.373 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:37:15.399 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:15 smithi138 ceph-mon[193606]: pgmap v1680: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 309 op/s; 23948/150318 objects degraded (15.932%); 2927/150318 objects misplaced (1.947%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T23:37:15.400 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:15 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:15.400 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:15 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:15.400 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:15 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3082524278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:15.400 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:15 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:37:15.400 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:15 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:37:15.400 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:15 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:15.416 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:15.514 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:15 smithi110 ceph-mon[221102]: pgmap v1680: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 309 op/s; 23948/150318 objects degraded (15.932%); 2927/150318 objects misplaced (1.947%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T23:37:15.514 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:15 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:15.514 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:15 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:15.514 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:15 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3082524278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:15.514 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:15 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:37:15.514 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:15 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:37:15.514 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:15 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:16.539 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:16.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:16.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:16.642 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:16.684 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:16.804 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:37:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:37:16] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-09-23T23:37:16.834 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:16.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:16.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:16.847 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:16.939 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:16.966 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:17.069 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:17.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:17.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:17.121 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:37:17.121 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:37:17.174 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:37:17.206 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:17.356 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:37:17.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:17 smithi138 ceph-mon[193606]: pgmap v1681: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 23856/146892 objects degraded (16.241%); 2927/146892 objects misplaced (1.993%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:37:17.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:17.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:37:17.460 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:17.486 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:17.529 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:17.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:17.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:17.632 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:37:17.670 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:17 smithi110 ceph-mon[212939]: pgmap v1681: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 23856/146892 objects degraded (16.241%); 2927/146892 objects misplaced (1.993%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:37:17.671 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:17 smithi110 ceph-mon[221102]: pgmap v1681: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 23856/146892 objects degraded (16.241%); 2927/146892 objects misplaced (1.993%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:37:17.680 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:17.728 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:37:17.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:17.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:37:17.831 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:17.875 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:17.947 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:17.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:17.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:18.045 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:37:18.051 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:18.121 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:18.174 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:18.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:18.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:18.283 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:37:18.286 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:37:18.332 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2809985594' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:37:18.332 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:18 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:37:18.338 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:18.398 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:37:18.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:18.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:37:18.504 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:37:18.522 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2809985594' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:37:18.523 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:18 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:37:18.524 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2809985594' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:37:18.524 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:18 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:37:18.543 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:37:18.550 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:18.618 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:37:18.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:18.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:37:18.725 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:37:18.754 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:37:18.759 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:18.817 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:37:18.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:18.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:37:18.921 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:18.948 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:18.992 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:18.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:18.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:19.095 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:19.125 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:19.173 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:19.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:19.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:19.281 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:19.307 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:19.349 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:19.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:19.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:19.419 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:19 smithi138 ceph-mon[193606]: pgmap v1682: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 23856/146892 objects degraded (16.241%); 2927/146892 objects misplaced (1.993%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:37:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:19 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:19 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:37:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:19 smithi138 ceph-mon[193606]: osdmap e600: 8 total, 8 up, 7 in 2023-09-23T23:37:19.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:19 smithi138 ceph-mon[193606]: failed to encode map e600 with expected crc 2023-09-23T23:37:19.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:19 smithi138 ceph-mon[193606]: failed to encode map e600 with expected crc 2023-09-23T23:37:19.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:19 smithi138 ceph-mon[193606]: failed to encode map e600 with expected crc 2023-09-23T23:37:19.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:19 smithi138 ceph-mon[193606]: failed to encode map e600 with expected crc 2023-09-23T23:37:19.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:19 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 23856/146892 objects degraded (16.241%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:19.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[221102]: pgmap v1682: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 23856/146892 objects degraded (16.241%); 2927/146892 objects misplaced (1.993%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:37:19.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:19.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:37:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[221102]: osdmap e600: 8 total, 8 up, 7 in 2023-09-23T23:37:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[221102]: failed to encode map e600 with expected crc 2023-09-23T23:37:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[221102]: failed to encode map e600 with expected crc 2023-09-23T23:37:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[221102]: failed to encode map e600 with expected crc 2023-09-23T23:37:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[221102]: failed to encode map e600 with expected crc 2023-09-23T23:37:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 23856/146892 objects degraded (16.241%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:19.453 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:19.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[212939]: pgmap v1682: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 23856/146892 objects degraded (16.241%); 2927/146892 objects misplaced (1.993%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:37:19.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:19.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:37:19.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[212939]: osdmap e600: 8 total, 8 up, 7 in 2023-09-23T23:37:19.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[212939]: failed to encode map e600 with expected crc 2023-09-23T23:37:19.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[212939]: failed to encode map e600 with expected crc 2023-09-23T23:37:19.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[212939]: failed to encode map e600 with expected crc 2023-09-23T23:37:19.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[212939]: failed to encode map e600 with expected crc 2023-09-23T23:37:19.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:19 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 23856/146892 objects degraded (16.241%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:19.480 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:19.570 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:19.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:19.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:19.675 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:37:19.707 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:37:19.720 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:37:19.725 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:19.791 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:37:19.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:19.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:37:19.897 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:19.932 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:20.036 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:20.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:20.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:20.141 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:20.176 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:20.386 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:20.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:20.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:20.426 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:37:20.426 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:37:20.490 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:37:20.531 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:21.008 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:37:21.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:21.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:37:21.058 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1683,"stamp":"2023-09-23T23:37:18.272606+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3182408949,"num_objects":48575,"num_object_clones":0,"num_object_copies":145725,"num_objects_missing_on_primary":1931,"num_objects_missing":1931,"num_objects_degraded":23747,"num_objects_misplaced":2826,"num_objects_unfound":0,"num_objects_dirty":48575,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":127082,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17024,"num_bytes_recovered":1112994216,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95129,"ondisk_log_size":95129,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14098028,"kb_used_data":8375988,"kb_used_omap":29,"kb_used_meta":5721890,"kb_avail":642118036,"statfs":{"total":671965249536,"available":657528868864,"internally_reserved":0,"allocated":8577011712,"data_stored":8560663696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5859216083},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-146079744,"num_objects":-2229,"num_object_clones":0,"num_object_copies":-6687,"num_objects_missing_on_primary":-200,"num_objects_missing":-200,"num_objects_degraded":-291,"num_objects_misplaced":-201,"num_objects_unfound":0,"num_objects_dirty":-2229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2229,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":493,"num_bytes_recovered":32309248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.336869"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":795,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.773787+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:36:59.773787+0000","last_peered":"2023-09-23T23:36:59.773787+0000","last_clean":"2023-09-23T23:36:59.773787+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:36:59.773787+0000","last_undegraded":"2023-09-23T23:36:59.773787+0000","last_fullsized":"2023-09-23T23:36:59.773787+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":354,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.782908+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:36:59.782908+0000","last_peered":"2023-09-23T23:36:59.782908+0000","last_clean":"2023-09-23T23:36:59.782908+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:36:59.782908+0000","last_undegraded":"2023-09-23T23:36:59.782908+0000","last_fullsized":"2023-09-23T23:36:59.782908+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":581,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.826157+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:36:59.826157+0000","last_peered":"2023-09-23T23:36:59.826157+0000","last_clean":"2023-09-23T23:36:59.826157+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:36:59.826157+0000","last_undegraded":"2023-09-23T23:36:59.826157+0000","last_fullsized":"2023-09-23T23:36:59.826157+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":748,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.775307+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:36:59.775307+0000","last_peered":"2023-09-23T23:36:59.775307+0000","last_clean":"2023-09-23T23:36:59.775307+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:36:59.775307+0000","last_undegraded":"2023-09-23T23:36:59.775307+0000","last_fullsized":"2023-09-23T23:36:59.775307+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":318,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.792342+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:36:59.792342+0000","last_peered":"2023-09-23T23:36:59.792342+0000","last_clean":"2023-09-23T23:36:59.792342+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:36:59.792342+0000","last_undegraded":"2023-09-23T23:36:59.792342+0000","last_fullsized":"2023-09-23T23:36:59.792342+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":560,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.793859+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:36:59.793859+0000","last_peered":"2023-09-23T23:36:59.793859+0000","last_clean":"2023-09-23T23:36:59.793859+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:36:59.793859+0000","last_undegraded":"2023-09-23T23:36:59.793859+0000","last_fullsized":"2023-09-23T23:36:59.793859+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":760,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.775146+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:36:59.775146+0000","last_peered":"2023-09-23T23:36:59.775146+0000","last_clean":"2023-09-23T23:36:59.775146+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:36:59.775146+0000","last_undegraded":"2023-09-23T23:36:59.775146+0000","last_fullsized":"2023-09-23T23:36:59.775146+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":617,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.791689+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:36:59.791689+0000","last_peered":"2023-09-23T23:36:59.791689+0000","last_clean":"2023-09-23T23:36:59.791689+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:36:59.791689+0000","last_undegraded":"2023-09-23T23:36:59.791689+0000","last_fullsized":"2023-09-23T23:36:59.791689+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":113,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.783822+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:36:59.783822+0000","last_peered":"2023-09-23T23:36:59.783822+0000","last_clean":"2023-09-23T23:36:59.783822+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:36:59.783822+0000","last_undegraded":"2023-09-23T23:36:59.783822+0000","last_fullsized":"2023-09-23T23:36:59.783822+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":919,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.775055+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:36:59.775055+0000","last_peered":"2023-09-23T23:36:59.775055+0000","last_clean":"2023-09-23T23:36:59.775055+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:36:59.775055+0000","last_undegraded":"2023-09-23T23:36:59.775055+0000","last_fullsized":"2023-09-23T23:36:59.775055+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":251,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.773669+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:36:59.773669+0000","last_peered":"2023-09-23T23:36:59.773669+0000","last_clean":"2023-09-23T23:36:59.773669+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:36:59.773669+0000","last_undegraded":"2023-09-23T23:36:59.773669+0000","last_fullsized":"2023-09-23T23:36:59.773669+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":600,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.791635+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:36:59.791635+0000","last_peered":"2023-09-23T23:36:59.791635+0000","last_clean":"2023-09-23T23:36:59.791635+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:36:59.791635+0000","last_undegraded":"2023-09-23T23:36:59.791635+0000","last_fullsized":"2023-09-23T23:36:59.791635+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":892,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.783260+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:36:59.783260+0000","last_peered":"2023-09-23T23:36:59.783260+0000","last_clean":"2023-09-23T23:36:59.783260+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:36:59.783260+0000","last_undegraded":"2023-09-23T23:36:59.783260+0000","last_fullsized":"2023-09-23T23:36:59.783260+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":230,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.775608+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:36:59.775608+0000","last_peered":"2023-09-23T23:36:59.775608+0000","last_clean":"2023-09-23T23:36:59.775608+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:36:59.775608+0000","last_undegraded":"2023-09-23T23:36:59.775608+0000","last_fullsized":"2023-09-23T23:36:59.775608+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":561,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.792137+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:36:59.792137+0000","last_peered":"2023-09-23T23:36:59.792137+0000","last_clean":"2023-09-23T23:36:59.792137+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:36:59.792137+0000","last_undegraded":"2023-09-23T23:36:59.792137+0000","last_fullsized":"2023-09-23T23:36:59.792137+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1053,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.773539+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:36:59.773539+0000","last_peered":"2023-09-23T23:36:59.773539+0000","last_clean":"2023-09-23T23:36:59.773539+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:36:59.773539+0000","last_undegraded":"2023-09-23T23:36:59.773539+0000","last_fullsized":"2023-09-23T23:36:59.773539+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":272,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.803083+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:36:59.803083+0000","last_peered":"2023-09-23T23:36:59.803083+0000","last_clean":"2023-09-23T23:36:59.803083+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:36:59.803083+0000","last_undegraded":"2023-09-23T23:36:59.803083+0000","last_fullsized":"2023-09-23T23:36:59.803083+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":132,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.776996+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:36:59.776996+0000","last_peered":"2023-09-23T23:36:59.776996+0000","last_clean":"2023-09-23T23:36:59.776996+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:36:59.776996+0000","last_undegraded":"2023-09-23T23:36:59.776996+0000","last_fullsized":"2023-09-23T23:36:59.776996+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":450,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.794000+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:36:59.794000+0000","last_peered":"2023-09-23T23:36:59.794000+0000","last_clean":"2023-09-23T23:36:59.794000+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:36:59.794000+0000","last_undegraded":"2023-09-23T23:36:59.794000+0000","last_fullsized":"2023-09-23T23:36:59.794000+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":173,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.777420+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:36:59.777420+0000","last_peered":"2023-09-23T23:36:59.777420+0000","last_clean":"2023-09-23T23:36:59.777420+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:36:59.777420+0000","last_undegraded":"2023-09-23T23:36:59.777420+0000","last_fullsized":"2023-09-23T23:36:59.777420+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":540,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.777267+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:36:59.777267+0000","last_peered":"2023-09-23T23:36:59.777267+0000","last_clean":"2023-09-23T23:36:59.777267+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:36:59.777267+0000","last_undegraded":"2023-09-23T23:36:59.777267+0000","last_fullsized":"2023-09-23T23:36:59.777267+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":754,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.774638+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:36:59.774638+0000","last_peered":"2023-09-23T23:36:59.774638+0000","last_clean":"2023-09-23T23:36:59.774638+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:36:59.774638+0000","last_undegraded":"2023-09-23T23:36:59.774638+0000","last_fullsized":"2023-09-23T23:36:59.774638+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":146,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.781309+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:36:59.781309+0000","last_peered":"2023-09-23T23:36:59.781309+0000","last_clean":"2023-09-23T23:36:59.781309+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:36:59.781309+0000","last_undegraded":"2023-09-23T23:36:59.781309+0000","last_fullsized":"2023-09-23T23:36:59.781309+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":599,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.793320+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:36:59.793320+0000","last_peered":"2023-09-23T23:36:59.793320+0000","last_clean":"2023-09-23T23:36:59.793320+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:36:59.793320+0000","last_undegraded":"2023-09-23T23:36:59.793320+0000","last_fullsized":"2023-09-23T23:36:59.793320+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":884,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.776309+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:36:59.776309+0000","last_peered":"2023-09-23T23:36:59.776309+0000","last_clean":"2023-09-23T23:36:59.776309+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:36:59.776309+0000","last_undegraded":"2023-09-23T23:36:59.776309+0000","last_fullsized":"2023-09-23T23:36:59.776309+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":194,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.792972+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:36:59.792972+0000","last_peered":"2023-09-23T23:36:59.792972+0000","last_clean":"2023-09-23T23:36:59.792972+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:36:59.792972+0000","last_undegraded":"2023-09-23T23:36:59.792972+0000","last_fullsized":"2023-09-23T23:36:59.792972+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":321,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.825980+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:36:59.825980+0000","last_peered":"2023-09-23T23:36:59.825980+0000","last_clean":"2023-09-23T23:36:59.825980+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:36:59.825980+0000","last_undegraded":"2023-09-23T23:36:59.825980+0000","last_fullsized":"2023-09-23T23:36:59.825980+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":562,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.794809+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:36:59.794809+0000","last_peered":"2023-09-23T23:36:59.794809+0000","last_clean":"2023-09-23T23:36:59.794809+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:36:59.794809+0000","last_undegraded":"2023-09-23T23:36:59.794809+0000","last_fullsized":"2023-09-23T23:36:59.794809+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":467,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.778958+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:36:59.778958+0000","last_peered":"2023-09-23T23:36:59.778958+0000","last_clean":"2023-09-23T23:36:59.778958+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:36:59.778958+0000","last_undegraded":"2023-09-23T23:36:59.778958+0000","last_fullsized":"2023-09-23T23:36:59.778958+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":555,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.794143+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:36:59.794143+0000","last_peered":"2023-09-23T23:36:59.794143+0000","last_clean":"2023-09-23T23:36:59.794143+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:36:59.794143+0000","last_undegraded":"2023-09-23T23:36:59.794143+0000","last_fullsized":"2023-09-23T23:36:59.794143+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":962,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.774278+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:36:59.774278+0000","last_peered":"2023-09-23T23:36:59.774278+0000","last_clean":"2023-09-23T23:36:59.774278+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:36:59.774278+0000","last_undegraded":"2023-09-23T23:36:59.774278+0000","last_fullsized":"2023-09-23T23:36:59.774278+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":163,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.776593+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:36:59.776593+0000","last_peered":"2023-09-23T23:36:59.776593+0000","last_clean":"2023-09-23T23:36:59.776593+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:36:59.776593+0000","last_undegraded":"2023-09-23T23:36:59.776593+0000","last_fullsized":"2023-09-23T23:36:59.776593+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":240,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.793663+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:36:59.793663+0000","last_peered":"2023-09-23T23:36:59.793663+0000","last_clean":"2023-09-23T23:36:59.793663+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:36:59.793663+0000","last_undegraded":"2023-09-23T23:36:59.793663+0000","last_fullsized":"2023-09-23T23:36:59.793663+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"599'7787","reported_seq":7869,"reported_epoch":599,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:14.128981+0000","last_change":"2023-09-23T23:33:52.884769+0000","last_active":"2023-09-23T23:37:14.128981+0000","last_peered":"2023-09-23T23:37:14.128981+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:37:14.128981+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"550'2400","ondisk_log_start":"550'2400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5387,"log_dups_size":0,"ondisk_log_size":5387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200605696,"num_objects":3061,"num_object_clones":0,"num_object_copies":9183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7787,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":2035},{"shards":"2,6","objects":-1337}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":933,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.783232+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:36:59.783232+0000","last_peered":"2023-09-23T23:36:59.783232+0000","last_clean":"2023-09-23T23:36:59.783232+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:36:59.783232+0000","last_undegraded":"2023-09-23T23:36:59.783232+0000","last_fullsized":"2023-09-23T23:36:59.783232+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":240,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.774151+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:36:59.774151+0000","last_peered":"2023-09-23T23:36:59.774151+0000","last_clean":"2023-09-23T23:36:59.774151+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:36:59.774151+0000","last_undegraded":"2023-09-23T23:36:59.774151+0000","last_fullsized":"2023-09-23T23:36:59.774151+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":230,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.773705+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:36:59.773705+0000","last_peered":"2023-09-23T23:36:59.773705+0000","last_clean":"2023-09-23T23:36:59.773705+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:36:59.773705+0000","last_undegraded":"2023-09-23T23:36:59.773705+0000","last_fullsized":"2023-09-23T23:36:59.773705+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"599'7948","reported_seq":8030,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:14.865997+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:37:14.865997+0000","last_peered":"2023-09-23T23:37:14.865997+0000","last_clean":"2023-09-23T23:37:14.865997+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:37:14.865997+0000","last_undegraded":"2023-09-23T23:37:14.865997+0000","last_fullsized":"2023-09-23T23:37:14.865997+0000","mapping_epoch":566,"log_start":"549'1800","ondisk_log_start":"549'1800","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6148,"log_dups_size":0,"ondisk_log_size":6148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202113024,"num_objects":3084,"num_object_clones":0,"num_object_copies":9252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7948,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1011,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.793001+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:36:59.793001+0000","last_peered":"2023-09-23T23:36:59.793001+0000","last_clean":"2023-09-23T23:36:59.793001+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:36:59.793001+0000","last_undegraded":"2023-09-23T23:36:59.793001+0000","last_fullsized":"2023-09-23T23:36:59.793001+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":230,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.774709+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:36:59.774709+0000","last_peered":"2023-09-23T23:36:59.774709+0000","last_clean":"2023-09-23T23:36:59.774709+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:36:59.774709+0000","last_undegraded":"2023-09-23T23:36:59.774709+0000","last_fullsized":"2023-09-23T23:36:59.774709+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":450,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.791735+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:36:59.791735+0000","last_peered":"2023-09-23T23:36:59.791735+0000","last_clean":"2023-09-23T23:36:59.791735+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:36:59.791735+0000","last_undegraded":"2023-09-23T23:36:59.791735+0000","last_fullsized":"2023-09-23T23:36:59.791735+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"599'8072","reported_seq":14285,"reported_epoch":599,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:37:18.047442+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:37:18.047442+0000","last_peered":"2023-09-23T23:37:18.047442+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:37:18.047442+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:37:18.047442+0000","mapping_epoch":571,"log_start":"551'2800","ondisk_log_start":"551'2800","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5272,"log_dups_size":0,"ondisk_log_size":5272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195035136,"num_objects":2976,"num_object_clones":0,"num_object_copies":8928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2081,"num_objects_misplaced":895,"num_objects_unfound":0,"num_objects_dirty":2976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8072,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3050,"num_bytes_recovered":199884800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":1959},{"shards":"3,5,7","objects":1017}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":909,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.783905+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:36:59.783905+0000","last_peered":"2023-09-23T23:36:59.783905+0000","last_clean":"2023-09-23T23:36:59.783905+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:36:59.783905+0000","last_undegraded":"2023-09-23T23:36:59.783905+0000","last_fullsized":"2023-09-23T23:36:59.783905+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":337,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.778238+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:36:59.778238+0000","last_peered":"2023-09-23T23:36:59.778238+0000","last_clean":"2023-09-23T23:36:59.778238+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:36:59.778238+0000","last_undegraded":"2023-09-23T23:36:59.778238+0000","last_fullsized":"2023-09-23T23:36:59.778238+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":113,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.777166+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:36:59.777166+0000","last_peered":"2023-09-23T23:36:59.777166+0000","last_clean":"2023-09-23T23:36:59.777166+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:36:59.777166+0000","last_undegraded":"2023-09-23T23:36:59.777166+0000","last_fullsized":"2023-09-23T23:36:59.777166+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"599'8046","reported_seq":10115,"reported_epoch":599,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:18.022562+0000","last_change":"2023-09-23T23:34:35.750428+0000","last_active":"2023-09-23T23:37:18.022562+0000","last_peered":"2023-09-23T23:37:18.022562+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:37:18.022562+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"551'2700","ondisk_log_start":"551'2700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5346,"log_dups_size":0,"ondisk_log_size":5346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199491584,"num_objects":3044,"num_object_clones":0,"num_object_copies":9132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8046,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":991,"num_bytes_recovered":64946176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":2034},{"shards":"3,5","objects":-500}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":506,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.794077+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:36:59.794077+0000","last_peered":"2023-09-23T23:36:59.794077+0000","last_clean":"2023-09-23T23:36:59.794077+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:36:59.794077+0000","last_undegraded":"2023-09-23T23:36:59.794077+0000","last_fullsized":"2023-09-23T23:36:59.794077+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":440,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:37:08.005799+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:37:08.005799+0000","last_peered":"2023-09-23T23:37:08.005799+0000","last_clean":"2023-09-23T23:37:08.005799+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:37:08.005799+0000","last_undegraded":"2023-09-23T23:37:08.005799+0000","last_fullsized":"2023-09-23T23:37:08.005799+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":596,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.824718+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:36:59.824718+0000","last_peered":"2023-09-23T23:36:59.824718+0000","last_clean":"2023-09-23T23:36:59.824718+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:36:59.824718+0000","last_undegraded":"2023-09-23T23:36:59.824718+0000","last_fullsized":"2023-09-23T23:36:59.824718+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"599'8075","reported_seq":9403,"reported_epoch":599,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:14.346136+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:37:14.346136+0000","last_peered":"2023-09-23T23:37:14.346136+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:37:14.346136+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"550'2400","ondisk_log_start":"550'2400","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5675,"log_dups_size":0,"ondisk_log_size":5675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202309632,"num_objects":3087,"num_object_clones":0,"num_object_copies":9261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8075,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":2041},{"shards":"3,5","objects":-2373},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":727,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.792064+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:36:59.792064+0000","last_peered":"2023-09-23T23:36:59.792064+0000","last_clean":"2023-09-23T23:36:59.792064+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:36:59.792064+0000","last_undegraded":"2023-09-23T23:36:59.792064+0000","last_fullsized":"2023-09-23T23:36:59.792064+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":609,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.794739+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:36:59.794739+0000","last_peered":"2023-09-23T23:36:59.794739+0000","last_clean":"2023-09-23T23:36:59.794739+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:36:59.794739+0000","last_undegraded":"2023-09-23T23:36:59.794739+0000","last_fullsized":"2023-09-23T23:36:59.794739+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":520,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.793081+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:36:59.793081+0000","last_peered":"2023-09-23T23:36:59.793081+0000","last_clean":"2023-09-23T23:36:59.793081+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:36:59.793081+0000","last_undegraded":"2023-09-23T23:36:59.793081+0000","last_fullsized":"2023-09-23T23:36:59.793081+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"599'7997","reported_seq":8070,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:14.153446+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:37:14.153446+0000","last_peered":"2023-09-23T23:37:14.153446+0000","last_clean":"2023-09-23T23:37:14.153446+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:37:14.153446+0000","last_undegraded":"2023-09-23T23:37:14.153446+0000","last_fullsized":"2023-09-23T23:37:14.153446+0000","mapping_epoch":534,"log_start":"551'2600","ondisk_log_start":"551'2600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5397,"log_dups_size":0,"ondisk_log_size":5397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202047488,"num_objects":3083,"num_object_clones":0,"num_object_copies":9249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3083,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7997,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":882,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.781594+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:36:59.781594+0000","last_peered":"2023-09-23T23:36:59.781594+0000","last_clean":"2023-09-23T23:36:59.781594+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:36:59.781594+0000","last_undegraded":"2023-09-23T23:36:59.781594+0000","last_fullsized":"2023-09-23T23:36:59.781594+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":123,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.777081+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:36:59.777081+0000","last_peered":"2023-09-23T23:36:59.777081+0000","last_clean":"2023-09-23T23:36:59.777081+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:36:59.777081+0000","last_undegraded":"2023-09-23T23:36:59.777081+0000","last_fullsized":"2023-09-23T23:36:59.777081+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":465,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.783443+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:36:59.783443+0000","last_peered":"2023-09-23T23:36:59.783443+0000","last_clean":"2023-09-23T23:36:59.783443+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:36:59.783443+0000","last_undegraded":"2023-09-23T23:36:59.783443+0000","last_fullsized":"2023-09-23T23:36:59.783443+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"599'8049","reported_seq":13136,"reported_epoch":599,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:37:14.129882+0000","last_change":"2023-09-23T23:34:35.785270+0000","last_active":"2023-09-23T23:37:14.129882+0000","last_peered":"2023-09-23T23:37:14.129882+0000","last_clean":"2023-09-23T23:33:50.504296+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:37:14.129882+0000","last_undegraded":"2023-09-23T23:34:35.780981+0000","last_fullsized":"2023-09-23T23:37:14.129882+0000","mapping_epoch":567,"log_start":"551'2600","ondisk_log_start":"551'2600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5449,"log_dups_size":0,"ondisk_log_size":5449,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200212480,"num_objects":3055,"num_object_clones":0,"num_object_copies":9165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2050,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8049,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2497,"num_bytes_recovered":163643392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":2050},{"shards":"5,6,7","objects":1005}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":607,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.775224+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:36:59.775224+0000","last_peered":"2023-09-23T23:36:59.775224+0000","last_clean":"2023-09-23T23:36:59.775224+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:36:59.775224+0000","last_undegraded":"2023-09-23T23:36:59.775224+0000","last_fullsized":"2023-09-23T23:36:59.775224+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":462,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.781819+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:36:59.781819+0000","last_peered":"2023-09-23T23:36:59.781819+0000","last_clean":"2023-09-23T23:36:59.781819+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:36:59.781819+0000","last_undegraded":"2023-09-23T23:36:59.781819+0000","last_fullsized":"2023-09-23T23:36:59.781819+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":550,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.779583+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:36:59.779583+0000","last_peered":"2023-09-23T23:36:59.779583+0000","last_clean":"2023-09-23T23:36:59.779583+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:36:59.779583+0000","last_undegraded":"2023-09-23T23:36:59.779583+0000","last_fullsized":"2023-09-23T23:36:59.779583+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"599'7952","reported_seq":16988,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:14.367453+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:37:14.367453+0000","last_peered":"2023-09-23T23:37:14.367453+0000","last_clean":"2023-09-23T23:37:14.367453+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:37:14.367453+0000","last_undegraded":"2023-09-23T23:37:14.367453+0000","last_fullsized":"2023-09-23T23:37:14.367453+0000","mapping_epoch":579,"log_start":"550'2200","ondisk_log_start":"550'2200","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5752,"log_dups_size":0,"ondisk_log_size":5752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200802304,"num_objects":3064,"num_object_clones":0,"num_object_copies":9192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7952,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":941,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.782743+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:36:59.782743+0000","last_peered":"2023-09-23T23:36:59.782743+0000","last_clean":"2023-09-23T23:36:59.782743+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:36:59.782743+0000","last_undegraded":"2023-09-23T23:36:59.782743+0000","last_fullsized":"2023-09-23T23:36:59.782743+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":241,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.791551+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:36:59.791551+0000","last_peered":"2023-09-23T23:36:59.791551+0000","last_clean":"2023-09-23T23:36:59.791551+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:36:59.791551+0000","last_undegraded":"2023-09-23T23:36:59.791551+0000","last_fullsized":"2023-09-23T23:36:59.791551+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":529,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.793901+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:36:59.793901+0000","last_peered":"2023-09-23T23:36:59.793901+0000","last_clean":"2023-09-23T23:36:59.793901+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:36:59.793901+0000","last_undegraded":"2023-09-23T23:36:59.793901+0000","last_fullsized":"2023-09-23T23:36:59.793901+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"599'7741","reported_seq":7817,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.053574+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:37:18.053574+0000","last_peered":"2023-09-23T23:37:18.053574+0000","last_clean":"2023-09-23T23:37:18.053574+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:37:18.053574+0000","last_undegraded":"2023-09-23T23:37:18.053574+0000","last_fullsized":"2023-09-23T23:37:18.053574+0000","mapping_epoch":534,"log_start":"551'2400","ondisk_log_start":"551'2400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5341,"log_dups_size":0,"ondisk_log_size":5341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195756032,"num_objects":2987,"num_object_clones":0,"num_object_copies":8961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7741,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":920,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.780382+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:36:59.780382+0000","last_peered":"2023-09-23T23:36:59.780382+0000","last_clean":"2023-09-23T23:36:59.780382+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:36:59.780382+0000","last_undegraded":"2023-09-23T23:36:59.780382+0000","last_fullsized":"2023-09-23T23:36:59.780382+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":309,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.303437+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:37:10.303437+0000","last_peered":"2023-09-23T23:37:10.303437+0000","last_clean":"2023-09-23T23:37:10.303437+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:37:10.303437+0000","last_undegraded":"2023-09-23T23:37:10.303437+0000","last_fullsized":"2023-09-23T23:37:10.303437+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":614,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.778424+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:36:59.778424+0000","last_peered":"2023-09-23T23:36:59.778424+0000","last_clean":"2023-09-23T23:36:59.778424+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:36:59.778424+0000","last_undegraded":"2023-09-23T23:36:59.778424+0000","last_fullsized":"2023-09-23T23:36:59.778424+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":195,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.774020+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:36:59.774020+0000","last_peered":"2023-09-23T23:36:59.774020+0000","last_clean":"2023-09-23T23:36:59.774020+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:36:59.774020+0000","last_undegraded":"2023-09-23T23:36:59.774020+0000","last_fullsized":"2023-09-23T23:36:59.774020+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"599'8159","reported_seq":13413,"reported_epoch":599,"state":"active+recovering","last_fresh":"2023-09-23T23:37:18.028194+0000","last_change":"2023-09-23T23:34:35.815566+0000","last_active":"2023-09-23T23:37:18.028194+0000","last_peered":"2023-09-23T23:37:18.028194+0000","last_clean":"2023-09-23T23:33:50.519308+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:37:18.028194+0000","last_undegraded":"2023-09-23T23:37:18.028194+0000","last_fullsized":"2023-09-23T23:37:18.028194+0000","mapping_epoch":566,"log_start":"551'2822","ondisk_log_start":"551'2822","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5337,"log_dups_size":0,"ondisk_log_size":5337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197132312,"num_objects":3009,"num_object_clones":0,"num_object_copies":9027,"num_objects_missing_on_primary":1931,"num_objects_missing":1931,"num_objects_degraded":0,"num_objects_misplaced":1931,"num_objects_unfound":0,"num_objects_dirty":3009,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8159,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2574,"num_bytes_recovered":168689664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,3,4,7","objects":10},{"shards":"1,3,7","objects":1068},{"shards":"1,4,7","objects":1931}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":683,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.826648+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:36:59.826648+0000","last_peered":"2023-09-23T23:36:59.826648+0000","last_clean":"2023-09-23T23:36:59.826648+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:36:59.826648+0000","last_undegraded":"2023-09-23T23:36:59.826648+0000","last_fullsized":"2023-09-23T23:36:59.826648+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":247,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.777039+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:36:59.777039+0000","last_peered":"2023-09-23T23:36:59.777039+0000","last_clean":"2023-09-23T23:36:59.777039+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:36:59.777039+0000","last_undegraded":"2023-09-23T23:36:59.777039+0000","last_fullsized":"2023-09-23T23:36:59.777039+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":531,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.777368+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:36:59.777368+0000","last_peered":"2023-09-23T23:36:59.777368+0000","last_clean":"2023-09-23T23:36:59.777368+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:36:59.777368+0000","last_undegraded":"2023-09-23T23:36:59.777368+0000","last_fullsized":"2023-09-23T23:36:59.777368+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":232,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.781060+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:36:59.781060+0000","last_peered":"2023-09-23T23:36:59.781060+0000","last_clean":"2023-09-23T23:36:59.781060+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:36:59.781060+0000","last_undegraded":"2023-09-23T23:36:59.781060+0000","last_fullsized":"2023-09-23T23:36:59.781060+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"599'7755","reported_seq":9807,"reported_epoch":599,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:14.237701+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:37:14.237701+0000","last_peered":"2023-09-23T23:37:14.237701+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:37:14.237701+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"550'2000","ondisk_log_start":"550'2000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5755,"log_dups_size":0,"ondisk_log_size":5755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197722112,"num_objects":3017,"num_object_clones":0,"num_object_copies":9051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7755,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":2006},{"shards":"5,7","objects":-375}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":901,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:37:09.941597+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:37:09.941597+0000","last_peered":"2023-09-23T23:37:09.941597+0000","last_clean":"2023-09-23T23:37:09.941597+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:37:09.941597+0000","last_undegraded":"2023-09-23T23:37:09.941597+0000","last_fullsized":"2023-09-23T23:37:09.941597+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":338,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.940052+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:36:59.940052+0000","last_peered":"2023-09-23T23:36:59.940052+0000","last_clean":"2023-09-23T23:36:59.940052+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:36:59.940052+0000","last_undegraded":"2023-09-23T23:36:59.940052+0000","last_fullsized":"2023-09-23T23:36:59.940052+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":247,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.781054+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:36:59.781054+0000","last_peered":"2023-09-23T23:36:59.781054+0000","last_clean":"2023-09-23T23:36:59.781054+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:36:59.781054+0000","last_undegraded":"2023-09-23T23:36:59.781054+0000","last_fullsized":"2023-09-23T23:36:59.781054+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":211,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.781819+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:36:59.781819+0000","last_peered":"2023-09-23T23:36:59.781819+0000","last_clean":"2023-09-23T23:36:59.781819+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:36:59.781819+0000","last_undegraded":"2023-09-23T23:36:59.781819+0000","last_fullsized":"2023-09-23T23:36:59.781819+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"599'7946","reported_seq":8029,"reported_epoch":599,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:37:14.311703+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:37:14.311703+0000","last_peered":"2023-09-23T23:37:14.311703+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:37:14.311703+0000","last_undegraded":"2023-09-23T23:37:14.311703+0000","last_fullsized":"2023-09-23T23:37:14.311703+0000","mapping_epoch":562,"log_start":"550'2200","ondisk_log_start":"550'2200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5746,"log_dups_size":0,"ondisk_log_size":5746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193724416,"num_objects":2956,"num_object_clones":0,"num_object_copies":8868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7946,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":2956}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":614,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.781381+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:36:59.781381+0000","last_peered":"2023-09-23T23:36:59.781381+0000","last_clean":"2023-09-23T23:36:59.781381+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:36:59.781381+0000","last_undegraded":"2023-09-23T23:36:59.781381+0000","last_fullsized":"2023-09-23T23:36:59.781381+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":885,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.776379+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:36:59.776379+0000","last_peered":"2023-09-23T23:36:59.776379+0000","last_clean":"2023-09-23T23:36:59.776379+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:36:59.776379+0000","last_undegraded":"2023-09-23T23:36:59.776379+0000","last_fullsized":"2023-09-23T23:36:59.776379+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":561,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.792775+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:36:59.792775+0000","last_peered":"2023-09-23T23:36:59.792775+0000","last_clean":"2023-09-23T23:36:59.792775+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:36:59.792775+0000","last_undegraded":"2023-09-23T23:36:59.792775+0000","last_fullsized":"2023-09-23T23:36:59.792775+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":123,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.779855+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:36:59.779855+0000","last_peered":"2023-09-23T23:36:59.779855+0000","last_clean":"2023-09-23T23:36:59.779855+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:36:59.779855+0000","last_undegraded":"2023-09-23T23:36:59.779855+0000","last_fullsized":"2023-09-23T23:36:59.779855+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"599'7726","reported_seq":7799,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:14.132014+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:37:14.132014+0000","last_peered":"2023-09-23T23:37:14.132014+0000","last_clean":"2023-09-23T23:37:14.132014+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:37:14.132014+0000","last_undegraded":"2023-09-23T23:37:14.132014+0000","last_fullsized":"2023-09-23T23:37:14.132014+0000","mapping_epoch":534,"log_start":"550'2300","ondisk_log_start":"550'2300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5426,"log_dups_size":0,"ondisk_log_size":5426,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197263360,"num_objects":3010,"num_object_clones":0,"num_object_copies":9030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7726,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":711,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.777279+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:36:59.777279+0000","last_peered":"2023-09-23T23:36:59.777279+0000","last_clean":"2023-09-23T23:36:59.777279+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:36:59.777279+0000","last_undegraded":"2023-09-23T23:36:59.777279+0000","last_fullsized":"2023-09-23T23:36:59.777279+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":526,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.773241+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:36:59.773241+0000","last_peered":"2023-09-23T23:36:59.773241+0000","last_clean":"2023-09-23T23:36:59.773241+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:36:59.773241+0000","last_undegraded":"2023-09-23T23:36:59.773241+0000","last_fullsized":"2023-09-23T23:36:59.773241+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":440,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.791816+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:36:59.791816+0000","last_peered":"2023-09-23T23:36:59.791816+0000","last_clean":"2023-09-23T23:36:59.791816+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:36:59.791816+0000","last_undegraded":"2023-09-23T23:36:59.791816+0000","last_fullsized":"2023-09-23T23:36:59.791816+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":40,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.825141+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:36:59.825141+0000","last_peered":"2023-09-23T23:36:59.825141+0000","last_clean":"2023-09-23T23:36:59.825141+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:36:59.825141+0000","last_undegraded":"2023-09-23T23:36:59.825141+0000","last_fullsized":"2023-09-23T23:36:59.825141+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"599'7973","reported_seq":11646,"reported_epoch":599,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:14.940678+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:37:14.940678+0000","last_peered":"2023-09-23T23:37:14.940678+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:37:14.940678+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"549'1800","ondisk_log_start":"549'1800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6173,"log_dups_size":0,"ondisk_log_size":6173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198246400,"num_objects":3025,"num_object_clones":0,"num_object_copies":9075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7973,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":321},{"shards":"1,2,4","objects":2023}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":594,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.779901+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:36:59.779901+0000","last_peered":"2023-09-23T23:36:59.779901+0000","last_clean":"2023-09-23T23:36:59.779901+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:36:59.779901+0000","last_undegraded":"2023-09-23T23:36:59.779901+0000","last_fullsized":"2023-09-23T23:36:59.779901+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":40,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.824793+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:36:59.824793+0000","last_peered":"2023-09-23T23:36:59.824793+0000","last_clean":"2023-09-23T23:36:59.824793+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:36:59.824793+0000","last_undegraded":"2023-09-23T23:36:59.824793+0000","last_fullsized":"2023-09-23T23:36:59.824793+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":139,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.783358+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:36:59.783358+0000","last_peered":"2023-09-23T23:36:59.783358+0000","last_clean":"2023-09-23T23:36:59.783358+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:36:59.783358+0000","last_undegraded":"2023-09-23T23:36:59.783358+0000","last_fullsized":"2023-09-23T23:36:59.783358+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":332,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.775430+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:36:59.775430+0000","last_peered":"2023-09-23T23:36:59.775430+0000","last_clean":"2023-09-23T23:36:59.775430+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:36:59.775430+0000","last_undegraded":"2023-09-23T23:36:59.775430+0000","last_fullsized":"2023-09-23T23:36:59.775430+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"599'7936","reported_seq":8009,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:14.349250+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:37:14.349250+0000","last_peered":"2023-09-23T23:37:14.349250+0000","last_clean":"2023-09-23T23:37:14.349250+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:37:14.349250+0000","last_undegraded":"2023-09-23T23:37:14.349250+0000","last_fullsized":"2023-09-23T23:37:14.349250+0000","mapping_epoch":534,"log_start":"550'2200","ondisk_log_start":"550'2200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5736,"log_dups_size":0,"ondisk_log_size":5736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198443008,"num_objects":3028,"num_object_clones":0,"num_object_copies":9084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7936,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":780,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.783535+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:36:59.783535+0000","last_peered":"2023-09-23T23:36:59.783535+0000","last_clean":"2023-09-23T23:36:59.783535+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:36:59.783535+0000","last_undegraded":"2023-09-23T23:36:59.783535+0000","last_fullsized":"2023-09-23T23:36:59.783535+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":597,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:37:02.990269+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:37:02.990269+0000","last_peered":"2023-09-23T23:37:02.990269+0000","last_clean":"2023-09-23T23:37:02.990269+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:37:02.990269+0000","last_undegraded":"2023-09-23T23:37:02.990269+0000","last_fullsized":"2023-09-23T23:37:02.990269+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":203,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.779694+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:36:59.779694+0000","last_peered":"2023-09-23T23:36:59.779694+0000","last_clean":"2023-09-23T23:36:59.779694+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:36:59.779694+0000","last_undegraded":"2023-09-23T23:36:59.779694+0000","last_fullsized":"2023-09-23T23:36:59.779694+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":151,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.781438+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:36:59.781438+0000","last_peered":"2023-09-23T23:36:59.781438+0000","last_clean":"2023-09-23T23:36:59.781438+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:36:59.781438+0000","last_undegraded":"2023-09-23T23:36:59.781438+0000","last_fullsized":"2023-09-23T23:36:59.781438+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"599'7843","reported_seq":7919,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:14.764756+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:37:14.764756+0000","last_peered":"2023-09-23T23:37:14.764756+0000","last_clean":"2023-09-23T23:37:14.764756+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:37:14.764756+0000","last_undegraded":"2023-09-23T23:37:14.764756+0000","last_fullsized":"2023-09-23T23:37:14.764756+0000","mapping_epoch":534,"log_start":"549'1700","ondisk_log_start":"549'1700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6143,"log_dups_size":0,"ondisk_log_size":6143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200867840,"num_objects":3065,"num_object_clones":0,"num_object_copies":9195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7843,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":986,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.793445+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:36:59.793445+0000","last_peered":"2023-09-23T23:36:59.793445+0000","last_clean":"2023-09-23T23:36:59.793445+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:36:59.793445+0000","last_undegraded":"2023-09-23T23:36:59.793445+0000","last_fullsized":"2023-09-23T23:36:59.793445+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":113,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.780496+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:36:59.780496+0000","last_peered":"2023-09-23T23:36:59.780496+0000","last_clean":"2023-09-23T23:36:59.780496+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:36:59.780496+0000","last_undegraded":"2023-09-23T23:36:59.780496+0000","last_fullsized":"2023-09-23T23:36:59.780496+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":44,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.825200+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:36:59.825200+0000","last_peered":"2023-09-23T23:36:59.825200+0000","last_clean":"2023-09-23T23:36:59.825200+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:36:59.825200+0000","last_undegraded":"2023-09-23T23:36:59.825200+0000","last_fullsized":"2023-09-23T23:36:59.825200+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":173,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.774713+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:36:59.774713+0000","last_peered":"2023-09-23T23:36:59.774713+0000","last_clean":"2023-09-23T23:36:59.774713+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:36:59.774713+0000","last_undegraded":"2023-09-23T23:36:59.774713+0000","last_fullsized":"2023-09-23T23:36:59.774713+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1035,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.791984+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:36:59.791984+0000","last_peered":"2023-09-23T23:36:59.791984+0000","last_clean":"2023-09-23T23:36:59.791984+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:36:59.791984+0000","last_undegraded":"2023-09-23T23:36:59.791984+0000","last_fullsized":"2023-09-23T23:36:59.791984+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":40,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:37:05.963403+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:37:05.963403+0000","last_peered":"2023-09-23T23:37:05.963403+0000","last_clean":"2023-09-23T23:37:05.963403+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:37:05.963403+0000","last_undegraded":"2023-09-23T23:37:05.963403+0000","last_fullsized":"2023-09-23T23:37:05.963403+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":354,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.783433+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:36:59.783433+0000","last_peered":"2023-09-23T23:36:59.783433+0000","last_clean":"2023-09-23T23:36:59.783433+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:36:59.783433+0000","last_undegraded":"2023-09-23T23:36:59.783433+0000","last_fullsized":"2023-09-23T23:36:59.783433+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":747,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.794293+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:36:59.794293+0000","last_peered":"2023-09-23T23:36:59.794293+0000","last_clean":"2023-09-23T23:36:59.794293+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:36:59.794293+0000","last_undegraded":"2023-09-23T23:36:59.794293+0000","last_fullsized":"2023-09-23T23:36:59.794293+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":101,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.777318+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:36:59.777318+0000","last_peered":"2023-09-23T23:36:59.777318+0000","last_clean":"2023-09-23T23:36:59.777318+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:36:59.777318+0000","last_undegraded":"2023-09-23T23:36:59.777318+0000","last_fullsized":"2023-09-23T23:36:59.777318+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":447,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.794623+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:36:59.794623+0000","last_peered":"2023-09-23T23:36:59.794623+0000","last_clean":"2023-09-23T23:36:59.794623+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:36:59.794623+0000","last_undegraded":"2023-09-23T23:36:59.794623+0000","last_fullsized":"2023-09-23T23:36:59.794623+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":297,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.779711+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:36:59.779711+0000","last_peered":"2023-09-23T23:36:59.779711+0000","last_clean":"2023-09-23T23:36:59.779711+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:36:59.779711+0000","last_undegraded":"2023-09-23T23:36:59.779711+0000","last_fullsized":"2023-09-23T23:36:59.779711+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":286,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.792855+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:36:59.792855+0000","last_peered":"2023-09-23T23:36:59.792855+0000","last_clean":"2023-09-23T23:36:59.792855+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:36:59.792855+0000","last_undegraded":"2023-09-23T23:36:59.792855+0000","last_fullsized":"2023-09-23T23:36:59.792855+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":811,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.779280+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:36:59.779280+0000","last_peered":"2023-09-23T23:36:59.779280+0000","last_clean":"2023-09-23T23:36:59.779280+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:36:59.779280+0000","last_undegraded":"2023-09-23T23:36:59.779280+0000","last_fullsized":"2023-09-23T23:36:59.779280+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":630,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:37:02.990226+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:37:02.990226+0000","last_peered":"2023-09-23T23:37:02.990226+0000","last_clean":"2023-09-23T23:37:02.990226+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:37:02.990226+0000","last_undegraded":"2023-09-23T23:37:02.990226+0000","last_fullsized":"2023-09-23T23:37:02.990226+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":613,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.792261+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:36:59.792261+0000","last_peered":"2023-09-23T23:36:59.792261+0000","last_clean":"2023-09-23T23:36:59.792261+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:36:59.792261+0000","last_undegraded":"2023-09-23T23:36:59.792261+0000","last_fullsized":"2023-09-23T23:36:59.792261+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":772,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.802989+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:36:59.802989+0000","last_peered":"2023-09-23T23:36:59.802989+0000","last_clean":"2023-09-23T23:36:59.802989+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:36:59.802989+0000","last_undegraded":"2023-09-23T23:36:59.802989+0000","last_fullsized":"2023-09-23T23:36:59.802989+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":443,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.783815+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:36:59.783815+0000","last_peered":"2023-09-23T23:36:59.783815+0000","last_clean":"2023-09-23T23:36:59.783815+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:36:59.783815+0000","last_undegraded":"2023-09-23T23:36:59.783815+0000","last_fullsized":"2023-09-23T23:36:59.783815+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":666,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T23:36:59.776191+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:36:59.776191+0000","last_peered":"2023-09-23T23:36:59.776191+0000","last_clean":"2023-09-23T23:36:59.776191+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:36:59.776191+0000","last_undegraded":"2023-09-23T23:36:59.776191+0000","last_fullsized":"2023-09-23T23:36:59.776191+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":3181772824,"num_objects":48551,"num_object_clones":0,"num_object_copies":145653,"num_objects_missing_on_primary":1931,"num_objects_missing":1931,"num_objects_degraded":23747,"num_objects_misplaced":2826,"num_objects_unfound":0,"num_objects_dirty":48551,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":127005,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16969,"num_bytes_recovered":1111883776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10883837952,"data_stored":10883825736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90083,"ondisk_log_size":90083,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782693,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1572964,"kb_used_data":1186780,"kb_used_omap":8,"kb_used_meta":386167,"kb_avail":92172188,"statfs":{"total":95995035648,"available":94384320512,"internally_reserved":0,"allocated":1215262720,"data_stored":1213592881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":395435077},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":1.048,"15min":2.1080000000000001},"min":{"1min":0.25900000000000001,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":0.51300000000000001,"5min":17.140000000000001,"15min":85.721000000000004},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.47199999999999998,"5min":1.0600000000000001,"15min":2.1829999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":2.331,"5min":11.069000000000001,"15min":85.965000000000003},"last":0.34000000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":2.532,"15min":3.1309999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":1.5780000000000001,"5min":129.11199999999999,"15min":198.58000000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.53200000000000003,"5min":2.3039999999999998,"15min":3.1440000000000001},"min":{"1min":0.31900000000000001,"5min":0.247,"15min":0.246},"max":{"1min":2.3690000000000002,"5min":129.02600000000001,"15min":198.71100000000001},"last":0.42099999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":3.3980000000000001,"5min":2.6880000000000002,"15min":3.8050000000000002},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":53.054000000000002,"5min":78.528999999999996,"15min":203.58000000000001},"last":0.26700000000000002},{"interface":"front","average":{"1min":3.5699999999999998,"5min":1.9299999999999999,"15min":3.2229999999999999},"min":{"1min":0.33400000000000002,"5min":0.27900000000000003,"15min":0.26200000000000001},"max":{"1min":53.023000000000003,"5min":53.023000000000003,"15min":198.69},"last":0.30399999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":8.6839999999999993,"5min":4.4390000000000001,"15min":4.0750000000000002},"min":{"1min":0.33300000000000002,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":100.40600000000001,"5min":125.511,"15min":279.26499999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":8.7420000000000009,"5min":4.8460000000000001,"15min":4.6180000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29599999999999999},"max":{"1min":100.42400000000001,"5min":125.48999999999999,"15min":279.29899999999998},"last":0.46500000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":1.546,"15min":1.0389999999999999},"min":{"1min":0.28999999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":0.58099999999999996,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":1.341,"15min":0.97599999999999998},"min":{"1min":0.32500000000000001,"5min":0.317,"15min":0.23899999999999999},"max":{"1min":2.3980000000000001,"5min":31.68,"15min":31.68},"last":0.53600000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.6220000000000001,"15min":1.863},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.26500000000000001},"max":{"1min":2.4510000000000001,"5min":60.463999999999999,"15min":210.83600000000001},"last":0.504},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.5840000000000001,"15min":1.742},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.5940000000000001,"5min":63.006,"15min":207.04599999999999},"last":0.438}]},{"osd":6,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.827,"15min":1.3320000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.29899999999999999},"max":{"1min":1.619,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":2.052,"15min":1.544},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.389,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.54600000000000004}]}]},{"osd":6,"up_from":46,"seq":197568496170,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1830612,"kb_used_data":1119436,"kb_used_omap":0,"kb_used_meta":711168,"kb_avail":91914540,"statfs":{"total":95995035648,"available":94120488960,"internally_reserved":0,"allocated":1146302464,"data_stored":1144648618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":728236032},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.93500000000000005,"15min":4.6799999999999997},"min":{"1min":0.27700000000000002,"5min":0.251,"15min":0.20399999999999999},"max":{"1min":2.8450000000000002,"5min":8.8119999999999994,"15min":135.31999999999999},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.73399999999999999,"15min":4.601},"min":{"1min":0.24399999999999999,"5min":0.23999999999999999,"15min":0.22900000000000001},"max":{"1min":1.8859999999999999,"5min":3.8239999999999998,"15min":135.34299999999999},"last":0.29799999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":1.7509999999999999,"15min":2.3759999999999999},"min":{"1min":0.253,"5min":0.251,"15min":0.245},"max":{"1min":1.944,"5min":65.891000000000005,"15min":89.605000000000004},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.64100000000000001,"5min":1.4390000000000001,"15min":2.4510000000000001},"min":{"1min":0.33400000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":2.1269999999999998,"5min":66.087000000000003,"15min":91.025999999999996},"last":0.34699999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:37:06 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":3.6930000000000001,"15min":4.4749999999999996},"min":{"1min":0.27900000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.1800000000000002,"5min":67.796999999999997,"15min":127.44499999999999},"last":0.5},{"interface":"front","average":{"1min":0.66600000000000004,"5min":3.6749999999999998,"15min":4.8680000000000003},"min":{"1min":0.316,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.2080000000000002,"5min":67.893000000000001,"15min":125.539},"last":0.48999999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:36:55 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":5.6589999999999998,"15min":4.3159999999999998},"min":{"1min":0.30299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.3610000000000002,"5min":87.283000000000001,"15min":138.87299999999999},"last":0.318},{"interface":"front","average":{"1min":0.47699999999999998,"5min":5.516,"15min":4.625},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0349999999999999,"5min":87.491,"15min":138.976},"last":0.57299999999999995}]},{"osd":4,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.76400000000000001,"15min":0.68300000000000005},"min":{"1min":0.27900000000000003,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":0.57799999999999996,"5min":7.101,"15min":8.1999999999999993},"last":0.38},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.88900000000000001,"15min":0.77600000000000002},"min":{"1min":0.32500000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.54600000000000004,"5min":8.3499999999999996,"15min":8.3499999999999996},"last":0.55100000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.82899999999999996,"15min":0.80700000000000005},"min":{"1min":0.34300000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.266,"5min":11.372,"15min":11.473000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.98999999999999999,"15min":0.83599999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26900000000000002},"max":{"1min":0.57699999999999996,"5min":7.0129999999999999,"15min":7.5670000000000002},"last":0.46899999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:37:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.1679999999999999,"15min":1.0029999999999999},"min":{"1min":0.32700000000000001,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":0.77000000000000002,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.204,"15min":1.056},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33500000000000002},"max":{"1min":1.792,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.52700000000000002}]}]},{"osd":1,"up_from":112,"seq":481036337511,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1890200,"kb_used_data":981264,"kb_used_omap":4,"kb_used_meta":908923,"kb_avail":91854952,"statfs":{"total":95995035648,"available":94059470848,"internally_reserved":0,"allocated":1004814336,"data_stored":1001988706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":930738062},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":1.881,"15min":3.4060000000000001},"min":{"1min":0.217,"5min":0.17799999999999999,"15min":0.16700000000000001},"max":{"1min":2.7970000000000002,"5min":47.195999999999998,"15min":130.768},"last":0.317},{"interface":"front","average":{"1min":0.40999999999999998,"5min":1.5469999999999999,"15min":2.726},"min":{"1min":0.24299999999999999,"5min":0.20499999999999999,"15min":0.19},"max":{"1min":1.016,"5min":46.582000000000001,"15min":134.929},"last":0.40300000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":3.5659999999999998,"15min":6.7649999999999997},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":1.401,"5min":201.03999999999999,"15min":231.67400000000001},"last":1.302},{"interface":"front","average":{"1min":0.66900000000000004,"5min":3.8849999999999998,"15min":6.8339999999999996},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.23,"5min":202.12,"15min":232.21000000000001},"last":1.8089999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":6.883,"5min":5.8460000000000001,"15min":5.4379999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22600000000000001},"max":{"1min":71.745000000000005,"5min":181.94399999999999,"15min":234.404},"last":0.504},{"interface":"front","average":{"1min":6.8639999999999999,"5min":5.8860000000000001,"15min":5.3899999999999997},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":71.786000000000001,"5min":181.98500000000001,"15min":236.33699999999999},"last":3.8319999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":1.23,"15min":1.423},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":2.1059999999999999,"5min":12.042999999999999,"15min":12.042999999999999},"last":1.9159999999999999},{"interface":"front","average":{"1min":0.624,"5min":1.129,"15min":1.46},"min":{"1min":0.48099999999999998,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":1.2669999999999999,"5min":9.5069999999999997,"15min":18.108000000000001},"last":0.371}]},{"osd":5,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":2.9209999999999998,"15min":1.796},"min":{"1min":0.34300000000000003,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66600000000000004,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.68200000000000005,"5min":2.9940000000000002,"15min":2.113},"min":{"1min":0.40899999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.044,"5min":151.821,"15min":151.821},"last":1.9390000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.3600000000000001,"15min":1.7529999999999999},"min":{"1min":0.377,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":1.1040000000000001,"5min":8.9269999999999996,"15min":74.087999999999994},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.97699999999999998,"5min":1.585,"15min":1.968},"min":{"1min":0.42999999999999999,"5min":0.35699999999999998,"15min":0.33100000000000002},"max":{"1min":5.367,"5min":11.927,"15min":73.974999999999994},"last":1.9610000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":1.5449999999999999,"15min":2.359},"min":{"1min":0.40200000000000002,"5min":0.38700000000000001,"15min":0.32900000000000001},"max":{"1min":2.504,"5min":44.241,"15min":78.308000000000007},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.556,"15min":2.2730000000000001},"min":{"1min":0.41299999999999998,"5min":0.37,"15min":0.32700000000000001},"max":{"1min":3.258,"5min":43.920999999999999,"15min":77.204999999999998},"last":1.8879999999999999}]}]},{"osd":0,"up_from":91,"seq":390842024335,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1467804,"kb_used_data":265748,"kb_used_omap":5,"kb_used_meta":1202042,"kb_avail":92277348,"statfs":{"total":95995035648,"available":94492004352,"internally_reserved":0,"allocated":272125952,"data_stored":269290349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1230891642},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:36:36 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":4.391,"15min":6.6539999999999999},"min":{"1min":0.219,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":1.7709999999999999,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.43099999999999999,"5min":4.1289999999999996,"15min":5.7969999999999997},"min":{"1min":0.245,"5min":0.20200000000000001,"15min":0.17599999999999999},"max":{"1min":1.0249999999999999,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.254}]},{"osd":2,"last update":"Sat Sep 23 23:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":6.9610000000000003,"15min":7.3559999999999999},"min":{"1min":0.26200000000000001,"5min":0.214,"15min":0.17199999999999999},"max":{"1min":4.0899999999999999,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.84599999999999997,"5min":6.6799999999999997,"15min":7.3869999999999996},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.17599999999999999},"max":{"1min":3.468,"5min":283.161,"15min":283.161},"last":0.28799999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:36:36 2023","interfaces":[{"interface":"back","average":{"1min":5.9699999999999998,"5min":5.1630000000000003,"15min":5.6079999999999997},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":95.655000000000001,"5min":114.381,"15min":220.732},"last":0.42599999999999999},{"interface":"front","average":{"1min":5.8300000000000001,"5min":5.2779999999999996,"15min":5.6669999999999998},"min":{"1min":0.26600000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":95.695999999999998,"5min":114.30500000000001,"15min":218.78200000000001},"last":0.317}]},{"osd":4,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.79900000000000004,"15min":2.3479999999999999},"min":{"1min":0.34399999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.60999999999999999,"5min":9.3140000000000001,"15min":138.81299999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.66800000000000004,"5min":1.02,"15min":3.0270000000000001},"min":{"1min":0.36799999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.661,"5min":9.3670000000000009,"15min":171.251},"last":0.89000000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.1850000000000001,"15min":2.7290000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":1.288,"5min":15.076000000000001,"15min":171.30699999999999},"last":1.288},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.097,"15min":1.9930000000000001},"min":{"1min":0.33600000000000002,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":4.1710000000000003,"5min":6.5279999999999996,"15min":139.40799999999999},"last":0.45300000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":1.143,"15min":3.0760000000000001},"min":{"1min":0.374,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":12.654999999999999,"5min":12.654999999999999,"15min":171.33600000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.155,"15min":3.2749999999999999},"min":{"1min":0.41199999999999998,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":4.6440000000000001,"5min":9.157,"15min":184.13499999999999},"last":0.51400000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":1.089,"15min":2.569},"min":{"1min":0.35099999999999998,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":2.6749999999999998,"5min":8.9510000000000005,"15min":138.77699999999999},"last":0.373},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.3260000000000001,"15min":2.508},"min":{"1min":0.373,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":4.2229999999999999,"5min":9.8870000000000005,"15min":139.37899999999999},"last":0.48399999999999999}]}]},{"osd":2,"up_from":199,"seq":854698492156,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2795612,"kb_used_data":1951700,"kb_used_omap":7,"kb_used_meta":843896,"kb_avail":90949540,"statfs":{"total":95995035648,"available":93132328960,"internally_reserved":0,"allocated":1998540800,"data_stored":1995655245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":864149585},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.7789999999999999,"15min":3.617},"min":{"1min":0.20200000000000001,"5min":0.186,"15min":0.16700000000000001},"max":{"1min":6.3940000000000001,"5min":61.950000000000003,"15min":119.277},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":1.8740000000000001,"15min":3.46},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.18099999999999999},"max":{"1min":5.0339999999999998,"5min":61.984999999999999,"15min":119.56100000000001},"last":0.24099999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":4.4960000000000004,"15min":4.5449999999999999},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":4.7060000000000004,"5min":106.765,"15min":157.98699999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.73899999999999999,"5min":4.1210000000000004,"15min":4.4950000000000001},"min":{"1min":0.23400000000000001,"5min":0.222,"15min":0.222},"max":{"1min":6.4340000000000002,"5min":106.732,"15min":158.03800000000001},"last":0.44700000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":5.3179999999999996,"15min":4.6900000000000004},"min":{"1min":0.26000000000000001,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":1.075,"5min":106.70399999999999,"15min":119.366},"last":0.309},{"interface":"front","average":{"1min":0.52100000000000002,"5min":5.0359999999999996,"15min":4.1440000000000001},"min":{"1min":0.246,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.6120000000000001,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.40999999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":1.756,"15min":2.1909999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.25},"max":{"1min":0.65600000000000003,"5min":62.081000000000003,"15min":119.88200000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.9710000000000001,"15min":1.829},"min":{"1min":0.42299999999999999,"5min":0.35599999999999998,"15min":0.28499999999999998},"max":{"1min":1.121,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.437}]},{"osd":5,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":2.8340000000000001,"15min":2.5840000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":2.6549999999999998,"5min":91.207999999999998,"15min":119.919},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.69399999999999995,"5min":2.919,"15min":2.5800000000000001},"min":{"1min":0.371,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.4289999999999998,"5min":91.150999999999996,"15min":119.306},"last":0.38}]},{"osd":6,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":3.254,"15min":2.9169999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":1.04,"5min":107.018,"15min":119.943},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.746,"5min":2.1779999999999999,"15min":2.1160000000000001},"min":{"1min":0.45100000000000001,"5min":0.371,"15min":0.34999999999999998},"max":{"1min":3.4950000000000001,"5min":62.493000000000002,"15min":62.493000000000002},"last":0.46200000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.917,"15min":1.883},"min":{"1min":0.41699999999999998,"5min":0.374,"15min":0.34000000000000002},"max":{"1min":1.831,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.496},{"interface":"front","average":{"1min":0.66800000000000004,"5min":3.1800000000000002,"15min":2.3220000000000001},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.36199999999999999},"max":{"1min":1.792,"5min":106.809,"15min":106.809},"last":0.51200000000000001}]}]},{"osd":3,"up_from":148,"seq":635655160125,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1971924,"kb_used_data":1038284,"kb_used_omap":2,"kb_used_meta":933629,"kb_avail":91773228,"statfs":{"total":95995035648,"available":93975785472,"internally_reserved":0,"allocated":1063202816,"data_stored":1060378902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":956036149},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.1359999999999999,"15min":2.6019999999999999},"min":{"1min":0.184,"5min":0.184,"15min":0.17699999999999999},"max":{"1min":6.7699999999999996,"5min":17.213000000000001,"15min":103.866},"last":0.20799999999999999},{"interface":"front","average":{"1min":0.72899999999999998,"5min":1.4139999999999999,"15min":2.855},"min":{"1min":0.218,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":3.4980000000000002,"5min":22.754000000000001,"15min":101.884},"last":0.247}]},{"osd":1,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":2.5960000000000001,"15min":2.903},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":1.649,"5min":56.119,"15min":85.242999999999995},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.42299999999999999,"5min":2.036,"15min":2.7469999999999999},"min":{"1min":0.254,"5min":0.224,"15min":0.20899999999999999},"max":{"1min":1.6990000000000001,"5min":53.962000000000003,"15min":85.302000000000007},"last":0.26500000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:36:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":2.2989999999999999,"15min":4.5999999999999996},"min":{"1min":0.25800000000000001,"5min":0.25,"15min":0.19800000000000001},"max":{"1min":14.965,"5min":56.226999999999997,"15min":101.279},"last":36.362000000000002},{"interface":"front","average":{"1min":1.2210000000000001,"5min":2.165,"15min":4.4370000000000003},"min":{"1min":0.315,"5min":0.29299999999999998,"15min":0.214},"max":{"1min":15.058999999999999,"5min":54.280999999999999,"15min":101.294},"last":36.158999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":0.68999999999999995,"5min":11.209,"15min":11.209},"last":0.372},{"interface":"front","average":{"1min":0.44500000000000001,"5min":1.3720000000000001,"15min":1.6719999999999999},"min":{"1min":0.33000000000000002,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":0.67900000000000005,"5min":11.494999999999999,"15min":24.782},"last":0.40899999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":6.3570000000000002,"15min":3.3220000000000001},"min":{"1min":0.33000000000000002,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":0.622,"5min":214.43700000000001,"15min":214.43700000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":6.3970000000000002,"15min":3.3969999999999998},"min":{"1min":0.36499999999999999,"5min":0.34699999999999998,"15min":0.30599999999999999},"max":{"1min":1.716,"5min":214.46600000000001,"15min":214.46600000000001},"last":0.39600000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":2.218,"15min":2.1739999999999999},"min":{"1min":0.35399999999999998,"5min":0.318,"15min":0.318},"max":{"1min":2.008,"5min":58.932000000000002,"15min":58.932000000000002},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.629,"5min":2.492,"15min":2.2210000000000001},"min":{"1min":0.36699999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.7450000000000001,"5min":59.186,"15min":59.186},"last":0.47499999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:36:56 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":4.5469999999999997,"15min":3.1829999999999998},"min":{"1min":0.42199999999999999,"5min":0.38700000000000001,"15min":0.32600000000000001},"max":{"1min":0.66700000000000004,"5min":192.69,"15min":192.69},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.54300000000000004,"5min":4.6280000000000001,"15min":3.649},"min":{"1min":0.40100000000000002,"5min":0.39200000000000002,"15min":0.36699999999999999},"max":{"1min":1.2769999999999999,"5min":192.65600000000001,"15min":192.65600000000001},"last":0.49099999999999999}]}]},{"osd":4,"up_from":0,"seq":128849019443,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757806,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2568912,"kb_used_data":1832776,"kb_used_omap":0,"kb_used_meta":736064,"kb_avail":91176240,"statfs":{"total":95995035648,"available":93364469760,"internally_reserved":0,"allocated":1876762624,"data_stored":1875108995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":753729536},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:14 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":1.141,"15min":3.198},"min":{"1min":0.252,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":1.5640000000000001,"5min":21.109000000000002,"15min":113.869},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.38,"5min":1.3939999999999999,"15min":3.5710000000000002},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.22700000000000001},"max":{"1min":0.76200000000000001,"5min":20.876999999999999,"15min":138.55199999999999},"last":0.28899999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:36:20 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.6200000000000001,"15min":2.258},"min":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.23100000000000001},"max":{"1min":1.9930000000000001,"5min":29.007999999999999,"15min":69.816999999999993},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.8,"15min":2.5840000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":2.8130000000000002,"5min":29.053999999999998,"15min":70.268000000000001},"last":0.30499999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:36:48 2023","interfaces":[{"interface":"back","average":{"1min":5.194,"5min":6.984,"15min":4.8159999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.26900000000000002},"max":{"1min":60.441000000000003,"5min":208.738,"15min":208.738},"last":3.2930000000000001},{"interface":"front","average":{"1min":5.2839999999999998,"5min":5.7190000000000003,"15min":4.3259999999999996},"min":{"1min":0.29999999999999999,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":60.520000000000003,"5min":208.631,"15min":208.631},"last":0.47699999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:36:48 2023","interfaces":[{"interface":"back","average":{"1min":4.1749999999999998,"5min":4.6360000000000001,"15min":3.8100000000000001},"min":{"1min":0.34999999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":37.878,"5min":122.071,"15min":122.071},"last":0.41799999999999998},{"interface":"front","average":{"1min":4.2460000000000004,"5min":4.9829999999999997,"15min":3.984},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.318},"max":{"1min":37.853000000000002,"5min":122.09699999999999,"15min":122.09699999999999},"last":0.46000000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.98699999999999999,"15min":0.90700000000000003},"min":{"1min":0.32900000000000001,"5min":0.25700000000000001,"15min":0.251},"max":{"1min":0.749,"5min":9.2690000000000001,"15min":19.962},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.502,"5min":1.0129999999999999,"15min":0.81599999999999995},"min":{"1min":0.314,"5min":0.29599999999999999,"15min":0.22900000000000001},"max":{"1min":1.595,"5min":10.273999999999999,"15min":20.829999999999998},"last":0.44700000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":2.29,"15min":1.5049999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.26900000000000002},"max":{"1min":1.619,"5min":148.36600000000001,"15min":148.36600000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.54000000000000004,"5min":2.798,"15min":1.7250000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.26000000000000001},"max":{"1min":2.1549999999999998,"5min":148.42500000000001,"15min":148.42500000000001},"last":0.37}]},{"osd":7,"last update":"Sat Sep 23 23:36:48 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.2629999999999999,"15min":2.3879999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":2.29,"5min":15.497,"15min":179.28800000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.64900000000000002,"5min":1.2729999999999999,"15min":2.46},"min":{"1min":0.36699999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.0630000000000002,"5min":15.653,"15min":179.428},"last":0.48399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":263192576,"data_stored":263192576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":992481280,"data_stored":992477208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1988362240,"data_stored":1988362240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1054806016,"data_stored":1054801944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2379022336,"data_stored":2379022336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1862074368,"data_stored":1862074368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1136525312,"data_stored":1136525312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1207373824,"data_stored":1207369752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:37:21.067 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:37:21.115 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:37:21.155 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:21.245 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:37:21.273 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:21 smithi110 ceph-mon[221102]: from='client.56869 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:37:21.273 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:21 smithi110 ceph-mon[221102]: pgmap v1684: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 288 op/s; 23640/143076 objects degraded (16.523%); 2826/143076 objects misplaced (1.975%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T23:37:21.278 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:37:21.345 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:37:21.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:21.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:37:21.453 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:21.507 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:21.644 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:21 smithi110 ceph-mon[212939]: from='client.56869 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:37:21.644 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:21 smithi110 ceph-mon[212939]: pgmap v1684: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 288 op/s; 23640/143076 objects degraded (16.523%); 2826/143076 objects misplaced (1.975%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T23:37:21.645 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:21.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:21.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:21 smithi138 ceph-mon[193606]: from='client.56869 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:37:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:21 smithi138 ceph-mon[193606]: pgmap v1684: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 288 op/s; 23640/143076 objects degraded (16.523%); 2826/143076 objects misplaced (1.975%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T23:37:21.711 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:37:21.756 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:21.788 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:21.802 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:21.866 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:21.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:21.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:21.952 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:37:21.973 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:37:21.978 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:37:21.993 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:22.007 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:37:22.030 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:22.034 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:37:22.066 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:37:22.085 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:37:22.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:22.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:37:22.096 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:37:22.131 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:37:22.158 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:37:22.159 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:37:22.160 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:37:22.160 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:37:22.160 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:37:22.160 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:37:22.160 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:37:22.160 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:37:22.160 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:37:22.190 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:22.224 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:22.362 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:22 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4276095130' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:22.362 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:22 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4276095130' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:22.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:22 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4276095130' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:22.729 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:22.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:22.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:22.832 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:22.859 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:23.155 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:23.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:23.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:23.263 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:23.300 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:23.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:23 smithi138 ceph-mon[193606]: pgmap v1685: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 23640/142194 objects degraded (16.625%); 2826/142194 objects misplaced (1.987%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T23:37:23.467 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:23.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:23.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:23.575 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:23.602 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:23 smithi110 ceph-mon[212939]: pgmap v1685: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 23640/142194 objects degraded (16.625%); 2826/142194 objects misplaced (1.987%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T23:37:23.603 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:23 smithi110 ceph-mon[221102]: pgmap v1685: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 23640/142194 objects degraded (16.625%); 2826/142194 objects misplaced (1.987%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T23:37:23.606 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:23.731 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:37:23.731 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:37:23.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:37:23.732 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:37:23.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.732 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:73408 12.10 12:0ed56545:::benchmark_data_smithi110_491193_object73407:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T23:37:23.732 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:29:44.726507+0000", 2023-09-23T23:37:23.732 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 459.00099642499998, 2023-09-23T23:37:23.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95402578900000001, 2023-09-23T23:37:23.732 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.732 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:37:23.732 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:23.732 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:23.733 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:23.733 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 73408 2023-09-23T23:37:23.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.733 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726507+0000", 2023-09-23T23:37:23.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726507+0000", 2023-09-23T23:37:23.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726505+0000", 2023-09-23T23:37:23.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:37:23.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726548+0000", 2023-09-23T23:37:23.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2936999999999999e-05 2023-09-23T23:37:23.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.735 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.735 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726565+0000", 2023-09-23T23:37:23.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6705e-05 2023-09-23T23:37:23.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.735 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.735 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.726572+0000", 2023-09-23T23:37:23.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9500000000000004e-06 2023-09-23T23:37:23.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.735 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.751276+0000", 2023-09-23T23:37:23.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.024704454000000001 2023-09-23T23:37:23.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:23.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:44.751287+0000", 2023-09-23T23:37:23.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0243e-05 2023-09-23T23:37:23.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.661352+0000", 2023-09-23T23:37:23.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91006557200000004 2023-09-23T23:37:23.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-09-23T23:37:23.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.661964+0000", 2023-09-23T23:37:23.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00061125199999999995 2023-09-23T23:37:23.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.680469+0000", 2023-09-23T23:37:23.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018505864 2023-09-23T23:37:23.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:29:45.680533+0000", 2023-09-23T23:37:23.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3732999999999994e-05 2023-09-23T23:37:23.738 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.738 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.738 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.738 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.738 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.738 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21483 13.5 13:a8fdedd1:::benchmark_data_smithi110_541928_object21482:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:37:23.738 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.045464+0000", 2023-09-23T23:37:23.738 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 253.68204013100001, 2023-09-23T23:37:23.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.135403025, 2023-09-23T23:37:23.739 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.739 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:23.739 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:23.739 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:23.739 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:23.739 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21483 2023-09-23T23:37:23.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.739 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.739 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.739 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045464+0000", 2023-09-23T23:37:23.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045464+0000", 2023-09-23T23:37:23.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045462+0000", 2023-09-23T23:37:23.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:37:23.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045507+0000", 2023-09-23T23:37:23.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4555999999999998e-05 2023-09-23T23:37:23.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045524+0000", 2023-09-23T23:37:23.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7450999999999999e-05 2023-09-23T23:37:23.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045531+0000", 2023-09-23T23:37:23.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7329999999999997e-06 2023-09-23T23:37:23.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045729+0000", 2023-09-23T23:37:23.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000197634 2023-09-23T23:37:23.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:23.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045739+0000", 2023-09-23T23:37:23.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.825e-06 2023-09-23T23:37:23.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.177900+0000", 2023-09-23T23:37:23.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.132160896 2023-09-23T23:37:23.743 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:23.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.178003+0000", 2023-09-23T23:37:23.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103212 2023-09-23T23:37:23.743 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:37:23.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.178112+0000", 2023-09-23T23:37:23.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109002 2023-09-23T23:37:23.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.744 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:23.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.179965+0000", 2023-09-23T23:37:23.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001853234 2023-09-23T23:37:23.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.744 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180695+0000", 2023-09-23T23:37:23.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000729913 2023-09-23T23:37:23.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180800+0000", 2023-09-23T23:37:23.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000104758 2023-09-23T23:37:23.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:23.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180834+0000", 2023-09-23T23:37:23.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4038000000000002e-05 2023-09-23T23:37:23.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180867+0000", 2023-09-23T23:37:23.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2935000000000002e-05 2023-09-23T23:37:23.746 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.746 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.746 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.746 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.746 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.746 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21498 13.5 13:ac1dd01c:::benchmark_data_smithi110_541928_object21497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:37:23.746 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.050241+0000", 2023-09-23T23:37:23.746 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 253.677262329, 2023-09-23T23:37:23.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.130775804, 2023-09-23T23:37:23.746 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.746 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:23.747 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:23.747 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:23.747 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:23.747 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21498 2023-09-23T23:37:23.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.747 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.747 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.747 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050241+0000", 2023-09-23T23:37:23.747 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050241+0000", 2023-09-23T23:37:23.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050240+0000", 2023-09-23T23:37:23.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:23.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050290+0000", 2023-09-23T23:37:23.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0144e-05 2023-09-23T23:37:23.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.749 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050304+0000", 2023-09-23T23:37:23.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3708e-05 2023-09-23T23:37:23.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.749 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050310+0000", 2023-09-23T23:37:23.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9290000000000003e-06 2023-09-23T23:37:23.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050369+0000", 2023-09-23T23:37:23.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9196e-05 2023-09-23T23:37:23.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:23.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050379+0000", 2023-09-23T23:37:23.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9639999999999998e-06 2023-09-23T23:37:23.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.179168+0000", 2023-09-23T23:37:23.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.128789193 2023-09-23T23:37:23.751 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:23.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.179266+0000", 2023-09-23T23:37:23.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7250000000000006e-05 2023-09-23T23:37:23.751 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:37:23.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.179375+0000", 2023-09-23T23:37:23.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109688 2023-09-23T23:37:23.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.752 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180921+0000", 2023-09-23T23:37:23.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015460490000000001 2023-09-23T23:37:23.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.752 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:23.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180927+0000", 2023-09-23T23:37:23.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9109999999999998e-06 2023-09-23T23:37:23.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180956+0000", 2023-09-23T23:37:23.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8926000000000001e-05 2023-09-23T23:37:23.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:23.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180995+0000", 2023-09-23T23:37:23.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8621999999999997e-05 2023-09-23T23:37:23.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181017+0000", 2023-09-23T23:37:23.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.251e-05 2023-09-23T23:37:23.754 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.754 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.754 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.754 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21499 13.5 13:a6d31f08:::benchmark_data_smithi110_541928_object21498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:37:23.754 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.054699+0000", 2023-09-23T23:37:23.754 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 253.67280493199999, 2023-09-23T23:37:23.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1271894090000001, 2023-09-23T23:37:23.754 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.754 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:23.754 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:23.755 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:23.755 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:23.755 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21499 2023-09-23T23:37:23.755 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.755 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054699+0000", 2023-09-23T23:37:23.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.755 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.756 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054699+0000", 2023-09-23T23:37:23.756 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.756 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.756 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.756 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.756 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054697+0000", 2023-09-23T23:37:23.756 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:23.756 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.756 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.756 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.756 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054738+0000", 2023-09-23T23:37:23.756 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0728e-05 2023-09-23T23:37:23.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.757 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.757 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054757+0000", 2023-09-23T23:37:23.757 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9037000000000001e-05 2023-09-23T23:37:23.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.757 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.757 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054765+0000", 2023-09-23T23:37:23.757 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8010000000000004e-06 2023-09-23T23:37:23.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054821+0000", 2023-09-23T23:37:23.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5702999999999998e-05 2023-09-23T23:37:23.758 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:23.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054829+0000", 2023-09-23T23:37:23.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9579999999999996e-06 2023-09-23T23:37:23.758 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.179986+0000", 2023-09-23T23:37:23.759 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.125156388 2023-09-23T23:37:23.759 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.759 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.759 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:23.759 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180059+0000", 2023-09-23T23:37:23.759 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3510999999999995e-05 2023-09-23T23:37:23.759 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.759 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.759 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:37:23.759 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180157+0000", 2023-09-23T23:37:23.759 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8039000000000001e-05 2023-09-23T23:37:23.759 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.760 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.760 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:23.760 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181753+0000", 2023-09-23T23:37:23.760 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001595602 2023-09-23T23:37:23.760 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.760 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.760 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.760 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181813+0000", 2023-09-23T23:37:23.760 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9984e-05 2023-09-23T23:37:23.760 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.760 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.760 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.761 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181836+0000", 2023-09-23T23:37:23.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3373000000000001e-05 2023-09-23T23:37:23.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.761 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:23.761 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181865+0000", 2023-09-23T23:37:23.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8147e-05 2023-09-23T23:37:23.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.761 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.761 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181888+0000", 2023-09-23T23:37:23.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.37e-05 2023-09-23T23:37:23.762 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.762 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.762 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.762 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.762 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.762 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21528 13.5 13:a8de34c9:::benchmark_data_smithi110_541928_object21527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:37:23.762 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.066977+0000", 2023-09-23T23:37:23.762 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 253.66052667599999, 2023-09-23T23:37:23.762 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.116824818, 2023-09-23T23:37:23.762 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.762 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:23.762 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:23.763 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:23.763 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:23.763 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21528 2023-09-23T23:37:23.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.763 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.763 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.763 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.763 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.066977+0000", 2023-09-23T23:37:23.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.763 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.763 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.764 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.066977+0000", 2023-09-23T23:37:23.764 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.764 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.764 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.764 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.764 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.066976+0000", 2023-09-23T23:37:23.764 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:37:23.764 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.764 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.764 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.764 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067009+0000", 2023-09-23T23:37:23.764 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2963999999999998e-05 2023-09-23T23:37:23.764 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.765 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.765 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.765 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067017+0000", 2023-09-23T23:37:23.765 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8350000000000004e-06 2023-09-23T23:37:23.765 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.765 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.765 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.765 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067022+0000", 2023-09-23T23:37:23.765 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9989999999999999e-06 2023-09-23T23:37:23.765 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.765 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.765 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.766 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.068428+0000", 2023-09-23T23:37:23.766 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014058860000000001 2023-09-23T23:37:23.766 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.766 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.766 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:23.766 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.068438+0000", 2023-09-23T23:37:23.766 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0033e-05 2023-09-23T23:37:23.766 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.766 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.766 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.766 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181053+0000", 2023-09-23T23:37:23.766 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.112614311 2023-09-23T23:37:23.767 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.767 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.767 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:23.767 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181184+0000", 2023-09-23T23:37:23.767 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000131314 2023-09-23T23:37:23.767 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.767 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.767 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:37:23.767 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181273+0000", 2023-09-23T23:37:23.767 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.954e-05 2023-09-23T23:37:23.767 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.767 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.768 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.768 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183686+0000", 2023-09-23T23:37:23.768 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002412871 2023-09-23T23:37:23.768 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.768 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.768 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.768 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183729+0000", 2023-09-23T23:37:23.768 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2329000000000001e-05 2023-09-23T23:37:23.768 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.768 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.768 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:23.768 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183736+0000", 2023-09-23T23:37:23.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0360000000000001e-06 2023-09-23T23:37:23.769 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.769 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.769 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:23.769 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183767+0000", 2023-09-23T23:37:23.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1133000000000002e-05 2023-09-23T23:37:23.769 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.769 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.769 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.769 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183802+0000", 2023-09-23T23:37:23.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5176000000000003e-05 2023-09-23T23:37:23.769 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.769 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.770 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.770 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.770 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.770 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21532 13.5 13:a4c16192:::benchmark_data_smithi110_541928_object21531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:37:23.770 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.067036+0000", 2023-09-23T23:37:23.770 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 253.66046812600001, 2023-09-23T23:37:23.770 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1169511729999999, 2023-09-23T23:37:23.770 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.770 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:23.770 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:23.770 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:23.770 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:23.771 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21532 2023-09-23T23:37:23.771 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.771 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.771 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.771 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.771 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067036+0000", 2023-09-23T23:37:23.771 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.771 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.771 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.771 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.771 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067036+0000", 2023-09-23T23:37:23.771 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.772 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.772 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.772 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.772 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067035+0000", 2023-09-23T23:37:23.772 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:37:23.772 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.772 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.772 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.772 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067063+0000", 2023-09-23T23:37:23.772 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7453e-05 2023-09-23T23:37:23.772 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.772 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.772 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.773 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067070+0000", 2023-09-23T23:37:23.773 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4699999999999996e-06 2023-09-23T23:37:23.773 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.773 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.773 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.773 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067075+0000", 2023-09-23T23:37:23.773 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2730000000000002e-06 2023-09-23T23:37:23.773 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.773 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.773 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.773 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.068458+0000", 2023-09-23T23:37:23.773 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001382505 2023-09-23T23:37:23.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.774 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:23.774 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.068464+0000", 2023-09-23T23:37:23.774 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0909999999999998e-06 2023-09-23T23:37:23.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.774 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.774 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181930+0000", 2023-09-23T23:37:23.774 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.113466209 2023-09-23T23:37:23.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:23.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.182050+0000", 2023-09-23T23:37:23.775 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120068 2023-09-23T23:37:23.775 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:37:23.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.182218+0000", 2023-09-23T23:37:23.775 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016763300000000001 2023-09-23T23:37:23.775 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:23.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183812+0000", 2023-09-23T23:37:23.775 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001594179 2023-09-23T23:37:23.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.776 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.776 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.776 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183872+0000", 2023-09-23T23:37:23.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.033e-05 2023-09-23T23:37:23.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.776 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.776 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.776 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183920+0000", 2023-09-23T23:37:23.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7956e-05 2023-09-23T23:37:23.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.776 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:23.777 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183953+0000", 2023-09-23T23:37:23.777 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2855999999999998e-05 2023-09-23T23:37:23.777 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.777 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183987+0000", 2023-09-23T23:37:23.777 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3695000000000002e-05 2023-09-23T23:37:23.777 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.777 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.777 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.777 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.778 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21564 13.5 13:a601eb05:::benchmark_data_smithi110_541928_object21563:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:37:23.778 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.088589+0000", 2023-09-23T23:37:23.778 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 253.63891514700001, 2023-09-23T23:37:23.778 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0965361170000001, 2023-09-23T23:37:23.778 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.778 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:23.778 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:23.778 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:23.778 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:23.778 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21564 2023-09-23T23:37:23.778 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.779 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.779 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.779 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.779 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088589+0000", 2023-09-23T23:37:23.779 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.779 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.779 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.779 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.779 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088589+0000", 2023-09-23T23:37:23.779 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.779 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.779 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.780 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.780 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088588+0000", 2023-09-23T23:37:23.780 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:37:23.780 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.780 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.780 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.780 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088625+0000", 2023-09-23T23:37:23.780 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7500999999999999e-05 2023-09-23T23:37:23.780 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.780 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.780 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.780 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088639+0000", 2023-09-23T23:37:23.781 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3568e-05 2023-09-23T23:37:23.781 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.781 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.781 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.781 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088643+0000", 2023-09-23T23:37:23.781 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7219999999999999e-06 2023-09-23T23:37:23.781 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.781 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.781 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.781 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088698+0000", 2023-09-23T23:37:23.781 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4797999999999999e-05 2023-09-23T23:37:23.781 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.782 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.782 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:23.782 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088713+0000", 2023-09-23T23:37:23.782 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4809e-05 2023-09-23T23:37:23.782 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.782 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.782 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.782 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.182903+0000", 2023-09-23T23:37:23.782 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.094190306 2023-09-23T23:37:23.782 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.782 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.782 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:23.783 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.182976+0000", 2023-09-23T23:37:23.783 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3059000000000006e-05 2023-09-23T23:37:23.783 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.783 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.783 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:37:23.783 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183121+0000", 2023-09-23T23:37:23.783 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000144797 2023-09-23T23:37:23.783 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.783 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.783 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:23.783 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183999+0000", 2023-09-23T23:37:23.783 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00087746999999999999 2023-09-23T23:37:23.783 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.784 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.784 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.784 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185012+0000", 2023-09-23T23:37:23.784 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001013735 2023-09-23T23:37:23.784 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.784 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.784 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.784 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185045+0000", 2023-09-23T23:37:23.784 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2339000000000003e-05 2023-09-23T23:37:23.784 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.784 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.784 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:23.784 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185088+0000", 2023-09-23T23:37:23.785 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2854e-05 2023-09-23T23:37:23.785 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.785 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.785 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.785 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185125+0000", 2023-09-23T23:37:23.785 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.714e-05 2023-09-23T23:37:23.785 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.785 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.785 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.785 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.785 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.785 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21580 13.5 13:afa4f38d:::benchmark_data_smithi110_541928_object21579:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:37:23.785 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.097520+0000", 2023-09-23T23:37:23.786 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 253.629984181, 2023-09-23T23:37:23.786 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.089325219, 2023-09-23T23:37:23.786 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.786 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:23.786 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:23.786 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:23.786 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:23.786 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21580 2023-09-23T23:37:23.786 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.786 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.786 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.786 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.787 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097520+0000", 2023-09-23T23:37:23.787 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.787 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.787 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.787 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.787 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097520+0000", 2023-09-23T23:37:23.787 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.787 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.787 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.787 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.787 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097518+0000", 2023-09-23T23:37:23.787 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:23.788 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.788 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.788 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.788 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097552+0000", 2023-09-23T23:37:23.788 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3627999999999997e-05 2023-09-23T23:37:23.788 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.788 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.788 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.788 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097567+0000", 2023-09-23T23:37:23.788 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5477000000000001e-05 2023-09-23T23:37:23.788 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.788 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.788 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.789 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097574+0000", 2023-09-23T23:37:23.789 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7309999999999996e-06 2023-09-23T23:37:23.789 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.789 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.789 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.789 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097718+0000", 2023-09-23T23:37:23.789 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014400700000000001 2023-09-23T23:37:23.789 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.789 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.789 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:23.789 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097740+0000", 2023-09-23T23:37:23.789 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2238999999999998e-05 2023-09-23T23:37:23.789 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.790 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.790 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.790 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.184022+0000", 2023-09-23T23:37:23.790 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0862815990000001 2023-09-23T23:37:23.790 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.790 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.790 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:23.790 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.184170+0000", 2023-09-23T23:37:23.790 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000147754 2023-09-23T23:37:23.790 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.790 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.790 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:37:23.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.184340+0000", 2023-09-23T23:37:23.791 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017066499999999999 2023-09-23T23:37:23.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.791 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186756+0000", 2023-09-23T23:37:23.791 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024154049999999998 2023-09-23T23:37:23.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.791 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186789+0000", 2023-09-23T23:37:23.791 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3621000000000002e-05 2023-09-23T23:37:23.792 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.792 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.792 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:23.792 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186795+0000", 2023-09-23T23:37:23.792 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4809999999999997e-06 2023-09-23T23:37:23.792 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.792 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.792 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:23.792 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186822+0000", 2023-09-23T23:37:23.792 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7166e-05 2023-09-23T23:37:23.792 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.792 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.792 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.793 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186845+0000", 2023-09-23T23:37:23.793 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2716000000000001e-05 2023-09-23T23:37:23.793 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.793 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.793 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.793 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.793 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.793 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21590 13.5 13:a913081b:::benchmark_data_smithi110_541928_object21589:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:37:23.793 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.101884+0000", 2023-09-23T23:37:23.793 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 253.62561987300001, 2023-09-23T23:37:23.793 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0874988189999999, 2023-09-23T23:37:23.794 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.794 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:23.794 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:23.794 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:23.794 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:23.794 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21590 2023-09-23T23:37:23.794 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.794 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.794 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.794 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.794 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101884+0000", 2023-09-23T23:37:23.794 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.795 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.795 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.795 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.795 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101884+0000", 2023-09-23T23:37:23.795 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.795 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.795 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.795 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.795 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101883+0000", 2023-09-23T23:37:23.795 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:37:23.795 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.795 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.795 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.796 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101943+0000", 2023-09-23T23:37:23.796 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9757999999999999e-05 2023-09-23T23:37:23.796 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.796 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.796 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.796 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101952+0000", 2023-09-23T23:37:23.796 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6420000000000002e-06 2023-09-23T23:37:23.796 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.796 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.796 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.796 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101958+0000", 2023-09-23T23:37:23.797 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0900000000000001e-06 2023-09-23T23:37:23.797 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.797 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.797 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.797 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102140+0000", 2023-09-23T23:37:23.797 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018145100000000001 2023-09-23T23:37:23.797 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.797 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.797 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:23.797 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102153+0000", 2023-09-23T23:37:23.797 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2676e-05 2023-09-23T23:37:23.797 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.797 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.798 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.798 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185166+0000", 2023-09-23T23:37:23.798 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0830130069999999 2023-09-23T23:37:23.798 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.798 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.798 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:23.798 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185286+0000", 2023-09-23T23:37:23.798 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120227 2023-09-23T23:37:23.798 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.798 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.798 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:37:23.799 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185443+0000", 2023-09-23T23:37:23.799 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000156844 2023-09-23T23:37:23.799 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.799 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.799 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:23.799 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186851+0000", 2023-09-23T23:37:23.799 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014085199999999999 2023-09-23T23:37:23.799 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.799 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.799 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.799 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188371+0000", 2023-09-23T23:37:23.799 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015201780000000001 2023-09-23T23:37:23.800 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.800 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.800 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.800 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189290+0000", 2023-09-23T23:37:23.800 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00091839000000000005 2023-09-23T23:37:23.800 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.800 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.800 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:23.800 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189342+0000", 2023-09-23T23:37:23.800 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2110000000000001e-05 2023-09-23T23:37:23.800 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.800 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.800 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.801 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189383+0000", 2023-09-23T23:37:23.801 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0859999999999998e-05 2023-09-23T23:37:23.801 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.801 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.801 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.801 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.801 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.801 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21591 13.5 13:ae681958:::benchmark_data_smithi110_541928_object21590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:37:23.801 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.102344+0000", 2023-09-23T23:37:23.801 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 253.625160042, 2023-09-23T23:37:23.801 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0912746840000001, 2023-09-23T23:37:23.801 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.801 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:23.802 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:23.802 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:23.802 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:23.802 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21591 2023-09-23T23:37:23.802 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.802 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.802 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.802 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.802 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102344+0000", 2023-09-23T23:37:23.802 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.802 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.803 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.803 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.803 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102344+0000", 2023-09-23T23:37:23.803 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.803 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.803 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.803 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.803 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102343+0000", 2023-09-23T23:37:23.803 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:37:23.803 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.803 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.803 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.803 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102429+0000", 2023-09-23T23:37:23.804 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6637000000000004e-05 2023-09-23T23:37:23.804 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.804 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.804 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.804 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102441+0000", 2023-09-23T23:37:23.804 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1902999999999999e-05 2023-09-23T23:37:23.804 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.804 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.804 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.804 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102450+0000", 2023-09-23T23:37:23.804 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2320000000000001e-06 2023-09-23T23:37:23.804 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.805 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.805 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.805 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102504+0000", 2023-09-23T23:37:23.805 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3981999999999999e-05 2023-09-23T23:37:23.805 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.805 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.805 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:23.805 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102511+0000", 2023-09-23T23:37:23.805 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8399999999999995e-06 2023-09-23T23:37:23.805 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.805 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.805 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.805 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186108+0000", 2023-09-23T23:37:23.806 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.083596351 2023-09-23T23:37:23.806 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.806 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.806 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:23.806 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186201+0000", 2023-09-23T23:37:23.806 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3474000000000007e-05 2023-09-23T23:37:23.806 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.806 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.806 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:37:23.806 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186283+0000", 2023-09-23T23:37:23.806 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1625999999999994e-05 2023-09-23T23:37:23.806 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.807 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.807 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.807 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188420+0000", 2023-09-23T23:37:23.807 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021368749999999999 2023-09-23T23:37:23.807 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.807 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.807 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:23.807 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188429+0000", 2023-09-23T23:37:23.807 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5370000000000003e-06 2023-09-23T23:37:23.807 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.807 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.807 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.808 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193525+0000", 2023-09-23T23:37:23.808 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050954379999999999 2023-09-23T23:37:23.808 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.808 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.808 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:23.808 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193578+0000", 2023-09-23T23:37:23.808 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2927999999999999e-05 2023-09-23T23:37:23.808 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.808 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.808 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.808 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193618+0000", 2023-09-23T23:37:23.808 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0822000000000003e-05 2023-09-23T23:37:23.809 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.809 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.809 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.809 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.809 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.809 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21598 13.5 13:a172c780:::benchmark_data_smithi110_541928_object21597:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:37:23.809 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.106175+0000", 2023-09-23T23:37:23.809 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 253.621328847, 2023-09-23T23:37:23.809 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0875860749999999, 2023-09-23T23:37:23.809 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.809 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:23.809 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:23.810 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:23.810 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:23.810 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21598 2023-09-23T23:37:23.810 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.810 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.810 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.810 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.810 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106175+0000", 2023-09-23T23:37:23.810 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.810 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.810 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.811 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.811 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106175+0000", 2023-09-23T23:37:23.811 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.811 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.811 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.811 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.811 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106174+0000", 2023-09-23T23:37:23.811 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:37:23.811 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.811 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.811 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.811 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106283+0000", 2023-09-23T23:37:23.811 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109133 2023-09-23T23:37:23.812 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.812 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.812 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.812 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106294+0000", 2023-09-23T23:37:23.812 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0994e-05 2023-09-23T23:37:23.812 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.812 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.812 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.812 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106301+0000", 2023-09-23T23:37:23.812 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5089999999999999e-06 2023-09-23T23:37:23.812 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.812 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.813 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.813 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106360+0000", 2023-09-23T23:37:23.813 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8828e-05 2023-09-23T23:37:23.813 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.813 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.813 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:23.813 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106369+0000", 2023-09-23T23:37:23.813 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1880000000000008e-06 2023-09-23T23:37:23.813 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.813 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.813 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.813 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186882+0000", 2023-09-23T23:37:23.814 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0805130059999999 2023-09-23T23:37:23.814 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.814 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.814 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:23.814 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186960+0000", 2023-09-23T23:37:23.814 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7863999999999999e-05 2023-09-23T23:37:23.814 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.814 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.814 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:37:23.814 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.187041+0000", 2023-09-23T23:37:23.814 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1542e-05 2023-09-23T23:37:23.815 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.815 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.815 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:23.815 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188432+0000", 2023-09-23T23:37:23.815 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001390797 2023-09-23T23:37:23.815 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.815 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.815 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.815 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188460+0000", 2023-09-23T23:37:23.815 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7668e-05 2023-09-23T23:37:23.815 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.815 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.816 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.816 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193681+0000", 2023-09-23T23:37:23.816 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0052214999999999996 2023-09-23T23:37:23.816 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.816 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.816 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:23.816 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193725+0000", 2023-09-23T23:37:23.816 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3377000000000002e-05 2023-09-23T23:37:23.816 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.816 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.816 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.816 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193761+0000", 2023-09-23T23:37:23.816 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6464000000000002e-05 2023-09-23T23:37:23.817 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.817 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.817 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.817 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.817 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.817 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21614 13.5 13:ab7b6c08:::benchmark_data_smithi110_541928_object21613:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:37:23.817 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.115290+0000", 2023-09-23T23:37:23.817 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 253.61221374199999, 2023-09-23T23:37:23.817 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.078720337, 2023-09-23T23:37:23.817 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.817 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:23.818 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:23.818 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:23.818 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:23.818 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21614 2023-09-23T23:37:23.818 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.818 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.818 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.818 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.818 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115290+0000", 2023-09-23T23:37:23.818 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.818 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.818 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.819 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.819 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115290+0000", 2023-09-23T23:37:23.819 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.819 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.819 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.819 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.819 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115289+0000", 2023-09-23T23:37:23.819 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:23.819 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.819 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.819 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.819 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115353+0000", 2023-09-23T23:37:23.820 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4473000000000004e-05 2023-09-23T23:37:23.820 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.820 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.820 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.820 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115365+0000", 2023-09-23T23:37:23.820 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.164e-05 2023-09-23T23:37:23.820 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.820 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.820 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.820 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115371+0000", 2023-09-23T23:37:23.820 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1500000000000004e-06 2023-09-23T23:37:23.820 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.820 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.821 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.821 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115421+0000", 2023-09-23T23:37:23.821 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0117999999999997e-05 2023-09-23T23:37:23.821 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.821 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.821 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:23.821 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115431+0000", 2023-09-23T23:37:23.821 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4350000000000003e-06 2023-09-23T23:37:23.821 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.821 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.821 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.821 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.187560+0000", 2023-09-23T23:37:23.822 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0721295150000001 2023-09-23T23:37:23.822 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.822 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.822 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:23.822 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.187686+0000", 2023-09-23T23:37:23.822 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012613699999999999 2023-09-23T23:37:23.822 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.822 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.822 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:37:23.822 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.187797+0000", 2023-09-23T23:37:23.822 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110852 2023-09-23T23:37:23.822 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.823 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:23.823 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188463+0000", 2023-09-23T23:37:23.823 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00066563799999999997 2023-09-23T23:37:23.823 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.823 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.823 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189424+0000", 2023-09-23T23:37:23.823 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00096155399999999999 2023-09-23T23:37:23.823 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.824 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.824 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193937+0000", 2023-09-23T23:37:23.824 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00451308 2023-09-23T23:37:23.824 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.824 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.824 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:23.824 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193979+0000", 2023-09-23T23:37:23.824 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1501999999999997e-05 2023-09-23T23:37:23.824 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.824 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.824 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.824 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194010+0000", 2023-09-23T23:37:23.825 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1535999999999998e-05 2023-09-23T23:37:23.825 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.825 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.825 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.825 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.825 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.825 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21623 13.5 13:ac3554a5:::benchmark_data_smithi110_541928_object21622:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:37:23.825 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.123936+0000", 2023-09-23T23:37:23.825 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 253.60356770199999, 2023-09-23T23:37:23.825 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0702429680000001, 2023-09-23T23:37:23.825 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.825 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:23.825 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:23.826 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:23.826 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:23.826 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21623 2023-09-23T23:37:23.826 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.826 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.826 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.826 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.826 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.123936+0000", 2023-09-23T23:37:23.826 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.826 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.826 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.826 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.827 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.123936+0000", 2023-09-23T23:37:23.827 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.827 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.827 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.827 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.827 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.123935+0000", 2023-09-23T23:37:23.827 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:23.827 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.827 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.827 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.827 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.124042+0000", 2023-09-23T23:37:23.827 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010689200000000001 2023-09-23T23:37:23.828 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.828 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.828 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.828 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.124053+0000", 2023-09-23T23:37:23.828 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1221e-05 2023-09-23T23:37:23.828 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.828 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.828 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.828 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.124058+0000", 2023-09-23T23:37:23.828 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.074e-06 2023-09-23T23:37:23.828 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.828 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.829 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.124114+0000", 2023-09-23T23:37:23.829 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6026999999999997e-05 2023-09-23T23:37:23.829 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.829 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.829 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:23.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.124124+0000", 2023-09-23T23:37:23.829 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0032e-05 2023-09-23T23:37:23.829 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.829 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.829 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188483+0000", 2023-09-23T23:37:23.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.064359354 2023-09-23T23:37:23.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.830 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.830 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:23.830 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188589+0000", 2023-09-23T23:37:23.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105889 2023-09-23T23:37:23.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.830 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.830 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:37:23.830 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188737+0000", 2023-09-23T23:37:23.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014783099999999999 2023-09-23T23:37:23.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.831 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.831 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:23.831 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191026+0000", 2023-09-23T23:37:23.831 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022893409999999999 2023-09-23T23:37:23.831 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.831 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.831 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.831 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191847+0000", 2023-09-23T23:37:23.831 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00082065100000000004 2023-09-23T23:37:23.831 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.831 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.831 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.831 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194089+0000", 2023-09-23T23:37:23.832 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022423339999999999 2023-09-23T23:37:23.832 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.832 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.832 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:23.832 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194143+0000", 2023-09-23T23:37:23.832 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3446999999999998e-05 2023-09-23T23:37:23.832 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.832 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.832 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.832 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194179+0000", 2023-09-23T23:37:23.832 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6216e-05 2023-09-23T23:37:23.832 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.833 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.833 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.833 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.833 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.833 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21639 13.5 13:afb98446:::benchmark_data_smithi110_541928_object21638:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:37:23.833 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.135557+0000", 2023-09-23T23:37:23.833 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 253.59194650399999, 2023-09-23T23:37:23.833 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0589459779999999, 2023-09-23T23:37:23.833 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.833 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:23.833 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:23.833 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:23.834 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:23.834 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21639 2023-09-23T23:37:23.834 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.834 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.834 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.834 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.834 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135557+0000", 2023-09-23T23:37:23.834 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.834 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.834 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.834 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.834 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135557+0000", 2023-09-23T23:37:23.835 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.835 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.835 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.835 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.835 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135557+0000", 2023-09-23T23:37:23.835 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:37:23.835 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.835 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.835 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.835 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135622+0000", 2023-09-23T23:37:23.835 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4926999999999997e-05 2023-09-23T23:37:23.835 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.836 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.836 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.836 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135628+0000", 2023-09-23T23:37:23.836 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0299999999999999e-06 2023-09-23T23:37:23.836 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.836 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.836 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.836 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135631+0000", 2023-09-23T23:37:23.836 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.841e-06 2023-09-23T23:37:23.836 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.836 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.836 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.837 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135680+0000", 2023-09-23T23:37:23.837 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9818999999999999e-05 2023-09-23T23:37:23.837 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.837 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.837 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:23.837 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135691+0000", 2023-09-23T23:37:23.837 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0976e-05 2023-09-23T23:37:23.837 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.837 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.837 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.837 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189484+0000", 2023-09-23T23:37:23.837 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.053793092 2023-09-23T23:37:23.838 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.838 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.838 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:23.838 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189619+0000", 2023-09-23T23:37:23.838 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013436399999999999 2023-09-23T23:37:23.838 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.838 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.838 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:37:23.838 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189777+0000", 2023-09-23T23:37:23.838 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015826999999999999 2023-09-23T23:37:23.838 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.838 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.839 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:23.839 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191857+0000", 2023-09-23T23:37:23.839 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020803929999999998 2023-09-23T23:37:23.839 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.839 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.839 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.839 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191914+0000", 2023-09-23T23:37:23.839 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6075000000000001e-05 2023-09-23T23:37:23.839 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.839 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.839 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.839 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194427+0000", 2023-09-23T23:37:23.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002513684 2023-09-23T23:37:23.840 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.840 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:23.840 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194468+0000", 2023-09-23T23:37:23.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1183999999999997e-05 2023-09-23T23:37:23.840 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.840 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.840 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194503+0000", 2023-09-23T23:37:23.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4867999999999999e-05 2023-09-23T23:37:23.841 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.841 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.841 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.841 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.841 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.841 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21659 13.5 13:aad23106:::benchmark_data_smithi110_541928_object21658:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:37:23.841 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.150190+0000", 2023-09-23T23:37:23.841 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 253.57731414400001, 2023-09-23T23:37:23.841 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0444529309999999, 2023-09-23T23:37:23.841 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.841 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:23.842 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:23.842 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:23.842 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:23.842 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21659 2023-09-23T23:37:23.842 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.842 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.842 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.842 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.842 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150190+0000", 2023-09-23T23:37:23.842 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.842 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.842 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.842 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150190+0000", 2023-09-23T23:37:23.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.843 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150189+0000", 2023-09-23T23:37:23.845 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:37:23.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150260+0000", 2023-09-23T23:37:23.845 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1550000000000004e-05 2023-09-23T23:37:23.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150272+0000", 2023-09-23T23:37:23.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1328000000000001e-05 2023-09-23T23:37:23.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150276+0000", 2023-09-23T23:37:23.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6700000000000002e-06 2023-09-23T23:37:23.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150329+0000", 2023-09-23T23:37:23.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3118999999999997e-05 2023-09-23T23:37:23.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.847 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:23.847 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150339+0000", 2023-09-23T23:37:23.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4660000000000005e-06 2023-09-23T23:37:23.847 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.847 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.847 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.190332+0000", 2023-09-23T23:37:23.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.039992633 2023-09-23T23:37:23.847 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.847 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:23.848 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.190453+0000", 2023-09-23T23:37:23.848 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012097899999999999 2023-09-23T23:37:23.848 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.848 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.848 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:37:23.848 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.190560+0000", 2023-09-23T23:37:23.848 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107587 2023-09-23T23:37:23.848 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.848 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.848 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:23.848 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.192645+0000", 2023-09-23T23:37:23.848 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020851329999999999 2023-09-23T23:37:23.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.849 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.849 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.849 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193487+0000", 2023-09-23T23:37:23.849 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00084168199999999996 2023-09-23T23:37:23.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.849 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.849 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.849 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194573+0000", 2023-09-23T23:37:23.849 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001085978 2023-09-23T23:37:23.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.849 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.850 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:23.850 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194610+0000", 2023-09-23T23:37:23.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7512000000000003e-05 2023-09-23T23:37:23.850 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.850 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.850 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.850 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194643+0000", 2023-09-23T23:37:23.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2147000000000002e-05 2023-09-23T23:37:23.850 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.850 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.850 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.851 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.851 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21680 13.5 13:ac31eb73:::benchmark_data_smithi110_541928_object21679:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:37:23.851 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.170348+0000", 2023-09-23T23:37:23.851 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 253.55715531800001, 2023-09-23T23:37:23.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0244998839999999, 2023-09-23T23:37:23.851 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.851 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:23.851 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:23.851 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:23.851 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:23.851 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21680 2023-09-23T23:37:23.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.852 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.852 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170348+0000", 2023-09-23T23:37:23.852 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.852 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170348+0000", 2023-09-23T23:37:23.852 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.853 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.853 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170347+0000", 2023-09-23T23:37:23.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:23.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.853 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.853 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170403+0000", 2023-09-23T23:37:23.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6042000000000003e-05 2023-09-23T23:37:23.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.853 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170415+0000", 2023-09-23T23:37:23.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2238e-05 2023-09-23T23:37:23.854 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.854 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170423+0000", 2023-09-23T23:37:23.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1829999999999997e-06 2023-09-23T23:37:23.854 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.854 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170480+0000", 2023-09-23T23:37:23.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7476999999999999e-05 2023-09-23T23:37:23.855 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.855 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.855 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:23.855 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170488+0000", 2023-09-23T23:37:23.855 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2390000000000002e-06 2023-09-23T23:37:23.855 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.855 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.855 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.855 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191062+0000", 2023-09-23T23:37:23.855 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0205732489999999 2023-09-23T23:37:23.855 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.855 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.855 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:23.856 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191141+0000", 2023-09-23T23:37:23.856 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9435000000000001e-05 2023-09-23T23:37:23.856 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.856 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.856 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:37:23.856 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191244+0000", 2023-09-23T23:37:23.856 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010341299999999999 2023-09-23T23:37:23.856 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.856 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.856 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:23.856 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.192657+0000", 2023-09-23T23:37:23.857 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014124599999999999 2023-09-23T23:37:23.857 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193656+0000", 2023-09-23T23:37:23.857 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00099945200000000007 2023-09-23T23:37:23.857 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194770+0000", 2023-09-23T23:37:23.857 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001114163 2023-09-23T23:37:23.857 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.858 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:23.858 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194815+0000", 2023-09-23T23:37:23.858 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4583999999999999e-05 2023-09-23T23:37:23.858 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.858 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.858 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.858 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194848+0000", 2023-09-23T23:37:23.858 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3272000000000003e-05 2023-09-23T23:37:23.858 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.858 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.858 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.859 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21693 13.5 13:a03052c3:::benchmark_data_smithi110_541928_object21692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:37:23.859 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.193329+0000", 2023-09-23T23:37:23.859 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 253.534175178, 2023-09-23T23:37:23.859 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.001638582, 2023-09-23T23:37:23.859 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.859 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:23.859 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:23.859 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:23.859 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:23.859 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21693 2023-09-23T23:37:23.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.860 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193329+0000", 2023-09-23T23:37:23.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193329+0000", 2023-09-23T23:37:23.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.861 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.861 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193328+0000", 2023-09-23T23:37:23.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:37:23.861 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.861 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.861 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.861 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193375+0000", 2023-09-23T23:37:23.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7132000000000002e-05 2023-09-23T23:37:23.861 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.861 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.861 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.861 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193390+0000", 2023-09-23T23:37:23.862 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4841000000000001e-05 2023-09-23T23:37:23.862 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.862 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.862 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.862 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193396+0000", 2023-09-23T23:37:23.862 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1299999999999998e-06 2023-09-23T23:37:23.862 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.862 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.862 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.862 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193442+0000", 2023-09-23T23:37:23.862 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6621e-05 2023-09-23T23:37:23.862 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.863 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.863 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:23.863 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193450+0000", 2023-09-23T23:37:23.863 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1079999999999993e-06 2023-09-23T23:37:23.863 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.863 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.863 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.863 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191950+0000", 2023-09-23T23:37:23.863 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99849948600000005 2023-09-23T23:37:23.863 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.863 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.863 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:23.863 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.192054+0000", 2023-09-23T23:37:23.864 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103933 2023-09-23T23:37:23.864 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.864 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.864 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:37:23.864 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.192153+0000", 2023-09-23T23:37:23.864 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9110999999999993e-05 2023-09-23T23:37:23.864 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.864 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.864 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:23.864 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193631+0000", 2023-09-23T23:37:23.864 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014778619999999999 2023-09-23T23:37:23.864 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.865 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.865 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193884+0000", 2023-09-23T23:37:23.865 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025308299999999998 2023-09-23T23:37:23.865 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.865 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:23.865 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194896+0000", 2023-09-23T23:37:23.865 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010124159999999999 2023-09-23T23:37:23.865 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.866 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:23.866 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194935+0000", 2023-09-23T23:37:23.866 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8470000000000003e-05 2023-09-23T23:37:23.866 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.866 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.866 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.866 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194967+0000", 2023-09-23T23:37:23.866 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2385999999999998e-05 2023-09-23T23:37:23.866 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.866 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.866 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.866 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.867 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPull(13.d e595/566 cost 1049576)", 2023-09-23T23:37:23.867 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:36:32.914368+0000", 2023-09-23T23:37:23.867 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 50.813136051000001, 2023-09-23T23:37:23.867 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023016680000000002, 2023-09-23T23:37:23.867 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.867 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:37:23.867 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.867 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.867 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:32.914368+0000", 2023-09-23T23:37:23.867 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.868 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.868 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.868 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.868 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:32.914368+0000", 2023-09-23T23:37:23.868 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.868 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.868 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.868 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.868 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:32.914367+0000", 2023-09-23T23:37:23.868 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:37:23.868 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.868 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.868 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.869 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:32.914382+0000", 2023-09-23T23:37:23.869 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.539e-05 2023-09-23T23:37:23.869 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.869 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.869 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.869 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:32.914403+0000", 2023-09-23T23:37:23.869 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0914999999999999e-05 2023-09-23T23:37:23.869 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.869 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.869 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.869 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:32.914412+0000", 2023-09-23T23:37:23.869 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2410000000000001e-06 2023-09-23T23:37:23.870 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.870 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.870 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.870 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:32.914464+0000", 2023-09-23T23:37:23.870 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1647999999999997e-05 2023-09-23T23:37:23.870 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.870 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.870 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.870 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:32.916669+0000", 2023-09-23T23:37:23.870 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022055669999999999 2023-09-23T23:37:23.870 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.871 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.871 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.871 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.871 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPull(13.d e597/566 cost 1049576)", 2023-09-23T23:37:23.871 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:36:54.414048+0000", 2023-09-23T23:37:23.871 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 29.313455544, 2023-09-23T23:37:23.871 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030373409999999998, 2023-09-23T23:37:23.871 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.871 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:37:23.871 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.871 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.872 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:54.414048+0000", 2023-09-23T23:37:23.872 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.872 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.872 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.872 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.872 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:54.414048+0000", 2023-09-23T23:37:23.872 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.872 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.872 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.872 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.872 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:54.414047+0000", 2023-09-23T23:37:23.872 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:23.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.873 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.873 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:54.414063+0000", 2023-09-23T23:37:23.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5695000000000001e-05 2023-09-23T23:37:23.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.873 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.873 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:54.414081+0000", 2023-09-23T23:37:23.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7971000000000001e-05 2023-09-23T23:37:23.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:54.414088+0000", 2023-09-23T23:37:23.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3579999999999997e-06 2023-09-23T23:37:23.874 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:54.414143+0000", 2023-09-23T23:37:23.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5237e-05 2023-09-23T23:37:23.874 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:36:54.417086+0000", 2023-09-23T23:37:23.875 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029423969999999998 2023-09-23T23:37:23.875 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.875 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.875 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.875 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.875 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPull(13.d e600/566 cost 1049576)", 2023-09-23T23:37:23.875 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:37:22.991227+0000", 2023-09-23T23:37:23.875 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.736276288, 2023-09-23T23:37:23.875 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00304579, 2023-09-23T23:37:23.875 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:23.875 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:37:23.875 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:23.876 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.876 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:23.876 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:22.991227+0000", 2023-09-23T23:37:23.876 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.876 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.876 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.876 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:23.876 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:22.991227+0000", 2023-09-23T23:37:23.876 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:23.876 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.876 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.876 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:23.877 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:22.991226+0000", 2023-09-23T23:37:23.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:37:23.877 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.877 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.877 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:23.877 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:22.991242+0000", 2023-09-23T23:37:23.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5936000000000001e-05 2023-09-23T23:37:23.877 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.877 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.877 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:23.877 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:22.991262+0000", 2023-09-23T23:37:23.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9868999999999999e-05 2023-09-23T23:37:23.877 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.878 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:23.878 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:22.991270+0000", 2023-09-23T23:37:23.878 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5449999999999998e-06 2023-09-23T23:37:23.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.878 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:23.878 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:22.991319+0000", 2023-09-23T23:37:23.878 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8928999999999999e-05 2023-09-23T23:37:23.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:23.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:23.878 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:23.879 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:22.994273+0000", 2023-09-23T23:37:23.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029546450000000001 2023-09-23T23:37:23.879 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.879 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.879 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.879 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:23.879 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:23.879 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:37:23.935 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:23.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:23.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:24.039 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:24.067 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:24.110 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:24.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:24.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:24.214 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:37:24.372 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:24.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:24 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 23640/142194 objects degraded (16.625%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:24.570 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:24 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 23640/142194 objects degraded (16.625%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:24.570 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:24 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 23640/142194 objects degraded (16.625%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:24.589 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:37:24.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:24.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:37:24.648 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:37:24.699 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:24.742 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:24.803 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:24.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:24.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:24.912 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:24.969 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:25.110 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:37:25.148 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:25.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:25.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:25.253 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:37:25.281 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:25.342 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:37:25.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:25.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:37:25.446 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:25.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:25 smithi110 ceph-mon[212939]: pgmap v1686: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 23536/140958 objects degraded (16.697%); 2725/140958 objects misplaced (1.933%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T23:37:25.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:25 smithi110 ceph-mon[221102]: pgmap v1686: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 23536/140958 objects degraded (16.697%); 2725/140958 objects misplaced (1.933%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T23:37:25.473 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:25.526 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:25.534 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:25 smithi138 ceph-mon[193606]: pgmap v1686: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 23536/140958 objects degraded (16.697%); 2725/140958 objects misplaced (1.933%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T23:37:25.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:25.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:25.635 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:25.673 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:25.734 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:25.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:25.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:25.839 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:37:25.865 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:37:25.865 INFO:teuthology.orchestra.run.smithi110.stdout:{"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 23536/140958 objects degraded (16.697%), 7 pgs degraded, 5 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2123,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":600,"num_osds":8,"num_up_osds":8,"osd_up_since":1695511414,"num_in_osds":7,"osd_in_since":1695512030,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":110},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+recovering","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":119,"num_pools":5,"num_objects":46986,"data_bytes":3078272245,"bytes_used":14160838656,"bytes_avail":657804410880,"bytes_total":671965249536,"degraded_objects":23536,"degraded_total":140958,"degraded_ratio":0.16697172207324168,"misplaced_objects":2725,"misplaced_total":140958,"misplaced_ratio":0.019331999602718541,"recovering_objects_per_sec":61,"recovering_bytes_per_sec":4035886,"recovering_keys_per_sec":0,"num_objects_recovered":616,"num_bytes_recovered":40370176,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":311},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi110.fuglpi","status":"up:active","gid":24572}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.110:8443/","prometheus":"http://172.21.15.110:9283/"}},"servicemap":{"epoch":32,"modified":"2023-09-23T23:32:57.852217+0000","services":{}},"progress_events":{"833ef15d-07ae-4299-b4e4-2cf94e068147":{"message":"Upgrade to 18.2.0-460-gbb0daeff (10m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true},"c67c6d89-9cd0-495b-b1de-fa24b5d12727":{"message":"Global Recovery Event (3m)\n [=========================...] (remaining: 15s)","progress":0.92436975240707397,"add_to_ceph_s":true}}} 2023-09-23T23:37:25.890 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:26.113 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:37:26.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:26.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:37:26.217 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:26.244 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:26.295 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:37:26.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:26 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2481386764' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:26.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:26 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:26.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:26 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/438465086' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:37:26.315 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:26 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2481386764' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:26.315 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:26 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:26.315 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:26 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/438465086' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:37:26.338 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:26.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:26.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:26.441 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:26.476 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:26.505 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:37:26.535 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:26 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2481386764' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:26.535 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:26 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:26.535 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:26 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/438465086' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:37:26.853 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:26.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:26.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:26.869 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:37:26.870 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:37:26.957 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:27.008 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:27.060 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:27.063 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:37:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:37:26] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-09-23T23:37:27.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:27.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:27.140 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:37:27.141 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:37:27.141 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:27.141 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:130972 13.5 13:aa5ee02f:::benchmark_data_smithi110_541928_object43192:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e601)", 2023-09-23T23:37:27.141 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:37:27.099367+0000", 2023-09-23T23:37:27.141 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.040011823000000002, 2023-09-23T23:37:27.141 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.040037413000000001, 2023-09-23T23:37:27.141 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:27.141 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:37:27.141 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:27.141 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:27.142 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:27.142 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 130972 2023-09-23T23:37:27.142 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:27.142 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:27.142 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:27.142 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:27.142 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:27.099367+0000", 2023-09-23T23:37:27.142 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:27.142 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:27.142 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:27.142 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:27.143 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:27.099367+0000", 2023-09-23T23:37:27.143 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:27.143 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:27.143 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:27.143 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:27.143 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:27.099365+0000", 2023-09-23T23:37:27.143 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:27.143 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:27.143 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:27.143 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:27.143 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:27.099375+0000", 2023-09-23T23:37:27.143 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5899999999999997e-06 2023-09-23T23:37:27.144 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:27.144 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:27.144 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:27.144 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:27.099396+0000", 2023-09-23T23:37:27.144 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1393999999999999e-05 2023-09-23T23:37:27.144 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:27.144 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:27.144 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:27.144 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:27.099404+0000", 2023-09-23T23:37:27.144 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8869999999999992e-06 2023-09-23T23:37:27.144 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:27.144 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:27.145 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:27.145 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:27.099449+0000", 2023-09-23T23:37:27.145 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4675000000000002e-05 2023-09-23T23:37:27.145 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:27.145 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:27.145 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:27.145 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:27.099584+0000", 2023-09-23T23:37:27.145 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013545699999999999 2023-09-23T23:37:27.145 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:27.145 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:27.145 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:37:27.145 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:27.099705+0000", 2023-09-23T23:37:27.146 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012067599999999999 2023-09-23T23:37:27.146 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:27.146 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:27.146 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:27.146 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:27.102389+0000", 2023-09-23T23:37:27.146 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0026840879999999998 2023-09-23T23:37:27.146 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:27.146 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:27.146 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:27.146 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:27.105686+0000", 2023-09-23T23:37:27.146 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032976339999999998 2023-09-23T23:37:27.146 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:27.147 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:27.147 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:27.147 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:27.147 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-09-23T23:37:27.147 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 1 2023-09-23T23:37:27.147 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:37:27.165 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:37:27.205 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:27 smithi138 ceph-mon[193606]: pgmap v1687: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 324 op/s; 23438/137148 objects degraded (17.090%); 2725/137148 objects misplaced (1.987%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T23:37:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:27 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:37:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:27 smithi138 ceph-mon[193606]: failed to encode map e601 with expected crc 2023-09-23T23:37:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:27 smithi138 ceph-mon[193606]: failed to encode map e601 with expected crc 2023-09-23T23:37:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:27 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:37:27.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:27 smithi138 ceph-mon[193606]: osdmap e601: 8 total, 8 up, 7 in 2023-09-23T23:37:27.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:27 smithi138 ceph-mon[193606]: failed to encode map e601 with expected crc 2023-09-23T23:37:27.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:27 smithi110 ceph-mon[212939]: pgmap v1687: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 324 op/s; 23438/137148 objects degraded (17.090%); 2725/137148 objects misplaced (1.987%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T23:37:27.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:27 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:37:27.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:27 smithi110 ceph-mon[212939]: failed to encode map e601 with expected crc 2023-09-23T23:37:27.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:27 smithi110 ceph-mon[212939]: failed to encode map e601 with expected crc 2023-09-23T23:37:27.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:27 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:37:27.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:27 smithi110 ceph-mon[212939]: osdmap e601: 8 total, 8 up, 7 in 2023-09-23T23:37:27.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:27 smithi110 ceph-mon[212939]: failed to encode map e601 with expected crc 2023-09-23T23:37:27.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:27 smithi110 ceph-mon[221102]: pgmap v1687: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 324 op/s; 23438/137148 objects degraded (17.090%); 2725/137148 objects misplaced (1.987%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T23:37:27.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:27 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:37:27.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:27 smithi110 ceph-mon[221102]: failed to encode map e601 with expected crc 2023-09-23T23:37:27.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:27 smithi110 ceph-mon[221102]: failed to encode map e601 with expected crc 2023-09-23T23:37:27.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:27 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:37:27.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:27 smithi110 ceph-mon[221102]: osdmap e601: 8 total, 8 up, 7 in 2023-09-23T23:37:27.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:27 smithi110 ceph-mon[221102]: failed to encode map e601 with expected crc 2023-09-23T23:37:27.521 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:37:27.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:27.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:37:27.584 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:27.621 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:27.627 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:27.676 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:27.933 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:27.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:27.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:27.977 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:37:28.043 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:28.104 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:28.213 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:28.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:28.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:28.321 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:37:28.361 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:28.475 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:37:29.033 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:29 smithi138 ceph-mon[193606]: pgmap v1689: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 23438/137148 objects degraded (17.090%); 2725/137148 objects misplaced (1.987%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:37:29.033 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:29 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:29.033 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:29 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:37:29.034 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:29 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 23438/137148 objects degraded (17.090%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:29.198 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:37:29.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:29.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:37:29.303 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:29.339 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:29.392 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:29.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:29.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:29 smithi110 ceph-mon[212939]: pgmap v1689: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 23438/137148 objects degraded (17.090%); 2725/137148 objects misplaced (1.987%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:37:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:29 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:29.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:29 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:37:29.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:29 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 23438/137148 objects degraded (17.090%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:29.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:29 smithi110 ceph-mon[221102]: pgmap v1689: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 23438/137148 objects degraded (17.090%); 2725/137148 objects misplaced (1.987%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:37:29.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:29 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:29.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:29 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:37:29.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:29 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 23438/137148 objects degraded (17.090%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:29.496 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:29.530 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:29.537 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:37:29.751 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:29.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:29.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:29.855 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:37:29.900 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:30.136 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:37:30.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:30.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:37:30.210 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:37:30.210 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:37:30.210 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:37:30.210 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:37:30.210 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:37:30.244 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:30.322 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:30.365 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:37:30.401 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:37:30.429 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:37:30.459 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:37:30.460 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:30.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:30.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:30.497 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:37:30.498 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:30.521 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:37:30.559 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:37:30.568 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:30.585 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:37:30.586 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:37:30.586 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:37:30.586 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:37:30.587 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:37:30.587 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:37:30.587 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:37:30.587 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:37:30.587 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:37:30.597 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:30.641 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:30.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:30.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:30.745 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:30.801 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:30.918 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:37:30.924 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:37:31.079 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:31 smithi110 ceph-mon[212939]: pgmap v1690: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 23188/132453 objects degraded (17.507%); 2623/132453 objects misplaced (1.980%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:37:31.080 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:31 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/380279081' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:31.080 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:31 smithi110 ceph-mon[221102]: pgmap v1690: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 23188/132453 objects degraded (17.507%); 2623/132453 objects misplaced (1.980%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:37:31.080 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:31 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/380279081' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:31.090 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:37:31.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:31 smithi138 ceph-mon[193606]: pgmap v1690: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 23188/132453 objects degraded (17.507%); 2623/132453 objects misplaced (1.980%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:37:31.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:31 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/380279081' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:31.501 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:37:31.909 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:31.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:31.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:32.016 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:32.055 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:32.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:32 smithi110 ceph-mon[212939]: from='client.36942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:37:32.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:32 smithi110 ceph-mon[221102]: from='client.36942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:37:32.209 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:32.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:32.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:32.313 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:37:32.346 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:32 smithi138 ceph-mon[193606]: from='client.36942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:37:33.015 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:37:33.021 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1690,"stamp":"2023-09-23T23:37:29.941343+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2892477685,"num_objects":44151,"num_object_clones":0,"num_object_copies":132453,"num_objects_missing_on_primary":1728,"num_objects_missing":1728,"num_objects_degraded":23188,"num_objects_misplaced":2623,"num_objects_unfound":0,"num_objects_dirty":44151,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":131506,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17788,"num_bytes_recovered":1163063720,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95053,"ondisk_log_size":95053,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13409164,"kb_used_data":7689364,"kb_used_omap":29,"kb_used_meta":5719650,"kb_avail":642806900,"statfs":{"total":671965249536,"available":658234265600,"internally_reserved":0,"allocated":7873908736,"data_stored":7857499884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5856922323},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,3,2,1,3,2,1,4,5],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-232062976,"num_objects":-3541,"num_object_clones":0,"num_object_copies":-10623,"num_objects_missing_on_primary":-203,"num_objects_missing":-203,"num_objects_degraded":-452,"num_objects_misplaced":-203,"num_objects_unfound":0,"num_objects_dirty":-3541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3541,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":657,"num_bytes_recovered":43057152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002903"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":797,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.289752+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:37:18.289752+0000","last_peered":"2023-09-23T23:37:18.289752+0000","last_clean":"2023-09-23T23:37:18.289752+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:37:18.289752+0000","last_undegraded":"2023-09-23T23:37:18.289752+0000","last_fullsized":"2023-09-23T23:37:18.289752+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":355,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.272326+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:37:10.272326+0000","last_peered":"2023-09-23T23:37:10.272326+0000","last_clean":"2023-09-23T23:37:10.272326+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:37:10.272326+0000","last_undegraded":"2023-09-23T23:37:10.272326+0000","last_fullsized":"2023-09-23T23:37:10.272326+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":583,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.312971+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:37:18.312971+0000","last_peered":"2023-09-23T23:37:18.312971+0000","last_clean":"2023-09-23T23:37:18.312971+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:37:18.312971+0000","last_undegraded":"2023-09-23T23:37:18.312971+0000","last_fullsized":"2023-09-23T23:37:18.312971+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":750,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.292834+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:37:18.292834+0000","last_peered":"2023-09-23T23:37:18.292834+0000","last_clean":"2023-09-23T23:37:18.292834+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:37:18.292834+0000","last_undegraded":"2023-09-23T23:37:18.292834+0000","last_fullsized":"2023-09-23T23:37:18.292834+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":320,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.291202+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:37:18.291202+0000","last_peered":"2023-09-23T23:37:18.291202+0000","last_clean":"2023-09-23T23:37:18.291202+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:37:18.291202+0000","last_undegraded":"2023-09-23T23:37:18.291202+0000","last_fullsized":"2023-09-23T23:37:18.291202+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":562,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.288956+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:37:18.288956+0000","last_peered":"2023-09-23T23:37:18.288956+0000","last_clean":"2023-09-23T23:37:18.288956+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:37:18.288956+0000","last_undegraded":"2023-09-23T23:37:18.288956+0000","last_fullsized":"2023-09-23T23:37:18.288956+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":762,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.281791+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:37:18.281791+0000","last_peered":"2023-09-23T23:37:18.281791+0000","last_clean":"2023-09-23T23:37:18.281791+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:37:18.281791+0000","last_undegraded":"2023-09-23T23:37:18.281791+0000","last_fullsized":"2023-09-23T23:37:18.281791+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":619,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.287694+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:37:18.287694+0000","last_peered":"2023-09-23T23:37:18.287694+0000","last_clean":"2023-09-23T23:37:18.287694+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:37:18.287694+0000","last_undegraded":"2023-09-23T23:37:18.287694+0000","last_fullsized":"2023-09-23T23:37:18.287694+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":115,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283996+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:37:18.283996+0000","last_peered":"2023-09-23T23:37:18.283996+0000","last_clean":"2023-09-23T23:37:18.283996+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:37:18.283996+0000","last_undegraded":"2023-09-23T23:37:18.283996+0000","last_fullsized":"2023-09-23T23:37:18.283996+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":921,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.281571+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:37:18.281571+0000","last_peered":"2023-09-23T23:37:18.281571+0000","last_clean":"2023-09-23T23:37:18.281571+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:37:18.281571+0000","last_undegraded":"2023-09-23T23:37:18.281571+0000","last_fullsized":"2023-09-23T23:37:18.281571+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":253,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290568+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:37:18.290568+0000","last_peered":"2023-09-23T23:37:18.290568+0000","last_clean":"2023-09-23T23:37:18.290568+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:37:18.290568+0000","last_undegraded":"2023-09-23T23:37:18.290568+0000","last_fullsized":"2023-09-23T23:37:18.290568+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":602,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.287609+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:37:18.287609+0000","last_peered":"2023-09-23T23:37:18.287609+0000","last_clean":"2023-09-23T23:37:18.287609+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:37:18.287609+0000","last_undegraded":"2023-09-23T23:37:18.287609+0000","last_fullsized":"2023-09-23T23:37:18.287609+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":893,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.275873+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:37:10.275873+0000","last_peered":"2023-09-23T23:37:10.275873+0000","last_clean":"2023-09-23T23:37:10.275873+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:37:10.275873+0000","last_undegraded":"2023-09-23T23:37:10.275873+0000","last_fullsized":"2023-09-23T23:37:10.275873+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":232,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.291541+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:37:18.291541+0000","last_peered":"2023-09-23T23:37:18.291541+0000","last_clean":"2023-09-23T23:37:18.291541+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:37:18.291541+0000","last_undegraded":"2023-09-23T23:37:18.291541+0000","last_fullsized":"2023-09-23T23:37:18.291541+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":563,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.291023+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:37:18.291023+0000","last_peered":"2023-09-23T23:37:18.291023+0000","last_clean":"2023-09-23T23:37:18.291023+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:37:18.291023+0000","last_undegraded":"2023-09-23T23:37:18.291023+0000","last_fullsized":"2023-09-23T23:37:18.291023+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1055,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290495+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:37:18.290495+0000","last_peered":"2023-09-23T23:37:18.290495+0000","last_clean":"2023-09-23T23:37:18.290495+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:37:18.290495+0000","last_undegraded":"2023-09-23T23:37:18.290495+0000","last_fullsized":"2023-09-23T23:37:18.290495+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":274,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.303655+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:37:18.303655+0000","last_peered":"2023-09-23T23:37:18.303655+0000","last_clean":"2023-09-23T23:37:18.303655+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:37:18.303655+0000","last_undegraded":"2023-09-23T23:37:18.303655+0000","last_fullsized":"2023-09-23T23:37:18.303655+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":134,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283969+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:37:18.283969+0000","last_peered":"2023-09-23T23:37:18.283969+0000","last_clean":"2023-09-23T23:37:18.283969+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:37:18.283969+0000","last_undegraded":"2023-09-23T23:37:18.283969+0000","last_fullsized":"2023-09-23T23:37:18.283969+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":452,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.289085+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:37:18.289085+0000","last_peered":"2023-09-23T23:37:18.289085+0000","last_clean":"2023-09-23T23:37:18.289085+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:37:18.289085+0000","last_undegraded":"2023-09-23T23:37:18.289085+0000","last_fullsized":"2023-09-23T23:37:18.289085+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":175,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283680+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:37:18.283680+0000","last_peered":"2023-09-23T23:37:18.283680+0000","last_clean":"2023-09-23T23:37:18.283680+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:37:18.283680+0000","last_undegraded":"2023-09-23T23:37:18.283680+0000","last_fullsized":"2023-09-23T23:37:18.283680+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":542,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283606+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:37:18.283606+0000","last_peered":"2023-09-23T23:37:18.283606+0000","last_clean":"2023-09-23T23:37:18.283606+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:37:18.283606+0000","last_undegraded":"2023-09-23T23:37:18.283606+0000","last_fullsized":"2023-09-23T23:37:18.283606+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":756,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.289973+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:37:18.289973+0000","last_peered":"2023-09-23T23:37:18.289973+0000","last_clean":"2023-09-23T23:37:18.289973+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:37:18.289973+0000","last_undegraded":"2023-09-23T23:37:18.289973+0000","last_fullsized":"2023-09-23T23:37:18.289973+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":147,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.272776+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:37:10.272776+0000","last_peered":"2023-09-23T23:37:10.272776+0000","last_clean":"2023-09-23T23:37:10.272776+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:37:10.272776+0000","last_undegraded":"2023-09-23T23:37:10.272776+0000","last_fullsized":"2023-09-23T23:37:10.272776+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":601,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.288033+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:37:18.288033+0000","last_peered":"2023-09-23T23:37:18.288033+0000","last_clean":"2023-09-23T23:37:18.288033+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:37:18.288033+0000","last_undegraded":"2023-09-23T23:37:18.288033+0000","last_fullsized":"2023-09-23T23:37:18.288033+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":886,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283181+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:37:18.283181+0000","last_peered":"2023-09-23T23:37:18.283181+0000","last_clean":"2023-09-23T23:37:18.283181+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:37:18.283181+0000","last_undegraded":"2023-09-23T23:37:18.283181+0000","last_fullsized":"2023-09-23T23:37:18.283181+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":196,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.293323+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:37:18.293323+0000","last_peered":"2023-09-23T23:37:18.293323+0000","last_clean":"2023-09-23T23:37:18.293323+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:37:18.293323+0000","last_undegraded":"2023-09-23T23:37:18.293323+0000","last_fullsized":"2023-09-23T23:37:18.293323+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":323,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.312860+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:37:18.312860+0000","last_peered":"2023-09-23T23:37:18.312860+0000","last_clean":"2023-09-23T23:37:18.312860+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:37:18.312860+0000","last_undegraded":"2023-09-23T23:37:18.312860+0000","last_fullsized":"2023-09-23T23:37:18.312860+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":564,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.291021+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:37:18.291021+0000","last_peered":"2023-09-23T23:37:18.291021+0000","last_clean":"2023-09-23T23:37:18.291021+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:37:18.291021+0000","last_undegraded":"2023-09-23T23:37:18.291021+0000","last_fullsized":"2023-09-23T23:37:18.291021+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":469,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283257+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:37:18.283257+0000","last_peered":"2023-09-23T23:37:18.283257+0000","last_clean":"2023-09-23T23:37:18.283257+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:37:18.283257+0000","last_undegraded":"2023-09-23T23:37:18.283257+0000","last_fullsized":"2023-09-23T23:37:18.283257+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":557,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.289226+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:37:18.289226+0000","last_peered":"2023-09-23T23:37:18.289226+0000","last_clean":"2023-09-23T23:37:18.289226+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:37:18.289226+0000","last_undegraded":"2023-09-23T23:37:18.289226+0000","last_fullsized":"2023-09-23T23:37:18.289226+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":964,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290395+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:37:18.290395+0000","last_peered":"2023-09-23T23:37:18.290395+0000","last_clean":"2023-09-23T23:37:18.290395+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:37:18.290395+0000","last_undegraded":"2023-09-23T23:37:18.290395+0000","last_fullsized":"2023-09-23T23:37:18.290395+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":165,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283571+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:37:18.283571+0000","last_peered":"2023-09-23T23:37:18.283571+0000","last_clean":"2023-09-23T23:37:18.283571+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:37:18.283571+0000","last_undegraded":"2023-09-23T23:37:18.283571+0000","last_fullsized":"2023-09-23T23:37:18.283571+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":242,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.288498+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:37:18.288498+0000","last_peered":"2023-09-23T23:37:18.288498+0000","last_clean":"2023-09-23T23:37:18.288498+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:37:18.288498+0000","last_undegraded":"2023-09-23T23:37:18.288498+0000","last_fullsized":"2023-09-23T23:37:18.288498+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"601'8135","reported_seq":8219,"reported_epoch":601,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:29.185298+0000","last_change":"2023-09-23T23:33:52.884769+0000","last_active":"2023-09-23T23:37:29.185298+0000","last_peered":"2023-09-23T23:37:29.185298+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:37:29.185298+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"551'2700","ondisk_log_start":"551'2700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5435,"log_dups_size":0,"ondisk_log_size":5435,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177799168,"num_objects":2713,"num_object_clones":0,"num_object_copies":8139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8135,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":1687},{"shards":"2,6","objects":-1685}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":934,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.272341+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:37:10.272341+0000","last_peered":"2023-09-23T23:37:10.272341+0000","last_clean":"2023-09-23T23:37:10.272341+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:37:10.272341+0000","last_undegraded":"2023-09-23T23:37:10.272341+0000","last_fullsized":"2023-09-23T23:37:10.272341+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":242,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.281540+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:37:18.281540+0000","last_peered":"2023-09-23T23:37:18.281540+0000","last_clean":"2023-09-23T23:37:18.281540+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:37:18.281540+0000","last_undegraded":"2023-09-23T23:37:18.281540+0000","last_fullsized":"2023-09-23T23:37:18.281540+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":232,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.289673+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:37:18.289673+0000","last_peered":"2023-09-23T23:37:18.289673+0000","last_clean":"2023-09-23T23:37:18.289673+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:37:18.289673+0000","last_undegraded":"2023-09-23T23:37:18.289673+0000","last_fullsized":"2023-09-23T23:37:18.289673+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"600'8151","reported_seq":8234,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:24.785403+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:37:24.785403+0000","last_peered":"2023-09-23T23:37:24.785403+0000","last_clean":"2023-09-23T23:37:24.785403+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:37:24.785403+0000","last_undegraded":"2023-09-23T23:37:24.785403+0000","last_fullsized":"2023-09-23T23:37:24.785403+0000","mapping_epoch":566,"log_start":"549'2000","ondisk_log_start":"549'2000","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6151,"log_dups_size":0,"ondisk_log_size":6151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188809216,"num_objects":2881,"num_object_clones":0,"num_object_copies":8643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8151,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1013,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.287758+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:37:18.287758+0000","last_peered":"2023-09-23T23:37:18.287758+0000","last_clean":"2023-09-23T23:37:18.287758+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:37:18.287758+0000","last_undegraded":"2023-09-23T23:37:18.287758+0000","last_fullsized":"2023-09-23T23:37:18.287758+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":232,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.282295+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:37:18.282295+0000","last_peered":"2023-09-23T23:37:18.282295+0000","last_clean":"2023-09-23T23:37:18.282295+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:37:18.282295+0000","last_undegraded":"2023-09-23T23:37:18.282295+0000","last_fullsized":"2023-09-23T23:37:18.282295+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":452,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290662+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:37:18.290662+0000","last_peered":"2023-09-23T23:37:18.290662+0000","last_clean":"2023-09-23T23:37:18.290662+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:37:18.290662+0000","last_undegraded":"2023-09-23T23:37:18.290662+0000","last_fullsized":"2023-09-23T23:37:18.290662+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"601'8298","reported_seq":14965,"reported_epoch":601,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:37:28.061837+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:37:28.061837+0000","last_peered":"2023-09-23T23:37:28.061837+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:37:28.061837+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:37:28.061837+0000","mapping_epoch":571,"log_start":"551'3000","ondisk_log_start":"551'3000","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5298,"log_dups_size":0,"ondisk_log_size":5298,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180224000,"num_objects":2750,"num_object_clones":0,"num_object_copies":8250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1855,"num_objects_misplaced":895,"num_objects_unfound":0,"num_objects_dirty":2750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8298,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3276,"num_bytes_recovered":214695936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":1733},{"shards":"3,5,7","objects":1017}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":911,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.284071+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:37:18.284071+0000","last_peered":"2023-09-23T23:37:18.284071+0000","last_clean":"2023-09-23T23:37:18.284071+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:37:18.284071+0000","last_undegraded":"2023-09-23T23:37:18.284071+0000","last_fullsized":"2023-09-23T23:37:18.284071+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":339,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.291692+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:37:18.291692+0000","last_peered":"2023-09-23T23:37:18.291692+0000","last_clean":"2023-09-23T23:37:18.291692+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:37:18.291692+0000","last_undegraded":"2023-09-23T23:37:18.291692+0000","last_fullsized":"2023-09-23T23:37:18.291692+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":115,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.284208+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:37:18.284208+0000","last_peered":"2023-09-23T23:37:18.284208+0000","last_clean":"2023-09-23T23:37:18.284208+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:37:18.284208+0000","last_undegraded":"2023-09-23T23:37:18.284208+0000","last_fullsized":"2023-09-23T23:37:18.284208+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"601'8275","reported_seq":10346,"reported_epoch":601,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:28.027131+0000","last_change":"2023-09-23T23:34:35.750428+0000","last_active":"2023-09-23T23:37:28.027131+0000","last_peered":"2023-09-23T23:37:28.027131+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:37:28.027131+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"551'3000","ondisk_log_start":"551'3000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5275,"log_dups_size":0,"ondisk_log_size":5275,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184483840,"num_objects":2815,"num_object_clones":0,"num_object_copies":8445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8275,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":991,"num_bytes_recovered":64946176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":1805},{"shards":"3,5","objects":-729}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":508,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.289159+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:37:18.289159+0000","last_peered":"2023-09-23T23:37:18.289159+0000","last_clean":"2023-09-23T23:37:18.289159+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:37:18.289159+0000","last_undegraded":"2023-09-23T23:37:18.289159+0000","last_fullsized":"2023-09-23T23:37:18.289159+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":442,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.287917+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:37:18.287917+0000","last_peered":"2023-09-23T23:37:18.287917+0000","last_clean":"2023-09-23T23:37:18.287917+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:37:18.287917+0000","last_undegraded":"2023-09-23T23:37:18.287917+0000","last_fullsized":"2023-09-23T23:37:18.287917+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":597,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.295471+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:37:10.295471+0000","last_peered":"2023-09-23T23:37:10.295471+0000","last_clean":"2023-09-23T23:37:10.295471+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:37:10.295471+0000","last_undegraded":"2023-09-23T23:37:10.295471+0000","last_fullsized":"2023-09-23T23:37:10.295471+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"601'8423","reported_seq":9754,"reported_epoch":601,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:29.455020+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:37:29.455020+0000","last_peered":"2023-09-23T23:37:29.455020+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:37:29.455020+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"551'2700","ondisk_log_start":"551'2700","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5723,"log_dups_size":0,"ondisk_log_size":5723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179503104,"num_objects":2739,"num_object_clones":0,"num_object_copies":8217,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2739,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8423,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":1693},{"shards":"3,5","objects":-2721},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":729,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290954+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:37:18.290954+0000","last_peered":"2023-09-23T23:37:18.290954+0000","last_clean":"2023-09-23T23:37:18.290954+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:37:18.290954+0000","last_undegraded":"2023-09-23T23:37:18.290954+0000","last_fullsized":"2023-09-23T23:37:18.290954+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":611,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290937+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:37:18.290937+0000","last_peered":"2023-09-23T23:37:18.290937+0000","last_clean":"2023-09-23T23:37:18.290937+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:37:18.290937+0000","last_undegraded":"2023-09-23T23:37:18.290937+0000","last_fullsized":"2023-09-23T23:37:18.290937+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":522,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.287839+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:37:18.287839+0000","last_peered":"2023-09-23T23:37:18.287839+0000","last_clean":"2023-09-23T23:37:18.287839+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:37:18.287839+0000","last_undegraded":"2023-09-23T23:37:18.287839+0000","last_fullsized":"2023-09-23T23:37:18.287839+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"601'8344","reported_seq":8419,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:29.194132+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:37:29.194132+0000","last_peered":"2023-09-23T23:37:29.194132+0000","last_clean":"2023-09-23T23:37:29.194132+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:37:29.194132+0000","last_undegraded":"2023-09-23T23:37:29.194132+0000","last_fullsized":"2023-09-23T23:37:29.194132+0000","mapping_epoch":534,"log_start":"551'2900","ondisk_log_start":"551'2900","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5444,"log_dups_size":0,"ondisk_log_size":5444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179306496,"num_objects":2736,"num_object_clones":0,"num_object_copies":8208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8344,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":883,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.272123+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:37:10.272123+0000","last_peered":"2023-09-23T23:37:10.272123+0000","last_clean":"2023-09-23T23:37:10.272123+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:37:10.272123+0000","last_undegraded":"2023-09-23T23:37:10.272123+0000","last_fullsized":"2023-09-23T23:37:10.272123+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":125,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.284125+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:37:18.284125+0000","last_peered":"2023-09-23T23:37:18.284125+0000","last_clean":"2023-09-23T23:37:18.284125+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:37:18.284125+0000","last_undegraded":"2023-09-23T23:37:18.284125+0000","last_fullsized":"2023-09-23T23:37:18.284125+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":466,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.272530+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:37:10.272530+0000","last_peered":"2023-09-23T23:37:10.272530+0000","last_clean":"2023-09-23T23:37:10.272530+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:37:10.272530+0000","last_undegraded":"2023-09-23T23:37:10.272530+0000","last_fullsized":"2023-09-23T23:37:10.272530+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"601'8382","reported_seq":14137,"reported_epoch":601,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:37:29.177475+0000","last_change":"2023-09-23T23:34:35.785270+0000","last_active":"2023-09-23T23:37:29.177475+0000","last_peered":"2023-09-23T23:37:29.177475+0000","last_clean":"2023-09-23T23:33:50.504296+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:37:29.177475+0000","last_undegraded":"2023-09-23T23:34:35.780981+0000","last_fullsized":"2023-09-23T23:37:29.177475+0000","mapping_epoch":567,"log_start":"551'3000","ondisk_log_start":"551'3000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5382,"log_dups_size":0,"ondisk_log_size":5382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178388992,"num_objects":2722,"num_object_clones":0,"num_object_copies":8166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1717,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2722,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8382,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2830,"num_bytes_recovered":185466880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":1717},{"shards":"5,6,7","objects":1005}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":609,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.292749+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:37:18.292749+0000","last_peered":"2023-09-23T23:37:18.292749+0000","last_clean":"2023-09-23T23:37:18.292749+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:37:18.292749+0000","last_undegraded":"2023-09-23T23:37:18.292749+0000","last_fullsized":"2023-09-23T23:37:18.292749+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":463,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.276018+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:37:10.276018+0000","last_peered":"2023-09-23T23:37:10.276018+0000","last_clean":"2023-09-23T23:37:10.276018+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:37:10.276018+0000","last_undegraded":"2023-09-23T23:37:10.276018+0000","last_fullsized":"2023-09-23T23:37:10.276018+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":552,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.285809+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:37:18.285809+0000","last_peered":"2023-09-23T23:37:18.285809+0000","last_clean":"2023-09-23T23:37:18.285809+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:37:18.285809+0000","last_undegraded":"2023-09-23T23:37:18.285809+0000","last_fullsized":"2023-09-23T23:37:18.285809+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"601'8292","reported_seq":17330,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:29.359812+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:37:29.359812+0000","last_peered":"2023-09-23T23:37:29.359812+0000","last_clean":"2023-09-23T23:37:29.359812+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:37:29.359812+0000","last_undegraded":"2023-09-23T23:37:29.359812+0000","last_fullsized":"2023-09-23T23:37:29.359812+0000","mapping_epoch":579,"log_start":"551'2600","ondisk_log_start":"551'2600","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5692,"log_dups_size":0,"ondisk_log_size":5692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178520064,"num_objects":2724,"num_object_clones":0,"num_object_copies":8172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8292,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":942,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.272232+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:37:10.272232+0000","last_peered":"2023-09-23T23:37:10.272232+0000","last_clean":"2023-09-23T23:37:10.272232+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:37:10.272232+0000","last_undegraded":"2023-09-23T23:37:10.272232+0000","last_fullsized":"2023-09-23T23:37:10.272232+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":243,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.287508+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:37:18.287508+0000","last_peered":"2023-09-23T23:37:18.287508+0000","last_clean":"2023-09-23T23:37:18.287508+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:37:18.287508+0000","last_undegraded":"2023-09-23T23:37:18.287508+0000","last_fullsized":"2023-09-23T23:37:18.287508+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":531,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.288728+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:37:18.288728+0000","last_peered":"2023-09-23T23:37:18.288728+0000","last_clean":"2023-09-23T23:37:18.288728+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:37:18.288728+0000","last_undegraded":"2023-09-23T23:37:18.288728+0000","last_fullsized":"2023-09-23T23:37:18.288728+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"601'7955","reported_seq":8033,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:28.071037+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:37:28.071037+0000","last_peered":"2023-09-23T23:37:28.071037+0000","last_clean":"2023-09-23T23:37:28.071037+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:37:28.071037+0000","last_undegraded":"2023-09-23T23:37:28.071037+0000","last_fullsized":"2023-09-23T23:37:28.071037+0000","mapping_epoch":534,"log_start":"551'2600","ondisk_log_start":"551'2600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5355,"log_dups_size":0,"ondisk_log_size":5355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181731328,"num_objects":2773,"num_object_clones":0,"num_object_copies":8319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7955,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":922,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283904+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:37:18.283904+0000","last_peered":"2023-09-23T23:37:18.283904+0000","last_clean":"2023-09-23T23:37:18.283904+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:37:18.283904+0000","last_undegraded":"2023-09-23T23:37:18.283904+0000","last_fullsized":"2023-09-23T23:37:18.283904+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":311,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.277973+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:37:18.277973+0000","last_peered":"2023-09-23T23:37:18.277973+0000","last_clean":"2023-09-23T23:37:18.277973+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:37:18.277973+0000","last_undegraded":"2023-09-23T23:37:18.277973+0000","last_fullsized":"2023-09-23T23:37:18.277973+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":616,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.291785+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:37:18.291785+0000","last_peered":"2023-09-23T23:37:18.291785+0000","last_clean":"2023-09-23T23:37:18.291785+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:37:18.291785+0000","last_undegraded":"2023-09-23T23:37:18.291785+0000","last_fullsized":"2023-09-23T23:37:18.291785+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":197,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290247+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:37:18.290247+0000","last_peered":"2023-09-23T23:37:18.290247+0000","last_clean":"2023-09-23T23:37:18.290247+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:37:18.290247+0000","last_undegraded":"2023-09-23T23:37:18.290247+0000","last_fullsized":"2023-09-23T23:37:18.290247+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"601'8364","reported_seq":14028,"reported_epoch":601,"state":"active+recovering","last_fresh":"2023-09-23T23:37:28.080425+0000","last_change":"2023-09-23T23:34:35.815566+0000","last_active":"2023-09-23T23:37:28.080425+0000","last_peered":"2023-09-23T23:37:28.080425+0000","last_clean":"2023-09-23T23:33:50.519308+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:37:28.080425+0000","last_undegraded":"2023-09-23T23:37:28.080425+0000","last_fullsized":"2023-09-23T23:37:28.080425+0000","mapping_epoch":566,"log_start":"551'3022","ondisk_log_start":"551'3022","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5342,"log_dups_size":0,"ondisk_log_size":5342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183697432,"num_objects":2804,"num_object_clones":0,"num_object_copies":8412,"num_objects_missing_on_primary":1728,"num_objects_missing":1728,"num_objects_degraded":0,"num_objects_misplaced":1728,"num_objects_unfound":0,"num_objects_dirty":2804,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8364,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2779,"num_bytes_recovered":182124544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,3,4,7","objects":8},{"shards":"1,3,7","objects":1068},{"shards":"1,4,7","objects":1728}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":684,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.294821+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:37:10.294821+0000","last_peered":"2023-09-23T23:37:10.294821+0000","last_clean":"2023-09-23T23:37:10.294821+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:37:10.294821+0000","last_undegraded":"2023-09-23T23:37:10.294821+0000","last_fullsized":"2023-09-23T23:37:10.294821+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":249,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.284045+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:37:18.284045+0000","last_peered":"2023-09-23T23:37:18.284045+0000","last_clean":"2023-09-23T23:37:18.284045+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:37:18.284045+0000","last_undegraded":"2023-09-23T23:37:18.284045+0000","last_fullsized":"2023-09-23T23:37:18.284045+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":533,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283689+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:37:18.283689+0000","last_peered":"2023-09-23T23:37:18.283689+0000","last_clean":"2023-09-23T23:37:18.283689+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:37:18.283689+0000","last_undegraded":"2023-09-23T23:37:18.283689+0000","last_fullsized":"2023-09-23T23:37:18.283689+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":233,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.275884+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:37:10.275884+0000","last_peered":"2023-09-23T23:37:10.275884+0000","last_clean":"2023-09-23T23:37:10.275884+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:37:10.275884+0000","last_undegraded":"2023-09-23T23:37:10.275884+0000","last_fullsized":"2023-09-23T23:37:10.275884+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"601'8044","reported_seq":10098,"reported_epoch":601,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:29.209160+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:37:29.209160+0000","last_peered":"2023-09-23T23:37:29.209160+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:37:29.209160+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"550'2300","ondisk_log_start":"550'2300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5744,"log_dups_size":0,"ondisk_log_size":5744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178782208,"num_objects":2728,"num_object_clones":0,"num_object_copies":8184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8044,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":1717},{"shards":"5,7","objects":-664}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":902,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.272167+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:37:10.272167+0000","last_peered":"2023-09-23T23:37:10.272167+0000","last_clean":"2023-09-23T23:37:10.272167+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:37:10.272167+0000","last_undegraded":"2023-09-23T23:37:10.272167+0000","last_fullsized":"2023-09-23T23:37:10.272167+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":339,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.272029+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:37:10.272029+0000","last_peered":"2023-09-23T23:37:10.272029+0000","last_clean":"2023-09-23T23:37:10.272029+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:37:10.272029+0000","last_undegraded":"2023-09-23T23:37:10.272029+0000","last_fullsized":"2023-09-23T23:37:10.272029+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":248,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.272173+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:37:10.272173+0000","last_peered":"2023-09-23T23:37:10.272173+0000","last_clean":"2023-09-23T23:37:10.272173+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:37:10.272173+0000","last_undegraded":"2023-09-23T23:37:10.272173+0000","last_fullsized":"2023-09-23T23:37:10.272173+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":213,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.277475+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:37:18.277475+0000","last_peered":"2023-09-23T23:37:18.277475+0000","last_clean":"2023-09-23T23:37:18.277475+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:37:18.277475+0000","last_undegraded":"2023-09-23T23:37:18.277475+0000","last_fullsized":"2023-09-23T23:37:18.277475+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"601'8261","reported_seq":8346,"reported_epoch":601,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:37:29.488544+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:37:29.488544+0000","last_peered":"2023-09-23T23:37:29.488544+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:37:29.488544+0000","last_undegraded":"2023-09-23T23:37:29.488544+0000","last_fullsized":"2023-09-23T23:37:29.488544+0000","mapping_epoch":562,"log_start":"551'2600","ondisk_log_start":"551'2600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5661,"log_dups_size":0,"ondisk_log_size":5661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173080576,"num_objects":2641,"num_object_clones":0,"num_object_copies":7923,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2641,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8261,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":2641}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":615,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.272848+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:37:10.272848+0000","last_peered":"2023-09-23T23:37:10.272848+0000","last_clean":"2023-09-23T23:37:10.272848+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:37:10.272848+0000","last_undegraded":"2023-09-23T23:37:10.272848+0000","last_fullsized":"2023-09-23T23:37:10.272848+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":887,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283331+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:37:18.283331+0000","last_peered":"2023-09-23T23:37:18.283331+0000","last_clean":"2023-09-23T23:37:18.283331+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:37:18.283331+0000","last_undegraded":"2023-09-23T23:37:18.283331+0000","last_fullsized":"2023-09-23T23:37:18.283331+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":563,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.293189+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:37:18.293189+0000","last_peered":"2023-09-23T23:37:18.293189+0000","last_clean":"2023-09-23T23:37:18.293189+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:37:18.293189+0000","last_undegraded":"2023-09-23T23:37:18.293189+0000","last_fullsized":"2023-09-23T23:37:18.293189+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":125,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.285739+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:37:18.285739+0000","last_peered":"2023-09-23T23:37:18.285739+0000","last_clean":"2023-09-23T23:37:18.285739+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:37:18.285739+0000","last_undegraded":"2023-09-23T23:37:18.285739+0000","last_fullsized":"2023-09-23T23:37:18.285739+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"601'8015","reported_seq":8090,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:29.216377+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:37:29.216377+0000","last_peered":"2023-09-23T23:37:29.216377+0000","last_clean":"2023-09-23T23:37:29.216377+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:37:29.216377+0000","last_undegraded":"2023-09-23T23:37:29.216377+0000","last_fullsized":"2023-09-23T23:37:29.216377+0000","mapping_epoch":534,"log_start":"551'2600","ondisk_log_start":"551'2600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5415,"log_dups_size":0,"ondisk_log_size":5415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178323456,"num_objects":2721,"num_object_clones":0,"num_object_copies":8163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8015,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":713,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283523+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:37:18.283523+0000","last_peered":"2023-09-23T23:37:18.283523+0000","last_clean":"2023-09-23T23:37:18.283523+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:37:18.283523+0000","last_undegraded":"2023-09-23T23:37:18.283523+0000","last_fullsized":"2023-09-23T23:37:18.283523+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":528,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.280933+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:37:18.280933+0000","last_peered":"2023-09-23T23:37:18.280933+0000","last_clean":"2023-09-23T23:37:18.280933+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:37:18.280933+0000","last_undegraded":"2023-09-23T23:37:18.280933+0000","last_fullsized":"2023-09-23T23:37:18.280933+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":442,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290744+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:37:18.290744+0000","last_peered":"2023-09-23T23:37:18.290744+0000","last_clean":"2023-09-23T23:37:18.290744+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:37:18.290744+0000","last_undegraded":"2023-09-23T23:37:18.290744+0000","last_fullsized":"2023-09-23T23:37:18.290744+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":41,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.295288+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:37:10.295288+0000","last_peered":"2023-09-23T23:37:10.295288+0000","last_clean":"2023-09-23T23:37:10.295288+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:37:10.295288+0000","last_undegraded":"2023-09-23T23:37:10.295288+0000","last_fullsized":"2023-09-23T23:37:10.295288+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"600'8181","reported_seq":11855,"reported_epoch":600,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:24.929330+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:37:24.929330+0000","last_peered":"2023-09-23T23:37:24.929330+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:37:24.929330+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"549'2000","ondisk_log_start":"549'2000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6181,"log_dups_size":0,"ondisk_log_size":6181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184614912,"num_objects":2817,"num_object_clones":0,"num_object_copies":8451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8181,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":113},{"shards":"1,2,4","objects":1815}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":596,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.286019+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:37:18.286019+0000","last_peered":"2023-09-23T23:37:18.286019+0000","last_clean":"2023-09-23T23:37:18.286019+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:37:18.286019+0000","last_undegraded":"2023-09-23T23:37:18.286019+0000","last_fullsized":"2023-09-23T23:37:18.286019+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":41,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.295557+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:37:10.295557+0000","last_peered":"2023-09-23T23:37:10.295557+0000","last_clean":"2023-09-23T23:37:10.295557+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:37:10.295557+0000","last_undegraded":"2023-09-23T23:37:10.295557+0000","last_fullsized":"2023-09-23T23:37:10.295557+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":140,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.275955+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:37:10.275955+0000","last_peered":"2023-09-23T23:37:10.275955+0000","last_clean":"2023-09-23T23:37:10.275955+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:37:10.275955+0000","last_undegraded":"2023-09-23T23:37:10.275955+0000","last_fullsized":"2023-09-23T23:37:10.275955+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":334,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.292967+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:37:18.292967+0000","last_peered":"2023-09-23T23:37:18.292967+0000","last_clean":"2023-09-23T23:37:18.292967+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:37:18.292967+0000","last_undegraded":"2023-09-23T23:37:18.292967+0000","last_fullsized":"2023-09-23T23:37:18.292967+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"601'8248","reported_seq":8323,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:29.455149+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:37:29.455149+0000","last_peered":"2023-09-23T23:37:29.455149+0000","last_clean":"2023-09-23T23:37:29.455149+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:37:29.455149+0000","last_undegraded":"2023-09-23T23:37:29.455149+0000","last_fullsized":"2023-09-23T23:37:29.455149+0000","mapping_epoch":534,"log_start":"550'2500","ondisk_log_start":"550'2500","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5748,"log_dups_size":0,"ondisk_log_size":5748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177995776,"num_objects":2716,"num_object_clones":0,"num_object_copies":8148,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2716,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8248,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":781,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.272619+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:37:10.272619+0000","last_peered":"2023-09-23T23:37:10.272619+0000","last_clean":"2023-09-23T23:37:10.272619+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:37:10.272619+0000","last_undegraded":"2023-09-23T23:37:10.272619+0000","last_fullsized":"2023-09-23T23:37:10.272619+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":599,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.288656+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:37:18.288656+0000","last_peered":"2023-09-23T23:37:18.288656+0000","last_clean":"2023-09-23T23:37:18.288656+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:37:18.288656+0000","last_undegraded":"2023-09-23T23:37:18.288656+0000","last_fullsized":"2023-09-23T23:37:18.288656+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":205,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.285866+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:37:18.285866+0000","last_peered":"2023-09-23T23:37:18.285866+0000","last_clean":"2023-09-23T23:37:18.285866+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:37:18.285866+0000","last_undegraded":"2023-09-23T23:37:18.285866+0000","last_fullsized":"2023-09-23T23:37:18.285866+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":152,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.272009+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:37:10.272009+0000","last_peered":"2023-09-23T23:37:10.272009+0000","last_clean":"2023-09-23T23:37:10.272009+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:37:10.272009+0000","last_undegraded":"2023-09-23T23:37:10.272009+0000","last_fullsized":"2023-09-23T23:37:10.272009+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"600'8061","reported_seq":8138,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:24.834833+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:37:24.834833+0000","last_peered":"2023-09-23T23:37:24.834833+0000","last_clean":"2023-09-23T23:37:24.834833+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:37:24.834833+0000","last_undegraded":"2023-09-23T23:37:24.834833+0000","last_fullsized":"2023-09-23T23:37:24.834833+0000","mapping_epoch":534,"log_start":"549'1900","ondisk_log_start":"549'1900","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6161,"log_dups_size":0,"ondisk_log_size":6161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186580992,"num_objects":2847,"num_object_clones":0,"num_object_copies":8541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8061,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":988,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.288160+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:37:18.288160+0000","last_peered":"2023-09-23T23:37:18.288160+0000","last_clean":"2023-09-23T23:37:18.288160+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:37:18.288160+0000","last_undegraded":"2023-09-23T23:37:18.288160+0000","last_fullsized":"2023-09-23T23:37:18.288160+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":115,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283930+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:37:18.283930+0000","last_peered":"2023-09-23T23:37:18.283930+0000","last_clean":"2023-09-23T23:37:18.283930+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:37:18.283930+0000","last_undegraded":"2023-09-23T23:37:18.283930+0000","last_fullsized":"2023-09-23T23:37:18.283930+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":45,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.295302+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:37:10.295302+0000","last_peered":"2023-09-23T23:37:10.295302+0000","last_clean":"2023-09-23T23:37:10.295302+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:37:10.295302+0000","last_undegraded":"2023-09-23T23:37:10.295302+0000","last_fullsized":"2023-09-23T23:37:10.295302+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":175,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290045+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:37:18.290045+0000","last_peered":"2023-09-23T23:37:18.290045+0000","last_clean":"2023-09-23T23:37:18.290045+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:37:18.290045+0000","last_undegraded":"2023-09-23T23:37:18.290045+0000","last_fullsized":"2023-09-23T23:37:18.290045+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1037,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290879+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:37:18.290879+0000","last_peered":"2023-09-23T23:37:18.290879+0000","last_clean":"2023-09-23T23:37:18.290879+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:37:18.290879+0000","last_undegraded":"2023-09-23T23:37:18.290879+0000","last_fullsized":"2023-09-23T23:37:18.290879+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":41,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.295488+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:37:10.295488+0000","last_peered":"2023-09-23T23:37:10.295488+0000","last_clean":"2023-09-23T23:37:10.295488+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:37:10.295488+0000","last_undegraded":"2023-09-23T23:37:10.295488+0000","last_fullsized":"2023-09-23T23:37:10.295488+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":355,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.275956+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:37:10.275956+0000","last_peered":"2023-09-23T23:37:10.275956+0000","last_clean":"2023-09-23T23:37:10.275956+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:37:10.275956+0000","last_undegraded":"2023-09-23T23:37:10.275956+0000","last_fullsized":"2023-09-23T23:37:10.275956+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":749,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.289384+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:37:18.289384+0000","last_peered":"2023-09-23T23:37:18.289384+0000","last_clean":"2023-09-23T23:37:18.289384+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:37:18.289384+0000","last_undegraded":"2023-09-23T23:37:18.289384+0000","last_fullsized":"2023-09-23T23:37:18.289384+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":103,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283558+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:37:18.283558+0000","last_peered":"2023-09-23T23:37:18.283558+0000","last_clean":"2023-09-23T23:37:18.283558+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:37:18.283558+0000","last_undegraded":"2023-09-23T23:37:18.283558+0000","last_fullsized":"2023-09-23T23:37:18.283558+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":449,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290854+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:37:18.290854+0000","last_peered":"2023-09-23T23:37:18.290854+0000","last_clean":"2023-09-23T23:37:18.290854+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:37:18.290854+0000","last_undegraded":"2023-09-23T23:37:18.290854+0000","last_fullsized":"2023-09-23T23:37:18.290854+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":299,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.285619+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:37:18.285619+0000","last_peered":"2023-09-23T23:37:18.285619+0000","last_clean":"2023-09-23T23:37:18.285619+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:37:18.285619+0000","last_undegraded":"2023-09-23T23:37:18.285619+0000","last_fullsized":"2023-09-23T23:37:18.285619+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":288,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.293254+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:37:18.293254+0000","last_peered":"2023-09-23T23:37:18.293254+0000","last_clean":"2023-09-23T23:37:18.293254+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:37:18.293254+0000","last_undegraded":"2023-09-23T23:37:18.293254+0000","last_fullsized":"2023-09-23T23:37:18.293254+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":813,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283743+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:37:18.283743+0000","last_peered":"2023-09-23T23:37:18.283743+0000","last_clean":"2023-09-23T23:37:18.283743+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:37:18.283743+0000","last_undegraded":"2023-09-23T23:37:18.283743+0000","last_fullsized":"2023-09-23T23:37:18.283743+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":632,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.289306+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:37:18.289306+0000","last_peered":"2023-09-23T23:37:18.289306+0000","last_clean":"2023-09-23T23:37:18.289306+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:37:18.289306+0000","last_undegraded":"2023-09-23T23:37:18.289306+0000","last_fullsized":"2023-09-23T23:37:18.289306+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":615,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.291133+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:37:18.291133+0000","last_peered":"2023-09-23T23:37:18.291133+0000","last_clean":"2023-09-23T23:37:18.291133+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:37:18.291133+0000","last_undegraded":"2023-09-23T23:37:18.291133+0000","last_fullsized":"2023-09-23T23:37:18.291133+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":774,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.300198+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:37:18.300198+0000","last_peered":"2023-09-23T23:37:18.300198+0000","last_clean":"2023-09-23T23:37:18.300198+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:37:18.300198+0000","last_undegraded":"2023-09-23T23:37:18.300198+0000","last_fullsized":"2023-09-23T23:37:18.300198+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":444,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T23:37:10.275955+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:37:10.275955+0000","last_peered":"2023-09-23T23:37:10.275955+0000","last_clean":"2023-09-23T23:37:10.275955+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:37:10.275955+0000","last_undegraded":"2023-09-23T23:37:10.275955+0000","last_fullsized":"2023-09-23T23:37:10.275955+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":668,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283248+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:37:18.283248+0000","last_peered":"2023-09-23T23:37:18.283248+0000","last_clean":"2023-09-23T23:37:18.283248+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:37:18.283248+0000","last_undegraded":"2023-09-23T23:37:18.283248+0000","last_fullsized":"2023-09-23T23:37:18.283248+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":2891841560,"num_objects":44127,"num_object_clones":0,"num_object_copies":132381,"num_objects_missing_on_primary":1728,"num_objects_missing":1728,"num_objects_degraded":23188,"num_objects_misplaced":2623,"num_objects_unfound":0,"num_objects_dirty":44127,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":131429,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17733,"num_bytes_recovered":1161953280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10146623488,"data_stored":10146611272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90007,"ondisk_log_size":90007,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782696,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1474812,"kb_used_data":1088820,"kb_used_omap":8,"kb_used_meta":385975,"kb_avail":92270340,"statfs":{"total":95995035648,"available":94484828160,"internally_reserved":0,"allocated":1114951680,"data_stored":1113274115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":395238469},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":1.048,"15min":2.1080000000000001},"min":{"1min":0.25900000000000001,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":0.51300000000000001,"5min":17.140000000000001,"15min":85.721000000000004},"last":1.405},{"interface":"front","average":{"1min":0.47199999999999998,"5min":1.0600000000000001,"15min":2.1829999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":2.331,"5min":11.069000000000001,"15min":85.965000000000003},"last":2.5499999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":2.532,"15min":3.1309999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":1.5780000000000001,"5min":129.11199999999999,"15min":198.58000000000001},"last":2.6739999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":2.3039999999999998,"15min":3.1440000000000001},"min":{"1min":0.31900000000000001,"5min":0.247,"15min":0.246},"max":{"1min":2.3690000000000002,"5min":129.02600000000001,"15min":198.71100000000001},"last":0.42599999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":3.3980000000000001,"5min":2.6880000000000002,"15min":3.8050000000000002},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":53.054000000000002,"5min":78.528999999999996,"15min":203.58000000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":3.5699999999999998,"5min":1.9299999999999999,"15min":3.2229999999999999},"min":{"1min":0.33400000000000002,"5min":0.27900000000000003,"15min":0.26200000000000001},"max":{"1min":53.023000000000003,"5min":53.023000000000003,"15min":198.69},"last":0.41499999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":8.6839999999999993,"5min":4.4390000000000001,"15min":4.0750000000000002},"min":{"1min":0.33300000000000002,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":100.40600000000001,"5min":125.511,"15min":279.26499999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":8.7420000000000009,"5min":4.8460000000000001,"15min":4.6180000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29599999999999999},"max":{"1min":100.42400000000001,"5min":125.48999999999999,"15min":279.29899999999998},"last":4.2370000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":1.546,"15min":1.0389999999999999},"min":{"1min":0.28999999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":0.58099999999999996,"5min":31.498999999999999,"15min":31.498999999999999},"last":1.3480000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":1.341,"15min":0.97599999999999998},"min":{"1min":0.32500000000000001,"5min":0.317,"15min":0.23899999999999999},"max":{"1min":2.3980000000000001,"5min":31.68,"15min":31.68},"last":0.36299999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.6220000000000001,"15min":1.863},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.26500000000000001},"max":{"1min":2.4510000000000001,"5min":60.463999999999999,"15min":210.83600000000001},"last":0.503},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.5840000000000001,"15min":1.742},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.5940000000000001,"5min":63.006,"15min":207.04599999999999},"last":0.34100000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.827,"15min":1.3320000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.29899999999999999},"max":{"1min":1.619,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":2.052,"15min":1.544},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.389,"5min":95.510999999999996,"15min":95.510999999999996},"last":1.911}]}]},{"osd":6,"up_from":46,"seq":197568496173,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1720876,"kb_used_data":1015780,"kb_used_omap":0,"kb_used_meta":705088,"kb_avail":92024276,"statfs":{"total":95995035648,"available":94232858624,"internally_reserved":0,"allocated":1040158720,"data_stored":1038497148,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":722010112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.73499999999999999,"15min":4.4370000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.21099999999999999},"max":{"1min":0.53700000000000003,"5min":6.4450000000000003,"15min":135.31999999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.498,"5min":0.63800000000000001,"15min":4.3869999999999996},"min":{"1min":0.27800000000000002,"5min":0.24399999999999999,"15min":0.23000000000000001},"max":{"1min":1.742,"5min":3.8239999999999998,"15min":135.34299999999999},"last":0.39600000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.90500000000000003,"15min":2.2890000000000001},"min":{"1min":0.28299999999999997,"5min":0.253,"15min":0.245},"max":{"1min":1.8759999999999999,"5min":6.9080000000000004,"15min":89.605000000000004},"last":0.435},{"interface":"front","average":{"1min":0.749,"5min":0.72499999999999998,"15min":2.391},"min":{"1min":0.29099999999999998,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":3.8980000000000001,"5min":3.8980000000000001,"15min":91.025999999999996},"last":0.46300000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:37:06 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":3.6930000000000001,"15min":4.4749999999999996},"min":{"1min":0.27900000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.1800000000000002,"5min":67.796999999999997,"15min":127.44499999999999},"last":43.765999999999998},{"interface":"front","average":{"1min":0.66600000000000004,"5min":3.6749999999999998,"15min":4.8680000000000003},"min":{"1min":0.316,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.2080000000000002,"5min":67.893000000000001,"15min":125.539},"last":43.750999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:36:55 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":5.6589999999999998,"15min":4.3159999999999998},"min":{"1min":0.30299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.3610000000000002,"5min":87.283000000000001,"15min":138.87299999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.47699999999999998,"5min":5.516,"15min":4.625},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0349999999999999,"5min":87.491,"15min":138.976},"last":0.47799999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.76400000000000001,"15min":0.68300000000000005},"min":{"1min":0.27900000000000003,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":0.57799999999999996,"5min":7.101,"15min":8.1999999999999993},"last":0.5},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.88900000000000001,"15min":0.77600000000000002},"min":{"1min":0.32500000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.54600000000000004,"5min":8.3499999999999996,"15min":8.3499999999999996},"last":0.45400000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.82899999999999996,"15min":0.80700000000000005},"min":{"1min":0.34300000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.266,"5min":11.372,"15min":11.473000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.98999999999999999,"15min":0.83599999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26900000000000002},"max":{"1min":0.57699999999999996,"5min":7.0129999999999999,"15min":7.5670000000000002},"last":0.42299999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:37:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.1679999999999999,"15min":1.0029999999999999},"min":{"1min":0.32700000000000001,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":0.77000000000000002,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.204,"15min":1.056},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33500000000000002},"max":{"1min":1.792,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.56399999999999995}]}]},{"osd":1,"up_from":112,"seq":481036337514,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1789748,"kb_used_data":893100,"kb_used_omap":4,"kb_used_meta":896635,"kb_avail":91955404,"statfs":{"total":95995035648,"available":94162333696,"internally_reserved":0,"allocated":914534400,"data_stored":911697424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":918155150},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.407,"15min":3.173},"min":{"1min":0.19,"5min":0.19,"15min":0.16700000000000001},"max":{"1min":3.5830000000000002,"5min":47.195999999999998,"15min":130.768},"last":2.1619999999999999},{"interface":"front","average":{"1min":0.377,"5min":1.119,"15min":2.5030000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19},"max":{"1min":1.143,"5min":46.582000000000001,"15min":134.929},"last":0.29099999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":3.5659999999999998,"15min":6.7649999999999997},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":1.401,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.66900000000000004,"5min":3.8849999999999998,"15min":6.8339999999999996},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.23,"5min":202.12,"15min":232.21000000000001},"last":0.442}]},{"osd":3,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":6.883,"5min":5.8460000000000001,"15min":5.4379999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22600000000000001},"max":{"1min":71.745000000000005,"5min":181.94399999999999,"15min":234.404},"last":2.0489999999999999},{"interface":"front","average":{"1min":6.8639999999999999,"5min":5.8860000000000001,"15min":5.3899999999999997},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":71.786000000000001,"5min":181.98500000000001,"15min":236.33699999999999},"last":3.673}]},{"osd":4,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":0.95099999999999996,"15min":1.399},"min":{"1min":0.32500000000000001,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":4.4880000000000004,"5min":8.0850000000000009,"15min":12.042999999999999},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.85699999999999998,"15min":1.45},"min":{"1min":0.316,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":4.6660000000000004,"5min":9.5069999999999997,"15min":18.108000000000001},"last":0.39500000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":2.6309999999999998,"15min":1.734},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28899999999999998},"max":{"1min":0.98199999999999998,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.024,"5min":2.661,"15min":2.0710000000000002},"min":{"1min":0.34399999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.194,"5min":151.821,"15min":151.821},"last":0.38800000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.0449999999999999,"15min":1.6659999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.27500000000000002},"max":{"1min":2.306,"5min":8.9269999999999996,"15min":74.087999999999994},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.2849999999999999,"15min":1.929},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33100000000000002},"max":{"1min":3.29,"5min":8.1400000000000006,"15min":73.974999999999994},"last":0.41499999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":1.5449999999999999,"15min":2.359},"min":{"1min":0.40200000000000002,"5min":0.38700000000000001,"15min":0.32900000000000001},"max":{"1min":2.504,"5min":44.241,"15min":78.308000000000007},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.556,"15min":2.2730000000000001},"min":{"1min":0.41299999999999998,"5min":0.37,"15min":0.32700000000000001},"max":{"1min":3.258,"5min":43.920999999999999,"15min":77.204999999999998},"last":0.47799999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024337,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1459500,"kb_used_data":253348,"kb_used_omap":5,"kb_used_meta":1206138,"kb_avail":92285652,"statfs":{"total":95995035648,"available":94500507648,"internally_reserved":0,"allocated":259428352,"data_stored":256585266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1235085946},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:36:36 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":4.391,"15min":6.6539999999999999},"min":{"1min":0.219,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":1.7709999999999999,"5min":301.12700000000001,"15min":301.12700000000001},"last":1.2549999999999999},{"interface":"front","average":{"1min":0.43099999999999999,"5min":4.1289999999999996,"15min":5.7969999999999997},"min":{"1min":0.245,"5min":0.20200000000000001,"15min":0.17599999999999999},"max":{"1min":1.0249999999999999,"5min":297.72399999999999,"15min":297.72399999999999},"last":1.1890000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":6.9610000000000003,"15min":7.3559999999999999},"min":{"1min":0.26200000000000001,"5min":0.214,"15min":0.17199999999999999},"max":{"1min":4.0899999999999999,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.84599999999999997,"5min":6.6799999999999997,"15min":7.3869999999999996},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.17599999999999999},"max":{"1min":3.468,"5min":283.161,"15min":283.161},"last":0.51400000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:36:36 2023","interfaces":[{"interface":"back","average":{"1min":5.9699999999999998,"5min":5.1630000000000003,"15min":5.6079999999999997},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":95.655000000000001,"5min":114.381,"15min":220.732},"last":1.3080000000000001},{"interface":"front","average":{"1min":5.8300000000000001,"5min":5.2779999999999996,"15min":5.6669999999999998},"min":{"1min":0.26600000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":95.695999999999998,"5min":114.30500000000001,"15min":218.78200000000001},"last":0.53800000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.79900000000000004,"15min":2.3479999999999999},"min":{"1min":0.34399999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.60999999999999999,"5min":9.3140000000000001,"15min":138.81299999999999},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.66800000000000004,"5min":1.02,"15min":3.0270000000000001},"min":{"1min":0.36799999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.661,"5min":9.3670000000000009,"15min":171.251},"last":0.81100000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.1850000000000001,"15min":2.7290000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":1.288,"5min":15.076000000000001,"15min":171.30699999999999},"last":0.84499999999999997},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.097,"15min":1.9930000000000001},"min":{"1min":0.33600000000000002,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":4.1710000000000003,"5min":6.5279999999999996,"15min":139.40799999999999},"last":0.42099999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":1.143,"15min":3.0760000000000001},"min":{"1min":0.374,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":12.654999999999999,"5min":12.654999999999999,"15min":171.33600000000001},"last":0.87},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.155,"15min":3.2749999999999999},"min":{"1min":0.41199999999999998,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":4.6440000000000001,"5min":9.157,"15min":184.13499999999999},"last":0.442}]},{"osd":7,"last update":"Sat Sep 23 23:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":1.089,"15min":2.569},"min":{"1min":0.35099999999999998,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":2.6749999999999998,"5min":8.9510000000000005,"15min":138.77699999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.3260000000000001,"15min":2.508},"min":{"1min":0.373,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":4.2229999999999999,"5min":9.8870000000000005,"15min":139.37899999999999},"last":0.73099999999999998}]}]},{"osd":2,"up_from":199,"seq":854698492158,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2672172,"kb_used_data":1820068,"kb_used_omap":7,"kb_used_meta":852088,"kb_avail":91072980,"statfs":{"total":95995035648,"available":93258731520,"internally_reserved":0,"allocated":1863749632,"data_stored":1860856594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":872538193},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.7789999999999999,"15min":3.617},"min":{"1min":0.20200000000000001,"5min":0.186,"15min":0.16700000000000001},"max":{"1min":6.3940000000000001,"5min":61.950000000000003,"15min":119.277},"last":0.22500000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":1.8740000000000001,"15min":3.46},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.18099999999999999},"max":{"1min":5.0339999999999998,"5min":61.984999999999999,"15min":119.56100000000001},"last":0.316}]},{"osd":1,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":4.4960000000000004,"15min":4.5449999999999999},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":4.7060000000000004,"5min":106.765,"15min":157.98699999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.73899999999999999,"5min":4.1210000000000004,"15min":4.4950000000000001},"min":{"1min":0.23400000000000001,"5min":0.222,"15min":0.222},"max":{"1min":6.4340000000000002,"5min":106.732,"15min":158.03800000000001},"last":0.26500000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":5.3179999999999996,"15min":4.6900000000000004},"min":{"1min":0.26000000000000001,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":1.075,"5min":106.70399999999999,"15min":119.366},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.52100000000000002,"5min":5.0359999999999996,"15min":4.1440000000000001},"min":{"1min":0.246,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.6120000000000001,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.28100000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":1.756,"15min":2.1909999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.25},"max":{"1min":0.65600000000000003,"5min":62.081000000000003,"15min":119.88200000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.9710000000000001,"15min":1.829},"min":{"1min":0.42299999999999999,"5min":0.35599999999999998,"15min":0.28499999999999998},"max":{"1min":1.121,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.41299999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":2.8340000000000001,"15min":2.5840000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":2.6549999999999998,"5min":91.207999999999998,"15min":119.919},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.69399999999999995,"5min":2.919,"15min":2.5800000000000001},"min":{"1min":0.371,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.4289999999999998,"5min":91.150999999999996,"15min":119.306},"last":0.36799999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":3.254,"15min":2.9169999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":1.04,"5min":107.018,"15min":119.943},"last":0.441},{"interface":"front","average":{"1min":0.746,"5min":2.1779999999999999,"15min":2.1160000000000001},"min":{"1min":0.45100000000000001,"5min":0.371,"15min":0.34999999999999998},"max":{"1min":3.4950000000000001,"5min":62.493000000000002,"15min":62.493000000000002},"last":0.38900000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.917,"15min":1.883},"min":{"1min":0.41699999999999998,"5min":0.374,"15min":0.34000000000000002},"max":{"1min":1.831,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.66800000000000004,"5min":3.1800000000000002,"15min":2.3220000000000001},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.36199999999999999},"max":{"1min":1.792,"5min":106.809,"15min":106.809},"last":0.373}]}]},{"osd":3,"up_from":148,"seq":635655160127,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1903088,"kb_used_data":965352,"kb_used_omap":2,"kb_used_meta":937725,"kb_avail":91842064,"statfs":{"total":95995035648,"available":94046273536,"internally_reserved":0,"allocated":988520448,"data_stored":985685188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":960230453},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,2,2,1,4,5],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.1359999999999999,"15min":2.6019999999999999},"min":{"1min":0.184,"5min":0.184,"15min":0.17699999999999999},"max":{"1min":6.7699999999999996,"5min":17.213000000000001,"15min":103.866},"last":2.9980000000000002},{"interface":"front","average":{"1min":0.72899999999999998,"5min":1.4139999999999999,"15min":2.855},"min":{"1min":0.218,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":3.4980000000000002,"5min":22.754000000000001,"15min":101.884},"last":0.75900000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":2.5960000000000001,"15min":2.903},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":1.649,"5min":56.119,"15min":85.242999999999995},"last":2.851},{"interface":"front","average":{"1min":0.42299999999999999,"5min":2.036,"15min":2.7469999999999999},"min":{"1min":0.254,"5min":0.224,"15min":0.20899999999999999},"max":{"1min":1.6990000000000001,"5min":53.962000000000003,"15min":85.302000000000007},"last":2.9830000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:36:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":2.2989999999999999,"15min":4.5999999999999996},"min":{"1min":0.25800000000000001,"5min":0.25,"15min":0.19800000000000001},"max":{"1min":14.965,"5min":56.226999999999997,"15min":101.279},"last":2.8010000000000002},{"interface":"front","average":{"1min":1.2210000000000001,"5min":2.165,"15min":4.4370000000000003},"min":{"1min":0.315,"5min":0.29299999999999998,"15min":0.214},"max":{"1min":15.058999999999999,"5min":54.280999999999999,"15min":101.294},"last":2.923}]},{"osd":4,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":0.68999999999999995,"5min":11.209,"15min":11.209},"last":2.9609999999999999},{"interface":"front","average":{"1min":0.44500000000000001,"5min":1.3720000000000001,"15min":1.6719999999999999},"min":{"1min":0.33000000000000002,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":0.67900000000000005,"5min":11.494999999999999,"15min":24.782},"last":0.51800000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":6.3570000000000002,"15min":3.3220000000000001},"min":{"1min":0.33000000000000002,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":0.622,"5min":214.43700000000001,"15min":214.43700000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.61099999999999999,"5min":6.3970000000000002,"15min":3.3969999999999998},"min":{"1min":0.36499999999999999,"5min":0.34699999999999998,"15min":0.30599999999999999},"max":{"1min":1.716,"5min":214.46600000000001,"15min":214.46600000000001},"last":3.0339999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":2.218,"15min":2.1739999999999999},"min":{"1min":0.35399999999999998,"5min":0.318,"15min":0.318},"max":{"1min":2.008,"5min":58.932000000000002,"15min":58.932000000000002},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.629,"5min":2.492,"15min":2.2210000000000001},"min":{"1min":0.36699999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.7450000000000001,"5min":59.186,"15min":59.186},"last":3.0649999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:36:56 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":4.5469999999999997,"15min":3.1829999999999998},"min":{"1min":0.42199999999999999,"5min":0.38700000000000001,"15min":0.32600000000000001},"max":{"1min":0.66700000000000004,"5min":192.69,"15min":192.69},"last":3.194},{"interface":"front","average":{"1min":0.54300000000000004,"5min":4.6280000000000001,"15min":3.649},"min":{"1min":0.40100000000000002,"5min":0.39200000000000002,"15min":0.36699999999999999},"max":{"1min":1.2769999999999999,"5min":192.65600000000001,"15min":192.65600000000001},"last":0.64700000000000002}]}]},{"osd":4,"up_from":0,"seq":128849019446,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757809,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2388968,"kb_used_data":1652896,"kb_used_omap":0,"kb_used_meta":736000,"kb_avail":91356184,"statfs":{"total":95995035648,"available":93548732416,"internally_reserved":0,"allocated":1692565504,"data_stored":1690904149,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":753664000},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.032,"15min":2.597},"min":{"1min":0.25,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":2.9279999999999999,"5min":21.109000000000002,"15min":113.869},"last":0.308},{"interface":"front","average":{"1min":1.123,"5min":1.407,"15min":3.0659999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":11.154999999999999,"5min":20.876999999999999,"15min":138.55199999999999},"last":0.33100000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:37:22 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.1459999999999999,"15min":2.1629999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.1360000000000001,"5min":29.007999999999999,"15min":69.816999999999993},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.212,"15min":2.3929999999999998},"min":{"1min":0.30499999999999999,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":1.452,"5min":29.053999999999998,"15min":70.268000000000001},"last":0.36699999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:36:48 2023","interfaces":[{"interface":"back","average":{"1min":5.194,"5min":6.984,"15min":4.8159999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.26900000000000002},"max":{"1min":60.441000000000003,"5min":208.738,"15min":208.738},"last":0.44600000000000001},{"interface":"front","average":{"1min":5.2839999999999998,"5min":5.7190000000000003,"15min":4.3259999999999996},"min":{"1min":0.29999999999999999,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":60.520000000000003,"5min":208.631,"15min":208.631},"last":0.70799999999999996}]},{"osd":3,"last update":"Sat Sep 23 23:36:48 2023","interfaces":[{"interface":"back","average":{"1min":4.1749999999999998,"5min":4.6360000000000001,"15min":3.8100000000000001},"min":{"1min":0.34999999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":37.878,"5min":122.071,"15min":122.071},"last":0.39800000000000002},{"interface":"front","average":{"1min":4.2460000000000004,"5min":4.9829999999999997,"15min":3.984},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.318},"max":{"1min":37.853000000000002,"5min":122.09699999999999,"15min":122.09699999999999},"last":0.48099999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.98699999999999999,"15min":0.90700000000000003},"min":{"1min":0.32900000000000001,"5min":0.25700000000000001,"15min":0.251},"max":{"1min":0.749,"5min":9.2690000000000001,"15min":19.962},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.502,"5min":1.0129999999999999,"15min":0.81599999999999995},"min":{"1min":0.314,"5min":0.29599999999999999,"15min":0.22900000000000001},"max":{"1min":1.595,"5min":10.273999999999999,"15min":20.829999999999998},"last":0.433}]},{"osd":6,"last update":"Sat Sep 23 23:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":2.29,"15min":1.5049999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.26900000000000002},"max":{"1min":1.619,"5min":148.36600000000001,"15min":148.36600000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.54000000000000004,"5min":2.798,"15min":1.7250000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.26000000000000001},"max":{"1min":2.1549999999999998,"5min":148.42500000000001,"15min":148.42500000000001},"last":0.42299999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:36:48 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.2629999999999999,"15min":2.3879999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":2.29,"5min":15.497,"15min":179.28800000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.64900000000000002,"5min":1.2729999999999999,"15min":2.46},"min":{"1min":0.36699999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.0630000000000002,"5min":15.653,"15min":179.428},"last":0.46400000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":250347520,"data_stored":250347520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":903614464,"data_stored":903610392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1850474496,"data_stored":1850474496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":979898368,"data_stored":979894296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2353397760,"data_stored":2353397760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1673527296,"data_stored":1673527296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1028849664,"data_stored":1028849664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1106513920,"data_stored":1106509848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:37:33.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:33.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:37:33.131 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:33.171 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:33 smithi138 ceph-mon[193606]: pgmap v1691: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 359 op/s; 23188/131397 objects degraded (17.647%); 2623/131397 objects misplaced (1.996%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:37:33.177 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:33.262 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:33.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:33.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:33.321 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:33 smithi110 ceph-mon[212939]: pgmap v1691: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 359 op/s; 23188/131397 objects degraded (17.647%); 2623/131397 objects misplaced (1.996%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:37:33.322 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:33 smithi110 ceph-mon[221102]: pgmap v1691: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 359 op/s; 23188/131397 objects degraded (17.647%); 2623/131397 objects misplaced (1.996%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:37:33.368 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:37:33.422 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:33.505 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:37:33.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:33.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:37:33.516 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:37:33.517 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:37:33.609 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:33.639 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:33.688 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:33.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:33.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:33.794 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:37:33.800 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:37:33.845 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:33.905 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:37:33.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:33.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:37:34.010 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:37:34.042 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:34.108 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:37:34.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:34.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:37:34.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:34 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 23188/131397 objects degraded (17.647%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:34.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:34 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1003231047' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:37:34.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:34 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 23188/131397 objects degraded (17.647%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:34.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:34 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1003231047' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:37:34.217 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:34.247 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:34.313 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:34.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:34 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 23188/131397 objects degraded (17.647%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:34.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:34 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1003231047' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:37:34.509 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:34.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:34.520 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:37:34.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:34.621 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:34.669 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:34.740 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:34.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:34.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:34.792 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:37:34.849 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:34.885 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:34.943 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:34.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:34.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:35.034 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:37:35.051 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:37:35.083 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:35.170 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:37:35.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:35.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:37:35.274 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:37:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:35 smithi138 ceph-mon[193606]: pgmap v1692: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 366 op/s; 23060/129951 objects degraded (17.745%); 2522/129951 objects misplaced (1.941%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:37:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:35 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:35 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1003231047' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:37:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:35 smithi138 ceph-mon[193606]: osdmap e602: 8 total, 8 up, 7 in 2023-09-23T23:37:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:35 smithi138 ceph-mon[193606]: failed to encode map e602 with expected crc 2023-09-23T23:37:35.313 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:35 smithi110 ceph-mon[221102]: pgmap v1692: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 366 op/s; 23060/129951 objects degraded (17.745%); 2522/129951 objects misplaced (1.941%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:37:35.313 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:35 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:35.313 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:35 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1003231047' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:37:35.314 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:35 smithi110 ceph-mon[221102]: osdmap e602: 8 total, 8 up, 7 in 2023-09-23T23:37:35.314 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:35 smithi110 ceph-mon[221102]: failed to encode map e602 with expected crc 2023-09-23T23:37:35.314 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:35 smithi110 ceph-mon[212939]: pgmap v1692: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 366 op/s; 23060/129951 objects degraded (17.745%); 2522/129951 objects misplaced (1.941%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:37:35.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:35 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:35.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:35 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1003231047' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:37:35.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:35 smithi110 ceph-mon[212939]: osdmap e602: 8 total, 8 up, 7 in 2023-09-23T23:37:35.315 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:35 smithi110 ceph-mon[212939]: failed to encode map e602 with expected crc 2023-09-23T23:37:35.318 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:35.390 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:37:35.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:35.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:37:35.495 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:35.548 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:35.754 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:35.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:35.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:35.859 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:35.903 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:36.030 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:37:36.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:36 smithi110 ceph-mon[212939]: failed to encode map e602 with expected crc 2023-09-23T23:37:36.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:36 smithi110 ceph-mon[221102]: failed to encode map e602 with expected crc 2023-09-23T23:37:36.346 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:36 smithi138 ceph-mon[193606]: failed to encode map e602 with expected crc 2023-09-23T23:37:36.377 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:36.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:36.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:36.486 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:36.520 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:36.604 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:36.682 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:36.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:36.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:36.785 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:36.833 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:36.911 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:36.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:36.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:36.985 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:37:36.985 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:37:37.016 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:37:37.050 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:37:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:37:36] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2023-09-23T23:37:37.055 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:37:37.056 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:37.086 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:37:37.114 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:37:37.135 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:37:37.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:37.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:37:37.144 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:37:37.167 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:37:37.188 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:37:37.208 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:37:37.231 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:37:37.233 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:37:37.233 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:37:37.233 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:37:37.233 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:37:37.233 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:37:37.233 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:37:37.233 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:37:37.233 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:37:37.240 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:37.266 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:37 smithi138 ceph-mon[193606]: pgmap v1694: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 404 op/s; 22923/125421 objects degraded (18.277%); 2522/125421 objects misplaced (2.011%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:37:37.266 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:37 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2449844893' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:37.269 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:37.312 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:37.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:37.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:37.416 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:37.441 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:37.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:37 smithi110 ceph-mon[212939]: pgmap v1694: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 404 op/s; 22923/125421 objects degraded (18.277%); 2522/125421 objects misplaced (2.011%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:37:37.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:37 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2449844893' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:37.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:37 smithi110 ceph-mon[221102]: pgmap v1694: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 404 op/s; 22923/125421 objects degraded (18.277%); 2522/125421 objects misplaced (2.011%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:37:37.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:37 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2449844893' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:37.499 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:37.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:37.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:37.607 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:37.633 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:37.831 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:37.833 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:37:37.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:37.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:37.941 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:37.997 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:38.195 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:38.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:38.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:38.299 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:38.305 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:37:38.321 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:37:38.336 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:38.396 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:38.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:38.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:38.499 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:38.532 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:38.587 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:38.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:38.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:38.692 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:38.729 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:38.853 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:38.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:38.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:38.958 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:39.003 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:39.151 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:39.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:39.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:39.173 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:39 smithi138 ceph-mon[193606]: pgmap v1695: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 390 op/s; 22923/125421 objects degraded (18.277%); 2522/125421 objects misplaced (2.011%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T23:37:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:39 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 22923/125421 objects degraded (18.277%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:39.258 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:39.327 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:39 smithi110 ceph-mon[221102]: pgmap v1695: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 390 op/s; 22923/125421 objects degraded (18.277%); 2522/125421 objects misplaced (2.011%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T23:37:39.327 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:39 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 22923/125421 objects degraded (18.277%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:39.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:39 smithi110 ceph-mon[212939]: pgmap v1695: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 390 op/s; 22923/125421 objects degraded (18.277%); 2522/125421 objects misplaced (2.011%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T23:37:39.327 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:39 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 22923/125421 objects degraded (18.277%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:39.333 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:39.408 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:39.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:39.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:39.512 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:37:39.539 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:37:39.541 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:37:39.548 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:39.612 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:37:39.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:39.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:37:39.719 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:39.752 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:39.813 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:39.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:39.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:39.917 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:39.951 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:40.005 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:40.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:40.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:40.066 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:37:40.067 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:37:40.067 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:37:40.067 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:37:40.067 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:37:40.114 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:37:40.145 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:40.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:40 smithi110 ceph-mon[212939]: from='client.47006 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:37:40.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:40 smithi110 ceph-mon[221102]: from='client.47006 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:37:40.308 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:37:40.310 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:40 smithi138 ceph-mon[193606]: from='client.47006 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:37:40.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:40.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:37:40.412 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:37:40.440 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:40.832 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1695,"stamp":"2023-09-23T23:37:37.944317+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2738861301,"num_objects":41807,"num_object_clones":0,"num_object_copies":125421,"num_objects_missing_on_primary":1627,"num_objects_missing":1627,"num_objects_degraded":22923,"num_objects_misplaced":2522,"num_objects_unfound":0,"num_objects_dirty":41807,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":133850,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18159,"num_bytes_recovered":1187377576,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95097,"ondisk_log_size":95097,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13016628,"kb_used_data":7284668,"kb_used_omap":29,"kb_used_meta":5731810,"kb_avail":643199436,"statfs":{"total":671965249536,"available":658636222464,"internally_reserved":0,"allocated":7459500032,"data_stored":7443075971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5869374163},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-256180224,"num_objects":-3909,"num_object_clones":0,"num_object_copies":-11727,"num_objects_missing_on_primary":-202,"num_objects_missing":-202,"num_objects_degraded":-515,"num_objects_misplaced":-203,"num_objects_unfound":0,"num_objects_dirty":-3909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3909,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":724,"num_bytes_recovered":47448064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003479"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":797,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.289752+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:37:18.289752+0000","last_peered":"2023-09-23T23:37:18.289752+0000","last_clean":"2023-09-23T23:37:18.289752+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:37:18.289752+0000","last_undegraded":"2023-09-23T23:37:18.289752+0000","last_fullsized":"2023-09-23T23:37:18.289752+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":357,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.400899+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:37:26.400899+0000","last_peered":"2023-09-23T23:37:26.400899+0000","last_clean":"2023-09-23T23:37:26.400899+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:37:26.400899+0000","last_undegraded":"2023-09-23T23:37:26.400899+0000","last_fullsized":"2023-09-23T23:37:26.400899+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":583,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.312971+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:37:18.312971+0000","last_peered":"2023-09-23T23:37:18.312971+0000","last_clean":"2023-09-23T23:37:18.312971+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:37:18.312971+0000","last_undegraded":"2023-09-23T23:37:18.312971+0000","last_fullsized":"2023-09-23T23:37:18.312971+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":750,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.292834+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:37:18.292834+0000","last_peered":"2023-09-23T23:37:18.292834+0000","last_clean":"2023-09-23T23:37:18.292834+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:37:18.292834+0000","last_undegraded":"2023-09-23T23:37:18.292834+0000","last_fullsized":"2023-09-23T23:37:18.292834+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":320,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.291202+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:37:18.291202+0000","last_peered":"2023-09-23T23:37:18.291202+0000","last_clean":"2023-09-23T23:37:18.291202+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:37:18.291202+0000","last_undegraded":"2023-09-23T23:37:18.291202+0000","last_fullsized":"2023-09-23T23:37:18.291202+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":562,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.288956+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:37:18.288956+0000","last_peered":"2023-09-23T23:37:18.288956+0000","last_clean":"2023-09-23T23:37:18.288956+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:37:18.288956+0000","last_undegraded":"2023-09-23T23:37:18.288956+0000","last_fullsized":"2023-09-23T23:37:18.288956+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":762,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.281791+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:37:18.281791+0000","last_peered":"2023-09-23T23:37:18.281791+0000","last_clean":"2023-09-23T23:37:18.281791+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:37:18.281791+0000","last_undegraded":"2023-09-23T23:37:18.281791+0000","last_fullsized":"2023-09-23T23:37:18.281791+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":619,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.287694+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:37:18.287694+0000","last_peered":"2023-09-23T23:37:18.287694+0000","last_clean":"2023-09-23T23:37:18.287694+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:37:18.287694+0000","last_undegraded":"2023-09-23T23:37:18.287694+0000","last_fullsized":"2023-09-23T23:37:18.287694+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":115,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283996+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:37:18.283996+0000","last_peered":"2023-09-23T23:37:18.283996+0000","last_clean":"2023-09-23T23:37:18.283996+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:37:18.283996+0000","last_undegraded":"2023-09-23T23:37:18.283996+0000","last_fullsized":"2023-09-23T23:37:18.283996+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":921,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.281571+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:37:18.281571+0000","last_peered":"2023-09-23T23:37:18.281571+0000","last_clean":"2023-09-23T23:37:18.281571+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:37:18.281571+0000","last_undegraded":"2023-09-23T23:37:18.281571+0000","last_fullsized":"2023-09-23T23:37:18.281571+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":253,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290568+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:37:18.290568+0000","last_peered":"2023-09-23T23:37:18.290568+0000","last_clean":"2023-09-23T23:37:18.290568+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:37:18.290568+0000","last_undegraded":"2023-09-23T23:37:18.290568+0000","last_fullsized":"2023-09-23T23:37:18.290568+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":602,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.287609+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:37:18.287609+0000","last_peered":"2023-09-23T23:37:18.287609+0000","last_clean":"2023-09-23T23:37:18.287609+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:37:18.287609+0000","last_undegraded":"2023-09-23T23:37:18.287609+0000","last_fullsized":"2023-09-23T23:37:18.287609+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":895,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.400106+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:37:26.400106+0000","last_peered":"2023-09-23T23:37:26.400106+0000","last_clean":"2023-09-23T23:37:26.400106+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:37:26.400106+0000","last_undegraded":"2023-09-23T23:37:26.400106+0000","last_fullsized":"2023-09-23T23:37:26.400106+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":232,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.291541+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:37:18.291541+0000","last_peered":"2023-09-23T23:37:18.291541+0000","last_clean":"2023-09-23T23:37:18.291541+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:37:18.291541+0000","last_undegraded":"2023-09-23T23:37:18.291541+0000","last_fullsized":"2023-09-23T23:37:18.291541+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":563,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.291023+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:37:18.291023+0000","last_peered":"2023-09-23T23:37:18.291023+0000","last_clean":"2023-09-23T23:37:18.291023+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:37:18.291023+0000","last_undegraded":"2023-09-23T23:37:18.291023+0000","last_fullsized":"2023-09-23T23:37:18.291023+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1055,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290495+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:37:18.290495+0000","last_peered":"2023-09-23T23:37:18.290495+0000","last_clean":"2023-09-23T23:37:18.290495+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:37:18.290495+0000","last_undegraded":"2023-09-23T23:37:18.290495+0000","last_fullsized":"2023-09-23T23:37:18.290495+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":274,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.303655+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:37:18.303655+0000","last_peered":"2023-09-23T23:37:18.303655+0000","last_clean":"2023-09-23T23:37:18.303655+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:37:18.303655+0000","last_undegraded":"2023-09-23T23:37:18.303655+0000","last_fullsized":"2023-09-23T23:37:18.303655+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":134,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283969+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:37:18.283969+0000","last_peered":"2023-09-23T23:37:18.283969+0000","last_clean":"2023-09-23T23:37:18.283969+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:37:18.283969+0000","last_undegraded":"2023-09-23T23:37:18.283969+0000","last_fullsized":"2023-09-23T23:37:18.283969+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":452,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.289085+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:37:18.289085+0000","last_peered":"2023-09-23T23:37:18.289085+0000","last_clean":"2023-09-23T23:37:18.289085+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:37:18.289085+0000","last_undegraded":"2023-09-23T23:37:18.289085+0000","last_fullsized":"2023-09-23T23:37:18.289085+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":175,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283680+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:37:18.283680+0000","last_peered":"2023-09-23T23:37:18.283680+0000","last_clean":"2023-09-23T23:37:18.283680+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:37:18.283680+0000","last_undegraded":"2023-09-23T23:37:18.283680+0000","last_fullsized":"2023-09-23T23:37:18.283680+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":542,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283606+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:37:18.283606+0000","last_peered":"2023-09-23T23:37:18.283606+0000","last_clean":"2023-09-23T23:37:18.283606+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:37:18.283606+0000","last_undegraded":"2023-09-23T23:37:18.283606+0000","last_fullsized":"2023-09-23T23:37:18.283606+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":756,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.289973+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:37:18.289973+0000","last_peered":"2023-09-23T23:37:18.289973+0000","last_clean":"2023-09-23T23:37:18.289973+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:37:18.289973+0000","last_undegraded":"2023-09-23T23:37:18.289973+0000","last_fullsized":"2023-09-23T23:37:18.289973+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":149,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.399396+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:37:26.399396+0000","last_peered":"2023-09-23T23:37:26.399396+0000","last_clean":"2023-09-23T23:37:26.399396+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:37:26.399396+0000","last_undegraded":"2023-09-23T23:37:26.399396+0000","last_fullsized":"2023-09-23T23:37:26.399396+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":601,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.288033+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:37:18.288033+0000","last_peered":"2023-09-23T23:37:18.288033+0000","last_clean":"2023-09-23T23:37:18.288033+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:37:18.288033+0000","last_undegraded":"2023-09-23T23:37:18.288033+0000","last_fullsized":"2023-09-23T23:37:18.288033+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":886,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283181+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:37:18.283181+0000","last_peered":"2023-09-23T23:37:18.283181+0000","last_clean":"2023-09-23T23:37:18.283181+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:37:18.283181+0000","last_undegraded":"2023-09-23T23:37:18.283181+0000","last_fullsized":"2023-09-23T23:37:18.283181+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":196,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.293323+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:37:18.293323+0000","last_peered":"2023-09-23T23:37:18.293323+0000","last_clean":"2023-09-23T23:37:18.293323+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:37:18.293323+0000","last_undegraded":"2023-09-23T23:37:18.293323+0000","last_fullsized":"2023-09-23T23:37:18.293323+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":323,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.312860+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:37:18.312860+0000","last_peered":"2023-09-23T23:37:18.312860+0000","last_clean":"2023-09-23T23:37:18.312860+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:37:18.312860+0000","last_undegraded":"2023-09-23T23:37:18.312860+0000","last_fullsized":"2023-09-23T23:37:18.312860+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":564,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.291021+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:37:18.291021+0000","last_peered":"2023-09-23T23:37:18.291021+0000","last_clean":"2023-09-23T23:37:18.291021+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:37:18.291021+0000","last_undegraded":"2023-09-23T23:37:18.291021+0000","last_fullsized":"2023-09-23T23:37:18.291021+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":469,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283257+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:37:18.283257+0000","last_peered":"2023-09-23T23:37:18.283257+0000","last_clean":"2023-09-23T23:37:18.283257+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:37:18.283257+0000","last_undegraded":"2023-09-23T23:37:18.283257+0000","last_fullsized":"2023-09-23T23:37:18.283257+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":557,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.289226+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:37:18.289226+0000","last_peered":"2023-09-23T23:37:18.289226+0000","last_clean":"2023-09-23T23:37:18.289226+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:37:18.289226+0000","last_undegraded":"2023-09-23T23:37:18.289226+0000","last_fullsized":"2023-09-23T23:37:18.289226+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":964,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290395+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:37:18.290395+0000","last_peered":"2023-09-23T23:37:18.290395+0000","last_clean":"2023-09-23T23:37:18.290395+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:37:18.290395+0000","last_undegraded":"2023-09-23T23:37:18.290395+0000","last_fullsized":"2023-09-23T23:37:18.290395+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":165,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283571+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:37:18.283571+0000","last_peered":"2023-09-23T23:37:18.283571+0000","last_clean":"2023-09-23T23:37:18.283571+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:37:18.283571+0000","last_undegraded":"2023-09-23T23:37:18.283571+0000","last_fullsized":"2023-09-23T23:37:18.283571+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":242,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.288498+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:37:18.288498+0000","last_peered":"2023-09-23T23:37:18.288498+0000","last_clean":"2023-09-23T23:37:18.288498+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:37:18.288498+0000","last_undegraded":"2023-09-23T23:37:18.288498+0000","last_fullsized":"2023-09-23T23:37:18.288498+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"601'8248","reported_seq":8332,"reported_epoch":601,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:34.210664+0000","last_change":"2023-09-23T23:33:52.884769+0000","last_active":"2023-09-23T23:37:34.210664+0000","last_peered":"2023-09-23T23:37:34.210664+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:37:34.210664+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"551'2800","ondisk_log_start":"551'2800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5448,"log_dups_size":0,"ondisk_log_size":5448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170393600,"num_objects":2600,"num_object_clones":0,"num_object_copies":7800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8248,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":1574},{"shards":"2,6","objects":-1798}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":936,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.399548+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:37:26.399548+0000","last_peered":"2023-09-23T23:37:26.399548+0000","last_clean":"2023-09-23T23:37:26.399548+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:37:26.399548+0000","last_undegraded":"2023-09-23T23:37:26.399548+0000","last_fullsized":"2023-09-23T23:37:26.399548+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":242,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.281540+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:37:18.281540+0000","last_peered":"2023-09-23T23:37:18.281540+0000","last_clean":"2023-09-23T23:37:18.281540+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:37:18.281540+0000","last_undegraded":"2023-09-23T23:37:18.281540+0000","last_fullsized":"2023-09-23T23:37:18.281540+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":232,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.289673+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:37:18.289673+0000","last_peered":"2023-09-23T23:37:18.289673+0000","last_clean":"2023-09-23T23:37:18.289673+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:37:18.289673+0000","last_undegraded":"2023-09-23T23:37:18.289673+0000","last_fullsized":"2023-09-23T23:37:18.289673+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"602'8406","reported_seq":8491,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.919329+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:37:34.919329+0000","last_peered":"2023-09-23T23:37:34.919329+0000","last_clean":"2023-09-23T23:37:34.919329+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:37:34.919329+0000","last_undegraded":"2023-09-23T23:37:34.919329+0000","last_fullsized":"2023-09-23T23:37:34.919329+0000","mapping_epoch":566,"log_start":"550'2200","ondisk_log_start":"550'2200","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6206,"log_dups_size":0,"ondisk_log_size":6206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172097536,"num_objects":2626,"num_object_clones":0,"num_object_copies":7878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8406,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1013,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.287758+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:37:18.287758+0000","last_peered":"2023-09-23T23:37:18.287758+0000","last_clean":"2023-09-23T23:37:18.287758+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:37:18.287758+0000","last_undegraded":"2023-09-23T23:37:18.287758+0000","last_fullsized":"2023-09-23T23:37:18.287758+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":232,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.282295+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:37:18.282295+0000","last_peered":"2023-09-23T23:37:18.282295+0000","last_clean":"2023-09-23T23:37:18.282295+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:37:18.282295+0000","last_undegraded":"2023-09-23T23:37:18.282295+0000","last_fullsized":"2023-09-23T23:37:18.282295+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":452,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290662+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:37:18.290662+0000","last_peered":"2023-09-23T23:37:18.290662+0000","last_clean":"2023-09-23T23:37:18.290662+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:37:18.290662+0000","last_undegraded":"2023-09-23T23:37:18.290662+0000","last_fullsized":"2023-09-23T23:37:18.290662+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"601'8426","reported_seq":15349,"reported_epoch":601,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:37:33.120251+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:37:33.120251+0000","last_peered":"2023-09-23T23:37:33.120251+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:37:33.120251+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:37:33.120251+0000","mapping_epoch":571,"log_start":"551'3100","ondisk_log_start":"551'3100","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5326,"log_dups_size":0,"ondisk_log_size":5326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171835392,"num_objects":2622,"num_object_clones":0,"num_object_copies":7866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1727,"num_objects_misplaced":895,"num_objects_unfound":0,"num_objects_dirty":2622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8426,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3404,"num_bytes_recovered":223084544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":1605},{"shards":"3,5,7","objects":1017}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":911,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.284071+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:37:18.284071+0000","last_peered":"2023-09-23T23:37:18.284071+0000","last_clean":"2023-09-23T23:37:18.284071+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:37:18.284071+0000","last_undegraded":"2023-09-23T23:37:18.284071+0000","last_fullsized":"2023-09-23T23:37:18.284071+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":339,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.291692+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:37:18.291692+0000","last_peered":"2023-09-23T23:37:18.291692+0000","last_clean":"2023-09-23T23:37:18.291692+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:37:18.291692+0000","last_undegraded":"2023-09-23T23:37:18.291692+0000","last_fullsized":"2023-09-23T23:37:18.291692+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":115,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.284208+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:37:18.284208+0000","last_peered":"2023-09-23T23:37:18.284208+0000","last_clean":"2023-09-23T23:37:18.284208+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:37:18.284208+0000","last_undegraded":"2023-09-23T23:37:18.284208+0000","last_fullsized":"2023-09-23T23:37:18.284208+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"601'8396","reported_seq":10467,"reported_epoch":601,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:33.110268+0000","last_change":"2023-09-23T23:34:35.750428+0000","last_active":"2023-09-23T23:37:33.110268+0000","last_peered":"2023-09-23T23:37:33.110268+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:37:33.110268+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"551'3100","ondisk_log_start":"551'3100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5296,"log_dups_size":0,"ondisk_log_size":5296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176553984,"num_objects":2694,"num_object_clones":0,"num_object_copies":8082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8396,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":991,"num_bytes_recovered":64946176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":1684},{"shards":"3,5","objects":-850}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":508,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.289159+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:37:18.289159+0000","last_peered":"2023-09-23T23:37:18.289159+0000","last_clean":"2023-09-23T23:37:18.289159+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:37:18.289159+0000","last_undegraded":"2023-09-23T23:37:18.289159+0000","last_fullsized":"2023-09-23T23:37:18.289159+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":442,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.287917+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:37:18.287917+0000","last_peered":"2023-09-23T23:37:18.287917+0000","last_clean":"2023-09-23T23:37:18.287917+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:37:18.287917+0000","last_undegraded":"2023-09-23T23:37:18.287917+0000","last_fullsized":"2023-09-23T23:37:18.287917+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":599,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.403083+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:37:26.403083+0000","last_peered":"2023-09-23T23:37:26.403083+0000","last_clean":"2023-09-23T23:37:26.403083+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:37:26.403083+0000","last_undegraded":"2023-09-23T23:37:26.403083+0000","last_fullsized":"2023-09-23T23:37:26.403083+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"601'8552","reported_seq":9882,"reported_epoch":601,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:34.462568+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:37:34.462568+0000","last_peered":"2023-09-23T23:37:34.462568+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:37:34.462568+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"551'2800","ondisk_log_start":"551'2800","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5752,"log_dups_size":0,"ondisk_log_size":5752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171048960,"num_objects":2610,"num_object_clones":0,"num_object_copies":7830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8552,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":1564},{"shards":"3,5","objects":-2850},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":729,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290954+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:37:18.290954+0000","last_peered":"2023-09-23T23:37:18.290954+0000","last_clean":"2023-09-23T23:37:18.290954+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:37:18.290954+0000","last_undegraded":"2023-09-23T23:37:18.290954+0000","last_fullsized":"2023-09-23T23:37:18.290954+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":611,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290937+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:37:18.290937+0000","last_peered":"2023-09-23T23:37:18.290937+0000","last_clean":"2023-09-23T23:37:18.290937+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:37:18.290937+0000","last_undegraded":"2023-09-23T23:37:18.290937+0000","last_fullsized":"2023-09-23T23:37:18.290937+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":522,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.287839+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:37:18.287839+0000","last_peered":"2023-09-23T23:37:18.287839+0000","last_clean":"2023-09-23T23:37:18.287839+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:37:18.287839+0000","last_undegraded":"2023-09-23T23:37:18.287839+0000","last_fullsized":"2023-09-23T23:37:18.287839+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"601'8460","reported_seq":8535,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.187192+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:37:34.187192+0000","last_peered":"2023-09-23T23:37:34.187192+0000","last_clean":"2023-09-23T23:37:34.187192+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:37:34.187192+0000","last_undegraded":"2023-09-23T23:37:34.187192+0000","last_fullsized":"2023-09-23T23:37:34.187192+0000","mapping_epoch":534,"log_start":"551'3100","ondisk_log_start":"551'3100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5360,"log_dups_size":0,"ondisk_log_size":5360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171704320,"num_objects":2620,"num_object_clones":0,"num_object_copies":7860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8460,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":885,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.399352+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:37:26.399352+0000","last_peered":"2023-09-23T23:37:26.399352+0000","last_clean":"2023-09-23T23:37:26.399352+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:37:26.399352+0000","last_undegraded":"2023-09-23T23:37:26.399352+0000","last_fullsized":"2023-09-23T23:37:26.399352+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":125,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.284125+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:37:18.284125+0000","last_peered":"2023-09-23T23:37:18.284125+0000","last_clean":"2023-09-23T23:37:18.284125+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:37:18.284125+0000","last_undegraded":"2023-09-23T23:37:18.284125+0000","last_fullsized":"2023-09-23T23:37:18.284125+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":468,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.399740+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:37:26.399740+0000","last_peered":"2023-09-23T23:37:26.399740+0000","last_clean":"2023-09-23T23:37:26.399740+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:37:26.399740+0000","last_undegraded":"2023-09-23T23:37:26.399740+0000","last_fullsized":"2023-09-23T23:37:26.399740+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"601'8519","reported_seq":14548,"reported_epoch":601,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:37:34.188147+0000","last_change":"2023-09-23T23:34:35.785270+0000","last_active":"2023-09-23T23:37:34.188147+0000","last_peered":"2023-09-23T23:37:34.188147+0000","last_clean":"2023-09-23T23:33:50.504296+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:37:34.188147+0000","last_undegraded":"2023-09-23T23:34:35.780981+0000","last_fullsized":"2023-09-23T23:37:34.188147+0000","mapping_epoch":567,"log_start":"551'3100","ondisk_log_start":"551'3100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5419,"log_dups_size":0,"ondisk_log_size":5419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169410560,"num_objects":2585,"num_object_clones":0,"num_object_copies":7755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1580,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8519,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2967,"num_bytes_recovered":194445312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":1580},{"shards":"5,6,7","objects":1005}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":609,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.292749+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:37:18.292749+0000","last_peered":"2023-09-23T23:37:18.292749+0000","last_clean":"2023-09-23T23:37:18.292749+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:37:18.292749+0000","last_undegraded":"2023-09-23T23:37:18.292749+0000","last_fullsized":"2023-09-23T23:37:18.292749+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":465,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.399400+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:37:26.399400+0000","last_peered":"2023-09-23T23:37:26.399400+0000","last_clean":"2023-09-23T23:37:26.399400+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:37:26.399400+0000","last_undegraded":"2023-09-23T23:37:26.399400+0000","last_fullsized":"2023-09-23T23:37:26.399400+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":552,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.285809+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:37:18.285809+0000","last_peered":"2023-09-23T23:37:18.285809+0000","last_clean":"2023-09-23T23:37:18.285809+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:37:18.285809+0000","last_undegraded":"2023-09-23T23:37:18.285809+0000","last_fullsized":"2023-09-23T23:37:18.285809+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"601'8415","reported_seq":17453,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.425854+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:37:34.425854+0000","last_peered":"2023-09-23T23:37:34.425854+0000","last_clean":"2023-09-23T23:37:34.425854+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:37:34.425854+0000","last_undegraded":"2023-09-23T23:37:34.425854+0000","last_fullsized":"2023-09-23T23:37:34.425854+0000","mapping_epoch":579,"log_start":"551'2700","ondisk_log_start":"551'2700","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5715,"log_dups_size":0,"ondisk_log_size":5715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170459136,"num_objects":2601,"num_object_clones":0,"num_object_copies":7803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8415,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":944,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.400745+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:37:26.400745+0000","last_peered":"2023-09-23T23:37:26.400745+0000","last_clean":"2023-09-23T23:37:26.400745+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:37:26.400745+0000","last_undegraded":"2023-09-23T23:37:26.400745+0000","last_fullsized":"2023-09-23T23:37:26.400745+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":243,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.287508+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:37:18.287508+0000","last_peered":"2023-09-23T23:37:18.287508+0000","last_clean":"2023-09-23T23:37:18.287508+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:37:18.287508+0000","last_undegraded":"2023-09-23T23:37:18.287508+0000","last_fullsized":"2023-09-23T23:37:18.287508+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":531,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.288728+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:37:18.288728+0000","last_peered":"2023-09-23T23:37:18.288728+0000","last_clean":"2023-09-23T23:37:18.288728+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:37:18.288728+0000","last_undegraded":"2023-09-23T23:37:18.288728+0000","last_fullsized":"2023-09-23T23:37:18.288728+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"601'8082","reported_seq":8160,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:33.149018+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:37:33.149018+0000","last_peered":"2023-09-23T23:37:33.149018+0000","last_clean":"2023-09-23T23:37:33.149018+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:37:33.149018+0000","last_undegraded":"2023-09-23T23:37:33.149018+0000","last_fullsized":"2023-09-23T23:37:33.149018+0000","mapping_epoch":534,"log_start":"551'2800","ondisk_log_start":"551'2800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5282,"log_dups_size":0,"ondisk_log_size":5282,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173408256,"num_objects":2646,"num_object_clones":0,"num_object_copies":7938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8082,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":922,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283904+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:37:18.283904+0000","last_peered":"2023-09-23T23:37:18.283904+0000","last_clean":"2023-09-23T23:37:18.283904+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:37:18.283904+0000","last_undegraded":"2023-09-23T23:37:18.283904+0000","last_fullsized":"2023-09-23T23:37:18.283904+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":311,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.277973+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:37:18.277973+0000","last_peered":"2023-09-23T23:37:18.277973+0000","last_clean":"2023-09-23T23:37:18.277973+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:37:18.277973+0000","last_undegraded":"2023-09-23T23:37:18.277973+0000","last_fullsized":"2023-09-23T23:37:18.277973+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":616,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.291785+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:37:18.291785+0000","last_peered":"2023-09-23T23:37:18.291785+0000","last_clean":"2023-09-23T23:37:18.291785+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:37:18.291785+0000","last_undegraded":"2023-09-23T23:37:18.291785+0000","last_fullsized":"2023-09-23T23:37:18.291785+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":197,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290247+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:37:18.290247+0000","last_peered":"2023-09-23T23:37:18.290247+0000","last_clean":"2023-09-23T23:37:18.290247+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:37:18.290247+0000","last_undegraded":"2023-09-23T23:37:18.290247+0000","last_fullsized":"2023-09-23T23:37:18.290247+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"601'8470","reported_seq":14340,"reported_epoch":601,"state":"active+recovering","last_fresh":"2023-09-23T23:37:33.133314+0000","last_change":"2023-09-23T23:34:35.815566+0000","last_active":"2023-09-23T23:37:33.133314+0000","last_peered":"2023-09-23T23:37:33.133314+0000","last_clean":"2023-09-23T23:33:50.519308+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:37:33.133314+0000","last_undegraded":"2023-09-23T23:37:33.133314+0000","last_fullsized":"2023-09-23T23:37:33.133314+0000","mapping_epoch":566,"log_start":"551'3122","ondisk_log_start":"551'3122","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5348,"log_dups_size":0,"ondisk_log_size":5348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176750616,"num_objects":2698,"num_object_clones":0,"num_object_copies":8094,"num_objects_missing_on_primary":1627,"num_objects_missing":1627,"num_objects_degraded":0,"num_objects_misplaced":1627,"num_objects_unfound":0,"num_objects_dirty":2698,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8470,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2885,"num_bytes_recovered":189071360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,3,4,7","objects":3},{"shards":"1,3,7","objects":1068},{"shards":"1,4,7","objects":1627}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":686,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.401728+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:37:26.401728+0000","last_peered":"2023-09-23T23:37:26.401728+0000","last_clean":"2023-09-23T23:37:26.401728+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:37:26.401728+0000","last_undegraded":"2023-09-23T23:37:26.401728+0000","last_fullsized":"2023-09-23T23:37:26.401728+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":249,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.284045+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:37:18.284045+0000","last_peered":"2023-09-23T23:37:18.284045+0000","last_clean":"2023-09-23T23:37:18.284045+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:37:18.284045+0000","last_undegraded":"2023-09-23T23:37:18.284045+0000","last_fullsized":"2023-09-23T23:37:18.284045+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":533,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283689+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:37:18.283689+0000","last_peered":"2023-09-23T23:37:18.283689+0000","last_clean":"2023-09-23T23:37:18.283689+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:37:18.283689+0000","last_undegraded":"2023-09-23T23:37:18.283689+0000","last_fullsized":"2023-09-23T23:37:18.283689+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":235,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.400200+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:37:26.400200+0000","last_peered":"2023-09-23T23:37:26.400200+0000","last_clean":"2023-09-23T23:37:26.400200+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:37:26.400200+0000","last_undegraded":"2023-09-23T23:37:26.400200+0000","last_fullsized":"2023-09-23T23:37:26.400200+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"601'8155","reported_seq":10209,"reported_epoch":601,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:34.475835+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:37:34.475835+0000","last_peered":"2023-09-23T23:37:34.475835+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:37:34.475835+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"550'2400","ondisk_log_start":"550'2400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5755,"log_dups_size":0,"ondisk_log_size":5755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171507712,"num_objects":2617,"num_object_clones":0,"num_object_copies":7851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8155,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":1606},{"shards":"5,7","objects":-775}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":904,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.400657+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:37:26.400657+0000","last_peered":"2023-09-23T23:37:26.400657+0000","last_clean":"2023-09-23T23:37:26.400657+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:37:26.400657+0000","last_undegraded":"2023-09-23T23:37:26.400657+0000","last_fullsized":"2023-09-23T23:37:26.400657+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":341,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.399306+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:37:26.399306+0000","last_peered":"2023-09-23T23:37:26.399306+0000","last_clean":"2023-09-23T23:37:26.399306+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:37:26.399306+0000","last_undegraded":"2023-09-23T23:37:26.399306+0000","last_fullsized":"2023-09-23T23:37:26.399306+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":250,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.399756+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:37:26.399756+0000","last_peered":"2023-09-23T23:37:26.399756+0000","last_clean":"2023-09-23T23:37:26.399756+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:37:26.399756+0000","last_undegraded":"2023-09-23T23:37:26.399756+0000","last_fullsized":"2023-09-23T23:37:26.399756+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":213,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.277475+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:37:18.277475+0000","last_peered":"2023-09-23T23:37:18.277475+0000","last_clean":"2023-09-23T23:37:18.277475+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:37:18.277475+0000","last_undegraded":"2023-09-23T23:37:18.277475+0000","last_fullsized":"2023-09-23T23:37:18.277475+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"601'8377","reported_seq":8461,"reported_epoch":601,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:37:34.499164+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:37:34.499164+0000","last_peered":"2023-09-23T23:37:34.499164+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:37:34.499164+0000","last_undegraded":"2023-09-23T23:37:34.499164+0000","last_fullsized":"2023-09-23T23:37:34.499164+0000","mapping_epoch":562,"log_start":"551'2700","ondisk_log_start":"551'2700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5677,"log_dups_size":0,"ondisk_log_size":5677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165478400,"num_objects":2525,"num_object_clones":0,"num_object_copies":7575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8377,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":2525}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":617,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.399483+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:37:26.399483+0000","last_peered":"2023-09-23T23:37:26.399483+0000","last_clean":"2023-09-23T23:37:26.399483+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:37:26.399483+0000","last_undegraded":"2023-09-23T23:37:26.399483+0000","last_fullsized":"2023-09-23T23:37:26.399483+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":887,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283331+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:37:18.283331+0000","last_peered":"2023-09-23T23:37:18.283331+0000","last_clean":"2023-09-23T23:37:18.283331+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:37:18.283331+0000","last_undegraded":"2023-09-23T23:37:18.283331+0000","last_fullsized":"2023-09-23T23:37:18.283331+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":563,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.293189+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:37:18.293189+0000","last_peered":"2023-09-23T23:37:18.293189+0000","last_clean":"2023-09-23T23:37:18.293189+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:37:18.293189+0000","last_undegraded":"2023-09-23T23:37:18.293189+0000","last_fullsized":"2023-09-23T23:37:18.293189+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":125,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.285739+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:37:18.285739+0000","last_peered":"2023-09-23T23:37:18.285739+0000","last_clean":"2023-09-23T23:37:18.285739+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:37:18.285739+0000","last_undegraded":"2023-09-23T23:37:18.285739+0000","last_fullsized":"2023-09-23T23:37:18.285739+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"601'8139","reported_seq":8214,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.214529+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:37:34.214529+0000","last_peered":"2023-09-23T23:37:34.214529+0000","last_clean":"2023-09-23T23:37:34.214529+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:37:34.214529+0000","last_undegraded":"2023-09-23T23:37:34.214529+0000","last_fullsized":"2023-09-23T23:37:34.214529+0000","mapping_epoch":534,"log_start":"551'2700","ondisk_log_start":"551'2700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5439,"log_dups_size":0,"ondisk_log_size":5439,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170196992,"num_objects":2597,"num_object_clones":0,"num_object_copies":7791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8139,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":713,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283523+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:37:18.283523+0000","last_peered":"2023-09-23T23:37:18.283523+0000","last_clean":"2023-09-23T23:37:18.283523+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:37:18.283523+0000","last_undegraded":"2023-09-23T23:37:18.283523+0000","last_fullsized":"2023-09-23T23:37:18.283523+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":528,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.280933+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:37:18.280933+0000","last_peered":"2023-09-23T23:37:18.280933+0000","last_clean":"2023-09-23T23:37:18.280933+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:37:18.280933+0000","last_undegraded":"2023-09-23T23:37:18.280933+0000","last_fullsized":"2023-09-23T23:37:18.280933+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":442,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290744+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:37:18.290744+0000","last_peered":"2023-09-23T23:37:18.290744+0000","last_clean":"2023-09-23T23:37:18.290744+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:37:18.290744+0000","last_undegraded":"2023-09-23T23:37:18.290744+0000","last_fullsized":"2023-09-23T23:37:18.290744+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":43,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.402227+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:37:26.402227+0000","last_peered":"2023-09-23T23:37:26.402227+0000","last_clean":"2023-09-23T23:37:26.402227+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:37:26.402227+0000","last_undegraded":"2023-09-23T23:37:26.402227+0000","last_fullsized":"2023-09-23T23:37:26.402227+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"602'8452","reported_seq":12128,"reported_epoch":602,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:34.924737+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:37:34.924737+0000","last_peered":"2023-09-23T23:37:34.924737+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:37:34.924737+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"550'2300","ondisk_log_start":"550'2300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6152,"log_dups_size":0,"ondisk_log_size":6152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166854656,"num_objects":2546,"num_object_clones":0,"num_object_copies":7638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8452,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":-158},{"shards":"1,2,4","objects":1544}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":596,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.286019+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:37:18.286019+0000","last_peered":"2023-09-23T23:37:18.286019+0000","last_clean":"2023-09-23T23:37:18.286019+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:37:18.286019+0000","last_undegraded":"2023-09-23T23:37:18.286019+0000","last_fullsized":"2023-09-23T23:37:18.286019+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":43,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.402630+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:37:26.402630+0000","last_peered":"2023-09-23T23:37:26.402630+0000","last_clean":"2023-09-23T23:37:26.402630+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:37:26.402630+0000","last_undegraded":"2023-09-23T23:37:26.402630+0000","last_fullsized":"2023-09-23T23:37:26.402630+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":142,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.400263+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:37:26.400263+0000","last_peered":"2023-09-23T23:37:26.400263+0000","last_clean":"2023-09-23T23:37:26.400263+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:37:26.400263+0000","last_undegraded":"2023-09-23T23:37:26.400263+0000","last_fullsized":"2023-09-23T23:37:26.400263+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":334,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.292967+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:37:18.292967+0000","last_peered":"2023-09-23T23:37:18.292967+0000","last_clean":"2023-09-23T23:37:18.292967+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:37:18.292967+0000","last_undegraded":"2023-09-23T23:37:18.292967+0000","last_fullsized":"2023-09-23T23:37:18.292967+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"601'8374","reported_seq":8449,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.425753+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:37:34.425753+0000","last_peered":"2023-09-23T23:37:34.425753+0000","last_clean":"2023-09-23T23:37:34.425753+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:37:34.425753+0000","last_undegraded":"2023-09-23T23:37:34.425753+0000","last_fullsized":"2023-09-23T23:37:34.425753+0000","mapping_epoch":534,"log_start":"551'2700","ondisk_log_start":"551'2700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5674,"log_dups_size":0,"ondisk_log_size":5674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169738240,"num_objects":2590,"num_object_clones":0,"num_object_copies":7770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8374,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":783,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.399820+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:37:26.399820+0000","last_peered":"2023-09-23T23:37:26.399820+0000","last_clean":"2023-09-23T23:37:26.399820+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:37:26.399820+0000","last_undegraded":"2023-09-23T23:37:26.399820+0000","last_fullsized":"2023-09-23T23:37:26.399820+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":599,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.288656+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:37:18.288656+0000","last_peered":"2023-09-23T23:37:18.288656+0000","last_clean":"2023-09-23T23:37:18.288656+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:37:18.288656+0000","last_undegraded":"2023-09-23T23:37:18.288656+0000","last_fullsized":"2023-09-23T23:37:18.288656+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":205,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.285866+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:37:18.285866+0000","last_peered":"2023-09-23T23:37:18.285866+0000","last_clean":"2023-09-23T23:37:18.285866+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:37:18.285866+0000","last_undegraded":"2023-09-23T23:37:18.285866+0000","last_fullsized":"2023-09-23T23:37:18.285866+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":154,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.399242+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:37:26.399242+0000","last_peered":"2023-09-23T23:37:26.399242+0000","last_clean":"2023-09-23T23:37:26.399242+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:37:26.399242+0000","last_undegraded":"2023-09-23T23:37:26.399242+0000","last_fullsized":"2023-09-23T23:37:26.399242+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"601'8302","reported_seq":8381,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.748851+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:37:34.748851+0000","last_peered":"2023-09-23T23:37:34.748851+0000","last_clean":"2023-09-23T23:37:34.748851+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:37:34.748851+0000","last_undegraded":"2023-09-23T23:37:34.748851+0000","last_fullsized":"2023-09-23T23:37:34.748851+0000","mapping_epoch":534,"log_start":"550'2100","ondisk_log_start":"550'2100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6202,"log_dups_size":0,"ondisk_log_size":6202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170786816,"num_objects":2606,"num_object_clones":0,"num_object_copies":7818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8302,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":988,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.288160+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:37:18.288160+0000","last_peered":"2023-09-23T23:37:18.288160+0000","last_clean":"2023-09-23T23:37:18.288160+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:37:18.288160+0000","last_undegraded":"2023-09-23T23:37:18.288160+0000","last_fullsized":"2023-09-23T23:37:18.288160+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":115,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283930+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:37:18.283930+0000","last_peered":"2023-09-23T23:37:18.283930+0000","last_clean":"2023-09-23T23:37:18.283930+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:37:18.283930+0000","last_undegraded":"2023-09-23T23:37:18.283930+0000","last_fullsized":"2023-09-23T23:37:18.283930+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":47,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.402271+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:37:26.402271+0000","last_peered":"2023-09-23T23:37:26.402271+0000","last_clean":"2023-09-23T23:37:26.402271+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:37:26.402271+0000","last_undegraded":"2023-09-23T23:37:26.402271+0000","last_fullsized":"2023-09-23T23:37:26.402271+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":175,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290045+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:37:18.290045+0000","last_peered":"2023-09-23T23:37:18.290045+0000","last_clean":"2023-09-23T23:37:18.290045+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:37:18.290045+0000","last_undegraded":"2023-09-23T23:37:18.290045+0000","last_fullsized":"2023-09-23T23:37:18.290045+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1038,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.393662+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:37:26.393662+0000","last_peered":"2023-09-23T23:37:26.393662+0000","last_clean":"2023-09-23T23:37:26.393662+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:37:26.393662+0000","last_undegraded":"2023-09-23T23:37:26.393662+0000","last_fullsized":"2023-09-23T23:37:26.393662+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":43,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.403170+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:37:26.403170+0000","last_peered":"2023-09-23T23:37:26.403170+0000","last_clean":"2023-09-23T23:37:26.403170+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:37:26.403170+0000","last_undegraded":"2023-09-23T23:37:26.403170+0000","last_fullsized":"2023-09-23T23:37:26.403170+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":357,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.400305+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:37:26.400305+0000","last_peered":"2023-09-23T23:37:26.400305+0000","last_clean":"2023-09-23T23:37:26.400305+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:37:26.400305+0000","last_undegraded":"2023-09-23T23:37:26.400305+0000","last_fullsized":"2023-09-23T23:37:26.400305+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":749,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.289384+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:37:18.289384+0000","last_peered":"2023-09-23T23:37:18.289384+0000","last_clean":"2023-09-23T23:37:18.289384+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:37:18.289384+0000","last_undegraded":"2023-09-23T23:37:18.289384+0000","last_fullsized":"2023-09-23T23:37:18.289384+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":103,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283558+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:37:18.283558+0000","last_peered":"2023-09-23T23:37:18.283558+0000","last_clean":"2023-09-23T23:37:18.283558+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:37:18.283558+0000","last_undegraded":"2023-09-23T23:37:18.283558+0000","last_fullsized":"2023-09-23T23:37:18.283558+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":449,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.290854+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:37:18.290854+0000","last_peered":"2023-09-23T23:37:18.290854+0000","last_clean":"2023-09-23T23:37:18.290854+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:37:18.290854+0000","last_undegraded":"2023-09-23T23:37:18.290854+0000","last_fullsized":"2023-09-23T23:37:18.290854+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":299,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.285619+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:37:18.285619+0000","last_peered":"2023-09-23T23:37:18.285619+0000","last_clean":"2023-09-23T23:37:18.285619+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:37:18.285619+0000","last_undegraded":"2023-09-23T23:37:18.285619+0000","last_fullsized":"2023-09-23T23:37:18.285619+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":288,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.293254+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:37:18.293254+0000","last_peered":"2023-09-23T23:37:18.293254+0000","last_clean":"2023-09-23T23:37:18.293254+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:37:18.293254+0000","last_undegraded":"2023-09-23T23:37:18.293254+0000","last_fullsized":"2023-09-23T23:37:18.293254+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":813,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283743+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:37:18.283743+0000","last_peered":"2023-09-23T23:37:18.283743+0000","last_clean":"2023-09-23T23:37:18.283743+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:37:18.283743+0000","last_undegraded":"2023-09-23T23:37:18.283743+0000","last_fullsized":"2023-09-23T23:37:18.283743+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":632,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.289306+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:37:18.289306+0000","last_peered":"2023-09-23T23:37:18.289306+0000","last_clean":"2023-09-23T23:37:18.289306+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:37:18.289306+0000","last_undegraded":"2023-09-23T23:37:18.289306+0000","last_fullsized":"2023-09-23T23:37:18.289306+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":615,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.291133+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:37:18.291133+0000","last_peered":"2023-09-23T23:37:18.291133+0000","last_clean":"2023-09-23T23:37:18.291133+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:37:18.291133+0000","last_undegraded":"2023-09-23T23:37:18.291133+0000","last_fullsized":"2023-09-23T23:37:18.291133+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":774,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.300198+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:37:18.300198+0000","last_peered":"2023-09-23T23:37:18.300198+0000","last_clean":"2023-09-23T23:37:18.300198+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:37:18.300198+0000","last_undegraded":"2023-09-23T23:37:18.300198+0000","last_fullsized":"2023-09-23T23:37:18.300198+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":446,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T23:37:26.399649+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:37:26.399649+0000","last_peered":"2023-09-23T23:37:26.399649+0000","last_clean":"2023-09-23T23:37:26.399649+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:37:26.399649+0000","last_undegraded":"2023-09-23T23:37:26.399649+0000","last_fullsized":"2023-09-23T23:37:26.399649+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":668,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T23:37:18.283248+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:37:18.283248+0000","last_peered":"2023-09-23T23:37:18.283248+0000","last_clean":"2023-09-23T23:37:18.283248+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:37:18.283248+0000","last_undegraded":"2023-09-23T23:37:18.283248+0000","last_fullsized":"2023-09-23T23:37:18.283248+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":2738225176,"num_objects":41783,"num_object_clones":0,"num_object_copies":125349,"num_objects_missing_on_primary":1627,"num_objects_missing":1627,"num_objects_degraded":22923,"num_objects_misplaced":2522,"num_objects_unfound":0,"num_objects_dirty":41783,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":133773,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18104,"num_bytes_recovered":1186267136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9734270976,"data_stored":9734258760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90051,"ondisk_log_size":90051,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782697,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1442172,"kb_used_data":1050548,"kb_used_omap":8,"kb_used_meta":391607,"kb_avail":92302980,"statfs":{"total":95995035648,"available":94518251520,"internally_reserved":0,"allocated":1075761152,"data_stored":1074083587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":401005637},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":1.048,"15min":2.1080000000000001},"min":{"1min":0.25900000000000001,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":0.51300000000000001,"5min":17.140000000000001,"15min":85.721000000000004},"last":0.307},{"interface":"front","average":{"1min":0.47199999999999998,"5min":1.0600000000000001,"15min":2.1829999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":2.331,"5min":11.069000000000001,"15min":85.965000000000003},"last":0.26800000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":2.532,"15min":3.1309999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":1.5780000000000001,"5min":129.11199999999999,"15min":198.58000000000001},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.53200000000000003,"5min":2.3039999999999998,"15min":3.1440000000000001},"min":{"1min":0.31900000000000001,"5min":0.247,"15min":0.246},"max":{"1min":2.3690000000000002,"5min":129.02600000000001,"15min":198.71100000000001},"last":0.29999999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":3.3980000000000001,"5min":2.6880000000000002,"15min":3.8050000000000002},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":53.054000000000002,"5min":78.528999999999996,"15min":203.58000000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":3.5699999999999998,"5min":1.9299999999999999,"15min":3.2229999999999999},"min":{"1min":0.33400000000000002,"5min":0.27900000000000003,"15min":0.26200000000000001},"max":{"1min":53.023000000000003,"5min":53.023000000000003,"15min":198.69},"last":0.372}]},{"osd":3,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":8.6839999999999993,"5min":4.4390000000000001,"15min":4.0750000000000002},"min":{"1min":0.33300000000000002,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":100.40600000000001,"5min":125.511,"15min":279.26499999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":8.7420000000000009,"5min":4.8460000000000001,"15min":4.6180000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29599999999999999},"max":{"1min":100.42400000000001,"5min":125.48999999999999,"15min":279.29899999999998},"last":0.38400000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":1.546,"15min":1.0389999999999999},"min":{"1min":0.28999999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":0.58099999999999996,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":1.341,"15min":0.97599999999999998},"min":{"1min":0.32500000000000001,"5min":0.317,"15min":0.23899999999999999},"max":{"1min":2.3980000000000001,"5min":31.68,"15min":31.68},"last":0.35999999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.6220000000000001,"15min":1.863},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.26500000000000001},"max":{"1min":2.4510000000000001,"5min":60.463999999999999,"15min":210.83600000000001},"last":119.417},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.5840000000000001,"15min":1.742},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.5940000000000001,"5min":63.006,"15min":207.04599999999999},"last":119.61499999999999}]},{"osd":6,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.827,"15min":1.3320000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.29899999999999999},"max":{"1min":1.619,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":2.052,"15min":1.544},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.389,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.41799999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496174,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1684204,"kb_used_data":974244,"kb_used_omap":0,"kb_used_meta":709952,"kb_avail":92060948,"statfs":{"total":95995035648,"available":94270410752,"internally_reserved":0,"allocated":997625856,"data_stored":995964284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":726990848},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.73499999999999999,"15min":4.4370000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.21099999999999999},"max":{"1min":0.53700000000000003,"5min":6.4450000000000003,"15min":135.31999999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.498,"5min":0.63800000000000001,"15min":4.3869999999999996},"min":{"1min":0.27800000000000002,"5min":0.24399999999999999,"15min":0.23000000000000001},"max":{"1min":1.742,"5min":3.8239999999999998,"15min":135.34299999999999},"last":2.79}]},{"osd":1,"last update":"Sat Sep 23 23:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.90500000000000003,"15min":2.2890000000000001},"min":{"1min":0.28299999999999997,"5min":0.253,"15min":0.245},"max":{"1min":1.8759999999999999,"5min":6.9080000000000004,"15min":89.605000000000004},"last":0.441},{"interface":"front","average":{"1min":0.749,"5min":0.72499999999999998,"15min":2.391},"min":{"1min":0.29099999999999998,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":3.8980000000000001,"5min":3.8980000000000001,"15min":91.025999999999996},"last":1.341}]},{"osd":2,"last update":"Sat Sep 23 23:37:06 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":3.6930000000000001,"15min":4.4749999999999996},"min":{"1min":0.27900000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.1800000000000002,"5min":67.796999999999997,"15min":127.44499999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.66600000000000004,"5min":3.6749999999999998,"15min":4.8680000000000003},"min":{"1min":0.316,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.2080000000000002,"5min":67.893000000000001,"15min":125.539},"last":1.6739999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:36:55 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":5.6589999999999998,"15min":4.3159999999999998},"min":{"1min":0.30299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.3610000000000002,"5min":87.283000000000001,"15min":138.87299999999999},"last":1.228},{"interface":"front","average":{"1min":0.47699999999999998,"5min":5.516,"15min":4.625},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0349999999999999,"5min":87.491,"15min":138.976},"last":1.782}]},{"osd":4,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.755,"15min":0.66700000000000004},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.23400000000000001},"max":{"1min":1.76,"5min":7.101,"15min":8.1999999999999993},"last":0.318},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.81399999999999995,"15min":0.76900000000000002},"min":{"1min":0.26700000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.865,"5min":8.3499999999999996,"15min":8.3499999999999996},"last":0.373}]},{"osd":5,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.76100000000000001,"15min":0.79800000000000004},"min":{"1min":0.27700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.847,"5min":11.372,"15min":11.473000000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.876,"15min":0.84999999999999998},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.26900000000000002},"max":{"1min":6.1500000000000004,"5min":7.0129999999999999,"15min":7.5670000000000002},"last":0.307}]},{"osd":7,"last update":"Sat Sep 23 23:37:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.1679999999999999,"15min":1.0029999999999999},"min":{"1min":0.32700000000000001,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":0.77000000000000002,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.204,"15min":1.056},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33500000000000002},"max":{"1min":1.792,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.54000000000000004}]}]},{"osd":1,"up_from":112,"seq":481036337515,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1754036,"kb_used_data":853292,"kb_used_omap":4,"kb_used_meta":900731,"kb_avail":91991116,"statfs":{"total":95995035648,"available":94198902784,"internally_reserved":0,"allocated":873771008,"data_stored":870934032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":922349454},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.407,"15min":3.173},"min":{"1min":0.19,"5min":0.19,"15min":0.16700000000000001},"max":{"1min":3.5830000000000002,"5min":47.195999999999998,"15min":130.768},"last":0.192},{"interface":"front","average":{"1min":0.377,"5min":1.119,"15min":2.5030000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19},"max":{"1min":1.143,"5min":46.582000000000001,"15min":134.929},"last":0.26600000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":3.5659999999999998,"15min":6.7649999999999997},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":1.401,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.308},{"interface":"front","average":{"1min":0.66900000000000004,"5min":3.8849999999999998,"15min":6.8339999999999996},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.23,"5min":202.12,"15min":232.21000000000001},"last":0.23300000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":6.883,"5min":5.8460000000000001,"15min":5.4379999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22600000000000001},"max":{"1min":71.745000000000005,"5min":181.94399999999999,"15min":234.404},"last":0.28699999999999998},{"interface":"front","average":{"1min":6.8639999999999999,"5min":5.8860000000000001,"15min":5.3899999999999997},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":71.786000000000001,"5min":181.98500000000001,"15min":236.33699999999999},"last":0.245}]},{"osd":4,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":0.95099999999999996,"15min":1.399},"min":{"1min":0.32500000000000001,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":4.4880000000000004,"5min":8.0850000000000009,"15min":12.042999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.85699999999999998,"15min":1.45},"min":{"1min":0.316,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":4.6660000000000004,"5min":9.5069999999999997,"15min":18.108000000000001},"last":0.34999999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":2.6309999999999998,"15min":1.734},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28899999999999998},"max":{"1min":0.98199999999999998,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.024,"5min":2.661,"15min":2.0710000000000002},"min":{"1min":0.34399999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.194,"5min":151.821,"15min":151.821},"last":0.29599999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.0449999999999999,"15min":1.6659999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.27500000000000002},"max":{"1min":2.306,"5min":8.9269999999999996,"15min":74.087999999999994},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.2849999999999999,"15min":1.929},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33100000000000002},"max":{"1min":3.29,"5min":8.1400000000000006,"15min":73.974999999999994},"last":0.41099999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":1.5449999999999999,"15min":2.359},"min":{"1min":0.40200000000000002,"5min":0.38700000000000001,"15min":0.32900000000000001},"max":{"1min":2.504,"5min":44.241,"15min":78.308000000000007},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.556,"15min":2.2730000000000001},"min":{"1min":0.41299999999999998,"5min":0.37,"15min":0.32700000000000001},"max":{"1min":3.258,"5min":43.920999999999999,"15min":77.204999999999998},"last":0.44500000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024339,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1445960,"kb_used_data":235712,"kb_used_omap":5,"kb_used_meta":1210234,"kb_avail":92299192,"statfs":{"total":95995035648,"available":94514372608,"internally_reserved":0,"allocated":241369088,"data_stored":238514656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1239280250},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:36:36 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":4.391,"15min":6.6539999999999999},"min":{"1min":0.219,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":1.7709999999999999,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.251},{"interface":"front","average":{"1min":0.43099999999999999,"5min":4.1289999999999996,"15min":5.7969999999999997},"min":{"1min":0.245,"5min":0.20200000000000001,"15min":0.17599999999999999},"max":{"1min":1.0249999999999999,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.26300000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":6.9610000000000003,"15min":7.3559999999999999},"min":{"1min":0.26200000000000001,"5min":0.214,"15min":0.17199999999999999},"max":{"1min":4.0899999999999999,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.84599999999999997,"5min":6.6799999999999997,"15min":7.3869999999999996},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.17599999999999999},"max":{"1min":3.468,"5min":283.161,"15min":283.161},"last":0.498}]},{"osd":3,"last update":"Sat Sep 23 23:36:36 2023","interfaces":[{"interface":"back","average":{"1min":5.9699999999999998,"5min":5.1630000000000003,"15min":5.6079999999999997},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":95.655000000000001,"5min":114.381,"15min":220.732},"last":0.312},{"interface":"front","average":{"1min":5.8300000000000001,"5min":5.2779999999999996,"15min":5.6669999999999998},"min":{"1min":0.26600000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":95.695999999999998,"5min":114.30500000000001,"15min":218.78200000000001},"last":0.30199999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.79900000000000004,"15min":2.3479999999999999},"min":{"1min":0.34399999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.60999999999999999,"5min":9.3140000000000001,"15min":138.81299999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.66800000000000004,"5min":1.02,"15min":3.0270000000000001},"min":{"1min":0.36799999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.661,"5min":9.3670000000000009,"15min":171.251},"last":0.39500000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.1850000000000001,"15min":2.7290000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":1.288,"5min":15.076000000000001,"15min":171.30699999999999},"last":1.9139999999999999},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.097,"15min":1.9930000000000001},"min":{"1min":0.33600000000000002,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":4.1710000000000003,"5min":6.5279999999999996,"15min":139.40799999999999},"last":0.50900000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":1.143,"15min":3.0760000000000001},"min":{"1min":0.374,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":12.654999999999999,"5min":12.654999999999999,"15min":171.33600000000001},"last":1.6719999999999999},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.155,"15min":3.2749999999999999},"min":{"1min":0.41199999999999998,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":4.6440000000000001,"5min":9.157,"15min":184.13499999999999},"last":1.661}]},{"osd":7,"last update":"Sat Sep 23 23:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":1.089,"15min":2.569},"min":{"1min":0.35099999999999998,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":2.6749999999999998,"5min":8.9510000000000005,"15min":138.77699999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.3260000000000001,"15min":2.508},"min":{"1min":0.373,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":4.2229999999999999,"5min":9.8870000000000005,"15min":139.37899999999999},"last":0.55400000000000005}]}]},{"osd":2,"up_from":199,"seq":854698492160,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2513848,"kb_used_data":1665840,"kb_used_omap":7,"kb_used_meta":847992,"kb_avail":91231304,"statfs":{"total":95995035648,"available":93420855296,"internally_reserved":0,"allocated":1705820160,"data_stored":1702923259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":868343889},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.7789999999999999,"15min":3.617},"min":{"1min":0.20200000000000001,"5min":0.186,"15min":0.16700000000000001},"max":{"1min":6.3940000000000001,"5min":61.950000000000003,"15min":119.277},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.54400000000000004,"5min":1.8740000000000001,"15min":3.46},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.18099999999999999},"max":{"1min":5.0339999999999998,"5min":61.984999999999999,"15min":119.56100000000001},"last":0.27100000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":4.4960000000000004,"15min":4.5449999999999999},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":4.7060000000000004,"5min":106.765,"15min":157.98699999999999},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.73899999999999999,"5min":4.1210000000000004,"15min":4.4950000000000001},"min":{"1min":0.23400000000000001,"5min":0.222,"15min":0.222},"max":{"1min":6.4340000000000002,"5min":106.732,"15min":158.03800000000001},"last":0.255}]},{"osd":3,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":5.3179999999999996,"15min":4.6900000000000004},"min":{"1min":0.26000000000000001,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":1.075,"5min":106.70399999999999,"15min":119.366},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.52100000000000002,"5min":5.0359999999999996,"15min":4.1440000000000001},"min":{"1min":0.246,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.6120000000000001,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.42199999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":1.756,"15min":2.1909999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.25},"max":{"1min":0.65600000000000003,"5min":62.081000000000003,"15min":119.88200000000001},"last":0.437},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.9710000000000001,"15min":1.829},"min":{"1min":0.42299999999999999,"5min":0.35599999999999998,"15min":0.28499999999999998},"max":{"1min":1.121,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.33400000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":2.8340000000000001,"15min":2.5840000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":2.6549999999999998,"5min":91.207999999999998,"15min":119.919},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.69399999999999995,"5min":2.919,"15min":2.5800000000000001},"min":{"1min":0.371,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.4289999999999998,"5min":91.150999999999996,"15min":119.306},"last":1.266}]},{"osd":6,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":3.254,"15min":2.9169999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":1.04,"5min":107.018,"15min":119.943},"last":1.1879999999999999},{"interface":"front","average":{"1min":0.746,"5min":2.1779999999999999,"15min":2.1160000000000001},"min":{"1min":0.45100000000000001,"5min":0.371,"15min":0.34999999999999998},"max":{"1min":3.4950000000000001,"5min":62.493000000000002,"15min":62.493000000000002},"last":0.375}]},{"osd":7,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.917,"15min":1.883},"min":{"1min":0.41699999999999998,"5min":0.374,"15min":0.34000000000000002},"max":{"1min":1.831,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.66800000000000004,"5min":3.1800000000000002,"15min":2.3220000000000001},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.36199999999999999},"max":{"1min":1.792,"5min":106.809,"15min":106.809},"last":0.35399999999999998}]}]},{"osd":3,"up_from":148,"seq":635655160128,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1851760,"kb_used_data":926248,"kb_used_omap":2,"kb_used_meta":925437,"kb_avail":91893392,"statfs":{"total":95995035648,"available":94098833408,"internally_reserved":0,"allocated":948477952,"data_stored":945642692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":947647541},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.1359999999999999,"15min":2.6019999999999999},"min":{"1min":0.184,"5min":0.184,"15min":0.17699999999999999},"max":{"1min":6.7699999999999996,"5min":17.213000000000001,"15min":103.866},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.72899999999999998,"5min":1.4139999999999999,"15min":2.855},"min":{"1min":0.218,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":3.4980000000000002,"5min":22.754000000000001,"15min":101.884},"last":0.255}]},{"osd":1,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":2.5960000000000001,"15min":2.903},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":1.649,"5min":56.119,"15min":85.242999999999995},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.42299999999999999,"5min":2.036,"15min":2.7469999999999999},"min":{"1min":0.254,"5min":0.224,"15min":0.20899999999999999},"max":{"1min":1.6990000000000001,"5min":53.962000000000003,"15min":85.302000000000007},"last":0.218}]},{"osd":2,"last update":"Sat Sep 23 23:36:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":2.2989999999999999,"15min":4.5999999999999996},"min":{"1min":0.25800000000000001,"5min":0.25,"15min":0.19800000000000001},"max":{"1min":14.965,"5min":56.226999999999997,"15min":101.279},"last":0.27700000000000002},{"interface":"front","average":{"1min":1.2210000000000001,"5min":2.165,"15min":4.4370000000000003},"min":{"1min":0.315,"5min":0.29299999999999998,"15min":0.214},"max":{"1min":15.058999999999999,"5min":54.280999999999999,"15min":101.294},"last":0.26800000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":0.68999999999999995,"5min":11.209,"15min":11.209},"last":0.318},{"interface":"front","average":{"1min":0.44500000000000001,"5min":1.3720000000000001,"15min":1.6719999999999999},"min":{"1min":0.33000000000000002,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":0.67900000000000005,"5min":11.494999999999999,"15min":24.782},"last":0.45000000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":6.3570000000000002,"15min":3.3220000000000001},"min":{"1min":0.33000000000000002,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":0.622,"5min":214.43700000000001,"15min":214.43700000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":6.3970000000000002,"15min":3.3969999999999998},"min":{"1min":0.36499999999999999,"5min":0.34699999999999998,"15min":0.30599999999999999},"max":{"1min":1.716,"5min":214.46600000000001,"15min":214.46600000000001},"last":0.40600000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":2.218,"15min":2.1739999999999999},"min":{"1min":0.35399999999999998,"5min":0.318,"15min":0.318},"max":{"1min":2.008,"5min":58.932000000000002,"15min":58.932000000000002},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.629,"5min":2.492,"15min":2.2210000000000001},"min":{"1min":0.36699999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.7450000000000001,"5min":59.186,"15min":59.186},"last":0.36499999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:36:56 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":4.5469999999999997,"15min":3.1829999999999998},"min":{"1min":0.42199999999999999,"5min":0.38700000000000001,"15min":0.32600000000000001},"max":{"1min":0.66700000000000004,"5min":192.69,"15min":192.69},"last":0.439},{"interface":"front","average":{"1min":0.54300000000000004,"5min":4.6280000000000001,"15min":3.649},"min":{"1min":0.40100000000000002,"5min":0.39200000000000002,"15min":0.36699999999999999},"max":{"1min":1.2769999999999999,"5min":192.65600000000001,"15min":192.65600000000001},"last":0.53100000000000003}]}]},{"osd":4,"up_from":0,"seq":128849019447,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757810,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2324648,"kb_used_data":1578784,"kb_used_omap":0,"kb_used_meta":745856,"kb_avail":91420504,"statfs":{"total":95995035648,"available":93614596096,"internally_reserved":0,"allocated":1616674816,"data_stored":1615013461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":763756544},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.032,"15min":2.597},"min":{"1min":0.25,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":2.9279999999999999,"5min":21.109000000000002,"15min":113.869},"last":0.23899999999999999},{"interface":"front","average":{"1min":1.123,"5min":1.407,"15min":3.0659999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":11.154999999999999,"5min":20.876999999999999,"15min":138.55199999999999},"last":0.28000000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:37:22 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.1459999999999999,"15min":2.1629999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.1360000000000001,"5min":29.007999999999999,"15min":69.816999999999993},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.212,"15min":2.3929999999999998},"min":{"1min":0.30499999999999999,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":1.452,"5min":29.053999999999998,"15min":70.268000000000001},"last":0.39100000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:36:48 2023","interfaces":[{"interface":"back","average":{"1min":5.194,"5min":6.984,"15min":4.8159999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.26900000000000002},"max":{"1min":60.441000000000003,"5min":208.738,"15min":208.738},"last":0.41299999999999998},{"interface":"front","average":{"1min":5.2839999999999998,"5min":5.7190000000000003,"15min":4.3259999999999996},"min":{"1min":0.29999999999999999,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":60.520000000000003,"5min":208.631,"15min":208.631},"last":1.585}]},{"osd":3,"last update":"Sat Sep 23 23:36:48 2023","interfaces":[{"interface":"back","average":{"1min":4.1749999999999998,"5min":4.6360000000000001,"15min":3.8100000000000001},"min":{"1min":0.34999999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":37.878,"5min":122.071,"15min":122.071},"last":1.319},{"interface":"front","average":{"1min":4.2460000000000004,"5min":4.9829999999999997,"15min":3.984},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.318},"max":{"1min":37.853000000000002,"5min":122.09699999999999,"15min":122.09699999999999},"last":0.52200000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.88200000000000001,"15min":0.877},"min":{"1min":0.316,"5min":0.315,"15min":0.251},"max":{"1min":2.2650000000000001,"5min":9.2690000000000001,"15min":19.962},"last":2.2650000000000001},{"interface":"front","average":{"1min":0.85499999999999998,"5min":1.016,"15min":0.84099999999999997},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.22900000000000001},"max":{"1min":3.1840000000000002,"5min":10.273999999999999,"15min":20.829999999999998},"last":1.3580000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.3340000000000001,"15min":1.5249999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.3100000000000001,"5min":148.36600000000001,"15min":148.36600000000001},"last":1.381},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.7389999999999999,"15min":1.71},"min":{"1min":0.32800000000000001,"5min":0.30399999999999999,"15min":0.26000000000000001},"max":{"1min":5.7240000000000002,"5min":148.42500000000001,"15min":148.42500000000001},"last":0.49299999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:36:48 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.2629999999999999,"15min":2.3879999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":2.29,"5min":15.497,"15min":179.28800000000001},"last":0.93300000000000005},{"interface":"front","average":{"1min":0.64900000000000002,"5min":1.2729999999999999,"15min":2.46},"min":{"1min":0.36699999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.0630000000000002,"5min":15.653,"15min":179.428},"last":0.85999999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":233177088,"data_stored":233177088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":863965184,"data_stored":863961112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1690697728,"data_stored":1690697728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":939134976,"data_stored":939130904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2353397760,"data_stored":2353397760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1599930368,"data_stored":1599930368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":988151808,"data_stored":988151808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1065816064,"data_stored":1065811992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:37:40.844 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:37:40.877 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:37:40.933 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:37:41.016 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:37:41.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:41.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:37:41.124 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:41.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:41 smithi138 ceph-mon[193606]: pgmap v1696: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 22665/120861 objects degraded (18.753%); 2400/120861 objects misplaced (1.986%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T23:37:41.194 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:41 smithi110 ceph-mon[212939]: pgmap v1696: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 22665/120861 objects degraded (18.753%); 2400/120861 objects misplaced (1.986%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T23:37:41.200 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:41 smithi110 ceph-mon[221102]: pgmap v1696: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 22665/120861 objects degraded (18.753%); 2400/120861 objects misplaced (1.986%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T23:37:41.200 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:41.386 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:37:41.543 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:41.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:41.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:41.651 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:41.662 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:41.715 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:41.723 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:41.864 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:41.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:41.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:41.969 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:37:42.000 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:42.082 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:37:42.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:42 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2046170257' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:42.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:42 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:42.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:42 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2046170257' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:42.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:42 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:42.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:42 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2046170257' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:42.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:42 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:37:42.594 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:37:42.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:42.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:37:42.699 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:42.746 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:43.328 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:37:43.328 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:37:43.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:37:43.328 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:37:43.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.328 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:143362 12.14 12:2df09e03:::benchmark_data_smithi110_491193_object48487:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T23:37:43.329 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:31:22.670541+0000", 2023-09-23T23:37:43.329 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 380.64977073799997, 2023-09-23T23:37:43.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.34082258900000001, 2023-09-23T23:37:43.329 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.329 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:43.329 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.329 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.329 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.329 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 143362 2023-09-23T23:37:43.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.329 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.670541+0000", 2023-09-23T23:37:43.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.670541+0000", 2023-09-23T23:37:43.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.331 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.331 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.670539+0000", 2023-09-23T23:37:43.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:37:43.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.331 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.331 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.670548+0000", 2023-09-23T23:37:43.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5199999999999997e-06 2023-09-23T23:37:43.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.670561+0000", 2023-09-23T23:37:43.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3543999999999999e-05 2023-09-23T23:37:43.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.670567+0000", 2023-09-23T23:37:43.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2900000000000002e-06 2023-09-23T23:37:43.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.670614+0000", 2023-09-23T23:37:43.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7338e-05 2023-09-23T23:37:43.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.670773+0000", 2023-09-23T23:37:43.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015908900000000001 2023-09-23T23:37:43.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:37:43.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.671188+0000", 2023-09-23T23:37:43.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000414659 2023-09-23T23:37:43.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.675806+0000", 2023-09-23T23:37:43.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0046179760000000002 2023-09-23T23:37:43.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.676124+0000", 2023-09-23T23:37:43.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00031816 2023-09-23T23:37:43.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011209+0000", 2023-09-23T23:37:43.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.33508485300000002 2023-09-23T23:37:43.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:43.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011265+0000", 2023-09-23T23:37:43.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5729999999999997e-05 2023-09-23T23:37:43.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011364+0000", 2023-09-23T23:37:43.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9328999999999997e-05 2023-09-23T23:37:43.336 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.336 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.336 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.337 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:143371 12.4 12:215a8f6d:::benchmark_data_smithi110_491193_object48496:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T23:37:43.337 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:31:22.673253+0000", 2023-09-23T23:37:43.337 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 380.64705940499999, 2023-09-23T23:37:43.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.33808128500000001, 2023-09-23T23:37:43.337 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.337 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:43.337 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.337 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.337 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.338 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 143371 2023-09-23T23:37:43.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.338 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673253+0000", 2023-09-23T23:37:43.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673253+0000", 2023-09-23T23:37:43.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673251+0000", 2023-09-23T23:37:43.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:43.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673259+0000", 2023-09-23T23:37:43.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5449999999999998e-06 2023-09-23T23:37:43.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.340 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673270+0000", 2023-09-23T23:37:43.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0791000000000001e-05 2023-09-23T23:37:43.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.340 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673276+0000", 2023-09-23T23:37:43.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7240000000000003e-06 2023-09-23T23:37:43.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673351+0000", 2023-09-23T23:37:43.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4734000000000006e-05 2023-09-23T23:37:43.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673454+0000", 2023-09-23T23:37:43.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000102908 2023-09-23T23:37:43.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:37:43.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673810+0000", 2023-09-23T23:37:43.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00035560399999999998 2023-09-23T23:37:43.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.676165+0000", 2023-09-23T23:37:43.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023558749999999999 2023-09-23T23:37:43.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.684242+0000", 2023-09-23T23:37:43.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0080765290000000007 2023-09-23T23:37:43.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011252+0000", 2023-09-23T23:37:43.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.32701007199999998 2023-09-23T23:37:43.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:43.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011292+0000", 2023-09-23T23:37:43.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9969000000000003e-05 2023-09-23T23:37:43.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011334+0000", 2023-09-23T23:37:43.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1872999999999997e-05 2023-09-23T23:37:43.344 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.345 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.345 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.345 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:143373 12.4 12:21483e46:::benchmark_data_smithi110_491193_object48498:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T23:37:43.345 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:31:22.678380+0000", 2023-09-23T23:37:43.345 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 380.641931634, 2023-09-23T23:37:43.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.33305675200000001, 2023-09-23T23:37:43.345 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.345 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:43.346 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.346 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.346 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.346 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 143373 2023-09-23T23:37:43.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.346 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.678380+0000", 2023-09-23T23:37:43.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.678380+0000", 2023-09-23T23:37:43.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.678379+0000", 2023-09-23T23:37:43.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:37:43.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.348 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.348 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.678384+0000", 2023-09-23T23:37:43.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.938e-06 2023-09-23T23:37:43.348 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.348 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.348 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.678398+0000", 2023-09-23T23:37:43.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4158999999999999e-05 2023-09-23T23:37:43.348 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.678404+0000", 2023-09-23T23:37:43.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7270000000000004e-06 2023-09-23T23:37:43.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.678455+0000", 2023-09-23T23:37:43.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0915999999999998e-05 2023-09-23T23:37:43.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.350 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.350 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.678597+0000", 2023-09-23T23:37:43.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014165599999999999 2023-09-23T23:37:43.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.350 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:37:43.350 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.679001+0000", 2023-09-23T23:37:43.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00040454400000000001 2023-09-23T23:37:43.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.681163+0000", 2023-09-23T23:37:43.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002161732 2023-09-23T23:37:43.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.684272+0000", 2023-09-23T23:37:43.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003108839 2023-09-23T23:37:43.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011366+0000", 2023-09-23T23:37:43.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.32709446599999997 2023-09-23T23:37:43.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:43.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011405+0000", 2023-09-23T23:37:43.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9167e-05 2023-09-23T23:37:43.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011437+0000", 2023-09-23T23:37:43.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1733000000000003e-05 2023-09-23T23:37:43.353 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.353 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.353 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.353 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:143384 12.14 12:2ec0ef03:::benchmark_data_smithi110_491193_object48509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T23:37:43.353 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:31:22.688389+0000", 2023-09-23T23:37:43.353 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 380.63192320899998, 2023-09-23T23:37:43.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.32313247099999998, 2023-09-23T23:37:43.354 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.354 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:43.354 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.354 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.354 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.354 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 143384 2023-09-23T23:37:43.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.354 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.688389+0000", 2023-09-23T23:37:43.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.688389+0000", 2023-09-23T23:37:43.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.688387+0000", 2023-09-23T23:37:43.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:43.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.688394+0000", 2023-09-23T23:37:43.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6479999999999997e-06 2023-09-23T23:37:43.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.688406+0000", 2023-09-23T23:37:43.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2323999999999999e-05 2023-09-23T23:37:43.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.688413+0000", 2023-09-23T23:37:43.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0960000000000003e-06 2023-09-23T23:37:43.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.688463+0000", 2023-09-23T23:37:43.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9302000000000003e-05 2023-09-23T23:37:43.358 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.358 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.358 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.688606+0000", 2023-09-23T23:37:43.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000142896 2023-09-23T23:37:43.358 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.358 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:37:43.358 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.689022+0000", 2023-09-23T23:37:43.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00041676899999999999 2023-09-23T23:37:43.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.359 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.359 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.359 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.691255+0000", 2023-09-23T23:37:43.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022322179999999998 2023-09-23T23:37:43.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.359 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.359 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.359 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.691512+0000", 2023-09-23T23:37:43.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025754199999999998 2023-09-23T23:37:43.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011396+0000", 2023-09-23T23:37:43.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.31988349700000002 2023-09-23T23:37:43.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:43.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011436+0000", 2023-09-23T23:37:43.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0778999999999997e-05 2023-09-23T23:37:43.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011521+0000", 2023-09-23T23:37:43.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4919999999999993e-05 2023-09-23T23:37:43.361 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.361 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.361 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.361 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172637 12.5 12:a0f0c82f:::benchmark_data_smithi110_491193_object77762:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:37:43.362 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.431822+0000", 2023-09-23T23:37:43.362 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 339.888490429, 2023-09-23T23:37:43.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0329259159999999, 2023-09-23T23:37:43.362 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.362 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:37:43.362 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.362 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.362 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.362 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172637 2023-09-23T23:37:43.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.363 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431822+0000", 2023-09-23T23:37:43.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431822+0000", 2023-09-23T23:37:43.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431820+0000", 2023-09-23T23:37:43.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:37:43.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431827+0000", 2023-09-23T23:37:43.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1779999999999997e-06 2023-09-23T23:37:43.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.365 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.365 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431843+0000", 2023-09-23T23:37:43.365 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5821e-05 2023-09-23T23:37:43.365 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.365 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.365 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431850+0000", 2023-09-23T23:37:43.365 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0010000000000004e-06 2023-09-23T23:37:43.365 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.366 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.366 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431916+0000", 2023-09-23T23:37:43.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5951000000000006e-05 2023-09-23T23:37:43.366 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.366 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.366 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.366 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431931+0000", 2023-09-23T23:37:43.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4979e-05 2023-09-23T23:37:43.366 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.366 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.464675+0000", 2023-09-23T23:37:43.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.032744629 2023-09-23T23:37:43.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.464748+0000", 2023-09-23T23:37:43.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2084999999999998e-05 2023-09-23T23:37:43.367 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.367 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.367 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.368 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172642 12.5 12:a6e87117:::benchmark_data_smithi110_491193_object77767:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:37:43.368 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.431864+0000", 2023-09-23T23:37:43.368 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 339.88844761899998, 2023-09-23T23:37:43.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.032980893, 2023-09-23T23:37:43.368 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.368 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:37:43.368 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.368 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.368 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.369 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172642 2023-09-23T23:37:43.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.369 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431864+0000", 2023-09-23T23:37:43.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.370 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431864+0000", 2023-09-23T23:37:43.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.370 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431864+0000", 2023-09-23T23:37:43.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:37:43.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431867+0000", 2023-09-23T23:37:43.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1099999999999999e-06 2023-09-23T23:37:43.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431881+0000", 2023-09-23T23:37:43.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3616e-05 2023-09-23T23:37:43.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431886+0000", 2023-09-23T23:37:43.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9899999999999997e-06 2023-09-23T23:37:43.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431964+0000", 2023-09-23T23:37:43.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8727999999999996e-05 2023-09-23T23:37:43.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431973+0000", 2023-09-23T23:37:43.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4789999999999996e-06 2023-09-23T23:37:43.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.464788+0000", 2023-09-23T23:37:43.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0328152420000001 2023-09-23T23:37:43.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.464845+0000", 2023-09-23T23:37:43.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7188000000000002e-05 2023-09-23T23:37:43.374 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.374 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.374 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.374 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172739 12.5 12:a6b54f33:::benchmark_data_smithi110_491193_object77864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:37:43.374 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.576924+0000", 2023-09-23T23:37:43.374 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 339.74338783100001, 2023-09-23T23:37:43.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91090562100000005, 2023-09-23T23:37:43.374 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.375 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:43.375 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.375 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.375 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.375 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172739 2023-09-23T23:37:43.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.375 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.576924+0000", 2023-09-23T23:37:43.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.576924+0000", 2023-09-23T23:37:43.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.576923+0000", 2023-09-23T23:37:43.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:43.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.576930+0000", 2023-09-23T23:37:43.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0820000000000001e-06 2023-09-23T23:37:43.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.576945+0000", 2023-09-23T23:37:43.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4877e-05 2023-09-23T23:37:43.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.576951+0000", 2023-09-23T23:37:43.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2029999999999997e-06 2023-09-23T23:37:43.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.577016+0000", 2023-09-23T23:37:43.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5184999999999993e-05 2023-09-23T23:37:43.379 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.379 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.379 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.577027+0000", 2023-09-23T23:37:43.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1542000000000001e-05 2023-09-23T23:37:43.379 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.379 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.379 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.464871+0000", 2023-09-23T23:37:43.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88784394600000005 2023-09-23T23:37:43.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.464949+0000", 2023-09-23T23:37:43.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7397000000000005e-05 2023-09-23T23:37:43.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:37:43.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.465320+0000", 2023-09-23T23:37:43.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00037099799999999999 2023-09-23T23:37:43.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.479792+0000", 2023-09-23T23:37:43.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014472485 2023-09-23T23:37:43.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487729+0000", 2023-09-23T23:37:43.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0079366209999999996 2023-09-23T23:37:43.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487760+0000", 2023-09-23T23:37:43.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1142000000000001e-05 2023-09-23T23:37:43.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:43.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487793+0000", 2023-09-23T23:37:43.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2712000000000001e-05 2023-09-23T23:37:43.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487830+0000", 2023-09-23T23:37:43.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7044e-05 2023-09-23T23:37:43.383 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.383 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.383 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.384 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172764 12.5 12:a4dfe0b2:::benchmark_data_smithi110_491193_object77889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:37:43.384 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.604922+0000", 2023-09-23T23:37:43.384 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 339.71539016399998, 2023-09-23T23:37:43.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88303478499999999, 2023-09-23T23:37:43.384 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.384 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:43.384 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.384 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.384 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.385 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172764 2023-09-23T23:37:43.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.385 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.604922+0000", 2023-09-23T23:37:43.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.604922+0000", 2023-09-23T23:37:43.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.386 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.386 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.386 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.604921+0000", 2023-09-23T23:37:43.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:43.386 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.386 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.604929+0000", 2023-09-23T23:37:43.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4419999999999998e-06 2023-09-23T23:37:43.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.604946+0000", 2023-09-23T23:37:43.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6554000000000001e-05 2023-09-23T23:37:43.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.604953+0000", 2023-09-23T23:37:43.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5120000000000002e-06 2023-09-23T23:37:43.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.605016+0000", 2023-09-23T23:37:43.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2572999999999999e-05 2023-09-23T23:37:43.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.605028+0000", 2023-09-23T23:37:43.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2335e-05 2023-09-23T23:37:43.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.467556+0000", 2023-09-23T23:37:43.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.86252822200000001 2023-09-23T23:37:43.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.467630+0000", 2023-09-23T23:37:43.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3326999999999998e-05 2023-09-23T23:37:43.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:37:43.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.467914+0000", 2023-09-23T23:37:43.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00028455700000000001 2023-09-23T23:37:43.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.479819+0000", 2023-09-23T23:37:43.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011905007 2023-09-23T23:37:43.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487890+0000", 2023-09-23T23:37:43.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0080704699999999997 2023-09-23T23:37:43.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487911+0000", 2023-09-23T23:37:43.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1698000000000001e-05 2023-09-23T23:37:43.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:43.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487936+0000", 2023-09-23T23:37:43.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4635e-05 2023-09-23T23:37:43.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487957+0000", 2023-09-23T23:37:43.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0724e-05 2023-09-23T23:37:43.393 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.393 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.393 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.393 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172776 12.5 12:a601ba3f:::benchmark_data_smithi110_491193_object77901:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:37:43.393 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.615330+0000", 2023-09-23T23:37:43.393 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 339.70498167400001, 2023-09-23T23:37:43.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.87300335500000004, 2023-09-23T23:37:43.393 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.393 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:43.394 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.394 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.394 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.394 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172776 2023-09-23T23:37:43.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.394 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.615330+0000", 2023-09-23T23:37:43.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.615330+0000", 2023-09-23T23:37:43.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.615329+0000", 2023-09-23T23:37:43.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:37:43.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.615337+0000", 2023-09-23T23:37:43.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6740000000000003e-06 2023-09-23T23:37:43.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.615354+0000", 2023-09-23T23:37:43.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6642000000000001e-05 2023-09-23T23:37:43.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.397 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.397 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.615363+0000", 2023-09-23T23:37:43.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7320000000000004e-06 2023-09-23T23:37:43.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.397 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.397 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.615431+0000", 2023-09-23T23:37:43.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8034999999999995e-05 2023-09-23T23:37:43.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.398 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.398 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.615442+0000", 2023-09-23T23:37:43.398 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1235e-05 2023-09-23T23:37:43.398 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.398 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.398 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.470198+0000", 2023-09-23T23:37:43.398 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.85475619300000005 2023-09-23T23:37:43.398 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.470310+0000", 2023-09-23T23:37:43.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112336 2023-09-23T23:37:43.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:37:43.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.470624+0000", 2023-09-23T23:37:43.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00031330399999999998 2023-09-23T23:37:43.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.479824+0000", 2023-09-23T23:37:43.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0092004470000000005 2023-09-23T23:37:43.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488042+0000", 2023-09-23T23:37:43.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0082178689999999992 2023-09-23T23:37:43.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488075+0000", 2023-09-23T23:37:43.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3098000000000001e-05 2023-09-23T23:37:43.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:43.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488119+0000", 2023-09-23T23:37:43.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3526999999999998e-05 2023-09-23T23:37:43.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488334+0000", 2023-09-23T23:37:43.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000214976 2023-09-23T23:37:43.402 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.402 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.402 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.402 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172861 12.5 12:a2093041:::benchmark_data_smithi110_491193_object77986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:37:43.402 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.736047+0000", 2023-09-23T23:37:43.403 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 339.58426452100002, 2023-09-23T23:37:43.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.75252812999999996, 2023-09-23T23:37:43.403 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.403 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:43.403 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.403 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.403 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.403 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172861 2023-09-23T23:37:43.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.403 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.404 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.736047+0000", 2023-09-23T23:37:43.404 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.404 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.736047+0000", 2023-09-23T23:37:43.404 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.736046+0000", 2023-09-23T23:37:43.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:37:43.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.736052+0000", 2023-09-23T23:37:43.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8329999999999999e-06 2023-09-23T23:37:43.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.406 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.736065+0000", 2023-09-23T23:37:43.406 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2875e-05 2023-09-23T23:37:43.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.406 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.406 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.736070+0000", 2023-09-23T23:37:43.406 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9169999999999997e-06 2023-09-23T23:37:43.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.406 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.736144+0000", 2023-09-23T23:37:43.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4196999999999995e-05 2023-09-23T23:37:43.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.736155+0000", 2023-09-23T23:37:43.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1036000000000001e-05 2023-09-23T23:37:43.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.472246+0000", 2023-09-23T23:37:43.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.73609044199999996 2023-09-23T23:37:43.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.472371+0000", 2023-09-23T23:37:43.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000125728 2023-09-23T23:37:43.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:37:43.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.472633+0000", 2023-09-23T23:37:43.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00026140100000000003 2023-09-23T23:37:43.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.482986+0000", 2023-09-23T23:37:43.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010353012999999999 2023-09-23T23:37:43.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488401+0000", 2023-09-23T23:37:43.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0054150520000000001 2023-09-23T23:37:43.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.410 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488505+0000", 2023-09-23T23:37:43.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010446 2023-09-23T23:37:43.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:43.410 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488553+0000", 2023-09-23T23:37:43.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7568000000000003e-05 2023-09-23T23:37:43.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.411 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488576+0000", 2023-09-23T23:37:43.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2682999999999999e-05 2023-09-23T23:37:43.411 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.411 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.411 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.412 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172904 12.5 12:a3ef1e59:::benchmark_data_smithi110_491193_object78029:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:37:43.412 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.795190+0000", 2023-09-23T23:37:43.412 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 339.52512185699999, 2023-09-23T23:37:43.412 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.69346639600000004, 2023-09-23T23:37:43.412 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.412 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:43.412 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.412 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.412 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.412 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172904 2023-09-23T23:37:43.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.413 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.795190+0000", 2023-09-23T23:37:43.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.795190+0000", 2023-09-23T23:37:43.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.795188+0000", 2023-09-23T23:37:43.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:37:43.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.795196+0000", 2023-09-23T23:37:43.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0489999999999998e-06 2023-09-23T23:37:43.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.795214+0000", 2023-09-23T23:37:43.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8461000000000002e-05 2023-09-23T23:37:43.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.795221+0000", 2023-09-23T23:37:43.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9890000000000003e-06 2023-09-23T23:37:43.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.416 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.795271+0000", 2023-09-23T23:37:43.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9607999999999997e-05 2023-09-23T23:37:43.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.416 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.795282+0000", 2023-09-23T23:37:43.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1039999999999999e-05 2023-09-23T23:37:43.417 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.474046+0000", 2023-09-23T23:37:43.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.67876398900000001 2023-09-23T23:37:43.417 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.474122+0000", 2023-09-23T23:37:43.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6681999999999995e-05 2023-09-23T23:37:43.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:37:43.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.474331+0000", 2023-09-23T23:37:43.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000209049 2023-09-23T23:37:43.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.482992+0000", 2023-09-23T23:37:43.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086603419999999997 2023-09-23T23:37:43.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488458+0000", 2023-09-23T23:37:43.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0054665870000000002 2023-09-23T23:37:43.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488611+0000", 2023-09-23T23:37:43.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015253099999999999 2023-09-23T23:37:43.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:43.420 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488637+0000", 2023-09-23T23:37:43.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6126e-05 2023-09-23T23:37:43.420 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.420 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488657+0000", 2023-09-23T23:37:43.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9629000000000001e-05 2023-09-23T23:37:43.420 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.420 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.421 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.421 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172905 12.5 12:a350bac5:::benchmark_data_smithi110_491193_object78030:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:37:43.421 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.801511+0000", 2023-09-23T23:37:43.421 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 339.51880094099999, 2023-09-23T23:37:43.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.68724216400000004, 2023-09-23T23:37:43.421 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.421 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:43.421 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.422 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.422 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.422 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172905 2023-09-23T23:37:43.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.422 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.801511+0000", 2023-09-23T23:37:43.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.801511+0000", 2023-09-23T23:37:43.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.801510+0000", 2023-09-23T23:37:43.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:43.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.801518+0000", 2023-09-23T23:37:43.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0390000000000005e-06 2023-09-23T23:37:43.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.801528+0000", 2023-09-23T23:37:43.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0577999999999999e-05 2023-09-23T23:37:43.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.801534+0000", 2023-09-23T23:37:43.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7119999999999997e-06 2023-09-23T23:37:43.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.801593+0000", 2023-09-23T23:37:43.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8794e-05 2023-09-23T23:37:43.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.801603+0000", 2023-09-23T23:37:43.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0423e-05 2023-09-23T23:37:43.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.475424+0000", 2023-09-23T23:37:43.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.67382106200000003 2023-09-23T23:37:43.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.427 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.475498+0000", 2023-09-23T23:37:43.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3946e-05 2023-09-23T23:37:43.427 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.427 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.427 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:37:43.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.475699+0000", 2023-09-23T23:37:43.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000200736 2023-09-23T23:37:43.427 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.427 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.427 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.428 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.483001+0000", 2023-09-23T23:37:43.428 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0073019419999999996 2023-09-23T23:37:43.428 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.428 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.428 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488685+0000", 2023-09-23T23:37:43.428 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0056844449999999998 2023-09-23T23:37:43.428 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.428 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488710+0000", 2023-09-23T23:37:43.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4898000000000001e-05 2023-09-23T23:37:43.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:43.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488734+0000", 2023-09-23T23:37:43.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3272999999999999e-05 2023-09-23T23:37:43.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488753+0000", 2023-09-23T23:37:43.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9647e-05 2023-09-23T23:37:43.430 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.430 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.430 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.430 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.430 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.430 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172917 12.5 12:a628e390:::benchmark_data_smithi110_491193_object78042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:37:43.430 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.813634+0000", 2023-09-23T23:37:43.430 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 339.506677663, 2023-09-23T23:37:43.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.68256179699999997, 2023-09-23T23:37:43.431 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.431 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:43.431 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.431 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.431 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.431 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172917 2023-09-23T23:37:43.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.431 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.431 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.431 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.431 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.813634+0000", 2023-09-23T23:37:43.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.813634+0000", 2023-09-23T23:37:43.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.813633+0000", 2023-09-23T23:37:43.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:37:43.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.433 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.813640+0000", 2023-09-23T23:37:43.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9399999999999996e-06 2023-09-23T23:37:43.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.433 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.813652+0000", 2023-09-23T23:37:43.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2490999999999999e-05 2023-09-23T23:37:43.434 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.813658+0000", 2023-09-23T23:37:43.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4439999999999999e-06 2023-09-23T23:37:43.434 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.813719+0000", 2023-09-23T23:37:43.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1523e-05 2023-09-23T23:37:43.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.435 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.435 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.813728+0000", 2023-09-23T23:37:43.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2450000000000004e-06 2023-09-23T23:37:43.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.435 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.435 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.476807+0000", 2023-09-23T23:37:43.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.66307843499999997 2023-09-23T23:37:43.436 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.436 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.436 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.436 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.476876+0000", 2023-09-23T23:37:43.436 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8788000000000006e-05 2023-09-23T23:37:43.436 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.436 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.436 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:37:43.436 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.477076+0000", 2023-09-23T23:37:43.436 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020076800000000001 2023-09-23T23:37:43.436 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.483006+0000", 2023-09-23T23:37:43.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0059295099999999998 2023-09-23T23:37:43.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488782+0000", 2023-09-23T23:37:43.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0057763950000000001 2023-09-23T23:37:43.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.438 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.438 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.438 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496102+0000", 2023-09-23T23:37:43.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0073202079999999999 2023-09-23T23:37:43.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.438 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.438 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:43.438 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496155+0000", 2023-09-23T23:37:43.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2954000000000002e-05 2023-09-23T23:37:43.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.439 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496196+0000", 2023-09-23T23:37:43.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0781999999999997e-05 2023-09-23T23:37:43.439 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.439 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.439 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.439 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:173047 12.5 12:a6b5faa1:::benchmark_data_smithi110_491193_object78172:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:37:43.439 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:04.026890+0000", 2023-09-23T23:37:43.440 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 339.29342190400001, 2023-09-23T23:37:43.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.46946490800000001, 2023-09-23T23:37:43.440 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.440 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:43.440 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.440 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.440 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.440 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 173047 2023-09-23T23:37:43.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.440 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.026890+0000", 2023-09-23T23:37:43.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.026890+0000", 2023-09-23T23:37:43.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.026889+0000", 2023-09-23T23:37:43.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:37:43.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.026893+0000", 2023-09-23T23:37:43.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9140000000000001e-06 2023-09-23T23:37:43.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.026903+0000", 2023-09-23T23:37:43.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.594e-06 2023-09-23T23:37:43.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.026909+0000", 2023-09-23T23:37:43.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8980000000000001e-06 2023-09-23T23:37:43.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.026971+0000", 2023-09-23T23:37:43.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2510999999999998e-05 2023-09-23T23:37:43.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.026980+0000", 2023-09-23T23:37:43.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9360000000000005e-06 2023-09-23T23:37:43.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.478207+0000", 2023-09-23T23:37:43.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.45122682600000003 2023-09-23T23:37:43.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.478311+0000", 2023-09-23T23:37:43.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000104338 2023-09-23T23:37:43.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:37:43.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.478579+0000", 2023-09-23T23:37:43.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00026786900000000001 2023-09-23T23:37:43.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.485830+0000", 2023-09-23T23:37:43.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0072510320000000001 2023-09-23T23:37:43.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488804+0000", 2023-09-23T23:37:43.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029741030000000001 2023-09-23T23:37:43.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496254+0000", 2023-09-23T23:37:43.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0074498450000000001 2023-09-23T23:37:43.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:43.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496316+0000", 2023-09-23T23:37:43.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1923999999999999e-05 2023-09-23T23:37:43.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.448 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.448 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496355+0000", 2023-09-23T23:37:43.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8903999999999999e-05 2023-09-23T23:37:43.448 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.448 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.448 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.449 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:173052 12.5 12:a47f877d:::benchmark_data_smithi110_491193_object78177:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:37:43.449 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:04.037247+0000", 2023-09-23T23:37:43.449 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 339.28306533799997, 2023-09-23T23:37:43.449 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.45921886099999998, 2023-09-23T23:37:43.449 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.449 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:43.449 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.449 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.449 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.449 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 173052 2023-09-23T23:37:43.449 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.450 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.037247+0000", 2023-09-23T23:37:43.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.037247+0000", 2023-09-23T23:37:43.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.037245+0000", 2023-09-23T23:37:43.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:37:43.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.037252+0000", 2023-09-23T23:37:43.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.486e-06 2023-09-23T23:37:43.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.037268+0000", 2023-09-23T23:37:43.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5919000000000001e-05 2023-09-23T23:37:43.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.037276+0000", 2023-09-23T23:37:43.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9540000000000004e-06 2023-09-23T23:37:43.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.037338+0000", 2023-09-23T23:37:43.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1952000000000006e-05 2023-09-23T23:37:43.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.037351+0000", 2023-09-23T23:37:43.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2221e-05 2023-09-23T23:37:43.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.454 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.479867+0000", 2023-09-23T23:37:43.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.44251611099999999 2023-09-23T23:37:43.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.454 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.479940+0000", 2023-09-23T23:37:43.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2977000000000003e-05 2023-09-23T23:37:43.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:37:43.455 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.480183+0000", 2023-09-23T23:37:43.455 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00024294 2023-09-23T23:37:43.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.455 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.485834+0000", 2023-09-23T23:37:43.455 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0056517110000000002 2023-09-23T23:37:43.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.490636+0000", 2023-09-23T23:37:43.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004801619 2023-09-23T23:37:43.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496406+0000", 2023-09-23T23:37:43.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005769723 2023-09-23T23:37:43.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:43.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496436+0000", 2023-09-23T23:37:43.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0195000000000001e-05 2023-09-23T23:37:43.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496466+0000", 2023-09-23T23:37:43.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9723999999999999e-05 2023-09-23T23:37:43.457 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.457 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.457 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.458 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.458 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.458 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:173087 12.5 12:a280e1c5:::benchmark_data_smithi110_491193_object78212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:37:43.458 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:04.089799+0000", 2023-09-23T23:37:43.458 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 339.23051261900002, 2023-09-23T23:37:43.458 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.40679320899999999, 2023-09-23T23:37:43.458 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.458 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:43.458 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.458 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.459 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.459 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 173087 2023-09-23T23:37:43.459 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.459 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.459 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.459 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.459 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.089799+0000", 2023-09-23T23:37:43.459 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.459 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.459 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.460 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.460 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.089799+0000", 2023-09-23T23:37:43.460 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.460 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.460 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.460 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.460 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.089798+0000", 2023-09-23T23:37:43.460 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:43.460 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.460 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.460 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.461 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.089806+0000", 2023-09-23T23:37:43.461 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4929999999999998e-06 2023-09-23T23:37:43.461 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.461 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.461 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.461 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.089821+0000", 2023-09-23T23:37:43.461 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4915e-05 2023-09-23T23:37:43.461 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.461 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.461 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.461 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.089829+0000", 2023-09-23T23:37:43.462 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9519999999999994e-06 2023-09-23T23:37:43.462 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.462 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.462 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.462 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.089892+0000", 2023-09-23T23:37:43.462 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3378000000000002e-05 2023-09-23T23:37:43.462 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.462 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.462 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.462 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.089900+0000", 2023-09-23T23:37:43.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7370000000000006e-06 2023-09-23T23:37:43.463 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.463 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.481362+0000", 2023-09-23T23:37:43.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.391462225 2023-09-23T23:37:43.463 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.463 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.481469+0000", 2023-09-23T23:37:43.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010607600000000001 2023-09-23T23:37:43.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.464 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:37:43.464 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.481656+0000", 2023-09-23T23:37:43.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018705800000000001 2023-09-23T23:37:43.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.464 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.464 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487429+0000", 2023-09-23T23:37:43.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0057732809999999999 2023-09-23T23:37:43.465 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.465 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.465 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.465 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.490672+0000", 2023-09-23T23:37:43.465 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032433050000000001 2023-09-23T23:37:43.465 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.465 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.465 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.465 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496511+0000", 2023-09-23T23:37:43.465 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0058392180000000002 2023-09-23T23:37:43.466 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.466 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.466 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:43.466 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496557+0000", 2023-09-23T23:37:43.466 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5278999999999998e-05 2023-09-23T23:37:43.466 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.466 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.466 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.466 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496593+0000", 2023-09-23T23:37:43.466 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5932000000000001e-05 2023-09-23T23:37:43.467 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.467 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.467 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.467 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.467 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.467 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:173110 12.5 12:a6c8a621:::benchmark_data_smithi110_491193_object78235:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:37:43.467 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:04.128868+0000", 2023-09-23T23:37:43.467 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 339.19144392099997, 2023-09-23T23:37:43.467 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.36781228999999999, 2023-09-23T23:37:43.467 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.467 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:43.468 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.468 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.468 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.468 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 173110 2023-09-23T23:37:43.468 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.468 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.468 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.468 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.468 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.128868+0000", 2023-09-23T23:37:43.468 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.468 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.469 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.469 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.469 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.128868+0000", 2023-09-23T23:37:43.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.469 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.469 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.469 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.469 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.128867+0000", 2023-09-23T23:37:43.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:43.469 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.470 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.470 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.128872+0000", 2023-09-23T23:37:43.470 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.31e-06 2023-09-23T23:37:43.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.470 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.470 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.128886+0000", 2023-09-23T23:37:43.470 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4049999999999999e-05 2023-09-23T23:37:43.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.128892+0000", 2023-09-23T23:37:43.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3240000000000002e-06 2023-09-23T23:37:43.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.128955+0000", 2023-09-23T23:37:43.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3304000000000002e-05 2023-09-23T23:37:43.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.128968+0000", 2023-09-23T23:37:43.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2736e-05 2023-09-23T23:37:43.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.483042+0000", 2023-09-23T23:37:43.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.35407397299999999 2023-09-23T23:37:43.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.483126+0000", 2023-09-23T23:37:43.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4703999999999994e-05 2023-09-23T23:37:43.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:37:43.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.483325+0000", 2023-09-23T23:37:43.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019903100000000001 2023-09-23T23:37:43.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487435+0000", 2023-09-23T23:37:43.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041099760000000004 2023-09-23T23:37:43.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.490751+0000", 2023-09-23T23:37:43.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0033156359999999998 2023-09-23T23:37:43.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.475 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.475 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496626+0000", 2023-09-23T23:37:43.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0058748419999999999 2023-09-23T23:37:43.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.475 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:43.475 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496650+0000", 2023-09-23T23:37:43.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4660000000000001e-05 2023-09-23T23:37:43.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.476 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.476 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496680+0000", 2023-09-23T23:37:43.476 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9950999999999998e-05 2023-09-23T23:37:43.476 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.476 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.476 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.476 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.476 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.476 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:173132 12.5 12:a42e1715:::benchmark_data_smithi110_491193_object78257:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:37:43.476 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:04.163750+0000", 2023-09-23T23:37:43.477 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 339.15656251299998, 2023-09-23T23:37:43.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.33305025599999999, 2023-09-23T23:37:43.477 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.477 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:43.477 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.477 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:37:43.477 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:37:43.477 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 173132 2023-09-23T23:37:43.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.477 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.163750+0000", 2023-09-23T23:37:43.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.163750+0000", 2023-09-23T23:37:43.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.163748+0000", 2023-09-23T23:37:43.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:43.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.163755+0000", 2023-09-23T23:37:43.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7649999999999996e-06 2023-09-23T23:37:43.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.163770+0000", 2023-09-23T23:37:43.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5336e-05 2023-09-23T23:37:43.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.163777+0000", 2023-09-23T23:37:43.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6279999999999999e-06 2023-09-23T23:37:43.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.481 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.481 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.163832+0000", 2023-09-23T23:37:43.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5615999999999997e-05 2023-09-23T23:37:43.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.481 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.481 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.163840+0000", 2023-09-23T23:37:43.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3100000000000003e-06 2023-09-23T23:37:43.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.482 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.482 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.482 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.484458+0000", 2023-09-23T23:37:43.482 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.320618393 2023-09-23T23:37:43.482 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.482 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.482 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.482 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.484538+0000", 2023-09-23T23:37:43.482 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9394000000000006e-05 2023-09-23T23:37:43.482 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:37:43.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.484731+0000", 2023-09-23T23:37:43.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019382000000000001 2023-09-23T23:37:43.483 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487685+0000", 2023-09-23T23:37:43.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029539240000000001 2023-09-23T23:37:43.483 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.484 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.494655+0000", 2023-09-23T23:37:43.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069701110000000002 2023-09-23T23:37:43.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.484 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496722+0000", 2023-09-23T23:37:43.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020667580000000001 2023-09-23T23:37:43.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:43.485 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496765+0000", 2023-09-23T23:37:43.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2719999999999998e-05 2023-09-23T23:37:43.485 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.485 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496800+0000", 2023-09-23T23:37:43.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4879999999999998e-05 2023-09-23T23:37:43.485 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.485 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.485 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.486 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.486 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71517 13.1 13:8b0a5480:::benchmark_data_smithi110_541928_object71516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e555)", 2023-09-23T23:37:43.486 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.526261+0000", 2023-09-23T23:37:43.486 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 232.794050545, 2023-09-23T23:37:43.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3549986289999998, 2023-09-23T23:37:43.486 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.486 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:37:43.486 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.486 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:43.487 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:43.487 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71517 2023-09-23T23:37:43.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.487 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.487 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.526261+0000", 2023-09-23T23:37:43.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.526261+0000", 2023-09-23T23:37:43.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.526260+0000", 2023-09-23T23:37:43.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:37:43.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.526294+0000", 2023-09-23T23:37:43.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3726000000000002e-05 2023-09-23T23:37:43.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.526306+0000", 2023-09-23T23:37:43.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1528999999999999e-05 2023-09-23T23:37:43.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.526311+0000", 2023-09-23T23:37:43.490 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7289999999999997e-06 2023-09-23T23:37:43.490 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.490 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531327+0000", 2023-09-23T23:37:43.490 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050161060000000002 2023-09-23T23:37:43.490 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.490 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531460+0000", 2023-09-23T23:37:43.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013231000000000001 2023-09-23T23:37:43.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.491 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:37:43.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531617+0000", 2023-09-23T23:37:43.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000157026 2023-09-23T23:37:43.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.491 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.534849+0000", 2023-09-23T23:37:43.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032324279999999999 2023-09-23T23:37:43.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.553986+0000", 2023-09-23T23:37:43.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019137105000000001 2023-09-23T23:37:43.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.558234+0000", 2023-09-23T23:37:43.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0042477089999999997 2023-09-23T23:37:43.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.558245+0000", 2023-09-23T23:37:43.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1037e-05 2023-09-23T23:37:43.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.546510+0000", 2023-09-23T23:37:43.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98826527500000005 2023-09-23T23:37:43.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.546524+0000", 2023-09-23T23:37:43.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3667e-05 2023-09-23T23:37:43.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.881146+0000", 2023-09-23T23:37:43.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3346218910000001 2023-09-23T23:37:43.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.881260+0000", 2023-09-23T23:37:43.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000114296 2023-09-23T23:37:43.495 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.495 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.495 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.495 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71539 13.1 13:8eed35ed:::benchmark_data_smithi110_541928_object71538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:37:43.495 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.584298+0000", 2023-09-23T23:37:43.496 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 232.736013907, 2023-09-23T23:37:43.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3176904839999999, 2023-09-23T23:37:43.496 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:37:43.496 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:37:43.496 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:37:43.496 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:37:43.496 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:37:43.496 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71539 2023-09-23T23:37:43.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.496 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:37:43.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:37:43.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.584298+0000", 2023-09-23T23:37:43.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:37:43.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.584298+0000", 2023-09-23T23:37:43.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:37:43.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:37:43.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.584297+0000", 2023-09-23T23:37:43.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:37:43.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:37:43.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.584362+0000", 2023-09-23T23:37:43.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4276999999999995e-05 2023-09-23T23:37:43.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:37:43.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.584371+0000", 2023-09-23T23:37:43.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3479999999999993e-06 2023-09-23T23:37:43.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:37:43.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.584377+0000", 2023-09-23T23:37:43.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6300000000000003e-06 2023-09-23T23:37:43.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.584429+0000", 2023-09-23T23:37:43.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2410000000000001e-05 2023-09-23T23:37:43.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.584437+0000", 2023-09-23T23:37:43.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2090000000000005e-06 2023-09-23T23:37:43.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.546551+0000", 2023-09-23T23:37:43.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96211402999999995 2023-09-23T23:37:43.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:37:43.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.546559+0000", 2023-09-23T23:37:43.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6480000000000003e-06 2023-09-23T23:37:43.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:37:43.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.881286+0000", 2023-09-23T23:37:43.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3347272050000001 2023-09-23T23:37:43.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:37:43.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.881451+0000", 2023-09-23T23:37:43.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016506 2023-09-23T23:37:43.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-09-23T23:37:43.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.881663+0000", 2023-09-23T23:37:43.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021163399999999999 2023-09-23T23:37:43.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:37:43.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.891533+0000", 2023-09-23T23:37:43.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0098703790000000003 2023-09-23T23:37:43.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.894429+0000", 2023-09-23T23:37:43.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028962229999999999 2023-09-23T23:37:43.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:37:43.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.897756+0000", 2023-09-23T23:37:43.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003327048 2023-09-23T23:37:43.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:37:43.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.897834+0000", 2023-09-23T23:37:43.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7341000000000004e-05 2023-09-23T23:37:43.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:37:43.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:37:43.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:37:43.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.901989+0000", 2023-09-23T23:37:43.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041549009999999999 2023-09-23T23:37:43.507 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.507 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.507 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.507 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:37:43.507 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:37:43.507 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:37:43.508 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:43 smithi138 ceph-mon[193606]: pgmap v1697: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 384 op/s; 22665/119847 objects degraded (18.912%); 2400/119847 objects misplaced (2.003%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T23:37:43.508 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:43 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:43.508 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:43 smithi138 ceph-mon[193606]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.508 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:43 smithi138 ceph-mon[193606]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.508 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:43 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:37:43.508 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:43 smithi138 ceph-mon[193606]: osdmap e603: 8 total, 8 up, 7 in 2023-09-23T23:37:43.509 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:43 smithi138 ceph-mon[193606]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.509 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:43 smithi138 ceph-mon[193606]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.509 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:43 smithi138 ceph-mon[193606]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.509 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:43 smithi138 ceph-mon[193606]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.509 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:43.509 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:37:43.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:43.510 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:43.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:43.511 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[221102]: pgmap v1697: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 384 op/s; 22665/119847 objects degraded (18.912%); 2400/119847 objects misplaced (2.003%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T23:37:43.511 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:43.511 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[221102]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.511 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[221102]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.511 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:37:43.511 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[221102]: osdmap e603: 8 total, 8 up, 7 in 2023-09-23T23:37:43.511 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[221102]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.511 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[221102]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.511 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[221102]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.511 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[221102]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.511 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:43.512 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:37:43.512 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[212939]: pgmap v1697: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 384 op/s; 22665/119847 objects degraded (18.912%); 2400/119847 objects misplaced (2.003%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T23:37:43.512 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:43.512 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[212939]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.512 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[212939]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.512 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:37:43.512 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[212939]: osdmap e603: 8 total, 8 up, 7 in 2023-09-23T23:37:43.512 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[212939]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.513 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[212939]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.513 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[212939]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.513 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[212939]: failed to encode map e603 with expected crc 2023-09-23T23:37:43.513 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:37:43.513 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:37:43.612 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:43.641 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:43.689 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:43.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:43.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:43.793 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:37:43.828 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:44.081 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:44 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 22665/119847 objects degraded (18.912%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:44.082 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:44 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 22665/119847 objects degraded (18.912%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:44.172 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:37:44.184 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:37:44.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:44.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:37:44.294 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:37:44.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:44 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 22665/119847 objects degraded (18.912%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:44.331 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:44.395 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:37:44.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:44.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:37:44.501 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:44.527 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:44.574 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:44.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:44.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:44.684 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:44.733 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:44.959 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:44.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:44.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:45.063 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:45.116 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:45.317 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:45.321 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:45.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:45.330 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:45 smithi110 ceph-mon[212939]: pgmap v1699: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 22540/118425 objects degraded (19.033%); 2283/118425 objects misplaced (1.928%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T23:37:45.330 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:45 smithi110 ceph-mon[221102]: pgmap v1699: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 22540/118425 objects degraded (19.033%); 2283/118425 objects misplaced (1.928%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T23:37:45.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:45.432 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:45.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:45 smithi138 ceph-mon[193606]: pgmap v1699: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 22540/118425 objects degraded (19.033%); 2283/118425 objects misplaced (1.928%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T23:37:45.450 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:37:45.481 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:45.486 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:37:45.522 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:37:45.553 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:37:45.553 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:45.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:45.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:45.579 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:37:45.598 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:37:45.618 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:37:45.644 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:37:45.646 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:37:45.646 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:37:45.646 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:37:45.646 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:37:45.646 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:37:45.646 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:37:45.646 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:37:45.646 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:37:45.662 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:45.694 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:45.810 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:45.811 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:37:45.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:45.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:45.917 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:45.978 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:46.153 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:37:46.153 INFO:teuthology.orchestra.run.smithi110.stdout:{"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 22423/113988 objects degraded (19.671%), 7 pgs degraded, 5 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2143,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":603,"num_osds":8,"num_up_osds":8,"osd_up_since":1695511414,"num_in_osds":7,"osd_in_since":1695512030,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":110},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+recovering","count":1},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":119,"num_pools":5,"num_objects":37996,"data_bytes":2489103605,"bytes_used":12752478208,"bytes_avail":659212771328,"bytes_total":671965249536,"degraded_objects":22423,"degraded_total":113988,"degraded_ratio":0.19671368916026249,"misplaced_objects":2283,"misplaced_total":113988,"misplaced_ratio":0.020028424044636277,"recovering_objects_per_sec":73,"recovering_bytes_per_sec":4848274,"recovering_keys_per_sec":0,"num_objects_recovered":740,"num_bytes_recovered":48496640,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":380},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi110.fuglpi","status":"up:active","gid":24572}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.110:8443/","prometheus":"http://172.21.15.110:9283/"}},"servicemap":{"epoch":32,"modified":"2023-09-23T23:32:57.852217+0000","services":{}},"progress_events":{"833ef15d-07ae-4299-b4e4-2cf94e068147":{"message":"Upgrade to 18.2.0-460-gbb0daeff (10m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true},"c67c6d89-9cd0-495b-b1de-fa24b5d12727":{"message":"Global Recovery Event (3m)\n [=========================...] (remaining: 17s)","progress":0.92436975240707397,"add_to_ceph_s":true}}} 2023-09-23T23:37:46.303 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:46 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2154980527' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:46.303 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:46 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2154980527' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:46 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2154980527' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:46.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:37:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:37:46] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2023-09-23T23:37:47.343 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:47.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:47 smithi110 ceph-mon[212939]: pgmap v1700: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 380 op/s; 22423/113988 objects degraded (19.671%); 2283/113988 objects misplaced (2.003%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:37:47.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:47 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3745875671' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:37:47.345 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:47 smithi110 ceph-mon[221102]: pgmap v1700: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 380 op/s; 22423/113988 objects degraded (19.671%); 2283/113988 objects misplaced (2.003%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:37:47.345 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:47 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3745875671' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:37:47.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:47.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:47.395 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:37:47.395 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:37:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:47 smithi138 ceph-mon[193606]: pgmap v1700: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 380 op/s; 22423/113988 objects degraded (19.671%); 2283/113988 objects misplaced (2.003%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:37:47.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:47 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3745875671' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:37:47.447 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:47.509 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:47.658 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:47.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:47.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:47.771 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:47.799 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:47.847 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:47.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:47.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:47.955 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:47.962 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:47.979 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:48.002 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:48.178 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:48.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:48.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:48.288 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:37:48.326 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:48.440 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:37:48.441 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:37:48.802 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:37:48.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:48.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:37:48.917 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:48.946 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:48.997 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:49.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:49.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:49.102 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:49.133 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:49 smithi138 ceph-mon[193606]: pgmap v1701: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 380 op/s; 22423/113988 objects degraded (19.671%); 2283/113988 objects misplaced (2.003%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:37:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:49 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 22423/113988 objects degraded (19.671%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:49.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:49 smithi110 ceph-mon[212939]: pgmap v1701: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 380 op/s; 22423/113988 objects degraded (19.671%); 2283/113988 objects misplaced (2.003%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:37:49.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:49 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 22423/113988 objects degraded (19.671%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:49 smithi110 ceph-mon[221102]: pgmap v1701: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 380 op/s; 22423/113988 objects degraded (19.671%); 2283/113988 objects misplaced (2.003%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:37:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:49 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 22423/113988 objects degraded (19.671%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:49.500 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:49.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:49.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:49.605 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:49.642 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:49.675 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:37:50.308 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:50.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:50.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:50.413 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:50.449 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:50.685 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:37:50.871 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:50.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:50.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:50.980 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:51.013 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:51.063 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:51.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:51.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:51.171 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:51.202 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:51.303 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:51.377 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:51 smithi110 ceph-mon[212939]: pgmap v1702: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 22216/109605 objects degraded (20.269%); 2160/109605 objects misplaced (1.971%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T23:37:51.377 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:51 smithi110 ceph-mon[221102]: pgmap v1702: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 22216/109605 objects degraded (20.269%); 2160/109605 objects misplaced (1.971%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T23:37:51.430 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:51.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:51.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:51 smithi138 ceph-mon[193606]: pgmap v1702: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 22216/109605 objects degraded (20.269%); 2160/109605 objects misplaced (1.971%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T23:37:51.537 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:51.572 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:51.632 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:51.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:51.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:51.718 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:37:51.740 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:51.762 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:37:51.771 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:51.844 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:51.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:51.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:51.951 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:51.997 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:52.117 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:37:52.164 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:52.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:52.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:52.272 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:52.297 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:52.341 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:52.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:52.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:52 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1048592401' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:37:52.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:52 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:37:52.446 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:52.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:52 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1048592401' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:37:52.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:52 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:37:52.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:52 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1048592401' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:37:52.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:52 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:37:52.478 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:52.535 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:52.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:52.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:52.641 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:52.672 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:52.717 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:52.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:52.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:52.823 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:37:52.852 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:52.893 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:37:52.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:52.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:37:53.000 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:53.059 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:53.397 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:53 smithi110 ceph-mon[212939]: from='client.56938 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:37:53.397 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:53 smithi110 ceph-mon[212939]: pgmap v1703: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 374 op/s; 22216/108597 objects degraded (20.457%); 2160/108597 objects misplaced (1.989%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T23:37:53.397 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:53 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:53.397 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:53 smithi110 ceph-mon[212939]: failed to encode map e604 with expected crc 2023-09-23T23:37:53.398 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:53 smithi110 ceph-mon[212939]: failed to encode map e604 with expected crc 2023-09-23T23:37:53.398 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:53 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:37:53.398 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:53 smithi110 ceph-mon[212939]: osdmap e604: 8 total, 8 up, 7 in 2023-09-23T23:37:53.399 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:53 smithi110 ceph-mon[221102]: from='client.56938 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:37:53.399 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:53 smithi110 ceph-mon[221102]: pgmap v1703: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 374 op/s; 22216/108597 objects degraded (20.457%); 2160/108597 objects misplaced (1.989%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T23:37:53.400 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:53 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:53.400 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:53 smithi110 ceph-mon[221102]: failed to encode map e604 with expected crc 2023-09-23T23:37:53.400 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:53 smithi110 ceph-mon[221102]: failed to encode map e604 with expected crc 2023-09-23T23:37:53.400 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:53 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:37:53.400 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:53 smithi110 ceph-mon[221102]: osdmap e604: 8 total, 8 up, 7 in 2023-09-23T23:37:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:53 smithi138 ceph-mon[193606]: from='client.56938 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:37:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:53 smithi138 ceph-mon[193606]: pgmap v1703: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 374 op/s; 22216/108597 objects degraded (20.457%); 2160/108597 objects misplaced (1.989%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T23:37:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:53 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:37:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:53 smithi138 ceph-mon[193606]: failed to encode map e604 with expected crc 2023-09-23T23:37:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:53 smithi138 ceph-mon[193606]: failed to encode map e604 with expected crc 2023-09-23T23:37:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:53 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:37:53.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:53 smithi138 ceph-mon[193606]: osdmap e604: 8 total, 8 up, 7 in 2023-09-23T23:37:53.557 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:53.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:53.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:53.662 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:53.697 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:53.808 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1702,"stamp":"2023-09-23T23:37:49.947944+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2393355509,"num_objects":36535,"num_object_clones":0,"num_object_copies":109605,"num_objects_missing_on_primary":1265,"num_objects_missing":1265,"num_objects_degraded":22216,"num_objects_misplaced":2160,"num_objects_unfound":0,"num_objects_dirty":36535,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":139122,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19220,"num_bytes_recovered":1256911272,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94869,"ondisk_log_size":94869,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12261288,"kb_used_data":6470064,"kb_used_omap":29,"kb_used_meta":5791138,"kb_avail":643954776,"statfs":{"total":671965249536,"available":659409690624,"internally_reserved":0,"allocated":6625345536,"data_stored":6608849536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5930126035},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-245891072,"num_objects":-3752,"num_object_clones":0,"num_object_copies":-11256,"num_objects_missing_on_primary":-240,"num_objects_missing":-240,"num_objects_degraded":-449,"num_objects_misplaced":-240,"num_objects_unfound":0,"num_objects_dirty":-3752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3752,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":691,"num_bytes_recovered":45285376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003094"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":799,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.783079+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:37:34.783079+0000","last_peered":"2023-09-23T23:37:34.783079+0000","last_clean":"2023-09-23T23:37:34.783079+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:37:34.783079+0000","last_undegraded":"2023-09-23T23:37:34.783079+0000","last_fullsized":"2023-09-23T23:37:34.783079+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":358,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.781707+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:37:34.781707+0000","last_peered":"2023-09-23T23:37:34.781707+0000","last_clean":"2023-09-23T23:37:34.781707+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:37:34.781707+0000","last_undegraded":"2023-09-23T23:37:34.781707+0000","last_fullsized":"2023-09-23T23:37:34.781707+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":585,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.795982+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:37:34.795982+0000","last_peered":"2023-09-23T23:37:34.795982+0000","last_clean":"2023-09-23T23:37:34.795982+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:37:34.795982+0000","last_undegraded":"2023-09-23T23:37:34.795982+0000","last_fullsized":"2023-09-23T23:37:34.795982+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":752,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.783652+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:37:34.783652+0000","last_peered":"2023-09-23T23:37:34.783652+0000","last_clean":"2023-09-23T23:37:34.783652+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:37:34.783652+0000","last_undegraded":"2023-09-23T23:37:34.783652+0000","last_fullsized":"2023-09-23T23:37:34.783652+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":322,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.794373+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:37:34.794373+0000","last_peered":"2023-09-23T23:37:34.794373+0000","last_clean":"2023-09-23T23:37:34.794373+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:37:34.794373+0000","last_undegraded":"2023-09-23T23:37:34.794373+0000","last_fullsized":"2023-09-23T23:37:34.794373+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":564,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.794763+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:37:34.794763+0000","last_peered":"2023-09-23T23:37:34.794763+0000","last_clean":"2023-09-23T23:37:34.794763+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:37:34.794763+0000","last_undegraded":"2023-09-23T23:37:34.794763+0000","last_fullsized":"2023-09-23T23:37:34.794763+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":764,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.787073+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:37:34.787073+0000","last_peered":"2023-09-23T23:37:34.787073+0000","last_clean":"2023-09-23T23:37:34.787073+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:37:34.787073+0000","last_undegraded":"2023-09-23T23:37:34.787073+0000","last_fullsized":"2023-09-23T23:37:34.787073+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":622,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:38.165929+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:37:38.165929+0000","last_peered":"2023-09-23T23:37:38.165929+0000","last_clean":"2023-09-23T23:37:38.165929+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:37:38.165929+0000","last_undegraded":"2023-09-23T23:37:38.165929+0000","last_fullsized":"2023-09-23T23:37:38.165929+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":117,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.792874+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:37:34.792874+0000","last_peered":"2023-09-23T23:37:34.792874+0000","last_clean":"2023-09-23T23:37:34.792874+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:37:34.792874+0000","last_undegraded":"2023-09-23T23:37:34.792874+0000","last_fullsized":"2023-09-23T23:37:34.792874+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":923,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.787822+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:37:34.787822+0000","last_peered":"2023-09-23T23:37:34.787822+0000","last_clean":"2023-09-23T23:37:34.787822+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:37:34.787822+0000","last_undegraded":"2023-09-23T23:37:34.787822+0000","last_fullsized":"2023-09-23T23:37:34.787822+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":255,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.784041+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:37:34.784041+0000","last_peered":"2023-09-23T23:37:34.784041+0000","last_clean":"2023-09-23T23:37:34.784041+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:37:34.784041+0000","last_undegraded":"2023-09-23T23:37:34.784041+0000","last_fullsized":"2023-09-23T23:37:34.784041+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":604,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.793981+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:37:34.793981+0000","last_peered":"2023-09-23T23:37:34.793981+0000","last_clean":"2023-09-23T23:37:34.793981+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:37:34.793981+0000","last_undegraded":"2023-09-23T23:37:34.793981+0000","last_fullsized":"2023-09-23T23:37:34.793981+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":896,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.779165+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:37:34.779165+0000","last_peered":"2023-09-23T23:37:34.779165+0000","last_clean":"2023-09-23T23:37:34.779165+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:37:34.779165+0000","last_undegraded":"2023-09-23T23:37:34.779165+0000","last_fullsized":"2023-09-23T23:37:34.779165+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":234,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.786679+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:37:34.786679+0000","last_peered":"2023-09-23T23:37:34.786679+0000","last_clean":"2023-09-23T23:37:34.786679+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:37:34.786679+0000","last_undegraded":"2023-09-23T23:37:34.786679+0000","last_fullsized":"2023-09-23T23:37:34.786679+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":565,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.794189+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:37:34.794189+0000","last_peered":"2023-09-23T23:37:34.794189+0000","last_clean":"2023-09-23T23:37:34.794189+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:37:34.794189+0000","last_undegraded":"2023-09-23T23:37:34.794189+0000","last_fullsized":"2023-09-23T23:37:34.794189+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1057,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.783896+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:37:34.783896+0000","last_peered":"2023-09-23T23:37:34.783896+0000","last_clean":"2023-09-23T23:37:34.783896+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:37:34.783896+0000","last_undegraded":"2023-09-23T23:37:34.783896+0000","last_fullsized":"2023-09-23T23:37:34.783896+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":276,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.794971+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:37:34.794971+0000","last_peered":"2023-09-23T23:37:34.794971+0000","last_clean":"2023-09-23T23:37:34.794971+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:37:34.794971+0000","last_undegraded":"2023-09-23T23:37:34.794971+0000","last_fullsized":"2023-09-23T23:37:34.794971+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":136,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.791474+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:37:34.791474+0000","last_peered":"2023-09-23T23:37:34.791474+0000","last_clean":"2023-09-23T23:37:34.791474+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:37:34.791474+0000","last_undegraded":"2023-09-23T23:37:34.791474+0000","last_fullsized":"2023-09-23T23:37:34.791474+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":454,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.794893+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:37:34.794893+0000","last_peered":"2023-09-23T23:37:34.794893+0000","last_clean":"2023-09-23T23:37:34.794893+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:37:34.794893+0000","last_undegraded":"2023-09-23T23:37:34.794893+0000","last_fullsized":"2023-09-23T23:37:34.794893+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":177,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.790819+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:37:34.790819+0000","last_peered":"2023-09-23T23:37:34.790819+0000","last_clean":"2023-09-23T23:37:34.790819+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:37:34.790819+0000","last_undegraded":"2023-09-23T23:37:34.790819+0000","last_fullsized":"2023-09-23T23:37:34.790819+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":544,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.789752+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:37:34.789752+0000","last_peered":"2023-09-23T23:37:34.789752+0000","last_clean":"2023-09-23T23:37:34.789752+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:37:34.789752+0000","last_undegraded":"2023-09-23T23:37:34.789752+0000","last_fullsized":"2023-09-23T23:37:34.789752+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":758,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.783225+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:37:34.783225+0000","last_peered":"2023-09-23T23:37:34.783225+0000","last_clean":"2023-09-23T23:37:34.783225+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:37:34.783225+0000","last_undegraded":"2023-09-23T23:37:34.783225+0000","last_fullsized":"2023-09-23T23:37:34.783225+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":150,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.778298+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:37:34.778298+0000","last_peered":"2023-09-23T23:37:34.778298+0000","last_clean":"2023-09-23T23:37:34.778298+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:37:34.778298+0000","last_undegraded":"2023-09-23T23:37:34.778298+0000","last_fullsized":"2023-09-23T23:37:34.778298+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":603,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.794402+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:37:34.794402+0000","last_peered":"2023-09-23T23:37:34.794402+0000","last_clean":"2023-09-23T23:37:34.794402+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:37:34.794402+0000","last_undegraded":"2023-09-23T23:37:34.794402+0000","last_fullsized":"2023-09-23T23:37:34.794402+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":888,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.791072+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:37:34.791072+0000","last_peered":"2023-09-23T23:37:34.791072+0000","last_clean":"2023-09-23T23:37:34.791072+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:37:34.791072+0000","last_undegraded":"2023-09-23T23:37:34.791072+0000","last_fullsized":"2023-09-23T23:37:34.791072+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":198,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.798480+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:37:34.798480+0000","last_peered":"2023-09-23T23:37:34.798480+0000","last_clean":"2023-09-23T23:37:34.798480+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:37:34.798480+0000","last_undegraded":"2023-09-23T23:37:34.798480+0000","last_fullsized":"2023-09-23T23:37:34.798480+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":325,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.795814+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:37:34.795814+0000","last_peered":"2023-09-23T23:37:34.795814+0000","last_clean":"2023-09-23T23:37:34.795814+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:37:34.795814+0000","last_undegraded":"2023-09-23T23:37:34.795814+0000","last_fullsized":"2023-09-23T23:37:34.795814+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":566,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.797974+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:37:34.797974+0000","last_peered":"2023-09-23T23:37:34.797974+0000","last_clean":"2023-09-23T23:37:34.797974+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:37:34.797974+0000","last_undegraded":"2023-09-23T23:37:34.797974+0000","last_fullsized":"2023-09-23T23:37:34.797974+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":471,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.793973+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:37:34.793973+0000","last_peered":"2023-09-23T23:37:34.793973+0000","last_clean":"2023-09-23T23:37:34.793973+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:37:34.793973+0000","last_undegraded":"2023-09-23T23:37:34.793973+0000","last_fullsized":"2023-09-23T23:37:34.793973+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":559,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.795023+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:37:34.795023+0000","last_peered":"2023-09-23T23:37:34.795023+0000","last_clean":"2023-09-23T23:37:34.795023+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:37:34.795023+0000","last_undegraded":"2023-09-23T23:37:34.795023+0000","last_fullsized":"2023-09-23T23:37:34.795023+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":966,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.783342+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:37:34.783342+0000","last_peered":"2023-09-23T23:37:34.783342+0000","last_clean":"2023-09-23T23:37:34.783342+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:37:34.783342+0000","last_undegraded":"2023-09-23T23:37:34.783342+0000","last_fullsized":"2023-09-23T23:37:34.783342+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":167,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.791290+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:37:34.791290+0000","last_peered":"2023-09-23T23:37:34.791290+0000","last_clean":"2023-09-23T23:37:34.791290+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:37:34.791290+0000","last_undegraded":"2023-09-23T23:37:34.791290+0000","last_fullsized":"2023-09-23T23:37:34.791290+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":244,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.794705+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:37:34.794705+0000","last_peered":"2023-09-23T23:37:34.794705+0000","last_clean":"2023-09-23T23:37:34.794705+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:37:34.794705+0000","last_undegraded":"2023-09-23T23:37:34.794705+0000","last_fullsized":"2023-09-23T23:37:34.794705+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"603'8601","reported_seq":8687,"reported_epoch":603,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:49.159426+0000","last_change":"2023-09-23T23:33:52.884769+0000","last_active":"2023-09-23T23:37:49.159426+0000","last_peered":"2023-09-23T23:37:49.159426+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:37:49.159426+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"552'3200","ondisk_log_start":"552'3200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5401,"log_dups_size":0,"ondisk_log_size":5401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147259392,"num_objects":2247,"num_object_clones":0,"num_object_copies":6741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8601,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":1221},{"shards":"2,6","objects":-2151}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":937,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.778794+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:37:34.778794+0000","last_peered":"2023-09-23T23:37:34.778794+0000","last_clean":"2023-09-23T23:37:34.778794+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:37:34.778794+0000","last_undegraded":"2023-09-23T23:37:34.778794+0000","last_fullsized":"2023-09-23T23:37:34.778794+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":244,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.788054+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:37:34.788054+0000","last_peered":"2023-09-23T23:37:34.788054+0000","last_clean":"2023-09-23T23:37:34.788054+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:37:34.788054+0000","last_undegraded":"2023-09-23T23:37:34.788054+0000","last_fullsized":"2023-09-23T23:37:34.788054+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":234,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.783023+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:37:34.783023+0000","last_peered":"2023-09-23T23:37:34.783023+0000","last_clean":"2023-09-23T23:37:34.783023+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:37:34.783023+0000","last_undegraded":"2023-09-23T23:37:34.783023+0000","last_fullsized":"2023-09-23T23:37:34.783023+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"603'8648","reported_seq":8734,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:44.883638+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:37:44.883638+0000","last_peered":"2023-09-23T23:37:44.883638+0000","last_clean":"2023-09-23T23:37:44.883638+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:37:44.883638+0000","last_undegraded":"2023-09-23T23:37:44.883638+0000","last_fullsized":"2023-09-23T23:37:44.883638+0000","mapping_epoch":566,"log_start":"551'2500","ondisk_log_start":"551'2500","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6148,"log_dups_size":0,"ondisk_log_size":6148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156237824,"num_objects":2384,"num_object_clones":0,"num_object_copies":7152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8648,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1015,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.794142+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:37:34.794142+0000","last_peered":"2023-09-23T23:37:34.794142+0000","last_clean":"2023-09-23T23:37:34.794142+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:37:34.794142+0000","last_undegraded":"2023-09-23T23:37:34.794142+0000","last_fullsized":"2023-09-23T23:37:34.794142+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":234,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.787438+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:37:34.787438+0000","last_peered":"2023-09-23T23:37:34.787438+0000","last_clean":"2023-09-23T23:37:34.787438+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:37:34.787438+0000","last_undegraded":"2023-09-23T23:37:34.787438+0000","last_fullsized":"2023-09-23T23:37:34.787438+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":454,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.794446+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:37:34.794446+0000","last_peered":"2023-09-23T23:37:34.794446+0000","last_clean":"2023-09-23T23:37:34.794446+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:37:34.794446+0000","last_undegraded":"2023-09-23T23:37:34.794446+0000","last_fullsized":"2023-09-23T23:37:34.794446+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"603'8789","reported_seq":16440,"reported_epoch":603,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:37:48.207843+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:37:48.207843+0000","last_peered":"2023-09-23T23:37:48.207843+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:37:48.207843+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:37:48.207843+0000","mapping_epoch":571,"log_start":"552'3500","ondisk_log_start":"552'3500","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5289,"log_dups_size":0,"ondisk_log_size":5289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148045824,"num_objects":2259,"num_object_clones":0,"num_object_copies":6777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1364,"num_objects_misplaced":895,"num_objects_unfound":0,"num_objects_dirty":2259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8789,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3767,"num_bytes_recovered":246874112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":1242},{"shards":"3,5,7","objects":1017}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":913,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.792938+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:37:34.792938+0000","last_peered":"2023-09-23T23:37:34.792938+0000","last_clean":"2023-09-23T23:37:34.792938+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:37:34.792938+0000","last_undegraded":"2023-09-23T23:37:34.792938+0000","last_fullsized":"2023-09-23T23:37:34.792938+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":341,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.783625+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:37:34.783625+0000","last_peered":"2023-09-23T23:37:34.783625+0000","last_clean":"2023-09-23T23:37:34.783625+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:37:34.783625+0000","last_undegraded":"2023-09-23T23:37:34.783625+0000","last_fullsized":"2023-09-23T23:37:34.783625+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":117,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.791604+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:37:34.791604+0000","last_peered":"2023-09-23T23:37:34.791604+0000","last_clean":"2023-09-23T23:37:34.791604+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:37:34.791604+0000","last_undegraded":"2023-09-23T23:37:34.791604+0000","last_fullsized":"2023-09-23T23:37:34.791604+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"603'8733","reported_seq":10806,"reported_epoch":603,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:48.209270+0000","last_change":"2023-09-23T23:34:35.750428+0000","last_active":"2023-09-23T23:37:48.209270+0000","last_peered":"2023-09-23T23:37:48.209270+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:37:48.209270+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"552'3400","ondisk_log_start":"552'3400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5333,"log_dups_size":0,"ondisk_log_size":5333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154468352,"num_objects":2357,"num_object_clones":0,"num_object_copies":7071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8733,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":991,"num_bytes_recovered":64946176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":1347},{"shards":"3,5","objects":-1187}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":510,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.794957+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:37:34.794957+0000","last_peered":"2023-09-23T23:37:34.794957+0000","last_clean":"2023-09-23T23:37:34.794957+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:37:34.794957+0000","last_undegraded":"2023-09-23T23:37:34.794957+0000","last_fullsized":"2023-09-23T23:37:34.794957+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":444,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.794290+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:37:34.794290+0000","last_peered":"2023-09-23T23:37:34.794290+0000","last_clean":"2023-09-23T23:37:34.794290+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:37:34.794290+0000","last_undegraded":"2023-09-23T23:37:34.794290+0000","last_fullsized":"2023-09-23T23:37:34.794290+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":600,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.779625+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:37:34.779625+0000","last_peered":"2023-09-23T23:37:34.779625+0000","last_clean":"2023-09-23T23:37:34.779625+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:37:34.779625+0000","last_undegraded":"2023-09-23T23:37:34.779625+0000","last_fullsized":"2023-09-23T23:37:34.779625+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"603'8896","reported_seq":10227,"reported_epoch":603,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:49.314353+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:37:49.314353+0000","last_peered":"2023-09-23T23:37:49.314353+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:37:49.314353+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"551'3200","ondisk_log_start":"551'3200","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5696,"log_dups_size":0,"ondisk_log_size":5696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148504576,"num_objects":2266,"num_object_clones":0,"num_object_copies":6798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8896,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":1220},{"shards":"3,5","objects":-3194},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":731,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.794115+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:37:34.794115+0000","last_peered":"2023-09-23T23:37:34.794115+0000","last_clean":"2023-09-23T23:37:34.794115+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:37:34.794115+0000","last_undegraded":"2023-09-23T23:37:34.794115+0000","last_fullsized":"2023-09-23T23:37:34.794115+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":613,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.797882+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:37:34.797882+0000","last_peered":"2023-09-23T23:37:34.797882+0000","last_clean":"2023-09-23T23:37:34.797882+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:37:34.797882+0000","last_undegraded":"2023-09-23T23:37:34.797882+0000","last_fullsized":"2023-09-23T23:37:34.797882+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":524,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.794215+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:37:34.794215+0000","last_peered":"2023-09-23T23:37:34.794215+0000","last_clean":"2023-09-23T23:37:34.794215+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:37:34.794215+0000","last_undegraded":"2023-09-23T23:37:34.794215+0000","last_fullsized":"2023-09-23T23:37:34.794215+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"603'8839","reported_seq":8916,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:48.936281+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:37:48.936281+0000","last_peered":"2023-09-23T23:37:48.936281+0000","last_clean":"2023-09-23T23:37:48.936281+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:37:48.936281+0000","last_undegraded":"2023-09-23T23:37:48.936281+0000","last_fullsized":"2023-09-23T23:37:48.936281+0000","mapping_epoch":534,"log_start":"552'3400","ondisk_log_start":"552'3400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5439,"log_dups_size":0,"ondisk_log_size":5439,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146866176,"num_objects":2241,"num_object_clones":0,"num_object_copies":6723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8839,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":887,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:39.978597+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:37:39.978597+0000","last_peered":"2023-09-23T23:37:39.978597+0000","last_clean":"2023-09-23T23:37:39.978597+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:37:39.978597+0000","last_undegraded":"2023-09-23T23:37:39.978597+0000","last_fullsized":"2023-09-23T23:37:39.978597+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":127,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.791560+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:37:34.791560+0000","last_peered":"2023-09-23T23:37:34.791560+0000","last_clean":"2023-09-23T23:37:34.791560+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:37:34.791560+0000","last_undegraded":"2023-09-23T23:37:34.791560+0000","last_fullsized":"2023-09-23T23:37:34.791560+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":469,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.778997+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:37:34.778997+0000","last_peered":"2023-09-23T23:37:34.778997+0000","last_clean":"2023-09-23T23:37:34.778997+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:37:34.778997+0000","last_undegraded":"2023-09-23T23:37:34.778997+0000","last_fullsized":"2023-09-23T23:37:34.778997+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"603'8863","reported_seq":15582,"reported_epoch":603,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:37:49.193478+0000","last_change":"2023-09-23T23:34:35.785270+0000","last_active":"2023-09-23T23:37:49.193478+0000","last_peered":"2023-09-23T23:37:49.193478+0000","last_clean":"2023-09-23T23:33:50.504296+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:37:49.193478+0000","last_undegraded":"2023-09-23T23:34:35.780981+0000","last_fullsized":"2023-09-23T23:37:49.193478+0000","mapping_epoch":567,"log_start":"552'3500","ondisk_log_start":"552'3500","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5363,"log_dups_size":0,"ondisk_log_size":5363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146866176,"num_objects":2241,"num_object_clones":0,"num_object_copies":6723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1236,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8863,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3311,"num_bytes_recovered":216989696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":1236},{"shards":"5,6,7","objects":1005}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":611,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.783553+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:37:34.783553+0000","last_peered":"2023-09-23T23:37:34.783553+0000","last_clean":"2023-09-23T23:37:34.783553+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:37:34.783553+0000","last_undegraded":"2023-09-23T23:37:34.783553+0000","last_fullsized":"2023-09-23T23:37:34.783553+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":466,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.779719+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:37:34.779719+0000","last_peered":"2023-09-23T23:37:34.779719+0000","last_clean":"2023-09-23T23:37:34.779719+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:37:34.779719+0000","last_undegraded":"2023-09-23T23:37:34.779719+0000","last_fullsized":"2023-09-23T23:37:34.779719+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":554,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.790365+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:37:34.790365+0000","last_peered":"2023-09-23T23:37:34.790365+0000","last_clean":"2023-09-23T23:37:34.790365+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:37:34.790365+0000","last_undegraded":"2023-09-23T23:37:34.790365+0000","last_fullsized":"2023-09-23T23:37:34.790365+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"603'8782","reported_seq":17822,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:49.291678+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:37:49.291678+0000","last_peered":"2023-09-23T23:37:49.291678+0000","last_clean":"2023-09-23T23:37:49.291678+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:37:49.291678+0000","last_undegraded":"2023-09-23T23:37:49.291678+0000","last_fullsized":"2023-09-23T23:37:49.291678+0000","mapping_epoch":579,"log_start":"551'3100","ondisk_log_start":"551'3100","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5682,"log_dups_size":0,"ondisk_log_size":5682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146407424,"num_objects":2234,"num_object_clones":0,"num_object_copies":6702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8782,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":945,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.781559+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:37:34.781559+0000","last_peered":"2023-09-23T23:37:34.781559+0000","last_clean":"2023-09-23T23:37:34.781559+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:37:34.781559+0000","last_undegraded":"2023-09-23T23:37:34.781559+0000","last_fullsized":"2023-09-23T23:37:34.781559+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":245,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.793901+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:37:34.793901+0000","last_peered":"2023-09-23T23:37:34.793901+0000","last_clean":"2023-09-23T23:37:34.793901+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:37:34.793901+0000","last_undegraded":"2023-09-23T23:37:34.793901+0000","last_fullsized":"2023-09-23T23:37:34.793901+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":533,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.794905+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:37:34.794905+0000","last_peered":"2023-09-23T23:37:34.794905+0000","last_clean":"2023-09-23T23:37:34.794905+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:37:34.794905+0000","last_undegraded":"2023-09-23T23:37:34.794905+0000","last_fullsized":"2023-09-23T23:37:34.794905+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"603'8430","reported_seq":8510,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:48.217163+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:37:48.217163+0000","last_peered":"2023-09-23T23:37:48.217163+0000","last_clean":"2023-09-23T23:37:48.217163+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:37:48.217163+0000","last_undegraded":"2023-09-23T23:37:48.217163+0000","last_fullsized":"2023-09-23T23:37:48.217163+0000","mapping_epoch":534,"log_start":"551'3100","ondisk_log_start":"551'3100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5330,"log_dups_size":0,"ondisk_log_size":5330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150601728,"num_objects":2298,"num_object_clones":0,"num_object_copies":6894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8430,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":924,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.791923+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:37:34.791923+0000","last_peered":"2023-09-23T23:37:34.791923+0000","last_clean":"2023-09-23T23:37:34.791923+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:37:34.791923+0000","last_undegraded":"2023-09-23T23:37:34.791923+0000","last_fullsized":"2023-09-23T23:37:34.791923+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":313,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.786044+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:37:34.786044+0000","last_peered":"2023-09-23T23:37:34.786044+0000","last_clean":"2023-09-23T23:37:34.786044+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:37:34.786044+0000","last_undegraded":"2023-09-23T23:37:34.786044+0000","last_fullsized":"2023-09-23T23:37:34.786044+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":618,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.783769+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:37:34.783769+0000","last_peered":"2023-09-23T23:37:34.783769+0000","last_clean":"2023-09-23T23:37:34.783769+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:37:34.783769+0000","last_undegraded":"2023-09-23T23:37:34.783769+0000","last_fullsized":"2023-09-23T23:37:34.783769+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":199,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.783146+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:37:34.783146+0000","last_peered":"2023-09-23T23:37:34.783146+0000","last_clean":"2023-09-23T23:37:34.783146+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:37:34.783146+0000","last_undegraded":"2023-09-23T23:37:34.783146+0000","last_fullsized":"2023-09-23T23:37:34.783146+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"603'8824","reported_seq":15413,"reported_epoch":603,"state":"active+recovering","last_fresh":"2023-09-23T23:37:48.230454+0000","last_change":"2023-09-23T23:34:35.815566+0000","last_active":"2023-09-23T23:37:48.230454+0000","last_peered":"2023-09-23T23:37:48.230454+0000","last_clean":"2023-09-23T23:33:50.519308+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:37:48.230454+0000","last_undegraded":"2023-09-23T23:37:48.230454+0000","last_fullsized":"2023-09-23T23:37:48.230454+0000","mapping_epoch":566,"log_start":"552'3522","ondisk_log_start":"552'3522","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5302,"log_dups_size":0,"ondisk_log_size":5302,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153550872,"num_objects":2344,"num_object_clones":0,"num_object_copies":7032,"num_objects_missing_on_primary":1265,"num_objects_missing":1265,"num_objects_degraded":0,"num_objects_misplaced":1265,"num_objects_unfound":0,"num_objects_dirty":2344,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8824,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3239,"num_bytes_recovered":212271104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,3,4,7","objects":11},{"shards":"1,3,7","objects":1068},{"shards":"1,4,7","objects":1265}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":687,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.778744+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:37:34.778744+0000","last_peered":"2023-09-23T23:37:34.778744+0000","last_clean":"2023-09-23T23:37:34.778744+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:37:34.778744+0000","last_undegraded":"2023-09-23T23:37:34.778744+0000","last_fullsized":"2023-09-23T23:37:34.778744+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":251,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.791549+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:37:34.791549+0000","last_peered":"2023-09-23T23:37:34.791549+0000","last_clean":"2023-09-23T23:37:34.791549+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:37:34.791549+0000","last_undegraded":"2023-09-23T23:37:34.791549+0000","last_fullsized":"2023-09-23T23:37:34.791549+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":535,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.789820+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:37:34.789820+0000","last_peered":"2023-09-23T23:37:34.789820+0000","last_clean":"2023-09-23T23:37:34.789820+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:37:34.789820+0000","last_undegraded":"2023-09-23T23:37:34.789820+0000","last_fullsized":"2023-09-23T23:37:34.789820+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":236,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.779248+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:37:34.779248+0000","last_peered":"2023-09-23T23:37:34.779248+0000","last_clean":"2023-09-23T23:37:34.779248+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:37:34.779248+0000","last_undegraded":"2023-09-23T23:37:34.779248+0000","last_fullsized":"2023-09-23T23:37:34.779248+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"603'8505","reported_seq":10561,"reported_epoch":603,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:49.275679+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:37:49.275679+0000","last_peered":"2023-09-23T23:37:49.275679+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:37:49.275679+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"551'2800","ondisk_log_start":"551'2800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5705,"log_dups_size":0,"ondisk_log_size":5705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148570112,"num_objects":2267,"num_object_clones":0,"num_object_copies":6801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8505,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":1256},{"shards":"5,7","objects":-1125}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":905,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.781467+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:37:34.781467+0000","last_peered":"2023-09-23T23:37:34.781467+0000","last_clean":"2023-09-23T23:37:34.781467+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:37:34.781467+0000","last_undegraded":"2023-09-23T23:37:34.781467+0000","last_fullsized":"2023-09-23T23:37:34.781467+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":342,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.779626+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:37:34.779626+0000","last_peered":"2023-09-23T23:37:34.779626+0000","last_clean":"2023-09-23T23:37:34.779626+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:37:34.779626+0000","last_undegraded":"2023-09-23T23:37:34.779626+0000","last_fullsized":"2023-09-23T23:37:34.779626+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":251,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.778457+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:37:34.778457+0000","last_peered":"2023-09-23T23:37:34.778457+0000","last_clean":"2023-09-23T23:37:34.778457+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:37:34.778457+0000","last_undegraded":"2023-09-23T23:37:34.778457+0000","last_fullsized":"2023-09-23T23:37:34.778457+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":215,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.785774+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:37:34.785774+0000","last_peered":"2023-09-23T23:37:34.785774+0000","last_clean":"2023-09-23T23:37:34.785774+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:37:34.785774+0000","last_undegraded":"2023-09-23T23:37:34.785774+0000","last_fullsized":"2023-09-23T23:37:34.785774+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"603'8698","reported_seq":8785,"reported_epoch":603,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:37:49.503044+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:37:49.503044+0000","last_peered":"2023-09-23T23:37:49.503044+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:37:49.503044+0000","last_undegraded":"2023-09-23T23:37:49.503044+0000","last_fullsized":"2023-09-23T23:37:49.503044+0000","mapping_epoch":562,"log_start":"551'3000","ondisk_log_start":"551'3000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5698,"log_dups_size":0,"ondisk_log_size":5698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144441344,"num_objects":2204,"num_object_clones":0,"num_object_copies":6612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8698,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":2204}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":618,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.778376+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:37:34.778376+0000","last_peered":"2023-09-23T23:37:34.778376+0000","last_clean":"2023-09-23T23:37:34.778376+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:37:34.778376+0000","last_undegraded":"2023-09-23T23:37:34.778376+0000","last_fullsized":"2023-09-23T23:37:34.778376+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":889,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.791114+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:37:34.791114+0000","last_peered":"2023-09-23T23:37:34.791114+0000","last_clean":"2023-09-23T23:37:34.791114+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:37:34.791114+0000","last_undegraded":"2023-09-23T23:37:34.791114+0000","last_fullsized":"2023-09-23T23:37:34.791114+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":565,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.798305+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:37:34.798305+0000","last_peered":"2023-09-23T23:37:34.798305+0000","last_clean":"2023-09-23T23:37:34.798305+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:37:34.798305+0000","last_undegraded":"2023-09-23T23:37:34.798305+0000","last_fullsized":"2023-09-23T23:37:34.798305+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":127,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.790288+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:37:34.790288+0000","last_peered":"2023-09-23T23:37:34.790288+0000","last_clean":"2023-09-23T23:37:34.790288+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:37:34.790288+0000","last_undegraded":"2023-09-23T23:37:34.790288+0000","last_fullsized":"2023-09-23T23:37:34.790288+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"603'8474","reported_seq":8551,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:49.021470+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:37:49.021470+0000","last_peered":"2023-09-23T23:37:49.021470+0000","last_clean":"2023-09-23T23:37:49.021470+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:37:49.021470+0000","last_undegraded":"2023-09-23T23:37:49.021470+0000","last_fullsized":"2023-09-23T23:37:49.021470+0000","mapping_epoch":534,"log_start":"551'3100","ondisk_log_start":"551'3100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5374,"log_dups_size":0,"ondisk_log_size":5374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148242432,"num_objects":2262,"num_object_clones":0,"num_object_copies":6786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8474,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":715,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.789683+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:37:34.789683+0000","last_peered":"2023-09-23T23:37:34.789683+0000","last_clean":"2023-09-23T23:37:34.789683+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:37:34.789683+0000","last_undegraded":"2023-09-23T23:37:34.789683+0000","last_fullsized":"2023-09-23T23:37:34.789683+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":530,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.791742+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:37:34.791742+0000","last_peered":"2023-09-23T23:37:34.791742+0000","last_clean":"2023-09-23T23:37:34.791742+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:37:34.791742+0000","last_undegraded":"2023-09-23T23:37:34.791742+0000","last_fullsized":"2023-09-23T23:37:34.791742+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":444,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.793881+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:37:34.793881+0000","last_peered":"2023-09-23T23:37:34.793881+0000","last_clean":"2023-09-23T23:37:34.793881+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:37:34.793881+0000","last_undegraded":"2023-09-23T23:37:34.793881+0000","last_fullsized":"2023-09-23T23:37:34.793881+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":44,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.779912+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:37:34.779912+0000","last_peered":"2023-09-23T23:37:34.779912+0000","last_clean":"2023-09-23T23:37:34.779912+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:37:34.779912+0000","last_undegraded":"2023-09-23T23:37:34.779912+0000","last_fullsized":"2023-09-23T23:37:34.779912+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"603'8694","reported_seq":12371,"reported_epoch":603,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:44.946459+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:37:44.946459+0000","last_peered":"2023-09-23T23:37:44.946459+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:37:44.946459+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"550'2500","ondisk_log_start":"550'2500","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6194,"log_dups_size":0,"ondisk_log_size":6194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150994944,"num_objects":2304,"num_object_clones":0,"num_object_copies":6912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8694,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":-400},{"shards":"1,2,4","objects":1302}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":598,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.790608+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:37:34.790608+0000","last_peered":"2023-09-23T23:37:34.790608+0000","last_clean":"2023-09-23T23:37:34.790608+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:37:34.790608+0000","last_undegraded":"2023-09-23T23:37:34.790608+0000","last_fullsized":"2023-09-23T23:37:34.790608+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":44,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.778716+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:37:34.778716+0000","last_peered":"2023-09-23T23:37:34.778716+0000","last_clean":"2023-09-23T23:37:34.778716+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:37:34.778716+0000","last_undegraded":"2023-09-23T23:37:34.778716+0000","last_fullsized":"2023-09-23T23:37:34.778716+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":143,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.779329+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:37:34.779329+0000","last_peered":"2023-09-23T23:37:34.779329+0000","last_clean":"2023-09-23T23:37:34.779329+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:37:34.779329+0000","last_undegraded":"2023-09-23T23:37:34.779329+0000","last_fullsized":"2023-09-23T23:37:34.779329+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":336,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.783803+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:37:34.783803+0000","last_peered":"2023-09-23T23:37:34.783803+0000","last_clean":"2023-09-23T23:37:34.783803+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:37:34.783803+0000","last_undegraded":"2023-09-23T23:37:34.783803+0000","last_fullsized":"2023-09-23T23:37:34.783803+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"603'8745","reported_seq":8822,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:49.200519+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:37:49.200519+0000","last_peered":"2023-09-23T23:37:49.200519+0000","last_clean":"2023-09-23T23:37:49.200519+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:37:49.200519+0000","last_undegraded":"2023-09-23T23:37:49.200519+0000","last_fullsized":"2023-09-23T23:37:49.200519+0000","mapping_epoch":534,"log_start":"551'3000","ondisk_log_start":"551'3000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5745,"log_dups_size":0,"ondisk_log_size":5745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145424384,"num_objects":2219,"num_object_clones":0,"num_object_copies":6657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8745,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":784,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.779091+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:37:34.779091+0000","last_peered":"2023-09-23T23:37:34.779091+0000","last_clean":"2023-09-23T23:37:34.779091+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:37:34.779091+0000","last_undegraded":"2023-09-23T23:37:34.779091+0000","last_fullsized":"2023-09-23T23:37:34.779091+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":601,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.794835+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:37:34.794835+0000","last_peered":"2023-09-23T23:37:34.794835+0000","last_clean":"2023-09-23T23:37:34.794835+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:37:34.794835+0000","last_undegraded":"2023-09-23T23:37:34.794835+0000","last_fullsized":"2023-09-23T23:37:34.794835+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":207,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.790429+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:37:34.790429+0000","last_peered":"2023-09-23T23:37:34.790429+0000","last_clean":"2023-09-23T23:37:34.790429+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:37:34.790429+0000","last_undegraded":"2023-09-23T23:37:34.790429+0000","last_fullsized":"2023-09-23T23:37:34.790429+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":155,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.778230+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:37:34.778230+0000","last_peered":"2023-09-23T23:37:34.778230+0000","last_clean":"2023-09-23T23:37:34.778230+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:37:34.778230+0000","last_undegraded":"2023-09-23T23:37:34.778230+0000","last_fullsized":"2023-09-23T23:37:34.778230+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"603'8524","reported_seq":8604,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:44.942189+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:37:44.942189+0000","last_peered":"2023-09-23T23:37:44.942189+0000","last_clean":"2023-09-23T23:37:44.942189+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:37:44.942189+0000","last_undegraded":"2023-09-23T23:37:44.942189+0000","last_fullsized":"2023-09-23T23:37:44.942189+0000","mapping_epoch":534,"log_start":"550'2400","ondisk_log_start":"550'2400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6124,"log_dups_size":0,"ondisk_log_size":6124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156237824,"num_objects":2384,"num_object_clones":0,"num_object_copies":7152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8524,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":991,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.085762+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:37:42.085762+0000","last_peered":"2023-09-23T23:37:42.085762+0000","last_clean":"2023-09-23T23:37:42.085762+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:37:42.085762+0000","last_undegraded":"2023-09-23T23:37:42.085762+0000","last_fullsized":"2023-09-23T23:37:42.085762+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":117,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.792049+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:37:34.792049+0000","last_peered":"2023-09-23T23:37:34.792049+0000","last_clean":"2023-09-23T23:37:34.792049+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:37:34.792049+0000","last_undegraded":"2023-09-23T23:37:34.792049+0000","last_fullsized":"2023-09-23T23:37:34.792049+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":48,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.779940+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:37:34.779940+0000","last_peered":"2023-09-23T23:37:34.779940+0000","last_clean":"2023-09-23T23:37:34.779940+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:37:34.779940+0000","last_undegraded":"2023-09-23T23:37:34.779940+0000","last_fullsized":"2023-09-23T23:37:34.779940+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":177,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.783297+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:37:34.783297+0000","last_peered":"2023-09-23T23:37:34.783297+0000","last_clean":"2023-09-23T23:37:34.783297+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:37:34.783297+0000","last_undegraded":"2023-09-23T23:37:34.783297+0000","last_fullsized":"2023-09-23T23:37:34.783297+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1040,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.794032+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:37:34.794032+0000","last_peered":"2023-09-23T23:37:34.794032+0000","last_clean":"2023-09-23T23:37:34.794032+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:37:34.794032+0000","last_undegraded":"2023-09-23T23:37:34.794032+0000","last_fullsized":"2023-09-23T23:37:34.794032+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":44,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.779705+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:37:34.779705+0000","last_peered":"2023-09-23T23:37:34.779705+0000","last_clean":"2023-09-23T23:37:34.779705+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:37:34.779705+0000","last_undegraded":"2023-09-23T23:37:34.779705+0000","last_fullsized":"2023-09-23T23:37:34.779705+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":358,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.779407+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:37:34.779407+0000","last_peered":"2023-09-23T23:37:34.779407+0000","last_clean":"2023-09-23T23:37:34.779407+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:37:34.779407+0000","last_undegraded":"2023-09-23T23:37:34.779407+0000","last_fullsized":"2023-09-23T23:37:34.779407+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":751,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.795192+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:37:34.795192+0000","last_peered":"2023-09-23T23:37:34.795192+0000","last_clean":"2023-09-23T23:37:34.795192+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:37:34.795192+0000","last_undegraded":"2023-09-23T23:37:34.795192+0000","last_fullsized":"2023-09-23T23:37:34.795192+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":105,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.790712+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:37:34.790712+0000","last_peered":"2023-09-23T23:37:34.790712+0000","last_clean":"2023-09-23T23:37:34.790712+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:37:34.790712+0000","last_undegraded":"2023-09-23T23:37:34.790712+0000","last_fullsized":"2023-09-23T23:37:34.790712+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":451,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.797778+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:37:34.797778+0000","last_peered":"2023-09-23T23:37:34.797778+0000","last_clean":"2023-09-23T23:37:34.797778+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:37:34.797778+0000","last_undegraded":"2023-09-23T23:37:34.797778+0000","last_fullsized":"2023-09-23T23:37:34.797778+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":301,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.790170+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:37:34.790170+0000","last_peered":"2023-09-23T23:37:34.790170+0000","last_clean":"2023-09-23T23:37:34.790170+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:37:34.790170+0000","last_undegraded":"2023-09-23T23:37:34.790170+0000","last_fullsized":"2023-09-23T23:37:34.790170+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":290,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.798393+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:37:34.798393+0000","last_peered":"2023-09-23T23:37:34.798393+0000","last_clean":"2023-09-23T23:37:34.798393+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:37:34.798393+0000","last_undegraded":"2023-09-23T23:37:34.798393+0000","last_fullsized":"2023-09-23T23:37:34.798393+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":815,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.790861+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:37:34.790861+0000","last_peered":"2023-09-23T23:37:34.790861+0000","last_clean":"2023-09-23T23:37:34.790861+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:37:34.790861+0000","last_undegraded":"2023-09-23T23:37:34.790861+0000","last_fullsized":"2023-09-23T23:37:34.790861+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":634,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.795100+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:37:34.795100+0000","last_peered":"2023-09-23T23:37:34.795100+0000","last_clean":"2023-09-23T23:37:34.795100+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:37:34.795100+0000","last_undegraded":"2023-09-23T23:37:34.795100+0000","last_fullsized":"2023-09-23T23:37:34.795100+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":618,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:38.165995+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:37:38.165995+0000","last_peered":"2023-09-23T23:37:38.165995+0000","last_clean":"2023-09-23T23:37:38.165995+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:37:38.165995+0000","last_undegraded":"2023-09-23T23:37:38.165995+0000","last_fullsized":"2023-09-23T23:37:38.165995+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":776,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.794477+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:37:34.794477+0000","last_peered":"2023-09-23T23:37:34.794477+0000","last_clean":"2023-09-23T23:37:34.794477+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:37:34.794477+0000","last_undegraded":"2023-09-23T23:37:34.794477+0000","last_fullsized":"2023-09-23T23:37:34.794477+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":447,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.779433+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:37:34.779433+0000","last_peered":"2023-09-23T23:37:34.779433+0000","last_clean":"2023-09-23T23:37:34.779433+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:37:34.779433+0000","last_undegraded":"2023-09-23T23:37:34.779433+0000","last_fullsized":"2023-09-23T23:37:34.779433+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":670,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T23:37:34.791030+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:37:34.791030+0000","last_peered":"2023-09-23T23:37:34.791030+0000","last_clean":"2023-09-23T23:37:34.791030+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:37:34.791030+0000","last_undegraded":"2023-09-23T23:37:34.791030+0000","last_fullsized":"2023-09-23T23:37:34.791030+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":2392719384,"num_objects":36511,"num_object_clones":0,"num_object_copies":109533,"num_objects_missing_on_primary":1265,"num_objects_missing":1265,"num_objects_degraded":22216,"num_objects_misplaced":2160,"num_objects_unfound":0,"num_objects_dirty":36511,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":139045,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19165,"num_bytes_recovered":1255800832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8899538944,"data_stored":8899526728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89823,"ondisk_log_size":89823,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782700,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1331156,"kb_used_data":939340,"kb_used_omap":8,"kb_used_meta":391799,"kb_avail":92413996,"statfs":{"total":95995035648,"available":94631931904,"internally_reserved":0,"allocated":961884160,"data_stored":960198869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":401202245},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":1.048,"15min":2.1080000000000001},"min":{"1min":0.25900000000000001,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":0.51300000000000001,"5min":17.140000000000001,"15min":85.721000000000004},"last":2.4159999999999999},{"interface":"front","average":{"1min":0.47199999999999998,"5min":1.0600000000000001,"15min":2.1829999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":2.331,"5min":11.069000000000001,"15min":85.965000000000003},"last":0.79200000000000004}]},{"osd":1,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":2.532,"15min":3.1309999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":1.5780000000000001,"5min":129.11199999999999,"15min":198.58000000000001},"last":0.438},{"interface":"front","average":{"1min":0.53200000000000003,"5min":2.3039999999999998,"15min":3.1440000000000001},"min":{"1min":0.31900000000000001,"5min":0.247,"15min":0.246},"max":{"1min":2.3690000000000002,"5min":129.02600000000001,"15min":198.71100000000001},"last":0.93300000000000005}]},{"osd":2,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":3.3980000000000001,"5min":2.6880000000000002,"15min":3.8050000000000002},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":53.054000000000002,"5min":78.528999999999996,"15min":203.58000000000001},"last":0.85599999999999998},{"interface":"front","average":{"1min":3.5699999999999998,"5min":1.9299999999999999,"15min":3.2229999999999999},"min":{"1min":0.33400000000000002,"5min":0.27900000000000003,"15min":0.26200000000000001},"max":{"1min":53.023000000000003,"5min":53.023000000000003,"15min":198.69},"last":0.48999999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":8.6839999999999993,"5min":4.4390000000000001,"15min":4.0750000000000002},"min":{"1min":0.33300000000000002,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":100.40600000000001,"5min":125.511,"15min":279.26499999999999},"last":4.8520000000000003},{"interface":"front","average":{"1min":8.7420000000000009,"5min":4.8460000000000001,"15min":4.6180000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29599999999999999},"max":{"1min":100.42400000000001,"5min":125.48999999999999,"15min":279.29899999999998},"last":1.0569999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":1.546,"15min":1.0389999999999999},"min":{"1min":0.28999999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":0.58099999999999996,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":1.341,"15min":0.97599999999999998},"min":{"1min":0.32500000000000001,"5min":0.317,"15min":0.23899999999999999},"max":{"1min":2.3980000000000001,"5min":31.68,"15min":31.68},"last":0.874}]},{"osd":5,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.6220000000000001,"15min":1.863},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.26500000000000001},"max":{"1min":2.4510000000000001,"5min":60.463999999999999,"15min":210.83600000000001},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.5840000000000001,"15min":1.742},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.5940000000000001,"5min":63.006,"15min":207.04599999999999},"last":0.63300000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.827,"15min":1.3320000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.29899999999999999},"max":{"1min":1.619,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":2.052,"15min":1.544},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.389,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.97699999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496177,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1582404,"kb_used_data":862012,"kb_used_omap":0,"kb_used_meta":720384,"kb_avail":92162748,"statfs":{"total":95995035648,"available":94374653952,"internally_reserved":0,"allocated":882700288,"data_stored":881030990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":737673216},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.73499999999999999,"15min":4.4370000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.21099999999999999},"max":{"1min":0.53700000000000003,"5min":6.4450000000000003,"15min":135.31999999999999},"last":2.0430000000000001},{"interface":"front","average":{"1min":0.498,"5min":0.63800000000000001,"15min":4.3869999999999996},"min":{"1min":0.27800000000000002,"5min":0.24399999999999999,"15min":0.23000000000000001},"max":{"1min":1.742,"5min":3.8239999999999998,"15min":135.34299999999999},"last":1.964}]},{"osd":1,"last update":"Sat Sep 23 23:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.90500000000000003,"15min":2.2890000000000001},"min":{"1min":0.28299999999999997,"5min":0.253,"15min":0.245},"max":{"1min":1.8759999999999999,"5min":6.9080000000000004,"15min":89.605000000000004},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.749,"5min":0.72499999999999998,"15min":2.391},"min":{"1min":0.29099999999999998,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":3.8980000000000001,"5min":3.8980000000000001,"15min":91.025999999999996},"last":3.0179999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:37:06 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":3.6930000000000001,"15min":4.4749999999999996},"min":{"1min":0.27900000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.1800000000000002,"5min":67.796999999999997,"15min":127.44499999999999},"last":3.1880000000000002},{"interface":"front","average":{"1min":0.66600000000000004,"5min":3.6749999999999998,"15min":4.8680000000000003},"min":{"1min":0.316,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.2080000000000002,"5min":67.893000000000001,"15min":125.539},"last":1.042}]},{"osd":3,"last update":"Sat Sep 23 23:36:55 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":5.6589999999999998,"15min":4.3159999999999998},"min":{"1min":0.30299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.3610000000000002,"5min":87.283000000000001,"15min":138.87299999999999},"last":3.113},{"interface":"front","average":{"1min":0.47699999999999998,"5min":5.516,"15min":4.625},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0349999999999999,"5min":87.491,"15min":138.976},"last":2.2290000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.755,"15min":0.66700000000000004},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.23400000000000001},"max":{"1min":1.76,"5min":7.101,"15min":8.1999999999999993},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.81399999999999995,"15min":0.76900000000000002},"min":{"1min":0.26700000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.865,"5min":8.3499999999999996,"15min":8.3499999999999996},"last":0.372}]},{"osd":5,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.76100000000000001,"15min":0.79800000000000004},"min":{"1min":0.27700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.847,"5min":11.372,"15min":11.473000000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.876,"15min":0.84999999999999998},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.26900000000000002},"max":{"1min":6.1500000000000004,"5min":7.0129999999999999,"15min":7.5670000000000002},"last":0.51800000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:37:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.1679999999999999,"15min":1.0029999999999999},"min":{"1min":0.32700000000000001,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":0.77000000000000002,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.204,"15min":1.056},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33500000000000002},"max":{"1min":1.792,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.53800000000000003}]}]},{"osd":1,"up_from":112,"seq":481036337518,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1648660,"kb_used_data":739724,"kb_used_omap":4,"kb_used_meta":908923,"kb_avail":92096492,"statfs":{"total":95995035648,"available":94306807808,"internally_reserved":0,"allocated":757477376,"data_stored":754625434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":930738062},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.407,"15min":3.173},"min":{"1min":0.19,"5min":0.19,"15min":0.16700000000000001},"max":{"1min":3.5830000000000002,"5min":47.195999999999998,"15min":130.768},"last":1.6319999999999999},{"interface":"front","average":{"1min":0.377,"5min":1.119,"15min":2.5030000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19},"max":{"1min":1.143,"5min":46.582000000000001,"15min":134.929},"last":5.726}]},{"osd":2,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":3.5659999999999998,"15min":6.7649999999999997},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":1.401,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.66900000000000004,"5min":3.8849999999999998,"15min":6.8339999999999996},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.23,"5min":202.12,"15min":232.21000000000001},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":6.883,"5min":5.8460000000000001,"15min":5.4379999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22600000000000001},"max":{"1min":71.745000000000005,"5min":181.94399999999999,"15min":234.404},"last":1.7809999999999999},{"interface":"front","average":{"1min":6.8639999999999999,"5min":5.8860000000000001,"15min":5.3899999999999997},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":71.786000000000001,"5min":181.98500000000001,"15min":236.33699999999999},"last":1.843}]},{"osd":4,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":0.95099999999999996,"15min":1.399},"min":{"1min":0.32500000000000001,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":4.4880000000000004,"5min":8.0850000000000009,"15min":12.042999999999999},"last":1.113},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.85699999999999998,"15min":1.45},"min":{"1min":0.316,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":4.6660000000000004,"5min":9.5069999999999997,"15min":18.108000000000001},"last":0.61399999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":2.6309999999999998,"15min":1.734},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28899999999999998},"max":{"1min":0.98199999999999998,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.024,"5min":2.661,"15min":2.0710000000000002},"min":{"1min":0.34399999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.194,"5min":151.821,"15min":151.821},"last":1.0780000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.0449999999999999,"15min":1.6659999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.27500000000000002},"max":{"1min":2.306,"5min":8.9269999999999996,"15min":74.087999999999994},"last":0.373},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.2849999999999999,"15min":1.929},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33100000000000002},"max":{"1min":3.29,"5min":8.1400000000000006,"15min":73.974999999999994},"last":1.0329999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":1.5449999999999999,"15min":2.359},"min":{"1min":0.40200000000000002,"5min":0.38700000000000001,"15min":0.32900000000000001},"max":{"1min":2.504,"5min":44.241,"15min":78.308000000000007},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.556,"15min":2.2730000000000001},"min":{"1min":0.41299999999999998,"5min":0.37,"15min":0.32700000000000001},"max":{"1min":3.258,"5min":43.920999999999999,"15min":77.204999999999998},"last":1.145}]}]},{"osd":0,"up_from":91,"seq":390842024341,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1436376,"kb_used_data":220368,"kb_used_omap":5,"kb_used_meta":1215994,"kb_avail":92308776,"statfs":{"total":95995035648,"available":94524186624,"internally_reserved":0,"allocated":225656832,"data_stored":222794917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1245178490},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:37:39 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":4.0620000000000003,"15min":6.4610000000000003},"min":{"1min":0.23200000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":8.4350000000000005,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.39600000000000002,"5min":3.718,"15min":5.6509999999999998},"min":{"1min":0.219,"5min":0.20200000000000001,"15min":0.17599999999999999},"max":{"1min":1.1890000000000001,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.34399999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":6.9610000000000003,"15min":7.3559999999999999},"min":{"1min":0.26200000000000001,"5min":0.214,"15min":0.17199999999999999},"max":{"1min":4.0899999999999999,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.84599999999999997,"5min":6.6799999999999997,"15min":7.3869999999999996},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.17599999999999999},"max":{"1min":3.468,"5min":283.161,"15min":283.161},"last":1.7689999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:37:39 2023","interfaces":[{"interface":"back","average":{"1min":16.741,"5min":6.8769999999999998,"15min":6.4880000000000004},"min":{"1min":0.27300000000000002,"5min":0.219,"15min":0.217},"max":{"1min":194.26300000000001,"5min":194.26300000000001,"15min":220.732},"last":0.76700000000000002},{"interface":"front","average":{"1min":16.422000000000001,"5min":6.75,"15min":6.5830000000000002},"min":{"1min":0.30199999999999999,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":194.56299999999999,"5min":194.56299999999999,"15min":218.78200000000001},"last":1.734}]},{"osd":4,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.79900000000000004,"15min":2.3479999999999999},"min":{"1min":0.34399999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.60999999999999999,"5min":9.3140000000000001,"15min":138.81299999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.66800000000000004,"5min":1.02,"15min":3.0270000000000001},"min":{"1min":0.36799999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.661,"5min":9.3670000000000009,"15min":171.251},"last":0.48799999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.1850000000000001,"15min":2.7290000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":1.288,"5min":15.076000000000001,"15min":171.30699999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.097,"15min":1.9930000000000001},"min":{"1min":0.33600000000000002,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":4.1710000000000003,"5min":6.5279999999999996,"15min":139.40799999999999},"last":0.45800000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":1.143,"15min":3.0760000000000001},"min":{"1min":0.374,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":12.654999999999999,"5min":12.654999999999999,"15min":171.33600000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.155,"15min":3.2749999999999999},"min":{"1min":0.41199999999999998,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":4.6440000000000001,"5min":9.157,"15min":184.13499999999999},"last":0.56599999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":1.089,"15min":2.569},"min":{"1min":0.35099999999999998,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":2.6749999999999998,"5min":8.9510000000000005,"15min":138.77699999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.3260000000000001,"15min":2.508},"min":{"1min":0.373,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":4.2229999999999999,"5min":9.8870000000000005,"15min":139.37899999999999},"last":0.503}]}]},{"osd":2,"up_from":199,"seq":854698492162,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2389656,"kb_used_data":1515408,"kb_used_omap":7,"kb_used_meta":874232,"kb_avail":91355496,"statfs":{"total":95995035648,"available":93548027904,"internally_reserved":0,"allocated":1551777792,"data_stored":1548865925,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":895213649},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.7789999999999999,"15min":3.617},"min":{"1min":0.20200000000000001,"5min":0.186,"15min":0.16700000000000001},"max":{"1min":6.3940000000000001,"5min":61.950000000000003,"15min":119.277},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":1.8740000000000001,"15min":3.46},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.18099999999999999},"max":{"1min":5.0339999999999998,"5min":61.984999999999999,"15min":119.56100000000001},"last":0.25800000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":4.4960000000000004,"15min":4.5449999999999999},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":4.7060000000000004,"5min":106.765,"15min":157.98699999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.73899999999999999,"5min":4.1210000000000004,"15min":4.4950000000000001},"min":{"1min":0.23400000000000001,"5min":0.222,"15min":0.222},"max":{"1min":6.4340000000000002,"5min":106.732,"15min":158.03800000000001},"last":0.34499999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":5.3179999999999996,"15min":4.6900000000000004},"min":{"1min":0.26000000000000001,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":1.075,"5min":106.70399999999999,"15min":119.366},"last":0.245},{"interface":"front","average":{"1min":0.52100000000000002,"5min":5.0359999999999996,"15min":4.1440000000000001},"min":{"1min":0.246,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.6120000000000001,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.46200000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":1.756,"15min":2.1909999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.25},"max":{"1min":0.65600000000000003,"5min":62.081000000000003,"15min":119.88200000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.9710000000000001,"15min":1.829},"min":{"1min":0.42299999999999999,"5min":0.35599999999999998,"15min":0.28499999999999998},"max":{"1min":1.121,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.45100000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":2.8340000000000001,"15min":2.5840000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":2.6549999999999998,"5min":91.207999999999998,"15min":119.919},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.69399999999999995,"5min":2.919,"15min":2.5800000000000001},"min":{"1min":0.371,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.4289999999999998,"5min":91.150999999999996,"15min":119.306},"last":0.46999999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":3.254,"15min":2.9169999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":1.04,"5min":107.018,"15min":119.943},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.746,"5min":2.1779999999999999,"15min":2.1160000000000001},"min":{"1min":0.45100000000000001,"5min":0.371,"15min":0.34999999999999998},"max":{"1min":3.4950000000000001,"5min":62.493000000000002,"15min":62.493000000000002},"last":0.49199999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.917,"15min":1.883},"min":{"1min":0.41699999999999998,"5min":0.374,"15min":0.34000000000000002},"max":{"1min":1.831,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.66800000000000004,"5min":3.1800000000000002,"15min":2.3220000000000001},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.36199999999999999},"max":{"1min":1.792,"5min":106.809,"15min":106.809},"last":2.3809999999999998}]}]},{"osd":3,"up_from":148,"seq":635655160131,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1751308,"kb_used_data":817668,"kb_used_omap":2,"kb_used_meta":933629,"kb_avail":91993844,"statfs":{"total":95995035648,"available":94201696256,"internally_reserved":0,"allocated":837292032,"data_stored":834445426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":956036149},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.1359999999999999,"15min":2.6019999999999999},"min":{"1min":0.184,"5min":0.184,"15min":0.17699999999999999},"max":{"1min":6.7699999999999996,"5min":17.213000000000001,"15min":103.866},"last":0.25},{"interface":"front","average":{"1min":0.72899999999999998,"5min":1.4139999999999999,"15min":2.855},"min":{"1min":0.218,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":3.4980000000000002,"5min":22.754000000000001,"15min":101.884},"last":0.65300000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":2.5960000000000001,"15min":2.903},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":1.649,"5min":56.119,"15min":85.242999999999995},"last":1.224},{"interface":"front","average":{"1min":0.42299999999999999,"5min":2.036,"15min":2.7469999999999999},"min":{"1min":0.254,"5min":0.224,"15min":0.20899999999999999},"max":{"1min":1.6990000000000001,"5min":53.962000000000003,"15min":85.302000000000007},"last":0.311}]},{"osd":2,"last update":"Sat Sep 23 23:36:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":2.2989999999999999,"15min":4.5999999999999996},"min":{"1min":0.25800000000000001,"5min":0.25,"15min":0.19800000000000001},"max":{"1min":14.965,"5min":56.226999999999997,"15min":101.279},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.2210000000000001,"5min":2.165,"15min":4.4370000000000003},"min":{"1min":0.315,"5min":0.29299999999999998,"15min":0.214},"max":{"1min":15.058999999999999,"5min":54.280999999999999,"15min":101.294},"last":0.33900000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":0.68999999999999995,"5min":11.209,"15min":11.209},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.44500000000000001,"5min":1.3720000000000001,"15min":1.6719999999999999},"min":{"1min":0.33000000000000002,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":0.67900000000000005,"5min":11.494999999999999,"15min":24.782},"last":0.57199999999999995}]},{"osd":5,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":6.3570000000000002,"15min":3.3220000000000001},"min":{"1min":0.33000000000000002,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":0.622,"5min":214.43700000000001,"15min":214.43700000000001},"last":0.627},{"interface":"front","average":{"1min":0.61099999999999999,"5min":6.3970000000000002,"15min":3.3969999999999998},"min":{"1min":0.36499999999999999,"5min":0.34699999999999998,"15min":0.30599999999999999},"max":{"1min":1.716,"5min":214.46600000000001,"15min":214.46600000000001},"last":0.52500000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":2.218,"15min":2.1739999999999999},"min":{"1min":0.35399999999999998,"5min":0.318,"15min":0.318},"max":{"1min":2.008,"5min":58.932000000000002,"15min":58.932000000000002},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.629,"5min":2.492,"15min":2.2210000000000001},"min":{"1min":0.36699999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.7450000000000001,"5min":59.186,"15min":59.186},"last":0.55100000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:36:56 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":4.5469999999999997,"15min":3.1829999999999998},"min":{"1min":0.42199999999999999,"5min":0.38700000000000001,"15min":0.32600000000000001},"max":{"1min":0.66700000000000004,"5min":192.69,"15min":192.69},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.54300000000000004,"5min":4.6280000000000001,"15min":3.649},"min":{"1min":0.40100000000000002,"5min":0.39200000000000002,"15min":0.36699999999999999},"max":{"1min":1.2769999999999999,"5min":192.65600000000001,"15min":192.65600000000001},"last":0.81299999999999994}]}]},{"osd":4,"up_from":0,"seq":128849019450,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757813,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2121728,"kb_used_data":1375544,"kb_used_omap":0,"kb_used_meta":746176,"kb_avail":91623424,"statfs":{"total":95995035648,"available":93822386176,"internally_reserved":0,"allocated":1408557056,"data_stored":1406887975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":764084224},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.032,"15min":2.597},"min":{"1min":0.25,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":2.9279999999999999,"5min":21.109000000000002,"15min":113.869},"last":0.245},{"interface":"front","average":{"1min":1.123,"5min":1.407,"15min":3.0659999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":11.154999999999999,"5min":20.876999999999999,"15min":138.55199999999999},"last":1.335}]},{"osd":1,"last update":"Sat Sep 23 23:37:22 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.1459999999999999,"15min":2.1629999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.1360000000000001,"5min":29.007999999999999,"15min":69.816999999999993},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.212,"15min":2.3929999999999998},"min":{"1min":0.30499999999999999,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":1.452,"5min":29.053999999999998,"15min":70.268000000000001},"last":3.0649999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:36:48 2023","interfaces":[{"interface":"back","average":{"1min":5.194,"5min":6.984,"15min":4.8159999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.26900000000000002},"max":{"1min":60.441000000000003,"5min":208.738,"15min":208.738},"last":0.56799999999999995},{"interface":"front","average":{"1min":5.2839999999999998,"5min":5.7190000000000003,"15min":4.3259999999999996},"min":{"1min":0.29999999999999999,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":60.520000000000003,"5min":208.631,"15min":208.631},"last":2.0979999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:36:48 2023","interfaces":[{"interface":"back","average":{"1min":4.1749999999999998,"5min":4.6360000000000001,"15min":3.8100000000000001},"min":{"1min":0.34999999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":37.878,"5min":122.071,"15min":122.071},"last":0.33400000000000002},{"interface":"front","average":{"1min":4.2460000000000004,"5min":4.9829999999999997,"15min":3.984},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.318},"max":{"1min":37.853000000000002,"5min":122.09699999999999,"15min":122.09699999999999},"last":1.3029999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.88200000000000001,"15min":0.877},"min":{"1min":0.316,"5min":0.315,"15min":0.251},"max":{"1min":2.2650000000000001,"5min":9.2690000000000001,"15min":19.962},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.85499999999999998,"5min":1.016,"15min":0.84099999999999997},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.22900000000000001},"max":{"1min":3.1840000000000002,"5min":10.273999999999999,"15min":20.829999999999998},"last":0.47199999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.3340000000000001,"15min":1.5249999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.3100000000000001,"5min":148.36600000000001,"15min":148.36600000000001},"last":0.505},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.7389999999999999,"15min":1.71},"min":{"1min":0.32800000000000001,"5min":0.30399999999999999,"15min":0.26000000000000001},"max":{"1min":5.7240000000000002,"5min":148.42500000000001,"15min":148.42500000000001},"last":1.4790000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:37:48 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.306,"15min":2.177},"min":{"1min":0.34699999999999998,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":4.3380000000000001,"5min":15.497,"15min":179.28800000000001},"last":1.5129999999999999},{"interface":"front","average":{"1min":1.157,"5min":1.339,"15min":2.2480000000000002},"min":{"1min":0.40600000000000003,"5min":0.35199999999999998,"15min":0.34999999999999998},"max":{"1min":6.1440000000000001,"5min":15.653,"15min":179.428},"last":1.536}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":216924160,"data_stored":216924160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":747376640,"data_stored":747372568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1536688128,"data_stored":1536688128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":822087680,"data_stored":822083608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2353397760,"data_stored":2353397760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1396441088,"data_stored":1396441088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":874905600,"data_stored":874905600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":951717888,"data_stored":951713816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:37:54.098 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:54.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:54.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:54.130 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:54.206 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:54.241 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:54.303 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:54.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:54.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:54.411 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:54.437 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:54 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 22216/108597 objects degraded (20.457%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:54.439 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:54.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:54 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 22216/108597 objects degraded (20.457%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:54.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:54 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 22216/108597 objects degraded (20.457%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:54.492 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:54.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:54.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:54.519 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:37:54.548 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:37:54.577 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:37:54.600 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:54.605 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:37:54.631 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:54.643 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:37:54.677 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:37:54.690 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:54.695 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:37:54.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:54.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:54.726 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:37:54.728 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:37:54.728 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:37:54.728 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:37:54.728 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:37:54.729 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:37:54.729 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:37:54.729 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:37:54.729 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:37:54.797 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:54.863 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:55.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:55 smithi138 ceph-mon[193606]: pgmap v1705: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 376 op/s; 22097/107124 objects degraded (20.627%); 2032/107124 objects misplaced (1.897%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:37:55.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:55 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3594758950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:55.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:55 smithi110 ceph-mon[221102]: pgmap v1705: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 376 op/s; 22097/107124 objects degraded (20.627%); 2032/107124 objects misplaced (1.897%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:37:55.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:55 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3594758950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:55.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:55 smithi110 ceph-mon[212939]: pgmap v1705: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 376 op/s; 22097/107124 objects degraded (20.627%); 2032/107124 objects misplaced (1.897%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:37:55.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:55 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3594758950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:37:56.040 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:56.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:56.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:56.105 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:37:56.147 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:37:56.206 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:56.317 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:37:56.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:56.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:37:56.429 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:56.465 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:56.532 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:56.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:56.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:56.642 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:56.683 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:56.768 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:56.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:56.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:56.818 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:37:56.858 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:56.876 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:37:56.890 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:37:56.890 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:37:56.890 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:37:56.890 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:37:56.890 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:37:56.905 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:56.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:37:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:37:56] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2023-09-23T23:37:56.969 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:37:56.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:56.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:37:57.075 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:57.107 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:57.156 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:57.162 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:57 smithi138 ceph-mon[193606]: pgmap v1706: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 21985/102588 objects degraded (21.430%); 2032/102588 objects misplaced (1.981%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:37:57.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:57.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:57.264 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:57.292 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:57.333 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:57.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:57.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:57.438 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:57.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:57 smithi110 ceph-mon[212939]: pgmap v1706: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 21985/102588 objects degraded (21.430%); 2032/102588 objects misplaced (1.981%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:37:57.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:57 smithi110 ceph-mon[221102]: pgmap v1706: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 21985/102588 objects degraded (21.430%); 2032/102588 objects misplaced (1.981%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:37:57.464 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:57.518 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:57.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:57.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:57.624 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:37:57.652 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:57.966 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:37:57.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:57.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:37:58.045 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:37:58.070 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:58.117 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:58.182 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:37:58.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:37:58.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:37:58.298 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:58.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:58.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:58.404 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:37:58.429 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:37:58.431 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:58.473 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:37:58.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:58.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:37:58.584 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:58.611 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:58.672 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:58.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:58.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:58.780 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:37:58.834 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:58.859 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:58.955 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:37:58.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:58.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:37:59.066 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:37:59.085 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:59.125 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:59 smithi110 ceph-mon[212939]: pgmap v1707: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 21985/102588 objects degraded (21.430%); 2032/102588 objects misplaced (1.981%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:37:59.126 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:37:59 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 21985/102588 objects degraded (21.430%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:59.153 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:59.392 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:59 smithi110 ceph-mon[221102]: pgmap v1707: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 21985/102588 objects degraded (21.430%); 2032/102588 objects misplaced (1.981%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:37:59.392 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:37:59 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 21985/102588 objects degraded (21.430%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:59.435 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:37:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:59 smithi138 ceph-mon[193606]: pgmap v1707: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 21985/102588 objects degraded (21.430%); 2032/102588 objects misplaced (1.981%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:37:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:37:59 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 21985/102588 objects degraded (21.430%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:37:59.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:59.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:37:59.550 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:37:59.596 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:59.662 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:37:59.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:59.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:37:59.768 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:37:59.804 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:37:59.812 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:37:59.875 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:37:59.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:37:59.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:37:59.981 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:38:00.066 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:00.134 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:00 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/517938382' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:00.135 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:00 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:00.155 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:38:00.318 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:00 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/517938382' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:00.318 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:00 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:00.318 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:00 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/517938382' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:00.318 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:00 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:01 smithi138 ceph-mon[193606]: pgmap v1708: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 419 op/s; 21709/97017 objects degraded (22.376%); 1889/97017 objects misplaced (1.947%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T23:38:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:01 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:38:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:01 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:38:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:01 smithi138 ceph-mon[193606]: osdmap e605: 8 total, 8 up, 7 in 2023-09-23T23:38:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:01 smithi138 ceph-mon[193606]: failed to encode map e605 with expected crc 2023-09-23T23:38:01.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:01 smithi138 ceph-mon[193606]: failed to encode map e605 with expected crc 2023-09-23T23:38:01.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:01 smithi138 ceph-mon[193606]: failed to encode map e605 with expected crc 2023-09-23T23:38:01.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:01 smithi138 ceph-mon[193606]: failed to encode map e605 with expected crc 2023-09-23T23:38:01.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:01 smithi138 ceph-mon[193606]: failed to encode map e605 with expected crc 2023-09-23T23:38:01.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[212939]: pgmap v1708: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 419 op/s; 21709/97017 objects degraded (22.376%); 1889/97017 objects misplaced (1.947%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T23:38:01.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:38:01.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:38:01.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[212939]: osdmap e605: 8 total, 8 up, 7 in 2023-09-23T23:38:01.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[212939]: failed to encode map e605 with expected crc 2023-09-23T23:38:01.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[212939]: failed to encode map e605 with expected crc 2023-09-23T23:38:01.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[212939]: failed to encode map e605 with expected crc 2023-09-23T23:38:01.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[212939]: failed to encode map e605 with expected crc 2023-09-23T23:38:01.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[212939]: failed to encode map e605 with expected crc 2023-09-23T23:38:01.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[221102]: pgmap v1708: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 419 op/s; 21709/97017 objects degraded (22.376%); 1889/97017 objects misplaced (1.947%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T23:38:01.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:38:01.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:38:01.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[221102]: osdmap e605: 8 total, 8 up, 7 in 2023-09-23T23:38:01.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[221102]: failed to encode map e605 with expected crc 2023-09-23T23:38:01.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[221102]: failed to encode map e605 with expected crc 2023-09-23T23:38:01.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[221102]: failed to encode map e605 with expected crc 2023-09-23T23:38:01.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[221102]: failed to encode map e605 with expected crc 2023-09-23T23:38:01.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:01 smithi110 ceph-mon[221102]: failed to encode map e605 with expected crc 2023-09-23T23:38:01.929 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:38:01.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:01.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:38:02.032 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:38:02.104 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:02.421 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:38:02.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:02.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:38:02.528 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:02.615 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:02.830 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:02.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:02.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:02.933 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:02.943 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:38:02.945 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:38:02.961 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:03.012 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:03.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:03.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:03.117 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:38:03.148 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:03.213 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:38:03.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:03.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:38:03.319 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:03.368 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:03 smithi110 ceph-mon[212939]: pgmap v1710: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 21709/95724 objects degraded (22.679%); 1889/95724 objects misplaced (1.973%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T23:38:03.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:03 smithi110 ceph-mon[221102]: pgmap v1710: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 21709/95724 objects degraded (22.679%); 1889/95724 objects misplaced (1.973%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T23:38:03.372 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:03 smithi138 ceph-mon[193606]: pgmap v1710: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 21709/95724 objects degraded (22.679%); 1889/95724 objects misplaced (1.973%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T23:38:04.000 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:04.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:04.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:04.106 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:38:04.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:04 smithi110 ceph-mon[221102]: from='client.47039 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:38:04.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:04 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/512966324' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:04.161 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:04 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 21709/95724 objects degraded (22.679%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:04.161 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:04 smithi110 ceph-mon[212939]: from='client.47039 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:38:04.161 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:04 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/512966324' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:04.162 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:04 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 21709/95724 objects degraded (22.679%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:04.166 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:04.306 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:38:04.306 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:38:04.306 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:38:04.306 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:38:04.306 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:38:04.306 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:38:04.306 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:38:04.307 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:38:04.307 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:38:04.307 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:38:04.307 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:38:04.307 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:38:04.307 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:38:04.307 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:38:04.307 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:38:04.307 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:38:04.358 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:38:04.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:04.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:38:04.397 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:38:04.402 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1710,"stamp":"2023-09-23T23:38:01.955024+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2090120437,"num_objects":31908,"num_object_clones":0,"num_object_copies":95724,"num_objects_missing_on_primary":994,"num_objects_missing":994,"num_objects_degraded":21709,"num_objects_misplaced":1889,"num_objects_unfound":0,"num_objects_dirty":31908,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":143749,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19999,"num_bytes_recovered":1307963816,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95196,"ondisk_log_size":95196,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11482572,"kb_used_data":5720084,"kb_used_omap":29,"kb_used_meta":5762402,"kb_avail":644733492,"statfs":{"total":671965249536,"available":660207095808,"internally_reserved":0,"allocated":5857366016,"data_stored":5840835492,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5900700371},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,0,0,0,2,0,9,7,4],"upper_bound":2048},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-281214976,"num_objects":-4291,"num_object_clones":0,"num_object_copies":-12873,"num_objects_missing_on_primary":-271,"num_objects_missing":-271,"num_objects_degraded":-507,"num_objects_misplaced":-271,"num_objects_unfound":0,"num_objects_dirty":-4291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4291,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":779,"num_bytes_recovered":51052544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.844400"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":800,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.238108+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:37:42.238108+0000","last_peered":"2023-09-23T23:37:42.238108+0000","last_clean":"2023-09-23T23:37:42.238108+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:37:42.238108+0000","last_undegraded":"2023-09-23T23:37:42.238108+0000","last_fullsized":"2023-09-23T23:37:42.238108+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":359,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.074888+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:37:42.074888+0000","last_peered":"2023-09-23T23:37:42.074888+0000","last_clean":"2023-09-23T23:37:42.074888+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:37:42.074888+0000","last_undegraded":"2023-09-23T23:37:42.074888+0000","last_fullsized":"2023-09-23T23:37:42.074888+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":586,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.099376+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:37:42.099376+0000","last_peered":"2023-09-23T23:37:42.099376+0000","last_clean":"2023-09-23T23:37:42.099376+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:37:42.099376+0000","last_undegraded":"2023-09-23T23:37:42.099376+0000","last_fullsized":"2023-09-23T23:37:42.099376+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":753,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.238793+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:37:42.238793+0000","last_peered":"2023-09-23T23:37:42.238793+0000","last_clean":"2023-09-23T23:37:42.238793+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:37:42.238793+0000","last_undegraded":"2023-09-23T23:37:42.238793+0000","last_fullsized":"2023-09-23T23:37:42.238793+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":323,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.085388+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:37:42.085388+0000","last_peered":"2023-09-23T23:37:42.085388+0000","last_clean":"2023-09-23T23:37:42.085388+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:37:42.085388+0000","last_undegraded":"2023-09-23T23:37:42.085388+0000","last_fullsized":"2023-09-23T23:37:42.085388+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":565,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.086231+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:37:42.086231+0000","last_peered":"2023-09-23T23:37:42.086231+0000","last_clean":"2023-09-23T23:37:42.086231+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:37:42.086231+0000","last_undegraded":"2023-09-23T23:37:42.086231+0000","last_fullsized":"2023-09-23T23:37:42.086231+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":765,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.234759+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:37:42.234759+0000","last_peered":"2023-09-23T23:37:42.234759+0000","last_clean":"2023-09-23T23:37:42.234759+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:37:42.234759+0000","last_undegraded":"2023-09-23T23:37:42.234759+0000","last_fullsized":"2023-09-23T23:37:42.234759+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":623,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.085172+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:37:42.085172+0000","last_peered":"2023-09-23T23:37:42.085172+0000","last_clean":"2023-09-23T23:37:42.085172+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:37:42.085172+0000","last_undegraded":"2023-09-23T23:37:42.085172+0000","last_fullsized":"2023-09-23T23:37:42.085172+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":118,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.246809+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:37:42.246809+0000","last_peered":"2023-09-23T23:37:42.246809+0000","last_clean":"2023-09-23T23:37:42.246809+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:37:42.246809+0000","last_undegraded":"2023-09-23T23:37:42.246809+0000","last_fullsized":"2023-09-23T23:37:42.246809+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":924,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.234254+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:37:42.234254+0000","last_peered":"2023-09-23T23:37:42.234254+0000","last_clean":"2023-09-23T23:37:42.234254+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:37:42.234254+0000","last_undegraded":"2023-09-23T23:37:42.234254+0000","last_fullsized":"2023-09-23T23:37:42.234254+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":256,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.239441+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:37:42.239441+0000","last_peered":"2023-09-23T23:37:42.239441+0000","last_clean":"2023-09-23T23:37:42.239441+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:37:42.239441+0000","last_undegraded":"2023-09-23T23:37:42.239441+0000","last_fullsized":"2023-09-23T23:37:42.239441+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":605,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.085153+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:37:42.085153+0000","last_peered":"2023-09-23T23:37:42.085153+0000","last_clean":"2023-09-23T23:37:42.085153+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:37:42.085153+0000","last_undegraded":"2023-09-23T23:37:42.085153+0000","last_fullsized":"2023-09-23T23:37:42.085153+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":898,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:50.079677+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:37:50.079677+0000","last_peered":"2023-09-23T23:37:50.079677+0000","last_clean":"2023-09-23T23:37:50.079677+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:37:50.079677+0000","last_undegraded":"2023-09-23T23:37:50.079677+0000","last_fullsized":"2023-09-23T23:37:50.079677+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":235,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.237810+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:37:42.237810+0000","last_peered":"2023-09-23T23:37:42.237810+0000","last_clean":"2023-09-23T23:37:42.237810+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:37:42.237810+0000","last_undegraded":"2023-09-23T23:37:42.237810+0000","last_fullsized":"2023-09-23T23:37:42.237810+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":566,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.085291+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:37:42.085291+0000","last_peered":"2023-09-23T23:37:42.085291+0000","last_clean":"2023-09-23T23:37:42.085291+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:37:42.085291+0000","last_undegraded":"2023-09-23T23:37:42.085291+0000","last_fullsized":"2023-09-23T23:37:42.085291+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1058,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.239180+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:37:42.239180+0000","last_peered":"2023-09-23T23:37:42.239180+0000","last_clean":"2023-09-23T23:37:42.239180+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:37:42.239180+0000","last_undegraded":"2023-09-23T23:37:42.239180+0000","last_fullsized":"2023-09-23T23:37:42.239180+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":277,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.089479+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:37:42.089479+0000","last_peered":"2023-09-23T23:37:42.089479+0000","last_clean":"2023-09-23T23:37:42.089479+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:37:42.089479+0000","last_undegraded":"2023-09-23T23:37:42.089479+0000","last_fullsized":"2023-09-23T23:37:42.089479+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":137,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.242814+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:37:42.242814+0000","last_peered":"2023-09-23T23:37:42.242814+0000","last_clean":"2023-09-23T23:37:42.242814+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:37:42.242814+0000","last_undegraded":"2023-09-23T23:37:42.242814+0000","last_fullsized":"2023-09-23T23:37:42.242814+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":455,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.086305+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:37:42.086305+0000","last_peered":"2023-09-23T23:37:42.086305+0000","last_clean":"2023-09-23T23:37:42.086305+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:37:42.086305+0000","last_undegraded":"2023-09-23T23:37:42.086305+0000","last_fullsized":"2023-09-23T23:37:42.086305+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":178,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.243368+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:37:42.243368+0000","last_peered":"2023-09-23T23:37:42.243368+0000","last_clean":"2023-09-23T23:37:42.243368+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:37:42.243368+0000","last_undegraded":"2023-09-23T23:37:42.243368+0000","last_fullsized":"2023-09-23T23:37:42.243368+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":545,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.243177+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:37:42.243177+0000","last_peered":"2023-09-23T23:37:42.243177+0000","last_clean":"2023-09-23T23:37:42.243177+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:37:42.243177+0000","last_undegraded":"2023-09-23T23:37:42.243177+0000","last_fullsized":"2023-09-23T23:37:42.243177+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":759,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.237044+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:37:42.237044+0000","last_peered":"2023-09-23T23:37:42.237044+0000","last_clean":"2023-09-23T23:37:42.237044+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:37:42.237044+0000","last_undegraded":"2023-09-23T23:37:42.237044+0000","last_fullsized":"2023-09-23T23:37:42.237044+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":151,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.072673+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:37:42.072673+0000","last_peered":"2023-09-23T23:37:42.072673+0000","last_clean":"2023-09-23T23:37:42.072673+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:37:42.072673+0000","last_undegraded":"2023-09-23T23:37:42.072673+0000","last_fullsized":"2023-09-23T23:37:42.072673+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":604,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.085696+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:37:42.085696+0000","last_peered":"2023-09-23T23:37:42.085696+0000","last_clean":"2023-09-23T23:37:42.085696+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:37:42.085696+0000","last_undegraded":"2023-09-23T23:37:42.085696+0000","last_fullsized":"2023-09-23T23:37:42.085696+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":889,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.242379+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:37:42.242379+0000","last_peered":"2023-09-23T23:37:42.242379+0000","last_clean":"2023-09-23T23:37:42.242379+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:37:42.242379+0000","last_undegraded":"2023-09-23T23:37:42.242379+0000","last_fullsized":"2023-09-23T23:37:42.242379+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":199,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.088384+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:37:42.088384+0000","last_peered":"2023-09-23T23:37:42.088384+0000","last_clean":"2023-09-23T23:37:42.088384+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:37:42.088384+0000","last_undegraded":"2023-09-23T23:37:42.088384+0000","last_fullsized":"2023-09-23T23:37:42.088384+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":326,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.099282+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:37:42.099282+0000","last_peered":"2023-09-23T23:37:42.099282+0000","last_clean":"2023-09-23T23:37:42.099282+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:37:42.099282+0000","last_undegraded":"2023-09-23T23:37:42.099282+0000","last_fullsized":"2023-09-23T23:37:42.099282+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":567,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.087437+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:37:42.087437+0000","last_peered":"2023-09-23T23:37:42.087437+0000","last_clean":"2023-09-23T23:37:42.087437+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:37:42.087437+0000","last_undegraded":"2023-09-23T23:37:42.087437+0000","last_fullsized":"2023-09-23T23:37:42.087437+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":472,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.242506+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:37:42.242506+0000","last_peered":"2023-09-23T23:37:42.242506+0000","last_clean":"2023-09-23T23:37:42.242506+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:37:42.242506+0000","last_undegraded":"2023-09-23T23:37:42.242506+0000","last_fullsized":"2023-09-23T23:37:42.242506+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":560,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.086387+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:37:42.086387+0000","last_peered":"2023-09-23T23:37:42.086387+0000","last_clean":"2023-09-23T23:37:42.086387+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:37:42.086387+0000","last_undegraded":"2023-09-23T23:37:42.086387+0000","last_fullsized":"2023-09-23T23:37:42.086387+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":967,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.237929+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:37:42.237929+0000","last_peered":"2023-09-23T23:37:42.237929+0000","last_clean":"2023-09-23T23:37:42.237929+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:37:42.237929+0000","last_undegraded":"2023-09-23T23:37:42.237929+0000","last_fullsized":"2023-09-23T23:37:42.237929+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":168,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.242629+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:37:42.242629+0000","last_peered":"2023-09-23T23:37:42.242629+0000","last_clean":"2023-09-23T23:37:42.242629+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:37:42.242629+0000","last_undegraded":"2023-09-23T23:37:42.242629+0000","last_fullsized":"2023-09-23T23:37:42.242629+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":245,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.085443+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:37:42.085443+0000","last_peered":"2023-09-23T23:37:42.085443+0000","last_clean":"2023-09-23T23:37:42.085443+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:37:42.085443+0000","last_undegraded":"2023-09-23T23:37:42.085443+0000","last_fullsized":"2023-09-23T23:37:42.085443+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"604'8852","reported_seq":8939,"reported_epoch":604,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:59.200963+0000","last_change":"2023-09-23T23:33:52.884769+0000","last_active":"2023-09-23T23:37:59.200963+0000","last_peered":"2023-09-23T23:37:59.200963+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:37:59.200963+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"552'3400","ondisk_log_start":"552'3400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5452,"log_dups_size":0,"ondisk_log_size":5452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130809856,"num_objects":1996,"num_object_clones":0,"num_object_copies":5988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8852,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":970},{"shards":"2,6","objects":-2402}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":938,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.071771+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:37:42.071771+0000","last_peered":"2023-09-23T23:37:42.071771+0000","last_clean":"2023-09-23T23:37:42.071771+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:37:42.071771+0000","last_undegraded":"2023-09-23T23:37:42.071771+0000","last_fullsized":"2023-09-23T23:37:42.071771+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":245,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.235147+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:37:42.235147+0000","last_peered":"2023-09-23T23:37:42.235147+0000","last_clean":"2023-09-23T23:37:42.235147+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:37:42.235147+0000","last_undegraded":"2023-09-23T23:37:42.235147+0000","last_fullsized":"2023-09-23T23:37:42.235147+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":235,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.238040+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:37:42.238040+0000","last_peered":"2023-09-23T23:37:42.238040+0000","last_clean":"2023-09-23T23:37:42.238040+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:37:42.238040+0000","last_undegraded":"2023-09-23T23:37:42.238040+0000","last_fullsized":"2023-09-23T23:37:42.238040+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"604'9059","reported_seq":9146,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.082197+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:38:00.082197+0000","last_peered":"2023-09-23T23:38:00.082197+0000","last_clean":"2023-09-23T23:38:00.082197+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:38:00.082197+0000","last_undegraded":"2023-09-23T23:38:00.082197+0000","last_fullsized":"2023-09-23T23:38:00.082197+0000","mapping_epoch":566,"log_start":"551'2900","ondisk_log_start":"551'2900","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6159,"log_dups_size":0,"ondisk_log_size":6159,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129302528,"num_objects":1973,"num_object_clones":0,"num_object_copies":5919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9059,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1016,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.085564+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:37:42.085564+0000","last_peered":"2023-09-23T23:37:42.085564+0000","last_clean":"2023-09-23T23:37:42.085564+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:37:42.085564+0000","last_undegraded":"2023-09-23T23:37:42.085564+0000","last_fullsized":"2023-09-23T23:37:42.085564+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":235,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.234510+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:37:42.234510+0000","last_peered":"2023-09-23T23:37:42.234510+0000","last_clean":"2023-09-23T23:37:42.234510+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:37:42.234510+0000","last_undegraded":"2023-09-23T23:37:42.234510+0000","last_fullsized":"2023-09-23T23:37:42.234510+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":455,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.085043+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:37:42.085043+0000","last_peered":"2023-09-23T23:37:42.085043+0000","last_clean":"2023-09-23T23:37:42.085043+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:37:42.085043+0000","last_undegraded":"2023-09-23T23:37:42.085043+0000","last_fullsized":"2023-09-23T23:37:42.085043+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"604'9034","reported_seq":17176,"reported_epoch":604,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:37:58.083124+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:37:58.083124+0000","last_peered":"2023-09-23T23:37:58.083124+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:37:58.083124+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:37:58.083124+0000","mapping_epoch":571,"log_start":"552'3700","ondisk_log_start":"552'3700","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5334,"log_dups_size":0,"ondisk_log_size":5334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131989504,"num_objects":2014,"num_object_clones":0,"num_object_copies":6042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1119,"num_objects_misplaced":895,"num_objects_unfound":0,"num_objects_dirty":2014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9034,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4012,"num_bytes_recovered":262930432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":997},{"shards":"3,5,7","objects":1017}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":914,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.246891+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:37:42.246891+0000","last_peered":"2023-09-23T23:37:42.246891+0000","last_clean":"2023-09-23T23:37:42.246891+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:37:42.246891+0000","last_undegraded":"2023-09-23T23:37:42.246891+0000","last_fullsized":"2023-09-23T23:37:42.246891+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":342,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.238789+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:37:42.238789+0000","last_peered":"2023-09-23T23:37:42.238789+0000","last_clean":"2023-09-23T23:37:42.238789+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:37:42.238789+0000","last_undegraded":"2023-09-23T23:37:42.238789+0000","last_fullsized":"2023-09-23T23:37:42.238789+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":118,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.243015+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:37:42.243015+0000","last_peered":"2023-09-23T23:37:42.243015+0000","last_clean":"2023-09-23T23:37:42.243015+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:37:42.243015+0000","last_undegraded":"2023-09-23T23:37:42.243015+0000","last_fullsized":"2023-09-23T23:37:42.243015+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"604'9024","reported_seq":11098,"reported_epoch":604,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:58.024783+0000","last_change":"2023-09-23T23:34:35.750428+0000","last_active":"2023-09-23T23:37:58.024783+0000","last_peered":"2023-09-23T23:37:58.024783+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:37:58.024783+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"552'3700","ondisk_log_start":"552'3700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5324,"log_dups_size":0,"ondisk_log_size":5324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135397376,"num_objects":2066,"num_object_clones":0,"num_object_copies":6198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9024,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":991,"num_bytes_recovered":64946176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":1056},{"shards":"3,5","objects":-1478}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":511,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.086348+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:37:42.086348+0000","last_peered":"2023-09-23T23:37:42.086348+0000","last_clean":"2023-09-23T23:37:42.086348+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:37:42.086348+0000","last_undegraded":"2023-09-23T23:37:42.086348+0000","last_fullsized":"2023-09-23T23:37:42.086348+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":445,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.085635+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:37:42.085635+0000","last_peered":"2023-09-23T23:37:42.085635+0000","last_clean":"2023-09-23T23:37:42.085635+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:37:42.085635+0000","last_undegraded":"2023-09-23T23:37:42.085635+0000","last_fullsized":"2023-09-23T23:37:42.085635+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":603,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:55.992857+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:37:55.992857+0000","last_peered":"2023-09-23T23:37:55.992857+0000","last_clean":"2023-09-23T23:37:55.992857+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:37:55.992857+0000","last_undegraded":"2023-09-23T23:37:55.992857+0000","last_fullsized":"2023-09-23T23:37:55.992857+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"604'9157","reported_seq":10491,"reported_epoch":604,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:59.467173+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:37:59.467173+0000","last_peered":"2023-09-23T23:37:59.467173+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:37:59.467173+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"552'3400","ondisk_log_start":"552'3400","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5757,"log_dups_size":0,"ondisk_log_size":5757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131399680,"num_objects":2005,"num_object_clones":0,"num_object_copies":6015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9157,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":959},{"shards":"3,5","objects":-3455},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":732,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.085196+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:37:42.085196+0000","last_peered":"2023-09-23T23:37:42.085196+0000","last_clean":"2023-09-23T23:37:42.085196+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:37:42.085196+0000","last_undegraded":"2023-09-23T23:37:42.085196+0000","last_fullsized":"2023-09-23T23:37:42.085196+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":615,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.140800+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:37:52.140800+0000","last_peered":"2023-09-23T23:37:52.140800+0000","last_clean":"2023-09-23T23:37:52.140800+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:37:52.140800+0000","last_undegraded":"2023-09-23T23:37:52.140800+0000","last_fullsized":"2023-09-23T23:37:52.140800+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":525,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.085602+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:37:42.085602+0000","last_peered":"2023-09-23T23:37:42.085602+0000","last_clean":"2023-09-23T23:37:42.085602+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:37:42.085602+0000","last_undegraded":"2023-09-23T23:37:42.085602+0000","last_fullsized":"2023-09-23T23:37:42.085602+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"604'9120","reported_seq":9198,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:59.225201+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:37:59.225201+0000","last_peered":"2023-09-23T23:37:59.225201+0000","last_clean":"2023-09-23T23:37:59.225201+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:37:59.225201+0000","last_undegraded":"2023-09-23T23:37:59.225201+0000","last_fullsized":"2023-09-23T23:37:59.225201+0000","mapping_epoch":534,"log_start":"552'3700","ondisk_log_start":"552'3700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5420,"log_dups_size":0,"ondisk_log_size":5420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128450560,"num_objects":1960,"num_object_clones":0,"num_object_copies":5880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1960,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9120,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":888,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.071615+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:37:42.071615+0000","last_peered":"2023-09-23T23:37:42.071615+0000","last_clean":"2023-09-23T23:37:42.071615+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:37:42.071615+0000","last_undegraded":"2023-09-23T23:37:42.071615+0000","last_fullsized":"2023-09-23T23:37:42.071615+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":128,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.242947+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:37:42.242947+0000","last_peered":"2023-09-23T23:37:42.242947+0000","last_clean":"2023-09-23T23:37:42.242947+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:37:42.242947+0000","last_undegraded":"2023-09-23T23:37:42.242947+0000","last_fullsized":"2023-09-23T23:37:42.242947+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":470,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.071938+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:37:42.071938+0000","last_peered":"2023-09-23T23:37:42.071938+0000","last_clean":"2023-09-23T23:37:42.071938+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:37:42.071938+0000","last_undegraded":"2023-09-23T23:37:42.071938+0000","last_fullsized":"2023-09-23T23:37:42.071938+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"604'9125","reported_seq":16369,"reported_epoch":604,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:37:59.136895+0000","last_change":"2023-09-23T23:34:35.785270+0000","last_active":"2023-09-23T23:37:59.136895+0000","last_peered":"2023-09-23T23:37:59.136895+0000","last_clean":"2023-09-23T23:33:50.504296+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:37:59.136895+0000","last_undegraded":"2023-09-23T23:34:35.780981+0000","last_fullsized":"2023-09-23T23:37:59.136895+0000","mapping_epoch":567,"log_start":"552'3700","ondisk_log_start":"552'3700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5425,"log_dups_size":0,"ondisk_log_size":5425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129695744,"num_objects":1979,"num_object_clones":0,"num_object_copies":5937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":974,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9125,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3573,"num_bytes_recovered":234160128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":974},{"shards":"5,6,7","objects":1005}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":612,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.238715+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:37:42.238715+0000","last_peered":"2023-09-23T23:37:42.238715+0000","last_clean":"2023-09-23T23:37:42.238715+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:37:42.238715+0000","last_undegraded":"2023-09-23T23:37:42.238715+0000","last_fullsized":"2023-09-23T23:37:42.238715+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":467,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.071618+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:37:42.071618+0000","last_peered":"2023-09-23T23:37:42.071618+0000","last_clean":"2023-09-23T23:37:42.071618+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:37:42.071618+0000","last_undegraded":"2023-09-23T23:37:42.071618+0000","last_fullsized":"2023-09-23T23:37:42.071618+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":555,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.243866+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:37:42.243866+0000","last_peered":"2023-09-23T23:37:42.243866+0000","last_clean":"2023-09-23T23:37:42.243866+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:37:42.243866+0000","last_undegraded":"2023-09-23T23:37:42.243866+0000","last_fullsized":"2023-09-23T23:37:42.243866+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"604'9034","reported_seq":18075,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:59.515494+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:37:59.515494+0000","last_peered":"2023-09-23T23:37:59.515494+0000","last_clean":"2023-09-23T23:37:59.515494+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:37:59.515494+0000","last_undegraded":"2023-09-23T23:37:59.515494+0000","last_fullsized":"2023-09-23T23:37:59.515494+0000","mapping_epoch":579,"log_start":"552'3300","ondisk_log_start":"552'3300","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5734,"log_dups_size":0,"ondisk_log_size":5734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129892352,"num_objects":1982,"num_object_clones":0,"num_object_copies":5946,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1982,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9034,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":946,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.074748+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:37:42.074748+0000","last_peered":"2023-09-23T23:37:42.074748+0000","last_clean":"2023-09-23T23:37:42.074748+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:37:42.074748+0000","last_undegraded":"2023-09-23T23:37:42.074748+0000","last_fullsized":"2023-09-23T23:37:42.074748+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":246,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.085084+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:37:42.085084+0000","last_peered":"2023-09-23T23:37:42.085084+0000","last_clean":"2023-09-23T23:37:42.085084+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:37:42.085084+0000","last_undegraded":"2023-09-23T23:37:42.085084+0000","last_fullsized":"2023-09-23T23:37:42.085084+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":534,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.088486+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:37:42.088486+0000","last_peered":"2023-09-23T23:37:42.088486+0000","last_clean":"2023-09-23T23:37:42.088486+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:37:42.088486+0000","last_undegraded":"2023-09-23T23:37:42.088486+0000","last_fullsized":"2023-09-23T23:37:42.088486+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"604'8696","reported_seq":8777,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:58.099320+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:37:58.099320+0000","last_peered":"2023-09-23T23:37:58.099320+0000","last_clean":"2023-09-23T23:37:58.099320+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:37:58.099320+0000","last_undegraded":"2023-09-23T23:37:58.099320+0000","last_fullsized":"2023-09-23T23:37:58.099320+0000","mapping_epoch":534,"log_start":"552'3400","ondisk_log_start":"552'3400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5296,"log_dups_size":0,"ondisk_log_size":5296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133169152,"num_objects":2032,"num_object_clones":0,"num_object_copies":6096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8696,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":925,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.244464+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:37:42.244464+0000","last_peered":"2023-09-23T23:37:42.244464+0000","last_clean":"2023-09-23T23:37:42.244464+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:37:42.244464+0000","last_undegraded":"2023-09-23T23:37:42.244464+0000","last_fullsized":"2023-09-23T23:37:42.244464+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":314,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.077596+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:37:42.077596+0000","last_peered":"2023-09-23T23:37:42.077596+0000","last_clean":"2023-09-23T23:37:42.077596+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:37:42.077596+0000","last_undegraded":"2023-09-23T23:37:42.077596+0000","last_fullsized":"2023-09-23T23:37:42.077596+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":619,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.238930+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:37:42.238930+0000","last_peered":"2023-09-23T23:37:42.238930+0000","last_clean":"2023-09-23T23:37:42.238930+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:37:42.238930+0000","last_undegraded":"2023-09-23T23:37:42.238930+0000","last_fullsized":"2023-09-23T23:37:42.238930+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":200,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.237719+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:37:42.237719+0000","last_peered":"2023-09-23T23:37:42.237719+0000","last_clean":"2023-09-23T23:37:42.237719+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:37:42.237719+0000","last_undegraded":"2023-09-23T23:37:42.237719+0000","last_fullsized":"2023-09-23T23:37:42.237719+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"604'9095","reported_seq":16227,"reported_epoch":604,"state":"active+recovering","last_fresh":"2023-09-23T23:37:58.259864+0000","last_change":"2023-09-23T23:34:35.815566+0000","last_active":"2023-09-23T23:37:58.259864+0000","last_peered":"2023-09-23T23:37:58.259864+0000","last_clean":"2023-09-23T23:33:50.519308+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:37:58.259864+0000","last_undegraded":"2023-09-23T23:37:58.259864+0000","last_fullsized":"2023-09-23T23:37:58.259864+0000","mapping_epoch":566,"log_start":"552'3822","ondisk_log_start":"552'3822","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5273,"log_dups_size":0,"ondisk_log_size":5273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135790616,"num_objects":2073,"num_object_clones":0,"num_object_copies":6219,"num_objects_missing_on_primary":994,"num_objects_missing":994,"num_objects_degraded":0,"num_objects_misplaced":994,"num_objects_unfound":0,"num_objects_dirty":2073,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9095,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3511,"num_bytes_recovered":230096896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,3,4,7","objects":10},{"shards":"1,3,7","objects":1069},{"shards":"1,4,7","objects":994}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":689,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.118552+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:37:52.118552+0000","last_peered":"2023-09-23T23:37:52.118552+0000","last_clean":"2023-09-23T23:37:52.118552+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:37:52.118552+0000","last_undegraded":"2023-09-23T23:37:52.118552+0000","last_fullsized":"2023-09-23T23:37:52.118552+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":252,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.242878+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:37:42.242878+0000","last_peered":"2023-09-23T23:37:42.242878+0000","last_clean":"2023-09-23T23:37:42.242878+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:37:42.242878+0000","last_undegraded":"2023-09-23T23:37:42.242878+0000","last_fullsized":"2023-09-23T23:37:42.242878+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":536,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.243221+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:37:42.243221+0000","last_peered":"2023-09-23T23:37:42.243221+0000","last_clean":"2023-09-23T23:37:42.243221+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:37:42.243221+0000","last_undegraded":"2023-09-23T23:37:42.243221+0000","last_fullsized":"2023-09-23T23:37:42.243221+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":237,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.072221+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:37:42.072221+0000","last_peered":"2023-09-23T23:37:42.072221+0000","last_clean":"2023-09-23T23:37:42.072221+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:37:42.072221+0000","last_undegraded":"2023-09-23T23:37:42.072221+0000","last_fullsized":"2023-09-23T23:37:42.072221+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"604'8817","reported_seq":10873,"reported_epoch":604,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:37:59.507027+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:37:59.507027+0000","last_peered":"2023-09-23T23:37:59.507027+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:37:59.507027+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"551'3100","ondisk_log_start":"551'3100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5717,"log_dups_size":0,"ondisk_log_size":5717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128122880,"num_objects":1955,"num_object_clones":0,"num_object_copies":5865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8817,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":944},{"shards":"5,7","objects":-1437}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":906,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.074673+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:37:42.074673+0000","last_peered":"2023-09-23T23:37:42.074673+0000","last_clean":"2023-09-23T23:37:42.074673+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:37:42.074673+0000","last_undegraded":"2023-09-23T23:37:42.074673+0000","last_fullsized":"2023-09-23T23:37:42.074673+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":343,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.071521+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:37:42.071521+0000","last_peered":"2023-09-23T23:37:42.071521+0000","last_clean":"2023-09-23T23:37:42.071521+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:37:42.071521+0000","last_undegraded":"2023-09-23T23:37:42.071521+0000","last_fullsized":"2023-09-23T23:37:42.071521+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":252,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.071654+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:37:42.071654+0000","last_peered":"2023-09-23T23:37:42.071654+0000","last_clean":"2023-09-23T23:37:42.071654+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:37:42.071654+0000","last_undegraded":"2023-09-23T23:37:42.071654+0000","last_fullsized":"2023-09-23T23:37:42.071654+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":216,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.076568+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:37:42.076568+0000","last_peered":"2023-09-23T23:37:42.076568+0000","last_clean":"2023-09-23T23:37:42.076568+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:37:42.076568+0000","last_undegraded":"2023-09-23T23:37:42.076568+0000","last_fullsized":"2023-09-23T23:37:42.076568+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"604'8960","reported_seq":9048,"reported_epoch":604,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:37:59.486228+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:37:59.486228+0000","last_peered":"2023-09-23T23:37:59.486228+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:37:59.486228+0000","last_undegraded":"2023-09-23T23:37:59.486228+0000","last_fullsized":"2023-09-23T23:37:59.486228+0000","mapping_epoch":562,"log_start":"551'3200","ondisk_log_start":"551'3200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5760,"log_dups_size":0,"ondisk_log_size":5760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127270912,"num_objects":1942,"num_object_clones":0,"num_object_copies":5826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8960,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":1942}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":619,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.072739+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:37:42.072739+0000","last_peered":"2023-09-23T23:37:42.072739+0000","last_clean":"2023-09-23T23:37:42.072739+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:37:42.072739+0000","last_undegraded":"2023-09-23T23:37:42.072739+0000","last_fullsized":"2023-09-23T23:37:42.072739+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":890,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.242429+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:37:42.242429+0000","last_peered":"2023-09-23T23:37:42.242429+0000","last_clean":"2023-09-23T23:37:42.242429+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:37:42.242429+0000","last_undegraded":"2023-09-23T23:37:42.242429+0000","last_fullsized":"2023-09-23T23:37:42.242429+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":566,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.088289+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:37:42.088289+0000","last_peered":"2023-09-23T23:37:42.088289+0000","last_clean":"2023-09-23T23:37:42.088289+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:37:42.088289+0000","last_undegraded":"2023-09-23T23:37:42.088289+0000","last_fullsized":"2023-09-23T23:37:42.088289+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":128,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.243836+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:37:42.243836+0000","last_peered":"2023-09-23T23:37:42.243836+0000","last_clean":"2023-09-23T23:37:42.243836+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:37:42.243836+0000","last_undegraded":"2023-09-23T23:37:42.243836+0000","last_fullsized":"2023-09-23T23:37:42.243836+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"604'8734","reported_seq":8812,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:59.195009+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:37:59.195009+0000","last_peered":"2023-09-23T23:37:59.195009+0000","last_clean":"2023-09-23T23:37:59.195009+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:37:59.195009+0000","last_undegraded":"2023-09-23T23:37:59.195009+0000","last_fullsized":"2023-09-23T23:37:59.195009+0000","mapping_epoch":534,"log_start":"552'3300","ondisk_log_start":"552'3300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5434,"log_dups_size":0,"ondisk_log_size":5434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131203072,"num_objects":2002,"num_object_clones":0,"num_object_copies":6006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8734,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":716,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.243124+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:37:42.243124+0000","last_peered":"2023-09-23T23:37:42.243124+0000","last_clean":"2023-09-23T23:37:42.243124+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:37:42.243124+0000","last_undegraded":"2023-09-23T23:37:42.243124+0000","last_fullsized":"2023-09-23T23:37:42.243124+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":531,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.078763+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:37:42.078763+0000","last_peered":"2023-09-23T23:37:42.078763+0000","last_clean":"2023-09-23T23:37:42.078763+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:37:42.078763+0000","last_undegraded":"2023-09-23T23:37:42.078763+0000","last_fullsized":"2023-09-23T23:37:42.078763+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":446,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.138569+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:37:52.138569+0000","last_peered":"2023-09-23T23:37:52.138569+0000","last_clean":"2023-09-23T23:37:52.138569+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:37:52.138569+0000","last_undegraded":"2023-09-23T23:37:52.138569+0000","last_fullsized":"2023-09-23T23:37:52.138569+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":46,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.119885+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:37:52.119885+0000","last_peered":"2023-09-23T23:37:52.119885+0000","last_clean":"2023-09-23T23:37:52.119885+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:37:52.119885+0000","last_undegraded":"2023-09-23T23:37:52.119885+0000","last_fullsized":"2023-09-23T23:37:52.119885+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"604'9074","reported_seq":12752,"reported_epoch":604,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:00.024262+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:38:00.024262+0000","last_peered":"2023-09-23T23:38:00.024262+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:38:00.024262+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"551'2900","ondisk_log_start":"551'2900","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6174,"log_dups_size":0,"ondisk_log_size":6174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126091264,"num_objects":1924,"num_object_clones":0,"num_object_copies":5772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9074,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":-780},{"shards":"1,2,4","objects":922}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":599,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.243972+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:37:42.243972+0000","last_peered":"2023-09-23T23:37:42.243972+0000","last_clean":"2023-09-23T23:37:42.243972+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:37:42.243972+0000","last_undegraded":"2023-09-23T23:37:42.243972+0000","last_fullsized":"2023-09-23T23:37:42.243972+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":46,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.118991+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:37:52.118991+0000","last_peered":"2023-09-23T23:37:52.118991+0000","last_clean":"2023-09-23T23:37:52.118991+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:37:52.118991+0000","last_undegraded":"2023-09-23T23:37:52.118991+0000","last_fullsized":"2023-09-23T23:37:52.118991+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":144,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.072300+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:37:42.072300+0000","last_peered":"2023-09-23T23:37:42.072300+0000","last_clean":"2023-09-23T23:37:42.072300+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:37:42.072300+0000","last_undegraded":"2023-09-23T23:37:42.072300+0000","last_fullsized":"2023-09-23T23:37:42.072300+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":337,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.238912+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:37:42.238912+0000","last_peered":"2023-09-23T23:37:42.238912+0000","last_clean":"2023-09-23T23:37:42.238912+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:37:42.238912+0000","last_undegraded":"2023-09-23T23:37:42.238912+0000","last_fullsized":"2023-09-23T23:37:42.238912+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"604'9023","reported_seq":9101,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:59.502146+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:37:59.502146+0000","last_peered":"2023-09-23T23:37:59.502146+0000","last_clean":"2023-09-23T23:37:59.502146+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:37:59.502146+0000","last_undegraded":"2023-09-23T23:37:59.502146+0000","last_fullsized":"2023-09-23T23:37:59.502146+0000","mapping_epoch":534,"log_start":"552'3300","ondisk_log_start":"552'3300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5723,"log_dups_size":0,"ondisk_log_size":5723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127205376,"num_objects":1941,"num_object_clones":0,"num_object_copies":5823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9023,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":785,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.072018+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:37:42.072018+0000","last_peered":"2023-09-23T23:37:42.072018+0000","last_clean":"2023-09-23T23:37:42.072018+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:37:42.072018+0000","last_undegraded":"2023-09-23T23:37:42.072018+0000","last_fullsized":"2023-09-23T23:37:42.072018+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":602,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.088482+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:37:42.088482+0000","last_peered":"2023-09-23T23:37:42.088482+0000","last_clean":"2023-09-23T23:37:42.088482+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:37:42.088482+0000","last_undegraded":"2023-09-23T23:37:42.088482+0000","last_fullsized":"2023-09-23T23:37:42.088482+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":208,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.243871+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:37:42.243871+0000","last_peered":"2023-09-23T23:37:42.243871+0000","last_clean":"2023-09-23T23:37:42.243871+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:37:42.243871+0000","last_undegraded":"2023-09-23T23:37:42.243871+0000","last_fullsized":"2023-09-23T23:37:42.243871+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":156,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.071495+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:37:42.071495+0000","last_peered":"2023-09-23T23:37:42.071495+0000","last_clean":"2023-09-23T23:37:42.071495+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:37:42.071495+0000","last_undegraded":"2023-09-23T23:37:42.071495+0000","last_fullsized":"2023-09-23T23:37:42.071495+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"604'8868","reported_seq":8949,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:59.943410+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:37:59.943410+0000","last_peered":"2023-09-23T23:37:59.943410+0000","last_clean":"2023-09-23T23:37:59.943410+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:37:59.943410+0000","last_undegraded":"2023-09-23T23:37:59.943410+0000","last_fullsized":"2023-09-23T23:37:59.943410+0000","mapping_epoch":534,"log_start":"551'2700","ondisk_log_start":"551'2700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6168,"log_dups_size":0,"ondisk_log_size":6168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133693440,"num_objects":2040,"num_object_clones":0,"num_object_copies":6120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8868,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":992,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:48.238809+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:37:48.238809+0000","last_peered":"2023-09-23T23:37:48.238809+0000","last_clean":"2023-09-23T23:37:48.238809+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:37:48.238809+0000","last_undegraded":"2023-09-23T23:37:48.238809+0000","last_fullsized":"2023-09-23T23:37:48.238809+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":118,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.244600+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:37:42.244600+0000","last_peered":"2023-09-23T23:37:42.244600+0000","last_clean":"2023-09-23T23:37:42.244600+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:37:42.244600+0000","last_undegraded":"2023-09-23T23:37:42.244600+0000","last_fullsized":"2023-09-23T23:37:42.244600+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":50,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.119969+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:37:52.119969+0000","last_peered":"2023-09-23T23:37:52.119969+0000","last_clean":"2023-09-23T23:37:52.119969+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:37:52.119969+0000","last_undegraded":"2023-09-23T23:37:52.119969+0000","last_fullsized":"2023-09-23T23:37:52.119969+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":178,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.237123+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:37:42.237123+0000","last_peered":"2023-09-23T23:37:42.237123+0000","last_clean":"2023-09-23T23:37:42.237123+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:37:42.237123+0000","last_undegraded":"2023-09-23T23:37:42.237123+0000","last_fullsized":"2023-09-23T23:37:42.237123+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1041,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.085205+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:37:42.085205+0000","last_peered":"2023-09-23T23:37:42.085205+0000","last_clean":"2023-09-23T23:37:42.085205+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:37:42.085205+0000","last_undegraded":"2023-09-23T23:37:42.085205+0000","last_fullsized":"2023-09-23T23:37:42.085205+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":46,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.118743+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:37:52.118743+0000","last_peered":"2023-09-23T23:37:52.118743+0000","last_clean":"2023-09-23T23:37:52.118743+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:37:52.118743+0000","last_undegraded":"2023-09-23T23:37:52.118743+0000","last_fullsized":"2023-09-23T23:37:52.118743+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":360,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:50.079713+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:37:50.079713+0000","last_peered":"2023-09-23T23:37:50.079713+0000","last_clean":"2023-09-23T23:37:50.079713+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:37:50.079713+0000","last_undegraded":"2023-09-23T23:37:50.079713+0000","last_fullsized":"2023-09-23T23:37:50.079713+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":753,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.140424+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:37:52.140424+0000","last_peered":"2023-09-23T23:37:52.140424+0000","last_clean":"2023-09-23T23:37:52.140424+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:37:52.140424+0000","last_undegraded":"2023-09-23T23:37:52.140424+0000","last_fullsized":"2023-09-23T23:37:52.140424+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":106,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.243277+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:37:42.243277+0000","last_peered":"2023-09-23T23:37:42.243277+0000","last_clean":"2023-09-23T23:37:42.243277+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:37:42.243277+0000","last_undegraded":"2023-09-23T23:37:42.243277+0000","last_fullsized":"2023-09-23T23:37:42.243277+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":452,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.087259+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:37:42.087259+0000","last_peered":"2023-09-23T23:37:42.087259+0000","last_clean":"2023-09-23T23:37:42.087259+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:37:42.087259+0000","last_undegraded":"2023-09-23T23:37:42.087259+0000","last_fullsized":"2023-09-23T23:37:42.087259+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":302,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.243773+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:37:42.243773+0000","last_peered":"2023-09-23T23:37:42.243773+0000","last_clean":"2023-09-23T23:37:42.243773+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:37:42.243773+0000","last_undegraded":"2023-09-23T23:37:42.243773+0000","last_fullsized":"2023-09-23T23:37:42.243773+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":291,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.088337+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:37:42.088337+0000","last_peered":"2023-09-23T23:37:42.088337+0000","last_clean":"2023-09-23T23:37:42.088337+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:37:42.088337+0000","last_undegraded":"2023-09-23T23:37:42.088337+0000","last_fullsized":"2023-09-23T23:37:42.088337+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":816,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.243410+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:37:42.243410+0000","last_peered":"2023-09-23T23:37:42.243410+0000","last_clean":"2023-09-23T23:37:42.243410+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:37:42.243410+0000","last_undegraded":"2023-09-23T23:37:42.243410+0000","last_fullsized":"2023-09-23T23:37:42.243410+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":635,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.086434+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:37:42.086434+0000","last_peered":"2023-09-23T23:37:42.086434+0000","last_clean":"2023-09-23T23:37:42.086434+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:37:42.086434+0000","last_undegraded":"2023-09-23T23:37:42.086434+0000","last_fullsized":"2023-09-23T23:37:42.086434+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":619,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.085354+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:37:42.085354+0000","last_peered":"2023-09-23T23:37:42.085354+0000","last_clean":"2023-09-23T23:37:42.085354+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:37:42.085354+0000","last_undegraded":"2023-09-23T23:37:42.085354+0000","last_fullsized":"2023-09-23T23:37:42.085354+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":777,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.089418+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:37:42.089418+0000","last_peered":"2023-09-23T23:37:42.089418+0000","last_clean":"2023-09-23T23:37:42.089418+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:37:42.089418+0000","last_undegraded":"2023-09-23T23:37:42.089418+0000","last_fullsized":"2023-09-23T23:37:42.089418+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":448,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.072528+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:37:42.072528+0000","last_peered":"2023-09-23T23:37:42.072528+0000","last_clean":"2023-09-23T23:37:42.072528+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:37:42.072528+0000","last_undegraded":"2023-09-23T23:37:42.072528+0000","last_fullsized":"2023-09-23T23:37:42.072528+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":671,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T23:37:42.242325+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:37:42.242325+0000","last_peered":"2023-09-23T23:37:42.242325+0000","last_clean":"2023-09-23T23:37:42.242325+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:37:42.242325+0000","last_undegraded":"2023-09-23T23:37:42.242325+0000","last_fullsized":"2023-09-23T23:37:42.242325+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":2089484312,"num_objects":31884,"num_object_clones":0,"num_object_copies":95652,"num_objects_missing_on_primary":994,"num_objects_missing":994,"num_objects_degraded":21709,"num_objects_misplaced":1889,"num_objects_unfound":0,"num_objects_dirty":31884,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":143672,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19944,"num_bytes_recovered":1306853376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8110682112,"data_stored":8110669896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90150,"ondisk_log_size":90150,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782702,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1243872,"kb_used_data":853016,"kb_used_omap":8,"kb_used_meta":390839,"kb_avail":92501280,"statfs":{"total":95995035648,"available":94721310720,"internally_reserved":0,"allocated":873488384,"data_stored":871799230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":400219205},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":1.048,"15min":2.1080000000000001},"min":{"1min":0.25900000000000001,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":0.51300000000000001,"5min":17.140000000000001,"15min":85.721000000000004},"last":0.435},{"interface":"front","average":{"1min":0.47199999999999998,"5min":1.0600000000000001,"15min":2.1829999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":2.331,"5min":11.069000000000001,"15min":85.965000000000003},"last":0.34599999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":2.532,"15min":3.1309999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":1.5780000000000001,"5min":129.11199999999999,"15min":198.58000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":2.3039999999999998,"15min":3.1440000000000001},"min":{"1min":0.31900000000000001,"5min":0.247,"15min":0.246},"max":{"1min":2.3690000000000002,"5min":129.02600000000001,"15min":198.71100000000001},"last":0.27200000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":3.3980000000000001,"5min":2.6880000000000002,"15min":3.8050000000000002},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":53.054000000000002,"5min":78.528999999999996,"15min":203.58000000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":3.5699999999999998,"5min":1.9299999999999999,"15min":3.2229999999999999},"min":{"1min":0.33400000000000002,"5min":0.27900000000000003,"15min":0.26200000000000001},"max":{"1min":53.023000000000003,"5min":53.023000000000003,"15min":198.69},"last":0.79900000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":8.6839999999999993,"5min":4.4390000000000001,"15min":4.0750000000000002},"min":{"1min":0.33300000000000002,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":100.40600000000001,"5min":125.511,"15min":279.26499999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":8.7420000000000009,"5min":4.8460000000000001,"15min":4.6180000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29599999999999999},"max":{"1min":100.42400000000001,"5min":125.48999999999999,"15min":279.29899999999998},"last":0.65200000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":1.546,"15min":1.0389999999999999},"min":{"1min":0.28999999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":0.58099999999999996,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":1.341,"15min":0.97599999999999998},"min":{"1min":0.32500000000000001,"5min":0.317,"15min":0.23899999999999999},"max":{"1min":2.3980000000000001,"5min":31.68,"15min":31.68},"last":0.371}]},{"osd":5,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.6220000000000001,"15min":1.863},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.26500000000000001},"max":{"1min":2.4510000000000001,"5min":60.463999999999999,"15min":210.83600000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.5840000000000001,"15min":1.742},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.5940000000000001,"5min":63.006,"15min":207.04599999999999},"last":0.41999999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.827,"15min":1.3320000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.29899999999999999},"max":{"1min":1.619,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":2.052,"15min":1.544},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.389,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.58299999999999996}]}]},{"osd":6,"up_from":46,"seq":197568496179,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1492368,"kb_used_data":782600,"kb_used_omap":0,"kb_used_meta":709760,"kb_avail":92252784,"statfs":{"total":95995035648,"available":94466850816,"internally_reserved":0,"allocated":801382400,"data_stored":799709239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":726794240},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.73499999999999999,"15min":4.4370000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.21099999999999999},"max":{"1min":0.53700000000000003,"5min":6.4450000000000003,"15min":135.31999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.498,"5min":0.63800000000000001,"15min":4.3869999999999996},"min":{"1min":0.27800000000000002,"5min":0.24399999999999999,"15min":0.23000000000000001},"max":{"1min":1.742,"5min":3.8239999999999998,"15min":135.34299999999999},"last":0.47899999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.90500000000000003,"15min":2.2890000000000001},"min":{"1min":0.28299999999999997,"5min":0.253,"15min":0.245},"max":{"1min":1.8759999999999999,"5min":6.9080000000000004,"15min":89.605000000000004},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.749,"5min":0.72499999999999998,"15min":2.391},"min":{"1min":0.29099999999999998,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":3.8980000000000001,"5min":3.8980000000000001,"15min":91.025999999999996},"last":0.46200000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:37:06 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":3.6930000000000001,"15min":4.4749999999999996},"min":{"1min":0.27900000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.1800000000000002,"5min":67.796999999999997,"15min":127.44499999999999},"last":0.441},{"interface":"front","average":{"1min":0.66600000000000004,"5min":3.6749999999999998,"15min":4.8680000000000003},"min":{"1min":0.316,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.2080000000000002,"5min":67.893000000000001,"15min":125.539},"last":0.373}]},{"osd":3,"last update":"Sat Sep 23 23:37:57 2023","interfaces":[{"interface":"back","average":{"1min":13.925000000000001,"5min":8.1059999999999999,"15min":5.2009999999999996},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":199.536,"5min":199.536,"15min":199.536},"last":199.536},{"interface":"front","average":{"1min":13.538,"5min":7.9660000000000002,"15min":5.4530000000000003},"min":{"1min":0.36799999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":199.59999999999999,"5min":199.59999999999999,"15min":199.59999999999999},"last":199.59999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.755,"15min":0.66700000000000004},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.23400000000000001},"max":{"1min":1.76,"5min":7.101,"15min":8.1999999999999993},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.81399999999999995,"15min":0.76900000000000002},"min":{"1min":0.26700000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.865,"5min":8.3499999999999996,"15min":8.3499999999999996},"last":0.42999999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.76100000000000001,"15min":0.79800000000000004},"min":{"1min":0.27700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.847,"5min":11.372,"15min":11.473000000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.876,"15min":0.84999999999999998},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.26900000000000002},"max":{"1min":6.1500000000000004,"5min":7.0129999999999999,"15min":7.5670000000000002},"last":0.40400000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:37:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.1679999999999999,"15min":1.0029999999999999},"min":{"1min":0.32700000000000001,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":0.77000000000000002,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.204,"15min":1.056},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33500000000000002},"max":{"1min":1.792,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.49399999999999999}]}]},{"osd":1,"up_from":112,"seq":481036337520,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1553120,"kb_used_data":656472,"kb_used_omap":4,"kb_used_meta":896635,"kb_avail":92192032,"statfs":{"total":95995035648,"available":94404640768,"internally_reserved":0,"allocated":672227328,"data_stored":669371523,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":918155150},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.407,"15min":3.173},"min":{"1min":0.19,"5min":0.19,"15min":0.16700000000000001},"max":{"1min":3.5830000000000002,"5min":47.195999999999998,"15min":130.768},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.377,"5min":1.119,"15min":2.5030000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19},"max":{"1min":1.143,"5min":46.582000000000001,"15min":134.929},"last":0.35799999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":3.5659999999999998,"15min":6.7649999999999997},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":1.401,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.66900000000000004,"5min":3.8849999999999998,"15min":6.8339999999999996},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.23,"5min":202.12,"15min":232.21000000000001},"last":0.46400000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":6.883,"5min":5.8460000000000001,"15min":5.4379999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22600000000000001},"max":{"1min":71.745000000000005,"5min":181.94399999999999,"15min":234.404},"last":1.7909999999999999},{"interface":"front","average":{"1min":6.8639999999999999,"5min":5.8860000000000001,"15min":5.3899999999999997},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":71.786000000000001,"5min":181.98500000000001,"15min":236.33699999999999},"last":0.69499999999999995}]},{"osd":4,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":0.95099999999999996,"15min":1.399},"min":{"1min":0.32500000000000001,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":4.4880000000000004,"5min":8.0850000000000009,"15min":12.042999999999999},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.85699999999999998,"15min":1.45},"min":{"1min":0.316,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":4.6660000000000004,"5min":9.5069999999999997,"15min":18.108000000000001},"last":0.42899999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":2.6309999999999998,"15min":1.734},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28899999999999998},"max":{"1min":0.98199999999999998,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.024,"5min":2.661,"15min":2.0710000000000002},"min":{"1min":0.34399999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.194,"5min":151.821,"15min":151.821},"last":0.70699999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.0449999999999999,"15min":1.6659999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.27500000000000002},"max":{"1min":2.306,"5min":8.9269999999999996,"15min":74.087999999999994},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.2849999999999999,"15min":1.929},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33100000000000002},"max":{"1min":3.29,"5min":8.1400000000000006,"15min":73.974999999999994},"last":0.67800000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":1.5449999999999999,"15min":2.359},"min":{"1min":0.40200000000000002,"5min":0.38700000000000001,"15min":0.32900000000000001},"max":{"1min":2.504,"5min":44.241,"15min":78.308000000000007},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.556,"15min":2.2730000000000001},"min":{"1min":0.41299999999999998,"5min":0.37,"15min":0.32700000000000001},"max":{"1min":3.258,"5min":43.920999999999999,"15min":77.204999999999998},"last":0.874}]}]},{"osd":0,"up_from":91,"seq":390842024344,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1417652,"kb_used_data":193452,"kb_used_omap":5,"kb_used_meta":1224186,"kb_avail":92327500,"statfs":{"total":95995035648,"available":94543360000,"internally_reserved":0,"allocated":198094848,"data_stored":195221587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1253567098},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:37:39 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":4.0620000000000003,"15min":6.4610000000000003},"min":{"1min":0.23200000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":8.4350000000000005,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.39600000000000002,"5min":3.718,"15min":5.6509999999999998},"min":{"1min":0.219,"5min":0.20200000000000001,"15min":0.17599999999999999},"max":{"1min":1.1890000000000001,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.26800000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":6.9610000000000003,"15min":7.3559999999999999},"min":{"1min":0.26200000000000001,"5min":0.214,"15min":0.17199999999999999},"max":{"1min":4.0899999999999999,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.214},{"interface":"front","average":{"1min":0.84599999999999997,"5min":6.6799999999999997,"15min":7.3869999999999996},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.17599999999999999},"max":{"1min":3.468,"5min":283.161,"15min":283.161},"last":0.24299999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:37:39 2023","interfaces":[{"interface":"back","average":{"1min":16.741,"5min":6.8769999999999998,"15min":6.4880000000000004},"min":{"1min":0.27300000000000002,"5min":0.219,"15min":0.217},"max":{"1min":194.26300000000001,"5min":194.26300000000001,"15min":220.732},"last":66.093000000000004},{"interface":"front","average":{"1min":16.422000000000001,"5min":6.75,"15min":6.5830000000000002},"min":{"1min":0.30199999999999999,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":194.56299999999999,"5min":194.56299999999999,"15min":218.78200000000001},"last":66.123999999999995}]},{"osd":4,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.79900000000000004,"15min":2.3479999999999999},"min":{"1min":0.34399999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.60999999999999999,"5min":9.3140000000000001,"15min":138.81299999999999},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.66800000000000004,"5min":1.02,"15min":3.0270000000000001},"min":{"1min":0.36799999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.661,"5min":9.3670000000000009,"15min":171.251},"last":0.68100000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.1850000000000001,"15min":2.7290000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":1.288,"5min":15.076000000000001,"15min":171.30699999999999},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.097,"15min":1.9930000000000001},"min":{"1min":0.33600000000000002,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":4.1710000000000003,"5min":6.5279999999999996,"15min":139.40799999999999},"last":0.67000000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":1.143,"15min":3.0760000000000001},"min":{"1min":0.374,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":12.654999999999999,"5min":12.654999999999999,"15min":171.33600000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.155,"15min":3.2749999999999999},"min":{"1min":0.41199999999999998,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":4.6440000000000001,"5min":9.157,"15min":184.13499999999999},"last":0.71099999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":1.089,"15min":2.569},"min":{"1min":0.35099999999999998,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":2.6749999999999998,"5min":8.9510000000000005,"15min":138.77699999999999},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.3260000000000001,"15min":2.508},"min":{"1min":0.373,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":4.2229999999999999,"5min":9.8870000000000005,"15min":139.37899999999999},"last":0.59199999999999997}]}]},{"osd":2,"up_from":199,"seq":854698492165,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2143012,"kb_used_data":1272860,"kb_used_omap":7,"kb_used_meta":870136,"kb_avail":91602140,"statfs":{"total":95995035648,"available":93800591360,"internally_reserved":0,"allocated":1303408640,"data_stored":1300492910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":891019345},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.7789999999999999,"15min":3.617},"min":{"1min":0.20200000000000001,"5min":0.186,"15min":0.16700000000000001},"max":{"1min":6.3940000000000001,"5min":61.950000000000003,"15min":119.277},"last":1.823},{"interface":"front","average":{"1min":0.54400000000000004,"5min":1.8740000000000001,"15min":3.46},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.18099999999999999},"max":{"1min":5.0339999999999998,"5min":61.984999999999999,"15min":119.56100000000001},"last":4.5739999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":4.4960000000000004,"15min":4.5449999999999999},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":4.7060000000000004,"5min":106.765,"15min":157.98699999999999},"last":2.9089999999999998},{"interface":"front","average":{"1min":0.73899999999999999,"5min":4.1210000000000004,"15min":4.4950000000000001},"min":{"1min":0.23400000000000001,"5min":0.222,"15min":0.222},"max":{"1min":6.4340000000000002,"5min":106.732,"15min":158.03800000000001},"last":1.849}]},{"osd":3,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":5.3179999999999996,"15min":4.6900000000000004},"min":{"1min":0.26000000000000001,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":1.075,"5min":106.70399999999999,"15min":119.366},"last":4.1740000000000004},{"interface":"front","average":{"1min":0.52100000000000002,"5min":5.0359999999999996,"15min":4.1440000000000001},"min":{"1min":0.246,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.6120000000000001,"5min":106.66800000000001,"15min":106.66800000000001},"last":1.792}]},{"osd":4,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":1.756,"15min":2.1909999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.25},"max":{"1min":0.65600000000000003,"5min":62.081000000000003,"15min":119.88200000000001},"last":3.5339999999999998},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.9710000000000001,"15min":1.829},"min":{"1min":0.42299999999999999,"5min":0.35599999999999998,"15min":0.28499999999999998},"max":{"1min":1.121,"5min":62.072000000000003,"15min":62.072000000000003},"last":2.9489999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":2.8340000000000001,"15min":2.5840000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":2.6549999999999998,"5min":91.207999999999998,"15min":119.919},"last":3.589},{"interface":"front","average":{"1min":0.69399999999999995,"5min":2.919,"15min":2.5800000000000001},"min":{"1min":0.371,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.4289999999999998,"5min":91.150999999999996,"15min":119.306},"last":2.9929999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":3.254,"15min":2.9169999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":1.04,"5min":107.018,"15min":119.943},"last":3.5649999999999999},{"interface":"front","average":{"1min":0.746,"5min":2.1779999999999999,"15min":2.1160000000000001},"min":{"1min":0.45100000000000001,"5min":0.371,"15min":0.34999999999999998},"max":{"1min":3.4950000000000001,"5min":62.493000000000002,"15min":62.493000000000002},"last":2.9710000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.917,"15min":1.883},"min":{"1min":0.41699999999999998,"5min":0.374,"15min":0.34000000000000002},"max":{"1min":1.831,"5min":62.603000000000002,"15min":62.603000000000002},"last":1.877},{"interface":"front","average":{"1min":0.66800000000000004,"5min":3.1800000000000002,"15min":2.3220000000000001},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.36199999999999999},"max":{"1min":1.792,"5min":106.809,"15min":106.809},"last":1.9039999999999999}]}]},{"osd":3,"up_from":148,"seq":635655160133,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1659608,"kb_used_data":734160,"kb_used_omap":2,"kb_used_meta":925437,"kb_avail":92085544,"statfs":{"total":95995035648,"available":94295597056,"internally_reserved":0,"allocated":751779840,"data_stored":748929371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":947647541},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,9,7,4],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":1.0009999999999999,"15min":2.5059999999999998},"min":{"1min":0.19500000000000001,"5min":0.184,"15min":0.17699999999999999},"max":{"1min":2.9980000000000002,"5min":17.213000000000001,"15min":103.866},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.995,"15min":2.8069999999999999},"min":{"1min":0.188,"5min":0.188,"15min":0.17199999999999999},"max":{"1min":2.8090000000000002,"5min":12.202,"15min":101.884},"last":0.47599999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.95999999999999996,"15min":2.819},"min":{"1min":0.23899999999999999,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":2.851,"5min":7.3319999999999999,"15min":85.242999999999995},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.88400000000000001,"15min":2.6920000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":2.9830000000000001,"5min":11.228,"15min":85.302000000000007},"last":0.26300000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:37:57 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":1.4359999999999999,"15min":4.0659999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.19800000000000001},"max":{"1min":36.362000000000002,"5min":36.362000000000002,"15min":84.724999999999994},"last":0.59299999999999997},{"interface":"front","average":{"1min":2.6110000000000002,"5min":1.431,"15min":3.931},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.214},"max":{"1min":36.158999999999999,"5min":36.158999999999999,"15min":84.567999999999998},"last":0.376}]},{"osd":4,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":1.032,"15min":1.202},"min":{"1min":0.318,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":2.9609999999999999,"5min":11.209,"15min":11.209},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.74199999999999999,"5min":1.1140000000000001,"15min":1.6160000000000001},"min":{"1min":0.34000000000000002,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":2.7639999999999998,"5min":7.681,"15min":24.782},"last":0.56899999999999995}]},{"osd":5,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":10.651,"5min":8.0739999999999998,"15min":3.9369999999999998},"min":{"1min":0.38500000000000001,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":196.358,"5min":214.43700000000001,"15min":214.43700000000001},"last":0.66000000000000003},{"interface":"front","average":{"1min":10.586,"5min":8.0500000000000007,"15min":3.984},"min":{"1min":0.35999999999999999,"5min":0.34699999999999998,"15min":0.30599999999999999},"max":{"1min":196.048,"5min":214.46600000000001,"15min":214.46600000000001},"last":0.64100000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":7.2190000000000003,"5min":3.2280000000000002,"15min":2.5649999999999999},"min":{"1min":0.35799999999999998,"5min":0.318,"15min":0.318},"max":{"1min":125.94,"5min":125.94,"15min":125.94},"last":0.68600000000000005},{"interface":"front","average":{"1min":7.1189999999999998,"5min":3.3559999999999999,"15min":2.5649999999999999},"min":{"1min":0.36499999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":125.955,"5min":125.955,"15min":125.955},"last":0.67300000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:37:57 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":4.5289999999999999,"15min":3.1909999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32600000000000001},"max":{"1min":6.5800000000000001,"5min":192.69,"15min":192.69},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.92400000000000004,"5min":4.5789999999999997,"15min":3.629},"min":{"1min":0.40699999999999997,"5min":0.40100000000000002,"15min":0.36699999999999999},"max":{"1min":3.6120000000000001,"5min":192.65600000000001,"15min":192.65600000000001},"last":0.624}]}]},{"osd":4,"up_from":0,"seq":128849019452,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757815,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1972940,"kb_used_data":1227524,"kb_used_omap":0,"kb_used_meta":745408,"kb_avail":91772212,"statfs":{"total":95995035648,"available":93974745088,"internally_reserved":0,"allocated":1256984576,"data_stored":1255311632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":763297792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.032,"15min":2.597},"min":{"1min":0.25,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":2.9279999999999999,"5min":21.109000000000002,"15min":113.869},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.123,"5min":1.407,"15min":3.0659999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":11.154999999999999,"5min":20.876999999999999,"15min":138.55199999999999},"last":1.0429999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:37:22 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.1459999999999999,"15min":2.1629999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.1360000000000001,"5min":29.007999999999999,"15min":69.816999999999993},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.212,"15min":2.3929999999999998},"min":{"1min":0.30499999999999999,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":1.452,"5min":29.053999999999998,"15min":70.268000000000001},"last":1.288}]},{"osd":2,"last update":"Sat Sep 23 23:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":6.8869999999999996,"15min":4.4939999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26900000000000002},"max":{"1min":6.2469999999999999,"5min":208.738,"15min":208.738},"last":2.8849999999999998},{"interface":"front","average":{"1min":0.77700000000000002,"5min":5.5220000000000002,"15min":4.0140000000000002},"min":{"1min":0.309,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":3.2290000000000001,"5min":208.631,"15min":208.631},"last":2.863}]},{"osd":3,"last update":"Sat Sep 23 23:37:53 2023","interfaces":[{"interface":"back","average":{"1min":6.6550000000000002,"5min":5.6689999999999996,"15min":4.1470000000000002},"min":{"1min":0.33400000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":143.66300000000001,"5min":143.66300000000001,"15min":143.66300000000001},"last":0.442},{"interface":"front","average":{"1min":6.7670000000000003,"5min":6.0010000000000003,"15min":4.327},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":143.62200000000001,"5min":143.62200000000001,"15min":143.62200000000001},"last":0.46100000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.88200000000000001,"15min":0.877},"min":{"1min":0.316,"5min":0.315,"15min":0.251},"max":{"1min":2.2650000000000001,"5min":9.2690000000000001,"15min":19.962},"last":0.497},{"interface":"front","average":{"1min":0.85499999999999998,"5min":1.016,"15min":0.84099999999999997},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.22900000000000001},"max":{"1min":3.1840000000000002,"5min":10.273999999999999,"15min":20.829999999999998},"last":0.378}]},{"osd":6,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.3340000000000001,"15min":1.5249999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.3100000000000001,"5min":148.36600000000001,"15min":148.36600000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.7389999999999999,"15min":1.71},"min":{"1min":0.32800000000000001,"5min":0.30399999999999999,"15min":0.26000000000000001},"max":{"1min":5.7240000000000002,"5min":148.42500000000001,"15min":148.42500000000001},"last":0.35599999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:37:48 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.306,"15min":2.177},"min":{"1min":0.34699999999999998,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":4.3380000000000001,"5min":15.497,"15min":179.28800000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.157,"5min":1.339,"15min":2.2480000000000002},"min":{"1min":0.40600000000000003,"5min":0.35199999999999998,"15min":0.34999999999999998},"max":{"1min":6.1440000000000001,"5min":15.653,"15min":179.428},"last":0.40300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":189464576,"data_stored":189464576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":660934656,"data_stored":660930584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1279918080,"data_stored":1279918080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":737284096,"data_stored":737280024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2353397760,"data_stored":2353397760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1235943424,"data_stored":1235943424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":790036480,"data_stored":790036480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":863703040,"data_stored":863698968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:38:04.410 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:38:04.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:04 smithi138 ceph-mon[193606]: from='client.47039 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:38:04.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:04 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/512966324' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:04.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:04 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 21709/95724 objects degraded (22.679%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:04.472 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:38:04.510 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:04.567 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:38:04.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:04.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:38:04.675 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:38:04.706 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:04.768 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:38:04.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:04.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:38:04.881 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:38:04.908 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:04.949 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:38:04.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:04.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:38:05.054 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:38:05.129 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:05.134 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:05 smithi138 ceph-mon[193606]: pgmap v1711: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 21554/93864 objects degraded (22.963%); 1741/93864 objects misplaced (1.855%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T23:38:05.222 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:38:05.222 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:38:05.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:05 smithi110 ceph-mon[212939]: pgmap v1711: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 21554/93864 objects degraded (22.963%); 1741/93864 objects misplaced (1.855%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T23:38:05.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:05 smithi110 ceph-mon[221102]: pgmap v1711: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 21554/93864 objects degraded (22.963%); 1741/93864 objects misplaced (1.855%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T23:38:05.626 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:38:05.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:05.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:38:05.733 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:38:05.795 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:06.147 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:38:06.149 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:38:06.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:06.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:38:06.253 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:38:06.270 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:06.331 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:06.344 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:06.478 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:38:06.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:06.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:38:06.586 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:06.646 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:06.732 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:38:06.848 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:38:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:38:06] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-09-23T23:38:07.144 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:07 smithi110 ceph-mon[221102]: pgmap v1712: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 461 op/s; 21433/88746 objects degraded (24.151%); 1741/88746 objects misplaced (1.962%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T23:38:07.300 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:07 smithi138 ceph-mon[193606]: pgmap v1712: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 461 op/s; 21433/88746 objects degraded (24.151%); 1741/88746 objects misplaced (1.962%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T23:38:07.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:07 smithi110 ceph-mon[212939]: pgmap v1712: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 461 op/s; 21433/88746 objects degraded (24.151%); 1741/88746 objects misplaced (1.962%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T23:38:07.882 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:38:08.471 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:08.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:08.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:08.589 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:38:08.639 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:08.707 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:38:08.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:08.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:38:08.793 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:08.811 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:08.873 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:09.062 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:09.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:09.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:09.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:09 smithi110 ceph-mon[212939]: pgmap v1713: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 461 op/s; 21433/88746 objects degraded (24.151%); 1741/88746 objects misplaced (1.962%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T23:38:09.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:09 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 21433/88746 objects degraded (24.151%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:09.145 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:09 smithi110 ceph-mon[221102]: pgmap v1713: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 461 op/s; 21433/88746 objects degraded (24.151%); 1741/88746 objects misplaced (1.962%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T23:38:09.146 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:09 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 21433/88746 objects degraded (24.151%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:09.166 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:38:09.193 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:09.235 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:38:09.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:09.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:38:09.338 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:38:09.366 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:09.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:09 smithi138 ceph-mon[193606]: pgmap v1713: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 461 op/s; 21433/88746 objects degraded (24.151%); 1741/88746 objects misplaced (1.962%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T23:38:09.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:09 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 21433/88746 objects degraded (24.151%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:09.589 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:38:09.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:09.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:38:09.634 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:38:09.634 INFO:teuthology.orchestra.run.smithi110.stdout:{"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 21433/88746 objects degraded (24.151%), 7 pgs degraded, 5 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2167,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":605,"num_osds":8,"num_up_osds":8,"osd_up_since":1695511414,"num_in_osds":7,"osd_in_since":1695512030,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":110},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+recovering","count":1},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":119,"num_pools":5,"num_objects":29582,"data_bytes":1937683701,"bytes_used":11410563072,"bytes_avail":660554686464,"bytes_total":671965249536,"degraded_objects":21433,"degraded_total":88746,"degraded_ratio":0.24150947648344714,"misplaced_objects":1741,"misplaced_total":88746,"misplaced_ratio":0.019617785590336466,"recovering_objects_per_sec":84,"recovering_bytes_per_sec":5522274,"recovering_keys_per_sec":0,"num_objects_recovered":843,"num_bytes_recovered":55246848,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":461},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi110.fuglpi","status":"up:active","gid":24572}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.110:8443/","prometheus":"http://172.21.15.110:9283/"}},"servicemap":{"epoch":32,"modified":"2023-09-23T23:32:57.852217+0000","services":{}},"progress_events":{"833ef15d-07ae-4299-b4e4-2cf94e068147":{"message":"Upgrade to 18.2.0-460-gbb0daeff (10m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true},"c67c6d89-9cd0-495b-b1de-fa24b5d12727":{"message":"Global Recovery Event (3m)\n [=========================...] (remaining: 18s)","progress":0.92436975240707397,"add_to_ceph_s":true}}} 2023-09-23T23:38:09.703 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:38:09.772 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:10.181 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:38:10.368 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:38:10.368 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:38:10.399 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:10 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/221292376' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:38:10.399 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:10 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1865203728' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:38:10.399 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:10 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:38:10.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:10 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/221292376' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:38:10.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:10 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1865203728' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:38:10.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:10 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:38:10.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:10 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/221292376' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:38:10.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:10 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1865203728' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:38:10.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:10 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:38:10.883 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:38:10.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:10.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:38:10.987 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:38:11.018 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:11.266 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:38:11.266 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:38:11.346 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:11 smithi138 ceph-mon[193606]: pgmap v1714: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 21071/81552 objects degraded (25.838%); 1566/81552 objects misplaced (1.920%); 6.1 MiB/s, 96 objects/s recovering 2023-09-23T23:38:11.346 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:11 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:38:11.346 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:11 smithi138 ceph-mon[193606]: failed to encode map e606 with expected crc 2023-09-23T23:38:11.346 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:11 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:38:11.346 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:11 smithi138 ceph-mon[193606]: osdmap e606: 8 total, 8 up, 7 in 2023-09-23T23:38:11.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:11 smithi138 ceph-mon[193606]: failed to encode map e606 with expected crc 2023-09-23T23:38:11.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:11 smithi138 ceph-mon[193606]: failed to encode map e606 with expected crc 2023-09-23T23:38:11.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:11 smithi138 ceph-mon[193606]: failed to encode map e606 with expected crc 2023-09-23T23:38:11.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:11 smithi138 ceph-mon[193606]: failed to encode map e606 with expected crc 2023-09-23T23:38:11.362 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T23:38:11.376 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:38:11.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:11.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:38:11.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[212939]: pgmap v1714: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 21071/81552 objects degraded (25.838%); 1566/81552 objects misplaced (1.920%); 6.1 MiB/s, 96 objects/s recovering 2023-09-23T23:38:11.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:38:11.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[212939]: failed to encode map e606 with expected crc 2023-09-23T23:38:11.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:38:11.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[212939]: osdmap e606: 8 total, 8 up, 7 in 2023-09-23T23:38:11.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[212939]: failed to encode map e606 with expected crc 2023-09-23T23:38:11.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[212939]: failed to encode map e606 with expected crc 2023-09-23T23:38:11.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[212939]: failed to encode map e606 with expected crc 2023-09-23T23:38:11.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[212939]: failed to encode map e606 with expected crc 2023-09-23T23:38:11.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[221102]: pgmap v1714: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 21071/81552 objects degraded (25.838%); 1566/81552 objects misplaced (1.920%); 6.1 MiB/s, 96 objects/s recovering 2023-09-23T23:38:11.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:38:11.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[221102]: failed to encode map e606 with expected crc 2023-09-23T23:38:11.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:38:11.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[221102]: osdmap e606: 8 total, 8 up, 7 in 2023-09-23T23:38:11.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[221102]: failed to encode map e606 with expected crc 2023-09-23T23:38:11.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[221102]: failed to encode map e606 with expected crc 2023-09-23T23:38:11.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[221102]: failed to encode map e606 with expected crc 2023-09-23T23:38:11.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:11 smithi110 ceph-mon[221102]: failed to encode map e606 with expected crc 2023-09-23T23:38:11.484 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:38:11.517 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:11.884 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:38:11.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:11.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:38:11.911 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:38:12.008 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:38:12.082 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:12.199 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:38:12.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:12.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:38:12.314 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:38:12.325 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:12.405 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:12.474 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:38:12.511 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:38:12.544 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:38:12.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:12.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:38:12.554 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:38:12.611 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:38:12.655 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:38:12.663 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:38:12.703 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:38:12.745 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:38:12.751 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:12.787 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:38:12.791 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:38:12.791 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:38:12.791 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:38:12.791 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:38:12.791 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:38:12.791 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:38:12.792 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:38:12.792 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:38:13.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:13 smithi138 ceph-mon[193606]: pgmap v1716: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 21071/79803 objects degraded (26.404%); 1566/79803 objects misplaced (1.962%); 6.1 MiB/s, 96 objects/s recovering 2023-09-23T23:38:13.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:13 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3978880858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:13.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:38:13.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:13 smithi110 ceph-mon[212939]: pgmap v1716: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 21071/79803 objects degraded (26.404%); 1566/79803 objects misplaced (1.962%); 6.1 MiB/s, 96 objects/s recovering 2023-09-23T23:38:13.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:13 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3978880858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:13.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:38:13.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:13 smithi110 ceph-mon[221102]: pgmap v1716: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 21071/79803 objects degraded (26.404%); 1566/79803 objects misplaced (1.962%); 6.1 MiB/s, 96 objects/s recovering 2023-09-23T23:38:13.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:13 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3978880858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:13.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:38:13.647 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:38:13.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:13.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:38:13.751 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:38:13.827 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:13.925 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:38:14.109 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:38:14.109 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:38:14.109 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:38:14.109 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:38:14.109 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:38:14.298 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:38:14.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:14.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:38:14.305 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:14 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 21071/79803 objects degraded (26.404%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:14.311 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:14 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 21071/79803 objects degraded (26.404%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:14.406 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:38:14.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:14 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 21071/79803 objects degraded (26.404%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:14.449 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:14.518 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:38:14.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:14.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:38:14.622 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:38:14.679 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:14.897 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:14.950 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:38:14.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:14.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:38:15.057 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:15.075 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T23:38:15.105 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:15.321 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:15.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:15.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:15.424 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:15 smithi110 ceph-mon[212939]: pgmap v1717: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 20896/77664 objects degraded (26.906%); 1350/77664 objects misplaced (1.738%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T23:38:15.426 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:15 smithi110 ceph-mon[221102]: pgmap v1717: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 20896/77664 objects degraded (26.906%); 1350/77664 objects misplaced (1.738%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T23:38:15.431 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:15.551 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:15.654 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:38:15.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:15 smithi138 ceph-mon[193606]: pgmap v1717: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 20896/77664 objects degraded (26.906%); 1350/77664 objects misplaced (1.738%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T23:38:15.818 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:38:15.835 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:38:16.346 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:16.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:16.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:16.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:16 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:38:16.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:16 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:38:16.430 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:38:16.458 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:38:16.485 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:16 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:38:16.488 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:16.532 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:38:16.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:16.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:38:16.636 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:38:16.663 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:16.714 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:38:16.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:16.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:38:16.819 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:38:16.845 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:16.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:38:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:38:16] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-09-23T23:38:17.364 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1717,"stamp":"2023-09-23T23:38:13.958942+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1695593717,"num_objects":25888,"num_object_clones":0,"num_object_copies":77664,"num_objects_missing_on_primary":454,"num_objects_missing":454,"num_objects_degraded":20896,"num_objects_misplaced":1350,"num_objects_unfound":0,"num_objects_dirty":25888,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":149769,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21361,"num_bytes_recovered":1397223848,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95116,"ondisk_log_size":95116,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10534732,"kb_used_data":4701076,"kb_used_omap":29,"kb_used_meta":5833570,"kb_avail":645681332,"statfs":{"total":671965249536,"available":661177683968,"internally_reserved":0,"allocated":4813901824,"data_stored":4797325430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5973576403},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-353894400,"num_objects":-5400,"num_object_clones":0,"num_object_copies":-16200,"num_objects_missing_on_primary":-391,"num_objects_missing":-391,"num_objects_degraded":-658,"num_objects_misplaced":-391,"num_objects_unfound":0,"num_objects_dirty":-5400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5400,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1056,"num_bytes_recovered":69206016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003332"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":801,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.113693+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:37:52.113693+0000","last_peered":"2023-09-23T23:37:52.113693+0000","last_clean":"2023-09-23T23:37:52.113693+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:37:52.113693+0000","last_undegraded":"2023-09-23T23:37:52.113693+0000","last_fullsized":"2023-09-23T23:37:52.113693+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":360,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.118728+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:37:52.118728+0000","last_peered":"2023-09-23T23:37:52.118728+0000","last_clean":"2023-09-23T23:37:52.118728+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:37:52.118728+0000","last_undegraded":"2023-09-23T23:37:52.118728+0000","last_fullsized":"2023-09-23T23:37:52.118728+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":588,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.158495+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:38:00.158495+0000","last_peered":"2023-09-23T23:38:00.158495+0000","last_clean":"2023-09-23T23:38:00.158495+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:38:00.158495+0000","last_undegraded":"2023-09-23T23:38:00.158495+0000","last_fullsized":"2023-09-23T23:38:00.158495+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":754,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.113799+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:37:52.113799+0000","last_peered":"2023-09-23T23:37:52.113799+0000","last_clean":"2023-09-23T23:37:52.113799+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:37:52.113799+0000","last_undegraded":"2023-09-23T23:37:52.113799+0000","last_fullsized":"2023-09-23T23:37:52.113799+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":325,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.146463+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:38:00.146463+0000","last_peered":"2023-09-23T23:38:00.146463+0000","last_clean":"2023-09-23T23:38:00.146463+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:38:00.146463+0000","last_undegraded":"2023-09-23T23:38:00.146463+0000","last_fullsized":"2023-09-23T23:38:00.146463+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":567,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.148886+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:38:00.148886+0000","last_peered":"2023-09-23T23:38:00.148886+0000","last_clean":"2023-09-23T23:38:00.148886+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:38:00.148886+0000","last_undegraded":"2023-09-23T23:38:00.148886+0000","last_fullsized":"2023-09-23T23:38:00.148886+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":767,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.150808+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:38:00.150808+0000","last_peered":"2023-09-23T23:38:00.150808+0000","last_clean":"2023-09-23T23:38:00.150808+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:38:00.150808+0000","last_undegraded":"2023-09-23T23:38:00.150808+0000","last_fullsized":"2023-09-23T23:38:00.150808+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":625,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.146056+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:38:00.146056+0000","last_peered":"2023-09-23T23:38:00.146056+0000","last_clean":"2023-09-23T23:38:00.146056+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:38:00.146056+0000","last_undegraded":"2023-09-23T23:38:00.146056+0000","last_fullsized":"2023-09-23T23:38:00.146056+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":119,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.119527+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:37:52.119527+0000","last_peered":"2023-09-23T23:37:52.119527+0000","last_clean":"2023-09-23T23:37:52.119527+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:37:52.119527+0000","last_undegraded":"2023-09-23T23:37:52.119527+0000","last_fullsized":"2023-09-23T23:37:52.119527+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":926,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.151307+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:38:00.151307+0000","last_peered":"2023-09-23T23:38:00.151307+0000","last_clean":"2023-09-23T23:38:00.151307+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:38:00.151307+0000","last_undegraded":"2023-09-23T23:38:00.151307+0000","last_fullsized":"2023-09-23T23:38:00.151307+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":257,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.114448+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:37:52.114448+0000","last_peered":"2023-09-23T23:37:52.114448+0000","last_clean":"2023-09-23T23:37:52.114448+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:37:52.114448+0000","last_undegraded":"2023-09-23T23:37:52.114448+0000","last_fullsized":"2023-09-23T23:37:52.114448+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":607,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.146009+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:38:00.146009+0000","last_peered":"2023-09-23T23:38:00.146009+0000","last_clean":"2023-09-23T23:38:00.146009+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:38:00.146009+0000","last_undegraded":"2023-09-23T23:38:00.146009+0000","last_fullsized":"2023-09-23T23:38:00.146009+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":899,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.117982+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:37:52.117982+0000","last_peered":"2023-09-23T23:37:52.117982+0000","last_clean":"2023-09-23T23:37:52.117982+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:37:52.117982+0000","last_undegraded":"2023-09-23T23:37:52.117982+0000","last_fullsized":"2023-09-23T23:37:52.117982+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":236,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.114285+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:37:52.114285+0000","last_peered":"2023-09-23T23:37:52.114285+0000","last_clean":"2023-09-23T23:37:52.114285+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:37:52.114285+0000","last_undegraded":"2023-09-23T23:37:52.114285+0000","last_fullsized":"2023-09-23T23:37:52.114285+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":568,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.146310+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:38:00.146310+0000","last_peered":"2023-09-23T23:38:00.146310+0000","last_clean":"2023-09-23T23:38:00.146310+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:38:00.146310+0000","last_undegraded":"2023-09-23T23:38:00.146310+0000","last_fullsized":"2023-09-23T23:38:00.146310+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1059,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.113772+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:37:52.113772+0000","last_peered":"2023-09-23T23:37:52.113772+0000","last_clean":"2023-09-23T23:37:52.113772+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:37:52.113772+0000","last_undegraded":"2023-09-23T23:37:52.113772+0000","last_fullsized":"2023-09-23T23:37:52.113772+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":279,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.148450+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:38:00.148450+0000","last_peered":"2023-09-23T23:38:00.148450+0000","last_clean":"2023-09-23T23:38:00.148450+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:38:00.148450+0000","last_undegraded":"2023-09-23T23:38:00.148450+0000","last_fullsized":"2023-09-23T23:38:00.148450+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":138,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.117195+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:37:52.117195+0000","last_peered":"2023-09-23T23:37:52.117195+0000","last_clean":"2023-09-23T23:37:52.117195+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:37:52.117195+0000","last_undegraded":"2023-09-23T23:37:52.117195+0000","last_fullsized":"2023-09-23T23:37:52.117195+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":457,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.149027+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:38:00.149027+0000","last_peered":"2023-09-23T23:38:00.149027+0000","last_clean":"2023-09-23T23:38:00.149027+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:38:00.149027+0000","last_undegraded":"2023-09-23T23:38:00.149027+0000","last_fullsized":"2023-09-23T23:38:00.149027+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":179,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.118219+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:37:52.118219+0000","last_peered":"2023-09-23T23:37:52.118219+0000","last_clean":"2023-09-23T23:37:52.118219+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:37:52.118219+0000","last_undegraded":"2023-09-23T23:37:52.118219+0000","last_fullsized":"2023-09-23T23:37:52.118219+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":546,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.118317+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:37:52.118317+0000","last_peered":"2023-09-23T23:37:52.118317+0000","last_clean":"2023-09-23T23:37:52.118317+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:37:52.118317+0000","last_undegraded":"2023-09-23T23:37:52.118317+0000","last_fullsized":"2023-09-23T23:37:52.118317+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":760,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.115364+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:37:52.115364+0000","last_peered":"2023-09-23T23:37:52.115364+0000","last_clean":"2023-09-23T23:37:52.115364+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:37:52.115364+0000","last_undegraded":"2023-09-23T23:37:52.115364+0000","last_fullsized":"2023-09-23T23:37:52.115364+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":152,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.117887+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:37:52.117887+0000","last_peered":"2023-09-23T23:37:52.117887+0000","last_clean":"2023-09-23T23:37:52.117887+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:37:52.117887+0000","last_undegraded":"2023-09-23T23:37:52.117887+0000","last_fullsized":"2023-09-23T23:37:52.117887+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":606,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.148113+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:38:00.148113+0000","last_peered":"2023-09-23T23:38:00.148113+0000","last_clean":"2023-09-23T23:38:00.148113+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:38:00.148113+0000","last_undegraded":"2023-09-23T23:38:00.148113+0000","last_fullsized":"2023-09-23T23:38:00.148113+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":890,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.116659+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:37:52.116659+0000","last_peered":"2023-09-23T23:37:52.116659+0000","last_clean":"2023-09-23T23:37:52.116659+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:37:52.116659+0000","last_undegraded":"2023-09-23T23:37:52.116659+0000","last_fullsized":"2023-09-23T23:37:52.116659+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":201,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.150817+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:38:00.150817+0000","last_peered":"2023-09-23T23:38:00.150817+0000","last_clean":"2023-09-23T23:38:00.150817+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:38:00.150817+0000","last_undegraded":"2023-09-23T23:38:00.150817+0000","last_fullsized":"2023-09-23T23:38:00.150817+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":328,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.158316+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:38:00.158316+0000","last_peered":"2023-09-23T23:38:00.158316+0000","last_clean":"2023-09-23T23:38:00.158316+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:38:00.158316+0000","last_undegraded":"2023-09-23T23:38:00.158316+0000","last_fullsized":"2023-09-23T23:38:00.158316+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":569,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.148826+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:38:00.148826+0000","last_peered":"2023-09-23T23:38:00.148826+0000","last_clean":"2023-09-23T23:38:00.148826+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:38:00.148826+0000","last_undegraded":"2023-09-23T23:38:00.148826+0000","last_fullsized":"2023-09-23T23:38:00.148826+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":473,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.118113+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:37:52.118113+0000","last_peered":"2023-09-23T23:37:52.118113+0000","last_clean":"2023-09-23T23:37:52.118113+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:37:52.118113+0000","last_undegraded":"2023-09-23T23:37:52.118113+0000","last_fullsized":"2023-09-23T23:37:52.118113+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":563,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:03.302379+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:38:03.302379+0000","last_peered":"2023-09-23T23:38:03.302379+0000","last_clean":"2023-09-23T23:38:03.302379+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:38:03.302379+0000","last_undegraded":"2023-09-23T23:38:03.302379+0000","last_fullsized":"2023-09-23T23:38:03.302379+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":968,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.114264+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:37:52.114264+0000","last_peered":"2023-09-23T23:37:52.114264+0000","last_clean":"2023-09-23T23:37:52.114264+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:37:52.114264+0000","last_undegraded":"2023-09-23T23:37:52.114264+0000","last_fullsized":"2023-09-23T23:37:52.114264+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":169,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.116880+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:37:52.116880+0000","last_peered":"2023-09-23T23:37:52.116880+0000","last_clean":"2023-09-23T23:37:52.116880+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:37:52.116880+0000","last_undegraded":"2023-09-23T23:37:52.116880+0000","last_fullsized":"2023-09-23T23:37:52.116880+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":248,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.374866+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:38:10.374866+0000","last_peered":"2023-09-23T23:38:10.374866+0000","last_clean":"2023-09-23T23:38:10.374866+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:38:10.374866+0000","last_undegraded":"2023-09-23T23:38:10.374866+0000","last_fullsized":"2023-09-23T23:38:10.374866+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"605'9187","reported_seq":9275,"reported_epoch":605,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:08.968688+0000","last_change":"2023-09-23T23:33:52.884769+0000","last_active":"2023-09-23T23:38:08.968688+0000","last_peered":"2023-09-23T23:38:08.968688+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:38:08.968688+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"552'3800","ondisk_log_start":"552'3800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5387,"log_dups_size":0,"ondisk_log_size":5387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108855296,"num_objects":1661,"num_object_clones":0,"num_object_copies":4983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9187,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":635},{"shards":"2,6","objects":-2737}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":939,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.117104+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:37:52.117104+0000","last_peered":"2023-09-23T23:37:52.117104+0000","last_clean":"2023-09-23T23:37:52.117104+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:37:52.117104+0000","last_undegraded":"2023-09-23T23:37:52.117104+0000","last_fullsized":"2023-09-23T23:37:52.117104+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":247,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.151648+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:38:00.151648+0000","last_peered":"2023-09-23T23:38:00.151648+0000","last_clean":"2023-09-23T23:38:00.151648+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:38:00.151648+0000","last_undegraded":"2023-09-23T23:38:00.151648+0000","last_fullsized":"2023-09-23T23:38:00.151648+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":236,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.113605+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:37:52.113605+0000","last_peered":"2023-09-23T23:37:52.113605+0000","last_clean":"2023-09-23T23:37:52.113605+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:37:52.113605+0000","last_undegraded":"2023-09-23T23:37:52.113605+0000","last_fullsized":"2023-09-23T23:37:52.113605+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"605'9392","reported_seq":9480,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.037720+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:38:10.037720+0000","last_peered":"2023-09-23T23:38:10.037720+0000","last_clean":"2023-09-23T23:38:10.037720+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:38:10.037720+0000","last_undegraded":"2023-09-23T23:38:10.037720+0000","last_fullsized":"2023-09-23T23:38:10.037720+0000","mapping_epoch":566,"log_start":"551'3200","ondisk_log_start":"551'3200","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6192,"log_dups_size":0,"ondisk_log_size":6192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107479040,"num_objects":1640,"num_object_clones":0,"num_object_copies":4920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9392,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1019,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:03.295904+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:38:03.295904+0000","last_peered":"2023-09-23T23:38:03.295904+0000","last_clean":"2023-09-23T23:38:03.295904+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:38:03.295904+0000","last_undegraded":"2023-09-23T23:38:03.295904+0000","last_fullsized":"2023-09-23T23:38:03.295904+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":237,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.153219+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:38:00.153219+0000","last_peered":"2023-09-23T23:38:00.153219+0000","last_clean":"2023-09-23T23:38:00.153219+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:38:00.153219+0000","last_undegraded":"2023-09-23T23:38:00.153219+0000","last_fullsized":"2023-09-23T23:38:00.153219+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":457,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.146110+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:38:00.146110+0000","last_peered":"2023-09-23T23:38:00.146110+0000","last_clean":"2023-09-23T23:38:00.146110+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:38:00.146110+0000","last_undegraded":"2023-09-23T23:38:00.146110+0000","last_fullsized":"2023-09-23T23:38:00.146110+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"606'9541","reported_seq":18702,"reported_epoch":606,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:38:13.329013+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:38:13.329013+0000","last_peered":"2023-09-23T23:38:13.329013+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:38:13.329013+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:38:13.329013+0000","mapping_epoch":571,"log_start":"554'4200","ondisk_log_start":"554'4200","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5341,"log_dups_size":0,"ondisk_log_size":5341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98762752,"num_objects":1507,"num_object_clones":0,"num_object_copies":4521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":611,"num_objects_misplaced":896,"num_objects_unfound":0,"num_objects_dirty":1507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9541,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4520,"num_bytes_recovered":296222720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":489},{"shards":"3,5,7","objects":1018}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":915,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.119601+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:37:52.119601+0000","last_peered":"2023-09-23T23:37:52.119601+0000","last_clean":"2023-09-23T23:37:52.119601+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:37:52.119601+0000","last_undegraded":"2023-09-23T23:37:52.119601+0000","last_fullsized":"2023-09-23T23:37:52.119601+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":343,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.114016+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:37:52.114016+0000","last_peered":"2023-09-23T23:37:52.114016+0000","last_clean":"2023-09-23T23:37:52.114016+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:37:52.114016+0000","last_undegraded":"2023-09-23T23:37:52.114016+0000","last_fullsized":"2023-09-23T23:37:52.114016+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":119,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.117483+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:37:52.117483+0000","last_peered":"2023-09-23T23:37:52.117483+0000","last_clean":"2023-09-23T23:37:52.117483+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:37:52.117483+0000","last_undegraded":"2023-09-23T23:37:52.117483+0000","last_fullsized":"2023-09-23T23:37:52.117483+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"606'9600","reported_seq":11676,"reported_epoch":606,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:13.302314+0000","last_change":"2023-09-23T23:34:35.750428+0000","last_active":"2023-09-23T23:38:13.302314+0000","last_peered":"2023-09-23T23:38:13.302314+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:38:13.302314+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"555'4300","ondisk_log_start":"555'4300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5300,"log_dups_size":0,"ondisk_log_size":5300,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97648640,"num_objects":1490,"num_object_clones":0,"num_object_copies":4470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9600,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":991,"num_bytes_recovered":64946176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":480},{"shards":"3,5","objects":-2054}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":513,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.149107+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:38:00.149107+0000","last_peered":"2023-09-23T23:38:00.149107+0000","last_clean":"2023-09-23T23:38:00.149107+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:38:00.149107+0000","last_undegraded":"2023-09-23T23:38:00.149107+0000","last_fullsized":"2023-09-23T23:38:00.149107+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":447,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.147993+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:38:00.147993+0000","last_peered":"2023-09-23T23:38:00.147993+0000","last_clean":"2023-09-23T23:38:00.147993+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:38:00.147993+0000","last_undegraded":"2023-09-23T23:38:00.147993+0000","last_fullsized":"2023-09-23T23:38:00.147993+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":604,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.143385+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:38:00.143385+0000","last_peered":"2023-09-23T23:38:00.143385+0000","last_clean":"2023-09-23T23:38:00.143385+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:38:00.143385+0000","last_undegraded":"2023-09-23T23:38:00.143385+0000","last_fullsized":"2023-09-23T23:38:00.143385+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"605'9476","reported_seq":10811,"reported_epoch":605,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:09.463009+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:38:09.463009+0000","last_peered":"2023-09-23T23:38:09.463009+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:38:09.463009+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"552'3800","ondisk_log_start":"552'3800","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5676,"log_dups_size":0,"ondisk_log_size":5676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110493696,"num_objects":1686,"num_object_clones":0,"num_object_copies":5058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9476,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":640},{"shards":"3,5","objects":-3774},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":734,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.146271+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:38:00.146271+0000","last_peered":"2023-09-23T23:38:00.146271+0000","last_clean":"2023-09-23T23:38:00.146271+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:38:00.146271+0000","last_undegraded":"2023-09-23T23:38:00.146271+0000","last_fullsized":"2023-09-23T23:38:00.146271+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":617,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.148733+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:38:00.148733+0000","last_peered":"2023-09-23T23:38:00.148733+0000","last_clean":"2023-09-23T23:38:00.148733+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:38:00.148733+0000","last_undegraded":"2023-09-23T23:38:00.148733+0000","last_fullsized":"2023-09-23T23:38:00.148733+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":527,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.148625+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:38:00.148625+0000","last_peered":"2023-09-23T23:38:00.148625+0000","last_clean":"2023-09-23T23:38:00.148625+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:38:00.148625+0000","last_undegraded":"2023-09-23T23:38:00.148625+0000","last_fullsized":"2023-09-23T23:38:00.148625+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"605'9472","reported_seq":9551,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:09.009720+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:38:09.009720+0000","last_peered":"2023-09-23T23:38:09.009720+0000","last_clean":"2023-09-23T23:38:09.009720+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:38:09.009720+0000","last_undegraded":"2023-09-23T23:38:09.009720+0000","last_fullsized":"2023-09-23T23:38:09.009720+0000","mapping_epoch":534,"log_start":"552'4100","ondisk_log_start":"552'4100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5372,"log_dups_size":0,"ondisk_log_size":5372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105381888,"num_objects":1608,"num_object_clones":0,"num_object_copies":4824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9472,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":889,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.117129+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:37:52.117129+0000","last_peered":"2023-09-23T23:37:52.117129+0000","last_clean":"2023-09-23T23:37:52.117129+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:37:52.117129+0000","last_undegraded":"2023-09-23T23:37:52.117129+0000","last_fullsized":"2023-09-23T23:37:52.117129+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":129,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.117410+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:37:52.117410+0000","last_peered":"2023-09-23T23:37:52.117410+0000","last_clean":"2023-09-23T23:37:52.117410+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:37:52.117410+0000","last_undegraded":"2023-09-23T23:37:52.117410+0000","last_fullsized":"2023-09-23T23:37:52.117410+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":471,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.117297+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:37:52.117297+0000","last_peered":"2023-09-23T23:37:52.117297+0000","last_clean":"2023-09-23T23:37:52.117297+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:37:52.117297+0000","last_undegraded":"2023-09-23T23:37:52.117297+0000","last_fullsized":"2023-09-23T23:37:52.117297+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"605'9429","reported_seq":17284,"reported_epoch":605,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:38:09.000101+0000","last_change":"2023-09-23T23:34:35.785270+0000","last_active":"2023-09-23T23:38:09.000101+0000","last_peered":"2023-09-23T23:38:09.000101+0000","last_clean":"2023-09-23T23:33:50.504296+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:38:09.000101+0000","last_undegraded":"2023-09-23T23:34:35.780981+0000","last_fullsized":"2023-09-23T23:38:09.000101+0000","mapping_epoch":567,"log_start":"552'4000","ondisk_log_start":"552'4000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5429,"log_dups_size":0,"ondisk_log_size":5429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109772800,"num_objects":1675,"num_object_clones":0,"num_object_copies":5025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":669,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9429,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3878,"num_bytes_recovered":254148608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":669},{"shards":"5,6,7","objects":1006}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":613,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.113720+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:37:52.113720+0000","last_peered":"2023-09-23T23:37:52.113720+0000","last_clean":"2023-09-23T23:37:52.113720+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:37:52.113720+0000","last_undegraded":"2023-09-23T23:37:52.113720+0000","last_fullsized":"2023-09-23T23:37:52.113720+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":468,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.119447+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:37:52.119447+0000","last_peered":"2023-09-23T23:37:52.119447+0000","last_clean":"2023-09-23T23:37:52.119447+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:37:52.119447+0000","last_undegraded":"2023-09-23T23:37:52.119447+0000","last_fullsized":"2023-09-23T23:37:52.119447+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":556,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.117288+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:37:52.117288+0000","last_peered":"2023-09-23T23:37:52.117288+0000","last_clean":"2023-09-23T23:37:52.117288+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:37:52.117288+0000","last_undegraded":"2023-09-23T23:37:52.117288+0000","last_fullsized":"2023-09-23T23:37:52.117288+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"605'9340","reported_seq":18382,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:09.420346+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:38:09.420346+0000","last_peered":"2023-09-23T23:38:09.420346+0000","last_clean":"2023-09-23T23:38:09.420346+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:38:09.420346+0000","last_undegraded":"2023-09-23T23:38:09.420346+0000","last_fullsized":"2023-09-23T23:38:09.420346+0000","mapping_epoch":579,"log_start":"552'3600","ondisk_log_start":"552'3600","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5740,"log_dups_size":0,"ondisk_log_size":5740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109838336,"num_objects":1676,"num_object_clones":0,"num_object_copies":5028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9340,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":947,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.118533+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:37:52.118533+0000","last_peered":"2023-09-23T23:37:52.118533+0000","last_clean":"2023-09-23T23:37:52.118533+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:37:52.118533+0000","last_undegraded":"2023-09-23T23:37:52.118533+0000","last_fullsized":"2023-09-23T23:37:52.118533+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":248,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.145888+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:38:00.145888+0000","last_peered":"2023-09-23T23:38:00.145888+0000","last_clean":"2023-09-23T23:38:00.145888+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:38:00.145888+0000","last_undegraded":"2023-09-23T23:38:00.145888+0000","last_fullsized":"2023-09-23T23:38:00.145888+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":536,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.148943+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:38:00.148943+0000","last_peered":"2023-09-23T23:38:00.148943+0000","last_clean":"2023-09-23T23:38:00.148943+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:38:00.148943+0000","last_undegraded":"2023-09-23T23:38:00.148943+0000","last_fullsized":"2023-09-23T23:38:00.148943+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"606'9211","reported_seq":9294,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:13.320749+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:38:13.320749+0000","last_peered":"2023-09-23T23:38:13.320749+0000","last_clean":"2023-09-23T23:38:13.320749+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:38:13.320749+0000","last_undegraded":"2023-09-23T23:38:13.320749+0000","last_fullsized":"2023-09-23T23:38:13.320749+0000","mapping_epoch":534,"log_start":"553'3900","ondisk_log_start":"553'3900","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5311,"log_dups_size":0,"ondisk_log_size":5311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99418112,"num_objects":1517,"num_object_clones":0,"num_object_copies":4551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9211,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":926,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.119241+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:37:52.119241+0000","last_peered":"2023-09-23T23:37:52.119241+0000","last_clean":"2023-09-23T23:37:52.119241+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:37:52.119241+0000","last_undegraded":"2023-09-23T23:37:52.119241+0000","last_fullsized":"2023-09-23T23:37:52.119241+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":316,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.162122+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:38:00.162122+0000","last_peered":"2023-09-23T23:38:00.162122+0000","last_clean":"2023-09-23T23:38:00.162122+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:38:00.162122+0000","last_undegraded":"2023-09-23T23:38:00.162122+0000","last_fullsized":"2023-09-23T23:38:00.162122+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":620,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.114148+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:37:52.114148+0000","last_peered":"2023-09-23T23:37:52.114148+0000","last_clean":"2023-09-23T23:37:52.114148+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:37:52.114148+0000","last_undegraded":"2023-09-23T23:37:52.114148+0000","last_fullsized":"2023-09-23T23:37:52.114148+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":201,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.113928+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:37:52.113928+0000","last_peered":"2023-09-23T23:37:52.113928+0000","last_clean":"2023-09-23T23:37:52.113928+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:37:52.113928+0000","last_undegraded":"2023-09-23T23:37:52.113928+0000","last_fullsized":"2023-09-23T23:37:52.113928+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"606'9599","reported_seq":17823,"reported_epoch":606,"state":"active+recovering","last_fresh":"2023-09-23T23:38:13.330541+0000","last_change":"2023-09-23T23:34:35.815566+0000","last_active":"2023-09-23T23:38:13.330541+0000","last_peered":"2023-09-23T23:38:13.330541+0000","last_clean":"2023-09-23T23:33:50.519308+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:38:13.330541+0000","last_undegraded":"2023-09-23T23:38:13.330541+0000","last_fullsized":"2023-09-23T23:38:13.330541+0000","mapping_epoch":566,"log_start":"555'4322","ondisk_log_start":"555'4322","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5277,"log_dups_size":0,"ondisk_log_size":5277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102760472,"num_objects":1569,"num_object_clones":0,"num_object_copies":4707,"num_objects_missing_on_primary":454,"num_objects_missing":454,"num_objects_degraded":0,"num_objects_misplaced":454,"num_objects_unfound":0,"num_objects_dirty":1569,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9599,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4060,"num_bytes_recovered":266076160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,3,4,7","objects":1},{"shards":"1,3,7","objects":1114},{"shards":"1,4,7","objects":454}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":690,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.143500+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:38:00.143500+0000","last_peered":"2023-09-23T23:38:00.143500+0000","last_clean":"2023-09-23T23:38:00.143500+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:38:00.143500+0000","last_undegraded":"2023-09-23T23:38:00.143500+0000","last_fullsized":"2023-09-23T23:38:00.143500+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":253,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.117306+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:37:52.117306+0000","last_peered":"2023-09-23T23:37:52.117306+0000","last_clean":"2023-09-23T23:37:52.117306+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:37:52.117306+0000","last_undegraded":"2023-09-23T23:37:52.117306+0000","last_fullsized":"2023-09-23T23:37:52.117306+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":537,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.118371+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:37:52.118371+0000","last_peered":"2023-09-23T23:37:52.118371+0000","last_clean":"2023-09-23T23:37:52.118371+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:37:52.118371+0000","last_undegraded":"2023-09-23T23:37:52.118371+0000","last_fullsized":"2023-09-23T23:37:52.118371+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":238,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.118048+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:37:52.118048+0000","last_peered":"2023-09-23T23:37:52.118048+0000","last_clean":"2023-09-23T23:37:52.118048+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:37:52.118048+0000","last_undegraded":"2023-09-23T23:37:52.118048+0000","last_fullsized":"2023-09-23T23:37:52.118048+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"605'9132","reported_seq":11190,"reported_epoch":605,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:09.444254+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:38:09.444254+0000","last_peered":"2023-09-23T23:38:09.444254+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:38:09.444254+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"552'3400","ondisk_log_start":"552'3400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5732,"log_dups_size":0,"ondisk_log_size":5732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107479040,"num_objects":1640,"num_object_clones":0,"num_object_copies":4920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9132,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":629},{"shards":"5,7","objects":-1752}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":907,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.118448+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:37:52.118448+0000","last_peered":"2023-09-23T23:37:52.118448+0000","last_clean":"2023-09-23T23:37:52.118448+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:37:52.118448+0000","last_undegraded":"2023-09-23T23:37:52.118448+0000","last_fullsized":"2023-09-23T23:37:52.118448+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":344,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.119360+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:37:52.119360+0000","last_peered":"2023-09-23T23:37:52.119360+0000","last_clean":"2023-09-23T23:37:52.119360+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:37:52.119360+0000","last_undegraded":"2023-09-23T23:37:52.119360+0000","last_fullsized":"2023-09-23T23:37:52.119360+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":254,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.155446+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:38:00.155446+0000","last_peered":"2023-09-23T23:38:00.155446+0000","last_clean":"2023-09-23T23:38:00.155446+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:38:00.155446+0000","last_undegraded":"2023-09-23T23:38:00.155446+0000","last_fullsized":"2023-09-23T23:38:00.155446+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":219,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.354947+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:38:10.354947+0000","last_peered":"2023-09-23T23:38:10.354947+0000","last_clean":"2023-09-23T23:38:10.354947+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:38:10.354947+0000","last_undegraded":"2023-09-23T23:38:10.354947+0000","last_fullsized":"2023-09-23T23:38:10.354947+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"605'9298","reported_seq":9387,"reported_epoch":605,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:38:09.521009+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:38:09.521009+0000","last_peered":"2023-09-23T23:38:09.521009+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:38:09.521009+0000","last_undegraded":"2023-09-23T23:38:09.521009+0000","last_fullsized":"2023-09-23T23:38:09.521009+0000","mapping_epoch":562,"log_start":"552'3600","ondisk_log_start":"552'3600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5698,"log_dups_size":0,"ondisk_log_size":5698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105119744,"num_objects":1604,"num_object_clones":0,"num_object_copies":4812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9298,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":1604}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":621,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.159989+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:38:00.159989+0000","last_peered":"2023-09-23T23:38:00.159989+0000","last_clean":"2023-09-23T23:38:00.159989+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:38:00.159989+0000","last_undegraded":"2023-09-23T23:38:00.159989+0000","last_fullsized":"2023-09-23T23:38:00.159989+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":891,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.116715+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:37:52.116715+0000","last_peered":"2023-09-23T23:37:52.116715+0000","last_clean":"2023-09-23T23:37:52.116715+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:37:52.116715+0000","last_undegraded":"2023-09-23T23:37:52.116715+0000","last_fullsized":"2023-09-23T23:37:52.116715+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":569,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:03.294440+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:38:03.294440+0000","last_peered":"2023-09-23T23:38:03.294440+0000","last_clean":"2023-09-23T23:38:03.294440+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:38:03.294440+0000","last_undegraded":"2023-09-23T23:38:03.294440+0000","last_fullsized":"2023-09-23T23:38:03.294440+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":129,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.117308+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:37:52.117308+0000","last_peered":"2023-09-23T23:37:52.117308+0000","last_clean":"2023-09-23T23:37:52.117308+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:37:52.117308+0000","last_undegraded":"2023-09-23T23:37:52.117308+0000","last_fullsized":"2023-09-23T23:37:52.117308+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"605'9052","reported_seq":9131,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:08.941368+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:38:08.941368+0000","last_peered":"2023-09-23T23:38:08.941368+0000","last_clean":"2023-09-23T23:38:08.941368+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:38:08.941368+0000","last_undegraded":"2023-09-23T23:38:08.941368+0000","last_fullsized":"2023-09-23T23:38:08.941368+0000","mapping_epoch":534,"log_start":"552'3600","ondisk_log_start":"552'3600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5452,"log_dups_size":0,"ondisk_log_size":5452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110362624,"num_objects":1684,"num_object_clones":0,"num_object_copies":5052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1684,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9052,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":717,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.118299+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:37:52.118299+0000","last_peered":"2023-09-23T23:37:52.118299+0000","last_clean":"2023-09-23T23:37:52.118299+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:37:52.118299+0000","last_undegraded":"2023-09-23T23:37:52.118299+0000","last_fullsized":"2023-09-23T23:37:52.118299+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":533,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.151945+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:38:00.151945+0000","last_peered":"2023-09-23T23:38:00.151945+0000","last_clean":"2023-09-23T23:38:00.151945+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:38:00.151945+0000","last_undegraded":"2023-09-23T23:38:00.151945+0000","last_fullsized":"2023-09-23T23:38:00.151945+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":448,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.146152+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:38:00.146152+0000","last_peered":"2023-09-23T23:38:00.146152+0000","last_clean":"2023-09-23T23:38:00.146152+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:38:00.146152+0000","last_undegraded":"2023-09-23T23:38:00.146152+0000","last_fullsized":"2023-09-23T23:38:00.146152+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":47,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.144347+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:38:00.144347+0000","last_peered":"2023-09-23T23:38:00.144347+0000","last_clean":"2023-09-23T23:38:00.144347+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:38:00.144347+0000","last_undegraded":"2023-09-23T23:38:00.144347+0000","last_fullsized":"2023-09-23T23:38:00.144347+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"605'9409","reported_seq":13088,"reported_epoch":605,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:10.102260+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:38:10.102260+0000","last_peered":"2023-09-23T23:38:10.102260+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:38:10.102260+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"551'3200","ondisk_log_start":"551'3200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6209,"log_dups_size":0,"ondisk_log_size":6209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104136704,"num_objects":1589,"num_object_clones":0,"num_object_copies":4767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9409,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":-1115},{"shards":"1,2,4","objects":587}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":600,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.117484+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:37:52.117484+0000","last_peered":"2023-09-23T23:37:52.117484+0000","last_clean":"2023-09-23T23:37:52.117484+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:37:52.117484+0000","last_undegraded":"2023-09-23T23:37:52.117484+0000","last_fullsized":"2023-09-23T23:37:52.117484+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":47,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.147730+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:38:00.147730+0000","last_peered":"2023-09-23T23:38:00.147730+0000","last_clean":"2023-09-23T23:38:00.147730+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:38:00.147730+0000","last_undegraded":"2023-09-23T23:38:00.147730+0000","last_fullsized":"2023-09-23T23:38:00.147730+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":145,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.118149+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:37:52.118149+0000","last_peered":"2023-09-23T23:37:52.118149+0000","last_clean":"2023-09-23T23:37:52.118149+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:37:52.118149+0000","last_undegraded":"2023-09-23T23:37:52.118149+0000","last_fullsized":"2023-09-23T23:37:52.118149+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":338,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.113869+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:37:52.113869+0000","last_peered":"2023-09-23T23:37:52.113869+0000","last_clean":"2023-09-23T23:37:52.113869+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:37:52.113869+0000","last_undegraded":"2023-09-23T23:37:52.113869+0000","last_fullsized":"2023-09-23T23:37:52.113869+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"605'9346","reported_seq":9425,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:09.513536+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:38:09.513536+0000","last_peered":"2023-09-23T23:38:09.513536+0000","last_clean":"2023-09-23T23:38:09.513536+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:38:09.513536+0000","last_undegraded":"2023-09-23T23:38:09.513536+0000","last_fullsized":"2023-09-23T23:38:09.513536+0000","mapping_epoch":534,"log_start":"552'3600","ondisk_log_start":"552'3600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5746,"log_dups_size":0,"ondisk_log_size":5746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106037248,"num_objects":1618,"num_object_clones":0,"num_object_copies":4854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9346,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":787,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.160615+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:38:00.160615+0000","last_peered":"2023-09-23T23:38:00.160615+0000","last_clean":"2023-09-23T23:38:00.160615+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:38:00.160615+0000","last_undegraded":"2023-09-23T23:38:00.160615+0000","last_fullsized":"2023-09-23T23:38:00.160615+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":604,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.148866+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:38:00.148866+0000","last_peered":"2023-09-23T23:38:00.148866+0000","last_clean":"2023-09-23T23:38:00.148866+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:38:00.148866+0000","last_undegraded":"2023-09-23T23:38:00.148866+0000","last_fullsized":"2023-09-23T23:38:00.148866+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":209,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.117342+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:37:52.117342+0000","last_peered":"2023-09-23T23:37:52.117342+0000","last_clean":"2023-09-23T23:37:52.117342+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:37:52.117342+0000","last_undegraded":"2023-09-23T23:37:52.117342+0000","last_fullsized":"2023-09-23T23:37:52.117342+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":157,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.116992+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:37:52.116992+0000","last_peered":"2023-09-23T23:37:52.116992+0000","last_clean":"2023-09-23T23:37:52.116992+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:37:52.116992+0000","last_undegraded":"2023-09-23T23:37:52.116992+0000","last_fullsized":"2023-09-23T23:37:52.116992+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"605'9208","reported_seq":9290,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.103078+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:38:10.103078+0000","last_peered":"2023-09-23T23:38:10.103078+0000","last_clean":"2023-09-23T23:38:10.103078+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:38:10.103078+0000","last_undegraded":"2023-09-23T23:38:10.103078+0000","last_fullsized":"2023-09-23T23:38:10.103078+0000","mapping_epoch":534,"log_start":"551'3000","ondisk_log_start":"551'3000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6208,"log_dups_size":0,"ondisk_log_size":6208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111411200,"num_objects":1700,"num_object_clones":0,"num_object_copies":5100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9208,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":994,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.148229+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:38:00.148229+0000","last_peered":"2023-09-23T23:38:00.148229+0000","last_clean":"2023-09-23T23:38:00.148229+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:38:00.148229+0000","last_undegraded":"2023-09-23T23:38:00.148229+0000","last_fullsized":"2023-09-23T23:38:00.148229+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":119,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.119337+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:37:52.119337+0000","last_peered":"2023-09-23T23:37:52.119337+0000","last_clean":"2023-09-23T23:37:52.119337+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:37:52.119337+0000","last_undegraded":"2023-09-23T23:37:52.119337+0000","last_fullsized":"2023-09-23T23:37:52.119337+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":51,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.144410+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:38:00.144410+0000","last_peered":"2023-09-23T23:38:00.144410+0000","last_clean":"2023-09-23T23:38:00.144410+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:38:00.144410+0000","last_undegraded":"2023-09-23T23:38:00.144410+0000","last_fullsized":"2023-09-23T23:38:00.144410+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":179,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.115445+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:37:52.115445+0000","last_peered":"2023-09-23T23:37:52.115445+0000","last_clean":"2023-09-23T23:37:52.115445+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:37:52.115445+0000","last_undegraded":"2023-09-23T23:37:52.115445+0000","last_fullsized":"2023-09-23T23:37:52.115445+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1043,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.146225+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:38:00.146225+0000","last_peered":"2023-09-23T23:38:00.146225+0000","last_clean":"2023-09-23T23:38:00.146225+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:38:00.146225+0000","last_undegraded":"2023-09-23T23:38:00.146225+0000","last_fullsized":"2023-09-23T23:38:00.146225+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":47,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.143471+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:38:00.143471+0000","last_peered":"2023-09-23T23:38:00.143471+0000","last_clean":"2023-09-23T23:38:00.143471+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:38:00.143471+0000","last_undegraded":"2023-09-23T23:38:00.143471+0000","last_fullsized":"2023-09-23T23:38:00.143471+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":361,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.118240+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:37:52.118240+0000","last_peered":"2023-09-23T23:37:52.118240+0000","last_clean":"2023-09-23T23:37:52.118240+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:37:52.118240+0000","last_undegraded":"2023-09-23T23:37:52.118240+0000","last_fullsized":"2023-09-23T23:37:52.118240+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":755,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.149350+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:38:00.149350+0000","last_peered":"2023-09-23T23:38:00.149350+0000","last_clean":"2023-09-23T23:38:00.149350+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:38:00.149350+0000","last_undegraded":"2023-09-23T23:38:00.149350+0000","last_fullsized":"2023-09-23T23:38:00.149350+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":107,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.117963+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:37:52.117963+0000","last_peered":"2023-09-23T23:37:52.117963+0000","last_clean":"2023-09-23T23:37:52.117963+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:37:52.117963+0000","last_undegraded":"2023-09-23T23:37:52.117963+0000","last_fullsized":"2023-09-23T23:37:52.117963+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":454,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.146821+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:38:00.146821+0000","last_peered":"2023-09-23T23:38:00.146821+0000","last_clean":"2023-09-23T23:38:00.146821+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:38:00.146821+0000","last_undegraded":"2023-09-23T23:38:00.146821+0000","last_fullsized":"2023-09-23T23:38:00.146821+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":303,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.117155+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:37:52.117155+0000","last_peered":"2023-09-23T23:37:52.117155+0000","last_clean":"2023-09-23T23:37:52.117155+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:37:52.117155+0000","last_undegraded":"2023-09-23T23:37:52.117155+0000","last_fullsized":"2023-09-23T23:37:52.117155+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":293,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.150773+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:38:00.150773+0000","last_peered":"2023-09-23T23:38:00.150773+0000","last_clean":"2023-09-23T23:38:00.150773+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:38:00.150773+0000","last_undegraded":"2023-09-23T23:38:00.150773+0000","last_fullsized":"2023-09-23T23:38:00.150773+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":817,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.118306+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:37:52.118306+0000","last_peered":"2023-09-23T23:37:52.118306+0000","last_clean":"2023-09-23T23:37:52.118306+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:37:52.118306+0000","last_undegraded":"2023-09-23T23:37:52.118306+0000","last_fullsized":"2023-09-23T23:37:52.118306+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":637,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.149267+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:38:00.149267+0000","last_peered":"2023-09-23T23:38:00.149267+0000","last_clean":"2023-09-23T23:38:00.149267+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:38:00.149267+0000","last_undegraded":"2023-09-23T23:38:00.149267+0000","last_fullsized":"2023-09-23T23:38:00.149267+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":621,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.146386+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:38:00.146386+0000","last_peered":"2023-09-23T23:38:00.146386+0000","last_clean":"2023-09-23T23:38:00.146386+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:38:00.146386+0000","last_undegraded":"2023-09-23T23:38:00.146386+0000","last_fullsized":"2023-09-23T23:38:00.146386+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":779,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T23:38:00.148357+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:38:00.148357+0000","last_peered":"2023-09-23T23:38:00.148357+0000","last_clean":"2023-09-23T23:38:00.148357+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:38:00.148357+0000","last_undegraded":"2023-09-23T23:38:00.148357+0000","last_fullsized":"2023-09-23T23:38:00.148357+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":449,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.119231+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:37:52.119231+0000","last_peered":"2023-09-23T23:37:52.119231+0000","last_clean":"2023-09-23T23:37:52.119231+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:37:52.119231+0000","last_undegraded":"2023-09-23T23:37:52.119231+0000","last_fullsized":"2023-09-23T23:37:52.119231+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":672,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T23:37:52.116631+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:37:52.116631+0000","last_peered":"2023-09-23T23:37:52.116631+0000","last_clean":"2023-09-23T23:37:52.116631+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:37:52.116631+0000","last_undegraded":"2023-09-23T23:37:52.116631+0000","last_fullsized":"2023-09-23T23:37:52.116631+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":1694957592,"num_objects":25864,"num_object_clones":0,"num_object_copies":77592,"num_objects_missing_on_primary":454,"num_objects_missing":454,"num_objects_degraded":20896,"num_objects_misplaced":1350,"num_objects_unfound":0,"num_objects_dirty":25864,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":149692,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21306,"num_bytes_recovered":1396113408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7076327424,"data_stored":7076315208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90070,"ondisk_log_size":90070,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782705,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1091576,"kb_used_data":694896,"kb_used_omap":8,"kb_used_meta":396663,"kb_avail":92653576,"statfs":{"total":95995035648,"available":94877261824,"internally_reserved":0,"allocated":711573504,"data_stored":709876624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":406182981},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.81599999999999995,"15min":2.0459999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":2.4159999999999999,"5min":11.173,"15min":85.721000000000004},"last":0.372},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.77300000000000002,"15min":2.145},"min":{"1min":0.24199999999999999,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":2.5499999999999998,"5min":5.2530000000000001,"15min":85.965000000000003},"last":1.3320000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":2.1549999999999998,"15min":3.105},"min":{"1min":0.255,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":2.6739999999999999,"5min":129.11199999999999,"15min":198.58000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.61299999999999999,"5min":2.036,"15min":3.0870000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.2850000000000001,"5min":129.02600000000001,"15min":198.71100000000001},"last":0.34499999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":2.4180000000000001,"15min":3.7330000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":3.3860000000000001,"5min":78.528999999999996,"15min":203.58000000000001},"last":0.433},{"interface":"front","average":{"1min":1.141,"5min":1.712,"15min":3.2200000000000002},"min":{"1min":0.30399999999999999,"5min":0.27900000000000003,"15min":0.26200000000000001},"max":{"1min":6.0700000000000003,"5min":53.023000000000003,"15min":198.69},"last":4.9340000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":14.901999999999999,"5min":7.1070000000000002,"15min":4.9690000000000003},"min":{"1min":0.33000000000000002,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":154.18100000000001,"5min":154.18100000000001,"15min":279.26499999999999},"last":0.71399999999999997},{"interface":"front","average":{"1min":14.157999999999999,"5min":7.3140000000000001,"15min":5.476},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":154.16300000000001,"5min":154.16300000000001,"15min":279.29899999999998},"last":0.66300000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.464,"15min":1.026},"min":{"1min":0.318,"5min":0.27100000000000002,"15min":0.246},"max":{"1min":2.0499999999999998,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.339,"15min":0.97399999999999998},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.23899999999999999},"max":{"1min":5.391,"5min":31.68,"15min":31.68},"last":0.41199999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":7.1280000000000001,"5min":2.7919999999999998,"15min":2.266},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.26500000000000001},"max":{"1min":119.417,"5min":119.417,"15min":210.83600000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":7.6669999999999998,"5min":3.016,"15min":2.1989999999999998},"min":{"1min":0.33600000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":119.61499999999999,"5min":119.61499999999999,"15min":207.04599999999999},"last":0.48399999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":1.8899999999999999,"15min":1.3420000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":6.2169999999999996,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.63,"5min":1.956,"15min":1.514},"min":{"1min":0.40100000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.911,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.496}]}]},{"osd":6,"up_from":46,"seq":197568496181,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1394140,"kb_used_data":674004,"kb_used_omap":0,"kb_used_meta":720128,"kb_avail":92351012,"statfs":{"total":95995035648,"available":94567436288,"internally_reserved":0,"allocated":690180096,"data_stored":688503072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":737411072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.73499999999999999,"15min":4.4370000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.21099999999999999},"max":{"1min":0.53700000000000003,"5min":6.4450000000000003,"15min":135.31999999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.498,"5min":0.63800000000000001,"15min":4.3869999999999996},"min":{"1min":0.27800000000000002,"5min":0.24399999999999999,"15min":0.23000000000000001},"max":{"1min":1.742,"5min":3.8239999999999998,"15min":135.34299999999999},"last":0.44700000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.90500000000000003,"15min":2.2890000000000001},"min":{"1min":0.28299999999999997,"5min":0.253,"15min":0.245},"max":{"1min":1.8759999999999999,"5min":6.9080000000000004,"15min":89.605000000000004},"last":1.3160000000000001},{"interface":"front","average":{"1min":0.749,"5min":0.72499999999999998,"15min":2.391},"min":{"1min":0.29099999999999998,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":3.8980000000000001,"5min":3.8980000000000001,"15min":91.025999999999996},"last":0.50600000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:37:06 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":3.6930000000000001,"15min":4.4749999999999996},"min":{"1min":0.27900000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.1800000000000002,"5min":67.796999999999997,"15min":127.44499999999999},"last":2.9020000000000001},{"interface":"front","average":{"1min":0.66600000000000004,"5min":3.6749999999999998,"15min":4.8680000000000003},"min":{"1min":0.316,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.2080000000000002,"5min":67.893000000000001,"15min":125.539},"last":0.52700000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:37:57 2023","interfaces":[{"interface":"back","average":{"1min":13.925000000000001,"5min":8.1059999999999999,"15min":5.2009999999999996},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":199.536,"5min":199.536,"15min":199.536},"last":0.60399999999999998},{"interface":"front","average":{"1min":13.538,"5min":7.9660000000000002,"15min":5.4530000000000003},"min":{"1min":0.36799999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":199.59999999999999,"5min":199.59999999999999,"15min":199.59999999999999},"last":0.83199999999999996}]},{"osd":4,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.755,"15min":0.66700000000000004},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.23400000000000001},"max":{"1min":1.76,"5min":7.101,"15min":8.1999999999999993},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.81399999999999995,"15min":0.76900000000000002},"min":{"1min":0.26700000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.865,"5min":8.3499999999999996,"15min":8.3499999999999996},"last":0.63}]},{"osd":5,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.76100000000000001,"15min":0.79800000000000004},"min":{"1min":0.27700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.847,"5min":11.372,"15min":11.473000000000001},"last":2.6269999999999998},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.876,"15min":0.84999999999999998},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.26900000000000002},"max":{"1min":6.1500000000000004,"5min":7.0129999999999999,"15min":7.5670000000000002},"last":7.5549999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:37:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.1679999999999999,"15min":1.0029999999999999},"min":{"1min":0.32700000000000001,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":0.77000000000000002,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.204,"15min":1.056},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33500000000000002},"max":{"1min":1.792,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.68100000000000005}]}]},{"osd":1,"up_from":112,"seq":481036337523,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1399548,"kb_used_data":494708,"kb_used_omap":4,"kb_used_meta":904827,"kb_avail":92345604,"statfs":{"total":95995035648,"available":94561898496,"internally_reserved":0,"allocated":506580992,"data_stored":503713841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":926543758},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.407,"15min":3.173},"min":{"1min":0.19,"5min":0.19,"15min":0.16700000000000001},"max":{"1min":3.5830000000000002,"5min":47.195999999999998,"15min":130.768},"last":7.1070000000000002},{"interface":"front","average":{"1min":0.377,"5min":1.119,"15min":2.5030000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19},"max":{"1min":1.143,"5min":46.582000000000001,"15min":134.929},"last":1.1140000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:38:05 2023","interfaces":[{"interface":"back","average":{"1min":3.2170000000000001,"5min":3.8460000000000001,"15min":6.9219999999999997},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":53.951999999999998,"5min":201.03999999999999,"15min":231.67400000000001},"last":7.0549999999999997},{"interface":"front","average":{"1min":3.6440000000000001,"5min":3.9420000000000002,"15min":7.0069999999999997},"min":{"1min":0.221,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":54.213000000000001,"5min":202.12,"15min":232.21000000000001},"last":17.902000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:38:05 2023","interfaces":[{"interface":"back","average":{"1min":7.2720000000000002,"5min":6.9420000000000002,"15min":5.8620000000000001},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22600000000000001},"max":{"1min":77.034999999999997,"5min":181.94399999999999,"15min":234.404},"last":0.44600000000000001},{"interface":"front","average":{"1min":7.5010000000000003,"5min":7.0190000000000001,"15min":5.8150000000000004},"min":{"1min":0.245,"5min":0.224,"15min":0.224},"max":{"1min":77.165999999999997,"5min":181.98500000000001,"15min":236.33699999999999},"last":9.5600000000000005}]},{"osd":4,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":0.95099999999999996,"15min":1.399},"min":{"1min":0.32500000000000001,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":4.4880000000000004,"5min":8.0850000000000009,"15min":12.042999999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.85699999999999998,"15min":1.45},"min":{"1min":0.316,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":4.6660000000000004,"5min":9.5069999999999997,"15min":18.108000000000001},"last":7.1779999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":2.6309999999999998,"15min":1.734},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28899999999999998},"max":{"1min":0.98199999999999998,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.503},{"interface":"front","average":{"1min":1.024,"5min":2.661,"15min":2.0710000000000002},"min":{"1min":0.34399999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.194,"5min":151.821,"15min":151.821},"last":0.60599999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.0449999999999999,"15min":1.6659999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.27500000000000002},"max":{"1min":2.306,"5min":8.9269999999999996,"15min":74.087999999999994},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.2849999999999999,"15min":1.929},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33100000000000002},"max":{"1min":3.29,"5min":8.1400000000000006,"15min":73.974999999999994},"last":1.988}]},{"osd":7,"last update":"Sat Sep 23 23:38:05 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":1.4590000000000001,"15min":2.101},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.7639999999999998,"5min":44.241,"15min":78.308000000000007},"last":7.1470000000000002},{"interface":"front","average":{"1min":1.657,"5min":1.514,"15min":2.113},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32700000000000001},"max":{"1min":5.7969999999999997,"5min":43.920999999999999,"15min":77.204999999999998},"last":0.57299999999999995}]}]},{"osd":0,"up_from":91,"seq":390842024346,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1401536,"kb_used_data":173240,"kb_used_omap":5,"kb_used_meta":1228282,"kb_avail":92343616,"statfs":{"total":95995035648,"available":94559862784,"internally_reserved":0,"allocated":177397760,"data_stored":174520636,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1257761402},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:37:39 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":4.0620000000000003,"15min":6.4610000000000003},"min":{"1min":0.23200000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":8.4350000000000005,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.39600000000000002,"5min":3.718,"15min":5.6509999999999998},"min":{"1min":0.219,"5min":0.20200000000000001,"15min":0.17599999999999999},"max":{"1min":1.1890000000000001,"5min":297.72399999999999,"15min":297.72399999999999},"last":3.6869999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":6.702,"15min":6.5780000000000003},"min":{"1min":0.214,"5min":0.214,"15min":0.17199999999999999},"max":{"1min":5.4340000000000002,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.77900000000000003,"5min":6.4569999999999999,"15min":6.5339999999999998},"min":{"1min":0.22600000000000001,"5min":0.20799999999999999,"15min":0.17599999999999999},"max":{"1min":3.4470000000000001,"5min":283.161,"15min":283.161},"last":3.6600000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:37:39 2023","interfaces":[{"interface":"back","average":{"1min":16.741,"5min":6.8769999999999998,"15min":6.4880000000000004},"min":{"1min":0.27300000000000002,"5min":0.219,"15min":0.217},"max":{"1min":194.26300000000001,"5min":194.26300000000001,"15min":220.732},"last":0.35799999999999998},{"interface":"front","average":{"1min":16.422000000000001,"5min":6.75,"15min":6.5830000000000002},"min":{"1min":0.30199999999999999,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":194.56299999999999,"5min":194.56299999999999,"15min":218.78200000000001},"last":3.7130000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.79900000000000004,"15min":2.3479999999999999},"min":{"1min":0.34399999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.60999999999999999,"5min":9.3140000000000001,"15min":138.81299999999999},"last":3.605},{"interface":"front","average":{"1min":0.66800000000000004,"5min":1.02,"15min":3.0270000000000001},"min":{"1min":0.36799999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.661,"5min":9.3670000000000009,"15min":171.251},"last":3.4830000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.1850000000000001,"15min":2.7290000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":1.288,"5min":15.076000000000001,"15min":171.30699999999999},"last":3.5169999999999999},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.097,"15min":1.9930000000000001},"min":{"1min":0.33600000000000002,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":4.1710000000000003,"5min":6.5279999999999996,"15min":139.40799999999999},"last":0.54100000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":1.143,"15min":3.0760000000000001},"min":{"1min":0.374,"5min":0.34499999999999997,"15min":0.29499999999999998},"max":{"1min":12.654999999999999,"5min":12.654999999999999,"15min":171.33600000000001},"last":3.5600000000000001},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.155,"15min":3.2749999999999999},"min":{"1min":0.41199999999999998,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":4.6440000000000001,"5min":9.157,"15min":184.13499999999999},"last":2.1360000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":1.012,"15min":2.5339999999999998},"min":{"1min":0.35999999999999999,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":3.4089999999999998,"5min":8.9510000000000005,"15min":138.77699999999999},"last":3.6320000000000001},{"interface":"front","average":{"1min":0.89100000000000001,"5min":1.3069999999999999,"15min":2.5089999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.33300000000000002},"max":{"1min":2.5,"5min":9.8870000000000005,"15min":139.37899999999999},"last":0.58299999999999996}]}]},{"osd":2,"up_from":199,"seq":854698492167,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1928176,"kb_used_data":1062120,"kb_used_omap":7,"kb_used_meta":866040,"kb_avail":91816976,"statfs":{"total":95995035648,"available":94020583424,"internally_reserved":0,"allocated":1087610880,"data_stored":1084691287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":886825041},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.496,"15min":2.968},"min":{"1min":0.20999999999999999,"5min":0.186,"15min":0.17699999999999999},"max":{"1min":2.9980000000000002,"5min":61.950000000000003,"15min":95.587999999999994},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.90700000000000003,"5min":1.7050000000000001,"15min":2.8319999999999999},"min":{"1min":0.22600000000000001,"5min":0.19700000000000001,"15min":0.18099999999999999},"max":{"1min":4.5739999999999998,"5min":61.984999999999999,"15min":95.796000000000006},"last":0.371}]},{"osd":1,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":4.3550000000000004,"15min":4.516},"min":{"1min":0.21299999999999999,"5min":0.192,"15min":0.192},"max":{"1min":4.4500000000000002,"5min":106.765,"15min":157.98699999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.76300000000000001,"5min":3.9319999999999999,"15min":4.4649999999999999},"min":{"1min":0.24399999999999999,"5min":0.222,"15min":0.222},"max":{"1min":4.468,"5min":106.732,"15min":158.03800000000001},"last":0.38600000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":5.1689999999999996,"15min":4.3070000000000004},"min":{"1min":0.245,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":4.1740000000000004,"5min":106.70399999999999,"15min":106.70399999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.0209999999999999,"5min":4.7800000000000002,"15min":4.1459999999999999},"min":{"1min":0.28100000000000003,"5min":0.246,"15min":0.23200000000000001},"max":{"1min":3.52,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.435}]},{"osd":4,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":1.6779999999999999,"15min":1.768},"min":{"1min":0.35499999999999998,"5min":0.33700000000000002,"15min":0.27700000000000002},"max":{"1min":3.5339999999999998,"5min":62.081000000000003,"15min":62.081000000000003},"last":0.62},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.875,"15min":1.802},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.28499999999999998},"max":{"1min":2.9489999999999998,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.53300000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":9.3179999999999996,"5min":4.476,"15min":2.6640000000000001},"min":{"1min":0.32200000000000001,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":101.104,"5min":101.104,"15min":101.104},"last":101.104},{"interface":"front","average":{"1min":9.7829999999999995,"5min":4.6280000000000001,"15min":2.6800000000000002},"min":{"1min":0.35199999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":101.404,"5min":101.404,"15min":101.404},"last":101.404}]},{"osd":6,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":9.1869999999999994,"5min":4.8730000000000002,"15min":2.9239999999999999},"min":{"1min":0.372,"5min":0.31,"15min":0.31},"max":{"1min":97.558999999999997,"5min":107.018,"15min":107.018},"last":97.558999999999997},{"interface":"front","average":{"1min":9.4420000000000002,"5min":3.7989999999999999,"15min":2.5219999999999998},"min":{"1min":0.375,"5min":0.371,"15min":0.36699999999999999},"max":{"1min":98.231999999999999,"5min":98.231999999999999,"15min":98.231999999999999},"last":98.231999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.7150000000000001,"15min":1.8680000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.4129999999999998,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.67200000000000004},{"interface":"front","average":{"1min":1.0840000000000001,"5min":3,"15min":2.3100000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":3.452,"5min":106.809,"15min":106.809},"last":0.70399999999999996}]}]},{"osd":3,"up_from":148,"seq":635655160136,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1528116,"kb_used_data":572332,"kb_used_omap":2,"kb_used_meta":955773,"kb_avail":92217036,"statfs":{"total":95995035648,"available":94430244864,"internally_reserved":0,"allocated":586067968,"data_stored":583206153,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":978711605},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":1.0009999999999999,"15min":2.5059999999999998},"min":{"1min":0.19500000000000001,"5min":0.184,"15min":0.17699999999999999},"max":{"1min":2.9980000000000002,"5min":17.213000000000001,"15min":103.866},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.995,"15min":2.8069999999999999},"min":{"1min":0.188,"5min":0.188,"15min":0.17199999999999999},"max":{"1min":2.8090000000000002,"5min":12.202,"15min":101.884},"last":0.84699999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.95999999999999996,"15min":2.819},"min":{"1min":0.23899999999999999,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":2.851,"5min":7.3319999999999999,"15min":85.242999999999995},"last":9.7170000000000005},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.88400000000000001,"15min":2.6920000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":2.9830000000000001,"5min":11.228,"15min":85.302000000000007},"last":0.38100000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:37:57 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":1.4359999999999999,"15min":4.0659999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.19800000000000001},"max":{"1min":36.362000000000002,"5min":36.362000000000002,"15min":84.724999999999994},"last":9.7750000000000004},{"interface":"front","average":{"1min":2.6110000000000002,"5min":1.431,"15min":3.931},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.214},"max":{"1min":36.158999999999999,"5min":36.158999999999999,"15min":84.567999999999998},"last":2.2170000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":1.032,"15min":1.202},"min":{"1min":0.318,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":2.9609999999999999,"5min":11.209,"15min":11.209},"last":0.308},{"interface":"front","average":{"1min":0.74199999999999999,"5min":1.1140000000000001,"15min":1.6160000000000001},"min":{"1min":0.34000000000000002,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":2.7639999999999998,"5min":7.681,"15min":24.782},"last":10.917}]},{"osd":5,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":10.651,"5min":8.0739999999999998,"15min":3.9369999999999998},"min":{"1min":0.38500000000000001,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":196.358,"5min":214.43700000000001,"15min":214.43700000000001},"last":0.73699999999999999},{"interface":"front","average":{"1min":10.586,"5min":8.0500000000000007,"15min":3.984},"min":{"1min":0.35999999999999999,"5min":0.34699999999999998,"15min":0.30599999999999999},"max":{"1min":196.048,"5min":214.46600000000001,"15min":214.46600000000001},"last":6.0650000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":7.2190000000000003,"5min":3.2280000000000002,"15min":2.5649999999999999},"min":{"1min":0.35799999999999998,"5min":0.318,"15min":0.318},"max":{"1min":125.94,"5min":125.94,"15min":125.94},"last":11.09},{"interface":"front","average":{"1min":7.1189999999999998,"5min":3.3559999999999999,"15min":2.5649999999999999},"min":{"1min":0.36499999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":125.955,"5min":125.955,"15min":125.955},"last":6.1210000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:37:57 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":4.5289999999999999,"15min":3.1909999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32600000000000001},"max":{"1min":6.5800000000000001,"5min":192.69,"15min":192.69},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.92400000000000004,"5min":4.5789999999999997,"15min":3.629},"min":{"1min":0.40699999999999997,"5min":0.40100000000000002,"15min":0.36699999999999999},"max":{"1min":3.6120000000000001,"5min":192.65600000000001,"15min":192.65600000000001},"last":10.951000000000001}]}]},{"osd":4,"up_from":0,"seq":128849019455,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757817,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1791640,"kb_used_data":1029776,"kb_used_omap":0,"kb_used_meta":761856,"kb_avail":91953512,"statfs":{"total":95995035648,"available":94160396288,"internally_reserved":0,"allocated":1054490624,"data_stored":1052813817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":780140544},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.032,"15min":2.597},"min":{"1min":0.25,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":2.9279999999999999,"5min":21.109000000000002,"15min":113.869},"last":0.28499999999999998},{"interface":"front","average":{"1min":1.123,"5min":1.407,"15min":3.0659999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":11.154999999999999,"5min":20.876999999999999,"15min":138.55199999999999},"last":1.0209999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:37:22 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.1459999999999999,"15min":2.1629999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.1360000000000001,"5min":29.007999999999999,"15min":69.816999999999993},"last":0.90300000000000002},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.212,"15min":2.3929999999999998},"min":{"1min":0.30499999999999999,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":1.452,"5min":29.053999999999998,"15min":70.268000000000001},"last":0.93200000000000005}]},{"osd":2,"last update":"Sat Sep 23 23:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":6.8869999999999996,"15min":4.4939999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26900000000000002},"max":{"1min":6.2469999999999999,"5min":208.738,"15min":208.738},"last":1.3280000000000001},{"interface":"front","average":{"1min":0.77700000000000002,"5min":5.5220000000000002,"15min":4.0140000000000002},"min":{"1min":0.309,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":3.2290000000000001,"5min":208.631,"15min":208.631},"last":0.34200000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:37:53 2023","interfaces":[{"interface":"back","average":{"1min":6.6550000000000002,"5min":5.6689999999999996,"15min":4.1470000000000002},"min":{"1min":0.33400000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":143.66300000000001,"5min":143.66300000000001,"15min":143.66300000000001},"last":1.181},{"interface":"front","average":{"1min":6.7670000000000003,"5min":6.0010000000000003,"15min":4.327},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":143.62200000000001,"5min":143.62200000000001,"15min":143.62200000000001},"last":1.2}]},{"osd":4,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.88200000000000001,"15min":0.877},"min":{"1min":0.316,"5min":0.315,"15min":0.251},"max":{"1min":2.2650000000000001,"5min":9.2690000000000001,"15min":19.962},"last":0.96899999999999997},{"interface":"front","average":{"1min":0.85499999999999998,"5min":1.016,"15min":0.84099999999999997},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.22900000000000001},"max":{"1min":3.1840000000000002,"5min":10.273999999999999,"15min":20.829999999999998},"last":2.0899999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.3340000000000001,"15min":1.5249999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.3100000000000001,"5min":148.36600000000001,"15min":148.36600000000001},"last":1.7290000000000001},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.7389999999999999,"15min":1.71},"min":{"1min":0.32800000000000001,"5min":0.30399999999999999,"15min":0.26000000000000001},"max":{"1min":5.7240000000000002,"5min":148.42500000000001,"15min":148.42500000000001},"last":0.504}]},{"osd":7,"last update":"Sat Sep 23 23:37:48 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.306,"15min":2.177},"min":{"1min":0.34699999999999998,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":4.3380000000000001,"5min":15.497,"15min":179.28800000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.157,"5min":1.339,"15min":2.2480000000000002},"min":{"1min":0.40600000000000003,"5min":0.35199999999999998,"15min":0.34999999999999998},"max":{"1min":6.1440000000000001,"5min":15.653,"15min":179.428},"last":0.47499999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":169082880,"data_stored":169082880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":494800896,"data_stored":494796824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1064239104,"data_stored":1064239104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":569905152,"data_stored":569901080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2353397760,"data_stored":2353397760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1042939904,"data_stored":1042939904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":683212800,"data_stored":683212800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":698748928,"data_stored":698744856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:38:17.394 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:38:17.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:17.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:38:17.506 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:38:17.596 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:17 smithi110 ceph-mon[221102]: from='client.37017 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:38:17.596 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:17 smithi110 ceph-mon[221102]: pgmap v1718: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 20709/70755 objects degraded (29.269%); 1350/70755 objects misplaced (1.908%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T23:38:17.600 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:17 smithi110 ceph-mon[212939]: from='client.37017 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:38:17.600 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:17 smithi110 ceph-mon[212939]: pgmap v1718: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 20709/70755 objects degraded (29.269%); 1350/70755 objects misplaced (1.908%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T23:38:17.605 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:17.638 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:17 smithi138 ceph-mon[193606]: from='client.37017 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:38:17.638 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:17 smithi138 ceph-mon[193606]: pgmap v1718: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 20709/70755 objects degraded (29.269%); 1350/70755 objects misplaced (1.908%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T23:38:17.670 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:17.797 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:38:17.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:17.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:38:17.907 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:38:18.023 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:18.054 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:38:18.054 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:38:18.054 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:38:18.055 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:38:18.055 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.055 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71516 13.e 13:7de122bf:::benchmark_data_smithi110_541928_object71515:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e555)", 2023-09-23T23:38:18.055 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.528660+0000", 2023-09-23T23:38:18.055 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 267.519461917, 2023-09-23T23:38:18.055 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2167170459999999, 2023-09-23T23:38:18.055 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.055 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:38:18.055 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:38:18.055 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:38:18.055 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:38:18.055 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71516 2023-09-23T23:38:18.056 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.056 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.056 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528660+0000", 2023-09-23T23:38:18.056 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.056 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.056 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528660+0000", 2023-09-23T23:38:18.056 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.056 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.057 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528659+0000", 2023-09-23T23:38:18.057 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:38:18.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.057 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528700+0000", 2023-09-23T23:38:18.057 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0859999999999998e-05 2023-09-23T23:38:18.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.057 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528712+0000", 2023-09-23T23:38:18.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2262000000000001e-05 2023-09-23T23:38:18.058 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.058 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.058 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.058 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528719+0000", 2023-09-23T23:38:18.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3339999999999998e-06 2023-09-23T23:38:18.058 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.058 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.058 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.058 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531320+0000", 2023-09-23T23:38:18.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002601506 2023-09-23T23:38:18.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.059 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531460+0000", 2023-09-23T23:38:18.059 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013956799999999999 2023-09-23T23:38:18.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.059 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:38:18.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531617+0000", 2023-09-23T23:38:18.059 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015701199999999999 2023-09-23T23:38:18.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.059 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:38:18.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.534591+0000", 2023-09-23T23:38:18.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029740880000000002 2023-09-23T23:38:18.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.060 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.060 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.545743+0000", 2023-09-23T23:38:18.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011152331 2023-09-23T23:38:18.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.060 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.060 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.545752+0000", 2023-09-23T23:38:18.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0000000000000002e-06 2023-09-23T23:38:18.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.061 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552315+0000", 2023-09-23T23:38:18.061 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.006563179 2023-09-23T23:38:18.061 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.061 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552330+0000", 2023-09-23T23:38:18.061 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4734e-05 2023-09-23T23:38:18.061 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.062 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745274+0000", 2023-09-23T23:38:18.062 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1929441679999999 2023-09-23T23:38:18.062 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.062 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.062 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.062 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745377+0000", 2023-09-23T23:38:18.062 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103303 2023-09-23T23:38:18.062 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.062 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.062 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.062 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.062 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.062 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71528 13.e 13:790d8dd9:::benchmark_data_smithi110_541928_object71527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:38:18.063 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.572698+0000", 2023-09-23T23:38:18.063 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 267.475424148, 2023-09-23T23:38:18.063 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2677280570000002, 2023-09-23T23:38:18.063 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.063 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:18.063 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:38:18.063 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:38:18.063 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:38:18.063 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71528 2023-09-23T23:38:18.063 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.063 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.063 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.063 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572698+0000", 2023-09-23T23:38:18.064 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.064 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.064 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.064 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572698+0000", 2023-09-23T23:38:18.064 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.064 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.064 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.064 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572697+0000", 2023-09-23T23:38:18.064 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:38:18.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.065 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.065 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572737+0000", 2023-09-23T23:38:18.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0278000000000003e-05 2023-09-23T23:38:18.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.065 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.065 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572746+0000", 2023-09-23T23:38:18.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3340000000000008e-06 2023-09-23T23:38:18.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.065 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572753+0000", 2023-09-23T23:38:18.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3899999999999998e-06 2023-09-23T23:38:18.066 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.066 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572817+0000", 2023-09-23T23:38:18.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4576e-05 2023-09-23T23:38:18.066 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.066 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572827+0000", 2023-09-23T23:38:18.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.978e-06 2023-09-23T23:38:18.067 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.067 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.067 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.067 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552357+0000", 2023-09-23T23:38:18.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97953002600000005 2023-09-23T23:38:18.067 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.067 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.067 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.067 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552365+0000", 2023-09-23T23:38:18.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5390000000000002e-06 2023-09-23T23:38:18.067 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.067 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.067 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745405+0000", 2023-09-23T23:38:18.068 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1930404109999999 2023-09-23T23:38:18.068 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.068 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745543+0000", 2023-09-23T23:38:18.068 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013780300000000001 2023-09-23T23:38:18.068 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.068 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:38:18.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745737+0000", 2023-09-23T23:38:18.068 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019410400000000001 2023-09-23T23:38:18.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.069 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.069 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:38:18.069 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.759566+0000", 2023-09-23T23:38:18.069 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013828844999999999 2023-09-23T23:38:18.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.069 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.069 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.069 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.768156+0000", 2023-09-23T23:38:18.069 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085902789999999993 2023-09-23T23:38:18.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.069 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.069 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.070 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.836351+0000", 2023-09-23T23:38:18.070 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.068194406999999999 2023-09-23T23:38:18.070 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.070 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:38:18.070 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.836426+0000", 2023-09-23T23:38:18.070 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4889999999999996e-05 2023-09-23T23:38:18.070 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.070 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.070 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.840426+0000", 2023-09-23T23:38:18.070 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040005659999999997 2023-09-23T23:38:18.071 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.071 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.071 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.071 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.071 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.071 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71532 13.e 13:7da393ae:::benchmark_data_smithi110_541928_object71531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:38:18.071 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.575009+0000", 2023-09-23T23:38:18.071 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 267.47311306099999, 2023-09-23T23:38:18.071 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2696883859999999, 2023-09-23T23:38:18.071 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.071 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:18.071 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:38:18.072 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:38:18.072 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:38:18.072 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71532 2023-09-23T23:38:18.072 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.072 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.072 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.072 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.072 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575009+0000", 2023-09-23T23:38:18.072 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.072 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.072 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.072 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.072 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575009+0000", 2023-09-23T23:38:18.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.073 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.073 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575004+0000", 2023-09-23T23:38:18.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999948 2023-09-23T23:38:18.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.073 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.073 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575068+0000", 2023-09-23T23:38:18.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3359000000000001e-05 2023-09-23T23:38:18.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.074 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.074 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.074 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575081+0000", 2023-09-23T23:38:18.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3434999999999999e-05 2023-09-23T23:38:18.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.074 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.074 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.074 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575090+0000", 2023-09-23T23:38:18.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7490000000000004e-06 2023-09-23T23:38:18.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.074 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.074 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.074 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575154+0000", 2023-09-23T23:38:18.075 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4023999999999996e-05 2023-09-23T23:38:18.075 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.075 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.075 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.075 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575166+0000", 2023-09-23T23:38:18.075 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2033e-05 2023-09-23T23:38:18.075 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.075 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.075 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.075 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552388+0000", 2023-09-23T23:38:18.076 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97722166399999999 2023-09-23T23:38:18.076 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.076 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.076 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.076 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552395+0000", 2023-09-23T23:38:18.076 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3429999999999998e-06 2023-09-23T23:38:18.076 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.076 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.076 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.076 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.748633+0000", 2023-09-23T23:38:18.076 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1962377799999999 2023-09-23T23:38:18.076 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.077 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.077 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.077 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.748799+0000", 2023-09-23T23:38:18.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016630200000000001 2023-09-23T23:38:18.077 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.077 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.077 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:38:18.077 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.748992+0000", 2023-09-23T23:38:18.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019316 2023-09-23T23:38:18.077 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.077 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:38:18.078 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.759572+0000", 2023-09-23T23:38:18.078 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010580032 2023-09-23T23:38:18.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.078 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.768191+0000", 2023-09-23T23:38:18.078 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086191280000000002 2023-09-23T23:38:18.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.078 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.840481+0000", 2023-09-23T23:38:18.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.072289500000000007 2023-09-23T23:38:18.079 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.079 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:38:18.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.840542+0000", 2023-09-23T23:38:18.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1509999999999999e-05 2023-09-23T23:38:18.079 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.079 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.844698+0000", 2023-09-23T23:38:18.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041553880000000003 2023-09-23T23:38:18.079 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.079 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.080 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.080 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77337 13.d 13:ba15f28f:::benchmark_data_smithi110_541928_object77336:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:38:18.080 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.706918+0000", 2023-09-23T23:38:18.080 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 254.341204086, 2023-09-23T23:38:18.080 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1549711069999999, 2023-09-23T23:38:18.080 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.080 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:38:18.080 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:38:18.080 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:38:18.080 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:38:18.081 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77337 2023-09-23T23:38:18.081 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.081 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.081 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.081 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.081 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706918+0000", 2023-09-23T23:38:18.081 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.081 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.081 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.081 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.081 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706918+0000", 2023-09-23T23:38:18.081 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.082 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.082 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706917+0000", 2023-09-23T23:38:18.082 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:38:18.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.082 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.082 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706970+0000", 2023-09-23T23:38:18.082 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2435000000000002e-05 2023-09-23T23:38:18.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.083 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.083 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706986+0000", 2023-09-23T23:38:18.083 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6252999999999998e-05 2023-09-23T23:38:18.083 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.083 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.083 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.083 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706993+0000", 2023-09-23T23:38:18.083 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1439999999999997e-06 2023-09-23T23:38:18.083 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.083 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.083 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.083 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709103+0000", 2023-09-23T23:38:18.083 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021098419999999998 2023-09-23T23:38:18.084 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.084 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.084 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.084 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709112+0000", 2023-09-23T23:38:18.084 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3640000000000005e-06 2023-09-23T23:38:18.084 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.084 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.084 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.084 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.860680+0000", 2023-09-23T23:38:18.084 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1515676990000001 2023-09-23T23:38:18.084 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.084 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.085 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.085 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.861889+0000", 2023-09-23T23:38:18.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012094930000000001 2023-09-23T23:38:18.085 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.085 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.085 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.085 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.085 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.085 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77365 13.d 13:bd3fed09:::benchmark_data_smithi110_541928_object77364:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:38:18.085 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.707869+0000", 2023-09-23T23:38:18.085 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 254.340253808, 2023-09-23T23:38:18.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1996441330000001, 2023-09-23T23:38:18.086 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.086 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:18.086 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:38:18.086 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:38:18.086 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:38:18.086 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77365 2023-09-23T23:38:18.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.086 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.086 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.086 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707869+0000", 2023-09-23T23:38:18.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707869+0000", 2023-09-23T23:38:18.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707868+0000", 2023-09-23T23:38:18.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:38:18.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707912+0000", 2023-09-23T23:38:18.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4286999999999998e-05 2023-09-23T23:38:18.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707922+0000", 2023-09-23T23:38:18.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0132000000000001e-05 2023-09-23T23:38:18.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707929+0000", 2023-09-23T23:38:18.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4690000000000002e-06 2023-09-23T23:38:18.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.089 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.089 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.089 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709143+0000", 2023-09-23T23:38:18.089 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012139539999999999 2023-09-23T23:38:18.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.089 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.089 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.089 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709148+0000", 2023-09-23T23:38:18.089 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8420000000000001e-06 2023-09-23T23:38:18.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.090 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.090 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.090 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.861913+0000", 2023-09-23T23:38:18.090 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.152764984 2023-09-23T23:38:18.090 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.090 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.090 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.090 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.862371+0000", 2023-09-23T23:38:18.090 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00045722599999999998 2023-09-23T23:38:18.090 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.090 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.090 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:38:18.091 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.862643+0000", 2023-09-23T23:38:18.091 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000271983 2023-09-23T23:38:18.091 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.091 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.091 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:38:18.091 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.872385+0000", 2023-09-23T23:38:18.091 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0097426379999999996 2023-09-23T23:38:18.091 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.091 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.091 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.091 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.905589+0000", 2023-09-23T23:38:18.091 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.033203780000000002 2023-09-23T23:38:18.092 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.092 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.092 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.092 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.905624+0000", 2023-09-23T23:38:18.092 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5058000000000002e-05 2023-09-23T23:38:18.092 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.092 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.092 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:38:18.092 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.905662+0000", 2023-09-23T23:38:18.092 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7523e-05 2023-09-23T23:38:18.092 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.093 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.093 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.093 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907513+0000", 2023-09-23T23:38:18.093 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0018510320000000001 2023-09-23T23:38:18.093 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.093 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.093 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.093 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.093 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.093 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77391 13.d 13:bfca221d:::benchmark_data_smithi110_541928_object77390:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:38:18.093 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.707935+0000", 2023-09-23T23:38:18.093 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 254.34018734700001, 2023-09-23T23:38:18.094 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2024361749999999, 2023-09-23T23:38:18.094 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.094 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:18.094 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:38:18.094 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:38:18.094 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:38:18.094 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77391 2023-09-23T23:38:18.094 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.094 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.094 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.094 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.094 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707935+0000", 2023-09-23T23:38:18.095 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.095 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.095 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.095 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707935+0000", 2023-09-23T23:38:18.095 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.095 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.095 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.095 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707935+0000", 2023-09-23T23:38:18.095 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:18.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.096 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.096 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707971+0000", 2023-09-23T23:38:18.096 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6470000000000001e-05 2023-09-23T23:38:18.096 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.096 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.096 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707977+0000", 2023-09-23T23:38:18.096 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1489999999999999e-06 2023-09-23T23:38:18.096 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.096 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.097 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707982+0000", 2023-09-23T23:38:18.097 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5349999999999998e-06 2023-09-23T23:38:18.097 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.097 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.097 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.097 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709170+0000", 2023-09-23T23:38:18.097 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011882080000000001 2023-09-23T23:38:18.097 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.097 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.097 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.097 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709176+0000", 2023-09-23T23:38:18.098 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.297e-06 2023-09-23T23:38:18.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.098 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.098 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.098 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.866177+0000", 2023-09-23T23:38:18.098 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1570007 2023-09-23T23:38:18.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.098 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.098 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.098 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.866317+0000", 2023-09-23T23:38:18.098 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013973499999999999 2023-09-23T23:38:18.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.099 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.099 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:38:18.099 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.866488+0000", 2023-09-23T23:38:18.099 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000171495 2023-09-23T23:38:18.099 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.099 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.099 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:38:18.099 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.875118+0000", 2023-09-23T23:38:18.099 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086296299999999992 2023-09-23T23:38:18.099 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.099 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.099 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.100 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.905607+0000", 2023-09-23T23:38:18.100 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.030489532999999999 2023-09-23T23:38:18.100 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.100 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.100 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.100 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907981+0000", 2023-09-23T23:38:18.100 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023734239999999998 2023-09-23T23:38:18.100 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.100 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.100 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:38:18.100 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.908040+0000", 2023-09-23T23:38:18.100 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9463000000000003e-05 2023-09-23T23:38:18.101 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.101 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.101 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.101 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910371+0000", 2023-09-23T23:38:18.101 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023309279999999999 2023-09-23T23:38:18.101 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.101 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.101 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.101 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.101 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.101 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77394 13.d 13:bd1f7538:::benchmark_data_smithi110_541928_object77393:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:38:18.102 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.707987+0000", 2023-09-23T23:38:18.102 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 254.340135605, 2023-09-23T23:38:18.102 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.204224143, 2023-09-23T23:38:18.102 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.102 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:18.102 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:38:18.102 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:38:18.102 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:38:18.102 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77394 2023-09-23T23:38:18.102 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.102 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.102 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.103 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.103 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707987+0000", 2023-09-23T23:38:18.103 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.103 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.103 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.103 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.103 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707987+0000", 2023-09-23T23:38:18.103 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.103 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.103 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.103 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707986+0000", 2023-09-23T23:38:18.104 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:18.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.104 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.104 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708018+0000", 2023-09-23T23:38:18.104 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1340999999999997e-05 2023-09-23T23:38:18.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.104 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.104 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708024+0000", 2023-09-23T23:38:18.104 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9050000000000002e-06 2023-09-23T23:38:18.105 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.105 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.105 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708027+0000", 2023-09-23T23:38:18.105 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6550000000000002e-06 2023-09-23T23:38:18.105 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.105 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.105 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709199+0000", 2023-09-23T23:38:18.105 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011713400000000001 2023-09-23T23:38:18.105 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.106 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.106 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709204+0000", 2023-09-23T23:38:18.106 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4480000000000002e-06 2023-09-23T23:38:18.106 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.106 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.106 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.106 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.869263+0000", 2023-09-23T23:38:18.106 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1600593370000001 2023-09-23T23:38:18.106 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.106 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.106 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.107 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.869419+0000", 2023-09-23T23:38:18.107 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015545899999999999 2023-09-23T23:38:18.107 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.107 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.107 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:38:18.107 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.869589+0000", 2023-09-23T23:38:18.107 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000170225 2023-09-23T23:38:18.107 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.107 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.107 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:38:18.107 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889913+0000", 2023-09-23T23:38:18.107 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020323536 2023-09-23T23:38:18.108 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.108 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.108 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.108 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907553+0000", 2023-09-23T23:38:18.108 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017640769000000001 2023-09-23T23:38:18.108 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.108 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.108 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.108 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910571+0000", 2023-09-23T23:38:18.108 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030181119999999999 2023-09-23T23:38:18.108 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.108 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.109 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:38:18.109 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910620+0000", 2023-09-23T23:38:18.109 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8891000000000003e-05 2023-09-23T23:38:18.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.109 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.109 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912211+0000", 2023-09-23T23:38:18.109 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015906270000000001 2023-09-23T23:38:18.109 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.109 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.109 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.110 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.110 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77463 13.d 13:b8eb00c3:::benchmark_data_smithi110_541928_object77462:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:38:18.110 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.708031+0000", 2023-09-23T23:38:18.110 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 254.34009090399999, 2023-09-23T23:38:18.110 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.206478288, 2023-09-23T23:38:18.110 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.110 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:18.110 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:38:18.110 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:38:18.110 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:38:18.110 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77463 2023-09-23T23:38:18.110 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.111 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.111 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708031+0000", 2023-09-23T23:38:18.111 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.111 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.111 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708031+0000", 2023-09-23T23:38:18.111 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.111 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708031+0000", 2023-09-23T23:38:18.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:18.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708086+0000", 2023-09-23T23:38:18.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4630999999999999e-05 2023-09-23T23:38:18.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708092+0000", 2023-09-23T23:38:18.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1800000000000001e-06 2023-09-23T23:38:18.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708096+0000", 2023-09-23T23:38:18.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.811e-06 2023-09-23T23:38:18.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709228+0000", 2023-09-23T23:38:18.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011318190000000001 2023-09-23T23:38:18.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.114 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.114 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709234+0000", 2023-09-23T23:38:18.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2979999999999997e-06 2023-09-23T23:38:18.114 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.114 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.114 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.872414+0000", 2023-09-23T23:38:18.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.163180543 2023-09-23T23:38:18.114 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.114 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.114 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.872546+0000", 2023-09-23T23:38:18.115 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013156499999999999 2023-09-23T23:38:18.115 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.115 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.115 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:38:18.115 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.872714+0000", 2023-09-23T23:38:18.115 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000167713 2023-09-23T23:38:18.115 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.115 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.115 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:38:18.115 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889920+0000", 2023-09-23T23:38:18.115 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017206255 2023-09-23T23:38:18.115 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.116 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.116 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.116 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907578+0000", 2023-09-23T23:38:18.116 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017658231 2023-09-23T23:38:18.116 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.116 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.116 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.116 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912462+0000", 2023-09-23T23:38:18.116 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048837949999999998 2023-09-23T23:38:18.116 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.116 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.116 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:38:18.116 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912506+0000", 2023-09-23T23:38:18.117 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4131999999999997e-05 2023-09-23T23:38:18.117 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.117 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.117 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.117 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.914510+0000", 2023-09-23T23:38:18.117 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002003632 2023-09-23T23:38:18.117 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.117 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.117 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.117 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.117 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.117 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77467 13.d 13:b265ce95:::benchmark_data_smithi110_541928_object77466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:38:18.118 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.708101+0000", 2023-09-23T23:38:18.118 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 254.34002149, 2023-09-23T23:38:18.118 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2086463730000001, 2023-09-23T23:38:18.118 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.118 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:18.118 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:38:18.118 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:38:18.118 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:38:18.118 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77467 2023-09-23T23:38:18.118 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.118 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.118 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.119 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.119 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708101+0000", 2023-09-23T23:38:18.119 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.119 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.119 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.119 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.119 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708101+0000", 2023-09-23T23:38:18.119 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.119 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.119 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.119 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.119 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708101+0000", 2023-09-23T23:38:18.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:18.120 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.120 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.120 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.120 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708157+0000", 2023-09-23T23:38:18.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6196999999999997e-05 2023-09-23T23:38:18.120 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.120 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.120 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.120 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708163+0000", 2023-09-23T23:38:18.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0939999999999996e-06 2023-09-23T23:38:18.120 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708168+0000", 2023-09-23T23:38:18.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1800000000000004e-06 2023-09-23T23:38:18.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709259+0000", 2023-09-23T23:38:18.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010911370000000001 2023-09-23T23:38:18.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709266+0000", 2023-09-23T23:38:18.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7870000000000004e-06 2023-09-23T23:38:18.122 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.122 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.122 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.875164+0000", 2023-09-23T23:38:18.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1658984800000001 2023-09-23T23:38:18.122 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.122 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.122 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.875272+0000", 2023-09-23T23:38:18.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010731299999999999 2023-09-23T23:38:18.122 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.123 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.123 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:38:18.123 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.875417+0000", 2023-09-23T23:38:18.123 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014540399999999999 2023-09-23T23:38:18.123 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.123 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.123 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:38:18.123 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889926+0000", 2023-09-23T23:38:18.123 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014509024000000001 2023-09-23T23:38:18.123 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.123 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.123 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.124 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907925+0000", 2023-09-23T23:38:18.124 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017998692 2023-09-23T23:38:18.124 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.124 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.124 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.124 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.914568+0000", 2023-09-23T23:38:18.124 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066434789999999999 2023-09-23T23:38:18.124 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.124 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.124 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:38:18.124 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.914630+0000", 2023-09-23T23:38:18.124 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1175999999999998e-05 2023-09-23T23:38:18.125 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.125 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.125 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.125 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.916747+0000", 2023-09-23T23:38:18.125 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021177470000000001 2023-09-23T23:38:18.125 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.125 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.125 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.125 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.125 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.125 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77471 13.d 13:bd1602ed:::benchmark_data_smithi110_541928_object77470:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:38:18.125 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.708173+0000", 2023-09-23T23:38:18.125 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 254.339949639, 2023-09-23T23:38:18.126 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2113011469999999, 2023-09-23T23:38:18.126 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.126 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:18.126 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:38:18.126 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:38:18.126 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:38:18.126 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77471 2023-09-23T23:38:18.126 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.126 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.126 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.126 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.127 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708173+0000", 2023-09-23T23:38:18.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.127 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.127 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.127 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.127 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708173+0000", 2023-09-23T23:38:18.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.127 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.127 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.127 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.127 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708172+0000", 2023-09-23T23:38:18.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:18.127 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.128 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.128 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708227+0000", 2023-09-23T23:38:18.128 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4215000000000002e-05 2023-09-23T23:38:18.128 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.128 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.128 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708234+0000", 2023-09-23T23:38:18.128 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0790000000000003e-06 2023-09-23T23:38:18.128 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.128 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.129 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708238+0000", 2023-09-23T23:38:18.129 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6450000000000004e-06 2023-09-23T23:38:18.129 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.129 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.129 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.129 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709564+0000", 2023-09-23T23:38:18.129 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0013253379999999999 2023-09-23T23:38:18.129 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.129 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.129 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.129 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709570+0000", 2023-09-23T23:38:18.129 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5570000000000001e-06 2023-09-23T23:38:18.130 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.130 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.130 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.130 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.877443+0000", 2023-09-23T23:38:18.130 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.167873216 2023-09-23T23:38:18.130 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.130 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.130 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.130 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.877572+0000", 2023-09-23T23:38:18.130 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012810999999999999 2023-09-23T23:38:18.130 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.130 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.131 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:38:18.131 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.877732+0000", 2023-09-23T23:38:18.131 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000160376 2023-09-23T23:38:18.131 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.131 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.131 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:38:18.131 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889933+0000", 2023-09-23T23:38:18.131 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012201033999999999 2023-09-23T23:38:18.131 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.131 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.131 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.131 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910445+0000", 2023-09-23T23:38:18.131 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020511715999999999 2023-09-23T23:38:18.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.132 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.132 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.132 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.916902+0000", 2023-09-23T23:38:18.132 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064569379999999997 2023-09-23T23:38:18.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.132 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.132 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:38:18.132 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.916958+0000", 2023-09-23T23:38:18.132 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6443000000000002e-05 2023-09-23T23:38:18.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.133 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.133 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.919474+0000", 2023-09-23T23:38:18.133 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002515804 2023-09-23T23:38:18.133 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.133 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.133 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.133 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.133 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77477 13.d 13:b03c7b50:::benchmark_data_smithi110_541928_object77476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:38:18.133 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.708243+0000", 2023-09-23T23:38:18.133 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 254.33987964900001, 2023-09-23T23:38:18.134 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.213803755, 2023-09-23T23:38:18.134 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.134 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:18.134 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:38:18.134 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:38:18.134 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:38:18.134 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77477 2023-09-23T23:38:18.134 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.134 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.134 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.134 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.134 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708243+0000", 2023-09-23T23:38:18.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.135 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708243+0000", 2023-09-23T23:38:18.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.135 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708242+0000", 2023-09-23T23:38:18.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:18.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.136 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.136 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.136 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708286+0000", 2023-09-23T23:38:18.136 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3625999999999999e-05 2023-09-23T23:38:18.136 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.136 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.136 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.136 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708292+0000", 2023-09-23T23:38:18.136 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0100000000000001e-06 2023-09-23T23:38:18.136 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.136 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.136 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.137 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708296+0000", 2023-09-23T23:38:18.137 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1860000000000002e-06 2023-09-23T23:38:18.137 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.137 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.137 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.137 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709595+0000", 2023-09-23T23:38:18.137 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012991089999999999 2023-09-23T23:38:18.137 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.137 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.137 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.137 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709602+0000", 2023-09-23T23:38:18.137 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6100000000000002e-06 2023-09-23T23:38:18.138 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.138 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.138 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.138 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.879894+0000", 2023-09-23T23:38:18.138 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.170292077 2023-09-23T23:38:18.138 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.138 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.138 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.138 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.879990+0000", 2023-09-23T23:38:18.138 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6212000000000007e-05 2023-09-23T23:38:18.138 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.139 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.139 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:38:18.139 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.880120+0000", 2023-09-23T23:38:18.139 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012987 2023-09-23T23:38:18.139 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.139 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.139 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:38:18.139 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889937+0000", 2023-09-23T23:38:18.139 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0098164270000000008 2023-09-23T23:38:18.139 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.139 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.139 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.140 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910473+0000", 2023-09-23T23:38:18.140 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020536688000000001 2023-09-23T23:38:18.140 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.140 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.140 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.140 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.919527+0000", 2023-09-23T23:38:18.140 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0090542190000000005 2023-09-23T23:38:18.140 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.140 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.140 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:38:18.140 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.919586+0000", 2023-09-23T23:38:18.140 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8749999999999998e-05 2023-09-23T23:38:18.141 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.141 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.141 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.141 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.922046+0000", 2023-09-23T23:38:18.141 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024603099999999998 2023-09-23T23:38:18.141 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.141 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.141 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.141 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.141 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.141 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77503 13.d 13:b2378296:::benchmark_data_smithi110_541928_object77502:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:38:18.141 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.781601+0000", 2023-09-23T23:38:18.141 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 254.26652132000001, 2023-09-23T23:38:18.142 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1428059340000001, 2023-09-23T23:38:18.142 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.142 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:18.142 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:38:18.142 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:38:18.142 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:38:18.142 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77503 2023-09-23T23:38:18.142 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.142 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.142 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.142 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.142 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781601+0000", 2023-09-23T23:38:18.143 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.143 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.143 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.143 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.143 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781601+0000", 2023-09-23T23:38:18.143 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.143 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.143 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.143 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.143 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781600+0000", 2023-09-23T23:38:18.143 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:38:18.143 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.144 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.144 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.144 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781628+0000", 2023-09-23T23:38:18.144 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8694999999999999e-05 2023-09-23T23:38:18.144 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.144 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.144 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.144 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781638+0000", 2023-09-23T23:38:18.144 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5959999999999993e-06 2023-09-23T23:38:18.144 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.144 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.144 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.144 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781643+0000", 2023-09-23T23:38:18.145 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4970000000000001e-06 2023-09-23T23:38:18.145 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.145 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.145 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.145 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781701+0000", 2023-09-23T23:38:18.145 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7757000000000003e-05 2023-09-23T23:38:18.145 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.145 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.145 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.145 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781711+0000", 2023-09-23T23:38:18.145 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4409999999999999e-06 2023-09-23T23:38:18.145 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.145 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.146 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.146 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.882289+0000", 2023-09-23T23:38:18.146 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1005785459999999 2023-09-23T23:38:18.146 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.146 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.146 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.146 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.882445+0000", 2023-09-23T23:38:18.146 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015557 2023-09-23T23:38:18.146 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.146 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.146 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:38:18.147 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.882635+0000", 2023-09-23T23:38:18.147 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000190087 2023-09-23T23:38:18.147 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.147 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.147 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:38:18.147 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.892019+0000", 2023-09-23T23:38:18.147 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0093840280000000009 2023-09-23T23:38:18.147 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.147 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.147 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.147 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910507+0000", 2023-09-23T23:38:18.147 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018488389000000001 2023-09-23T23:38:18.148 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.148 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.148 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.148 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.922095+0000", 2023-09-23T23:38:18.148 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011588144999999999 2023-09-23T23:38:18.148 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.148 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.148 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:38:18.148 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.922148+0000", 2023-09-23T23:38:18.148 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2661000000000003e-05 2023-09-23T23:38:18.148 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.148 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.148 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.149 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.924407+0000", 2023-09-23T23:38:18.149 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022589189999999999 2023-09-23T23:38:18.149 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.149 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.149 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.149 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.149 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.149 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77524 13.d 13:bc4b3c00:::benchmark_data_smithi110_541928_object77523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:38:18.149 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.815029+0000", 2023-09-23T23:38:18.149 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 254.233093129, 2023-09-23T23:38:18.149 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.112009437, 2023-09-23T23:38:18.149 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.150 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:18.150 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:38:18.150 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:38:18.150 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:38:18.150 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77524 2023-09-23T23:38:18.150 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.150 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.150 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.151 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.151 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815029+0000", 2023-09-23T23:38:18.151 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.151 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.151 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.151 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.151 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815029+0000", 2023-09-23T23:38:18.151 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.151 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.151 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.151 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.151 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815028+0000", 2023-09-23T23:38:18.152 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:38:18.152 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.152 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.152 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.152 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815060+0000", 2023-09-23T23:38:18.152 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2218999999999998e-05 2023-09-23T23:38:18.152 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.152 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.152 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.152 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815074+0000", 2023-09-23T23:38:18.152 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.43e-05 2023-09-23T23:38:18.152 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.153 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.153 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.153 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815081+0000", 2023-09-23T23:38:18.153 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0700000000000003e-06 2023-09-23T23:38:18.153 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.153 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.153 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.153 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815152+0000", 2023-09-23T23:38:18.153 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1408999999999993e-05 2023-09-23T23:38:18.153 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.153 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.153 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.154 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815162+0000", 2023-09-23T23:38:18.154 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9459999999999993e-06 2023-09-23T23:38:18.154 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.154 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.154 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.154 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.884789+0000", 2023-09-23T23:38:18.154 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.069627511 2023-09-23T23:38:18.154 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.154 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.154 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.154 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.884895+0000", 2023-09-23T23:38:18.155 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105283 2023-09-23T23:38:18.155 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.155 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.155 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:38:18.155 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.885012+0000", 2023-09-23T23:38:18.155 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011697900000000001 2023-09-23T23:38:18.155 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.155 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.155 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:38:18.155 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.892026+0000", 2023-09-23T23:38:18.155 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0070145779999999996 2023-09-23T23:38:18.155 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.155 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.156 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.156 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910539+0000", 2023-09-23T23:38:18.156 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018512857000000001 2023-09-23T23:38:18.156 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.156 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.156 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.156 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.924468+0000", 2023-09-23T23:38:18.156 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01392849 2023-09-23T23:38:18.156 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.156 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.156 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:38:18.156 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.924523+0000", 2023-09-23T23:38:18.157 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4948000000000003e-05 2023-09-23T23:38:18.157 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.157 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.157 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.157 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.927039+0000", 2023-09-23T23:38:18.157 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002516108 2023-09-23T23:38:18.157 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.157 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.157 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.157 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.157 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.157 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77547 13.d 13:bc3de976:::benchmark_data_smithi110_541928_object77546:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:38:18.158 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.859759+0000", 2023-09-23T23:38:18.158 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 254.188362934, 2023-09-23T23:38:18.158 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0697625900000001, 2023-09-23T23:38:18.158 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.158 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:18.158 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:38:18.158 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:38:18.158 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:38:18.158 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77547 2023-09-23T23:38:18.158 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.158 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.159 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.159 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.159 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859759+0000", 2023-09-23T23:38:18.159 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.159 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.159 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.159 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.159 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859759+0000", 2023-09-23T23:38:18.159 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.159 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.159 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.159 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.160 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859758+0000", 2023-09-23T23:38:18.160 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:38:18.160 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.160 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.160 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.160 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859847+0000", 2023-09-23T23:38:18.160 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8799000000000001e-05 2023-09-23T23:38:18.160 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.160 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.160 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.160 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859861+0000", 2023-09-23T23:38:18.160 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4229e-05 2023-09-23T23:38:18.161 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.161 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.161 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.161 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859868+0000", 2023-09-23T23:38:18.161 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4949999999999996e-06 2023-09-23T23:38:18.161 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.161 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.161 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.161 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859928+0000", 2023-09-23T23:38:18.161 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0738e-05 2023-09-23T23:38:18.161 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.162 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.162 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.162 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859941+0000", 2023-09-23T23:38:18.162 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.24e-05 2023-09-23T23:38:18.162 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.162 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.162 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.162 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.887534+0000", 2023-09-23T23:38:18.162 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0275931309999999 2023-09-23T23:38:18.162 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.162 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.162 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.163 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.887728+0000", 2023-09-23T23:38:18.163 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000194165 2023-09-23T23:38:18.163 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.163 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.163 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:38:18.163 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.887881+0000", 2023-09-23T23:38:18.163 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000153263 2023-09-23T23:38:18.163 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.163 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.163 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:38:18.163 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.903469+0000", 2023-09-23T23:38:18.163 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015587215 2023-09-23T23:38:18.164 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.164 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.164 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.164 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912270+0000", 2023-09-23T23:38:18.164 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088015479999999993 2023-09-23T23:38:18.164 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.164 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.164 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.164 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.927101+0000", 2023-09-23T23:38:18.164 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014831195 2023-09-23T23:38:18.164 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.164 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.165 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:38:18.165 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.927169+0000", 2023-09-23T23:38:18.165 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8172999999999999e-05 2023-09-23T23:38:18.165 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.165 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.165 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.165 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.929522+0000", 2023-09-23T23:38:18.165 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023525769999999998 2023-09-23T23:38:18.165 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.165 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.165 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.165 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.166 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.166 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77576 13.d 13:b73edecb:::benchmark_data_smithi110_541928_object77575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:38:18.166 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.919341+0000", 2023-09-23T23:38:18.166 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 254.128781594, 2023-09-23T23:38:18.166 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0128839199999999, 2023-09-23T23:38:18.166 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.166 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:18.166 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:38:18.166 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:38:18.166 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:38:18.166 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77576 2023-09-23T23:38:18.166 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.167 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.167 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.167 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.167 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919341+0000", 2023-09-23T23:38:18.167 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.167 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.167 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.167 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.167 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919341+0000", 2023-09-23T23:38:18.167 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.167 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.167 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.168 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.168 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919339+0000", 2023-09-23T23:38:18.168 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:38:18.168 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.168 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.168 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.168 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919384+0000", 2023-09-23T23:38:18.168 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5042e-05 2023-09-23T23:38:18.168 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.168 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.168 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.168 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919403+0000", 2023-09-23T23:38:18.169 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9136000000000001e-05 2023-09-23T23:38:18.169 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.169 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.169 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.169 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919412+0000", 2023-09-23T23:38:18.169 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4589999999999998e-06 2023-09-23T23:38:18.169 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.169 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.169 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.169 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.924505+0000", 2023-09-23T23:38:18.169 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050933469999999998 2023-09-23T23:38:18.169 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.170 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.170 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:38:18.170 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.924519+0000", 2023-09-23T23:38:18.170 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3852000000000001e-05 2023-09-23T23:38:18.170 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.170 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.170 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.170 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889955+0000", 2023-09-23T23:38:18.170 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96543663400000002 2023-09-23T23:38:18.170 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.170 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.170 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.171 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.890057+0000", 2023-09-23T23:38:18.171 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010134899999999999 2023-09-23T23:38:18.171 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.171 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.171 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:38:18.171 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.890180+0000", 2023-09-23T23:38:18.171 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012325699999999999 2023-09-23T23:38:18.171 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.171 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.171 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:38:18.171 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.903476+0000", 2023-09-23T23:38:18.171 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013296290000000001 2023-09-23T23:38:18.172 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.172 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.172 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.172 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912329+0000", 2023-09-23T23:38:18.172 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088524599999999995 2023-09-23T23:38:18.172 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.172 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.172 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:18.172 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.929574+0000", 2023-09-23T23:38:18.172 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017245415 2023-09-23T23:38:18.172 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.173 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.173 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:38:18.173 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.929630+0000", 2023-09-23T23:38:18.173 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6097999999999997e-05 2023-09-23T23:38:18.173 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.173 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.173 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.173 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.932225+0000", 2023-09-23T23:38:18.173 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002594332 2023-09-23T23:38:18.173 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.173 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.173 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.173 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.174 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.174 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46514.0:135027 13.d e602/566 13:bef361b6:::benchmark_data_smithi110_541928_object47247:head v 602'8573, mlcod=0'0)", 2023-09-23T23:38:18.174 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:37:37.426291+0000", 2023-09-23T23:38:18.174 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 40.621831467, 2023-09-23T23:38:18.174 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.060379475000000002, 2023-09-23T23:38:18.174 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.174 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:18.174 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.174 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.174 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.174 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:37.426291+0000", 2023-09-23T23:38:18.175 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.175 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.175 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.175 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.175 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:37.426291+0000", 2023-09-23T23:38:18.175 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.175 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.175 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.175 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.175 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:37.426290+0000", 2023-09-23T23:38:18.175 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:18.175 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.175 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.176 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.176 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:37.426298+0000", 2023-09-23T23:38:18.176 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9729999999999997e-06 2023-09-23T23:38:18.176 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.176 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.176 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.176 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:37.478041+0000", 2023-09-23T23:38:18.176 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.051742429999999999 2023-09-23T23:38:18.176 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.176 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.176 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.176 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:37.478055+0000", 2023-09-23T23:38:18.177 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4411e-05 2023-09-23T23:38:18.177 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.177 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.177 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.177 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:37.478208+0000", 2023-09-23T23:38:18.177 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015337 2023-09-23T23:38:18.177 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.177 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.177 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.177 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:37.478238+0000", 2023-09-23T23:38:18.177 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9825999999999999e-05 2023-09-23T23:38:18.177 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.178 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.178 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:38:18.178 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:37.486621+0000", 2023-09-23T23:38:18.178 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0083832060000000007 2023-09-23T23:38:18.178 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.178 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.178 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.178 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:37.486670+0000", 2023-09-23T23:38:18.178 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8899000000000001e-05 2023-09-23T23:38:18.178 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.178 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.178 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.179 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.179 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.179 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46514.0:136629 13.a e603/557 13:5a36f06e:::benchmark_data_smithi110_541928_object48849:head v 603'8604, mlcod=549'1793)", 2023-09-23T23:38:18.179 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:37:42.080718+0000", 2023-09-23T23:38:18.179 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 35.967404788000003, 2023-09-23T23:38:18.179 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.15968157199999999, 2023-09-23T23:38:18.179 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.180 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:18.180 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.180 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.180 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.180 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.080718+0000", 2023-09-23T23:38:18.180 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.180 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.180 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.180 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.181 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.080718+0000", 2023-09-23T23:38:18.181 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.181 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.181 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.181 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.181 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.080717+0000", 2023-09-23T23:38:18.181 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:18.181 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.181 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.181 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.181 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.080723+0000", 2023-09-23T23:38:18.181 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4600000000000002e-06 2023-09-23T23:38:18.182 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.182 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.182 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.182 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.233482+0000", 2023-09-23T23:38:18.182 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.15275956500000001 2023-09-23T23:38:18.182 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.182 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.182 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.182 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.233496+0000", 2023-09-23T23:38:18.182 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3757000000000001e-05 2023-09-23T23:38:18.182 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.182 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.183 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.183 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.234982+0000", 2023-09-23T23:38:18.183 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014856870000000001 2023-09-23T23:38:18.183 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.183 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.183 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.183 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.235009+0000", 2023-09-23T23:38:18.183 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7449000000000001e-05 2023-09-23T23:38:18.183 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.183 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.183 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:38:18.183 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.240361+0000", 2023-09-23T23:38:18.184 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0053522359999999998 2023-09-23T23:38:18.184 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.184 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.184 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.184 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.240399+0000", 2023-09-23T23:38:18.184 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7944000000000001e-05 2023-09-23T23:38:18.184 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.184 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.184 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.184 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.184 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.184 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46514.0:136501 13.d e602/566 13:b5d13e05:::benchmark_data_smithi110_541928_object48721:head v 602'8678, mlcod=0'0)", 2023-09-23T23:38:18.185 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:37:42.096194+0000", 2023-09-23T23:38:18.185 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 35.951928379999998, 2023-09-23T23:38:18.185 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.15669300799999999, 2023-09-23T23:38:18.185 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.185 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:18.185 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.185 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.185 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.185 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.096194+0000", 2023-09-23T23:38:18.185 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.185 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.185 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.186 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.186 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.096194+0000", 2023-09-23T23:38:18.186 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.186 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.186 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.186 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.186 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.096193+0000", 2023-09-23T23:38:18.186 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:18.186 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.186 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.186 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.186 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.096203+0000", 2023-09-23T23:38:18.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2459999999999992e-06 2023-09-23T23:38:18.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.187 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.187 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.234813+0000", 2023-09-23T23:38:18.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.138610486 2023-09-23T23:38:18.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.187 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.187 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.234821+0000", 2023-09-23T23:38:18.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6909999999999997e-06 2023-09-23T23:38:18.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.188 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.235456+0000", 2023-09-23T23:38:18.188 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00063495000000000001 2023-09-23T23:38:18.188 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.188 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.188 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.235481+0000", 2023-09-23T23:38:18.188 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5316999999999998e-05 2023-09-23T23:38:18.188 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.188 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:38:18.188 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.252844+0000", 2023-09-23T23:38:18.189 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017363390999999999 2023-09-23T23:38:18.189 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.189 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.189 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.189 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:37:42.252887+0000", 2023-09-23T23:38:18.189 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2500000000000003e-05 2023-09-23T23:38:18.189 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.189 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.189 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.189 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.189 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.189 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46514.0:153463 13.3 e606/571 13:cf006423:::benchmark_data_smithi110_541928_object65683:head v 606'9682, mlcod=0'0)", 2023-09-23T23:38:18.190 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:38:16.630961+0000", 2023-09-23T23:38:18.190 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.41716171, 2023-09-23T23:38:18.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25026810500000002, 2023-09-23T23:38:18.190 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.190 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:18.190 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.190 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.190 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.190 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.630961+0000", 2023-09-23T23:38:18.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.190 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.190 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.191 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.630961+0000", 2023-09-23T23:38:18.191 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.191 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.630959+0000", 2023-09-23T23:38:18.191 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:38:18.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.191 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.630971+0000", 2023-09-23T23:38:18.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.207e-05 2023-09-23T23:38:18.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.630992+0000", 2023-09-23T23:38:18.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0177e-05 2023-09-23T23:38:18.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.631002+0000", 2023-09-23T23:38:18.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0772e-05 2023-09-23T23:38:18.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.193 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.193 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.631072+0000", 2023-09-23T23:38:18.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9306000000000004e-05 2023-09-23T23:38:18.193 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.193 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.193 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.631110+0000", 2023-09-23T23:38:18.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8408999999999998e-05 2023-09-23T23:38:18.193 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.193 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:38:18.194 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.881180+0000", 2023-09-23T23:38:18.194 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25006991499999998 2023-09-23T23:38:18.194 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.194 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.194 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.194 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.881229+0000", 2023-09-23T23:38:18.194 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8729000000000001e-05 2023-09-23T23:38:18.194 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.194 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.194 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.194 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.194 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.195 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(13.3 606/571 [PushOp(13:ce83adce:::benchmark_data_smithi110_541928_object65693:head, version: 554'4159, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(13:ce83adce:::benchmark_data_smithi110_541928_object65693:head@554'4159, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:38:18.195 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:38:16.632618+0000", 2023-09-23T23:38:18.195 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.415504004, 2023-09-23T23:38:18.195 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.248557219, 2023-09-23T23:38:18.195 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:38:18.195 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:38:18.195 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:38:18.195 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.195 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:38:18.195 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.632618+0000", 2023-09-23T23:38:18.195 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.196 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.196 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.196 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:38:18.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.632618+0000", 2023-09-23T23:38:18.196 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:38:18.196 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.196 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.196 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:38:18.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.632618+0000", 2023-09-23T23:38:18.196 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:18.196 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.197 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:38:18.197 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.632645+0000", 2023-09-23T23:38:18.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7268e-05 2023-09-23T23:38:18.197 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.197 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:38:18.197 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.875012+0000", 2023-09-23T23:38:18.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24236708800000001 2023-09-23T23:38:18.197 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.197 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:38:18.198 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.875052+0000", 2023-09-23T23:38:18.198 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9263e-05 2023-09-23T23:38:18.198 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.198 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.198 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:38:18.198 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.878339+0000", 2023-09-23T23:38:18.198 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032871229999999999 2023-09-23T23:38:18.198 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.198 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.198 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:38:18.198 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.878347+0000", 2023-09-23T23:38:18.198 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9410000000000007e-06 2023-09-23T23:38:18.199 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:38:18.199 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:38:18.199 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:38:18.199 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.881176+0000", 2023-09-23T23:38:18.199 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028289719999999999 2023-09-23T23:38:18.199 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.199 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.199 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.199 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:38:18.199 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:38:18.199 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:38:18.386 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:38:18.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:18.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:38:18.492 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:38:18.558 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:18.673 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:38:18.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:18.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:38:18.777 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:18.852 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:19.174 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:38:19.237 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:38:19.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:19 smithi138 ceph-mon[193606]: pgmap v1719: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 20709/70755 objects degraded (29.269%); 1350/70755 objects misplaced (1.908%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T23:38:19.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:19 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/95249878' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:19.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:19 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:19.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:19 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 20709/70755 objects degraded (29.269%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:19.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:19 smithi110 ceph-mon[221102]: pgmap v1719: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 20709/70755 objects degraded (29.269%); 1350/70755 objects misplaced (1.908%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T23:38:19.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:19 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/95249878' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:19 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:19 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 20709/70755 objects degraded (29.269%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:19.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:19 smithi110 ceph-mon[212939]: pgmap v1719: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 20709/70755 objects degraded (29.269%); 1350/70755 objects misplaced (1.908%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T23:38:19.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:19 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/95249878' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:19.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:19 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:19.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:19 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 20709/70755 objects degraded (29.269%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:19.817 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:19.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:19.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:19.925 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:38:20.011 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:20.143 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:38:20.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:20.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:38:20.265 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:38:20.345 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:20 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:38:20.345 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:20 smithi138 ceph-mon[193606]: failed to encode map e607 with expected crc 2023-09-23T23:38:20.345 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:20 smithi138 ceph-mon[193606]: failed to encode map e607 with expected crc 2023-09-23T23:38:20.345 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:20 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:38:20.345 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:20 smithi138 ceph-mon[193606]: osdmap e607: 8 total, 8 up, 7 in 2023-09-23T23:38:20.350 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:20.373 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:38:20.439 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:38:20.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:20.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:38:20.551 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:38:20.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:20 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:38:20.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:20 smithi110 ceph-mon[212939]: failed to encode map e607 with expected crc 2023-09-23T23:38:20.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:20 smithi110 ceph-mon[212939]: failed to encode map e607 with expected crc 2023-09-23T23:38:20.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:20 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:38:20.611 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:20 smithi110 ceph-mon[212939]: osdmap e607: 8 total, 8 up, 7 in 2023-09-23T23:38:20.613 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:20 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:38:20.614 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:20 smithi110 ceph-mon[221102]: failed to encode map e607 with expected crc 2023-09-23T23:38:20.614 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:20 smithi110 ceph-mon[221102]: failed to encode map e607 with expected crc 2023-09-23T23:38:20.614 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:20 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:38:20.614 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:20 smithi110 ceph-mon[221102]: osdmap e607: 8 total, 8 up, 7 in 2023-09-23T23:38:20.614 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:20.683 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:20.798 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:38:20.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:20.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:38:20.905 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:38:20.980 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:21.149 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:38:21.187 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:38:21.214 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:38:21.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:21.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:38:21.233 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:38:21.277 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:38:21.308 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:38:21.321 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:38:21.360 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:38:21.401 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:38:21.402 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:21.437 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:38:21.440 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:38:21.441 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:38:21.441 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:38:21.441 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:38:21.441 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:38:21.441 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:38:21.441 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:38:21.441 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:38:21.544 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:38:21.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:21.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:38:21.552 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:21 smithi138 ceph-mon[193606]: pgmap v1721: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.3 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 20290/63171 objects degraded (32.119%); 1058/63171 objects misplaced (1.675%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T23:38:21.552 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:21 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4187200156' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:21.653 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:38:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:21 smithi110 ceph-mon[212939]: pgmap v1721: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.3 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 20290/63171 objects degraded (32.119%); 1058/63171 objects misplaced (1.675%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T23:38:21.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:21 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4187200156' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:21.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:21 smithi110 ceph-mon[221102]: pgmap v1721: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.3 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 20290/63171 objects degraded (32.119%); 1058/63171 objects misplaced (1.675%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T23:38:21.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:21 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4187200156' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:21.712 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:22.357 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:38:22.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:22.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:38:22.436 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:38:22.470 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:38:22.506 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:23.056 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:23.066 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:38:23.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:23.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:38:23.199 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:38:23.260 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:23.570 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:38:23.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:23.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:38:23.679 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:38:23.721 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:23.739 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:23.824 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:38:23.828 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:23 smithi138 ceph-mon[193606]: pgmap v1722: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 20290/61308 objects degraded (33.095%); 1058/61308 objects misplaced (1.726%); 8.1 MiB/s, 128 objects/s recovering 2023-09-23T23:38:23.828 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:23 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:38:23.828 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:23 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:38:23.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:23.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:38:23.930 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:38:23.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:23 smithi110 ceph-mon[212939]: pgmap v1722: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 20290/61308 objects degraded (33.095%); 1058/61308 objects misplaced (1.726%); 8.1 MiB/s, 128 objects/s recovering 2023-09-23T23:38:23.956 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:23 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:38:23.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:23 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:38:23.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:23 smithi110 ceph-mon[221102]: pgmap v1722: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 1.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 20290/61308 objects degraded (33.095%); 1058/61308 objects misplaced (1.726%); 8.1 MiB/s, 128 objects/s recovering 2023-09-23T23:38:23.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:23 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:38:23.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:23 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:38:24.006 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:24.254 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:38:24.254 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:38:24.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:24 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 20290/61308 objects degraded (33.095%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:24.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:24 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 20290/61308 objects degraded (33.095%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:24.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:24 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 20290/61308 objects degraded (33.095%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:25.103 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:38:25.947 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:25 smithi138 ceph-mon[193606]: pgmap v1723: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 19967/58779 objects degraded (33.970%); 968/58779 objects misplaced (1.647%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T23:38:25.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:25 smithi110 ceph-mon[221102]: pgmap v1723: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 19967/58779 objects degraded (33.970%); 968/58779 objects misplaced (1.647%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T23:38:25.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:25 smithi110 ceph-mon[212939]: pgmap v1723: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 1.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 19967/58779 objects degraded (33.970%); 968/58779 objects misplaced (1.647%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T23:38:26.500 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:38:26.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:26.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:38:26.608 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:38:26.664 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:38:26.680 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:26.778 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:38:26.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:26.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:38:26.852 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:38:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:38:26] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-09-23T23:38:26.901 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:38:27.051 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:27.564 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:38:27.586 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:38:27.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:27 smithi138 ceph-mon[193606]: pgmap v1724: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 1.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 19564/50685 objects degraded (38.599%); 1015/50685 objects misplaced (2.003%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T23:38:27.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:27 smithi110 ceph-mon[212939]: pgmap v1724: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 1.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 19564/50685 objects degraded (38.599%); 1015/50685 objects misplaced (2.003%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T23:38:27.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:27 smithi110 ceph-mon[221102]: pgmap v1724: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 1.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 19564/50685 objects degraded (38.599%); 1015/50685 objects misplaced (2.003%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T23:38:28.456 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:38:28.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:28.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:38:28.568 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:38:28.621 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:28.726 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:38:28.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:28.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:38:28.835 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:28.883 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:28 smithi110 ceph-mon[212939]: from='client.47075 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:38:28.883 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:38:28.883 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:28 smithi110 ceph-mon[221102]: from='client.47075 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:38:28.883 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:38:28.895 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:28 smithi138 ceph-mon[193606]: from='client.47075 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:38:28.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:38:29.455 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:29.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:29.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:29.463 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:29.475 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1724,"stamp":"2023-09-23T23:38:25.962715+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1106228469,"num_objects":16895,"num_object_clones":0,"num_object_copies":50685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19564,"num_objects_misplaced":1015,"num_objects_unfound":0,"num_objects_dirty":16895,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":158762,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23029,"num_bytes_recovered":1506537896,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95209,"ondisk_log_size":95209,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9168772,"kb_used_data":3356108,"kb_used_omap":29,"kb_used_meta":5812130,"kb_avail":647047292,"statfs":{"total":671965249536,"available":662576427008,"internally_reserved":0,"allocated":3436654592,"data_stored":3420039490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":5951621843},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,2,5,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-438435840,"num_objects":-6690,"num_object_clones":0,"num_object_copies":-20070,"num_objects_missing_on_primary":-454,"num_objects_missing":-454,"num_objects_degraded":-1145,"num_objects_misplaced":-335,"num_objects_unfound":0,"num_objects_dirty":-6690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6690,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1481,"num_bytes_recovered":97058816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003122"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":803,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.178058+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:38:10.178058+0000","last_peered":"2023-09-23T23:38:10.178058+0000","last_clean":"2023-09-23T23:38:10.178058+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:38:10.178058+0000","last_undegraded":"2023-09-23T23:38:10.178058+0000","last_fullsized":"2023-09-23T23:38:10.178058+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":362,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.165183+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:38:10.165183+0000","last_peered":"2023-09-23T23:38:10.165183+0000","last_clean":"2023-09-23T23:38:10.165183+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:38:10.165183+0000","last_undegraded":"2023-09-23T23:38:10.165183+0000","last_fullsized":"2023-09-23T23:38:10.165183+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":589,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.373817+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:38:10.373817+0000","last_peered":"2023-09-23T23:38:10.373817+0000","last_clean":"2023-09-23T23:38:10.373817+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:38:10.373817+0000","last_undegraded":"2023-09-23T23:38:10.373817+0000","last_fullsized":"2023-09-23T23:38:10.373817+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":756,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.180146+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:38:10.180146+0000","last_peered":"2023-09-23T23:38:10.180146+0000","last_clean":"2023-09-23T23:38:10.180146+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:38:10.180146+0000","last_undegraded":"2023-09-23T23:38:10.180146+0000","last_fullsized":"2023-09-23T23:38:10.180146+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":326,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.369062+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:38:10.369062+0000","last_peered":"2023-09-23T23:38:10.369062+0000","last_clean":"2023-09-23T23:38:10.369062+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:38:10.369062+0000","last_undegraded":"2023-09-23T23:38:10.369062+0000","last_fullsized":"2023-09-23T23:38:10.369062+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":569,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.381084+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:38:19.381084+0000","last_peered":"2023-09-23T23:38:19.381084+0000","last_clean":"2023-09-23T23:38:19.381084+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:38:19.381084+0000","last_undegraded":"2023-09-23T23:38:19.381084+0000","last_fullsized":"2023-09-23T23:38:19.381084+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":768,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.189078+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:38:10.189078+0000","last_peered":"2023-09-23T23:38:10.189078+0000","last_clean":"2023-09-23T23:38:10.189078+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:38:10.189078+0000","last_undegraded":"2023-09-23T23:38:10.189078+0000","last_fullsized":"2023-09-23T23:38:10.189078+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":626,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.368172+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:38:10.368172+0000","last_peered":"2023-09-23T23:38:10.368172+0000","last_clean":"2023-09-23T23:38:10.368172+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:38:10.368172+0000","last_undegraded":"2023-09-23T23:38:10.368172+0000","last_fullsized":"2023-09-23T23:38:10.368172+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":121,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.196775+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:38:10.196775+0000","last_peered":"2023-09-23T23:38:10.196775+0000","last_clean":"2023-09-23T23:38:10.196775+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:38:10.196775+0000","last_undegraded":"2023-09-23T23:38:10.196775+0000","last_fullsized":"2023-09-23T23:38:10.196775+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":927,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.190599+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:38:10.190599+0000","last_peered":"2023-09-23T23:38:10.190599+0000","last_clean":"2023-09-23T23:38:10.190599+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:38:10.190599+0000","last_undegraded":"2023-09-23T23:38:10.190599+0000","last_fullsized":"2023-09-23T23:38:10.190599+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":259,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.178044+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:38:10.178044+0000","last_peered":"2023-09-23T23:38:10.178044+0000","last_clean":"2023-09-23T23:38:10.178044+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:38:10.178044+0000","last_undegraded":"2023-09-23T23:38:10.178044+0000","last_fullsized":"2023-09-23T23:38:10.178044+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":608,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.368126+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:38:10.368126+0000","last_peered":"2023-09-23T23:38:10.368126+0000","last_clean":"2023-09-23T23:38:10.368126+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:38:10.368126+0000","last_undegraded":"2023-09-23T23:38:10.368126+0000","last_fullsized":"2023-09-23T23:38:10.368126+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":901,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.163598+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:38:10.163598+0000","last_peered":"2023-09-23T23:38:10.163598+0000","last_clean":"2023-09-23T23:38:10.163598+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:38:10.163598+0000","last_undegraded":"2023-09-23T23:38:10.163598+0000","last_fullsized":"2023-09-23T23:38:10.163598+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":238,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.180550+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:38:10.180550+0000","last_peered":"2023-09-23T23:38:10.180550+0000","last_clean":"2023-09-23T23:38:10.180550+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:38:10.180550+0000","last_undegraded":"2023-09-23T23:38:10.180550+0000","last_fullsized":"2023-09-23T23:38:10.180550+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":569,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.368881+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:38:10.368881+0000","last_peered":"2023-09-23T23:38:10.368881+0000","last_clean":"2023-09-23T23:38:10.368881+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:38:10.368881+0000","last_undegraded":"2023-09-23T23:38:10.368881+0000","last_fullsized":"2023-09-23T23:38:10.368881+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1061,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.177902+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:38:10.177902+0000","last_peered":"2023-09-23T23:38:10.177902+0000","last_clean":"2023-09-23T23:38:10.177902+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:38:10.177902+0000","last_undegraded":"2023-09-23T23:38:10.177902+0000","last_fullsized":"2023-09-23T23:38:10.177902+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":280,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.371733+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:38:10.371733+0000","last_peered":"2023-09-23T23:38:10.371733+0000","last_clean":"2023-09-23T23:38:10.371733+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:38:10.371733+0000","last_undegraded":"2023-09-23T23:38:10.371733+0000","last_fullsized":"2023-09-23T23:38:10.371733+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":140,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.182550+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:38:10.182550+0000","last_peered":"2023-09-23T23:38:10.182550+0000","last_clean":"2023-09-23T23:38:10.182550+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:38:10.182550+0000","last_undegraded":"2023-09-23T23:38:10.182550+0000","last_fullsized":"2023-09-23T23:38:10.182550+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":458,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.375319+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:38:10.375319+0000","last_peered":"2023-09-23T23:38:10.375319+0000","last_clean":"2023-09-23T23:38:10.375319+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:38:10.375319+0000","last_undegraded":"2023-09-23T23:38:10.375319+0000","last_fullsized":"2023-09-23T23:38:10.375319+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":181,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.179435+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:38:10.179435+0000","last_peered":"2023-09-23T23:38:10.179435+0000","last_clean":"2023-09-23T23:38:10.179435+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:38:10.179435+0000","last_undegraded":"2023-09-23T23:38:10.179435+0000","last_fullsized":"2023-09-23T23:38:10.179435+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":548,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.179150+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:38:10.179150+0000","last_peered":"2023-09-23T23:38:10.179150+0000","last_clean":"2023-09-23T23:38:10.179150+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:38:10.179150+0000","last_undegraded":"2023-09-23T23:38:10.179150+0000","last_fullsized":"2023-09-23T23:38:10.179150+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":762,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.177562+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:38:10.177562+0000","last_peered":"2023-09-23T23:38:10.177562+0000","last_clean":"2023-09-23T23:38:10.177562+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:38:10.177562+0000","last_undegraded":"2023-09-23T23:38:10.177562+0000","last_fullsized":"2023-09-23T23:38:10.177562+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":154,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.165352+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:38:10.165352+0000","last_peered":"2023-09-23T23:38:10.165352+0000","last_clean":"2023-09-23T23:38:10.165352+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:38:10.165352+0000","last_undegraded":"2023-09-23T23:38:10.165352+0000","last_fullsized":"2023-09-23T23:38:10.165352+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":607,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.369245+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:38:10.369245+0000","last_peered":"2023-09-23T23:38:10.369245+0000","last_clean":"2023-09-23T23:38:10.369245+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:38:10.369245+0000","last_undegraded":"2023-09-23T23:38:10.369245+0000","last_fullsized":"2023-09-23T23:38:10.369245+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":892,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.178807+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:38:10.178807+0000","last_peered":"2023-09-23T23:38:10.178807+0000","last_clean":"2023-09-23T23:38:10.178807+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:38:10.178807+0000","last_undegraded":"2023-09-23T23:38:10.178807+0000","last_fullsized":"2023-09-23T23:38:10.178807+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":202,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.373840+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:38:10.373840+0000","last_peered":"2023-09-23T23:38:10.373840+0000","last_clean":"2023-09-23T23:38:10.373840+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:38:10.373840+0000","last_undegraded":"2023-09-23T23:38:10.373840+0000","last_fullsized":"2023-09-23T23:38:10.373840+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":329,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.373692+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:38:10.373692+0000","last_peered":"2023-09-23T23:38:10.373692+0000","last_clean":"2023-09-23T23:38:10.373692+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:38:10.373692+0000","last_undegraded":"2023-09-23T23:38:10.373692+0000","last_fullsized":"2023-09-23T23:38:10.373692+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":570,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.371174+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:38:10.371174+0000","last_peered":"2023-09-23T23:38:10.371174+0000","last_clean":"2023-09-23T23:38:10.371174+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:38:10.371174+0000","last_undegraded":"2023-09-23T23:38:10.371174+0000","last_fullsized":"2023-09-23T23:38:10.371174+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":475,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.187273+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:38:10.187273+0000","last_peered":"2023-09-23T23:38:10.187273+0000","last_clean":"2023-09-23T23:38:10.187273+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:38:10.187273+0000","last_undegraded":"2023-09-23T23:38:10.187273+0000","last_fullsized":"2023-09-23T23:38:10.187273+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":564,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.375375+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:38:10.375375+0000","last_peered":"2023-09-23T23:38:10.375375+0000","last_clean":"2023-09-23T23:38:10.375375+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:38:10.375375+0000","last_undegraded":"2023-09-23T23:38:10.375375+0000","last_fullsized":"2023-09-23T23:38:10.375375+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":970,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.177636+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:38:10.177636+0000","last_peered":"2023-09-23T23:38:10.177636+0000","last_clean":"2023-09-23T23:38:10.177636+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:38:10.177636+0000","last_undegraded":"2023-09-23T23:38:10.177636+0000","last_fullsized":"2023-09-23T23:38:10.177636+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":171,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.179119+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:38:10.179119+0000","last_peered":"2023-09-23T23:38:10.179119+0000","last_clean":"2023-09-23T23:38:10.179119+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:38:10.179119+0000","last_undegraded":"2023-09-23T23:38:10.179119+0000","last_fullsized":"2023-09-23T23:38:10.179119+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":249,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:13.340311+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:38:13.340311+0000","last_peered":"2023-09-23T23:38:13.340311+0000","last_clean":"2023-09-23T23:38:13.340311+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:38:13.340311+0000","last_undegraded":"2023-09-23T23:38:13.340311+0000","last_fullsized":"2023-09-23T23:38:13.340311+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"607'9774","reported_seq":10055,"reported_epoch":607,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:24.239730+0000","last_change":"2023-09-23T23:38:22.063474+0000","last_active":"2023-09-23T23:38:24.239730+0000","last_peered":"2023-09-23T23:38:24.239730+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:38:24.239730+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"555'4300","ondisk_log_start":"555'4300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5474,"log_dups_size":0,"ondisk_log_size":5474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70385664,"num_objects":1074,"num_object_clones":0,"num_object_copies":3222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4256,"num_objects_misplaced":47,"num_objects_unfound":0,"num_objects_dirty":1074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9774,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":95,"num_bytes_recovered":6225920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":47},{"shards":"2,6","objects":-3229}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":941,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.163472+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:38:10.163472+0000","last_peered":"2023-09-23T23:38:10.163472+0000","last_clean":"2023-09-23T23:38:10.163472+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:38:10.163472+0000","last_undegraded":"2023-09-23T23:38:10.163472+0000","last_fullsized":"2023-09-23T23:38:10.163472+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":248,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.190554+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:38:10.190554+0000","last_peered":"2023-09-23T23:38:10.190554+0000","last_clean":"2023-09-23T23:38:10.190554+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:38:10.190554+0000","last_undegraded":"2023-09-23T23:38:10.190554+0000","last_fullsized":"2023-09-23T23:38:10.190554+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":238,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.177986+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:38:10.177986+0000","last_peered":"2023-09-23T23:38:10.177986+0000","last_clean":"2023-09-23T23:38:10.177986+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:38:10.177986+0000","last_undegraded":"2023-09-23T23:38:10.177986+0000","last_fullsized":"2023-09-23T23:38:10.177986+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"607'10037","reported_seq":10128,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:25.100769+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:38:25.100769+0000","last_peered":"2023-09-23T23:38:25.100769+0000","last_clean":"2023-09-23T23:38:25.100769+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:38:25.100769+0000","last_undegraded":"2023-09-23T23:38:25.100769+0000","last_fullsized":"2023-09-23T23:38:25.100769+0000","mapping_epoch":566,"log_start":"552'3900","ondisk_log_start":"552'3900","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6137,"log_dups_size":0,"ondisk_log_size":6137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65208320,"num_objects":995,"num_object_clones":0,"num_object_copies":2985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10037,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1020,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.368613+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:38:10.368613+0000","last_peered":"2023-09-23T23:38:10.368613+0000","last_clean":"2023-09-23T23:38:10.368613+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:38:10.368613+0000","last_undegraded":"2023-09-23T23:38:10.368613+0000","last_fullsized":"2023-09-23T23:38:10.368613+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":238,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.189400+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:38:10.189400+0000","last_peered":"2023-09-23T23:38:10.189400+0000","last_clean":"2023-09-23T23:38:10.189400+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:38:10.189400+0000","last_undegraded":"2023-09-23T23:38:10.189400+0000","last_fullsized":"2023-09-23T23:38:10.189400+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":458,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.368099+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:38:10.368099+0000","last_peered":"2023-09-23T23:38:10.368099+0000","last_clean":"2023-09-23T23:38:10.368099+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:38:10.368099+0000","last_undegraded":"2023-09-23T23:38:10.368099+0000","last_fullsized":"2023-09-23T23:38:10.368099+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"607'9944","reported_seq":19911,"reported_epoch":607,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:38:23.356830+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:38:23.356830+0000","last_peered":"2023-09-23T23:38:23.356830+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:38:23.356830+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:38:23.356830+0000","mapping_epoch":571,"log_start":"560'4600","ondisk_log_start":"560'4600","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5344,"log_dups_size":0,"ondisk_log_size":5344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72351744,"num_objects":1104,"num_object_clones":0,"num_object_copies":3312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":208,"num_objects_misplaced":896,"num_objects_unfound":0,"num_objects_dirty":1104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9944,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4923,"num_bytes_recovered":322633728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":86},{"shards":"3,5,7","objects":1018}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":917,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.196853+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:38:10.196853+0000","last_peered":"2023-09-23T23:38:10.196853+0000","last_clean":"2023-09-23T23:38:10.196853+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:38:10.196853+0000","last_undegraded":"2023-09-23T23:38:10.196853+0000","last_fullsized":"2023-09-23T23:38:10.196853+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":345,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.185155+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:38:10.185155+0000","last_peered":"2023-09-23T23:38:10.185155+0000","last_clean":"2023-09-23T23:38:10.185155+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:38:10.185155+0000","last_undegraded":"2023-09-23T23:38:10.185155+0000","last_fullsized":"2023-09-23T23:38:10.185155+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":121,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.180879+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:38:10.180879+0000","last_peered":"2023-09-23T23:38:10.180879+0000","last_clean":"2023-09-23T23:38:10.180879+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:38:10.180879+0000","last_undegraded":"2023-09-23T23:38:10.180879+0000","last_fullsized":"2023-09-23T23:38:10.180879+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"607'10006","reported_seq":12176,"reported_epoch":607,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:23.339601+0000","last_change":"2023-09-23T23:38:22.122318+0000","last_active":"2023-09-23T23:38:23.339601+0000","last_peered":"2023-09-23T23:38:23.339601+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:38:23.339601+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"560'4700","ondisk_log_start":"560'4700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5306,"log_dups_size":0,"ondisk_log_size":5306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71041024,"num_objects":1084,"num_object_clones":0,"num_object_copies":3252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3426,"num_objects_misplaced":72,"num_objects_unfound":0,"num_objects_dirty":1084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10006,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1037,"num_bytes_recovered":67960832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":72},{"shards":"3,5","objects":-2414}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":514,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.375301+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:38:10.375301+0000","last_peered":"2023-09-23T23:38:10.375301+0000","last_clean":"2023-09-23T23:38:10.375301+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:38:10.375301+0000","last_undegraded":"2023-09-23T23:38:10.375301+0000","last_fullsized":"2023-09-23T23:38:10.375301+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":448,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.369171+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:38:10.369171+0000","last_peered":"2023-09-23T23:38:10.369171+0000","last_clean":"2023-09-23T23:38:10.369171+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:38:10.369171+0000","last_undegraded":"2023-09-23T23:38:10.369171+0000","last_fullsized":"2023-09-23T23:38:10.369171+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":605,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.183807+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:38:10.183807+0000","last_peered":"2023-09-23T23:38:10.183807+0000","last_clean":"2023-09-23T23:38:10.183807+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:38:10.183807+0000","last_undegraded":"2023-09-23T23:38:10.183807+0000","last_fullsized":"2023-09-23T23:38:10.183807+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"607'10095","reported_seq":11432,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:24.611443+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:38:24.611443+0000","last_peered":"2023-09-23T23:38:24.611443+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:38:24.611443+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"555'4400","ondisk_log_start":"555'4400","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5695,"log_dups_size":0,"ondisk_log_size":5695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69926912,"num_objects":1067,"num_object_clones":0,"num_object_copies":3201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10095,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":21},{"shards":"3,5","objects":-4393},{"shards":"3,5,6","objects":1046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":735,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.368812+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:38:10.368812+0000","last_peered":"2023-09-23T23:38:10.368812+0000","last_clean":"2023-09-23T23:38:10.368812+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:38:10.368812+0000","last_undegraded":"2023-09-23T23:38:10.368812+0000","last_fullsized":"2023-09-23T23:38:10.368812+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":618,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.371087+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:38:10.371087+0000","last_peered":"2023-09-23T23:38:10.371087+0000","last_clean":"2023-09-23T23:38:10.371087+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:38:10.371087+0000","last_undegraded":"2023-09-23T23:38:10.371087+0000","last_fullsized":"2023-09-23T23:38:10.371087+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":529,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.376804+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:38:19.376804+0000","last_peered":"2023-09-23T23:38:19.376804+0000","last_clean":"2023-09-23T23:38:19.376804+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:38:19.376804+0000","last_undegraded":"2023-09-23T23:38:19.376804+0000","last_fullsized":"2023-09-23T23:38:19.376804+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"607'10058","reported_seq":10139,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:24.199148+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:38:24.199148+0000","last_peered":"2023-09-23T23:38:24.199148+0000","last_clean":"2023-09-23T23:38:24.199148+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:38:24.199148+0000","last_undegraded":"2023-09-23T23:38:24.199148+0000","last_fullsized":"2023-09-23T23:38:24.199148+0000","mapping_epoch":534,"log_start":"559'4600","ondisk_log_start":"559'4600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5458,"log_dups_size":0,"ondisk_log_size":5458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66977792,"num_objects":1022,"num_object_clones":0,"num_object_copies":3066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10058,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":891,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.163374+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:38:10.163374+0000","last_peered":"2023-09-23T23:38:10.163374+0000","last_clean":"2023-09-23T23:38:10.163374+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:38:10.163374+0000","last_undegraded":"2023-09-23T23:38:10.163374+0000","last_fullsized":"2023-09-23T23:38:10.163374+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":131,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.180815+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:38:10.180815+0000","last_peered":"2023-09-23T23:38:10.180815+0000","last_clean":"2023-09-23T23:38:10.180815+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:38:10.180815+0000","last_undegraded":"2023-09-23T23:38:10.180815+0000","last_fullsized":"2023-09-23T23:38:10.180815+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":474,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.393122+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:38:19.393122+0000","last_peered":"2023-09-23T23:38:19.393122+0000","last_clean":"2023-09-23T23:38:19.393122+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:38:19.393122+0000","last_undegraded":"2023-09-23T23:38:19.393122+0000","last_fullsized":"2023-09-23T23:38:19.393122+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"607'10057","reported_seq":19256,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:24.213461+0000","last_change":"2023-09-23T23:38:22.063236+0000","last_active":"2023-09-23T23:38:24.213461+0000","last_peered":"2023-09-23T23:38:24.213461+0000","last_clean":"2023-09-23T23:38:24.213461+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:38:24.213461+0000","last_undegraded":"2023-09-23T23:38:24.213461+0000","last_fullsized":"2023-09-23T23:38:24.213461+0000","mapping_epoch":567,"log_start":"559'4600","ondisk_log_start":"559'4600","created":534,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5457,"log_dups_size":0,"ondisk_log_size":5457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68616192,"num_objects":1047,"num_object_clones":0,"num_object_copies":3141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10057,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4547,"num_bytes_recovered":297992192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":615,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.180074+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:38:10.180074+0000","last_peered":"2023-09-23T23:38:10.180074+0000","last_clean":"2023-09-23T23:38:10.180074+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:38:10.180074+0000","last_undegraded":"2023-09-23T23:38:10.180074+0000","last_fullsized":"2023-09-23T23:38:10.180074+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":471,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:15.117217+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:38:15.117217+0000","last_peered":"2023-09-23T23:38:15.117217+0000","last_clean":"2023-09-23T23:38:15.117217+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:38:15.117217+0000","last_undegraded":"2023-09-23T23:38:15.117217+0000","last_fullsized":"2023-09-23T23:38:15.117217+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":558,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.182327+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:38:10.182327+0000","last_peered":"2023-09-23T23:38:10.182327+0000","last_clean":"2023-09-23T23:38:10.182327+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:38:10.182327+0000","last_undegraded":"2023-09-23T23:38:10.182327+0000","last_fullsized":"2023-09-23T23:38:10.182327+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"607'9961","reported_seq":19005,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:24.604165+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:38:24.604165+0000","last_peered":"2023-09-23T23:38:24.604165+0000","last_clean":"2023-09-23T23:38:24.604165+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:38:24.604165+0000","last_undegraded":"2023-09-23T23:38:24.604165+0000","last_fullsized":"2023-09-23T23:38:24.604165+0000","mapping_epoch":579,"log_start":"555'4300","ondisk_log_start":"555'4300","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5661,"log_dups_size":0,"ondisk_log_size":5661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69140480,"num_objects":1055,"num_object_clones":0,"num_object_copies":3165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9961,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":949,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.165035+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:38:10.165035+0000","last_peered":"2023-09-23T23:38:10.165035+0000","last_clean":"2023-09-23T23:38:10.165035+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:38:10.165035+0000","last_undegraded":"2023-09-23T23:38:10.165035+0000","last_fullsized":"2023-09-23T23:38:10.165035+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":249,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.368061+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:38:10.368061+0000","last_peered":"2023-09-23T23:38:10.368061+0000","last_clean":"2023-09-23T23:38:10.368061+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:38:10.368061+0000","last_undegraded":"2023-09-23T23:38:10.368061+0000","last_fullsized":"2023-09-23T23:38:10.368061+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":537,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.374994+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:38:10.374994+0000","last_peered":"2023-09-23T23:38:10.374994+0000","last_clean":"2023-09-23T23:38:10.374994+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:38:10.374994+0000","last_undegraded":"2023-09-23T23:38:10.374994+0000","last_fullsized":"2023-09-23T23:38:10.374994+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"607'9630","reported_seq":9714,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:23.319139+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:38:23.319139+0000","last_peered":"2023-09-23T23:38:23.319139+0000","last_clean":"2023-09-23T23:38:23.319139+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:38:23.319139+0000","last_undegraded":"2023-09-23T23:38:23.319139+0000","last_fullsized":"2023-09-23T23:38:23.319139+0000","mapping_epoch":534,"log_start":"555'4300","ondisk_log_start":"555'4300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5330,"log_dups_size":0,"ondisk_log_size":5330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71958528,"num_objects":1098,"num_object_clones":0,"num_object_copies":3294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9630,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":928,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.192883+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:38:10.192883+0000","last_peered":"2023-09-23T23:38:10.192883+0000","last_clean":"2023-09-23T23:38:10.192883+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:38:10.192883+0000","last_undegraded":"2023-09-23T23:38:10.192883+0000","last_fullsized":"2023-09-23T23:38:10.192883+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":317,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.355245+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:38:10.355245+0000","last_peered":"2023-09-23T23:38:10.355245+0000","last_clean":"2023-09-23T23:38:10.355245+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:38:10.355245+0000","last_undegraded":"2023-09-23T23:38:10.355245+0000","last_fullsized":"2023-09-23T23:38:10.355245+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":622,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.185279+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:38:10.185279+0000","last_peered":"2023-09-23T23:38:10.185279+0000","last_clean":"2023-09-23T23:38:10.185279+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:38:10.185279+0000","last_undegraded":"2023-09-23T23:38:10.185279+0000","last_fullsized":"2023-09-23T23:38:10.185279+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":203,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.177351+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:38:10.177351+0000","last_peered":"2023-09-23T23:38:10.177351+0000","last_clean":"2023-09-23T23:38:10.177351+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:38:10.177351+0000","last_undegraded":"2023-09-23T23:38:10.177351+0000","last_fullsized":"2023-09-23T23:38:10.177351+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"607'10014","reported_seq":19151,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:23.329664+0000","last_change":"2023-09-23T23:38:20.181875+0000","last_active":"2023-09-23T23:38:23.329664+0000","last_peered":"2023-09-23T23:38:23.329664+0000","last_clean":"2023-09-23T23:38:23.329664+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:38:23.329664+0000","last_undegraded":"2023-09-23T23:38:23.329664+0000","last_fullsized":"2023-09-23T23:38:23.329664+0000","mapping_epoch":566,"log_start":"560'4722","ondisk_log_start":"560'4722","created":534,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5292,"log_dups_size":0,"ondisk_log_size":5292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75563032,"num_objects":1154,"num_object_clones":0,"num_object_copies":3462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1154,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10014,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4515,"num_bytes_recovered":295895040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":691,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.183199+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:38:10.183199+0000","last_peered":"2023-09-23T23:38:10.183199+0000","last_clean":"2023-09-23T23:38:10.183199+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:38:10.183199+0000","last_undegraded":"2023-09-23T23:38:10.183199+0000","last_fullsized":"2023-09-23T23:38:10.183199+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":255,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.180741+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:38:10.180741+0000","last_peered":"2023-09-23T23:38:10.180741+0000","last_clean":"2023-09-23T23:38:10.180741+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:38:10.180741+0000","last_undegraded":"2023-09-23T23:38:10.180741+0000","last_fullsized":"2023-09-23T23:38:10.180741+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":539,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.179231+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:38:10.179231+0000","last_peered":"2023-09-23T23:38:10.179231+0000","last_clean":"2023-09-23T23:38:10.179231+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:38:10.179231+0000","last_undegraded":"2023-09-23T23:38:10.179231+0000","last_fullsized":"2023-09-23T23:38:10.179231+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":241,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:15.117301+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:38:15.117301+0000","last_peered":"2023-09-23T23:38:15.117301+0000","last_clean":"2023-09-23T23:38:15.117301+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:38:15.117301+0000","last_undegraded":"2023-09-23T23:38:15.117301+0000","last_fullsized":"2023-09-23T23:38:15.117301+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"607'9733","reported_seq":11793,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:24.621280+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:38:24.621280+0000","last_peered":"2023-09-23T23:38:24.621280+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:38:24.621280+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"553'4000","ondisk_log_start":"553'4000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5733,"log_dups_size":0,"ondisk_log_size":5733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68091904,"num_objects":1039,"num_object_clones":0,"num_object_copies":3117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9733,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":28},{"shards":"5,7","objects":-2353}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":909,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.164952+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:38:10.164952+0000","last_peered":"2023-09-23T23:38:10.164952+0000","last_clean":"2023-09-23T23:38:10.164952+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:38:10.164952+0000","last_undegraded":"2023-09-23T23:38:10.164952+0000","last_fullsized":"2023-09-23T23:38:10.164952+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":346,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.167861+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:38:10.167861+0000","last_peered":"2023-09-23T23:38:10.167861+0000","last_clean":"2023-09-23T23:38:10.167861+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:38:10.167861+0000","last_undegraded":"2023-09-23T23:38:10.167861+0000","last_fullsized":"2023-09-23T23:38:10.167861+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":256,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.163367+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:38:10.163367+0000","last_peered":"2023-09-23T23:38:10.163367+0000","last_clean":"2023-09-23T23:38:10.163367+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:38:10.163367+0000","last_undegraded":"2023-09-23T23:38:10.163367+0000","last_fullsized":"2023-09-23T23:38:10.163367+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":220,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:13.067392+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:38:13.067392+0000","last_peered":"2023-09-23T23:38:13.067392+0000","last_clean":"2023-09-23T23:38:13.067392+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:38:13.067392+0000","last_undegraded":"2023-09-23T23:38:13.067392+0000","last_fullsized":"2023-09-23T23:38:13.067392+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"607'9908","reported_seq":9999,"reported_epoch":607,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:38:24.605951+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:38:24.605951+0000","last_peered":"2023-09-23T23:38:24.605951+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:38:24.605951+0000","last_undegraded":"2023-09-23T23:38:24.605951+0000","last_fullsized":"2023-09-23T23:38:24.605951+0000","mapping_epoch":562,"log_start":"555'4200","ondisk_log_start":"555'4200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5708,"log_dups_size":0,"ondisk_log_size":5708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65142784,"num_objects":994,"num_object_clones":0,"num_object_copies":2982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9908,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":994}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":623,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.165424+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:38:10.165424+0000","last_peered":"2023-09-23T23:38:10.165424+0000","last_clean":"2023-09-23T23:38:10.165424+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:38:10.165424+0000","last_undegraded":"2023-09-23T23:38:10.165424+0000","last_fullsized":"2023-09-23T23:38:10.165424+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":893,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.178886+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:38:10.178886+0000","last_peered":"2023-09-23T23:38:10.178886+0000","last_clean":"2023-09-23T23:38:10.178886+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:38:10.178886+0000","last_undegraded":"2023-09-23T23:38:10.178886+0000","last_fullsized":"2023-09-23T23:38:10.178886+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":570,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.373679+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:38:10.373679+0000","last_peered":"2023-09-23T23:38:10.373679+0000","last_clean":"2023-09-23T23:38:10.373679+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:38:10.373679+0000","last_undegraded":"2023-09-23T23:38:10.373679+0000","last_fullsized":"2023-09-23T23:38:10.373679+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":131,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.180654+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:38:10.180654+0000","last_peered":"2023-09-23T23:38:10.180654+0000","last_clean":"2023-09-23T23:38:10.180654+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:38:10.180654+0000","last_undegraded":"2023-09-23T23:38:10.180654+0000","last_fullsized":"2023-09-23T23:38:10.180654+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"607'9668","reported_seq":9749,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:24.205699+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:38:24.205699+0000","last_peered":"2023-09-23T23:38:24.205699+0000","last_clean":"2023-09-23T23:38:24.205699+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:38:24.205699+0000","last_undegraded":"2023-09-23T23:38:24.205699+0000","last_fullsized":"2023-09-23T23:38:24.205699+0000","mapping_epoch":534,"log_start":"555'4200","ondisk_log_start":"555'4200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5468,"log_dups_size":0,"ondisk_log_size":5468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69992448,"num_objects":1068,"num_object_clones":0,"num_object_copies":3204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9668,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":719,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.179056+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:38:10.179056+0000","last_peered":"2023-09-23T23:38:10.179056+0000","last_clean":"2023-09-23T23:38:10.179056+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:38:10.179056+0000","last_undegraded":"2023-09-23T23:38:10.179056+0000","last_fullsized":"2023-09-23T23:38:10.179056+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":534,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.188837+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:38:10.188837+0000","last_peered":"2023-09-23T23:38:10.188837+0000","last_clean":"2023-09-23T23:38:10.188837+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:38:10.188837+0000","last_undegraded":"2023-09-23T23:38:10.188837+0000","last_fullsized":"2023-09-23T23:38:10.188837+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":449,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.368606+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:38:10.368606+0000","last_peered":"2023-09-23T23:38:10.368606+0000","last_clean":"2023-09-23T23:38:10.368606+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:38:10.368606+0000","last_undegraded":"2023-09-23T23:38:10.368606+0000","last_fullsized":"2023-09-23T23:38:10.368606+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":48,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.350845+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:38:10.350845+0000","last_peered":"2023-09-23T23:38:10.350845+0000","last_clean":"2023-09-23T23:38:10.350845+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:38:10.350845+0000","last_undegraded":"2023-09-23T23:38:10.350845+0000","last_fullsized":"2023-09-23T23:38:10.350845+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"607'10002","reported_seq":13683,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:25.099014+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:38:25.099014+0000","last_peered":"2023-09-23T23:38:25.099014+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:38:25.099014+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"552'3800","ondisk_log_start":"552'3800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6202,"log_dups_size":0,"ondisk_log_size":6202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65273856,"num_objects":996,"num_object_clones":0,"num_object_copies":2988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10002,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":-1708}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":602,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.182477+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:38:10.182477+0000","last_peered":"2023-09-23T23:38:10.182477+0000","last_clean":"2023-09-23T23:38:10.182477+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:38:10.182477+0000","last_undegraded":"2023-09-23T23:38:10.182477+0000","last_fullsized":"2023-09-23T23:38:10.182477+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":48,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.183727+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:38:10.183727+0000","last_peered":"2023-09-23T23:38:10.183727+0000","last_clean":"2023-09-23T23:38:10.183727+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:38:10.183727+0000","last_undegraded":"2023-09-23T23:38:10.183727+0000","last_fullsized":"2023-09-23T23:38:10.183727+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":148,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:15.117318+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:38:15.117318+0000","last_peered":"2023-09-23T23:38:15.117318+0000","last_clean":"2023-09-23T23:38:15.117318+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:38:15.117318+0000","last_undegraded":"2023-09-23T23:38:15.117318+0000","last_fullsized":"2023-09-23T23:38:15.117318+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":340,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.180255+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:38:10.180255+0000","last_peered":"2023-09-23T23:38:10.180255+0000","last_clean":"2023-09-23T23:38:10.180255+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:38:10.180255+0000","last_undegraded":"2023-09-23T23:38:10.180255+0000","last_fullsized":"2023-09-23T23:38:10.180255+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"607'9945","reported_seq":10025,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:24.621321+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:38:24.621321+0000","last_peered":"2023-09-23T23:38:24.621321+0000","last_clean":"2023-09-23T23:38:24.621321+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:38:24.621321+0000","last_undegraded":"2023-09-23T23:38:24.621321+0000","last_fullsized":"2023-09-23T23:38:24.621321+0000","mapping_epoch":534,"log_start":"555'4200","ondisk_log_start":"555'4200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5745,"log_dups_size":0,"ondisk_log_size":5745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66781184,"num_objects":1019,"num_object_clones":0,"num_object_copies":3057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9945,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":789,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.163766+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:38:10.163766+0000","last_peered":"2023-09-23T23:38:10.163766+0000","last_clean":"2023-09-23T23:38:10.163766+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:38:10.163766+0000","last_undegraded":"2023-09-23T23:38:10.163766+0000","last_fullsized":"2023-09-23T23:38:10.163766+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":605,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.374991+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:38:10.374991+0000","last_peered":"2023-09-23T23:38:10.374991+0000","last_clean":"2023-09-23T23:38:10.374991+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:38:10.374991+0000","last_undegraded":"2023-09-23T23:38:10.374991+0000","last_fullsized":"2023-09-23T23:38:10.374991+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":211,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.182350+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:38:10.182350+0000","last_peered":"2023-09-23T23:38:10.182350+0000","last_clean":"2023-09-23T23:38:10.182350+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:38:10.182350+0000","last_undegraded":"2023-09-23T23:38:10.182350+0000","last_fullsized":"2023-09-23T23:38:10.182350+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":159,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.163271+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:38:10.163271+0000","last_peered":"2023-09-23T23:38:10.163271+0000","last_clean":"2023-09-23T23:38:10.163271+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:38:10.163271+0000","last_undegraded":"2023-09-23T23:38:10.163271+0000","last_fullsized":"2023-09-23T23:38:10.163271+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"607'9853","reported_seq":9937,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:25.067836+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:38:25.067836+0000","last_peered":"2023-09-23T23:38:25.067836+0000","last_clean":"2023-09-23T23:38:25.067836+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:38:25.067836+0000","last_undegraded":"2023-09-23T23:38:25.067836+0000","last_fullsized":"2023-09-23T23:38:25.067836+0000","mapping_epoch":534,"log_start":"552'3700","ondisk_log_start":"552'3700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6153,"log_dups_size":0,"ondisk_log_size":6153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69140480,"num_objects":1055,"num_object_clones":0,"num_object_copies":3165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9853,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":995,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.369319+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:38:10.369319+0000","last_peered":"2023-09-23T23:38:10.369319+0000","last_clean":"2023-09-23T23:38:10.369319+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:38:10.369319+0000","last_undegraded":"2023-09-23T23:38:10.369319+0000","last_fullsized":"2023-09-23T23:38:10.369319+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":121,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.193004+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:38:10.193004+0000","last_peered":"2023-09-23T23:38:10.193004+0000","last_clean":"2023-09-23T23:38:10.193004+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:38:10.193004+0000","last_undegraded":"2023-09-23T23:38:10.193004+0000","last_fullsized":"2023-09-23T23:38:10.193004+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":52,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.350964+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:38:10.350964+0000","last_peered":"2023-09-23T23:38:10.350964+0000","last_clean":"2023-09-23T23:38:10.350964+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:38:10.350964+0000","last_undegraded":"2023-09-23T23:38:10.350964+0000","last_fullsized":"2023-09-23T23:38:10.350964+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":181,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.177653+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:38:10.177653+0000","last_peered":"2023-09-23T23:38:10.177653+0000","last_clean":"2023-09-23T23:38:10.177653+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:38:10.177653+0000","last_undegraded":"2023-09-23T23:38:10.177653+0000","last_fullsized":"2023-09-23T23:38:10.177653+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1044,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.368740+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:38:10.368740+0000","last_peered":"2023-09-23T23:38:10.368740+0000","last_clean":"2023-09-23T23:38:10.368740+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:38:10.368740+0000","last_undegraded":"2023-09-23T23:38:10.368740+0000","last_fullsized":"2023-09-23T23:38:10.368740+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":48,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.350729+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:38:10.350729+0000","last_peered":"2023-09-23T23:38:10.350729+0000","last_clean":"2023-09-23T23:38:10.350729+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:38:10.350729+0000","last_undegraded":"2023-09-23T23:38:10.350729+0000","last_fullsized":"2023-09-23T23:38:10.350729+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":363,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.163751+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:38:10.163751+0000","last_peered":"2023-09-23T23:38:10.163751+0000","last_clean":"2023-09-23T23:38:10.163751+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:38:10.163751+0000","last_undegraded":"2023-09-23T23:38:10.163751+0000","last_fullsized":"2023-09-23T23:38:10.163751+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":756,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.375449+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:38:10.375449+0000","last_peered":"2023-09-23T23:38:10.375449+0000","last_clean":"2023-09-23T23:38:10.375449+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:38:10.375449+0000","last_undegraded":"2023-09-23T23:38:10.375449+0000","last_fullsized":"2023-09-23T23:38:10.375449+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":109,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.179274+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:38:10.179274+0000","last_peered":"2023-09-23T23:38:10.179274+0000","last_clean":"2023-09-23T23:38:10.179274+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:38:10.179274+0000","last_undegraded":"2023-09-23T23:38:10.179274+0000","last_fullsized":"2023-09-23T23:38:10.179274+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":455,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.373584+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:38:10.373584+0000","last_peered":"2023-09-23T23:38:10.373584+0000","last_clean":"2023-09-23T23:38:10.373584+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:38:10.373584+0000","last_undegraded":"2023-09-23T23:38:10.373584+0000","last_fullsized":"2023-09-23T23:38:10.373584+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":305,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.179609+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:38:10.179609+0000","last_peered":"2023-09-23T23:38:10.179609+0000","last_clean":"2023-09-23T23:38:10.179609+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:38:10.179609+0000","last_undegraded":"2023-09-23T23:38:10.179609+0000","last_fullsized":"2023-09-23T23:38:10.179609+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":294,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.373762+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:38:10.373762+0000","last_peered":"2023-09-23T23:38:10.373762+0000","last_clean":"2023-09-23T23:38:10.373762+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:38:10.373762+0000","last_undegraded":"2023-09-23T23:38:10.373762+0000","last_fullsized":"2023-09-23T23:38:10.373762+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":819,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.179519+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:38:10.179519+0000","last_peered":"2023-09-23T23:38:10.179519+0000","last_clean":"2023-09-23T23:38:10.179519+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:38:10.179519+0000","last_undegraded":"2023-09-23T23:38:10.179519+0000","last_fullsized":"2023-09-23T23:38:10.179519+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":638,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.375408+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:38:10.375408+0000","last_peered":"2023-09-23T23:38:10.375408+0000","last_clean":"2023-09-23T23:38:10.375408+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:38:10.375408+0000","last_undegraded":"2023-09-23T23:38:10.375408+0000","last_fullsized":"2023-09-23T23:38:10.375408+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":622,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.368990+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:38:10.368990+0000","last_peered":"2023-09-23T23:38:10.368990+0000","last_clean":"2023-09-23T23:38:10.368990+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:38:10.368990+0000","last_undegraded":"2023-09-23T23:38:10.368990+0000","last_fullsized":"2023-09-23T23:38:10.368990+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":780,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.371635+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:38:10.371635+0000","last_peered":"2023-09-23T23:38:10.371635+0000","last_clean":"2023-09-23T23:38:10.371635+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:38:10.371635+0000","last_undegraded":"2023-09-23T23:38:10.371635+0000","last_fullsized":"2023-09-23T23:38:10.371635+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":451,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.167656+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:38:10.167656+0000","last_peered":"2023-09-23T23:38:10.167656+0000","last_clean":"2023-09-23T23:38:10.167656+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:38:10.167656+0000","last_undegraded":"2023-09-23T23:38:10.167656+0000","last_fullsized":"2023-09-23T23:38:10.167656+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":674,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T23:38:10.178727+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:38:10.178727+0000","last_peered":"2023-09-23T23:38:10.178727+0000","last_clean":"2023-09-23T23:38:10.178727+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:38:10.178727+0000","last_undegraded":"2023-09-23T23:38:10.178727+0000","last_fullsized":"2023-09-23T23:38:10.178727+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":1105592344,"num_objects":16871,"num_object_clones":0,"num_object_copies":50613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19564,"num_objects_misplaced":1015,"num_objects_unfound":0,"num_objects_dirty":16871,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":158685,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22974,"num_bytes_recovered":1505427456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5660553216,"data_stored":5660541000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90163,"ondisk_log_size":90163,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782707,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":959236,"kb_used_data":561980,"kb_used_omap":8,"kb_used_meta":397111,"kb_avail":92785916,"statfs":{"total":95995035648,"available":95012777984,"internally_reserved":0,"allocated":575467520,"data_stored":573766777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":406641733},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.81599999999999995,"15min":2.0459999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":2.4159999999999999,"5min":11.173,"15min":85.721000000000004},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.77300000000000002,"15min":2.145},"min":{"1min":0.24199999999999999,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":2.5499999999999998,"5min":5.2530000000000001,"15min":85.965000000000003},"last":1.21}]},{"osd":1,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":2.1549999999999998,"15min":3.105},"min":{"1min":0.255,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":2.6739999999999999,"5min":129.11199999999999,"15min":198.58000000000001},"last":4.9699999999999998},{"interface":"front","average":{"1min":0.61299999999999999,"5min":2.036,"15min":3.0870000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.2850000000000001,"5min":129.02600000000001,"15min":198.71100000000001},"last":13.145}]},{"osd":2,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":2.4180000000000001,"15min":3.7330000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":3.3860000000000001,"5min":78.528999999999996,"15min":203.58000000000001},"last":2.0019999999999998},{"interface":"front","average":{"1min":1.141,"5min":1.712,"15min":3.2200000000000002},"min":{"1min":0.30399999999999999,"5min":0.27900000000000003,"15min":0.26200000000000001},"max":{"1min":6.0700000000000003,"5min":53.023000000000003,"15min":198.69},"last":0.49099999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":14.901999999999999,"5min":7.1070000000000002,"15min":4.9690000000000003},"min":{"1min":0.33000000000000002,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":154.18100000000001,"5min":154.18100000000001,"15min":279.26499999999999},"last":4.375},{"interface":"front","average":{"1min":14.157999999999999,"5min":7.3140000000000001,"15min":5.476},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":154.16300000000001,"5min":154.16300000000001,"15min":279.29899999999998},"last":4.0759999999999996}]},{"osd":4,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.464,"15min":1.026},"min":{"1min":0.318,"5min":0.27100000000000002,"15min":0.246},"max":{"1min":2.0499999999999998,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.339,"15min":0.97399999999999998},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.23899999999999999},"max":{"1min":5.391,"5min":31.68,"15min":31.68},"last":1.141}]},{"osd":5,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":7.1280000000000001,"5min":2.7919999999999998,"15min":2.266},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.26500000000000001},"max":{"1min":119.417,"5min":119.417,"15min":210.83600000000001},"last":1.085},{"interface":"front","average":{"1min":7.6669999999999998,"5min":3.016,"15min":2.1989999999999998},"min":{"1min":0.33600000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":119.61499999999999,"5min":119.61499999999999,"15min":207.04599999999999},"last":0.53200000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":1.8899999999999999,"15min":1.3420000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":6.2169999999999996,"5min":94.852999999999994,"15min":94.852999999999994},"last":4.2839999999999998},{"interface":"front","average":{"1min":0.63,"5min":1.956,"15min":1.514},"min":{"1min":0.40100000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.911,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.50800000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496184,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1203504,"kb_used_data":483880,"kb_used_omap":0,"kb_used_meta":719616,"kb_avail":92541648,"statfs":{"total":95995035648,"available":94762647552,"internally_reserved":0,"allocated":495493120,"data_stored":493811912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":736886784},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.73499999999999999,"15min":4.4370000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.21099999999999999},"max":{"1min":0.53700000000000003,"5min":6.4450000000000003,"15min":135.31999999999999},"last":3.1629999999999998},{"interface":"front","average":{"1min":0.498,"5min":0.63800000000000001,"15min":4.3869999999999996},"min":{"1min":0.27800000000000002,"5min":0.24399999999999999,"15min":0.23000000000000001},"max":{"1min":1.742,"5min":3.8239999999999998,"15min":135.34299999999999},"last":6.8529999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.90500000000000003,"15min":2.2890000000000001},"min":{"1min":0.28299999999999997,"5min":0.253,"15min":0.245},"max":{"1min":1.8759999999999999,"5min":6.9080000000000004,"15min":89.605000000000004},"last":4.1440000000000001},{"interface":"front","average":{"1min":0.749,"5min":0.72499999999999998,"15min":2.391},"min":{"1min":0.29099999999999998,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":3.8980000000000001,"5min":3.8980000000000001,"15min":91.025999999999996},"last":2.468}]},{"osd":2,"last update":"Sat Sep 23 23:38:10 2023","interfaces":[{"interface":"back","average":{"1min":7.9569999999999999,"5min":5.032,"15min":4.9539999999999997},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":85.995999999999995,"5min":85.995999999999995,"15min":127.44499999999999},"last":2.5099999999999998},{"interface":"front","average":{"1min":7.6959999999999997,"5min":5.0250000000000004,"15min":5.3319999999999999},"min":{"1min":0.32300000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":85.266999999999996,"5min":85.266999999999996,"15min":125.539},"last":4.2800000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:37:57 2023","interfaces":[{"interface":"back","average":{"1min":13.925000000000001,"5min":8.1059999999999999,"15min":5.2009999999999996},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":199.536,"5min":199.536,"15min":199.536},"last":6.9240000000000004},{"interface":"front","average":{"1min":13.538,"5min":7.9660000000000002,"15min":5.4530000000000003},"min":{"1min":0.36799999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":199.59999999999999,"5min":199.59999999999999,"15min":199.59999999999999},"last":2.6349999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.755,"15min":0.66700000000000004},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.23400000000000001},"max":{"1min":1.76,"5min":7.101,"15min":8.1999999999999993},"last":2.3679999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.81399999999999995,"15min":0.76900000000000002},"min":{"1min":0.26700000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.865,"5min":8.3499999999999996,"15min":8.3499999999999996},"last":0.34100000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.76100000000000001,"15min":0.79800000000000004},"min":{"1min":0.27700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.847,"5min":11.372,"15min":11.473000000000001},"last":5.8490000000000002},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.876,"15min":0.84999999999999998},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.26900000000000002},"max":{"1min":6.1500000000000004,"5min":7.0129999999999999,"15min":7.5670000000000002},"last":0.31}]},{"osd":7,"last update":"Sat Sep 23 23:38:10 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.25,"15min":1.012},"min":{"1min":0.33600000000000002,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":6.8259999999999996,"5min":35.792999999999999,"15min":35.792999999999999},"last":2.4849999999999999},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.331,"15min":1.079},"min":{"1min":0.38300000000000001,"5min":0.33800000000000002,"15min":0.33500000000000002},"max":{"1min":12.531000000000001,"5min":40.012999999999998,"15min":40.012999999999998},"last":2.415}]}]},{"osd":1,"up_from":112,"seq":481036337525,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1260040,"kb_used_data":363392,"kb_used_omap":4,"kb_used_meta":896635,"kb_avail":92485112,"statfs":{"total":95995035648,"available":94704754688,"internally_reserved":0,"allocated":372113408,"data_stored":369242394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":918155150},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.407,"15min":3.173},"min":{"1min":0.19,"5min":0.19,"15min":0.16700000000000001},"max":{"1min":3.5830000000000002,"5min":47.195999999999998,"15min":130.768},"last":2.1200000000000001},{"interface":"front","average":{"1min":0.377,"5min":1.119,"15min":2.5030000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19},"max":{"1min":1.143,"5min":46.582000000000001,"15min":134.929},"last":0.33300000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:38:05 2023","interfaces":[{"interface":"back","average":{"1min":3.2170000000000001,"5min":3.8460000000000001,"15min":6.9219999999999997},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":53.951999999999998,"5min":201.03999999999999,"15min":231.67400000000001},"last":8.1379999999999999},{"interface":"front","average":{"1min":3.6440000000000001,"5min":3.9420000000000002,"15min":7.0069999999999997},"min":{"1min":0.221,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":54.213000000000001,"5min":202.12,"15min":232.21000000000001},"last":2.0640000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:38:05 2023","interfaces":[{"interface":"back","average":{"1min":7.2720000000000002,"5min":6.9420000000000002,"15min":5.8620000000000001},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22600000000000001},"max":{"1min":77.034999999999997,"5min":181.94399999999999,"15min":234.404},"last":3.073},{"interface":"front","average":{"1min":7.5010000000000003,"5min":7.0190000000000001,"15min":5.8150000000000004},"min":{"1min":0.245,"5min":0.224,"15min":0.224},"max":{"1min":77.165999999999997,"5min":181.98500000000001,"15min":236.33699999999999},"last":3.8730000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":0.95099999999999996,"15min":1.399},"min":{"1min":0.32500000000000001,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":4.4880000000000004,"5min":8.0850000000000009,"15min":12.042999999999999},"last":2.1360000000000001},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.85699999999999998,"15min":1.45},"min":{"1min":0.316,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":4.6660000000000004,"5min":9.5069999999999997,"15min":18.108000000000001},"last":0.52100000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":2.6309999999999998,"15min":1.734},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28899999999999998},"max":{"1min":0.98199999999999998,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.024,"5min":2.661,"15min":2.0710000000000002},"min":{"1min":0.34399999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.194,"5min":151.821,"15min":151.821},"last":2.105}]},{"osd":6,"last update":"Sat Sep 23 23:37:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.0449999999999999,"15min":1.6659999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.27500000000000002},"max":{"1min":2.306,"5min":8.9269999999999996,"15min":74.087999999999994},"last":1.0589999999999999},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.2849999999999999,"15min":1.929},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33100000000000002},"max":{"1min":3.29,"5min":8.1400000000000006,"15min":73.974999999999994},"last":2.1600000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:38:05 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":1.4590000000000001,"15min":2.101},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.7639999999999998,"5min":44.241,"15min":78.308000000000007},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.657,"5min":1.514,"15min":2.113},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32700000000000001},"max":{"1min":5.7969999999999997,"5min":43.920999999999999,"15min":77.204999999999998},"last":2.181}]}]},{"osd":0,"up_from":91,"seq":390842024348,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1365132,"kb_used_data":149124,"kb_used_omap":5,"kb_used_meta":1215994,"kb_avail":92380020,"statfs":{"total":95995035648,"available":94597140480,"internally_reserved":0,"allocated":152702976,"data_stored":149821989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1245178490},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:37:39 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":4.0620000000000003,"15min":6.4610000000000003},"min":{"1min":0.23200000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":8.4350000000000005,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.39600000000000002,"5min":3.718,"15min":5.6509999999999998},"min":{"1min":0.219,"5min":0.20200000000000001,"15min":0.17599999999999999},"max":{"1min":1.1890000000000001,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.314}]},{"osd":2,"last update":"Sat Sep 23 23:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":6.702,"15min":6.5780000000000003},"min":{"1min":0.214,"5min":0.214,"15min":0.17199999999999999},"max":{"1min":5.4340000000000002,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.77900000000000003,"5min":6.4569999999999999,"15min":6.5339999999999998},"min":{"1min":0.22600000000000001,"5min":0.20799999999999999,"15min":0.17599999999999999},"max":{"1min":3.4470000000000001,"5min":283.161,"15min":283.161},"last":0.41899999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:37:39 2023","interfaces":[{"interface":"back","average":{"1min":16.741,"5min":6.8769999999999998,"15min":6.4880000000000004},"min":{"1min":0.27300000000000002,"5min":0.219,"15min":0.217},"max":{"1min":194.26300000000001,"5min":194.26300000000001,"15min":220.732},"last":0.28599999999999998},{"interface":"front","average":{"1min":16.422000000000001,"5min":6.75,"15min":6.5830000000000002},"min":{"1min":0.30199999999999999,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":194.56299999999999,"5min":194.56299999999999,"15min":218.78200000000001},"last":0.36099999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.748,"15min":2.2799999999999998},"min":{"1min":0.33000000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.605,"5min":4.1310000000000002,"15min":138.81299999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.095,"15min":3.0369999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.27200000000000002},"max":{"1min":4.2030000000000003,"5min":9.3670000000000009,"15min":171.251},"last":0.40899999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":1.3029999999999999,"15min":2.75},"min":{"1min":0.38100000000000001,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":4.4880000000000004,"5min":15.076000000000001,"15min":171.30699999999999},"last":67.010000000000005},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.1000000000000001,"15min":1.9570000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.253},"max":{"1min":2.7029999999999998,"5min":6.5279999999999996,"15min":139.40799999999999},"last":67.090000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.1870000000000001,"15min":3.0569999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.29499999999999998},"max":{"1min":3.5939999999999999,"5min":12.654999999999999,"15min":171.33600000000001},"last":69.043999999999997},{"interface":"front","average":{"1min":0.80300000000000005,"5min":1.1659999999999999,"15min":3.2360000000000002},"min":{"1min":0.39000000000000001,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":2.1360000000000001,"5min":9.157,"15min":184.13499999999999},"last":69.028999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":1.012,"15min":2.5339999999999998},"min":{"1min":0.35999999999999999,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":3.4089999999999998,"5min":8.9510000000000005,"15min":138.77699999999999},"last":66.620000000000005},{"interface":"front","average":{"1min":0.89100000000000001,"5min":1.3069999999999999,"15min":2.5089999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.33300000000000002},"max":{"1min":2.5,"5min":9.8870000000000005,"15min":139.37899999999999},"last":66.701999999999998}]}]},{"osd":2,"up_from":199,"seq":854698492170,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1534732,"kb_used_data":664580,"kb_used_omap":7,"kb_used_meta":870136,"kb_avail":92210420,"statfs":{"total":95995035648,"available":94423470080,"internally_reserved":0,"allocated":680529920,"data_stored":677598981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":891019345},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.496,"15min":2.968},"min":{"1min":0.20999999999999999,"5min":0.186,"15min":0.17699999999999999},"max":{"1min":2.9980000000000002,"5min":61.950000000000003,"15min":95.587999999999994},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.90700000000000003,"5min":1.7050000000000001,"15min":2.8319999999999999},"min":{"1min":0.22600000000000001,"5min":0.19700000000000001,"15min":0.18099999999999999},"max":{"1min":4.5739999999999998,"5min":61.984999999999999,"15min":95.796000000000006},"last":6.5220000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":4.3550000000000004,"15min":4.516},"min":{"1min":0.21299999999999999,"5min":0.192,"15min":0.192},"max":{"1min":4.4500000000000002,"5min":106.765,"15min":157.98699999999999},"last":2.875},{"interface":"front","average":{"1min":0.76300000000000001,"5min":3.9319999999999999,"15min":4.4649999999999999},"min":{"1min":0.24399999999999999,"5min":0.222,"15min":0.222},"max":{"1min":4.468,"5min":106.732,"15min":158.03800000000001},"last":0.443}]},{"osd":3,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":5.1689999999999996,"15min":4.3070000000000004},"min":{"1min":0.245,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":4.1740000000000004,"5min":106.70399999999999,"15min":106.70399999999999},"last":11.042999999999999},{"interface":"front","average":{"1min":1.0209999999999999,"5min":4.7800000000000002,"15min":4.1459999999999999},"min":{"1min":0.28100000000000003,"5min":0.246,"15min":0.23200000000000001},"max":{"1min":3.52,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.46400000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":1.6779999999999999,"15min":1.768},"min":{"1min":0.35499999999999998,"5min":0.33700000000000002,"15min":0.27700000000000002},"max":{"1min":3.5339999999999998,"5min":62.081000000000003,"15min":62.081000000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.875,"15min":1.802},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.28499999999999998},"max":{"1min":2.9489999999999998,"5min":62.072000000000003,"15min":62.072000000000003},"last":1.6919999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":9.3179999999999996,"5min":4.476,"15min":2.6640000000000001},"min":{"1min":0.32200000000000001,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":101.104,"5min":101.104,"15min":101.104},"last":0.56799999999999995},{"interface":"front","average":{"1min":9.7829999999999995,"5min":4.6280000000000001,"15min":2.6800000000000002},"min":{"1min":0.35199999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":101.404,"5min":101.404,"15min":101.404},"last":1.6259999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":9.1869999999999994,"5min":4.8730000000000002,"15min":2.9239999999999999},"min":{"1min":0.372,"5min":0.31,"15min":0.31},"max":{"1min":97.558999999999997,"5min":107.018,"15min":107.018},"last":0.55300000000000005},{"interface":"front","average":{"1min":9.4420000000000002,"5min":3.7989999999999999,"15min":2.5219999999999998},"min":{"1min":0.375,"5min":0.371,"15min":0.36699999999999999},"max":{"1min":98.231999999999999,"5min":98.231999999999999,"15min":98.231999999999999},"last":1.6639999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.7150000000000001,"15min":1.8680000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.4129999999999998,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.0840000000000001,"5min":3,"15min":2.3100000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":3.452,"5min":106.809,"15min":106.809},"last":0.5}]}]},{"osd":3,"up_from":148,"seq":635655160138,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1399296,"kb_used_data":447288,"kb_used_omap":2,"kb_used_meta":951677,"kb_avail":92345856,"statfs":{"total":95995035648,"available":94562156544,"internally_reserved":0,"allocated":458022912,"data_stored":455157234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":974517301},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,3,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":1.0009999999999999,"15min":2.5059999999999998},"min":{"1min":0.19500000000000001,"5min":0.184,"15min":0.17699999999999999},"max":{"1min":2.9980000000000002,"5min":17.213000000000001,"15min":103.866},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.995,"15min":2.8069999999999999},"min":{"1min":0.188,"5min":0.188,"15min":0.17199999999999999},"max":{"1min":2.8090000000000002,"5min":12.202,"15min":101.884},"last":7.843}]},{"osd":1,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.95999999999999996,"15min":2.819},"min":{"1min":0.23899999999999999,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":2.851,"5min":7.3319999999999999,"15min":85.242999999999995},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.88400000000000001,"15min":2.6920000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":2.9830000000000001,"5min":11.228,"15min":85.302000000000007},"last":0.42299999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:37:57 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":1.4359999999999999,"15min":4.0659999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.19800000000000001},"max":{"1min":36.362000000000002,"5min":36.362000000000002,"15min":84.724999999999994},"last":0.373},{"interface":"front","average":{"1min":2.6110000000000002,"5min":1.431,"15min":3.931},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.214},"max":{"1min":36.158999999999999,"5min":36.158999999999999,"15min":84.567999999999998},"last":0.45900000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":1.032,"15min":1.202},"min":{"1min":0.318,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":2.9609999999999999,"5min":11.209,"15min":11.209},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.74199999999999999,"5min":1.1140000000000001,"15min":1.6160000000000001},"min":{"1min":0.34000000000000002,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":2.7639999999999998,"5min":7.681,"15min":24.782},"last":7.8920000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":10.651,"5min":8.0739999999999998,"15min":3.9369999999999998},"min":{"1min":0.38500000000000001,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":196.358,"5min":214.43700000000001,"15min":214.43700000000001},"last":7.7169999999999996},{"interface":"front","average":{"1min":10.586,"5min":8.0500000000000007,"15min":3.984},"min":{"1min":0.35999999999999999,"5min":0.34699999999999998,"15min":0.30599999999999999},"max":{"1min":196.048,"5min":214.46600000000001,"15min":214.46600000000001},"last":4.7930000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":7.2190000000000003,"5min":3.2280000000000002,"15min":2.5649999999999999},"min":{"1min":0.35799999999999998,"5min":0.318,"15min":0.318},"max":{"1min":125.94,"5min":125.94,"15min":125.94},"last":7.8159999999999998},{"interface":"front","average":{"1min":7.1189999999999998,"5min":3.3559999999999999,"15min":2.5649999999999999},"min":{"1min":0.36499999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":125.955,"5min":125.955,"15min":125.955},"last":0.441}]},{"osd":7,"last update":"Sat Sep 23 23:37:57 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":4.5289999999999999,"15min":3.1909999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32600000000000001},"max":{"1min":6.5800000000000001,"5min":192.69,"15min":192.69},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.92400000000000004,"5min":4.5789999999999997,"15min":3.629},"min":{"1min":0.40699999999999997,"5min":0.40100000000000002,"15min":0.36699999999999999},"max":{"1min":3.6120000000000001,"5min":192.65600000000001,"15min":192.65600000000001},"last":7.8689999999999998}]}]},{"osd":4,"up_from":0,"seq":128849019457,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757820,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1446832,"kb_used_data":685864,"kb_used_omap":0,"kb_used_meta":760960,"kb_avail":92298320,"statfs":{"total":95995035648,"available":94513479680,"internally_reserved":0,"allocated":702324736,"data_stored":700640203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":779223040},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:19 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.78200000000000003,"15min":1.454},"min":{"1min":0.223,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.2890000000000001,"5min":6.1609999999999996,"15min":63.741999999999997},"last":0.26200000000000001},{"interface":"front","average":{"1min":1,"5min":1.1779999999999999,"15min":1.964},"min":{"1min":0.26200000000000001,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":4.8579999999999997,"5min":15.414,"15min":138.55199999999999},"last":4.8579999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:37:22 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.1459999999999999,"15min":2.1629999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.1360000000000001,"5min":29.007999999999999,"15min":69.816999999999993},"last":3.7269999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.212,"15min":2.3929999999999998},"min":{"1min":0.30499999999999999,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":1.452,"5min":29.053999999999998,"15min":70.268000000000001},"last":3.7570000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":6.8869999999999996,"15min":4.4939999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26900000000000002},"max":{"1min":6.2469999999999999,"5min":208.738,"15min":208.738},"last":1.2869999999999999},{"interface":"front","average":{"1min":0.77700000000000002,"5min":5.5220000000000002,"15min":4.0140000000000002},"min":{"1min":0.309,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":3.2290000000000001,"5min":208.631,"15min":208.631},"last":3.2040000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:37:53 2023","interfaces":[{"interface":"back","average":{"1min":6.6550000000000002,"5min":5.6689999999999996,"15min":4.1470000000000002},"min":{"1min":0.33400000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":143.66300000000001,"5min":143.66300000000001,"15min":143.66300000000001},"last":2.9359999999999999},{"interface":"front","average":{"1min":6.7670000000000003,"5min":6.0010000000000003,"15min":4.327},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":143.62200000000001,"5min":143.62200000000001,"15min":143.62200000000001},"last":4.9089999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.88200000000000001,"15min":0.877},"min":{"1min":0.316,"5min":0.315,"15min":0.251},"max":{"1min":2.2650000000000001,"5min":9.2690000000000001,"15min":19.962},"last":3.7909999999999999},{"interface":"front","average":{"1min":0.85499999999999998,"5min":1.016,"15min":0.84099999999999997},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.22900000000000001},"max":{"1min":3.1840000000000002,"5min":10.273999999999999,"15min":20.829999999999998},"last":0.44500000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:37:31 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.3340000000000001,"15min":1.5249999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.3100000000000001,"5min":148.36600000000001,"15min":148.36600000000001},"last":3.1699999999999999},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.7389999999999999,"15min":1.71},"min":{"1min":0.32800000000000001,"5min":0.30399999999999999,"15min":0.26000000000000001},"max":{"1min":5.7240000000000002,"5min":148.42500000000001,"15min":148.42500000000001},"last":5.0220000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:37:48 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.306,"15min":2.177},"min":{"1min":0.34699999999999998,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":4.3380000000000001,"5min":15.497,"15min":179.28800000000001},"last":4.9429999999999996},{"interface":"front","average":{"1min":1.157,"5min":1.339,"15min":2.2480000000000002},"min":{"1min":0.40600000000000003,"5min":0.35199999999999998,"15min":0.34999999999999998},"max":{"1min":6.1440000000000001,"5min":15.653,"15min":179.428},"last":4.9740000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":143458304,"data_stored":143458304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":358092800,"data_stored":358088728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":658702336,"data_stored":658702336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":438243328,"data_stored":438239256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2341732352,"data_stored":2341732352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":678363136,"data_stored":678363136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":481755136,"data_stored":481755136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":560205824,"data_stored":560201752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:38:29.484 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:38:29.565 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:38:29.611 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:29.678 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:38:29.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:29.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:38:29.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:29 smithi110 ceph-mon[212939]: pgmap v1725: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 1.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 19564/50685 objects degraded (38.599%); 1015/50685 objects misplaced (2.003%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T23:38:29.707 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:29 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 19564/50685 objects degraded (38.599%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:29.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:29 smithi110 ceph-mon[221102]: pgmap v1725: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 1.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 19564/50685 objects degraded (38.599%); 1015/50685 objects misplaced (2.003%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T23:38:29.707 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:29 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 19564/50685 objects degraded (38.599%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:29.759 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:29.792 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:38:29.847 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:29 smithi138 ceph-mon[193606]: pgmap v1725: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 1.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 19564/50685 objects degraded (38.599%); 1015/50685 objects misplaced (2.003%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T23:38:29.847 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:29 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 19564/50685 objects degraded (38.599%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:29.856 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:29.932 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:38:29.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:29.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:38:30.043 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:38:30.090 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:30.180 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:38:30.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:30.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:38:30.292 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:38:30.395 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:30.686 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:38:30.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:30.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:38:30.803 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:38:30.850 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:30.940 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:38:30.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:30.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:38:31.051 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:38:31.112 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:31.202 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:38:31.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:31.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:38:31.307 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:38:31.381 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:31 smithi138 ceph-mon[193606]: pgmap v1726: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 860 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 839 op/s; 19124/41307 objects degraded (46.297%); 734/41307 objects misplaced (1.777%); 6.8 MiB/s, 144 objects/s recovering 2023-09-23T23:38:31.381 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:31 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:38:31.381 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:31 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:38:31.381 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:31 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/327437584' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:38:31.381 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:31 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:38:31.402 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:31.619 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:38:31.655 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:31 smithi110 ceph-mon[221102]: pgmap v1726: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 860 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 839 op/s; 19124/41307 objects degraded (46.297%); 734/41307 objects misplaced (1.777%); 6.8 MiB/s, 144 objects/s recovering 2023-09-23T23:38:31.655 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:31 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:38:31.655 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:31 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:38:31.655 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:31 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/327437584' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:38:31.655 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:31 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:38:31.657 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:31 smithi110 ceph-mon[212939]: pgmap v1726: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 860 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 839 op/s; 19124/41307 objects degraded (46.297%); 734/41307 objects misplaced (1.777%); 6.8 MiB/s, 144 objects/s recovering 2023-09-23T23:38:31.657 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:31 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:38:31.658 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:31 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:38:31.658 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:31 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/327437584' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:38:31.658 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:31 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:38:32.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:32 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:38:32.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:32 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:38:32.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:32 smithi138 ceph-mon[193606]: osdmap e608: 8 total, 8 up, 7 in 2023-09-23T23:38:32.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:32 smithi138 ceph-mon[193606]: failed to encode map e608 with expected crc 2023-09-23T23:38:32.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:32 smithi138 ceph-mon[193606]: failed to encode map e608 with expected crc 2023-09-23T23:38:32.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:32 smithi138 ceph-mon[193606]: failed to encode map e608 with expected crc 2023-09-23T23:38:32.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:32 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:38:32.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:32 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:38:32.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:32 smithi110 ceph-mon[212939]: osdmap e608: 8 total, 8 up, 7 in 2023-09-23T23:38:32.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:32 smithi110 ceph-mon[212939]: failed to encode map e608 with expected crc 2023-09-23T23:38:32.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:32 smithi110 ceph-mon[212939]: failed to encode map e608 with expected crc 2023-09-23T23:38:32.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:32 smithi110 ceph-mon[212939]: failed to encode map e608 with expected crc 2023-09-23T23:38:32.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:32 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:38:32.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:32 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:38:32.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:32 smithi110 ceph-mon[221102]: osdmap e608: 8 total, 8 up, 7 in 2023-09-23T23:38:32.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:32 smithi110 ceph-mon[221102]: failed to encode map e608 with expected crc 2023-09-23T23:38:32.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:32 smithi110 ceph-mon[221102]: failed to encode map e608 with expected crc 2023-09-23T23:38:32.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:32 smithi110 ceph-mon[221102]: failed to encode map e608 with expected crc 2023-09-23T23:38:33.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:33 smithi110 ceph-mon[221102]: pgmap v1728: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 815 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 737 op/s; 19124/39177 objects degraded (48.814%); 734/39177 objects misplaced (1.874%); 6.0 MiB/s, 132 objects/s recovering 2023-09-23T23:38:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:33 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:38:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:33 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:38:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:33 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:38:33.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:33 smithi110 ceph-mon[212939]: pgmap v1728: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 815 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 737 op/s; 19124/39177 objects degraded (48.814%); 734/39177 objects misplaced (1.874%); 6.0 MiB/s, 132 objects/s recovering 2023-09-23T23:38:33.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:33 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:38:33.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:33 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:38:33.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:33 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:38:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:33 smithi138 ceph-mon[193606]: pgmap v1728: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 815 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 737 op/s; 19124/39177 objects degraded (48.814%); 734/39177 objects misplaced (1.874%); 6.0 MiB/s, 132 objects/s recovering 2023-09-23T23:38:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:33 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:38:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:33 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:38:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:33 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:38:33.927 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:33.995 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:38:34.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:34.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:38:34.122 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:38:34.200 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:34.298 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:38:34.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:34.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:38:34.407 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:38:34.447 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:34 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 19124/39177 objects degraded (48.814%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:34.485 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:34.578 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:34 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 19124/39177 objects degraded (48.814%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:34.579 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:34 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 19124/39177 objects degraded (48.814%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:34.755 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:38:34.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:34.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:38:34.859 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:38:34.902 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:34.912 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:38:34.912 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:38:34.974 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:38:34.985 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:38:35.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:35.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:38:35.024 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:38:35.102 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:38:35.128 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:38:35.136 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:35.154 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:38:35.191 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:38:35.239 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:38:35.257 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:38:35.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:35.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:38:35.277 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:38:35.307 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:38:35.313 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:38:35.313 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:38:35.313 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:38:35.313 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:38:35.313 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:38:35.313 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:38:35.313 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:38:35.313 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:38:35.370 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:38:35.437 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:35.533 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:38:35.543 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:35 smithi138 ceph-mon[193606]: pgmap v1729: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 757 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 746 op/s; 18870/36360 objects degraded (51.898%); 508/36360 objects misplaced (1.397%); 3.4 MiB/s, 116 objects/s recovering 2023-09-23T23:38:35.543 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:35 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2845339065' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:35.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:35.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:38:35.645 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:38:35.688 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:35.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:35 smithi110 ceph-mon[212939]: pgmap v1729: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 757 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 746 op/s; 18870/36360 objects degraded (51.898%); 508/36360 objects misplaced (1.397%); 3.4 MiB/s, 116 objects/s recovering 2023-09-23T23:38:35.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:35 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2845339065' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:35.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:35 smithi110 ceph-mon[221102]: pgmap v1729: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 757 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 746 op/s; 18870/36360 objects degraded (51.898%); 508/36360 objects misplaced (1.397%); 3.4 MiB/s, 116 objects/s recovering 2023-09-23T23:38:35.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:35 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2845339065' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:35.800 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:38:35.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:35.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:38:35.914 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:38:35.918 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:35.989 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:36.492 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:36.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:36.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:36.606 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:36.632 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:37.086 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:37.102 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:38:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:38:36] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.43.0" 2023-09-23T23:38:37.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:37.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:37.132 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:38:37.204 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:38:37.253 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:37.354 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:38:37.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:37.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:38:37.420 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:37 smithi110 ceph-mon[212939]: pgmap v1730: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 574 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 18609/27600 objects degraded (67.424%); 508/27600 objects misplaced (1.841%); 1.1 MiB/s, 107 objects/s recovering 2023-09-23T23:38:37.464 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:38:37.523 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:37.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:37 smithi138 ceph-mon[193606]: pgmap v1730: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 574 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 18609/27600 objects degraded (67.424%); 508/27600 objects misplaced (1.841%); 1.1 MiB/s, 107 objects/s recovering 2023-09-23T23:38:37.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:37 smithi110 ceph-mon[221102]: pgmap v1730: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 574 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 18609/27600 objects degraded (67.424%); 508/27600 objects misplaced (1.841%); 1.1 MiB/s, 107 objects/s recovering 2023-09-23T23:38:37.999 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:38:38.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:38.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:38:38.113 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:38:38.180 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:38.241 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:38.253 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:38:38.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:38.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:38:38.352 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:38.359 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:38:38.483 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:38.809 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:38:38.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:38.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:38:38.829 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:38:38.829 INFO:teuthology.orchestra.run.smithi110.stdout:{"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 18609/27600 objects degraded (67.424%), 6 pgs degraded, 5 pgs undersized","count":11},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2196,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":608,"num_osds":8,"num_up_osds":8,"osd_up_since":1695511414,"num_in_osds":7,"osd_in_since":1695512030,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":112},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":3},{"state_name":"active+recovering+undersized+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+recovery_wait","count":1}],"num_pgs":119,"num_pools":5,"num_objects":9200,"data_bytes":601928949,"bytes_used":7999164416,"bytes_avail":663966085120,"bytes_total":671965249536,"degraded_objects":18609,"degraded_total":27600,"degraded_ratio":0.67423913043478256,"misplaced_objects":508,"misplaced_total":27600,"misplaced_ratio":0.018405797101449274,"recovering_objects_per_sec":107,"recovering_bytes_per_sec":1191857,"recovering_keys_per_sec":0,"num_objects_recovered":1074,"num_bytes_recovered":11927552,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":768},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi110.fuglpi","status":"up:active","gid":24572}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.110:8443/","prometheus":"http://172.21.15.110:9283/"}},"servicemap":{"epoch":33,"modified":"2023-09-23T23:38:29.965572+0000","services":{}},"progress_events":{"833ef15d-07ae-4299-b4e4-2cf94e068147":{"message":"Upgrade to 18.2.0-460-gbb0daeff (10m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true},"c67c6d89-9cd0-495b-b1de-fa24b5d12727":{"message":"Global Recovery Event (4m)\n [==========================..] (remaining: 16s)","progress":0.94117647409439087,"add_to_ceph_s":true}}} 2023-09-23T23:38:38.926 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:38:38.989 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:39.111 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:38:39.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:39.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:38:39.222 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:38:39.277 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:39.377 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:38:39.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:39.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:38:39.427 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:39 smithi110 ceph-mon[212939]: pgmap v1731: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 574 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 18609/27600 objects degraded (67.424%); 508/27600 objects misplaced (1.841%); 1.1 MiB/s, 107 objects/s recovering 2023-09-23T23:38:39.428 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:39 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 18609/27600 objects degraded (67.424%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:39.428 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:39 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2580404898' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:38:39.493 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:38:39.549 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:39 smithi138 ceph-mon[193606]: pgmap v1731: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 574 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 18609/27600 objects degraded (67.424%); 508/27600 objects misplaced (1.841%); 1.1 MiB/s, 107 objects/s recovering 2023-09-23T23:38:39.549 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:39 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 18609/27600 objects degraded (67.424%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:39.549 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:39 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2580404898' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:38:39.565 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:39.663 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:38:39.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:39.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:38:39.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:39 smithi110 ceph-mon[221102]: pgmap v1731: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 574 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 18609/27600 objects degraded (67.424%); 508/27600 objects misplaced (1.841%); 1.1 MiB/s, 107 objects/s recovering 2023-09-23T23:38:39.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:39 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 18609/27600 objects degraded (67.424%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:39.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:39 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2580404898' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:38:39.781 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:39.858 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:40.045 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:38:41.305 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:41.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:41.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:41.362 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:38:41.362 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:38:41.411 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:38:41.473 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:41.567 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:38:41.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:41.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:38:41.623 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:41 smithi110 ceph-mon[212939]: pgmap v1732: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 386 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 757 op/s; 18086/18576 objects degraded (97.362%); 266/18576 objects misplaced (1.432%); 0 B/s, 103 objects/s recovering 2023-09-23T23:38:41.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:41 smithi110 ceph-mon[221102]: pgmap v1732: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 386 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 757 op/s; 18086/18576 objects degraded (97.362%); 266/18576 objects misplaced (1.432%); 0 B/s, 103 objects/s recovering 2023-09-23T23:38:41.674 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:38:41.732 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:41 smithi138 ceph-mon[193606]: pgmap v1732: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 386 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 757 op/s; 18086/18576 objects degraded (97.362%); 266/18576 objects misplaced (1.432%); 0 B/s, 103 objects/s recovering 2023-09-23T23:38:41.743 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:41.858 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:38:41.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:41.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:38:41.979 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:38:42.049 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:42.235 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:38:42.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:42.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:38:42.343 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:38:42.394 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:42.488 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:38:42.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:42.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:38:42.601 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:38:42.676 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:42.680 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:42.746 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:42.930 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:38:42.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:42.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:38:43.047 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:43.081 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:38:43.081 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:38:43.081 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:38:43.081 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:38:43.081 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:175261 12.10 12:0f9d8e44:::benchmark_data_smithi110_491193_object80386:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2023-09-23T23:38:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.428760+0000", 2023-09-23T23:38:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 395.62985535000001, 2023-09-23T23:38:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.279936254, 2023-09-23T23:38:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:38:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:38:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:38:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175261 2023-09-23T23:38:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428760+0000", 2023-09-23T23:38:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428760+0000", 2023-09-23T23:38:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428758+0000", 2023-09-23T23:38:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:38:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428765+0000", 2023-09-23T23:38:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8680000000000001e-06 2023-09-23T23:38:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428778+0000", 2023-09-23T23:38:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2687000000000001e-05 2023-09-23T23:38:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428785+0000", 2023-09-23T23:38:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1679999999999998e-06 2023-09-23T23:38:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428858+0000", 2023-09-23T23:38:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3231000000000004e-05 2023-09-23T23:38:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428968+0000", 2023-09-23T23:38:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010950699999999999 2023-09-23T23:38:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:38:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.429420+0000", 2023-09-23T23:38:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045183000000000002 2023-09-23T23:38:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:38:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.432669+0000", 2023-09-23T23:38:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032491830000000001 2023-09-23T23:38:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708584+0000", 2023-09-23T23:38:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.27591494900000002 2023-09-23T23:38:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708613+0000", 2023-09-23T23:38:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9178000000000001e-05 2023-09-23T23:38:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708659+0000", 2023-09-23T23:38:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6415000000000002e-05 2023-09-23T23:38:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708696+0000", 2023-09-23T23:38:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6832000000000002e-05 2023-09-23T23:38:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:175263 12.8 e524/490 12:11714bff:::benchmark_data_smithi110_491193_object80388:head v 524'7666, mlcod=524'7664)", 2023-09-23T23:38:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.433325+0000", 2023-09-23T23:38:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 395.62528994500002, 2023-09-23T23:38:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.282317341, 2023-09-23T23:38:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433325+0000", 2023-09-23T23:38:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433325+0000", 2023-09-23T23:38:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433325+0000", 2023-09-23T23:38:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:38:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433333+0000", 2023-09-23T23:38:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5420000000000005e-06 2023-09-23T23:38:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433344+0000", 2023-09-23T23:38:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1358e-05 2023-09-23T23:38:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433351+0000", 2023-09-23T23:38:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7209999999999997e-06 2023-09-23T23:38:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433384+0000", 2023-09-23T23:38:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3065000000000003e-05 2023-09-23T23:38:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433404+0000", 2023-09-23T23:38:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9762000000000002e-05 2023-09-23T23:38:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.715624+0000", 2023-09-23T23:38:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28222047 2023-09-23T23:38:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.715643+0000", 2023-09-23T23:38:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8238999999999999e-05 2023-09-23T23:38:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:175265 12.10 12:09b7804b:::benchmark_data_smithi110_491193_object80390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2023-09-23T23:38:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.434661+0000", 2023-09-23T23:38:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 395.62395394999999, 2023-09-23T23:38:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28935229200000001, 2023-09-23T23:38:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:38:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:38:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:38:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175265 2023-09-23T23:38:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.434661+0000", 2023-09-23T23:38:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.434661+0000", 2023-09-23T23:38:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.434660+0000", 2023-09-23T23:38:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:38:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.434667+0000", 2023-09-23T23:38:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6919999999999996e-06 2023-09-23T23:38:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708315+0000", 2023-09-23T23:38:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.27364855599999999 2023-09-23T23:38:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708328+0000", 2023-09-23T23:38:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2444e-05 2023-09-23T23:38:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.712241+0000", 2023-09-23T23:38:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039134829999999997 2023-09-23T23:38:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.712351+0000", 2023-09-23T23:38:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110096 2023-09-23T23:38:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:38:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.712806+0000", 2023-09-23T23:38:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000454604 2023-09-23T23:38:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.718758+0000", 2023-09-23T23:38:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059521080000000002 2023-09-23T23:38:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:38:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.718791+0000", 2023-09-23T23:38:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3618000000000002e-05 2023-09-23T23:38:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:38:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.723932+0000", 2023-09-23T23:38:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051407349999999996 2023-09-23T23:38:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.723976+0000", 2023-09-23T23:38:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3618000000000001e-05 2023-09-23T23:38:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.724014+0000", 2023-09-23T23:38:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7829e-05 2023-09-23T23:38:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:175259 12.c e524/476 12:3a3a4253:::benchmark_data_smithi110_491193_object80384:head v 524'10995, mlcod=524'10989)", 2023-09-23T23:38:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.436466+0000", 2023-09-23T23:38:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 395.622148907, 2023-09-23T23:38:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28907641299999998, 2023-09-23T23:38:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436466+0000", 2023-09-23T23:38:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436466+0000", 2023-09-23T23:38:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436466+0000", 2023-09-23T23:38:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436473+0000", 2023-09-23T23:38:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4479999999999997e-06 2023-09-23T23:38:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708281+0000", 2023-09-23T23:38:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.27180817299999999 2023-09-23T23:38:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708296+0000", 2023-09-23T23:38:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4875000000000001e-05 2023-09-23T23:38:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708345+0000", 2023-09-23T23:38:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9089999999999999e-05 2023-09-23T23:38:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708371+0000", 2023-09-23T23:38:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5809e-05 2023-09-23T23:38:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.725504+0000", 2023-09-23T23:38:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017133116 2023-09-23T23:38:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.725543+0000", 2023-09-23T23:38:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8574e-05 2023-09-23T23:38:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:136767 13.5 e603/570 13:a2c45377:::benchmark_data_smithi110_541928_object48987:head v 603'8740, mlcod=538'122)", 2023-09-23T23:38:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:37:42.444137+0000", 2023-09-23T23:38:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.614478421000001, 2023-09-23T23:38:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038863414999999998, 2023-09-23T23:38:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444137+0000", 2023-09-23T23:38:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444137+0000", 2023-09-23T23:38:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444136+0000", 2023-09-23T23:38:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:38:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444145+0000", 2023-09-23T23:38:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3510000000000008e-06 2023-09-23T23:38:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444161+0000", 2023-09-23T23:38:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6041000000000001e-05 2023-09-23T23:38:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444170+0000", 2023-09-23T23:38:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5590000000000006e-06 2023-09-23T23:38:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444219+0000", 2023-09-23T23:38:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8640000000000002e-05 2023-09-23T23:38:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444246+0000", 2023-09-23T23:38:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7104e-05 2023-09-23T23:38:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.482876+0000", 2023-09-23T23:38:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038630039999999997 2023-09-23T23:38:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.483000+0000", 2023-09-23T23:38:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012453 2023-09-23T23:38:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:136768 13.5 e603/570 13:ac9b8559:::benchmark_data_smithi110_541928_object48988:head v 603'8741, mlcod=538'122)", 2023-09-23T23:38:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:37:42.444186+0000", 2023-09-23T23:38:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.614429114000004, 2023-09-23T23:38:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038839388000000002, 2023-09-23T23:38:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444186+0000", 2023-09-23T23:38:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444186+0000", 2023-09-23T23:38:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444186+0000", 2023-09-23T23:38:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444191+0000", 2023-09-23T23:38:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2440000000000001e-06 2023-09-23T23:38:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444198+0000", 2023-09-23T23:38:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4200000000000004e-06 2023-09-23T23:38:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444202+0000", 2023-09-23T23:38:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7790000000000003e-06 2023-09-23T23:38:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.451799+0000", 2023-09-23T23:38:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075971290000000002 2023-09-23T23:38:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.451827+0000", 2023-09-23T23:38:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7872999999999999e-05 2023-09-23T23:38:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.483005+0000", 2023-09-23T23:38:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031177269000000001 2023-09-23T23:38:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.483026+0000", 2023-09-23T23:38:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0948e-05 2023-09-23T23:38:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:136772 13.5 e603/570 13:a2b484f3:::benchmark_data_smithi110_541928_object48992:head v 603'8742, mlcod=538'122)", 2023-09-23T23:38:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:37:42.444382+0000", 2023-09-23T23:38:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.614233003999999, 2023-09-23T23:38:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038666736, 2023-09-23T23:38:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444382+0000", 2023-09-23T23:38:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444382+0000", 2023-09-23T23:38:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444382+0000", 2023-09-23T23:38:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444391+0000", 2023-09-23T23:38:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3549999999999994e-06 2023-09-23T23:38:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444401+0000", 2023-09-23T23:38:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5140000000000007e-06 2023-09-23T23:38:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.444407+0000", 2023-09-23T23:38:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1569999999999997e-06 2023-09-23T23:38:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.460154+0000", 2023-09-23T23:38:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015747068999999999 2023-09-23T23:38:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.460179+0000", 2023-09-23T23:38:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5429e-05 2023-09-23T23:38:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.483028+0000", 2023-09-23T23:38:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022848730000000001 2023-09-23T23:38:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:42.483049+0000", 2023-09-23T23:38:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1140000000000001e-05 2023-09-23T23:38:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:150836 13.5 e606/570 13:a2d92cee:::benchmark_data_smithi110_541928_object63056:head v 606'9588, mlcod=538'122)", 2023-09-23T23:38:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:38:12.495903+0000", 2023-09-23T23:38:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 30.562712453, 2023-09-23T23:38:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.045501689999999997, 2023-09-23T23:38:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.495903+0000", 2023-09-23T23:38:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.495903+0000", 2023-09-23T23:38:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.495902+0000", 2023-09-23T23:38:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:38:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.495913+0000", 2023-09-23T23:38:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1181e-05 2023-09-23T23:38:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.495926+0000", 2023-09-23T23:38:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2938999999999999e-05 2023-09-23T23:38:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.495934+0000", 2023-09-23T23:38:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4460000000000004e-06 2023-09-23T23:38:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.498806+0000", 2023-09-23T23:38:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028721939999999998 2023-09-23T23:38:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.498834+0000", 2023-09-23T23:38:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8484e-05 2023-09-23T23:38:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.541360+0000", 2023-09-23T23:38:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042526248000000003 2023-09-23T23:38:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.541405+0000", 2023-09-23T23:38:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4100000000000001e-05 2023-09-23T23:38:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:150837 13.5 e606/570 13:ab984c3d:::benchmark_data_smithi110_541928_object63057:head v 606'9589, mlcod=538'122)", 2023-09-23T23:38:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:38:12.498686+0000", 2023-09-23T23:38:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 30.559929564000001, 2023-09-23T23:38:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042743299999999998, 2023-09-23T23:38:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.498686+0000", 2023-09-23T23:38:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.498686+0000", 2023-09-23T23:38:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.498685+0000", 2023-09-23T23:38:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.498695+0000", 2023-09-23T23:38:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5340000000000005e-06 2023-09-23T23:38:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.498705+0000", 2023-09-23T23:38:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0134e-05 2023-09-23T23:38:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.498711+0000", 2023-09-23T23:38:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6800000000000004e-06 2023-09-23T23:38:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.506269+0000", 2023-09-23T23:38:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075579590000000004 2023-09-23T23:38:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.506299+0000", 2023-09-23T23:38:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9782999999999999e-05 2023-09-23T23:38:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.541411+0000", 2023-09-23T23:38:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.035111699000000003 2023-09-23T23:38:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:12.541429+0000", 2023-09-23T23:38:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8213999999999999e-05 2023-09-23T23:38:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:151703 13.5 e606/570 13:aefb0dd7:::benchmark_data_smithi110_541928_object63923:head v 606'9641, mlcod=538'122)", 2023-09-23T23:38:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:38:13.841682+0000", 2023-09-23T23:38:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 29.216933770000001, 2023-09-23T23:38:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040556536999999997, 2023-09-23T23:38:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:13.841682+0000", 2023-09-23T23:38:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:13.841682+0000", 2023-09-23T23:38:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:13.841681+0000", 2023-09-23T23:38:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:13.841694+0000", 2023-09-23T23:38:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2779000000000001e-05 2023-09-23T23:38:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:13.841702+0000", 2023-09-23T23:38:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8089999999999993e-06 2023-09-23T23:38:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:13.841705+0000", 2023-09-23T23:38:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1300000000000001e-06 2023-09-23T23:38:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:13.849128+0000", 2023-09-23T23:38:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074230019999999997 2023-09-23T23:38:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:13.849149+0000", 2023-09-23T23:38:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0846999999999999e-05 2023-09-23T23:38:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:13.882195+0000", 2023-09-23T23:38:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.033046064999999999 2023-09-23T23:38:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:13.882238+0000", 2023-09-23T23:38:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3231e-05 2023-09-23T23:38:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:153979 13.5 e606/570 13:ac75a5c1:::benchmark_data_smithi110_541928_object66199:head v 606'9791, mlcod=538'122)", 2023-09-23T23:38:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:38:17.510671+0000", 2023-09-23T23:38:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 25.547944474000001, 2023-09-23T23:38:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.043624652999999999, 2023-09-23T23:38:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.510671+0000", 2023-09-23T23:38:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.510671+0000", 2023-09-23T23:38:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.510671+0000", 2023-09-23T23:38:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.510675+0000", 2023-09-23T23:38:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2590000000000002e-06 2023-09-23T23:38:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.510681+0000", 2023-09-23T23:38:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2339999999999999e-06 2023-09-23T23:38:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.510686+0000", 2023-09-23T23:38:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6689999999999997e-06 2023-09-23T23:38:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.514602+0000", 2023-09-23T23:38:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039164990000000004 2023-09-23T23:38:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.514633+0000", 2023-09-23T23:38:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0343999999999999e-05 2023-09-23T23:38:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.554245+0000", 2023-09-23T23:38:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.039612055 2023-09-23T23:38:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.554295+0000", 2023-09-23T23:38:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0903999999999999e-05 2023-09-23T23:38:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:153985 13.5 e606/570 13:a949d1f0:::benchmark_data_smithi110_541928_object66205:head v 606'9792, mlcod=538'122)", 2023-09-23T23:38:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:38:17.510728+0000", 2023-09-23T23:38:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 25.547887629000002, 2023-09-23T23:38:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.04359238, 2023-09-23T23:38:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.510728+0000", 2023-09-23T23:38:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.510728+0000", 2023-09-23T23:38:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.510727+0000", 2023-09-23T23:38:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-09-23T23:38:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.510731+0000", 2023-09-23T23:38:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2820000000000001e-06 2023-09-23T23:38:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.510736+0000", 2023-09-23T23:38:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0340000000000004e-06 2023-09-23T23:38:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.510740+0000", 2023-09-23T23:38:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0330000000000002e-06 2023-09-23T23:38:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.522390+0000", 2023-09-23T23:38:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011650305 2023-09-23T23:38:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.522420+0000", 2023-09-23T23:38:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9723e-05 2023-09-23T23:38:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.554300+0000", 2023-09-23T23:38:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031880296000000002 2023-09-23T23:38:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.554320+0000", 2023-09-23T23:38:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9942999999999999e-05 2023-09-23T23:38:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:154197 13.5 e606/570 13:a9c58133:::benchmark_data_smithi110_541928_object66417:head v 606'9809, mlcod=538'122)", 2023-09-23T23:38:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:38:17.822704+0000", 2023-09-23T23:38:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 25.235911092999999, 2023-09-23T23:38:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041510327999999999, 2023-09-23T23:38:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.822704+0000", 2023-09-23T23:38:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.822704+0000", 2023-09-23T23:38:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.822703+0000", 2023-09-23T23:38:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:38:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.822716+0000", 2023-09-23T23:38:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3159999999999999e-05 2023-09-23T23:38:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.822734+0000", 2023-09-23T23:38:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7932e-05 2023-09-23T23:38:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.822743+0000", 2023-09-23T23:38:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3389999999999993e-06 2023-09-23T23:38:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.822809+0000", 2023-09-23T23:38:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6878999999999994e-05 2023-09-23T23:38:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.822838+0000", 2023-09-23T23:38:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8958e-05 2023-09-23T23:38:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.864171+0000", 2023-09-23T23:38:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041332979999999998 2023-09-23T23:38:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:17.864215+0000", 2023-09-23T23:38:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3099999999999997e-05 2023-09-23T23:38:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:154495 13.5 e606/570 13:a50ebe71:::benchmark_data_smithi110_541928_object66715:head v 606'9829, mlcod=538'122)", 2023-09-23T23:38:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:38:18.258133+0000", 2023-09-23T23:38:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 24.800481813000001, 2023-09-23T23:38:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041089220000000003, 2023-09-23T23:38:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.258133+0000", 2023-09-23T23:38:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.258133+0000", 2023-09-23T23:38:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.258132+0000", 2023-09-23T23:38:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:38:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.258148+0000", 2023-09-23T23:38:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5608e-05 2023-09-23T23:38:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.258170+0000", 2023-09-23T23:38:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2232999999999999e-05 2023-09-23T23:38:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.258180+0000", 2023-09-23T23:38:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0108999999999999e-05 2023-09-23T23:38:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.258253+0000", 2023-09-23T23:38:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2825999999999997e-05 2023-09-23T23:38:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.258285+0000", 2023-09-23T23:38:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2280000000000003e-05 2023-09-23T23:38:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.299174+0000", 2023-09-23T23:38:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040888971000000003 2023-09-23T23:38:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.299223+0000", 2023-09-23T23:38:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8542999999999999e-05 2023-09-23T23:38:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:154501 13.5 e606/570 13:a807b0fe:::benchmark_data_smithi110_541928_object66721:head v 606'9830, mlcod=538'122)", 2023-09-23T23:38:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:38:18.261215+0000", 2023-09-23T23:38:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 24.797400481, 2023-09-23T23:38:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038032707999999998, 2023-09-23T23:38:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.261215+0000", 2023-09-23T23:38:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.261215+0000", 2023-09-23T23:38:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.261214+0000", 2023-09-23T23:38:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:38:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.261224+0000", 2023-09-23T23:38:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0254e-05 2023-09-23T23:38:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.261243+0000", 2023-09-23T23:38:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9553000000000001e-05 2023-09-23T23:38:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.261252+0000", 2023-09-23T23:38:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6589999999999996e-06 2023-09-23T23:38:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.266580+0000", 2023-09-23T23:38:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053279240000000004 2023-09-23T23:38:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.266610+0000", 2023-09-23T23:38:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0003999999999999e-05 2023-09-23T23:38:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.299229+0000", 2023-09-23T23:38:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.032619170000000003 2023-09-23T23:38:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:18.299248+0000", 2023-09-23T23:38:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8289000000000001e-05 2023-09-23T23:38:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:170876 13.5 e608/570 13:a24e6f33:::benchmark_data_smithi110_541928_object83096:head v 608'10861, mlcod=538'122)", 2023-09-23T23:38:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:38:40.307314+0000", 2023-09-23T23:38:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 2.7513017909999999, 2023-09-23T23:38:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.055158410999999997, 2023-09-23T23:38:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307314+0000", 2023-09-23T23:38:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307314+0000", 2023-09-23T23:38:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307313+0000", 2023-09-23T23:38:43.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:38:43.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307321+0000", 2023-09-23T23:38:43.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7660000000000004e-06 2023-09-23T23:38:43.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307335+0000", 2023-09-23T23:38:43.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3811999999999999e-05 2023-09-23T23:38:43.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307343+0000", 2023-09-23T23:38:43.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6860000000000006e-06 2023-09-23T23:38:43.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307396+0000", 2023-09-23T23:38:43.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2679999999999997e-05 2023-09-23T23:38:43.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307423+0000", 2023-09-23T23:38:43.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6829e-05 2023-09-23T23:38:43.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.362419+0000", 2023-09-23T23:38:43.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.054996239000000002 2023-09-23T23:38:43.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.362472+0000", 2023-09-23T23:38:43.170 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3173000000000001e-05 2023-09-23T23:38:43.170 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.170 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.170 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.170 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:170882 13.5 e608/570 13:a85b78a2:::benchmark_data_smithi110_541928_object83102:head v 608'10862, mlcod=538'122)", 2023-09-23T23:38:43.170 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:38:40.307357+0000", 2023-09-23T23:38:43.170 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 2.751258107, 2023-09-23T23:38:43.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.055138026999999999, 2023-09-23T23:38:43.171 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.171 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.171 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307357+0000", 2023-09-23T23:38:43.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307357+0000", 2023-09-23T23:38:43.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.172 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307357+0000", 2023-09-23T23:38:43.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:43.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.172 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307361+0000", 2023-09-23T23:38:43.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8809999999999998e-06 2023-09-23T23:38:43.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307368+0000", 2023-09-23T23:38:43.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4030000000000001e-06 2023-09-23T23:38:43.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307372+0000", 2023-09-23T23:38:43.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0640000000000004e-06 2023-09-23T23:38:43.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.314831+0000", 2023-09-23T23:38:43.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074583119999999999 2023-09-23T23:38:43.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.314856+0000", 2023-09-23T23:38:43.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.535e-05 2023-09-23T23:38:43.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.362477+0000", 2023-09-23T23:38:43.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047620607000000002 2023-09-23T23:38:43.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.362495+0000", 2023-09-23T23:38:43.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8683000000000001e-05 2023-09-23T23:38:43.175 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.175 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.175 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.175 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:170886 13.5 e608/570 13:af22f136:::benchmark_data_smithi110_541928_object83106:head v 608'10863, mlcod=538'122)", 2023-09-23T23:38:43.175 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:38:40.307428+0000", 2023-09-23T23:38:43.176 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 2.7511869689999999, 2023-09-23T23:38:43.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.055089358999999997, 2023-09-23T23:38:43.176 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.176 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.176 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.176 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307428+0000", 2023-09-23T23:38:43.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.176 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307428+0000", 2023-09-23T23:38:43.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307428+0000", 2023-09-23T23:38:43.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:38:43.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307432+0000", 2023-09-23T23:38:43.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1219999999999997e-06 2023-09-23T23:38:43.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307439+0000", 2023-09-23T23:38:43.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6939999999999998e-06 2023-09-23T23:38:43.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.307443+0000", 2023-09-23T23:38:43.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.668e-06 2023-09-23T23:38:43.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.329437+0000", 2023-09-23T23:38:43.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021993181000000001 2023-09-23T23:38:43.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.329461+0000", 2023-09-23T23:38:43.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4011000000000001e-05 2023-09-23T23:38:43.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.362497+0000", 2023-09-23T23:38:43.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.033036744 2023-09-23T23:38:43.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:40.362518+0000", 2023-09-23T23:38:43.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0313e-05 2023-09-23T23:38:43.180 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.180 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.180 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.180 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:172348 13.5 e608/570 13:a0329dbd:::benchmark_data_smithi110_541928_object84568:head v 608'10959, mlcod=538'122)", 2023-09-23T23:38:43.181 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:38:42.201195+0000", 2023-09-23T23:38:43.181 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.85742000299999999, 2023-09-23T23:38:43.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.057286776999999997, 2023-09-23T23:38:43.181 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.181 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.181 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.201195+0000", 2023-09-23T23:38:43.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.201195+0000", 2023-09-23T23:38:43.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.201195+0000", 2023-09-23T23:38:43.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-09-23T23:38:43.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.201201+0000", 2023-09-23T23:38:43.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1739999999999997e-06 2023-09-23T23:38:43.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.201206+0000", 2023-09-23T23:38:43.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1460000000000004e-06 2023-09-23T23:38:43.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.201210+0000", 2023-09-23T23:38:43.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6500000000000002e-06 2023-09-23T23:38:43.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.211303+0000", 2023-09-23T23:38:43.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010092810000000001 2023-09-23T23:38:43.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.211329+0000", 2023-09-23T23:38:43.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6211e-05 2023-09-23T23:38:43.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.258439+0000", 2023-09-23T23:38:43.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047109905000000001 2023-09-23T23:38:43.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.258482+0000", 2023-09-23T23:38:43.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3047999999999998e-05 2023-09-23T23:38:43.185 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.185 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.185 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.186 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46514.0:172351 13.5 e608/570 13:ad1c0604:::benchmark_data_smithi110_541928_object84571:head v 608'10960, mlcod=538'122)", 2023-09-23T23:38:43.186 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:38:42.214609+0000", 2023-09-23T23:38:43.186 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.84400668700000003, 2023-09-23T23:38:43.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.043903931, 2023-09-23T23:38:43.186 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:38:43.186 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:38:43.186 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:38:43.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:38:43.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.214609+0000", 2023-09-23T23:38:43.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:38:43.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.214609+0000", 2023-09-23T23:38:43.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:38:43.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:38:43.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.214608+0000", 2023-09-23T23:38:43.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:38:43.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:38:43.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.214614+0000", 2023-09-23T23:38:43.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3400000000000003e-06 2023-09-23T23:38:43.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:38:43.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.214636+0000", 2023-09-23T23:38:43.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2019e-05 2023-09-23T23:38:43.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:38:43.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.214645+0000", 2023-09-23T23:38:43.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3359999999999995e-06 2023-09-23T23:38:43.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:38:43.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.222460+0000", 2023-09-23T23:38:43.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078154890000000001 2023-09-23T23:38:43.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:38:43.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.222490+0000", 2023-09-23T23:38:43.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0448000000000001e-05 2023-09-23T23:38:43.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:38:43.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.258485+0000", 2023-09-23T23:38:43.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.035994959 2023-09-23T23:38:43.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:38:43.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:38:43.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:38:43.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:38:42.258513+0000", 2023-09-23T23:38:43.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7101e-05 2023-09-23T23:38:43.190 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.190 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.190 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.190 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:38:43.191 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:38:43.191 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:38:43.191 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:43.462 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:38:43.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:43 smithi110 ceph-mon[212939]: pgmap v1733: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 342 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 18086/16458 objects degraded (109.892%); 266/16458 objects misplaced (1.616%); 0 B/s, 100 objects/s recovering 2023-09-23T23:38:43.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:38:43.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:43 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/278731792' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:43.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:43 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:43.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:43 smithi110 ceph-mon[221102]: pgmap v1733: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 342 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 18086/16458 objects degraded (109.892%); 266/16458 objects misplaced (1.616%); 0 B/s, 100 objects/s recovering 2023-09-23T23:38:43.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:38:43.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:43 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/278731792' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:43.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:43 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:43.713 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:43.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:43.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:43.840 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:43.891 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:43.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:43 smithi138 ceph-mon[193606]: pgmap v1733: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 342 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 18086/16458 objects degraded (109.892%); 266/16458 objects misplaced (1.616%); 0 B/s, 100 objects/s recovering 2023-09-23T23:38:43.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:38:43.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:43 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/278731792' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:43.945 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:43 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:38:44.854 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:44 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:38:44.854 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:44 smithi110 ceph-mon[221102]: failed to encode map e609 with expected crc 2023-09-23T23:38:44.854 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:44 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:38:44.854 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:44 smithi110 ceph-mon[221102]: osdmap e609: 8 total, 8 up, 7 in 2023-09-23T23:38:44.854 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:44 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 18086/16458 objects degraded (109.892%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:44.861 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:44 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:38:44.862 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:44 smithi110 ceph-mon[212939]: failed to encode map e609 with expected crc 2023-09-23T23:38:44.862 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:44 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:38:44.862 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:44 smithi110 ceph-mon[212939]: osdmap e609: 8 total, 8 up, 7 in 2023-09-23T23:38:44.862 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:44 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 18086/16458 objects degraded (109.892%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:44.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:44 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:38:44.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:44 smithi138 ceph-mon[193606]: failed to encode map e609 with expected crc 2023-09-23T23:38:44.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:44 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:38:44.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:44 smithi138 ceph-mon[193606]: osdmap e609: 8 total, 8 up, 7 in 2023-09-23T23:38:44.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:44 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 18086/16458 objects degraded (109.892%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:45.495 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:45.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:45.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:45.509 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:45 smithi110 ceph-mon[212939]: pgmap v1735: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 282 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 759 op/s; 17819/13569 objects degraded (131.321%); 25/13569 objects misplaced (0.184%); 0 B/s, 105 objects/s recovering 2023-09-23T23:38:45.513 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:45 smithi110 ceph-mon[221102]: pgmap v1735: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 282 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 759 op/s; 17819/13569 objects degraded (131.321%); 25/13569 objects misplaced (0.184%); 0 B/s, 105 objects/s recovering 2023-09-23T23:38:45.535 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:38:45.610 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:45.698 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:45.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:45 smithi138 ceph-mon[193606]: pgmap v1735: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 282 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 759 op/s; 17819/13569 objects degraded (131.321%); 25/13569 objects misplaced (0.184%); 0 B/s, 105 objects/s recovering 2023-09-23T23:38:46.433 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Removed 87777 objects 2023-09-23T23:38:46.433 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Clean up completed and total clean up time :257.509 2023-09-23T23:38:46.814 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:46.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:46.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:46.918 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:38:46.945 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:47.090 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:38:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:38:46] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2023-09-23T23:38:47.301 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:38:47.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:47.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:38:47.406 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:38:47.440 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:47.496 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:38:47.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:47.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:38:47.600 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:38:47.630 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:47.649 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:47.675 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:38:47.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:47.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:38:47.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:47 smithi110 ceph-mon[212939]: pgmap v1736: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 103 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 17551/4989 objects degraded (351.794%); 25/4989 objects misplaced (0.501%); 0 B/s, 105 objects/s recovering 2023-09-23T23:38:47.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:47 smithi110 ceph-mon[221102]: pgmap v1736: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 103 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 17551/4989 objects degraded (351.794%); 25/4989 objects misplaced (0.501%); 0 B/s, 105 objects/s recovering 2023-09-23T23:38:47.787 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:38:47.811 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:47 smithi138 ceph-mon[193606]: pgmap v1736: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 103 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 17551/4989 objects degraded (351.794%); 25/4989 objects misplaced (0.501%); 0 B/s, 105 objects/s recovering 2023-09-23T23:38:47.814 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:47.860 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:38:47.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:47.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:38:47.969 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:38:47.971 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:38:47.972 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:38:47.998 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:48.013 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:38:48.035 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:38:48.044 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:38:48.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:48.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:38:48.061 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:38:48.084 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:38:48.107 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:38:48.129 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:38:48.150 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:38:48.153 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:38:48.176 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:48.178 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:38:48.179 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:38:48.179 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:38:48.179 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:38:48.180 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:38:48.180 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:38:48.180 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:38:48.180 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:38:48.180 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:38:48.221 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:38:48.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:48.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:38:48.327 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:38:48.358 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:48.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:48 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2100493023' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:48.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:48 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2100493023' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:48.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:48 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2100493023' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:49.419 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:38:49.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:49 smithi110 ceph-mon[212939]: from='client.47099 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:38:49.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:49 smithi110 ceph-mon[212939]: pgmap v1737: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 103 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 17551/4989 objects degraded (351.794%); 25/4989 objects misplaced (0.501%); 0 B/s, 105 objects/s recovering 2023-09-23T23:38:49.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:49 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 17551/4989 objects degraded (351.794%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:49.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:49 smithi110 ceph-mon[221102]: from='client.47099 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:38:49.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:49 smithi110 ceph-mon[221102]: pgmap v1737: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 103 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 17551/4989 objects degraded (351.794%); 25/4989 objects misplaced (0.501%); 0 B/s, 105 objects/s recovering 2023-09-23T23:38:49.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:49 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 17551/4989 objects degraded (351.794%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:49.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:49 smithi138 ceph-mon[193606]: from='client.47099 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:38:49.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:49 smithi138 ceph-mon[193606]: pgmap v1737: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 103 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 17551/4989 objects degraded (351.794%); 25/4989 objects misplaced (0.501%); 0 B/s, 105 objects/s recovering 2023-09-23T23:38:49.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:49 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 17551/4989 objects degraded (351.794%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:50.461 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:38:50.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:50.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:38:50.564 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:38:50.592 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:50.650 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1736,"stamp":"2023-09-23T23:38:45.975284+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":107984117,"num_objects":1663,"num_object_clones":0,"num_object_copies":4989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17551,"num_objects_misplaced":25,"num_objects_unfound":0,"num_objects_dirty":1663,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":173994,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25161,"num_bytes_recovered":1518465448,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96041,"ondisk_log_size":96041,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":367,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6463052,"kb_used_data":563028,"kb_used_omap":29,"kb_used_meta":5899490,"kb_avail":649753012,"statfs":{"total":671965249536,"available":665347084288,"internally_reserved":0,"allocated":576540672,"data_stored":559872460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":6041078483},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,5,8,8,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-493944832,"num_objects":-7537,"num_object_clones":0,"num_object_copies":-22611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1058,"num_objects_misplaced":-483,"num_objects_unfound":0,"num_objects_dirty":-7537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7537,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1058,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005178"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":805,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597761+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:38:31.597761+0000","last_peered":"2023-09-23T23:38:31.597761+0000","last_clean":"2023-09-23T23:38:31.597761+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:38:31.597761+0000","last_undegraded":"2023-09-23T23:38:31.597761+0000","last_fullsized":"2023-09-23T23:38:31.597761+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":364,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.595132+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:38:31.595132+0000","last_peered":"2023-09-23T23:38:31.595132+0000","last_clean":"2023-09-23T23:38:31.595132+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:38:31.595132+0000","last_undegraded":"2023-09-23T23:38:31.595132+0000","last_fullsized":"2023-09-23T23:38:31.595132+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":591,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613388+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:38:31.613388+0000","last_peered":"2023-09-23T23:38:31.613388+0000","last_clean":"2023-09-23T23:38:31.613388+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:38:31.613388+0000","last_undegraded":"2023-09-23T23:38:31.613388+0000","last_fullsized":"2023-09-23T23:38:31.613388+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":758,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597404+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:38:31.597404+0000","last_peered":"2023-09-23T23:38:31.597404+0000","last_clean":"2023-09-23T23:38:31.597404+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:38:31.597404+0000","last_undegraded":"2023-09-23T23:38:31.597404+0000","last_fullsized":"2023-09-23T23:38:31.597404+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":327,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.376555+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:38:19.376555+0000","last_peered":"2023-09-23T23:38:19.376555+0000","last_clean":"2023-09-23T23:38:19.376555+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:38:19.376555+0000","last_undegraded":"2023-09-23T23:38:19.376555+0000","last_fullsized":"2023-09-23T23:38:19.376555+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":570,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:23.360582+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:38:23.360582+0000","last_peered":"2023-09-23T23:38:23.360582+0000","last_clean":"2023-09-23T23:38:23.360582+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:38:23.360582+0000","last_undegraded":"2023-09-23T23:38:23.360582+0000","last_fullsized":"2023-09-23T23:38:23.360582+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":769,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.233887+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:38:19.233887+0000","last_peered":"2023-09-23T23:38:19.233887+0000","last_clean":"2023-09-23T23:38:19.233887+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:38:19.233887+0000","last_undegraded":"2023-09-23T23:38:19.233887+0000","last_fullsized":"2023-09-23T23:38:19.233887+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":627,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.376293+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:38:19.376293+0000","last_peered":"2023-09-23T23:38:19.376293+0000","last_clean":"2023-09-23T23:38:19.376293+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:38:19.376293+0000","last_undegraded":"2023-09-23T23:38:19.376293+0000","last_fullsized":"2023-09-23T23:38:19.376293+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":123,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.615668+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:38:31.615668+0000","last_peered":"2023-09-23T23:38:31.615668+0000","last_clean":"2023-09-23T23:38:31.615668+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:38:31.615668+0000","last_undegraded":"2023-09-23T23:38:31.615668+0000","last_fullsized":"2023-09-23T23:38:31.615668+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":928,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.232807+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:38:19.232807+0000","last_peered":"2023-09-23T23:38:19.232807+0000","last_clean":"2023-09-23T23:38:19.232807+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:38:19.232807+0000","last_undegraded":"2023-09-23T23:38:19.232807+0000","last_fullsized":"2023-09-23T23:38:19.232807+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":261,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.604422+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:38:31.604422+0000","last_peered":"2023-09-23T23:38:31.604422+0000","last_clean":"2023-09-23T23:38:31.604422+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:38:31.604422+0000","last_undegraded":"2023-09-23T23:38:31.604422+0000","last_fullsized":"2023-09-23T23:38:31.604422+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":609,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.376221+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:38:19.376221+0000","last_peered":"2023-09-23T23:38:19.376221+0000","last_clean":"2023-09-23T23:38:19.376221+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:38:19.376221+0000","last_undegraded":"2023-09-23T23:38:19.376221+0000","last_fullsized":"2023-09-23T23:38:19.376221+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":903,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.594579+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:38:31.594579+0000","last_peered":"2023-09-23T23:38:31.594579+0000","last_clean":"2023-09-23T23:38:31.594579+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:38:31.594579+0000","last_undegraded":"2023-09-23T23:38:31.594579+0000","last_fullsized":"2023-09-23T23:38:31.594579+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":240,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.601308+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:38:31.601308+0000","last_peered":"2023-09-23T23:38:31.601308+0000","last_clean":"2023-09-23T23:38:31.601308+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:38:31.601308+0000","last_undegraded":"2023-09-23T23:38:31.601308+0000","last_fullsized":"2023-09-23T23:38:31.601308+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":570,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.376459+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:38:19.376459+0000","last_peered":"2023-09-23T23:38:19.376459+0000","last_clean":"2023-09-23T23:38:19.376459+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:38:19.376459+0000","last_undegraded":"2023-09-23T23:38:19.376459+0000","last_fullsized":"2023-09-23T23:38:19.376459+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1063,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597525+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:38:31.597525+0000","last_peered":"2023-09-23T23:38:31.597525+0000","last_clean":"2023-09-23T23:38:31.597525+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:38:31.597525+0000","last_undegraded":"2023-09-23T23:38:31.597525+0000","last_fullsized":"2023-09-23T23:38:31.597525+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":281,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.378644+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:38:19.378644+0000","last_peered":"2023-09-23T23:38:19.378644+0000","last_clean":"2023-09-23T23:38:19.378644+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:38:19.378644+0000","last_undegraded":"2023-09-23T23:38:19.378644+0000","last_fullsized":"2023-09-23T23:38:19.378644+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":142,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613588+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:38:31.613588+0000","last_peered":"2023-09-23T23:38:31.613588+0000","last_clean":"2023-09-23T23:38:31.613588+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:38:31.613588+0000","last_undegraded":"2023-09-23T23:38:31.613588+0000","last_fullsized":"2023-09-23T23:38:31.613588+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":459,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.381268+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:38:19.381268+0000","last_peered":"2023-09-23T23:38:19.381268+0000","last_clean":"2023-09-23T23:38:19.381268+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:38:19.381268+0000","last_undegraded":"2023-09-23T23:38:19.381268+0000","last_fullsized":"2023-09-23T23:38:19.381268+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":183,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613560+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:38:31.613560+0000","last_peered":"2023-09-23T23:38:31.613560+0000","last_clean":"2023-09-23T23:38:31.613560+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:38:31.613560+0000","last_undegraded":"2023-09-23T23:38:31.613560+0000","last_fullsized":"2023-09-23T23:38:31.613560+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":550,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612160+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:38:31.612160+0000","last_peered":"2023-09-23T23:38:31.612160+0000","last_clean":"2023-09-23T23:38:31.612160+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:38:31.612160+0000","last_undegraded":"2023-09-23T23:38:31.612160+0000","last_fullsized":"2023-09-23T23:38:31.612160+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":764,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597394+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:38:31.597394+0000","last_peered":"2023-09-23T23:38:31.597394+0000","last_clean":"2023-09-23T23:38:31.597394+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:38:31.597394+0000","last_undegraded":"2023-09-23T23:38:31.597394+0000","last_fullsized":"2023-09-23T23:38:31.597394+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":156,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.593780+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:38:31.593780+0000","last_peered":"2023-09-23T23:38:31.593780+0000","last_clean":"2023-09-23T23:38:31.593780+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:38:31.593780+0000","last_undegraded":"2023-09-23T23:38:31.593780+0000","last_fullsized":"2023-09-23T23:38:31.593780+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":608,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.376999+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:38:19.376999+0000","last_peered":"2023-09-23T23:38:19.376999+0000","last_clean":"2023-09-23T23:38:19.376999+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:38:19.376999+0000","last_undegraded":"2023-09-23T23:38:19.376999+0000","last_fullsized":"2023-09-23T23:38:19.376999+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":894,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612443+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:38:31.612443+0000","last_peered":"2023-09-23T23:38:31.612443+0000","last_clean":"2023-09-23T23:38:31.612443+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:38:31.612443+0000","last_undegraded":"2023-09-23T23:38:31.612443+0000","last_fullsized":"2023-09-23T23:38:31.612443+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":203,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.382958+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:38:19.382958+0000","last_peered":"2023-09-23T23:38:19.382958+0000","last_clean":"2023-09-23T23:38:19.382958+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:38:19.382958+0000","last_undegraded":"2023-09-23T23:38:19.382958+0000","last_fullsized":"2023-09-23T23:38:19.382958+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":330,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.378815+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:38:19.378815+0000","last_peered":"2023-09-23T23:38:19.378815+0000","last_clean":"2023-09-23T23:38:19.378815+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:38:19.378815+0000","last_undegraded":"2023-09-23T23:38:19.378815+0000","last_fullsized":"2023-09-23T23:38:19.378815+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":571,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.377422+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:38:19.377422+0000","last_peered":"2023-09-23T23:38:19.377422+0000","last_clean":"2023-09-23T23:38:19.377422+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:38:19.377422+0000","last_undegraded":"2023-09-23T23:38:19.377422+0000","last_fullsized":"2023-09-23T23:38:19.377422+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":477,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613545+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:38:31.613545+0000","last_peered":"2023-09-23T23:38:31.613545+0000","last_clean":"2023-09-23T23:38:31.613545+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:38:31.613545+0000","last_undegraded":"2023-09-23T23:38:31.613545+0000","last_fullsized":"2023-09-23T23:38:31.613545+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":565,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.381413+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:38:19.381413+0000","last_peered":"2023-09-23T23:38:19.381413+0000","last_clean":"2023-09-23T23:38:19.381413+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:38:19.381413+0000","last_undegraded":"2023-09-23T23:38:19.381413+0000","last_fullsized":"2023-09-23T23:38:19.381413+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":972,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597871+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:38:31.597871+0000","last_peered":"2023-09-23T23:38:31.597871+0000","last_clean":"2023-09-23T23:38:31.597871+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:38:31.597871+0000","last_undegraded":"2023-09-23T23:38:31.597871+0000","last_fullsized":"2023-09-23T23:38:31.597871+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":173,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612655+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:38:31.612655+0000","last_peered":"2023-09-23T23:38:31.612655+0000","last_clean":"2023-09-23T23:38:31.612655+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:38:31.612655+0000","last_undegraded":"2023-09-23T23:38:31.612655+0000","last_fullsized":"2023-09-23T23:38:31.612655+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":250,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.378320+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:38:19.378320+0000","last_peered":"2023-09-23T23:38:19.378320+0000","last_clean":"2023-09-23T23:38:19.378320+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:38:19.378320+0000","last_undegraded":"2023-09-23T23:38:19.378320+0000","last_fullsized":"2023-09-23T23:38:19.378320+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"609'10748","reported_seq":13147,"reported_epoch":609,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:44.261305+0000","last_change":"2023-09-23T23:38:22.063474+0000","last_active":"2023-09-23T23:38:44.261305+0000","last_peered":"2023-09-23T23:38:44.261305+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:38:44.261305+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"565'5000","ondisk_log_start":"565'5000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5748,"log_dups_size":0,"ondisk_log_size":5748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6553600,"num_objects":100,"num_object_clones":0,"num_object_copies":300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3245,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10748,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1153,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,6","objects":-3145}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":943,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.594116+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:38:31.594116+0000","last_peered":"2023-09-23T23:38:31.594116+0000","last_clean":"2023-09-23T23:38:31.594116+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:38:31.594116+0000","last_undegraded":"2023-09-23T23:38:31.594116+0000","last_fullsized":"2023-09-23T23:38:31.594116+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":249,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.233533+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:38:19.233533+0000","last_peered":"2023-09-23T23:38:19.233533+0000","last_clean":"2023-09-23T23:38:19.233533+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:38:19.233533+0000","last_undegraded":"2023-09-23T23:38:19.233533+0000","last_fullsized":"2023-09-23T23:38:19.233533+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":240,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597728+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:38:31.597728+0000","last_peered":"2023-09-23T23:38:31.597728+0000","last_clean":"2023-09-23T23:38:31.597728+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:38:31.597728+0000","last_undegraded":"2023-09-23T23:38:31.597728+0000","last_fullsized":"2023-09-23T23:38:31.597728+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"609'10983","reported_seq":11076,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:45.101704+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:38:45.101704+0000","last_peered":"2023-09-23T23:38:45.101704+0000","last_clean":"2023-09-23T23:38:45.101704+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:38:45.101704+0000","last_undegraded":"2023-09-23T23:38:45.101704+0000","last_fullsized":"2023-09-23T23:38:45.101704+0000","mapping_epoch":566,"log_start":"561'4800","ondisk_log_start":"561'4800","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6183,"log_dups_size":0,"ondisk_log_size":6183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3211264,"num_objects":49,"num_object_clones":0,"num_object_copies":147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":49,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10983,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1021,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.376730+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:38:19.376730+0000","last_peered":"2023-09-23T23:38:19.376730+0000","last_clean":"2023-09-23T23:38:19.376730+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:38:19.376730+0000","last_undegraded":"2023-09-23T23:38:19.376730+0000","last_fullsized":"2023-09-23T23:38:19.376730+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":239,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.233227+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:38:19.233227+0000","last_peered":"2023-09-23T23:38:19.233227+0000","last_clean":"2023-09-23T23:38:19.233227+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:38:19.233227+0000","last_undegraded":"2023-09-23T23:38:19.233227+0000","last_fullsized":"2023-09-23T23:38:19.233227+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":460,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:28.366684+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:38:28.366684+0000","last_peered":"2023-09-23T23:38:28.366684+0000","last_clean":"2023-09-23T23:38:28.366684+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:38:28.366684+0000","last_undegraded":"2023-09-23T23:38:28.366684+0000","last_fullsized":"2023-09-23T23:38:28.366684+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"608'10901","reported_seq":21042,"reported_epoch":608,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:38:43.424487+0000","last_change":"2023-09-23T23:34:37.372432+0000","last_active":"2023-09-23T23:38:43.424487+0000","last_peered":"2023-09-23T23:38:43.424487+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:38:43.424487+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:38:43.424487+0000","mapping_epoch":571,"log_start":"569'5600","ondisk_log_start":"569'5600","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5301,"log_dups_size":0,"ondisk_log_size":5301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9633792,"num_objects":147,"num_object_clones":0,"num_object_copies":441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":122,"num_objects_misplaced":25,"num_objects_unfound":0,"num_objects_dirty":147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10901,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5009,"num_bytes_recovered":328269824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,5,7","objects":147}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":919,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.615696+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:38:31.615696+0000","last_peered":"2023-09-23T23:38:31.615696+0000","last_clean":"2023-09-23T23:38:31.615696+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:38:31.615696+0000","last_undegraded":"2023-09-23T23:38:31.615696+0000","last_fullsized":"2023-09-23T23:38:31.615696+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":347,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.598441+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:38:31.598441+0000","last_peered":"2023-09-23T23:38:31.598441+0000","last_clean":"2023-09-23T23:38:31.598441+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:38:31.598441+0000","last_undegraded":"2023-09-23T23:38:31.598441+0000","last_fullsized":"2023-09-23T23:38:31.598441+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":123,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613712+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:38:31.613712+0000","last_peered":"2023-09-23T23:38:31.613712+0000","last_clean":"2023-09-23T23:38:31.613712+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:38:31.613712+0000","last_undegraded":"2023-09-23T23:38:31.613712+0000","last_fullsized":"2023-09-23T23:38:31.613712+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"608'10942","reported_seq":15089,"reported_epoch":608,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:43.449007+0000","last_change":"2023-09-23T23:38:22.122318+0000","last_active":"2023-09-23T23:38:43.449007+0000","last_peered":"2023-09-23T23:38:43.449007+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:38:43.449007+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"569'5600","ondisk_log_start":"569'5600","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5342,"log_dups_size":0,"ondisk_log_size":5342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9699328,"num_objects":148,"num_object_clones":0,"num_object_copies":444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2510,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10942,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2025,"num_bytes_recovered":72089600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,5","objects":-2362}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":515,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.381357+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:38:19.381357+0000","last_peered":"2023-09-23T23:38:19.381357+0000","last_clean":"2023-09-23T23:38:19.381357+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:38:19.381357+0000","last_undegraded":"2023-09-23T23:38:19.381357+0000","last_fullsized":"2023-09-23T23:38:19.381357+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":449,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.376882+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:38:19.376882+0000","last_peered":"2023-09-23T23:38:19.376882+0000","last_clean":"2023-09-23T23:38:19.376882+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:38:19.376882+0000","last_undegraded":"2023-09-23T23:38:19.376882+0000","last_fullsized":"2023-09-23T23:38:19.376882+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":606,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.379708+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:38:19.379708+0000","last_peered":"2023-09-23T23:38:19.379708+0000","last_clean":"2023-09-23T23:38:19.379708+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:38:19.379708+0000","last_undegraded":"2023-09-23T23:38:19.379708+0000","last_fullsized":"2023-09-23T23:38:19.379708+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"609'11066","reported_seq":12405,"reported_epoch":609,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:44.624923+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:38:44.624923+0000","last_peered":"2023-09-23T23:38:44.624923+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:38:44.624923+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"565'5400","ondisk_log_start":"565'5400","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5666,"log_dups_size":0,"ondisk_log_size":5666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6291456,"num_objects":96,"num_object_clones":0,"num_object_copies":288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":96,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11066,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,5","objects":-5364},{"shards":"3,5,6","objects":96}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":737,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.610763+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:38:31.610763+0000","last_peered":"2023-09-23T23:38:31.610763+0000","last_clean":"2023-09-23T23:38:31.610763+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:38:31.610763+0000","last_undegraded":"2023-09-23T23:38:31.610763+0000","last_fullsized":"2023-09-23T23:38:31.610763+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":619,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.377276+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:38:19.377276+0000","last_peered":"2023-09-23T23:38:19.377276+0000","last_clean":"2023-09-23T23:38:19.377276+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:38:19.377276+0000","last_undegraded":"2023-09-23T23:38:19.377276+0000","last_fullsized":"2023-09-23T23:38:19.377276+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":530,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:23.360484+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:38:23.360484+0000","last_peered":"2023-09-23T23:38:23.360484+0000","last_clean":"2023-09-23T23:38:23.360484+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:38:23.360484+0000","last_undegraded":"2023-09-23T23:38:23.360484+0000","last_fullsized":"2023-09-23T23:38:23.360484+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"609'10986","reported_seq":11069,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:44.265310+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:38:44.265310+0000","last_peered":"2023-09-23T23:38:44.265310+0000","last_clean":"2023-09-23T23:38:44.265310+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:38:44.265310+0000","last_undegraded":"2023-09-23T23:38:44.265310+0000","last_fullsized":"2023-09-23T23:38:44.265310+0000","mapping_epoch":534,"log_start":"565'5300","ondisk_log_start":"565'5300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5686,"log_dups_size":0,"ondisk_log_size":5686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6160384,"num_objects":94,"num_object_clones":0,"num_object_copies":282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":94,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10986,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":893,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.593859+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:38:31.593859+0000","last_peered":"2023-09-23T23:38:31.593859+0000","last_clean":"2023-09-23T23:38:31.593859+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:38:31.593859+0000","last_undegraded":"2023-09-23T23:38:31.593859+0000","last_fullsized":"2023-09-23T23:38:31.593859+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":133,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612048+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:38:31.612048+0000","last_peered":"2023-09-23T23:38:31.612048+0000","last_clean":"2023-09-23T23:38:31.612048+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:38:31.612048+0000","last_undegraded":"2023-09-23T23:38:31.612048+0000","last_fullsized":"2023-09-23T23:38:31.612048+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":476,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.594324+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:38:31.594324+0000","last_peered":"2023-09-23T23:38:31.594324+0000","last_clean":"2023-09-23T23:38:31.594324+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:38:31.594324+0000","last_undegraded":"2023-09-23T23:38:31.594324+0000","last_fullsized":"2023-09-23T23:38:31.594324+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"609'11011","reported_seq":20212,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:44.252597+0000","last_change":"2023-09-23T23:38:22.063236+0000","last_active":"2023-09-23T23:38:44.252597+0000","last_peered":"2023-09-23T23:38:44.252597+0000","last_clean":"2023-09-23T23:38:44.252597+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:38:44.252597+0000","last_undegraded":"2023-09-23T23:38:44.252597+0000","last_fullsized":"2023-09-23T23:38:44.252597+0000","mapping_epoch":567,"log_start":"565'5300","ondisk_log_start":"565'5300","created":534,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5711,"log_dups_size":0,"ondisk_log_size":5711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6094848,"num_objects":93,"num_object_clones":0,"num_object_copies":279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":93,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11011,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4547,"num_bytes_recovered":297992192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":617,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597981+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:38:31.597981+0000","last_peered":"2023-09-23T23:38:31.597981+0000","last_clean":"2023-09-23T23:38:31.597981+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:38:31.597981+0000","last_undegraded":"2023-09-23T23:38:31.597981+0000","last_fullsized":"2023-09-23T23:38:31.597981+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":473,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.593707+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:38:31.593707+0000","last_peered":"2023-09-23T23:38:31.593707+0000","last_clean":"2023-09-23T23:38:31.593707+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:38:31.593707+0000","last_undegraded":"2023-09-23T23:38:31.593707+0000","last_fullsized":"2023-09-23T23:38:31.593707+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":560,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612121+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:38:31.612121+0000","last_peered":"2023-09-23T23:38:31.612121+0000","last_clean":"2023-09-23T23:38:31.612121+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:38:31.612121+0000","last_undegraded":"2023-09-23T23:38:31.612121+0000","last_fullsized":"2023-09-23T23:38:31.612121+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"609'10929","reported_seq":19975,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:44.599264+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:38:44.599264+0000","last_peered":"2023-09-23T23:38:44.599264+0000","last_clean":"2023-09-23T23:38:44.599264+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:38:44.599264+0000","last_undegraded":"2023-09-23T23:38:44.599264+0000","last_fullsized":"2023-09-23T23:38:44.599264+0000","mapping_epoch":579,"log_start":"565'5200","ondisk_log_start":"565'5200","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5729,"log_dups_size":0,"ondisk_log_size":5729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5701632,"num_objects":87,"num_object_clones":0,"num_object_copies":261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10929,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":951,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.595007+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:38:31.595007+0000","last_peered":"2023-09-23T23:38:31.595007+0000","last_clean":"2023-09-23T23:38:31.595007+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:38:31.595007+0000","last_undegraded":"2023-09-23T23:38:31.595007+0000","last_fullsized":"2023-09-23T23:38:31.595007+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":250,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.376055+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:38:19.376055+0000","last_peered":"2023-09-23T23:38:19.376055+0000","last_clean":"2023-09-23T23:38:19.376055+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:38:19.376055+0000","last_undegraded":"2023-09-23T23:38:19.376055+0000","last_fullsized":"2023-09-23T23:38:19.376055+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":539,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612161+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:38:31.612161+0000","last_peered":"2023-09-23T23:38:31.612161+0000","last_clean":"2023-09-23T23:38:31.612161+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:38:31.612161+0000","last_undegraded":"2023-09-23T23:38:31.612161+0000","last_fullsized":"2023-09-23T23:38:31.612161+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"608'10592","reported_seq":10677,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:43.432608+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:38:43.432608+0000","last_peered":"2023-09-23T23:38:43.432608+0000","last_clean":"2023-09-23T23:38:43.432608+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:38:43.432608+0000","last_undegraded":"2023-09-23T23:38:43.432608+0000","last_fullsized":"2023-09-23T23:38:43.432608+0000","mapping_epoch":534,"log_start":"569'5300","ondisk_log_start":"569'5300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5292,"log_dups_size":0,"ondisk_log_size":5292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8912896,"num_objects":136,"num_object_clones":0,"num_object_copies":408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10592,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":930,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.615532+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:38:31.615532+0000","last_peered":"2023-09-23T23:38:31.615532+0000","last_clean":"2023-09-23T23:38:31.615532+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:38:31.615532+0000","last_undegraded":"2023-09-23T23:38:31.615532+0000","last_fullsized":"2023-09-23T23:38:31.615532+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":318,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.384192+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:38:19.384192+0000","last_peered":"2023-09-23T23:38:19.384192+0000","last_clean":"2023-09-23T23:38:19.384192+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:38:19.384192+0000","last_undegraded":"2023-09-23T23:38:19.384192+0000","last_fullsized":"2023-09-23T23:38:19.384192+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":624,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.598620+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:38:31.598620+0000","last_peered":"2023-09-23T23:38:31.598620+0000","last_clean":"2023-09-23T23:38:31.598620+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:38:31.598620+0000","last_undegraded":"2023-09-23T23:38:31.598620+0000","last_fullsized":"2023-09-23T23:38:31.598620+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":205,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597694+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:38:31.597694+0000","last_peered":"2023-09-23T23:38:31.597694+0000","last_clean":"2023-09-23T23:38:31.597694+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:38:31.597694+0000","last_undegraded":"2023-09-23T23:38:31.597694+0000","last_fullsized":"2023-09-23T23:38:31.597694+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"608'10992","reported_seq":20130,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:43.440452+0000","last_change":"2023-09-23T23:38:20.181875+0000","last_active":"2023-09-23T23:38:43.440452+0000","last_peered":"2023-09-23T23:38:43.440452+0000","last_clean":"2023-09-23T23:38:43.440452+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:38:43.440452+0000","last_undegraded":"2023-09-23T23:38:43.440452+0000","last_fullsized":"2023-09-23T23:38:43.440452+0000","mapping_epoch":566,"log_start":"571'5722","ondisk_log_start":"571'5722","created":534,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5270,"log_dups_size":0,"ondisk_log_size":5270,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11468824,"num_objects":176,"num_object_clones":0,"num_object_copies":528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":176,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10992,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4515,"num_bytes_recovered":295895040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":692,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.378183+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:38:19.378183+0000","last_peered":"2023-09-23T23:38:19.378183+0000","last_clean":"2023-09-23T23:38:19.378183+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:38:19.378183+0000","last_undegraded":"2023-09-23T23:38:19.378183+0000","last_fullsized":"2023-09-23T23:38:19.378183+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":257,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613655+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:38:31.613655+0000","last_peered":"2023-09-23T23:38:31.613655+0000","last_clean":"2023-09-23T23:38:31.613655+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:38:31.613655+0000","last_undegraded":"2023-09-23T23:38:31.613655+0000","last_fullsized":"2023-09-23T23:38:31.613655+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":541,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612055+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:38:31.612055+0000","last_peered":"2023-09-23T23:38:31.612055+0000","last_clean":"2023-09-23T23:38:31.612055+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:38:31.612055+0000","last_undegraded":"2023-09-23T23:38:31.612055+0000","last_fullsized":"2023-09-23T23:38:31.612055+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":243,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.594649+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:38:31.594649+0000","last_peered":"2023-09-23T23:38:31.594649+0000","last_clean":"2023-09-23T23:38:31.594649+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:38:31.594649+0000","last_undegraded":"2023-09-23T23:38:31.594649+0000","last_fullsized":"2023-09-23T23:38:31.594649+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"609'10678","reported_seq":12740,"reported_epoch":609,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:44.635389+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:38:44.635389+0000","last_peered":"2023-09-23T23:38:44.635389+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:38:44.635389+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"565'5000","ondisk_log_start":"565'5000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5678,"log_dups_size":0,"ondisk_log_size":5678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6160384,"num_objects":94,"num_object_clones":0,"num_object_copies":282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":94,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10678,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"5,7","objects":-3298}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":911,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.594903+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:38:31.594903+0000","last_peered":"2023-09-23T23:38:31.594903+0000","last_clean":"2023-09-23T23:38:31.594903+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:38:31.594903+0000","last_undegraded":"2023-09-23T23:38:31.594903+0000","last_fullsized":"2023-09-23T23:38:31.594903+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":348,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.593586+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:38:31.593586+0000","last_peered":"2023-09-23T23:38:31.593586+0000","last_clean":"2023-09-23T23:38:31.593586+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:38:31.593586+0000","last_undegraded":"2023-09-23T23:38:31.593586+0000","last_fullsized":"2023-09-23T23:38:31.593586+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":258,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.593938+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:38:31.593938+0000","last_peered":"2023-09-23T23:38:31.593938+0000","last_clean":"2023-09-23T23:38:31.593938+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:38:31.593938+0000","last_undegraded":"2023-09-23T23:38:31.593938+0000","last_fullsized":"2023-09-23T23:38:31.593938+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":221,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.384506+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:38:19.384506+0000","last_peered":"2023-09-23T23:38:19.384506+0000","last_clean":"2023-09-23T23:38:19.384506+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:38:19.384506+0000","last_undegraded":"2023-09-23T23:38:19.384506+0000","last_fullsized":"2023-09-23T23:38:19.384506+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"609'10815","reported_seq":10908,"reported_epoch":609,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:38:44.606459+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:38:44.606459+0000","last_peered":"2023-09-23T23:38:44.606459+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:38:44.606459+0000","last_undegraded":"2023-09-23T23:38:44.606459+0000","last_fullsized":"2023-09-23T23:38:44.606459+0000","mapping_epoch":562,"log_start":"565'5100","ondisk_log_start":"565'5100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5715,"log_dups_size":0,"ondisk_log_size":5715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5701632,"num_objects":87,"num_object_clones":0,"num_object_copies":261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10815,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":87}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":625,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.593897+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:38:31.593897+0000","last_peered":"2023-09-23T23:38:31.593897+0000","last_clean":"2023-09-23T23:38:31.593897+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:38:31.593897+0000","last_undegraded":"2023-09-23T23:38:31.593897+0000","last_fullsized":"2023-09-23T23:38:31.593897+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":895,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612470+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:38:31.612470+0000","last_peered":"2023-09-23T23:38:31.612470+0000","last_clean":"2023-09-23T23:38:31.612470+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:38:31.612470+0000","last_undegraded":"2023-09-23T23:38:31.612470+0000","last_fullsized":"2023-09-23T23:38:31.612470+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":571,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.380814+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:38:19.380814+0000","last_peered":"2023-09-23T23:38:19.380814+0000","last_clean":"2023-09-23T23:38:19.380814+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:38:19.380814+0000","last_undegraded":"2023-09-23T23:38:19.380814+0000","last_fullsized":"2023-09-23T23:38:19.380814+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":133,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.614522+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:38:31.614522+0000","last_peered":"2023-09-23T23:38:31.614522+0000","last_clean":"2023-09-23T23:38:31.614522+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:38:31.614522+0000","last_undegraded":"2023-09-23T23:38:31.614522+0000","last_fullsized":"2023-09-23T23:38:31.614522+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"609'10616","reported_seq":10699,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:44.215534+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:38:44.215534+0000","last_peered":"2023-09-23T23:38:44.215534+0000","last_clean":"2023-09-23T23:38:44.215534+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:38:44.215534+0000","last_undegraded":"2023-09-23T23:38:44.215534+0000","last_fullsized":"2023-09-23T23:38:44.215534+0000","mapping_epoch":534,"log_start":"564'4900","ondisk_log_start":"564'4900","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5716,"log_dups_size":0,"ondisk_log_size":5716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7864320,"num_objects":120,"num_object_clones":0,"num_object_copies":360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10616,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":721,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613387+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:38:31.613387+0000","last_peered":"2023-09-23T23:38:31.613387+0000","last_clean":"2023-09-23T23:38:31.613387+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:38:31.613387+0000","last_undegraded":"2023-09-23T23:38:31.613387+0000","last_fullsized":"2023-09-23T23:38:31.613387+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":535,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.232094+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:38:19.232094+0000","last_peered":"2023-09-23T23:38:19.232094+0000","last_clean":"2023-09-23T23:38:19.232094+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:38:19.232094+0000","last_undegraded":"2023-09-23T23:38:19.232094+0000","last_fullsized":"2023-09-23T23:38:19.232094+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":450,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.376163+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:38:19.376163+0000","last_peered":"2023-09-23T23:38:19.376163+0000","last_clean":"2023-09-23T23:38:19.376163+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:38:19.376163+0000","last_undegraded":"2023-09-23T23:38:19.376163+0000","last_fullsized":"2023-09-23T23:38:19.376163+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":49,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.378322+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:38:19.378322+0000","last_peered":"2023-09-23T23:38:19.378322+0000","last_clean":"2023-09-23T23:38:19.378322+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:38:19.378322+0000","last_undegraded":"2023-09-23T23:38:19.378322+0000","last_fullsized":"2023-09-23T23:38:19.378322+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"609'10934","reported_seq":14617,"reported_epoch":609,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:45.093688+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:38:45.093688+0000","last_peered":"2023-09-23T23:38:45.093688+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:38:45.093688+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"561'4800","ondisk_log_start":"561'4800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6134,"log_dups_size":0,"ondisk_log_size":6134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":64,"num_object_clones":0,"num_object_copies":192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10934,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":-2640}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":604,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.615468+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:38:31.615468+0000","last_peered":"2023-09-23T23:38:31.615468+0000","last_clean":"2023-09-23T23:38:31.615468+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:38:31.615468+0000","last_undegraded":"2023-09-23T23:38:31.615468+0000","last_fullsized":"2023-09-23T23:38:31.615468+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":49,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.378986+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:38:19.378986+0000","last_peered":"2023-09-23T23:38:19.378986+0000","last_clean":"2023-09-23T23:38:19.378986+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:38:19.378986+0000","last_undegraded":"2023-09-23T23:38:19.378986+0000","last_fullsized":"2023-09-23T23:38:19.378986+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":150,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.594719+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:38:31.594719+0000","last_peered":"2023-09-23T23:38:31.594719+0000","last_clean":"2023-09-23T23:38:31.594719+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:38:31.594719+0000","last_undegraded":"2023-09-23T23:38:31.594719+0000","last_fullsized":"2023-09-23T23:38:31.594719+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":342,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.598071+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:38:31.598071+0000","last_peered":"2023-09-23T23:38:31.598071+0000","last_clean":"2023-09-23T23:38:31.598071+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:38:31.598071+0000","last_undegraded":"2023-09-23T23:38:31.598071+0000","last_fullsized":"2023-09-23T23:38:31.598071+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"609'10874","reported_seq":10957,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:44.458372+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:38:44.458372+0000","last_peered":"2023-09-23T23:38:44.458372+0000","last_clean":"2023-09-23T23:38:44.458372+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:38:44.458372+0000","last_undegraded":"2023-09-23T23:38:44.458372+0000","last_fullsized":"2023-09-23T23:38:44.458372+0000","mapping_epoch":534,"log_start":"565'5200","ondisk_log_start":"565'5200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5674,"log_dups_size":0,"ondisk_log_size":5674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5898240,"num_objects":90,"num_object_clones":0,"num_object_copies":270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10874,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":791,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.594393+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:38:31.594393+0000","last_peered":"2023-09-23T23:38:31.594393+0000","last_clean":"2023-09-23T23:38:31.594393+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:38:31.594393+0000","last_undegraded":"2023-09-23T23:38:31.594393+0000","last_fullsized":"2023-09-23T23:38:31.594393+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":606,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.378464+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:38:19.378464+0000","last_peered":"2023-09-23T23:38:19.378464+0000","last_clean":"2023-09-23T23:38:19.378464+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:38:19.378464+0000","last_undegraded":"2023-09-23T23:38:19.378464+0000","last_fullsized":"2023-09-23T23:38:19.378464+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":213,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.614288+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:38:31.614288+0000","last_peered":"2023-09-23T23:38:31.614288+0000","last_clean":"2023-09-23T23:38:31.614288+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:38:31.614288+0000","last_undegraded":"2023-09-23T23:38:31.614288+0000","last_fullsized":"2023-09-23T23:38:31.614288+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":161,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.593722+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:38:31.593722+0000","last_peered":"2023-09-23T23:38:31.593722+0000","last_clean":"2023-09-23T23:38:31.593722+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:38:31.593722+0000","last_undegraded":"2023-09-23T23:38:31.593722+0000","last_fullsized":"2023-09-23T23:38:31.593722+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"609'10850","reported_seq":10936,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:45.210239+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:38:45.210239+0000","last_peered":"2023-09-23T23:38:45.210239+0000","last_clean":"2023-09-23T23:38:45.210239+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:38:45.210239+0000","last_undegraded":"2023-09-23T23:38:45.210239+0000","last_fullsized":"2023-09-23T23:38:45.210239+0000","mapping_epoch":534,"log_start":"561'4700","ondisk_log_start":"561'4700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6150,"log_dups_size":0,"ondisk_log_size":6150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3801088,"num_objects":58,"num_object_clones":0,"num_object_copies":174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10850,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":996,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.377121+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:38:19.377121+0000","last_peered":"2023-09-23T23:38:19.377121+0000","last_clean":"2023-09-23T23:38:19.377121+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:38:19.377121+0000","last_undegraded":"2023-09-23T23:38:19.377121+0000","last_fullsized":"2023-09-23T23:38:19.377121+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":123,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.615595+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:38:31.615595+0000","last_peered":"2023-09-23T23:38:31.615595+0000","last_clean":"2023-09-23T23:38:31.615595+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:38:31.615595+0000","last_undegraded":"2023-09-23T23:38:31.615595+0000","last_fullsized":"2023-09-23T23:38:31.615595+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":53,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.378409+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:38:19.378409+0000","last_peered":"2023-09-23T23:38:19.378409+0000","last_clean":"2023-09-23T23:38:19.378409+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:38:19.378409+0000","last_undegraded":"2023-09-23T23:38:19.378409+0000","last_fullsized":"2023-09-23T23:38:19.378409+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":183,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597378+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:38:31.597378+0000","last_peered":"2023-09-23T23:38:31.597378+0000","last_clean":"2023-09-23T23:38:31.597378+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:38:31.597378+0000","last_undegraded":"2023-09-23T23:38:31.597378+0000","last_fullsized":"2023-09-23T23:38:31.597378+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1045,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.376303+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:38:19.376303+0000","last_peered":"2023-09-23T23:38:19.376303+0000","last_clean":"2023-09-23T23:38:19.376303+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:38:19.376303+0000","last_undegraded":"2023-09-23T23:38:19.376303+0000","last_fullsized":"2023-09-23T23:38:19.376303+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":49,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.379739+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:38:19.379739+0000","last_peered":"2023-09-23T23:38:19.379739+0000","last_clean":"2023-09-23T23:38:19.379739+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:38:19.379739+0000","last_undegraded":"2023-09-23T23:38:19.379739+0000","last_fullsized":"2023-09-23T23:38:19.379739+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":365,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.594786+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:38:31.594786+0000","last_peered":"2023-09-23T23:38:31.594786+0000","last_clean":"2023-09-23T23:38:31.594786+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:38:31.594786+0000","last_undegraded":"2023-09-23T23:38:31.594786+0000","last_fullsized":"2023-09-23T23:38:31.594786+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":757,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.381568+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:38:19.381568+0000","last_peered":"2023-09-23T23:38:19.381568+0000","last_clean":"2023-09-23T23:38:19.381568+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:38:19.381568+0000","last_undegraded":"2023-09-23T23:38:19.381568+0000","last_fullsized":"2023-09-23T23:38:19.381568+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":111,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613302+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:38:31.613302+0000","last_peered":"2023-09-23T23:38:31.613302+0000","last_clean":"2023-09-23T23:38:31.613302+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:38:31.613302+0000","last_undegraded":"2023-09-23T23:38:31.613302+0000","last_fullsized":"2023-09-23T23:38:31.613302+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":456,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.377202+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:38:19.377202+0000","last_peered":"2023-09-23T23:38:19.377202+0000","last_clean":"2023-09-23T23:38:19.377202+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:38:19.377202+0000","last_undegraded":"2023-09-23T23:38:19.377202+0000","last_fullsized":"2023-09-23T23:38:19.377202+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":307,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612256+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:38:31.612256+0000","last_peered":"2023-09-23T23:38:31.612256+0000","last_clean":"2023-09-23T23:38:31.612256+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:38:31.612256+0000","last_undegraded":"2023-09-23T23:38:31.612256+0000","last_fullsized":"2023-09-23T23:38:31.612256+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":295,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.382956+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:38:19.382956+0000","last_peered":"2023-09-23T23:38:19.382956+0000","last_clean":"2023-09-23T23:38:19.382956+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:38:19.382956+0000","last_undegraded":"2023-09-23T23:38:19.382956+0000","last_fullsized":"2023-09-23T23:38:19.382956+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":821,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613682+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:38:31.613682+0000","last_peered":"2023-09-23T23:38:31.613682+0000","last_clean":"2023-09-23T23:38:31.613682+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:38:31.613682+0000","last_undegraded":"2023-09-23T23:38:31.613682+0000","last_fullsized":"2023-09-23T23:38:31.613682+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":639,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.381495+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:38:19.381495+0000","last_peered":"2023-09-23T23:38:19.381495+0000","last_clean":"2023-09-23T23:38:19.381495+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:38:19.381495+0000","last_undegraded":"2023-09-23T23:38:19.381495+0000","last_fullsized":"2023-09-23T23:38:19.381495+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":623,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.376533+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:38:19.376533+0000","last_peered":"2023-09-23T23:38:19.376533+0000","last_clean":"2023-09-23T23:38:19.376533+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:38:19.376533+0000","last_undegraded":"2023-09-23T23:38:19.376533+0000","last_fullsized":"2023-09-23T23:38:19.376533+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":781,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T23:38:19.376729+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:38:19.376729+0000","last_peered":"2023-09-23T23:38:19.376729+0000","last_clean":"2023-09-23T23:38:19.376729+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:38:19.376729+0000","last_undegraded":"2023-09-23T23:38:19.376729+0000","last_fullsized":"2023-09-23T23:38:19.376729+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":453,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.594266+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:38:31.594266+0000","last_peered":"2023-09-23T23:38:31.594266+0000","last_clean":"2023-09-23T23:38:31.594266+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:38:31.594266+0000","last_undegraded":"2023-09-23T23:38:31.594266+0000","last_fullsized":"2023-09-23T23:38:31.594266+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":676,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612386+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:38:31.612386+0000","last_peered":"2023-09-23T23:38:31.612386+0000","last_clean":"2023-09-23T23:38:31.612386+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:38:31.612386+0000","last_undegraded":"2023-09-23T23:38:31.612386+0000","last_fullsized":"2023-09-23T23:38:31.612386+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":107347992,"num_objects":1639,"num_object_clones":0,"num_object_copies":4917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17551,"num_objects_misplaced":25,"num_objects_unfound":0,"num_objects_dirty":1639,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":173917,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25106,"num_bytes_recovered":1517355008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2703175680,"data_stored":2703163464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90995,"ondisk_log_size":90995,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782711,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":518672,"kb_used_data":113288,"kb_used_omap":8,"kb_used_meta":405239,"kb_avail":93226480,"statfs":{"total":95995035648,"available":95463915520,"internally_reserved":0,"allocated":116006912,"data_stored":114302306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":414964805},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.81599999999999995,"15min":2.0459999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":2.4159999999999999,"5min":11.173,"15min":85.721000000000004},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.77300000000000002,"15min":2.145},"min":{"1min":0.24199999999999999,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":2.5499999999999998,"5min":5.2530000000000001,"15min":85.965000000000003},"last":0.39700000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":2.1549999999999998,"15min":3.105},"min":{"1min":0.255,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":2.6739999999999999,"5min":129.11199999999999,"15min":198.58000000000001},"last":0.313},{"interface":"front","average":{"1min":0.61299999999999999,"5min":2.036,"15min":3.0870000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.2850000000000001,"5min":129.02600000000001,"15min":198.71100000000001},"last":0.41599999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":2.4180000000000001,"15min":3.7330000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":3.3860000000000001,"5min":78.528999999999996,"15min":203.58000000000001},"last":1.536},{"interface":"front","average":{"1min":1.141,"5min":1.712,"15min":3.2200000000000002},"min":{"1min":0.30399999999999999,"5min":0.27900000000000003,"15min":0.26200000000000001},"max":{"1min":6.0700000000000003,"5min":53.023000000000003,"15min":198.69},"last":0.46800000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":14.901999999999999,"5min":7.1070000000000002,"15min":4.9690000000000003},"min":{"1min":0.33000000000000002,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":154.18100000000001,"5min":154.18100000000001,"15min":279.26499999999999},"last":1.6639999999999999},{"interface":"front","average":{"1min":14.157999999999999,"5min":7.3140000000000001,"15min":5.476},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":154.16300000000001,"5min":154.16300000000001,"15min":279.29899999999998},"last":1.4470000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.464,"15min":1.026},"min":{"1min":0.318,"5min":0.27100000000000002,"15min":0.246},"max":{"1min":2.0499999999999998,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.339,"15min":0.97399999999999998},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.23899999999999999},"max":{"1min":5.391,"5min":31.68,"15min":31.68},"last":0.33100000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":7.1280000000000001,"5min":2.7919999999999998,"15min":2.266},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.26500000000000001},"max":{"1min":119.417,"5min":119.417,"15min":210.83600000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":7.6669999999999998,"5min":3.016,"15min":2.1989999999999998},"min":{"1min":0.33600000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":119.61499999999999,"5min":119.61499999999999,"15min":207.04599999999999},"last":0.29999999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":1.8899999999999999,"15min":1.3420000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":6.2169999999999996,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.434},{"interface":"front","average":{"1min":0.63,"5min":1.956,"15min":1.514},"min":{"1min":0.40100000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.911,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.34899999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496188,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":860232,"kb_used_data":122368,"kb_used_omap":0,"kb_used_meta":737856,"kb_avail":92884920,"statfs":{"total":95995035648,"available":95114158080,"internally_reserved":0,"allocated":125304832,"data_stored":123615898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":755564544},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:26 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":0.90400000000000003,"15min":3.956},"min":{"1min":0.224,"5min":0.224,"15min":0.21099999999999999},"max":{"1min":6.5579999999999998,"5min":6.5579999999999998,"15min":135.31999999999999},"last":0.27400000000000002},{"interface":"front","average":{"1min":1.1850000000000001,"5min":0.70899999999999996,"15min":3.891},"min":{"1min":0.28599999999999998,"5min":0.24399999999999999,"15min":0.23000000000000001},"max":{"1min":6.8529999999999998,"5min":6.8529999999999998,"15min":135.34299999999999},"last":0.31900000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:38:26 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":1.018,"15min":2.1720000000000002},"min":{"1min":0.28499999999999998,"5min":0.253,"15min":0.245},"max":{"1min":6.6200000000000001,"5min":6.9080000000000004,"15min":89.605000000000004},"last":1.9399999999999999},{"interface":"front","average":{"1min":1.4990000000000001,"5min":0.873,"15min":2.2749999999999999},"min":{"1min":0.26300000000000001,"5min":0.249,"15min":0.23300000000000001},"max":{"1min":12.577,"5min":12.577,"15min":91.025999999999996},"last":0.254}]},{"osd":2,"last update":"Sat Sep 23 23:38:10 2023","interfaces":[{"interface":"back","average":{"1min":7.9569999999999999,"5min":5.032,"15min":4.9539999999999997},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":85.995999999999995,"5min":85.995999999999995,"15min":127.44499999999999},"last":0.90000000000000002},{"interface":"front","average":{"1min":7.6959999999999997,"5min":5.0250000000000004,"15min":5.3319999999999999},"min":{"1min":0.32300000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":85.266999999999996,"5min":85.266999999999996,"15min":125.539},"last":2.0779999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:37:57 2023","interfaces":[{"interface":"back","average":{"1min":13.925000000000001,"5min":8.1059999999999999,"15min":5.2009999999999996},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":199.536,"5min":199.536,"15min":199.536},"last":0.51600000000000001},{"interface":"front","average":{"1min":13.538,"5min":7.9660000000000002,"15min":5.4530000000000003},"min":{"1min":0.36799999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":199.59999999999999,"5min":199.59999999999999,"15min":199.59999999999999},"last":0.41999999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:38:31 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.73599999999999999,"15min":0.69499999999999995},"min":{"1min":0.29799999999999999,"5min":0.27900000000000003,"15min":0.23699999999999999},"max":{"1min":5.7619999999999996,"5min":5.7619999999999996,"15min":8.1999999999999993},"last":0.876},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.085,"15min":0.85899999999999999},"min":{"1min":0.32400000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":12.124000000000001,"5min":12.124000000000001,"15min":12.124000000000001},"last":0.47399999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:38:31 2023","interfaces":[{"interface":"back","average":{"1min":1.637,"5min":0.96899999999999997,"15min":0.871},"min":{"1min":0.32400000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.8490000000000002,"5min":11.372,"15min":11.473000000000001},"last":0.84299999999999997},{"interface":"front","average":{"1min":2.133,"5min":1.1679999999999999,"15min":0.94999999999999996},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26900000000000002},"max":{"1min":12.323,"5min":12.323,"15min":12.323},"last":3.4769999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:38:10 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.25,"15min":1.012},"min":{"1min":0.33600000000000002,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":6.8259999999999996,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.331,"15min":1.079},"min":{"1min":0.38300000000000001,"5min":0.33800000000000002,"15min":0.33500000000000002},"max":{"1min":12.531000000000001,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.92300000000000004}]}]},{"osd":1,"up_from":112,"seq":481036337529,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":980824,"kb_used_data":66064,"kb_used_omap":4,"kb_used_meta":914683,"kb_avail":92764328,"statfs":{"total":95995035648,"available":94990671872,"internally_reserved":0,"allocated":67649536,"data_stored":64771039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":936636302},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":0.98699999999999999,"15min":3.1669999999999998},"min":{"1min":0.16400000000000001,"5min":0.16400000000000001,"15min":0.16400000000000001},"max":{"1min":7.1070000000000002,"5min":15.983000000000001,"15min":130.768},"last":0.23200000000000001},{"interface":"front","average":{"1min":0.78800000000000003,"5min":0.65200000000000002,"15min":2.5150000000000001},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":5.726,"5min":5.8099999999999996,"15min":134.929},"last":0.32500000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:38:05 2023","interfaces":[{"interface":"back","average":{"1min":3.2170000000000001,"5min":3.8460000000000001,"15min":6.9219999999999997},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":53.951999999999998,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":3.6440000000000001,"5min":3.9420000000000002,"15min":7.0069999999999997},"min":{"1min":0.221,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":54.213000000000001,"5min":202.12,"15min":232.21000000000001},"last":0.30399999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:38:05 2023","interfaces":[{"interface":"back","average":{"1min":7.2720000000000002,"5min":6.9420000000000002,"15min":5.8620000000000001},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22600000000000001},"max":{"1min":77.034999999999997,"5min":181.94399999999999,"15min":234.404},"last":0.38300000000000001},{"interface":"front","average":{"1min":7.5010000000000003,"5min":7.0190000000000001,"15min":5.8150000000000004},"min":{"1min":0.245,"5min":0.224,"15min":0.224},"max":{"1min":77.165999999999997,"5min":181.98500000000001,"15min":236.33699999999999},"last":0.436}]},{"osd":4,"last update":"Sat Sep 23 23:38:23 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":0.998,"15min":1.4039999999999999},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.28699999999999998},"max":{"1min":3.5939999999999999,"5min":4.7069999999999999,"15min":12.042999999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.034,"5min":0.874,"15min":1.4399999999999999},"min":{"1min":0.33300000000000002,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":7.1779999999999999,"5min":7.1779999999999999,"15min":18.108000000000001},"last":0.35599999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:38:23 2023","interfaces":[{"interface":"back","average":{"1min":4.2169999999999996,"5min":3.2690000000000001,"15min":1.962},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":80.302999999999997,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":4.7610000000000001,"5min":3.4239999999999999,"15min":2.3130000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":80.254000000000005,"5min":151.821,"15min":151.821},"last":0.371}]},{"osd":6,"last update":"Sat Sep 23 23:38:23 2023","interfaces":[{"interface":"back","average":{"1min":4.5700000000000003,"5min":1.732,"15min":1.921},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.27500000000000002},"max":{"1min":87.837999999999994,"5min":87.837999999999994,"15min":87.837999999999994},"last":0.45600000000000002},{"interface":"front","average":{"1min":5.0910000000000002,"5min":2.1080000000000001,"15min":2.198},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":85.518000000000001,"5min":85.518000000000001,"15min":85.518000000000001},"last":0.47699999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:38:05 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":1.4590000000000001,"15min":2.101},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.7639999999999998,"5min":44.241,"15min":78.308000000000007},"last":35.487000000000002},{"interface":"front","average":{"1min":1.657,"5min":1.514,"15min":2.113},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32700000000000001},"max":{"1min":5.7969999999999997,"5min":43.920999999999999,"15min":77.204999999999998},"last":35.146000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024352,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1269336,"kb_used_data":41040,"kb_used_omap":5,"kb_used_meta":1228282,"kb_avail":92475816,"statfs":{"total":95995035648,"available":94695235584,"internally_reserved":0,"allocated":42024960,"data_stored":39140110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1257761402},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:38:40 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":4.0839999999999996,"15min":6.4660000000000002},"min":{"1min":0.22500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":3.9769999999999999,"5min":301.12700000000001,"15min":301.12700000000001},"last":3.403},{"interface":"front","average":{"1min":1.579,"5min":3.8439999999999999,"15min":5.6849999999999996},"min":{"1min":0.23499999999999999,"5min":0.20200000000000001,"15min":0.17599999999999999},"max":{"1min":13.141,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.253}]},{"osd":2,"last update":"Sat Sep 23 23:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":6.702,"15min":6.5780000000000003},"min":{"1min":0.214,"5min":0.214,"15min":0.17199999999999999},"max":{"1min":5.4340000000000002,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.218},{"interface":"front","average":{"1min":0.77900000000000003,"5min":6.4569999999999999,"15min":6.5339999999999998},"min":{"1min":0.22600000000000001,"5min":0.20799999999999999,"15min":0.17599999999999999},"max":{"1min":3.4470000000000001,"5min":283.161,"15min":283.161},"last":4.0289999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:38:40 2023","interfaces":[{"interface":"back","average":{"1min":6.2859999999999996,"5min":7.3879999999999999,"15min":6.8490000000000002},"min":{"1min":0.28599999999999998,"5min":0.219,"15min":0.217},"max":{"1min":66.093000000000004,"5min":194.26300000000001,"15min":220.732},"last":0.29999999999999999},{"interface":"front","average":{"1min":6.8170000000000002,"5min":7.3819999999999997,"15min":6.9580000000000002},"min":{"1min":0.36099999999999999,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":66.123999999999995,"5min":194.56299999999999,"15min":218.78200000000001},"last":4.0629999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.748,"15min":2.2799999999999998},"min":{"1min":0.33000000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.605,"5min":4.1310000000000002,"15min":138.81299999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.095,"15min":3.0369999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.27200000000000002},"max":{"1min":4.2030000000000003,"5min":9.3670000000000009,"15min":171.251},"last":0.95199999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":1.3029999999999999,"15min":2.75},"min":{"1min":0.38100000000000001,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":4.4880000000000004,"5min":15.076000000000001,"15min":171.30699999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.1000000000000001,"15min":1.9570000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.253},"max":{"1min":2.7029999999999998,"5min":6.5279999999999996,"15min":139.40799999999999},"last":1.1579999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.1870000000000001,"15min":3.0569999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.29499999999999998},"max":{"1min":3.5939999999999999,"5min":12.654999999999999,"15min":171.33600000000001},"last":2.1160000000000001},{"interface":"front","average":{"1min":0.80300000000000005,"5min":1.1659999999999999,"15min":3.2360000000000002},"min":{"1min":0.39000000000000001,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":2.1360000000000001,"5min":9.157,"15min":184.13499999999999},"last":2.1259999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":1.012,"15min":2.5339999999999998},"min":{"1min":0.35999999999999999,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":3.4089999999999998,"5min":8.9510000000000005,"15min":138.77699999999999},"last":3.794},{"interface":"front","average":{"1min":0.89100000000000001,"5min":1.3069999999999999,"15min":2.5089999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.33300000000000002},"max":{"1min":2.5,"5min":9.8870000000000005,"15min":139.37899999999999},"last":2.3540000000000001}]}]},{"osd":2,"up_from":199,"seq":854698492174,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":950892,"kb_used_data":62436,"kb_used_omap":7,"kb_used_meta":888184,"kb_avail":92794260,"statfs":{"total":95995035648,"available":95021322240,"internally_reserved":0,"allocated":63934464,"data_stored":60988559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":909500497},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,4,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.496,"15min":2.968},"min":{"1min":0.20999999999999999,"5min":0.186,"15min":0.17699999999999999},"max":{"1min":2.9980000000000002,"5min":61.950000000000003,"15min":95.587999999999994},"last":1.8540000000000001},{"interface":"front","average":{"1min":0.90700000000000003,"5min":1.7050000000000001,"15min":2.8319999999999999},"min":{"1min":0.22600000000000001,"5min":0.19700000000000001,"15min":0.18099999999999999},"max":{"1min":4.5739999999999998,"5min":61.984999999999999,"15min":95.796000000000006},"last":1.9530000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":4.3550000000000004,"15min":4.516},"min":{"1min":0.21299999999999999,"5min":0.192,"15min":0.192},"max":{"1min":4.4500000000000002,"5min":106.765,"15min":157.98699999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.76300000000000001,"5min":3.9319999999999999,"15min":4.4649999999999999},"min":{"1min":0.24399999999999999,"5min":0.222,"15min":0.222},"max":{"1min":4.468,"5min":106.732,"15min":158.03800000000001},"last":0.41599999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":5.1689999999999996,"15min":4.3070000000000004},"min":{"1min":0.245,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":4.1740000000000004,"5min":106.70399999999999,"15min":106.70399999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.0209999999999999,"5min":4.7800000000000002,"15min":4.1459999999999999},"min":{"1min":0.28100000000000003,"5min":0.246,"15min":0.23200000000000001},"max":{"1min":3.52,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.27600000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":1.6779999999999999,"15min":1.768},"min":{"1min":0.35499999999999998,"5min":0.33700000000000002,"15min":0.27700000000000002},"max":{"1min":3.5339999999999998,"5min":62.081000000000003,"15min":62.081000000000003},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.875,"15min":1.802},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.28499999999999998},"max":{"1min":2.9489999999999998,"5min":62.072000000000003,"15min":62.072000000000003},"last":5.1630000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":9.3179999999999996,"5min":4.476,"15min":2.6640000000000001},"min":{"1min":0.32200000000000001,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":101.104,"5min":101.104,"15min":101.104},"last":0.52700000000000002},{"interface":"front","average":{"1min":9.7829999999999995,"5min":4.6280000000000001,"15min":2.6800000000000002},"min":{"1min":0.35199999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":101.404,"5min":101.404,"15min":101.404},"last":0.58499999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":9.1869999999999994,"5min":4.8730000000000002,"15min":2.9239999999999999},"min":{"1min":0.372,"5min":0.31,"15min":0.31},"max":{"1min":97.558999999999997,"5min":107.018,"15min":107.018},"last":0.56999999999999995},{"interface":"front","average":{"1min":9.4420000000000002,"5min":3.7989999999999999,"15min":2.5219999999999998},"min":{"1min":0.375,"5min":0.371,"15min":0.36699999999999999},"max":{"1min":98.231999999999999,"5min":98.231999999999999,"15min":98.231999999999999},"last":2.399}]},{"osd":7,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.7150000000000001,"15min":1.8680000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.4129999999999998,"5min":62.603000000000002,"15min":62.603000000000002},"last":5.266},{"interface":"front","average":{"1min":1.0840000000000001,"5min":3,"15min":2.3100000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":3.452,"5min":106.809,"15min":106.809},"last":2.714}]}]},{"osd":3,"up_from":148,"seq":635655160142,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1032080,"kb_used_data":76296,"kb_used_omap":2,"kb_used_meta":955773,"kb_avail":92713072,"statfs":{"total":95995035648,"available":94938185728,"internally_reserved":0,"allocated":78127104,"data_stored":75253943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":978711605},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":1.0009999999999999,"15min":2.5059999999999998},"min":{"1min":0.19500000000000001,"5min":0.184,"15min":0.17699999999999999},"max":{"1min":2.9980000000000002,"5min":17.213000000000001,"15min":103.866},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.995,"15min":2.8069999999999999},"min":{"1min":0.188,"5min":0.188,"15min":0.17199999999999999},"max":{"1min":2.8090000000000002,"5min":12.202,"15min":101.884},"last":4.0960000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.95999999999999996,"15min":2.819},"min":{"1min":0.23899999999999999,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":2.851,"5min":7.3319999999999999,"15min":85.242999999999995},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.88400000000000001,"15min":2.6920000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":2.9830000000000001,"5min":11.228,"15min":85.302000000000007},"last":0.64600000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:37:57 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":1.4359999999999999,"15min":4.0659999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.19800000000000001},"max":{"1min":36.362000000000002,"5min":36.362000000000002,"15min":84.724999999999994},"last":0.432},{"interface":"front","average":{"1min":2.6110000000000002,"5min":1.431,"15min":3.931},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.214},"max":{"1min":36.158999999999999,"5min":36.158999999999999,"15min":84.567999999999998},"last":0.58899999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":1.032,"15min":1.202},"min":{"1min":0.318,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":2.9609999999999999,"5min":11.209,"15min":11.209},"last":0.62},{"interface":"front","average":{"1min":0.74199999999999999,"5min":1.1140000000000001,"15min":1.6160000000000001},"min":{"1min":0.34000000000000002,"5min":0.316,"15min":0.26800000000000002},"max":{"1min":2.7639999999999998,"5min":7.681,"15min":24.782},"last":0.99199999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":10.651,"5min":8.0739999999999998,"15min":3.9369999999999998},"min":{"1min":0.38500000000000001,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":196.358,"5min":214.43700000000001,"15min":214.43700000000001},"last":1.0309999999999999},{"interface":"front","average":{"1min":10.586,"5min":8.0500000000000007,"15min":3.984},"min":{"1min":0.35999999999999999,"5min":0.34699999999999998,"15min":0.30599999999999999},"max":{"1min":196.048,"5min":214.46600000000001,"15min":214.46600000000001},"last":0.45900000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:37:50 2023","interfaces":[{"interface":"back","average":{"1min":7.2190000000000003,"5min":3.2280000000000002,"15min":2.5649999999999999},"min":{"1min":0.35799999999999998,"5min":0.318,"15min":0.318},"max":{"1min":125.94,"5min":125.94,"15min":125.94},"last":1.056},{"interface":"front","average":{"1min":7.1189999999999998,"5min":3.3559999999999999,"15min":2.5649999999999999},"min":{"1min":0.36499999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":125.955,"5min":125.955,"15min":125.955},"last":0.47499999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:37:57 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":4.5289999999999999,"15min":3.1909999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32600000000000001},"max":{"1min":6.5800000000000001,"5min":192.69,"15min":192.69},"last":4.2309999999999999},{"interface":"front","average":{"1min":0.92400000000000004,"5min":4.5789999999999997,"15min":3.629},"min":{"1min":0.40699999999999997,"5min":0.40100000000000002,"15min":0.36699999999999999},"max":{"1min":3.6120000000000001,"5min":192.65600000000001,"15min":192.65600000000001},"last":1.0760000000000001}]}]},{"osd":4,"up_from":0,"seq":128849019461,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757824,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":851016,"kb_used_data":81536,"kb_used_omap":0,"kb_used_meta":769472,"kb_avail":92894136,"statfs":{"total":95995035648,"available":95123595264,"internally_reserved":0,"allocated":83492864,"data_stored":81800605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":787939328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,6,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:19 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.78200000000000003,"15min":1.454},"min":{"1min":0.223,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.2890000000000001,"5min":6.1609999999999996,"15min":63.741999999999997},"last":2.0470000000000002},{"interface":"front","average":{"1min":1,"5min":1.1779999999999999,"15min":1.964},"min":{"1min":0.26200000000000001,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":4.8579999999999997,"5min":15.414,"15min":138.55199999999999},"last":0.38900000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:38:25 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":0.85599999999999998,"15min":2.21},"min":{"1min":0.27000000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.7480000000000002,"5min":6.1360000000000001,"15min":69.816999999999993},"last":0.70899999999999996},{"interface":"front","average":{"1min":1.1120000000000001,"5min":0.89300000000000002,"15min":2.4249999999999998},"min":{"1min":0.307,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":3.7570000000000001,"5min":4.4859999999999998,"15min":70.268000000000001},"last":0.752}]},{"osd":2,"last update":"Sat Sep 23 23:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":6.8869999999999996,"15min":4.4939999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26900000000000002},"max":{"1min":6.2469999999999999,"5min":208.738,"15min":208.738},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.77700000000000002,"5min":5.5220000000000002,"15min":4.0140000000000002},"min":{"1min":0.309,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":3.2290000000000001,"5min":208.631,"15min":208.631},"last":2.0179999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:37:53 2023","interfaces":[{"interface":"back","average":{"1min":6.6550000000000002,"5min":5.6689999999999996,"15min":4.1470000000000002},"min":{"1min":0.33400000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":143.66300000000001,"5min":143.66300000000001,"15min":143.66300000000001},"last":4.8449999999999998},{"interface":"front","average":{"1min":6.7670000000000003,"5min":6.0010000000000003,"15min":4.327},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":143.62200000000001,"5min":143.62200000000001,"15min":143.62200000000001},"last":0.36299999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:38:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":0.90800000000000003,"15min":0.92300000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.251},"max":{"1min":4.4269999999999996,"5min":9.2690000000000001,"15min":19.962},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.92200000000000004,"15min":0.87},"min":{"1min":0.32300000000000001,"5min":0.28899999999999998,"15min":0.22900000000000001},"max":{"1min":4.4169999999999998,"5min":10.057,"15min":20.829999999999998},"last":1.9330000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":2.3490000000000002,"15min":1.556},"min":{"1min":0.28899999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.2350000000000003,"5min":148.36600000000001,"15min":148.36600000000001},"last":1.974},{"interface":"front","average":{"1min":1.1639999999999999,"5min":2.5870000000000002,"15min":1.754},"min":{"1min":0.35599999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.7919999999999998,"5min":148.42500000000001,"15min":148.42500000000001},"last":0.30199999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:37:48 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.306,"15min":2.177},"min":{"1min":0.34699999999999998,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":4.3380000000000001,"5min":15.497,"15min":179.28800000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.157,"5min":1.339,"15min":2.2480000000000002},"min":{"1min":0.40600000000000003,"5min":0.35199999999999998,"15min":0.34999999999999998},"max":{"1min":6.1440000000000001,"5min":15.653,"15min":179.428},"last":0.40899999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32309248,"data_stored":32309248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":54792192,"data_stored":54788120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":39649280,"data_stored":39649280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":58789888,"data_stored":58785816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2263089152,"data_stored":2263089152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":56950784,"data_stored":56950784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":109379584,"data_stored":109379584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":88215552,"data_stored":88211480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:38:50.665 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:38:50.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:50.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:38:50.768 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:38:50.796 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:51.513 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:38:51.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:51.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:38:51.618 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:51.644 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_8 2023-09-23T23:38:51.644 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_8 unique_pool_8 --yes-i-really-really-mean-it 2023-09-23T23:38:51.651 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:51.695 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:51.703 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:51.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:51.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:51.800 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:51 smithi110 ceph-mon[212939]: pgmap v1738: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 11 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 16378/585 objects degraded (2799.658%); 0 B/s, 158 objects/s recovering 2023-09-23T23:38:51.801 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:51 smithi110 ceph-mon[221102]: pgmap v1738: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 11 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 16378/585 objects degraded (2799.658%); 0 B/s, 158 objects/s recovering 2023-09-23T23:38:51.807 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:38:51.838 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:51.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:51 smithi138 ceph-mon[193606]: pgmap v1738: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 11 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 16378/585 objects degraded (2799.658%); 0 B/s, 158 objects/s recovering 2023-09-23T23:38:52.052 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:38:52.053 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:38:52.095 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:38:52.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:52.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:38:52.200 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:38:52.227 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:52.280 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:38:52.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:52.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:38:52.387 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:38:52.415 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:52.462 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:38:52.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:52.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:38:52.568 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:38:52.595 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:52.643 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:38:52.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:52.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:38:52.749 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:38:52.776 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:52.823 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:38:52.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:52.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:38:52.929 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:52.959 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:53.413 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:53.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:53.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:53.512 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:38:53.517 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:38:53.544 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:53.577 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:38:53.594 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:38:53.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:53.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:38:53.658 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:38:53.684 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:53 smithi110 ceph-mon[221102]: pgmap v1739: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 16378/72 objects degraded (22747.222%); 0 B/s, 158 objects/s recovering 2023-09-23T23:38:53.685 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:53 smithi110 ceph-mon[212939]: pgmap v1739: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 16378/72 objects degraded (22747.222%); 0 B/s, 158 objects/s recovering 2023-09-23T23:38:53.703 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:38:53.727 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:53 smithi138 ceph-mon[193606]: pgmap v1739: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 16378/72 objects degraded (22747.222%); 0 B/s, 158 objects/s recovering 2023-09-23T23:38:53.730 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:53.776 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:38:53.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:53.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:38:53.883 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:38:53.909 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:54.235 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:38:54.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:54.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:38:54.316 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:54.338 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:38:54.366 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:54.411 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:38:54.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:54.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:38:54.477 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:54.521 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:38:54.535 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:54 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 16378/72 objects degraded (22747.222%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:54.535 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:54 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 16378/72 objects degraded (22747.222%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:54.548 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:54.594 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:38:54.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:54.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:38:54.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:54 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 16378/72 objects degraded (22747.222%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:54.702 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:38:54.734 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:54.963 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:55.041 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:38:55.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:55.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:38:55.145 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:38:55.173 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:55.204 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:38:55.218 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:38:55.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:55.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:38:55.325 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:38:55.355 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:55.671 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:55 smithi110 ceph-mon[212939]: pgmap v1740: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 15264/72 objects degraded (21200.000%); 0 B/s, 256 objects/s recovering 2023-09-23T23:38:55.671 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:55 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/768247519' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:38:55.671 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:55 smithi110 ceph-mon[221102]: pgmap v1740: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 15264/72 objects degraded (21200.000%); 0 B/s, 256 objects/s recovering 2023-09-23T23:38:55.671 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:55 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/768247519' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:38:55.801 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:38:55.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:55 smithi138 ceph-mon[193606]: pgmap v1740: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 15264/72 objects degraded (21200.000%); 0 B/s, 256 objects/s recovering 2023-09-23T23:38:55.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:55 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/768247519' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:38:56.802 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:56 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:38:56.802 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:56 smithi110 ceph-mon[212939]: failed to encode map e610 with expected crc 2023-09-23T23:38:56.802 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:56 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/768247519' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:38:56.802 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:56 smithi110 ceph-mon[212939]: osdmap e610: 8 total, 8 up, 7 in 2023-09-23T23:38:56.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:56 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:38:56.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:56 smithi110 ceph-mon[221102]: failed to encode map e610 with expected crc 2023-09-23T23:38:56.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:56 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/768247519' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:38:56.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:56 smithi110 ceph-mon[221102]: osdmap e610: 8 total, 8 up, 7 in 2023-09-23T23:38:56.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:56 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:38:56.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:56 smithi138 ceph-mon[193606]: failed to encode map e610 with expected crc 2023-09-23T23:38:56.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:56 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/768247519' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:38:56.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:56 smithi138 ceph-mon[193606]: osdmap e610: 8 total, 8 up, 7 in 2023-09-23T23:38:57.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:38:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:38:56] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2023-09-23T23:38:57.413 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:38:57.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:57.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:38:57.516 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:38:57.545 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:57.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:57 smithi110 ceph-mon[212939]: pgmap v1742: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 163 op/s; 14396/72 objects degraded (19994.444%); 0 B/s, 303 objects/s recovering 2023-09-23T23:38:57.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:57 smithi110 ceph-mon[212939]: failed to encode map e611 with expected crc 2023-09-23T23:38:57.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:57 smithi110 ceph-mon[212939]: osdmap e611: 8 total, 8 up, 7 in 2023-09-23T23:38:57.587 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:57 smithi110 ceph-mon[212939]: failed to encode map e611 with expected crc 2023-09-23T23:38:57.587 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:57 smithi110 ceph-mon[221102]: pgmap v1742: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 163 op/s; 14396/72 objects degraded (19994.444%); 0 B/s, 303 objects/s recovering 2023-09-23T23:38:57.587 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:57 smithi110 ceph-mon[221102]: failed to encode map e611 with expected crc 2023-09-23T23:38:57.588 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:57 smithi110 ceph-mon[221102]: osdmap e611: 8 total, 8 up, 7 in 2023-09-23T23:38:57.588 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:57 smithi110 ceph-mon[221102]: failed to encode map e611 with expected crc 2023-09-23T23:38:57.592 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:38:57.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:57.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:38:57.698 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:38:57.727 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:57.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:57 smithi138 ceph-mon[193606]: pgmap v1742: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 163 op/s; 14396/72 objects degraded (19994.444%); 0 B/s, 303 objects/s recovering 2023-09-23T23:38:57.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:57 smithi138 ceph-mon[193606]: failed to encode map e611 with expected crc 2023-09-23T23:38:57.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:57 smithi138 ceph-mon[193606]: osdmap e611: 8 total, 8 up, 7 in 2023-09-23T23:38:57.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:57 smithi138 ceph-mon[193606]: failed to encode map e611 with expected crc 2023-09-23T23:38:58.573 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:58 smithi110 ceph-mon[212939]: failed to encode map e611 with expected crc 2023-09-23T23:38:58.573 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:58 smithi110 ceph-mon[212939]: failed to encode map e612 with expected crc 2023-09-23T23:38:58.573 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:58 smithi110 ceph-mon[212939]: osdmap e612: 8 total, 8 up, 7 in 2023-09-23T23:38:58.573 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:58 smithi110 ceph-mon[212939]: failed to encode map e612 with expected crc 2023-09-23T23:38:58.573 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:38:58.575 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:58 smithi110 ceph-mon[221102]: failed to encode map e611 with expected crc 2023-09-23T23:38:58.576 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:58 smithi110 ceph-mon[221102]: failed to encode map e612 with expected crc 2023-09-23T23:38:58.576 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:58 smithi110 ceph-mon[221102]: osdmap e612: 8 total, 8 up, 7 in 2023-09-23T23:38:58.576 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:58 smithi110 ceph-mon[221102]: failed to encode map e612 with expected crc 2023-09-23T23:38:58.576 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:38:58.585 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:38:58.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:58.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:38:58.689 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:38:58.716 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:58.758 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:38:58.764 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:58 smithi138 ceph-mon[193606]: failed to encode map e611 with expected crc 2023-09-23T23:38:58.764 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:58 smithi138 ceph-mon[193606]: failed to encode map e612 with expected crc 2023-09-23T23:38:58.764 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:58 smithi138 ceph-mon[193606]: osdmap e612: 8 total, 8 up, 7 in 2023-09-23T23:38:58.764 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:58 smithi138 ceph-mon[193606]: failed to encode map e612 with expected crc 2023-09-23T23:38:58.764 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:38:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:58.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:38:58.777 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:38:58.779 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:38:58.784 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:38:58.807 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:38:58.827 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:38:58.848 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:38:58.866 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:38:58.873 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:38:58.892 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:58.897 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:38:58.918 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:38:58.936 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:38:58.937 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:38:58.939 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:38:58.939 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:38:58.939 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:38:58.939 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:38:58.939 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:38:58.939 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:38:58.939 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:38:58.939 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:38:58.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:58.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:38:59.041 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:38:59.067 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:59.112 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:38:59.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:59.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:38:59.218 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:38:59.244 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:59.287 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:38:59.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:59.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:38:59.394 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:38:59.414 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:38:59.421 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:59.464 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:38:59.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:38:59.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:38:59.570 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:38:59.599 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:38:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:59 smithi138 ceph-mon[193606]: pgmap v1745: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 14396/72 objects degraded (19994.444%); 0 B/s, 330 objects/s recovering 2023-09-23T23:38:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:59 smithi138 ceph-mon[193606]: failed to encode map e612 with expected crc 2023-09-23T23:38:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:59 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1268477391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:59 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 14396/72 objects degraded (19994.444%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:59 smithi138 ceph-mon[193606]: failed to encode map e613 with expected crc 2023-09-23T23:38:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:59 smithi138 ceph-mon[193606]: failed to encode map e613 with expected crc 2023-09-23T23:38:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:59 smithi138 ceph-mon[193606]: osdmap e613: 8 total, 8 up, 7 in 2023-09-23T23:38:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:38:59 smithi138 ceph-mon[193606]: failed to encode map e613 with expected crc 2023-09-23T23:38:59.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:59 smithi110 ceph-mon[212939]: pgmap v1745: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 14396/72 objects degraded (19994.444%); 0 B/s, 330 objects/s recovering 2023-09-23T23:38:59.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:59 smithi110 ceph-mon[212939]: failed to encode map e612 with expected crc 2023-09-23T23:38:59.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:59 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1268477391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:59.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:59 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 14396/72 objects degraded (19994.444%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:59.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:59 smithi110 ceph-mon[212939]: failed to encode map e613 with expected crc 2023-09-23T23:38:59.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:59 smithi110 ceph-mon[212939]: failed to encode map e613 with expected crc 2023-09-23T23:38:59.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:59 smithi110 ceph-mon[212939]: osdmap e613: 8 total, 8 up, 7 in 2023-09-23T23:38:59.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:38:59 smithi110 ceph-mon[212939]: failed to encode map e613 with expected crc 2023-09-23T23:38:59.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:59 smithi110 ceph-mon[221102]: pgmap v1745: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 14396/72 objects degraded (19994.444%); 0 B/s, 330 objects/s recovering 2023-09-23T23:38:59.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:59 smithi110 ceph-mon[221102]: failed to encode map e612 with expected crc 2023-09-23T23:38:59.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:59 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1268477391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:38:59.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:59 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 14396/72 objects degraded (19994.444%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T23:38:59.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:59 smithi110 ceph-mon[221102]: failed to encode map e613 with expected crc 2023-09-23T23:38:59.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:59 smithi110 ceph-mon[221102]: failed to encode map e613 with expected crc 2023-09-23T23:38:59.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:59 smithi110 ceph-mon[221102]: osdmap e613: 8 total, 8 up, 7 in 2023-09-23T23:38:59.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:38:59 smithi110 ceph-mon[221102]: failed to encode map e613 with expected crc 2023-09-23T23:39:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:00 smithi138 ceph-mon[193606]: from='client.47117 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:39:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:00 smithi138 ceph-mon[193606]: osdmap e614: 8 total, 8 up, 7 in 2023-09-23T23:39:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:00 smithi138 ceph-mon[193606]: failed to encode map e614 with expected crc 2023-09-23T23:39:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:00 smithi138 ceph-mon[193606]: failed to encode map e614 with expected crc 2023-09-23T23:39:00.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:00 smithi138 ceph-mon[193606]: failed to encode map e614 with expected crc 2023-09-23T23:39:00.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:00 smithi138 ceph-mon[193606]: failed to encode map e614 with expected crc 2023-09-23T23:39:00.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:00 smithi110 ceph-mon[212939]: from='client.47117 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:39:00.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:00 smithi110 ceph-mon[212939]: osdmap e614: 8 total, 8 up, 7 in 2023-09-23T23:39:00.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:00 smithi110 ceph-mon[212939]: failed to encode map e614 with expected crc 2023-09-23T23:39:00.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:00 smithi110 ceph-mon[212939]: failed to encode map e614 with expected crc 2023-09-23T23:39:00.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:00 smithi110 ceph-mon[212939]: failed to encode map e614 with expected crc 2023-09-23T23:39:00.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:00 smithi110 ceph-mon[212939]: failed to encode map e614 with expected crc 2023-09-23T23:39:00.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:00 smithi110 ceph-mon[221102]: from='client.47117 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:39:00.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:00 smithi110 ceph-mon[221102]: osdmap e614: 8 total, 8 up, 7 in 2023-09-23T23:39:00.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:00 smithi110 ceph-mon[221102]: failed to encode map e614 with expected crc 2023-09-23T23:39:00.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:00 smithi110 ceph-mon[221102]: failed to encode map e614 with expected crc 2023-09-23T23:39:00.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:00 smithi110 ceph-mon[221102]: failed to encode map e614 with expected crc 2023-09-23T23:39:00.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:00 smithi110 ceph-mon[221102]: failed to encode map e614 with expected crc 2023-09-23T23:39:01.062 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:01.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:01.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:01.167 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:39:01.193 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:01.238 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:39:01.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:01.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:39:01.344 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:39:01.370 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:01.413 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:39:01.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:01.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:39:01.466 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1745,"stamp":"2023-09-23T23:38:57.978624+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":636125,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14396,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":34,"num_read_kb":64,"num_write":175633,"num_write_kb":5617922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28194,"num_bytes_recovered":1518465448,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96180,"ondisk_log_size":96180,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":367,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6020604,"kb_used_data":118340,"kb_used_omap":29,"kb_used_meta":5902178,"kb_avail":650195460,"statfs":{"total":671965249536,"available":665800151040,"internally_reserved":0,"allocated":121180160,"data_stored":104504849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":6043830995},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1982,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1982,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001454"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":805,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597761+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:38:31.597761+0000","last_peered":"2023-09-23T23:38:31.597761+0000","last_clean":"2023-09-23T23:38:31.597761+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:38:31.597761+0000","last_undegraded":"2023-09-23T23:38:31.597761+0000","last_fullsized":"2023-09-23T23:38:31.597761+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":364,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.595132+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:38:31.595132+0000","last_peered":"2023-09-23T23:38:31.595132+0000","last_clean":"2023-09-23T23:38:31.595132+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:38:31.595132+0000","last_undegraded":"2023-09-23T23:38:31.595132+0000","last_fullsized":"2023-09-23T23:38:31.595132+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":592,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:33.414678+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:38:33.414678+0000","last_peered":"2023-09-23T23:38:33.414678+0000","last_clean":"2023-09-23T23:38:33.414678+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:38:33.414678+0000","last_undegraded":"2023-09-23T23:38:33.414678+0000","last_fullsized":"2023-09-23T23:38:33.414678+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":758,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597404+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:38:31.597404+0000","last_peered":"2023-09-23T23:38:31.597404+0000","last_clean":"2023-09-23T23:38:31.597404+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:38:31.597404+0000","last_undegraded":"2023-09-23T23:38:31.597404+0000","last_fullsized":"2023-09-23T23:38:31.597404+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":328,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.611225+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:38:31.611225+0000","last_peered":"2023-09-23T23:38:31.611225+0000","last_clean":"2023-09-23T23:38:31.611225+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:38:31.611225+0000","last_undegraded":"2023-09-23T23:38:31.611225+0000","last_fullsized":"2023-09-23T23:38:31.611225+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":571,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612102+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:38:31.612102+0000","last_peered":"2023-09-23T23:38:31.612102+0000","last_clean":"2023-09-23T23:38:31.612102+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:38:31.612102+0000","last_undegraded":"2023-09-23T23:38:31.612102+0000","last_fullsized":"2023-09-23T23:38:31.612102+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":770,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.614779+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:38:31.614779+0000","last_peered":"2023-09-23T23:38:31.614779+0000","last_clean":"2023-09-23T23:38:31.614779+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:38:31.614779+0000","last_undegraded":"2023-09-23T23:38:31.614779+0000","last_fullsized":"2023-09-23T23:38:31.614779+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":628,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.610827+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:38:31.610827+0000","last_peered":"2023-09-23T23:38:31.610827+0000","last_clean":"2023-09-23T23:38:31.610827+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:38:31.610827+0000","last_undegraded":"2023-09-23T23:38:31.610827+0000","last_fullsized":"2023-09-23T23:38:31.610827+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":123,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.615668+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:38:31.615668+0000","last_peered":"2023-09-23T23:38:31.615668+0000","last_clean":"2023-09-23T23:38:31.615668+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:38:31.615668+0000","last_undegraded":"2023-09-23T23:38:31.615668+0000","last_fullsized":"2023-09-23T23:38:31.615668+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":929,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.616621+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:38:31.616621+0000","last_peered":"2023-09-23T23:38:31.616621+0000","last_clean":"2023-09-23T23:38:31.616621+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:38:31.616621+0000","last_undegraded":"2023-09-23T23:38:31.616621+0000","last_fullsized":"2023-09-23T23:38:31.616621+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":261,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.604422+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:38:31.604422+0000","last_peered":"2023-09-23T23:38:31.604422+0000","last_clean":"2023-09-23T23:38:31.604422+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:38:31.604422+0000","last_undegraded":"2023-09-23T23:38:31.604422+0000","last_fullsized":"2023-09-23T23:38:31.604422+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":610,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.610758+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:38:31.610758+0000","last_peered":"2023-09-23T23:38:31.610758+0000","last_clean":"2023-09-23T23:38:31.610758+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:38:31.610758+0000","last_undegraded":"2023-09-23T23:38:31.610758+0000","last_fullsized":"2023-09-23T23:38:31.610758+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":903,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.594579+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:38:31.594579+0000","last_peered":"2023-09-23T23:38:31.594579+0000","last_clean":"2023-09-23T23:38:31.594579+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:38:31.594579+0000","last_undegraded":"2023-09-23T23:38:31.594579+0000","last_fullsized":"2023-09-23T23:38:31.594579+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":240,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.601308+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:38:31.601308+0000","last_peered":"2023-09-23T23:38:31.601308+0000","last_clean":"2023-09-23T23:38:31.601308+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:38:31.601308+0000","last_undegraded":"2023-09-23T23:38:31.601308+0000","last_fullsized":"2023-09-23T23:38:31.601308+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":571,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.610871+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:38:31.610871+0000","last_peered":"2023-09-23T23:38:31.610871+0000","last_clean":"2023-09-23T23:38:31.610871+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:38:31.610871+0000","last_undegraded":"2023-09-23T23:38:31.610871+0000","last_fullsized":"2023-09-23T23:38:31.610871+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1063,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597525+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:38:31.597525+0000","last_peered":"2023-09-23T23:38:31.597525+0000","last_clean":"2023-09-23T23:38:31.597525+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:38:31.597525+0000","last_undegraded":"2023-09-23T23:38:31.597525+0000","last_fullsized":"2023-09-23T23:38:31.597525+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":282,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613154+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:38:31.613154+0000","last_peered":"2023-09-23T23:38:31.613154+0000","last_clean":"2023-09-23T23:38:31.613154+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:38:31.613154+0000","last_undegraded":"2023-09-23T23:38:31.613154+0000","last_fullsized":"2023-09-23T23:38:31.613154+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":142,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613588+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:38:31.613588+0000","last_peered":"2023-09-23T23:38:31.613588+0000","last_clean":"2023-09-23T23:38:31.613588+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:38:31.613588+0000","last_undegraded":"2023-09-23T23:38:31.613588+0000","last_fullsized":"2023-09-23T23:38:31.613588+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":461,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:43.465866+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:38:43.465866+0000","last_peered":"2023-09-23T23:38:43.465866+0000","last_clean":"2023-09-23T23:38:43.465866+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:38:43.465866+0000","last_undegraded":"2023-09-23T23:38:43.465866+0000","last_fullsized":"2023-09-23T23:38:43.465866+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":183,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613560+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:38:31.613560+0000","last_peered":"2023-09-23T23:38:31.613560+0000","last_clean":"2023-09-23T23:38:31.613560+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:38:31.613560+0000","last_undegraded":"2023-09-23T23:38:31.613560+0000","last_fullsized":"2023-09-23T23:38:31.613560+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":550,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612160+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:38:31.612160+0000","last_peered":"2023-09-23T23:38:31.612160+0000","last_clean":"2023-09-23T23:38:31.612160+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:38:31.612160+0000","last_undegraded":"2023-09-23T23:38:31.612160+0000","last_fullsized":"2023-09-23T23:38:31.612160+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":764,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597394+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:38:31.597394+0000","last_peered":"2023-09-23T23:38:31.597394+0000","last_clean":"2023-09-23T23:38:31.597394+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:38:31.597394+0000","last_undegraded":"2023-09-23T23:38:31.597394+0000","last_fullsized":"2023-09-23T23:38:31.597394+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":156,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.593780+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:38:31.593780+0000","last_peered":"2023-09-23T23:38:31.593780+0000","last_clean":"2023-09-23T23:38:31.593780+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:38:31.593780+0000","last_undegraded":"2023-09-23T23:38:31.593780+0000","last_fullsized":"2023-09-23T23:38:31.593780+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":609,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.611782+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:38:31.611782+0000","last_peered":"2023-09-23T23:38:31.611782+0000","last_clean":"2023-09-23T23:38:31.611782+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:38:31.611782+0000","last_undegraded":"2023-09-23T23:38:31.611782+0000","last_fullsized":"2023-09-23T23:38:31.611782+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":894,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612443+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:38:31.612443+0000","last_peered":"2023-09-23T23:38:31.612443+0000","last_clean":"2023-09-23T23:38:31.612443+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:38:31.612443+0000","last_undegraded":"2023-09-23T23:38:31.612443+0000","last_fullsized":"2023-09-23T23:38:31.612443+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":204,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613714+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:38:31.613714+0000","last_peered":"2023-09-23T23:38:31.613714+0000","last_clean":"2023-09-23T23:38:31.613714+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:38:31.613714+0000","last_undegraded":"2023-09-23T23:38:31.613714+0000","last_fullsized":"2023-09-23T23:38:31.613714+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":331,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613280+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:38:31.613280+0000","last_peered":"2023-09-23T23:38:31.613280+0000","last_clean":"2023-09-23T23:38:31.613280+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:38:31.613280+0000","last_undegraded":"2023-09-23T23:38:31.613280+0000","last_fullsized":"2023-09-23T23:38:31.613280+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":573,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:43.465647+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:38:43.465647+0000","last_peered":"2023-09-23T23:38:43.465647+0000","last_clean":"2023-09-23T23:38:43.465647+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:38:43.465647+0000","last_undegraded":"2023-09-23T23:38:43.465647+0000","last_fullsized":"2023-09-23T23:38:43.465647+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":477,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613545+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:38:31.613545+0000","last_peered":"2023-09-23T23:38:31.613545+0000","last_clean":"2023-09-23T23:38:31.613545+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:38:31.613545+0000","last_undegraded":"2023-09-23T23:38:31.613545+0000","last_fullsized":"2023-09-23T23:38:31.613545+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":566,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612293+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:38:31.612293+0000","last_peered":"2023-09-23T23:38:31.612293+0000","last_clean":"2023-09-23T23:38:31.612293+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:38:31.612293+0000","last_undegraded":"2023-09-23T23:38:31.612293+0000","last_fullsized":"2023-09-23T23:38:31.612293+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":972,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597871+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:38:31.597871+0000","last_peered":"2023-09-23T23:38:31.597871+0000","last_clean":"2023-09-23T23:38:31.597871+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:38:31.597871+0000","last_undegraded":"2023-09-23T23:38:31.597871+0000","last_fullsized":"2023-09-23T23:38:31.597871+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":173,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612655+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:38:31.612655+0000","last_peered":"2023-09-23T23:38:31.612655+0000","last_clean":"2023-09-23T23:38:31.612655+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:38:31.612655+0000","last_undegraded":"2023-09-23T23:38:31.612655+0000","last_fullsized":"2023-09-23T23:38:31.612655+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":251,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612067+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:38:31.612067+0000","last_peered":"2023-09-23T23:38:31.612067+0000","last_clean":"2023-09-23T23:38:31.612067+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:38:31.612067+0000","last_undegraded":"2023-09-23T23:38:31.612067+0000","last_fullsized":"2023-09-23T23:38:31.612067+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"609'10848","reported_seq":16065,"reported_epoch":609,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:54.282484+0000","last_change":"2023-09-23T23:38:22.063474+0000","last_active":"2023-09-23T23:38:54.282484+0000","last_peered":"2023-09-23T23:38:54.282484+0000","last_clean":"2023-09-23T23:33:50.553902+0000","last_became_active":"2023-09-23T23:33:52.866697+0000","last_became_peered":"2023-09-23T23:33:52.866697+0000","last_unstale":"2023-09-23T23:38:54.282484+0000","last_undegraded":"2023-09-23T23:33:52.572459+0000","last_fullsized":"2023-09-23T23:33:52.536666+0000","mapping_epoch":557,"log_start":"565'5100","ondisk_log_start":"565'5100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5748,"log_dups_size":0,"ondisk_log_size":5748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:37.042308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1836,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10848,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2562,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,6","objects":-1836}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":944,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:43.453504+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:38:43.453504+0000","last_peered":"2023-09-23T23:38:43.453504+0000","last_clean":"2023-09-23T23:38:43.453504+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:38:43.453504+0000","last_undegraded":"2023-09-23T23:38:43.453504+0000","last_fullsized":"2023-09-23T23:38:43.453504+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":250,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.615264+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:38:31.615264+0000","last_peered":"2023-09-23T23:38:31.615264+0000","last_clean":"2023-09-23T23:38:31.615264+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:38:31.615264+0000","last_undegraded":"2023-09-23T23:38:31.615264+0000","last_fullsized":"2023-09-23T23:38:31.615264+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":240,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597728+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:38:31.597728+0000","last_peered":"2023-09-23T23:38:31.597728+0000","last_clean":"2023-09-23T23:38:31.597728+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:38:31.597728+0000","last_undegraded":"2023-09-23T23:38:31.597728+0000","last_fullsized":"2023-09-23T23:38:31.597728+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"609'11032","reported_seq":11125,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:46.371636+0000","last_change":"2023-09-23T23:34:18.729202+0000","last_active":"2023-09-23T23:38:46.371636+0000","last_peered":"2023-09-23T23:38:46.371636+0000","last_clean":"2023-09-23T23:38:46.371636+0000","last_became_active":"2023-09-23T23:34:18.722367+0000","last_became_peered":"2023-09-23T23:34:18.722367+0000","last_unstale":"2023-09-23T23:38:46.371636+0000","last_undegraded":"2023-09-23T23:38:46.371636+0000","last_fullsized":"2023-09-23T23:38:46.371636+0000","mapping_epoch":566,"log_start":"564'4900","ondisk_log_start":"564'4900","created":534,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6132,"log_dups_size":0,"ondisk_log_size":6132,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:05:32.270047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11032,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1022,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.611496+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:38:31.611496+0000","last_peered":"2023-09-23T23:38:31.611496+0000","last_clean":"2023-09-23T23:38:31.611496+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:38:31.611496+0000","last_undegraded":"2023-09-23T23:38:31.611496+0000","last_fullsized":"2023-09-23T23:38:31.611496+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":240,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.615094+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:38:31.615094+0000","last_peered":"2023-09-23T23:38:31.615094+0000","last_clean":"2023-09-23T23:38:31.615094+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:38:31.615094+0000","last_undegraded":"2023-09-23T23:38:31.615094+0000","last_fullsized":"2023-09-23T23:38:31.615094+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":461,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.611010+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:38:31.611010+0000","last_peered":"2023-09-23T23:38:31.611010+0000","last_clean":"2023-09-23T23:38:31.611010+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:38:31.611010+0000","last_undegraded":"2023-09-23T23:38:31.611010+0000","last_fullsized":"2023-09-23T23:38:31.611010+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"609'11048","reported_seq":21190,"reported_epoch":609,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T23:38:46.414279+0000","last_change":"2023-09-23T23:38:46.418846+0000","last_active":"2023-09-23T23:38:46.414279+0000","last_peered":"2023-09-23T23:38:46.414279+0000","last_clean":"2023-09-23T23:33:50.512365+0000","last_became_active":"2023-09-23T23:34:37.362103+0000","last_became_peered":"2023-09-23T23:34:37.362103+0000","last_unstale":"2023-09-23T23:38:46.414279+0000","last_undegraded":"2023-09-23T23:34:36.896222+0000","last_fullsized":"2023-09-23T23:38:46.414279+0000","mapping_epoch":571,"log_start":"573'5700","ondisk_log_start":"573'5700","created":534,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5348,"log_dups_size":0,"ondisk_log_size":5348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:14.510556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11048,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5009,"num_bytes_recovered":328269824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5,7],"avail_no_missing":["3","0","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":919,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.615696+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:38:31.615696+0000","last_peered":"2023-09-23T23:38:31.615696+0000","last_clean":"2023-09-23T23:38:31.615696+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:38:31.615696+0000","last_undegraded":"2023-09-23T23:38:31.615696+0000","last_fullsized":"2023-09-23T23:38:31.615696+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":347,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.598441+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:38:31.598441+0000","last_peered":"2023-09-23T23:38:31.598441+0000","last_clean":"2023-09-23T23:38:31.598441+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:38:31.598441+0000","last_undegraded":"2023-09-23T23:38:31.598441+0000","last_fullsized":"2023-09-23T23:38:31.598441+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":123,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613712+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:38:31.613712+0000","last_peered":"2023-09-23T23:38:31.613712+0000","last_clean":"2023-09-23T23:38:31.613712+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:38:31.613712+0000","last_undegraded":"2023-09-23T23:38:31.613712+0000","last_fullsized":"2023-09-23T23:38:31.613712+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"609'11090","reported_seq":18486,"reported_epoch":609,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:53.457245+0000","last_change":"2023-09-23T23:38:22.122318+0000","last_active":"2023-09-23T23:38:53.457245+0000","last_peered":"2023-09-23T23:38:53.457245+0000","last_clean":"2023-09-23T23:33:50.518482+0000","last_became_active":"2023-09-23T23:33:52.808345+0000","last_became_peered":"2023-09-23T23:33:52.808345+0000","last_unstale":"2023-09-23T23:38:53.457245+0000","last_undegraded":"2023-09-23T23:34:35.747711+0000","last_fullsized":"2023-09-23T23:33:52.530393+0000","mapping_epoch":557,"log_start":"575'5800","ondisk_log_start":"575'5800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5290,"log_dups_size":0,"ondisk_log_size":5290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:52.500612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":886,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11090,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3649,"num_bytes_recovered":72089600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,5","objects":-886}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":517,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:43.465945+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:38:43.465945+0000","last_peered":"2023-09-23T23:38:43.465945+0000","last_clean":"2023-09-23T23:38:43.465945+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:38:43.465945+0000","last_undegraded":"2023-09-23T23:38:43.465945+0000","last_fullsized":"2023-09-23T23:38:43.465945+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":450,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.611665+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:38:31.611665+0000","last_peered":"2023-09-23T23:38:31.611665+0000","last_clean":"2023-09-23T23:38:31.611665+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:38:31.611665+0000","last_undegraded":"2023-09-23T23:38:31.611665+0000","last_fullsized":"2023-09-23T23:38:31.611665+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":608,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:43.480605+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:38:43.480605+0000","last_peered":"2023-09-23T23:38:43.480605+0000","last_clean":"2023-09-23T23:38:43.480605+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:38:43.480605+0000","last_undegraded":"2023-09-23T23:38:43.480605+0000","last_fullsized":"2023-09-23T23:38:43.480605+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"609'11162","reported_seq":12501,"reported_epoch":609,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:46.411094+0000","last_change":"2023-09-23T23:34:35.916516+0000","last_active":"2023-09-23T23:38:46.411094+0000","last_peered":"2023-09-23T23:38:46.411094+0000","last_clean":"2023-09-23T23:33:50.521236+0000","last_became_active":"2023-09-23T23:34:35.907119+0000","last_became_peered":"2023-09-23T23:34:35.907119+0000","last_unstale":"2023-09-23T23:38:46.411094+0000","last_undegraded":"2023-09-23T23:34:35.766965+0000","last_fullsized":"2023-09-23T23:34:35.727968+0000","mapping_epoch":570,"log_start":"569'5500","ondisk_log_start":"569'5500","created":534,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5662,"log_dups_size":0,"ondisk_log_size":5662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:31.591128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11162,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,5","objects":-5460}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":738,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:43.456190+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:38:43.456190+0000","last_peered":"2023-09-23T23:38:43.456190+0000","last_clean":"2023-09-23T23:38:43.456190+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:38:43.456190+0000","last_undegraded":"2023-09-23T23:38:43.456190+0000","last_fullsized":"2023-09-23T23:38:43.456190+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":620,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.611852+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:38:31.611852+0000","last_peered":"2023-09-23T23:38:31.611852+0000","last_clean":"2023-09-23T23:38:31.611852+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:38:31.611852+0000","last_undegraded":"2023-09-23T23:38:31.611852+0000","last_fullsized":"2023-09-23T23:38:31.611852+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":531,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.611590+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:38:31.611590+0000","last_peered":"2023-09-23T23:38:31.611590+0000","last_clean":"2023-09-23T23:38:31.611590+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:38:31.611590+0000","last_undegraded":"2023-09-23T23:38:31.611590+0000","last_fullsized":"2023-09-23T23:38:31.611590+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"609'11080","reported_seq":11163,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:46.406092+0000","last_change":"2023-09-23T23:32:51.928785+0000","last_active":"2023-09-23T23:38:46.406092+0000","last_peered":"2023-09-23T23:38:46.406092+0000","last_clean":"2023-09-23T23:38:46.406092+0000","last_became_active":"2023-09-23T23:32:51.928405+0000","last_became_peered":"2023-09-23T23:32:51.928405+0000","last_unstale":"2023-09-23T23:38:46.406092+0000","last_undegraded":"2023-09-23T23:38:46.406092+0000","last_fullsized":"2023-09-23T23:38:46.406092+0000","mapping_epoch":534,"log_start":"565'5400","ondisk_log_start":"565'5400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5680,"log_dups_size":0,"ondisk_log_size":5680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:02.696843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11080,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":893,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.593859+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:38:31.593859+0000","last_peered":"2023-09-23T23:38:31.593859+0000","last_clean":"2023-09-23T23:38:31.593859+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:38:31.593859+0000","last_undegraded":"2023-09-23T23:38:31.593859+0000","last_fullsized":"2023-09-23T23:38:31.593859+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":133,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612048+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:38:31.612048+0000","last_peered":"2023-09-23T23:38:31.612048+0000","last_clean":"2023-09-23T23:38:31.612048+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:38:31.612048+0000","last_undegraded":"2023-09-23T23:38:31.612048+0000","last_fullsized":"2023-09-23T23:38:31.612048+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":476,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.594324+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:38:31.594324+0000","last_peered":"2023-09-23T23:38:31.594324+0000","last_clean":"2023-09-23T23:38:31.594324+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:38:31.594324+0000","last_undegraded":"2023-09-23T23:38:31.594324+0000","last_fullsized":"2023-09-23T23:38:31.594324+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"609'11104","reported_seq":20305,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:46.360379+0000","last_change":"2023-09-23T23:38:22.063236+0000","last_active":"2023-09-23T23:38:46.360379+0000","last_peered":"2023-09-23T23:38:46.360379+0000","last_clean":"2023-09-23T23:38:46.360379+0000","last_became_active":"2023-09-23T23:34:19.733501+0000","last_became_peered":"2023-09-23T23:34:19.733501+0000","last_unstale":"2023-09-23T23:38:46.360379+0000","last_undegraded":"2023-09-23T23:38:46.360379+0000","last_fullsized":"2023-09-23T23:38:46.360379+0000","mapping_epoch":567,"log_start":"565'5400","ondisk_log_start":"565'5400","created":534,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5704,"log_dups_size":0,"ondisk_log_size":5704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:21.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11104,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4547,"num_bytes_recovered":297992192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":617,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597981+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:38:31.597981+0000","last_peered":"2023-09-23T23:38:31.597981+0000","last_clean":"2023-09-23T23:38:31.597981+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:38:31.597981+0000","last_undegraded":"2023-09-23T23:38:31.597981+0000","last_fullsized":"2023-09-23T23:38:31.597981+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":473,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.593707+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:38:31.593707+0000","last_peered":"2023-09-23T23:38:31.593707+0000","last_clean":"2023-09-23T23:38:31.593707+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:38:31.593707+0000","last_undegraded":"2023-09-23T23:38:31.593707+0000","last_fullsized":"2023-09-23T23:38:31.593707+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":560,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612121+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:38:31.612121+0000","last_peered":"2023-09-23T23:38:31.612121+0000","last_clean":"2023-09-23T23:38:31.612121+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:38:31.612121+0000","last_undegraded":"2023-09-23T23:38:31.612121+0000","last_fullsized":"2023-09-23T23:38:31.612121+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"609'11016","reported_seq":20062,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:46.376901+0000","last_change":"2023-09-23T23:34:58.341045+0000","last_active":"2023-09-23T23:38:46.376901+0000","last_peered":"2023-09-23T23:38:46.376901+0000","last_clean":"2023-09-23T23:38:46.376901+0000","last_became_active":"2023-09-23T23:34:58.331978+0000","last_became_peered":"2023-09-23T23:34:58.331978+0000","last_unstale":"2023-09-23T23:38:46.376901+0000","last_undegraded":"2023-09-23T23:38:46.376901+0000","last_fullsized":"2023-09-23T23:38:46.376901+0000","mapping_epoch":579,"log_start":"565'5300","ondisk_log_start":"565'5300","created":534,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5716,"log_dups_size":0,"ondisk_log_size":5716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:18.148301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11016,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4470,"num_bytes_recovered":292945920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":952,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:43.453756+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:38:43.453756+0000","last_peered":"2023-09-23T23:38:43.453756+0000","last_clean":"2023-09-23T23:38:43.453756+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:38:43.453756+0000","last_undegraded":"2023-09-23T23:38:43.453756+0000","last_fullsized":"2023-09-23T23:38:43.453756+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":251,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.610635+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:38:31.610635+0000","last_peered":"2023-09-23T23:38:31.610635+0000","last_clean":"2023-09-23T23:38:31.610635+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:38:31.610635+0000","last_undegraded":"2023-09-23T23:38:31.610635+0000","last_fullsized":"2023-09-23T23:38:31.610635+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":540,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:38.455196+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:38:38.455196+0000","last_peered":"2023-09-23T23:38:38.455196+0000","last_clean":"2023-09-23T23:38:38.455196+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:38:38.455196+0000","last_undegraded":"2023-09-23T23:38:38.455196+0000","last_fullsized":"2023-09-23T23:38:38.455196+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"609'10728","reported_seq":10814,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:46.414499+0000","last_change":"2023-09-23T23:32:51.925789+0000","last_active":"2023-09-23T23:38:46.414499+0000","last_peered":"2023-09-23T23:38:46.414499+0000","last_clean":"2023-09-23T23:38:46.414499+0000","last_became_active":"2023-09-23T23:32:51.925405+0000","last_became_peered":"2023-09-23T23:32:51.925405+0000","last_unstale":"2023-09-23T23:38:46.414499+0000","last_undegraded":"2023-09-23T23:38:46.414499+0000","last_fullsized":"2023-09-23T23:38:46.414499+0000","mapping_epoch":534,"log_start":"569'5400","ondisk_log_start":"569'5400","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5328,"log_dups_size":0,"ondisk_log_size":5328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.782130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10728,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":930,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.615532+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:38:31.615532+0000","last_peered":"2023-09-23T23:38:31.615532+0000","last_clean":"2023-09-23T23:38:31.615532+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:38:31.615532+0000","last_undegraded":"2023-09-23T23:38:31.615532+0000","last_fullsized":"2023-09-23T23:38:31.615532+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":319,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.605763+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:38:31.605763+0000","last_peered":"2023-09-23T23:38:31.605763+0000","last_clean":"2023-09-23T23:38:31.605763+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:38:31.605763+0000","last_undegraded":"2023-09-23T23:38:31.605763+0000","last_fullsized":"2023-09-23T23:38:31.605763+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":624,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.598620+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:38:31.598620+0000","last_peered":"2023-09-23T23:38:31.598620+0000","last_clean":"2023-09-23T23:38:31.598620+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:38:31.598620+0000","last_undegraded":"2023-09-23T23:38:31.598620+0000","last_fullsized":"2023-09-23T23:38:31.598620+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":205,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597694+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:38:31.597694+0000","last_peered":"2023-09-23T23:38:31.597694+0000","last_clean":"2023-09-23T23:38:31.597694+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:38:31.597694+0000","last_undegraded":"2023-09-23T23:38:31.597694+0000","last_fullsized":"2023-09-23T23:38:31.597694+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"609'11168","reported_seq":20308,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:46.435202+0000","last_change":"2023-09-23T23:38:20.181875+0000","last_active":"2023-09-23T23:38:46.435202+0000","last_peered":"2023-09-23T23:38:46.435202+0000","last_clean":"2023-09-23T23:38:46.435202+0000","last_became_active":"2023-09-23T23:34:18.783681+0000","last_became_peered":"2023-09-23T23:34:18.783681+0000","last_unstale":"2023-09-23T23:38:46.435202+0000","last_undegraded":"2023-09-23T23:38:46.435202+0000","last_fullsized":"2023-09-23T23:38:46.435202+0000","mapping_epoch":566,"log_start":"574'5822","ondisk_log_start":"574'5822","created":534,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5346,"log_dups_size":0,"ondisk_log_size":5346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:13.899574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11168,"num_write_kb":357313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4515,"num_bytes_recovered":295895040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":695,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:51.022954+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:38:51.022954+0000","last_peered":"2023-09-23T23:38:51.022954+0000","last_clean":"2023-09-23T23:38:51.022954+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:38:51.022954+0000","last_undegraded":"2023-09-23T23:38:51.022954+0000","last_fullsized":"2023-09-23T23:38:51.022954+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":257,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613655+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:38:31.613655+0000","last_peered":"2023-09-23T23:38:31.613655+0000","last_clean":"2023-09-23T23:38:31.613655+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:38:31.613655+0000","last_undegraded":"2023-09-23T23:38:31.613655+0000","last_fullsized":"2023-09-23T23:38:31.613655+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":541,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612055+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:38:31.612055+0000","last_peered":"2023-09-23T23:38:31.612055+0000","last_clean":"2023-09-23T23:38:31.612055+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:38:31.612055+0000","last_undegraded":"2023-09-23T23:38:31.612055+0000","last_fullsized":"2023-09-23T23:38:31.612055+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":243,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.594649+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:38:31.594649+0000","last_peered":"2023-09-23T23:38:31.594649+0000","last_clean":"2023-09-23T23:38:31.594649+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:38:31.594649+0000","last_undegraded":"2023-09-23T23:38:31.594649+0000","last_fullsized":"2023-09-23T23:38:31.594649+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"609'10772","reported_seq":12834,"reported_epoch":609,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:46.408361+0000","last_change":"2023-09-23T23:34:19.778826+0000","last_active":"2023-09-23T23:38:46.408361+0000","last_peered":"2023-09-23T23:38:46.408361+0000","last_clean":"2023-09-23T23:33:50.534824+0000","last_became_active":"2023-09-23T23:34:19.732934+0000","last_became_peered":"2023-09-23T23:34:19.732934+0000","last_unstale":"2023-09-23T23:38:46.408361+0000","last_undegraded":"2023-09-23T23:34:19.718545+0000","last_fullsized":"2023-09-23T23:34:19.712090+0000","mapping_epoch":567,"log_start":"565'5100","ondisk_log_start":"565'5100","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5672,"log_dups_size":0,"ondisk_log_size":5672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:00.567551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10772,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":982,"num_bytes_recovered":64356352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"5,7","objects":-3392}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":911,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.594903+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:38:31.594903+0000","last_peered":"2023-09-23T23:38:31.594903+0000","last_clean":"2023-09-23T23:38:31.594903+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:38:31.594903+0000","last_undegraded":"2023-09-23T23:38:31.594903+0000","last_fullsized":"2023-09-23T23:38:31.594903+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":348,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.593586+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:38:31.593586+0000","last_peered":"2023-09-23T23:38:31.593586+0000","last_clean":"2023-09-23T23:38:31.593586+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:38:31.593586+0000","last_undegraded":"2023-09-23T23:38:31.593586+0000","last_fullsized":"2023-09-23T23:38:31.593586+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":258,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.593938+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:38:31.593938+0000","last_peered":"2023-09-23T23:38:31.593938+0000","last_clean":"2023-09-23T23:38:31.593938+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:38:31.593938+0000","last_undegraded":"2023-09-23T23:38:31.593938+0000","last_fullsized":"2023-09-23T23:38:31.593938+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":222,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.604955+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:38:31.604955+0000","last_peered":"2023-09-23T23:38:31.604955+0000","last_clean":"2023-09-23T23:38:31.604955+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:38:31.604955+0000","last_undegraded":"2023-09-23T23:38:31.604955+0000","last_fullsized":"2023-09-23T23:38:31.604955+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"609'10902","reported_seq":10995,"reported_epoch":609,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:38:46.401764+0000","last_change":"2023-09-23T23:34:03.750999+0000","last_active":"2023-09-23T23:38:46.401764+0000","last_peered":"2023-09-23T23:38:46.401764+0000","last_clean":"2023-09-23T23:34:03.045435+0000","last_became_active":"2023-09-23T23:34:03.736073+0000","last_became_peered":"2023-09-23T23:34:03.736073+0000","last_unstale":"2023-09-23T23:38:46.401764+0000","last_undegraded":"2023-09-23T23:38:46.401764+0000","last_fullsized":"2023-09-23T23:38:46.401764+0000","mapping_epoch":562,"log_start":"565'5200","ondisk_log_start":"565'5200","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5702,"log_dups_size":0,"ondisk_log_size":5702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.251405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10902,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":625,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.593897+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:38:31.593897+0000","last_peered":"2023-09-23T23:38:31.593897+0000","last_clean":"2023-09-23T23:38:31.593897+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:38:31.593897+0000","last_undegraded":"2023-09-23T23:38:31.593897+0000","last_fullsized":"2023-09-23T23:38:31.593897+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":895,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612470+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:38:31.612470+0000","last_peered":"2023-09-23T23:38:31.612470+0000","last_clean":"2023-09-23T23:38:31.612470+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:38:31.612470+0000","last_undegraded":"2023-09-23T23:38:31.612470+0000","last_fullsized":"2023-09-23T23:38:31.612470+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":572,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613543+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:38:31.613543+0000","last_peered":"2023-09-23T23:38:31.613543+0000","last_clean":"2023-09-23T23:38:31.613543+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:38:31.613543+0000","last_undegraded":"2023-09-23T23:38:31.613543+0000","last_fullsized":"2023-09-23T23:38:31.613543+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":133,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.614522+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:38:31.614522+0000","last_peered":"2023-09-23T23:38:31.614522+0000","last_clean":"2023-09-23T23:38:31.614522+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:38:31.614522+0000","last_undegraded":"2023-09-23T23:38:31.614522+0000","last_fullsized":"2023-09-23T23:38:31.614522+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"609'10736","reported_seq":10819,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:46.399103+0000","last_change":"2023-09-23T23:32:51.926043+0000","last_active":"2023-09-23T23:38:46.399103+0000","last_peered":"2023-09-23T23:38:46.399103+0000","last_clean":"2023-09-23T23:38:46.399103+0000","last_became_active":"2023-09-23T23:32:51.925758+0000","last_became_peered":"2023-09-23T23:32:51.925758+0000","last_unstale":"2023-09-23T23:38:46.399103+0000","last_undegraded":"2023-09-23T23:38:46.399103+0000","last_fullsized":"2023-09-23T23:38:46.399103+0000","mapping_epoch":534,"log_start":"565'5000","ondisk_log_start":"565'5000","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5736,"log_dups_size":0,"ondisk_log_size":5736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:22.291931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10736,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":721,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613387+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:38:31.613387+0000","last_peered":"2023-09-23T23:38:31.613387+0000","last_clean":"2023-09-23T23:38:31.613387+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:38:31.613387+0000","last_undegraded":"2023-09-23T23:38:31.613387+0000","last_fullsized":"2023-09-23T23:38:31.613387+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":536,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.614217+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:38:31.614217+0000","last_peered":"2023-09-23T23:38:31.614217+0000","last_clean":"2023-09-23T23:38:31.614217+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:38:31.614217+0000","last_undegraded":"2023-09-23T23:38:31.614217+0000","last_fullsized":"2023-09-23T23:38:31.614217+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":451,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.610592+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:38:31.610592+0000","last_peered":"2023-09-23T23:38:31.610592+0000","last_clean":"2023-09-23T23:38:31.610592+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:38:31.610592+0000","last_undegraded":"2023-09-23T23:38:31.610592+0000","last_fullsized":"2023-09-23T23:38:31.610592+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":52,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T23:38:55.905129+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:38:55.905129+0000","last_peered":"2023-09-23T23:38:55.905129+0000","last_clean":"2023-09-23T23:38:55.905129+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:38:55.905129+0000","last_undegraded":"2023-09-23T23:38:55.905129+0000","last_fullsized":"2023-09-23T23:38:55.905129+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"609'10998","reported_seq":14681,"reported_epoch":609,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:38:46.429355+0000","last_change":"2023-09-23T23:34:35.761115+0000","last_active":"2023-09-23T23:38:46.429355+0000","last_peered":"2023-09-23T23:38:46.429355+0000","last_clean":"2023-09-23T23:33:50.529318+0000","last_became_active":"2023-09-23T23:33:52.885366+0000","last_became_peered":"2023-09-23T23:33:52.885366+0000","last_unstale":"2023-09-23T23:38:46.429355+0000","last_undegraded":"2023-09-23T23:34:35.758765+0000","last_fullsized":"2023-09-23T23:33:52.527167+0000","mapping_epoch":557,"log_start":"561'4800","ondisk_log_start":"561'4800","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6198,"log_dups_size":0,"ondisk_log_size":6198,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:20.010592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10998,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1793,"num_bytes_recovered":117506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2","objects":-2704}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":604,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.615468+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:38:31.615468+0000","last_peered":"2023-09-23T23:38:31.615468+0000","last_clean":"2023-09-23T23:38:31.615468+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:38:31.615468+0000","last_undegraded":"2023-09-23T23:38:31.615468+0000","last_fullsized":"2023-09-23T23:38:31.615468+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":52,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:46.022475+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:38:46.022475+0000","last_peered":"2023-09-23T23:38:46.022475+0000","last_clean":"2023-09-23T23:38:46.022475+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:38:46.022475+0000","last_undegraded":"2023-09-23T23:38:46.022475+0000","last_fullsized":"2023-09-23T23:38:46.022475+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":150,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.594719+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:38:31.594719+0000","last_peered":"2023-09-23T23:38:31.594719+0000","last_clean":"2023-09-23T23:38:31.594719+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:38:31.594719+0000","last_undegraded":"2023-09-23T23:38:31.594719+0000","last_fullsized":"2023-09-23T23:38:31.594719+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":342,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.598071+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:38:31.598071+0000","last_peered":"2023-09-23T23:38:31.598071+0000","last_clean":"2023-09-23T23:38:31.598071+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:38:31.598071+0000","last_undegraded":"2023-09-23T23:38:31.598071+0000","last_fullsized":"2023-09-23T23:38:31.598071+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"609'10964","reported_seq":11047,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:46.410874+0000","last_change":"2023-09-23T23:32:52.424405+0000","last_active":"2023-09-23T23:38:46.410874+0000","last_peered":"2023-09-23T23:38:46.410874+0000","last_clean":"2023-09-23T23:38:46.410874+0000","last_became_active":"2023-09-23T23:32:52.424081+0000","last_became_peered":"2023-09-23T23:32:52.424081+0000","last_unstale":"2023-09-23T23:38:46.410874+0000","last_undegraded":"2023-09-23T23:38:46.410874+0000","last_fullsized":"2023-09-23T23:38:46.410874+0000","mapping_epoch":534,"log_start":"565'5300","ondisk_log_start":"565'5300","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":5664,"log_dups_size":0,"ondisk_log_size":5664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:07.823580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10964,"num_write_kb":350848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":791,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.594393+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:38:31.594393+0000","last_peered":"2023-09-23T23:38:31.594393+0000","last_clean":"2023-09-23T23:38:31.594393+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:38:31.594393+0000","last_undegraded":"2023-09-23T23:38:31.594393+0000","last_fullsized":"2023-09-23T23:38:31.594393+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":607,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612160+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:38:31.612160+0000","last_peered":"2023-09-23T23:38:31.612160+0000","last_clean":"2023-09-23T23:38:31.612160+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:38:31.612160+0000","last_undegraded":"2023-09-23T23:38:31.612160+0000","last_fullsized":"2023-09-23T23:38:31.612160+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":213,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.614288+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:38:31.614288+0000","last_peered":"2023-09-23T23:38:31.614288+0000","last_clean":"2023-09-23T23:38:31.614288+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:38:31.614288+0000","last_undegraded":"2023-09-23T23:38:31.614288+0000","last_fullsized":"2023-09-23T23:38:31.614288+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":162,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:43.451834+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:38:43.451834+0000","last_peered":"2023-09-23T23:38:43.451834+0000","last_clean":"2023-09-23T23:38:43.451834+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:38:43.451834+0000","last_undegraded":"2023-09-23T23:38:43.451834+0000","last_fullsized":"2023-09-23T23:38:43.451834+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"609'10908","reported_seq":10994,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:46.351718+0000","last_change":"2023-09-23T23:32:51.927337+0000","last_active":"2023-09-23T23:38:46.351718+0000","last_peered":"2023-09-23T23:38:46.351718+0000","last_clean":"2023-09-23T23:38:46.351718+0000","last_became_active":"2023-09-23T23:32:51.927047+0000","last_became_peered":"2023-09-23T23:32:51.927047+0000","last_unstale":"2023-09-23T23:38:46.351718+0000","last_undegraded":"2023-09-23T23:38:46.351718+0000","last_fullsized":"2023-09-23T23:38:46.351718+0000","mapping_epoch":534,"log_start":"561'4700","ondisk_log_start":"561'4700","created":534,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:32:50.904612+0000","last_clean_scrub_stamp":"2023-09-23T23:32:50.904612+0000","objects_scrubbed":0,"log_size":6208,"log_dups_size":0,"ondisk_log_size":6208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:21.536679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10908,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":997,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.611904+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:38:31.611904+0000","last_peered":"2023-09-23T23:38:31.611904+0000","last_clean":"2023-09-23T23:38:31.611904+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:38:31.611904+0000","last_undegraded":"2023-09-23T23:38:31.611904+0000","last_fullsized":"2023-09-23T23:38:31.611904+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":123,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.615595+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:38:31.615595+0000","last_peered":"2023-09-23T23:38:31.615595+0000","last_clean":"2023-09-23T23:38:31.615595+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:38:31.615595+0000","last_undegraded":"2023-09-23T23:38:31.615595+0000","last_fullsized":"2023-09-23T23:38:31.615595+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":56,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T23:38:55.905174+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:38:55.905174+0000","last_peered":"2023-09-23T23:38:55.905174+0000","last_clean":"2023-09-23T23:38:55.905174+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:38:55.905174+0000","last_undegraded":"2023-09-23T23:38:55.905174+0000","last_fullsized":"2023-09-23T23:38:55.905174+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":183,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.597378+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:38:31.597378+0000","last_peered":"2023-09-23T23:38:31.597378+0000","last_clean":"2023-09-23T23:38:31.597378+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:38:31.597378+0000","last_undegraded":"2023-09-23T23:38:31.597378+0000","last_fullsized":"2023-09-23T23:38:31.597378+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1046,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.610687+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:38:31.610687+0000","last_peered":"2023-09-23T23:38:31.610687+0000","last_clean":"2023-09-23T23:38:31.610687+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:38:31.610687+0000","last_undegraded":"2023-09-23T23:38:31.610687+0000","last_fullsized":"2023-09-23T23:38:31.610687+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":51,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:43.480683+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:38:43.480683+0000","last_peered":"2023-09-23T23:38:43.480683+0000","last_clean":"2023-09-23T23:38:43.480683+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:38:43.480683+0000","last_undegraded":"2023-09-23T23:38:43.480683+0000","last_fullsized":"2023-09-23T23:38:43.480683+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":365,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.594786+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:38:31.594786+0000","last_peered":"2023-09-23T23:38:31.594786+0000","last_clean":"2023-09-23T23:38:31.594786+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:38:31.594786+0000","last_undegraded":"2023-09-23T23:38:31.594786+0000","last_fullsized":"2023-09-23T23:38:31.594786+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":758,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612382+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:38:31.612382+0000","last_peered":"2023-09-23T23:38:31.612382+0000","last_clean":"2023-09-23T23:38:31.612382+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:38:31.612382+0000","last_undegraded":"2023-09-23T23:38:31.612382+0000","last_fullsized":"2023-09-23T23:38:31.612382+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":111,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613302+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:38:31.613302+0000","last_peered":"2023-09-23T23:38:31.613302+0000","last_clean":"2023-09-23T23:38:31.613302+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:38:31.613302+0000","last_undegraded":"2023-09-23T23:38:31.613302+0000","last_fullsized":"2023-09-23T23:38:31.613302+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":458,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:43.465466+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:38:43.465466+0000","last_peered":"2023-09-23T23:38:43.465466+0000","last_clean":"2023-09-23T23:38:43.465466+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:38:43.465466+0000","last_undegraded":"2023-09-23T23:38:43.465466+0000","last_fullsized":"2023-09-23T23:38:43.465466+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":307,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612256+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:38:31.612256+0000","last_peered":"2023-09-23T23:38:31.612256+0000","last_clean":"2023-09-23T23:38:31.612256+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:38:31.612256+0000","last_undegraded":"2023-09-23T23:38:31.612256+0000","last_fullsized":"2023-09-23T23:38:31.612256+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":297,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:43.465460+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:38:43.465460+0000","last_peered":"2023-09-23T23:38:43.465460+0000","last_clean":"2023-09-23T23:38:43.465460+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:38:43.465460+0000","last_undegraded":"2023-09-23T23:38:43.465460+0000","last_fullsized":"2023-09-23T23:38:43.465460+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":821,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613682+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:38:31.613682+0000","last_peered":"2023-09-23T23:38:31.613682+0000","last_clean":"2023-09-23T23:38:31.613682+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:38:31.613682+0000","last_undegraded":"2023-09-23T23:38:31.613682+0000","last_fullsized":"2023-09-23T23:38:31.613682+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":640,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612339+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:38:31.612339+0000","last_peered":"2023-09-23T23:38:31.612339+0000","last_clean":"2023-09-23T23:38:31.612339+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:38:31.612339+0000","last_undegraded":"2023-09-23T23:38:31.612339+0000","last_fullsized":"2023-09-23T23:38:31.612339+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":624,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.611128+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:38:31.611128+0000","last_peered":"2023-09-23T23:38:31.611128+0000","last_clean":"2023-09-23T23:38:31.611128+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:38:31.611128+0000","last_undegraded":"2023-09-23T23:38:31.611128+0000","last_fullsized":"2023-09-23T23:38:31.611128+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":782,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.613110+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:38:31.613110+0000","last_peered":"2023-09-23T23:38:31.613110+0000","last_clean":"2023-09-23T23:38:31.613110+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:38:31.613110+0000","last_undegraded":"2023-09-23T23:38:31.613110+0000","last_fullsized":"2023-09-23T23:38:31.613110+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":454,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T23:38:43.453665+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:38:43.453665+0000","last_peered":"2023-09-23T23:38:43.453665+0000","last_clean":"2023-09-23T23:38:43.453665+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:38:43.453665+0000","last_undegraded":"2023-09-23T23:38:43.453665+0000","last_fullsized":"2023-09-23T23:38:43.453665+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":676,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T23:38:31.612386+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:38:31.612386+0000","last_peered":"2023-09-23T23:38:31.612386+0000","last_clean":"2023-09-23T23:38:31.612386+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:38:31.612386+0000","last_undegraded":"2023-09-23T23:38:31.612386+0000","last_fullsized":"2023-09-23T23:38:31.612386+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14396,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":175556,"num_write_kb":5617729,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28139,"num_bytes_recovered":1517355008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2292318208,"data_stored":2292318208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91134,"ondisk_log_size":91134,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782713,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":427096,"kb_used_data":6608,"kb_used_omap":8,"kb_used_meta":420471,"kb_avail":93318056,"statfs":{"total":95995035648,"available":95557689344,"internally_reserved":0,"allocated":6766592,"data_stored":5062195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":430562373},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.81599999999999995,"15min":2.0459999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":2.4159999999999999,"5min":11.173,"15min":85.721000000000004},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.77300000000000002,"15min":2.145},"min":{"1min":0.24199999999999999,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":2.5499999999999998,"5min":5.2530000000000001,"15min":85.965000000000003},"last":0.60299999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":2.1549999999999998,"15min":3.105},"min":{"1min":0.255,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":2.6739999999999999,"5min":129.11199999999999,"15min":198.58000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.61299999999999999,"5min":2.036,"15min":3.0870000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.2850000000000001,"5min":129.02600000000001,"15min":198.71100000000001},"last":0.41299999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":2.4180000000000001,"15min":3.7330000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":3.3860000000000001,"5min":78.528999999999996,"15min":203.58000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.141,"5min":1.712,"15min":3.2200000000000002},"min":{"1min":0.30399999999999999,"5min":0.27900000000000003,"15min":0.26200000000000001},"max":{"1min":6.0700000000000003,"5min":53.023000000000003,"15min":198.69},"last":0.54900000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":14.901999999999999,"5min":7.1070000000000002,"15min":4.9690000000000003},"min":{"1min":0.33000000000000002,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":154.18100000000001,"5min":154.18100000000001,"15min":279.26499999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":14.157999999999999,"5min":7.3140000000000001,"15min":5.476},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":154.16300000000001,"5min":154.16300000000001,"15min":279.29899999999998},"last":0.43099999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.464,"15min":1.026},"min":{"1min":0.318,"5min":0.27100000000000002,"15min":0.246},"max":{"1min":2.0499999999999998,"5min":31.498999999999999,"15min":31.498999999999999},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.339,"15min":0.97399999999999998},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.23899999999999999},"max":{"1min":5.391,"5min":31.68,"15min":31.68},"last":0.56299999999999994}]},{"osd":5,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":7.1280000000000001,"5min":2.7919999999999998,"15min":2.266},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.26500000000000001},"max":{"1min":119.417,"5min":119.417,"15min":210.83600000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":7.6669999999999998,"5min":3.016,"15min":2.1989999999999998},"min":{"1min":0.33600000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":119.61499999999999,"5min":119.61499999999999,"15min":207.04599999999999},"last":0.58099999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":1.8899999999999999,"15min":1.3420000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":6.2169999999999996,"5min":94.852999999999994,"15min":94.852999999999994},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.63,"5min":1.956,"15min":1.514},"min":{"1min":0.40100000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.911,"5min":95.510999999999996,"15min":95.510999999999996},"last":0.53900000000000003}]}]},{"osd":6,"up_from":46,"seq":197568496190,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":814216,"kb_used_data":73600,"kb_used_omap":0,"kb_used_meta":740608,"kb_avail":92930936,"statfs":{"total":95995035648,"available":95161278464,"internally_reserved":0,"allocated":75366400,"data_stored":73677466,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":758382592},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:26 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":0.90400000000000003,"15min":3.956},"min":{"1min":0.224,"5min":0.224,"15min":0.21099999999999999},"max":{"1min":6.5579999999999998,"5min":6.5579999999999998,"15min":135.31999999999999},"last":0.28499999999999998},{"interface":"front","average":{"1min":1.1850000000000001,"5min":0.70899999999999996,"15min":3.891},"min":{"1min":0.28599999999999998,"5min":0.24399999999999999,"15min":0.23000000000000001},"max":{"1min":6.8529999999999998,"5min":6.8529999999999998,"15min":135.34299999999999},"last":0.47799999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:38:26 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":1.018,"15min":2.1720000000000002},"min":{"1min":0.28499999999999998,"5min":0.253,"15min":0.245},"max":{"1min":6.6200000000000001,"5min":6.9080000000000004,"15min":89.605000000000004},"last":0.25},{"interface":"front","average":{"1min":1.4990000000000001,"5min":0.873,"15min":2.2749999999999999},"min":{"1min":0.26300000000000001,"5min":0.249,"15min":0.23300000000000001},"max":{"1min":12.577,"5min":12.577,"15min":91.025999999999996},"last":0.55700000000000005}]},{"osd":2,"last update":"Sat Sep 23 23:38:10 2023","interfaces":[{"interface":"back","average":{"1min":7.9569999999999999,"5min":5.032,"15min":4.9539999999999997},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":85.995999999999995,"5min":85.995999999999995,"15min":127.44499999999999},"last":0.501},{"interface":"front","average":{"1min":7.6959999999999997,"5min":5.0250000000000004,"15min":5.3319999999999999},"min":{"1min":0.32300000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":85.266999999999996,"5min":85.266999999999996,"15min":125.539},"last":0.312}]},{"osd":3,"last update":"Sat Sep 23 23:37:57 2023","interfaces":[{"interface":"back","average":{"1min":13.925000000000001,"5min":8.1059999999999999,"15min":5.2009999999999996},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":199.536,"5min":199.536,"15min":199.536},"last":0.38300000000000001},{"interface":"front","average":{"1min":13.538,"5min":7.9660000000000002,"15min":5.4530000000000003},"min":{"1min":0.36799999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":199.59999999999999,"5min":199.59999999999999,"15min":199.59999999999999},"last":0.375}]},{"osd":4,"last update":"Sat Sep 23 23:38:31 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.73599999999999999,"15min":0.69499999999999995},"min":{"1min":0.29799999999999999,"5min":0.27900000000000003,"15min":0.23699999999999999},"max":{"1min":5.7619999999999996,"5min":5.7619999999999996,"15min":8.1999999999999993},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.085,"15min":0.85899999999999999},"min":{"1min":0.32400000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":12.124000000000001,"5min":12.124000000000001,"15min":12.124000000000001},"last":0.42799999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:38:31 2023","interfaces":[{"interface":"back","average":{"1min":1.637,"5min":0.96899999999999997,"15min":0.871},"min":{"1min":0.32400000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.8490000000000002,"5min":11.372,"15min":11.473000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":2.133,"5min":1.1679999999999999,"15min":0.94999999999999996},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26900000000000002},"max":{"1min":12.323,"5min":12.323,"15min":12.323},"last":0.41199999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:38:10 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.25,"15min":1.012},"min":{"1min":0.33600000000000002,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":6.8259999999999996,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.436},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.331,"15min":1.079},"min":{"1min":0.38300000000000001,"5min":0.33800000000000002,"15min":0.33500000000000002},"max":{"1min":12.531000000000001,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.52100000000000002}]}]},{"osd":1,"up_from":112,"seq":481036337531,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":926432,"kb_used_data":7640,"kb_used_omap":4,"kb_used_meta":918779,"kb_avail":92818720,"statfs":{"total":95995035648,"available":95046369280,"internally_reserved":0,"allocated":7823360,"data_stored":4945072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":940830606},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":0.98699999999999999,"15min":3.1669999999999998},"min":{"1min":0.16400000000000001,"5min":0.16400000000000001,"15min":0.16400000000000001},"max":{"1min":7.1070000000000002,"5min":15.983000000000001,"15min":130.768},"last":0.253},{"interface":"front","average":{"1min":0.78800000000000003,"5min":0.65200000000000002,"15min":2.5150000000000001},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":5.726,"5min":5.8099999999999996,"15min":134.929},"last":0.32400000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:38:05 2023","interfaces":[{"interface":"back","average":{"1min":3.2170000000000001,"5min":3.8460000000000001,"15min":6.9219999999999997},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":53.951999999999998,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":3.6440000000000001,"5min":3.9420000000000002,"15min":7.0069999999999997},"min":{"1min":0.221,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":54.213000000000001,"5min":202.12,"15min":232.21000000000001},"last":0.28799999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:38:05 2023","interfaces":[{"interface":"back","average":{"1min":7.2720000000000002,"5min":6.9420000000000002,"15min":5.8620000000000001},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22600000000000001},"max":{"1min":77.034999999999997,"5min":181.94399999999999,"15min":234.404},"last":0.36199999999999999},{"interface":"front","average":{"1min":7.5010000000000003,"5min":7.0190000000000001,"15min":5.8150000000000004},"min":{"1min":0.245,"5min":0.224,"15min":0.224},"max":{"1min":77.165999999999997,"5min":181.98500000000001,"15min":236.33699999999999},"last":0.38400000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:38:23 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":0.998,"15min":1.4039999999999999},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.28699999999999998},"max":{"1min":3.5939999999999999,"5min":4.7069999999999999,"15min":12.042999999999999},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.034,"5min":0.874,"15min":1.4399999999999999},"min":{"1min":0.33300000000000002,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":7.1779999999999999,"5min":7.1779999999999999,"15min":18.108000000000001},"last":0.39200000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:38:23 2023","interfaces":[{"interface":"back","average":{"1min":4.2169999999999996,"5min":3.2690000000000001,"15min":1.962},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":80.302999999999997,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":4.7610000000000001,"5min":3.4239999999999999,"15min":2.3130000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":80.254000000000005,"5min":151.821,"15min":151.821},"last":0.46100000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:38:23 2023","interfaces":[{"interface":"back","average":{"1min":4.5700000000000003,"5min":1.732,"15min":1.921},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.27500000000000002},"max":{"1min":87.837999999999994,"5min":87.837999999999994,"15min":87.837999999999994},"last":0.40000000000000002},{"interface":"front","average":{"1min":5.0910000000000002,"5min":2.1080000000000001,"15min":2.198},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":85.518000000000001,"5min":85.518000000000001,"15min":85.518000000000001},"last":0.47799999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:38:05 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":1.4590000000000001,"15min":2.101},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.7639999999999998,"5min":44.241,"15min":78.308000000000007},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.657,"5min":1.514,"15min":2.113},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32700000000000001},"max":{"1min":5.7969999999999997,"5min":43.920999999999999,"15min":77.204999999999998},"last":0.495}]}]},{"osd":0,"up_from":91,"seq":390842024355,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1223984,"kb_used_data":7976,"kb_used_omap":5,"kb_used_meta":1215994,"kb_avail":92521168,"statfs":{"total":95995035648,"available":94741676032,"internally_reserved":0,"allocated":8167424,"data_stored":5274848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1245178490},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:38:40 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":4.0839999999999996,"15min":6.4660000000000002},"min":{"1min":0.22500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":3.9769999999999999,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.579,"5min":3.8439999999999999,"15min":5.6849999999999996},"min":{"1min":0.23499999999999999,"5min":0.20200000000000001,"15min":0.17599999999999999},"max":{"1min":13.141,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.307}]},{"osd":2,"last update":"Sat Sep 23 23:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":6.702,"15min":6.5780000000000003},"min":{"1min":0.214,"5min":0.214,"15min":0.17199999999999999},"max":{"1min":5.4340000000000002,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.77900000000000003,"5min":6.4569999999999999,"15min":6.5339999999999998},"min":{"1min":0.22600000000000001,"5min":0.20799999999999999,"15min":0.17599999999999999},"max":{"1min":3.4470000000000001,"5min":283.161,"15min":283.161},"last":0.49099999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:38:40 2023","interfaces":[{"interface":"back","average":{"1min":6.2859999999999996,"5min":7.3879999999999999,"15min":6.8490000000000002},"min":{"1min":0.28599999999999998,"5min":0.219,"15min":0.217},"max":{"1min":66.093000000000004,"5min":194.26300000000001,"15min":220.732},"last":0.746},{"interface":"front","average":{"1min":6.8170000000000002,"5min":7.3819999999999997,"15min":6.9580000000000002},"min":{"1min":0.36099999999999999,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":66.123999999999995,"5min":194.56299999999999,"15min":218.78200000000001},"last":0.35999999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.748,"15min":2.2799999999999998},"min":{"1min":0.33000000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.605,"5min":4.1310000000000002,"15min":138.81299999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.095,"15min":3.0369999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.27200000000000002},"max":{"1min":4.2030000000000003,"5min":9.3670000000000009,"15min":171.251},"last":0.41599999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":1.3029999999999999,"15min":2.75},"min":{"1min":0.38100000000000001,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":4.4880000000000004,"5min":15.076000000000001,"15min":171.30699999999999},"last":0.501},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.1000000000000001,"15min":1.9570000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.253},"max":{"1min":2.7029999999999998,"5min":6.5279999999999996,"15min":139.40799999999999},"last":0.77400000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.1870000000000001,"15min":3.0569999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.29499999999999998},"max":{"1min":3.5939999999999999,"5min":12.654999999999999,"15min":171.33600000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.80300000000000005,"5min":1.1659999999999999,"15min":3.2360000000000002},"min":{"1min":0.39000000000000001,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":2.1360000000000001,"5min":9.157,"15min":184.13499999999999},"last":0.79500000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":1.012,"15min":2.5339999999999998},"min":{"1min":0.35999999999999999,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":3.4089999999999998,"5min":8.9510000000000005,"15min":138.77699999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.89100000000000001,"5min":1.3069999999999999,"15min":2.5089999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.33300000000000002},"max":{"1min":2.5,"5min":9.8870000000000005,"15min":139.37899999999999},"last":0.81200000000000006}]}]},{"osd":2,"up_from":199,"seq":854698492176,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":896172,"kb_used_data":7972,"kb_used_omap":7,"kb_used_meta":888184,"kb_avail":92848980,"statfs":{"total":95995035648,"available":95077355520,"internally_reserved":0,"allocated":8163328,"data_stored":5217423,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":909500497},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.496,"15min":2.968},"min":{"1min":0.20999999999999999,"5min":0.186,"15min":0.17699999999999999},"max":{"1min":2.9980000000000002,"5min":61.950000000000003,"15min":95.587999999999994},"last":0.23499999999999999},{"interface":"front","average":{"1min":0.90700000000000003,"5min":1.7050000000000001,"15min":2.8319999999999999},"min":{"1min":0.22600000000000001,"5min":0.19700000000000001,"15min":0.18099999999999999},"max":{"1min":4.5739999999999998,"5min":61.984999999999999,"15min":95.796000000000006},"last":0.252}]},{"osd":1,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":4.3550000000000004,"15min":4.516},"min":{"1min":0.21299999999999999,"5min":0.192,"15min":0.192},"max":{"1min":4.4500000000000002,"5min":106.765,"15min":157.98699999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.76300000000000001,"5min":3.9319999999999999,"15min":4.4649999999999999},"min":{"1min":0.24399999999999999,"5min":0.222,"15min":0.222},"max":{"1min":4.468,"5min":106.732,"15min":158.03800000000001},"last":0.308}]},{"osd":3,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":5.1689999999999996,"15min":4.3070000000000004},"min":{"1min":0.245,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":4.1740000000000004,"5min":106.70399999999999,"15min":106.70399999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.0209999999999999,"5min":4.7800000000000002,"15min":4.1459999999999999},"min":{"1min":0.28100000000000003,"5min":0.246,"15min":0.23200000000000001},"max":{"1min":3.52,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.38100000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":1.6779999999999999,"15min":1.768},"min":{"1min":0.35499999999999998,"5min":0.33700000000000002,"15min":0.27700000000000002},"max":{"1min":3.5339999999999998,"5min":62.081000000000003,"15min":62.081000000000003},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.875,"15min":1.802},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.28499999999999998},"max":{"1min":2.9489999999999998,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.44}]},{"osd":5,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":9.3179999999999996,"5min":4.476,"15min":2.6640000000000001},"min":{"1min":0.32200000000000001,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":101.104,"5min":101.104,"15min":101.104},"last":0.39500000000000002},{"interface":"front","average":{"1min":9.7829999999999995,"5min":4.6280000000000001,"15min":2.6800000000000002},"min":{"1min":0.35199999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":101.404,"5min":101.404,"15min":101.404},"last":0.40400000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":9.1869999999999994,"5min":4.8730000000000002,"15min":2.9239999999999999},"min":{"1min":0.372,"5min":0.31,"15min":0.31},"max":{"1min":97.558999999999997,"5min":107.018,"15min":107.018},"last":0.499},{"interface":"front","average":{"1min":9.4420000000000002,"5min":3.7989999999999999,"15min":2.5219999999999998},"min":{"1min":0.375,"5min":0.371,"15min":0.36699999999999999},"max":{"1min":98.231999999999999,"5min":98.231999999999999,"15min":98.231999999999999},"last":0.42499999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.7150000000000001,"15min":1.8680000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.4129999999999998,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.0840000000000001,"5min":3,"15min":2.3100000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":3.452,"5min":106.809,"15min":106.809},"last":0.50900000000000001}]}]},{"osd":3,"up_from":148,"seq":635655160144,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":951448,"kb_used_data":7952,"kb_used_omap":2,"kb_used_meta":943485,"kb_avail":92793704,"statfs":{"total":95995035648,"available":95020752896,"internally_reserved":0,"allocated":8142848,"data_stored":5269896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":966128693},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.101,"15min":2.1000000000000001},"min":{"1min":0.22700000000000001,"5min":0.184,"15min":0.17699999999999999},"max":{"1min":6.4400000000000004,"5min":17.213000000000001,"15min":62.914000000000001},"last":0.29099999999999998},{"interface":"front","average":{"1min":1.6259999999999999,"5min":1.1399999999999999,"15min":2.4689999999999999},"min":{"1min":0.20300000000000001,"5min":0.188,"15min":0.17199999999999999},"max":{"1min":7.843,"5min":12.202,"15min":63.276000000000003},"last":0.32700000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:38:50 2023","interfaces":[{"interface":"back","average":{"1min":1.694,"5min":1.155,"15min":2.766},"min":{"1min":0.217,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":9.7170000000000005,"5min":9.7170000000000005,"15min":85.242999999999995},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.86899999999999999,"15min":2.6459999999999999},"min":{"1min":0.251,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":6.4169999999999998,"5min":11.228,"15min":85.302000000000007},"last":0.315}]},{"osd":2,"last update":"Sat Sep 23 23:37:57 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":1.4359999999999999,"15min":4.0659999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.19800000000000001},"max":{"1min":36.362000000000002,"5min":36.362000000000002,"15min":84.724999999999994},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.6110000000000002,"5min":1.431,"15min":3.931},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.214},"max":{"1min":36.158999999999999,"5min":36.158999999999999,"15min":84.567999999999998},"last":0.33900000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:38:50 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":1.0069999999999999,"15min":1.1879999999999999},"min":{"1min":0.308,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":6.3330000000000002,"5min":11.209,"15min":11.209},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.9019999999999999,"5min":1.167,"15min":1.6499999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.26800000000000002},"max":{"1min":10.917,"5min":10.917,"15min":24.782},"last":0.37}]},{"osd":5,"last update":"Sat Sep 23 23:38:50 2023","interfaces":[{"interface":"back","average":{"1min":4.8259999999999996,"5min":3.831,"15min":4.1909999999999998},"min":{"1min":0.373,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":79.572999999999993,"5min":196.358,"15min":214.43700000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":4.8499999999999996,"5min":3.8740000000000001,"15min":4.1890000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.30599999999999999},"max":{"1min":79.426000000000002,"5min":196.048,"15min":214.46600000000001},"last":0.51900000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:38:50 2023","interfaces":[{"interface":"back","average":{"1min":1.877,"5min":2.6779999999999999,"15min":2.5950000000000002},"min":{"1min":0.36899999999999999,"5min":0.318,"15min":0.318},"max":{"1min":11.09,"5min":125.94,"15min":125.94},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.855,"5min":2.7679999999999998,"15min":2.5739999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.6319999999999997,"5min":125.955,"15min":125.955},"last":0.497}]},{"osd":7,"last update":"Sat Sep 23 23:37:57 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":4.5289999999999999,"15min":3.1909999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32600000000000001},"max":{"1min":6.5800000000000001,"5min":192.69,"15min":192.69},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.92400000000000004,"5min":4.5789999999999997,"15min":3.629},"min":{"1min":0.40699999999999997,"5min":0.40100000000000002,"15min":0.36699999999999999},"max":{"1min":3.6120000000000001,"5min":192.65600000000001,"15min":192.65600000000001},"last":0.53000000000000003}]}]},{"osd":4,"up_from":0,"seq":128849019463,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757826,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":781256,"kb_used_data":6592,"kb_used_omap":0,"kb_used_meta":774656,"kb_avail":92963896,"statfs":{"total":95995035648,"available":95195029504,"internally_reserved":0,"allocated":6750208,"data_stored":5057949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":793247744},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:19 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.78200000000000003,"15min":1.454},"min":{"1min":0.223,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.2890000000000001,"5min":6.1609999999999996,"15min":63.741999999999997},"last":0.27800000000000002},{"interface":"front","average":{"1min":1,"5min":1.1779999999999999,"15min":1.964},"min":{"1min":0.26200000000000001,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":4.8579999999999997,"5min":15.414,"15min":138.55199999999999},"last":0.39000000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:38:25 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":0.85599999999999998,"15min":2.21},"min":{"1min":0.27000000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.7480000000000002,"5min":6.1360000000000001,"15min":69.816999999999993},"last":0.29199999999999998},{"interface":"front","average":{"1min":1.1120000000000001,"5min":0.89300000000000002,"15min":2.4249999999999998},"min":{"1min":0.307,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":3.7570000000000001,"5min":4.4859999999999998,"15min":70.268000000000001},"last":0.33500000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":6.8869999999999996,"15min":4.4939999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26900000000000002},"max":{"1min":6.2469999999999999,"5min":208.738,"15min":208.738},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.77700000000000002,"5min":5.5220000000000002,"15min":4.0140000000000002},"min":{"1min":0.309,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":3.2290000000000001,"5min":208.631,"15min":208.631},"last":0.496}]},{"osd":3,"last update":"Sat Sep 23 23:37:53 2023","interfaces":[{"interface":"back","average":{"1min":6.6550000000000002,"5min":5.6689999999999996,"15min":4.1470000000000002},"min":{"1min":0.33400000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":143.66300000000001,"5min":143.66300000000001,"15min":143.66300000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":6.7670000000000003,"5min":6.0010000000000003,"15min":4.327},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":143.62200000000001,"5min":143.62200000000001,"15min":143.62200000000001},"last":0.57399999999999995}]},{"osd":4,"last update":"Sat Sep 23 23:38:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":0.90800000000000003,"15min":0.92300000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.251},"max":{"1min":4.4269999999999996,"5min":9.2690000000000001,"15min":19.962},"last":0.443},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.92200000000000004,"15min":0.87},"min":{"1min":0.32300000000000001,"5min":0.28899999999999998,"15min":0.22900000000000001},"max":{"1min":4.4169999999999998,"5min":10.057,"15min":20.829999999999998},"last":0.52600000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":2.3490000000000002,"15min":1.556},"min":{"1min":0.28899999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.2350000000000003,"5min":148.36600000000001,"15min":148.36600000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.1639999999999999,"5min":2.5870000000000002,"15min":1.754},"min":{"1min":0.35599999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.7919999999999998,"5min":148.42500000000001,"15min":148.42500000000001},"last":0.59599999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:38:51 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.2310000000000001,"15min":1.9970000000000001},"min":{"1min":0.33600000000000002,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":5.8559999999999999,"5min":15.497,"15min":179.28800000000001},"last":0.61499999999999999},{"interface":"front","average":{"1min":1.492,"5min":1.252,"15min":2.0579999999999998},"min":{"1min":0.40300000000000002,"5min":0.36699999999999999,"15min":0.34999999999999998},"max":{"1min":5.7729999999999997,"5min":15.653,"15min":179.428},"last":0.63100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2223767552,"data_stored":2223767552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":68550656,"data_stored":68550656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:39:01.473 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:39:01.521 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:01.552 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:01.613 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:01.615 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:01 smithi110 ceph-mon[221102]: pgmap v1748: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 11674/72 objects degraded (16213.889%); 0 B/s, 680 objects/s recovering 2023-09-23T23:39:01.615 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:01 smithi110 ceph-mon[221102]: failed to encode map e614 with expected crc 2023-09-23T23:39:01.615 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:01 smithi110 ceph-mon[221102]: failed to encode map e614 with expected crc 2023-09-23T23:39:01.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:01 smithi110 ceph-mon[212939]: pgmap v1748: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 11674/72 objects degraded (16213.889%); 0 B/s, 680 objects/s recovering 2023-09-23T23:39:01.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:01 smithi110 ceph-mon[212939]: failed to encode map e614 with expected crc 2023-09-23T23:39:01.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:01 smithi110 ceph-mon[212939]: failed to encode map e614 with expected crc 2023-09-23T23:39:01.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:01.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:01.717 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:01.741 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:01 smithi138 ceph-mon[193606]: pgmap v1748: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 11674/72 objects degraded (16213.889%); 0 B/s, 680 objects/s recovering 2023-09-23T23:39:01.741 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:01 smithi138 ceph-mon[193606]: failed to encode map e614 with expected crc 2023-09-23T23:39:01.741 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:01 smithi138 ceph-mon[193606]: failed to encode map e614 with expected crc 2023-09-23T23:39:01.743 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:01.786 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:01.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:01.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:01.892 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:01.918 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:01.960 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:01.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:01.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:02.066 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:02.093 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:02.136 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:02.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:02.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:02.244 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:02.270 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:02.313 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:02.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:02.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:02.419 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:02.447 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:02.522 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_8' removed 2023-09-23T23:39:02.635 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:02.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:02.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:02.697 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:02 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4040588419' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:39:02.698 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:02 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4040588419' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:39:02.740 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:02.765 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:02 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4040588419' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:39:02.767 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:02.807 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:02.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:02.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:02.905 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:39:02.905 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:39:02.905 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:39:02.905 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:39:02.905 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:39:02.913 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:02.949 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:02.993 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:02.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:02.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:03.098 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:03.126 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:03.420 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:39:03.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:03.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:39:03.524 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:39:03.552 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:03.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:03 smithi110 ceph-mon[212939]: pgmap v1749: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 11231/72 objects degraded (15598.611%); 0 B/s, 610 objects/s recovering 2023-09-23T23:39:03.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:03 smithi110 ceph-mon[212939]: failed to encode map e615 with expected crc 2023-09-23T23:39:03.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:03 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4040588419' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:39:03.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:03 smithi110 ceph-mon[212939]: osdmap e615: 8 total, 8 up, 7 in 2023-09-23T23:39:03.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:03 smithi110 ceph-mon[221102]: pgmap v1749: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 11231/72 objects degraded (15598.611%); 0 B/s, 610 objects/s recovering 2023-09-23T23:39:03.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:03 smithi110 ceph-mon[221102]: failed to encode map e615 with expected crc 2023-09-23T23:39:03.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:03 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4040588419' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:39:03.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:03 smithi110 ceph-mon[221102]: osdmap e615: 8 total, 8 up, 7 in 2023-09-23T23:39:03.746 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:39:03.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:03.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:39:03.850 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:03.877 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:03 smithi138 ceph-mon[193606]: pgmap v1749: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 11231/72 objects degraded (15598.611%); 0 B/s, 610 objects/s recovering 2023-09-23T23:39:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:03 smithi138 ceph-mon[193606]: failed to encode map e615 with expected crc 2023-09-23T23:39:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:03 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4040588419' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:39:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:03 smithi138 ceph-mon[193606]: osdmap e615: 8 total, 8 up, 7 in 2023-09-23T23:39:04.150 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:39:04.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:04.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:39:04.172 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T23:39:04.172 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T23:39:04.173 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_9 2023-09-23T23:39:04.173 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_9 16 2023-09-23T23:39:04.197 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:39:04.203 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:04.255 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:04.256 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:04.287 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:04.335 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:04.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:04.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:04.442 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:04.471 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:04.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:04 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 11231/72 objects degraded (15598.611%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:39:04.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:04 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 11231/72 objects degraded (15598.611%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:39:04.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:04 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 11231/72 objects degraded (15598.611%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:39:04.721 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:39:04.801 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:04.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:04.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:04.905 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:04.937 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:04.985 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:04.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:04.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:05.090 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:39:05.117 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:05.551 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:39:05.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:05.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:39:05.654 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:05.686 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:05.735 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:05.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:05.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:05.830 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:05 smithi110 ceph-mon[212939]: pgmap v1751: 103 pgs: 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:05.831 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:05 smithi110 ceph-mon[212939]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11231/72 objects degraded (15598.611%), 3 pgs degraded, 3 pgs undersized) 2023-09-23T23:39:05.831 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:05 smithi110 ceph-mon[221102]: pgmap v1751: 103 pgs: 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:05.831 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:05 smithi110 ceph-mon[221102]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11231/72 objects degraded (15598.611%), 3 pgs degraded, 3 pgs undersized) 2023-09-23T23:39:05.843 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:05.873 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:05.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:05 smithi138 ceph-mon[193606]: pgmap v1751: 103 pgs: 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:05.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:05 smithi138 ceph-mon[193606]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11231/72 objects degraded (15598.611%), 3 pgs degraded, 3 pgs undersized) 2023-09-23T23:39:06.026 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:06.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:06.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:06.131 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:39:06.163 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:06.211 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:39:06.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:06.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:39:06.317 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:06.346 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:06.347 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:06.383 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:06.390 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:06.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:06.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:06.397 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:06.498 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:06.532 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:06.694 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:06.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:06.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:06.798 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:06.826 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:06.869 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:06.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:06.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:06.977 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:07.007 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:07.033 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:39:07.033 INFO:teuthology.orchestra.run.smithi110.stdout:{"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2224,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":615,"num_osds":8,"num_up_osds":8,"osd_up_since":1695511414,"num_in_osds":7,"osd_in_since":1695512030,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":103}],"num_pgs":103,"num_pools":4,"num_objects":24,"data_bytes":636125,"bytes_used":6158872576,"bytes_avail":665806376960,"bytes_total":671965249536},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi110.fuglpi","status":"up:active","gid":24572}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.110:8443/","prometheus":"http://172.21.15.110:9283/"}},"servicemap":{"epoch":33,"modified":"2023-09-23T23:38:29.965572+0000","services":{}},"progress_events":{"833ef15d-07ae-4299-b4e4-2cf94e068147":{"message":"Upgrade to 18.2.0-460-gbb0daeff (10m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T23:39:07.056 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:39:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:39:06] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.43.0" 2023-09-23T23:39:07.059 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:07.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:07.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:07.163 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:07.190 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:07.231 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:07.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:07.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:07.338 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:07.366 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:07.541 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:39:07.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:07 smithi110 ceph-mon[212939]: pgmap v1752: 103 pgs: 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:07.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:07 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3853413377' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:39:07.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:07 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:39:07.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:07 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/268034360' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:39:07.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:07 smithi110 ceph-mon[221102]: pgmap v1752: 103 pgs: 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:07.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:07 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3853413377' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:39:07.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:07 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:39:07.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:07 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/268034360' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:39:07.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:07 smithi138 ceph-mon[193606]: pgmap v1752: 103 pgs: 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:07.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:07 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3853413377' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:39:07.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:07 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:39:07.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:07 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/268034360' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:39:08.764 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:08.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:08 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:39:08.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:08 smithi110 ceph-mon[212939]: failed to encode map e616 with expected crc 2023-09-23T23:39:08.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:08 smithi110 ceph-mon[212939]: failed to encode map e616 with expected crc 2023-09-23T23:39:08.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:08 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:39:08.767 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:08 smithi110 ceph-mon[212939]: osdmap e616: 8 total, 8 up, 7 in 2023-09-23T23:39:08.767 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:08 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:39:08.767 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:08 smithi110 ceph-mon[221102]: failed to encode map e616 with expected crc 2023-09-23T23:39:08.768 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:08 smithi110 ceph-mon[221102]: failed to encode map e616 with expected crc 2023-09-23T23:39:08.768 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:08 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:39:08.768 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:08 smithi110 ceph-mon[221102]: osdmap e616: 8 total, 8 up, 7 in 2023-09-23T23:39:08.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:08.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:08.797 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-09-23T23:39:08.798 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:39:08.870 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:39:08.908 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:08 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:39:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:08 smithi138 ceph-mon[193606]: failed to encode map e616 with expected crc 2023-09-23T23:39:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:08 smithi138 ceph-mon[193606]: failed to encode map e616 with expected crc 2023-09-23T23:39:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:08 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:39:08.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:08 smithi138 ceph-mon[193606]: osdmap e616: 8 total, 8 up, 7 in 2023-09-23T23:39:09.837 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:09 smithi110 ceph-mon[212939]: pgmap v1754: 103 pgs: 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:09.837 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:09 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:09.837 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:09 smithi110 ceph-mon[221102]: pgmap v1754: 103 pgs: 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:09.837 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:09 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:09.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:09 smithi138 ceph-mon[193606]: pgmap v1754: 103 pgs: 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:09.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:09 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:09.960 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:39:09.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:09.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:39:10.066 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:10.098 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:10.116 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:10.146 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:10.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:10.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:10.253 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:10.282 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:10.326 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:10.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:10.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:10.432 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:39:10.462 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:10.462 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:39:10.484 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:39:10.512 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:39:10.533 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:39:10.553 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:39:10.573 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:39:10.592 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:39:10.611 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:39:10.612 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:39:10.612 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:39:10.613 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:39:10.613 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:39:10.613 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:39:10.613 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:39:10.613 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:39:10.613 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:39:10.623 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:39:10.623 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:39:10.711 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:39:10.713 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:10 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2531494681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:39:10.713 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:10 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2531494681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:39:10.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:10.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:39:10.816 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:10.841 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:10 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2531494681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:39:10.844 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:10.889 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:10.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:10.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:10.996 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:11.022 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:11.063 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:11.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:11.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:11.169 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:11.196 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:11.236 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:11.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:11.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:11.342 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:11.373 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:11.523 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:11.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:11.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:11.629 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:39:11.656 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:11.905 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:39:11.931 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:11 smithi110 ceph-mon[212939]: pgmap v1755: 103 pgs: 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:11.931 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:11 smithi110 ceph-mon[221102]: pgmap v1755: 103 pgs: 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:11 smithi138 ceph-mon[193606]: pgmap v1755: 103 pgs: 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:12.471 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:39:12.525 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_9' created 2023-09-23T23:39:12.555 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:39:12.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:12.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:39:12.597 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:12.661 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:12.690 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:12.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:12 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1803360094' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-09-23T23:39:12.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:12 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1803360094' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-09-23T23:39:12.734 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:12.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:12.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:12.838 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:39:12.864 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:12.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:12 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1803360094' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-09-23T23:39:13.312 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:39:13.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:13.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:39:13.340 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_9 rados --yes-i-really-mean-it || true 2023-09-23T23:39:13.415 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:13.458 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:13.516 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:39:13.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:13.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:39:13.622 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:39:13.653 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:13.662 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:13.754 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:39:13.756 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:39:13.777 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:13 smithi110 ceph-mon[212939]: pgmap v1756: 103 pgs: 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:13.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:13 smithi110 ceph-mon[212939]: failed to encode map e617 with expected crc 2023-09-23T23:39:13.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:13 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1803360094' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-09-23T23:39:13.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:13 smithi110 ceph-mon[212939]: osdmap e617: 8 total, 8 up, 7 in 2023-09-23T23:39:13.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:13 smithi110 ceph-mon[212939]: failed to encode map e617 with expected crc 2023-09-23T23:39:13.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:13 smithi110 ceph-mon[212939]: failed to encode map e617 with expected crc 2023-09-23T23:39:13.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:13.778 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:39:13.779 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:13 smithi110 ceph-mon[221102]: pgmap v1756: 103 pgs: 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:13.779 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:13 smithi110 ceph-mon[221102]: failed to encode map e617 with expected crc 2023-09-23T23:39:13.779 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:13 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1803360094' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-09-23T23:39:13.779 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:13 smithi110 ceph-mon[221102]: osdmap e617: 8 total, 8 up, 7 in 2023-09-23T23:39:13.779 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:13 smithi110 ceph-mon[221102]: failed to encode map e617 with expected crc 2023-09-23T23:39:13.779 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:13 smithi110 ceph-mon[221102]: failed to encode map e617 with expected crc 2023-09-23T23:39:13.779 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:13.779 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:39:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:13 smithi138 ceph-mon[193606]: pgmap v1756: 103 pgs: 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:13 smithi138 ceph-mon[193606]: failed to encode map e617 with expected crc 2023-09-23T23:39:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:13 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1803360094' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-09-23T23:39:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:13 smithi138 ceph-mon[193606]: osdmap e617: 8 total, 8 up, 7 in 2023-09-23T23:39:13.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:13 smithi138 ceph-mon[193606]: failed to encode map e617 with expected crc 2023-09-23T23:39:13.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:13 smithi138 ceph-mon[193606]: failed to encode map e617 with expected crc 2023-09-23T23:39:13.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:13.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:39:14.490 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:39:14.592 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:39:14.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:14.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:39:14.696 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:14.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:14 smithi110 ceph-mon[212939]: failed to encode map e618 with expected crc 2023-09-23T23:39:14.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:14 smithi110 ceph-mon[212939]: osdmap e618: 8 total, 8 up, 7 in 2023-09-23T23:39:14.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:14 smithi110 ceph-mon[212939]: failed to encode map e618 with expected crc 2023-09-23T23:39:14.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:14 smithi110 ceph-mon[212939]: failed to encode map e618 with expected crc 2023-09-23T23:39:14.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:14 smithi110 ceph-mon[212939]: failed to encode map e618 with expected crc 2023-09-23T23:39:14.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:14 smithi110 ceph-mon[221102]: failed to encode map e618 with expected crc 2023-09-23T23:39:14.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:14 smithi110 ceph-mon[221102]: osdmap e618: 8 total, 8 up, 7 in 2023-09-23T23:39:14.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:14 smithi110 ceph-mon[221102]: failed to encode map e618 with expected crc 2023-09-23T23:39:14.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:14 smithi110 ceph-mon[221102]: failed to encode map e618 with expected crc 2023-09-23T23:39:14.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:14 smithi110 ceph-mon[221102]: failed to encode map e618 with expected crc 2023-09-23T23:39:14.729 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:14.779 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:14.782 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:14 smithi138 ceph-mon[193606]: failed to encode map e618 with expected crc 2023-09-23T23:39:14.782 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:14 smithi138 ceph-mon[193606]: osdmap e618: 8 total, 8 up, 7 in 2023-09-23T23:39:14.782 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:14 smithi138 ceph-mon[193606]: failed to encode map e618 with expected crc 2023-09-23T23:39:14.783 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:14 smithi138 ceph-mon[193606]: failed to encode map e618 with expected crc 2023-09-23T23:39:14.783 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:14 smithi138 ceph-mon[193606]: failed to encode map e618 with expected crc 2023-09-23T23:39:14.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:14.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:14.884 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:39:14.912 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:14.958 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1758,"stamp":"2023-09-23T23:39:13.519810+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":636125,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":77,"num_write_kb":193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":1110440,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5046,"ondisk_log_size":5046,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":316,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6007284,"kb_used_data":105596,"kb_used_omap":29,"kb_used_meta":5901602,"kb_avail":650208780,"statfs":{"total":671965249536,"available":665813790720,"internally_reserved":0,"allocated":108130304,"data_stored":91230670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":6043241171},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.539510"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":812,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.842348+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:39:02.842348+0000","last_peered":"2023-09-23T23:39:02.842348+0000","last_clean":"2023-09-23T23:39:02.842348+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:39:02.842348+0000","last_undegraded":"2023-09-23T23:39:02.842348+0000","last_fullsized":"2023-09-23T23:39:02.842348+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":372,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:05.217242+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:39:05.217242+0000","last_peered":"2023-09-23T23:39:05.217242+0000","last_clean":"2023-09-23T23:39:05.217242+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:39:05.217242+0000","last_undegraded":"2023-09-23T23:39:05.217242+0000","last_fullsized":"2023-09-23T23:39:05.217242+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":600,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.001042+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:39:08.001042+0000","last_peered":"2023-09-23T23:39:08.001042+0000","last_clean":"2023-09-23T23:39:08.001042+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:39:08.001042+0000","last_undegraded":"2023-09-23T23:39:08.001042+0000","last_fullsized":"2023-09-23T23:39:08.001042+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":765,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.843811+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:39:02.843811+0000","last_peered":"2023-09-23T23:39:02.843811+0000","last_clean":"2023-09-23T23:39:02.843811+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:39:02.843811+0000","last_undegraded":"2023-09-23T23:39:02.843811+0000","last_fullsized":"2023-09-23T23:39:02.843811+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":337,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000275+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:39:08.000275+0000","last_peered":"2023-09-23T23:39:08.000275+0000","last_clean":"2023-09-23T23:39:08.000275+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:39:08.000275+0000","last_undegraded":"2023-09-23T23:39:08.000275+0000","last_fullsized":"2023-09-23T23:39:08.000275+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":579,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000487+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:39:08.000487+0000","last_peered":"2023-09-23T23:39:08.000487+0000","last_clean":"2023-09-23T23:39:08.000487+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:39:08.000487+0000","last_undegraded":"2023-09-23T23:39:08.000487+0000","last_fullsized":"2023-09-23T23:39:08.000487+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":778,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:07.500138+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:39:07.500138+0000","last_peered":"2023-09-23T23:39:07.500138+0000","last_clean":"2023-09-23T23:39:07.500138+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:39:07.500138+0000","last_undegraded":"2023-09-23T23:39:07.500138+0000","last_fullsized":"2023-09-23T23:39:07.500138+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":636,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000272+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:39:08.000272+0000","last_peered":"2023-09-23T23:39:08.000272+0000","last_clean":"2023-09-23T23:39:08.000272+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:39:08.000272+0000","last_undegraded":"2023-09-23T23:39:08.000272+0000","last_fullsized":"2023-09-23T23:39:08.000272+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":130,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.844688+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:39:02.844688+0000","last_peered":"2023-09-23T23:39:02.844688+0000","last_clean":"2023-09-23T23:39:02.844688+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:39:02.844688+0000","last_undegraded":"2023-09-23T23:39:02.844688+0000","last_fullsized":"2023-09-23T23:39:02.844688+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":937,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:07.500618+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:39:07.500618+0000","last_peered":"2023-09-23T23:39:07.500618+0000","last_clean":"2023-09-23T23:39:07.500618+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:39:07.500618+0000","last_undegraded":"2023-09-23T23:39:07.500618+0000","last_fullsized":"2023-09-23T23:39:07.500618+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":268,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.842532+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:39:02.842532+0000","last_peered":"2023-09-23T23:39:02.842532+0000","last_clean":"2023-09-23T23:39:02.842532+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:39:02.842532+0000","last_undegraded":"2023-09-23T23:39:02.842532+0000","last_fullsized":"2023-09-23T23:39:02.842532+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":619,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000649+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:39:08.000649+0000","last_peered":"2023-09-23T23:39:08.000649+0000","last_clean":"2023-09-23T23:39:08.000649+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:39:08.000649+0000","last_undegraded":"2023-09-23T23:39:08.000649+0000","last_fullsized":"2023-09-23T23:39:08.000649+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":910,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.491648+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:39:02.491648+0000","last_peered":"2023-09-23T23:39:02.491648+0000","last_clean":"2023-09-23T23:39:02.491648+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:39:02.491648+0000","last_undegraded":"2023-09-23T23:39:02.491648+0000","last_fullsized":"2023-09-23T23:39:02.491648+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":247,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.844086+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:39:02.844086+0000","last_peered":"2023-09-23T23:39:02.844086+0000","last_clean":"2023-09-23T23:39:02.844086+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:39:02.844086+0000","last_undegraded":"2023-09-23T23:39:02.844086+0000","last_fullsized":"2023-09-23T23:39:02.844086+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":580,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000803+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:39:08.000803+0000","last_peered":"2023-09-23T23:39:08.000803+0000","last_clean":"2023-09-23T23:39:08.000803+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:39:08.000803+0000","last_undegraded":"2023-09-23T23:39:08.000803+0000","last_fullsized":"2023-09-23T23:39:08.000803+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1070,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.842363+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:39:02.842363+0000","last_peered":"2023-09-23T23:39:02.842363+0000","last_clean":"2023-09-23T23:39:02.842363+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:39:02.842363+0000","last_undegraded":"2023-09-23T23:39:02.842363+0000","last_fullsized":"2023-09-23T23:39:02.842363+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":291,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.461204+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:39:08.461204+0000","last_peered":"2023-09-23T23:39:08.461204+0000","last_clean":"2023-09-23T23:39:08.461204+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:39:08.461204+0000","last_undegraded":"2023-09-23T23:39:08.461204+0000","last_fullsized":"2023-09-23T23:39:08.461204+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":149,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.839356+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:39:02.839356+0000","last_peered":"2023-09-23T23:39:02.839356+0000","last_clean":"2023-09-23T23:39:02.839356+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:39:02.839356+0000","last_undegraded":"2023-09-23T23:39:02.839356+0000","last_fullsized":"2023-09-23T23:39:02.839356+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":469,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000417+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:39:08.000417+0000","last_peered":"2023-09-23T23:39:08.000417+0000","last_clean":"2023-09-23T23:39:08.000417+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:39:08.000417+0000","last_undegraded":"2023-09-23T23:39:08.000417+0000","last_fullsized":"2023-09-23T23:39:08.000417+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":190,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.837570+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:39:02.837570+0000","last_peered":"2023-09-23T23:39:02.837570+0000","last_clean":"2023-09-23T23:39:02.837570+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:39:02.837570+0000","last_undegraded":"2023-09-23T23:39:02.837570+0000","last_fullsized":"2023-09-23T23:39:02.837570+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":557,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.837406+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:39:02.837406+0000","last_peered":"2023-09-23T23:39:02.837406+0000","last_clean":"2023-09-23T23:39:02.837406+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:39:02.837406+0000","last_undegraded":"2023-09-23T23:39:02.837406+0000","last_fullsized":"2023-09-23T23:39:02.837406+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":771,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.837235+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:39:02.837235+0000","last_peered":"2023-09-23T23:39:02.837235+0000","last_clean":"2023-09-23T23:39:02.837235+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:39:02.837235+0000","last_undegraded":"2023-09-23T23:39:02.837235+0000","last_fullsized":"2023-09-23T23:39:02.837235+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":164,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.489571+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:39:02.489571+0000","last_peered":"2023-09-23T23:39:02.489571+0000","last_clean":"2023-09-23T23:39:02.489571+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:39:02.489571+0000","last_undegraded":"2023-09-23T23:39:02.489571+0000","last_fullsized":"2023-09-23T23:39:02.489571+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":618,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000926+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:39:08.000926+0000","last_peered":"2023-09-23T23:39:08.000926+0000","last_clean":"2023-09-23T23:39:08.000926+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:39:08.000926+0000","last_undegraded":"2023-09-23T23:39:08.000926+0000","last_fullsized":"2023-09-23T23:39:08.000926+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":901,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.838836+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:39:02.838836+0000","last_peered":"2023-09-23T23:39:02.838836+0000","last_clean":"2023-09-23T23:39:02.838836+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:39:02.838836+0000","last_undegraded":"2023-09-23T23:39:02.838836+0000","last_fullsized":"2023-09-23T23:39:02.838836+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":213,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000296+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:39:08.000296+0000","last_peered":"2023-09-23T23:39:08.000296+0000","last_clean":"2023-09-23T23:39:08.000296+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:39:08.000296+0000","last_undegraded":"2023-09-23T23:39:08.000296+0000","last_fullsized":"2023-09-23T23:39:08.000296+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":340,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.461223+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:39:08.461223+0000","last_peered":"2023-09-23T23:39:08.461223+0000","last_clean":"2023-09-23T23:39:08.461223+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:39:08.461223+0000","last_undegraded":"2023-09-23T23:39:08.461223+0000","last_fullsized":"2023-09-23T23:39:08.461223+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":581,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000318+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:39:08.000318+0000","last_peered":"2023-09-23T23:39:08.000318+0000","last_clean":"2023-09-23T23:39:08.000318+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:39:08.000318+0000","last_undegraded":"2023-09-23T23:39:08.000318+0000","last_fullsized":"2023-09-23T23:39:08.000318+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":484,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.836801+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:39:02.836801+0000","last_peered":"2023-09-23T23:39:02.836801+0000","last_clean":"2023-09-23T23:39:02.836801+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:39:02.836801+0000","last_undegraded":"2023-09-23T23:39:02.836801+0000","last_fullsized":"2023-09-23T23:39:02.836801+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":574,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000387+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:39:08.000387+0000","last_peered":"2023-09-23T23:39:08.000387+0000","last_clean":"2023-09-23T23:39:08.000387+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:39:08.000387+0000","last_undegraded":"2023-09-23T23:39:08.000387+0000","last_fullsized":"2023-09-23T23:39:08.000387+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":979,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.837950+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:39:02.837950+0000","last_peered":"2023-09-23T23:39:02.837950+0000","last_clean":"2023-09-23T23:39:02.837950+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:39:02.837950+0000","last_undegraded":"2023-09-23T23:39:02.837950+0000","last_fullsized":"2023-09-23T23:39:02.837950+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":180,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.839179+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:39:02.839179+0000","last_peered":"2023-09-23T23:39:02.839179+0000","last_clean":"2023-09-23T23:39:02.839179+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:39:02.839179+0000","last_undegraded":"2023-09-23T23:39:02.839179+0000","last_fullsized":"2023-09-23T23:39:02.839179+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":259,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000960+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:39:08.000960+0000","last_peered":"2023-09-23T23:39:08.000960+0000","last_clean":"2023-09-23T23:39:08.000960+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:39:08.000960+0000","last_undegraded":"2023-09-23T23:39:08.000960+0000","last_fullsized":"2023-09-23T23:39:08.000960+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:39:12.504732+0000","last_change":"2023-09-23T23:39:12.504732+0000","last_active":"2023-09-23T23:39:12.504732+0000","last_peered":"2023-09-23T23:39:12.504732+0000","last_clean":"2023-09-23T23:39:12.504732+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:39:12.504732+0000","last_undegraded":"2023-09-23T23:39:12.504732+0000","last_fullsized":"2023-09-23T23:39:12.504732+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":951,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.491885+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:39:02.491885+0000","last_peered":"2023-09-23T23:39:02.491885+0000","last_clean":"2023-09-23T23:39:02.491885+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:39:02.491885+0000","last_undegraded":"2023-09-23T23:39:02.491885+0000","last_fullsized":"2023-09-23T23:39:02.491885+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":258,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:07.500489+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:39:07.500489+0000","last_peered":"2023-09-23T23:39:07.500489+0000","last_clean":"2023-09-23T23:39:07.500489+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:39:07.500489+0000","last_undegraded":"2023-09-23T23:39:07.500489+0000","last_fullsized":"2023-09-23T23:39:07.500489+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":247,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.842268+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:39:02.842268+0000","last_peered":"2023-09-23T23:39:02.842268+0000","last_clean":"2023-09-23T23:39:02.842268+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:39:02.842268+0000","last_undegraded":"2023-09-23T23:39:02.842268+0000","last_fullsized":"2023-09-23T23:39:02.842268+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"0'0","reported_seq":1,"reported_epoch":617,"state":"creating+peering","last_fresh":"2023-09-23T23:39:12.523882+0000","last_change":"2023-09-23T23:39:12.523882+0000","last_active":"2023-09-23T23:39:12.504732+0000","last_peered":"2023-09-23T23:39:12.504732+0000","last_clean":"2023-09-23T23:39:12.504732+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:39:12.523882+0000","last_undegraded":"2023-09-23T23:39:12.523882+0000","last_fullsized":"2023-09-23T23:39:12.523882+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:44.833933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1030,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000706+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:39:08.000706+0000","last_peered":"2023-09-23T23:39:08.000706+0000","last_clean":"2023-09-23T23:39:08.000706+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:39:08.000706+0000","last_undegraded":"2023-09-23T23:39:08.000706+0000","last_fullsized":"2023-09-23T23:39:08.000706+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":248,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:07.500518+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:39:07.500518+0000","last_peered":"2023-09-23T23:39:07.500518+0000","last_clean":"2023-09-23T23:39:07.500518+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:39:07.500518+0000","last_undegraded":"2023-09-23T23:39:07.500518+0000","last_fullsized":"2023-09-23T23:39:07.500518+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":469,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000303+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:39:08.000303+0000","last_peered":"2023-09-23T23:39:08.000303+0000","last_clean":"2023-09-23T23:39:08.000303+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:39:08.000303+0000","last_undegraded":"2023-09-23T23:39:08.000303+0000","last_fullsized":"2023-09-23T23:39:08.000303+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:39:12.504732+0000","last_change":"2023-09-23T23:39:12.504732+0000","last_active":"2023-09-23T23:39:12.504732+0000","last_peered":"2023-09-23T23:39:12.504732+0000","last_clean":"2023-09-23T23:39:12.504732+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:39:12.504732+0000","last_undegraded":"2023-09-23T23:39:12.504732+0000","last_fullsized":"2023-09-23T23:39:12.504732+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":926,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.844740+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:39:02.844740+0000","last_peered":"2023-09-23T23:39:02.844740+0000","last_clean":"2023-09-23T23:39:02.844740+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:39:02.844740+0000","last_undegraded":"2023-09-23T23:39:02.844740+0000","last_fullsized":"2023-09-23T23:39:02.844740+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":354,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.844073+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:39:02.844073+0000","last_peered":"2023-09-23T23:39:02.844073+0000","last_clean":"2023-09-23T23:39:02.844073+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:39:02.844073+0000","last_undegraded":"2023-09-23T23:39:02.844073+0000","last_fullsized":"2023-09-23T23:39:02.844073+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":130,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.839453+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:39:02.839453+0000","last_peered":"2023-09-23T23:39:02.839453+0000","last_clean":"2023-09-23T23:39:02.839453+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:39:02.839453+0000","last_undegraded":"2023-09-23T23:39:02.839453+0000","last_fullsized":"2023-09-23T23:39:02.839453+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"0'0","reported_seq":1,"reported_epoch":617,"state":"creating+peering","last_fresh":"2023-09-23T23:39:12.524406+0000","last_change":"2023-09-23T23:39:12.524406+0000","last_active":"2023-09-23T23:39:12.504732+0000","last_peered":"2023-09-23T23:39:12.504732+0000","last_clean":"2023-09-23T23:39:12.504732+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:39:12.524406+0000","last_undegraded":"2023-09-23T23:39:12.524406+0000","last_fullsized":"2023-09-23T23:39:12.524406+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:50.120382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":525,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.001196+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:39:08.001196+0000","last_peered":"2023-09-23T23:39:08.001196+0000","last_clean":"2023-09-23T23:39:08.001196+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:39:08.001196+0000","last_undegraded":"2023-09-23T23:39:08.001196+0000","last_fullsized":"2023-09-23T23:39:08.001196+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":458,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000796+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:39:08.000796+0000","last_peered":"2023-09-23T23:39:08.000796+0000","last_clean":"2023-09-23T23:39:08.000796+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:39:08.000796+0000","last_undegraded":"2023-09-23T23:39:08.000796+0000","last_fullsized":"2023-09-23T23:39:08.000796+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":614,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.838563+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:39:02.838563+0000","last_peered":"2023-09-23T23:39:02.838563+0000","last_clean":"2023-09-23T23:39:02.838563+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:39:02.838563+0000","last_undegraded":"2023-09-23T23:39:02.838563+0000","last_fullsized":"2023-09-23T23:39:02.838563+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:39:12.504732+0000","last_change":"2023-09-23T23:39:12.504732+0000","last_active":"2023-09-23T23:39:12.504732+0000","last_peered":"2023-09-23T23:39:12.504732+0000","last_clean":"2023-09-23T23:39:12.504732+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:39:12.504732+0000","last_undegraded":"2023-09-23T23:39:12.504732+0000","last_fullsized":"2023-09-23T23:39:12.504732+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":746,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000765+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:39:08.000765+0000","last_peered":"2023-09-23T23:39:08.000765+0000","last_clean":"2023-09-23T23:39:08.000765+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:39:08.000765+0000","last_undegraded":"2023-09-23T23:39:08.000765+0000","last_fullsized":"2023-09-23T23:39:08.000765+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":628,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000424+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:39:08.000424+0000","last_peered":"2023-09-23T23:39:08.000424+0000","last_clean":"2023-09-23T23:39:08.000424+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:39:08.000424+0000","last_undegraded":"2023-09-23T23:39:08.000424+0000","last_fullsized":"2023-09-23T23:39:08.000424+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":539,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000886+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:39:08.000886+0000","last_peered":"2023-09-23T23:39:08.000886+0000","last_clean":"2023-09-23T23:39:08.000886+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:39:08.000886+0000","last_undegraded":"2023-09-23T23:39:08.000886+0000","last_fullsized":"2023-09-23T23:39:08.000886+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:39:12.504732+0000","last_change":"2023-09-23T23:39:12.504732+0000","last_active":"2023-09-23T23:39:12.504732+0000","last_peered":"2023-09-23T23:39:12.504732+0000","last_clean":"2023-09-23T23:39:12.504732+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:39:12.504732+0000","last_undegraded":"2023-09-23T23:39:12.504732+0000","last_fullsized":"2023-09-23T23:39:12.504732+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":900,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.493577+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:39:02.493577+0000","last_peered":"2023-09-23T23:39:02.493577+0000","last_clean":"2023-09-23T23:39:02.493577+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:39:02.493577+0000","last_undegraded":"2023-09-23T23:39:02.493577+0000","last_fullsized":"2023-09-23T23:39:02.493577+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":140,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.839444+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:39:02.839444+0000","last_peered":"2023-09-23T23:39:02.839444+0000","last_clean":"2023-09-23T23:39:02.839444+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:39:02.839444+0000","last_undegraded":"2023-09-23T23:39:02.839444+0000","last_fullsized":"2023-09-23T23:39:02.839444+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":483,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.491988+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:39:02.491988+0000","last_peered":"2023-09-23T23:39:02.491988+0000","last_clean":"2023-09-23T23:39:02.491988+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:39:02.491988+0000","last_undegraded":"2023-09-23T23:39:02.491988+0000","last_fullsized":"2023-09-23T23:39:02.491988+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:39:12.504732+0000","last_change":"2023-09-23T23:39:12.504732+0000","last_active":"2023-09-23T23:39:12.504732+0000","last_peered":"2023-09-23T23:39:12.504732+0000","last_clean":"2023-09-23T23:39:12.504732+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:39:12.504732+0000","last_undegraded":"2023-09-23T23:39:12.504732+0000","last_fullsized":"2023-09-23T23:39:12.504732+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":624,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.843713+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:39:02.843713+0000","last_peered":"2023-09-23T23:39:02.843713+0000","last_clean":"2023-09-23T23:39:02.843713+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:39:02.843713+0000","last_undegraded":"2023-09-23T23:39:02.843713+0000","last_fullsized":"2023-09-23T23:39:02.843713+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":480,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.489204+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:39:02.489204+0000","last_peered":"2023-09-23T23:39:02.489204+0000","last_clean":"2023-09-23T23:39:02.489204+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:39:02.489204+0000","last_undegraded":"2023-09-23T23:39:02.489204+0000","last_fullsized":"2023-09-23T23:39:02.489204+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":567,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.836697+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:39:02.836697+0000","last_peered":"2023-09-23T23:39:02.836697+0000","last_clean":"2023-09-23T23:39:02.836697+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:39:02.836697+0000","last_undegraded":"2023-09-23T23:39:02.836697+0000","last_fullsized":"2023-09-23T23:39:02.836697+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"0'0","reported_seq":1,"reported_epoch":617,"state":"creating+peering","last_fresh":"2023-09-23T23:39:12.523925+0000","last_change":"2023-09-23T23:39:12.523925+0000","last_active":"2023-09-23T23:39:12.504732+0000","last_peered":"2023-09-23T23:39:12.504732+0000","last_clean":"2023-09-23T23:39:12.504732+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:39:12.523925+0000","last_undegraded":"2023-09-23T23:39:12.523925+0000","last_fullsized":"2023-09-23T23:39:12.523925+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:51.872071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":959,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.490724+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:39:02.490724+0000","last_peered":"2023-09-23T23:39:02.490724+0000","last_clean":"2023-09-23T23:39:02.490724+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:39:02.490724+0000","last_undegraded":"2023-09-23T23:39:02.490724+0000","last_fullsized":"2023-09-23T23:39:02.490724+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":260,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.460925+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:39:08.460925+0000","last_peered":"2023-09-23T23:39:08.460925+0000","last_clean":"2023-09-23T23:39:08.460925+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:39:08.460925+0000","last_undegraded":"2023-09-23T23:39:08.460925+0000","last_fullsized":"2023-09-23T23:39:08.460925+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":548,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000403+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:39:08.000403+0000","last_peered":"2023-09-23T23:39:08.000403+0000","last_clean":"2023-09-23T23:39:08.000403+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:39:08.000403+0000","last_undegraded":"2023-09-23T23:39:08.000403+0000","last_fullsized":"2023-09-23T23:39:08.000403+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:39:12.504732+0000","last_change":"2023-09-23T23:39:12.504732+0000","last_active":"2023-09-23T23:39:12.504732+0000","last_peered":"2023-09-23T23:39:12.504732+0000","last_clean":"2023-09-23T23:39:12.504732+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:39:12.504732+0000","last_undegraded":"2023-09-23T23:39:12.504732+0000","last_fullsized":"2023-09-23T23:39:12.504732+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":937,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.840752+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:39:02.840752+0000","last_peered":"2023-09-23T23:39:02.840752+0000","last_clean":"2023-09-23T23:39:02.840752+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:39:02.840752+0000","last_undegraded":"2023-09-23T23:39:02.840752+0000","last_fullsized":"2023-09-23T23:39:02.840752+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":327,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:07.766520+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:39:07.766520+0000","last_peered":"2023-09-23T23:39:07.766520+0000","last_clean":"2023-09-23T23:39:07.766520+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:39:07.766520+0000","last_undegraded":"2023-09-23T23:39:07.766520+0000","last_fullsized":"2023-09-23T23:39:07.766520+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":631,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.844919+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:39:02.844919+0000","last_peered":"2023-09-23T23:39:02.844919+0000","last_clean":"2023-09-23T23:39:02.844919+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:39:02.844919+0000","last_undegraded":"2023-09-23T23:39:02.844919+0000","last_fullsized":"2023-09-23T23:39:02.844919+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":212,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.837805+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:39:02.837805+0000","last_peered":"2023-09-23T23:39:02.837805+0000","last_clean":"2023-09-23T23:39:02.837805+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:39:02.837805+0000","last_undegraded":"2023-09-23T23:39:02.837805+0000","last_fullsized":"2023-09-23T23:39:02.837805+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:39:12.504732+0000","last_change":"2023-09-23T23:39:12.504732+0000","last_active":"2023-09-23T23:39:12.504732+0000","last_peered":"2023-09-23T23:39:12.504732+0000","last_clean":"2023-09-23T23:39:12.504732+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:39:12.504732+0000","last_undegraded":"2023-09-23T23:39:12.504732+0000","last_fullsized":"2023-09-23T23:39:12.504732+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":701,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.841839+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:39:02.841839+0000","last_peered":"2023-09-23T23:39:02.841839+0000","last_clean":"2023-09-23T23:39:02.841839+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:39:02.841839+0000","last_undegraded":"2023-09-23T23:39:02.841839+0000","last_fullsized":"2023-09-23T23:39:02.841839+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":264,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.839382+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:39:02.839382+0000","last_peered":"2023-09-23T23:39:02.839382+0000","last_clean":"2023-09-23T23:39:02.839382+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:39:02.839382+0000","last_undegraded":"2023-09-23T23:39:02.839382+0000","last_fullsized":"2023-09-23T23:39:02.839382+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":548,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.840166+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:39:02.840166+0000","last_peered":"2023-09-23T23:39:02.840166+0000","last_clean":"2023-09-23T23:39:02.840166+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:39:02.840166+0000","last_undegraded":"2023-09-23T23:39:02.840166+0000","last_fullsized":"2023-09-23T23:39:02.840166+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":250,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.491703+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:39:02.491703+0000","last_peered":"2023-09-23T23:39:02.491703+0000","last_clean":"2023-09-23T23:39:02.491703+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:39:02.491703+0000","last_undegraded":"2023-09-23T23:39:02.491703+0000","last_fullsized":"2023-09-23T23:39:02.491703+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:39:12.504732+0000","last_change":"2023-09-23T23:39:12.504732+0000","last_active":"2023-09-23T23:39:12.504732+0000","last_peered":"2023-09-23T23:39:12.504732+0000","last_clean":"2023-09-23T23:39:12.504732+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:39:12.504732+0000","last_undegraded":"2023-09-23T23:39:12.504732+0000","last_fullsized":"2023-09-23T23:39:12.504732+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":918,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.490632+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:39:02.490632+0000","last_peered":"2023-09-23T23:39:02.490632+0000","last_clean":"2023-09-23T23:39:02.490632+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:39:02.490632+0000","last_undegraded":"2023-09-23T23:39:02.490632+0000","last_fullsized":"2023-09-23T23:39:02.490632+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":356,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:05.217172+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:39:05.217172+0000","last_peered":"2023-09-23T23:39:05.217172+0000","last_clean":"2023-09-23T23:39:05.217172+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:39:05.217172+0000","last_undegraded":"2023-09-23T23:39:05.217172+0000","last_fullsized":"2023-09-23T23:39:05.217172+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":265,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.493661+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:39:02.493661+0000","last_peered":"2023-09-23T23:39:02.493661+0000","last_clean":"2023-09-23T23:39:02.493661+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:39:02.493661+0000","last_undegraded":"2023-09-23T23:39:02.493661+0000","last_fullsized":"2023-09-23T23:39:02.493661+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":230,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:07.766681+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:39:07.766681+0000","last_peered":"2023-09-23T23:39:07.766681+0000","last_clean":"2023-09-23T23:39:07.766681+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:39:07.766681+0000","last_undegraded":"2023-09-23T23:39:07.766681+0000","last_fullsized":"2023-09-23T23:39:07.766681+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"0'0","reported_seq":1,"reported_epoch":617,"state":"creating+peering","last_fresh":"2023-09-23T23:39:12.523963+0000","last_change":"2023-09-23T23:39:12.523963+0000","last_active":"2023-09-23T23:39:12.504732+0000","last_peered":"2023-09-23T23:39:12.504732+0000","last_clean":"2023-09-23T23:39:12.504732+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:39:12.523963+0000","last_undegraded":"2023-09-23T23:39:12.523963+0000","last_fullsized":"2023-09-23T23:39:12.523963+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:34.450587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":632,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.489640+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:39:02.489640+0000","last_peered":"2023-09-23T23:39:02.489640+0000","last_clean":"2023-09-23T23:39:02.489640+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:39:02.489640+0000","last_undegraded":"2023-09-23T23:39:02.489640+0000","last_fullsized":"2023-09-23T23:39:02.489640+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":902,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.838909+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:39:02.838909+0000","last_peered":"2023-09-23T23:39:02.838909+0000","last_clean":"2023-09-23T23:39:02.838909+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:39:02.838909+0000","last_undegraded":"2023-09-23T23:39:02.838909+0000","last_fullsized":"2023-09-23T23:39:02.838909+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":580,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000601+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:39:08.000601+0000","last_peered":"2023-09-23T23:39:08.000601+0000","last_clean":"2023-09-23T23:39:08.000601+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:39:08.000601+0000","last_undegraded":"2023-09-23T23:39:08.000601+0000","last_fullsized":"2023-09-23T23:39:08.000601+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":140,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.837499+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:39:02.837499+0000","last_peered":"2023-09-23T23:39:02.837499+0000","last_clean":"2023-09-23T23:39:02.837499+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:39:02.837499+0000","last_undegraded":"2023-09-23T23:39:02.837499+0000","last_fullsized":"2023-09-23T23:39:02.837499+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:39:12.504732+0000","last_change":"2023-09-23T23:39:12.504732+0000","last_active":"2023-09-23T23:39:12.504732+0000","last_peered":"2023-09-23T23:39:12.504732+0000","last_clean":"2023-09-23T23:39:12.504732+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:39:12.504732+0000","last_undegraded":"2023-09-23T23:39:12.504732+0000","last_fullsized":"2023-09-23T23:39:12.504732+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":728,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.837322+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:39:02.837322+0000","last_peered":"2023-09-23T23:39:02.837322+0000","last_clean":"2023-09-23T23:39:02.837322+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:39:02.837322+0000","last_undegraded":"2023-09-23T23:39:02.837322+0000","last_fullsized":"2023-09-23T23:39:02.837322+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":544,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:07.499996+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:39:07.499996+0000","last_peered":"2023-09-23T23:39:07.499996+0000","last_clean":"2023-09-23T23:39:07.499996+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:39:07.499996+0000","last_undegraded":"2023-09-23T23:39:07.499996+0000","last_fullsized":"2023-09-23T23:39:07.499996+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":459,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000616+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:39:08.000616+0000","last_peered":"2023-09-23T23:39:08.000616+0000","last_clean":"2023-09-23T23:39:08.000616+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:39:08.000616+0000","last_undegraded":"2023-09-23T23:39:08.000616+0000","last_fullsized":"2023-09-23T23:39:08.000616+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":58,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.839068+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:39:02.839068+0000","last_peered":"2023-09-23T23:39:02.839068+0000","last_clean":"2023-09-23T23:39:02.839068+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:39:02.839068+0000","last_undegraded":"2023-09-23T23:39:02.839068+0000","last_fullsized":"2023-09-23T23:39:02.839068+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"0'0","reported_seq":1,"reported_epoch":617,"state":"creating+peering","last_fresh":"2023-09-23T23:39:12.523910+0000","last_change":"2023-09-23T23:39:12.523910+0000","last_active":"2023-09-23T23:39:12.504732+0000","last_peered":"2023-09-23T23:39:12.504732+0000","last_clean":"2023-09-23T23:39:12.504732+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:39:12.523910+0000","last_undegraded":"2023-09-23T23:39:12.523910+0000","last_fullsized":"2023-09-23T23:39:12.523910+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:15.737461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":611,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.840588+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:39:02.840588+0000","last_peered":"2023-09-23T23:39:02.840588+0000","last_clean":"2023-09-23T23:39:02.840588+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:39:02.840588+0000","last_undegraded":"2023-09-23T23:39:02.840588+0000","last_fullsized":"2023-09-23T23:39:02.840588+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":58,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.841695+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:39:02.841695+0000","last_peered":"2023-09-23T23:39:02.841695+0000","last_clean":"2023-09-23T23:39:02.841695+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:39:02.841695+0000","last_undegraded":"2023-09-23T23:39:02.841695+0000","last_fullsized":"2023-09-23T23:39:02.841695+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":157,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.491753+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:39:02.491753+0000","last_peered":"2023-09-23T23:39:02.491753+0000","last_clean":"2023-09-23T23:39:02.491753+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:39:02.491753+0000","last_undegraded":"2023-09-23T23:39:02.491753+0000","last_fullsized":"2023-09-23T23:39:02.491753+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":349,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.843947+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:39:02.843947+0000","last_peered":"2023-09-23T23:39:02.843947+0000","last_clean":"2023-09-23T23:39:02.843947+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:39:02.843947+0000","last_undegraded":"2023-09-23T23:39:02.843947+0000","last_fullsized":"2023-09-23T23:39:02.843947+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.a","version":"0'0","reported_seq":1,"reported_epoch":617,"state":"creating+peering","last_fresh":"2023-09-23T23:39:12.524107+0000","last_change":"2023-09-23T23:39:12.524107+0000","last_active":"2023-09-23T23:39:12.504732+0000","last_peered":"2023-09-23T23:39:12.504732+0000","last_clean":"2023-09-23T23:39:12.504732+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:39:12.524107+0000","last_undegraded":"2023-09-23T23:39:12.524107+0000","last_fullsized":"2023-09-23T23:39:12.524107+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:39.662209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":798,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.492033+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:39:02.492033+0000","last_peered":"2023-09-23T23:39:02.492033+0000","last_clean":"2023-09-23T23:39:02.492033+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:39:02.492033+0000","last_undegraded":"2023-09-23T23:39:02.492033+0000","last_fullsized":"2023-09-23T23:39:02.492033+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":616,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T23:39:12.525957+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:39:12.525957+0000","last_peered":"2023-09-23T23:39:12.525957+0000","last_clean":"2023-09-23T23:39:12.525957+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:39:12.525957+0000","last_undegraded":"2023-09-23T23:39:12.525957+0000","last_fullsized":"2023-09-23T23:39:12.525957+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":220,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.840414+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:39:02.840414+0000","last_peered":"2023-09-23T23:39:02.840414+0000","last_clean":"2023-09-23T23:39:02.840414+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:39:02.840414+0000","last_undegraded":"2023-09-23T23:39:02.840414+0000","last_fullsized":"2023-09-23T23:39:02.840414+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":169,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.493420+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:39:02.493420+0000","last_peered":"2023-09-23T23:39:02.493420+0000","last_clean":"2023-09-23T23:39:02.493420+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:39:02.493420+0000","last_undegraded":"2023-09-23T23:39:02.493420+0000","last_fullsized":"2023-09-23T23:39:02.493420+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:39:12.504732+0000","last_change":"2023-09-23T23:39:12.504732+0000","last_active":"2023-09-23T23:39:12.504732+0000","last_peered":"2023-09-23T23:39:12.504732+0000","last_clean":"2023-09-23T23:39:12.504732+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:39:12.504732+0000","last_undegraded":"2023-09-23T23:39:12.504732+0000","last_fullsized":"2023-09-23T23:39:12.504732+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":1005,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000317+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:39:08.000317+0000","last_peered":"2023-09-23T23:39:08.000317+0000","last_clean":"2023-09-23T23:39:08.000317+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:39:08.000317+0000","last_undegraded":"2023-09-23T23:39:08.000317+0000","last_fullsized":"2023-09-23T23:39:08.000317+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":130,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.840852+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:39:02.840852+0000","last_peered":"2023-09-23T23:39:02.840852+0000","last_clean":"2023-09-23T23:39:02.840852+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:39:02.840852+0000","last_undegraded":"2023-09-23T23:39:02.840852+0000","last_fullsized":"2023-09-23T23:39:02.840852+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":62,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.839136+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:39:02.839136+0000","last_peered":"2023-09-23T23:39:02.839136+0000","last_clean":"2023-09-23T23:39:02.839136+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:39:02.839136+0000","last_undegraded":"2023-09-23T23:39:02.839136+0000","last_fullsized":"2023-09-23T23:39:02.839136+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":190,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.837341+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:39:02.837341+0000","last_peered":"2023-09-23T23:39:02.837341+0000","last_clean":"2023-09-23T23:39:02.837341+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:39:02.837341+0000","last_undegraded":"2023-09-23T23:39:02.837341+0000","last_fullsized":"2023-09-23T23:39:02.837341+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1054,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000711+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:39:08.000711+0000","last_peered":"2023-09-23T23:39:08.000711+0000","last_clean":"2023-09-23T23:39:08.000711+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:39:08.000711+0000","last_undegraded":"2023-09-23T23:39:08.000711+0000","last_fullsized":"2023-09-23T23:39:08.000711+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":57,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.838638+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:39:02.838638+0000","last_peered":"2023-09-23T23:39:02.838638+0000","last_clean":"2023-09-23T23:39:02.838638+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:39:02.838638+0000","last_undegraded":"2023-09-23T23:39:02.838638+0000","last_fullsized":"2023-09-23T23:39:02.838638+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":372,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.491794+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:39:02.491794+0000","last_peered":"2023-09-23T23:39:02.491794+0000","last_clean":"2023-09-23T23:39:02.491794+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:39:02.491794+0000","last_undegraded":"2023-09-23T23:39:02.491794+0000","last_fullsized":"2023-09-23T23:39:02.491794+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":766,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.001277+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:39:08.001277+0000","last_peered":"2023-09-23T23:39:08.001277+0000","last_clean":"2023-09-23T23:39:08.001277+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:39:08.001277+0000","last_undegraded":"2023-09-23T23:39:08.001277+0000","last_fullsized":"2023-09-23T23:39:08.001277+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":118,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.837520+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:39:02.837520+0000","last_peered":"2023-09-23T23:39:02.837520+0000","last_clean":"2023-09-23T23:39:02.837520+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:39:02.837520+0000","last_undegraded":"2023-09-23T23:39:02.837520+0000","last_fullsized":"2023-09-23T23:39:02.837520+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":466,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000760+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:39:08.000760+0000","last_peered":"2023-09-23T23:39:08.000760+0000","last_clean":"2023-09-23T23:39:08.000760+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:39:08.000760+0000","last_undegraded":"2023-09-23T23:39:08.000760+0000","last_fullsized":"2023-09-23T23:39:08.000760+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":314,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.837398+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:39:02.837398+0000","last_peered":"2023-09-23T23:39:02.837398+0000","last_clean":"2023-09-23T23:39:02.837398+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:39:02.837398+0000","last_undegraded":"2023-09-23T23:39:02.837398+0000","last_fullsized":"2023-09-23T23:39:02.837398+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":305,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000681+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:39:08.000681+0000","last_peered":"2023-09-23T23:39:08.000681+0000","last_clean":"2023-09-23T23:39:08.000681+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:39:08.000681+0000","last_undegraded":"2023-09-23T23:39:08.000681+0000","last_fullsized":"2023-09-23T23:39:08.000681+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":828,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.837613+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:39:02.837613+0000","last_peered":"2023-09-23T23:39:02.837613+0000","last_clean":"2023-09-23T23:39:02.837613+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:39:02.837613+0000","last_undegraded":"2023-09-23T23:39:02.837613+0000","last_fullsized":"2023-09-23T23:39:02.837613+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":649,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.001241+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:39:08.001241+0000","last_peered":"2023-09-23T23:39:08.001241+0000","last_clean":"2023-09-23T23:39:08.001241+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:39:08.001241+0000","last_undegraded":"2023-09-23T23:39:08.001241+0000","last_fullsized":"2023-09-23T23:39:08.001241+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":632,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.000879+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:39:08.000879+0000","last_peered":"2023-09-23T23:39:08.000879+0000","last_clean":"2023-09-23T23:39:08.000879+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:39:08.000879+0000","last_undegraded":"2023-09-23T23:39:08.000879+0000","last_fullsized":"2023-09-23T23:39:08.000879+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":791,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T23:39:08.461181+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:39:08.461181+0000","last_peered":"2023-09-23T23:39:08.461181+0000","last_clean":"2023-09-23T23:39:08.461181+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:39:08.461181+0000","last_undegraded":"2023-09-23T23:39:08.461181+0000","last_fullsized":"2023-09-23T23:39:08.461181+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":461,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.490486+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:39:02.490486+0000","last_peered":"2023-09-23T23:39:02.490486+0000","last_clean":"2023-09-23T23:39:02.490486+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:39:02.490486+0000","last_undegraded":"2023-09-23T23:39:02.490486+0000","last_fullsized":"2023-09-23T23:39:02.490486+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":683,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T23:39:02.838762+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:39:02.838762+0000","last_peered":"2023-09-23T23:39:02.838762+0000","last_clean":"2023-09-23T23:39:02.838762+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:39:02.838762+0000","last_undegraded":"2023-09-23T23:39:02.838762+0000","last_fullsized":"2023-09-23T23:39:02.838762+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":18,"acting":18,"num_store_stats":1},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782717,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":429288,"kb_used_data":6688,"kb_used_omap":8,"kb_used_meta":422583,"kb_avail":93315864,"statfs":{"total":95995035648,"available":95555444736,"internally_reserved":0,"allocated":6848512,"data_stored":5120275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":432725061},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:39:02 2023","interfaces":[{"interface":"back","average":{"1min":1.893,"5min":1.03,"15min":2.073},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":12.432,"5min":12.432,"15min":85.721000000000004},"last":0.495},{"interface":"front","average":{"1min":1.8859999999999999,"5min":0.90000000000000002,"15min":2.198},"min":{"1min":0.22900000000000001,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":12.717000000000001,"5min":12.717000000000001,"15min":85.965000000000003},"last":0.56499999999999995}]},{"osd":1,"last update":"Sat Sep 23 23:39:02 2023","interfaces":[{"interface":"back","average":{"1min":1.6759999999999999,"5min":1.044,"15min":3.1469999999999998},"min":{"1min":0.27000000000000002,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":12.484,"5min":12.484,"15min":198.58000000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":2.71,"5min":1.1379999999999999,"15min":3.1699999999999999},"min":{"1min":0.246,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":13.145,"5min":13.145,"15min":198.71100000000001},"last":0.46800000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:39:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5870000000000002,"5min":2.6739999999999999,"15min":3.7829999999999999},"min":{"1min":0.314,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":12.795999999999999,"5min":78.528999999999996,"15min":203.58000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":2.0630000000000002,"5min":1.839,"15min":3.278},"min":{"1min":0.30599999999999999,"5min":0.27900000000000003,"15min":0.26200000000000001},"max":{"1min":9.3010000000000002,"5min":53.023000000000003,"15min":198.69},"last":0.53500000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:39:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1299999999999999,"5min":5.5960000000000001,"15min":5.0110000000000001},"min":{"1min":0.309,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":9.0660000000000007,"5min":154.18100000000001,"15min":279.26499999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":2.165,"5min":5.7789999999999999,"15min":5.5339999999999998},"min":{"1min":0.35699999999999998,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":12.551,"5min":154.16300000000001,"15min":279.29899999999998},"last":0.54500000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:39:02 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":0.95199999999999996,"15min":1.048},"min":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.246},"max":{"1min":12.510999999999999,"5min":12.510999999999999,"15min":31.498999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.7230000000000001,"5min":0.89600000000000002,"15min":1.026},"min":{"1min":0.31900000000000001,"5min":0.317,"15min":0.23899999999999999},"max":{"1min":12.821999999999999,"5min":12.821999999999999,"15min":31.68},"last":0.372}]},{"osd":5,"last update":"Sat Sep 23 23:39:02 2023","interfaces":[{"interface":"back","average":{"1min":1.798,"5min":2.8969999999999998,"15min":2.3140000000000001},"min":{"1min":0.28899999999999998,"5min":0.27900000000000003,"15min":0.26500000000000001},"max":{"1min":13.849,"5min":119.417,"15min":210.83600000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.7629999999999999,"5min":3.129,"15min":2.2629999999999999},"min":{"1min":0.29999999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.8040000000000003,"5min":119.61499999999999,"15min":207.04599999999999},"last":0.51200000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:39:02 2023","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":1.0700000000000001,"15min":1.4019999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.29899999999999999},"max":{"1min":5.3799999999999999,"5min":8.6440000000000001,"15min":94.852999999999994},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.522,"5min":1.097,"15min":1.5429999999999999},"min":{"1min":0.34899999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":12.529999999999999,"5min":12.529999999999999,"15min":95.510999999999996},"last":0.57999999999999996}]}]},{"osd":6,"up_from":46,"seq":197568496193,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":791640,"kb_used_data":60304,"kb_used_omap":0,"kb_used_meta":731328,"kb_avail":92953512,"statfs":{"total":95995035648,"available":95184396288,"internally_reserved":0,"allocated":61751296,"data_stored":60038522,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":748879872},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:26 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":0.90400000000000003,"15min":3.956},"min":{"1min":0.224,"5min":0.224,"15min":0.21099999999999999},"max":{"1min":6.5579999999999998,"5min":6.5579999999999998,"15min":135.31999999999999},"last":0.31900000000000001},{"interface":"front","average":{"1min":1.1850000000000001,"5min":0.70899999999999996,"15min":3.891},"min":{"1min":0.28599999999999998,"5min":0.24399999999999999,"15min":0.23000000000000001},"max":{"1min":6.8529999999999998,"5min":6.8529999999999998,"15min":135.34299999999999},"last":0.54900000000000004}]},{"osd":1,"last update":"Sat Sep 23 23:38:26 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":1.018,"15min":2.1720000000000002},"min":{"1min":0.28499999999999998,"5min":0.253,"15min":0.245},"max":{"1min":6.6200000000000001,"5min":6.9080000000000004,"15min":89.605000000000004},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.4990000000000001,"5min":0.873,"15min":2.2749999999999999},"min":{"1min":0.26300000000000001,"5min":0.249,"15min":0.23300000000000001},"max":{"1min":12.577,"5min":12.577,"15min":91.025999999999996},"last":0.46500000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:38:10 2023","interfaces":[{"interface":"back","average":{"1min":7.9569999999999999,"5min":5.032,"15min":4.9539999999999997},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":85.995999999999995,"5min":85.995999999999995,"15min":127.44499999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":7.6959999999999997,"5min":5.0250000000000004,"15min":5.3319999999999999},"min":{"1min":0.32300000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":85.266999999999996,"5min":85.266999999999996,"15min":125.539},"last":0.40400000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:39:00 2023","interfaces":[{"interface":"back","average":{"1min":3.109,"5min":6.8689999999999998,"15min":5.3520000000000003},"min":{"1min":0.35599999999999998,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":12.166,"5min":199.536,"15min":199.536},"last":0.73399999999999999},{"interface":"front","average":{"1min":1.998,"5min":6.4969999999999999,"15min":5.5229999999999997},"min":{"1min":0.29999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":10.826000000000001,"5min":199.59999999999999,"15min":199.59999999999999},"last":0.47999999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:38:31 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.73599999999999999,"15min":0.69499999999999995},"min":{"1min":0.29799999999999999,"5min":0.27900000000000003,"15min":0.23699999999999999},"max":{"1min":5.7619999999999996,"5min":5.7619999999999996,"15min":8.1999999999999993},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.085,"15min":0.85899999999999999},"min":{"1min":0.32400000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":12.124000000000001,"5min":12.124000000000001,"15min":12.124000000000001},"last":0.53800000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:38:31 2023","interfaces":[{"interface":"back","average":{"1min":1.637,"5min":0.96899999999999997,"15min":0.871},"min":{"1min":0.32400000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.8490000000000002,"5min":11.372,"15min":11.473000000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":2.133,"5min":1.1679999999999999,"15min":0.94999999999999996},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26900000000000002},"max":{"1min":12.323,"5min":12.323,"15min":12.323},"last":0.48899999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:38:10 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.25,"15min":1.012},"min":{"1min":0.33600000000000002,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":6.8259999999999996,"5min":35.792999999999999,"15min":35.792999999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.331,"15min":1.079},"min":{"1min":0.38300000000000001,"5min":0.33800000000000002,"15min":0.33500000000000002},"max":{"1min":12.531000000000001,"5min":40.012999999999998,"15min":40.012999999999998},"last":0.42399999999999999}]}]},{"osd":1,"up_from":112,"seq":481036337535,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":926544,"kb_used_data":7752,"kb_used_omap":4,"kb_used_meta":918779,"kb_avail":92818608,"statfs":{"total":95995035648,"available":95046254592,"internally_reserved":0,"allocated":7938048,"data_stored":5014183,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":940830606},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":0.98699999999999999,"15min":3.1669999999999998},"min":{"1min":0.16400000000000001,"5min":0.16400000000000001,"15min":0.16400000000000001},"max":{"1min":7.1070000000000002,"5min":15.983000000000001,"15min":130.768},"last":0.254},{"interface":"front","average":{"1min":0.78800000000000003,"5min":0.65200000000000002,"15min":2.5150000000000001},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":5.726,"5min":5.8099999999999996,"15min":134.929},"last":0.246}]},{"osd":2,"last update":"Sat Sep 23 23:39:08 2023","interfaces":[{"interface":"back","average":{"1min":2.0169999999999999,"5min":3.3479999999999999,"15min":6.7489999999999997},"min":{"1min":0.28799999999999998,"5min":0.184,"15min":0.184},"max":{"1min":8.1379999999999999,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.9489999999999998,"5min":3.7050000000000001,"15min":6.8520000000000003},"min":{"1min":0.27000000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":20.202999999999999,"5min":202.12,"15min":232.21000000000001},"last":0.29299999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:39:08 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":5.1740000000000004,"15min":5.9109999999999996},"min":{"1min":0.34100000000000003,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.6579999999999995,"5min":163.94300000000001,"15min":234.404},"last":0.21299999999999999},{"interface":"front","average":{"1min":2.2919999999999998,"5min":5.4690000000000003,"15min":5.9050000000000002},"min":{"1min":0.25600000000000001,"5min":0.224,"15min":0.224},"max":{"1min":9.625,"5min":163.98699999999999,"15min":236.33699999999999},"last":0.27500000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:38:23 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":0.998,"15min":1.4039999999999999},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.28699999999999998},"max":{"1min":3.5939999999999999,"5min":4.7069999999999999,"15min":12.042999999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.034,"5min":0.874,"15min":1.4399999999999999},"min":{"1min":0.33300000000000002,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":7.1779999999999999,"5min":7.1779999999999999,"15min":18.108000000000001},"last":0.625}]},{"osd":5,"last update":"Sat Sep 23 23:38:23 2023","interfaces":[{"interface":"back","average":{"1min":4.2169999999999996,"5min":3.2690000000000001,"15min":1.962},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":80.302999999999997,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":4.7610000000000001,"5min":3.4239999999999999,"15min":2.3130000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":80.254000000000005,"5min":151.821,"15min":151.821},"last":0.63400000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:38:23 2023","interfaces":[{"interface":"back","average":{"1min":4.5700000000000003,"5min":1.732,"15min":1.921},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.27500000000000002},"max":{"1min":87.837999999999994,"5min":87.837999999999994,"15min":87.837999999999994},"last":0.53900000000000003},{"interface":"front","average":{"1min":5.0910000000000002,"5min":2.1080000000000001,"15min":2.198},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":85.518000000000001,"5min":85.518000000000001,"15min":85.518000000000001},"last":0.60899999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:39:08 2023","interfaces":[{"interface":"back","average":{"1min":3.331,"5min":1.4490000000000001,"15min":2.0150000000000001},"min":{"1min":0.38600000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":35.487000000000002,"5min":35.487000000000002,"15min":78.308000000000007},"last":0.59899999999999998},{"interface":"front","average":{"1min":3.0750000000000002,"5min":1.5089999999999999,"15min":2.0459999999999998},"min":{"1min":0.44400000000000001,"5min":0.36699999999999999,"15min":0.32700000000000001},"max":{"1min":35.146000000000001,"5min":35.146000000000001,"15min":77.204999999999998},"last":0.55500000000000005}]}]},{"osd":0,"up_from":91,"seq":390842024358,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1228160,"kb_used_data":8056,"kb_used_omap":5,"kb_used_meta":1220090,"kb_avail":92516992,"statfs":{"total":95995035648,"available":94737399808,"internally_reserved":0,"allocated":8249344,"data_stored":5325968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1249372794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:38:40 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":4.0839999999999996,"15min":6.4660000000000002},"min":{"1min":0.22500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":3.9769999999999999,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.23699999999999999},{"interface":"front","average":{"1min":1.579,"5min":3.8439999999999999,"15min":5.6849999999999996},"min":{"1min":0.23499999999999999,"5min":0.20200000000000001,"15min":0.17599999999999999},"max":{"1min":13.141,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.41699999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:39:06 2023","interfaces":[{"interface":"back","average":{"1min":6.6310000000000002,"5min":6.1769999999999996,"15min":6.3029999999999999},"min":{"1min":0.218,"5min":0.214,"15min":0.17199999999999999},"max":{"1min":102.279,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.37},{"interface":"front","average":{"1min":7.2030000000000003,"5min":6.0519999999999996,"15min":6.3360000000000003},"min":{"1min":0.23899999999999999,"5min":0.20799999999999999,"15min":0.17599999999999999},"max":{"1min":102.218,"5min":283.161,"15min":283.161},"last":0.39000000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:38:40 2023","interfaces":[{"interface":"back","average":{"1min":6.2859999999999996,"5min":7.3879999999999999,"15min":6.8490000000000002},"min":{"1min":0.28599999999999998,"5min":0.219,"15min":0.217},"max":{"1min":66.093000000000004,"5min":194.26300000000001,"15min":220.732},"last":0.32000000000000001},{"interface":"front","average":{"1min":6.8170000000000002,"5min":7.3819999999999997,"15min":6.9580000000000002},"min":{"1min":0.36099999999999999,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":66.123999999999995,"5min":194.56299999999999,"15min":218.78200000000001},"last":0.35099999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.748,"15min":2.2799999999999998},"min":{"1min":0.33000000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.605,"5min":4.1310000000000002,"15min":138.81299999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.095,"15min":3.0369999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.27200000000000002},"max":{"1min":4.2030000000000003,"5min":9.3670000000000009,"15min":171.251},"last":0.58199999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":1.3029999999999999,"15min":2.75},"min":{"1min":0.38100000000000001,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":4.4880000000000004,"5min":15.076000000000001,"15min":171.30699999999999},"last":0.504},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.1000000000000001,"15min":1.9570000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.253},"max":{"1min":2.7029999999999998,"5min":6.5279999999999996,"15min":139.40799999999999},"last":0.57399999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.1870000000000001,"15min":3.0569999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.29499999999999998},"max":{"1min":3.5939999999999999,"5min":12.654999999999999,"15min":171.33600000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.80300000000000005,"5min":1.1659999999999999,"15min":3.2360000000000002},"min":{"1min":0.39000000000000001,"5min":0.30199999999999999,"15min":0.27100000000000002},"max":{"1min":2.1360000000000001,"5min":9.157,"15min":184.13499999999999},"last":0.59599999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:39:06 2023","interfaces":[{"interface":"back","average":{"1min":5.3540000000000001,"5min":1.72,"15min":2.802},"min":{"1min":0.38100000000000001,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":66.620000000000005,"5min":66.620000000000005,"15min":138.77699999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":4.9279999999999999,"5min":1.9850000000000001,"15min":2.778},"min":{"1min":0.47799999999999998,"5min":0.35399999999999998,"15min":0.33300000000000002},"max":{"1min":66.701999999999998,"5min":66.701999999999998,"15min":139.37899999999999},"last":0.61499999999999999}]}]},{"osd":2,"up_from":199,"seq":854698492179,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":896268,"kb_used_data":8068,"kb_used_omap":7,"kb_used_meta":888184,"kb_avail":92848884,"statfs":{"total":95995035648,"available":95077257216,"internally_reserved":0,"allocated":8261632,"data_stored":5277536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":909500497},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.496,"15min":2.968},"min":{"1min":0.20999999999999999,"5min":0.186,"15min":0.17699999999999999},"max":{"1min":2.9980000000000002,"5min":61.950000000000003,"15min":95.587999999999994},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.90700000000000003,"5min":1.7050000000000001,"15min":2.8319999999999999},"min":{"1min":0.22600000000000001,"5min":0.19700000000000001,"15min":0.18099999999999999},"max":{"1min":4.5739999999999998,"5min":61.984999999999999,"15min":95.796000000000006},"last":0.41899999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":4.3550000000000004,"15min":4.516},"min":{"1min":0.21299999999999999,"5min":0.192,"15min":0.192},"max":{"1min":4.4500000000000002,"5min":106.765,"15min":157.98699999999999},"last":0.318},{"interface":"front","average":{"1min":0.76300000000000001,"5min":3.9319999999999999,"15min":4.4649999999999999},"min":{"1min":0.24399999999999999,"5min":0.222,"15min":0.222},"max":{"1min":4.468,"5min":106.732,"15min":158.03800000000001},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":5.1689999999999996,"15min":4.3070000000000004},"min":{"1min":0.245,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":4.1740000000000004,"5min":106.70399999999999,"15min":106.70399999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.0209999999999999,"5min":4.7800000000000002,"15min":4.1459999999999999},"min":{"1min":0.28100000000000003,"5min":0.246,"15min":0.23200000000000001},"max":{"1min":3.52,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.38700000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":1.6779999999999999,"15min":1.768},"min":{"1min":0.35499999999999998,"5min":0.33700000000000002,"15min":0.27700000000000002},"max":{"1min":3.5339999999999998,"5min":62.081000000000003,"15min":62.081000000000003},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.875,"15min":1.802},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.28499999999999998},"max":{"1min":2.9489999999999998,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.84699999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":9.3179999999999996,"5min":4.476,"15min":2.6640000000000001},"min":{"1min":0.32200000000000001,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":101.104,"5min":101.104,"15min":101.104},"last":0.86799999999999999},{"interface":"front","average":{"1min":9.7829999999999995,"5min":4.6280000000000001,"15min":2.6800000000000002},"min":{"1min":0.35199999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":101.404,"5min":101.404,"15min":101.404},"last":0.66400000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":9.1869999999999994,"5min":4.8730000000000002,"15min":2.9239999999999999},"min":{"1min":0.372,"5min":0.31,"15min":0.31},"max":{"1min":97.558999999999997,"5min":107.018,"15min":107.018},"last":0.67700000000000005},{"interface":"front","average":{"1min":9.4420000000000002,"5min":3.7989999999999999,"15min":2.5219999999999998},"min":{"1min":0.375,"5min":0.371,"15min":0.36699999999999999},"max":{"1min":98.231999999999999,"5min":98.231999999999999,"15min":98.231999999999999},"last":0.79200000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.7150000000000001,"15min":1.8680000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.4129999999999998,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.71499999999999997},{"interface":"front","average":{"1min":1.0840000000000001,"5min":3,"15min":2.3100000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":3.452,"5min":106.809,"15min":106.809},"last":0.64700000000000002}]}]},{"osd":3,"up_from":148,"seq":635655160148,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":951556,"kb_used_data":8060,"kb_used_omap":2,"kb_used_meta":943485,"kb_avail":92793596,"statfs":{"total":95995035648,"available":95020642304,"internally_reserved":0,"allocated":8253440,"data_stored":5338179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":966128693},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.101,"15min":2.1000000000000001},"min":{"1min":0.22700000000000001,"5min":0.184,"15min":0.17699999999999999},"max":{"1min":6.4400000000000004,"5min":17.213000000000001,"15min":62.914000000000001},"last":0.309},{"interface":"front","average":{"1min":1.6259999999999999,"5min":1.1399999999999999,"15min":2.4689999999999999},"min":{"1min":0.20300000000000001,"5min":0.188,"15min":0.17199999999999999},"max":{"1min":7.843,"5min":12.202,"15min":63.276000000000003},"last":0.26900000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:38:50 2023","interfaces":[{"interface":"back","average":{"1min":1.694,"5min":1.155,"15min":2.766},"min":{"1min":0.217,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":9.7170000000000005,"5min":9.7170000000000005,"15min":85.242999999999995},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.86899999999999999,"15min":2.6459999999999999},"min":{"1min":0.251,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":6.4169999999999998,"5min":11.228,"15min":85.302000000000007},"last":0.34399999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:39:01 2023","interfaces":[{"interface":"back","average":{"1min":2.6789999999999998,"5min":1.776,"15min":4.0330000000000004},"min":{"1min":0.29099999999999998,"5min":0.23999999999999999,"15min":0.19800000000000001},"max":{"1min":21.725999999999999,"5min":36.362000000000002,"15min":84.724999999999994},"last":0.432},{"interface":"front","average":{"1min":2.403,"5min":1.7350000000000001,"15min":3.9279999999999999},"min":{"1min":0.33900000000000002,"5min":0.26800000000000002,"15min":0.214},"max":{"1min":21.687000000000001,"5min":36.158999999999999,"15min":84.567999999999998},"last":0.496}]},{"osd":4,"last update":"Sat Sep 23 23:38:50 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":1.0069999999999999,"15min":1.1879999999999999},"min":{"1min":0.308,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":6.3330000000000002,"5min":11.209,"15min":11.209},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.9019999999999999,"5min":1.167,"15min":1.6499999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.26800000000000002},"max":{"1min":10.917,"5min":10.917,"15min":24.782},"last":0.50700000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:38:50 2023","interfaces":[{"interface":"back","average":{"1min":4.8259999999999996,"5min":3.831,"15min":4.1909999999999998},"min":{"1min":0.373,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":79.572999999999993,"5min":196.358,"15min":214.43700000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":4.8499999999999996,"5min":3.8740000000000001,"15min":4.1890000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.30599999999999999},"max":{"1min":79.426000000000002,"5min":196.048,"15min":214.46600000000001},"last":0.47299999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:38:50 2023","interfaces":[{"interface":"back","average":{"1min":1.877,"5min":2.6779999999999999,"15min":2.5950000000000002},"min":{"1min":0.36899999999999999,"5min":0.318,"15min":0.318},"max":{"1min":11.09,"5min":125.94,"15min":125.94},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.855,"5min":2.7679999999999998,"15min":2.5739999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.6319999999999997,"5min":125.955,"15min":125.955},"last":0.52100000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5,"5min":1.1399999999999999,"15min":3.234},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":6.3869999999999996,"5min":11.191000000000001,"15min":192.69},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.3009999999999999,"15min":3.6600000000000001},"min":{"1min":0.42599999999999999,"5min":0.40100000000000002,"15min":0.36699999999999999},"max":{"1min":10.951000000000001,"5min":10.951000000000001,"15min":192.65600000000001},"last":0.54500000000000004}]}]},{"osd":4,"up_from":0,"seq":128849019466,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757829,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":783828,"kb_used_data":6668,"kb_used_omap":0,"kb_used_meta":777152,"kb_avail":92961324,"statfs":{"total":95995035648,"available":95192395776,"internally_reserved":0,"allocated":6828032,"data_stored":5116007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":795803648},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:19 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.78200000000000003,"15min":1.454},"min":{"1min":0.223,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.2890000000000001,"5min":6.1609999999999996,"15min":63.741999999999997},"last":0.69299999999999995},{"interface":"front","average":{"1min":1,"5min":1.1779999999999999,"15min":1.964},"min":{"1min":0.26200000000000001,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":4.8579999999999997,"5min":15.414,"15min":138.55199999999999},"last":0.70999999999999996}]},{"osd":1,"last update":"Sat Sep 23 23:38:25 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":0.85599999999999998,"15min":2.21},"min":{"1min":0.27000000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.7480000000000002,"5min":6.1360000000000001,"15min":69.816999999999993},"last":0.76700000000000002},{"interface":"front","average":{"1min":1.1120000000000001,"5min":0.89300000000000002,"15min":2.4249999999999998},"min":{"1min":0.307,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":3.7570000000000001,"5min":4.4859999999999998,"15min":70.268000000000001},"last":0.73099999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:38:55 2023","interfaces":[{"interface":"back","average":{"1min":2.6499999999999999,"5min":4.992,"15min":4.609},"min":{"1min":0.36599999999999999,"5min":0.28799999999999998,"15min":0.26900000000000002},"max":{"1min":8.7230000000000008,"5min":145.93199999999999,"15min":208.738},"last":0.66000000000000003},{"interface":"front","average":{"1min":1.7649999999999999,"5min":3.4049999999999998,"15min":4.0759999999999996},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.4100000000000001,"5min":66.421000000000006,"15min":208.631},"last":0.77800000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:38:55 2023","interfaces":[{"interface":"back","average":{"1min":4.0339999999999998,"5min":5.9790000000000001,"15min":4.3490000000000002},"min":{"1min":0.34899999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":46.768999999999998,"5min":143.66300000000001,"15min":143.66300000000001},"last":0.81000000000000005},{"interface":"front","average":{"1min":3.9870000000000001,"5min":6.2720000000000002,"15min":4.5229999999999997},"min":{"1min":0.36299999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":46.795000000000002,"5min":143.62200000000001,"15min":143.62200000000001},"last":0.79300000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:38:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":0.90800000000000003,"15min":0.92300000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.251},"max":{"1min":4.4269999999999996,"5min":9.2690000000000001,"15min":19.962},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.92200000000000004,"15min":0.87},"min":{"1min":0.32300000000000001,"5min":0.28899999999999998,"15min":0.22900000000000001},"max":{"1min":4.4169999999999998,"5min":10.057,"15min":20.829999999999998},"last":0.74199999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":2.3490000000000002,"15min":1.556},"min":{"1min":0.28899999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.2350000000000003,"5min":148.36600000000001,"15min":148.36600000000001},"last":0.622},{"interface":"front","average":{"1min":1.1639999999999999,"5min":2.5870000000000002,"15min":1.754},"min":{"1min":0.35599999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.7919999999999998,"5min":148.42500000000001,"15min":148.42500000000001},"last":0.83499999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:38:51 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.2310000000000001,"15min":1.9970000000000001},"min":{"1min":0.33600000000000002,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":5.8559999999999999,"5min":15.497,"15min":179.28800000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.492,"5min":1.252,"15min":2.0579999999999998},"min":{"1min":0.40300000000000002,"5min":0.36699999999999999,"15min":0.34999999999999998},"max":{"1min":5.7729999999999997,"5min":15.653,"15min":179.428},"last":0.754}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:39:14.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:14.968 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:39:14.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:39:15.068 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:15.095 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:15.136 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:15.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:15.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:15.242 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:39:15.276 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:15.475 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:39:15.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:15.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:39:15.579 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:15.606 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:15.648 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:15.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:15.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:15.753 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:15.777 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:15 smithi138 ceph-mon[193606]: from='client.37110 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:39:15.777 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:15 smithi138 ceph-mon[193606]: pgmap v1759: 119 pgs: 6 creating+peering, 10 unknown, 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:15.777 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:15 smithi138 ceph-mon[193606]: failed to encode map e619 with expected crc 2023-09-23T23:39:15.778 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:15 smithi138 ceph-mon[193606]: failed to encode map e619 with expected crc 2023-09-23T23:39:15.778 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:15 smithi138 ceph-mon[193606]: osdmap e619: 8 total, 8 up, 7 in 2023-09-23T23:39:15.778 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:15.820 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:15.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:15.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:15.927 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:15.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:15 smithi110 ceph-mon[221102]: from='client.37110 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:39:15.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:15 smithi110 ceph-mon[221102]: pgmap v1759: 119 pgs: 6 creating+peering, 10 unknown, 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:15.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:15 smithi110 ceph-mon[221102]: failed to encode map e619 with expected crc 2023-09-23T23:39:15.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:15 smithi110 ceph-mon[221102]: failed to encode map e619 with expected crc 2023-09-23T23:39:15.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:15 smithi110 ceph-mon[221102]: osdmap e619: 8 total, 8 up, 7 in 2023-09-23T23:39:15.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:15 smithi110 ceph-mon[212939]: from='client.37110 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:39:15.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:15 smithi110 ceph-mon[212939]: pgmap v1759: 119 pgs: 6 creating+peering, 10 unknown, 103 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:15.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:15 smithi110 ceph-mon[212939]: failed to encode map e619 with expected crc 2023-09-23T23:39:15.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:15 smithi110 ceph-mon[212939]: failed to encode map e619 with expected crc 2023-09-23T23:39:15.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:15 smithi110 ceph-mon[212939]: osdmap e619: 8 total, 8 up, 7 in 2023-09-23T23:39:15.955 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:15.994 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:15.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:15.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:16.099 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:16.125 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:16.166 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:16.170 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:16.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:16.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:16.183 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:16.272 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:16.301 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:16.454 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:16.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:16.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:16.539 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:39:16.559 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:16.586 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:16.629 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:16.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:16.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:16.664 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:16 smithi110 ceph-mon[212939]: failed to encode map e619 with expected crc 2023-09-23T23:39:16.664 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:16 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1969399701' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:39:16.665 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:16 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:39:16.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:16 smithi110 ceph-mon[221102]: failed to encode map e619 with expected crc 2023-09-23T23:39:16.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:16 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1969399701' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:39:16.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:16 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:39:16.735 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:16.767 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:16 smithi138 ceph-mon[193606]: failed to encode map e619 with expected crc 2023-09-23T23:39:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:16 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1969399701' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:39:16.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:16 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:39:16.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:39:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:39:16] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.43.0" 2023-09-23T23:39:17.340 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:39:17.340 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:39:17.466 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:17.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:17.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:17.570 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:17.602 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:17.649 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:17.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:17.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:17.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[212939]: failed to encode map e619 with expected crc 2023-09-23T23:39:17.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[212939]: pgmap v1761: 119 pgs: 6 creating+peering, 113 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:17.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:39:17.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:39:17.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[212939]: osdmap e620: 8 total, 8 up, 7 in 2023-09-23T23:39:17.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[212939]: failed to encode map e620 with expected crc 2023-09-23T23:39:17.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[212939]: failed to encode map e620 with expected crc 2023-09-23T23:39:17.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[212939]: failed to encode map e620 with expected crc 2023-09-23T23:39:17.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[212939]: failed to encode map e620 with expected crc 2023-09-23T23:39:17.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[221102]: failed to encode map e619 with expected crc 2023-09-23T23:39:17.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[221102]: pgmap v1761: 119 pgs: 6 creating+peering, 113 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:17.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:39:17.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:39:17.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[221102]: osdmap e620: 8 total, 8 up, 7 in 2023-09-23T23:39:17.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[221102]: failed to encode map e620 with expected crc 2023-09-23T23:39:17.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[221102]: failed to encode map e620 with expected crc 2023-09-23T23:39:17.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[221102]: failed to encode map e620 with expected crc 2023-09-23T23:39:17.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:17 smithi110 ceph-mon[221102]: failed to encode map e620 with expected crc 2023-09-23T23:39:17.756 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:17.783 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:17.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:17 smithi138 ceph-mon[193606]: failed to encode map e619 with expected crc 2023-09-23T23:39:17.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:17 smithi138 ceph-mon[193606]: pgmap v1761: 119 pgs: 6 creating+peering, 113 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:17.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:17 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:39:17.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:17 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:39:17.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:17 smithi138 ceph-mon[193606]: osdmap e620: 8 total, 8 up, 7 in 2023-09-23T23:39:17.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:17 smithi138 ceph-mon[193606]: failed to encode map e620 with expected crc 2023-09-23T23:39:17.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:17 smithi138 ceph-mon[193606]: failed to encode map e620 with expected crc 2023-09-23T23:39:17.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:17 smithi138 ceph-mon[193606]: failed to encode map e620 with expected crc 2023-09-23T23:39:17.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:17 smithi138 ceph-mon[193606]: failed to encode map e620 with expected crc 2023-09-23T23:39:17.967 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:39:18.361 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:39:18.683 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:18.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:18.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:18.788 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:39:18.827 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:19.385 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:39:19.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:19.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:39:19.489 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:19.518 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:19.636 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:19.653 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:19 smithi110 ceph-mon[212939]: pgmap v1763: 119 pgs: 6 creating+peering, 113 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:19.654 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:19 smithi110 ceph-mon[221102]: pgmap v1763: 119 pgs: 6 creating+peering, 113 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:19.666 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:19.764 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:19.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:19.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:19.869 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:19.898 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:19 smithi138 ceph-mon[193606]: pgmap v1763: 119 pgs: 6 creating+peering, 113 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:19.901 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:19.949 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:19.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:19.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:20.056 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:39:20.084 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:20.131 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:39:20.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:20.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:39:20.217 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:39:20.236 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:39:20.263 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:20.304 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:39:20.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:20.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:39:20.409 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:20.435 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:20.477 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:20.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:20.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:20.543 INFO:teuthology.orchestra.run.smithi110.stderr:enabled application 'rados' on pool 'unique_pool_9' 2023-09-23T23:39:20.584 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:20.615 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:20.823 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:20 smithi110 ceph-mon[212939]: 1.16 scrub starts 2023-09-23T23:39:20.823 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:20 smithi110 ceph-mon[212939]: 1.16 scrub ok 2023-09-23T23:39:20.823 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:20 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2143982021' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:39:20.823 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:20 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:39:20.824 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:20 smithi110 ceph-mon[221102]: 1.16 scrub starts 2023-09-23T23:39:20.824 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:20 smithi110 ceph-mon[221102]: 1.16 scrub ok 2023-09-23T23:39:20.824 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:20 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2143982021' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:39:20.824 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:20 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:39:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:20 smithi138 ceph-mon[193606]: 1.16 scrub starts 2023-09-23T23:39:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:20 smithi138 ceph-mon[193606]: 1.16 scrub ok 2023-09-23T23:39:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:20 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2143982021' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:39:20.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:20 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:39:20.964 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:39:20.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:20.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:39:21.067 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:21.099 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:21.148 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:21.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:21.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:21.253 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:21.284 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:21.332 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:21.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:21.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:21.439 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:21.467 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:21.512 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:21.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:21.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:21.520 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:39:21.520 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:39:21.521 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:39:21.521 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:39:21.521 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:39:21.521 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:39:21.521 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:39:21.521 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:39:21.521 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:39:21.521 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:39:21.521 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:39:21.521 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:39:21.522 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:39:21.522 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:39:21.522 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:39:21.522 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:39:21.618 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:21.645 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:21.685 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:21.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:21.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:21 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:39:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:21 smithi110 ceph-mon[212939]: pgmap v1764: 119 pgs: 119 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:21 smithi110 ceph-mon[212939]: 1.9 scrub ok 2023-09-23T23:39:21.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:21 smithi110 ceph-mon[212939]: failed to encode map e621 with expected crc 2023-09-23T23:39:21.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:21 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:39:21.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:21 smithi110 ceph-mon[212939]: osdmap e621: 8 total, 8 up, 7 in 2023-09-23T23:39:21.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:21 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1937901153' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:39:21.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:21 smithi110 ceph-mon[212939]: failed to encode map e621 with expected crc 2023-09-23T23:39:21.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:21 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:39:21.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:21 smithi110 ceph-mon[221102]: pgmap v1764: 119 pgs: 119 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:21.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:21 smithi110 ceph-mon[221102]: 1.9 scrub ok 2023-09-23T23:39:21.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:21 smithi110 ceph-mon[221102]: failed to encode map e621 with expected crc 2023-09-23T23:39:21.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:21 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:39:21.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:21 smithi110 ceph-mon[221102]: osdmap e621: 8 total, 8 up, 7 in 2023-09-23T23:39:21.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:21 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1937901153' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:39:21.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:21 smithi110 ceph-mon[221102]: failed to encode map e621 with expected crc 2023-09-23T23:39:21.790 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:21.819 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:21 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:39:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:21 smithi138 ceph-mon[193606]: pgmap v1764: 119 pgs: 119 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:21 smithi138 ceph-mon[193606]: 1.9 scrub ok 2023-09-23T23:39:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:21 smithi138 ceph-mon[193606]: failed to encode map e621 with expected crc 2023-09-23T23:39:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:21 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:39:21.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:21 smithi138 ceph-mon[193606]: osdmap e621: 8 total, 8 up, 7 in 2023-09-23T23:39:21.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:21 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1937901153' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:39:21.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:21 smithi138 ceph-mon[193606]: failed to encode map e621 with expected crc 2023-09-23T23:39:22.420 DEBUG:teuthology.orchestra.run.smithi110:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_9 bench 90 write' 2023-09-23T23:39:22.422 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T23:39:22.423 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T23:39:22.628 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:22.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:22.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:22.657 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:hints = 1 2023-09-23T23:39:22.657 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T23:39:22.657 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object prefix: benchmark_data_smithi110_640334 2023-09-23T23:39:22.658 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:39:22.658 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 0 0 0 0 0 0 - 0 2023-09-23T23:39:22.733 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:22.801 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:23.046 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:23.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:23.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:23.152 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:39:23.182 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:23.237 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:39:23.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:23.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:39:23.272 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:23.343 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:23.373 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:23.426 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:23.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:23.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:23.440 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:39:23.440 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:39:23.440 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:39:23.441 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:39:23.441 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:39:23.531 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:39:23.561 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:23.607 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:39:23.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:23.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:39:23.658 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 1 16 2507 2491 155.678 155.688 0.00403259 0.00639459 2023-09-23T23:39:23.713 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:23.742 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:23.798 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:23.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:23.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:23.863 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:39:23.872 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:39:23.904 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:39:23.910 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:23 smithi110 ceph-mon[221102]: failed to encode map e621 with expected crc 2023-09-23T23:39:23.911 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:23 smithi110 ceph-mon[221102]: pgmap v1766: 119 pgs: 119 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:23.912 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:23 smithi110 ceph-mon[212939]: failed to encode map e621 with expected crc 2023-09-23T23:39:23.912 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:23 smithi110 ceph-mon[212939]: pgmap v1766: 119 pgs: 119 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:23.931 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:23 smithi138 ceph-mon[193606]: failed to encode map e621 with expected crc 2023-09-23T23:39:23.932 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:23 smithi138 ceph-mon[193606]: pgmap v1766: 119 pgs: 119 active+clean; 621 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-09-23T23:39:23.934 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:23.990 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:39:23.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:23.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:39:24.098 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:24.163 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:24.658 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 2 16 5421 5405 168.894 182.125 0.00213779 0.00589061 2023-09-23T23:39:24.790 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:39:25.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:25 smithi110 ceph-mon[221102]: from='client.57088 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:39:25.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:25 smithi110 ceph-mon[221102]: pgmap v1767: 119 pgs: 119 active+clean; 49 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 6.0 MiB/s wr, 94 op/s 2023-09-23T23:39:25.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:25 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:25.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:25 smithi110 ceph-mon[212939]: from='client.57088 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:39:25.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:25 smithi110 ceph-mon[212939]: pgmap v1767: 119 pgs: 119 active+clean; 49 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 6.0 MiB/s wr, 94 op/s 2023-09-23T23:39:25.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:25 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:25.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:25 smithi138 ceph-mon[193606]: from='client.57088 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:39:25.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:25 smithi138 ceph-mon[193606]: pgmap v1767: 119 pgs: 119 active+clean; 49 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 6.0 MiB/s wr, 94 op/s 2023-09-23T23:39:25.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:25 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:25.658 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 3 16 7706 7690 160.196 142.812 0.00128396 0.00617136 2023-09-23T23:39:26.434 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:26.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:26.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:26.541 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:26.573 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:26.633 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:26.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:26.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:26.658 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 4 16 10011 9995 156.159 144.062 0.00701807 0.0063972 2023-09-23T23:39:26.740 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:26.796 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:27.154 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:27 smithi110 ceph-mon[221102]: pgmap v1768: 119 pgs: 119 active+clean; 275 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 29 MiB/s wr, 463 op/s 2023-09-23T23:39:27.154 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:27 smithi110 ceph-mon[212939]: pgmap v1768: 119 pgs: 119 active+clean; 275 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 29 MiB/s wr, 463 op/s 2023-09-23T23:39:27.154 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:39:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:39:26] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.43.0" 2023-09-23T23:39:27.203 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1766,"stamp":"2023-09-23T23:39:21.983989+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":636125,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":77,"num_write_kb":193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":1110440,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5046,"ondisk_log_size":5046,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5990556,"kb_used_data":88676,"kb_used_omap":29,"kb_used_meta":5901794,"kb_avail":650225508,"statfs":{"total":671965249536,"available":665830920192,"internally_reserved":0,"allocated":90804224,"data_stored":73757922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":6043437779},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.459117"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":816,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.102909+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:39:15.102909+0000","last_peered":"2023-09-23T23:39:15.102909+0000","last_clean":"2023-09-23T23:39:15.102909+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:39:15.102909+0000","last_undegraded":"2023-09-23T23:39:15.102909+0000","last_fullsized":"2023-09-23T23:39:15.102909+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":376,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562623+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:39:15.562623+0000","last_peered":"2023-09-23T23:39:15.562623+0000","last_clean":"2023-09-23T23:39:15.562623+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:39:15.562623+0000","last_undegraded":"2023-09-23T23:39:15.562623+0000","last_fullsized":"2023-09-23T23:39:15.562623+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":603,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.005113+0000","last_change":"2023-09-23T23:33:51.556410+0000","last_active":"2023-09-23T23:39:15.005113+0000","last_peered":"2023-09-23T23:39:15.005113+0000","last_clean":"2023-09-23T23:39:15.005113+0000","last_became_active":"2023-09-23T23:33:51.556084+0000","last_became_peered":"2023-09-23T23:33:51.556084+0000","last_unstale":"2023-09-23T23:39:15.005113+0000","last_undegraded":"2023-09-23T23:39:15.005113+0000","last_fullsized":"2023-09-23T23:39:15.005113+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:33:35.328097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":769,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.103441+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:39:15.103441+0000","last_peered":"2023-09-23T23:39:15.103441+0000","last_clean":"2023-09-23T23:39:15.103441+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:39:15.103441+0000","last_undegraded":"2023-09-23T23:39:15.103441+0000","last_fullsized":"2023-09-23T23:39:15.103441+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":340,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004373+0000","last_change":"2023-09-23T23:34:03.724390+0000","last_active":"2023-09-23T23:39:15.004373+0000","last_peered":"2023-09-23T23:39:15.004373+0000","last_clean":"2023-09-23T23:39:15.004373+0000","last_became_active":"2023-09-23T23:34:03.724114+0000","last_became_peered":"2023-09-23T23:34:03.724114+0000","last_unstale":"2023-09-23T23:39:15.004373+0000","last_undegraded":"2023-09-23T23:39:15.004373+0000","last_fullsized":"2023-09-23T23:39:15.004373+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:13.717303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":582,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.005011+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:39:15.005011+0000","last_peered":"2023-09-23T23:39:15.005011+0000","last_clean":"2023-09-23T23:39:15.005011+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:39:15.005011+0000","last_undegraded":"2023-09-23T23:39:15.005011+0000","last_fullsized":"2023-09-23T23:39:15.005011+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":781,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.528436+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:39:14.528436+0000","last_peered":"2023-09-23T23:39:14.528436+0000","last_clean":"2023-09-23T23:39:14.528436+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:39:14.528436+0000","last_undegraded":"2023-09-23T23:39:14.528436+0000","last_fullsized":"2023-09-23T23:39:14.528436+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":639,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.003961+0000","last_change":"2023-09-23T23:34:35.736206+0000","last_active":"2023-09-23T23:39:15.003961+0000","last_peered":"2023-09-23T23:39:15.003961+0000","last_clean":"2023-09-23T23:39:15.003961+0000","last_became_active":"2023-09-23T23:34:35.735931+0000","last_became_peered":"2023-09-23T23:34:35.735931+0000","last_unstale":"2023-09-23T23:39:15.003961+0000","last_undegraded":"2023-09-23T23:39:15.003961+0000","last_fullsized":"2023-09-23T23:39:15.003961+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:28.817820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":134,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.529815+0000","last_change":"2023-09-23T23:29:54.994862+0000","last_active":"2023-09-23T23:39:14.529815+0000","last_peered":"2023-09-23T23:39:14.529815+0000","last_clean":"2023-09-23T23:39:14.529815+0000","last_became_active":"2023-09-23T23:29:54.994562+0000","last_became_peered":"2023-09-23T23:29:54.994562+0000","last_unstale":"2023-09-23T23:39:14.529815+0000","last_undegraded":"2023-09-23T23:39:14.529815+0000","last_fullsized":"2023-09-23T23:39:14.529815+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:35.700558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":940,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.529381+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:39:14.529381+0000","last_peered":"2023-09-23T23:39:14.529381+0000","last_clean":"2023-09-23T23:39:14.529381+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:39:14.529381+0000","last_undegraded":"2023-09-23T23:39:14.529381+0000","last_fullsized":"2023-09-23T23:39:14.529381+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":272,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.103634+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:39:15.103634+0000","last_peered":"2023-09-23T23:39:15.103634+0000","last_clean":"2023-09-23T23:39:15.103634+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:39:15.103634+0000","last_undegraded":"2023-09-23T23:39:15.103634+0000","last_fullsized":"2023-09-23T23:39:15.103634+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":622,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.003953+0000","last_change":"2023-09-23T23:29:54.992861+0000","last_active":"2023-09-23T23:39:15.003953+0000","last_peered":"2023-09-23T23:39:15.003953+0000","last_clean":"2023-09-23T23:39:15.003953+0000","last_became_active":"2023-09-23T23:29:54.992455+0000","last_became_peered":"2023-09-23T23:29:54.992455+0000","last_unstale":"2023-09-23T23:39:15.003953+0000","last_undegraded":"2023-09-23T23:39:15.003953+0000","last_fullsized":"2023-09-23T23:39:15.003953+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:13.541875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":914,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562026+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:39:15.562026+0000","last_peered":"2023-09-23T23:39:15.562026+0000","last_clean":"2023-09-23T23:39:15.562026+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:39:15.562026+0000","last_undegraded":"2023-09-23T23:39:15.562026+0000","last_fullsized":"2023-09-23T23:39:15.562026+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":251,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.103276+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:39:15.103276+0000","last_peered":"2023-09-23T23:39:15.103276+0000","last_clean":"2023-09-23T23:39:15.103276+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:39:15.103276+0000","last_undegraded":"2023-09-23T23:39:15.103276+0000","last_fullsized":"2023-09-23T23:39:15.103276+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":583,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004238+0000","last_change":"2023-09-23T23:29:55.002915+0000","last_active":"2023-09-23T23:39:15.004238+0000","last_peered":"2023-09-23T23:39:15.004238+0000","last_clean":"2023-09-23T23:39:15.004238+0000","last_became_active":"2023-09-23T23:29:55.002589+0000","last_became_peered":"2023-09-23T23:29:55.002589+0000","last_unstale":"2023-09-23T23:39:15.004238+0000","last_undegraded":"2023-09-23T23:39:15.004238+0000","last_fullsized":"2023-09-23T23:39:15.004238+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:32.239667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1074,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.103547+0000","last_change":"2023-09-23T23:34:39.280358+0000","last_active":"2023-09-23T23:39:15.103547+0000","last_peered":"2023-09-23T23:39:15.103547+0000","last_clean":"2023-09-23T23:39:15.103547+0000","last_became_active":"2023-09-23T23:34:39.279869+0000","last_became_peered":"2023-09-23T23:34:39.279869+0000","last_unstale":"2023-09-23T23:39:15.103547+0000","last_undegraded":"2023-09-23T23:39:15.103547+0000","last_fullsized":"2023-09-23T23:39:15.103547+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:26.704638+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":294,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004910+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:39:15.004910+0000","last_peered":"2023-09-23T23:39:15.004910+0000","last_clean":"2023-09-23T23:39:15.004910+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:39:15.004910+0000","last_undegraded":"2023-09-23T23:39:15.004910+0000","last_fullsized":"2023-09-23T23:39:15.004910+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":153,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.528496+0000","last_change":"2023-09-23T23:34:03.722502+0000","last_active":"2023-09-23T23:39:14.528496+0000","last_peered":"2023-09-23T23:39:14.528496+0000","last_clean":"2023-09-23T23:39:14.528496+0000","last_became_active":"2023-09-23T23:34:03.722107+0000","last_became_peered":"2023-09-23T23:34:03.722107+0000","last_unstale":"2023-09-23T23:39:14.528496+0000","last_undegraded":"2023-09-23T23:39:14.528496+0000","last_fullsized":"2023-09-23T23:39:14.528496+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:36.655390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":472,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.005090+0000","last_change":"2023-09-23T23:34:03.720481+0000","last_active":"2023-09-23T23:39:15.005090+0000","last_peered":"2023-09-23T23:39:15.005090+0000","last_clean":"2023-09-23T23:39:15.005090+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:39:15.005090+0000","last_undegraded":"2023-09-23T23:39:15.005090+0000","last_fullsized":"2023-09-23T23:39:15.005090+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:02.267981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:02.267981+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":194,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.528411+0000","last_change":"2023-09-23T23:34:18.725792+0000","last_active":"2023-09-23T23:39:14.528411+0000","last_peered":"2023-09-23T23:39:14.528411+0000","last_clean":"2023-09-23T23:39:14.528411+0000","last_became_active":"2023-09-23T23:34:18.725332+0000","last_became_peered":"2023-09-23T23:34:18.725332+0000","last_unstale":"2023-09-23T23:39:14.528411+0000","last_undegraded":"2023-09-23T23:39:14.528411+0000","last_fullsized":"2023-09-23T23:39:14.528411+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:28.812462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":561,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.528014+0000","last_change":"2023-09-23T23:33:51.563366+0000","last_active":"2023-09-23T23:39:14.528014+0000","last_peered":"2023-09-23T23:39:14.528014+0000","last_clean":"2023-09-23T23:39:14.528014+0000","last_became_active":"2023-09-23T23:33:51.563055+0000","last_became_peered":"2023-09-23T23:33:51.563055+0000","last_unstale":"2023-09-23T23:39:14.528014+0000","last_undegraded":"2023-09-23T23:39:14.528014+0000","last_fullsized":"2023-09-23T23:39:14.528014+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:22.137152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":775,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.103250+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:39:15.103250+0000","last_peered":"2023-09-23T23:39:15.103250+0000","last_clean":"2023-09-23T23:39:15.103250+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:39:15.103250+0000","last_undegraded":"2023-09-23T23:39:15.103250+0000","last_fullsized":"2023-09-23T23:39:15.103250+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":168,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562159+0000","last_change":"2023-09-23T23:34:18.729715+0000","last_active":"2023-09-23T23:39:15.562159+0000","last_peered":"2023-09-23T23:39:15.562159+0000","last_clean":"2023-09-23T23:39:15.562159+0000","last_became_active":"2023-09-23T23:34:18.729085+0000","last_became_peered":"2023-09-23T23:34:18.729085+0000","last_unstale":"2023-09-23T23:39:15.562159+0000","last_undegraded":"2023-09-23T23:39:15.562159+0000","last_fullsized":"2023-09-23T23:39:15.562159+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:14.650374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":621,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004837+0000","last_change":"2023-09-23T23:27:20.553991+0000","last_active":"2023-09-23T23:39:15.004837+0000","last_peered":"2023-09-23T23:39:15.004837+0000","last_clean":"2023-09-23T23:39:15.004837+0000","last_became_active":"2023-09-23T23:27:20.553518+0000","last_became_peered":"2023-09-23T23:27:20.553518+0000","last_unstale":"2023-09-23T23:39:15.004837+0000","last_undegraded":"2023-09-23T23:39:15.004837+0000","last_fullsized":"2023-09-23T23:39:15.004837+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.806583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":905,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.527788+0000","last_change":"2023-09-23T23:34:18.722982+0000","last_active":"2023-09-23T23:39:14.527788+0000","last_peered":"2023-09-23T23:39:14.527788+0000","last_clean":"2023-09-23T23:39:14.527788+0000","last_became_active":"2023-09-23T23:34:18.722662+0000","last_became_peered":"2023-09-23T23:34:18.722662+0000","last_unstale":"2023-09-23T23:39:14.527788+0000","last_undegraded":"2023-09-23T23:39:14.527788+0000","last_fullsized":"2023-09-23T23:39:14.527788+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:30.952688+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":216,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004602+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:39:15.004602+0000","last_peered":"2023-09-23T23:39:15.004602+0000","last_clean":"2023-09-23T23:39:15.004602+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:39:15.004602+0000","last_undegraded":"2023-09-23T23:39:15.004602+0000","last_fullsized":"2023-09-23T23:39:15.004602+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":343,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.005037+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:39:15.005037+0000","last_peered":"2023-09-23T23:39:15.005037+0000","last_clean":"2023-09-23T23:39:15.005037+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:39:15.005037+0000","last_undegraded":"2023-09-23T23:39:15.005037+0000","last_fullsized":"2023-09-23T23:39:15.005037+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":584,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004462+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:39:15.004462+0000","last_peered":"2023-09-23T23:39:15.004462+0000","last_clean":"2023-09-23T23:39:15.004462+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:39:15.004462+0000","last_undegraded":"2023-09-23T23:39:15.004462+0000","last_fullsized":"2023-09-23T23:39:15.004462+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":488,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.528008+0000","last_change":"2023-09-23T23:34:18.721745+0000","last_active":"2023-09-23T23:39:14.528008+0000","last_peered":"2023-09-23T23:39:14.528008+0000","last_clean":"2023-09-23T23:39:14.528008+0000","last_became_active":"2023-09-23T23:34:18.721472+0000","last_became_peered":"2023-09-23T23:34:18.721472+0000","last_unstale":"2023-09-23T23:39:14.528008+0000","last_undegraded":"2023-09-23T23:39:14.528008+0000","last_fullsized":"2023-09-23T23:39:14.528008+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:46.999497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":577,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.005169+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:39:15.005169+0000","last_peered":"2023-09-23T23:39:15.005169+0000","last_clean":"2023-09-23T23:39:15.005169+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:39:15.005169+0000","last_undegraded":"2023-09-23T23:39:15.005169+0000","last_fullsized":"2023-09-23T23:39:15.005169+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":983,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.103121+0000","last_change":"2023-09-23T23:29:58.259202+0000","last_active":"2023-09-23T23:39:15.103121+0000","last_peered":"2023-09-23T23:39:15.103121+0000","last_clean":"2023-09-23T23:39:15.103121+0000","last_became_active":"2023-09-23T23:29:58.258776+0000","last_became_peered":"2023-09-23T23:29:58.258776+0000","last_unstale":"2023-09-23T23:39:15.103121+0000","last_undegraded":"2023-09-23T23:39:15.103121+0000","last_fullsized":"2023-09-23T23:39:15.103121+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:35.711328+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":184,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.528440+0000","last_change":"2023-09-23T23:34:35.768405+0000","last_active":"2023-09-23T23:39:14.528440+0000","last_peered":"2023-09-23T23:39:14.528440+0000","last_clean":"2023-09-23T23:39:14.528440+0000","last_became_active":"2023-09-23T23:34:35.743442+0000","last_became_peered":"2023-09-23T23:34:35.743442+0000","last_unstale":"2023-09-23T23:39:14.528440+0000","last_undegraded":"2023-09-23T23:39:14.528440+0000","last_fullsized":"2023-09-23T23:39:14.528440+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:31.632567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":262,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.005221+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:39:15.005221+0000","last_peered":"2023-09-23T23:39:15.005221+0000","last_clean":"2023-09-23T23:39:15.005221+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:39:15.005221+0000","last_undegraded":"2023-09-23T23:39:15.005221+0000","last_fullsized":"2023-09-23T23:39:15.005221+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"0'0","reported_seq":11,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.528202+0000","last_change":"2023-09-23T23:39:13.745036+0000","last_active":"2023-09-23T23:39:14.528202+0000","last_peered":"2023-09-23T23:39:14.528202+0000","last_clean":"2023-09-23T23:39:14.528202+0000","last_became_active":"2023-09-23T23:39:13.744702+0000","last_became_peered":"2023-09-23T23:39:13.744702+0000","last_unstale":"2023-09-23T23:39:14.528202+0000","last_undegraded":"2023-09-23T23:39:14.528202+0000","last_fullsized":"2023-09-23T23:39:14.528202+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:34.829751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":955,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562302+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:39:15.562302+0000","last_peered":"2023-09-23T23:39:15.562302+0000","last_clean":"2023-09-23T23:39:15.562302+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:39:15.562302+0000","last_undegraded":"2023-09-23T23:39:15.562302+0000","last_fullsized":"2023-09-23T23:39:15.562302+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":261,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.529550+0000","last_change":"2023-09-23T23:34:35.772783+0000","last_active":"2023-09-23T23:39:14.529550+0000","last_peered":"2023-09-23T23:39:14.529550+0000","last_clean":"2023-09-23T23:39:14.529550+0000","last_became_active":"2023-09-23T23:34:35.744746+0000","last_became_peered":"2023-09-23T23:34:35.744746+0000","last_unstale":"2023-09-23T23:39:14.529550+0000","last_undegraded":"2023-09-23T23:39:14.529550+0000","last_fullsized":"2023-09-23T23:39:14.529550+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:44.248537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":251,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.103216+0000","last_change":"2023-09-23T23:29:54.997168+0000","last_active":"2023-09-23T23:39:15.103216+0000","last_peered":"2023-09-23T23:39:15.103216+0000","last_clean":"2023-09-23T23:39:15.103216+0000","last_became_active":"2023-09-23T23:29:54.996850+0000","last_became_peered":"2023-09-23T23:29:54.996850+0000","last_unstale":"2023-09-23T23:39:15.103216+0000","last_undegraded":"2023-09-23T23:39:15.103216+0000","last_fullsized":"2023-09-23T23:39:15.103216+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:41.241130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"0'0","reported_seq":12,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004140+0000","last_change":"2023-09-23T23:39:13.746676+0000","last_active":"2023-09-23T23:39:15.004140+0000","last_peered":"2023-09-23T23:39:15.004140+0000","last_clean":"2023-09-23T23:39:15.004140+0000","last_became_active":"2023-09-23T23:39:13.746389+0000","last_became_peered":"2023-09-23T23:39:13.746389+0000","last_unstale":"2023-09-23T23:39:15.004140+0000","last_undegraded":"2023-09-23T23:39:15.004140+0000","last_fullsized":"2023-09-23T23:39:15.004140+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:44.833933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1033,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004684+0000","last_change":"2023-09-23T23:31:09.816087+0000","last_active":"2023-09-23T23:39:15.004684+0000","last_peered":"2023-09-23T23:39:15.004684+0000","last_clean":"2023-09-23T23:39:15.004684+0000","last_became_active":"2023-09-23T23:31:09.815680+0000","last_became_peered":"2023-09-23T23:31:09.815680+0000","last_unstale":"2023-09-23T23:39:15.004684+0000","last_undegraded":"2023-09-23T23:39:15.004684+0000","last_fullsized":"2023-09-23T23:39:15.004684+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":511,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:38.796636+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":251,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.529526+0000","last_change":"2023-09-23T23:34:35.754692+0000","last_active":"2023-09-23T23:39:14.529526+0000","last_peered":"2023-09-23T23:39:14.529526+0000","last_clean":"2023-09-23T23:39:14.529526+0000","last_became_active":"2023-09-23T23:34:35.744681+0000","last_became_peered":"2023-09-23T23:34:35.744681+0000","last_unstale":"2023-09-23T23:39:14.529526+0000","last_undegraded":"2023-09-23T23:39:14.529526+0000","last_fullsized":"2023-09-23T23:39:14.529526+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:58.640268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":472,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004022+0000","last_change":"2023-09-23T23:34:03.724341+0000","last_active":"2023-09-23T23:39:15.004022+0000","last_peered":"2023-09-23T23:39:15.004022+0000","last_clean":"2023-09-23T23:39:15.004022+0000","last_became_active":"2023-09-23T23:34:03.724052+0000","last_became_peered":"2023-09-23T23:34:03.724052+0000","last_unstale":"2023-09-23T23:39:15.004022+0000","last_undegraded":"2023-09-23T23:39:15.004022+0000","last_fullsized":"2023-09-23T23:39:15.004022+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:20.895783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.0","version":"0'0","reported_seq":12,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562449+0000","last_change":"2023-09-23T23:39:13.741363+0000","last_active":"2023-09-23T23:39:15.562449+0000","last_peered":"2023-09-23T23:39:15.562449+0000","last_clean":"2023-09-23T23:39:15.562449+0000","last_became_active":"2023-09-23T23:39:13.741098+0000","last_became_peered":"2023-09-23T23:39:13.741098+0000","last_unstale":"2023-09-23T23:39:15.562449+0000","last_undegraded":"2023-09-23T23:39:15.562449+0000","last_fullsized":"2023-09-23T23:39:15.562449+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:31.131594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":930,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.529901+0000","last_change":"2023-09-23T23:33:54.612562+0000","last_active":"2023-09-23T23:39:14.529901+0000","last_peered":"2023-09-23T23:39:14.529901+0000","last_clean":"2023-09-23T23:39:14.529901+0000","last_became_active":"2023-09-23T23:33:54.612249+0000","last_became_peered":"2023-09-23T23:33:54.612249+0000","last_unstale":"2023-09-23T23:39:14.529901+0000","last_undegraded":"2023-09-23T23:39:14.529901+0000","last_fullsized":"2023-09-23T23:39:14.529901+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:49:11.338795+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":358,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.102816+0000","last_change":"2023-09-23T23:34:18.730906+0000","last_active":"2023-09-23T23:39:15.102816+0000","last_peered":"2023-09-23T23:39:15.102816+0000","last_clean":"2023-09-23T23:39:15.102816+0000","last_became_active":"2023-09-23T23:34:18.730525+0000","last_became_peered":"2023-09-23T23:34:18.730525+0000","last_unstale":"2023-09-23T23:39:15.102816+0000","last_undegraded":"2023-09-23T23:39:15.102816+0000","last_fullsized":"2023-09-23T23:39:15.102816+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:16.692547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":134,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.528717+0000","last_change":"2023-09-23T23:29:54.995268+0000","last_active":"2023-09-23T23:39:14.528717+0000","last_peered":"2023-09-23T23:39:14.528717+0000","last_clean":"2023-09-23T23:39:14.528717+0000","last_became_active":"2023-09-23T23:29:54.994730+0000","last_became_peered":"2023-09-23T23:29:54.994730+0000","last_unstale":"2023-09-23T23:39:14.528717+0000","last_undegraded":"2023-09-23T23:39:14.528717+0000","last_fullsized":"2023-09-23T23:39:14.528717+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.685819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"0'0","reported_seq":12,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.003921+0000","last_change":"2023-09-23T23:39:13.744449+0000","last_active":"2023-09-23T23:39:15.003921+0000","last_peered":"2023-09-23T23:39:15.003921+0000","last_clean":"2023-09-23T23:39:15.003921+0000","last_became_active":"2023-09-23T23:39:13.744163+0000","last_became_peered":"2023-09-23T23:39:13.744163+0000","last_unstale":"2023-09-23T23:39:15.003921+0000","last_undegraded":"2023-09-23T23:39:15.003921+0000","last_fullsized":"2023-09-23T23:39:15.003921+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:50.120382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":528,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.005126+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:39:15.005126+0000","last_peered":"2023-09-23T23:39:15.005126+0000","last_clean":"2023-09-23T23:39:15.005126+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:39:15.005126+0000","last_undegraded":"2023-09-23T23:39:15.005126+0000","last_fullsized":"2023-09-23T23:39:15.005126+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":461,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004777+0000","last_change":"2023-09-23T23:29:54.999264+0000","last_active":"2023-09-23T23:39:15.004777+0000","last_peered":"2023-09-23T23:39:15.004777+0000","last_clean":"2023-09-23T23:39:15.004777+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:39:15.004777+0000","last_undegraded":"2023-09-23T23:39:15.004777+0000","last_fullsized":"2023-09-23T23:39:15.004777+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:57.984768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":619,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T23:39:16.534844+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:39:16.534844+0000","last_peered":"2023-09-23T23:39:16.534844+0000","last_clean":"2023-09-23T23:39:16.534844+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:39:16.534844+0000","last_undegraded":"2023-09-23T23:39:16.534844+0000","last_fullsized":"2023-09-23T23:39:16.534844+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"0'0","reported_seq":12,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562984+0000","last_change":"2023-09-23T23:39:13.745430+0000","last_active":"2023-09-23T23:39:15.562984+0000","last_peered":"2023-09-23T23:39:15.562984+0000","last_clean":"2023-09-23T23:39:15.562984+0000","last_became_active":"2023-09-23T23:39:13.744751+0000","last_became_peered":"2023-09-23T23:39:13.744751+0000","last_unstale":"2023-09-23T23:39:15.562984+0000","last_undegraded":"2023-09-23T23:39:15.562984+0000","last_fullsized":"2023-09-23T23:39:15.562984+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:23.195329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":749,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004228+0000","last_change":"2023-09-23T23:34:18.725607+0000","last_active":"2023-09-23T23:39:15.004228+0000","last_peered":"2023-09-23T23:39:15.004228+0000","last_clean":"2023-09-23T23:39:15.004228+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:39:15.004228+0000","last_undegraded":"2023-09-23T23:39:15.004228+0000","last_fullsized":"2023-09-23T23:39:15.004228+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.830308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.830308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":631,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004371+0000","last_change":"2023-09-23T23:34:35.770226+0000","last_active":"2023-09-23T23:39:15.004371+0000","last_peered":"2023-09-23T23:39:15.004371+0000","last_clean":"2023-09-23T23:39:15.004371+0000","last_became_active":"2023-09-23T23:34:35.740506+0000","last_became_peered":"2023-09-23T23:34:35.740506+0000","last_unstale":"2023-09-23T23:39:15.004371+0000","last_undegraded":"2023-09-23T23:39:15.004371+0000","last_fullsized":"2023-09-23T23:39:15.004371+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:00.070701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":542,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004674+0000","last_change":"2023-09-23T23:33:51.548795+0000","last_active":"2023-09-23T23:39:15.004674+0000","last_peered":"2023-09-23T23:39:15.004674+0000","last_clean":"2023-09-23T23:39:15.004674+0000","last_became_active":"2023-09-23T23:33:51.548532+0000","last_became_peered":"2023-09-23T23:33:51.548532+0000","last_unstale":"2023-09-23T23:39:15.004674+0000","last_undegraded":"2023-09-23T23:39:15.004674+0000","last_fullsized":"2023-09-23T23:39:15.004674+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:31.136292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.7","version":"0'0","reported_seq":11,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.103504+0000","last_change":"2023-09-23T23:39:13.743009+0000","last_active":"2023-09-23T23:39:15.103504+0000","last_peered":"2023-09-23T23:39:15.103504+0000","last_clean":"2023-09-23T23:39:15.103504+0000","last_became_active":"2023-09-23T23:39:13.742719+0000","last_became_peered":"2023-09-23T23:39:13.742719+0000","last_unstale":"2023-09-23T23:39:15.103504+0000","last_undegraded":"2023-09-23T23:39:15.103504+0000","last_fullsized":"2023-09-23T23:39:15.103504+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:50.189642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":904,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.561981+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:39:15.561981+0000","last_peered":"2023-09-23T23:39:15.561981+0000","last_clean":"2023-09-23T23:39:15.561981+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:39:15.561981+0000","last_undegraded":"2023-09-23T23:39:15.561981+0000","last_fullsized":"2023-09-23T23:39:15.561981+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":144,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.528636+0000","last_change":"2023-09-23T23:33:51.568351+0000","last_active":"2023-09-23T23:39:14.528636+0000","last_peered":"2023-09-23T23:39:14.528636+0000","last_clean":"2023-09-23T23:39:14.528636+0000","last_became_active":"2023-09-23T23:33:51.567926+0000","last_became_peered":"2023-09-23T23:33:51.567926+0000","last_unstale":"2023-09-23T23:39:14.528636+0000","last_undegraded":"2023-09-23T23:39:14.528636+0000","last_fullsized":"2023-09-23T23:39:14.528636+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:44.765868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":487,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562959+0000","last_change":"2023-09-23T23:33:51.545806+0000","last_active":"2023-09-23T23:39:15.562959+0000","last_peered":"2023-09-23T23:39:15.562959+0000","last_clean":"2023-09-23T23:39:15.562959+0000","last_became_active":"2023-09-23T23:33:51.545469+0000","last_became_peered":"2023-09-23T23:33:51.545469+0000","last_unstale":"2023-09-23T23:39:15.562959+0000","last_undegraded":"2023-09-23T23:39:15.562959+0000","last_fullsized":"2023-09-23T23:39:15.562959+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:33.192586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"0'0","reported_seq":11,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.528225+0000","last_change":"2023-09-23T23:39:13.531316+0000","last_active":"2023-09-23T23:39:14.528225+0000","last_peered":"2023-09-23T23:39:14.528225+0000","last_clean":"2023-09-23T23:39:14.528225+0000","last_became_active":"2023-09-23T23:39:13.531001+0000","last_became_peered":"2023-09-23T23:39:13.531001+0000","last_unstale":"2023-09-23T23:39:14.528225+0000","last_undegraded":"2023-09-23T23:39:14.528225+0000","last_fullsized":"2023-09-23T23:39:14.528225+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:48:57.407908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":628,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.103353+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:39:15.103353+0000","last_peered":"2023-09-23T23:39:15.103353+0000","last_clean":"2023-09-23T23:39:15.103353+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:39:15.103353+0000","last_undegraded":"2023-09-23T23:39:15.103353+0000","last_fullsized":"2023-09-23T23:39:15.103353+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":484,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562212+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:39:15.562212+0000","last_peered":"2023-09-23T23:39:15.562212+0000","last_clean":"2023-09-23T23:39:15.562212+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:39:15.562212+0000","last_undegraded":"2023-09-23T23:39:15.562212+0000","last_fullsized":"2023-09-23T23:39:15.562212+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":571,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.529438+0000","last_change":"2023-09-23T23:33:51.572103+0000","last_active":"2023-09-23T23:39:14.529438+0000","last_peered":"2023-09-23T23:39:14.529438+0000","last_clean":"2023-09-23T23:39:14.529438+0000","last_became_active":"2023-09-23T23:33:51.571811+0000","last_became_peered":"2023-09-23T23:33:51.571811+0000","last_unstale":"2023-09-23T23:39:14.529438+0000","last_undegraded":"2023-09-23T23:39:14.529438+0000","last_fullsized":"2023-09-23T23:39:14.529438+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:19.528707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"0'0","reported_seq":12,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004765+0000","last_change":"2023-09-23T23:39:13.742854+0000","last_active":"2023-09-23T23:39:15.004765+0000","last_peered":"2023-09-23T23:39:15.004765+0000","last_clean":"2023-09-23T23:39:15.004765+0000","last_became_active":"2023-09-23T23:39:13.742579+0000","last_became_peered":"2023-09-23T23:39:13.742579+0000","last_unstale":"2023-09-23T23:39:15.004765+0000","last_undegraded":"2023-09-23T23:39:15.004765+0000","last_fullsized":"2023-09-23T23:39:15.004765+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:51.872071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":963,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562456+0000","last_change":"2023-09-23T23:29:58.258931+0000","last_active":"2023-09-23T23:39:15.562456+0000","last_peered":"2023-09-23T23:39:15.562456+0000","last_clean":"2023-09-23T23:39:15.562456+0000","last_became_active":"2023-09-23T23:29:58.258580+0000","last_became_peered":"2023-09-23T23:29:58.258580+0000","last_unstale":"2023-09-23T23:39:15.562456+0000","last_undegraded":"2023-09-23T23:39:15.562456+0000","last_fullsized":"2023-09-23T23:39:15.562456+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:29.620685+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":263,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.003868+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:39:15.003868+0000","last_peered":"2023-09-23T23:39:15.003868+0000","last_clean":"2023-09-23T23:39:15.003868+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:39:15.003868+0000","last_undegraded":"2023-09-23T23:39:15.003868+0000","last_fullsized":"2023-09-23T23:39:15.003868+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":551,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.005292+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:39:15.005292+0000","last_peered":"2023-09-23T23:39:15.005292+0000","last_clean":"2023-09-23T23:39:15.005292+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:39:15.005292+0000","last_undegraded":"2023-09-23T23:39:15.005292+0000","last_fullsized":"2023-09-23T23:39:15.005292+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.8","version":"0'0","reported_seq":12,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562357+0000","last_change":"2023-09-23T23:39:13.747857+0000","last_active":"2023-09-23T23:39:15.562357+0000","last_peered":"2023-09-23T23:39:15.562357+0000","last_clean":"2023-09-23T23:39:15.562357+0000","last_became_active":"2023-09-23T23:39:13.747570+0000","last_became_peered":"2023-09-23T23:39:13.747570+0000","last_unstale":"2023-09-23T23:39:15.562357+0000","last_undegraded":"2023-09-23T23:39:15.562357+0000","last_fullsized":"2023-09-23T23:39:15.562357+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:58.863599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":941,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.529620+0000","last_change":"2023-09-23T23:29:58.259631+0000","last_active":"2023-09-23T23:39:14.529620+0000","last_peered":"2023-09-23T23:39:14.529620+0000","last_clean":"2023-09-23T23:39:14.529620+0000","last_became_active":"2023-09-23T23:29:58.259293+0000","last_became_peered":"2023-09-23T23:29:58.259293+0000","last_unstale":"2023-09-23T23:39:14.529620+0000","last_undegraded":"2023-09-23T23:39:14.529620+0000","last_fullsized":"2023-09-23T23:39:14.529620+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:03:27.498604+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":330,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.773078+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:39:14.773078+0000","last_peered":"2023-09-23T23:39:14.773078+0000","last_clean":"2023-09-23T23:39:14.773078+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:39:14.773078+0000","last_undegraded":"2023-09-23T23:39:14.773078+0000","last_fullsized":"2023-09-23T23:39:14.773078+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":635,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.103442+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:39:15.103442+0000","last_peered":"2023-09-23T23:39:15.103442+0000","last_clean":"2023-09-23T23:39:15.103442+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:39:15.103442+0000","last_undegraded":"2023-09-23T23:39:15.103442+0000","last_fullsized":"2023-09-23T23:39:15.103442+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":216,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.103187+0000","last_change":"2023-09-23T23:34:03.728269+0000","last_active":"2023-09-23T23:39:15.103187+0000","last_peered":"2023-09-23T23:39:15.103187+0000","last_clean":"2023-09-23T23:39:15.103187+0000","last_became_active":"2023-09-23T23:34:03.727930+0000","last_became_peered":"2023-09-23T23:34:03.727930+0000","last_unstale":"2023-09-23T23:39:15.103187+0000","last_undegraded":"2023-09-23T23:39:15.103187+0000","last_fullsized":"2023-09-23T23:39:15.103187+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:46.406135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.e","version":"0'0","reported_seq":12,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562929+0000","last_change":"2023-09-23T23:39:13.745603+0000","last_active":"2023-09-23T23:39:15.562929+0000","last_peered":"2023-09-23T23:39:15.562929+0000","last_clean":"2023-09-23T23:39:15.562929+0000","last_became_active":"2023-09-23T23:39:13.745132+0000","last_became_peered":"2023-09-23T23:39:13.745132+0000","last_unstale":"2023-09-23T23:39:15.562929+0000","last_undegraded":"2023-09-23T23:39:15.562929+0000","last_fullsized":"2023-09-23T23:39:15.562929+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:30.212782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":706,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T23:39:16.534798+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:39:16.534798+0000","last_peered":"2023-09-23T23:39:16.534798+0000","last_clean":"2023-09-23T23:39:16.534798+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:39:16.534798+0000","last_undegraded":"2023-09-23T23:39:16.534798+0000","last_fullsized":"2023-09-23T23:39:16.534798+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":268,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.528545+0000","last_change":"2023-09-23T23:34:03.729102+0000","last_active":"2023-09-23T23:39:14.528545+0000","last_peered":"2023-09-23T23:39:14.528545+0000","last_clean":"2023-09-23T23:39:14.528545+0000","last_became_active":"2023-09-23T23:34:03.728788+0000","last_became_peered":"2023-09-23T23:34:03.728788+0000","last_unstale":"2023-09-23T23:39:14.528545+0000","last_undegraded":"2023-09-23T23:39:14.528545+0000","last_fullsized":"2023-09-23T23:39:14.528545+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:21.094972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":552,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.528087+0000","last_change":"2023-09-23T23:33:51.568728+0000","last_active":"2023-09-23T23:39:14.528087+0000","last_peered":"2023-09-23T23:39:14.528087+0000","last_clean":"2023-09-23T23:39:14.528087+0000","last_became_active":"2023-09-23T23:33:51.568436+0000","last_became_peered":"2023-09-23T23:33:51.568436+0000","last_unstale":"2023-09-23T23:39:14.528087+0000","last_undegraded":"2023-09-23T23:39:14.528087+0000","last_fullsized":"2023-09-23T23:39:14.528087+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:28.272085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":254,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562828+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:39:15.562828+0000","last_peered":"2023-09-23T23:39:15.562828+0000","last_clean":"2023-09-23T23:39:15.562828+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:39:15.562828+0000","last_undegraded":"2023-09-23T23:39:15.562828+0000","last_fullsized":"2023-09-23T23:39:15.562828+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"0'0","reported_seq":11,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.529004+0000","last_change":"2023-09-23T23:39:13.531150+0000","last_active":"2023-09-23T23:39:14.529004+0000","last_peered":"2023-09-23T23:39:14.529004+0000","last_clean":"2023-09-23T23:39:14.529004+0000","last_became_active":"2023-09-23T23:39:13.530808+0000","last_became_peered":"2023-09-23T23:39:13.530808+0000","last_unstale":"2023-09-23T23:39:14.529004+0000","last_undegraded":"2023-09-23T23:39:14.529004+0000","last_fullsized":"2023-09-23T23:39:14.529004+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:06.705448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":923,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562065+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:39:15.562065+0000","last_peered":"2023-09-23T23:39:15.562065+0000","last_clean":"2023-09-23T23:39:15.562065+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:39:15.562065+0000","last_undegraded":"2023-09-23T23:39:15.562065+0000","last_fullsized":"2023-09-23T23:39:15.562065+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":360,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562130+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:39:15.562130+0000","last_peered":"2023-09-23T23:39:15.562130+0000","last_clean":"2023-09-23T23:39:15.562130+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:39:15.562130+0000","last_undegraded":"2023-09-23T23:39:15.562130+0000","last_fullsized":"2023-09-23T23:39:15.562130+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":269,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562594+0000","last_change":"2023-09-23T23:34:03.718697+0000","last_active":"2023-09-23T23:39:15.562594+0000","last_peered":"2023-09-23T23:39:15.562594+0000","last_clean":"2023-09-23T23:39:15.562594+0000","last_became_active":"2023-09-23T23:34:03.718420+0000","last_became_peered":"2023-09-23T23:34:03.718420+0000","last_unstale":"2023-09-23T23:39:15.562594+0000","last_undegraded":"2023-09-23T23:39:15.562594+0000","last_fullsized":"2023-09-23T23:39:15.562594+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:45.965727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":233,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.774668+0000","last_change":"2023-09-23T23:34:35.750118+0000","last_active":"2023-09-23T23:39:14.774668+0000","last_peered":"2023-09-23T23:39:14.774668+0000","last_clean":"2023-09-23T23:39:14.774668+0000","last_became_active":"2023-09-23T23:34:35.749696+0000","last_became_peered":"2023-09-23T23:34:35.749696+0000","last_unstale":"2023-09-23T23:39:14.774668+0000","last_undegraded":"2023-09-23T23:39:14.774668+0000","last_fullsized":"2023-09-23T23:39:14.774668+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:35.237207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"0'0","reported_seq":12,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.005092+0000","last_change":"2023-09-23T23:39:13.747297+0000","last_active":"2023-09-23T23:39:15.005092+0000","last_peered":"2023-09-23T23:39:15.005092+0000","last_clean":"2023-09-23T23:39:15.005092+0000","last_became_active":"2023-09-23T23:39:13.747068+0000","last_became_peered":"2023-09-23T23:39:13.747068+0000","last_unstale":"2023-09-23T23:39:15.005092+0000","last_undegraded":"2023-09-23T23:39:15.005092+0000","last_fullsized":"2023-09-23T23:39:15.005092+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:34.450587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":636,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562745+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:39:15.562745+0000","last_peered":"2023-09-23T23:39:15.562745+0000","last_clean":"2023-09-23T23:39:15.562745+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:39:15.562745+0000","last_undegraded":"2023-09-23T23:39:15.562745+0000","last_fullsized":"2023-09-23T23:39:15.562745+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":906,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.527894+0000","last_change":"2023-09-23T23:34:39.277858+0000","last_active":"2023-09-23T23:39:14.527894+0000","last_peered":"2023-09-23T23:39:14.527894+0000","last_clean":"2023-09-23T23:39:14.527894+0000","last_became_active":"2023-09-23T23:34:39.277506+0000","last_became_peered":"2023-09-23T23:34:39.277506+0000","last_unstale":"2023-09-23T23:39:14.527894+0000","last_undegraded":"2023-09-23T23:39:14.527894+0000","last_fullsized":"2023-09-23T23:39:14.527894+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:16.576006+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":583,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004499+0000","last_change":"2023-09-23T23:29:54.992143+0000","last_active":"2023-09-23T23:39:15.004499+0000","last_peered":"2023-09-23T23:39:15.004499+0000","last_clean":"2023-09-23T23:39:15.004499+0000","last_became_active":"2023-09-23T23:29:54.991836+0000","last_became_peered":"2023-09-23T23:29:54.991836+0000","last_unstale":"2023-09-23T23:39:15.004499+0000","last_undegraded":"2023-09-23T23:39:15.004499+0000","last_fullsized":"2023-09-23T23:39:15.004499+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:38.665699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":144,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.529613+0000","last_change":"2023-09-23T23:33:51.555593+0000","last_active":"2023-09-23T23:39:14.529613+0000","last_peered":"2023-09-23T23:39:14.529613+0000","last_clean":"2023-09-23T23:39:14.529613+0000","last_became_active":"2023-09-23T23:33:51.555340+0000","last_became_peered":"2023-09-23T23:33:51.555340+0000","last_unstale":"2023-09-23T23:39:14.529613+0000","last_undegraded":"2023-09-23T23:39:14.529613+0000","last_fullsized":"2023-09-23T23:39:14.529613+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:49:15.542249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"0'0","reported_seq":12,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562455+0000","last_change":"2023-09-23T23:39:13.742798+0000","last_active":"2023-09-23T23:39:15.562455+0000","last_peered":"2023-09-23T23:39:15.562455+0000","last_clean":"2023-09-23T23:39:15.562455+0000","last_became_active":"2023-09-23T23:39:13.742484+0000","last_became_peered":"2023-09-23T23:39:13.742484+0000","last_unstale":"2023-09-23T23:39:15.562455+0000","last_undegraded":"2023-09-23T23:39:15.562455+0000","last_fullsized":"2023-09-23T23:39:15.562455+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:34.260853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":732,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.527977+0000","last_change":"2023-09-23T23:34:03.724998+0000","last_active":"2023-09-23T23:39:14.527977+0000","last_peered":"2023-09-23T23:39:14.527977+0000","last_clean":"2023-09-23T23:39:14.527977+0000","last_became_active":"2023-09-23T23:34:03.724613+0000","last_became_peered":"2023-09-23T23:34:03.724613+0000","last_unstale":"2023-09-23T23:39:14.527977+0000","last_undegraded":"2023-09-23T23:39:14.527977+0000","last_fullsized":"2023-09-23T23:39:14.527977+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:14:15.210483+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":547,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.528758+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:39:14.528758+0000","last_peered":"2023-09-23T23:39:14.528758+0000","last_clean":"2023-09-23T23:39:14.528758+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:39:14.528758+0000","last_undegraded":"2023-09-23T23:39:14.528758+0000","last_fullsized":"2023-09-23T23:39:14.528758+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":462,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004042+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:39:15.004042+0000","last_peered":"2023-09-23T23:39:15.004042+0000","last_clean":"2023-09-23T23:39:15.004042+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:39:15.004042+0000","last_undegraded":"2023-09-23T23:39:15.004042+0000","last_fullsized":"2023-09-23T23:39:15.004042+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":63,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T23:39:16.534389+0000","last_change":"2023-09-23T23:34:35.750050+0000","last_active":"2023-09-23T23:39:16.534389+0000","last_peered":"2023-09-23T23:39:16.534389+0000","last_clean":"2023-09-23T23:39:16.534389+0000","last_became_active":"2023-09-23T23:34:35.749743+0000","last_became_peered":"2023-09-23T23:34:35.749743+0000","last_unstale":"2023-09-23T23:39:16.534389+0000","last_undegraded":"2023-09-23T23:39:16.534389+0000","last_fullsized":"2023-09-23T23:39:16.534389+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:31.973964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"0'0","reported_seq":12,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004134+0000","last_change":"2023-09-23T23:39:13.742801+0000","last_active":"2023-09-23T23:39:15.004134+0000","last_peered":"2023-09-23T23:39:15.004134+0000","last_clean":"2023-09-23T23:39:15.004134+0000","last_became_active":"2023-09-23T23:39:13.742482+0000","last_became_peered":"2023-09-23T23:39:13.742482+0000","last_unstale":"2023-09-23T23:39:15.004134+0000","last_undegraded":"2023-09-23T23:39:15.004134+0000","last_fullsized":"2023-09-23T23:39:15.004134+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:15.737461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":615,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.529750+0000","last_change":"2023-09-23T23:34:39.267248+0000","last_active":"2023-09-23T23:39:14.529750+0000","last_peered":"2023-09-23T23:39:14.529750+0000","last_clean":"2023-09-23T23:39:14.529750+0000","last_became_active":"2023-09-23T23:34:39.266786+0000","last_became_peered":"2023-09-23T23:34:39.266786+0000","last_unstale":"2023-09-23T23:39:14.529750+0000","last_undegraded":"2023-09-23T23:39:14.529750+0000","last_fullsized":"2023-09-23T23:39:14.529750+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:14.245135+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":63,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T23:39:16.535133+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:39:16.535133+0000","last_peered":"2023-09-23T23:39:16.535133+0000","last_clean":"2023-09-23T23:39:16.535133+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:39:16.535133+0000","last_undegraded":"2023-09-23T23:39:16.535133+0000","last_fullsized":"2023-09-23T23:39:16.535133+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":161,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562909+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:39:15.562909+0000","last_peered":"2023-09-23T23:39:15.562909+0000","last_clean":"2023-09-23T23:39:15.562909+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:39:15.562909+0000","last_undegraded":"2023-09-23T23:39:15.562909+0000","last_fullsized":"2023-09-23T23:39:15.562909+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":353,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.103588+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:39:15.103588+0000","last_peered":"2023-09-23T23:39:15.103588+0000","last_clean":"2023-09-23T23:39:15.103588+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:39:15.103588+0000","last_undegraded":"2023-09-23T23:39:15.103588+0000","last_fullsized":"2023-09-23T23:39:15.103588+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.a","version":"0'0","reported_seq":12,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004435+0000","last_change":"2023-09-23T23:39:13.746851+0000","last_active":"2023-09-23T23:39:15.004435+0000","last_peered":"2023-09-23T23:39:15.004435+0000","last_clean":"2023-09-23T23:39:15.004435+0000","last_became_active":"2023-09-23T23:39:13.746412+0000","last_became_peered":"2023-09-23T23:39:13.746412+0000","last_unstale":"2023-09-23T23:39:15.004435+0000","last_undegraded":"2023-09-23T23:39:15.004435+0000","last_fullsized":"2023-09-23T23:39:15.004435+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:39.662209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":802,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562205+0000","last_change":"2023-09-23T23:33:54.597912+0000","last_active":"2023-09-23T23:39:15.562205+0000","last_peered":"2023-09-23T23:39:15.562205+0000","last_clean":"2023-09-23T23:39:15.562205+0000","last_became_active":"2023-09-23T23:33:54.597583+0000","last_became_peered":"2023-09-23T23:33:54.597583+0000","last_unstale":"2023-09-23T23:39:15.562205+0000","last_undegraded":"2023-09-23T23:39:15.562205+0000","last_fullsized":"2023-09-23T23:39:15.562205+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:45.752989+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":619,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.005299+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:39:15.005299+0000","last_peered":"2023-09-23T23:39:15.005299+0000","last_clean":"2023-09-23T23:39:15.005299+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:39:15.005299+0000","last_undegraded":"2023-09-23T23:39:15.005299+0000","last_fullsized":"2023-09-23T23:39:15.005299+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":224,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.529510+0000","last_change":"2023-09-23T23:34:03.722490+0000","last_active":"2023-09-23T23:39:14.529510+0000","last_peered":"2023-09-23T23:39:14.529510+0000","last_clean":"2023-09-23T23:39:14.529510+0000","last_became_active":"2023-09-23T23:34:03.722213+0000","last_became_peered":"2023-09-23T23:34:03.722213+0000","last_unstale":"2023-09-23T23:39:14.529510+0000","last_undegraded":"2023-09-23T23:39:14.529510+0000","last_fullsized":"2023-09-23T23:39:14.529510+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:03.310130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":173,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562482+0000","last_change":"2023-09-23T23:34:35.747919+0000","last_active":"2023-09-23T23:39:15.562482+0000","last_peered":"2023-09-23T23:39:15.562482+0000","last_clean":"2023-09-23T23:39:15.562482+0000","last_became_active":"2023-09-23T23:34:35.747595+0000","last_became_peered":"2023-09-23T23:34:35.747595+0000","last_unstale":"2023-09-23T23:39:15.562482+0000","last_undegraded":"2023-09-23T23:39:15.562482+0000","last_fullsized":"2023-09-23T23:39:15.562482+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:30.036991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.b","version":"0'0","reported_seq":12,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562661+0000","last_change":"2023-09-23T23:39:13.744730+0000","last_active":"2023-09-23T23:39:15.562661+0000","last_peered":"2023-09-23T23:39:15.562661+0000","last_clean":"2023-09-23T23:39:15.562661+0000","last_became_active":"2023-09-23T23:39:13.744442+0000","last_became_peered":"2023-09-23T23:39:13.744442+0000","last_unstale":"2023-09-23T23:39:15.562661+0000","last_undegraded":"2023-09-23T23:39:15.562661+0000","last_fullsized":"2023-09-23T23:39:15.562661+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:56:43.437044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":1008,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004916+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:39:15.004916+0000","last_peered":"2023-09-23T23:39:15.004916+0000","last_clean":"2023-09-23T23:39:15.004916+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:39:15.004916+0000","last_undegraded":"2023-09-23T23:39:15.004916+0000","last_fullsized":"2023-09-23T23:39:15.004916+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":134,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.529733+0000","last_change":"2023-09-23T23:29:54.996285+0000","last_active":"2023-09-23T23:39:14.529733+0000","last_peered":"2023-09-23T23:39:14.529733+0000","last_clean":"2023-09-23T23:39:14.529733+0000","last_became_active":"2023-09-23T23:29:54.993872+0000","last_became_peered":"2023-09-23T23:29:54.993872+0000","last_unstale":"2023-09-23T23:39:14.529733+0000","last_undegraded":"2023-09-23T23:39:14.529733+0000","last_fullsized":"2023-09-23T23:39:14.529733+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:03.848786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":67,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T23:39:16.534435+0000","last_change":"2023-09-23T23:34:35.749898+0000","last_active":"2023-09-23T23:39:16.534435+0000","last_peered":"2023-09-23T23:39:16.534435+0000","last_clean":"2023-09-23T23:39:16.534435+0000","last_became_active":"2023-09-23T23:34:35.746238+0000","last_became_peered":"2023-09-23T23:34:35.746238+0000","last_unstale":"2023-09-23T23:39:16.534435+0000","last_undegraded":"2023-09-23T23:39:16.534435+0000","last_fullsized":"2023-09-23T23:39:16.534435+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:26.177655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":194,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.103331+0000","last_change":"2023-09-23T23:33:51.555845+0000","last_active":"2023-09-23T23:39:15.103331+0000","last_peered":"2023-09-23T23:39:15.103331+0000","last_clean":"2023-09-23T23:39:15.103331+0000","last_became_active":"2023-09-23T23:33:51.555486+0000","last_became_peered":"2023-09-23T23:33:51.555486+0000","last_unstale":"2023-09-23T23:39:15.103331+0000","last_undegraded":"2023-09-23T23:39:15.103331+0000","last_fullsized":"2023-09-23T23:39:15.103331+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:17.541067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1057,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004181+0000","last_change":"2023-09-23T23:34:39.279631+0000","last_active":"2023-09-23T23:39:15.004181+0000","last_peered":"2023-09-23T23:39:15.004181+0000","last_clean":"2023-09-23T23:39:15.004181+0000","last_became_active":"2023-09-23T23:34:39.279287+0000","last_became_peered":"2023-09-23T23:34:39.279287+0000","last_unstale":"2023-09-23T23:39:15.004181+0000","last_undegraded":"2023-09-23T23:39:15.004181+0000","last_fullsized":"2023-09-23T23:39:15.004181+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:06.289629+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":62,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T23:39:16.534922+0000","last_change":"2023-09-23T23:34:35.734570+0000","last_active":"2023-09-23T23:39:16.534922+0000","last_peered":"2023-09-23T23:39:16.534922+0000","last_clean":"2023-09-23T23:39:16.534922+0000","last_became_active":"2023-09-23T23:34:35.733857+0000","last_became_peered":"2023-09-23T23:34:35.733857+0000","last_unstale":"2023-09-23T23:39:16.534922+0000","last_undegraded":"2023-09-23T23:39:16.534922+0000","last_fullsized":"2023-09-23T23:39:16.534922+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.964918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":376,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562999+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:39:15.562999+0000","last_peered":"2023-09-23T23:39:15.562999+0000","last_clean":"2023-09-23T23:39:15.562999+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:39:15.562999+0000","last_undegraded":"2023-09-23T23:39:15.562999+0000","last_fullsized":"2023-09-23T23:39:15.562999+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":769,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.005225+0000","last_change":"2023-09-23T23:29:58.261889+0000","last_active":"2023-09-23T23:39:15.005225+0000","last_peered":"2023-09-23T23:39:15.005225+0000","last_clean":"2023-09-23T23:39:15.005225+0000","last_became_active":"2023-09-23T23:29:58.261560+0000","last_became_peered":"2023-09-23T23:29:58.261560+0000","last_unstale":"2023-09-23T23:39:15.005225+0000","last_undegraded":"2023-09-23T23:39:15.005225+0000","last_fullsized":"2023-09-23T23:39:15.005225+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:39.184915+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":122,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.528316+0000","last_change":"2023-09-23T23:34:03.727887+0000","last_active":"2023-09-23T23:39:14.528316+0000","last_peered":"2023-09-23T23:39:14.528316+0000","last_clean":"2023-09-23T23:39:14.528316+0000","last_became_active":"2023-09-23T23:34:03.727563+0000","last_became_peered":"2023-09-23T23:34:03.727563+0000","last_unstale":"2023-09-23T23:39:14.528316+0000","last_undegraded":"2023-09-23T23:39:14.528316+0000","last_fullsized":"2023-09-23T23:39:14.528316+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.673639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":469,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004280+0000","last_change":"2023-09-23T23:33:51.541223+0000","last_active":"2023-09-23T23:39:15.004280+0000","last_peered":"2023-09-23T23:39:15.004280+0000","last_clean":"2023-09-23T23:39:15.004280+0000","last_became_active":"2023-09-23T23:33:51.540659+0000","last_became_peered":"2023-09-23T23:33:51.540659+0000","last_unstale":"2023-09-23T23:39:15.004280+0000","last_undegraded":"2023-09-23T23:39:15.004280+0000","last_fullsized":"2023-09-23T23:39:15.004280+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:07.948647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":318,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.529407+0000","last_change":"2023-09-23T23:33:51.593735+0000","last_active":"2023-09-23T23:39:14.529407+0000","last_peered":"2023-09-23T23:39:14.529407+0000","last_clean":"2023-09-23T23:39:14.529407+0000","last_became_active":"2023-09-23T23:33:51.571579+0000","last_became_peered":"2023-09-23T23:33:51.571579+0000","last_unstale":"2023-09-23T23:39:14.529407+0000","last_undegraded":"2023-09-23T23:39:14.529407+0000","last_fullsized":"2023-09-23T23:39:14.529407+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:52.229384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":308,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004522+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:39:15.004522+0000","last_peered":"2023-09-23T23:39:15.004522+0000","last_clean":"2023-09-23T23:39:15.004522+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:39:15.004522+0000","last_undegraded":"2023-09-23T23:39:15.004522+0000","last_fullsized":"2023-09-23T23:39:15.004522+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":832,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.528991+0000","last_change":"2023-09-23T23:34:18.720948+0000","last_active":"2023-09-23T23:39:14.528991+0000","last_peered":"2023-09-23T23:39:14.528991+0000","last_clean":"2023-09-23T23:39:14.528991+0000","last_became_active":"2023-09-23T23:34:18.720627+0000","last_became_peered":"2023-09-23T23:34:18.720627+0000","last_unstale":"2023-09-23T23:39:14.528991+0000","last_undegraded":"2023-09-23T23:39:14.528991+0000","last_fullsized":"2023-09-23T23:39:14.528991+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:41.468566+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":652,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.005224+0000","last_change":"2023-09-23T23:29:54.999316+0000","last_active":"2023-09-23T23:39:15.005224+0000","last_peered":"2023-09-23T23:39:15.005224+0000","last_clean":"2023-09-23T23:39:15.005224+0000","last_became_active":"2023-09-23T23:29:54.998956+0000","last_became_peered":"2023-09-23T23:29:54.998956+0000","last_unstale":"2023-09-23T23:39:15.005224+0000","last_undegraded":"2023-09-23T23:39:15.005224+0000","last_fullsized":"2023-09-23T23:39:15.005224+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:00.922808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":635,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004364+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:39:15.004364+0000","last_peered":"2023-09-23T23:39:15.004364+0000","last_clean":"2023-09-23T23:39:15.004364+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:39:15.004364+0000","last_undegraded":"2023-09-23T23:39:15.004364+0000","last_fullsized":"2023-09-23T23:39:15.004364+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":794,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.004833+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:39:15.004833+0000","last_peered":"2023-09-23T23:39:15.004833+0000","last_clean":"2023-09-23T23:39:15.004833+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:39:15.004833+0000","last_undegraded":"2023-09-23T23:39:15.004833+0000","last_fullsized":"2023-09-23T23:39:15.004833+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":465,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:15.562965+0000","last_change":"2023-09-23T23:34:35.776638+0000","last_active":"2023-09-23T23:39:15.562965+0000","last_peered":"2023-09-23T23:39:15.562965+0000","last_clean":"2023-09-23T23:39:15.562965+0000","last_became_active":"2023-09-23T23:34:35.748276+0000","last_became_peered":"2023-09-23T23:34:35.748276+0000","last_unstale":"2023-09-23T23:39:15.562965+0000","last_undegraded":"2023-09-23T23:39:15.562965+0000","last_fullsized":"2023-09-23T23:39:15.562965+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:38.828586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":687,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T23:39:14.527670+0000","last_change":"2023-09-23T23:33:51.559732+0000","last_active":"2023-09-23T23:39:14.527670+0000","last_peered":"2023-09-23T23:39:14.527670+0000","last_clean":"2023-09-23T23:39:14.527670+0000","last_became_active":"2023-09-23T23:33:51.559347+0000","last_became_peered":"2023-09-23T23:33:51.559347+0000","last_unstale":"2023-09-23T23:39:14.527670+0000","last_undegraded":"2023-09-23T23:39:14.527670+0000","last_fullsized":"2023-09-23T23:39:14.527670+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:42.694342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":6644,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":16,"num_write_kb":164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1103796,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782718,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":429400,"kb_used_data":6736,"kb_used_omap":8,"kb_used_meta":422647,"kb_avail":93315752,"statfs":{"total":95995035648,"available":95555330048,"internally_reserved":0,"allocated":6897664,"data_stored":5153231,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":432790597},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:39:02 2023","interfaces":[{"interface":"back","average":{"1min":1.893,"5min":1.03,"15min":2.073},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":12.432,"5min":12.432,"15min":85.721000000000004},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.8859999999999999,"5min":0.90000000000000002,"15min":2.198},"min":{"1min":0.22900000000000001,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":12.717000000000001,"5min":12.717000000000001,"15min":85.965000000000003},"last":0.56000000000000005}]},{"osd":1,"last update":"Sat Sep 23 23:39:02 2023","interfaces":[{"interface":"back","average":{"1min":1.6759999999999999,"5min":1.044,"15min":3.1469999999999998},"min":{"1min":0.27000000000000002,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":12.484,"5min":12.484,"15min":198.58000000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":2.71,"5min":1.1379999999999999,"15min":3.1699999999999999},"min":{"1min":0.246,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":13.145,"5min":13.145,"15min":198.71100000000001},"last":0.52400000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:39:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5870000000000002,"5min":2.6739999999999999,"15min":3.7829999999999999},"min":{"1min":0.314,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":12.795999999999999,"5min":78.528999999999996,"15min":203.58000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.0630000000000002,"5min":1.839,"15min":3.278},"min":{"1min":0.30599999999999999,"5min":0.27900000000000003,"15min":0.26200000000000001},"max":{"1min":9.3010000000000002,"5min":53.023000000000003,"15min":198.69},"last":0.46500000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:39:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1299999999999999,"5min":5.5960000000000001,"15min":5.0110000000000001},"min":{"1min":0.309,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":9.0660000000000007,"5min":154.18100000000001,"15min":279.26499999999999},"last":0.502},{"interface":"front","average":{"1min":2.165,"5min":5.7789999999999999,"15min":5.5339999999999998},"min":{"1min":0.35699999999999998,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":12.551,"5min":154.16300000000001,"15min":279.29899999999998},"last":0.47599999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:39:02 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":0.95199999999999996,"15min":1.048},"min":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.246},"max":{"1min":12.510999999999999,"5min":12.510999999999999,"15min":31.498999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.7230000000000001,"5min":0.89600000000000002,"15min":1.026},"min":{"1min":0.31900000000000001,"5min":0.317,"15min":0.23899999999999999},"max":{"1min":12.821999999999999,"5min":12.821999999999999,"15min":31.68},"last":0.58799999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:39:02 2023","interfaces":[{"interface":"back","average":{"1min":1.798,"5min":2.8969999999999998,"15min":2.3140000000000001},"min":{"1min":0.28899999999999998,"5min":0.27900000000000003,"15min":0.26500000000000001},"max":{"1min":13.849,"5min":119.417,"15min":210.83600000000001},"last":0.44},{"interface":"front","average":{"1min":1.7629999999999999,"5min":3.129,"15min":2.2629999999999999},"min":{"1min":0.29999999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.8040000000000003,"5min":119.61499999999999,"15min":207.04599999999999},"last":0.39800000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:39:02 2023","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":1.0700000000000001,"15min":1.4019999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.29899999999999999},"max":{"1min":5.3799999999999999,"5min":8.6440000000000001,"15min":94.852999999999994},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.522,"5min":1.097,"15min":1.5429999999999999},"min":{"1min":0.34899999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":12.529999999999999,"5min":12.529999999999999,"15min":95.510999999999996},"last":0.51500000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496195,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":774472,"kb_used_data":43072,"kb_used_omap":0,"kb_used_meta":731392,"kb_avail":92970680,"statfs":{"total":95995035648,"available":95201976320,"internally_reserved":0,"allocated":44105728,"data_stored":42376758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":748945408},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:26 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":0.90400000000000003,"15min":3.956},"min":{"1min":0.224,"5min":0.224,"15min":0.21099999999999999},"max":{"1min":6.5579999999999998,"5min":6.5579999999999998,"15min":135.31999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.1850000000000001,"5min":0.70899999999999996,"15min":3.891},"min":{"1min":0.28599999999999998,"5min":0.24399999999999999,"15min":0.23000000000000001},"max":{"1min":6.8529999999999998,"5min":6.8529999999999998,"15min":135.34299999999999},"last":0.40699999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:38:26 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":1.018,"15min":2.1720000000000002},"min":{"1min":0.28499999999999998,"5min":0.253,"15min":0.245},"max":{"1min":6.6200000000000001,"5min":6.9080000000000004,"15min":89.605000000000004},"last":0.375},{"interface":"front","average":{"1min":1.4990000000000001,"5min":0.873,"15min":2.2749999999999999},"min":{"1min":0.26300000000000001,"5min":0.249,"15min":0.23300000000000001},"max":{"1min":12.577,"5min":12.577,"15min":91.025999999999996},"last":0.434}]},{"osd":2,"last update":"Sat Sep 23 23:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":4.0960000000000001,"15min":4.6950000000000003},"min":{"1min":0.31900000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.7919999999999998,"5min":85.995999999999995,"15min":127.44499999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.0369999999999999,"5min":3.915,"15min":5.0860000000000003},"min":{"1min":0.312,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.8220000000000001,"5min":85.266999999999996,"15min":125.539},"last":0.39700000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:39:00 2023","interfaces":[{"interface":"back","average":{"1min":3.109,"5min":6.8689999999999998,"15min":5.3520000000000003},"min":{"1min":0.35599999999999998,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":12.166,"5min":199.536,"15min":199.536},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.998,"5min":6.4969999999999999,"15min":5.5229999999999997},"min":{"1min":0.29999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":10.826000000000001,"5min":199.59999999999999,"15min":199.59999999999999},"last":0.42299999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:38:31 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.73599999999999999,"15min":0.69499999999999995},"min":{"1min":0.29799999999999999,"5min":0.27900000000000003,"15min":0.23699999999999999},"max":{"1min":5.7619999999999996,"5min":5.7619999999999996,"15min":8.1999999999999993},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.085,"15min":0.85899999999999999},"min":{"1min":0.32400000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":12.124000000000001,"5min":12.124000000000001,"15min":12.124000000000001},"last":0.38600000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:38:31 2023","interfaces":[{"interface":"back","average":{"1min":1.637,"5min":0.96899999999999997,"15min":0.871},"min":{"1min":0.32400000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.8490000000000002,"5min":11.372,"15min":11.473000000000001},"last":0.29599999999999999},{"interface":"front","average":{"1min":2.133,"5min":1.1679999999999999,"15min":0.94999999999999996},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26900000000000002},"max":{"1min":12.323,"5min":12.323,"15min":12.323},"last":0.33400000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":0.81100000000000005,"15min":1.0449999999999999},"min":{"1min":0.312,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":7.6630000000000003,"5min":7.6630000000000003,"15min":35.792999999999999},"last":0.312},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.88200000000000001,"15min":1.081},"min":{"1min":0.34300000000000003,"5min":0.33800000000000002,"15min":0.33500000000000002},"max":{"1min":2.6579999999999999,"5min":12.531000000000001,"15min":40.012999999999998},"last":0.46200000000000002}]}]},{"osd":1,"up_from":112,"seq":481036337536,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":926588,"kb_used_data":7796,"kb_used_omap":4,"kb_used_meta":918779,"kb_avail":92818564,"statfs":{"total":95995035648,"available":95046209536,"internally_reserved":0,"allocated":7983104,"data_stored":5040164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":940830606},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":0.98699999999999999,"15min":3.1669999999999998},"min":{"1min":0.16400000000000001,"5min":0.16400000000000001,"15min":0.16400000000000001},"max":{"1min":7.1070000000000002,"5min":15.983000000000001,"15min":130.768},"last":0.378},{"interface":"front","average":{"1min":0.78800000000000003,"5min":0.65200000000000002,"15min":2.5150000000000001},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":5.726,"5min":5.8099999999999996,"15min":134.929},"last":0.27300000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:39:08 2023","interfaces":[{"interface":"back","average":{"1min":2.0169999999999999,"5min":3.3479999999999999,"15min":6.7489999999999997},"min":{"1min":0.28799999999999998,"5min":0.184,"15min":0.184},"max":{"1min":8.1379999999999999,"5min":201.03999999999999,"15min":231.67400000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":2.9489999999999998,"5min":3.7050000000000001,"15min":6.8520000000000003},"min":{"1min":0.27000000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":20.202999999999999,"5min":202.12,"15min":232.21000000000001},"last":0.29699999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:39:08 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":5.1740000000000004,"15min":5.9109999999999996},"min":{"1min":0.34100000000000003,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.6579999999999995,"5min":163.94300000000001,"15min":234.404},"last":0.35599999999999998},{"interface":"front","average":{"1min":2.2919999999999998,"5min":5.4690000000000003,"15min":5.9050000000000002},"min":{"1min":0.25600000000000001,"5min":0.224,"15min":0.224},"max":{"1min":9.625,"5min":163.98699999999999,"15min":236.33699999999999},"last":0.39100000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:38:23 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":0.998,"15min":1.4039999999999999},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.28699999999999998},"max":{"1min":3.5939999999999999,"5min":4.7069999999999999,"15min":12.042999999999999},"last":0.71499999999999997},{"interface":"front","average":{"1min":1.034,"5min":0.874,"15min":1.4399999999999999},"min":{"1min":0.33300000000000002,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":7.1779999999999999,"5min":7.1779999999999999,"15min":18.108000000000001},"last":0.58599999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:38:23 2023","interfaces":[{"interface":"back","average":{"1min":4.2169999999999996,"5min":3.2690000000000001,"15min":1.962},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":80.302999999999997,"5min":151.94800000000001,"15min":151.94800000000001},"last":0.72499999999999998},{"interface":"front","average":{"1min":4.7610000000000001,"5min":3.4239999999999999,"15min":2.3130000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":80.254000000000005,"5min":151.821,"15min":151.821},"last":0.626}]},{"osd":6,"last update":"Sat Sep 23 23:38:23 2023","interfaces":[{"interface":"back","average":{"1min":4.5700000000000003,"5min":1.732,"15min":1.921},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.27500000000000002},"max":{"1min":87.837999999999994,"5min":87.837999999999994,"15min":87.837999999999994},"last":0.57899999999999996},{"interface":"front","average":{"1min":5.0910000000000002,"5min":2.1080000000000001,"15min":2.198},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":85.518000000000001,"5min":85.518000000000001,"15min":85.518000000000001},"last":0.68799999999999994}]},{"osd":7,"last update":"Sat Sep 23 23:39:08 2023","interfaces":[{"interface":"back","average":{"1min":3.331,"5min":1.4490000000000001,"15min":2.0150000000000001},"min":{"1min":0.38600000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":35.487000000000002,"5min":35.487000000000002,"15min":78.308000000000007},"last":0.69999999999999996},{"interface":"front","average":{"1min":3.0750000000000002,"5min":1.5089999999999999,"15min":2.0459999999999998},"min":{"1min":0.44400000000000001,"5min":0.36699999999999999,"15min":0.32700000000000001},"max":{"1min":35.146000000000001,"5min":35.146000000000001,"15min":77.204999999999998},"last":0.59899999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024360,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1228220,"kb_used_data":8116,"kb_used_omap":5,"kb_used_meta":1220090,"kb_avail":92516932,"statfs":{"total":95995035648,"available":94737338368,"internally_reserved":0,"allocated":8310784,"data_stored":5361031,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1249372794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:38:40 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":4.0839999999999996,"15min":6.4660000000000002},"min":{"1min":0.22500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":3.9769999999999999,"5min":301.12700000000001,"15min":301.12700000000001},"last":0.25700000000000001},{"interface":"front","average":{"1min":1.579,"5min":3.8439999999999999,"15min":5.6849999999999996},"min":{"1min":0.23499999999999999,"5min":0.20200000000000001,"15min":0.17599999999999999},"max":{"1min":13.141,"5min":297.72399999999999,"15min":297.72399999999999},"last":0.32700000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:39:06 2023","interfaces":[{"interface":"back","average":{"1min":6.6310000000000002,"5min":6.1769999999999996,"15min":6.3029999999999999},"min":{"1min":0.218,"5min":0.214,"15min":0.17199999999999999},"max":{"1min":102.279,"5min":283.07400000000001,"15min":283.07400000000001},"last":0.307},{"interface":"front","average":{"1min":7.2030000000000003,"5min":6.0519999999999996,"15min":6.3360000000000003},"min":{"1min":0.23899999999999999,"5min":0.20799999999999999,"15min":0.17599999999999999},"max":{"1min":102.218,"5min":283.161,"15min":283.161},"last":0.36099999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:38:40 2023","interfaces":[{"interface":"back","average":{"1min":6.2859999999999996,"5min":7.3879999999999999,"15min":6.8490000000000002},"min":{"1min":0.28599999999999998,"5min":0.219,"15min":0.217},"max":{"1min":66.093000000000004,"5min":194.26300000000001,"15min":220.732},"last":0.34200000000000003},{"interface":"front","average":{"1min":6.8170000000000002,"5min":7.3819999999999997,"15min":6.9580000000000002},"min":{"1min":0.36099999999999999,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":66.123999999999995,"5min":194.56299999999999,"15min":218.78200000000001},"last":0.28499999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3560000000000001,"5min":0.75800000000000001,"15min":2.3100000000000001},"min":{"1min":0.32600000000000001,"5min":0.312,"15min":0.28499999999999998},"max":{"1min":13.207000000000001,"5min":13.207000000000001,"15min":138.81299999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.0920000000000001,"15min":2.476},"min":{"1min":0.38200000000000001,"5min":0.32600000000000001,"15min":0.27200000000000002},"max":{"1min":8.718,"5min":9.3670000000000009,"15min":139.19800000000001},"last":0.46899999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:39:16 2023","interfaces":[{"interface":"back","average":{"1min":4.3899999999999997,"5min":1.8089999999999999,"15min":2.3650000000000002},"min":{"1min":0.36699999999999999,"5min":0.32600000000000001,"15min":0.32200000000000001},"max":{"1min":67.010000000000005,"5min":67.010000000000005,"15min":139.119},"last":0.42299999999999999},{"interface":"front","average":{"1min":5.0369999999999999,"5min":1.7809999999999999,"15min":2.2109999999999999},"min":{"1min":0.46000000000000002,"5min":0.32900000000000001,"15min":0.253},"max":{"1min":67.090000000000003,"5min":67.090000000000003,"15min":139.40799999999999},"last":0.503}]},{"osd":6,"last update":"Sat Sep 23 23:39:16 2023","interfaces":[{"interface":"back","average":{"1min":5.0629999999999997,"5min":2.0209999999999999,"15min":2.734},"min":{"1min":0.34899999999999998,"5min":0.33000000000000002,"15min":0.29499999999999998},"max":{"1min":69.043999999999997,"5min":69.043999999999997,"15min":139.16399999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":5.3609999999999998,"5min":1.986,"15min":3.512},"min":{"1min":0.435,"5min":0.37,"15min":0.27100000000000002},"max":{"1min":69.028999999999996,"5min":69.028999999999996,"15min":184.13499999999999},"last":0.52400000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:39:06 2023","interfaces":[{"interface":"back","average":{"1min":5.3540000000000001,"5min":1.72,"15min":2.802},"min":{"1min":0.38100000000000001,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":66.620000000000005,"5min":66.620000000000005,"15min":138.77699999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":4.9279999999999999,"5min":1.9850000000000001,"15min":2.778},"min":{"1min":0.47799999999999998,"5min":0.35399999999999998,"15min":0.33300000000000002},"max":{"1min":66.701999999999998,"5min":66.701999999999998,"15min":139.37899999999999},"last":0.54200000000000004}]}]},{"osd":2,"up_from":199,"seq":854698492181,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":896332,"kb_used_data":8132,"kb_used_omap":7,"kb_used_meta":888184,"kb_avail":92848820,"statfs":{"total":95995035648,"available":95077191680,"internally_reserved":0,"allocated":8327168,"data_stored":5313107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":909500497},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:39:10 2023","interfaces":[{"interface":"back","average":{"1min":1.476,"5min":1.6759999999999999,"15min":2.6589999999999998},"min":{"1min":0.217,"5min":0.186,"15min":0.182},"max":{"1min":8.0050000000000008,"5min":61.950000000000003,"15min":76.942999999999998},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.911,"5min":1.8,"15min":2.6269999999999998},"min":{"1min":0.23699999999999999,"5min":0.19700000000000001,"15min":0.192},"max":{"1min":9.9860000000000007,"5min":61.984999999999999,"15min":76.564999999999998},"last":0.41499999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":3.222,"15min":4.0170000000000003},"min":{"1min":0.27700000000000002,"5min":0.192,"15min":0.192},"max":{"1min":3.7320000000000002,"5min":106.765,"15min":106.765},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.999,"5min":2.8809999999999998,"15min":3.9009999999999998},"min":{"1min":0.29199999999999998,"5min":0.23400000000000001,"15min":0.222},"max":{"1min":4.0449999999999999,"5min":106.732,"15min":106.732},"last":0.39900000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:39:10 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":3.351,"15min":4.343},"min":{"1min":0.30499999999999999,"5min":0.24099999999999999,"15min":0.191},"max":{"1min":11.042999999999999,"5min":106.70399999999999,"15min":106.70399999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.3759999999999999,"5min":3,"15min":4.0880000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":6.0309999999999997,"5min":106.66800000000001,"15min":106.66800000000001},"last":0.432}]},{"osd":4,"last update":"Sat Sep 23 23:39:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.619,"15min":1.7649999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.27700000000000002},"max":{"1min":5.0190000000000001,"5min":62.081000000000003,"15min":62.081000000000003},"last":0.69899999999999995},{"interface":"front","average":{"1min":1.02,"5min":1.778,"15min":1.8320000000000001},"min":{"1min":0.33500000000000002,"5min":0.33400000000000002,"15min":0.28499999999999998},"max":{"1min":5.1630000000000003,"5min":62.072000000000003,"15min":62.072000000000003},"last":0.68799999999999994}]},{"osd":5,"last update":"Sat Sep 23 23:39:10 2023","interfaces":[{"interface":"back","average":{"1min":2.3969999999999998,"5min":3.6560000000000001,"15min":2.7429999999999999},"min":{"1min":0.34599999999999997,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":20.283000000000001,"5min":101.104,"15min":101.104},"last":0.65000000000000002},{"interface":"front","average":{"1min":2.3889999999999998,"5min":3.8420000000000001,"15min":2.7850000000000001},"min":{"1min":0.38700000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":17.318000000000001,"5min":101.404,"15min":101.404},"last":0.71699999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:39:10 2023","interfaces":[{"interface":"back","average":{"1min":1.907,"5min":5.0259999999999998,"15min":2.9609999999999999},"min":{"1min":0.36399999999999999,"5min":0.31,"15min":0.31},"max":{"1min":12.753,"5min":107.018,"15min":107.018},"last":0.74199999999999999},{"interface":"front","average":{"1min":2.286,"5min":3.9460000000000002,"15min":2.6309999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":12.779999999999999,"5min":98.231999999999999,"15min":98.231999999999999},"last":0.61899999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:39:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.673,"15min":1.8779999999999999},"min":{"1min":0.35099999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.266,"5min":62.603000000000002,"15min":62.603000000000002},"last":0.71099999999999997},{"interface":"front","average":{"1min":1.347,"5min":3.089,"15min":2.3149999999999999},"min":{"1min":0.379,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":6.1449999999999996,"5min":106.809,"15min":106.809},"last":0.66000000000000003}]}]},{"osd":3,"up_from":148,"seq":635655160149,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":951604,"kb_used_data":8108,"kb_used_omap":2,"kb_used_meta":943485,"kb_avail":92793548,"statfs":{"total":95995035648,"available":95020593152,"internally_reserved":0,"allocated":8302592,"data_stored":5364668,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":966128693},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.101,"15min":2.1000000000000001},"min":{"1min":0.22700000000000001,"5min":0.184,"15min":0.17699999999999999},"max":{"1min":6.4400000000000004,"5min":17.213000000000001,"15min":62.914000000000001},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.6259999999999999,"5min":1.1399999999999999,"15min":2.4689999999999999},"min":{"1min":0.20300000000000001,"5min":0.188,"15min":0.17199999999999999},"max":{"1min":7.843,"5min":12.202,"15min":63.276000000000003},"last":0.32100000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:38:50 2023","interfaces":[{"interface":"back","average":{"1min":1.694,"5min":1.155,"15min":2.766},"min":{"1min":0.217,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":9.7170000000000005,"5min":9.7170000000000005,"15min":85.242999999999995},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.86899999999999999,"15min":2.6459999999999999},"min":{"1min":0.251,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":6.4169999999999998,"5min":11.228,"15min":85.302000000000007},"last":0.28799999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:39:01 2023","interfaces":[{"interface":"back","average":{"1min":2.6789999999999998,"5min":1.776,"15min":4.0330000000000004},"min":{"1min":0.29099999999999998,"5min":0.23999999999999999,"15min":0.19800000000000001},"max":{"1min":21.725999999999999,"5min":36.362000000000002,"15min":84.724999999999994},"last":0.36199999999999999},{"interface":"front","average":{"1min":2.403,"5min":1.7350000000000001,"15min":3.9279999999999999},"min":{"1min":0.33900000000000002,"5min":0.26800000000000002,"15min":0.214},"max":{"1min":21.687000000000001,"5min":36.158999999999999,"15min":84.567999999999998},"last":0.379}]},{"osd":4,"last update":"Sat Sep 23 23:38:50 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":1.0069999999999999,"15min":1.1879999999999999},"min":{"1min":0.308,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":6.3330000000000002,"5min":11.209,"15min":11.209},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.9019999999999999,"5min":1.167,"15min":1.6499999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.26800000000000002},"max":{"1min":10.917,"5min":10.917,"15min":24.782},"last":0.63800000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:38:50 2023","interfaces":[{"interface":"back","average":{"1min":4.8259999999999996,"5min":3.831,"15min":4.1909999999999998},"min":{"1min":0.373,"5min":0.29499999999999998,"15min":0.28000000000000003},"max":{"1min":79.572999999999993,"5min":196.358,"15min":214.43700000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":4.8499999999999996,"5min":3.8740000000000001,"15min":4.1890000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.30599999999999999},"max":{"1min":79.426000000000002,"5min":196.048,"15min":214.46600000000001},"last":0.64900000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:38:50 2023","interfaces":[{"interface":"back","average":{"1min":1.877,"5min":2.6779999999999999,"15min":2.5950000000000002},"min":{"1min":0.36899999999999999,"5min":0.318,"15min":0.318},"max":{"1min":11.09,"5min":125.94,"15min":125.94},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.855,"5min":2.7679999999999998,"15min":2.5739999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.6319999999999997,"5min":125.955,"15min":125.955},"last":0.60399999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5,"5min":1.1399999999999999,"15min":3.234},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":6.3869999999999996,"5min":11.191000000000001,"15min":192.69},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.3009999999999999,"15min":3.6600000000000001},"min":{"1min":0.42599999999999999,"5min":0.40100000000000002,"15min":0.36699999999999999},"max":{"1min":10.951000000000001,"5min":10.951000000000001,"15min":192.65600000000001},"last":0.58799999999999997}]}]},{"osd":4,"up_from":0,"seq":128849019468,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757831,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":783940,"kb_used_data":6716,"kb_used_omap":0,"kb_used_meta":777216,"kb_avail":92961212,"statfs":{"total":95995035648,"available":95192281088,"internally_reserved":0,"allocated":6877184,"data_stored":5148963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":795869184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:38:19 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.78200000000000003,"15min":1.454},"min":{"1min":0.223,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.2890000000000001,"5min":6.1609999999999996,"15min":63.741999999999997},"last":0.442},{"interface":"front","average":{"1min":1,"5min":1.1779999999999999,"15min":1.964},"min":{"1min":0.26200000000000001,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":4.8579999999999997,"5min":15.414,"15min":138.55199999999999},"last":0.47399999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:38:25 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":0.85599999999999998,"15min":2.21},"min":{"1min":0.27000000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.7480000000000002,"5min":6.1360000000000001,"15min":69.816999999999993},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.1120000000000001,"5min":0.89300000000000002,"15min":2.4249999999999998},"min":{"1min":0.307,"5min":0.28599999999999998,"15min":0.249},"max":{"1min":3.7570000000000001,"5min":4.4859999999999998,"15min":70.268000000000001},"last":0.377}]},{"osd":2,"last update":"Sat Sep 23 23:38:55 2023","interfaces":[{"interface":"back","average":{"1min":2.6499999999999999,"5min":4.992,"15min":4.609},"min":{"1min":0.36599999999999999,"5min":0.28799999999999998,"15min":0.26900000000000002},"max":{"1min":8.7230000000000008,"5min":145.93199999999999,"15min":208.738},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.7649999999999999,"5min":3.4049999999999998,"15min":4.0759999999999996},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.4100000000000001,"5min":66.421000000000006,"15min":208.631},"last":0.498}]},{"osd":3,"last update":"Sat Sep 23 23:38:55 2023","interfaces":[{"interface":"back","average":{"1min":4.0339999999999998,"5min":5.9790000000000001,"15min":4.3490000000000002},"min":{"1min":0.34899999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":46.768999999999998,"5min":143.66300000000001,"15min":143.66300000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":3.9870000000000001,"5min":6.2720000000000002,"15min":4.5229999999999997},"min":{"1min":0.36299999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":46.795000000000002,"5min":143.62200000000001,"15min":143.62200000000001},"last":0.61199999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:38:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":0.90800000000000003,"15min":0.92300000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.251},"max":{"1min":4.4269999999999996,"5min":9.2690000000000001,"15min":19.962},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.92200000000000004,"15min":0.87},"min":{"1min":0.32300000000000001,"5min":0.28899999999999998,"15min":0.22900000000000001},"max":{"1min":4.4169999999999998,"5min":10.057,"15min":20.829999999999998},"last":0.58899999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":2.3490000000000002,"15min":1.556},"min":{"1min":0.28899999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.2350000000000003,"5min":148.36600000000001,"15min":148.36600000000001},"last":0.626},{"interface":"front","average":{"1min":1.1639999999999999,"5min":2.5870000000000002,"15min":1.754},"min":{"1min":0.35599999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.7919999999999998,"5min":148.42500000000001,"15min":148.42500000000001},"last":0.53100000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:38:51 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.2310000000000001,"15min":1.9970000000000001},"min":{"1min":0.33600000000000002,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":5.8559999999999999,"5min":15.497,"15min":179.28800000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.492,"5min":1.252,"15min":2.0579999999999998},"min":{"1min":0.40300000000000002,"5min":0.36699999999999999,"15min":0.34999999999999998},"max":{"1min":5.7729999999999997,"5min":15.653,"15min":179.428},"last":0.65100000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:39:27.211 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T23:39:27.211 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:39:27.212 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T23:39:27.212 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T23:39:27.212 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T23:39:27.212 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T23:39:27.232 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:39:27.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:27.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:39:27.255 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:39:27.350 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:27.378 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:27 smithi138 ceph-mon[193606]: pgmap v1768: 119 pgs: 119 active+clean; 275 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 29 MiB/s wr, 463 op/s 2023-09-23T23:39:27.381 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:27.436 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:27.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:27.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:27.544 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:27.617 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:27.647 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:39:27.658 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 5 16 12175 12159 151.975 135.25 0.00959908 0.00657138 2023-09-23T23:39:27.721 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:27.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:27.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:27.826 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:27.861 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:27.929 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:27.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:27.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:28.033 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:28.064 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:28.123 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:28.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:28.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:28.229 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:28.296 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:28.641 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:28.650 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:39:28.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:28.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:39:28.658 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 6 16 14493 14477 150.789 144.875 0.013392 0.00662248 2023-09-23T23:39:28.756 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:28.823 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:28.837 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:29.196 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:29.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:29 smithi110 ceph-mon[221102]: pgmap v1769: 119 pgs: 119 active+clean; 275 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 27 MiB/s wr, 437 op/s 2023-09-23T23:39:29.197 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:29 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:39:29.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:29 smithi110 ceph-mon[212939]: pgmap v1769: 119 pgs: 119 active+clean; 275 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 27 MiB/s wr, 437 op/s 2023-09-23T23:39:29.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:29 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:39:29.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:29.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:29.309 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:39:29.378 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:29 smithi138 ceph-mon[193606]: pgmap v1769: 119 pgs: 119 active+clean; 275 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 27 MiB/s wr, 437 op/s 2023-09-23T23:39:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:29 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:39:29.658 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 7 16 16118 16102 143.756 101.562 0.00692096 0.00694966 2023-09-23T23:39:30.629 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:39:30.629 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "" 2023-09-23T23:39:30.629 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:39:30.660 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 8 16 18216 18200 142.151 131.125 0.00837491 0.0070274 2023-09-23T23:39:30.907 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:39:30.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:30.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:39:31.013 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:39:31.066 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:39:31.076 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:31.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:31 smithi110 ceph-mon[212939]: pgmap v1770: 119 pgs: 119 active+clean; 727 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-09-23T23:39:31.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:31 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4284905360' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:39:31.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:31 smithi110 ceph-mon[221102]: pgmap v1770: 119 pgs: 119 active+clean; 727 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-09-23T23:39:31.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:31 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4284905360' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:39:31.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:31 smithi138 ceph-mon[193606]: pgmap v1770: 119 pgs: 119 active+clean; 727 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-09-23T23:39:31.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:31 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4284905360' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:39:31.662 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 9 16 19380 19364 134.412 72.75 0.00548491 0.00743267 2023-09-23T23:39:32.040 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:39:32.040 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:39:32.040 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4284905360' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:39:32.040 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[212939]: osdmap e622: 8 total, 8 up, 7 in 2023-09-23T23:39:32.040 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[212939]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.041 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[212939]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.041 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[212939]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.041 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[212939]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.041 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[212939]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.041 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[212939]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:32 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:39:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:32 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:39:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:32 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4284905360' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:39:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:32 smithi138 ceph-mon[193606]: osdmap e622: 8 total, 8 up, 7 in 2023-09-23T23:39:32.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:32 smithi138 ceph-mon[193606]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:32 smithi138 ceph-mon[193606]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:32 smithi138 ceph-mon[193606]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:32 smithi138 ceph-mon[193606]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:32 smithi138 ceph-mon[193606]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:32 smithi138 ceph-mon[193606]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:39:32.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:39:32.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4284905360' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:39:32.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[221102]: osdmap e622: 8 total, 8 up, 7 in 2023-09-23T23:39:32.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[221102]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[221102]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[221102]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[221102]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[221102]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:32 smithi110 ceph-mon[221102]: failed to encode map e622 with expected crc 2023-09-23T23:39:32.662 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 10 16 20793 20777 129.8 88.3125 0.00926481 0.00769806 2023-09-23T23:39:33.046 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:39:33.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:33.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:39:33.152 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:33.234 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:33.308 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:33 smithi110 ceph-mon[221102]: pgmap v1772: 119 pgs: 119 active+clean; 983 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T23:39:33.308 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:33 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:39:33.310 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:33 smithi110 ceph-mon[212939]: pgmap v1772: 119 pgs: 119 active+clean; 983 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T23:39:33.310 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:33 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:39:33.415 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:33.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:33.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:33 smithi138 ceph-mon[193606]: pgmap v1772: 119 pgs: 119 active+clean; 983 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T23:39:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:33 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:39:33.526 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:33.563 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:33.627 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:33.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:33.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:33.663 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 11 16 22348 22332 126.827 97.1875 0.00954124 0.00787879 2023-09-23T23:39:33.735 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:33.766 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:33.827 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:33.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:33.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:33.887 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:39:33.887 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:39:33.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:39:33.887 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:39:33.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.887 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71524 13.2 13:443908cd:::benchmark_data_smithi110_541928_object71523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e555)", 2023-09-23T23:39:33.888 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.543571+0000", 2023-09-23T23:39:33.888 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 343.33791142799998, 2023-09-23T23:39:33.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.309790005, 2023-09-23T23:39:33.888 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:33.888 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:33.888 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:33.888 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:33.888 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:33.888 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71524 2023-09-23T23:39:33.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.888 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:33.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:33.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543571+0000", 2023-09-23T23:39:33.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:33.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543571+0000", 2023-09-23T23:39:33.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:33.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543570+0000", 2023-09-23T23:39:33.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:39:33.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:33.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543592+0000", 2023-09-23T23:39:33.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1195e-05 2023-09-23T23:39:33.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:33.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543598+0000", 2023-09-23T23:39:33.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8190000000000003e-06 2023-09-23T23:39:33.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:33.891 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543604+0000", 2023-09-23T23:39:33.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.451e-06 2023-09-23T23:39:33.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.891 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.544968+0000", 2023-09-23T23:39:33.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001364007 2023-09-23T23:39:33.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.891 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.544984+0000", 2023-09-23T23:39:33.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5699999999999999e-05 2023-09-23T23:39:33.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527908+0000", 2023-09-23T23:39:33.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98292410900000005 2023-09-23T23:39:33.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527920+0000", 2023-09-23T23:39:33.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2852e-05 2023-09-23T23:39:33.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.815907+0000", 2023-09-23T23:39:33.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.287986606 2023-09-23T23:39:33.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:33.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.816011+0000", 2023-09-23T23:39:33.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103791 2023-09-23T23:39:33.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:39:33.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.816116+0000", 2023-09-23T23:39:33.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104955 2023-09-23T23:39:33.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:33.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.825412+0000", 2023-09-23T23:39:33.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.009295757 2023-09-23T23:39:33.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.847154+0000", 2023-09-23T23:39:33.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021742053000000001 2023-09-23T23:39:33.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.895 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.850549+0000", 2023-09-23T23:39:33.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033955470000000001 2023-09-23T23:39:33.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:33.895 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.850599+0000", 2023-09-23T23:39:33.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9645000000000003e-05 2023-09-23T23:39:33.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:33.895 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.853361+0000", 2023-09-23T23:39:33.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027620710000000001 2023-09-23T23:39:33.895 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.896 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:33.896 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.896 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71514 13.3 13:cdfd1184:::benchmark_data_smithi110_541928_object71513:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:39:33.896 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.547244+0000", 2023-09-23T23:39:33.896 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 343.334238304, 2023-09-23T23:39:33.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4756996080000002, 2023-09-23T23:39:33.896 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:33.896 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:39:33.896 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:33.896 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:33.896 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:33.897 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71514 2023-09-23T23:39:33.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.897 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:33.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:33.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547244+0000", 2023-09-23T23:39:33.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:33.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547244+0000", 2023-09-23T23:39:33.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:33.898 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547242+0000", 2023-09-23T23:39:33.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:39:33.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:33.898 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547281+0000", 2023-09-23T23:39:33.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8605000000000001e-05 2023-09-23T23:39:33.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:33.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547302+0000", 2023-09-23T23:39:33.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0873000000000002e-05 2023-09-23T23:39:33.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:33.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547312+0000", 2023-09-23T23:39:33.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.859e-06 2023-09-23T23:39:33.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.548286+0000", 2023-09-23T23:39:33.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00097434599999999996 2023-09-23T23:39:33.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.548298+0000", 2023-09-23T23:39:33.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1881e-05 2023-09-23T23:39:33.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527773+0000", 2023-09-23T23:39:33.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97947559500000003 2023-09-23T23:39:33.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527787+0000", 2023-09-23T23:39:33.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3329999999999999e-05 2023-09-23T23:39:33.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.901 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.021377+0000", 2023-09-23T23:39:33.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.493589799 2023-09-23T23:39:33.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.901 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:33.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.022944+0000", 2023-09-23T23:39:33.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015670910000000001 2023-09-23T23:39:33.901 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.901 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:33.902 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.902 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71522 13.3 13:c49d694d:::benchmark_data_smithi110_541928_object71521:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:39:33.902 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.547319+0000", 2023-09-23T23:39:33.902 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 343.33416295699999, 2023-09-23T23:39:33.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5115680760000001, 2023-09-23T23:39:33.902 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:33.902 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:33.902 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:33.902 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:33.902 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:33.903 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71522 2023-09-23T23:39:33.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.903 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:33.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:33.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547319+0000", 2023-09-23T23:39:33.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:33.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547319+0000", 2023-09-23T23:39:33.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:33.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547319+0000", 2023-09-23T23:39:33.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:39:33.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:33.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547342+0000", 2023-09-23T23:39:33.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2857000000000002e-05 2023-09-23T23:39:33.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:33.905 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547348+0000", 2023-09-23T23:39:33.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8129999999999998e-06 2023-09-23T23:39:33.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.905 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:33.905 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547354+0000", 2023-09-23T23:39:33.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5090000000000001e-06 2023-09-23T23:39:33.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.905 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.905 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.548319+0000", 2023-09-23T23:39:33.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00096521499999999995 2023-09-23T23:39:33.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.548327+0000", 2023-09-23T23:39:33.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7079999999999997e-06 2023-09-23T23:39:33.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527817+0000", 2023-09-23T23:39:33.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97948980500000005 2023-09-23T23:39:33.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527825+0000", 2023-09-23T23:39:33.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3440000000000001e-06 2023-09-23T23:39:33.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.026234+0000", 2023-09-23T23:39:33.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4984093380000001 2023-09-23T23:39:33.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:33.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.027045+0000", 2023-09-23T23:39:33.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00081087800000000001 2023-09-23T23:39:33.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,7", 2023-09-23T23:39:33.908 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.027252+0000", 2023-09-23T23:39:33.908 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000206553 2023-09-23T23:39:33.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.908 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.038654+0000", 2023-09-23T23:39:33.908 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011401669 2023-09-23T23:39:33.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:33.909 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.046827+0000", 2023-09-23T23:39:33.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081732569999999997 2023-09-23T23:39:33.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.909 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.909 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.909 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.055749+0000", 2023-09-23T23:39:33.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0089217600000000008 2023-09-23T23:39:33.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.909 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.909 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:33.909 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.055791+0000", 2023-09-23T23:39:33.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2493999999999997e-05 2023-09-23T23:39:33.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:33.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.058887+0000", 2023-09-23T23:39:33.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030964320000000001 2023-09-23T23:39:33.910 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.910 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:33.910 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.910 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.910 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71530 13.2 13:4cd5e89c:::benchmark_data_smithi110_541928_object71529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:39:33.910 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.571622+0000", 2023-09-23T23:39:33.910 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 343.309860426, 2023-09-23T23:39:33.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2848813030000001, 2023-09-23T23:39:33.911 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:33.911 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:33.911 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:33.911 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:33.911 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:33.911 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71530 2023-09-23T23:39:33.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.911 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:33.911 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.911 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:33.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571622+0000", 2023-09-23T23:39:33.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:33.912 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571622+0000", 2023-09-23T23:39:33.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:33.912 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571620+0000", 2023-09-23T23:39:33.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:33.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:33.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571678+0000", 2023-09-23T23:39:33.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7633000000000002e-05 2023-09-23T23:39:33.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.913 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:33.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571696+0000", 2023-09-23T23:39:33.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8215000000000001e-05 2023-09-23T23:39:33.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.913 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:33.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571714+0000", 2023-09-23T23:39:33.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7839e-05 2023-09-23T23:39:33.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571774+0000", 2023-09-23T23:39:33.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0322999999999998e-05 2023-09-23T23:39:33.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571784+0000", 2023-09-23T23:39:33.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1719999999999996e-06 2023-09-23T23:39:33.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527947+0000", 2023-09-23T23:39:33.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95616350999999999 2023-09-23T23:39:33.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527957+0000", 2023-09-23T23:39:33.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9369999999999999e-06 2023-09-23T23:39:33.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.820763+0000", 2023-09-23T23:39:33.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.292805757 2023-09-23T23:39:33.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:33.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.820924+0000", 2023-09-23T23:39:33.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016086599999999999 2023-09-23T23:39:33.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:39:33.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.821115+0000", 2023-09-23T23:39:33.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000191137 2023-09-23T23:39:33.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:33.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.825434+0000", 2023-09-23T23:39:33.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043196479999999997 2023-09-23T23:39:33.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.847182+0000", 2023-09-23T23:39:33.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021747295999999999 2023-09-23T23:39:33.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.853421+0000", 2023-09-23T23:39:33.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.006239306 2023-09-23T23:39:33.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:33.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.853476+0000", 2023-09-23T23:39:33.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4929000000000002e-05 2023-09-23T23:39:33.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.918 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:33.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.856503+0000", 2023-09-23T23:39:33.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030272390000000001 2023-09-23T23:39:33.918 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.918 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:33.918 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.919 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71537 13.3 13:cd57dd3d:::benchmark_data_smithi110_541928_object71536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:39:33.919 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.580860+0000", 2023-09-23T23:39:33.919 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 343.30062264700001, 2023-09-23T23:39:33.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4840317629999999, 2023-09-23T23:39:33.919 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:33.919 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:33.919 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:33.919 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:33.919 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:33.919 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71537 2023-09-23T23:39:33.919 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.920 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:33.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.920 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:33.920 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580860+0000", 2023-09-23T23:39:33.920 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.920 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:33.920 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580860+0000", 2023-09-23T23:39:33.920 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.920 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:33.920 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580858+0000", 2023-09-23T23:39:33.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:39:33.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:33.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580883+0000", 2023-09-23T23:39:33.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5388000000000002e-05 2023-09-23T23:39:33.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:33.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580901+0000", 2023-09-23T23:39:33.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7496e-05 2023-09-23T23:39:33.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:33.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580911+0000", 2023-09-23T23:39:33.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0093e-05 2023-09-23T23:39:33.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580968+0000", 2023-09-23T23:39:33.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6971000000000001e-05 2023-09-23T23:39:33.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580979+0000", 2023-09-23T23:39:33.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1423999999999999e-05 2023-09-23T23:39:33.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527852+0000", 2023-09-23T23:39:33.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94687271200000001 2023-09-23T23:39:33.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527860+0000", 2023-09-23T23:39:33.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8159999999999994e-06 2023-09-23T23:39:33.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.029524+0000", 2023-09-23T23:39:33.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.501664385 2023-09-23T23:39:33.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:33.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.029705+0000", 2023-09-23T23:39:33.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000181423 2023-09-23T23:39:33.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,7", 2023-09-23T23:39:33.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.029886+0000", 2023-09-23T23:39:33.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018058199999999999 2023-09-23T23:39:33.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.038692+0000", 2023-09-23T23:39:33.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0088061979999999995 2023-09-23T23:39:33.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:33.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.046831+0000", 2023-09-23T23:39:33.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081387630000000002 2023-09-23T23:39:33.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.058949+0000", 2023-09-23T23:39:33.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012117909 2023-09-23T23:39:33.926 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:33.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.059022+0000", 2023-09-23T23:39:33.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2758999999999999e-05 2023-09-23T23:39:33.926 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:33.927 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.064891+0000", 2023-09-23T23:39:33.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058699099999999999 2023-09-23T23:39:33.927 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.927 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:33.927 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.927 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.927 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.927 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71540 13.2 13:4d0dc391:::benchmark_data_smithi110_541928_object71539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:39:33.927 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.586761+0000", 2023-09-23T23:39:33.927 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 343.29472163100002, 2023-09-23T23:39:33.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4196291790000002, 2023-09-23T23:39:33.927 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:33.928 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:33.928 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:33.928 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:33.928 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:33.928 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71540 2023-09-23T23:39:33.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.928 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:33.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.928 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:33.928 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586761+0000", 2023-09-23T23:39:33.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:33.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586761+0000", 2023-09-23T23:39:33.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:33.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586758+0000", 2023-09-23T23:39:33.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-09-23T23:39:33.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:33.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586795+0000", 2023-09-23T23:39:33.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6177999999999998e-05 2023-09-23T23:39:33.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:33.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586821+0000", 2023-09-23T23:39:33.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6081999999999999e-05 2023-09-23T23:39:33.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:33.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586834+0000", 2023-09-23T23:39:33.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3441999999999999e-05 2023-09-23T23:39:33.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586917+0000", 2023-09-23T23:39:33.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2551000000000003e-05 2023-09-23T23:39:33.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586938+0000", 2023-09-23T23:39:33.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1016e-05 2023-09-23T23:39:33.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527980+0000", 2023-09-23T23:39:33.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94104277599999997 2023-09-23T23:39:33.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.932 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.932 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527989+0000", 2023-09-23T23:39:33.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7439999999999996e-06 2023-09-23T23:39:33.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.932 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.932 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.823717+0000", 2023-09-23T23:39:33.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.295727691 2023-09-23T23:39:33.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:33.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.823855+0000", 2023-09-23T23:39:33.933 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013797700000000001 2023-09-23T23:39:33.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:39:33.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.823956+0000", 2023-09-23T23:39:33.933 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010142899999999999 2023-09-23T23:39:33.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:33.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.825554+0000", 2023-09-23T23:39:33.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015978839999999999 2023-09-23T23:39:33.934 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.934 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.934 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.934 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.849319+0000", 2023-09-23T23:39:33.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023764928000000001 2023-09-23T23:39:33.934 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.934 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.934 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.934 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.856559+0000", 2023-09-23T23:39:33.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0072398460000000003 2023-09-23T23:39:33.934 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:33.935 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.856611+0000", 2023-09-23T23:39:33.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1752999999999997e-05 2023-09-23T23:39:33.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:33.935 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.006390+0000", 2023-09-23T23:39:33.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14977916799999999 2023-09-23T23:39:33.935 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.935 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:33.935 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.936 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89738 13.3 13:cade2564:::benchmark_data_smithi110_541928_object1958:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:39:33.936 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:34.801877+0000", 2023-09-23T23:39:33.936 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 299.07960504900001, 2023-09-23T23:39:33.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5955823369999997, 2023-09-23T23:39:33.936 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:33.936 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:33.936 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:33.936 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:33.936 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:33.936 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89738 2023-09-23T23:39:33.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.936 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:33.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:33.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801877+0000", 2023-09-23T23:39:33.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:33.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801877+0000", 2023-09-23T23:39:33.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:33.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801876+0000", 2023-09-23T23:39:33.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:33.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:33.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801883+0000", 2023-09-23T23:39:33.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8959999999999997e-06 2023-09-23T23:39:33.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:33.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801895+0000", 2023-09-23T23:39:33.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2709e-05 2023-09-23T23:39:33.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:33.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801905+0000", 2023-09-23T23:39:33.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1589999999999999e-06 2023-09-23T23:39:33.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801984+0000", 2023-09-23T23:39:33.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9043999999999997e-05 2023-09-23T23:39:33.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801997+0000", 2023-09-23T23:39:33.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3294e-05 2023-09-23T23:39:33.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731046+0000", 2023-09-23T23:39:33.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.92904877500000005 2023-09-23T23:39:33.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731060+0000", 2023-09-23T23:39:33.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4042e-05 2023-09-23T23:39:33.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.941 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.941 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.371733+0000", 2023-09-23T23:39:33.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6406730409999999 2023-09-23T23:39:33.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.941 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:39:33.941 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.372202+0000", 2023-09-23T23:39:33.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000469529 2023-09-23T23:39:33.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.941 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.941 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.381823+0000", 2023-09-23T23:39:33.942 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0096204110000000006 2023-09-23T23:39:33.942 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.942 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:33.942 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.381932+0000", 2023-09-23T23:39:33.942 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010885 2023-09-23T23:39:33.942 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.942 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:39:33.942 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.382080+0000", 2023-09-23T23:39:33.942 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000148555 2023-09-23T23:39:33.942 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.943 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394009+0000", 2023-09-23T23:39:33.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011928887000000001 2023-09-23T23:39:33.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.943 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394057+0000", 2023-09-23T23:39:33.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8157e-05 2023-09-23T23:39:33.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.943 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:33.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394066+0000", 2023-09-23T23:39:33.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7320000000000004e-06 2023-09-23T23:39:33.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394132+0000", 2023-09-23T23:39:33.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5845000000000004e-05 2023-09-23T23:39:33.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:33.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394208+0000", 2023-09-23T23:39:33.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6352000000000004e-05 2023-09-23T23:39:33.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:33.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397460+0000", 2023-09-23T23:39:33.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032514520000000002 2023-09-23T23:39:33.945 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.945 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:33.945 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.945 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89748 13.3 13:c6535c6c:::benchmark_data_smithi110_541928_object1968:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:39:33.945 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:34.962737+0000", 2023-09-23T23:39:33.945 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 298.91874554999998, 2023-09-23T23:39:33.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.437414017, 2023-09-23T23:39:33.945 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:33.945 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:33.946 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:33.946 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:33.946 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:33.946 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89748 2023-09-23T23:39:33.946 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.946 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:33.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.946 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:33.946 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962737+0000", 2023-09-23T23:39:33.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.946 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.946 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:33.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962737+0000", 2023-09-23T23:39:33.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:33.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962735+0000", 2023-09-23T23:39:33.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:39:33.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:33.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962743+0000", 2023-09-23T23:39:33.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7700000000000001e-06 2023-09-23T23:39:33.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:33.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962756+0000", 2023-09-23T23:39:33.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3013999999999999e-05 2023-09-23T23:39:33.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:33.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962763+0000", 2023-09-23T23:39:33.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7260000000000002e-06 2023-09-23T23:39:33.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962867+0000", 2023-09-23T23:39:33.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103978 2023-09-23T23:39:33.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962878+0000", 2023-09-23T23:39:33.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0927999999999999e-05 2023-09-23T23:39:33.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731084+0000", 2023-09-23T23:39:33.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.76820582999999998 2023-09-23T23:39:33.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731093+0000", 2023-09-23T23:39:33.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6149999999999997e-06 2023-09-23T23:39:33.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.375727+0000", 2023-09-23T23:39:33.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6446342889999999 2023-09-23T23:39:33.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:39:33.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.376199+0000", 2023-09-23T23:39:33.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047155199999999997 2023-09-23T23:39:33.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.390857+0000", 2023-09-23T23:39:33.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014658200999999999 2023-09-23T23:39:33.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:33.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.390980+0000", 2023-09-23T23:39:33.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000122774 2023-09-23T23:39:33.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:39:33.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.391164+0000", 2023-09-23T23:39:33.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018492700000000001 2023-09-23T23:39:33.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397524+0000", 2023-09-23T23:39:33.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063597879999999999 2023-09-23T23:39:33.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:33.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397564+0000", 2023-09-23T23:39:33.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9746999999999997e-05 2023-09-23T23:39:33.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397611+0000", 2023-09-23T23:39:33.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7403e-05 2023-09-23T23:39:33.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397668+0000", 2023-09-23T23:39:33.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6178999999999999e-05 2023-09-23T23:39:33.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.954 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:33.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397732+0000", 2023-09-23T23:39:33.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4548999999999995e-05 2023-09-23T23:39:33.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.954 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:33.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.400151+0000", 2023-09-23T23:39:33.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024186429999999998 2023-09-23T23:39:33.954 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.954 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:33.954 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.955 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89756 13.3 13:cf60a43c:::benchmark_data_smithi110_541928_object1976:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:39:33.955 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.051526+0000", 2023-09-23T23:39:33.955 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 298.82995677700001, 2023-09-23T23:39:33.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4865002390000002, 2023-09-23T23:39:33.955 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:33.955 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:33.955 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:33.955 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:33.955 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:33.955 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89756 2023-09-23T23:39:33.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.956 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:33.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:33.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051526+0000", 2023-09-23T23:39:33.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:33.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051526+0000", 2023-09-23T23:39:33.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:33.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051524+0000", 2023-09-23T23:39:33.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:33.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:33.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051532+0000", 2023-09-23T23:39:33.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6310000000000003e-06 2023-09-23T23:39:33.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:33.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051559+0000", 2023-09-23T23:39:33.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7444999999999999e-05 2023-09-23T23:39:33.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:33.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051568+0000", 2023-09-23T23:39:33.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8310000000000007e-06 2023-09-23T23:39:33.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051616+0000", 2023-09-23T23:39:33.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8168999999999999e-05 2023-09-23T23:39:33.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051624+0000", 2023-09-23T23:39:33.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2290000000000003e-06 2023-09-23T23:39:33.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731118+0000", 2023-09-23T23:39:33.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.67949406400000001 2023-09-23T23:39:33.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731126+0000", 2023-09-23T23:39:33.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0949999999999996e-06 2023-09-23T23:39:33.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.400190+0000", 2023-09-23T23:39:33.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.669063559 2023-09-23T23:39:33.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:39:33.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.400555+0000", 2023-09-23T23:39:33.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036511099999999999 2023-09-23T23:39:33.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.416430+0000", 2023-09-23T23:39:33.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015875185999999999 2023-09-23T23:39:33.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:33.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.416532+0000", 2023-09-23T23:39:33.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000101456 2023-09-23T23:39:33.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:39:33.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.416657+0000", 2023-09-23T23:39:33.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012510200000000001 2023-09-23T23:39:33.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:33.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.434768+0000", 2023-09-23T23:39:33.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018111008000000001 2023-09-23T23:39:33.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.434835+0000", 2023-09-23T23:39:33.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7475e-05 2023-09-23T23:39:33.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.534925+0000", 2023-09-23T23:39:33.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.100090106 2023-09-23T23:39:33.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.534976+0000", 2023-09-23T23:39:33.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0603999999999999e-05 2023-09-23T23:39:33.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:33.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.535014+0000", 2023-09-23T23:39:33.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8379e-05 2023-09-23T23:39:33.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:33.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.538026+0000", 2023-09-23T23:39:33.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003011391 2023-09-23T23:39:33.964 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.964 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:33.964 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.964 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89769 13.3 13:c7ccd55e:::benchmark_data_smithi110_541928_object1989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:39:33.964 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.139666+0000", 2023-09-23T23:39:33.964 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 298.74181624200003, 2023-09-23T23:39:33.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4017686810000001, 2023-09-23T23:39:33.964 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:33.964 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:33.965 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:33.965 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:33.965 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:33.965 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89769 2023-09-23T23:39:33.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.965 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:33.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:33.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139666+0000", 2023-09-23T23:39:33.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:33.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139666+0000", 2023-09-23T23:39:33.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:33.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139665+0000", 2023-09-23T23:39:33.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:33.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:33.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139673+0000", 2023-09-23T23:39:33.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1049999999999995e-06 2023-09-23T23:39:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139689+0000", 2023-09-23T23:39:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6656000000000001e-05 2023-09-23T23:39:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139701+0000", 2023-09-23T23:39:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1127e-05 2023-09-23T23:39:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139754+0000", 2023-09-23T23:39:33.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3362000000000002e-05 2023-09-23T23:39:33.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139764+0000", 2023-09-23T23:39:33.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0474e-05 2023-09-23T23:39:33.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731150+0000", 2023-09-23T23:39:33.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.59138585200000005 2023-09-23T23:39:33.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.735349+0000", 2023-09-23T23:39:33.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041989519999999997 2023-09-23T23:39:33.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.400620+0000", 2023-09-23T23:39:33.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6652705239999999 2023-09-23T23:39:33.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:39:33.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.401065+0000", 2023-09-23T23:39:33.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00044538800000000001 2023-09-23T23:39:33.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.424058+0000", 2023-09-23T23:39:33.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022993085999999999 2023-09-23T23:39:33.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:33.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.424146+0000", 2023-09-23T23:39:33.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7412999999999998e-05 2023-09-23T23:39:33.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:39:33.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.424253+0000", 2023-09-23T23:39:33.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107246 2023-09-23T23:39:33.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:33.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.440042+0000", 2023-09-23T23:39:33.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015789306999999999 2023-09-23T23:39:33.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.443102+0000", 2023-09-23T23:39:33.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030595700000000002 2023-09-23T23:39:33.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.538201+0000", 2023-09-23T23:39:33.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.095099702999999994 2023-09-23T23:39:33.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.538249+0000", 2023-09-23T23:39:33.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7549000000000002e-05 2023-09-23T23:39:33.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:33.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.538328+0000", 2023-09-23T23:39:33.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8671000000000006e-05 2023-09-23T23:39:33.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:33.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.541435+0000", 2023-09-23T23:39:33.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031071649999999998 2023-09-23T23:39:33.973 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.973 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:33.973 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.974 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89790 13.3 13:ce3e2df6:::benchmark_data_smithi110_541928_object2010:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:39:33.974 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.273997+0000", 2023-09-23T23:39:33.974 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 298.60748565300003, 2023-09-23T23:39:33.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.270580657, 2023-09-23T23:39:33.974 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:33.974 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:33.974 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:33.974 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:33.974 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:33.974 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89790 2023-09-23T23:39:33.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.975 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:33.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:33.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.273997+0000", 2023-09-23T23:39:33.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:33.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.273997+0000", 2023-09-23T23:39:33.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:33.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.273995+0000", 2023-09-23T23:39:33.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:33.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:33.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274004+0000", 2023-09-23T23:39:33.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7940000000000008e-06 2023-09-23T23:39:33.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:33.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274017+0000", 2023-09-23T23:39:33.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3405e-05 2023-09-23T23:39:33.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:33.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274027+0000", 2023-09-23T23:39:33.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7200000000000001e-06 2023-09-23T23:39:33.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274086+0000", 2023-09-23T23:39:33.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.94e-05 2023-09-23T23:39:33.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274101+0000", 2023-09-23T23:39:33.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4109e-05 2023-09-23T23:39:33.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.735405+0000", 2023-09-23T23:39:33.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.46130481400000001 2023-09-23T23:39:33.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.735419+0000", 2023-09-23T23:39:33.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3217000000000001e-05 2023-09-23T23:39:33.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.401121+0000", 2023-09-23T23:39:33.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6657021570000001 2023-09-23T23:39:33.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:39:33.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.401570+0000", 2023-09-23T23:39:33.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000449343 2023-09-23T23:39:33.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.431522+0000", 2023-09-23T23:39:33.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029951951000000001 2023-09-23T23:39:33.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:33.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.431654+0000", 2023-09-23T23:39:33.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013153199999999999 2023-09-23T23:39:33.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:39:33.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.431813+0000", 2023-09-23T23:39:33.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000159937 2023-09-23T23:39:33.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:33.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.451910+0000", 2023-09-23T23:39:33.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02009613 2023-09-23T23:39:33.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.451987+0000", 2023-09-23T23:39:33.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7597999999999998e-05 2023-09-23T23:39:33.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.541568+0000", 2023-09-23T23:39:33.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.089580504000000005 2023-09-23T23:39:33.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.541608+0000", 2023-09-23T23:39:33.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0274000000000001e-05 2023-09-23T23:39:33.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:33.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.541675+0000", 2023-09-23T23:39:33.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6890000000000005e-05 2023-09-23T23:39:33.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:33.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.544577+0000", 2023-09-23T23:39:33.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029024509999999999 2023-09-23T23:39:33.983 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.983 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:33.983 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.983 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89805 13.3 13:c4e9783b:::benchmark_data_smithi110_541928_object2025:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e571)", 2023-09-23T23:39:33.983 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.955885+0000", 2023-09-23T23:39:33.984 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 297.92559746199998, 2023-09-23T23:39:33.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.592296927, 2023-09-23T23:39:33.984 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:33.984 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:33.984 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:33.984 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:33.984 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:33.984 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89805 2023-09-23T23:39:33.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.984 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:33.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:33.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955885+0000", 2023-09-23T23:39:33.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:33.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955885+0000", 2023-09-23T23:39:33.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:33.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955883+0000", 2023-09-23T23:39:33.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:33.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:33.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955891+0000", 2023-09-23T23:39:33.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0209999999999999e-06 2023-09-23T23:39:33.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:33.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955908+0000", 2023-09-23T23:39:33.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.643e-05 2023-09-23T23:39:33.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:33.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955917+0000", 2023-09-23T23:39:33.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2089999999999994e-06 2023-09-23T23:39:33.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955987+0000", 2023-09-23T23:39:33.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9494999999999998e-05 2023-09-23T23:39:33.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.956002+0000", 2023-09-23T23:39:33.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5223e-05 2023-09-23T23:39:33.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.401618+0000", 2023-09-23T23:39:33.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4456160360000001 2023-09-23T23:39:33.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:39:33.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.402009+0000", 2023-09-23T23:39:33.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039078399999999998 2023-09-23T23:39:33.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.440091+0000", 2023-09-23T23:39:33.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038082498999999999 2023-09-23T23:39:33.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:33.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.440216+0000", 2023-09-23T23:39:33.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012535399999999999 2023-09-23T23:39:33.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:39:33.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.440388+0000", 2023-09-23T23:39:33.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017125300000000001 2023-09-23T23:39:33.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:33.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.457767+0000", 2023-09-23T23:39:33.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017379131999999999 2023-09-23T23:39:33.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.460556+0000", 2023-09-23T23:39:33.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027891050000000001 2023-09-23T23:39:33.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.544650+0000", 2023-09-23T23:39:33.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.084093672999999994 2023-09-23T23:39:33.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.544694+0000", 2023-09-23T23:39:33.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4366000000000002e-05 2023-09-23T23:39:33.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:33.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.544759+0000", 2023-09-23T23:39:33.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5371999999999997e-05 2023-09-23T23:39:33.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:33.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.548182+0000", 2023-09-23T23:39:33.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034224870000000001 2023-09-23T23:39:33.992 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.992 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:33.992 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:33.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.992 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89823 13.3 13:c65a1b7a:::benchmark_data_smithi110_541928_object2043:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e571)", 2023-09-23T23:39:33.992 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.990158+0000", 2023-09-23T23:39:33.992 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 297.89132446799999, 2023-09-23T23:39:33.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5615935460000001, 2023-09-23T23:39:33.992 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:33.992 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:33.992 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:33.992 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:33.993 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:33.993 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89823 2023-09-23T23:39:33.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.993 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:33.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:33.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990158+0000", 2023-09-23T23:39:33.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:33.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990158+0000", 2023-09-23T23:39:33.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:33.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:33.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990157+0000", 2023-09-23T23:39:33.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:39:33.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:33.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990162+0000", 2023-09-23T23:39:33.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1359999999999996e-06 2023-09-23T23:39:33.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:33.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990174+0000", 2023-09-23T23:39:33.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1695000000000001e-05 2023-09-23T23:39:33.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:33.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990184+0000", 2023-09-23T23:39:33.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0465000000000001e-05 2023-09-23T23:39:33.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990231+0000", 2023-09-23T23:39:33.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6329000000000003e-05 2023-09-23T23:39:33.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:39:33.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990240+0000", 2023-09-23T23:39:33.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5310000000000007e-06 2023-09-23T23:39:33.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.402069+0000", 2023-09-23T23:39:33.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.411828474 2023-09-23T23:39:33.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:39:33.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.402423+0000", 2023-09-23T23:39:33.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035466000000000002 2023-09-23T23:39:33.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:33.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.448598+0000", 2023-09-23T23:39:33.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.046175040000000001 2023-09-23T23:39:33.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:33.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.448732+0000", 2023-09-23T23:39:33.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013384599999999999 2023-09-23T23:39:33.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:39:33.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.448935+0000", 2023-09-23T23:39:33.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020236700000000001 2023-09-23T23:39:33.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:33.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.460571+0000", 2023-09-23T23:39:33.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011636069000000001 2023-09-23T23:39:33.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.468058+0000", 2023-09-23T23:39:33.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074870570000000001 2023-09-23T23:39:33.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.548273+0000", 2023-09-23T23:39:33.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.080215195000000003 2023-09-23T23:39:33.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:33.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:33.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:33.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.548328+0000", 2023-09-23T23:39:33.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4493000000000002e-05 2023-09-23T23:39:33.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:34.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.548424+0000", 2023-09-23T23:39:34.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6016999999999999e-05 2023-09-23T23:39:34.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:34.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.551751+0000", 2023-09-23T23:39:34.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033278750000000001 2023-09-23T23:39:34.000 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:34.000 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:34.000 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:34.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.001 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:95149 13.d 13:bc7a1628:::benchmark_data_smithi110_541928_object7369:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T23:39:34.001 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:53.232568+0000", 2023-09-23T23:39:34.001 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 280.64891415900001, 2023-09-23T23:39:34.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4660194620000002, 2023-09-23T23:39:34.001 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:34.001 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:34.001 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:34.001 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:34.001 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:34.001 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95149 2023-09-23T23:39:34.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.002 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:34.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:34.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232568+0000", 2023-09-23T23:39:34.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:34.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:34.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232568+0000", 2023-09-23T23:39:34.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:34.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:34.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232567+0000", 2023-09-23T23:39:34.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:34.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:34.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232574+0000", 2023-09-23T23:39:34.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.824e-06 2023-09-23T23:39:34.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:34.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232588+0000", 2023-09-23T23:39:34.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3597e-05 2023-09-23T23:39:34.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:34.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232596+0000", 2023-09-23T23:39:34.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8120000000000008e-06 2023-09-23T23:39:34.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.737972+0000", 2023-09-23T23:39:34.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.50537641300000002 2023-09-23T23:39:34.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:39:34.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.741095+0000", 2023-09-23T23:39:34.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031224780000000001 2023-09-23T23:39:34.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.346073+0000", 2023-09-23T23:39:34.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.60497817499999995 2023-09-23T23:39:34.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:39:34.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.351441+0000", 2023-09-23T23:39:34.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053685449999999997 2023-09-23T23:39:34.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.606909+0000", 2023-09-23T23:39:34.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2554679260000001 2023-09-23T23:39:34.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:34.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.612108+0000", 2023-09-23T23:39:34.008 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:34.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051983919999999996 2023-09-23T23:39:34.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:39:34.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.632573+0000", 2023-09-23T23:39:34.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020465006000000001 2023-09-23T23:39:34.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:34.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.668063+0000", 2023-09-23T23:39:34.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.035490090000000002 2023-09-23T23:39:34.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:34.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.680799+0000", 2023-09-23T23:39:34.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012736032 2023-09-23T23:39:34.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:34.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.682749+0000", 2023-09-23T23:39:34.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019503509999999999 2023-09-23T23:39:34.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:34.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.685937+0000", 2023-09-23T23:39:34.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031873520000000001 2023-09-23T23:39:34.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:34.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.698588+0000", 2023-09-23T23:39:34.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012651051999999999 2023-09-23T23:39:34.012 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:34.012 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:34.012 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:34.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.012 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:95282 13.d 13:b9f8e077:::benchmark_data_smithi110_541928_object7502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T23:39:34.012 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:53.626451+0000", 2023-09-23T23:39:34.012 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 280.25503106399998, 2023-09-23T23:39:34.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5002010320000001, 2023-09-23T23:39:34.013 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:34.013 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:34.013 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:34.013 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:34.013 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:34.013 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95282 2023-09-23T23:39:34.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.013 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:34.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:34.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626451+0000", 2023-09-23T23:39:34.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:34.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:34.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626451+0000", 2023-09-23T23:39:34.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:34.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:34.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626449+0000", 2023-09-23T23:39:34.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:39:34.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:34.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626459+0000", 2023-09-23T23:39:34.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7790000000000007e-06 2023-09-23T23:39:34.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:34.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626473+0000", 2023-09-23T23:39:34.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3655000000000001e-05 2023-09-23T23:39:34.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:34.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626483+0000", 2023-09-23T23:39:34.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.013e-05 2023-09-23T23:39:34.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.973570+0000", 2023-09-23T23:39:34.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.34708731599999998 2023-09-23T23:39:34.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:39:34.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.978239+0000", 2023-09-23T23:39:34.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046685850000000003 2023-09-23T23:39:34.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.019769+0000", 2023-09-23T23:39:34.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041529772 2023-09-23T23:39:34.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:39:34.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.024313+0000", 2023-09-23T23:39:34.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045445479999999998 2023-09-23T23:39:34.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.064268+0000", 2023-09-23T23:39:34.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.039955124 2023-09-23T23:39:34.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:34.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.067970+0000", 2023-09-23T23:39:34.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00370164 2023-09-23T23:39:34.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:39:34.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.076658+0000", 2023-09-23T23:39:34.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086883770000000006 2023-09-23T23:39:34.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:34.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.104140+0000", 2023-09-23T23:39:34.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02748134 2023-09-23T23:39:34.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:34.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.113193+0000", 2023-09-23T23:39:34.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0090528350000000004 2023-09-23T23:39:34.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:34.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.114921+0000", 2023-09-23T23:39:34.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017289149999999999 2023-09-23T23:39:34.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:34.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.117081+0000", 2023-09-23T23:39:34.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021593519999999998 2023-09-23T23:39:34.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:34.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.126652+0000", 2023-09-23T23:39:34.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.009571474 2023-09-23T23:39:34.020 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:34.020 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:34.020 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:34.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.021 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:95302 13.d 13:b56e9a5b:::benchmark_data_smithi110_541928_object7522:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T23:39:34.021 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:53.699709+0000", 2023-09-23T23:39:34.021 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 280.18177290699998, 2023-09-23T23:39:34.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.504555552, 2023-09-23T23:39:34.021 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:34.021 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:34.021 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:34.021 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:34.021 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:34.021 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95302 2023-09-23T23:39:34.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.021 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:34.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:34.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699709+0000", 2023-09-23T23:39:34.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:34.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:34.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699709+0000", 2023-09-23T23:39:34.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:34.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:34.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699708+0000", 2023-09-23T23:39:34.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:39:34.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:34.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699717+0000", 2023-09-23T23:39:34.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0899999999999994e-06 2023-09-23T23:39:34.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:34.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699730+0000", 2023-09-23T23:39:34.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3392e-05 2023-09-23T23:39:34.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:34.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699740+0000", 2023-09-23T23:39:34.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5559999999999996e-06 2023-09-23T23:39:34.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.025484+0000", 2023-09-23T23:39:34.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.325744649 2023-09-23T23:39:34.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:39:34.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.030494+0000", 2023-09-23T23:39:34.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.005009365 2023-09-23T23:39:34.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.166809+0000", 2023-09-23T23:39:34.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.13631576300000001 2023-09-23T23:39:34.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:39:34.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.171655+0000", 2023-09-23T23:39:34.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048457680000000003 2023-09-23T23:39:34.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.138451+0000", 2023-09-23T23:39:34.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96679576300000003 2023-09-23T23:39:34.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:34.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.142480+0000", 2023-09-23T23:39:34.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040295679999999999 2023-09-23T23:39:34.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:39:34.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.150317+0000", 2023-09-23T23:39:34.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078367109999999997 2023-09-23T23:39:34.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:34.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.186750+0000", 2023-09-23T23:39:34.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036433140000000003 2023-09-23T23:39:34.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:34.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.188250+0000", 2023-09-23T23:39:34.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001500145 2023-09-23T23:39:34.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:34.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.188845+0000", 2023-09-23T23:39:34.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00059478099999999996 2023-09-23T23:39:34.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:34.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.191339+0000", 2023-09-23T23:39:34.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024934559999999998 2023-09-23T23:39:34.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:34.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.204265+0000", 2023-09-23T23:39:34.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012926272000000001 2023-09-23T23:39:34.029 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:34.029 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:34.029 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:34.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.029 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:95356 13.d 13:b7510188:::benchmark_data_smithi110_541928_object7576:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T23:39:34.029 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:53.853143+0000", 2023-09-23T23:39:34.029 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 280.028339152, 2023-09-23T23:39:34.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5531699410000002, 2023-09-23T23:39:34.029 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:34.029 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:34.030 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:34.030 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:34.030 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:34.030 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95356 2023-09-23T23:39:34.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.030 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:34.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:34.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853143+0000", 2023-09-23T23:39:34.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:34.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:34.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853143+0000", 2023-09-23T23:39:34.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:34.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:34.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853142+0000", 2023-09-23T23:39:34.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:34.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:34.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853150+0000", 2023-09-23T23:39:34.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8709999999999997e-06 2023-09-23T23:39:34.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:34.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853166+0000", 2023-09-23T23:39:34.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6568000000000001e-05 2023-09-23T23:39:34.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:34.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853176+0000", 2023-09-23T23:39:34.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9520000000000006e-06 2023-09-23T23:39:34.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.200709+0000", 2023-09-23T23:39:34.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.347532703 2023-09-23T23:39:34.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:39:34.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.205641+0000", 2023-09-23T23:39:34.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.004932199 2023-09-23T23:39:34.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.252766+0000", 2023-09-23T23:39:34.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047124528999999998 2023-09-23T23:39:34.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:39:34.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.257597+0000", 2023-09-23T23:39:34.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048309980000000004 2023-09-23T23:39:34.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.269273+0000", 2023-09-23T23:39:34.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.01167611 2023-09-23T23:39:34.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:34.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.276153+0000", 2023-09-23T23:39:34.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068806689999999998 2023-09-23T23:39:34.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:39:34.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.296160+0000", 2023-09-23T23:39:34.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020006954 2023-09-23T23:39:34.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:34.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.310175+0000", 2023-09-23T23:39:34.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014014235 2023-09-23T23:39:34.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:34.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.313239+0000", 2023-09-23T23:39:34.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030640490000000001 2023-09-23T23:39:34.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:34.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.383950+0000", 2023-09-23T23:39:34.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.070711100999999998 2023-09-23T23:39:34.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:34.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.388728+0000", 2023-09-23T23:39:34.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047779540000000001 2023-09-23T23:39:34.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:34.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.406313+0000", 2023-09-23T23:39:34.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017585478000000002 2023-09-23T23:39:34.037 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:34.037 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:34.037 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:34.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.037 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:129500 13.d 13:b573bb55:::benchmark_data_smithi110_541928_object41720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T23:39:34.037 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:37:23.150568+0000", 2023-09-23T23:39:34.037 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 130.730914791, 2023-09-23T23:39:34.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.78439364499999997, 2023-09-23T23:39:34.038 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:34.038 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:34.038 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:34.038 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:34.038 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:34.038 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 129500 2023-09-23T23:39:34.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.038 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:34.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:34.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.150568+0000", 2023-09-23T23:39:34.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:34.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:34.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.150568+0000", 2023-09-23T23:39:34.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:34.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:34.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.150566+0000", 2023-09-23T23:39:34.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:34.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:34.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.150574+0000", 2023-09-23T23:39:34.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5719999999999997e-06 2023-09-23T23:39:34.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:34.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.150590+0000", 2023-09-23T23:39:34.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5976000000000001e-05 2023-09-23T23:39:34.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:34.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.150598+0000", 2023-09-23T23:39:34.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0020000000000006e-06 2023-09-23T23:39:34.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.214013+0000", 2023-09-23T23:39:34.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.063415213999999998 2023-09-23T23:39:34.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:39:34.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.216759+0000", 2023-09-23T23:39:34.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027466669999999999 2023-09-23T23:39:34.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.264593+0000", 2023-09-23T23:39:34.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047833345999999999 2023-09-23T23:39:34.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:39:34.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.266933+0000", 2023-09-23T23:39:34.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023401849999999998 2023-09-23T23:39:34.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.898241+0000", 2023-09-23T23:39:34.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.63130840799999999 2023-09-23T23:39:34.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:34.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.900207+0000", 2023-09-23T23:39:34.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019656840000000001 2023-09-23T23:39:34.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:39:34.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.904494+0000", 2023-09-23T23:39:34.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042864660000000001 2023-09-23T23:39:34.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:34.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.927324+0000", 2023-09-23T23:39:34.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022830100999999998 2023-09-23T23:39:34.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:34.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.927651+0000", 2023-09-23T23:39:34.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000327752 2023-09-23T23:39:34.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:34.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.928327+0000", 2023-09-23T23:39:34.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00067513700000000002 2023-09-23T23:39:34.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:34.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.929405+0000", 2023-09-23T23:39:34.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010787220000000001 2023-09-23T23:39:34.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:34.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.934961+0000", 2023-09-23T23:39:34.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055559620000000002 2023-09-23T23:39:34.045 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:34.046 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:34.046 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:34.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.046 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:129660 13.d 13:b2f7b05f:::benchmark_data_smithi110_541928_object41880:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T23:39:34.046 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:37:23.464683+0000", 2023-09-23T23:39:34.046 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 130.41679931799999, 2023-09-23T23:39:34.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.79308129900000002, 2023-09-23T23:39:34.046 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:34.046 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:34.046 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:34.046 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:34.047 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:34.047 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 129660 2023-09-23T23:39:34.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.047 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:34.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:34.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.464683+0000", 2023-09-23T23:39:34.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:34.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:34.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.464683+0000", 2023-09-23T23:39:34.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:34.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:34.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.464682+0000", 2023-09-23T23:39:34.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:34.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:34.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.464688+0000", 2023-09-23T23:39:34.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6409999999999996e-06 2023-09-23T23:39:34.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:34.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.464703+0000", 2023-09-23T23:39:34.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4999e-05 2023-09-23T23:39:34.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:34.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.464714+0000", 2023-09-23T23:39:34.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0584000000000001e-05 2023-09-23T23:39:34.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.508771+0000", 2023-09-23T23:39:34.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.044057223999999999 2023-09-23T23:39:34.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:39:34.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.510213+0000", 2023-09-23T23:39:34.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014423400000000001 2023-09-23T23:39:34.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.847358+0000", 2023-09-23T23:39:34.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.33714445100000001 2023-09-23T23:39:34.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:39:34.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:23.849872+0000", 2023-09-23T23:39:34.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025140470000000002 2023-09-23T23:39:34.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:24.225158+0000", 2023-09-23T23:39:34.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37528565600000002 2023-09-23T23:39:34.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:34.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:24.227067+0000", 2023-09-23T23:39:34.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019089770000000001 2023-09-23T23:39:34.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:39:34.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:24.230799+0000", 2023-09-23T23:39:34.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037329379999999999 2023-09-23T23:39:34.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:34.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:24.251393+0000", 2023-09-23T23:39:34.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020593634999999999 2023-09-23T23:39:34.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:34.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:24.251616+0000", 2023-09-23T23:39:34.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000222993 2023-09-23T23:39:34.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:34.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:24.252305+0000", 2023-09-23T23:39:34.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00068871199999999996 2023-09-23T23:39:34.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:34.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:24.253342+0000", 2023-09-23T23:39:34.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001036705 2023-09-23T23:39:34.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:34.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:24.257764+0000", 2023-09-23T23:39:34.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044228039999999998 2023-09-23T23:39:34.054 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:34.054 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:34.054 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:34.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.054 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:142679 13.d 13:be67433e:::benchmark_data_smithi110_541928_object54899:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T23:39:34.054 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:37:56.791462+0000", 2023-09-23T23:39:34.054 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 97.090020386000006, 2023-09-23T23:39:34.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.751677131, 2023-09-23T23:39:34.054 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:39:34.054 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:34.054 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:39:34.055 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:39:34.055 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:34.055 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 142679 2023-09-23T23:39:34.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.055 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:39:34.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:39:34.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:56.791462+0000", 2023-09-23T23:39:34.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:34.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:39:34.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:56.791462+0000", 2023-09-23T23:39:34.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:39:34.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:39:34.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:56.791461+0000", 2023-09-23T23:39:34.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:34.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:39:34.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:56.791467+0000", 2023-09-23T23:39:34.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1290000000000001e-06 2023-09-23T23:39:34.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:39:34.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:56.791477+0000", 2023-09-23T23:39:34.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0601000000000001e-05 2023-09-23T23:39:34.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:39:34.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:56.791484+0000", 2023-09-23T23:39:34.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5989999999999998e-06 2023-09-23T23:39:34.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:56.806018+0000", 2023-09-23T23:39:34.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014534109999999999 2023-09-23T23:39:34.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:39:34.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:56.806876+0000", 2023-09-23T23:39:34.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00085761599999999996 2023-09-23T23:39:34.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:57.000905+0000", 2023-09-23T23:39:34.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.19402907899999999 2023-09-23T23:39:34.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:39:34.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:57.002957+0000", 2023-09-23T23:39:34.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020521710000000002 2023-09-23T23:39:34.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:39:34.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:57.517128+0000", 2023-09-23T23:39:34.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.51417098000000006 2023-09-23T23:39:34.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:39:34.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:57.518557+0000", 2023-09-23T23:39:34.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014290749999999999 2023-09-23T23:39:34.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:39:34.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:57.521575+0000", 2023-09-23T23:39:34.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030175359999999999 2023-09-23T23:39:34.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:34.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:57.538353+0000", 2023-09-23T23:39:34.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016778966999999999 2023-09-23T23:39:34.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.061 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:34.061 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:57.538782+0000", 2023-09-23T23:39:34.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00042809100000000002 2023-09-23T23:39:34.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.061 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:39:34.061 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:57.538956+0000", 2023-09-23T23:39:34.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000174699 2023-09-23T23:39:34.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.061 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:39:34.061 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:57.539810+0000", 2023-09-23T23:39:34.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00085412800000000001 2023-09-23T23:39:34.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:39:34.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:39:34.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:39:34.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:37:57.543139+0000", 2023-09-23T23:39:34.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033286819999999999 2023-09-23T23:39:34.062 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:34.062 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:34.062 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:34.062 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:39:34.062 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:39:34.062 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:39:34.063 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:34.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:34 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:34.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:34 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:34.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:34 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:34.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:34 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:34.197 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:34.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:34.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:34.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:34 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:34.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:34 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:34.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:34 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:34.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:34 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:34.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:34 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:34.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:34 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:34.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:34 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:34.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:34 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:34.210 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:39:34.251 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:39:34.297 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:39:34.305 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:34.341 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:34.347 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:39:34.386 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:39:34.406 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:34.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:34.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:34.423 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:39:34.469 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:39:34.504 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:39:34.512 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:39:34.512 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:39:34.512 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:39:34.513 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:39:34.513 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:39:34.513 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:39:34.513 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:39:34.513 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:39:34.516 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:39:34.585 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:34.664 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 12 16 23963 23947 124.669 100.938 0.00843116 0.00801696 2023-09-23T23:39:34.688 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:39:34.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:34.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:39:34.793 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:34.830 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:34.900 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:34.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:34.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:35.007 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:35.079 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:35.171 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:39:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:35 smithi138 ceph-mon[193606]: pgmap v1773: 119 pgs: 119 active+clean; 1.1 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T23:39:35.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:35 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3436245462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:39:35.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:35 smithi110 ceph-mon[212939]: pgmap v1773: 119 pgs: 119 active+clean; 1.1 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T23:39:35.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:35 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3436245462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:39:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:35 smithi110 ceph-mon[221102]: pgmap v1773: 119 pgs: 119 active+clean; 1.1 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T23:39:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:35 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3436245462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:39:35.663 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 13 16 25733 25717 123.589 110.625 0.00530432 0.00808546 2023-09-23T23:39:36.112 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:39:36.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:36.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:39:36.224 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:36.281 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:36.621 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 5, 7, 2, 1, 6] out_osds: [4] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:39:36.622 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:39:36.622 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T23:39:36.622 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T23:39:36.622 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T23:39:36.622 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T23:39:36.663 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 14 16 27016 27000 120.485 80.1875 0.00764142 0.00829475 2023-09-23T23:39:36.787 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:39:36.940 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:39:36.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:36.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:39:37.048 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:37.074 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:39:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:39:36] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.43.0" 2023-09-23T23:39:37.076 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:37 smithi110 ceph-mon[212939]: pgmap v1774: 119 pgs: 119 active+clean; 1.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-09-23T23:39:37.116 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:37.134 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:39:37.290 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:37.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:37.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:37.350 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:37.395 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:37.423 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:37 smithi138 ceph-mon[193606]: pgmap v1774: 119 pgs: 119 active+clean; 1.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-09-23T23:39:37.426 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:37.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:37 smithi110 ceph-mon[221102]: pgmap v1774: 119 pgs: 119 active+clean; 1.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-09-23T23:39:37.539 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:37.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:37.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:37.647 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:37.669 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 15 16 28475 28459 118.526 91.1875 0.0133119 0.00842712 2023-09-23T23:39:37.686 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:37.790 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:37.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:37.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:37.857 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:39:37.895 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:37.975 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:38.238 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:38.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:38.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:38.342 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:38.392 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:38.664 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 16 16 29619 29603 115.587 71.5 0.00238594 0.00862119 2023-09-23T23:39:38.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:38 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:38.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:38 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:38.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:38 smithi110 ceph-mon[212939]: pgmap v1775: 119 pgs: 119 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-09-23T23:39:38.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:38 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:38.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:38 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:38.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:38 smithi110 ceph-mon[221102]: pgmap v1775: 119 pgs: 119 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-09-23T23:39:39.006 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:38 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:39.006 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:38 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:39.006 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:38 smithi138 ceph-mon[193606]: pgmap v1775: 119 pgs: 119 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-09-23T23:39:39.146 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:39:39.291 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:39:39.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:39.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:39:39.396 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:39.425 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:39.665 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 17 16 30940 30924 113.645 82.5625 0.00490826 0.00879189 2023-09-23T23:39:39.717 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:39.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:39.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:39.821 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:39.903 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:40.038 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:40.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:40.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:40.114 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:40.149 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:40.216 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:39:40.216 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "" 2023-09-23T23:39:40.217 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:39:40.232 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:40.321 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:40.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:40.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:40.430 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:39:40.492 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:40.607 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:39:40.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:40.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:39:40.667 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 18 16 32187 32171 111.646 77.9375 0.00925495 0.00895176 2023-09-23T23:39:40.715 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:40.744 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:40.842 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:40.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:40.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:40.946 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:39:40.981 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:41.095 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:39:41.110 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:39:41.110 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:39:41.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:41 smithi110 ceph-mon[212939]: pgmap v1776: 119 pgs: 119 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 102 MiB/s wr, 1.64k op/s 2023-09-23T23:39:41.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:41 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1149403911' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:39:41.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:41 smithi110 ceph-mon[221102]: pgmap v1776: 119 pgs: 119 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 102 MiB/s wr, 1.64k op/s 2023-09-23T23:39:41.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:41 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1149403911' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:39:41.410 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:41 smithi138 ceph-mon[193606]: pgmap v1776: 119 pgs: 119 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 102 MiB/s wr, 1.64k op/s 2023-09-23T23:39:41.410 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:41 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1149403911' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:39:41.510 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:39:41.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:41.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:39:41.614 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:41.642 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:41.667 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 19 16 33303 33287 109.442 69.75 0.00172253 0.00906426 2023-09-23T23:39:42.039 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:39:42.356 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:42 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:39:42.356 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:42 smithi110 ceph-mon[221102]: failed to encode map e623 with expected crc 2023-09-23T23:39:42.357 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:42 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1149403911' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:39:42.357 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:42 smithi110 ceph-mon[221102]: osdmap e623: 8 total, 8 up, 7 in 2023-09-23T23:39:42.357 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:42 smithi110 ceph-mon[221102]: failed to encode map e623 with expected crc 2023-09-23T23:39:42.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:42 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:39:42.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:42 smithi110 ceph-mon[212939]: failed to encode map e623 with expected crc 2023-09-23T23:39:42.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:42 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1149403911' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:39:42.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:42 smithi110 ceph-mon[212939]: osdmap e623: 8 total, 8 up, 7 in 2023-09-23T23:39:42.357 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:42 smithi110 ceph-mon[212939]: failed to encode map e623 with expected crc 2023-09-23T23:39:42.408 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:39:42.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:42 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:39:42.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:42 smithi138 ceph-mon[193606]: failed to encode map e623 with expected crc 2023-09-23T23:39:42.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:42 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1149403911' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:39:42.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:42 smithi138 ceph-mon[193606]: osdmap e623: 8 total, 8 up, 7 in 2023-09-23T23:39:42.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:42 smithi138 ceph-mon[193606]: failed to encode map e623 with expected crc 2023-09-23T23:39:42.667 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:39:42.666438+0000 min lat: 0.00116231 max lat: 0.238552 avg lat: 0.00945572 2023-09-23T23:39:42.668 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:39:42.668 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 20 16 33854 33838 105.693 34.4375 0.0115486 0.00945572 2023-09-23T23:39:42.984 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:42.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:42.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:43.095 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:43.126 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:43.180 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:43.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:43.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:43.284 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:43.358 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:43 smithi110 ceph-mon[212939]: pgmap v1778: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-09-23T23:39:43.358 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:39:43.359 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:43 smithi110 ceph-mon[221102]: pgmap v1778: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-09-23T23:39:43.359 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:39:43.367 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:43 smithi138 ceph-mon[193606]: pgmap v1778: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-09-23T23:39:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:39:43.667 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 21 16 34911 34895 103.806 66.0625 0.0126831 0.00962751 2023-09-23T23:39:43.689 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:43.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:43.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:43.796 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:39:43.829 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:43.935 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:39:43.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:43.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:39:43.992 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:39:44.039 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:44.043 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:39:44.082 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:39:44.107 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:44.140 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:39:44.165 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:39:44.165 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:39:44.181 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:39:44.215 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:39:44.233 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:44.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:44.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:44.248 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:39:44.284 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:39:44.305 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:39:44.306 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:39:44.306 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:39:44.306 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:39:44.306 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:39:44.306 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:39:44.306 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:39:44.306 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:39:44.342 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:44.383 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:44.435 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:44.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:44.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:44.539 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:44.567 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:44.667 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 22 16 35977 35961 102.116 66.625 0.00221351 0.00969961 2023-09-23T23:39:44.842 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:44 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1818285202' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:39:44.842 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:44 smithi110 ceph-mon[212939]: pgmap v1779: 119 pgs: 119 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T23:39:44.842 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:44 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 2]}]: dispatch 2023-09-23T23:39:44.842 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:44 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 2]}]: dispatch 2023-09-23T23:39:44.842 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:44 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1818285202' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:39:44.843 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:44 smithi110 ceph-mon[221102]: pgmap v1779: 119 pgs: 119 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T23:39:44.843 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:44 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 2]}]: dispatch 2023-09-23T23:39:44.843 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:44 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 2]}]: dispatch 2023-09-23T23:39:45.089 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:44 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1818285202' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:39:45.089 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:44 smithi138 ceph-mon[193606]: pgmap v1779: 119 pgs: 119 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T23:39:45.089 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:44 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 2]}]: dispatch 2023-09-23T23:39:45.089 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:44 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 2]}]: dispatch 2023-09-23T23:39:45.090 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:39:45.143 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:39:45.483 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:39:45.619 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:45.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:45.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:45.667 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 23 16 36905 36889 100.199 58 0.00729459 0.00997529 2023-09-23T23:39:45.724 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:45.810 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:45.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:45 smithi110 ceph-mon[221102]: failed to encode map e624 with expected crc 2023-09-23T23:39:45.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:45 smithi110 ceph-mon[221102]: failed to encode map e624 with expected crc 2023-09-23T23:39:45.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:45 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 2]}]': finished 2023-09-23T23:39:45.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:45 smithi110 ceph-mon[221102]: osdmap e624: 8 total, 8 up, 7 in 2023-09-23T23:39:45.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:45 smithi110 ceph-mon[212939]: failed to encode map e624 with expected crc 2023-09-23T23:39:45.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:45 smithi110 ceph-mon[212939]: failed to encode map e624 with expected crc 2023-09-23T23:39:45.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:45 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 2]}]': finished 2023-09-23T23:39:45.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:45 smithi110 ceph-mon[212939]: osdmap e624: 8 total, 8 up, 7 in 2023-09-23T23:39:46.033 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:46.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:46.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:46.075 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:45 smithi138 ceph-mon[193606]: failed to encode map e624 with expected crc 2023-09-23T23:39:46.075 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:45 smithi138 ceph-mon[193606]: failed to encode map e624 with expected crc 2023-09-23T23:39:46.075 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:45 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 2]}]': finished 2023-09-23T23:39:46.075 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:45 smithi138 ceph-mon[193606]: osdmap e624: 8 total, 8 up, 7 in 2023-09-23T23:39:46.149 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:46.172 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:46.213 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:46.435 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 5, 7, 2, 1, 6] out_osds: [4] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:39:46.436 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:39:46.436 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 3, 5, 7, 2, 1, 6] 2023-09-23T23:39:46.436 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd out 6 2023-09-23T23:39:46.551 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:46.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:46.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:46.661 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:46.667 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 24 16 37828 37812 98.4279 57.6875 0.0155723 0.0101536 2023-09-23T23:39:46.692 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:46.762 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:46.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:46.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:46.865 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:46.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:46 smithi138 ceph-mon[193606]: osdmap e625: 8 total, 8 up, 7 in 2023-09-23T23:39:46.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:46 smithi138 ceph-mon[193606]: failed to encode map e625 with expected crc 2023-09-23T23:39:46.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:46 smithi138 ceph-mon[193606]: failed to encode map e625 with expected crc 2023-09-23T23:39:46.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:46 smithi138 ceph-mon[193606]: failed to encode map e625 with expected crc 2023-09-23T23:39:46.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:46 smithi138 ceph-mon[193606]: failed to encode map e625 with expected crc 2023-09-23T23:39:46.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:46 smithi138 ceph-mon[193606]: failed to encode map e625 with expected crc 2023-09-23T23:39:46.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:46 smithi138 ceph-mon[193606]: failed to encode map e625 with expected crc 2023-09-23T23:39:46.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:46 smithi138 ceph-mon[193606]: pgmap v1782: 119 pgs: 1 peering, 118 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T23:39:46.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:46 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:46.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:46 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:46.973 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:47.203 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:39:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:39:46] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.43.0" 2023-09-23T23:39:47.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[212939]: osdmap e625: 8 total, 8 up, 7 in 2023-09-23T23:39:47.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[212939]: failed to encode map e625 with expected crc 2023-09-23T23:39:47.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[212939]: failed to encode map e625 with expected crc 2023-09-23T23:39:47.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[212939]: failed to encode map e625 with expected crc 2023-09-23T23:39:47.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[212939]: failed to encode map e625 with expected crc 2023-09-23T23:39:47.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[212939]: failed to encode map e625 with expected crc 2023-09-23T23:39:47.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[212939]: failed to encode map e625 with expected crc 2023-09-23T23:39:47.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[212939]: pgmap v1782: 119 pgs: 1 peering, 118 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T23:39:47.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:47.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:47.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[221102]: osdmap e625: 8 total, 8 up, 7 in 2023-09-23T23:39:47.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[221102]: failed to encode map e625 with expected crc 2023-09-23T23:39:47.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[221102]: failed to encode map e625 with expected crc 2023-09-23T23:39:47.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[221102]: failed to encode map e625 with expected crc 2023-09-23T23:39:47.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[221102]: failed to encode map e625 with expected crc 2023-09-23T23:39:47.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[221102]: failed to encode map e625 with expected crc 2023-09-23T23:39:47.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[221102]: failed to encode map e625 with expected crc 2023-09-23T23:39:47.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[221102]: pgmap v1782: 119 pgs: 1 peering, 118 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T23:39:47.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:47.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:46 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:47.236 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:47.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:47.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:47.314 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:39:47.315 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:39:47.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:39:47.315 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:39:47.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.315 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21483 13.5 13:a8fdedd1:::benchmark_data_smithi110_541928_object21482:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:39:47.315 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.045464+0000", 2023-09-23T23:39:47.315 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 397.265010889, 2023-09-23T23:39:47.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.135403025, 2023-09-23T23:39:47.315 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.315 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:47.316 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:39:47.316 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:39:47.316 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:47.316 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21483 2023-09-23T23:39:47.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.316 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.316 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.316 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045464+0000", 2023-09-23T23:39:47.316 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045464+0000", 2023-09-23T23:39:47.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.317 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045462+0000", 2023-09-23T23:39:47.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:39:47.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.317 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045507+0000", 2023-09-23T23:39:47.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4555999999999998e-05 2023-09-23T23:39:47.318 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.318 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.318 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.318 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045524+0000", 2023-09-23T23:39:47.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7450999999999999e-05 2023-09-23T23:39:47.318 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.318 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.318 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.318 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045531+0000", 2023-09-23T23:39:47.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7329999999999997e-06 2023-09-23T23:39:47.318 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.319 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.319 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045729+0000", 2023-09-23T23:39:47.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000197634 2023-09-23T23:39:47.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.319 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:39:47.319 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.045739+0000", 2023-09-23T23:39:47.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.825e-06 2023-09-23T23:39:47.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.319 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.177900+0000", 2023-09-23T23:39:47.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.132160896 2023-09-23T23:39:47.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:39:47.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.178003+0000", 2023-09-23T23:39:47.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103212 2023-09-23T23:39:47.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:39:47.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.178112+0000", 2023-09-23T23:39:47.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109002 2023-09-23T23:39:47.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:39:47.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.179965+0000", 2023-09-23T23:39:47.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001853234 2023-09-23T23:39:47.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180695+0000", 2023-09-23T23:39:47.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000729913 2023-09-23T23:39:47.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180800+0000", 2023-09-23T23:39:47.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000104758 2023-09-23T23:39:47.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:39:47.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180834+0000", 2023-09-23T23:39:47.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4038000000000002e-05 2023-09-23T23:39:47.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180867+0000", 2023-09-23T23:39:47.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2935000000000002e-05 2023-09-23T23:39:47.323 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.323 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.323 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.323 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.323 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.323 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21498 13.5 13:ac1dd01c:::benchmark_data_smithi110_541928_object21497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:39:47.323 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.050241+0000", 2023-09-23T23:39:47.323 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 397.26023308700002, 2023-09-23T23:39:47.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.130775804, 2023-09-23T23:39:47.323 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.323 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:47.324 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:39:47.324 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:39:47.324 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:47.324 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21498 2023-09-23T23:39:47.324 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.324 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.324 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.324 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.324 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050241+0000", 2023-09-23T23:39:47.324 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.324 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.324 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.325 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.325 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050241+0000", 2023-09-23T23:39:47.325 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.325 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.325 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.325 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.325 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050240+0000", 2023-09-23T23:39:47.325 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:47.325 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.325 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.325 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.325 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050290+0000", 2023-09-23T23:39:47.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0144e-05 2023-09-23T23:39:47.326 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050304+0000", 2023-09-23T23:39:47.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3708e-05 2023-09-23T23:39:47.326 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050310+0000", 2023-09-23T23:39:47.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9290000000000003e-06 2023-09-23T23:39:47.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050369+0000", 2023-09-23T23:39:47.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9196e-05 2023-09-23T23:39:47.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:39:47.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.050379+0000", 2023-09-23T23:39:47.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9639999999999998e-06 2023-09-23T23:39:47.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.179168+0000", 2023-09-23T23:39:47.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.128789193 2023-09-23T23:39:47.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:39:47.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.179266+0000", 2023-09-23T23:39:47.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7250000000000006e-05 2023-09-23T23:39:47.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:39:47.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.179375+0000", 2023-09-23T23:39:47.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109688 2023-09-23T23:39:47.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.329 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.329 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180921+0000", 2023-09-23T23:39:47.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015460490000000001 2023-09-23T23:39:47.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.329 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:39:47.329 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180927+0000", 2023-09-23T23:39:47.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9109999999999998e-06 2023-09-23T23:39:47.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.329 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180956+0000", 2023-09-23T23:39:47.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8926000000000001e-05 2023-09-23T23:39:47.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:39:47.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180995+0000", 2023-09-23T23:39:47.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8621999999999997e-05 2023-09-23T23:39:47.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181017+0000", 2023-09-23T23:39:47.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.251e-05 2023-09-23T23:39:47.331 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.331 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.331 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.331 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21499 13.5 13:a6d31f08:::benchmark_data_smithi110_541928_object21498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:39:47.331 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.054699+0000", 2023-09-23T23:39:47.331 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 397.25577569000001, 2023-09-23T23:39:47.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1271894090000001, 2023-09-23T23:39:47.331 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.331 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:47.331 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:39:47.332 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:39:47.332 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:47.332 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21499 2023-09-23T23:39:47.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.332 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054699+0000", 2023-09-23T23:39:47.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054699+0000", 2023-09-23T23:39:47.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054697+0000", 2023-09-23T23:39:47.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:47.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054738+0000", 2023-09-23T23:39:47.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0728e-05 2023-09-23T23:39:47.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054757+0000", 2023-09-23T23:39:47.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9037000000000001e-05 2023-09-23T23:39:47.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054765+0000", 2023-09-23T23:39:47.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8010000000000004e-06 2023-09-23T23:39:47.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054821+0000", 2023-09-23T23:39:47.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5702999999999998e-05 2023-09-23T23:39:47.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:39:47.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.054829+0000", 2023-09-23T23:39:47.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9579999999999996e-06 2023-09-23T23:39:47.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.179986+0000", 2023-09-23T23:39:47.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.125156388 2023-09-23T23:39:47.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:39:47.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180059+0000", 2023-09-23T23:39:47.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3510999999999995e-05 2023-09-23T23:39:47.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:39:47.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.180157+0000", 2023-09-23T23:39:47.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8039000000000001e-05 2023-09-23T23:39:47.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:39:47.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181753+0000", 2023-09-23T23:39:47.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001595602 2023-09-23T23:39:47.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181813+0000", 2023-09-23T23:39:47.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9984e-05 2023-09-23T23:39:47.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181836+0000", 2023-09-23T23:39:47.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3373000000000001e-05 2023-09-23T23:39:47.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:39:47.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181865+0000", 2023-09-23T23:39:47.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8147e-05 2023-09-23T23:39:47.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181888+0000", 2023-09-23T23:39:47.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.37e-05 2023-09-23T23:39:47.339 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.339 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.339 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.340 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21528 13.5 13:a8de34c9:::benchmark_data_smithi110_541928_object21527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:39:47.340 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.066977+0000", 2023-09-23T23:39:47.340 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 397.24349743400001, 2023-09-23T23:39:47.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.116824818, 2023-09-23T23:39:47.340 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.340 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:47.340 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:39:47.340 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:39:47.340 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:47.340 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21528 2023-09-23T23:39:47.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.341 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.066977+0000", 2023-09-23T23:39:47.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.066977+0000", 2023-09-23T23:39:47.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.066976+0000", 2023-09-23T23:39:47.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:39:47.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067009+0000", 2023-09-23T23:39:47.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2963999999999998e-05 2023-09-23T23:39:47.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067017+0000", 2023-09-23T23:39:47.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8350000000000004e-06 2023-09-23T23:39:47.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067022+0000", 2023-09-23T23:39:47.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9989999999999999e-06 2023-09-23T23:39:47.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.068428+0000", 2023-09-23T23:39:47.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014058860000000001 2023-09-23T23:39:47.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:39:47.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.068438+0000", 2023-09-23T23:39:47.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0033e-05 2023-09-23T23:39:47.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181053+0000", 2023-09-23T23:39:47.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.112614311 2023-09-23T23:39:47.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:39:47.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181184+0000", 2023-09-23T23:39:47.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000131314 2023-09-23T23:39:47.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:39:47.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181273+0000", 2023-09-23T23:39:47.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.954e-05 2023-09-23T23:39:47.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183686+0000", 2023-09-23T23:39:47.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002412871 2023-09-23T23:39:47.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183729+0000", 2023-09-23T23:39:47.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2329000000000001e-05 2023-09-23T23:39:47.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:39:47.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183736+0000", 2023-09-23T23:39:47.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0360000000000001e-06 2023-09-23T23:39:47.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:39:47.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183767+0000", 2023-09-23T23:39:47.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1133000000000002e-05 2023-09-23T23:39:47.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183802+0000", 2023-09-23T23:39:47.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5176000000000003e-05 2023-09-23T23:39:47.347 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.348 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.348 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.348 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.348 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21532 13.5 13:a4c16192:::benchmark_data_smithi110_541928_object21531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:39:47.348 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.067036+0000", 2023-09-23T23:39:47.348 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 397.243438884, 2023-09-23T23:39:47.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1169511729999999, 2023-09-23T23:39:47.348 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.348 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:47.348 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:39:47.348 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:39:47.349 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:47.349 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21532 2023-09-23T23:39:47.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.349 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067036+0000", 2023-09-23T23:39:47.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067036+0000", 2023-09-23T23:39:47.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.350 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.350 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067035+0000", 2023-09-23T23:39:47.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:39:47.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.350 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.350 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067063+0000", 2023-09-23T23:39:47.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7453e-05 2023-09-23T23:39:47.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067070+0000", 2023-09-23T23:39:47.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4699999999999996e-06 2023-09-23T23:39:47.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.067075+0000", 2023-09-23T23:39:47.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2730000000000002e-06 2023-09-23T23:39:47.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.068458+0000", 2023-09-23T23:39:47.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001382505 2023-09-23T23:39:47.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:39:47.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.068464+0000", 2023-09-23T23:39:47.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0909999999999998e-06 2023-09-23T23:39:47.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.181930+0000", 2023-09-23T23:39:47.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.113466209 2023-09-23T23:39:47.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:39:47.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.182050+0000", 2023-09-23T23:39:47.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120068 2023-09-23T23:39:47.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:39:47.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.182218+0000", 2023-09-23T23:39:47.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016763300000000001 2023-09-23T23:39:47.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:39:47.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183812+0000", 2023-09-23T23:39:47.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001594179 2023-09-23T23:39:47.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183872+0000", 2023-09-23T23:39:47.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.033e-05 2023-09-23T23:39:47.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183920+0000", 2023-09-23T23:39:47.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7956e-05 2023-09-23T23:39:47.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:39:47.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183953+0000", 2023-09-23T23:39:47.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2855999999999998e-05 2023-09-23T23:39:47.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183987+0000", 2023-09-23T23:39:47.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3695000000000002e-05 2023-09-23T23:39:47.356 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.356 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.356 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.356 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21564 13.5 13:a601eb05:::benchmark_data_smithi110_541928_object21563:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:39:47.356 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.088589+0000", 2023-09-23T23:39:47.356 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 397.22188590500002, 2023-09-23T23:39:47.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0965361170000001, 2023-09-23T23:39:47.356 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.356 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:47.356 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:39:47.357 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:39:47.357 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:47.357 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21564 2023-09-23T23:39:47.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.357 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088589+0000", 2023-09-23T23:39:47.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.358 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088589+0000", 2023-09-23T23:39:47.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.358 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.358 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.358 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088588+0000", 2023-09-23T23:39:47.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:39:47.358 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.358 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.358 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088625+0000", 2023-09-23T23:39:47.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7500999999999999e-05 2023-09-23T23:39:47.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.359 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.359 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.359 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088639+0000", 2023-09-23T23:39:47.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3568e-05 2023-09-23T23:39:47.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.359 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.359 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.359 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088643+0000", 2023-09-23T23:39:47.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7219999999999999e-06 2023-09-23T23:39:47.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.359 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088698+0000", 2023-09-23T23:39:47.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4797999999999999e-05 2023-09-23T23:39:47.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:39:47.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.088713+0000", 2023-09-23T23:39:47.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4809e-05 2023-09-23T23:39:47.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.182903+0000", 2023-09-23T23:39:47.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.094190306 2023-09-23T23:39:47.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:39:47.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.182976+0000", 2023-09-23T23:39:47.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3059000000000006e-05 2023-09-23T23:39:47.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:39:47.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183121+0000", 2023-09-23T23:39:47.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000144797 2023-09-23T23:39:47.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:39:47.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.183999+0000", 2023-09-23T23:39:47.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00087746999999999999 2023-09-23T23:39:47.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185012+0000", 2023-09-23T23:39:47.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001013735 2023-09-23T23:39:47.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185045+0000", 2023-09-23T23:39:47.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2339000000000003e-05 2023-09-23T23:39:47.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:39:47.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185088+0000", 2023-09-23T23:39:47.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2854e-05 2023-09-23T23:39:47.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185125+0000", 2023-09-23T23:39:47.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.714e-05 2023-09-23T23:39:47.364 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.364 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.364 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.364 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21580 13.5 13:afa4f38d:::benchmark_data_smithi110_541928_object21579:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:39:47.364 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.097520+0000", 2023-09-23T23:39:47.364 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 397.21295493899999, 2023-09-23T23:39:47.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.089325219, 2023-09-23T23:39:47.365 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.365 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:47.365 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:39:47.365 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:39:47.365 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:47.365 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21580 2023-09-23T23:39:47.365 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.365 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.365 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.365 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097520+0000", 2023-09-23T23:39:47.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.366 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.366 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.366 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.366 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097520+0000", 2023-09-23T23:39:47.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.366 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.366 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.366 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.366 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097518+0000", 2023-09-23T23:39:47.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:47.366 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097552+0000", 2023-09-23T23:39:47.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3627999999999997e-05 2023-09-23T23:39:47.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097567+0000", 2023-09-23T23:39:47.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5477000000000001e-05 2023-09-23T23:39:47.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097574+0000", 2023-09-23T23:39:47.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7309999999999996e-06 2023-09-23T23:39:47.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097718+0000", 2023-09-23T23:39:47.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014400700000000001 2023-09-23T23:39:47.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:39:47.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.097740+0000", 2023-09-23T23:39:47.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2238999999999998e-05 2023-09-23T23:39:47.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.184022+0000", 2023-09-23T23:39:47.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0862815990000001 2023-09-23T23:39:47.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:39:47.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.184170+0000", 2023-09-23T23:39:47.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000147754 2023-09-23T23:39:47.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:39:47.370 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.184340+0000", 2023-09-23T23:39:47.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017066499999999999 2023-09-23T23:39:47.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.370 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186756+0000", 2023-09-23T23:39:47.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024154049999999998 2023-09-23T23:39:47.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.370 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186789+0000", 2023-09-23T23:39:47.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3621000000000002e-05 2023-09-23T23:39:47.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:39:47.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186795+0000", 2023-09-23T23:39:47.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4809999999999997e-06 2023-09-23T23:39:47.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:39:47.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186822+0000", 2023-09-23T23:39:47.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7166e-05 2023-09-23T23:39:47.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186845+0000", 2023-09-23T23:39:47.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2716000000000001e-05 2023-09-23T23:39:47.372 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.372 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.372 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.372 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21590 13.5 13:a913081b:::benchmark_data_smithi110_541928_object21589:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:39:47.372 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.101884+0000", 2023-09-23T23:39:47.372 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 397.20859063099999, 2023-09-23T23:39:47.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0874988189999999, 2023-09-23T23:39:47.373 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.373 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:47.373 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:39:47.373 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:39:47.373 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:47.373 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21590 2023-09-23T23:39:47.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.373 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101884+0000", 2023-09-23T23:39:47.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101884+0000", 2023-09-23T23:39:47.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101883+0000", 2023-09-23T23:39:47.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:39:47.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101943+0000", 2023-09-23T23:39:47.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9757999999999999e-05 2023-09-23T23:39:47.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101952+0000", 2023-09-23T23:39:47.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6420000000000002e-06 2023-09-23T23:39:47.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.101958+0000", 2023-09-23T23:39:47.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0900000000000001e-06 2023-09-23T23:39:47.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102140+0000", 2023-09-23T23:39:47.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018145100000000001 2023-09-23T23:39:47.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:39:47.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102153+0000", 2023-09-23T23:39:47.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2676e-05 2023-09-23T23:39:47.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185166+0000", 2023-09-23T23:39:47.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0830130069999999 2023-09-23T23:39:47.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:39:47.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185286+0000", 2023-09-23T23:39:47.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120227 2023-09-23T23:39:47.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:39:47.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.185443+0000", 2023-09-23T23:39:47.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000156844 2023-09-23T23:39:47.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:39:47.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186851+0000", 2023-09-23T23:39:47.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014085199999999999 2023-09-23T23:39:47.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188371+0000", 2023-09-23T23:39:47.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015201780000000001 2023-09-23T23:39:47.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.379 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189290+0000", 2023-09-23T23:39:47.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00091839000000000005 2023-09-23T23:39:47.379 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.379 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:39:47.379 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189342+0000", 2023-09-23T23:39:47.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2110000000000001e-05 2023-09-23T23:39:47.379 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.379 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.379 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189383+0000", 2023-09-23T23:39:47.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0859999999999998e-05 2023-09-23T23:39:47.380 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.380 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.380 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.380 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21591 13.5 13:ae681958:::benchmark_data_smithi110_541928_object21590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:39:47.380 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.102344+0000", 2023-09-23T23:39:47.380 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 397.20813079999999, 2023-09-23T23:39:47.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0912746840000001, 2023-09-23T23:39:47.380 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.381 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:47.381 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:39:47.381 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:39:47.381 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:47.381 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21591 2023-09-23T23:39:47.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.381 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102344+0000", 2023-09-23T23:39:47.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102344+0000", 2023-09-23T23:39:47.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102343+0000", 2023-09-23T23:39:47.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:39:47.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102429+0000", 2023-09-23T23:39:47.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6637000000000004e-05 2023-09-23T23:39:47.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102441+0000", 2023-09-23T23:39:47.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1902999999999999e-05 2023-09-23T23:39:47.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102450+0000", 2023-09-23T23:39:47.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2320000000000001e-06 2023-09-23T23:39:47.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.384 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102504+0000", 2023-09-23T23:39:47.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3981999999999999e-05 2023-09-23T23:39:47.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.384 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:39:47.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.102511+0000", 2023-09-23T23:39:47.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8399999999999995e-06 2023-09-23T23:39:47.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.384 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186108+0000", 2023-09-23T23:39:47.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.083596351 2023-09-23T23:39:47.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:39:47.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186201+0000", 2023-09-23T23:39:47.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3474000000000007e-05 2023-09-23T23:39:47.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:39:47.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186283+0000", 2023-09-23T23:39:47.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1625999999999994e-05 2023-09-23T23:39:47.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.386 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.386 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188420+0000", 2023-09-23T23:39:47.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021368749999999999 2023-09-23T23:39:47.386 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.386 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:39:47.386 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188429+0000", 2023-09-23T23:39:47.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5370000000000003e-06 2023-09-23T23:39:47.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193525+0000", 2023-09-23T23:39:47.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050954379999999999 2023-09-23T23:39:47.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:39:47.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193578+0000", 2023-09-23T23:39:47.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2927999999999999e-05 2023-09-23T23:39:47.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193618+0000", 2023-09-23T23:39:47.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0822000000000003e-05 2023-09-23T23:39:47.388 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.388 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.388 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.388 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21598 13.5 13:a172c780:::benchmark_data_smithi110_541928_object21597:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:39:47.388 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.106175+0000", 2023-09-23T23:39:47.388 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 397.20429960500002, 2023-09-23T23:39:47.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0875860749999999, 2023-09-23T23:39:47.389 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.389 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:47.389 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:39:47.389 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:39:47.389 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:47.389 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21598 2023-09-23T23:39:47.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.389 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106175+0000", 2023-09-23T23:39:47.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106175+0000", 2023-09-23T23:39:47.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106174+0000", 2023-09-23T23:39:47.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:39:47.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106283+0000", 2023-09-23T23:39:47.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109133 2023-09-23T23:39:47.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106294+0000", 2023-09-23T23:39:47.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0994e-05 2023-09-23T23:39:47.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106301+0000", 2023-09-23T23:39:47.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5089999999999999e-06 2023-09-23T23:39:47.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106360+0000", 2023-09-23T23:39:47.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8828e-05 2023-09-23T23:39:47.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:39:47.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.106369+0000", 2023-09-23T23:39:47.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1880000000000008e-06 2023-09-23T23:39:47.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.393 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186882+0000", 2023-09-23T23:39:47.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0805130059999999 2023-09-23T23:39:47.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.393 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:39:47.393 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.186960+0000", 2023-09-23T23:39:47.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7863999999999999e-05 2023-09-23T23:39:47.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.393 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:39:47.393 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.187041+0000", 2023-09-23T23:39:47.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1542e-05 2023-09-23T23:39:47.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:39:47.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188432+0000", 2023-09-23T23:39:47.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001390797 2023-09-23T23:39:47.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188460+0000", 2023-09-23T23:39:47.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7668e-05 2023-09-23T23:39:47.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193681+0000", 2023-09-23T23:39:47.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0052214999999999996 2023-09-23T23:39:47.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:39:47.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193725+0000", 2023-09-23T23:39:47.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3377000000000002e-05 2023-09-23T23:39:47.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193761+0000", 2023-09-23T23:39:47.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6464000000000002e-05 2023-09-23T23:39:47.396 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.396 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.396 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.396 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21614 13.5 13:ab7b6c08:::benchmark_data_smithi110_541928_object21613:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:39:47.396 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.115290+0000", 2023-09-23T23:39:47.396 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 397.19518449999998, 2023-09-23T23:39:47.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.078720337, 2023-09-23T23:39:47.396 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.396 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:47.396 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:39:47.396 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:39:47.397 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:47.397 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21614 2023-09-23T23:39:47.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.397 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.397 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.397 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115290+0000", 2023-09-23T23:39:47.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.397 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.397 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115290+0000", 2023-09-23T23:39:47.398 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.398 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.398 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.398 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115289+0000", 2023-09-23T23:39:47.398 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:47.398 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.398 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.398 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115353+0000", 2023-09-23T23:39:47.398 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4473000000000004e-05 2023-09-23T23:39:47.398 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115365+0000", 2023-09-23T23:39:47.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.164e-05 2023-09-23T23:39:47.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115371+0000", 2023-09-23T23:39:47.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1500000000000004e-06 2023-09-23T23:39:47.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115421+0000", 2023-09-23T23:39:47.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0117999999999997e-05 2023-09-23T23:39:47.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:39:47.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.115431+0000", 2023-09-23T23:39:47.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4350000000000003e-06 2023-09-23T23:39:47.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.187560+0000", 2023-09-23T23:39:47.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0721295150000001 2023-09-23T23:39:47.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:39:47.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.187686+0000", 2023-09-23T23:39:47.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012613699999999999 2023-09-23T23:39:47.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:39:47.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.187797+0000", 2023-09-23T23:39:47.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110852 2023-09-23T23:39:47.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:39:47.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188463+0000", 2023-09-23T23:39:47.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00066563799999999997 2023-09-23T23:39:47.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189424+0000", 2023-09-23T23:39:47.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00096155399999999999 2023-09-23T23:39:47.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193937+0000", 2023-09-23T23:39:47.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00451308 2023-09-23T23:39:47.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.403 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:39:47.403 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193979+0000", 2023-09-23T23:39:47.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1501999999999997e-05 2023-09-23T23:39:47.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.403 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.403 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194010+0000", 2023-09-23T23:39:47.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1535999999999998e-05 2023-09-23T23:39:47.403 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.404 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.404 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.404 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21623 13.5 13:ac3554a5:::benchmark_data_smithi110_541928_object21622:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:39:47.404 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.123936+0000", 2023-09-23T23:39:47.404 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 397.18653846000001, 2023-09-23T23:39:47.404 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0702429680000001, 2023-09-23T23:39:47.404 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.404 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:47.404 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:39:47.404 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:39:47.404 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:47.405 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21623 2023-09-23T23:39:47.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.405 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.123936+0000", 2023-09-23T23:39:47.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.123936+0000", 2023-09-23T23:39:47.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.406 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.406 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.123935+0000", 2023-09-23T23:39:47.406 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:47.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.406 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.406 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.124042+0000", 2023-09-23T23:39:47.406 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010689200000000001 2023-09-23T23:39:47.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.406 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.124053+0000", 2023-09-23T23:39:47.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1221e-05 2023-09-23T23:39:47.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.124058+0000", 2023-09-23T23:39:47.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.074e-06 2023-09-23T23:39:47.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.124114+0000", 2023-09-23T23:39:47.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6026999999999997e-05 2023-09-23T23:39:47.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:39:47.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.124124+0000", 2023-09-23T23:39:47.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0032e-05 2023-09-23T23:39:47.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188483+0000", 2023-09-23T23:39:47.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.064359354 2023-09-23T23:39:47.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:39:47.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188589+0000", 2023-09-23T23:39:47.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105889 2023-09-23T23:39:47.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:39:47.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.188737+0000", 2023-09-23T23:39:47.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014783099999999999 2023-09-23T23:39:47.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:39:47.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191026+0000", 2023-09-23T23:39:47.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022893409999999999 2023-09-23T23:39:47.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.410 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191847+0000", 2023-09-23T23:39:47.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00082065100000000004 2023-09-23T23:39:47.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.410 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194089+0000", 2023-09-23T23:39:47.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022423339999999999 2023-09-23T23:39:47.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:39:47.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194143+0000", 2023-09-23T23:39:47.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3446999999999998e-05 2023-09-23T23:39:47.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.411 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194179+0000", 2023-09-23T23:39:47.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6216e-05 2023-09-23T23:39:47.411 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.411 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.411 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.412 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21639 13.5 13:afb98446:::benchmark_data_smithi110_541928_object21638:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:39:47.412 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.135557+0000", 2023-09-23T23:39:47.412 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 397.17491726200001, 2023-09-23T23:39:47.412 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0589459779999999, 2023-09-23T23:39:47.412 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.412 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:47.412 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:39:47.412 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:39:47.412 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:47.412 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21639 2023-09-23T23:39:47.412 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.412 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135557+0000", 2023-09-23T23:39:47.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135557+0000", 2023-09-23T23:39:47.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135557+0000", 2023-09-23T23:39:47.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:39:47.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135622+0000", 2023-09-23T23:39:47.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4926999999999997e-05 2023-09-23T23:39:47.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135628+0000", 2023-09-23T23:39:47.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0299999999999999e-06 2023-09-23T23:39:47.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135631+0000", 2023-09-23T23:39:47.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.841e-06 2023-09-23T23:39:47.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135680+0000", 2023-09-23T23:39:47.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9818999999999999e-05 2023-09-23T23:39:47.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:39:47.416 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.135691+0000", 2023-09-23T23:39:47.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0976e-05 2023-09-23T23:39:47.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.416 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189484+0000", 2023-09-23T23:39:47.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.053793092 2023-09-23T23:39:47.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:39:47.416 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189619+0000", 2023-09-23T23:39:47.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013436399999999999 2023-09-23T23:39:47.417 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:39:47.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.189777+0000", 2023-09-23T23:39:47.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015826999999999999 2023-09-23T23:39:47.417 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:39:47.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191857+0000", 2023-09-23T23:39:47.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020803929999999998 2023-09-23T23:39:47.417 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191914+0000", 2023-09-23T23:39:47.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6075000000000001e-05 2023-09-23T23:39:47.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194427+0000", 2023-09-23T23:39:47.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002513684 2023-09-23T23:39:47.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:39:47.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194468+0000", 2023-09-23T23:39:47.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1183999999999997e-05 2023-09-23T23:39:47.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194503+0000", 2023-09-23T23:39:47.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4867999999999999e-05 2023-09-23T23:39:47.419 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.419 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.419 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.419 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21659 13.5 13:aad23106:::benchmark_data_smithi110_541928_object21658:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:39:47.420 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.150190+0000", 2023-09-23T23:39:47.420 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 397.160284902, 2023-09-23T23:39:47.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0444529309999999, 2023-09-23T23:39:47.420 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.420 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:47.420 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:39:47.420 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:39:47.420 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:47.420 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21659 2023-09-23T23:39:47.420 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.420 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.421 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.421 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150190+0000", 2023-09-23T23:39:47.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.421 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.421 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150190+0000", 2023-09-23T23:39:47.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150189+0000", 2023-09-23T23:39:47.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:39:47.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150260+0000", 2023-09-23T23:39:47.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1550000000000004e-05 2023-09-23T23:39:47.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150272+0000", 2023-09-23T23:39:47.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1328000000000001e-05 2023-09-23T23:39:47.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150276+0000", 2023-09-23T23:39:47.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6700000000000002e-06 2023-09-23T23:39:47.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150329+0000", 2023-09-23T23:39:47.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3118999999999997e-05 2023-09-23T23:39:47.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:39:47.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.150339+0000", 2023-09-23T23:39:47.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4660000000000005e-06 2023-09-23T23:39:47.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.190332+0000", 2023-09-23T23:39:47.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.039992633 2023-09-23T23:39:47.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:39:47.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.190453+0000", 2023-09-23T23:39:47.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012097899999999999 2023-09-23T23:39:47.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:39:47.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.190560+0000", 2023-09-23T23:39:47.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107587 2023-09-23T23:39:47.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:39:47.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.192645+0000", 2023-09-23T23:39:47.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020851329999999999 2023-09-23T23:39:47.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193487+0000", 2023-09-23T23:39:47.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00084168199999999996 2023-09-23T23:39:47.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194573+0000", 2023-09-23T23:39:47.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001085978 2023-09-23T23:39:47.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.427 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.427 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:39:47.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194610+0000", 2023-09-23T23:39:47.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7512000000000003e-05 2023-09-23T23:39:47.427 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.427 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.427 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194643+0000", 2023-09-23T23:39:47.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2147000000000002e-05 2023-09-23T23:39:47.427 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.427 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.427 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.428 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.428 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21680 13.5 13:ac31eb73:::benchmark_data_smithi110_541928_object21679:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:39:47.428 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.170348+0000", 2023-09-23T23:39:47.428 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 397.140126076, 2023-09-23T23:39:47.428 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0244998839999999, 2023-09-23T23:39:47.428 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.428 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:47.428 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:39:47.428 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:39:47.428 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:47.428 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21680 2023-09-23T23:39:47.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.429 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170348+0000", 2023-09-23T23:39:47.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170348+0000", 2023-09-23T23:39:47.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.430 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.430 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.430 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.430 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170347+0000", 2023-09-23T23:39:47.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:47.430 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.430 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.430 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.430 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170403+0000", 2023-09-23T23:39:47.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6042000000000003e-05 2023-09-23T23:39:47.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.431 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.431 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.431 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170415+0000", 2023-09-23T23:39:47.431 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2238e-05 2023-09-23T23:39:47.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.431 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.431 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.431 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170423+0000", 2023-09-23T23:39:47.431 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1829999999999997e-06 2023-09-23T23:39:47.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.431 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170480+0000", 2023-09-23T23:39:47.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7476999999999999e-05 2023-09-23T23:39:47.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:39:47.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.170488+0000", 2023-09-23T23:39:47.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2390000000000002e-06 2023-09-23T23:39:47.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191062+0000", 2023-09-23T23:39:47.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0205732489999999 2023-09-23T23:39:47.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.433 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:39:47.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191141+0000", 2023-09-23T23:39:47.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9435000000000001e-05 2023-09-23T23:39:47.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.433 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:39:47.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191244+0000", 2023-09-23T23:39:47.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010341299999999999 2023-09-23T23:39:47.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:39:47.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.192657+0000", 2023-09-23T23:39:47.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014124599999999999 2023-09-23T23:39:47.434 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193656+0000", 2023-09-23T23:39:47.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00099945200000000007 2023-09-23T23:39:47.434 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.435 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194770+0000", 2023-09-23T23:39:47.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001114163 2023-09-23T23:39:47.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.435 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:39:47.435 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194815+0000", 2023-09-23T23:39:47.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4583999999999999e-05 2023-09-23T23:39:47.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.435 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.435 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194848+0000", 2023-09-23T23:39:47.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3272000000000003e-05 2023-09-23T23:39:47.436 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.436 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.436 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.436 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.436 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.436 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:21693 13.5 13:a03052c3:::benchmark_data_smithi110_541928_object21692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e547)", 2023-09-23T23:39:47.436 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:10.193329+0000", 2023-09-23T23:39:47.436 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 397.11714593599999, 2023-09-23T23:39:47.436 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.001638582, 2023-09-23T23:39:47.436 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.436 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:39:47.436 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:39:47.437 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:39:47.437 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:39:47.437 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21693 2023-09-23T23:39:47.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.437 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193329+0000", 2023-09-23T23:39:47.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193329+0000", 2023-09-23T23:39:47.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.438 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.438 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.438 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193328+0000", 2023-09-23T23:39:47.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:39:47.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.438 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.438 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.438 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193375+0000", 2023-09-23T23:39:47.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7132000000000002e-05 2023-09-23T23:39:47.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.439 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193390+0000", 2023-09-23T23:39:47.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4841000000000001e-05 2023-09-23T23:39:47.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.439 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193396+0000", 2023-09-23T23:39:47.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1299999999999998e-06 2023-09-23T23:39:47.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193442+0000", 2023-09-23T23:39:47.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6621e-05 2023-09-23T23:39:47.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:39:47.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:10.193450+0000", 2023-09-23T23:39:47.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1079999999999993e-06 2023-09-23T23:39:47.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.191950+0000", 2023-09-23T23:39:47.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99849948600000005 2023-09-23T23:39:47.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:39:47.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.192054+0000", 2023-09-23T23:39:47.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103933 2023-09-23T23:39:47.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:39:47.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.192153+0000", 2023-09-23T23:39:47.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9110999999999993e-05 2023-09-23T23:39:47.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:39:47.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193631+0000", 2023-09-23T23:39:47.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014778619999999999 2023-09-23T23:39:47.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.193884+0000", 2023-09-23T23:39:47.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025308299999999998 2023-09-23T23:39:47.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:39:47.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194896+0000", 2023-09-23T23:39:47.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010124159999999999 2023-09-23T23:39:47.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:39:47.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194935+0000", 2023-09-23T23:39:47.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8470000000000003e-05 2023-09-23T23:39:47.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:11.194967+0000", 2023-09-23T23:39:47.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2385999999999998e-05 2023-09-23T23:39:47.443 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.444 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.444 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.444 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPull(13.d e606/566 cost 1049576)", 2023-09-23T23:39:47.444 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:38:16.311683+0000", 2023-09-23T23:39:47.444 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 90.998792021, 2023-09-23T23:39:47.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006598759, 2023-09-23T23:39:47.444 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.444 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:39:47.444 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.311683+0000", 2023-09-23T23:39:47.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.311683+0000", 2023-09-23T23:39:47.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.311681+0000", 2023-09-23T23:39:47.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:47.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.311697+0000", 2023-09-23T23:39:47.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5739999999999998e-05 2023-09-23T23:39:47.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.311717+0000", 2023-09-23T23:39:47.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0480999999999999e-05 2023-09-23T23:39:47.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.311726+0000", 2023-09-23T23:39:47.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1550000000000007e-06 2023-09-23T23:39:47.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.311787+0000", 2023-09-23T23:39:47.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0949000000000002e-05 2023-09-23T23:39:47.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.448 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.448 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.318281+0000", 2023-09-23T23:39:47.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064947169999999997 2023-09-23T23:39:47.448 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.448 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.448 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.448 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPull(13.d e607/566 cost 1049576)", 2023-09-23T23:39:47.448 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:38:19.622224+0000", 2023-09-23T23:39:47.449 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 87.688250202999996, 2023-09-23T23:39:47.449 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088886050000000008, 2023-09-23T23:39:47.449 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.449 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:39:47.449 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.449 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.449 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:19.622224+0000", 2023-09-23T23:39:47.449 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.449 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.449 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:19.622224+0000", 2023-09-23T23:39:47.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:19.622223+0000", 2023-09-23T23:39:47.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:39:47.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:19.622238+0000", 2023-09-23T23:39:47.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4346e-05 2023-09-23T23:39:47.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:19.622257+0000", 2023-09-23T23:39:47.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.906e-05 2023-09-23T23:39:47.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:19.622264+0000", 2023-09-23T23:39:47.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1500000000000002e-06 2023-09-23T23:39:47.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:19.622325+0000", 2023-09-23T23:39:47.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0883e-05 2023-09-23T23:39:47.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:19.631113+0000", 2023-09-23T23:39:47.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0087880570000000002 2023-09-23T23:39:47.453 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.453 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.453 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.454 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPull(13.d e607/566 cost 1049576)", 2023-09-23T23:39:47.454 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:38:19.980917+0000", 2023-09-23T23:39:47.454 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 87.329557429000005, 2023-09-23T23:39:47.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0046271699999999999, 2023-09-23T23:39:47.454 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:39:47.454 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:39:47.454 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:39:47.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:39:47.454 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:19.980917+0000", 2023-09-23T23:39:47.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:39:47.455 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:19.980917+0000", 2023-09-23T23:39:47.455 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:39:47.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:39:47.455 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:19.980916+0000", 2023-09-23T23:39:47.455 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:39:47.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:39:47.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:19.980932+0000", 2023-09-23T23:39:47.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6727000000000001e-05 2023-09-23T23:39:47.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:39:47.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:19.980953+0000", 2023-09-23T23:39:47.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0662999999999999e-05 2023-09-23T23:39:47.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:39:47.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:19.980961+0000", 2023-09-23T23:39:47.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6490000000000008e-06 2023-09-23T23:39:47.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:39:47.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:19.981026+0000", 2023-09-23T23:39:47.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5524000000000005e-05 2023-09-23T23:39:47.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:39:47.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:39:47.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:39:47.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:19.985544+0000", 2023-09-23T23:39:47.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004518022 2023-09-23T23:39:47.457 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.457 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.457 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.458 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:39:47.458 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:39:47.458 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:39:47.458 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:47.492 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:47.668 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 25 16 38978 38962 97.3658 71.875 0.00303659 0.0102437 2023-09-23T23:39:47.763 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:39:48.187 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:39:48.240 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:48.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:48.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:48.351 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:39:48.382 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:48.438 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:39:48.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:48.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:39:48.545 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:48.554 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:48.613 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:39:48.636 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:48.651 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:48.668 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 26 16 40020 40004 96.1259 65.125 0.0128096 0.0103959 2023-09-23T23:39:48.791 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:39:48.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:48.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:39:48.902 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:48.940 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:49.183 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:49 smithi138 ceph-mon[193606]: pgmap v1783: 119 pgs: 1 peering, 118 active+clean; 2.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 46 MiB/s wr, 729 op/s 2023-09-23T23:39:49.183 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:49 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:39:49.183 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:49 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:39:49.183 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:49 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:49.198 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:49.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:49.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:49.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:49 smithi110 ceph-mon[212939]: pgmap v1783: 119 pgs: 1 peering, 118 active+clean; 2.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 46 MiB/s wr, 729 op/s 2023-09-23T23:39:49.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:49 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:39:49.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:49 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:39:49.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:49 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:49.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:49 smithi110 ceph-mon[221102]: pgmap v1783: 119 pgs: 1 peering, 118 active+clean; 2.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 46 MiB/s wr, 729 op/s 2023-09-23T23:39:49.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:49 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:39:49.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:49 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:39:49.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:49 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:49.303 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:49.360 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:49.419 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:39:49.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:49.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:39:49.525 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:49.553 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:49.668 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 27 16 40900 40884 94.603 55 0.0101522 0.0105637 2023-09-23T23:39:49.855 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:49.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:49.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:49.964 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:39:49.998 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:50.101 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:39:50.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:50 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/37218311' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:39:50.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:50 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:39:50.155 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:39:50.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:50.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:39:50.260 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:50.299 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:50.363 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:50.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:50.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:50.423 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:39:50.423 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:39:50.423 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-09-23T23:39:50.423 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:39:50.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:50 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/37218311' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:39:50.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:50 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:39:50.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:50 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/37218311' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:39:50.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:50 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:39:50.467 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:50.497 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:50.590 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:50.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:50.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:50.668 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 28 16 41996 41980 93.6708 68.5 0.00192483 0.0106401 2023-09-23T23:39:50.694 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:50.749 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:50.958 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:39:50.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:50.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:39:51.062 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:51.134 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:51.246 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:39:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:51 smithi138 ceph-mon[193606]: pgmap v1784: 119 pgs: 1 peering, 118 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T23:39:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:51 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:39:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:51 smithi138 ceph-mon[193606]: failed to encode map e626 with expected crc 2023-09-23T23:39:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:51 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:39:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:51 smithi138 ceph-mon[193606]: osdmap e626: 8 total, 8 up, 7 in 2023-09-23T23:39:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:51 smithi138 ceph-mon[193606]: failed to encode map e626 with expected crc 2023-09-23T23:39:51.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:51 smithi110 ceph-mon[212939]: pgmap v1784: 119 pgs: 1 peering, 118 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T23:39:51.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:51 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:39:51.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:51 smithi110 ceph-mon[212939]: failed to encode map e626 with expected crc 2023-09-23T23:39:51.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:51 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:39:51.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:51 smithi110 ceph-mon[212939]: osdmap e626: 8 total, 8 up, 7 in 2023-09-23T23:39:51.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:51 smithi110 ceph-mon[212939]: failed to encode map e626 with expected crc 2023-09-23T23:39:51.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:51 smithi110 ceph-mon[221102]: pgmap v1784: 119 pgs: 1 peering, 118 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T23:39:51.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:51 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:39:51.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:51 smithi110 ceph-mon[221102]: failed to encode map e626 with expected crc 2023-09-23T23:39:51.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:51 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:39:51.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:51 smithi110 ceph-mon[221102]: osdmap e626: 8 total, 8 up, 7 in 2023-09-23T23:39:51.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:51 smithi110 ceph-mon[221102]: failed to encode map e626 with expected crc 2023-09-23T23:39:51.598 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:51.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:51.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:51.668 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 29 16 43091 43075 92.8006 68.4375 0.0154039 0.0107678 2023-09-23T23:39:51.681 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:39:51.707 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:51.743 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:51.825 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:51.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:51.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:51.929 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:51.958 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:52.000 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:52.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:52.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:52.104 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:39:52.130 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:52.188 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:39:52.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:52.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:39:52.294 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:52.326 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:52.668 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 30 16 44129 44113 91.8699 64.875 0.0125466 0.0108794 2023-09-23T23:39:52.896 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:52.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:52.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:52.999 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:53.100 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:53.174 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:39:53.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:53 smithi138 ceph-mon[193606]: pgmap v1786: 119 pgs: 119 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s 2023-09-23T23:39:53.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:53 smithi110 ceph-mon[212939]: pgmap v1786: 119 pgs: 119 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s 2023-09-23T23:39:53.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:53 smithi110 ceph-mon[221102]: pgmap v1786: 119 pgs: 119 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s 2023-09-23T23:39:53.218 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:39:53.255 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:53.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:53.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:53.275 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:39:53.323 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:39:53.358 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:39:53.363 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:53.395 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:53.396 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:39:53.435 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:39:53.473 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:39:53.479 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:53.479 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:39:53.480 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:39:53.480 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:39:53.481 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:39:53.481 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:39:53.481 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:39:53.481 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:39:53.481 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:39:53.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:53.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:53.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:53.656 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:53.668 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 31 16 45243 45227 91.1523 69.625 0.00902692 0.0109644 2023-09-23T23:39:53.775 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:39:53.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:53.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:39:53.782 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:53 smithi110 conmon[212914]: 2023-09-23T23:39:53.541+0000 7f29aa464700 -1 mon.a@0(leader).osd e626 definitely_dead 0 2023-09-23T23:39:53.883 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:53.897 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:39:53.897 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:39:53.947 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:54.055 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:54.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:54.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:54.093 INFO:teuthology.orchestra.run.smithi110.stderr:marked out osd.6. 2023-09-23T23:39:54.159 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:54.185 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:54.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:54 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/20990242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:39:54.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:54 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2598369394' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T23:39:54.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:54 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T23:39:54.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:54 smithi138 ceph-mon[193606]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T23:39:54.227 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:54.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:54.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:54.330 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:39:54.359 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:54.402 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:39:54.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:54.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:39:54.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:54 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/20990242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:39:54.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:54 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2598369394' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T23:39:54.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:54 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T23:39:54.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:54 smithi110 ceph-mon[212939]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T23:39:54.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:54 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/20990242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:39:54.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:54 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2598369394' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T23:39:54.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:54 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T23:39:54.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:54 smithi110 ceph-mon[221102]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T23:39:54.506 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:54.531 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:54.593 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:39:54.668 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 32 16 45742 45726 89.2787 31.1875 0.00210287 0.0110478 2023-09-23T23:39:54.699 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:54.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:54.707 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:39:54.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:54.809 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:54.836 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:55.139 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:39:55.332 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:55.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:55.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:55.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:55 smithi138 ceph-mon[193606]: pgmap v1787: 119 pgs: 119 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 55 MiB/s wr, 879 op/s 2023-09-23T23:39:55.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:55 smithi138 ceph-mon[193606]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:55 smithi138 ceph-mon[193606]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:55 smithi138 ceph-mon[193606]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:55 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T23:39:55.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:55 smithi138 ceph-mon[193606]: osdmap e627: 8 total, 8 up, 6 in 2023-09-23T23:39:55.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:55 smithi138 ceph-mon[193606]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:55 smithi138 ceph-mon[193606]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:55 smithi138 ceph-mon[193606]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:55 smithi138 ceph-mon[193606]: osdmap e628: 8 total, 8 up, 6 in 2023-09-23T23:39:55.435 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:39:55.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[212939]: pgmap v1787: 119 pgs: 119 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 55 MiB/s wr, 879 op/s 2023-09-23T23:39:55.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[212939]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[212939]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[212939]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T23:39:55.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[212939]: osdmap e627: 8 total, 8 up, 6 in 2023-09-23T23:39:55.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[212939]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[212939]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[212939]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[212939]: osdmap e628: 8 total, 8 up, 6 in 2023-09-23T23:39:55.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[221102]: pgmap v1787: 119 pgs: 119 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 55 MiB/s wr, 879 op/s 2023-09-23T23:39:55.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[221102]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[221102]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[221102]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T23:39:55.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[221102]: osdmap e627: 8 total, 8 up, 6 in 2023-09-23T23:39:55.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[221102]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[221102]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[221102]: failed to encode map e627 with expected crc 2023-09-23T23:39:55.456 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:55 smithi110 ceph-mon[221102]: osdmap e628: 8 total, 8 up, 6 in 2023-09-23T23:39:55.456 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:55.466 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:55.511 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:39:55.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:55.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:39:55.615 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:55.643 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:55.668 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 33 16 45742 45726 86.5739 0 - 0.0110478 2023-09-23T23:39:55.689 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:55.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:55.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:55.794 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:55.822 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:55.866 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:55.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:55.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:55.970 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:56.000 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:56.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:56 smithi138 ceph-mon[193606]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:56 smithi138 ceph-mon[193606]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:56 smithi138 ceph-mon[193606]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:56 smithi138 ceph-mon[193606]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:56 smithi138 ceph-mon[193606]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:56 smithi138 ceph-mon[193606]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:56 smithi138 ceph-mon[193606]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:56 smithi138 ceph-mon[193606]: osdmap e629: 8 total, 8 up, 6 in 2023-09-23T23:39:56.154 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:56.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:56.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:56.258 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:56.283 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:56 smithi110 ceph-mon[212939]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.283 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:56 smithi110 ceph-mon[212939]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.283 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:56 smithi110 ceph-mon[212939]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.284 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:56 smithi110 ceph-mon[212939]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.284 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:56 smithi110 ceph-mon[212939]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.284 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:56 smithi110 ceph-mon[212939]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.284 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:56 smithi110 ceph-mon[212939]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.284 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:56 smithi110 ceph-mon[212939]: osdmap e629: 8 total, 8 up, 6 in 2023-09-23T23:39:56.285 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:56 smithi110 ceph-mon[221102]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.285 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:56 smithi110 ceph-mon[221102]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.285 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:56 smithi110 ceph-mon[221102]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.285 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:56 smithi110 ceph-mon[221102]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.285 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:56 smithi110 ceph-mon[221102]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.285 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:56 smithi110 ceph-mon[221102]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.286 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:56 smithi110 ceph-mon[221102]: failed to encode map e628 with expected crc 2023-09-23T23:39:56.286 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:56 smithi110 ceph-mon[221102]: osdmap e629: 8 total, 8 up, 6 in 2023-09-23T23:39:56.286 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:56.326 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:56.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:56.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:56.429 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:56.461 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:56.570 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:56.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:56.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:56.668 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 34 16 46025 46009 84.5483 8.84375 0.0192522 0.0118214 2023-09-23T23:39:56.677 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:56.714 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:56.784 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:56.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:56.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:56.828 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:39:56.828 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:39:56.889 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:56.992 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:57.113 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:39:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:39:56] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.43.0" 2023-09-23T23:39:57.115 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[212939]: pgmap v1790: 119 pgs: 6 remapped+peering, 11 peering, 102 active+clean; 2.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 794 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T23:39:57.115 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[212939]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.115 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[212939]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.115 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[212939]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.115 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[212939]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.115 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[212939]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.115 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[212939]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.115 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[212939]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.115 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T23:39:57.116 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[212939]: osdmap e630: 8 total, 8 up, 6 in 2023-09-23T23:39:57.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:57 smithi138 ceph-mon[193606]: pgmap v1790: 119 pgs: 6 remapped+peering, 11 peering, 102 active+clean; 2.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 794 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T23:39:57.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:57 smithi138 ceph-mon[193606]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:57 smithi138 ceph-mon[193606]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:57 smithi138 ceph-mon[193606]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:57 smithi138 ceph-mon[193606]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:57 smithi138 ceph-mon[193606]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:57 smithi138 ceph-mon[193606]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.202 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:39:57.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:57.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:39:57.283 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:39:57.315 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:39:57.345 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:57 smithi138 ceph-mon[193606]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.345 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:57 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T23:39:57.345 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:57 smithi138 ceph-mon[193606]: osdmap e630: 8 total, 8 up, 6 in 2023-09-23T23:39:57.347 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:57.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[221102]: pgmap v1790: 119 pgs: 6 remapped+peering, 11 peering, 102 active+clean; 2.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 794 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T23:39:57.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[221102]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[221102]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[221102]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[221102]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[221102]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[221102]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[221102]: failed to encode map e629 with expected crc 2023-09-23T23:39:57.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T23:39:57.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:57 smithi110 ceph-mon[221102]: osdmap e630: 8 total, 8 up, 6 in 2023-09-23T23:39:57.601 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:39:57.602 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:39:57.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:57.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:39:57.671 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 35 16 46874 46858 83.6428 53.0625 0.0142535 0.0119484 2023-09-23T23:39:57.706 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:57.798 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:57.933 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:57.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:57.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:58.014 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:39:58.041 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:58.077 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:39:58.125 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:39:58.133 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[212939]: failed to encode map e630 with expected crc 2023-09-23T23:39:58.133 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[212939]: failed to encode map e630 with expected crc 2023-09-23T23:39:58.134 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[212939]: failed to encode map e630 with expected crc 2023-09-23T23:39:58.134 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/364294766' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:39:58.134 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:58.134 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:39:58.134 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:39:58.134 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/364294766' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:39:58.134 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[212939]: osdmap e631: 8 total, 8 up, 6 in 2023-09-23T23:39:58.157 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:58.414 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:39:58.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:58.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:39:58.421 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[221102]: failed to encode map e630 with expected crc 2023-09-23T23:39:58.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[221102]: failed to encode map e630 with expected crc 2023-09-23T23:39:58.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[221102]: failed to encode map e630 with expected crc 2023-09-23T23:39:58.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/364294766' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:39:58.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:58.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:39:58.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:39:58.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/364294766' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:39:58.422 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:58 smithi110 ceph-mon[221102]: osdmap e631: 8 total, 8 up, 6 in 2023-09-23T23:39:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:58 smithi138 ceph-mon[193606]: failed to encode map e630 with expected crc 2023-09-23T23:39:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:58 smithi138 ceph-mon[193606]: failed to encode map e630 with expected crc 2023-09-23T23:39:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:58 smithi138 ceph-mon[193606]: failed to encode map e630 with expected crc 2023-09-23T23:39:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:58 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/364294766' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:39:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:58.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:39:58.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:58 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:39:58.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:58 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/364294766' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:39:58.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:58 smithi138 ceph-mon[193606]: osdmap e631: 8 total, 8 up, 6 in 2023-09-23T23:39:58.522 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:39:58.564 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:58.671 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 36 16 47730 47714 82.8057 53.5 0.0139963 0.0120704 2023-09-23T23:39:58.675 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:39:58.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:58.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:39:58.780 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:39:58.814 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:58.858 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:39:58.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:58.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:39:58.962 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:59.044 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:59.134 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:59 smithi110 ceph-mon[212939]: pgmap v1793: 119 pgs: 6 remapped+peering, 11 peering, 102 active+clean; 2.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 33 MiB/s wr, 522 op/s; 1.2 KiB/s, 2 keys/s, 1 objects/s recovering 2023-09-23T23:39:59.134 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:59 smithi110 ceph-mon[212939]: failed to encode map e631 with expected crc 2023-09-23T23:39:59.135 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:59 smithi110 ceph-mon[212939]: failed to encode map e631 with expected crc 2023-09-23T23:39:59.135 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:59 smithi110 ceph-mon[212939]: failed to encode map e631 with expected crc 2023-09-23T23:39:59.135 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:59 smithi110 ceph-mon[212939]: failed to encode map e631 with expected crc 2023-09-23T23:39:59.135 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:59 smithi110 ceph-mon[212939]: failed to encode map e631 with expected crc 2023-09-23T23:39:59.135 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:39:59 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:59.144 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:39:59.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:59.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:39:59.254 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:39:59.291 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:59.339 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:39:59.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:59.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:39:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:59 smithi138 ceph-mon[193606]: pgmap v1793: 119 pgs: 6 remapped+peering, 11 peering, 102 active+clean; 2.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 33 MiB/s wr, 522 op/s; 1.2 KiB/s, 2 keys/s, 1 objects/s recovering 2023-09-23T23:39:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:59 smithi138 ceph-mon[193606]: failed to encode map e631 with expected crc 2023-09-23T23:39:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:59 smithi138 ceph-mon[193606]: failed to encode map e631 with expected crc 2023-09-23T23:39:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:59 smithi138 ceph-mon[193606]: failed to encode map e631 with expected crc 2023-09-23T23:39:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:59 smithi138 ceph-mon[193606]: failed to encode map e631 with expected crc 2023-09-23T23:39:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:59 smithi138 ceph-mon[193606]: failed to encode map e631 with expected crc 2023-09-23T23:39:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:39:59 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:59.443 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:59.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:59 smithi110 ceph-mon[221102]: pgmap v1793: 119 pgs: 6 remapped+peering, 11 peering, 102 active+clean; 2.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 33 MiB/s wr, 522 op/s; 1.2 KiB/s, 2 keys/s, 1 objects/s recovering 2023-09-23T23:39:59.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:59 smithi110 ceph-mon[221102]: failed to encode map e631 with expected crc 2023-09-23T23:39:59.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:59 smithi110 ceph-mon[221102]: failed to encode map e631 with expected crc 2023-09-23T23:39:59.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:59 smithi110 ceph-mon[221102]: failed to encode map e631 with expected crc 2023-09-23T23:39:59.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:59 smithi110 ceph-mon[221102]: failed to encode map e631 with expected crc 2023-09-23T23:39:59.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:59 smithi110 ceph-mon[221102]: failed to encode map e631 with expected crc 2023-09-23T23:39:59.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:39:59 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:39:59.469 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:59.567 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:39:59.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:59.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:39:59.672 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 37 16 48313 48297 81.55 36.4375 0.0934451 0.0122484 2023-09-23T23:39:59.673 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:39:59.709 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:39:59.778 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:39:59.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:39:59.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:39:59.830 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:39:59.830 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:39:59.885 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:39:59.960 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:00.152 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[212939]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Reduced data availability: 4 pgs inactive, 6 pgs peering 2023-09-23T23:40:00.152 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[212939]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T23:40:00.152 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[212939]: [WRN] PG_AVAILABILITY: Reduced data availability: 4 pgs inactive, 6 pgs peering 2023-09-23T23:40:00.152 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[212939]: pg 1.f is stuck peering for 10m, current state remapped+peering, last acting [2,3] 2023-09-23T23:40:00.152 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[212939]: pg 1.10 is stuck peering for 13m, current state peering, last acting [5,7,0] 2023-09-23T23:40:00.152 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[212939]: pg 2.2 is stuck peering for 10m, current state peering, last acting [5,1,0] 2023-09-23T23:40:00.152 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[212939]: pg 3.1c is stuck peering for 18m, current state peering, last acting [5,1,7] 2023-09-23T23:40:00.152 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[212939]: pg 4.13 is stuck peering for 10m, current state peering, last acting [5,1,2] 2023-09-23T23:40:00.152 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[212939]: pg 4.1f is stuck peering for 17m, current state peering, last acting [5,1,0] 2023-09-23T23:40:00.220 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:40:00.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:00.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:40:00.313 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 5, 7, 2, 1] out_osds: [4, 6] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:40:00.313 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:40:00.314 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-09-23T23:40:00.314 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T23:40:00.314 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T23:40:00.314 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-09-23T23:40:00.324 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:40:00.357 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:00 smithi138 ceph-mon[193606]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Reduced data availability: 4 pgs inactive, 6 pgs peering 2023-09-23T23:40:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:00 smithi138 ceph-mon[193606]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T23:40:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:00 smithi138 ceph-mon[193606]: [WRN] PG_AVAILABILITY: Reduced data availability: 4 pgs inactive, 6 pgs peering 2023-09-23T23:40:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:00 smithi138 ceph-mon[193606]: pg 1.f is stuck peering for 10m, current state remapped+peering, last acting [2,3] 2023-09-23T23:40:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:00 smithi138 ceph-mon[193606]: pg 1.10 is stuck peering for 13m, current state peering, last acting [5,7,0] 2023-09-23T23:40:00.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:00 smithi138 ceph-mon[193606]: pg 2.2 is stuck peering for 10m, current state peering, last acting [5,1,0] 2023-09-23T23:40:00.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:00 smithi138 ceph-mon[193606]: pg 3.1c is stuck peering for 18m, current state peering, last acting [5,1,7] 2023-09-23T23:40:00.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:00 smithi138 ceph-mon[193606]: pg 4.13 is stuck peering for 10m, current state peering, last acting [5,1,2] 2023-09-23T23:40:00.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:00 smithi138 ceph-mon[193606]: pg 4.1f is stuck peering for 17m, current state peering, last acting [5,1,0] 2023-09-23T23:40:00.463 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[221102]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Reduced data availability: 4 pgs inactive, 6 pgs peering 2023-09-23T23:40:00.464 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[221102]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T23:40:00.464 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[221102]: [WRN] PG_AVAILABILITY: Reduced data availability: 4 pgs inactive, 6 pgs peering 2023-09-23T23:40:00.464 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[221102]: pg 1.f is stuck peering for 10m, current state remapped+peering, last acting [2,3] 2023-09-23T23:40:00.464 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[221102]: pg 1.10 is stuck peering for 13m, current state peering, last acting [5,7,0] 2023-09-23T23:40:00.464 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[221102]: pg 2.2 is stuck peering for 10m, current state peering, last acting [5,1,0] 2023-09-23T23:40:00.464 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[221102]: pg 3.1c is stuck peering for 18m, current state peering, last acting [5,1,7] 2023-09-23T23:40:00.464 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[221102]: pg 4.13 is stuck peering for 10m, current state peering, last acting [5,1,2] 2023-09-23T23:40:00.464 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:00 smithi110 ceph-mon[221102]: pg 4.1f is stuck peering for 17m, current state peering, last acting [5,1,0] 2023-09-23T23:40:00.579 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:40:00.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:00.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:40:00.654 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:40:00.674 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 38 16 49181 49165 80.8288 54.25 0.0193807 0.0123649 2023-09-23T23:40:00.683 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:40:00.769 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:01.076 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:40:01.156 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:01 smithi110 ceph-mon[221102]: pgmap v1795: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 3 peering, 109 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 48 MiB/s wr, 766 op/s; 2837/140484 objects degraded (2.019%); 2767/140484 objects misplaced (1.970%); 1.4 MiB/s, 2 keys/s, 23 objects/s recovering 2023-09-23T23:40:01.157 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:01 smithi110 ceph-mon[221102]: Health check failed: Degraded data redundancy: 2837/140484 objects degraded (2.019%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:40:01.157 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:01 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2023-09-23T23:40:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:01 smithi138 ceph-mon[193606]: pgmap v1795: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 3 peering, 109 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 48 MiB/s wr, 766 op/s; 2837/140484 objects degraded (2.019%); 2767/140484 objects misplaced (1.970%); 1.4 MiB/s, 2 keys/s, 23 objects/s recovering 2023-09-23T23:40:01.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:01 smithi138 ceph-mon[193606]: Health check failed: Degraded data redundancy: 2837/140484 objects degraded (2.019%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:40:01.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:01 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2023-09-23T23:40:01.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:01 smithi110 ceph-mon[212939]: pgmap v1795: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 3 peering, 109 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 48 MiB/s wr, 766 op/s; 2837/140484 objects degraded (2.019%); 2767/140484 objects misplaced (1.970%); 1.4 MiB/s, 2 keys/s, 23 objects/s recovering 2023-09-23T23:40:01.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:01 smithi110 ceph-mon[212939]: Health check failed: Degraded data redundancy: 2837/140484 objects degraded (2.019%), 1 pg degraded (PG_DEGRADED) 2023-09-23T23:40:01.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:01 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2023-09-23T23:40:01.675 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 39 16 50123 50107 80.2632 58.875 0.0199459 0.0124532 2023-09-23T23:40:01.740 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:40:01.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:01.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:40:01.844 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:40:01.876 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:01.971 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:40:01.989 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:40:02.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:02.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:40:02.108 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:40:02.140 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:02.226 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:40:02.259 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:40:02.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:02.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:40:02.367 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:40:02.400 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:02.450 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:40:02.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:02.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:40:02.554 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:40:02.641 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:02.642 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:40:02.675 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:40:02.674646+0000 min lat: 0.00116231 max lat: 2.39686 avg lat: 0.0126028 2023-09-23T23:40:02.675 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:40:02.676 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 40 16 50803 50787 79.3193 42.5 0.0269667 0.0126028 2023-09-23T23:40:02.692 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:40:02.742 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:40:02.764 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:40:02.765 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:40:02.799 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:40:02.838 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:40:02.884 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:40:02.910 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:40:02.930 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:40:02.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:02.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:40:02.941 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:40:02.956 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:40:02.956 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:40:02.956 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:40:02.957 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:40:02.957 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:40:02.957 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:40:02.957 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:40:02.957 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:40:03.039 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:40:03.069 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:03.113 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:40:03.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:03.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:40:03.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:03 smithi110 ceph-mon[221102]: pgmap v1796: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 peering, 111 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 34 MiB/s wr, 547 op/s; 11405/144381 objects degraded (7.899%); 2767/144381 objects misplaced (1.916%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T23:40:03.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:03 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/539353539' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:40:03.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:03 smithi110 ceph-mon[212939]: pgmap v1796: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 peering, 111 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 34 MiB/s wr, 547 op/s; 11405/144381 objects degraded (7.899%); 2767/144381 objects misplaced (1.916%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T23:40:03.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:03 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/539353539' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:40:03.216 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:40:03.247 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:03 smithi138 ceph-mon[193606]: pgmap v1796: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 peering, 111 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 34 MiB/s wr, 547 op/s; 11405/144381 objects degraded (7.899%); 2767/144381 objects misplaced (1.916%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T23:40:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:03 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/539353539' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:40:03.501 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:40:03.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:03.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:40:03.514 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:40:03.609 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:40:03.645 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:03.675 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 41 16 51471 51455 78.4032 41.75 0.0121058 0.0127314 2023-09-23T23:40:03.959 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:40:04.676 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 42 16 52368 52352 77.87 56.0625 0.0098258 0.0128363 2023-09-23T23:40:04.996 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:40:05.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:05.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:40:05.102 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:40:05.128 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:04 smithi138 ceph-mon[193606]: pgmap v1797: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 41 MiB/s wr, 662 op/s; 11405/150279 objects degraded (7.589%); 2480/150279 objects misplaced (1.650%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:40:05.131 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:05.175 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:40:05.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:05.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:40:05.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:04 smithi110 ceph-mon[212939]: pgmap v1797: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 41 MiB/s wr, 662 op/s; 11405/150279 objects degraded (7.589%); 2480/150279 objects misplaced (1.650%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:40:05.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:04 smithi110 ceph-mon[221102]: pgmap v1797: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 41 MiB/s wr, 662 op/s; 11405/150279 objects degraded (7.589%); 2480/150279 objects misplaced (1.650%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:40:05.280 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:40:05.352 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:05.513 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:40:05.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:05.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:40:05.594 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:40:05.626 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:40:05.662 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:05.677 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 43 16 52941 52925 76.8909 35.8125 0.0152671 0.0129992 2023-09-23T23:40:05.708 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:40:05.708 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:40:05.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:40:05.708 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:40:05.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.708 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:143362 12.14 12:2df09e03:::benchmark_data_smithi110_491193_object48487:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T23:40:05.709 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:31:22.670541+0000", 2023-09-23T23:40:05.709 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 523.03261356799999, 2023-09-23T23:40:05.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.34082258900000001, 2023-09-23T23:40:05.709 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.709 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:05.709 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.709 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.709 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.709 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 143362 2023-09-23T23:40:05.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.709 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.670541+0000", 2023-09-23T23:40:05.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.670541+0000", 2023-09-23T23:40:05.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.670539+0000", 2023-09-23T23:40:05.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:40:05.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.711 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.711 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.670548+0000", 2023-09-23T23:40:05.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5199999999999997e-06 2023-09-23T23:40:05.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.711 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.711 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.670561+0000", 2023-09-23T23:40:05.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3543999999999999e-05 2023-09-23T23:40:05.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.670567+0000", 2023-09-23T23:40:05.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2900000000000002e-06 2023-09-23T23:40:05.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.670614+0000", 2023-09-23T23:40:05.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7338e-05 2023-09-23T23:40:05.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.713 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.713 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.670773+0000", 2023-09-23T23:40:05.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015908900000000001 2023-09-23T23:40:05.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.713 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:40:05.713 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.671188+0000", 2023-09-23T23:40:05.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000414659 2023-09-23T23:40:05.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.675806+0000", 2023-09-23T23:40:05.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0046179760000000002 2023-09-23T23:40:05.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.676124+0000", 2023-09-23T23:40:05.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00031816 2023-09-23T23:40:05.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011209+0000", 2023-09-23T23:40:05.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.33508485300000002 2023-09-23T23:40:05.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:40:05.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011265+0000", 2023-09-23T23:40:05.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5729999999999997e-05 2023-09-23T23:40:05.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011364+0000", 2023-09-23T23:40:05.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9328999999999997e-05 2023-09-23T23:40:05.715 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.716 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.716 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.716 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.716 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:143371 12.4 12:215a8f6d:::benchmark_data_smithi110_491193_object48496:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T23:40:05.716 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:31:22.673253+0000", 2023-09-23T23:40:05.716 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 523.02990223500001, 2023-09-23T23:40:05.716 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.33808128500000001, 2023-09-23T23:40:05.716 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.716 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:05.716 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.716 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.717 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.717 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 143371 2023-09-23T23:40:05.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.717 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673253+0000", 2023-09-23T23:40:05.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673253+0000", 2023-09-23T23:40:05.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673251+0000", 2023-09-23T23:40:05.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:40:05.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673259+0000", 2023-09-23T23:40:05.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5449999999999998e-06 2023-09-23T23:40:05.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.719 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673270+0000", 2023-09-23T23:40:05.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0791000000000001e-05 2023-09-23T23:40:05.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.719 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673276+0000", 2023-09-23T23:40:05.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7240000000000003e-06 2023-09-23T23:40:05.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.719 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.720 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673351+0000", 2023-09-23T23:40:05.720 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4734000000000006e-05 2023-09-23T23:40:05.720 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.720 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.720 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.720 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673454+0000", 2023-09-23T23:40:05.720 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000102908 2023-09-23T23:40:05.720 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.720 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.720 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:40:05.720 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.673810+0000", 2023-09-23T23:40:05.720 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00035560399999999998 2023-09-23T23:40:05.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.721 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.721 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.676165+0000", 2023-09-23T23:40:05.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023558749999999999 2023-09-23T23:40:05.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.721 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.721 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.684242+0000", 2023-09-23T23:40:05.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0080765290000000007 2023-09-23T23:40:05.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.722 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011252+0000", 2023-09-23T23:40:05.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.32701007199999998 2023-09-23T23:40:05.722 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:40:05.722 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011292+0000", 2023-09-23T23:40:05.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9969000000000003e-05 2023-09-23T23:40:05.722 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011334+0000", 2023-09-23T23:40:05.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1872999999999997e-05 2023-09-23T23:40:05.723 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.723 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.723 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.723 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:143373 12.4 12:21483e46:::benchmark_data_smithi110_491193_object48498:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T23:40:05.723 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:31:22.678380+0000", 2023-09-23T23:40:05.723 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 523.02477446399996, 2023-09-23T23:40:05.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.33305675200000001, 2023-09-23T23:40:05.724 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.724 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:05.724 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.724 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.724 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.724 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 143373 2023-09-23T23:40:05.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.724 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.678380+0000", 2023-09-23T23:40:05.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.678380+0000", 2023-09-23T23:40:05.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.678379+0000", 2023-09-23T23:40:05.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:40:05.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.678384+0000", 2023-09-23T23:40:05.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.938e-06 2023-09-23T23:40:05.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.678398+0000", 2023-09-23T23:40:05.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4158999999999999e-05 2023-09-23T23:40:05.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.678404+0000", 2023-09-23T23:40:05.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7270000000000004e-06 2023-09-23T23:40:05.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.678455+0000", 2023-09-23T23:40:05.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0915999999999998e-05 2023-09-23T23:40:05.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.678597+0000", 2023-09-23T23:40:05.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014165599999999999 2023-09-23T23:40:05.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.728 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:40:05.728 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.679001+0000", 2023-09-23T23:40:05.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00040454400000000001 2023-09-23T23:40:05.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.728 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.728 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.681163+0000", 2023-09-23T23:40:05.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002161732 2023-09-23T23:40:05.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.728 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.684272+0000", 2023-09-23T23:40:05.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003108839 2023-09-23T23:40:05.729 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011366+0000", 2023-09-23T23:40:05.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.32709446599999997 2023-09-23T23:40:05.729 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:40:05.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011405+0000", 2023-09-23T23:40:05.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9167e-05 2023-09-23T23:40:05.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011437+0000", 2023-09-23T23:40:05.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1733000000000003e-05 2023-09-23T23:40:05.730 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.730 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.730 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.730 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:143384 12.14 12:2ec0ef03:::benchmark_data_smithi110_491193_object48509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T23:40:05.730 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:31:22.688389+0000", 2023-09-23T23:40:05.731 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 523.01476603900005, 2023-09-23T23:40:05.731 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.32313247099999998, 2023-09-23T23:40:05.731 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.731 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:05.731 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.731 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.731 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.731 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 143384 2023-09-23T23:40:05.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.731 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.731 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.731 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.688389+0000", 2023-09-23T23:40:05.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.732 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.688389+0000", 2023-09-23T23:40:05.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.732 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.688387+0000", 2023-09-23T23:40:05.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:40:05.732 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.688394+0000", 2023-09-23T23:40:05.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6479999999999997e-06 2023-09-23T23:40:05.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.688406+0000", 2023-09-23T23:40:05.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2323999999999999e-05 2023-09-23T23:40:05.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.688413+0000", 2023-09-23T23:40:05.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0960000000000003e-06 2023-09-23T23:40:05.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.688463+0000", 2023-09-23T23:40:05.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9302000000000003e-05 2023-09-23T23:40:05.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.688606+0000", 2023-09-23T23:40:05.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000142896 2023-09-23T23:40:05.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.735 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:40:05.735 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.689022+0000", 2023-09-23T23:40:05.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00041676899999999999 2023-09-23T23:40:05.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.735 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.735 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.691255+0000", 2023-09-23T23:40:05.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022322179999999998 2023-09-23T23:40:05.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:22.691512+0000", 2023-09-23T23:40:05.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025754199999999998 2023-09-23T23:40:05.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011396+0000", 2023-09-23T23:40:05.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.31988349700000002 2023-09-23T23:40:05.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:40:05.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011436+0000", 2023-09-23T23:40:05.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0778999999999997e-05 2023-09-23T23:40:05.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:31:23.011521+0000", 2023-09-23T23:40:05.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4919999999999993e-05 2023-09-23T23:40:05.737 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.737 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.737 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.737 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172637 12.5 12:a0f0c82f:::benchmark_data_smithi110_491193_object77762:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:40:05.738 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.431822+0000", 2023-09-23T23:40:05.738 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 482.27133325900002, 2023-09-23T23:40:05.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0329259159999999, 2023-09-23T23:40:05.738 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.738 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:40:05.738 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.738 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.738 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.738 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172637 2023-09-23T23:40:05.738 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.738 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.738 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.739 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.739 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431822+0000", 2023-09-23T23:40:05.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.739 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.739 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431822+0000", 2023-09-23T23:40:05.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.739 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.739 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431820+0000", 2023-09-23T23:40:05.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:40:05.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431827+0000", 2023-09-23T23:40:05.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1779999999999997e-06 2023-09-23T23:40:05.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431843+0000", 2023-09-23T23:40:05.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5821e-05 2023-09-23T23:40:05.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431850+0000", 2023-09-23T23:40:05.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0010000000000004e-06 2023-09-23T23:40:05.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431916+0000", 2023-09-23T23:40:05.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5951000000000006e-05 2023-09-23T23:40:05.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431931+0000", 2023-09-23T23:40:05.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4979e-05 2023-09-23T23:40:05.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.464675+0000", 2023-09-23T23:40:05.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.032744629 2023-09-23T23:40:05.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.464748+0000", 2023-09-23T23:40:05.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2084999999999998e-05 2023-09-23T23:40:05.742 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.743 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.743 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.743 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.743 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172642 12.5 12:a6e87117:::benchmark_data_smithi110_491193_object77767:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:40:05.743 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.431864+0000", 2023-09-23T23:40:05.743 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 482.27129044899999, 2023-09-23T23:40:05.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.032980893, 2023-09-23T23:40:05.743 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.743 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:40:05.743 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.743 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.744 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.744 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172642 2023-09-23T23:40:05.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.744 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.744 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431864+0000", 2023-09-23T23:40:05.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.744 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431864+0000", 2023-09-23T23:40:05.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431864+0000", 2023-09-23T23:40:05.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:40:05.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431867+0000", 2023-09-23T23:40:05.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1099999999999999e-06 2023-09-23T23:40:05.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.746 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.746 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431881+0000", 2023-09-23T23:40:05.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3616e-05 2023-09-23T23:40:05.746 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.746 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.746 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.746 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431886+0000", 2023-09-23T23:40:05.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9899999999999997e-06 2023-09-23T23:40:05.746 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.746 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.746 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.746 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431964+0000", 2023-09-23T23:40:05.747 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8727999999999996e-05 2023-09-23T23:40:05.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.747 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.747 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.431973+0000", 2023-09-23T23:40:05.747 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4789999999999996e-06 2023-09-23T23:40:05.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.747 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.747 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.464788+0000", 2023-09-23T23:40:05.747 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0328152420000001 2023-09-23T23:40:05.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.464845+0000", 2023-09-23T23:40:05.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7188000000000002e-05 2023-09-23T23:40:05.748 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.748 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.748 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.748 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172739 12.5 12:a6b54f33:::benchmark_data_smithi110_491193_object77864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:40:05.748 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.576924+0000", 2023-09-23T23:40:05.748 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 482.12623066100002, 2023-09-23T23:40:05.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91090562100000005, 2023-09-23T23:40:05.749 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.749 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:05.749 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.749 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.749 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.749 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172739 2023-09-23T23:40:05.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.749 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.749 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.576924+0000", 2023-09-23T23:40:05.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.576924+0000", 2023-09-23T23:40:05.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.576923+0000", 2023-09-23T23:40:05.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:40:05.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.576930+0000", 2023-09-23T23:40:05.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0820000000000001e-06 2023-09-23T23:40:05.751 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.576945+0000", 2023-09-23T23:40:05.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4877e-05 2023-09-23T23:40:05.751 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.576951+0000", 2023-09-23T23:40:05.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2029999999999997e-06 2023-09-23T23:40:05.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.752 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.577016+0000", 2023-09-23T23:40:05.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5184999999999993e-05 2023-09-23T23:40:05.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.752 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.577027+0000", 2023-09-23T23:40:05.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1542000000000001e-05 2023-09-23T23:40:05.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.464871+0000", 2023-09-23T23:40:05.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88784394600000005 2023-09-23T23:40:05.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.464949+0000", 2023-09-23T23:40:05.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7397000000000005e-05 2023-09-23T23:40:05.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:40:05.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.465320+0000", 2023-09-23T23:40:05.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00037099799999999999 2023-09-23T23:40:05.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.479792+0000", 2023-09-23T23:40:05.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014472485 2023-09-23T23:40:05.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487729+0000", 2023-09-23T23:40:05.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0079366209999999996 2023-09-23T23:40:05.755 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487760+0000", 2023-09-23T23:40:05.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1142000000000001e-05 2023-09-23T23:40:05.755 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:40:05.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487793+0000", 2023-09-23T23:40:05.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2712000000000001e-05 2023-09-23T23:40:05.755 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.756 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.756 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.756 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487830+0000", 2023-09-23T23:40:05.756 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7044e-05 2023-09-23T23:40:05.756 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.756 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.756 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.756 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.756 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.756 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172764 12.5 12:a4dfe0b2:::benchmark_data_smithi110_491193_object77889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:40:05.756 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.604922+0000", 2023-09-23T23:40:05.756 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 482.098232994, 2023-09-23T23:40:05.757 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88303478499999999, 2023-09-23T23:40:05.757 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.757 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:05.757 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.757 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.757 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.757 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172764 2023-09-23T23:40:05.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.757 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.757 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.757 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.604922+0000", 2023-09-23T23:40:05.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.758 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.604922+0000", 2023-09-23T23:40:05.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.758 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.604921+0000", 2023-09-23T23:40:05.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:40:05.759 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.759 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.759 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.759 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.604929+0000", 2023-09-23T23:40:05.759 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4419999999999998e-06 2023-09-23T23:40:05.759 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.759 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.759 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.759 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.604946+0000", 2023-09-23T23:40:05.759 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6554000000000001e-05 2023-09-23T23:40:05.759 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.759 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.760 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.760 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.604953+0000", 2023-09-23T23:40:05.760 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5120000000000002e-06 2023-09-23T23:40:05.760 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.760 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.760 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.760 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.605016+0000", 2023-09-23T23:40:05.760 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2572999999999999e-05 2023-09-23T23:40:05.760 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.760 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.760 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.760 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.605028+0000", 2023-09-23T23:40:05.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2335e-05 2023-09-23T23:40:05.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.761 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.761 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.467556+0000", 2023-09-23T23:40:05.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.86252822200000001 2023-09-23T23:40:05.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.761 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.761 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.467630+0000", 2023-09-23T23:40:05.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3326999999999998e-05 2023-09-23T23:40:05.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.762 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.762 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:40:05.762 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.467914+0000", 2023-09-23T23:40:05.762 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00028455700000000001 2023-09-23T23:40:05.762 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.762 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.762 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.762 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.479819+0000", 2023-09-23T23:40:05.762 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011905007 2023-09-23T23:40:05.762 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.762 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.762 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.763 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487890+0000", 2023-09-23T23:40:05.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0080704699999999997 2023-09-23T23:40:05.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.763 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.763 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.763 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487911+0000", 2023-09-23T23:40:05.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1698000000000001e-05 2023-09-23T23:40:05.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.763 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.763 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:40:05.763 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487936+0000", 2023-09-23T23:40:05.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4635e-05 2023-09-23T23:40:05.764 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.764 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.764 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.764 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487957+0000", 2023-09-23T23:40:05.764 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0724e-05 2023-09-23T23:40:05.764 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.764 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.764 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.764 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.764 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.764 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172776 12.5 12:a601ba3f:::benchmark_data_smithi110_491193_object77901:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:40:05.764 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.615330+0000", 2023-09-23T23:40:05.765 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 482.08782450400003, 2023-09-23T23:40:05.765 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.87300335500000004, 2023-09-23T23:40:05.765 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.765 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:05.765 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.765 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.765 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.765 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172776 2023-09-23T23:40:05.765 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.765 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.765 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.765 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.765 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.615330+0000", 2023-09-23T23:40:05.766 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.766 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.766 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.766 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.766 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.615330+0000", 2023-09-23T23:40:05.766 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.766 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.766 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.766 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.766 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.615329+0000", 2023-09-23T23:40:05.766 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:40:05.766 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.767 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.767 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.767 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.615337+0000", 2023-09-23T23:40:05.767 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6740000000000003e-06 2023-09-23T23:40:05.767 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.767 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.767 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.767 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.615354+0000", 2023-09-23T23:40:05.767 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6642000000000001e-05 2023-09-23T23:40:05.767 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.767 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.767 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.768 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.615363+0000", 2023-09-23T23:40:05.768 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7320000000000004e-06 2023-09-23T23:40:05.768 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.768 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.768 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.768 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.615431+0000", 2023-09-23T23:40:05.768 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8034999999999995e-05 2023-09-23T23:40:05.768 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.768 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.768 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.768 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.615442+0000", 2023-09-23T23:40:05.768 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1235e-05 2023-09-23T23:40:05.769 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.769 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.769 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.769 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.470198+0000", 2023-09-23T23:40:05.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.85475619300000005 2023-09-23T23:40:05.769 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.769 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.769 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.769 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.470310+0000", 2023-09-23T23:40:05.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112336 2023-09-23T23:40:05.769 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.769 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.769 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:40:05.770 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.470624+0000", 2023-09-23T23:40:05.770 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00031330399999999998 2023-09-23T23:40:05.770 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.770 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.770 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.770 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.479824+0000", 2023-09-23T23:40:05.770 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0092004470000000005 2023-09-23T23:40:05.770 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.770 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.770 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.770 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488042+0000", 2023-09-23T23:40:05.770 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0082178689999999992 2023-09-23T23:40:05.771 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.771 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.771 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.771 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488075+0000", 2023-09-23T23:40:05.771 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3098000000000001e-05 2023-09-23T23:40:05.771 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.771 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.771 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:40:05.771 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488119+0000", 2023-09-23T23:40:05.771 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3526999999999998e-05 2023-09-23T23:40:05.771 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.771 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.772 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.772 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488334+0000", 2023-09-23T23:40:05.772 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000214976 2023-09-23T23:40:05.772 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.772 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.772 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.772 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.772 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.772 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172861 12.5 12:a2093041:::benchmark_data_smithi110_491193_object77986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:40:05.772 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.736047+0000", 2023-09-23T23:40:05.772 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 481.96710735099998, 2023-09-23T23:40:05.772 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.75252812999999996, 2023-09-23T23:40:05.773 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.773 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:05.773 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.773 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.773 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.773 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172861 2023-09-23T23:40:05.773 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.773 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.773 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.773 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.773 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.736047+0000", 2023-09-23T23:40:05.773 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.774 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.774 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.736047+0000", 2023-09-23T23:40:05.774 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.774 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.774 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.736046+0000", 2023-09-23T23:40:05.774 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:40:05.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.736052+0000", 2023-09-23T23:40:05.775 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8329999999999999e-06 2023-09-23T23:40:05.775 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.736065+0000", 2023-09-23T23:40:05.775 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2875e-05 2023-09-23T23:40:05.775 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.736070+0000", 2023-09-23T23:40:05.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9169999999999997e-06 2023-09-23T23:40:05.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.776 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.776 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.776 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.736144+0000", 2023-09-23T23:40:05.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4196999999999995e-05 2023-09-23T23:40:05.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.776 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.776 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.776 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.736155+0000", 2023-09-23T23:40:05.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1036000000000001e-05 2023-09-23T23:40:05.777 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.777 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.472246+0000", 2023-09-23T23:40:05.777 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.73609044199999996 2023-09-23T23:40:05.777 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.777 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.472371+0000", 2023-09-23T23:40:05.777 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000125728 2023-09-23T23:40:05.777 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:40:05.778 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.472633+0000", 2023-09-23T23:40:05.778 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00026140100000000003 2023-09-23T23:40:05.778 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.778 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.778 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.778 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.482986+0000", 2023-09-23T23:40:05.778 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010353012999999999 2023-09-23T23:40:05.778 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.778 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.778 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.778 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488401+0000", 2023-09-23T23:40:05.779 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0054150520000000001 2023-09-23T23:40:05.779 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.779 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.779 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.779 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488505+0000", 2023-09-23T23:40:05.779 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010446 2023-09-23T23:40:05.779 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.779 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.779 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:40:05.779 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488553+0000", 2023-09-23T23:40:05.779 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7568000000000003e-05 2023-09-23T23:40:05.779 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.780 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.780 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.780 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488576+0000", 2023-09-23T23:40:05.780 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2682999999999999e-05 2023-09-23T23:40:05.780 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.780 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.780 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.780 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.780 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.780 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172904 12.5 12:a3ef1e59:::benchmark_data_smithi110_491193_object78029:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:40:05.780 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.795190+0000", 2023-09-23T23:40:05.780 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 481.907964687, 2023-09-23T23:40:05.781 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.69346639600000004, 2023-09-23T23:40:05.781 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.781 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:05.781 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.781 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.781 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.781 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172904 2023-09-23T23:40:05.781 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.781 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.781 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.781 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.781 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.795190+0000", 2023-09-23T23:40:05.782 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.782 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.782 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.782 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.782 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.795190+0000", 2023-09-23T23:40:05.782 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.782 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.782 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.782 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.782 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.795188+0000", 2023-09-23T23:40:05.782 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:40:05.782 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.783 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.783 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.783 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.795196+0000", 2023-09-23T23:40:05.783 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0489999999999998e-06 2023-09-23T23:40:05.783 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.783 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.783 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.783 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.795214+0000", 2023-09-23T23:40:05.783 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8461000000000002e-05 2023-09-23T23:40:05.783 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.783 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.783 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.784 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.795221+0000", 2023-09-23T23:40:05.784 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9890000000000003e-06 2023-09-23T23:40:05.784 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.784 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.784 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.784 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.795271+0000", 2023-09-23T23:40:05.784 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9607999999999997e-05 2023-09-23T23:40:05.784 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.784 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.784 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.784 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.795282+0000", 2023-09-23T23:40:05.784 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1039999999999999e-05 2023-09-23T23:40:05.785 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.785 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.785 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.785 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.474046+0000", 2023-09-23T23:40:05.785 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.67876398900000001 2023-09-23T23:40:05.785 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.785 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.785 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.785 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.474122+0000", 2023-09-23T23:40:05.785 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6681999999999995e-05 2023-09-23T23:40:05.785 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.785 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.786 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:40:05.786 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.474331+0000", 2023-09-23T23:40:05.786 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000209049 2023-09-23T23:40:05.786 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.786 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.786 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.786 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.482992+0000", 2023-09-23T23:40:05.786 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086603419999999997 2023-09-23T23:40:05.786 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.786 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.786 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.786 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488458+0000", 2023-09-23T23:40:05.786 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0054665870000000002 2023-09-23T23:40:05.787 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.787 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.787 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.787 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488611+0000", 2023-09-23T23:40:05.787 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015253099999999999 2023-09-23T23:40:05.787 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.787 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.787 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:40:05.787 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488637+0000", 2023-09-23T23:40:05.787 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6126e-05 2023-09-23T23:40:05.787 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.787 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.788 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.788 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488657+0000", 2023-09-23T23:40:05.788 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9629000000000001e-05 2023-09-23T23:40:05.788 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.788 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.788 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.788 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.788 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.788 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172905 12.5 12:a350bac5:::benchmark_data_smithi110_491193_object78030:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:40:05.788 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.801511+0000", 2023-09-23T23:40:05.788 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 481.90164377100001, 2023-09-23T23:40:05.788 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.68724216400000004, 2023-09-23T23:40:05.789 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.789 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:05.789 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.789 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.789 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.789 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172905 2023-09-23T23:40:05.789 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.789 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.789 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.789 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.789 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.801511+0000", 2023-09-23T23:40:05.789 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.790 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.790 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.790 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.790 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.801511+0000", 2023-09-23T23:40:05.790 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.790 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.790 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.790 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.790 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.801510+0000", 2023-09-23T23:40:05.790 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:40:05.790 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.791 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.801518+0000", 2023-09-23T23:40:05.791 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0390000000000005e-06 2023-09-23T23:40:05.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.791 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.801528+0000", 2023-09-23T23:40:05.791 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0577999999999999e-05 2023-09-23T23:40:05.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.791 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.801534+0000", 2023-09-23T23:40:05.792 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7119999999999997e-06 2023-09-23T23:40:05.792 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.792 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.792 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.792 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.801593+0000", 2023-09-23T23:40:05.792 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8794e-05 2023-09-23T23:40:05.792 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.792 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.792 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.792 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.801603+0000", 2023-09-23T23:40:05.792 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0423e-05 2023-09-23T23:40:05.792 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.793 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.793 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.793 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.475424+0000", 2023-09-23T23:40:05.793 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.67382106200000003 2023-09-23T23:40:05.793 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.793 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.793 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.793 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.475498+0000", 2023-09-23T23:40:05.793 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3946e-05 2023-09-23T23:40:05.793 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.793 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.793 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:40:05.794 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.475699+0000", 2023-09-23T23:40:05.794 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000200736 2023-09-23T23:40:05.794 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.794 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.794 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.794 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.483001+0000", 2023-09-23T23:40:05.794 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0073019419999999996 2023-09-23T23:40:05.794 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.794 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.794 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.794 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488685+0000", 2023-09-23T23:40:05.794 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0056844449999999998 2023-09-23T23:40:05.795 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.795 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.795 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.795 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488710+0000", 2023-09-23T23:40:05.795 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4898000000000001e-05 2023-09-23T23:40:05.795 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.795 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.795 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:40:05.795 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488734+0000", 2023-09-23T23:40:05.795 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3272999999999999e-05 2023-09-23T23:40:05.795 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.795 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.795 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.796 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488753+0000", 2023-09-23T23:40:05.796 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9647e-05 2023-09-23T23:40:05.796 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.796 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.796 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.796 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.796 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.796 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:172917 12.5 12:a628e390:::benchmark_data_smithi110_491193_object78042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:40:05.796 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:03.813634+0000", 2023-09-23T23:40:05.796 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 481.88952049300002, 2023-09-23T23:40:05.796 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.68256179699999997, 2023-09-23T23:40:05.796 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.797 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:05.797 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.797 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.797 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.797 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 172917 2023-09-23T23:40:05.797 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.797 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.797 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.797 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.797 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.813634+0000", 2023-09-23T23:40:05.797 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.797 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.797 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.798 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.798 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.813634+0000", 2023-09-23T23:40:05.798 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.798 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.798 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.798 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.798 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.813633+0000", 2023-09-23T23:40:05.798 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:40:05.798 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.798 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.798 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.799 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.813640+0000", 2023-09-23T23:40:05.799 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9399999999999996e-06 2023-09-23T23:40:05.799 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.799 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.799 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.799 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.813652+0000", 2023-09-23T23:40:05.799 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2490999999999999e-05 2023-09-23T23:40:05.799 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.799 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.799 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.799 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.813658+0000", 2023-09-23T23:40:05.799 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4439999999999999e-06 2023-09-23T23:40:05.799 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.800 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.800 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.800 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.813719+0000", 2023-09-23T23:40:05.800 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1523e-05 2023-09-23T23:40:05.800 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.800 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.800 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.800 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:03.813728+0000", 2023-09-23T23:40:05.800 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2450000000000004e-06 2023-09-23T23:40:05.800 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.800 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.800 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.476807+0000", 2023-09-23T23:40:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.66307843499999997 2023-09-23T23:40:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.476876+0000", 2023-09-23T23:40:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8788000000000006e-05 2023-09-23T23:40:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:40:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.477076+0000", 2023-09-23T23:40:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020076800000000001 2023-09-23T23:40:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.483006+0000", 2023-09-23T23:40:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0059295099999999998 2023-09-23T23:40:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488782+0000", 2023-09-23T23:40:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0057763950000000001 2023-09-23T23:40:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496102+0000", 2023-09-23T23:40:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0073202079999999999 2023-09-23T23:40:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:40:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496155+0000", 2023-09-23T23:40:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2954000000000002e-05 2023-09-23T23:40:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496196+0000", 2023-09-23T23:40:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0781999999999997e-05 2023-09-23T23:40:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:173047 12.5 12:a6b5faa1:::benchmark_data_smithi110_491193_object78172:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:40:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:04.026890+0000", 2023-09-23T23:40:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 481.67626473399997, 2023-09-23T23:40:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.46946490800000001, 2023-09-23T23:40:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 173047 2023-09-23T23:40:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.026890+0000", 2023-09-23T23:40:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.026890+0000", 2023-09-23T23:40:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.026889+0000", 2023-09-23T23:40:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:40:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.026893+0000", 2023-09-23T23:40:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9140000000000001e-06 2023-09-23T23:40:05.807 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.807 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.807 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.807 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.026903+0000", 2023-09-23T23:40:05.807 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.594e-06 2023-09-23T23:40:05.807 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.807 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.807 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.807 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.026909+0000", 2023-09-23T23:40:05.807 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8980000000000001e-06 2023-09-23T23:40:05.807 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.807 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.808 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.808 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.026971+0000", 2023-09-23T23:40:05.808 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2510999999999998e-05 2023-09-23T23:40:05.808 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.808 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.808 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.808 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.026980+0000", 2023-09-23T23:40:05.808 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9360000000000005e-06 2023-09-23T23:40:05.808 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.808 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.808 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.808 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.478207+0000", 2023-09-23T23:40:05.809 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.45122682600000003 2023-09-23T23:40:05.809 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.809 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.809 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.809 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.478311+0000", 2023-09-23T23:40:05.809 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000104338 2023-09-23T23:40:05.809 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.809 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.809 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:40:05.809 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.478579+0000", 2023-09-23T23:40:05.809 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00026786900000000001 2023-09-23T23:40:05.809 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.810 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.810 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.810 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.485830+0000", 2023-09-23T23:40:05.810 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0072510320000000001 2023-09-23T23:40:05.810 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.810 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.810 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.810 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.488804+0000", 2023-09-23T23:40:05.810 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029741030000000001 2023-09-23T23:40:05.810 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.810 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.810 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.811 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496254+0000", 2023-09-23T23:40:05.811 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0074498450000000001 2023-09-23T23:40:05.811 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.811 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.811 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:40:05.811 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496316+0000", 2023-09-23T23:40:05.811 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1923999999999999e-05 2023-09-23T23:40:05.811 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.811 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.811 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.811 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496355+0000", 2023-09-23T23:40:05.811 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8903999999999999e-05 2023-09-23T23:40:05.812 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.812 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.812 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.812 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.812 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.812 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:173052 12.5 12:a47f877d:::benchmark_data_smithi110_491193_object78177:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:40:05.812 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:04.037247+0000", 2023-09-23T23:40:05.812 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 481.66590816799999, 2023-09-23T23:40:05.812 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.45921886099999998, 2023-09-23T23:40:05.812 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.812 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:05.812 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.812 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.813 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.813 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 173052 2023-09-23T23:40:05.813 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.813 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.813 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.813 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.813 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.037247+0000", 2023-09-23T23:40:05.813 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.813 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.813 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.813 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.814 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.037247+0000", 2023-09-23T23:40:05.814 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.814 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.814 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.814 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.814 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.037245+0000", 2023-09-23T23:40:05.814 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:40:05.814 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.814 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.814 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.814 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.037252+0000", 2023-09-23T23:40:05.814 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.486e-06 2023-09-23T23:40:05.815 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.815 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.815 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.815 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.037268+0000", 2023-09-23T23:40:05.815 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5919000000000001e-05 2023-09-23T23:40:05.815 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.815 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.815 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.815 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.037276+0000", 2023-09-23T23:40:05.815 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9540000000000004e-06 2023-09-23T23:40:05.815 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.815 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.815 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.816 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.037338+0000", 2023-09-23T23:40:05.816 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1952000000000006e-05 2023-09-23T23:40:05.816 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.816 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.816 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.816 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.037351+0000", 2023-09-23T23:40:05.816 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2221e-05 2023-09-23T23:40:05.816 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.816 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.816 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.816 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.479867+0000", 2023-09-23T23:40:05.816 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.44251611099999999 2023-09-23T23:40:05.817 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.817 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.817 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.817 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.479940+0000", 2023-09-23T23:40:05.817 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2977000000000003e-05 2023-09-23T23:40:05.817 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.817 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.817 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:40:05.817 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.480183+0000", 2023-09-23T23:40:05.817 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00024294 2023-09-23T23:40:05.817 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.817 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.818 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.818 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.485834+0000", 2023-09-23T23:40:05.818 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0056517110000000002 2023-09-23T23:40:05.818 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.818 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.818 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.818 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.490636+0000", 2023-09-23T23:40:05.818 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004801619 2023-09-23T23:40:05.818 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.818 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.818 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.818 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496406+0000", 2023-09-23T23:40:05.819 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005769723 2023-09-23T23:40:05.819 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.819 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.819 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:40:05.819 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496436+0000", 2023-09-23T23:40:05.819 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0195000000000001e-05 2023-09-23T23:40:05.819 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.819 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.819 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.819 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496466+0000", 2023-09-23T23:40:05.819 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9723999999999999e-05 2023-09-23T23:40:05.819 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.820 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.820 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.820 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.820 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.820 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:173087 12.5 12:a280e1c5:::benchmark_data_smithi110_491193_object78212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:40:05.820 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:04.089799+0000", 2023-09-23T23:40:05.820 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 481.61335544899998, 2023-09-23T23:40:05.820 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.40679320899999999, 2023-09-23T23:40:05.820 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.820 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:05.820 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.820 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.821 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.821 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 173087 2023-09-23T23:40:05.821 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.821 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.821 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.821 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.821 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.089799+0000", 2023-09-23T23:40:05.821 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.821 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.821 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.821 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.821 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.089799+0000", 2023-09-23T23:40:05.822 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.822 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.822 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.822 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.822 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.089798+0000", 2023-09-23T23:40:05.822 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:40:05.822 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.822 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.822 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.822 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.089806+0000", 2023-09-23T23:40:05.822 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4929999999999998e-06 2023-09-23T23:40:05.822 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.823 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.823 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.089821+0000", 2023-09-23T23:40:05.823 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4915e-05 2023-09-23T23:40:05.823 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.823 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.823 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.089829+0000", 2023-09-23T23:40:05.823 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9519999999999994e-06 2023-09-23T23:40:05.823 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.823 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.824 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.089892+0000", 2023-09-23T23:40:05.824 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3378000000000002e-05 2023-09-23T23:40:05.824 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.824 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.824 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.824 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.089900+0000", 2023-09-23T23:40:05.824 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7370000000000006e-06 2023-09-23T23:40:05.824 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.824 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.824 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.824 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.481362+0000", 2023-09-23T23:40:05.824 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.391462225 2023-09-23T23:40:05.825 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.825 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.825 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.825 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.481469+0000", 2023-09-23T23:40:05.825 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010607600000000001 2023-09-23T23:40:05.825 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.825 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.825 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:40:05.825 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.481656+0000", 2023-09-23T23:40:05.825 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018705800000000001 2023-09-23T23:40:05.825 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.825 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.826 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.826 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487429+0000", 2023-09-23T23:40:05.826 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0057732809999999999 2023-09-23T23:40:05.826 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.826 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.826 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.826 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.490672+0000", 2023-09-23T23:40:05.826 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032433050000000001 2023-09-23T23:40:05.826 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.826 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.826 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.826 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496511+0000", 2023-09-23T23:40:05.827 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0058392180000000002 2023-09-23T23:40:05.827 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.827 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.827 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:40:05.827 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496557+0000", 2023-09-23T23:40:05.827 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5278999999999998e-05 2023-09-23T23:40:05.827 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.827 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.827 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.827 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496593+0000", 2023-09-23T23:40:05.827 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5932000000000001e-05 2023-09-23T23:40:05.827 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.828 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.828 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.828 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.828 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.828 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:173110 12.5 12:a6c8a621:::benchmark_data_smithi110_491193_object78235:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:40:05.828 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:04.128868+0000", 2023-09-23T23:40:05.828 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 481.57428675099999, 2023-09-23T23:40:05.828 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.36781228999999999, 2023-09-23T23:40:05.828 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.828 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:05.828 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.828 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.829 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.829 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 173110 2023-09-23T23:40:05.829 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.829 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.829 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.829 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.128868+0000", 2023-09-23T23:40:05.829 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.829 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.829 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.829 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.128868+0000", 2023-09-23T23:40:05.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.830 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.830 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.830 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.128867+0000", 2023-09-23T23:40:05.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:40:05.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.830 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.830 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.830 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.128872+0000", 2023-09-23T23:40:05.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.31e-06 2023-09-23T23:40:05.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.831 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.831 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.831 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.128886+0000", 2023-09-23T23:40:05.831 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4049999999999999e-05 2023-09-23T23:40:05.831 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.831 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.831 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.831 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.128892+0000", 2023-09-23T23:40:05.831 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3240000000000002e-06 2023-09-23T23:40:05.831 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.831 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.831 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.832 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.128955+0000", 2023-09-23T23:40:05.832 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3304000000000002e-05 2023-09-23T23:40:05.832 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.832 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.832 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.832 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.128968+0000", 2023-09-23T23:40:05.832 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2736e-05 2023-09-23T23:40:05.832 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.832 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.832 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.832 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.483042+0000", 2023-09-23T23:40:05.832 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.35407397299999999 2023-09-23T23:40:05.833 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.833 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.833 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.833 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.483126+0000", 2023-09-23T23:40:05.833 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4703999999999994e-05 2023-09-23T23:40:05.833 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.833 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.833 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:40:05.833 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.483325+0000", 2023-09-23T23:40:05.833 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019903100000000001 2023-09-23T23:40:05.833 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.833 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.834 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.834 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487435+0000", 2023-09-23T23:40:05.834 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041099760000000004 2023-09-23T23:40:05.834 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.834 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.834 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.834 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.490751+0000", 2023-09-23T23:40:05.834 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0033156359999999998 2023-09-23T23:40:05.834 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.834 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.835 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.835 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496626+0000", 2023-09-23T23:40:05.835 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0058748419999999999 2023-09-23T23:40:05.835 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.835 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.835 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:40:05.835 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496650+0000", 2023-09-23T23:40:05.835 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4660000000000001e-05 2023-09-23T23:40:05.835 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.835 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.835 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.835 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496680+0000", 2023-09-23T23:40:05.836 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9950999999999998e-05 2023-09-23T23:40:05.836 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.836 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.836 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.836 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.836 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.836 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46229.0:173132 12.5 12:a42e1715:::benchmark_data_smithi110_491193_object78257:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T23:40:05.836 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:32:04.163750+0000", 2023-09-23T23:40:05.836 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 481.539405343, 2023-09-23T23:40:05.836 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.33305025599999999, 2023-09-23T23:40:05.836 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.836 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:05.837 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.837 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46229", 2023-09-23T23:40:05.837 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:05.837 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 173132 2023-09-23T23:40:05.837 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.837 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.837 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.837 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.837 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.163750+0000", 2023-09-23T23:40:05.837 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.837 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.837 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.838 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.838 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.163750+0000", 2023-09-23T23:40:05.838 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.838 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.838 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.838 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.838 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.163748+0000", 2023-09-23T23:40:05.838 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:40:05.838 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.838 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.838 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.839 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.163755+0000", 2023-09-23T23:40:05.839 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7649999999999996e-06 2023-09-23T23:40:05.839 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.839 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.839 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.839 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.163770+0000", 2023-09-23T23:40:05.839 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5336e-05 2023-09-23T23:40:05.839 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.839 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.839 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.839 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.163777+0000", 2023-09-23T23:40:05.839 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6279999999999999e-06 2023-09-23T23:40:05.839 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.840 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.840 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.163832+0000", 2023-09-23T23:40:05.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5615999999999997e-05 2023-09-23T23:40:05.840 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.840 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.840 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.163840+0000", 2023-09-23T23:40:05.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3100000000000003e-06 2023-09-23T23:40:05.840 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.841 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.841 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.484458+0000", 2023-09-23T23:40:05.841 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.320618393 2023-09-23T23:40:05.841 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.841 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.841 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.841 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.484538+0000", 2023-09-23T23:40:05.841 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9394000000000006e-05 2023-09-23T23:40:05.841 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.841 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.841 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:40:05.841 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.484731+0000", 2023-09-23T23:40:05.842 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019382000000000001 2023-09-23T23:40:05.842 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.842 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.842 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.842 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.487685+0000", 2023-09-23T23:40:05.842 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029539240000000001 2023-09-23T23:40:05.842 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.842 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.842 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.842 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.494655+0000", 2023-09-23T23:40:05.842 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069701110000000002 2023-09-23T23:40:05.843 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496722+0000", 2023-09-23T23:40:05.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020667580000000001 2023-09-23T23:40:05.843 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:40:05.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496765+0000", 2023-09-23T23:40:05.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2719999999999998e-05 2023-09-23T23:40:05.843 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.844 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:32:04.496800+0000", 2023-09-23T23:40:05.844 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4879999999999998e-05 2023-09-23T23:40:05.844 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.844 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.844 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.844 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.844 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71517 13.1 13:8b0a5480:::benchmark_data_smithi110_541928_object71516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e555)", 2023-09-23T23:40:05.844 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.526261+0000", 2023-09-23T23:40:05.844 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 375.17689337500002, 2023-09-23T23:40:05.844 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3549986289999998, 2023-09-23T23:40:05.844 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.845 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:40:05.845 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.845 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:40:05.845 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:40:05.845 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71517 2023-09-23T23:40:05.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.845 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.526261+0000", 2023-09-23T23:40:05.845 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.526261+0000", 2023-09-23T23:40:05.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.526260+0000", 2023-09-23T23:40:05.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:40:05.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.847 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.526294+0000", 2023-09-23T23:40:05.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3726000000000002e-05 2023-09-23T23:40:05.847 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.847 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.847 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.526306+0000", 2023-09-23T23:40:05.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1528999999999999e-05 2023-09-23T23:40:05.847 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.847 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.847 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.526311+0000", 2023-09-23T23:40:05.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7289999999999997e-06 2023-09-23T23:40:05.848 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.848 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.848 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.848 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531327+0000", 2023-09-23T23:40:05.848 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050161060000000002 2023-09-23T23:40:05.848 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.848 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.848 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.848 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531460+0000", 2023-09-23T23:40:05.848 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013231000000000001 2023-09-23T23:40:05.848 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.848 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.849 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,4", 2023-09-23T23:40:05.849 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531617+0000", 2023-09-23T23:40:05.849 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000157026 2023-09-23T23:40:05.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.849 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.849 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.849 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.534849+0000", 2023-09-23T23:40:05.849 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032324279999999999 2023-09-23T23:40:05.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.849 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.849 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.849 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.553986+0000", 2023-09-23T23:40:05.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019137105000000001 2023-09-23T23:40:05.850 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.850 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.850 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.850 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.558234+0000", 2023-09-23T23:40:05.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0042477089999999997 2023-09-23T23:40:05.850 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.850 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.850 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.850 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.558245+0000", 2023-09-23T23:40:05.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1037e-05 2023-09-23T23:40:05.850 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.851 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.546510+0000", 2023-09-23T23:40:05.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98826527500000005 2023-09-23T23:40:05.851 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.851 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.546524+0000", 2023-09-23T23:40:05.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3667e-05 2023-09-23T23:40:05.851 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.851 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.852 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.881146+0000", 2023-09-23T23:40:05.852 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3346218910000001 2023-09-23T23:40:05.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.852 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.881260+0000", 2023-09-23T23:40:05.852 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000114296 2023-09-23T23:40:05.852 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.852 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.852 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.853 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71539 13.1 13:8eed35ed:::benchmark_data_smithi110_541928_object71538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:40:05.853 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.584298+0000", 2023-09-23T23:40:05.853 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 375.11885673699999, 2023-09-23T23:40:05.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3176904839999999, 2023-09-23T23:40:05.853 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:40:05.853 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:05.853 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:40:05.853 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:40:05.853 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:40:05.853 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71539 2023-09-23T23:40:05.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.854 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:40:05.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.854 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:40:05.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.584298+0000", 2023-09-23T23:40:05.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.854 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.854 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:40:05.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.584298+0000", 2023-09-23T23:40:05.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:40:05.854 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.855 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:40:05.855 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.584297+0000", 2023-09-23T23:40:05.855 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:40:05.855 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.855 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.855 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:40:05.855 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.584362+0000", 2023-09-23T23:40:05.855 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4276999999999995e-05 2023-09-23T23:40:05.855 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.855 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.855 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:40:05.855 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.584371+0000", 2023-09-23T23:40:05.856 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3479999999999993e-06 2023-09-23T23:40:05.856 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.856 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.856 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:40:05.856 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.584377+0000", 2023-09-23T23:40:05.856 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6300000000000003e-06 2023-09-23T23:40:05.856 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.856 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.856 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.856 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.584429+0000", 2023-09-23T23:40:05.856 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2410000000000001e-05 2023-09-23T23:40:05.856 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.584437+0000", 2023-09-23T23:40:05.857 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2090000000000005e-06 2023-09-23T23:40:05.857 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.546551+0000", 2023-09-23T23:40:05.857 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96211402999999995 2023-09-23T23:40:05.857 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:40:05.858 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.546559+0000", 2023-09-23T23:40:05.858 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6480000000000003e-06 2023-09-23T23:40:05.858 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.858 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.858 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:40:05.858 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.881286+0000", 2023-09-23T23:40:05.858 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3347272050000001 2023-09-23T23:40:05.858 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.858 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.858 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:40:05.858 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.881451+0000", 2023-09-23T23:40:05.858 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016506 2023-09-23T23:40:05.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.859 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-09-23T23:40:05.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.881663+0000", 2023-09-23T23:40:05.859 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021163399999999999 2023-09-23T23:40:05.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.859 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:40:05.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.891533+0000", 2023-09-23T23:40:05.859 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0098703790000000003 2023-09-23T23:40:05.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.894429+0000", 2023-09-23T23:40:05.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028962229999999999 2023-09-23T23:40:05.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:05.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.897756+0000", 2023-09-23T23:40:05.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003327048 2023-09-23T23:40:05.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:40:05.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.897834+0000", 2023-09-23T23:40:05.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7341000000000004e-05 2023-09-23T23:40:05.861 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:40:05.861 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:40:05.861 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:40:05.861 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.901989+0000", 2023-09-23T23:40:05.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041549009999999999 2023-09-23T23:40:05.861 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.861 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.861 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.861 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:40:05.861 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:40:05.861 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:40:05.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:05.863 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:40:05.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:40:05.964 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:40:05.991 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:06.038 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:40:06.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:06.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:40:06.113 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:06 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 11405/155958 objects degraded (7.313%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:06.142 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:40:06.171 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:06.224 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:40:06.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:06.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:40:06.235 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:40:06.235 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T23:40:06.235 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:40:06.329 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:40:06.395 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:06 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 11405/155958 objects degraded (7.313%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:06.406 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:06.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:06 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 11405/155958 objects degraded (7.313%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:06.512 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:40:06.677 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 44 16 53714 53698 76.2415 48.3125 0.0337235 0.0131105 2023-09-23T23:40:07.113 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:40:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:40:06] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.43.0" 2023-09-23T23:40:07.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:07 smithi138 ceph-mon[193606]: pgmap v1798: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 50 MiB/s wr, 800 op/s; 11405/155958 objects degraded (7.313%); 2480/155958 objects misplaced (1.590%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:40:07.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:07 smithi110 ceph-mon[212939]: pgmap v1798: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 50 MiB/s wr, 800 op/s; 11405/155958 objects degraded (7.313%); 2480/155958 objects misplaced (1.590%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:40:07.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:07 smithi110 ceph-mon[221102]: pgmap v1798: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 50 MiB/s wr, 800 op/s; 11405/155958 objects degraded (7.313%); 2480/155958 objects misplaced (1.590%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:40:07.476 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:40:07.677 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 45 16 54442 54426 75.5585 45.5 0.0284024 0.0132288 2023-09-23T23:40:08.226 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:40:08.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:08.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:40:08.332 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:40:08.396 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:08.680 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 46 16 55013 54997 74.689 35.6875 0.03329 0.0133817 2023-09-23T23:40:08.731 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:40:08.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:08.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:40:08.843 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:40:08.855 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:40:08.887 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:08.935 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:40:08.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:08.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:40:09.008 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:40:09.040 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:40:09.069 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:09.120 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:40:09.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:09.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:40:09.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:09 smithi138 ceph-mon[193606]: pgmap v1799: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 45 MiB/s wr, 713 op/s; 11405/155958 objects degraded (7.313%); 2480/155958 objects misplaced (1.590%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:40:09.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:09 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:40:09.227 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:40:09.256 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:09.309 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:40:09.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:09.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:40:09.417 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:40:09.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:09 smithi110 ceph-mon[221102]: pgmap v1799: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 45 MiB/s wr, 713 op/s; 11405/155958 objects degraded (7.313%); 2480/155958 objects misplaced (1.590%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:40:09.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:09 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:40:09.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:09 smithi110 ceph-mon[212939]: pgmap v1799: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 45 MiB/s wr, 713 op/s; 11405/155958 objects degraded (7.313%); 2480/155958 objects misplaced (1.590%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:40:09.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:09 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:40:09.525 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:09.682 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 47 16 55768 55752 74.0989 47.1875 0.0175501 0.0134855 2023-09-23T23:40:09.714 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:40:09.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:09.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:40:09.822 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:40:09.855 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:09.909 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:40:09.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:09.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:40:10.018 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:40:10.048 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:10.099 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:40:10.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:10.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:40:10.179 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:40:10.210 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:40:10.275 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:10 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1360587855' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:40:10.275 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:10 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:40:10.275 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:10 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1360587855' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:40:10.275 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:10 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:40:10.277 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:10.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:10 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1360587855' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:40:10.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:10 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:40:10.683 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 48 16 56443 56427 73.4343 42.1875 0.00701347 0.0135644 2023-09-23T23:40:11.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:11 smithi138 ceph-mon[193606]: pgmap v1800: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 48 MiB/s wr, 762 op/s; 11405/161760 objects degraded (7.051%); 2184/161760 objects misplaced (1.350%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T23:40:11.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:11 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:40:11.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:11 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:40:11.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:11 smithi138 ceph-mon[193606]: osdmap e632: 8 total, 8 up, 6 in 2023-09-23T23:40:11.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:11 smithi138 ceph-mon[193606]: failed to encode map e632 with expected crc 2023-09-23T23:40:11.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:11 smithi138 ceph-mon[193606]: failed to encode map e632 with expected crc 2023-09-23T23:40:11.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:11 smithi138 ceph-mon[193606]: failed to encode map e632 with expected crc 2023-09-23T23:40:11.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:11 smithi138 ceph-mon[193606]: failed to encode map e632 with expected crc 2023-09-23T23:40:11.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:11 smithi110 ceph-mon[212939]: pgmap v1800: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 48 MiB/s wr, 762 op/s; 11405/161760 objects degraded (7.051%); 2184/161760 objects misplaced (1.350%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T23:40:11.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:11 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:40:11.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:11 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:40:11.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:11 smithi110 ceph-mon[212939]: osdmap e632: 8 total, 8 up, 6 in 2023-09-23T23:40:11.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:11 smithi110 ceph-mon[212939]: failed to encode map e632 with expected crc 2023-09-23T23:40:11.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:11 smithi110 ceph-mon[212939]: failed to encode map e632 with expected crc 2023-09-23T23:40:11.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:11 smithi110 ceph-mon[212939]: failed to encode map e632 with expected crc 2023-09-23T23:40:11.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:11 smithi110 ceph-mon[212939]: failed to encode map e632 with expected crc 2023-09-23T23:40:11.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:11 smithi110 ceph-mon[221102]: pgmap v1800: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 48 MiB/s wr, 762 op/s; 11405/161760 objects degraded (7.051%); 2184/161760 objects misplaced (1.350%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T23:40:11.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:11 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:40:11.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:11 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:40:11.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:11 smithi110 ceph-mon[221102]: osdmap e632: 8 total, 8 up, 6 in 2023-09-23T23:40:11.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:11 smithi110 ceph-mon[221102]: failed to encode map e632 with expected crc 2023-09-23T23:40:11.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:11 smithi110 ceph-mon[221102]: failed to encode map e632 with expected crc 2023-09-23T23:40:11.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:11 smithi110 ceph-mon[221102]: failed to encode map e632 with expected crc 2023-09-23T23:40:11.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:11 smithi110 ceph-mon[221102]: failed to encode map e632 with expected crc 2023-09-23T23:40:11.586 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:40:11.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:11.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:40:11.683 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 49 16 57337 57321 73.076 55.875 0.014148 0.0136798 2023-09-23T23:40:11.700 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:40:11.787 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:12.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:12 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 11405/166527 objects degraded (6.849%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:12.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:12 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 11405/166527 objects degraded (6.849%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:12.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:12 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 11405/166527 objects degraded (6.849%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:12.683 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 50 16 58144 58128 72.6232 50.4375 0.0200133 0.0137627 2023-09-23T23:40:13.024 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:40:13.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:13.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:40:13.134 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:40:13.162 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:13.216 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:40:13.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:13.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:40:13.326 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:40:13.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:13 smithi110 ceph-mon[212939]: pgmap v1802: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 46 MiB/s wr, 737 op/s; 11405/166527 objects degraded (6.849%); 2184/166527 objects misplaced (1.311%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T23:40:13.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:40:13.345 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:40:13.345 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:13 smithi110 ceph-mon[221102]: pgmap v1802: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 46 MiB/s wr, 737 op/s; 11405/166527 objects degraded (6.849%); 2184/166527 objects misplaced (1.311%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T23:40:13.345 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:40:13.345 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:40:13.355 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:13.402 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:40:13.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:13.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:40:13.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:13 smithi138 ceph-mon[193606]: pgmap v1802: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 46 MiB/s wr, 737 op/s; 11405/166527 objects degraded (6.849%); 2184/166527 objects misplaced (1.311%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T23:40:13.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:40:13.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:40:13.511 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:40:13.540 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:13.598 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:40:13.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:13.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:40:13.681 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 5, 7, 2, 1] out_osds: [4, 6] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:40:13.681 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:40:13.681 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T23:40:13.681 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T23:40:13.683 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 51 16 58950 58934 72.1871 50.375 0.0137654 0.013846 2023-09-23T23:40:13.705 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:40:13.735 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:13.785 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:40:13.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:13.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:40:13.891 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:40:13.989 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:14.176 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:40:14.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:14.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:40:14.240 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:40:14.285 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:40:14.288 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:40:14.316 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:14.334 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:40:14.367 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:40:14.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:14.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:40:14.372 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:40:14.405 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:40:14.454 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:40:14.454 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:40:14.454 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:40:14.473 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:40:14.501 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:40:14.505 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:14.531 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:40:14.544 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:40:14.551 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:40:14.551 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:40:14.551 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:40:14.552 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:40:14.552 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:40:14.552 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:40:14.552 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:40:14.552 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:40:14.554 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:40:14.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:14.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:40:14.661 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:40:14.683 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 52 16 59617 59601 71.6007 41.6875 0.0267288 0.0139598 2023-09-23T23:40:14.690 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:14.742 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:40:14.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:14.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:40:14.847 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:40:14.876 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:14.931 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:40:14.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:14.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:40:14.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:14 smithi138 ceph-mon[193606]: pgmap v1803: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 744 op/s; 11405/172638 objects degraded (6.606%); 1816/172638 objects misplaced (1.052%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T23:40:14.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:14 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1835669921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:40:15.040 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:40:15.068 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:15.118 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:40:15.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:15.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:40:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:14 smithi110 ceph-mon[212939]: pgmap v1803: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 744 op/s; 11405/172638 objects degraded (6.606%); 1816/172638 objects misplaced (1.052%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T23:40:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:14 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1835669921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:40:15.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:14 smithi110 ceph-mon[221102]: pgmap v1803: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 744 op/s; 11405/172638 objects degraded (6.606%); 1816/172638 objects misplaced (1.052%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T23:40:15.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:14 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1835669921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:40:15.225 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:40:15.257 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:15.304 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:40:15.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:15.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:40:15.380 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:40:15.410 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:40:15.498 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:15.683 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 53 16 60402 60386 71.1755 49.0625 0.0105108 0.0140435 2023-09-23T23:40:16.560 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:40:16.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:16.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:40:16.664 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:40:16.683 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 54 16 61121 61105 70.6897 44.9375 0.0214316 0.01414 2023-09-23T23:40:16.731 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:16.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:40:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:40:16] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.43.0" 2023-09-23T23:40:17.141 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:40:17.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:17.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:40:17.164 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:40:17.264 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:40:17.334 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:17 smithi110 ceph-mon[221102]: pgmap v1804: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 747 op/s; 11405/178401 objects degraded (6.393%); 1816/178401 objects misplaced (1.018%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T23:40:17.334 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:17 smithi110 ceph-mon[212939]: pgmap v1804: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 747 op/s; 11405/178401 objects degraded (6.393%); 1816/178401 objects misplaced (1.018%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T23:40:17.403 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:17.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:17 smithi138 ceph-mon[193606]: pgmap v1804: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 747 op/s; 11405/178401 objects degraded (6.393%); 1816/178401 objects misplaced (1.018%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T23:40:17.596 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:40:17.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:17.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:40:17.648 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:40:17.685 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 55 16 61875 61859 70.2596 47.125 0.0226751 0.0142265 2023-09-23T23:40:17.702 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:40:17.729 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:17.780 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:40:17.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:17.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:40:17.885 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:40:17.951 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:18.077 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:40:18.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:18.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:40:18.181 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:40:18.232 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:18.274 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:40:18.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:18.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:40:18.377 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:40:18.404 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:18.446 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:40:18.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:18.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:40:18.553 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:40:18.579 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:18.620 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:40:18.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:18.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:40:18.685 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 56 16 62339 62323 69.5225 29 0.00924568 0.0143775 2023-09-23T23:40:18.726 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:40:18.792 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:19.052 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:40:19.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:19.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:40:19.160 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:40:19.191 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:19.199 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:40:19.239 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:40:19.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:19.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:40:19.344 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:40:19.362 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:19 smithi110 ceph-mon[212939]: pgmap v1805: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 747 op/s; 11405/178401 objects degraded (6.393%); 1816/178401 objects misplaced (1.018%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T23:40:19.362 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:19 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 11405/178401 objects degraded (6.393%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:19.362 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:19 smithi110 ceph-mon[221102]: pgmap v1805: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 747 op/s; 11405/178401 objects degraded (6.393%); 1816/178401 objects misplaced (1.018%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T23:40:19.362 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:19 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 11405/178401 objects degraded (6.393%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:19 smithi138 ceph-mon[193606]: pgmap v1805: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 747 op/s; 11405/178401 objects degraded (6.393%); 1816/178401 objects misplaced (1.018%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T23:40:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:19 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 11405/178401 objects degraded (6.393%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:19.452 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:19.654 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:40:19.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:19.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:40:19.688 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 57 16 63088 63072 69.1204 46.8125 0.0122897 0.0144613 2023-09-23T23:40:19.759 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:40:19.787 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:19.840 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:40:19.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:19.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:40:19.946 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:40:20.036 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:20.689 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 58 16 63808 63792 68.7046 45 0.00644637 0.0145495 2023-09-23T23:40:20.714 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:40:20.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:20.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:40:20.821 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:40:20.898 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:40:20.920 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:21.085 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:40:21.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:21.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:40:21.197 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:40:21.224 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:21.273 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:40:21.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:21.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:40:21.323 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:21 smithi110 ceph-mon[212939]: pgmap v1806: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 749 op/s; 11405/184257 objects degraded (6.190%); 1424/184257 objects misplaced (0.773%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T23:40:21.323 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:21 smithi110 ceph-mon[221102]: pgmap v1806: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 749 op/s; 11405/184257 objects degraded (6.190%); 1424/184257 objects misplaced (0.773%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T23:40:21.379 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:40:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:21 smithi138 ceph-mon[193606]: pgmap v1806: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 749 op/s; 11405/184257 objects degraded (6.190%); 1424/184257 objects misplaced (0.773%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T23:40:21.502 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:21.689 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 59 16 64491 64475 68.2638 42.6875 0.0145555 0.0146417 2023-09-23T23:40:21.694 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:40:21.709 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:40:21.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:21.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:40:21.781 INFO:teuthology.orchestra.run.smithi110.stdout:moved 27 / 357 (7.56302%) 2023-09-23T23:40:21.781 INFO:teuthology.orchestra.run.smithi110.stdout:avg 59.5 2023-09-23T23:40:21.781 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 6.44851 -> 5.56028 (expected baseline 7.04154) 2023-09-23T23:40:21.782 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.2 with 47 -> 53 pgs (0.789916 -> 0.890756 * mean) 2023-09-23T23:40:21.782 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.0 with 68 -> 55 pgs (1.14286 -> 0.92437 * mean) 2023-09-23T23:40:21.782 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:40:21.782 INFO:teuthology.orchestra.run.smithi110.stdout:oload 110 2023-09-23T23:40:21.782 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 1 2023-09-23T23:40:21.782 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:40:21.782 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0294 2023-09-23T23:40:21.782 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.0324 2023-09-23T23:40:21.782 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 weight 1.0000 -> 0.7745 2023-09-23T23:40:21.783 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 weight 1.0000 -> 0.8489 2023-09-23T23:40:21.783 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 weight 1.0000 -> 0.9060 2023-09-23T23:40:21.788 INFO:teuthology.orchestra.run.smithi110.stderr:no change 2023-09-23T23:40:21.823 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:40:21.854 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:21.902 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:40:21.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:21.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:40:21.959 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:40:21 smithi110 conmon[198813]: 2023-09-23T23:40:21.769+0000 7ff0150b6700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T23:40:22.008 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:40:22.082 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:22.626 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:40:22.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:22.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:40:22.689 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:40:22.688022+0000 min lat: 0.00116231 max lat: 2.39686 avg lat: 0.014715 2023-09-23T23:40:22.689 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:40:22.689 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 60 16 65262 65246 67.9293 48.1875 0.0105814 0.014715 2023-09-23T23:40:22.739 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:40:22.769 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:22.822 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:40:22.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:22.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:40:22.929 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:40:22.958 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:23.001 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:40:23.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:23.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:40:23.106 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:40:23.162 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:40:23.201 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:23.412 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:23 smithi110 ceph-mon[212939]: from='client.37197 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:40:23.413 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:23 smithi110 ceph-mon[212939]: pgmap v1807: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 47 MiB/s wr, 755 op/s; 11405/188682 objects degraded (6.045%); 1424/188682 objects misplaced (0.755%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T23:40:23.413 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:23 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/357434470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:40:23.413 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:23 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:40:23.413 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:23 smithi110 ceph-mon[221102]: from='client.37197 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:40:23.413 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:23 smithi110 ceph-mon[221102]: pgmap v1807: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 47 MiB/s wr, 755 op/s; 11405/188682 objects degraded (6.045%); 1424/188682 objects misplaced (0.755%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T23:40:23.413 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:23 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/357434470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:40:23.413 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:23 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:40:23.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:23 smithi138 ceph-mon[193606]: from='client.37197 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:40:23.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:23 smithi138 ceph-mon[193606]: pgmap v1807: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 47 MiB/s wr, 755 op/s; 11405/188682 objects degraded (6.045%); 1424/188682 objects misplaced (0.755%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T23:40:23.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:23 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/357434470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:40:23.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:23 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:40:23.689 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 61 16 65876 65860 67.4449 38.375 0.0185531 0.014821 2023-09-23T23:40:23.869 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:40:23.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:23.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:40:23.979 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:40:24.062 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:24.150 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:24 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:40:24.150 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:24 smithi110 ceph-mon[212939]: failed to encode map e633 with expected crc 2023-09-23T23:40:24.150 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:24 smithi110 ceph-mon[212939]: failed to encode map e633 with expected crc 2023-09-23T23:40:24.150 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:24 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:40:24.150 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:24 smithi110 ceph-mon[212939]: osdmap e633: 8 total, 8 up, 6 in 2023-09-23T23:40:24.150 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:24 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:40:24.150 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:24 smithi110 ceph-mon[221102]: failed to encode map e633 with expected crc 2023-09-23T23:40:24.151 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:24 smithi110 ceph-mon[221102]: failed to encode map e633 with expected crc 2023-09-23T23:40:24.151 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:24 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:40:24.151 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:24 smithi110 ceph-mon[221102]: osdmap e633: 8 total, 8 up, 6 in 2023-09-23T23:40:24.151 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:24 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 11405/188682 objects degraded (6.045%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:24.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:24 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:40:24.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:24 smithi138 ceph-mon[193606]: failed to encode map e633 with expected crc 2023-09-23T23:40:24.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:24 smithi138 ceph-mon[193606]: failed to encode map e633 with expected crc 2023-09-23T23:40:24.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:24 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:40:24.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:24 smithi138 ceph-mon[193606]: osdmap e633: 8 total, 8 up, 6 in 2023-09-23T23:40:24.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:24 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 11405/188682 objects degraded (6.045%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:24.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:24 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 11405/188682 objects degraded (6.045%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:24.689 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 62 16 66735 66719 67.223 53.6875 0.0186329 0.0148701 2023-09-23T23:40:25.232 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T23:40:25.232 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-09-23T23:40:25.332 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:40:25.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:25.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:40:25.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:25 smithi138 ceph-mon[193606]: pgmap v1809: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 45 MiB/s wr, 712 op/s; 11405/194052 objects degraded (5.877%); 916/194052 objects misplaced (0.472%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T23:40:25.445 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:40:25.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:25 smithi110 ceph-mon[212939]: pgmap v1809: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 45 MiB/s wr, 712 op/s; 11405/194052 objects degraded (5.877%); 916/194052 objects misplaced (0.472%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T23:40:25.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:25 smithi110 ceph-mon[221102]: pgmap v1809: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 45 MiB/s wr, 712 op/s; 11405/194052 objects degraded (5.877%); 916/194052 objects misplaced (0.472%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T23:40:25.472 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:25.519 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:40:25.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:25.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:40:25.625 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:40:25.694 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 63 16 67370 67354 66.7811 39.6875 0.0275825 0.0149661 2023-09-23T23:40:25.699 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:26.262 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:40:26.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:26.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:40:26.386 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:40:26.413 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:26.455 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:40:26.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:26.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:40:26.559 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:40:26.586 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:26.682 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:40:26.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:26.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:40:26.694 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 64 16 67939 67923 66.2935 35.5625 0.0202782 0.0150788 2023-09-23T23:40:26.789 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:40:26.819 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:26.874 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:40:26.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:26.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:40:26.954 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:40:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:40:26] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.43.0" 2023-09-23T23:40:26.980 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:40:27.010 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:27.060 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:40:27.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:27.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:40:27.165 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:40:27.217 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:40:27.217 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:40:27.287 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:27 smithi138 ceph-mon[193606]: pgmap v1810: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 44 MiB/s wr, 701 op/s; 11405/199467 objects degraded (5.718%); 916/199467 objects misplaced (0.459%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T23:40:27.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:27 smithi110 ceph-mon[212939]: pgmap v1810: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 44 MiB/s wr, 701 op/s; 11405/199467 objects degraded (5.718%); 916/199467 objects misplaced (0.459%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T23:40:27.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:27 smithi110 ceph-mon[221102]: pgmap v1810: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 44 MiB/s wr, 701 op/s; 11405/199467 objects degraded (5.718%); 916/199467 objects misplaced (0.459%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T23:40:27.621 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:40:27.694 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 65 16 68704 68688 66.0091 47.8125 0.00887247 0.0151426 2023-09-23T23:40:27.938 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:40:27.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:27.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:40:28.043 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:40:28.071 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:28.124 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:40:28.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:28.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:40:28.229 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:40:28.297 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:28.694 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 66 16 69412 69396 65.6795 44.25 0.0034366 0.0152196 2023-09-23T23:40:28.698 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:40:28.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:28.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:40:28.716 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:40:28.758 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:40:28.806 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:40:28.968 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:40:28.995 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:29.014 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:40:29.070 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:40:29.124 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:40:29.136 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:29 smithi110 ceph-mon[212939]: pgmap v1811: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 44 MiB/s wr, 701 op/s; 11405/199467 objects degraded (5.718%); 916/199467 objects misplaced (0.459%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T23:40:29.136 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:29 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:40:29.136 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:29 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:40:29.136 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:29 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 11405/199467 objects degraded (5.718%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:29.136 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:29 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4182540338' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:40:29.136 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:29 smithi110 ceph-mon[221102]: pgmap v1811: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 44 MiB/s wr, 701 op/s; 11405/199467 objects degraded (5.718%); 916/199467 objects misplaced (0.459%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T23:40:29.137 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:29 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:40:29.137 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:29 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:40:29.137 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:29 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 11405/199467 objects degraded (5.718%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:29.137 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:29 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4182540338' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:40:29.158 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:40:29.159 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:40:29.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:29.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:40:29.203 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:40:29.246 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:40:29.276 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:40:29.287 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:40:29.292 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:40:29.293 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:40:29.293 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:40:29.293 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:40:29.293 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:40:29.293 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:40:29.293 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:40:29.293 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:40:29.301 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:29 smithi138 ceph-mon[193606]: pgmap v1811: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 44 MiB/s wr, 701 op/s; 11405/199467 objects degraded (5.718%); 916/199467 objects misplaced (0.459%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T23:40:29.301 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:29 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:40:29.302 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:29 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:40:29.302 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:29 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 11405/199467 objects degraded (5.718%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:29.302 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:29 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4182540338' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:40:29.304 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:29.359 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:40:29.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:29.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:40:29.466 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:40:29.572 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:29.694 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 67 16 70047 70031 65.2917 39.6875 0.0269575 0.0153095 2023-09-23T23:40:30.574 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:40:30.694 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 68 16 70737 70721 64.9658 43.125 0.0130106 0.015387 2023-09-23T23:40:31.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:31 smithi110 ceph-mon[221102]: pgmap v1812: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 702 op/s; 11405/205338 objects degraded (5.554%); 369/205338 objects misplaced (0.180%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T23:40:31.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:31 smithi110 ceph-mon[212939]: pgmap v1812: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 702 op/s; 11405/205338 objects degraded (5.554%); 369/205338 objects misplaced (0.180%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T23:40:31.441 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:40:31.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:31 smithi138 ceph-mon[193606]: pgmap v1812: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 702 op/s; 11405/205338 objects degraded (5.554%); 369/205338 objects misplaced (0.180%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T23:40:31.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:31.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:40:31.546 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:40:31.684 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:31.694 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 69 16 71425 71409 64.6475 43 0.0263854 0.0154605 2023-09-23T23:40:31.816 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:40:31.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:31.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:40:31.936 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:40:31.972 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:32.032 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:40:32.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:32.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:40:32.125 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:40:32.136 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:40:32.248 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:32.695 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 70 16 71861 71845 64.1135 27.25 0.00355423 0.0154995 2023-09-23T23:40:32.961 INFO:teuthology.orchestra.run.smithi110.stdout:moved 0 / 357 (0%) 2023-09-23T23:40:32.961 INFO:teuthology.orchestra.run.smithi110.stdout:avg 59.5 2023-09-23T23:40:32.961 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 6.44851 -> 6.44851 (expected baseline 7.04154) 2023-09-23T23:40:32.961 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.2 with 47 -> 47 pgs (0.789916 -> 0.789916 * mean) 2023-09-23T23:40:32.961 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.0 with 68 -> 68 pgs (1.14286 -> 1.14286 * mean) 2023-09-23T23:40:32.961 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:40:32.962 INFO:teuthology.orchestra.run.smithi110.stdout:oload 1000 2023-09-23T23:40:32.962 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 0.05 2023-09-23T23:40:32.962 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:40:32.962 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0324 2023-09-23T23:40:32.962 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.3240 2023-09-23T23:40:32.963 INFO:teuthology.orchestra.run.smithi110.stderr:no change 2023-09-23T23:40:33.295 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:40:33.370 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:33 smithi110 ceph-mon[212939]: pgmap v1813: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 708 op/s; 11405/209958 objects degraded (5.432%); 369/209958 objects misplaced (0.176%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T23:40:33.370 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:33 smithi110 ceph-mon[212939]: osdmap e634: 8 total, 8 up, 6 in 2023-09-23T23:40:33.370 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:33 smithi110 ceph-mon[212939]: failed to encode map e634 with expected crc 2023-09-23T23:40:33.370 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:33 smithi110 ceph-mon[212939]: failed to encode map e634 with expected crc 2023-09-23T23:40:33.370 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:33 smithi110 ceph-mon[212939]: failed to encode map e634 with expected crc 2023-09-23T23:40:33.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:33 smithi110 ceph-mon[221102]: pgmap v1813: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 708 op/s; 11405/209958 objects degraded (5.432%); 369/209958 objects misplaced (0.176%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T23:40:33.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:33 smithi110 ceph-mon[221102]: osdmap e634: 8 total, 8 up, 6 in 2023-09-23T23:40:33.370 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:33 smithi110 ceph-mon[221102]: failed to encode map e634 with expected crc 2023-09-23T23:40:33.371 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:33 smithi110 ceph-mon[221102]: failed to encode map e634 with expected crc 2023-09-23T23:40:33.371 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:33 smithi110 ceph-mon[221102]: failed to encode map e634 with expected crc 2023-09-23T23:40:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:33 smithi138 ceph-mon[193606]: pgmap v1813: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 708 op/s; 11405/209958 objects degraded (5.432%); 369/209958 objects misplaced (0.176%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T23:40:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:33 smithi138 ceph-mon[193606]: osdmap e634: 8 total, 8 up, 6 in 2023-09-23T23:40:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:33 smithi138 ceph-mon[193606]: failed to encode map e634 with expected crc 2023-09-23T23:40:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:33 smithi138 ceph-mon[193606]: failed to encode map e634 with expected crc 2023-09-23T23:40:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:33 smithi138 ceph-mon[193606]: failed to encode map e634 with expected crc 2023-09-23T23:40:33.697 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 71 16 72254 72238 63.5541 24.5625 0.0324783 0.0157272 2023-09-23T23:40:33.828 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:40:33.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:33.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:40:33.941 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:40:34.031 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:34.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:34 smithi110 ceph-mon[212939]: from='client.47243 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:40:34.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:34 smithi110 ceph-mon[212939]: failed to encode map e635 with expected crc 2023-09-23T23:40:34.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:34 smithi110 ceph-mon[212939]: osdmap e635: 8 total, 8 up, 6 in 2023-09-23T23:40:34.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:34 smithi110 ceph-mon[212939]: failed to encode map e635 with expected crc 2023-09-23T23:40:34.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:34 smithi110 ceph-mon[212939]: failed to encode map e635 with expected crc 2023-09-23T23:40:34.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:34 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 11405/209958 objects degraded (5.432%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:34.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:34 smithi110 ceph-mon[221102]: from='client.47243 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:40:34.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:34 smithi110 ceph-mon[221102]: failed to encode map e635 with expected crc 2023-09-23T23:40:34.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:34 smithi110 ceph-mon[221102]: osdmap e635: 8 total, 8 up, 6 in 2023-09-23T23:40:34.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:34 smithi110 ceph-mon[221102]: failed to encode map e635 with expected crc 2023-09-23T23:40:34.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:34 smithi110 ceph-mon[221102]: failed to encode map e635 with expected crc 2023-09-23T23:40:34.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:34 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 11405/209958 objects degraded (5.432%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:34 smithi138 ceph-mon[193606]: from='client.47243 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:40:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:34 smithi138 ceph-mon[193606]: failed to encode map e635 with expected crc 2023-09-23T23:40:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:34 smithi138 ceph-mon[193606]: osdmap e635: 8 total, 8 up, 6 in 2023-09-23T23:40:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:34 smithi138 ceph-mon[193606]: failed to encode map e635 with expected crc 2023-09-23T23:40:34.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:34 smithi138 ceph-mon[193606]: failed to encode map e635 with expected crc 2023-09-23T23:40:34.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:34 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 11405/209958 objects degraded (5.432%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:34.697 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 72 16 73001 72985 63.3199 46.6875 0.00649901 0.0157875 2023-09-23T23:40:35.404 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:40:35.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:35.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:40:35.413 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:35 smithi110 ceph-mon[212939]: pgmap v1816: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 38 MiB/s wr, 607 op/s; 8568/214050 objects degraded (4.003%); 2675/214050 objects misplaced (1.250%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T23:40:35.413 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:35 smithi110 ceph-mon[221102]: pgmap v1816: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 38 MiB/s wr, 607 op/s; 8568/214050 objects degraded (4.003%); 2675/214050 objects misplaced (1.250%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T23:40:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:35 smithi138 ceph-mon[193606]: pgmap v1816: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 38 MiB/s wr, 607 op/s; 8568/214050 objects degraded (4.003%); 2675/214050 objects misplaced (1.250%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T23:40:35.508 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T23:40:35.508 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-09-23T23:40:35.513 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:40:35.641 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:35.698 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 73 16 73594 73578 62.9599 37.0625 0.0202578 0.0158759 2023-09-23T23:40:35.883 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:40:35.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:35.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:40:36.001 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:40:36.098 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:36.698 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 74 16 74274 74258 62.6835 42.5 0.0181974 0.0159481 2023-09-23T23:40:37.084 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:37 smithi110 ceph-mon[212939]: pgmap v1817: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 50 MiB/s wr, 794 op/s; 5734/218535 objects degraded (2.624%); 5274/218535 objects misplaced (2.413%); 10 MiB/s, 164 objects/s recovering 2023-09-23T23:40:37.084 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:40:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:40:36] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.43.0" 2023-09-23T23:40:37.084 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:37 smithi110 ceph-mon[221102]: pgmap v1817: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 50 MiB/s wr, 794 op/s; 5734/218535 objects degraded (2.624%); 5274/218535 objects misplaced (2.413%); 10 MiB/s, 164 objects/s recovering 2023-09-23T23:40:37.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:37 smithi138 ceph-mon[193606]: pgmap v1817: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 50 MiB/s wr, 794 op/s; 5734/218535 objects degraded (2.624%); 5274/218535 objects misplaced (2.413%); 10 MiB/s, 164 objects/s recovering 2023-09-23T23:40:37.697 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:40:37.698 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 75 16 75029 75013 62.4769 47.1875 0.0165732 0.0159969 2023-09-23T23:40:37.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:37.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:40:37.814 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:40:37.914 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:38.023 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:40:38.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:38.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:40:38.116 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:40:38.145 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:40:38.174 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:38.222 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:40:38.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:38.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:40:38.327 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:40:38.429 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:38.573 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:40:38.627 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:40:38.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:38.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:40:38.706 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 76 16 75555 75539 62.0876 32.875 0.03133 0.0160999 2023-09-23T23:40:38.719 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:40:38.719 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:40:38.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:40:38.719 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:40:38.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.719 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:175259 12.c 12:3a3a4253:::benchmark_data_smithi110_491193_object80384:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2023-09-23T23:40:38.720 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.423074+0000", 2023-09-23T23:40:38.720 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 511.28611681500001, 2023-09-23T23:40:38.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.30677402500000001, 2023-09-23T23:40:38.720 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.720 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:38.720 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:40:38.720 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:40:38.720 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:40:38.720 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175259 2023-09-23T23:40:38.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.721 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.423074+0000", 2023-09-23T23:40:38.721 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.423074+0000", 2023-09-23T23:40:38.722 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.722 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.722 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.722 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.722 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.423073+0000", 2023-09-23T23:40:38.722 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:40:38.722 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.722 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.722 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.722 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.423079+0000", 2023-09-23T23:40:38.723 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5300000000000004e-06 2023-09-23T23:40:38.723 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.723 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.723 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.723 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.423093+0000", 2023-09-23T23:40:38.723 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4314999999999999e-05 2023-09-23T23:40:38.723 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.723 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.723 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.723 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.423100+0000", 2023-09-23T23:40:38.723 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2840000000000001e-06 2023-09-23T23:40:38.723 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.724 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.724 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.724 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433657+0000", 2023-09-23T23:40:38.724 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010556528000000001 2023-09-23T23:40:38.724 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.724 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.724 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.724 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433752+0000", 2023-09-23T23:40:38.724 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5221000000000002e-05 2023-09-23T23:40:38.724 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.724 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:40:38.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433841+0000", 2023-09-23T23:40:38.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8955999999999993e-05 2023-09-23T23:40:38.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:40:38.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.437065+0000", 2023-09-23T23:40:38.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003224219 2023-09-23T23:40:38.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.441479+0000", 2023-09-23T23:40:38.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.004414177 2023-09-23T23:40:38.726 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.729759+0000", 2023-09-23T23:40:38.726 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28828009199999999 2023-09-23T23:40:38.726 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:40:38.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.729805+0000", 2023-09-23T23:40:38.726 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5633999999999997e-05 2023-09-23T23:40:38.726 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.727 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.727 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.729848+0000", 2023-09-23T23:40:38.727 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3220999999999998e-05 2023-09-23T23:40:38.727 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.727 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.727 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.727 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.727 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.727 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71529 13.a 13:5f357adb:::benchmark_data_smithi110_541928_object71528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:40:38.727 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.570405+0000", 2023-09-23T23:40:38.727 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 408.13878591600002, 2023-09-23T23:40:38.728 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4516865019999998, 2023-09-23T23:40:38.728 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.728 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:38.728 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:40:38.728 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:40:38.728 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:40:38.728 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71529 2023-09-23T23:40:38.728 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.728 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.728 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.728 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.728 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.570405+0000", 2023-09-23T23:40:38.728 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.570405+0000", 2023-09-23T23:40:38.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.570403+0000", 2023-09-23T23:40:38.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:40:38.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.570432+0000", 2023-09-23T23:40:38.730 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9042000000000001e-05 2023-09-23T23:40:38.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.570449+0000", 2023-09-23T23:40:38.730 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6860000000000001e-05 2023-09-23T23:40:38.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.570459+0000", 2023-09-23T23:40:38.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5380000000000008e-06 2023-09-23T23:40:38.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.570523+0000", 2023-09-23T23:40:38.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3935000000000004e-05 2023-09-23T23:40:38.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.732 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:40:38.732 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.570554+0000", 2023-09-23T23:40:38.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1585000000000003e-05 2023-09-23T23:40:38.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.732 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.732 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.533184+0000", 2023-09-23T23:40:38.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96262950599999997 2023-09-23T23:40:38.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.732 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:40:38.732 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.533197+0000", 2023-09-23T23:40:38.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3514e-05 2023-09-23T23:40:38.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.890671+0000", 2023-09-23T23:40:38.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.357473919 2023-09-23T23:40:38.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.890773+0000", 2023-09-23T23:40:38.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000101548 2023-09-23T23:40:38.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:40:38.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.890880+0000", 2023-09-23T23:40:38.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107626 2023-09-23T23:40:38.734 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.734 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:40:38.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.897164+0000", 2023-09-23T23:40:38.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0062841499999999996 2023-09-23T23:40:38.734 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.734 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.911366+0000", 2023-09-23T23:40:38.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014201217 2023-09-23T23:40:38.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.018718+0000", 2023-09-23T23:40:38.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.107352738 2023-09-23T23:40:38.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:40:38.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.018792+0000", 2023-09-23T23:40:38.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3300000000000006e-05 2023-09-23T23:40:38.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.022092+0000", 2023-09-23T23:40:38.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032998910000000001 2023-09-23T23:40:38.736 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.736 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.736 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.736 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:36244 14.6 14:6c63a03b:::benchmark_data_smithi110_640334_object36243:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e623)", 2023-09-23T23:40:38.736 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.847989+0000", 2023-09-23T23:40:38.736 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.861202480000003, 2023-09-23T23:40:38.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.390174772, 2023-09-23T23:40:38.737 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.737 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:38.737 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:40:38.737 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:40:38.737 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:40:38.737 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 36244 2023-09-23T23:40:38.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.737 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.737 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.737 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.847989+0000", 2023-09-23T23:40:38.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.847989+0000", 2023-09-23T23:40:38.738 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.847987+0000", 2023-09-23T23:40:38.738 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:40:38.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.739 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848016+0000", 2023-09-23T23:40:38.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8796999999999999e-05 2023-09-23T23:40:38.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.739 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848027+0000", 2023-09-23T23:40:38.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0730000000000001e-05 2023-09-23T23:40:38.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.739 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848034+0000", 2023-09-23T23:40:38.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.627e-06 2023-09-23T23:40:38.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.740 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.740 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.740 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848085+0000", 2023-09-23T23:40:38.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0226000000000003e-05 2023-09-23T23:40:38.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.740 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.740 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.740 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848239+0000", 2023-09-23T23:40:38.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015469400000000001 2023-09-23T23:40:38.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.741 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:40:38.741 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848393+0000", 2023-09-23T23:40:38.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000153781 2023-09-23T23:40:38.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.741 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:40:38.741 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.857088+0000", 2023-09-23T23:40:38.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086951730000000005 2023-09-23T23:40:38.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.237918+0000", 2023-09-23T23:40:38.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.380829947 2023-09-23T23:40:38.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.238098+0000", 2023-09-23T23:40:38.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017939900000000001 2023-09-23T23:40:38.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:40:38.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.238134+0000", 2023-09-23T23:40:38.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6148e-05 2023-09-23T23:40:38.743 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.238163+0000", 2023-09-23T23:40:38.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9507000000000001e-05 2023-09-23T23:40:38.743 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.743 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.743 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.744 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:36248 14.b 14:d7f3c884:::benchmark_data_smithi110_640334_object36247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e623)", 2023-09-23T23:40:38.744 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.848052+0000", 2023-09-23T23:40:38.744 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.861139123000001, 2023-09-23T23:40:38.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.38857067000000001, 2023-09-23T23:40:38.744 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.744 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:38.744 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:40:38.744 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:40:38.744 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:40:38.744 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 36248 2023-09-23T23:40:38.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.745 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848052+0000", 2023-09-23T23:40:38.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848052+0000", 2023-09-23T23:40:38.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.746 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.746 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848051+0000", 2023-09-23T23:40:38.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:40:38.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.746 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.746 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848075+0000", 2023-09-23T23:40:38.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3703999999999999e-05 2023-09-23T23:40:38.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.746 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.746 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848082+0000", 2023-09-23T23:40:38.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1849999999999998e-06 2023-09-23T23:40:38.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848088+0000", 2023-09-23T23:40:38.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3689999999999998e-06 2023-09-23T23:40:38.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848203+0000", 2023-09-23T23:40:38.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000115149 2023-09-23T23:40:38.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.748 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.748 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848327+0000", 2023-09-23T23:40:38.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012472199999999999 2023-09-23T23:40:38.748 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.748 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:40:38.748 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848467+0000", 2023-09-23T23:40:38.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000139195 2023-09-23T23:40:38.748 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.748 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:40:38.748 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.856999+0000", 2023-09-23T23:40:38.749 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0085324749999999994 2023-09-23T23:40:38.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.749 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236149+0000", 2023-09-23T23:40:38.749 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37914956799999999 2023-09-23T23:40:38.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.749 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236524+0000", 2023-09-23T23:40:38.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00037518700000000002 2023-09-23T23:40:38.750 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.750 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.750 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:40:38.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236581+0000", 2023-09-23T23:40:38.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7522999999999998e-05 2023-09-23T23:40:38.750 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.750 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.751 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236623+0000", 2023-09-23T23:40:38.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1109999999999998e-05 2023-09-23T23:40:38.751 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.751 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.751 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.752 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.752 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.752 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:36250 14.b 14:dd0a45e1:::benchmark_data_smithi110_640334_object36249:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e623)", 2023-09-23T23:40:38.752 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.848101+0000", 2023-09-23T23:40:38.752 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.861089747999998, 2023-09-23T23:40:38.752 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.38862088299999997, 2023-09-23T23:40:38.752 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.753 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:38.753 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:40:38.753 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:40:38.753 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:40:38.753 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 36250 2023-09-23T23:40:38.753 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.753 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.754 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848101+0000", 2023-09-23T23:40:38.754 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.754 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.754 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848101+0000", 2023-09-23T23:40:38.755 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.755 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.755 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.755 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848101+0000", 2023-09-23T23:40:38.755 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:40:38.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.755 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.756 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.756 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848123+0000", 2023-09-23T23:40:38.756 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2585000000000001e-05 2023-09-23T23:40:38.756 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.756 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.756 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.756 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848131+0000", 2023-09-23T23:40:38.757 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7740000000000004e-06 2023-09-23T23:40:38.757 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.757 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.757 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.757 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.848137+0000", 2023-09-23T23:40:38.757 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6210000000000001e-06 2023-09-23T23:40:38.757 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.757 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.757 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.758 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.849928+0000", 2023-09-23T23:40:38.758 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017915800000000001 2023-09-23T23:40:38.758 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.758 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.758 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.758 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.850046+0000", 2023-09-23T23:40:38.758 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117786 2023-09-23T23:40:38.758 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.758 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.758 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:40:38.758 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.850167+0000", 2023-09-23T23:40:38.759 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000121321 2023-09-23T23:40:38.759 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.759 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.759 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:40:38.759 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.857011+0000", 2023-09-23T23:40:38.759 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068439750000000004 2023-09-23T23:40:38.759 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.759 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.759 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.759 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236187+0000", 2023-09-23T23:40:38.759 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.379175337 2023-09-23T23:40:38.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.760 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236657+0000", 2023-09-23T23:40:38.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047074399999999999 2023-09-23T23:40:38.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.760 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:40:38.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236695+0000", 2023-09-23T23:40:38.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7228999999999999e-05 2023-09-23T23:40:38.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.760 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236722+0000", 2023-09-23T23:40:38.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7501e-05 2023-09-23T23:40:38.761 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.761 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.761 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.761 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:36256 14.c 14:3fc29556:::benchmark_data_smithi110_640334_object36255:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T23:40:38.761 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.856035+0000", 2023-09-23T23:40:38.761 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.853155817999998, 2023-09-23T23:40:38.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.383245269, 2023-09-23T23:40:38.761 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.762 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:38.762 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:40:38.762 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:40:38.762 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:40:38.762 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 36256 2023-09-23T23:40:38.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.762 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.856035+0000", 2023-09-23T23:40:38.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.856035+0000", 2023-09-23T23:40:38.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.856033+0000", 2023-09-23T23:40:38.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:40:38.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.856059+0000", 2023-09-23T23:40:38.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5783e-05 2023-09-23T23:40:38.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.856069+0000", 2023-09-23T23:40:38.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8339999999999994e-06 2023-09-23T23:40:38.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.856078+0000", 2023-09-23T23:40:38.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0429999999999996e-06 2023-09-23T23:40:38.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.856932+0000", 2023-09-23T23:40:38.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00085346499999999997 2023-09-23T23:40:38.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.857070+0000", 2023-09-23T23:40:38.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013888799999999999 2023-09-23T23:40:38.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.766 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6", 2023-09-23T23:40:38.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.857221+0000", 2023-09-23T23:40:38.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015082600000000001 2023-09-23T23:40:38.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.766 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:40:38.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.858724+0000", 2023-09-23T23:40:38.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015025640000000001 2023-09-23T23:40:38.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.766 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.239173+0000", 2023-09-23T23:40:38.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.38044936299999998 2023-09-23T23:40:38.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.239203+0000", 2023-09-23T23:40:38.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9450000000000001e-05 2023-09-23T23:40:38.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:40:38.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.239247+0000", 2023-09-23T23:40:38.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4190999999999998e-05 2023-09-23T23:40:38.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.768 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.768 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.768 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.239280+0000", 2023-09-23T23:40:38.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3559000000000002e-05 2023-09-23T23:40:38.768 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.768 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.768 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.768 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.768 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:36259 14.e 14:779fd9c8:::benchmark_data_smithi110_640334_object36258:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T23:40:38.768 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.859633+0000", 2023-09-23T23:40:38.768 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.849558455999997, 2023-09-23T23:40:38.769 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37839893299999999, 2023-09-23T23:40:38.769 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.769 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:38.769 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:40:38.769 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:40:38.769 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:40:38.769 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 36259 2023-09-23T23:40:38.769 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.769 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.769 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.769 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.769 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859633+0000", 2023-09-23T23:40:38.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.770 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.770 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.770 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859633+0000", 2023-09-23T23:40:38.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.770 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.770 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.770 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859631+0000", 2023-09-23T23:40:38.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:40:38.770 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859673+0000", 2023-09-23T23:40:38.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.214e-05 2023-09-23T23:40:38.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859687+0000", 2023-09-23T23:40:38.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3696000000000001e-05 2023-09-23T23:40:38.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859696+0000", 2023-09-23T23:40:38.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6239999999999997e-06 2023-09-23T23:40:38.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.772 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.772 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859953+0000", 2023-09-23T23:40:38.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025695900000000002 2023-09-23T23:40:38.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.772 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.772 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.860074+0000", 2023-09-23T23:40:38.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0001205 2023-09-23T23:40:38.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-09-23T23:40:38.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.860204+0000", 2023-09-23T23:40:38.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000130741 2023-09-23T23:40:38.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:40:38.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863045+0000", 2023-09-23T23:40:38.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002840221 2023-09-23T23:40:38.774 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.237876+0000", 2023-09-23T23:40:38.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37483171300000001 2023-09-23T23:40:38.774 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.237947+0000", 2023-09-23T23:40:38.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0698000000000006e-05 2023-09-23T23:40:38.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:40:38.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.237996+0000", 2023-09-23T23:40:38.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8959999999999999e-05 2023-09-23T23:40:38.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.238031+0000", 2023-09-23T23:40:38.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5379000000000001e-05 2023-09-23T23:40:38.776 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.776 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.776 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.777 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:36261 14.8 14:1efac221:::benchmark_data_smithi110_640334_object36260:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T23:40:38.777 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.859718+0000", 2023-09-23T23:40:38.777 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.849473002000003, 2023-09-23T23:40:38.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37840712500000001, 2023-09-23T23:40:38.777 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.777 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:38.778 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:40:38.778 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:40:38.778 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:40:38.778 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 36261 2023-09-23T23:40:38.778 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.778 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.778 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.778 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859718+0000", 2023-09-23T23:40:38.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859718+0000", 2023-09-23T23:40:38.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859717+0000", 2023-09-23T23:40:38.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:40:38.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.780 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859759+0000", 2023-09-23T23:40:38.780 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1767000000000002e-05 2023-09-23T23:40:38.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.780 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859767+0000", 2023-09-23T23:40:38.780 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1920000000000005e-06 2023-09-23T23:40:38.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.780 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859773+0000", 2023-09-23T23:40:38.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1009999999999997e-06 2023-09-23T23:40:38.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859827+0000", 2023-09-23T23:40:38.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3368999999999997e-05 2023-09-23T23:40:38.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859970+0000", 2023-09-23T23:40:38.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000143354 2023-09-23T23:40:38.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:40:38.782 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.860107+0000", 2023-09-23T23:40:38.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013744199999999999 2023-09-23T23:40:38.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:40:38.782 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862181+0000", 2023-09-23T23:40:38.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020735100000000002 2023-09-23T23:40:38.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236281+0000", 2023-09-23T23:40:38.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37410006299999998 2023-09-23T23:40:38.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.238051+0000", 2023-09-23T23:40:38.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017699510000000001 2023-09-23T23:40:38.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:40:38.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.238092+0000", 2023-09-23T23:40:38.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1161000000000001e-05 2023-09-23T23:40:38.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.238125+0000", 2023-09-23T23:40:38.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2999e-05 2023-09-23T23:40:38.785 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.785 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.785 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.786 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:36262 14.0 14:05a9d568:::benchmark_data_smithi110_640334_object36261:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T23:40:38.786 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.859789+0000", 2023-09-23T23:40:38.786 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.849401993000001, 2023-09-23T23:40:38.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37828298900000001, 2023-09-23T23:40:38.786 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.786 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:38.786 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:40:38.786 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:40:38.786 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:40:38.787 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 36262 2023-09-23T23:40:38.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.787 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859789+0000", 2023-09-23T23:40:38.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859789+0000", 2023-09-23T23:40:38.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859788+0000", 2023-09-23T23:40:38.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:40:38.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859815+0000", 2023-09-23T23:40:38.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6287000000000001e-05 2023-09-23T23:40:38.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859823+0000", 2023-09-23T23:40:38.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5539999999999998e-06 2023-09-23T23:40:38.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859829+0000", 2023-09-23T23:40:38.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6729999999999997e-06 2023-09-23T23:40:38.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859866+0000", 2023-09-23T23:40:38.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7048999999999997e-05 2023-09-23T23:40:38.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859992+0000", 2023-09-23T23:40:38.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012596100000000001 2023-09-23T23:40:38.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.791 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:40:38.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.860126+0000", 2023-09-23T23:40:38.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013418999999999999 2023-09-23T23:40:38.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:40:38.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862329+0000", 2023-09-23T23:40:38.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002202702 2023-09-23T23:40:38.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236246+0000", 2023-09-23T23:40:38.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37391674499999999 2023-09-23T23:40:38.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.237997+0000", 2023-09-23T23:40:38.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001750894 2023-09-23T23:40:38.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:40:38.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.238037+0000", 2023-09-23T23:40:38.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0790000000000001e-05 2023-09-23T23:40:38.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.238072+0000", 2023-09-23T23:40:38.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4690999999999997e-05 2023-09-23T23:40:38.794 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.795 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.795 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.795 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:36263 14.6 14:695974ef:::benchmark_data_smithi110_640334_object36262:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T23:40:38.795 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.859838+0000", 2023-09-23T23:40:38.795 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.849352555999999, 2023-09-23T23:40:38.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.379995584, 2023-09-23T23:40:38.795 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.795 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:38.795 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:40:38.795 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:40:38.795 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:40:38.796 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 36263 2023-09-23T23:40:38.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.796 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859838+0000", 2023-09-23T23:40:38.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859838+0000", 2023-09-23T23:40:38.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859838+0000", 2023-09-23T23:40:38.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:40:38.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859858+0000", 2023-09-23T23:40:38.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0517e-05 2023-09-23T23:40:38.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859866+0000", 2023-09-23T23:40:38.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8830000000000006e-06 2023-09-23T23:40:38.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859872+0000", 2023-09-23T23:40:38.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2780000000000001e-06 2023-09-23T23:40:38.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.861524+0000", 2023-09-23T23:40:38.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001652504 2023-09-23T23:40:38.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.861653+0000", 2023-09-23T23:40:38.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012924400000000001 2023-09-23T23:40:38.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:40:38.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.861774+0000", 2023-09-23T23:40:38.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012105 2023-09-23T23:40:38.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:40:38.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.237884+0000", 2023-09-23T23:40:38.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37610977899999998 2023-09-23T23:40:38.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.238067+0000", 2023-09-23T23:40:38.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000182399 2023-09-23T23:40:38.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.239750+0000", 2023-09-23T23:40:38.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001683969 2023-09-23T23:40:38.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.801 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:40:38.801 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.239799+0000", 2023-09-23T23:40:38.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8643999999999997e-05 2023-09-23T23:40:38.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.802 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.802 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.239834+0000", 2023-09-23T23:40:38.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4881e-05 2023-09-23T23:40:38.802 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.802 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.802 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.802 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:36264 14.6 14:6cba1907:::benchmark_data_smithi110_640334_object36263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T23:40:38.802 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.859877+0000", 2023-09-23T23:40:38.802 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.849313590000001, 2023-09-23T23:40:38.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.38178853000000001, 2023-09-23T23:40:38.803 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.803 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:38.803 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:40:38.803 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:40:38.803 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:40:38.804 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 36264 2023-09-23T23:40:38.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.804 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859877+0000", 2023-09-23T23:40:38.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859877+0000", 2023-09-23T23:40:38.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859877+0000", 2023-09-23T23:40:38.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:40:38.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859899+0000", 2023-09-23T23:40:38.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2171000000000002e-05 2023-09-23T23:40:38.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859906+0000", 2023-09-23T23:40:38.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1380000000000001e-06 2023-09-23T23:40:38.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.859912+0000", 2023-09-23T23:40:38.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5629999999999999e-06 2023-09-23T23:40:38.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863102+0000", 2023-09-23T23:40:38.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031902039999999999 2023-09-23T23:40:38.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236319+0000", 2023-09-23T23:40:38.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37321713000000001 2023-09-23T23:40:38.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:40:38.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236478+0000", 2023-09-23T23:40:38.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000158715 2023-09-23T23:40:38.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.240341+0000", 2023-09-23T23:40:38.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038635829999999999 2023-09-23T23:40:38.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.240372+0000", 2023-09-23T23:40:38.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0190999999999999e-05 2023-09-23T23:40:38.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:40:38.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.241587+0000", 2023-09-23T23:40:38.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012151589999999999 2023-09-23T23:40:38.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:40:38.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.241630+0000", 2023-09-23T23:40:38.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2892000000000002e-05 2023-09-23T23:40:38.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.810 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.810 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.241666+0000", 2023-09-23T23:40:38.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6306000000000001e-05 2023-09-23T23:40:38.810 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.810 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.810 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.810 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.810 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:36266 14.c 14:362e3d67:::benchmark_data_smithi110_640334_object36265:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T23:40:38.810 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.862680+0000", 2023-09-23T23:40:38.810 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.846510608000003, 2023-09-23T23:40:38.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.379989154, 2023-09-23T23:40:38.811 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.811 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:38.811 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:40:38.811 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:40:38.811 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:40:38.811 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 36266 2023-09-23T23:40:38.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.811 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862680+0000", 2023-09-23T23:40:38.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862680+0000", 2023-09-23T23:40:38.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862678+0000", 2023-09-23T23:40:38.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-09-23T23:40:38.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862719+0000", 2023-09-23T23:40:38.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1066999999999998e-05 2023-09-23T23:40:38.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862732+0000", 2023-09-23T23:40:38.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2490999999999999e-05 2023-09-23T23:40:38.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862741+0000", 2023-09-23T23:40:38.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1309999999999995e-06 2023-09-23T23:40:38.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862791+0000", 2023-09-23T23:40:38.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0512e-05 2023-09-23T23:40:38.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862967+0000", 2023-09-23T23:40:38.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017527299999999999 2023-09-23T23:40:38.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6", 2023-09-23T23:40:38.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863119+0000", 2023-09-23T23:40:38.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015238099999999999 2023-09-23T23:40:38.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.239421+0000", 2023-09-23T23:40:38.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.376301999 2023-09-23T23:40:38.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:40:38.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.239434+0000", 2023-09-23T23:40:38.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2882e-05 2023-09-23T23:40:38.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.242565+0000", 2023-09-23T23:40:38.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031315959999999999 2023-09-23T23:40:38.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:40:38.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.242636+0000", 2023-09-23T23:40:38.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0401999999999994e-05 2023-09-23T23:40:38.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.242670+0000", 2023-09-23T23:40:38.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3643000000000003e-05 2023-09-23T23:40:38.817 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.817 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.817 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.817 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:36267 14.c 14:3f8b12fd:::benchmark_data_smithi110_640334_object36266:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T23:40:38.817 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.862760+0000", 2023-09-23T23:40:38.817 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.846430726000001, 2023-09-23T23:40:38.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.38002651999999998, 2023-09-23T23:40:38.818 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.818 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:38.818 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:40:38.818 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:40:38.818 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:40:38.818 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 36267 2023-09-23T23:40:38.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.818 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862760+0000", 2023-09-23T23:40:38.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.819 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.819 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862760+0000", 2023-09-23T23:40:38.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.819 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.819 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862760+0000", 2023-09-23T23:40:38.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:40:38.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862782+0000", 2023-09-23T23:40:38.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2192e-05 2023-09-23T23:40:38.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862791+0000", 2023-09-23T23:40:38.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6299999999999993e-06 2023-09-23T23:40:38.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862798+0000", 2023-09-23T23:40:38.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4110000000000001e-06 2023-09-23T23:40:38.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.237624+0000", 2023-09-23T23:40:38.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37482587299999998 2023-09-23T23:40:38.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.237738+0000", 2023-09-23T23:40:38.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000114577 2023-09-23T23:40:38.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6", 2023-09-23T23:40:38.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.237875+0000", 2023-09-23T23:40:38.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013675300000000001 2023-09-23T23:40:38.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:40:38.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.239546+0000", 2023-09-23T23:40:38.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00167121 2023-09-23T23:40:38.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.239906+0000", 2023-09-23T23:40:38.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035955999999999998 2023-09-23T23:40:38.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.242717+0000", 2023-09-23T23:40:38.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028112459999999999 2023-09-23T23:40:38.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:40:38.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.242756+0000", 2023-09-23T23:40:38.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8525999999999997e-05 2023-09-23T23:40:38.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.242787+0000", 2023-09-23T23:40:38.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1149999999999998e-05 2023-09-23T23:40:38.824 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.824 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.824 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.825 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.47186.0:36256 14.c e624/617 ondisk, result = 0)", 2023-09-23T23:40:38.825 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.862797+0000", 2023-09-23T23:40:38.825 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.846393851999999, 2023-09-23T23:40:38.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37637868800000002, 2023-09-23T23:40:38.825 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.825 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:40:38.825 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862797+0000", 2023-09-23T23:40:38.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862797+0000", 2023-09-23T23:40:38.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862796+0000", 2023-09-23T23:40:38.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:40:38.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862802+0000", 2023-09-23T23:40:38.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0639999999999999e-06 2023-09-23T23:40:38.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862813+0000", 2023-09-23T23:40:38.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0764000000000001e-05 2023-09-23T23:40:38.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.862821+0000", 2023-09-23T23:40:38.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2209999999999997e-06 2023-09-23T23:40:38.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.239161+0000", 2023-09-23T23:40:38.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37633977299999999 2023-09-23T23:40:38.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.239168+0000", 2023-09-23T23:40:38.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7530000000000004e-06 2023-09-23T23:40:38.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.239176+0000", 2023-09-23T23:40:38.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1750000000000005e-06 2023-09-23T23:40:38.829 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.829 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.829 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.829 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.47186.0:36244 14.6 e624/617 ondisk, result = 0)", 2023-09-23T23:40:38.829 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.863043+0000", 2023-09-23T23:40:38.829 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.846147608999999, 2023-09-23T23:40:38.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.374877126, 2023-09-23T23:40:38.830 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.830 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:40:38.830 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863043+0000", 2023-09-23T23:40:38.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863043+0000", 2023-09-23T23:40:38.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863043+0000", 2023-09-23T23:40:38.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:40:38.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863048+0000", 2023-09-23T23:40:38.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.412e-06 2023-09-23T23:40:38.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863059+0000", 2023-09-23T23:40:38.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0521e-05 2023-09-23T23:40:38.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863066+0000", 2023-09-23T23:40:38.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1770000000000001e-06 2023-09-23T23:40:38.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.237907+0000", 2023-09-23T23:40:38.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37484078599999998 2023-09-23T23:40:38.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.237913+0000", 2023-09-23T23:40:38.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4980000000000003e-06 2023-09-23T23:40:38.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.237921+0000", 2023-09-23T23:40:38.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3599999999999998e-06 2023-09-23T23:40:38.833 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.834 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.834 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.834 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.47186.0:36248 14.b e624/617 ondisk, result = 0)", 2023-09-23T23:40:38.834 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.863084+0000", 2023-09-23T23:40:38.834 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.846106986000002, 2023-09-23T23:40:38.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37306815300000001, 2023-09-23T23:40:38.834 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.834 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:40:38.834 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863084+0000", 2023-09-23T23:40:38.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863084+0000", 2023-09-23T23:40:38.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863084+0000", 2023-09-23T23:40:38.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:40:38.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863086+0000", 2023-09-23T23:40:38.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6819999999999999e-06 2023-09-23T23:40:38.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863093+0000", 2023-09-23T23:40:38.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4139999999999999e-06 2023-09-23T23:40:38.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863098+0000", 2023-09-23T23:40:38.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9590000000000003e-06 2023-09-23T23:40:38.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236131+0000", 2023-09-23T23:40:38.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37303322799999999 2023-09-23T23:40:38.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236141+0000", 2023-09-23T23:40:38.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0424e-05 2023-09-23T23:40:38.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236152+0000", 2023-09-23T23:40:38.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.075e-05 2023-09-23T23:40:38.841 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.841 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.841 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.841 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.47186.0:36250 14.b e624/617 ondisk, result = 0)", 2023-09-23T23:40:38.842 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.863109+0000", 2023-09-23T23:40:38.842 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.846082340000002, 2023-09-23T23:40:38.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37308003200000001, 2023-09-23T23:40:38.842 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.842 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:40:38.842 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863109+0000", 2023-09-23T23:40:38.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863109+0000", 2023-09-23T23:40:38.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.844 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863108+0000", 2023-09-23T23:40:38.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:40:38.844 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863110+0000", 2023-09-23T23:40:38.845 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9309999999999998e-06 2023-09-23T23:40:38.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.845 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863116+0000", 2023-09-23T23:40:38.845 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.366e-06 2023-09-23T23:40:38.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.845 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.863120+0000", 2023-09-23T23:40:38.845 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4989999999999996e-06 2023-09-23T23:40:38.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.846 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236175+0000", 2023-09-23T23:40:38.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37305437899999999 2023-09-23T23:40:38.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.846 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236181+0000", 2023-09-23T23:40:38.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6250000000000001e-06 2023-09-23T23:40:38.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.846 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.847 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236189+0000", 2023-09-23T23:40:38.847 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4769999999999997e-06 2023-09-23T23:40:38.847 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.847 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.847 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.847 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.47186.0:36259 14.e e624/617 ondisk, result = 0)", 2023-09-23T23:40:38.847 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.864399+0000", 2023-09-23T23:40:38.847 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.844792216999998, 2023-09-23T23:40:38.847 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37348628299999997, 2023-09-23T23:40:38.847 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.848 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:40:38.848 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.848 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.864399+0000", 2023-09-23T23:40:38.848 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.848 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.848 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.864399+0000", 2023-09-23T23:40:38.848 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.849 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.849 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.849 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.864398+0000", 2023-09-23T23:40:38.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:40:38.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.849 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.849 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.849 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.864402+0000", 2023-09-23T23:40:38.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1919999999999998e-06 2023-09-23T23:40:38.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236094+0000", 2023-09-23T23:40:38.850 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37169160099999998 2023-09-23T23:40:38.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236109+0000", 2023-09-23T23:40:38.850 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5495999999999999e-05 2023-09-23T23:40:38.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.851 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.851 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.237864+0000", 2023-09-23T23:40:38.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017545950000000001 2023-09-23T23:40:38.851 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.851 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.851 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.237871+0000", 2023-09-23T23:40:38.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0299999999999996e-06 2023-09-23T23:40:38.851 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.851 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.851 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.852 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.237885+0000", 2023-09-23T23:40:38.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3937000000000001e-05 2023-09-23T23:40:38.852 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.852 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.852 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.852 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.852 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.852 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.47186.0:36262 14.0 e624/617 ondisk, result = 0)", 2023-09-23T23:40:38.852 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.864487+0000", 2023-09-23T23:40:38.852 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.844704096000001, 2023-09-23T23:40:38.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37176142099999998, 2023-09-23T23:40:38.852 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.853 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:40:38.853 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.853 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.853 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.864487+0000", 2023-09-23T23:40:38.853 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.853 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.853 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.853 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.864487+0000", 2023-09-23T23:40:38.853 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.853 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.864486+0000", 2023-09-23T23:40:38.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:40:38.854 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.854 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.864490+0000", 2023-09-23T23:40:38.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7280000000000001e-06 2023-09-23T23:40:38.854 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.854 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.855 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236109+0000", 2023-09-23T23:40:38.855 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37161849299999999 2023-09-23T23:40:38.855 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.855 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.855 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.855 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236120+0000", 2023-09-23T23:40:38.855 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1344e-05 2023-09-23T23:40:38.855 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.855 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.855 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.855 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236232+0000", 2023-09-23T23:40:38.855 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000112409 2023-09-23T23:40:38.856 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.856 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.856 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.856 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236240+0000", 2023-09-23T23:40:38.856 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4710000000000001e-06 2023-09-23T23:40:38.856 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.856 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.856 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.856 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236248+0000", 2023-09-23T23:40:38.856 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5250000000000005e-06 2023-09-23T23:40:38.856 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.856 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.856 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.857 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.857 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.857 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:36268 14.e 14:7c8bdcb2:::benchmark_data_smithi110_640334_object36267:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T23:40:38.857 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.864498+0000", 2023-09-23T23:40:38.857 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.844692946000002, 2023-09-23T23:40:38.857 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.38032344699999998, 2023-09-23T23:40:38.857 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:40:38.857 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:40:38.857 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:40:38.857 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:40:38.857 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:40:38.857 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 36268 2023-09-23T23:40:38.857 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.858 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:40:38.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.858 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:40:38.858 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.864498+0000", 2023-09-23T23:40:38.858 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.858 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.858 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:40:38.858 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.864498+0000", 2023-09-23T23:40:38.858 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:40:38.858 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.859 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:40:38.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.864497+0000", 2023-09-23T23:40:38.859 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:40:38.859 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.859 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:40:38.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.864522+0000", 2023-09-23T23:40:38.859 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4977000000000002e-05 2023-09-23T23:40:38.859 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:40:38.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236095+0000", 2023-09-23T23:40:38.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.37157304400000002 2023-09-23T23:40:38.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:40:38.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.236109+0000", 2023-09-23T23:40:38.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4419e-05 2023-09-23T23:40:38.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:40:38.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.238197+0000", 2023-09-23T23:40:38.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020881060000000002 2023-09-23T23:40:38.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:40:38.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.238326+0000", 2023-09-23T23:40:38.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012886199999999999 2023-09-23T23:40:38.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-09-23T23:40:38.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.238461+0000", 2023-09-23T23:40:38.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013457700000000001 2023-09-23T23:40:38.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:40:38.863 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.241679+0000", 2023-09-23T23:40:38.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003218491 2023-09-23T23:40:38.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.864 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.243624+0000", 2023-09-23T23:40:38.864 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001944376 2023-09-23T23:40:38.864 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.864 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.864 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:40:38.865 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.244735+0000", 2023-09-23T23:40:38.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011111090000000001 2023-09-23T23:40:38.865 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:40:38.865 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.244784+0000", 2023-09-23T23:40:38.866 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9254e-05 2023-09-23T23:40:38.866 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:40:38.866 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:40:38.866 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:40:38.866 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.244821+0000", 2023-09-23T23:40:38.866 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7481000000000002e-05 2023-09-23T23:40:38.867 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.867 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.867 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.867 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:40:38.867 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:40:38.867 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:40:38.868 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:40:38.895 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:38.941 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:40:38.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:38.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:40:39.048 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:40:39.080 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:39.117 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:40:39.130 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:40:39.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:39.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:40:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:39 smithi138 ceph-mon[193606]: pgmap v1818: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 34 MiB/s wr, 549 op/s; 5734/218535 objects degraded (2.624%); 5274/218535 objects misplaced (2.413%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T23:40:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:39 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3445274041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:40:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:39 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:40:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:39 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 5734/218535 objects degraded (2.624%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:39.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:39 smithi110 ceph-mon[212939]: pgmap v1818: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 34 MiB/s wr, 549 op/s; 5734/218535 objects degraded (2.624%); 5274/218535 objects misplaced (2.413%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T23:40:39.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:39 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3445274041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:40:39.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:39 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:40:39.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:39 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 5734/218535 objects degraded (2.624%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:39.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:39 smithi110 ceph-mon[221102]: pgmap v1818: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 34 MiB/s wr, 549 op/s; 5734/218535 objects degraded (2.624%); 5274/218535 objects misplaced (2.413%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T23:40:39.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:39 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3445274041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:40:39.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:39 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:40:39.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:39 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 5734/218535 objects degraded (2.624%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:39.240 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:40:39.287 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:39.701 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 77 16 76100 76084 61.7221 34.0625 0.0169275 0.0161958 2023-09-23T23:40:40.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:40 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:40:40.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:40 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:40:40.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:40 smithi138 ceph-mon[193606]: osdmap e636: 8 total, 8 up, 6 in 2023-09-23T23:40:40.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:40 smithi138 ceph-mon[193606]: failed to encode map e636 with expected crc 2023-09-23T23:40:40.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:40 smithi138 ceph-mon[193606]: failed to encode map e636 with expected crc 2023-09-23T23:40:40.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:40 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:40:40.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:40 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:40:40.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:40 smithi110 ceph-mon[212939]: osdmap e636: 8 total, 8 up, 6 in 2023-09-23T23:40:40.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:40 smithi110 ceph-mon[212939]: failed to encode map e636 with expected crc 2023-09-23T23:40:40.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:40 smithi110 ceph-mon[212939]: failed to encode map e636 with expected crc 2023-09-23T23:40:40.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:40 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:40:40.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:40 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:40:40.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:40 smithi110 ceph-mon[221102]: osdmap e636: 8 total, 8 up, 6 in 2023-09-23T23:40:40.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:40 smithi110 ceph-mon[221102]: failed to encode map e636 with expected crc 2023-09-23T23:40:40.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:40 smithi110 ceph-mon[221102]: failed to encode map e636 with expected crc 2023-09-23T23:40:40.701 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 78 16 76762 76746 61.4613 41.375 0.0177854 0.0162643 2023-09-23T23:40:41.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:41 smithi138 ceph-mon[193606]: pgmap v1820: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 37 MiB/s wr, 584 op/s; 5734/223923 objects degraded (2.561%); 5016/223923 objects misplaced (2.240%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T23:40:41.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:41 smithi110 ceph-mon[221102]: pgmap v1820: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 37 MiB/s wr, 584 op/s; 5734/223923 objects degraded (2.561%); 5016/223923 objects misplaced (2.240%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T23:40:41.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:41 smithi110 ceph-mon[212939]: pgmap v1820: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 37 MiB/s wr, 584 op/s; 5734/223923 objects degraded (2.561%); 5016/223923 objects misplaced (2.240%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T23:40:41.701 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 79 16 77481 77465 61.2522 44.9375 0.023135 0.0163198 2023-09-23T23:40:42.219 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:40:42.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:42.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:40:42.341 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:40:42.370 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:42.417 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:40:42.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:42.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:40:42.523 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:40:42.564 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:42.705 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:2023-09-23T23:40:42.704818+0000 min lat: 0.00116231 max lat: 2.39686 avg lat: 0.016375 2023-09-23T23:40:42.706 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:40:42.706 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 80 16 78200 78184 61.0446 44.9375 0.0189638 0.016375 2023-09-23T23:40:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:43 smithi138 ceph-mon[193606]: pgmap v1821: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.6 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 42 MiB/s wr, 676 op/s; 5734/228165 objects degraded (2.513%); 4765/228165 objects misplaced (2.088%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T23:40:43.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:40:43.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:43 smithi110 ceph-mon[212939]: pgmap v1821: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.6 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 42 MiB/s wr, 676 op/s; 5734/228165 objects degraded (2.513%); 4765/228165 objects misplaced (2.088%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T23:40:43.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:40:43.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:43 smithi110 ceph-mon[221102]: pgmap v1821: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.6 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 42 MiB/s wr, 676 op/s; 5734/228165 objects degraded (2.513%); 4765/228165 objects misplaced (2.088%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T23:40:43.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:40:43.706 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 81 16 78797 78781 60.7517 37.3125 0.0283573 0.0164532 2023-09-23T23:40:44.144 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:44 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 5734/228165 objects degraded (2.513%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:44.145 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:44 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 5734/228165 objects degraded (2.513%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:44.375 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:40:44.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:44.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:40:44.437 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:40:44.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:44 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 5734/228165 objects degraded (2.513%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:44.493 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:40:44.522 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:44.576 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:40:44.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:44.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:40:44.681 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:40:44.706 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 82 16 79352 79336 60.434 34.6875 0.0254569 0.0165393 2023-09-23T23:40:44.772 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:44.886 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:40:44.924 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:40:44.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:44.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:40:45.034 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:40:45.063 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:45.169 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:40:45.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:45.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:40:45.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:45 smithi110 ceph-mon[212939]: pgmap v1822: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 39 MiB/s wr, 631 op/s; 5734/233016 objects degraded (2.461%); 4428/233016 objects misplaced (1.900%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T23:40:45.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:45 smithi110 ceph-mon[221102]: pgmap v1822: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 39 MiB/s wr, 631 op/s; 5734/233016 objects degraded (2.461%); 4428/233016 objects misplaced (1.900%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T23:40:45.263 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:40:45.276 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:40:45.317 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:40:45.378 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:45.382 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:45 smithi138 ceph-mon[193606]: pgmap v1822: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 39 MiB/s wr, 631 op/s; 5734/233016 objects degraded (2.461%); 4428/233016 objects misplaced (1.900%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T23:40:45.395 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:40:45.453 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:40:45.489 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:40:45.545 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:40:45.556 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:40:45.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:45.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:40:45.589 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:40:45.639 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:40:45.669 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:40:45.672 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:40:45.672 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:40:45.673 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:40:45.673 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:40:45.673 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:40:45.673 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:40:45.673 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:40:45.673 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:40:45.712 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 83 16 79939 79923 60.1433 36.6875 0.0214975 0.0166191 2023-09-23T23:40:45.773 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:45.960 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:40:45.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:45.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:40:46.020 INFO:teuthology.orchestra.run.smithi110.stdout:moved 0 / 357 (0%) 2023-09-23T23:40:46.020 INFO:teuthology.orchestra.run.smithi110.stdout:avg 59.5 2023-09-23T23:40:46.020 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 6.44851 -> 6.44851 (expected baseline 7.04154) 2023-09-23T23:40:46.020 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.2 with 47 -> 47 pgs (0.789916 -> 0.789916 * mean) 2023-09-23T23:40:46.021 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.0 with 68 -> 68 pgs (1.14286 -> 1.14286 * mean) 2023-09-23T23:40:46.021 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:40:46.021 INFO:teuthology.orchestra.run.smithi110.stdout:oload 1000 2023-09-23T23:40:46.021 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 3 2023-09-23T23:40:46.021 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:40:46.021 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0353 2023-09-23T23:40:46.021 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.3530 2023-09-23T23:40:46.031 INFO:teuthology.orchestra.run.smithi110.stderr:no change 2023-09-23T23:40:46.073 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:40:46.103 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:46.148 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:40:46.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:46.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:40:46.252 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:40:46.322 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:46 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/473174490' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:40:46.459 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:46 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/473174490' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:40:46.460 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:46 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/473174490' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:40:46.697 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:40:46.712 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 84 16 80523 80507 59.862 36.5 0.0199703 0.0166993 2023-09-23T23:40:47.162 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:47 smithi110 ceph-mon[212939]: from='client.47255 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:40:47.162 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:47 smithi110 ceph-mon[212939]: pgmap v1823: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 40 MiB/s wr, 641 op/s; 5734/237804 objects degraded (2.411%); 4119/237804 objects misplaced (1.732%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:40:47.163 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:40:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:40:46] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.43.0" 2023-09-23T23:40:47.209 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:40:47.209 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:40:47.235 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:47 smithi138 ceph-mon[193606]: from='client.47255 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:40:47.235 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:47 smithi138 ceph-mon[193606]: pgmap v1823: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 40 MiB/s wr, 641 op/s; 5734/237804 objects degraded (2.411%); 4119/237804 objects misplaced (1.732%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:40:47.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:47 smithi110 ceph-mon[221102]: from='client.47255 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:40:47.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:47 smithi110 ceph-mon[221102]: pgmap v1823: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 40 MiB/s wr, 641 op/s; 5734/237804 objects degraded (2.411%); 4119/237804 objects misplaced (1.732%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:40:47.713 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 85 16 81043 81027 59.5402 32.5 0.0282614 0.0167903 2023-09-23T23:40:47.976 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T23:40:48.147 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:40:48.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:48.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:40:48.252 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:40:48.290 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:48.411 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:40:48.450 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:40:48.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:48.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:40:48.558 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:40:48.586 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:48.633 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:40:48.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:48.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:40:48.719 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 86 16 81625 81609 59.2663 36.375 0.0156745 0.0168627 2023-09-23T23:40:48.737 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:40:48.870 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:48.950 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T23:40:48.950 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T23:40:49.026 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:40:49.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:49.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:40:49.145 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:40:49.153 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:40:49.212 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:49.334 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:40:49.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:49.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:40:49.442 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:40:49.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:49 smithi138 ceph-mon[193606]: pgmap v1824: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 40 MiB/s wr, 641 op/s; 5734/237804 objects degraded (2.411%); 4119/237804 objects misplaced (1.732%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:40:49.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:49 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:40:49.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:49 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 5734/237804 objects degraded (2.411%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:49.460 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:49 smithi110 ceph-mon[221102]: pgmap v1824: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 40 MiB/s wr, 641 op/s; 5734/237804 objects degraded (2.411%); 4119/237804 objects misplaced (1.732%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:40:49.460 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:49 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:40:49.460 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:49 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 5734/237804 objects degraded (2.411%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:49.461 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:49 smithi110 ceph-mon[212939]: pgmap v1824: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 40 MiB/s wr, 641 op/s; 5734/237804 objects degraded (2.411%); 4119/237804 objects misplaced (1.732%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:40:49.462 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:49 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:40:49.462 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:49 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 5734/237804 objects degraded (2.411%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:49.541 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:49.654 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:40:49.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:49.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:40:49.719 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 87 16 82129 82113 58.9473 31.5 0.0294348 0.0169575 2023-09-23T23:40:49.763 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:40:49.840 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:50.161 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:40:50.163 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:40:50.164 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:40:50.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:50.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:40:50.269 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:40:50.380 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:50.524 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:40:50.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:50.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:40:50.627 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:40:50.658 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:50.722 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 88 16 82700 82684 58.6815 35.6875 0.0208506 0.0170363 2023-09-23T23:40:50.885 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:40:50.967 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:40:50.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:50.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:40:51.071 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:40:51.106 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T23:40:51.153 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:51 smithi138 ceph-mon[193606]: pgmap v1825: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 36 MiB/s wr, 583 op/s; 5734/242532 objects degraded (2.364%); 3841/242532 objects misplaced (1.584%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T23:40:51.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:51 smithi110 ceph-mon[221102]: pgmap v1825: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 36 MiB/s wr, 583 op/s; 5734/242532 objects degraded (2.364%); 3841/242532 objects misplaced (1.584%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T23:40:51.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:51 smithi110 ceph-mon[212939]: pgmap v1825: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 36 MiB/s wr, 583 op/s; 5734/242532 objects degraded (2.364%); 3841/242532 objects misplaced (1.584%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T23:40:51.722 INFO:tasks.radosbench.radosbench.0.smithi110.stdout: 89 16 83337 83321 58.4696 39.8125 0.0213997 0.0170972 2023-09-23T23:40:51.729 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:40:51.858 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:40:51.873 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:40:51.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:51.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:40:51.989 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:40:52.025 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:52.185 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:40:52.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:52.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:40:52.295 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:40:52.331 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:52.493 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:40:52.722 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total time run: 90.0142 2023-09-23T23:40:52.722 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Total writes made: 83876 2023-09-23T23:40:52.722 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Write size: 65536 2023-09-23T23:40:52.722 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Object size: 65536 2023-09-23T23:40:52.722 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Bandwidth (MB/sec): 58.2381 2023-09-23T23:40:52.723 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Bandwidth: 32.8331 2023-09-23T23:40:52.723 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max bandwidth (MB/sec): 182.125 2023-09-23T23:40:52.723 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min bandwidth (MB/sec): 0 2023-09-23T23:40:52.723 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average IOPS: 931 2023-09-23T23:40:52.723 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev IOPS: 525.338 2023-09-23T23:40:52.723 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max IOPS: 2914 2023-09-23T23:40:52.723 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min IOPS: 0 2023-09-23T23:40:52.723 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Average Latency(s): 0.0171664 2023-09-23T23:40:52.723 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Stddev Latency(s): 0.0377322 2023-09-23T23:40:52.723 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Max latency(s): 2.39686 2023-09-23T23:40:52.724 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Min latency(s): 0.00116231 2023-09-23T23:40:52.731 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T23:40:53.079 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:40:53.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:53.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:40:53.185 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:40:53.235 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:40:53.286 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:53.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:53 smithi138 ceph-mon[193606]: pgmap v1826: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.0 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 39 MiB/s wr, 618 op/s; 5734/246216 objects degraded (2.329%); 3550/246216 objects misplaced (1.442%); 7.6 MiB/s, 122 objects/s recovering 2023-09-23T23:40:53.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:53 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1016845254' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:40:53.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:53 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:40:53.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:53 smithi110 ceph-mon[212939]: pgmap v1826: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.0 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 39 MiB/s wr, 618 op/s; 5734/246216 objects degraded (2.329%); 3550/246216 objects misplaced (1.442%); 7.6 MiB/s, 122 objects/s recovering 2023-09-23T23:40:53.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:53 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1016845254' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:40:53.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:53 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:40:53.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:53 smithi110 ceph-mon[221102]: pgmap v1826: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.0 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 39 MiB/s wr, 618 op/s; 5734/246216 objects degraded (2.329%); 3550/246216 objects misplaced (1.442%); 7.6 MiB/s, 122 objects/s recovering 2023-09-23T23:40:53.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:53 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1016845254' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:40:53.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:53 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:40:53.733 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:40:53.733 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:40:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:54 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:40:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:54 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:40:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:54 smithi138 ceph-mon[193606]: osdmap e637: 8 total, 8 up, 6 in 2023-09-23T23:40:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:54 smithi138 ceph-mon[193606]: failed to encode map e637 with expected crc 2023-09-23T23:40:54.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:54 smithi138 ceph-mon[193606]: failed to encode map e637 with expected crc 2023-09-23T23:40:54.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:54 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 5734/246216 objects degraded (2.329%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:54.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:54 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:40:54.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:54 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:40:54.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:54 smithi110 ceph-mon[212939]: osdmap e637: 8 total, 8 up, 6 in 2023-09-23T23:40:54.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:54 smithi110 ceph-mon[212939]: failed to encode map e637 with expected crc 2023-09-23T23:40:54.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:54 smithi110 ceph-mon[212939]: failed to encode map e637 with expected crc 2023-09-23T23:40:54.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:54 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 5734/246216 objects degraded (2.329%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:54.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:54 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:40:54.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:54 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:40:54.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:54 smithi110 ceph-mon[221102]: osdmap e637: 8 total, 8 up, 6 in 2023-09-23T23:40:54.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:54 smithi110 ceph-mon[221102]: failed to encode map e637 with expected crc 2023-09-23T23:40:54.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:54 smithi110 ceph-mon[221102]: failed to encode map e637 with expected crc 2023-09-23T23:40:54.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:54 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 5734/246216 objects degraded (2.329%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:40:54.655 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:40:54.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:54.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:40:54.767 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:40:54.792 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:40:54.852 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:55.349 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:40:55.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:55.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:40:55.461 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:40:55.491 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:55 smithi138 ceph-mon[193606]: pgmap v1828: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.1 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 34 MiB/s wr, 587 op/s; 5734/248475 objects degraded (2.308%); 3256/248475 objects misplaced (1.310%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T23:40:55.498 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:55.559 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:40:55.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:55.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:40:55.667 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:40:55.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:55 smithi110 ceph-mon[212939]: pgmap v1828: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.1 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 34 MiB/s wr, 587 op/s; 5734/248475 objects degraded (2.308%); 3256/248475 objects misplaced (1.310%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T23:40:55.710 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:55 smithi110 ceph-mon[221102]: pgmap v1828: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.1 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 34 MiB/s wr, 587 op/s; 5734/248475 objects degraded (2.308%); 3256/248475 objects misplaced (1.310%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T23:40:55.747 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:56.125 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:40:56.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:56.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:40:56.233 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:40:56.348 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:56.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:56 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:40:56.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:56 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:40:56.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:56 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:40:56.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:56 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:40:56.537 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:40:56.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:56.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:40:56.644 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:40:56.673 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:56 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:40:56.673 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:56 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:40:56.676 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:56.727 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:40:56.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:56.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:40:56.809 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:40:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:40:56] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.43.0" 2023-09-23T23:40:56.834 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:40:56.866 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:56.927 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:40:56.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:56.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:40:57.034 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:40:57.130 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:57.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:57 smithi110 ceph-mon[221102]: pgmap v1829: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 29 MiB/s wr, 585 op/s; 5734/248043 objects degraded (2.312%); 2931/248043 objects misplaced (1.182%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T23:40:57.369 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:57 smithi110 ceph-mon[212939]: pgmap v1829: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 29 MiB/s wr, 585 op/s; 5734/248043 objects degraded (2.312%); 2931/248043 objects misplaced (1.182%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T23:40:57.640 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:40:57.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:57.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:40:57.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:57 smithi138 ceph-mon[193606]: pgmap v1829: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 29 MiB/s wr, 585 op/s; 5734/248043 objects degraded (2.312%); 2931/248043 objects misplaced (1.182%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T23:40:57.746 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:40:57.853 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:58.082 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:40:58.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:58.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:40:58.145 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:40:58.210 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:40:58.238 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:58.247 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:40:58.284 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:40:58.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:58.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:40:58.392 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:40:58.419 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:58.469 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:40:58.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:58.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:40:58.576 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:40:58.600 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:40:58.603 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:58.651 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:40:58.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:58.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:40:58.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:40:58.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:40:58.758 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:40:58.858 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:58.870 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:40:58.905 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:40:58.952 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:40:59.007 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:40:59.043 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:40:59.073 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:40:59.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:59.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:40:59.086 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:40:59.132 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:40:59.181 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:40:59.192 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:40:59.203 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:40:59.203 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:40:59.204 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:40:59.204 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:40:59.204 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:40:59.204 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:40:59.204 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:40:59.204 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:40:59.290 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:59.449 INFO:teuthology.orchestra.run.smithi110.stdout:moved 6 / 357 (1.68067%) 2023-09-23T23:40:59.449 INFO:teuthology.orchestra.run.smithi110.stdout:avg 59.5 2023-09-23T23:40:59.449 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 6.44851 -> 5.76628 (expected baseline 7.04154) 2023-09-23T23:40:59.449 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.2 with 47 -> 48 pgs (0.789916 -> 0.806723 * mean) 2023-09-23T23:40:59.449 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.0 with 68 -> 66 pgs (1.14286 -> 1.10924 * mean) 2023-09-23T23:40:59.449 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:40:59.450 INFO:teuthology.orchestra.run.smithi110.stdout:oload 110 2023-09-23T23:40:59.450 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 0.05 2023-09-23T23:40:59.450 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:40:59.450 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0375 2023-09-23T23:40:59.450 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.0412 2023-09-23T23:40:59.450 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 weight 1.0000 -> 0.9500 2023-09-23T23:40:59.450 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 weight 1.0000 -> 0.9500 2023-09-23T23:40:59.642 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:40:59.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:59.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:40:59.650 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:59 smithi110 ceph-mon[221102]: pgmap v1830: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 29 MiB/s wr, 703 op/s; 5734/244494 objects degraded (2.345%); 2629/244494 objects misplaced (1.075%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T23:40:59.650 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:59 smithi110 ceph-mon[221102]: from='client.57181 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:40:59.650 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:59 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','1':'62260'}"}]: dispatch 2023-09-23T23:40:59.650 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:59 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','1':'62260'}"}]: dispatch 2023-09-23T23:40:59.650 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:59 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/911587537' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:40:59.650 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:40:59 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 5734/244494 objects degraded (2.345%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:40:59.651 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:59 smithi110 ceph-mon[212939]: pgmap v1830: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 29 MiB/s wr, 703 op/s; 5734/244494 objects degraded (2.345%); 2629/244494 objects misplaced (1.075%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T23:40:59.651 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:59 smithi110 ceph-mon[212939]: from='client.57181 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:40:59.651 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:59 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','1':'62260'}"}]: dispatch 2023-09-23T23:40:59.651 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:59 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','1':'62260'}"}]: dispatch 2023-09-23T23:40:59.651 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:59 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/911587537' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:40:59.651 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:40:59 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 5734/244494 objects degraded (2.345%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:40:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:59 smithi138 ceph-mon[193606]: pgmap v1830: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 29 MiB/s wr, 703 op/s; 5734/244494 objects degraded (2.345%); 2629/244494 objects misplaced (1.075%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T23:40:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:59 smithi138 ceph-mon[193606]: from='client.57181 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:40:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:59 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','1':'62260'}"}]: dispatch 2023-09-23T23:40:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:59 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','1':'62260'}"}]: dispatch 2023-09-23T23:40:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:59 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/911587537' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:40:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:40:59 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 5734/244494 objects degraded (2.345%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:40:59.751 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:40:59.778 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:40:59.832 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:40:59.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:40:59.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:40:59.937 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:41:00.030 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:00.422 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:00 smithi110 ceph-mon[212939]: failed to encode map e638 with expected crc 2023-09-23T23:41:00.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:00 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','1':'62260'}"}]': finished 2023-09-23T23:41:00.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:00 smithi110 ceph-mon[212939]: osdmap e638: 8 total, 8 up, 6 in 2023-09-23T23:41:00.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:00 smithi110 ceph-mon[212939]: failed to encode map e638 with expected crc 2023-09-23T23:41:00.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:00 smithi110 ceph-mon[212939]: failed to encode map e638 with expected crc 2023-09-23T23:41:00.423 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:00 smithi110 ceph-mon[221102]: failed to encode map e638 with expected crc 2023-09-23T23:41:00.423 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:00 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','1':'62260'}"}]': finished 2023-09-23T23:41:00.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:00 smithi110 ceph-mon[221102]: osdmap e638: 8 total, 8 up, 6 in 2023-09-23T23:41:00.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:00 smithi110 ceph-mon[221102]: failed to encode map e638 with expected crc 2023-09-23T23:41:00.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:00 smithi110 ceph-mon[221102]: failed to encode map e638 with expected crc 2023-09-23T23:41:00.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:00 smithi138 ceph-mon[193606]: failed to encode map e638 with expected crc 2023-09-23T23:41:00.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:00 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','1':'62260'}"}]': finished 2023-09-23T23:41:00.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:00 smithi138 ceph-mon[193606]: osdmap e638: 8 total, 8 up, 6 in 2023-09-23T23:41:00.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:00 smithi138 ceph-mon[193606]: failed to encode map e638 with expected crc 2023-09-23T23:41:00.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:00 smithi138 ceph-mon[193606]: failed to encode map e638 with expected crc 2023-09-23T23:41:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:01 smithi138 ceph-mon[193606]: pgmap v1832: 119 pgs: 1 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 127 B/s rd, 14 MiB/s wr, 551 op/s; 5734/243942 objects degraded (2.351%); 2629/243942 objects misplaced (1.078%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:41:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:01 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:41:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:01 smithi138 ceph-mon[193606]: osdmap e639: 8 total, 8 up, 6 in 2023-09-23T23:41:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:01 smithi138 ceph-mon[193606]: failed to encode map e639 with expected crc 2023-09-23T23:41:01.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:01 smithi138 ceph-mon[193606]: failed to encode map e639 with expected crc 2023-09-23T23:41:01.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:01 smithi138 ceph-mon[193606]: failed to encode map e639 with expected crc 2023-09-23T23:41:01.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:01 smithi138 ceph-mon[193606]: failed to encode map e639 with expected crc 2023-09-23T23:41:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:01 smithi110 ceph-mon[221102]: pgmap v1832: 119 pgs: 1 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 127 B/s rd, 14 MiB/s wr, 551 op/s; 5734/243942 objects degraded (2.351%); 2629/243942 objects misplaced (1.078%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:41:01.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:01 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:41:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:01 smithi110 ceph-mon[221102]: osdmap e639: 8 total, 8 up, 6 in 2023-09-23T23:41:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:01 smithi110 ceph-mon[221102]: failed to encode map e639 with expected crc 2023-09-23T23:41:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:01 smithi110 ceph-mon[221102]: failed to encode map e639 with expected crc 2023-09-23T23:41:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:01 smithi110 ceph-mon[221102]: failed to encode map e639 with expected crc 2023-09-23T23:41:01.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:01 smithi110 ceph-mon[221102]: failed to encode map e639 with expected crc 2023-09-23T23:41:01.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:01 smithi110 ceph-mon[212939]: pgmap v1832: 119 pgs: 1 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 127 B/s rd, 14 MiB/s wr, 551 op/s; 5734/243942 objects degraded (2.351%); 2629/243942 objects misplaced (1.078%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:41:01.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:01 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:41:01.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:01 smithi110 ceph-mon[212939]: osdmap e639: 8 total, 8 up, 6 in 2023-09-23T23:41:01.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:01 smithi110 ceph-mon[212939]: failed to encode map e639 with expected crc 2023-09-23T23:41:01.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:01 smithi110 ceph-mon[212939]: failed to encode map e639 with expected crc 2023-09-23T23:41:01.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:01 smithi110 ceph-mon[212939]: failed to encode map e639 with expected crc 2023-09-23T23:41:01.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:01 smithi110 ceph-mon[212939]: failed to encode map e639 with expected crc 2023-09-23T23:41:02.275 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:41:02.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:02.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:41:02.382 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:41:02.410 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:02.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:02 smithi110 ceph-mon[221102]: osdmap e640: 8 total, 8 up, 6 in 2023-09-23T23:41:02.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:02 smithi110 ceph-mon[221102]: failed to encode map e640 with expected crc 2023-09-23T23:41:02.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:02 smithi110 ceph-mon[221102]: failed to encode map e640 with expected crc 2023-09-23T23:41:02.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:02 smithi110 ceph-mon[221102]: failed to encode map e640 with expected crc 2023-09-23T23:41:02.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:02 smithi110 ceph-mon[212939]: osdmap e640: 8 total, 8 up, 6 in 2023-09-23T23:41:02.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:02 smithi110 ceph-mon[212939]: failed to encode map e640 with expected crc 2023-09-23T23:41:02.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:02 smithi110 ceph-mon[212939]: failed to encode map e640 with expected crc 2023-09-23T23:41:02.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:02 smithi110 ceph-mon[212939]: failed to encode map e640 with expected crc 2023-09-23T23:41:02.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:02 smithi138 ceph-mon[193606]: osdmap e640: 8 total, 8 up, 6 in 2023-09-23T23:41:02.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:02 smithi138 ceph-mon[193606]: failed to encode map e640 with expected crc 2023-09-23T23:41:02.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:02 smithi138 ceph-mon[193606]: failed to encode map e640 with expected crc 2023-09-23T23:41:02.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:02 smithi138 ceph-mon[193606]: failed to encode map e640 with expected crc 2023-09-23T23:41:03.579 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:41:03.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:03.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:41:03.646 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:41:03.683 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T23:41:03.683 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 1.0 2023-09-23T23:41:03.691 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:41:03.818 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:03 smithi110 ceph-mon[212939]: failed to encode map e640 with expected crc 2023-09-23T23:41:03.818 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:03 smithi110 ceph-mon[212939]: failed to encode map e640 with expected crc 2023-09-23T23:41:03.818 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:03 smithi110 ceph-mon[212939]: pgmap v1835: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 474 op/s; 5734/239490 objects degraded (2.394%); 2268/239490 objects misplaced (0.947%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T23:41:03.818 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:03 smithi110 ceph-mon[212939]: failed to encode map e641 with expected crc 2023-09-23T23:41:03.818 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:03 smithi110 ceph-mon[212939]: failed to encode map e641 with expected crc 2023-09-23T23:41:03.818 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:03 smithi110 ceph-mon[212939]: osdmap e641: 8 total, 8 up, 6 in 2023-09-23T23:41:03.822 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:03 smithi110 ceph-mon[221102]: failed to encode map e640 with expected crc 2023-09-23T23:41:03.823 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:03 smithi110 ceph-mon[221102]: failed to encode map e640 with expected crc 2023-09-23T23:41:03.823 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:03 smithi110 ceph-mon[221102]: pgmap v1835: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 474 op/s; 5734/239490 objects degraded (2.394%); 2268/239490 objects misplaced (0.947%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T23:41:03.823 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:03 smithi110 ceph-mon[221102]: failed to encode map e641 with expected crc 2023-09-23T23:41:03.823 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:03 smithi110 ceph-mon[221102]: failed to encode map e641 with expected crc 2023-09-23T23:41:03.823 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:03 smithi110 ceph-mon[221102]: osdmap e641: 8 total, 8 up, 6 in 2023-09-23T23:41:03.859 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:03 smithi138 ceph-mon[193606]: failed to encode map e640 with expected crc 2023-09-23T23:41:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:03 smithi138 ceph-mon[193606]: failed to encode map e640 with expected crc 2023-09-23T23:41:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:03 smithi138 ceph-mon[193606]: pgmap v1835: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 474 op/s; 5734/239490 objects degraded (2.394%); 2268/239490 objects misplaced (0.947%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T23:41:03.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:03 smithi138 ceph-mon[193606]: failed to encode map e641 with expected crc 2023-09-23T23:41:03.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:03 smithi138 ceph-mon[193606]: failed to encode map e641 with expected crc 2023-09-23T23:41:03.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:03 smithi138 ceph-mon[193606]: osdmap e641: 8 total, 8 up, 6 in 2023-09-23T23:41:04.304 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:41:04.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:04.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:41:04.419 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:41:04.447 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:04.496 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:41:04.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:04.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:41:04.528 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:04 smithi110 ceph-mon[212939]: failed to encode map e642 with expected crc 2023-09-23T23:41:04.529 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:04 smithi110 ceph-mon[212939]: osdmap e642: 8 total, 8 up, 6 in 2023-09-23T23:41:04.529 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:04 smithi110 ceph-mon[212939]: failed to encode map e642 with expected crc 2023-09-23T23:41:04.529 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:04 smithi110 ceph-mon[212939]: failed to encode map e642 with expected crc 2023-09-23T23:41:04.529 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:04 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 5734/239490 objects degraded (2.394%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:04.529 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:41:04.529 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:41:04.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.529 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:91462 14.b e642/617 14:dae8676e:::benchmark_data_smithi110_640334_object7583:head v 642'5666, mlcod=642'5664)", 2023-09-23T23:41:04.529 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:04.516716+0000", 2023-09-23T23:41:04.530 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00089240800000000003, 2023-09-23T23:41:04.530 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00092146399999999999, 2023-09-23T23:41:04.530 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:04.530 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:41:04.530 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:04.530 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.530 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:04.530 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516716+0000", 2023-09-23T23:41:04.530 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:04.530 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.530 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:04.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516716+0000", 2023-09-23T23:41:04.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:04.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:04.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516715+0000", 2023-09-23T23:41:04.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:41:04.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:04.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516725+0000", 2023-09-23T23:41:04.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8730000000000002e-06 2023-09-23T23:41:04.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:04.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516744+0000", 2023-09-23T23:41:04.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9700000000000001e-05 2023-09-23T23:41:04.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:04.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516754+0000", 2023-09-23T23:41:04.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6630000000000005e-06 2023-09-23T23:41:04.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.533 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:04.533 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516797+0000", 2023-09-23T23:41:04.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2920999999999998e-05 2023-09-23T23:41:04.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.533 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:04.533 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516821+0000", 2023-09-23T23:41:04.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3893999999999999e-05 2023-09-23T23:41:04.533 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:04.533 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:04.533 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:04.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.534 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:91455 14.8 e642/617 14:19ef060f:::benchmark_data_smithi110_640334_object7576:head v 642'5791, mlcod=642'5787)", 2023-09-23T23:41:04.534 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:04.516773+0000", 2023-09-23T23:41:04.534 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00083518999999999998, 2023-09-23T23:41:04.534 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00093157799999999996, 2023-09-23T23:41:04.534 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:04.534 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:41:04.534 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:04.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:04.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516773+0000", 2023-09-23T23:41:04.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:04.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:04.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516773+0000", 2023-09-23T23:41:04.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:04.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:04.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516773+0000", 2023-09-23T23:41:04.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:41:04.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:04.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516777+0000", 2023-09-23T23:41:04.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1400000000000002e-06 2023-09-23T23:41:04.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:04.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516783+0000", 2023-09-23T23:41:04.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4169999999999997e-06 2023-09-23T23:41:04.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:04.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516788+0000", 2023-09-23T23:41:04.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0690000000000001e-06 2023-09-23T23:41:04.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:04.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516907+0000", 2023-09-23T23:41:04.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118656 2023-09-23T23:41:04.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:04.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516935+0000", 2023-09-23T23:41:04.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8262000000000001e-05 2023-09-23T23:41:04.537 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:04.538 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:04.538 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:04.538 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.538 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:91464 14.b e642/617)", 2023-09-23T23:41:04.538 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:04.516800+0000", 2023-09-23T23:41:04.538 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00080789200000000005, 2023-09-23T23:41:04.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00096035999999999995, 2023-09-23T23:41:04.538 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:04.538 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:41:04.538 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:04.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.539 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:04.539 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516800+0000", 2023-09-23T23:41:04.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:04.539 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.539 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:04.539 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516800+0000", 2023-09-23T23:41:04.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:04.539 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.539 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:04.539 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516800+0000", 2023-09-23T23:41:04.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:41:04.540 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.540 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.540 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:04.540 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516807+0000", 2023-09-23T23:41:04.540 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5610000000000004e-06 2023-09-23T23:41:04.540 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.540 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.540 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:04.540 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516813+0000", 2023-09-23T23:41:04.540 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9869999999999996e-06 2023-09-23T23:41:04.540 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.540 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.540 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:04.541 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516817+0000", 2023-09-23T23:41:04.541 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1699999999999999e-06 2023-09-23T23:41:04.541 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:04.541 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:04.541 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:04.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.541 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:91460 14.8 e642/617)", 2023-09-23T23:41:04.541 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:04.516822+0000", 2023-09-23T23:41:04.541 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00078668100000000001, 2023-09-23T23:41:04.541 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00098293699999999992, 2023-09-23T23:41:04.541 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:04.541 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:41:04.542 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:04.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:04.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516822+0000", 2023-09-23T23:41:04.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:04.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:04.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516822+0000", 2023-09-23T23:41:04.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:04.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:04.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516821+0000", 2023-09-23T23:41:04.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-09-23T23:41:04.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:04.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516825+0000", 2023-09-23T23:41:04.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0860000000000002e-06 2023-09-23T23:41:04.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:04.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516829+0000", 2023-09-23T23:41:04.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6530000000000002e-06 2023-09-23T23:41:04.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:04.544 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:04.544 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:04.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:04.516833+0000", 2023-09-23T23:41:04.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7570000000000002e-06 2023-09-23T23:41:04.544 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:04.544 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:04.544 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:04.544 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:04.544 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-09-23T23:41:04.544 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 4 2023-09-23T23:41:04.544 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:41:04.602 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:41:04.678 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:04.901 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:04 smithi110 ceph-mon[221102]: failed to encode map e642 with expected crc 2023-09-23T23:41:04.902 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:04 smithi110 ceph-mon[221102]: osdmap e642: 8 total, 8 up, 6 in 2023-09-23T23:41:04.902 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:04 smithi110 ceph-mon[221102]: failed to encode map e642 with expected crc 2023-09-23T23:41:04.902 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:04 smithi110 ceph-mon[221102]: failed to encode map e642 with expected crc 2023-09-23T23:41:04.902 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:04 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 5734/239490 objects degraded (2.394%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:04.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:04 smithi138 ceph-mon[193606]: failed to encode map e642 with expected crc 2023-09-23T23:41:04.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:04 smithi138 ceph-mon[193606]: osdmap e642: 8 total, 8 up, 6 in 2023-09-23T23:41:04.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:04 smithi138 ceph-mon[193606]: failed to encode map e642 with expected crc 2023-09-23T23:41:04.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:04 smithi138 ceph-mon[193606]: failed to encode map e642 with expected crc 2023-09-23T23:41:04.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:04 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 5734/239490 objects degraded (2.394%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:05.620 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:41:05.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:05 smithi138 ceph-mon[193606]: pgmap v1838: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 747 op/s; 5734/234963 objects degraded (2.440%); 1886/234963 objects misplaced (0.803%); 12 MiB/s, 185 objects/s recovering 2023-09-23T23:41:05.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:05 smithi138 ceph-mon[193606]: failed to encode map e642 with expected crc 2023-09-23T23:41:05.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:05 smithi110 ceph-mon[212939]: pgmap v1838: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 747 op/s; 5734/234963 objects degraded (2.440%); 1886/234963 objects misplaced (0.803%); 12 MiB/s, 185 objects/s recovering 2023-09-23T23:41:05.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:05 smithi110 ceph-mon[212939]: failed to encode map e642 with expected crc 2023-09-23T23:41:05.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:05 smithi110 ceph-mon[221102]: pgmap v1838: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 747 op/s; 5734/234963 objects degraded (2.440%); 1886/234963 objects misplaced (0.803%); 12 MiB/s, 185 objects/s recovering 2023-09-23T23:41:05.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:05 smithi110 ceph-mon[221102]: failed to encode map e642 with expected crc 2023-09-23T23:41:06.794 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:41:06.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:06.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:41:06.911 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:41:06.937 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:06 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:41:06.940 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:06.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:06 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:41:06.957 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:41:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:41:06] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.43.0" 2023-09-23T23:41:06.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:06 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:41:06.993 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:41:07.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:07.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:41:07.102 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:41:07.109 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:41:07.177 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:07.284 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:41:07.415 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:41:07.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:07.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:41:07.519 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:41:07.547 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:07.595 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:41:07.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:07.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:41:07.668 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:41:07.703 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:41:07.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:07 smithi110 ceph-mon[221102]: pgmap v1839: 119 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 557 op/s; 5734/230166 objects degraded (2.491%); 1354/230166 objects misplaced (0.588%); 10 MiB/s, 163 objects/s recovering 2023-09-23T23:41:07.711 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:07 smithi110 ceph-mon[212939]: pgmap v1839: 119 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 557 op/s; 5734/230166 objects degraded (2.491%); 1354/230166 objects misplaced (0.588%); 10 MiB/s, 163 objects/s recovering 2023-09-23T23:41:07.731 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:07.782 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:41:07.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:07.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:41:07.888 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:41:07.913 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:07 smithi138 ceph-mon[193606]: pgmap v1839: 119 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 557 op/s; 5734/230166 objects degraded (2.491%); 1354/230166 objects misplaced (0.588%); 10 MiB/s, 163 objects/s recovering 2023-09-23T23:41:07.915 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:07.963 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:41:07.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:07.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:41:08.071 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:41:08.099 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:08.149 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:41:08.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:08.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:41:08.256 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:41:08.283 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:08.330 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:41:08.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:08.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:41:08.436 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:41:08.523 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:08.968 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:41:09.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:08 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:09.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:08 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:09.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:08 smithi138 ceph-mon[193606]: pgmap v1840: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 724 op/s; 5734/225201 objects degraded (2.546%); 844/225201 objects misplaced (0.375%); 14 MiB/s, 216 objects/s recovering 2023-09-23T23:41:09.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:08 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4283263077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:41:09.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:08 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 5734/230166 objects degraded (2.491%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:09.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:08 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:09.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:08 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:09.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:08 smithi110 ceph-mon[221102]: pgmap v1840: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 724 op/s; 5734/225201 objects degraded (2.546%); 844/225201 objects misplaced (0.375%); 14 MiB/s, 216 objects/s recovering 2023-09-23T23:41:09.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:08 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4283263077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:41:09.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:08 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 5734/230166 objects degraded (2.491%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:09.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:08 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:09.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:08 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:09.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:08 smithi110 ceph-mon[212939]: pgmap v1840: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 724 op/s; 5734/225201 objects degraded (2.546%); 844/225201 objects misplaced (0.375%); 14 MiB/s, 216 objects/s recovering 2023-09-23T23:41:09.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:08 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4283263077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:41:09.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:08 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 5734/230166 objects degraded (2.491%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:10.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:09 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:41:10.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:09 smithi138 ceph-mon[193606]: failed to encode map e643 with expected crc 2023-09-23T23:41:10.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:09 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4283263077' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:41:10.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:09 smithi138 ceph-mon[193606]: osdmap e643: 8 total, 8 up, 6 in 2023-09-23T23:41:10.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:09 smithi138 ceph-mon[193606]: failed to encode map e643 with expected crc 2023-09-23T23:41:10.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:09 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:41:10.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:09 smithi110 ceph-mon[212939]: failed to encode map e643 with expected crc 2023-09-23T23:41:10.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:09 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4283263077' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:41:10.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:09 smithi110 ceph-mon[212939]: osdmap e643: 8 total, 8 up, 6 in 2023-09-23T23:41:10.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:09 smithi110 ceph-mon[212939]: failed to encode map e643 with expected crc 2023-09-23T23:41:10.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:09 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:41:10.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:09 smithi110 ceph-mon[221102]: failed to encode map e643 with expected crc 2023-09-23T23:41:10.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:09 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4283263077' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:41:10.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:09 smithi110 ceph-mon[221102]: osdmap e643: 8 total, 8 up, 6 in 2023-09-23T23:41:10.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:09 smithi110 ceph-mon[221102]: failed to encode map e643 with expected crc 2023-09-23T23:41:11.014 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:41:11.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:11.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:41:11.127 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:41:11.156 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:11.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:10 smithi110 ceph-mon[212939]: failed to encode map e644 with expected crc 2023-09-23T23:41:11.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:10 smithi110 ceph-mon[212939]: failed to encode map e644 with expected crc 2023-09-23T23:41:11.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:10 smithi110 ceph-mon[212939]: osdmap e644: 8 total, 8 up, 6 in 2023-09-23T23:41:11.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:10 smithi110 ceph-mon[212939]: failed to encode map e644 with expected crc 2023-09-23T23:41:11.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:10 smithi110 ceph-mon[212939]: pgmap v1843: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.6 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 527 op/s; 5734/224580 objects degraded (2.553%); 844/224580 objects misplaced (0.376%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T23:41:11.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:10 smithi110 ceph-mon[221102]: failed to encode map e644 with expected crc 2023-09-23T23:41:11.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:10 smithi110 ceph-mon[221102]: failed to encode map e644 with expected crc 2023-09-23T23:41:11.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:10 smithi110 ceph-mon[221102]: osdmap e644: 8 total, 8 up, 6 in 2023-09-23T23:41:11.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:10 smithi110 ceph-mon[221102]: failed to encode map e644 with expected crc 2023-09-23T23:41:11.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:10 smithi110 ceph-mon[221102]: pgmap v1843: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.6 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 527 op/s; 5734/224580 objects degraded (2.553%); 844/224580 objects misplaced (0.376%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T23:41:11.208 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:41:11.211 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:10 smithi138 ceph-mon[193606]: failed to encode map e644 with expected crc 2023-09-23T23:41:11.211 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:10 smithi138 ceph-mon[193606]: failed to encode map e644 with expected crc 2023-09-23T23:41:11.211 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:10 smithi138 ceph-mon[193606]: osdmap e644: 8 total, 8 up, 6 in 2023-09-23T23:41:11.211 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:10 smithi138 ceph-mon[193606]: failed to encode map e644 with expected crc 2023-09-23T23:41:11.211 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:10 smithi138 ceph-mon[193606]: pgmap v1843: 119 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.6 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 527 op/s; 5734/224580 objects degraded (2.553%); 844/224580 objects misplaced (0.376%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T23:41:11.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:11.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:41:11.313 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:41:11.341 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:11.383 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:41:11.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:11.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:41:11.489 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:41:11.517 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:11.889 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:41:11.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:11.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:41:11.903 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:41:11.994 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:41:12.021 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:12.066 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:41:12.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:12.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:41:12.103 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:11 smithi110 ceph-mon[221102]: failed to encode map e645 with expected crc 2023-09-23T23:41:12.103 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:11 smithi110 ceph-mon[221102]: osdmap e645: 8 total, 8 up, 6 in 2023-09-23T23:41:12.103 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:11 smithi110 ceph-mon[221102]: failed to encode map e645 with expected crc 2023-09-23T23:41:12.103 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:11 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:12.103 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:11 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:12.106 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:11 smithi110 ceph-mon[212939]: failed to encode map e645 with expected crc 2023-09-23T23:41:12.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:11 smithi110 ceph-mon[212939]: osdmap e645: 8 total, 8 up, 6 in 2023-09-23T23:41:12.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:11 smithi110 ceph-mon[212939]: failed to encode map e645 with expected crc 2023-09-23T23:41:12.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:11 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:12.107 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:11 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:12.170 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:41:12.263 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:12.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:11 smithi138 ceph-mon[193606]: failed to encode map e645 with expected crc 2023-09-23T23:41:12.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:11 smithi138 ceph-mon[193606]: osdmap e645: 8 total, 8 up, 6 in 2023-09-23T23:41:12.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:11 smithi138 ceph-mon[193606]: failed to encode map e645 with expected crc 2023-09-23T23:41:12.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:11 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:12.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:11 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:12.534 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:41:12.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:12.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:41:12.647 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:41:12.765 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:12.869 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:41:12.920 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:41:12.966 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:41:12.995 INFO:teuthology.orchestra.run.smithi110.stdout:moved 20 / 357 (5.60224%) 2023-09-23T23:41:12.995 INFO:teuthology.orchestra.run.smithi110.stdout:avg 59.5 2023-09-23T23:41:12.995 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 5.76628 -> 6.92219 (expected baseline 7.04154) 2023-09-23T23:41:12.995 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.2 with 48 -> 52 pgs (0.806723 -> 0.87395 * mean) 2023-09-23T23:41:12.995 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.0 with 66 -> 54 pgs (1.10924 -> 0.907563 * mean) 2023-09-23T23:41:12.995 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:41:12.995 INFO:teuthology.orchestra.run.smithi110.stdout:oload 110 2023-09-23T23:41:12.996 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 1 2023-09-23T23:41:12.996 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:41:12.996 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0349 2023-09-23T23:41:12.996 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.0384 2023-09-23T23:41:12.996 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 weight 0.9500 -> 0.7523 2023-09-23T23:41:12.996 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1 weight 0.9500 -> 0.8396 2023-09-23T23:41:13.028 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:41:13.032 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:41:13.039 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[212939]: failed to encode map e645 with expected crc 2023-09-23T23:41:13.039 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[212939]: failed to encode map e646 with expected crc 2023-09-23T23:41:13.039 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[212939]: failed to encode map e646 with expected crc 2023-09-23T23:41:13.039 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[212939]: osdmap e646: 8 total, 8 up, 6 in 2023-09-23T23:41:13.039 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[212939]: failed to encode map e646 with expected crc 2023-09-23T23:41:13.039 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[212939]: pgmap v1846: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 373 op/s; 5734/220713 objects degraded (2.598%); 314/220713 objects misplaced (0.142%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T23:41:13.039 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[212939]: failed to encode map e646 with expected crc 2023-09-23T23:41:13.039 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'49302','1':'55023'}"}]: dispatch 2023-09-23T23:41:13.039 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'49302','1':'55023'}"}]: dispatch 2023-09-23T23:41:13.039 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3025321830' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:41:13.040 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[221102]: failed to encode map e645 with expected crc 2023-09-23T23:41:13.040 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[221102]: failed to encode map e646 with expected crc 2023-09-23T23:41:13.040 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[221102]: failed to encode map e646 with expected crc 2023-09-23T23:41:13.040 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[221102]: osdmap e646: 8 total, 8 up, 6 in 2023-09-23T23:41:13.040 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[221102]: failed to encode map e646 with expected crc 2023-09-23T23:41:13.040 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[221102]: pgmap v1846: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 373 op/s; 5734/220713 objects degraded (2.598%); 314/220713 objects misplaced (0.142%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T23:41:13.040 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[221102]: failed to encode map e646 with expected crc 2023-09-23T23:41:13.040 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'49302','1':'55023'}"}]: dispatch 2023-09-23T23:41:13.040 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'49302','1':'55023'}"}]: dispatch 2023-09-23T23:41:13.040 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:12 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3025321830' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:41:13.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:13.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:41:13.085 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:41:13.111 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:41:13.149 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:41:13.156 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:41:13.186 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:41:13.204 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:41:13.204 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:41:13.204 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:41:13.204 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:41:13.204 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:41:13.204 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:41:13.204 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:41:13.204 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:41:13.228 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:13.311 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:41:13.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:13.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:41:13.416 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:41:13.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:12 smithi138 ceph-mon[193606]: failed to encode map e645 with expected crc 2023-09-23T23:41:13.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:12 smithi138 ceph-mon[193606]: failed to encode map e646 with expected crc 2023-09-23T23:41:13.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:12 smithi138 ceph-mon[193606]: failed to encode map e646 with expected crc 2023-09-23T23:41:13.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:12 smithi138 ceph-mon[193606]: osdmap e646: 8 total, 8 up, 6 in 2023-09-23T23:41:13.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:12 smithi138 ceph-mon[193606]: failed to encode map e646 with expected crc 2023-09-23T23:41:13.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:12 smithi138 ceph-mon[193606]: pgmap v1846: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 373 op/s; 5734/220713 objects degraded (2.598%); 314/220713 objects misplaced (0.142%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T23:41:13.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:12 smithi138 ceph-mon[193606]: failed to encode map e646 with expected crc 2023-09-23T23:41:13.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:12 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'49302','1':'55023'}"}]: dispatch 2023-09-23T23:41:13.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:12 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'49302','1':'55023'}"}]: dispatch 2023-09-23T23:41:13.441 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:12 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3025321830' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:41:13.442 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:13.484 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:41:13.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:13.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:41:13.594 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:41:13.619 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:13.662 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:41:13.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:13.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:41:13.768 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:41:13.797 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:13.830 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:41:14.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[212939]: from='client.47279 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:41:14.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[212939]: failed to encode map e647 with expected crc 2023-09-23T23:41:14.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'49302','1':'55023'}"}]': finished 2023-09-23T23:41:14.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[212939]: osdmap e647: 8 total, 8 up, 6 in 2023-09-23T23:41:14.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:14.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:41:14.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 5734/220713 objects degraded (2.598%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:14.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[212939]: osdmap e648: 8 total, 8 up, 6 in 2023-09-23T23:41:14.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[212939]: failed to encode map e648 with expected crc 2023-09-23T23:41:14.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[221102]: from='client.47279 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:41:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[221102]: failed to encode map e647 with expected crc 2023-09-23T23:41:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'49302','1':'55023'}"}]': finished 2023-09-23T23:41:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[221102]: osdmap e647: 8 total, 8 up, 6 in 2023-09-23T23:41:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:41:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 5734/220713 objects degraded (2.598%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[221102]: osdmap e648: 8 total, 8 up, 6 in 2023-09-23T23:41:14.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:13 smithi110 ceph-mon[221102]: failed to encode map e648 with expected crc 2023-09-23T23:41:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:13 smithi138 ceph-mon[193606]: from='client.47279 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:41:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:13 smithi138 ceph-mon[193606]: failed to encode map e647 with expected crc 2023-09-23T23:41:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'49302','1':'55023'}"}]': finished 2023-09-23T23:41:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:13 smithi138 ceph-mon[193606]: osdmap e647: 8 total, 8 up, 6 in 2023-09-23T23:41:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:14.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:41:14.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:13 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 5734/220713 objects degraded (2.598%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:14.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:13 smithi138 ceph-mon[193606]: osdmap e648: 8 total, 8 up, 6 in 2023-09-23T23:41:14.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:13 smithi138 ceph-mon[193606]: failed to encode map e648 with expected crc 2023-09-23T23:41:15.047 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:41:15.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:15.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:41:15.151 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:41:15.229 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:14 smithi110 ceph-mon[221102]: failed to encode map e648 with expected crc 2023-09-23T23:41:15.229 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:14 smithi110 ceph-mon[221102]: pgmap v1849: 119 pgs: 3 peering, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.4 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 5734/217332 objects degraded (2.638%); 13 MiB/s, 210 objects/s recovering 2023-09-23T23:41:15.230 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:14 smithi110 ceph-mon[221102]: failed to encode map e648 with expected crc 2023-09-23T23:41:15.230 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:14 smithi110 ceph-mon[221102]: failed to encode map e648 with expected crc 2023-09-23T23:41:15.230 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:14 smithi110 ceph-mon[221102]: failed to encode map e649 with expected crc 2023-09-23T23:41:15.230 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:14 smithi110 ceph-mon[221102]: failed to encode map e649 with expected crc 2023-09-23T23:41:15.230 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:14 smithi110 ceph-mon[221102]: osdmap e649: 8 total, 8 up, 6 in 2023-09-23T23:41:15.231 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:14 smithi110 ceph-mon[212939]: failed to encode map e648 with expected crc 2023-09-23T23:41:15.231 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:14 smithi110 ceph-mon[212939]: pgmap v1849: 119 pgs: 3 peering, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.4 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 5734/217332 objects degraded (2.638%); 13 MiB/s, 210 objects/s recovering 2023-09-23T23:41:15.231 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:14 smithi110 ceph-mon[212939]: failed to encode map e648 with expected crc 2023-09-23T23:41:15.232 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:14 smithi110 ceph-mon[212939]: failed to encode map e648 with expected crc 2023-09-23T23:41:15.232 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:14 smithi110 ceph-mon[212939]: failed to encode map e649 with expected crc 2023-09-23T23:41:15.232 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:14 smithi110 ceph-mon[212939]: failed to encode map e649 with expected crc 2023-09-23T23:41:15.232 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:14 smithi110 ceph-mon[212939]: osdmap e649: 8 total, 8 up, 6 in 2023-09-23T23:41:15.233 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:15.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:14 smithi138 ceph-mon[193606]: failed to encode map e648 with expected crc 2023-09-23T23:41:15.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:14 smithi138 ceph-mon[193606]: pgmap v1849: 119 pgs: 3 peering, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.4 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 5734/217332 objects degraded (2.638%); 13 MiB/s, 210 objects/s recovering 2023-09-23T23:41:15.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:14 smithi138 ceph-mon[193606]: failed to encode map e648 with expected crc 2023-09-23T23:41:15.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:14 smithi138 ceph-mon[193606]: failed to encode map e648 with expected crc 2023-09-23T23:41:15.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:14 smithi138 ceph-mon[193606]: failed to encode map e649 with expected crc 2023-09-23T23:41:15.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:14 smithi138 ceph-mon[193606]: failed to encode map e649 with expected crc 2023-09-23T23:41:15.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:14 smithi138 ceph-mon[193606]: osdmap e649: 8 total, 8 up, 6 in 2023-09-23T23:41:16.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:15 smithi110 ceph-mon[212939]: failed to encode map e649 with expected crc 2023-09-23T23:41:16.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:15 smithi110 ceph-mon[212939]: failed to encode map e649 with expected crc 2023-09-23T23:41:16.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:15 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:41:16.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:15 smithi110 ceph-mon[212939]: failed to encode map e650 with expected crc 2023-09-23T23:41:16.423 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:15 smithi110 ceph-mon[212939]: osdmap e650: 8 total, 8 up, 6 in 2023-09-23T23:41:16.423 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:15 smithi110 ceph-mon[221102]: failed to encode map e649 with expected crc 2023-09-23T23:41:16.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:15 smithi110 ceph-mon[221102]: failed to encode map e649 with expected crc 2023-09-23T23:41:16.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:15 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:41:16.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:15 smithi110 ceph-mon[221102]: failed to encode map e650 with expected crc 2023-09-23T23:41:16.424 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:15 smithi110 ceph-mon[221102]: osdmap e650: 8 total, 8 up, 6 in 2023-09-23T23:41:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:15 smithi138 ceph-mon[193606]: failed to encode map e649 with expected crc 2023-09-23T23:41:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:15 smithi138 ceph-mon[193606]: failed to encode map e649 with expected crc 2023-09-23T23:41:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:15 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:41:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:15 smithi138 ceph-mon[193606]: failed to encode map e650 with expected crc 2023-09-23T23:41:16.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:15 smithi138 ceph-mon[193606]: osdmap e650: 8 total, 8 up, 6 in 2023-09-23T23:41:16.565 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:41:16.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:16.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:41:16.669 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:41:16.768 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:16.954 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:41:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:41:16] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.43.0" 2023-09-23T23:41:17.166 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:41:17.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:17.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:41:17.233 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:41:17.233 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:41:17.274 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:41:17.298 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:17 smithi138 ceph-mon[193606]: failed to encode map e650 with expected crc 2023-09-23T23:41:17.299 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:17 smithi138 ceph-mon[193606]: pgmap v1852: 119 pgs: 1 remapped+peering, 2 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 442 op/s; 8622/215400 objects degraded (4.003%); 2001/215400 objects misplaced (0.929%); 12 MiB/s, 187 objects/s recovering 2023-09-23T23:41:17.299 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:17 smithi138 ceph-mon[193606]: failed to encode map e650 with expected crc 2023-09-23T23:41:17.299 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:17 smithi138 ceph-mon[193606]: osdmap e651: 8 total, 8 up, 6 in 2023-09-23T23:41:17.301 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:17.347 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:41:17.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:17.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:41:17.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:17 smithi110 ceph-mon[212939]: failed to encode map e650 with expected crc 2023-09-23T23:41:17.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:17 smithi110 ceph-mon[212939]: pgmap v1852: 119 pgs: 1 remapped+peering, 2 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 442 op/s; 8622/215400 objects degraded (4.003%); 2001/215400 objects misplaced (0.929%); 12 MiB/s, 187 objects/s recovering 2023-09-23T23:41:17.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:17 smithi110 ceph-mon[212939]: failed to encode map e650 with expected crc 2023-09-23T23:41:17.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:17 smithi110 ceph-mon[212939]: osdmap e651: 8 total, 8 up, 6 in 2023-09-23T23:41:17.435 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:17 smithi110 ceph-mon[221102]: failed to encode map e650 with expected crc 2023-09-23T23:41:17.435 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:17 smithi110 ceph-mon[221102]: pgmap v1852: 119 pgs: 1 remapped+peering, 2 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 442 op/s; 8622/215400 objects degraded (4.003%); 2001/215400 objects misplaced (0.929%); 12 MiB/s, 187 objects/s recovering 2023-09-23T23:41:17.436 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:17 smithi110 ceph-mon[221102]: failed to encode map e650 with expected crc 2023-09-23T23:41:17.436 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:17 smithi110 ceph-mon[221102]: osdmap e651: 8 total, 8 up, 6 in 2023-09-23T23:41:17.453 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:41:17.479 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:17.529 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:41:17.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:17.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:41:17.634 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:41:17.683 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:18 smithi138 ceph-mon[193606]: failed to encode map e651 with expected crc 2023-09-23T23:41:18.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:18 smithi138 ceph-mon[193606]: failed to encode map e651 with expected crc 2023-09-23T23:41:18.458 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:18 smithi110 ceph-mon[212939]: failed to encode map e651 with expected crc 2023-09-23T23:41:18.458 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:18 smithi110 ceph-mon[212939]: failed to encode map e651 with expected crc 2023-09-23T23:41:18.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:18 smithi110 ceph-mon[221102]: failed to encode map e651 with expected crc 2023-09-23T23:41:18.458 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:18 smithi110 ceph-mon[221102]: failed to encode map e651 with expected crc 2023-09-23T23:41:18.614 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:41:18.682 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:41:18.904 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:41:18.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:18.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:41:19.018 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:41:19.046 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:19.096 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:41:19.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:19.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:41:19.202 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:41:19.229 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:19 smithi138 ceph-mon[193606]: pgmap v1854: 119 pgs: 1 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 407 op/s; 8622/212214 objects degraded (4.063%); 2001/212214 objects misplaced (0.943%); 6.5 MiB/s, 105 objects/s recovering 2023-09-23T23:41:19.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:19 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 8622/215400 objects degraded (4.003%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:19.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:19 smithi110 ceph-mon[212939]: pgmap v1854: 119 pgs: 1 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 407 op/s; 8622/212214 objects degraded (4.063%); 2001/212214 objects misplaced (0.943%); 6.5 MiB/s, 105 objects/s recovering 2023-09-23T23:41:19.457 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:19 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 8622/215400 objects degraded (4.003%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:19.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:19 smithi110 ceph-mon[221102]: pgmap v1854: 119 pgs: 1 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 407 op/s; 8622/212214 objects degraded (4.063%); 2001/212214 objects misplaced (0.943%); 6.5 MiB/s, 105 objects/s recovering 2023-09-23T23:41:19.457 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:19 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 8622/215400 objects degraded (4.003%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:19.511 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:41:19.666 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:41:19.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:19.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:41:19.794 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:41:19.821 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:19.866 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:41:19.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:19.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:41:19.974 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:41:20.059 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:20.168 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:20 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T23:41:20.168 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:20 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T23:41:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:20 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T23:41:20.483 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:41:20.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:20.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:41:20.607 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:41:20.688 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:20.939 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:41:20.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:20.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:41:21.051 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:41:21.080 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:21.128 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:41:21.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:21.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:41:21.234 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:41:21.262 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:21.310 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:41:21.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:21.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:41:21.416 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:41:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:21 smithi138 ceph-mon[193606]: pgmap v1855: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 311 op/s; 8622/211716 objects degraded (4.072%); 2001/211716 objects misplaced (0.945%); 4.5 MiB/s, 73 objects/s recovering 2023-09-23T23:41:21.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:21 smithi110 ceph-mon[212939]: pgmap v1855: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 311 op/s; 8622/211716 objects degraded (4.072%); 2001/211716 objects misplaced (0.945%); 4.5 MiB/s, 73 objects/s recovering 2023-09-23T23:41:21.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:21 smithi110 ceph-mon[221102]: pgmap v1855: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 311 op/s; 8622/211716 objects degraded (4.072%); 2001/211716 objects misplaced (0.945%); 4.5 MiB/s, 73 objects/s recovering 2023-09-23T23:41:21.458 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:21.551 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 5, 7, 2, 1] out_osds: [4, 6] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:41:21.552 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T23:41:22.240 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:41:22.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:22.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:41:22.349 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:41:22.371 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:41:22.375 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:22.424 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:41:22.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:22.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:41:22.532 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:41:22.549 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:41:22.558 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:22.605 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:41:22.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:22.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:41:22.704 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:41:22.710 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:41:22.736 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:22.781 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:41:22.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:22.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:41:22.889 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:41:22.971 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:23.182 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:41:23.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:23.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:41:23.299 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:41:23.327 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:23.373 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:41:23.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:23.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:41:23.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:23 smithi110 ceph-mon[212939]: pgmap v1856: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 453 op/s; 8622/207342 objects degraded (4.158%); 1830/207342 objects misplaced (0.883%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:41:23.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:23 smithi110 ceph-mon[221102]: pgmap v1856: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 453 op/s; 8622/207342 objects degraded (4.158%); 1830/207342 objects misplaced (0.883%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:41:23.482 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:41:23.574 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:23.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:23 smithi138 ceph-mon[193606]: pgmap v1856: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 453 op/s; 8622/207342 objects degraded (4.158%); 1830/207342 objects misplaced (0.883%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T23:41:24.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:24 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 8622/207342 objects degraded (4.158%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:24.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:24 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 8622/207342 objects degraded (4.158%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:24 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 8622/207342 objects degraded (4.158%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:25.155 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:41:25.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:25.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:41:25.265 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:41:25.372 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:25.420 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:25 smithi110 ceph-mon[212939]: pgmap v1857: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 516 op/s; 8622/202764 objects degraded (4.252%); 1830/202764 objects misplaced (0.903%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T23:41:25.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:25 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3806000688' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:41:25.421 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:25 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:41:25.421 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:25 smithi110 ceph-mon[221102]: pgmap v1857: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 516 op/s; 8622/202764 objects degraded (4.252%); 1830/202764 objects misplaced (0.903%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T23:41:25.421 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:25 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3806000688' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:41:25.421 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:25 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:41:25.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:25 smithi138 ceph-mon[193606]: pgmap v1857: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 516 op/s; 8622/202764 objects degraded (4.252%); 1830/202764 objects misplaced (0.903%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T23:41:25.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:25 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3806000688' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:41:25.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:25 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:41:25.910 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:41:26.087 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:41:26.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:26.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:41:26.191 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:41:26.221 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:26.270 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:41:26.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:26.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:41:26.377 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:41:26.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:26 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:41:26.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:26 smithi110 ceph-mon[221102]: failed to encode map e652 with expected crc 2023-09-23T23:41:26.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:26 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:41:26.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:26 smithi110 ceph-mon[221102]: osdmap e652: 8 total, 8 up, 6 in 2023-09-23T23:41:26.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:26 smithi110 ceph-mon[221102]: failed to encode map e652 with expected crc 2023-09-23T23:41:26.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:26 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:41:26.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:26 smithi110 ceph-mon[212939]: failed to encode map e652 with expected crc 2023-09-23T23:41:26.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:26 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:41:26.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:26 smithi110 ceph-mon[212939]: osdmap e652: 8 total, 8 up, 6 in 2023-09-23T23:41:26.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:26 smithi110 ceph-mon[212939]: failed to encode map e652 with expected crc 2023-09-23T23:41:26.482 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:26 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:41:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:26 smithi138 ceph-mon[193606]: failed to encode map e652 with expected crc 2023-09-23T23:41:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:26 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:41:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:26 smithi138 ceph-mon[193606]: osdmap e652: 8 total, 8 up, 6 in 2023-09-23T23:41:26.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:26 smithi138 ceph-mon[193606]: failed to encode map e652 with expected crc 2023-09-23T23:41:26.804 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:41:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:41:26] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.43.0" 2023-09-23T23:41:27.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:27 smithi138 ceph-mon[193606]: pgmap v1859: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 622 op/s; 8622/198309 objects degraded (4.348%); 1683/198309 objects misplaced (0.849%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T23:41:27.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:27 smithi110 ceph-mon[212939]: pgmap v1859: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 622 op/s; 8622/198309 objects degraded (4.348%); 1683/198309 objects misplaced (0.849%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T23:41:27.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:27 smithi110 ceph-mon[221102]: pgmap v1859: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 622 op/s; 8622/198309 objects degraded (4.348%); 1683/198309 objects misplaced (0.849%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T23:41:27.896 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:41:27.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:27.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:41:28.003 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:41:28.032 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:29.304 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:41:29.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:29.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:41:29.405 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:29 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:29.405 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:29 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:41:29.406 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:29 smithi110 ceph-mon[212939]: pgmap v1860: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 621 op/s; 8622/193557 objects degraded (4.455%); 1683/193557 objects misplaced (0.870%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T23:41:29.406 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:29 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 8622/198309 objects degraded (4.348%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:29.406 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:29 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:29.406 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:29 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:41:29.407 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:29 smithi110 ceph-mon[221102]: pgmap v1860: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 621 op/s; 8622/193557 objects degraded (4.455%); 1683/193557 objects misplaced (0.870%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T23:41:29.407 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:29 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 8622/198309 objects degraded (4.348%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:29.414 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:41:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:29 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:29 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:41:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:29 smithi138 ceph-mon[193606]: pgmap v1860: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 621 op/s; 8622/193557 objects degraded (4.455%); 1683/193557 objects misplaced (0.870%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T23:41:29.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:29 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 8622/198309 objects degraded (4.348%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:29.512 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:29.617 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:41:29.652 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:41:29.658 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:41:29.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:29.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:41:29.717 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:41:29.751 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:41:29.772 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:41:29.799 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:41:29.841 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:41:29.896 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:29.904 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:41:29.947 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:41:29.966 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:41:29.966 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:41:29.966 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:41:29.966 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:41:29.966 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:41:29.966 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:41:29.966 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:41:29.966 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:41:30.013 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:41:30.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:30.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:41:30.122 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:41:30.203 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:30.303 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:41:30.303 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:41:30.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:41:30.303 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:41:30.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.304 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:175261 12.10 12:0f9d8e44:::benchmark_data_smithi110_491193_object80386:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2023-09-23T23:41:30.304 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.428760+0000", 2023-09-23T23:41:30.304 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 562.86810356000001, 2023-09-23T23:41:30.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.279936254, 2023-09-23T23:41:30.304 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.304 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.304 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:41:30.304 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:41:30.304 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:41:30.304 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175261 2023-09-23T23:41:30.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.304 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428760+0000", 2023-09-23T23:41:30.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428760+0000", 2023-09-23T23:41:30.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428758+0000", 2023-09-23T23:41:30.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:41:30.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428765+0000", 2023-09-23T23:41:30.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8680000000000001e-06 2023-09-23T23:41:30.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428778+0000", 2023-09-23T23:41:30.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2687000000000001e-05 2023-09-23T23:41:30.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428785+0000", 2023-09-23T23:41:30.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1679999999999998e-06 2023-09-23T23:41:30.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428858+0000", 2023-09-23T23:41:30.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3231000000000004e-05 2023-09-23T23:41:30.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.428968+0000", 2023-09-23T23:41:30.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010950699999999999 2023-09-23T23:41:30.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:41:30.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.429420+0000", 2023-09-23T23:41:30.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045183000000000002 2023-09-23T23:41:30.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:41:30.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.432669+0000", 2023-09-23T23:41:30.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032491830000000001 2023-09-23T23:41:30.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:41:30.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708584+0000", 2023-09-23T23:41:30.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.27591494900000002 2023-09-23T23:41:30.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:41:30.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708613+0000", 2023-09-23T23:41:30.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9178000000000001e-05 2023-09-23T23:41:30.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708659+0000", 2023-09-23T23:41:30.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6415000000000002e-05 2023-09-23T23:41:30.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708696+0000", 2023-09-23T23:41:30.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6832000000000002e-05 2023-09-23T23:41:30.310 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.310 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.310 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.311 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:175263 12.8 e524/490 12:11714bff:::benchmark_data_smithi110_491193_object80388:head v 524'7666, mlcod=524'7664)", 2023-09-23T23:41:30.311 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.433325+0000", 2023-09-23T23:41:30.311 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 562.86353815500001, 2023-09-23T23:41:30.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.282317341, 2023-09-23T23:41:30.311 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.311 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.311 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433325+0000", 2023-09-23T23:41:30.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433325+0000", 2023-09-23T23:41:30.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433325+0000", 2023-09-23T23:41:30.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:41:30.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433333+0000", 2023-09-23T23:41:30.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5420000000000005e-06 2023-09-23T23:41:30.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433344+0000", 2023-09-23T23:41:30.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1358e-05 2023-09-23T23:41:30.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433351+0000", 2023-09-23T23:41:30.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7209999999999997e-06 2023-09-23T23:41:30.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433384+0000", 2023-09-23T23:41:30.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3065000000000003e-05 2023-09-23T23:41:30.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.433404+0000", 2023-09-23T23:41:30.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9762000000000002e-05 2023-09-23T23:41:30.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.715624+0000", 2023-09-23T23:41:30.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28222047 2023-09-23T23:41:30.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.715643+0000", 2023-09-23T23:41:30.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8238999999999999e-05 2023-09-23T23:41:30.315 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.315 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.316 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.316 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46229.0:175265 12.10 12:09b7804b:::benchmark_data_smithi110_491193_object80390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2023-09-23T23:41:30.316 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.434661+0000", 2023-09-23T23:41:30.316 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 562.86220216000004, 2023-09-23T23:41:30.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28935229200000001, 2023-09-23T23:41:30.316 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.316 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.316 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:41:30.316 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46229", 2023-09-23T23:41:30.316 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4240424848", 2023-09-23T23:41:30.317 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 175265 2023-09-23T23:41:30.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.317 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.434661+0000", 2023-09-23T23:41:30.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.434661+0000", 2023-09-23T23:41:30.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.434660+0000", 2023-09-23T23:41:30.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:41:30.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.434667+0000", 2023-09-23T23:41:30.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6919999999999996e-06 2023-09-23T23:41:30.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708315+0000", 2023-09-23T23:41:30.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.27364855599999999 2023-09-23T23:41:30.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708328+0000", 2023-09-23T23:41:30.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2444e-05 2023-09-23T23:41:30.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.712241+0000", 2023-09-23T23:41:30.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039134829999999997 2023-09-23T23:41:30.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.712351+0000", 2023-09-23T23:41:30.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110096 2023-09-23T23:41:30.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:41:30.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.712806+0000", 2023-09-23T23:41:30.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000454604 2023-09-23T23:41:30.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:41:30.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.718758+0000", 2023-09-23T23:41:30.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059521080000000002 2023-09-23T23:41:30.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:41:30.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.718791+0000", 2023-09-23T23:41:30.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3618000000000002e-05 2023-09-23T23:41:30.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:41:30.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.723932+0000", 2023-09-23T23:41:30.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051407349999999996 2023-09-23T23:41:30.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.723976+0000", 2023-09-23T23:41:30.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3618000000000001e-05 2023-09-23T23:41:30.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.724014+0000", 2023-09-23T23:41:30.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7829e-05 2023-09-23T23:41:30.322 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.322 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.323 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.323 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.46229.0:175259 12.c e524/476 12:3a3a4253:::benchmark_data_smithi110_491193_object80384:head v 524'10995, mlcod=524'10989)", 2023-09-23T23:41:30.323 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:32:07.436466+0000", 2023-09-23T23:41:30.323 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 562.86039711700005, 2023-09-23T23:41:30.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28907641299999998, 2023-09-23T23:41:30.323 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.323 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.323 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436466+0000", 2023-09-23T23:41:30.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436466+0000", 2023-09-23T23:41:30.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436466+0000", 2023-09-23T23:41:30.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:41:30.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.436473+0000", 2023-09-23T23:41:30.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4479999999999997e-06 2023-09-23T23:41:30.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708281+0000", 2023-09-23T23:41:30.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.27180817299999999 2023-09-23T23:41:30.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708296+0000", 2023-09-23T23:41:30.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4875000000000001e-05 2023-09-23T23:41:30.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708345+0000", 2023-09-23T23:41:30.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9089999999999999e-05 2023-09-23T23:41:30.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.708371+0000", 2023-09-23T23:41:30.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5809e-05 2023-09-23T23:41:30.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.327 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.327 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.725504+0000", 2023-09-23T23:41:30.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017133116 2023-09-23T23:41:30.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.327 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.327 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:32:07.725543+0000", 2023-09-23T23:41:30.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8574e-05 2023-09-23T23:41:30.327 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.327 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.328 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.328 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:20745 14.7 e622/617 14:e2e79db3:::benchmark_data_smithi110_640334_object20744:head v 622'1260, mlcod=622'1258)", 2023-09-23T23:41:30.328 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:32.543221+0000", 2023-09-23T23:41:30.328 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 117.753642359, 2023-09-23T23:41:30.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.102787935, 2023-09-23T23:41:30.328 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.328 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.328 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.543221+0000", 2023-09-23T23:41:30.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.329 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.543221+0000", 2023-09-23T23:41:30.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.329 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.543220+0000", 2023-09-23T23:41:30.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:41:30.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.543275+0000", 2023-09-23T23:41:30.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4375e-05 2023-09-23T23:41:30.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.543285+0000", 2023-09-23T23:41:30.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0638e-05 2023-09-23T23:41:30.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.543292+0000", 2023-09-23T23:41:30.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7159999999999997e-06 2023-09-23T23:41:30.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.544178+0000", 2023-09-23T23:41:30.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00088605300000000004 2023-09-23T23:41:30.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.544196+0000", 2023-09-23T23:41:30.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8309999999999999e-05 2023-09-23T23:41:30.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.546352+0000", 2023-09-23T23:41:30.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021560189999999999 2023-09-23T23:41:30.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.646009+0000", 2023-09-23T23:41:30.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.099656588000000004 2023-09-23T23:41:30.332 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.332 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.332 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.333 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:20738 14.3 e622/617 14:c744d650:::benchmark_data_smithi110_640334_object20737:head v 622'1275, mlcod=622'1270)", 2023-09-23T23:41:30.333 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:32.544195+0000", 2023-09-23T23:41:30.333 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 117.752668888, 2023-09-23T23:41:30.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.103316703, 2023-09-23T23:41:30.333 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.333 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.333 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.544195+0000", 2023-09-23T23:41:30.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.544195+0000", 2023-09-23T23:41:30.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.544194+0000", 2023-09-23T23:41:30.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:41:30.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.335 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.335 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.544215+0000", 2023-09-23T23:41:30.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0965999999999999e-05 2023-09-23T23:41:30.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.335 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.335 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.544225+0000", 2023-09-23T23:41:30.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0236e-05 2023-09-23T23:41:30.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.544232+0000", 2023-09-23T23:41:30.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5239999999999997e-06 2023-09-23T23:41:30.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.544270+0000", 2023-09-23T23:41:30.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7954999999999998e-05 2023-09-23T23:41:30.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.544288+0000", 2023-09-23T23:41:30.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8269000000000001e-05 2023-09-23T23:41:30.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.647472+0000", 2023-09-23T23:41:30.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.103183699 2023-09-23T23:41:30.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.647511+0000", 2023-09-23T23:41:30.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9739e-05 2023-09-23T23:41:30.337 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.338 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.338 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.338 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:20744 14.3 e622/617 14:cb505301:::benchmark_data_smithi110_640334_object20743:head v 622'1276, mlcod=622'1270)", 2023-09-23T23:41:30.338 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:32.544853+0000", 2023-09-23T23:41:30.338 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 117.752010377, 2023-09-23T23:41:30.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.102712711, 2023-09-23T23:41:30.338 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.338 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.338 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.544853+0000", 2023-09-23T23:41:30.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.544853+0000", 2023-09-23T23:41:30.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.544852+0000", 2023-09-23T23:41:30.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:41:30.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.544875+0000", 2023-09-23T23:41:30.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2594000000000001e-05 2023-09-23T23:41:30.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.544885+0000", 2023-09-23T23:41:30.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0165e-05 2023-09-23T23:41:30.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.544892+0000", 2023-09-23T23:41:30.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8140000000000003e-06 2023-09-23T23:41:30.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.645991+0000", 2023-09-23T23:41:30.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.101099414 2023-09-23T23:41:30.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.646014+0000", 2023-09-23T23:41:30.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2971000000000001e-05 2023-09-23T23:41:30.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.647548+0000", 2023-09-23T23:41:30.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015333180000000001 2023-09-23T23:41:30.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.647566+0000", 2023-09-23T23:41:30.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8139e-05 2023-09-23T23:41:30.342 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.343 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.343 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.343 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:20743 14.b e622/617 14:dcca59c6:::benchmark_data_smithi110_640334_object20742:head v 622'1302, mlcod=622'1300)", 2023-09-23T23:41:30.343 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:32.545156+0000", 2023-09-23T23:41:30.343 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 117.75170707300001, 2023-09-23T23:41:30.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10238944899999999, 2023-09-23T23:41:30.343 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.343 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.343 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545156+0000", 2023-09-23T23:41:30.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545156+0000", 2023-09-23T23:41:30.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545156+0000", 2023-09-23T23:41:30.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:41:30.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545187+0000", 2023-09-23T23:41:30.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1167999999999997e-05 2023-09-23T23:41:30.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545196+0000", 2023-09-23T23:41:30.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8130000000000002e-06 2023-09-23T23:41:30.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545203+0000", 2023-09-23T23:41:30.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.109e-06 2023-09-23T23:41:30.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545406+0000", 2023-09-23T23:41:30.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020331 2023-09-23T23:41:30.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545422+0000", 2023-09-23T23:41:30.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6211000000000001e-05 2023-09-23T23:41:30.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.647516+0000", 2023-09-23T23:41:30.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10209351999999999 2023-09-23T23:41:30.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.647546+0000", 2023-09-23T23:41:30.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0049999999999999e-05 2023-09-23T23:41:30.347 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.347 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.348 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.348 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:20746 14.6 e622/617 14:65e46a2e:::benchmark_data_smithi110_640334_object20745:head v 622'1318, mlcod=622'1316)", 2023-09-23T23:41:30.348 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:32.545215+0000", 2023-09-23T23:41:30.348 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 117.751648037, 2023-09-23T23:41:30.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.100812318, 2023-09-23T23:41:30.348 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.348 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.348 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545215+0000", 2023-09-23T23:41:30.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545215+0000", 2023-09-23T23:41:30.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545215+0000", 2023-09-23T23:41:30.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:41:30.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545244+0000", 2023-09-23T23:41:30.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9294000000000001e-05 2023-09-23T23:41:30.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545251+0000", 2023-09-23T23:41:30.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9940000000000003e-06 2023-09-23T23:41:30.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545256+0000", 2023-09-23T23:41:30.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0200000000000002e-06 2023-09-23T23:41:30.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545300+0000", 2023-09-23T23:41:30.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3896999999999997e-05 2023-09-23T23:41:30.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545320+0000", 2023-09-23T23:41:30.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9168e-05 2023-09-23T23:41:30.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.645998+0000", 2023-09-23T23:41:30.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.100678455 2023-09-23T23:41:30.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.646028+0000", 2023-09-23T23:41:30.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9774999999999999e-05 2023-09-23T23:41:30.353 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.353 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.353 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.353 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:20747 14.c e622/617 14:3d61ae78:::benchmark_data_smithi110_640334_object20746:head v 622'1294, mlcod=622'1292)", 2023-09-23T23:41:30.353 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:32.545270+0000", 2023-09-23T23:41:30.353 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 117.75159363900001, 2023-09-23T23:41:30.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.102387715, 2023-09-23T23:41:30.353 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.353 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.353 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545270+0000", 2023-09-23T23:41:30.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545270+0000", 2023-09-23T23:41:30.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545269+0000", 2023-09-23T23:41:30.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:41:30.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545291+0000", 2023-09-23T23:41:30.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1722e-05 2023-09-23T23:41:30.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545297+0000", 2023-09-23T23:41:30.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.139e-06 2023-09-23T23:41:30.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545302+0000", 2023-09-23T23:41:30.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4469999999999999e-06 2023-09-23T23:41:30.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545344+0000", 2023-09-23T23:41:30.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1931999999999998e-05 2023-09-23T23:41:30.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545364+0000", 2023-09-23T23:41:30.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9961000000000001e-05 2023-09-23T23:41:30.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.647624+0000", 2023-09-23T23:41:30.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.102260618 2023-09-23T23:41:30.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.647658+0000", 2023-09-23T23:41:30.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3262000000000001e-05 2023-09-23T23:41:30.358 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.358 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.358 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.358 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:20742 14.8 e622/617 14:161d6a3c:::benchmark_data_smithi110_640334_object20741:head v 622'1333, mlcod=622'1330)", 2023-09-23T23:41:30.358 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:32.545315+0000", 2023-09-23T23:41:30.358 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 117.751548668, 2023-09-23T23:41:30.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.102128384, 2023-09-23T23:41:30.358 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.358 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.359 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545315+0000", 2023-09-23T23:41:30.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545315+0000", 2023-09-23T23:41:30.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545315+0000", 2023-09-23T23:41:30.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:41:30.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545336+0000", 2023-09-23T23:41:30.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1328999999999999e-05 2023-09-23T23:41:30.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545342+0000", 2023-09-23T23:41:30.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.421e-06 2023-09-23T23:41:30.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.545347+0000", 2023-09-23T23:41:30.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7199999999999997e-06 2023-09-23T23:41:30.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.645964+0000", 2023-09-23T23:41:30.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.100616499 2023-09-23T23:41:30.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.645990+0000", 2023-09-23T23:41:30.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6310000000000001e-05 2023-09-23T23:41:30.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.647411+0000", 2023-09-23T23:41:30.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001421248 2023-09-23T23:41:30.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.647443+0000", 2023-09-23T23:41:30.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2156999999999997e-05 2023-09-23T23:41:30.363 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.363 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.363 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.363 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:20751 14.c e622/617 14:3ef9f1e1:::benchmark_data_smithi110_640334_object20750:head v 622'1295, mlcod=622'1292)", 2023-09-23T23:41:30.363 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:32.546441+0000", 2023-09-23T23:41:30.363 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 117.750422872, 2023-09-23T23:41:30.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.101744362, 2023-09-23T23:41:30.364 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.364 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.364 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.546441+0000", 2023-09-23T23:41:30.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.546441+0000", 2023-09-23T23:41:30.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.546440+0000", 2023-09-23T23:41:30.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:41:30.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.546467+0000", 2023-09-23T23:41:30.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7509e-05 2023-09-23T23:41:30.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.645939+0000", 2023-09-23T23:41:30.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.099471991999999995 2023-09-23T23:41:30.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.645952+0000", 2023-09-23T23:41:30.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2486e-05 2023-09-23T23:41:30.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.646929+0000", 2023-09-23T23:41:30.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00097667600000000011 2023-09-23T23:41:30.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.646947+0000", 2023-09-23T23:41:30.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8168999999999998e-05 2023-09-23T23:41:30.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.648153+0000", 2023-09-23T23:41:30.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012062780000000001 2023-09-23T23:41:30.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.648185+0000", 2023-09-23T23:41:30.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1955000000000002e-05 2023-09-23T23:41:30.368 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.368 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.368 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.368 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:20753 14.7 e622/617 14:e7da5c21:::benchmark_data_smithi110_640334_object20752:head v 622'1261, mlcod=622'1258)", 2023-09-23T23:41:30.368 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:32.546504+0000", 2023-09-23T23:41:30.368 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 117.750359102, 2023-09-23T23:41:30.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.101195984, 2023-09-23T23:41:30.369 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.369 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.369 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.546504+0000", 2023-09-23T23:41:30.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.546504+0000", 2023-09-23T23:41:30.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.546504+0000", 2023-09-23T23:41:30.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:41:30.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.546561+0000", 2023-09-23T23:41:30.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6609e-05 2023-09-23T23:41:30.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.645940+0000", 2023-09-23T23:41:30.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.099379751000000002 2023-09-23T23:41:30.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.645952+0000", 2023-09-23T23:41:30.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2203e-05 2023-09-23T23:41:30.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.646065+0000", 2023-09-23T23:41:30.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000112878 2023-09-23T23:41:30.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.646078+0000", 2023-09-23T23:41:30.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3162e-05 2023-09-23T23:41:30.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.647670+0000", 2023-09-23T23:41:30.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015917170000000001 2023-09-23T23:41:30.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:32.647700+0000", 2023-09-23T23:41:30.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0173999999999999e-05 2023-09-23T23:41:30.373 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.373 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.373 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.373 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:36258 14.7 e624/617 14:e0ceb5cd:::benchmark_data_smithi110_640334_object36257:head v 624'2253, mlcod=623'2251)", 2023-09-23T23:41:30.373 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.854454+0000", 2023-09-23T23:41:30.373 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.442409881, 2023-09-23T23:41:30.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.361132012, 2023-09-23T23:41:30.374 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.374 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.374 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.854454+0000", 2023-09-23T23:41:30.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.854454+0000", 2023-09-23T23:41:30.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.854453+0000", 2023-09-23T23:41:30.375 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:41:30.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.854490+0000", 2023-09-23T23:41:30.375 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7032000000000001e-05 2023-09-23T23:41:30.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.854499+0000", 2023-09-23T23:41:30.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6580000000000008e-06 2023-09-23T23:41:30.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.854506+0000", 2023-09-23T23:41:30.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7360000000000004e-06 2023-09-23T23:41:30.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.854566+0000", 2023-09-23T23:41:30.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0486999999999999e-05 2023-09-23T23:41:30.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.854587+0000", 2023-09-23T23:41:30.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0795e-05 2023-09-23T23:41:30.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.215544+0000", 2023-09-23T23:41:30.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.36095729999999998 2023-09-23T23:41:30.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.215586+0000", 2023-09-23T23:41:30.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1295999999999999e-05 2023-09-23T23:41:30.379 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.379 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.379 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.379 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:36244 14.6 e623/617 14:6c63a03b:::benchmark_data_smithi110_640334_object36243:head v 623'2310, mlcod=623'2308)", 2023-09-23T23:41:30.379 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.857036+0000", 2023-09-23T23:41:30.379 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.43982722699999, 2023-09-23T23:41:30.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.36113023900000002, 2023-09-23T23:41:30.379 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.379 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.379 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.857036+0000", 2023-09-23T23:41:30.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.857036+0000", 2023-09-23T23:41:30.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.857036+0000", 2023-09-23T23:41:30.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:41:30.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.857056+0000", 2023-09-23T23:41:30.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9850000000000001e-05 2023-09-23T23:41:30.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.215107+0000", 2023-09-23T23:41:30.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.358051595 2023-09-23T23:41:30.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.215124+0000", 2023-09-23T23:41:30.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6506000000000001e-05 2023-09-23T23:41:30.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.215807+0000", 2023-09-23T23:41:30.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00068340499999999999 2023-09-23T23:41:30.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.215824+0000", 2023-09-23T23:41:30.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6331999999999999e-05 2023-09-23T23:41:30.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.218129+0000", 2023-09-23T23:41:30.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023048840000000001 2023-09-23T23:41:30.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.218167+0000", 2023-09-23T23:41:30.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7948999999999999e-05 2023-09-23T23:41:30.383 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.384 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.384 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.384 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:36254 14.3 e624/617 14:c0f10be9:::benchmark_data_smithi110_640334_object36253:head v 624'2236, mlcod=623'2234)", 2023-09-23T23:41:30.384 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:44.860110+0000", 2023-09-23T23:41:30.384 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.436753283, 2023-09-23T23:41:30.384 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.35919823899999997, 2023-09-23T23:41:30.384 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.384 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.384 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.385 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.860110+0000", 2023-09-23T23:41:30.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.385 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.860110+0000", 2023-09-23T23:41:30.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.385 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.860109+0000", 2023-09-23T23:41:30.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:41:30.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:44.860140+0000", 2023-09-23T23:41:30.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0454999999999999e-05 2023-09-23T23:41:30.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.215106+0000", 2023-09-23T23:41:30.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.35496614500000001 2023-09-23T23:41:30.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.215125+0000", 2023-09-23T23:41:30.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9451000000000001e-05 2023-09-23T23:41:30.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.215354+0000", 2023-09-23T23:41:30.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022907299999999999 2023-09-23T23:41:30.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.215380+0000", 2023-09-23T23:41:30.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.563e-05 2023-09-23T23:41:30.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.219265+0000", 2023-09-23T23:41:30.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038846810000000001 2023-09-23T23:41:30.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:45.219308+0000", 2023-09-23T23:41:30.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3789999999999999e-05 2023-09-23T23:41:30.388 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.389 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.389 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.389 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.389 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:41905 14.c e626/617 14:3506c616:::benchmark_data_smithi110_640334_object41904:head v 626'2584, mlcod=626'2581)", 2023-09-23T23:41:30.389 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:50.398056+0000", 2023-09-23T23:41:30.389 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 99.898807740999999, 2023-09-23T23:41:30.389 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12603710100000001, 2023-09-23T23:41:30.389 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.389 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.389 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.390 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.390 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.398056+0000", 2023-09-23T23:41:30.390 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.390 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.390 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.390 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.390 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.398056+0000", 2023-09-23T23:41:30.390 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.390 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.390 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.390 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.390 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.398055+0000", 2023-09-23T23:41:30.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:41:30.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.391 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.398097+0000", 2023-09-23T23:41:30.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1727999999999998e-05 2023-09-23T23:41:30.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.391 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.398108+0000", 2023-09-23T23:41:30.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1592e-05 2023-09-23T23:41:30.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.398116+0000", 2023-09-23T23:41:30.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0269999999999995e-06 2023-09-23T23:41:30.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.398162+0000", 2023-09-23T23:41:30.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5720000000000003e-05 2023-09-23T23:41:30.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.398183+0000", 2023-09-23T23:41:30.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0684999999999999e-05 2023-09-23T23:41:30.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.393 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.524060+0000", 2023-09-23T23:41:30.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.125877302 2023-09-23T23:41:30.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.393 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.524093+0000", 2023-09-23T23:41:30.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3036e-05 2023-09-23T23:41:30.393 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.394 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.394 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.394 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:41900 14.6 e626/617 14:6e61a616:::benchmark_data_smithi110_640334_object41899:head v 626'2670, mlcod=626'2667)", 2023-09-23T23:41:30.394 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:50.398136+0000", 2023-09-23T23:41:30.394 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 99.898727381, 2023-09-23T23:41:30.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12732508200000001, 2023-09-23T23:41:30.394 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.394 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.394 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.398136+0000", 2023-09-23T23:41:30.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.398136+0000", 2023-09-23T23:41:30.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.398136+0000", 2023-09-23T23:41:30.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:41:30.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.398160+0000", 2023-09-23T23:41:30.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4130000000000001e-05 2023-09-23T23:41:30.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.398167+0000", 2023-09-23T23:41:30.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2969999999999998e-06 2023-09-23T23:41:30.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.397 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.397 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.398172+0000", 2023-09-23T23:41:30.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2220000000000001e-06 2023-09-23T23:41:30.397 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.397 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.397 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.397 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.521157+0000", 2023-09-23T23:41:30.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12298458900000001 2023-09-23T23:41:30.397 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.397 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.397 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.397 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.521181+0000", 2023-09-23T23:41:30.398 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4488999999999999e-05 2023-09-23T23:41:30.398 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.398 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.398 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.398 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.525437+0000", 2023-09-23T23:41:30.398 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042551029999999997 2023-09-23T23:41:30.398 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.398 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.398 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.398 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.525461+0000", 2023-09-23T23:41:30.398 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4607999999999999e-05 2023-09-23T23:41:30.398 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.399 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.399 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.399 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:41904 14.6 e626/617 14:67176013:::benchmark_data_smithi110_640334_object41903:head v 626'2671, mlcod=626'2667)", 2023-09-23T23:41:30.399 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:50.399117+0000", 2023-09-23T23:41:30.399 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 99.897746088000005, 2023-09-23T23:41:30.399 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12637540899999999, 2023-09-23T23:41:30.399 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.399 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.399 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.400 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.399117+0000", 2023-09-23T23:41:30.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.400 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.400 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.399117+0000", 2023-09-23T23:41:30.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.400 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.400 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.399117+0000", 2023-09-23T23:41:30.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:41:30.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.399143+0000", 2023-09-23T23:41:30.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6432e-05 2023-09-23T23:41:30.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.521124+0000", 2023-09-23T23:41:30.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.121981278 2023-09-23T23:41:30.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.521137+0000", 2023-09-23T23:41:30.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2659e-05 2023-09-23T23:41:30.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.522740+0000", 2023-09-23T23:41:30.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016029449999999999 2023-09-23T23:41:30.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.522758+0000", 2023-09-23T23:41:30.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7992999999999999e-05 2023-09-23T23:41:30.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.525465+0000", 2023-09-23T23:41:30.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002707096 2023-09-23T23:41:30.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.525493+0000", 2023-09-23T23:41:30.404 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7712000000000002e-05 2023-09-23T23:41:30.404 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.404 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.404 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.404 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.404 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.404 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:41896 14.9 e626/617 14:9fdd985a:::benchmark_data_smithi110_640334_object41895:head v 626'2608, mlcod=626'2605)", 2023-09-23T23:41:30.404 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:50.404036+0000", 2023-09-23T23:41:30.404 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 99.892827467000004, 2023-09-23T23:41:30.404 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.120294215, 2023-09-23T23:41:30.404 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:30.404 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:41:30.405 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:30.405 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.405 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:30.405 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.404036+0000", 2023-09-23T23:41:30.405 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.405 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.405 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.405 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:30.405 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.404036+0000", 2023-09-23T23:41:30.405 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:30.405 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.405 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.405 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:30.406 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.404035+0000", 2023-09-23T23:41:30.406 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:41:30.406 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.406 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.406 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:30.406 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.404062+0000", 2023-09-23T23:41:30.406 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6834000000000001e-05 2023-09-23T23:41:30.406 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.406 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.406 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:30.406 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.521127+0000", 2023-09-23T23:41:30.406 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.117064768 2023-09-23T23:41:30.407 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.407 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.407 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:30.407 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.521138+0000", 2023-09-23T23:41:30.407 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1440999999999999e-05 2023-09-23T23:41:30.407 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.407 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.407 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:30.407 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.521184+0000", 2023-09-23T23:41:30.407 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6068e-05 2023-09-23T23:41:30.407 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.407 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.408 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:30.408 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.521201+0000", 2023-09-23T23:41:30.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7073000000000001e-05 2023-09-23T23:41:30.408 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.408 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:41:30.408 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.524297+0000", 2023-09-23T23:41:30.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030956830000000001 2023-09-23T23:41:30.408 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:30.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:30.408 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:41:30.408 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:50.524330+0000", 2023-09-23T23:41:30.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3420000000000002e-05 2023-09-23T23:41:30.409 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.409 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.409 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.409 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:30.409 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:30.409 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:41:30.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:30.410 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:41:30.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:41:30.511 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:41:30.570 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:30.637 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T23:41:30.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:30 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:30.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:30 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:30.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:30 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:41:30.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:30 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:41:30.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:30 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1180746831' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:41:30.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:30 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:30.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:30 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3594141178' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:41:30.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:30 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:41:30.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:30 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:30.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:30 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:30.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:30 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:41:30.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:30 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:41:30.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:30 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1180746831' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:41:30.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:30 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:30.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:30 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3594141178' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:41:30.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:30 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:41:30.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:30 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:30.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:30 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:30.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:30 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:41:30.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:30 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:41:30.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:30 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1180746831' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:41:30.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:30 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:41:30.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:30 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3594141178' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:41:30.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:30 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:41:31.290 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:41:31.581 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:31 smithi110 ceph-mon[221102]: pgmap v1861: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 623 op/s; 8622/192999 objects degraded (4.467%); 1683/192999 objects misplaced (0.872%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T23:41:31.582 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:31 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:41:31.582 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:31 smithi110 ceph-mon[221102]: osdmap e653: 8 total, 8 up, 6 in 2023-09-23T23:41:31.582 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:31 smithi110 ceph-mon[221102]: failed to encode map e653 with expected crc 2023-09-23T23:41:31.582 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:31 smithi110 ceph-mon[221102]: failed to encode map e653 with expected crc 2023-09-23T23:41:31.582 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:31 smithi110 ceph-mon[221102]: failed to encode map e653 with expected crc 2023-09-23T23:41:31.827 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:41:31.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:31.840 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:31 smithi110 ceph-mon[212939]: pgmap v1861: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 623 op/s; 8622/192999 objects degraded (4.467%); 1683/192999 objects misplaced (0.872%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T23:41:31.840 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:31 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:41:31.841 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:31 smithi110 ceph-mon[212939]: osdmap e653: 8 total, 8 up, 6 in 2023-09-23T23:41:31.841 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:31 smithi110 ceph-mon[212939]: failed to encode map e653 with expected crc 2023-09-23T23:41:31.841 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:31 smithi110 ceph-mon[212939]: failed to encode map e653 with expected crc 2023-09-23T23:41:31.841 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:31 smithi110 ceph-mon[212939]: failed to encode map e653 with expected crc 2023-09-23T23:41:31.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:41:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:31 smithi138 ceph-mon[193606]: pgmap v1861: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 623 op/s; 8622/192999 objects degraded (4.467%); 1683/192999 objects misplaced (0.872%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T23:41:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:31 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:41:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:31 smithi138 ceph-mon[193606]: osdmap e653: 8 total, 8 up, 6 in 2023-09-23T23:41:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:31 smithi138 ceph-mon[193606]: failed to encode map e653 with expected crc 2023-09-23T23:41:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:31 smithi138 ceph-mon[193606]: failed to encode map e653 with expected crc 2023-09-23T23:41:31.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:31 smithi138 ceph-mon[193606]: failed to encode map e653 with expected crc 2023-09-23T23:41:31.943 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:41:31.970 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:32.018 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:41:32.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:32.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:41:32.127 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:41:32.153 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:32.202 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:41:32.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:32.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:41:32.310 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:41:32.339 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:32.385 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:41:32.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:32.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:41:32.492 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:41:32.519 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:32.824 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:32 smithi110 ceph-mon[221102]: failed to encode map e653 with expected crc 2023-09-23T23:41:32.824 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:32 smithi110 ceph-mon[221102]: failed to encode map e653 with expected crc 2023-09-23T23:41:32.824 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:32 smithi110 ceph-mon[221102]: failed to encode map e654 with expected crc 2023-09-23T23:41:32.824 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:32 smithi110 ceph-mon[221102]: failed to encode map e654 with expected crc 2023-09-23T23:41:32.825 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:32 smithi110 ceph-mon[221102]: osdmap e654: 8 total, 8 up, 6 in 2023-09-23T23:41:32.825 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:32 smithi110 ceph-mon[221102]: failed to encode map e654 with expected crc 2023-09-23T23:41:32.825 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:32 smithi110 ceph-mon[221102]: failed to encode map e654 with expected crc 2023-09-23T23:41:32.834 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:32 smithi110 ceph-mon[212939]: failed to encode map e653 with expected crc 2023-09-23T23:41:32.834 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:32 smithi110 ceph-mon[212939]: failed to encode map e653 with expected crc 2023-09-23T23:41:32.834 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:32 smithi110 ceph-mon[212939]: failed to encode map e654 with expected crc 2023-09-23T23:41:32.834 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:32 smithi110 ceph-mon[212939]: failed to encode map e654 with expected crc 2023-09-23T23:41:32.835 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:32 smithi110 ceph-mon[212939]: osdmap e654: 8 total, 8 up, 6 in 2023-09-23T23:41:32.835 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:32 smithi110 ceph-mon[212939]: failed to encode map e654 with expected crc 2023-09-23T23:41:32.835 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:32 smithi110 ceph-mon[212939]: failed to encode map e654 with expected crc 2023-09-23T23:41:32.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:32 smithi138 ceph-mon[193606]: failed to encode map e653 with expected crc 2023-09-23T23:41:32.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:32 smithi138 ceph-mon[193606]: failed to encode map e653 with expected crc 2023-09-23T23:41:32.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:32 smithi138 ceph-mon[193606]: failed to encode map e654 with expected crc 2023-09-23T23:41:32.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:32 smithi138 ceph-mon[193606]: failed to encode map e654 with expected crc 2023-09-23T23:41:32.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:32 smithi138 ceph-mon[193606]: osdmap e654: 8 total, 8 up, 6 in 2023-09-23T23:41:32.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:32 smithi138 ceph-mon[193606]: failed to encode map e654 with expected crc 2023-09-23T23:41:32.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:32 smithi138 ceph-mon[193606]: failed to encode map e654 with expected crc 2023-09-23T23:41:33.416 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:41:33.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:33.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:41:33.522 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:41:33.549 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:33.596 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:41:33.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:33.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:41:33.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:33 smithi110 ceph-mon[212939]: pgmap v1864: 119 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 505 op/s; 8622/188943 objects degraded (4.563%); 1513/188943 objects misplaced (0.801%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:41:33.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:33 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:41:33.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:33 smithi110 ceph-mon[212939]: failed to encode map e655 with expected crc 2023-09-23T23:41:33.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:33 smithi110 ceph-mon[212939]: osdmap e655: 8 total, 8 up, 6 in 2023-09-23T23:41:33.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:33 smithi110 ceph-mon[212939]: failed to encode map e655 with expected crc 2023-09-23T23:41:33.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:33 smithi110 ceph-mon[212939]: failed to encode map e655 with expected crc 2023-09-23T23:41:33.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:33 smithi110 ceph-mon[212939]: failed to encode map e655 with expected crc 2023-09-23T23:41:33.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:33 smithi110 ceph-mon[212939]: failed to encode map e655 with expected crc 2023-09-23T23:41:33.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:33 smithi110 ceph-mon[221102]: pgmap v1864: 119 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 505 op/s; 8622/188943 objects degraded (4.563%); 1513/188943 objects misplaced (0.801%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:41:33.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:33 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:41:33.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:33 smithi110 ceph-mon[221102]: failed to encode map e655 with expected crc 2023-09-23T23:41:33.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:33 smithi110 ceph-mon[221102]: osdmap e655: 8 total, 8 up, 6 in 2023-09-23T23:41:33.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:33 smithi110 ceph-mon[221102]: failed to encode map e655 with expected crc 2023-09-23T23:41:33.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:33 smithi110 ceph-mon[221102]: failed to encode map e655 with expected crc 2023-09-23T23:41:33.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:33 smithi110 ceph-mon[221102]: failed to encode map e655 with expected crc 2023-09-23T23:41:33.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:33 smithi110 ceph-mon[221102]: failed to encode map e655 with expected crc 2023-09-23T23:41:33.708 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:41:33.799 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:33.889 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T23:41:33.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:33 smithi138 ceph-mon[193606]: pgmap v1864: 119 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 505 op/s; 8622/188943 objects degraded (4.563%); 1513/188943 objects misplaced (0.801%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:41:33.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:33 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T23:41:33.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:33 smithi138 ceph-mon[193606]: failed to encode map e655 with expected crc 2023-09-23T23:41:33.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:33 smithi138 ceph-mon[193606]: osdmap e655: 8 total, 8 up, 6 in 2023-09-23T23:41:33.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:33 smithi138 ceph-mon[193606]: failed to encode map e655 with expected crc 2023-09-23T23:41:33.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:33 smithi138 ceph-mon[193606]: failed to encode map e655 with expected crc 2023-09-23T23:41:33.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:33 smithi138 ceph-mon[193606]: failed to encode map e655 with expected crc 2023-09-23T23:41:33.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:33 smithi138 ceph-mon[193606]: failed to encode map e655 with expected crc 2023-09-23T23:41:33.953 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:41:33.977 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:41:33.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:33.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:41:34.090 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:41:34.117 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:34.124 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:41:34.166 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:41:34.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:34.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:41:34.274 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:41:34.301 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:34.348 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:41:34.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:34.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:41:34.455 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:41:34.481 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:34.532 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:41:34.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:34.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:41:34.633 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:34 smithi110 ceph-mon[212939]: failed to encode map e655 with expected crc 2023-09-23T23:41:34.633 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:34 smithi110 ceph-mon[212939]: failed to encode map e655 with expected crc 2023-09-23T23:41:34.633 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:34 smithi110 ceph-mon[212939]: osdmap e656: 8 total, 8 up, 6 in 2023-09-23T23:41:34.633 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:34 smithi110 ceph-mon[212939]: failed to encode map e656 with expected crc 2023-09-23T23:41:34.633 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:34 smithi110 ceph-mon[212939]: failed to encode map e656 with expected crc 2023-09-23T23:41:34.633 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:34 smithi110 ceph-mon[212939]: failed to encode map e656 with expected crc 2023-09-23T23:41:34.634 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:34 smithi110 ceph-mon[212939]: failed to encode map e656 with expected crc 2023-09-23T23:41:34.634 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:34 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 8622/188943 objects degraded (4.563%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:34.634 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:34 smithi110 ceph-mon[221102]: failed to encode map e655 with expected crc 2023-09-23T23:41:34.634 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:34 smithi110 ceph-mon[221102]: failed to encode map e655 with expected crc 2023-09-23T23:41:34.634 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:34 smithi110 ceph-mon[221102]: osdmap e656: 8 total, 8 up, 6 in 2023-09-23T23:41:34.634 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:34 smithi110 ceph-mon[221102]: failed to encode map e656 with expected crc 2023-09-23T23:41:34.634 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:34 smithi110 ceph-mon[221102]: failed to encode map e656 with expected crc 2023-09-23T23:41:34.634 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:34 smithi110 ceph-mon[221102]: failed to encode map e656 with expected crc 2023-09-23T23:41:34.634 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:34 smithi110 ceph-mon[221102]: failed to encode map e656 with expected crc 2023-09-23T23:41:34.634 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:34 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 8622/188943 objects degraded (4.563%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:34.639 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:41:34.739 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:34.871 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:41:34.905 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:41:34.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:34.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:41:34.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:34 smithi138 ceph-mon[193606]: failed to encode map e655 with expected crc 2023-09-23T23:41:34.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:34 smithi138 ceph-mon[193606]: failed to encode map e655 with expected crc 2023-09-23T23:41:34.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:34 smithi138 ceph-mon[193606]: osdmap e656: 8 total, 8 up, 6 in 2023-09-23T23:41:34.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:34 smithi138 ceph-mon[193606]: failed to encode map e656 with expected crc 2023-09-23T23:41:34.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:34 smithi138 ceph-mon[193606]: failed to encode map e656 with expected crc 2023-09-23T23:41:34.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:34 smithi138 ceph-mon[193606]: failed to encode map e656 with expected crc 2023-09-23T23:41:34.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:34 smithi138 ceph-mon[193606]: failed to encode map e656 with expected crc 2023-09-23T23:41:34.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:34 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 8622/188943 objects degraded (4.563%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:35.025 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:41:35.146 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:35.258 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:41:35.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:35.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:41:35.371 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:41:35.444 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:35.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:35 smithi110 ceph-mon[212939]: pgmap v1867: 119 pgs: 2 activating+undersized+remapped, 3 peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 661 op/s; 8622/185058 objects degraded (4.659%); 1513/185058 objects misplaced (0.818%); 2.7 MiB/s, 41 objects/s recovering 2023-09-23T23:41:35.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:35 smithi110 ceph-mon[212939]: osdmap e657: 8 total, 8 up, 6 in 2023-09-23T23:41:35.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:35 smithi110 ceph-mon[212939]: failed to encode map e657 with expected crc 2023-09-23T23:41:35.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:35 smithi110 ceph-mon[212939]: failed to encode map e657 with expected crc 2023-09-23T23:41:35.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:35 smithi110 ceph-mon[221102]: pgmap v1867: 119 pgs: 2 activating+undersized+remapped, 3 peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 661 op/s; 8622/185058 objects degraded (4.659%); 1513/185058 objects misplaced (0.818%); 2.7 MiB/s, 41 objects/s recovering 2023-09-23T23:41:35.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:35 smithi110 ceph-mon[221102]: osdmap e657: 8 total, 8 up, 6 in 2023-09-23T23:41:35.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:35 smithi110 ceph-mon[221102]: failed to encode map e657 with expected crc 2023-09-23T23:41:35.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:35 smithi110 ceph-mon[221102]: failed to encode map e657 with expected crc 2023-09-23T23:41:35.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:35 smithi138 ceph-mon[193606]: pgmap v1867: 119 pgs: 2 activating+undersized+remapped, 3 peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 661 op/s; 8622/185058 objects degraded (4.659%); 1513/185058 objects misplaced (0.818%); 2.7 MiB/s, 41 objects/s recovering 2023-09-23T23:41:35.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:35 smithi138 ceph-mon[193606]: osdmap e657: 8 total, 8 up, 6 in 2023-09-23T23:41:35.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:35 smithi138 ceph-mon[193606]: failed to encode map e657 with expected crc 2023-09-23T23:41:35.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:35 smithi138 ceph-mon[193606]: failed to encode map e657 with expected crc 2023-09-23T23:41:36.483 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:41:36.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:36.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:41:36.563 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:41:36.604 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:41:36.625 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:41:36.631 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:36.676 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:41:36.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:36.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:41:36.785 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:41:36.802 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:36 smithi110 ceph-mon[221102]: failed to encode map e657 with expected crc 2023-09-23T23:41:36.802 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:36 smithi110 ceph-mon[221102]: failed to encode map e657 with expected crc 2023-09-23T23:41:36.808 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:36 smithi110 ceph-mon[212939]: failed to encode map e657 with expected crc 2023-09-23T23:41:36.809 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:36 smithi110 ceph-mon[212939]: failed to encode map e657 with expected crc 2023-09-23T23:41:36.867 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:36 smithi138 ceph-mon[193606]: failed to encode map e657 with expected crc 2023-09-23T23:41:36.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:36 smithi138 ceph-mon[193606]: failed to encode map e657 with expected crc 2023-09-23T23:41:37.053 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:41:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:41:36] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.43.0" 2023-09-23T23:41:37.094 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:41:37.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:37.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:41:37.204 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:41:37.279 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:41:37.308 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:37.673 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:37 smithi110 ceph-mon[221102]: pgmap v1869: 119 pgs: 2 activating+undersized+remapped, 3 peering, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 602 op/s; 2514/180837 objects degraded (1.390%); 2517/180837 objects misplaced (1.392%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T23:41:37.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:37 smithi138 ceph-mon[193606]: pgmap v1869: 119 pgs: 2 activating+undersized+remapped, 3 peering, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 602 op/s; 2514/180837 objects degraded (1.390%); 2517/180837 objects misplaced (1.392%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T23:41:37.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:37 smithi110 ceph-mon[212939]: pgmap v1869: 119 pgs: 2 activating+undersized+remapped, 3 peering, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 602 op/s; 2514/180837 objects degraded (1.390%); 2517/180837 objects misplaced (1.392%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T23:41:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:38 smithi138 ceph-mon[193606]: pgmap v1870: 119 pgs: 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 2514/176286 objects degraded (1.426%); 2517/176286 objects misplaced (1.428%); 4.5 MiB/s, 74 objects/s recovering 2023-09-23T23:41:39.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:38 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T23:41:39.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:38 smithi110 ceph-mon[212939]: pgmap v1870: 119 pgs: 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 2514/176286 objects degraded (1.426%); 2517/176286 objects misplaced (1.428%); 4.5 MiB/s, 74 objects/s recovering 2023-09-23T23:41:39.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:38 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T23:41:39.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:38 smithi110 ceph-mon[221102]: pgmap v1870: 119 pgs: 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 2514/176286 objects degraded (1.426%); 2517/176286 objects misplaced (1.428%); 4.5 MiB/s, 74 objects/s recovering 2023-09-23T23:41:39.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:38 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T23:41:39.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:39 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2514/176286 objects degraded (1.426%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:39.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:39 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2514/176286 objects degraded (1.426%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:40.126 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:41:40.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:40.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:41:40.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:39 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2514/176286 objects degraded (1.426%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:40.237 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:41:40.265 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:40.316 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:41:40.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:40.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:41:40.423 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:41:40.518 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:40.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:40 smithi110 ceph-mon[221102]: pgmap v1871: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 592 op/s; 2514/175686 objects degraded (1.431%); 2517/175686 objects misplaced (1.433%); 3.6 MiB/s, 59 objects/s recovering 2023-09-23T23:41:40.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:40 smithi110 ceph-mon[212939]: pgmap v1871: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 592 op/s; 2514/175686 objects degraded (1.431%); 2517/175686 objects misplaced (1.433%); 3.6 MiB/s, 59 objects/s recovering 2023-09-23T23:41:41.032 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:41:41.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:41.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:41:41.145 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:41:41.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:40 smithi138 ceph-mon[193606]: pgmap v1871: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 592 op/s; 2514/175686 objects degraded (1.431%); 2517/175686 objects misplaced (1.433%); 3.6 MiB/s, 59 objects/s recovering 2023-09-23T23:41:41.271 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:41.622 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:41:41.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:41.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:41:41.728 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:41:41.755 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:41.807 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:41:41.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:41.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:41:41.812 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:41:41.912 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:41:41.939 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:41.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:41 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2480233603' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:41:41.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:41 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:41:41.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:41 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2480233603' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:41:41.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:41 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:41:41.986 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:41:41.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:41.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:41:42.093 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:41:42.119 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:41 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2480233603' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:41:42.119 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:41 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:41:42.121 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:42.171 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:41:42.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:42.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:41:42.279 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:41:42.308 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:42.353 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:41:42.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:42.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:41:42.459 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:41:42.485 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:42.526 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:41:42.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:42.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:41:42.631 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:41:42.661 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:42.958 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:42 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:41:42.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:42 smithi110 ceph-mon[212939]: failed to encode map e658 with expected crc 2023-09-23T23:41:42.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:42 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:41:42.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:42 smithi110 ceph-mon[212939]: osdmap e658: 8 total, 8 up, 6 in 2023-09-23T23:41:42.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:42 smithi110 ceph-mon[212939]: failed to encode map e658 with expected crc 2023-09-23T23:41:42.959 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:42 smithi110 ceph-mon[212939]: pgmap v1873: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 566 op/s; 2514/171453 objects degraded (1.466%); 1940/171453 objects misplaced (1.132%); 7.9 MiB/s, 127 objects/s recovering 2023-09-23T23:41:42.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:42 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:41:42.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:42 smithi110 ceph-mon[221102]: failed to encode map e658 with expected crc 2023-09-23T23:41:42.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:42 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:41:42.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:42 smithi110 ceph-mon[221102]: osdmap e658: 8 total, 8 up, 6 in 2023-09-23T23:41:42.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:42 smithi110 ceph-mon[221102]: failed to encode map e658 with expected crc 2023-09-23T23:41:42.960 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:42 smithi110 ceph-mon[221102]: pgmap v1873: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 566 op/s; 2514/171453 objects degraded (1.466%); 1940/171453 objects misplaced (1.132%); 7.9 MiB/s, 127 objects/s recovering 2023-09-23T23:41:43.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:42 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:41:43.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:42 smithi138 ceph-mon[193606]: failed to encode map e658 with expected crc 2023-09-23T23:41:43.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:42 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:41:43.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:42 smithi138 ceph-mon[193606]: osdmap e658: 8 total, 8 up, 6 in 2023-09-23T23:41:43.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:42 smithi138 ceph-mon[193606]: failed to encode map e658 with expected crc 2023-09-23T23:41:43.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:42 smithi138 ceph-mon[193606]: pgmap v1873: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 566 op/s; 2514/171453 objects degraded (1.466%); 1940/171453 objects misplaced (1.132%); 7.9 MiB/s, 127 objects/s recovering 2023-09-23T23:41:43.663 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:41:43.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:43.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:41:43.770 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:41:43.800 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:43.853 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:41:43.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:43.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:41:43.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:41:43.957 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:41:43.961 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:41:44.032 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:44.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:41:44.626 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:41:44.626 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:41:44.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.626 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:114155 14.6 e658/617 14:67c57852:::benchmark_data_smithi110_640334_object30276:head v 658'7286, mlcod=658'7283)", 2023-09-23T23:41:44.626 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:44.478184+0000", 2023-09-23T23:41:44.626 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.022820363, 2023-09-23T23:41:44.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022854192999999998, 2023-09-23T23:41:44.626 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:44.626 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:41:44.626 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:44.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:44.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:44.478184+0000", 2023-09-23T23:41:44.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:44.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:44.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:44.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:44.478184+0000", 2023-09-23T23:41:44.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:44.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:44.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:44.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:44.478183+0000", 2023-09-23T23:41:44.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-09-23T23:41:44.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:44.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:44.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:44.478187+0000", 2023-09-23T23:41:44.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.343e-06 2023-09-23T23:41:44.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:44.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:44.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:44.478192+0000", 2023-09-23T23:41:44.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.242e-06 2023-09-23T23:41:44.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:44.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:44.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:44.478196+0000", 2023-09-23T23:41:44.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7419999999999999e-06 2023-09-23T23:41:44.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:44.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:44.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:44.490631+0000", 2023-09-23T23:41:44.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012435636 2023-09-23T23:41:44.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:44.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:44.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:44.490649+0000", 2023-09-23T23:41:44.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7442e-05 2023-09-23T23:41:44.631 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:44.631 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:44.632 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:44.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:44.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.632 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:114162 14.6 e658/617 14:6f5e0e94:::benchmark_data_smithi110_640334_object30283:head v 658'7287, mlcod=658'7283)", 2023-09-23T23:41:44.632 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:44.487059+0000", 2023-09-23T23:41:44.632 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.013945459, 2023-09-23T23:41:44.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014048211, 2023-09-23T23:41:44.632 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:41:44.632 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:41:44.632 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:41:44.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:41:44.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:44.487059+0000", 2023-09-23T23:41:44.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:44.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:44.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:41:44.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:44.487059+0000", 2023-09-23T23:41:44.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:41:44.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:44.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:41:44.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:44.487058+0000", 2023-09-23T23:41:44.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:41:44.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:44.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:41:44.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:44.487065+0000", 2023-09-23T23:41:44.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6199999999999999e-06 2023-09-23T23:41:44.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:44.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:41:44.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:44.487080+0000", 2023-09-23T23:41:44.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4285e-05 2023-09-23T23:41:44.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:44.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:41:44.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:44.487088+0000", 2023-09-23T23:41:44.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4449999999999996e-06 2023-09-23T23:41:44.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:44.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:41:44.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:44.493687+0000", 2023-09-23T23:41:44.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0065991770000000003 2023-09-23T23:41:44.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:41:44.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:41:44.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:41:44.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:44.493706+0000", 2023-09-23T23:41:44.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8746999999999999e-05 2023-09-23T23:41:44.636 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:44.636 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:41:44.636 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:44.636 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:41:44.636 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-09-23T23:41:44.636 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 2 2023-09-23T23:41:44.636 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:41:44.687 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:41:44.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:44.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:41:44.795 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:41:44.881 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:44 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2514/171453 objects degraded (1.466%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:44.882 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:44 smithi110 ceph-mon[212939]: pgmap v1874: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.4 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 651 op/s; 2514/166626 objects degraded (1.509%); 1940/166626 objects misplaced (1.164%); 6.7 MiB/s, 108 objects/s recovering 2023-09-23T23:41:44.882 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:44 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/776712443' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:41:44.882 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:44 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2514/171453 objects degraded (1.466%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:44.882 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:44 smithi110 ceph-mon[221102]: pgmap v1874: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.4 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 651 op/s; 2514/166626 objects degraded (1.509%); 1940/166626 objects misplaced (1.164%); 6.7 MiB/s, 108 objects/s recovering 2023-09-23T23:41:44.882 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:44 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/776712443' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:41:44.888 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:44.949 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T23:41:45.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:44 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2514/171453 objects degraded (1.466%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:45.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:44 smithi138 ceph-mon[193606]: pgmap v1874: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.4 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 651 op/s; 2514/166626 objects degraded (1.509%); 1940/166626 objects misplaced (1.164%); 6.7 MiB/s, 108 objects/s recovering 2023-09-23T23:41:45.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:44 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/776712443' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:41:45.307 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:41:45.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:45.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:41:45.413 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:41:45.441 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:45.491 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:41:45.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:45.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:41:45.602 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:41:45.670 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:45.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:45 smithi110 ceph-mon[221102]: failed to encode map e659 with expected crc 2023-09-23T23:41:45.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:45 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/776712443' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:41:45.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:45 smithi110 ceph-mon[221102]: osdmap e659: 8 total, 8 up, 6 in 2023-09-23T23:41:45.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:45 smithi110 ceph-mon[221102]: failed to encode map e659 with expected crc 2023-09-23T23:41:45.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:45 smithi110 ceph-mon[221102]: failed to encode map e659 with expected crc 2023-09-23T23:41:45.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:45 smithi110 ceph-mon[212939]: failed to encode map e659 with expected crc 2023-09-23T23:41:45.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:45 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/776712443' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:41:45.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:45 smithi110 ceph-mon[212939]: osdmap e659: 8 total, 8 up, 6 in 2023-09-23T23:41:45.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:45 smithi110 ceph-mon[212939]: failed to encode map e659 with expected crc 2023-09-23T23:41:45.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:45 smithi110 ceph-mon[212939]: failed to encode map e659 with expected crc 2023-09-23T23:41:46.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:45 smithi138 ceph-mon[193606]: failed to encode map e659 with expected crc 2023-09-23T23:41:46.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:45 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/776712443' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:41:46.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:45 smithi138 ceph-mon[193606]: osdmap e659: 8 total, 8 up, 6 in 2023-09-23T23:41:46.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:45 smithi138 ceph-mon[193606]: failed to encode map e659 with expected crc 2023-09-23T23:41:46.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:45 smithi138 ceph-mon[193606]: failed to encode map e659 with expected crc 2023-09-23T23:41:46.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:46.263 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:41:46.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:41:46.364 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:41:46.394 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:46.442 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:41:46.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:46.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:41:46.551 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:41:46.577 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:46.621 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:41:46.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:46.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:41:46.726 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:41:46.754 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:46.806 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:41:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:41:46] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.43.0" 2023-09-23T23:41:47.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:46 smithi138 ceph-mon[193606]: pgmap v1876: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 609 op/s; 2529/161646 objects degraded (1.565%); 1384/161646 objects misplaced (0.856%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T23:41:47.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:46 smithi110 ceph-mon[212939]: pgmap v1876: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 609 op/s; 2529/161646 objects degraded (1.565%); 1384/161646 objects misplaced (0.856%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T23:41:47.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:46 smithi110 ceph-mon[221102]: pgmap v1876: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 609 op/s; 2529/161646 objects degraded (1.565%); 1384/161646 objects misplaced (0.856%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T23:41:47.257 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:41:47.586 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:41:47.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:47.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:41:47.703 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:41:47.730 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:47.746 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T23:41:47.777 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:41:47.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:47.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:41:47.801 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:41:47.883 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:41:48.050 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:48.110 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:41:48.151 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:41:48.164 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:41:48.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:48 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/488501681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:41:48.205 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:48 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/488501681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:41:48.205 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:41:48.209 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:41:48.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:48.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:41:48.254 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:41:48.295 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:41:48.327 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:41:48.352 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:48 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/488501681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:41:48.355 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:48.360 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:41:48.404 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:41:48.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:48.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:41:48.412 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:41:48.457 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:41:48.459 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:41:48.459 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:41:48.459 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:41:48.459 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:41:48.460 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:41:48.460 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:41:48.460 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:41:48.460 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:41:48.512 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:41:48.539 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:48.591 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:41:48.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:48.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:41:48.697 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:41:48.723 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:48.770 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:41:48.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:48.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:41:48.878 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:41:48.907 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:48.955 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:41:48.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:48.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:41:49.063 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:41:49.090 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:49.136 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:41:49.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:49.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:41:49.165 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:49 smithi110 ceph-mon[212939]: pgmap v1877: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 795 op/s; 2529/156573 objects degraded (1.615%); 1384/156573 objects misplaced (0.884%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T23:41:49.165 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:49 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2529/156573 objects degraded (1.615%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:49.165 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:49 smithi110 ceph-mon[221102]: pgmap v1877: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 795 op/s; 2529/156573 objects degraded (1.615%); 1384/156573 objects misplaced (0.884%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T23:41:49.165 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:49 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2529/156573 objects degraded (1.615%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:49.242 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:41:49.270 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:49.374 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:41:49.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:49.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:41:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:49 smithi138 ceph-mon[193606]: pgmap v1877: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 795 op/s; 2529/156573 objects degraded (1.615%); 1384/156573 objects misplaced (0.884%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T23:41:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:49 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2529/156573 objects degraded (1.615%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:49.541 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:41:49.570 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:49.617 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:41:49.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:49.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:41:49.724 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:41:49.753 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:49.795 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:41:49.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:49.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:41:49.902 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:41:49.956 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:50.569 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:41:50.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:50.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:41:50.679 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:41:50.707 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:50.755 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:41:50.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:50.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:41:50.863 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:41:50.930 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:51.198 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:51 smithi110 ceph-mon[212939]: pgmap v1878: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 622 op/s; 2529/155922 objects degraded (1.622%); 1384/155922 objects misplaced (0.888%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:41:51.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:51 smithi110 ceph-mon[221102]: pgmap v1878: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 622 op/s; 2529/155922 objects degraded (1.622%); 1384/155922 objects misplaced (0.888%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:41:51.589 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:41:51.613 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:41:51.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:51.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:41:51.676 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:41:51.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:51 smithi138 ceph-mon[193606]: pgmap v1878: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 622 op/s; 2529/155922 objects degraded (1.622%); 1384/155922 objects misplaced (0.888%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:41:51.723 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:41:51.797 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:52.224 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:41:52.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:52.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:41:52.337 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:41:52.443 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:53.494 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:41:53.519 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:53 smithi110 ceph-mon[212939]: pgmap v1879: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 2538/151671 objects degraded (1.673%); 762/151671 objects misplaced (0.502%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:41:53.520 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:53 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3432325817' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:41:53.520 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:53 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:41:53.520 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:53 smithi110 ceph-mon[221102]: pgmap v1879: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 2538/151671 objects degraded (1.673%); 762/151671 objects misplaced (0.502%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:41:53.520 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:53 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3432325817' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:41:53.520 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:53 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:41:53.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:53 smithi138 ceph-mon[193606]: pgmap v1879: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 2538/151671 objects degraded (1.673%); 762/151671 objects misplaced (0.502%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:41:53.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:53 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3432325817' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:41:53.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:53 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:41:54.354 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:41:54.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:54.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:41:54.471 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:41:54.543 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:54 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:41:54.543 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:54 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:41:54.543 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:54 smithi110 ceph-mon[221102]: osdmap e660: 8 total, 8 up, 6 in 2023-09-23T23:41:54.543 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:54 smithi110 ceph-mon[221102]: failed to encode map e660 with expected crc 2023-09-23T23:41:54.543 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:54 smithi110 ceph-mon[221102]: failed to encode map e660 with expected crc 2023-09-23T23:41:54.543 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:54 smithi110 ceph-mon[221102]: failed to encode map e660 with expected crc 2023-09-23T23:41:54.543 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:54 smithi110 ceph-mon[221102]: failed to encode map e660 with expected crc 2023-09-23T23:41:54.544 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:54 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2538/151671 objects degraded (1.673%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:54.544 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:54 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:41:54.544 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:54 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:41:54.545 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:54 smithi110 ceph-mon[212939]: osdmap e660: 8 total, 8 up, 6 in 2023-09-23T23:41:54.545 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:54 smithi110 ceph-mon[212939]: failed to encode map e660 with expected crc 2023-09-23T23:41:54.545 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:54 smithi110 ceph-mon[212939]: failed to encode map e660 with expected crc 2023-09-23T23:41:54.545 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:54 smithi110 ceph-mon[212939]: failed to encode map e660 with expected crc 2023-09-23T23:41:54.545 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:54 smithi110 ceph-mon[212939]: failed to encode map e660 with expected crc 2023-09-23T23:41:54.545 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:54 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2538/151671 objects degraded (1.673%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:54.556 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:54.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:54 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:41:54.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:54 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:41:54.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:54 smithi138 ceph-mon[193606]: osdmap e660: 8 total, 8 up, 6 in 2023-09-23T23:41:54.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:54 smithi138 ceph-mon[193606]: failed to encode map e660 with expected crc 2023-09-23T23:41:54.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:54 smithi138 ceph-mon[193606]: failed to encode map e660 with expected crc 2023-09-23T23:41:54.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:54 smithi138 ceph-mon[193606]: failed to encode map e660 with expected crc 2023-09-23T23:41:54.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:54 smithi138 ceph-mon[193606]: failed to encode map e660 with expected crc 2023-09-23T23:41:54.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:54 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2538/151671 objects degraded (1.673%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:54.788 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:41:55.525 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:41:55.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:55.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:41:55.535 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:55 smithi110 ceph-mon[212939]: pgmap v1881: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 699 op/s; 2538/146763 objects degraded (1.729%); 762/146763 objects misplaced (0.519%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T23:41:55.535 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:55 smithi110 ceph-mon[221102]: pgmap v1881: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 699 op/s; 2538/146763 objects degraded (1.729%); 762/146763 objects misplaced (0.519%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T23:41:55.632 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:41:55.658 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:55 smithi138 ceph-mon[193606]: pgmap v1881: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 699 op/s; 2538/146763 objects degraded (1.729%); 762/146763 objects misplaced (0.519%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T23:41:55.661 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:55.712 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:41:55.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:55.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:41:55.821 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:41:55.912 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:56.611 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:41:56.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:56.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:41:56.722 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:41:56.749 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:56.797 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:41:56.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:56.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:41:56.904 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:41:56.931 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:56.953 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:41:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:41:56] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.43.0" 2023-09-23T23:41:56.977 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:41:56.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:56.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:41:57.082 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:41:57.155 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:57.263 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:57 smithi110 ceph-mon[212939]: pgmap v1882: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 2.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 660 op/s; 2059/141807 objects degraded (1.452%); 499/141807 objects misplaced (0.352%); 5.3 MiB/s, 135 objects/s recovering 2023-09-23T23:41:57.264 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:57 smithi110 ceph-mon[221102]: pgmap v1882: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 2.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 660 op/s; 2059/141807 objects degraded (1.452%); 499/141807 objects misplaced (0.352%); 5.3 MiB/s, 135 objects/s recovering 2023-09-23T23:41:57.345 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:41:57.364 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:41:57.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:57.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:41:57.670 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:41:57.670 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:41:57.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:57 smithi138 ceph-mon[193606]: pgmap v1882: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 2.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 660 op/s; 2059/141807 objects degraded (1.452%); 499/141807 objects misplaced (0.352%); 5.3 MiB/s, 135 objects/s recovering 2023-09-23T23:41:57.770 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:41:57.798 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:57.845 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:41:57.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:57.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:41:57.953 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:41:57.982 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:58.035 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:41:58.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:58.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:41:58.141 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:41:58.209 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:58.291 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T23:41:58.341 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:41:58.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:41:58.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:41:58.453 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:41:58.528 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:58 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1771786764' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:41:58.528 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:58 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:41:58.528 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:41:58.528 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:58 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1771786764' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:41:58.529 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:58 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:41:58.529 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:41:58.529 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:41:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:58 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1771786764' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:41:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:58 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:41:58.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:41:59.123 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:41:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:59 smithi110 ceph-mon[212939]: pgmap v1883: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 2.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 2059/136911 objects degraded (1.504%); 499/136911 objects misplaced (0.364%); 5.3 MiB/s, 135 objects/s recovering 2023-09-23T23:41:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:59 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:41:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:59 smithi110 ceph-mon[212939]: osdmap e661: 8 total, 8 up, 6 in 2023-09-23T23:41:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:59 smithi110 ceph-mon[212939]: failed to encode map e661 with expected crc 2023-09-23T23:41:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:59 smithi110 ceph-mon[212939]: failed to encode map e661 with expected crc 2023-09-23T23:41:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:59 smithi110 ceph-mon[212939]: failed to encode map e661 with expected crc 2023-09-23T23:41:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:41:59 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 2059/136911 objects degraded (1.504%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:59.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:59 smithi110 ceph-mon[221102]: pgmap v1883: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 2.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 2059/136911 objects degraded (1.504%); 499/136911 objects misplaced (0.364%); 5.3 MiB/s, 135 objects/s recovering 2023-09-23T23:41:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:59 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:41:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:59 smithi110 ceph-mon[221102]: osdmap e661: 8 total, 8 up, 6 in 2023-09-23T23:41:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:59 smithi110 ceph-mon[221102]: failed to encode map e661 with expected crc 2023-09-23T23:41:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:59 smithi110 ceph-mon[221102]: failed to encode map e661 with expected crc 2023-09-23T23:41:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:59 smithi110 ceph-mon[221102]: failed to encode map e661 with expected crc 2023-09-23T23:41:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:41:59 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 2059/136911 objects degraded (1.504%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:41:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:59 smithi138 ceph-mon[193606]: pgmap v1883: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 2.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 2059/136911 objects degraded (1.504%); 499/136911 objects misplaced (0.364%); 5.3 MiB/s, 135 objects/s recovering 2023-09-23T23:41:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:59 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:41:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:59 smithi138 ceph-mon[193606]: osdmap e661: 8 total, 8 up, 6 in 2023-09-23T23:41:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:59 smithi138 ceph-mon[193606]: failed to encode map e661 with expected crc 2023-09-23T23:41:59.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:59 smithi138 ceph-mon[193606]: failed to encode map e661 with expected crc 2023-09-23T23:41:59.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:59 smithi138 ceph-mon[193606]: failed to encode map e661 with expected crc 2023-09-23T23:41:59.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:41:59 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 2059/136911 objects degraded (1.504%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T23:42:00.060 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:42:00.078 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:00.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:00.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:00.189 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:00.268 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:00.848 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T23:42:01.028 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:01.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:01.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:01.132 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:01.159 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:01.200 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:01.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:01.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:01.209 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:01.306 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:01.333 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:01.335 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:01.376 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:01.385 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:01.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:01.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:01 smithi138 ceph-mon[193606]: pgmap v1885: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 2.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 2059/136185 objects degraded (1.512%); 499/136185 objects misplaced (0.366%); 1.8 MiB/s, 92 objects/s recovering 2023-09-23T23:42:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:01 smithi138 ceph-mon[193606]: failed to encode map e662 with expected crc 2023-09-23T23:42:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:01 smithi138 ceph-mon[193606]: osdmap e662: 8 total, 8 up, 6 in 2023-09-23T23:42:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:01 smithi138 ceph-mon[193606]: failed to encode map e662 with expected crc 2023-09-23T23:42:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:01 smithi138 ceph-mon[193606]: failed to encode map e662 with expected crc 2023-09-23T23:42:01.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:01 smithi138 ceph-mon[193606]: failed to encode map e662 with expected crc 2023-09-23T23:42:01.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:01 smithi138 ceph-mon[193606]: failed to encode map e662 with expected crc 2023-09-23T23:42:01.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:01 smithi110 ceph-mon[221102]: pgmap v1885: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 2.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 2059/136185 objects degraded (1.512%); 499/136185 objects misplaced (0.366%); 1.8 MiB/s, 92 objects/s recovering 2023-09-23T23:42:01.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:01 smithi110 ceph-mon[221102]: failed to encode map e662 with expected crc 2023-09-23T23:42:01.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:01 smithi110 ceph-mon[221102]: osdmap e662: 8 total, 8 up, 6 in 2023-09-23T23:42:01.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:01 smithi110 ceph-mon[221102]: failed to encode map e662 with expected crc 2023-09-23T23:42:01.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:01 smithi110 ceph-mon[221102]: failed to encode map e662 with expected crc 2023-09-23T23:42:01.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:01 smithi110 ceph-mon[221102]: failed to encode map e662 with expected crc 2023-09-23T23:42:01.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:01 smithi110 ceph-mon[221102]: failed to encode map e662 with expected crc 2023-09-23T23:42:01.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:01 smithi110 ceph-mon[212939]: pgmap v1885: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 116 active+clean; 2.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 2059/136185 objects degraded (1.512%); 499/136185 objects misplaced (0.366%); 1.8 MiB/s, 92 objects/s recovering 2023-09-23T23:42:01.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:01 smithi110 ceph-mon[212939]: failed to encode map e662 with expected crc 2023-09-23T23:42:01.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:01 smithi110 ceph-mon[212939]: osdmap e662: 8 total, 8 up, 6 in 2023-09-23T23:42:01.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:01 smithi110 ceph-mon[212939]: failed to encode map e662 with expected crc 2023-09-23T23:42:01.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:01 smithi110 ceph-mon[212939]: failed to encode map e662 with expected crc 2023-09-23T23:42:01.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:01 smithi110 ceph-mon[212939]: failed to encode map e662 with expected crc 2023-09-23T23:42:01.455 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:01 smithi110 ceph-mon[212939]: failed to encode map e662 with expected crc 2023-09-23T23:42:01.493 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:01.560 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:01.700 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:01.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:01.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:01.805 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:01.832 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:01.879 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:01.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:01.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:01.986 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:42:02.012 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:02.062 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:42:02.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:02.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:42:02.169 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:02.170 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:42:02.208 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:42:02.252 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:42:02.256 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:02.313 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:42:02.352 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:42:02.398 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:42:02.432 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:42:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:02 smithi110 ceph-mon[212939]: failed to encode map e663 with expected crc 2023-09-23T23:42:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:02 smithi110 ceph-mon[212939]: osdmap e663: 8 total, 8 up, 6 in 2023-09-23T23:42:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:02 smithi110 ceph-mon[212939]: failed to encode map e663 with expected crc 2023-09-23T23:42:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:02 smithi110 ceph-mon[212939]: failed to encode map e663 with expected crc 2023-09-23T23:42:02.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:02 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3479723269' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:02.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:02 smithi110 ceph-mon[221102]: failed to encode map e663 with expected crc 2023-09-23T23:42:02.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:02 smithi110 ceph-mon[221102]: osdmap e663: 8 total, 8 up, 6 in 2023-09-23T23:42:02.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:02 smithi110 ceph-mon[221102]: failed to encode map e663 with expected crc 2023-09-23T23:42:02.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:02 smithi110 ceph-mon[221102]: failed to encode map e663 with expected crc 2023-09-23T23:42:02.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:02 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3479723269' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:02.469 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:42:02.500 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:42:02.501 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:42:02.501 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:42:02.501 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:42:02.501 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:42:02.501 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:42:02.501 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:42:02.501 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:42:02.635 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:02.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:02.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:02.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:02 smithi138 ceph-mon[193606]: failed to encode map e663 with expected crc 2023-09-23T23:42:02.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:02 smithi138 ceph-mon[193606]: osdmap e663: 8 total, 8 up, 6 in 2023-09-23T23:42:02.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:02 smithi138 ceph-mon[193606]: failed to encode map e663 with expected crc 2023-09-23T23:42:02.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:02 smithi138 ceph-mon[193606]: failed to encode map e663 with expected crc 2023-09-23T23:42:02.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:02 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3479723269' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:02.746 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:02.773 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:02.826 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:02.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:02.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:02.932 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:42:02.999 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:03.333 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:03 smithi110 ceph-mon[221102]: pgmap v1888: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 568 op/s; 1545/131574 objects degraded (1.174%); 284/131574 objects misplaced (0.216%); 2.1 MiB/s, 121 objects/s recovering 2023-09-23T23:42:03.333 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:03 smithi110 ceph-mon[212939]: pgmap v1888: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 568 op/s; 1545/131574 objects degraded (1.174%); 284/131574 objects misplaced (0.216%); 2.1 MiB/s, 121 objects/s recovering 2023-09-23T23:42:03.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:03 smithi138 ceph-mon[193606]: pgmap v1888: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 568 op/s; 1545/131574 objects degraded (1.174%); 284/131574 objects misplaced (0.216%); 2.1 MiB/s, 121 objects/s recovering 2023-09-23T23:42:04.607 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:04 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1545/131574 objects degraded (1.174%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:42:04.615 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:04 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1545/131574 objects degraded (1.174%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:42:04.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:04 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1545/131574 objects degraded (1.174%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:42:05.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:05 smithi110 ceph-mon[212939]: pgmap v1889: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 552 op/s; 1545/126966 objects degraded (1.217%); 284/126966 objects misplaced (0.224%); 2.1 MiB/s, 121 objects/s recovering 2023-09-23T23:42:05.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:05 smithi110 ceph-mon[221102]: pgmap v1889: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 552 op/s; 1545/126966 objects degraded (1.217%); 284/126966 objects misplaced (0.224%); 2.1 MiB/s, 121 objects/s recovering 2023-09-23T23:42:05.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:05 smithi138 ceph-mon[193606]: pgmap v1889: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 552 op/s; 1545/126966 objects degraded (1.217%); 284/126966 objects misplaced (0.224%); 2.1 MiB/s, 121 objects/s recovering 2023-09-23T23:42:05.701 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:42:05.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:05.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:42:05.809 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:05.845 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:06.335 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:06.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:06.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:06.452 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:42:06.479 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:06.524 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:42:06.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:06.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:42:06.631 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:42:06.658 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:06.708 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:42:06.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:06.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:42:06.815 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:42:06.842 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:06.888 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:42:06.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:06.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:42:06.953 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:42:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:42:06] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.43.0" 2023-09-23T23:42:06.993 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:42:07.054 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:42:07.055 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:42:07.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:42:07.055 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:42:07.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.055 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71508 13.d 13:b7936a93:::benchmark_data_smithi110_541928_object71507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:42:07.055 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.552274+0000", 2023-09-23T23:42:07.055 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 496.48763971199998, 2023-09-23T23:42:07.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1860481599999999, 2023-09-23T23:42:07.055 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.055 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:42:07.056 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:07.056 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:07.056 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:07.056 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71508 2023-09-23T23:42:07.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.056 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.552274+0000", 2023-09-23T23:42:07.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.552274+0000", 2023-09-23T23:42:07.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.552273+0000", 2023-09-23T23:42:07.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:42:07.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.552302+0000", 2023-09-23T23:42:07.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9275999999999999e-05 2023-09-23T23:42:07.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.552314+0000", 2023-09-23T23:42:07.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1657999999999999e-05 2023-09-23T23:42:07.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.552322+0000", 2023-09-23T23:42:07.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8289999999999997e-06 2023-09-23T23:42:07.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.556287+0000", 2023-09-23T23:42:07.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039647220000000004 2023-09-23T23:42:07.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:07.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.556297+0000", 2023-09-23T23:42:07.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8099999999999992e-06 2023-09-23T23:42:07.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.537459+0000", 2023-09-23T23:42:07.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98116200399999998 2023-09-23T23:42:07.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:07.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.537473+0000", 2023-09-23T23:42:07.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4430000000000001e-05 2023-09-23T23:42:07.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.737038+0000", 2023-09-23T23:42:07.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1995647469999999 2023-09-23T23:42:07.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.061 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.061 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.738322+0000", 2023-09-23T23:42:07.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012843780000000001 2023-09-23T23:42:07.061 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.061 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.061 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.061 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:7674 14.8 e621/617 14:166d5947:::benchmark_data_smithi110_640334_object7673:head v 621'472, mlcod=621'470)", 2023-09-23T23:42:07.061 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:25.621719+0000", 2023-09-23T23:42:07.061 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 161.418195516, 2023-09-23T23:42:07.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.122259012, 2023-09-23T23:42:07.062 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.062 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.062 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.621719+0000", 2023-09-23T23:42:07.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.621719+0000", 2023-09-23T23:42:07.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.621717+0000", 2023-09-23T23:42:07.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:42:07.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.621746+0000", 2023-09-23T23:42:07.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9260000000000001e-05 2023-09-23T23:42:07.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.621760+0000", 2023-09-23T23:42:07.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3935e-05 2023-09-23T23:42:07.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.621768+0000", 2023-09-23T23:42:07.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2900000000000002e-06 2023-09-23T23:42:07.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.621824+0000", 2023-09-23T23:42:07.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5594999999999999e-05 2023-09-23T23:42:07.065 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.065 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.065 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.065 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.621840+0000", 2023-09-23T23:42:07.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5875999999999998e-05 2023-09-23T23:42:07.065 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.065 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.065 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.065 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743927+0000", 2023-09-23T23:42:07.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.122086954 2023-09-23T23:42:07.065 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.065 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.066 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.066 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743978+0000", 2023-09-23T23:42:07.066 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0899999999999997e-05 2023-09-23T23:42:07.066 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.066 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.066 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.066 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.066 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.066 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:7678 14.1 e621/617 14:8d451f58:::benchmark_data_smithi110_640334_object7677:head v 621'503, mlcod=621'499)", 2023-09-23T23:42:07.066 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:25.622495+0000", 2023-09-23T23:42:07.066 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 161.41741858699999, 2023-09-23T23:42:07.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.121498043, 2023-09-23T23:42:07.067 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.067 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.067 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.067 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.067 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.067 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622495+0000", 2023-09-23T23:42:07.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.067 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.067 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.067 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.067 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622495+0000", 2023-09-23T23:42:07.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.068 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.068 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.068 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.068 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622495+0000", 2023-09-23T23:42:07.068 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:42:07.068 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.068 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.068 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.068 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622530+0000", 2023-09-23T23:42:07.068 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5311000000000001e-05 2023-09-23T23:42:07.068 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.068 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.069 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622547+0000", 2023-09-23T23:42:07.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7088e-05 2023-09-23T23:42:07.069 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.069 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.069 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622555+0000", 2023-09-23T23:42:07.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6210000000000004e-06 2023-09-23T23:42:07.069 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.069 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.069 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622793+0000", 2023-09-23T23:42:07.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023872499999999999 2023-09-23T23:42:07.070 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.070 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.070 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.070 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622821+0000", 2023-09-23T23:42:07.070 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7518e-05 2023-09-23T23:42:07.070 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.070 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.070 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.070 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743951+0000", 2023-09-23T23:42:07.070 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.121129903 2023-09-23T23:42:07.070 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.070 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.071 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.071 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743993+0000", 2023-09-23T23:42:07.071 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2607000000000001e-05 2023-09-23T23:42:07.071 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.071 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.071 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.071 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.071 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.071 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:7680 14.f e621/617 14:fec04f59:::benchmark_data_smithi110_640334_object7679:head v 621'505, mlcod=621'502)", 2023-09-23T23:42:07.071 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:25.622573+0000", 2023-09-23T23:42:07.071 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 161.417341037, 2023-09-23T23:42:07.071 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.121410935, 2023-09-23T23:42:07.072 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.072 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.072 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.072 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.072 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.072 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622573+0000", 2023-09-23T23:42:07.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.072 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.072 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.072 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.072 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622573+0000", 2023-09-23T23:42:07.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622573+0000", 2023-09-23T23:42:07.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:07.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622603+0000", 2023-09-23T23:42:07.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0352e-05 2023-09-23T23:42:07.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.074 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622610+0000", 2023-09-23T23:42:07.074 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2370000000000003e-06 2023-09-23T23:42:07.074 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.074 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.074 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622615+0000", 2023-09-23T23:42:07.074 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0660000000000003e-06 2023-09-23T23:42:07.074 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.074 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.074 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622691+0000", 2023-09-23T23:42:07.075 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5326000000000004e-05 2023-09-23T23:42:07.075 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.075 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.075 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.075 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622713+0000", 2023-09-23T23:42:07.075 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2010000000000001e-05 2023-09-23T23:42:07.075 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.075 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.075 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.075 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743932+0000", 2023-09-23T23:42:07.075 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.1212198 2023-09-23T23:42:07.075 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.076 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.076 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.076 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743984+0000", 2023-09-23T23:42:07.076 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1487999999999999e-05 2023-09-23T23:42:07.076 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.076 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.076 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.076 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.076 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.076 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:7685 14.a e621/617 14:55b7902e:::benchmark_data_smithi110_640334_object7684:head v 621'455, mlcod=621'452)", 2023-09-23T23:42:07.076 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:25.622628+0000", 2023-09-23T23:42:07.077 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 161.41728647799999, 2023-09-23T23:42:07.077 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.121849387, 2023-09-23T23:42:07.077 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.077 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.077 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.077 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.077 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.077 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622628+0000", 2023-09-23T23:42:07.077 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.077 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.077 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.077 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.078 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622628+0000", 2023-09-23T23:42:07.078 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.078 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.078 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.078 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.078 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622627+0000", 2023-09-23T23:42:07.078 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:07.078 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.078 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.078 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.078 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622648+0000", 2023-09-23T23:42:07.078 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0874e-05 2023-09-23T23:42:07.078 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.079 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.079 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.079 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622653+0000", 2023-09-23T23:42:07.079 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.293e-06 2023-09-23T23:42:07.079 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.079 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.079 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.079 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622657+0000", 2023-09-23T23:42:07.079 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9310000000000001e-06 2023-09-23T23:42:07.079 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.079 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622781+0000", 2023-09-23T23:42:07.080 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012415700000000001 2023-09-23T23:42:07.080 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622803+0000", 2023-09-23T23:42:07.080 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1413e-05 2023-09-23T23:42:07.080 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.744451+0000", 2023-09-23T23:42:07.081 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12164800000000001 2023-09-23T23:42:07.081 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.081 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.081 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.081 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.744477+0000", 2023-09-23T23:42:07.081 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6030999999999999e-05 2023-09-23T23:42:07.081 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.081 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.081 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.081 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.081 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.081 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:7690 14.2 e621/617 14:4a64c483:::benchmark_data_smithi110_640334_object7689:head v 621'469, mlcod=621'467)", 2023-09-23T23:42:07.081 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:25.624477+0000", 2023-09-23T23:42:07.082 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 161.415437473, 2023-09-23T23:42:07.082 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12058026299999999, 2023-09-23T23:42:07.082 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.082 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.082 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.082 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.082 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.082 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.624477+0000", 2023-09-23T23:42:07.082 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.082 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.082 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.082 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.083 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.624477+0000", 2023-09-23T23:42:07.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.083 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.083 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.083 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.624476+0000", 2023-09-23T23:42:07.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:42:07.083 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.083 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.083 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.624518+0000", 2023-09-23T23:42:07.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2144999999999997e-05 2023-09-23T23:42:07.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.084 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743899+0000", 2023-09-23T23:42:07.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.119381112 2023-09-23T23:42:07.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.084 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743910+0000", 2023-09-23T23:42:07.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0789e-05 2023-09-23T23:42:07.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.084 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743973+0000", 2023-09-23T23:42:07.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3007000000000002e-05 2023-09-23T23:42:07.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743996+0000", 2023-09-23T23:42:07.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2906000000000001e-05 2023-09-23T23:42:07.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.745028+0000", 2023-09-23T23:42:07.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001032921 2023-09-23T23:42:07.086 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.086 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.087 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.745057+0000", 2023-09-23T23:42:07.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8388999999999999e-05 2023-09-23T23:42:07.088 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.088 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.088 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.089 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:7689 14.1 e621/617 14:8f840b9f:::benchmark_data_smithi110_640334_object7688:head v 621'504, mlcod=621'500)", 2023-09-23T23:42:07.089 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:25.625092+0000", 2023-09-23T23:42:07.089 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 161.414821617, 2023-09-23T23:42:07.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11995934599999999, 2023-09-23T23:42:07.089 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.089 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.089 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.625092+0000", 2023-09-23T23:42:07.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.625092+0000", 2023-09-23T23:42:07.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.625091+0000", 2023-09-23T23:42:07.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:42:07.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.625116+0000", 2023-09-23T23:42:07.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4726e-05 2023-09-23T23:42:07.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743861+0000", 2023-09-23T23:42:07.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11874531200000001 2023-09-23T23:42:07.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.092 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743915+0000", 2023-09-23T23:42:07.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3533999999999999e-05 2023-09-23T23:42:07.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.092 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.744015+0000", 2023-09-23T23:42:07.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9842999999999999e-05 2023-09-23T23:42:07.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.744028+0000", 2023-09-23T23:42:07.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3351e-05 2023-09-23T23:42:07.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.745031+0000", 2023-09-23T23:42:07.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001003114 2023-09-23T23:42:07.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.745052+0000", 2023-09-23T23:42:07.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0460000000000001e-05 2023-09-23T23:42:07.094 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.094 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.094 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.094 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:76679 14.6 e636/617 14:63b1009c:::benchmark_data_smithi110_640334_object76678:head v 636'4879, mlcod=636'4877)", 2023-09-23T23:42:07.094 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:40:40.502261+0000", 2023-09-23T23:42:07.095 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 86.537652968000003, 2023-09-23T23:42:07.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.114979202, 2023-09-23T23:42:07.095 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.095 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.095 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.095 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.502261+0000", 2023-09-23T23:42:07.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.502261+0000", 2023-09-23T23:42:07.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.096 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.502260+0000", 2023-09-23T23:42:07.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:42:07.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.096 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.502290+0000", 2023-09-23T23:42:07.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9986999999999999e-05 2023-09-23T23:42:07.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.502304+0000", 2023-09-23T23:42:07.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3762e-05 2023-09-23T23:42:07.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.502324+0000", 2023-09-23T23:42:07.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0387999999999999e-05 2023-09-23T23:42:07.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.502398+0000", 2023-09-23T23:42:07.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3547000000000006e-05 2023-09-23T23:42:07.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.502427+0000", 2023-09-23T23:42:07.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9654000000000001e-05 2023-09-23T23:42:07.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.617199+0000", 2023-09-23T23:42:07.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11477135099999999 2023-09-23T23:42:07.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.617240+0000", 2023-09-23T23:42:07.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1597999999999997e-05 2023-09-23T23:42:07.099 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.099 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.099 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.099 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:76677 14.a e636/617 14:5b35bc31:::benchmark_data_smithi110_640334_object76676:head v 636'4690, mlcod=636'4686)", 2023-09-23T23:42:07.100 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:40:40.502659+0000", 2023-09-23T23:42:07.100 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 86.537255049999999, 2023-09-23T23:42:07.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.111865535, 2023-09-23T23:42:07.100 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.100 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.100 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.502659+0000", 2023-09-23T23:42:07.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.502659+0000", 2023-09-23T23:42:07.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.502658+0000", 2023-09-23T23:42:07.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:42:07.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.502683+0000", 2023-09-23T23:42:07.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4700000000000001e-05 2023-09-23T23:42:07.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.103 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.103 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.502691+0000", 2023-09-23T23:42:07.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9840000000000001e-06 2023-09-23T23:42:07.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.103 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.103 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.502698+0000", 2023-09-23T23:42:07.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7000000000000002e-06 2023-09-23T23:42:07.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.502734+0000", 2023-09-23T23:42:07.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6851999999999999e-05 2023-09-23T23:42:07.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.502752+0000", 2023-09-23T23:42:07.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7465999999999998e-05 2023-09-23T23:42:07.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.614487+0000", 2023-09-23T23:42:07.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11173551599999999 2023-09-23T23:42:07.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.614525+0000", 2023-09-23T23:42:07.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7054000000000001e-05 2023-09-23T23:42:07.105 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.105 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.105 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.105 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:76681 14.6 e636/617 14:66e04a98:::benchmark_data_smithi110_640334_object76680:head v 636'4880, mlcod=636'4877)", 2023-09-23T23:42:07.105 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:40:40.503861+0000", 2023-09-23T23:42:07.106 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 86.536052679999997, 2023-09-23T23:42:07.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.121150234, 2023-09-23T23:42:07.106 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.106 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.106 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.503861+0000", 2023-09-23T23:42:07.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.503861+0000", 2023-09-23T23:42:07.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.503861+0000", 2023-09-23T23:42:07.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:42:07.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.108 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.503886+0000", 2023-09-23T23:42:07.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5160999999999998e-05 2023-09-23T23:42:07.108 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.108 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.503898+0000", 2023-09-23T23:42:07.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2356e-05 2023-09-23T23:42:07.108 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.108 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.503905+0000", 2023-09-23T23:42:07.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0960000000000003e-06 2023-09-23T23:42:07.109 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.109 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.109 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.505169+0000", 2023-09-23T23:42:07.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012638390000000001 2023-09-23T23:42:07.109 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.109 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.109 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.614431+0000", 2023-09-23T23:42:07.110 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.109262383 2023-09-23T23:42:07.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.110 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.110 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.624959+0000", 2023-09-23T23:42:07.110 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010528087 2023-09-23T23:42:07.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.625012+0000", 2023-09-23T23:42:07.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2160999999999997e-05 2023-09-23T23:42:07.111 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.111 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.111 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.111 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:76687 14.f e636/617 14:feac0c90:::benchmark_data_smithi110_640334_object76686:head v 636'4833, mlcod=636'4831)", 2023-09-23T23:42:07.112 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:40:40.508633+0000", 2023-09-23T23:42:07.112 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 86.531280757000005, 2023-09-23T23:42:07.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11117706099999999, 2023-09-23T23:42:07.112 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.112 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.112 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.508633+0000", 2023-09-23T23:42:07.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.508633+0000", 2023-09-23T23:42:07.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.508632+0000", 2023-09-23T23:42:07.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:07.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.508667+0000", 2023-09-23T23:42:07.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5235999999999999e-05 2023-09-23T23:42:07.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.115 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.115 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.614427+0000", 2023-09-23T23:42:07.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.105759625 2023-09-23T23:42:07.115 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.115 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.116 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.614443+0000", 2023-09-23T23:42:07.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5692000000000002e-05 2023-09-23T23:42:07.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.116 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.116 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.614481+0000", 2023-09-23T23:42:07.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8143999999999999e-05 2023-09-23T23:42:07.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.116 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.614507+0000", 2023-09-23T23:42:07.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6757000000000001e-05 2023-09-23T23:42:07.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.619740+0000", 2023-09-23T23:42:07.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052330340000000001 2023-09-23T23:42:07.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.619810+0000", 2023-09-23T23:42:07.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9895999999999996e-05 2023-09-23T23:42:07.118 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.118 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.118 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.118 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.118 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.118 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:92258 14.3 e642/617 14:c938b5e2:::benchmark_data_smithi110_640334_object8379:head v 642'5671, mlcod=642'5669)", 2023-09-23T23:42:07.118 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:05.597280+0000", 2023-09-23T23:42:07.118 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 61.442633717, 2023-09-23T23:42:07.118 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.17337343499999999, 2023-09-23T23:42:07.118 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.118 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.119 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.119 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597280+0000", 2023-09-23T23:42:07.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.119 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597280+0000", 2023-09-23T23:42:07.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.120 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.120 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.120 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597279+0000", 2023-09-23T23:42:07.120 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:42:07.120 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.120 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.121 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.121 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597285+0000", 2023-09-23T23:42:07.121 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.144e-06 2023-09-23T23:42:07.121 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.121 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.121 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.121 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597298+0000", 2023-09-23T23:42:07.122 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2916e-05 2023-09-23T23:42:07.122 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.122 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.122 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.122 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597306+0000", 2023-09-23T23:42:07.122 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7090000000000002e-06 2023-09-23T23:42:07.122 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.122 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.122 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.122 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597435+0000", 2023-09-23T23:42:07.122 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000128498 2023-09-23T23:42:07.123 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.123 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.123 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.123 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597454+0000", 2023-09-23T23:42:07.123 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9933999999999999e-05 2023-09-23T23:42:07.123 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.123 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.123 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.123 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.770624+0000", 2023-09-23T23:42:07.123 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.173169447 2023-09-23T23:42:07.123 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.123 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.123 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.124 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.770654+0000", 2023-09-23T23:42:07.124 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9832000000000002e-05 2023-09-23T23:42:07.124 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.124 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.124 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.124 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.124 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.124 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:92266 14.1 e642/617 14:8946a29f:::benchmark_data_smithi110_640334_object8387:head v 642'5829, mlcod=642'5826)", 2023-09-23T23:42:07.124 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:05.597320+0000", 2023-09-23T23:42:07.124 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 61.442593948000003, 2023-09-23T23:42:07.125 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.173397724, 2023-09-23T23:42:07.125 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.125 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.125 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.125 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.125 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.125 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597320+0000", 2023-09-23T23:42:07.125 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.125 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.125 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.125 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.125 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597320+0000", 2023-09-23T23:42:07.126 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.126 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.126 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.126 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.126 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597320+0000", 2023-09-23T23:42:07.126 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:07.126 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.126 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.126 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.126 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597324+0000", 2023-09-23T23:42:07.126 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8029999999999998e-06 2023-09-23T23:42:07.127 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.127 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.127 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.127 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597329+0000", 2023-09-23T23:42:07.127 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1340000000000003e-06 2023-09-23T23:42:07.127 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.127 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.127 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.127 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597333+0000", 2023-09-23T23:42:07.127 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9169999999999999e-06 2023-09-23T23:42:07.127 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.128 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.128 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.128 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.762583+0000", 2023-09-23T23:42:07.128 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.16525079100000001 2023-09-23T23:42:07.128 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.128 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.128 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.128 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.762608+0000", 2023-09-23T23:42:07.128 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4746999999999999e-05 2023-09-23T23:42:07.128 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.128 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.128 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.129 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.770669+0000", 2023-09-23T23:42:07.129 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080605199999999998 2023-09-23T23:42:07.129 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.129 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.129 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.129 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.770718+0000", 2023-09-23T23:42:07.129 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9098000000000003e-05 2023-09-23T23:42:07.129 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.129 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.129 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.129 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.129 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.130 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:92264 14.b e642/617 14:d4f12831:::benchmark_data_smithi110_640334_object8385:head v 642'5716, mlcod=642'5714)", 2023-09-23T23:42:07.130 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:05.597352+0000", 2023-09-23T23:42:07.130 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 61.442561728000001, 2023-09-23T23:42:07.130 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.17333879499999999, 2023-09-23T23:42:07.130 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.130 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.130 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.130 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.130 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.130 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597352+0000", 2023-09-23T23:42:07.131 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.131 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.131 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.131 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.131 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597352+0000", 2023-09-23T23:42:07.131 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.131 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.131 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.131 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.131 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597352+0000", 2023-09-23T23:42:07.131 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:07.131 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.132 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.132 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.132 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597356+0000", 2023-09-23T23:42:07.132 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9859999999999996e-06 2023-09-23T23:42:07.132 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.132 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.132 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.132 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597361+0000", 2023-09-23T23:42:07.132 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.254e-06 2023-09-23T23:42:07.132 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.132 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.133 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.133 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597365+0000", 2023-09-23T23:42:07.133 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0130000000000004e-06 2023-09-23T23:42:07.133 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.133 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.133 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.133 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.762579+0000", 2023-09-23T23:42:07.133 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.165213839 2023-09-23T23:42:07.133 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.133 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.133 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.134 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.762605+0000", 2023-09-23T23:42:07.134 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6086000000000001e-05 2023-09-23T23:42:07.134 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.134 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.134 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.134 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.770663+0000", 2023-09-23T23:42:07.134 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080581279999999995 2023-09-23T23:42:07.134 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.134 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.134 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.134 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.770691+0000", 2023-09-23T23:42:07.134 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7778999999999999e-05 2023-09-23T23:42:07.135 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.135 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.135 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.135 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.135 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.135 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:92267 14.6 e642/617 14:6427f456:::benchmark_data_smithi110_640334_object8388:head v 642'5896, mlcod=642'5894)", 2023-09-23T23:42:07.135 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:05.601294+0000", 2023-09-23T23:42:07.135 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 61.438619901000003, 2023-09-23T23:42:07.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.17304376499999999, 2023-09-23T23:42:07.136 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.136 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.136 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.136 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.136 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.136 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.601294+0000", 2023-09-23T23:42:07.136 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.136 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.136 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.136 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.601294+0000", 2023-09-23T23:42:07.136 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.137 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.137 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.601293+0000", 2023-09-23T23:42:07.137 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:42:07.137 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.137 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.138 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.601302+0000", 2023-09-23T23:42:07.138 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3989999999999993e-06 2023-09-23T23:42:07.138 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.138 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.138 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.138 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.762549+0000", 2023-09-23T23:42:07.138 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.16124624000000001 2023-09-23T23:42:07.138 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.138 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.138 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.138 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.762562+0000", 2023-09-23T23:42:07.139 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2853e-05 2023-09-23T23:42:07.139 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.139 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.139 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.139 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.762698+0000", 2023-09-23T23:42:07.139 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000136126 2023-09-23T23:42:07.139 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.139 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.139 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.139 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.762717+0000", 2023-09-23T23:42:07.140 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9259e-05 2023-09-23T23:42:07.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.140 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.140 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.774303+0000", 2023-09-23T23:42:07.140 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01158578 2023-09-23T23:42:07.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.140 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.774338+0000", 2023-09-23T23:42:07.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5163999999999997e-05 2023-09-23T23:42:07.141 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.141 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.141 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.141 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.141 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.141 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:92719 14.f e642/617 14:f97a3607:::benchmark_data_smithi110_640334_object8840:head v 642'5872, mlcod=642'5870)", 2023-09-23T23:42:07.141 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:06.360791+0000", 2023-09-23T23:42:07.141 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.679122775000003, 2023-09-23T23:42:07.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12632665400000001, 2023-09-23T23:42:07.141 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.142 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.142 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.142 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.142 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.142 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.360791+0000", 2023-09-23T23:42:07.142 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.142 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.360791+0000", 2023-09-23T23:42:07.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.360790+0000", 2023-09-23T23:42:07.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:42:07.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.144 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.360799+0000", 2023-09-23T23:42:07.144 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6319999999999997e-06 2023-09-23T23:42:07.144 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.144 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.144 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.360812+0000", 2023-09-23T23:42:07.144 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3040000000000001e-05 2023-09-23T23:42:07.144 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.145 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.145 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.360820+0000", 2023-09-23T23:42:07.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1780000000000003e-06 2023-09-23T23:42:07.145 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.145 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.145 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.145 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.360866+0000", 2023-09-23T23:42:07.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6371e-05 2023-09-23T23:42:07.145 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.145 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.146 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.146 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.360889+0000", 2023-09-23T23:42:07.146 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2673000000000001e-05 2023-09-23T23:42:07.146 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.146 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.146 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371548+0000", 2023-09-23T23:42:07.146 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010658978 2023-09-23T23:42:07.146 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.147 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.147 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.487118+0000", 2023-09-23T23:42:07.147 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.115569828 2023-09-23T23:42:07.147 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.147 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.147 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.147 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.148 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:92721 14.8 e642/617 14:17bf1845:::benchmark_data_smithi110_640334_object8842:head v 642'5875, mlcod=642'5873)", 2023-09-23T23:42:07.148 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:06.365471+0000", 2023-09-23T23:42:07.148 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.674442923000001, 2023-09-23T23:42:07.148 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12165229599999999, 2023-09-23T23:42:07.148 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.148 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.148 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.149 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.149 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.365471+0000", 2023-09-23T23:42:07.149 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.149 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.149 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.149 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.149 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.365471+0000", 2023-09-23T23:42:07.150 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.150 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.150 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.150 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.150 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.365471+0000", 2023-09-23T23:42:07.150 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:07.150 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.150 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.150 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.151 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.365475+0000", 2023-09-23T23:42:07.151 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9999999999999998e-06 2023-09-23T23:42:07.151 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.151 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.151 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.151 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.365481+0000", 2023-09-23T23:42:07.151 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8200000000000002e-06 2023-09-23T23:42:07.151 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.151 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.151 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.151 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.365485+0000", 2023-09-23T23:42:07.151 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.216e-06 2023-09-23T23:42:07.152 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.152 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.152 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.365560+0000", 2023-09-23T23:42:07.152 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5581999999999996e-05 2023-09-23T23:42:07.152 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.152 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.152 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.365581+0000", 2023-09-23T23:42:07.152 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0581000000000002e-05 2023-09-23T23:42:07.152 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.152 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.153 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371604+0000", 2023-09-23T23:42:07.153 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060231779999999997 2023-09-23T23:42:07.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.153 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.153 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.153 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.487123+0000", 2023-09-23T23:42:07.153 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.115519344 2023-09-23T23:42:07.153 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.153 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.153 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.153 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.154 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:92722 14.3 e642/617 14:c87b4832:::benchmark_data_smithi110_640334_object8843:head v 642'5709, mlcod=642'5707)", 2023-09-23T23:42:07.154 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:06.368831+0000", 2023-09-23T23:42:07.154 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.671082681000001, 2023-09-23T23:42:07.154 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11828602000000001, 2023-09-23T23:42:07.154 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.154 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.154 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.154 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.154 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.154 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.368831+0000", 2023-09-23T23:42:07.154 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.154 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.154 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.155 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.368831+0000", 2023-09-23T23:42:07.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.155 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.155 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.368830+0000", 2023-09-23T23:42:07.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:42:07.155 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.155 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.368838+0000", 2023-09-23T23:42:07.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5650000000000004e-06 2023-09-23T23:42:07.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.156 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.156 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.156 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.368849+0000", 2023-09-23T23:42:07.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1253000000000001e-05 2023-09-23T23:42:07.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.156 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.156 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.156 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.368855+0000", 2023-09-23T23:42:07.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6470000000000003e-06 2023-09-23T23:42:07.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.157 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.368921+0000", 2023-09-23T23:42:07.157 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5729999999999996e-05 2023-09-23T23:42:07.157 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.157 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.368941+0000", 2023-09-23T23:42:07.157 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0301000000000001e-05 2023-09-23T23:42:07.157 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.374127+0000", 2023-09-23T23:42:07.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051863559999999996 2023-09-23T23:42:07.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.158 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.487117+0000", 2023-09-23T23:42:07.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.112990123 2023-09-23T23:42:07.158 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.158 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.158 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.159 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:92723 14.6 e642/617 14:6a0ee19f:::benchmark_data_smithi110_640334_object8844:head v 642'5921, mlcod=642'5919)", 2023-09-23T23:42:07.159 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:06.371410+0000", 2023-09-23T23:42:07.159 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.668503905999998, 2023-09-23T23:42:07.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.13841763800000001, 2023-09-23T23:42:07.159 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.159 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.159 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371410+0000", 2023-09-23T23:42:07.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.160 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.160 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.160 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.160 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371410+0000", 2023-09-23T23:42:07.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.160 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.160 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.160 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.160 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371409+0000", 2023-09-23T23:42:07.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:42:07.160 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.161 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371417+0000", 2023-09-23T23:42:07.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5179999999999998e-06 2023-09-23T23:42:07.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.161 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371426+0000", 2023-09-23T23:42:07.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6709999999999994e-06 2023-09-23T23:42:07.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371433+0000", 2023-09-23T23:42:07.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.024e-06 2023-09-23T23:42:07.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.487124+0000", 2023-09-23T23:42:07.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.115690628 2023-09-23T23:42:07.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.487144+0000", 2023-09-23T23:42:07.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9564000000000001e-05 2023-09-23T23:42:07.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.509781+0000", 2023-09-23T23:42:07.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022637066000000001 2023-09-23T23:42:07.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.509828+0000", 2023-09-23T23:42:07.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7086000000000003e-05 2023-09-23T23:42:07.164 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.164 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.164 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.164 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:92726 14.b e642/617 14:db54547b:::benchmark_data_smithi110_640334_object8847:head v 642'5747, mlcod=642'5744)", 2023-09-23T23:42:07.164 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:06.371446+0000", 2023-09-23T23:42:07.164 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.668468277000002, 2023-09-23T23:42:07.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.119250561, 2023-09-23T23:42:07.164 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:07.164 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:07.164 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:07.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:07.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371446+0000", 2023-09-23T23:42:07.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:07.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371446+0000", 2023-09-23T23:42:07.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:07.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:07.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371445+0000", 2023-09-23T23:42:07.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:07.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:07.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371449+0000", 2023-09-23T23:42:07.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8500000000000004e-06 2023-09-23T23:42:07.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:07.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371455+0000", 2023-09-23T23:42:07.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9660000000000001e-06 2023-09-23T23:42:07.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:07.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.487089+0000", 2023-09-23T23:42:07.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.115633662 2023-09-23T23:42:07.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:07.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.487130+0000", 2023-09-23T23:42:07.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1440999999999998e-05 2023-09-23T23:42:07.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:07.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.487146+0000", 2023-09-23T23:42:07.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5934999999999999e-05 2023-09-23T23:42:07.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:07.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.490680+0000", 2023-09-23T23:42:07.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035332739999999999 2023-09-23T23:42:07.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:07.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:07.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:07.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.490696+0000", 2023-09-23T23:42:07.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6799e-05 2023-09-23T23:42:07.169 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.169 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.169 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.169 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:07.169 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:07.169 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:42:07.170 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:07.373 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:42:07.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:07.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:42:07.388 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:07 smithi110 ceph-mon[221102]: pgmap v1890: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 607 op/s; 1464/122646 objects degraded (1.194%); 3.9 MiB/s, 139 objects/s recovering 2023-09-23T23:42:07.388 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:07 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/311434781' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:42:07.393 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T23:42:07.489 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:42:07.518 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:07.560 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:42:07.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:07.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:42:07.665 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:07.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:07 smithi138 ceph-mon[193606]: pgmap v1890: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 607 op/s; 1464/122646 objects degraded (1.194%); 3.9 MiB/s, 139 objects/s recovering 2023-09-23T23:42:07.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:07 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/311434781' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:42:07.694 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:07.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:07 smithi110 ceph-mon[212939]: pgmap v1890: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 607 op/s; 1464/122646 objects degraded (1.194%); 3.9 MiB/s, 139 objects/s recovering 2023-09-23T23:42:07.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:07 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/311434781' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:42:07.738 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:07.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:07.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:07.845 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:42:07.874 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:07.923 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:42:07.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:07.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:42:08.030 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:42:08.058 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:08.076 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:42:08.106 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:42:08.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:08.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:42:08.213 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:42:08.327 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:08.413 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:08 smithi110 ceph-mon[212939]: failed to encode map e664 with expected crc 2023-09-23T23:42:08.414 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:08 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/311434781' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:42:08.414 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:08 smithi110 ceph-mon[212939]: osdmap e664: 8 total, 8 up, 6 in 2023-09-23T23:42:08.414 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:08 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/38126554' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:42:08.414 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:08 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:42:08.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:08 smithi110 ceph-mon[221102]: failed to encode map e664 with expected crc 2023-09-23T23:42:08.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:08 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/311434781' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:42:08.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:08 smithi110 ceph-mon[221102]: osdmap e664: 8 total, 8 up, 6 in 2023-09-23T23:42:08.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:08 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/38126554' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:42:08.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:08 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:42:08.464 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:42:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:08 smithi138 ceph-mon[193606]: failed to encode map e664 with expected crc 2023-09-23T23:42:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:08 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/311434781' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:42:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:08 smithi138 ceph-mon[193606]: osdmap e664: 8 total, 8 up, 6 in 2023-09-23T23:42:08.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:08 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/38126554' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:42:08.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:08 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:42:09.027 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:42:09.540 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:42:09.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:09.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:42:09.551 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[221102]: failed to encode map e664 with expected crc 2023-09-23T23:42:09.551 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[221102]: failed to encode map e664 with expected crc 2023-09-23T23:42:09.552 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[221102]: pgmap v1892: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 787 op/s; 1464/117816 objects degraded (1.243%); 3.9 MiB/s, 140 objects/s recovering 2023-09-23T23:42:09.552 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:42:09.552 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[221102]: failed to encode map e665 with expected crc 2023-09-23T23:42:09.552 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[221102]: failed to encode map e665 with expected crc 2023-09-23T23:42:09.552 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:42:09.552 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[221102]: osdmap e665: 8 total, 8 up, 6 in 2023-09-23T23:42:09.552 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1464/117816 objects degraded (1.243%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:42:09.552 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[212939]: failed to encode map e664 with expected crc 2023-09-23T23:42:09.552 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[212939]: failed to encode map e664 with expected crc 2023-09-23T23:42:09.553 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[212939]: pgmap v1892: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 787 op/s; 1464/117816 objects degraded (1.243%); 3.9 MiB/s, 140 objects/s recovering 2023-09-23T23:42:09.553 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:42:09.553 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[212939]: failed to encode map e665 with expected crc 2023-09-23T23:42:09.553 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[212939]: failed to encode map e665 with expected crc 2023-09-23T23:42:09.553 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:42:09.553 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[212939]: osdmap e665: 8 total, 8 up, 6 in 2023-09-23T23:42:09.553 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:09 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1464/117816 objects degraded (1.243%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:42:09.649 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:42:09.672 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:09 smithi138 ceph-mon[193606]: failed to encode map e664 with expected crc 2023-09-23T23:42:09.673 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:09 smithi138 ceph-mon[193606]: failed to encode map e664 with expected crc 2023-09-23T23:42:09.673 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:09 smithi138 ceph-mon[193606]: pgmap v1892: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 787 op/s; 1464/117816 objects degraded (1.243%); 3.9 MiB/s, 140 objects/s recovering 2023-09-23T23:42:09.673 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:09 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:42:09.673 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:09 smithi138 ceph-mon[193606]: failed to encode map e665 with expected crc 2023-09-23T23:42:09.673 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:09 smithi138 ceph-mon[193606]: failed to encode map e665 with expected crc 2023-09-23T23:42:09.673 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:09 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:42:09.673 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:09 smithi138 ceph-mon[193606]: osdmap e665: 8 total, 8 up, 6 in 2023-09-23T23:42:09.673 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:09 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1464/117816 objects degraded (1.243%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:42:09.675 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:09.727 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:42:09.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:09.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:42:09.834 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:42:09.876 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T23:42:09.993 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:10.050 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:10.136 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:10.143 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:42:10.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:10.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:42:10.253 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:10.331 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:10.450 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:10.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:10.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:10.559 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:10.644 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:11.096 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:11.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:11.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:11.209 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:11.311 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:11.577 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:11.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:11.589 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:11 smithi110 ceph-mon[212939]: pgmap v1894: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 601 op/s; 1464/117135 objects degraded (1.250%); 2.2 MiB/s, 45 objects/s recovering 2023-09-23T23:42:11.590 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:11 smithi110 ceph-mon[221102]: pgmap v1894: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 601 op/s; 1464/117135 objects degraded (1.250%); 2.2 MiB/s, 45 objects/s recovering 2023-09-23T23:42:11.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:11.691 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:42:11.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:11 smithi138 ceph-mon[193606]: pgmap v1894: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 601 op/s; 1464/117135 objects degraded (1.250%); 2.2 MiB/s, 45 objects/s recovering 2023-09-23T23:42:11.722 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:11.764 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:42:11.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:11.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:42:11.868 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:11.900 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:42:11.914 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:12.008 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:12.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:12.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:12.126 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:42:12.155 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:12.199 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:42:12.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:12.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:42:12.306 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:12.335 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:12.386 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:12.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:12.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:12.491 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:12.520 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:12.566 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:12.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:12.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:12.672 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:42:12.700 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:12.961 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:13.007 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:42:13.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:13.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:42:13.126 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:42:13.208 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:13.487 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:42:13.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:13.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:42:13.576 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:13 smithi110 ceph-mon[221102]: pgmap v1895: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 578 op/s; 1202/113064 objects degraded (1.063%); 2.2 MiB/s, 78 objects/s recovering 2023-09-23T23:42:13.576 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:13.576 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:42:13.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:13 smithi110 ceph-mon[212939]: pgmap v1895: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 578 op/s; 1202/113064 objects degraded (1.063%); 2.2 MiB/s, 78 objects/s recovering 2023-09-23T23:42:13.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:13.577 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:42:13.594 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:42:13.672 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:13 smithi138 ceph-mon[193606]: pgmap v1895: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 578 op/s; 1202/113064 objects degraded (1.063%); 2.2 MiB/s, 78 objects/s recovering 2023-09-23T23:42:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:13.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:42:13.728 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:14.611 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:14 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 1202/113064 objects degraded (1.063%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:42:14.612 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:14 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 1202/113064 objects degraded (1.063%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:42:14.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:14 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 1202/113064 objects degraded (1.063%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:42:15.004 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:42:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:42:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:133632 14.3 14:c7d69c4a:::benchmark_data_smithi110_640334_object49753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e665)", 2023-09-23T23:42:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:42:14.548019+0000", 2023-09-23T23:42:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.023905464000000001, 2023-09-23T23:42:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023938045000000002, 2023-09-23T23:42:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:42:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:42:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:42:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 133632 2023-09-23T23:42:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.548019+0000", 2023-09-23T23:42:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.548019+0000", 2023-09-23T23:42:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.548018+0000", 2023-09-23T23:42:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:42:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.548024+0000", 2023-09-23T23:42:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2450000000000003e-06 2023-09-23T23:42:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.548037+0000", 2023-09-23T23:42:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.255e-05 2023-09-23T23:42:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.548045+0000", 2023-09-23T23:42:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.174e-06 2023-09-23T23:42:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.548117+0000", 2023-09-23T23:42:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1885999999999996e-05 2023-09-23T23:42:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.551218+0000", 2023-09-23T23:42:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031010970000000001 2023-09-23T23:42:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:42:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.551324+0000", 2023-09-23T23:42:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105486 2023-09-23T23:42:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.564643+0000", 2023-09-23T23:42:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013319938 2023-09-23T23:42:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.565475+0000", 2023-09-23T23:42:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00083151799999999995 2023-09-23T23:42:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:133633 14.2 14:411fdd16:::benchmark_data_smithi110_640334_object49754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e665)", 2023-09-23T23:42:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:42:14.548060+0000", 2023-09-23T23:42:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.023864864999999999, 2023-09-23T23:42:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023991934999999999, 2023-09-23T23:42:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:42:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:42:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:42:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 133633 2023-09-23T23:42:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.548060+0000", 2023-09-23T23:42:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.548060+0000", 2023-09-23T23:42:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.548059+0000", 2023-09-23T23:42:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.548062+0000", 2023-09-23T23:42:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9569999999999998e-06 2023-09-23T23:42:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.548072+0000", 2023-09-23T23:42:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6220000000000004e-06 2023-09-23T23:42:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.548077+0000", 2023-09-23T23:42:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3589999999999999e-06 2023-09-23T23:42:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.548158+0000", 2023-09-23T23:42:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0659000000000001e-05 2023-09-23T23:42:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.548266+0000", 2023-09-23T23:42:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010765899999999999 2023-09-23T23:42:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-09-23T23:42:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.548373+0000", 2023-09-23T23:42:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000106913 2023-09-23T23:42:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.561349+0000", 2023-09-23T23:42:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012976577 2023-09-23T23:42:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.565411+0000", 2023-09-23T23:42:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040619100000000002 2023-09-23T23:42:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:133641 14.3 14:c207706b:::benchmark_data_smithi110_640334_object49762:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e665)", 2023-09-23T23:42:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:42:14.553790+0000", 2023-09-23T23:42:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.018134705000000001, 2023-09-23T23:42:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018348449999999999, 2023-09-23T23:42:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:42:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:42:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:42:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 133641 2023-09-23T23:42:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553790+0000", 2023-09-23T23:42:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553790+0000", 2023-09-23T23:42:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553789+0000", 2023-09-23T23:42:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:42:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553795+0000", 2023-09-23T23:42:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6890000000000001e-06 2023-09-23T23:42:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553806+0000", 2023-09-23T23:42:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1256e-05 2023-09-23T23:42:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553813+0000", 2023-09-23T23:42:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8040000000000004e-06 2023-09-23T23:42:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.555709+0000", 2023-09-23T23:42:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001896071 2023-09-23T23:42:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.555820+0000", 2023-09-23T23:42:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011146600000000001 2023-09-23T23:42:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:42:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.555927+0000", 2023-09-23T23:42:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000106282 2023-09-23T23:42:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.564689+0000", 2023-09-23T23:42:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0087619989999999995 2023-09-23T23:42:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.565480+0000", 2023-09-23T23:42:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00079112200000000001 2023-09-23T23:42:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:133643 14.a 14:5862081a:::benchmark_data_smithi110_640334_object49764:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e665)", 2023-09-23T23:42:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:42:14.553830+0000", 2023-09-23T23:42:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.018094545, 2023-09-23T23:42:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018397828000000001, 2023-09-23T23:42:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:42:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:42:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:42:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 133643 2023-09-23T23:42:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553830+0000", 2023-09-23T23:42:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553830+0000", 2023-09-23T23:42:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553830+0000", 2023-09-23T23:42:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553833+0000", 2023-09-23T23:42:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2959999999999999e-06 2023-09-23T23:42:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553839+0000", 2023-09-23T23:42:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.285e-06 2023-09-23T23:42:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553845+0000", 2023-09-23T23:42:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6980000000000003e-06 2023-09-23T23:42:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.556230+0000", 2023-09-23T23:42:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023851570000000002 2023-09-23T23:42:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.556326+0000", 2023-09-23T23:42:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6317000000000006e-05 2023-09-23T23:42:15.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-09-23T23:42:15.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.556424+0000", 2023-09-23T23:42:15.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8043999999999998e-05 2023-09-23T23:42:15.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:15.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.561382+0000", 2023-09-23T23:42:15.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0049570680000000002 2023-09-23T23:42:15.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:15.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.565712+0000", 2023-09-23T23:42:15.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043306860000000003 2023-09-23T23:42:15.028 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.028 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:15.028 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.028 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:133645 14.3 14:c72c9659:::benchmark_data_smithi110_640334_object49766:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e665)", 2023-09-23T23:42:15.028 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:42:14.553852+0000", 2023-09-23T23:42:15.028 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.018072568000000001, 2023-09-23T23:42:15.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018460206, 2023-09-23T23:42:15.028 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:15.029 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:42:15.029 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:15.029 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:42:15.029 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:42:15.029 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 133645 2023-09-23T23:42:15.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.029 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:15.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:15.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553852+0000", 2023-09-23T23:42:15.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:15.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553852+0000", 2023-09-23T23:42:15.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:15.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553852+0000", 2023-09-23T23:42:15.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:15.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:15.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553854+0000", 2023-09-23T23:42:15.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4200000000000001e-06 2023-09-23T23:42:15.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:15.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553860+0000", 2023-09-23T23:42:15.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9750000000000004e-06 2023-09-23T23:42:15.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:15.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.553865+0000", 2023-09-23T23:42:15.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8590000000000004e-06 2023-09-23T23:42:15.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:15.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.558599+0000", 2023-09-23T23:42:15.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047343760000000002 2023-09-23T23:42:15.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:15.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.558704+0000", 2023-09-23T23:42:15.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010461100000000001 2023-09-23T23:42:15.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:42:15.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.558807+0000", 2023-09-23T23:42:15.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103217 2023-09-23T23:42:15.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:15.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.564717+0000", 2023-09-23T23:42:15.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059101240000000001 2023-09-23T23:42:15.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:15.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.565727+0000", 2023-09-23T23:42:15.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001009788 2023-09-23T23:42:15.034 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.034 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:15.034 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.034 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:133647 14.3 14:ccc902e5:::benchmark_data_smithi110_640334_object49768:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e665)", 2023-09-23T23:42:15.034 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:42:14.559604+0000", 2023-09-23T23:42:15.034 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.012320283, 2023-09-23T23:42:15.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01279226, 2023-09-23T23:42:15.034 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:15.034 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:42:15.035 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:15.035 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:42:15.035 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:42:15.035 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 133647 2023-09-23T23:42:15.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.035 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:15.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:15.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.559604+0000", 2023-09-23T23:42:15.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:15.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.559604+0000", 2023-09-23T23:42:15.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:15.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.559603+0000", 2023-09-23T23:42:15.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:15.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:15.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.559611+0000", 2023-09-23T23:42:15.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1210000000000007e-06 2023-09-23T23:42:15.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:15.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.559624+0000", 2023-09-23T23:42:15.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2982e-05 2023-09-23T23:42:15.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:15.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.559633+0000", 2023-09-23T23:42:15.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4659999999999999e-06 2023-09-23T23:42:15.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:15.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.561118+0000", 2023-09-23T23:42:15.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001485002 2023-09-23T23:42:15.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:15.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.561192+0000", 2023-09-23T23:42:15.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4556999999999998e-05 2023-09-23T23:42:15.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:42:15.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.561258+0000", 2023-09-23T23:42:15.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5931000000000003e-05 2023-09-23T23:42:15.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:15.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.565732+0000", 2023-09-23T23:42:15.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044738169999999997 2023-09-23T23:42:15.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:15.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.567431+0000", 2023-09-23T23:42:15.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001698644 2023-09-23T23:42:15.040 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.040 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:15.040 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.040 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:133649 14.9 14:9311d8db:::benchmark_data_smithi110_640334_object49770:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e665)", 2023-09-23T23:42:15.040 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:42:14.567767+0000", 2023-09-23T23:42:15.040 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0041577519999999998, 2023-09-23T23:42:15.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047162460000000003, 2023-09-23T23:42:15.040 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:15.041 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:42:15.041 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:15.041 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:42:15.041 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:42:15.041 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 133649 2023-09-23T23:42:15.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.041 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:15.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:15.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.567767+0000", 2023-09-23T23:42:15.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:15.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.567767+0000", 2023-09-23T23:42:15.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:15.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.567766+0000", 2023-09-23T23:42:15.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:15.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:15.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.567773+0000", 2023-09-23T23:42:15.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9539999999999998e-06 2023-09-23T23:42:15.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:15.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.567788+0000", 2023-09-23T23:42:15.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5781e-05 2023-09-23T23:42:15.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:15.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.567796+0000", 2023-09-23T23:42:15.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7200000000000006e-06 2023-09-23T23:42:15.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:15.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.567838+0000", 2023-09-23T23:42:15.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1950000000000003e-05 2023-09-23T23:42:15.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:15.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.567969+0000", 2023-09-23T23:42:15.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013131300000000001 2023-09-23T23:42:15.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:42:15.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.568086+0000", 2023-09-23T23:42:15.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000116811 2023-09-23T23:42:15.045 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.045 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:15.045 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.046 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:133652 14.3 14:c86df565:::benchmark_data_smithi110_640334_object49773:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e665)", 2023-09-23T23:42:15.046 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:42:14.567818+0000", 2023-09-23T23:42:15.046 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0041070270000000001, 2023-09-23T23:42:15.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.004745066, 2023-09-23T23:42:15.046 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:15.046 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:42:15.046 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:15.046 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:42:15.046 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:42:15.046 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 133652 2023-09-23T23:42:15.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.047 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:15.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:15.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.567818+0000", 2023-09-23T23:42:15.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:15.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.567818+0000", 2023-09-23T23:42:15.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:15.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.567817+0000", 2023-09-23T23:42:15.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:15.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:15.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.567821+0000", 2023-09-23T23:42:15.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5899999999999999e-06 2023-09-23T23:42:15.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:15.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.567827+0000", 2023-09-23T23:42:15.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6529999999999997e-06 2023-09-23T23:42:15.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:15.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.567833+0000", 2023-09-23T23:42:15.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5890000000000002e-06 2023-09-23T23:42:15.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:15.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.567893+0000", 2023-09-23T23:42:15.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0024e-05 2023-09-23T23:42:15.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:15.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.567998+0000", 2023-09-23T23:42:15.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104493 2023-09-23T23:42:15.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:42:15.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.568109+0000", 2023-09-23T23:42:15.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000111218 2023-09-23T23:42:15.050 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.050 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:15.050 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.051 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:133653 14.1 14:845dbb3c:::benchmark_data_smithi110_640334_object49774:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e665)", 2023-09-23T23:42:15.051 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:42:14.568271+0000", 2023-09-23T23:42:15.051 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.003653496, 2023-09-23T23:42:15.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043631659999999999, 2023-09-23T23:42:15.051 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:15.051 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:42:15.051 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:15.051 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:42:15.051 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:42:15.051 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 133653 2023-09-23T23:42:15.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.051 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:15.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:15.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.568271+0000", 2023-09-23T23:42:15.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:15.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.568271+0000", 2023-09-23T23:42:15.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:15.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.568271+0000", 2023-09-23T23:42:15.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:42:15.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:15.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.568275+0000", 2023-09-23T23:42:15.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8690000000000003e-06 2023-09-23T23:42:15.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:15.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.568286+0000", 2023-09-23T23:42:15.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0581000000000001e-05 2023-09-23T23:42:15.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:15.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.568293+0000", 2023-09-23T23:42:15.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3830000000000003e-06 2023-09-23T23:42:15.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:15.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.568334+0000", 2023-09-23T23:42:15.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0161999999999999e-05 2023-09-23T23:42:15.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:15.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.568444+0000", 2023-09-23T23:42:15.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110612 2023-09-23T23:42:15.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:42:15.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.568562+0000", 2023-09-23T23:42:15.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011767 2023-09-23T23:42:15.055 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.055 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:15.055 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.056 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.47186.0:133649 14.9 e665/617)", 2023-09-23T23:42:15.056 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:42:14.571857+0000", 2023-09-23T23:42:15.056 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 6.7711000000000003e-05, 2023-09-23T23:42:15.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000844701, 2023-09-23T23:42:15.056 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:15.056 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-09-23T23:42:15.056 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:15.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:15.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.571857+0000", 2023-09-23T23:42:15.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:15.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.571857+0000", 2023-09-23T23:42:15.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:15.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:15.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.571856+0000", 2023-09-23T23:42:15.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:42:15.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:15.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.571862+0000", 2023-09-23T23:42:15.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6319999999999996e-06 2023-09-23T23:42:15.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:15.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.571875+0000", 2023-09-23T23:42:15.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3241000000000001e-05 2023-09-23T23:42:15.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:15.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:15.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:15.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.571881+0000", 2023-09-23T23:42:15.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8799999999999996e-06 2023-09-23T23:42:15.059 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.059 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:15.059 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.059 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:15.059 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-09-23T23:42:15.059 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 10 2023-09-23T23:42:15.059 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:42:15.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:15.060 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:42:15.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:42:15.161 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:15.189 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:15.236 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:15.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:15.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:15.343 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:42:15.370 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:15.414 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:42:15.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:15.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:42:15.519 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:42:15.633 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:15.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:15 smithi110 ceph-mon[212939]: pgmap v1896: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 606 op/s; 1202/108081 objects degraded (1.112%); 0 B/s, 32 objects/s recovering 2023-09-23T23:42:15.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:15 smithi110 ceph-mon[221102]: pgmap v1896: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 606 op/s; 1202/108081 objects degraded (1.112%); 0 B/s, 32 objects/s recovering 2023-09-23T23:42:15.745 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:42:15.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:15.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:42:15.854 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:42:15.879 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:15 smithi138 ceph-mon[193606]: pgmap v1896: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 606 op/s; 1202/108081 objects degraded (1.112%); 0 B/s, 32 objects/s recovering 2023-09-23T23:42:15.881 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:15.930 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:42:15.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:15.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:42:16.036 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:42:16.063 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:16.188 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:42:16.188 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:42:16.188 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:42:16.212 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:42:16.247 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:42:16.283 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:42:16.313 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:42:16.349 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:42:16.375 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:42:16.375 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:42:16.375 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:42:16.375 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:42:16.375 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:42:16.375 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:42:16.375 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:42:16.375 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:42:16.583 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:42:16.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:16.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:42:16.693 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:42:16.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:16 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/293630326' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:16.706 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:16 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/293630326' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:16.718 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:16 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/293630326' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:16.723 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:16.769 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:42:16.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:16.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:42:16.875 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:16.903 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:17.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:42:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:42:16] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2023-09-23T23:42:17.342 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:42:17.498 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:17.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:17.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:17.585 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:17 smithi110 ceph-mon[212939]: pgmap v1897: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 729 op/s; 870/103533 objects degraded (0.840%); 0 B/s, 68 objects/s recovering 2023-09-23T23:42:17.585 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:17 smithi110 ceph-mon[221102]: pgmap v1897: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 729 op/s; 870/103533 objects degraded (0.840%); 0 B/s, 68 objects/s recovering 2023-09-23T23:42:17.608 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:17.722 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:17.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:17 smithi138 ceph-mon[193606]: pgmap v1897: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 729 op/s; 870/103533 objects degraded (0.840%); 0 B/s, 68 objects/s recovering 2023-09-23T23:42:18.179 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:18.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:18.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:18.289 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:42:18.372 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:18.465 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:42:18.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:18.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:42:18.509 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T23:42:18.585 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:42:18.679 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2931154892' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:42:18.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:18 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:42:18.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2931154892' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:42:18.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:18 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:42:18.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2931154892' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:42:18.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:18 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:42:19.377 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:42:19.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:19.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:42:19.489 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:42:19.542 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:19.658 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:19 smithi110 ceph-mon[212939]: pgmap v1898: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 636 op/s; 870/98709 objects degraded (0.881%); 0 B/s, 59 objects/s recovering 2023-09-23T23:42:19.659 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:19 smithi110 ceph-mon[212939]: failed to encode map e666 with expected crc 2023-09-23T23:42:19.659 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:19 smithi110 ceph-mon[212939]: failed to encode map e666 with expected crc 2023-09-23T23:42:19.659 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:19 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:42:19.659 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:19 smithi110 ceph-mon[212939]: osdmap e666: 8 total, 8 up, 6 in 2023-09-23T23:42:19.659 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:19 smithi110 ceph-mon[212939]: failed to encode map e666 with expected crc 2023-09-23T23:42:19.659 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:19 smithi110 ceph-mon[212939]: Health check update: Degraded data redundancy: 870/98709 objects degraded (0.881%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:42:19.664 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:19 smithi110 ceph-mon[221102]: pgmap v1898: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 636 op/s; 870/98709 objects degraded (0.881%); 0 B/s, 59 objects/s recovering 2023-09-23T23:42:19.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:19 smithi110 ceph-mon[221102]: failed to encode map e666 with expected crc 2023-09-23T23:42:19.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:19 smithi110 ceph-mon[221102]: failed to encode map e666 with expected crc 2023-09-23T23:42:19.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:19 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:42:19.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:19 smithi110 ceph-mon[221102]: osdmap e666: 8 total, 8 up, 6 in 2023-09-23T23:42:19.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:19 smithi110 ceph-mon[221102]: failed to encode map e666 with expected crc 2023-09-23T23:42:19.665 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:19 smithi110 ceph-mon[221102]: Health check update: Degraded data redundancy: 870/98709 objects degraded (0.881%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:42:19.691 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:42:19.836 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:42:19.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:19.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:42:19.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:19 smithi138 ceph-mon[193606]: pgmap v1898: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 636 op/s; 870/98709 objects degraded (0.881%); 0 B/s, 59 objects/s recovering 2023-09-23T23:42:19.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:19 smithi138 ceph-mon[193606]: failed to encode map e666 with expected crc 2023-09-23T23:42:19.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:19 smithi138 ceph-mon[193606]: failed to encode map e666 with expected crc 2023-09-23T23:42:19.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:19 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:42:19.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:19 smithi138 ceph-mon[193606]: osdmap e666: 8 total, 8 up, 6 in 2023-09-23T23:42:19.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:19 smithi138 ceph-mon[193606]: failed to encode map e666 with expected crc 2023-09-23T23:42:19.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:19 smithi138 ceph-mon[193606]: Health check update: Degraded data redundancy: 870/98709 objects degraded (0.881%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:42:19.951 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:19.958 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:19.971 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T23:42:19.978 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:20.022 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:20.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:20.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:20.129 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:20.156 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:20.200 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:20.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:20.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:20.311 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:42:20.338 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:20.383 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:42:20.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:20.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:42:20.489 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:20.581 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:20.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:20 smithi138 ceph-mon[193606]: failed to encode map e666 with expected crc 2023-09-23T23:42:20.839 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:20.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:20.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:20.845 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:20 smithi110 ceph-mon[212939]: failed to encode map e666 with expected crc 2023-09-23T23:42:20.845 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:20 smithi110 ceph-mon[221102]: failed to encode map e666 with expected crc 2023-09-23T23:42:20.946 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:42:20.949 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:20.979 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:21.130 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:42:21.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:21.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:42:21.233 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:21.260 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:21.303 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:21.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:21.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:21.408 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:42:21.436 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:21.506 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:42:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:21 smithi138 ceph-mon[193606]: pgmap v1900: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 636 op/s; 870/98037 objects degraded (0.887%); 0 B/s, 59 objects/s recovering 2023-09-23T23:42:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:21 smithi138 ceph-mon[193606]: failed to encode map e667 with expected crc 2023-09-23T23:42:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:21 smithi138 ceph-mon[193606]: failed to encode map e667 with expected crc 2023-09-23T23:42:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:21 smithi138 ceph-mon[193606]: osdmap e667: 8 total, 8 up, 6 in 2023-09-23T23:42:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:21 smithi138 ceph-mon[193606]: failed to encode map e667 with expected crc 2023-09-23T23:42:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:21 smithi138 ceph-mon[193606]: failed to encode map e667 with expected crc 2023-09-23T23:42:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:21 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1118007992' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:42:21.700 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:42:21.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:21 smithi110 ceph-mon[212939]: pgmap v1900: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 636 op/s; 870/98037 objects degraded (0.887%); 0 B/s, 59 objects/s recovering 2023-09-23T23:42:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:21 smithi110 ceph-mon[212939]: failed to encode map e667 with expected crc 2023-09-23T23:42:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:21 smithi110 ceph-mon[212939]: failed to encode map e667 with expected crc 2023-09-23T23:42:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:21 smithi110 ceph-mon[212939]: osdmap e667: 8 total, 8 up, 6 in 2023-09-23T23:42:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:21 smithi110 ceph-mon[212939]: failed to encode map e667 with expected crc 2023-09-23T23:42:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:21 smithi110 ceph-mon[212939]: failed to encode map e667 with expected crc 2023-09-23T23:42:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:21 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1118007992' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:42:21.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:21 smithi110 ceph-mon[221102]: pgmap v1900: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 636 op/s; 870/98037 objects degraded (0.887%); 0 B/s, 59 objects/s recovering 2023-09-23T23:42:21.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:21 smithi110 ceph-mon[221102]: failed to encode map e667 with expected crc 2023-09-23T23:42:21.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:21 smithi110 ceph-mon[221102]: failed to encode map e667 with expected crc 2023-09-23T23:42:21.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:21 smithi110 ceph-mon[221102]: osdmap e667: 8 total, 8 up, 6 in 2023-09-23T23:42:21.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:21 smithi110 ceph-mon[221102]: failed to encode map e667 with expected crc 2023-09-23T23:42:21.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:21 smithi110 ceph-mon[221102]: failed to encode map e667 with expected crc 2023-09-23T23:42:21.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:21 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1118007992' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:42:21.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:21.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:42:21.804 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:21.874 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:22.174 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:22.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:22.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:22.280 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:22.307 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:22.348 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:22.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:22.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:22.454 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:22.480 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:22.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:22 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:42:22.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:22 smithi138 ceph-mon[193606]: failed to encode map e668 with expected crc 2023-09-23T23:42:22.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:22 smithi138 ceph-mon[193606]: failed to encode map e668 with expected crc 2023-09-23T23:42:22.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:22 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1118007992' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:42:22.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:22 smithi138 ceph-mon[193606]: osdmap e668: 8 total, 8 up, 6 in 2023-09-23T23:42:22.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:22 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:42:22.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:22 smithi110 ceph-mon[221102]: failed to encode map e668 with expected crc 2023-09-23T23:42:22.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:22 smithi110 ceph-mon[221102]: failed to encode map e668 with expected crc 2023-09-23T23:42:22.935 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:22 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1118007992' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:42:22.936 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:22 smithi110 ceph-mon[221102]: osdmap e668: 8 total, 8 up, 6 in 2023-09-23T23:42:22.936 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:22 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:42:22.936 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:22 smithi110 ceph-mon[212939]: failed to encode map e668 with expected crc 2023-09-23T23:42:22.936 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:22 smithi110 ceph-mon[212939]: failed to encode map e668 with expected crc 2023-09-23T23:42:22.936 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:22 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1118007992' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:42:22.936 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:22 smithi110 ceph-mon[212939]: osdmap e668: 8 total, 8 up, 6 in 2023-09-23T23:42:23.004 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:23.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:23.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:23.055 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:23.108 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:42:23.137 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:23.187 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:42:23.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:23.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:42:23.293 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:42:23.319 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:23.371 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:42:23.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:23.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:42:23.477 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:42:23.563 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:23 smithi138 ceph-mon[193606]: pgmap v1903: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 117 active+clean; 1.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 553 op/s; 0 B/s, 40 objects/s recovering 2023-09-23T23:42:23.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:23 smithi138 ceph-mon[193606]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 870/98037 objects degraded (0.887%), 2 pgs degraded, 1 pg undersized) 2023-09-23T23:42:23.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:23 smithi110 ceph-mon[212939]: pgmap v1903: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 117 active+clean; 1.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 553 op/s; 0 B/s, 40 objects/s recovering 2023-09-23T23:42:23.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:23 smithi110 ceph-mon[212939]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 870/98037 objects degraded (0.887%), 2 pgs degraded, 1 pg undersized) 2023-09-23T23:42:23.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:23 smithi110 ceph-mon[221102]: pgmap v1903: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 117 active+clean; 1.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 553 op/s; 0 B/s, 40 objects/s recovering 2023-09-23T23:42:23.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:23 smithi110 ceph-mon[221102]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 870/98037 objects degraded (0.887%), 2 pgs degraded, 1 pg undersized) 2023-09-23T23:42:24.669 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:42:24.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:24.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:42:24.784 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:42:24.813 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:24.864 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:42:24.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:24.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:42:24.919 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:42:24.919 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:42:24.959 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:24 smithi110 ceph-mon[221102]: pgmap v1904: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 117 active+clean; 1.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 511 op/s; 0 B/s, 40 objects/s recovering 2023-09-23T23:42:24.960 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:24 smithi110 ceph-mon[212939]: pgmap v1904: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 117 active+clean; 1.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 511 op/s; 0 B/s, 40 objects/s recovering 2023-09-23T23:42:24.973 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:25.047 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:25.087 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:42:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:24 smithi138 ceph-mon[193606]: pgmap v1904: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 117 active+clean; 1.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 511 op/s; 0 B/s, 40 objects/s recovering 2023-09-23T23:42:26.671 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:26.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:26.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:26.684 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:42:26.781 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:26.907 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:26.954 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:42:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:42:26] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2023-09-23T23:42:27.090 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:27.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:27.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:27.202 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:27.277 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:27 smithi110 ceph-mon[212939]: pgmap v1905: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 592 op/s; 0 B/s, 95 objects/s recovering 2023-09-23T23:42:27.280 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:27 smithi110 ceph-mon[221102]: pgmap v1905: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 592 op/s; 0 B/s, 95 objects/s recovering 2023-09-23T23:42:27.289 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:27.341 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:27.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:27 smithi138 ceph-mon[193606]: pgmap v1905: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 592 op/s; 0 B/s, 95 objects/s recovering 2023-09-23T23:42:27.461 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:42:27.467 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:27.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:27.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:27.500 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:42:27.544 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:42:27.577 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:42:27.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:27.611 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:42:27.663 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:42:27.688 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:27.719 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:42:27.755 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:42:27.789 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:42:27.789 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:42:27.790 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:42:27.790 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:42:27.790 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:42:27.790 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:42:27.790 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:42:27.790 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:42:27.812 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:27.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:27.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:27.928 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:42:28.000 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:28.249 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T23:42:28.276 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:28 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/769931794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:28.276 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:28.276 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:42:28.276 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:28 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3700888988' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:42:28.276 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:28 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:42:28.277 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:28 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/769931794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:28.277 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:28.277 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:42:28.277 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:28 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3700888988' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:42:28.277 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:28 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:42:28.632 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:28 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/769931794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:28.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:42:28.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:28 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3700888988' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:42:28.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:28 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:42:29.684 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:29 smithi110 ceph-mon[221102]: pgmap v1906: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 759 op/s; 0 B/s, 90 objects/s recovering 2023-09-23T23:42:29.685 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:29 smithi110 ceph-mon[221102]: failed to encode map e669 with expected crc 2023-09-23T23:42:29.685 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:29 smithi110 ceph-mon[221102]: failed to encode map e669 with expected crc 2023-09-23T23:42:29.685 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:29 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:42:29.685 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:29 smithi110 ceph-mon[221102]: osdmap e669: 8 total, 8 up, 6 in 2023-09-23T23:42:29.685 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:29 smithi110 ceph-mon[221102]: osdmap e670: 8 total, 8 up, 6 in 2023-09-23T23:42:29.685 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:29 smithi110 ceph-mon[221102]: failed to encode map e670 with expected crc 2023-09-23T23:42:29.686 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:29 smithi110 ceph-mon[212939]: pgmap v1906: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 759 op/s; 0 B/s, 90 objects/s recovering 2023-09-23T23:42:29.686 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:29 smithi110 ceph-mon[212939]: failed to encode map e669 with expected crc 2023-09-23T23:42:29.686 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:29 smithi110 ceph-mon[212939]: failed to encode map e669 with expected crc 2023-09-23T23:42:29.686 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:29 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:42:29.686 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:29 smithi110 ceph-mon[212939]: osdmap e669: 8 total, 8 up, 6 in 2023-09-23T23:42:29.686 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:29 smithi110 ceph-mon[212939]: osdmap e670: 8 total, 8 up, 6 in 2023-09-23T23:42:29.686 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:29 smithi110 ceph-mon[212939]: failed to encode map e670 with expected crc 2023-09-23T23:42:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:29 smithi138 ceph-mon[193606]: pgmap v1906: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 759 op/s; 0 B/s, 90 objects/s recovering 2023-09-23T23:42:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:29 smithi138 ceph-mon[193606]: failed to encode map e669 with expected crc 2023-09-23T23:42:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:29 smithi138 ceph-mon[193606]: failed to encode map e669 with expected crc 2023-09-23T23:42:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:29 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:42:29.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:29 smithi138 ceph-mon[193606]: osdmap e669: 8 total, 8 up, 6 in 2023-09-23T23:42:29.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:29 smithi138 ceph-mon[193606]: osdmap e670: 8 total, 8 up, 6 in 2023-09-23T23:42:29.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:29 smithi138 ceph-mon[193606]: failed to encode map e670 with expected crc 2023-09-23T23:42:30.146 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:42:30.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:30.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:42:30.249 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:30.314 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:30.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:30 smithi110 ceph-mon[221102]: failed to encode map e669 with expected crc 2023-09-23T23:42:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:30 smithi110 ceph-mon[221102]: failed to encode map e670 with expected crc 2023-09-23T23:42:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:30 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:42:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:30 smithi110 ceph-mon[221102]: osdmap e671: 8 total, 8 up, 6 in 2023-09-23T23:42:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:30 smithi110 ceph-mon[221102]: failed to encode map e671 with expected crc 2023-09-23T23:42:30.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:30 smithi110 ceph-mon[212939]: failed to encode map e669 with expected crc 2023-09-23T23:42:30.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:30 smithi110 ceph-mon[212939]: failed to encode map e670 with expected crc 2023-09-23T23:42:30.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:30 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:42:30.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:30 smithi110 ceph-mon[212939]: osdmap e671: 8 total, 8 up, 6 in 2023-09-23T23:42:30.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:30 smithi110 ceph-mon[212939]: failed to encode map e671 with expected crc 2023-09-23T23:42:30.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:30 smithi138 ceph-mon[193606]: failed to encode map e669 with expected crc 2023-09-23T23:42:30.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:30 smithi138 ceph-mon[193606]: failed to encode map e670 with expected crc 2023-09-23T23:42:30.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:30 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:42:30.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:30 smithi138 ceph-mon[193606]: osdmap e671: 8 total, 8 up, 6 in 2023-09-23T23:42:30.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:30 smithi138 ceph-mon[193606]: failed to encode map e671 with expected crc 2023-09-23T23:42:31.054 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:31.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:31.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:31.084 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T23:42:31.084 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:42:31.160 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:42:31.193 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:31.363 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:42:31.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:31.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:42:31.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:31 smithi138 ceph-mon[193606]: failed to encode map e671 with expected crc 2023-09-23T23:42:31.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:31 smithi138 ceph-mon[193606]: pgmap v1910: 119 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 117 active+clean; 1.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 566 op/s; 0 B/s, 80 objects/s recovering 2023-09-23T23:42:31.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:31 smithi138 ceph-mon[193606]: failed to encode map e671 with expected crc 2023-09-23T23:42:31.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:31 smithi138 ceph-mon[193606]: failed to encode map e671 with expected crc 2023-09-23T23:42:31.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:31 smithi138 ceph-mon[193606]: failed to encode map e672 with expected crc 2023-09-23T23:42:31.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:31 smithi138 ceph-mon[193606]: osdmap e672: 8 total, 8 up, 6 in 2023-09-23T23:42:31.468 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:31.474 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:31 smithi110 ceph-mon[212939]: failed to encode map e671 with expected crc 2023-09-23T23:42:31.474 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:31 smithi110 ceph-mon[212939]: pgmap v1910: 119 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 117 active+clean; 1.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 566 op/s; 0 B/s, 80 objects/s recovering 2023-09-23T23:42:31.474 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:31 smithi110 ceph-mon[212939]: failed to encode map e671 with expected crc 2023-09-23T23:42:31.474 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:31 smithi110 ceph-mon[212939]: failed to encode map e671 with expected crc 2023-09-23T23:42:31.474 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:31 smithi110 ceph-mon[212939]: failed to encode map e672 with expected crc 2023-09-23T23:42:31.474 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:31 smithi110 ceph-mon[212939]: osdmap e672: 8 total, 8 up, 6 in 2023-09-23T23:42:31.474 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:31 smithi110 ceph-mon[221102]: failed to encode map e671 with expected crc 2023-09-23T23:42:31.475 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:31 smithi110 ceph-mon[221102]: pgmap v1910: 119 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 117 active+clean; 1.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 566 op/s; 0 B/s, 80 objects/s recovering 2023-09-23T23:42:31.475 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:31 smithi110 ceph-mon[221102]: failed to encode map e671 with expected crc 2023-09-23T23:42:31.475 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:31 smithi110 ceph-mon[221102]: failed to encode map e671 with expected crc 2023-09-23T23:42:31.475 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:31 smithi110 ceph-mon[221102]: failed to encode map e672 with expected crc 2023-09-23T23:42:31.475 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:31 smithi110 ceph-mon[221102]: osdmap e672: 8 total, 8 up, 6 in 2023-09-23T23:42:31.496 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:31.696 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:31.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:31.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:31.802 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:42:31.834 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:31.885 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:42:31.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:31.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:42:31.994 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:32.022 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:32.290 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:42:32.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:32 smithi110 ceph-mon[221102]: failed to encode map e672 with expected crc 2023-09-23T23:42:32.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:32 smithi110 ceph-mon[221102]: failed to encode map e673 with expected crc 2023-09-23T23:42:32.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:32 smithi110 ceph-mon[221102]: osdmap e673: 8 total, 8 up, 6 in 2023-09-23T23:42:32.358 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:32 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2877779293' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:42:32.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:32 smithi110 ceph-mon[212939]: failed to encode map e672 with expected crc 2023-09-23T23:42:32.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:32 smithi110 ceph-mon[212939]: failed to encode map e673 with expected crc 2023-09-23T23:42:32.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:32 smithi110 ceph-mon[212939]: osdmap e673: 8 total, 8 up, 6 in 2023-09-23T23:42:32.359 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:32 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2877779293' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:42:32.450 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:32.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:32.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:32.559 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:42:32.595 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:32 smithi138 ceph-mon[193606]: failed to encode map e672 with expected crc 2023-09-23T23:42:32.595 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:32 smithi138 ceph-mon[193606]: failed to encode map e673 with expected crc 2023-09-23T23:42:32.595 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:32 smithi138 ceph-mon[193606]: osdmap e673: 8 total, 8 up, 6 in 2023-09-23T23:42:32.596 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:32 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2877779293' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:42:32.597 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:32.694 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:42:32.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:32.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:42:32.716 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:32.798 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:32.905 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:33.014 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:42:33.310 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:33.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:33.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:33.427 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:42:33.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:33 smithi110 ceph-mon[212939]: failed to encode map e673 with expected crc 2023-09-23T23:42:33.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:33 smithi110 ceph-mon[212939]: pgmap v1913: 119 pgs: 1 remapped+peering, 118 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 385 op/s; 0 B/s, 36 objects/s recovering 2023-09-23T23:42:33.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:33 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:42:33.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:33 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2877779293' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:42:33.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:33 smithi110 ceph-mon[212939]: osdmap e674: 8 total, 8 up, 6 in 2023-09-23T23:42:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:33 smithi110 ceph-mon[221102]: failed to encode map e673 with expected crc 2023-09-23T23:42:33.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:33 smithi110 ceph-mon[221102]: pgmap v1913: 119 pgs: 1 remapped+peering, 118 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 385 op/s; 0 B/s, 36 objects/s recovering 2023-09-23T23:42:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:33 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:42:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:33 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2877779293' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:42:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:33 smithi110 ceph-mon[221102]: osdmap e674: 8 total, 8 up, 6 in 2023-09-23T23:42:33.524 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:33 smithi138 ceph-mon[193606]: failed to encode map e673 with expected crc 2023-09-23T23:42:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:33 smithi138 ceph-mon[193606]: pgmap v1913: 119 pgs: 1 remapped+peering, 118 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 385 op/s; 0 B/s, 36 objects/s recovering 2023-09-23T23:42:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:33 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:42:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:33 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2877779293' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:42:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:33 smithi138 ceph-mon[193606]: osdmap e674: 8 total, 8 up, 6 in 2023-09-23T23:42:34.796 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:42:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:42:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:42:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:42:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71524 13.2 13:443908cd:::benchmark_data_smithi110_541928_object71523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e555)", 2023-09-23T23:42:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.543571+0000", 2023-09-23T23:42:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 523.08144325000001, 2023-09-23T23:42:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.309790005, 2023-09-23T23:42:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71524 2023-09-23T23:42:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543571+0000", 2023-09-23T23:42:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543571+0000", 2023-09-23T23:42:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543570+0000", 2023-09-23T23:42:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543592+0000", 2023-09-23T23:42:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1195e-05 2023-09-23T23:42:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543598+0000", 2023-09-23T23:42:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8190000000000003e-06 2023-09-23T23:42:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.543604+0000", 2023-09-23T23:42:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.451e-06 2023-09-23T23:42:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.544968+0000", 2023-09-23T23:42:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001364007 2023-09-23T23:42:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.544984+0000", 2023-09-23T23:42:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5699999999999999e-05 2023-09-23T23:42:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527908+0000", 2023-09-23T23:42:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98292410900000005 2023-09-23T23:42:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527920+0000", 2023-09-23T23:42:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2852e-05 2023-09-23T23:42:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.815907+0000", 2023-09-23T23:42:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.287986606 2023-09-23T23:42:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.816011+0000", 2023-09-23T23:42:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103791 2023-09-23T23:42:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:42:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.816116+0000", 2023-09-23T23:42:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104955 2023-09-23T23:42:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.825412+0000", 2023-09-23T23:42:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.009295757 2023-09-23T23:42:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.847154+0000", 2023-09-23T23:42:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021742053000000001 2023-09-23T23:42:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.850549+0000", 2023-09-23T23:42:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033955470000000001 2023-09-23T23:42:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.850599+0000", 2023-09-23T23:42:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9645000000000003e-05 2023-09-23T23:42:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.853361+0000", 2023-09-23T23:42:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027620710000000001 2023-09-23T23:42:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71514 13.3 13:cdfd1184:::benchmark_data_smithi110_541928_object71513:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:42:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.547244+0000", 2023-09-23T23:42:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 523.07777012600002, 2023-09-23T23:42:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4756996080000002, 2023-09-23T23:42:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-09-23T23:42:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71514 2023-09-23T23:42:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547244+0000", 2023-09-23T23:42:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547244+0000", 2023-09-23T23:42:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547242+0000", 2023-09-23T23:42:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:42:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547281+0000", 2023-09-23T23:42:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8605000000000001e-05 2023-09-23T23:42:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547302+0000", 2023-09-23T23:42:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0873000000000002e-05 2023-09-23T23:42:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547312+0000", 2023-09-23T23:42:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.859e-06 2023-09-23T23:42:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.548286+0000", 2023-09-23T23:42:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00097434599999999996 2023-09-23T23:42:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.548298+0000", 2023-09-23T23:42:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1881e-05 2023-09-23T23:42:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527773+0000", 2023-09-23T23:42:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97947559500000003 2023-09-23T23:42:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527787+0000", 2023-09-23T23:42:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3329999999999999e-05 2023-09-23T23:42:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.021377+0000", 2023-09-23T23:42:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.493589799 2023-09-23T23:42:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.022944+0000", 2023-09-23T23:42:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015670910000000001 2023-09-23T23:42:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71522 13.3 13:c49d694d:::benchmark_data_smithi110_541928_object71521:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:42:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.547319+0000", 2023-09-23T23:42:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 523.07769477900001, 2023-09-23T23:42:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5115680760000001, 2023-09-23T23:42:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71522 2023-09-23T23:42:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547319+0000", 2023-09-23T23:42:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547319+0000", 2023-09-23T23:42:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547319+0000", 2023-09-23T23:42:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547342+0000", 2023-09-23T23:42:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2857000000000002e-05 2023-09-23T23:42:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547348+0000", 2023-09-23T23:42:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8129999999999998e-06 2023-09-23T23:42:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.547354+0000", 2023-09-23T23:42:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5090000000000001e-06 2023-09-23T23:42:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.548319+0000", 2023-09-23T23:42:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00096521499999999995 2023-09-23T23:42:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.548327+0000", 2023-09-23T23:42:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7079999999999997e-06 2023-09-23T23:42:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527817+0000", 2023-09-23T23:42:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97948980500000005 2023-09-23T23:42:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527825+0000", 2023-09-23T23:42:34.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3440000000000001e-06 2023-09-23T23:42:34.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.026234+0000", 2023-09-23T23:42:34.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4984093380000001 2023-09-23T23:42:34.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.027045+0000", 2023-09-23T23:42:34.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00081087800000000001 2023-09-23T23:42:34.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,7", 2023-09-23T23:42:34.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.027252+0000", 2023-09-23T23:42:34.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000206553 2023-09-23T23:42:34.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.038654+0000", 2023-09-23T23:42:34.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011401669 2023-09-23T23:42:34.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.819 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.819 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.046827+0000", 2023-09-23T23:42:34.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081732569999999997 2023-09-23T23:42:34.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.819 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.819 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.055749+0000", 2023-09-23T23:42:34.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0089217600000000008 2023-09-23T23:42:34.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.819 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.819 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.055791+0000", 2023-09-23T23:42:34.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2493999999999997e-05 2023-09-23T23:42:34.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.058887+0000", 2023-09-23T23:42:34.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030964320000000001 2023-09-23T23:42:34.820 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.820 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.820 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.820 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71530 13.2 13:4cd5e89c:::benchmark_data_smithi110_541928_object71529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:42:34.821 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.571622+0000", 2023-09-23T23:42:34.821 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 523.05339224800002, 2023-09-23T23:42:34.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2848813030000001, 2023-09-23T23:42:34.821 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.821 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.821 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.821 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:34.821 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:34.821 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71530 2023-09-23T23:42:34.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.821 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571622+0000", 2023-09-23T23:42:34.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571622+0000", 2023-09-23T23:42:34.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571620+0000", 2023-09-23T23:42:34.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:34.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571678+0000", 2023-09-23T23:42:34.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7633000000000002e-05 2023-09-23T23:42:34.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571696+0000", 2023-09-23T23:42:34.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8215000000000001e-05 2023-09-23T23:42:34.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571714+0000", 2023-09-23T23:42:34.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7839e-05 2023-09-23T23:42:34.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571774+0000", 2023-09-23T23:42:34.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0322999999999998e-05 2023-09-23T23:42:34.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.571784+0000", 2023-09-23T23:42:34.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1719999999999996e-06 2023-09-23T23:42:34.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527947+0000", 2023-09-23T23:42:34.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95616350999999999 2023-09-23T23:42:34.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527957+0000", 2023-09-23T23:42:34.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9369999999999999e-06 2023-09-23T23:42:34.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.820763+0000", 2023-09-23T23:42:34.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.292805757 2023-09-23T23:42:34.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.820924+0000", 2023-09-23T23:42:34.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016086599999999999 2023-09-23T23:42:34.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:42:34.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.821115+0000", 2023-09-23T23:42:34.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000191137 2023-09-23T23:42:34.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.825434+0000", 2023-09-23T23:42:34.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043196479999999997 2023-09-23T23:42:34.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.847182+0000", 2023-09-23T23:42:34.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021747295999999999 2023-09-23T23:42:34.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.853421+0000", 2023-09-23T23:42:34.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.006239306 2023-09-23T23:42:34.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.853476+0000", 2023-09-23T23:42:34.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4929000000000002e-05 2023-09-23T23:42:34.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.856503+0000", 2023-09-23T23:42:34.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030272390000000001 2023-09-23T23:42:34.828 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.829 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.829 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.829 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71537 13.3 13:cd57dd3d:::benchmark_data_smithi110_541928_object71536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:42:34.829 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.580860+0000", 2023-09-23T23:42:34.829 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 523.04415446899998, 2023-09-23T23:42:34.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4840317629999999, 2023-09-23T23:42:34.829 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.829 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.829 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.829 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:34.829 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:34.829 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71537 2023-09-23T23:42:34.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.830 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580860+0000", 2023-09-23T23:42:34.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580860+0000", 2023-09-23T23:42:34.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580858+0000", 2023-09-23T23:42:34.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:42:34.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580883+0000", 2023-09-23T23:42:34.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5388000000000002e-05 2023-09-23T23:42:34.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580901+0000", 2023-09-23T23:42:34.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7496e-05 2023-09-23T23:42:34.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580911+0000", 2023-09-23T23:42:34.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0093e-05 2023-09-23T23:42:34.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580968+0000", 2023-09-23T23:42:34.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6971000000000001e-05 2023-09-23T23:42:34.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.580979+0000", 2023-09-23T23:42:34.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1423999999999999e-05 2023-09-23T23:42:34.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527852+0000", 2023-09-23T23:42:34.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94687271200000001 2023-09-23T23:42:34.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527860+0000", 2023-09-23T23:42:34.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8159999999999994e-06 2023-09-23T23:42:34.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.029524+0000", 2023-09-23T23:42:34.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.501664385 2023-09-23T23:42:34.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.029705+0000", 2023-09-23T23:42:34.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000181423 2023-09-23T23:42:34.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,7", 2023-09-23T23:42:34.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.029886+0000", 2023-09-23T23:42:34.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018058199999999999 2023-09-23T23:42:34.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.038692+0000", 2023-09-23T23:42:34.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0088061979999999995 2023-09-23T23:42:34.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.046831+0000", 2023-09-23T23:42:34.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081387630000000002 2023-09-23T23:42:34.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.058949+0000", 2023-09-23T23:42:34.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012117909 2023-09-23T23:42:34.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.059022+0000", 2023-09-23T23:42:34.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2758999999999999e-05 2023-09-23T23:42:34.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.064891+0000", 2023-09-23T23:42:34.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058699099999999999 2023-09-23T23:42:34.837 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.837 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.837 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.837 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:71540 13.2 13:4d0dc391:::benchmark_data_smithi110_541928_object71539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:42:34.837 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:33:50.586761+0000", 2023-09-23T23:42:34.837 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 523.03825345300004, 2023-09-23T23:42:34.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4196291790000002, 2023-09-23T23:42:34.838 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.838 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.838 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.838 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:34.838 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:34.838 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71540 2023-09-23T23:42:34.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.838 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586761+0000", 2023-09-23T23:42:34.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586761+0000", 2023-09-23T23:42:34.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586758+0000", 2023-09-23T23:42:34.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-09-23T23:42:34.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586795+0000", 2023-09-23T23:42:34.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6177999999999998e-05 2023-09-23T23:42:34.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586821+0000", 2023-09-23T23:42:34.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6081999999999999e-05 2023-09-23T23:42:34.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586834+0000", 2023-09-23T23:42:34.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3441999999999999e-05 2023-09-23T23:42:34.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.841 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.841 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586917+0000", 2023-09-23T23:42:34.841 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2551000000000003e-05 2023-09-23T23:42:34.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.841 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.841 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:50.586938+0000", 2023-09-23T23:42:34.841 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1016e-05 2023-09-23T23:42:34.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.842 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.842 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527980+0000", 2023-09-23T23:42:34.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94104277599999997 2023-09-23T23:42:34.842 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.842 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.842 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:51.527989+0000", 2023-09-23T23:42:34.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7439999999999996e-06 2023-09-23T23:42:34.842 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.842 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.842 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.823717+0000", 2023-09-23T23:42:34.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.295727691 2023-09-23T23:42:34.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.823855+0000", 2023-09-23T23:42:34.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013797700000000001 2023-09-23T23:42:34.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:42:34.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.823956+0000", 2023-09-23T23:42:34.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010142899999999999 2023-09-23T23:42:34.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.825554+0000", 2023-09-23T23:42:34.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015978839999999999 2023-09-23T23:42:34.844 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.849319+0000", 2023-09-23T23:42:34.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023764928000000001 2023-09-23T23:42:34.844 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.856559+0000", 2023-09-23T23:42:34.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0072398460000000003 2023-09-23T23:42:34.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.845 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:52.856611+0000", 2023-09-23T23:42:34.845 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1752999999999997e-05 2023-09-23T23:42:34.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.845 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:33:53.006390+0000", 2023-09-23T23:42:34.845 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14977916799999999 2023-09-23T23:42:34.845 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.845 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.845 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.846 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89738 13.3 13:cade2564:::benchmark_data_smithi110_541928_object1958:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:42:34.846 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:34.801877+0000", 2023-09-23T23:42:34.846 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 478.82313687099997, 2023-09-23T23:42:34.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5955823369999997, 2023-09-23T23:42:34.846 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.846 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.846 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.846 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:34.846 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:34.846 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89738 2023-09-23T23:42:34.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.847 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.847 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.847 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801877+0000", 2023-09-23T23:42:34.847 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.847 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.847 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801877+0000", 2023-09-23T23:42:34.847 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801876+0000", 2023-09-23T23:42:34.848 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:34.848 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.848 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801883+0000", 2023-09-23T23:42:34.848 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8959999999999997e-06 2023-09-23T23:42:34.848 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.848 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801895+0000", 2023-09-23T23:42:34.848 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2709e-05 2023-09-23T23:42:34.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.849 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.849 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.849 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801905+0000", 2023-09-23T23:42:34.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1589999999999999e-06 2023-09-23T23:42:34.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.849 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.849 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.849 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801984+0000", 2023-09-23T23:42:34.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9043999999999997e-05 2023-09-23T23:42:34.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.849 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801997+0000", 2023-09-23T23:42:34.850 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3294e-05 2023-09-23T23:42:34.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731046+0000", 2023-09-23T23:42:34.850 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.92904877500000005 2023-09-23T23:42:34.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731060+0000", 2023-09-23T23:42:34.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4042e-05 2023-09-23T23:42:34.851 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.851 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.851 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.371733+0000", 2023-09-23T23:42:34.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6406730409999999 2023-09-23T23:42:34.851 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.851 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.851 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:42:34.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.372202+0000", 2023-09-23T23:42:34.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000469529 2023-09-23T23:42:34.851 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.851 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.852 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.852 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.381823+0000", 2023-09-23T23:42:34.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0096204110000000006 2023-09-23T23:42:34.852 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.852 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.852 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.852 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.381932+0000", 2023-09-23T23:42:34.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010885 2023-09-23T23:42:34.852 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.852 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.852 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:42:34.852 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.382080+0000", 2023-09-23T23:42:34.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000148555 2023-09-23T23:42:34.853 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.853 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.853 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394009+0000", 2023-09-23T23:42:34.853 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011928887000000001 2023-09-23T23:42:34.853 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.853 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.853 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394057+0000", 2023-09-23T23:42:34.853 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8157e-05 2023-09-23T23:42:34.853 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394066+0000", 2023-09-23T23:42:34.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7320000000000004e-06 2023-09-23T23:42:34.854 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.854 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394132+0000", 2023-09-23T23:42:34.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5845000000000004e-05 2023-09-23T23:42:34.854 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.854 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394208+0000", 2023-09-23T23:42:34.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6352000000000004e-05 2023-09-23T23:42:34.855 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.855 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.855 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.855 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397460+0000", 2023-09-23T23:42:34.855 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032514520000000002 2023-09-23T23:42:34.855 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.855 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.855 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.855 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.855 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.855 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89748 13.3 13:c6535c6c:::benchmark_data_smithi110_541928_object1968:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:42:34.855 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:34.962737+0000", 2023-09-23T23:42:34.855 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 478.66227737200001, 2023-09-23T23:42:34.856 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.437414017, 2023-09-23T23:42:34.856 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.856 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.856 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.856 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:34.856 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:34.856 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89748 2023-09-23T23:42:34.856 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.856 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.856 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.856 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.856 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962737+0000", 2023-09-23T23:42:34.856 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.857 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.857 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.857 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.857 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962737+0000", 2023-09-23T23:42:34.857 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.857 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.857 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.857 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.857 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962735+0000", 2023-09-23T23:42:34.857 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:42:34.857 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.857 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.858 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.858 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962743+0000", 2023-09-23T23:42:34.858 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7700000000000001e-06 2023-09-23T23:42:34.858 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.858 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.858 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962756+0000", 2023-09-23T23:42:34.858 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3013999999999999e-05 2023-09-23T23:42:34.858 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.858 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.858 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962763+0000", 2023-09-23T23:42:34.859 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7260000000000002e-06 2023-09-23T23:42:34.859 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.859 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962867+0000", 2023-09-23T23:42:34.859 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103978 2023-09-23T23:42:34.859 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.859 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962878+0000", 2023-09-23T23:42:34.859 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0927999999999999e-05 2023-09-23T23:42:34.859 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731084+0000", 2023-09-23T23:42:34.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.76820582999999998 2023-09-23T23:42:34.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731093+0000", 2023-09-23T23:42:34.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6149999999999997e-06 2023-09-23T23:42:34.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.375727+0000", 2023-09-23T23:42:34.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6446342889999999 2023-09-23T23:42:34.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:42:34.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.376199+0000", 2023-09-23T23:42:34.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047155199999999997 2023-09-23T23:42:34.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.390857+0000", 2023-09-23T23:42:34.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014658200999999999 2023-09-23T23:42:34.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.390980+0000", 2023-09-23T23:42:34.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000122774 2023-09-23T23:42:34.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:42:34.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.391164+0000", 2023-09-23T23:42:34.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018492700000000001 2023-09-23T23:42:34.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397524+0000", 2023-09-23T23:42:34.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063597879999999999 2023-09-23T23:42:34.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.863 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397564+0000", 2023-09-23T23:42:34.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9746999999999997e-05 2023-09-23T23:42:34.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.863 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397611+0000", 2023-09-23T23:42:34.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7403e-05 2023-09-23T23:42:34.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.864 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.864 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.864 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397668+0000", 2023-09-23T23:42:34.864 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6178999999999999e-05 2023-09-23T23:42:34.864 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.864 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.864 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.864 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397732+0000", 2023-09-23T23:42:34.864 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4548999999999995e-05 2023-09-23T23:42:34.864 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.864 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.864 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.864 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.400151+0000", 2023-09-23T23:42:34.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024186429999999998 2023-09-23T23:42:34.865 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.865 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.865 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.865 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.865 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89756 13.3 13:cf60a43c:::benchmark_data_smithi110_541928_object1976:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:42:34.865 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.051526+0000", 2023-09-23T23:42:34.865 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 478.57348859899997, 2023-09-23T23:42:34.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4865002390000002, 2023-09-23T23:42:34.865 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.865 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.865 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.866 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:34.866 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:34.866 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89756 2023-09-23T23:42:34.866 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.866 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.866 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.866 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.866 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051526+0000", 2023-09-23T23:42:34.866 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.866 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.866 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.866 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.867 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051526+0000", 2023-09-23T23:42:34.867 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.867 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.867 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.867 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.867 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051524+0000", 2023-09-23T23:42:34.867 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:34.867 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.867 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.867 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.867 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051532+0000", 2023-09-23T23:42:34.867 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6310000000000003e-06 2023-09-23T23:42:34.867 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.868 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051559+0000", 2023-09-23T23:42:34.868 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7444999999999999e-05 2023-09-23T23:42:34.868 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.868 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051568+0000", 2023-09-23T23:42:34.868 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8310000000000007e-06 2023-09-23T23:42:34.868 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.868 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051616+0000", 2023-09-23T23:42:34.869 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8168999999999999e-05 2023-09-23T23:42:34.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.869 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.869 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.869 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051624+0000", 2023-09-23T23:42:34.869 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2290000000000003e-06 2023-09-23T23:42:34.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.869 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.869 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.869 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731118+0000", 2023-09-23T23:42:34.869 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.67949406400000001 2023-09-23T23:42:34.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.869 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.870 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.870 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731126+0000", 2023-09-23T23:42:34.870 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0949999999999996e-06 2023-09-23T23:42:34.870 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.870 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.870 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.870 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.400190+0000", 2023-09-23T23:42:34.870 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.669063559 2023-09-23T23:42:34.870 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.870 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.870 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:42:34.870 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.400555+0000", 2023-09-23T23:42:34.871 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036511099999999999 2023-09-23T23:42:34.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.871 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.871 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.416430+0000", 2023-09-23T23:42:34.871 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015875185999999999 2023-09-23T23:42:34.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.871 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.871 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.416532+0000", 2023-09-23T23:42:34.871 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000101456 2023-09-23T23:42:34.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.872 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:42:34.872 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.416657+0000", 2023-09-23T23:42:34.872 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012510200000000001 2023-09-23T23:42:34.872 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.872 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.872 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.872 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.434768+0000", 2023-09-23T23:42:34.872 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018111008000000001 2023-09-23T23:42:34.872 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.872 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.872 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.872 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.434835+0000", 2023-09-23T23:42:34.873 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7475e-05 2023-09-23T23:42:34.873 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.873 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.873 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.534925+0000", 2023-09-23T23:42:34.873 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.100090106 2023-09-23T23:42:34.873 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.873 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.873 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.534976+0000", 2023-09-23T23:42:34.873 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0603999999999999e-05 2023-09-23T23:42:34.873 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.874 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.874 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.535014+0000", 2023-09-23T23:42:34.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8379e-05 2023-09-23T23:42:34.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.874 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.874 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.874 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.538026+0000", 2023-09-23T23:42:34.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003011391 2023-09-23T23:42:34.874 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.874 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.874 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.875 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.875 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89769 13.3 13:c7ccd55e:::benchmark_data_smithi110_541928_object1989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:42:34.875 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.139666+0000", 2023-09-23T23:42:34.875 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 478.48534806399999, 2023-09-23T23:42:34.875 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4017686810000001, 2023-09-23T23:42:34.875 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.875 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.875 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.875 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:34.875 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:34.875 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89769 2023-09-23T23:42:34.875 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.875 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139666+0000", 2023-09-23T23:42:34.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139666+0000", 2023-09-23T23:42:34.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139665+0000", 2023-09-23T23:42:34.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:34.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.877 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139673+0000", 2023-09-23T23:42:34.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1049999999999995e-06 2023-09-23T23:42:34.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.877 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139689+0000", 2023-09-23T23:42:34.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6656000000000001e-05 2023-09-23T23:42:34.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.878 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139701+0000", 2023-09-23T23:42:34.878 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1127e-05 2023-09-23T23:42:34.878 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.878 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139754+0000", 2023-09-23T23:42:34.878 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3362000000000002e-05 2023-09-23T23:42:34.878 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.878 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139764+0000", 2023-09-23T23:42:34.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0474e-05 2023-09-23T23:42:34.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.879 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.879 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731150+0000", 2023-09-23T23:42:34.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.59138585200000005 2023-09-23T23:42:34.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.879 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.879 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.735349+0000", 2023-09-23T23:42:34.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041989519999999997 2023-09-23T23:42:34.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.879 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.880 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.880 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.400620+0000", 2023-09-23T23:42:34.880 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6652705239999999 2023-09-23T23:42:34.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.880 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:42:34.880 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.401065+0000", 2023-09-23T23:42:34.880 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00044538800000000001 2023-09-23T23:42:34.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.880 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.880 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.424058+0000", 2023-09-23T23:42:34.880 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022993085999999999 2023-09-23T23:42:34.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.881 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.881 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.424146+0000", 2023-09-23T23:42:34.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7412999999999998e-05 2023-09-23T23:42:34.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.881 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.881 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:42:34.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.424253+0000", 2023-09-23T23:42:34.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107246 2023-09-23T23:42:34.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.881 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.881 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.882 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.440042+0000", 2023-09-23T23:42:34.882 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015789306999999999 2023-09-23T23:42:34.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.882 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.882 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.443102+0000", 2023-09-23T23:42:34.882 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030595700000000002 2023-09-23T23:42:34.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.882 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.882 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.538201+0000", 2023-09-23T23:42:34.882 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.095099702999999994 2023-09-23T23:42:34.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.538249+0000", 2023-09-23T23:42:34.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7549000000000002e-05 2023-09-23T23:42:34.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.538328+0000", 2023-09-23T23:42:34.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8671000000000006e-05 2023-09-23T23:42:34.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.884 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.541435+0000", 2023-09-23T23:42:34.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031071649999999998 2023-09-23T23:42:34.884 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.884 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.884 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.884 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.884 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89790 13.3 13:ce3e2df6:::benchmark_data_smithi110_541928_object2010:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:42:34.884 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.273997+0000", 2023-09-23T23:42:34.884 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 478.35101747499999, 2023-09-23T23:42:34.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.270580657, 2023-09-23T23:42:34.884 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.885 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.885 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.885 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:34.885 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:34.885 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89790 2023-09-23T23:42:34.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.885 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.885 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.273997+0000", 2023-09-23T23:42:34.885 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.273997+0000", 2023-09-23T23:42:34.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.273995+0000", 2023-09-23T23:42:34.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:34.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274004+0000", 2023-09-23T23:42:34.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7940000000000008e-06 2023-09-23T23:42:34.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274017+0000", 2023-09-23T23:42:34.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3405e-05 2023-09-23T23:42:34.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274027+0000", 2023-09-23T23:42:34.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7200000000000001e-06 2023-09-23T23:42:34.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274086+0000", 2023-09-23T23:42:34.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.94e-05 2023-09-23T23:42:34.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274101+0000", 2023-09-23T23:42:34.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4109e-05 2023-09-23T23:42:34.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.735405+0000", 2023-09-23T23:42:34.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.46130481400000001 2023-09-23T23:42:34.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.735419+0000", 2023-09-23T23:42:34.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3217000000000001e-05 2023-09-23T23:42:34.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.401121+0000", 2023-09-23T23:42:34.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6657021570000001 2023-09-23T23:42:34.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:42:34.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.401570+0000", 2023-09-23T23:42:34.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000449343 2023-09-23T23:42:34.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.431522+0000", 2023-09-23T23:42:34.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029951951000000001 2023-09-23T23:42:34.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.431654+0000", 2023-09-23T23:42:34.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013153199999999999 2023-09-23T23:42:34.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:42:34.891 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.431813+0000", 2023-09-23T23:42:34.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000159937 2023-09-23T23:42:34.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.891 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.451910+0000", 2023-09-23T23:42:34.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02009613 2023-09-23T23:42:34.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.451987+0000", 2023-09-23T23:42:34.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7597999999999998e-05 2023-09-23T23:42:34.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.541568+0000", 2023-09-23T23:42:34.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.089580504000000005 2023-09-23T23:42:34.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.541608+0000", 2023-09-23T23:42:34.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0274000000000001e-05 2023-09-23T23:42:34.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.541675+0000", 2023-09-23T23:42:34.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6890000000000005e-05 2023-09-23T23:42:34.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.544577+0000", 2023-09-23T23:42:34.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029024509999999999 2023-09-23T23:42:34.893 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.893 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.893 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.894 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89805 13.3 13:c4e9783b:::benchmark_data_smithi110_541928_object2025:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e571)", 2023-09-23T23:42:34.894 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.955885+0000", 2023-09-23T23:42:34.894 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 477.66912928400001, 2023-09-23T23:42:34.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.592296927, 2023-09-23T23:42:34.894 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.894 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.894 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.894 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:34.894 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:34.894 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89805 2023-09-23T23:42:34.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.895 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.895 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955885+0000", 2023-09-23T23:42:34.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.895 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955885+0000", 2023-09-23T23:42:34.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955883+0000", 2023-09-23T23:42:34.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:34.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955891+0000", 2023-09-23T23:42:34.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0209999999999999e-06 2023-09-23T23:42:34.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955908+0000", 2023-09-23T23:42:34.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.643e-05 2023-09-23T23:42:34.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955917+0000", 2023-09-23T23:42:34.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2089999999999994e-06 2023-09-23T23:42:34.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955987+0000", 2023-09-23T23:42:34.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9494999999999998e-05 2023-09-23T23:42:34.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.956002+0000", 2023-09-23T23:42:34.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5223e-05 2023-09-23T23:42:34.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.898 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.401618+0000", 2023-09-23T23:42:34.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4456160360000001 2023-09-23T23:42:34.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:42:34.898 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.402009+0000", 2023-09-23T23:42:34.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039078399999999998 2023-09-23T23:42:34.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.440091+0000", 2023-09-23T23:42:34.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038082498999999999 2023-09-23T23:42:34.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.440216+0000", 2023-09-23T23:42:34.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012535399999999999 2023-09-23T23:42:34.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:42:34.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.440388+0000", 2023-09-23T23:42:34.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017125300000000001 2023-09-23T23:42:34.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.457767+0000", 2023-09-23T23:42:34.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017379131999999999 2023-09-23T23:42:34.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.460556+0000", 2023-09-23T23:42:34.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027891050000000001 2023-09-23T23:42:34.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.901 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.544650+0000", 2023-09-23T23:42:34.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.084093672999999994 2023-09-23T23:42:34.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.901 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.544694+0000", 2023-09-23T23:42:34.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4366000000000002e-05 2023-09-23T23:42:34.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.901 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.544759+0000", 2023-09-23T23:42:34.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5371999999999997e-05 2023-09-23T23:42:34.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.902 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.902 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.548182+0000", 2023-09-23T23:42:34.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034224870000000001 2023-09-23T23:42:34.902 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.902 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.902 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.902 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89823 13.3 13:c65a1b7a:::benchmark_data_smithi110_541928_object2043:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e571)", 2023-09-23T23:42:34.903 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.990158+0000", 2023-09-23T23:42:34.903 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 477.63485629000002, 2023-09-23T23:42:34.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5615935460000001, 2023-09-23T23:42:34.903 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.903 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.903 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.903 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:34.903 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:34.903 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89823 2023-09-23T23:42:34.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.903 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990158+0000", 2023-09-23T23:42:34.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990158+0000", 2023-09-23T23:42:34.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990157+0000", 2023-09-23T23:42:34.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:34.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.905 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.905 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990162+0000", 2023-09-23T23:42:34.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1359999999999996e-06 2023-09-23T23:42:34.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.905 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.905 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990174+0000", 2023-09-23T23:42:34.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1695000000000001e-05 2023-09-23T23:42:34.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990184+0000", 2023-09-23T23:42:34.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0465000000000001e-05 2023-09-23T23:42:34.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990231+0000", 2023-09-23T23:42:34.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6329000000000003e-05 2023-09-23T23:42:34.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990240+0000", 2023-09-23T23:42:34.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5310000000000007e-06 2023-09-23T23:42:34.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.402069+0000", 2023-09-23T23:42:34.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.411828474 2023-09-23T23:42:34.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:42:34.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.402423+0000", 2023-09-23T23:42:34.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035466000000000002 2023-09-23T23:42:34.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.908 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.448598+0000", 2023-09-23T23:42:34.908 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.046175040000000001 2023-09-23T23:42:34.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.908 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.448732+0000", 2023-09-23T23:42:34.908 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013384599999999999 2023-09-23T23:42:34.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:42:34.908 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.448935+0000", 2023-09-23T23:42:34.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020236700000000001 2023-09-23T23:42:34.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.909 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.909 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.909 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.460571+0000", 2023-09-23T23:42:34.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011636069000000001 2023-09-23T23:42:34.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.909 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.909 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.909 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.468058+0000", 2023-09-23T23:42:34.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074870570000000001 2023-09-23T23:42:34.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.909 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.548273+0000", 2023-09-23T23:42:34.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.080215195000000003 2023-09-23T23:42:34.910 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.548328+0000", 2023-09-23T23:42:34.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4493000000000002e-05 2023-09-23T23:42:34.910 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.548424+0000", 2023-09-23T23:42:34.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6016999999999999e-05 2023-09-23T23:42:34.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.911 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.911 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.551751+0000", 2023-09-23T23:42:34.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033278750000000001 2023-09-23T23:42:34.911 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.911 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.911 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.911 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.911 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:95149 13.d 13:bc7a1628:::benchmark_data_smithi110_541928_object7369:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T23:42:34.912 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:53.232568+0000", 2023-09-23T23:42:34.912 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 460.39244598099998, 2023-09-23T23:42:34.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4660194620000002, 2023-09-23T23:42:34.912 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.912 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.912 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.912 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:34.912 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:34.912 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95149 2023-09-23T23:42:34.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.912 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232568+0000", 2023-09-23T23:42:34.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.913 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232568+0000", 2023-09-23T23:42:34.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.913 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232567+0000", 2023-09-23T23:42:34.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:34.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232574+0000", 2023-09-23T23:42:34.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.824e-06 2023-09-23T23:42:34.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232588+0000", 2023-09-23T23:42:34.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3597e-05 2023-09-23T23:42:34.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232596+0000", 2023-09-23T23:42:34.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8120000000000008e-06 2023-09-23T23:42:34.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.737972+0000", 2023-09-23T23:42:34.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.50537641300000002 2023-09-23T23:42:34.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:42:34.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.741095+0000", 2023-09-23T23:42:34.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031224780000000001 2023-09-23T23:42:34.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.346073+0000", 2023-09-23T23:42:34.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.60497817499999995 2023-09-23T23:42:34.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:42:34.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.351441+0000", 2023-09-23T23:42:34.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053685449999999997 2023-09-23T23:42:34.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.606909+0000", 2023-09-23T23:42:34.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2554679260000001 2023-09-23T23:42:34.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.612108+0000", 2023-09-23T23:42:34.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051983919999999996 2023-09-23T23:42:34.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:42:34.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.632573+0000", 2023-09-23T23:42:34.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020465006000000001 2023-09-23T23:42:34.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.918 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.668063+0000", 2023-09-23T23:42:34.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.035490090000000002 2023-09-23T23:42:34.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.918 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.680799+0000", 2023-09-23T23:42:34.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012736032 2023-09-23T23:42:34.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.919 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.919 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.682749+0000", 2023-09-23T23:42:34.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019503509999999999 2023-09-23T23:42:34.919 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.919 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.919 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.685937+0000", 2023-09-23T23:42:34.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031873520000000001 2023-09-23T23:42:34.919 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.919 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.919 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.698588+0000", 2023-09-23T23:42:34.920 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012651051999999999 2023-09-23T23:42:34.920 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.920 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.920 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.920 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:95282 13.d 13:b9f8e077:::benchmark_data_smithi110_541928_object7502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T23:42:34.920 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:53.626451+0000", 2023-09-23T23:42:34.920 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 459.998562886, 2023-09-23T23:42:34.920 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5002010320000001, 2023-09-23T23:42:34.920 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.920 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.920 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.921 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:34.921 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:34.921 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95282 2023-09-23T23:42:34.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.921 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626451+0000", 2023-09-23T23:42:34.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626451+0000", 2023-09-23T23:42:34.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626449+0000", 2023-09-23T23:42:34.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:42:34.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626459+0000", 2023-09-23T23:42:34.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7790000000000007e-06 2023-09-23T23:42:34.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626473+0000", 2023-09-23T23:42:34.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3655000000000001e-05 2023-09-23T23:42:34.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626483+0000", 2023-09-23T23:42:34.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.013e-05 2023-09-23T23:42:34.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.973570+0000", 2023-09-23T23:42:34.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.34708731599999998 2023-09-23T23:42:34.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:42:34.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.978239+0000", 2023-09-23T23:42:34.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046685850000000003 2023-09-23T23:42:34.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.019769+0000", 2023-09-23T23:42:34.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041529772 2023-09-23T23:42:34.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:42:34.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.024313+0000", 2023-09-23T23:42:34.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045445479999999998 2023-09-23T23:42:34.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.064268+0000", 2023-09-23T23:42:34.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.039955124 2023-09-23T23:42:34.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.067970+0000", 2023-09-23T23:42:34.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00370164 2023-09-23T23:42:34.926 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:42:34.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.076658+0000", 2023-09-23T23:42:34.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086883770000000006 2023-09-23T23:42:34.926 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.104140+0000", 2023-09-23T23:42:34.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02748134 2023-09-23T23:42:34.927 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.927 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.927 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.927 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.113193+0000", 2023-09-23T23:42:34.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0090528350000000004 2023-09-23T23:42:34.927 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.927 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.927 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.927 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.114921+0000", 2023-09-23T23:42:34.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017289149999999999 2023-09-23T23:42:34.927 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.927 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.927 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.928 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.117081+0000", 2023-09-23T23:42:34.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021593519999999998 2023-09-23T23:42:34.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.928 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.928 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.126652+0000", 2023-09-23T23:42:34.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.009571474 2023-09-23T23:42:34.928 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.928 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.928 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.928 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:95302 13.d 13:b56e9a5b:::benchmark_data_smithi110_541928_object7522:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T23:42:34.929 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:53.699709+0000", 2023-09-23T23:42:34.929 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 459.925304729, 2023-09-23T23:42:34.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.504555552, 2023-09-23T23:42:34.929 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.929 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.929 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.929 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:34.929 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:34.929 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95302 2023-09-23T23:42:34.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.929 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699709+0000", 2023-09-23T23:42:34.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699709+0000", 2023-09-23T23:42:34.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699708+0000", 2023-09-23T23:42:34.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:42:34.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699717+0000", 2023-09-23T23:42:34.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0899999999999994e-06 2023-09-23T23:42:34.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699730+0000", 2023-09-23T23:42:34.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3392e-05 2023-09-23T23:42:34.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.932 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699740+0000", 2023-09-23T23:42:34.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5559999999999996e-06 2023-09-23T23:42:34.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.932 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.932 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.025484+0000", 2023-09-23T23:42:34.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.325744649 2023-09-23T23:42:34.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.932 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:42:34.932 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.030494+0000", 2023-09-23T23:42:34.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.005009365 2023-09-23T23:42:34.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.166809+0000", 2023-09-23T23:42:34.933 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.13631576300000001 2023-09-23T23:42:34.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:42:34.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.171655+0000", 2023-09-23T23:42:34.933 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048457680000000003 2023-09-23T23:42:34.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.934 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.138451+0000", 2023-09-23T23:42:34.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96679576300000003 2023-09-23T23:42:34.934 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.934 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.934 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.934 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.142480+0000", 2023-09-23T23:42:34.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040295679999999999 2023-09-23T23:42:34.934 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.934 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.934 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:42:34.934 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.150317+0000", 2023-09-23T23:42:34.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078367109999999997 2023-09-23T23:42:34.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.935 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.186750+0000", 2023-09-23T23:42:34.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036433140000000003 2023-09-23T23:42:34.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.935 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.188250+0000", 2023-09-23T23:42:34.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001500145 2023-09-23T23:42:34.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.936 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.188845+0000", 2023-09-23T23:42:34.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00059478099999999996 2023-09-23T23:42:34.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.936 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.936 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.191339+0000", 2023-09-23T23:42:34.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024934559999999998 2023-09-23T23:42:34.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.936 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.936 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.204265+0000", 2023-09-23T23:42:34.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012926272000000001 2023-09-23T23:42:34.937 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.937 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.937 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.937 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:95356 13.d 13:b7510188:::benchmark_data_smithi110_541928_object7576:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T23:42:34.937 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:53.853143+0000", 2023-09-23T23:42:34.937 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 459.77187097400002, 2023-09-23T23:42:34.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5531699410000002, 2023-09-23T23:42:34.937 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.937 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.937 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.937 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:42:34.938 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:34.938 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95356 2023-09-23T23:42:34.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.938 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853143+0000", 2023-09-23T23:42:34.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853143+0000", 2023-09-23T23:42:34.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853142+0000", 2023-09-23T23:42:34.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:34.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853150+0000", 2023-09-23T23:42:34.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8709999999999997e-06 2023-09-23T23:42:34.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853166+0000", 2023-09-23T23:42:34.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6568000000000001e-05 2023-09-23T23:42:34.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853176+0000", 2023-09-23T23:42:34.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9520000000000006e-06 2023-09-23T23:42:34.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.200709+0000", 2023-09-23T23:42:34.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.347532703 2023-09-23T23:42:34.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.941 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:42:34.941 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.205641+0000", 2023-09-23T23:42:34.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.004932199 2023-09-23T23:42:34.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.941 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.941 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.252766+0000", 2023-09-23T23:42:34.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047124528999999998 2023-09-23T23:42:34.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.942 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:42:34.942 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.257597+0000", 2023-09-23T23:42:34.942 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048309980000000004 2023-09-23T23:42:34.942 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.942 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.942 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.269273+0000", 2023-09-23T23:42:34.942 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.01167611 2023-09-23T23:42:34.942 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.942 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.942 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.276153+0000", 2023-09-23T23:42:34.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068806689999999998 2023-09-23T23:42:34.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.943 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:42:34.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.296160+0000", 2023-09-23T23:42:34.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020006954 2023-09-23T23:42:34.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.943 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.310175+0000", 2023-09-23T23:42:34.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014014235 2023-09-23T23:42:34.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.313239+0000", 2023-09-23T23:42:34.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030640490000000001 2023-09-23T23:42:34.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.383950+0000", 2023-09-23T23:42:34.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.070711100999999998 2023-09-23T23:42:34.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.388728+0000", 2023-09-23T23:42:34.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047779540000000001 2023-09-23T23:42:34.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.406313+0000", 2023-09-23T23:42:34.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017585478000000002 2023-09-23T23:42:34.945 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.945 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.945 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.945 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:71676 14.4 14:23cfb94e:::benchmark_data_smithi110_640334_object71675:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e634)", 2023-09-23T23:42:34.946 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:40:32.090762+0000", 2023-09-23T23:42:34.946 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 121.53425174500001, 2023-09-23T23:42:34.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.000094357, 2023-09-23T23:42:34.946 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.946 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.946 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.946 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:42:34.946 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:42:34.946 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71676 2023-09-23T23:42:34.946 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.946 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.946 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.090762+0000", 2023-09-23T23:42:34.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.090762+0000", 2023-09-23T23:42:34.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.090762+0000", 2023-09-23T23:42:34.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:34.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.090781+0000", 2023-09-23T23:42:34.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9534e-05 2023-09-23T23:42:34.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.090786+0000", 2023-09-23T23:42:34.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7299999999999996e-06 2023-09-23T23:42:34.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.090790+0000", 2023-09-23T23:42:34.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4400000000000001e-06 2023-09-23T23:42:34.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.090844+0000", 2023-09-23T23:42:34.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4425000000000002e-05 2023-09-23T23:42:34.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.090853+0000", 2023-09-23T23:42:34.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9069999999999996e-06 2023-09-23T23:42:34.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.080184+0000", 2023-09-23T23:42:34.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98933127700000001 2023-09-23T23:42:34.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.080350+0000", 2023-09-23T23:42:34.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016558100000000001 2023-09-23T23:42:34.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,7", 2023-09-23T23:42:34.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.080521+0000", 2023-09-23T23:42:34.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017122099999999999 2023-09-23T23:42:34.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.090629+0000", 2023-09-23T23:42:34.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010108112000000001 2023-09-23T23:42:34.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.090688+0000", 2023-09-23T23:42:34.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9299999999999998e-05 2023-09-23T23:42:34.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.090751+0000", 2023-09-23T23:42:34.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2228000000000001e-05 2023-09-23T23:42:34.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.090814+0000", 2023-09-23T23:42:34.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3442999999999995e-05 2023-09-23T23:42:34.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.090857+0000", 2023-09-23T23:42:34.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2725999999999997e-05 2023-09-23T23:42:34.953 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.953 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.953 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.953 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:71689 14.4 14:26b4eb17:::benchmark_data_smithi110_640334_object71688:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e634)", 2023-09-23T23:42:34.953 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:40:32.119139+0000", 2023-09-23T23:42:34.953 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 121.505875223, 2023-09-23T23:42:34.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97705882799999999, 2023-09-23T23:42:34.953 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.954 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.954 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.954 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:42:34.954 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:42:34.954 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71689 2023-09-23T23:42:34.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.954 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.954 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.119139+0000", 2023-09-23T23:42:34.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.119139+0000", 2023-09-23T23:42:34.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.119138+0000", 2023-09-23T23:42:34.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:34.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.119162+0000", 2023-09-23T23:42:34.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3254000000000001e-05 2023-09-23T23:42:34.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.119170+0000", 2023-09-23T23:42:34.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3629999999999994e-06 2023-09-23T23:42:34.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.119176+0000", 2023-09-23T23:42:34.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.314e-06 2023-09-23T23:42:34.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.119238+0000", 2023-09-23T23:42:34.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1598000000000002e-05 2023-09-23T23:42:34.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.119250+0000", 2023-09-23T23:42:34.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1725999999999999e-05 2023-09-23T23:42:34.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.082887+0000", 2023-09-23T23:42:34.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96363742299999999 2023-09-23T23:42:34.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.083034+0000", 2023-09-23T23:42:34.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014674999999999999 2023-09-23T23:42:34.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,7", 2023-09-23T23:42:34.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.083190+0000", 2023-09-23T23:42:34.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000156334 2023-09-23T23:42:34.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.090739+0000", 2023-09-23T23:42:34.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075486320000000004 2023-09-23T23:42:34.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.090970+0000", 2023-09-23T23:42:34.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023142400000000001 2023-09-23T23:42:34.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.096138+0000", 2023-09-23T23:42:34.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051678319999999998 2023-09-23T23:42:34.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.096173+0000", 2023-09-23T23:42:34.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5423999999999998e-05 2023-09-23T23:42:34.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.096198+0000", 2023-09-23T23:42:34.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.429e-05 2023-09-23T23:42:34.960 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.960 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.961 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.961 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:95152 14.2 14:4c8232cd:::benchmark_data_smithi110_640334_object11273:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e645)", 2023-09-23T23:42:34.961 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:11.037166+0000", 2023-09-23T23:42:34.961 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 82.587848191999996, 2023-09-23T23:42:34.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.963686143, 2023-09-23T23:42:34.961 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:42:34.961 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:34.961 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:42:34.961 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:42:34.961 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:42:34.961 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95152 2023-09-23T23:42:34.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.962 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:42:34.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:42:34.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.037166+0000", 2023-09-23T23:42:34.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:42:34.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.037166+0000", 2023-09-23T23:42:34.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:42:34.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:42:34.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.037165+0000", 2023-09-23T23:42:34.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:34.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:42:34.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.037171+0000", 2023-09-23T23:42:34.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7680000000000003e-06 2023-09-23T23:42:34.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:42:34.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.037186+0000", 2023-09-23T23:42:34.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5013e-05 2023-09-23T23:42:34.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:42:34.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.037194+0000", 2023-09-23T23:42:34.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4880000000000001e-06 2023-09-23T23:42:34.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.037255+0000", 2023-09-23T23:42:34.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1377000000000005e-05 2023-09-23T23:42:34.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:42:34.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.037268+0000", 2023-09-23T23:42:34.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2731999999999999e-05 2023-09-23T23:42:34.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:42:34.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.970837+0000", 2023-09-23T23:42:34.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93356932800000003 2023-09-23T23:42:34.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:42:34.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.971001+0000", 2023-09-23T23:42:34.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016338999999999999 2023-09-23T23:42:34.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-09-23T23:42:34.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.971105+0000", 2023-09-23T23:42:34.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104168 2023-09-23T23:42:34.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.973794+0000", 2023-09-23T23:42:34.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026890479999999999 2023-09-23T23:42:34.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:34.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.992482+0000", 2023-09-23T23:42:34.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018688201000000002 2023-09-23T23:42:34.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:42:34.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:12.000770+0000", 2023-09-23T23:42:34.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0082880490000000005 2023-09-23T23:42:34.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:42:34.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:12.000814+0000", 2023-09-23T23:42:34.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4251e-05 2023-09-23T23:42:34.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:42:34.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:42:34.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:42:34.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:12.000852+0000", 2023-09-23T23:42:34.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7608999999999999e-05 2023-09-23T23:42:34.968 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.968 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.968 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.968 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:42:34.968 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:42:34.968 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:42:34.969 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:34 smithi138 ceph-mon[193606]: failed to encode map e674 with expected crc 2023-09-23T23:42:34.969 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:34 smithi138 ceph-mon[193606]: failed to encode map e674 with expected crc 2023-09-23T23:42:34.969 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:34 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:34.969 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:34 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:34.970 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:42:34.970 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:34 smithi110 ceph-mon[221102]: failed to encode map e674 with expected crc 2023-09-23T23:42:34.970 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:34 smithi110 ceph-mon[221102]: failed to encode map e674 with expected crc 2023-09-23T23:42:34.970 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:34 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:34.970 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:34 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:34.970 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:34 smithi110 ceph-mon[212939]: failed to encode map e674 with expected crc 2023-09-23T23:42:34.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:34 smithi110 ceph-mon[212939]: failed to encode map e674 with expected crc 2023-09-23T23:42:34.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:34 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:34.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:34 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:34.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:34.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:42:35.072 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:42:35.099 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:35.149 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:42:35.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:35.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:42:35.255 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:35.319 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:35.590 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:35 smithi110 ceph-mon[212939]: pgmap v1915: 119 pgs: 1 remapped+peering, 118 active+clean; 1.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 0 B/s, 34 objects/s recovering 2023-09-23T23:42:35.592 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:35 smithi110 ceph-mon[221102]: pgmap v1915: 119 pgs: 1 remapped+peering, 118 active+clean; 1.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 0 B/s, 34 objects/s recovering 2023-09-23T23:42:35.664 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:35.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:35.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:35.775 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:35.866 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:35.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:35 smithi138 ceph-mon[193606]: pgmap v1915: 119 pgs: 1 remapped+peering, 118 active+clean; 1.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 0 B/s, 34 objects/s recovering 2023-09-23T23:42:36.226 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:36.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:36.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:36.292 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:42:36.334 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:36.426 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:36.628 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:36.646 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:36.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:36.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:36.672 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:42:36.757 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:42:36.785 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:36.892 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:42:36.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:36.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:42:36.942 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:42:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:42:36] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.43.0" 2023-09-23T23:42:36.996 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:37.034 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:37.084 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:37.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:37.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:37.109 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:42:37.126 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:42:37.188 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:37.287 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:37.451 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:42:37.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:37 smithi138 ceph-mon[193606]: pgmap v1916: 119 pgs: 119 active+clean; 1.3 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 776 op/s; 0 B/s, 23 objects/s recovering 2023-09-23T23:42:37.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:37 smithi110 ceph-mon[212939]: pgmap v1916: 119 pgs: 119 active+clean; 1.3 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 776 op/s; 0 B/s, 23 objects/s recovering 2023-09-23T23:42:37.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:37 smithi110 ceph-mon[221102]: pgmap v1916: 119 pgs: 119 active+clean; 1.3 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 776 op/s; 0 B/s, 23 objects/s recovering 2023-09-23T23:42:38.593 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:38.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:38.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:38.702 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:38.728 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:38.869 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:38 smithi138 ceph-mon[193606]: from='client.47339 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:42:38.942 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:38 smithi110 ceph-mon[212939]: from='client.47339 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:42:38.943 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:38 smithi110 ceph-mon[221102]: from='client.47339 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:42:38.978 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1916,"stamp":"2023-09-23T23:42:36.108678+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1425807288,"num_objects":21772,"num_object_clones":0,"num_object_copies":65316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21772,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":146067,"num_write_kb":5368094,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15009,"num_bytes_recovered":810955076,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91701,"ondisk_log_size":91701,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":10281656,"kb_used_data":4380936,"kb_used_omap":29,"kb_used_meta":5900194,"kb_avail":552189256,"statfs":{"total":575970213888,"available":565441798144,"internally_reserved":0,"allocated":4486078464,"data_stored":4468968042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":6041799379},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,8,8,4,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-305463296,"num_objects":-4661,"num_object_clones":0,"num_object_copies":-13983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4661,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001129"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":870,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.949604+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:42:31.949604+0000","last_peered":"2023-09-23T23:42:31.949604+0000","last_clean":"2023-09-23T23:42:31.949604+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:42:31.949604+0000","last_undegraded":"2023-09-23T23:42:31.949604+0000","last_fullsized":"2023-09-23T23:42:31.949604+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":431,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.071033+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:42:32.071033+0000","last_peered":"2023-09-23T23:42:32.071033+0000","last_clean":"2023-09-23T23:42:32.071033+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:42:32.071033+0000","last_undegraded":"2023-09-23T23:42:32.071033+0000","last_fullsized":"2023-09-23T23:42:32.071033+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":666,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.976879+0000","last_change":"2023-09-23T23:39:55.102353+0000","last_active":"2023-09-23T23:42:30.976879+0000","last_peered":"2023-09-23T23:42:30.976879+0000","last_clean":"2023-09-23T23:42:30.976879+0000","last_became_active":"2023-09-23T23:39:55.102017+0000","last_became_peered":"2023-09-23T23:39:55.102017+0000","last_unstale":"2023-09-23T23:42:30.976879+0000","last_undegraded":"2023-09-23T23:42:30.976879+0000","last_fullsized":"2023-09-23T23:42:30.976879+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:48.870443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":823,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.950005+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:42:31.950005+0000","last_peered":"2023-09-23T23:42:31.950005+0000","last_clean":"2023-09-23T23:42:31.950005+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:42:31.950005+0000","last_undegraded":"2023-09-23T23:42:31.950005+0000","last_fullsized":"2023-09-23T23:42:31.950005+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":403,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.956286+0000","last_change":"2023-09-23T23:39:55.104571+0000","last_active":"2023-09-23T23:42:30.956286+0000","last_peered":"2023-09-23T23:42:30.956286+0000","last_clean":"2023-09-23T23:42:30.956286+0000","last_became_active":"2023-09-23T23:39:55.104152+0000","last_became_peered":"2023-09-23T23:39:55.104152+0000","last_unstale":"2023-09-23T23:42:30.956286+0000","last_undegraded":"2023-09-23T23:42:30.956286+0000","last_fullsized":"2023-09-23T23:42:30.956286+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:31:52.551048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":637,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.970810+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:42:30.970810+0000","last_peered":"2023-09-23T23:42:30.970810+0000","last_clean":"2023-09-23T23:42:30.970810+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:42:30.970810+0000","last_undegraded":"2023-09-23T23:42:30.970810+0000","last_fullsized":"2023-09-23T23:42:30.970810+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":835,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.948452+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:42:31.948452+0000","last_peered":"2023-09-23T23:42:31.948452+0000","last_clean":"2023-09-23T23:42:31.948452+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:42:31.948452+0000","last_undegraded":"2023-09-23T23:42:31.948452+0000","last_fullsized":"2023-09-23T23:42:31.948452+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":721,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.955839+0000","last_change":"2023-09-23T23:42:28.951264+0000","last_active":"2023-09-23T23:42:30.955839+0000","last_peered":"2023-09-23T23:42:30.955839+0000","last_clean":"2023-09-23T23:42:30.955839+0000","last_became_active":"2023-09-23T23:42:28.950966+0000","last_became_peered":"2023-09-23T23:42:28.950966+0000","last_unstale":"2023-09-23T23:42:30.955839+0000","last_undegraded":"2023-09-23T23:42:30.955839+0000","last_fullsized":"2023-09-23T23:42:30.955839+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:30.571439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":353,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.950935+0000","last_change":"2023-09-23T23:39:55.092989+0000","last_active":"2023-09-23T23:42:31.950935+0000","last_peered":"2023-09-23T23:42:31.950935+0000","last_clean":"2023-09-23T23:42:31.950935+0000","last_became_active":"2023-09-23T23:39:55.092680+0000","last_became_peered":"2023-09-23T23:39:55.092680+0000","last_unstale":"2023-09-23T23:42:31.950935+0000","last_undegraded":"2023-09-23T23:42:31.950935+0000","last_fullsized":"2023-09-23T23:42:31.950935+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:44.685469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":994,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.949149+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:42:31.949149+0000","last_peered":"2023-09-23T23:42:31.949149+0000","last_clean":"2023-09-23T23:42:31.949149+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:42:31.949149+0000","last_undegraded":"2023-09-23T23:42:31.949149+0000","last_fullsized":"2023-09-23T23:42:31.949149+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":326,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.950819+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:42:31.950819+0000","last_peered":"2023-09-23T23:42:31.950819+0000","last_clean":"2023-09-23T23:42:31.950819+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:42:31.950819+0000","last_undegraded":"2023-09-23T23:42:31.950819+0000","last_fullsized":"2023-09-23T23:42:31.950819+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":685,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.955804+0000","last_change":"2023-09-23T23:39:55.100428+0000","last_active":"2023-09-23T23:42:30.955804+0000","last_peered":"2023-09-23T23:42:30.955804+0000","last_clean":"2023-09-23T23:42:30.955804+0000","last_became_active":"2023-09-23T23:39:55.100094+0000","last_became_peered":"2023-09-23T23:39:55.100094+0000","last_unstale":"2023-09-23T23:42:30.955804+0000","last_undegraded":"2023-09-23T23:42:30.955804+0000","last_fullsized":"2023-09-23T23:42:30.955804+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:43.132618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":970,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.072863+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:42:32.072863+0000","last_peered":"2023-09-23T23:42:32.072863+0000","last_clean":"2023-09-23T23:42:32.072863+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:42:32.072863+0000","last_undegraded":"2023-09-23T23:42:32.072863+0000","last_fullsized":"2023-09-23T23:42:32.072863+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":305,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.950564+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:42:31.950564+0000","last_peered":"2023-09-23T23:42:31.950564+0000","last_clean":"2023-09-23T23:42:31.950564+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:42:31.950564+0000","last_undegraded":"2023-09-23T23:42:31.950564+0000","last_fullsized":"2023-09-23T23:42:31.950564+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":70,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.071107+0000","last_change":"2023-09-23T23:39:55.091152+0000","last_active":"2023-09-23T23:42:32.071107+0000","last_peered":"2023-09-23T23:42:32.071107+0000","last_clean":"2023-09-23T23:42:32.071107+0000","last_became_active":"2023-09-23T23:39:55.090833+0000","last_became_peered":"2023-09-23T23:39:55.090833+0000","last_unstale":"2023-09-23T23:42:32.071107+0000","last_undegraded":"2023-09-23T23:42:32.071107+0000","last_fullsized":"2023-09-23T23:42:32.071107+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:33.156366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1148,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.950677+0000","last_change":"2023-09-23T23:39:58.110527+0000","last_active":"2023-09-23T23:42:31.950677+0000","last_peered":"2023-09-23T23:42:31.950677+0000","last_clean":"2023-09-23T23:42:31.950677+0000","last_became_active":"2023-09-23T23:39:58.110194+0000","last_became_peered":"2023-09-23T23:39:58.110194+0000","last_unstale":"2023-09-23T23:42:31.950677+0000","last_undegraded":"2023-09-23T23:42:31.950677+0000","last_fullsized":"2023-09-23T23:42:31.950677+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:21:19.332102+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":348,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.976659+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:42:30.976659+0000","last_peered":"2023-09-23T23:42:30.976659+0000","last_clean":"2023-09-23T23:42:30.976659+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:42:30.976659+0000","last_undegraded":"2023-09-23T23:42:30.976659+0000","last_fullsized":"2023-09-23T23:42:30.976659+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":360,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.070956+0000","last_change":"2023-09-23T23:42:28.956958+0000","last_active":"2023-09-23T23:42:32.070956+0000","last_peered":"2023-09-23T23:42:32.070956+0000","last_clean":"2023-09-23T23:42:32.070956+0000","last_became_active":"2023-09-23T23:42:28.956645+0000","last_became_peered":"2023-09-23T23:42:28.956645+0000","last_unstale":"2023-09-23T23:42:32.070956+0000","last_undegraded":"2023-09-23T23:42:32.070956+0000","last_fullsized":"2023-09-23T23:42:32.070956+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:16:28.472945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":538,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.970905+0000","last_change":"2023-09-23T23:39:19.011304+0000","last_active":"2023-09-23T23:42:30.970905+0000","last_peered":"2023-09-23T23:42:30.970905+0000","last_clean":"2023-09-23T23:42:30.970905+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:42:30.970905+0000","last_undegraded":"2023-09-23T23:42:30.970905+0000","last_fullsized":"2023-09-23T23:42:30.970905+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:39:19.011223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:39:19.011223+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:48:49.078060+0000","scrub_duration":0.001220575,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":194,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.072767+0000","last_change":"2023-09-23T23:39:55.129350+0000","last_active":"2023-09-23T23:42:32.072767+0000","last_peered":"2023-09-23T23:42:32.072767+0000","last_clean":"2023-09-23T23:42:32.072767+0000","last_became_active":"2023-09-23T23:39:55.092677+0000","last_became_peered":"2023-09-23T23:39:55.092677+0000","last_unstale":"2023-09-23T23:42:32.072767+0000","last_undegraded":"2023-09-23T23:42:32.072767+0000","last_fullsized":"2023-09-23T23:42:32.072767+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:38.925371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":548,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.949805+0000","last_change":"2023-09-23T23:39:55.097595+0000","last_active":"2023-09-23T23:42:31.949805+0000","last_peered":"2023-09-23T23:42:31.949805+0000","last_clean":"2023-09-23T23:42:31.949805+0000","last_became_active":"2023-09-23T23:39:55.097301+0000","last_became_peered":"2023-09-23T23:39:55.097301+0000","last_unstale":"2023-09-23T23:42:31.949805+0000","last_undegraded":"2023-09-23T23:42:31.949805+0000","last_fullsized":"2023-09-23T23:42:31.949805+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:26.508461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":829,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.950090+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:42:31.950090+0000","last_peered":"2023-09-23T23:42:31.950090+0000","last_clean":"2023-09-23T23:42:31.950090+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:42:31.950090+0000","last_undegraded":"2023-09-23T23:42:31.950090+0000","last_fullsized":"2023-09-23T23:42:31.950090+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":238,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.071706+0000","last_change":"2023-09-23T23:39:55.137107+0000","last_active":"2023-09-23T23:42:32.071706+0000","last_peered":"2023-09-23T23:42:32.071706+0000","last_clean":"2023-09-23T23:42:32.071706+0000","last_became_active":"2023-09-23T23:39:55.094309+0000","last_became_peered":"2023-09-23T23:39:55.094309+0000","last_unstale":"2023-09-23T23:42:32.071706+0000","last_undegraded":"2023-09-23T23:42:32.071706+0000","last_fullsized":"2023-09-23T23:42:32.071706+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:49.766841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":693,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.976193+0000","last_change":"2023-09-23T23:42:28.954052+0000","last_active":"2023-09-23T23:42:30.976193+0000","last_peered":"2023-09-23T23:42:30.976193+0000","last_clean":"2023-09-23T23:42:30.976193+0000","last_became_active":"2023-09-23T23:42:28.953681+0000","last_became_peered":"2023-09-23T23:42:28.953681+0000","last_unstale":"2023-09-23T23:42:30.976193+0000","last_undegraded":"2023-09-23T23:42:30.976193+0000","last_fullsized":"2023-09-23T23:42:30.976193+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:07.367387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":791,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.949515+0000","last_change":"2023-09-23T23:39:58.117169+0000","last_active":"2023-09-23T23:42:31.949515+0000","last_peered":"2023-09-23T23:42:31.949515+0000","last_clean":"2023-09-23T23:42:31.949515+0000","last_became_active":"2023-09-23T23:39:58.116815+0000","last_became_peered":"2023-09-23T23:39:58.116815+0000","last_unstale":"2023-09-23T23:42:31.949515+0000","last_undegraded":"2023-09-23T23:42:31.949515+0000","last_fullsized":"2023-09-23T23:42:31.949515+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:51.175189+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":270,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.967881+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:42:30.967881+0000","last_peered":"2023-09-23T23:42:30.967881+0000","last_clean":"2023-09-23T23:42:30.967881+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:42:30.967881+0000","last_undegraded":"2023-09-23T23:42:30.967881+0000","last_fullsized":"2023-09-23T23:42:30.967881+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":397,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.976793+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:42:30.976793+0000","last_peered":"2023-09-23T23:42:30.976793+0000","last_clean":"2023-09-23T23:42:30.976793+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:42:30.976793+0000","last_undegraded":"2023-09-23T23:42:30.976793+0000","last_fullsized":"2023-09-23T23:42:30.976793+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":638,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.971996+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:42:30.971996+0000","last_peered":"2023-09-23T23:42:30.971996+0000","last_clean":"2023-09-23T23:42:30.971996+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:42:30.971996+0000","last_undegraded":"2023-09-23T23:42:30.971996+0000","last_fullsized":"2023-09-23T23:42:30.971996+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":483,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T23:42:29.938664+0000","last_change":"2023-09-23T23:39:55.100855+0000","last_active":"2023-09-23T23:42:29.938664+0000","last_peered":"2023-09-23T23:42:29.938664+0000","last_clean":"2023-09-23T23:42:29.938664+0000","last_became_active":"2023-09-23T23:39:55.100577+0000","last_became_peered":"2023-09-23T23:39:55.100577+0000","last_unstale":"2023-09-23T23:42:29.938664+0000","last_undegraded":"2023-09-23T23:42:29.938664+0000","last_fullsized":"2023-09-23T23:42:29.938664+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:02.548093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":631,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.970983+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:42:30.970983+0000","last_peered":"2023-09-23T23:42:30.970983+0000","last_clean":"2023-09-23T23:42:30.970983+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:42:30.970983+0000","last_undegraded":"2023-09-23T23:42:30.970983+0000","last_fullsized":"2023-09-23T23:42:30.970983+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":1097,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.073622+0000","last_change":"2023-09-23T23:42:32.073277+0000","last_active":"2023-09-23T23:42:32.073622+0000","last_peered":"2023-09-23T23:42:32.073622+0000","last_clean":"2023-09-23T23:42:32.073622+0000","last_became_active":"2023-09-23T23:42:32.072952+0000","last_became_peered":"2023-09-23T23:42:32.072952+0000","last_unstale":"2023-09-23T23:42:32.073622+0000","last_undegraded":"2023-09-23T23:42:32.073622+0000","last_fullsized":"2023-09-23T23:42:32.073622+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:19.983734+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":39,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T23:42:29.942162+0000","last_change":"2023-09-23T23:41:31.621623+0000","last_active":"2023-09-23T23:42:29.942162+0000","last_peered":"2023-09-23T23:42:29.942162+0000","last_clean":"2023-09-23T23:42:29.942162+0000","last_became_active":"2023-09-23T23:41:31.596928+0000","last_became_peered":"2023-09-23T23:41:31.596928+0000","last_unstale":"2023-09-23T23:42:29.942162+0000","last_undegraded":"2023-09-23T23:42:29.942162+0000","last_fullsized":"2023-09-23T23:42:29.942162+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:01.308674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":316,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.971341+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:42:30.971341+0000","last_peered":"2023-09-23T23:42:30.971341+0000","last_clean":"2023-09-23T23:42:30.971341+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:42:30.971341+0000","last_undegraded":"2023-09-23T23:42:30.971341+0000","last_fullsized":"2023-09-23T23:42:30.971341+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"673'9020","reported_seq":14777,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.743427+0000","last_change":"2023-09-23T23:41:11.967390+0000","last_active":"2023-09-23T23:42:32.743427+0000","last_peered":"2023-09-23T23:42:32.743427+0000","last_clean":"2023-09-23T23:42:32.743427+0000","last_became_active":"2023-09-23T23:41:11.955563+0000","last_became_peered":"2023-09-23T23:41:11.955563+0000","last_unstale":"2023-09-23T23:42:32.743427+0000","last_undegraded":"2023-09-23T23:42:32.743427+0000","last_fullsized":"2023-09-23T23:42:32.743427+0000","mapping_epoch":645,"log_start":"633'4186","ondisk_log_start":"633'4186","created":617,"last_epoch_clean":646,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4834,"log_dups_size":0,"ondisk_log_size":4834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:03.686224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92536832,"num_objects":1412,"num_object_clones":0,"num_object_copies":4236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9020,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2837,"num_bytes_recovered":185925632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":1010,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.076525+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:42:32.076525+0000","last_peered":"2023-09-23T23:42:32.076525+0000","last_clean":"2023-09-23T23:42:32.076525+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:42:32.076525+0000","last_undegraded":"2023-09-23T23:42:32.076525+0000","last_fullsized":"2023-09-23T23:42:32.076525+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":307,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.949123+0000","last_change":"2023-09-23T23:42:28.954407+0000","last_active":"2023-09-23T23:42:31.949123+0000","last_peered":"2023-09-23T23:42:31.949123+0000","last_clean":"2023-09-23T23:42:31.949123+0000","last_became_active":"2023-09-23T23:42:28.950556+0000","last_became_peered":"2023-09-23T23:42:28.950556+0000","last_unstale":"2023-09-23T23:42:31.949123+0000","last_undegraded":"2023-09-23T23:42:31.949123+0000","last_fullsized":"2023-09-23T23:42:31.949123+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:58.736585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":315,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.949537+0000","last_change":"2023-09-23T23:39:55.095319+0000","last_active":"2023-09-23T23:42:31.949537+0000","last_peered":"2023-09-23T23:42:31.949537+0000","last_clean":"2023-09-23T23:42:31.949537+0000","last_became_active":"2023-09-23T23:39:55.094677+0000","last_became_peered":"2023-09-23T23:39:55.094677+0000","last_unstale":"2023-09-23T23:42:31.949537+0000","last_undegraded":"2023-09-23T23:42:31.949537+0000","last_fullsized":"2023-09-23T23:42:31.949537+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:21.525905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"673'8942","reported_seq":9008,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.774834+0000","last_change":"2023-09-23T23:39:13.746676+0000","last_active":"2023-09-23T23:42:32.774834+0000","last_peered":"2023-09-23T23:42:32.774834+0000","last_clean":"2023-09-23T23:42:32.774834+0000","last_became_active":"2023-09-23T23:39:13.746389+0000","last_became_peered":"2023-09-23T23:39:13.746389+0000","last_unstale":"2023-09-23T23:42:32.774834+0000","last_undegraded":"2023-09-23T23:42:32.774834+0000","last_fullsized":"2023-09-23T23:42:32.774834+0000","mapping_epoch":617,"log_start":"633'4097","ondisk_log_start":"633'4097","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4845,"log_dups_size":0,"ondisk_log_size":4845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:44.833933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92012544,"num_objects":1404,"num_object_clones":0,"num_object_copies":4212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8942,"num_write_kb":331072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1109,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066677+0000","last_change":"2023-09-23T23:39:58.119994+0000","last_active":"2023-09-23T23:42:32.066677+0000","last_peered":"2023-09-23T23:42:32.066677+0000","last_clean":"2023-09-23T23:42:32.066677+0000","last_became_active":"2023-09-23T23:39:58.119654+0000","last_became_peered":"2023-09-23T23:39:58.119654+0000","last_unstale":"2023-09-23T23:42:32.066677+0000","last_undegraded":"2023-09-23T23:42:32.066677+0000","last_fullsized":"2023-09-23T23:42:32.066677+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:02:41.967366+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":333,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.948982+0000","last_change":"2023-09-23T23:41:31.603533+0000","last_active":"2023-09-23T23:42:31.948982+0000","last_peered":"2023-09-23T23:42:31.948982+0000","last_clean":"2023-09-23T23:42:31.948982+0000","last_became_active":"2023-09-23T23:41:31.593485+0000","last_became_peered":"2023-09-23T23:41:31.593485+0000","last_unstale":"2023-09-23T23:42:31.948982+0000","last_undegraded":"2023-09-23T23:42:31.948982+0000","last_fullsized":"2023-09-23T23:42:31.948982+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:10:56.208677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":535,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.955968+0000","last_change":"2023-09-23T23:39:55.101024+0000","last_active":"2023-09-23T23:42:30.955968+0000","last_peered":"2023-09-23T23:42:30.955968+0000","last_clean":"2023-09-23T23:42:30.955968+0000","last_became_active":"2023-09-23T23:39:55.100745+0000","last_became_peered":"2023-09-23T23:39:55.100745+0000","last_unstale":"2023-09-23T23:42:30.955968+0000","last_undegraded":"2023-09-23T23:42:30.955968+0000","last_fullsized":"2023-09-23T23:42:30.955968+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:28.387953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.0","version":"674'9281","reported_seq":9348,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:35.570580+0000","last_change":"2023-09-23T23:39:13.741363+0000","last_active":"2023-09-23T23:42:35.570580+0000","last_peered":"2023-09-23T23:42:35.570580+0000","last_clean":"2023-09-23T23:42:35.570580+0000","last_became_active":"2023-09-23T23:39:13.741098+0000","last_became_peered":"2023-09-23T23:39:13.741098+0000","last_unstale":"2023-09-23T23:42:35.570580+0000","last_undegraded":"2023-09-23T23:42:35.570580+0000","last_fullsized":"2023-09-23T23:42:35.570580+0000","mapping_epoch":617,"log_start":"631'3123","ondisk_log_start":"631'3123","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6158,"log_dups_size":0,"ondisk_log_size":6158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:31.131594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89063424,"num_objects":1359,"num_object_clones":0,"num_object_copies":4077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9281,"num_write_kb":340480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":841,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.950491+0000","last_change":"2023-09-23T23:39:58.122462+0000","last_active":"2023-09-23T23:42:31.950491+0000","last_peered":"2023-09-23T23:42:31.950491+0000","last_clean":"2023-09-23T23:42:31.950491+0000","last_became_active":"2023-09-23T23:39:58.122163+0000","last_became_peered":"2023-09-23T23:39:58.122163+0000","last_unstale":"2023-09-23T23:42:31.950491+0000","last_undegraded":"2023-09-23T23:42:31.950491+0000","last_fullsized":"2023-09-23T23:42:31.950491+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:41.511681+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":426,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.950342+0000","last_change":"2023-09-23T23:39:55.141520+0000","last_active":"2023-09-23T23:42:31.950342+0000","last_peered":"2023-09-23T23:42:31.950342+0000","last_clean":"2023-09-23T23:42:31.950342+0000","last_became_active":"2023-09-23T23:39:55.099176+0000","last_became_peered":"2023-09-23T23:39:55.099176+0000","last_unstale":"2023-09-23T23:42:31.950342+0000","last_undegraded":"2023-09-23T23:42:31.950342+0000","last_fullsized":"2023-09-23T23:42:31.950342+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:12.860173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":56,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.949816+0000","last_change":"2023-09-23T23:39:55.094219+0000","last_active":"2023-09-23T23:42:31.949816+0000","last_peered":"2023-09-23T23:42:31.949816+0000","last_clean":"2023-09-23T23:42:31.949816+0000","last_became_active":"2023-09-23T23:39:55.093849+0000","last_became_peered":"2023-09-23T23:39:55.093849+0000","last_unstale":"2023-09-23T23:42:31.949816+0000","last_undegraded":"2023-09-23T23:42:31.949816+0000","last_fullsized":"2023-09-23T23:42:31.949816+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:54.576029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.1","version":"673'9147","reported_seq":9213,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.781864+0000","last_change":"2023-09-23T23:39:13.744449+0000","last_active":"2023-09-23T23:42:32.781864+0000","last_peered":"2023-09-23T23:42:32.781864+0000","last_clean":"2023-09-23T23:42:32.781864+0000","last_became_active":"2023-09-23T23:39:13.744163+0000","last_became_peered":"2023-09-23T23:39:13.744163+0000","last_unstale":"2023-09-23T23:42:32.781864+0000","last_undegraded":"2023-09-23T23:42:32.781864+0000","last_fullsized":"2023-09-23T23:42:32.781864+0000","mapping_epoch":617,"log_start":"633'4346","ondisk_log_start":"633'4346","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4801,"log_dups_size":0,"ondisk_log_size":4801,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:50.120382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93782016,"num_objects":1431,"num_object_clones":0,"num_object_copies":4293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9147,"num_write_kb":338496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":582,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.970946+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:42:30.970946+0000","last_peered":"2023-09-23T23:42:30.970946+0000","last_clean":"2023-09-23T23:42:30.970946+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:42:30.970946+0000","last_undegraded":"2023-09-23T23:42:30.970946+0000","last_fullsized":"2023-09-23T23:42:30.970946+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":525,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.976053+0000","last_change":"2023-09-23T23:39:55.099560+0000","last_active":"2023-09-23T23:42:30.976053+0000","last_peered":"2023-09-23T23:42:30.976053+0000","last_clean":"2023-09-23T23:42:30.976053+0000","last_became_active":"2023-09-23T23:39:55.099237+0000","last_became_peered":"2023-09-23T23:39:55.099237+0000","last_unstale":"2023-09-23T23:42:30.976053+0000","last_undegraded":"2023-09-23T23:42:30.976053+0000","last_fullsized":"2023-09-23T23:42:30.976053+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:12.522331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":672,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T23:42:29.940159+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:42:29.940159+0000","last_peered":"2023-09-23T23:42:29.940159+0000","last_clean":"2023-09-23T23:42:29.940159+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:42:29.940159+0000","last_undegraded":"2023-09-23T23:42:29.940159+0000","last_fullsized":"2023-09-23T23:42:29.940159+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"674'9380","reported_seq":9447,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:35.604455+0000","last_change":"2023-09-23T23:39:13.745430+0000","last_active":"2023-09-23T23:42:35.604455+0000","last_peered":"2023-09-23T23:42:35.604455+0000","last_clean":"2023-09-23T23:42:35.604455+0000","last_became_active":"2023-09-23T23:39:13.744751+0000","last_became_peered":"2023-09-23T23:39:13.744751+0000","last_unstale":"2023-09-23T23:42:35.604455+0000","last_undegraded":"2023-09-23T23:42:35.604455+0000","last_fullsized":"2023-09-23T23:42:35.604455+0000","mapping_epoch":617,"log_start":"631'3154","ondisk_log_start":"631'3154","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6226,"log_dups_size":0,"ondisk_log_size":6226,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:23.195329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86638592,"num_objects":1322,"num_object_clones":0,"num_object_copies":3966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9380,"num_write_kb":342464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":815,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.956171+0000","last_change":"2023-09-23T23:39:19.985630+0000","last_active":"2023-09-23T23:42:30.956171+0000","last_peered":"2023-09-23T23:42:30.956171+0000","last_clean":"2023-09-23T23:42:30.956171+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:42:30.956171+0000","last_undegraded":"2023-09-23T23:42:30.956171+0000","last_fullsized":"2023-09-23T23:42:30.956171+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:39:19.985512+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:39:19.985512+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:52.750567+0000","scrub_duration":0.001852713,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":712,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.971921+0000","last_change":"2023-09-23T23:41:31.618278+0000","last_active":"2023-09-23T23:42:30.971921+0000","last_peered":"2023-09-23T23:42:30.971921+0000","last_clean":"2023-09-23T23:42:30.971921+0000","last_became_active":"2023-09-23T23:41:31.602519+0000","last_became_peered":"2023-09-23T23:41:31.602519+0000","last_unstale":"2023-09-23T23:42:30.971921+0000","last_undegraded":"2023-09-23T23:42:30.971921+0000","last_fullsized":"2023-09-23T23:42:30.971921+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:20.395010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":605,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.975966+0000","last_change":"2023-09-23T23:39:55.099345+0000","last_active":"2023-09-23T23:42:30.975966+0000","last_peered":"2023-09-23T23:42:30.975966+0000","last_clean":"2023-09-23T23:42:30.975966+0000","last_became_active":"2023-09-23T23:39:55.098875+0000","last_became_peered":"2023-09-23T23:39:55.098875+0000","last_unstale":"2023-09-23T23:42:30.975966+0000","last_undegraded":"2023-09-23T23:42:30.975966+0000","last_fullsized":"2023-09-23T23:42:30.975966+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:09.169232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.7","version":"674'9271","reported_seq":9336,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:34.709469+0000","last_change":"2023-09-23T23:39:13.743009+0000","last_active":"2023-09-23T23:42:34.709469+0000","last_peered":"2023-09-23T23:42:34.709469+0000","last_clean":"2023-09-23T23:42:34.709469+0000","last_became_active":"2023-09-23T23:39:13.742719+0000","last_became_peered":"2023-09-23T23:39:13.742719+0000","last_unstale":"2023-09-23T23:42:34.709469+0000","last_undegraded":"2023-09-23T23:42:34.709469+0000","last_fullsized":"2023-09-23T23:42:34.709469+0000","mapping_epoch":617,"log_start":"632'4076","ondisk_log_start":"632'4076","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":5195,"log_dups_size":0,"ondisk_log_size":5195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:50.189642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90112000,"num_objects":1375,"num_object_clones":0,"num_object_copies":4125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9271,"num_write_kb":340672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":960,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.075870+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:42:32.075870+0000","last_peered":"2023-09-23T23:42:32.075870+0000","last_clean":"2023-09-23T23:42:32.075870+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:42:32.075870+0000","last_undegraded":"2023-09-23T23:42:32.075870+0000","last_fullsized":"2023-09-23T23:42:32.075870+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":148,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.958378+0000","last_change":"2023-09-23T23:42:28.953222+0000","last_active":"2023-09-23T23:42:30.958378+0000","last_peered":"2023-09-23T23:42:30.958378+0000","last_clean":"2023-09-23T23:42:30.958378+0000","last_became_active":"2023-09-23T23:42:28.952902+0000","last_became_peered":"2023-09-23T23:42:28.952902+0000","last_unstale":"2023-09-23T23:42:30.958378+0000","last_undegraded":"2023-09-23T23:42:30.958378+0000","last_fullsized":"2023-09-23T23:42:30.958378+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:21.715883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":569,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.076724+0000","last_change":"2023-09-23T23:41:31.604354+0000","last_active":"2023-09-23T23:42:32.076724+0000","last_peered":"2023-09-23T23:42:32.076724+0000","last_clean":"2023-09-23T23:42:32.076724+0000","last_became_active":"2023-09-23T23:41:31.604035+0000","last_became_peered":"2023-09-23T23:41:31.604035+0000","last_unstale":"2023-09-23T23:42:32.076724+0000","last_undegraded":"2023-09-23T23:42:32.076724+0000","last_fullsized":"2023-09-23T23:42:32.076724+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:55.798632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"673'9062","reported_seq":14891,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.801270+0000","last_change":"2023-09-23T23:40:33.082798+0000","last_active":"2023-09-23T23:42:32.801270+0000","last_peered":"2023-09-23T23:42:32.801270+0000","last_clean":"2023-09-23T23:42:32.801270+0000","last_became_active":"2023-09-23T23:40:33.076970+0000","last_became_peered":"2023-09-23T23:40:33.076970+0000","last_unstale":"2023-09-23T23:42:32.801270+0000","last_undegraded":"2023-09-23T23:42:32.801270+0000","last_fullsized":"2023-09-23T23:42:32.801270+0000","mapping_epoch":634,"log_start":"633'4269","ondisk_log_start":"633'4269","created":617,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4793,"log_dups_size":0,"ondisk_log_size":4793,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:08.992451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90963968,"num_objects":1388,"num_object_clones":0,"num_object_copies":4164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9062,"num_write_kb":334400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2873,"num_bytes_recovered":188284928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":682,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.949897+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:42:31.949897+0000","last_peered":"2023-09-23T23:42:31.949897+0000","last_clean":"2023-09-23T23:42:31.949897+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:42:31.949897+0000","last_undegraded":"2023-09-23T23:42:31.949897+0000","last_fullsized":"2023-09-23T23:42:31.949897+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":540,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.010759+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:42:33.010759+0000","last_peered":"2023-09-23T23:42:33.010759+0000","last_clean":"2023-09-23T23:42:33.010759+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:42:33.010759+0000","last_undegraded":"2023-09-23T23:42:33.010759+0000","last_fullsized":"2023-09-23T23:42:33.010759+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":575,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.970696+0000","last_change":"2023-09-23T23:42:28.951851+0000","last_active":"2023-09-23T23:42:30.970696+0000","last_peered":"2023-09-23T23:42:30.970696+0000","last_clean":"2023-09-23T23:42:30.970696+0000","last_became_active":"2023-09-23T23:42:28.951542+0000","last_became_peered":"2023-09-23T23:42:28.951542+0000","last_unstale":"2023-09-23T23:42:30.970696+0000","last_undegraded":"2023-09-23T23:42:30.970696+0000","last_fullsized":"2023-09-23T23:42:30.970696+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:13.264990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.5","version":"673'9097","reported_seq":9163,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.760376+0000","last_change":"2023-09-23T23:39:13.742854+0000","last_active":"2023-09-23T23:42:32.760376+0000","last_peered":"2023-09-23T23:42:32.760376+0000","last_clean":"2023-09-23T23:42:32.760376+0000","last_became_active":"2023-09-23T23:39:13.742579+0000","last_became_peered":"2023-09-23T23:39:13.742579+0000","last_unstale":"2023-09-23T23:42:32.760376+0000","last_undegraded":"2023-09-23T23:42:32.760376+0000","last_fullsized":"2023-09-23T23:42:32.760376+0000","mapping_epoch":617,"log_start":"633'4326","ondisk_log_start":"633'4326","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4771,"log_dups_size":0,"ondisk_log_size":4771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:51.872071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93913088,"num_objects":1433,"num_object_clones":0,"num_object_copies":4299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9097,"num_write_kb":336960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":1035,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.070893+0000","last_change":"2023-09-23T23:39:58.119762+0000","last_active":"2023-09-23T23:42:32.070893+0000","last_peered":"2023-09-23T23:42:32.070893+0000","last_clean":"2023-09-23T23:42:32.070893+0000","last_became_active":"2023-09-23T23:39:58.119422+0000","last_became_peered":"2023-09-23T23:39:58.119422+0000","last_unstale":"2023-09-23T23:42:32.070893+0000","last_undegraded":"2023-09-23T23:42:32.070893+0000","last_fullsized":"2023-09-23T23:42:32.070893+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:03.564930+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":317,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.955672+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:42:30.955672+0000","last_peered":"2023-09-23T23:42:30.955672+0000","last_clean":"2023-09-23T23:42:30.955672+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:42:30.955672+0000","last_undegraded":"2023-09-23T23:42:30.955672+0000","last_fullsized":"2023-09-23T23:42:30.955672+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":605,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.971475+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:42:30.971475+0000","last_peered":"2023-09-23T23:42:30.971475+0000","last_clean":"2023-09-23T23:42:30.971475+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:42:30.971475+0000","last_undegraded":"2023-09-23T23:42:30.971475+0000","last_fullsized":"2023-09-23T23:42:30.971475+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.8","version":"674'9376","reported_seq":10781,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:35.616744+0000","last_change":"2023-09-23T23:42:28.980646+0000","last_active":"2023-09-23T23:42:35.616744+0000","last_peered":"2023-09-23T23:42:35.616744+0000","last_clean":"2023-09-23T23:42:35.616744+0000","last_became_active":"2023-09-23T23:42:28.958020+0000","last_became_peered":"2023-09-23T23:42:28.958020+0000","last_unstale":"2023-09-23T23:42:35.616744+0000","last_undegraded":"2023-09-23T23:42:35.616744+0000","last_fullsized":"2023-09-23T23:42:35.616744+0000","mapping_epoch":669,"log_start":"631'3198","ondisk_log_start":"631'3198","created":617,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6178,"log_dups_size":0,"ondisk_log_size":6178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:21.232496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83492864,"num_objects":1274,"num_object_clones":0,"num_object_copies":3822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9376,"num_write_kb":340800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":656,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":878,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.971256+0000","last_change":"2023-09-23T23:39:58.120057+0000","last_active":"2023-09-23T23:42:30.971256+0000","last_peered":"2023-09-23T23:42:30.971256+0000","last_clean":"2023-09-23T23:42:30.971256+0000","last_became_active":"2023-09-23T23:39:58.119647+0000","last_became_peered":"2023-09-23T23:39:58.119647+0000","last_unstale":"2023-09-23T23:42:30.971256+0000","last_undegraded":"2023-09-23T23:42:30.971256+0000","last_fullsized":"2023-09-23T23:42:30.971256+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:50.965369+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":386,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.072200+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:42:32.072200+0000","last_peered":"2023-09-23T23:42:32.072200+0000","last_clean":"2023-09-23T23:42:32.072200+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:42:32.072200+0000","last_undegraded":"2023-09-23T23:42:32.072200+0000","last_fullsized":"2023-09-23T23:42:32.072200+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":689,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.950676+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:42:31.950676+0000","last_peered":"2023-09-23T23:42:31.950676+0000","last_clean":"2023-09-23T23:42:31.950676+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:42:31.950676+0000","last_undegraded":"2023-09-23T23:42:31.950676+0000","last_fullsized":"2023-09-23T23:42:31.950676+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":290,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.949898+0000","last_change":"2023-09-23T23:42:28.950994+0000","last_active":"2023-09-23T23:42:31.949898+0000","last_peered":"2023-09-23T23:42:31.949898+0000","last_clean":"2023-09-23T23:42:31.949898+0000","last_became_active":"2023-09-23T23:42:28.950464+0000","last_became_peered":"2023-09-23T23:42:28.950464+0000","last_unstale":"2023-09-23T23:42:31.949898+0000","last_undegraded":"2023-09-23T23:42:31.949898+0000","last_fullsized":"2023-09-23T23:42:31.949898+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:57.975234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.e","version":"674'9236","reported_seq":15214,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:35.533177+0000","last_change":"2023-09-23T23:42:21.783482+0000","last_active":"2023-09-23T23:42:35.533177+0000","last_peered":"2023-09-23T23:42:35.533177+0000","last_clean":"2023-09-23T23:42:35.533177+0000","last_became_active":"2023-09-23T23:42:21.754054+0000","last_became_peered":"2023-09-23T23:42:21.754054+0000","last_unstale":"2023-09-23T23:42:35.533177+0000","last_undegraded":"2023-09-23T23:42:35.533177+0000","last_fullsized":"2023-09-23T23:42:35.533177+0000","mapping_epoch":667,"log_start":"631'3104","ondisk_log_start":"631'3104","created":617,"last_epoch_clean":668,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6132,"log_dups_size":0,"ondisk_log_size":6132,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:33.850501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82575360,"num_objects":1260,"num_object_clones":0,"num_object_copies":3780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9236,"num_write_kb":335872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2945,"num_bytes_recovered":134676480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":758,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T23:42:29.938582+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:42:29.938582+0000","last_peered":"2023-09-23T23:42:29.938582+0000","last_clean":"2023-09-23T23:42:29.938582+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:42:29.938582+0000","last_undegraded":"2023-09-23T23:42:29.938582+0000","last_fullsized":"2023-09-23T23:42:29.938582+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":327,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.949954+0000","last_change":"2023-09-23T23:39:55.089555+0000","last_active":"2023-09-23T23:42:31.949954+0000","last_peered":"2023-09-23T23:42:31.949954+0000","last_clean":"2023-09-23T23:42:31.949954+0000","last_became_active":"2023-09-23T23:39:55.089184+0000","last_became_peered":"2023-09-23T23:39:55.089184+0000","last_unstale":"2023-09-23T23:42:31.949954+0000","last_undegraded":"2023-09-23T23:42:31.949954+0000","last_fullsized":"2023-09-23T23:42:31.949954+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:19.775188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":537,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T23:42:29.940368+0000","last_change":"2023-09-23T23:39:55.089351+0000","last_active":"2023-09-23T23:42:29.940368+0000","last_peered":"2023-09-23T23:42:29.940368+0000","last_clean":"2023-09-23T23:42:29.940368+0000","last_became_active":"2023-09-23T23:39:55.089016+0000","last_became_peered":"2023-09-23T23:39:55.089016+0000","last_unstale":"2023-09-23T23:42:29.940368+0000","last_undegraded":"2023-09-23T23:42:29.940368+0000","last_fullsized":"2023-09-23T23:42:29.940368+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:05:23.856265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":310,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.072948+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:42:32.072948+0000","last_peered":"2023-09-23T23:42:32.072948+0000","last_clean":"2023-09-23T23:42:32.072948+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:42:32.072948+0000","last_undegraded":"2023-09-23T23:42:32.072948+0000","last_fullsized":"2023-09-23T23:42:32.072948+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"674'9037","reported_seq":14702,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:35.616171+0000","last_change":"2023-09-23T23:42:01.335232+0000","last_active":"2023-09-23T23:42:35.616171+0000","last_peered":"2023-09-23T23:42:35.616171+0000","last_clean":"2023-09-23T23:42:35.616171+0000","last_became_active":"2023-09-23T23:42:01.316912+0000","last_became_peered":"2023-09-23T23:42:01.316912+0000","last_unstale":"2023-09-23T23:42:35.616171+0000","last_undegraded":"2023-09-23T23:42:35.616171+0000","last_fullsized":"2023-09-23T23:42:35.616171+0000","mapping_epoch":662,"log_start":"631'2939","ondisk_log_start":"631'2939","created":617,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6098,"log_dups_size":0,"ondisk_log_size":6098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:34:25.843454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85852184,"num_objects":1311,"num_object_clones":0,"num_object_copies":3933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1311,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9037,"num_write_kb":331073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2789,"num_bytes_recovered":115015680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":978,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.070820+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:42:32.070820+0000","last_peered":"2023-09-23T23:42:32.070820+0000","last_clean":"2023-09-23T23:42:32.070820+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:42:32.070820+0000","last_undegraded":"2023-09-23T23:42:32.070820+0000","last_fullsized":"2023-09-23T23:42:32.070820+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":415,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.071220+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:42:32.071220+0000","last_peered":"2023-09-23T23:42:32.071220+0000","last_clean":"2023-09-23T23:42:32.071220+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:42:32.071220+0000","last_undegraded":"2023-09-23T23:42:32.071220+0000","last_fullsized":"2023-09-23T23:42:32.071220+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":334,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.075961+0000","last_change":"2023-09-23T23:39:55.093320+0000","last_active":"2023-09-23T23:42:32.075961+0000","last_peered":"2023-09-23T23:42:32.075961+0000","last_clean":"2023-09-23T23:42:32.075961+0000","last_became_active":"2023-09-23T23:39:55.092998+0000","last_became_peered":"2023-09-23T23:39:55.092998+0000","last_unstale":"2023-09-23T23:42:32.075961+0000","last_undegraded":"2023-09-23T23:42:32.075961+0000","last_fullsized":"2023-09-23T23:42:32.075961+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:38.763321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":15,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.026533+0000","last_change":"2023-09-23T23:42:28.952433+0000","last_active":"2023-09-23T23:42:33.026533+0000","last_peered":"2023-09-23T23:42:33.026533+0000","last_clean":"2023-09-23T23:42:33.026533+0000","last_became_active":"2023-09-23T23:42:28.952139+0000","last_became_peered":"2023-09-23T23:42:28.952139+0000","last_unstale":"2023-09-23T23:42:33.026533+0000","last_undegraded":"2023-09-23T23:42:33.026533+0000","last_fullsized":"2023-09-23T23:42:33.026533+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:49.952980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"673'9183","reported_seq":9249,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.781613+0000","last_change":"2023-09-23T23:39:13.747297+0000","last_active":"2023-09-23T23:42:32.781613+0000","last_peered":"2023-09-23T23:42:32.781613+0000","last_clean":"2023-09-23T23:42:32.781613+0000","last_became_active":"2023-09-23T23:39:13.747068+0000","last_became_peered":"2023-09-23T23:39:13.747068+0000","last_unstale":"2023-09-23T23:42:32.781613+0000","last_undegraded":"2023-09-23T23:42:32.781613+0000","last_fullsized":"2023-09-23T23:42:32.781613+0000","mapping_epoch":617,"log_start":"633'4381","ondisk_log_start":"633'4381","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4802,"log_dups_size":0,"ondisk_log_size":4802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:34.450587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91947008,"num_objects":1403,"num_object_clones":0,"num_object_copies":4209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9183,"num_write_kb":338752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":692,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.071771+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:42:32.071771+0000","last_peered":"2023-09-23T23:42:32.071771+0000","last_clean":"2023-09-23T23:42:32.071771+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:42:32.071771+0000","last_undegraded":"2023-09-23T23:42:32.071771+0000","last_fullsized":"2023-09-23T23:42:32.071771+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":993,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.948186+0000","last_change":"2023-09-23T23:39:58.122811+0000","last_active":"2023-09-23T23:42:31.948186+0000","last_peered":"2023-09-23T23:42:31.948186+0000","last_clean":"2023-09-23T23:42:31.948186+0000","last_became_active":"2023-09-23T23:39:58.122461+0000","last_became_peered":"2023-09-23T23:39:58.122461+0000","last_unstale":"2023-09-23T23:42:31.948186+0000","last_undegraded":"2023-09-23T23:42:31.948186+0000","last_fullsized":"2023-09-23T23:42:31.948186+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:20:59.163746+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":647,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.967713+0000","last_change":"2023-09-23T23:39:55.101203+0000","last_active":"2023-09-23T23:42:30.967713+0000","last_peered":"2023-09-23T23:42:30.967713+0000","last_clean":"2023-09-23T23:42:30.967713+0000","last_became_active":"2023-09-23T23:39:55.099575+0000","last_became_peered":"2023-09-23T23:39:55.099575+0000","last_unstale":"2023-09-23T23:42:30.967713+0000","last_undegraded":"2023-09-23T23:42:30.967713+0000","last_fullsized":"2023-09-23T23:42:30.967713+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:40.288184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":57,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.976380+0000","last_change":"2023-09-23T23:39:55.103713+0000","last_active":"2023-09-23T23:42:30.976380+0000","last_peered":"2023-09-23T23:42:30.976380+0000","last_clean":"2023-09-23T23:42:30.976380+0000","last_became_active":"2023-09-23T23:39:55.103402+0000","last_became_peered":"2023-09-23T23:39:55.103402+0000","last_unstale":"2023-09-23T23:42:30.976380+0000","last_undegraded":"2023-09-23T23:42:30.976380+0000","last_fullsized":"2023-09-23T23:42:30.976380+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:44.949292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.c","version":"674'9108","reported_seq":14859,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:35.601236+0000","last_change":"2023-09-23T23:41:10.964608+0000","last_active":"2023-09-23T23:42:35.601236+0000","last_peered":"2023-09-23T23:42:35.601236+0000","last_clean":"2023-09-23T23:42:35.601236+0000","last_became_active":"2023-09-23T23:41:10.952515+0000","last_became_peered":"2023-09-23T23:41:10.952515+0000","last_unstale":"2023-09-23T23:42:35.601236+0000","last_undegraded":"2023-09-23T23:42:35.601236+0000","last_fullsized":"2023-09-23T23:42:35.601236+0000","mapping_epoch":644,"log_start":"630'2926","ondisk_log_start":"630'2926","created":617,"last_epoch_clean":645,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6182,"log_dups_size":0,"ondisk_log_size":6182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:30.860608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88342528,"num_objects":1348,"num_object_clones":0,"num_object_copies":4044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9108,"num_write_kb":334592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2834,"num_bytes_recovered":185729024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":837,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.072123+0000","last_change":"2023-09-23T23:41:34.668439+0000","last_active":"2023-09-23T23:42:32.072123+0000","last_peered":"2023-09-23T23:42:32.072123+0000","last_clean":"2023-09-23T23:42:32.072123+0000","last_became_active":"2023-09-23T23:41:34.668104+0000","last_became_peered":"2023-09-23T23:41:34.668104+0000","last_unstale":"2023-09-23T23:42:32.072123+0000","last_undegraded":"2023-09-23T23:42:32.072123+0000","last_fullsized":"2023-09-23T23:42:32.072123+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:42:24.264118+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":601,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.947923+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:42:31.947923+0000","last_peered":"2023-09-23T23:42:31.947923+0000","last_clean":"2023-09-23T23:42:31.947923+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:42:31.947923+0000","last_undegraded":"2023-09-23T23:42:31.947923+0000","last_fullsized":"2023-09-23T23:42:31.947923+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":517,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.956008+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:42:30.956008+0000","last_peered":"2023-09-23T23:42:30.956008+0000","last_clean":"2023-09-23T23:42:30.956008+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:42:30.956008+0000","last_undegraded":"2023-09-23T23:42:30.956008+0000","last_fullsized":"2023-09-23T23:42:30.956008+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":29,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T23:42:29.941041+0000","last_change":"2023-09-23T23:41:31.605087+0000","last_active":"2023-09-23T23:42:29.941041+0000","last_peered":"2023-09-23T23:42:29.941041+0000","last_clean":"2023-09-23T23:42:29.941041+0000","last_became_active":"2023-09-23T23:41:31.604777+0000","last_became_peered":"2023-09-23T23:41:31.604777+0000","last_unstale":"2023-09-23T23:42:29.941041+0000","last_undegraded":"2023-09-23T23:42:29.941041+0000","last_fullsized":"2023-09-23T23:42:29.941041+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:57.416801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"673'8912","reported_seq":8978,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.692697+0000","last_change":"2023-09-23T23:39:13.742801+0000","last_active":"2023-09-23T23:42:32.692697+0000","last_peered":"2023-09-23T23:42:32.692697+0000","last_clean":"2023-09-23T23:42:32.692697+0000","last_became_active":"2023-09-23T23:39:13.742482+0000","last_became_peered":"2023-09-23T23:39:13.742482+0000","last_unstale":"2023-09-23T23:42:32.692697+0000","last_undegraded":"2023-09-23T23:42:32.692697+0000","last_fullsized":"2023-09-23T23:42:32.692697+0000","mapping_epoch":617,"log_start":"633'4142","ondisk_log_start":"633'4142","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4770,"log_dups_size":0,"ondisk_log_size":4770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:15.737461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86900736,"num_objects":1326,"num_object_clones":0,"num_object_copies":3978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8912,"num_write_kb":327616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":600,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.970823+0000","last_change":"2023-09-23T23:39:58.112420+0000","last_active":"2023-09-23T23:42:30.970823+0000","last_peered":"2023-09-23T23:42:30.970823+0000","last_clean":"2023-09-23T23:42:30.970823+0000","last_became_active":"2023-09-23T23:39:58.111949+0000","last_became_peered":"2023-09-23T23:39:58.111949+0000","last_unstale":"2023-09-23T23:42:30.970823+0000","last_undegraded":"2023-09-23T23:42:30.970823+0000","last_fullsized":"2023-09-23T23:42:30.970823+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:20.354620+0000","scrub_duration":0.0047245100000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":115,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T23:42:29.939254+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:42:29.939254+0000","last_peered":"2023-09-23T23:42:29.939254+0000","last_clean":"2023-09-23T23:42:29.939254+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:42:29.939254+0000","last_undegraded":"2023-09-23T23:42:29.939254+0000","last_fullsized":"2023-09-23T23:42:29.939254+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":217,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.073031+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:42:32.073031+0000","last_peered":"2023-09-23T23:42:32.073031+0000","last_clean":"2023-09-23T23:42:32.073031+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:42:32.073031+0000","last_undegraded":"2023-09-23T23:42:32.073031+0000","last_fullsized":"2023-09-23T23:42:32.073031+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":407,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.950092+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:42:31.950092+0000","last_peered":"2023-09-23T23:42:31.950092+0000","last_clean":"2023-09-23T23:42:31.950092+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:42:31.950092+0000","last_undegraded":"2023-09-23T23:42:31.950092+0000","last_fullsized":"2023-09-23T23:42:31.950092+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.a","version":"673'8859","reported_seq":8925,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.787472+0000","last_change":"2023-09-23T23:39:13.746851+0000","last_active":"2023-09-23T23:42:32.787472+0000","last_peered":"2023-09-23T23:42:32.787472+0000","last_clean":"2023-09-23T23:42:32.787472+0000","last_became_active":"2023-09-23T23:39:13.746412+0000","last_became_peered":"2023-09-23T23:39:13.746412+0000","last_unstale":"2023-09-23T23:42:32.787472+0000","last_undegraded":"2023-09-23T23:42:32.787472+0000","last_fullsized":"2023-09-23T23:42:32.787472+0000","mapping_epoch":617,"log_start":"633'4072","ondisk_log_start":"633'4072","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4787,"log_dups_size":0,"ondisk_log_size":4787,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:39.662209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93650944,"num_objects":1429,"num_object_clones":0,"num_object_copies":4287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8859,"num_write_kb":329216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":875,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.076809+0000","last_change":"2023-09-23T23:39:58.122659+0000","last_active":"2023-09-23T23:42:32.076809+0000","last_peered":"2023-09-23T23:42:32.076809+0000","last_clean":"2023-09-23T23:42:32.076809+0000","last_became_active":"2023-09-23T23:39:58.122334+0000","last_became_peered":"2023-09-23T23:39:58.122334+0000","last_unstale":"2023-09-23T23:42:32.076809+0000","last_undegraded":"2023-09-23T23:42:32.076809+0000","last_fullsized":"2023-09-23T23:42:32.076809+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:29.154690+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":673,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.971433+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:42:30.971433+0000","last_peered":"2023-09-23T23:42:30.971433+0000","last_clean":"2023-09-23T23:42:30.971433+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:42:30.971433+0000","last_undegraded":"2023-09-23T23:42:30.971433+0000","last_fullsized":"2023-09-23T23:42:30.971433+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":216,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.970711+0000","last_change":"2023-09-23T23:39:55.100517+0000","last_active":"2023-09-23T23:42:30.970711+0000","last_peered":"2023-09-23T23:42:30.970711+0000","last_clean":"2023-09-23T23:42:30.970711+0000","last_became_active":"2023-09-23T23:39:55.098755+0000","last_became_peered":"2023-09-23T23:39:55.098755+0000","last_unstale":"2023-09-23T23:42:30.970711+0000","last_undegraded":"2023-09-23T23:42:30.970711+0000","last_fullsized":"2023-09-23T23:42:30.970711+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:50.086720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":246,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.075720+0000","last_change":"2023-09-23T23:41:31.604932+0000","last_active":"2023-09-23T23:42:32.075720+0000","last_peered":"2023-09-23T23:42:32.075720+0000","last_clean":"2023-09-23T23:42:32.075720+0000","last_became_active":"2023-09-23T23:41:31.604686+0000","last_became_peered":"2023-09-23T23:41:31.604686+0000","last_unstale":"2023-09-23T23:42:32.075720+0000","last_undegraded":"2023-09-23T23:42:32.075720+0000","last_fullsized":"2023-09-23T23:42:32.075720+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.931148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.b","version":"674'9095","reported_seq":9162,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:35.601271+0000","last_change":"2023-09-23T23:39:13.744730+0000","last_active":"2023-09-23T23:42:35.601271+0000","last_peered":"2023-09-23T23:42:35.601271+0000","last_clean":"2023-09-23T23:42:35.601271+0000","last_became_active":"2023-09-23T23:39:13.744442+0000","last_became_peered":"2023-09-23T23:39:13.744442+0000","last_unstale":"2023-09-23T23:42:35.601271+0000","last_undegraded":"2023-09-23T23:42:35.601271+0000","last_fullsized":"2023-09-23T23:42:35.601271+0000","mapping_epoch":617,"log_start":"631'3012","ondisk_log_start":"631'3012","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6083,"log_dups_size":0,"ondisk_log_size":6083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:56:43.437044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83427328,"num_objects":1273,"num_object_clones":0,"num_object_copies":3819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9095,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":1063,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.976339+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:42:30.976339+0000","last_peered":"2023-09-23T23:42:30.976339+0000","last_clean":"2023-09-23T23:42:30.976339+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:42:30.976339+0000","last_undegraded":"2023-09-23T23:42:30.976339+0000","last_fullsized":"2023-09-23T23:42:30.976339+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":390,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.075790+0000","last_change":"2023-09-23T23:39:55.096945+0000","last_active":"2023-09-23T23:42:32.075790+0000","last_peered":"2023-09-23T23:42:32.075790+0000","last_clean":"2023-09-23T23:42:32.075790+0000","last_became_active":"2023-09-23T23:39:55.096655+0000","last_became_peered":"2023-09-23T23:39:55.096655+0000","last_unstale":"2023-09-23T23:42:32.075790+0000","last_undegraded":"2023-09-23T23:42:32.075790+0000","last_fullsized":"2023-09-23T23:42:32.075790+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:27:24.290604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":33,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T23:42:29.940962+0000","last_change":"2023-09-23T23:41:31.599981+0000","last_active":"2023-09-23T23:42:29.940962+0000","last_peered":"2023-09-23T23:42:29.940962+0000","last_clean":"2023-09-23T23:42:29.940962+0000","last_became_active":"2023-09-23T23:41:31.597250+0000","last_became_peered":"2023-09-23T23:41:31.597250+0000","last_unstale":"2023-09-23T23:42:29.940962+0000","last_undegraded":"2023-09-23T23:42:29.940962+0000","last_fullsized":"2023-09-23T23:42:29.940962+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:06.361776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":258,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.949573+0000","last_change":"2023-09-23T23:39:55.092828+0000","last_active":"2023-09-23T23:42:31.949573+0000","last_peered":"2023-09-23T23:42:31.949573+0000","last_clean":"2023-09-23T23:42:31.949573+0000","last_became_active":"2023-09-23T23:39:55.092504+0000","last_became_peered":"2023-09-23T23:39:55.092504+0000","last_unstale":"2023-09-23T23:42:31.949573+0000","last_undegraded":"2023-09-23T23:42:31.949573+0000","last_fullsized":"2023-09-23T23:42:31.949573+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:59.997875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1163,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.956127+0000","last_change":"2023-09-23T23:41:34.675056+0000","last_active":"2023-09-23T23:42:30.956127+0000","last_peered":"2023-09-23T23:42:30.956127+0000","last_clean":"2023-09-23T23:42:30.956127+0000","last_became_active":"2023-09-23T23:41:34.674727+0000","last_became_peered":"2023-09-23T23:41:34.674727+0000","last_unstale":"2023-09-23T23:42:30.956127+0000","last_undegraded":"2023-09-23T23:42:30.956127+0000","last_fullsized":"2023-09-23T23:42:30.956127+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:01.911339+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":40,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.955430+0000","last_change":"2023-09-23T23:42:28.952637+0000","last_active":"2023-09-23T23:42:30.955430+0000","last_peered":"2023-09-23T23:42:30.955430+0000","last_clean":"2023-09-23T23:42:30.955430+0000","last_became_active":"2023-09-23T23:42:28.952283+0000","last_became_peered":"2023-09-23T23:42:28.952283+0000","last_unstale":"2023-09-23T23:42:30.955430+0000","last_undegraded":"2023-09-23T23:42:30.955430+0000","last_fullsized":"2023-09-23T23:42:30.955430+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:14.296297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":431,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.073108+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:42:32.073108+0000","last_peered":"2023-09-23T23:42:32.073108+0000","last_clean":"2023-09-23T23:42:32.073108+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:42:32.073108+0000","last_undegraded":"2023-09-23T23:42:32.073108+0000","last_fullsized":"2023-09-23T23:42:32.073108+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":875,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.971070+0000","last_change":"2023-09-23T23:41:34.667657+0000","last_active":"2023-09-23T23:42:30.971070+0000","last_peered":"2023-09-23T23:42:30.971070+0000","last_clean":"2023-09-23T23:42:30.971070+0000","last_became_active":"2023-09-23T23:41:34.659982+0000","last_became_peered":"2023-09-23T23:41:34.659982+0000","last_unstale":"2023-09-23T23:42:30.971070+0000","last_undegraded":"2023-09-23T23:42:30.971070+0000","last_fullsized":"2023-09-23T23:42:30.971070+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:49.342830+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":333,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.950272+0000","last_change":"2023-09-23T23:39:55.115869+0000","last_active":"2023-09-23T23:42:31.950272+0000","last_peered":"2023-09-23T23:42:31.950272+0000","last_clean":"2023-09-23T23:42:31.950272+0000","last_became_active":"2023-09-23T23:39:55.092827+0000","last_became_peered":"2023-09-23T23:39:55.092827+0000","last_unstale":"2023-09-23T23:42:31.950272+0000","last_undegraded":"2023-09-23T23:42:31.950272+0000","last_fullsized":"2023-09-23T23:42:31.950272+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:03.590720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":541,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.971843+0000","last_change":"2023-09-23T23:42:28.956777+0000","last_active":"2023-09-23T23:42:30.971843+0000","last_peered":"2023-09-23T23:42:30.971843+0000","last_clean":"2023-09-23T23:42:30.971843+0000","last_became_active":"2023-09-23T23:42:28.956436+0000","last_became_peered":"2023-09-23T23:42:28.956436+0000","last_unstale":"2023-09-23T23:42:30.971843+0000","last_undegraded":"2023-09-23T23:42:30.971843+0000","last_fullsized":"2023-09-23T23:42:30.971843+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:42.633268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":61,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.967780+0000","last_change":"2023-09-23T23:39:55.128319+0000","last_active":"2023-09-23T23:42:30.967780+0000","last_peered":"2023-09-23T23:42:30.967780+0000","last_clean":"2023-09-23T23:42:30.967780+0000","last_became_active":"2023-09-23T23:39:55.098936+0000","last_became_peered":"2023-09-23T23:39:55.098936+0000","last_unstale":"2023-09-23T23:42:30.967780+0000","last_undegraded":"2023-09-23T23:42:30.967780+0000","last_fullsized":"2023-09-23T23:42:30.967780+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:03.570174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":362,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.967795+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:42:30.967795+0000","last_peered":"2023-09-23T23:42:30.967795+0000","last_clean":"2023-09-23T23:42:30.967795+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:42:30.967795+0000","last_undegraded":"2023-09-23T23:42:30.967795+0000","last_fullsized":"2023-09-23T23:42:30.967795+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":919,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.948573+0000","last_change":"2023-09-23T23:39:58.111742+0000","last_active":"2023-09-23T23:42:31.948573+0000","last_peered":"2023-09-23T23:42:31.948573+0000","last_clean":"2023-09-23T23:42:31.948573+0000","last_became_active":"2023-09-23T23:39:58.111358+0000","last_became_peered":"2023-09-23T23:39:58.111358+0000","last_unstale":"2023-09-23T23:42:31.948573+0000","last_undegraded":"2023-09-23T23:42:31.948573+0000","last_fullsized":"2023-09-23T23:42:31.948573+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:54.588394+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":715,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.971033+0000","last_change":"2023-09-23T23:39:55.101668+0000","last_active":"2023-09-23T23:42:30.971033+0000","last_peered":"2023-09-23T23:42:30.971033+0000","last_clean":"2023-09-23T23:42:30.971033+0000","last_became_active":"2023-09-23T23:39:55.100664+0000","last_became_peered":"2023-09-23T23:39:55.100664+0000","last_unstale":"2023-09-23T23:42:30.971033+0000","last_undegraded":"2023-09-23T23:42:30.971033+0000","last_fullsized":"2023-09-23T23:42:30.971033+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:43.354111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":690,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.067317+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:42:32.067317+0000","last_peered":"2023-09-23T23:42:32.067317+0000","last_clean":"2023-09-23T23:42:32.067317+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:42:32.067317+0000","last_undegraded":"2023-09-23T23:42:32.067317+0000","last_fullsized":"2023-09-23T23:42:32.067317+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":848,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.976566+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:42:30.976566+0000","last_peered":"2023-09-23T23:42:30.976566+0000","last_clean":"2023-09-23T23:42:30.976566+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:42:30.976566+0000","last_undegraded":"2023-09-23T23:42:30.976566+0000","last_fullsized":"2023-09-23T23:42:30.976566+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":550,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.075880+0000","last_change":"2023-09-23T23:41:31.624156+0000","last_active":"2023-09-23T23:42:32.075880+0000","last_peered":"2023-09-23T23:42:32.075880+0000","last_clean":"2023-09-23T23:42:32.075880+0000","last_became_active":"2023-09-23T23:41:31.612962+0000","last_became_peered":"2023-09-23T23:41:31.612962+0000","last_unstale":"2023-09-23T23:42:32.075880+0000","last_undegraded":"2023-09-23T23:42:32.075880+0000","last_fullsized":"2023-09-23T23:42:32.075880+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:31.042576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":673,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T23:42:30.955592+0000","last_change":"2023-09-23T23:39:55.099784+0000","last_active":"2023-09-23T23:42:30.955592+0000","last_peered":"2023-09-23T23:42:30.955592+0000","last_clean":"2023-09-23T23:42:30.955592+0000","last_became_active":"2023-09-23T23:39:55.099483+0000","last_became_peered":"2023-09-23T23:39:55.099483+0000","last_unstale":"2023-09-23T23:42:30.955592+0000","last_undegraded":"2023-09-23T23:42:30.955592+0000","last_fullsized":"2023-09-23T23:42:30.955592+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:48.012026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":1425211416,"num_objects":21748,"num_object_clones":0,"num_object_copies":65244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21748,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":146006,"num_write_kb":5368065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14934,"num_bytes_recovered":809631744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4714672128,"data_stored":4714659912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86655,"ondisk_log_size":86655,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":71,"num_bytes_recovered":12612,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":143360,"data_stored":26230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782757,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1042456,"kb_used_data":556240,"kb_used_omap":8,"kb_used_meta":486199,"kb_avail":92702696,"statfs":{"total":95995035648,"available":94927560704,"internally_reserved":0,"allocated":569589760,"data_stored":567633673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":497867845},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.0900000000000001,"15min":1.6180000000000001},"min":{"1min":0.23799999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":3.528,"5min":12.432,"15min":44.094999999999999},"last":0.255},{"interface":"front","average":{"1min":1.353,"5min":1.3320000000000001,"15min":1.76},"min":{"1min":0.27100000000000002,"5min":0.22700000000000001,"15min":0.21299999999999999},"max":{"1min":4.4720000000000004,"5min":12.717000000000001,"15min":44.335999999999999},"last":1.0840000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":1.377,"15min":3.0409999999999999},"min":{"1min":0.28499999999999998,"5min":0.255,"15min":0.23200000000000001},"max":{"1min":5.2229999999999999,"5min":12.484,"15min":198.58000000000001},"last":0.31},{"interface":"front","average":{"1min":1.575,"5min":1.526,"15min":3.0030000000000001},"min":{"1min":0.28999999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.6749999999999998,"5min":13.145,"15min":198.71100000000001},"last":0.32500000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":2.524,"5min":2.3599999999999999,"15min":3.798},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":8.9779999999999998,"5min":46.374000000000002,"15min":203.58000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.909,"5min":2.5230000000000001,"15min":3.4399999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26200000000000001},"max":{"1min":9.9670000000000005,"5min":46.412999999999997,"15min":198.69},"last":1.24}]},{"osd":3,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":2.2759999999999998,"5min":5.1429999999999998,"15min":5.2770000000000001},"min":{"1min":0.31,"5min":0.309,"15min":0.26700000000000002},"max":{"1min":8.1479999999999997,"5min":154.18100000000001,"15min":279.26499999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":2.931,"5min":5.0919999999999996,"15min":5.8150000000000004},"min":{"1min":0.32600000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":8.5779999999999994,"5min":154.16300000000001,"15min":279.29899999999998},"last":0.45600000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.69199999999999995,"15min":1.008},"min":{"1min":0.29599999999999999,"5min":0.28999999999999998,"15min":0.246},"max":{"1min":1.5169999999999999,"5min":12.510999999999999,"15min":31.498999999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.435,"5min":0.80100000000000005,"15min":0.93999999999999995},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.23899999999999999},"max":{"1min":0.82399999999999995,"5min":12.821999999999999,"15min":31.68},"last":0.40300000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":2.0659999999999998,"15min":2.2090000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.88300000000000001,"5min":119.417,"15min":210.83600000000001},"last":0.438},{"interface":"front","average":{"1min":0.40999999999999998,"5min":2.1669999999999998,"15min":2.1949999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.27700000000000002},"max":{"1min":0.54900000000000004,"5min":119.61499999999999,"15min":207.04599999999999},"last":0.39300000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.84199999999999997,"15min":1.149},"min":{"1min":0.34499999999999997,"5min":0.32800000000000001,"15min":0.29899999999999999},"max":{"1min":0.77900000000000003,"5min":6.2169999999999996,"15min":94.852999999999994},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.81899999999999995,"15min":1.274},"min":{"1min":0.36099999999999999,"5min":0.31,"15min":0.30599999999999999},"max":{"1min":0.84999999999999998,"5min":12.529999999999999,"15min":95.510999999999996},"last":0.47699999999999998}]}]},{"osd":6,"up_from":0,"seq":197568496234,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":112,"seq":481036337575,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1848616,"kb_used_data":815904,"kb_used_omap":4,"kb_used_meta":1032507,"kb_avail":91896536,"statfs":{"total":95995035648,"available":94102052864,"internally_reserved":0,"allocated":835485696,"data_stored":832187997,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1057288078},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:42:31 2023","interfaces":[{"interface":"back","average":{"1min":3.0979999999999999,"5min":5.2750000000000004,"15min":3.9780000000000002},"min":{"1min":0.20999999999999999,"5min":0.16400000000000001,"15min":0.16400000000000001},"max":{"1min":9.1319999999999997,"5min":156.37799999999999,"15min":156.37799999999999},"last":5.8239999999999998},{"interface":"front","average":{"1min":1.821,"5min":4.7320000000000002,"15min":3.4220000000000002},"min":{"1min":0.221,"5min":0.189,"15min":0.189},"max":{"1min":8.8049999999999997,"5min":161.05199999999999,"15min":161.05199999999999},"last":0.46400000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":5.9349999999999996,"5min":3.1960000000000002,"15min":6.0010000000000003},"min":{"1min":0.22900000000000001,"5min":0.184,"15min":0.184},"max":{"1min":49.311,"5min":53.951999999999998,"15min":231.67400000000001},"last":1.351},{"interface":"front","average":{"1min":6.383,"5min":3.7789999999999999,"15min":6.2679999999999998},"min":{"1min":0.23400000000000001,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":48.991999999999997,"5min":54.213000000000001,"15min":232.21000000000001},"last":9.2110000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":18.759,"5min":6.4210000000000003,"15min":6.6829999999999998},"min":{"1min":0.25,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":272.839,"5min":272.839,"15min":272.839},"last":4.1829999999999998},{"interface":"front","average":{"1min":17.308,"5min":6.3499999999999996,"15min":6.5949999999999998},"min":{"1min":0.27700000000000002,"5min":0.245,"15min":0.224},"max":{"1min":272.76400000000001,"5min":272.76400000000001,"15min":272.76400000000001},"last":5.5540000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:42:31 2023","interfaces":[{"interface":"back","average":{"1min":3.194,"5min":2.004,"15min":1.5600000000000001},"min":{"1min":0.42999999999999999,"5min":0.34000000000000002,"15min":0.29799999999999999},"max":{"1min":10.183,"5min":10.413,"15min":12.042999999999999},"last":5.0640000000000001},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.2549999999999999,"15min":1.2669999999999999},"min":{"1min":0.36699999999999999,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":7.8760000000000003,"5min":13.622999999999999,"15min":18.108000000000001},"last":0.54700000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:42:31 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":2.23,"15min":2.1150000000000002},"min":{"1min":0.31,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":2.7890000000000001,"5min":80.302999999999997,"15min":151.94800000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":3.3090000000000002,"5min":2.8879999999999999,"15min":2.4279999999999999},"min":{"1min":0.39200000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.141999999999999,"5min":80.254000000000005,"15min":151.821},"last":5.0359999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:42:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":2.129,"15min":1.5740000000000001},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.32200000000000001},"max":{"1min":2.8180000000000001,"5min":87.837999999999994,"15min":87.837999999999994},"last":0.59799999999999998},{"interface":"front","average":{"1min":3.2200000000000002,"5min":2.8570000000000002,"15min":1.986},"min":{"1min":0.42999999999999999,"5min":0.312,"15min":0.312},"max":{"1min":10.233000000000001,"5min":85.518000000000001,"15min":85.518000000000001},"last":4.9969999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0169999999999999,"5min":1.6779999999999999,"15min":1.6319999999999999},"min":{"1min":0.40300000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":13.587999999999999,"5min":35.487000000000002,"15min":44.241},"last":0.56999999999999995},{"interface":"front","average":{"1min":3.069,"5min":2.3410000000000002,"15min":1.9350000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.210000000000001,"5min":35.146000000000001,"15min":43.920999999999999},"last":5.0899999999999999}]}]},{"osd":0,"up_from":91,"seq":390842024398,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2351100,"kb_used_data":970804,"kb_used_omap":5,"kb_used_meta":1380090,"kb_avail":91394052,"statfs":{"total":95995035648,"available":93587509248,"internally_reserved":0,"allocated":994103296,"data_stored":990820332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1413212794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,4],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:41:44 2023","interfaces":[{"interface":"back","average":{"1min":2.8180000000000001,"5min":1.8839999999999999,"15min":5.7699999999999996},"min":{"1min":0.23000000000000001,"5min":0.219,"15min":0.17799999999999999},"max":{"1min":7.1890000000000001,"5min":15.589,"15min":301.12700000000001},"last":0.19700000000000001},{"interface":"front","average":{"1min":3.0609999999999999,"5min":1.9770000000000001,"15min":5.851},"min":{"1min":0.23899999999999999,"5min":0.219,"15min":0.186},"max":{"1min":9.7159999999999993,"5min":15.446999999999999,"15min":297.72399999999999},"last":2.407}]},{"osd":2,"last update":"Sat Sep 23 23:42:13 2023","interfaces":[{"interface":"back","average":{"1min":5.476,"5min":3.8759999999999999,"15min":5.7949999999999999},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.17199999999999999},"max":{"1min":20.542000000000002,"5min":102.279,"15min":283.07400000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":4.3719999999999999,"5min":3.8420000000000001,"15min":5.9340000000000002},"min":{"1min":0.252,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":18.529,"5min":102.218,"15min":283.161},"last":0.41299999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:41:44 2023","interfaces":[{"interface":"back","average":{"1min":5.5359999999999996,"5min":7.4420000000000002,"15min":6.5110000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.217},"max":{"1min":36.186999999999998,"5min":194.26300000000001,"15min":220.732},"last":3.6760000000000002},{"interface":"front","average":{"1min":6.0629999999999997,"5min":7.7750000000000004,"15min":6.7880000000000003},"min":{"1min":0.27500000000000002,"5min":0.254,"15min":0.19800000000000001},"max":{"1min":33.616,"5min":194.56299999999999,"15min":218.78200000000001},"last":0.375}]},{"osd":4,"last update":"Sat Sep 23 23:42:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":1.7,"15min":1.784},"min":{"1min":0.36899999999999999,"5min":0.32600000000000001,"15min":0.28499999999999998},"max":{"1min":18.274999999999999,"5min":18.274999999999999,"15min":108.01900000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.554,"5min":1.5629999999999999,"15min":1.7729999999999999},"min":{"1min":0.39300000000000002,"5min":0.32600000000000001,"15min":0.27200000000000002},"max":{"1min":6.1310000000000002,"5min":8.718,"15min":107.898},"last":2.8370000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:42:24 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":3.117,"15min":1.9610000000000001},"min":{"1min":0.36799999999999999,"5min":0.36699999999999999,"15min":0.32600000000000001},"max":{"1min":6.0650000000000004,"5min":94.254000000000005,"15min":94.254000000000005},"last":2.798},{"interface":"front","average":{"1min":3.1200000000000001,"5min":3.3780000000000001,"15min":1.962},"min":{"1min":0.45200000000000001,"5min":0.32900000000000001,"15min":0.253},"max":{"1min":9.5869999999999997,"5min":94.837000000000003,"15min":94.837000000000003},"last":0.45800000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:42:24 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":2.3690000000000002,"15min":2.0659999999999998},"min":{"1min":0.33500000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.0979999999999999,"5min":69.043999999999997,"15min":108.67},"last":2.8650000000000002},{"interface":"front","average":{"1min":3.1629999999999998,"5min":2.597,"15min":2.121},"min":{"1min":0.52900000000000003,"5min":0.38200000000000001,"15min":0.27100000000000002},"max":{"1min":9.6310000000000002,"5min":69.028999999999996,"15min":108.804},"last":0.47499999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:42:13 2023","interfaces":[{"interface":"back","average":{"1min":2.5339999999999998,"5min":2.5129999999999999,"15min":2.2200000000000002},"min":{"1min":0.41299999999999998,"5min":0.35899999999999999,"15min":0.32900000000000001},"max":{"1min":18.469999999999999,"5min":66.620000000000005,"15min":111.71299999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":3.0710000000000002,"5min":2.4180000000000001,"15min":2.1840000000000002},"min":{"1min":0.496,"5min":0.35399999999999998,"15min":0.33300000000000002},"max":{"1min":9.6590000000000007,"5min":66.701999999999998,"15min":111.108},"last":0.42599999999999999}]}]},{"osd":2,"up_from":199,"seq":854698492220,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1721400,"kb_used_data":679472,"kb_used_omap":7,"kb_used_meta":1041912,"kb_avail":92023752,"statfs":{"total":95995035648,"available":94232322048,"internally_reserved":0,"allocated":695779328,"data_stored":692410329,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1066917969},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,6,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":5.1479999999999997,"5min":3.2290000000000001,"15min":2.7490000000000001},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":34.648000000000003,"5min":65.001000000000005,"15min":65.001000000000005},"last":3.0030000000000001},{"interface":"front","average":{"1min":3.1379999999999999,"5min":2.5259999999999998,"15min":2.4670000000000001},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19700000000000001},"max":{"1min":13.132,"5min":64.346000000000004,"15min":64.346000000000004},"last":1.5549999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":2.8220000000000001,"5min":2.8279999999999998,"15min":3.8719999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.192},"max":{"1min":14.369,"5min":41.43,"15min":106.765},"last":0.69599999999999995},{"interface":"front","average":{"1min":3.4129999999999998,"5min":2.6200000000000001,"15min":3.7290000000000001},"min":{"1min":0.36299999999999999,"5min":0.24399999999999999,"15min":0.222},"max":{"1min":11.821999999999999,"5min":53.268000000000001,"15min":106.732},"last":10.276999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":6.2300000000000004,"5min":3.1909999999999998,"15min":4.3090000000000002},"min":{"1min":0.315,"5min":0.245,"15min":0.24099999999999999},"max":{"1min":54.441000000000003,"5min":54.441000000000003,"15min":106.70399999999999},"last":4.0979999999999999},{"interface":"front","average":{"1min":6.1120000000000001,"5min":2.6720000000000002,"15min":3.9910000000000001},"min":{"1min":0.23899999999999999,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":53.734000000000002,"5min":53.734000000000002,"15min":106.66800000000001},"last":2.048}]},{"osd":4,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":3.23,"5min":1.5669999999999999,"15min":1.7929999999999999},"min":{"1min":0.41199999999999998,"5min":0.314,"15min":0.27700000000000002},"max":{"1min":16.001000000000001,"5min":16.001000000000001,"15min":62.081000000000003},"last":1.6000000000000001},{"interface":"front","average":{"1min":1.6870000000000001,"5min":1.77,"15min":1.9259999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.28899999999999998},"max":{"1min":6.6849999999999996,"5min":15.055999999999999,"15min":62.072000000000003},"last":0.76200000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":4.2229999999999999,"15min":3.0110000000000001},"min":{"1min":0.38600000000000001,"5min":0.32200000000000001,"15min":0.26200000000000001},"max":{"1min":13.023,"5min":101.104,"15min":101.104},"last":1.6259999999999999},{"interface":"front","average":{"1min":1.6830000000000001,"5min":4.681,"15min":3.1589999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.641,"5min":101.404,"15min":101.404},"last":0.78800000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2559999999999998,"5min":4.3710000000000004,"15min":3.3039999999999998},"min":{"1min":0.437,"5min":0.33400000000000002,"15min":0.31},"max":{"1min":15.898999999999999,"5min":97.558999999999997,"15min":107.018},"last":1.794},{"interface":"front","average":{"1min":1.71,"5min":4.7530000000000001,"15min":3.0590000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.7130000000000001,"5min":98.231999999999999,"15min":98.231999999999999},"last":0.80500000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":2.238,"15min":2.085},"min":{"1min":0.41899999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.9470000000000001,"5min":68.814999999999998,"15min":68.814999999999998},"last":2.1179999999999999},{"interface":"front","average":{"1min":2.3879999999999999,"5min":2.4580000000000002,"15min":2.5659999999999998},"min":{"1min":0.45900000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":6.923,"5min":69.918999999999997,"15min":106.809},"last":3.9369999999999998}]}]},{"osd":3,"up_from":148,"seq":635655160188,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1847540,"kb_used_data":736940,"kb_used_omap":2,"kb_used_meta":1110589,"kb_avail":91897612,"statfs":{"total":95995035648,"available":94103154688,"internally_reserved":0,"allocated":754626560,"data_stored":751365774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1137243189},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,2,1,0,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:41:58 2023","interfaces":[{"interface":"back","average":{"1min":8.3770000000000007,"5min":3.1070000000000002,"15min":2.3029999999999999},"min":{"1min":0.20300000000000001,"5min":0.19500000000000001,"15min":0.17699999999999999},"max":{"1min":69.281999999999996,"5min":69.281999999999996,"15min":69.281999999999996},"last":1.631},{"interface":"front","average":{"1min":4.3600000000000003,"5min":2.2690000000000001,"15min":2.4209999999999998},"min":{"1min":0.247,"5min":0.188,"15min":0.17199999999999999},"max":{"1min":20.071000000000002,"5min":20.071000000000002,"15min":46.204000000000001},"last":2.5569999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:41:58 2023","interfaces":[{"interface":"back","average":{"1min":3.6040000000000001,"5min":1.738,"15min":2.7000000000000002},"min":{"1min":0.27000000000000002,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":9.1649999999999991,"5min":11.911,"15min":85.242999999999995},"last":15.525},{"interface":"front","average":{"1min":4.774,"5min":2.2759999999999998,"15min":2.7919999999999998},"min":{"1min":0.30099999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":20.779,"5min":20.779,"15min":85.302000000000007},"last":1.881}]},{"osd":2,"last update":"Sat Sep 23 23:42:06 2023","interfaces":[{"interface":"back","average":{"1min":3.835,"5min":6.1420000000000003,"15min":5.0220000000000002},"min":{"1min":0.26100000000000001,"5min":0.23999999999999999,"15min":0.219},"max":{"1min":9.4100000000000001,"5min":139.96299999999999,"15min":139.96299999999999},"last":14.622999999999999},{"interface":"front","average":{"1min":7.3789999999999996,"5min":7.0220000000000002,"15min":5.2649999999999997},"min":{"1min":0.30099999999999999,"5min":0.26800000000000002,"15min":0.25700000000000001},"max":{"1min":69.405000000000001,"5min":141.93199999999999,"15min":141.93199999999999},"last":3.4580000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:41:58 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":1.3939999999999999,"15min":1.3300000000000001},"min":{"1min":0.34100000000000003,"5min":0.308,"15min":0.26800000000000002},"max":{"1min":7.9790000000000001,"5min":11.414,"15min":11.414},"last":1.736},{"interface":"front","average":{"1min":2.3079999999999998,"5min":1.839,"15min":1.786},"min":{"1min":0.37,"5min":0.32900000000000001,"15min":0.26800000000000002},"max":{"1min":6.7439999999999998,"5min":14.967000000000001,"15min":24.782},"last":2.4849999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:41:58 2023","interfaces":[{"interface":"back","average":{"1min":2.363,"5min":5.4210000000000003,"15min":4.6289999999999996},"min":{"1min":0.36499999999999999,"5min":0.33500000000000002,"15min":0.29499999999999998},"max":{"1min":6.6630000000000003,"5min":196.358,"15min":214.43700000000001},"last":15.56},{"interface":"front","average":{"1min":2.415,"5min":5.0709999999999997,"15min":4.5220000000000002},"min":{"1min":0.38500000000000001,"5min":0.32700000000000001,"15min":0.30599999999999999},"max":{"1min":9.2210000000000001,"5min":196.048,"15min":214.46600000000001},"last":14.478999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:41:58 2023","interfaces":[{"interface":"back","average":{"1min":2.3330000000000002,"5min":3.1560000000000001,"15min":2.5270000000000001},"min":{"1min":0.443,"5min":0.35799999999999998,"15min":0.318},"max":{"1min":6.718,"5min":125.94,"15min":125.94},"last":2.444},{"interface":"front","average":{"1min":2.4580000000000002,"5min":2.8660000000000001,"15min":2.4209999999999998},"min":{"1min":0.40200000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.2469999999999999,"5min":125.955,"15min":125.955},"last":14.513}]},{"osd":7,"last update":"Sat Sep 23 23:42:06 2023","interfaces":[{"interface":"back","average":{"1min":5.4770000000000003,"5min":2.27,"15min":3.5859999999999999},"min":{"1min":0.38900000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":69.197999999999993,"5min":69.197999999999993,"15min":192.69},"last":3.4900000000000002},{"interface":"front","average":{"1min":3.254,"5min":2.1070000000000002,"15min":3.8410000000000002},"min":{"1min":0.48899999999999999,"5min":0.39500000000000002,"15min":0.36699999999999999},"max":{"1min":12.568,"5min":14.923999999999999,"15min":192.65600000000001},"last":2.528}]}]},{"osd":4,"up_from":0,"seq":128849019507,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757870,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1470544,"kb_used_data":621576,"kb_used_omap":0,"kb_used_meta":848896,"kb_avail":92274608,"statfs":{"total":95995035648,"available":94489198592,"internally_reserved":0,"allocated":636493824,"data_stored":634549937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":869269504},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:42:33 2023","interfaces":[{"interface":"back","average":{"1min":15.054,"5min":4.782,"15min":2.4399999999999999},"min":{"1min":0.251,"5min":0.223,"15min":0.20899999999999999},"max":{"1min":174.58600000000001,"5min":174.58600000000001,"15min":174.58600000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":14.362,"5min":5.0739999999999998,"15min":2.5750000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.22700000000000001},"max":{"1min":174.69499999999999,"5min":174.69499999999999,"15min":174.69499999999999},"last":0.36699999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:42:33 2023","interfaces":[{"interface":"back","average":{"1min":3.2789999999999999,"5min":3.0510000000000002,"15min":2.0489999999999999},"min":{"1min":0.28000000000000003,"5min":0.25600000000000001,"15min":0.23100000000000001},"max":{"1min":38.468000000000004,"5min":95.709000000000003,"15min":95.709000000000003},"last":0.39200000000000002},{"interface":"front","average":{"1min":3.4159999999999999,"5min":3.2789999999999999,"15min":2.198},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.249},"max":{"1min":38.841999999999999,"5min":94.966999999999999,"15min":94.966999999999999},"last":2.5049999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:42:01 2023","interfaces":[{"interface":"back","average":{"1min":2.8780000000000001,"5min":3.6030000000000002,"15min":4.4500000000000002},"min":{"1min":0.40100000000000002,"5min":0.28799999999999998,"15min":0.27200000000000002},"max":{"1min":9.5879999999999992,"5min":132.81999999999999,"15min":208.738},"last":0.434},{"interface":"front","average":{"1min":2.472,"5min":3.169,"15min":3.8690000000000002},"min":{"1min":0.313,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.3360000000000003,"5min":133.09200000000001,"15min":208.631},"last":2.9319999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:42:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":3.5550000000000002,"15min":4.1849999999999996},"min":{"1min":0.34599999999999997,"5min":0.33100000000000002,"15min":0.27800000000000002},"max":{"1min":14.288,"5min":143.66300000000001,"15min":143.66300000000001},"last":3.7799999999999998},{"interface":"front","average":{"1min":2.552,"5min":3.6139999999999999,"15min":4.3410000000000002},"min":{"1min":0.44500000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":14.307,"5min":143.62200000000001,"15min":143.62200000000001},"last":3.8140000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.90200000000000002,"15min":0.92600000000000005},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.252},"max":{"1min":6.9390000000000001,"5min":6.9390000000000001,"15min":19.962},"last":1.0580000000000001},{"interface":"front","average":{"1min":0.82599999999999996,"5min":0.78400000000000003,"15min":0.89400000000000002},"min":{"1min":0.31900000000000001,"5min":0.28899999999999998,"15min":0.22900000000000001},"max":{"1min":6.7590000000000003,"5min":6.7590000000000003,"15min":20.829999999999998},"last":0.58299999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.95699999999999996,"15min":1.3380000000000001},"min":{"1min":0.32900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.617,"5min":14.252000000000001,"15min":148.36600000000001},"last":0.624},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.94499999999999995,"15min":1.5229999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.8040000000000003,"5min":6.8040000000000003,"15min":148.42500000000001},"last":0.59999999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.91400000000000003,"15min":1.7190000000000001},"min":{"1min":0.316,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.56299999999999994,"5min":6.4199999999999999,"15min":179.28800000000001},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.0049999999999999,"15min":1.796},"min":{"1min":0.34899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.9279999999999999,"5min":6.4649999999999999,"15min":179.428},"last":0.88600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":975306752,"data_stored":975306752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":823656448,"data_stored":823656448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":684658688,"data_stored":684654616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":735313920,"data_stored":735313920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":610275328,"data_stored":610271256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":326959104,"data_stored":326959104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":558501888,"data_stored":558497816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:42:38.985 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:42:39.109 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:42:39.110 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:42:39.110 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-09-23T23:42:39.110 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:42:39.116 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:39.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:39.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:39.224 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:39.365 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:39.416 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:39.476 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:39.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:39.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:39.582 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:42:39.618 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:39.703 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:42:39.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:39.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:42:39.754 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:39 smithi110 ceph-mon[212939]: pgmap v1917: 119 pgs: 119 active+clean; 1.2 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 717 op/s 2023-09-23T23:42:39.755 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:39 smithi110 ceph-mon[221102]: pgmap v1917: 119 pgs: 119 active+clean; 1.2 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 717 op/s 2023-09-23T23:42:39.789 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:39 smithi138 ceph-mon[193606]: pgmap v1917: 119 pgs: 119 active+clean; 1.2 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 717 op/s 2023-09-23T23:42:39.805 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:42:39.808 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:39.884 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:39.934 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:39.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:39.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:40.040 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:42:40.118 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:40.131 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:42:40.168 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:42:40.216 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:42:40.226 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:42:40.301 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:42:40.337 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:40.337 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:42:40.346 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:42:40.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:40.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:42:40.381 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:42:40.427 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:42:40.458 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:40.466 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:42:40.489 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:40.489 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:42:40.489 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:42:40.489 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:42:40.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:42:40.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:42:40.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:42:40.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:42:40.490 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:42:40.623 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:40 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1668963909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:40.623 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:40 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1668963909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:40.655 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:40.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:40.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:40.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:40 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1668963909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:40.759 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:40.825 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:41.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:41 smithi138 ceph-mon[193606]: pgmap v1918: 119 pgs: 119 active+clean; 1.2 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 655 op/s 2023-09-23T23:42:41.824 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:42:41.824 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:42:41.850 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:41.852 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:41 smithi110 ceph-mon[212939]: pgmap v1918: 119 pgs: 119 active+clean; 1.2 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 655 op/s 2023-09-23T23:42:41.852 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:41 smithi110 ceph-mon[221102]: pgmap v1918: 119 pgs: 119 active+clean; 1.2 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 655 op/s 2023-09-23T23:42:41.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:41.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:41.953 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:42:42.042 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:42.442 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:42:42.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:42.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:42:42.555 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:42.577 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:42:42.590 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:42.656 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:42.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:42.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:42.768 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:42.804 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:42.963 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:42.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:42.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:42.995 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:42:43.072 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:43.203 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:43.314 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:43.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:43.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:43.426 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:43.504 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:43.594 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:42:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:43 smithi138 ceph-mon[193606]: pgmap v1919: 119 pgs: 119 active+clean; 1.1 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 677 op/s 2023-09-23T23:42:43.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:42:43.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:43 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2082033838' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:42:43.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:43 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:42:43.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:43 smithi110 ceph-mon[212939]: pgmap v1919: 119 pgs: 119 active+clean; 1.1 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 677 op/s 2023-09-23T23:42:43.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:42:43.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:43 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2082033838' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:42:43.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:43 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:42:43.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:43 smithi110 ceph-mon[221102]: pgmap v1919: 119 pgs: 119 active+clean; 1.1 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 677 op/s 2023-09-23T23:42:43.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:42:43.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:43 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2082033838' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:42:43.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:43 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:42:44.258 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:44.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:44.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:44.363 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:42:44.390 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:44.431 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:42:44.571 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:44 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:42:44.571 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:44 smithi110 ceph-mon[212939]: failed to encode map e675 with expected crc 2023-09-23T23:42:44.571 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:44 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:42:44.571 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:44 smithi110 ceph-mon[212939]: osdmap e675: 8 total, 8 up, 6 in 2023-09-23T23:42:44.572 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:44 smithi110 ceph-mon[212939]: failed to encode map e675 with expected crc 2023-09-23T23:42:44.572 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:44 smithi110 ceph-mon[212939]: failed to encode map e675 with expected crc 2023-09-23T23:42:44.572 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:44 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:44.620 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:42:44.620 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:42:44.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:44 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:42:44.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:44 smithi138 ceph-mon[193606]: failed to encode map e675 with expected crc 2023-09-23T23:42:44.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:44 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:42:44.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:44 smithi138 ceph-mon[193606]: osdmap e675: 8 total, 8 up, 6 in 2023-09-23T23:42:44.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:44 smithi138 ceph-mon[193606]: failed to encode map e675 with expected crc 2023-09-23T23:42:44.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:44 smithi138 ceph-mon[193606]: failed to encode map e675 with expected crc 2023-09-23T23:42:44.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:44 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:44.849 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:42:44.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:44.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:42:44.855 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:44 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:42:44.855 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:44 smithi110 ceph-mon[221102]: failed to encode map e675 with expected crc 2023-09-23T23:42:44.855 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:44 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:42:44.855 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:44 smithi110 ceph-mon[221102]: osdmap e675: 8 total, 8 up, 6 in 2023-09-23T23:42:44.855 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:44 smithi110 ceph-mon[221102]: failed to encode map e675 with expected crc 2023-09-23T23:42:44.855 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:44 smithi110 ceph-mon[221102]: failed to encode map e675 with expected crc 2023-09-23T23:42:44.856 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:44 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:44.954 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:42:44.984 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:45.035 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:42:45.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:45.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:42:45.139 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:42:45.167 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:45.303 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:45.315 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:42:45.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:45.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:42:45.382 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:42:45.419 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:42:45.454 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:45.504 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:42:45.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:45.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:42:45.610 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:45.733 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:45.784 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:42:45.792 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:45 smithi110 ceph-mon[212939]: pgmap v1921: 119 pgs: 119 active+clean; 1.0 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T23:42:45.792 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:45 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]}]: dispatch 2023-09-23T23:42:45.792 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:45 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.16", "id": [1, 2]}]: dispatch 2023-09-23T23:42:45.792 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:45 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]}]: dispatch 2023-09-23T23:42:45.792 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:45 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.16", "id": [1, 2]}]: dispatch 2023-09-23T23:42:45.794 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:45 smithi110 ceph-mon[221102]: pgmap v1921: 119 pgs: 119 active+clean; 1.0 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T23:42:45.794 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:45 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]}]: dispatch 2023-09-23T23:42:45.794 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:45 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.16", "id": [1, 2]}]: dispatch 2023-09-23T23:42:45.794 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:45 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]}]: dispatch 2023-09-23T23:42:45.794 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:45 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.16", "id": [1, 2]}]: dispatch 2023-09-23T23:42:45.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:45 smithi138 ceph-mon[193606]: pgmap v1921: 119 pgs: 119 active+clean; 1.0 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T23:42:45.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:45 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]}]: dispatch 2023-09-23T23:42:45.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:45 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.16", "id": [1, 2]}]: dispatch 2023-09-23T23:42:45.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:45 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]}]: dispatch 2023-09-23T23:42:45.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:45 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.16", "id": [1, 2]}]: dispatch 2023-09-23T23:42:46.174 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:46.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:46.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:46.283 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:46.315 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:42:46.336 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:42:46.338 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:46.806 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:46 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]}]': finished 2023-09-23T23:42:46.807 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:46 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.16", "id": [1, 2]}]': finished 2023-09-23T23:42:46.807 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:46 smithi110 ceph-mon[221102]: osdmap e676: 8 total, 8 up, 6 in 2023-09-23T23:42:46.807 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:46 smithi110 ceph-mon[221102]: failed to encode map e676 with expected crc 2023-09-23T23:42:46.807 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:46 smithi110 ceph-mon[221102]: failed to encode map e676 with expected crc 2023-09-23T23:42:46.807 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:46 smithi110 ceph-mon[221102]: failed to encode map e676 with expected crc 2023-09-23T23:42:46.807 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:46 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]}]': finished 2023-09-23T23:42:46.807 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:46 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.16", "id": [1, 2]}]': finished 2023-09-23T23:42:46.808 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:46 smithi110 ceph-mon[212939]: osdmap e676: 8 total, 8 up, 6 in 2023-09-23T23:42:46.808 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:46 smithi110 ceph-mon[212939]: failed to encode map e676 with expected crc 2023-09-23T23:42:46.808 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:46 smithi110 ceph-mon[212939]: failed to encode map e676 with expected crc 2023-09-23T23:42:46.808 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:46 smithi110 ceph-mon[212939]: failed to encode map e676 with expected crc 2023-09-23T23:42:46.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:46 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]}]': finished 2023-09-23T23:42:46.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:46 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.16", "id": [1, 2]}]': finished 2023-09-23T23:42:46.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:46 smithi138 ceph-mon[193606]: osdmap e676: 8 total, 8 up, 6 in 2023-09-23T23:42:46.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:46 smithi138 ceph-mon[193606]: failed to encode map e676 with expected crc 2023-09-23T23:42:46.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:46 smithi138 ceph-mon[193606]: failed to encode map e676 with expected crc 2023-09-23T23:42:46.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:46 smithi138 ceph-mon[193606]: failed to encode map e676 with expected crc 2023-09-23T23:42:47.070 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:42:47.136 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:42:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:42:46] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.43.0" 2023-09-23T23:42:47.370 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:42:47.371 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:42:47.538 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:47.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:47.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:47.663 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:47.687 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[212939]: failed to encode map e676 with expected crc 2023-09-23T23:42:47.688 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[212939]: failed to encode map e676 with expected crc 2023-09-23T23:42:47.688 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[212939]: pgmap v1923: 119 pgs: 1 peering, 118 active+clean; 947 MiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 599 op/s 2023-09-23T23:42:47.688 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[212939]: from='client.57298 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:42:47.688 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:42:47.688 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[212939]: failed to encode map e677 with expected crc 2023-09-23T23:42:47.688 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[212939]: osdmap e677: 8 total, 8 up, 6 in 2023-09-23T23:42:47.688 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[212939]: failed to encode map e677 with expected crc 2023-09-23T23:42:47.688 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:47.688 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:47.688 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[221102]: failed to encode map e676 with expected crc 2023-09-23T23:42:47.689 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[221102]: failed to encode map e676 with expected crc 2023-09-23T23:42:47.689 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[221102]: pgmap v1923: 119 pgs: 1 peering, 118 active+clean; 947 MiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 599 op/s 2023-09-23T23:42:47.689 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[221102]: from='client.57298 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:42:47.689 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:42:47.689 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[221102]: failed to encode map e677 with expected crc 2023-09-23T23:42:47.689 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[221102]: osdmap e677: 8 total, 8 up, 6 in 2023-09-23T23:42:47.689 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[221102]: failed to encode map e677 with expected crc 2023-09-23T23:42:47.689 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:47.689 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:47 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:47.690 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:47 smithi138 ceph-mon[193606]: failed to encode map e676 with expected crc 2023-09-23T23:42:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:47 smithi138 ceph-mon[193606]: failed to encode map e676 with expected crc 2023-09-23T23:42:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:47 smithi138 ceph-mon[193606]: pgmap v1923: 119 pgs: 1 peering, 118 active+clean; 947 MiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 599 op/s 2023-09-23T23:42:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:47 smithi138 ceph-mon[193606]: from='client.57298 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:42:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:47 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:42:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:47 smithi138 ceph-mon[193606]: failed to encode map e677 with expected crc 2023-09-23T23:42:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:47 smithi138 ceph-mon[193606]: osdmap e677: 8 total, 8 up, 6 in 2023-09-23T23:42:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:47 smithi138 ceph-mon[193606]: failed to encode map e677 with expected crc 2023-09-23T23:42:47.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:47 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:47.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:47 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:48.091 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:42:48.155 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:48.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:48.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:48.261 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:48.290 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:48.304 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1923,"stamp":"2023-09-23T23:42:46.113595+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":992810936,"num_objects":15165,"num_object_clones":0,"num_object_copies":45495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15165,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152674,"num_write_kb":5368094,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15009,"num_bytes_recovered":810955076,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93012,"ondisk_log_size":93012,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":8975456,"kb_used_data":3100016,"kb_used_omap":29,"kb_used_meta":5874914,"kb_avail":553495456,"statfs":{"total":575970213888,"available":566779346944,"internally_reserved":0,"allocated":3174416384,"data_stored":3157266800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":6015912659},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,5,0,1,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-314376192,"num_objects":-4797,"num_object_clones":0,"num_object_copies":-14391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4797,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.004480"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":871,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.018729+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:42:33.018729+0000","last_peered":"2023-09-23T23:42:33.018729+0000","last_clean":"2023-09-23T23:42:33.018729+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:42:33.018729+0000","last_undegraded":"2023-09-23T23:42:33.018729+0000","last_fullsized":"2023-09-23T23:42:33.018729+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":433,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.910195+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:42:43.910195+0000","last_peered":"2023-09-23T23:42:43.910195+0000","last_clean":"2023-09-23T23:42:43.910195+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:42:43.910195+0000","last_undegraded":"2023-09-23T23:42:43.910195+0000","last_fullsized":"2023-09-23T23:42:43.910195+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":667,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066768+0000","last_change":"2023-09-23T23:39:55.102353+0000","last_active":"2023-09-23T23:42:32.066768+0000","last_peered":"2023-09-23T23:42:32.066768+0000","last_clean":"2023-09-23T23:42:32.066768+0000","last_became_active":"2023-09-23T23:39:55.102017+0000","last_became_peered":"2023-09-23T23:39:55.102017+0000","last_unstale":"2023-09-23T23:42:32.066768+0000","last_undegraded":"2023-09-23T23:42:32.066768+0000","last_fullsized":"2023-09-23T23:42:32.066768+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:48.870443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":824,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.017126+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:42:33.017126+0000","last_peered":"2023-09-23T23:42:33.017126+0000","last_clean":"2023-09-23T23:42:33.017126+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:42:33.017126+0000","last_undegraded":"2023-09-23T23:42:33.017126+0000","last_fullsized":"2023-09-23T23:42:33.017126+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":404,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066884+0000","last_change":"2023-09-23T23:39:55.104571+0000","last_active":"2023-09-23T23:42:32.066884+0000","last_peered":"2023-09-23T23:42:32.066884+0000","last_clean":"2023-09-23T23:42:32.066884+0000","last_became_active":"2023-09-23T23:39:55.104152+0000","last_became_peered":"2023-09-23T23:39:55.104152+0000","last_unstale":"2023-09-23T23:42:32.066884+0000","last_undegraded":"2023-09-23T23:42:32.066884+0000","last_fullsized":"2023-09-23T23:42:32.066884+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:31:52.551048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":638,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066459+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:42:32.066459+0000","last_peered":"2023-09-23T23:42:32.066459+0000","last_clean":"2023-09-23T23:42:32.066459+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:42:32.066459+0000","last_undegraded":"2023-09-23T23:42:32.066459+0000","last_fullsized":"2023-09-23T23:42:32.066459+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":835,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.948452+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:42:31.948452+0000","last_peered":"2023-09-23T23:42:31.948452+0000","last_clean":"2023-09-23T23:42:31.948452+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:42:31.948452+0000","last_undegraded":"2023-09-23T23:42:31.948452+0000","last_fullsized":"2023-09-23T23:42:31.948452+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":722,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.067163+0000","last_change":"2023-09-23T23:42:28.951264+0000","last_active":"2023-09-23T23:42:32.067163+0000","last_peered":"2023-09-23T23:42:32.067163+0000","last_clean":"2023-09-23T23:42:32.067163+0000","last_became_active":"2023-09-23T23:42:28.950966+0000","last_became_peered":"2023-09-23T23:42:28.950966+0000","last_unstale":"2023-09-23T23:42:32.067163+0000","last_undegraded":"2023-09-23T23:42:32.067163+0000","last_fullsized":"2023-09-23T23:42:32.067163+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:30.571439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":354,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.017888+0000","last_change":"2023-09-23T23:39:55.092989+0000","last_active":"2023-09-23T23:42:33.017888+0000","last_peered":"2023-09-23T23:42:33.017888+0000","last_clean":"2023-09-23T23:42:33.017888+0000","last_became_active":"2023-09-23T23:39:55.092680+0000","last_became_peered":"2023-09-23T23:39:55.092680+0000","last_unstale":"2023-09-23T23:42:33.017888+0000","last_undegraded":"2023-09-23T23:42:33.017888+0000","last_fullsized":"2023-09-23T23:42:33.017888+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:44.685469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":994,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.949149+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:42:31.949149+0000","last_peered":"2023-09-23T23:42:31.949149+0000","last_clean":"2023-09-23T23:42:31.949149+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:42:31.949149+0000","last_undegraded":"2023-09-23T23:42:31.949149+0000","last_fullsized":"2023-09-23T23:42:31.949149+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":327,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.017830+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:42:33.017830+0000","last_peered":"2023-09-23T23:42:33.017830+0000","last_clean":"2023-09-23T23:42:33.017830+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:42:33.017830+0000","last_undegraded":"2023-09-23T23:42:33.017830+0000","last_fullsized":"2023-09-23T23:42:33.017830+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":686,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.067075+0000","last_change":"2023-09-23T23:39:55.100428+0000","last_active":"2023-09-23T23:42:32.067075+0000","last_peered":"2023-09-23T23:42:32.067075+0000","last_clean":"2023-09-23T23:42:32.067075+0000","last_became_active":"2023-09-23T23:39:55.100094+0000","last_became_peered":"2023-09-23T23:39:55.100094+0000","last_unstale":"2023-09-23T23:42:32.067075+0000","last_undegraded":"2023-09-23T23:42:32.067075+0000","last_fullsized":"2023-09-23T23:42:32.067075+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:43.132618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":972,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.601144+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:42:43.601144+0000","last_peered":"2023-09-23T23:42:43.601144+0000","last_clean":"2023-09-23T23:42:43.601144+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:42:43.601144+0000","last_undegraded":"2023-09-23T23:42:43.601144+0000","last_fullsized":"2023-09-23T23:42:43.601144+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":306,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.017633+0000","last_change":"2023-09-23T23:27:20.551633+0000","last_active":"2023-09-23T23:42:33.017633+0000","last_peered":"2023-09-23T23:42:33.017633+0000","last_clean":"2023-09-23T23:42:33.017633+0000","last_became_active":"2023-09-23T23:27:20.551366+0000","last_became_peered":"2023-09-23T23:27:20.551366+0000","last_unstale":"2023-09-23T23:42:33.017633+0000","last_undegraded":"2023-09-23T23:42:33.017633+0000","last_fullsized":"2023-09-23T23:42:33.017633+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:21.265500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":72,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.600480+0000","last_change":"2023-09-23T23:39:55.091152+0000","last_active":"2023-09-23T23:42:43.600480+0000","last_peered":"2023-09-23T23:42:43.600480+0000","last_clean":"2023-09-23T23:42:43.600480+0000","last_became_active":"2023-09-23T23:39:55.090833+0000","last_became_peered":"2023-09-23T23:39:55.090833+0000","last_unstale":"2023-09-23T23:42:43.600480+0000","last_undegraded":"2023-09-23T23:42:43.600480+0000","last_fullsized":"2023-09-23T23:42:43.600480+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:33.156366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1149,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.017756+0000","last_change":"2023-09-23T23:39:58.110527+0000","last_active":"2023-09-23T23:42:33.017756+0000","last_peered":"2023-09-23T23:42:33.017756+0000","last_clean":"2023-09-23T23:42:33.017756+0000","last_became_active":"2023-09-23T23:39:58.110194+0000","last_became_peered":"2023-09-23T23:39:58.110194+0000","last_unstale":"2023-09-23T23:42:33.017756+0000","last_undegraded":"2023-09-23T23:42:33.017756+0000","last_fullsized":"2023-09-23T23:42:33.017756+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:21:19.332102+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":349,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066565+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:42:32.066565+0000","last_peered":"2023-09-23T23:42:32.066565+0000","last_clean":"2023-09-23T23:42:32.066565+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:42:32.066565+0000","last_undegraded":"2023-09-23T23:42:32.066565+0000","last_fullsized":"2023-09-23T23:42:32.066565+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":362,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.602139+0000","last_change":"2023-09-23T23:42:28.956958+0000","last_active":"2023-09-23T23:42:43.602139+0000","last_peered":"2023-09-23T23:42:43.602139+0000","last_clean":"2023-09-23T23:42:43.602139+0000","last_became_active":"2023-09-23T23:42:28.956645+0000","last_became_peered":"2023-09-23T23:42:28.956645+0000","last_unstale":"2023-09-23T23:42:43.602139+0000","last_undegraded":"2023-09-23T23:42:43.602139+0000","last_fullsized":"2023-09-23T23:42:43.602139+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:16:28.472945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":539,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066655+0000","last_change":"2023-09-23T23:39:19.011304+0000","last_active":"2023-09-23T23:42:32.066655+0000","last_peered":"2023-09-23T23:42:32.066655+0000","last_clean":"2023-09-23T23:42:32.066655+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:42:32.066655+0000","last_undegraded":"2023-09-23T23:42:32.066655+0000","last_fullsized":"2023-09-23T23:42:32.066655+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:39:19.011223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:39:19.011223+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:48:49.078060+0000","scrub_duration":0.001220575,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":196,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.601072+0000","last_change":"2023-09-23T23:39:55.129350+0000","last_active":"2023-09-23T23:42:43.601072+0000","last_peered":"2023-09-23T23:42:43.601072+0000","last_clean":"2023-09-23T23:42:43.601072+0000","last_became_active":"2023-09-23T23:39:55.092677+0000","last_became_peered":"2023-09-23T23:39:55.092677+0000","last_unstale":"2023-09-23T23:42:43.601072+0000","last_undegraded":"2023-09-23T23:42:43.601072+0000","last_fullsized":"2023-09-23T23:42:43.601072+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:38.925371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":549,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.017247+0000","last_change":"2023-09-23T23:39:55.097595+0000","last_active":"2023-09-23T23:42:33.017247+0000","last_peered":"2023-09-23T23:42:33.017247+0000","last_clean":"2023-09-23T23:42:33.017247+0000","last_became_active":"2023-09-23T23:39:55.097301+0000","last_became_peered":"2023-09-23T23:39:55.097301+0000","last_unstale":"2023-09-23T23:42:33.017247+0000","last_undegraded":"2023-09-23T23:42:33.017247+0000","last_fullsized":"2023-09-23T23:42:33.017247+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:26.508461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":830,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.016840+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:42:33.016840+0000","last_peered":"2023-09-23T23:42:33.016840+0000","last_clean":"2023-09-23T23:42:33.016840+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:42:33.016840+0000","last_undegraded":"2023-09-23T23:42:33.016840+0000","last_fullsized":"2023-09-23T23:42:33.016840+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":240,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.602051+0000","last_change":"2023-09-23T23:39:55.137107+0000","last_active":"2023-09-23T23:42:43.602051+0000","last_peered":"2023-09-23T23:42:43.602051+0000","last_clean":"2023-09-23T23:42:43.602051+0000","last_became_active":"2023-09-23T23:39:55.094309+0000","last_became_peered":"2023-09-23T23:39:55.094309+0000","last_unstale":"2023-09-23T23:42:43.602051+0000","last_undegraded":"2023-09-23T23:42:43.602051+0000","last_fullsized":"2023-09-23T23:42:43.602051+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:49.766841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":694,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.067037+0000","last_change":"2023-09-23T23:42:28.954052+0000","last_active":"2023-09-23T23:42:32.067037+0000","last_peered":"2023-09-23T23:42:32.067037+0000","last_clean":"2023-09-23T23:42:32.067037+0000","last_became_active":"2023-09-23T23:42:28.953681+0000","last_became_peered":"2023-09-23T23:42:28.953681+0000","last_unstale":"2023-09-23T23:42:32.067037+0000","last_undegraded":"2023-09-23T23:42:32.067037+0000","last_fullsized":"2023-09-23T23:42:32.067037+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:07.367387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":792,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.018671+0000","last_change":"2023-09-23T23:39:58.117169+0000","last_active":"2023-09-23T23:42:33.018671+0000","last_peered":"2023-09-23T23:42:33.018671+0000","last_clean":"2023-09-23T23:42:33.018671+0000","last_became_active":"2023-09-23T23:39:58.116815+0000","last_became_peered":"2023-09-23T23:39:58.116815+0000","last_unstale":"2023-09-23T23:42:33.018671+0000","last_undegraded":"2023-09-23T23:42:33.018671+0000","last_fullsized":"2023-09-23T23:42:33.018671+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:51.175189+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":271,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.067408+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:42:32.067408+0000","last_peered":"2023-09-23T23:42:32.067408+0000","last_clean":"2023-09-23T23:42:32.067408+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:42:32.067408+0000","last_undegraded":"2023-09-23T23:42:32.067408+0000","last_fullsized":"2023-09-23T23:42:32.067408+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":398,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066698+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:42:32.066698+0000","last_peered":"2023-09-23T23:42:32.066698+0000","last_clean":"2023-09-23T23:42:32.066698+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:42:32.066698+0000","last_undegraded":"2023-09-23T23:42:32.066698+0000","last_fullsized":"2023-09-23T23:42:32.066698+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":639,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066609+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:42:32.066609+0000","last_peered":"2023-09-23T23:42:32.066609+0000","last_clean":"2023-09-23T23:42:32.066609+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:42:32.066609+0000","last_undegraded":"2023-09-23T23:42:32.066609+0000","last_fullsized":"2023-09-23T23:42:32.066609+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":485,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.078508+0000","last_change":"2023-09-23T23:39:55.100855+0000","last_active":"2023-09-23T23:42:32.078508+0000","last_peered":"2023-09-23T23:42:32.078508+0000","last_clean":"2023-09-23T23:42:32.078508+0000","last_became_active":"2023-09-23T23:39:55.100577+0000","last_became_peered":"2023-09-23T23:39:55.100577+0000","last_unstale":"2023-09-23T23:42:32.078508+0000","last_undegraded":"2023-09-23T23:42:32.078508+0000","last_fullsized":"2023-09-23T23:42:32.078508+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:02.548093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":632,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066882+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:42:32.066882+0000","last_peered":"2023-09-23T23:42:32.066882+0000","last_clean":"2023-09-23T23:42:32.066882+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:42:32.066882+0000","last_undegraded":"2023-09-23T23:42:32.066882+0000","last_fullsized":"2023-09-23T23:42:32.066882+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":1098,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.017104+0000","last_change":"2023-09-23T23:42:32.073277+0000","last_active":"2023-09-23T23:42:33.017104+0000","last_peered":"2023-09-23T23:42:33.017104+0000","last_clean":"2023-09-23T23:42:33.017104+0000","last_became_active":"2023-09-23T23:42:32.072952+0000","last_became_peered":"2023-09-23T23:42:32.072952+0000","last_unstale":"2023-09-23T23:42:33.017104+0000","last_undegraded":"2023-09-23T23:42:33.017104+0000","last_fullsized":"2023-09-23T23:42:33.017104+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:19.983734+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":41,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.080190+0000","last_change":"2023-09-23T23:41:31.621623+0000","last_active":"2023-09-23T23:42:32.080190+0000","last_peered":"2023-09-23T23:42:32.080190+0000","last_clean":"2023-09-23T23:42:32.080190+0000","last_became_active":"2023-09-23T23:41:31.596928+0000","last_became_peered":"2023-09-23T23:41:31.596928+0000","last_unstale":"2023-09-23T23:42:32.080190+0000","last_undegraded":"2023-09-23T23:42:32.080190+0000","last_fullsized":"2023-09-23T23:42:32.080190+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:01.308674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":317,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.071248+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:42:32.071248+0000","last_peered":"2023-09-23T23:42:32.071248+0000","last_clean":"2023-09-23T23:42:32.071248+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:42:32.071248+0000","last_undegraded":"2023-09-23T23:42:32.071248+0000","last_fullsized":"2023-09-23T23:42:32.071248+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"674'9450","reported_seq":15208,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:42.793820+0000","last_change":"2023-09-23T23:41:11.967390+0000","last_active":"2023-09-23T23:42:42.793820+0000","last_peered":"2023-09-23T23:42:42.793820+0000","last_clean":"2023-09-23T23:42:42.793820+0000","last_became_active":"2023-09-23T23:41:11.955563+0000","last_became_peered":"2023-09-23T23:41:11.955563+0000","last_unstale":"2023-09-23T23:42:42.793820+0000","last_undegraded":"2023-09-23T23:42:42.793820+0000","last_fullsized":"2023-09-23T23:42:42.793820+0000","mapping_epoch":645,"log_start":"635'4686","ondisk_log_start":"635'4686","created":617,"last_epoch_clean":646,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4764,"log_dups_size":0,"ondisk_log_size":4764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:03.686224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64356352,"num_objects":982,"num_object_clones":0,"num_object_copies":2946,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":982,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9450,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2837,"num_bytes_recovered":185925632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":1012,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.601549+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:42:43.601549+0000","last_peered":"2023-09-23T23:42:43.601549+0000","last_clean":"2023-09-23T23:42:43.601549+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:42:43.601549+0000","last_undegraded":"2023-09-23T23:42:43.601549+0000","last_fullsized":"2023-09-23T23:42:43.601549+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":307,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.949123+0000","last_change":"2023-09-23T23:42:28.954407+0000","last_active":"2023-09-23T23:42:31.949123+0000","last_peered":"2023-09-23T23:42:31.949123+0000","last_clean":"2023-09-23T23:42:31.949123+0000","last_became_active":"2023-09-23T23:42:28.950556+0000","last_became_peered":"2023-09-23T23:42:28.950556+0000","last_unstale":"2023-09-23T23:42:31.949123+0000","last_undegraded":"2023-09-23T23:42:31.949123+0000","last_fullsized":"2023-09-23T23:42:31.949123+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:58.736585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":316,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.018671+0000","last_change":"2023-09-23T23:39:55.095319+0000","last_active":"2023-09-23T23:42:33.018671+0000","last_peered":"2023-09-23T23:42:33.018671+0000","last_clean":"2023-09-23T23:42:33.018671+0000","last_became_active":"2023-09-23T23:39:55.094677+0000","last_became_peered":"2023-09-23T23:39:55.094677+0000","last_unstale":"2023-09-23T23:42:33.018671+0000","last_undegraded":"2023-09-23T23:42:33.018671+0000","last_fullsized":"2023-09-23T23:42:33.018671+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:21.525905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"674'9372","reported_seq":9439,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:42.801644+0000","last_change":"2023-09-23T23:39:13.746676+0000","last_active":"2023-09-23T23:42:42.801644+0000","last_peered":"2023-09-23T23:42:42.801644+0000","last_clean":"2023-09-23T23:42:42.801644+0000","last_became_active":"2023-09-23T23:39:13.746389+0000","last_became_peered":"2023-09-23T23:39:13.746389+0000","last_unstale":"2023-09-23T23:42:42.801644+0000","last_undegraded":"2023-09-23T23:42:42.801644+0000","last_fullsized":"2023-09-23T23:42:42.801644+0000","mapping_epoch":617,"log_start":"635'4597","ondisk_log_start":"635'4597","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4775,"log_dups_size":0,"ondisk_log_size":4775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:44.833933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63832064,"num_objects":974,"num_object_clones":0,"num_object_copies":2922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9372,"num_write_kb":331072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1110,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.805874+0000","last_change":"2023-09-23T23:39:58.119994+0000","last_active":"2023-09-23T23:42:32.805874+0000","last_peered":"2023-09-23T23:42:32.805874+0000","last_clean":"2023-09-23T23:42:32.805874+0000","last_became_active":"2023-09-23T23:39:58.119654+0000","last_became_peered":"2023-09-23T23:39:58.119654+0000","last_unstale":"2023-09-23T23:42:32.805874+0000","last_undegraded":"2023-09-23T23:42:32.805874+0000","last_fullsized":"2023-09-23T23:42:32.805874+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:02:41.967366+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":333,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.948982+0000","last_change":"2023-09-23T23:41:31.603533+0000","last_active":"2023-09-23T23:42:31.948982+0000","last_peered":"2023-09-23T23:42:31.948982+0000","last_clean":"2023-09-23T23:42:31.948982+0000","last_became_active":"2023-09-23T23:41:31.593485+0000","last_became_peered":"2023-09-23T23:41:31.593485+0000","last_unstale":"2023-09-23T23:42:31.948982+0000","last_undegraded":"2023-09-23T23:42:31.948982+0000","last_fullsized":"2023-09-23T23:42:31.948982+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:10:56.208677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":536,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066419+0000","last_change":"2023-09-23T23:39:55.101024+0000","last_active":"2023-09-23T23:42:32.066419+0000","last_peered":"2023-09-23T23:42:32.066419+0000","last_clean":"2023-09-23T23:42:32.066419+0000","last_became_active":"2023-09-23T23:39:55.100745+0000","last_became_peered":"2023-09-23T23:39:55.100745+0000","last_unstale":"2023-09-23T23:42:32.066419+0000","last_undegraded":"2023-09-23T23:42:32.066419+0000","last_fullsized":"2023-09-23T23:42:32.066419+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:28.387953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.0","version":"676'9715","reported_seq":9784,"reported_epoch":676,"state":"active+clean","last_fresh":"2023-09-23T23:42:45.588915+0000","last_change":"2023-09-23T23:39:13.741363+0000","last_active":"2023-09-23T23:42:45.588915+0000","last_peered":"2023-09-23T23:42:45.588915+0000","last_clean":"2023-09-23T23:42:45.588915+0000","last_became_active":"2023-09-23T23:39:13.741098+0000","last_became_peered":"2023-09-23T23:39:13.741098+0000","last_unstale":"2023-09-23T23:42:45.588915+0000","last_undegraded":"2023-09-23T23:42:45.588915+0000","last_fullsized":"2023-09-23T23:42:45.588915+0000","mapping_epoch":617,"log_start":"631'3323","ondisk_log_start":"631'3323","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6392,"log_dups_size":0,"ondisk_log_size":6392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:31.131594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60620800,"num_objects":925,"num_object_clones":0,"num_object_copies":2775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9715,"num_write_kb":340480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":842,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.017680+0000","last_change":"2023-09-23T23:39:58.122462+0000","last_active":"2023-09-23T23:42:33.017680+0000","last_peered":"2023-09-23T23:42:33.017680+0000","last_clean":"2023-09-23T23:42:33.017680+0000","last_became_active":"2023-09-23T23:39:58.122163+0000","last_became_peered":"2023-09-23T23:39:58.122163+0000","last_unstale":"2023-09-23T23:42:33.017680+0000","last_undegraded":"2023-09-23T23:42:33.017680+0000","last_fullsized":"2023-09-23T23:42:33.017680+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:41.511681+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":427,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.017254+0000","last_change":"2023-09-23T23:39:55.141520+0000","last_active":"2023-09-23T23:42:33.017254+0000","last_peered":"2023-09-23T23:42:33.017254+0000","last_clean":"2023-09-23T23:42:33.017254+0000","last_became_active":"2023-09-23T23:39:55.099176+0000","last_became_peered":"2023-09-23T23:39:55.099176+0000","last_unstale":"2023-09-23T23:42:33.017254+0000","last_undegraded":"2023-09-23T23:42:33.017254+0000","last_fullsized":"2023-09-23T23:42:33.017254+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:12.860173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":57,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.016871+0000","last_change":"2023-09-23T23:39:55.094219+0000","last_active":"2023-09-23T23:42:33.016871+0000","last_peered":"2023-09-23T23:42:33.016871+0000","last_clean":"2023-09-23T23:42:33.016871+0000","last_became_active":"2023-09-23T23:39:55.093849+0000","last_became_peered":"2023-09-23T23:39:55.093849+0000","last_unstale":"2023-09-23T23:42:33.016871+0000","last_undegraded":"2023-09-23T23:42:33.016871+0000","last_fullsized":"2023-09-23T23:42:33.016871+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:54.576029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.1","version":"674'9586","reported_seq":9652,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:42.794011+0000","last_change":"2023-09-23T23:39:13.744449+0000","last_active":"2023-09-23T23:42:42.794011+0000","last_peered":"2023-09-23T23:42:42.794011+0000","last_clean":"2023-09-23T23:42:42.794011+0000","last_became_active":"2023-09-23T23:39:13.744163+0000","last_became_peered":"2023-09-23T23:39:13.744163+0000","last_unstale":"2023-09-23T23:42:42.794011+0000","last_undegraded":"2023-09-23T23:42:42.794011+0000","last_fullsized":"2023-09-23T23:42:42.794011+0000","mapping_epoch":617,"log_start":"635'4746","ondisk_log_start":"635'4746","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4840,"log_dups_size":0,"ondisk_log_size":4840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:50.120382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65011712,"num_objects":992,"num_object_clones":0,"num_object_copies":2976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9586,"num_write_kb":338496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":583,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066737+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:42:32.066737+0000","last_peered":"2023-09-23T23:42:32.066737+0000","last_clean":"2023-09-23T23:42:32.066737+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:42:32.066737+0000","last_undegraded":"2023-09-23T23:42:32.066737+0000","last_fullsized":"2023-09-23T23:42:32.066737+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":2,"reported_epoch":676,"state":"peering","last_fresh":"2023-09-23T23:39:55.084696+0000","last_change":"2023-09-23T23:39:54.078123+0000","last_active":"2023-09-23T23:39:54.077970+0000","last_peered":"2023-09-23T23:39:50.101853+0000","last_clean":"2023-09-23T23:39:50.101853+0000","last_became_active":"2023-09-23T23:29:54.998930+0000","last_became_peered":"2023-09-23T23:29:54.998930+0000","last_unstale":"2023-09-23T23:39:55.084696+0000","last_undegraded":"2023-09-23T23:39:55.084696+0000","last_fullsized":"2023-09-23T23:39:55.084696+0000","mapping_epoch":676,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:11.272589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":674,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.078163+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:42:32.078163+0000","last_peered":"2023-09-23T23:42:32.078163+0000","last_clean":"2023-09-23T23:42:32.078163+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:42:32.078163+0000","last_undegraded":"2023-09-23T23:42:32.078163+0000","last_fullsized":"2023-09-23T23:42:32.078163+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"675'9763","reported_seq":9832,"reported_epoch":676,"state":"active+clean","last_fresh":"2023-09-23T23:42:45.584147+0000","last_change":"2023-09-23T23:39:13.745430+0000","last_active":"2023-09-23T23:42:45.584147+0000","last_peered":"2023-09-23T23:42:45.584147+0000","last_clean":"2023-09-23T23:42:45.584147+0000","last_became_active":"2023-09-23T23:39:13.744751+0000","last_became_peered":"2023-09-23T23:39:13.744751+0000","last_unstale":"2023-09-23T23:42:45.584147+0000","last_undegraded":"2023-09-23T23:42:45.584147+0000","last_fullsized":"2023-09-23T23:42:45.584147+0000","mapping_epoch":617,"log_start":"631'3354","ondisk_log_start":"631'3354","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6409,"log_dups_size":0,"ondisk_log_size":6409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:23.195329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61538304,"num_objects":939,"num_object_clones":0,"num_object_copies":2817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9763,"num_write_kb":342464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":816,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066799+0000","last_change":"2023-09-23T23:39:19.985630+0000","last_active":"2023-09-23T23:42:32.066799+0000","last_peered":"2023-09-23T23:42:32.066799+0000","last_clean":"2023-09-23T23:42:32.066799+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:42:32.066799+0000","last_undegraded":"2023-09-23T23:42:32.066799+0000","last_fullsized":"2023-09-23T23:42:32.066799+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:39:19.985512+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:39:19.985512+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:52.750567+0000","scrub_duration":0.001852713,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":713,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066531+0000","last_change":"2023-09-23T23:41:31.618278+0000","last_active":"2023-09-23T23:42:32.066531+0000","last_peered":"2023-09-23T23:42:32.066531+0000","last_clean":"2023-09-23T23:42:32.066531+0000","last_became_active":"2023-09-23T23:41:31.602519+0000","last_became_peered":"2023-09-23T23:41:31.602519+0000","last_unstale":"2023-09-23T23:42:32.066531+0000","last_undegraded":"2023-09-23T23:42:32.066531+0000","last_fullsized":"2023-09-23T23:42:32.066531+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:20.395010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":606,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066845+0000","last_change":"2023-09-23T23:39:55.099345+0000","last_active":"2023-09-23T23:42:32.066845+0000","last_peered":"2023-09-23T23:42:32.066845+0000","last_clean":"2023-09-23T23:42:32.066845+0000","last_became_active":"2023-09-23T23:39:55.098875+0000","last_became_peered":"2023-09-23T23:39:55.098875+0000","last_unstale":"2023-09-23T23:42:32.066845+0000","last_undegraded":"2023-09-23T23:42:32.066845+0000","last_fullsized":"2023-09-23T23:42:32.066845+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:09.169232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.7","version":"675'9694","reported_seq":9760,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:44.798833+0000","last_change":"2023-09-23T23:39:13.743009+0000","last_active":"2023-09-23T23:42:44.798833+0000","last_peered":"2023-09-23T23:42:44.798833+0000","last_clean":"2023-09-23T23:42:44.798833+0000","last_became_active":"2023-09-23T23:39:13.742719+0000","last_became_peered":"2023-09-23T23:39:13.742719+0000","last_unstale":"2023-09-23T23:42:44.798833+0000","last_undegraded":"2023-09-23T23:42:44.798833+0000","last_fullsized":"2023-09-23T23:42:44.798833+0000","mapping_epoch":617,"log_start":"633'4476","ondisk_log_start":"633'4476","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":5218,"log_dups_size":0,"ondisk_log_size":5218,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:50.189642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62390272,"num_objects":952,"num_object_clones":0,"num_object_copies":2856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9694,"num_write_kb":340672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":962,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.910855+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:42:43.910855+0000","last_peered":"2023-09-23T23:42:43.910855+0000","last_clean":"2023-09-23T23:42:43.910855+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:42:43.910855+0000","last_undegraded":"2023-09-23T23:42:43.910855+0000","last_fullsized":"2023-09-23T23:42:43.910855+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":149,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.067193+0000","last_change":"2023-09-23T23:42:28.953222+0000","last_active":"2023-09-23T23:42:32.067193+0000","last_peered":"2023-09-23T23:42:32.067193+0000","last_clean":"2023-09-23T23:42:32.067193+0000","last_became_active":"2023-09-23T23:42:28.952902+0000","last_became_peered":"2023-09-23T23:42:28.952902+0000","last_unstale":"2023-09-23T23:42:32.067193+0000","last_undegraded":"2023-09-23T23:42:32.067193+0000","last_fullsized":"2023-09-23T23:42:32.067193+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:21.715883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":571,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.601712+0000","last_change":"2023-09-23T23:41:31.604354+0000","last_active":"2023-09-23T23:42:43.601712+0000","last_peered":"2023-09-23T23:42:43.601712+0000","last_clean":"2023-09-23T23:42:43.601712+0000","last_became_active":"2023-09-23T23:41:31.604035+0000","last_became_peered":"2023-09-23T23:41:31.604035+0000","last_unstale":"2023-09-23T23:42:43.601712+0000","last_undegraded":"2023-09-23T23:42:43.601712+0000","last_fullsized":"2023-09-23T23:42:43.601712+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:55.798632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"674'9453","reported_seq":15283,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:42.810052+0000","last_change":"2023-09-23T23:40:33.082798+0000","last_active":"2023-09-23T23:42:42.810052+0000","last_peered":"2023-09-23T23:42:42.810052+0000","last_clean":"2023-09-23T23:42:42.810052+0000","last_became_active":"2023-09-23T23:40:33.076970+0000","last_became_peered":"2023-09-23T23:40:33.076970+0000","last_unstale":"2023-09-23T23:42:42.810052+0000","last_undegraded":"2023-09-23T23:42:42.810052+0000","last_fullsized":"2023-09-23T23:42:42.810052+0000","mapping_epoch":634,"log_start":"635'4669","ondisk_log_start":"635'4669","created":617,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4784,"log_dups_size":0,"ondisk_log_size":4784,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:08.992451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65339392,"num_objects":997,"num_object_clones":0,"num_object_copies":2991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9453,"num_write_kb":334400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2873,"num_bytes_recovered":188284928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":683,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.017076+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:42:33.017076+0000","last_peered":"2023-09-23T23:42:33.017076+0000","last_clean":"2023-09-23T23:42:33.017076+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:42:33.017076+0000","last_undegraded":"2023-09-23T23:42:33.017076+0000","last_fullsized":"2023-09-23T23:42:33.017076+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":542,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.600705+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:42:43.600705+0000","last_peered":"2023-09-23T23:42:43.600705+0000","last_clean":"2023-09-23T23:42:43.600705+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:42:43.600705+0000","last_undegraded":"2023-09-23T23:42:43.600705+0000","last_fullsized":"2023-09-23T23:42:43.600705+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":576,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066307+0000","last_change":"2023-09-23T23:42:28.951851+0000","last_active":"2023-09-23T23:42:32.066307+0000","last_peered":"2023-09-23T23:42:32.066307+0000","last_clean":"2023-09-23T23:42:32.066307+0000","last_became_active":"2023-09-23T23:42:28.951542+0000","last_became_peered":"2023-09-23T23:42:28.951542+0000","last_unstale":"2023-09-23T23:42:32.066307+0000","last_undegraded":"2023-09-23T23:42:32.066307+0000","last_fullsized":"2023-09-23T23:42:32.066307+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:13.264990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.5","version":"674'9552","reported_seq":9619,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:42.770892+0000","last_change":"2023-09-23T23:39:13.742854+0000","last_active":"2023-09-23T23:42:42.770892+0000","last_peered":"2023-09-23T23:42:42.770892+0000","last_clean":"2023-09-23T23:42:42.770892+0000","last_became_active":"2023-09-23T23:39:13.742579+0000","last_became_peered":"2023-09-23T23:39:13.742579+0000","last_unstale":"2023-09-23T23:42:42.770892+0000","last_undegraded":"2023-09-23T23:42:42.770892+0000","last_fullsized":"2023-09-23T23:42:42.770892+0000","mapping_epoch":617,"log_start":"635'4726","ondisk_log_start":"635'4726","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4826,"log_dups_size":0,"ondisk_log_size":4826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:51.872071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64094208,"num_objects":978,"num_object_clones":0,"num_object_copies":2934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":978,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9552,"num_write_kb":336960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":1037,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.602065+0000","last_change":"2023-09-23T23:39:58.119762+0000","last_active":"2023-09-23T23:42:43.602065+0000","last_peered":"2023-09-23T23:42:43.602065+0000","last_clean":"2023-09-23T23:42:43.602065+0000","last_became_active":"2023-09-23T23:39:58.119422+0000","last_became_peered":"2023-09-23T23:39:58.119422+0000","last_unstale":"2023-09-23T23:42:43.602065+0000","last_undegraded":"2023-09-23T23:42:43.602065+0000","last_fullsized":"2023-09-23T23:42:43.602065+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:03.564930+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":318,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.067248+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:42:32.067248+0000","last_peered":"2023-09-23T23:42:32.067248+0000","last_clean":"2023-09-23T23:42:32.067248+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:42:32.067248+0000","last_undegraded":"2023-09-23T23:42:32.067248+0000","last_fullsized":"2023-09-23T23:42:32.067248+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":606,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.071445+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:42:32.071445+0000","last_peered":"2023-09-23T23:42:32.071445+0000","last_clean":"2023-09-23T23:42:32.071445+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:42:32.071445+0000","last_undegraded":"2023-09-23T23:42:32.071445+0000","last_fullsized":"2023-09-23T23:42:32.071445+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.8","version":"675'9792","reported_seq":11199,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:45.563335+0000","last_change":"2023-09-23T23:42:28.980646+0000","last_active":"2023-09-23T23:42:45.563335+0000","last_peered":"2023-09-23T23:42:45.563335+0000","last_clean":"2023-09-23T23:42:45.563335+0000","last_became_active":"2023-09-23T23:42:28.958020+0000","last_became_peered":"2023-09-23T23:42:28.958020+0000","last_unstale":"2023-09-23T23:42:45.563335+0000","last_undegraded":"2023-09-23T23:42:45.563335+0000","last_fullsized":"2023-09-23T23:42:45.563335+0000","mapping_epoch":669,"log_start":"631'3398","ondisk_log_start":"631'3398","created":617,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6394,"log_dups_size":0,"ondisk_log_size":6394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:21.232496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56229888,"num_objects":858,"num_object_clones":0,"num_object_copies":2574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9792,"num_write_kb":340800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":656,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":879,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.071177+0000","last_change":"2023-09-23T23:39:58.120057+0000","last_active":"2023-09-23T23:42:32.071177+0000","last_peered":"2023-09-23T23:42:32.071177+0000","last_clean":"2023-09-23T23:42:32.071177+0000","last_became_active":"2023-09-23T23:39:58.119647+0000","last_became_peered":"2023-09-23T23:39:58.119647+0000","last_unstale":"2023-09-23T23:42:32.071177+0000","last_undegraded":"2023-09-23T23:42:32.071177+0000","last_fullsized":"2023-09-23T23:42:32.071177+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:50.965369+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":386,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.072200+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:42:32.072200+0000","last_peered":"2023-09-23T23:42:32.072200+0000","last_clean":"2023-09-23T23:42:32.072200+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:42:32.072200+0000","last_undegraded":"2023-09-23T23:42:32.072200+0000","last_fullsized":"2023-09-23T23:42:32.072200+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":690,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.017406+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:42:33.017406+0000","last_peered":"2023-09-23T23:42:33.017406+0000","last_clean":"2023-09-23T23:42:33.017406+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:42:33.017406+0000","last_undegraded":"2023-09-23T23:42:33.017406+0000","last_fullsized":"2023-09-23T23:42:33.017406+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":291,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.016948+0000","last_change":"2023-09-23T23:42:28.950994+0000","last_active":"2023-09-23T23:42:33.016948+0000","last_peered":"2023-09-23T23:42:33.016948+0000","last_clean":"2023-09-23T23:42:33.016948+0000","last_became_active":"2023-09-23T23:42:28.950464+0000","last_became_peered":"2023-09-23T23:42:28.950464+0000","last_unstale":"2023-09-23T23:42:33.016948+0000","last_undegraded":"2023-09-23T23:42:33.016948+0000","last_fullsized":"2023-09-23T23:42:33.016948+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:57.975234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.e","version":"675'9609","reported_seq":15589,"reported_epoch":676,"state":"active+clean","last_fresh":"2023-09-23T23:42:45.584311+0000","last_change":"2023-09-23T23:42:21.783482+0000","last_active":"2023-09-23T23:42:45.584311+0000","last_peered":"2023-09-23T23:42:45.584311+0000","last_clean":"2023-09-23T23:42:45.584311+0000","last_became_active":"2023-09-23T23:42:21.754054+0000","last_became_peered":"2023-09-23T23:42:21.754054+0000","last_unstale":"2023-09-23T23:42:45.584311+0000","last_undegraded":"2023-09-23T23:42:45.584311+0000","last_fullsized":"2023-09-23T23:42:45.584311+0000","mapping_epoch":667,"log_start":"631'3204","ondisk_log_start":"631'3204","created":617,"last_epoch_clean":668,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6405,"log_dups_size":0,"ondisk_log_size":6405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:33.850501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58130432,"num_objects":887,"num_object_clones":0,"num_object_copies":2661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9609,"num_write_kb":335872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2945,"num_bytes_recovered":134676480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":760,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.077725+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:42:32.077725+0000","last_peered":"2023-09-23T23:42:32.077725+0000","last_clean":"2023-09-23T23:42:32.077725+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:42:32.077725+0000","last_undegraded":"2023-09-23T23:42:32.077725+0000","last_fullsized":"2023-09-23T23:42:32.077725+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":328,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.017024+0000","last_change":"2023-09-23T23:39:55.089555+0000","last_active":"2023-09-23T23:42:33.017024+0000","last_peered":"2023-09-23T23:42:33.017024+0000","last_clean":"2023-09-23T23:42:33.017024+0000","last_became_active":"2023-09-23T23:39:55.089184+0000","last_became_peered":"2023-09-23T23:39:55.089184+0000","last_unstale":"2023-09-23T23:42:33.017024+0000","last_undegraded":"2023-09-23T23:42:33.017024+0000","last_fullsized":"2023-09-23T23:42:33.017024+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:19.775188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":539,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.078339+0000","last_change":"2023-09-23T23:39:55.089351+0000","last_active":"2023-09-23T23:42:32.078339+0000","last_peered":"2023-09-23T23:42:32.078339+0000","last_clean":"2023-09-23T23:42:32.078339+0000","last_became_active":"2023-09-23T23:39:55.089016+0000","last_became_peered":"2023-09-23T23:39:55.089016+0000","last_unstale":"2023-09-23T23:42:32.078339+0000","last_undegraded":"2023-09-23T23:42:32.078339+0000","last_fullsized":"2023-09-23T23:42:32.078339+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:05:23.856265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":312,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.601209+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:42:43.601209+0000","last_peered":"2023-09-23T23:42:43.601209+0000","last_clean":"2023-09-23T23:42:43.601209+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:42:43.601209+0000","last_undegraded":"2023-09-23T23:42:43.601209+0000","last_fullsized":"2023-09-23T23:42:43.601209+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"676'9412","reported_seq":15080,"reported_epoch":676,"state":"active+clean","last_fresh":"2023-09-23T23:42:45.602321+0000","last_change":"2023-09-23T23:42:01.335232+0000","last_active":"2023-09-23T23:42:45.602321+0000","last_peered":"2023-09-23T23:42:45.602321+0000","last_clean":"2023-09-23T23:42:45.602321+0000","last_became_active":"2023-09-23T23:42:01.316912+0000","last_became_peered":"2023-09-23T23:42:01.316912+0000","last_unstale":"2023-09-23T23:42:45.602321+0000","last_undegraded":"2023-09-23T23:42:45.602321+0000","last_fullsized":"2023-09-23T23:42:45.602321+0000","mapping_epoch":662,"log_start":"631'3161","ondisk_log_start":"631'3161","created":617,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6251,"log_dups_size":0,"ondisk_log_size":6251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:34:25.843454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61276184,"num_objects":936,"num_object_clones":0,"num_object_copies":2808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":936,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9412,"num_write_kb":331073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2789,"num_bytes_recovered":115015680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":980,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.601986+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:42:43.601986+0000","last_peered":"2023-09-23T23:42:43.601986+0000","last_clean":"2023-09-23T23:42:43.601986+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:42:43.601986+0000","last_undegraded":"2023-09-23T23:42:43.601986+0000","last_fullsized":"2023-09-23T23:42:43.601986+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":417,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.600626+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:42:43.600626+0000","last_peered":"2023-09-23T23:42:43.600626+0000","last_clean":"2023-09-23T23:42:43.600626+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:42:43.600626+0000","last_undegraded":"2023-09-23T23:42:43.600626+0000","last_fullsized":"2023-09-23T23:42:43.600626+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":336,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.910938+0000","last_change":"2023-09-23T23:39:55.093320+0000","last_active":"2023-09-23T23:42:43.910938+0000","last_peered":"2023-09-23T23:42:43.910938+0000","last_clean":"2023-09-23T23:42:43.910938+0000","last_became_active":"2023-09-23T23:39:55.092998+0000","last_became_peered":"2023-09-23T23:39:55.092998+0000","last_unstale":"2023-09-23T23:42:43.910938+0000","last_undegraded":"2023-09-23T23:42:43.910938+0000","last_fullsized":"2023-09-23T23:42:43.910938+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:38.763321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":15,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.026533+0000","last_change":"2023-09-23T23:42:28.952433+0000","last_active":"2023-09-23T23:42:33.026533+0000","last_peered":"2023-09-23T23:42:33.026533+0000","last_clean":"2023-09-23T23:42:33.026533+0000","last_became_active":"2023-09-23T23:42:28.952139+0000","last_became_peered":"2023-09-23T23:42:28.952139+0000","last_unstale":"2023-09-23T23:42:33.026533+0000","last_undegraded":"2023-09-23T23:42:33.026533+0000","last_fullsized":"2023-09-23T23:42:33.026533+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:49.952980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"674'9602","reported_seq":9669,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:42.748943+0000","last_change":"2023-09-23T23:39:13.747297+0000","last_active":"2023-09-23T23:42:42.748943+0000","last_peered":"2023-09-23T23:42:42.748943+0000","last_clean":"2023-09-23T23:42:42.748943+0000","last_became_active":"2023-09-23T23:39:13.747068+0000","last_became_peered":"2023-09-23T23:39:13.747068+0000","last_unstale":"2023-09-23T23:42:42.748943+0000","last_undegraded":"2023-09-23T23:42:42.748943+0000","last_fullsized":"2023-09-23T23:42:42.748943+0000","mapping_epoch":617,"log_start":"636'4781","ondisk_log_start":"636'4781","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4821,"log_dups_size":0,"ondisk_log_size":4821,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:34.450587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64487424,"num_objects":984,"num_object_clones":0,"num_object_copies":2952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9602,"num_write_kb":338752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":694,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.602119+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:42:43.602119+0000","last_peered":"2023-09-23T23:42:43.602119+0000","last_clean":"2023-09-23T23:42:43.602119+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:42:43.602119+0000","last_undegraded":"2023-09-23T23:42:43.602119+0000","last_fullsized":"2023-09-23T23:42:43.602119+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":993,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.948186+0000","last_change":"2023-09-23T23:39:58.122811+0000","last_active":"2023-09-23T23:42:31.948186+0000","last_peered":"2023-09-23T23:42:31.948186+0000","last_clean":"2023-09-23T23:42:31.948186+0000","last_became_active":"2023-09-23T23:39:58.122461+0000","last_became_peered":"2023-09-23T23:39:58.122461+0000","last_unstale":"2023-09-23T23:42:31.948186+0000","last_undegraded":"2023-09-23T23:42:31.948186+0000","last_fullsized":"2023-09-23T23:42:31.948186+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:20:59.163746+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":648,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.067267+0000","last_change":"2023-09-23T23:39:55.101203+0000","last_active":"2023-09-23T23:42:32.067267+0000","last_peered":"2023-09-23T23:42:32.067267+0000","last_clean":"2023-09-23T23:42:32.067267+0000","last_became_active":"2023-09-23T23:39:55.099575+0000","last_became_peered":"2023-09-23T23:39:55.099575+0000","last_unstale":"2023-09-23T23:42:32.067267+0000","last_undegraded":"2023-09-23T23:42:32.067267+0000","last_fullsized":"2023-09-23T23:42:32.067267+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:40.288184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":58,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066321+0000","last_change":"2023-09-23T23:39:55.103713+0000","last_active":"2023-09-23T23:42:32.066321+0000","last_peered":"2023-09-23T23:42:32.066321+0000","last_clean":"2023-09-23T23:42:32.066321+0000","last_became_active":"2023-09-23T23:39:55.103402+0000","last_became_peered":"2023-09-23T23:39:55.103402+0000","last_unstale":"2023-09-23T23:42:32.066321+0000","last_undegraded":"2023-09-23T23:42:32.066321+0000","last_fullsized":"2023-09-23T23:42:32.066321+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:44.949292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.c","version":"676'9539","reported_seq":15292,"reported_epoch":676,"state":"active+clean","last_fresh":"2023-09-23T23:42:45.595881+0000","last_change":"2023-09-23T23:41:10.964608+0000","last_active":"2023-09-23T23:42:45.595881+0000","last_peered":"2023-09-23T23:42:45.595881+0000","last_clean":"2023-09-23T23:42:45.595881+0000","last_became_active":"2023-09-23T23:41:10.952515+0000","last_became_peered":"2023-09-23T23:41:10.952515+0000","last_unstale":"2023-09-23T23:42:45.595881+0000","last_undegraded":"2023-09-23T23:42:45.595881+0000","last_fullsized":"2023-09-23T23:42:45.595881+0000","mapping_epoch":644,"log_start":"631'3288","ondisk_log_start":"631'3288","created":617,"last_epoch_clean":645,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6251,"log_dups_size":0,"ondisk_log_size":6251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:30.860608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60096512,"num_objects":917,"num_object_clones":0,"num_object_copies":2751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9539,"num_write_kb":334592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2834,"num_bytes_recovered":185729024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":837,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.072123+0000","last_change":"2023-09-23T23:41:34.668439+0000","last_active":"2023-09-23T23:42:32.072123+0000","last_peered":"2023-09-23T23:42:32.072123+0000","last_clean":"2023-09-23T23:42:32.072123+0000","last_became_active":"2023-09-23T23:41:34.668104+0000","last_became_peered":"2023-09-23T23:41:34.668104+0000","last_unstale":"2023-09-23T23:42:32.072123+0000","last_undegraded":"2023-09-23T23:42:32.072123+0000","last_fullsized":"2023-09-23T23:42:32.072123+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:42:24.264118+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":601,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.947923+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:42:31.947923+0000","last_peered":"2023-09-23T23:42:31.947923+0000","last_clean":"2023-09-23T23:42:31.947923+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:42:31.947923+0000","last_undegraded":"2023-09-23T23:42:31.947923+0000","last_fullsized":"2023-09-23T23:42:31.947923+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":518,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066494+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:42:32.066494+0000","last_peered":"2023-09-23T23:42:32.066494+0000","last_clean":"2023-09-23T23:42:32.066494+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:42:32.066494+0000","last_undegraded":"2023-09-23T23:42:32.066494+0000","last_fullsized":"2023-09-23T23:42:32.066494+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":31,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.078603+0000","last_change":"2023-09-23T23:41:31.605087+0000","last_active":"2023-09-23T23:42:32.078603+0000","last_peered":"2023-09-23T23:42:32.078603+0000","last_clean":"2023-09-23T23:42:32.078603+0000","last_became_active":"2023-09-23T23:41:31.604777+0000","last_became_peered":"2023-09-23T23:41:31.604777+0000","last_unstale":"2023-09-23T23:42:32.078603+0000","last_undegraded":"2023-09-23T23:42:32.078603+0000","last_fullsized":"2023-09-23T23:42:32.078603+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:57.416801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"674'9310","reported_seq":9377,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:42.793813+0000","last_change":"2023-09-23T23:39:13.742801+0000","last_active":"2023-09-23T23:42:42.793813+0000","last_peered":"2023-09-23T23:42:42.793813+0000","last_clean":"2023-09-23T23:42:42.793813+0000","last_became_active":"2023-09-23T23:39:13.742482+0000","last_became_peered":"2023-09-23T23:39:13.742482+0000","last_unstale":"2023-09-23T23:42:42.793813+0000","last_undegraded":"2023-09-23T23:42:42.793813+0000","last_fullsized":"2023-09-23T23:42:42.793813+0000","mapping_epoch":617,"log_start":"635'4542","ondisk_log_start":"635'4542","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4768,"log_dups_size":0,"ondisk_log_size":4768,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:15.737461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60817408,"num_objects":928,"num_object_clones":0,"num_object_copies":2784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9310,"num_write_kb":327616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":601,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066563+0000","last_change":"2023-09-23T23:39:58.112420+0000","last_active":"2023-09-23T23:42:32.066563+0000","last_peered":"2023-09-23T23:42:32.066563+0000","last_clean":"2023-09-23T23:42:32.066563+0000","last_became_active":"2023-09-23T23:39:58.111949+0000","last_became_peered":"2023-09-23T23:39:58.111949+0000","last_unstale":"2023-09-23T23:42:32.066563+0000","last_undegraded":"2023-09-23T23:42:32.066563+0000","last_fullsized":"2023-09-23T23:42:32.066563+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:20.354620+0000","scrub_duration":0.0047245100000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":117,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.078201+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:42:32.078201+0000","last_peered":"2023-09-23T23:42:32.078201+0000","last_clean":"2023-09-23T23:42:32.078201+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:42:32.078201+0000","last_undegraded":"2023-09-23T23:42:32.078201+0000","last_fullsized":"2023-09-23T23:42:32.078201+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":219,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.601275+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:42:43.601275+0000","last_peered":"2023-09-23T23:42:43.601275+0000","last_clean":"2023-09-23T23:42:43.601275+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:42:43.601275+0000","last_undegraded":"2023-09-23T23:42:43.601275+0000","last_fullsized":"2023-09-23T23:42:43.601275+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":408,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.017213+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:42:33.017213+0000","last_peered":"2023-09-23T23:42:33.017213+0000","last_clean":"2023-09-23T23:42:33.017213+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:42:33.017213+0000","last_undegraded":"2023-09-23T23:42:33.017213+0000","last_fullsized":"2023-09-23T23:42:33.017213+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.a","version":"674'9289","reported_seq":9356,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:42.785048+0000","last_change":"2023-09-23T23:39:13.746851+0000","last_active":"2023-09-23T23:42:42.785048+0000","last_peered":"2023-09-23T23:42:42.785048+0000","last_clean":"2023-09-23T23:42:42.785048+0000","last_became_active":"2023-09-23T23:39:13.746412+0000","last_became_peered":"2023-09-23T23:39:13.746412+0000","last_unstale":"2023-09-23T23:42:42.785048+0000","last_undegraded":"2023-09-23T23:42:42.785048+0000","last_fullsized":"2023-09-23T23:42:42.785048+0000","mapping_epoch":617,"log_start":"635'4472","ondisk_log_start":"635'4472","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4817,"log_dups_size":0,"ondisk_log_size":4817,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:39.662209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65470464,"num_objects":999,"num_object_clones":0,"num_object_copies":2997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9289,"num_write_kb":329216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":877,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.601782+0000","last_change":"2023-09-23T23:39:58.122659+0000","last_active":"2023-09-23T23:42:43.601782+0000","last_peered":"2023-09-23T23:42:43.601782+0000","last_clean":"2023-09-23T23:42:43.601782+0000","last_became_active":"2023-09-23T23:39:58.122334+0000","last_became_peered":"2023-09-23T23:39:58.122334+0000","last_unstale":"2023-09-23T23:42:43.601782+0000","last_undegraded":"2023-09-23T23:42:43.601782+0000","last_fullsized":"2023-09-23T23:42:43.601782+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:29.154690+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":674,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.071377+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:42:32.071377+0000","last_peered":"2023-09-23T23:42:32.071377+0000","last_clean":"2023-09-23T23:42:32.071377+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:42:32.071377+0000","last_undegraded":"2023-09-23T23:42:32.071377+0000","last_fullsized":"2023-09-23T23:42:32.071377+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":217,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066381+0000","last_change":"2023-09-23T23:39:55.100517+0000","last_active":"2023-09-23T23:42:32.066381+0000","last_peered":"2023-09-23T23:42:32.066381+0000","last_clean":"2023-09-23T23:42:32.066381+0000","last_became_active":"2023-09-23T23:39:55.098755+0000","last_became_peered":"2023-09-23T23:39:55.098755+0000","last_unstale":"2023-09-23T23:42:32.066381+0000","last_undegraded":"2023-09-23T23:42:32.066381+0000","last_fullsized":"2023-09-23T23:42:32.066381+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:50.086720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":248,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.910700+0000","last_change":"2023-09-23T23:41:31.604932+0000","last_active":"2023-09-23T23:42:43.910700+0000","last_peered":"2023-09-23T23:42:43.910700+0000","last_clean":"2023-09-23T23:42:43.910700+0000","last_became_active":"2023-09-23T23:41:31.604686+0000","last_became_peered":"2023-09-23T23:41:31.604686+0000","last_unstale":"2023-09-23T23:42:43.910700+0000","last_undegraded":"2023-09-23T23:42:43.910700+0000","last_fullsized":"2023-09-23T23:42:43.910700+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.931148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.b","version":"676'9475","reported_seq":9544,"reported_epoch":676,"state":"active+clean","last_fresh":"2023-09-23T23:42:45.584763+0000","last_change":"2023-09-23T23:39:13.744730+0000","last_active":"2023-09-23T23:42:45.584763+0000","last_peered":"2023-09-23T23:42:45.584763+0000","last_clean":"2023-09-23T23:42:45.584763+0000","last_became_active":"2023-09-23T23:39:13.744442+0000","last_became_peered":"2023-09-23T23:39:13.744442+0000","last_unstale":"2023-09-23T23:42:45.584763+0000","last_undegraded":"2023-09-23T23:42:45.584763+0000","last_fullsized":"2023-09-23T23:42:45.584763+0000","mapping_epoch":617,"log_start":"631'3224","ondisk_log_start":"631'3224","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6251,"log_dups_size":0,"ondisk_log_size":6251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:56:43.437044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58523648,"num_objects":893,"num_object_clones":0,"num_object_copies":2679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9475,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":1064,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066967+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:42:32.066967+0000","last_peered":"2023-09-23T23:42:32.066967+0000","last_clean":"2023-09-23T23:42:32.066967+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:42:32.066967+0000","last_undegraded":"2023-09-23T23:42:32.066967+0000","last_fullsized":"2023-09-23T23:42:32.066967+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":392,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.600790+0000","last_change":"2023-09-23T23:39:55.096945+0000","last_active":"2023-09-23T23:42:43.600790+0000","last_peered":"2023-09-23T23:42:43.600790+0000","last_clean":"2023-09-23T23:42:43.600790+0000","last_became_active":"2023-09-23T23:39:55.096655+0000","last_became_peered":"2023-09-23T23:39:55.096655+0000","last_unstale":"2023-09-23T23:42:43.600790+0000","last_undegraded":"2023-09-23T23:42:43.600790+0000","last_fullsized":"2023-09-23T23:42:43.600790+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:27:24.290604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":35,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.078515+0000","last_change":"2023-09-23T23:41:31.599981+0000","last_active":"2023-09-23T23:42:32.078515+0000","last_peered":"2023-09-23T23:42:32.078515+0000","last_clean":"2023-09-23T23:42:32.078515+0000","last_became_active":"2023-09-23T23:41:31.597250+0000","last_became_peered":"2023-09-23T23:41:31.597250+0000","last_unstale":"2023-09-23T23:42:32.078515+0000","last_undegraded":"2023-09-23T23:42:32.078515+0000","last_fullsized":"2023-09-23T23:42:32.078515+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:06.361776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":259,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.016839+0000","last_change":"2023-09-23T23:39:55.092828+0000","last_active":"2023-09-23T23:42:33.016839+0000","last_peered":"2023-09-23T23:42:33.016839+0000","last_clean":"2023-09-23T23:42:33.016839+0000","last_became_active":"2023-09-23T23:39:55.092504+0000","last_became_peered":"2023-09-23T23:39:55.092504+0000","last_unstale":"2023-09-23T23:42:33.016839+0000","last_undegraded":"2023-09-23T23:42:33.016839+0000","last_fullsized":"2023-09-23T23:42:33.016839+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:59.997875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1164,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066709+0000","last_change":"2023-09-23T23:41:34.675056+0000","last_active":"2023-09-23T23:42:32.066709+0000","last_peered":"2023-09-23T23:42:32.066709+0000","last_clean":"2023-09-23T23:42:32.066709+0000","last_became_active":"2023-09-23T23:41:34.674727+0000","last_became_peered":"2023-09-23T23:41:34.674727+0000","last_unstale":"2023-09-23T23:42:32.066709+0000","last_undegraded":"2023-09-23T23:42:32.066709+0000","last_fullsized":"2023-09-23T23:42:32.066709+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:01.911339+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":42,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.078247+0000","last_change":"2023-09-23T23:42:28.952637+0000","last_active":"2023-09-23T23:42:32.078247+0000","last_peered":"2023-09-23T23:42:32.078247+0000","last_clean":"2023-09-23T23:42:32.078247+0000","last_became_active":"2023-09-23T23:42:28.952283+0000","last_became_peered":"2023-09-23T23:42:28.952283+0000","last_unstale":"2023-09-23T23:42:32.078247+0000","last_undegraded":"2023-09-23T23:42:32.078247+0000","last_fullsized":"2023-09-23T23:42:32.078247+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:14.296297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":433,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.601338+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:42:43.601338+0000","last_peered":"2023-09-23T23:42:43.601338+0000","last_clean":"2023-09-23T23:42:43.601338+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:42:43.601338+0000","last_undegraded":"2023-09-23T23:42:43.601338+0000","last_fullsized":"2023-09-23T23:42:43.601338+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":876,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.067045+0000","last_change":"2023-09-23T23:41:34.667657+0000","last_active":"2023-09-23T23:42:32.067045+0000","last_peered":"2023-09-23T23:42:32.067045+0000","last_clean":"2023-09-23T23:42:32.067045+0000","last_became_active":"2023-09-23T23:41:34.659982+0000","last_became_peered":"2023-09-23T23:41:34.659982+0000","last_unstale":"2023-09-23T23:42:32.067045+0000","last_undegraded":"2023-09-23T23:42:32.067045+0000","last_fullsized":"2023-09-23T23:42:32.067045+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:49.342830+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":334,"reported_epoch":674,"state":"active+clean","last_fresh":"2023-09-23T23:42:33.017439+0000","last_change":"2023-09-23T23:39:55.115869+0000","last_active":"2023-09-23T23:42:33.017439+0000","last_peered":"2023-09-23T23:42:33.017439+0000","last_clean":"2023-09-23T23:42:33.017439+0000","last_became_active":"2023-09-23T23:39:55.092827+0000","last_became_peered":"2023-09-23T23:39:55.092827+0000","last_unstale":"2023-09-23T23:42:33.017439+0000","last_undegraded":"2023-09-23T23:42:33.017439+0000","last_fullsized":"2023-09-23T23:42:33.017439+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:03.590720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":542,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066451+0000","last_change":"2023-09-23T23:42:28.956777+0000","last_active":"2023-09-23T23:42:32.066451+0000","last_peered":"2023-09-23T23:42:32.066451+0000","last_clean":"2023-09-23T23:42:32.066451+0000","last_became_active":"2023-09-23T23:42:28.956436+0000","last_became_peered":"2023-09-23T23:42:28.956436+0000","last_unstale":"2023-09-23T23:42:32.066451+0000","last_undegraded":"2023-09-23T23:42:32.066451+0000","last_fullsized":"2023-09-23T23:42:32.066451+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:42.633268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":62,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066246+0000","last_change":"2023-09-23T23:39:55.128319+0000","last_active":"2023-09-23T23:42:32.066246+0000","last_peered":"2023-09-23T23:42:32.066246+0000","last_clean":"2023-09-23T23:42:32.066246+0000","last_became_active":"2023-09-23T23:39:55.098936+0000","last_became_peered":"2023-09-23T23:39:55.098936+0000","last_unstale":"2023-09-23T23:42:32.066246+0000","last_undegraded":"2023-09-23T23:42:32.066246+0000","last_fullsized":"2023-09-23T23:42:32.066246+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:03.570174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":363,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.067340+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:42:32.067340+0000","last_peered":"2023-09-23T23:42:32.067340+0000","last_clean":"2023-09-23T23:42:32.067340+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:42:32.067340+0000","last_undegraded":"2023-09-23T23:42:32.067340+0000","last_fullsized":"2023-09-23T23:42:32.067340+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":919,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:31.948573+0000","last_change":"2023-09-23T23:39:58.111742+0000","last_active":"2023-09-23T23:42:31.948573+0000","last_peered":"2023-09-23T23:42:31.948573+0000","last_clean":"2023-09-23T23:42:31.948573+0000","last_became_active":"2023-09-23T23:39:58.111358+0000","last_became_peered":"2023-09-23T23:39:58.111358+0000","last_unstale":"2023-09-23T23:42:31.948573+0000","last_undegraded":"2023-09-23T23:42:31.948573+0000","last_fullsized":"2023-09-23T23:42:31.948573+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:54.588394+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":716,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066966+0000","last_change":"2023-09-23T23:39:55.101668+0000","last_active":"2023-09-23T23:42:32.066966+0000","last_peered":"2023-09-23T23:42:32.066966+0000","last_clean":"2023-09-23T23:42:32.066966+0000","last_became_active":"2023-09-23T23:39:55.100664+0000","last_became_peered":"2023-09-23T23:39:55.100664+0000","last_unstale":"2023-09-23T23:42:32.066966+0000","last_undegraded":"2023-09-23T23:42:32.066966+0000","last_fullsized":"2023-09-23T23:42:32.066966+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:43.354111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":691,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.805709+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:42:32.805709+0000","last_peered":"2023-09-23T23:42:32.805709+0000","last_clean":"2023-09-23T23:42:32.805709+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:42:32.805709+0000","last_undegraded":"2023-09-23T23:42:32.805709+0000","last_fullsized":"2023-09-23T23:42:32.805709+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":849,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066469+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:42:32.066469+0000","last_peered":"2023-09-23T23:42:32.066469+0000","last_clean":"2023-09-23T23:42:32.066469+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:42:32.066469+0000","last_undegraded":"2023-09-23T23:42:32.066469+0000","last_fullsized":"2023-09-23T23:42:32.066469+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":552,"reported_epoch":675,"state":"active+clean","last_fresh":"2023-09-23T23:42:43.600874+0000","last_change":"2023-09-23T23:41:31.624156+0000","last_active":"2023-09-23T23:42:43.600874+0000","last_peered":"2023-09-23T23:42:43.600874+0000","last_clean":"2023-09-23T23:42:43.600874+0000","last_became_active":"2023-09-23T23:41:31.612962+0000","last_became_peered":"2023-09-23T23:41:31.612962+0000","last_unstale":"2023-09-23T23:42:43.600874+0000","last_undegraded":"2023-09-23T23:42:43.600874+0000","last_fullsized":"2023-09-23T23:42:43.600874+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:31.042576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":674,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-09-23T23:42:32.066907+0000","last_change":"2023-09-23T23:39:55.099784+0000","last_active":"2023-09-23T23:42:32.066907+0000","last_peered":"2023-09-23T23:42:32.066907+0000","last_clean":"2023-09-23T23:42:32.066907+0000","last_became_active":"2023-09-23T23:39:55.099483+0000","last_became_peered":"2023-09-23T23:39:55.099483+0000","last_unstale":"2023-09-23T23:42:32.066907+0000","last_undegraded":"2023-09-23T23:42:32.066907+0000","last_fullsized":"2023-09-23T23:42:32.066907+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:48.012026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":992215064,"num_objects":15141,"num_object_clones":0,"num_object_copies":45423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15141,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152613,"num_write_kb":5368065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14934,"num_bytes_recovered":809631744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3328782336,"data_stored":3328770120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87966,"ondisk_log_size":87966,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":71,"num_bytes_recovered":12612,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":143360,"data_stored":26230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782759,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":879268,"kb_used_data":395548,"kb_used_omap":8,"kb_used_meta":483703,"kb_avail":92865884,"statfs":{"total":95995035648,"available":95094665216,"internally_reserved":0,"allocated":405041152,"data_stored":403081053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":495311941},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.0900000000000001,"15min":1.6180000000000001},"min":{"1min":0.23799999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":3.528,"5min":12.432,"15min":44.094999999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.353,"5min":1.3320000000000001,"15min":1.76},"min":{"1min":0.27100000000000002,"5min":0.22700000000000001,"15min":0.21299999999999999},"max":{"1min":4.4720000000000004,"5min":12.717000000000001,"15min":44.335999999999999},"last":2.7290000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":1.377,"15min":3.0409999999999999},"min":{"1min":0.28499999999999998,"5min":0.255,"15min":0.23200000000000001},"max":{"1min":5.2229999999999999,"5min":12.484,"15min":198.58000000000001},"last":0.496},{"interface":"front","average":{"1min":1.575,"5min":1.526,"15min":3.0030000000000001},"min":{"1min":0.28999999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.6749999999999998,"5min":13.145,"15min":198.71100000000001},"last":0.34599999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":2.524,"5min":2.3599999999999999,"15min":3.798},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":8.9779999999999998,"5min":46.374000000000002,"15min":203.58000000000001},"last":4.5949999999999998},{"interface":"front","average":{"1min":1.909,"5min":2.5230000000000001,"15min":3.4399999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26200000000000001},"max":{"1min":9.9670000000000005,"5min":46.412999999999997,"15min":198.69},"last":3.1960000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":2.2759999999999998,"5min":5.1429999999999998,"15min":5.2770000000000001},"min":{"1min":0.31,"5min":0.309,"15min":0.26700000000000002},"max":{"1min":8.1479999999999997,"5min":154.18100000000001,"15min":279.26499999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":2.931,"5min":5.0919999999999996,"15min":5.8150000000000004},"min":{"1min":0.32600000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":8.5779999999999994,"5min":154.16300000000001,"15min":279.29899999999998},"last":2.8029999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.69199999999999995,"15min":1.008},"min":{"1min":0.29599999999999999,"5min":0.28999999999999998,"15min":0.246},"max":{"1min":1.5169999999999999,"5min":12.510999999999999,"15min":31.498999999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.435,"5min":0.80100000000000005,"15min":0.93999999999999995},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.23899999999999999},"max":{"1min":0.82399999999999995,"5min":12.821999999999999,"15min":31.68},"last":0.38100000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":2.0659999999999998,"15min":2.2090000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.88300000000000001,"5min":119.417,"15min":210.83600000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.40999999999999998,"5min":2.1669999999999998,"15min":2.1949999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.27700000000000002},"max":{"1min":0.54900000000000004,"5min":119.61499999999999,"15min":207.04599999999999},"last":0.51000000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.84199999999999997,"15min":1.149},"min":{"1min":0.34499999999999997,"5min":0.32800000000000001,"15min":0.29899999999999999},"max":{"1min":0.77900000000000003,"5min":6.2169999999999996,"15min":94.852999999999994},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.81899999999999995,"15min":1.274},"min":{"1min":0.36099999999999999,"5min":0.31,"15min":0.30599999999999999},"max":{"1min":0.84999999999999998,"5min":12.529999999999999,"15min":95.510999999999996},"last":0.66200000000000003}]}]},{"osd":6,"up_from":0,"seq":197568496236,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":112,"seq":481036337577,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1603304,"kb_used_data":574880,"kb_used_omap":4,"kb_used_meta":1028411,"kb_avail":92141848,"statfs":{"total":95995035648,"available":94353252352,"internally_reserved":0,"allocated":588677120,"data_stored":585379421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1053093774},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:42:31 2023","interfaces":[{"interface":"back","average":{"1min":3.0979999999999999,"5min":5.2750000000000004,"15min":3.9780000000000002},"min":{"1min":0.20999999999999999,"5min":0.16400000000000001,"15min":0.16400000000000001},"max":{"1min":9.1319999999999997,"5min":156.37799999999999,"15min":156.37799999999999},"last":0.254},{"interface":"front","average":{"1min":1.821,"5min":4.7320000000000002,"15min":3.4220000000000002},"min":{"1min":0.221,"5min":0.189,"15min":0.189},"max":{"1min":8.8049999999999997,"5min":161.05199999999999,"15min":161.05199999999999},"last":0.224}]},{"osd":2,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":5.9349999999999996,"5min":3.1960000000000002,"15min":6.0010000000000003},"min":{"1min":0.22900000000000001,"5min":0.184,"15min":0.184},"max":{"1min":49.311,"5min":53.951999999999998,"15min":231.67400000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":6.383,"5min":3.7789999999999999,"15min":6.2679999999999998},"min":{"1min":0.23400000000000001,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":48.991999999999997,"5min":54.213000000000001,"15min":232.21000000000001},"last":0.24099999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":18.759,"5min":6.4210000000000003,"15min":6.6829999999999998},"min":{"1min":0.25,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":272.839,"5min":272.839,"15min":272.839},"last":43.515000000000001},{"interface":"front","average":{"1min":17.308,"5min":6.3499999999999996,"15min":6.5949999999999998},"min":{"1min":0.27700000000000002,"5min":0.245,"15min":0.224},"max":{"1min":272.76400000000001,"5min":272.76400000000001,"15min":272.76400000000001},"last":43.482999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:42:31 2023","interfaces":[{"interface":"back","average":{"1min":3.194,"5min":2.004,"15min":1.5600000000000001},"min":{"1min":0.42999999999999999,"5min":0.34000000000000002,"15min":0.29799999999999999},"max":{"1min":10.183,"5min":10.413,"15min":12.042999999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.2549999999999999,"15min":1.2669999999999999},"min":{"1min":0.36699999999999999,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":7.8760000000000003,"5min":13.622999999999999,"15min":18.108000000000001},"last":0.40999999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:42:31 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":2.23,"15min":2.1150000000000002},"min":{"1min":0.31,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":2.7890000000000001,"5min":80.302999999999997,"15min":151.94800000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":3.3090000000000002,"5min":2.8879999999999999,"15min":2.4279999999999999},"min":{"1min":0.39200000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.141999999999999,"5min":80.254000000000005,"15min":151.821},"last":0.48899999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:42:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":2.129,"15min":1.5740000000000001},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.32200000000000001},"max":{"1min":2.8180000000000001,"5min":87.837999999999994,"15min":87.837999999999994},"last":0.45600000000000002},{"interface":"front","average":{"1min":3.2200000000000002,"5min":2.8570000000000002,"15min":1.986},"min":{"1min":0.42999999999999999,"5min":0.312,"15min":0.312},"max":{"1min":10.233000000000001,"5min":85.518000000000001,"15min":85.518000000000001},"last":0.39900000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0169999999999999,"5min":1.6779999999999999,"15min":1.6319999999999999},"min":{"1min":0.40300000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":13.587999999999999,"5min":35.487000000000002,"15min":44.241},"last":0.38500000000000001},{"interface":"front","average":{"1min":3.069,"5min":2.3410000000000002,"15min":1.9350000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.210000000000001,"5min":35.146000000000001,"15min":43.920999999999999},"last":0.433}]}]},{"osd":0,"up_from":91,"seq":390842024400,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2072288,"kb_used_data":696280,"kb_used_omap":5,"kb_used_meta":1375994,"kb_avail":91672864,"statfs":{"total":95995035648,"available":93873012736,"internally_reserved":0,"allocated":712990720,"data_stored":709695720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1409018490},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:41:44 2023","interfaces":[{"interface":"back","average":{"1min":2.8180000000000001,"5min":1.8839999999999999,"15min":5.7699999999999996},"min":{"1min":0.23000000000000001,"5min":0.219,"15min":0.17799999999999999},"max":{"1min":7.1890000000000001,"5min":15.589,"15min":301.12700000000001},"last":0.255},{"interface":"front","average":{"1min":3.0609999999999999,"5min":1.9770000000000001,"15min":5.851},"min":{"1min":0.23899999999999999,"5min":0.219,"15min":0.186},"max":{"1min":9.7159999999999993,"5min":15.446999999999999,"15min":297.72399999999999},"last":0.221}]},{"osd":2,"last update":"Sat Sep 23 23:42:13 2023","interfaces":[{"interface":"back","average":{"1min":5.476,"5min":3.8759999999999999,"15min":5.7949999999999999},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.17199999999999999},"max":{"1min":20.542000000000002,"5min":102.279,"15min":283.07400000000001},"last":89.935000000000002},{"interface":"front","average":{"1min":4.3719999999999999,"5min":3.8420000000000001,"15min":5.9340000000000002},"min":{"1min":0.252,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":18.529,"5min":102.218,"15min":283.161},"last":90.25}]},{"osd":3,"last update":"Sat Sep 23 23:41:44 2023","interfaces":[{"interface":"back","average":{"1min":5.5359999999999996,"5min":7.4420000000000002,"15min":6.5110000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.217},"max":{"1min":36.186999999999998,"5min":194.26300000000001,"15min":220.732},"last":0.23200000000000001},{"interface":"front","average":{"1min":6.0629999999999997,"5min":7.7750000000000004,"15min":6.7880000000000003},"min":{"1min":0.27500000000000002,"5min":0.254,"15min":0.19800000000000001},"max":{"1min":33.616,"5min":194.56299999999999,"15min":218.78200000000001},"last":0.245}]},{"osd":4,"last update":"Sat Sep 23 23:42:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":1.7,"15min":1.784},"min":{"1min":0.36899999999999999,"5min":0.32600000000000001,"15min":0.28499999999999998},"max":{"1min":18.274999999999999,"5min":18.274999999999999,"15min":108.01900000000001},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.554,"5min":1.5629999999999999,"15min":1.7729999999999999},"min":{"1min":0.39300000000000002,"5min":0.32600000000000001,"15min":0.27200000000000002},"max":{"1min":6.1310000000000002,"5min":8.718,"15min":107.898},"last":0.56499999999999995}]},{"osd":5,"last update":"Sat Sep 23 23:42:24 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":3.117,"15min":1.9610000000000001},"min":{"1min":0.36799999999999999,"5min":0.36699999999999999,"15min":0.32600000000000001},"max":{"1min":6.0650000000000004,"5min":94.254000000000005,"15min":94.254000000000005},"last":0.73199999999999998},{"interface":"front","average":{"1min":3.1200000000000001,"5min":3.3780000000000001,"15min":1.962},"min":{"1min":0.45200000000000001,"5min":0.32900000000000001,"15min":0.253},"max":{"1min":9.5869999999999997,"5min":94.837000000000003,"15min":94.837000000000003},"last":0.55400000000000005}]},{"osd":6,"last update":"Sat Sep 23 23:42:24 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":2.3690000000000002,"15min":2.0659999999999998},"min":{"1min":0.33500000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.0979999999999999,"5min":69.043999999999997,"15min":108.67},"last":0.67100000000000004},{"interface":"front","average":{"1min":3.1629999999999998,"5min":2.597,"15min":2.121},"min":{"1min":0.52900000000000003,"5min":0.38200000000000001,"15min":0.27100000000000002},"max":{"1min":9.6310000000000002,"5min":69.028999999999996,"15min":108.804},"last":0.623}]},{"osd":7,"last update":"Sat Sep 23 23:42:13 2023","interfaces":[{"interface":"back","average":{"1min":2.5339999999999998,"5min":2.5129999999999999,"15min":2.2200000000000002},"min":{"1min":0.41299999999999998,"5min":0.35899999999999999,"15min":0.32900000000000001},"max":{"1min":18.469999999999999,"5min":66.620000000000005,"15min":111.71299999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":3.0710000000000002,"5min":2.4180000000000001,"15min":2.1840000000000002},"min":{"1min":0.496,"5min":0.35399999999999998,"15min":0.33300000000000002},"max":{"1min":9.6590000000000007,"5min":66.701999999999998,"15min":111.108},"last":0.70399999999999996}]}]},{"osd":2,"up_from":199,"seq":854698492222,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1507096,"kb_used_data":473232,"kb_used_omap":7,"kb_used_meta":1033720,"kb_avail":92238056,"statfs":{"total":95995035648,"available":94451769344,"internally_reserved":0,"allocated":484589568,"data_stored":481205479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1058529361},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,5,0,1,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":5.1479999999999997,"5min":3.2290000000000001,"15min":2.7490000000000001},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":34.648000000000003,"5min":65.001000000000005,"15min":65.001000000000005},"last":2.782},{"interface":"front","average":{"1min":3.1379999999999999,"5min":2.5259999999999998,"15min":2.4670000000000001},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19700000000000001},"max":{"1min":13.132,"5min":64.346000000000004,"15min":64.346000000000004},"last":0.38800000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":2.8220000000000001,"5min":2.8279999999999998,"15min":3.8719999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.192},"max":{"1min":14.369,"5min":41.43,"15min":106.765},"last":8.8239999999999998},{"interface":"front","average":{"1min":3.4129999999999998,"5min":2.6200000000000001,"15min":3.7290000000000001},"min":{"1min":0.36299999999999999,"5min":0.24399999999999999,"15min":0.222},"max":{"1min":11.821999999999999,"5min":53.268000000000001,"15min":106.732},"last":3.222}]},{"osd":3,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":6.2300000000000004,"5min":3.1909999999999998,"15min":4.3090000000000002},"min":{"1min":0.315,"5min":0.245,"15min":0.24099999999999999},"max":{"1min":54.441000000000003,"5min":54.441000000000003,"15min":106.70399999999999},"last":4.8140000000000001},{"interface":"front","average":{"1min":6.1120000000000001,"5min":2.6720000000000002,"15min":3.9910000000000001},"min":{"1min":0.23899999999999999,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":53.734000000000002,"5min":53.734000000000002,"15min":106.66800000000001},"last":2.8130000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":3.23,"5min":1.5669999999999999,"15min":1.7929999999999999},"min":{"1min":0.41199999999999998,"5min":0.314,"15min":0.27700000000000002},"max":{"1min":16.001000000000001,"5min":16.001000000000001,"15min":62.081000000000003},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.6870000000000001,"5min":1.77,"15min":1.9259999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.28899999999999998},"max":{"1min":6.6849999999999996,"5min":15.055999999999999,"15min":62.072000000000003},"last":4.8579999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":4.2229999999999999,"15min":3.0110000000000001},"min":{"1min":0.38600000000000001,"5min":0.32200000000000001,"15min":0.26200000000000001},"max":{"1min":13.023,"5min":101.104,"15min":101.104},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.6830000000000001,"5min":4.681,"15min":3.1589999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.641,"5min":101.404,"15min":101.404},"last":4.891}]},{"osd":6,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2559999999999998,"5min":4.3710000000000004,"15min":3.3039999999999998},"min":{"1min":0.437,"5min":0.33400000000000002,"15min":0.31},"max":{"1min":15.898999999999999,"5min":97.558999999999997,"15min":107.018},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.71,"5min":4.7530000000000001,"15min":3.0590000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.7130000000000001,"5min":98.231999999999999,"15min":98.231999999999999},"last":4.9260000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":2.238,"15min":2.085},"min":{"1min":0.41899999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.9470000000000001,"5min":68.814999999999998,"15min":68.814999999999998},"last":1.2609999999999999},{"interface":"front","average":{"1min":2.3879999999999999,"5min":2.4580000000000002,"15min":2.5659999999999998},"min":{"1min":0.45900000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":6.923,"5min":69.918999999999997,"15min":106.809},"last":1.2869999999999999}]}]},{"osd":3,"up_from":148,"seq":635655160190,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1628096,"kb_used_data":521336,"kb_used_omap":2,"kb_used_meta":1106493,"kb_avail":92117056,"statfs":{"total":95995035648,"available":94327865344,"internally_reserved":0,"allocated":533848064,"data_stored":530583266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1133048885},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:41:58 2023","interfaces":[{"interface":"back","average":{"1min":8.3770000000000007,"5min":3.1070000000000002,"15min":2.3029999999999999},"min":{"1min":0.20300000000000001,"5min":0.19500000000000001,"15min":0.17699999999999999},"max":{"1min":69.281999999999996,"5min":69.281999999999996,"15min":69.281999999999996},"last":4.8570000000000002},{"interface":"front","average":{"1min":4.3600000000000003,"5min":2.2690000000000001,"15min":2.4209999999999998},"min":{"1min":0.247,"5min":0.188,"15min":0.17199999999999999},"max":{"1min":20.071000000000002,"5min":20.071000000000002,"15min":46.204000000000001},"last":5.0869999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:41:58 2023","interfaces":[{"interface":"back","average":{"1min":3.6040000000000001,"5min":1.738,"15min":2.7000000000000002},"min":{"1min":0.27000000000000002,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":9.1649999999999991,"5min":11.911,"15min":85.242999999999995},"last":5.4260000000000002},{"interface":"front","average":{"1min":4.774,"5min":2.2759999999999998,"15min":2.7919999999999998},"min":{"1min":0.30099999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":20.779,"5min":20.779,"15min":85.302000000000007},"last":15.315}]},{"osd":2,"last update":"Sat Sep 23 23:42:06 2023","interfaces":[{"interface":"back","average":{"1min":3.835,"5min":6.1420000000000003,"15min":5.0220000000000002},"min":{"1min":0.26100000000000001,"5min":0.23999999999999999,"15min":0.219},"max":{"1min":9.4100000000000001,"5min":139.96299999999999,"15min":139.96299999999999},"last":5.4500000000000002},{"interface":"front","average":{"1min":7.3789999999999996,"5min":7.0220000000000002,"15min":5.2649999999999997},"min":{"1min":0.30099999999999999,"5min":0.26800000000000002,"15min":0.25700000000000001},"max":{"1min":69.405000000000001,"5min":141.93199999999999,"15min":141.93199999999999},"last":4.8719999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:41:58 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":1.3939999999999999,"15min":1.3300000000000001},"min":{"1min":0.34100000000000003,"5min":0.308,"15min":0.26800000000000002},"max":{"1min":7.9790000000000001,"5min":11.414,"15min":11.414},"last":4.8200000000000003},{"interface":"front","average":{"1min":2.3079999999999998,"5min":1.839,"15min":1.786},"min":{"1min":0.37,"5min":0.32900000000000001,"15min":0.26800000000000002},"max":{"1min":6.7439999999999998,"5min":14.967000000000001,"15min":24.782},"last":5.1600000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:41:58 2023","interfaces":[{"interface":"back","average":{"1min":2.363,"5min":5.4210000000000003,"15min":4.6289999999999996},"min":{"1min":0.36499999999999999,"5min":0.33500000000000002,"15min":0.29499999999999998},"max":{"1min":6.6630000000000003,"5min":196.358,"15min":214.43700000000001},"last":5.2320000000000002},{"interface":"front","average":{"1min":2.415,"5min":5.0709999999999997,"15min":4.5220000000000002},"min":{"1min":0.38500000000000001,"5min":0.32700000000000001,"15min":0.30599999999999999},"max":{"1min":9.2210000000000001,"5min":196.048,"15min":214.46600000000001},"last":5.3689999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:41:58 2023","interfaces":[{"interface":"back","average":{"1min":2.3330000000000002,"5min":3.1560000000000001,"15min":2.5270000000000001},"min":{"1min":0.443,"5min":0.35799999999999998,"15min":0.318},"max":{"1min":6.718,"5min":125.94,"15min":125.94},"last":5.2119999999999997},{"interface":"front","average":{"1min":2.4580000000000002,"5min":2.8660000000000001,"15min":2.4209999999999998},"min":{"1min":0.40200000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.2469999999999999,"5min":125.955,"15min":125.955},"last":5.4009999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:42:06 2023","interfaces":[{"interface":"back","average":{"1min":5.4770000000000003,"5min":2.27,"15min":3.5859999999999999},"min":{"1min":0.38900000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":69.197999999999993,"5min":69.197999999999993,"15min":192.69},"last":4.8419999999999996},{"interface":"front","average":{"1min":3.254,"5min":2.1070000000000002,"15min":3.8410000000000002},"min":{"1min":0.48899999999999999,"5min":0.39500000000000002,"15min":0.36699999999999999},"max":{"1min":12.568,"5min":14.923999999999999,"15min":192.65600000000001},"last":5.1859999999999999}]}]},{"osd":4,"up_from":0,"seq":128849019509,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757872,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1285404,"kb_used_data":438740,"kb_used_omap":0,"kb_used_meta":846592,"kb_avail":92459748,"statfs":{"total":95995035648,"available":94678781952,"internally_reserved":0,"allocated":449269760,"data_stored":447321861,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":866910208},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:42:33 2023","interfaces":[{"interface":"back","average":{"1min":15.054,"5min":4.782,"15min":2.4399999999999999},"min":{"1min":0.251,"5min":0.223,"15min":0.20899999999999999},"max":{"1min":174.58600000000001,"5min":174.58600000000001,"15min":174.58600000000001},"last":0.68100000000000005},{"interface":"front","average":{"1min":14.362,"5min":5.0739999999999998,"15min":2.5750000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.22700000000000001},"max":{"1min":174.69499999999999,"5min":174.69499999999999,"15min":174.69499999999999},"last":0.50600000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:42:33 2023","interfaces":[{"interface":"back","average":{"1min":3.2789999999999999,"5min":3.0510000000000002,"15min":2.0489999999999999},"min":{"1min":0.28000000000000003,"5min":0.25600000000000001,"15min":0.23100000000000001},"max":{"1min":38.468000000000004,"5min":95.709000000000003,"15min":95.709000000000003},"last":0.72199999999999998},{"interface":"front","average":{"1min":3.4159999999999999,"5min":3.2789999999999999,"15min":2.198},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.249},"max":{"1min":38.841999999999999,"5min":94.966999999999999,"15min":94.966999999999999},"last":0.748}]},{"osd":2,"last update":"Sat Sep 23 23:42:01 2023","interfaces":[{"interface":"back","average":{"1min":2.8780000000000001,"5min":3.6030000000000002,"15min":4.4500000000000002},"min":{"1min":0.40100000000000002,"5min":0.28799999999999998,"15min":0.27200000000000002},"max":{"1min":9.5879999999999992,"5min":132.81999999999999,"15min":208.738},"last":0.77500000000000002},{"interface":"front","average":{"1min":2.472,"5min":3.169,"15min":3.8690000000000002},"min":{"1min":0.313,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.3360000000000003,"5min":133.09200000000001,"15min":208.631},"last":0.73599999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:42:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":3.5550000000000002,"15min":4.1849999999999996},"min":{"1min":0.34599999999999997,"5min":0.33100000000000002,"15min":0.27800000000000002},"max":{"1min":14.288,"5min":143.66300000000001,"15min":143.66300000000001},"last":0.71299999999999997},{"interface":"front","average":{"1min":2.552,"5min":3.6139999999999999,"15min":4.3410000000000002},"min":{"1min":0.44500000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":14.307,"5min":143.62200000000001,"15min":143.62200000000001},"last":0.54200000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.90200000000000002,"15min":0.92600000000000005},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.252},"max":{"1min":6.9390000000000001,"5min":6.9390000000000001,"15min":19.962},"last":4.0629999999999997},{"interface":"front","average":{"1min":0.82599999999999996,"5min":0.78400000000000003,"15min":0.89400000000000002},"min":{"1min":0.31900000000000001,"5min":0.28899999999999998,"15min":0.22900000000000001},"max":{"1min":6.7590000000000003,"5min":6.7590000000000003,"15min":20.829999999999998},"last":0.94099999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.95699999999999996,"15min":1.3380000000000001},"min":{"1min":0.32900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.617,"5min":14.252000000000001,"15min":148.36600000000001},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.94499999999999995,"15min":1.5229999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.8040000000000003,"5min":6.8040000000000003,"15min":148.42500000000001},"last":0.78300000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.91400000000000003,"15min":1.7190000000000001},"min":{"1min":0.316,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.56299999999999994,"5min":6.4199999999999999,"15min":179.28800000000001},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.0049999999999999,"15min":1.796},"min":{"1min":0.34899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.9279999999999999,"5min":6.4649999999999999,"15min":179.428},"last":2.8690000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":700645376,"data_stored":700645376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":575668224,"data_stored":575668224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":474288128,"data_stored":474284056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":513081344,"data_stored":513081344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":425267200,"data_stored":425263128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":248315904,"data_stored":248315904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":391516160,"data_stored":391512088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:42:48.312 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:42:48.347 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:48.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:48.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:48.453 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:42:48.483 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:48.500 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:42:48.541 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:42:48.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:48.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:42:48.607 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:48.648 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:48.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:48 smithi110 ceph-mon[221102]: failed to encode map e677 with expected crc 2023-09-23T23:42:48.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:48 smithi110 ceph-mon[221102]: failed to encode map e677 with expected crc 2023-09-23T23:42:48.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:48 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:42:48.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:48 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:42:48.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:48 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:48.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:48 smithi110 ceph-mon[212939]: failed to encode map e677 with expected crc 2023-09-23T23:42:48.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:48 smithi110 ceph-mon[212939]: failed to encode map e677 with expected crc 2023-09-23T23:42:48.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:48 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:42:48.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:48 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:42:48.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:48.742 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:48.790 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:42:48.833 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:42:48.877 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:48.877 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:42:48.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:48.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:48.937 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:42:48.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:48 smithi138 ceph-mon[193606]: failed to encode map e677 with expected crc 2023-09-23T23:42:48.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:48 smithi138 ceph-mon[193606]: failed to encode map e677 with expected crc 2023-09-23T23:42:48.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:48 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:42:48.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:48 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:42:48.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:48 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:48.992 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:42:49.003 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:42:49.026 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:49.039 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:42:49.075 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:42:49.083 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:42:49.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:49.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:42:49.137 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:42:49.159 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:42:49.160 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:42:49.160 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:42:49.160 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:42:49.160 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:42:49.160 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:42:49.160 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:42:49.160 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:42:49.187 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:49.216 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:49.313 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:49.315 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:49.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:49.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:49.421 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:49.512 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:49.817 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:49 smithi138 ceph-mon[193606]: pgmap v1925: 119 pgs: 1 peering, 118 active+clean; 855 MiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 771 op/s 2023-09-23T23:42:49.817 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:49 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3222784983' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:49.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:49 smithi110 ceph-mon[212939]: pgmap v1925: 119 pgs: 1 peering, 118 active+clean; 855 MiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 771 op/s 2023-09-23T23:42:49.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:49 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3222784983' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:49.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:49 smithi110 ceph-mon[221102]: pgmap v1925: 119 pgs: 1 peering, 118 active+clean; 855 MiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 771 op/s 2023-09-23T23:42:49.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:49 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3222784983' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:50.050 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:42:50.051 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:42:50.051 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:42:50.051 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:42:50.051 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:42:50.751 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:42:51.136 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:42:51.571 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:51.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:51.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:51.676 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:42:51.707 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:51.761 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:42:51.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:51.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:42:51.865 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:42:51.892 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:51 smithi110 ceph-mon[212939]: pgmap v1926: 119 pgs: 1 peering, 118 active+clean; 840 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 500 op/s 2023-09-23T23:42:51.892 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:51 smithi110 ceph-mon[221102]: pgmap v1926: 119 pgs: 1 peering, 118 active+clean; 840 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 500 op/s 2023-09-23T23:42:51.898 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:51.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:51 smithi138 ceph-mon[193606]: pgmap v1926: 119 pgs: 1 peering, 118 active+clean; 840 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 500 op/s 2023-09-23T23:42:52.003 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:42:52.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:52.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:42:52.112 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:52.137 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:52.178 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:52.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:52.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:52.282 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:52.375 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:52.722 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:42:52.722 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:42:52.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:42:52.723 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:42:52.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.723 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71506 13.5 13:afea6fb0:::benchmark_data_smithi110_541928_object71505:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:42:52.723 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.551723+0000", 2023-09-23T23:42:52.723 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 542.16641939099998, 2023-09-23T23:42:52.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2818913190000001, 2023-09-23T23:42:52.723 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.723 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:42:52.723 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.723 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:42:52.724 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:52.724 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71506 2023-09-23T23:42:52.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.724 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551723+0000", 2023-09-23T23:42:52.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551723+0000", 2023-09-23T23:42:52.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551721+0000", 2023-09-23T23:42:52.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:42:52.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551766+0000", 2023-09-23T23:42:52.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4861000000000003e-05 2023-09-23T23:42:52.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551821+0000", 2023-09-23T23:42:52.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4308999999999998e-05 2023-09-23T23:42:52.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551836+0000", 2023-09-23T23:42:52.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5704000000000001e-05 2023-09-23T23:42:52.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.552046+0000", 2023-09-23T23:42:52.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021007999999999999 2023-09-23T23:42:52.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.728 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op must wait for map", 2023-09-23T23:42:52.728 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.552054+0000", 2023-09-23T23:42:52.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3429999999999998e-06 2023-09-23T23:42:52.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.728 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.728 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.566691+0000", 2023-09-23T23:42:52.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014637636000000001 2023-09-23T23:42:52.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:42:52.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.566701+0000", 2023-09-23T23:42:52.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4339999999999998e-06 2023-09-23T23:42:52.729 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.532653+0000", 2023-09-23T23:42:52.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96595198800000004 2023-09-23T23:42:52.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:42:52.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.532669+0000", 2023-09-23T23:42:52.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5919000000000001e-05 2023-09-23T23:42:52.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.731 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.831588+0000", 2023-09-23T23:42:52.731 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2989191230000001 2023-09-23T23:42:52.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.731 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.731 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.833615+0000", 2023-09-23T23:42:52.731 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020266519999999999 2023-09-23T23:42:52.731 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.731 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.731 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.732 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71510 13.5 13:a6f614de:::benchmark_data_smithi110_541928_object71509:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:42:52.732 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.551852+0000", 2023-09-23T23:42:52.732 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 542.16629062000004, 2023-09-23T23:42:52.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2819083340000001, 2023-09-23T23:42:52.732 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.732 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:42:52.732 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.732 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:42:52.733 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:52.733 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71510 2023-09-23T23:42:52.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.733 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551852+0000", 2023-09-23T23:42:52.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551852+0000", 2023-09-23T23:42:52.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551851+0000", 2023-09-23T23:42:52.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:52.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551896+0000", 2023-09-23T23:42:52.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4605999999999999e-05 2023-09-23T23:42:52.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.735 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.735 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551905+0000", 2023-09-23T23:42:52.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7940000000000008e-06 2023-09-23T23:42:52.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.735 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.735 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551911+0000", 2023-09-23T23:42:52.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9630000000000003e-06 2023-09-23T23:42:52.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.552096+0000", 2023-09-23T23:42:52.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018562799999999999 2023-09-23T23:42:52.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting_for_map not empty", 2023-09-23T23:42:52.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.552102+0000", 2023-09-23T23:42:52.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4940000000000003e-06 2023-09-23T23:42:52.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.567882+0000", 2023-09-23T23:42:52.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015779791000000001 2023-09-23T23:42:52.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:42:52.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.567890+0000", 2023-09-23T23:42:52.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6710000000000005e-06 2023-09-23T23:42:52.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.738 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.532701+0000", 2023-09-23T23:42:52.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96481083899999998 2023-09-23T23:42:52.738 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.738 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.738 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:42:52.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.532709+0000", 2023-09-23T23:42:52.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5989999999999996e-06 2023-09-23T23:42:52.738 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.738 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.738 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.833653+0000", 2023-09-23T23:42:52.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3009445150000001 2023-09-23T23:42:52.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.739 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.739 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.833760+0000", 2023-09-23T23:42:52.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107042 2023-09-23T23:42:52.739 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.739 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.739 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.739 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71519 13.6 13:66147c0c:::benchmark_data_smithi110_541928_object71518:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:42:52.739 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.551916+0000", 2023-09-23T23:42:52.740 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 542.16622616500001, 2023-09-23T23:42:52.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.26976434, 2023-09-23T23:42:52.740 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.740 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:42:52.740 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.740 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:42:52.740 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:52.740 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71519 2023-09-23T23:42:52.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.740 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551916+0000", 2023-09-23T23:42:52.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551916+0000", 2023-09-23T23:42:52.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551916+0000", 2023-09-23T23:42:52.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:52.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551943+0000", 2023-09-23T23:42:52.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6628999999999999e-05 2023-09-23T23:42:52.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551950+0000", 2023-09-23T23:42:52.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7440000000000007e-06 2023-09-23T23:42:52.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.551955+0000", 2023-09-23T23:42:52.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8940000000000001e-06 2023-09-23T23:42:52.743 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.554085+0000", 2023-09-23T23:42:52.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021293810000000001 2023-09-23T23:42:52.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.744 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op must wait for map", 2023-09-23T23:42:52.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.554092+0000", 2023-09-23T23:42:52.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4630000000000004e-06 2023-09-23T23:42:52.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.744 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.559055+0000", 2023-09-23T23:42:52.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049632790000000001 2023-09-23T23:42:52.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:42:52.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.559063+0000", 2023-09-23T23:42:52.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6310000000000003e-06 2023-09-23T23:42:52.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.746 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.535471+0000", 2023-09-23T23:42:52.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97640791500000002 2023-09-23T23:42:52.746 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.746 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.746 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:42:52.746 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.535479+0000", 2023-09-23T23:42:52.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0409999999999998e-06 2023-09-23T23:42:52.746 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.747 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.747 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.805321+0000", 2023-09-23T23:42:52.747 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2698418789999999 2023-09-23T23:42:52.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.747 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-09-23T23:42:52.747 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.810871+0000", 2023-09-23T23:42:52.747 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005550007 2023-09-23T23:42:52.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.820016+0000", 2023-09-23T23:42:52.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0091451060000000001 2023-09-23T23:42:52.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.821681+0000", 2023-09-23T23:42:52.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001664888 2023-09-23T23:42:52.748 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.748 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.748 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.749 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:89710 13.5 13:a3a20adb:::benchmark_data_smithi110_541928_object1930:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-23T23:42:52.749 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:34.617023+0000", 2023-09-23T23:42:52.749 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 498.10111945400001, 2023-09-23T23:42:52.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3090113269999999, 2023-09-23T23:42:52.749 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.749 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:42:52.749 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.749 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:42:52.749 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:52.749 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 89710 2023-09-23T23:42:52.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.749 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.617023+0000", 2023-09-23T23:42:52.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.751 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.617023+0000", 2023-09-23T23:42:52.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.752 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.617021+0000", 2023-09-23T23:42:52.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:42:52.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.752 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.617028+0000", 2023-09-23T23:42:52.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2810000000000003e-06 2023-09-23T23:42:52.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.617042+0000", 2023-09-23T23:42:52.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4185999999999999e-05 2023-09-23T23:42:52.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.617049+0000", 2023-09-23T23:42:52.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6200000000000001e-06 2023-09-23T23:42:52.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.617092+0000", 2023-09-23T23:42:52.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3242999999999999e-05 2023-09-23T23:42:52.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:42:52.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.617227+0000", 2023-09-23T23:42:52.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013476499999999999 2023-09-23T23:42:52.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:42:52.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.617403+0000", 2023-09-23T23:42:52.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017603599999999999 2023-09-23T23:42:52.755 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:42:52.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.620583+0000", 2023-09-23T23:42:52.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031794459999999998 2023-09-23T23:42:52.755 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.623541+0000", 2023-09-23T23:42:52.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029586130000000001 2023-09-23T23:42:52.756 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.756 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.756 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.756 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.747937+0000", 2023-09-23T23:42:52.756 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.124395831 2023-09-23T23:42:52.756 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.756 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.756 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:42:52.756 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.747951+0000", 2023-09-23T23:42:52.756 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3511e-05 2023-09-23T23:42:52.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.757 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.757 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.916287+0000", 2023-09-23T23:42:52.757 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1683368350000001 2023-09-23T23:42:52.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.757 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-09-23T23:42:52.757 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.916409+0000", 2023-09-23T23:42:52.757 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012187400000000001 2023-09-23T23:42:52.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.925964+0000", 2023-09-23T23:42:52.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0095546439999999993 2023-09-23T23:42:52.758 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.926034+0000", 2023-09-23T23:42:52.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0554000000000002e-05 2023-09-23T23:42:52.758 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.758 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.758 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.759 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.759 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.759 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:89716 13.5 13:aea1ffc2:::benchmark_data_smithi110_541928_object1936:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-23T23:42:52.759 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:34.639817+0000", 2023-09-23T23:42:52.759 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 498.07832521799997, 2023-09-23T23:42:52.759 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.311516924, 2023-09-23T23:42:52.759 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.759 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:42:52.759 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.759 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:42:52.759 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:52.759 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 89716 2023-09-23T23:42:52.760 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.760 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.760 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.760 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.760 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.639817+0000", 2023-09-23T23:42:52.760 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.760 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.760 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.760 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.760 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.639817+0000", 2023-09-23T23:42:52.760 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.761 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.761 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.639817+0000", 2023-09-23T23:42:52.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:52.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.761 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.761 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.639819+0000", 2023-09-23T23:42:52.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9209999999999999e-06 2023-09-23T23:42:52.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.762 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.762 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.639822+0000", 2023-09-23T23:42:52.762 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4359999999999998e-06 2023-09-23T23:42:52.762 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.762 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.762 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.762 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.639825+0000", 2023-09-23T23:42:52.762 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.503e-06 2023-09-23T23:42:52.762 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.762 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.762 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.762 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.639863+0000", 2023-09-23T23:42:52.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8176000000000002e-05 2023-09-23T23:42:52.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.763 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.763 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:42:52.763 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.639966+0000", 2023-09-23T23:42:52.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000102769 2023-09-23T23:42:52.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.763 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.763 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:42:52.763 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.640064+0000", 2023-09-23T23:42:52.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8486999999999994e-05 2023-09-23T23:42:52.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.764 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.764 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:42:52.764 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.642750+0000", 2023-09-23T23:42:52.764 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0026851449999999999 2023-09-23T23:42:52.764 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.764 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.764 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.764 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.645468+0000", 2023-09-23T23:42:52.764 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0027181810000000001 2023-09-23T23:42:52.764 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.764 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.764 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.765 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.748427+0000", 2023-09-23T23:42:52.765 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.10295940200000001 2023-09-23T23:42:52.765 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.765 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.765 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:42:52.765 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.748435+0000", 2023-09-23T23:42:52.765 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9529999999999999e-06 2023-09-23T23:42:52.765 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.765 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.765 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.765 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.918623+0000", 2023-09-23T23:42:52.765 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1701884220000001 2023-09-23T23:42:52.766 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.766 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.766 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-09-23T23:42:52.766 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.918708+0000", 2023-09-23T23:42:52.766 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4462999999999994e-05 2023-09-23T23:42:52.766 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.766 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.766 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.766 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.951261+0000", 2023-09-23T23:42:52.766 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.032553460999999999 2023-09-23T23:42:52.766 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.767 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.767 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.767 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.951334+0000", 2023-09-23T23:42:52.767 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2857999999999999e-05 2023-09-23T23:42:52.767 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.767 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.767 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.767 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.767 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.767 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:89721 13.5 13:af9caa70:::benchmark_data_smithi110_541928_object1941:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e569)", 2023-09-23T23:42:52.767 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:34.659711+0000", 2023-09-23T23:42:52.767 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 498.058431461, 2023-09-23T23:42:52.768 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3024725930000001, 2023-09-23T23:42:52.768 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.768 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:42:52.768 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.768 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:42:52.768 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:52.768 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 89721 2023-09-23T23:42:52.768 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.768 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.768 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.768 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.768 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.659711+0000", 2023-09-23T23:42:52.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.769 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.769 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.769 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.769 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.659711+0000", 2023-09-23T23:42:52.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.769 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.769 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.769 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.769 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.659710+0000", 2023-09-23T23:42:52.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:52.769 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.770 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.770 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.770 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.659717+0000", 2023-09-23T23:42:52.770 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.396e-06 2023-09-23T23:42:52.770 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.770 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.770 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.770 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.659732+0000", 2023-09-23T23:42:52.770 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4902000000000001e-05 2023-09-23T23:42:52.770 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.770 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.770 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.771 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.659738+0000", 2023-09-23T23:42:52.771 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0379999999999996e-06 2023-09-23T23:42:52.771 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.771 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.771 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.771 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.659792+0000", 2023-09-23T23:42:52.771 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3813999999999997e-05 2023-09-23T23:42:52.771 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.772 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.772 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:42:52.772 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.659950+0000", 2023-09-23T23:42:52.772 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015842800000000001 2023-09-23T23:42:52.772 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.772 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.772 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:42:52.772 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.660109+0000", 2023-09-23T23:42:52.772 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015913200000000001 2023-09-23T23:42:52.772 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.772 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.773 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:42:52.773 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.663293+0000", 2023-09-23T23:42:52.773 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031838690000000002 2023-09-23T23:42:52.773 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.773 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.773 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.773 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.665690+0000", 2023-09-23T23:42:52.773 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023967530000000002 2023-09-23T23:42:52.773 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.773 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.773 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.773 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.748455+0000", 2023-09-23T23:42:52.774 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.082764669999999999 2023-09-23T23:42:52.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.774 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:42:52.774 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.748461+0000", 2023-09-23T23:42:52.774 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2740000000000004e-06 2023-09-23T23:42:52.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.774 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.774 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.918737+0000", 2023-09-23T23:42:52.774 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1702763680000001 2023-09-23T23:42:52.775 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-09-23T23:42:52.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.918802+0000", 2023-09-23T23:42:52.775 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4813000000000004e-05 2023-09-23T23:42:52.775 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.962104+0000", 2023-09-23T23:42:52.775 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.043301740999999998 2023-09-23T23:42:52.775 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.776 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.776 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.962184+0000", 2023-09-23T23:42:52.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9864999999999995e-05 2023-09-23T23:42:52.776 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.776 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.776 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.776 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.776 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:89739 13.5 13:a1d796e6:::benchmark_data_smithi110_541928_object1959:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:42:52.776 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:34.808515+0000", 2023-09-23T23:42:52.776 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 497.90962740499998, 2023-09-23T23:42:52.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.145114076, 2023-09-23T23:42:52.777 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.777 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:52.777 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.777 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:42:52.777 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:52.777 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 89739 2023-09-23T23:42:52.777 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.777 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.777 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.808515+0000", 2023-09-23T23:42:52.777 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.777 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.778 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.778 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.778 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.808515+0000", 2023-09-23T23:42:52.778 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.778 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.778 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.778 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.778 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.808513+0000", 2023-09-23T23:42:52.778 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:42:52.778 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.778 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.779 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.779 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.808522+0000", 2023-09-23T23:42:52.779 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3510000000000002e-06 2023-09-23T23:42:52.779 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.779 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.779 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.779 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.808539+0000", 2023-09-23T23:42:52.779 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.732e-05 2023-09-23T23:42:52.779 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.779 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.779 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.779 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.808546+0000", 2023-09-23T23:42:52.779 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0249999999999997e-06 2023-09-23T23:42:52.780 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.780 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.780 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.780 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.808615+0000", 2023-09-23T23:42:52.780 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8652000000000006e-05 2023-09-23T23:42:52.780 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.780 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.780 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:42:52.780 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.808629+0000", 2023-09-23T23:42:52.780 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3957e-05 2023-09-23T23:42:52.780 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.781 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.781 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.781 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.918830+0000", 2023-09-23T23:42:52.781 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.110201583 2023-09-23T23:42:52.781 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.781 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.781 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:42:52.781 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.918919+0000", 2023-09-23T23:42:52.781 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8877999999999998e-05 2023-09-23T23:42:52.781 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.781 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.781 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:42:52.782 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.919025+0000", 2023-09-23T23:42:52.782 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105757 2023-09-23T23:42:52.782 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.782 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.782 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.782 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.926094+0000", 2023-09-23T23:42:52.782 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.007069517 2023-09-23T23:42:52.782 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.782 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.782 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:42:52.782 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.929415+0000", 2023-09-23T23:42:52.783 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0033203289999999999 2023-09-23T23:42:52.783 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.783 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.783 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.783 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.951370+0000", 2023-09-23T23:42:52.783 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.021955585 2023-09-23T23:42:52.783 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.783 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.783 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:42:52.783 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.951409+0000", 2023-09-23T23:42:52.783 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8637999999999998e-05 2023-09-23T23:42:52.783 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.784 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.784 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.784 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.953629+0000", 2023-09-23T23:42:52.784 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022203639999999998 2023-09-23T23:42:52.784 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.784 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.784 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.784 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.784 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.784 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:89744 13.5 13:a76fde0b:::benchmark_data_smithi110_541928_object1964:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:42:52.784 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:34.893405+0000", 2023-09-23T23:42:52.785 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 497.82473730800001, 2023-09-23T23:42:52.785 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0726365330000001, 2023-09-23T23:42:52.785 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.785 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:52.785 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.785 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:42:52.785 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:52.785 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 89744 2023-09-23T23:42:52.785 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.785 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.785 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.785 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.785 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.893405+0000", 2023-09-23T23:42:52.786 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.786 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.786 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.786 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.786 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.893405+0000", 2023-09-23T23:42:52.786 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.786 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.786 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.786 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.786 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.893404+0000", 2023-09-23T23:42:52.786 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:42:52.786 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.787 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.787 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.787 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.893410+0000", 2023-09-23T23:42:52.787 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2900000000000002e-06 2023-09-23T23:42:52.787 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.787 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.787 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.787 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.893423+0000", 2023-09-23T23:42:52.787 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3210999999999999e-05 2023-09-23T23:42:52.787 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.787 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.787 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.788 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.893428+0000", 2023-09-23T23:42:52.788 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1850000000000003e-06 2023-09-23T23:42:52.788 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.788 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.788 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.788 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.893491+0000", 2023-09-23T23:42:52.788 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.313e-05 2023-09-23T23:42:52.788 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.788 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.788 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:42:52.788 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.893512+0000", 2023-09-23T23:42:52.788 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0418999999999999e-05 2023-09-23T23:42:52.789 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.789 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.789 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.789 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.926122+0000", 2023-09-23T23:42:52.789 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0326101860000001 2023-09-23T23:42:52.789 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.789 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.789 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:42:52.789 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.926229+0000", 2023-09-23T23:42:52.789 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107493 2023-09-23T23:42:52.789 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.789 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.789 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:42:52.790 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.926344+0000", 2023-09-23T23:42:52.790 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000115226 2023-09-23T23:42:52.790 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.790 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.790 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:42:52.790 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.936873+0000", 2023-09-23T23:42:52.790 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010528871 2023-09-23T23:42:52.790 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.790 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.790 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.790 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.953700+0000", 2023-09-23T23:42:52.790 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016826563999999999 2023-09-23T23:42:52.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.791 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.963121+0000", 2023-09-23T23:42:52.791 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0094214469999999995 2023-09-23T23:42:52.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.791 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:42:52.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.963167+0000", 2023-09-23T23:42:52.791 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5417000000000002e-05 2023-09-23T23:42:52.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.792 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.792 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.966042+0000", 2023-09-23T23:42:52.792 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028750920000000001 2023-09-23T23:42:52.792 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.792 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.792 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.792 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.792 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.793 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:89745 13.5 13:a95b12a2:::benchmark_data_smithi110_541928_object1965:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:42:52.793 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:34.910244+0000", 2023-09-23T23:42:52.793 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 497.80789811099999, 2023-09-23T23:42:52.793 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0596367, 2023-09-23T23:42:52.793 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.793 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:52.793 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.793 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:42:52.793 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:52.793 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 89745 2023-09-23T23:42:52.794 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.794 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.794 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.794 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.794 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.910244+0000", 2023-09-23T23:42:52.794 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.794 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.794 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.794 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.794 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.910244+0000", 2023-09-23T23:42:52.794 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.794 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.795 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.795 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.795 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.910243+0000", 2023-09-23T23:42:52.795 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:52.795 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.795 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.795 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.795 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.910253+0000", 2023-09-23T23:42:52.795 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0363999999999999e-05 2023-09-23T23:42:52.795 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.795 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.796 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.796 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.910269+0000", 2023-09-23T23:42:52.796 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5281000000000002e-05 2023-09-23T23:42:52.796 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.796 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.796 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.796 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.910275+0000", 2023-09-23T23:42:52.796 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4370000000000004e-06 2023-09-23T23:42:52.796 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.797 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.797 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.797 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.910330+0000", 2023-09-23T23:42:52.797 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4332000000000001e-05 2023-09-23T23:42:52.797 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.797 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.797 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:42:52.797 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.910338+0000", 2023-09-23T23:42:52.797 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4500000000000004e-06 2023-09-23T23:42:52.797 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.797 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.797 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.798 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.929446+0000", 2023-09-23T23:42:52.798 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0191080859999999 2023-09-23T23:42:52.798 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.798 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.798 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:42:52.798 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.929601+0000", 2023-09-23T23:42:52.798 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000155131 2023-09-23T23:42:52.798 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.798 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.798 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:42:52.798 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.929762+0000", 2023-09-23T23:42:52.799 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016065400000000001 2023-09-23T23:42:52.799 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.799 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.799 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:42:52.799 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.939508+0000", 2023-09-23T23:42:52.799 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0097459529999999999 2023-09-23T23:42:52.799 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.799 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.799 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.799 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.953727+0000", 2023-09-23T23:42:52.799 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014219341 2023-09-23T23:42:52.799 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.800 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.800 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.800 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.966118+0000", 2023-09-23T23:42:52.800 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012391312999999999 2023-09-23T23:42:52.800 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.800 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.800 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:42:52.800 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.966390+0000", 2023-09-23T23:42:52.800 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00027137400000000001 2023-09-23T23:42:52.800 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.800 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.801 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.801 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.969881+0000", 2023-09-23T23:42:52.801 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034913560000000001 2023-09-23T23:42:52.801 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.801 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.801 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.801 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.801 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.801 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:89749 13.5 13:a310cdec:::benchmark_data_smithi110_541928_object1969:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:42:52.801 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:34.965229+0000", 2023-09-23T23:42:52.801 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 497.75291334299999, 2023-09-23T23:42:52.801 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0084807309999999, 2023-09-23T23:42:52.802 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.802 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:52.802 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.802 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:42:52.802 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:52.802 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 89749 2023-09-23T23:42:52.802 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.802 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.802 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.802 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.802 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.965229+0000", 2023-09-23T23:42:52.803 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.803 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.803 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.803 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.803 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.965229+0000", 2023-09-23T23:42:52.803 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.803 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.803 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.803 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.803 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.965228+0000", 2023-09-23T23:42:52.803 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:52.803 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.804 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.804 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.804 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.965235+0000", 2023-09-23T23:42:52.804 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6410000000000002e-06 2023-09-23T23:42:52.804 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.804 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.804 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.804 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.965252+0000", 2023-09-23T23:42:52.804 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6413e-05 2023-09-23T23:42:52.804 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.804 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.804 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.805 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.965258+0000", 2023-09-23T23:42:52.805 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5969999999999997e-06 2023-09-23T23:42:52.805 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.805 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.805 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.805 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.965334+0000", 2023-09-23T23:42:52.805 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5895000000000004e-05 2023-09-23T23:42:52.805 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.805 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.805 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:42:52.805 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.965350+0000", 2023-09-23T23:42:52.805 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5293000000000001e-05 2023-09-23T23:42:52.806 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.806 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.806 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.806 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.933637+0000", 2023-09-23T23:42:52.806 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96828758000000004 2023-09-23T23:42:52.806 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.806 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.806 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:42:52.806 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.933744+0000", 2023-09-23T23:42:52.806 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107054 2023-09-23T23:42:52.806 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.806 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.807 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:42:52.807 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.933858+0000", 2023-09-23T23:42:52.807 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000113535 2023-09-23T23:42:52.807 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.807 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.807 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:42:52.807 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.951233+0000", 2023-09-23T23:42:52.807 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017375096 2023-09-23T23:42:52.807 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.807 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.807 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.807 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.953760+0000", 2023-09-23T23:42:52.808 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025267269999999999 2023-09-23T23:42:52.808 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.808 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.808 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.808 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.969953+0000", 2023-09-23T23:42:52.808 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016193056000000001 2023-09-23T23:42:52.808 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.808 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.808 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:42:52.808 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.969994+0000", 2023-09-23T23:42:52.808 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1465999999999999e-05 2023-09-23T23:42:52.808 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.808 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.809 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.809 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.973710+0000", 2023-09-23T23:42:52.809 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037158709999999999 2023-09-23T23:42:52.809 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.809 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.809 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.809 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.809 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.809 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:89754 13.5 13:a28fe27c:::benchmark_data_smithi110_541928_object1974:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:42:52.809 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:35.047672+0000", 2023-09-23T23:42:52.809 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 497.67047013899997, 2023-09-23T23:42:52.809 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92934757800000001, 2023-09-23T23:42:52.810 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.810 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:52.810 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.810 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:42:52.810 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:52.810 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 89754 2023-09-23T23:42:52.810 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.810 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.810 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.810 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.810 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.047672+0000", 2023-09-23T23:42:52.811 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.811 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.811 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.811 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.811 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.047672+0000", 2023-09-23T23:42:52.811 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.811 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.811 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.811 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.811 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.047671+0000", 2023-09-23T23:42:52.811 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:42:52.811 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.811 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.812 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.812 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.047679+0000", 2023-09-23T23:42:52.812 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5730000000000008e-06 2023-09-23T23:42:52.812 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.812 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.812 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.812 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.047698+0000", 2023-09-23T23:42:52.812 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8508999999999998e-05 2023-09-23T23:42:52.812 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.812 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.812 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.812 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.047708+0000", 2023-09-23T23:42:52.813 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0369e-05 2023-09-23T23:42:52.813 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.813 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.813 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.813 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.047763+0000", 2023-09-23T23:42:52.813 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4969999999999997e-05 2023-09-23T23:42:52.813 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.813 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.813 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:42:52.813 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.047776+0000", 2023-09-23T23:42:52.813 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.29e-05 2023-09-23T23:42:52.813 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.814 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.814 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.814 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.936908+0000", 2023-09-23T23:42:52.814 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88913176000000005 2023-09-23T23:42:52.814 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.814 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.814 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:42:52.814 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.937081+0000", 2023-09-23T23:42:52.814 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017318999999999999 2023-09-23T23:42:52.814 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.814 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.814 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:42:52.815 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.937234+0000", 2023-09-23T23:42:52.815 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015310000000000001 2023-09-23T23:42:52.815 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.815 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.815 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:42:52.815 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.953643+0000", 2023-09-23T23:42:52.815 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016409353000000002 2023-09-23T23:42:52.815 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.815 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.815 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.815 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.953793+0000", 2023-09-23T23:42:52.815 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014953000000000001 2023-09-23T23:42:52.816 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.816 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.816 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.816 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.973759+0000", 2023-09-23T23:42:52.816 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019966133 2023-09-23T23:42:52.816 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.816 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.816 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:42:52.816 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.973801+0000", 2023-09-23T23:42:52.816 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1754e-05 2023-09-23T23:42:52.816 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.817 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.817 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.817 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.977020+0000", 2023-09-23T23:42:52.817 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032191400000000001 2023-09-23T23:42:52.817 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.817 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.817 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.817 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.817 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.817 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:95946 13.6 13:64dac656:::benchmark_data_smithi110_541928_object8166:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e579)", 2023-09-23T23:42:52.817 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:57.359617+0000", 2023-09-23T23:42:52.817 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 475.35852576500002, 2023-09-23T23:42:52.818 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98724013700000002, 2023-09-23T23:42:52.818 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.818 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:52.818 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.818 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:42:52.818 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:52.818 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95946 2023-09-23T23:42:52.818 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.818 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.818 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.818 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.818 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.359617+0000", 2023-09-23T23:42:52.819 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.819 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.819 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.819 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.819 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.359617+0000", 2023-09-23T23:42:52.819 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.819 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.819 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.819 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.819 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.359615+0000", 2023-09-23T23:42:52.819 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:52.819 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.820 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.820 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.820 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.359622+0000", 2023-09-23T23:42:52.820 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1049999999999997e-06 2023-09-23T23:42:52.820 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.820 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.820 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.820 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.359640+0000", 2023-09-23T23:42:52.820 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7756000000000001e-05 2023-09-23T23:42:52.820 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.820 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.820 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.821 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.359647+0000", 2023-09-23T23:42:52.821 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8600000000000004e-06 2023-09-23T23:42:52.821 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.821 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.821 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.821 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.359718+0000", 2023-09-23T23:42:52.821 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0860999999999998e-05 2023-09-23T23:42:52.821 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.821 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.821 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:42:52.821 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.359733+0000", 2023-09-23T23:42:52.821 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5349000000000002e-05 2023-09-23T23:42:52.822 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.822 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.822 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.822 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.338363+0000", 2023-09-23T23:42:52.822 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97862939999999998 2023-09-23T23:42:52.822 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.822 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.822 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:42:52.822 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.338460+0000", 2023-09-23T23:42:52.822 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6921000000000003e-05 2023-09-23T23:42:52.822 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.823 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:42:52.823 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.338540+0000", 2023-09-23T23:42:52.823 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9913999999999994e-05 2023-09-23T23:42:52.823 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.823 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.823 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.342614+0000", 2023-09-23T23:42:52.823 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040745290000000003 2023-09-23T23:42:52.823 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.823 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.824 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.342633+0000", 2023-09-23T23:42:52.824 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8618999999999999e-05 2023-09-23T23:42:52.824 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.824 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.824 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:42:52.824 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.346789+0000", 2023-09-23T23:42:52.824 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041560929999999996 2023-09-23T23:42:52.824 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.824 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.824 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:42:52.824 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.346827+0000", 2023-09-23T23:42:52.825 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8006000000000002e-05 2023-09-23T23:42:52.825 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.825 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.825 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.825 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.346857+0000", 2023-09-23T23:42:52.825 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0154e-05 2023-09-23T23:42:52.825 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.825 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.825 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.825 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.825 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.825 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:95960 13.6 13:605ca08f:::benchmark_data_smithi110_541928_object8180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e579)", 2023-09-23T23:42:52.826 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:57.442651+0000", 2023-09-23T23:42:52.826 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 475.27549110400003, 2023-09-23T23:42:52.826 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.90870927599999995, 2023-09-23T23:42:52.826 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.826 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:52.826 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.826 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:42:52.826 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:52.826 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95960 2023-09-23T23:42:52.826 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.826 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.827 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.827 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.827 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.442651+0000", 2023-09-23T23:42:52.827 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.827 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.827 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.827 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.827 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.442651+0000", 2023-09-23T23:42:52.827 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.827 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.827 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.827 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.828 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.442650+0000", 2023-09-23T23:42:52.828 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:52.828 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.828 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.828 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.828 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.442657+0000", 2023-09-23T23:42:52.828 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2540000000000003e-06 2023-09-23T23:42:52.828 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.828 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.828 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.828 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.442671+0000", 2023-09-23T23:42:52.829 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3985000000000001e-05 2023-09-23T23:42:52.829 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.829 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.829 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.442678+0000", 2023-09-23T23:42:52.829 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5749999999999997e-06 2023-09-23T23:42:52.829 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.829 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.829 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.442740+0000", 2023-09-23T23:42:52.829 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1989000000000006e-05 2023-09-23T23:42:52.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.830 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.830 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:42:52.830 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.442750+0000", 2023-09-23T23:42:52.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0499000000000001e-05 2023-09-23T23:42:52.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.830 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.830 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.830 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.341144+0000", 2023-09-23T23:42:52.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.89839374900000002 2023-09-23T23:42:52.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.830 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.831 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:42:52.831 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.341231+0000", 2023-09-23T23:42:52.831 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7334000000000001e-05 2023-09-23T23:42:52.831 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.831 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.831 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:42:52.831 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.341295+0000", 2023-09-23T23:42:52.831 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3958e-05 2023-09-23T23:42:52.831 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.831 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.831 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.831 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.348387+0000", 2023-09-23T23:42:52.832 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0070911200000000002 2023-09-23T23:42:52.832 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.832 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.832 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:42:52.832 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.348458+0000", 2023-09-23T23:42:52.832 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1619000000000007e-05 2023-09-23T23:42:52.832 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.832 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.832 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.832 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.351293+0000", 2023-09-23T23:42:52.832 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028346270000000002 2023-09-23T23:42:52.833 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.833 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.833 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:42:52.833 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.351325+0000", 2023-09-23T23:42:52.833 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2329999999999997e-05 2023-09-23T23:42:52.833 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.833 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.833 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.833 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.351361+0000", 2023-09-23T23:42:52.833 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5657000000000001e-05 2023-09-23T23:42:52.833 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.833 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.834 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.834 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.834 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.834 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:153444 13.e 13:7c68890f:::benchmark_data_smithi110_541928_object65664:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-23T23:42:52.834 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:38:16.592187+0000", 2023-09-23T23:42:52.834 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 276.12595603099999, 2023-09-23T23:42:52.834 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.29196677500000001, 2023-09-23T23:42:52.834 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.834 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:52.834 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.834 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:42:52.834 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:52.835 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 153444 2023-09-23T23:42:52.835 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.835 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.835 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.835 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.835 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592187+0000", 2023-09-23T23:42:52.835 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.835 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.835 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.835 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.835 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592187+0000", 2023-09-23T23:42:52.835 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.836 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.836 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.836 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.836 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592185+0000", 2023-09-23T23:42:52.836 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:52.836 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.836 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.836 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.836 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592192+0000", 2023-09-23T23:42:52.836 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5209999999999997e-06 2023-09-23T23:42:52.836 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.836 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.837 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.837 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592208+0000", 2023-09-23T23:42:52.837 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5746000000000001e-05 2023-09-23T23:42:52.837 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.837 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.837 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.837 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592215+0000", 2023-09-23T23:42:52.837 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5490000000000003e-06 2023-09-23T23:42:52.837 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.837 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.837 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.837 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592370+0000", 2023-09-23T23:42:52.838 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015496499999999999 2023-09-23T23:42:52.838 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.838 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.838 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:42:52.838 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592515+0000", 2023-09-23T23:42:52.838 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000145771 2023-09-23T23:42:52.838 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.838 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.838 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:42:52.838 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592664+0000", 2023-09-23T23:42:52.838 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014883999999999999 2023-09-23T23:42:52.838 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.839 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.839 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:42:52.839 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.602612+0000", 2023-09-23T23:42:52.839 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0099480739999999995 2023-09-23T23:42:52.839 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.839 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.839 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.839 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.602670+0000", 2023-09-23T23:42:52.839 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7840000000000002e-05 2023-09-23T23:42:52.839 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.839 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.839 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.840 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.882423+0000", 2023-09-23T23:42:52.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.27975320599999998 2023-09-23T23:42:52.840 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.840 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:42:52.840 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.882477+0000", 2023-09-23T23:42:52.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3607999999999999e-05 2023-09-23T23:42:52.840 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.840 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.840 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.884153+0000", 2023-09-23T23:42:52.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001676262 2023-09-23T23:42:52.841 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.841 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.841 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.841 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.841 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.841 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:153449 13.e 13:7decf6a5:::benchmark_data_smithi110_541928_object65669:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-23T23:42:52.841 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:38:16.597143+0000", 2023-09-23T23:42:52.841 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 276.12100002, 2023-09-23T23:42:52.841 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.28863325099999998, 2023-09-23T23:42:52.841 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.841 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:52.841 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.842 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:42:52.842 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:42:52.842 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 153449 2023-09-23T23:42:52.842 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.842 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.842 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.842 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.842 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.597143+0000", 2023-09-23T23:42:52.842 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.842 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.842 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.842 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.597143+0000", 2023-09-23T23:42:52.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.843 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.597141+0000", 2023-09-23T23:42:52.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:42:52.843 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.597148+0000", 2023-09-23T23:42:52.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5070000000000003e-06 2023-09-23T23:42:52.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.844 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.844 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.844 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.597161+0000", 2023-09-23T23:42:52.844 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2843000000000001e-05 2023-09-23T23:42:52.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.844 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.844 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.844 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.597167+0000", 2023-09-23T23:42:52.844 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2140000000000001e-06 2023-09-23T23:42:52.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.844 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.599181+0000", 2023-09-23T23:42:52.845 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002014007 2023-09-23T23:42:52.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:42:52.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.599369+0000", 2023-09-23T23:42:52.845 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018732800000000001 2023-09-23T23:42:52.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:42:52.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.599530+0000", 2023-09-23T23:42:52.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000161461 2023-09-23T23:42:52.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.605636+0000", 2023-09-23T23:42:52.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0061053599999999998 2023-09-23T23:42:52.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:42:52.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.605643+0000", 2023-09-23T23:42:52.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2609999999999996e-06 2023-09-23T23:42:52.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.847 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.847 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.884203+0000", 2023-09-23T23:42:52.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.27855972499999998 2023-09-23T23:42:52.847 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.847 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:42:52.847 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.884253+0000", 2023-09-23T23:42:52.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0427999999999999e-05 2023-09-23T23:42:52.847 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.847 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.848 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.885776+0000", 2023-09-23T23:42:52.848 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001522816 2023-09-23T23:42:52.848 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.848 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.848 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.848 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.848 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.848 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.47186.0:36258 14.7 14:e0ceb5cd:::benchmark_data_smithi110_640334_object36257:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T23:42:52.848 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:39:44.854122+0000", 2023-09-23T23:42:52.848 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 187.86402020200001, 2023-09-23T23:42:52.848 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.36212670400000002, 2023-09-23T23:42:52.848 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.849 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:52.849 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.849 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.47186", 2023-09-23T23:42:52.849 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:42:52.849 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 36258 2023-09-23T23:42:52.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.849 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.849 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.849 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.849 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854122+0000", 2023-09-23T23:42:52.849 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.850 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.850 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.850 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854122+0000", 2023-09-23T23:42:52.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.850 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.850 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.850 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.850 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854121+0000", 2023-09-23T23:42:52.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:52.850 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.850 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.850 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854177+0000", 2023-09-23T23:42:52.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5659999999999999e-05 2023-09-23T23:42:52.851 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.851 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854189+0000", 2023-09-23T23:42:52.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2468e-05 2023-09-23T23:42:52.851 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.851 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854199+0000", 2023-09-23T23:42:52.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8570000000000007e-06 2023-09-23T23:42:52.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.852 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854258+0000", 2023-09-23T23:42:52.852 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9049000000000003e-05 2023-09-23T23:42:52.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:42:52.852 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854381+0000", 2023-09-23T23:42:52.852 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000122373 2023-09-23T23:42:52.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.853 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:42:52.853 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854530+0000", 2023-09-23T23:42:52.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014989699999999999 2023-09-23T23:42:52.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.853 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:42:52.853 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.855861+0000", 2023-09-23T23:42:52.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0013305630000000001 2023-09-23T23:42:52.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.853 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.856075+0000", 2023-09-23T23:42:52.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000214195 2023-09-23T23:42:52.854 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.854 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:45.216123+0000", 2023-09-23T23:42:52.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.360047426 2023-09-23T23:42:52.854 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.854 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:42:52.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:45.216200+0000", 2023-09-23T23:42:52.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6913e-05 2023-09-23T23:42:52.855 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.855 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.855 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.855 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:45.216249+0000", 2023-09-23T23:42:52.855 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9527999999999997e-05 2023-09-23T23:42:52.855 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.855 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.855 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.855 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.855 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.855 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.47186.0:37504 14.7 14:e20c8f2d:::benchmark_data_smithi110_640334_object37503:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T23:42:52.855 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:39:46.194924+0000", 2023-09-23T23:42:52.856 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 186.52321896800001, 2023-09-23T23:42:52.856 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22139715600000001, 2023-09-23T23:42:52.856 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.856 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:52.856 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.856 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.47186", 2023-09-23T23:42:52.856 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:42:52.856 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 37504 2023-09-23T23:42:52.856 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.856 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.856 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.194924+0000", 2023-09-23T23:42:52.857 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.857 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.194924+0000", 2023-09-23T23:42:52.857 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.857 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.194923+0000", 2023-09-23T23:42:52.858 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:42:52.858 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.858 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.858 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.858 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.194980+0000", 2023-09-23T23:42:52.858 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6762999999999998e-05 2023-09-23T23:42:52.858 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.858 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.858 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411523+0000", 2023-09-23T23:42:52.859 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21654374000000001 2023-09-23T23:42:52.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.859 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411534+0000", 2023-09-23T23:42:52.859 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0794e-05 2023-09-23T23:42:52.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.859 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411576+0000", 2023-09-23T23:42:52.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1700999999999999e-05 2023-09-23T23:42:52.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:42:52.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411702+0000", 2023-09-23T23:42:52.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000126239 2023-09-23T23:42:52.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:42:52.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411827+0000", 2023-09-23T23:42:52.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000124991 2023-09-23T23:42:52.861 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.861 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.861 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:42:52.861 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.415986+0000", 2023-09-23T23:42:52.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041592870000000002 2023-09-23T23:42:52.861 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.861 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.861 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.861 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.416043+0000", 2023-09-23T23:42:52.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6314000000000003e-05 2023-09-23T23:42:52.861 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.861 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.862 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.862 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.416222+0000", 2023-09-23T23:42:52.862 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017949799999999999 2023-09-23T23:42:52.862 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.862 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.862 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:42:52.862 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.416279+0000", 2023-09-23T23:42:52.862 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6972000000000003e-05 2023-09-23T23:42:52.862 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.862 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.862 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.863 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.416321+0000", 2023-09-23T23:42:52.863 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1499999999999999e-05 2023-09-23T23:42:52.863 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.863 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.863 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.863 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.863 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.863 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.47186.0:37498 14.a e625/617 14:56947ea7:::benchmark_data_smithi110_640334_object37497:head v 625'2249, mlcod=625'2247)", 2023-09-23T23:42:52.863 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:39:46.196021+0000", 2023-09-23T23:42:52.863 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 186.522121607, 2023-09-23T23:42:52.863 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21694238800000001, 2023-09-23T23:42:52.864 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.864 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:52.864 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.864 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.864 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.864 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.196021+0000", 2023-09-23T23:42:52.864 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.864 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.864 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.864 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.864 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.196021+0000", 2023-09-23T23:42:52.864 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.865 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.865 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.865 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.196020+0000", 2023-09-23T23:42:52.865 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:52.865 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.865 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.865 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.196065+0000", 2023-09-23T23:42:52.865 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4861999999999998e-05 2023-09-23T23:42:52.865 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.866 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.866 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411523+0000", 2023-09-23T23:42:52.866 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21545887599999999 2023-09-23T23:42:52.866 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.866 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.866 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.866 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411533+0000", 2023-09-23T23:42:52.866 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5980000000000003e-06 2023-09-23T23:42:52.866 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.866 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.866 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.867 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411583+0000", 2023-09-23T23:42:52.867 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9599999999999999e-05 2023-09-23T23:42:52.867 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.867 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:42:52.867 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411604+0000", 2023-09-23T23:42:52.867 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1741000000000001e-05 2023-09-23T23:42:52.867 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.867 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:42:52.867 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.412924+0000", 2023-09-23T23:42:52.867 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0013196600000000001 2023-09-23T23:42:52.868 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.868 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.868 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.868 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.412963+0000", 2023-09-23T23:42:52.868 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9288000000000001e-05 2023-09-23T23:42:52.868 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.868 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.868 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.868 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.868 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.868 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.47186.0:37512 14.0 e625/617 14:07e6ea4d:::benchmark_data_smithi110_640334_object37511:head v 625'2372, mlcod=625'2370)", 2023-09-23T23:42:52.868 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:39:46.208752+0000", 2023-09-23T23:42:52.869 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 186.50939019399999, 2023-09-23T23:42:52.869 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20513990300000001, 2023-09-23T23:42:52.869 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.869 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:52.869 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.869 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.869 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.869 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.208752+0000", 2023-09-23T23:42:52.869 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.869 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.869 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.869 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.869 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.208752+0000", 2023-09-23T23:42:52.870 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.870 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.870 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.870 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.870 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.208751+0000", 2023-09-23T23:42:52.870 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:42:52.870 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.870 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.870 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.870 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.208799+0000", 2023-09-23T23:42:52.870 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8372999999999999e-05 2023-09-23T23:42:52.871 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.871 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.871 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411548+0000", 2023-09-23T23:42:52.871 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20274924699999999 2023-09-23T23:42:52.871 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.871 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.871 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411556+0000", 2023-09-23T23:42:52.871 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4730000000000003e-06 2023-09-23T23:42:52.871 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.873 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.873 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411600+0000", 2023-09-23T23:42:52.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.375e-05 2023-09-23T23:42:52.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.873 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:42:52.873 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411617+0000", 2023-09-23T23:42:52.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7363e-05 2023-09-23T23:42:52.874 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:42:52.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.413860+0000", 2023-09-23T23:42:52.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022428890000000001 2023-09-23T23:42:52.874 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.413892+0000", 2023-09-23T23:42:52.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2335000000000001e-05 2023-09-23T23:42:52.874 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.874 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.875 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.875 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.875 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.47186.0:131926 14.7 14:ecd0bdd0:::benchmark_data_smithi110_640334_object48047:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e665)", 2023-09-23T23:42:52.875 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:42:11.684341+0000", 2023-09-23T23:42:52.875 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 41.033801998999998, 2023-09-23T23:42:52.875 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.204595685, 2023-09-23T23:42:52.875 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:42:52.875 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:42:52.875 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:42:52.875 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.47186", 2023-09-23T23:42:52.876 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:42:52.876 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 131926 2023-09-23T23:42:52.876 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.876 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:42:52.876 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.876 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:42:52.876 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:11.684341+0000", 2023-09-23T23:42:52.876 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.876 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.876 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.876 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:42:52.876 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:11.684341+0000", 2023-09-23T23:42:52.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:42:52.877 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.877 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.877 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:42:52.877 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:11.684338+0000", 2023-09-23T23:42:52.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-09-23T23:42:52.877 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.877 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.877 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:42:52.877 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:11.684351+0000", 2023-09-23T23:42:52.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2676e-05 2023-09-23T23:42:52.877 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.878 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:42:52.878 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:11.684397+0000", 2023-09-23T23:42:52.878 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6285999999999997e-05 2023-09-23T23:42:52.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.878 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:42:52.878 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:11.684403+0000", 2023-09-23T23:42:52.878 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3310000000000003e-06 2023-09-23T23:42:52.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.878 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:42:52.879 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:11.684448+0000", 2023-09-23T23:42:52.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5148999999999998e-05 2023-09-23T23:42:52.879 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.879 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.879 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:42:52.879 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:11.684546+0000", 2023-09-23T23:42:52.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8406000000000006e-05 2023-09-23T23:42:52.879 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.879 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.879 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:42:52.879 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:11.684631+0000", 2023-09-23T23:42:52.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4851000000000005e-05 2023-09-23T23:42:52.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.880 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:42:52.880 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:11.686735+0000", 2023-09-23T23:42:52.880 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021039779999999998 2023-09-23T23:42:52.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.880 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.880 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:11.687109+0000", 2023-09-23T23:42:52.880 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00037349300000000002 2023-09-23T23:42:52.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:42:52.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:11.888839+0000", 2023-09-23T23:42:52.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20173020799999999 2023-09-23T23:42:52.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:42:52.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:11.888893+0000", 2023-09-23T23:42:52.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4311000000000002e-05 2023-09-23T23:42:52.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:42:52.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:42:52.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:42:52.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:11.888936+0000", 2023-09-23T23:42:52.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3189000000000003e-05 2023-09-23T23:42:52.882 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.882 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.882 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.882 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:42:52.882 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:42:52.882 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:42:52.883 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:52 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:42:52.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:52.883 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:52.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:52.984 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:42:53.014 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:53.056 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:42:53.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:53.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:42:53.161 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:53.167 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:42:53.189 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:53.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:52 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:42:53.208 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:52 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:42:53.441 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:53.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:53.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:53.454 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:42:53.549 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:42:53.643 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:53.755 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:42:53.786 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:42:53.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:53.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:42:53.886 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:42:53.892 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:53.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:53 smithi110 ceph-mon[221102]: pgmap v1927: 119 pgs: 119 active+clean; 743 MiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T23:42:53.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:53 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/859066041' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:42:53.949 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:53 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:42:53.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:53 smithi110 ceph-mon[212939]: pgmap v1927: 119 pgs: 119 active+clean; 743 MiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T23:42:53.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:53 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/859066041' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:42:53.950 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:53 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:42:53.988 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:54.087 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:53 smithi138 ceph-mon[193606]: pgmap v1927: 119 pgs: 119 active+clean; 743 MiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T23:42:54.087 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:53 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/859066041' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:42:54.087 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:53 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:42:54.133 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:42:54.133 INFO:teuthology.orchestra.run.smithi110.stdout:{"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2451,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":678,"num_osds":8,"num_up_osds":8,"osd_up_since":1695511414,"num_in_osds":6,"osd_in_since":1695512394,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":119}],"num_pgs":119,"num_pools":5,"num_objects":11904,"data_bytes":779098040,"bytes_used":8472227840,"bytes_avail":567497986048,"bytes_total":575970213888,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":604},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi110.fuglpi","status":"up:active","gid":24572}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.110:8443/","prometheus":"http://172.21.15.110:9283/"}},"servicemap":{"epoch":35,"modified":"2023-09-23T23:41:58.090092+0000","services":{}},"progress_events":{"833ef15d-07ae-4299-b4e4-2cf94e068147":{"message":"Upgrade to 18.2.0-460-gbb0daeff (10m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true},"a6961007-fb3a-4fa7-ac57-f1c904a2cf75":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true}}} 2023-09-23T23:42:54.148 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:54.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:54.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:54.260 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:42:54.287 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:54.381 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:42:54.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:54.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:42:54.484 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:54.559 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:54 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:42:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:54 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:42:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:54 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:42:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:54 smithi110 ceph-mon[212939]: osdmap e678: 8 total, 8 up, 6 in 2023-09-23T23:42:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:54 smithi110 ceph-mon[212939]: failed to encode map e678 with expected crc 2023-09-23T23:42:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:54 smithi110 ceph-mon[212939]: failed to encode map e678 with expected crc 2023-09-23T23:42:54.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:54 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4213436742' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:42:54.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:54 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:42:54.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:54 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:42:54.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:54 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:42:54.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:54 smithi110 ceph-mon[221102]: osdmap e678: 8 total, 8 up, 6 in 2023-09-23T23:42:54.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:54 smithi110 ceph-mon[221102]: failed to encode map e678 with expected crc 2023-09-23T23:42:54.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:54 smithi110 ceph-mon[221102]: failed to encode map e678 with expected crc 2023-09-23T23:42:54.957 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:54 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4213436742' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:42:55.154 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:54 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:42:55.155 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:54 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:42:55.155 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:54 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:42:55.155 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:54 smithi138 ceph-mon[193606]: osdmap e678: 8 total, 8 up, 6 in 2023-09-23T23:42:55.155 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:54 smithi138 ceph-mon[193606]: failed to encode map e678 with expected crc 2023-09-23T23:42:55.155 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:54 smithi138 ceph-mon[193606]: failed to encode map e678 with expected crc 2023-09-23T23:42:55.155 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:54 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4213436742' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:42:55.290 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:55.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:55.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:55.363 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:42:55.363 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:42:55.363 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-09-23T23:42:55.364 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:42:55.394 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:42:55.487 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:55.668 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:42:55.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:55.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:42:55.717 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-09-23T23:42:55.717 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:42:55.784 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:42:55.824 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:55.883 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:55.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:55 smithi138 ceph-mon[193606]: pgmap v1929: 119 pgs: 119 active+clean; 637 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T23:42:55.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:55 smithi110 ceph-mon[221102]: pgmap v1929: 119 pgs: 119 active+clean; 637 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T23:42:55.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:55 smithi110 ceph-mon[212939]: pgmap v1929: 119 pgs: 119 active+clean; 637 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T23:42:55.990 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:42:55.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:55.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:42:56.089 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:42:56.098 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:42:56.162 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:56.216 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:42:56.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:56.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:42:56.326 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:56.404 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:56.479 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:42:56.699 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:56.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:56.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:56.805 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:42:56.886 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:56.993 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:42:57.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:57.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:42:57.008 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:42:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:42:56] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.43.0" 2023-09-23T23:42:57.107 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:42:57.140 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:57.188 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:42:57.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:57.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:42:57.291 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:42:57.320 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:57.405 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:57.454 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:42:57.570 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:42:57.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:57.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:42:57.673 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:57.830 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:57.839 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:57 smithi138 ceph-mon[193606]: pgmap v1930: 119 pgs: 119 active+clean; 532 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 698 op/s 2023-09-23T23:42:57.978 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:57.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:57.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:57.990 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:57 smithi110 ceph-mon[212939]: pgmap v1930: 119 pgs: 119 active+clean; 532 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 698 op/s 2023-09-23T23:42:57.996 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:57 smithi110 ceph-mon[221102]: pgmap v1930: 119 pgs: 119 active+clean; 532 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 698 op/s 2023-09-23T23:42:58.049 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:42:58.049 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:42:58.049 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:42:58.049 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:42:58.049 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:42:58.091 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:42:58.198 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:58.298 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:42:58.345 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:42:58.373 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:42:58.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:58.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:42:58.383 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:42:58.407 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:42:58.426 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:42:58.480 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:42:58.498 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:42:58.515 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:58.537 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:42:58.578 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:42:58.590 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:42:58.590 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:42:58.590 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:42:58.591 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:42:58.591 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:42:58.591 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:42:58.591 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:42:58.591 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:42:58.800 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:42:58.809 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:42:58.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:58.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:42:58.920 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:42:58.951 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:59.004 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:42:59.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:59.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:42:59.073 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:59 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:59.073 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:59 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:42:59.074 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:59 smithi138 ceph-mon[193606]: pgmap v1931: 119 pgs: 119 active+clean; 430 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T23:42:59.074 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:42:59 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/209176933' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:59.119 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:42:59.149 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:59.203 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:42:59.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:59.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:42:59.234 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:42:59.310 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:42:59.374 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:59 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:59.374 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:59 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:42:59.374 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:59 smithi110 ceph-mon[221102]: pgmap v1931: 119 pgs: 119 active+clean; 430 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T23:42:59.374 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:42:59 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/209176933' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:59.374 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:59 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:42:59.374 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:59 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:42:59.375 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:59 smithi110 ceph-mon[212939]: pgmap v1931: 119 pgs: 119 active+clean; 430 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T23:42:59.375 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:42:59 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/209176933' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:42:59.377 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:42:59.824 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:42:59.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:42:59.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:42:59.939 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:42:59.951 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:42:59.988 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:00.148 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:00.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:00.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:00.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:00 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:00.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:00 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:00.260 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:00.347 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:00.411 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:00 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:00.474 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:00.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:00.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:00.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:00.706 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:00.868 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:43:00.897 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:43:00.931 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:43:00.932 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:43:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:01 smithi138 ceph-mon[193606]: pgmap v1932: 119 pgs: 119 active+clean; 416 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 677 op/s 2023-09-23T23:43:01.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:01 smithi110 ceph-mon[212939]: pgmap v1932: 119 pgs: 119 active+clean; 416 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 677 op/s 2023-09-23T23:43:01.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:01 smithi110 ceph-mon[221102]: pgmap v1932: 119 pgs: 119 active+clean; 416 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 677 op/s 2023-09-23T23:43:01.576 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:43:01.958 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:01.963 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:43:01.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:01.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:02.066 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:02.098 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:02.149 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:02.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:02.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:02.255 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:43:02.278 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:02 smithi110 ceph-mon[212939]: from='client.37329 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:43:02.279 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:02 smithi110 ceph-mon[221102]: from='client.37329 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:43:02.281 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:02.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:02 smithi138 ceph-mon[193606]: from='client.37329 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:43:02.607 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:43:02.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:02.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:43:02.712 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:02.731 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1932,"stamp":"2023-09-23T23:43:00.122642+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":436279224,"num_objects":6673,"num_object_clones":0,"num_object_copies":20019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6673,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":161166,"num_write_kb":5368094,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15009,"num_bytes_recovered":810955076,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92703,"ondisk_log_size":92703,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":7212884,"kb_used_data":1308324,"kb_used_omap":29,"kb_used_meta":5904034,"kb_avail":555258028,"statfs":{"total":575970213888,"available":568584220672,"internally_reserved":0,"allocated":1339723776,"data_stored":1322441354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":6045731539},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,3,4,7],"upper_bound":32},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-444399616,"num_objects":-6781,"num_object_clones":0,"num_object_copies":-20343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6781,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007897"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":874,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.625366+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:42:46.625366+0000","last_peered":"2023-09-23T23:42:46.625366+0000","last_clean":"2023-09-23T23:42:46.625366+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:42:46.625366+0000","last_undegraded":"2023-09-23T23:42:46.625366+0000","last_fullsized":"2023-09-23T23:42:46.625366+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":435,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.626257+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:42:46.626257+0000","last_peered":"2023-09-23T23:42:46.626257+0000","last_clean":"2023-09-23T23:42:46.626257+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:42:46.626257+0000","last_undegraded":"2023-09-23T23:42:46.626257+0000","last_fullsized":"2023-09-23T23:42:46.626257+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":671,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.623622+0000","last_change":"2023-09-23T23:39:55.102353+0000","last_active":"2023-09-23T23:42:46.623622+0000","last_peered":"2023-09-23T23:42:46.623622+0000","last_clean":"2023-09-23T23:42:46.623622+0000","last_became_active":"2023-09-23T23:39:55.102017+0000","last_became_peered":"2023-09-23T23:39:55.102017+0000","last_unstale":"2023-09-23T23:42:46.623622+0000","last_undegraded":"2023-09-23T23:42:46.623622+0000","last_fullsized":"2023-09-23T23:42:46.623622+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:48.870443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":827,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.624423+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:42:46.624423+0000","last_peered":"2023-09-23T23:42:46.624423+0000","last_clean":"2023-09-23T23:42:46.624423+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:42:46.624423+0000","last_undegraded":"2023-09-23T23:42:46.624423+0000","last_fullsized":"2023-09-23T23:42:46.624423+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":408,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622402+0000","last_change":"2023-09-23T23:39:55.104571+0000","last_active":"2023-09-23T23:42:46.622402+0000","last_peered":"2023-09-23T23:42:46.622402+0000","last_clean":"2023-09-23T23:42:46.622402+0000","last_became_active":"2023-09-23T23:39:55.104152+0000","last_became_peered":"2023-09-23T23:39:55.104152+0000","last_unstale":"2023-09-23T23:42:46.622402+0000","last_undegraded":"2023-09-23T23:42:46.622402+0000","last_fullsized":"2023-09-23T23:42:46.622402+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:31:52.551048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":642,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622098+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:42:46.622098+0000","last_peered":"2023-09-23T23:42:46.622098+0000","last_clean":"2023-09-23T23:42:46.622098+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:42:46.622098+0000","last_undegraded":"2023-09-23T23:42:46.622098+0000","last_fullsized":"2023-09-23T23:42:46.622098+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":839,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622251+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:42:46.622251+0000","last_peered":"2023-09-23T23:42:46.622251+0000","last_clean":"2023-09-23T23:42:46.622251+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:42:46.622251+0000","last_undegraded":"2023-09-23T23:42:46.622251+0000","last_fullsized":"2023-09-23T23:42:46.622251+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":726,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622264+0000","last_change":"2023-09-23T23:42:28.951264+0000","last_active":"2023-09-23T23:42:46.622264+0000","last_peered":"2023-09-23T23:42:46.622264+0000","last_clean":"2023-09-23T23:42:46.622264+0000","last_became_active":"2023-09-23T23:42:28.950966+0000","last_became_peered":"2023-09-23T23:42:28.950966+0000","last_unstale":"2023-09-23T23:42:46.622264+0000","last_undegraded":"2023-09-23T23:42:46.622264+0000","last_fullsized":"2023-09-23T23:42:46.622264+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:30.571439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":357,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.624829+0000","last_change":"2023-09-23T23:39:55.092989+0000","last_active":"2023-09-23T23:42:46.624829+0000","last_peered":"2023-09-23T23:42:46.624829+0000","last_clean":"2023-09-23T23:42:46.624829+0000","last_became_active":"2023-09-23T23:39:55.092680+0000","last_became_peered":"2023-09-23T23:39:55.092680+0000","last_unstale":"2023-09-23T23:42:46.624829+0000","last_undegraded":"2023-09-23T23:42:46.624829+0000","last_fullsized":"2023-09-23T23:42:46.624829+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:44.685469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":998,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622700+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:42:46.622700+0000","last_peered":"2023-09-23T23:42:46.622700+0000","last_clean":"2023-09-23T23:42:46.622700+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:42:46.622700+0000","last_undegraded":"2023-09-23T23:42:46.622700+0000","last_fullsized":"2023-09-23T23:42:46.622700+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":330,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.624780+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:42:46.624780+0000","last_peered":"2023-09-23T23:42:46.624780+0000","last_clean":"2023-09-23T23:42:46.624780+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:42:46.624780+0000","last_undegraded":"2023-09-23T23:42:46.624780+0000","last_fullsized":"2023-09-23T23:42:46.624780+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":690,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622943+0000","last_change":"2023-09-23T23:39:55.100428+0000","last_active":"2023-09-23T23:42:46.622943+0000","last_peered":"2023-09-23T23:42:46.622943+0000","last_clean":"2023-09-23T23:42:46.622943+0000","last_became_active":"2023-09-23T23:39:55.100094+0000","last_became_peered":"2023-09-23T23:39:55.100094+0000","last_unstale":"2023-09-23T23:42:46.622943+0000","last_undegraded":"2023-09-23T23:42:46.622943+0000","last_fullsized":"2023-09-23T23:42:46.622943+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:43.132618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":974,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.630044+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:42:46.630044+0000","last_peered":"2023-09-23T23:42:46.630044+0000","last_clean":"2023-09-23T23:42:46.630044+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:42:46.630044+0000","last_undegraded":"2023-09-23T23:42:46.630044+0000","last_fullsized":"2023-09-23T23:42:46.630044+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":319,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.646021+0000","last_change":"2023-09-23T23:42:46.645764+0000","last_active":"2023-09-23T23:42:46.646021+0000","last_peered":"2023-09-23T23:42:46.646021+0000","last_clean":"2023-09-23T23:42:46.646021+0000","last_became_active":"2023-09-23T23:42:46.645462+0000","last_became_peered":"2023-09-23T23:42:46.645462+0000","last_unstale":"2023-09-23T23:42:46.646021+0000","last_undegraded":"2023-09-23T23:42:46.646021+0000","last_fullsized":"2023-09-23T23:42:46.646021+0000","mapping_epoch":676,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":677,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:04.694952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":74,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.625694+0000","last_change":"2023-09-23T23:39:55.091152+0000","last_active":"2023-09-23T23:42:46.625694+0000","last_peered":"2023-09-23T23:42:46.625694+0000","last_clean":"2023-09-23T23:42:46.625694+0000","last_became_active":"2023-09-23T23:39:55.090833+0000","last_became_peered":"2023-09-23T23:39:55.090833+0000","last_unstale":"2023-09-23T23:42:46.625694+0000","last_undegraded":"2023-09-23T23:42:46.625694+0000","last_fullsized":"2023-09-23T23:42:46.625694+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:33.156366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1152,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.624735+0000","last_change":"2023-09-23T23:39:58.110527+0000","last_active":"2023-09-23T23:42:46.624735+0000","last_peered":"2023-09-23T23:42:46.624735+0000","last_clean":"2023-09-23T23:42:46.624735+0000","last_became_active":"2023-09-23T23:39:58.110194+0000","last_became_peered":"2023-09-23T23:39:58.110194+0000","last_unstale":"2023-09-23T23:42:46.624735+0000","last_undegraded":"2023-09-23T23:42:46.624735+0000","last_fullsized":"2023-09-23T23:42:46.624735+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:21:19.332102+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":353,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.623439+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:42:46.623439+0000","last_peered":"2023-09-23T23:42:46.623439+0000","last_clean":"2023-09-23T23:42:46.623439+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:42:46.623439+0000","last_undegraded":"2023-09-23T23:42:46.623439+0000","last_fullsized":"2023-09-23T23:42:46.623439+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":364,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.626178+0000","last_change":"2023-09-23T23:42:28.956958+0000","last_active":"2023-09-23T23:42:46.626178+0000","last_peered":"2023-09-23T23:42:46.626178+0000","last_clean":"2023-09-23T23:42:46.626178+0000","last_became_active":"2023-09-23T23:42:28.956645+0000","last_became_peered":"2023-09-23T23:42:28.956645+0000","last_unstale":"2023-09-23T23:42:46.626178+0000","last_undegraded":"2023-09-23T23:42:46.626178+0000","last_fullsized":"2023-09-23T23:42:46.626178+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:16:28.472945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":543,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622258+0000","last_change":"2023-09-23T23:39:19.011304+0000","last_active":"2023-09-23T23:42:46.622258+0000","last_peered":"2023-09-23T23:42:46.622258+0000","last_clean":"2023-09-23T23:42:46.622258+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:42:46.622258+0000","last_undegraded":"2023-09-23T23:42:46.622258+0000","last_fullsized":"2023-09-23T23:42:46.622258+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:39:19.011223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:39:19.011223+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:48:49.078060+0000","scrub_duration":0.001220575,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":198,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.629960+0000","last_change":"2023-09-23T23:39:55.129350+0000","last_active":"2023-09-23T23:42:46.629960+0000","last_peered":"2023-09-23T23:42:46.629960+0000","last_clean":"2023-09-23T23:42:46.629960+0000","last_became_active":"2023-09-23T23:39:55.092677+0000","last_became_peered":"2023-09-23T23:39:55.092677+0000","last_unstale":"2023-09-23T23:42:46.629960+0000","last_undegraded":"2023-09-23T23:42:46.629960+0000","last_fullsized":"2023-09-23T23:42:46.629960+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:38.925371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":552,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.624107+0000","last_change":"2023-09-23T23:39:55.097595+0000","last_active":"2023-09-23T23:42:46.624107+0000","last_peered":"2023-09-23T23:42:46.624107+0000","last_clean":"2023-09-23T23:42:46.624107+0000","last_became_active":"2023-09-23T23:39:55.097301+0000","last_became_peered":"2023-09-23T23:39:55.097301+0000","last_unstale":"2023-09-23T23:42:46.624107+0000","last_undegraded":"2023-09-23T23:42:46.624107+0000","last_fullsized":"2023-09-23T23:42:46.624107+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:26.508461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":833,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.623909+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:42:46.623909+0000","last_peered":"2023-09-23T23:42:46.623909+0000","last_clean":"2023-09-23T23:42:46.623909+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:42:46.623909+0000","last_undegraded":"2023-09-23T23:42:46.623909+0000","last_fullsized":"2023-09-23T23:42:46.623909+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":242,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.626103+0000","last_change":"2023-09-23T23:39:55.137107+0000","last_active":"2023-09-23T23:42:46.626103+0000","last_peered":"2023-09-23T23:42:46.626103+0000","last_clean":"2023-09-23T23:42:46.626103+0000","last_became_active":"2023-09-23T23:39:55.094309+0000","last_became_peered":"2023-09-23T23:39:55.094309+0000","last_unstale":"2023-09-23T23:42:46.626103+0000","last_undegraded":"2023-09-23T23:42:46.626103+0000","last_fullsized":"2023-09-23T23:42:46.626103+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:49.766841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":698,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.623952+0000","last_change":"2023-09-23T23:42:28.954052+0000","last_active":"2023-09-23T23:42:46.623952+0000","last_peered":"2023-09-23T23:42:46.623952+0000","last_clean":"2023-09-23T23:42:46.623952+0000","last_became_active":"2023-09-23T23:42:28.953681+0000","last_became_peered":"2023-09-23T23:42:28.953681+0000","last_unstale":"2023-09-23T23:42:46.623952+0000","last_undegraded":"2023-09-23T23:42:46.623952+0000","last_fullsized":"2023-09-23T23:42:46.623952+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:07.367387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":795,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.625312+0000","last_change":"2023-09-23T23:39:58.117169+0000","last_active":"2023-09-23T23:42:46.625312+0000","last_peered":"2023-09-23T23:42:46.625312+0000","last_clean":"2023-09-23T23:42:46.625312+0000","last_became_active":"2023-09-23T23:39:58.116815+0000","last_became_peered":"2023-09-23T23:39:58.116815+0000","last_unstale":"2023-09-23T23:42:46.625312+0000","last_undegraded":"2023-09-23T23:42:46.625312+0000","last_fullsized":"2023-09-23T23:42:46.625312+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:51.175189+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":275,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622251+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:42:46.622251+0000","last_peered":"2023-09-23T23:42:46.622251+0000","last_clean":"2023-09-23T23:42:46.622251+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:42:46.622251+0000","last_undegraded":"2023-09-23T23:42:46.622251+0000","last_fullsized":"2023-09-23T23:42:46.622251+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":402,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.623872+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:42:46.623872+0000","last_peered":"2023-09-23T23:42:46.623872+0000","last_clean":"2023-09-23T23:42:46.623872+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:42:46.623872+0000","last_undegraded":"2023-09-23T23:42:46.623872+0000","last_fullsized":"2023-09-23T23:42:46.623872+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":643,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.623435+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:42:46.623435+0000","last_peered":"2023-09-23T23:42:46.623435+0000","last_clean":"2023-09-23T23:42:46.623435+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:42:46.623435+0000","last_undegraded":"2023-09-23T23:42:46.623435+0000","last_fullsized":"2023-09-23T23:42:46.623435+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":489,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.630995+0000","last_change":"2023-09-23T23:39:55.100855+0000","last_active":"2023-09-23T23:42:46.630995+0000","last_peered":"2023-09-23T23:42:46.630995+0000","last_clean":"2023-09-23T23:42:46.630995+0000","last_became_active":"2023-09-23T23:39:55.100577+0000","last_became_peered":"2023-09-23T23:39:55.100577+0000","last_unstale":"2023-09-23T23:42:46.630995+0000","last_undegraded":"2023-09-23T23:42:46.630995+0000","last_fullsized":"2023-09-23T23:42:46.630995+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:02.548093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":637,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:52.832468+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:42:52.832468+0000","last_peered":"2023-09-23T23:42:52.832468+0000","last_clean":"2023-09-23T23:42:52.832468+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:42:52.832468+0000","last_undegraded":"2023-09-23T23:42:52.832468+0000","last_fullsized":"2023-09-23T23:42:52.832468+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":1101,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.624454+0000","last_change":"2023-09-23T23:42:32.073277+0000","last_active":"2023-09-23T23:42:46.624454+0000","last_peered":"2023-09-23T23:42:46.624454+0000","last_clean":"2023-09-23T23:42:46.624454+0000","last_became_active":"2023-09-23T23:42:32.072952+0000","last_became_peered":"2023-09-23T23:42:32.072952+0000","last_unstale":"2023-09-23T23:42:46.624454+0000","last_undegraded":"2023-09-23T23:42:46.624454+0000","last_fullsized":"2023-09-23T23:42:46.624454+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:19.983734+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":45,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.634348+0000","last_change":"2023-09-23T23:41:31.621623+0000","last_active":"2023-09-23T23:42:46.634348+0000","last_peered":"2023-09-23T23:42:46.634348+0000","last_clean":"2023-09-23T23:42:46.634348+0000","last_became_active":"2023-09-23T23:41:31.596928+0000","last_became_peered":"2023-09-23T23:41:31.596928+0000","last_unstale":"2023-09-23T23:42:46.634348+0000","last_undegraded":"2023-09-23T23:42:46.634348+0000","last_fullsized":"2023-09-23T23:42:46.634348+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:01.308674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":321,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622838+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:42:46.622838+0000","last_peered":"2023-09-23T23:42:46.622838+0000","last_clean":"2023-09-23T23:42:46.622838+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:42:46.622838+0000","last_undegraded":"2023-09-23T23:42:46.622838+0000","last_fullsized":"2023-09-23T23:42:46.622838+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"678'10045","reported_seq":15807,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:57.773522+0000","last_change":"2023-09-23T23:41:11.967390+0000","last_active":"2023-09-23T23:42:57.773522+0000","last_peered":"2023-09-23T23:42:57.773522+0000","last_clean":"2023-09-23T23:42:57.773522+0000","last_became_active":"2023-09-23T23:41:11.955563+0000","last_became_peered":"2023-09-23T23:41:11.955563+0000","last_unstale":"2023-09-23T23:42:57.773522+0000","last_undegraded":"2023-09-23T23:42:57.773522+0000","last_fullsized":"2023-09-23T23:42:57.773522+0000","mapping_epoch":645,"log_start":"636'5186","ondisk_log_start":"636'5186","created":617,"last_epoch_clean":646,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4859,"log_dups_size":0,"ondisk_log_size":4859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:03.686224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25362432,"num_objects":387,"num_object_clones":0,"num_object_copies":1161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10045,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2837,"num_bytes_recovered":185925632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":1014,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.626497+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:42:46.626497+0000","last_peered":"2023-09-23T23:42:46.626497+0000","last_clean":"2023-09-23T23:42:46.626497+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:42:46.626497+0000","last_undegraded":"2023-09-23T23:42:46.626497+0000","last_fullsized":"2023-09-23T23:42:46.626497+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":311,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622807+0000","last_change":"2023-09-23T23:42:28.954407+0000","last_active":"2023-09-23T23:42:46.622807+0000","last_peered":"2023-09-23T23:42:46.622807+0000","last_clean":"2023-09-23T23:42:46.622807+0000","last_became_active":"2023-09-23T23:42:28.950556+0000","last_became_peered":"2023-09-23T23:42:28.950556+0000","last_unstale":"2023-09-23T23:42:46.622807+0000","last_undegraded":"2023-09-23T23:42:46.622807+0000","last_fullsized":"2023-09-23T23:42:46.622807+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:58.736585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":319,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.625383+0000","last_change":"2023-09-23T23:39:55.095319+0000","last_active":"2023-09-23T23:42:46.625383+0000","last_peered":"2023-09-23T23:42:46.625383+0000","last_clean":"2023-09-23T23:42:46.625383+0000","last_became_active":"2023-09-23T23:39:55.094677+0000","last_became_peered":"2023-09-23T23:39:55.094677+0000","last_unstale":"2023-09-23T23:42:46.625383+0000","last_undegraded":"2023-09-23T23:42:46.625383+0000","last_fullsized":"2023-09-23T23:42:46.625383+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:21.525905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"678'9940","reported_seq":10011,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:57.777830+0000","last_change":"2023-09-23T23:39:13.746676+0000","last_active":"2023-09-23T23:42:57.777830+0000","last_peered":"2023-09-23T23:42:57.777830+0000","last_clean":"2023-09-23T23:42:57.777830+0000","last_became_active":"2023-09-23T23:39:13.746389+0000","last_became_peered":"2023-09-23T23:39:13.746389+0000","last_unstale":"2023-09-23T23:42:57.777830+0000","last_undegraded":"2023-09-23T23:42:57.777830+0000","last_fullsized":"2023-09-23T23:42:57.777830+0000","mapping_epoch":617,"log_start":"636'5097","ondisk_log_start":"636'5097","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4843,"log_dups_size":0,"ondisk_log_size":4843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:44.833933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26607616,"num_objects":406,"num_object_clones":0,"num_object_copies":1218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9940,"num_write_kb":331072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1114,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.623799+0000","last_change":"2023-09-23T23:39:58.119994+0000","last_active":"2023-09-23T23:42:46.623799+0000","last_peered":"2023-09-23T23:42:46.623799+0000","last_clean":"2023-09-23T23:42:46.623799+0000","last_became_active":"2023-09-23T23:39:58.119654+0000","last_became_peered":"2023-09-23T23:39:58.119654+0000","last_unstale":"2023-09-23T23:42:46.623799+0000","last_undegraded":"2023-09-23T23:42:46.623799+0000","last_fullsized":"2023-09-23T23:42:46.623799+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:02:41.967366+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":337,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622999+0000","last_change":"2023-09-23T23:41:31.603533+0000","last_active":"2023-09-23T23:42:46.622999+0000","last_peered":"2023-09-23T23:42:46.622999+0000","last_clean":"2023-09-23T23:42:46.622999+0000","last_became_active":"2023-09-23T23:41:31.593485+0000","last_became_peered":"2023-09-23T23:41:31.593485+0000","last_unstale":"2023-09-23T23:42:46.622999+0000","last_undegraded":"2023-09-23T23:42:46.622999+0000","last_fullsized":"2023-09-23T23:42:46.622999+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:10:56.208677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":541,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:47.826801+0000","last_change":"2023-09-23T23:39:55.101024+0000","last_active":"2023-09-23T23:42:47.826801+0000","last_peered":"2023-09-23T23:42:47.826801+0000","last_clean":"2023-09-23T23:42:47.826801+0000","last_became_active":"2023-09-23T23:39:55.100745+0000","last_became_peered":"2023-09-23T23:39:55.100745+0000","last_unstale":"2023-09-23T23:42:47.826801+0000","last_undegraded":"2023-09-23T23:42:47.826801+0000","last_fullsized":"2023-09-23T23:42:47.826801+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:28.387953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.0","version":"678'10143","reported_seq":10214,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:55.617643+0000","last_change":"2023-09-23T23:39:13.741363+0000","last_active":"2023-09-23T23:42:55.617643+0000","last_peered":"2023-09-23T23:42:55.617643+0000","last_clean":"2023-09-23T23:42:55.617643+0000","last_became_active":"2023-09-23T23:39:13.741098+0000","last_became_peered":"2023-09-23T23:39:13.741098+0000","last_unstale":"2023-09-23T23:42:55.617643+0000","last_undegraded":"2023-09-23T23:42:55.617643+0000","last_fullsized":"2023-09-23T23:42:55.617643+0000","mapping_epoch":617,"log_start":"632'3947","ondisk_log_start":"632'3947","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6196,"log_dups_size":0,"ondisk_log_size":6196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:31.131594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32571392,"num_objects":497,"num_object_clones":0,"num_object_copies":1491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10143,"num_write_kb":340480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":845,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.624644+0000","last_change":"2023-09-23T23:39:58.122462+0000","last_active":"2023-09-23T23:42:46.624644+0000","last_peered":"2023-09-23T23:42:46.624644+0000","last_clean":"2023-09-23T23:42:46.624644+0000","last_became_active":"2023-09-23T23:39:58.122163+0000","last_became_peered":"2023-09-23T23:39:58.122163+0000","last_unstale":"2023-09-23T23:42:46.624644+0000","last_undegraded":"2023-09-23T23:42:46.624644+0000","last_fullsized":"2023-09-23T23:42:46.624644+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:41.511681+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":430,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.624249+0000","last_change":"2023-09-23T23:39:55.141520+0000","last_active":"2023-09-23T23:42:46.624249+0000","last_peered":"2023-09-23T23:42:46.624249+0000","last_clean":"2023-09-23T23:42:46.624249+0000","last_became_active":"2023-09-23T23:39:55.099176+0000","last_became_peered":"2023-09-23T23:39:55.099176+0000","last_unstale":"2023-09-23T23:42:46.624249+0000","last_undegraded":"2023-09-23T23:42:46.624249+0000","last_fullsized":"2023-09-23T23:42:46.624249+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:12.860173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":60,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.623948+0000","last_change":"2023-09-23T23:39:55.094219+0000","last_active":"2023-09-23T23:42:46.623948+0000","last_peered":"2023-09-23T23:42:46.623948+0000","last_clean":"2023-09-23T23:42:46.623948+0000","last_became_active":"2023-09-23T23:39:55.093849+0000","last_became_peered":"2023-09-23T23:39:55.093849+0000","last_unstale":"2023-09-23T23:42:46.623948+0000","last_undegraded":"2023-09-23T23:42:46.623948+0000","last_fullsized":"2023-09-23T23:42:46.623948+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:54.576029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.1","version":"678'10219","reported_seq":10290,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:57.800858+0000","last_change":"2023-09-23T23:39:13.744449+0000","last_active":"2023-09-23T23:42:57.800858+0000","last_peered":"2023-09-23T23:42:57.800858+0000","last_clean":"2023-09-23T23:42:57.800858+0000","last_became_active":"2023-09-23T23:39:13.744163+0000","last_became_peered":"2023-09-23T23:39:13.744163+0000","last_unstale":"2023-09-23T23:42:57.800858+0000","last_undegraded":"2023-09-23T23:42:57.800858+0000","last_fullsized":"2023-09-23T23:42:57.800858+0000","mapping_epoch":617,"log_start":"637'5346","ondisk_log_start":"637'5346","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4873,"log_dups_size":0,"ondisk_log_size":4873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:50.120382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23527424,"num_objects":359,"num_object_clones":0,"num_object_copies":1077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10219,"num_write_kb":338496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":587,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622968+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:42:46.622968+0000","last_peered":"2023-09-23T23:42:46.622968+0000","last_clean":"2023-09-23T23:42:46.622968+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:42:46.622968+0000","last_undegraded":"2023-09-23T23:42:46.622968+0000","last_fullsized":"2023-09-23T23:42:46.622968+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.702778+0000","last_change":"2023-09-23T23:42:46.697762+0000","last_active":"2023-09-23T23:42:46.702778+0000","last_peered":"2023-09-23T23:42:46.702778+0000","last_clean":"2023-09-23T23:42:46.702778+0000","last_became_active":"2023-09-23T23:42:46.697414+0000","last_became_peered":"2023-09-23T23:42:46.697414+0000","last_unstale":"2023-09-23T23:42:46.702778+0000","last_undegraded":"2023-09-23T23:42:46.702778+0000","last_fullsized":"2023-09-23T23:42:46.702778+0000","mapping_epoch":676,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":677,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:11.272589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":678,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.631015+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:42:46.631015+0000","last_peered":"2023-09-23T23:42:46.631015+0000","last_clean":"2023-09-23T23:42:46.631015+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:42:46.631015+0000","last_undegraded":"2023-09-23T23:42:46.631015+0000","last_fullsized":"2023-09-23T23:42:46.631015+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"678'10201","reported_seq":10270,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:55.645340+0000","last_change":"2023-09-23T23:39:13.745430+0000","last_active":"2023-09-23T23:42:55.645340+0000","last_peered":"2023-09-23T23:42:55.645340+0000","last_clean":"2023-09-23T23:42:55.645340+0000","last_became_active":"2023-09-23T23:39:13.744751+0000","last_became_peered":"2023-09-23T23:39:13.744751+0000","last_unstale":"2023-09-23T23:42:55.645340+0000","last_undegraded":"2023-09-23T23:42:55.645340+0000","last_fullsized":"2023-09-23T23:42:55.645340+0000","mapping_epoch":617,"log_start":"632'3981","ondisk_log_start":"632'3981","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6220,"log_dups_size":0,"ondisk_log_size":6220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:23.195329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32833536,"num_objects":501,"num_object_clones":0,"num_object_copies":1503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10201,"num_write_kb":342464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":820,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622079+0000","last_change":"2023-09-23T23:39:19.985630+0000","last_active":"2023-09-23T23:42:46.622079+0000","last_peered":"2023-09-23T23:42:46.622079+0000","last_clean":"2023-09-23T23:42:46.622079+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:42:46.622079+0000","last_undegraded":"2023-09-23T23:42:46.622079+0000","last_fullsized":"2023-09-23T23:42:46.622079+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:39:19.985512+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:39:19.985512+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:52.750567+0000","scrub_duration":0.001852713,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":717,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.623349+0000","last_change":"2023-09-23T23:41:31.618278+0000","last_active":"2023-09-23T23:42:46.623349+0000","last_peered":"2023-09-23T23:42:46.623349+0000","last_clean":"2023-09-23T23:42:46.623349+0000","last_became_active":"2023-09-23T23:41:31.602519+0000","last_became_peered":"2023-09-23T23:41:31.602519+0000","last_unstale":"2023-09-23T23:42:46.623349+0000","last_undegraded":"2023-09-23T23:42:46.623349+0000","last_fullsized":"2023-09-23T23:42:46.623349+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:20.395010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":611,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:52.832314+0000","last_change":"2023-09-23T23:39:55.099345+0000","last_active":"2023-09-23T23:42:52.832314+0000","last_peered":"2023-09-23T23:42:52.832314+0000","last_clean":"2023-09-23T23:42:52.832314+0000","last_became_active":"2023-09-23T23:39:55.098875+0000","last_became_peered":"2023-09-23T23:39:55.098875+0000","last_unstale":"2023-09-23T23:42:52.832314+0000","last_undegraded":"2023-09-23T23:42:52.832314+0000","last_fullsized":"2023-09-23T23:42:52.832314+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:09.169232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.7","version":"678'10370","reported_seq":10439,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:59.792693+0000","last_change":"2023-09-23T23:39:13.743009+0000","last_active":"2023-09-23T23:42:59.792693+0000","last_peered":"2023-09-23T23:42:59.792693+0000","last_clean":"2023-09-23T23:42:59.792693+0000","last_became_active":"2023-09-23T23:39:13.742719+0000","last_became_peered":"2023-09-23T23:39:13.742719+0000","last_unstale":"2023-09-23T23:42:59.792693+0000","last_undegraded":"2023-09-23T23:42:59.792693+0000","last_fullsized":"2023-09-23T23:42:59.792693+0000","mapping_epoch":617,"log_start":"636'5176","ondisk_log_start":"636'5176","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":5194,"log_dups_size":0,"ondisk_log_size":5194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:50.189642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18087936,"num_objects":276,"num_object_clones":0,"num_object_copies":828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10370,"num_write_kb":340672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":964,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.626738+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:42:46.626738+0000","last_peered":"2023-09-23T23:42:46.626738+0000","last_clean":"2023-09-23T23:42:46.626738+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:42:46.626738+0000","last_undegraded":"2023-09-23T23:42:46.626738+0000","last_fullsized":"2023-09-23T23:42:46.626738+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":153,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.621991+0000","last_change":"2023-09-23T23:42:28.953222+0000","last_active":"2023-09-23T23:42:46.621991+0000","last_peered":"2023-09-23T23:42:46.621991+0000","last_clean":"2023-09-23T23:42:46.621991+0000","last_became_active":"2023-09-23T23:42:28.952902+0000","last_became_peered":"2023-09-23T23:42:28.952902+0000","last_unstale":"2023-09-23T23:42:46.621991+0000","last_undegraded":"2023-09-23T23:42:46.621991+0000","last_fullsized":"2023-09-23T23:42:46.621991+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:21.715883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":574,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.753231+0000","last_change":"2023-09-23T23:41:31.604354+0000","last_active":"2023-09-23T23:42:53.753231+0000","last_peered":"2023-09-23T23:42:53.753231+0000","last_clean":"2023-09-23T23:42:53.753231+0000","last_became_active":"2023-09-23T23:41:31.604035+0000","last_became_peered":"2023-09-23T23:41:31.604035+0000","last_unstale":"2023-09-23T23:42:53.753231+0000","last_undegraded":"2023-09-23T23:42:53.753231+0000","last_fullsized":"2023-09-23T23:42:53.753231+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:55.798632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"678'10064","reported_seq":15898,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:57.753925+0000","last_change":"2023-09-23T23:40:33.082798+0000","last_active":"2023-09-23T23:42:57.753925+0000","last_peered":"2023-09-23T23:42:57.753925+0000","last_clean":"2023-09-23T23:42:57.753925+0000","last_became_active":"2023-09-23T23:40:33.076970+0000","last_became_peered":"2023-09-23T23:40:33.076970+0000","last_unstale":"2023-09-23T23:42:57.753925+0000","last_undegraded":"2023-09-23T23:42:57.753925+0000","last_fullsized":"2023-09-23T23:42:57.753925+0000","mapping_epoch":634,"log_start":"636'5169","ondisk_log_start":"636'5169","created":617,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4895,"log_dups_size":0,"ondisk_log_size":4895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:08.992451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25296896,"num_objects":386,"num_object_clones":0,"num_object_copies":1158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":386,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10064,"num_write_kb":334400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2873,"num_bytes_recovered":188284928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":686,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.624196+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:42:46.624196+0000","last_peered":"2023-09-23T23:42:46.624196+0000","last_clean":"2023-09-23T23:42:46.624196+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:42:46.624196+0000","last_undegraded":"2023-09-23T23:42:46.624196+0000","last_fullsized":"2023-09-23T23:42:46.624196+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":544,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.631230+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:42:46.631230+0000","last_peered":"2023-09-23T23:42:46.631230+0000","last_clean":"2023-09-23T23:42:46.631230+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:42:46.631230+0000","last_undegraded":"2023-09-23T23:42:46.631230+0000","last_fullsized":"2023-09-23T23:42:46.631230+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":580,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.621955+0000","last_change":"2023-09-23T23:42:28.951851+0000","last_active":"2023-09-23T23:42:46.621955+0000","last_peered":"2023-09-23T23:42:46.621955+0000","last_clean":"2023-09-23T23:42:46.621955+0000","last_became_active":"2023-09-23T23:42:28.951542+0000","last_became_peered":"2023-09-23T23:42:28.951542+0000","last_unstale":"2023-09-23T23:42:46.621955+0000","last_undegraded":"2023-09-23T23:42:46.621955+0000","last_fullsized":"2023-09-23T23:42:46.621955+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:13.264990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.5","version":"678'10161","reported_seq":10232,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:57.812984+0000","last_change":"2023-09-23T23:39:13.742854+0000","last_active":"2023-09-23T23:42:57.812984+0000","last_peered":"2023-09-23T23:42:57.812984+0000","last_clean":"2023-09-23T23:42:57.812984+0000","last_became_active":"2023-09-23T23:39:13.742579+0000","last_became_peered":"2023-09-23T23:39:13.742579+0000","last_unstale":"2023-09-23T23:42:57.812984+0000","last_undegraded":"2023-09-23T23:42:57.812984+0000","last_fullsized":"2023-09-23T23:42:57.812984+0000","mapping_epoch":617,"log_start":"636'5226","ondisk_log_start":"636'5226","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4935,"log_dups_size":0,"ondisk_log_size":4935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:51.872071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24182784,"num_objects":369,"num_object_clones":0,"num_object_copies":1107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10161,"num_write_kb":336960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":1039,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.626110+0000","last_change":"2023-09-23T23:39:58.119762+0000","last_active":"2023-09-23T23:42:46.626110+0000","last_peered":"2023-09-23T23:42:46.626110+0000","last_clean":"2023-09-23T23:42:46.626110+0000","last_became_active":"2023-09-23T23:39:58.119422+0000","last_became_peered":"2023-09-23T23:39:58.119422+0000","last_unstale":"2023-09-23T23:42:46.626110+0000","last_undegraded":"2023-09-23T23:42:46.626110+0000","last_fullsized":"2023-09-23T23:42:46.626110+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:03.564930+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":322,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.621795+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:42:46.621795+0000","last_peered":"2023-09-23T23:42:46.621795+0000","last_clean":"2023-09-23T23:42:46.621795+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:42:46.621795+0000","last_undegraded":"2023-09-23T23:42:46.621795+0000","last_fullsized":"2023-09-23T23:42:46.621795+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":610,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.623056+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:42:46.623056+0000","last_peered":"2023-09-23T23:42:46.623056+0000","last_clean":"2023-09-23T23:42:46.623056+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:42:46.623056+0000","last_undegraded":"2023-09-23T23:42:46.623056+0000","last_fullsized":"2023-09-23T23:42:46.623056+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.8","version":"678'10199","reported_seq":11607,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:55.648891+0000","last_change":"2023-09-23T23:42:28.980646+0000","last_active":"2023-09-23T23:42:55.648891+0000","last_peered":"2023-09-23T23:42:55.648891+0000","last_clean":"2023-09-23T23:42:55.648891+0000","last_became_active":"2023-09-23T23:42:28.958020+0000","last_became_peered":"2023-09-23T23:42:28.958020+0000","last_unstale":"2023-09-23T23:42:55.648891+0000","last_undegraded":"2023-09-23T23:42:55.648891+0000","last_fullsized":"2023-09-23T23:42:55.648891+0000","mapping_epoch":669,"log_start":"632'4017","ondisk_log_start":"632'4017","created":617,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6182,"log_dups_size":0,"ondisk_log_size":6182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:21.232496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29556736,"num_objects":451,"num_object_clones":0,"num_object_copies":1353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10199,"num_write_kb":340800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":656,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":883,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622762+0000","last_change":"2023-09-23T23:39:58.120057+0000","last_active":"2023-09-23T23:42:46.622762+0000","last_peered":"2023-09-23T23:42:46.622762+0000","last_clean":"2023-09-23T23:42:46.622762+0000","last_became_active":"2023-09-23T23:39:58.119647+0000","last_became_peered":"2023-09-23T23:39:58.119647+0000","last_unstale":"2023-09-23T23:42:46.622762+0000","last_undegraded":"2023-09-23T23:42:46.622762+0000","last_fullsized":"2023-09-23T23:42:46.622762+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:50.965369+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":390,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.691391+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:42:46.691391+0000","last_peered":"2023-09-23T23:42:46.691391+0000","last_clean":"2023-09-23T23:42:46.691391+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:42:46.691391+0000","last_undegraded":"2023-09-23T23:42:46.691391+0000","last_fullsized":"2023-09-23T23:42:46.691391+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":693,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.624329+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:42:46.624329+0000","last_peered":"2023-09-23T23:42:46.624329+0000","last_clean":"2023-09-23T23:42:46.624329+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:42:46.624329+0000","last_undegraded":"2023-09-23T23:42:46.624329+0000","last_fullsized":"2023-09-23T23:42:46.624329+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":294,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.624153+0000","last_change":"2023-09-23T23:42:28.950994+0000","last_active":"2023-09-23T23:42:46.624153+0000","last_peered":"2023-09-23T23:42:46.624153+0000","last_clean":"2023-09-23T23:42:46.624153+0000","last_became_active":"2023-09-23T23:42:28.950464+0000","last_became_peered":"2023-09-23T23:42:28.950464+0000","last_unstale":"2023-09-23T23:42:46.624153+0000","last_undegraded":"2023-09-23T23:42:46.624153+0000","last_fullsized":"2023-09-23T23:42:46.624153+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:57.975234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.e","version":"678'10010","reported_seq":15991,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:55.632978+0000","last_change":"2023-09-23T23:42:21.783482+0000","last_active":"2023-09-23T23:42:55.632978+0000","last_peered":"2023-09-23T23:42:55.632978+0000","last_clean":"2023-09-23T23:42:55.632978+0000","last_became_active":"2023-09-23T23:42:21.754054+0000","last_became_peered":"2023-09-23T23:42:21.754054+0000","last_unstale":"2023-09-23T23:42:55.632978+0000","last_undegraded":"2023-09-23T23:42:55.632978+0000","last_fullsized":"2023-09-23T23:42:55.632978+0000","mapping_epoch":667,"log_start":"632'3818","ondisk_log_start":"632'3818","created":617,"last_epoch_clean":668,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6192,"log_dups_size":0,"ondisk_log_size":6192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:33.850501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31850496,"num_objects":486,"num_object_clones":0,"num_object_copies":1458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10010,"num_write_kb":335872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2945,"num_bytes_recovered":134676480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":764,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.629301+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:42:46.629301+0000","last_peered":"2023-09-23T23:42:46.629301+0000","last_clean":"2023-09-23T23:42:46.629301+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:42:46.629301+0000","last_undegraded":"2023-09-23T23:42:46.629301+0000","last_fullsized":"2023-09-23T23:42:46.629301+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":331,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.624299+0000","last_change":"2023-09-23T23:39:55.089555+0000","last_active":"2023-09-23T23:42:46.624299+0000","last_peered":"2023-09-23T23:42:46.624299+0000","last_clean":"2023-09-23T23:42:46.624299+0000","last_became_active":"2023-09-23T23:39:55.089184+0000","last_became_peered":"2023-09-23T23:39:55.089184+0000","last_unstale":"2023-09-23T23:42:46.624299+0000","last_undegraded":"2023-09-23T23:42:46.624299+0000","last_fullsized":"2023-09-23T23:42:46.624299+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:19.775188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":543,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.631202+0000","last_change":"2023-09-23T23:39:55.089351+0000","last_active":"2023-09-23T23:42:46.631202+0000","last_peered":"2023-09-23T23:42:46.631202+0000","last_clean":"2023-09-23T23:42:46.631202+0000","last_became_active":"2023-09-23T23:39:55.089016+0000","last_became_peered":"2023-09-23T23:39:55.089016+0000","last_unstale":"2023-09-23T23:42:46.631202+0000","last_undegraded":"2023-09-23T23:42:46.631202+0000","last_fullsized":"2023-09-23T23:42:46.631202+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:05:23.856265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":314,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.630138+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:42:46.630138+0000","last_peered":"2023-09-23T23:42:46.630138+0000","last_clean":"2023-09-23T23:42:46.630138+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:42:46.630138+0000","last_undegraded":"2023-09-23T23:42:46.630138+0000","last_fullsized":"2023-09-23T23:42:46.630138+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"678'9865","reported_seq":15535,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:55.627156+0000","last_change":"2023-09-23T23:42:01.335232+0000","last_active":"2023-09-23T23:42:55.627156+0000","last_peered":"2023-09-23T23:42:55.627156+0000","last_clean":"2023-09-23T23:42:55.627156+0000","last_became_active":"2023-09-23T23:42:01.316912+0000","last_became_peered":"2023-09-23T23:42:01.316912+0000","last_unstale":"2023-09-23T23:42:55.627156+0000","last_undegraded":"2023-09-23T23:42:55.627156+0000","last_fullsized":"2023-09-23T23:42:55.627156+0000","mapping_epoch":662,"log_start":"632'3733","ondisk_log_start":"632'3733","created":617,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6132,"log_dups_size":0,"ondisk_log_size":6132,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:34:25.843454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31588376,"num_objects":483,"num_object_clones":0,"num_object_copies":1449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":483,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9865,"num_write_kb":331073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2789,"num_bytes_recovered":115015680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":982,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.626033+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:42:46.626033+0000","last_peered":"2023-09-23T23:42:46.626033+0000","last_clean":"2023-09-23T23:42:46.626033+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:42:46.626033+0000","last_undegraded":"2023-09-23T23:42:46.626033+0000","last_fullsized":"2023-09-23T23:42:46.626033+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":419,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.631142+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:42:46.631142+0000","last_peered":"2023-09-23T23:42:46.631142+0000","last_clean":"2023-09-23T23:42:46.631142+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:42:46.631142+0000","last_undegraded":"2023-09-23T23:42:46.631142+0000","last_fullsized":"2023-09-23T23:42:46.631142+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":338,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.626811+0000","last_change":"2023-09-23T23:39:55.093320+0000","last_active":"2023-09-23T23:42:46.626811+0000","last_peered":"2023-09-23T23:42:46.626811+0000","last_clean":"2023-09-23T23:42:46.626811+0000","last_became_active":"2023-09-23T23:39:55.092998+0000","last_became_peered":"2023-09-23T23:39:55.092998+0000","last_unstale":"2023-09-23T23:42:46.626811+0000","last_undegraded":"2023-09-23T23:42:46.626811+0000","last_fullsized":"2023-09-23T23:42:46.626811+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:38.763321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":19,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.690470+0000","last_change":"2023-09-23T23:42:28.952433+0000","last_active":"2023-09-23T23:42:46.690470+0000","last_peered":"2023-09-23T23:42:46.690470+0000","last_clean":"2023-09-23T23:42:46.690470+0000","last_became_active":"2023-09-23T23:42:28.952139+0000","last_became_peered":"2023-09-23T23:42:28.952139+0000","last_unstale":"2023-09-23T23:42:46.690470+0000","last_undegraded":"2023-09-23T23:42:46.690470+0000","last_fullsized":"2023-09-23T23:42:46.690470+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:49.952980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"678'10200","reported_seq":10271,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:57.788771+0000","last_change":"2023-09-23T23:39:13.747297+0000","last_active":"2023-09-23T23:42:57.788771+0000","last_peered":"2023-09-23T23:42:57.788771+0000","last_clean":"2023-09-23T23:42:57.788771+0000","last_became_active":"2023-09-23T23:39:13.747068+0000","last_became_peered":"2023-09-23T23:39:13.747068+0000","last_unstale":"2023-09-23T23:42:57.788771+0000","last_undegraded":"2023-09-23T23:42:57.788771+0000","last_fullsized":"2023-09-23T23:42:57.788771+0000","mapping_epoch":617,"log_start":"636'5281","ondisk_log_start":"636'5281","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4919,"log_dups_size":0,"ondisk_log_size":4919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:34.450587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25296896,"num_objects":386,"num_object_clones":0,"num_object_copies":1158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":386,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10200,"num_write_kb":338752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":696,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.626173+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:42:46.626173+0000","last_peered":"2023-09-23T23:42:46.626173+0000","last_clean":"2023-09-23T23:42:46.626173+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:42:46.626173+0000","last_undegraded":"2023-09-23T23:42:46.626173+0000","last_fullsized":"2023-09-23T23:42:46.626173+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":997,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622189+0000","last_change":"2023-09-23T23:39:58.122811+0000","last_active":"2023-09-23T23:42:46.622189+0000","last_peered":"2023-09-23T23:42:46.622189+0000","last_clean":"2023-09-23T23:42:46.622189+0000","last_became_active":"2023-09-23T23:39:58.122461+0000","last_became_peered":"2023-09-23T23:39:58.122461+0000","last_unstale":"2023-09-23T23:42:46.622189+0000","last_undegraded":"2023-09-23T23:42:46.622189+0000","last_fullsized":"2023-09-23T23:42:46.622189+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:20:59.163746+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":652,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622069+0000","last_change":"2023-09-23T23:39:55.101203+0000","last_active":"2023-09-23T23:42:46.622069+0000","last_peered":"2023-09-23T23:42:46.622069+0000","last_clean":"2023-09-23T23:42:46.622069+0000","last_became_active":"2023-09-23T23:39:55.099575+0000","last_became_peered":"2023-09-23T23:39:55.099575+0000","last_unstale":"2023-09-23T23:42:46.622069+0000","last_undegraded":"2023-09-23T23:42:46.622069+0000","last_fullsized":"2023-09-23T23:42:46.622069+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:40.288184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":62,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.623197+0000","last_change":"2023-09-23T23:39:55.103713+0000","last_active":"2023-09-23T23:42:46.623197+0000","last_peered":"2023-09-23T23:42:46.623197+0000","last_clean":"2023-09-23T23:42:46.623197+0000","last_became_active":"2023-09-23T23:39:55.103402+0000","last_became_peered":"2023-09-23T23:39:55.103402+0000","last_unstale":"2023-09-23T23:42:46.623197+0000","last_undegraded":"2023-09-23T23:42:46.623197+0000","last_fullsized":"2023-09-23T23:42:46.623197+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:44.949292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.c","version":"678'9984","reported_seq":15738,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:55.616969+0000","last_change":"2023-09-23T23:41:10.964608+0000","last_active":"2023-09-23T23:42:55.616969+0000","last_peered":"2023-09-23T23:42:55.616969+0000","last_clean":"2023-09-23T23:42:55.616969+0000","last_became_active":"2023-09-23T23:41:10.952515+0000","last_became_peered":"2023-09-23T23:41:10.952515+0000","last_unstale":"2023-09-23T23:42:55.616969+0000","last_undegraded":"2023-09-23T23:42:55.616969+0000","last_fullsized":"2023-09-23T23:42:55.616969+0000","mapping_epoch":644,"log_start":"632'3762","ondisk_log_start":"632'3762","created":617,"last_epoch_clean":645,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6222,"log_dups_size":0,"ondisk_log_size":6222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:30.860608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30932992,"num_objects":472,"num_object_clones":0,"num_object_copies":1416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9984,"num_write_kb":334592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2834,"num_bytes_recovered":185729024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":841,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.691572+0000","last_change":"2023-09-23T23:41:34.668439+0000","last_active":"2023-09-23T23:42:46.691572+0000","last_peered":"2023-09-23T23:42:46.691572+0000","last_clean":"2023-09-23T23:42:46.691572+0000","last_became_active":"2023-09-23T23:41:34.668104+0000","last_became_peered":"2023-09-23T23:41:34.668104+0000","last_unstale":"2023-09-23T23:42:46.691572+0000","last_undegraded":"2023-09-23T23:42:46.691572+0000","last_fullsized":"2023-09-23T23:42:46.691572+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:42:24.264118+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":605,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622036+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:42:46.622036+0000","last_peered":"2023-09-23T23:42:46.622036+0000","last_clean":"2023-09-23T23:42:46.622036+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:42:46.622036+0000","last_undegraded":"2023-09-23T23:42:46.622036+0000","last_fullsized":"2023-09-23T23:42:46.622036+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":522,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.621767+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:42:46.621767+0000","last_peered":"2023-09-23T23:42:46.621767+0000","last_clean":"2023-09-23T23:42:46.621767+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:42:46.621767+0000","last_undegraded":"2023-09-23T23:42:46.621767+0000","last_fullsized":"2023-09-23T23:42:46.621767+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":35,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.630660+0000","last_change":"2023-09-23T23:41:31.605087+0000","last_active":"2023-09-23T23:42:46.630660+0000","last_peered":"2023-09-23T23:42:46.630660+0000","last_clean":"2023-09-23T23:42:46.630660+0000","last_became_active":"2023-09-23T23:41:31.604777+0000","last_became_peered":"2023-09-23T23:41:31.604777+0000","last_unstale":"2023-09-23T23:42:46.630660+0000","last_undegraded":"2023-09-23T23:42:46.630660+0000","last_fullsized":"2023-09-23T23:42:46.630660+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:57.416801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"678'9889","reported_seq":9960,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:57.800961+0000","last_change":"2023-09-23T23:39:13.742801+0000","last_active":"2023-09-23T23:42:57.800961+0000","last_peered":"2023-09-23T23:42:57.800961+0000","last_clean":"2023-09-23T23:42:57.800961+0000","last_became_active":"2023-09-23T23:39:13.742482+0000","last_became_peered":"2023-09-23T23:39:13.742482+0000","last_unstale":"2023-09-23T23:42:57.800961+0000","last_undegraded":"2023-09-23T23:42:57.800961+0000","last_fullsized":"2023-09-23T23:42:57.800961+0000","mapping_epoch":617,"log_start":"636'5042","ondisk_log_start":"636'5042","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4847,"log_dups_size":0,"ondisk_log_size":4847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:15.737461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22872064,"num_objects":349,"num_object_clones":0,"num_object_copies":1047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9889,"num_write_kb":327616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":605,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622179+0000","last_change":"2023-09-23T23:39:58.112420+0000","last_active":"2023-09-23T23:42:46.622179+0000","last_peered":"2023-09-23T23:42:46.622179+0000","last_clean":"2023-09-23T23:42:46.622179+0000","last_became_active":"2023-09-23T23:39:58.111949+0000","last_became_peered":"2023-09-23T23:39:58.111949+0000","last_unstale":"2023-09-23T23:42:46.622179+0000","last_undegraded":"2023-09-23T23:42:46.622179+0000","last_fullsized":"2023-09-23T23:42:46.622179+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:20.354620+0000","scrub_duration":0.0047245100000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":122,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.760614+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:42:53.760614+0000","last_peered":"2023-09-23T23:42:53.760614+0000","last_clean":"2023-09-23T23:42:53.760614+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:42:53.760614+0000","last_undegraded":"2023-09-23T23:42:53.760614+0000","last_fullsized":"2023-09-23T23:42:53.760614+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":221,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.630229+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:42:46.630229+0000","last_peered":"2023-09-23T23:42:46.630229+0000","last_clean":"2023-09-23T23:42:46.630229+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:42:46.630229+0000","last_undegraded":"2023-09-23T23:42:46.630229+0000","last_fullsized":"2023-09-23T23:42:46.630229+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":411,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.624386+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:42:46.624386+0000","last_peered":"2023-09-23T23:42:46.624386+0000","last_clean":"2023-09-23T23:42:46.624386+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:42:46.624386+0000","last_undegraded":"2023-09-23T23:42:46.624386+0000","last_fullsized":"2023-09-23T23:42:46.624386+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.a","version":"678'9907","reported_seq":9978,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:57.817183+0000","last_change":"2023-09-23T23:39:13.746851+0000","last_active":"2023-09-23T23:42:57.817183+0000","last_peered":"2023-09-23T23:42:57.817183+0000","last_clean":"2023-09-23T23:42:57.817183+0000","last_became_active":"2023-09-23T23:39:13.746412+0000","last_became_peered":"2023-09-23T23:39:13.746412+0000","last_unstale":"2023-09-23T23:42:57.817183+0000","last_undegraded":"2023-09-23T23:42:57.817183+0000","last_fullsized":"2023-09-23T23:42:57.817183+0000","mapping_epoch":617,"log_start":"636'4972","ondisk_log_start":"636'4972","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4935,"log_dups_size":0,"ondisk_log_size":4935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:39.662209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24969216,"num_objects":381,"num_object_clones":0,"num_object_copies":1143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9907,"num_write_kb":329216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":879,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.626758+0000","last_change":"2023-09-23T23:39:58.122659+0000","last_active":"2023-09-23T23:42:46.626758+0000","last_peered":"2023-09-23T23:42:46.626758+0000","last_clean":"2023-09-23T23:42:46.626758+0000","last_became_active":"2023-09-23T23:39:58.122334+0000","last_became_peered":"2023-09-23T23:39:58.122334+0000","last_unstale":"2023-09-23T23:42:46.626758+0000","last_undegraded":"2023-09-23T23:42:46.626758+0000","last_fullsized":"2023-09-23T23:42:46.626758+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:29.154690+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":678,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622975+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:42:46.622975+0000","last_peered":"2023-09-23T23:42:46.622975+0000","last_clean":"2023-09-23T23:42:46.622975+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:42:46.622975+0000","last_undegraded":"2023-09-23T23:42:46.622975+0000","last_fullsized":"2023-09-23T23:42:46.622975+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":221,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622028+0000","last_change":"2023-09-23T23:39:55.100517+0000","last_active":"2023-09-23T23:42:46.622028+0000","last_peered":"2023-09-23T23:42:46.622028+0000","last_clean":"2023-09-23T23:42:46.622028+0000","last_became_active":"2023-09-23T23:39:55.098755+0000","last_became_peered":"2023-09-23T23:39:55.098755+0000","last_unstale":"2023-09-23T23:42:46.622028+0000","last_undegraded":"2023-09-23T23:42:46.622028+0000","last_fullsized":"2023-09-23T23:42:46.622028+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:50.086720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":251,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:50.644873+0000","last_change":"2023-09-23T23:41:31.604932+0000","last_active":"2023-09-23T23:42:50.644873+0000","last_peered":"2023-09-23T23:42:50.644873+0000","last_clean":"2023-09-23T23:42:50.644873+0000","last_became_active":"2023-09-23T23:41:31.604686+0000","last_became_peered":"2023-09-23T23:41:31.604686+0000","last_unstale":"2023-09-23T23:42:50.644873+0000","last_undegraded":"2023-09-23T23:42:50.644873+0000","last_fullsized":"2023-09-23T23:42:50.644873+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.931148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.b","version":"678'9908","reported_seq":9979,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:55.636471+0000","last_change":"2023-09-23T23:39:13.744730+0000","last_active":"2023-09-23T23:42:55.636471+0000","last_peered":"2023-09-23T23:42:55.636471+0000","last_clean":"2023-09-23T23:42:55.636471+0000","last_became_active":"2023-09-23T23:39:13.744442+0000","last_became_peered":"2023-09-23T23:39:13.744442+0000","last_unstale":"2023-09-23T23:42:55.636471+0000","last_undegraded":"2023-09-23T23:42:55.636471+0000","last_fullsized":"2023-09-23T23:42:55.636471+0000","mapping_epoch":617,"log_start":"632'3695","ondisk_log_start":"632'3695","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6213,"log_dups_size":0,"ondisk_log_size":6213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:56:43.437044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30146560,"num_objects":460,"num_object_clones":0,"num_object_copies":1380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9908,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":1068,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.624075+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:42:46.624075+0000","last_peered":"2023-09-23T23:42:46.624075+0000","last_clean":"2023-09-23T23:42:46.624075+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:42:46.624075+0000","last_undegraded":"2023-09-23T23:42:46.624075+0000","last_fullsized":"2023-09-23T23:42:46.624075+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":394,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.629625+0000","last_change":"2023-09-23T23:39:55.096945+0000","last_active":"2023-09-23T23:42:46.629625+0000","last_peered":"2023-09-23T23:42:46.629625+0000","last_clean":"2023-09-23T23:42:46.629625+0000","last_became_active":"2023-09-23T23:39:55.096655+0000","last_became_peered":"2023-09-23T23:39:55.096655+0000","last_unstale":"2023-09-23T23:42:46.629625+0000","last_undegraded":"2023-09-23T23:42:46.629625+0000","last_fullsized":"2023-09-23T23:42:46.629625+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:27:24.290604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":39,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.630580+0000","last_change":"2023-09-23T23:41:31.599981+0000","last_active":"2023-09-23T23:42:46.630580+0000","last_peered":"2023-09-23T23:42:46.630580+0000","last_clean":"2023-09-23T23:42:46.630580+0000","last_became_active":"2023-09-23T23:41:31.597250+0000","last_became_peered":"2023-09-23T23:41:31.597250+0000","last_unstale":"2023-09-23T23:42:46.630580+0000","last_undegraded":"2023-09-23T23:42:46.630580+0000","last_fullsized":"2023-09-23T23:42:46.630580+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:06.361776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":262,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.623911+0000","last_change":"2023-09-23T23:39:55.092828+0000","last_active":"2023-09-23T23:42:46.623911+0000","last_peered":"2023-09-23T23:42:46.623911+0000","last_clean":"2023-09-23T23:42:46.623911+0000","last_became_active":"2023-09-23T23:39:55.092504+0000","last_became_peered":"2023-09-23T23:39:55.092504+0000","last_unstale":"2023-09-23T23:42:46.623911+0000","last_undegraded":"2023-09-23T23:42:46.623911+0000","last_fullsized":"2023-09-23T23:42:46.623911+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:59.997875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1168,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622000+0000","last_change":"2023-09-23T23:41:34.675056+0000","last_active":"2023-09-23T23:42:46.622000+0000","last_peered":"2023-09-23T23:42:46.622000+0000","last_clean":"2023-09-23T23:42:46.622000+0000","last_became_active":"2023-09-23T23:41:34.674727+0000","last_became_peered":"2023-09-23T23:41:34.674727+0000","last_unstale":"2023-09-23T23:42:46.622000+0000","last_undegraded":"2023-09-23T23:42:46.622000+0000","last_fullsized":"2023-09-23T23:42:46.622000+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:01.911339+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":46,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.631087+0000","last_change":"2023-09-23T23:42:28.952637+0000","last_active":"2023-09-23T23:42:46.631087+0000","last_peered":"2023-09-23T23:42:46.631087+0000","last_clean":"2023-09-23T23:42:46.631087+0000","last_became_active":"2023-09-23T23:42:28.952283+0000","last_became_peered":"2023-09-23T23:42:28.952283+0000","last_unstale":"2023-09-23T23:42:46.631087+0000","last_undegraded":"2023-09-23T23:42:46.631087+0000","last_fullsized":"2023-09-23T23:42:46.631087+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:14.296297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":436,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:50.647301+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:42:50.647301+0000","last_peered":"2023-09-23T23:42:50.647301+0000","last_clean":"2023-09-23T23:42:50.647301+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:42:50.647301+0000","last_undegraded":"2023-09-23T23:42:50.647301+0000","last_fullsized":"2023-09-23T23:42:50.647301+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":880,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622877+0000","last_change":"2023-09-23T23:41:34.667657+0000","last_active":"2023-09-23T23:42:46.622877+0000","last_peered":"2023-09-23T23:42:46.622877+0000","last_clean":"2023-09-23T23:42:46.622877+0000","last_became_active":"2023-09-23T23:41:34.659982+0000","last_became_peered":"2023-09-23T23:41:34.659982+0000","last_unstale":"2023-09-23T23:42:46.622877+0000","last_undegraded":"2023-09-23T23:42:46.622877+0000","last_fullsized":"2023-09-23T23:42:46.622877+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:49.342830+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":337,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.624378+0000","last_change":"2023-09-23T23:39:55.115869+0000","last_active":"2023-09-23T23:42:46.624378+0000","last_peered":"2023-09-23T23:42:46.624378+0000","last_clean":"2023-09-23T23:42:46.624378+0000","last_became_active":"2023-09-23T23:39:55.092827+0000","last_became_peered":"2023-09-23T23:39:55.092827+0000","last_unstale":"2023-09-23T23:42:46.624378+0000","last_undegraded":"2023-09-23T23:42:46.624378+0000","last_fullsized":"2023-09-23T23:42:46.624378+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:03.590720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":546,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.623263+0000","last_change":"2023-09-23T23:42:28.956777+0000","last_active":"2023-09-23T23:42:46.623263+0000","last_peered":"2023-09-23T23:42:46.623263+0000","last_clean":"2023-09-23T23:42:46.623263+0000","last_became_active":"2023-09-23T23:42:28.956436+0000","last_became_peered":"2023-09-23T23:42:28.956436+0000","last_unstale":"2023-09-23T23:42:46.623263+0000","last_undegraded":"2023-09-23T23:42:46.623263+0000","last_fullsized":"2023-09-23T23:42:46.623263+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:42.633268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":66,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.623126+0000","last_change":"2023-09-23T23:39:55.128319+0000","last_active":"2023-09-23T23:42:46.623126+0000","last_peered":"2023-09-23T23:42:46.623126+0000","last_clean":"2023-09-23T23:42:46.623126+0000","last_became_active":"2023-09-23T23:39:55.098936+0000","last_became_peered":"2023-09-23T23:39:55.098936+0000","last_unstale":"2023-09-23T23:42:46.623126+0000","last_undegraded":"2023-09-23T23:42:46.623126+0000","last_fullsized":"2023-09-23T23:42:46.623126+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:03.570174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":367,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622161+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:42:46.622161+0000","last_peered":"2023-09-23T23:42:46.622161+0000","last_clean":"2023-09-23T23:42:46.622161+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:42:46.622161+0000","last_undegraded":"2023-09-23T23:42:46.622161+0000","last_fullsized":"2023-09-23T23:42:46.622161+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":923,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622684+0000","last_change":"2023-09-23T23:39:58.111742+0000","last_active":"2023-09-23T23:42:46.622684+0000","last_peered":"2023-09-23T23:42:46.622684+0000","last_clean":"2023-09-23T23:42:46.622684+0000","last_became_active":"2023-09-23T23:39:58.111358+0000","last_became_peered":"2023-09-23T23:39:58.111358+0000","last_unstale":"2023-09-23T23:42:46.622684+0000","last_undegraded":"2023-09-23T23:42:46.622684+0000","last_fullsized":"2023-09-23T23:42:46.622684+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:54.588394+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":720,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622793+0000","last_change":"2023-09-23T23:39:55.101668+0000","last_active":"2023-09-23T23:42:46.622793+0000","last_peered":"2023-09-23T23:42:46.622793+0000","last_clean":"2023-09-23T23:42:46.622793+0000","last_became_active":"2023-09-23T23:39:55.100664+0000","last_became_peered":"2023-09-23T23:39:55.100664+0000","last_unstale":"2023-09-23T23:42:46.622793+0000","last_undegraded":"2023-09-23T23:42:46.622793+0000","last_fullsized":"2023-09-23T23:42:46.622793+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:43.354111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":695,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.622296+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:42:46.622296+0000","last_peered":"2023-09-23T23:42:46.622296+0000","last_clean":"2023-09-23T23:42:46.622296+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:42:46.622296+0000","last_undegraded":"2023-09-23T23:42:46.622296+0000","last_fullsized":"2023-09-23T23:42:46.622296+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":853,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.623357+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:42:46.623357+0000","last_peered":"2023-09-23T23:42:46.623357+0000","last_clean":"2023-09-23T23:42:46.623357+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:42:46.623357+0000","last_undegraded":"2023-09-23T23:42:46.623357+0000","last_fullsized":"2023-09-23T23:42:46.623357+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":554,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.629707+0000","last_change":"2023-09-23T23:41:31.624156+0000","last_active":"2023-09-23T23:42:46.629707+0000","last_peered":"2023-09-23T23:42:46.629707+0000","last_clean":"2023-09-23T23:42:46.629707+0000","last_became_active":"2023-09-23T23:41:31.612962+0000","last_became_peered":"2023-09-23T23:41:31.612962+0000","last_unstale":"2023-09-23T23:42:46.629707+0000","last_undegraded":"2023-09-23T23:42:46.629707+0000","last_fullsized":"2023-09-23T23:42:46.629707+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:31.042576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":678,"reported_epoch":677,"state":"active+clean","last_fresh":"2023-09-23T23:42:46.621724+0000","last_change":"2023-09-23T23:39:55.099784+0000","last_active":"2023-09-23T23:42:46.621724+0000","last_peered":"2023-09-23T23:42:46.621724+0000","last_clean":"2023-09-23T23:42:46.621724+0000","last_became_active":"2023-09-23T23:39:55.099483+0000","last_became_peered":"2023-09-23T23:39:55.099483+0000","last_unstale":"2023-09-23T23:42:46.621724+0000","last_undegraded":"2023-09-23T23:42:46.621724+0000","last_fullsized":"2023-09-23T23:42:46.621724+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:48.012026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":435683352,"num_objects":6649,"num_object_clones":0,"num_object_copies":19947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6649,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":161105,"num_write_kb":5368065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14934,"num_bytes_recovered":809631744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1416114176,"data_stored":1416101960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87657,"ondisk_log_size":87657,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":71,"num_bytes_recovered":12612,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":143360,"data_stored":26230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782762,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":636436,"kb_used_data":154380,"kb_used_omap":8,"kb_used_meta":482039,"kb_avail":93108716,"statfs":{"total":95995035648,"available":95343325184,"internally_reserved":0,"allocated":158085120,"data_stored":156109147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":493608005},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.0900000000000001,"15min":1.6180000000000001},"min":{"1min":0.23799999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":3.528,"5min":12.432,"15min":44.094999999999999},"last":0.77400000000000002},{"interface":"front","average":{"1min":1.353,"5min":1.3320000000000001,"15min":1.76},"min":{"1min":0.27100000000000002,"5min":0.22700000000000001,"15min":0.21299999999999999},"max":{"1min":4.4720000000000004,"5min":12.717000000000001,"15min":44.335999999999999},"last":5.7430000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":1.377,"15min":3.0409999999999999},"min":{"1min":0.28499999999999998,"5min":0.255,"15min":0.23200000000000001},"max":{"1min":5.2229999999999999,"5min":12.484,"15min":198.58000000000001},"last":1.268},{"interface":"front","average":{"1min":1.575,"5min":1.526,"15min":3.0030000000000001},"min":{"1min":0.28999999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.6749999999999998,"5min":13.145,"15min":198.71100000000001},"last":6.6970000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":2.524,"5min":2.3599999999999999,"15min":3.798},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":8.9779999999999998,"5min":46.374000000000002,"15min":203.58000000000001},"last":6.3399999999999999},{"interface":"front","average":{"1min":1.909,"5min":2.5230000000000001,"15min":3.4399999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26200000000000001},"max":{"1min":9.9670000000000005,"5min":46.412999999999997,"15min":198.69},"last":10.145}]},{"osd":3,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":2.2759999999999998,"5min":5.1429999999999998,"15min":5.2770000000000001},"min":{"1min":0.31,"5min":0.309,"15min":0.26700000000000002},"max":{"1min":8.1479999999999997,"5min":154.18100000000001,"15min":279.26499999999999},"last":6.4290000000000003},{"interface":"front","average":{"1min":2.931,"5min":5.0919999999999996,"15min":5.8150000000000004},"min":{"1min":0.32600000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":8.5779999999999994,"5min":154.16300000000001,"15min":279.29899999999998},"last":0.80200000000000005}]},{"osd":4,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.69199999999999995,"15min":1.008},"min":{"1min":0.29599999999999999,"5min":0.28999999999999998,"15min":0.246},"max":{"1min":1.5169999999999999,"5min":12.510999999999999,"15min":31.498999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.435,"5min":0.80100000000000005,"15min":0.93999999999999995},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.23899999999999999},"max":{"1min":0.82399999999999995,"5min":12.821999999999999,"15min":31.68},"last":0.373}]},{"osd":5,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":2.0659999999999998,"15min":2.2090000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.88300000000000001,"5min":119.417,"15min":210.83600000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":2.1669999999999998,"15min":2.1949999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.27700000000000002},"max":{"1min":0.54900000000000004,"5min":119.61499999999999,"15min":207.04599999999999},"last":0.26600000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.84199999999999997,"15min":1.149},"min":{"1min":0.34499999999999997,"5min":0.32800000000000001,"15min":0.29899999999999999},"max":{"1min":0.77900000000000003,"5min":6.2169999999999996,"15min":94.852999999999994},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.81899999999999995,"15min":1.274},"min":{"1min":0.36099999999999999,"5min":0.31,"15min":0.30599999999999999},"max":{"1min":0.84999999999999998,"5min":12.529999999999999,"15min":95.510999999999996},"last":0.38600000000000001}]}]},{"osd":6,"up_from":0,"seq":197568496239,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":112,"seq":481036337580,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1252520,"kb_used_data":224096,"kb_used_omap":4,"kb_used_meta":1028411,"kb_avail":92492632,"statfs":{"total":95995035648,"available":94712455168,"internally_reserved":0,"allocated":229474304,"data_stored":226146557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1053093774},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:42:31 2023","interfaces":[{"interface":"back","average":{"1min":3.0979999999999999,"5min":5.2750000000000004,"15min":3.9780000000000002},"min":{"1min":0.20999999999999999,"5min":0.16400000000000001,"15min":0.16400000000000001},"max":{"1min":9.1319999999999997,"5min":156.37799999999999,"15min":156.37799999999999},"last":1.006},{"interface":"front","average":{"1min":1.821,"5min":4.7320000000000002,"15min":3.4220000000000002},"min":{"1min":0.221,"5min":0.189,"15min":0.189},"max":{"1min":8.8049999999999997,"5min":161.05199999999999,"15min":161.05199999999999},"last":8.359}]},{"osd":2,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":5.9349999999999996,"5min":3.1960000000000002,"15min":6.0010000000000003},"min":{"1min":0.22900000000000001,"5min":0.184,"15min":0.184},"max":{"1min":49.311,"5min":53.951999999999998,"15min":231.67400000000001},"last":0.97699999999999998},{"interface":"front","average":{"1min":6.383,"5min":3.7789999999999999,"15min":6.2679999999999998},"min":{"1min":0.23400000000000001,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":48.991999999999997,"5min":54.213000000000001,"15min":232.21000000000001},"last":5.9880000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":18.759,"5min":6.4210000000000003,"15min":6.6829999999999998},"min":{"1min":0.25,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":272.839,"5min":272.839,"15min":272.839},"last":8.3239999999999998},{"interface":"front","average":{"1min":17.308,"5min":6.3499999999999996,"15min":6.5949999999999998},"min":{"1min":0.27700000000000002,"5min":0.245,"15min":0.224},"max":{"1min":272.76400000000001,"5min":272.76400000000001,"15min":272.76400000000001},"last":8.3900000000000006}]},{"osd":4,"last update":"Sat Sep 23 23:42:31 2023","interfaces":[{"interface":"back","average":{"1min":3.194,"5min":2.004,"15min":1.5600000000000001},"min":{"1min":0.42999999999999999,"5min":0.34000000000000002,"15min":0.29799999999999999},"max":{"1min":10.183,"5min":10.413,"15min":12.042999999999999},"last":2.657},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.2549999999999999,"15min":1.2669999999999999},"min":{"1min":0.36699999999999999,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":7.8760000000000003,"5min":13.622999999999999,"15min":18.108000000000001},"last":0.91700000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:42:31 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":2.23,"15min":2.1150000000000002},"min":{"1min":0.31,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":2.7890000000000001,"5min":80.302999999999997,"15min":151.94800000000001},"last":8.2400000000000002},{"interface":"front","average":{"1min":3.3090000000000002,"5min":2.8879999999999999,"15min":2.4279999999999999},"min":{"1min":0.39200000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.141999999999999,"5min":80.254000000000005,"15min":151.821},"last":2.7229999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:42:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":2.129,"15min":1.5740000000000001},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.32200000000000001},"max":{"1min":2.8180000000000001,"5min":87.837999999999994,"15min":87.837999999999994},"last":8.2889999999999997},{"interface":"front","average":{"1min":3.2200000000000002,"5min":2.8570000000000002,"15min":1.986},"min":{"1min":0.42999999999999999,"5min":0.312,"15min":0.312},"max":{"1min":10.233000000000001,"5min":85.518000000000001,"15min":85.518000000000001},"last":2.698}]},{"osd":7,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0169999999999999,"5min":1.6779999999999999,"15min":1.6319999999999999},"min":{"1min":0.40300000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":13.587999999999999,"5min":35.487000000000002,"15min":44.241},"last":0.95199999999999996},{"interface":"front","average":{"1min":3.069,"5min":2.3410000000000002,"15min":1.9350000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.210000000000001,"5min":35.146000000000001,"15min":43.920999999999999},"last":2.7450000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024403,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1698016,"kb_used_data":303768,"kb_used_omap":5,"kb_used_meta":1394042,"kb_avail":92047136,"statfs":{"total":95995035648,"available":94256267264,"internally_reserved":0,"allocated":311058432,"data_stored":307733384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1427499642},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:42:46 2023","interfaces":[{"interface":"back","average":{"1min":3.7349999999999999,"5min":2.3069999999999999,"15min":5.7999999999999998},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.188},"max":{"1min":10.502000000000001,"5min":15.589,"15min":301.12700000000001},"last":4.1040000000000001},{"interface":"front","average":{"1min":2.7799999999999998,"5min":2.4540000000000002,"15min":5.8029999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.186},"max":{"1min":18.503,"5min":18.503,"15min":297.72399999999999},"last":3.286}]},{"osd":2,"last update":"Sat Sep 23 23:42:13 2023","interfaces":[{"interface":"back","average":{"1min":5.476,"5min":3.8759999999999999,"15min":5.7949999999999999},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.17199999999999999},"max":{"1min":20.542000000000002,"5min":102.279,"15min":283.07400000000001},"last":4.4489999999999998},{"interface":"front","average":{"1min":4.3719999999999999,"5min":3.8420000000000001,"15min":5.9340000000000002},"min":{"1min":0.252,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":18.529,"5min":102.218,"15min":283.161},"last":7.6440000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:42:46 2023","interfaces":[{"interface":"back","average":{"1min":4.968,"5min":5.0869999999999997,"15min":6.6349999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.217},"max":{"1min":20.498999999999999,"5min":99.122,"15min":220.732},"last":4.2629999999999999},{"interface":"front","average":{"1min":5.2960000000000003,"5min":5.5499999999999998,"15min":6.9249999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.19800000000000001},"max":{"1min":28.460999999999999,"5min":98.373999999999995,"15min":218.78200000000001},"last":7.5999999999999996}]},{"osd":4,"last update":"Sat Sep 23 23:42:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":1.7,"15min":1.784},"min":{"1min":0.36899999999999999,"5min":0.32600000000000001,"15min":0.28499999999999998},"max":{"1min":18.274999999999999,"5min":18.274999999999999,"15min":108.01900000000001},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.554,"5min":1.5629999999999999,"15min":1.7729999999999999},"min":{"1min":0.39300000000000002,"5min":0.32600000000000001,"15min":0.27200000000000002},"max":{"1min":6.1310000000000002,"5min":8.718,"15min":107.898},"last":0.64800000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:42:24 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":3.117,"15min":1.9610000000000001},"min":{"1min":0.36799999999999999,"5min":0.36699999999999999,"15min":0.32600000000000001},"max":{"1min":6.0650000000000004,"5min":94.254000000000005,"15min":94.254000000000005},"last":0.75900000000000001},{"interface":"front","average":{"1min":3.1200000000000001,"5min":3.3780000000000001,"15min":1.962},"min":{"1min":0.45200000000000001,"5min":0.32900000000000001,"15min":0.253},"max":{"1min":9.5869999999999997,"5min":94.837000000000003,"15min":94.837000000000003},"last":4.1479999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:42:24 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":2.3690000000000002,"15min":2.0659999999999998},"min":{"1min":0.33500000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.0979999999999999,"5min":69.043999999999997,"15min":108.67},"last":0.621},{"interface":"front","average":{"1min":3.1629999999999998,"5min":2.597,"15min":2.121},"min":{"1min":0.52900000000000003,"5min":0.38200000000000001,"15min":0.27100000000000002},"max":{"1min":9.6310000000000002,"5min":69.028999999999996,"15min":108.804},"last":4.1929999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:42:13 2023","interfaces":[{"interface":"back","average":{"1min":2.5339999999999998,"5min":2.5129999999999999,"15min":2.2200000000000002},"min":{"1min":0.41299999999999998,"5min":0.35899999999999999,"15min":0.32900000000000001},"max":{"1min":18.469999999999999,"5min":66.620000000000005,"15min":111.71299999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":3.0710000000000002,"5min":2.4180000000000001,"15min":2.1840000000000002},"min":{"1min":0.496,"5min":0.35399999999999998,"15min":0.33300000000000002},"max":{"1min":9.6590000000000007,"5min":66.701999999999998,"15min":111.108},"last":4.2279999999999998}]}]},{"osd":2,"up_from":199,"seq":854698492224,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1285752,"kb_used_data":256048,"kb_used_omap":7,"kb_used_meta":1029624,"kb_avail":92459400,"statfs":{"total":95995035648,"available":94678425600,"internally_reserved":0,"allocated":262193152,"data_stored":258794105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1054335057},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":5.1479999999999997,"5min":3.2290000000000001,"15min":2.7490000000000001},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":34.648000000000003,"5min":65.001000000000005,"15min":65.001000000000005},"last":1.0029999999999999},{"interface":"front","average":{"1min":3.1379999999999999,"5min":2.5259999999999998,"15min":2.4670000000000001},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19700000000000001},"max":{"1min":13.132,"5min":64.346000000000004,"15min":64.346000000000004},"last":11.589}]},{"osd":1,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":2.8220000000000001,"5min":2.8279999999999998,"15min":3.8719999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.192},"max":{"1min":14.369,"5min":41.43,"15min":106.765},"last":5.6500000000000004},{"interface":"front","average":{"1min":3.4129999999999998,"5min":2.6200000000000001,"15min":3.7290000000000001},"min":{"1min":0.36299999999999999,"5min":0.24399999999999999,"15min":0.222},"max":{"1min":11.821999999999999,"5min":53.268000000000001,"15min":106.732},"last":6.4560000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":6.2300000000000004,"5min":3.1909999999999998,"15min":4.3090000000000002},"min":{"1min":0.315,"5min":0.245,"15min":0.24099999999999999},"max":{"1min":54.441000000000003,"5min":54.441000000000003,"15min":106.70399999999999},"last":5.4409999999999998},{"interface":"front","average":{"1min":6.1120000000000001,"5min":2.6720000000000002,"15min":3.9910000000000001},"min":{"1min":0.23899999999999999,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":53.734000000000002,"5min":53.734000000000002,"15min":106.66800000000001},"last":1.0429999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":3.23,"5min":1.5669999999999999,"15min":1.7929999999999999},"min":{"1min":0.41199999999999998,"5min":0.314,"15min":0.27700000000000002},"max":{"1min":16.001000000000001,"5min":16.001000000000001,"15min":62.081000000000003},"last":11.653},{"interface":"front","average":{"1min":1.6870000000000001,"5min":1.77,"15min":1.9259999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.28899999999999998},"max":{"1min":6.6849999999999996,"5min":15.055999999999999,"15min":62.072000000000003},"last":5.5149999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":4.2229999999999999,"15min":3.0110000000000001},"min":{"1min":0.38600000000000001,"5min":0.32200000000000001,"15min":0.26200000000000001},"max":{"1min":13.023,"5min":101.104,"15min":101.104},"last":11.683},{"interface":"front","average":{"1min":1.6830000000000001,"5min":4.681,"15min":3.1589999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.641,"5min":101.404,"15min":101.404},"last":5.4859999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2559999999999998,"5min":4.3710000000000004,"15min":3.3039999999999998},"min":{"1min":0.437,"5min":0.33400000000000002,"15min":0.31},"max":{"1min":15.898999999999999,"5min":97.558999999999997,"15min":107.018},"last":13.073},{"interface":"front","average":{"1min":1.71,"5min":4.7530000000000001,"15min":3.0590000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.7130000000000001,"5min":98.231999999999999,"15min":98.231999999999999},"last":5.54}]},{"osd":7,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":2.238,"15min":2.085},"min":{"1min":0.41899999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.9470000000000001,"5min":68.814999999999998,"15min":68.814999999999998},"last":1.069},{"interface":"front","average":{"1min":2.3879999999999999,"5min":2.4580000000000002,"15min":2.5659999999999998},"min":{"1min":0.45900000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":6.923,"5min":69.918999999999997,"15min":106.809},"last":1.0920000000000001}]}]},{"osd":3,"up_from":148,"seq":635655160193,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1335232,"kb_used_data":214584,"kb_used_omap":2,"kb_used_meta":1120445,"kb_avail":92409920,"statfs":{"total":95995035648,"available":94627758080,"internally_reserved":0,"allocated":219734016,"data_stored":216439170,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1147335733},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,1,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:41:58 2023","interfaces":[{"interface":"back","average":{"1min":8.3770000000000007,"5min":3.1070000000000002,"15min":2.3029999999999999},"min":{"1min":0.20300000000000001,"5min":0.19500000000000001,"15min":0.17699999999999999},"max":{"1min":69.281999999999996,"5min":69.281999999999996,"15min":69.281999999999996},"last":2.5990000000000002},{"interface":"front","average":{"1min":4.3600000000000003,"5min":2.2690000000000001,"15min":2.4209999999999998},"min":{"1min":0.247,"5min":0.188,"15min":0.17199999999999999},"max":{"1min":20.071000000000002,"5min":20.071000000000002,"15min":46.204000000000001},"last":1.343}]},{"osd":1,"last update":"Sat Sep 23 23:41:58 2023","interfaces":[{"interface":"back","average":{"1min":3.6040000000000001,"5min":1.738,"15min":2.7000000000000002},"min":{"1min":0.27000000000000002,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":9.1649999999999991,"5min":11.911,"15min":85.242999999999995},"last":21.824999999999999},{"interface":"front","average":{"1min":4.774,"5min":2.2759999999999998,"15min":2.7919999999999998},"min":{"1min":0.30099999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":20.779,"5min":20.779,"15min":85.302000000000007},"last":0.23599999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:42:06 2023","interfaces":[{"interface":"back","average":{"1min":3.835,"5min":6.1420000000000003,"15min":5.0220000000000002},"min":{"1min":0.26100000000000001,"5min":0.23999999999999999,"15min":0.219},"max":{"1min":9.4100000000000001,"5min":139.96299999999999,"15min":139.96299999999999},"last":31.564},{"interface":"front","average":{"1min":7.3789999999999996,"5min":7.0220000000000002,"15min":5.2649999999999997},"min":{"1min":0.30099999999999999,"5min":0.26800000000000002,"15min":0.25700000000000001},"max":{"1min":69.405000000000001,"5min":141.93199999999999,"15min":141.93199999999999},"last":0.309}]},{"osd":4,"last update":"Sat Sep 23 23:41:58 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":1.3939999999999999,"15min":1.3300000000000001},"min":{"1min":0.34100000000000003,"5min":0.308,"15min":0.26800000000000002},"max":{"1min":7.9790000000000001,"5min":11.414,"15min":11.414},"last":0.379},{"interface":"front","average":{"1min":2.3079999999999998,"5min":1.839,"15min":1.786},"min":{"1min":0.37,"5min":0.32900000000000001,"15min":0.26800000000000002},"max":{"1min":6.7439999999999998,"5min":14.967000000000001,"15min":24.782},"last":1.387}]},{"osd":5,"last update":"Sat Sep 23 23:41:58 2023","interfaces":[{"interface":"back","average":{"1min":2.363,"5min":5.4210000000000003,"15min":4.6289999999999996},"min":{"1min":0.36499999999999999,"5min":0.33500000000000002,"15min":0.29499999999999998},"max":{"1min":6.6630000000000003,"5min":196.358,"15min":214.43700000000001},"last":1.4419999999999999},{"interface":"front","average":{"1min":2.415,"5min":5.0709999999999997,"15min":4.5220000000000002},"min":{"1min":0.38500000000000001,"5min":0.32700000000000001,"15min":0.30599999999999999},"max":{"1min":9.2210000000000001,"5min":196.048,"15min":214.46600000000001},"last":11.765000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:41:58 2023","interfaces":[{"interface":"back","average":{"1min":2.3330000000000002,"5min":3.1560000000000001,"15min":2.5270000000000001},"min":{"1min":0.443,"5min":0.35799999999999998,"15min":0.318},"max":{"1min":6.718,"5min":125.94,"15min":125.94},"last":1.4179999999999999},{"interface":"front","average":{"1min":2.4580000000000002,"5min":2.8660000000000001,"15min":2.4209999999999998},"min":{"1min":0.40200000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.2469999999999999,"5min":125.955,"15min":125.955},"last":11.808999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:42:06 2023","interfaces":[{"interface":"back","average":{"1min":5.4770000000000003,"5min":2.27,"15min":3.5859999999999999},"min":{"1min":0.38900000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":69.197999999999993,"5min":69.197999999999993,"15min":192.69},"last":0.42699999999999999},{"interface":"front","average":{"1min":3.254,"5min":2.1070000000000002,"15min":3.8410000000000002},"min":{"1min":0.48899999999999999,"5min":0.39500000000000002,"15min":0.36699999999999999},"max":{"1min":12.568,"5min":14.923999999999999,"15min":192.65600000000001},"last":1.474}]}]},{"osd":4,"up_from":0,"seq":128849019512,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757875,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1004928,"kb_used_data":155448,"kb_used_omap":0,"kb_used_meta":849472,"kb_avail":92740224,"statfs":{"total":95995035648,"available":94965989376,"internally_reserved":0,"allocated":159178752,"data_stored":157218991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":869859328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:42:33 2023","interfaces":[{"interface":"back","average":{"1min":15.054,"5min":4.782,"15min":2.4399999999999999},"min":{"1min":0.251,"5min":0.223,"15min":0.20899999999999999},"max":{"1min":174.58600000000001,"5min":174.58600000000001,"15min":174.58600000000001},"last":0.23799999999999999},{"interface":"front","average":{"1min":14.362,"5min":5.0739999999999998,"15min":2.5750000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.22700000000000001},"max":{"1min":174.69499999999999,"5min":174.69499999999999,"15min":174.69499999999999},"last":0.28899999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:42:33 2023","interfaces":[{"interface":"back","average":{"1min":3.2789999999999999,"5min":3.0510000000000002,"15min":2.0489999999999999},"min":{"1min":0.28000000000000003,"5min":0.25600000000000001,"15min":0.23100000000000001},"max":{"1min":38.468000000000004,"5min":95.709000000000003,"15min":95.709000000000003},"last":3.5569999999999999},{"interface":"front","average":{"1min":3.4159999999999999,"5min":3.2789999999999999,"15min":2.198},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.249},"max":{"1min":38.841999999999999,"5min":94.966999999999999,"15min":94.966999999999999},"last":3.7000000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:42:01 2023","interfaces":[{"interface":"back","average":{"1min":2.8780000000000001,"5min":3.6030000000000002,"15min":4.4500000000000002},"min":{"1min":0.40100000000000002,"5min":0.28799999999999998,"15min":0.27200000000000002},"max":{"1min":9.5879999999999992,"5min":132.81999999999999,"15min":208.738},"last":1.8120000000000001},{"interface":"front","average":{"1min":2.472,"5min":3.169,"15min":3.8690000000000002},"min":{"1min":0.313,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.3360000000000003,"5min":133.09200000000001,"15min":208.631},"last":23.899000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:42:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":3.5550000000000002,"15min":4.1849999999999996},"min":{"1min":0.34599999999999997,"5min":0.33100000000000002,"15min":0.27800000000000002},"max":{"1min":14.288,"5min":143.66300000000001,"15min":143.66300000000001},"last":2.25},{"interface":"front","average":{"1min":2.552,"5min":3.6139999999999999,"15min":4.3410000000000002},"min":{"1min":0.44500000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":14.307,"5min":143.62200000000001,"15min":143.62200000000001},"last":2.2639999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:42:45 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.90200000000000002,"15min":0.93799999999999994},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.252},"max":{"1min":4.0629999999999997,"5min":6.9390000000000001,"15min":19.962},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.72199999999999998,"15min":0.879},"min":{"1min":0.316,"5min":0.307,"15min":0.22900000000000001},"max":{"1min":1.8560000000000001,"5min":6.7590000000000003,"15min":20.829999999999998},"last":0.33700000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:42:45 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.88400000000000001,"15min":1.327},"min":{"1min":0.29499999999999998,"5min":0.28899999999999998,"15min":0.26100000000000001},"max":{"1min":0.98099999999999998,"5min":14.252000000000001,"15min":148.36600000000001},"last":0.37},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.88900000000000001,"15min":1.522},"min":{"1min":0.35599999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.9610000000000001,"5min":6.8040000000000003,"15min":148.42500000000001},"last":0.40899999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:42:58 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.83199999999999996,"15min":1.7150000000000001},"min":{"1min":0.312,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.109,"5min":6.4199999999999999,"15min":179.28800000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.93400000000000005,"15min":1.8049999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":2.8690000000000002,"5min":6.4649999999999999,"15min":179.428},"last":0.436}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":294453248,"data_stored":294453248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":213909504,"data_stored":213909504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":249958400,"data_stored":249954328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":197787648,"data_stored":197787648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":129699840,"data_stored":129695768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":185204736,"data_stored":185204736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":145100800,"data_stored":145096728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:43:02.740 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:02.785 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:02.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:02.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:02.891 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:02.994 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:03.152 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:03.153 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:03.178 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:03 smithi110 ceph-mon[212939]: pgmap v1933: 119 pgs: 119 active+clean; 318 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T23:43:03.203 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:03.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:03.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:03.322 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:03.433 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:03 smithi110 ceph-mon[221102]: pgmap v1933: 119 pgs: 119 active+clean; 318 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T23:43:03.436 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:03.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:03 smithi138 ceph-mon[193606]: pgmap v1933: 119 pgs: 119 active+clean; 318 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 679 op/s 2023-09-23T23:43:03.675 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:03.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:03.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:03.782 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:03.811 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:03.859 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:03.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:03.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:03.966 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:03.994 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:04.048 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:04.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:04.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:04.156 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:43:04.218 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:43:04.218 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:04.255 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:04 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2106364519' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:43:04.259 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:04 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2106364519' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:43:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:04 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2106364519' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:43:05.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:05 smithi138 ceph-mon[193606]: pgmap v1934: 119 pgs: 119 active+clean; 212 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 652 op/s 2023-09-23T23:43:05.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:05 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:43:05.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:05 smithi138 ceph-mon[193606]: failed to encode map e679 with expected crc 2023-09-23T23:43:05.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:05 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2106364519' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:43:05.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:05 smithi138 ceph-mon[193606]: osdmap e679: 8 total, 8 up, 6 in 2023-09-23T23:43:05.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:05 smithi138 ceph-mon[193606]: failed to encode map e679 with expected crc 2023-09-23T23:43:05.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:05 smithi110 ceph-mon[221102]: pgmap v1934: 119 pgs: 119 active+clean; 212 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 652 op/s 2023-09-23T23:43:05.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:05 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:43:05.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:05 smithi110 ceph-mon[221102]: failed to encode map e679 with expected crc 2023-09-23T23:43:05.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:05 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2106364519' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:43:05.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:05 smithi110 ceph-mon[221102]: osdmap e679: 8 total, 8 up, 6 in 2023-09-23T23:43:05.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:05 smithi110 ceph-mon[221102]: failed to encode map e679 with expected crc 2023-09-23T23:43:05.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:05 smithi110 ceph-mon[212939]: pgmap v1934: 119 pgs: 119 active+clean; 212 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 652 op/s 2023-09-23T23:43:05.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:05 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:43:05.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:05 smithi110 ceph-mon[212939]: failed to encode map e679 with expected crc 2023-09-23T23:43:05.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:05 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2106364519' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:43:05.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:05 smithi110 ceph-mon[212939]: osdmap e679: 8 total, 8 up, 6 in 2023-09-23T23:43:05.456 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:05 smithi110 ceph-mon[212939]: failed to encode map e679 with expected crc 2023-09-23T23:43:05.741 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:43:06.232 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:43:06.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:06.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:43:06.342 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:06.431 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:06.682 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:06.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:06.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:06.693 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Removed 83876 objects 2023-09-23T23:43:06.693 INFO:tasks.radosbench.radosbench.0.smithi110.stdout:Clean up completed and total clean up time :133.962 2023-09-23T23:43:06.791 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:06.818 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:06.830 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:06.860 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:06.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:06.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:06.925 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:43:06.945 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:43:06.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:43:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:43:06] "GET /metrics HTTP/1.1" 200 36519 "" "Prometheus/2.43.0" 2023-09-23T23:43:06.964 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:43:06.967 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:06.985 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:43:06.997 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:07.008 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:43:07.034 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:43:07.048 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:07.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:07.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:07.062 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:43:07.100 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:43:07.100 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:43:07.100 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:43:07.101 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:43:07.101 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:43:07.101 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:43:07.101 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:43:07.101 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:43:07.101 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:43:07.152 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:07.181 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:07.222 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:07.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:07.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:07.261 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:07 smithi110 ceph-mon[221102]: pgmap v1936: 119 pgs: 119 active+clean; 110 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T23:43:07.261 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:07 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2484697034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:07.262 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:07 smithi110 ceph-mon[212939]: pgmap v1936: 119 pgs: 119 active+clean; 110 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T23:43:07.262 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:07 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2484697034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:07.271 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:43:07.271 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:43:07.271 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-09-23T23:43:07.271 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:43:07.331 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:07.360 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:07.402 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:07.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:07.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:07 smithi138 ceph-mon[193606]: pgmap v1936: 119 pgs: 119 active+clean; 110 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 675 op/s 2023-09-23T23:43:07.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:07 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2484697034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:07.507 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:07.533 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:07.575 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:07.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:07.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:07.682 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:07.711 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:08.185 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_9 2023-09-23T23:43:08.185 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_9 unique_pool_9 --yes-i-really-really-mean-it 2023-09-23T23:43:08.454 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:43:08.536 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:08.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:08.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:08.640 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:08.668 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:08.684 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:43:08.712 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:08.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:08.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:08.819 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:08.846 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:08.888 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:08.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:08.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:08.996 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:09.022 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:09.064 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:09.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:09.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:09.172 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:09.199 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:09.412 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:09.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:09 smithi138 ceph-mon[193606]: pgmap v1937: 119 pgs: 119 active+clean; 26 MiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T23:43:09.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:09 smithi110 ceph-mon[212939]: pgmap v1937: 119 pgs: 119 active+clean; 26 MiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T23:43:09.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:09 smithi110 ceph-mon[221102]: pgmap v1937: 119 pgs: 119 active+clean; 26 MiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T23:43:09.474 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:09.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:09.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:09.532 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:09.559 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:09.577 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:09.604 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:09.647 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:09.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:09.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:09.753 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:09.787 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:10.636 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:43:11.038 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:11.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:11.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:11.141 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:11.176 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:11.227 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:11.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:11.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:11.333 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:11.361 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:11.388 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:43:11.397 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:43:11.405 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:11.410 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:11 smithi110 ceph-mon[221102]: pgmap v1938: 119 pgs: 119 active+clean; 21 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T23:43:11.410 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:11 smithi110 ceph-mon[212939]: pgmap v1938: 119 pgs: 119 active+clean; 21 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T23:43:11.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:11.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:11.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:11 smithi138 ceph-mon[193606]: pgmap v1938: 119 pgs: 119 active+clean; 21 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T23:43:11.514 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:11.546 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:12.309 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:12.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:12.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:12.414 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:43:12.446 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:12.494 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:43:12.497 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:12 smithi138 ceph-mon[193606]: from='client.37347 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:43:12.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:12.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:43:12.600 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:12.628 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:12 smithi110 ceph-mon[212939]: from='client.37347 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:43:12.629 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:12 smithi110 ceph-mon[221102]: from='client.37347 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:43:12.631 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:12.903 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":1938,"stamp":"2023-09-23T23:43:10.129071+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":22222776,"num_objects":355,"num_object_clones":0,"num_object_copies":1065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":355,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":167484,"num_write_kb":5368094,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15009,"num_bytes_recovered":810955076,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92521,"ondisk_log_size":92521,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":6076820,"kb_used_data":114724,"kb_used_omap":29,"kb_used_meta":5961890,"kb_avail":556394092,"statfs":{"total":575970213888,"available":569747550208,"internally_reserved":0,"allocated":117477376,"data_stored":100179374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29997,"internal_metadata":6104976083},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-414056448,"num_objects":-6318,"num_object_clones":0,"num_object_copies":-18954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6318,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006429"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":875,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.755248+0000","last_change":"2023-09-23T23:28:06.356814+0000","last_active":"2023-09-23T23:42:53.755248+0000","last_peered":"2023-09-23T23:42:53.755248+0000","last_clean":"2023-09-23T23:42:53.755248+0000","last_became_active":"2023-09-23T23:26:50.556913+0000","last_became_peered":"2023-09-23T23:26:50.556913+0000","last_unstale":"2023-09-23T23:42:53.755248+0000","last_undegraded":"2023-09-23T23:42:53.755248+0000","last_fullsized":"2023-09-23T23:42:53.755248+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:04.901415+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":436,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.750580+0000","last_change":"2023-09-23T23:34:35.760917+0000","last_active":"2023-09-23T23:42:53.750580+0000","last_peered":"2023-09-23T23:42:53.750580+0000","last_clean":"2023-09-23T23:42:53.750580+0000","last_became_active":"2023-09-23T23:34:35.754679+0000","last_became_peered":"2023-09-23T23:34:35.754679+0000","last_unstale":"2023-09-23T23:42:53.750580+0000","last_undegraded":"2023-09-23T23:42:53.750580+0000","last_fullsized":"2023-09-23T23:42:53.750580+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:53.166818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":672,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.747361+0000","last_change":"2023-09-23T23:39:55.102353+0000","last_active":"2023-09-23T23:42:53.747361+0000","last_peered":"2023-09-23T23:42:53.747361+0000","last_clean":"2023-09-23T23:42:53.747361+0000","last_became_active":"2023-09-23T23:39:55.102017+0000","last_became_peered":"2023-09-23T23:39:55.102017+0000","last_unstale":"2023-09-23T23:42:53.747361+0000","last_undegraded":"2023-09-23T23:42:53.747361+0000","last_fullsized":"2023-09-23T23:42:53.747361+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:48.870443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":828,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.755344+0000","last_change":"2023-09-23T23:34:39.267534+0000","last_active":"2023-09-23T23:42:53.755344+0000","last_peered":"2023-09-23T23:42:53.755344+0000","last_clean":"2023-09-23T23:42:53.755344+0000","last_became_active":"2023-09-23T23:34:39.267228+0000","last_became_peered":"2023-09-23T23:34:39.267228+0000","last_unstale":"2023-09-23T23:42:53.755344+0000","last_undegraded":"2023-09-23T23:42:53.755344+0000","last_fullsized":"2023-09-23T23:42:53.755344+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:46.158995+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":409,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.748213+0000","last_change":"2023-09-23T23:39:55.104571+0000","last_active":"2023-09-23T23:42:53.748213+0000","last_peered":"2023-09-23T23:42:53.748213+0000","last_clean":"2023-09-23T23:42:53.748213+0000","last_became_active":"2023-09-23T23:39:55.104152+0000","last_became_peered":"2023-09-23T23:39:55.104152+0000","last_unstale":"2023-09-23T23:42:53.748213+0000","last_undegraded":"2023-09-23T23:42:53.748213+0000","last_fullsized":"2023-09-23T23:42:53.748213+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:31:52.551048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":643,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.743737+0000","last_change":"2023-09-23T23:34:35.753441+0000","last_active":"2023-09-23T23:42:53.743737+0000","last_peered":"2023-09-23T23:42:53.743737+0000","last_clean":"2023-09-23T23:42:53.743737+0000","last_became_active":"2023-09-23T23:34:35.752758+0000","last_became_peered":"2023-09-23T23:34:35.752758+0000","last_unstale":"2023-09-23T23:42:53.743737+0000","last_undegraded":"2023-09-23T23:42:53.743737+0000","last_fullsized":"2023-09-23T23:42:53.743737+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:43.554013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":840,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.756687+0000","last_change":"2023-09-23T23:33:54.579115+0000","last_active":"2023-09-23T23:42:53.756687+0000","last_peered":"2023-09-23T23:42:53.756687+0000","last_clean":"2023-09-23T23:42:53.756687+0000","last_became_active":"2023-09-23T23:33:54.578765+0000","last_became_peered":"2023-09-23T23:33:54.578765+0000","last_unstale":"2023-09-23T23:42:53.756687+0000","last_undegraded":"2023-09-23T23:42:53.756687+0000","last_fullsized":"2023-09-23T23:42:53.756687+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:05.634728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:05.634728+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:35.410721+0000","scrub_duration":0.0014175360000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":727,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.743679+0000","last_change":"2023-09-23T23:42:28.951264+0000","last_active":"2023-09-23T23:42:53.743679+0000","last_peered":"2023-09-23T23:42:53.743679+0000","last_clean":"2023-09-23T23:42:53.743679+0000","last_became_active":"2023-09-23T23:42:28.950966+0000","last_became_peered":"2023-09-23T23:42:28.950966+0000","last_unstale":"2023-09-23T23:42:53.743679+0000","last_undegraded":"2023-09-23T23:42:53.743679+0000","last_fullsized":"2023-09-23T23:42:53.743679+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:30.571439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":358,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.759821+0000","last_change":"2023-09-23T23:39:55.092989+0000","last_active":"2023-09-23T23:42:53.759821+0000","last_peered":"2023-09-23T23:42:53.759821+0000","last_clean":"2023-09-23T23:42:53.759821+0000","last_became_active":"2023-09-23T23:39:55.092680+0000","last_became_peered":"2023-09-23T23:39:55.092680+0000","last_unstale":"2023-09-23T23:42:53.759821+0000","last_undegraded":"2023-09-23T23:42:53.759821+0000","last_fullsized":"2023-09-23T23:42:53.759821+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:44.685469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":999,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.757253+0000","last_change":"2023-09-23T23:23:46.781144+0000","last_active":"2023-09-23T23:42:53.757253+0000","last_peered":"2023-09-23T23:42:53.757253+0000","last_clean":"2023-09-23T23:42:53.757253+0000","last_became_active":"2023-09-23T23:23:45.459387+0000","last_became_peered":"2023-09-23T23:23:45.459387+0000","last_unstale":"2023-09-23T23:42:53.757253+0000","last_undegraded":"2023-09-23T23:42:53.757253+0000","last_fullsized":"2023-09-23T23:42:53.757253+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.781045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.781045+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.087500+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":331,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.759759+0000","last_change":"2023-09-23T23:34:35.753857+0000","last_active":"2023-09-23T23:42:53.759759+0000","last_peered":"2023-09-23T23:42:53.759759+0000","last_clean":"2023-09-23T23:42:53.759759+0000","last_became_active":"2023-09-23T23:34:35.753556+0000","last_became_peered":"2023-09-23T23:34:35.753556+0000","last_unstale":"2023-09-23T23:42:53.759759+0000","last_undegraded":"2023-09-23T23:42:53.759759+0000","last_fullsized":"2023-09-23T23:42:53.759759+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:25.362861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":691,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.743605+0000","last_change":"2023-09-23T23:39:55.100428+0000","last_active":"2023-09-23T23:42:53.743605+0000","last_peered":"2023-09-23T23:42:53.743605+0000","last_clean":"2023-09-23T23:42:53.743605+0000","last_became_active":"2023-09-23T23:39:55.100094+0000","last_became_peered":"2023-09-23T23:39:55.100094+0000","last_unstale":"2023-09-23T23:42:53.743605+0000","last_undegraded":"2023-09-23T23:42:53.743605+0000","last_fullsized":"2023-09-23T23:42:53.743605+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:43.132618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":975,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.748704+0000","last_change":"2023-09-23T23:33:54.596799+0000","last_active":"2023-09-23T23:42:53.748704+0000","last_peered":"2023-09-23T23:42:53.748704+0000","last_clean":"2023-09-23T23:42:53.748704+0000","last_became_active":"2023-09-23T23:33:54.596476+0000","last_became_peered":"2023-09-23T23:33:54.596476+0000","last_unstale":"2023-09-23T23:42:53.748704+0000","last_undegraded":"2023-09-23T23:42:53.748704+0000","last_fullsized":"2023-09-23T23:42:53.748704+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:50.112268+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":320,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.755123+0000","last_change":"2023-09-23T23:42:46.645764+0000","last_active":"2023-09-23T23:42:53.755123+0000","last_peered":"2023-09-23T23:42:53.755123+0000","last_clean":"2023-09-23T23:42:53.755123+0000","last_became_active":"2023-09-23T23:42:46.645462+0000","last_became_peered":"2023-09-23T23:42:46.645462+0000","last_unstale":"2023-09-23T23:42:53.755123+0000","last_undegraded":"2023-09-23T23:42:53.755123+0000","last_fullsized":"2023-09-23T23:42:53.755123+0000","mapping_epoch":676,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":677,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:04.694952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":75,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.748126+0000","last_change":"2023-09-23T23:39:55.091152+0000","last_active":"2023-09-23T23:42:53.748126+0000","last_peered":"2023-09-23T23:42:53.748126+0000","last_clean":"2023-09-23T23:42:53.748126+0000","last_became_active":"2023-09-23T23:39:55.090833+0000","last_became_peered":"2023-09-23T23:39:55.090833+0000","last_unstale":"2023-09-23T23:42:53.748126+0000","last_undegraded":"2023-09-23T23:42:53.748126+0000","last_fullsized":"2023-09-23T23:42:53.748126+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:33.156366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1153,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.759635+0000","last_change":"2023-09-23T23:39:58.110527+0000","last_active":"2023-09-23T23:42:53.759635+0000","last_peered":"2023-09-23T23:42:53.759635+0000","last_clean":"2023-09-23T23:42:53.759635+0000","last_became_active":"2023-09-23T23:39:58.110194+0000","last_became_peered":"2023-09-23T23:39:58.110194+0000","last_unstale":"2023-09-23T23:42:53.759635+0000","last_undegraded":"2023-09-23T23:42:53.759635+0000","last_fullsized":"2023-09-23T23:42:53.759635+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T11:21:19.332102+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":354,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.747151+0000","last_change":"2023-09-23T23:34:18.771902+0000","last_active":"2023-09-23T23:42:53.747151+0000","last_peered":"2023-09-23T23:42:53.747151+0000","last_clean":"2023-09-23T23:42:53.747151+0000","last_became_active":"2023-09-23T23:34:18.746542+0000","last_became_peered":"2023-09-23T23:34:18.746542+0000","last_unstale":"2023-09-23T23:42:53.747151+0000","last_undegraded":"2023-09-23T23:42:53.747151+0000","last_fullsized":"2023-09-23T23:42:53.747151+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.503213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":365,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.748355+0000","last_change":"2023-09-23T23:42:28.956958+0000","last_active":"2023-09-23T23:42:53.748355+0000","last_peered":"2023-09-23T23:42:53.748355+0000","last_clean":"2023-09-23T23:42:53.748355+0000","last_became_active":"2023-09-23T23:42:28.956645+0000","last_became_peered":"2023-09-23T23:42:28.956645+0000","last_unstale":"2023-09-23T23:42:53.748355+0000","last_undegraded":"2023-09-23T23:42:53.748355+0000","last_fullsized":"2023-09-23T23:42:53.748355+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:16:28.472945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":544,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.743891+0000","last_change":"2023-09-23T23:39:19.011304+0000","last_active":"2023-09-23T23:42:53.743891+0000","last_peered":"2023-09-23T23:42:53.743891+0000","last_clean":"2023-09-23T23:42:53.743891+0000","last_became_active":"2023-09-23T23:34:03.720152+0000","last_became_peered":"2023-09-23T23:34:03.720152+0000","last_unstale":"2023-09-23T23:42:53.743891+0000","last_undegraded":"2023-09-23T23:42:53.743891+0000","last_fullsized":"2023-09-23T23:42:53.743891+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:39:19.011223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:39:19.011223+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:48:49.078060+0000","scrub_duration":0.001220575,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":199,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.748625+0000","last_change":"2023-09-23T23:39:55.129350+0000","last_active":"2023-09-23T23:42:53.748625+0000","last_peered":"2023-09-23T23:42:53.748625+0000","last_clean":"2023-09-23T23:42:53.748625+0000","last_became_active":"2023-09-23T23:39:55.092677+0000","last_became_peered":"2023-09-23T23:39:55.092677+0000","last_unstale":"2023-09-23T23:42:53.748625+0000","last_undegraded":"2023-09-23T23:42:53.748625+0000","last_fullsized":"2023-09-23T23:42:53.748625+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:38.925371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":553,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.755175+0000","last_change":"2023-09-23T23:39:55.097595+0000","last_active":"2023-09-23T23:42:53.755175+0000","last_peered":"2023-09-23T23:42:53.755175+0000","last_clean":"2023-09-23T23:42:53.755175+0000","last_became_active":"2023-09-23T23:39:55.097301+0000","last_became_peered":"2023-09-23T23:39:55.097301+0000","last_unstale":"2023-09-23T23:42:53.755175+0000","last_undegraded":"2023-09-23T23:42:53.755175+0000","last_fullsized":"2023-09-23T23:42:53.755175+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:26.508461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":834,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.756702+0000","last_change":"2023-09-23T23:34:39.269881+0000","last_active":"2023-09-23T23:42:53.756702+0000","last_peered":"2023-09-23T23:42:53.756702+0000","last_clean":"2023-09-23T23:42:53.756702+0000","last_became_active":"2023-09-23T23:34:39.269571+0000","last_became_peered":"2023-09-23T23:34:39.269571+0000","last_unstale":"2023-09-23T23:42:53.756702+0000","last_undegraded":"2023-09-23T23:42:53.756702+0000","last_fullsized":"2023-09-23T23:42:53.756702+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:38.631376+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":243,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.752204+0000","last_change":"2023-09-23T23:39:55.137107+0000","last_active":"2023-09-23T23:42:53.752204+0000","last_peered":"2023-09-23T23:42:53.752204+0000","last_clean":"2023-09-23T23:42:53.752204+0000","last_became_active":"2023-09-23T23:39:55.094309+0000","last_became_peered":"2023-09-23T23:39:55.094309+0000","last_unstale":"2023-09-23T23:42:53.752204+0000","last_undegraded":"2023-09-23T23:42:53.752204+0000","last_fullsized":"2023-09-23T23:42:53.752204+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:49.766841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":699,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.747789+0000","last_change":"2023-09-23T23:42:28.954052+0000","last_active":"2023-09-23T23:42:53.747789+0000","last_peered":"2023-09-23T23:42:53.747789+0000","last_clean":"2023-09-23T23:42:53.747789+0000","last_became_active":"2023-09-23T23:42:28.953681+0000","last_became_peered":"2023-09-23T23:42:28.953681+0000","last_unstale":"2023-09-23T23:42:53.747789+0000","last_undegraded":"2023-09-23T23:42:53.747789+0000","last_fullsized":"2023-09-23T23:42:53.747789+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:07.367387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":796,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.755093+0000","last_change":"2023-09-23T23:39:58.117169+0000","last_active":"2023-09-23T23:42:53.755093+0000","last_peered":"2023-09-23T23:42:53.755093+0000","last_clean":"2023-09-23T23:42:53.755093+0000","last_became_active":"2023-09-23T23:39:58.116815+0000","last_became_peered":"2023-09-23T23:39:58.116815+0000","last_unstale":"2023-09-23T23:42:53.755093+0000","last_undegraded":"2023-09-23T23:42:53.755093+0000","last_fullsized":"2023-09-23T23:42:53.755093+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:51.175189+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":276,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.746634+0000","last_change":"2023-09-23T23:34:35.752916+0000","last_active":"2023-09-23T23:42:53.746634+0000","last_peered":"2023-09-23T23:42:53.746634+0000","last_clean":"2023-09-23T23:42:53.746634+0000","last_became_active":"2023-09-23T23:34:35.752583+0000","last_became_peered":"2023-09-23T23:34:35.752583+0000","last_unstale":"2023-09-23T23:42:53.746634+0000","last_undegraded":"2023-09-23T23:42:53.746634+0000","last_fullsized":"2023-09-23T23:42:53.746634+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.865717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":403,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.747285+0000","last_change":"2023-09-23T23:34:18.772184+0000","last_active":"2023-09-23T23:42:53.747285+0000","last_peered":"2023-09-23T23:42:53.747285+0000","last_clean":"2023-09-23T23:42:53.747285+0000","last_became_active":"2023-09-23T23:34:18.746905+0000","last_became_peered":"2023-09-23T23:34:18.746905+0000","last_unstale":"2023-09-23T23:42:53.747285+0000","last_undegraded":"2023-09-23T23:42:53.747285+0000","last_fullsized":"2023-09-23T23:42:53.747285+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:16.599164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":644,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.749920+0000","last_change":"2023-09-23T23:34:03.720676+0000","last_active":"2023-09-23T23:42:53.749920+0000","last_peered":"2023-09-23T23:42:53.749920+0000","last_clean":"2023-09-23T23:42:53.749920+0000","last_became_active":"2023-09-23T23:34:03.720151+0000","last_became_peered":"2023-09-23T23:34:03.720151+0000","last_unstale":"2023-09-23T23:42:53.749920+0000","last_undegraded":"2023-09-23T23:42:53.749920+0000","last_fullsized":"2023-09-23T23:42:53.749920+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:40.915099+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":490,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.755155+0000","last_change":"2023-09-23T23:39:55.100855+0000","last_active":"2023-09-23T23:42:53.755155+0000","last_peered":"2023-09-23T23:42:53.755155+0000","last_clean":"2023-09-23T23:42:53.755155+0000","last_became_active":"2023-09-23T23:39:55.100577+0000","last_became_peered":"2023-09-23T23:39:55.100577+0000","last_unstale":"2023-09-23T23:42:53.755155+0000","last_undegraded":"2023-09-23T23:42:53.755155+0000","last_fullsized":"2023-09-23T23:42:53.755155+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:02.548093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":638,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.747651+0000","last_change":"2023-09-23T23:34:35.753644+0000","last_active":"2023-09-23T23:42:53.747651+0000","last_peered":"2023-09-23T23:42:53.747651+0000","last_clean":"2023-09-23T23:42:53.747651+0000","last_became_active":"2023-09-23T23:34:35.753167+0000","last_became_peered":"2023-09-23T23:34:35.753167+0000","last_unstale":"2023-09-23T23:42:53.747651+0000","last_undegraded":"2023-09-23T23:42:53.747651+0000","last_fullsized":"2023-09-23T23:42:53.747651+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:30.663254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":1102,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.757664+0000","last_change":"2023-09-23T23:42:32.073277+0000","last_active":"2023-09-23T23:42:53.757664+0000","last_peered":"2023-09-23T23:42:53.757664+0000","last_clean":"2023-09-23T23:42:53.757664+0000","last_became_active":"2023-09-23T23:42:32.072952+0000","last_became_peered":"2023-09-23T23:42:32.072952+0000","last_unstale":"2023-09-23T23:42:53.757664+0000","last_undegraded":"2023-09-23T23:42:53.757664+0000","last_fullsized":"2023-09-23T23:42:53.757664+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:19.983734+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":46,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.755091+0000","last_change":"2023-09-23T23:41:31.621623+0000","last_active":"2023-09-23T23:42:53.755091+0000","last_peered":"2023-09-23T23:42:53.755091+0000","last_clean":"2023-09-23T23:42:53.755091+0000","last_became_active":"2023-09-23T23:41:31.596928+0000","last_became_peered":"2023-09-23T23:41:31.596928+0000","last_unstale":"2023-09-23T23:42:53.755091+0000","last_undegraded":"2023-09-23T23:42:53.755091+0000","last_fullsized":"2023-09-23T23:42:53.755091+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:01.308674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":323,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:43:02.842166+0000","last_change":"2023-09-23T23:26:48.538301+0000","last_active":"2023-09-23T23:43:02.842166+0000","last_peered":"2023-09-23T23:43:02.842166+0000","last_clean":"2023-09-23T23:43:02.842166+0000","last_became_active":"2023-09-23T23:26:48.537844+0000","last_became_peered":"2023-09-23T23:26:48.537844+0000","last_unstale":"2023-09-23T23:43:02.842166+0000","last_undegraded":"2023-09-23T23:43:02.842166+0000","last_fullsized":"2023-09-23T23:43:02.842166+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:43.066957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"679'10432","reported_seq":16195,"reported_epoch":679,"state":"active+clean","last_fresh":"2023-09-23T23:43:06.621431+0000","last_change":"2023-09-23T23:41:11.967390+0000","last_active":"2023-09-23T23:43:06.621431+0000","last_peered":"2023-09-23T23:43:06.621431+0000","last_clean":"2023-09-23T23:43:06.621431+0000","last_became_active":"2023-09-23T23:41:11.955563+0000","last_became_peered":"2023-09-23T23:41:11.955563+0000","last_unstale":"2023-09-23T23:43:06.621431+0000","last_undegraded":"2023-09-23T23:43:06.621431+0000","last_fullsized":"2023-09-23T23:43:06.621431+0000","mapping_epoch":645,"log_start":"640'5586","ondisk_log_start":"640'5586","created":617,"last_epoch_clean":646,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4846,"log_dups_size":0,"ondisk_log_size":4846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:03.686224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10432,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2837,"num_bytes_recovered":185925632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":1015,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.753035+0000","last_change":"2023-09-23T23:34:18.725735+0000","last_active":"2023-09-23T23:42:53.753035+0000","last_peered":"2023-09-23T23:42:53.753035+0000","last_clean":"2023-09-23T23:42:53.753035+0000","last_became_active":"2023-09-23T23:34:18.725343+0000","last_became_peered":"2023-09-23T23:34:18.725343+0000","last_unstale":"2023-09-23T23:42:53.753035+0000","last_undegraded":"2023-09-23T23:42:53.753035+0000","last_fullsized":"2023-09-23T23:42:53.753035+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:55.185657+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":312,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.757665+0000","last_change":"2023-09-23T23:42:28.954407+0000","last_active":"2023-09-23T23:42:53.757665+0000","last_peered":"2023-09-23T23:42:53.757665+0000","last_clean":"2023-09-23T23:42:53.757665+0000","last_became_active":"2023-09-23T23:42:28.950556+0000","last_became_peered":"2023-09-23T23:42:28.950556+0000","last_unstale":"2023-09-23T23:42:53.757665+0000","last_undegraded":"2023-09-23T23:42:53.757665+0000","last_fullsized":"2023-09-23T23:42:53.757665+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:58.736585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":320,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.755199+0000","last_change":"2023-09-23T23:39:55.095319+0000","last_active":"2023-09-23T23:42:53.755199+0000","last_peered":"2023-09-23T23:42:53.755199+0000","last_clean":"2023-09-23T23:42:53.755199+0000","last_became_active":"2023-09-23T23:39:55.094677+0000","last_became_peered":"2023-09-23T23:39:55.094677+0000","last_unstale":"2023-09-23T23:42:53.755199+0000","last_undegraded":"2023-09-23T23:42:53.755199+0000","last_fullsized":"2023-09-23T23:42:53.755199+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:21.525905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"679'10346","reported_seq":10418,"reported_epoch":679,"state":"active+clean","last_fresh":"2023-09-23T23:43:06.621901+0000","last_change":"2023-09-23T23:39:13.746676+0000","last_active":"2023-09-23T23:43:06.621901+0000","last_peered":"2023-09-23T23:43:06.621901+0000","last_clean":"2023-09-23T23:43:06.621901+0000","last_became_active":"2023-09-23T23:39:13.746389+0000","last_became_peered":"2023-09-23T23:39:13.746389+0000","last_unstale":"2023-09-23T23:43:06.621901+0000","last_undegraded":"2023-09-23T23:43:06.621901+0000","last_fullsized":"2023-09-23T23:43:06.621901+0000","mapping_epoch":617,"log_start":"639'5497","ondisk_log_start":"639'5497","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4849,"log_dups_size":0,"ondisk_log_size":4849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:44.833933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10346,"num_write_kb":331072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1115,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.747572+0000","last_change":"2023-09-23T23:39:58.119994+0000","last_active":"2023-09-23T23:42:53.747572+0000","last_peered":"2023-09-23T23:42:53.747572+0000","last_clean":"2023-09-23T23:42:53.747572+0000","last_became_active":"2023-09-23T23:39:58.119654+0000","last_became_peered":"2023-09-23T23:39:58.119654+0000","last_unstale":"2023-09-23T23:42:53.747572+0000","last_undegraded":"2023-09-23T23:42:53.747572+0000","last_fullsized":"2023-09-23T23:42:53.747572+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.906679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.906679+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:02:41.967366+0000","scrub_duration":0.001686295,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":338,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.757485+0000","last_change":"2023-09-23T23:41:31.603533+0000","last_active":"2023-09-23T23:42:53.757485+0000","last_peered":"2023-09-23T23:42:53.757485+0000","last_clean":"2023-09-23T23:42:53.757485+0000","last_became_active":"2023-09-23T23:41:31.593485+0000","last_became_peered":"2023-09-23T23:41:31.593485+0000","last_unstale":"2023-09-23T23:42:53.757485+0000","last_undegraded":"2023-09-23T23:42:53.757485+0000","last_fullsized":"2023-09-23T23:42:53.757485+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:10:56.208677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":542,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.743637+0000","last_change":"2023-09-23T23:39:55.101024+0000","last_active":"2023-09-23T23:42:53.743637+0000","last_peered":"2023-09-23T23:42:53.743637+0000","last_clean":"2023-09-23T23:42:53.743637+0000","last_became_active":"2023-09-23T23:39:55.100745+0000","last_became_peered":"2023-09-23T23:39:55.100745+0000","last_unstale":"2023-09-23T23:42:53.743637+0000","last_undegraded":"2023-09-23T23:42:53.743637+0000","last_fullsized":"2023-09-23T23:42:53.743637+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:28.387953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.0","version":"679'10591","reported_seq":10663,"reported_epoch":679,"state":"active+clean","last_fresh":"2023-09-23T23:43:05.507642+0000","last_change":"2023-09-23T23:39:13.741363+0000","last_active":"2023-09-23T23:43:05.507642+0000","last_peered":"2023-09-23T23:43:05.507642+0000","last_clean":"2023-09-23T23:43:05.507642+0000","last_became_active":"2023-09-23T23:39:13.741098+0000","last_became_peered":"2023-09-23T23:39:13.741098+0000","last_unstale":"2023-09-23T23:43:05.507642+0000","last_undegraded":"2023-09-23T23:43:05.507642+0000","last_fullsized":"2023-09-23T23:43:05.507642+0000","mapping_epoch":617,"log_start":"633'4447","ondisk_log_start":"633'4447","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6144,"log_dups_size":0,"ondisk_log_size":6144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:31.131594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3211264,"num_objects":49,"num_object_clones":0,"num_object_copies":147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":49,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10591,"num_write_kb":340480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":846,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.757440+0000","last_change":"2023-09-23T23:39:58.122462+0000","last_active":"2023-09-23T23:42:53.757440+0000","last_peered":"2023-09-23T23:42:53.757440+0000","last_clean":"2023-09-23T23:42:53.757440+0000","last_became_active":"2023-09-23T23:39:58.122163+0000","last_became_peered":"2023-09-23T23:39:58.122163+0000","last_unstale":"2023-09-23T23:42:53.757440+0000","last_undegraded":"2023-09-23T23:42:53.757440+0000","last_fullsized":"2023-09-23T23:42:53.757440+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:41.511681+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":431,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.756206+0000","last_change":"2023-09-23T23:39:55.141520+0000","last_active":"2023-09-23T23:42:53.756206+0000","last_peered":"2023-09-23T23:42:53.756206+0000","last_clean":"2023-09-23T23:42:53.756206+0000","last_became_active":"2023-09-23T23:39:55.099176+0000","last_became_peered":"2023-09-23T23:39:55.099176+0000","last_unstale":"2023-09-23T23:42:53.756206+0000","last_undegraded":"2023-09-23T23:42:53.756206+0000","last_fullsized":"2023-09-23T23:42:53.756206+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:12.860173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":61,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.757222+0000","last_change":"2023-09-23T23:39:55.094219+0000","last_active":"2023-09-23T23:42:53.757222+0000","last_peered":"2023-09-23T23:42:53.757222+0000","last_clean":"2023-09-23T23:42:53.757222+0000","last_became_active":"2023-09-23T23:39:55.093849+0000","last_became_peered":"2023-09-23T23:39:55.093849+0000","last_unstale":"2023-09-23T23:42:53.757222+0000","last_undegraded":"2023-09-23T23:42:53.757222+0000","last_fullsized":"2023-09-23T23:42:53.757222+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:54.576029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.1","version":"679'10578","reported_seq":10650,"reported_epoch":679,"state":"active+clean","last_fresh":"2023-09-23T23:43:06.641882+0000","last_change":"2023-09-23T23:39:13.744449+0000","last_active":"2023-09-23T23:43:06.641882+0000","last_peered":"2023-09-23T23:43:06.641882+0000","last_clean":"2023-09-23T23:43:06.641882+0000","last_became_active":"2023-09-23T23:39:13.744163+0000","last_became_peered":"2023-09-23T23:39:13.744163+0000","last_unstale":"2023-09-23T23:43:06.641882+0000","last_undegraded":"2023-09-23T23:43:06.641882+0000","last_fullsized":"2023-09-23T23:43:06.641882+0000","mapping_epoch":617,"log_start":"639'5646","ondisk_log_start":"639'5646","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4932,"log_dups_size":0,"ondisk_log_size":4932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:50.120382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10578,"num_write_kb":338496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":588,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.743968+0000","last_change":"2023-09-23T23:34:18.727022+0000","last_active":"2023-09-23T23:42:53.743968+0000","last_peered":"2023-09-23T23:42:53.743968+0000","last_clean":"2023-09-23T23:42:53.743968+0000","last_became_active":"2023-09-23T23:34:18.726723+0000","last_became_peered":"2023-09-23T23:34:18.726723+0000","last_unstale":"2023-09-23T23:42:53.743968+0000","last_undegraded":"2023-09-23T23:42:53.743968+0000","last_fullsized":"2023-09-23T23:42:53.743968+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:03.629519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:03.629519+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:59.323943+0000","scrub_duration":0.0061887540000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.748428+0000","last_change":"2023-09-23T23:42:46.697762+0000","last_active":"2023-09-23T23:42:53.748428+0000","last_peered":"2023-09-23T23:42:53.748428+0000","last_clean":"2023-09-23T23:42:53.748428+0000","last_became_active":"2023-09-23T23:42:46.697414+0000","last_became_peered":"2023-09-23T23:42:46.697414+0000","last_unstale":"2023-09-23T23:42:53.748428+0000","last_undegraded":"2023-09-23T23:42:53.748428+0000","last_fullsized":"2023-09-23T23:42:53.748428+0000","mapping_epoch":676,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":677,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:11.272589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":679,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.757127+0000","last_change":"2023-09-23T23:33:51.541200+0000","last_active":"2023-09-23T23:42:53.757127+0000","last_peered":"2023-09-23T23:42:53.757127+0000","last_clean":"2023-09-23T23:42:53.757127+0000","last_became_active":"2023-09-23T23:33:51.540876+0000","last_became_peered":"2023-09-23T23:33:51.540876+0000","last_unstale":"2023-09-23T23:42:53.757127+0000","last_undegraded":"2023-09-23T23:42:53.757127+0000","last_fullsized":"2023-09-23T23:42:53.757127+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:23.746996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"679'10648","reported_seq":10720,"reported_epoch":679,"state":"active+clean","last_fresh":"2023-09-23T23:43:05.449781+0000","last_change":"2023-09-23T23:39:13.745430+0000","last_active":"2023-09-23T23:43:05.449781+0000","last_peered":"2023-09-23T23:43:05.449781+0000","last_clean":"2023-09-23T23:43:05.449781+0000","last_became_active":"2023-09-23T23:39:13.744751+0000","last_became_peered":"2023-09-23T23:39:13.744751+0000","last_unstale":"2023-09-23T23:43:05.449781+0000","last_undegraded":"2023-09-23T23:43:05.449781+0000","last_fullsized":"2023-09-23T23:43:05.449781+0000","mapping_epoch":617,"log_start":"633'4481","ondisk_log_start":"633'4481","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6167,"log_dups_size":0,"ondisk_log_size":6167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:23.195329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3538944,"num_objects":54,"num_object_clones":0,"num_object_copies":162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":54,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10648,"num_write_kb":342464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":822,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:43:02.841870+0000","last_change":"2023-09-23T23:39:19.985630+0000","last_active":"2023-09-23T23:43:02.841870+0000","last_peered":"2023-09-23T23:43:02.841870+0000","last_clean":"2023-09-23T23:43:02.841870+0000","last_became_active":"2023-09-23T23:34:18.725270+0000","last_became_peered":"2023-09-23T23:34:18.725270+0000","last_unstale":"2023-09-23T23:43:02.841870+0000","last_undegraded":"2023-09-23T23:43:02.841870+0000","last_fullsized":"2023-09-23T23:43:02.841870+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:39:19.985512+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:39:19.985512+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:52.750567+0000","scrub_duration":0.001852713,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":718,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.749534+0000","last_change":"2023-09-23T23:41:31.618278+0000","last_active":"2023-09-23T23:42:53.749534+0000","last_peered":"2023-09-23T23:42:53.749534+0000","last_clean":"2023-09-23T23:42:53.749534+0000","last_became_active":"2023-09-23T23:41:31.602519+0000","last_became_peered":"2023-09-23T23:41:31.602519+0000","last_unstale":"2023-09-23T23:42:53.749534+0000","last_undegraded":"2023-09-23T23:42:53.749534+0000","last_fullsized":"2023-09-23T23:42:53.749534+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:20.395010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":612,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.747658+0000","last_change":"2023-09-23T23:39:55.099345+0000","last_active":"2023-09-23T23:42:53.747658+0000","last_peered":"2023-09-23T23:42:53.747658+0000","last_clean":"2023-09-23T23:42:53.747658+0000","last_became_active":"2023-09-23T23:39:55.098875+0000","last_became_peered":"2023-09-23T23:39:55.098875+0000","last_unstale":"2023-09-23T23:42:53.747658+0000","last_undegraded":"2023-09-23T23:42:53.747658+0000","last_fullsized":"2023-09-23T23:42:53.747658+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:09.169232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.7","version":"679'10646","reported_seq":10716,"reported_epoch":679,"state":"active+clean","last_fresh":"2023-09-23T23:43:06.645962+0000","last_change":"2023-09-23T23:39:13.743009+0000","last_active":"2023-09-23T23:43:06.645962+0000","last_peered":"2023-09-23T23:43:06.645962+0000","last_clean":"2023-09-23T23:43:06.645962+0000","last_became_active":"2023-09-23T23:39:13.742719+0000","last_became_peered":"2023-09-23T23:39:13.742719+0000","last_unstale":"2023-09-23T23:43:06.645962+0000","last_undegraded":"2023-09-23T23:43:06.645962+0000","last_fullsized":"2023-09-23T23:43:06.645962+0000","mapping_epoch":617,"log_start":"637'5376","ondisk_log_start":"637'5376","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":5270,"log_dups_size":0,"ondisk_log_size":5270,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:50.189642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10646,"num_write_kb":340672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":965,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.749142+0000","last_change":"2023-09-23T23:26:34.563423+0000","last_active":"2023-09-23T23:42:53.749142+0000","last_peered":"2023-09-23T23:42:53.749142+0000","last_clean":"2023-09-23T23:42:53.749142+0000","last_became_active":"2023-09-23T23:26:34.563089+0000","last_became_peered":"2023-09-23T23:26:34.563089+0000","last_unstale":"2023-09-23T23:42:53.749142+0000","last_undegraded":"2023-09-23T23:42:53.749142+0000","last_fullsized":"2023-09-23T23:42:53.749142+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:51.712550+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":154,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.746373+0000","last_change":"2023-09-23T23:42:28.953222+0000","last_active":"2023-09-23T23:42:53.746373+0000","last_peered":"2023-09-23T23:42:53.746373+0000","last_clean":"2023-09-23T23:42:53.746373+0000","last_became_active":"2023-09-23T23:42:28.952902+0000","last_became_peered":"2023-09-23T23:42:28.952902+0000","last_unstale":"2023-09-23T23:42:53.746373+0000","last_undegraded":"2023-09-23T23:42:53.746373+0000","last_fullsized":"2023-09-23T23:42:53.746373+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:21.715883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":575,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:55.663559+0000","last_change":"2023-09-23T23:41:31.604354+0000","last_active":"2023-09-23T23:42:55.663559+0000","last_peered":"2023-09-23T23:42:55.663559+0000","last_clean":"2023-09-23T23:42:55.663559+0000","last_became_active":"2023-09-23T23:41:31.604035+0000","last_became_peered":"2023-09-23T23:41:31.604035+0000","last_unstale":"2023-09-23T23:42:55.663559+0000","last_undegraded":"2023-09-23T23:42:55.663559+0000","last_fullsized":"2023-09-23T23:42:55.663559+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:55.798632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"679'10450","reported_seq":16285,"reported_epoch":679,"state":"active+clean","last_fresh":"2023-09-23T23:43:06.548935+0000","last_change":"2023-09-23T23:40:33.082798+0000","last_active":"2023-09-23T23:43:06.548935+0000","last_peered":"2023-09-23T23:43:06.548935+0000","last_clean":"2023-09-23T23:43:06.548935+0000","last_became_active":"2023-09-23T23:40:33.076970+0000","last_became_peered":"2023-09-23T23:40:33.076970+0000","last_unstale":"2023-09-23T23:43:06.548935+0000","last_undegraded":"2023-09-23T23:43:06.548935+0000","last_fullsized":"2023-09-23T23:43:06.548935+0000","mapping_epoch":634,"log_start":"640'5569","ondisk_log_start":"640'5569","created":617,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4881,"log_dups_size":0,"ondisk_log_size":4881,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:08.992451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10450,"num_write_kb":334400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2873,"num_bytes_recovered":188284928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":687,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.755261+0000","last_change":"2023-09-23T23:34:39.293194+0000","last_active":"2023-09-23T23:42:53.755261+0000","last_peered":"2023-09-23T23:42:53.755261+0000","last_clean":"2023-09-23T23:42:53.755261+0000","last_became_active":"2023-09-23T23:34:39.292804+0000","last_became_peered":"2023-09-23T23:34:39.292804+0000","last_unstale":"2023-09-23T23:42:53.755261+0000","last_undegraded":"2023-09-23T23:42:53.755261+0000","last_fullsized":"2023-09-23T23:42:53.755261+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.322881+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":545,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.748349+0000","last_change":"2023-09-23T23:33:51.549125+0000","last_active":"2023-09-23T23:42:53.748349+0000","last_peered":"2023-09-23T23:42:53.748349+0000","last_clean":"2023-09-23T23:42:53.748349+0000","last_became_active":"2023-09-23T23:33:51.548819+0000","last_became_peered":"2023-09-23T23:33:51.548819+0000","last_unstale":"2023-09-23T23:42:53.748349+0000","last_undegraded":"2023-09-23T23:42:53.748349+0000","last_fullsized":"2023-09-23T23:42:53.748349+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:17.074717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":581,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.743461+0000","last_change":"2023-09-23T23:42:28.951851+0000","last_active":"2023-09-23T23:42:53.743461+0000","last_peered":"2023-09-23T23:42:53.743461+0000","last_clean":"2023-09-23T23:42:53.743461+0000","last_became_active":"2023-09-23T23:42:28.951542+0000","last_became_peered":"2023-09-23T23:42:28.951542+0000","last_unstale":"2023-09-23T23:42:53.743461+0000","last_undegraded":"2023-09-23T23:42:53.743461+0000","last_fullsized":"2023-09-23T23:42:53.743461+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:13.264990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.5","version":"679'10530","reported_seq":10602,"reported_epoch":679,"state":"active+clean","last_fresh":"2023-09-23T23:43:06.682168+0000","last_change":"2023-09-23T23:39:13.742854+0000","last_active":"2023-09-23T23:43:06.682168+0000","last_peered":"2023-09-23T23:43:06.682168+0000","last_clean":"2023-09-23T23:43:06.682168+0000","last_became_active":"2023-09-23T23:39:13.742579+0000","last_became_peered":"2023-09-23T23:39:13.742579+0000","last_unstale":"2023-09-23T23:43:06.682168+0000","last_undegraded":"2023-09-23T23:43:06.682168+0000","last_fullsized":"2023-09-23T23:43:06.682168+0000","mapping_epoch":617,"log_start":"640'5626","ondisk_log_start":"640'5626","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4904,"log_dups_size":0,"ondisk_log_size":4904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:51.872071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10530,"num_write_kb":336960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":1040,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.748275+0000","last_change":"2023-09-23T23:39:58.119762+0000","last_active":"2023-09-23T23:42:53.748275+0000","last_peered":"2023-09-23T23:42:53.748275+0000","last_clean":"2023-09-23T23:42:53.748275+0000","last_became_active":"2023-09-23T23:39:58.119422+0000","last_became_peered":"2023-09-23T23:39:58.119422+0000","last_unstale":"2023-09-23T23:42:53.748275+0000","last_undegraded":"2023-09-23T23:42:53.748275+0000","last_fullsized":"2023-09-23T23:42:53.748275+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.876891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.876891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:03.564930+0000","scrub_duration":0.001627912,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":323,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.743447+0000","last_change":"2023-09-23T23:26:48.538510+0000","last_active":"2023-09-23T23:42:53.743447+0000","last_peered":"2023-09-23T23:42:53.743447+0000","last_clean":"2023-09-23T23:42:53.743447+0000","last_became_active":"2023-09-23T23:26:48.534965+0000","last_became_peered":"2023-09-23T23:26:48.534965+0000","last_unstale":"2023-09-23T23:42:53.743447+0000","last_undegraded":"2023-09-23T23:42:53.743447+0000","last_fullsized":"2023-09-23T23:42:53.743447+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:39.151258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":611,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.748243+0000","last_change":"2023-09-23T23:34:03.723555+0000","last_active":"2023-09-23T23:42:53.748243+0000","last_peered":"2023-09-23T23:42:53.748243+0000","last_clean":"2023-09-23T23:42:53.748243+0000","last_became_active":"2023-09-23T23:34:03.723220+0000","last_became_peered":"2023-09-23T23:34:03.723220+0000","last_unstale":"2023-09-23T23:42:53.748243+0000","last_undegraded":"2023-09-23T23:42:53.748243+0000","last_fullsized":"2023-09-23T23:42:53.748243+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.003020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.8","version":"679'10606","reported_seq":12016,"reported_epoch":679,"state":"active+clean","last_fresh":"2023-09-23T23:43:05.488625+0000","last_change":"2023-09-23T23:42:28.980646+0000","last_active":"2023-09-23T23:43:05.488625+0000","last_peered":"2023-09-23T23:43:05.488625+0000","last_clean":"2023-09-23T23:43:05.488625+0000","last_became_active":"2023-09-23T23:42:28.958020+0000","last_became_peered":"2023-09-23T23:42:28.958020+0000","last_unstale":"2023-09-23T23:43:05.488625+0000","last_undegraded":"2023-09-23T23:43:05.488625+0000","last_fullsized":"2023-09-23T23:43:05.488625+0000","mapping_epoch":669,"log_start":"633'4417","ondisk_log_start":"633'4417","created":617,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6189,"log_dups_size":0,"ondisk_log_size":6189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:21.232496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2883584,"num_objects":44,"num_object_clones":0,"num_object_copies":132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":44,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10606,"num_write_kb":340800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":656,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":884,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.747856+0000","last_change":"2023-09-23T23:39:58.120057+0000","last_active":"2023-09-23T23:42:53.747856+0000","last_peered":"2023-09-23T23:42:53.747856+0000","last_clean":"2023-09-23T23:42:53.747856+0000","last_became_active":"2023-09-23T23:39:58.119647+0000","last_became_peered":"2023-09-23T23:39:58.119647+0000","last_unstale":"2023-09-23T23:42:53.747856+0000","last_undegraded":"2023-09-23T23:42:53.747856+0000","last_fullsized":"2023-09-23T23:42:53.747856+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:50.965369+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":391,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.735281+0000","last_change":"2023-09-23T23:34:18.723907+0000","last_active":"2023-09-23T23:42:53.735281+0000","last_peered":"2023-09-23T23:42:53.735281+0000","last_clean":"2023-09-23T23:42:53.735281+0000","last_became_active":"2023-09-23T23:34:18.723372+0000","last_became_peered":"2023-09-23T23:34:18.723372+0000","last_unstale":"2023-09-23T23:42:53.735281+0000","last_undegraded":"2023-09-23T23:42:53.735281+0000","last_fullsized":"2023-09-23T23:42:53.735281+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:01.020839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":694,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.756344+0000","last_change":"2023-09-23T23:20:15.777896+0000","last_active":"2023-09-23T23:42:53.756344+0000","last_peered":"2023-09-23T23:42:53.756344+0000","last_clean":"2023-09-23T23:42:53.756344+0000","last_became_active":"2023-09-23T23:20:15.777221+0000","last_became_peered":"2023-09-23T23:20:15.777221+0000","last_unstale":"2023-09-23T23:42:53.756344+0000","last_undegraded":"2023-09-23T23:42:53.756344+0000","last_fullsized":"2023-09-23T23:42:53.756344+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:50:14.737628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":295,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.757403+0000","last_change":"2023-09-23T23:42:28.950994+0000","last_active":"2023-09-23T23:42:53.757403+0000","last_peered":"2023-09-23T23:42:53.757403+0000","last_clean":"2023-09-23T23:42:53.757403+0000","last_became_active":"2023-09-23T23:42:28.950464+0000","last_became_peered":"2023-09-23T23:42:28.950464+0000","last_unstale":"2023-09-23T23:42:53.757403+0000","last_undegraded":"2023-09-23T23:42:53.757403+0000","last_fullsized":"2023-09-23T23:42:53.757403+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:57.975234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.e","version":"679'10453","reported_seq":16436,"reported_epoch":679,"state":"active+clean","last_fresh":"2023-09-23T23:43:05.449832+0000","last_change":"2023-09-23T23:42:21.783482+0000","last_active":"2023-09-23T23:43:05.449832+0000","last_peered":"2023-09-23T23:43:05.449832+0000","last_clean":"2023-09-23T23:43:05.449832+0000","last_became_active":"2023-09-23T23:42:21.754054+0000","last_became_peered":"2023-09-23T23:42:21.754054+0000","last_unstale":"2023-09-23T23:43:05.449832+0000","last_undegraded":"2023-09-23T23:43:05.449832+0000","last_fullsized":"2023-09-23T23:43:05.449832+0000","mapping_epoch":667,"log_start":"633'4318","ondisk_log_start":"633'4318","created":617,"last_epoch_clean":668,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6135,"log_dups_size":0,"ondisk_log_size":6135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:33.850501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2818048,"num_objects":43,"num_object_clones":0,"num_object_copies":129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10453,"num_write_kb":335872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2945,"num_bytes_recovered":134676480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":766,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:43:01.200385+0000","last_change":"2023-09-23T23:34:39.262614+0000","last_active":"2023-09-23T23:43:01.200385+0000","last_peered":"2023-09-23T23:43:01.200385+0000","last_clean":"2023-09-23T23:43:01.200385+0000","last_became_active":"2023-09-23T23:34:39.262102+0000","last_became_peered":"2023-09-23T23:34:39.262102+0000","last_unstale":"2023-09-23T23:43:01.200385+0000","last_undegraded":"2023-09-23T23:43:01.200385+0000","last_fullsized":"2023-09-23T23:43:01.200385+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:33:09.592806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:33:09.592806+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:03.064305+0000","scrub_duration":0.0023362629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":332,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.757530+0000","last_change":"2023-09-23T23:39:55.089555+0000","last_active":"2023-09-23T23:42:53.757530+0000","last_peered":"2023-09-23T23:42:53.757530+0000","last_clean":"2023-09-23T23:42:53.757530+0000","last_became_active":"2023-09-23T23:39:55.089184+0000","last_became_peered":"2023-09-23T23:39:55.089184+0000","last_unstale":"2023-09-23T23:42:53.757530+0000","last_undegraded":"2023-09-23T23:42:53.757530+0000","last_fullsized":"2023-09-23T23:42:53.757530+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:19.775188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":544,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.757249+0000","last_change":"2023-09-23T23:39:55.089351+0000","last_active":"2023-09-23T23:42:53.757249+0000","last_peered":"2023-09-23T23:42:53.757249+0000","last_clean":"2023-09-23T23:42:53.757249+0000","last_became_active":"2023-09-23T23:39:55.089016+0000","last_became_peered":"2023-09-23T23:39:55.089016+0000","last_unstale":"2023-09-23T23:42:53.757249+0000","last_undegraded":"2023-09-23T23:42:53.757249+0000","last_fullsized":"2023-09-23T23:42:53.757249+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:05:23.856265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":315,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.748776+0000","last_change":"2023-09-23T23:34:03.717226+0000","last_active":"2023-09-23T23:42:53.748776+0000","last_peered":"2023-09-23T23:42:53.748776+0000","last_clean":"2023-09-23T23:42:53.748776+0000","last_became_active":"2023-09-23T23:34:03.716951+0000","last_became_peered":"2023-09-23T23:34:03.716951+0000","last_unstale":"2023-09-23T23:42:53.748776+0000","last_undegraded":"2023-09-23T23:42:53.748776+0000","last_fullsized":"2023-09-23T23:42:53.748776+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:19.129564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"679'10294","reported_seq":15965,"reported_epoch":679,"state":"active+clean","last_fresh":"2023-09-23T23:43:05.507836+0000","last_change":"2023-09-23T23:42:01.335232+0000","last_active":"2023-09-23T23:43:05.507836+0000","last_peered":"2023-09-23T23:43:05.507836+0000","last_clean":"2023-09-23T23:43:05.507836+0000","last_became_active":"2023-09-23T23:42:01.316912+0000","last_became_peered":"2023-09-23T23:42:01.316912+0000","last_unstale":"2023-09-23T23:43:05.507836+0000","last_undegraded":"2023-09-23T23:43:05.507836+0000","last_fullsized":"2023-09-23T23:43:05.507836+0000","mapping_epoch":662,"log_start":"633'4133","ondisk_log_start":"633'4133","created":617,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6161,"log_dups_size":0,"ondisk_log_size":6161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:34:25.843454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3473432,"num_objects":54,"num_object_clones":0,"num_object_copies":162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":54,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10294,"num_write_kb":331073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2789,"num_bytes_recovered":115015680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":983,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.748144+0000","last_change":"2023-09-23T23:34:18.733037+0000","last_active":"2023-09-23T23:42:53.748144+0000","last_peered":"2023-09-23T23:42:53.748144+0000","last_clean":"2023-09-23T23:42:53.748144+0000","last_became_active":"2023-09-23T23:34:18.732737+0000","last_became_peered":"2023-09-23T23:34:18.732737+0000","last_unstale":"2023-09-23T23:42:53.748144+0000","last_undegraded":"2023-09-23T23:42:53.748144+0000","last_fullsized":"2023-09-23T23:42:53.748144+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:35.040185+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":420,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.748263+0000","last_change":"2023-09-23T23:20:15.738116+0000","last_active":"2023-09-23T23:42:53.748263+0000","last_peered":"2023-09-23T23:42:53.748263+0000","last_clean":"2023-09-23T23:42:53.748263+0000","last_became_active":"2023-09-23T23:20:15.737849+0000","last_became_peered":"2023-09-23T23:20:15.737849+0000","last_unstale":"2023-09-23T23:42:53.748263+0000","last_undegraded":"2023-09-23T23:42:53.748263+0000","last_fullsized":"2023-09-23T23:42:53.748263+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:42.577115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":339,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.749220+0000","last_change":"2023-09-23T23:39:55.093320+0000","last_active":"2023-09-23T23:42:53.749220+0000","last_peered":"2023-09-23T23:42:53.749220+0000","last_clean":"2023-09-23T23:42:53.749220+0000","last_became_active":"2023-09-23T23:39:55.092998+0000","last_became_peered":"2023-09-23T23:39:55.092998+0000","last_unstale":"2023-09-23T23:42:53.749220+0000","last_undegraded":"2023-09-23T23:42:53.749220+0000","last_fullsized":"2023-09-23T23:42:53.749220+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:38.763321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":20,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.735800+0000","last_change":"2023-09-23T23:42:28.952433+0000","last_active":"2023-09-23T23:42:53.735800+0000","last_peered":"2023-09-23T23:42:53.735800+0000","last_clean":"2023-09-23T23:42:53.735800+0000","last_became_active":"2023-09-23T23:42:28.952139+0000","last_became_peered":"2023-09-23T23:42:28.952139+0000","last_unstale":"2023-09-23T23:42:53.735800+0000","last_undegraded":"2023-09-23T23:42:53.735800+0000","last_fullsized":"2023-09-23T23:42:53.735800+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:49.952980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"679'10586","reported_seq":10658,"reported_epoch":679,"state":"active+clean","last_fresh":"2023-09-23T23:43:06.681063+0000","last_change":"2023-09-23T23:39:13.747297+0000","last_active":"2023-09-23T23:43:06.681063+0000","last_peered":"2023-09-23T23:43:06.681063+0000","last_clean":"2023-09-23T23:43:06.681063+0000","last_became_active":"2023-09-23T23:39:13.747068+0000","last_became_peered":"2023-09-23T23:39:13.747068+0000","last_unstale":"2023-09-23T23:43:06.681063+0000","last_undegraded":"2023-09-23T23:43:06.681063+0000","last_fullsized":"2023-09-23T23:43:06.681063+0000","mapping_epoch":617,"log_start":"640'5681","ondisk_log_start":"640'5681","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4905,"log_dups_size":0,"ondisk_log_size":4905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:34.450587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10586,"num_write_kb":338752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":697,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.752280+0000","last_change":"2023-09-23T23:26:48.530664+0000","last_active":"2023-09-23T23:42:53.752280+0000","last_peered":"2023-09-23T23:42:53.752280+0000","last_clean":"2023-09-23T23:42:53.752280+0000","last_became_active":"2023-09-23T23:26:48.530276+0000","last_became_peered":"2023-09-23T23:26:48.530276+0000","last_unstale":"2023-09-23T23:42:53.752280+0000","last_undegraded":"2023-09-23T23:42:53.752280+0000","last_fullsized":"2023-09-23T23:42:53.752280+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:27.428398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":998,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.756113+0000","last_change":"2023-09-23T23:39:58.122811+0000","last_active":"2023-09-23T23:42:53.756113+0000","last_peered":"2023-09-23T23:42:53.756113+0000","last_clean":"2023-09-23T23:42:53.756113+0000","last_became_active":"2023-09-23T23:39:58.122461+0000","last_became_peered":"2023-09-23T23:39:58.122461+0000","last_unstale":"2023-09-23T23:42:53.756113+0000","last_undegraded":"2023-09-23T23:42:53.756113+0000","last_fullsized":"2023-09-23T23:42:53.756113+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:20:59.163746+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":653,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.746456+0000","last_change":"2023-09-23T23:39:55.101203+0000","last_active":"2023-09-23T23:42:53.746456+0000","last_peered":"2023-09-23T23:42:53.746456+0000","last_clean":"2023-09-23T23:42:53.746456+0000","last_became_active":"2023-09-23T23:39:55.099575+0000","last_became_peered":"2023-09-23T23:39:55.099575+0000","last_unstale":"2023-09-23T23:42:53.746456+0000","last_undegraded":"2023-09-23T23:42:53.746456+0000","last_fullsized":"2023-09-23T23:42:53.746456+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:40.288184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":63,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.746887+0000","last_change":"2023-09-23T23:39:55.103713+0000","last_active":"2023-09-23T23:42:53.746887+0000","last_peered":"2023-09-23T23:42:53.746887+0000","last_clean":"2023-09-23T23:42:53.746887+0000","last_became_active":"2023-09-23T23:39:55.103402+0000","last_became_peered":"2023-09-23T23:39:55.103402+0000","last_unstale":"2023-09-23T23:42:53.746887+0000","last_undegraded":"2023-09-23T23:42:53.746887+0000","last_fullsized":"2023-09-23T23:42:53.746887+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:44.949292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.c","version":"679'10406","reported_seq":16162,"reported_epoch":679,"state":"active+clean","last_fresh":"2023-09-23T23:43:05.516555+0000","last_change":"2023-09-23T23:41:10.964608+0000","last_active":"2023-09-23T23:43:05.516555+0000","last_peered":"2023-09-23T23:43:05.516555+0000","last_clean":"2023-09-23T23:43:05.516555+0000","last_became_active":"2023-09-23T23:41:10.952515+0000","last_became_peered":"2023-09-23T23:41:10.952515+0000","last_unstale":"2023-09-23T23:43:05.516555+0000","last_undegraded":"2023-09-23T23:43:05.516555+0000","last_fullsized":"2023-09-23T23:43:05.516555+0000","mapping_epoch":644,"log_start":"633'4262","ondisk_log_start":"633'4262","created":617,"last_epoch_clean":645,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6144,"log_dups_size":0,"ondisk_log_size":6144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:30.860608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3276800,"num_objects":50,"num_object_clones":0,"num_object_copies":150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":50,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10406,"num_write_kb":334592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2834,"num_bytes_recovered":185729024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":842,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.736977+0000","last_change":"2023-09-23T23:41:34.668439+0000","last_active":"2023-09-23T23:42:53.736977+0000","last_peered":"2023-09-23T23:42:53.736977+0000","last_clean":"2023-09-23T23:42:53.736977+0000","last_became_active":"2023-09-23T23:41:34.668104+0000","last_became_peered":"2023-09-23T23:41:34.668104+0000","last_unstale":"2023-09-23T23:42:53.736977+0000","last_undegraded":"2023-09-23T23:42:53.736977+0000","last_fullsized":"2023-09-23T23:42:53.736977+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:42:24.264118+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":606,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.755876+0000","last_change":"2023-09-23T23:27:20.552566+0000","last_active":"2023-09-23T23:42:53.755876+0000","last_peered":"2023-09-23T23:42:53.755876+0000","last_clean":"2023-09-23T23:42:53.755876+0000","last_became_active":"2023-09-23T23:27:20.552260+0000","last_became_peered":"2023-09-23T23:27:20.552260+0000","last_unstale":"2023-09-23T23:42:53.755876+0000","last_undegraded":"2023-09-23T23:42:53.755876+0000","last_fullsized":"2023-09-23T23:42:53.755876+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:34.428824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":523,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.743719+0000","last_change":"2023-09-23T23:33:51.541920+0000","last_active":"2023-09-23T23:42:53.743719+0000","last_peered":"2023-09-23T23:42:53.743719+0000","last_clean":"2023-09-23T23:42:53.743719+0000","last_became_active":"2023-09-23T23:33:51.541560+0000","last_became_peered":"2023-09-23T23:33:51.541560+0000","last_unstale":"2023-09-23T23:42:53.743719+0000","last_undegraded":"2023-09-23T23:42:53.743719+0000","last_fullsized":"2023-09-23T23:42:53.743719+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:29:10.647360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":36,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.761084+0000","last_change":"2023-09-23T23:41:31.605087+0000","last_active":"2023-09-23T23:42:53.761084+0000","last_peered":"2023-09-23T23:42:53.761084+0000","last_clean":"2023-09-23T23:42:53.761084+0000","last_became_active":"2023-09-23T23:41:31.604777+0000","last_became_peered":"2023-09-23T23:41:31.604777+0000","last_unstale":"2023-09-23T23:42:53.761084+0000","last_undegraded":"2023-09-23T23:42:53.761084+0000","last_fullsized":"2023-09-23T23:42:53.761084+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:57.416801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"679'10238","reported_seq":10310,"reported_epoch":679,"state":"active+clean","last_fresh":"2023-09-23T23:43:06.664055+0000","last_change":"2023-09-23T23:39:13.742801+0000","last_active":"2023-09-23T23:43:06.664055+0000","last_peered":"2023-09-23T23:43:06.664055+0000","last_clean":"2023-09-23T23:43:06.664055+0000","last_became_active":"2023-09-23T23:39:13.742482+0000","last_became_peered":"2023-09-23T23:39:13.742482+0000","last_unstale":"2023-09-23T23:43:06.664055+0000","last_undegraded":"2023-09-23T23:43:06.664055+0000","last_fullsized":"2023-09-23T23:43:06.664055+0000","mapping_epoch":617,"log_start":"637'5342","ondisk_log_start":"637'5342","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4896,"log_dups_size":0,"ondisk_log_size":4896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:15.737461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10238,"num_write_kb":327616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":606,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.743816+0000","last_change":"2023-09-23T23:39:58.112420+0000","last_active":"2023-09-23T23:42:53.743816+0000","last_peered":"2023-09-23T23:42:53.743816+0000","last_clean":"2023-09-23T23:42:53.743816+0000","last_became_active":"2023-09-23T23:39:58.111949+0000","last_became_peered":"2023-09-23T23:39:58.111949+0000","last_unstale":"2023-09-23T23:42:53.743816+0000","last_undegraded":"2023-09-23T23:42:53.743816+0000","last_fullsized":"2023-09-23T23:42:53.743816+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:20.354620+0000","scrub_duration":0.0047245100000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":123,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:56.196027+0000","last_change":"2023-09-23T23:34:35.734263+0000","last_active":"2023-09-23T23:42:56.196027+0000","last_peered":"2023-09-23T23:42:56.196027+0000","last_clean":"2023-09-23T23:42:56.196027+0000","last_became_active":"2023-09-23T23:34:35.733955+0000","last_became_peered":"2023-09-23T23:34:35.733955+0000","last_unstale":"2023-09-23T23:42:56.196027+0000","last_undegraded":"2023-09-23T23:42:56.196027+0000","last_fullsized":"2023-09-23T23:42:56.196027+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:05.450682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":222,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.748845+0000","last_change":"2023-09-23T23:34:03.729346+0000","last_active":"2023-09-23T23:42:53.748845+0000","last_peered":"2023-09-23T23:42:53.748845+0000","last_clean":"2023-09-23T23:42:53.748845+0000","last_became_active":"2023-09-23T23:34:03.729026+0000","last_became_peered":"2023-09-23T23:34:03.729026+0000","last_unstale":"2023-09-23T23:42:53.748845+0000","last_undegraded":"2023-09-23T23:42:53.748845+0000","last_fullsized":"2023-09-23T23:42:53.748845+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:44.371219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":412,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.755466+0000","last_change":"2023-09-23T23:23:36.022532+0000","last_active":"2023-09-23T23:42:53.755466+0000","last_peered":"2023-09-23T23:42:53.755466+0000","last_clean":"2023-09-23T23:42:53.755466+0000","last_became_active":"2023-09-23T23:23:36.022172+0000","last_became_peered":"2023-09-23T23:23:36.022172+0000","last_unstale":"2023-09-23T23:42:53.755466+0000","last_undegraded":"2023-09-23T23:42:53.755466+0000","last_fullsized":"2023-09-23T23:42:53.755466+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:03.976855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.a","version":"679'10288","reported_seq":10360,"reported_epoch":679,"state":"active+clean","last_fresh":"2023-09-23T23:43:06.650164+0000","last_change":"2023-09-23T23:39:13.746851+0000","last_active":"2023-09-23T23:43:06.650164+0000","last_peered":"2023-09-23T23:43:06.650164+0000","last_clean":"2023-09-23T23:43:06.650164+0000","last_became_active":"2023-09-23T23:39:13.746412+0000","last_became_peered":"2023-09-23T23:39:13.746412+0000","last_unstale":"2023-09-23T23:43:06.650164+0000","last_undegraded":"2023-09-23T23:43:06.650164+0000","last_fullsized":"2023-09-23T23:43:06.650164+0000","mapping_epoch":617,"log_start":"637'5372","ondisk_log_start":"637'5372","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":4916,"log_dups_size":0,"ondisk_log_size":4916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:39.662209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10288,"num_write_kb":329216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":880,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.753311+0000","last_change":"2023-09-23T23:39:58.122659+0000","last_active":"2023-09-23T23:42:53.753311+0000","last_peered":"2023-09-23T23:42:53.753311+0000","last_clean":"2023-09-23T23:42:53.753311+0000","last_became_active":"2023-09-23T23:39:58.122334+0000","last_became_peered":"2023-09-23T23:39:58.122334+0000","last_unstale":"2023-09-23T23:42:53.753311+0000","last_undegraded":"2023-09-23T23:42:53.753311+0000","last_fullsized":"2023-09-23T23:42:53.753311+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:29.154690+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":679,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.747958+0000","last_change":"2023-09-23T23:26:48.566664+0000","last_active":"2023-09-23T23:42:53.747958+0000","last_peered":"2023-09-23T23:42:53.747958+0000","last_clean":"2023-09-23T23:42:53.747958+0000","last_became_active":"2023-09-23T23:26:48.551798+0000","last_became_peered":"2023-09-23T23:26:48.551798+0000","last_unstale":"2023-09-23T23:42:53.747958+0000","last_undegraded":"2023-09-23T23:42:53.747958+0000","last_fullsized":"2023-09-23T23:42:53.747958+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:11.567600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":222,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.743665+0000","last_change":"2023-09-23T23:39:55.100517+0000","last_active":"2023-09-23T23:42:53.743665+0000","last_peered":"2023-09-23T23:42:53.743665+0000","last_clean":"2023-09-23T23:42:53.743665+0000","last_became_active":"2023-09-23T23:39:55.098755+0000","last_became_peered":"2023-09-23T23:39:55.098755+0000","last_unstale":"2023-09-23T23:42:53.743665+0000","last_undegraded":"2023-09-23T23:42:53.743665+0000","last_fullsized":"2023-09-23T23:42:53.743665+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:50.086720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":252,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.749014+0000","last_change":"2023-09-23T23:41:31.604932+0000","last_active":"2023-09-23T23:42:53.749014+0000","last_peered":"2023-09-23T23:42:53.749014+0000","last_clean":"2023-09-23T23:42:53.749014+0000","last_became_active":"2023-09-23T23:41:31.604686+0000","last_became_peered":"2023-09-23T23:41:31.604686+0000","last_unstale":"2023-09-23T23:42:53.749014+0000","last_undegraded":"2023-09-23T23:42:53.749014+0000","last_fullsized":"2023-09-23T23:42:53.749014+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:48.931148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.b","version":"679'10331","reported_seq":10403,"reported_epoch":679,"state":"active+clean","last_fresh":"2023-09-23T23:43:05.504994+0000","last_change":"2023-09-23T23:39:13.744730+0000","last_active":"2023-09-23T23:43:05.504994+0000","last_peered":"2023-09-23T23:43:05.504994+0000","last_clean":"2023-09-23T23:43:05.504994+0000","last_became_active":"2023-09-23T23:39:13.744442+0000","last_became_peered":"2023-09-23T23:39:13.744442+0000","last_unstale":"2023-09-23T23:43:05.504994+0000","last_undegraded":"2023-09-23T23:43:05.504994+0000","last_fullsized":"2023-09-23T23:43:05.504994+0000","mapping_epoch":617,"log_start":"633'4195","ondisk_log_start":"633'4195","created":617,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:39:12.504732+0000","last_clean_scrub_stamp":"2023-09-23T23:39:12.504732+0000","objects_scrubbed":0,"log_size":6136,"log_dups_size":0,"ondisk_log_size":6136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:56:43.437044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2424832,"num_objects":37,"num_object_clones":0,"num_object_copies":111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10331,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":1069,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.747930+0000","last_change":"2023-09-23T23:26:51.562604+0000","last_active":"2023-09-23T23:42:53.747930+0000","last_peered":"2023-09-23T23:42:53.747930+0000","last_clean":"2023-09-23T23:42:53.747930+0000","last_became_active":"2023-09-23T23:26:51.562285+0000","last_became_peered":"2023-09-23T23:26:51.562285+0000","last_unstale":"2023-09-23T23:42:53.747930+0000","last_undegraded":"2023-09-23T23:42:53.747930+0000","last_fullsized":"2023-09-23T23:42:53.747930+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:14.530373+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":395,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.749119+0000","last_change":"2023-09-23T23:39:55.096945+0000","last_active":"2023-09-23T23:42:53.749119+0000","last_peered":"2023-09-23T23:42:53.749119+0000","last_clean":"2023-09-23T23:42:53.749119+0000","last_became_active":"2023-09-23T23:39:55.096655+0000","last_became_peered":"2023-09-23T23:39:55.096655+0000","last_unstale":"2023-09-23T23:42:53.749119+0000","last_undegraded":"2023-09-23T23:42:53.749119+0000","last_fullsized":"2023-09-23T23:42:53.749119+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:27:24.290604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":40,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.761100+0000","last_change":"2023-09-23T23:41:31.599981+0000","last_active":"2023-09-23T23:42:53.761100+0000","last_peered":"2023-09-23T23:42:53.761100+0000","last_clean":"2023-09-23T23:42:53.761100+0000","last_became_active":"2023-09-23T23:41:31.597250+0000","last_became_peered":"2023-09-23T23:41:31.597250+0000","last_unstale":"2023-09-23T23:42:53.761100+0000","last_undegraded":"2023-09-23T23:42:53.761100+0000","last_fullsized":"2023-09-23T23:42:53.761100+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:06.361776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":263,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.756778+0000","last_change":"2023-09-23T23:39:55.092828+0000","last_active":"2023-09-23T23:42:53.756778+0000","last_peered":"2023-09-23T23:42:53.756778+0000","last_clean":"2023-09-23T23:42:53.756778+0000","last_became_active":"2023-09-23T23:39:55.092504+0000","last_became_peered":"2023-09-23T23:39:55.092504+0000","last_unstale":"2023-09-23T23:42:53.756778+0000","last_undegraded":"2023-09-23T23:42:53.756778+0000","last_fullsized":"2023-09-23T23:42:53.756778+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:59.997875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1169,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.743927+0000","last_change":"2023-09-23T23:41:34.675056+0000","last_active":"2023-09-23T23:42:53.743927+0000","last_peered":"2023-09-23T23:42:53.743927+0000","last_clean":"2023-09-23T23:42:53.743927+0000","last_became_active":"2023-09-23T23:41:34.674727+0000","last_became_peered":"2023-09-23T23:41:34.674727+0000","last_unstale":"2023-09-23T23:42:53.743927+0000","last_undegraded":"2023-09-23T23:42:53.743927+0000","last_fullsized":"2023-09-23T23:42:53.743927+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:01.911339+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":47,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.757175+0000","last_change":"2023-09-23T23:42:28.952637+0000","last_active":"2023-09-23T23:42:53.757175+0000","last_peered":"2023-09-23T23:42:53.757175+0000","last_clean":"2023-09-23T23:42:53.757175+0000","last_became_active":"2023-09-23T23:42:28.952283+0000","last_became_peered":"2023-09-23T23:42:28.952283+0000","last_unstale":"2023-09-23T23:42:53.757175+0000","last_undegraded":"2023-09-23T23:42:53.757175+0000","last_fullsized":"2023-09-23T23:42:53.757175+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:14.296297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":437,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.748913+0000","last_change":"2023-09-23T23:28:45.069329+0000","last_active":"2023-09-23T23:42:53.748913+0000","last_peered":"2023-09-23T23:42:53.748913+0000","last_clean":"2023-09-23T23:42:53.748913+0000","last_became_active":"2023-09-23T23:28:45.069058+0000","last_became_peered":"2023-09-23T23:28:45.069058+0000","last_unstale":"2023-09-23T23:42:53.748913+0000","last_undegraded":"2023-09-23T23:42:53.748913+0000","last_fullsized":"2023-09-23T23:42:53.748913+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:49.916257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":881,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.747609+0000","last_change":"2023-09-23T23:41:34.667657+0000","last_active":"2023-09-23T23:42:53.747609+0000","last_peered":"2023-09-23T23:42:53.747609+0000","last_clean":"2023-09-23T23:42:53.747609+0000","last_became_active":"2023-09-23T23:41:34.659982+0000","last_became_peered":"2023-09-23T23:41:34.659982+0000","last_unstale":"2023-09-23T23:42:53.747609+0000","last_undegraded":"2023-09-23T23:42:53.747609+0000","last_fullsized":"2023-09-23T23:42:53.747609+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:49.342830+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":338,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.755786+0000","last_change":"2023-09-23T23:39:55.115869+0000","last_active":"2023-09-23T23:42:53.755786+0000","last_peered":"2023-09-23T23:42:53.755786+0000","last_clean":"2023-09-23T23:42:53.755786+0000","last_became_active":"2023-09-23T23:39:55.092827+0000","last_became_peered":"2023-09-23T23:39:55.092827+0000","last_unstale":"2023-09-23T23:42:53.755786+0000","last_undegraded":"2023-09-23T23:42:53.755786+0000","last_fullsized":"2023-09-23T23:42:53.755786+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:03.590720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":547,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.749456+0000","last_change":"2023-09-23T23:42:28.956777+0000","last_active":"2023-09-23T23:42:53.749456+0000","last_peered":"2023-09-23T23:42:53.749456+0000","last_clean":"2023-09-23T23:42:53.749456+0000","last_became_active":"2023-09-23T23:42:28.956436+0000","last_became_peered":"2023-09-23T23:42:28.956436+0000","last_unstale":"2023-09-23T23:42:53.749456+0000","last_undegraded":"2023-09-23T23:42:53.749456+0000","last_fullsized":"2023-09-23T23:42:53.749456+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:42.633268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":67,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.746807+0000","last_change":"2023-09-23T23:39:55.128319+0000","last_active":"2023-09-23T23:42:53.746807+0000","last_peered":"2023-09-23T23:42:53.746807+0000","last_clean":"2023-09-23T23:42:53.746807+0000","last_became_active":"2023-09-23T23:39:55.098936+0000","last_became_peered":"2023-09-23T23:39:55.098936+0000","last_unstale":"2023-09-23T23:42:53.746807+0000","last_undegraded":"2023-09-23T23:42:53.746807+0000","last_fullsized":"2023-09-23T23:42:53.746807+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:03.570174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":368,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.746532+0000","last_change":"2023-09-23T23:33:51.542154+0000","last_active":"2023-09-23T23:42:53.746532+0000","last_peered":"2023-09-23T23:42:53.746532+0000","last_clean":"2023-09-23T23:42:53.746532+0000","last_became_active":"2023-09-23T23:33:51.541834+0000","last_became_peered":"2023-09-23T23:33:51.541834+0000","last_unstale":"2023-09-23T23:42:53.746532+0000","last_undegraded":"2023-09-23T23:42:53.746532+0000","last_fullsized":"2023-09-23T23:42:53.746532+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:56.136276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":924,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.756690+0000","last_change":"2023-09-23T23:39:58.111742+0000","last_active":"2023-09-23T23:42:53.756690+0000","last_peered":"2023-09-23T23:42:53.756690+0000","last_clean":"2023-09-23T23:42:53.756690+0000","last_became_active":"2023-09-23T23:39:58.111358+0000","last_became_peered":"2023-09-23T23:39:58.111358+0000","last_unstale":"2023-09-23T23:42:53.756690+0000","last_undegraded":"2023-09-23T23:42:53.756690+0000","last_fullsized":"2023-09-23T23:42:53.756690+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:51.871617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:51.871617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:54.588394+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":722,"reported_epoch":679,"state":"active+clean","last_fresh":"2023-09-23T23:43:04.204703+0000","last_change":"2023-09-23T23:39:55.101668+0000","last_active":"2023-09-23T23:43:04.204703+0000","last_peered":"2023-09-23T23:43:04.204703+0000","last_clean":"2023-09-23T23:43:04.204703+0000","last_became_active":"2023-09-23T23:39:55.100664+0000","last_became_peered":"2023-09-23T23:39:55.100664+0000","last_unstale":"2023-09-23T23:43:04.204703+0000","last_undegraded":"2023-09-23T23:43:04.204703+0000","last_fullsized":"2023-09-23T23:43:04.204703+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:43.354111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":696,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.748131+0000","last_change":"2023-09-23T23:20:15.734047+0000","last_active":"2023-09-23T23:42:53.748131+0000","last_peered":"2023-09-23T23:42:53.748131+0000","last_clean":"2023-09-23T23:42:53.748131+0000","last_became_active":"2023-09-23T23:20:15.733738+0000","last_became_peered":"2023-09-23T23:20:15.733738+0000","last_unstale":"2023-09-23T23:42:53.748131+0000","last_undegraded":"2023-09-23T23:42:53.748131+0000","last_fullsized":"2023-09-23T23:42:53.748131+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.015112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":854,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.747060+0000","last_change":"2023-09-23T23:34:03.717989+0000","last_active":"2023-09-23T23:42:53.747060+0000","last_peered":"2023-09-23T23:42:53.747060+0000","last_clean":"2023-09-23T23:42:53.747060+0000","last_became_active":"2023-09-23T23:34:03.717728+0000","last_became_peered":"2023-09-23T23:34:03.717728+0000","last_unstale":"2023-09-23T23:42:53.747060+0000","last_undegraded":"2023-09-23T23:42:53.747060+0000","last_fullsized":"2023-09-23T23:42:53.747060+0000","mapping_epoch":562,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":563,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:10.987781+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":556,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:43:00.681065+0000","last_change":"2023-09-23T23:41:31.624156+0000","last_active":"2023-09-23T23:43:00.681065+0000","last_peered":"2023-09-23T23:43:00.681065+0000","last_clean":"2023-09-23T23:43:00.681065+0000","last_became_active":"2023-09-23T23:41:31.612962+0000","last_became_peered":"2023-09-23T23:41:31.612962+0000","last_unstale":"2023-09-23T23:43:00.681065+0000","last_undegraded":"2023-09-23T23:43:00.681065+0000","last_fullsized":"2023-09-23T23:43:00.681065+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:31.042576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":679,"reported_epoch":678,"state":"active+clean","last_fresh":"2023-09-23T23:42:53.743369+0000","last_change":"2023-09-23T23:39:55.099784+0000","last_active":"2023-09-23T23:42:53.743369+0000","last_peered":"2023-09-23T23:42:53.743369+0000","last_clean":"2023-09-23T23:42:53.743369+0000","last_became_active":"2023-09-23T23:39:55.099483+0000","last_became_peered":"2023-09-23T23:39:55.099483+0000","last_unstale":"2023-09-23T23:42:53.743369+0000","last_undegraded":"2023-09-23T23:42:53.743369+0000","last_fullsized":"2023-09-23T23:42:53.743369+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:48.012026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":21626904,"num_objects":331,"num_object_clones":0,"num_object_copies":993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":331,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":167423,"num_write_kb":5368065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14934,"num_bytes_recovered":809631744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":183635968,"data_stored":183631896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87475,"ondisk_log_size":87475,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":71,"num_bytes_recovered":12612,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":143360,"data_stored":26230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27708,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782764,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":503260,"kb_used_data":7444,"kb_used_omap":8,"kb_used_meta":495799,"kb_avail":93241892,"statfs":{"total":95995035648,"available":95479697408,"internally_reserved":0,"allocated":7622656,"data_stored":5646801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":507698245},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.0900000000000001,"15min":1.6180000000000001},"min":{"1min":0.23799999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":3.528,"5min":12.432,"15min":44.094999999999999},"last":0.252},{"interface":"front","average":{"1min":1.353,"5min":1.3320000000000001,"15min":1.76},"min":{"1min":0.27100000000000002,"5min":0.22700000000000001,"15min":0.21299999999999999},"max":{"1min":4.4720000000000004,"5min":12.717000000000001,"15min":44.335999999999999},"last":0.26800000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":1.377,"15min":3.0409999999999999},"min":{"1min":0.28499999999999998,"5min":0.255,"15min":0.23200000000000001},"max":{"1min":5.2229999999999999,"5min":12.484,"15min":198.58000000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.575,"5min":1.526,"15min":3.0030000000000001},"min":{"1min":0.28999999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.6749999999999998,"5min":13.145,"15min":198.71100000000001},"last":2.1419999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":2.524,"5min":2.3599999999999999,"15min":3.798},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":8.9779999999999998,"5min":46.374000000000002,"15min":203.58000000000001},"last":1.4870000000000001},{"interface":"front","average":{"1min":1.909,"5min":2.5230000000000001,"15min":3.4399999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26200000000000001},"max":{"1min":9.9670000000000005,"5min":46.412999999999997,"15min":198.69},"last":0.68000000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":2.2759999999999998,"5min":5.1429999999999998,"15min":5.2770000000000001},"min":{"1min":0.31,"5min":0.309,"15min":0.26700000000000002},"max":{"1min":8.1479999999999997,"5min":154.18100000000001,"15min":279.26499999999999},"last":0.85599999999999998},{"interface":"front","average":{"1min":2.931,"5min":5.0919999999999996,"15min":5.8150000000000004},"min":{"1min":0.32600000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":8.5779999999999994,"5min":154.16300000000001,"15min":279.29899999999998},"last":0.58799999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.69199999999999995,"15min":1.008},"min":{"1min":0.29599999999999999,"5min":0.28999999999999998,"15min":0.246},"max":{"1min":1.5169999999999999,"5min":12.510999999999999,"15min":31.498999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.435,"5min":0.80100000000000005,"15min":0.93999999999999995},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.23899999999999999},"max":{"1min":0.82399999999999995,"5min":12.821999999999999,"15min":31.68},"last":0.34399999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":2.0659999999999998,"15min":2.2090000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.88300000000000001,"5min":119.417,"15min":210.83600000000001},"last":0.314},{"interface":"front","average":{"1min":0.40999999999999998,"5min":2.1669999999999998,"15min":2.1949999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.27700000000000002},"max":{"1min":0.54900000000000004,"5min":119.61499999999999,"15min":207.04599999999999},"last":0.46000000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.84199999999999997,"15min":1.149},"min":{"1min":0.34499999999999997,"5min":0.32800000000000001,"15min":0.29899999999999999},"max":{"1min":0.77900000000000003,"5min":6.2169999999999996,"15min":94.852999999999994},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.81899999999999995,"15min":1.274},"min":{"1min":0.36099999999999999,"5min":0.31,"15min":0.30599999999999999},"max":{"1min":0.84999999999999998,"5min":12.529999999999999,"15min":95.510999999999996},"last":0.44600000000000001}]}]},{"osd":6,"up_from":0,"seq":197568496241,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":112,"seq":481036337582,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1067444,"kb_used_data":8684,"kb_used_omap":4,"kb_used_meta":1058747,"kb_avail":92677708,"statfs":{"total":95995035648,"available":94901972992,"internally_reserved":0,"allocated":8892416,"data_stored":5560715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1084157838},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:42:31 2023","interfaces":[{"interface":"back","average":{"1min":3.0979999999999999,"5min":5.2750000000000004,"15min":3.9780000000000002},"min":{"1min":0.20999999999999999,"5min":0.16400000000000001,"15min":0.16400000000000001},"max":{"1min":9.1319999999999997,"5min":156.37799999999999,"15min":156.37799999999999},"last":5.2519999999999998},{"interface":"front","average":{"1min":1.821,"5min":4.7320000000000002,"15min":3.4220000000000002},"min":{"1min":0.221,"5min":0.189,"15min":0.189},"max":{"1min":8.8049999999999997,"5min":161.05199999999999,"15min":161.05199999999999},"last":0.33900000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":5.9349999999999996,"5min":3.1960000000000002,"15min":6.0010000000000003},"min":{"1min":0.22900000000000001,"5min":0.184,"15min":0.184},"max":{"1min":49.311,"5min":53.951999999999998,"15min":231.67400000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":6.383,"5min":3.7789999999999999,"15min":6.2679999999999998},"min":{"1min":0.23400000000000001,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":48.991999999999997,"5min":54.213000000000001,"15min":232.21000000000001},"last":4.5549999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":18.759,"5min":6.4210000000000003,"15min":6.6829999999999998},"min":{"1min":0.25,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":272.839,"5min":272.839,"15min":272.839},"last":3.5760000000000001},{"interface":"front","average":{"1min":17.308,"5min":6.3499999999999996,"15min":6.5949999999999998},"min":{"1min":0.27700000000000002,"5min":0.245,"15min":0.224},"max":{"1min":272.76400000000001,"5min":272.76400000000001,"15min":272.76400000000001},"last":3.6070000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:42:31 2023","interfaces":[{"interface":"back","average":{"1min":3.194,"5min":2.004,"15min":1.5600000000000001},"min":{"1min":0.42999999999999999,"5min":0.34000000000000002,"15min":0.29799999999999999},"max":{"1min":10.183,"5min":10.413,"15min":12.042999999999999},"last":4.4720000000000004},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.2549999999999999,"15min":1.2669999999999999},"min":{"1min":0.36699999999999999,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":7.8760000000000003,"5min":13.622999999999999,"15min":18.108000000000001},"last":0.45800000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:42:31 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":2.23,"15min":2.1150000000000002},"min":{"1min":0.31,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":2.7890000000000001,"5min":80.302999999999997,"15min":151.94800000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":3.3090000000000002,"5min":2.8879999999999999,"15min":2.4279999999999999},"min":{"1min":0.39200000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.141999999999999,"5min":80.254000000000005,"15min":151.821},"last":4.4370000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:42:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":2.129,"15min":1.5740000000000001},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.32200000000000001},"max":{"1min":2.8180000000000001,"5min":87.837999999999994,"15min":87.837999999999994},"last":0.44700000000000001},{"interface":"front","average":{"1min":3.2200000000000002,"5min":2.8570000000000002,"15min":1.986},"min":{"1min":0.42999999999999999,"5min":0.312,"15min":0.312},"max":{"1min":10.233000000000001,"5min":85.518000000000001,"15min":85.518000000000001},"last":4.5010000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:42:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0169999999999999,"5min":1.6779999999999999,"15min":1.6319999999999999},"min":{"1min":0.40300000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":13.587999999999999,"5min":35.487000000000002,"15min":44.241},"last":0.48699999999999999},{"interface":"front","average":{"1min":3.069,"5min":2.3410000000000002,"15min":1.9350000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.210000000000001,"5min":35.146000000000001,"15min":43.920999999999999},"last":4.5279999999999996}]}]},{"osd":0,"up_from":91,"seq":390842024405,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1425580,"kb_used_data":31460,"kb_used_omap":5,"kb_used_meta":1394042,"kb_avail":92319572,"statfs":{"total":95995035648,"available":94535241728,"internally_reserved":0,"allocated":32215040,"data_stored":28886038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1427499642},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:42:46 2023","interfaces":[{"interface":"back","average":{"1min":3.7349999999999999,"5min":2.3069999999999999,"15min":5.7999999999999998},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.188},"max":{"1min":10.502000000000001,"5min":15.589,"15min":301.12700000000001},"last":9.6519999999999992},{"interface":"front","average":{"1min":2.7799999999999998,"5min":2.4540000000000002,"15min":5.8029999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.186},"max":{"1min":18.503,"5min":18.503,"15min":297.72399999999999},"last":0.379}]},{"osd":2,"last update":"Sat Sep 23 23:42:13 2023","interfaces":[{"interface":"back","average":{"1min":5.476,"5min":3.8759999999999999,"15min":5.7949999999999999},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.17199999999999999},"max":{"1min":20.542000000000002,"5min":102.279,"15min":283.07400000000001},"last":11.156000000000001},{"interface":"front","average":{"1min":4.3719999999999999,"5min":3.8420000000000001,"15min":5.9340000000000002},"min":{"1min":0.252,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":18.529,"5min":102.218,"15min":283.161},"last":0.26700000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:42:46 2023","interfaces":[{"interface":"back","average":{"1min":4.968,"5min":5.0869999999999997,"15min":6.6349999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.217},"max":{"1min":20.498999999999999,"5min":99.122,"15min":220.732},"last":10.531000000000001},{"interface":"front","average":{"1min":5.2960000000000003,"5min":5.5499999999999998,"15min":6.9249999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.19800000000000001},"max":{"1min":28.460999999999999,"5min":98.373999999999995,"15min":218.78200000000001},"last":0.30499999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:42:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":1.7,"15min":1.784},"min":{"1min":0.36899999999999999,"5min":0.32600000000000001,"15min":0.28499999999999998},"max":{"1min":18.274999999999999,"5min":18.274999999999999,"15min":108.01900000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.554,"5min":1.5629999999999999,"15min":1.7729999999999999},"min":{"1min":0.39300000000000002,"5min":0.32600000000000001,"15min":0.27200000000000002},"max":{"1min":6.1310000000000002,"5min":8.718,"15min":107.898},"last":0.36199999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:42:24 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":3.117,"15min":1.9610000000000001},"min":{"1min":0.36799999999999999,"5min":0.36699999999999999,"15min":0.32600000000000001},"max":{"1min":6.0650000000000004,"5min":94.254000000000005,"15min":94.254000000000005},"last":0.39500000000000002},{"interface":"front","average":{"1min":3.1200000000000001,"5min":3.3780000000000001,"15min":1.962},"min":{"1min":0.45200000000000001,"5min":0.32900000000000001,"15min":0.253},"max":{"1min":9.5869999999999997,"5min":94.837000000000003,"15min":94.837000000000003},"last":9.5470000000000006}]},{"osd":6,"last update":"Sat Sep 23 23:42:24 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":2.3690000000000002,"15min":2.0659999999999998},"min":{"1min":0.33500000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.0979999999999999,"5min":69.043999999999997,"15min":108.67},"last":0.41099999999999998},{"interface":"front","average":{"1min":3.1629999999999998,"5min":2.597,"15min":2.121},"min":{"1min":0.52900000000000003,"5min":0.38200000000000001,"15min":0.27100000000000002},"max":{"1min":9.6310000000000002,"5min":69.028999999999996,"15min":108.804},"last":9.6240000000000006}]},{"osd":7,"last update":"Sat Sep 23 23:42:13 2023","interfaces":[{"interface":"back","average":{"1min":2.5339999999999998,"5min":2.5129999999999999,"15min":2.2200000000000002},"min":{"1min":0.41299999999999998,"5min":0.35899999999999999,"15min":0.32900000000000001},"max":{"1min":18.469999999999999,"5min":66.620000000000005,"15min":111.71299999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":3.0710000000000002,"5min":2.4180000000000001,"15min":2.1840000000000002},"min":{"1min":0.496,"5min":0.35399999999999998,"15min":0.33300000000000002},"max":{"1min":9.6590000000000007,"5min":66.701999999999998,"15min":111.108},"last":9.5920000000000005}]}]},{"osd":2,"up_from":199,"seq":854698492226,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1088324,"kb_used_data":50428,"kb_used_omap":7,"kb_used_meta":1037816,"kb_avail":92656828,"statfs":{"total":95995035648,"available":94880591872,"internally_reserved":0,"allocated":51638272,"data_stored":48235271,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1062723665},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":5.1479999999999997,"5min":3.2290000000000001,"15min":2.7490000000000001},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":34.648000000000003,"5min":65.001000000000005,"15min":65.001000000000005},"last":4.1799999999999997},{"interface":"front","average":{"1min":3.1379999999999999,"5min":2.5259999999999998,"15min":2.4670000000000001},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19700000000000001},"max":{"1min":13.132,"5min":64.346000000000004,"15min":64.346000000000004},"last":0.39400000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":2.8220000000000001,"5min":2.8279999999999998,"15min":3.8719999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.192},"max":{"1min":14.369,"5min":41.43,"15min":106.765},"last":0.29399999999999998},{"interface":"front","average":{"1min":3.4129999999999998,"5min":2.6200000000000001,"15min":3.7290000000000001},"min":{"1min":0.36299999999999999,"5min":0.24399999999999999,"15min":0.222},"max":{"1min":11.821999999999999,"5min":53.268000000000001,"15min":106.732},"last":3.157}]},{"osd":3,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":6.2300000000000004,"5min":3.1909999999999998,"15min":4.3090000000000002},"min":{"1min":0.315,"5min":0.245,"15min":0.24099999999999999},"max":{"1min":54.441000000000003,"5min":54.441000000000003,"15min":106.70399999999999},"last":0.32200000000000001},{"interface":"front","average":{"1min":6.1120000000000001,"5min":2.6720000000000002,"15min":3.9910000000000001},"min":{"1min":0.23899999999999999,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":53.734000000000002,"5min":53.734000000000002,"15min":106.66800000000001},"last":2.6190000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":3.23,"5min":1.5669999999999999,"15min":1.7929999999999999},"min":{"1min":0.41199999999999998,"5min":0.314,"15min":0.27700000000000002},"max":{"1min":16.001000000000001,"5min":16.001000000000001,"15min":62.081000000000003},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.6870000000000001,"5min":1.77,"15min":1.9259999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.28899999999999998},"max":{"1min":6.6849999999999996,"5min":15.055999999999999,"15min":62.072000000000003},"last":0.35499999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":4.2229999999999999,"15min":3.0110000000000001},"min":{"1min":0.38600000000000001,"5min":0.32200000000000001,"15min":0.26200000000000001},"max":{"1min":13.023,"5min":101.104,"15min":101.104},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.6830000000000001,"5min":4.681,"15min":3.1589999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.641,"5min":101.404,"15min":101.404},"last":0.34200000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2559999999999998,"5min":4.3710000000000004,"15min":3.3039999999999998},"min":{"1min":0.437,"5min":0.33400000000000002,"15min":0.31},"max":{"1min":15.898999999999999,"5min":97.558999999999997,"15min":107.018},"last":0.432},{"interface":"front","average":{"1min":1.71,"5min":4.7530000000000001,"15min":3.0590000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.7130000000000001,"5min":98.231999999999999,"15min":98.231999999999999},"last":0.36899999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:42:17 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":2.238,"15min":2.085},"min":{"1min":0.41899999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.9470000000000001,"5min":68.814999999999998,"15min":68.814999999999998},"last":2.581},{"interface":"front","average":{"1min":2.3879999999999999,"5min":2.4580000000000002,"15min":2.5659999999999998},"min":{"1min":0.45900000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":6.923,"5min":69.918999999999997,"15min":106.809},"last":2.657}]}]},{"osd":3,"up_from":148,"seq":635655160195,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1141708,"kb_used_data":8964,"kb_used_omap":2,"kb_used_meta":1132733,"kb_avail":92603444,"statfs":{"total":95995035648,"available":94825926656,"internally_reserved":0,"allocated":9179136,"data_stored":5880336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1159918645},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:42:59 2023","interfaces":[{"interface":"back","average":{"1min":25.039999999999999,"5min":8.0139999999999993,"15min":3.71},"min":{"1min":0.28399999999999997,"5min":0.20300000000000001,"15min":0.184},"max":{"1min":230.53899999999999,"5min":230.53899999999999,"15min":230.53899999999999},"last":2.415},{"interface":"front","average":{"1min":25.998999999999999,"5min":7.3079999999999998,"15min":3.8300000000000001},"min":{"1min":0.20599999999999999,"5min":0.20300000000000001,"15min":0.17199999999999999},"max":{"1min":230.38800000000001,"5min":230.38800000000001,"15min":230.38800000000001},"last":5.4400000000000004}]},{"osd":1,"last update":"Sat Sep 23 23:42:59 2023","interfaces":[{"interface":"back","average":{"1min":5.0620000000000003,"5min":2.6299999999999999,"15min":2.8010000000000002},"min":{"1min":0.314,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":21.824999999999999,"5min":21.824999999999999,"15min":85.242999999999995},"last":1.879},{"interface":"front","average":{"1min":3.1629999999999998,"5min":2.7799999999999998,"15min":2.6680000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20799999999999999},"max":{"1min":15.315,"5min":20.779,"15min":85.302000000000007},"last":1.4970000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:42:06 2023","interfaces":[{"interface":"back","average":{"1min":3.835,"5min":6.1420000000000003,"15min":5.0220000000000002},"min":{"1min":0.26100000000000001,"5min":0.23999999999999999,"15min":0.219},"max":{"1min":9.4100000000000001,"5min":139.96299999999999,"15min":139.96299999999999},"last":3.0190000000000001},{"interface":"front","average":{"1min":7.3789999999999996,"5min":7.0220000000000002,"15min":5.2649999999999997},"min":{"1min":0.30099999999999999,"5min":0.26800000000000002,"15min":0.25700000000000001},"max":{"1min":69.405000000000001,"5min":141.93199999999999,"15min":141.93199999999999},"last":5.4790000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:42:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":1.5189999999999999,"15min":1.3180000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":4.8200000000000003,"5min":11.414,"15min":11.414},"last":2.3849999999999998},{"interface":"front","average":{"1min":2.4700000000000002,"5min":2.1850000000000001,"15min":1.8500000000000001},"min":{"1min":0.34200000000000003,"5min":0.32900000000000001,"15min":0.30599999999999999},"max":{"1min":12.26,"5min":14.967000000000001,"15min":24.782},"last":2.3109999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:42:59 2023","interfaces":[{"interface":"back","average":{"1min":3.1960000000000002,"5min":3.9300000000000002,"15min":4.7229999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29499999999999998},"max":{"1min":15.56,"5min":84.298000000000002,"15min":214.43700000000001},"last":2.2759999999999998},{"interface":"front","average":{"1min":3.3690000000000002,"5min":3.6269999999999998,"15min":4.633},"min":{"1min":0.40699999999999997,"5min":0.32700000000000001,"15min":0.30599999999999999},"max":{"1min":14.478999999999999,"5min":83.912999999999997,"15min":214.46600000000001},"last":0.40799999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:42:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":2.2040000000000002,"15min":2.5830000000000002},"min":{"1min":0.45800000000000002,"5min":0.36599999999999999,"15min":0.318},"max":{"1min":12.215,"5min":15.167999999999999,"15min":125.94},"last":2.3719999999999999},{"interface":"front","average":{"1min":3.4790000000000001,"5min":2.1379999999999999,"15min":2.544},"min":{"1min":0.35499999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":14.513,"5min":14.513,"15min":125.955},"last":0.441}]},{"osd":7,"last update":"Sat Sep 23 23:42:06 2023","interfaces":[{"interface":"back","average":{"1min":5.4770000000000003,"5min":2.27,"15min":3.5859999999999999},"min":{"1min":0.38900000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":69.197999999999993,"5min":69.197999999999993,"15min":192.69},"last":2.262},{"interface":"front","average":{"1min":3.254,"5min":2.1070000000000002,"15min":3.8410000000000002},"min":{"1min":0.48899999999999999,"5min":0.39500000000000002,"15min":0.36699999999999999},"max":{"1min":12.568,"5min":14.923999999999999,"15min":192.65600000000001},"last":2.339}]}]},{"osd":4,"up_from":0,"seq":128849019514,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757877,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":850504,"kb_used_data":7744,"kb_used_omap":0,"kb_used_meta":842752,"kb_avail":92894648,"statfs":{"total":95995035648,"available":95124119552,"internally_reserved":0,"allocated":7929856,"data_stored":5970213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":862978048},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:42:33 2023","interfaces":[{"interface":"back","average":{"1min":15.054,"5min":4.782,"15min":2.4399999999999999},"min":{"1min":0.251,"5min":0.223,"15min":0.20899999999999999},"max":{"1min":174.58600000000001,"5min":174.58600000000001,"15min":174.58600000000001},"last":0.33500000000000002},{"interface":"front","average":{"1min":14.362,"5min":5.0739999999999998,"15min":2.5750000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.22700000000000001},"max":{"1min":174.69499999999999,"5min":174.69499999999999,"15min":174.69499999999999},"last":0.46000000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:42:33 2023","interfaces":[{"interface":"back","average":{"1min":3.2789999999999999,"5min":3.0510000000000002,"15min":2.0489999999999999},"min":{"1min":0.28000000000000003,"5min":0.25600000000000001,"15min":0.23100000000000001},"max":{"1min":38.468000000000004,"5min":95.709000000000003,"15min":95.709000000000003},"last":0.47099999999999997},{"interface":"front","average":{"1min":3.4159999999999999,"5min":3.2789999999999999,"15min":2.198},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.249},"max":{"1min":38.841999999999999,"5min":94.966999999999999,"15min":94.966999999999999},"last":0.437}]},{"osd":2,"last update":"Sat Sep 23 23:43:02 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":3.7810000000000001,"15min":4.4370000000000003},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27200000000000002},"max":{"1min":9.2159999999999993,"5min":132.81999999999999,"15min":208.738},"last":0.35199999999999998},{"interface":"front","average":{"1min":4.625,"5min":3.9380000000000002,"15min":4.0590000000000002},"min":{"1min":0.33800000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":23.899000000000001,"5min":133.09200000000001,"15min":208.631},"last":0.41199999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:43:02 2023","interfaces":[{"interface":"back","average":{"1min":2.254,"5min":2.6749999999999998,"15min":4.0359999999999996},"min":{"1min":0.38300000000000001,"5min":0.33100000000000002,"15min":0.27800000000000002},"max":{"1min":6.0149999999999997,"5min":46.768999999999998,"15min":143.66300000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":2.254,"5min":2.7109999999999999,"15min":4.1980000000000004},"min":{"1min":0.41599999999999998,"5min":0.36299999999999999,"15min":0.30399999999999999},"max":{"1min":6.0039999999999996,"5min":46.795000000000002,"15min":143.62200000000001},"last":0.36899999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:42:45 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.90200000000000002,"15min":0.93799999999999994},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.252},"max":{"1min":4.0629999999999997,"5min":6.9390000000000001,"15min":19.962},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.72199999999999998,"15min":0.879},"min":{"1min":0.316,"5min":0.307,"15min":0.22900000000000001},"max":{"1min":1.8560000000000001,"5min":6.7590000000000003,"15min":20.829999999999998},"last":0.38400000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:42:45 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.88400000000000001,"15min":1.327},"min":{"1min":0.29499999999999998,"5min":0.28899999999999998,"15min":0.26100000000000001},"max":{"1min":0.98099999999999998,"5min":14.252000000000001,"15min":148.36600000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.88900000000000001,"15min":1.522},"min":{"1min":0.35599999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.9610000000000001,"5min":6.8040000000000003,"15min":148.42500000000001},"last":0.48299999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:42:58 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.83199999999999996,"15min":1.7150000000000001},"min":{"1min":0.312,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.109,"5min":6.4199999999999999,"15min":179.28800000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.93400000000000005,"15min":1.8049999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":2.8690000000000002,"5min":6.4649999999999999,"15min":179.428},"last":0.501}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":13893632,"data_stored":13893632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":23859200,"data_stored":23855128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":145883136,"data_stored":145883136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:43:12.911 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T23:43:12.912 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:43:12.912 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 3, 5, 7, 2, 1] 2023-09-23T23:43:12.912 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd out 3 2023-09-23T23:43:13.049 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:13.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:13.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:13.154 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:13.195 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:13.263 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:43:13.263 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:43:13.264 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:43:13.264 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:43:13.264 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:43:13.296 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:13.381 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:13.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:13.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:13.437 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:13 smithi110 ceph-mon[212939]: pgmap v1939: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 507 op/s 2023-09-23T23:43:13.437 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:13 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:13.437 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:43:13.437 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:13 smithi110 ceph-mon[221102]: pgmap v1939: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 507 op/s 2023-09-23T23:43:13.437 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:13 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:13.438 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:43:13.486 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:13.516 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:13 smithi138 ceph-mon[193606]: pgmap v1939: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 507 op/s 2023-09-23T23:43:13.516 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:13 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:13.517 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:43:13.518 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:13.567 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:13.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:13.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:13.674 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:13.704 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:13.851 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:13.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:13.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:13.955 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:13.983 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:14.263 INFO:teuthology.orchestra.run.smithi110.stderr:pool 'unique_pool_9' removed 2023-09-23T23:43:14.269 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:43:14.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:14 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1104802075' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:43:14.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:14 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4135475921' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:43:14.306 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:14 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:43:14.306 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:14 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1104802075' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:43:14.306 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:14 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4135475921' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:43:14.306 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:14 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:43:14.587 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:14.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:14.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:14.690 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:14.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:14 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1104802075' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:43:14.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:14 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4135475921' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:43:14.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:14 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:43:14.724 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:14.779 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:14.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:14.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:14.888 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:14.918 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:14.937 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:43:15.636 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:15 smithi110 ceph-mon[212939]: pgmap v1940: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 338 op/s 2023-09-23T23:43:15.636 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:15 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:43:15.636 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:15 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1104802075' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:43:15.636 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:15 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:43:15.637 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:15 smithi110 ceph-mon[212939]: osdmap e680: 8 total, 8 up, 6 in 2023-09-23T23:43:15.637 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:15 smithi110 ceph-mon[212939]: failed to encode map e680 with expected crc 2023-09-23T23:43:15.637 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:15 smithi110 ceph-mon[212939]: failed to encode map e680 with expected crc 2023-09-23T23:43:15.637 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:15 smithi110 ceph-mon[212939]: failed to encode map e680 with expected crc 2023-09-23T23:43:15.637 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:15 smithi110 ceph-mon[221102]: pgmap v1940: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 338 op/s 2023-09-23T23:43:15.637 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:15 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:43:15.637 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:15 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1104802075' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:43:15.637 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:15 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:43:15.637 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:15 smithi110 ceph-mon[221102]: osdmap e680: 8 total, 8 up, 6 in 2023-09-23T23:43:15.638 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:15 smithi110 ceph-mon[221102]: failed to encode map e680 with expected crc 2023-09-23T23:43:15.638 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:15 smithi110 ceph-mon[221102]: failed to encode map e680 with expected crc 2023-09-23T23:43:15.638 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:15 smithi110 ceph-mon[221102]: failed to encode map e680 with expected crc 2023-09-23T23:43:15.661 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:15.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:15.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:15 smithi138 ceph-mon[193606]: pgmap v1940: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 338 op/s 2023-09-23T23:43:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:15 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:43:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:15 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1104802075' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:43:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:15 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:43:15.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:15 smithi138 ceph-mon[193606]: osdmap e680: 8 total, 8 up, 6 in 2023-09-23T23:43:15.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:15 smithi138 ceph-mon[193606]: failed to encode map e680 with expected crc 2023-09-23T23:43:15.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:15 smithi138 ceph-mon[193606]: failed to encode map e680 with expected crc 2023-09-23T23:43:15.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:15 smithi138 ceph-mon[193606]: failed to encode map e680 with expected crc 2023-09-23T23:43:15.724 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T23:43:15.725 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-09-23T23:43:15.725 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T23:43:15.729 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T23:43:15.729 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T23:43:15.729 DEBUG:teuthology.orchestra.run.smithi110:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T23:43:15.766 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:15.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T23:43:15.775 INFO:teuthology.orchestra.run.smithi110.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T23:43:15.776 DEBUG:teuthology.orchestra.run.smithi110:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T23:43:15.798 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:15.841 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T23:43:15.841 DEBUG:teuthology.orchestra.run.smithi110:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T23:43:15.842 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:15.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:15.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:15.858 INFO:tasks.workunit:timeout=3h 2023-09-23T23:43:15.858 INFO:tasks.workunit:cleanup=True 2023-09-23T23:43:15.859 DEBUG:teuthology.orchestra.run.smithi110:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T23:43:15.947 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:43:15.973 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:16.014 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:43:16.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:16.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:43:16.118 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:43:16.147 INFO:tasks.workunit.client.0.smithi110.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T23:43:16.150 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:16.199 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:43:16.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:16.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:43:16.306 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:43:16.334 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:16.378 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:43:16.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:16.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:43:16.485 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:16.512 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:16.563 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:16.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:16.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:16.574 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:16.591 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:16.669 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:16.697 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:16.739 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:16.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:16.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:16.844 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:16.874 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:17.133 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:43:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:43:16] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.43.0" 2023-09-23T23:43:17.387 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:17 smithi110 ceph-mon[212939]: pgmap v1942: 103 pgs: 103 active+clean; 582 KiB data, 5.8 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:43:17.388 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:17 smithi110 ceph-mon[221102]: pgmap v1942: 103 pgs: 103 active+clean; 582 KiB data, 5.8 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:43:17.639 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:43:17.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:17 smithi138 ceph-mon[193606]: pgmap v1942: 103 pgs: 103 active+clean; 582 KiB data, 5.8 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:43:17.796 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:43:17.814 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:43:17.824 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:17.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:17.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:17.842 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:43:17.866 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:43:17.889 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:43:17.910 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:43:17.929 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:17.929 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:43:17.950 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:43:17.961 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:43:17.961 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:43:17.962 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:43:17.962 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:43:17.962 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:43:17.962 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:43:17.962 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:43:17.962 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:43:17.963 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:18.009 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:18.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:18.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:18.115 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:18.146 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:18.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1669550919' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:18.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1669550919' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:18.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1669550919' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:18.693 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:18.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:18.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:18.796 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:18.824 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:19.192 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:19.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:19.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:19.296 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:19.328 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:19.379 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:19.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:19.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:19.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:19 smithi110 ceph-mon[212939]: pgmap v1943: 103 pgs: 103 active+clean; 582 KiB data, 5.8 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:43:19.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:19 smithi110 ceph-mon[221102]: pgmap v1943: 103 pgs: 103 active+clean; 582 KiB data, 5.8 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:43:19.486 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:19.517 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:19.622 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:19.630 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:19.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:19.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:19 smithi138 ceph-mon[193606]: pgmap v1943: 103 pgs: 103 active+clean; 582 KiB data, 5.8 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:43:19.735 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:43:19.767 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:19.814 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:43:19.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:19.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:43:19.852 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:43:19.852 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:43:19.919 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:19.948 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:20.045 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:20.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:20.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:20.149 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:20.178 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:20.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:19 smithi110 conmon[212914]: 2023-09-23T23:43:19.921+0000 7f29aa464700 -1 mon.a@0(leader).osd e680 definitely_dead 0 2023-09-23T23:43:20.222 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:20.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:20.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:20.272 INFO:teuthology.orchestra.run.smithi110.stderr:marked out osd.3. 2023-09-23T23:43:20.329 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:20.361 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:20 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2435791556' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T23:43:20.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:20 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T23:43:20.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:20 smithi138 ceph-mon[193606]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T23:43:20.492 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:20 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2435791556' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T23:43:20.493 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:20 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T23:43:20.493 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:20 smithi110 ceph-mon[212939]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T23:43:20.493 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:20 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2435791556' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T23:43:20.493 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:20 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T23:43:20.493 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:20 smithi110 ceph-mon[221102]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T23:43:21.017 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:21.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:21.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:21.121 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:21.136 DEBUG:teuthology.orchestra.run.smithi110:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T23:43:21.153 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:21.201 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:21.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:21.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:21.308 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:21.335 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:21.340 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:43:21.364 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:21 smithi110 ceph-mon[221102]: pgmap v1944: 103 pgs: 103 active+clean; 582 KiB data, 5.8 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:43:21.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:21 smithi110 ceph-mon[221102]: failed to encode map e681 with expected crc 2023-09-23T23:43:21.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:21 smithi110 ceph-mon[221102]: failed to encode map e681 with expected crc 2023-09-23T23:43:21.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:21 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T23:43:21.365 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:21 smithi110 ceph-mon[221102]: osdmap e681: 8 total, 8 up, 5 in 2023-09-23T23:43:21.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:21 smithi110 ceph-mon[212939]: pgmap v1944: 103 pgs: 103 active+clean; 582 KiB data, 5.8 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:43:21.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:21 smithi110 ceph-mon[212939]: failed to encode map e681 with expected crc 2023-09-23T23:43:21.365 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:21 smithi110 ceph-mon[212939]: failed to encode map e681 with expected crc 2023-09-23T23:43:21.366 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:21 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T23:43:21.366 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:21 smithi110 ceph-mon[212939]: osdmap e681: 8 total, 8 up, 5 in 2023-09-23T23:43:21.378 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:21.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:21.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:21 smithi138 ceph-mon[193606]: pgmap v1944: 103 pgs: 103 active+clean; 582 KiB data, 5.8 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:43:21.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:21 smithi138 ceph-mon[193606]: failed to encode map e681 with expected crc 2023-09-23T23:43:21.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:21 smithi138 ceph-mon[193606]: failed to encode map e681 with expected crc 2023-09-23T23:43:21.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:21 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T23:43:21.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:21 smithi138 ceph-mon[193606]: osdmap e681: 8 total, 8 up, 5 in 2023-09-23T23:43:21.466 INFO:tasks.workunit.client.0.smithi110.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T23:43:21.485 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:21.511 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:21.554 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:21.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:21.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:21.560 INFO:tasks.workunit.client.0.smithi110.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T23:43:21.560 INFO:tasks.workunit.client.0.smithi110.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T23:43:21.660 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:21.686 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:21.728 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:21.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:21.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:21.836 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:21.862 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:21.904 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:21.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:21.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:22.010 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:43:22.036 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:22.078 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:43:22.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:22.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:43:22.185 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:22.211 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:22.258 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:22.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:22.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:22.364 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:22.390 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:22.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:22 smithi138 ceph-mon[193606]: failed to encode map e682 with expected crc 2023-09-23T23:43:22.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:22 smithi138 ceph-mon[193606]: failed to encode map e682 with expected crc 2023-09-23T23:43:22.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:22 smithi138 ceph-mon[193606]: osdmap e682: 8 total, 8 up, 5 in 2023-09-23T23:43:22.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:22 smithi138 ceph-mon[193606]: failed to encode map e682 with expected crc 2023-09-23T23:43:22.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:22 smithi138 ceph-mon[193606]: failed to encode map e682 with expected crc 2023-09-23T23:43:22.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:22 smithi138 ceph-mon[193606]: failed to encode map e682 with expected crc 2023-09-23T23:43:22.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:22 smithi110 ceph-mon[212939]: failed to encode map e682 with expected crc 2023-09-23T23:43:22.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:22 smithi110 ceph-mon[212939]: failed to encode map e682 with expected crc 2023-09-23T23:43:22.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:22 smithi110 ceph-mon[212939]: osdmap e682: 8 total, 8 up, 5 in 2023-09-23T23:43:22.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:22 smithi110 ceph-mon[212939]: failed to encode map e682 with expected crc 2023-09-23T23:43:22.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:22 smithi110 ceph-mon[212939]: failed to encode map e682 with expected crc 2023-09-23T23:43:22.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:22 smithi110 ceph-mon[212939]: failed to encode map e682 with expected crc 2023-09-23T23:43:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:22 smithi110 ceph-mon[221102]: failed to encode map e682 with expected crc 2023-09-23T23:43:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:22 smithi110 ceph-mon[221102]: failed to encode map e682 with expected crc 2023-09-23T23:43:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:22 smithi110 ceph-mon[221102]: osdmap e682: 8 total, 8 up, 5 in 2023-09-23T23:43:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:22 smithi110 ceph-mon[221102]: failed to encode map e682 with expected crc 2023-09-23T23:43:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:22 smithi110 ceph-mon[221102]: failed to encode map e682 with expected crc 2023-09-23T23:43:22.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:22 smithi110 ceph-mon[221102]: failed to encode map e682 with expected crc 2023-09-23T23:43:23.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:23 smithi138 ceph-mon[193606]: failed to encode map e682 with expected crc 2023-09-23T23:43:23.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:23 smithi138 ceph-mon[193606]: failed to encode map e682 with expected crc 2023-09-23T23:43:23.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:23 smithi138 ceph-mon[193606]: pgmap v1947: 103 pgs: 26 peering, 4 unknown, 73 active+clean; 621 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 8.0 KiB/s rd, 21 KiB/s wr, 6 op/s; 15 KiB/s, 0 objects/s recovering 2023-09-23T23:43:23.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:23 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 18 pgs inactive, 17 pgs peering (PG_AVAILABILITY) 2023-09-23T23:43:23.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:23 smithi138 ceph-mon[193606]: failed to encode map e683 with expected crc 2023-09-23T23:43:23.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:23 smithi138 ceph-mon[193606]: failed to encode map e683 with expected crc 2023-09-23T23:43:23.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:23 smithi138 ceph-mon[193606]: osdmap e683: 8 total, 8 up, 5 in 2023-09-23T23:43:23.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:23 smithi138 ceph-mon[193606]: failed to encode map e683 with expected crc 2023-09-23T23:43:23.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:23 smithi138 ceph-mon[193606]: failed to encode map e683 with expected crc 2023-09-23T23:43:23.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:23 smithi138 ceph-mon[193606]: failed to encode map e683 with expected crc 2023-09-23T23:43:23.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[212939]: failed to encode map e682 with expected crc 2023-09-23T23:43:23.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[212939]: failed to encode map e682 with expected crc 2023-09-23T23:43:23.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[212939]: pgmap v1947: 103 pgs: 26 peering, 4 unknown, 73 active+clean; 621 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 8.0 KiB/s rd, 21 KiB/s wr, 6 op/s; 15 KiB/s, 0 objects/s recovering 2023-09-23T23:43:23.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 18 pgs inactive, 17 pgs peering (PG_AVAILABILITY) 2023-09-23T23:43:23.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[212939]: failed to encode map e683 with expected crc 2023-09-23T23:43:23.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[212939]: failed to encode map e683 with expected crc 2023-09-23T23:43:23.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[212939]: osdmap e683: 8 total, 8 up, 5 in 2023-09-23T23:43:23.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[212939]: failed to encode map e683 with expected crc 2023-09-23T23:43:23.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[212939]: failed to encode map e683 with expected crc 2023-09-23T23:43:23.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[212939]: failed to encode map e683 with expected crc 2023-09-23T23:43:23.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[221102]: failed to encode map e682 with expected crc 2023-09-23T23:43:23.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[221102]: failed to encode map e682 with expected crc 2023-09-23T23:43:23.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[221102]: pgmap v1947: 103 pgs: 26 peering, 4 unknown, 73 active+clean; 621 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 8.0 KiB/s rd, 21 KiB/s wr, 6 op/s; 15 KiB/s, 0 objects/s recovering 2023-09-23T23:43:23.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 18 pgs inactive, 17 pgs peering (PG_AVAILABILITY) 2023-09-23T23:43:23.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[221102]: failed to encode map e683 with expected crc 2023-09-23T23:43:23.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[221102]: failed to encode map e683 with expected crc 2023-09-23T23:43:23.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[221102]: osdmap e683: 8 total, 8 up, 5 in 2023-09-23T23:43:23.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[221102]: failed to encode map e683 with expected crc 2023-09-23T23:43:23.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[221102]: failed to encode map e683 with expected crc 2023-09-23T23:43:23.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:23 smithi110 ceph-mon[221102]: failed to encode map e683 with expected crc 2023-09-23T23:43:23.727 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:23.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:23.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:23.832 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:43:23.863 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:23.911 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:43:23.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:23.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:43:24.018 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:24.045 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:24.089 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:24.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:24.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:24.198 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:24.226 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:24 smithi138 ceph-mon[193606]: failed to encode map e684 with expected crc 2023-09-23T23:43:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:24 smithi138 ceph-mon[193606]: failed to encode map e684 with expected crc 2023-09-23T23:43:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:24 smithi138 ceph-mon[193606]: osdmap e684: 8 total, 8 up, 5 in 2023-09-23T23:43:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:24 smithi138 ceph-mon[193606]: failed to encode map e684 with expected crc 2023-09-23T23:43:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:24 smithi138 ceph-mon[193606]: failed to encode map e684 with expected crc 2023-09-23T23:43:24.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:24 smithi138 ceph-mon[193606]: failed to encode map e684 with expected crc 2023-09-23T23:43:24.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:24 smithi110 ceph-mon[212939]: failed to encode map e684 with expected crc 2023-09-23T23:43:24.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:24 smithi110 ceph-mon[212939]: failed to encode map e684 with expected crc 2023-09-23T23:43:24.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:24 smithi110 ceph-mon[212939]: osdmap e684: 8 total, 8 up, 5 in 2023-09-23T23:43:24.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:24 smithi110 ceph-mon[212939]: failed to encode map e684 with expected crc 2023-09-23T23:43:24.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:24 smithi110 ceph-mon[212939]: failed to encode map e684 with expected crc 2023-09-23T23:43:24.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:24 smithi110 ceph-mon[212939]: failed to encode map e684 with expected crc 2023-09-23T23:43:24.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:24 smithi110 ceph-mon[221102]: failed to encode map e684 with expected crc 2023-09-23T23:43:24.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:24 smithi110 ceph-mon[221102]: failed to encode map e684 with expected crc 2023-09-23T23:43:24.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:24 smithi110 ceph-mon[221102]: osdmap e684: 8 total, 8 up, 5 in 2023-09-23T23:43:24.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:24 smithi110 ceph-mon[221102]: failed to encode map e684 with expected crc 2023-09-23T23:43:24.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:24 smithi110 ceph-mon[221102]: failed to encode map e684 with expected crc 2023-09-23T23:43:24.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:24 smithi110 ceph-mon[221102]: failed to encode map e684 with expected crc 2023-09-23T23:43:24.970 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:43:24.977 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:24.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:24.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:25.084 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:25.094 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:25.115 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:25.272 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:25.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:25.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:25.322 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:43:25.377 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:25.405 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:25.455 INFO:tasks.workunit.client.0.smithi110.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T23:43:25.530 INFO:tasks.workunit.client.0.smithi110.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T23:43:25.540 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:25 smithi138 ceph-mon[193606]: failed to encode map e684 with expected crc 2023-09-23T23:43:25.540 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:25 smithi138 ceph-mon[193606]: pgmap v1950: 103 pgs: 1 active+undersized+remapped, 26 peering, 4 unknown, 72 active+clean; 621 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 16 KiB/s rd, 41 KiB/s wr, 11 op/s; 29 KiB/s, 0 objects/s recovering 2023-09-23T23:43:25.540 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:25 smithi138 ceph-mon[193606]: failed to encode map e685 with expected crc 2023-09-23T23:43:25.540 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:25 smithi138 ceph-mon[193606]: failed to encode map e685 with expected crc 2023-09-23T23:43:25.540 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:25 smithi138 ceph-mon[193606]: osdmap e685: 8 total, 8 up, 5 in 2023-09-23T23:43:25.540 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:25 smithi138 ceph-mon[193606]: failed to encode map e685 with expected crc 2023-09-23T23:43:25.569 INFO:tasks.workunit.client.0.smithi110.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T23:43:25.570 INFO:tasks.workunit.client.0.smithi110.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T23:43:25.570 INFO:tasks.workunit.client.0.smithi110.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T23:43:25.600 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:25 smithi110 ceph-mon[212939]: failed to encode map e684 with expected crc 2023-09-23T23:43:25.600 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:25 smithi110 ceph-mon[212939]: pgmap v1950: 103 pgs: 1 active+undersized+remapped, 26 peering, 4 unknown, 72 active+clean; 621 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 16 KiB/s rd, 41 KiB/s wr, 11 op/s; 29 KiB/s, 0 objects/s recovering 2023-09-23T23:43:25.600 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:25 smithi110 ceph-mon[212939]: failed to encode map e685 with expected crc 2023-09-23T23:43:25.600 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:25 smithi110 ceph-mon[212939]: failed to encode map e685 with expected crc 2023-09-23T23:43:25.601 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:25 smithi110 ceph-mon[212939]: osdmap e685: 8 total, 8 up, 5 in 2023-09-23T23:43:25.601 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:25 smithi110 ceph-mon[212939]: failed to encode map e685 with expected crc 2023-09-23T23:43:25.601 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:25 smithi110 ceph-mon[221102]: failed to encode map e684 with expected crc 2023-09-23T23:43:25.601 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:25 smithi110 ceph-mon[221102]: pgmap v1950: 103 pgs: 1 active+undersized+remapped, 26 peering, 4 unknown, 72 active+clean; 621 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 16 KiB/s rd, 41 KiB/s wr, 11 op/s; 29 KiB/s, 0 objects/s recovering 2023-09-23T23:43:25.601 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:25 smithi110 ceph-mon[221102]: failed to encode map e685 with expected crc 2023-09-23T23:43:25.601 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:25 smithi110 ceph-mon[221102]: failed to encode map e685 with expected crc 2023-09-23T23:43:25.601 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:25 smithi110 ceph-mon[221102]: osdmap e685: 8 total, 8 up, 5 in 2023-09-23T23:43:25.601 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:25 smithi110 ceph-mon[221102]: failed to encode map e685 with expected crc 2023-09-23T23:43:25.605 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:25.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:25.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:25.613 INFO:tasks.workunit.client.0.smithi110.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T23:43:25.644 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-09-23T23:43:25.644 DEBUG:teuthology.orchestra.run.smithi110:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T23:43:25.710 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:25.714 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-09-23T23:43:25.715 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-09-23T23:43:25.715 DEBUG:teuthology.orchestra.run.smithi110:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-09-23T23:43:25.737 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:25.853 INFO:tasks.workunit.client.0.smithi110.stdout:Running main() from gmock_main.cc 2023-09-23T23:43:25.853 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] Running 70 tests from 1 test suite. 2023-09-23T23:43:25.853 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment set-up. 2023-09-23T23:43:25.854 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 70 tests from TestClsRbd 2023-09-23T23:43:26.036 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:26.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:26.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:26.140 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:26.167 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:26.212 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:26.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:26.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:26.267 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:43:26.290 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:26 smithi110 ceph-mon[221102]: failed to encode map e685 with expected crc 2023-09-23T23:43:26.290 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:26 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3280446790' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:43:26.290 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:26 smithi110 ceph-mon[212939]: failed to encode map e685 with expected crc 2023-09-23T23:43:26.290 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:26 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3280446790' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:43:26.316 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:26.345 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:26.389 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:26.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:26.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:26.493 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:26.517 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:26 smithi138 ceph-mon[193606]: failed to encode map e685 with expected crc 2023-09-23T23:43:26.517 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:26 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3280446790' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:43:26.519 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:26.739 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:26.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:26.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:26.843 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:43:26.871 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:26.941 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:43:26.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:26.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:43:27.044 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:27.072 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:43:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:43:26] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.43.0" 2023-09-23T23:43:27.075 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:27.120 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:27.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:27.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:27.223 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:43:27.245 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-09-23T23:43:27.245 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:43:27.250 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:27.276 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_all_features 2023-09-23T23:43:27.288 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_all_features (12 ms) 2023-09-23T23:43:27.288 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.copyup 2023-09-23T23:43:27.337 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:27 smithi110 ceph-mon[212939]: pgmap v1952: 103 pgs: 1 active+undersized+remapped, 7 peering, 95 active+clean; 621 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 32 B/s, 0 objects/s recovering 2023-09-23T23:43:27.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:27 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:43:27.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:27 smithi110 ceph-mon[212939]: failed to encode map e686 with expected crc 2023-09-23T23:43:27.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:27 smithi110 ceph-mon[212939]: failed to encode map e686 with expected crc 2023-09-23T23:43:27.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:27 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3280446790' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:43:27.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:27 smithi110 ceph-mon[212939]: osdmap e686: 8 total, 8 up, 5 in 2023-09-23T23:43:27.338 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:27 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1665273682' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-691271-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:43:27.338 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:27 smithi110 ceph-mon[221102]: pgmap v1952: 103 pgs: 1 active+undersized+remapped, 7 peering, 95 active+clean; 621 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 32 B/s, 0 objects/s recovering 2023-09-23T23:43:27.338 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:27 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:43:27.338 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:27 smithi110 ceph-mon[221102]: failed to encode map e686 with expected crc 2023-09-23T23:43:27.339 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:27 smithi110 ceph-mon[221102]: failed to encode map e686 with expected crc 2023-09-23T23:43:27.339 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:27 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3280446790' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:43:27.339 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:27 smithi110 ceph-mon[221102]: osdmap e686: 8 total, 8 up, 5 in 2023-09-23T23:43:27.339 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:27 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1665273682' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-691271-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:43:27.370 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:43:27.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:27.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:43:27.431 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:27 smithi138 ceph-mon[193606]: pgmap v1952: 103 pgs: 1 active+undersized+remapped, 7 peering, 95 active+clean; 621 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 32 B/s, 0 objects/s recovering 2023-09-23T23:43:27.431 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:27 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:43:27.431 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:27 smithi138 ceph-mon[193606]: failed to encode map e686 with expected crc 2023-09-23T23:43:27.431 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:27 smithi138 ceph-mon[193606]: failed to encode map e686 with expected crc 2023-09-23T23:43:27.431 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:27 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3280446790' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:43:27.431 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:27 smithi138 ceph-mon[193606]: osdmap e686: 8 total, 8 up, 5 in 2023-09-23T23:43:27.432 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:27 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1665273682' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-691271-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:43:27.474 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:27.480 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.copyup (192 ms) 2023-09-23T23:43:27.480 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-09-23T23:43:27.502 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.sparse_copyup (22 ms) 2023-09-23T23:43:27.502 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-09-23T23:43:27.504 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:27.509 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2023-09-23T23:43:27.509 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-09-23T23:43:27.521 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.add_remove_child (11 ms) 2023-09-23T23:43:27.521 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.directory_methods 2023-09-23T23:43:27.556 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.directory_methods (35 ms) 2023-09-23T23:43:27.556 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.create 2023-09-23T23:43:27.562 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:27.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:27.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:27.583 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.create (27 ms) 2023-09-23T23:43:27.583 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_features 2023-09-23T23:43:27.584 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2023-09-23T23:43:27.584 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-09-23T23:43:27.585 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_object_prefix (1 ms) 2023-09-23T23:43:27.585 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-09-23T23:43:27.587 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-09-23T23:43:27.587 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-09-23T23:43:27.588 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2023-09-23T23:43:27.588 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-09-23T23:43:27.590 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-09-23T23:43:27.590 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-09-23T23:43:27.594 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2023-09-23T23:43:27.594 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.get_size 2023-09-23T23:43:27.596 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.get_size (3 ms) 2023-09-23T23:43:27.597 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.set_size 2023-09-23T23:43:27.601 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.set_size (6 ms) 2023-09-23T23:43:27.602 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.protection_status 2023-09-23T23:43:27.617 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.protection_status (15 ms) 2023-09-23T23:43:27.618 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-09-23T23:43:27.624 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2023-09-23T23:43:27.624 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.parents_v1 2023-09-23T23:43:27.668 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:27.668 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.parents_v1 (45 ms) 2023-09-23T23:43:27.669 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.parents_v2 2023-09-23T23:43:27.699 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.parents_v2 (30 ms) 2023-09-23T23:43:27.699 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.snapshots 2023-09-23T23:43:27.710 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:27.721 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.snapshots (22 ms) 2023-09-23T23:43:27.721 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.snapid_race 2023-09-23T23:43:27.726 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.snapid_race (5 ms) 2023-09-23T23:43:27.726 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.stripingv2 2023-09-23T23:43:27.736 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.stripingv2 (10 ms) 2023-09-23T23:43:27.736 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.object_map_save 2023-09-23T23:43:27.739 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.object_map_save (3 ms) 2023-09-23T23:43:27.739 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-09-23T23:43:27.745 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.object_map_resize (6 ms) 2023-09-23T23:43:27.745 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.object_map_update 2023-09-23T23:43:27.749 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.object_map_update (4 ms) 2023-09-23T23:43:27.749 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-09-23T23:43:27.749 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-09-23T23:43:27.749 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-09-23T23:43:27.755 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.object_map_snap_add (6 ms) 2023-09-23T23:43:27.755 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-09-23T23:43:27.763 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.object_map_snap_remove (8 ms) 2023-09-23T23:43:27.763 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.flags 2023-09-23T23:43:27.768 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.flags (5 ms) 2023-09-23T23:43:27.768 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.metadata 2023-09-23T23:43:27.784 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:43:27.798 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.metadata (30 ms) 2023-09-23T23:43:27.799 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.set_features 2023-09-23T23:43:27.805 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.set_features (7 ms) 2023-09-23T23:43:27.805 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.mirror 2023-09-23T23:43:27.810 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:43:27.835 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:43:27.843 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.mirror (38 ms) 2023-09-23T23:43:27.843 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.mirror_image 2023-09-23T23:43:27.859 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:43:27.860 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.mirror_image (16 ms) 2023-09-23T23:43:27.860 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-09-23T23:43:27.883 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:43:27.903 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:27.905 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:43:27.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:27.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:27.929 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:43:27.954 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:43:27.955 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:43:27.956 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:43:27.956 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:43:27.956 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:43:27.956 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:43:27.956 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:43:27.956 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:43:27.956 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:43:27.957 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T23:43:28.008 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:28.044 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:28.152 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:28.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:28.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:28.257 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:28.287 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:28.331 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:28.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:28.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:28.382 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:43:28.435 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:28.460 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:28.530 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:28.531 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:28 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1665273682' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-691271-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T23:43:28.531 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:28 smithi138 ceph-mon[193606]: osdmap e687: 8 total, 8 up, 5 in 2023-09-23T23:43:28.531 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:28 smithi138 ceph-mon[193606]: failed to encode map e687 with expected crc 2023-09-23T23:43:28.531 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:28 smithi138 ceph-mon[193606]: failed to encode map e687 with expected crc 2023-09-23T23:43:28.531 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:28 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3663830064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:28.531 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:28 smithi138 ceph-mon[193606]: failed to encode map e687 with expected crc 2023-09-23T23:43:28.531 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:28 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:28.531 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:43:28.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:28.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:28.633 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:28.664 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:28.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:28 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1665273682' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-691271-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T23:43:28.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:28 smithi110 ceph-mon[221102]: osdmap e687: 8 total, 8 up, 5 in 2023-09-23T23:43:28.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:28 smithi110 ceph-mon[221102]: failed to encode map e687 with expected crc 2023-09-23T23:43:28.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:28 smithi110 ceph-mon[221102]: failed to encode map e687 with expected crc 2023-09-23T23:43:28.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:28 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3663830064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:28.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:28 smithi110 ceph-mon[221102]: failed to encode map e687 with expected crc 2023-09-23T23:43:28.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:28 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:28.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:43:28.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:28 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1665273682' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi110-691271-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T23:43:28.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:28 smithi110 ceph-mon[212939]: osdmap e687: 8 total, 8 up, 5 in 2023-09-23T23:43:28.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:28 smithi110 ceph-mon[212939]: failed to encode map e687 with expected crc 2023-09-23T23:43:28.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:28 smithi110 ceph-mon[212939]: failed to encode map e687 with expected crc 2023-09-23T23:43:28.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:28 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3663830064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:28.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:28 smithi110 ceph-mon[212939]: failed to encode map e687 with expected crc 2023-09-23T23:43:28.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:28 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:28.706 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:43:28.709 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:28.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:28.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:28.814 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:28.841 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:28.889 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:28.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:28.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:28.993 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:43:29.022 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:29.068 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:43:29.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:29.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:43:29.172 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:29.202 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:29.253 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:29.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:29.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:29.356 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:43:29.360 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:29.384 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:29 smithi138 ceph-mon[193606]: 1.9 scrub starts 2023-09-23T23:43:29.384 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:29 smithi138 ceph-mon[193606]: 1.9 scrub ok 2023-09-23T23:43:29.384 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:29 smithi138 ceph-mon[193606]: failed to encode map e687 with expected crc 2023-09-23T23:43:29.385 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:29 smithi138 ceph-mon[193606]: pgmap v1955: 135 pgs: 29 unknown, 1 active+undersized+remapped, 105 active+clean; 621 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 1.6 MiB/s rd, 840 KiB/s wr, 6 op/s; 32 B/s, 1 objects/s recovering 2023-09-23T23:43:29.385 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:29 smithi138 ceph-mon[193606]: failed to encode map e688 with expected crc 2023-09-23T23:43:29.385 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:29 smithi138 ceph-mon[193606]: osdmap e688: 8 total, 8 up, 5 in 2023-09-23T23:43:29.385 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:29 smithi138 ceph-mon[193606]: failed to encode map e688 with expected crc 2023-09-23T23:43:29.385 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:29 smithi138 ceph-mon[193606]: Health check update: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T23:43:29.385 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:29 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-09-23T23:43:29.387 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[212939]: 1.9 scrub starts 2023-09-23T23:43:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[212939]: 1.9 scrub ok 2023-09-23T23:43:29.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[212939]: failed to encode map e687 with expected crc 2023-09-23T23:43:29.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[212939]: pgmap v1955: 135 pgs: 29 unknown, 1 active+undersized+remapped, 105 active+clean; 621 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 1.6 MiB/s rd, 840 KiB/s wr, 6 op/s; 32 B/s, 1 objects/s recovering 2023-09-23T23:43:29.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[212939]: failed to encode map e688 with expected crc 2023-09-23T23:43:29.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[212939]: osdmap e688: 8 total, 8 up, 5 in 2023-09-23T23:43:29.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[212939]: failed to encode map e688 with expected crc 2023-09-23T23:43:29.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[212939]: Health check update: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T23:43:29.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-09-23T23:43:29.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[221102]: 1.9 scrub starts 2023-09-23T23:43:29.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[221102]: 1.9 scrub ok 2023-09-23T23:43:29.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[221102]: failed to encode map e687 with expected crc 2023-09-23T23:43:29.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[221102]: pgmap v1955: 135 pgs: 29 unknown, 1 active+undersized+remapped, 105 active+clean; 621 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 1.6 MiB/s rd, 840 KiB/s wr, 6 op/s; 32 B/s, 1 objects/s recovering 2023-09-23T23:43:29.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[221102]: failed to encode map e688 with expected crc 2023-09-23T23:43:29.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[221102]: osdmap e688: 8 total, 8 up, 5 in 2023-09-23T23:43:29.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[221102]: failed to encode map e688 with expected crc 2023-09-23T23:43:29.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[221102]: Health check update: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T23:43:29.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:29 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-09-23T23:43:29.625 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:29.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:29.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:29.731 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:43:29.764 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:29.769 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:29.818 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:43:29.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:29.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:43:29.899 INFO:tasks.thrashosds.thrasher:in_osds: [0, 5, 7, 2, 1] out_osds: [4, 6, 3] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:43:29.899 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:43:29.899 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-09-23T23:43:29.899 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd in 3 2023-09-23T23:43:29.925 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:29.956 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:30.003 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:30.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:30.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:30.074 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:43:30.074 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:43:30.074 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:43:30.074 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:43:30.074 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:43:30.108 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:30.136 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:30.286 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:30.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:30.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:30.390 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:30.421 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:30.472 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:30.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:30.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:30.576 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:30.604 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:30.654 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:30.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:30.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:30.758 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:30.783 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:30.825 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:30.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:30.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:30.828 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T23:43:30.920 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:43:30.928 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:30.990 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:31.019 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.mirror_image_status (3160 ms) 2023-09-23T23:43:31.019 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-09-23T23:43:31.043 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:31.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:31.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:31.122 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.mirror_image_map (103 ms) 2023-09-23T23:43:31.123 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-09-23T23:43:31.130 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2023-09-23T23:43:31.130 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-09-23T23:43:31.142 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.mirror_snapshot (12 ms) 2023-09-23T23:43:31.142 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-09-23T23:43:31.145 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2023-09-23T23:43:31.145 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-09-23T23:43:31.146 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2023-09-23T23:43:31.147 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-09-23T23:43:31.148 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:31.150 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-09-23T23:43:31.151 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-09-23T23:43:31.156 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_dir_rename (7 ms) 2023-09-23T23:43:31.156 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-09-23T23:43:31.160 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_dir_remove (4 ms) 2023-09-23T23:43:31.160 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-09-23T23:43:31.164 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2023-09-23T23:43:31.165 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_image_add 2023-09-23T23:43:31.167 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2023-09-23T23:43:31.167 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-09-23T23:43:31.171 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_image_remove (4 ms) 2023-09-23T23:43:31.171 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_image_list 2023-09-23T23:43:31.173 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2023-09-23T23:43:31.174 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-09-23T23:43:31.179 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_image_clean (5 ms) 2023-09-23T23:43:31.179 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.image_group_add 2023-09-23T23:43:31.180 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:31.181 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:43:31.181 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2023-09-23T23:43:31.182 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-09-23T23:43:31.186 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.image_group_remove (5 ms) 2023-09-23T23:43:31.186 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.image_group_get 2023-09-23T23:43:31.188 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-09-23T23:43:31.188 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-09-23T23:43:31.190 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (3 ms) 2023-09-23T23:43:31.191 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-09-23T23:43:31.193 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-09-23T23:43:31.193 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-09-23T23:43:31.196 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-09-23T23:43:31.196 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-09-23T23:43:31.201 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-09-23T23:43:31.202 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-09-23T23:43:31.202 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_set (2 ms) 2023-09-23T23:43:31.202 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-09-23T23:43:31.205 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_list (4 ms) 2023-09-23T23:43:31.205 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-09-23T23:43:31.223 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (18 ms) 2023-09-23T23:43:31.223 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-09-23T23:43:31.357 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:31.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:31.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:31.426 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (202 ms) 2023-09-23T23:43:31.426 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-09-23T23:43:31.429 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_remove (4 ms) 2023-09-23T23:43:31.429 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-09-23T23:43:31.431 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-09-23T23:43:31.431 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.trash_methods 2023-09-23T23:43:31.438 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2023-09-23T23:43:31.438 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.op_features 2023-09-23T23:43:31.446 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.op_features (8 ms) 2023-09-23T23:43:31.446 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.clone_parent 2023-09-23T23:43:31.461 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:31.487 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.clone_parent (40 ms) 2023-09-23T23:43:31.487 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-09-23T23:43:31.489 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:31.499 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.clone_parent_ns (12 ms) 2023-09-23T23:43:31.499 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.clone_child 2023-09-23T23:43:31.524 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.clone_child (26 ms) 2023-09-23T23:43:31.524 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-09-23T23:43:31.532 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.namespace_methods (8 ms) 2023-09-23T23:43:31.532 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.migration 2023-09-23T23:43:31.533 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:31.535 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:31 smithi138 ceph-mon[193606]: pgmap v1957: 135 pgs: 23 unknown, 112 active+clean; 621 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 1.4 MiB/s rd, 699 KiB/s wr, 9 op/s; 27 B/s, 1 objects/s recovering 2023-09-23T23:43:31.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:31.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:31.537 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:31.543 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.migration (11 ms) 2023-09-23T23:43:31.543 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.migration_v1 2023-09-23T23:43:31.552 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.migration_v1 (9 ms) 2023-09-23T23:43:31.552 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-09-23T23:43:31.602 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:31.638 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:31.663 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:31 smithi110 ceph-mon[212939]: pgmap v1957: 135 pgs: 23 unknown, 112 active+clean; 621 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 1.4 MiB/s rd, 699 KiB/s wr, 9 op/s; 27 B/s, 1 objects/s recovering 2023-09-23T23:43:31.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:31 smithi110 ceph-mon[221102]: pgmap v1957: 135 pgs: 23 unknown, 112 active+clean; 621 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 1.4 MiB/s rd, 699 KiB/s wr, 9 op/s; 27 B/s, 1 objects/s recovering 2023-09-23T23:43:31.665 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:31.822 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:31.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:31.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:31.925 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:31.958 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:32.158 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:32.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:32.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:32.261 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:32.291 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:32.337 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:32.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:32.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:32.445 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:32.472 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:32 smithi110 ceph-mon[212939]: 1.e scrub starts 2023-09-23T23:43:32.472 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:32 smithi110 ceph-mon[212939]: 1.e scrub ok 2023-09-23T23:43:32.472 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:32 smithi110 ceph-mon[221102]: 1.e scrub starts 2023-09-23T23:43:32.472 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:32 smithi110 ceph-mon[221102]: 1.e scrub ok 2023-09-23T23:43:32.475 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:32.617 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:32 smithi138 ceph-mon[193606]: 1.e scrub starts 2023-09-23T23:43:32.618 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:32 smithi138 ceph-mon[193606]: 1.e scrub ok 2023-09-23T23:43:32.672 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:32.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:32.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:32.777 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:32.803 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:32.838 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:43:32.838 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:43:32.838 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:43:32.838 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:43:32.838 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71516 13.e 13:7de122bf:::benchmark_data_smithi110_541928_object71515:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e555)", 2023-09-23T23:43:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.528660+0000", 2023-09-23T23:43:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 582.30387272999997, 2023-09-23T23:43:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2167170459999999, 2023-09-23T23:43:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:43:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:43:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:43:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:43:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71516 2023-09-23T23:43:32.839 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528660+0000", 2023-09-23T23:43:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528660+0000", 2023-09-23T23:43:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.840 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528659+0000", 2023-09-23T23:43:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:43:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528700+0000", 2023-09-23T23:43:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0859999999999998e-05 2023-09-23T23:43:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528712+0000", 2023-09-23T23:43:32.841 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2262000000000001e-05 2023-09-23T23:43:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.528719+0000", 2023-09-23T23:43:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3339999999999998e-06 2023-09-23T23:43:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531320+0000", 2023-09-23T23:43:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002601506 2023-09-23T23:43:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.842 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531460+0000", 2023-09-23T23:43:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013956799999999999 2023-09-23T23:43:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:43:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.531617+0000", 2023-09-23T23:43:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015701199999999999 2023-09-23T23:43:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:43:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.534591+0000", 2023-09-23T23:43:32.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029740880000000002 2023-09-23T23:43:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.545743+0000", 2023-09-23T23:43:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011152331 2023-09-23T23:43:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.545752+0000", 2023-09-23T23:43:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0000000000000002e-06 2023-09-23T23:43:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.844 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552315+0000", 2023-09-23T23:43:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.006563179 2023-09-23T23:43:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552330+0000", 2023-09-23T23:43:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4734e-05 2023-09-23T23:43:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745274+0000", 2023-09-23T23:43:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1929441679999999 2023-09-23T23:43:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745377+0000", 2023-09-23T23:43:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103303 2023-09-23T23:43:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71528 13.e 13:790d8dd9:::benchmark_data_smithi110_541928_object71527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:43:32.846 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.572698+0000", 2023-09-23T23:43:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 582.25983496100002, 2023-09-23T23:43:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2677280570000002, 2023-09-23T23:43:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:43:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:43:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:43:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:43:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71528 2023-09-23T23:43:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.847 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572698+0000", 2023-09-23T23:43:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572698+0000", 2023-09-23T23:43:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572697+0000", 2023-09-23T23:43:32.848 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:43:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572737+0000", 2023-09-23T23:43:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0278000000000003e-05 2023-09-23T23:43:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572746+0000", 2023-09-23T23:43:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3340000000000008e-06 2023-09-23T23:43:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.849 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572753+0000", 2023-09-23T23:43:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3899999999999998e-06 2023-09-23T23:43:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572817+0000", 2023-09-23T23:43:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4576e-05 2023-09-23T23:43:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.572827+0000", 2023-09-23T23:43:32.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.978e-06 2023-09-23T23:43:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552357+0000", 2023-09-23T23:43:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97953002600000005 2023-09-23T23:43:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552365+0000", 2023-09-23T23:43:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5390000000000002e-06 2023-09-23T23:43:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745405+0000", 2023-09-23T23:43:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1930404109999999 2023-09-23T23:43:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745543+0000", 2023-09-23T23:43:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013780300000000001 2023-09-23T23:43:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:43:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.745737+0000", 2023-09-23T23:43:32.852 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019410400000000001 2023-09-23T23:43:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:43:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.759566+0000", 2023-09-23T23:43:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013828844999999999 2023-09-23T23:43:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.768156+0000", 2023-09-23T23:43:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085902789999999993 2023-09-23T23:43:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.836351+0000", 2023-09-23T23:43:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.068194406999999999 2023-09-23T23:43:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:43:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.836426+0000", 2023-09-23T23:43:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4889999999999996e-05 2023-09-23T23:43:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.840426+0000", 2023-09-23T23:43:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040005659999999997 2023-09-23T23:43:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:71532 13.e 13:7da393ae:::benchmark_data_smithi110_541928_object71531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e556)", 2023-09-23T23:43:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:33:50.575009+0000", 2023-09-23T23:43:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 582.25752387399996, 2023-09-23T23:43:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2696883859999999, 2023-09-23T23:43:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.855 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:43:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:43:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:43:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:43:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 71532 2023-09-23T23:43:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575009+0000", 2023-09-23T23:43:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.856 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575009+0000", 2023-09-23T23:43:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575004+0000", 2023-09-23T23:43:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999948 2023-09-23T23:43:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575068+0000", 2023-09-23T23:43:32.857 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3359000000000001e-05 2023-09-23T23:43:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575081+0000", 2023-09-23T23:43:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3434999999999999e-05 2023-09-23T23:43:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575090+0000", 2023-09-23T23:43:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7490000000000004e-06 2023-09-23T23:43:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.858 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575154+0000", 2023-09-23T23:43:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4023999999999996e-05 2023-09-23T23:43:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:50.575166+0000", 2023-09-23T23:43:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2033e-05 2023-09-23T23:43:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552388+0000", 2023-09-23T23:43:32.859 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97722166399999999 2023-09-23T23:43:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:51.552395+0000", 2023-09-23T23:43:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3429999999999998e-06 2023-09-23T23:43:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.748633+0000", 2023-09-23T23:43:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1962377799999999 2023-09-23T23:43:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.748799+0000", 2023-09-23T23:43:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016630200000000001 2023-09-23T23:43:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-09-23T23:43:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.748992+0000", 2023-09-23T23:43:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019316 2023-09-23T23:43:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:43:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.759572+0000", 2023-09-23T23:43:32.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010580032 2023-09-23T23:43:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.768191+0000", 2023-09-23T23:43:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086191280000000002 2023-09-23T23:43:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.840481+0000", 2023-09-23T23:43:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.072289500000000007 2023-09-23T23:43:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.862 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:43:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.840542+0000", 2023-09-23T23:43:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1509999999999999e-05 2023-09-23T23:43:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:33:52.844698+0000", 2023-09-23T23:43:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041553880000000003 2023-09-23T23:43:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.863 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77337 13.d 13:ba15f28f:::benchmark_data_smithi110_541928_object77336:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:43:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.706918+0000", 2023-09-23T23:43:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.12561489899997, 2023-09-23T23:43:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1549711069999999, 2023-09-23T23:43:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:43:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:43:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:43:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:43:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77337 2023-09-23T23:43:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.864 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706918+0000", 2023-09-23T23:43:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706918+0000", 2023-09-23T23:43:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.865 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706917+0000", 2023-09-23T23:43:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:43:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706970+0000", 2023-09-23T23:43:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2435000000000002e-05 2023-09-23T23:43:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706986+0000", 2023-09-23T23:43:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6252999999999998e-05 2023-09-23T23:43:32.866 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.706993+0000", 2023-09-23T23:43:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1439999999999997e-06 2023-09-23T23:43:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709103+0000", 2023-09-23T23:43:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021098419999999998 2023-09-23T23:43:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.867 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709112+0000", 2023-09-23T23:43:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3640000000000005e-06 2023-09-23T23:43:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.860680+0000", 2023-09-23T23:43:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1515676990000001 2023-09-23T23:43:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.861889+0000", 2023-09-23T23:43:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012094930000000001 2023-09-23T23:43:32.868 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77365 13.d 13:bd3fed09:::benchmark_data_smithi110_541928_object77364:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:43:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.707869+0000", 2023-09-23T23:43:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.12466462099997, 2023-09-23T23:43:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1996441330000001, 2023-09-23T23:43:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:43:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:43:32.869 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:43:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:43:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77365 2023-09-23T23:43:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707869+0000", 2023-09-23T23:43:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.870 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707869+0000", 2023-09-23T23:43:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707868+0000", 2023-09-23T23:43:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:43:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707912+0000", 2023-09-23T23:43:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4286999999999998e-05 2023-09-23T23:43:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707922+0000", 2023-09-23T23:43:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0132000000000001e-05 2023-09-23T23:43:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707929+0000", 2023-09-23T23:43:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4690000000000002e-06 2023-09-23T23:43:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.872 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709143+0000", 2023-09-23T23:43:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012139539999999999 2023-09-23T23:43:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709148+0000", 2023-09-23T23:43:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8420000000000001e-06 2023-09-23T23:43:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.861913+0000", 2023-09-23T23:43:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.152764984 2023-09-23T23:43:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.862371+0000", 2023-09-23T23:43:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00045722599999999998 2023-09-23T23:43:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:43:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.862643+0000", 2023-09-23T23:43:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000271983 2023-09-23T23:43:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:43:32.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.872385+0000", 2023-09-23T23:43:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0097426379999999996 2023-09-23T23:43:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.905589+0000", 2023-09-23T23:43:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.033203780000000002 2023-09-23T23:43:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.905624+0000", 2023-09-23T23:43:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5058000000000002e-05 2023-09-23T23:43:32.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:43:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.905662+0000", 2023-09-23T23:43:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7523e-05 2023-09-23T23:43:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907513+0000", 2023-09-23T23:43:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0018510320000000001 2023-09-23T23:43:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.876 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77391 13.d 13:bfca221d:::benchmark_data_smithi110_541928_object77390:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:43:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.707935+0000", 2023-09-23T23:43:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.12459816000001, 2023-09-23T23:43:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2024361749999999, 2023-09-23T23:43:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:43:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:43:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:43:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:43:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77391 2023-09-23T23:43:32.877 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707935+0000", 2023-09-23T23:43:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707935+0000", 2023-09-23T23:43:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707935+0000", 2023-09-23T23:43:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:43:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707971+0000", 2023-09-23T23:43:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6470000000000001e-05 2023-09-23T23:43:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707977+0000", 2023-09-23T23:43:32.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1489999999999999e-06 2023-09-23T23:43:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707982+0000", 2023-09-23T23:43:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5349999999999998e-06 2023-09-23T23:43:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709170+0000", 2023-09-23T23:43:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011882080000000001 2023-09-23T23:43:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709176+0000", 2023-09-23T23:43:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.297e-06 2023-09-23T23:43:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.866177+0000", 2023-09-23T23:43:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1570007 2023-09-23T23:43:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.866317+0000", 2023-09-23T23:43:32.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013973499999999999 2023-09-23T23:43:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:43:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.866488+0000", 2023-09-23T23:43:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000171495 2023-09-23T23:43:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:43:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.875118+0000", 2023-09-23T23:43:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086296299999999992 2023-09-23T23:43:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.882 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.905607+0000", 2023-09-23T23:43:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.030489532999999999 2023-09-23T23:43:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907981+0000", 2023-09-23T23:43:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023734239999999998 2023-09-23T23:43:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:43:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.908040+0000", 2023-09-23T23:43:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9463000000000003e-05 2023-09-23T23:43:32.883 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910371+0000", 2023-09-23T23:43:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023309279999999999 2023-09-23T23:43:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77394 13.d 13:bd1f7538:::benchmark_data_smithi110_541928_object77393:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:43:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.707987+0000", 2023-09-23T23:43:32.884 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.12454641800002, 2023-09-23T23:43:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.204224143, 2023-09-23T23:43:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:43:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:43:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:43:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:43:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77394 2023-09-23T23:43:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.885 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707987+0000", 2023-09-23T23:43:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707987+0000", 2023-09-23T23:43:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.707986+0000", 2023-09-23T23:43:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:43:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.886 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708018+0000", 2023-09-23T23:43:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1340999999999997e-05 2023-09-23T23:43:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708024+0000", 2023-09-23T23:43:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9050000000000002e-06 2023-09-23T23:43:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708027+0000", 2023-09-23T23:43:32.887 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6550000000000002e-06 2023-09-23T23:43:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709199+0000", 2023-09-23T23:43:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011713400000000001 2023-09-23T23:43:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709204+0000", 2023-09-23T23:43:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4480000000000002e-06 2023-09-23T23:43:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.888 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.869263+0000", 2023-09-23T23:43:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1600593370000001 2023-09-23T23:43:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.869419+0000", 2023-09-23T23:43:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015545899999999999 2023-09-23T23:43:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:43:32.889 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.869589+0000", 2023-09-23T23:43:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000170225 2023-09-23T23:43:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:43:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889913+0000", 2023-09-23T23:43:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020323536 2023-09-23T23:43:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907553+0000", 2023-09-23T23:43:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017640769000000001 2023-09-23T23:43:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.890 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910571+0000", 2023-09-23T23:43:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030181119999999999 2023-09-23T23:43:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:43:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910620+0000", 2023-09-23T23:43:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8891000000000003e-05 2023-09-23T23:43:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.891 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912211+0000", 2023-09-23T23:43:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015906270000000001 2023-09-23T23:43:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77463 13.d 13:b8eb00c3:::benchmark_data_smithi110_541928_object77462:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:43:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.708031+0000", 2023-09-23T23:43:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.12450171700004, 2023-09-23T23:43:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.206478288, 2023-09-23T23:43:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:43:32.892 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:43:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:43:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:43:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77463 2023-09-23T23:43:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708031+0000", 2023-09-23T23:43:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.893 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708031+0000", 2023-09-23T23:43:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708031+0000", 2023-09-23T23:43:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:43:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708086+0000", 2023-09-23T23:43:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4630999999999999e-05 2023-09-23T23:43:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.894 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708092+0000", 2023-09-23T23:43:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1800000000000001e-06 2023-09-23T23:43:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708096+0000", 2023-09-23T23:43:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.811e-06 2023-09-23T23:43:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709228+0000", 2023-09-23T23:43:32.895 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011318190000000001 2023-09-23T23:43:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709234+0000", 2023-09-23T23:43:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2979999999999997e-06 2023-09-23T23:43:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.872414+0000", 2023-09-23T23:43:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.163180543 2023-09-23T23:43:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.896 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.872546+0000", 2023-09-23T23:43:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013156499999999999 2023-09-23T23:43:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:43:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.872714+0000", 2023-09-23T23:43:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000167713 2023-09-23T23:43:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:43:32.897 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889920+0000", 2023-09-23T23:43:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017206255 2023-09-23T23:43:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907578+0000", 2023-09-23T23:43:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017658231 2023-09-23T23:43:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912462+0000", 2023-09-23T23:43:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048837949999999998 2023-09-23T23:43:32.898 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:43:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912506+0000", 2023-09-23T23:43:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4131999999999997e-05 2023-09-23T23:43:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.914510+0000", 2023-09-23T23:43:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002003632 2023-09-23T23:43:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.899 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77467 13.d 13:b265ce95:::benchmark_data_smithi110_541928_object77466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:43:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.708101+0000", 2023-09-23T23:43:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.12443230300005, 2023-09-23T23:43:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2086463730000001, 2023-09-23T23:43:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:43:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:43:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:43:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:43:32.900 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77467 2023-09-23T23:43:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708101+0000", 2023-09-23T23:43:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708101+0000", 2023-09-23T23:43:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.901 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708101+0000", 2023-09-23T23:43:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:43:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708157+0000", 2023-09-23T23:43:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6196999999999997e-05 2023-09-23T23:43:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.902 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708163+0000", 2023-09-23T23:43:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0939999999999996e-06 2023-09-23T23:43:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708168+0000", 2023-09-23T23:43:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1800000000000004e-06 2023-09-23T23:43:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709259+0000", 2023-09-23T23:43:32.903 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010911370000000001 2023-09-23T23:43:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709266+0000", 2023-09-23T23:43:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7870000000000004e-06 2023-09-23T23:43:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.875164+0000", 2023-09-23T23:43:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1658984800000001 2023-09-23T23:43:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.904 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.875272+0000", 2023-09-23T23:43:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010731299999999999 2023-09-23T23:43:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:43:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.875417+0000", 2023-09-23T23:43:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014540399999999999 2023-09-23T23:43:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:43:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889926+0000", 2023-09-23T23:43:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014509024000000001 2023-09-23T23:43:32.905 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.907925+0000", 2023-09-23T23:43:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017998692 2023-09-23T23:43:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.914568+0000", 2023-09-23T23:43:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066434789999999999 2023-09-23T23:43:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.906 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:43:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.914630+0000", 2023-09-23T23:43:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1175999999999998e-05 2023-09-23T23:43:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.916747+0000", 2023-09-23T23:43:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021177470000000001 2023-09-23T23:43:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.907 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77471 13.d 13:bd1602ed:::benchmark_data_smithi110_541928_object77470:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:43:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.708173+0000", 2023-09-23T23:43:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.12436045200002, 2023-09-23T23:43:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2113011469999999, 2023-09-23T23:43:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:43:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:43:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:43:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:43:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77471 2023-09-23T23:43:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.908 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708173+0000", 2023-09-23T23:43:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708173+0000", 2023-09-23T23:43:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708172+0000", 2023-09-23T23:43:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:43:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.909 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708227+0000", 2023-09-23T23:43:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4215000000000002e-05 2023-09-23T23:43:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708234+0000", 2023-09-23T23:43:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0790000000000003e-06 2023-09-23T23:43:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.910 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708238+0000", 2023-09-23T23:43:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6450000000000004e-06 2023-09-23T23:43:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709564+0000", 2023-09-23T23:43:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0013253379999999999 2023-09-23T23:43:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709570+0000", 2023-09-23T23:43:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5570000000000001e-06 2023-09-23T23:43:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.911 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.877443+0000", 2023-09-23T23:43:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.167873216 2023-09-23T23:43:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.877572+0000", 2023-09-23T23:43:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012810999999999999 2023-09-23T23:43:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:43:32.912 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.877732+0000", 2023-09-23T23:43:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000160376 2023-09-23T23:43:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:43:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889933+0000", 2023-09-23T23:43:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012201033999999999 2023-09-23T23:43:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910445+0000", 2023-09-23T23:43:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020511715999999999 2023-09-23T23:43:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.913 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.916902+0000", 2023-09-23T23:43:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064569379999999997 2023-09-23T23:43:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:43:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.916958+0000", 2023-09-23T23:43:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6443000000000002e-05 2023-09-23T23:43:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.914 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.919474+0000", 2023-09-23T23:43:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002515804 2023-09-23T23:43:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77477 13.d 13:b03c7b50:::benchmark_data_smithi110_541928_object77476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:43:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.708243+0000", 2023-09-23T23:43:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.12429046199998, 2023-09-23T23:43:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.213803755, 2023-09-23T23:43:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.915 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:43:32.916 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:43:32.916 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:43:32.916 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:43:32.916 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77477 2023-09-23T23:43:32.916 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.916 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.916 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.916 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.916 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708243+0000", 2023-09-23T23:43:32.916 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.916 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.916 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.916 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708243+0000", 2023-09-23T23:43:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708242+0000", 2023-09-23T23:43:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:43:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708286+0000", 2023-09-23T23:43:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3625999999999999e-05 2023-09-23T23:43:32.917 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708292+0000", 2023-09-23T23:43:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0100000000000001e-06 2023-09-23T23:43:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.708296+0000", 2023-09-23T23:43:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1860000000000002e-06 2023-09-23T23:43:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.918 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709595+0000", 2023-09-23T23:43:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012991089999999999 2023-09-23T23:43:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.709602+0000", 2023-09-23T23:43:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6100000000000002e-06 2023-09-23T23:43:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.879894+0000", 2023-09-23T23:43:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.170292077 2023-09-23T23:43:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.919 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.879990+0000", 2023-09-23T23:43:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6212000000000007e-05 2023-09-23T23:43:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:43:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.880120+0000", 2023-09-23T23:43:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012987 2023-09-23T23:43:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:43:32.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889937+0000", 2023-09-23T23:43:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0098164270000000008 2023-09-23T23:43:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910473+0000", 2023-09-23T23:43:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020536688000000001 2023-09-23T23:43:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.919527+0000", 2023-09-23T23:43:32.921 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0090542190000000005 2023-09-23T23:43:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:43:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.919586+0000", 2023-09-23T23:43:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8749999999999998e-05 2023-09-23T23:43:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.922046+0000", 2023-09-23T23:43:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024603099999999998 2023-09-23T23:43:32.922 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77503 13.d 13:b2378296:::benchmark_data_smithi110_541928_object77502:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:43:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.781601+0000", 2023-09-23T23:43:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.05093213299995, 2023-09-23T23:43:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1428059340000001, 2023-09-23T23:43:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:43:32.923 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:43:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:43:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:43:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77503 2023-09-23T23:43:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781601+0000", 2023-09-23T23:43:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.924 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781601+0000", 2023-09-23T23:43:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781600+0000", 2023-09-23T23:43:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:43:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781628+0000", 2023-09-23T23:43:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8694999999999999e-05 2023-09-23T23:43:32.925 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781638+0000", 2023-09-23T23:43:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5959999999999993e-06 2023-09-23T23:43:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781643+0000", 2023-09-23T23:43:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4970000000000001e-06 2023-09-23T23:43:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.926 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781701+0000", 2023-09-23T23:43:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7757000000000003e-05 2023-09-23T23:43:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.781711+0000", 2023-09-23T23:43:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4409999999999999e-06 2023-09-23T23:43:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.882289+0000", 2023-09-23T23:43:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1005785459999999 2023-09-23T23:43:32.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.882445+0000", 2023-09-23T23:43:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015557 2023-09-23T23:43:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:43:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.882635+0000", 2023-09-23T23:43:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000190087 2023-09-23T23:43:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.928 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:43:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.892019+0000", 2023-09-23T23:43:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0093840280000000009 2023-09-23T23:43:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910507+0000", 2023-09-23T23:43:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018488389000000001 2023-09-23T23:43:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.922095+0000", 2023-09-23T23:43:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011588144999999999 2023-09-23T23:43:32.929 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:43:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.922148+0000", 2023-09-23T23:43:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2661000000000003e-05 2023-09-23T23:43:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.924407+0000", 2023-09-23T23:43:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022589189999999999 2023-09-23T23:43:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.930 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77524 13.d 13:bc4b3c00:::benchmark_data_smithi110_541928_object77523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:43:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.815029+0000", 2023-09-23T23:43:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.01750394199996, 2023-09-23T23:43:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.112009437, 2023-09-23T23:43:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:43:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:43:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:43:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:43:32.931 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77524 2023-09-23T23:43:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815029+0000", 2023-09-23T23:43:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815029+0000", 2023-09-23T23:43:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.932 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815028+0000", 2023-09-23T23:43:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:43:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815060+0000", 2023-09-23T23:43:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2218999999999998e-05 2023-09-23T23:43:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.933 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815074+0000", 2023-09-23T23:43:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.43e-05 2023-09-23T23:43:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815081+0000", 2023-09-23T23:43:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0700000000000003e-06 2023-09-23T23:43:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815152+0000", 2023-09-23T23:43:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1408999999999993e-05 2023-09-23T23:43:32.934 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.815162+0000", 2023-09-23T23:43:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9459999999999993e-06 2023-09-23T23:43:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.884789+0000", 2023-09-23T23:43:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.069627511 2023-09-23T23:43:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.935 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.884895+0000", 2023-09-23T23:43:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105283 2023-09-23T23:43:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:43:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.885012+0000", 2023-09-23T23:43:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011697900000000001 2023-09-23T23:43:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:43:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.892026+0000", 2023-09-23T23:43:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0070145779999999996 2023-09-23T23:43:32.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.910539+0000", 2023-09-23T23:43:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018512857000000001 2023-09-23T23:43:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.924468+0000", 2023-09-23T23:43:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01392849 2023-09-23T23:43:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:43:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.924523+0000", 2023-09-23T23:43:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4948000000000003e-05 2023-09-23T23:43:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.927039+0000", 2023-09-23T23:43:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002516108 2023-09-23T23:43:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77547 13.d 13:bc3de976:::benchmark_data_smithi110_541928_object77546:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:43:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.859759+0000", 2023-09-23T23:43:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 568.97277374700002, 2023-09-23T23:43:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0697625900000001, 2023-09-23T23:43:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:43:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:43:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:43:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:43:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77547 2023-09-23T23:43:32.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859759+0000", 2023-09-23T23:43:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859759+0000", 2023-09-23T23:43:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859758+0000", 2023-09-23T23:43:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:43:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859847+0000", 2023-09-23T23:43:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8799000000000001e-05 2023-09-23T23:43:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859861+0000", 2023-09-23T23:43:32.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4229e-05 2023-09-23T23:43:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859868+0000", 2023-09-23T23:43:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4949999999999996e-06 2023-09-23T23:43:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859928+0000", 2023-09-23T23:43:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0738e-05 2023-09-23T23:43:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.942 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.859941+0000", 2023-09-23T23:43:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.24e-05 2023-09-23T23:43:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.887534+0000", 2023-09-23T23:43:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0275931309999999 2023-09-23T23:43:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.887728+0000", 2023-09-23T23:43:32.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000194165 2023-09-23T23:43:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:43:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.887881+0000", 2023-09-23T23:43:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000153263 2023-09-23T23:43:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:43:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.903469+0000", 2023-09-23T23:43:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015587215 2023-09-23T23:43:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.944 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912270+0000", 2023-09-23T23:43:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088015479999999993 2023-09-23T23:43:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.927101+0000", 2023-09-23T23:43:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014831195 2023-09-23T23:43:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:43:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.927169+0000", 2023-09-23T23:43:32.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8172999999999999e-05 2023-09-23T23:43:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.929522+0000", 2023-09-23T23:43:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023525769999999998 2023-09-23T23:43:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:77576 13.d 13:b73edecb:::benchmark_data_smithi110_541928_object77575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e563)", 2023-09-23T23:43:32.946 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:03.919341+0000", 2023-09-23T23:43:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 568.913192407, 2023-09-23T23:43:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0128839199999999, 2023-09-23T23:43:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:43:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:43:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:43:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:43:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77576 2023-09-23T23:43:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.947 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919341+0000", 2023-09-23T23:43:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919341+0000", 2023-09-23T23:43:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919339+0000", 2023-09-23T23:43:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:43:32.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919384+0000", 2023-09-23T23:43:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5042e-05 2023-09-23T23:43:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919403+0000", 2023-09-23T23:43:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9136000000000001e-05 2023-09-23T23:43:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.919412+0000", 2023-09-23T23:43:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4589999999999998e-06 2023-09-23T23:43:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.924505+0000", 2023-09-23T23:43:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050933469999999998 2023-09-23T23:43:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:43:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:03.924519+0000", 2023-09-23T23:43:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3852000000000001e-05 2023-09-23T23:43:32.950 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.889955+0000", 2023-09-23T23:43:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96543663400000002 2023-09-23T23:43:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.890057+0000", 2023-09-23T23:43:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010134899999999999 2023-09-23T23:43:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:43:32.951 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.890180+0000", 2023-09-23T23:43:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012325699999999999 2023-09-23T23:43:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:43:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.903476+0000", 2023-09-23T23:43:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013296290000000001 2023-09-23T23:43:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.912329+0000", 2023-09-23T23:43:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088524599999999995 2023-09-23T23:43:32.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:43:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.929574+0000", 2023-09-23T23:43:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017245415 2023-09-23T23:43:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:43:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.929630+0000", 2023-09-23T23:43:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6097999999999997e-05 2023-09-23T23:43:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:04.932225+0000", 2023-09-23T23:43:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002594332 2023-09-23T23:43:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46514.0:153463 13.3 e606/571 13:cf006423:::benchmark_data_smithi110_541928_object65683:head v 606'9682, mlcod=0'0)", 2023-09-23T23:43:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:38:16.630961+0000", 2023-09-23T23:43:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 316.20157252299998, 2023-09-23T23:43:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25026810500000002, 2023-09-23T23:43:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.954 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:43:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.630961+0000", 2023-09-23T23:43:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.630961+0000", 2023-09-23T23:43:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.956 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.630959+0000", 2023-09-23T23:43:32.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:43:32.956 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.956 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.956 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.630971+0000", 2023-09-23T23:43:32.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.207e-05 2023-09-23T23:43:32.956 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.956 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.956 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.630992+0000", 2023-09-23T23:43:32.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0177e-05 2023-09-23T23:43:32.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.631002+0000", 2023-09-23T23:43:32.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0772e-05 2023-09-23T23:43:32.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.631072+0000", 2023-09-23T23:43:32.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9306000000000004e-05 2023-09-23T23:43:32.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.631110+0000", 2023-09-23T23:43:32.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8408999999999998e-05 2023-09-23T23:43:32.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:43:32.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.881180+0000", 2023-09-23T23:43:32.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25006991499999998 2023-09-23T23:43:32.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.881229+0000", 2023-09-23T23:43:32.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8729000000000001e-05 2023-09-23T23:43:32.959 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.959 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.959 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.959 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.959 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(13.3 606/571 [PushOp(13:ce83adce:::benchmark_data_smithi110_541928_object65693:head, version: 554'4159, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(13:ce83adce:::benchmark_data_smithi110_541928_object65693:head@554'4159, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:43:32.959 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:38:16.632618+0000", 2023-09-23T23:43:32.959 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 316.19991481699998, 2023-09-23T23:43:32.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.248557219, 2023-09-23T23:43:32.959 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.959 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:43:32.960 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.632618+0000", 2023-09-23T23:43:32.960 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.632618+0000", 2023-09-23T23:43:32.960 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.632618+0000", 2023-09-23T23:43:32.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:43:32.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.632645+0000", 2023-09-23T23:43:32.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7268e-05 2023-09-23T23:43:32.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.875012+0000", 2023-09-23T23:43:32.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24236708800000001 2023-09-23T23:43:32.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.875052+0000", 2023-09-23T23:43:32.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9263e-05 2023-09-23T23:43:32.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.878339+0000", 2023-09-23T23:43:32.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032871229999999999 2023-09-23T23:43:32.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.963 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.963 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.878347+0000", 2023-09-23T23:43:32.963 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9410000000000007e-06 2023-09-23T23:43:32.963 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.963 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.963 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.881176+0000", 2023-09-23T23:43:32.963 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028289719999999999 2023-09-23T23:43:32.963 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.963 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.963 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.964 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.964 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.964 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.47186.0:15760 14.f e621/617 14:f2ab693e:::benchmark_data_smithi110_640334_object15759:head v 621'1010, mlcod=621'1008)", 2023-09-23T23:43:32.964 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:39:29.338854+0000", 2023-09-23T23:43:32.964 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 243.493678727, 2023-09-23T23:43:32.964 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17898309600000001, 2023-09-23T23:43:32.964 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.964 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:43:32.964 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.964 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.964 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.964 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.338854+0000", 2023-09-23T23:43:32.964 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.965 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.965 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.965 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.965 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.338854+0000", 2023-09-23T23:43:32.965 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.965 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.965 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.965 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.965 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.338852+0000", 2023-09-23T23:43:32.965 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:43:32.965 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.965 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.965 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.338913+0000", 2023-09-23T23:43:32.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0566000000000003e-05 2023-09-23T23:43:32.966 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.966 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.966 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.338941+0000", 2023-09-23T23:43:32.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8419e-05 2023-09-23T23:43:32.966 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.966 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.966 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.338953+0000", 2023-09-23T23:43:32.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1163e-05 2023-09-23T23:43:32.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.339066+0000", 2023-09-23T23:43:32.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011311099999999999 2023-09-23T23:43:32.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.339120+0000", 2023-09-23T23:43:32.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4236e-05 2023-09-23T23:43:32.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:43:32.968 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.517791+0000", 2023-09-23T23:43:32.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.178671474 2023-09-23T23:43:32.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.968 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.517838+0000", 2023-09-23T23:43:32.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6100000000000002e-05 2023-09-23T23:43:32.968 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.968 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.968 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.969 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.47186.0:15762 14.5 e621/617 14:a375336d:::benchmark_data_smithi110_640334_object15761:head v 621'992, mlcod=621'990)", 2023-09-23T23:43:32.969 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:39:29.340320+0000", 2023-09-23T23:43:32.969 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 243.49221270800001, 2023-09-23T23:43:32.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.176954217, 2023-09-23T23:43:32.969 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.969 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:43:32.969 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.340320+0000", 2023-09-23T23:43:32.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.340320+0000", 2023-09-23T23:43:32.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.340320+0000", 2023-09-23T23:43:32.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:43:32.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.340377+0000", 2023-09-23T23:43:32.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7104000000000001e-05 2023-09-23T23:43:32.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.515478+0000", 2023-09-23T23:43:32.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17510123999999999 2023-09-23T23:43:32.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.515486+0000", 2023-09-23T23:43:32.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6279999999999996e-06 2023-09-23T23:43:32.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.515567+0000", 2023-09-23T23:43:32.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1081999999999994e-05 2023-09-23T23:43:32.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.515588+0000", 2023-09-23T23:43:32.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1427e-05 2023-09-23T23:43:32.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:43:32.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.517248+0000", 2023-09-23T23:43:32.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001659811 2023-09-23T23:43:32.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.517275+0000", 2023-09-23T23:43:32.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6557e-05 2023-09-23T23:43:32.973 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.973 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.973 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.974 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.47186.0:18768 14.5 e622/617 14:a2f3bbdf:::benchmark_data_smithi110_640334_object18767:head v 622'1177, mlcod=621'1175)", 2023-09-23T23:43:32.974 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:39:31.069697+0000", 2023-09-23T23:43:32.974 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 241.76283610199999, 2023-09-23T23:43:32.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19107984, 2023-09-23T23:43:32.974 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:43:32.974 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:43:32.974 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:43:32.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.974 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:43:32.974 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.069697+0000", 2023-09-23T23:43:32.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.975 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.975 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:43:32.975 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.069697+0000", 2023-09-23T23:43:32.975 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:43:32.975 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.975 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.975 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:43:32.975 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.069697+0000", 2023-09-23T23:43:32.975 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:43:32.975 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.975 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.975 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:43:32.976 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.069770+0000", 2023-09-23T23:43:32.976 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3033000000000004e-05 2023-09-23T23:43:32.976 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.976 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.976 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:43:32.976 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.259501+0000", 2023-09-23T23:43:32.976 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.189730864 2023-09-23T23:43:32.976 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.976 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.976 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:43:32.976 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.259510+0000", 2023-09-23T23:43:32.976 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0920000000000004e-06 2023-09-23T23:43:32.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:43:32.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.259613+0000", 2023-09-23T23:43:32.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010329599999999999 2023-09-23T23:43:32.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:43:32.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.259633+0000", 2023-09-23T23:43:32.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0126999999999999e-05 2023-09-23T23:43:32.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:43:32.978 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.260743+0000", 2023-09-23T23:43:32.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011101730000000001 2023-09-23T23:43:32.978 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:43:32.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:43:32.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:43:32.978 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.260777+0000", 2023-09-23T23:43:32.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3704999999999997e-05 2023-09-23T23:43:32.978 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.978 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.978 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.978 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:43:32.979 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:43:32.979 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:43:33.286 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1734 ms) 2023-09-23T23:43:33.286 INFO:tasks.workunit.client.0.smithi110.stdout:[ RUN ] TestClsRbd.sparsify 2023-09-23T23:43:33.294 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:43:33.323 INFO:tasks.workunit.client.0.smithi110.stdout:[ OK ] TestClsRbd.sparsify (37 ms) 2023-09-23T23:43:33.520 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:43:33.546 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:33 smithi110 ceph-mon[212939]: pgmap v1958: 135 pgs: 135 active+clean; 582 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 2.8 MiB/s rd, 1.3 MiB/s wr, 2.26k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T23:43:33.547 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:33 smithi110 ceph-mon[212939]: failed to encode map e689 with expected crc 2023-09-23T23:43:33.547 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:33 smithi110 ceph-mon[212939]: failed to encode map e689 with expected crc 2023-09-23T23:43:33.547 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:33 smithi110 ceph-mon[212939]: osdmap e689: 8 total, 8 up, 5 in 2023-09-23T23:43:33.547 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:33 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4006464562' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:43:33.547 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:33 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:43:33.547 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:33 smithi110 ceph-mon[212939]: failed to encode map e689 with expected crc 2023-09-23T23:43:33.547 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:33 smithi110 ceph-mon[212939]: failed to encode map e689 with expected crc 2023-09-23T23:43:33.547 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:33 smithi110 ceph-mon[221102]: pgmap v1958: 135 pgs: 135 active+clean; 582 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 2.8 MiB/s rd, 1.3 MiB/s wr, 2.26k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T23:43:33.548 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:33 smithi110 ceph-mon[221102]: failed to encode map e689 with expected crc 2023-09-23T23:43:33.548 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:33 smithi110 ceph-mon[221102]: failed to encode map e689 with expected crc 2023-09-23T23:43:33.548 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:33 smithi110 ceph-mon[221102]: osdmap e689: 8 total, 8 up, 5 in 2023-09-23T23:43:33.548 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:33 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4006464562' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:43:33.548 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:33 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:43:33.548 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:33 smithi110 ceph-mon[221102]: failed to encode map e689 with expected crc 2023-09-23T23:43:33.548 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:33 smithi110 ceph-mon[221102]: failed to encode map e689 with expected crc 2023-09-23T23:43:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:33 smithi138 ceph-mon[193606]: pgmap v1958: 135 pgs: 135 active+clean; 582 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 2.8 MiB/s rd, 1.3 MiB/s wr, 2.26k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T23:43:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:33 smithi138 ceph-mon[193606]: failed to encode map e689 with expected crc 2023-09-23T23:43:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:33 smithi138 ceph-mon[193606]: failed to encode map e689 with expected crc 2023-09-23T23:43:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:33 smithi138 ceph-mon[193606]: osdmap e689: 8 total, 8 up, 5 in 2023-09-23T23:43:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:33 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4006464562' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:43:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:33 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:43:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:33 smithi138 ceph-mon[193606]: failed to encode map e689 with expected crc 2023-09-23T23:43:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:33 smithi138 ceph-mon[193606]: failed to encode map e689 with expected crc 2023-09-23T23:43:33.806 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:33.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:33.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:33.910 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:43:33.937 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:34.293 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] 70 tests from TestClsRbd (6047 ms total) 2023-09-23T23:43:34.293 INFO:tasks.workunit.client.0.smithi110.stdout: 2023-09-23T23:43:34.293 INFO:tasks.workunit.client.0.smithi110.stdout:[----------] Global test environment tear-down 2023-09-23T23:43:34.293 INFO:tasks.workunit.client.0.smithi110.stdout:[==========] 70 tests from 1 test suite ran. (8439 ms total) 2023-09-23T23:43:34.293 INFO:tasks.workunit.client.0.smithi110.stdout:[ PASSED ] 70 tests. 2023-09-23T23:43:34.295 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T23:43:34.295 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T23:43:34.363 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-09-23T23:43:34.364 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T23:43:34.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:34 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:43:34.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:34 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:43:34.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:34 smithi110 ceph-mon[212939]: failed to encode map e690 with expected crc 2023-09-23T23:43:34.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:34 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:43:34.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:34 smithi110 ceph-mon[212939]: osdmap e690: 8 total, 8 up, 5 in 2023-09-23T23:43:34.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:34 smithi110 ceph-mon[212939]: failed to encode map e690 with expected crc 2023-09-23T23:43:34.434 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:34 smithi110 ceph-mon[212939]: failed to encode map e690 with expected crc 2023-09-23T23:43:34.434 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:34 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:43:34.435 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:34 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:43:34.435 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:34 smithi110 ceph-mon[221102]: failed to encode map e690 with expected crc 2023-09-23T23:43:34.435 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:34 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:43:34.435 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:34 smithi110 ceph-mon[221102]: osdmap e690: 8 total, 8 up, 5 in 2023-09-23T23:43:34.435 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:34 smithi110 ceph-mon[221102]: failed to encode map e690 with expected crc 2023-09-23T23:43:34.435 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:34 smithi110 ceph-mon[221102]: failed to encode map e690 with expected crc 2023-09-23T23:43:34.460 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:43:34.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:34.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:43:34.564 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:34.590 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:34.646 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:34.648 DEBUG:teuthology.parallel:result is None 2023-09-23T23:43:34.649 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T23:43:34.655 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:34.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:34.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:34.683 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T23:43:34.684 DEBUG:teuthology.orchestra.run.smithi110:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T23:43:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:34 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:43:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:34 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:43:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:34 smithi138 ceph-mon[193606]: failed to encode map e690 with expected crc 2023-09-23T23:43:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:34 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:43:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:34 smithi138 ceph-mon[193606]: osdmap e690: 8 total, 8 up, 5 in 2023-09-23T23:43:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:34 smithi138 ceph-mon[193606]: failed to encode map e690 with expected crc 2023-09-23T23:43:34.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:34 smithi138 ceph-mon[193606]: failed to encode map e690 with expected crc 2023-09-23T23:43:34.761 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:34.772 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T23:43:34.772 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T23:43:34.772 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-09-23T23:43:34.772 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T23:43:34.772 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T23:43:34.772 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T23:43:34.773 DEBUG:teuthology.orchestra.run.smithi110:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T23:43:34.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T23:43:34.792 INFO:teuthology.orchestra.run.smithi110.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T23:43:34.792 DEBUG:teuthology.orchestra.run.smithi110:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T23:43:34.797 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:34.848 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:34.849 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T23:43:34.850 DEBUG:teuthology.orchestra.run.smithi110:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T23:43:34.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:34.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:34.907 INFO:tasks.workunit:timeout=3h 2023-09-23T23:43:34.907 INFO:tasks.workunit:cleanup=True 2023-09-23T23:43:34.908 DEBUG:teuthology.orchestra.run.smithi110:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T23:43:34.953 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:43:34.965 INFO:tasks.workunit.client.0.smithi110.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T23:43:34.992 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:35.202 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:43:35.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:35.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:43:35.214 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:43:35.247 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:43:35.266 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:43:35.285 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:43:35.306 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:35.309 INFO:teuthology.orchestra.run.smithi110.stdout:osd.4: {} 2023-09-23T23:43:35.330 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:43:35.335 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:35.350 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:43:35.373 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:43:35.415 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:43:35.416 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:43:35.416 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:43:35.416 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:43:35.417 INFO:teuthology.orchestra.run.smithi110.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:43:35.417 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:43:35.418 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:43:35.418 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:43:35.430 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:35.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:35.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:35.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:35 smithi110 ceph-mon[212939]: failed to encode map e690 with expected crc 2023-09-23T23:43:35.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:35 smithi110 ceph-mon[212939]: pgmap v1961: 135 pgs: 135 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 1.7 MiB/s rd, 719 KiB/s wr, 2.35k op/s 2023-09-23T23:43:35.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:35 smithi110 ceph-mon[212939]: failed to encode map e691 with expected crc 2023-09-23T23:43:35.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:35 smithi110 ceph-mon[212939]: osdmap e691: 8 total, 8 up, 5 in 2023-09-23T23:43:35.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:35 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1229678335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:35.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:35 smithi110 conmon[212914]: 2023-09-23T23:43:35.289+0000 7f29aa464700 -1 mon.a@0(leader).osd e691 definitely_dead 0 2023-09-23T23:43:35.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:35 smithi110 ceph-mon[221102]: failed to encode map e690 with expected crc 2023-09-23T23:43:35.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:35 smithi110 ceph-mon[221102]: pgmap v1961: 135 pgs: 135 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 1.7 MiB/s rd, 719 KiB/s wr, 2.35k op/s 2023-09-23T23:43:35.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:35 smithi110 ceph-mon[221102]: failed to encode map e691 with expected crc 2023-09-23T23:43:35.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:35 smithi110 ceph-mon[221102]: osdmap e691: 8 total, 8 up, 5 in 2023-09-23T23:43:35.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:35 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1229678335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:35.534 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:43:35.563 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:35 smithi138 ceph-mon[193606]: failed to encode map e690 with expected crc 2023-09-23T23:43:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:35 smithi138 ceph-mon[193606]: pgmap v1961: 135 pgs: 135 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 1.7 MiB/s rd, 719 KiB/s wr, 2.35k op/s 2023-09-23T23:43:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:35 smithi138 ceph-mon[193606]: failed to encode map e691 with expected crc 2023-09-23T23:43:35.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:35 smithi138 ceph-mon[193606]: osdmap e691: 8 total, 8 up, 5 in 2023-09-23T23:43:35.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:35 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1229678335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:36.217 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:43:36.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:36.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:43:36.300 INFO:teuthology.orchestra.run.smithi110.stderr:marked in osd.3. 2023-09-23T23:43:36.320 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:36.352 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:36.401 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:36.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:36.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:36.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:36 smithi110 ceph-mon[212939]: failed to encode map e691 with expected crc 2023-09-23T23:43:36.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:36 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3245538497' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T23:43:36.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:36 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T23:43:36.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:36 smithi110 ceph-mon[221102]: failed to encode map e691 with expected crc 2023-09-23T23:43:36.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:36 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3245538497' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T23:43:36.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:36 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T23:43:36.454 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:43:36.507 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:36.534 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:36.577 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:36.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:36 smithi138 ceph-mon[193606]: failed to encode map e691 with expected crc 2023-09-23T23:43:36.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:36 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3245538497' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T23:43:36.581 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:36 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T23:43:36.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:36.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:36.683 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:36.709 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:36.750 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:36.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:36.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:36.855 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:36.883 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:36.932 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:36.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:36.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:37.037 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:37.063 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:37.105 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:37.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:37.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:37.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:43:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:43:36] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.43.0" 2023-09-23T23:43:37.213 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:43:37.240 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:37.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:37 smithi138 ceph-mon[193606]: failed to encode map e691 with expected crc 2023-09-23T23:43:37.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:37 smithi138 ceph-mon[193606]: failed to encode map e691 with expected crc 2023-09-23T23:43:37.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:37 smithi138 ceph-mon[193606]: failed to encode map e691 with expected crc 2023-09-23T23:43:37.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:37 smithi138 ceph-mon[193606]: failed to encode map e691 with expected crc 2023-09-23T23:43:37.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:37 smithi138 ceph-mon[193606]: pgmap v1963: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail 2023-09-23T23:43:37.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:37 smithi138 ceph-mon[193606]: failed to encode map e692 with expected crc 2023-09-23T23:43:37.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:37 smithi138 ceph-mon[193606]: failed to encode map e692 with expected crc 2023-09-23T23:43:37.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:37 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T23:43:37.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:37 smithi138 ceph-mon[193606]: osdmap e692: 8 total, 8 up, 6 in 2023-09-23T23:43:37.495 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:43:37.497 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[212939]: failed to encode map e691 with expected crc 2023-09-23T23:43:37.497 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[212939]: failed to encode map e691 with expected crc 2023-09-23T23:43:37.497 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[212939]: failed to encode map e691 with expected crc 2023-09-23T23:43:37.497 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[212939]: failed to encode map e691 with expected crc 2023-09-23T23:43:37.497 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[212939]: pgmap v1963: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail 2023-09-23T23:43:37.498 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[212939]: failed to encode map e692 with expected crc 2023-09-23T23:43:37.498 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[212939]: failed to encode map e692 with expected crc 2023-09-23T23:43:37.498 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T23:43:37.498 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[212939]: osdmap e692: 8 total, 8 up, 6 in 2023-09-23T23:43:37.498 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[221102]: failed to encode map e691 with expected crc 2023-09-23T23:43:37.498 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[221102]: failed to encode map e691 with expected crc 2023-09-23T23:43:37.498 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[221102]: failed to encode map e691 with expected crc 2023-09-23T23:43:37.498 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[221102]: failed to encode map e691 with expected crc 2023-09-23T23:43:37.499 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[221102]: pgmap v1963: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail 2023-09-23T23:43:37.499 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[221102]: failed to encode map e692 with expected crc 2023-09-23T23:43:37.499 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[221102]: failed to encode map e692 with expected crc 2023-09-23T23:43:37.499 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T23:43:37.499 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:37 smithi110 ceph-mon[221102]: osdmap e692: 8 total, 8 up, 6 in 2023-09-23T23:43:37.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:37.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:43:37.601 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:37.614 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-09-23T23:43:37.631 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:37.729 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:37.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:37.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:37.834 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:37.835 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:37.869 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:37.938 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:43:37.939 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:43:37.972 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:37.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:37.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:38.076 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:38.105 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:38.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:38 smithi110 ceph-mon[212939]: failed to encode map e693 with expected crc 2023-09-23T23:43:38.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:38 smithi110 ceph-mon[212939]: osdmap e693: 8 total, 8 up, 6 in 2023-09-23T23:43:38.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:38 smithi110 ceph-mon[212939]: failed to encode map e693 with expected crc 2023-09-23T23:43:38.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:38 smithi110 ceph-mon[212939]: failed to encode map e693 with expected crc 2023-09-23T23:43:38.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:38 smithi110 ceph-mon[212939]: failed to encode map e693 with expected crc 2023-09-23T23:43:38.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:38 smithi110 ceph-mon[212939]: failed to encode map e693 with expected crc 2023-09-23T23:43:38.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:38 smithi110 ceph-mon[221102]: failed to encode map e693 with expected crc 2023-09-23T23:43:38.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:38 smithi110 ceph-mon[221102]: osdmap e693: 8 total, 8 up, 6 in 2023-09-23T23:43:38.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:38 smithi110 ceph-mon[221102]: failed to encode map e693 with expected crc 2023-09-23T23:43:38.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:38 smithi110 ceph-mon[221102]: failed to encode map e693 with expected crc 2023-09-23T23:43:38.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:38 smithi110 ceph-mon[221102]: failed to encode map e693 with expected crc 2023-09-23T23:43:38.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:38 smithi110 ceph-mon[221102]: failed to encode map e693 with expected crc 2023-09-23T23:43:38.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:38 smithi138 ceph-mon[193606]: failed to encode map e693 with expected crc 2023-09-23T23:43:38.694 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:38 smithi138 ceph-mon[193606]: osdmap e693: 8 total, 8 up, 6 in 2023-09-23T23:43:38.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:38 smithi138 ceph-mon[193606]: failed to encode map e693 with expected crc 2023-09-23T23:43:38.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:38 smithi138 ceph-mon[193606]: failed to encode map e693 with expected crc 2023-09-23T23:43:38.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:38 smithi138 ceph-mon[193606]: failed to encode map e693 with expected crc 2023-09-23T23:43:38.696 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:38 smithi138 ceph-mon[193606]: failed to encode map e693 with expected crc 2023-09-23T23:43:38.997 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:43:39.005 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:39.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:39.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:39.109 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:39.142 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:39.192 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:39.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:39.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:39.303 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:39.320 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:43:39.331 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:39.375 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:39.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:39.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:39.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:39 smithi138 ceph-mon[193606]: pgmap v1966: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail 2023-09-23T23:43:39.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:39 smithi138 ceph-mon[193606]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:39 smithi138 ceph-mon[193606]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:39 smithi138 ceph-mon[193606]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:39 smithi138 ceph-mon[193606]: osdmap e694: 8 total, 8 up, 6 in 2023-09-23T23:43:39.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:39 smithi138 ceph-mon[193606]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:39 smithi138 ceph-mon[193606]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:39 smithi138 ceph-mon[193606]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.445 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:39 smithi138 ceph-mon[193606]: failed to encode map e695 with expected crc 2023-09-23T23:43:39.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:39 smithi138 ceph-mon[193606]: osdmap e695: 8 total, 8 up, 6 in 2023-09-23T23:43:39.446 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:39 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:39.480 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:39.507 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:39.550 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:39.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:39.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:39.658 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:39.684 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[212939]: pgmap v1966: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail 2023-09-23T23:43:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[212939]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[212939]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[212939]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[212939]: osdmap e694: 8 total, 8 up, 6 in 2023-09-23T23:43:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[212939]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[212939]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[212939]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[212939]: failed to encode map e695 with expected crc 2023-09-23T23:43:39.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[212939]: osdmap e695: 8 total, 8 up, 6 in 2023-09-23T23:43:39.704 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[221102]: pgmap v1966: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail 2023-09-23T23:43:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[221102]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[221102]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[221102]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[221102]: osdmap e694: 8 total, 8 up, 6 in 2023-09-23T23:43:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[221102]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[221102]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[221102]: failed to encode map e694 with expected crc 2023-09-23T23:43:39.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[221102]: failed to encode map e695 with expected crc 2023-09-23T23:43:39.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[221102]: osdmap e695: 8 total, 8 up, 6 in 2023-09-23T23:43:39.705 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:39 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:39.726 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:39.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:39.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:39.833 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:39.841 DEBUG:teuthology.orchestra.run.smithi110:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T23:43:39.858 INFO:tasks.workunit.client.0.smithi110.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T23:43:39.859 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:39.860 INFO:tasks.workunit.client.0.smithi110.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T23:43:39.860 INFO:tasks.workunit.client.0.smithi110.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T23:43:39.901 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:39.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:39.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:40.007 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:43:40.034 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:40.076 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:43:40.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:40.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:43:40.181 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:43:40.208 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:40.249 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:43:40.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:40.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:43:40.356 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:40.383 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:40.424 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:40.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:40.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:40.532 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:40 smithi138 ceph-mon[193606]: failed to encode map e695 with expected crc 2023-09-23T23:43:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:40 smithi138 ceph-mon[193606]: failed to encode map e695 with expected crc 2023-09-23T23:43:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:40 smithi138 ceph-mon[193606]: failed to encode map e695 with expected crc 2023-09-23T23:43:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:40 smithi138 ceph-mon[193606]: failed to encode map e696 with expected crc 2023-09-23T23:43:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:40 smithi138 ceph-mon[193606]: osdmap e696: 8 total, 8 up, 6 in 2023-09-23T23:43:40.559 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:40.582 INFO:tasks.workunit.client.0.smithi110.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T23:43:40.601 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:40.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:40.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:40.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:40 smithi110 ceph-mon[212939]: failed to encode map e695 with expected crc 2023-09-23T23:43:40.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:40 smithi110 ceph-mon[212939]: failed to encode map e695 with expected crc 2023-09-23T23:43:40.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:40 smithi110 ceph-mon[212939]: failed to encode map e695 with expected crc 2023-09-23T23:43:40.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:40 smithi110 ceph-mon[212939]: failed to encode map e696 with expected crc 2023-09-23T23:43:40.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:40 smithi110 ceph-mon[212939]: osdmap e696: 8 total, 8 up, 6 in 2023-09-23T23:43:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:40 smithi110 ceph-mon[221102]: failed to encode map e695 with expected crc 2023-09-23T23:43:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:40 smithi110 ceph-mon[221102]: failed to encode map e695 with expected crc 2023-09-23T23:43:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:40 smithi110 ceph-mon[221102]: failed to encode map e695 with expected crc 2023-09-23T23:43:40.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:40 smithi110 ceph-mon[221102]: failed to encode map e696 with expected crc 2023-09-23T23:43:40.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:40 smithi110 ceph-mon[221102]: osdmap e696: 8 total, 8 up, 6 in 2023-09-23T23:43:40.709 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:43:40.743 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:40.833 INFO:tasks.workunit.client.0.smithi110.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T23:43:40.932 INFO:tasks.workunit.client.0.smithi110.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T23:43:40.934 INFO:tasks.workunit.client.0.smithi110.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T23:43:40.934 INFO:tasks.workunit.client.0.smithi110.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T23:43:40.976 INFO:tasks.workunit.client.0.smithi110.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T23:43:40.979 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-09-23T23:43:40.979 DEBUG:teuthology.orchestra.run.smithi110:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T23:43:41.037 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-23T23:43:41.039 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-23T23:43:41.039 DEBUG:teuthology.orchestra.run.smithi110:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-23T23:43:41.099 INFO:tasks.workunit.client.0.smithi110.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-23T23:43:41.099 INFO:tasks.workunit.client.0.smithi110.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-09-23T23:43:41.099 INFO:tasks.workunit.client.0.smithi110.stderr:+ TMPDIR=/tmp/rbd_import_export_695376 2023-09-23T23:43:41.099 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm -rf /tmp/rbd_import_export_695376 2023-09-23T23:43:41.100 INFO:tasks.workunit.client.0.smithi110.stderr:+ mkdir /tmp/rbd_import_export_695376 2023-09-23T23:43:41.100 INFO:tasks.workunit.client.0.smithi110.stderr:+ trap 'rm -rf /tmp/rbd_import_export_695376' INT TERM EXIT 2023-09-23T23:43:41.100 INFO:tasks.workunit.client.0.smithi110.stderr:+ mkdir foo.695376 2023-09-23T23:43:41.101 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import foo.695376 foo.dir 2023-09-23T23:43:41.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:41 smithi138 ceph-mon[193606]: failed to encode map e696 with expected crc 2023-09-23T23:43:41.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:41 smithi138 ceph-mon[193606]: failed to encode map e696 with expected crc 2023-09-23T23:43:41.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:41 smithi138 ceph-mon[193606]: failed to encode map e696 with expected crc 2023-09-23T23:43:41.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:41 smithi138 ceph-mon[193606]: failed to encode map e696 with expected crc 2023-09-23T23:43:41.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:41 smithi138 ceph-mon[193606]: pgmap v1970: 103 pgs: 2 peering, 1 active+undersized+remapped, 2 active+remapped, 98 active+clean; 595 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 5.5 KiB/s rd, 14 KiB/s wr, 4 op/s; 10 KiB/s, 0 objects/s recovering 2023-09-23T23:43:41.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:41 smithi138 ceph-mon[193606]: 1.c scrub starts 2023-09-23T23:43:41.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:41 smithi138 ceph-mon[193606]: 1.c scrub ok 2023-09-23T23:43:41.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:41 smithi110 ceph-mon[212939]: failed to encode map e696 with expected crc 2023-09-23T23:43:41.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:41 smithi110 ceph-mon[212939]: failed to encode map e696 with expected crc 2023-09-23T23:43:41.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:41 smithi110 ceph-mon[212939]: failed to encode map e696 with expected crc 2023-09-23T23:43:41.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:41 smithi110 ceph-mon[212939]: failed to encode map e696 with expected crc 2023-09-23T23:43:41.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:41 smithi110 ceph-mon[212939]: pgmap v1970: 103 pgs: 2 peering, 1 active+undersized+remapped, 2 active+remapped, 98 active+clean; 595 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 5.5 KiB/s rd, 14 KiB/s wr, 4 op/s; 10 KiB/s, 0 objects/s recovering 2023-09-23T23:43:41.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:41 smithi110 ceph-mon[212939]: 1.c scrub starts 2023-09-23T23:43:41.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:41 smithi110 ceph-mon[212939]: 1.c scrub ok 2023-09-23T23:43:41.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:41 smithi110 ceph-mon[221102]: failed to encode map e696 with expected crc 2023-09-23T23:43:41.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:41 smithi110 ceph-mon[221102]: failed to encode map e696 with expected crc 2023-09-23T23:43:41.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:41 smithi110 ceph-mon[221102]: failed to encode map e696 with expected crc 2023-09-23T23:43:41.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:41 smithi110 ceph-mon[221102]: failed to encode map e696 with expected crc 2023-09-23T23:43:41.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:41 smithi110 ceph-mon[221102]: pgmap v1970: 103 pgs: 2 peering, 1 active+undersized+remapped, 2 active+remapped, 98 active+clean; 595 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 5.5 KiB/s rd, 14 KiB/s wr, 4 op/s; 10 KiB/s, 0 objects/s recovering 2023-09-23T23:43:41.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:41 smithi110 ceph-mon[221102]: 1.c scrub starts 2023-09-23T23:43:41.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:41 smithi110 ceph-mon[221102]: 1.c scrub ok 2023-09-23T23:43:41.808 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:43:41.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:41.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:43:41.865 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: cannot import a directory 2023-09-23T23:43:41.865 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 0% complete...failed. 2023-09-23T23:43:41.866 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: import failed: (21) Is a directory 2023-09-23T23:43:41.868 INFO:tasks.workunit.client.0.smithi110.stderr:+ true 2023-09-23T23:43:41.868 INFO:tasks.workunit.client.0.smithi110.stderr:+ rmdir foo.695376 2023-09-23T23:43:41.869 INFO:tasks.workunit.client.0.smithi110.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_695376/img bs=1k count=1 seek=10 2023-09-23T23:43:41.870 INFO:tasks.workunit.client.0.smithi110.stderr:1+0 records in 2023-09-23T23:43:41.870 INFO:tasks.workunit.client.0.smithi110.stderr:1+0 records out 2023-09-23T23:43:41.870 INFO:tasks.workunit.client.0.smithi110.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 5.6641e-05 s, 18.1 MB/s 2023-09-23T23:43:41.870 INFO:tasks.workunit.client.0.smithi110.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_695376/img bs=1k count=10 seek=100 2023-09-23T23:43:41.871 INFO:tasks.workunit.client.0.smithi110.stderr:10+0 records in 2023-09-23T23:43:41.871 INFO:tasks.workunit.client.0.smithi110.stderr:10+0 records out 2023-09-23T23:43:41.871 INFO:tasks.workunit.client.0.smithi110.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.3006e-05 s, 163 MB/s 2023-09-23T23:43:41.871 INFO:tasks.workunit.client.0.smithi110.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_695376/img bs=1k count=100 seek=1000 2023-09-23T23:43:41.872 INFO:tasks.workunit.client.0.smithi110.stderr:70+1 records in 2023-09-23T23:43:41.872 INFO:tasks.workunit.client.0.smithi110.stderr:70+1 records out 2023-09-23T23:43:41.872 INFO:tasks.workunit.client.0.smithi110.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000211246 s, 341 MB/s 2023-09-23T23:43:41.873 INFO:tasks.workunit.client.0.smithi110.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_695376/img bs=1k seek=10000 2023-09-23T23:43:41.873 INFO:tasks.workunit.client.0.smithi110.stderr:139+1 records in 2023-09-23T23:43:41.873 INFO:tasks.workunit.client.0.smithi110.stderr:139+1 records out 2023-09-23T23:43:41.874 INFO:tasks.workunit.client.0.smithi110.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000352103 s, 407 MB/s 2023-09-23T23:43:41.874 INFO:tasks.workunit.client.0.smithi110.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_695376/img bs=1k seek=100000 2023-09-23T23:43:41.894 INFO:tasks.workunit.client.0.smithi110.stderr:70+1 records in 2023-09-23T23:43:41.894 INFO:tasks.workunit.client.0.smithi110.stderr:70+1 records out 2023-09-23T23:43:41.894 INFO:tasks.workunit.client.0.smithi110.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.0191214 s, 3.8 MB/s 2023-09-23T23:43:41.894 INFO:tasks.workunit.client.0.smithi110.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_695376/img bs=1k seek=1000000 2023-09-23T23:43:41.895 INFO:tasks.workunit.client.0.smithi110.stderr:189+1 records in 2023-09-23T23:43:41.895 INFO:tasks.workunit.client.0.smithi110.stderr:189+1 records out 2023-09-23T23:43:41.895 INFO:tasks.workunit.client.0.smithi110.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000413844 s, 468 MB/s 2023-09-23T23:43:41.895 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd rm testimg 2023-09-23T23:43:41.912 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:41.939 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 0% complete...failed. 2023-09-23T23:43:41.939 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T23:43:41.944 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:41.945 INFO:tasks.workunit.client.0.smithi110.stderr:+ true 2023-09-23T23:43:41.945 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --new-format /tmp/rbd_import_export_695376/img testimg 2023-09-23T23:43:41.998 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:42.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:42.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:42.034 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:42.082 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:42.102 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:42.132 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:42.220 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:42.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:42.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:42.325 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:42.352 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:42.449 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:42.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:42.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:42.553 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:43:42.582 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:42.615 INFO:tasks.thrashosds.thrasher:in_osds: [0, 5, 7, 2, 1, 3] out_osds: [4, 6] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:43:42.615 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:43:42.615 INFO:tasks.thrashosds.thrasher:Killing osd 4, live_osds are [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T23:43:42.615 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-09-23T23:43:42.615 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.4 2023-09-23T23:43:42.896 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:43:42 smithi138 systemd[1]: Stopping Ceph osd.4 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:43:43.192 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:43:42 smithi138 conmon[112407]: 2023-09-23T23:43:42.893+0000 7f933c4b9700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:43:43.192 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:43:42 smithi138 conmon[112407]: 2023-09-23T23:43:42.893+0000 7f933c4b9700 -1 osd.4 697 *** Got signal Terminated *** 2023-09-23T23:43:43.192 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:43:42 smithi138 conmon[112407]: 2023-09-23T23:43:42.893+0000 7f933c4b9700 -1 osd.4 697 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:43:43.325 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is set 2023-09-23T23:43:43.531 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:43:43.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:43.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:43:43.637 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:43.661 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[212939]: pgmap v1971: 103 pgs: 2 peering, 1 active+undersized+remapped, 2 active+remapped, 98 active+clean; 595 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 4.3 KiB/s rd, 11 KiB/s wr, 3 op/s; 8.1 KiB/s, 1 objects/s recovering 2023-09-23T23:43:43.661 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[212939]: 1.e scrub starts 2023-09-23T23:43:43.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[212939]: 1.e scrub ok 2023-09-23T23:43:43.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[212939]: failed to encode map e697 with expected crc 2023-09-23T23:43:43.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[212939]: osdmap e697: 8 total, 8 up, 6 in 2023-09-23T23:43:43.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[212939]: failed to encode map e697 with expected crc 2023-09-23T23:43:43.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[212939]: osd.4 marked itself down and dead 2023-09-23T23:43:43.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:43.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:43:43.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3219867681' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:43:43.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:43:43.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[221102]: pgmap v1971: 103 pgs: 2 peering, 1 active+undersized+remapped, 2 active+remapped, 98 active+clean; 595 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 4.3 KiB/s rd, 11 KiB/s wr, 3 op/s; 8.1 KiB/s, 1 objects/s recovering 2023-09-23T23:43:43.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[221102]: 1.e scrub starts 2023-09-23T23:43:43.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[221102]: 1.e scrub ok 2023-09-23T23:43:43.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[221102]: failed to encode map e697 with expected crc 2023-09-23T23:43:43.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[221102]: osdmap e697: 8 total, 8 up, 6 in 2023-09-23T23:43:43.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[221102]: failed to encode map e697 with expected crc 2023-09-23T23:43:43.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[221102]: osd.4 marked itself down and dead 2023-09-23T23:43:43.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:43.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:43:43.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3219867681' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:43:43.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:43 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:43:43.677 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:43.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:43 smithi138 ceph-mon[193606]: pgmap v1971: 103 pgs: 2 peering, 1 active+undersized+remapped, 2 active+remapped, 98 active+clean; 595 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 4.3 KiB/s rd, 11 KiB/s wr, 3 op/s; 8.1 KiB/s, 1 objects/s recovering 2023-09-23T23:43:43.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:43 smithi138 ceph-mon[193606]: 1.e scrub starts 2023-09-23T23:43:43.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:43 smithi138 ceph-mon[193606]: 1.e scrub ok 2023-09-23T23:43:43.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:43 smithi138 ceph-mon[193606]: failed to encode map e697 with expected crc 2023-09-23T23:43:43.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:43 smithi138 ceph-mon[193606]: osdmap e697: 8 total, 8 up, 6 in 2023-09-23T23:43:43.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:43 smithi138 ceph-mon[193606]: failed to encode map e697 with expected crc 2023-09-23T23:43:43.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:43 smithi138 ceph-mon[193606]: osd.4 marked itself down and dead 2023-09-23T23:43:43.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:43 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:43.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:43:43.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:43 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3219867681' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:43:43.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:43 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:43:43.938 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-23T23:43:43.938 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:43:43.941 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd export testimg /tmp/rbd_import_export_695376/img2 2023-09-23T23:43:44.262 INFO:tasks.workunit.client.0.smithi110.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T23:43:44.266 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd export testimg - 2023-09-23T23:43:44.442 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:43:44 smithi138 bash[415755]: 47f7252bf067f686d1f0abe36f9eb29cdc4d9de14cac9da602acb57c7375929d 2023-09-23T23:43:44.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:44 smithi138 ceph-mon[193606]: 1.16 scrub starts 2023-09-23T23:43:44.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:44 smithi138 ceph-mon[193606]: 1.16 scrub ok 2023-09-23T23:43:44.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:44 smithi138 ceph-mon[193606]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:43:44.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:44 smithi138 ceph-mon[193606]: failed to encode map e698 with expected crc 2023-09-23T23:43:44.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:44 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:43:44.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:44 smithi138 ceph-mon[193606]: osdmap e698: 8 total, 7 up, 6 in 2023-09-23T23:43:44.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:44 smithi138 ceph-mon[193606]: failed to encode map e698 with expected crc 2023-09-23T23:43:44.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:44 smithi110 ceph-mon[212939]: 1.16 scrub starts 2023-09-23T23:43:44.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:44 smithi110 ceph-mon[212939]: 1.16 scrub ok 2023-09-23T23:43:44.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:44 smithi110 ceph-mon[212939]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:43:44.616 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:44 smithi110 ceph-mon[212939]: failed to encode map e698 with expected crc 2023-09-23T23:43:44.617 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:44 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:43:44.617 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:44 smithi110 ceph-mon[212939]: osdmap e698: 8 total, 7 up, 6 in 2023-09-23T23:43:44.617 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:44 smithi110 ceph-mon[212939]: failed to encode map e698 with expected crc 2023-09-23T23:43:44.617 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:44 smithi110 ceph-mon[221102]: 1.16 scrub starts 2023-09-23T23:43:44.617 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:44 smithi110 ceph-mon[221102]: 1.16 scrub ok 2023-09-23T23:43:44.617 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:44 smithi110 ceph-mon[221102]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:43:44.618 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:44 smithi110 ceph-mon[221102]: failed to encode map e698 with expected crc 2023-09-23T23:43:44.618 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:44 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:43:44.618 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:44 smithi110 ceph-mon[221102]: osdmap e698: 8 total, 7 up, 6 in 2023-09-23T23:43:44.618 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:44 smithi110 ceph-mon[221102]: failed to encode map e698 with expected crc 2023-09-23T23:43:44.622 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:44.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:44.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:44.727 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:44.765 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:45.140 INFO:tasks.workunit.client.0.smithi110.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T23:43:45.144 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd rm testimg 2023-09-23T23:43:45.277 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T23:43:45.280 INFO:tasks.workunit.client.0.smithi110.stderr:+ cmp /tmp/rbd_import_export_695376/img /tmp/rbd_import_export_695376/img2 2023-09-23T23:43:45.477 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:45.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:45.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:45.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:45.609 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:45 smithi110 ceph-mon[212939]: pgmap v1974: 103 pgs: 103 active+clean; 1.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 6.9 KiB/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T23:43:45.609 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:45 smithi110 ceph-mon[221102]: pgmap v1974: 103 pgs: 103 active+clean; 1.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 6.9 KiB/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T23:43:45.611 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:45.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:45 smithi138 ceph-mon[193606]: pgmap v1974: 103 pgs: 103 active+clean; 1.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 6.9 KiB/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T23:43:46.014 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:46.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:46.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:46.082 INFO:tasks.workunit.client.0.smithi110.stderr:+ cmp /tmp/rbd_import_export_695376/img /tmp/rbd_import_export_695376/img3 2023-09-23T23:43:46.119 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:46.153 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:46.201 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:46.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:46.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:46.304 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:46.334 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:46.377 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:46.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:46.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:46.481 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:46.509 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:46.537 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm /tmp/rbd_import_export_695376/img2 /tmp/rbd_import_export_695376/img3 2023-09-23T23:43:46.638 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:43:46.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:46 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [0, 2]}]: dispatch 2023-09-23T23:43:46.662 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:46 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]: dispatch 2023-09-23T23:43:46.663 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:46 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [0, 2]}]: dispatch 2023-09-23T23:43:46.663 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:46 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]: dispatch 2023-09-23T23:43:46.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:46 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [0, 2]}]: dispatch 2023-09-23T23:43:46.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:46 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]: dispatch 2023-09-23T23:43:46.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:46 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [0, 2]}]: dispatch 2023-09-23T23:43:46.663 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:46 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]: dispatch 2023-09-23T23:43:46.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:46 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [0, 2]}]: dispatch 2023-09-23T23:43:46.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:46 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]: dispatch 2023-09-23T23:43:46.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:46 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [0, 2]}]: dispatch 2023-09-23T23:43:46.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:46 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]: dispatch 2023-09-23T23:43:46.746 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.4.service' 2023-09-23T23:43:46.787 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:43:46.787 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-09-23T23:43:46.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:43:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:43:46] "GET /metrics HTTP/1.1" 200 34125 "" "Prometheus/2.43.0" 2023-09-23T23:43:47.160 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --new-format - testimg 2023-09-23T23:43:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:47 smithi138 ceph-mon[193606]: pgmap v1975: 103 pgs: 103 active+clean; 1.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 50 KiB/s rd, 31 KiB/s wr, 15 op/s 2023-09-23T23:43:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:47 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [0, 2]}]': finished 2023-09-23T23:43:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:47 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]': finished 2023-09-23T23:43:47.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:47 smithi138 ceph-mon[193606]: osdmap e699: 8 total, 7 up, 6 in 2023-09-23T23:43:47.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:47 smithi138 ceph-mon[193606]: failed to encode map e699 with expected crc 2023-09-23T23:43:47.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:47 smithi138 ceph-mon[193606]: failed to encode map e699 with expected crc 2023-09-23T23:43:47.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:47 smithi138 ceph-mon[193606]: failed to encode map e699 with expected crc 2023-09-23T23:43:47.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:47 smithi110 ceph-mon[212939]: pgmap v1975: 103 pgs: 103 active+clean; 1.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 50 KiB/s rd, 31 KiB/s wr, 15 op/s 2023-09-23T23:43:47.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [0, 2]}]': finished 2023-09-23T23:43:47.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]': finished 2023-09-23T23:43:47.694 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:47 smithi110 ceph-mon[212939]: osdmap e699: 8 total, 7 up, 6 in 2023-09-23T23:43:47.695 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:47 smithi110 ceph-mon[212939]: failed to encode map e699 with expected crc 2023-09-23T23:43:47.695 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:47 smithi110 ceph-mon[212939]: failed to encode map e699 with expected crc 2023-09-23T23:43:47.695 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:47 smithi110 ceph-mon[212939]: failed to encode map e699 with expected crc 2023-09-23T23:43:47.695 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:47 smithi110 ceph-mon[221102]: pgmap v1975: 103 pgs: 103 active+clean; 1.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 50 KiB/s rd, 31 KiB/s wr, 15 op/s 2023-09-23T23:43:47.695 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:47 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [0, 2]}]': finished 2023-09-23T23:43:47.695 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:47 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]': finished 2023-09-23T23:43:47.695 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:47 smithi110 ceph-mon[221102]: osdmap e699: 8 total, 7 up, 6 in 2023-09-23T23:43:47.695 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:47 smithi110 ceph-mon[221102]: failed to encode map e699 with expected crc 2023-09-23T23:43:47.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:47 smithi110 ceph-mon[221102]: failed to encode map e699 with expected crc 2023-09-23T23:43:47.696 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:47 smithi110 ceph-mon[221102]: failed to encode map e699 with expected crc 2023-09-23T23:43:47.707 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:47.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:47.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:47.813 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:47.844 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:47.948 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:47.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:47.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:47.989 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:43:48.005 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:48.014 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:43:48.041 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:43:48.056 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:48.065 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:43:48.086 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:43:48.090 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:48.106 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:43:48.127 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:43:48.127 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:43:48.127 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:43:48.127 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:43:48.128 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:43:48.128 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-09-23T23:43:48.128 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:43:48.128 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:43:48.128 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:43:48.138 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 100% complete...done. 2023-09-23T23:43:48.138 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:43:48.141 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd export testimg /tmp/rbd_import_export_695376/img2 2023-09-23T23:43:48.143 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:48.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:48.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:48.251 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:48.278 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:48.322 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:48.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:48.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:48.426 INFO:tasks.workunit.client.0.smithi110.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T23:43:48.429 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:48.432 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd export testimg - 2023-09-23T23:43:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:48 smithi138 ceph-mon[193606]: failed to encode map e699 with expected crc 2023-09-23T23:43:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:48 smithi138 ceph-mon[193606]: failed to encode map e700 with expected crc 2023-09-23T23:43:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:48 smithi138 ceph-mon[193606]: failed to encode map e700 with expected crc 2023-09-23T23:43:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:48 smithi138 ceph-mon[193606]: osdmap e700: 8 total, 7 up, 6 in 2023-09-23T23:43:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:48 smithi138 ceph-mon[193606]: failed to encode map e700 with expected crc 2023-09-23T23:43:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:48 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2841447621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:48.454 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:48.497 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:48.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:48.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:48.513 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:48 smithi110 ceph-mon[212939]: failed to encode map e699 with expected crc 2023-09-23T23:43:48.513 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:48 smithi110 ceph-mon[212939]: failed to encode map e700 with expected crc 2023-09-23T23:43:48.513 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:48 smithi110 ceph-mon[212939]: failed to encode map e700 with expected crc 2023-09-23T23:43:48.513 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:48 smithi110 ceph-mon[212939]: osdmap e700: 8 total, 7 up, 6 in 2023-09-23T23:43:48.513 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:48 smithi110 ceph-mon[212939]: failed to encode map e700 with expected crc 2023-09-23T23:43:48.513 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:48 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2841447621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:48.514 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:48 smithi110 ceph-mon[221102]: failed to encode map e699 with expected crc 2023-09-23T23:43:48.514 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:48 smithi110 ceph-mon[221102]: failed to encode map e700 with expected crc 2023-09-23T23:43:48.514 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:48 smithi110 ceph-mon[221102]: failed to encode map e700 with expected crc 2023-09-23T23:43:48.514 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:48 smithi110 ceph-mon[221102]: osdmap e700: 8 total, 7 up, 6 in 2023-09-23T23:43:48.514 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:48 smithi110 ceph-mon[221102]: failed to encode map e700 with expected crc 2023-09-23T23:43:48.514 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:48 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2841447621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:48.551 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:43:48.551 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:43:48.551 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:43:48.552 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:43:48.552 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:43:48.604 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:48.630 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:48.674 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:48.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:48.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:48.779 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:48.806 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:48.852 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:48.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:48.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:48.958 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:48.987 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:49.034 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:49.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:49.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:49.141 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:49.172 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:49.233 INFO:tasks.workunit.client.0.smithi110.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T23:43:49.236 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd rm testimg 2023-09-23T23:43:49.319 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T23:43:49.323 INFO:tasks.workunit.client.0.smithi110.stderr:+ cmp /tmp/rbd_import_export_695376/img /tmp/rbd_import_export_695376/img2 2023-09-23T23:43:49.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:49 smithi110 ceph-mon[212939]: pgmap v1978: 103 pgs: 103 active+clean; 2.7 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 168 KiB/s rd, 93 KiB/s wr, 47 op/s 2023-09-23T23:43:49.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:49 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:43:49.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:49 smithi110 ceph-mon[212939]: osdmap e701: 8 total, 7 up, 6 in 2023-09-23T23:43:49.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:49 smithi110 ceph-mon[212939]: failed to encode map e701 with expected crc 2023-09-23T23:43:49.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:49 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:49 smithi110 ceph-mon[221102]: pgmap v1978: 103 pgs: 103 active+clean; 2.7 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 168 KiB/s rd, 93 KiB/s wr, 47 op/s 2023-09-23T23:43:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:49 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:43:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:49 smithi110 ceph-mon[221102]: osdmap e701: 8 total, 7 up, 6 in 2023-09-23T23:43:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:49 smithi110 ceph-mon[221102]: failed to encode map e701 with expected crc 2023-09-23T23:43:49.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:49 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:49.584 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:43:49.585 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:43:49.638 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:49.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:49.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:49.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:49 smithi138 ceph-mon[193606]: pgmap v1978: 103 pgs: 103 active+clean; 2.7 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 168 KiB/s rd, 93 KiB/s wr, 47 op/s 2023-09-23T23:43:49.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:49 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:43:49.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:49 smithi138 ceph-mon[193606]: osdmap e701: 8 total, 7 up, 6 in 2023-09-23T23:43:49.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:49 smithi138 ceph-mon[193606]: failed to encode map e701 with expected crc 2023-09-23T23:43:49.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:49 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:49.742 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:49.774 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:49.824 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:49.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:49.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:49.928 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:49.958 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:49.987 INFO:tasks.workunit.client.0.smithi110.stderr:+ cmp /tmp/rbd_import_export_695376/img /tmp/rbd_import_export_695376/img3 2023-09-23T23:43:50.005 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:50.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:50.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:50.109 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:50.138 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:50.184 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:50.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:50.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:50.293 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:50.320 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:50.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:50 smithi138 ceph-mon[193606]: failed to encode map e701 with expected crc 2023-09-23T23:43:50.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:50 smithi138 ceph-mon[193606]: failed to encode map e701 with expected crc 2023-09-23T23:43:50.448 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm /tmp/rbd_import_export_695376/img /tmp/rbd_import_export_695376/img2 /tmp/rbd_import_export_695376/img3 2023-09-23T23:43:50.700 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:50.701 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:50 smithi110 ceph-mon[212939]: failed to encode map e701 with expected crc 2023-09-23T23:43:50.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:50 smithi110 ceph-mon[212939]: failed to encode map e701 with expected crc 2023-09-23T23:43:50.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:50 smithi110 ceph-mon[221102]: failed to encode map e701 with expected crc 2023-09-23T23:43:50.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:50 smithi110 ceph-mon[221102]: failed to encode map e701 with expected crc 2023-09-23T23:43:50.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:50.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:50.804 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:50.830 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:50.872 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:50.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:50.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:50.975 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:51.002 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:51.044 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:51.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:51.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:51.147 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:51.176 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:51.273 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:51.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:51.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:51.384 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:51.417 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:51 smithi138 ceph-mon[193606]: pgmap v1980: 103 pgs: 103 active+clean; 3.5 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 253 KiB/s rd, 111 KiB/s wr, 65 op/s 2023-09-23T23:43:51.466 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:51.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:51.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:51.570 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:51.586 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:43:51.594 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:51 smithi110 ceph-mon[212939]: pgmap v1980: 103 pgs: 103 active+clean; 3.5 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 253 KiB/s rd, 111 KiB/s wr, 65 op/s 2023-09-23T23:43:51.595 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:51 smithi110 ceph-mon[221102]: pgmap v1980: 103 pgs: 103 active+clean; 3.5 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 253 KiB/s rd, 111 KiB/s wr, 65 op/s 2023-09-23T23:43:51.597 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:51.643 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:51.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:51.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:51.747 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:51.773 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:51.788 INFO:tasks.thrashosds.thrasher:in_osds: [0, 5, 7, 2, 1, 3] out_osds: [4, 6] dead_osds: [4] live_osds: [0, 1, 3, 5, 6, 2, 7] 2023-09-23T23:43:51.788 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:43:51.788 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T23:43:51.789 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.355864 2023-09-23T23:43:51.789 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 0.3558638796668582 2023-09-23T23:43:51.821 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:51.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:51.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:51.925 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:51.951 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:51.994 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:51.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:51.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:52.098 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:43:52.123 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:52.167 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:43:52.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:52.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:43:52.270 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:52.274 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd help export 2023-09-23T23:43:52.275 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep -q export-format 2023-09-23T23:43:52.295 INFO:tasks.workunit.client.0.smithi110.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_695376/img bs=1k count=10 seek=100 2023-09-23T23:43:52.297 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:52.334 INFO:tasks.workunit.client.0.smithi110.stderr:10+0 records in 2023-09-23T23:43:52.334 INFO:tasks.workunit.client.0.smithi110.stderr:10+0 records out 2023-09-23T23:43:52.335 INFO:tasks.workunit.client.0.smithi110.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.7899e-05 s, 151 MB/s 2023-09-23T23:43:52.335 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --new-format /tmp/rbd_import_export_695376/img testimg 2023-09-23T23:43:52.339 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:52.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:52.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:52.385 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 100% complete...done. 2023-09-23T23:43:52.385 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:43:52.387 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd snap create testimg@snap 2023-09-23T23:43:52.443 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:52.470 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:52.514 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:52.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:52.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:52.619 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:52.645 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:52.686 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:52.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:52.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:52.722 INFO:tasks.workunit.client.0.smithi110.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T23:43:52.727 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd image-meta set testimg key1 value1 2023-09-23T23:43:52.776 INFO:tasks.workunit.client.0.smithi110.stderr:++ rbd image-meta list testimg 2023-09-23T23:43:52.792 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:52.816 INFO:tasks.workunit.client.0.smithi110.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-09-23T23:43:52.817 INFO:tasks.workunit.client.0.smithi110.stderr: 2023-09-23T23:43:52.817 INFO:tasks.workunit.client.0.smithi110.stderr:Key Value 2023-09-23T23:43:52.817 INFO:tasks.workunit.client.0.smithi110.stderr:key1 value1' 2023-09-23T23:43:52.817 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_695376/img_v2 2023-09-23T23:43:52.823 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:52.863 INFO:tasks.workunit.client.0.smithi110.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T23:43:52.866 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_695376/img_v2 testimg_import 2023-09-23T23:43:52.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:52 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:52.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:52 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:52.971 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:52 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:52.971 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:52 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:52.971 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:52 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:52.972 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:52 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:52.978 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:52.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:52.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:53.082 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:53.109 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:53.151 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:53.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:53.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:53.257 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:53.285 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:53.326 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:53.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:53.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:53.434 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:53.460 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:53.732 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T23:43:53.735 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd info testimg_import 2023-09-23T23:43:53.770 INFO:tasks.workunit.client.0.smithi110.stdout:rbd image 'testimg_import': 2023-09-23T23:43:53.771 INFO:tasks.workunit.client.0.smithi110.stdout: size 110 KiB in 1 objects 2023-09-23T23:43:53.771 INFO:tasks.workunit.client.0.smithi110.stdout: order 22 (4 MiB objects) 2023-09-23T23:43:53.771 INFO:tasks.workunit.client.0.smithi110.stdout: snapshot_count: 1 2023-09-23T23:43:53.771 INFO:tasks.workunit.client.0.smithi110.stdout: id: b9bd868198c1 2023-09-23T23:43:53.771 INFO:tasks.workunit.client.0.smithi110.stdout: block_name_prefix: rbd_data.b9bd868198c1 2023-09-23T23:43:53.771 INFO:tasks.workunit.client.0.smithi110.stdout: format: 2 2023-09-23T23:43:53.771 INFO:tasks.workunit.client.0.smithi110.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T23:43:53.771 INFO:tasks.workunit.client.0.smithi110.stdout: op_features: 2023-09-23T23:43:53.771 INFO:tasks.workunit.client.0.smithi110.stdout: flags: 2023-09-23T23:43:53.772 INFO:tasks.workunit.client.0.smithi110.stdout: create_timestamp: Sat Sep 23 23:43:52 2023 2023-09-23T23:43:53.772 INFO:tasks.workunit.client.0.smithi110.stdout: access_timestamp: Sat Sep 23 23:43:52 2023 2023-09-23T23:43:53.772 INFO:tasks.workunit.client.0.smithi110.stdout: modify_timestamp: Sat Sep 23 23:43:52 2023 2023-09-23T23:43:53.773 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd info testimg_import@snap 2023-09-23T23:43:53.808 INFO:tasks.workunit.client.0.smithi110.stdout:rbd image 'testimg_import': 2023-09-23T23:43:53.809 INFO:tasks.workunit.client.0.smithi110.stdout: size 110 KiB in 1 objects 2023-09-23T23:43:53.809 INFO:tasks.workunit.client.0.smithi110.stdout: order 22 (4 MiB objects) 2023-09-23T23:43:53.809 INFO:tasks.workunit.client.0.smithi110.stdout: snapshot_count: 1 2023-09-23T23:43:53.809 INFO:tasks.workunit.client.0.smithi110.stdout: id: b9bd868198c1 2023-09-23T23:43:53.809 INFO:tasks.workunit.client.0.smithi110.stdout: block_name_prefix: rbd_data.b9bd868198c1 2023-09-23T23:43:53.809 INFO:tasks.workunit.client.0.smithi110.stdout: format: 2 2023-09-23T23:43:53.809 INFO:tasks.workunit.client.0.smithi110.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T23:43:53.809 INFO:tasks.workunit.client.0.smithi110.stdout: op_features: 2023-09-23T23:43:53.809 INFO:tasks.workunit.client.0.smithi110.stdout: flags: 2023-09-23T23:43:53.809 INFO:tasks.workunit.client.0.smithi110.stdout: create_timestamp: Sat Sep 23 23:43:52 2023 2023-09-23T23:43:53.810 INFO:tasks.workunit.client.0.smithi110.stdout: access_timestamp: Sat Sep 23 23:43:52 2023 2023-09-23T23:43:53.810 INFO:tasks.workunit.client.0.smithi110.stdout: modify_timestamp: Sat Sep 23 23:43:52 2023 2023-09-23T23:43:53.810 INFO:tasks.workunit.client.0.smithi110.stdout: protected: False 2023-09-23T23:43:53.812 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:53.813 INFO:tasks.workunit.client.0.smithi110.stderr:++ rbd image-meta list testimg_import 2023-09-23T23:43:53.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:53.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:53.837 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:43:53.853 INFO:tasks.workunit.client.0.smithi110.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-09-23T23:43:53.853 INFO:tasks.workunit.client.0.smithi110.stderr: 2023-09-23T23:43:53.853 INFO:tasks.workunit.client.0.smithi110.stderr:Key Value 2023-09-23T23:43:53.853 INFO:tasks.workunit.client.0.smithi110.stderr:key1 value1' 2023-09-23T23:43:53.853 INFO:tasks.workunit.client.0.smithi110.stderr:+ '[' 'There is 1 metadatum on this image: 2023-09-23T23:43:53.853 INFO:tasks.workunit.client.0.smithi110.stderr: 2023-09-23T23:43:53.853 INFO:tasks.workunit.client.0.smithi110.stderr:Key Value 2023-09-23T23:43:53.853 INFO:tasks.workunit.client.0.smithi110.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-09-23T23:43:53.853 INFO:tasks.workunit.client.0.smithi110.stderr: 2023-09-23T23:43:53.854 INFO:tasks.workunit.client.0.smithi110.stderr:Key Value 2023-09-23T23:43:53.854 INFO:tasks.workunit.client.0.smithi110.stderr:key1 value1' ']' 2023-09-23T23:43:53.854 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd export testimg_import /tmp/rbd_import_export_695376/img_import 2023-09-23T23:43:53.893 INFO:tasks.workunit.client.0.smithi110.stderr: Exporting image: 100% complete...done. 2023-09-23T23:43:53.896 INFO:tasks.workunit.client.0.smithi110.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_695376/img /tmp/rbd_import_export_695376/img_import 2023-09-23T23:43:53.897 INFO:tasks.workunit.client.0.smithi110.stderr:+ cmp -l /tmp/rbd_import_export_695376/img /tmp/rbd_import_export_695376/img_import 2023-09-23T23:43:53.897 INFO:tasks.workunit.client.0.smithi110.stderr:++ stat /tmp/rbd_import_export_695376/img --format %b 2023-09-23T23:43:53.903 INFO:tasks.workunit.client.0.smithi110.stderr:+ origsize=24 2023-09-23T23:43:53.903 INFO:tasks.workunit.client.0.smithi110.stderr:++ stat /tmp/rbd_import_export_695376/img_import --format %b 2023-09-23T23:43:53.905 INFO:tasks.workunit.client.0.smithi110.stderr:+ exportsize=224 2023-09-23T23:43:53.905 INFO:tasks.workunit.client.0.smithi110.stderr:+ difference=200 2023-09-23T23:43:53.905 INFO:tasks.workunit.client.0.smithi110.stderr:+ difference=200 2023-09-23T23:43:53.905 INFO:tasks.workunit.client.0.smithi110.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-23T23:43:53.905 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_695376/img_snap 2023-09-23T23:43:53.917 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:53.919 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:53.944 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:53.947 INFO:tasks.workunit.client.0.smithi110.stderr: Exporting image: 100% complete...done. 2023-09-23T23:43:53.955 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_695376/img_snap_import 2023-09-23T23:43:53.974 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:53.987 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:53.993 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:53 smithi138 ceph-mon[193606]: pgmap v1981: 103 pgs: 103 active+clean; 3.5 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 331 KiB/s rd, 131 KiB/s wr, 107 op/s 2023-09-23T23:43:53.993 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:53 smithi138 ceph-mon[193606]: failed to encode map e702 with expected crc 2023-09-23T23:43:53.993 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:53 smithi138 ceph-mon[193606]: osdmap e702: 8 total, 7 up, 6 in 2023-09-23T23:43:53.994 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:53 smithi138 ceph-mon[193606]: failed to encode map e702 with expected crc 2023-09-23T23:43:53.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:53.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:54.009 INFO:tasks.workunit.client.0.smithi110.stderr: Exporting image: 100% complete...done. 2023-09-23T23:43:54.019 INFO:tasks.workunit.client.0.smithi110.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_695376/img_snap /tmp/rbd_import_export_695376/img_snap_import 2023-09-23T23:43:54.019 INFO:tasks.workunit.client.0.smithi110.stderr:+ cmp -l /tmp/rbd_import_export_695376/img_snap /tmp/rbd_import_export_695376/img_snap_import 2023-09-23T23:43:54.021 INFO:tasks.workunit.client.0.smithi110.stderr:++ stat /tmp/rbd_import_export_695376/img_snap --format %b 2023-09-23T23:43:54.022 INFO:tasks.workunit.client.0.smithi110.stderr:+ origsize=224 2023-09-23T23:43:54.022 INFO:tasks.workunit.client.0.smithi110.stderr:++ stat /tmp/rbd_import_export_695376/img_snap_import --format %b 2023-09-23T23:43:54.023 INFO:tasks.workunit.client.0.smithi110.stderr:+ exportsize=224 2023-09-23T23:43:54.023 INFO:tasks.workunit.client.0.smithi110.stderr:+ difference=0 2023-09-23T23:43:54.023 INFO:tasks.workunit.client.0.smithi110.stderr:+ difference=0 2023-09-23T23:43:54.023 INFO:tasks.workunit.client.0.smithi110.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T23:43:54.023 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm /tmp/rbd_import_export_695376/img_v2 2023-09-23T23:43:54.024 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm /tmp/rbd_import_export_695376/img_import 2023-09-23T23:43:54.025 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm /tmp/rbd_import_export_695376/img_snap 2023-09-23T23:43:54.026 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm /tmp/rbd_import_export_695376/img_snap_import 2023-09-23T23:43:54.027 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd snap rm testimg_import@snap 2023-09-23T23:43:54.095 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:43:54.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:53 smithi110 ceph-mon[212939]: pgmap v1981: 103 pgs: 103 active+clean; 3.5 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 331 KiB/s rd, 131 KiB/s wr, 107 op/s 2023-09-23T23:43:54.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:53 smithi110 ceph-mon[212939]: failed to encode map e702 with expected crc 2023-09-23T23:43:54.127 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:53 smithi110 ceph-mon[212939]: osdmap e702: 8 total, 7 up, 6 in 2023-09-23T23:43:54.128 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:53 smithi110 ceph-mon[212939]: failed to encode map e702 with expected crc 2023-09-23T23:43:54.128 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:53 smithi110 ceph-mon[221102]: pgmap v1981: 103 pgs: 103 active+clean; 3.5 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 331 KiB/s rd, 131 KiB/s wr, 107 op/s 2023-09-23T23:43:54.128 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:53 smithi110 ceph-mon[221102]: failed to encode map e702 with expected crc 2023-09-23T23:43:54.128 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:53 smithi110 ceph-mon[221102]: osdmap e702: 8 total, 7 up, 6 in 2023-09-23T23:43:54.128 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:53 smithi110 ceph-mon[221102]: failed to encode map e702 with expected crc 2023-09-23T23:43:54.130 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:54.642 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:43:54.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:54.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:43:54.749 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:54.780 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:54 smithi110 ceph-mon[212939]: failed to encode map e702 with expected crc 2023-09-23T23:43:54.780 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:54 smithi110 ceph-mon[212939]: failed to encode map e703 with expected crc 2023-09-23T23:43:54.780 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:54 smithi110 ceph-mon[212939]: failed to encode map e703 with expected crc 2023-09-23T23:43:54.780 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:54 smithi110 ceph-mon[212939]: osdmap e703: 8 total, 7 up, 6 in 2023-09-23T23:43:54.780 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:54 smithi110 ceph-mon[221102]: failed to encode map e702 with expected crc 2023-09-23T23:43:54.780 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:54 smithi110 ceph-mon[221102]: failed to encode map e703 with expected crc 2023-09-23T23:43:54.780 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:54 smithi110 ceph-mon[221102]: failed to encode map e703 with expected crc 2023-09-23T23:43:54.780 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:54 smithi110 ceph-mon[221102]: osdmap e703: 8 total, 7 up, 6 in 2023-09-23T23:43:54.783 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:54.961 INFO:tasks.workunit.client.0.smithi110.stderr: Removing snap: 100% complete...done. 2023-09-23T23:43:54.965 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd remove testimg_import 2023-09-23T23:43:55.062 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 100% complete...done. 2023-09-23T23:43:55.064 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd snap rm testimg@snap 2023-09-23T23:43:55.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:54 smithi138 ceph-mon[193606]: failed to encode map e702 with expected crc 2023-09-23T23:43:55.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:54 smithi138 ceph-mon[193606]: failed to encode map e703 with expected crc 2023-09-23T23:43:55.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:54 smithi138 ceph-mon[193606]: failed to encode map e703 with expected crc 2023-09-23T23:43:55.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:54 smithi138 ceph-mon[193606]: osdmap e703: 8 total, 7 up, 6 in 2023-09-23T23:43:55.687 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:55.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:55.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:55.792 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:55.824 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:55.872 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:55.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:55.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:55.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:55 smithi110 ceph-mon[212939]: pgmap v1984: 103 pgs: 103 active+clean; 582 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 259 KiB/s rd, 74 KiB/s wr, 92 op/s 2023-09-23T23:43:55.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:55 smithi110 ceph-mon[212939]: failed to encode map e704 with expected crc 2023-09-23T23:43:55.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:55 smithi110 ceph-mon[212939]: osdmap e704: 8 total, 7 up, 6 in 2023-09-23T23:43:55.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:55 smithi110 ceph-mon[221102]: pgmap v1984: 103 pgs: 103 active+clean; 582 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 259 KiB/s rd, 74 KiB/s wr, 92 op/s 2023-09-23T23:43:55.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:55 smithi110 ceph-mon[221102]: failed to encode map e704 with expected crc 2023-09-23T23:43:55.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:55 smithi110 ceph-mon[221102]: osdmap e704: 8 total, 7 up, 6 in 2023-09-23T23:43:55.960 INFO:tasks.workunit.client.0.smithi110.stderr: Removing snap: 100% complete...done. 2023-09-23T23:43:55.967 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd rm testimg 2023-09-23T23:43:55.981 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:43:56.013 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:56.067 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 100% complete...done. 2023-09-23T23:43:56.069 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --order 20 /tmp/rbd_import_export_695376/img testimg 2023-09-23T23:43:56.125 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 100% complete...done. 2023-09-23T23:43:56.125 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T23:43:56.127 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_695376/img_v2 2023-09-23T23:43:56.185 INFO:tasks.workunit.client.0.smithi110.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T23:43:56.188 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_695376/img_v2 testimg_import 2023-09-23T23:43:56.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:55 smithi138 ceph-mon[193606]: pgmap v1984: 103 pgs: 103 active+clean; 582 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 259 KiB/s rd, 74 KiB/s wr, 92 op/s 2023-09-23T23:43:56.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:55 smithi138 ceph-mon[193606]: failed to encode map e704 with expected crc 2023-09-23T23:43:56.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:55 smithi138 ceph-mon[193606]: osdmap e704: 8 total, 7 up, 6 in 2023-09-23T23:43:56.246 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T23:43:56.250 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd info testimg_import 2023-09-23T23:43:56.250 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep order 2023-09-23T23:43:56.250 INFO:tasks.workunit.client.0.smithi110.stderr:+ awk '{print $2}' 2023-09-23T23:43:56.250 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep 20 2023-09-23T23:43:56.285 INFO:tasks.workunit.client.0.smithi110.stdout:20 2023-09-23T23:43:56.285 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm /tmp/rbd_import_export_695376/img_v2 2023-09-23T23:43:56.285 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd remove testimg_import 2023-09-23T23:43:56.386 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 100% complete...done. 2023-09-23T23:43:56.391 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd remove testimg 2023-09-23T23:43:56.393 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:43:56.425 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:43:56.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:56.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:43:56.518 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 100% complete...done. 2023-09-23T23:43:56.522 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_695376/img testimg 2023-09-23T23:43:56.531 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:56.562 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:56.572 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 100% complete...done. 2023-09-23T23:43:56.575 INFO:tasks.workunit.client.0.smithi110.stderr:++ rbd info testimg 2023-09-23T23:43:56.575 INFO:tasks.workunit.client.0.smithi110.stderr:++ grep features 2023-09-23T23:43:56.622 INFO:tasks.workunit.client.0.smithi110.stderr:+ FEATURES_BEFORE=' features: layering 2023-09-23T23:43:56.622 INFO:tasks.workunit.client.0.smithi110.stderr: op_features: ' 2023-09-23T23:43:56.622 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_695376/img_v2 2023-09-23T23:43:56.660 INFO:tasks.workunit.client.0.smithi110.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T23:43:56.662 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_695376/img_v2 testimg_import 2023-09-23T23:43:56.690 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:43:56.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:56.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:43:56.728 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T23:43:56.731 INFO:tasks.workunit.client.0.smithi110.stderr:++ rbd info testimg_import 2023-09-23T23:43:56.731 INFO:tasks.workunit.client.0.smithi110.stderr:++ grep features 2023-09-23T23:43:56.776 INFO:tasks.workunit.client.0.smithi110.stderr:+ FEATURES_AFTER=' features: layering 2023-09-23T23:43:56.776 INFO:tasks.workunit.client.0.smithi110.stderr: op_features: ' 2023-09-23T23:43:56.776 INFO:tasks.workunit.client.0.smithi110.stderr:+ '[' ' features: layering 2023-09-23T23:43:56.776 INFO:tasks.workunit.client.0.smithi110.stderr: op_features: ' '!=' ' features: layering 2023-09-23T23:43:56.776 INFO:tasks.workunit.client.0.smithi110.stderr: op_features: ' ']' 2023-09-23T23:43:56.776 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm /tmp/rbd_import_export_695376/img_v2 2023-09-23T23:43:56.777 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd remove testimg_import 2023-09-23T23:43:56.795 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:43:56.864 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:56.879 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 100% complete...done. 2023-09-23T23:43:56.881 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd remove testimg 2023-09-23T23:43:56.918 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:43:56.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:56.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:43:56.953 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:43:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:43:56] "GET /metrics HTTP/1.1" 200 34125 "" "Prometheus/2.43.0" 2023-09-23T23:43:56.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:56 smithi110 ceph-mon[221102]: failed to encode map e703 with expected crc 2023-09-23T23:43:56.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:56 smithi110 ceph-mon[221102]: failed to encode map e704 with expected crc 2023-09-23T23:43:56.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:56 smithi110 ceph-mon[221102]: failed to encode map e705 with expected crc 2023-09-23T23:43:56.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:56 smithi110 ceph-mon[221102]: osdmap e705: 8 total, 7 up, 6 in 2023-09-23T23:43:56.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:56 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3589637234' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:43:56.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:56 smithi110 ceph-mon[212939]: failed to encode map e703 with expected crc 2023-09-23T23:43:56.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:56 smithi110 ceph-mon[212939]: failed to encode map e704 with expected crc 2023-09-23T23:43:56.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:56 smithi110 ceph-mon[212939]: failed to encode map e705 with expected crc 2023-09-23T23:43:56.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:56 smithi110 ceph-mon[212939]: osdmap e705: 8 total, 7 up, 6 in 2023-09-23T23:43:56.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:56 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3589637234' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:43:56.975 INFO:teuthology.orchestra.run.smithi110.stderr:set osd.5 primary-affinity to 0.355864 (8233212) 2023-09-23T23:43:56.976 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 100% complete...done. 2023-09-23T23:43:56.983 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_695376/img testimg 2023-09-23T23:43:56.991 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is set 2023-09-23T23:43:57.015 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:43:57.024 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:43:57.038 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:43:57.049 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:56 smithi138 ceph-mon[193606]: failed to encode map e703 with expected crc 2023-09-23T23:43:57.049 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:56 smithi138 ceph-mon[193606]: failed to encode map e704 with expected crc 2023-09-23T23:43:57.049 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:56 smithi138 ceph-mon[193606]: failed to encode map e705 with expected crc 2023-09-23T23:43:57.049 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:56 smithi138 ceph-mon[193606]: osdmap e705: 8 total, 7 up, 6 in 2023-09-23T23:43:57.050 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:56 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3589637234' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:43:57.052 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 100% complete...done. 2023-09-23T23:43:57.052 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:57.057 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_695376/img_v2 2023-09-23T23:43:57.070 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:43:57.093 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:43:57.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:57.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:43:57.101 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:43:57.113 INFO:tasks.workunit.client.0.smithi110.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T23:43:57.116 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_695376/img_v2 testimg_import 2023-09-23T23:43:57.123 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:43:57.142 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:43:57.163 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:43:57.163 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:43:57.164 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:43:57.164 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:43:57.164 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:43:57.164 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-09-23T23:43:57.164 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:43:57.164 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:43:57.164 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:43:57.181 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T23:43:57.184 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd info testimg_import 2023-09-23T23:43:57.185 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep 'stripe unit' 2023-09-23T23:43:57.185 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep -Ei '(4 KiB|4096)' 2023-09-23T23:43:57.198 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:57.221 INFO:tasks.workunit.client.0.smithi110.stdout: stripe unit: 4 KiB 2023-09-23T23:43:57.222 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd info testimg_import 2023-09-23T23:43:57.222 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep 'stripe count' 2023-09-23T23:43:57.222 INFO:tasks.workunit.client.0.smithi110.stderr:+ awk '{print $3}' 2023-09-23T23:43:57.222 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep 1000 2023-09-23T23:43:57.224 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:57.264 INFO:tasks.workunit.client.0.smithi110.stdout:1000 2023-09-23T23:43:57.264 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm /tmp/rbd_import_export_695376/img_v2 2023-09-23T23:43:57.264 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd remove testimg_import 2023-09-23T23:43:57.266 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:57.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:57.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:57.351 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T23:43:57.354 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd remove testimg 2023-09-23T23:43:57.372 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:57.398 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:57.438 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T23:43:57.441 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_695376/img testimg 2023-09-23T23:43:57.443 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:57.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:57.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:57.494 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 100% complete...done. 2023-09-23T23:43:57.497 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd snap create testimg@snap1 2023-09-23T23:43:57.550 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:43:57.578 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:57.869 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[221102]: failed to encode map e705 with expected crc 2023-09-23T23:43:57.870 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[221102]: pgmap v1987: 103 pgs: 103 active+clean; 692 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 107 op/s 2023-09-23T23:43:57.870 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[221102]: failed to encode map e705 with expected crc 2023-09-23T23:43:57.870 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/108343801' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.3558638796668582}]: dispatch 2023-09-23T23:43:57.870 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.3558638796668582}]: dispatch 2023-09-23T23:43:57.870 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[221102]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:43:57.870 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[221102]: failed to encode map e706 with expected crc 2023-09-23T23:43:57.870 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3589637234' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:43:57.870 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.3558638796668582}]': finished 2023-09-23T23:43:57.870 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[221102]: osdmap e706: 8 total, 7 up, 6 in 2023-09-23T23:43:57.870 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/4216777968' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:57.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[212939]: failed to encode map e705 with expected crc 2023-09-23T23:43:57.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[212939]: pgmap v1987: 103 pgs: 103 active+clean; 692 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 107 op/s 2023-09-23T23:43:57.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[212939]: failed to encode map e705 with expected crc 2023-09-23T23:43:57.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/108343801' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.3558638796668582}]: dispatch 2023-09-23T23:43:57.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.3558638796668582}]: dispatch 2023-09-23T23:43:57.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[212939]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:43:57.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[212939]: failed to encode map e706 with expected crc 2023-09-23T23:43:57.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3589637234' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:43:57.871 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.3558638796668582}]': finished 2023-09-23T23:43:57.872 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[212939]: osdmap e706: 8 total, 7 up, 6 in 2023-09-23T23:43:57.872 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:57 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/4216777968' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:58.052 INFO:tasks.workunit.client.0.smithi110.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T23:43:58.061 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd snap create testimg@snap2 2023-09-23T23:43:58.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:57 smithi138 ceph-mon[193606]: failed to encode map e705 with expected crc 2023-09-23T23:43:58.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:57 smithi138 ceph-mon[193606]: pgmap v1987: 103 pgs: 103 active+clean; 692 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 107 op/s 2023-09-23T23:43:58.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:57 smithi138 ceph-mon[193606]: failed to encode map e705 with expected crc 2023-09-23T23:43:58.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:57 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/108343801' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.3558638796668582}]: dispatch 2023-09-23T23:43:58.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:57 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.3558638796668582}]: dispatch 2023-09-23T23:43:58.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:57 smithi138 ceph-mon[193606]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:43:58.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:57 smithi138 ceph-mon[193606]: failed to encode map e706 with expected crc 2023-09-23T23:43:58.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:57 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3589637234' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:43:58.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:57 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.3558638796668582}]': finished 2023-09-23T23:43:58.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:57 smithi138 ceph-mon[193606]: osdmap e706: 8 total, 7 up, 6 in 2023-09-23T23:43:58.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:57 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/4216777968' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:43:58.527 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:43:58.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:58.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:43:58.630 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:43:58.657 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:58.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:58 smithi110 ceph-mon[212939]: failed to encode map e706 with expected crc 2023-09-23T23:43:58.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:58 smithi110 ceph-mon[212939]: failed to encode map e707 with expected crc 2023-09-23T23:43:58.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:58 smithi110 ceph-mon[212939]: osdmap e707: 8 total, 7 up, 6 in 2023-09-23T23:43:58.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:58 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:58.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:43:58.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:58 smithi110 ceph-mon[221102]: failed to encode map e706 with expected crc 2023-09-23T23:43:58.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:58 smithi110 ceph-mon[221102]: failed to encode map e707 with expected crc 2023-09-23T23:43:58.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:58 smithi110 ceph-mon[221102]: osdmap e707: 8 total, 7 up, 6 in 2023-09-23T23:43:58.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:58 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:58.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:43:59.069 INFO:tasks.workunit.client.0.smithi110.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T23:43:59.076 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd snap protect testimg@snap2 2023-09-23T23:43:59.117 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_695376/snap_protect 2023-09-23T23:43:59.157 INFO:tasks.workunit.client.0.smithi110.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T23:43:59.160 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_695376/snap_protect testimg_import 2023-09-23T23:43:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:58 smithi138 ceph-mon[193606]: failed to encode map e706 with expected crc 2023-09-23T23:43:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:58 smithi138 ceph-mon[193606]: failed to encode map e707 with expected crc 2023-09-23T23:43:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:58 smithi138 ceph-mon[193606]: osdmap e707: 8 total, 7 up, 6 in 2023-09-23T23:43:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:58 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:43:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:43:59.212 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:43:59.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:59.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:43:59.319 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:43:59.350 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:59.396 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:43:59.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:59.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:43:59.460 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:43:59.461 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:43:59.502 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:43:59.529 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:59.573 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:43:59.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:43:59.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:43:59.680 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:43:59.717 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:43:59.848 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:43:59.849 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:43:59.872 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:59 smithi110 ceph-mon[212939]: failed to encode map e707 with expected crc 2023-09-23T23:43:59.872 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:59 smithi110 ceph-mon[212939]: failed to encode map e707 with expected crc 2023-09-23T23:43:59.872 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:59 smithi110 ceph-mon[212939]: pgmap v1990: 103 pgs: 103 active+clean; 692 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 199 KiB/s rd, 16 KiB/s wr, 250 op/s 2023-09-23T23:43:59.872 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:59 smithi110 ceph-mon[212939]: failed to encode map e708 with expected crc 2023-09-23T23:43:59.873 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:43:59 smithi110 ceph-mon[212939]: osdmap e708: 8 total, 7 up, 6 in 2023-09-23T23:43:59.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:59 smithi110 ceph-mon[221102]: failed to encode map e707 with expected crc 2023-09-23T23:43:59.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:59 smithi110 ceph-mon[221102]: failed to encode map e707 with expected crc 2023-09-23T23:43:59.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:59 smithi110 ceph-mon[221102]: pgmap v1990: 103 pgs: 103 active+clean; 692 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 199 KiB/s rd, 16 KiB/s wr, 250 op/s 2023-09-23T23:43:59.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:59 smithi110 ceph-mon[221102]: failed to encode map e708 with expected crc 2023-09-23T23:43:59.873 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:43:59 smithi110 ceph-mon[221102]: osdmap e708: 8 total, 7 up, 6 in 2023-09-23T23:44:00.068 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:44:00.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:00.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:44:00.178 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:44:00.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:59 smithi138 ceph-mon[193606]: failed to encode map e707 with expected crc 2023-09-23T23:44:00.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:59 smithi138 ceph-mon[193606]: failed to encode map e707 with expected crc 2023-09-23T23:44:00.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:59 smithi138 ceph-mon[193606]: pgmap v1990: 103 pgs: 103 active+clean; 692 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 199 KiB/s rd, 16 KiB/s wr, 250 op/s 2023-09-23T23:44:00.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:59 smithi138 ceph-mon[193606]: failed to encode map e708 with expected crc 2023-09-23T23:44:00.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:43:59 smithi138 ceph-mon[193606]: osdmap e708: 8 total, 7 up, 6 in 2023-09-23T23:44:00.206 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:00.248 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:44:00.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:00.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:44:00.356 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:44:00.382 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:00.424 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:44:00.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:00.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:44:00.530 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:44:00.560 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:00.864 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:44:00.864 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:00 smithi110 ceph-mon[212939]: failed to encode map e708 with expected crc 2023-09-23T23:44:00.864 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:00 smithi110 ceph-mon[212939]: failed to encode map e708 with expected crc 2023-09-23T23:44:00.865 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:00 smithi110 ceph-mon[212939]: failed to encode map e708 with expected crc 2023-09-23T23:44:00.865 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:00 smithi110 ceph-mon[212939]: osdmap e709: 8 total, 7 up, 6 in 2023-09-23T23:44:00.865 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:00 smithi110 ceph-mon[212939]: failed to encode map e709 with expected crc 2023-09-23T23:44:00.865 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:00 smithi110 ceph-mon[221102]: failed to encode map e708 with expected crc 2023-09-23T23:44:00.865 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:00 smithi110 ceph-mon[221102]: failed to encode map e708 with expected crc 2023-09-23T23:44:00.865 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:00 smithi110 ceph-mon[221102]: failed to encode map e708 with expected crc 2023-09-23T23:44:00.866 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:00 smithi110 ceph-mon[221102]: osdmap e709: 8 total, 7 up, 6 in 2023-09-23T23:44:00.866 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:00 smithi110 ceph-mon[221102]: failed to encode map e709 with expected crc 2023-09-23T23:44:00.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:00.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:44:00.968 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:44:00.993 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:00 smithi138 ceph-mon[193606]: failed to encode map e708 with expected crc 2023-09-23T23:44:00.993 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:00 smithi138 ceph-mon[193606]: failed to encode map e708 with expected crc 2023-09-23T23:44:00.993 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:00 smithi138 ceph-mon[193606]: failed to encode map e708 with expected crc 2023-09-23T23:44:00.993 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:00 smithi138 ceph-mon[193606]: osdmap e709: 8 total, 7 up, 6 in 2023-09-23T23:44:00.993 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:00 smithi138 ceph-mon[193606]: failed to encode map e709 with expected crc 2023-09-23T23:44:00.995 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:01.037 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:44:01.038 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:44:01.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:01.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:44:01.094 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T23:44:01.097 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd info testimg_import@snap1 2023-09-23T23:44:01.097 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep 'protected: False' 2023-09-23T23:44:01.140 INFO:tasks.workunit.client.0.smithi110.stdout: protected: False 2023-09-23T23:44:01.140 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd info testimg_import@snap2 2023-09-23T23:44:01.140 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep 'protected: True' 2023-09-23T23:44:01.145 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:44:01.176 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:01.187 INFO:tasks.workunit.client.0.smithi110.stdout: protected: True 2023-09-23T23:44:01.187 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm /tmp/rbd_import_export_695376/snap_protect 2023-09-23T23:44:01.188 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd snap unprotect testimg@snap2 2023-09-23T23:44:01.233 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-23T23:44:01.277 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:44:01.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:01.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:44:01.280 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd snap purge testimg 2023-09-23T23:44:01.312 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:44:01.381 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:44:01.408 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:01.449 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:44:01.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:01.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:44:01.462 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:44:01.555 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:44:01.559 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:01.588 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:01.648 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:44:01.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:01.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:44:01.753 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:44:01.780 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:01.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:01 smithi110 ceph-mon[221102]: failed to encode map e709 with expected crc 2023-09-23T23:44:01.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:01 smithi110 ceph-mon[221102]: pgmap v1993: 103 pgs: 103 active+clean; 802 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 200 KiB/s rd, 19 KiB/s wr, 262 op/s 2023-09-23T23:44:01.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:01 smithi110 ceph-mon[221102]: failed to encode map e710 with expected crc 2023-09-23T23:44:01.804 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:01 smithi110 ceph-mon[221102]: osdmap e710: 8 total, 7 up, 6 in 2023-09-23T23:44:01.804 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:01 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:01.804 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:01 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:01.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:01 smithi110 ceph-mon[212939]: failed to encode map e709 with expected crc 2023-09-23T23:44:01.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:01 smithi110 ceph-mon[212939]: pgmap v1993: 103 pgs: 103 active+clean; 802 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 200 KiB/s rd, 19 KiB/s wr, 262 op/s 2023-09-23T23:44:01.804 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:01 smithi110 ceph-mon[212939]: failed to encode map e710 with expected crc 2023-09-23T23:44:01.805 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:01 smithi110 ceph-mon[212939]: osdmap e710: 8 total, 7 up, 6 in 2023-09-23T23:44:01.805 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:01 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:01.805 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:01 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:01.820 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:44:01.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:01.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:44:01.925 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:44:01.956 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:01.979 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:02.116 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:44:02.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:02.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:44:02.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:01 smithi138 ceph-mon[193606]: failed to encode map e709 with expected crc 2023-09-23T23:44:02.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:01 smithi138 ceph-mon[193606]: pgmap v1993: 103 pgs: 103 active+clean; 802 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 200 KiB/s rd, 19 KiB/s wr, 262 op/s 2023-09-23T23:44:02.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:01 smithi138 ceph-mon[193606]: failed to encode map e710 with expected crc 2023-09-23T23:44:02.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:01 smithi138 ceph-mon[193606]: osdmap e710: 8 total, 7 up, 6 in 2023-09-23T23:44:02.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:01 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:02.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:01 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:02.220 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:44:02.249 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:02.292 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:44:02.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:02.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:44:02.396 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:44:02.423 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:02.621 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:44:02.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:02.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:44:02.725 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:44:02.756 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:02.782 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:02.805 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:44:02.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:02.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:44:02.909 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:44:02.941 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:03.076 INFO:tasks.workunit.client.0.smithi110.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T23:44:03.084 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd snap purge testimg_import 2023-09-23T23:44:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:02 smithi138 ceph-mon[193606]: failed to encode map e710 with expected crc 2023-09-23T23:44:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:02 smithi138 ceph-mon[193606]: failed to encode map e711 with expected crc 2023-09-23T23:44:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:02 smithi138 ceph-mon[193606]: osdmap e711: 8 total, 7 up, 6 in 2023-09-23T23:44:03.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:02 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:44:03.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:02 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:44:03.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:02 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:03.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:02 smithi110 ceph-mon[212939]: failed to encode map e710 with expected crc 2023-09-23T23:44:03.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:02 smithi110 ceph-mon[212939]: failed to encode map e711 with expected crc 2023-09-23T23:44:03.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:02 smithi110 ceph-mon[212939]: osdmap e711: 8 total, 7 up, 6 in 2023-09-23T23:44:03.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:02 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:44:03.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:02 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:44:03.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:02 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:03.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:02 smithi110 ceph-mon[221102]: failed to encode map e710 with expected crc 2023-09-23T23:44:03.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:02 smithi110 ceph-mon[221102]: failed to encode map e711 with expected crc 2023-09-23T23:44:03.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:02 smithi110 ceph-mon[221102]: osdmap e711: 8 total, 7 up, 6 in 2023-09-23T23:44:03.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:02 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:44:03.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:02 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:44:03.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:02 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:03.747 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:03 smithi110 ceph-mon[212939]: failed to encode map e711 with expected crc 2023-09-23T23:44:03.747 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:03 smithi110 ceph-mon[212939]: pgmap v1996: 103 pgs: 103 active+clean; 692 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 330 KiB/s rd, 41 KiB/s wr, 448 op/s 2023-09-23T23:44:03.747 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:03 smithi110 ceph-mon[212939]: failed to encode map e711 with expected crc 2023-09-23T23:44:03.747 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:03 smithi110 ceph-mon[212939]: failed to encode map e712 with expected crc 2023-09-23T23:44:03.747 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:03 smithi110 ceph-mon[212939]: osdmap e712: 8 total, 7 up, 6 in 2023-09-23T23:44:03.748 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:03 smithi110 ceph-mon[221102]: failed to encode map e711 with expected crc 2023-09-23T23:44:03.748 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:03 smithi110 ceph-mon[221102]: pgmap v1996: 103 pgs: 103 active+clean; 692 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 330 KiB/s rd, 41 KiB/s wr, 448 op/s 2023-09-23T23:44:03.748 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:03 smithi110 ceph-mon[221102]: failed to encode map e711 with expected crc 2023-09-23T23:44:03.748 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:03 smithi110 ceph-mon[221102]: failed to encode map e712 with expected crc 2023-09-23T23:44:03.748 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:03 smithi110 ceph-mon[221102]: osdmap e712: 8 total, 7 up, 6 in 2023-09-23T23:44:03.748 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:44:03.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:03.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:44:03.852 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:44:03.881 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-09-23T23:44:03.881 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:44:03.884 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:03.932 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:44:03.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:03.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:44:04.038 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:44:04.064 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:03 smithi138 ceph-mon[193606]: failed to encode map e711 with expected crc 2023-09-23T23:44:04.064 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:03 smithi138 ceph-mon[193606]: pgmap v1996: 103 pgs: 103 active+clean; 692 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 330 KiB/s rd, 41 KiB/s wr, 448 op/s 2023-09-23T23:44:04.064 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:03 smithi138 ceph-mon[193606]: failed to encode map e711 with expected crc 2023-09-23T23:44:04.064 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:03 smithi138 ceph-mon[193606]: failed to encode map e712 with expected crc 2023-09-23T23:44:04.064 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:03 smithi138 ceph-mon[193606]: osdmap e712: 8 total, 7 up, 6 in 2023-09-23T23:44:04.066 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:04.110 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:44:04.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:04.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:44:04.215 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:44:04.231 INFO:tasks.thrashosds.thrasher:in_osds: [0, 5, 7, 2, 1, 3] out_osds: [4, 6] dead_osds: [4] live_osds: [0, 1, 3, 5, 6, 2, 7] 2023-09-23T23:44:04.231 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:44:04.232 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T23:44:04.232 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T23:44:04.242 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:04.283 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:44:04.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:04.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:44:04.389 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:44:04.417 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:04.918 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:04 smithi110 ceph-mon[212939]: failed to encode map e712 with expected crc 2023-09-23T23:44:04.918 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:04 smithi110 ceph-mon[212939]: failed to encode map e712 with expected crc 2023-09-23T23:44:04.918 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:04 smithi110 ceph-mon[212939]: failed to encode map e713 with expected crc 2023-09-23T23:44:04.918 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:04 smithi110 ceph-mon[212939]: osdmap e713: 8 total, 7 up, 6 in 2023-09-23T23:44:04.918 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:04 smithi110 ceph-mon[221102]: failed to encode map e712 with expected crc 2023-09-23T23:44:04.919 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:04 smithi110 ceph-mon[221102]: failed to encode map e712 with expected crc 2023-09-23T23:44:04.919 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:04 smithi110 ceph-mon[221102]: failed to encode map e713 with expected crc 2023-09-23T23:44:04.919 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:04 smithi110 ceph-mon[221102]: osdmap e713: 8 total, 7 up, 6 in 2023-09-23T23:44:04.964 INFO:tasks.workunit.client.0.smithi110.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T23:44:04.976 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd remove testimg 2023-09-23T23:44:05.021 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:44:05.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:05.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:44:05.072 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 100% complete...done. 2023-09-23T23:44:05.075 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd remove testimg_import 2023-09-23T23:44:05.126 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:44:05.158 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:05.167 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 100% complete...done. 2023-09-23T23:44:05.172 INFO:tasks.workunit.client.0.smithi110.stderr:+ tiered=0 2023-09-23T23:44:05.172 INFO:tasks.workunit.client.0.smithi110.stderr:+ ceph osd dump 2023-09-23T23:44:05.172 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep '^pool' 2023-09-23T23:44:05.172 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep ''\''rbd'\''' 2023-09-23T23:44:05.173 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep tier 2023-09-23T23:44:05.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:04 smithi138 ceph-mon[193606]: failed to encode map e712 with expected crc 2023-09-23T23:44:05.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:04 smithi138 ceph-mon[193606]: failed to encode map e712 with expected crc 2023-09-23T23:44:05.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:04 smithi138 ceph-mon[193606]: failed to encode map e713 with expected crc 2023-09-23T23:44:05.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:04 smithi138 ceph-mon[193606]: osdmap e713: 8 total, 7 up, 6 in 2023-09-23T23:44:05.217 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:44:05.307 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:44:05.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:05.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:44:05.411 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:44:05.456 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:05.602 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:05.665 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:44:05.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:05.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:44:05.771 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:44:05.805 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:05.889 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:44:05.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:05.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:44:05.906 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:05.916 INFO:tasks.workunit.client.0.smithi110.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_695376/sparse1 2023-09-23T23:44:05.919 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0: {} 2023-09-23T23:44:05.923 INFO:tasks.workunit.client.0.smithi110.stderr:1+0 records in 2023-09-23T23:44:05.923 INFO:tasks.workunit.client.0.smithi110.stderr:1+0 records out 2023-09-23T23:44:05.923 INFO:tasks.workunit.client.0.smithi110.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00599808 s, 175 MB/s 2023-09-23T23:44:05.923 INFO:tasks.workunit.client.0.smithi110.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_695376/sparse2 2023-09-23T23:44:05.930 INFO:tasks.workunit.client.0.smithi110.stderr:1+0 records in 2023-09-23T23:44:05.930 INFO:tasks.workunit.client.0.smithi110.stderr:1+0 records out 2023-09-23T23:44:05.930 INFO:tasks.workunit.client.0.smithi110.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00562116 s, 187 MB/s 2023-09-23T23:44:05.930 INFO:tasks.workunit.client.0.smithi110.stderr:+ truncate /tmp/rbd_import_export_695376/sparse2 -s 2M 2023-09-23T23:44:05.943 INFO:teuthology.orchestra.run.smithi110.stdout:osd.1: {} 2023-09-23T23:44:05.962 INFO:teuthology.orchestra.run.smithi110.stdout:osd.2: {} 2023-09-23T23:44:05.964 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd rm sparse1 2023-09-23T23:44:05.983 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3: {} 2023-09-23T23:44:05.985 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:05 smithi110 ceph-mon[212939]: failed to encode map e713 with expected crc 2023-09-23T23:44:05.985 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:05 smithi110 ceph-mon[212939]: pgmap v1999: 103 pgs: 103 active+clean; 802 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 325 KiB/s rd, 36 KiB/s wr, 435 op/s 2023-09-23T23:44:05.985 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:05 smithi110 ceph-mon[212939]: failed to encode map e713 with expected crc 2023-09-23T23:44:05.985 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:05 smithi110 ceph-mon[212939]: failed to encode map e714 with expected crc 2023-09-23T23:44:05.985 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:05 smithi110 ceph-mon[212939]: osdmap e714: 8 total, 7 up, 6 in 2023-09-23T23:44:05.985 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:05 smithi110 ceph-mon[221102]: failed to encode map e713 with expected crc 2023-09-23T23:44:05.985 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:05 smithi110 ceph-mon[221102]: pgmap v1999: 103 pgs: 103 active+clean; 802 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 325 KiB/s rd, 36 KiB/s wr, 435 op/s 2023-09-23T23:44:05.985 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:05 smithi110 ceph-mon[221102]: failed to encode map e713 with expected crc 2023-09-23T23:44:05.986 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:05 smithi110 ceph-mon[221102]: failed to encode map e714 with expected crc 2023-09-23T23:44:05.986 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:05 smithi110 ceph-mon[221102]: osdmap e714: 8 total, 7 up, 6 in 2023-09-23T23:44:05.993 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:44:06.009 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 0% complete...failed. 2023-09-23T23:44:06.009 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T23:44:06.013 INFO:teuthology.orchestra.run.smithi110.stdout:osd.5: {} 2023-09-23T23:44:06.013 INFO:tasks.workunit.client.0.smithi110.stderr:+ true 2023-09-23T23:44:06.014 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_695376/sparse1 2023-09-23T23:44:06.022 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:05 smithi138 ceph-mon[193606]: failed to encode map e713 with expected crc 2023-09-23T23:44:06.022 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:05 smithi138 ceph-mon[193606]: pgmap v1999: 103 pgs: 103 active+clean; 802 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 325 KiB/s rd, 36 KiB/s wr, 435 op/s 2023-09-23T23:44:06.022 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:05 smithi138 ceph-mon[193606]: failed to encode map e713 with expected crc 2023-09-23T23:44:06.022 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:05 smithi138 ceph-mon[193606]: failed to encode map e714 with expected crc 2023-09-23T23:44:06.023 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:05 smithi138 ceph-mon[193606]: osdmap e714: 8 total, 7 up, 6 in 2023-09-23T23:44:06.025 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:06.047 INFO:teuthology.orchestra.run.smithi110.stdout:osd.6: {} 2023-09-23T23:44:06.069 INFO:teuthology.orchestra.run.smithi110.stdout:osd.7: {} 2023-09-23T23:44:06.069 INFO:teuthology.orchestra.run.smithi110.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:44:06.069 INFO:teuthology.orchestra.run.smithi110.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:44:06.070 INFO:teuthology.orchestra.run.smithi110.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:44:06.070 INFO:teuthology.orchestra.run.smithi110.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:44:06.070 INFO:teuthology.orchestra.run.smithi110.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-09-23T23:44:06.070 INFO:teuthology.orchestra.run.smithi110.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:44:06.070 INFO:teuthology.orchestra.run.smithi110.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:44:06.070 INFO:teuthology.orchestra.run.smithi110.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:44:06.075 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:44:06.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:06.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:44:06.087 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T23:44:06.088 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:44:06.088 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T23:44:06.091 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd ls -l 2023-09-23T23:44:06.091 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep sparse1 2023-09-23T23:44:06.091 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T23:44:06.127 INFO:tasks.workunit.client.0.smithi110.stdout:sparse1 2 MiB 2 2023-09-23T23:44:06.128 INFO:tasks.workunit.client.0.smithi110.stderr:++ objects sparse1 2023-09-23T23:44:06.128 INFO:tasks.workunit.client.0.smithi110.stderr:++ image=sparse1 2023-09-23T23:44:06.128 INFO:tasks.workunit.client.0.smithi110.stderr:+++ rbd info sparse1 2023-09-23T23:44:06.129 INFO:tasks.workunit.client.0.smithi110.stderr:+++ grep block_name_prefix 2023-09-23T23:44:06.129 INFO:tasks.workunit.client.0.smithi110.stderr:+++ awk '{print $NF;}' 2023-09-23T23:44:06.170 INFO:tasks.workunit.client.0.smithi110.stderr:++ prefix=rbd_data.93b77b1d51fe 2023-09-23T23:44:06.171 INFO:tasks.workunit.client.0.smithi110.stderr:+++ grep rbd_data.93b77b1d51fe 2023-09-23T23:44:06.171 INFO:tasks.workunit.client.0.smithi110.stderr:++++ get_image_data_pool sparse1 2023-09-23T23:44:06.172 INFO:tasks.workunit.client.0.smithi110.stderr:++++ image=sparse1 2023-09-23T23:44:06.172 INFO:tasks.workunit.client.0.smithi110.stderr:+++ sed -e 's/rbd_data.93b77b1d51fe\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T23:44:06.172 INFO:tasks.workunit.client.0.smithi110.stderr:+++ sort -u 2023-09-23T23:44:06.172 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ rbd info sparse1 2023-09-23T23:44:06.172 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ grep 'data_pool: ' 2023-09-23T23:44:06.172 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T23:44:06.182 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:44:06.210 INFO:tasks.workunit.client.0.smithi110.stderr:++++ data_pool= 2023-09-23T23:44:06.210 INFO:tasks.workunit.client.0.smithi110.stderr:++++ '[' -z ']' 2023-09-23T23:44:06.210 INFO:tasks.workunit.client.0.smithi110.stderr:++++ data_pool=rbd 2023-09-23T23:44:06.210 INFO:tasks.workunit.client.0.smithi110.stderr:++++ echo rbd 2023-09-23T23:44:06.210 INFO:tasks.workunit.client.0.smithi110.stderr:+++ rados ls -p rbd 2023-09-23T23:44:06.213 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:06.297 INFO:tasks.workunit.client.0.smithi110.stderr:++ objects=1 2023-09-23T23:44:06.297 INFO:tasks.workunit.client.0.smithi110.stderr:++ echo 1 2023-09-23T23:44:06.297 INFO:tasks.workunit.client.0.smithi110.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T23:44:06.297 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd export sparse1 /tmp/rbd_import_export_695376/sparse1.out 2023-09-23T23:44:06.337 INFO:tasks.workunit.client.0.smithi110.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T23:44:06.341 INFO:tasks.workunit.client.0.smithi110.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_695376/sparse1 /tmp/rbd_import_export_695376/sparse1.out 2023-09-23T23:44:06.341 INFO:tasks.workunit.client.0.smithi110.stderr:+ cmp -l /tmp/rbd_import_export_695376/sparse1 /tmp/rbd_import_export_695376/sparse1.out 2023-09-23T23:44:06.344 INFO:tasks.workunit.client.0.smithi110.stderr:++ stat /tmp/rbd_import_export_695376/sparse1 --format %b 2023-09-23T23:44:06.345 INFO:tasks.workunit.client.0.smithi110.stderr:+ origsize=2048 2023-09-23T23:44:06.345 INFO:tasks.workunit.client.0.smithi110.stderr:++ stat /tmp/rbd_import_export_695376/sparse1.out --format %b 2023-09-23T23:44:06.346 INFO:tasks.workunit.client.0.smithi110.stderr:+ exportsize=2048 2023-09-23T23:44:06.346 INFO:tasks.workunit.client.0.smithi110.stderr:+ difference=0 2023-09-23T23:44:06.346 INFO:tasks.workunit.client.0.smithi110.stderr:+ difference=0 2023-09-23T23:44:06.346 INFO:tasks.workunit.client.0.smithi110.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T23:44:06.346 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm /tmp/rbd_import_export_695376/sparse1.out 2023-09-23T23:44:06.346 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd rm sparse1 2023-09-23T23:44:06.436 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T23:44:06.440 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd rm sparse2 2023-09-23T23:44:06.464 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:44:06.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:06.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:44:06.477 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 0% complete...failed. 2023-09-23T23:44:06.477 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T23:44:06.480 INFO:tasks.workunit.client.0.smithi110.stderr:+ true 2023-09-23T23:44:06.480 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_695376/sparse2 2023-09-23T23:44:06.568 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:44:06.595 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:06.691 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T23:44:06.692 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:44:06.692 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T23:44:06.695 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd ls -l 2023-09-23T23:44:06.695 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep sparse2 2023-09-23T23:44:06.695 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T23:44:06.730 INFO:tasks.workunit.client.0.smithi110.stdout:sparse2 2 MiB 2 2023-09-23T23:44:06.730 INFO:tasks.workunit.client.0.smithi110.stderr:++ objects sparse2 2023-09-23T23:44:06.731 INFO:tasks.workunit.client.0.smithi110.stderr:++ image=sparse2 2023-09-23T23:44:06.731 INFO:tasks.workunit.client.0.smithi110.stderr:+++ rbd info sparse2 2023-09-23T23:44:06.731 INFO:tasks.workunit.client.0.smithi110.stderr:+++ grep block_name_prefix 2023-09-23T23:44:06.731 INFO:tasks.workunit.client.0.smithi110.stderr:+++ awk '{print $NF;}' 2023-09-23T23:44:06.745 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:44:06.777 INFO:tasks.workunit.client.0.smithi110.stderr:++ prefix=rbd_data.bb1cea0e5cc5 2023-09-23T23:44:06.778 INFO:tasks.workunit.client.0.smithi110.stderr:+++ grep rbd_data.bb1cea0e5cc5 2023-09-23T23:44:06.778 INFO:tasks.workunit.client.0.smithi110.stderr:++++ get_image_data_pool sparse2 2023-09-23T23:44:06.778 INFO:tasks.workunit.client.0.smithi110.stderr:++++ image=sparse2 2023-09-23T23:44:06.778 INFO:tasks.workunit.client.0.smithi110.stderr:+++ sed -e 's/rbd_data.bb1cea0e5cc5\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T23:44:06.783 INFO:tasks.workunit.client.0.smithi110.stderr:+++ sort -u 2023-09-23T23:44:06.783 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ rbd info sparse2 2023-09-23T23:44:06.783 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ grep 'data_pool: ' 2023-09-23T23:44:06.783 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T23:44:06.818 INFO:tasks.workunit.client.0.smithi110.stderr:++++ data_pool= 2023-09-23T23:44:06.818 INFO:tasks.workunit.client.0.smithi110.stderr:++++ '[' -z ']' 2023-09-23T23:44:06.818 INFO:tasks.workunit.client.0.smithi110.stderr:++++ data_pool=rbd 2023-09-23T23:44:06.818 INFO:tasks.workunit.client.0.smithi110.stderr:++++ echo rbd 2023-09-23T23:44:06.818 INFO:tasks.workunit.client.0.smithi110.stderr:+++ rados ls -p rbd 2023-09-23T23:44:06.853 INFO:tasks.workunit.client.0.smithi110.stderr:++ objects=0 2023-09-23T23:44:06.853 INFO:tasks.workunit.client.0.smithi110.stderr:++ echo 0 2023-09-23T23:44:06.853 INFO:tasks.workunit.client.0.smithi110.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T23:44:06.853 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd export sparse2 /tmp/rbd_import_export_695376/sparse2.out 2023-09-23T23:44:06.888 INFO:tasks.workunit.client.0.smithi110.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T23:44:06.892 INFO:tasks.workunit.client.0.smithi110.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_695376/sparse2 /tmp/rbd_import_export_695376/sparse2.out 2023-09-23T23:44:06.892 INFO:tasks.workunit.client.0.smithi110.stderr:+ cmp -l /tmp/rbd_import_export_695376/sparse2 /tmp/rbd_import_export_695376/sparse2.out 2023-09-23T23:44:06.894 INFO:tasks.workunit.client.0.smithi110.stderr:++ stat /tmp/rbd_import_export_695376/sparse2 --format %b 2023-09-23T23:44:06.895 INFO:tasks.workunit.client.0.smithi110.stderr:+ origsize=2048 2023-09-23T23:44:06.895 INFO:tasks.workunit.client.0.smithi110.stderr:++ stat /tmp/rbd_import_export_695376/sparse2.out --format %b 2023-09-23T23:44:06.896 INFO:tasks.workunit.client.0.smithi110.stderr:+ exportsize=2048 2023-09-23T23:44:06.896 INFO:tasks.workunit.client.0.smithi110.stderr:+ difference=0 2023-09-23T23:44:06.896 INFO:tasks.workunit.client.0.smithi110.stderr:+ difference=0 2023-09-23T23:44:06.896 INFO:tasks.workunit.client.0.smithi110.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T23:44:06.897 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm /tmp/rbd_import_export_695376/sparse2.out 2023-09-23T23:44:06.897 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd rm sparse2 2023-09-23T23:44:06.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:44:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:44:06] "GET /metrics HTTP/1.1" 200 34152 "" "Prometheus/2.43.0" 2023-09-23T23:44:06.952 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:06 smithi110 ceph-mon[221102]: failed to encode map e714 with expected crc 2023-09-23T23:44:06.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:06 smithi110 ceph-mon[221102]: failed to encode map e714 with expected crc 2023-09-23T23:44:06.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:06 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2346379897' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T23:44:06.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:06 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/454685645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:44:06.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:06 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2346379897' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T23:44:06.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:06 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/776505097' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:44:06.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:06 smithi110 ceph-mon[212939]: failed to encode map e714 with expected crc 2023-09-23T23:44:06.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:06 smithi110 ceph-mon[212939]: failed to encode map e714 with expected crc 2023-09-23T23:44:06.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:06 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2346379897' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T23:44:06.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:06 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/454685645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:44:06.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:06 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2346379897' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T23:44:06.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:06 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/776505097' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:44:06.982 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T23:44:06.986 INFO:tasks.workunit.client.0.smithi110.stderr:+ truncate /tmp/rbd_import_export_695376/sparse1 -s 10M 2023-09-23T23:44:06.987 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-23T23:44:07.076 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 100% complete...done. 2023-09-23T23:44:07.076 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:44:07.076 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T23:44:07.079 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd ls -l 2023-09-23T23:44:07.080 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep sparse1 2023-09-23T23:44:07.080 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-09-23T23:44:07.115 INFO:tasks.workunit.client.0.smithi110.stdout:sparse1 10 MiB 2 2023-09-23T23:44:07.116 INFO:tasks.workunit.client.0.smithi110.stderr:++ objects sparse1 2023-09-23T23:44:07.116 INFO:tasks.workunit.client.0.smithi110.stderr:++ image=sparse1 2023-09-23T23:44:07.116 INFO:tasks.workunit.client.0.smithi110.stderr:+++ rbd info sparse1 2023-09-23T23:44:07.116 INFO:tasks.workunit.client.0.smithi110.stderr:+++ grep block_name_prefix 2023-09-23T23:44:07.116 INFO:tasks.workunit.client.0.smithi110.stderr:+++ awk '{print $NF;}' 2023-09-23T23:44:07.151 INFO:tasks.workunit.client.0.smithi110.stderr:++ prefix=rbd_data.bb49fbbf69e8 2023-09-23T23:44:07.152 INFO:tasks.workunit.client.0.smithi110.stderr:+++ grep rbd_data.bb49fbbf69e8 2023-09-23T23:44:07.152 INFO:tasks.workunit.client.0.smithi110.stderr:++++ get_image_data_pool sparse1 2023-09-23T23:44:07.152 INFO:tasks.workunit.client.0.smithi110.stderr:++++ image=sparse1 2023-09-23T23:44:07.152 INFO:tasks.workunit.client.0.smithi110.stderr:+++ sort -u 2023-09-23T23:44:07.152 INFO:tasks.workunit.client.0.smithi110.stderr:+++ sed -e 's/rbd_data.bb49fbbf69e8\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T23:44:07.154 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ rbd info sparse1 2023-09-23T23:44:07.154 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T23:44:07.154 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ grep 'data_pool: ' 2023-09-23T23:44:07.190 INFO:tasks.workunit.client.0.smithi110.stderr:++++ data_pool= 2023-09-23T23:44:07.190 INFO:tasks.workunit.client.0.smithi110.stderr:++++ '[' -z ']' 2023-09-23T23:44:07.190 INFO:tasks.workunit.client.0.smithi110.stderr:++++ data_pool=rbd 2023-09-23T23:44:07.190 INFO:tasks.workunit.client.0.smithi110.stderr:++++ echo rbd 2023-09-23T23:44:07.190 INFO:tasks.workunit.client.0.smithi110.stderr:+++ rados ls -p rbd 2023-09-23T23:44:07.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:06 smithi138 ceph-mon[193606]: failed to encode map e714 with expected crc 2023-09-23T23:44:07.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:06 smithi138 ceph-mon[193606]: failed to encode map e714 with expected crc 2023-09-23T23:44:07.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:06 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2346379897' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T23:44:07.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:06 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/454685645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:44:07.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:06 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2346379897' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T23:44:07.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:06 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/776505097' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:44:07.220 INFO:tasks.workunit.client.0.smithi110.stderr:++ objects=1 2023-09-23T23:44:07.220 INFO:tasks.workunit.client.0.smithi110.stderr:++ echo 1 2023-09-23T23:44:07.221 INFO:tasks.workunit.client.0.smithi110.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T23:44:07.221 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd export sparse1 /tmp/rbd_import_export_695376/sparse1.out 2023-09-23T23:44:07.354 INFO:tasks.workunit.client.0.smithi110.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-23T23:44:07.357 INFO:tasks.workunit.client.0.smithi110.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_695376/sparse1 /tmp/rbd_import_export_695376/sparse1.out 2023-09-23T23:44:07.358 INFO:tasks.workunit.client.0.smithi110.stderr:+ cmp -l /tmp/rbd_import_export_695376/sparse1 /tmp/rbd_import_export_695376/sparse1.out 2023-09-23T23:44:07.365 INFO:tasks.workunit.client.0.smithi110.stderr:++ stat /tmp/rbd_import_export_695376/sparse1 --format %b 2023-09-23T23:44:07.366 INFO:tasks.workunit.client.0.smithi110.stderr:+ origsize=2048 2023-09-23T23:44:07.366 INFO:tasks.workunit.client.0.smithi110.stderr:++ stat /tmp/rbd_import_export_695376/sparse1.out --format %b 2023-09-23T23:44:07.367 INFO:tasks.workunit.client.0.smithi110.stderr:+ exportsize=2048 2023-09-23T23:44:07.367 INFO:tasks.workunit.client.0.smithi110.stderr:+ difference=0 2023-09-23T23:44:07.367 INFO:tasks.workunit.client.0.smithi110.stderr:+ difference=0 2023-09-23T23:44:07.367 INFO:tasks.workunit.client.0.smithi110.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T23:44:07.367 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm /tmp/rbd_import_export_695376/sparse1.out 2023-09-23T23:44:07.368 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd rm sparse1 2023-09-23T23:44:07.453 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T23:44:07.456 INFO:tasks.workunit.client.0.smithi110.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_695376/sparse2 oflag=append conv=notrunc 2023-09-23T23:44:07.467 INFO:tasks.workunit.client.0.smithi110.stderr:1+0 records in 2023-09-23T23:44:07.467 INFO:tasks.workunit.client.0.smithi110.stderr:1+0 records out 2023-09-23T23:44:07.467 INFO:tasks.workunit.client.0.smithi110.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00991515 s, 212 MB/s 2023-09-23T23:44:07.467 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-23T23:44:07.542 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 100% complete...done. 2023-09-23T23:44:07.543 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:44:07.543 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T23:44:07.545 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd ls -l 2023-09-23T23:44:07.546 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep sparse2 2023-09-23T23:44:07.546 INFO:tasks.workunit.client.0.smithi110.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-09-23T23:44:07.581 INFO:tasks.workunit.client.0.smithi110.stdout:sparse2 4 MiB 2 2023-09-23T23:44:07.582 INFO:tasks.workunit.client.0.smithi110.stderr:++ objects sparse2 2023-09-23T23:44:07.582 INFO:tasks.workunit.client.0.smithi110.stderr:++ image=sparse2 2023-09-23T23:44:07.582 INFO:tasks.workunit.client.0.smithi110.stderr:+++ rbd info sparse2 2023-09-23T23:44:07.582 INFO:tasks.workunit.client.0.smithi110.stderr:+++ grep block_name_prefix 2023-09-23T23:44:07.582 INFO:tasks.workunit.client.0.smithi110.stderr:+++ awk '{print $NF;}' 2023-09-23T23:44:07.618 INFO:tasks.workunit.client.0.smithi110.stderr:++ prefix=rbd_data.94444472da1a 2023-09-23T23:44:07.619 INFO:tasks.workunit.client.0.smithi110.stderr:+++ grep rbd_data.94444472da1a 2023-09-23T23:44:07.619 INFO:tasks.workunit.client.0.smithi110.stderr:++++ get_image_data_pool sparse2 2023-09-23T23:44:07.619 INFO:tasks.workunit.client.0.smithi110.stderr:++++ image=sparse2 2023-09-23T23:44:07.619 INFO:tasks.workunit.client.0.smithi110.stderr:+++ sed -e 's/rbd_data.94444472da1a\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T23:44:07.620 INFO:tasks.workunit.client.0.smithi110.stderr:+++ sort -u 2023-09-23T23:44:07.620 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ rbd info sparse2 2023-09-23T23:44:07.620 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ grep 'data_pool: ' 2023-09-23T23:44:07.620 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T23:44:07.656 INFO:tasks.workunit.client.0.smithi110.stderr:++++ data_pool= 2023-09-23T23:44:07.656 INFO:tasks.workunit.client.0.smithi110.stderr:++++ '[' -z ']' 2023-09-23T23:44:07.656 INFO:tasks.workunit.client.0.smithi110.stderr:++++ data_pool=rbd 2023-09-23T23:44:07.656 INFO:tasks.workunit.client.0.smithi110.stderr:++++ echo rbd 2023-09-23T23:44:07.656 INFO:tasks.workunit.client.0.smithi110.stderr:+++ rados ls -p rbd 2023-09-23T23:44:07.689 INFO:tasks.workunit.client.0.smithi110.stderr:++ objects='0 2023-09-23T23:44:07.689 INFO:tasks.workunit.client.0.smithi110.stderr:2 2023-09-23T23:44:07.689 INFO:tasks.workunit.client.0.smithi110.stderr:3' 2023-09-23T23:44:07.689 INFO:tasks.workunit.client.0.smithi110.stderr:++ echo 0 2 3 2023-09-23T23:44:07.689 INFO:tasks.workunit.client.0.smithi110.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-09-23T23:44:07.689 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd export sparse2 /tmp/rbd_import_export_695376/sparse2.out 2023-09-23T23:44:07.725 INFO:tasks.workunit.client.0.smithi110.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-23T23:44:07.728 INFO:tasks.workunit.client.0.smithi110.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_695376/sparse2 /tmp/rbd_import_export_695376/sparse2.out 2023-09-23T23:44:07.728 INFO:tasks.workunit.client.0.smithi110.stderr:+ cmp -l /tmp/rbd_import_export_695376/sparse2 /tmp/rbd_import_export_695376/sparse2.out 2023-09-23T23:44:07.731 INFO:tasks.workunit.client.0.smithi110.stderr:++ stat /tmp/rbd_import_export_695376/sparse2 --format %b 2023-09-23T23:44:07.732 INFO:tasks.workunit.client.0.smithi110.stderr:+ origsize=6144 2023-09-23T23:44:07.733 INFO:tasks.workunit.client.0.smithi110.stderr:++ stat /tmp/rbd_import_export_695376/sparse2.out --format %b 2023-09-23T23:44:07.734 INFO:tasks.workunit.client.0.smithi110.stderr:+ exportsize=6144 2023-09-23T23:44:07.734 INFO:tasks.workunit.client.0.smithi110.stderr:+ difference=0 2023-09-23T23:44:07.734 INFO:tasks.workunit.client.0.smithi110.stderr:+ difference=0 2023-09-23T23:44:07.734 INFO:tasks.workunit.client.0.smithi110.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T23:44:07.734 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm /tmp/rbd_import_export_695376/sparse2.out 2023-09-23T23:44:07.735 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd rm sparse2 2023-09-23T23:44:07.818 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-23T23:44:07.821 INFO:tasks.workunit.client.0.smithi110.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-09-23T23:44:07.822 INFO:tasks.workunit.client.0.smithi110.stdout:partially-sparse file imports to partially-sparse image 2023-09-23T23:44:07.822 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_695376/sparse1 sparse 2023-09-23T23:44:07.882 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-23T23:44:07.883 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:44:07.883 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T23:44:07.886 INFO:tasks.workunit.client.0.smithi110.stderr:++ objects sparse 2023-09-23T23:44:07.886 INFO:tasks.workunit.client.0.smithi110.stderr:++ image=sparse 2023-09-23T23:44:07.887 INFO:tasks.workunit.client.0.smithi110.stderr:+++ rbd info sparse 2023-09-23T23:44:07.887 INFO:tasks.workunit.client.0.smithi110.stderr:+++ grep block_name_prefix 2023-09-23T23:44:07.887 INFO:tasks.workunit.client.0.smithi110.stderr:+++ awk '{print $NF;}' 2023-09-23T23:44:07.922 INFO:tasks.workunit.client.0.smithi110.stderr:++ prefix=rbd_data.9471508d9fa2 2023-09-23T23:44:07.924 INFO:tasks.workunit.client.0.smithi110.stderr:+++ grep rbd_data.9471508d9fa2 2023-09-23T23:44:07.924 INFO:tasks.workunit.client.0.smithi110.stderr:++++ get_image_data_pool sparse 2023-09-23T23:44:07.924 INFO:tasks.workunit.client.0.smithi110.stderr:++++ image=sparse 2023-09-23T23:44:07.924 INFO:tasks.workunit.client.0.smithi110.stderr:+++ sed -e 's/rbd_data.9471508d9fa2\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T23:44:07.924 INFO:tasks.workunit.client.0.smithi110.stderr:+++ sort -u 2023-09-23T23:44:07.924 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ rbd info sparse 2023-09-23T23:44:07.924 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ grep 'data_pool: ' 2023-09-23T23:44:07.924 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T23:44:07.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:07 smithi110 ceph-mon[212939]: pgmap v2001: 103 pgs: 103 active+clean; 802 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 126 KiB/s rd, 13 KiB/s wr, 170 op/s 2023-09-23T23:44:07.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:07 smithi110 ceph-mon[212939]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:44:07.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:07 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/776505097' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:44:07.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:07 smithi110 ceph-mon[212939]: osdmap e715: 8 total, 7 up, 6 in 2023-09-23T23:44:07.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:07 smithi110 ceph-mon[212939]: failed to encode map e715 with expected crc 2023-09-23T23:44:07.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:07 smithi110 ceph-mon[212939]: failed to encode map e715 with expected crc 2023-09-23T23:44:07.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:07 smithi110 ceph-mon[212939]: failed to encode map e715 with expected crc 2023-09-23T23:44:07.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:07 smithi110 ceph-mon[212939]: failed to encode map e715 with expected crc 2023-09-23T23:44:07.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:07 smithi110 ceph-mon[221102]: pgmap v2001: 103 pgs: 103 active+clean; 802 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 126 KiB/s rd, 13 KiB/s wr, 170 op/s 2023-09-23T23:44:07.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:07 smithi110 ceph-mon[221102]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:44:07.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:07 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/776505097' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:44:07.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:07 smithi110 ceph-mon[221102]: osdmap e715: 8 total, 7 up, 6 in 2023-09-23T23:44:07.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:07 smithi110 ceph-mon[221102]: failed to encode map e715 with expected crc 2023-09-23T23:44:07.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:07 smithi110 ceph-mon[221102]: failed to encode map e715 with expected crc 2023-09-23T23:44:07.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:07 smithi110 ceph-mon[221102]: failed to encode map e715 with expected crc 2023-09-23T23:44:07.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:07 smithi110 ceph-mon[221102]: failed to encode map e715 with expected crc 2023-09-23T23:44:07.959 INFO:tasks.workunit.client.0.smithi110.stderr:++++ data_pool= 2023-09-23T23:44:07.959 INFO:tasks.workunit.client.0.smithi110.stderr:++++ '[' -z ']' 2023-09-23T23:44:07.960 INFO:tasks.workunit.client.0.smithi110.stderr:++++ data_pool=rbd 2023-09-23T23:44:07.960 INFO:tasks.workunit.client.0.smithi110.stderr:++++ echo rbd 2023-09-23T23:44:07.960 INFO:tasks.workunit.client.0.smithi110.stderr:+++ rados ls -p rbd 2023-09-23T23:44:07.993 INFO:tasks.workunit.client.0.smithi110.stderr:++ objects=1 2023-09-23T23:44:07.993 INFO:tasks.workunit.client.0.smithi110.stderr:++ echo 1 2023-09-23T23:44:07.993 INFO:tasks.workunit.client.0.smithi110.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T23:44:07.993 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd rm sparse 2023-09-23T23:44:08.078 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T23:44:08.081 INFO:tasks.workunit.client.0.smithi110.stderr:+ echo 'zeros import through stdin to sparse image' 2023-09-23T23:44:08.081 INFO:tasks.workunit.client.0.smithi110.stdout:zeros import through stdin to sparse image 2023-09-23T23:44:08.082 INFO:tasks.workunit.client.0.smithi110.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-23T23:44:08.082 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd import --new-format - sparse 2023-09-23T23:44:08.128 INFO:tasks.workunit.client.0.smithi110.stderr:4+0 records in 2023-09-23T23:44:08.128 INFO:tasks.workunit.client.0.smithi110.stderr:4+0 records out 2023-09-23T23:44:08.128 INFO:tasks.workunit.client.0.smithi110.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0454594 s, 92.3 MB/s 2023-09-23T23:44:08.138 INFO:tasks.workunit.client.0.smithi110.stderr: Importing image: 100% complete...done. 2023-09-23T23:44:08.139 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:44:08.141 INFO:tasks.workunit.client.0.smithi110.stderr:++ objects sparse 2023-09-23T23:44:08.141 INFO:tasks.workunit.client.0.smithi110.stderr:++ image=sparse 2023-09-23T23:44:08.142 INFO:tasks.workunit.client.0.smithi110.stderr:+++ rbd info sparse 2023-09-23T23:44:08.142 INFO:tasks.workunit.client.0.smithi110.stderr:+++ grep block_name_prefix 2023-09-23T23:44:08.142 INFO:tasks.workunit.client.0.smithi110.stderr:+++ awk '{print $NF;}' 2023-09-23T23:44:08.178 INFO:tasks.workunit.client.0.smithi110.stderr:++ prefix=rbd_data.949280edd016 2023-09-23T23:44:08.179 INFO:tasks.workunit.client.0.smithi110.stderr:+++ grep rbd_data.949280edd016 2023-09-23T23:44:08.179 INFO:tasks.workunit.client.0.smithi110.stderr:++++ get_image_data_pool sparse 2023-09-23T23:44:08.179 INFO:tasks.workunit.client.0.smithi110.stderr:++++ image=sparse 2023-09-23T23:44:08.179 INFO:tasks.workunit.client.0.smithi110.stderr:+++ sed -e 's/rbd_data.949280edd016\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T23:44:08.179 INFO:tasks.workunit.client.0.smithi110.stderr:+++ sort -u 2023-09-23T23:44:08.179 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ rbd info sparse 2023-09-23T23:44:08.179 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ grep 'data_pool: ' 2023-09-23T23:44:08.179 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T23:44:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:07 smithi138 ceph-mon[193606]: pgmap v2001: 103 pgs: 103 active+clean; 802 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 126 KiB/s rd, 13 KiB/s wr, 170 op/s 2023-09-23T23:44:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:07 smithi138 ceph-mon[193606]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:44:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:07 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/776505097' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:44:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:07 smithi138 ceph-mon[193606]: osdmap e715: 8 total, 7 up, 6 in 2023-09-23T23:44:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:07 smithi138 ceph-mon[193606]: failed to encode map e715 with expected crc 2023-09-23T23:44:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:07 smithi138 ceph-mon[193606]: failed to encode map e715 with expected crc 2023-09-23T23:44:08.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:07 smithi138 ceph-mon[193606]: failed to encode map e715 with expected crc 2023-09-23T23:44:08.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:07 smithi138 ceph-mon[193606]: failed to encode map e715 with expected crc 2023-09-23T23:44:08.194 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:44:08.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:08.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:44:08.216 INFO:tasks.workunit.client.0.smithi110.stderr:++++ data_pool= 2023-09-23T23:44:08.216 INFO:tasks.workunit.client.0.smithi110.stderr:++++ '[' -z ']' 2023-09-23T23:44:08.216 INFO:tasks.workunit.client.0.smithi110.stderr:++++ data_pool=rbd 2023-09-23T23:44:08.216 INFO:tasks.workunit.client.0.smithi110.stderr:++++ echo rbd 2023-09-23T23:44:08.216 INFO:tasks.workunit.client.0.smithi110.stderr:+++ rados ls -p rbd 2023-09-23T23:44:08.249 INFO:tasks.workunit.client.0.smithi110.stderr:++ objects= 2023-09-23T23:44:08.249 INFO:tasks.workunit.client.0.smithi110.stderr:++ echo 2023-09-23T23:44:08.249 INFO:tasks.workunit.client.0.smithi110.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-09-23T23:44:08.249 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd rm sparse 2023-09-23T23:44:08.299 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:44:08.328 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:08.577 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 100% complete...done. 2023-09-23T23:44:08.581 INFO:tasks.workunit.client.0.smithi110.stderr:+ echo 'zeros export to sparse file' 2023-09-23T23:44:08.581 INFO:tasks.workunit.client.0.smithi110.stdout:zeros export to sparse file 2023-09-23T23:44:08.581 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd create --new-format sparse --size 4 2023-09-23T23:44:08.619 INFO:tasks.workunit.client.0.smithi110.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:44:08.622 INFO:tasks.workunit.client.0.smithi110.stderr:++ rbd info sparse 2023-09-23T23:44:08.622 INFO:tasks.workunit.client.0.smithi110.stderr:++ grep block_name_prefix 2023-09-23T23:44:08.622 INFO:tasks.workunit.client.0.smithi110.stderr:++ awk '{print $NF;}' 2023-09-23T23:44:08.658 INFO:tasks.workunit.client.0.smithi110.stderr:+ prefix=rbd_data.94b38896dfdf 2023-09-23T23:44:08.658 INFO:tasks.workunit.client.0.smithi110.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-23T23:44:08.658 INFO:tasks.workunit.client.0.smithi110.stderr:++ get_image_data_pool sparse 2023-09-23T23:44:08.658 INFO:tasks.workunit.client.0.smithi110.stderr:++ image=sparse 2023-09-23T23:44:08.659 INFO:tasks.workunit.client.0.smithi110.stderr:+++ rbd info sparse 2023-09-23T23:44:08.659 INFO:tasks.workunit.client.0.smithi110.stderr:+++ grep 'data_pool: ' 2023-09-23T23:44:08.659 INFO:tasks.workunit.client.0.smithi110.stderr:+++ awk -F: '{ print $NF }' 2023-09-23T23:44:08.694 INFO:tasks.workunit.client.0.smithi110.stderr:++ data_pool= 2023-09-23T23:44:08.694 INFO:tasks.workunit.client.0.smithi110.stderr:++ '[' -z ']' 2023-09-23T23:44:08.694 INFO:tasks.workunit.client.0.smithi110.stderr:++ data_pool=rbd 2023-09-23T23:44:08.695 INFO:tasks.workunit.client.0.smithi110.stderr:++ echo rbd 2023-09-23T23:44:08.695 INFO:tasks.workunit.client.0.smithi110.stderr:+ rados -p rbd put rbd_data.94b38896dfdf.000000000000 - 2023-09-23T23:44:08.723 INFO:tasks.workunit.client.0.smithi110.stderr:1+0 records in 2023-09-23T23:44:08.723 INFO:tasks.workunit.client.0.smithi110.stderr:1+0 records out 2023-09-23T23:44:08.724 INFO:tasks.workunit.client.0.smithi110.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0650262 s, 64.5 MB/s 2023-09-23T23:44:08.739 INFO:tasks.workunit.client.0.smithi110.stderr:++ objects sparse 2023-09-23T23:44:08.739 INFO:tasks.workunit.client.0.smithi110.stderr:++ image=sparse 2023-09-23T23:44:08.740 INFO:tasks.workunit.client.0.smithi110.stderr:+++ rbd info sparse 2023-09-23T23:44:08.740 INFO:tasks.workunit.client.0.smithi110.stderr:+++ grep block_name_prefix 2023-09-23T23:44:08.740 INFO:tasks.workunit.client.0.smithi110.stderr:+++ awk '{print $NF;}' 2023-09-23T23:44:08.775 INFO:tasks.workunit.client.0.smithi110.stderr:++ prefix=rbd_data.94b38896dfdf 2023-09-23T23:44:08.777 INFO:tasks.workunit.client.0.smithi110.stderr:+++ grep rbd_data.94b38896dfdf 2023-09-23T23:44:08.777 INFO:tasks.workunit.client.0.smithi110.stderr:++++ get_image_data_pool sparse 2023-09-23T23:44:08.777 INFO:tasks.workunit.client.0.smithi110.stderr:+++ sed -e 's/rbd_data.94b38896dfdf\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T23:44:08.777 INFO:tasks.workunit.client.0.smithi110.stderr:++++ image=sparse 2023-09-23T23:44:08.777 INFO:tasks.workunit.client.0.smithi110.stderr:+++ sort -u 2023-09-23T23:44:08.777 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ rbd info sparse 2023-09-23T23:44:08.777 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ grep 'data_pool: ' 2023-09-23T23:44:08.777 INFO:tasks.workunit.client.0.smithi110.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T23:44:08.813 INFO:tasks.workunit.client.0.smithi110.stderr:++++ data_pool= 2023-09-23T23:44:08.813 INFO:tasks.workunit.client.0.smithi110.stderr:++++ '[' -z ']' 2023-09-23T23:44:08.813 INFO:tasks.workunit.client.0.smithi110.stderr:++++ data_pool=rbd 2023-09-23T23:44:08.813 INFO:tasks.workunit.client.0.smithi110.stderr:++++ echo rbd 2023-09-23T23:44:08.813 INFO:tasks.workunit.client.0.smithi110.stderr:+++ rados ls -p rbd 2023-09-23T23:44:08.846 INFO:tasks.workunit.client.0.smithi110.stderr:++ objects=0 2023-09-23T23:44:08.847 INFO:tasks.workunit.client.0.smithi110.stderr:++ echo 0 2023-09-23T23:44:08.847 INFO:tasks.workunit.client.0.smithi110.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T23:44:08.847 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm /tmp/rbd_import_export_695376/sparse 2023-09-23T23:44:08.847 INFO:tasks.workunit.client.0.smithi110.stderr:rm: cannot remove '/tmp/rbd_import_export_695376/sparse': No such file or directory 2023-09-23T23:44:08.848 INFO:tasks.workunit.client.0.smithi110.stderr:+ true 2023-09-23T23:44:08.848 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd export sparse /tmp/rbd_import_export_695376/sparse 2023-09-23T23:44:08.876 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:44:08.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:08.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:44:08.896 INFO:tasks.workunit.client.0.smithi110.stderr: Exporting image: 100% complete...done. 2023-09-23T23:44:08.901 INFO:tasks.workunit.client.0.smithi110.stderr:++ stat /tmp/rbd_import_export_695376/sparse --format=%b 2023-09-23T23:44:08.902 INFO:tasks.workunit.client.0.smithi110.stderr:+ '[' 0 = 0 ']' 2023-09-23T23:44:08.902 INFO:tasks.workunit.client.0.smithi110.stderr:+ rbd rm sparse 2023-09-23T23:44:08.978 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T23:44:08.979 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T23:44:08.983 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:44:09.018 INFO:tasks.workunit.client.0.smithi110.stderr: Removing image: 100% complete...done. 2023-09-23T23:44:09.021 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm /tmp/rbd_import_export_695376/sparse /tmp/rbd_import_export_695376/sparse1 /tmp/rbd_import_export_695376/sparse2 /tmp/rbd_import_export_695376/sparse3 2023-09-23T23:44:09.023 INFO:tasks.workunit.client.0.smithi110.stderr:rm: cannot remove '/tmp/rbd_import_export_695376/sparse3': No such file or directory 2023-09-23T23:44:09.025 INFO:tasks.workunit.client.0.smithi110.stderr:+ true 2023-09-23T23:44:09.025 INFO:tasks.workunit.client.0.smithi110.stdout:OK 2023-09-23T23:44:09.026 INFO:tasks.workunit.client.0.smithi110.stderr:+ echo OK 2023-09-23T23:44:09.026 INFO:tasks.workunit.client.0.smithi110.stderr:+ rm -rf /tmp/rbd_import_export_695376 2023-09-23T23:44:09.026 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T23:44:09.026 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T23:44:09.045 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:09.075 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-23T23:44:09.075 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T23:44:09.356 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:44:09.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:09.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:44:09.451 DEBUG:teuthology.parallel:result is None 2023-09-23T23:44:09.451 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T23:44:09.460 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:44:09.479 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T23:44:09.480 DEBUG:teuthology.orchestra.run.smithi110:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T23:44:09.547 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T23:44:09.548 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T23:44:09.548 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-23T23:44:09.548 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T23:44:09.548 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T23:44:09.548 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T23:44:09.549 DEBUG:teuthology.orchestra.run.smithi110:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T23:44:09.549 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:44:09.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T23:44:09.606 INFO:teuthology.orchestra.run.smithi110.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T23:44:09.606 DEBUG:teuthology.orchestra.run.smithi110:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T23:44:09.644 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:44:09.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:44:09.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:44:09.674 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T23:44:09.674 DEBUG:teuthology.orchestra.run.smithi110:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T23:44:09.703 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:44:09.704 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:44:09.704 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:44:09.704 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:44:09.704 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:44:09.720 INFO:tasks.thrashosds:joining thrashosds 2023-09-23T23:44:10.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:09 smithi138 ceph-mon[193606]: pgmap v2003: 103 pgs: 103 active+clean; 802 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 338 KiB/s rd, 417 KiB/s wr, 188 op/s 2023-09-23T23:44:10.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:09 smithi138 ceph-mon[193606]: failed to encode map e716 with expected crc 2023-09-23T23:44:10.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:09 smithi138 ceph-mon[193606]: osdmap e716: 8 total, 7 up, 6 in 2023-09-23T23:44:10.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:09 smithi110 ceph-mon[212939]: pgmap v2003: 103 pgs: 103 active+clean; 802 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 338 KiB/s rd, 417 KiB/s wr, 188 op/s 2023-09-23T23:44:10.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:09 smithi110 ceph-mon[212939]: failed to encode map e716 with expected crc 2023-09-23T23:44:10.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:09 smithi110 ceph-mon[212939]: osdmap e716: 8 total, 7 up, 6 in 2023-09-23T23:44:10.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:09 smithi110 ceph-mon[221102]: pgmap v2003: 103 pgs: 103 active+clean; 802 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 338 KiB/s rd, 417 KiB/s wr, 188 op/s 2023-09-23T23:44:10.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:09 smithi110 ceph-mon[221102]: failed to encode map e716 with expected crc 2023-09-23T23:44:10.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:09 smithi110 ceph-mon[221102]: osdmap e716: 8 total, 7 up, 6 in 2023-09-23T23:44:10.599 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:44:10.603 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:44:10.749 INFO:teuthology.orchestra.run.smithi110.stdout:moved 3 / 309 (0.970874%) 2023-09-23T23:44:10.750 INFO:teuthology.orchestra.run.smithi110.stdout:avg 51.5 2023-09-23T23:44:10.750 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 5.56028 -> 5.79511 (expected baseline 6.55108) 2023-09-23T23:44:10.750 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.2 with 40 -> 39 pgs (0.776699 -> 0.757282 * mean) 2023-09-23T23:44:10.750 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.0 with 57 -> 56 pgs (1.1068 -> 1.08738 * mean) 2023-09-23T23:44:10.750 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:44:10.750 INFO:teuthology.orchestra.run.smithi110.stdout:oload 110 2023-09-23T23:44:10.750 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 0.05 2023-09-23T23:44:10.750 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:44:10.750 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0107 2023-09-23T23:44:10.750 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.0118 2023-09-23T23:44:10.750 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 weight 1.0000 -> 0.9500 2023-09-23T23:44:10.750 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 weight 1.0000 -> 0.9500 2023-09-23T23:44:10.926 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:10 smithi110 ceph-mon[212939]: failed to encode map e716 with expected crc 2023-09-23T23:44:10.927 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:10 smithi110 ceph-mon[212939]: failed to encode map e716 with expected crc 2023-09-23T23:44:10.927 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:10 smithi110 ceph-mon[212939]: from='client.38127 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:44:10.927 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:10 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260'}"}]: dispatch 2023-09-23T23:44:10.927 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:10 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260'}"}]: dispatch 2023-09-23T23:44:10.927 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:10 smithi110 ceph-mon[221102]: failed to encode map e716 with expected crc 2023-09-23T23:44:10.927 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:10 smithi110 ceph-mon[221102]: failed to encode map e716 with expected crc 2023-09-23T23:44:10.927 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:10 smithi110 ceph-mon[221102]: from='client.38127 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:44:10.927 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:10 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260'}"}]: dispatch 2023-09-23T23:44:10.928 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:10 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260'}"}]: dispatch 2023-09-23T23:44:11.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:10 smithi138 ceph-mon[193606]: failed to encode map e716 with expected crc 2023-09-23T23:44:11.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:10 smithi138 ceph-mon[193606]: failed to encode map e716 with expected crc 2023-09-23T23:44:11.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:10 smithi138 ceph-mon[193606]: from='client.38127 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:44:11.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:10 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260'}"}]: dispatch 2023-09-23T23:44:11.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:10 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260'}"}]: dispatch 2023-09-23T23:44:11.403 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T23:44:11.404 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T23:44:11.487 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:11.540 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:11.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:11 smithi110 ceph-mon[212939]: pgmap v2005: 103 pgs: 103 active+clean; 582 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 726 KiB/s rd, 695 KiB/s wr, 283 op/s 2023-09-23T23:44:11.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:11 smithi110 ceph-mon[212939]: failed to encode map e717 with expected crc 2023-09-23T23:44:11.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:11 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260'}"}]': finished 2023-09-23T23:44:11.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:11 smithi110 ceph-mon[212939]: failed to encode map e717 with expected crc 2023-09-23T23:44:11.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:11 smithi110 ceph-mon[212939]: osdmap e717: 8 total, 7 up, 6 in 2023-09-23T23:44:11.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:11 smithi110 ceph-mon[221102]: pgmap v2005: 103 pgs: 103 active+clean; 582 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 726 KiB/s rd, 695 KiB/s wr, 283 op/s 2023-09-23T23:44:11.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:11 smithi110 ceph-mon[221102]: failed to encode map e717 with expected crc 2023-09-23T23:44:11.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:11 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260'}"}]': finished 2023-09-23T23:44:11.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:11 smithi110 ceph-mon[221102]: failed to encode map e717 with expected crc 2023-09-23T23:44:11.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:11 smithi110 ceph-mon[221102]: osdmap e717: 8 total, 7 up, 6 in 2023-09-23T23:44:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:11 smithi138 ceph-mon[193606]: pgmap v2005: 103 pgs: 103 active+clean; 582 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 726 KiB/s rd, 695 KiB/s wr, 283 op/s 2023-09-23T23:44:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:11 smithi138 ceph-mon[193606]: failed to encode map e717 with expected crc 2023-09-23T23:44:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:11 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260'}"}]': finished 2023-09-23T23:44:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:11 smithi138 ceph-mon[193606]: failed to encode map e717 with expected crc 2023-09-23T23:44:12.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:11 smithi138 ceph-mon[193606]: osdmap e717: 8 total, 7 up, 6 in 2023-09-23T23:44:13.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:12 smithi138 ceph-mon[193606]: failed to encode map e718 with expected crc 2023-09-23T23:44:13.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:12 smithi138 ceph-mon[193606]: osdmap e718: 8 total, 7 up, 6 in 2023-09-23T23:44:13.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:12 smithi138 ceph-mon[193606]: failed to encode map e718 with expected crc 2023-09-23T23:44:13.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:12 smithi138 ceph-mon[193606]: failed to encode map e718 with expected crc 2023-09-23T23:44:13.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:12 smithi110 ceph-mon[212939]: failed to encode map e718 with expected crc 2023-09-23T23:44:13.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:12 smithi110 ceph-mon[212939]: osdmap e718: 8 total, 7 up, 6 in 2023-09-23T23:44:13.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:12 smithi110 ceph-mon[212939]: failed to encode map e718 with expected crc 2023-09-23T23:44:13.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:12 smithi110 ceph-mon[212939]: failed to encode map e718 with expected crc 2023-09-23T23:44:13.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:12 smithi110 ceph-mon[221102]: failed to encode map e718 with expected crc 2023-09-23T23:44:13.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:12 smithi110 ceph-mon[221102]: osdmap e718: 8 total, 7 up, 6 in 2023-09-23T23:44:13.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:12 smithi110 ceph-mon[221102]: failed to encode map e718 with expected crc 2023-09-23T23:44:13.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:12 smithi110 ceph-mon[221102]: failed to encode map e718 with expected crc 2023-09-23T23:44:13.519 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:44:13.520 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:44:13.520 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:44:13.520 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:44:13.520 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.520 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:7674 14.8 e621/617 14:166d5947:::benchmark_data_smithi110_640334_object7673:head v 621'472, mlcod=621'470)", 2023-09-23T23:44:13.520 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:25.621719+0000", 2023-09-23T23:44:13.520 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 287.89398431699999, 2023-09-23T23:44:13.520 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.122259012, 2023-09-23T23:44:13.520 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.520 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.520 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.521 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.521 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.521 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.621719+0000", 2023-09-23T23:44:13.521 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.521 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.521 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.521 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.521 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.621719+0000", 2023-09-23T23:44:13.521 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.521 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.521 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.521 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.522 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.621717+0000", 2023-09-23T23:44:13.522 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:44:13.522 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.522 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.522 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.522 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.621746+0000", 2023-09-23T23:44:13.522 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9260000000000001e-05 2023-09-23T23:44:13.522 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.522 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.522 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.522 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.621760+0000", 2023-09-23T23:44:13.522 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3935e-05 2023-09-23T23:44:13.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.523 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.523 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.621768+0000", 2023-09-23T23:44:13.523 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2900000000000002e-06 2023-09-23T23:44:13.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.523 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.523 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.621824+0000", 2023-09-23T23:44:13.523 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5594999999999999e-05 2023-09-23T23:44:13.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.621840+0000", 2023-09-23T23:44:13.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5875999999999998e-05 2023-09-23T23:44:13.524 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743927+0000", 2023-09-23T23:44:13.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.122086954 2023-09-23T23:44:13.524 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743978+0000", 2023-09-23T23:44:13.525 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0899999999999997e-05 2023-09-23T23:44:13.525 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.525 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.525 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.525 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:7678 14.1 e621/617 14:8d451f58:::benchmark_data_smithi110_640334_object7677:head v 621'503, mlcod=621'499)", 2023-09-23T23:44:13.525 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:25.622495+0000", 2023-09-23T23:44:13.525 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 287.89320738800001, 2023-09-23T23:44:13.525 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.121498043, 2023-09-23T23:44:13.525 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.525 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.525 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.526 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.526 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.526 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622495+0000", 2023-09-23T23:44:13.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.526 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.526 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.526 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.526 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622495+0000", 2023-09-23T23:44:13.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.526 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.526 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.526 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622495+0000", 2023-09-23T23:44:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:44:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622530+0000", 2023-09-23T23:44:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5311000000000001e-05 2023-09-23T23:44:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622547+0000", 2023-09-23T23:44:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7088e-05 2023-09-23T23:44:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.528 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.528 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622555+0000", 2023-09-23T23:44:13.528 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6210000000000004e-06 2023-09-23T23:44:13.528 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.528 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.528 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622793+0000", 2023-09-23T23:44:13.528 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023872499999999999 2023-09-23T23:44:13.528 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.528 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622821+0000", 2023-09-23T23:44:13.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7518e-05 2023-09-23T23:44:13.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.529 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743951+0000", 2023-09-23T23:44:13.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.121129903 2023-09-23T23:44:13.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.529 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743993+0000", 2023-09-23T23:44:13.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2607000000000001e-05 2023-09-23T23:44:13.529 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.530 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.530 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.530 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.530 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.530 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:7680 14.f e621/617 14:fec04f59:::benchmark_data_smithi110_640334_object7679:head v 621'505, mlcod=621'502)", 2023-09-23T23:44:13.530 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:25.622573+0000", 2023-09-23T23:44:13.530 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 287.89312983799999, 2023-09-23T23:44:13.530 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.121410935, 2023-09-23T23:44:13.530 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.530 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.530 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.530 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622573+0000", 2023-09-23T23:44:13.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622573+0000", 2023-09-23T23:44:13.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622573+0000", 2023-09-23T23:44:13.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:13.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622603+0000", 2023-09-23T23:44:13.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0352e-05 2023-09-23T23:44:13.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622610+0000", 2023-09-23T23:44:13.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2370000000000003e-06 2023-09-23T23:44:13.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.533 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622615+0000", 2023-09-23T23:44:13.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0660000000000003e-06 2023-09-23T23:44:13.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.533 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.533 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622691+0000", 2023-09-23T23:44:13.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5326000000000004e-05 2023-09-23T23:44:13.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.533 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.533 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622713+0000", 2023-09-23T23:44:13.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2010000000000001e-05 2023-09-23T23:44:13.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.534 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743932+0000", 2023-09-23T23:44:13.534 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.1212198 2023-09-23T23:44:13.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.534 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743984+0000", 2023-09-23T23:44:13.534 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1487999999999999e-05 2023-09-23T23:44:13.534 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.534 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.535 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.535 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:7685 14.a e621/617 14:55b7902e:::benchmark_data_smithi110_640334_object7684:head v 621'455, mlcod=621'452)", 2023-09-23T23:44:13.535 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:25.622628+0000", 2023-09-23T23:44:13.535 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 287.89307527900002, 2023-09-23T23:44:13.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.121849387, 2023-09-23T23:44:13.535 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.535 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.535 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622628+0000", 2023-09-23T23:44:13.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622628+0000", 2023-09-23T23:44:13.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622627+0000", 2023-09-23T23:44:13.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:13.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622648+0000", 2023-09-23T23:44:13.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0874e-05 2023-09-23T23:44:13.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622653+0000", 2023-09-23T23:44:13.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.293e-06 2023-09-23T23:44:13.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622657+0000", 2023-09-23T23:44:13.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9310000000000001e-06 2023-09-23T23:44:13.538 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.538 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.538 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622781+0000", 2023-09-23T23:44:13.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012415700000000001 2023-09-23T23:44:13.538 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.538 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.538 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.622803+0000", 2023-09-23T23:44:13.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1413e-05 2023-09-23T23:44:13.538 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.539 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.539 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.744451+0000", 2023-09-23T23:44:13.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12164800000000001 2023-09-23T23:44:13.539 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.539 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.539 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.744477+0000", 2023-09-23T23:44:13.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6030999999999999e-05 2023-09-23T23:44:13.539 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.539 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.539 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.540 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.540 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.540 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:7690 14.2 e621/617 14:4a64c483:::benchmark_data_smithi110_640334_object7689:head v 621'469, mlcod=621'467)", 2023-09-23T23:44:13.540 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:39:25.624477+0000", 2023-09-23T23:44:13.540 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 287.89122627400002, 2023-09-23T23:44:13.540 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12058026299999999, 2023-09-23T23:44:13.540 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.540 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.540 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.540 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.540 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.540 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.624477+0000", 2023-09-23T23:44:13.540 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.541 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.541 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.624477+0000", 2023-09-23T23:44:13.541 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.541 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.541 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.624476+0000", 2023-09-23T23:44:13.541 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:44:13.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.624518+0000", 2023-09-23T23:44:13.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2144999999999997e-05 2023-09-23T23:44:13.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743899+0000", 2023-09-23T23:44:13.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.119381112 2023-09-23T23:44:13.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743910+0000", 2023-09-23T23:44:13.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0789e-05 2023-09-23T23:44:13.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743973+0000", 2023-09-23T23:44:13.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3007000000000002e-05 2023-09-23T23:44:13.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.743996+0000", 2023-09-23T23:44:13.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2906000000000001e-05 2023-09-23T23:44:13.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.544 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.745028+0000", 2023-09-23T23:44:13.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001032921 2023-09-23T23:44:13.544 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.544 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.544 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:39:25.745057+0000", 2023-09-23T23:44:13.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8388999999999999e-05 2023-09-23T23:44:13.544 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.544 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.544 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.544 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.545 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:76681 14.6 e636/617 14:66e04a98:::benchmark_data_smithi110_640334_object76680:head v 636'4880, mlcod=636'4877)", 2023-09-23T23:44:13.545 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:40:40.503861+0000", 2023-09-23T23:44:13.545 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 213.011841481, 2023-09-23T23:44:13.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.121150234, 2023-09-23T23:44:13.545 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.545 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.545 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.545 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.503861+0000", 2023-09-23T23:44:13.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.546 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.546 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.546 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.503861+0000", 2023-09-23T23:44:13.546 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.546 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.546 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.546 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.503861+0000", 2023-09-23T23:44:13.546 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:44:13.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.546 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.547 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.547 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.503886+0000", 2023-09-23T23:44:13.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5160999999999998e-05 2023-09-23T23:44:13.547 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.547 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.547 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.503898+0000", 2023-09-23T23:44:13.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2356e-05 2023-09-23T23:44:13.547 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.547 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.547 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.503905+0000", 2023-09-23T23:44:13.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0960000000000003e-06 2023-09-23T23:44:13.548 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.548 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.548 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.548 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.505169+0000", 2023-09-23T23:44:13.548 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012638390000000001 2023-09-23T23:44:13.548 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.548 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.548 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.548 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.614431+0000", 2023-09-23T23:44:13.548 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.109262383 2023-09-23T23:44:13.548 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.548 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.549 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.549 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.624959+0000", 2023-09-23T23:44:13.549 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010528087 2023-09-23T23:44:13.549 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.549 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.549 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.549 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:40.625012+0000", 2023-09-23T23:44:13.549 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2160999999999997e-05 2023-09-23T23:44:13.549 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.549 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.549 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.549 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.550 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:92258 14.3 e642/617 14:c938b5e2:::benchmark_data_smithi110_640334_object8379:head v 642'5671, mlcod=642'5669)", 2023-09-23T23:44:13.550 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:05.597280+0000", 2023-09-23T23:44:13.550 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 187.918422518, 2023-09-23T23:44:13.550 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.17337343499999999, 2023-09-23T23:44:13.550 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.550 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.550 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.550 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597280+0000", 2023-09-23T23:44:13.550 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597280+0000", 2023-09-23T23:44:13.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597279+0000", 2023-09-23T23:44:13.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:44:13.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597285+0000", 2023-09-23T23:44:13.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.144e-06 2023-09-23T23:44:13.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597298+0000", 2023-09-23T23:44:13.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2916e-05 2023-09-23T23:44:13.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597306+0000", 2023-09-23T23:44:13.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7090000000000002e-06 2023-09-23T23:44:13.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.553 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.553 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597435+0000", 2023-09-23T23:44:13.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000128498 2023-09-23T23:44:13.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.553 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.553 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597454+0000", 2023-09-23T23:44:13.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9933999999999999e-05 2023-09-23T23:44:13.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.554 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.554 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.770624+0000", 2023-09-23T23:44:13.554 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.173169447 2023-09-23T23:44:13.554 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.554 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.554 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.554 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.770654+0000", 2023-09-23T23:44:13.554 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9832000000000002e-05 2023-09-23T23:44:13.554 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.554 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.554 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.554 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.555 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:92266 14.1 e642/617 14:8946a29f:::benchmark_data_smithi110_640334_object8387:head v 642'5829, mlcod=642'5826)", 2023-09-23T23:44:13.555 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:05.597320+0000", 2023-09-23T23:44:13.555 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 187.91838274899999, 2023-09-23T23:44:13.555 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.173397724, 2023-09-23T23:44:13.555 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.555 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.555 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.555 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.555 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597320+0000", 2023-09-23T23:44:13.555 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.555 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.556 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597320+0000", 2023-09-23T23:44:13.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.556 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.556 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597320+0000", 2023-09-23T23:44:13.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:13.556 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.557 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597324+0000", 2023-09-23T23:44:13.557 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8029999999999998e-06 2023-09-23T23:44:13.557 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.557 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.557 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.557 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597329+0000", 2023-09-23T23:44:13.557 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1340000000000003e-06 2023-09-23T23:44:13.557 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.557 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.557 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.557 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597333+0000", 2023-09-23T23:44:13.557 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9169999999999999e-06 2023-09-23T23:44:13.558 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.558 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.558 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.762583+0000", 2023-09-23T23:44:13.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.16525079100000001 2023-09-23T23:44:13.558 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.558 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.558 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.762608+0000", 2023-09-23T23:44:13.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4746999999999999e-05 2023-09-23T23:44:13.558 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.770669+0000", 2023-09-23T23:44:13.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080605199999999998 2023-09-23T23:44:13.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.770718+0000", 2023-09-23T23:44:13.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9098000000000003e-05 2023-09-23T23:44:13.559 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.559 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.559 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.560 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:92264 14.b e642/617 14:d4f12831:::benchmark_data_smithi110_640334_object8385:head v 642'5716, mlcod=642'5714)", 2023-09-23T23:44:13.560 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:05.597352+0000", 2023-09-23T23:44:13.560 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 187.91835052900001, 2023-09-23T23:44:13.560 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.17333879499999999, 2023-09-23T23:44:13.560 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.560 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.560 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.560 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.560 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.560 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597352+0000", 2023-09-23T23:44:13.560 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.560 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.561 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.561 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.561 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597352+0000", 2023-09-23T23:44:13.561 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.561 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.561 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.561 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.561 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597352+0000", 2023-09-23T23:44:13.561 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:13.561 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.561 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.561 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.561 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597356+0000", 2023-09-23T23:44:13.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9859999999999996e-06 2023-09-23T23:44:13.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.562 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597361+0000", 2023-09-23T23:44:13.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.254e-06 2023-09-23T23:44:13.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.562 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.597365+0000", 2023-09-23T23:44:13.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0130000000000004e-06 2023-09-23T23:44:13.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.762579+0000", 2023-09-23T23:44:13.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.165213839 2023-09-23T23:44:13.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.762605+0000", 2023-09-23T23:44:13.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6086000000000001e-05 2023-09-23T23:44:13.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.770663+0000", 2023-09-23T23:44:13.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080581279999999995 2023-09-23T23:44:13.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.770691+0000", 2023-09-23T23:44:13.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7778999999999999e-05 2023-09-23T23:44:13.564 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.564 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.564 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.565 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:92267 14.6 e642/617 14:6427f456:::benchmark_data_smithi110_640334_object8388:head v 642'5896, mlcod=642'5894)", 2023-09-23T23:44:13.565 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:05.601294+0000", 2023-09-23T23:44:13.565 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 187.914408702, 2023-09-23T23:44:13.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.17304376499999999, 2023-09-23T23:44:13.565 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.565 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.565 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.565 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.565 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.601294+0000", 2023-09-23T23:44:13.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.566 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.566 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.566 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.601294+0000", 2023-09-23T23:44:13.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.566 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.566 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.566 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.566 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.601293+0000", 2023-09-23T23:44:13.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:44:13.566 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.566 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.566 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.566 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.601302+0000", 2023-09-23T23:44:13.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3989999999999993e-06 2023-09-23T23:44:13.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.567 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.567 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.762549+0000", 2023-09-23T23:44:13.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.16124624000000001 2023-09-23T23:44:13.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.567 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.567 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.762562+0000", 2023-09-23T23:44:13.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2853e-05 2023-09-23T23:44:13.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.762698+0000", 2023-09-23T23:44:13.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000136126 2023-09-23T23:44:13.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.762717+0000", 2023-09-23T23:44:13.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9259e-05 2023-09-23T23:44:13.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.774303+0000", 2023-09-23T23:44:13.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01158578 2023-09-23T23:44:13.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:05.774338+0000", 2023-09-23T23:44:13.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5163999999999997e-05 2023-09-23T23:44:13.569 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.569 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.569 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.570 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:92719 14.f e642/617 14:f97a3607:::benchmark_data_smithi110_640334_object8840:head v 642'5872, mlcod=642'5870)", 2023-09-23T23:44:13.570 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:06.360791+0000", 2023-09-23T23:44:13.570 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 187.15491157599999, 2023-09-23T23:44:13.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12632665400000001, 2023-09-23T23:44:13.570 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.570 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.570 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.570 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.360791+0000", 2023-09-23T23:44:13.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.360791+0000", 2023-09-23T23:44:13.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.360790+0000", 2023-09-23T23:44:13.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:44:13.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.360799+0000", 2023-09-23T23:44:13.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6319999999999997e-06 2023-09-23T23:44:13.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.360812+0000", 2023-09-23T23:44:13.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3040000000000001e-05 2023-09-23T23:44:13.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.360820+0000", 2023-09-23T23:44:13.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1780000000000003e-06 2023-09-23T23:44:13.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.360866+0000", 2023-09-23T23:44:13.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6371e-05 2023-09-23T23:44:13.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.360889+0000", 2023-09-23T23:44:13.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2673000000000001e-05 2023-09-23T23:44:13.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371548+0000", 2023-09-23T23:44:13.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010658978 2023-09-23T23:44:13.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.487118+0000", 2023-09-23T23:44:13.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.115569828 2023-09-23T23:44:13.574 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.574 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.574 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.574 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:92721 14.8 e642/617 14:17bf1845:::benchmark_data_smithi110_640334_object8842:head v 642'5875, mlcod=642'5873)", 2023-09-23T23:44:13.575 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:06.365471+0000", 2023-09-23T23:44:13.575 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 187.15023172400001, 2023-09-23T23:44:13.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12165229599999999, 2023-09-23T23:44:13.575 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.575 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.575 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.575 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.365471+0000", 2023-09-23T23:44:13.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.575 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.576 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.365471+0000", 2023-09-23T23:44:13.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.576 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.576 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.576 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.365471+0000", 2023-09-23T23:44:13.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:13.576 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.576 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.576 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.365475+0000", 2023-09-23T23:44:13.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9999999999999998e-06 2023-09-23T23:44:13.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.365481+0000", 2023-09-23T23:44:13.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8200000000000002e-06 2023-09-23T23:44:13.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.365485+0000", 2023-09-23T23:44:13.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.216e-06 2023-09-23T23:44:13.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.365560+0000", 2023-09-23T23:44:13.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5581999999999996e-05 2023-09-23T23:44:13.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.365581+0000", 2023-09-23T23:44:13.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0581000000000002e-05 2023-09-23T23:44:13.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371604+0000", 2023-09-23T23:44:13.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060231779999999997 2023-09-23T23:44:13.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.487123+0000", 2023-09-23T23:44:13.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.115519344 2023-09-23T23:44:13.579 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.579 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.579 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.579 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:92723 14.6 e642/617 14:6a0ee19f:::benchmark_data_smithi110_640334_object8844:head v 642'5921, mlcod=642'5919)", 2023-09-23T23:44:13.579 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:06.371410+0000", 2023-09-23T23:44:13.580 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 187.14429270700001, 2023-09-23T23:44:13.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.13841763800000001, 2023-09-23T23:44:13.580 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.580 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.580 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371410+0000", 2023-09-23T23:44:13.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371410+0000", 2023-09-23T23:44:13.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371409+0000", 2023-09-23T23:44:13.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:44:13.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371417+0000", 2023-09-23T23:44:13.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5179999999999998e-06 2023-09-23T23:44:13.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371426+0000", 2023-09-23T23:44:13.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6709999999999994e-06 2023-09-23T23:44:13.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.371433+0000", 2023-09-23T23:44:13.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.024e-06 2023-09-23T23:44:13.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.487124+0000", 2023-09-23T23:44:13.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.115690628 2023-09-23T23:44:13.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.487144+0000", 2023-09-23T23:44:13.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9564000000000001e-05 2023-09-23T23:44:13.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.509781+0000", 2023-09-23T23:44:13.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022637066000000001 2023-09-23T23:44:13.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:06.509828+0000", 2023-09-23T23:44:13.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7086000000000003e-05 2023-09-23T23:44:13.584 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.584 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.584 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.584 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:133600 14.f e665/617 14:f5f82a89:::benchmark_data_smithi110_640334_object49721:head v 665'8467, mlcod=665'8464)", 2023-09-23T23:44:13.584 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:42:14.374681+0000", 2023-09-23T23:44:13.584 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 119.14102228199999, 2023-09-23T23:44:13.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14911405, 2023-09-23T23:44:13.585 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.585 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.585 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.585 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.585 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374681+0000", 2023-09-23T23:44:13.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.585 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.585 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374681+0000", 2023-09-23T23:44:13.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374679+0000", 2023-09-23T23:44:13.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:44:13.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374688+0000", 2023-09-23T23:44:13.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8070000000000006e-06 2023-09-23T23:44:13.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374699+0000", 2023-09-23T23:44:13.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1062e-05 2023-09-23T23:44:13.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374708+0000", 2023-09-23T23:44:13.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4130000000000006e-06 2023-09-23T23:44:13.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374804+0000", 2023-09-23T23:44:13.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6218999999999995e-05 2023-09-23T23:44:13.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374833+0000", 2023-09-23T23:44:13.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8546000000000001e-05 2023-09-23T23:44:13.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.523754+0000", 2023-09-23T23:44:13.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.148921737 2023-09-23T23:44:13.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.523795+0000", 2023-09-23T23:44:13.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0355999999999998e-05 2023-09-23T23:44:13.589 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.589 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.589 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.589 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:133605 14.1 e665/617 14:8fb0bb53:::benchmark_data_smithi110_640334_object49726:head v 665'8376, mlcod=665'8374)", 2023-09-23T23:44:13.589 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:42:14.374722+0000", 2023-09-23T23:44:13.589 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 119.14098038500001, 2023-09-23T23:44:13.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.150819441, 2023-09-23T23:44:13.590 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.590 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.590 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374722+0000", 2023-09-23T23:44:13.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374722+0000", 2023-09-23T23:44:13.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374722+0000", 2023-09-23T23:44:13.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:13.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374726+0000", 2023-09-23T23:44:13.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4059999999999998e-06 2023-09-23T23:44:13.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374733+0000", 2023-09-23T23:44:13.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6610000000000003e-06 2023-09-23T23:44:13.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374738+0000", 2023-09-23T23:44:13.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1039999999999998e-06 2023-09-23T23:44:13.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374814+0000", 2023-09-23T23:44:13.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5872999999999997e-05 2023-09-23T23:44:13.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374835+0000", 2023-09-23T23:44:13.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0455e-05 2023-09-23T23:44:13.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.525504+0000", 2023-09-23T23:44:13.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.150668986 2023-09-23T23:44:13.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.525542+0000", 2023-09-23T23:44:13.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8352000000000001e-05 2023-09-23T23:44:13.594 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.594 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.594 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.594 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:133606 14.2 e665/645 14:40d905cb:::benchmark_data_smithi110_640334_object49727:head v 665'8330, mlcod=665'8328)", 2023-09-23T23:44:13.594 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:42:14.374749+0000", 2023-09-23T23:44:13.594 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 119.14095385500001, 2023-09-23T23:44:13.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.15799634400000001, 2023-09-23T23:44:13.595 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.595 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.595 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374749+0000", 2023-09-23T23:44:13.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374749+0000", 2023-09-23T23:44:13.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374749+0000", 2023-09-23T23:44:13.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-09-23T23:44:13.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374755+0000", 2023-09-23T23:44:13.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7519999999999999e-06 2023-09-23T23:44:13.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374761+0000", 2023-09-23T23:44:13.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8710000000000002e-06 2023-09-23T23:44:13.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374766+0000", 2023-09-23T23:44:13.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3839999999999999e-06 2023-09-23T23:44:13.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374813+0000", 2023-09-23T23:44:13.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7635e-05 2023-09-23T23:44:13.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374832+0000", 2023-09-23T23:44:13.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9068000000000001e-05 2023-09-23T23:44:13.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.532698+0000", 2023-09-23T23:44:13.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.157865433 2023-09-23T23:44:13.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.532745+0000", 2023-09-23T23:44:13.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7438999999999998e-05 2023-09-23T23:44:13.599 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.599 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.599 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.599 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:133607 14.a e665/617 14:5dffaf0f:::benchmark_data_smithi110_640334_object49728:head v 665'8148, mlcod=665'8145)", 2023-09-23T23:44:13.599 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:42:14.374777+0000", 2023-09-23T23:44:13.599 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 119.140926112, 2023-09-23T23:44:13.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.15591643599999999, 2023-09-23T23:44:13.600 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.600 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.600 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374777+0000", 2023-09-23T23:44:13.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374777+0000", 2023-09-23T23:44:13.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374776+0000", 2023-09-23T23:44:13.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:13.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374780+0000", 2023-09-23T23:44:13.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5669999999999999e-06 2023-09-23T23:44:13.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374786+0000", 2023-09-23T23:44:13.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5679999999999999e-06 2023-09-23T23:44:13.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.374791+0000", 2023-09-23T23:44:13.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.181e-06 2023-09-23T23:44:13.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.521162+0000", 2023-09-23T23:44:13.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14637107599999999 2023-09-23T23:44:13.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.521188+0000", 2023-09-23T23:44:13.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6205999999999999e-05 2023-09-23T23:44:13.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.530657+0000", 2023-09-23T23:44:13.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0094686200000000005 2023-09-23T23:44:13.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.530693+0000", 2023-09-23T23:44:13.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6495999999999998e-05 2023-09-23T23:44:13.604 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.604 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.604 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.604 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47186.0:133601 14.6 e665/617 14:69f9b4be:::benchmark_data_smithi110_640334_object49722:head v 665'8492, mlcod=665'8490)", 2023-09-23T23:44:13.604 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:42:14.375756+0000", 2023-09-23T23:44:13.604 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 119.139946826, 2023-09-23T23:44:13.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.15297572200000001, 2023-09-23T23:44:13.605 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.605 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.605 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.375756+0000", 2023-09-23T23:44:13.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.375756+0000", 2023-09-23T23:44:13.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.375755+0000", 2023-09-23T23:44:13.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:13.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.375762+0000", 2023-09-23T23:44:13.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5409999999999997e-06 2023-09-23T23:44:13.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.521137+0000", 2023-09-23T23:44:13.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14537471399999999 2023-09-23T23:44:13.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.521153+0000", 2023-09-23T23:44:13.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6799999999999998e-05 2023-09-23T23:44:13.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.521198+0000", 2023-09-23T23:44:13.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4787000000000003e-05 2023-09-23T23:44:13.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.521220+0000", 2023-09-23T23:44:13.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1457000000000001e-05 2023-09-23T23:44:13.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.528711+0000", 2023-09-23T23:44:13.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074912750000000004 2023-09-23T23:44:13.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:42:14.528732+0000", 2023-09-23T23:44:13.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0837999999999999e-05 2023-09-23T23:44:13.609 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.609 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.609 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.609 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.37383.0:889 15.6 e687/686 15:67cae2ed:::rbd_mirroring:head v 687'395, mlcod=687'393)", 2023-09-23T23:44:13.609 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:43:28.212660+0000", 2023-09-23T23:44:13.609 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 45.303042365000003, 2023-09-23T23:44:13.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.29256364800000001, 2023-09-23T23:44:13.610 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.610 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.610 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:28.212660+0000", 2023-09-23T23:44:13.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:28.212660+0000", 2023-09-23T23:44:13.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:28.212660+0000", 2023-09-23T23:44:13.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:13.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:28.212666+0000", 2023-09-23T23:44:13.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6679999999999998e-06 2023-09-23T23:44:13.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:28.212672+0000", 2023-09-23T23:44:13.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9149999999999999e-06 2023-09-23T23:44:13.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:28.212677+0000", 2023-09-23T23:44:13.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1230000000000002e-06 2023-09-23T23:44:13.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:28.212713+0000", 2023-09-23T23:44:13.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6072000000000003e-05 2023-09-23T23:44:13.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:28.212724+0000", 2023-09-23T23:44:13.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.164e-05 2023-09-23T23:44:13.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:28.505183+0000", 2023-09-23T23:44:13.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.29245907900000001 2023-09-23T23:44:13.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:28.505224+0000", 2023-09-23T23:44:13.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0738000000000002e-05 2023-09-23T23:44:13.614 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.614 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.614 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.614 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.38061.0:53 2.4 2:3831950c:::rbd_directory:head [call rbd.dir_remove_image in=19b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e715)", 2023-09-23T23:44:13.614 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:44:08.312996+0000", 2023-09-23T23:44:13.614 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 5.2027067819999999, 2023-09-23T23:44:13.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.24064371700000001, 2023-09-23T23:44:13.615 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:13.615 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:13.615 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:13.615 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.38061", 2023-09-23T23:44:13.615 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1837403620", 2023-09-23T23:44:13.615 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 53 2023-09-23T23:44:13.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.615 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:13.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:13.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.312996+0000", 2023-09-23T23:44:13.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:13.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.312996+0000", 2023-09-23T23:44:13.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:13.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:13.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.312995+0000", 2023-09-23T23:44:13.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:13.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:13.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.313000+0000", 2023-09-23T23:44:13.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3470000000000001e-06 2023-09-23T23:44:13.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:13.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.313007+0000", 2023-09-23T23:44:13.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1019999999999999e-06 2023-09-23T23:44:13.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:13.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.313012+0000", 2023-09-23T23:44:13.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7040000000000002e-06 2023-09-23T23:44:13.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:13.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.313038+0000", 2023-09-23T23:44:13.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6486e-05 2023-09-23T23:44:13.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:13.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.313084+0000", 2023-09-23T23:44:13.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6412999999999998e-05 2023-09-23T23:44:13.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:44:13.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.552833+0000", 2023-09-23T23:44:13.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.23974848700000001 2023-09-23T23:44:13.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:13.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.553268+0000", 2023-09-23T23:44:13.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00043517799999999999 2023-09-23T23:44:13.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:13.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.553554+0000", 2023-09-23T23:44:13.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028603 2023-09-23T23:44:13.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:13.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.553585+0000", 2023-09-23T23:44:13.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1191e-05 2023-09-23T23:44:13.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:13.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.553616+0000", 2023-09-23T23:44:13.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0494999999999999e-05 2023-09-23T23:44:13.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:13.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:13.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:13.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.553640+0000", 2023-09-23T23:44:13.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3952e-05 2023-09-23T23:44:13.621 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.621 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.621 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.621 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:13.621 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:13.621 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:44:13.622 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:13 smithi110 ceph-mon[212939]: pgmap v2008: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 1.4 MiB/s rd, 2.1 MiB/s wr, 441 op/s 2023-09-23T23:44:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:13 smithi110 ceph-mon[212939]: failed to encode map e718 with expected crc 2023-09-23T23:44:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:13 smithi110 ceph-mon[212939]: failed to encode map e719 with expected crc 2023-09-23T23:44:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:13 smithi110 ceph-mon[212939]: failed to encode map e719 with expected crc 2023-09-23T23:44:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:13 smithi110 ceph-mon[212939]: osdmap e719: 8 total, 7 up, 6 in 2023-09-23T23:44:13.955 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:44:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:13 smithi110 ceph-mon[221102]: pgmap v2008: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 1.4 MiB/s rd, 2.1 MiB/s wr, 441 op/s 2023-09-23T23:44:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:13 smithi110 ceph-mon[221102]: failed to encode map e718 with expected crc 2023-09-23T23:44:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:13 smithi110 ceph-mon[221102]: failed to encode map e719 with expected crc 2023-09-23T23:44:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:13 smithi110 ceph-mon[221102]: failed to encode map e719 with expected crc 2023-09-23T23:44:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:13 smithi110 ceph-mon[221102]: osdmap e719: 8 total, 7 up, 6 in 2023-09-23T23:44:13.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:44:14.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:13 smithi138 ceph-mon[193606]: pgmap v2008: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 1.4 MiB/s rd, 2.1 MiB/s wr, 441 op/s 2023-09-23T23:44:14.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:13 smithi138 ceph-mon[193606]: failed to encode map e718 with expected crc 2023-09-23T23:44:14.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:13 smithi138 ceph-mon[193606]: failed to encode map e719 with expected crc 2023-09-23T23:44:14.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:13 smithi138 ceph-mon[193606]: failed to encode map e719 with expected crc 2023-09-23T23:44:14.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:13 smithi138 ceph-mon[193606]: osdmap e719: 8 total, 7 up, 6 in 2023-09-23T23:44:14.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:44:14.346 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:44:15.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:14 smithi138 ceph-mon[193606]: failed to encode map e719 with expected crc 2023-09-23T23:44:15.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:14 smithi138 ceph-mon[193606]: failed to encode map e720 with expected crc 2023-09-23T23:44:15.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:14 smithi138 ceph-mon[193606]: osdmap e720: 8 total, 7 up, 6 in 2023-09-23T23:44:15.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:14 smithi138 ceph-mon[193606]: failed to encode map e720 with expected crc 2023-09-23T23:44:15.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:14 smithi138 ceph-mon[193606]: failed to encode map e721 with expected crc 2023-09-23T23:44:15.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:14 smithi138 ceph-mon[193606]: osdmap e721: 8 total, 7 up, 6 in 2023-09-23T23:44:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:14 smithi110 ceph-mon[212939]: failed to encode map e719 with expected crc 2023-09-23T23:44:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:14 smithi110 ceph-mon[212939]: failed to encode map e720 with expected crc 2023-09-23T23:44:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:14 smithi110 ceph-mon[212939]: osdmap e720: 8 total, 7 up, 6 in 2023-09-23T23:44:15.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:14 smithi110 ceph-mon[212939]: failed to encode map e720 with expected crc 2023-09-23T23:44:15.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:14 smithi110 ceph-mon[212939]: failed to encode map e721 with expected crc 2023-09-23T23:44:15.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:14 smithi110 ceph-mon[212939]: osdmap e721: 8 total, 7 up, 6 in 2023-09-23T23:44:15.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:14 smithi110 ceph-mon[221102]: failed to encode map e719 with expected crc 2023-09-23T23:44:15.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:14 smithi110 ceph-mon[221102]: failed to encode map e720 with expected crc 2023-09-23T23:44:15.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:14 smithi110 ceph-mon[221102]: osdmap e720: 8 total, 7 up, 6 in 2023-09-23T23:44:15.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:14 smithi110 ceph-mon[221102]: failed to encode map e720 with expected crc 2023-09-23T23:44:15.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:14 smithi110 ceph-mon[221102]: failed to encode map e721 with expected crc 2023-09-23T23:44:15.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:14 smithi110 ceph-mon[221102]: osdmap e721: 8 total, 7 up, 6 in 2023-09-23T23:44:15.403 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:15.771 INFO:teuthology.orchestra.run.smithi110.stderr:noscrub is unset 2023-09-23T23:44:15.864 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:15 smithi110 ceph-mon[212939]: failed to encode map e721 with expected crc 2023-09-23T23:44:15.864 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:15 smithi110 ceph-mon[212939]: pgmap v2012: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 92 KiB/s rd, 2.9 KiB/s wr, 122 op/s 2023-09-23T23:44:15.864 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:15 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3280216843' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:44:15.864 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:15 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:44:15.864 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:15 smithi110 ceph-mon[221102]: failed to encode map e721 with expected crc 2023-09-23T23:44:15.864 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:15 smithi110 ceph-mon[221102]: pgmap v2012: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 92 KiB/s rd, 2.9 KiB/s wr, 122 op/s 2023-09-23T23:44:15.864 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:15 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3280216843' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:44:15.865 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:15 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:44:16.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:15 smithi138 ceph-mon[193606]: failed to encode map e721 with expected crc 2023-09-23T23:44:16.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:15 smithi138 ceph-mon[193606]: pgmap v2012: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 92 KiB/s rd, 2.9 KiB/s wr, 122 op/s 2023-09-23T23:44:16.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:15 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3280216843' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:44:16.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:15 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:44:16.775 INFO:teuthology.orchestra.run.smithi110.stdout:moved 6 / 309 (1.94175%) 2023-09-23T23:44:16.775 INFO:teuthology.orchestra.run.smithi110.stdout:avg 51.5 2023-09-23T23:44:16.775 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 5.79511 -> 4.71699 (expected baseline 6.55108) 2023-09-23T23:44:16.776 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.2 with 39 -> 41 pgs (0.757282 -> 0.796117 * mean) 2023-09-23T23:44:16.776 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.0 with 56 -> 53 pgs (1.08738 -> 1.02913 * mean) 2023-09-23T23:44:16.776 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:44:16.776 INFO:teuthology.orchestra.run.smithi110.stdout:oload 110 2023-09-23T23:44:16.776 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 0.05 2023-09-23T23:44:16.776 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:44:16.776 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0107 2023-09-23T23:44:16.776 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.0118 2023-09-23T23:44:16.776 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 weight 0.9500 -> 0.9000 2023-09-23T23:44:16.776 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 weight 0.9500 -> 0.9000 2023-09-23T23:44:16.846 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:44:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:44:16] "GET /metrics HTTP/1.1" 200 34189 "" "Prometheus/2.43.0" 2023-09-23T23:44:16.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:16 smithi110 ceph-mon[221102]: failed to encode map e722 with expected crc 2023-09-23T23:44:16.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:16 smithi110 ceph-mon[221102]: failed to encode map e722 with expected crc 2023-09-23T23:44:16.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:16 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:44:16.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:16 smithi110 ceph-mon[221102]: osdmap e722: 8 total, 7 up, 6 in 2023-09-23T23:44:16.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:16 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'58984','3':'58984'}"}]: dispatch 2023-09-23T23:44:16.846 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:16 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'58984','3':'58984'}"}]: dispatch 2023-09-23T23:44:16.847 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:16 smithi110 ceph-mon[212939]: failed to encode map e722 with expected crc 2023-09-23T23:44:16.847 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:16 smithi110 ceph-mon[212939]: failed to encode map e722 with expected crc 2023-09-23T23:44:16.847 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:16 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:44:16.847 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:16 smithi110 ceph-mon[212939]: osdmap e722: 8 total, 7 up, 6 in 2023-09-23T23:44:16.847 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:16 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'58984','3':'58984'}"}]: dispatch 2023-09-23T23:44:16.847 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:16 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'58984','3':'58984'}"}]: dispatch 2023-09-23T23:44:16.865 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:44:17.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:16 smithi138 ceph-mon[193606]: failed to encode map e722 with expected crc 2023-09-23T23:44:17.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:16 smithi138 ceph-mon[193606]: failed to encode map e722 with expected crc 2023-09-23T23:44:17.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:16 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:44:17.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:16 smithi138 ceph-mon[193606]: osdmap e722: 8 total, 7 up, 6 in 2023-09-23T23:44:17.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:16 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'58984','3':'58984'}"}]: dispatch 2023-09-23T23:44:17.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:16 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'58984','3':'58984'}"}]: dispatch 2023-09-23T23:44:17.653 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T23:44:17.654 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-09-23T23:44:17.839 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:17.941 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:44:17.941 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:44:17.941 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-09-23T23:44:17.941 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:44:17.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:17 smithi110 ceph-mon[212939]: from='client.38139 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:44:17.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:17 smithi110 ceph-mon[212939]: pgmap v2014: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 14 KiB/s rd, 767 B/s wr, 24 op/s 2023-09-23T23:44:17.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:17 smithi110 ceph-mon[212939]: failed to encode map e723 with expected crc 2023-09-23T23:44:17.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:17 smithi110 ceph-mon[212939]: failed to encode map e723 with expected crc 2023-09-23T23:44:17.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:17 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'58984','3':'58984'}"}]': finished 2023-09-23T23:44:17.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:17 smithi110 ceph-mon[212939]: osdmap e723: 8 total, 7 up, 6 in 2023-09-23T23:44:17.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:17 smithi110 ceph-mon[221102]: from='client.38139 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:44:17.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:17 smithi110 ceph-mon[221102]: pgmap v2014: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 14 KiB/s rd, 767 B/s wr, 24 op/s 2023-09-23T23:44:17.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:17 smithi110 ceph-mon[221102]: failed to encode map e723 with expected crc 2023-09-23T23:44:17.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:17 smithi110 ceph-mon[221102]: failed to encode map e723 with expected crc 2023-09-23T23:44:17.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:17 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'58984','3':'58984'}"}]': finished 2023-09-23T23:44:17.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:17 smithi110 ceph-mon[221102]: osdmap e723: 8 total, 7 up, 6 in 2023-09-23T23:44:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:17 smithi138 ceph-mon[193606]: from='client.38139 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:44:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:17 smithi138 ceph-mon[193606]: pgmap v2014: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 14 KiB/s rd, 767 B/s wr, 24 op/s 2023-09-23T23:44:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:17 smithi138 ceph-mon[193606]: failed to encode map e723 with expected crc 2023-09-23T23:44:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:17 smithi138 ceph-mon[193606]: failed to encode map e723 with expected crc 2023-09-23T23:44:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:17 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'58984','3':'58984'}"}]': finished 2023-09-23T23:44:18.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:17 smithi138 ceph-mon[193606]: osdmap e723: 8 total, 7 up, 6 in 2023-09-23T23:44:18.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:18 smithi110 ceph-mon[212939]: failed to encode map e724 with expected crc 2023-09-23T23:44:18.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:18 smithi110 ceph-mon[212939]: osdmap e724: 8 total, 7 up, 6 in 2023-09-23T23:44:18.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:18 smithi110 ceph-mon[212939]: failed to encode map e724 with expected crc 2023-09-23T23:44:18.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:18 smithi110 ceph-mon[212939]: failed to encode map e724 with expected crc 2023-09-23T23:44:18.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:18 smithi110 ceph-mon[221102]: failed to encode map e724 with expected crc 2023-09-23T23:44:18.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:18 smithi110 ceph-mon[221102]: osdmap e724: 8 total, 7 up, 6 in 2023-09-23T23:44:18.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:18 smithi110 ceph-mon[221102]: failed to encode map e724 with expected crc 2023-09-23T23:44:18.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:18 smithi110 ceph-mon[221102]: failed to encode map e724 with expected crc 2023-09-23T23:44:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:18 smithi138 ceph-mon[193606]: failed to encode map e724 with expected crc 2023-09-23T23:44:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:18 smithi138 ceph-mon[193606]: osdmap e724: 8 total, 7 up, 6 in 2023-09-23T23:44:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:18 smithi138 ceph-mon[193606]: failed to encode map e724 with expected crc 2023-09-23T23:44:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:18 smithi138 ceph-mon[193606]: failed to encode map e724 with expected crc 2023-09-23T23:44:19.241 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:44:19.344 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:19.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:19 smithi110 ceph-mon[212939]: pgmap v2017: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:19.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:19 smithi110 ceph-mon[212939]: failed to encode map e725 with expected crc 2023-09-23T23:44:19.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:19 smithi110 ceph-mon[212939]: failed to encode map e725 with expected crc 2023-09-23T23:44:19.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:19 smithi110 ceph-mon[212939]: osdmap e725: 8 total, 7 up, 6 in 2023-09-23T23:44:19.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:19 smithi110 ceph-mon[212939]: failed to encode map e726 with expected crc 2023-09-23T23:44:19.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:19 smithi110 ceph-mon[212939]: osdmap e726: 8 total, 7 up, 6 in 2023-09-23T23:44:19.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:19 smithi110 ceph-mon[221102]: pgmap v2017: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:19.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:19 smithi110 ceph-mon[221102]: failed to encode map e725 with expected crc 2023-09-23T23:44:19.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:19 smithi110 ceph-mon[221102]: failed to encode map e725 with expected crc 2023-09-23T23:44:19.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:19 smithi110 ceph-mon[221102]: osdmap e725: 8 total, 7 up, 6 in 2023-09-23T23:44:19.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:19 smithi110 ceph-mon[221102]: failed to encode map e726 with expected crc 2023-09-23T23:44:19.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:19 smithi110 ceph-mon[221102]: osdmap e726: 8 total, 7 up, 6 in 2023-09-23T23:44:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:19 smithi138 ceph-mon[193606]: pgmap v2017: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:19 smithi138 ceph-mon[193606]: failed to encode map e725 with expected crc 2023-09-23T23:44:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:19 smithi138 ceph-mon[193606]: failed to encode map e725 with expected crc 2023-09-23T23:44:20.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:19 smithi138 ceph-mon[193606]: osdmap e725: 8 total, 7 up, 6 in 2023-09-23T23:44:20.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:19 smithi138 ceph-mon[193606]: failed to encode map e726 with expected crc 2023-09-23T23:44:20.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:19 smithi138 ceph-mon[193606]: osdmap e726: 8 total, 7 up, 6 in 2023-09-23T23:44:20.581 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:20.836 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:20 smithi110 ceph-mon[212939]: failed to encode map e726 with expected crc 2023-09-23T23:44:20.836 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:20 smithi110 ceph-mon[212939]: failed to encode map e727 with expected crc 2023-09-23T23:44:20.836 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:20 smithi110 ceph-mon[212939]: osdmap e727: 8 total, 7 up, 6 in 2023-09-23T23:44:20.836 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:20 smithi110 ceph-mon[212939]: failed to encode map e727 with expected crc 2023-09-23T23:44:20.836 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:20 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2064276032' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:44:20.836 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:20 smithi110 ceph-mon[221102]: failed to encode map e726 with expected crc 2023-09-23T23:44:20.836 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:20 smithi110 ceph-mon[221102]: failed to encode map e727 with expected crc 2023-09-23T23:44:20.837 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:20 smithi110 ceph-mon[221102]: osdmap e727: 8 total, 7 up, 6 in 2023-09-23T23:44:20.837 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:20 smithi110 ceph-mon[221102]: failed to encode map e727 with expected crc 2023-09-23T23:44:20.837 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:20 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2064276032' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:44:20.967 INFO:teuthology.orchestra.run.smithi110.stderr:nodeep-scrub is unset 2023-09-23T23:44:21.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:20 smithi138 ceph-mon[193606]: failed to encode map e726 with expected crc 2023-09-23T23:44:21.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:20 smithi138 ceph-mon[193606]: failed to encode map e727 with expected crc 2023-09-23T23:44:21.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:20 smithi138 ceph-mon[193606]: osdmap e727: 8 total, 7 up, 6 in 2023-09-23T23:44:21.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:20 smithi138 ceph-mon[193606]: failed to encode map e727 with expected crc 2023-09-23T23:44:21.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:20 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2064276032' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:44:21.703 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:44:21 smithi110 conmon[198813]: 2023-09-23T23:44:21.357+0000 7ff0150b6700 -1 mgr.server reply reply (2) No such file or directory no change 2023-09-23T23:44:21.885 INFO:teuthology.orchestra.run.smithi110.stdout:moved 6 / 309 (1.94175%) 2023-09-23T23:44:21.885 INFO:teuthology.orchestra.run.smithi110.stdout:avg 51.5 2023-09-23T23:44:21.885 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 4.71699 -> 4.89047 (expected baseline 6.55108) 2023-09-23T23:44:21.885 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.2 with 41 -> 42 pgs (0.796117 -> 0.815534 * mean) 2023-09-23T23:44:21.885 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.1 with 54 -> 54 pgs (1.04854 -> 1.04854 * mean) 2023-09-23T23:44:21.885 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:44:21.885 INFO:teuthology.orchestra.run.smithi110.stdout:oload 110 2023-09-23T23:44:21.885 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 0.05 2023-09-23T23:44:21.885 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:44:21.885 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0107 2023-09-23T23:44:21.886 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.0118 2023-09-23T23:44:21.886 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 weight 0.9000 -> 0.8500 2023-09-23T23:44:21.886 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 weight 0.9000 -> 0.8500 2023-09-23T23:44:21.886 INFO:teuthology.orchestra.run.smithi110.stderr:no change 2023-09-23T23:44:22.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:21 smithi138 ceph-mon[193606]: failed to encode map e727 with expected crc 2023-09-23T23:44:22.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:21 smithi138 ceph-mon[193606]: pgmap v2021: 103 pgs: 1 remapped+peering, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:22.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:21 smithi138 ceph-mon[193606]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:44:22.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:21 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:44:22.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:21 smithi138 ceph-mon[193606]: failed to encode map e728 with expected crc 2023-09-23T23:44:22.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:21 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2064276032' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:44:22.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:21 smithi138 ceph-mon[193606]: osdmap e728: 8 total, 7 up, 6 in 2023-09-23T23:44:22.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:21 smithi110 ceph-mon[212939]: failed to encode map e727 with expected crc 2023-09-23T23:44:22.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:21 smithi110 ceph-mon[212939]: pgmap v2021: 103 pgs: 1 remapped+peering, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:22.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:21 smithi110 ceph-mon[212939]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:44:22.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:21 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:44:22.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:21 smithi110 ceph-mon[212939]: failed to encode map e728 with expected crc 2023-09-23T23:44:22.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:21 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2064276032' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:44:22.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:21 smithi110 ceph-mon[212939]: osdmap e728: 8 total, 7 up, 6 in 2023-09-23T23:44:22.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:21 smithi110 ceph-mon[221102]: failed to encode map e727 with expected crc 2023-09-23T23:44:22.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:21 smithi110 ceph-mon[221102]: pgmap v2021: 103 pgs: 1 remapped+peering, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:22.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:21 smithi110 ceph-mon[221102]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:44:22.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:21 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:44:22.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:21 smithi110 ceph-mon[221102]: failed to encode map e728 with expected crc 2023-09-23T23:44:22.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:21 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2064276032' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:44:22.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:21 smithi110 ceph-mon[221102]: osdmap e728: 8 total, 7 up, 6 in 2023-09-23T23:44:23.022 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T23:44:23.023 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-09-23T23:44:23.052 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:22 smithi110 ceph-mon[212939]: from='client.48185 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:44:23.052 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:22 smithi110 ceph-mon[221102]: from='client.48185 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:44:23.159 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:44:23.159 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:44:23.159 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:44:23.159 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:44:23.159 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:44:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:22 smithi138 ceph-mon[193606]: from='client.48185 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:44:23.830 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:44:23.909 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:24.108 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:23 smithi110 ceph-mon[212939]: pgmap v2023: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 138 B/s, 0 objects/s recovering 2023-09-23T23:44:24.109 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:23 smithi110 ceph-mon[221102]: pgmap v2023: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 138 B/s, 0 objects/s recovering 2023-09-23T23:44:24.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:23 smithi138 ceph-mon[193606]: pgmap v2023: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 138 B/s, 0 objects/s recovering 2023-09-23T23:44:24.998 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:25.201 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:24 smithi110 ceph-mon[212939]: pgmap v2024: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 103 B/s, 0 objects/s recovering 2023-09-23T23:44:25.201 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:24 smithi110 ceph-mon[221102]: pgmap v2024: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 103 B/s, 0 objects/s recovering 2023-09-23T23:44:25.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:24 smithi138 ceph-mon[193606]: pgmap v2024: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 103 B/s, 0 objects/s recovering 2023-09-23T23:44:25.625 INFO:teuthology.orchestra.run.smithi110.stdout:moved 0 / 309 (0%) 2023-09-23T23:44:25.625 INFO:teuthology.orchestra.run.smithi110.stdout:avg 51.5 2023-09-23T23:44:25.626 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 4.71699 -> 4.71699 (expected baseline 6.55108) 2023-09-23T23:44:25.626 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.2 with 41 -> 41 pgs (0.796117 -> 0.796117 * mean) 2023-09-23T23:44:25.626 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.1 with 54 -> 54 pgs (1.04854 -> 1.04854 * mean) 2023-09-23T23:44:25.626 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:44:25.626 INFO:teuthology.orchestra.run.smithi110.stdout:oload 1000 2023-09-23T23:44:25.627 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 3 2023-09-23T23:44:25.627 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:44:25.627 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0107 2023-09-23T23:44:25.627 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.1071 2023-09-23T23:44:25.628 INFO:teuthology.orchestra.run.smithi110.stderr:no change 2023-09-23T23:44:26.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:25 smithi110 ceph-mon[212939]: from='client.48191 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:44:26.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:25 smithi110 ceph-mon[221102]: from='client.48191 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:44:26.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:25 smithi138 ceph-mon[193606]: from='client.48191 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:44:26.855 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T23:44:26.855 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-09-23T23:44:27.072 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:26 smithi110 ceph-mon[212939]: pgmap v2025: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 77 B/s, 0 objects/s recovering 2023-09-23T23:44:27.072 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:44:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:44:26] "GET /metrics HTTP/1.1" 200 34189 "" "Prometheus/2.43.0" 2023-09-23T23:44:27.072 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:26 smithi110 ceph-mon[221102]: pgmap v2025: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 77 B/s, 0 objects/s recovering 2023-09-23T23:44:27.258 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:44:27.258 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:44:27.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:44:27.258 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:44:27.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.258 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89738 13.3 13:cade2564:::benchmark_data_smithi110_541928_object1958:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:44:27.259 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:34.801877+0000", 2023-09-23T23:44:27.259 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 592.45090792899998, 2023-09-23T23:44:27.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5955823369999997, 2023-09-23T23:44:27.259 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.259 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.259 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.259 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:44:27.259 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:44:27.259 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89738 2023-09-23T23:44:27.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.259 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801877+0000", 2023-09-23T23:44:27.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801877+0000", 2023-09-23T23:44:27.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801876+0000", 2023-09-23T23:44:27.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:27.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801883+0000", 2023-09-23T23:44:27.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8959999999999997e-06 2023-09-23T23:44:27.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801895+0000", 2023-09-23T23:44:27.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2709e-05 2023-09-23T23:44:27.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801905+0000", 2023-09-23T23:44:27.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1589999999999999e-06 2023-09-23T23:44:27.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801984+0000", 2023-09-23T23:44:27.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9043999999999997e-05 2023-09-23T23:44:27.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:27.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.801997+0000", 2023-09-23T23:44:27.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3294e-05 2023-09-23T23:44:27.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731046+0000", 2023-09-23T23:44:27.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.92904877500000005 2023-09-23T23:44:27.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:27.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731060+0000", 2023-09-23T23:44:27.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4042e-05 2023-09-23T23:44:27.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.371733+0000", 2023-09-23T23:44:27.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6406730409999999 2023-09-23T23:44:27.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:44:27.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.372202+0000", 2023-09-23T23:44:27.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000469529 2023-09-23T23:44:27.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.381823+0000", 2023-09-23T23:44:27.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0096204110000000006 2023-09-23T23:44:27.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.381932+0000", 2023-09-23T23:44:27.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010885 2023-09-23T23:44:27.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:44:27.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.382080+0000", 2023-09-23T23:44:27.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000148555 2023-09-23T23:44:27.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394009+0000", 2023-09-23T23:44:27.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011928887000000001 2023-09-23T23:44:27.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394057+0000", 2023-09-23T23:44:27.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8157e-05 2023-09-23T23:44:27.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394066+0000", 2023-09-23T23:44:27.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7320000000000004e-06 2023-09-23T23:44:27.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394132+0000", 2023-09-23T23:44:27.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5845000000000004e-05 2023-09-23T23:44:27.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.394208+0000", 2023-09-23T23:44:27.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6352000000000004e-05 2023-09-23T23:44:27.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397460+0000", 2023-09-23T23:44:27.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032514520000000002 2023-09-23T23:44:27.269 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.269 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.270 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.270 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89748 13.3 13:c6535c6c:::benchmark_data_smithi110_541928_object1968:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:44:27.270 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:34.962737+0000", 2023-09-23T23:44:27.270 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 592.29004842999996, 2023-09-23T23:44:27.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.437414017, 2023-09-23T23:44:27.270 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.270 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.270 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.271 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:44:27.271 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:44:27.271 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89748 2023-09-23T23:44:27.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.271 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962737+0000", 2023-09-23T23:44:27.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962737+0000", 2023-09-23T23:44:27.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962735+0000", 2023-09-23T23:44:27.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:44:27.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962743+0000", 2023-09-23T23:44:27.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7700000000000001e-06 2023-09-23T23:44:27.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962756+0000", 2023-09-23T23:44:27.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3013999999999999e-05 2023-09-23T23:44:27.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962763+0000", 2023-09-23T23:44:27.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7260000000000002e-06 2023-09-23T23:44:27.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962867+0000", 2023-09-23T23:44:27.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103978 2023-09-23T23:44:27.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:27.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:34.962878+0000", 2023-09-23T23:44:27.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0927999999999999e-05 2023-09-23T23:44:27.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731084+0000", 2023-09-23T23:44:27.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.76820582999999998 2023-09-23T23:44:27.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:27.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731093+0000", 2023-09-23T23:44:27.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6149999999999997e-06 2023-09-23T23:44:27.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.375727+0000", 2023-09-23T23:44:27.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6446342889999999 2023-09-23T23:44:27.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:44:27.277 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.376199+0000", 2023-09-23T23:44:27.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047155199999999997 2023-09-23T23:44:27.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.277 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.277 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.390857+0000", 2023-09-23T23:44:27.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014658200999999999 2023-09-23T23:44:27.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.277 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.277 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.390980+0000", 2023-09-23T23:44:27.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000122774 2023-09-23T23:44:27.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:44:27.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.391164+0000", 2023-09-23T23:44:27.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018492700000000001 2023-09-23T23:44:27.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397524+0000", 2023-09-23T23:44:27.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063597879999999999 2023-09-23T23:44:27.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397564+0000", 2023-09-23T23:44:27.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9746999999999997e-05 2023-09-23T23:44:27.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397611+0000", 2023-09-23T23:44:27.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7403e-05 2023-09-23T23:44:27.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397668+0000", 2023-09-23T23:44:27.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6178999999999999e-05 2023-09-23T23:44:27.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.397732+0000", 2023-09-23T23:44:27.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4548999999999995e-05 2023-09-23T23:44:27.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.400151+0000", 2023-09-23T23:44:27.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024186429999999998 2023-09-23T23:44:27.281 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.281 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.281 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.282 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89756 13.3 13:cf60a43c:::benchmark_data_smithi110_541928_object1976:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:44:27.282 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.051526+0000", 2023-09-23T23:44:27.282 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 592.20125965700004, 2023-09-23T23:44:27.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4865002390000002, 2023-09-23T23:44:27.282 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.282 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.282 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.282 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:44:27.282 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:44:27.282 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89756 2023-09-23T23:44:27.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.283 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051526+0000", 2023-09-23T23:44:27.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051526+0000", 2023-09-23T23:44:27.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051524+0000", 2023-09-23T23:44:27.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:27.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051532+0000", 2023-09-23T23:44:27.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6310000000000003e-06 2023-09-23T23:44:27.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051559+0000", 2023-09-23T23:44:27.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7444999999999999e-05 2023-09-23T23:44:27.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051568+0000", 2023-09-23T23:44:27.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8310000000000007e-06 2023-09-23T23:44:27.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051616+0000", 2023-09-23T23:44:27.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8168999999999999e-05 2023-09-23T23:44:27.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:27.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.051624+0000", 2023-09-23T23:44:27.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2290000000000003e-06 2023-09-23T23:44:27.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731118+0000", 2023-09-23T23:44:27.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.67949406400000001 2023-09-23T23:44:27.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:27.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731126+0000", 2023-09-23T23:44:27.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0949999999999996e-06 2023-09-23T23:44:27.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.400190+0000", 2023-09-23T23:44:27.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.669063559 2023-09-23T23:44:27.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:44:27.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.400555+0000", 2023-09-23T23:44:27.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036511099999999999 2023-09-23T23:44:27.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.416430+0000", 2023-09-23T23:44:27.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015875185999999999 2023-09-23T23:44:27.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.416532+0000", 2023-09-23T23:44:27.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000101456 2023-09-23T23:44:27.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:44:27.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.416657+0000", 2023-09-23T23:44:27.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012510200000000001 2023-09-23T23:44:27.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.434768+0000", 2023-09-23T23:44:27.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018111008000000001 2023-09-23T23:44:27.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.434835+0000", 2023-09-23T23:44:27.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7475e-05 2023-09-23T23:44:27.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.534925+0000", 2023-09-23T23:44:27.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.100090106 2023-09-23T23:44:27.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.534976+0000", 2023-09-23T23:44:27.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0603999999999999e-05 2023-09-23T23:44:27.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.535014+0000", 2023-09-23T23:44:27.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8379e-05 2023-09-23T23:44:27.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.538026+0000", 2023-09-23T23:44:27.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003011391 2023-09-23T23:44:27.293 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.293 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.293 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.293 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89769 13.3 13:c7ccd55e:::benchmark_data_smithi110_541928_object1989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:44:27.294 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.139666+0000", 2023-09-23T23:44:27.294 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 592.11311912199994, 2023-09-23T23:44:27.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4017686810000001, 2023-09-23T23:44:27.294 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.294 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.294 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.294 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:44:27.294 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:44:27.294 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89769 2023-09-23T23:44:27.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.295 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139666+0000", 2023-09-23T23:44:27.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139666+0000", 2023-09-23T23:44:27.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139665+0000", 2023-09-23T23:44:27.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:27.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139673+0000", 2023-09-23T23:44:27.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1049999999999995e-06 2023-09-23T23:44:27.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139689+0000", 2023-09-23T23:44:27.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6656000000000001e-05 2023-09-23T23:44:27.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139701+0000", 2023-09-23T23:44:27.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1127e-05 2023-09-23T23:44:27.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139754+0000", 2023-09-23T23:44:27.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3362000000000002e-05 2023-09-23T23:44:27.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:27.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.139764+0000", 2023-09-23T23:44:27.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0474e-05 2023-09-23T23:44:27.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.731150+0000", 2023-09-23T23:44:27.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.59138585200000005 2023-09-23T23:44:27.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:27.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.735349+0000", 2023-09-23T23:44:27.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041989519999999997 2023-09-23T23:44:27.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.400620+0000", 2023-09-23T23:44:27.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6652705239999999 2023-09-23T23:44:27.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:44:27.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.401065+0000", 2023-09-23T23:44:27.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00044538800000000001 2023-09-23T23:44:27.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.424058+0000", 2023-09-23T23:44:27.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022993085999999999 2023-09-23T23:44:27.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.424146+0000", 2023-09-23T23:44:27.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7412999999999998e-05 2023-09-23T23:44:27.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:44:27.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.424253+0000", 2023-09-23T23:44:27.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107246 2023-09-23T23:44:27.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.440042+0000", 2023-09-23T23:44:27.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015789306999999999 2023-09-23T23:44:27.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.443102+0000", 2023-09-23T23:44:27.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030595700000000002 2023-09-23T23:44:27.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.538201+0000", 2023-09-23T23:44:27.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.095099702999999994 2023-09-23T23:44:27.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.538249+0000", 2023-09-23T23:44:27.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7549000000000002e-05 2023-09-23T23:44:27.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.538328+0000", 2023-09-23T23:44:27.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8671000000000006e-05 2023-09-23T23:44:27.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.541435+0000", 2023-09-23T23:44:27.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031071649999999998 2023-09-23T23:44:27.305 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.305 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.305 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.305 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89790 13.3 13:ce3e2df6:::benchmark_data_smithi110_541928_object2010:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:44:27.305 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.273997+0000", 2023-09-23T23:44:27.305 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 591.97878853300006, 2023-09-23T23:44:27.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.270580657, 2023-09-23T23:44:27.306 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.306 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.306 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.306 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:44:27.306 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:44:27.306 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89790 2023-09-23T23:44:27.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.306 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.273997+0000", 2023-09-23T23:44:27.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.273997+0000", 2023-09-23T23:44:27.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.273995+0000", 2023-09-23T23:44:27.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:27.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274004+0000", 2023-09-23T23:44:27.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7940000000000008e-06 2023-09-23T23:44:27.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274017+0000", 2023-09-23T23:44:27.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3405e-05 2023-09-23T23:44:27.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274027+0000", 2023-09-23T23:44:27.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7200000000000001e-06 2023-09-23T23:44:27.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274086+0000", 2023-09-23T23:44:27.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.94e-05 2023-09-23T23:44:27.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:27.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.274101+0000", 2023-09-23T23:44:27.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4109e-05 2023-09-23T23:44:27.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.735405+0000", 2023-09-23T23:44:27.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.46130481400000001 2023-09-23T23:44:27.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:27.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.735419+0000", 2023-09-23T23:44:27.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3217000000000001e-05 2023-09-23T23:44:27.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.401121+0000", 2023-09-23T23:44:27.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6657021570000001 2023-09-23T23:44:27.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:44:27.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.401570+0000", 2023-09-23T23:44:27.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000449343 2023-09-23T23:44:27.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.431522+0000", 2023-09-23T23:44:27.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029951951000000001 2023-09-23T23:44:27.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.431654+0000", 2023-09-23T23:44:27.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013153199999999999 2023-09-23T23:44:27.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:44:27.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.431813+0000", 2023-09-23T23:44:27.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000159937 2023-09-23T23:44:27.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.451910+0000", 2023-09-23T23:44:27.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02009613 2023-09-23T23:44:27.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.451987+0000", 2023-09-23T23:44:27.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7597999999999998e-05 2023-09-23T23:44:27.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.541568+0000", 2023-09-23T23:44:27.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.089580504000000005 2023-09-23T23:44:27.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.541608+0000", 2023-09-23T23:44:27.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0274000000000001e-05 2023-09-23T23:44:27.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.541675+0000", 2023-09-23T23:44:27.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6890000000000005e-05 2023-09-23T23:44:27.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.544577+0000", 2023-09-23T23:44:27.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029024509999999999 2023-09-23T23:44:27.316 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.316 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.317 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.317 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89805 13.3 13:c4e9783b:::benchmark_data_smithi110_541928_object2025:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e571)", 2023-09-23T23:44:27.317 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.955885+0000", 2023-09-23T23:44:27.317 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 591.29690034199996, 2023-09-23T23:44:27.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.592296927, 2023-09-23T23:44:27.317 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.317 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.317 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.317 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:44:27.318 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:44:27.318 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89805 2023-09-23T23:44:27.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.318 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955885+0000", 2023-09-23T23:44:27.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955885+0000", 2023-09-23T23:44:27.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955883+0000", 2023-09-23T23:44:27.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:27.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955891+0000", 2023-09-23T23:44:27.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0209999999999999e-06 2023-09-23T23:44:27.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955908+0000", 2023-09-23T23:44:27.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.643e-05 2023-09-23T23:44:27.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955917+0000", 2023-09-23T23:44:27.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2089999999999994e-06 2023-09-23T23:44:27.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.955987+0000", 2023-09-23T23:44:27.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9494999999999998e-05 2023-09-23T23:44:27.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:27.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.956002+0000", 2023-09-23T23:44:27.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5223e-05 2023-09-23T23:44:27.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.401618+0000", 2023-09-23T23:44:27.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4456160360000001 2023-09-23T23:44:27.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:44:27.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.402009+0000", 2023-09-23T23:44:27.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039078399999999998 2023-09-23T23:44:27.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.440091+0000", 2023-09-23T23:44:27.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038082498999999999 2023-09-23T23:44:27.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.440216+0000", 2023-09-23T23:44:27.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012535399999999999 2023-09-23T23:44:27.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:44:27.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.440388+0000", 2023-09-23T23:44:27.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017125300000000001 2023-09-23T23:44:27.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.457767+0000", 2023-09-23T23:44:27.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017379131999999999 2023-09-23T23:44:27.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.460556+0000", 2023-09-23T23:44:27.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027891050000000001 2023-09-23T23:44:27.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.544650+0000", 2023-09-23T23:44:27.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.084093672999999994 2023-09-23T23:44:27.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.544694+0000", 2023-09-23T23:44:27.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4366000000000002e-05 2023-09-23T23:44:27.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.544759+0000", 2023-09-23T23:44:27.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5371999999999997e-05 2023-09-23T23:44:27.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.327 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.327 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.548182+0000", 2023-09-23T23:44:27.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034224870000000001 2023-09-23T23:44:27.327 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.327 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.327 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.327 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:89823 13.3 13:c65a1b7a:::benchmark_data_smithi110_541928_object2043:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e571)", 2023-09-23T23:44:27.327 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:35.990158+0000", 2023-09-23T23:44:27.328 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 591.26262734800002, 2023-09-23T23:44:27.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5615935460000001, 2023-09-23T23:44:27.328 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.328 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.328 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.328 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:44:27.329 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:44:27.329 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 89823 2023-09-23T23:44:27.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.329 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990158+0000", 2023-09-23T23:44:27.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990158+0000", 2023-09-23T23:44:27.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990157+0000", 2023-09-23T23:44:27.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:27.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990162+0000", 2023-09-23T23:44:27.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1359999999999996e-06 2023-09-23T23:44:27.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990174+0000", 2023-09-23T23:44:27.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1695000000000001e-05 2023-09-23T23:44:27.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990184+0000", 2023-09-23T23:44:27.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0465000000000001e-05 2023-09-23T23:44:27.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990231+0000", 2023-09-23T23:44:27.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6329000000000003e-05 2023-09-23T23:44:27.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:27.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:35.990240+0000", 2023-09-23T23:44:27.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5310000000000007e-06 2023-09-23T23:44:27.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.402069+0000", 2023-09-23T23:44:27.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.411828474 2023-09-23T23:44:27.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-09-23T23:44:27.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.402423+0000", 2023-09-23T23:44:27.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035466000000000002 2023-09-23T23:44:27.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.448598+0000", 2023-09-23T23:44:27.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.046175040000000001 2023-09-23T23:44:27.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.335 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.335 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.448732+0000", 2023-09-23T23:44:27.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013384599999999999 2023-09-23T23:44:27.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.335 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T23:44:27.335 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.448935+0000", 2023-09-23T23:44:27.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020236700000000001 2023-09-23T23:44:27.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.460571+0000", 2023-09-23T23:44:27.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011636069000000001 2023-09-23T23:44:27.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.468058+0000", 2023-09-23T23:44:27.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074870570000000001 2023-09-23T23:44:27.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.548273+0000", 2023-09-23T23:44:27.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.080215195000000003 2023-09-23T23:44:27.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.548328+0000", 2023-09-23T23:44:27.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4493000000000002e-05 2023-09-23T23:44:27.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.548424+0000", 2023-09-23T23:44:27.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6016999999999999e-05 2023-09-23T23:44:27.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:37.551751+0000", 2023-09-23T23:44:27.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033278750000000001 2023-09-23T23:44:27.339 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.339 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.339 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.339 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:95149 13.d 13:bc7a1628:::benchmark_data_smithi110_541928_object7369:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T23:44:27.339 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:53.232568+0000", 2023-09-23T23:44:27.339 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 574.02021703900004, 2023-09-23T23:44:27.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4660194620000002, 2023-09-23T23:44:27.339 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.339 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.340 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.340 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:44:27.340 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:44:27.340 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95149 2023-09-23T23:44:27.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.340 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232568+0000", 2023-09-23T23:44:27.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232568+0000", 2023-09-23T23:44:27.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232567+0000", 2023-09-23T23:44:27.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:27.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232574+0000", 2023-09-23T23:44:27.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.824e-06 2023-09-23T23:44:27.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232588+0000", 2023-09-23T23:44:27.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3597e-05 2023-09-23T23:44:27.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.232596+0000", 2023-09-23T23:44:27.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8120000000000008e-06 2023-09-23T23:44:27.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.737972+0000", 2023-09-23T23:44:27.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.50537641300000002 2023-09-23T23:44:27.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:44:27.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.741095+0000", 2023-09-23T23:44:27.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031224780000000001 2023-09-23T23:44:27.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.346073+0000", 2023-09-23T23:44:27.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.60497817499999995 2023-09-23T23:44:27.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:44:27.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.351441+0000", 2023-09-23T23:44:27.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053685449999999997 2023-09-23T23:44:27.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.606909+0000", 2023-09-23T23:44:27.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2554679260000001 2023-09-23T23:44:27.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.612108+0000", 2023-09-23T23:44:27.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051983919999999996 2023-09-23T23:44:27.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:44:27.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.632573+0000", 2023-09-23T23:44:27.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020465006000000001 2023-09-23T23:44:27.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.668063+0000", 2023-09-23T23:44:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.035490090000000002 2023-09-23T23:44:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.680799+0000", 2023-09-23T23:44:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012736032 2023-09-23T23:44:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.682749+0000", 2023-09-23T23:44:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019503509999999999 2023-09-23T23:44:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.685937+0000", 2023-09-23T23:44:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031873520000000001 2023-09-23T23:44:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.698588+0000", 2023-09-23T23:44:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012651051999999999 2023-09-23T23:44:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:95282 13.d 13:b9f8e077:::benchmark_data_smithi110_541928_object7502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T23:44:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:53.626451+0000", 2023-09-23T23:44:27.349 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 573.62633394399995, 2023-09-23T23:44:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5002010320000001, 2023-09-23T23:44:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:44:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:44:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95282 2023-09-23T23:44:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.350 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626451+0000", 2023-09-23T23:44:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626451+0000", 2023-09-23T23:44:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626449+0000", 2023-09-23T23:44:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:44:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626459+0000", 2023-09-23T23:44:27.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7790000000000007e-06 2023-09-23T23:44:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626473+0000", 2023-09-23T23:44:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3655000000000001e-05 2023-09-23T23:44:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.626483+0000", 2023-09-23T23:44:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.013e-05 2023-09-23T23:44:27.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.973570+0000", 2023-09-23T23:44:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.34708731599999998 2023-09-23T23:44:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:44:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.978239+0000", 2023-09-23T23:44:27.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046685850000000003 2023-09-23T23:44:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.019769+0000", 2023-09-23T23:44:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041529772 2023-09-23T23:44:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:44:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.024313+0000", 2023-09-23T23:44:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045445479999999998 2023-09-23T23:44:27.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.064268+0000", 2023-09-23T23:44:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.039955124 2023-09-23T23:44:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.067970+0000", 2023-09-23T23:44:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00370164 2023-09-23T23:44:27.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:44:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.076658+0000", 2023-09-23T23:44:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086883770000000006 2023-09-23T23:44:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.104140+0000", 2023-09-23T23:44:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02748134 2023-09-23T23:44:27.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.113193+0000", 2023-09-23T23:44:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0090528350000000004 2023-09-23T23:44:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.114921+0000", 2023-09-23T23:44:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017289149999999999 2023-09-23T23:44:27.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.117081+0000", 2023-09-23T23:44:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021593519999999998 2023-09-23T23:44:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.126652+0000", 2023-09-23T23:44:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.009571474 2023-09-23T23:44:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.359 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:95302 13.d 13:b56e9a5b:::benchmark_data_smithi110_541928_object7522:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T23:44:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:53.699709+0000", 2023-09-23T23:44:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 573.55307578700001, 2023-09-23T23:44:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.504555552, 2023-09-23T23:44:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.360 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:44:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:44:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95302 2023-09-23T23:44:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699709+0000", 2023-09-23T23:44:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699709+0000", 2023-09-23T23:44:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699708+0000", 2023-09-23T23:44:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:44:27.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699717+0000", 2023-09-23T23:44:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0899999999999994e-06 2023-09-23T23:44:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699730+0000", 2023-09-23T23:44:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3392e-05 2023-09-23T23:44:27.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.699740+0000", 2023-09-23T23:44:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5559999999999996e-06 2023-09-23T23:44:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.025484+0000", 2023-09-23T23:44:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.325744649 2023-09-23T23:44:27.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:44:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.030494+0000", 2023-09-23T23:44:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.005009365 2023-09-23T23:44:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.166809+0000", 2023-09-23T23:44:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.13631576300000001 2023-09-23T23:44:27.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:44:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.171655+0000", 2023-09-23T23:44:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048457680000000003 2023-09-23T23:44:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.138451+0000", 2023-09-23T23:44:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96679576300000003 2023-09-23T23:44:27.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.142480+0000", 2023-09-23T23:44:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040295679999999999 2023-09-23T23:44:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:44:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.150317+0000", 2023-09-23T23:44:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078367109999999997 2023-09-23T23:44:27.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.186750+0000", 2023-09-23T23:44:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036433140000000003 2023-09-23T23:44:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.188250+0000", 2023-09-23T23:44:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001500145 2023-09-23T23:44:27.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.188845+0000", 2023-09-23T23:44:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00059478099999999996 2023-09-23T23:44:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.191339+0000", 2023-09-23T23:44:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024934559999999998 2023-09-23T23:44:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.204265+0000", 2023-09-23T23:44:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012926272000000001 2023-09-23T23:44:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.46514.0:95356 13.d 13:b7510188:::benchmark_data_smithi110_541928_object7576:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T23:44:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:34:53.853143+0000", 2023-09-23T23:44:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 573.39964203199997, 2023-09-23T23:44:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5531699410000002, 2023-09-23T23:44:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.46514", 2023-09-23T23:44:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:44:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95356 2023-09-23T23:44:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853143+0000", 2023-09-23T23:44:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853143+0000", 2023-09-23T23:44:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853142+0000", 2023-09-23T23:44:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853150+0000", 2023-09-23T23:44:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8709999999999997e-06 2023-09-23T23:44:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853166+0000", 2023-09-23T23:44:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6568000000000001e-05 2023-09-23T23:44:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:53.853176+0000", 2023-09-23T23:44:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9520000000000006e-06 2023-09-23T23:44:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.200709+0000", 2023-09-23T23:44:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.347532703 2023-09-23T23:44:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-09-23T23:44:27.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.205641+0000", 2023-09-23T23:44:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.004932199 2023-09-23T23:44:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.252766+0000", 2023-09-23T23:44:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047124528999999998 2023-09-23T23:44:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-09-23T23:44:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:54.257597+0000", 2023-09-23T23:44:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048309980000000004 2023-09-23T23:44:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.269273+0000", 2023-09-23T23:44:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.01167611 2023-09-23T23:44:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.276153+0000", 2023-09-23T23:44:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068806689999999998 2023-09-23T23:44:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-09-23T23:44:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.296160+0000", 2023-09-23T23:44:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020006954 2023-09-23T23:44:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.310175+0000", 2023-09-23T23:44:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014014235 2023-09-23T23:44:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.313239+0000", 2023-09-23T23:44:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030640490000000001 2023-09-23T23:44:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.379 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.383950+0000", 2023-09-23T23:44:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.070711100999999998 2023-09-23T23:44:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.388728+0000", 2023-09-23T23:44:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047779540000000001 2023-09-23T23:44:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:34:55.406313+0000", 2023-09-23T23:44:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017585478000000002 2023-09-23T23:44:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:71676 14.4 14:23cfb94e:::benchmark_data_smithi110_640334_object71675:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e634)", 2023-09-23T23:44:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:40:32.090762+0000", 2023-09-23T23:44:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 235.16202280300001, 2023-09-23T23:44:27.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.000094357, 2023-09-23T23:44:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71676 2023-09-23T23:44:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.090762+0000", 2023-09-23T23:44:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.090762+0000", 2023-09-23T23:44:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.090762+0000", 2023-09-23T23:44:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.090781+0000", 2023-09-23T23:44:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9534e-05 2023-09-23T23:44:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.384 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.090786+0000", 2023-09-23T23:44:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7299999999999996e-06 2023-09-23T23:44:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.090790+0000", 2023-09-23T23:44:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4400000000000001e-06 2023-09-23T23:44:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.090844+0000", 2023-09-23T23:44:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4425000000000002e-05 2023-09-23T23:44:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.090853+0000", 2023-09-23T23:44:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9069999999999996e-06 2023-09-23T23:44:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.080184+0000", 2023-09-23T23:44:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98933127700000001 2023-09-23T23:44:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.080350+0000", 2023-09-23T23:44:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016558100000000001 2023-09-23T23:44:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,7", 2023-09-23T23:44:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.080521+0000", 2023-09-23T23:44:27.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017122099999999999 2023-09-23T23:44:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.090629+0000", 2023-09-23T23:44:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010108112000000001 2023-09-23T23:44:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.090688+0000", 2023-09-23T23:44:27.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9299999999999998e-05 2023-09-23T23:44:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.090751+0000", 2023-09-23T23:44:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2228000000000001e-05 2023-09-23T23:44:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.090814+0000", 2023-09-23T23:44:27.389 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3442999999999995e-05 2023-09-23T23:44:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.090857+0000", 2023-09-23T23:44:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2725999999999997e-05 2023-09-23T23:44:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.390 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:71689 14.4 14:26b4eb17:::benchmark_data_smithi110_640334_object71688:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e634)", 2023-09-23T23:44:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:40:32.119139+0000", 2023-09-23T23:44:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 235.13364628100001, 2023-09-23T23:44:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97705882799999999, 2023-09-23T23:44:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 71689 2023-09-23T23:44:27.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.119139+0000", 2023-09-23T23:44:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.119139+0000", 2023-09-23T23:44:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.119138+0000", 2023-09-23T23:44:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.119162+0000", 2023-09-23T23:44:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3254000000000001e-05 2023-09-23T23:44:27.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.119170+0000", 2023-09-23T23:44:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3629999999999994e-06 2023-09-23T23:44:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.119176+0000", 2023-09-23T23:44:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.314e-06 2023-09-23T23:44:27.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.119238+0000", 2023-09-23T23:44:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1598000000000002e-05 2023-09-23T23:44:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:32.119250+0000", 2023-09-23T23:44:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1725999999999999e-05 2023-09-23T23:44:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.082887+0000", 2023-09-23T23:44:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96363742299999999 2023-09-23T23:44:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.083034+0000", 2023-09-23T23:44:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014674999999999999 2023-09-23T23:44:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 5,7", 2023-09-23T23:44:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.083190+0000", 2023-09-23T23:44:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000156334 2023-09-23T23:44:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.090739+0000", 2023-09-23T23:44:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075486320000000004 2023-09-23T23:44:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.397 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.090970+0000", 2023-09-23T23:44:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023142400000000001 2023-09-23T23:44:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.096138+0000", 2023-09-23T23:44:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051678319999999998 2023-09-23T23:44:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.398 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.096173+0000", 2023-09-23T23:44:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5423999999999998e-05 2023-09-23T23:44:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:40:33.096198+0000", 2023-09-23T23:44:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.429e-05 2023-09-23T23:44:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:95152 14.2 14:4c8232cd:::benchmark_data_smithi110_640334_object11273:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e645)", 2023-09-23T23:44:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:11.037166+0000", 2023-09-23T23:44:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 196.21561925, 2023-09-23T23:44:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.963686143, 2023-09-23T23:44:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:27.400 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95152 2023-09-23T23:44:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.037166+0000", 2023-09-23T23:44:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.037166+0000", 2023-09-23T23:44:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.037165+0000", 2023-09-23T23:44:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.037171+0000", 2023-09-23T23:44:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7680000000000003e-06 2023-09-23T23:44:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.037186+0000", 2023-09-23T23:44:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5013e-05 2023-09-23T23:44:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.037194+0000", 2023-09-23T23:44:27.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4880000000000001e-06 2023-09-23T23:44:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.037255+0000", 2023-09-23T23:44:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1377000000000005e-05 2023-09-23T23:44:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.037268+0000", 2023-09-23T23:44:27.404 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2731999999999999e-05 2023-09-23T23:44:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.970837+0000", 2023-09-23T23:44:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93356932800000003 2023-09-23T23:44:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.971001+0000", 2023-09-23T23:44:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016338999999999999 2023-09-23T23:44:27.405 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-09-23T23:44:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.971105+0000", 2023-09-23T23:44:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104168 2023-09-23T23:44:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.973794+0000", 2023-09-23T23:44:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026890479999999999 2023-09-23T23:44:27.406 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.992482+0000", 2023-09-23T23:44:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018688201000000002 2023-09-23T23:44:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:12.000770+0000", 2023-09-23T23:44:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0082880490000000005 2023-09-23T23:44:27.407 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:12.000814+0000", 2023-09-23T23:44:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4251e-05 2023-09-23T23:44:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:12.000852+0000", 2023-09-23T23:44:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7608999999999999e-05 2023-09-23T23:44:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.408 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.37521.0:1 2.6 2:7d15d1b1:::rbd_id.testimg_import:head [stat] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e703)", 2023-09-23T23:44:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:43:53.761677+0000", 2023-09-23T23:44:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 33.491108050000001, 2023-09-23T23:44:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028258530000000001, 2023-09-23T23:44:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-09-23T23:44:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.409 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.37521", 2023-09-23T23:44:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/134598945", 2023-09-23T23:44:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 1 2023-09-23T23:44:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:53.761677+0000", 2023-09-23T23:44:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.410 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:53.761677+0000", 2023-09-23T23:44:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:53.761676+0000", 2023-09-23T23:44:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:53.761682+0000", 2023-09-23T23:44:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1999999999999999e-06 2023-09-23T23:44:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:53.761693+0000", 2023-09-23T23:44:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0882e-05 2023-09-23T23:44:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.412 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:53.761703+0000", 2023-09-23T23:44:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0519e-05 2023-09-23T23:44:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:53.761754+0000", 2023-09-23T23:44:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1115000000000001e-05 2023-09-23T23:44:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.413 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op must wait for map", 2023-09-23T23:44:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:53.761761+0000", 2023-09-23T23:44:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.046e-06 2023-09-23T23:44:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:53.764398+0000", 2023-09-23T23:44:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026367119999999998 2023-09-23T23:44:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.414 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:53.764452+0000", 2023-09-23T23:44:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4249000000000002e-05 2023-09-23T23:44:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:53.764503+0000", 2023-09-23T23:44:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0615999999999998e-05 2023-09-23T23:44:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.415 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47642.0:52 2.6 2:7d15d1b1:::rbd_id.testimg_import:head [delete] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e705)", 2023-09-23T23:44:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:43:56.354148+0000", 2023-09-23T23:44:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 30.898637620999999, 2023-09-23T23:44:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027446810000000001, 2023-09-23T23:44:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47642", 2023-09-23T23:44:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/4250001659", 2023-09-23T23:44:27.416 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 52 2023-09-23T23:44:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.354148+0000", 2023-09-23T23:44:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.417 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.354148+0000", 2023-09-23T23:44:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.354146+0000", 2023-09-23T23:44:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:44:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.418 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.354153+0000", 2023-09-23T23:44:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1890000000000004e-06 2023-09-23T23:44:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.354166+0000", 2023-09-23T23:44:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3491e-05 2023-09-23T23:44:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.419 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.354174+0000", 2023-09-23T23:44:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8120000000000008e-06 2023-09-23T23:44:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.354230+0000", 2023-09-23T23:44:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5905000000000001e-05 2023-09-23T23:44:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.354311+0000", 2023-09-23T23:44:27.420 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1349999999999999e-05 2023-09-23T23:44:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:44:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.354416+0000", 2023-09-23T23:44:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010508700000000001 2023-09-23T23:44:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.354766+0000", 2023-09-23T23:44:27.421 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034934300000000001 2023-09-23T23:44:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.355086+0000", 2023-09-23T23:44:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032067800000000002 2023-09-23T23:44:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.356802+0000", 2023-09-23T23:44:27.422 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001715998 2023-09-23T23:44:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.356847+0000", 2023-09-23T23:44:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4904000000000002e-05 2023-09-23T23:44:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.423 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.356892+0000", 2023-09-23T23:44:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5061000000000001e-05 2023-09-23T23:44:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.37647.0:6 2.6 2:7d15d1b1:::rbd_id.testimg_import:head [create,call rbd.set_id in=9b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e705)", 2023-09-23T23:44:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:43:56.701955+0000", 2023-09-23T23:44:27.425 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 30.550830394999998, 2023-09-23T23:44:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0049566619999999997, 2023-09-23T23:44:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.37647", 2023-09-23T23:44:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/653353877", 2023-09-23T23:44:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 6 2023-09-23T23:44:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.426 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.701955+0000", 2023-09-23T23:44:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.701955+0000", 2023-09-23T23:44:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.427 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.701953+0000", 2023-09-23T23:44:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:44:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.701964+0000", 2023-09-23T23:44:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1408e-05 2023-09-23T23:44:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.428 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.701986+0000", 2023-09-23T23:44:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1991e-05 2023-09-23T23:44:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.701998+0000", 2023-09-23T23:44:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1107000000000001e-05 2023-09-23T23:44:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.429 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.702045+0000", 2023-09-23T23:44:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7302999999999998e-05 2023-09-23T23:44:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.702146+0000", 2023-09-23T23:44:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000100844 2023-09-23T23:44:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.430 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:44:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.702405+0000", 2023-09-23T23:44:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000259647 2023-09-23T23:44:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.703010+0000", 2023-09-23T23:44:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00060454599999999997 2023-09-23T23:44:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.431 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.703173+0000", 2023-09-23T23:44:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000163311 2023-09-23T23:44:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.706778+0000", 2023-09-23T23:44:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036044029999999999 2023-09-23T23:44:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.706856+0000", 2023-09-23T23:44:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8096000000000006e-05 2023-09-23T23:44:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:43:56.706911+0000", 2023-09-23T23:44:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5810999999999998e-05 2023-09-23T23:44:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47900.0:29 2.6 2:7640c296:::rbd_data.bb1cea0e5cc5.0000000000000000:head [set-alloc-hint object_size 1048576 write_size 1048576,writefull 0~1048576 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e714)", 2023-09-23T23:44:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:44:06.527668+0000", 2023-09-23T23:44:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 20.725117071, 2023-09-23T23:44:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.15864550999999999, 2023-09-23T23:44:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47900", 2023-09-23T23:44:27.434 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/2789370275", 2023-09-23T23:44:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 29 2023-09-23T23:44:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:06.527668+0000", 2023-09-23T23:44:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.435 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:06.527668+0000", 2023-09-23T23:44:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:06.527667+0000", 2023-09-23T23:44:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:44:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.436 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:06.528089+0000", 2023-09-23T23:44:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00042117100000000002 2023-09-23T23:44:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:06.528099+0000", 2023-09-23T23:44:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0509000000000001e-05 2023-09-23T23:44:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.437 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:06.528105+0000", 2023-09-23T23:44:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4949999999999999e-06 2023-09-23T23:44:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:06.528138+0000", 2023-09-23T23:44:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3840000000000001e-05 2023-09-23T23:44:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:06.528228+0000", 2023-09-23T23:44:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9438999999999995e-05 2023-09-23T23:44:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:44:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:06.528313+0000", 2023-09-23T23:44:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5667000000000005e-05 2023-09-23T23:44:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.439 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:06.684960+0000", 2023-09-23T23:44:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.15664637000000001 2023-09-23T23:44:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:06.686143+0000", 2023-09-23T23:44:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001182722 2023-09-23T23:44:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:06.686234+0000", 2023-09-23T23:44:27.440 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1682999999999996e-05 2023-09-23T23:44:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:06.686280+0000", 2023-09-23T23:44:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5395000000000002e-05 2023-09-23T23:44:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:06.686314+0000", 2023-09-23T23:44:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4010999999999997e-05 2023-09-23T23:44:27.441 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.37956.0:29 2.6 2:62e3e79d:::rbd_data.94444472da1a.0000000000000000:head [set-alloc-hint object_size 1048576 write_size 1048576,writefull 0~1048576 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e715)", 2023-09-23T23:44:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:44:07.517241+0000", 2023-09-23T23:44:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 19.735543929999999, 2023-09-23T23:44:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032250059999999999, 2023-09-23T23:44:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.442 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.37956", 2023-09-23T23:44:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/768881474", 2023-09-23T23:44:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 29 2023-09-23T23:44:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:07.517241+0000", 2023-09-23T23:44:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:07.517241+0000", 2023-09-23T23:44:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:07.517240+0000", 2023-09-23T23:44:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:44:27.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:07.517682+0000", 2023-09-23T23:44:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000442244 2023-09-23T23:44:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:07.517693+0000", 2023-09-23T23:44:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1039000000000001e-05 2023-09-23T23:44:27.445 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:07.517699+0000", 2023-09-23T23:44:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9329999999999998e-06 2023-09-23T23:44:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:07.517741+0000", 2023-09-23T23:44:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1822e-05 2023-09-23T23:44:27.446 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:07.517855+0000", 2023-09-23T23:44:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000113532 2023-09-23T23:44:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:44:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:07.517951+0000", 2023-09-23T23:44:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6426000000000001e-05 2023-09-23T23:44:27.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:07.520151+0000", 2023-09-23T23:44:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022001350000000002 2023-09-23T23:44:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:07.520350+0000", 2023-09-23T23:44:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019858399999999999 2023-09-23T23:44:27.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:07.520392+0000", 2023-09-23T23:44:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1981999999999999e-05 2023-09-23T23:44:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:07.520437+0000", 2023-09-23T23:44:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5111999999999998e-05 2023-09-23T23:44:27.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:07.520466+0000", 2023-09-23T23:44:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9315999999999998e-05 2023-09-23T23:44:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.38088.0:1 2.1 e715/692 2:9916b262:::rbd_data.94b38896dfdf.000000000000:head v 715'96, mlcod=715'94)", 2023-09-23T23:44:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:44:08.726090+0000", 2023-09-23T23:44:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 18.526695234000002, 2023-09-23T23:44:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0091944350000000008, 2023-09-23T23:44:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:27.451 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.726090+0000", 2023-09-23T23:44:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.726090+0000", 2023-09-23T23:44:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:27.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.726089+0000", 2023-09-23T23:44:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:44:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.728349+0000", 2023-09-23T23:44:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022596259999999998 2023-09-23T23:44:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.728377+0000", 2023-09-23T23:44:27.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8050000000000001e-05 2023-09-23T23:44:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.728385+0000", 2023-09-23T23:44:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9680000000000006e-06 2023-09-23T23:44:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.728443+0000", 2023-09-23T23:44:27.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8102e-05 2023-09-23T23:44:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.728465+0000", 2023-09-23T23:44:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1815e-05 2023-09-23T23:44:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.735243+0000", 2023-09-23T23:44:27.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067782659999999998 2023-09-23T23:44:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.735284+0000", 2023-09-23T23:44:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1625999999999998e-05 2023-09-23T23:44:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.456 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:27.457 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:27.457 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:44:27.457 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:26 smithi138 ceph-mon[193606]: pgmap v2025: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 77 B/s, 0 objects/s recovering 2023-09-23T23:44:27.457 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:28.104 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:44:28.458 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:44:29.129 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:29 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:29.129 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:29 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:44:29.129 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:29 smithi110 ceph-mon[212939]: pgmap v2026: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 67 B/s, 0 objects/s recovering 2023-09-23T23:44:29.129 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:29 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:29.129 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:29 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:44:29.129 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:29 smithi110 ceph-mon[221102]: pgmap v2026: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 67 B/s, 0 objects/s recovering 2023-09-23T23:44:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:29 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:29 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:44:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:29 smithi138 ceph-mon[193606]: pgmap v2026: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 67 B/s, 0 objects/s recovering 2023-09-23T23:44:30.116 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:44:30.116 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:44:30.117 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-09-23T23:44:30.117 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:44:30.190 INFO:teuthology.orchestra.run.smithi110.stdout:moved 26 / 309 (8.41424%) 2023-09-23T23:44:30.190 INFO:teuthology.orchestra.run.smithi110.stdout:avg 51.5 2023-09-23T23:44:30.190 INFO:teuthology.orchestra.run.smithi110.stdout:stddev 4.71699 -> 9.46485 (expected baseline 6.55108) 2023-09-23T23:44:30.190 INFO:teuthology.orchestra.run.smithi110.stdout:min osd.2 with 41 -> 47 pgs (0.796117 -> 0.912621 * mean) 2023-09-23T23:44:30.190 INFO:teuthology.orchestra.run.smithi110.stdout:max osd.1 with 54 -> 56 pgs (1.04854 -> 1.08738 * mean) 2023-09-23T23:44:30.190 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:44:30.191 INFO:teuthology.orchestra.run.smithi110.stdout:oload 110 2023-09-23T23:44:30.191 INFO:teuthology.orchestra.run.smithi110.stdout:max_change 3 2023-09-23T23:44:30.191 INFO:teuthology.orchestra.run.smithi110.stdout:max_change_osds 4 2023-09-23T23:44:30.191 INFO:teuthology.orchestra.run.smithi110.stdout:average_utilization 0.0107 2023-09-23T23:44:30.191 INFO:teuthology.orchestra.run.smithi110.stdout:overload_utilization 0.0118 2023-09-23T23:44:30.191 INFO:teuthology.orchestra.run.smithi110.stdout:osd.0 weight 0.9000 -> 0.6437 2023-09-23T23:44:30.191 INFO:teuthology.orchestra.run.smithi110.stdout:osd.3 weight 0.9000 -> 0.7996 2023-09-23T23:44:30.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:30 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:30.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:30 smithi138 ceph-mon[193606]: from='client.38163 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:44:30.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:30 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'42187','3':'52401'}"}]: dispatch 2023-09-23T23:44:30.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:30 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'42187','3':'52401'}"}]: dispatch 2023-09-23T23:44:30.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:30 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:30.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:30 smithi110 ceph-mon[212939]: from='client.38163 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:44:30.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:30 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'42187','3':'52401'}"}]: dispatch 2023-09-23T23:44:30.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:30 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'42187','3':'52401'}"}]: dispatch 2023-09-23T23:44:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:30 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:30 smithi110 ceph-mon[221102]: from='client.38163 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:44:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:30 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'42187','3':'52401'}"}]: dispatch 2023-09-23T23:44:30.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:30 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'42187','3':'52401'}"}]: dispatch 2023-09-23T23:44:30.721 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:44:31.063 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:44:31.439 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:31 smithi138 ceph-mon[193606]: pgmap v2027: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 55 B/s, 0 objects/s recovering 2023-09-23T23:44:31.439 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:31 smithi138 ceph-mon[193606]: failed to encode map e729 with expected crc 2023-09-23T23:44:31.439 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:31 smithi138 ceph-mon[193606]: failed to encode map e729 with expected crc 2023-09-23T23:44:31.439 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:31 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'42187','3':'52401'}"}]': finished 2023-09-23T23:44:31.439 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:31 smithi138 ceph-mon[193606]: osdmap e729: 8 total, 7 up, 6 in 2023-09-23T23:44:31.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:31 smithi110 ceph-mon[212939]: pgmap v2027: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 55 B/s, 0 objects/s recovering 2023-09-23T23:44:31.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:31 smithi110 ceph-mon[212939]: failed to encode map e729 with expected crc 2023-09-23T23:44:31.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:31 smithi110 ceph-mon[212939]: failed to encode map e729 with expected crc 2023-09-23T23:44:31.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:31 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'42187','3':'52401'}"}]': finished 2023-09-23T23:44:31.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:31 smithi110 ceph-mon[212939]: osdmap e729: 8 total, 7 up, 6 in 2023-09-23T23:44:31.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:31 smithi110 ceph-mon[221102]: pgmap v2027: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 55 B/s, 0 objects/s recovering 2023-09-23T23:44:31.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:31 smithi110 ceph-mon[221102]: failed to encode map e729 with expected crc 2023-09-23T23:44:31.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:31 smithi110 ceph-mon[221102]: failed to encode map e729 with expected crc 2023-09-23T23:44:31.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:31 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'42187','3':'52401'}"}]': finished 2023-09-23T23:44:31.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:31 smithi110 ceph-mon[221102]: osdmap e729: 8 total, 7 up, 6 in 2023-09-23T23:44:32.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:32 smithi138 ceph-mon[193606]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.348 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:32 smithi138 ceph-mon[193606]: osdmap e730: 8 total, 7 up, 6 in 2023-09-23T23:44:32.348 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:32 smithi138 ceph-mon[193606]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.348 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:32 smithi138 ceph-mon[193606]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.348 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:32 smithi138 ceph-mon[193606]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.348 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:32 smithi138 ceph-mon[193606]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.348 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:32 smithi138 ceph-mon[193606]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.348 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:32 smithi138 ceph-mon[193606]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:32 smithi110 ceph-mon[212939]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:32 smithi110 ceph-mon[212939]: osdmap e730: 8 total, 7 up, 6 in 2023-09-23T23:44:32.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:32 smithi110 ceph-mon[212939]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:32 smithi110 ceph-mon[212939]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:32 smithi110 ceph-mon[212939]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:32 smithi110 ceph-mon[212939]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:32 smithi110 ceph-mon[212939]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:32 smithi110 ceph-mon[212939]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:32 smithi110 ceph-mon[221102]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:32 smithi110 ceph-mon[221102]: osdmap e730: 8 total, 7 up, 6 in 2023-09-23T23:44:32.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:32 smithi110 ceph-mon[221102]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:32 smithi110 ceph-mon[221102]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:32 smithi110 ceph-mon[221102]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:32 smithi110 ceph-mon[221102]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:32 smithi110 ceph-mon[221102]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:32 smithi110 ceph-mon[221102]: failed to encode map e730 with expected crc 2023-09-23T23:44:32.551 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:44:32.551 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:44:32.551 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:44:32.551 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:44:32.551 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:44:33.186 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:44:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:33 smithi138 ceph-mon[193606]: pgmap v2030: 103 pgs: 2 remapped+peering, 6 peering, 1 unknown, 94 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:33 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:44:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:33 smithi138 ceph-mon[193606]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:33 smithi138 ceph-mon[193606]: osdmap e731: 8 total, 7 up, 6 in 2023-09-23T23:44:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:33 smithi138 ceph-mon[193606]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:33 smithi138 ceph-mon[193606]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:33 smithi138 ceph-mon[193606]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:33 smithi138 ceph-mon[193606]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:33 smithi138 ceph-mon[193606]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:33 smithi138 ceph-mon[193606]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[212939]: pgmap v2030: 103 pgs: 2 remapped+peering, 6 peering, 1 unknown, 94 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:33.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:44:33.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[212939]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[212939]: osdmap e731: 8 total, 7 up, 6 in 2023-09-23T23:44:33.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[212939]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[212939]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[212939]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[212939]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[212939]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[212939]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[221102]: pgmap v2030: 103 pgs: 2 remapped+peering, 6 peering, 1 unknown, 94 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:44:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[221102]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[221102]: osdmap e731: 8 total, 7 up, 6 in 2023-09-23T23:44:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[221102]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[221102]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[221102]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[221102]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[221102]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:33 smithi110 ceph-mon[221102]: failed to encode map e731 with expected crc 2023-09-23T23:44:33.533 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:44:34.436 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:34 smithi138 ceph-mon[193606]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.436 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:34 smithi138 ceph-mon[193606]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.436 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:34 smithi138 ceph-mon[193606]: osdmap e732: 8 total, 7 up, 6 in 2023-09-23T23:44:34.436 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:34 smithi138 ceph-mon[193606]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.436 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:34 smithi138 ceph-mon[193606]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.437 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:34 smithi138 ceph-mon[193606]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.437 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:34 smithi138 ceph-mon[193606]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.437 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:34 smithi138 ceph-mon[193606]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:34 smithi110 ceph-mon[212939]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:34 smithi110 ceph-mon[212939]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:34 smithi110 ceph-mon[212939]: osdmap e732: 8 total, 7 up, 6 in 2023-09-23T23:44:34.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:34 smithi110 ceph-mon[212939]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:34 smithi110 ceph-mon[212939]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:34 smithi110 ceph-mon[212939]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:34 smithi110 ceph-mon[212939]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:34 smithi110 ceph-mon[212939]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:34 smithi110 ceph-mon[221102]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:34 smithi110 ceph-mon[221102]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:34 smithi110 ceph-mon[221102]: osdmap e732: 8 total, 7 up, 6 in 2023-09-23T23:44:34.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:34 smithi110 ceph-mon[221102]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:34 smithi110 ceph-mon[221102]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:34 smithi110 ceph-mon[221102]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:34 smithi110 ceph-mon[221102]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:34 smithi110 ceph-mon[221102]: failed to encode map e732 with expected crc 2023-09-23T23:44:34.925 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:44:34.925 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:44:34.925 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:44:34.925 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:44:34.925 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.925 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:89749 13.5 13:a310cdec:::benchmark_data_smithi110_541928_object1969:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:44:34.925 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:34.965229+0000", 2023-09-23T23:44:34.926 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 599.95610779100002, 2023-09-23T23:44:34.926 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0084807309999999, 2023-09-23T23:44:34.926 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:44:34.926 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:34.926 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:44:34.926 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:44:34.926 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:44:34.926 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 89749 2023-09-23T23:44:34.926 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.926 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:44:34.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.927 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:44:34.927 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.965229+0000", 2023-09-23T23:44:34.927 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.927 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:44:34.927 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.965229+0000", 2023-09-23T23:44:34.927 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.928 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.928 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:44:34.928 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.965228+0000", 2023-09-23T23:44:34.928 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:34.928 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.928 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.928 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:44:34.928 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.965235+0000", 2023-09-23T23:44:34.928 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6410000000000002e-06 2023-09-23T23:44:34.928 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.929 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.929 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:44:34.929 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.965252+0000", 2023-09-23T23:44:34.929 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6413e-05 2023-09-23T23:44:34.929 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.929 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.929 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:44:34.929 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.965258+0000", 2023-09-23T23:44:34.929 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5969999999999997e-06 2023-09-23T23:44:34.929 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.930 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.930 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:34.930 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.965334+0000", 2023-09-23T23:44:34.930 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5895000000000004e-05 2023-09-23T23:44:34.930 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.930 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.930 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:44:34.930 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:34.965350+0000", 2023-09-23T23:44:34.930 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5293000000000001e-05 2023-09-23T23:44:34.930 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.931 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.931 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:34.931 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.933637+0000", 2023-09-23T23:44:34.931 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96828758000000004 2023-09-23T23:44:34.931 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.931 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.931 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:44:34.931 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.933744+0000", 2023-09-23T23:44:34.931 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107054 2023-09-23T23:44:34.931 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.931 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.932 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:44:34.932 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.933858+0000", 2023-09-23T23:44:34.932 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000113535 2023-09-23T23:44:34.932 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.932 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.932 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:44:34.932 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.951233+0000", 2023-09-23T23:44:34.932 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017375096 2023-09-23T23:44:34.932 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.932 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.932 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:34.932 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.953760+0000", 2023-09-23T23:44:34.933 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025267269999999999 2023-09-23T23:44:34.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.933 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.933 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:34.933 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.969953+0000", 2023-09-23T23:44:34.933 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016193056000000001 2023-09-23T23:44:34.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.933 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.934 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:44:34.934 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.969994+0000", 2023-09-23T23:44:34.934 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1465999999999999e-05 2023-09-23T23:44:34.934 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.934 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.934 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:44:34.934 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.973710+0000", 2023-09-23T23:44:34.934 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037158709999999999 2023-09-23T23:44:34.934 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.934 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:44:34.935 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.935 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.935 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.935 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:89754 13.5 13:a28fe27c:::benchmark_data_smithi110_541928_object1974:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e570)", 2023-09-23T23:44:34.935 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:35.047672+0000", 2023-09-23T23:44:34.935 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 599.87366458700001, 2023-09-23T23:44:34.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92934757800000001, 2023-09-23T23:44:34.935 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:44:34.936 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:34.936 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:44:34.936 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:44:34.936 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:44:34.936 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 89754 2023-09-23T23:44:34.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.936 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:44:34.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.936 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:44:34.936 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.047672+0000", 2023-09-23T23:44:34.936 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:44:34.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.047672+0000", 2023-09-23T23:44:34.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.937 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:44:34.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.047671+0000", 2023-09-23T23:44:34.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:44:34.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.938 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:44:34.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.047679+0000", 2023-09-23T23:44:34.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5730000000000008e-06 2023-09-23T23:44:34.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.938 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:44:34.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.047698+0000", 2023-09-23T23:44:34.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8508999999999998e-05 2023-09-23T23:44:34.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.939 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:44:34.939 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.047708+0000", 2023-09-23T23:44:34.939 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0369e-05 2023-09-23T23:44:34.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.939 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:34.939 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.047763+0000", 2023-09-23T23:44:34.939 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4969999999999997e-05 2023-09-23T23:44:34.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:44:34.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.047776+0000", 2023-09-23T23:44:34.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.29e-05 2023-09-23T23:44:34.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:34.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.936908+0000", 2023-09-23T23:44:34.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88913176000000005 2023-09-23T23:44:34.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.941 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:44:34.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.937081+0000", 2023-09-23T23:44:34.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017318999999999999 2023-09-23T23:44:34.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.941 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:44:34.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.937234+0000", 2023-09-23T23:44:34.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015310000000000001 2023-09-23T23:44:34.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.942 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.942 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:44:34.942 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.953643+0000", 2023-09-23T23:44:34.942 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016409353000000002 2023-09-23T23:44:34.942 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.942 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.942 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:34.942 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.953793+0000", 2023-09-23T23:44:34.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014953000000000001 2023-09-23T23:44:34.943 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.943 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:34.943 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.973759+0000", 2023-09-23T23:44:34.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019966133 2023-09-23T23:44:34.943 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.943 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:44:34.943 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.973801+0000", 2023-09-23T23:44:34.944 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1754e-05 2023-09-23T23:44:34.944 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.944 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.944 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:44:34.944 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:35.977020+0000", 2023-09-23T23:44:34.944 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032191400000000001 2023-09-23T23:44:34.944 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.944 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:44:34.945 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.945 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.945 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:95946 13.6 13:64dac656:::benchmark_data_smithi110_541928_object8166:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e579)", 2023-09-23T23:44:34.945 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:57.359617+0000", 2023-09-23T23:44:34.945 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 577.56172021299994, 2023-09-23T23:44:34.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98724013700000002, 2023-09-23T23:44:34.945 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:44:34.945 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:34.945 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:44:34.945 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:44:34.945 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:44:34.946 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95946 2023-09-23T23:44:34.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.946 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:44:34.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.946 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:44:34.946 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.359617+0000", 2023-09-23T23:44:34.946 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.946 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:44:34.946 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.359617+0000", 2023-09-23T23:44:34.946 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:44:34.947 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.359615+0000", 2023-09-23T23:44:34.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:34.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:44:34.947 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.359622+0000", 2023-09-23T23:44:34.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1049999999999997e-06 2023-09-23T23:44:34.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:44:34.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.359640+0000", 2023-09-23T23:44:34.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7756000000000001e-05 2023-09-23T23:44:34.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:44:34.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.359647+0000", 2023-09-23T23:44:34.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8600000000000004e-06 2023-09-23T23:44:34.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:34.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.359718+0000", 2023-09-23T23:44:34.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0860999999999998e-05 2023-09-23T23:44:34.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:44:34.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.359733+0000", 2023-09-23T23:44:34.949 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5349000000000002e-05 2023-09-23T23:44:34.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:34.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.338363+0000", 2023-09-23T23:44:34.949 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97862939999999998 2023-09-23T23:44:34.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:44:34.950 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.338460+0000", 2023-09-23T23:44:34.950 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6921000000000003e-05 2023-09-23T23:44:34.950 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.950 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.950 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:44:34.950 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.338540+0000", 2023-09-23T23:44:34.950 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9913999999999994e-05 2023-09-23T23:44:34.950 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.950 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.950 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:34.950 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.342614+0000", 2023-09-23T23:44:34.950 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040745290000000003 2023-09-23T23:44:34.950 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.951 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.951 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:34.951 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.342633+0000", 2023-09-23T23:44:34.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8618999999999999e-05 2023-09-23T23:44:34.951 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.951 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.951 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:44:34.951 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.346789+0000", 2023-09-23T23:44:34.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041560929999999996 2023-09-23T23:44:34.951 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.951 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.951 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:44:34.952 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.346827+0000", 2023-09-23T23:44:34.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8006000000000002e-05 2023-09-23T23:44:34.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.952 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.952 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:44:34.952 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.346857+0000", 2023-09-23T23:44:34.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0154e-05 2023-09-23T23:44:34.952 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.952 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:44:34.952 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.953 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:95960 13.6 13:605ca08f:::benchmark_data_smithi110_541928_object8180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e579)", 2023-09-23T23:44:34.953 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:34:57.442651+0000", 2023-09-23T23:44:34.953 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 577.478685552, 2023-09-23T23:44:34.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.90870927599999995, 2023-09-23T23:44:34.953 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:44:34.953 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:34.953 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:44:34.953 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:44:34.953 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:44:34.953 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95960 2023-09-23T23:44:34.953 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.954 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:44:34.954 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.954 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:44:34.954 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.442651+0000", 2023-09-23T23:44:34.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.954 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.954 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.954 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:44:34.954 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.442651+0000", 2023-09-23T23:44:34.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.954 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.954 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.954 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:44:34.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.442650+0000", 2023-09-23T23:44:34.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:34.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:44:34.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.442657+0000", 2023-09-23T23:44:34.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2540000000000003e-06 2023-09-23T23:44:34.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:44:34.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.442671+0000", 2023-09-23T23:44:34.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3985000000000001e-05 2023-09-23T23:44:34.956 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.956 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:44:34.956 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.442678+0000", 2023-09-23T23:44:34.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5749999999999997e-06 2023-09-23T23:44:34.956 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.956 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:34.956 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.442740+0000", 2023-09-23T23:44:34.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1989000000000006e-05 2023-09-23T23:44:34.956 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.956 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-09-23T23:44:34.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:57.442750+0000", 2023-09-23T23:44:34.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0499000000000001e-05 2023-09-23T23:44:34.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:34.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.341144+0000", 2023-09-23T23:44:34.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.89839374900000002 2023-09-23T23:44:34.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:44:34.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.341231+0000", 2023-09-23T23:44:34.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7334000000000001e-05 2023-09-23T23:44:34.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-09-23T23:44:34.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.341295+0000", 2023-09-23T23:44:34.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3958e-05 2023-09-23T23:44:34.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:34.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.348387+0000", 2023-09-23T23:44:34.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0070911200000000002 2023-09-23T23:44:34.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:44:34.959 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.348458+0000", 2023-09-23T23:44:34.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1619000000000007e-05 2023-09-23T23:44:34.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.959 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.959 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:34.959 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.351293+0000", 2023-09-23T23:44:34.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028346270000000002 2023-09-23T23:44:34.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.959 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.959 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:44:34.959 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.351325+0000", 2023-09-23T23:44:34.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2329999999999997e-05 2023-09-23T23:44:34.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:44:34.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:34:58.351361+0000", 2023-09-23T23:44:34.960 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5657000000000001e-05 2023-09-23T23:44:34.960 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.960 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:44:34.960 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.960 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:153444 13.e 13:7c68890f:::benchmark_data_smithi110_541928_object65664:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-23T23:44:34.961 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:38:16.592187+0000", 2023-09-23T23:44:34.961 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 378.32915047900002, 2023-09-23T23:44:34.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.29196677500000001, 2023-09-23T23:44:34.961 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:44:34.961 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:34.961 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:44:34.961 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:44:34.961 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:44:34.961 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 153444 2023-09-23T23:44:34.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.961 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:44:34.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:44:34.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592187+0000", 2023-09-23T23:44:34.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:44:34.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592187+0000", 2023-09-23T23:44:34.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:44:34.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592185+0000", 2023-09-23T23:44:34.963 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:34.963 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.963 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.963 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:44:34.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592192+0000", 2023-09-23T23:44:34.963 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5209999999999997e-06 2023-09-23T23:44:34.963 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.963 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.963 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:44:34.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592208+0000", 2023-09-23T23:44:34.963 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5746000000000001e-05 2023-09-23T23:44:34.963 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.964 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.964 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:44:34.964 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592215+0000", 2023-09-23T23:44:34.964 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5490000000000003e-06 2023-09-23T23:44:34.964 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.964 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.964 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:34.964 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592370+0000", 2023-09-23T23:44:34.964 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015496499999999999 2023-09-23T23:44:34.964 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.964 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.964 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:44:34.964 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592515+0000", 2023-09-23T23:44:34.965 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000145771 2023-09-23T23:44:34.965 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.965 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.965 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:44:34.965 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.592664+0000", 2023-09-23T23:44:34.965 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014883999999999999 2023-09-23T23:44:34.965 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.965 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.965 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:44:34.965 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.602612+0000", 2023-09-23T23:44:34.965 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0099480739999999995 2023-09-23T23:44:34.965 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.966 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.966 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:34.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.602670+0000", 2023-09-23T23:44:34.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7840000000000002e-05 2023-09-23T23:44:34.966 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.966 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.966 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:34.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.882423+0000", 2023-09-23T23:44:34.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.27975320599999998 2023-09-23T23:44:34.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:44:34.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.882477+0000", 2023-09-23T23:44:34.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3607999999999999e-05 2023-09-23T23:44:34.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:44:34.968 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.884153+0000", 2023-09-23T23:44:34.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001676262 2023-09-23T23:44:34.968 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.968 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:44:34.968 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.968 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46514.0:153449 13.e 13:7decf6a5:::benchmark_data_smithi110_541928_object65669:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-23T23:44:34.968 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:38:16.597143+0000", 2023-09-23T23:44:34.968 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 378.32419446799997, 2023-09-23T23:44:34.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.28863325099999998, 2023-09-23T23:44:34.969 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:44:34.969 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:34.969 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:44:34.969 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46514", 2023-09-23T23:44:34.969 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/17031442", 2023-09-23T23:44:34.969 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 153449 2023-09-23T23:44:34.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.969 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:44:34.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:44:34.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.597143+0000", 2023-09-23T23:44:34.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:44:34.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.597143+0000", 2023-09-23T23:44:34.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:44:34.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.597141+0000", 2023-09-23T23:44:34.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:44:34.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:44:34.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.597148+0000", 2023-09-23T23:44:34.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5070000000000003e-06 2023-09-23T23:44:34.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:44:34.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.597161+0000", 2023-09-23T23:44:34.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2843000000000001e-05 2023-09-23T23:44:34.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:44:34.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.597167+0000", 2023-09-23T23:44:34.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2140000000000001e-06 2023-09-23T23:44:34.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:34.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.599181+0000", 2023-09-23T23:44:34.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002014007 2023-09-23T23:44:34.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:44:34.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.599369+0000", 2023-09-23T23:44:34.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018732800000000001 2023-09-23T23:44:34.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:44:34.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.599530+0000", 2023-09-23T23:44:34.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000161461 2023-09-23T23:44:34.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:34.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.605636+0000", 2023-09-23T23:44:34.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0061053599999999998 2023-09-23T23:44:34.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:44:34.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.605643+0000", 2023-09-23T23:44:34.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2609999999999996e-06 2023-09-23T23:44:34.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.974 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:34.974 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.884203+0000", 2023-09-23T23:44:34.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.27855972499999998 2023-09-23T23:44:34.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.974 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:44:34.974 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.884253+0000", 2023-09-23T23:44:34.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0427999999999999e-05 2023-09-23T23:44:34.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.975 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:44:34.975 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.885776+0000", 2023-09-23T23:44:34.975 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001522816 2023-09-23T23:44:34.975 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.975 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:44:34.975 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.975 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.975 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.975 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.47186.0:36258 14.7 14:e0ceb5cd:::benchmark_data_smithi110_640334_object36257:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T23:44:34.975 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:39:44.854122+0000", 2023-09-23T23:44:34.975 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 290.06721464999998, 2023-09-23T23:44:34.975 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.36212670400000002, 2023-09-23T23:44:34.975 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:44:34.976 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:34.976 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:44:34.976 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.47186", 2023-09-23T23:44:34.976 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:34.976 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 36258 2023-09-23T23:44:34.976 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.976 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:44:34.976 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.976 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:44:34.976 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854122+0000", 2023-09-23T23:44:34.976 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.976 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:44:34.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854122+0000", 2023-09-23T23:44:34.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:44:34.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854121+0000", 2023-09-23T23:44:34.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:34.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:44:34.978 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854177+0000", 2023-09-23T23:44:34.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5659999999999999e-05 2023-09-23T23:44:34.978 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:44:34.978 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854189+0000", 2023-09-23T23:44:34.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2468e-05 2023-09-23T23:44:34.978 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:44:34.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854199+0000", 2023-09-23T23:44:34.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8570000000000007e-06 2023-09-23T23:44:34.979 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.979 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.979 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:34.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854258+0000", 2023-09-23T23:44:34.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9049000000000003e-05 2023-09-23T23:44:34.979 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.979 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.979 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:44:34.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854381+0000", 2023-09-23T23:44:34.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000122373 2023-09-23T23:44:34.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.980 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:44:34.980 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.854530+0000", 2023-09-23T23:44:34.980 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014989699999999999 2023-09-23T23:44:34.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.980 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:44:34.980 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.855861+0000", 2023-09-23T23:44:34.980 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0013305630000000001 2023-09-23T23:44:34.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:34.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:44.856075+0000", 2023-09-23T23:44:34.981 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000214195 2023-09-23T23:44:34.981 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.981 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:34.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:45.216123+0000", 2023-09-23T23:44:34.981 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.360047426 2023-09-23T23:44:34.981 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.981 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:44:34.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:45.216200+0000", 2023-09-23T23:44:34.981 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6913e-05 2023-09-23T23:44:34.981 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.982 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.982 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:44:34.982 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:45.216249+0000", 2023-09-23T23:44:34.982 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9527999999999997e-05 2023-09-23T23:44:34.982 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.982 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:44:34.982 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.982 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.982 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.982 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.47186.0:37504 14.7 14:e20c8f2d:::benchmark_data_smithi110_640334_object37503:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T23:44:34.982 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:39:46.194924+0000", 2023-09-23T23:44:34.983 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 288.72641341600001, 2023-09-23T23:44:34.983 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22139715600000001, 2023-09-23T23:44:34.983 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:44:34.983 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:34.983 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:44:34.983 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.47186", 2023-09-23T23:44:34.983 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:34.983 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 37504 2023-09-23T23:44:34.983 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.983 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:44:34.983 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.983 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:44:34.984 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.194924+0000", 2023-09-23T23:44:34.984 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.984 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.984 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.984 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:44:34.984 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.194924+0000", 2023-09-23T23:44:34.984 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.984 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.984 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.984 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:44:34.984 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.194923+0000", 2023-09-23T23:44:34.984 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:34.985 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.985 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.985 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:44:34.985 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.194980+0000", 2023-09-23T23:44:34.985 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6762999999999998e-05 2023-09-23T23:44:34.985 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.985 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.985 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:44:34.985 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411523+0000", 2023-09-23T23:44:34.985 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21654374000000001 2023-09-23T23:44:34.985 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.985 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.985 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:44:34.986 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411534+0000", 2023-09-23T23:44:34.986 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0794e-05 2023-09-23T23:44:34.986 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.986 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.986 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:34.986 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411576+0000", 2023-09-23T23:44:34.986 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1700999999999999e-05 2023-09-23T23:44:34.986 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.986 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.986 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:44:34.986 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411702+0000", 2023-09-23T23:44:34.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000126239 2023-09-23T23:44:34.987 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.987 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.987 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:44:34.987 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411827+0000", 2023-09-23T23:44:34.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000124991 2023-09-23T23:44:34.987 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.987 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.987 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:44:34.987 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.415986+0000", 2023-09-23T23:44:34.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041592870000000002 2023-09-23T23:44:34.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.988 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:34.988 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.416043+0000", 2023-09-23T23:44:34.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6314000000000003e-05 2023-09-23T23:44:34.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.988 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:34.988 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.416222+0000", 2023-09-23T23:44:34.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017949799999999999 2023-09-23T23:44:34.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.988 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:44:34.988 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.416279+0000", 2023-09-23T23:44:34.989 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6972000000000003e-05 2023-09-23T23:44:34.989 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.989 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:44:34.989 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.416321+0000", 2023-09-23T23:44:34.989 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1499999999999999e-05 2023-09-23T23:44:34.989 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.989 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:44:34.989 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.989 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.989 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.47186.0:37498 14.a e625/617 14:56947ea7:::benchmark_data_smithi110_640334_object37497:head v 625'2249, mlcod=625'2247)", 2023-09-23T23:44:34.990 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:39:46.196021+0000", 2023-09-23T23:44:34.990 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 288.72531605500001, 2023-09-23T23:44:34.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21694238800000001, 2023-09-23T23:44:34.990 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:44:34.990 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:34.990 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:44:34.990 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.990 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:44:34.990 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.196021+0000", 2023-09-23T23:44:34.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.990 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.990 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.991 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:44:34.991 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.196021+0000", 2023-09-23T23:44:34.991 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.991 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.991 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.991 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:44:34.991 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.196020+0000", 2023-09-23T23:44:34.991 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:34.991 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.991 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.991 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:44:34.991 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.196065+0000", 2023-09-23T23:44:34.991 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4861999999999998e-05 2023-09-23T23:44:34.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:44:34.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411523+0000", 2023-09-23T23:44:34.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21545887599999999 2023-09-23T23:44:34.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:44:34.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411533+0000", 2023-09-23T23:44:34.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5980000000000003e-06 2023-09-23T23:44:34.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:34.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411583+0000", 2023-09-23T23:44:34.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9599999999999999e-05 2023-09-23T23:44:34.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:44:34.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411604+0000", 2023-09-23T23:44:34.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1741000000000001e-05 2023-09-23T23:44:34.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:44:34.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.412924+0000", 2023-09-23T23:44:34.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0013196600000000001 2023-09-23T23:44:34.994 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.994 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:44:34.994 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.412963+0000", 2023-09-23T23:44:34.994 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9288000000000001e-05 2023-09-23T23:44:34.994 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.994 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:44:34.994 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.994 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.994 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.47186.0:37512 14.0 e625/617 14:07e6ea4d:::benchmark_data_smithi110_640334_object37511:head v 625'2372, mlcod=625'2370)", 2023-09-23T23:44:34.995 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:39:46.208752+0000", 2023-09-23T23:44:34.995 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 288.71258464200002, 2023-09-23T23:44:34.995 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20513990300000001, 2023-09-23T23:44:34.995 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:44:34.995 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:34.995 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:44:34.995 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.995 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:44:34.995 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.208752+0000", 2023-09-23T23:44:34.995 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.996 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:44:34.996 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.208752+0000", 2023-09-23T23:44:34.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:34.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.996 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:44:34.996 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.208751+0000", 2023-09-23T23:44:34.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:34.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:44:34.997 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.208799+0000", 2023-09-23T23:44:34.997 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8372999999999999e-05 2023-09-23T23:44:34.997 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.997 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:44:34.997 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411548+0000", 2023-09-23T23:44:34.997 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20274924699999999 2023-09-23T23:44:34.997 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.997 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:44:34.997 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411556+0000", 2023-09-23T23:44:34.997 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4730000000000003e-06 2023-09-23T23:44:34.998 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.998 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.998 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:34.998 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411600+0000", 2023-09-23T23:44:34.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.375e-05 2023-09-23T23:44:34.998 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.998 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.998 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:44:34.998 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.411617+0000", 2023-09-23T23:44:34.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7363e-05 2023-09-23T23:44:34.998 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.998 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.998 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:44:34.999 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.413860+0000", 2023-09-23T23:44:34.999 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022428890000000001 2023-09-23T23:44:34.999 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:34.999 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:34.999 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:44:34.999 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:46.413892+0000", 2023-09-23T23:44:34.999 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2335000000000001e-05 2023-09-23T23:44:34.999 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.999 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:44:34.999 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:34.999 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.000 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.000 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.47186.0:159302 14.7 14:ecb3d147:::benchmark_data_smithi110_640334_object75423:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e678)", 2023-09-23T23:44:35.000 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:42:54.233777+0000", 2023-09-23T23:44:35.000 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 100.68755982099999, 2023-09-23T23:44:35.000 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.249138268, 2023-09-23T23:44:35.000 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:44:35.000 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:35.000 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:44:35.000 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.47186", 2023-09-23T23:44:35.000 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:35.000 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 159302 2023-09-23T23:44:35.000 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.000 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:44:35.001 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:44:35.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:54.233777+0000", 2023-09-23T23:44:35.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:35.001 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.001 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:44:35.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:54.233777+0000", 2023-09-23T23:44:35.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:35.001 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.001 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:44:35.002 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:54.233776+0000", 2023-09-23T23:44:35.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:44:35.002 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.002 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.002 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:44:35.002 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:54.233781+0000", 2023-09-23T23:44:35.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2739999999999998e-06 2023-09-23T23:44:35.002 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.002 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.002 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:44:35.002 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:54.233792+0000", 2023-09-23T23:44:35.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1009000000000001e-05 2023-09-23T23:44:35.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.003 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.003 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:44:35.003 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:54.233797+0000", 2023-09-23T23:44:35.003 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3800000000000004e-06 2023-09-23T23:44:35.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.003 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.003 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:35.003 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:54.233842+0000", 2023-09-23T23:44:35.003 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5006000000000002e-05 2023-09-23T23:44:35.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.003 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.003 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:44:35.004 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:54.233944+0000", 2023-09-23T23:44:35.004 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010202199999999999 2023-09-23T23:44:35.004 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.004 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.004 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:44:35.004 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:54.234029+0000", 2023-09-23T23:44:35.004 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5314000000000003e-05 2023-09-23T23:44:35.004 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.004 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.004 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:44:35.004 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:54.236823+0000", 2023-09-23T23:44:35.004 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0027939100000000001 2023-09-23T23:44:35.004 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.005 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:35.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:54.237015+0000", 2023-09-23T23:44:35.005 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019181600000000001 2023-09-23T23:44:35.005 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.005 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:35.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:54.482824+0000", 2023-09-23T23:44:35.005 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24580887000000001 2023-09-23T23:44:35.005 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.005 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:44:35.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:54.482866+0000", 2023-09-23T23:44:35.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2648999999999998e-05 2023-09-23T23:44:35.006 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.006 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:44:35.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:42:54.482915+0000", 2023-09-23T23:44:35.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9138999999999998e-05 2023-09-23T23:44:35.006 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:35.006 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:44:35.006 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:35.006 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.006 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.006 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.47594.0:2 2.2 2:5cea7035:::rbd_info:head [call rbd.metadata_list in=16b] snapc 0=[] ondisk+read+known_if_redirected+full_try+supports_pool_eio e704)", 2023-09-23T23:44:35.007 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:43:54.993108+0000", 2023-09-23T23:44:35.007 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 39.928228957000002, 2023-09-23T23:44:35.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040669429999999999, 2023-09-23T23:44:35.007 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:44:35.007 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:44:35.007 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:44:35.007 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.47594", 2023-09-23T23:44:35.007 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2819178997", 2023-09-23T23:44:35.007 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 2 2023-09-23T23:44:35.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.007 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:44:35.007 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:44:35.008 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:54.993108+0000", 2023-09-23T23:44:35.008 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:35.008 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:44:35.008 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:54.993108+0000", 2023-09-23T23:44:35.008 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:35.008 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:44:35.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:54.993107+0000", 2023-09-23T23:44:35.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:35.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.009 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:44:35.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:54.993112+0000", 2023-09-23T23:44:35.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6069999999999998e-06 2023-09-23T23:44:35.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.009 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:44:35.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:54.993123+0000", 2023-09-23T23:44:35.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1229999999999999e-05 2023-09-23T23:44:35.010 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.010 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:44:35.010 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:54.993138+0000", 2023-09-23T23:44:35.010 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4718999999999999e-05 2023-09-23T23:44:35.010 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.010 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:35.010 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:54.993220+0000", 2023-09-23T23:44:35.010 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1661000000000003e-05 2023-09-23T23:44:35.010 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.010 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op must wait for map", 2023-09-23T23:44:35.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:54.993228+0000", 2023-09-23T23:44:35.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1210000000000007e-06 2023-09-23T23:44:35.011 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:35.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:54.996947+0000", 2023-09-23T23:44:35.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037193500000000002 2023-09-23T23:44:35.011 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:44:35.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:54.997046+0000", 2023-09-23T23:44:35.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8653000000000006e-05 2023-09-23T23:44:35.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.012 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:44:35.012 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:54.997175+0000", 2023-09-23T23:44:35.012 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000129152 2023-09-23T23:44:35.012 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:35.012 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:44:35.012 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:35.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.012 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.47912.0:10 2.2 2:5cea7035:::rbd_info:head [call rbd.metadata_list in=16b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e715)", 2023-09-23T23:44:35.012 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:06.766724+0000", 2023-09-23T23:44:35.012 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 28.154613300000001, 2023-09-23T23:44:35.013 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0044177460000000002, 2023-09-23T23:44:35.013 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:44:35.013 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:44:35.013 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:44:35.013 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.47912", 2023-09-23T23:44:35.013 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1701361345", 2023-09-23T23:44:35.013 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 10 2023-09-23T23:44:35.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.013 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:44:35.013 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.013 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:44:35.013 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.766724+0000", 2023-09-23T23:44:35.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:35.014 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:44:35.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.766724+0000", 2023-09-23T23:44:35.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:35.014 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:44:35.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.766723+0000", 2023-09-23T23:44:35.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:44:35.014 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.015 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:44:35.015 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.766727+0000", 2023-09-23T23:44:35.015 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4290000000000003e-06 2023-09-23T23:44:35.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.015 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:44:35.015 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.766735+0000", 2023-09-23T23:44:35.015 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4039999999999995e-06 2023-09-23T23:44:35.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.015 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:44:35.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.766741+0000", 2023-09-23T23:44:35.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4600000000000002e-06 2023-09-23T23:44:35.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:35.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.766801+0000", 2023-09-23T23:44:35.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9815000000000002e-05 2023-09-23T23:44:35.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op must wait for map", 2023-09-23T23:44:35.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.766805+0000", 2023-09-23T23:44:35.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6210000000000003e-06 2023-09-23T23:44:35.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.017 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:35.017 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.770935+0000", 2023-09-23T23:44:35.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041299939999999997 2023-09-23T23:44:35.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.017 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:44:35.017 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.771031+0000", 2023-09-23T23:44:35.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5985999999999999e-05 2023-09-23T23:44:35.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.018 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:44:35.018 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.771141+0000", 2023-09-23T23:44:35.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110097 2023-09-23T23:44:35.018 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:35.018 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:44:35.018 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:35.018 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.018 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.018 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.38121.0:36 2.2 2:5cea7035:::rbd_info:head [call rbd.metadata_list in=16b] snapc 0=[] ondisk+read+known_if_redirected+full_try+supports_pool_eio e716)", 2023-09-23T23:44:35.018 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:08.975167+0000", 2023-09-23T23:44:35.018 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 25.946169542, 2023-09-23T23:44:35.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043619890000000001, 2023-09-23T23:44:35.019 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:44:35.019 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:44:35.019 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:44:35.019 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.38121", 2023-09-23T23:44:35.019 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1147984989", 2023-09-23T23:44:35.019 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 36 2023-09-23T23:44:35.019 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.019 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:44:35.019 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.019 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:44:35.019 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.975167+0000", 2023-09-23T23:44:35.019 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:35.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.020 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:44:35.020 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.975167+0000", 2023-09-23T23:44:35.020 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:44:35.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.020 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:44:35.020 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.975166+0000", 2023-09-23T23:44:35.020 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:35.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:44:35.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.975173+0000", 2023-09-23T23:44:35.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3799999999999999e-06 2023-09-23T23:44:35.021 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.021 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:44:35.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.975187+0000", 2023-09-23T23:44:35.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4183e-05 2023-09-23T23:44:35.021 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.021 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:44:35.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.975209+0000", 2023-09-23T23:44:35.022 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2575999999999999e-05 2023-09-23T23:44:35.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.022 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:35.022 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.975258+0000", 2023-09-23T23:44:35.022 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8560999999999998e-05 2023-09-23T23:44:35.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.022 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op must wait for map", 2023-09-23T23:44:35.022 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.975263+0000", 2023-09-23T23:44:35.022 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6909999999999996e-06 2023-09-23T23:44:35.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.023 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:44:35.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.979337+0000", 2023-09-23T23:44:35.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004074846 2023-09-23T23:44:35.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.023 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:44:35.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.979438+0000", 2023-09-23T23:44:35.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010094699999999999 2023-09-23T23:44:35.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:44:35.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:44:35.024 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:44:35.024 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.979529+0000", 2023-09-23T23:44:35.024 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1095999999999997e-05 2023-09-23T23:44:35.024 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:35.024 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:44:35.024 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:35.024 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:44:35.024 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:44:35.024 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:44:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:35 smithi138 ceph-mon[193606]: failed to encode map e732 with expected crc 2023-09-23T23:44:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:35 smithi138 ceph-mon[193606]: pgmap v2033: 103 pgs: 1 remapped+peering, 11 peering, 1 unknown, 90 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s, 3 objects/s recovering 2023-09-23T23:44:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:35 smithi138 ceph-mon[193606]: failed to encode map e733 with expected crc 2023-09-23T23:44:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:35 smithi138 ceph-mon[193606]: osdmap e733: 8 total, 7 up, 6 in 2023-09-23T23:44:35.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:35 smithi138 ceph-mon[193606]: failed to encode map e733 with expected crc 2023-09-23T23:44:35.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:35 smithi138 ceph-mon[193606]: failed to encode map e733 with expected crc 2023-09-23T23:44:35.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:35 smithi138 ceph-mon[193606]: failed to encode map e733 with expected crc 2023-09-23T23:44:35.552 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:44:35.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:35 smithi110 ceph-mon[212939]: failed to encode map e732 with expected crc 2023-09-23T23:44:35.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:35 smithi110 ceph-mon[212939]: pgmap v2033: 103 pgs: 1 remapped+peering, 11 peering, 1 unknown, 90 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s, 3 objects/s recovering 2023-09-23T23:44:35.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:35 smithi110 ceph-mon[212939]: failed to encode map e733 with expected crc 2023-09-23T23:44:35.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:35 smithi110 ceph-mon[212939]: osdmap e733: 8 total, 7 up, 6 in 2023-09-23T23:44:35.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:35 smithi110 ceph-mon[212939]: failed to encode map e733 with expected crc 2023-09-23T23:44:35.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:35 smithi110 ceph-mon[212939]: failed to encode map e733 with expected crc 2023-09-23T23:44:35.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:35 smithi110 ceph-mon[212939]: failed to encode map e733 with expected crc 2023-09-23T23:44:35.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:35 smithi110 ceph-mon[221102]: failed to encode map e732 with expected crc 2023-09-23T23:44:35.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:35 smithi110 ceph-mon[221102]: pgmap v2033: 103 pgs: 1 remapped+peering, 11 peering, 1 unknown, 90 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s, 3 objects/s recovering 2023-09-23T23:44:35.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:35 smithi110 ceph-mon[221102]: failed to encode map e733 with expected crc 2023-09-23T23:44:35.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:35 smithi110 ceph-mon[221102]: osdmap e733: 8 total, 7 up, 6 in 2023-09-23T23:44:35.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:35 smithi110 ceph-mon[221102]: failed to encode map e733 with expected crc 2023-09-23T23:44:35.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:35 smithi110 ceph-mon[221102]: failed to encode map e733 with expected crc 2023-09-23T23:44:35.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:35 smithi110 ceph-mon[221102]: failed to encode map e733 with expected crc 2023-09-23T23:44:35.894 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:44:35.915 INFO:tasks.thrashosds.thrasher:reviving osd 2023-09-23T23:44:35.915 INFO:tasks.thrashosds.thrasher:Reviving osd 4 2023-09-23T23:44:35.915 INFO:tasks.cephadm.osd.4:Restarting osd.4 (starting--it wasn't running)... 2023-09-23T23:44:35.915 DEBUG:teuthology.orchestra.run.smithi138:osd.4> sudo journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.4.service 2023-09-23T23:44:35.917 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl start ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.4 2023-09-23T23:44:35.945 INFO:journalctl@ceph.osd.4.smithi138.stdout:-- Logs begin at Sat 2023-09-23 22:31:02 UTC. -- 2023-09-23T23:44:36.192 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:44:35 smithi138 systemd[1]: Starting Ceph osd.4 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:44:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:36 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T23:44:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:36 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:44:36.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:36 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T23:44:36.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:36 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:44:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:36 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T23:44:36.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:36 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:44:37.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:44:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:44:36] "GET /metrics HTTP/1.1" 200 34192 "" "Prometheus/2.43.0" 2023-09-23T23:44:37.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:37 smithi138 ceph-mon[193606]: pgmap v2035: 103 pgs: 5 peering, 98 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 331 KiB/s rd, 84 KiB/s wr, 217 op/s; 1.0 KiB/s, 6 objects/s recovering 2023-09-23T23:44:37.555 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:44:37 smithi138 bash[420887]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T23:44:37.555 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:44:37 smithi138 bash[420887]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-09-23T23:44:37.555 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:44:37 smithi138 bash[420887]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2023-09-23T23:44:37.556 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:44:37 smithi138 bash[420887]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2023-09-23T23:44:37.556 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:44:37 smithi138 bash[420887]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2023-09-23T23:44:37.556 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:44:37 smithi138 bash[420887]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T23:44:37.556 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:44:37 smithi138 bash[420887]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-09-23T23:44:37.556 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:44:37 smithi138 bash[420887]: --> ceph-volume lvm activate successful for osd ID: 4 2023-09-23T23:44:37.556 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:44:37.556 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:44:37.556 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-09-23T23:44:37.557 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:44:37.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:37 smithi110 ceph-mon[212939]: pgmap v2035: 103 pgs: 5 peering, 98 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 331 KiB/s rd, 84 KiB/s wr, 217 op/s; 1.0 KiB/s, 6 objects/s recovering 2023-09-23T23:44:37.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:37 smithi110 ceph-mon[221102]: pgmap v2035: 103 pgs: 5 peering, 98 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 331 KiB/s rd, 84 KiB/s wr, 217 op/s; 1.0 KiB/s, 6 objects/s recovering 2023-09-23T23:44:38.643 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:44:39.059 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:44:39.172 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:44:39.378 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:44:39 smithi138 bash[421498]: b664c4d251ad92701b68d043ba4b697be6110f9690a87ca5eecd6bed010482a1 2023-09-23T23:44:39.378 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:44:39 smithi138 ceph-osd[421615]: -- 172.21.15.138:0/847499018 <== mon.1 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56132576ab60 con 0x561325730c00 2023-09-23T23:44:39.378 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:44:39 smithi138 systemd[1]: Started Ceph osd.4 for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:44:39.378 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:39 smithi138 ceph-mon[193606]: pgmap v2036: 103 pgs: 5 peering, 98 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 273 KiB/s rd, 69 KiB/s wr, 179 op/s; 881 B/s, 4 objects/s recovering 2023-09-23T23:44:39.692 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:44:39 smithi138 conmon[421553]: 2023-09-23T23:44:39.579+0000 7f11a6852540 -1 Falling back to public interface 2023-09-23T23:44:39.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:39 smithi110 ceph-mon[212939]: pgmap v2036: 103 pgs: 5 peering, 98 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 273 KiB/s rd, 69 KiB/s wr, 179 op/s; 881 B/s, 4 objects/s recovering 2023-09-23T23:44:39.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:39 smithi110 ceph-mon[221102]: pgmap v2036: 103 pgs: 5 peering, 98 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 273 KiB/s rd, 69 KiB/s wr, 179 op/s; 881 B/s, 4 objects/s recovering 2023-09-23T23:44:40.196 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:44:41.362 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:44:41.363 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:44:41.363 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:44:41.363 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:44:41.363 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:44:41.664 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:41 smithi138 ceph-mon[193606]: pgmap v2037: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 206 KiB/s rd, 52 KiB/s wr, 135 op/s; 664 B/s, 3 objects/s recovering 2023-09-23T23:44:41.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:41 smithi110 ceph-mon[212939]: pgmap v2037: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 206 KiB/s rd, 52 KiB/s wr, 135 op/s; 664 B/s, 3 objects/s recovering 2023-09-23T23:44:41.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:41 smithi110 ceph-mon[221102]: pgmap v2037: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 206 KiB/s rd, 52 KiB/s wr, 135 op/s; 664 B/s, 3 objects/s recovering 2023-09-23T23:44:41.892 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:44:42.353 INFO:teuthology.orchestra.run.smithi138.stderr:no valid command found; 10 closest matches: 2023-09-23T23:44:42.354 INFO:teuthology.orchestra.run.smithi138.stderr:0 2023-09-23T23:44:42.354 INFO:teuthology.orchestra.run.smithi138.stderr:1 2023-09-23T23:44:42.354 INFO:teuthology.orchestra.run.smithi138.stderr:2 2023-09-23T23:44:42.354 INFO:teuthology.orchestra.run.smithi138.stderr:abort 2023-09-23T23:44:42.354 INFO:teuthology.orchestra.run.smithi138.stderr:assert 2023-09-23T23:44:42.354 INFO:teuthology.orchestra.run.smithi138.stderr:bluefs debug_inject_read_zeros 2023-09-23T23:44:42.354 INFO:teuthology.orchestra.run.smithi138.stderr:bluefs files list 2023-09-23T23:44:42.354 INFO:teuthology.orchestra.run.smithi138.stderr:bluefs stats 2023-09-23T23:44:42.354 INFO:teuthology.orchestra.run.smithi138.stderr:bluestore bluefs device info [] 2023-09-23T23:44:42.354 INFO:teuthology.orchestra.run.smithi138.stderr:config diff 2023-09-23T23:44:42.355 INFO:teuthology.orchestra.run.smithi138.stderr:admin_socket: invalid command 2023-09-23T23:44:42.357 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:44:43.293 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:43 smithi138 ceph-mon[193606]: pgmap v2038: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 183 KiB/s rd, 46 KiB/s wr, 120 op/s; 590 B/s, 3 objects/s recovering 2023-09-23T23:44:43.294 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:44:43.295 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T23:44:43.295 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-4, ['dump_ops_in_flight'] 2023-09-23T23:44:43.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:43 smithi110 ceph-mon[212939]: pgmap v2038: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 183 KiB/s rd, 46 KiB/s wr, 120 op/s; 590 B/s, 3 objects/s recovering 2023-09-23T23:44:43.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:44:43.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:43 smithi110 ceph-mon[221102]: pgmap v2038: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 183 KiB/s rd, 46 KiB/s wr, 120 op/s; 590 B/s, 3 objects/s recovering 2023-09-23T23:44:43.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:44:44.306 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:44:44.306 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:44:44.306 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:44:44.307 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [] 2023-09-23T23:44:44.307 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:44:44.960 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:44:45.322 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:45.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:45 smithi110 ceph-mon[212939]: pgmap v2039: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 164 KiB/s rd, 42 KiB/s wr, 107 op/s; 528 B/s, 1 objects/s recovering 2023-09-23T23:44:45.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:45 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:45.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:45 smithi110 ceph-mon[221102]: pgmap v2039: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 164 KiB/s rd, 42 KiB/s wr, 107 op/s; 528 B/s, 1 objects/s recovering 2023-09-23T23:44:45.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:45 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:45.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:45 smithi138 ceph-mon[193606]: pgmap v2039: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 164 KiB/s rd, 42 KiB/s wr, 107 op/s; 528 B/s, 1 objects/s recovering 2023-09-23T23:44:45.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:45 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:44:46.823 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:44:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:44:46] "GET /metrics HTTP/1.1" 200 34199 "" "Prometheus/2.43.0" 2023-09-23T23:44:47.045 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:44:47.045 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:44:47.045 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-09-23T23:44:47.045 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:44:47.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:47 smithi110 ceph-mon[212939]: pgmap v2040: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 137 KiB/s rd, 35 KiB/s wr, 90 op/s; 442 B/s, 1 objects/s recovering 2023-09-23T23:44:47.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:47 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1d", "id": [5, 3]}]: dispatch 2023-09-23T23:44:47.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:47 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [1, 2]}]: dispatch 2023-09-23T23:44:47.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1d", "id": [5, 3]}]: dispatch 2023-09-23T23:44:47.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:47 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 0]}]: dispatch 2023-09-23T23:44:47.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [1, 2]}]: dispatch 2023-09-23T23:44:47.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:47 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 0]}]: dispatch 2023-09-23T23:44:47.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:47 smithi110 ceph-mon[221102]: pgmap v2040: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 137 KiB/s rd, 35 KiB/s wr, 90 op/s; 442 B/s, 1 objects/s recovering 2023-09-23T23:44:47.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:47 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1d", "id": [5, 3]}]: dispatch 2023-09-23T23:44:47.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:47 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [1, 2]}]: dispatch 2023-09-23T23:44:47.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:47 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1d", "id": [5, 3]}]: dispatch 2023-09-23T23:44:47.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:47 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 0]}]: dispatch 2023-09-23T23:44:47.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:47 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [1, 2]}]: dispatch 2023-09-23T23:44:47.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:47 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 0]}]: dispatch 2023-09-23T23:44:47.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:47 smithi138 ceph-mon[193606]: pgmap v2040: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 137 KiB/s rd, 35 KiB/s wr, 90 op/s; 442 B/s, 1 objects/s recovering 2023-09-23T23:44:47.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:47 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1d", "id": [5, 3]}]: dispatch 2023-09-23T23:44:47.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:47 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [1, 2]}]: dispatch 2023-09-23T23:44:47.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:47 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1d", "id": [5, 3]}]: dispatch 2023-09-23T23:44:47.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:47 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 0]}]: dispatch 2023-09-23T23:44:47.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:47 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [1, 2]}]: dispatch 2023-09-23T23:44:47.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:47 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 0]}]: dispatch 2023-09-23T23:44:47.695 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:44:48.043 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:48.192 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:44:47 smithi138 conmon[421553]: 2023-09-23T23:44:47.936+0000 7f11a6852540 -1 osd.4 697 log_to_monitors true 2023-09-23T23:44:48.296 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:44:48.571 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:48 smithi110 ceph-mon[212939]: failed to encode map e734 with expected crc 2023-09-23T23:44:48.571 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1d", "id": [5, 3]}]': finished 2023-09-23T23:44:48.571 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [1, 2]}]': finished 2023-09-23T23:44:48.572 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:48 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 0]}]': finished 2023-09-23T23:44:48.572 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:48 smithi110 ceph-mon[212939]: osdmap e734: 8 total, 7 up, 6 in 2023-09-23T23:44:48.572 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:48 smithi110 ceph-mon[212939]: from='osd.4 [v2:172.21.15.138:6800/524427801,v1:172.21.15.138:6801/524427801]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T23:44:48.572 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:48 smithi110 ceph-mon[212939]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T23:44:48.572 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:48 smithi110 ceph-mon[221102]: failed to encode map e734 with expected crc 2023-09-23T23:44:48.572 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:48 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1d", "id": [5, 3]}]': finished 2023-09-23T23:44:48.572 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:48 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [1, 2]}]': finished 2023-09-23T23:44:48.572 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:48 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 0]}]': finished 2023-09-23T23:44:48.572 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:48 smithi110 ceph-mon[221102]: osdmap e734: 8 total, 7 up, 6 in 2023-09-23T23:44:48.572 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:48 smithi110 ceph-mon[221102]: from='osd.4 [v2:172.21.15.138:6800/524427801,v1:172.21.15.138:6801/524427801]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T23:44:48.573 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:48 smithi110 ceph-mon[221102]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T23:44:48.660 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:44:48.692 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:44:48 smithi138 conmon[421553]: 2023-09-23T23:44:48.214+0000 7f1199e37700 -1 osd.4 697 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T23:44:48.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:48 smithi138 ceph-mon[193606]: failed to encode map e734 with expected crc 2023-09-23T23:44:48.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:48 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1d", "id": [5, 3]}]': finished 2023-09-23T23:44:48.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:48 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [1, 2]}]': finished 2023-09-23T23:44:48.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:48 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 0]}]': finished 2023-09-23T23:44:48.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:48 smithi138 ceph-mon[193606]: osdmap e734: 8 total, 7 up, 6 in 2023-09-23T23:44:48.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:48 smithi138 ceph-mon[193606]: from='osd.4 [v2:172.21.15.138:6800/524427801,v1:172.21.15.138:6801/524427801]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T23:44:48.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:48 smithi138 ceph-mon[193606]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T23:44:49.368 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:49 smithi110 ceph-mon[212939]: pgmap v2042: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:49.368 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:49 smithi110 ceph-mon[212939]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T23:44:49.368 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:49 smithi110 ceph-mon[212939]: osdmap e735: 8 total, 7 up, 6 in 2023-09-23T23:44:49.368 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:49 smithi110 ceph-mon[212939]: failed to encode map e735 with expected crc 2023-09-23T23:44:49.368 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:49 smithi110 ceph-mon[212939]: from='osd.4 [v2:172.21.15.138:6800/524427801,v1:172.21.15.138:6801/524427801]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T23:44:49.368 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:49 smithi110 ceph-mon[212939]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T23:44:49.368 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:49 smithi110 ceph-mon[212939]: failed to encode map e735 with expected crc 2023-09-23T23:44:49.368 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:49 smithi110 ceph-mon[212939]: failed to encode map e735 with expected crc 2023-09-23T23:44:49.368 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:49 smithi110 ceph-mon[221102]: pgmap v2042: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:49.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:49 smithi110 ceph-mon[221102]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T23:44:49.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:49 smithi110 ceph-mon[221102]: osdmap e735: 8 total, 7 up, 6 in 2023-09-23T23:44:49.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:49 smithi110 ceph-mon[221102]: failed to encode map e735 with expected crc 2023-09-23T23:44:49.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:49 smithi110 ceph-mon[221102]: from='osd.4 [v2:172.21.15.138:6800/524427801,v1:172.21.15.138:6801/524427801]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T23:44:49.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:49 smithi110 ceph-mon[221102]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T23:44:49.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:49 smithi110 ceph-mon[221102]: failed to encode map e735 with expected crc 2023-09-23T23:44:49.369 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:49 smithi110 ceph-mon[221102]: failed to encode map e735 with expected crc 2023-09-23T23:44:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:49 smithi138 ceph-mon[193606]: pgmap v2042: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:49.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:49 smithi138 ceph-mon[193606]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T23:44:49.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:49 smithi138 ceph-mon[193606]: osdmap e735: 8 total, 7 up, 6 in 2023-09-23T23:44:49.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:49 smithi138 ceph-mon[193606]: failed to encode map e735 with expected crc 2023-09-23T23:44:49.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:49 smithi138 ceph-mon[193606]: from='osd.4 [v2:172.21.15.138:6800/524427801,v1:172.21.15.138:6801/524427801]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T23:44:49.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:49 smithi138 ceph-mon[193606]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-09-23T23:44:49.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:49 smithi138 ceph-mon[193606]: failed to encode map e735 with expected crc 2023-09-23T23:44:49.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:49 smithi138 ceph-mon[193606]: failed to encode map e735 with expected crc 2023-09-23T23:44:49.552 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:44:49.552 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-09-23T23:44:49.552 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-09-23T23:44:49.552 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-09-23T23:44:49.552 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:44:50.252 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:44:50.310 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:44:50.310 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:44:50.310 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-09-23T23:44:50.310 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:44:50.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:50 smithi138 ceph-mon[193606]: failed to encode map e736 with expected crc 2023-09-23T23:44:50.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:50 smithi138 ceph-mon[193606]: failed to encode map e736 with expected crc 2023-09-23T23:44:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:50 smithi138 ceph-mon[193606]: osd.4 [v2:172.21.15.138:6800/524427801,v1:172.21.15.138:6801/524427801] boot 2023-09-23T23:44:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:50 smithi138 ceph-mon[193606]: osdmap e736: 8 total, 8 up, 6 in 2023-09-23T23:44:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:50 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T23:44:50.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:50 smithi138 ceph-mon[193606]: failed to encode map e736 with expected crc 2023-09-23T23:44:50.603 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:50.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:50 smithi110 ceph-mon[212939]: failed to encode map e736 with expected crc 2023-09-23T23:44:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:50 smithi110 ceph-mon[212939]: failed to encode map e736 with expected crc 2023-09-23T23:44:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:50 smithi110 ceph-mon[212939]: osd.4 [v2:172.21.15.138:6800/524427801,v1:172.21.15.138:6801/524427801] boot 2023-09-23T23:44:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:50 smithi110 ceph-mon[212939]: osdmap e736: 8 total, 8 up, 6 in 2023-09-23T23:44:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:50 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T23:44:50.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:50 smithi110 ceph-mon[212939]: failed to encode map e736 with expected crc 2023-09-23T23:44:50.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:50 smithi110 ceph-mon[221102]: failed to encode map e736 with expected crc 2023-09-23T23:44:50.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:50 smithi110 ceph-mon[221102]: failed to encode map e736 with expected crc 2023-09-23T23:44:50.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:50 smithi110 ceph-mon[221102]: osd.4 [v2:172.21.15.138:6800/524427801,v1:172.21.15.138:6801/524427801] boot 2023-09-23T23:44:50.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:50 smithi110 ceph-mon[221102]: osdmap e736: 8 total, 8 up, 6 in 2023-09-23T23:44:50.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:50 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T23:44:50.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:50 smithi110 ceph-mon[221102]: failed to encode map e736 with expected crc 2023-09-23T23:44:51.017 INFO:tasks.thrashosds.thrasher:inning osd 2023-09-23T23:44:51.017 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-09-23T23:44:51.017 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd in 4 2023-09-23T23:44:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:51 smithi138 ceph-mon[193606]: failed to encode map e698 with expected crc 2023-09-23T23:44:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:51 smithi138 ceph-mon[193606]: failed to encode map e736 with expected crc 2023-09-23T23:44:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:51 smithi138 ceph-mon[193606]: failed to encode map e736 with expected crc 2023-09-23T23:44:51.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:51 smithi138 ceph-mon[193606]: pgmap v2045: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:51.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:51 smithi138 ceph-mon[193606]: failed to encode map e737 with expected crc 2023-09-23T23:44:51.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:51 smithi138 ceph-mon[193606]: failed to encode map e737 with expected crc 2023-09-23T23:44:51.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:51 smithi138 ceph-mon[193606]: failed to encode map e737 with expected crc 2023-09-23T23:44:51.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:51 smithi138 ceph-mon[193606]: osdmap e737: 8 total, 8 up, 6 in 2023-09-23T23:44:51.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:51 smithi138 ceph-mon[193606]: failed to encode map e737 with expected crc 2023-09-23T23:44:51.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[212939]: failed to encode map e698 with expected crc 2023-09-23T23:44:51.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[212939]: failed to encode map e736 with expected crc 2023-09-23T23:44:51.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[212939]: failed to encode map e736 with expected crc 2023-09-23T23:44:51.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[212939]: pgmap v2045: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:51.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[212939]: failed to encode map e737 with expected crc 2023-09-23T23:44:51.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[212939]: failed to encode map e737 with expected crc 2023-09-23T23:44:51.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[212939]: failed to encode map e737 with expected crc 2023-09-23T23:44:51.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[212939]: osdmap e737: 8 total, 8 up, 6 in 2023-09-23T23:44:51.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[212939]: failed to encode map e737 with expected crc 2023-09-23T23:44:51.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[221102]: failed to encode map e698 with expected crc 2023-09-23T23:44:51.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[221102]: failed to encode map e736 with expected crc 2023-09-23T23:44:51.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[221102]: failed to encode map e736 with expected crc 2023-09-23T23:44:51.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[221102]: pgmap v2045: 103 pgs: 103 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:51.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[221102]: failed to encode map e737 with expected crc 2023-09-23T23:44:51.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[221102]: failed to encode map e737 with expected crc 2023-09-23T23:44:51.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[221102]: failed to encode map e737 with expected crc 2023-09-23T23:44:51.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[221102]: osdmap e737: 8 total, 8 up, 6 in 2023-09-23T23:44:51.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:51 smithi110 ceph-mon[221102]: failed to encode map e737 with expected crc 2023-09-23T23:44:51.795 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:52.223 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-09-23T23:44:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-09-23T23:44:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-09-23T23:44:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-09-23T23:44:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:94913 14.c 14:36b9aa44:::benchmark_data_smithi110_640334_object11034:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e644)", 2023-09-23T23:44:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:09.950582+0000", 2023-09-23T23:44:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 222.269733914, 2023-09-23T23:44:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.019290596, 2023-09-23T23:44:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 94913 2023-09-23T23:44:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.950582+0000", 2023-09-23T23:44:52.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.950582+0000", 2023-09-23T23:44:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.950581+0000", 2023-09-23T23:44:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:44:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.950586+0000", 2023-09-23T23:44:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5709999999999999e-06 2023-09-23T23:44:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.950593+0000", 2023-09-23T23:44:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9000000000000006e-06 2023-09-23T23:44:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.950598+0000", 2023-09-23T23:44:52.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9579999999999998e-06 2023-09-23T23:44:52.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.950638+0000", 2023-09-23T23:44:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9524999999999999e-05 2023-09-23T23:44:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.950645+0000", 2023-09-23T23:44:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.092e-06 2023-09-23T23:44:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.961771+0000", 2023-09-23T23:44:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0111260049999999 2023-09-23T23:44:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.961885+0000", 2023-09-23T23:44:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000113613 2023-09-23T23:44:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:44:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.961973+0000", 2023-09-23T23:44:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7934000000000002e-05 2023-09-23T23:44:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.969595+0000", 2023-09-23T23:44:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076220849999999998 2023-09-23T23:44:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.969631+0000", 2023-09-23T23:44:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6202000000000003e-05 2023-09-23T23:44:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.969774+0000", 2023-09-23T23:44:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014359799999999999 2023-09-23T23:44:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.969829+0000", 2023-09-23T23:44:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4394000000000001e-05 2023-09-23T23:44:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.969872+0000", 2023-09-23T23:44:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3624000000000001e-05 2023-09-23T23:44:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:94917 14.c 14:38b79d93:::benchmark_data_smithi110_640334_object11038:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e644)", 2023-09-23T23:44:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:09.992063+0000", 2023-09-23T23:44:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 222.22825244800001, 2023-09-23T23:44:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98694669599999996, 2023-09-23T23:44:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 94917 2023-09-23T23:44:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.992063+0000", 2023-09-23T23:44:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.992063+0000", 2023-09-23T23:44:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.992063+0000", 2023-09-23T23:44:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.992066+0000", 2023-09-23T23:44:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.976e-06 2023-09-23T23:44:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.992072+0000", 2023-09-23T23:44:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3249999999999996e-06 2023-09-23T23:44:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.992077+0000", 2023-09-23T23:44:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1769999999999997e-06 2023-09-23T23:44:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.992238+0000", 2023-09-23T23:44:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016106599999999999 2023-09-23T23:44:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.992249+0000", 2023-09-23T23:44:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0699e-05 2023-09-23T23:44:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.964709+0000", 2023-09-23T23:44:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97245993200000003 2023-09-23T23:44:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.964832+0000", 2023-09-23T23:44:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012310700000000001 2023-09-23T23:44:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:44:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.964906+0000", 2023-09-23T23:44:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4097999999999994e-05 2023-09-23T23:44:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.969661+0000", 2023-09-23T23:44:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047551019999999998 2023-09-23T23:44:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.969723+0000", 2023-09-23T23:44:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1947999999999997e-05 2023-09-23T23:44:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.978928+0000", 2023-09-23T23:44:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0092045189999999995 2023-09-23T23:44:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.978978+0000", 2023-09-23T23:44:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0340000000000003e-05 2023-09-23T23:44:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979010+0000", 2023-09-23T23:44:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1868e-05 2023-09-23T23:44:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:94918 14.c 14:3f0efaa9:::benchmark_data_smithi110_640334_object11039:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e644)", 2023-09-23T23:44:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:09.999372+0000", 2023-09-23T23:44:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 222.22094426300001, 2023-09-23T23:44:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97974634000000005, 2023-09-23T23:44:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 94918 2023-09-23T23:44:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.999372+0000", 2023-09-23T23:44:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.999372+0000", 2023-09-23T23:44:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.999370+0000", 2023-09-23T23:44:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.999377+0000", 2023-09-23T23:44:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.593e-06 2023-09-23T23:44:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.999388+0000", 2023-09-23T23:44:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0905e-05 2023-09-23T23:44:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.999395+0000", 2023-09-23T23:44:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7599999999999997e-06 2023-09-23T23:44:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.999665+0000", 2023-09-23T23:44:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027007600000000001 2023-09-23T23:44:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.999675+0000", 2023-09-23T23:44:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0139000000000001e-05 2023-09-23T23:44:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.967221+0000", 2023-09-23T23:44:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96754572900000002 2023-09-23T23:44:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.967302+0000", 2023-09-23T23:44:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0811999999999998e-05 2023-09-23T23:44:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:44:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.967376+0000", 2023-09-23T23:44:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4296999999999997e-05 2023-09-23T23:44:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.969919+0000", 2023-09-23T23:44:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025428709999999999 2023-09-23T23:44:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.969925+0000", 2023-09-23T23:44:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9850000000000003e-06 2023-09-23T23:44:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979058+0000", 2023-09-23T23:44:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0091330620000000008 2023-09-23T23:44:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979095+0000", 2023-09-23T23:44:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6511000000000003e-05 2023-09-23T23:44:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979118+0000", 2023-09-23T23:44:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.323e-05 2023-09-23T23:44:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:94919 14.c 14:3e08badf:::benchmark_data_smithi110_640334_object11040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e644)", 2023-09-23T23:44:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:09.999411+0000", 2023-09-23T23:44:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 222.22090453000001, 2023-09-23T23:44:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97987423699999998, 2023-09-23T23:44:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 94919 2023-09-23T23:44:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.999411+0000", 2023-09-23T23:44:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.999411+0000", 2023-09-23T23:44:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.999411+0000", 2023-09-23T23:44:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.999420+0000", 2023-09-23T23:44:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0799999999999995e-06 2023-09-23T23:44:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.999427+0000", 2023-09-23T23:44:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9750000000000001e-06 2023-09-23T23:44:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.999432+0000", 2023-09-23T23:44:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9640000000000002e-06 2023-09-23T23:44:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.999700+0000", 2023-09-23T23:44:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026831899999999999 2023-09-23T23:44:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:09.999708+0000", 2023-09-23T23:44:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9880000000000004e-06 2023-09-23T23:44:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.969954+0000", 2023-09-23T23:44:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97024537899999996 2023-09-23T23:44:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.970049+0000", 2023-09-23T23:44:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5390999999999995e-05 2023-09-23T23:44:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:44:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.970117+0000", 2023-09-23T23:44:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7976000000000001e-05 2023-09-23T23:44:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979133+0000", 2023-09-23T23:44:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0090165130000000003 2023-09-23T23:44:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979173+0000", 2023-09-23T23:44:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9598999999999998e-05 2023-09-23T23:44:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979213+0000", 2023-09-23T23:44:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0397e-05 2023-09-23T23:44:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979252+0000", 2023-09-23T23:44:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8596000000000001e-05 2023-09-23T23:44:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979285+0000", 2023-09-23T23:44:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3507999999999998e-05 2023-09-23T23:44:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:94934 14.c 14:327c3f95:::benchmark_data_smithi110_640334_object11055:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e644)", 2023-09-23T23:44:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:10.013268+0000", 2023-09-23T23:44:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 222.20704812899999, 2023-09-23T23:44:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96622860300000002, 2023-09-23T23:44:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 94934 2023-09-23T23:44:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.013268+0000", 2023-09-23T23:44:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.013268+0000", 2023-09-23T23:44:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.013267+0000", 2023-09-23T23:44:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.013270+0000", 2023-09-23T23:44:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7410000000000001e-06 2023-09-23T23:44:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.013276+0000", 2023-09-23T23:44:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1090000000000003e-06 2023-09-23T23:44:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.013281+0000", 2023-09-23T23:44:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0839999999999999e-06 2023-09-23T23:44:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.013333+0000", 2023-09-23T23:44:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1496999999999998e-05 2023-09-23T23:44:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.013342+0000", 2023-09-23T23:44:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7899999999999994e-06 2023-09-23T23:44:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.973069+0000", 2023-09-23T23:44:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95972629600000003 2023-09-23T23:44:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.973187+0000", 2023-09-23T23:44:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118695 2023-09-23T23:44:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:44:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.973298+0000", 2023-09-23T23:44:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110375 2023-09-23T23:44:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979140+0000", 2023-09-23T23:44:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058425109999999999 2023-09-23T23:44:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979374+0000", 2023-09-23T23:44:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000233935 2023-09-23T23:44:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979414+0000", 2023-09-23T23:44:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0232999999999999e-05 2023-09-23T23:44:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979460+0000", 2023-09-23T23:44:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5878999999999999e-05 2023-09-23T23:44:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979496+0000", 2023-09-23T23:44:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5898000000000001e-05 2023-09-23T23:44:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:94958 14.c 14:30cee45a:::benchmark_data_smithi110_640334_object11079:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e644)", 2023-09-23T23:44:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:10.126899+0000", 2023-09-23T23:44:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 222.09341666899999, 2023-09-23T23:44:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86075324200000003, 2023-09-23T23:44:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 94958 2023-09-23T23:44:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.126899+0000", 2023-09-23T23:44:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.126899+0000", 2023-09-23T23:44:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.126898+0000", 2023-09-23T23:44:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:44:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.126902+0000", 2023-09-23T23:44:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0890000000000002e-06 2023-09-23T23:44:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.126914+0000", 2023-09-23T23:44:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1776e-05 2023-09-23T23:44:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.126920+0000", 2023-09-23T23:44:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4629999999999998e-06 2023-09-23T23:44:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.126999+0000", 2023-09-23T23:44:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8658000000000005e-05 2023-09-23T23:44:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.127009+0000", 2023-09-23T23:44:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0261000000000001e-05 2023-09-23T23:44:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.976237+0000", 2023-09-23T23:44:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.84922787200000005 2023-09-23T23:44:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.976350+0000", 2023-09-23T23:44:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000112732 2023-09-23T23:44:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:44:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.976433+0000", 2023-09-23T23:44:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2663999999999993e-05 2023-09-23T23:44:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979305+0000", 2023-09-23T23:44:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028724929999999998 2023-09-23T23:44:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979522+0000", 2023-09-23T23:44:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000217378 2023-09-23T23:44:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.987566+0000", 2023-09-23T23:44:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080439320000000002 2023-09-23T23:44:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.987607+0000", 2023-09-23T23:44:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0595e-05 2023-09-23T23:44:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.987652+0000", 2023-09-23T23:44:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5426000000000002e-05 2023-09-23T23:44:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:94982 14.c 14:326bb662:::benchmark_data_smithi110_640334_object11103:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e644)", 2023-09-23T23:44:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:10.149349+0000", 2023-09-23T23:44:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 222.070967169, 2023-09-23T23:44:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.854996545, 2023-09-23T23:44:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 94982 2023-09-23T23:44:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.149349+0000", 2023-09-23T23:44:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.149349+0000", 2023-09-23T23:44:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.149348+0000", 2023-09-23T23:44:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.149351+0000", 2023-09-23T23:44:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0599999999999999e-06 2023-09-23T23:44:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.152570+0000", 2023-09-23T23:44:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032187629999999999 2023-09-23T23:44:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.152581+0000", 2023-09-23T23:44:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1046999999999999e-05 2023-09-23T23:44:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.152697+0000", 2023-09-23T23:44:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000115573 2023-09-23T23:44:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.152707+0000", 2023-09-23T23:44:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0564000000000001e-05 2023-09-23T23:44:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979541+0000", 2023-09-23T23:44:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.82683367799999996 2023-09-23T23:44:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979641+0000", 2023-09-23T23:44:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9792999999999998e-05 2023-09-23T23:44:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:44:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.979740+0000", 2023-09-23T23:44:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9761999999999998e-05 2023-09-23T23:44:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.997073+0000", 2023-09-23T23:44:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017332941000000001 2023-09-23T23:44:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.997200+0000", 2023-09-23T23:44:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000127069 2023-09-23T23:44:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.004268+0000", 2023-09-23T23:44:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070677989999999996 2023-09-23T23:44:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.004310+0000", 2023-09-23T23:44:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1477000000000003e-05 2023-09-23T23:44:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.004345+0000", 2023-09-23T23:44:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5460999999999998e-05 2023-09-23T23:44:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:95003 14.c 14:33227595:::benchmark_data_smithi110_640334_object11124:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e644)", 2023-09-23T23:44:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:10.177179+0000", 2023-09-23T23:44:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 222.04313717799999, 2023-09-23T23:44:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.82723874200000003, 2023-09-23T23:44:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 95003 2023-09-23T23:44:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.177179+0000", 2023-09-23T23:44:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.177179+0000", 2023-09-23T23:44:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.177178+0000", 2023-09-23T23:44:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.177181+0000", 2023-09-23T23:44:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5639999999999999e-06 2023-09-23T23:44:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.177189+0000", 2023-09-23T23:44:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7220000000000005e-06 2023-09-23T23:44:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.177195+0000", 2023-09-23T23:44:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2379999999999997e-06 2023-09-23T23:44:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.177259+0000", 2023-09-23T23:44:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3917999999999993e-05 2023-09-23T23:44:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.177276+0000", 2023-09-23T23:44:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7507999999999999e-05 2023-09-23T23:44:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.982349+0000", 2023-09-23T23:44:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.80507250799999996 2023-09-23T23:44:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.982452+0000", 2023-09-23T23:44:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103541 2023-09-23T23:44:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-09-23T23:44:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.982571+0000", 2023-09-23T23:44:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118743 2023-09-23T23:44:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.997106+0000", 2023-09-23T23:44:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014535027000000001 2023-09-23T23:44:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:10.997231+0000", 2023-09-23T23:44:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012541200000000001 2023-09-23T23:44:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.004357+0000", 2023-09-23T23:44:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071256189999999997 2023-09-23T23:44:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.004389+0000", 2023-09-23T23:44:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1591999999999998e-05 2023-09-23T23:44:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:11.004417+0000", 2023-09-23T23:44:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8819999999999999e-05 2023-09-23T23:44:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:105753 14.8 14:10c439de:::benchmark_data_smithi110_640334_object21874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e653)", 2023-09-23T23:44:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:30.649181+0000", 2023-09-23T23:44:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 201.57113504899999, 2023-09-23T23:44:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.040734802, 2023-09-23T23:44:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105753 2023-09-23T23:44:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.649181+0000", 2023-09-23T23:44:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.649181+0000", 2023-09-23T23:44:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.649180+0000", 2023-09-23T23:44:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.649183+0000", 2023-09-23T23:44:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9280000000000002e-06 2023-09-23T23:44:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.649192+0000", 2023-09-23T23:44:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4829999999999999e-06 2023-09-23T23:44:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.649197+0000", 2023-09-23T23:44:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2120000000000002e-06 2023-09-23T23:44:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.649271+0000", 2023-09-23T23:44:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4349000000000003e-05 2023-09-23T23:44:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.649281+0000", 2023-09-23T23:44:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7790000000000007e-06 2023-09-23T23:44:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.641637+0000", 2023-09-23T23:44:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99235584499999996 2023-09-23T23:44:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.641769+0000", 2023-09-23T23:44:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013189399999999999 2023-09-23T23:44:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:44:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.641922+0000", 2023-09-23T23:44:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015290400000000001 2023-09-23T23:44:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.682855+0000", 2023-09-23T23:44:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040933561 2023-09-23T23:44:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.682886+0000", 2023-09-23T23:44:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1315000000000001e-05 2023-09-23T23:44:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.689706+0000", 2023-09-23T23:44:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068190760000000003 2023-09-23T23:44:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.689756+0000", 2023-09-23T23:44:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0776999999999999e-05 2023-09-23T23:44:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.689916+0000", 2023-09-23T23:44:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015922000000000001 2023-09-23T23:44:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:105777 14.8 14:12637292:::benchmark_data_smithi110_640334_object21898:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e653)", 2023-09-23T23:44:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:30.684050+0000", 2023-09-23T23:44:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 201.53626593199999, 2023-09-23T23:44:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0128159539999999, 2023-09-23T23:44:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105777 2023-09-23T23:44:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.684050+0000", 2023-09-23T23:44:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.684050+0000", 2023-09-23T23:44:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.684048+0000", 2023-09-23T23:44:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.684056+0000", 2023-09-23T23:44:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2760000000000003e-06 2023-09-23T23:44:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.684067+0000", 2023-09-23T23:44:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1888e-05 2023-09-23T23:44:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.684076+0000", 2023-09-23T23:44:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1599999999999998e-06 2023-09-23T23:44:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.684132+0000", 2023-09-23T23:44:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6208000000000001e-05 2023-09-23T23:44:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.684140+0000", 2023-09-23T23:44:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7680000000000008e-06 2023-09-23T23:44:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.682909+0000", 2023-09-23T23:44:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99876935 2023-09-23T23:44:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.683057+0000", 2023-09-23T23:44:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014810999999999999 2023-09-23T23:44:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:44:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.683214+0000", 2023-09-23T23:44:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015667799999999999 2023-09-23T23:44:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.696573+0000", 2023-09-23T23:44:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013358925000000001 2023-09-23T23:44:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.696630+0000", 2023-09-23T23:44:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7059999999999999e-05 2023-09-23T23:44:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.696643+0000", 2023-09-23T23:44:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2914999999999999e-05 2023-09-23T23:44:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.696697+0000", 2023-09-23T23:44:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4710000000000003e-05 2023-09-23T23:44:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.696866+0000", 2023-09-23T23:44:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016851100000000001 2023-09-23T23:44:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:105797 14.8 14:1f868cae:::benchmark_data_smithi110_640334_object21918:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e653)", 2023-09-23T23:44:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:30.708246+0000", 2023-09-23T23:44:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 201.51207015200001, 2023-09-23T23:44:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.000688501, 2023-09-23T23:44:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105797 2023-09-23T23:44:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.708246+0000", 2023-09-23T23:44:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.708246+0000", 2023-09-23T23:44:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.708245+0000", 2023-09-23T23:44:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.708249+0000", 2023-09-23T23:44:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6440000000000002e-06 2023-09-23T23:44:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.708256+0000", 2023-09-23T23:44:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9809999999999997e-06 2023-09-23T23:44:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.708261+0000", 2023-09-23T23:44:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9729999999999997e-06 2023-09-23T23:44:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.708322+0000", 2023-09-23T23:44:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1094999999999997e-05 2023-09-23T23:44:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.708332+0000", 2023-09-23T23:44:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0594000000000001e-05 2023-09-23T23:44:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.686269+0000", 2023-09-23T23:44:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97793702699999996 2023-09-23T23:44:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.686419+0000", 2023-09-23T23:44:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000149118 2023-09-23T23:44:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:44:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.686586+0000", 2023-09-23T23:44:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016720999999999999 2023-09-23T23:44:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.696878+0000", 2023-09-23T23:44:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010292163999999999 2023-09-23T23:44:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.708603+0000", 2023-09-23T23:44:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011724864 2023-09-23T23:44:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.708691+0000", 2023-09-23T23:44:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7774000000000004e-05 2023-09-23T23:44:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.708750+0000", 2023-09-23T23:44:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9512999999999998e-05 2023-09-23T23:44:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.708934+0000", 2023-09-23T23:44:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018400799999999999 2023-09-23T23:44:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:105808 14.8 14:1ef45467:::benchmark_data_smithi110_640334_object21929:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e653)", 2023-09-23T23:44:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:30.717695+0000", 2023-09-23T23:44:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 201.50262041900001, 2023-09-23T23:44:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99147217200000004, 2023-09-23T23:44:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105808 2023-09-23T23:44:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.717695+0000", 2023-09-23T23:44:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.717695+0000", 2023-09-23T23:44:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.717695+0000", 2023-09-23T23:44:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.717697+0000", 2023-09-23T23:44:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0870000000000001e-06 2023-09-23T23:44:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.717703+0000", 2023-09-23T23:44:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.463e-06 2023-09-23T23:44:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.717708+0000", 2023-09-23T23:44:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0250000000000002e-06 2023-09-23T23:44:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.724371+0000", 2023-09-23T23:44:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.006663146 2023-09-23T23:44:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.724380+0000", 2023-09-23T23:44:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5380000000000008e-06 2023-09-23T23:44:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.689956+0000", 2023-09-23T23:44:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96557568500000002 2023-09-23T23:44:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.690045+0000", 2023-09-23T23:44:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9355000000000001e-05 2023-09-23T23:44:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:44:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.690131+0000", 2023-09-23T23:44:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5699999999999996e-05 2023-09-23T23:44:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.706463+0000", 2023-09-23T23:44:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016332260000000001 2023-09-23T23:44:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.708653+0000", 2023-09-23T23:44:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021898339999999999 2023-09-23T23:44:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.708983+0000", 2023-09-23T23:44:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032998100000000002 2023-09-23T23:44:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.709031+0000", 2023-09-23T23:44:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7818999999999997e-05 2023-09-23T23:44:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.709168+0000", 2023-09-23T23:44:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013661899999999999 2023-09-23T23:44:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:105826 14.8 14:121eaa4a:::benchmark_data_smithi110_640334_object21947:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e653)", 2023-09-23T23:44:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:30.745668+0000", 2023-09-23T23:44:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 201.474647361, 2023-09-23T23:44:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97070416800000003, 2023-09-23T23:44:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105826 2023-09-23T23:44:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.745668+0000", 2023-09-23T23:44:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.745668+0000", 2023-09-23T23:44:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.745667+0000", 2023-09-23T23:44:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.745674+0000", 2023-09-23T23:44:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7739999999999999e-06 2023-09-23T23:44:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.745685+0000", 2023-09-23T23:44:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1824000000000001e-05 2023-09-23T23:44:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.745693+0000", 2023-09-23T23:44:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9019999999999999e-06 2023-09-23T23:44:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.745948+0000", 2023-09-23T23:44:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025477300000000002 2023-09-23T23:44:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.745959+0000", 2023-09-23T23:44:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0961999999999999e-05 2023-09-23T23:44:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.693200+0000", 2023-09-23T23:44:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94724060499999996 2023-09-23T23:44:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.693353+0000", 2023-09-23T23:44:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015279000000000001 2023-09-23T23:44:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:44:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.693522+0000", 2023-09-23T23:44:52.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016985200000000001 2023-09-23T23:44:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.709216+0000", 2023-09-23T23:44:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015694044000000001 2023-09-23T23:44:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.709256+0000", 2023-09-23T23:44:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9839000000000003e-05 2023-09-23T23:44:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.716170+0000", 2023-09-23T23:44:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0069136290000000001 2023-09-23T23:44:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.716206+0000", 2023-09-23T23:44:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5855000000000001e-05 2023-09-23T23:44:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.716373+0000", 2023-09-23T23:44:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000166863 2023-09-23T23:44:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:105833 14.8 14:1220db32:::benchmark_data_smithi110_640334_object21954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e653)", 2023-09-23T23:44:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:30.750699+0000", 2023-09-23T23:44:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 201.46961659600001, 2023-09-23T23:44:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96589858500000003, 2023-09-23T23:44:52.335 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105833 2023-09-23T23:44:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.750699+0000", 2023-09-23T23:44:52.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.750699+0000", 2023-09-23T23:44:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.750699+0000", 2023-09-23T23:44:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.750702+0000", 2023-09-23T23:44:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1410000000000001e-06 2023-09-23T23:44:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.750711+0000", 2023-09-23T23:44:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8019999999999998e-06 2023-09-23T23:44:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.750716+0000", 2023-09-23T23:44:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1630000000000004e-06 2023-09-23T23:44:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.750793+0000", 2023-09-23T23:44:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7426e-05 2023-09-23T23:44:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.750807+0000", 2023-09-23T23:44:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3647e-05 2023-09-23T23:44:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.696925+0000", 2023-09-23T23:44:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.946117702 2023-09-23T23:44:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.697082+0000", 2023-09-23T23:44:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015701699999999999 2023-09-23T23:44:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:44:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.697228+0000", 2023-09-23T23:44:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014616600000000001 2023-09-23T23:44:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.709290+0000", 2023-09-23T23:44:52.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012061886000000001 2023-09-23T23:44:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.716384+0000", 2023-09-23T23:44:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007094777 2023-09-23T23:44:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.716441+0000", 2023-09-23T23:44:52.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6609e-05 2023-09-23T23:44:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.716476+0000", 2023-09-23T23:44:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4765999999999999e-05 2023-09-23T23:44:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.716598+0000", 2023-09-23T23:44:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012197399999999999 2023-09-23T23:44:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.343 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:105864 14.8 14:19b933b7:::benchmark_data_smithi110_640334_object21985:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e653)", 2023-09-23T23:44:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:30.792894+0000", 2023-09-23T23:44:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 201.42742201499999, 2023-09-23T23:44:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.92392346000000003, 2023-09-23T23:44:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.344 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105864 2023-09-23T23:44:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.792894+0000", 2023-09-23T23:44:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.792894+0000", 2023-09-23T23:44:52.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.792893+0000", 2023-09-23T23:44:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:44:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.792898+0000", 2023-09-23T23:44:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5879999999999997e-06 2023-09-23T23:44:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.792908+0000", 2023-09-23T23:44:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3090000000000001e-06 2023-09-23T23:44:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.792914+0000", 2023-09-23T23:44:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.404e-06 2023-09-23T23:44:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.792967+0000", 2023-09-23T23:44:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3093999999999997e-05 2023-09-23T23:44:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.792978+0000", 2023-09-23T23:44:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0526000000000001e-05 2023-09-23T23:44:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.700100+0000", 2023-09-23T23:44:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.90712207 2023-09-23T23:44:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.700255+0000", 2023-09-23T23:44:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015522600000000001 2023-09-23T23:44:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:44:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.700400+0000", 2023-09-23T23:44:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014510700000000001 2023-09-23T23:44:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.709325+0000", 2023-09-23T23:44:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0089251869999999994 2023-09-23T23:44:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.716396+0000", 2023-09-23T23:44:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070703169999999996 2023-09-23T23:44:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.716673+0000", 2023-09-23T23:44:52.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027716299999999998 2023-09-23T23:44:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.716715+0000", 2023-09-23T23:44:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2725999999999997e-05 2023-09-23T23:44:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.716817+0000", 2023-09-23T23:44:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000101844 2023-09-23T23:44:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.351 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:105878 14.8 14:1efca29e:::benchmark_data_smithi110_640334_object21999:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e653)", 2023-09-23T23:44:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:30.810920+0000", 2023-09-23T23:44:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 201.409395518, 2023-09-23T23:44:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.91319983800000004, 2023-09-23T23:44:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.352 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105878 2023-09-23T23:44:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.810920+0000", 2023-09-23T23:44:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.810920+0000", 2023-09-23T23:44:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.810919+0000", 2023-09-23T23:44:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-09-23T23:44:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.810926+0000", 2023-09-23T23:44:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.562e-06 2023-09-23T23:44:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.810935+0000", 2023-09-23T23:44:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7780000000000002e-06 2023-09-23T23:44:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.810943+0000", 2023-09-23T23:44:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8520000000000004e-06 2023-09-23T23:44:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.811008+0000", 2023-09-23T23:44:52.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4768000000000001e-05 2023-09-23T23:44:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.811019+0000", 2023-09-23T23:44:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1080999999999999e-05 2023-09-23T23:44:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.703208+0000", 2023-09-23T23:44:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.89218864499999995 2023-09-23T23:44:52.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.703316+0000", 2023-09-23T23:44:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000108119 2023-09-23T23:44:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:44:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.703414+0000", 2023-09-23T23:44:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8301000000000006e-05 2023-09-23T23:44:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.716831+0000", 2023-09-23T23:44:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013417314 2023-09-23T23:44:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.716896+0000", 2023-09-23T23:44:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4777999999999995e-05 2023-09-23T23:44:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.723882+0000", 2023-09-23T23:44:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0069853490000000001 2023-09-23T23:44:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.723942+0000", 2023-09-23T23:44:52.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0313999999999999e-05 2023-09-23T23:44:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.724120+0000", 2023-09-23T23:44:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000178238 2023-09-23T23:44:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:105894 14.8 14:1060548c:::benchmark_data_smithi110_640334_object22015:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e653)", 2023-09-23T23:44:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:30.827433+0000", 2023-09-23T23:44:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 201.39288263500001, 2023-09-23T23:44:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.89697113900000003, 2023-09-23T23:44:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105894 2023-09-23T23:44:52.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.827433+0000", 2023-09-23T23:44:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.827433+0000", 2023-09-23T23:44:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.827433+0000", 2023-09-23T23:44:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.827435+0000", 2023-09-23T23:44:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4380000000000002e-06 2023-09-23T23:44:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.827441+0000", 2023-09-23T23:44:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7620000000000001e-06 2023-09-23T23:44:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.827446+0000", 2023-09-23T23:44:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6269999999999999e-06 2023-09-23T23:44:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.827493+0000", 2023-09-23T23:44:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7639999999999998e-05 2023-09-23T23:44:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.827501+0000", 2023-09-23T23:44:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0309999999999998e-06 2023-09-23T23:44:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.709350+0000", 2023-09-23T23:44:52.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.88184879400000005 2023-09-23T23:44:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.709496+0000", 2023-09-23T23:44:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000145873 2023-09-23T23:44:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:44:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.709646+0000", 2023-09-23T23:44:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000150486 2023-09-23T23:44:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.716841+0000", 2023-09-23T23:44:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071943040000000003 2023-09-23T23:44:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.724173+0000", 2023-09-23T23:44:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0073320670000000003 2023-09-23T23:44:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.724242+0000", 2023-09-23T23:44:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9300000000000004e-05 2023-09-23T23:44:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.724287+0000", 2023-09-23T23:44:52.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5391e-05 2023-09-23T23:44:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.724404+0000", 2023-09-23T23:44:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011682499999999999 2023-09-23T23:44:52.369 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:105925 14.8 14:17b0f557:::benchmark_data_smithi110_640334_object22046:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e653)", 2023-09-23T23:44:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:30.878834+0000", 2023-09-23T23:44:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 201.341481458, 2023-09-23T23:44:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.85509302700000001, 2023-09-23T23:44:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.370 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105925 2023-09-23T23:44:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.878834+0000", 2023-09-23T23:44:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.878834+0000", 2023-09-23T23:44:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.878833+0000", 2023-09-23T23:44:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-09-23T23:44:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.878839+0000", 2023-09-23T23:44:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8000000000000004e-06 2023-09-23T23:44:52.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.878856+0000", 2023-09-23T23:44:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6660999999999999e-05 2023-09-23T23:44:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.878865+0000", 2023-09-23T23:44:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0389999999999993e-06 2023-09-23T23:44:52.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.880608+0000", 2023-09-23T23:44:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001742674 2023-09-23T23:44:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.880621+0000", 2023-09-23T23:44:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3259e-05 2023-09-23T23:44:52.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.712776+0000", 2023-09-23T23:44:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.83215499900000001 2023-09-23T23:44:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.712900+0000", 2023-09-23T23:44:52.375 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012425500000000001 2023-09-23T23:44:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:44:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.713051+0000", 2023-09-23T23:44:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000150784 2023-09-23T23:44:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.716849+0000", 2023-09-23T23:44:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037980900000000001 2023-09-23T23:44:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.724206+0000", 2023-09-23T23:44:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0073568039999999998 2023-09-23T23:44:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.733723+0000", 2023-09-23T23:44:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0095177509999999996 2023-09-23T23:44:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.733776+0000", 2023-09-23T23:44:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2784000000000001e-05 2023-09-23T23:44:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.733927+0000", 2023-09-23T23:44:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000151089 2023-09-23T23:44:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47186.0:105962 14.8 14:1c1a1e3b:::benchmark_data_smithi110_640334_object22083:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e653)", 2023-09-23T23:44:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:41:30.929213+0000", 2023-09-23T23:44:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 201.29110310900001, 2023-09-23T23:44:52.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.81173547700000004, 2023-09-23T23:44:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47186", 2023-09-23T23:44:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/1419217895", 2023-09-23T23:44:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105962 2023-09-23T23:44:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.929213+0000", 2023-09-23T23:44:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.929213+0000", 2023-09-23T23:44:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.929212+0000", 2023-09-23T23:44:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-09-23T23:44:52.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.929215+0000", 2023-09-23T23:44:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1549999999999999e-06 2023-09-23T23:44:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.929222+0000", 2023-09-23T23:44:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1720000000000004e-06 2023-09-23T23:44:52.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.929227+0000", 2023-09-23T23:44:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3040000000000004e-06 2023-09-23T23:44:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.929282+0000", 2023-09-23T23:44:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5531000000000001e-05 2023-09-23T23:44:52.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-09-23T23:44:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:30.929293+0000", 2023-09-23T23:44:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0730000000000001e-05 2023-09-23T23:44:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.717023+0000", 2023-09-23T23:44:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.78772967599999999 2023-09-23T23:44:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.717146+0000", 2023-09-23T23:44:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012312 2023-09-23T23:44:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:44:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.717268+0000", 2023-09-23T23:44:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012234799999999999 2023-09-23T23:44:52.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.724134+0000", 2023-09-23T23:44:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068654950000000001 2023-09-23T23:44:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.733972+0000", 2023-09-23T23:44:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0098385260000000002 2023-09-23T23:44:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.740747+0000", 2023-09-23T23:44:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067750589999999999 2023-09-23T23:44:52.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.740794+0000", 2023-09-23T23:44:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.702e-05 2023-09-23T23:44:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:41:31.740948+0000", 2023-09-23T23:44:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015371199999999999 2023-09-23T23:44:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.38088.0:1 2.1 2:9916b262:::rbd_data.94b38896dfdf.000000000000:head [writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e715)", 2023-09-23T23:44:52.389 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-09-23T23:44:08.723521+0000", 2023-09-23T23:44:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 43.496794526999999, 2023-09-23T23:44:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011954507, 2023-09-23T23:44:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-09-23T23:44:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:44:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-09-23T23:44:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.38088", 2023-09-23T23:44:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.110:0/955820458", 2023-09-23T23:44:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 1 2023-09-23T23:44:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.390 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-09-23T23:44:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-09-23T23:44:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.723521+0000", 2023-09-23T23:44:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-09-23T23:44:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.723521+0000", 2023-09-23T23:44:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-09-23T23:44:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-09-23T23:44:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.723519+0000", 2023-09-23T23:44:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-09-23T23:44:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-09-23T23:44:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.725585+0000", 2023-09-23T23:44:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020660100000000001 2023-09-23T23:44:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-09-23T23:44:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.725610+0000", 2023-09-23T23:44:52.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4715999999999998e-05 2023-09-23T23:44:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-09-23T23:44:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.725621+0000", 2023-09-23T23:44:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0589e-05 2023-09-23T23:44:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-09-23T23:44:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.725689+0000", 2023-09-23T23:44:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8701999999999994e-05 2023-09-23T23:44:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-09-23T23:44:52.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.725823+0000", 2023-09-23T23:44:52.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013397999999999999 2023-09-23T23:44:52.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:44:52.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.725936+0000", 2023-09-23T23:44:52.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011267700000000001 2023-09-23T23:44:52.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-09-23T23:44:52.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.735066+0000", 2023-09-23T23:44:52.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0091300359999999994 2023-09-23T23:44:52.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.735380+0000", 2023-09-23T23:44:52.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00031400299999999998 2023-09-23T23:44:52.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:44:52.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.735405+0000", 2023-09-23T23:44:52.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5384e-05 2023-09-23T23:44:52.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-09-23T23:44:52.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.735444+0000", 2023-09-23T23:44:52.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8529000000000003e-05 2023-09-23T23:44:52.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-09-23T23:44:52.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-09-23T23:44:52.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-09-23T23:44:52.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-09-23T23:44:08.735476+0000", 2023-09-23T23:44:52.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1779999999999997e-05 2023-09-23T23:44:52.396 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.396 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.396 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.397 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-09-23T23:44:52.397 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-09-23T23:44:52.397 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-09-23T23:44:52.397 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:52 smithi110 ceph-mon[212939]: failed to encode map e738 with expected crc 2023-09-23T23:44:52.397 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:52 smithi110 ceph-mon[212939]: failed to encode map e738 with expected crc 2023-09-23T23:44:52.397 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:52 smithi110 ceph-mon[212939]: osdmap e738: 8 total, 8 up, 6 in 2023-09-23T23:44:52.398 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:52 smithi110 ceph-mon[212939]: failed to encode map e738 with expected crc 2023-09-23T23:44:52.398 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:52 smithi110 ceph-mon[212939]: failed to encode map e738 with expected crc 2023-09-23T23:44:52.398 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:52 smithi110 ceph-mon[212939]: failed to encode map e738 with expected crc 2023-09-23T23:44:52.398 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:52 smithi110 ceph-mon[221102]: failed to encode map e738 with expected crc 2023-09-23T23:44:52.398 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:52 smithi110 ceph-mon[221102]: failed to encode map e738 with expected crc 2023-09-23T23:44:52.398 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:52 smithi110 ceph-mon[221102]: osdmap e738: 8 total, 8 up, 6 in 2023-09-23T23:44:52.398 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:52 smithi110 ceph-mon[221102]: failed to encode map e738 with expected crc 2023-09-23T23:44:52.398 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:52 smithi110 ceph-mon[221102]: failed to encode map e738 with expected crc 2023-09-23T23:44:52.398 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:52 smithi110 ceph-mon[221102]: failed to encode map e738 with expected crc 2023-09-23T23:44:52.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:52 smithi138 ceph-mon[193606]: failed to encode map e738 with expected crc 2023-09-23T23:44:52.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:52 smithi138 ceph-mon[193606]: failed to encode map e738 with expected crc 2023-09-23T23:44:52.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:52 smithi138 ceph-mon[193606]: osdmap e738: 8 total, 8 up, 6 in 2023-09-23T23:44:52.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:52 smithi138 ceph-mon[193606]: failed to encode map e738 with expected crc 2023-09-23T23:44:52.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:52 smithi138 ceph-mon[193606]: failed to encode map e738 with expected crc 2023-09-23T23:44:52.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:52 smithi138 ceph-mon[193606]: failed to encode map e738 with expected crc 2023-09-23T23:44:53.016 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:44:53.377 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:44:53.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:53 smithi138 ceph-mon[193606]: pgmap v2048: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:53.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:53 smithi110 ceph-mon[212939]: pgmap v2048: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:53.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:53 smithi110 ceph-mon[221102]: pgmap v2048: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:54.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:54 smithi110 conmon[212914]: 2023-09-23T23:44:54.174+0000 7f29aa464700 -1 mon.a@0(leader).osd e738 definitely_dead 0 2023-09-23T23:44:54.923 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:44:54.924 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:44:54.924 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-09-23T23:44:54.924 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:44:54.986 INFO:teuthology.orchestra.run.smithi110.stderr:marked in osd.4. 2023-09-23T23:44:55.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:54 smithi138 ceph-mon[193606]: pgmap v2049: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:55.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:54 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3589203774' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T23:44:55.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:54 smithi110 ceph-mon[212939]: pgmap v2049: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:55.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:54 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3589203774' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T23:44:55.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:54 smithi110 ceph-mon[221102]: pgmap v2049: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:55.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:54 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3589203774' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T23:44:55.692 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T23:44:55.807 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-09-23T23:44:55.807 INFO:tasks.thrashosds.thrasher:inning osd 2023-09-23T23:44:55.807 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-09-23T23:44:55.808 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd in 6 2023-09-23T23:44:56.058 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:44:56.186 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:44:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:55 smithi110 ceph-mon[212939]: failed to encode map e739 with expected crc 2023-09-23T23:44:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:55 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3589203774' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T23:44:56.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:55 smithi110 ceph-mon[212939]: osdmap e739: 8 total, 8 up, 7 in 2023-09-23T23:44:56.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:55 smithi110 ceph-mon[212939]: failed to encode map e739 with expected crc 2023-09-23T23:44:56.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:55 smithi110 ceph-mon[212939]: failed to encode map e739 with expected crc 2023-09-23T23:44:56.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:55 smithi110 ceph-mon[212939]: failed to encode map e739 with expected crc 2023-09-23T23:44:56.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:55 smithi110 ceph-mon[212939]: failed to encode map e739 with expected crc 2023-09-23T23:44:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:55 smithi110 ceph-mon[221102]: failed to encode map e739 with expected crc 2023-09-23T23:44:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:55 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3589203774' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T23:44:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:55 smithi110 ceph-mon[221102]: osdmap e739: 8 total, 8 up, 7 in 2023-09-23T23:44:56.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:55 smithi110 ceph-mon[221102]: failed to encode map e739 with expected crc 2023-09-23T23:44:56.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:55 smithi110 ceph-mon[221102]: failed to encode map e739 with expected crc 2023-09-23T23:44:56.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:55 smithi110 ceph-mon[221102]: failed to encode map e739 with expected crc 2023-09-23T23:44:56.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:55 smithi110 ceph-mon[221102]: failed to encode map e739 with expected crc 2023-09-23T23:44:56.275 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:55 smithi138 ceph-mon[193606]: failed to encode map e739 with expected crc 2023-09-23T23:44:56.276 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:55 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3589203774' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T23:44:56.276 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:55 smithi138 ceph-mon[193606]: osdmap e739: 8 total, 8 up, 7 in 2023-09-23T23:44:56.276 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:55 smithi138 ceph-mon[193606]: failed to encode map e739 with expected crc 2023-09-23T23:44:56.276 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:55 smithi138 ceph-mon[193606]: failed to encode map e739 with expected crc 2023-09-23T23:44:56.276 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:55 smithi138 ceph-mon[193606]: failed to encode map e739 with expected crc 2023-09-23T23:44:56.276 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:55 smithi138 ceph-mon[193606]: failed to encode map e739 with expected crc 2023-09-23T23:44:57.203 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:44:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:44:56] "GET /metrics HTTP/1.1" 200 34199 "" "Prometheus/2.43.0" 2023-09-23T23:44:57.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[212939]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[212939]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[212939]: osdmap e740: 8 total, 8 up, 7 in 2023-09-23T23:44:57.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[212939]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[212939]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[212939]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[212939]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[212939]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[212939]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[212939]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[212939]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[212939]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[212939]: pgmap v2052: 103 pgs: 1 unknown, 6 peering, 96 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:57.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[212939]: osdmap e741: 8 total, 8 up, 7 in 2023-09-23T23:44:57.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[221102]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[221102]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[221102]: osdmap e740: 8 total, 8 up, 7 in 2023-09-23T23:44:57.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[221102]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[221102]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[221102]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[221102]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[221102]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.205 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[221102]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[221102]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[221102]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[221102]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[221102]: pgmap v2052: 103 pgs: 1 unknown, 6 peering, 96 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:57.206 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:56 smithi110 ceph-mon[221102]: osdmap e741: 8 total, 8 up, 7 in 2023-09-23T23:44:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:56 smithi138 ceph-mon[193606]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:56 smithi138 ceph-mon[193606]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:56 smithi138 ceph-mon[193606]: osdmap e740: 8 total, 8 up, 7 in 2023-09-23T23:44:57.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:56 smithi138 ceph-mon[193606]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:56 smithi138 ceph-mon[193606]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:56 smithi138 ceph-mon[193606]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:56 smithi138 ceph-mon[193606]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:56 smithi138 ceph-mon[193606]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:56 smithi138 ceph-mon[193606]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:56 smithi138 ceph-mon[193606]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:56 smithi138 ceph-mon[193606]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:56 smithi138 ceph-mon[193606]: failed to encode map e740 with expected crc 2023-09-23T23:44:57.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:56 smithi138 ceph-mon[193606]: pgmap v2052: 103 pgs: 1 unknown, 6 peering, 96 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:57.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:56 smithi138 ceph-mon[193606]: osdmap e741: 8 total, 8 up, 7 in 2023-09-23T23:44:57.670 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:44:57.671 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:44:57.671 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:44:57.671 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:44:57.671 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:44:58.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:57 smithi138 ceph-mon[193606]: failed to encode map e741 with expected crc 2023-09-23T23:44:58.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:57 smithi138 ceph-mon[193606]: failed to encode map e741 with expected crc 2023-09-23T23:44:58.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:57 smithi138 ceph-mon[193606]: failed to encode map e741 with expected crc 2023-09-23T23:44:58.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:57 smithi138 ceph-mon[193606]: failed to encode map e741 with expected crc 2023-09-23T23:44:58.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:57 smithi138 ceph-mon[193606]: osdmap e742: 8 total, 8 up, 7 in 2023-09-23T23:44:58.341 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T23:44:58.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:57 smithi110 ceph-mon[212939]: failed to encode map e741 with expected crc 2023-09-23T23:44:58.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:57 smithi110 ceph-mon[212939]: failed to encode map e741 with expected crc 2023-09-23T23:44:58.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:57 smithi110 ceph-mon[212939]: failed to encode map e741 with expected crc 2023-09-23T23:44:58.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:57 smithi110 ceph-mon[212939]: failed to encode map e741 with expected crc 2023-09-23T23:44:58.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:57 smithi110 ceph-mon[212939]: osdmap e742: 8 total, 8 up, 7 in 2023-09-23T23:44:58.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:57 smithi110 conmon[212914]: 2023-09-23T23:44:57.990+0000 7f29aa464700 -1 mon.a@0(leader).osd e742 definitely_dead 0 2023-09-23T23:44:58.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:57 smithi110 ceph-mon[221102]: failed to encode map e741 with expected crc 2023-09-23T23:44:58.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:57 smithi110 ceph-mon[221102]: failed to encode map e741 with expected crc 2023-09-23T23:44:58.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:57 smithi110 ceph-mon[221102]: failed to encode map e741 with expected crc 2023-09-23T23:44:58.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:57 smithi110 ceph-mon[221102]: failed to encode map e741 with expected crc 2023-09-23T23:44:58.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:57 smithi110 ceph-mon[221102]: osdmap e742: 8 total, 8 up, 7 in 2023-09-23T23:44:58.686 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:44:58.990 INFO:teuthology.orchestra.run.smithi110.stderr:marked in osd.6. 2023-09-23T23:44:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:59 smithi138 ceph-mon[193606]: failed to encode map e742 with expected crc 2023-09-23T23:44:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:59 smithi138 ceph-mon[193606]: failed to encode map e742 with expected crc 2023-09-23T23:44:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:59 smithi138 ceph-mon[193606]: failed to encode map e742 with expected crc 2023-09-23T23:44:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:59 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/372749540' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T23:44:59.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:59 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T23:44:59.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:59 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:44:59.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:59 smithi138 ceph-mon[193606]: pgmap v2055: 103 pgs: 3 remapped+peering, 1 unknown, 6 peering, 93 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:59.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:59 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T23:44:59.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:44:59 smithi138 ceph-mon[193606]: osdmap e743: 8 total, 8 up, 8 in 2023-09-23T23:44:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[212939]: failed to encode map e742 with expected crc 2023-09-23T23:44:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[212939]: failed to encode map e742 with expected crc 2023-09-23T23:44:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[212939]: failed to encode map e742 with expected crc 2023-09-23T23:44:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/372749540' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T23:44:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T23:44:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:44:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[212939]: pgmap v2055: 103 pgs: 3 remapped+peering, 1 unknown, 6 peering, 93 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:59.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T23:44:59.454 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[212939]: osdmap e743: 8 total, 8 up, 8 in 2023-09-23T23:44:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[221102]: failed to encode map e742 with expected crc 2023-09-23T23:44:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[221102]: failed to encode map e742 with expected crc 2023-09-23T23:44:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[221102]: failed to encode map e742 with expected crc 2023-09-23T23:44:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/372749540' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T23:44:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T23:44:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:44:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[221102]: pgmap v2055: 103 pgs: 3 remapped+peering, 1 unknown, 6 peering, 93 active+clean; 4.6 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail 2023-09-23T23:44:59.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T23:44:59.455 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:44:59 smithi110 ceph-mon[221102]: osdmap e743: 8 total, 8 up, 8 in 2023-09-23T23:44:59.822 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-09-23T23:44:59.822 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T23:44:59.822 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 71 2023-09-23T23:45:00.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:00 smithi138 ceph-mon[193606]: failed to encode map e743 with expected crc 2023-09-23T23:45:00.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:00 smithi138 ceph-mon[193606]: failed to encode map e743 with expected crc 2023-09-23T23:45:00.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:00 smithi138 ceph-mon[193606]: failed to encode map e742 with expected crc 2023-09-23T23:45:00.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:00 smithi138 ceph-mon[193606]: failed to encode map e743 with expected crc 2023-09-23T23:45:00.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:00 smithi138 ceph-mon[193606]: failed to encode map e743 with expected crc 2023-09-23T23:45:00.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:00 smithi138 ceph-mon[193606]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:45:00.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:00 smithi138 ceph-mon[193606]: osdmap e744: 8 total, 8 up, 8 in 2023-09-23T23:45:00.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:00 smithi110 ceph-mon[212939]: failed to encode map e743 with expected crc 2023-09-23T23:45:00.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:00 smithi110 ceph-mon[212939]: failed to encode map e743 with expected crc 2023-09-23T23:45:00.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:00 smithi110 ceph-mon[212939]: failed to encode map e742 with expected crc 2023-09-23T23:45:00.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:00 smithi110 ceph-mon[212939]: failed to encode map e743 with expected crc 2023-09-23T23:45:00.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:00 smithi110 ceph-mon[212939]: failed to encode map e743 with expected crc 2023-09-23T23:45:00.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:00 smithi110 ceph-mon[212939]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:45:00.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:00 smithi110 ceph-mon[212939]: osdmap e744: 8 total, 8 up, 8 in 2023-09-23T23:45:00.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:00 smithi110 ceph-mon[221102]: failed to encode map e743 with expected crc 2023-09-23T23:45:00.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:00 smithi110 ceph-mon[221102]: failed to encode map e743 with expected crc 2023-09-23T23:45:00.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:00 smithi110 ceph-mon[221102]: failed to encode map e742 with expected crc 2023-09-23T23:45:00.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:00 smithi110 ceph-mon[221102]: failed to encode map e743 with expected crc 2023-09-23T23:45:00.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:00 smithi110 ceph-mon[221102]: failed to encode map e743 with expected crc 2023-09-23T23:45:00.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:00 smithi110 ceph-mon[221102]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:45:00.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:00 smithi110 ceph-mon[221102]: osdmap e744: 8 total, 8 up, 8 in 2023-09-23T23:45:00.218 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:00.324 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:45:00.324 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:45:00.324 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:45:00.324 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:45:00.324 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.325 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46514.0:153463 13.3 e606/571 13:cf006423:::benchmark_data_smithi110_541928_object65683:head v 606'9682, mlcod=0'0)", 2023-09-23T23:45:00.325 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:38:16.630961+0000", 2023-09-23T23:45:00.325 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 403.68946608200002, 2023-09-23T23:45:00.325 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25026810500000002, 2023-09-23T23:45:00.325 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.325 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.325 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.630961+0000", 2023-09-23T23:45:00.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.326 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.630961+0000", 2023-09-23T23:45:00.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.630959+0000", 2023-09-23T23:45:00.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:45:00.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.630971+0000", 2023-09-23T23:45:00.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.207e-05 2023-09-23T23:45:00.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.630992+0000", 2023-09-23T23:45:00.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0177e-05 2023-09-23T23:45:00.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.329 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.329 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.631002+0000", 2023-09-23T23:45:00.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0772e-05 2023-09-23T23:45:00.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.329 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.631072+0000", 2023-09-23T23:45:00.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9306000000000004e-05 2023-09-23T23:45:00.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.631110+0000", 2023-09-23T23:45:00.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8408999999999998e-05 2023-09-23T23:45:00.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.331 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.331 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.881180+0000", 2023-09-23T23:45:00.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25006991499999998 2023-09-23T23:45:00.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.331 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.881229+0000", 2023-09-23T23:45:00.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8729000000000001e-05 2023-09-23T23:45:00.332 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.332 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.332 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.333 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(13.3 606/571 [PushOp(13:ce83adce:::benchmark_data_smithi110_541928_object65693:head, version: 554'4159, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(13:ce83adce:::benchmark_data_smithi110_541928_object65693:head@554'4159, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:45:00.333 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:38:16.632618+0000", 2023-09-23T23:45:00.333 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 403.68780837600002, 2023-09-23T23:45:00.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.248557219, 2023-09-23T23:45:00.333 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.333 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:00.333 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.632618+0000", 2023-09-23T23:45:00.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.632618+0000", 2023-09-23T23:45:00.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.632618+0000", 2023-09-23T23:45:00.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:00.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.632645+0000", 2023-09-23T23:45:00.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7268e-05 2023-09-23T23:45:00.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.875012+0000", 2023-09-23T23:45:00.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24236708800000001 2023-09-23T23:45:00.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.875052+0000", 2023-09-23T23:45:00.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9263e-05 2023-09-23T23:45:00.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.878339+0000", 2023-09-23T23:45:00.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032871229999999999 2023-09-23T23:45:00.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.878347+0000", 2023-09-23T23:45:00.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9410000000000007e-06 2023-09-23T23:45:00.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:38:16.881176+0000", 2023-09-23T23:45:00.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028289719999999999 2023-09-23T23:45:00.339 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.339 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.339 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.339 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.47186.0:15760 14.f e621/617 14:f2ab693e:::benchmark_data_smithi110_640334_object15759:head v 621'1010, mlcod=621'1008)", 2023-09-23T23:45:00.339 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:39:29.338854+0000", 2023-09-23T23:45:00.339 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 330.98157228600002, 2023-09-23T23:45:00.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17898309600000001, 2023-09-23T23:45:00.341 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.341 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.341 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.338854+0000", 2023-09-23T23:45:00.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.338854+0000", 2023-09-23T23:45:00.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.338852+0000", 2023-09-23T23:45:00.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:45:00.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.338913+0000", 2023-09-23T23:45:00.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0566000000000003e-05 2023-09-23T23:45:00.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.338941+0000", 2023-09-23T23:45:00.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8419e-05 2023-09-23T23:45:00.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.338953+0000", 2023-09-23T23:45:00.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1163e-05 2023-09-23T23:45:00.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.339066+0000", 2023-09-23T23:45:00.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011311099999999999 2023-09-23T23:45:00.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.339120+0000", 2023-09-23T23:45:00.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4236e-05 2023-09-23T23:45:00.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.348 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.348 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.517791+0000", 2023-09-23T23:45:00.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.178671474 2023-09-23T23:45:00.348 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.348 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.348 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.517838+0000", 2023-09-23T23:45:00.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6100000000000002e-05 2023-09-23T23:45:00.349 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.349 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.349 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.349 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.47186.0:15762 14.5 e621/617 14:a375336d:::benchmark_data_smithi110_640334_object15761:head v 621'992, mlcod=621'990)", 2023-09-23T23:45:00.349 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:39:29.340320+0000", 2023-09-23T23:45:00.350 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 330.980106267, 2023-09-23T23:45:00.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.176954217, 2023-09-23T23:45:00.350 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.350 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.350 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.350 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.350 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.340320+0000", 2023-09-23T23:45:00.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.340320+0000", 2023-09-23T23:45:00.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.340320+0000", 2023-09-23T23:45:00.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:00.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.340377+0000", 2023-09-23T23:45:00.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7104000000000001e-05 2023-09-23T23:45:00.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.515478+0000", 2023-09-23T23:45:00.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17510123999999999 2023-09-23T23:45:00.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.515486+0000", 2023-09-23T23:45:00.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6279999999999996e-06 2023-09-23T23:45:00.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.515567+0000", 2023-09-23T23:45:00.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1081999999999994e-05 2023-09-23T23:45:00.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.515588+0000", 2023-09-23T23:45:00.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1427e-05 2023-09-23T23:45:00.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.517248+0000", 2023-09-23T23:45:00.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001659811 2023-09-23T23:45:00.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:29.517275+0000", 2023-09-23T23:45:00.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6557e-05 2023-09-23T23:45:00.356 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.356 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.357 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.357 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.47186.0:18768 14.5 e622/617 14:a2f3bbdf:::benchmark_data_smithi110_640334_object18767:head v 622'1177, mlcod=621'1175)", 2023-09-23T23:45:00.357 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:39:31.069697+0000", 2023-09-23T23:45:00.357 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 329.25072966099998, 2023-09-23T23:45:00.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19107984, 2023-09-23T23:45:00.357 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.358 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.358 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.358 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.358 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.069697+0000", 2023-09-23T23:45:00.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.358 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.358 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.359 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.069697+0000", 2023-09-23T23:45:00.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.359 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.359 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.359 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.069697+0000", 2023-09-23T23:45:00.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:00.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.069770+0000", 2023-09-23T23:45:00.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3033000000000004e-05 2023-09-23T23:45:00.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.259501+0000", 2023-09-23T23:45:00.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.189730864 2023-09-23T23:45:00.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.259510+0000", 2023-09-23T23:45:00.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0920000000000004e-06 2023-09-23T23:45:00.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.259613+0000", 2023-09-23T23:45:00.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010329599999999999 2023-09-23T23:45:00.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.259633+0000", 2023-09-23T23:45:00.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0126999999999999e-05 2023-09-23T23:45:00.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.260743+0000", 2023-09-23T23:45:00.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011101730000000001 2023-09-23T23:45:00.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:39:31.260777+0000", 2023-09-23T23:45:00.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3704999999999997e-05 2023-09-23T23:45:00.364 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.364 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.364 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.365 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.57565.0:69 2.0 2:167b983a:::rbd_header.ba0228068ca6:head [watch watch cookie 140552267986528] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e705)", 2023-09-23T23:45:00.365 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:43:56.483474+0000", 2023-09-23T23:45:00.365 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 63.836952992000001, 2023-09-23T23:45:00.365 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001402354, 2023-09-23T23:45:00.365 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.365 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.365 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:45:00.365 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.57565", 2023-09-23T23:45:00.366 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3724050471", 2023-09-23T23:45:00.366 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 69 2023-09-23T23:45:00.366 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.366 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.366 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.366 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.366 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.483474+0000", 2023-09-23T23:45:00.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.483474+0000", 2023-09-23T23:45:00.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.483473+0000", 2023-09-23T23:45:00.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:00.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.483482+0000", 2023-09-23T23:45:00.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0219999999999993e-06 2023-09-23T23:45:00.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.483491+0000", 2023-09-23T23:45:00.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5259999999999993e-06 2023-09-23T23:45:00.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.483496+0000", 2023-09-23T23:45:00.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9910000000000002e-06 2023-09-23T23:45:00.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.370 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.483528+0000", 2023-09-23T23:45:00.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2166999999999999e-05 2023-09-23T23:45:00.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.483583+0000", 2023-09-23T23:45:00.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4653e-05 2023-09-23T23:45:00.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:45:00.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.483675+0000", 2023-09-23T23:45:00.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2250999999999995e-05 2023-09-23T23:45:00.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:45:00.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.483957+0000", 2023-09-23T23:45:00.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00028186999999999998 2023-09-23T23:45:00.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.484541+0000", 2023-09-23T23:45:00.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00058418800000000002 2023-09-23T23:45:00.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.484801+0000", 2023-09-23T23:45:00.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00026039699999999999 2023-09-23T23:45:00.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.484832+0000", 2023-09-23T23:45:00.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0961999999999999e-05 2023-09-23T23:45:00.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.484876+0000", 2023-09-23T23:45:00.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3869000000000003e-05 2023-09-23T23:45:00.375 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.375 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.375 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.375 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.57565.0:86 2.0 2:167b983a:::rbd_header.ba0228068ca6:head [watch unwatch cookie 140552267986528] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e705)", 2023-09-23T23:45:00.376 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:43:56.505975+0000", 2023-09-23T23:45:00.376 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 63.814451986000002, 2023-09-23T23:45:00.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0013434689999999999, 2023-09-23T23:45:00.376 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.376 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.376 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:45:00.376 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.57565", 2023-09-23T23:45:00.376 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3724050471", 2023-09-23T23:45:00.377 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 86 2023-09-23T23:45:00.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.377 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.505975+0000", 2023-09-23T23:45:00.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.505975+0000", 2023-09-23T23:45:00.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.379 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.379 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.505974+0000", 2023-09-23T23:45:00.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:00.379 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.505976+0000", 2023-09-23T23:45:00.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6360000000000001e-06 2023-09-23T23:45:00.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.505980+0000", 2023-09-23T23:45:00.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7850000000000002e-06 2023-09-23T23:45:00.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.505982+0000", 2023-09-23T23:45:00.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6280000000000001e-06 2023-09-23T23:45:00.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.506238+0000", 2023-09-23T23:45:00.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025527599999999997 2023-09-23T23:45:00.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.506288+0000", 2023-09-23T23:45:00.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0658000000000002e-05 2023-09-23T23:45:00.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:45:00.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.506370+0000", 2023-09-23T23:45:00.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1269999999999999e-05 2023-09-23T23:45:00.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:45:00.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.506661+0000", 2023-09-23T23:45:00.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00029135700000000001 2023-09-23T23:45:00.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.384 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.507200+0000", 2023-09-23T23:45:00.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00053865399999999996 2023-09-23T23:45:00.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.507226+0000", 2023-09-23T23:45:00.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5933000000000001e-05 2023-09-23T23:45:00.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.507259+0000", 2023-09-23T23:45:00.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3086999999999997e-05 2023-09-23T23:45:00.386 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.386 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.386 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.507318+0000", 2023-09-23T23:45:00.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9506999999999998e-05 2023-09-23T23:45:00.386 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.386 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.387 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.387 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.57565.0:88 2.0 2:167b983a:::rbd_header.ba0228068ca6:head [delete] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e705)", 2023-09-23T23:45:00.387 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:43:56.507739+0000", 2023-09-23T23:45:00.387 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 63.812687898999997, 2023-09-23T23:45:00.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0016601470000000001, 2023-09-23T23:45:00.388 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.388 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.388 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:45:00.388 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.57565", 2023-09-23T23:45:00.388 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3724050471", 2023-09-23T23:45:00.388 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 88 2023-09-23T23:45:00.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.388 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.507739+0000", 2023-09-23T23:45:00.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.507739+0000", 2023-09-23T23:45:00.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.507738+0000", 2023-09-23T23:45:00.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:00.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.507741+0000", 2023-09-23T23:45:00.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6979999999999998e-06 2023-09-23T23:45:00.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.507750+0000", 2023-09-23T23:45:00.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7269999999999996e-06 2023-09-23T23:45:00.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.507753+0000", 2023-09-23T23:45:00.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7610000000000001e-06 2023-09-23T23:45:00.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.393 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.393 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.507785+0000", 2023-09-23T23:45:00.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1049e-05 2023-09-23T23:45:00.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.393 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.507859+0000", 2023-09-23T23:45:00.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4634000000000004e-05 2023-09-23T23:45:00.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:45:00.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.507969+0000", 2023-09-23T23:45:00.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010974100000000001 2023-09-23T23:45:00.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:45:00.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.508324+0000", 2023-09-23T23:45:00.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00035504599999999997 2023-09-23T23:45:00.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.508882+0000", 2023-09-23T23:45:00.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00055798899999999995 2023-09-23T23:45:00.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.509250+0000", 2023-09-23T23:45:00.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000368038 2023-09-23T23:45:00.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.397 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.397 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.509333+0000", 2023-09-23T23:45:00.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3108000000000005e-05 2023-09-23T23:45:00.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.398 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.398 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:43:56.509399+0000", 2023-09-23T23:45:00.398 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5885999999999999e-05 2023-09-23T23:45:00.398 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.398 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.398 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.398 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.398 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.37815.0:29 2.3 e714/410 2:d47d8c36:::rbd_data.93b77b1d51fe.0000000000000001:head v 714'190, mlcod=714'188)", 2023-09-23T23:45:00.399 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:06.079121+0000", 2023-09-23T23:45:00.399 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 54.241306109999996, 2023-09-23T23:45:00.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034993049999999999, 2023-09-23T23:45:00.399 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.399 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.399 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.079121+0000", 2023-09-23T23:45:00.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.079121+0000", 2023-09-23T23:45:00.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.079120+0000", 2023-09-23T23:45:00.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:45:00.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.081013+0000", 2023-09-23T23:45:00.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0018926749999999999 2023-09-23T23:45:00.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.081030+0000", 2023-09-23T23:45:00.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7385999999999999e-05 2023-09-23T23:45:00.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.081035+0000", 2023-09-23T23:45:00.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4960000000000004e-06 2023-09-23T23:45:00.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.081094+0000", 2023-09-23T23:45:00.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8631999999999997e-05 2023-09-23T23:45:00.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.081121+0000", 2023-09-23T23:45:00.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6444999999999998e-05 2023-09-23T23:45:00.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.082583+0000", 2023-09-23T23:45:00.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014623139999999999 2023-09-23T23:45:00.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.403 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.403 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.082620+0000", 2023-09-23T23:45:00.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7103e-05 2023-09-23T23:45:00.403 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.403 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.403 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.403 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.47900.0:9 2.0 2:061d1dd3:::rbd_object_map.bb1cea0e5cc5:head [call rbd.object_map_resize in=20b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e714)", 2023-09-23T23:45:00.403 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:06.514722+0000", 2023-09-23T23:45:00.404 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 53.805704773999999, 2023-09-23T23:45:00.404 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001151652, 2023-09-23T23:45:00.404 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.404 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.404 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:45:00.404 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.47900", 2023-09-23T23:45:00.404 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2789370275", 2023-09-23T23:45:00.404 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 9 2023-09-23T23:45:00.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.404 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.404 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.514722+0000", 2023-09-23T23:45:00.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.514722+0000", 2023-09-23T23:45:00.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.514721+0000", 2023-09-23T23:45:00.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:45:00.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.406 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.406 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.514726+0000", 2023-09-23T23:45:00.406 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2869999999999998e-06 2023-09-23T23:45:00.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.406 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.406 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.514734+0000", 2023-09-23T23:45:00.406 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1620000000000008e-06 2023-09-23T23:45:00.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.406 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.406 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.514740+0000", 2023-09-23T23:45:00.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3679999999999998e-06 2023-09-23T23:45:00.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.514788+0000", 2023-09-23T23:45:00.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7970000000000003e-05 2023-09-23T23:45:00.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.514885+0000", 2023-09-23T23:45:00.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6986999999999998e-05 2023-09-23T23:45:00.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:45:00.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.515025+0000", 2023-09-23T23:45:00.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014017000000000001 2023-09-23T23:45:00.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:45:00.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.515484+0000", 2023-09-23T23:45:00.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000458939 2023-09-23T23:45:00.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.515794+0000", 2023-09-23T23:45:00.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00031035999999999998 2023-09-23T23:45:00.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.515815+0000", 2023-09-23T23:45:00.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0217999999999999e-05 2023-09-23T23:45:00.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.515847+0000", 2023-09-23T23:45:00.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2168000000000001e-05 2023-09-23T23:45:00.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.410 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.515874+0000", 2023-09-23T23:45:00.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6772e-05 2023-09-23T23:45:00.410 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.410 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.410 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.410 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.47900.0:24 2.0 2:061d1dd3:::rbd_object_map.bb1cea0e5cc5:head [call lock.lock in=8b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e714)", 2023-09-23T23:45:00.410 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:06.524544+0000", 2023-09-23T23:45:00.410 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 53.795882503999998, 2023-09-23T23:45:00.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011508320000000001, 2023-09-23T23:45:00.410 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.411 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.411 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:45:00.411 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.47900", 2023-09-23T23:45:00.411 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2789370275", 2023-09-23T23:45:00.411 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 24 2023-09-23T23:45:00.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.411 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.411 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.524544+0000", 2023-09-23T23:45:00.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.412 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.524544+0000", 2023-09-23T23:45:00.412 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.412 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.412 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.412 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.524543+0000", 2023-09-23T23:45:00.412 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:45:00.412 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.412 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.412 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.524548+0000", 2023-09-23T23:45:00.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7439999999999998e-06 2023-09-23T23:45:00.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.524561+0000", 2023-09-23T23:45:00.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3148e-05 2023-09-23T23:45:00.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.524568+0000", 2023-09-23T23:45:00.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5350000000000001e-06 2023-09-23T23:45:00.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.524634+0000", 2023-09-23T23:45:00.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6344e-05 2023-09-23T23:45:00.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.524695+0000", 2023-09-23T23:45:00.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0772e-05 2023-09-23T23:45:00.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:45:00.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.524822+0000", 2023-09-23T23:45:00.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012693399999999999 2023-09-23T23:45:00.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:45:00.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.525182+0000", 2023-09-23T23:45:00.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00035968600000000001 2023-09-23T23:45:00.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.525601+0000", 2023-09-23T23:45:00.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00041895099999999998 2023-09-23T23:45:00.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.416 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.525636+0000", 2023-09-23T23:45:00.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5169999999999997e-05 2023-09-23T23:45:00.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.416 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.525672+0000", 2023-09-23T23:45:00.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5781000000000002e-05 2023-09-23T23:45:00.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.525695+0000", 2023-09-23T23:45:00.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3532000000000001e-05 2023-09-23T23:45:00.417 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.417 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.417 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.417 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.417 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.47900.0:30 2.0 2:061d1dd3:::rbd_object_map.bb1cea0e5cc5:head [call lock.unlock in=10b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e714)", 2023-09-23T23:45:00.417 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:06.687090+0000", 2023-09-23T23:45:00.417 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 53.633336731999997, 2023-09-23T23:45:00.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0013206159999999999, 2023-09-23T23:45:00.417 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.417 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.418 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:45:00.418 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.47900", 2023-09-23T23:45:00.418 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2789370275", 2023-09-23T23:45:00.418 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 30 2023-09-23T23:45:00.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.418 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.687090+0000", 2023-09-23T23:45:00.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.687090+0000", 2023-09-23T23:45:00.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.687088+0000", 2023-09-23T23:45:00.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:45:00.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.687100+0000", 2023-09-23T23:45:00.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1420000000000001e-05 2023-09-23T23:45:00.420 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.420 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.687156+0000", 2023-09-23T23:45:00.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6848999999999997e-05 2023-09-23T23:45:00.420 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.420 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.687176+0000", 2023-09-23T23:45:00.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9340000000000001e-05 2023-09-23T23:45:00.420 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.421 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.687268+0000", 2023-09-23T23:45:00.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2328000000000001e-05 2023-09-23T23:45:00.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.421 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.421 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.687327+0000", 2023-09-23T23:45:00.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9224000000000001e-05 2023-09-23T23:45:00.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.421 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:45:00.421 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.687438+0000", 2023-09-23T23:45:00.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110319 2023-09-23T23:45:00.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:45:00.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.687796+0000", 2023-09-23T23:45:00.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00035839699999999999 2023-09-23T23:45:00.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.688284+0000", 2023-09-23T23:45:00.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000487877 2023-09-23T23:45:00.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.688350+0000", 2023-09-23T23:45:00.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6129000000000003e-05 2023-09-23T23:45:00.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.688386+0000", 2023-09-23T23:45:00.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5898000000000001e-05 2023-09-23T23:45:00.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.688411+0000", 2023-09-23T23:45:00.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4695e-05 2023-09-23T23:45:00.423 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.424 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.424 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.424 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.37905.0:72 2.0 2:061d1dd3:::rbd_object_map.bb1cea0e5cc5:head [call lock.lock in=8b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e715)", 2023-09-23T23:45:00.424 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:06.966108+0000", 2023-09-23T23:45:00.424 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 53.354318587999998, 2023-09-23T23:45:00.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00119641, 2023-09-23T23:45:00.424 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.424 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.424 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:45:00.424 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.37905", 2023-09-23T23:45:00.425 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2247100638", 2023-09-23T23:45:00.425 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 72 2023-09-23T23:45:00.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.425 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.966108+0000", 2023-09-23T23:45:00.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.966108+0000", 2023-09-23T23:45:00.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.966106+0000", 2023-09-23T23:45:00.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:45:00.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.966118+0000", 2023-09-23T23:45:00.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1687e-05 2023-09-23T23:45:00.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.427 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.966169+0000", 2023-09-23T23:45:00.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0979000000000001e-05 2023-09-23T23:45:00.427 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.427 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.427 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.966182+0000", 2023-09-23T23:45:00.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2699e-05 2023-09-23T23:45:00.427 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.427 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.427 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.966279+0000", 2023-09-23T23:45:00.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6968999999999999e-05 2023-09-23T23:45:00.428 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.428 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.428 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.966430+0000", 2023-09-23T23:45:00.428 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015175 2023-09-23T23:45:00.428 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.428 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:45:00.428 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.966607+0000", 2023-09-23T23:45:00.428 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017707399999999999 2023-09-23T23:45:00.428 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.428 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:45:00.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.966957+0000", 2023-09-23T23:45:00.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00034925199999999998 2023-09-23T23:45:00.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.967210+0000", 2023-09-23T23:45:00.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025377800000000002 2023-09-23T23:45:00.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.967234+0000", 2023-09-23T23:45:00.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3521000000000001e-05 2023-09-23T23:45:00.430 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.430 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.430 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.430 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.967265+0000", 2023-09-23T23:45:00.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0885e-05 2023-09-23T23:45:00.430 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.430 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.430 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.430 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:06.967305+0000", 2023-09-23T23:45:00.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9730000000000001e-05 2023-09-23T23:45:00.430 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.431 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.431 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.431 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.431 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.47945.0:38 2.0 2:166f20f4:::rbd_data.bb49fbbf69e8.0000000000000001:head [set-alloc-hint object_size 1048576 write_size 1048576,writefull 0~1048576 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e715)", 2023-09-23T23:45:00.431 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:07.041715+0000", 2023-09-23T23:45:00.431 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 53.278712065000001, 2023-09-23T23:45:00.431 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064356910000000003, 2023-09-23T23:45:00.431 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.431 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.431 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:45:00.431 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.47945", 2023-09-23T23:45:00.431 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/328333598", 2023-09-23T23:45:00.432 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 38 2023-09-23T23:45:00.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.432 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.041715+0000", 2023-09-23T23:45:00.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.041715+0000", 2023-09-23T23:45:00.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.433 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.041714+0000", 2023-09-23T23:45:00.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:45:00.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.433 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.043962+0000", 2023-09-23T23:45:00.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022477980000000001 2023-09-23T23:45:00.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.043973+0000", 2023-09-23T23:45:00.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1131000000000001e-05 2023-09-23T23:45:00.434 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.043978+0000", 2023-09-23T23:45:00.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9089999999999999e-06 2023-09-23T23:45:00.434 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.044024+0000", 2023-09-23T23:45:00.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6307999999999998e-05 2023-09-23T23:45:00.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.435 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.435 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.044120+0000", 2023-09-23T23:45:00.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6073e-05 2023-09-23T23:45:00.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.435 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:45:00.435 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.044219+0000", 2023-09-23T23:45:00.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9267999999999999e-05 2023-09-23T23:45:00.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.436 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.436 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:45:00.436 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.045527+0000", 2023-09-23T23:45:00.436 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0013074860000000001 2023-09-23T23:45:00.436 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.436 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.436 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.436 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.047991+0000", 2023-09-23T23:45:00.436 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024643709999999999 2023-09-23T23:45:00.436 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.436 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.436 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.048064+0000", 2023-09-23T23:45:00.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2677999999999998e-05 2023-09-23T23:45:00.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.048101+0000", 2023-09-23T23:45:00.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7505000000000001e-05 2023-09-23T23:45:00.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.048150+0000", 2023-09-23T23:45:00.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8912999999999997e-05 2023-09-23T23:45:00.438 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.438 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.438 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.438 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.438 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.47978.0:13 2.0 2:166f20f4:::rbd_data.bb49fbbf69e8.0000000000000001:head [sparse-read 0~1048576 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e715)", 2023-09-23T23:45:00.438 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:07.347213+0000", 2023-09-23T23:45:00.438 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 52.973213860999998, 2023-09-23T23:45:00.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032900830000000001, 2023-09-23T23:45:00.438 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.438 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:00.438 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:45:00.439 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.47978", 2023-09-23T23:45:00.439 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/3887577341", 2023-09-23T23:45:00.439 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 13 2023-09-23T23:45:00.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.439 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.439 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.347213+0000", 2023-09-23T23:45:00.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.439 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.347213+0000", 2023-09-23T23:45:00.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.347212+0000", 2023-09-23T23:45:00.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:00.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.347216+0000", 2023-09-23T23:45:00.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4489999999999999e-06 2023-09-23T23:45:00.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.347224+0000", 2023-09-23T23:45:00.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0809999999999994e-06 2023-09-23T23:45:00.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.347228+0000", 2023-09-23T23:45:00.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7700000000000001e-06 2023-09-23T23:45:00.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.347274+0000", 2023-09-23T23:45:00.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5698999999999997e-05 2023-09-23T23:45:00.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.347334+0000", 2023-09-23T23:45:00.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9998999999999999e-05 2023-09-23T23:45:00.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.350503+0000", 2023-09-23T23:45:00.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031687709999999999 2023-09-23T23:45:00.442 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.443 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.443 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.443 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.37956.0:51 2.7 e715/692 2:e1440a79:::rbd_data.94444472da1a.0000000000000003:head v 715'131, mlcod=715'129)", 2023-09-23T23:45:00.443 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:07.535230+0000", 2023-09-23T23:45:00.443 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 52.78519644, 2023-09-23T23:45:00.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002161638, 2023-09-23T23:45:00.443 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.443 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.443 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.535230+0000", 2023-09-23T23:45:00.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.535230+0000", 2023-09-23T23:45:00.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.535229+0000", 2023-09-23T23:45:00.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-09-23T23:45:00.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.536419+0000", 2023-09-23T23:45:00.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011900229999999999 2023-09-23T23:45:00.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.536430+0000", 2023-09-23T23:45:00.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1620999999999999e-05 2023-09-23T23:45:00.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.536435+0000", 2023-09-23T23:45:00.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4050000000000002e-06 2023-09-23T23:45:00.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.536486+0000", 2023-09-23T23:45:00.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1641000000000002e-05 2023-09-23T23:45:00.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.536501+0000", 2023-09-23T23:45:00.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4674e-05 2023-09-23T23:45:00.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.537364+0000", 2023-09-23T23:45:00.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00086294900000000001 2023-09-23T23:45:00.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.537392+0000", 2023-09-23T23:45:00.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7827999999999999e-05 2023-09-23T23:45:00.447 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.448 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.448 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.448 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.38001.0:29 2.0 2:0b674657:::rbd_data.9471508d9fa2.0000000000000001:head [set-alloc-hint object_size 1048576 write_size 1048576,writefull 0~1048576 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e715)", 2023-09-23T23:45:00.448 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:07.873042+0000", 2023-09-23T23:45:00.448 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 52.447384315000001, 2023-09-23T23:45:00.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0054184619999999998, 2023-09-23T23:45:00.448 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.448 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.448 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:45:00.448 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.38001", 2023-09-23T23:45:00.448 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/795940386", 2023-09-23T23:45:00.449 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 29 2023-09-23T23:45:00.449 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.449 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.449 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.449 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.873042+0000", 2023-09-23T23:45:00.449 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.449 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.449 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.449 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.873042+0000", 2023-09-23T23:45:00.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.873042+0000", 2023-09-23T23:45:00.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-09-23T23:45:00.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.874053+0000", 2023-09-23T23:45:00.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010113629999999999 2023-09-23T23:45:00.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.874063+0000", 2023-09-23T23:45:00.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0396e-05 2023-09-23T23:45:00.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.874070+0000", 2023-09-23T23:45:00.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2500000000000003e-06 2023-09-23T23:45:00.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.874163+0000", 2023-09-23T23:45:00.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3401999999999998e-05 2023-09-23T23:45:00.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.874260+0000", 2023-09-23T23:45:00.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7019000000000001e-05 2023-09-23T23:45:00.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:45:00.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.874356+0000", 2023-09-23T23:45:00.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6087000000000003e-05 2023-09-23T23:45:00.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:45:00.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.875410+0000", 2023-09-23T23:45:00.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010538419999999999 2023-09-23T23:45:00.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.878232+0000", 2023-09-23T23:45:00.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002821872 2023-09-23T23:45:00.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.878303+0000", 2023-09-23T23:45:00.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1389000000000004e-05 2023-09-23T23:45:00.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.454 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.878397+0000", 2023-09-23T23:45:00.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3370999999999998e-05 2023-09-23T23:45:00.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.454 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:07.878461+0000", 2023-09-23T23:45:00.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4372e-05 2023-09-23T23:45:00.454 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.454 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.455 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.455 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.48062.0:80 2.0 2:0b674657:::rbd_data.9471508d9fa2.0000000000000001:head [delete] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e715)", 2023-09-23T23:45:00.455 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:08.066736+0000", 2023-09-23T23:45:00.455 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 52.253690992000003, 2023-09-23T23:45:00.455 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011582039999999999, 2023-09-23T23:45:00.455 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.455 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.455 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:45:00.455 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.48062", 2023-09-23T23:45:00.455 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/2630528106", 2023-09-23T23:45:00.456 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 80 2023-09-23T23:45:00.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.456 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.066736+0000", 2023-09-23T23:45:00.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.066736+0000", 2023-09-23T23:45:00.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.066734+0000", 2023-09-23T23:45:00.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:45:00.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.066743+0000", 2023-09-23T23:45:00.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2259999999999994e-06 2023-09-23T23:45:00.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.458 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.066765+0000", 2023-09-23T23:45:00.458 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1648999999999999e-05 2023-09-23T23:45:00.458 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.458 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.458 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.458 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.066782+0000", 2023-09-23T23:45:00.458 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7496999999999999e-05 2023-09-23T23:45:00.458 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.458 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.458 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.458 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.066955+0000", 2023-09-23T23:45:00.458 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017252000000000001 2023-09-23T23:45:00.459 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.459 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.459 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.459 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.067060+0000", 2023-09-23T23:45:00.459 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105618 2023-09-23T23:45:00.459 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.459 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.459 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:45:00.459 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.067170+0000", 2023-09-23T23:45:00.459 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110006 2023-09-23T23:45:00.459 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.459 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.459 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:45:00.460 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.067555+0000", 2023-09-23T23:45:00.460 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00038498500000000001 2023-09-23T23:45:00.460 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.460 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.460 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.460 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.067810+0000", 2023-09-23T23:45:00.460 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025504199999999997 2023-09-23T23:45:00.460 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.460 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.460 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.460 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.067839+0000", 2023-09-23T23:45:00.460 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8941999999999998e-05 2023-09-23T23:45:00.461 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.461 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.461 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.461 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.067870+0000", 2023-09-23T23:45:00.461 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0857e-05 2023-09-23T23:45:00.461 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.461 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.461 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.461 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.067894+0000", 2023-09-23T23:45:00.461 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3708000000000001e-05 2023-09-23T23:45:00.461 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.461 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.461 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.462 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.462 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.462 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.38034.0:30 2.0 2:069c7064:::rbd_object_map.949280edd016:head [call lock.unlock in=10b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e715)", 2023-09-23T23:45:00.462 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:08.134158+0000", 2023-09-23T23:45:00.462 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 52.18626854, 2023-09-23T23:45:00.462 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001131068, 2023-09-23T23:45:00.462 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.462 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.462 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:45:00.462 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.38034", 2023-09-23T23:45:00.462 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/115858723", 2023-09-23T23:45:00.462 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 30 2023-09-23T23:45:00.463 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.463 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.463 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.134158+0000", 2023-09-23T23:45:00.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.463 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.463 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.134158+0000", 2023-09-23T23:45:00.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.463 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.464 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.464 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.134157+0000", 2023-09-23T23:45:00.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:00.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.464 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.464 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.134162+0000", 2023-09-23T23:45:00.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5539999999999999e-06 2023-09-23T23:45:00.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.464 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.464 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.134175+0000", 2023-09-23T23:45:00.465 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3341e-05 2023-09-23T23:45:00.465 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.465 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.465 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.465 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.134182+0000", 2023-09-23T23:45:00.465 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.426e-06 2023-09-23T23:45:00.465 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.465 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.465 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.465 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.134213+0000", 2023-09-23T23:45:00.465 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1566000000000002e-05 2023-09-23T23:45:00.465 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.466 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.466 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.466 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.134266+0000", 2023-09-23T23:45:00.466 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2845e-05 2023-09-23T23:45:00.466 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.466 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.466 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:45:00.466 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.134378+0000", 2023-09-23T23:45:00.466 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000111674 2023-09-23T23:45:00.466 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.466 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.466 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:45:00.467 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.134682+0000", 2023-09-23T23:45:00.467 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00030434700000000001 2023-09-23T23:45:00.467 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.467 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.467 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.467 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.135159+0000", 2023-09-23T23:45:00.467 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00047689999999999999 2023-09-23T23:45:00.467 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.467 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.467 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.467 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.135235+0000", 2023-09-23T23:45:00.467 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5722000000000004e-05 2023-09-23T23:45:00.468 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.468 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.468 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.468 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.135266+0000", 2023-09-23T23:45:00.468 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1535999999999998e-05 2023-09-23T23:45:00.468 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.468 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.468 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.468 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.135289+0000", 2023-09-23T23:45:00.468 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2860000000000001e-05 2023-09-23T23:45:00.468 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.468 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.468 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.469 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.469 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.469 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.38061.0:72 2.0 2:069c7064:::rbd_object_map.949280edd016:head [call lock.lock in=8b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e715)", 2023-09-23T23:45:00.469 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:08.564929+0000", 2023-09-23T23:45:00.469 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 51.755498045000003, 2023-09-23T23:45:00.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00115661, 2023-09-23T23:45:00.469 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:00.469 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:45:00.469 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-09-23T23:45:00.469 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.38061", 2023-09-23T23:45:00.469 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.110:0/1837403620", 2023-09-23T23:45:00.469 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 72 2023-09-23T23:45:00.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.470 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:00.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.470 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:00.470 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.564929+0000", 2023-09-23T23:45:00.470 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.470 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:00.470 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.564929+0000", 2023-09-23T23:45:00.470 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:00.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:00.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.564927+0000", 2023-09-23T23:45:00.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-09-23T23:45:00.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:00.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.564939+0000", 2023-09-23T23:45:00.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1919999999999999e-05 2023-09-23T23:45:00.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:00.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.564971+0000", 2023-09-23T23:45:00.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2608999999999999e-05 2023-09-23T23:45:00.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:00.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.564985+0000", 2023-09-23T23:45:00.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.343e-05 2023-09-23T23:45:00.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:00.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.565095+0000", 2023-09-23T23:45:00.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110461 2023-09-23T23:45:00.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:00.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.565262+0000", 2023-09-23T23:45:00.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000166701 2023-09-23T23:45:00.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:45:00.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.565421+0000", 2023-09-23T23:45:00.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015916699999999999 2023-09-23T23:45:00.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-09-23T23:45:00.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.565772+0000", 2023-09-23T23:45:00.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000350899 2023-09-23T23:45:00.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.566003+0000", 2023-09-23T23:45:00.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000231406 2023-09-23T23:45:00.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:45:00.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.566029+0000", 2023-09-23T23:45:00.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5395999999999999e-05 2023-09-23T23:45:00.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.475 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-09-23T23:45:00.475 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.566061+0000", 2023-09-23T23:45:00.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2230999999999997e-05 2023-09-23T23:45:00.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:00.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:00.475 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:00.475 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:08.566085+0000", 2023-09-23T23:45:00.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4386999999999999e-05 2023-09-23T23:45:00.475 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.475 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.475 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.475 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:00.476 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:00.476 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:45:01.012 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[221102]: 1.1a scrub starts 2023-09-23T23:45:01.012 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[221102]: 1.1a scrub ok 2023-09-23T23:45:01.013 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[221102]: failed to encode map e744 with expected crc 2023-09-23T23:45:01.013 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[221102]: failed to encode map e744 with expected crc 2023-09-23T23:45:01.013 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[221102]: failed to encode map e744 with expected crc 2023-09-23T23:45:01.013 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[221102]: failed to encode map e744 with expected crc 2023-09-23T23:45:01.013 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[221102]: failed to encode map e744 with expected crc 2023-09-23T23:45:01.013 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[221102]: pgmap v2058: 103 pgs: 1 unknown, 28 peering, 74 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2023-09-23T23:45:01.013 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[221102]: osdmap e745: 8 total, 8 up, 8 in 2023-09-23T23:45:01.040 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:45:01.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[212939]: 1.1a scrub starts 2023-09-23T23:45:01.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[212939]: 1.1a scrub ok 2023-09-23T23:45:01.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[212939]: failed to encode map e744 with expected crc 2023-09-23T23:45:01.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[212939]: failed to encode map e744 with expected crc 2023-09-23T23:45:01.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[212939]: failed to encode map e744 with expected crc 2023-09-23T23:45:01.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[212939]: failed to encode map e744 with expected crc 2023-09-23T23:45:01.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[212939]: failed to encode map e744 with expected crc 2023-09-23T23:45:01.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[212939]: pgmap v2058: 103 pgs: 1 unknown, 28 peering, 74 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2023-09-23T23:45:01.301 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:01 smithi110 ceph-mon[212939]: osdmap e745: 8 total, 8 up, 8 in 2023-09-23T23:45:01.394 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:45:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:01 smithi138 ceph-mon[193606]: 1.1a scrub starts 2023-09-23T23:45:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:01 smithi138 ceph-mon[193606]: 1.1a scrub ok 2023-09-23T23:45:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:01 smithi138 ceph-mon[193606]: failed to encode map e744 with expected crc 2023-09-23T23:45:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:01 smithi138 ceph-mon[193606]: failed to encode map e744 with expected crc 2023-09-23T23:45:01.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:01 smithi138 ceph-mon[193606]: failed to encode map e744 with expected crc 2023-09-23T23:45:01.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:01 smithi138 ceph-mon[193606]: failed to encode map e744 with expected crc 2023-09-23T23:45:01.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:01 smithi138 ceph-mon[193606]: failed to encode map e744 with expected crc 2023-09-23T23:45:01.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:01 smithi138 ceph-mon[193606]: pgmap v2058: 103 pgs: 1 unknown, 28 peering, 74 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2023-09-23T23:45:01.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:01 smithi138 ceph-mon[193606]: osdmap e745: 8 total, 8 up, 8 in 2023-09-23T23:45:01.951 INFO:teuthology.orchestra.run.smithi110.stderr:Error EINVAL: specified pgp_num 71 > pg_num 31 2023-09-23T23:45:02.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:02 smithi138 ceph-mon[193606]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:02 smithi138 ceph-mon[193606]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:02 smithi138 ceph-mon[193606]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:02 smithi138 ceph-mon[193606]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:02 smithi138 ceph-mon[193606]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:02 smithi138 ceph-mon[193606]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:02 smithi138 ceph-mon[193606]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:02 smithi138 ceph-mon[193606]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:02 smithi138 ceph-mon[193606]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:02 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1443349609' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "71"}]: dispatch 2023-09-23T23:45:02.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[212939]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[212939]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[212939]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[212939]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[212939]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[212939]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[212939]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[212939]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[212939]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1443349609' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "71"}]: dispatch 2023-09-23T23:45:02.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[221102]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[221102]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[221102]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[221102]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[221102]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[221102]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[221102]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[221102]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[221102]: failed to encode map e745 with expected crc 2023-09-23T23:45:02.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:02 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1443349609' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "71"}]: dispatch 2023-09-23T23:45:02.725 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T23:45:02.726 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-09-23T23:45:02.989 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:45:02.990 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:45:02.991 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-09-23T23:45:02.991 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:45:03.025 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[212939]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.026 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[212939]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.026 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[212939]: osdmap e746: 8 total, 8 up, 8 in 2023-09-23T23:45:03.026 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[212939]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.026 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[212939]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.026 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[212939]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.027 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[212939]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.027 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[212939]: pgmap v2061: 103 pgs: 2 active+undersized+remapped, 4 remapped+peering, 21 peering, 1 active+remapped, 75 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2023-09-23T23:45:03.027 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:45:03.027 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[221102]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.027 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[221102]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.028 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[221102]: osdmap e746: 8 total, 8 up, 8 in 2023-09-23T23:45:03.028 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[221102]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.028 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[221102]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.028 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[221102]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.028 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[221102]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.028 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[221102]: pgmap v2061: 103 pgs: 2 active+undersized+remapped, 4 remapped+peering, 21 peering, 1 active+remapped, 75 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2023-09-23T23:45:03.029 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:03 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:45:03.188 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:03 smithi138 ceph-mon[193606]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:03 smithi138 ceph-mon[193606]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:03 smithi138 ceph-mon[193606]: osdmap e746: 8 total, 8 up, 8 in 2023-09-23T23:45:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:03 smithi138 ceph-mon[193606]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:03 smithi138 ceph-mon[193606]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:03 smithi138 ceph-mon[193606]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:03 smithi138 ceph-mon[193606]: failed to encode map e746 with expected crc 2023-09-23T23:45:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:03 smithi138 ceph-mon[193606]: pgmap v2061: 103 pgs: 2 active+undersized+remapped, 4 remapped+peering, 21 peering, 1 active+remapped, 75 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2023-09-23T23:45:03.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:03 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:45:04.030 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:45:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:04 smithi138 ceph-mon[193606]: failed to encode map e747 with expected crc 2023-09-23T23:45:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:04 smithi138 ceph-mon[193606]: failed to encode map e747 with expected crc 2023-09-23T23:45:04.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:04 smithi138 ceph-mon[193606]: failed to encode map e747 with expected crc 2023-09-23T23:45:04.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:04 smithi138 ceph-mon[193606]: osdmap e747: 8 total, 8 up, 8 in 2023-09-23T23:45:04.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:04 smithi138 ceph-mon[193606]: failed to encode map e747 with expected crc 2023-09-23T23:45:04.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:04 smithi138 ceph-mon[193606]: failed to encode map e747 with expected crc 2023-09-23T23:45:04.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:04 smithi110 ceph-mon[212939]: failed to encode map e747 with expected crc 2023-09-23T23:45:04.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:04 smithi110 ceph-mon[212939]: failed to encode map e747 with expected crc 2023-09-23T23:45:04.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:04 smithi110 ceph-mon[212939]: failed to encode map e747 with expected crc 2023-09-23T23:45:04.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:04 smithi110 ceph-mon[212939]: osdmap e747: 8 total, 8 up, 8 in 2023-09-23T23:45:04.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:04 smithi110 ceph-mon[212939]: failed to encode map e747 with expected crc 2023-09-23T23:45:04.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:04 smithi110 ceph-mon[212939]: failed to encode map e747 with expected crc 2023-09-23T23:45:04.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:04 smithi110 ceph-mon[221102]: failed to encode map e747 with expected crc 2023-09-23T23:45:04.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:04 smithi110 ceph-mon[221102]: failed to encode map e747 with expected crc 2023-09-23T23:45:04.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:04 smithi110 ceph-mon[221102]: failed to encode map e747 with expected crc 2023-09-23T23:45:04.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:04 smithi110 ceph-mon[221102]: osdmap e747: 8 total, 8 up, 8 in 2023-09-23T23:45:04.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:04 smithi110 ceph-mon[221102]: failed to encode map e747 with expected crc 2023-09-23T23:45:04.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:04 smithi110 ceph-mon[221102]: failed to encode map e747 with expected crc 2023-09-23T23:45:04.501 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:45:05.365 INFO:teuthology.orchestra.run.smithi110.stdout:mon.a: {} 2023-09-23T23:45:05.377 INFO:teuthology.orchestra.run.smithi110.stdout:mon.b: {} 2023-09-23T23:45:05.388 INFO:teuthology.orchestra.run.smithi110.stdout:mon.c: {} 2023-09-23T23:45:05.389 INFO:teuthology.orchestra.run.smithi110.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T23:45:05.389 INFO:teuthology.orchestra.run.smithi110.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T23:45:05.389 INFO:teuthology.orchestra.run.smithi110.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T23:45:05.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:05 smithi110 ceph-mon[221102]: pgmap v2063: 103 pgs: 2 active+undersized+remapped, 4 remapped+peering, 10 peering, 1 active+remapped, 86 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 78 KiB/s, 5 objects/s recovering 2023-09-23T23:45:05.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:05 smithi110 ceph-mon[221102]: 1.1c scrub starts 2023-09-23T23:45:05.415 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:05 smithi110 ceph-mon[221102]: 1.1c scrub ok 2023-09-23T23:45:05.416 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:05 smithi110 ceph-mon[212939]: pgmap v2063: 103 pgs: 2 active+undersized+remapped, 4 remapped+peering, 10 peering, 1 active+remapped, 86 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 78 KiB/s, 5 objects/s recovering 2023-09-23T23:45:05.416 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:05 smithi110 ceph-mon[212939]: 1.1c scrub starts 2023-09-23T23:45:05.416 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:05 smithi110 ceph-mon[212939]: 1.1c scrub ok 2023-09-23T23:45:05.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:05 smithi138 ceph-mon[193606]: pgmap v2063: 103 pgs: 2 active+undersized+remapped, 4 remapped+peering, 10 peering, 1 active+remapped, 86 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 78 KiB/s, 5 objects/s recovering 2023-09-23T23:45:05.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:05 smithi138 ceph-mon[193606]: 1.1c scrub starts 2023-09-23T23:45:05.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:05 smithi138 ceph-mon[193606]: 1.1c scrub ok 2023-09-23T23:45:06.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:06 smithi110 ceph-mon[212939]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 6 pgs peering) 2023-09-23T23:45:06.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:06 smithi110 ceph-mon[212939]: Cluster is now healthy 2023-09-23T23:45:06.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:06 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1462693720' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T23:45:06.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:06 smithi110 ceph-mon[221102]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 6 pgs peering) 2023-09-23T23:45:06.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:06 smithi110 ceph-mon[221102]: Cluster is now healthy 2023-09-23T23:45:06.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:06 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1462693720' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T23:45:06.318 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T23:45:06.425 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:06 smithi138 ceph-mon[193606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 6 pgs peering) 2023-09-23T23:45:06.426 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:06 smithi138 ceph-mon[193606]: Cluster is now healthy 2023-09-23T23:45:06.426 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:06 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1462693720' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T23:45:06.655 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:45:06.656 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-09-23T23:45:06.656 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-09-23T23:45:06.656 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-09-23T23:45:06.656 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:45:06.845 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:07.062 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:45:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:45:06] "GET /metrics HTTP/1.1" 200 34194 "" "Prometheus/2.43.0" 2023-09-23T23:45:07.347 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:45:07.693 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:45:08.103 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:07 smithi138 ceph-mon[193606]: pgmap v2064: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 110 KiB/s rd, 27 KiB/s wr, 78 op/s; 54 KiB/s, 14 objects/s recovering 2023-09-23T23:45:08.103 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:07 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:08.103 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:07 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:08.103 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:07 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:08.103 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:07 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:08.103 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:07 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:45:08.103 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:07 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:45:08.103 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:07 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:08.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:07 smithi110 ceph-mon[212939]: pgmap v2064: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 110 KiB/s rd, 27 KiB/s wr, 78 op/s; 54 KiB/s, 14 objects/s recovering 2023-09-23T23:45:08.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:07 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:08.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:07 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:08.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:07 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:08.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:07 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:08.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:07 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:45:08.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:07 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:45:08.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:07 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:08.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:07 smithi110 ceph-mon[221102]: pgmap v2064: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 110 KiB/s rd, 27 KiB/s wr, 78 op/s; 54 KiB/s, 14 objects/s recovering 2023-09-23T23:45:08.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:07 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:08.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:07 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:08.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:07 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:08.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:07 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:08.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:07 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:45:08.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:07 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:45:08.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:07 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:08.764 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T23:45:08.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:08 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3774166592' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:45:09.161 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-09-23T23:45:09.162 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-09-23T23:45:09.162 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-09-23T23:45:09.162 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-09-23T23:45:09.162 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.162 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(3.b 740/739 [PushOp(3:d0630e4c:::mds0_sessionmap:head, version: 68'1, data_included: [], data_size: 0, omap_header_size: 14, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:d0630e4c:::mds0_sessionmap:head@68'1, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:45:09.162 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:55.999375+0000", 2023-09-23T23:45:09.162 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 13.158906889000001, 2023-09-23T23:45:09.162 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00031091400000000003, 2023-09-23T23:45:09.162 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.162 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:09.163 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.163 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.163 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.163 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:55.999375+0000", 2023-09-23T23:45:09.163 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.163 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.163 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.163 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.163 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:55.999375+0000", 2023-09-23T23:45:09.163 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.163 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.163 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.163 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.164 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:55.999375+0000", 2023-09-23T23:45:09.164 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.164 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.164 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.164 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.164 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:55.999392+0000", 2023-09-23T23:45:09.164 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7640000000000001e-05 2023-09-23T23:45:09.164 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.164 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.164 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.164 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:55.999405+0000", 2023-09-23T23:45:09.164 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2714000000000001e-05 2023-09-23T23:45:09.165 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.165 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.165 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.165 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:55.999417+0000", 2023-09-23T23:45:09.165 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2064e-05 2023-09-23T23:45:09.165 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.165 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.165 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.165 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:55.999491+0000", 2023-09-23T23:45:09.165 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3442000000000006e-05 2023-09-23T23:45:09.165 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.165 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.165 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:09.166 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:55.999506+0000", 2023-09-23T23:45:09.166 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5497000000000001e-05 2023-09-23T23:45:09.166 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.166 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.166 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.166 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:55.999686+0000", 2023-09-23T23:45:09.166 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000179858 2023-09-23T23:45:09.166 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.166 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.166 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.166 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.166 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.167 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(3.1d 740/739 [PushOp(3:b871830b:::605.00000000:head, version: 68'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:b871830b:::605.00000000:head@68'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:45:09.167 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:56.006048+0000", 2023-09-23T23:45:09.167 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 13.152233486, 2023-09-23T23:45:09.167 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000422922, 2023-09-23T23:45:09.167 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.167 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:09.167 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.167 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.167 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.167 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.006048+0000", 2023-09-23T23:45:09.167 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.167 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.167 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.168 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.168 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.006048+0000", 2023-09-23T23:45:09.168 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.168 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.168 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.168 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.168 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.006048+0000", 2023-09-23T23:45:09.168 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.168 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.168 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.168 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.168 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.006069+0000", 2023-09-23T23:45:09.168 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1418999999999999e-05 2023-09-23T23:45:09.169 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.169 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.169 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.169 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.006083+0000", 2023-09-23T23:45:09.169 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.362e-05 2023-09-23T23:45:09.169 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.169 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.169 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.169 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.006094+0000", 2023-09-23T23:45:09.169 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0812000000000001e-05 2023-09-23T23:45:09.169 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.169 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.169 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.170 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.006151+0000", 2023-09-23T23:45:09.170 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7056999999999999e-05 2023-09-23T23:45:09.170 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.170 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.170 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:09.170 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.006161+0000", 2023-09-23T23:45:09.170 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7270000000000002e-06 2023-09-23T23:45:09.170 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.170 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.170 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.170 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.006471+0000", 2023-09-23T23:45:09.170 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00031083199999999998 2023-09-23T23:45:09.171 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.171 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.171 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.171 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.171 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.171 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(3.1f 740/739 [PushOp(3:fe07bba1:::100.00000000.inode:head, version: 68'3, data_included: [0~554], data_size: 554, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:fe07bba1:::100.00000000.inode:head@68'3, size: 554, copy_subset: [0~554], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:554, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:45:09.171 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:56.008524+0000", 2023-09-23T23:45:09.171 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 13.149758302, 2023-09-23T23:45:09.171 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00029750600000000001, 2023-09-23T23:45:09.171 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.171 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:09.171 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.171 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.172 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.172 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.008524+0000", 2023-09-23T23:45:09.172 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.172 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.172 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.172 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.172 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.008524+0000", 2023-09-23T23:45:09.172 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.172 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.172 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.172 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.172 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.008523+0000", 2023-09-23T23:45:09.173 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.173 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.173 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.173 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.173 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.008531+0000", 2023-09-23T23:45:09.173 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2169999999999994e-06 2023-09-23T23:45:09.173 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.173 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.173 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.173 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.008545+0000", 2023-09-23T23:45:09.173 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3263e-05 2023-09-23T23:45:09.173 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.173 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.174 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.174 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.008551+0000", 2023-09-23T23:45:09.174 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.477e-06 2023-09-23T23:45:09.174 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.174 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.174 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.174 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.008608+0000", 2023-09-23T23:45:09.174 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6418000000000001e-05 2023-09-23T23:45:09.174 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.174 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.174 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:09.175 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.008615+0000", 2023-09-23T23:45:09.175 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1749999999999999e-06 2023-09-23T23:45:09.175 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.175 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.175 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.175 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.008821+0000", 2023-09-23T23:45:09.175 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020634200000000001 2023-09-23T23:45:09.175 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.175 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.175 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.175 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.175 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.175 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(3.1c 740/739 [PushOp(3:3ed09add:::607.00000000:head, version: 68'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:3ed09add:::607.00000000:head@68'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:45:09.176 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:56.010341+0000", 2023-09-23T23:45:09.176 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 13.147941439, 2023-09-23T23:45:09.176 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00046862800000000001, 2023-09-23T23:45:09.176 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.176 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:09.176 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.176 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.176 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.176 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.010341+0000", 2023-09-23T23:45:09.176 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.176 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.176 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.176 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.177 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.010341+0000", 2023-09-23T23:45:09.177 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.177 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.177 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.177 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.177 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.010340+0000", 2023-09-23T23:45:09.177 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.177 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.177 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.177 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.177 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.010355+0000", 2023-09-23T23:45:09.177 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5113000000000001e-05 2023-09-23T23:45:09.178 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.178 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.178 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.178 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.010372+0000", 2023-09-23T23:45:09.178 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6813999999999999e-05 2023-09-23T23:45:09.178 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.178 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.178 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.178 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.010391+0000", 2023-09-23T23:45:09.178 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9465000000000001e-05 2023-09-23T23:45:09.178 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.178 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.178 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.179 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.010461+0000", 2023-09-23T23:45:09.179 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9859999999999999e-05 2023-09-23T23:45:09.179 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.179 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.179 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:09.179 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.010473+0000", 2023-09-23T23:45:09.179 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2157000000000001e-05 2023-09-23T23:45:09.179 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.179 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.179 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.179 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.010809+0000", 2023-09-23T23:45:09.179 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00033588299999999998 2023-09-23T23:45:09.180 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.180 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.180 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.180 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.180 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.180 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(3.15 740/739 [PushOp(3:a93a17c2:::604.00000000:head, version: 68'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:a93a17c2:::604.00000000:head@68'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:45:09.180 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:56.010985+0000", 2023-09-23T23:45:09.180 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 13.147297092000001, 2023-09-23T23:45:09.180 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00024586299999999997, 2023-09-23T23:45:09.180 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.180 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:09.180 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.181 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.181 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.181 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.010985+0000", 2023-09-23T23:45:09.181 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.181 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.181 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.181 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.181 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.010985+0000", 2023-09-23T23:45:09.181 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.181 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.181 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.181 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.181 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.010984+0000", 2023-09-23T23:45:09.182 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.182 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.182 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.182 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.182 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.010995+0000", 2023-09-23T23:45:09.182 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1153e-05 2023-09-23T23:45:09.182 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.182 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.182 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.182 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.011014+0000", 2023-09-23T23:45:09.182 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9023999999999999e-05 2023-09-23T23:45:09.182 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.183 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.183 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.183 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.011024+0000", 2023-09-23T23:45:09.183 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4979999999999995e-06 2023-09-23T23:45:09.183 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.183 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.183 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.183 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.011070+0000", 2023-09-23T23:45:09.183 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5602000000000001e-05 2023-09-23T23:45:09.183 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.183 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.183 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:09.184 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.011079+0000", 2023-09-23T23:45:09.184 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1849999999999993e-06 2023-09-23T23:45:09.184 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.184 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.184 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.184 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.011231+0000", 2023-09-23T23:45:09.184 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015195900000000001 2023-09-23T23:45:09.184 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.184 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.184 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.184 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.184 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.184 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(3.15 740/739 [PushOp(3:aa448500:::500.00000000:head, version: 68'3, data_included: [0~90], data_size: 90, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:aa448500:::500.00000000:head@68'3, size: 90, copy_subset: [0~90], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:90, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:45:09.185 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:56.012205+0000", 2023-09-23T23:45:09.185 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 13.146077096999999, 2023-09-23T23:45:09.185 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000259564, 2023-09-23T23:45:09.185 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.185 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:09.185 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.185 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.185 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.185 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.012205+0000", 2023-09-23T23:45:09.185 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.185 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.185 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.186 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.186 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.012205+0000", 2023-09-23T23:45:09.186 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.186 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.186 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.186 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.186 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.012204+0000", 2023-09-23T23:45:09.186 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.186 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.186 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.186 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.186 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.012214+0000", 2023-09-23T23:45:09.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3700000000000001e-06 2023-09-23T23:45:09.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.187 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.187 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.012233+0000", 2023-09-23T23:45:09.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9718999999999999e-05 2023-09-23T23:45:09.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.187 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.187 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.012243+0000", 2023-09-23T23:45:09.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5510000000000005e-06 2023-09-23T23:45:09.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.188 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.012301+0000", 2023-09-23T23:45:09.188 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8065000000000001e-05 2023-09-23T23:45:09.188 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.188 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:09.188 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.012305+0000", 2023-09-23T23:45:09.188 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3839999999999999e-06 2023-09-23T23:45:09.188 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.188 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.188 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.012464+0000", 2023-09-23T23:45:09.189 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015894499999999999 2023-09-23T23:45:09.189 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.189 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.189 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.189 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.189 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.189 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(3.13 740/739 [PushOp(3:cd5a64a3:::100.00000000:head, version: 68'1, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 10, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:cd5a64a3:::100.00000000:head@68'1, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:45:09.189 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:56.014003+0000", 2023-09-23T23:45:09.189 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 13.144278751, 2023-09-23T23:45:09.189 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00045590099999999998, 2023-09-23T23:45:09.189 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.189 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:09.189 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.190 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.190 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.190 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.014003+0000", 2023-09-23T23:45:09.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.190 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.190 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.190 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.190 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.014003+0000", 2023-09-23T23:45:09.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.190 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.190 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.190 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.191 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.014003+0000", 2023-09-23T23:45:09.191 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.191 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.014016+0000", 2023-09-23T23:45:09.191 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3426e-05 2023-09-23T23:45:09.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.191 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.014033+0000", 2023-09-23T23:45:09.191 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6971999999999999e-05 2023-09-23T23:45:09.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.014044+0000", 2023-09-23T23:45:09.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0611000000000001e-05 2023-09-23T23:45:09.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.014108+0000", 2023-09-23T23:45:09.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4580999999999997e-05 2023-09-23T23:45:09.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:09.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.014117+0000", 2023-09-23T23:45:09.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8289999999999997e-06 2023-09-23T23:45:09.193 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.193 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.193 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.014459+0000", 2023-09-23T23:45:09.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00034189000000000002 2023-09-23T23:45:09.193 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.193 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.193 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.193 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.193 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(3.13 740/739 [PushOp(3:c91d4a1d:::mds0_inotable:head, version: 68'2, data_included: [0~34], data_size: 34, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:c91d4a1d:::mds0_inotable:head@68'2, size: 34, copy_subset: [0~34], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:34, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:45:09.194 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:56.015173+0000", 2023-09-23T23:45:09.194 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 13.143109375, 2023-09-23T23:45:09.194 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000407067, 2023-09-23T23:45:09.194 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.194 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:09.194 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.194 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.194 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.194 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.015173+0000", 2023-09-23T23:45:09.194 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.194 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.195 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.195 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.195 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.015173+0000", 2023-09-23T23:45:09.195 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.195 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.195 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.195 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.195 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.015172+0000", 2023-09-23T23:45:09.195 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.195 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.195 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.195 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.015181+0000", 2023-09-23T23:45:09.196 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4549999999999995e-06 2023-09-23T23:45:09.196 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.196 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.196 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.015193+0000", 2023-09-23T23:45:09.196 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2636000000000001e-05 2023-09-23T23:45:09.196 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.196 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.196 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.015203+0000", 2023-09-23T23:45:09.196 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2949999999999999e-06 2023-09-23T23:45:09.196 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.197 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.197 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.015259+0000", 2023-09-23T23:45:09.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6842999999999998e-05 2023-09-23T23:45:09.197 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.197 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:09.197 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.015266+0000", 2023-09-23T23:45:09.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2910000000000004e-06 2023-09-23T23:45:09.197 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.197 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.197 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.015580+0000", 2023-09-23T23:45:09.198 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00031398199999999998 2023-09-23T23:45:09.198 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.198 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.198 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.198 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.198 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.198 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(3.14 740/739 [PushOp(3:2f55791f:::606.00000000:head, version: 68'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:2f55791f:::606.00000000:head@68'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:45:09.198 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:56.017163+0000", 2023-09-23T23:45:09.198 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 13.141119206000001, 2023-09-23T23:45:09.198 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000381393, 2023-09-23T23:45:09.198 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.198 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:09.199 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.199 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.199 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.199 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.017163+0000", 2023-09-23T23:45:09.199 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.199 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.199 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.199 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.199 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.017163+0000", 2023-09-23T23:45:09.199 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.199 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.199 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.199 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.200 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.017162+0000", 2023-09-23T23:45:09.200 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.200 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.200 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.200 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.200 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.017173+0000", 2023-09-23T23:45:09.200 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0346999999999999e-05 2023-09-23T23:45:09.200 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.200 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.200 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.200 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.017187+0000", 2023-09-23T23:45:09.200 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4568e-05 2023-09-23T23:45:09.201 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.201 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.201 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.201 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.017194+0000", 2023-09-23T23:45:09.201 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2930000000000003e-06 2023-09-23T23:45:09.201 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.201 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.201 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.201 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.017238+0000", 2023-09-23T23:45:09.201 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3970000000000001e-05 2023-09-23T23:45:09.201 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.201 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.201 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:09.202 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.017244+0000", 2023-09-23T23:45:09.202 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0619999999999997e-06 2023-09-23T23:45:09.202 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.202 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.202 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.202 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.017544+0000", 2023-09-23T23:45:09.202 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00029972400000000002 2023-09-23T23:45:09.203 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.203 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.203 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.203 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.203 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.203 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(3.14 740/739 [PushOp(3:287034c4:::1.00000000.inode:head, version: 68'4, data_included: [0~554], data_size: 554, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:287034c4:::1.00000000.inode:head@68'4, size: 554, copy_subset: [0~554], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:554, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:45:09.203 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:56.018237+0000", 2023-09-23T23:45:09.203 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 13.140044518, 2023-09-23T23:45:09.203 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000498684, 2023-09-23T23:45:09.203 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.203 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:09.203 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.204 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.204 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.204 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.018237+0000", 2023-09-23T23:45:09.204 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.204 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.204 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.204 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.204 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.018237+0000", 2023-09-23T23:45:09.204 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.204 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.204 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.205 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.205 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.018237+0000", 2023-09-23T23:45:09.205 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.205 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.205 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.205 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.205 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.018256+0000", 2023-09-23T23:45:09.205 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9349000000000001e-05 2023-09-23T23:45:09.205 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.205 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.205 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.205 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.018270+0000", 2023-09-23T23:45:09.205 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3220999999999999e-05 2023-09-23T23:45:09.206 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.206 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.206 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.206 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.018277+0000", 2023-09-23T23:45:09.206 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2320000000000004e-06 2023-09-23T23:45:09.206 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.206 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.206 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.206 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.018343+0000", 2023-09-23T23:45:09.206 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5871999999999995e-05 2023-09-23T23:45:09.206 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.206 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.207 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:09.207 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.018352+0000", 2023-09-23T23:45:09.207 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7620000000000007e-06 2023-09-23T23:45:09.207 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.207 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.207 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.207 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.018736+0000", 2023-09-23T23:45:09.207 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000383707 2023-09-23T23:45:09.207 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.207 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.207 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.207 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.207 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.208 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(3.14 740/739 [PushOp(3:292cf221:::200.00000000:head, version: 82'21, data_included: [0~90], data_size: 90, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:292cf221:::200.00000000:head@82'21, size: 90, copy_subset: [0~90], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:90, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:45:09.208 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:56.019494+0000", 2023-09-23T23:45:09.208 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 13.138787729000001, 2023-09-23T23:45:09.208 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000481597, 2023-09-23T23:45:09.208 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.208 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:09.208 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.208 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.208 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.208 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.019494+0000", 2023-09-23T23:45:09.208 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.208 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.209 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.209 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.209 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.019494+0000", 2023-09-23T23:45:09.209 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.209 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.209 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.209 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.209 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.019494+0000", 2023-09-23T23:45:09.209 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.209 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.209 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.209 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.209 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.019501+0000", 2023-09-23T23:45:09.210 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7090000000000002e-06 2023-09-23T23:45:09.210 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.210 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.210 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.210 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.019514+0000", 2023-09-23T23:45:09.210 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2469999999999999e-05 2023-09-23T23:45:09.210 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.210 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.210 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.210 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.019522+0000", 2023-09-23T23:45:09.210 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0260000000000007e-06 2023-09-23T23:45:09.210 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.211 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.211 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.211 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.019581+0000", 2023-09-23T23:45:09.211 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8964e-05 2023-09-23T23:45:09.211 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.211 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.211 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:09.211 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.019590+0000", 2023-09-23T23:45:09.211 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8370000000000003e-06 2023-09-23T23:45:09.211 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.211 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.211 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.211 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.019976+0000", 2023-09-23T23:45:09.212 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00038605099999999999 2023-09-23T23:45:09.212 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.212 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.212 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.212 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.212 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.212 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(3.14 740/739 [PushOp(3:2e2fa760:::200.00000001:head, version: 82'22, data_included: [0~4090], data_size: 4090, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:2e2fa760:::200.00000001:head@82'22, size: 4090, copy_subset: [0~4090], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4090, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:45:09.212 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:56.020694+0000", 2023-09-23T23:45:09.212 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 13.137587830999999, 2023-09-23T23:45:09.212 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00049678600000000004, 2023-09-23T23:45:09.212 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.213 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:09.213 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.213 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.213 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.213 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.020694+0000", 2023-09-23T23:45:09.213 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.213 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.213 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.213 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.213 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.020694+0000", 2023-09-23T23:45:09.213 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.213 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.214 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.214 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.214 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.020694+0000", 2023-09-23T23:45:09.214 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.214 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.214 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.214 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.214 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.020706+0000", 2023-09-23T23:45:09.214 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2639999999999999e-05 2023-09-23T23:45:09.214 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.214 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.214 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.215 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.020719+0000", 2023-09-23T23:45:09.215 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2972e-05 2023-09-23T23:45:09.215 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.215 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.215 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.215 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.020732+0000", 2023-09-23T23:45:09.215 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2607e-05 2023-09-23T23:45:09.215 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.215 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.215 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.215 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.020797+0000", 2023-09-23T23:45:09.215 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5548000000000003e-05 2023-09-23T23:45:09.215 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.216 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.216 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:09.216 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.020807+0000", 2023-09-23T23:45:09.216 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3109999999999995e-06 2023-09-23T23:45:09.216 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.216 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.216 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.216 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.021191+0000", 2023-09-23T23:45:09.216 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00038422 2023-09-23T23:45:09.216 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.216 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.216 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.217 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.217 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.217 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(3.f 740/739 [PushOp(3:f1f69726:::400.00000000:head, version: 68'1, data_included: [0~22], data_size: 22, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:f1f69726:::400.00000000:head@68'1, size: 22, copy_subset: [0~22], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:22, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:45:09.217 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:56.023049+0000", 2023-09-23T23:45:09.217 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 13.135233283, 2023-09-23T23:45:09.217 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00030344200000000001, 2023-09-23T23:45:09.217 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.217 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:09.217 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.217 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.217 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.217 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.023049+0000", 2023-09-23T23:45:09.217 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.218 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.218 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.218 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.218 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.023049+0000", 2023-09-23T23:45:09.218 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.218 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.218 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.218 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.218 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.023048+0000", 2023-09-23T23:45:09.218 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.218 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.218 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.219 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.219 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.023064+0000", 2023-09-23T23:45:09.219 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.626e-05 2023-09-23T23:45:09.219 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.219 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.219 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.219 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.023080+0000", 2023-09-23T23:45:09.219 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5169e-05 2023-09-23T23:45:09.219 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.219 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.219 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.219 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.023086+0000", 2023-09-23T23:45:09.219 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5699999999999998e-06 2023-09-23T23:45:09.220 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.220 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.220 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.220 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.023140+0000", 2023-09-23T23:45:09.220 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4259999999999999e-05 2023-09-23T23:45:09.220 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.220 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.220 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:09.220 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.023145+0000", 2023-09-23T23:45:09.220 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8500000000000002e-06 2023-09-23T23:45:09.220 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.220 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.220 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.221 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.023352+0000", 2023-09-23T23:45:09.221 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020687599999999999 2023-09-23T23:45:09.221 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.221 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.221 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.221 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.221 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.221 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(3.6 740/739 [PushOp(3:654134d2:::mds0_openfiles.0:head, version: 68'1, data_included: [], data_size: 0, omap_header_size: 42, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:654134d2:::mds0_openfiles.0:head@68'1, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:45:09.221 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:56.025368+0000", 2023-09-23T23:45:09.221 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 13.132913801999999, 2023-09-23T23:45:09.221 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00031000299999999999, 2023-09-23T23:45:09.221 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.221 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:09.221 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.222 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.222 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.222 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.025368+0000", 2023-09-23T23:45:09.222 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.222 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.222 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.222 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.222 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.025368+0000", 2023-09-23T23:45:09.222 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.222 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.222 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.222 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.223 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.025368+0000", 2023-09-23T23:45:09.223 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.223 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.223 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.223 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.223 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.025379+0000", 2023-09-23T23:45:09.223 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1574e-05 2023-09-23T23:45:09.223 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.223 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.223 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.223 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.025396+0000", 2023-09-23T23:45:09.223 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6909e-05 2023-09-23T23:45:09.224 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.224 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.224 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.224 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.025405+0000", 2023-09-23T23:45:09.224 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6349999999999995e-06 2023-09-23T23:45:09.224 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.224 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.224 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.224 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.025459+0000", 2023-09-23T23:45:09.224 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4685999999999998e-05 2023-09-23T23:45:09.224 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.224 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.224 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:09.225 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.025466+0000", 2023-09-23T23:45:09.225 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2439999999999998e-06 2023-09-23T23:45:09.225 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.225 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.225 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.225 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.025678+0000", 2023-09-23T23:45:09.225 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021250899999999999 2023-09-23T23:45:09.225 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.225 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.225 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.225 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.225 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.225 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(3.e 740/739 [PushOp(3:717a0223:::608.00000000:head, version: 68'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:717a0223:::608.00000000:head@68'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:45:09.226 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:56.027232+0000", 2023-09-23T23:45:09.226 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 13.131050001, 2023-09-23T23:45:09.226 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00032332800000000001, 2023-09-23T23:45:09.226 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.226 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:09.226 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.226 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.226 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.226 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.027232+0000", 2023-09-23T23:45:09.226 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.226 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.226 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.226 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.227 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.027232+0000", 2023-09-23T23:45:09.227 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.227 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.227 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.227 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.227 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.027231+0000", 2023-09-23T23:45:09.227 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.227 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.227 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.227 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.227 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.027242+0000", 2023-09-23T23:45:09.227 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0073e-05 2023-09-23T23:45:09.228 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.228 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.228 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.228 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.027253+0000", 2023-09-23T23:45:09.228 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1576000000000001e-05 2023-09-23T23:45:09.228 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.228 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.228 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.228 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.027260+0000", 2023-09-23T23:45:09.228 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5740000000000001e-06 2023-09-23T23:45:09.228 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.228 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.229 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.229 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.027312+0000", 2023-09-23T23:45:09.229 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2045000000000001e-05 2023-09-23T23:45:09.229 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.229 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.229 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:09.229 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.027320+0000", 2023-09-23T23:45:09.229 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8080000000000005e-06 2023-09-23T23:45:09.229 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.229 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.229 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.229 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.027555+0000", 2023-09-23T23:45:09.229 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000235725 2023-09-23T23:45:09.230 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.230 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.230 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.230 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.230 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.230 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(1.c 741/740 [PushOp(1:30306672:devicehealth::main.db.0000000000000000:head, version: 77'161, data_included: [0~327680], data_size: 327680, omap_header_size: 0, omap_entries_size: 0, attrset_size: 10, recovery_info: ObjectRecoveryInfo(1:30306672:devicehealth::main.db.0000000000000000:head@77'161, size: 327680, copy_subset: [0~327680], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:327680, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:45:09.230 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:44:56.997660+0000", 2023-09-23T23:45:09.230 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 12.160621710999999, 2023-09-23T23:45:09.230 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001072321, 2023-09-23T23:45:09.230 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.230 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-09-23T23:45:09.230 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.231 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.231 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.231 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.997660+0000", 2023-09-23T23:45:09.231 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.231 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.231 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.231 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.231 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.997660+0000", 2023-09-23T23:45:09.231 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.231 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.231 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.231 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.232 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.997660+0000", 2023-09-23T23:45:09.232 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.232 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.232 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.232 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.232 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.998392+0000", 2023-09-23T23:45:09.232 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00073201600000000005 2023-09-23T23:45:09.232 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.232 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.232 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.232 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.998407+0000", 2023-09-23T23:45:09.232 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5036e-05 2023-09-23T23:45:09.233 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.233 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.233 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.233 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.998414+0000", 2023-09-23T23:45:09.233 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.294e-06 2023-09-23T23:45:09.233 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.233 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.233 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.233 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.998462+0000", 2023-09-23T23:45:09.233 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7494999999999998e-05 2023-09-23T23:45:09.233 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.233 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.233 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-09-23T23:45:09.233 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.998468+0000", 2023-09-23T23:45:09.234 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7190000000000004e-06 2023-09-23T23:45:09.234 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.234 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.234 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.234 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:44:56.998733+0000", 2023-09-23T23:45:09.234 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00026421099999999998 2023-09-23T23:45:09.234 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.234 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.234 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.234 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.234 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.234 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPushReply(3.1d 744/743 [PushReplyOp(3:b871830b:::605.00000000:head)])", 2023-09-23T23:45:09.235 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:45:00.000359+0000", 2023-09-23T23:45:09.235 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 9.1579232289999997, 2023-09-23T23:45:09.235 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00024111400000000001, 2023-09-23T23:45:09.235 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.235 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:45:09.235 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.235 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.235 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.235 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.000359+0000", 2023-09-23T23:45:09.235 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.235 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.235 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.236 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.236 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.000359+0000", 2023-09-23T23:45:09.236 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.236 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.236 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.236 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.236 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.000358+0000", 2023-09-23T23:45:09.236 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.236 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.236 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.236 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.236 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.000362+0000", 2023-09-23T23:45:09.237 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7570000000000002e-06 2023-09-23T23:45:09.237 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.237 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.237 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.237 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.000369+0000", 2023-09-23T23:45:09.237 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.255e-06 2023-09-23T23:45:09.237 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.237 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.237 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.237 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.000375+0000", 2023-09-23T23:45:09.237 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9989999999999997e-06 2023-09-23T23:45:09.237 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.237 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.238 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.238 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.000412+0000", 2023-09-23T23:45:09.238 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6667e-05 2023-09-23T23:45:09.238 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.238 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.238 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.238 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.000600+0000", 2023-09-23T23:45:09.238 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018777600000000001 2023-09-23T23:45:09.238 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.238 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.238 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.238 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.238 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.239 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPushReply(3.14 744/743 [PushReplyOp(3:2f55791f:::606.00000000:head)])", 2023-09-23T23:45:09.239 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:45:00.002122+0000", 2023-09-23T23:45:09.239 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 9.1561594639999999, 2023-09-23T23:45:09.239 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000241225, 2023-09-23T23:45:09.239 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.239 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:45:09.239 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.239 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.239 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.239 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.002122+0000", 2023-09-23T23:45:09.239 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.239 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.240 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.240 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.240 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.002122+0000", 2023-09-23T23:45:09.240 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.240 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.240 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.240 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.240 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.002122+0000", 2023-09-23T23:45:09.240 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.240 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.240 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.240 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.240 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.002125+0000", 2023-09-23T23:45:09.241 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0709999999999999e-06 2023-09-23T23:45:09.241 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.241 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.241 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.241 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.002138+0000", 2023-09-23T23:45:09.241 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2799000000000001e-05 2023-09-23T23:45:09.241 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.241 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.241 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.241 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.002143+0000", 2023-09-23T23:45:09.241 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5260000000000004e-06 2023-09-23T23:45:09.241 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.241 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.242 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.242 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.002175+0000", 2023-09-23T23:45:09.242 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2341000000000001e-05 2023-09-23T23:45:09.242 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.242 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.242 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.242 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.002364+0000", 2023-09-23T23:45:09.242 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000188834 2023-09-23T23:45:09.242 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.242 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.242 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.242 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.243 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.243 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPushReply(3.14 744/743 [PushReplyOp(3:287034c4:::1.00000000.inode:head)])", 2023-09-23T23:45:09.243 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:45:00.003109+0000", 2023-09-23T23:45:09.243 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 9.1551731210000007, 2023-09-23T23:45:09.243 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00024123499999999999, 2023-09-23T23:45:09.243 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.243 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:45:09.243 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.243 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.243 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.243 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.003109+0000", 2023-09-23T23:45:09.243 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.243 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.244 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.244 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.003109+0000", 2023-09-23T23:45:09.244 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.244 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.244 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.244 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.003108+0000", 2023-09-23T23:45:09.244 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.244 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.244 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.245 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.003114+0000", 2023-09-23T23:45:09.245 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6200000000000004e-06 2023-09-23T23:45:09.245 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.245 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.245 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.245 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.003124+0000", 2023-09-23T23:45:09.245 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8900000000000002e-06 2023-09-23T23:45:09.245 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.245 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.245 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.245 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.003133+0000", 2023-09-23T23:45:09.245 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2189999999999993e-06 2023-09-23T23:45:09.245 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.246 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.246 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.246 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.003166+0000", 2023-09-23T23:45:09.246 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2716999999999999e-05 2023-09-23T23:45:09.246 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.246 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.246 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.246 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.003350+0000", 2023-09-23T23:45:09.246 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018433600000000001 2023-09-23T23:45:09.246 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.246 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.246 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.247 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.247 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.247 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPushReply(3.14 744/743 [PushReplyOp(3:2e2fa760:::200.00000001:head)])", 2023-09-23T23:45:09.247 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-09-23T23:45:00.005097+0000", 2023-09-23T23:45:09.247 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 9.1531845359999995, 2023-09-23T23:45:09.247 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00023307199999999999, 2023-09-23T23:45:09.247 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-09-23T23:45:09.247 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-09-23T23:45:09.247 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-09-23T23:45:09.247 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.247 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-09-23T23:45:09.247 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.005097+0000", 2023-09-23T23:45:09.248 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.248 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.248 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.248 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-09-23T23:45:09.248 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.005097+0000", 2023-09-23T23:45:09.248 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-09-23T23:45:09.248 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.248 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.248 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-09-23T23:45:09.248 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.005097+0000", 2023-09-23T23:45:09.248 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-09-23T23:45:09.248 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.249 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.249 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-09-23T23:45:09.249 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.005100+0000", 2023-09-23T23:45:09.249 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3270000000000001e-06 2023-09-23T23:45:09.249 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.249 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.249 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-09-23T23:45:09.249 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.005108+0000", 2023-09-23T23:45:09.249 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2470000000000002e-06 2023-09-23T23:45:09.249 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.249 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.249 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-09-23T23:45:09.249 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.005114+0000", 2023-09-23T23:45:09.250 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0109999999999997e-06 2023-09-23T23:45:09.250 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.250 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.250 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-09-23T23:45:09.250 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.005150+0000", 2023-09-23T23:45:09.250 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6189999999999997e-05 2023-09-23T23:45:09.250 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-09-23T23:45:09.250 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-09-23T23:45:09.250 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-09-23T23:45:09.250 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-09-23T23:45:00.005330+0000", 2023-09-23T23:45:09.250 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018062500000000001 2023-09-23T23:45:09.250 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.251 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.251 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.251 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-09-23T23:45:09.251 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-09-23T23:45:09.251 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-09-23T23:45:09.251 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:08 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3774166592' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:45:09.252 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:08 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3774166592' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:45:09.535 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-09-23T23:45:09.885 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:09.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[212939]: pgmap v2065: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 92 KiB/s rd, 23 KiB/s wr, 65 op/s; 45 KiB/s, 11 objects/s recovering 2023-09-23T23:45:09.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3774166592' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:45:09.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[212939]: osdmap e748: 8 total, 8 up, 8 in 2023-09-23T23:45:09.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[212939]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[212939]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[212939]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[212939]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[212939]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[212939]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[212939]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[212939]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[212939]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[212939]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[212939]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[212939]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[212939]: failed to encode map e749 with expected crc 2023-09-23T23:45:09.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[212939]: osdmap e749: 8 total, 8 up, 8 in 2023-09-23T23:45:09.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[221102]: pgmap v2065: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 92 KiB/s rd, 23 KiB/s wr, 65 op/s; 45 KiB/s, 11 objects/s recovering 2023-09-23T23:45:09.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3774166592' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:45:09.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[221102]: osdmap e748: 8 total, 8 up, 8 in 2023-09-23T23:45:09.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[221102]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[221102]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[221102]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[221102]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[221102]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[221102]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[221102]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[221102]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[221102]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[221102]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.955 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[221102]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[221102]: failed to encode map e748 with expected crc 2023-09-23T23:45:09.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[221102]: failed to encode map e749 with expected crc 2023-09-23T23:45:09.956 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:09 smithi110 ceph-mon[221102]: osdmap e749: 8 total, 8 up, 8 in 2023-09-23T23:45:10.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:09 smithi138 ceph-mon[193606]: pgmap v2065: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 92 KiB/s rd, 23 KiB/s wr, 65 op/s; 45 KiB/s, 11 objects/s recovering 2023-09-23T23:45:10.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:09 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3774166592' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:45:10.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:09 smithi138 ceph-mon[193606]: osdmap e748: 8 total, 8 up, 8 in 2023-09-23T23:45:10.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:09 smithi138 ceph-mon[193606]: failed to encode map e748 with expected crc 2023-09-23T23:45:10.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:09 smithi138 ceph-mon[193606]: failed to encode map e748 with expected crc 2023-09-23T23:45:10.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:09 smithi138 ceph-mon[193606]: failed to encode map e748 with expected crc 2023-09-23T23:45:10.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:09 smithi138 ceph-mon[193606]: failed to encode map e748 with expected crc 2023-09-23T23:45:10.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:09 smithi138 ceph-mon[193606]: failed to encode map e748 with expected crc 2023-09-23T23:45:10.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:09 smithi138 ceph-mon[193606]: failed to encode map e748 with expected crc 2023-09-23T23:45:10.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:09 smithi138 ceph-mon[193606]: failed to encode map e748 with expected crc 2023-09-23T23:45:10.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:09 smithi138 ceph-mon[193606]: failed to encode map e748 with expected crc 2023-09-23T23:45:10.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:09 smithi138 ceph-mon[193606]: failed to encode map e748 with expected crc 2023-09-23T23:45:10.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:09 smithi138 ceph-mon[193606]: failed to encode map e748 with expected crc 2023-09-23T23:45:10.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:09 smithi138 ceph-mon[193606]: failed to encode map e748 with expected crc 2023-09-23T23:45:10.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:09 smithi138 ceph-mon[193606]: failed to encode map e748 with expected crc 2023-09-23T23:45:10.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:09 smithi138 ceph-mon[193606]: failed to encode map e749 with expected crc 2023-09-23T23:45:10.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:09 smithi138 ceph-mon[193606]: osdmap e749: 8 total, 8 up, 8 in 2023-09-23T23:45:11.042 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[221102]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.043 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[221102]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.043 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[221102]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.043 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[221102]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.043 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[221102]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.043 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[221102]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.043 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[221102]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.043 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[221102]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.043 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[221102]: osdmap e750: 8 total, 8 up, 8 in 2023-09-23T23:45:11.043 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[221102]: failed to encode map e750 with expected crc 2023-09-23T23:45:11.043 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[212939]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.044 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[212939]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.044 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[212939]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.044 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[212939]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.044 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[212939]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.044 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[212939]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.044 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[212939]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.044 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[212939]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.044 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[212939]: osdmap e750: 8 total, 8 up, 8 in 2023-09-23T23:45:11.044 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:10 smithi110 ceph-mon[212939]: failed to encode map e750 with expected crc 2023-09-23T23:45:11.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:10 smithi138 ceph-mon[193606]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:10 smithi138 ceph-mon[193606]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:10 smithi138 ceph-mon[193606]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:10 smithi138 ceph-mon[193606]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:10 smithi138 ceph-mon[193606]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:10 smithi138 ceph-mon[193606]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:10 smithi138 ceph-mon[193606]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:10 smithi138 ceph-mon[193606]: failed to encode map e749 with expected crc 2023-09-23T23:45:11.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:10 smithi138 ceph-mon[193606]: osdmap e750: 8 total, 8 up, 8 in 2023-09-23T23:45:11.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:10 smithi138 ceph-mon[193606]: failed to encode map e750 with expected crc 2023-09-23T23:45:11.995 INFO:teuthology.orchestra.run.smithi110.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-09-23T23:45:12.014 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[212939]: failed to encode map e750 with expected crc 2023-09-23T23:45:12.014 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[212939]: failed to encode map e750 with expected crc 2023-09-23T23:45:12.014 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[212939]: failed to encode map e750 with expected crc 2023-09-23T23:45:12.015 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[212939]: failed to encode map e750 with expected crc 2023-09-23T23:45:12.015 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[212939]: failed to encode map e750 with expected crc 2023-09-23T23:45:12.015 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[212939]: pgmap v2069: 103 pgs: 1 remapped+peering, 1 peering, 101 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 110 KiB/s rd, 27 KiB/s wr, 78 op/s; 3 B/s, 11 objects/s recovering 2023-09-23T23:45:12.015 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[212939]: failed to encode map e751 with expected crc 2023-09-23T23:45:12.015 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[212939]: osdmap e751: 8 total, 8 up, 8 in 2023-09-23T23:45:12.015 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3987525225' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:45:12.015 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:45:12.015 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[221102]: failed to encode map e750 with expected crc 2023-09-23T23:45:12.015 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[221102]: failed to encode map e750 with expected crc 2023-09-23T23:45:12.015 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[221102]: failed to encode map e750 with expected crc 2023-09-23T23:45:12.016 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[221102]: failed to encode map e750 with expected crc 2023-09-23T23:45:12.016 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[221102]: failed to encode map e750 with expected crc 2023-09-23T23:45:12.016 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[221102]: pgmap v2069: 103 pgs: 1 remapped+peering, 1 peering, 101 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 110 KiB/s rd, 27 KiB/s wr, 78 op/s; 3 B/s, 11 objects/s recovering 2023-09-23T23:45:12.016 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[221102]: failed to encode map e751 with expected crc 2023-09-23T23:45:12.016 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[221102]: osdmap e751: 8 total, 8 up, 8 in 2023-09-23T23:45:12.016 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3987525225' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:45:12.016 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:11 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:45:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:11 smithi138 ceph-mon[193606]: failed to encode map e750 with expected crc 2023-09-23T23:45:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:11 smithi138 ceph-mon[193606]: failed to encode map e750 with expected crc 2023-09-23T23:45:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:11 smithi138 ceph-mon[193606]: failed to encode map e750 with expected crc 2023-09-23T23:45:12.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:11 smithi138 ceph-mon[193606]: failed to encode map e750 with expected crc 2023-09-23T23:45:12.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:11 smithi138 ceph-mon[193606]: failed to encode map e750 with expected crc 2023-09-23T23:45:12.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:11 smithi138 ceph-mon[193606]: pgmap v2069: 103 pgs: 1 remapped+peering, 1 peering, 101 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 110 KiB/s rd, 27 KiB/s wr, 78 op/s; 3 B/s, 11 objects/s recovering 2023-09-23T23:45:12.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:11 smithi138 ceph-mon[193606]: failed to encode map e751 with expected crc 2023-09-23T23:45:12.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:11 smithi138 ceph-mon[193606]: osdmap e751: 8 total, 8 up, 8 in 2023-09-23T23:45:12.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:11 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3987525225' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:45:12.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:11 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:45:12.750 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T23:45:13.108 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:13.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:12 smithi138 ceph-mon[193606]: failed to encode map e751 with expected crc 2023-09-23T23:45:13.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:12 smithi138 ceph-mon[193606]: failed to encode map e751 with expected crc 2023-09-23T23:45:13.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:12 smithi138 ceph-mon[193606]: failed to encode map e751 with expected crc 2023-09-23T23:45:13.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:12 smithi138 ceph-mon[193606]: failed to encode map e751 with expected crc 2023-09-23T23:45:13.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:12 smithi138 ceph-mon[193606]: failed to encode map e752 with expected crc 2023-09-23T23:45:13.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:12 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:45:13.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:12 smithi138 ceph-mon[193606]: osdmap e752: 8 total, 8 up, 8 in 2023-09-23T23:45:13.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:12 smithi110 ceph-mon[221102]: failed to encode map e751 with expected crc 2023-09-23T23:45:13.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:12 smithi110 ceph-mon[221102]: failed to encode map e751 with expected crc 2023-09-23T23:45:13.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:12 smithi110 ceph-mon[221102]: failed to encode map e751 with expected crc 2023-09-23T23:45:13.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:12 smithi110 ceph-mon[221102]: failed to encode map e751 with expected crc 2023-09-23T23:45:13.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:12 smithi110 ceph-mon[221102]: failed to encode map e752 with expected crc 2023-09-23T23:45:13.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:12 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:45:13.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:12 smithi110 ceph-mon[221102]: osdmap e752: 8 total, 8 up, 8 in 2023-09-23T23:45:13.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:12 smithi110 ceph-mon[212939]: failed to encode map e751 with expected crc 2023-09-23T23:45:13.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:12 smithi110 ceph-mon[212939]: failed to encode map e751 with expected crc 2023-09-23T23:45:13.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:12 smithi110 ceph-mon[212939]: failed to encode map e751 with expected crc 2023-09-23T23:45:13.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:12 smithi110 ceph-mon[212939]: failed to encode map e751 with expected crc 2023-09-23T23:45:13.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:12 smithi110 ceph-mon[212939]: failed to encode map e752 with expected crc 2023-09-23T23:45:13.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:12 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:45:13.204 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:12 smithi110 ceph-mon[212939]: osdmap e752: 8 total, 8 up, 8 in 2023-09-23T23:45:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[221102]: failed to encode map e752 with expected crc 2023-09-23T23:45:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[221102]: failed to encode map e752 with expected crc 2023-09-23T23:45:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[221102]: failed to encode map e752 with expected crc 2023-09-23T23:45:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[221102]: failed to encode map e752 with expected crc 2023-09-23T23:45:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[221102]: failed to encode map e752 with expected crc 2023-09-23T23:45:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[221102]: pgmap v2072: 103 pgs: 1 remapped+peering, 2 peering, 100 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 162 B/s, 4 objects/s recovering 2023-09-23T23:45:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[221102]: failed to encode map e753 with expected crc 2023-09-23T23:45:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[221102]: osdmap e753: 8 total, 8 up, 8 in 2023-09-23T23:45:13.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:45:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[212939]: failed to encode map e752 with expected crc 2023-09-23T23:45:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[212939]: failed to encode map e752 with expected crc 2023-09-23T23:45:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[212939]: failed to encode map e752 with expected crc 2023-09-23T23:45:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[212939]: failed to encode map e752 with expected crc 2023-09-23T23:45:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[212939]: failed to encode map e752 with expected crc 2023-09-23T23:45:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[212939]: pgmap v2072: 103 pgs: 1 remapped+peering, 2 peering, 100 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 162 B/s, 4 objects/s recovering 2023-09-23T23:45:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[212939]: failed to encode map e753 with expected crc 2023-09-23T23:45:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[212939]: osdmap e753: 8 total, 8 up, 8 in 2023-09-23T23:45:13.954 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:45:14.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:13 smithi138 ceph-mon[193606]: failed to encode map e752 with expected crc 2023-09-23T23:45:14.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:13 smithi138 ceph-mon[193606]: failed to encode map e752 with expected crc 2023-09-23T23:45:14.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:13 smithi138 ceph-mon[193606]: failed to encode map e752 with expected crc 2023-09-23T23:45:14.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:13 smithi138 ceph-mon[193606]: failed to encode map e752 with expected crc 2023-09-23T23:45:14.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:13 smithi138 ceph-mon[193606]: failed to encode map e752 with expected crc 2023-09-23T23:45:14.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:13 smithi138 ceph-mon[193606]: pgmap v2072: 103 pgs: 1 remapped+peering, 2 peering, 100 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 162 B/s, 4 objects/s recovering 2023-09-23T23:45:14.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:13 smithi138 ceph-mon[193606]: failed to encode map e753 with expected crc 2023-09-23T23:45:14.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:13 smithi138 ceph-mon[193606]: osdmap e753: 8 total, 8 up, 8 in 2023-09-23T23:45:14.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:45:15.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:14 smithi138 ceph-mon[193606]: failed to encode map e753 with expected crc 2023-09-23T23:45:15.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:14 smithi138 ceph-mon[193606]: failed to encode map e753 with expected crc 2023-09-23T23:45:15.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:14 smithi138 ceph-mon[193606]: failed to encode map e753 with expected crc 2023-09-23T23:45:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:14 smithi110 ceph-mon[212939]: failed to encode map e753 with expected crc 2023-09-23T23:45:15.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:14 smithi110 ceph-mon[212939]: failed to encode map e753 with expected crc 2023-09-23T23:45:15.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:14 smithi110 ceph-mon[212939]: failed to encode map e753 with expected crc 2023-09-23T23:45:15.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:14 smithi110 ceph-mon[221102]: failed to encode map e753 with expected crc 2023-09-23T23:45:15.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:14 smithi110 ceph-mon[221102]: failed to encode map e753 with expected crc 2023-09-23T23:45:15.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:14 smithi110 ceph-mon[221102]: failed to encode map e753 with expected crc 2023-09-23T23:45:15.769 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T23:45:16.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:15 smithi138 ceph-mon[193606]: pgmap v2074: 103 pgs: 1 remapped+peering, 2 peering, 100 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 137 B/s, 3 objects/s recovering 2023-09-23T23:45:16.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:15 smithi138 ceph-mon[193606]: 1.16 scrub starts 2023-09-23T23:45:16.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:15 smithi138 ceph-mon[193606]: 1.16 scrub ok 2023-09-23T23:45:16.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:15 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1739372683' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:45:16.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:15 smithi110 ceph-mon[221102]: pgmap v2074: 103 pgs: 1 remapped+peering, 2 peering, 100 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 137 B/s, 3 objects/s recovering 2023-09-23T23:45:16.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:15 smithi110 ceph-mon[221102]: 1.16 scrub starts 2023-09-23T23:45:16.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:15 smithi110 ceph-mon[221102]: 1.16 scrub ok 2023-09-23T23:45:16.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:15 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1739372683' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:45:16.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:15 smithi110 ceph-mon[212939]: pgmap v2074: 103 pgs: 1 remapped+peering, 2 peering, 100 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 137 B/s, 3 objects/s recovering 2023-09-23T23:45:16.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:15 smithi110 ceph-mon[212939]: 1.16 scrub starts 2023-09-23T23:45:16.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:15 smithi110 ceph-mon[212939]: 1.16 scrub ok 2023-09-23T23:45:16.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:15 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1739372683' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:45:16.569 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-09-23T23:45:16.802 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:16 smithi110 ceph-mon[212939]: failed to encode map e754 with expected crc 2023-09-23T23:45:16.802 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:16 smithi110 ceph-mon[212939]: failed to encode map e754 with expected crc 2023-09-23T23:45:16.802 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:16 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1739372683' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:45:16.802 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:16 smithi110 ceph-mon[212939]: osdmap e754: 8 total, 8 up, 8 in 2023-09-23T23:45:16.802 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:16 smithi110 ceph-mon[212939]: failed to encode map e754 with expected crc 2023-09-23T23:45:16.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:16 smithi110 ceph-mon[221102]: failed to encode map e754 with expected crc 2023-09-23T23:45:16.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:16 smithi110 ceph-mon[221102]: failed to encode map e754 with expected crc 2023-09-23T23:45:16.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:16 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1739372683' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:45:16.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:16 smithi110 ceph-mon[221102]: osdmap e754: 8 total, 8 up, 8 in 2023-09-23T23:45:16.803 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:16 smithi110 ceph-mon[221102]: failed to encode map e754 with expected crc 2023-09-23T23:45:16.923 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:17.125 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:45:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:45:16] "GET /metrics HTTP/1.1" 200 34185 "" "Prometheus/2.43.0" 2023-09-23T23:45:17.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:16 smithi138 ceph-mon[193606]: failed to encode map e754 with expected crc 2023-09-23T23:45:17.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:16 smithi138 ceph-mon[193606]: failed to encode map e754 with expected crc 2023-09-23T23:45:17.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:16 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1739372683' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:45:17.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:16 smithi138 ceph-mon[193606]: osdmap e754: 8 total, 8 up, 8 in 2023-09-23T23:45:17.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:16 smithi138 ceph-mon[193606]: failed to encode map e754 with expected crc 2023-09-23T23:45:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:17 smithi138 ceph-mon[193606]: pgmap v2076: 103 pgs: 1 remapped+peering, 2 peering, 100 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 4 B/s, 2 objects/s recovering 2023-09-23T23:45:18.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:17 smithi138 ceph-mon[193606]: failed to encode map e754 with expected crc 2023-09-23T23:45:18.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:17 smithi110 ceph-mon[212939]: pgmap v2076: 103 pgs: 1 remapped+peering, 2 peering, 100 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 4 B/s, 2 objects/s recovering 2023-09-23T23:45:18.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:17 smithi110 ceph-mon[212939]: failed to encode map e754 with expected crc 2023-09-23T23:45:18.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:17 smithi110 ceph-mon[221102]: pgmap v2076: 103 pgs: 1 remapped+peering, 2 peering, 100 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 4 B/s, 2 objects/s recovering 2023-09-23T23:45:18.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:17 smithi110 ceph-mon[221102]: failed to encode map e754 with expected crc 2023-09-23T23:45:18.786 INFO:teuthology.orchestra.run.smithi110.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-09-23T23:45:19.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:18 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/636585426' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:45:19.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:18 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/636585426' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:45:19.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:18 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/636585426' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:45:19.482 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T23:45:19.839 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:20.041 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:19 smithi110 ceph-mon[212939]: pgmap v2077: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:20.041 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:19 smithi110 ceph-mon[212939]: failed to encode map e755 with expected crc 2023-09-23T23:45:20.042 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:19 smithi110 ceph-mon[212939]: failed to encode map e755 with expected crc 2023-09-23T23:45:20.042 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:19 smithi110 ceph-mon[212939]: failed to encode map e755 with expected crc 2023-09-23T23:45:20.042 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:19 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/636585426' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:45:20.042 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:19 smithi110 ceph-mon[212939]: osdmap e755: 8 total, 8 up, 8 in 2023-09-23T23:45:20.042 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:19 smithi110 ceph-mon[221102]: pgmap v2077: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:20.042 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:19 smithi110 ceph-mon[221102]: failed to encode map e755 with expected crc 2023-09-23T23:45:20.042 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:19 smithi110 ceph-mon[221102]: failed to encode map e755 with expected crc 2023-09-23T23:45:20.042 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:19 smithi110 ceph-mon[221102]: failed to encode map e755 with expected crc 2023-09-23T23:45:20.042 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:19 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/636585426' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:45:20.042 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:19 smithi110 ceph-mon[221102]: osdmap e755: 8 total, 8 up, 8 in 2023-09-23T23:45:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:19 smithi138 ceph-mon[193606]: pgmap v2077: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:19 smithi138 ceph-mon[193606]: failed to encode map e755 with expected crc 2023-09-23T23:45:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:19 smithi138 ceph-mon[193606]: failed to encode map e755 with expected crc 2023-09-23T23:45:20.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:19 smithi138 ceph-mon[193606]: failed to encode map e755 with expected crc 2023-09-23T23:45:20.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:19 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/636585426' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:45:20.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:19 smithi138 ceph-mon[193606]: osdmap e755: 8 total, 8 up, 8 in 2023-09-23T23:45:21.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:20 smithi138 ceph-mon[193606]: failed to encode map e756 with expected crc 2023-09-23T23:45:21.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:20 smithi138 ceph-mon[193606]: failed to encode map e756 with expected crc 2023-09-23T23:45:21.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:20 smithi138 ceph-mon[193606]: failed to encode map e756 with expected crc 2023-09-23T23:45:21.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:20 smithi138 ceph-mon[193606]: osdmap e756: 8 total, 8 up, 8 in 2023-09-23T23:45:21.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:20 smithi138 ceph-mon[193606]: failed to encode map e756 with expected crc 2023-09-23T23:45:21.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:20 smithi110 ceph-mon[212939]: failed to encode map e756 with expected crc 2023-09-23T23:45:21.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:20 smithi110 ceph-mon[212939]: failed to encode map e756 with expected crc 2023-09-23T23:45:21.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:20 smithi110 ceph-mon[212939]: failed to encode map e756 with expected crc 2023-09-23T23:45:21.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:20 smithi110 ceph-mon[212939]: osdmap e756: 8 total, 8 up, 8 in 2023-09-23T23:45:21.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:20 smithi110 ceph-mon[212939]: failed to encode map e756 with expected crc 2023-09-23T23:45:21.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:20 smithi110 ceph-mon[221102]: failed to encode map e756 with expected crc 2023-09-23T23:45:21.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:20 smithi110 ceph-mon[221102]: failed to encode map e756 with expected crc 2023-09-23T23:45:21.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:20 smithi110 ceph-mon[221102]: failed to encode map e756 with expected crc 2023-09-23T23:45:21.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:20 smithi110 ceph-mon[221102]: osdmap e756: 8 total, 8 up, 8 in 2023-09-23T23:45:21.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:20 smithi110 ceph-mon[221102]: failed to encode map e756 with expected crc 2023-09-23T23:45:21.808 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T23:45:22.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:21 smithi138 ceph-mon[193606]: pgmap v2080: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:22.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:21 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3774364106' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:45:22.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:21 smithi110 ceph-mon[212939]: pgmap v2080: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:22.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:21 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3774364106' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:45:22.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:21 smithi110 ceph-mon[221102]: pgmap v2080: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:22.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:21 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3774364106' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:45:22.524 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-09-23T23:45:22.883 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:23.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:22 smithi110 ceph-mon[212939]: failed to encode map e757 with expected crc 2023-09-23T23:45:23.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:22 smithi110 ceph-mon[212939]: failed to encode map e757 with expected crc 2023-09-23T23:45:23.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:22 smithi110 ceph-mon[212939]: failed to encode map e757 with expected crc 2023-09-23T23:45:23.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:22 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3774364106' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:45:23.092 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:22 smithi110 ceph-mon[212939]: osdmap e757: 8 total, 8 up, 8 in 2023-09-23T23:45:23.092 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:22 smithi110 ceph-mon[221102]: failed to encode map e757 with expected crc 2023-09-23T23:45:23.092 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:22 smithi110 ceph-mon[221102]: failed to encode map e757 with expected crc 2023-09-23T23:45:23.092 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:22 smithi110 ceph-mon[221102]: failed to encode map e757 with expected crc 2023-09-23T23:45:23.092 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:22 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3774364106' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:45:23.093 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:22 smithi110 ceph-mon[221102]: osdmap e757: 8 total, 8 up, 8 in 2023-09-23T23:45:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:22 smithi138 ceph-mon[193606]: failed to encode map e757 with expected crc 2023-09-23T23:45:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:22 smithi138 ceph-mon[193606]: failed to encode map e757 with expected crc 2023-09-23T23:45:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:22 smithi138 ceph-mon[193606]: failed to encode map e757 with expected crc 2023-09-23T23:45:23.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:22 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3774364106' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:45:23.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:22 smithi138 ceph-mon[193606]: osdmap e757: 8 total, 8 up, 8 in 2023-09-23T23:45:24.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:23 smithi138 ceph-mon[193606]: pgmap v2082: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:24.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:23 smithi138 ceph-mon[193606]: failed to encode map e758 with expected crc 2023-09-23T23:45:24.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:23 smithi138 ceph-mon[193606]: failed to encode map e758 with expected crc 2023-09-23T23:45:24.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:23 smithi138 ceph-mon[193606]: failed to encode map e758 with expected crc 2023-09-23T23:45:24.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:23 smithi138 ceph-mon[193606]: osdmap e758: 8 total, 8 up, 8 in 2023-09-23T23:45:24.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:23 smithi138 ceph-mon[193606]: failed to encode map e758 with expected crc 2023-09-23T23:45:24.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:23 smithi138 ceph-mon[193606]: failed to encode map e758 with expected crc 2023-09-23T23:45:24.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:23 smithi138 ceph-mon[193606]: 1.15 scrub starts 2023-09-23T23:45:24.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:23 smithi138 ceph-mon[193606]: 1.15 scrub ok 2023-09-23T23:45:24.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[212939]: pgmap v2082: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:24.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[212939]: failed to encode map e758 with expected crc 2023-09-23T23:45:24.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[212939]: failed to encode map e758 with expected crc 2023-09-23T23:45:24.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[212939]: failed to encode map e758 with expected crc 2023-09-23T23:45:24.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[212939]: osdmap e758: 8 total, 8 up, 8 in 2023-09-23T23:45:24.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[212939]: failed to encode map e758 with expected crc 2023-09-23T23:45:24.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[212939]: failed to encode map e758 with expected crc 2023-09-23T23:45:24.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[212939]: 1.15 scrub starts 2023-09-23T23:45:24.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[212939]: 1.15 scrub ok 2023-09-23T23:45:24.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[221102]: pgmap v2082: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:24.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[221102]: failed to encode map e758 with expected crc 2023-09-23T23:45:24.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[221102]: failed to encode map e758 with expected crc 2023-09-23T23:45:24.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[221102]: failed to encode map e758 with expected crc 2023-09-23T23:45:24.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[221102]: osdmap e758: 8 total, 8 up, 8 in 2023-09-23T23:45:24.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[221102]: failed to encode map e758 with expected crc 2023-09-23T23:45:24.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[221102]: failed to encode map e758 with expected crc 2023-09-23T23:45:24.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[221102]: 1.15 scrub starts 2023-09-23T23:45:24.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:23 smithi110 ceph-mon[221102]: 1.15 scrub ok 2023-09-23T23:45:24.987 INFO:teuthology.orchestra.run.smithi110.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-09-23T23:45:25.166 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:24 smithi110 ceph-mon[212939]: failed to encode map e759 with expected crc 2023-09-23T23:45:25.166 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:24 smithi110 ceph-mon[212939]: failed to encode map e759 with expected crc 2023-09-23T23:45:25.167 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:24 smithi110 ceph-mon[212939]: osdmap e759: 8 total, 8 up, 8 in 2023-09-23T23:45:25.167 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:24 smithi110 ceph-mon[212939]: failed to encode map e759 with expected crc 2023-09-23T23:45:25.167 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:24 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:25.167 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:24 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/519444739' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:45:25.167 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:24 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:45:25.167 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:24 smithi110 ceph-mon[221102]: failed to encode map e759 with expected crc 2023-09-23T23:45:25.167 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:24 smithi110 ceph-mon[221102]: failed to encode map e759 with expected crc 2023-09-23T23:45:25.167 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:24 smithi110 ceph-mon[221102]: osdmap e759: 8 total, 8 up, 8 in 2023-09-23T23:45:25.167 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:24 smithi110 ceph-mon[221102]: failed to encode map e759 with expected crc 2023-09-23T23:45:25.167 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:24 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:25.168 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:24 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/519444739' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:45:25.168 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:24 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:45:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:24 smithi138 ceph-mon[193606]: failed to encode map e759 with expected crc 2023-09-23T23:45:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:24 smithi138 ceph-mon[193606]: failed to encode map e759 with expected crc 2023-09-23T23:45:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:24 smithi138 ceph-mon[193606]: osdmap e759: 8 total, 8 up, 8 in 2023-09-23T23:45:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:24 smithi138 ceph-mon[193606]: failed to encode map e759 with expected crc 2023-09-23T23:45:25.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:24 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:25.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:24 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/519444739' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:45:25.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:24 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:45:25.664 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T23:45:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:25 smithi110 ceph-mon[212939]: pgmap v2085: 103 pgs: 2 remapped+peering, 101 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:25.952 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:25 smithi110 ceph-mon[212939]: failed to encode map e760 with expected crc 2023-09-23T23:45:25.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:25 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:45:25.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:25 smithi110 ceph-mon[212939]: osdmap e760: 8 total, 8 up, 8 in 2023-09-23T23:45:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:25 smithi110 ceph-mon[221102]: pgmap v2085: 103 pgs: 2 remapped+peering, 101 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:25 smithi110 ceph-mon[221102]: failed to encode map e760 with expected crc 2023-09-23T23:45:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:25 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:45:25.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:25 smithi110 ceph-mon[221102]: osdmap e760: 8 total, 8 up, 8 in 2023-09-23T23:45:26.017 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:26.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:25 smithi138 ceph-mon[193606]: pgmap v2085: 103 pgs: 2 remapped+peering, 101 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:26.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:25 smithi138 ceph-mon[193606]: failed to encode map e760 with expected crc 2023-09-23T23:45:26.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:25 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:45:26.193 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:25 smithi138 ceph-mon[193606]: osdmap e760: 8 total, 8 up, 8 in 2023-09-23T23:45:26.953 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:45:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:45:26] "GET /metrics HTTP/1.1" 200 34185 "" "Prometheus/2.43.0" 2023-09-23T23:45:26.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:26 smithi110 ceph-mon[212939]: failed to encode map e760 with expected crc 2023-09-23T23:45:26.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:26 smithi110 ceph-mon[212939]: failed to encode map e760 with expected crc 2023-09-23T23:45:26.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:26 smithi110 ceph-mon[212939]: osdmap e761: 8 total, 8 up, 8 in 2023-09-23T23:45:26.953 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:26 smithi110 ceph-mon[212939]: failed to encode map e761 with expected crc 2023-09-23T23:45:26.953 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:26 smithi110 ceph-mon[221102]: failed to encode map e760 with expected crc 2023-09-23T23:45:26.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:26 smithi110 ceph-mon[221102]: failed to encode map e760 with expected crc 2023-09-23T23:45:26.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:26 smithi110 ceph-mon[221102]: osdmap e761: 8 total, 8 up, 8 in 2023-09-23T23:45:26.954 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:26 smithi110 ceph-mon[221102]: failed to encode map e761 with expected crc 2023-09-23T23:45:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:26 smithi138 ceph-mon[193606]: failed to encode map e760 with expected crc 2023-09-23T23:45:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:26 smithi138 ceph-mon[193606]: failed to encode map e760 with expected crc 2023-09-23T23:45:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:26 smithi138 ceph-mon[193606]: osdmap e761: 8 total, 8 up, 8 in 2023-09-23T23:45:27.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:26 smithi138 ceph-mon[193606]: failed to encode map e761 with expected crc 2023-09-23T23:45:27.987 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T23:45:28.182 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:27 smithi110 ceph-mon[212939]: failed to encode map e761 with expected crc 2023-09-23T23:45:28.183 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:27 smithi110 ceph-mon[212939]: failed to encode map e761 with expected crc 2023-09-23T23:45:28.183 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:27 smithi110 ceph-mon[212939]: failed to encode map e761 with expected crc 2023-09-23T23:45:28.183 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:27 smithi110 ceph-mon[212939]: pgmap v2088: 103 pgs: 1 active+undersized+remapped, 2 remapped+peering, 100 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:28.183 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:27 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1704911013' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:45:28.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:27 smithi110 ceph-mon[221102]: failed to encode map e761 with expected crc 2023-09-23T23:45:28.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:27 smithi110 ceph-mon[221102]: failed to encode map e761 with expected crc 2023-09-23T23:45:28.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:27 smithi110 ceph-mon[221102]: failed to encode map e761 with expected crc 2023-09-23T23:45:28.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:27 smithi110 ceph-mon[221102]: pgmap v2088: 103 pgs: 1 active+undersized+remapped, 2 remapped+peering, 100 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:28.183 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:27 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1704911013' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:45:28.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:27 smithi138 ceph-mon[193606]: failed to encode map e761 with expected crc 2023-09-23T23:45:28.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:27 smithi138 ceph-mon[193606]: failed to encode map e761 with expected crc 2023-09-23T23:45:28.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:27 smithi138 ceph-mon[193606]: failed to encode map e761 with expected crc 2023-09-23T23:45:28.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:27 smithi138 ceph-mon[193606]: pgmap v2088: 103 pgs: 1 active+undersized+remapped, 2 remapped+peering, 100 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:28.192 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:27 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1704911013' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:45:28.697 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-09-23T23:45:29.049 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:29.265 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:28 smithi110 ceph-mon[212939]: failed to encode map e762 with expected crc 2023-09-23T23:45:29.265 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:28 smithi110 ceph-mon[212939]: failed to encode map e762 with expected crc 2023-09-23T23:45:29.265 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:28 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1704911013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:45:29.265 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:28 smithi110 ceph-mon[212939]: osdmap e762: 8 total, 8 up, 8 in 2023-09-23T23:45:29.265 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:28 smithi110 ceph-mon[212939]: failed to encode map e762 with expected crc 2023-09-23T23:45:29.265 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:45:29.266 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:28 smithi110 ceph-mon[212939]: pgmap v2090: 103 pgs: 1 active+undersized+remapped, 2 remapped+peering, 100 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:29.266 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:28 smithi110 ceph-mon[221102]: failed to encode map e762 with expected crc 2023-09-23T23:45:29.266 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:28 smithi110 ceph-mon[221102]: failed to encode map e762 with expected crc 2023-09-23T23:45:29.266 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:28 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1704911013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:45:29.266 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:28 smithi110 ceph-mon[221102]: osdmap e762: 8 total, 8 up, 8 in 2023-09-23T23:45:29.266 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:28 smithi110 ceph-mon[221102]: failed to encode map e762 with expected crc 2023-09-23T23:45:29.266 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:45:29.266 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:28 smithi110 ceph-mon[221102]: pgmap v2090: 103 pgs: 1 active+undersized+remapped, 2 remapped+peering, 100 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:28 smithi138 ceph-mon[193606]: failed to encode map e762 with expected crc 2023-09-23T23:45:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:28 smithi138 ceph-mon[193606]: failed to encode map e762 with expected crc 2023-09-23T23:45:29.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:28 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1704911013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:45:29.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:28 smithi138 ceph-mon[193606]: osdmap e762: 8 total, 8 up, 8 in 2023-09-23T23:45:29.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:28 smithi138 ceph-mon[193606]: failed to encode map e762 with expected crc 2023-09-23T23:45:29.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:45:29.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:28 smithi138 ceph-mon[193606]: pgmap v2090: 103 pgs: 1 active+undersized+remapped, 2 remapped+peering, 100 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:31.231 INFO:teuthology.orchestra.run.smithi110.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-09-23T23:45:31.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:31 smithi138 ceph-mon[193606]: pgmap v2091: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:31.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:31 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2952035127' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:45:31.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:31 smithi110 ceph-mon[212939]: pgmap v2091: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:31.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:31 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2952035127' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:45:31.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:31 smithi110 ceph-mon[221102]: pgmap v2091: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:31.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:31 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2952035127' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:45:31.947 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T23:45:32.299 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:32.516 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:32 smithi110 ceph-mon[212939]: failed to encode map e763 with expected crc 2023-09-23T23:45:32.516 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:32 smithi110 ceph-mon[212939]: failed to encode map e763 with expected crc 2023-09-23T23:45:32.516 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:32 smithi110 ceph-mon[212939]: failed to encode map e763 with expected crc 2023-09-23T23:45:32.516 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:32 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2952035127' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:45:32.516 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:32 smithi110 ceph-mon[212939]: osdmap e763: 8 total, 8 up, 8 in 2023-09-23T23:45:32.517 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:32 smithi110 ceph-mon[221102]: failed to encode map e763 with expected crc 2023-09-23T23:45:32.517 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:32 smithi110 ceph-mon[221102]: failed to encode map e763 with expected crc 2023-09-23T23:45:32.517 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:32 smithi110 ceph-mon[221102]: failed to encode map e763 with expected crc 2023-09-23T23:45:32.517 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:32 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2952035127' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:45:32.517 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:32 smithi110 ceph-mon[221102]: osdmap e763: 8 total, 8 up, 8 in 2023-09-23T23:45:32.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:32 smithi138 ceph-mon[193606]: failed to encode map e763 with expected crc 2023-09-23T23:45:32.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:32 smithi138 ceph-mon[193606]: failed to encode map e763 with expected crc 2023-09-23T23:45:32.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:32 smithi138 ceph-mon[193606]: failed to encode map e763 with expected crc 2023-09-23T23:45:32.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:32 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2952035127' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:45:32.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:32 smithi138 ceph-mon[193606]: osdmap e763: 8 total, 8 up, 8 in 2023-09-23T23:45:33.626 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:33 smithi110 ceph-mon[212939]: pgmap v2093: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:33.626 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:33 smithi110 ceph-mon[212939]: failed to encode map e764 with expected crc 2023-09-23T23:45:33.626 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:33 smithi110 ceph-mon[212939]: failed to encode map e764 with expected crc 2023-09-23T23:45:33.626 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:33 smithi110 ceph-mon[212939]: failed to encode map e764 with expected crc 2023-09-23T23:45:33.626 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:33 smithi110 ceph-mon[212939]: osdmap e764: 8 total, 8 up, 8 in 2023-09-23T23:45:33.626 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:33 smithi110 ceph-mon[212939]: failed to encode map e764 with expected crc 2023-09-23T23:45:33.627 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:33 smithi110 ceph-mon[212939]: 1.a scrub starts 2023-09-23T23:45:33.627 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:33 smithi110 ceph-mon[212939]: 1.a scrub ok 2023-09-23T23:45:33.627 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:33 smithi110 ceph-mon[221102]: pgmap v2093: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:33.627 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:33 smithi110 ceph-mon[221102]: failed to encode map e764 with expected crc 2023-09-23T23:45:33.627 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:33 smithi110 ceph-mon[221102]: failed to encode map e764 with expected crc 2023-09-23T23:45:33.627 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:33 smithi110 ceph-mon[221102]: failed to encode map e764 with expected crc 2023-09-23T23:45:33.627 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:33 smithi110 ceph-mon[221102]: osdmap e764: 8 total, 8 up, 8 in 2023-09-23T23:45:33.627 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:33 smithi110 ceph-mon[221102]: failed to encode map e764 with expected crc 2023-09-23T23:45:33.627 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:33 smithi110 ceph-mon[221102]: 1.a scrub starts 2023-09-23T23:45:33.627 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:33 smithi110 ceph-mon[221102]: 1.a scrub ok 2023-09-23T23:45:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:33 smithi138 ceph-mon[193606]: pgmap v2093: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:33 smithi138 ceph-mon[193606]: failed to encode map e764 with expected crc 2023-09-23T23:45:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:33 smithi138 ceph-mon[193606]: failed to encode map e764 with expected crc 2023-09-23T23:45:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:33 smithi138 ceph-mon[193606]: failed to encode map e764 with expected crc 2023-09-23T23:45:33.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:33 smithi138 ceph-mon[193606]: osdmap e764: 8 total, 8 up, 8 in 2023-09-23T23:45:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:33 smithi138 ceph-mon[193606]: failed to encode map e764 with expected crc 2023-09-23T23:45:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:33 smithi138 ceph-mon[193606]: 1.a scrub starts 2023-09-23T23:45:33.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:33 smithi138 ceph-mon[193606]: 1.a scrub ok 2023-09-23T23:45:34.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:34 smithi138 ceph-mon[193606]: 1.15 scrub starts 2023-09-23T23:45:34.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:34 smithi138 ceph-mon[193606]: 1.15 scrub ok 2023-09-23T23:45:34.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:34 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2864387246' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:45:34.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:34 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:45:34.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:34 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:34.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:34 smithi110 ceph-mon[212939]: 1.15 scrub starts 2023-09-23T23:45:34.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:34 smithi110 ceph-mon[212939]: 1.15 scrub ok 2023-09-23T23:45:34.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:34 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2864387246' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:45:34.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:34 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:45:34.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:34 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:34.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:34 smithi110 ceph-mon[221102]: 1.15 scrub starts 2023-09-23T23:45:34.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:34 smithi110 ceph-mon[221102]: 1.15 scrub ok 2023-09-23T23:45:34.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:34 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2864387246' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:45:34.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:34 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:45:34.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:34 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:45:34.987 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T23:45:35.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:35 smithi110 ceph-mon[212939]: pgmap v2095: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2023-09-23T23:45:35.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:35 smithi110 ceph-mon[212939]: failed to encode map e765 with expected crc 2023-09-23T23:45:35.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:35 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:45:35.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:35 smithi110 ceph-mon[212939]: osdmap e765: 8 total, 8 up, 8 in 2023-09-23T23:45:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:35 smithi110 ceph-mon[221102]: pgmap v2095: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2023-09-23T23:45:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:35 smithi110 ceph-mon[221102]: failed to encode map e765 with expected crc 2023-09-23T23:45:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:35 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:45:35.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:35 smithi110 ceph-mon[221102]: osdmap e765: 8 total, 8 up, 8 in 2023-09-23T23:45:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:35 smithi138 ceph-mon[193606]: pgmap v2095: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2023-09-23T23:45:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:35 smithi138 ceph-mon[193606]: failed to encode map e765 with expected crc 2023-09-23T23:45:35.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:35 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:45:35.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:35 smithi138 ceph-mon[193606]: osdmap e765: 8 total, 8 up, 8 in 2023-09-23T23:45:35.752 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-09-23T23:45:36.105 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:36.493 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:36 smithi110 ceph-mon[221102]: failed to encode map e765 with expected crc 2023-09-23T23:45:36.493 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:36 smithi110 ceph-mon[221102]: 1.1 scrub starts 2023-09-23T23:45:36.493 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:36 smithi110 ceph-mon[221102]: 1.1 scrub ok 2023-09-23T23:45:36.493 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:36 smithi110 ceph-mon[212939]: failed to encode map e765 with expected crc 2023-09-23T23:45:36.493 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:36 smithi110 ceph-mon[212939]: 1.1 scrub starts 2023-09-23T23:45:36.493 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:36 smithi110 ceph-mon[212939]: 1.1 scrub ok 2023-09-23T23:45:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:36 smithi138 ceph-mon[193606]: failed to encode map e765 with expected crc 2023-09-23T23:45:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:36 smithi138 ceph-mon[193606]: 1.1 scrub starts 2023-09-23T23:45:36.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:36 smithi138 ceph-mon[193606]: 1.1 scrub ok 2023-09-23T23:45:37.174 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:45:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:45:36] "GET /metrics HTTP/1.1" 200 34162 "" "Prometheus/2.43.0" 2023-09-23T23:45:37.443 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:37 smithi110 ceph-mon[212939]: pgmap v2097: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2023-09-23T23:45:37.443 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:37 smithi110 ceph-mon[221102]: pgmap v2097: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2023-09-23T23:45:37.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:37 smithi138 ceph-mon[193606]: pgmap v2097: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2023-09-23T23:45:38.248 INFO:teuthology.orchestra.run.smithi110.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-09-23T23:45:38.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:38 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3385430140' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:45:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:38 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3385430140' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:45:38.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:38 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3385430140' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:45:38.990 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T23:45:39.245 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:39 smithi110 ceph-mon[221102]: pgmap v2098: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2023-09-23T23:45:39.245 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:39 smithi110 ceph-mon[221102]: failed to encode map e766 with expected crc 2023-09-23T23:45:39.245 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:39 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3385430140' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:45:39.245 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:39 smithi110 ceph-mon[221102]: osdmap e766: 8 total, 8 up, 8 in 2023-09-23T23:45:39.245 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:39 smithi110 ceph-mon[221102]: failed to encode map e766 with expected crc 2023-09-23T23:45:39.336 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:39.539 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:39 smithi110 ceph-mon[212939]: pgmap v2098: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2023-09-23T23:45:39.540 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:39 smithi110 ceph-mon[212939]: failed to encode map e766 with expected crc 2023-09-23T23:45:39.540 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:39 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3385430140' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:45:39.540 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:39 smithi110 ceph-mon[212939]: osdmap e766: 8 total, 8 up, 8 in 2023-09-23T23:45:39.540 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:39 smithi110 ceph-mon[212939]: failed to encode map e766 with expected crc 2023-09-23T23:45:39.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:39 smithi138 ceph-mon[193606]: pgmap v2098: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2023-09-23T23:45:39.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:39 smithi138 ceph-mon[193606]: failed to encode map e766 with expected crc 2023-09-23T23:45:39.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:39 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3385430140' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:45:39.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:39 smithi138 ceph-mon[193606]: osdmap e766: 8 total, 8 up, 8 in 2023-09-23T23:45:39.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:39 smithi138 ceph-mon[193606]: failed to encode map e766 with expected crc 2023-09-23T23:45:41.252 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T23:45:41.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:41 smithi138 ceph-mon[193606]: pgmap v2100: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2023-09-23T23:45:41.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:41 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3502099661' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:45:41.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:41 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:45:41.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:41 smithi110 ceph-mon[212939]: pgmap v2100: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2023-09-23T23:45:41.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:41 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3502099661' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:45:41.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:41 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:45:41.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:41 smithi110 ceph-mon[221102]: pgmap v2100: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2023-09-23T23:45:41.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:41 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3502099661' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:45:41.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:41 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:45:41.980 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-23T23:45:42.258 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:42 smithi110 ceph-mon[221102]: failed to encode map e767 with expected crc 2023-09-23T23:45:42.258 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:42 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:45:42.258 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:42 smithi110 ceph-mon[221102]: osdmap e767: 8 total, 8 up, 8 in 2023-09-23T23:45:42.258 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:42 smithi110 ceph-mon[221102]: failed to encode map e767 with expected crc 2023-09-23T23:45:42.329 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:42.539 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:42 smithi110 ceph-mon[212939]: failed to encode map e767 with expected crc 2023-09-23T23:45:42.539 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:42 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:45:42.539 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:42 smithi110 ceph-mon[212939]: osdmap e767: 8 total, 8 up, 8 in 2023-09-23T23:45:42.539 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:42 smithi110 ceph-mon[212939]: failed to encode map e767 with expected crc 2023-09-23T23:45:42.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:42 smithi138 ceph-mon[193606]: failed to encode map e767 with expected crc 2023-09-23T23:45:42.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:42 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:45:42.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:42 smithi138 ceph-mon[193606]: osdmap e767: 8 total, 8 up, 8 in 2023-09-23T23:45:42.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:42 smithi138 ceph-mon[193606]: failed to encode map e767 with expected crc 2023-09-23T23:45:43.642 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:43 smithi110 ceph-mon[212939]: failed to encode map e767 with expected crc 2023-09-23T23:45:43.642 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:43 smithi110 ceph-mon[212939]: pgmap v2102: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:43.642 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:43 smithi110 ceph-mon[212939]: failed to encode map e767 with expected crc 2023-09-23T23:45:43.642 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:43 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:45:43.643 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:43 smithi110 ceph-mon[221102]: failed to encode map e767 with expected crc 2023-09-23T23:45:43.643 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:43 smithi110 ceph-mon[221102]: pgmap v2102: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:43.643 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:43 smithi110 ceph-mon[221102]: failed to encode map e767 with expected crc 2023-09-23T23:45:43.643 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:43 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:45:43.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:43 smithi138 ceph-mon[193606]: failed to encode map e767 with expected crc 2023-09-23T23:45:43.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:43 smithi138 ceph-mon[193606]: pgmap v2102: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:43.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:43 smithi138 ceph-mon[193606]: failed to encode map e767 with expected crc 2023-09-23T23:45:43.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:43 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:45:44.991 INFO:teuthology.orchestra.run.smithi110.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-23T23:45:45.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:44 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3779450382' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:45:45.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:44 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:45:45.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:44 smithi138 ceph-mon[193606]: pgmap v2103: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:45.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:44 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3779450382' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:45:45.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:44 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:45:45.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:44 smithi110 ceph-mon[221102]: pgmap v2103: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:45.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:44 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3779450382' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:45:45.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:44 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:45:45.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:44 smithi110 ceph-mon[212939]: pgmap v2103: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:45.726 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T23:45:45.985 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:45 smithi110 ceph-mon[212939]: failed to encode map e768 with expected crc 2023-09-23T23:45:45.985 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:45 smithi110 ceph-mon[212939]: failed to encode map e768 with expected crc 2023-09-23T23:45:45.985 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:45 smithi110 ceph-mon[212939]: failed to encode map e768 with expected crc 2023-09-23T23:45:45.985 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:45 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:45:45.985 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:45 smithi110 ceph-mon[212939]: osdmap e768: 8 total, 8 up, 8 in 2023-09-23T23:45:45.985 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:45 smithi110 ceph-mon[212939]: failed to encode map e768 with expected crc 2023-09-23T23:45:45.985 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:45 smithi110 ceph-mon[212939]: failed to encode map e768 with expected crc 2023-09-23T23:45:45.986 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:45 smithi110 ceph-mon[221102]: failed to encode map e768 with expected crc 2023-09-23T23:45:45.986 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:45 smithi110 ceph-mon[221102]: failed to encode map e768 with expected crc 2023-09-23T23:45:45.986 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:45 smithi110 ceph-mon[221102]: failed to encode map e768 with expected crc 2023-09-23T23:45:45.986 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:45 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:45:45.986 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:45 smithi110 ceph-mon[221102]: osdmap e768: 8 total, 8 up, 8 in 2023-09-23T23:45:45.986 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:45 smithi110 ceph-mon[221102]: failed to encode map e768 with expected crc 2023-09-23T23:45:45.986 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:45 smithi110 ceph-mon[221102]: failed to encode map e768 with expected crc 2023-09-23T23:45:46.083 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:45 smithi138 ceph-mon[193606]: failed to encode map e768 with expected crc 2023-09-23T23:45:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:45 smithi138 ceph-mon[193606]: failed to encode map e768 with expected crc 2023-09-23T23:45:46.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:45 smithi138 ceph-mon[193606]: failed to encode map e768 with expected crc 2023-09-23T23:45:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:45 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:45:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:45 smithi138 ceph-mon[193606]: osdmap e768: 8 total, 8 up, 8 in 2023-09-23T23:45:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:45 smithi138 ceph-mon[193606]: failed to encode map e768 with expected crc 2023-09-23T23:45:46.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:45 smithi138 ceph-mon[193606]: failed to encode map e768 with expected crc 2023-09-23T23:45:46.952 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:45:46 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:45:46] "GET /metrics HTTP/1.1" 200 34161 "" "Prometheus/2.43.0" 2023-09-23T23:45:47.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:46 smithi138 ceph-mon[193606]: pgmap v2105: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:47.449 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:46 smithi110 ceph-mon[212939]: pgmap v2105: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:47.449 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:46 smithi110 ceph-mon[221102]: pgmap v2105: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:48.001 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T23:45:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:47 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/358913069' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:45:48.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:47 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:45:48.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:47 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/358913069' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:45:48.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:47 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:45:48.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:47 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/358913069' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:45:48.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:47 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:45:48.754 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-09-23T23:45:49.104 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:49.321 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:48 smithi110 ceph-mon[212939]: failed to encode map e769 with expected crc 2023-09-23T23:45:49.321 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:48 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:45:49.321 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:48 smithi110 ceph-mon[212939]: osdmap e769: 8 total, 8 up, 8 in 2023-09-23T23:45:49.321 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:48 smithi110 ceph-mon[212939]: failed to encode map e769 with expected crc 2023-09-23T23:45:49.321 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:48 smithi110 ceph-mon[212939]: pgmap v2107: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:49.322 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:48 smithi110 ceph-mon[212939]: failed to encode map e769 with expected crc 2023-09-23T23:45:49.322 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:48 smithi110 ceph-mon[221102]: failed to encode map e769 with expected crc 2023-09-23T23:45:49.322 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:48 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:45:49.322 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:48 smithi110 ceph-mon[221102]: osdmap e769: 8 total, 8 up, 8 in 2023-09-23T23:45:49.322 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:48 smithi110 ceph-mon[221102]: failed to encode map e769 with expected crc 2023-09-23T23:45:49.322 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:48 smithi110 ceph-mon[221102]: pgmap v2107: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:49.322 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:48 smithi110 ceph-mon[221102]: failed to encode map e769 with expected crc 2023-09-23T23:45:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:48 smithi138 ceph-mon[193606]: failed to encode map e769 with expected crc 2023-09-23T23:45:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:48 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:45:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:48 smithi138 ceph-mon[193606]: osdmap e769: 8 total, 8 up, 8 in 2023-09-23T23:45:49.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:48 smithi138 ceph-mon[193606]: failed to encode map e769 with expected crc 2023-09-23T23:45:49.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:48 smithi138 ceph-mon[193606]: pgmap v2107: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:49.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:48 smithi138 ceph-mon[193606]: failed to encode map e769 with expected crc 2023-09-23T23:45:51.235 INFO:teuthology.orchestra.run.smithi110.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-09-23T23:45:51.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:51 smithi138 ceph-mon[193606]: pgmap v2108: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:51.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:51 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/11845984' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:45:51.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:51 smithi110 ceph-mon[212939]: pgmap v2108: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:51.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:51 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/11845984' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:45:51.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:51 smithi110 ceph-mon[221102]: pgmap v2108: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:51.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:51 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/11845984' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:45:51.919 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T23:45:52.274 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:52.503 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:52 smithi110 ceph-mon[212939]: failed to encode map e770 with expected crc 2023-09-23T23:45:52.503 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:52 smithi110 ceph-mon[212939]: failed to encode map e770 with expected crc 2023-09-23T23:45:52.503 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:52 smithi110 ceph-mon[212939]: failed to encode map e770 with expected crc 2023-09-23T23:45:52.503 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:52 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/11845984' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:45:52.503 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:52 smithi110 ceph-mon[212939]: osdmap e770: 8 total, 8 up, 8 in 2023-09-23T23:45:52.504 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:52 smithi110 ceph-mon[221102]: failed to encode map e770 with expected crc 2023-09-23T23:45:52.504 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:52 smithi110 ceph-mon[221102]: failed to encode map e770 with expected crc 2023-09-23T23:45:52.504 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:52 smithi110 ceph-mon[221102]: failed to encode map e770 with expected crc 2023-09-23T23:45:52.504 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:52 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/11845984' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:45:52.504 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:52 smithi110 ceph-mon[221102]: osdmap e770: 8 total, 8 up, 8 in 2023-09-23T23:45:52.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:52 smithi138 ceph-mon[193606]: failed to encode map e770 with expected crc 2023-09-23T23:45:52.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:52 smithi138 ceph-mon[193606]: failed to encode map e770 with expected crc 2023-09-23T23:45:52.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:52 smithi138 ceph-mon[193606]: failed to encode map e770 with expected crc 2023-09-23T23:45:52.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:52 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/11845984' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:45:52.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:52 smithi138 ceph-mon[193606]: osdmap e770: 8 total, 8 up, 8 in 2023-09-23T23:45:53.657 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:53 smithi110 ceph-mon[212939]: pgmap v2110: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:53.657 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:53 smithi110 ceph-mon[212939]: failed to encode map e771 with expected crc 2023-09-23T23:45:53.657 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:53 smithi110 ceph-mon[212939]: failed to encode map e771 with expected crc 2023-09-23T23:45:53.657 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:53 smithi110 ceph-mon[212939]: failed to encode map e771 with expected crc 2023-09-23T23:45:53.657 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:53 smithi110 ceph-mon[212939]: osdmap e771: 8 total, 8 up, 8 in 2023-09-23T23:45:53.658 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:53 smithi110 ceph-mon[221102]: pgmap v2110: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:53.658 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:53 smithi110 ceph-mon[221102]: failed to encode map e771 with expected crc 2023-09-23T23:45:53.658 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:53 smithi110 ceph-mon[221102]: failed to encode map e771 with expected crc 2023-09-23T23:45:53.658 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:53 smithi110 ceph-mon[221102]: failed to encode map e771 with expected crc 2023-09-23T23:45:53.658 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:53 smithi110 ceph-mon[221102]: osdmap e771: 8 total, 8 up, 8 in 2023-09-23T23:45:53.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:53 smithi138 ceph-mon[193606]: pgmap v2110: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:45:53.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:53 smithi138 ceph-mon[193606]: failed to encode map e771 with expected crc 2023-09-23T23:45:53.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:53 smithi138 ceph-mon[193606]: failed to encode map e771 with expected crc 2023-09-23T23:45:53.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:53 smithi138 ceph-mon[193606]: failed to encode map e771 with expected crc 2023-09-23T23:45:53.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:53 smithi138 ceph-mon[193606]: osdmap e771: 8 total, 8 up, 8 in 2023-09-23T23:45:54.992 INFO:teuthology.orchestra.run.smithi110.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T23:45:55.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:54 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3575339697' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:45:55.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:54 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:45:55.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:54 smithi138 ceph-mon[193606]: pgmap v2112: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 3 keys/s, 0 objects/s recovering 2023-09-23T23:45:55.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:54 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3575339697' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:45:55.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:54 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:45:55.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:54 smithi110 ceph-mon[212939]: pgmap v2112: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 3 keys/s, 0 objects/s recovering 2023-09-23T23:45:55.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:54 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3575339697' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:45:55.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:54 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:45:55.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:54 smithi110 ceph-mon[221102]: pgmap v2112: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 3 keys/s, 0 objects/s recovering 2023-09-23T23:45:55.757 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-09-23T23:45:56.107 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:56.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:55 smithi110 ceph-mon[212939]: failed to encode map e772 with expected crc 2023-09-23T23:45:56.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:55 smithi110 ceph-mon[212939]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:45:56.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:55 smithi110 ceph-mon[212939]: osdmap e772: 8 total, 8 up, 8 in 2023-09-23T23:45:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:55 smithi110 ceph-mon[221102]: failed to encode map e772 with expected crc 2023-09-23T23:45:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:55 smithi110 ceph-mon[221102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:45:56.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:55 smithi110 ceph-mon[221102]: osdmap e772: 8 total, 8 up, 8 in 2023-09-23T23:45:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:55 smithi138 ceph-mon[193606]: failed to encode map e772 with expected crc 2023-09-23T23:45:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:55 smithi138 ceph-mon[193606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:45:56.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:55 smithi138 ceph-mon[193606]: osdmap e772: 8 total, 8 up, 8 in 2023-09-23T23:45:56.989 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:45:56 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:45:56] "GET /metrics HTTP/1.1" 200 34161 "" "Prometheus/2.43.0" 2023-09-23T23:45:57.246 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:56 smithi110 ceph-mon[221102]: pgmap v2114: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 3 keys/s, 0 objects/s recovering 2023-09-23T23:45:57.247 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:56 smithi110 ceph-mon[212939]: pgmap v2114: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 3 keys/s, 0 objects/s recovering 2023-09-23T23:45:57.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:56 smithi138 ceph-mon[193606]: pgmap v2114: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 3 keys/s, 0 objects/s recovering 2023-09-23T23:45:58.001 INFO:teuthology.orchestra.run.smithi110.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-09-23T23:45:58.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:57 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1012851800' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:45:58.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:57 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1012851800' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:45:58.452 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:57 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1012851800' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:45:58.686 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-09-23T23:45:58.686 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-09-23T23:45:58.686 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-09-23T23:45:58.687 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-09-23T23:45:58.687 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T23:45:58.687 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T23:45:59.036 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:45:59.261 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:58 smithi110 ceph-mon[212939]: failed to encode map e773 with expected crc 2023-09-23T23:45:59.262 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:58 smithi110 ceph-mon[212939]: failed to encode map e773 with expected crc 2023-09-23T23:45:59.262 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:58 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1012851800' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:45:59.262 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:58 smithi110 ceph-mon[212939]: osdmap e773: 8 total, 8 up, 8 in 2023-09-23T23:45:59.262 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:58 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:45:59.262 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:45:58 smithi110 ceph-mon[212939]: pgmap v2116: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 3 keys/s, 0 objects/s recovering 2023-09-23T23:45:59.263 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:58 smithi110 ceph-mon[221102]: failed to encode map e773 with expected crc 2023-09-23T23:45:59.263 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:58 smithi110 ceph-mon[221102]: failed to encode map e773 with expected crc 2023-09-23T23:45:59.263 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:58 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1012851800' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:45:59.263 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:58 smithi110 ceph-mon[221102]: osdmap e773: 8 total, 8 up, 8 in 2023-09-23T23:45:59.263 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:58 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:45:59.263 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:45:58 smithi110 ceph-mon[221102]: pgmap v2116: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 3 keys/s, 0 objects/s recovering 2023-09-23T23:45:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:58 smithi138 ceph-mon[193606]: failed to encode map e773 with expected crc 2023-09-23T23:45:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:58 smithi138 ceph-mon[193606]: failed to encode map e773 with expected crc 2023-09-23T23:45:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:58 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1012851800' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:45:59.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:58 smithi138 ceph-mon[193606]: osdmap e773: 8 total, 8 up, 8 in 2023-09-23T23:45:59.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:58 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:45:59.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:45:58 smithi138 ceph-mon[193606]: pgmap v2116: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 3 keys/s, 0 objects/s recovering 2023-09-23T23:46:00.790 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:46:00.790 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":773,"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:46:49.032269+0000","modified":"2023-09-23T23:45:57.988164+0000","last_up_change":"2023-09-23T23:44:49.203428+0000","last_in_change":"2023-09-23T23:44:58.968345+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:49:49.263399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"348","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"333","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:52:19.908332+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"714","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":714,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:53:15.155816+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:53:16.112056+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"78791acb-c93c-4e4f-bf04-ea3eee46408d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":757,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":983471373},{"type":"v1","addr":"172.21.15.110:6803","nonce":983471373}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":983471373},{"type":"v1","addr":"172.21.15.110:6805","nonce":983471373}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":983471373},{"type":"v1","addr":"172.21.15.110:6809","nonce":983471373}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":983471373},{"type":"v1","addr":"172.21.15.110:6807","nonce":983471373}]},"public_addr":"172.21.15.110:6803/983471373","cluster_addr":"172.21.15.110:6805/983471373","heartbeat_back_addr":"172.21.15.110:6809/983471373","heartbeat_front_addr":"172.21.15.110:6807/983471373","state":["exists","up"]},{"osd":1,"uuid":"6793178f-2ef1-4c48-927c-831a083a3356","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":107,"up_from":112,"up_thru":760,"down_at":108,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":594715751},{"type":"v1","addr":"172.21.15.110:6811","nonce":594715751}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":594715751},{"type":"v1","addr":"172.21.15.110:6813","nonce":594715751}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":594715751},{"type":"v1","addr":"172.21.15.110:6817","nonce":594715751}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":594715751},{"type":"v1","addr":"172.21.15.110:6815","nonce":594715751}]},"public_addr":"172.21.15.110:6811/594715751","cluster_addr":"172.21.15.110:6813/594715751","heartbeat_back_addr":"172.21.15.110:6817/594715751","heartbeat_front_addr":"172.21.15.110:6815/594715751","state":["exists","up"]},{"osd":2,"uuid":"146fbd28-a365-4545-8e97-d629c6ddba21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":130,"last_clean_end":193,"up_from":199,"up_thru":748,"down_at":194,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":618908095},{"type":"v1","addr":"172.21.15.110:6819","nonce":618908095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":618908095},{"type":"v1","addr":"172.21.15.110:6821","nonce":618908095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":618908095},{"type":"v1","addr":"172.21.15.110:6825","nonce":618908095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":618908095},{"type":"v1","addr":"172.21.15.110:6823","nonce":618908095}]},"public_addr":"172.21.15.110:6819/618908095","cluster_addr":"172.21.15.110:6821/618908095","heartbeat_back_addr":"172.21.15.110:6825/618908095","heartbeat_front_addr":"172.21.15.110:6823/618908095","state":["exists","up"]},{"osd":3,"uuid":"6fcf80d5-6977-4929-b7ba-0e9ab65df0a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":144,"up_from":148,"up_thru":757,"down_at":145,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":2333701749},{"type":"v1","addr":"172.21.15.110:6827","nonce":2333701749}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":2333701749},{"type":"v1","addr":"172.21.15.110:6829","nonce":2333701749}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":2333701749},{"type":"v1","addr":"172.21.15.110:6833","nonce":2333701749}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":2333701749},{"type":"v1","addr":"172.21.15.110:6831","nonce":2333701749}]},"public_addr":"172.21.15.110:6827/2333701749","cluster_addr":"172.21.15.110:6829/2333701749","heartbeat_back_addr":"172.21.15.110:6833/2333701749","heartbeat_front_addr":"172.21.15.110:6831/2333701749","state":["exists","up"]},{"osd":4,"uuid":"131d0749-c46c-44f0-ab7f-03f8ad6ed502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":697,"up_from":736,"up_thru":757,"down_at":698,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":524427801},{"type":"v1","addr":"172.21.15.138:6801","nonce":524427801}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":524427801},{"type":"v1","addr":"172.21.15.138:6803","nonce":524427801}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":524427801},{"type":"v1","addr":"172.21.15.138:6807","nonce":524427801}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":524427801},{"type":"v1","addr":"172.21.15.138:6805","nonce":524427801}]},"public_addr":"172.21.15.138:6801/524427801","cluster_addr":"172.21.15.138:6803/524427801","heartbeat_back_addr":"172.21.15.138:6807/524427801","heartbeat_front_addr":"172.21.15.138:6805/524427801","state":["exists","up"]},{"osd":5,"uuid":"8cb496e5-d242-4e41-b279-e8d5ecea2996","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":763,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6809","nonce":3098195887}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6811","nonce":3098195887}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6815","nonce":3098195887}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6813","nonce":3098195887}]},"public_addr":"172.21.15.138:6809/3098195887","cluster_addr":"172.21.15.138:6811/3098195887","heartbeat_back_addr":"172.21.15.138:6815/3098195887","heartbeat_front_addr":"172.21.15.138:6813/3098195887","state":["exists","up"]},{"osd":6,"uuid":"c709e629-85ab-4273-9ca8-f2c4e8fc9074","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":760,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6817","nonce":187413403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6819","nonce":187413403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6823","nonce":187413403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6821","nonce":187413403}]},"public_addr":"172.21.15.138:6817/187413403","cluster_addr":"172.21.15.138:6819/187413403","heartbeat_back_addr":"172.21.15.138:6823/187413403","heartbeat_front_addr":"172.21.15.138:6821/187413403","state":["exists","up"]},{"osd":7,"uuid":"ed15f3b0-cd2e-43b7-b803-e45252b62a45","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":54,"last_clean_end":255,"up_from":343,"up_thru":770,"down_at":256,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":675905607},{"type":"v1","addr":"172.21.15.138:6825","nonce":675905607}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":675905607},{"type":"v1","addr":"172.21.15.138:6827","nonce":675905607}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":675905607},{"type":"v1","addr":"172.21.15.138:6831","nonce":675905607}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":675905607},{"type":"v1","addr":"172.21.15.138:6829","nonce":675905607}]},"public_addr":"172.21.15.138:6825/675905607","cluster_addr":"172.21.15.138:6827/675905607","heartbeat_back_addr":"172.21.15.138:6831/675905607","heartbeat_front_addr":"172.21.15.138:6829/675905607","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:48:58.523930+0000","dead_epoch":87},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:22.993201+0000","dead_epoch":107},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:46.747264+0000","dead_epoch":193},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:11.206564+0000","dead_epoch":144},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:33.511236+0000","dead_epoch":697},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:56.364297+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:18.141750+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:41.242882+0000","dead_epoch":255}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"1.2","mappings":[{"from":7,"to":2}]},{"pgid":"1.f","mappings":[{"from":5,"to":2}]},{"pgid":"3.a","mappings":[{"from":1,"to":2}]},{"pgid":"3.16","mappings":[{"from":1,"to":2}]},{"pgid":"3.18","mappings":[{"from":3,"to":0}]},{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.f","mappings":[{"from":3,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":5}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:0/3250066431":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/3705879030":"2023-09-24T22:58:42.842267+0000","172.21.15.110:0/1399491671":"2023-09-24T22:56:16.309980+0000","172.21.15.110:0/1649304640":"2023-09-24T22:47:38.935467+0000","172.21.15.138:0/2647044257":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/1326657364":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1731974664":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/1506915348":"2023-09-24T22:47:15.528545+0000","172.21.15.138:6834/867141825":"2023-09-24T22:57:33.926232+0000","172.21.15.110:6800/1771333922":"2023-09-24T22:58:42.842267+0000","172.21.15.110:6801/4228126008":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/478775577":"2023-09-24T22:58:42.842267+0000","172.21.15.110:0/380987145":"2023-09-24T22:56:16.309980+0000","172.21.15.110:6800/4228126008":"2023-09-24T22:47:38.935467+0000","172.21.15.110:6800/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.138:0/1613829186":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/3889938710":"2023-09-24T22:58:42.842267+0000","172.21.15.110:6801/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/657635177":"2023-09-24T22:52:17.775108+0000","172.21.15.138:0/331679217":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/410644117":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6801/2914393040":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/3674528218":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6801/576619896":"2023-09-24T22:56:16.309980+0000","172.21.15.110:0/2850586742":"2023-09-24T22:56:16.309980+0000","172.21.15.110:0/2523943302":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6801/1771333922":"2023-09-24T22:58:42.842267+0000","172.21.15.110:0/653682064":"2023-09-24T22:58:42.842267+0000","172.21.15.138:0/3577646480":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/3669919402":"2023-09-24T22:58:42.842267+0000","172.21.15.110:6800/576619896":"2023-09-24T22:56:16.309980+0000","172.21.15.138:0/3320282773":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/3464832160":"2023-09-24T22:56:16.309980+0000","172.21.15.138:6835/867141825":"2023-09-24T22:57:33.926232+0000","172.21.15.110:6800/2914393040":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/3412430555":"2023-09-24T22:47:38.935467+0000","172.21.15.138:0/3258575069":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/2804253682":"2023-09-24T22:58:42.842267+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T23:46:01.483 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:01 smithi110 ceph-mon[212939]: pgmap v2117: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 keys/s, 0 objects/s recovering 2023-09-23T23:46:01.483 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:01 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1100192878' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:46:01.484 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:01 smithi110 ceph-mon[221102]: pgmap v2117: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 keys/s, 0 objects/s recovering 2023-09-23T23:46:01.484 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:01 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1100192878' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:46:01.485 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T23:46:01.485 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T23:46:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:01 smithi138 ceph-mon[193606]: pgmap v2117: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 keys/s, 0 objects/s recovering 2023-09-23T23:46:01.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:01 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1100192878' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:46:01.832 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:03.512 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:46:03.512 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":773,"fsid":"f5f05148-5a62-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:46:49.032269+0000","modified":"2023-09-23T23:45:57.988164+0000","last_up_change":"2023-09-23T23:44:49.203428+0000","last_in_change":"2023-09-23T23:44:58.968345+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:49:49.263399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"348","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"333","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:52:19.908332+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"714","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":714,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:53:15.155816+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:53:16.112056+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"78791acb-c93c-4e4f-bf04-ea3eee46408d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":757,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":983471373},{"type":"v1","addr":"172.21.15.110:6803","nonce":983471373}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":983471373},{"type":"v1","addr":"172.21.15.110:6805","nonce":983471373}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":983471373},{"type":"v1","addr":"172.21.15.110:6809","nonce":983471373}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":983471373},{"type":"v1","addr":"172.21.15.110:6807","nonce":983471373}]},"public_addr":"172.21.15.110:6803/983471373","cluster_addr":"172.21.15.110:6805/983471373","heartbeat_back_addr":"172.21.15.110:6809/983471373","heartbeat_front_addr":"172.21.15.110:6807/983471373","state":["exists","up"]},{"osd":1,"uuid":"6793178f-2ef1-4c48-927c-831a083a3356","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":107,"up_from":112,"up_thru":760,"down_at":108,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":594715751},{"type":"v1","addr":"172.21.15.110:6811","nonce":594715751}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":594715751},{"type":"v1","addr":"172.21.15.110:6813","nonce":594715751}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":594715751},{"type":"v1","addr":"172.21.15.110:6817","nonce":594715751}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":594715751},{"type":"v1","addr":"172.21.15.110:6815","nonce":594715751}]},"public_addr":"172.21.15.110:6811/594715751","cluster_addr":"172.21.15.110:6813/594715751","heartbeat_back_addr":"172.21.15.110:6817/594715751","heartbeat_front_addr":"172.21.15.110:6815/594715751","state":["exists","up"]},{"osd":2,"uuid":"146fbd28-a365-4545-8e97-d629c6ddba21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":130,"last_clean_end":193,"up_from":199,"up_thru":748,"down_at":194,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":618908095},{"type":"v1","addr":"172.21.15.110:6819","nonce":618908095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":618908095},{"type":"v1","addr":"172.21.15.110:6821","nonce":618908095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":618908095},{"type":"v1","addr":"172.21.15.110:6825","nonce":618908095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":618908095},{"type":"v1","addr":"172.21.15.110:6823","nonce":618908095}]},"public_addr":"172.21.15.110:6819/618908095","cluster_addr":"172.21.15.110:6821/618908095","heartbeat_back_addr":"172.21.15.110:6825/618908095","heartbeat_front_addr":"172.21.15.110:6823/618908095","state":["exists","up"]},{"osd":3,"uuid":"6fcf80d5-6977-4929-b7ba-0e9ab65df0a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":144,"up_from":148,"up_thru":757,"down_at":145,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":2333701749},{"type":"v1","addr":"172.21.15.110:6827","nonce":2333701749}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":2333701749},{"type":"v1","addr":"172.21.15.110:6829","nonce":2333701749}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6832","nonce":2333701749},{"type":"v1","addr":"172.21.15.110:6833","nonce":2333701749}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":2333701749},{"type":"v1","addr":"172.21.15.110:6831","nonce":2333701749}]},"public_addr":"172.21.15.110:6827/2333701749","cluster_addr":"172.21.15.110:6829/2333701749","heartbeat_back_addr":"172.21.15.110:6833/2333701749","heartbeat_front_addr":"172.21.15.110:6831/2333701749","state":["exists","up"]},{"osd":4,"uuid":"131d0749-c46c-44f0-ab7f-03f8ad6ed502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":697,"up_from":736,"up_thru":757,"down_at":698,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":524427801},{"type":"v1","addr":"172.21.15.138:6801","nonce":524427801}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":524427801},{"type":"v1","addr":"172.21.15.138:6803","nonce":524427801}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":524427801},{"type":"v1","addr":"172.21.15.138:6807","nonce":524427801}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":524427801},{"type":"v1","addr":"172.21.15.138:6805","nonce":524427801}]},"public_addr":"172.21.15.138:6801/524427801","cluster_addr":"172.21.15.138:6803/524427801","heartbeat_back_addr":"172.21.15.138:6807/524427801","heartbeat_front_addr":"172.21.15.138:6805/524427801","state":["exists","up"]},{"osd":5,"uuid":"8cb496e5-d242-4e41-b279-e8d5ecea2996","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":763,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6809","nonce":3098195887}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6811","nonce":3098195887}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6815","nonce":3098195887}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":3098195887},{"type":"v1","addr":"172.21.15.138:6813","nonce":3098195887}]},"public_addr":"172.21.15.138:6809/3098195887","cluster_addr":"172.21.15.138:6811/3098195887","heartbeat_back_addr":"172.21.15.138:6815/3098195887","heartbeat_front_addr":"172.21.15.138:6813/3098195887","state":["exists","up"]},{"osd":6,"uuid":"c709e629-85ab-4273-9ca8-f2c4e8fc9074","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":760,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6817","nonce":187413403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6819","nonce":187413403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6823","nonce":187413403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":187413403},{"type":"v1","addr":"172.21.15.138:6821","nonce":187413403}]},"public_addr":"172.21.15.138:6817/187413403","cluster_addr":"172.21.15.138:6819/187413403","heartbeat_back_addr":"172.21.15.138:6823/187413403","heartbeat_front_addr":"172.21.15.138:6821/187413403","state":["exists","up"]},{"osd":7,"uuid":"ed15f3b0-cd2e-43b7-b803-e45252b62a45","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":54,"last_clean_end":255,"up_from":343,"up_thru":770,"down_at":256,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":675905607},{"type":"v1","addr":"172.21.15.138:6825","nonce":675905607}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":675905607},{"type":"v1","addr":"172.21.15.138:6827","nonce":675905607}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":675905607},{"type":"v1","addr":"172.21.15.138:6831","nonce":675905607}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":675905607},{"type":"v1","addr":"172.21.15.138:6829","nonce":675905607}]},"public_addr":"172.21.15.138:6825/675905607","cluster_addr":"172.21.15.138:6827/675905607","heartbeat_back_addr":"172.21.15.138:6831/675905607","heartbeat_front_addr":"172.21.15.138:6829/675905607","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:48:58.523930+0000","dead_epoch":87},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:22.993201+0000","dead_epoch":107},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:49:46.747264+0000","dead_epoch":193},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:11.206564+0000","dead_epoch":144},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:33.511236+0000","dead_epoch":697},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:50:56.364297+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:18.141750+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:51:41.242882+0000","dead_epoch":255}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"1.2","mappings":[{"from":7,"to":2}]},{"pgid":"1.f","mappings":[{"from":5,"to":2}]},{"pgid":"3.a","mappings":[{"from":1,"to":2}]},{"pgid":"3.16","mappings":[{"from":1,"to":2}]},{"pgid":"3.18","mappings":[{"from":3,"to":0}]},{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.f","mappings":[{"from":3,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":5}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.110:0/3705879030":"2023-09-24T22:58:42.842267+0000","172.21.15.110:0/2804253682":"2023-09-24T22:58:42.842267+0000","172.21.15.110:6800/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/3250066431":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/1399491671":"2023-09-24T22:56:16.309980+0000","172.21.15.110:0/1649304640":"2023-09-24T22:47:38.935467+0000","172.21.15.138:0/2647044257":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/1326657364":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/1731974664":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/1506915348":"2023-09-24T22:47:15.528545+0000","172.21.15.138:6834/867141825":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/380987145":"2023-09-24T22:56:16.309980+0000","172.21.15.110:0/2850586742":"2023-09-24T22:56:16.309980+0000","172.21.15.110:0/2523943302":"2023-09-24T22:52:17.775108+0000","172.21.15.138:0/1613829186":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/3889938710":"2023-09-24T22:58:42.842267+0000","172.21.15.110:6801/4112551188":"2023-09-24T22:47:15.528545+0000","172.21.15.110:0/657635177":"2023-09-24T22:52:17.775108+0000","172.21.15.138:0/331679217":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/410644117":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6801/2914393040":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/3674528218":"2023-09-24T22:52:17.775108+0000","172.21.15.110:6801/576619896":"2023-09-24T22:56:16.309980+0000","172.21.15.110:6800/4228126008":"2023-09-24T22:47:38.935467+0000","172.21.15.138:0/3577646480":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/3669919402":"2023-09-24T22:58:42.842267+0000","172.21.15.110:6800/576619896":"2023-09-24T22:56:16.309980+0000","172.21.15.138:0/3320282773":"2023-09-24T22:57:33.926232+0000","172.21.15.110:0/3464832160":"2023-09-24T22:56:16.309980+0000","172.21.15.138:6835/867141825":"2023-09-24T22:57:33.926232+0000","172.21.15.110:6800/2914393040":"2023-09-24T22:52:17.775108+0000","172.21.15.110:0/3412430555":"2023-09-24T22:47:38.935467+0000","172.21.15.138:0/3258575069":"2023-09-24T22:57:33.926232+0000","172.21.15.110:6801/4228126008":"2023-09-24T22:47:38.935467+0000","172.21.15.110:0/478775577":"2023-09-24T22:58:42.842267+0000","172.21.15.110:6800/1771333922":"2023-09-24T22:58:42.842267+0000","172.21.15.110:6801/1771333922":"2023-09-24T22:58:42.842267+0000","172.21.15.110:0/653682064":"2023-09-24T22:58:42.842267+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T23:46:03.542 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:03 smithi110 ceph-mon[212939]: pgmap v2118: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:03.542 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:03 smithi110 ceph-mon[221102]: pgmap v2118: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:03.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:03 smithi138 ceph-mon[193606]: pgmap v2118: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:04.219 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T23:46:04.220 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T23:46:04.220 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T23:46:04.221 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T23:46:04.221 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T23:46:04.222 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T23:46:04.222 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T23:46:04.222 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T23:46:04.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:04 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1790554842' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:46:04.704 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:04 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1790554842' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:46:04.705 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:04 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1790554842' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:46:05.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:05 smithi138 ceph-mon[193606]: pgmap v2119: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:05.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:05 smithi110 ceph-mon[212939]: pgmap v2119: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:05.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:05 smithi110 ceph-mon[221102]: pgmap v2119: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:05.808 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:05.860 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:05.919 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:06.114 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:06.363 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:06.799 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:07.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:46:06 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:46:06] "GET /metrics HTTP/1.1" 200 34164 "" "Prometheus/2.43.0" 2023-09-23T23:46:07.589 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:07 smithi110 ceph-mon[212939]: pgmap v2120: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:07.590 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:07 smithi110 ceph-mon[221102]: pgmap v2120: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:07.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:07 smithi138 ceph-mon[193606]: pgmap v2120: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:07.887 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:07.889 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:08.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:08 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:46:08.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:08 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:46:08.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:08 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:46:09.503 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:09 smithi110 ceph-mon[212939]: pgmap v2121: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:09.503 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:09 smithi110 ceph-mon[221102]: pgmap v2121: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:09.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:09 smithi138 ceph-mon[193606]: pgmap v2121: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:11.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:11 smithi110 ceph-mon[212939]: pgmap v2122: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:11.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:11 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:46:11.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:11 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:46:11.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:11 smithi110 ceph-mon[221102]: pgmap v2122: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:11.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:11 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:46:11.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:11 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:46:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:11 smithi138 ceph-mon[193606]: pgmap v2122: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:11 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:46:11.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:11 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:46:12.582 INFO:teuthology.orchestra.run.smithi110.stdout:3161095929874 2023-09-23T23:46:12.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T23:46:13.380 INFO:teuthology.orchestra.run.smithi110.stdout:197568496278 2023-09-23T23:46:13.380 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T23:46:13.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:13 smithi110 ceph-mon[212939]: pgmap v2123: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:13.703 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:13 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:46:13.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:13 smithi110 ceph-mon[221102]: pgmap v2123: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:13.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:13 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:46:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:13 smithi138 ceph-mon[193606]: pgmap v2123: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:13.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:13 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:46:14.197 INFO:teuthology.orchestra.run.smithi110.stdout:635655160232 2023-09-23T23:46:14.197 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T23:46:15.285 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:14 smithi110 ceph-mon[212939]: pgmap v2124: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:15.286 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:14 smithi110 ceph-mon[221102]: pgmap v2124: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:15.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:14 smithi138 ceph-mon[193606]: pgmap v2124: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:15.674 INFO:teuthology.orchestra.run.smithi110.stdout:481036337619 2023-09-23T23:46:15.675 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T23:46:16.144 INFO:teuthology.orchestra.run.smithi110.stdout:854698492264 2023-09-23T23:46:16.144 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T23:46:17.202 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:46:16 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:46:16] "GET /metrics HTTP/1.1" 200 34164 "" "Prometheus/2.43.0" 2023-09-23T23:46:17.250 INFO:teuthology.orchestra.run.smithi110.stdout:1473173782802 2023-09-23T23:46:17.250 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T23:46:17.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:17 smithi138 ceph-mon[193606]: pgmap v2125: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:17.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:17 smithi110 ceph-mon[212939]: pgmap v2125: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:17.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:17 smithi110 ceph-mon[221102]: pgmap v2125: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:18.247 INFO:teuthology.orchestra.run.smithi110.stdout:163208757915 2023-09-23T23:46:18.248 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T23:46:18.489 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:18.539 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:18.557 INFO:teuthology.orchestra.run.smithi110.stdout:390842024444 2023-09-23T23:46:18.558 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T23:46:18.644 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:18.760 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:19.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:19 smithi138 ceph-mon[193606]: pgmap v2126: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:19.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:19 smithi110 ceph-mon[212939]: pgmap v2126: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:19.703 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:19 smithi110 ceph-mon[221102]: pgmap v2126: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:20.139 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:21 smithi138 ceph-mon[193606]: pgmap v2127: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:21.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:21 smithi110 ceph-mon[212939]: pgmap v2127: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:21.702 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:21 smithi110 ceph-mon[221102]: pgmap v2127: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:21.778 INFO:teuthology.orchestra.run.smithi110.stdout:3161095929876 2023-09-23T23:46:22.576 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:22 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/292459228' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T23:46:22.576 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:22 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/292459228' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T23:46:22.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:22 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/292459228' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T23:46:23.060 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:23.100 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3161095929874 got 3161095929876 for osd.4 2023-09-23T23:46:23.100 DEBUG:teuthology.parallel:result is None 2023-09-23T23:46:23.273 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:23 smithi110 ceph-mon[212939]: pgmap v2128: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:23.273 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:23 smithi110 ceph-mon[221102]: pgmap v2128: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:23.547 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:23.583 INFO:teuthology.orchestra.run.smithi110.stdout:635655160234 2023-09-23T23:46:23.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:23 smithi138 ceph-mon[193606]: pgmap v2128: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:23.734 INFO:teuthology.orchestra.run.smithi110.stdout:854698492266 2023-09-23T23:46:24.576 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:24 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1585758497' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T23:46:24.576 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:24 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1150095156' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T23:46:24.577 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:24 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1585758497' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T23:46:24.577 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:24 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1150095156' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T23:46:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:24 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1585758497' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T23:46:24.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:24 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1150095156' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T23:46:24.745 INFO:teuthology.orchestra.run.smithi110.stdout:197568496280 2023-09-23T23:46:25.055 INFO:tasks.cephadm.ceph_manager.ceph:need seq 854698492264 got 854698492266 for osd.2 2023-09-23T23:46:25.055 DEBUG:teuthology.parallel:result is None 2023-09-23T23:46:25.262 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:25 smithi110 ceph-mon[212939]: pgmap v2129: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:25.262 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:25 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1677363142' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T23:46:25.263 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:25 smithi110 ceph-mon[221102]: pgmap v2129: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:25.263 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:25 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1677363142' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T23:46:25.275 INFO:tasks.cephadm.ceph_manager.ceph:need seq 635655160232 got 635655160234 for osd.3 2023-09-23T23:46:25.276 DEBUG:teuthology.parallel:result is None 2023-09-23T23:46:25.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:25 smithi138 ceph-mon[193606]: pgmap v2129: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:25.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:25 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1677363142' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T23:46:26.680 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568496278 got 197568496280 for osd.6 2023-09-23T23:46:26.680 DEBUG:teuthology.parallel:result is None 2023-09-23T23:46:26.891 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:27.001 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:46:26 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:46:26] "GET /metrics HTTP/1.1" 200 34164 "" "Prometheus/2.43.0" 2023-09-23T23:46:27.227 INFO:teuthology.orchestra.run.smithi110.stdout:481036337622 2023-09-23T23:46:27.257 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:27 smithi110 ceph-mon[212939]: pgmap v2130: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:27.257 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:27 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2210935818' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T23:46:27.257 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:27 smithi110 ceph-mon[221102]: pgmap v2130: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:27.257 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:27 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2210935818' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T23:46:27.465 INFO:teuthology.orchestra.run.smithi110.stdout:1473173782804 2023-09-23T23:46:27.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:27 smithi138 ceph-mon[193606]: pgmap v2130: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:27.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:27 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2210935818' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T23:46:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:28 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/2056060261' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T23:46:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:28 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:46:28.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:28 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/3058177500' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T23:46:28.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:28 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/2056060261' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T23:46:28.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:28 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:46:28.454 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:28 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/3058177500' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T23:46:28.618 INFO:teuthology.orchestra.run.smithi110.stdout:163208757917 2023-09-23T23:46:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:28 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/2056060261' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T23:46:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:28 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:46:28.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:28 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/3058177500' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T23:46:28.755 INFO:tasks.cephadm.ceph_manager.ceph:need seq 481036337619 got 481036337622 for osd.1 2023-09-23T23:46:28.756 DEBUG:teuthology.parallel:result is None 2023-09-23T23:46:29.187 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1473173782802 got 1473173782804 for osd.7 2023-09-23T23:46:29.187 DEBUG:teuthology.parallel:result is None 2023-09-23T23:46:29.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:29 smithi110 ceph-mon[212939]: pgmap v2131: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:29.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:29 smithi110 ceph-mon[221102]: pgmap v2131: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:29.472 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757915 got 163208757917 for osd.5 2023-09-23T23:46:29.472 DEBUG:teuthology.parallel:result is None 2023-09-23T23:46:29.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:29 smithi138 ceph-mon[193606]: pgmap v2131: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:31.087 INFO:teuthology.orchestra.run.smithi110.stdout:390842024446 2023-09-23T23:46:31.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:31 smithi110 ceph-mon[212939]: pgmap v2132: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:31.453 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:31 smithi110 ceph-mon[212939]: from='client.? 172.21.15.110:0/1578194152' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T23:46:31.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:31 smithi110 ceph-mon[221102]: pgmap v2132: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:31.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:31 smithi110 ceph-mon[221102]: from='client.? 172.21.15.110:0/1578194152' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T23:46:31.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:31 smithi138 ceph-mon[193606]: pgmap v2132: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:31.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:31 smithi138 ceph-mon[193606]: from='client.? 172.21.15.110:0/1578194152' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T23:46:31.984 INFO:tasks.cephadm.ceph_manager.ceph:need seq 390842024444 got 390842024446 for osd.0 2023-09-23T23:46:31.985 DEBUG:teuthology.parallel:result is None 2023-09-23T23:46:31.985 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T23:46:31.985 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:46:32.351 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:33.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:32 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:46:33.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:32 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:46:33.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:32 smithi110 ceph-mon[212939]: pgmap v2133: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:33.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:32 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:46:33.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:32 smithi110 ceph-mon[212939]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:46:33.203 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:32 smithi110 ceph-mon[212939]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:46:33.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:32 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:46:33.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:32 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:46:33.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:32 smithi110 ceph-mon[221102]: pgmap v2133: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:33.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:32 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:46:33.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:32 smithi110 ceph-mon[221102]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:46:33.204 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:32 smithi110 ceph-mon[221102]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:46:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:32 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:46:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:32 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:46:33.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:32 smithi138 ceph-mon[193606]: pgmap v2133: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:32 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:46:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:32 smithi138 ceph-mon[193606]: from='mgr.25208 172.21.15.110:0/408195058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:46:33.443 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:32 smithi138 ceph-mon[193606]: from='mgr.25208 ' entity='mgr.y' 2023-09-23T23:46:34.111 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:46:34.113 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:46:34.807 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":2133,"stamp":"2023-09-23T23:46:32.178096+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5994,"num_read_kb":13211,"num_write":907,"num_write_kb":12725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":149,"num_bytes_recovered":684618,"num_keys_recovered":30,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6838004,"kb_used_data":64308,"kb_used_omap":33,"kb_used_meta":6773598,"kb_avail":743123212,"statfs":{"total":767960285184,"available":760958169088,"internally_reserved":0,"allocated":65851392,"data_stored":49854189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34157,"internal_metadata":6936165011},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003007"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":922,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051419+0000","last_change":"2023-09-23T23:45:32.240149+0000","last_active":"2023-09-23T23:45:55.051419+0000","last_peered":"2023-09-23T23:45:55.051419+0000","last_clean":"2023-09-23T23:45:55.051419+0000","last_became_active":"2023-09-23T23:45:32.239216+0000","last_became_peered":"2023-09-23T23:45:32.239216+0000","last_unstale":"2023-09-23T23:45:55.051419+0000","last_undegraded":"2023-09-23T23:45:55.051419+0000","last_fullsized":"2023-09-23T23:45:55.051419+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:42.973963+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":43,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988722+0000","last_change":"2023-09-23T23:44:55.986638+0000","last_active":"2023-09-23T23:45:54.988722+0000","last_peered":"2023-09-23T23:45:54.988722+0000","last_clean":"2023-09-23T23:45:54.988722+0000","last_became_active":"2023-09-23T23:44:55.986291+0000","last_became_peered":"2023-09-23T23:44:55.986291+0000","last_unstale":"2023-09-23T23:45:54.988722+0000","last_undegraded":"2023-09-23T23:45:54.988722+0000","last_fullsized":"2023-09-23T23:45:54.988722+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:53.344884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":100,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:57.995142+0000","last_change":"2023-09-23T23:44:55.986672+0000","last_active":"2023-09-23T23:45:57.995142+0000","last_peered":"2023-09-23T23:45:57.995142+0000","last_clean":"2023-09-23T23:45:57.995142+0000","last_became_active":"2023-09-23T23:44:55.986200+0000","last_became_peered":"2023-09-23T23:44:55.986200+0000","last_unstale":"2023-09-23T23:45:57.995142+0000","last_undegraded":"2023-09-23T23:45:57.995142+0000","last_fullsized":"2023-09-23T23:45:57.995142+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:09.840032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":875,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052204+0000","last_change":"2023-09-23T23:45:32.240569+0000","last_active":"2023-09-23T23:45:55.052204+0000","last_peered":"2023-09-23T23:45:55.052204+0000","last_clean":"2023-09-23T23:45:55.052204+0000","last_became_active":"2023-09-23T23:45:32.240057+0000","last_became_peered":"2023-09-23T23:45:32.240057+0000","last_unstale":"2023-09-23T23:45:55.052204+0000","last_undegraded":"2023-09-23T23:45:55.052204+0000","last_fullsized":"2023-09-23T23:45:55.052204+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:21.618355+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":377,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.985203+0000","last_change":"2023-09-23T23:45:19.799761+0000","last_active":"2023-09-23T23:45:54.985203+0000","last_peered":"2023-09-23T23:45:54.985203+0000","last_clean":"2023-09-23T23:45:54.985203+0000","last_became_active":"2023-09-23T23:45:19.799456+0000","last_became_peered":"2023-09-23T23:45:19.799456+0000","last_unstale":"2023-09-23T23:45:54.985203+0000","last_undegraded":"2023-09-23T23:45:54.985203+0000","last_fullsized":"2023-09-23T23:45:54.985203+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:11.868474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":612,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052125+0000","last_change":"2023-09-23T23:45:22.811728+0000","last_active":"2023-09-23T23:45:55.052125+0000","last_peered":"2023-09-23T23:45:55.052125+0000","last_clean":"2023-09-23T23:45:55.052125+0000","last_became_active":"2023-09-23T23:45:22.811392+0000","last_became_peered":"2023-09-23T23:45:22.811392+0000","last_unstale":"2023-09-23T23:45:55.052125+0000","last_undegraded":"2023-09-23T23:45:55.052125+0000","last_fullsized":"2023-09-23T23:45:55.052125+0000","mapping_epoch":757,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":758,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.109854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":755,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988777+0000","last_change":"2023-09-23T23:44:59.944358+0000","last_active":"2023-09-23T23:45:54.988777+0000","last_peered":"2023-09-23T23:45:54.988777+0000","last_clean":"2023-09-23T23:45:54.988777+0000","last_became_active":"2023-09-23T23:44:58.998549+0000","last_became_peered":"2023-09-23T23:44:58.998549+0000","last_unstale":"2023-09-23T23:45:54.988777+0000","last_undegraded":"2023-09-23T23:45:54.988777+0000","last_fullsized":"2023-09-23T23:45:54.988777+0000","mapping_epoch":742,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":743,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:44:59.944277+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:44:59.944277+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:34:51.937588+0000","scrub_duration":0.0017553779999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":35,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051469+0000","last_change":"2023-09-23T23:45:08.975463+0000","last_active":"2023-09-23T23:45:55.051469+0000","last_peered":"2023-09-23T23:45:55.051469+0000","last_clean":"2023-09-23T23:45:55.051469+0000","last_became_active":"2023-09-23T23:45:08.975141+0000","last_became_peered":"2023-09-23T23:45:08.975141+0000","last_unstale":"2023-09-23T23:45:55.051469+0000","last_undegraded":"2023-09-23T23:45:55.051469+0000","last_fullsized":"2023-09-23T23:45:55.051469+0000","mapping_epoch":748,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":749,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:54.909147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":39,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.054055+0000","last_change":"2023-09-23T23:44:59.991084+0000","last_active":"2023-09-23T23:45:55.054055+0000","last_peered":"2023-09-23T23:45:55.054055+0000","last_clean":"2023-09-23T23:45:55.054055+0000","last_became_active":"2023-09-23T23:44:59.990783+0000","last_became_peered":"2023-09-23T23:44:59.990783+0000","last_unstale":"2023-09-23T23:45:55.054055+0000","last_undegraded":"2023-09-23T23:45:55.054055+0000","last_fullsized":"2023-09-23T23:45:55.054055+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:22.521680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":1088,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052362+0000","last_change":"2023-09-23T23:45:52.247423+0000","last_active":"2023-09-23T23:45:55.052362+0000","last_peered":"2023-09-23T23:45:55.052362+0000","last_clean":"2023-09-23T23:45:55.052362+0000","last_became_active":"2023-09-23T23:45:52.246596+0000","last_became_peered":"2023-09-23T23:45:52.246596+0000","last_unstale":"2023-09-23T23:45:55.052362+0000","last_undegraded":"2023-09-23T23:45:55.052362+0000","last_fullsized":"2023-09-23T23:45:55.052362+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:45:33.483204+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:45:33.483204+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:38.294415+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":388,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052479+0000","last_change":"2023-09-23T23:45:12.996691+0000","last_active":"2023-09-23T23:45:55.052479+0000","last_peered":"2023-09-23T23:45:55.052479+0000","last_clean":"2023-09-23T23:45:55.052479+0000","last_became_active":"2023-09-23T23:45:12.996313+0000","last_became_peered":"2023-09-23T23:45:12.996313+0000","last_unstale":"2023-09-23T23:45:55.052479+0000","last_undegraded":"2023-09-23T23:45:55.052479+0000","last_fullsized":"2023-09-23T23:45:55.052479+0000","mapping_epoch":752,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":753,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:45:41.498031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":109,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:57.994672+0000","last_change":"2023-09-23T23:44:59.986735+0000","last_active":"2023-09-23T23:45:57.994672+0000","last_peered":"2023-09-23T23:45:57.994672+0000","last_clean":"2023-09-23T23:45:57.994672+0000","last_became_active":"2023-09-23T23:44:59.986416+0000","last_became_peered":"2023-09-23T23:44:59.986416+0000","last_unstale":"2023-09-23T23:45:57.994672+0000","last_undegraded":"2023-09-23T23:45:57.994672+0000","last_fullsized":"2023-09-23T23:45:57.994672+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:37.959581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":1087,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.985797+0000","last_change":"2023-09-23T23:44:58.994659+0000","last_active":"2023-09-23T23:45:54.985797+0000","last_peered":"2023-09-23T23:45:54.985797+0000","last_clean":"2023-09-23T23:45:54.985797+0000","last_became_active":"2023-09-23T23:44:58.994260+0000","last_became_peered":"2023-09-23T23:44:58.994260+0000","last_unstale":"2023-09-23T23:45:54.985797+0000","last_undegraded":"2023-09-23T23:45:54.985797+0000","last_fullsized":"2023-09-23T23:45:54.985797+0000","mapping_epoch":742,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":743,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:55.907214+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":414,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.053172+0000","last_change":"2023-09-23T23:42:46.645764+0000","last_active":"2023-09-23T23:45:55.053172+0000","last_peered":"2023-09-23T23:45:55.053172+0000","last_clean":"2023-09-23T23:45:55.053172+0000","last_became_active":"2023-09-23T23:42:46.645462+0000","last_became_peered":"2023-09-23T23:42:46.645462+0000","last_unstale":"2023-09-23T23:45:55.053172+0000","last_undegraded":"2023-09-23T23:45:55.053172+0000","last_fullsized":"2023-09-23T23:45:55.053172+0000","mapping_epoch":676,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":677,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:04.694952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":100,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:57.994803+0000","last_change":"2023-09-23T23:44:59.987681+0000","last_active":"2023-09-23T23:45:57.994803+0000","last_peered":"2023-09-23T23:45:57.994803+0000","last_clean":"2023-09-23T23:45:57.994803+0000","last_became_active":"2023-09-23T23:44:59.987328+0000","last_became_peered":"2023-09-23T23:44:59.987328+0000","last_unstale":"2023-09-23T23:45:57.994803+0000","last_undegraded":"2023-09-23T23:45:57.994803+0000","last_fullsized":"2023-09-23T23:45:57.994803+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:11.777866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1237,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.053209+0000","last_change":"2023-09-23T23:45:32.240700+0000","last_active":"2023-09-23T23:45:55.053209+0000","last_peered":"2023-09-23T23:45:55.053209+0000","last_clean":"2023-09-23T23:45:55.053209+0000","last_became_active":"2023-09-23T23:45:32.240319+0000","last_became_peered":"2023-09-23T23:45:32.240319+0000","last_unstale":"2023-09-23T23:45:55.053209+0000","last_undegraded":"2023-09-23T23:45:55.053209+0000","last_fullsized":"2023-09-23T23:45:55.053209+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:22.146123+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":200,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051737+0000","last_change":"2023-09-23T23:45:52.245063+0000","last_active":"2023-09-23T23:45:55.051737+0000","last_peered":"2023-09-23T23:45:55.051737+0000","last_clean":"2023-09-23T23:45:55.051737+0000","last_became_active":"2023-09-23T23:45:52.244601+0000","last_became_peered":"2023-09-23T23:45:52.244601+0000","last_unstale":"2023-09-23T23:45:55.051737+0000","last_undegraded":"2023-09-23T23:45:55.051737+0000","last_fullsized":"2023-09-23T23:45:55.051737+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:12.350381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":445,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.986820+0000","last_change":"2023-09-23T23:45:19.799305+0000","last_active":"2023-09-23T23:45:54.986820+0000","last_peered":"2023-09-23T23:45:54.986820+0000","last_clean":"2023-09-23T23:45:54.986820+0000","last_became_active":"2023-09-23T23:45:19.798819+0000","last_became_peered":"2023-09-23T23:45:19.798819+0000","last_unstale":"2023-09-23T23:45:54.986820+0000","last_undegraded":"2023-09-23T23:45:54.986820+0000","last_fullsized":"2023-09-23T23:45:54.986820+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:48.039469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":694,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.986028+0000","last_change":"2023-09-23T23:45:19.799651+0000","last_active":"2023-09-23T23:45:54.986028+0000","last_peered":"2023-09-23T23:45:54.986028+0000","last_clean":"2023-09-23T23:45:54.986028+0000","last_became_active":"2023-09-23T23:45:19.799301+0000","last_became_peered":"2023-09-23T23:45:19.799301+0000","last_unstale":"2023-09-23T23:45:54.986028+0000","last_undegraded":"2023-09-23T23:45:54.986028+0000","last_fullsized":"2023-09-23T23:45:54.986028+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:45:14.789802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:45:14.789802+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:17.255907+0000","scrub_duration":0.0018535310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":73,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988906+0000","last_change":"2023-09-23T23:45:00.005548+0000","last_active":"2023-09-23T23:45:54.988906+0000","last_peered":"2023-09-23T23:45:54.988906+0000","last_clean":"2023-09-23T23:45:54.988906+0000","last_became_active":"2023-09-23T23:44:59.992651+0000","last_became_peered":"2023-09-23T23:44:59.992651+0000","last_unstale":"2023-09-23T23:45:54.988906+0000","last_undegraded":"2023-09-23T23:45:54.988906+0000","last_fullsized":"2023-09-23T23:45:54.988906+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:01.163903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":53,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988934+0000","last_change":"2023-09-23T23:44:59.994663+0000","last_active":"2023-09-23T23:45:54.988934+0000","last_peered":"2023-09-23T23:45:54.988934+0000","last_clean":"2023-09-23T23:45:54.988934+0000","last_became_active":"2023-09-23T23:44:59.994425+0000","last_became_peered":"2023-09-23T23:44:59.994425+0000","last_unstale":"2023-09-23T23:45:54.988934+0000","last_undegraded":"2023-09-23T23:45:54.988934+0000","last_fullsized":"2023-09-23T23:45:54.988934+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:22.831850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":1001,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051858+0000","last_change":"2023-09-23T23:45:12.996741+0000","last_active":"2023-09-23T23:45:55.051858+0000","last_peered":"2023-09-23T23:45:55.051858+0000","last_clean":"2023-09-23T23:45:55.051858+0000","last_became_active":"2023-09-23T23:45:12.996317+0000","last_became_peered":"2023-09-23T23:45:12.996317+0000","last_unstale":"2023-09-23T23:45:55.051858+0000","last_undegraded":"2023-09-23T23:45:55.051858+0000","last_fullsized":"2023-09-23T23:45:55.051858+0000","mapping_epoch":752,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":753,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:19.589426+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":527,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051789+0000","last_change":"2023-09-23T23:45:52.245725+0000","last_active":"2023-09-23T23:45:55.051789+0000","last_peered":"2023-09-23T23:45:55.051789+0000","last_clean":"2023-09-23T23:45:55.051789+0000","last_became_active":"2023-09-23T23:45:52.245190+0000","last_became_peered":"2023-09-23T23:45:52.245190+0000","last_unstale":"2023-09-23T23:45:55.051789+0000","last_undegraded":"2023-09-23T23:45:55.051789+0000","last_fullsized":"2023-09-23T23:45:55.051789+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:12.914713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":91,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:57.995037+0000","last_change":"2023-09-23T23:43:37.337600+0000","last_active":"2023-09-23T23:45:57.995037+0000","last_peered":"2023-09-23T23:45:57.995037+0000","last_clean":"2023-09-23T23:45:57.995037+0000","last_became_active":"2023-09-23T23:43:37.337098+0000","last_became_peered":"2023-09-23T23:43:37.337098+0000","last_unstale":"2023-09-23T23:45:57.995037+0000","last_undegraded":"2023-09-23T23:45:57.995037+0000","last_fullsized":"2023-09-23T23:45:57.995037+0000","mapping_epoch":692,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":693,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:40.723292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":876,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051296+0000","last_change":"2023-09-23T23:45:32.239630+0000","last_active":"2023-09-23T23:45:55.051296+0000","last_peered":"2023-09-23T23:45:55.051296+0000","last_clean":"2023-09-23T23:45:55.051296+0000","last_became_active":"2023-09-23T23:45:32.238728+0000","last_became_peered":"2023-09-23T23:45:32.238728+0000","last_unstale":"2023-09-23T23:45:55.051296+0000","last_undegraded":"2023-09-23T23:45:55.051296+0000","last_fullsized":"2023-09-23T23:45:55.051296+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:23.346950+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":240,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051694+0000","last_change":"2023-09-23T23:45:22.812226+0000","last_active":"2023-09-23T23:45:55.051694+0000","last_peered":"2023-09-23T23:45:55.051694+0000","last_clean":"2023-09-23T23:45:55.051694+0000","last_became_active":"2023-09-23T23:45:22.811926+0000","last_became_peered":"2023-09-23T23:45:22.811926+0000","last_unstale":"2023-09-23T23:45:55.051694+0000","last_undegraded":"2023-09-23T23:45:55.051694+0000","last_fullsized":"2023-09-23T23:45:55.051694+0000","mapping_epoch":757,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":758,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:17:43.890183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":321,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051511+0000","last_change":"2023-09-23T23:45:32.242077+0000","last_active":"2023-09-23T23:45:55.051511+0000","last_peered":"2023-09-23T23:45:55.051511+0000","last_clean":"2023-09-23T23:45:55.051511+0000","last_became_active":"2023-09-23T23:45:32.240435+0000","last_became_peered":"2023-09-23T23:45:32.240435+0000","last_unstale":"2023-09-23T23:45:55.051511+0000","last_undegraded":"2023-09-23T23:45:55.051511+0000","last_fullsized":"2023-09-23T23:45:55.051511+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:20.255228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":814,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.985773+0000","last_change":"2023-09-23T23:45:25.984337+0000","last_active":"2023-09-23T23:45:54.985773+0000","last_peered":"2023-09-23T23:45:54.985773+0000","last_clean":"2023-09-23T23:45:54.985773+0000","last_became_active":"2023-09-23T23:45:25.984028+0000","last_became_peered":"2023-09-23T23:45:25.984028+0000","last_unstale":"2023-09-23T23:45:54.985773+0000","last_undegraded":"2023-09-23T23:45:54.985773+0000","last_fullsized":"2023-09-23T23:45:54.985773+0000","mapping_epoch":760,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":761,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:44.207154+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":528,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052489+0000","last_change":"2023-09-23T23:45:52.242787+0000","last_active":"2023-09-23T23:45:55.052489+0000","last_peered":"2023-09-23T23:45:55.052489+0000","last_clean":"2023-09-23T23:45:55.052489+0000","last_became_active":"2023-09-23T23:45:52.242370+0000","last_became_peered":"2023-09-23T23:45:52.242370+0000","last_unstale":"2023-09-23T23:45:55.052489+0000","last_undegraded":"2023-09-23T23:45:55.052489+0000","last_fullsized":"2023-09-23T23:45:55.052489+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:45:25.958298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":107,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051478+0000","last_change":"2023-09-23T23:45:12.996789+0000","last_active":"2023-09-23T23:45:55.051478+0000","last_peered":"2023-09-23T23:45:55.051478+0000","last_clean":"2023-09-23T23:45:55.051478+0000","last_became_active":"2023-09-23T23:45:12.996316+0000","last_became_peered":"2023-09-23T23:45:12.996316+0000","last_unstale":"2023-09-23T23:45:55.051478+0000","last_undegraded":"2023-09-23T23:45:55.051478+0000","last_fullsized":"2023-09-23T23:45:55.051478+0000","mapping_epoch":752,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":753,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:18.402324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":1216,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051683+0000","last_change":"2023-09-23T23:45:03.033754+0000","last_active":"2023-09-23T23:45:55.051683+0000","last_peered":"2023-09-23T23:45:55.051683+0000","last_clean":"2023-09-23T23:45:55.051683+0000","last_became_active":"2023-09-23T23:45:03.033401+0000","last_became_peered":"2023-09-23T23:45:03.033401+0000","last_unstale":"2023-09-23T23:45:55.051683+0000","last_undegraded":"2023-09-23T23:45:55.051683+0000","last_fullsized":"2023-09-23T23:45:55.051683+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:27.436501+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":69,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.053466+0000","last_change":"2023-09-23T23:45:08.995797+0000","last_active":"2023-09-23T23:45:55.053466+0000","last_peered":"2023-09-23T23:45:55.053466+0000","last_clean":"2023-09-23T23:45:55.053466+0000","last_became_active":"2023-09-23T23:45:08.982500+0000","last_became_peered":"2023-09-23T23:45:08.982500+0000","last_unstale":"2023-09-23T23:45:55.053466+0000","last_undegraded":"2023-09-23T23:45:55.053466+0000","last_fullsized":"2023-09-23T23:45:55.053466+0000","mapping_epoch":748,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":749,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:56.451333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":26,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052783+0000","last_change":"2023-09-23T23:45:22.812030+0000","last_active":"2023-09-23T23:45:55.052783+0000","last_peered":"2023-09-23T23:45:55.052783+0000","last_clean":"2023-09-23T23:45:55.052783+0000","last_became_active":"2023-09-23T23:45:22.811724+0000","last_became_peered":"2023-09-23T23:45:22.811724+0000","last_unstale":"2023-09-23T23:45:55.052783+0000","last_undegraded":"2023-09-23T23:45:55.052783+0000","last_fullsized":"2023-09-23T23:45:55.052783+0000","mapping_epoch":757,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":758,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:19:17.650196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":926,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051854+0000","last_change":"2023-09-23T23:45:52.243550+0000","last_active":"2023-09-23T23:45:55.051854+0000","last_peered":"2023-09-23T23:45:55.051854+0000","last_clean":"2023-09-23T23:45:55.051854+0000","last_became_active":"2023-09-23T23:45:52.242990+0000","last_became_peered":"2023-09-23T23:45:52.242990+0000","last_unstale":"2023-09-23T23:45:55.051854+0000","last_undegraded":"2023-09-23T23:45:55.051854+0000","last_fullsized":"2023-09-23T23:45:55.051854+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:15.663145+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":375,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052855+0000","last_change":"2023-09-23T23:45:52.243970+0000","last_active":"2023-09-23T23:45:55.052855+0000","last_peered":"2023-09-23T23:45:55.052855+0000","last_clean":"2023-09-23T23:45:55.052855+0000","last_became_active":"2023-09-23T23:45:52.243703+0000","last_became_peered":"2023-09-23T23:45:52.243703+0000","last_unstale":"2023-09-23T23:45:55.052855+0000","last_undegraded":"2023-09-23T23:45:55.052855+0000","last_fullsized":"2023-09-23T23:45:55.052855+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:23.366439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":387,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051363+0000","last_change":"2023-09-23T23:45:32.239212+0000","last_active":"2023-09-23T23:45:55.051363+0000","last_peered":"2023-09-23T23:45:55.051363+0000","last_clean":"2023-09-23T23:45:55.051363+0000","last_became_active":"2023-09-23T23:45:32.238690+0000","last_became_peered":"2023-09-23T23:45:32.238690+0000","last_unstale":"2023-09-23T23:45:55.051363+0000","last_undegraded":"2023-09-23T23:45:55.051363+0000","last_fullsized":"2023-09-23T23:45:55.051363+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:14.425469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":799,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:57.995011+0000","last_change":"2023-09-23T23:45:03.038443+0000","last_active":"2023-09-23T23:45:57.995011+0000","last_peered":"2023-09-23T23:45:57.995011+0000","last_clean":"2023-09-23T23:45:57.995011+0000","last_became_active":"2023-09-23T23:45:03.038076+0000","last_became_peered":"2023-09-23T23:45:03.038076+0000","last_unstale":"2023-09-23T23:45:57.995011+0000","last_undegraded":"2023-09-23T23:45:57.995011+0000","last_fullsized":"2023-09-23T23:45:57.995011+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:43:40.189087+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:43:40.189087+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:51:18.436998+0000","scrub_duration":0.001887503,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":425,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052713+0000","last_change":"2023-09-23T23:45:52.243030+0000","last_active":"2023-09-23T23:45:55.052713+0000","last_peered":"2023-09-23T23:45:55.052713+0000","last_clean":"2023-09-23T23:45:55.052713+0000","last_became_active":"2023-09-23T23:45:52.242318+0000","last_became_peered":"2023-09-23T23:45:52.242318+0000","last_unstale":"2023-09-23T23:45:55.052713+0000","last_undegraded":"2023-09-23T23:45:55.052713+0000","last_fullsized":"2023-09-23T23:45:55.052713+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:48:15.424032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":63,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988447+0000","last_change":"2023-09-23T23:45:22.814032+0000","last_active":"2023-09-23T23:45:54.988447+0000","last_peered":"2023-09-23T23:45:54.988447+0000","last_clean":"2023-09-23T23:45:54.988447+0000","last_became_active":"2023-09-23T23:45:22.813596+0000","last_became_peered":"2023-09-23T23:45:22.813596+0000","last_unstale":"2023-09-23T23:45:54.988447+0000","last_undegraded":"2023-09-23T23:45:54.988447+0000","last_fullsized":"2023-09-23T23:45:54.988447+0000","mapping_epoch":757,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":758,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:04.270071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":845,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.054146+0000","last_change":"2023-09-23T23:45:03.041640+0000","last_active":"2023-09-23T23:45:55.054146+0000","last_peered":"2023-09-23T23:45:55.054146+0000","last_clean":"2023-09-23T23:45:55.054146+0000","last_became_active":"2023-09-23T23:45:03.040104+0000","last_became_peered":"2023-09-23T23:45:03.040104+0000","last_unstale":"2023-09-23T23:45:55.054146+0000","last_undegraded":"2023-09-23T23:45:55.054146+0000","last_fullsized":"2023-09-23T23:45:55.054146+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:35.920724+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":499,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052121+0000","last_change":"2023-09-23T23:45:52.247754+0000","last_active":"2023-09-23T23:45:55.052121+0000","last_peered":"2023-09-23T23:45:55.052121+0000","last_clean":"2023-09-23T23:45:55.052121+0000","last_became_active":"2023-09-23T23:45:52.247004+0000","last_became_peered":"2023-09-23T23:45:52.247004+0000","last_unstale":"2023-09-23T23:45:55.052121+0000","last_undegraded":"2023-09-23T23:45:55.052121+0000","last_fullsized":"2023-09-23T23:45:55.052121+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:39:41.941465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":39,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.053541+0000","last_change":"2023-09-23T23:44:59.996784+0000","last_active":"2023-09-23T23:45:55.053541+0000","last_peered":"2023-09-23T23:45:55.053541+0000","last_clean":"2023-09-23T23:45:55.053541+0000","last_became_active":"2023-09-23T23:44:59.996229+0000","last_became_peered":"2023-09-23T23:44:59.996229+0000","last_unstale":"2023-09-23T23:45:55.053541+0000","last_undegraded":"2023-09-23T23:45:55.053541+0000","last_fullsized":"2023-09-23T23:45:55.053541+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:52.941708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":533,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988976+0000","last_change":"2023-09-23T23:44:59.000492+0000","last_active":"2023-09-23T23:45:54.988976+0000","last_peered":"2023-09-23T23:45:54.988976+0000","last_clean":"2023-09-23T23:45:54.988976+0000","last_became_active":"2023-09-23T23:44:59.000157+0000","last_became_peered":"2023-09-23T23:44:59.000157+0000","last_unstale":"2023-09-23T23:45:54.988976+0000","last_undegraded":"2023-09-23T23:45:54.988976+0000","last_fullsized":"2023-09-23T23:45:54.988976+0000","mapping_epoch":742,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":743,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:43:42.149317+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:43:42.149317+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:06.104330+0000","scrub_duration":0.0023427560000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":513,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051521+0000","last_change":"2023-09-23T23:45:32.240799+0000","last_active":"2023-09-23T23:45:55.051521+0000","last_peered":"2023-09-23T23:45:55.051521+0000","last_clean":"2023-09-23T23:45:55.051521+0000","last_became_active":"2023-09-23T23:45:32.239349+0000","last_became_peered":"2023-09-23T23:45:32.239349+0000","last_unstale":"2023-09-23T23:45:55.051521+0000","last_undegraded":"2023-09-23T23:45:55.051521+0000","last_fullsized":"2023-09-23T23:45:55.051521+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:33.985138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":800,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052286+0000","last_change":"2023-09-23T23:44:55.985533+0000","last_active":"2023-09-23T23:45:55.052286+0000","last_peered":"2023-09-23T23:45:55.052286+0000","last_clean":"2023-09-23T23:45:55.052286+0000","last_became_active":"2023-09-23T23:44:55.985154+0000","last_became_peered":"2023-09-23T23:44:55.985154+0000","last_unstale":"2023-09-23T23:45:55.052286+0000","last_undegraded":"2023-09-23T23:45:55.052286+0000","last_fullsized":"2023-09-23T23:45:55.052286+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:31.396248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":914,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052540+0000","last_change":"2023-09-23T23:45:52.245509+0000","last_active":"2023-09-23T23:45:55.052540+0000","last_peered":"2023-09-23T23:45:55.052540+0000","last_clean":"2023-09-23T23:45:55.052540+0000","last_became_active":"2023-09-23T23:45:52.245064+0000","last_became_peered":"2023-09-23T23:45:52.245064+0000","last_unstale":"2023-09-23T23:45:55.052540+0000","last_undegraded":"2023-09-23T23:45:55.052540+0000","last_fullsized":"2023-09-23T23:45:55.052540+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:43:27.592049+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:43:27.592049+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:17.167970+0000","scrub_duration":0.001659259,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":134,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051834+0000","last_change":"2023-09-23T23:45:08.989599+0000","last_active":"2023-09-23T23:45:55.051834+0000","last_peered":"2023-09-23T23:45:55.051834+0000","last_clean":"2023-09-23T23:45:55.051834+0000","last_became_active":"2023-09-23T23:45:08.981090+0000","last_became_peered":"2023-09-23T23:45:08.981090+0000","last_unstale":"2023-09-23T23:45:55.051834+0000","last_undegraded":"2023-09-23T23:45:55.051834+0000","last_fullsized":"2023-09-23T23:45:55.051834+0000","mapping_epoch":748,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":749,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:13.105965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":53,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988886+0000","last_change":"2023-09-23T23:44:59.997689+0000","last_active":"2023-09-23T23:45:54.988886+0000","last_peered":"2023-09-23T23:45:54.988886+0000","last_clean":"2023-09-23T23:45:54.988886+0000","last_became_active":"2023-09-23T23:44:59.997370+0000","last_became_peered":"2023-09-23T23:44:59.997370+0000","last_unstale":"2023-09-23T23:45:54.988886+0000","last_undegraded":"2023-09-23T23:45:54.988886+0000","last_fullsized":"2023-09-23T23:45:54.988886+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:46:54.721468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":684,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051252+0000","last_change":"2023-09-23T23:45:32.241274+0000","last_active":"2023-09-23T23:45:55.051252+0000","last_peered":"2023-09-23T23:45:55.051252+0000","last_clean":"2023-09-23T23:45:55.051252+0000","last_became_active":"2023-09-23T23:45:32.240417+0000","last_became_peered":"2023-09-23T23:45:32.240417+0000","last_unstale":"2023-09-23T23:45:55.051252+0000","last_undegraded":"2023-09-23T23:45:55.051252+0000","last_fullsized":"2023-09-23T23:45:55.051252+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:44.056296+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":39,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.053782+0000","last_change":"2023-09-23T23:44:59.988998+0000","last_active":"2023-09-23T23:45:55.053782+0000","last_peered":"2023-09-23T23:45:55.053782+0000","last_clean":"2023-09-23T23:45:55.053782+0000","last_became_active":"2023-09-23T23:44:59.988635+0000","last_became_peered":"2023-09-23T23:44:59.988635+0000","last_unstale":"2023-09-23T23:45:55.053782+0000","last_undegraded":"2023-09-23T23:45:55.053782+0000","last_fullsized":"2023-09-23T23:45:55.053782+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:06:39.231374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":43,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.989121+0000","last_change":"2023-09-23T23:44:55.990783+0000","last_active":"2023-09-23T23:45:54.989121+0000","last_peered":"2023-09-23T23:45:54.989121+0000","last_clean":"2023-09-23T23:45:54.989121+0000","last_became_active":"2023-09-23T23:44:55.990445+0000","last_became_peered":"2023-09-23T23:44:55.990445+0000","last_unstale":"2023-09-23T23:45:54.989121+0000","last_undegraded":"2023-09-23T23:45:54.989121+0000","last_fullsized":"2023-09-23T23:45:54.989121+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:21.475623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":819,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:58.452852+0000","last_change":"2023-09-23T23:45:19.800745+0000","last_active":"2023-09-23T23:45:58.452852+0000","last_peered":"2023-09-23T23:45:58.452852+0000","last_clean":"2023-09-23T23:45:58.452852+0000","last_became_active":"2023-09-23T23:45:19.800473+0000","last_became_peered":"2023-09-23T23:45:19.800473+0000","last_unstale":"2023-09-23T23:45:58.452852+0000","last_undegraded":"2023-09-23T23:45:58.452852+0000","last_fullsized":"2023-09-23T23:45:58.452852+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:45.672099+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":43,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988533+0000","last_change":"2023-09-23T23:44:56.001669+0000","last_active":"2023-09-23T23:45:54.988533+0000","last_peered":"2023-09-23T23:45:54.988533+0000","last_clean":"2023-09-23T23:45:54.988533+0000","last_became_active":"2023-09-23T23:44:56.001107+0000","last_became_peered":"2023-09-23T23:44:56.001107+0000","last_unstale":"2023-09-23T23:45:54.988533+0000","last_undegraded":"2023-09-23T23:45:54.988533+0000","last_fullsized":"2023-09-23T23:45:54.988533+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:01.851610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":53,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.989213+0000","last_change":"2023-09-23T23:44:59.995182+0000","last_active":"2023-09-23T23:45:54.989213+0000","last_peered":"2023-09-23T23:45:54.989213+0000","last_clean":"2023-09-23T23:45:54.989213+0000","last_became_active":"2023-09-23T23:44:59.994869+0000","last_became_peered":"2023-09-23T23:44:59.994869+0000","last_unstale":"2023-09-23T23:45:54.989213+0000","last_undegraded":"2023-09-23T23:45:54.989213+0000","last_fullsized":"2023-09-23T23:45:54.989213+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:35.582441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":625,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051589+0000","last_change":"2023-09-23T23:45:32.526297+0000","last_active":"2023-09-23T23:45:55.051589+0000","last_peered":"2023-09-23T23:45:55.051589+0000","last_clean":"2023-09-23T23:45:55.051589+0000","last_became_active":"2023-09-23T23:45:32.240832+0000","last_became_peered":"2023-09-23T23:45:32.240832+0000","last_unstale":"2023-09-23T23:45:55.051589+0000","last_undegraded":"2023-09-23T23:45:55.051589+0000","last_fullsized":"2023-09-23T23:45:55.051589+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:45:32.526219+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:45:32.526219+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:42:01.551486+0000","scrub_duration":0.001811336,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":26,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051300+0000","last_change":"2023-09-23T23:45:22.812891+0000","last_active":"2023-09-23T23:45:55.051300+0000","last_peered":"2023-09-23T23:45:55.051300+0000","last_clean":"2023-09-23T23:45:55.051300+0000","last_became_active":"2023-09-23T23:45:22.812026+0000","last_became_peered":"2023-09-23T23:45:22.812026+0000","last_unstale":"2023-09-23T23:45:55.051300+0000","last_undegraded":"2023-09-23T23:45:55.051300+0000","last_fullsized":"2023-09-23T23:45:55.051300+0000","mapping_epoch":757,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":758,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:39:57.403005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":542,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.986311+0000","last_change":"2023-09-23T23:45:19.799247+0000","last_active":"2023-09-23T23:45:54.986311+0000","last_peered":"2023-09-23T23:45:54.986311+0000","last_clean":"2023-09-23T23:45:54.986311+0000","last_became_active":"2023-09-23T23:45:19.798967+0000","last_became_peered":"2023-09-23T23:45:19.798967+0000","last_unstale":"2023-09-23T23:45:54.986311+0000","last_undegraded":"2023-09-23T23:45:54.986311+0000","last_fullsized":"2023-09-23T23:45:54.986311+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:01.817851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":953,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.054204+0000","last_change":"2023-09-23T23:45:25.990842+0000","last_active":"2023-09-23T23:45:55.054204+0000","last_peered":"2023-09-23T23:45:55.054204+0000","last_clean":"2023-09-23T23:45:55.054204+0000","last_became_active":"2023-09-23T23:45:25.990381+0000","last_became_peered":"2023-09-23T23:45:25.990381+0000","last_unstale":"2023-09-23T23:45:55.054204+0000","last_undegraded":"2023-09-23T23:45:55.054204+0000","last_fullsized":"2023-09-23T23:45:55.054204+0000","mapping_epoch":760,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":761,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:21.240299+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"716'77","reported_seq":786,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.986776+0000","last_change":"2023-09-23T23:45:19.801381+0000","last_active":"2023-09-23T23:45:54.986776+0000","last_peered":"2023-09-23T23:45:54.986776+0000","last_clean":"2023-09-23T23:45:54.986776+0000","last_became_active":"2023-09-23T23:45:19.800975+0000","last_became_peered":"2023-09-23T23:45:19.800975+0000","last_unstale":"2023-09-23T23:45:54.986776+0000","last_undegraded":"2023-09-23T23:45:54.986776+0000","last_fullsized":"2023-09-23T23:45:54.986776+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":77,"log_dups_size":0,"ondisk_log_size":77,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:20:28.335306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033286999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":355,"num_read_kb":483,"num_write":81,"num_write_kb":122,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":761,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.053076+0000","last_change":"2023-09-23T23:45:32.243261+0000","last_active":"2023-09-23T23:45:55.053076+0000","last_peered":"2023-09-23T23:45:55.053076+0000","last_clean":"2023-09-23T23:45:55.053076+0000","last_became_active":"2023-09-23T23:45:32.242919+0000","last_became_peered":"2023-09-23T23:45:32.242919+0000","last_unstale":"2023-09-23T23:45:55.053076+0000","last_undegraded":"2023-09-23T23:45:55.053076+0000","last_fullsized":"2023-09-23T23:45:55.053076+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:20.140239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":386,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.986735+0000","last_change":"2023-09-23T23:45:19.801084+0000","last_active":"2023-09-23T23:45:54.986735+0000","last_peered":"2023-09-23T23:45:54.986735+0000","last_clean":"2023-09-23T23:45:54.986735+0000","last_became_active":"2023-09-23T23:45:19.800739+0000","last_became_peered":"2023-09-23T23:45:19.800739+0000","last_unstale":"2023-09-23T23:45:54.986735+0000","last_undegraded":"2023-09-23T23:45:54.986735+0000","last_fullsized":"2023-09-23T23:45:54.986735+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:49.105815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":625,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051422+0000","last_change":"2023-09-23T23:45:35.481801+0000","last_active":"2023-09-23T23:45:55.051422+0000","last_peered":"2023-09-23T23:45:55.051422+0000","last_clean":"2023-09-23T23:45:55.051422+0000","last_became_active":"2023-09-23T23:45:32.239773+0000","last_became_peered":"2023-09-23T23:45:32.239773+0000","last_unstale":"2023-09-23T23:45:55.051422+0000","last_undegraded":"2023-09-23T23:45:55.051422+0000","last_fullsized":"2023-09-23T23:45:55.051422+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:45:35.481699+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:45:35.481699+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:59.093100+0000","scrub_duration":0.0020465320000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"715'48","reported_seq":795,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051485+0000","last_change":"2023-09-23T23:45:32.241569+0000","last_active":"2023-09-23T23:45:55.051485+0000","last_peered":"2023-09-23T23:45:55.051485+0000","last_clean":"2023-09-23T23:45:55.051485+0000","last_became_active":"2023-09-23T23:45:32.240975+0000","last_became_peered":"2023-09-23T23:45:32.240975+0000","last_unstale":"2023-09-23T23:45:55.051485+0000","last_undegraded":"2023-09-23T23:45:55.051485+0000","last_fullsized":"2023-09-23T23:45:55.051485+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":48,"log_dups_size":0,"ondisk_log_size":48,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:48.198846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039284299999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":440,"num_read_kb":682,"num_write":42,"num_write_kb":162,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":53,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.989118+0000","last_change":"2023-09-23T23:44:59.991849+0000","last_active":"2023-09-23T23:45:54.989118+0000","last_peered":"2023-09-23T23:45:54.989118+0000","last_clean":"2023-09-23T23:45:54.989118+0000","last_became_active":"2023-09-23T23:44:59.991523+0000","last_became_peered":"2023-09-23T23:44:59.991523+0000","last_unstale":"2023-09-23T23:45:54.989118+0000","last_undegraded":"2023-09-23T23:45:54.989118+0000","last_fullsized":"2023-09-23T23:45:54.989118+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:48:44.930913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":367,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.986580+0000","last_change":"2023-09-23T23:45:19.800908+0000","last_active":"2023-09-23T23:45:54.986580+0000","last_peered":"2023-09-23T23:45:54.986580+0000","last_clean":"2023-09-23T23:45:54.986580+0000","last_became_active":"2023-09-23T23:45:19.800226+0000","last_became_peered":"2023-09-23T23:45:19.800226+0000","last_unstale":"2023-09-23T23:45:54.986580+0000","last_undegraded":"2023-09-23T23:45:54.986580+0000","last_fullsized":"2023-09-23T23:45:54.986580+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.081896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":1096,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.985575+0000","last_change":"2023-09-23T23:44:58.990040+0000","last_active":"2023-09-23T23:45:54.985575+0000","last_peered":"2023-09-23T23:45:54.985575+0000","last_clean":"2023-09-23T23:45:54.985575+0000","last_became_active":"2023-09-23T23:44:58.989206+0000","last_became_peered":"2023-09-23T23:44:58.989206+0000","last_unstale":"2023-09-23T23:45:54.985575+0000","last_undegraded":"2023-09-23T23:45:54.985575+0000","last_fullsized":"2023-09-23T23:45:54.985575+0000","mapping_epoch":742,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":743,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:39.172287+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"716'117","reported_seq":871,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.985345+0000","last_change":"2023-09-23T23:44:04.966130+0000","last_active":"2023-09-23T23:45:54.985345+0000","last_peered":"2023-09-23T23:45:54.985345+0000","last_clean":"2023-09-23T23:45:54.985345+0000","last_became_active":"2023-09-23T23:43:37.337816+0000","last_became_peered":"2023-09-23T23:43:37.337816+0000","last_unstale":"2023-09-23T23:45:54.985345+0000","last_undegraded":"2023-09-23T23:45:54.985345+0000","last_fullsized":"2023-09-23T23:45:54.985345+0000","mapping_epoch":692,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":693,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:42.250477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092634300000000004,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":558,"num_read_kb":1548,"num_write":124,"num_write_kb":5249,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":434,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.985317+0000","last_change":"2023-09-23T23:45:19.799510+0000","last_active":"2023-09-23T23:45:54.985317+0000","last_peered":"2023-09-23T23:45:54.985317+0000","last_clean":"2023-09-23T23:45:54.985317+0000","last_became_active":"2023-09-23T23:45:19.798822+0000","last_became_peered":"2023-09-23T23:45:19.798822+0000","last_unstale":"2023-09-23T23:45:54.985317+0000","last_undegraded":"2023-09-23T23:45:54.985317+0000","last_fullsized":"2023-09-23T23:45:54.985317+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:42:21.032363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":138,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.986239+0000","last_change":"2023-09-23T23:45:19.801143+0000","last_active":"2023-09-23T23:45:54.986239+0000","last_peered":"2023-09-23T23:45:54.986239+0000","last_clean":"2023-09-23T23:45:54.986239+0000","last_became_active":"2023-09-23T23:45:19.800221+0000","last_became_peered":"2023-09-23T23:45:19.800221+0000","last_unstale":"2023-09-23T23:45:54.986239+0000","last_undegraded":"2023-09-23T23:45:54.986239+0000","last_fullsized":"2023-09-23T23:45:54.986239+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:29.306589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"716'220","reported_seq":1532,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052257+0000","last_change":"2023-09-23T23:45:32.242090+0000","last_active":"2023-09-23T23:45:55.052257+0000","last_peered":"2023-09-23T23:45:55.052257+0000","last_clean":"2023-09-23T23:45:55.052257+0000","last_became_active":"2023-09-23T23:45:32.241092+0000","last_became_peered":"2023-09-23T23:45:32.241092+0000","last_unstale":"2023-09-23T23:45:55.052257+0000","last_undegraded":"2023-09-23T23:45:55.052257+0000","last_fullsized":"2023-09-23T23:45:55.052257+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":220,"log_dups_size":0,"ondisk_log_size":220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:42.511039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035346800000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1240,"num_read_kb":2147,"num_write":187,"num_write_kb":1152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":1061,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.053009+0000","last_change":"2023-09-23T23:45:52.244147+0000","last_active":"2023-09-23T23:45:55.053009+0000","last_peered":"2023-09-23T23:45:55.053009+0000","last_clean":"2023-09-23T23:45:55.053009+0000","last_became_active":"2023-09-23T23:45:52.243102+0000","last_became_peered":"2023-09-23T23:45:52.243102+0000","last_unstale":"2023-09-23T23:45:55.053009+0000","last_undegraded":"2023-09-23T23:45:55.053009+0000","last_fullsized":"2023-09-23T23:45:55.053009+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:56.522753+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":109,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051486+0000","last_change":"2023-09-23T23:44:59.989992+0000","last_active":"2023-09-23T23:45:55.051486+0000","last_peered":"2023-09-23T23:45:55.051486+0000","last_clean":"2023-09-23T23:45:55.051486+0000","last_became_active":"2023-09-23T23:44:59.989677+0000","last_became_peered":"2023-09-23T23:44:59.989677+0000","last_unstale":"2023-09-23T23:45:55.051486+0000","last_undegraded":"2023-09-23T23:45:55.051486+0000","last_fullsized":"2023-09-23T23:45:55.051486+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:05.406361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":39,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.054050+0000","last_change":"2023-09-23T23:44:59.995791+0000","last_active":"2023-09-23T23:45:55.054050+0000","last_peered":"2023-09-23T23:45:55.054050+0000","last_clean":"2023-09-23T23:45:55.054050+0000","last_became_active":"2023-09-23T23:44:59.995498+0000","last_became_peered":"2023-09-23T23:44:59.995498+0000","last_unstale":"2023-09-23T23:45:55.054050+0000","last_undegraded":"2023-09-23T23:45:55.054050+0000","last_fullsized":"2023-09-23T23:45:55.054050+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:58.895160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":936,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:58.452886+0000","last_change":"2023-09-23T23:45:19.797135+0000","last_active":"2023-09-23T23:45:58.452886+0000","last_peered":"2023-09-23T23:45:58.452886+0000","last_clean":"2023-09-23T23:45:58.452886+0000","last_became_active":"2023-09-23T23:45:19.796815+0000","last_became_peered":"2023-09-23T23:45:19.796815+0000","last_unstale":"2023-09-23T23:45:58.452886+0000","last_undegraded":"2023-09-23T23:45:58.452886+0000","last_fullsized":"2023-09-23T23:45:58.452886+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:11.893318+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"715'124","reported_seq":1083,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052842+0000","last_change":"2023-09-23T23:45:52.244360+0000","last_active":"2023-09-23T23:45:55.052842+0000","last_peered":"2023-09-23T23:45:55.052842+0000","last_clean":"2023-09-23T23:45:55.052842+0000","last_became_active":"2023-09-23T23:45:52.243651+0000","last_became_peered":"2023-09-23T23:45:52.243651+0000","last_unstale":"2023-09-23T23:45:55.052842+0000","last_undegraded":"2023-09-23T23:45:55.052842+0000","last_fullsized":"2023-09-23T23:45:55.052842+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:57.939576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012476480000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":727,"num_read_kb":1949,"num_write":120,"num_write_kb":2256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":546,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052056+0000","last_change":"2023-09-23T23:45:22.812953+0000","last_active":"2023-09-23T23:45:55.052056+0000","last_peered":"2023-09-23T23:45:55.052056+0000","last_clean":"2023-09-23T23:45:55.052056+0000","last_became_active":"2023-09-23T23:45:22.812658+0000","last_became_peered":"2023-09-23T23:45:22.812658+0000","last_unstale":"2023-09-23T23:45:55.052056+0000","last_undegraded":"2023-09-23T23:45:55.052056+0000","last_fullsized":"2023-09-23T23:45:55.052056+0000","mapping_epoch":757,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":758,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:49.688942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":35,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051873+0000","last_change":"2023-09-23T23:45:08.978721+0000","last_active":"2023-09-23T23:45:55.051873+0000","last_peered":"2023-09-23T23:45:55.051873+0000","last_clean":"2023-09-23T23:45:55.051873+0000","last_became_active":"2023-09-23T23:45:08.978419+0000","last_became_peered":"2023-09-23T23:45:08.978419+0000","last_unstale":"2023-09-23T23:45:55.051873+0000","last_undegraded":"2023-09-23T23:45:55.051873+0000","last_fullsized":"2023-09-23T23:45:55.051873+0000","mapping_epoch":748,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":749,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:28.678292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":608,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.054103+0000","last_change":"2023-09-23T23:45:03.040803+0000","last_active":"2023-09-23T23:45:55.054103+0000","last_peered":"2023-09-23T23:45:55.054103+0000","last_clean":"2023-09-23T23:45:55.054103+0000","last_became_active":"2023-09-23T23:45:03.040337+0000","last_became_peered":"2023-09-23T23:45:03.040337+0000","last_unstale":"2023-09-23T23:45:55.054103+0000","last_undegraded":"2023-09-23T23:45:55.054103+0000","last_fullsized":"2023-09-23T23:45:55.054103+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:36.187578+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"715'102","reported_seq":717,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052007+0000","last_change":"2023-09-23T23:45:52.243612+0000","last_active":"2023-09-23T23:45:55.052007+0000","last_peered":"2023-09-23T23:45:55.052007+0000","last_clean":"2023-09-23T23:45:55.052007+0000","last_became_active":"2023-09-23T23:45:52.243149+0000","last_became_peered":"2023-09-23T23:45:52.243149+0000","last_unstale":"2023-09-23T23:45:55.052007+0000","last_undegraded":"2023-09-23T23:45:55.052007+0000","last_fullsized":"2023-09-23T23:45:55.052007+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:08.282580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037965999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1014,"num_read_kb":1641,"num_write":66,"num_write_kb":416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":293,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.986864+0000","last_change":"2023-09-23T23:45:19.801071+0000","last_active":"2023-09-23T23:45:54.986864+0000","last_peered":"2023-09-23T23:45:54.986864+0000","last_clean":"2023-09-23T23:45:54.986864+0000","last_became_active":"2023-09-23T23:45:19.800637+0000","last_became_peered":"2023-09-23T23:45:19.800637+0000","last_unstale":"2023-09-23T23:45:54.986864+0000","last_undegraded":"2023-09-23T23:45:54.986864+0000","last_fullsized":"2023-09-23T23:45:54.986864+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:45.026051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":267,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052465+0000","last_change":"2023-09-23T23:45:12.997379+0000","last_active":"2023-09-23T23:45:55.052465+0000","last_peered":"2023-09-23T23:45:55.052465+0000","last_clean":"2023-09-23T23:45:55.052465+0000","last_became_active":"2023-09-23T23:45:12.995469+0000","last_became_peered":"2023-09-23T23:45:12.995469+0000","last_unstale":"2023-09-23T23:45:55.052465+0000","last_undegraded":"2023-09-23T23:45:55.052465+0000","last_fullsized":"2023-09-23T23:45:55.052465+0000","mapping_epoch":752,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":753,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:04.637951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":1014,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.989248+0000","last_change":"2023-09-23T23:45:03.038729+0000","last_active":"2023-09-23T23:45:54.989248+0000","last_peered":"2023-09-23T23:45:54.989248+0000","last_clean":"2023-09-23T23:45:54.989248+0000","last_became_active":"2023-09-23T23:45:03.038334+0000","last_became_peered":"2023-09-23T23:45:03.038334+0000","last_unstale":"2023-09-23T23:45:54.989248+0000","last_undegraded":"2023-09-23T23:45:54.989248+0000","last_fullsized":"2023-09-23T23:45:54.989248+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:16.965706+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":95,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052868+0000","last_change":"2023-09-23T23:43:37.369294+0000","last_active":"2023-09-23T23:45:55.052868+0000","last_peered":"2023-09-23T23:45:55.052868+0000","last_clean":"2023-09-23T23:45:55.052868+0000","last_became_active":"2023-09-23T23:43:37.342878+0000","last_became_peered":"2023-09-23T23:43:37.342878+0000","last_unstale":"2023-09-23T23:45:55.052868+0000","last_undegraded":"2023-09-23T23:45:55.052868+0000","last_fullsized":"2023-09-23T23:45:55.052868+0000","mapping_epoch":692,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":693,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:52.844816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"716'140","reported_seq":590,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.985933+0000","last_change":"2023-09-23T23:45:19.797178+0000","last_active":"2023-09-23T23:45:54.985933+0000","last_peered":"2023-09-23T23:45:54.985933+0000","last_clean":"2023-09-23T23:45:54.985933+0000","last_became_active":"2023-09-23T23:45:19.796843+0000","last_became_peered":"2023-09-23T23:45:19.796843+0000","last_unstale":"2023-09-23T23:45:54.985933+0000","last_undegraded":"2023-09-23T23:45:54.985933+0000","last_fullsized":"2023-09-23T23:45:54.985933+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:45.644839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000432328,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":762,"num_read_kb":2671,"num_write":98,"num_write_kb":2116,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":26,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051387+0000","last_change":"2023-09-23T23:45:22.812861+0000","last_active":"2023-09-23T23:45:55.051387+0000","last_peered":"2023-09-23T23:45:55.051387+0000","last_clean":"2023-09-23T23:45:55.051387+0000","last_became_active":"2023-09-23T23:45:22.812453+0000","last_became_peered":"2023-09-23T23:45:22.812453+0000","last_unstale":"2023-09-23T23:45:55.051387+0000","last_undegraded":"2023-09-23T23:45:55.051387+0000","last_fullsized":"2023-09-23T23:45:55.051387+0000","mapping_epoch":757,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":758,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:09.336586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":848,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051799+0000","last_change":"2023-09-23T23:43:39.961347+0000","last_active":"2023-09-23T23:45:55.051799+0000","last_peered":"2023-09-23T23:45:55.051799+0000","last_clean":"2023-09-23T23:45:55.051799+0000","last_became_active":"2023-09-23T23:43:39.961019+0000","last_became_peered":"2023-09-23T23:43:39.961019+0000","last_unstale":"2023-09-23T23:45:55.051799+0000","last_undegraded":"2023-09-23T23:45:55.051799+0000","last_fullsized":"2023-09-23T23:45:55.051799+0000","mapping_epoch":695,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":696,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:01.621631+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"715'148","reported_seq":1024,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.054127+0000","last_change":"2023-09-23T23:45:03.042180+0000","last_active":"2023-09-23T23:45:55.054127+0000","last_peered":"2023-09-23T23:45:55.054127+0000","last_clean":"2023-09-23T23:45:55.054127+0000","last_became_active":"2023-09-23T23:45:03.041364+0000","last_became_peered":"2023-09-23T23:45:03.041364+0000","last_unstale":"2023-09-23T23:45:55.054127+0000","last_undegraded":"2023-09-23T23:45:55.054127+0000","last_fullsized":"2023-09-23T23:45:55.054127+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:49.057745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033161300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":898,"num_read_kb":2090,"num_write":128,"num_write_kb":1223,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":39,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051974+0000","last_change":"2023-09-23T23:45:08.984591+0000","last_active":"2023-09-23T23:45:55.051974+0000","last_peered":"2023-09-23T23:45:55.051974+0000","last_clean":"2023-09-23T23:45:55.051974+0000","last_became_active":"2023-09-23T23:45:08.982188+0000","last_became_peered":"2023-09-23T23:45:08.982188+0000","last_unstale":"2023-09-23T23:45:55.051974+0000","last_undegraded":"2023-09-23T23:45:55.051974+0000","last_fullsized":"2023-09-23T23:45:55.051974+0000","mapping_epoch":748,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":749,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:57.031770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988650+0000","last_change":"2023-09-23T23:44:59.992453+0000","last_active":"2023-09-23T23:45:54.988650+0000","last_peered":"2023-09-23T23:45:54.988650+0000","last_clean":"2023-09-23T23:45:54.988650+0000","last_became_active":"2023-09-23T23:44:59.992101+0000","last_became_peered":"2023-09-23T23:44:59.992101+0000","last_unstale":"2023-09-23T23:45:54.988650+0000","last_undegraded":"2023-09-23T23:45:54.988650+0000","last_fullsized":"2023-09-23T23:45:54.988650+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:09.758075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1288,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:57.994865+0000","last_change":"2023-09-23T23:45:11.988445+0000","last_active":"2023-09-23T23:45:57.994865+0000","last_peered":"2023-09-23T23:45:57.994865+0000","last_clean":"2023-09-23T23:45:57.994865+0000","last_became_active":"2023-09-23T23:45:11.988080+0000","last_became_peered":"2023-09-23T23:45:11.988080+0000","last_unstale":"2023-09-23T23:45:57.994865+0000","last_undegraded":"2023-09-23T23:45:57.994865+0000","last_fullsized":"2023-09-23T23:45:57.994865+0000","mapping_epoch":751,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":752,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:18.929804+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":52,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052381+0000","last_change":"2023-09-23T23:45:12.997680+0000","last_active":"2023-09-23T23:45:55.052381+0000","last_peered":"2023-09-23T23:45:55.052381+0000","last_clean":"2023-09-23T23:45:55.052381+0000","last_became_active":"2023-09-23T23:45:12.995943+0000","last_became_peered":"2023-09-23T23:45:12.995943+0000","last_unstale":"2023-09-23T23:45:55.052381+0000","last_undegraded":"2023-09-23T23:45:55.052381+0000","last_fullsized":"2023-09-23T23:45:55.052381+0000","mapping_epoch":752,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":753,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:32:37.694678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":550,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.985906+0000","last_change":"2023-09-23T23:43:37.332588+0000","last_active":"2023-09-23T23:45:54.985906+0000","last_peered":"2023-09-23T23:45:54.985906+0000","last_clean":"2023-09-23T23:45:54.985906+0000","last_became_active":"2023-09-23T23:43:37.331837+0000","last_became_peered":"2023-09-23T23:43:37.331837+0000","last_unstale":"2023-09-23T23:45:54.985906+0000","last_undegraded":"2023-09-23T23:45:54.985906+0000","last_fullsized":"2023-09-23T23:45:54.985906+0000","mapping_epoch":692,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":693,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:14.466130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":983,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052711+0000","last_change":"2023-09-23T23:45:03.038381+0000","last_active":"2023-09-23T23:45:55.052711+0000","last_peered":"2023-09-23T23:45:55.052711+0000","last_clean":"2023-09-23T23:45:55.052711+0000","last_became_active":"2023-09-23T23:45:03.038081+0000","last_became_peered":"2023-09-23T23:45:03.038081+0000","last_unstale":"2023-09-23T23:45:55.052711+0000","last_undegraded":"2023-09-23T23:45:55.052711+0000","last_fullsized":"2023-09-23T23:45:55.052711+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:00.921011+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":418,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051623+0000","last_change":"2023-09-23T23:45:32.242007+0000","last_active":"2023-09-23T23:45:55.051623+0000","last_peered":"2023-09-23T23:45:55.051623+0000","last_clean":"2023-09-23T23:45:55.051623+0000","last_became_active":"2023-09-23T23:45:32.240828+0000","last_became_peered":"2023-09-23T23:45:32.240828+0000","last_unstale":"2023-09-23T23:45:55.051623+0000","last_undegraded":"2023-09-23T23:45:55.051623+0000","last_fullsized":"2023-09-23T23:45:55.051623+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:44.441348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":43,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988856+0000","last_change":"2023-09-23T23:44:55.988280+0000","last_active":"2023-09-23T23:45:54.988856+0000","last_peered":"2023-09-23T23:45:54.988856+0000","last_clean":"2023-09-23T23:45:54.988856+0000","last_became_active":"2023-09-23T23:44:55.987930+0000","last_became_peered":"2023-09-23T23:44:55.987930+0000","last_unstale":"2023-09-23T23:45:54.988856+0000","last_undegraded":"2023-09-23T23:45:54.988856+0000","last_fullsized":"2023-09-23T23:45:54.988856+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:24.234351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":286,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.053022+0000","last_change":"2023-09-23T23:45:32.241873+0000","last_active":"2023-09-23T23:45:55.053022+0000","last_peered":"2023-09-23T23:45:55.053022+0000","last_clean":"2023-09-23T23:45:55.053022+0000","last_became_active":"2023-09-23T23:45:32.239852+0000","last_became_peered":"2023-09-23T23:45:32.239852+0000","last_unstale":"2023-09-23T23:45:55.053022+0000","last_undegraded":"2023-09-23T23:45:55.053022+0000","last_fullsized":"2023-09-23T23:45:55.053022+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:15.224411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":43,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988609+0000","last_change":"2023-09-23T23:44:55.987739+0000","last_active":"2023-09-23T23:45:54.988609+0000","last_peered":"2023-09-23T23:45:54.988609+0000","last_clean":"2023-09-23T23:45:54.988609+0000","last_became_active":"2023-09-23T23:44:55.987506+0000","last_became_peered":"2023-09-23T23:44:55.987506+0000","last_unstale":"2023-09-23T23:45:54.988609+0000","last_undegraded":"2023-09-23T23:45:54.988609+0000","last_fullsized":"2023-09-23T23:45:54.988609+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:09.672401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":1001,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052196+0000","last_change":"2023-09-23T23:45:52.243605+0000","last_active":"2023-09-23T23:45:55.052196+0000","last_peered":"2023-09-23T23:45:55.052196+0000","last_clean":"2023-09-23T23:45:55.052196+0000","last_became_active":"2023-09-23T23:45:52.243247+0000","last_became_peered":"2023-09-23T23:45:52.243247+0000","last_unstale":"2023-09-23T23:45:55.052196+0000","last_undegraded":"2023-09-23T23:45:55.052196+0000","last_fullsized":"2023-09-23T23:45:55.052196+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:45:04.205935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:45:04.205935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:16.253214+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":100,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052811+0000","last_change":"2023-09-23T23:44:59.990843+0000","last_active":"2023-09-23T23:45:55.052811+0000","last_peered":"2023-09-23T23:45:55.052811+0000","last_clean":"2023-09-23T23:45:55.052811+0000","last_became_active":"2023-09-23T23:44:59.990488+0000","last_became_peered":"2023-09-23T23:44:59.990488+0000","last_unstale":"2023-09-23T23:45:55.052811+0000","last_undegraded":"2023-09-23T23:45:55.052811+0000","last_fullsized":"2023-09-23T23:45:55.052811+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:06.824244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":91,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:57.994729+0000","last_change":"2023-09-23T23:43:37.342518+0000","last_active":"2023-09-23T23:45:57.994729+0000","last_peered":"2023-09-23T23:45:57.994729+0000","last_clean":"2023-09-23T23:45:57.994729+0000","last_became_active":"2023-09-23T23:43:37.341672+0000","last_became_peered":"2023-09-23T23:43:37.341672+0000","last_unstale":"2023-09-23T23:45:57.994729+0000","last_undegraded":"2023-09-23T23:45:57.994729+0000","last_fullsized":"2023-09-23T23:45:57.994729+0000","mapping_epoch":692,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":693,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:09.991757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":1016,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:58.452705+0000","last_change":"2023-09-23T23:45:25.984339+0000","last_active":"2023-09-23T23:45:58.452705+0000","last_peered":"2023-09-23T23:45:58.452705+0000","last_clean":"2023-09-23T23:45:58.452705+0000","last_became_active":"2023-09-23T23:45:25.984004+0000","last_became_peered":"2023-09-23T23:45:25.984004+0000","last_unstale":"2023-09-23T23:45:58.452705+0000","last_undegraded":"2023-09-23T23:45:58.452705+0000","last_fullsized":"2023-09-23T23:45:58.452705+0000","mapping_epoch":760,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":761,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:48.011518+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":719,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052359+0000","last_change":"2023-09-23T23:45:12.995656+0000","last_active":"2023-09-23T23:45:55.052359+0000","last_peered":"2023-09-23T23:45:55.052359+0000","last_clean":"2023-09-23T23:45:55.052359+0000","last_became_active":"2023-09-23T23:45:12.995246+0000","last_became_peered":"2023-09-23T23:45:12.995246+0000","last_unstale":"2023-09-23T23:45:55.052359+0000","last_undegraded":"2023-09-23T23:45:55.052359+0000","last_fullsized":"2023-09-23T23:45:55.052359+0000","mapping_epoch":752,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":753,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:48.479413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":53,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988431+0000","last_change":"2023-09-23T23:44:59.991420+0000","last_active":"2023-09-23T23:45:54.988431+0000","last_peered":"2023-09-23T23:45:54.988431+0000","last_clean":"2023-09-23T23:45:54.988431+0000","last_became_active":"2023-09-23T23:44:59.991042+0000","last_became_peered":"2023-09-23T23:44:59.991042+0000","last_unstale":"2023-09-23T23:45:54.988431+0000","last_undegraded":"2023-09-23T23:45:54.988431+0000","last_fullsized":"2023-09-23T23:45:54.988431+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:51.331723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":114,"num_bytes_recovered":29258,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31868,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":13211,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782805,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":498644,"kb_used_data":5836,"kb_used_omap":8,"kb_used_meta":492791,"kb_avail":93246508,"statfs":{"total":95995035648,"available":95484424192,"internally_reserved":0,"allocated":5976064,"data_stored":4608295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":504618053},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:46:24 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.755,"15min":1.0900000000000001},"min":{"1min":0.44700000000000001,"5min":0.23799999999999999,"15min":0.20499999999999999},"max":{"1min":2.7109999999999999,"5min":6.5510000000000002,"15min":36.956000000000003},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.86299999999999999,"15min":1.1799999999999999},"min":{"1min":0.39000000000000001,"5min":0.25800000000000001,"15min":0.21299999999999999},"max":{"1min":2.6579999999999999,"5min":6.4489999999999998,"15min":37.247999999999998},"last":0.46600000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:46:24 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.81799999999999995,"15min":1.96},"min":{"1min":0.39300000000000002,"5min":0.252,"15min":0.245},"max":{"1min":2.4889999999999999,"5min":5.9660000000000002,"15min":129.11199999999999},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.59399999999999997,"5min":1.143,"15min":2.0419999999999998},"min":{"1min":0.40200000000000002,"5min":0.28999999999999998,"15min":0.24399999999999999},"max":{"1min":1.661,"5min":8.8460000000000001,"15min":129.02600000000001},"last":0.49299999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:46:24 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.2070000000000001,"15min":2.2040000000000002},"min":{"1min":0.36899999999999999,"5min":0.251,"15min":0.251},"max":{"1min":1.411,"5min":8.9779999999999998,"15min":78.528999999999996},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.69799999999999995,"5min":1.135,"15min":1.9810000000000001},"min":{"1min":0.40699999999999997,"5min":0.28000000000000003,"15min":0.26200000000000001},"max":{"1min":2.3370000000000002,"5min":10.145,"15min":53.023000000000003},"last":0.57099999999999995}]},{"osd":3,"last update":"Sat Sep 23 23:46:24 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":1.1639999999999999,"15min":3.6389999999999998},"min":{"1min":0.371,"5min":0.31,"15min":0.29599999999999999},"max":{"1min":5.4080000000000004,"5min":8.1479999999999997,"15min":154.18100000000001},"last":0.442},{"interface":"front","average":{"1min":0.80000000000000004,"5min":1.234,"15min":4.2649999999999997},"min":{"1min":0.505,"5min":0.32600000000000001,"15min":0.30099999999999999},"max":{"1min":4.4630000000000001,"5min":8.5779999999999994,"15min":154.16300000000001},"last":0.53000000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:45:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":15.500999999999999,"5min":15.500999999999999,"15min":15.500999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.6389999999999998,"5min":3.6389999999999998,"15min":3.6389999999999998},"last":0.72399999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:46:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.61099999999999999,"15min":1.452},"min":{"1min":0.33100000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.96299999999999997,"5min":9.0709999999999997,"15min":119.417},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.60899999999999999,"15min":1.4690000000000001},"min":{"1min":0.38300000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.79200000000000004,"5min":8.9339999999999993,"15min":119.61499999999999},"last":0.51400000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:46:24 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.53300000000000003,"15min":1.0880000000000001},"min":{"1min":0.48299999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":94.852999999999994},"last":0.621},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.64700000000000002,"15min":1.236},"min":{"1min":0.41599999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.0569999999999999,"5min":5.9299999999999997,"15min":95.510999999999996},"last":0.80900000000000005}]}]},{"osd":6,"up_from":46,"seq":197568496282,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":767816,"kb_used_data":5888,"kb_used_omap":0,"kb_used_meta":761920,"kb_avail":92977336,"statfs":{"total":95995035648,"available":95208792064,"internally_reserved":0,"allocated":6029312,"data_stored":4673160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":780206080},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.84099999999999997,"15min":1.081},"min":{"1min":0.35299999999999998,"5min":0.23899999999999999,"15min":0.223},"max":{"1min":0.79100000000000004,"5min":9.4060000000000006,"15min":60.252000000000002},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":1.0700000000000001,"15min":1.1299999999999999},"min":{"1min":0.33500000000000002,"5min":0.26200000000000001,"15min":0.23999999999999999},"max":{"1min":0.77700000000000002,"5min":7.1920000000000002,"15min":60.130000000000003},"last":0.54800000000000004}]},{"osd":1,"last update":"Sat Sep 23 23:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":1.5920000000000001,"15min":2.0939999999999999},"min":{"1min":0.32200000000000001,"5min":0.24399999999999999,"15min":0.24299999999999999},"max":{"1min":0.75600000000000001,"5min":31.670000000000002,"15min":89.605000000000004},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.8500000000000001,"15min":2.1970000000000001},"min":{"1min":0.374,"5min":0.33700000000000002,"15min":0.24099999999999999},"max":{"1min":0.77100000000000002,"5min":32.363999999999997,"15min":91.025999999999996},"last":0.68700000000000006}]},{"osd":2,"last update":"Sat Sep 23 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":2.1400000000000001,"15min":3.5950000000000002},"min":{"1min":0.39500000000000002,"5min":0.318,"15min":0.26700000000000002},"max":{"1min":0.76800000000000002,"5min":73.483000000000004,"15min":97.408000000000001},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.8740000000000001,"15min":3.8159999999999998},"min":{"1min":0.32700000000000001,"5min":0.29499999999999998,"15min":0.26100000000000001},"max":{"1min":0.86699999999999999,"5min":73.135000000000005,"15min":125.539},"last":0.81100000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":3.923,"15min":6.6130000000000004},"min":{"1min":0.39300000000000002,"5min":0.32400000000000001,"15min":0.28100000000000003},"max":{"1min":1.3049999999999999,"5min":84.466999999999999,"15min":319.22699999999998},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.64100000000000001,"5min":3.681,"15min":6.3710000000000004},"min":{"1min":0.377,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.83799999999999997,"5min":84.730000000000004,"15min":319.42899999999997},"last":0.86299999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.90300000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":1.131,"15min":0.998},"min":{"1min":0.36699999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.82599999999999996,"5min":68.947999999999993,"15min":68.947999999999993},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.59299999999999997,"5min":1.1479999999999999,"15min":1.073},"min":{"1min":0.32900000000000001,"5min":0.28999999999999998,"15min":0.28399999999999997},"max":{"1min":0.76000000000000001,"5min":67.546000000000006,"15min":67.546000000000006},"last":0.71299999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":1.2689999999999999,"15min":1.127},"min":{"1min":0.503,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.94399999999999995,"5min":70.463999999999999,"15min":70.463999999999999},"last":0.94399999999999995},{"interface":"front","average":{"1min":0.68899999999999995,"5min":1.2729999999999999,"15min":1.1499999999999999},"min":{"1min":0.52100000000000002,"5min":0.312,"15min":0.28299999999999997},"max":{"1min":0.93200000000000005,"5min":70.563999999999993,"15min":70.563999999999993},"last":0.93200000000000005}]}]},{"osd":1,"up_from":112,"seq":481036337623,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1065716,"kb_used_data":6956,"kb_used_omap":4,"kb_used_meta":1058747,"kb_avail":92679436,"statfs":{"total":95995035648,"available":94903742464,"internally_reserved":0,"allocated":7122944,"data_stored":4527163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1084157838},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:45:38 2023","interfaces":[{"interface":"back","average":{"1min":0.34100000000000003,"5min":2.2519999999999998,"15min":2.968},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.16400000000000001},"max":{"1min":0.58299999999999996,"5min":13.766,"15min":156.37799999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.39000000000000001,"5min":1.4750000000000001,"15min":2.3969999999999998},"min":{"1min":0.23699999999999999,"5min":0.22,"15min":0.189},"max":{"1min":0.77400000000000002,"5min":13.548999999999999,"15min":161.05199999999999},"last":0.46899999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:46:25 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":1.9990000000000001,"15min":3.762},"min":{"1min":0.313,"5min":0.22900000000000001,"15min":0.184},"max":{"1min":0.51500000000000001,"5min":49.311,"15min":201.03999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.34399999999999997,"5min":2.0390000000000001,"15min":4.0940000000000003},"min":{"1min":0.26700000000000002,"5min":0.23000000000000001,"15min":0.20899999999999999},"max":{"1min":0.55300000000000005,"5min":48.991999999999997,"15min":202.12},"last":0.45200000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:46:25 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":5.0970000000000004,"15min":4.7800000000000002},"min":{"1min":0.26800000000000002,"5min":0.25,"15min":0.21299999999999999},"max":{"1min":0.52000000000000002,"5min":272.839,"15min":272.839},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.44400000000000001,"5min":4.8689999999999998,"15min":4.8070000000000004},"min":{"1min":0.31,"5min":0.27700000000000002,"15min":0.224},"max":{"1min":1.2450000000000001,"5min":272.76400000000001,"15min":272.76400000000001},"last":1.2450000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:45:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.93799999999999994},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.70999999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:45:38 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":1.127,"15min":2.044},"min":{"1min":0.38300000000000001,"5min":0.31,"15min":0.28899999999999998},"max":{"1min":0.85099999999999998,"5min":8.2639999999999993,"15min":151.94800000000001},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.6339999999999999,"15min":2.3290000000000002},"min":{"1min":0.40300000000000002,"5min":0.36499999999999999,"15min":0.29599999999999999},"max":{"1min":0.81499999999999995,"5min":10.141999999999999,"15min":151.821},"last":0.74099999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:45:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.161,"15min":1.496},"min":{"1min":0.42499999999999999,"5min":0.33200000000000002,"15min":0.32200000000000001},"max":{"1min":0.873,"5min":8.2910000000000004,"15min":87.837999999999994},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.657,"15min":1.8740000000000001},"min":{"1min":0.41699999999999998,"5min":0.38400000000000001,"15min":0.312},"max":{"1min":0.78100000000000003,"5min":10.233000000000001,"15min":85.518000000000001},"last":0.94999999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:46:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.0329999999999999,"15min":1.405},"min":{"1min":0.52000000000000002,"5min":0.38,"15min":0.28799999999999998},"max":{"1min":1.0269999999999999,"5min":13.587999999999999,"15min":44.241},"last":1.0269999999999999},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1.2969999999999999,"15min":1.677},"min":{"1min":0.52100000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.77900000000000003,"5min":10.210000000000001,"15min":43.920999999999999},"last":0.77600000000000002}]}]},{"osd":0,"up_from":91,"seq":390842024447,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1405392,"kb_used_data":11336,"kb_used_omap":5,"kb_used_meta":1394042,"kb_avail":92339760,"statfs":{"total":95995035648,"available":94555914240,"internally_reserved":0,"allocated":11608064,"data_stored":9037781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1427499642},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:45:50 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":1.8160000000000001,"15min":4.5979999999999999},"min":{"1min":0.24099999999999999,"5min":0.19700000000000001,"15min":0.188},"max":{"1min":0.59699999999999998,"5min":10.502000000000001,"15min":301.12700000000001},"last":0.245},{"interface":"front","average":{"1min":0.34200000000000003,"5min":1.8029999999999999,"15min":4.5540000000000003},"min":{"1min":0.23999999999999999,"5min":0.221,"15min":0.19700000000000001},"max":{"1min":0.52700000000000002,"5min":18.503,"15min":297.72399999999999},"last":0.312}]},{"osd":2,"last update":"Sat Sep 23 23:46:23 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":2.907,"15min":5.282},"min":{"1min":0.26200000000000001,"5min":0.19400000000000001,"15min":0.17699999999999999},"max":{"1min":0.55500000000000005,"5min":89.935000000000002,"15min":283.07400000000001},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.39300000000000002,"5min":2.8220000000000001,"15min":5.3799999999999999},"min":{"1min":0.26300000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.65100000000000002,"5min":90.25,"15min":283.161},"last":0.27900000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:45:50 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":2.5990000000000002,"15min":6.2590000000000003},"min":{"1min":0.27500000000000002,"5min":0.23200000000000001,"15min":0.217},"max":{"1min":0.59299999999999997,"5min":36.186999999999998,"15min":220.732},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.435,"5min":2.7949999999999999,"15min":6.5220000000000002},"min":{"1min":0.30499999999999999,"5min":0.245,"15min":0.19800000000000001},"max":{"1min":0.64900000000000002,"5min":33.616,"15min":218.78200000000001},"last":0.34899999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:45:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.60699999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":1.3979999999999999,"15min":1.923},"min":{"1min":0.34999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.80300000000000005,"5min":20.629999999999999,"15min":94.254000000000005},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.56499999999999995,"5min":1.986,"15min":2.036},"min":{"1min":0.38200000000000001,"5min":0.34200000000000003,"15min":0.30599999999999999},"max":{"1min":0.81899999999999995,"5min":20.867000000000001,"15min":94.837000000000003},"last":0.67000000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.254,"15min":1.9610000000000001},"min":{"1min":0.40100000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.83899999999999997,"5min":7.4359999999999999,"15min":108.67},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.8280000000000001,"15min":2.117},"min":{"1min":0.439,"5min":0.42299999999999999,"15min":0.30199999999999999},"max":{"1min":0.97199999999999998,"5min":9.6310000000000002,"15min":108.804},"last":0.70399999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:46:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":1.4339999999999999,"15min":1.6519999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.875,"5min":21.132999999999999,"15min":66.620000000000005},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.621,"5min":1.7609999999999999,"15min":1.798},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.33400000000000002},"max":{"1min":0.93000000000000005,"5min":23.489000000000001,"15min":66.701999999999998},"last":0.73399999999999999}]}]},{"osd":2,"up_from":199,"seq":854698492268,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1063248,"kb_used_data":11464,"kb_used_omap":7,"kb_used_meta":1051768,"kb_avail":92681904,"statfs":{"total":95995035648,"available":94906269696,"internally_reserved":0,"allocated":11739136,"data_stored":9001924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1077010513},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":1.879,"15min":2.1389999999999998},"min":{"1min":0.20000000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.49399999999999999,"5min":34.648000000000003,"15min":65.001000000000005},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.33400000000000002,"5min":1.3169999999999999,"15min":1.8260000000000001},"min":{"1min":0.215,"5min":0.19900000000000001,"15min":0.19700000000000001},"max":{"1min":0.45100000000000001,"5min":13.506,"15min":64.346000000000004},"last":0.29799999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":1.3069999999999999,"15min":3.0009999999999999},"min":{"1min":0.254,"5min":0.20999999999999999,"15min":0.192},"max":{"1min":0.53600000000000003,"5min":14.433999999999999,"15min":106.765},"last":0.312},{"interface":"front","average":{"1min":0.38800000000000001,"5min":1.698,"15min":2.8929999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.222},"max":{"1min":0.57199999999999995,"5min":13.353,"15min":106.732},"last":0.40999999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":2.254,"15min":3.3599999999999999},"min":{"1min":0.251,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.626,"5min":54.441000000000003,"15min":106.70399999999999},"last":0.371},{"interface":"front","average":{"1min":0.40000000000000002,"5min":2.448,"15min":3.1400000000000001},"min":{"1min":0.28399999999999997,"5min":0.23899999999999999,"15min":0.23200000000000001},"max":{"1min":0.64700000000000002,"5min":53.734000000000002,"15min":106.66800000000001},"last":0.38200000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:45:53 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.70199999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":1.54,"15min":2.8149999999999999},"min":{"1min":0.373,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.83399999999999996,"5min":13.023,"15min":101.104},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.56000000000000005,"5min":1.494,"15min":2.944},"min":{"1min":0.315,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.78200000000000003,"5min":15.074999999999999,"15min":101.404},"last":0.78100000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.651,"15min":3.1240000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.31},"max":{"1min":0.80200000000000005,"5min":15.898999999999999,"15min":107.018},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.55200000000000005,"5min":1.4770000000000001,"15min":2.8279999999999998},"min":{"1min":0.34999999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.76800000000000002,"5min":15.025,"15min":98.231999999999999},"last":0.54100000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.4119999999999999,"15min":2.0049999999999999},"min":{"1min":0.40500000000000003,"5min":0.379,"15min":0.32200000000000001},"max":{"1min":0.79100000000000004,"5min":7.4349999999999996,"15min":68.814999999999998},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.58899999999999997,"5min":1.538,"15min":2.4860000000000002},"min":{"1min":0.435,"5min":0.39900000000000002,"15min":0.35399999999999998},"max":{"1min":0.82199999999999995,"5min":7.4649999999999999,"15min":106.809},"last":0.55000000000000004}]}]},{"osd":3,"up_from":148,"seq":635655160236,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1131892,"kb_used_data":11436,"kb_used_omap":2,"kb_used_meta":1120445,"kb_avail":92613260,"statfs":{"total":95995035648,"available":94835978240,"internally_reserved":0,"allocated":11710464,"data_stored":9056341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1147335733},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:46:05 2023","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":6.915,"15min":3.2919999999999998},"min":{"1min":0.26000000000000001,"5min":0.20300000000000001,"15min":0.184},"max":{"1min":0.58899999999999997,"5min":230.53899999999999,"15min":230.53899999999999},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.35699999999999998,"5min":6.3460000000000001,"15min":3.2549999999999999},"min":{"1min":0.23100000000000001,"5min":0.20599999999999999,"15min":0.17199999999999999},"max":{"1min":0.60599999999999998,"5min":230.38800000000001,"15min":230.38800000000001},"last":0.35599999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:46:05 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":1.974,"15min":2.1019999999999999},"min":{"1min":0.25600000000000001,"5min":0.219,"15min":0.17599999999999999},"max":{"1min":1.204,"5min":21.824999999999999,"15min":56.119},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.38300000000000001,"5min":1.837,"15min":1.8899999999999999},"min":{"1min":0.25,"5min":0.22900000000000001,"15min":0.20799999999999999},"max":{"1min":0.625,"5min":20.779,"15min":53.962000000000003},"last":0.29099999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":2.0409999999999999,"15min":4.1550000000000002},"min":{"1min":0.29799999999999999,"5min":0.25,"15min":0.23999999999999999},"max":{"1min":1.0820000000000001,"5min":31.564,"15min":139.96299999999999},"last":0.375},{"interface":"front","average":{"1min":0.433,"5min":2.3620000000000001,"15min":4.2629999999999999},"min":{"1min":0.32400000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.82399999999999995,"5min":69.405000000000001,"15min":141.93199999999999},"last":0.318}]},{"osd":4,"last update":"Sat Sep 23 23:45:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.55900000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:46:05 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":1.4930000000000001,"15min":4.4489999999999998},"min":{"1min":0.44500000000000001,"5min":0.29899999999999999,"15min":0.29499999999999998},"max":{"1min":0.85499999999999998,"5min":15.56,"15min":214.43700000000001},"last":0.628},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.5169999999999999,"15min":4.3280000000000003},"min":{"1min":0.46400000000000002,"5min":0.38500000000000001,"15min":0.32600000000000001},"max":{"1min":1.151,"5min":14.478999999999999,"15min":214.46600000000001},"last":0.54300000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:46:05 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":1.339,"15min":2.2810000000000001},"min":{"1min":0.499,"5min":0.38900000000000001,"15min":0.318},"max":{"1min":0.85099999999999998,"5min":12.215,"15min":125.94},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.64400000000000002,"5min":1.546,"15min":2.3039999999999998},"min":{"1min":0.46800000000000003,"5min":0.35499999999999998,"15min":0.32300000000000001},"max":{"1min":1.1779999999999999,"5min":14.513,"15min":125.955},"last":0.51700000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.7989999999999999,"15min":2.6070000000000002},"min":{"1min":0.497,"5min":0.313,"15min":0.313},"max":{"1min":0.88,"5min":69.197999999999993,"15min":192.69},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.65900000000000003,"5min":1.401,"15min":2.698},"min":{"1min":0.44400000000000001,"5min":0.40899999999999997,"15min":0.39200000000000002},"max":{"1min":0.81699999999999995,"5min":12.568,"15min":192.65600000000001},"last":0.65800000000000003}]}]},{"osd":4,"up_from":736,"seq":3161095929878,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33356,"kb_used_data":5572,"kb_used_omap":4,"kb_used_meta":27771,"kb_avail":93711796,"statfs":{"total":95995035648,"available":95960879104,"internally_reserved":0,"allocated":5705728,"data_stored":4345537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4160,"internal_metadata":28438464},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.54100000000000004}]},{"osd":1,"last update":"Sat Sep 23 23:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.38300000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.51100000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.433}]},{"osd":5,"last update":"Sat Sep 23 23:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.40100000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.55500000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.53000000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757918,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":871940,"kb_used_data":5820,"kb_used_omap":0,"kb_used_meta":866112,"kb_avail":92873212,"statfs":{"total":95995035648,"available":95102169088,"internally_reserved":0,"allocated":5959680,"data_stored":4603988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":886898688},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:45:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":4.9539999999999997,"15min":2.3370000000000002},"min":{"1min":0.34599999999999997,"5min":0.22900000000000001,"15min":0.20899999999999999},"max":{"1min":0.81499999999999995,"5min":174.58600000000001,"15min":174.58600000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":5.0709999999999997,"15min":2.5739999999999998},"min":{"1min":0.38300000000000001,"5min":0.247,"15min":0.23699999999999999},"max":{"1min":0.68999999999999995,"5min":174.69499999999999,"15min":174.69499999999999},"last":0.47199999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:45:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":3.2570000000000001,"15min":2.02},"min":{"1min":0.32900000000000001,"5min":0.28000000000000003,"15min":0.23100000000000001},"max":{"1min":0.751,"5min":95.709000000000003,"15min":95.709000000000003},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.495,"5min":3.1459999999999999,"15min":2.0920000000000001},"min":{"1min":0.36399999999999999,"5min":0.27800000000000002,"15min":0.249},"max":{"1min":0.80200000000000005,"5min":94.966999999999999,"15min":94.966999999999999},"last":0.52600000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:46:04 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.331,"15min":3.9769999999999999},"min":{"1min":0.39200000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.82599999999999996,"5min":9.5879999999999992,"15min":208.738},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.7350000000000001,"15min":3.5049999999999999},"min":{"1min":0.41199999999999998,"5min":0.313,"15min":0.23999999999999999},"max":{"1min":0.68799999999999994,"5min":23.899000000000001,"15min":208.631},"last":0.50700000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:46:04 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.335,"15min":3.1230000000000002},"min":{"1min":0.435,"5min":0.311,"15min":0.27800000000000002},"max":{"1min":1.1719999999999999,"5min":14.288,"15min":143.66300000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.357,"15min":3.2690000000000001},"min":{"1min":0.40999999999999998,"5min":0.35799999999999998,"15min":0.30399999999999999},"max":{"1min":1.157,"5min":14.307,"15min":143.62200000000001},"last":0.66500000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.60199999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.57599999999999996,"15min":1.2829999999999999},"min":{"1min":0.38700000000000001,"5min":0.29499999999999998,"15min":0.26100000000000001},"max":{"1min":0.79000000000000004,"5min":6.617,"15min":148.36600000000001},"last":0.628},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.627,"15min":1.4750000000000001},"min":{"1min":0.35299999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.81299999999999994,"5min":6.8040000000000003,"15min":148.42500000000001},"last":0.63900000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:46:04 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.52700000000000002,"15min":0.93000000000000005},"min":{"1min":0.46100000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.879,"5min":1.109,"15min":15.497},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.63400000000000001,"15min":1.0009999999999999},"min":{"1min":0.49199999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.98099999999999998,"5min":2.8690000000000002,"15min":15.653},"last":0.69199999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4160,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:46:34.815 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:46:35.164 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:46:35.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:34 smithi110 ceph-mon[212939]: from='client.38409 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:46:35.202 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:34 smithi110 ceph-mon[212939]: pgmap v2134: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:35.202 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:34 smithi110 ceph-mon[221102]: from='client.38409 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:46:35.203 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:34 smithi110 ceph-mon[221102]: pgmap v2134: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:34 smithi138 ceph-mon[193606]: from='client.38409 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:46:35.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:34 smithi138 ceph-mon[193606]: pgmap v2134: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:36.934 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-09-23T23:46:36.936 INFO:teuthology.orchestra.run.smithi110.stderr:dumped all 2023-09-23T23:46:37.115 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:46:36 smithi110 conmon[198813]: ::ffff:172.21.15.138 - - [23/Sep/2023:23:46:36] "GET /metrics HTTP/1.1" 200 34165 "" "Prometheus/2.43.0" 2023-09-23T23:46:37.452 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:37 smithi110 ceph-mon[212939]: pgmap v2135: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:37.453 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:37 smithi110 ceph-mon[221102]: pgmap v2135: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:37.573 INFO:teuthology.orchestra.run.smithi110.stdout:{"pg_ready":true,"pg_map":{"version":2135,"stamp":"2023-09-23T23:46:36.178978+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5994,"num_read_kb":13211,"num_write":907,"num_write_kb":12725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":149,"num_bytes_recovered":684618,"num_keys_recovered":30,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6838004,"kb_used_data":64308,"kb_used_omap":33,"kb_used_meta":6773598,"kb_avail":743123212,"statfs":{"total":767960285184,"available":760958169088,"internally_reserved":0,"allocated":65851392,"data_stored":49854189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34157,"internal_metadata":6936165011},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002881"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":922,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051419+0000","last_change":"2023-09-23T23:45:32.240149+0000","last_active":"2023-09-23T23:45:55.051419+0000","last_peered":"2023-09-23T23:45:55.051419+0000","last_clean":"2023-09-23T23:45:55.051419+0000","last_became_active":"2023-09-23T23:45:32.239216+0000","last_became_peered":"2023-09-23T23:45:32.239216+0000","last_unstale":"2023-09-23T23:45:55.051419+0000","last_undegraded":"2023-09-23T23:45:55.051419+0000","last_fullsized":"2023-09-23T23:45:55.051419+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:06.356716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:06.356716+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:42.973963+0000","scrub_duration":0.0069586200000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":43,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988722+0000","last_change":"2023-09-23T23:44:55.986638+0000","last_active":"2023-09-23T23:45:54.988722+0000","last_peered":"2023-09-23T23:45:54.988722+0000","last_clean":"2023-09-23T23:45:54.988722+0000","last_became_active":"2023-09-23T23:44:55.986291+0000","last_became_peered":"2023-09-23T23:44:55.986291+0000","last_unstale":"2023-09-23T23:45:54.988722+0000","last_undegraded":"2023-09-23T23:45:54.988722+0000","last_fullsized":"2023-09-23T23:45:54.988722+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:21:53.344884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":100,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:57.995142+0000","last_change":"2023-09-23T23:44:55.986672+0000","last_active":"2023-09-23T23:45:57.995142+0000","last_peered":"2023-09-23T23:45:57.995142+0000","last_clean":"2023-09-23T23:45:57.995142+0000","last_became_active":"2023-09-23T23:44:55.986200+0000","last_became_peered":"2023-09-23T23:44:55.986200+0000","last_unstale":"2023-09-23T23:45:57.995142+0000","last_undegraded":"2023-09-23T23:45:57.995142+0000","last_fullsized":"2023-09-23T23:45:57.995142+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:09.840032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":875,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052204+0000","last_change":"2023-09-23T23:45:32.240569+0000","last_active":"2023-09-23T23:45:55.052204+0000","last_peered":"2023-09-23T23:45:55.052204+0000","last_clean":"2023-09-23T23:45:55.052204+0000","last_became_active":"2023-09-23T23:45:32.240057+0000","last_became_peered":"2023-09-23T23:45:32.240057+0000","last_unstale":"2023-09-23T23:45:55.052204+0000","last_undegraded":"2023-09-23T23:45:55.052204+0000","last_fullsized":"2023-09-23T23:45:55.052204+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.459553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.459553+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:21.618355+0000","scrub_duration":0.0012968929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":377,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.985203+0000","last_change":"2023-09-23T23:45:19.799761+0000","last_active":"2023-09-23T23:45:54.985203+0000","last_peered":"2023-09-23T23:45:54.985203+0000","last_clean":"2023-09-23T23:45:54.985203+0000","last_became_active":"2023-09-23T23:45:19.799456+0000","last_became_peered":"2023-09-23T23:45:19.799456+0000","last_unstale":"2023-09-23T23:45:54.985203+0000","last_undegraded":"2023-09-23T23:45:54.985203+0000","last_fullsized":"2023-09-23T23:45:54.985203+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:11.868474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":612,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052125+0000","last_change":"2023-09-23T23:45:22.811728+0000","last_active":"2023-09-23T23:45:55.052125+0000","last_peered":"2023-09-23T23:45:55.052125+0000","last_clean":"2023-09-23T23:45:55.052125+0000","last_became_active":"2023-09-23T23:45:22.811392+0000","last_became_peered":"2023-09-23T23:45:22.811392+0000","last_unstale":"2023-09-23T23:45:55.052125+0000","last_undegraded":"2023-09-23T23:45:55.052125+0000","last_fullsized":"2023-09-23T23:45:55.052125+0000","mapping_epoch":757,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":758,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.109854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":755,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988777+0000","last_change":"2023-09-23T23:44:59.944358+0000","last_active":"2023-09-23T23:45:54.988777+0000","last_peered":"2023-09-23T23:45:54.988777+0000","last_clean":"2023-09-23T23:45:54.988777+0000","last_became_active":"2023-09-23T23:44:58.998549+0000","last_became_peered":"2023-09-23T23:44:58.998549+0000","last_unstale":"2023-09-23T23:45:54.988777+0000","last_undegraded":"2023-09-23T23:45:54.988777+0000","last_fullsized":"2023-09-23T23:45:54.988777+0000","mapping_epoch":742,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":743,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:44:59.944277+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:44:59.944277+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:34:51.937588+0000","scrub_duration":0.0017553779999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":35,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051469+0000","last_change":"2023-09-23T23:45:08.975463+0000","last_active":"2023-09-23T23:45:55.051469+0000","last_peered":"2023-09-23T23:45:55.051469+0000","last_clean":"2023-09-23T23:45:55.051469+0000","last_became_active":"2023-09-23T23:45:08.975141+0000","last_became_peered":"2023-09-23T23:45:08.975141+0000","last_unstale":"2023-09-23T23:45:55.051469+0000","last_undegraded":"2023-09-23T23:45:55.051469+0000","last_fullsized":"2023-09-23T23:45:55.051469+0000","mapping_epoch":748,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":749,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:54.909147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":39,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.054055+0000","last_change":"2023-09-23T23:44:59.991084+0000","last_active":"2023-09-23T23:45:55.054055+0000","last_peered":"2023-09-23T23:45:55.054055+0000","last_clean":"2023-09-23T23:45:55.054055+0000","last_became_active":"2023-09-23T23:44:59.990783+0000","last_became_peered":"2023-09-23T23:44:59.990783+0000","last_unstale":"2023-09-23T23:45:55.054055+0000","last_undegraded":"2023-09-23T23:45:55.054055+0000","last_fullsized":"2023-09-23T23:45:55.054055+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:22.521680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":1088,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052362+0000","last_change":"2023-09-23T23:45:52.247423+0000","last_active":"2023-09-23T23:45:55.052362+0000","last_peered":"2023-09-23T23:45:55.052362+0000","last_clean":"2023-09-23T23:45:55.052362+0000","last_became_active":"2023-09-23T23:45:52.246596+0000","last_became_peered":"2023-09-23T23:45:52.246596+0000","last_unstale":"2023-09-23T23:45:55.052362+0000","last_undegraded":"2023-09-23T23:45:55.052362+0000","last_fullsized":"2023-09-23T23:45:55.052362+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:45:33.483204+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:45:33.483204+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:38.294415+0000","scrub_duration":0.0016975289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":388,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052479+0000","last_change":"2023-09-23T23:45:12.996691+0000","last_active":"2023-09-23T23:45:55.052479+0000","last_peered":"2023-09-23T23:45:55.052479+0000","last_clean":"2023-09-23T23:45:55.052479+0000","last_became_active":"2023-09-23T23:45:12.996313+0000","last_became_peered":"2023-09-23T23:45:12.996313+0000","last_unstale":"2023-09-23T23:45:55.052479+0000","last_undegraded":"2023-09-23T23:45:55.052479+0000","last_fullsized":"2023-09-23T23:45:55.052479+0000","mapping_epoch":752,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":753,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:45:41.498031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":109,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:57.994672+0000","last_change":"2023-09-23T23:44:59.986735+0000","last_active":"2023-09-23T23:45:57.994672+0000","last_peered":"2023-09-23T23:45:57.994672+0000","last_clean":"2023-09-23T23:45:57.994672+0000","last_became_active":"2023-09-23T23:44:59.986416+0000","last_became_peered":"2023-09-23T23:44:59.986416+0000","last_unstale":"2023-09-23T23:45:57.994672+0000","last_undegraded":"2023-09-23T23:45:57.994672+0000","last_fullsized":"2023-09-23T23:45:57.994672+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:37.959581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":1087,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.985797+0000","last_change":"2023-09-23T23:44:58.994659+0000","last_active":"2023-09-23T23:45:54.985797+0000","last_peered":"2023-09-23T23:45:54.985797+0000","last_clean":"2023-09-23T23:45:54.985797+0000","last_became_active":"2023-09-23T23:44:58.994260+0000","last_became_peered":"2023-09-23T23:44:58.994260+0000","last_unstale":"2023-09-23T23:45:54.985797+0000","last_undegraded":"2023-09-23T23:45:54.985797+0000","last_fullsized":"2023-09-23T23:45:54.985797+0000","mapping_epoch":742,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":743,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.137407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.137407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:55.907214+0000","scrub_duration":0.045338919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":414,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.053172+0000","last_change":"2023-09-23T23:42:46.645764+0000","last_active":"2023-09-23T23:45:55.053172+0000","last_peered":"2023-09-23T23:45:55.053172+0000","last_clean":"2023-09-23T23:45:55.053172+0000","last_became_active":"2023-09-23T23:42:46.645462+0000","last_became_peered":"2023-09-23T23:42:46.645462+0000","last_unstale":"2023-09-23T23:45:55.053172+0000","last_undegraded":"2023-09-23T23:45:55.053172+0000","last_fullsized":"2023-09-23T23:45:55.053172+0000","mapping_epoch":676,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":677,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:04.694952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":100,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:57.994803+0000","last_change":"2023-09-23T23:44:59.987681+0000","last_active":"2023-09-23T23:45:57.994803+0000","last_peered":"2023-09-23T23:45:57.994803+0000","last_clean":"2023-09-23T23:45:57.994803+0000","last_became_active":"2023-09-23T23:44:59.987328+0000","last_became_peered":"2023-09-23T23:44:59.987328+0000","last_unstale":"2023-09-23T23:45:57.994803+0000","last_undegraded":"2023-09-23T23:45:57.994803+0000","last_fullsized":"2023-09-23T23:45:57.994803+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:11.777866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":1237,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.053209+0000","last_change":"2023-09-23T23:45:32.240700+0000","last_active":"2023-09-23T23:45:55.053209+0000","last_peered":"2023-09-23T23:45:55.053209+0000","last_clean":"2023-09-23T23:45:55.053209+0000","last_became_active":"2023-09-23T23:45:32.240319+0000","last_became_peered":"2023-09-23T23:45:32.240319+0000","last_unstale":"2023-09-23T23:45:55.053209+0000","last_undegraded":"2023-09-23T23:45:55.053209+0000","last_fullsized":"2023-09-23T23:45:55.053209+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:09.362990+0000","last_clean_scrub_stamp":"2023-09-23T23:28:09.362990+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:22.146123+0000","scrub_duration":0.011155854999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":200,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051737+0000","last_change":"2023-09-23T23:45:52.245063+0000","last_active":"2023-09-23T23:45:55.051737+0000","last_peered":"2023-09-23T23:45:55.051737+0000","last_clean":"2023-09-23T23:45:55.051737+0000","last_became_active":"2023-09-23T23:45:52.244601+0000","last_became_peered":"2023-09-23T23:45:52.244601+0000","last_unstale":"2023-09-23T23:45:55.051737+0000","last_undegraded":"2023-09-23T23:45:55.051737+0000","last_fullsized":"2023-09-23T23:45:55.051737+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:12.350381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":445,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.986820+0000","last_change":"2023-09-23T23:45:19.799305+0000","last_active":"2023-09-23T23:45:54.986820+0000","last_peered":"2023-09-23T23:45:54.986820+0000","last_clean":"2023-09-23T23:45:54.986820+0000","last_became_active":"2023-09-23T23:45:19.798819+0000","last_became_peered":"2023-09-23T23:45:19.798819+0000","last_unstale":"2023-09-23T23:45:54.986820+0000","last_undegraded":"2023-09-23T23:45:54.986820+0000","last_fullsized":"2023-09-23T23:45:54.986820+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:48.039469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":694,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.986028+0000","last_change":"2023-09-23T23:45:19.799651+0000","last_active":"2023-09-23T23:45:54.986028+0000","last_peered":"2023-09-23T23:45:54.986028+0000","last_clean":"2023-09-23T23:45:54.986028+0000","last_became_active":"2023-09-23T23:45:19.799301+0000","last_became_peered":"2023-09-23T23:45:19.799301+0000","last_unstale":"2023-09-23T23:45:54.986028+0000","last_undegraded":"2023-09-23T23:45:54.986028+0000","last_fullsized":"2023-09-23T23:45:54.986028+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:45:14.789802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:45:14.789802+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:17.255907+0000","scrub_duration":0.0018535310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":73,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988906+0000","last_change":"2023-09-23T23:45:00.005548+0000","last_active":"2023-09-23T23:45:54.988906+0000","last_peered":"2023-09-23T23:45:54.988906+0000","last_clean":"2023-09-23T23:45:54.988906+0000","last_became_active":"2023-09-23T23:44:59.992651+0000","last_became_peered":"2023-09-23T23:44:59.992651+0000","last_unstale":"2023-09-23T23:45:54.988906+0000","last_undegraded":"2023-09-23T23:45:54.988906+0000","last_fullsized":"2023-09-23T23:45:54.988906+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:01.163903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":53,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988934+0000","last_change":"2023-09-23T23:44:59.994663+0000","last_active":"2023-09-23T23:45:54.988934+0000","last_peered":"2023-09-23T23:45:54.988934+0000","last_clean":"2023-09-23T23:45:54.988934+0000","last_became_active":"2023-09-23T23:44:59.994425+0000","last_became_peered":"2023-09-23T23:44:59.994425+0000","last_unstale":"2023-09-23T23:45:54.988934+0000","last_undegraded":"2023-09-23T23:45:54.988934+0000","last_fullsized":"2023-09-23T23:45:54.988934+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:22.831850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":1001,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051858+0000","last_change":"2023-09-23T23:45:12.996741+0000","last_active":"2023-09-23T23:45:55.051858+0000","last_peered":"2023-09-23T23:45:55.051858+0000","last_clean":"2023-09-23T23:45:55.051858+0000","last_became_active":"2023-09-23T23:45:12.996317+0000","last_became_peered":"2023-09-23T23:45:12.996317+0000","last_unstale":"2023-09-23T23:45:55.051858+0000","last_undegraded":"2023-09-23T23:45:55.051858+0000","last_fullsized":"2023-09-23T23:45:55.051858+0000","mapping_epoch":752,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":753,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-09-23T23:28:14.378257+0000","last_clean_scrub_stamp":"2023-09-23T23:28:14.378257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:19.589426+0000","scrub_duration":0.0017680059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":527,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051789+0000","last_change":"2023-09-23T23:45:52.245725+0000","last_active":"2023-09-23T23:45:55.051789+0000","last_peered":"2023-09-23T23:45:55.051789+0000","last_clean":"2023-09-23T23:45:55.051789+0000","last_became_active":"2023-09-23T23:45:52.245190+0000","last_became_peered":"2023-09-23T23:45:52.245190+0000","last_unstale":"2023-09-23T23:45:55.051789+0000","last_undegraded":"2023-09-23T23:45:55.051789+0000","last_fullsized":"2023-09-23T23:45:55.051789+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:12.914713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":91,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:57.995037+0000","last_change":"2023-09-23T23:43:37.337600+0000","last_active":"2023-09-23T23:45:57.995037+0000","last_peered":"2023-09-23T23:45:57.995037+0000","last_clean":"2023-09-23T23:45:57.995037+0000","last_became_active":"2023-09-23T23:43:37.337098+0000","last_became_peered":"2023-09-23T23:43:37.337098+0000","last_unstale":"2023-09-23T23:45:57.995037+0000","last_undegraded":"2023-09-23T23:45:57.995037+0000","last_fullsized":"2023-09-23T23:45:57.995037+0000","mapping_epoch":692,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":693,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:40.723292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":876,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051296+0000","last_change":"2023-09-23T23:45:32.239630+0000","last_active":"2023-09-23T23:45:55.051296+0000","last_peered":"2023-09-23T23:45:55.051296+0000","last_clean":"2023-09-23T23:45:55.051296+0000","last_became_active":"2023-09-23T23:45:32.238728+0000","last_became_peered":"2023-09-23T23:45:32.238728+0000","last_unstale":"2023-09-23T23:45:55.051296+0000","last_undegraded":"2023-09-23T23:45:55.051296+0000","last_fullsized":"2023-09-23T23:45:55.051296+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.876817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.876817+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:23.346950+0000","scrub_duration":0.0014661159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":240,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051694+0000","last_change":"2023-09-23T23:45:22.812226+0000","last_active":"2023-09-23T23:45:55.051694+0000","last_peered":"2023-09-23T23:45:55.051694+0000","last_clean":"2023-09-23T23:45:55.051694+0000","last_became_active":"2023-09-23T23:45:22.811926+0000","last_became_peered":"2023-09-23T23:45:22.811926+0000","last_unstale":"2023-09-23T23:45:55.051694+0000","last_undegraded":"2023-09-23T23:45:55.051694+0000","last_fullsized":"2023-09-23T23:45:55.051694+0000","mapping_epoch":757,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":758,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:17:43.890183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":321,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051511+0000","last_change":"2023-09-23T23:45:32.242077+0000","last_active":"2023-09-23T23:45:55.051511+0000","last_peered":"2023-09-23T23:45:55.051511+0000","last_clean":"2023-09-23T23:45:55.051511+0000","last_became_active":"2023-09-23T23:45:32.240435+0000","last_became_peered":"2023-09-23T23:45:32.240435+0000","last_unstale":"2023-09-23T23:45:55.051511+0000","last_undegraded":"2023-09-23T23:45:55.051511+0000","last_fullsized":"2023-09-23T23:45:55.051511+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:20.255228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":814,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.985773+0000","last_change":"2023-09-23T23:45:25.984337+0000","last_active":"2023-09-23T23:45:54.985773+0000","last_peered":"2023-09-23T23:45:54.985773+0000","last_clean":"2023-09-23T23:45:54.985773+0000","last_became_active":"2023-09-23T23:45:25.984028+0000","last_became_peered":"2023-09-23T23:45:25.984028+0000","last_unstale":"2023-09-23T23:45:54.985773+0000","last_undegraded":"2023-09-23T23:45:54.985773+0000","last_fullsized":"2023-09-23T23:45:54.985773+0000","mapping_epoch":760,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":761,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:24:05.304836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:24:05.304836+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:44.207154+0000","scrub_duration":0.0071485029999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":528,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052489+0000","last_change":"2023-09-23T23:45:52.242787+0000","last_active":"2023-09-23T23:45:55.052489+0000","last_peered":"2023-09-23T23:45:55.052489+0000","last_clean":"2023-09-23T23:45:55.052489+0000","last_became_active":"2023-09-23T23:45:52.242370+0000","last_became_peered":"2023-09-23T23:45:52.242370+0000","last_unstale":"2023-09-23T23:45:55.052489+0000","last_undegraded":"2023-09-23T23:45:55.052489+0000","last_fullsized":"2023-09-23T23:45:55.052489+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:45:25.958298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":107,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051478+0000","last_change":"2023-09-23T23:45:12.996789+0000","last_active":"2023-09-23T23:45:55.051478+0000","last_peered":"2023-09-23T23:45:55.051478+0000","last_clean":"2023-09-23T23:45:55.051478+0000","last_became_active":"2023-09-23T23:45:12.996316+0000","last_became_peered":"2023-09-23T23:45:12.996316+0000","last_unstale":"2023-09-23T23:45:55.051478+0000","last_undegraded":"2023-09-23T23:45:55.051478+0000","last_fullsized":"2023-09-23T23:45:55.051478+0000","mapping_epoch":752,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":753,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:18.402324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":1216,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051683+0000","last_change":"2023-09-23T23:45:03.033754+0000","last_active":"2023-09-23T23:45:55.051683+0000","last_peered":"2023-09-23T23:45:55.051683+0000","last_clean":"2023-09-23T23:45:55.051683+0000","last_became_active":"2023-09-23T23:45:03.033401+0000","last_became_peered":"2023-09-23T23:45:03.033401+0000","last_unstale":"2023-09-23T23:45:55.051683+0000","last_undegraded":"2023-09-23T23:45:55.051683+0000","last_fullsized":"2023-09-23T23:45:55.051683+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:48.533450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:48.533450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:27.436501+0000","scrub_duration":0.0014917929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":69,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.053466+0000","last_change":"2023-09-23T23:45:08.995797+0000","last_active":"2023-09-23T23:45:55.053466+0000","last_peered":"2023-09-23T23:45:55.053466+0000","last_clean":"2023-09-23T23:45:55.053466+0000","last_became_active":"2023-09-23T23:45:08.982500+0000","last_became_peered":"2023-09-23T23:45:08.982500+0000","last_unstale":"2023-09-23T23:45:55.053466+0000","last_undegraded":"2023-09-23T23:45:55.053466+0000","last_fullsized":"2023-09-23T23:45:55.053466+0000","mapping_epoch":748,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":749,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:56.451333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":26,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052783+0000","last_change":"2023-09-23T23:45:22.812030+0000","last_active":"2023-09-23T23:45:55.052783+0000","last_peered":"2023-09-23T23:45:55.052783+0000","last_clean":"2023-09-23T23:45:55.052783+0000","last_became_active":"2023-09-23T23:45:22.811724+0000","last_became_peered":"2023-09-23T23:45:22.811724+0000","last_unstale":"2023-09-23T23:45:55.052783+0000","last_undegraded":"2023-09-23T23:45:55.052783+0000","last_fullsized":"2023-09-23T23:45:55.052783+0000","mapping_epoch":757,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":758,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:19:17.650196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":926,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051854+0000","last_change":"2023-09-23T23:45:52.243550+0000","last_active":"2023-09-23T23:45:55.051854+0000","last_peered":"2023-09-23T23:45:55.051854+0000","last_clean":"2023-09-23T23:45:55.051854+0000","last_became_active":"2023-09-23T23:45:52.242990+0000","last_became_peered":"2023-09-23T23:45:52.242990+0000","last_unstale":"2023-09-23T23:45:55.051854+0000","last_undegraded":"2023-09-23T23:45:55.051854+0000","last_fullsized":"2023-09-23T23:45:55.051854+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:57.266973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:57.266973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:15.663145+0000","scrub_duration":0.012443027000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":375,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052855+0000","last_change":"2023-09-23T23:45:52.243970+0000","last_active":"2023-09-23T23:45:55.052855+0000","last_peered":"2023-09-23T23:45:55.052855+0000","last_clean":"2023-09-23T23:45:55.052855+0000","last_became_active":"2023-09-23T23:45:52.243703+0000","last_became_peered":"2023-09-23T23:45:52.243703+0000","last_unstale":"2023-09-23T23:45:55.052855+0000","last_undegraded":"2023-09-23T23:45:55.052855+0000","last_fullsized":"2023-09-23T23:45:55.052855+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:23.366439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":387,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051363+0000","last_change":"2023-09-23T23:45:32.239212+0000","last_active":"2023-09-23T23:45:55.051363+0000","last_peered":"2023-09-23T23:45:55.051363+0000","last_clean":"2023-09-23T23:45:55.051363+0000","last_became_active":"2023-09-23T23:45:32.238690+0000","last_became_peered":"2023-09-23T23:45:32.238690+0000","last_unstale":"2023-09-23T23:45:55.051363+0000","last_undegraded":"2023-09-23T23:45:55.051363+0000","last_fullsized":"2023-09-23T23:45:55.051363+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:14.425469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":799,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:57.995011+0000","last_change":"2023-09-23T23:45:03.038443+0000","last_active":"2023-09-23T23:45:57.995011+0000","last_peered":"2023-09-23T23:45:57.995011+0000","last_clean":"2023-09-23T23:45:57.995011+0000","last_became_active":"2023-09-23T23:45:03.038076+0000","last_became_peered":"2023-09-23T23:45:03.038076+0000","last_unstale":"2023-09-23T23:45:57.995011+0000","last_undegraded":"2023-09-23T23:45:57.995011+0000","last_fullsized":"2023-09-23T23:45:57.995011+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:43:40.189087+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:43:40.189087+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:51:18.436998+0000","scrub_duration":0.001887503,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":425,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052713+0000","last_change":"2023-09-23T23:45:52.243030+0000","last_active":"2023-09-23T23:45:55.052713+0000","last_peered":"2023-09-23T23:45:55.052713+0000","last_clean":"2023-09-23T23:45:55.052713+0000","last_became_active":"2023-09-23T23:45:52.242318+0000","last_became_peered":"2023-09-23T23:45:52.242318+0000","last_unstale":"2023-09-23T23:45:55.052713+0000","last_undegraded":"2023-09-23T23:45:55.052713+0000","last_fullsized":"2023-09-23T23:45:55.052713+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:48:15.424032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":63,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988447+0000","last_change":"2023-09-23T23:45:22.814032+0000","last_active":"2023-09-23T23:45:54.988447+0000","last_peered":"2023-09-23T23:45:54.988447+0000","last_clean":"2023-09-23T23:45:54.988447+0000","last_became_active":"2023-09-23T23:45:22.813596+0000","last_became_peered":"2023-09-23T23:45:22.813596+0000","last_unstale":"2023-09-23T23:45:54.988447+0000","last_undegraded":"2023-09-23T23:45:54.988447+0000","last_fullsized":"2023-09-23T23:45:54.988447+0000","mapping_epoch":757,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":758,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:04.270071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":845,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.054146+0000","last_change":"2023-09-23T23:45:03.041640+0000","last_active":"2023-09-23T23:45:55.054146+0000","last_peered":"2023-09-23T23:45:55.054146+0000","last_clean":"2023-09-23T23:45:55.054146+0000","last_became_active":"2023-09-23T23:45:03.040104+0000","last_became_peered":"2023-09-23T23:45:03.040104+0000","last_unstale":"2023-09-23T23:45:55.054146+0000","last_undegraded":"2023-09-23T23:45:55.054146+0000","last_fullsized":"2023-09-23T23:45:55.054146+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:29:15.716489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:29:15.716489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:35.920724+0000","scrub_duration":0.0072989020000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":499,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052121+0000","last_change":"2023-09-23T23:45:52.247754+0000","last_active":"2023-09-23T23:45:55.052121+0000","last_peered":"2023-09-23T23:45:55.052121+0000","last_clean":"2023-09-23T23:45:55.052121+0000","last_became_active":"2023-09-23T23:45:52.247004+0000","last_became_peered":"2023-09-23T23:45:52.247004+0000","last_unstale":"2023-09-23T23:45:55.052121+0000","last_undegraded":"2023-09-23T23:45:55.052121+0000","last_fullsized":"2023-09-23T23:45:55.052121+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:39:41.941465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":39,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.053541+0000","last_change":"2023-09-23T23:44:59.996784+0000","last_active":"2023-09-23T23:45:55.053541+0000","last_peered":"2023-09-23T23:45:55.053541+0000","last_clean":"2023-09-23T23:45:55.053541+0000","last_became_active":"2023-09-23T23:44:59.996229+0000","last_became_peered":"2023-09-23T23:44:59.996229+0000","last_unstale":"2023-09-23T23:45:55.053541+0000","last_undegraded":"2023-09-23T23:45:55.053541+0000","last_fullsized":"2023-09-23T23:45:55.053541+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:52.941708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":533,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988976+0000","last_change":"2023-09-23T23:44:59.000492+0000","last_active":"2023-09-23T23:45:54.988976+0000","last_peered":"2023-09-23T23:45:54.988976+0000","last_clean":"2023-09-23T23:45:54.988976+0000","last_became_active":"2023-09-23T23:44:59.000157+0000","last_became_peered":"2023-09-23T23:44:59.000157+0000","last_unstale":"2023-09-23T23:45:54.988976+0000","last_undegraded":"2023-09-23T23:45:54.988976+0000","last_fullsized":"2023-09-23T23:45:54.988976+0000","mapping_epoch":742,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":743,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:43:42.149317+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:43:42.149317+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:06.104330+0000","scrub_duration":0.0023427560000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":513,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051521+0000","last_change":"2023-09-23T23:45:32.240799+0000","last_active":"2023-09-23T23:45:55.051521+0000","last_peered":"2023-09-23T23:45:55.051521+0000","last_clean":"2023-09-23T23:45:55.051521+0000","last_became_active":"2023-09-23T23:45:32.239349+0000","last_became_peered":"2023-09-23T23:45:32.239349+0000","last_unstale":"2023-09-23T23:45:55.051521+0000","last_undegraded":"2023-09-23T23:45:55.051521+0000","last_fullsized":"2023-09-23T23:45:55.051521+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:33.985138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":800,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052286+0000","last_change":"2023-09-23T23:44:55.985533+0000","last_active":"2023-09-23T23:45:55.052286+0000","last_peered":"2023-09-23T23:45:55.052286+0000","last_clean":"2023-09-23T23:45:55.052286+0000","last_became_active":"2023-09-23T23:44:55.985154+0000","last_became_peered":"2023-09-23T23:44:55.985154+0000","last_unstale":"2023-09-23T23:45:55.052286+0000","last_undegraded":"2023-09-23T23:45:55.052286+0000","last_fullsized":"2023-09-23T23:45:55.052286+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:31.396248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":914,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052540+0000","last_change":"2023-09-23T23:45:52.245509+0000","last_active":"2023-09-23T23:45:55.052540+0000","last_peered":"2023-09-23T23:45:55.052540+0000","last_clean":"2023-09-23T23:45:55.052540+0000","last_became_active":"2023-09-23T23:45:52.245064+0000","last_became_peered":"2023-09-23T23:45:52.245064+0000","last_unstale":"2023-09-23T23:45:55.052540+0000","last_undegraded":"2023-09-23T23:45:55.052540+0000","last_fullsized":"2023-09-23T23:45:55.052540+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:43:27.592049+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:43:27.592049+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:03:17.167970+0000","scrub_duration":0.001659259,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":134,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051834+0000","last_change":"2023-09-23T23:45:08.989599+0000","last_active":"2023-09-23T23:45:55.051834+0000","last_peered":"2023-09-23T23:45:55.051834+0000","last_clean":"2023-09-23T23:45:55.051834+0000","last_became_active":"2023-09-23T23:45:08.981090+0000","last_became_peered":"2023-09-23T23:45:08.981090+0000","last_unstale":"2023-09-23T23:45:55.051834+0000","last_undegraded":"2023-09-23T23:45:55.051834+0000","last_fullsized":"2023-09-23T23:45:55.051834+0000","mapping_epoch":748,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":749,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:13.105965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":53,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988886+0000","last_change":"2023-09-23T23:44:59.997689+0000","last_active":"2023-09-23T23:45:54.988886+0000","last_peered":"2023-09-23T23:45:54.988886+0000","last_clean":"2023-09-23T23:45:54.988886+0000","last_became_active":"2023-09-23T23:44:59.997370+0000","last_became_peered":"2023-09-23T23:44:59.997370+0000","last_unstale":"2023-09-23T23:45:54.988886+0000","last_undegraded":"2023-09-23T23:45:54.988886+0000","last_fullsized":"2023-09-23T23:45:54.988886+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:46:54.721468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":684,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051252+0000","last_change":"2023-09-23T23:45:32.241274+0000","last_active":"2023-09-23T23:45:55.051252+0000","last_peered":"2023-09-23T23:45:55.051252+0000","last_clean":"2023-09-23T23:45:55.051252+0000","last_became_active":"2023-09-23T23:45:32.240417+0000","last_became_peered":"2023-09-23T23:45:32.240417+0000","last_unstale":"2023-09-23T23:45:55.051252+0000","last_undegraded":"2023-09-23T23:45:55.051252+0000","last_fullsized":"2023-09-23T23:45:55.051252+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.862562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.862562+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:44.056296+0000","scrub_duration":0.001586018,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":39,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.053782+0000","last_change":"2023-09-23T23:44:59.988998+0000","last_active":"2023-09-23T23:45:55.053782+0000","last_peered":"2023-09-23T23:45:55.053782+0000","last_clean":"2023-09-23T23:45:55.053782+0000","last_became_active":"2023-09-23T23:44:59.988635+0000","last_became_peered":"2023-09-23T23:44:59.988635+0000","last_unstale":"2023-09-23T23:45:55.053782+0000","last_undegraded":"2023-09-23T23:45:55.053782+0000","last_fullsized":"2023-09-23T23:45:55.053782+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:06:39.231374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":43,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.989121+0000","last_change":"2023-09-23T23:44:55.990783+0000","last_active":"2023-09-23T23:45:54.989121+0000","last_peered":"2023-09-23T23:45:54.989121+0000","last_clean":"2023-09-23T23:45:54.989121+0000","last_became_active":"2023-09-23T23:44:55.990445+0000","last_became_peered":"2023-09-23T23:44:55.990445+0000","last_unstale":"2023-09-23T23:45:54.989121+0000","last_undegraded":"2023-09-23T23:45:54.989121+0000","last_fullsized":"2023-09-23T23:45:54.989121+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:21.475623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":819,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:58.452852+0000","last_change":"2023-09-23T23:45:19.800745+0000","last_active":"2023-09-23T23:45:58.452852+0000","last_peered":"2023-09-23T23:45:58.452852+0000","last_clean":"2023-09-23T23:45:58.452852+0000","last_became_active":"2023-09-23T23:45:19.800473+0000","last_became_peered":"2023-09-23T23:45:19.800473+0000","last_unstale":"2023-09-23T23:45:58.452852+0000","last_undegraded":"2023-09-23T23:45:58.452852+0000","last_fullsized":"2023-09-23T23:45:58.452852+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":310,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.178592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.178592+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:45.672099+0000","scrub_duration":0.0024270810000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":43,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988533+0000","last_change":"2023-09-23T23:44:56.001669+0000","last_active":"2023-09-23T23:45:54.988533+0000","last_peered":"2023-09-23T23:45:54.988533+0000","last_clean":"2023-09-23T23:45:54.988533+0000","last_became_active":"2023-09-23T23:44:56.001107+0000","last_became_peered":"2023-09-23T23:44:56.001107+0000","last_unstale":"2023-09-23T23:45:54.988533+0000","last_undegraded":"2023-09-23T23:45:54.988533+0000","last_fullsized":"2023-09-23T23:45:54.988533+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:01.851610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":53,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.989213+0000","last_change":"2023-09-23T23:44:59.995182+0000","last_active":"2023-09-23T23:45:54.989213+0000","last_peered":"2023-09-23T23:45:54.989213+0000","last_clean":"2023-09-23T23:45:54.989213+0000","last_became_active":"2023-09-23T23:44:59.994869+0000","last_became_peered":"2023-09-23T23:44:59.994869+0000","last_unstale":"2023-09-23T23:45:54.989213+0000","last_undegraded":"2023-09-23T23:45:54.989213+0000","last_fullsized":"2023-09-23T23:45:54.989213+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:35.582441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":625,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051589+0000","last_change":"2023-09-23T23:45:32.526297+0000","last_active":"2023-09-23T23:45:55.051589+0000","last_peered":"2023-09-23T23:45:55.051589+0000","last_clean":"2023-09-23T23:45:55.051589+0000","last_became_active":"2023-09-23T23:45:32.240832+0000","last_became_peered":"2023-09-23T23:45:32.240832+0000","last_unstale":"2023-09-23T23:45:55.051589+0000","last_undegraded":"2023-09-23T23:45:55.051589+0000","last_fullsized":"2023-09-23T23:45:55.051589+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:45:32.526219+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:45:32.526219+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:42:01.551486+0000","scrub_duration":0.001811336,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":26,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051300+0000","last_change":"2023-09-23T23:45:22.812891+0000","last_active":"2023-09-23T23:45:55.051300+0000","last_peered":"2023-09-23T23:45:55.051300+0000","last_clean":"2023-09-23T23:45:55.051300+0000","last_became_active":"2023-09-23T23:45:22.812026+0000","last_became_peered":"2023-09-23T23:45:22.812026+0000","last_unstale":"2023-09-23T23:45:55.051300+0000","last_undegraded":"2023-09-23T23:45:55.051300+0000","last_fullsized":"2023-09-23T23:45:55.051300+0000","mapping_epoch":757,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":758,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:39:57.403005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":542,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.986311+0000","last_change":"2023-09-23T23:45:19.799247+0000","last_active":"2023-09-23T23:45:54.986311+0000","last_peered":"2023-09-23T23:45:54.986311+0000","last_clean":"2023-09-23T23:45:54.986311+0000","last_became_active":"2023-09-23T23:45:19.798967+0000","last_became_peered":"2023-09-23T23:45:19.798967+0000","last_unstale":"2023-09-23T23:45:54.986311+0000","last_undegraded":"2023-09-23T23:45:54.986311+0000","last_fullsized":"2023-09-23T23:45:54.986311+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:01.817851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":953,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.054204+0000","last_change":"2023-09-23T23:45:25.990842+0000","last_active":"2023-09-23T23:45:55.054204+0000","last_peered":"2023-09-23T23:45:55.054204+0000","last_clean":"2023-09-23T23:45:55.054204+0000","last_became_active":"2023-09-23T23:45:25.990381+0000","last_became_peered":"2023-09-23T23:45:25.990381+0000","last_unstale":"2023-09-23T23:45:55.054204+0000","last_undegraded":"2023-09-23T23:45:55.054204+0000","last_fullsized":"2023-09-23T23:45:55.054204+0000","mapping_epoch":760,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":761,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:28:11.839577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:28:11.839577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:21.240299+0000","scrub_duration":0.001707702,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"716'77","reported_seq":786,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.986776+0000","last_change":"2023-09-23T23:45:19.801381+0000","last_active":"2023-09-23T23:45:54.986776+0000","last_peered":"2023-09-23T23:45:54.986776+0000","last_clean":"2023-09-23T23:45:54.986776+0000","last_became_active":"2023-09-23T23:45:19.800975+0000","last_became_peered":"2023-09-23T23:45:19.800975+0000","last_unstale":"2023-09-23T23:45:54.986776+0000","last_undegraded":"2023-09-23T23:45:54.986776+0000","last_fullsized":"2023-09-23T23:45:54.986776+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":77,"log_dups_size":0,"ondisk_log_size":77,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:20:28.335306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033286999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":355,"num_read_kb":483,"num_write":81,"num_write_kb":122,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":761,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.053076+0000","last_change":"2023-09-23T23:45:32.243261+0000","last_active":"2023-09-23T23:45:55.053076+0000","last_peered":"2023-09-23T23:45:55.053076+0000","last_clean":"2023-09-23T23:45:55.053076+0000","last_became_active":"2023-09-23T23:45:32.242919+0000","last_became_peered":"2023-09-23T23:45:32.242919+0000","last_unstale":"2023-09-23T23:45:55.053076+0000","last_undegraded":"2023-09-23T23:45:55.053076+0000","last_fullsized":"2023-09-23T23:45:55.053076+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:20.140239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":386,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.986735+0000","last_change":"2023-09-23T23:45:19.801084+0000","last_active":"2023-09-23T23:45:54.986735+0000","last_peered":"2023-09-23T23:45:54.986735+0000","last_clean":"2023-09-23T23:45:54.986735+0000","last_became_active":"2023-09-23T23:45:19.800739+0000","last_became_peered":"2023-09-23T23:45:19.800739+0000","last_unstale":"2023-09-23T23:45:54.986735+0000","last_undegraded":"2023-09-23T23:45:54.986735+0000","last_fullsized":"2023-09-23T23:45:54.986735+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:49.105815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":625,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051422+0000","last_change":"2023-09-23T23:45:35.481801+0000","last_active":"2023-09-23T23:45:55.051422+0000","last_peered":"2023-09-23T23:45:55.051422+0000","last_clean":"2023-09-23T23:45:55.051422+0000","last_became_active":"2023-09-23T23:45:32.239773+0000","last_became_peered":"2023-09-23T23:45:32.239773+0000","last_unstale":"2023-09-23T23:45:55.051422+0000","last_undegraded":"2023-09-23T23:45:55.051422+0000","last_fullsized":"2023-09-23T23:45:55.051422+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:45:35.481699+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:45:35.481699+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:59.093100+0000","scrub_duration":0.0020465320000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"715'48","reported_seq":795,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051485+0000","last_change":"2023-09-23T23:45:32.241569+0000","last_active":"2023-09-23T23:45:55.051485+0000","last_peered":"2023-09-23T23:45:55.051485+0000","last_clean":"2023-09-23T23:45:55.051485+0000","last_became_active":"2023-09-23T23:45:32.240975+0000","last_became_peered":"2023-09-23T23:45:32.240975+0000","last_unstale":"2023-09-23T23:45:55.051485+0000","last_undegraded":"2023-09-23T23:45:55.051485+0000","last_fullsized":"2023-09-23T23:45:55.051485+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":48,"log_dups_size":0,"ondisk_log_size":48,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:48.198846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039284299999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":440,"num_read_kb":682,"num_write":42,"num_write_kb":162,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":53,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.989118+0000","last_change":"2023-09-23T23:44:59.991849+0000","last_active":"2023-09-23T23:45:54.989118+0000","last_peered":"2023-09-23T23:45:54.989118+0000","last_clean":"2023-09-23T23:45:54.989118+0000","last_became_active":"2023-09-23T23:44:59.991523+0000","last_became_peered":"2023-09-23T23:44:59.991523+0000","last_unstale":"2023-09-23T23:45:54.989118+0000","last_undegraded":"2023-09-23T23:45:54.989118+0000","last_fullsized":"2023-09-23T23:45:54.989118+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:48:44.930913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":367,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.986580+0000","last_change":"2023-09-23T23:45:19.800908+0000","last_active":"2023-09-23T23:45:54.986580+0000","last_peered":"2023-09-23T23:45:54.986580+0000","last_clean":"2023-09-23T23:45:54.986580+0000","last_became_active":"2023-09-23T23:45:19.800226+0000","last_became_peered":"2023-09-23T23:45:19.800226+0000","last_unstale":"2023-09-23T23:45:54.986580+0000","last_undegraded":"2023-09-23T23:45:54.986580+0000","last_fullsized":"2023-09-23T23:45:54.986580+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.081896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":1096,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.985575+0000","last_change":"2023-09-23T23:44:58.990040+0000","last_active":"2023-09-23T23:45:54.985575+0000","last_peered":"2023-09-23T23:45:54.985575+0000","last_clean":"2023-09-23T23:45:54.985575+0000","last_became_active":"2023-09-23T23:44:58.989206+0000","last_became_peered":"2023-09-23T23:44:58.989206+0000","last_unstale":"2023-09-23T23:45:54.985575+0000","last_undegraded":"2023-09-23T23:45:54.985575+0000","last_fullsized":"2023-09-23T23:45:54.985575+0000","mapping_epoch":742,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":743,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:52.342058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:52.342058+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:39.172287+0000","scrub_duration":0.0050123399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"716'117","reported_seq":871,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.985345+0000","last_change":"2023-09-23T23:44:04.966130+0000","last_active":"2023-09-23T23:45:54.985345+0000","last_peered":"2023-09-23T23:45:54.985345+0000","last_clean":"2023-09-23T23:45:54.985345+0000","last_became_active":"2023-09-23T23:43:37.337816+0000","last_became_peered":"2023-09-23T23:43:37.337816+0000","last_unstale":"2023-09-23T23:45:54.985345+0000","last_undegraded":"2023-09-23T23:45:54.985345+0000","last_fullsized":"2023-09-23T23:45:54.985345+0000","mapping_epoch":692,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":693,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:42.250477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092634300000000004,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":558,"num_read_kb":1548,"num_write":124,"num_write_kb":5249,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":434,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.985317+0000","last_change":"2023-09-23T23:45:19.799510+0000","last_active":"2023-09-23T23:45:54.985317+0000","last_peered":"2023-09-23T23:45:54.985317+0000","last_clean":"2023-09-23T23:45:54.985317+0000","last_became_active":"2023-09-23T23:45:19.798822+0000","last_became_peered":"2023-09-23T23:45:19.798822+0000","last_unstale":"2023-09-23T23:45:54.985317+0000","last_undegraded":"2023-09-23T23:45:54.985317+0000","last_fullsized":"2023-09-23T23:45:54.985317+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:42:21.032363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":138,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.986239+0000","last_change":"2023-09-23T23:45:19.801143+0000","last_active":"2023-09-23T23:45:54.986239+0000","last_peered":"2023-09-23T23:45:54.986239+0000","last_clean":"2023-09-23T23:45:54.986239+0000","last_became_active":"2023-09-23T23:45:19.800221+0000","last_became_peered":"2023-09-23T23:45:19.800221+0000","last_unstale":"2023-09-23T23:45:54.986239+0000","last_undegraded":"2023-09-23T23:45:54.986239+0000","last_fullsized":"2023-09-23T23:45:54.986239+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:29.306589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"716'220","reported_seq":1532,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052257+0000","last_change":"2023-09-23T23:45:32.242090+0000","last_active":"2023-09-23T23:45:55.052257+0000","last_peered":"2023-09-23T23:45:55.052257+0000","last_clean":"2023-09-23T23:45:55.052257+0000","last_became_active":"2023-09-23T23:45:32.241092+0000","last_became_peered":"2023-09-23T23:45:32.241092+0000","last_unstale":"2023-09-23T23:45:55.052257+0000","last_undegraded":"2023-09-23T23:45:55.052257+0000","last_fullsized":"2023-09-23T23:45:55.052257+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":220,"log_dups_size":0,"ondisk_log_size":220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:42.511039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035346800000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1240,"num_read_kb":2147,"num_write":187,"num_write_kb":1152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":1061,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.053009+0000","last_change":"2023-09-23T23:45:52.244147+0000","last_active":"2023-09-23T23:45:55.053009+0000","last_peered":"2023-09-23T23:45:55.053009+0000","last_clean":"2023-09-23T23:45:55.053009+0000","last_became_active":"2023-09-23T23:45:52.243102+0000","last_became_peered":"2023-09-23T23:45:52.243102+0000","last_unstale":"2023-09-23T23:45:55.053009+0000","last_undegraded":"2023-09-23T23:45:55.053009+0000","last_fullsized":"2023-09-23T23:45:55.053009+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:55.342903+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:55.342903+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:56.522753+0000","scrub_duration":0.0026289199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":109,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051486+0000","last_change":"2023-09-23T23:44:59.989992+0000","last_active":"2023-09-23T23:45:55.051486+0000","last_peered":"2023-09-23T23:45:55.051486+0000","last_clean":"2023-09-23T23:45:55.051486+0000","last_became_active":"2023-09-23T23:44:59.989677+0000","last_became_peered":"2023-09-23T23:44:59.989677+0000","last_unstale":"2023-09-23T23:45:55.051486+0000","last_undegraded":"2023-09-23T23:45:55.051486+0000","last_fullsized":"2023-09-23T23:45:55.051486+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:05.406361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":39,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.054050+0000","last_change":"2023-09-23T23:44:59.995791+0000","last_active":"2023-09-23T23:45:55.054050+0000","last_peered":"2023-09-23T23:45:55.054050+0000","last_clean":"2023-09-23T23:45:55.054050+0000","last_became_active":"2023-09-23T23:44:59.995498+0000","last_became_peered":"2023-09-23T23:44:59.995498+0000","last_unstale":"2023-09-23T23:45:55.054050+0000","last_undegraded":"2023-09-23T23:45:55.054050+0000","last_fullsized":"2023-09-23T23:45:55.054050+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:58.895160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":936,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:58.452886+0000","last_change":"2023-09-23T23:45:19.797135+0000","last_active":"2023-09-23T23:45:58.452886+0000","last_peered":"2023-09-23T23:45:58.452886+0000","last_clean":"2023-09-23T23:45:58.452886+0000","last_became_active":"2023-09-23T23:45:19.796815+0000","last_became_peered":"2023-09-23T23:45:19.796815+0000","last_unstale":"2023-09-23T23:45:58.452886+0000","last_undegraded":"2023-09-23T23:45:58.452886+0000","last_fullsized":"2023-09-23T23:45:58.452886+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.214286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.214286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:11.893318+0000","scrub_duration":0.001784304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"715'124","reported_seq":1083,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052842+0000","last_change":"2023-09-23T23:45:52.244360+0000","last_active":"2023-09-23T23:45:55.052842+0000","last_peered":"2023-09-23T23:45:55.052842+0000","last_clean":"2023-09-23T23:45:55.052842+0000","last_became_active":"2023-09-23T23:45:52.243651+0000","last_became_peered":"2023-09-23T23:45:52.243651+0000","last_unstale":"2023-09-23T23:45:55.052842+0000","last_undegraded":"2023-09-23T23:45:55.052842+0000","last_fullsized":"2023-09-23T23:45:55.052842+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:57.939576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012476480000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":727,"num_read_kb":1949,"num_write":120,"num_write_kb":2256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":546,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052056+0000","last_change":"2023-09-23T23:45:22.812953+0000","last_active":"2023-09-23T23:45:55.052056+0000","last_peered":"2023-09-23T23:45:55.052056+0000","last_clean":"2023-09-23T23:45:55.052056+0000","last_became_active":"2023-09-23T23:45:22.812658+0000","last_became_peered":"2023-09-23T23:45:22.812658+0000","last_unstale":"2023-09-23T23:45:55.052056+0000","last_undegraded":"2023-09-23T23:45:55.052056+0000","last_fullsized":"2023-09-23T23:45:55.052056+0000","mapping_epoch":757,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":758,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:49.688942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":35,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051873+0000","last_change":"2023-09-23T23:45:08.978721+0000","last_active":"2023-09-23T23:45:55.051873+0000","last_peered":"2023-09-23T23:45:55.051873+0000","last_clean":"2023-09-23T23:45:55.051873+0000","last_became_active":"2023-09-23T23:45:08.978419+0000","last_became_peered":"2023-09-23T23:45:08.978419+0000","last_unstale":"2023-09-23T23:45:55.051873+0000","last_undegraded":"2023-09-23T23:45:55.051873+0000","last_fullsized":"2023-09-23T23:45:55.051873+0000","mapping_epoch":748,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":749,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:28.678292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":608,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.054103+0000","last_change":"2023-09-23T23:45:03.040803+0000","last_active":"2023-09-23T23:45:55.054103+0000","last_peered":"2023-09-23T23:45:55.054103+0000","last_clean":"2023-09-23T23:45:55.054103+0000","last_became_active":"2023-09-23T23:45:03.040337+0000","last_became_peered":"2023-09-23T23:45:03.040337+0000","last_unstale":"2023-09-23T23:45:55.054103+0000","last_undegraded":"2023-09-23T23:45:55.054103+0000","last_fullsized":"2023-09-23T23:45:55.054103+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:32:41.025609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:32:41.025609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:36.187578+0000","scrub_duration":0.0016965490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"715'102","reported_seq":717,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052007+0000","last_change":"2023-09-23T23:45:52.243612+0000","last_active":"2023-09-23T23:45:55.052007+0000","last_peered":"2023-09-23T23:45:55.052007+0000","last_clean":"2023-09-23T23:45:55.052007+0000","last_became_active":"2023-09-23T23:45:52.243149+0000","last_became_peered":"2023-09-23T23:45:52.243149+0000","last_unstale":"2023-09-23T23:45:55.052007+0000","last_undegraded":"2023-09-23T23:45:55.052007+0000","last_fullsized":"2023-09-23T23:45:55.052007+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:08.282580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037965999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1014,"num_read_kb":1641,"num_write":66,"num_write_kb":416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":293,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.986864+0000","last_change":"2023-09-23T23:45:19.801071+0000","last_active":"2023-09-23T23:45:54.986864+0000","last_peered":"2023-09-23T23:45:54.986864+0000","last_clean":"2023-09-23T23:45:54.986864+0000","last_became_active":"2023-09-23T23:45:19.800637+0000","last_became_peered":"2023-09-23T23:45:19.800637+0000","last_unstale":"2023-09-23T23:45:54.986864+0000","last_undegraded":"2023-09-23T23:45:54.986864+0000","last_fullsized":"2023-09-23T23:45:54.986864+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:45.026051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":267,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052465+0000","last_change":"2023-09-23T23:45:12.997379+0000","last_active":"2023-09-23T23:45:55.052465+0000","last_peered":"2023-09-23T23:45:55.052465+0000","last_clean":"2023-09-23T23:45:55.052465+0000","last_became_active":"2023-09-23T23:45:12.995469+0000","last_became_peered":"2023-09-23T23:45:12.995469+0000","last_unstale":"2023-09-23T23:45:55.052465+0000","last_undegraded":"2023-09-23T23:45:55.052465+0000","last_fullsized":"2023-09-23T23:45:55.052465+0000","mapping_epoch":752,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":753,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:04.637951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":1014,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.989248+0000","last_change":"2023-09-23T23:45:03.038729+0000","last_active":"2023-09-23T23:45:54.989248+0000","last_peered":"2023-09-23T23:45:54.989248+0000","last_clean":"2023-09-23T23:45:54.989248+0000","last_became_active":"2023-09-23T23:45:03.038334+0000","last_became_peered":"2023-09-23T23:45:03.038334+0000","last_unstale":"2023-09-23T23:45:54.989248+0000","last_undegraded":"2023-09-23T23:45:54.989248+0000","last_fullsized":"2023-09-23T23:45:54.989248+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:46.892563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:46.892563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:16.965706+0000","scrub_duration":0.00148774,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":95,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052868+0000","last_change":"2023-09-23T23:43:37.369294+0000","last_active":"2023-09-23T23:45:55.052868+0000","last_peered":"2023-09-23T23:45:55.052868+0000","last_clean":"2023-09-23T23:45:55.052868+0000","last_became_active":"2023-09-23T23:43:37.342878+0000","last_became_peered":"2023-09-23T23:43:37.342878+0000","last_unstale":"2023-09-23T23:45:55.052868+0000","last_undegraded":"2023-09-23T23:45:55.052868+0000","last_fullsized":"2023-09-23T23:45:55.052868+0000","mapping_epoch":692,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":693,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:52.844816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"716'140","reported_seq":590,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.985933+0000","last_change":"2023-09-23T23:45:19.797178+0000","last_active":"2023-09-23T23:45:54.985933+0000","last_peered":"2023-09-23T23:45:54.985933+0000","last_clean":"2023-09-23T23:45:54.985933+0000","last_became_active":"2023-09-23T23:45:19.796843+0000","last_became_peered":"2023-09-23T23:45:19.796843+0000","last_unstale":"2023-09-23T23:45:54.985933+0000","last_undegraded":"2023-09-23T23:45:54.985933+0000","last_fullsized":"2023-09-23T23:45:54.985933+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":140,"log_dups_size":0,"ondisk_log_size":140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:45.644839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000432328,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":762,"num_read_kb":2671,"num_write":98,"num_write_kb":2116,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":26,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051387+0000","last_change":"2023-09-23T23:45:22.812861+0000","last_active":"2023-09-23T23:45:55.051387+0000","last_peered":"2023-09-23T23:45:55.051387+0000","last_clean":"2023-09-23T23:45:55.051387+0000","last_became_active":"2023-09-23T23:45:22.812453+0000","last_became_peered":"2023-09-23T23:45:22.812453+0000","last_unstale":"2023-09-23T23:45:55.051387+0000","last_undegraded":"2023-09-23T23:45:55.051387+0000","last_fullsized":"2023-09-23T23:45:55.051387+0000","mapping_epoch":757,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":758,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:09.336586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":848,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051799+0000","last_change":"2023-09-23T23:43:39.961347+0000","last_active":"2023-09-23T23:45:55.051799+0000","last_peered":"2023-09-23T23:45:55.051799+0000","last_clean":"2023-09-23T23:45:55.051799+0000","last_became_active":"2023-09-23T23:43:39.961019+0000","last_became_peered":"2023-09-23T23:43:39.961019+0000","last_unstale":"2023-09-23T23:45:55.051799+0000","last_undegraded":"2023-09-23T23:45:55.051799+0000","last_fullsized":"2023-09-23T23:45:55.051799+0000","mapping_epoch":695,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":696,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:45.832190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:45.832190+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:01.621631+0000","scrub_duration":0.00107891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"715'148","reported_seq":1024,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.054127+0000","last_change":"2023-09-23T23:45:03.042180+0000","last_active":"2023-09-23T23:45:55.054127+0000","last_peered":"2023-09-23T23:45:55.054127+0000","last_clean":"2023-09-23T23:45:55.054127+0000","last_became_active":"2023-09-23T23:45:03.041364+0000","last_became_peered":"2023-09-23T23:45:03.041364+0000","last_unstale":"2023-09-23T23:45:55.054127+0000","last_undegraded":"2023-09-23T23:45:55.054127+0000","last_fullsized":"2023-09-23T23:45:55.054127+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:52:20.297695+0000","last_clean_scrub_stamp":"2023-09-23T22:52:20.297695+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:49.057745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033161300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":898,"num_read_kb":2090,"num_write":128,"num_write_kb":1223,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":39,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051974+0000","last_change":"2023-09-23T23:45:08.984591+0000","last_active":"2023-09-23T23:45:55.051974+0000","last_peered":"2023-09-23T23:45:55.051974+0000","last_clean":"2023-09-23T23:45:55.051974+0000","last_became_active":"2023-09-23T23:45:08.982188+0000","last_became_peered":"2023-09-23T23:45:08.982188+0000","last_unstale":"2023-09-23T23:45:55.051974+0000","last_undegraded":"2023-09-23T23:45:55.051974+0000","last_fullsized":"2023-09-23T23:45:55.051974+0000","mapping_epoch":748,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":749,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:57.031770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988650+0000","last_change":"2023-09-23T23:44:59.992453+0000","last_active":"2023-09-23T23:45:54.988650+0000","last_peered":"2023-09-23T23:45:54.988650+0000","last_clean":"2023-09-23T23:45:54.988650+0000","last_became_active":"2023-09-23T23:44:59.992101+0000","last_became_peered":"2023-09-23T23:44:59.992101+0000","last_unstale":"2023-09-23T23:45:54.988650+0000","last_undegraded":"2023-09-23T23:45:54.988650+0000","last_fullsized":"2023-09-23T23:45:54.988650+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:09.758075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":1288,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:57.994865+0000","last_change":"2023-09-23T23:45:11.988445+0000","last_active":"2023-09-23T23:45:57.994865+0000","last_peered":"2023-09-23T23:45:57.994865+0000","last_clean":"2023-09-23T23:45:57.994865+0000","last_became_active":"2023-09-23T23:45:11.988080+0000","last_became_peered":"2023-09-23T23:45:11.988080+0000","last_unstale":"2023-09-23T23:45:57.994865+0000","last_undegraded":"2023-09-23T23:45:57.994865+0000","last_fullsized":"2023-09-23T23:45:57.994865+0000","mapping_epoch":751,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":752,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:50.872334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:50.872334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:18.929804+0000","scrub_duration":0.001866024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":52,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052381+0000","last_change":"2023-09-23T23:45:12.997680+0000","last_active":"2023-09-23T23:45:55.052381+0000","last_peered":"2023-09-23T23:45:55.052381+0000","last_clean":"2023-09-23T23:45:55.052381+0000","last_became_active":"2023-09-23T23:45:12.995943+0000","last_became_peered":"2023-09-23T23:45:12.995943+0000","last_unstale":"2023-09-23T23:45:55.052381+0000","last_undegraded":"2023-09-23T23:45:55.052381+0000","last_fullsized":"2023-09-23T23:45:55.052381+0000","mapping_epoch":752,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":753,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:32:37.694678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":550,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.985906+0000","last_change":"2023-09-23T23:43:37.332588+0000","last_active":"2023-09-23T23:45:54.985906+0000","last_peered":"2023-09-23T23:45:54.985906+0000","last_clean":"2023-09-23T23:45:54.985906+0000","last_became_active":"2023-09-23T23:43:37.331837+0000","last_became_peered":"2023-09-23T23:43:37.331837+0000","last_unstale":"2023-09-23T23:45:54.985906+0000","last_undegraded":"2023-09-23T23:45:54.985906+0000","last_fullsized":"2023-09-23T23:45:54.985906+0000","mapping_epoch":692,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":693,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:14.466130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":983,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052711+0000","last_change":"2023-09-23T23:45:03.038381+0000","last_active":"2023-09-23T23:45:55.052711+0000","last_peered":"2023-09-23T23:45:55.052711+0000","last_clean":"2023-09-23T23:45:55.052711+0000","last_became_active":"2023-09-23T23:45:03.038081+0000","last_became_peered":"2023-09-23T23:45:03.038081+0000","last_unstale":"2023-09-23T23:45:55.052711+0000","last_undegraded":"2023-09-23T23:45:55.052711+0000","last_fullsized":"2023-09-23T23:45:55.052711+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:58.725500+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:58.725500+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:00.921011+0000","scrub_duration":0.0020644489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":418,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.051623+0000","last_change":"2023-09-23T23:45:32.242007+0000","last_active":"2023-09-23T23:45:55.051623+0000","last_peered":"2023-09-23T23:45:55.051623+0000","last_clean":"2023-09-23T23:45:55.051623+0000","last_became_active":"2023-09-23T23:45:32.240828+0000","last_became_peered":"2023-09-23T23:45:32.240828+0000","last_unstale":"2023-09-23T23:45:55.051623+0000","last_undegraded":"2023-09-23T23:45:55.051623+0000","last_fullsized":"2023-09-23T23:45:55.051623+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:44.441348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":43,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988856+0000","last_change":"2023-09-23T23:44:55.988280+0000","last_active":"2023-09-23T23:45:54.988856+0000","last_peered":"2023-09-23T23:45:54.988856+0000","last_clean":"2023-09-23T23:45:54.988856+0000","last_became_active":"2023-09-23T23:44:55.987930+0000","last_became_peered":"2023-09-23T23:44:55.987930+0000","last_unstale":"2023-09-23T23:45:54.988856+0000","last_undegraded":"2023-09-23T23:45:54.988856+0000","last_fullsized":"2023-09-23T23:45:54.988856+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:24.234351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":286,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.053022+0000","last_change":"2023-09-23T23:45:32.241873+0000","last_active":"2023-09-23T23:45:55.053022+0000","last_peered":"2023-09-23T23:45:55.053022+0000","last_clean":"2023-09-23T23:45:55.053022+0000","last_became_active":"2023-09-23T23:45:32.239852+0000","last_became_peered":"2023-09-23T23:45:32.239852+0000","last_unstale":"2023-09-23T23:45:55.053022+0000","last_undegraded":"2023-09-23T23:45:55.053022+0000","last_fullsized":"2023-09-23T23:45:55.053022+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:15.224411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":43,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988609+0000","last_change":"2023-09-23T23:44:55.987739+0000","last_active":"2023-09-23T23:45:54.988609+0000","last_peered":"2023-09-23T23:45:54.988609+0000","last_clean":"2023-09-23T23:45:54.988609+0000","last_became_active":"2023-09-23T23:44:55.987506+0000","last_became_peered":"2023-09-23T23:44:55.987506+0000","last_unstale":"2023-09-23T23:45:54.988609+0000","last_undegraded":"2023-09-23T23:45:54.988609+0000","last_fullsized":"2023-09-23T23:45:54.988609+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:09.672401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":1001,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052196+0000","last_change":"2023-09-23T23:45:52.243605+0000","last_active":"2023-09-23T23:45:55.052196+0000","last_peered":"2023-09-23T23:45:55.052196+0000","last_clean":"2023-09-23T23:45:55.052196+0000","last_became_active":"2023-09-23T23:45:52.243247+0000","last_became_peered":"2023-09-23T23:45:52.243247+0000","last_unstale":"2023-09-23T23:45:55.052196+0000","last_undegraded":"2023-09-23T23:45:55.052196+0000","last_fullsized":"2023-09-23T23:45:55.052196+0000","mapping_epoch":770,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":771,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:45:04.205935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:45:04.205935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:16.253214+0000","scrub_duration":0.0015440289999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":100,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052811+0000","last_change":"2023-09-23T23:44:59.990843+0000","last_active":"2023-09-23T23:45:55.052811+0000","last_peered":"2023-09-23T23:45:55.052811+0000","last_clean":"2023-09-23T23:45:55.052811+0000","last_became_active":"2023-09-23T23:44:59.990488+0000","last_became_peered":"2023-09-23T23:44:59.990488+0000","last_unstale":"2023-09-23T23:45:55.052811+0000","last_undegraded":"2023-09-23T23:45:55.052811+0000","last_fullsized":"2023-09-23T23:45:55.052811+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:06.824244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":91,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:57.994729+0000","last_change":"2023-09-23T23:43:37.342518+0000","last_active":"2023-09-23T23:45:57.994729+0000","last_peered":"2023-09-23T23:45:57.994729+0000","last_clean":"2023-09-23T23:45:57.994729+0000","last_became_active":"2023-09-23T23:43:37.341672+0000","last_became_peered":"2023-09-23T23:43:37.341672+0000","last_unstale":"2023-09-23T23:45:57.994729+0000","last_undegraded":"2023-09-23T23:45:57.994729+0000","last_fullsized":"2023-09-23T23:45:57.994729+0000","mapping_epoch":692,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":693,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:09.991757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":1016,"reported_epoch":773,"state":"active+clean","last_fresh":"2023-09-23T23:45:58.452705+0000","last_change":"2023-09-23T23:45:25.984339+0000","last_active":"2023-09-23T23:45:58.452705+0000","last_peered":"2023-09-23T23:45:58.452705+0000","last_clean":"2023-09-23T23:45:58.452705+0000","last_became_active":"2023-09-23T23:45:25.984004+0000","last_became_peered":"2023-09-23T23:45:25.984004+0000","last_unstale":"2023-09-23T23:45:58.452705+0000","last_undegraded":"2023-09-23T23:45:58.452705+0000","last_fullsized":"2023-09-23T23:45:58.452705+0000","mapping_epoch":760,"log_start":"0'0","ondisk_log_start":"0'0","created":333,"last_epoch_clean":761,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-09-23T23:23:56.246039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:49:49.548021+0000","last_clean_scrub_stamp":"2023-09-23T23:23:56.246039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:48.011518+0000","scrub_duration":0.0024015680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":719,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:55.052359+0000","last_change":"2023-09-23T23:45:12.995656+0000","last_active":"2023-09-23T23:45:55.052359+0000","last_peered":"2023-09-23T23:45:55.052359+0000","last_clean":"2023-09-23T23:45:55.052359+0000","last_became_active":"2023-09-23T23:45:12.995246+0000","last_became_peered":"2023-09-23T23:45:12.995246+0000","last_unstale":"2023-09-23T23:45:55.052359+0000","last_undegraded":"2023-09-23T23:45:55.052359+0000","last_fullsized":"2023-09-23T23:45:55.052359+0000","mapping_epoch":752,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":753,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:15.863481+0000","last_clean_scrub_stamp":"2023-09-23T22:53:15.863481+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:48.479413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":53,"reported_epoch":772,"state":"active+clean","last_fresh":"2023-09-23T23:45:54.988431+0000","last_change":"2023-09-23T23:44:59.991420+0000","last_active":"2023-09-23T23:45:54.988431+0000","last_peered":"2023-09-23T23:45:54.988431+0000","last_clean":"2023-09-23T23:45:54.988431+0000","last_became_active":"2023-09-23T23:44:59.991042+0000","last_became_peered":"2023-09-23T23:44:59.991042+0000","last_unstale":"2023-09-23T23:45:54.988431+0000","last_undegraded":"2023-09-23T23:45:54.988431+0000","last_fullsized":"2023-09-23T23:45:54.988431+0000","mapping_epoch":743,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":744,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:53:16.868862+0000","last_clean_scrub_stamp":"2023-09-23T22:53:16.868862+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:51.331723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":114,"num_bytes_recovered":29258,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31868,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":13211,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":343,"seq":1473173782806,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":498644,"kb_used_data":5836,"kb_used_omap":8,"kb_used_meta":492791,"kb_avail":93246508,"statfs":{"total":95995035648,"available":95484424192,"internally_reserved":0,"allocated":5976064,"data_stored":4608295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9147,"internal_metadata":504618053},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:46:24 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.755,"15min":1.0900000000000001},"min":{"1min":0.44700000000000001,"5min":0.23799999999999999,"15min":0.20499999999999999},"max":{"1min":2.7109999999999999,"5min":6.5510000000000002,"15min":36.956000000000003},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.86299999999999999,"15min":1.1799999999999999},"min":{"1min":0.39000000000000001,"5min":0.25800000000000001,"15min":0.21299999999999999},"max":{"1min":2.6579999999999999,"5min":6.4489999999999998,"15min":37.247999999999998},"last":0.66700000000000004}]},{"osd":1,"last update":"Sat Sep 23 23:46:24 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.81799999999999995,"15min":1.96},"min":{"1min":0.39300000000000002,"5min":0.252,"15min":0.245},"max":{"1min":2.4889999999999999,"5min":5.9660000000000002,"15min":129.11199999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.59399999999999997,"5min":1.143,"15min":2.0419999999999998},"min":{"1min":0.40200000000000002,"5min":0.28999999999999998,"15min":0.24399999999999999},"max":{"1min":1.661,"5min":8.8460000000000001,"15min":129.02600000000001},"last":0.56599999999999995}]},{"osd":2,"last update":"Sat Sep 23 23:46:24 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.2070000000000001,"15min":2.2040000000000002},"min":{"1min":0.36899999999999999,"5min":0.251,"15min":0.251},"max":{"1min":1.411,"5min":8.9779999999999998,"15min":78.528999999999996},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.69799999999999995,"5min":1.135,"15min":1.9810000000000001},"min":{"1min":0.40699999999999997,"5min":0.28000000000000003,"15min":0.26200000000000001},"max":{"1min":2.3370000000000002,"5min":10.145,"15min":53.023000000000003},"last":0.67900000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:46:24 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":1.1639999999999999,"15min":3.6389999999999998},"min":{"1min":0.371,"5min":0.31,"15min":0.29599999999999999},"max":{"1min":5.4080000000000004,"5min":8.1479999999999997,"15min":154.18100000000001},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.80000000000000004,"5min":1.234,"15min":4.2649999999999997},"min":{"1min":0.505,"5min":0.32600000000000001,"15min":0.30099999999999999},"max":{"1min":4.4630000000000001,"5min":8.5779999999999994,"15min":154.16300000000001},"last":0.50600000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:45:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":15.500999999999999,"5min":15.500999999999999,"15min":15.500999999999999},"last":0.443},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.6389999999999998,"5min":3.6389999999999998,"15min":3.6389999999999998},"last":0.40999999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:46:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.61099999999999999,"15min":1.452},"min":{"1min":0.33100000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.96299999999999997,"5min":9.0709999999999997,"15min":119.417},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.60899999999999999,"15min":1.4690000000000001},"min":{"1min":0.38300000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.79200000000000004,"5min":8.9339999999999993,"15min":119.61499999999999},"last":0.53600000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:46:24 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.53300000000000003,"15min":1.0880000000000001},"min":{"1min":0.48299999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":94.852999999999994},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.64700000000000002,"15min":1.236},"min":{"1min":0.41599999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.0569999999999999,"5min":5.9299999999999997,"15min":95.510999999999996},"last":0.46400000000000002}]}]},{"osd":6,"up_from":46,"seq":197568496283,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":767816,"kb_used_data":5888,"kb_used_omap":0,"kb_used_meta":761920,"kb_avail":92977336,"statfs":{"total":95995035648,"available":95208792064,"internally_reserved":0,"allocated":6029312,"data_stored":4673160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":780206080},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.84099999999999997,"15min":1.081},"min":{"1min":0.35299999999999998,"5min":0.23899999999999999,"15min":0.223},"max":{"1min":0.79100000000000004,"5min":9.4060000000000006,"15min":60.252000000000002},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.55200000000000005,"5min":1.0700000000000001,"15min":1.1299999999999999},"min":{"1min":0.33500000000000002,"5min":0.26200000000000001,"15min":0.23999999999999999},"max":{"1min":0.77700000000000002,"5min":7.1920000000000002,"15min":60.130000000000003},"last":0.503}]},{"osd":1,"last update":"Sat Sep 23 23:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":1.5920000000000001,"15min":2.0939999999999999},"min":{"1min":0.32200000000000001,"5min":0.24399999999999999,"15min":0.24299999999999999},"max":{"1min":0.75600000000000001,"5min":31.670000000000002,"15min":89.605000000000004},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.8500000000000001,"15min":2.1970000000000001},"min":{"1min":0.374,"5min":0.33700000000000002,"15min":0.24099999999999999},"max":{"1min":0.77100000000000002,"5min":32.363999999999997,"15min":91.025999999999996},"last":0.61699999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":2.1400000000000001,"15min":3.5950000000000002},"min":{"1min":0.39500000000000002,"5min":0.318,"15min":0.26700000000000002},"max":{"1min":0.76800000000000002,"5min":73.483000000000004,"15min":97.408000000000001},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.8740000000000001,"15min":3.8159999999999998},"min":{"1min":0.32700000000000001,"5min":0.29499999999999998,"15min":0.26100000000000001},"max":{"1min":0.86699999999999999,"5min":73.135000000000005,"15min":125.539},"last":0.73099999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":3.923,"15min":6.6130000000000004},"min":{"1min":0.39300000000000002,"5min":0.32400000000000001,"15min":0.28100000000000003},"max":{"1min":1.3049999999999999,"5min":84.466999999999999,"15min":319.22699999999998},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.64100000000000001,"5min":3.681,"15min":6.3710000000000004},"min":{"1min":0.377,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.83799999999999997,"5min":84.730000000000004,"15min":319.42899999999997},"last":0.79900000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.85899999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":1.131,"15min":0.998},"min":{"1min":0.36699999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.82599999999999996,"5min":68.947999999999993,"15min":68.947999999999993},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.59299999999999997,"5min":1.1479999999999999,"15min":1.073},"min":{"1min":0.32900000000000001,"5min":0.28999999999999998,"15min":0.28399999999999997},"max":{"1min":0.76000000000000001,"5min":67.546000000000006,"15min":67.546000000000006},"last":0.83999999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":1.2689999999999999,"15min":1.127},"min":{"1min":0.503,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.94399999999999995,"5min":70.463999999999999,"15min":70.463999999999999},"last":0.89400000000000002},{"interface":"front","average":{"1min":0.68899999999999995,"5min":1.2729999999999999,"15min":1.1499999999999999},"min":{"1min":0.52100000000000002,"5min":0.312,"15min":0.28299999999999997},"max":{"1min":0.93200000000000005,"5min":70.563999999999993,"15min":70.563999999999993},"last":0.82099999999999995}]}]},{"osd":1,"up_from":112,"seq":481036337624,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1065716,"kb_used_data":6956,"kb_used_omap":4,"kb_used_meta":1058747,"kb_avail":92679436,"statfs":{"total":95995035648,"available":94903742464,"internally_reserved":0,"allocated":7122944,"data_stored":4527163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1084157838},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:45:38 2023","interfaces":[{"interface":"back","average":{"1min":0.34100000000000003,"5min":2.2519999999999998,"15min":2.968},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.16400000000000001},"max":{"1min":0.58299999999999996,"5min":13.766,"15min":156.37799999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.39000000000000001,"5min":1.4750000000000001,"15min":2.3969999999999998},"min":{"1min":0.23699999999999999,"5min":0.22,"15min":0.189},"max":{"1min":0.77400000000000002,"5min":13.548999999999999,"15min":161.05199999999999},"last":0.41099999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:46:25 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":1.9990000000000001,"15min":3.762},"min":{"1min":0.313,"5min":0.22900000000000001,"15min":0.184},"max":{"1min":0.51500000000000001,"5min":49.311,"15min":201.03999999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.34399999999999997,"5min":2.0390000000000001,"15min":4.0940000000000003},"min":{"1min":0.26700000000000002,"5min":0.23000000000000001,"15min":0.20899999999999999},"max":{"1min":0.55300000000000005,"5min":48.991999999999997,"15min":202.12},"last":0.55900000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:46:25 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":5.0970000000000004,"15min":4.7800000000000002},"min":{"1min":0.26800000000000002,"5min":0.25,"15min":0.21299999999999999},"max":{"1min":0.52000000000000002,"5min":272.839,"15min":272.839},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.44400000000000001,"5min":4.8689999999999998,"15min":4.8070000000000004},"min":{"1min":0.31,"5min":0.27700000000000002,"15min":0.224},"max":{"1min":1.2450000000000001,"5min":272.76400000000001,"15min":272.76400000000001},"last":0.38}]},{"osd":4,"last update":"Sat Sep 23 23:45:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.81299999999999994},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.71899999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:45:38 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":1.127,"15min":2.044},"min":{"1min":0.38300000000000001,"5min":0.31,"15min":0.28899999999999998},"max":{"1min":0.85099999999999998,"5min":8.2639999999999993,"15min":151.94800000000001},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.6339999999999999,"15min":2.3290000000000002},"min":{"1min":0.40300000000000002,"5min":0.36499999999999999,"15min":0.29599999999999999},"max":{"1min":0.81499999999999995,"5min":10.141999999999999,"15min":151.821},"last":0.83399999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:45:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.161,"15min":1.496},"min":{"1min":0.42499999999999999,"5min":0.33200000000000002,"15min":0.32200000000000001},"max":{"1min":0.873,"5min":8.2910000000000004,"15min":87.837999999999994},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.657,"15min":1.8740000000000001},"min":{"1min":0.41699999999999998,"5min":0.38400000000000001,"15min":0.312},"max":{"1min":0.78100000000000003,"5min":10.233000000000001,"15min":85.518000000000001},"last":0.88100000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:46:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.0329999999999999,"15min":1.405},"min":{"1min":0.52000000000000002,"5min":0.38,"15min":0.28799999999999998},"max":{"1min":1.0269999999999999,"5min":13.587999999999999,"15min":44.241},"last":0.86099999999999999},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1.2969999999999999,"15min":1.677},"min":{"1min":0.52100000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.77900000000000003,"5min":10.210000000000001,"15min":43.920999999999999},"last":0.70399999999999996}]}]},{"osd":0,"up_from":91,"seq":390842024447,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1405392,"kb_used_data":11336,"kb_used_omap":5,"kb_used_meta":1394042,"kb_avail":92339760,"statfs":{"total":95995035648,"available":94555914240,"internally_reserved":0,"allocated":11608064,"data_stored":9037781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1427499642},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:45:50 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":1.8160000000000001,"15min":4.5979999999999999},"min":{"1min":0.24099999999999999,"5min":0.19700000000000001,"15min":0.188},"max":{"1min":0.59699999999999998,"5min":10.502000000000001,"15min":301.12700000000001},"last":0.245},{"interface":"front","average":{"1min":0.34200000000000003,"5min":1.8029999999999999,"15min":4.5540000000000003},"min":{"1min":0.23999999999999999,"5min":0.221,"15min":0.19700000000000001},"max":{"1min":0.52700000000000002,"5min":18.503,"15min":297.72399999999999},"last":0.312}]},{"osd":2,"last update":"Sat Sep 23 23:46:23 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":2.907,"15min":5.282},"min":{"1min":0.26200000000000001,"5min":0.19400000000000001,"15min":0.17699999999999999},"max":{"1min":0.55500000000000005,"5min":89.935000000000002,"15min":283.07400000000001},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.39300000000000002,"5min":2.8220000000000001,"15min":5.3799999999999999},"min":{"1min":0.26300000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.65100000000000002,"5min":90.25,"15min":283.161},"last":0.27900000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:45:50 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":2.5990000000000002,"15min":6.2590000000000003},"min":{"1min":0.27500000000000002,"5min":0.23200000000000001,"15min":0.217},"max":{"1min":0.59299999999999997,"5min":36.186999999999998,"15min":220.732},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.435,"5min":2.7949999999999999,"15min":6.5220000000000002},"min":{"1min":0.30499999999999999,"5min":0.245,"15min":0.19800000000000001},"max":{"1min":0.64900000000000002,"5min":33.616,"15min":218.78200000000001},"last":0.34899999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:45:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.60699999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":1.3979999999999999,"15min":1.923},"min":{"1min":0.34999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.80300000000000005,"5min":20.629999999999999,"15min":94.254000000000005},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.56499999999999995,"5min":1.986,"15min":2.036},"min":{"1min":0.38200000000000001,"5min":0.34200000000000003,"15min":0.30599999999999999},"max":{"1min":0.81899999999999995,"5min":20.867000000000001,"15min":94.837000000000003},"last":0.67000000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.254,"15min":1.9610000000000001},"min":{"1min":0.40100000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.83899999999999997,"5min":7.4359999999999999,"15min":108.67},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.8280000000000001,"15min":2.117},"min":{"1min":0.439,"5min":0.42299999999999999,"15min":0.30199999999999999},"max":{"1min":0.97199999999999998,"5min":9.6310000000000002,"15min":108.804},"last":0.70399999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:46:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":1.4339999999999999,"15min":1.6519999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.875,"5min":21.132999999999999,"15min":66.620000000000005},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.621,"5min":1.7609999999999999,"15min":1.798},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.33400000000000002},"max":{"1min":0.93000000000000005,"5min":23.489000000000001,"15min":66.701999999999998},"last":0.73399999999999999}]}]},{"osd":2,"up_from":199,"seq":854698492269,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1063248,"kb_used_data":11464,"kb_used_omap":7,"kb_used_meta":1051768,"kb_avail":92681904,"statfs":{"total":95995035648,"available":94906269696,"internally_reserved":0,"allocated":11739136,"data_stored":9001924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1077010513},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":1.879,"15min":2.1389999999999998},"min":{"1min":0.20000000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.49399999999999999,"5min":34.648000000000003,"15min":65.001000000000005},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.33400000000000002,"5min":1.3169999999999999,"15min":1.8260000000000001},"min":{"1min":0.215,"5min":0.19900000000000001,"15min":0.19700000000000001},"max":{"1min":0.45100000000000001,"5min":13.506,"15min":64.346000000000004},"last":0.25700000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":1.3069999999999999,"15min":3.0009999999999999},"min":{"1min":0.254,"5min":0.20999999999999999,"15min":0.192},"max":{"1min":0.53600000000000003,"5min":14.433999999999999,"15min":106.765},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.38800000000000001,"5min":1.698,"15min":2.8929999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.222},"max":{"1min":0.57199999999999995,"5min":13.353,"15min":106.732},"last":0.372}]},{"osd":3,"last update":"Sat Sep 23 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":2.254,"15min":3.3599999999999999},"min":{"1min":0.251,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.626,"5min":54.441000000000003,"15min":106.70399999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":2.448,"15min":3.1400000000000001},"min":{"1min":0.28399999999999997,"5min":0.23899999999999999,"15min":0.23200000000000001},"max":{"1min":0.64700000000000002,"5min":53.734000000000002,"15min":106.66800000000001},"last":0.441}]},{"osd":4,"last update":"Sat Sep 23 23:45:53 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.69699999999999995}]},{"osd":5,"last update":"Sat Sep 23 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.252,"15min":2.6869999999999998},"min":{"1min":0.35799999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.81999999999999995,"5min":13.023,"15min":101.104},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.93300000000000005,"15min":2.8969999999999998},"min":{"1min":0.42999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.84499999999999997,"5min":6.641,"15min":101.404},"last":0.72999999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.383,"15min":2.9289999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.31},"max":{"1min":0.82399999999999995,"5min":15.898999999999999,"15min":107.018},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.94299999999999995,"15min":2.7160000000000002},"min":{"1min":0.377,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.78900000000000003,"5min":6.7130000000000001,"15min":98.231999999999999},"last":0.76900000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.242,"15min":1.78},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.32200000000000001},"max":{"1min":0.86399999999999999,"5min":7.4349999999999996,"15min":68.814999999999998},"last":0.622},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.2809999999999999,"15min":2.2869999999999999},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.35399999999999998},"max":{"1min":0.81000000000000005,"5min":7.4649999999999999,"15min":106.809},"last":0.68700000000000006}]}]},{"osd":3,"up_from":148,"seq":635655160237,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1131892,"kb_used_data":11436,"kb_used_omap":2,"kb_used_meta":1120445,"kb_avail":92613260,"statfs":{"total":95995035648,"available":94835978240,"internally_reserved":0,"allocated":11710464,"data_stored":9056341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1147335733},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:46:05 2023","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":6.915,"15min":3.2919999999999998},"min":{"1min":0.26000000000000001,"5min":0.20300000000000001,"15min":0.184},"max":{"1min":0.58899999999999997,"5min":230.53899999999999,"15min":230.53899999999999},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.35699999999999998,"5min":6.3460000000000001,"15min":3.2549999999999999},"min":{"1min":0.23100000000000001,"5min":0.20599999999999999,"15min":0.17199999999999999},"max":{"1min":0.60599999999999998,"5min":230.38800000000001,"15min":230.38800000000001},"last":0.35599999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:46:05 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":1.974,"15min":2.1019999999999999},"min":{"1min":0.25600000000000001,"5min":0.219,"15min":0.17599999999999999},"max":{"1min":1.204,"5min":21.824999999999999,"15min":56.119},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.38300000000000001,"5min":1.837,"15min":1.8899999999999999},"min":{"1min":0.25,"5min":0.22900000000000001,"15min":0.20799999999999999},"max":{"1min":0.625,"5min":20.779,"15min":53.962000000000003},"last":0.29099999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":2.0409999999999999,"15min":4.1550000000000002},"min":{"1min":0.29799999999999999,"5min":0.25,"15min":0.23999999999999999},"max":{"1min":1.0820000000000001,"5min":31.564,"15min":139.96299999999999},"last":0.375},{"interface":"front","average":{"1min":0.433,"5min":2.3620000000000001,"15min":4.2629999999999999},"min":{"1min":0.32400000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.82399999999999995,"5min":69.405000000000001,"15min":141.93199999999999},"last":0.318}]},{"osd":4,"last update":"Sat Sep 23 23:45:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.55900000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:46:05 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":1.4930000000000001,"15min":4.4489999999999998},"min":{"1min":0.44500000000000001,"5min":0.29899999999999999,"15min":0.29499999999999998},"max":{"1min":0.85499999999999998,"5min":15.56,"15min":214.43700000000001},"last":0.628},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.5169999999999999,"15min":4.3280000000000003},"min":{"1min":0.46400000000000002,"5min":0.38500000000000001,"15min":0.32600000000000001},"max":{"1min":1.151,"5min":14.478999999999999,"15min":214.46600000000001},"last":0.54300000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:46:05 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":1.339,"15min":2.2810000000000001},"min":{"1min":0.499,"5min":0.38900000000000001,"15min":0.318},"max":{"1min":0.85099999999999998,"5min":12.215,"15min":125.94},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.64400000000000002,"5min":1.546,"15min":2.3039999999999998},"min":{"1min":0.46800000000000003,"5min":0.35499999999999998,"15min":0.32300000000000001},"max":{"1min":1.1779999999999999,"5min":14.513,"15min":125.955},"last":0.51700000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.7989999999999999,"15min":2.6070000000000002},"min":{"1min":0.497,"5min":0.313,"15min":0.313},"max":{"1min":0.88,"5min":69.197999999999993,"15min":192.69},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.65900000000000003,"5min":1.401,"15min":2.698},"min":{"1min":0.44400000000000001,"5min":0.40899999999999997,"15min":0.39200000000000002},"max":{"1min":0.81699999999999995,"5min":12.568,"15min":192.65600000000001},"last":0.65800000000000003}]}]},{"osd":4,"up_from":736,"seq":3161095929879,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33356,"kb_used_data":5572,"kb_used_omap":4,"kb_used_meta":27771,"kb_avail":93711796,"statfs":{"total":95995035648,"available":95960879104,"internally_reserved":0,"allocated":5705728,"data_stored":4345537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4160,"internal_metadata":28438464},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.47699999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.63200000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.497},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.51500000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.80800000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.60799999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.79400000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.621}]}]},{"osd":5,"up_from":38,"seq":163208757919,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":871940,"kb_used_data":5820,"kb_used_omap":0,"kb_used_meta":866112,"kb_avail":92873212,"statfs":{"total":95995035648,"available":95102169088,"internally_reserved":0,"allocated":5959680,"data_stored":4603988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":886898688},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:45:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":4.9539999999999997,"15min":2.3370000000000002},"min":{"1min":0.34599999999999997,"5min":0.22900000000000001,"15min":0.20899999999999999},"max":{"1min":0.81499999999999995,"5min":174.58600000000001,"15min":174.58600000000001},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.52300000000000002,"5min":5.0709999999999997,"15min":2.5739999999999998},"min":{"1min":0.38300000000000001,"5min":0.247,"15min":0.23699999999999999},"max":{"1min":0.68999999999999995,"5min":174.69499999999999,"15min":174.69499999999999},"last":0.72399999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:45:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":3.2570000000000001,"15min":2.02},"min":{"1min":0.32900000000000001,"5min":0.28000000000000003,"15min":0.23100000000000001},"max":{"1min":0.751,"5min":95.709000000000003,"15min":95.709000000000003},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.495,"5min":3.1459999999999999,"15min":2.0920000000000001},"min":{"1min":0.36399999999999999,"5min":0.27800000000000002,"15min":0.249},"max":{"1min":0.80200000000000005,"5min":94.966999999999999,"15min":94.966999999999999},"last":0.73599999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:46:04 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.331,"15min":3.9769999999999999},"min":{"1min":0.39200000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.82599999999999996,"5min":9.5879999999999992,"15min":208.738},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.7350000000000001,"15min":3.5049999999999999},"min":{"1min":0.41199999999999998,"5min":0.313,"15min":0.23999999999999999},"max":{"1min":0.68799999999999994,"5min":23.899000000000001,"15min":208.631},"last":0.58899999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:46:04 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.335,"15min":3.1230000000000002},"min":{"1min":0.435,"5min":0.311,"15min":0.27800000000000002},"max":{"1min":1.1719999999999999,"5min":14.288,"15min":143.66300000000001},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.357,"15min":3.2690000000000001},"min":{"1min":0.40999999999999998,"5min":0.35799999999999998,"15min":0.30399999999999999},"max":{"1min":1.157,"5min":14.307,"15min":143.62200000000001},"last":0.75700000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.48399999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.57599999999999996,"15min":1.2829999999999999},"min":{"1min":0.38700000000000001,"5min":0.29499999999999998,"15min":0.26100000000000001},"max":{"1min":0.79000000000000004,"5min":6.617,"15min":148.36600000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.627,"15min":1.4750000000000001},"min":{"1min":0.35299999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.81299999999999994,"5min":6.8040000000000003,"15min":148.42500000000001},"last":0.53100000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:46:04 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.52700000000000002,"15min":0.93000000000000005},"min":{"1min":0.46100000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.879,"5min":1.109,"15min":15.497},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.63400000000000001,"15min":1.0009999999999999},"min":{"1min":0.49199999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.98099999999999998,"5min":2.8690000000000002,"15min":15.653},"last":0.65500000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4160,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8041,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:46:37.580 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T23:46:37.580 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T23:46:37.643 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=2f2227164bc64a648135d6056d83d7a9 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T23:46:37.645 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-09-23T23:46:37.655 INFO:tasks.cephadm:Teardown begin 2023-09-23T23:46:37.655 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T23:46:37.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:37 smithi138 ceph-mon[193606]: pgmap v2135: 103 pgs: 103 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:46:37.730 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T23:46:37.778 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-09-23T23:46:37.779 DEBUG:teuthology.orchestra.run.smithi110:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T23:46:37.796 DEBUG:teuthology.orchestra.run.smithi138:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T23:46:37.815 INFO:tasks.cephadm:Stopping all daemons... 2023-09-23T23:46:37.815 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-09-23T23:46:37.816 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T23:46:38.113 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:37 smithi110 systemd[1]: Stopping Ceph mon.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:46:38.113 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:38 smithi110 conmon[212914]: 2023-09-23T23:46:38.110+0000 7f29afc6f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:46:38.113 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:38 smithi110 conmon[212914]: 2023-09-23T23:46:38.110+0000 7f29afc6f700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-09-23T23:46:38.702 INFO:journalctl@ceph.mon.a.smithi110.stdout:Sep 23 23:46:38 smithi110 bash[730218]: d6c089d81bde0ea9dbc676c3c112ab790973c04dd0cd29f29cd2c7fa95d19d08 2023-09-23T23:46:39.023 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mon.a.service' 2023-09-23T23:46:39.075 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:46:39.075 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-09-23T23:46:39.075 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-09-23T23:46:39.075 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mon.c 2023-09-23T23:46:39.385 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:39 smithi110 systemd[1]: Stopping Ceph mon.c for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:46:39.385 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:39 smithi110 conmon[221078]: 2023-09-23T23:46:39.381+0000 7fb7170a1700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:46:39.385 INFO:journalctl@ceph.mon.c.smithi110.stdout:Sep 23 23:46:39 smithi110 conmon[221078]: 2023-09-23T23:46:39.381+0000 7fb7170a1700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-09-23T23:46:39.699 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:46:39 smithi110 conmon[198813]: [23/Sep/2023:23:46:39] ENGINE Bus STOPPING 2023-09-23T23:46:39.699 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:46:39 smithi110 conmon[198813]: [23/Sep/2023:23:46:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T23:46:39.699 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:46:39 smithi110 conmon[198813]: [23/Sep/2023:23:46:39] ENGINE Bus STOPPED 2023-09-23T23:46:39.700 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:46:39 smithi110 conmon[198813]: [23/Sep/2023:23:46:39] ENGINE Bus STARTING 2023-09-23T23:46:39.700 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:46:39 smithi110 conmon[198813]: [23/Sep/2023:23:46:39] ENGINE Serving on http://:::9283 2023-09-23T23:46:39.700 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:46:39 smithi110 conmon[198813]: [23/Sep/2023:23:46:39] ENGINE Bus STARTED 2023-09-23T23:46:40.450 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mon.c.service' 2023-09-23T23:46:40.489 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:46:40.489 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-09-23T23:46:40.489 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-09-23T23:46:40.490 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mon.b 2023-09-23T23:46:40.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:40 smithi138 systemd[1]: Stopping Ceph mon.b for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:46:40.942 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:40 smithi138 conmon[193581]: 2023-09-23T23:46:40.753+0000 7ff82829b700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:46:40.943 INFO:journalctl@ceph.mon.b.smithi138.stdout:Sep 23 23:46:40 smithi138 conmon[193581]: 2023-09-23T23:46:40.753+0000 7ff82829b700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-09-23T23:46:41.748 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mon.b.service' 2023-09-23T23:46:41.788 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:46:41.788 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-09-23T23:46:41.788 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-09-23T23:46:41.789 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T23:46:42.084 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:46:41 smithi110 systemd[1]: Stopping Ceph mgr.y for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:46:42.084 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:46:42 smithi110 conmon[198813]: 2023-09-23T23:46:42.054+0000 7ff050424700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:46:42.084 INFO:journalctl@ceph.mgr.y.smithi110.stdout:Sep 23 23:46:42 smithi110 conmon[198813]: 2023-09-23T23:46:42.054+0000 7ff050424700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2023-09-23T23:46:42.839 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mgr.y.service' 2023-09-23T23:46:42.876 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:46:42.877 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-09-23T23:46:42.877 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-09-23T23:46:42.877 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mgr.x 2023-09-23T23:46:43.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Sep 23 23:46:42 smithi138 systemd[1]: Stopping Ceph mgr.x for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:46:43.950 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@mgr.x.service' 2023-09-23T23:46:43.987 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:46:43.987 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-09-23T23:46:43.987 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-09-23T23:46:43.988 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T23:46:44.452 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:46:44 smithi110 systemd[1]: Stopping Ceph osd.0 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:46:44.453 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:46:44 smithi110 conmon[238972]: 2023-09-23T23:46:44.262+0000 7fb1a85a4700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:46:44.453 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:46:44 smithi110 conmon[238972]: 2023-09-23T23:46:44.262+0000 7fb1a85a4700 -1 osd.0 773 *** Got signal Terminated *** 2023-09-23T23:46:44.453 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:46:44 smithi110 conmon[238972]: 2023-09-23T23:46:44.262+0000 7fb1a85a4700 -1 osd.0 773 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:46:50.202 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:46:49 smithi110 bash[731122]: 0c340b7feaa9953c955694b859f2ec198c46966573902eb017dc1c54aa851f5c 2023-09-23T23:46:52.446 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.0.service' 2023-09-23T23:46:52.472 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:46:52 smithi110 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.0.service: Succeeded. 2023-09-23T23:46:52.473 INFO:journalctl@ceph.osd.0.smithi110.stdout:Sep 23 23:46:52 smithi110 systemd[1]: Stopped Ceph osd.0 for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:46:52.487 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:46:52.487 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-09-23T23:46:52.487 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T23:46:52.487 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T23:46:52.785 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:46:52 smithi110 systemd[1]: Stopping Ceph osd.1 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:46:53.202 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:46:52 smithi110 conmon[250401]: 2023-09-23T23:46:52.783+0000 7fdf8b73f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:46:53.203 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:46:52 smithi110 conmon[250401]: 2023-09-23T23:46:52.783+0000 7fdf8b73f700 -1 osd.1 773 *** Got signal Terminated *** 2023-09-23T23:46:53.203 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:46:52 smithi110 conmon[250401]: 2023-09-23T23:46:52.783+0000 7fdf8b73f700 -1 osd.1 773 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:46:57.693 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:46:57 smithi138 conmon[177361]: ts=2023-09-23T23:46:57.429Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.110:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.110:8765: connect: connection refused" 2023-09-23T23:46:57.693 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:46:57 smithi138 conmon[177361]: ts=2023-09-23T23:46:57.429Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.110:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.110:8765: connect: connection refused" 2023-09-23T23:46:57.693 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:46:57 smithi138 conmon[177361]: ts=2023-09-23T23:46:57.429Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.110:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.110:8765: connect: connection refused" 2023-09-23T23:46:57.693 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:46:57 smithi138 conmon[177361]: ts=2023-09-23T23:46:57.429Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.110:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.110:8765: connect: connection refused" 2023-09-23T23:46:59.453 INFO:journalctl@ceph.osd.1.smithi110.stdout:Sep 23 23:46:59 smithi110 bash[731704]: 038c0580de94c2fe3da0388aa26ae41baf6833e17db7460b1d2cebe6c8bc1315 2023-09-23T23:47:00.711 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T23:47:00.752 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:47:00.753 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T23:47:00.753 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T23:47:00.753 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T23:47:01.055 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:47:00 smithi110 systemd[1]: Stopping Ceph osd.2 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:47:01.452 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:47:01 smithi110 conmon[328908]: 2023-09-23T23:47:01.051+0000 7fa5f9c14700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:47:01.453 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:47:01 smithi110 conmon[328908]: 2023-09-23T23:47:01.051+0000 7fa5f9c14700 -1 osd.2 773 *** Got signal Terminated *** 2023-09-23T23:47:01.453 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:47:01 smithi110 conmon[328908]: 2023-09-23T23:47:01.051+0000 7fa5f9c14700 -1 osd.2 773 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:47:06.952 INFO:journalctl@ceph.osd.2.smithi110.stdout:Sep 23 23:47:06 smithi110 bash[732283]: ebe9f18b6270c3d57f236c8f1c0da9938009320a4e1d397c3dace0938ae496e9 2023-09-23T23:47:08.442 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 23:47:08 smithi138 conmon[172649]: logger=cleanup t=2023-09-23T23:47:08.025715302Z level=info msg="Completed cleanup jobs" duration=90.465454ms 2023-09-23T23:47:08.552 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T23:47:08.592 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:47:08.592 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T23:47:08.592 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-09-23T23:47:08.593 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.3 2023-09-23T23:47:08.883 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:47:08 smithi110 systemd[1]: Stopping Ceph osd.3 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:47:09.202 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:47:08 smithi110 conmon[270750]: 2023-09-23T23:47:08.880+0000 7f5fdb0f2700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:47:09.202 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:47:08 smithi110 conmon[270750]: 2023-09-23T23:47:08.880+0000 7f5fdb0f2700 -1 osd.3 773 *** Got signal Terminated *** 2023-09-23T23:47:09.203 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:47:08 smithi110 conmon[270750]: 2023-09-23T23:47:08.880+0000 7f5fdb0f2700 -1 osd.3 773 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:47:11.443 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:10 smithi138 conmon[131081]: 2023-09-23T23:47:10.951+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:11.452 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:47:11 smithi110 conmon[270750]: 2023-09-23T23:47:11.023+0000 7f5fd5ee5700 -1 osd.3 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.491709+0000 front 2023-09-23T23:46:48.491833+0000 (oldest deadline 2023-09-23T23:47:10.791884+0000) 2023-09-23T23:47:12.442 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:11 smithi138 conmon[131081]: 2023-09-23T23:47:11.982+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:12.452 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:47:12 smithi110 conmon[270750]: 2023-09-23T23:47:12.003+0000 7f5fd5ee5700 -1 osd.3 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.491709+0000 front 2023-09-23T23:46:48.491833+0000 (oldest deadline 2023-09-23T23:47:10.791884+0000) 2023-09-23T23:47:13.389 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:12 smithi138 conmon[131081]: 2023-09-23T23:47:12.983+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:13.452 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:47:12 smithi110 conmon[270750]: 2023-09-23T23:47:12.960+0000 7f5fd5ee5700 -1 osd.3 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.491709+0000 front 2023-09-23T23:46:48.491833+0000 (oldest deadline 2023-09-23T23:47:10.791884+0000) 2023-09-23T23:47:13.692 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:13 smithi138 conmon[275015]: 2023-09-23T23:47:13.559+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:13.693 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:47:13 smithi138 conmon[421553]: 2023-09-23T23:47:13.599+0000 7f119b63a700 -1 osd.4 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:49.161962+0000 front 2023-09-23T23:46:49.162015+0000 (oldest deadline 2023-09-23T23:47:13.261891+0000) 2023-09-23T23:47:13.693 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:13 smithi138 conmon[122074]: 2023-09-23T23:47:13.387+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:47.336182+0000 front 2023-09-23T23:46:47.336268+0000 (oldest deadline 2023-09-23T23:47:13.236199+0000) 2023-09-23T23:47:14.371 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:13 smithi138 conmon[131081]: 2023-09-23T23:47:13.953+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:14.692 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:14 smithi138 conmon[275015]: 2023-09-23T23:47:14.538+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:14.693 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:47:14 smithi138 conmon[421553]: 2023-09-23T23:47:14.586+0000 7f119b63a700 -1 osd.4 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:49.161962+0000 front 2023-09-23T23:46:49.162015+0000 (oldest deadline 2023-09-23T23:47:13.261891+0000) 2023-09-23T23:47:14.693 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:14 smithi138 conmon[122074]: 2023-09-23T23:47:14.369+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:47.336182+0000 front 2023-09-23T23:46:47.336268+0000 (oldest deadline 2023-09-23T23:47:13.236199+0000) 2023-09-23T23:47:14.952 INFO:journalctl@ceph.osd.3.smithi110.stdout:Sep 23 23:47:14 smithi110 bash[732872]: ff8e0fe10bdc86c51c90e979e94a6df14944096d4a2105d6081f57165af9d11c 2023-09-23T23:47:15.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:14 smithi138 conmon[131081]: 2023-09-23T23:47:14.906+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:15.692 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:15 smithi138 conmon[122074]: 2023-09-23T23:47:15.378+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:47.336182+0000 front 2023-09-23T23:46:47.336268+0000 (oldest deadline 2023-09-23T23:47:13.236199+0000) 2023-09-23T23:47:15.693 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:15 smithi138 conmon[275015]: 2023-09-23T23:47:15.541+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:15.693 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:47:15 smithi138 conmon[421553]: 2023-09-23T23:47:15.590+0000 7f119b63a700 -1 osd.4 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:49.161962+0000 front 2023-09-23T23:46:49.162015+0000 (oldest deadline 2023-09-23T23:47:13.261891+0000) 2023-09-23T23:47:16.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:15 smithi138 conmon[131081]: 2023-09-23T23:47:15.859+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:16.401 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.3.service' 2023-09-23T23:47:16.442 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:47:16.442 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-09-23T23:47:16.443 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-09-23T23:47:16.443 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.4 2023-09-23T23:47:16.692 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:16 smithi138 conmon[122074]: 2023-09-23T23:47:16.368+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:47.336182+0000 front 2023-09-23T23:46:47.336268+0000 (oldest deadline 2023-09-23T23:47:13.236199+0000) 2023-09-23T23:47:16.693 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:16 smithi138 conmon[275015]: 2023-09-23T23:47:16.494+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:16.693 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:47:16 smithi138 systemd[1]: Stopping Ceph osd.4 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:47:16.693 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:47:16 smithi138 conmon[421553]: 2023-09-23T23:47:16.598+0000 7f119b63a700 -1 osd.4 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:49.161962+0000 front 2023-09-23T23:46:49.162015+0000 (oldest deadline 2023-09-23T23:47:13.261891+0000) 2023-09-23T23:47:17.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:16 smithi138 conmon[131081]: 2023-09-23T23:47:16.849+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:17.193 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:47:16 smithi138 conmon[421553]: 2023-09-23T23:47:16.726+0000 7f119e03a700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:47:17.193 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:47:16 smithi138 conmon[421553]: 2023-09-23T23:47:16.726+0000 7f119e03a700 -1 osd.4 773 *** Got signal Terminated *** 2023-09-23T23:47:17.193 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:47:16 smithi138 conmon[421553]: 2023-09-23T23:47:16.726+0000 7f119e03a700 -1 osd.4 773 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:47:17.600 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:17 smithi138 conmon[275015]: 2023-09-23T23:47:17.465+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:17.600 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:17 smithi138 conmon[122074]: 2023-09-23T23:47:17.322+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:47.336182+0000 front 2023-09-23T23:46:47.336268+0000 (oldest deadline 2023-09-23T23:47:13.236199+0000) 2023-09-23T23:47:17.893 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:47:17 smithi138 conmon[421553]: 2023-09-23T23:47:17.598+0000 7f119b63a700 -1 osd.4 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:49.161962+0000 front 2023-09-23T23:46:49.162015+0000 (oldest deadline 2023-09-23T23:47:13.261891+0000) 2023-09-23T23:47:18.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:17 smithi138 conmon[131081]: 2023-09-23T23:47:17.891+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:18.560 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:18 smithi138 conmon[275015]: 2023-09-23T23:47:18.423+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:18.561 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:18 smithi138 conmon[122074]: 2023-09-23T23:47:18.302+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:47.336182+0000 front 2023-09-23T23:46:47.336268+0000 (oldest deadline 2023-09-23T23:47:13.236199+0000) 2023-09-23T23:47:18.874 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:47:18 smithi138 conmon[421553]: 2023-09-23T23:47:18.559+0000 7f119b63a700 -1 osd.4 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:49.161962+0000 front 2023-09-23T23:46:49.162015+0000 (oldest deadline 2023-09-23T23:47:13.261891+0000) 2023-09-23T23:47:19.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:18 smithi138 conmon[131081]: 2023-09-23T23:47:18.873+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:19.526 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:19 smithi138 conmon[275015]: 2023-09-23T23:47:19.456+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:19.526 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:19 smithi138 conmon[122074]: 2023-09-23T23:47:19.266+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:47.336182+0000 front 2023-09-23T23:46:47.336268+0000 (oldest deadline 2023-09-23T23:47:13.236199+0000) 2023-09-23T23:47:19.843 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:47:19 smithi138 conmon[421553]: 2023-09-23T23:47:19.524+0000 7f119b63a700 -1 osd.4 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:49.161962+0000 front 2023-09-23T23:46:49.162015+0000 (oldest deadline 2023-09-23T23:47:13.261891+0000) 2023-09-23T23:47:20.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:19 smithi138 conmon[131081]: 2023-09-23T23:47:19.841+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:20.193 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:19 smithi138 conmon[131081]: 2023-09-23T23:47:19.841+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:20.692 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:20 smithi138 conmon[275015]: 2023-09-23T23:47:20.453+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:20.692 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:20 smithi138 conmon[122074]: 2023-09-23T23:47:20.265+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:47.336182+0000 front 2023-09-23T23:46:47.336268+0000 (oldest deadline 2023-09-23T23:47:13.236199+0000) 2023-09-23T23:47:20.693 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:47:20 smithi138 conmon[421553]: 2023-09-23T23:47:20.475+0000 7f119b63a700 -1 osd.4 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:49.161962+0000 front 2023-09-23T23:46:49.162015+0000 (oldest deadline 2023-09-23T23:47:13.261891+0000) 2023-09-23T23:47:21.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:20 smithi138 conmon[131081]: 2023-09-23T23:47:20.864+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:21.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:20 smithi138 conmon[131081]: 2023-09-23T23:47:20.864+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:21.692 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:21 smithi138 conmon[275015]: 2023-09-23T23:47:21.455+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:21.693 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:47:21 smithi138 conmon[421553]: 2023-09-23T23:47:21.464+0000 7f119b63a700 -1 osd.4 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:49.161962+0000 front 2023-09-23T23:46:49.162015+0000 (oldest deadline 2023-09-23T23:47:13.261891+0000) 2023-09-23T23:47:21.693 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:21 smithi138 conmon[122074]: 2023-09-23T23:47:21.240+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:47.336182+0000 front 2023-09-23T23:46:47.336268+0000 (oldest deadline 2023-09-23T23:47:13.236199+0000) 2023-09-23T23:47:21.693 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:21 smithi138 conmon[122074]: 2023-09-23T23:47:21.240+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:57.737949+0000 front 2023-09-23T23:46:57.738033+0000 (oldest deadline 2023-09-23T23:47:21.138552+0000) 2023-09-23T23:47:22.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:21 smithi138 conmon[131081]: 2023-09-23T23:47:21.903+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:22.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:21 smithi138 conmon[131081]: 2023-09-23T23:47:21.903+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:22.671 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:22 smithi138 conmon[275015]: 2023-09-23T23:47:22.469+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:22.671 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:22 smithi138 conmon[275015]: 2023-09-23T23:47:22.469+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:22.671 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:22 smithi138 conmon[122074]: 2023-09-23T23:47:22.230+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:47.336182+0000 front 2023-09-23T23:46:47.336268+0000 (oldest deadline 2023-09-23T23:47:13.236199+0000) 2023-09-23T23:47:22.671 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:22 smithi138 conmon[122074]: 2023-09-23T23:47:22.230+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:57.737949+0000 front 2023-09-23T23:46:57.738033+0000 (oldest deadline 2023-09-23T23:47:21.138552+0000) 2023-09-23T23:47:22.942 INFO:journalctl@ceph.osd.4.smithi138.stdout:Sep 23 23:47:22 smithi138 bash[429323]: b664c4d251ad92701b68d043ba4b697be6110f9690a87ca5eecd6bed010482a1 2023-09-23T23:47:22.943 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:22 smithi138 conmon[131081]: 2023-09-23T23:47:22.904+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:22.943 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:22 smithi138 conmon[131081]: 2023-09-23T23:47:22.904+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:23.692 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:23 smithi138 conmon[122074]: 2023-09-23T23:47:23.246+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:47.336182+0000 front 2023-09-23T23:46:47.336268+0000 (oldest deadline 2023-09-23T23:47:13.236199+0000) 2023-09-23T23:47:23.693 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:23 smithi138 conmon[122074]: 2023-09-23T23:47:23.246+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:57.737949+0000 front 2023-09-23T23:46:57.738033+0000 (oldest deadline 2023-09-23T23:47:21.138552+0000) 2023-09-23T23:47:23.693 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:23 smithi138 conmon[275015]: 2023-09-23T23:47:23.465+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:23.693 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:23 smithi138 conmon[275015]: 2023-09-23T23:47:23.465+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:24.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:23 smithi138 conmon[131081]: 2023-09-23T23:47:23.933+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:24.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:23 smithi138 conmon[131081]: 2023-09-23T23:47:23.933+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:24.443 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.4.service' 2023-09-23T23:47:24.483 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:47:24.483 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-09-23T23:47:24.483 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-09-23T23:47:24.483 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.5 2023-09-23T23:47:24.547 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:24 smithi138 conmon[275015]: 2023-09-23T23:47:24.471+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:24.547 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:24 smithi138 conmon[275015]: 2023-09-23T23:47:24.471+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:24.547 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:24 smithi138 conmon[122074]: 2023-09-23T23:47:24.282+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:47.336182+0000 front 2023-09-23T23:46:47.336268+0000 (oldest deadline 2023-09-23T23:47:13.236199+0000) 2023-09-23T23:47:24.547 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:24 smithi138 conmon[122074]: 2023-09-23T23:47:24.282+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:57.737949+0000 front 2023-09-23T23:46:57.738033+0000 (oldest deadline 2023-09-23T23:47:21.138552+0000) 2023-09-23T23:47:24.801 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:24 smithi138 systemd[1]: Stopping Ceph osd.5 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:47:24.802 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:24 smithi138 conmon[122074]: 2023-09-23T23:47:24.799+0000 7fb5de0a6700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:47:25.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:24 smithi138 conmon[131081]: 2023-09-23T23:47:24.908+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:25.193 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:24 smithi138 conmon[131081]: 2023-09-23T23:47:24.908+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:25.193 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:24 smithi138 conmon[122074]: 2023-09-23T23:47:24.799+0000 7fb5de0a6700 -1 osd.5 773 *** Got signal Terminated *** 2023-09-23T23:47:25.193 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:24 smithi138 conmon[122074]: 2023-09-23T23:47:24.799+0000 7fb5de0a6700 -1 osd.5 773 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:47:25.692 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:25 smithi138 conmon[275015]: 2023-09-23T23:47:25.433+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:25.692 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:25 smithi138 conmon[275015]: 2023-09-23T23:47:25.433+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:25.693 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:25 smithi138 conmon[122074]: 2023-09-23T23:47:25.242+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:47.336182+0000 front 2023-09-23T23:46:47.336268+0000 (oldest deadline 2023-09-23T23:47:13.236199+0000) 2023-09-23T23:47:25.693 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:25 smithi138 conmon[122074]: 2023-09-23T23:47:25.242+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:57.737949+0000 front 2023-09-23T23:46:57.738033+0000 (oldest deadline 2023-09-23T23:47:21.138552+0000) 2023-09-23T23:47:26.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:25 smithi138 conmon[131081]: 2023-09-23T23:47:25.936+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:26.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:25 smithi138 conmon[131081]: 2023-09-23T23:47:25.936+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:26.692 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:26 smithi138 conmon[122074]: 2023-09-23T23:47:26.253+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:47.336182+0000 front 2023-09-23T23:46:47.336268+0000 (oldest deadline 2023-09-23T23:47:13.236199+0000) 2023-09-23T23:47:26.692 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:26 smithi138 conmon[122074]: 2023-09-23T23:47:26.253+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:57.737949+0000 front 2023-09-23T23:46:57.738033+0000 (oldest deadline 2023-09-23T23:47:21.138552+0000) 2023-09-23T23:47:26.693 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:26 smithi138 conmon[275015]: 2023-09-23T23:47:26.394+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:26.693 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:26 smithi138 conmon[275015]: 2023-09-23T23:47:26.394+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:27.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:26 smithi138 conmon[131081]: 2023-09-23T23:47:26.924+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:27.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:26 smithi138 conmon[131081]: 2023-09-23T23:47:26.924+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:27.692 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:27 smithi138 conmon[275015]: 2023-09-23T23:47:27.378+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:27.692 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:27 smithi138 conmon[275015]: 2023-09-23T23:47:27.378+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:27.693 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:27 smithi138 conmon[122074]: 2023-09-23T23:47:27.238+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:47.336182+0000 front 2023-09-23T23:46:47.336268+0000 (oldest deadline 2023-09-23T23:47:13.236199+0000) 2023-09-23T23:47:27.693 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:27 smithi138 conmon[122074]: 2023-09-23T23:47:27.238+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:57.737949+0000 front 2023-09-23T23:46:57.738033+0000 (oldest deadline 2023-09-23T23:47:21.138552+0000) 2023-09-23T23:47:28.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:27 smithi138 conmon[131081]: 2023-09-23T23:47:27.875+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:28.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:27 smithi138 conmon[131081]: 2023-09-23T23:47:27.875+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:28.693 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:28 smithi138 conmon[275015]: 2023-09-23T23:47:28.381+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:28.694 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:28 smithi138 conmon[275015]: 2023-09-23T23:47:28.381+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:28.694 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:28 smithi138 conmon[122074]: 2023-09-23T23:47:28.192+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:47.336182+0000 front 2023-09-23T23:46:47.336268+0000 (oldest deadline 2023-09-23T23:47:13.236199+0000) 2023-09-23T23:47:28.694 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:28 smithi138 conmon[122074]: 2023-09-23T23:47:28.192+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:57.737949+0000 front 2023-09-23T23:46:57.738033+0000 (oldest deadline 2023-09-23T23:47:21.138552+0000) 2023-09-23T23:47:28.694 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:28 smithi138 conmon[122074]: 2023-09-23T23:47:28.192+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:03.439801+0000 front 2023-09-23T23:47:03.439768+0000 (oldest deadline 2023-09-23T23:47:27.539678+0000) 2023-09-23T23:47:29.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:28 smithi138 conmon[131081]: 2023-09-23T23:47:28.920+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:29.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:28 smithi138 conmon[131081]: 2023-09-23T23:47:28.920+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:29.692 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:29 smithi138 conmon[275015]: 2023-09-23T23:47:29.389+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:29.692 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:29 smithi138 conmon[275015]: 2023-09-23T23:47:29.389+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:29.692 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:29 smithi138 conmon[122074]: 2023-09-23T23:47:29.194+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:47.336182+0000 front 2023-09-23T23:46:47.336268+0000 (oldest deadline 2023-09-23T23:47:13.236199+0000) 2023-09-23T23:47:29.693 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:29 smithi138 conmon[122074]: 2023-09-23T23:47:29.194+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:57.737949+0000 front 2023-09-23T23:46:57.738033+0000 (oldest deadline 2023-09-23T23:47:21.138552+0000) 2023-09-23T23:47:29.693 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:29 smithi138 conmon[122074]: 2023-09-23T23:47:29.194+0000 7fb5d8ea1700 -1 osd.5 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:03.439801+0000 front 2023-09-23T23:47:03.439768+0000 (oldest deadline 2023-09-23T23:47:27.539678+0000) 2023-09-23T23:47:30.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:29 smithi138 conmon[131081]: 2023-09-23T23:47:29.878+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:30.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:29 smithi138 conmon[131081]: 2023-09-23T23:47:29.878+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:30.692 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:30 smithi138 conmon[275015]: 2023-09-23T23:47:30.436+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:30.693 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:30 smithi138 conmon[275015]: 2023-09-23T23:47:30.436+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:30.693 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:30 smithi138 conmon[275015]: 2023-09-23T23:47:30.436+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:01.605083+0000 front 2023-09-23T23:47:01.605211+0000 (oldest deadline 2023-09-23T23:47:30.405570+0000) 2023-09-23T23:47:31.192 INFO:journalctl@ceph.osd.5.smithi138.stdout:Sep 23 23:47:30 smithi138 bash[429898]: fb3decabc2c4e6124bcf5679209afb2b1371a6879897246efe7401e5a0af1496 2023-09-23T23:47:31.193 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:30 smithi138 conmon[131081]: 2023-09-23T23:47:30.839+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:31.193 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:30 smithi138 conmon[131081]: 2023-09-23T23:47:30.839+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:31.836 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:31 smithi138 conmon[275015]: 2023-09-23T23:47:31.474+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:31.836 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:31 smithi138 conmon[275015]: 2023-09-23T23:47:31.474+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:31.836 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:31 smithi138 conmon[275015]: 2023-09-23T23:47:31.474+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:01.605083+0000 front 2023-09-23T23:47:01.605211+0000 (oldest deadline 2023-09-23T23:47:30.405570+0000) 2023-09-23T23:47:32.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:31 smithi138 conmon[131081]: 2023-09-23T23:47:31.880+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:32.193 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:31 smithi138 conmon[131081]: 2023-09-23T23:47:31.880+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:32.193 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:31 smithi138 conmon[131081]: 2023-09-23T23:47:31.880+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:05.579553+0000 front 2023-09-23T23:47:05.579589+0000 (oldest deadline 2023-09-23T23:47:31.479523+0000) 2023-09-23T23:47:32.475 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.5.service' 2023-09-23T23:47:32.558 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:47:32.558 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-09-23T23:47:32.558 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-23T23:47:32.558 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T23:47:32.848 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:32 smithi138 systemd[1]: Stopping Ceph osd.6 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:47:32.848 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:32 smithi138 conmon[131081]: 2023-09-23T23:47:32.845+0000 7fb611134700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:47:32.849 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:32 smithi138 conmon[275015]: 2023-09-23T23:47:32.445+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:32.849 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:32 smithi138 conmon[275015]: 2023-09-23T23:47:32.445+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:32.849 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:32 smithi138 conmon[275015]: 2023-09-23T23:47:32.445+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:01.605083+0000 front 2023-09-23T23:47:01.605211+0000 (oldest deadline 2023-09-23T23:47:30.405570+0000) 2023-09-23T23:47:33.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:32 smithi138 conmon[131081]: 2023-09-23T23:47:32.845+0000 7fb611134700 -1 osd.6 773 *** Got signal Terminated *** 2023-09-23T23:47:33.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:32 smithi138 conmon[131081]: 2023-09-23T23:47:32.845+0000 7fb611134700 -1 osd.6 773 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:47:33.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:32 smithi138 conmon[131081]: 2023-09-23T23:47:32.918+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:33.193 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:32 smithi138 conmon[131081]: 2023-09-23T23:47:32.918+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:33.193 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:32 smithi138 conmon[131081]: 2023-09-23T23:47:32.918+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:05.579553+0000 front 2023-09-23T23:47:05.579589+0000 (oldest deadline 2023-09-23T23:47:31.479523+0000) 2023-09-23T23:47:33.940 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:33 smithi138 conmon[275015]: 2023-09-23T23:47:33.441+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:33.940 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:33 smithi138 conmon[275015]: 2023-09-23T23:47:33.441+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:33.940 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:33 smithi138 conmon[275015]: 2023-09-23T23:47:33.441+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:01.605083+0000 front 2023-09-23T23:47:01.605211+0000 (oldest deadline 2023-09-23T23:47:30.405570+0000) 2023-09-23T23:47:34.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:33 smithi138 conmon[131081]: 2023-09-23T23:47:33.938+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:34.192 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:33 smithi138 conmon[131081]: 2023-09-23T23:47:33.938+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:34.193 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:33 smithi138 conmon[131081]: 2023-09-23T23:47:33.938+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:05.579553+0000 front 2023-09-23T23:47:05.579589+0000 (oldest deadline 2023-09-23T23:47:31.479523+0000) 2023-09-23T23:47:34.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:34 smithi138 conmon[275015]: 2023-09-23T23:47:34.472+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:34.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:34 smithi138 conmon[275015]: 2023-09-23T23:47:34.472+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:34.943 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:34 smithi138 conmon[275015]: 2023-09-23T23:47:34.472+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:01.605083+0000 front 2023-09-23T23:47:01.605211+0000 (oldest deadline 2023-09-23T23:47:30.405570+0000) 2023-09-23T23:47:35.442 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:34 smithi138 conmon[131081]: 2023-09-23T23:47:34.964+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:35.442 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:34 smithi138 conmon[131081]: 2023-09-23T23:47:34.964+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:35.443 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:34 smithi138 conmon[131081]: 2023-09-23T23:47:34.964+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:05.579553+0000 front 2023-09-23T23:47:05.579589+0000 (oldest deadline 2023-09-23T23:47:31.479523+0000) 2023-09-23T23:47:35.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:35 smithi138 conmon[275015]: 2023-09-23T23:47:35.441+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:35.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:35 smithi138 conmon[275015]: 2023-09-23T23:47:35.441+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:35.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:35 smithi138 conmon[275015]: 2023-09-23T23:47:35.441+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:01.605083+0000 front 2023-09-23T23:47:01.605211+0000 (oldest deadline 2023-09-23T23:47:30.405570+0000) 2023-09-23T23:47:36.442 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:35 smithi138 conmon[131081]: 2023-09-23T23:47:35.961+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:36.442 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:35 smithi138 conmon[131081]: 2023-09-23T23:47:35.961+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:36.442 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:35 smithi138 conmon[131081]: 2023-09-23T23:47:35.961+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:05.579553+0000 front 2023-09-23T23:47:05.579589+0000 (oldest deadline 2023-09-23T23:47:31.479523+0000) 2023-09-23T23:47:36.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:36 smithi138 conmon[275015]: 2023-09-23T23:47:36.471+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:36.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:36 smithi138 conmon[275015]: 2023-09-23T23:47:36.471+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:36.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:36 smithi138 conmon[275015]: 2023-09-23T23:47:36.471+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:01.605083+0000 front 2023-09-23T23:47:01.605211+0000 (oldest deadline 2023-09-23T23:47:30.405570+0000) 2023-09-23T23:47:37.438 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:36 smithi138 conmon[131081]: 2023-09-23T23:47:36.990+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:46.877026+0000 front 2023-09-23T23:46:46.877054+0000 (oldest deadline 2023-09-23T23:47:10.376909+0000) 2023-09-23T23:47:37.438 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:36 smithi138 conmon[131081]: 2023-09-23T23:47:36.990+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:53.277943+0000 front 2023-09-23T23:46:53.278073+0000 (oldest deadline 2023-09-23T23:47:19.177885+0000) 2023-09-23T23:47:37.438 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:36 smithi138 conmon[131081]: 2023-09-23T23:47:36.990+0000 7fb60a72c700 -1 osd.6 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:05.579553+0000 front 2023-09-23T23:47:05.579589+0000 (oldest deadline 2023-09-23T23:47:31.479523+0000) 2023-09-23T23:47:37.692 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:37 smithi138 conmon[275015]: 2023-09-23T23:47:37.435+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:37.692 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:37 smithi138 conmon[275015]: 2023-09-23T23:47:37.435+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:37.692 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:37 smithi138 conmon[275015]: 2023-09-23T23:47:37.435+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:01.605083+0000 front 2023-09-23T23:47:01.605211+0000 (oldest deadline 2023-09-23T23:47:30.405570+0000) 2023-09-23T23:47:37.693 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:37 smithi138 conmon[275015]: 2023-09-23T23:47:37.435+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2023-09-23T23:47:11.506691+0000 front 2023-09-23T23:47:11.506675+0000 (oldest deadline 2023-09-23T23:47:36.706984+0000) 2023-09-23T23:47:38.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:38 smithi138 conmon[275015]: 2023-09-23T23:47:38.467+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:38.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:38 smithi138 conmon[275015]: 2023-09-23T23:47:38.467+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:38.943 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:38 smithi138 conmon[275015]: 2023-09-23T23:47:38.467+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:01.605083+0000 front 2023-09-23T23:47:01.605211+0000 (oldest deadline 2023-09-23T23:47:30.405570+0000) 2023-09-23T23:47:38.943 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:38 smithi138 conmon[275015]: 2023-09-23T23:47:38.467+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2023-09-23T23:47:11.506691+0000 front 2023-09-23T23:47:11.506675+0000 (oldest deadline 2023-09-23T23:47:36.706984+0000) 2023-09-23T23:47:39.442 INFO:journalctl@ceph.osd.6.smithi138.stdout:Sep 23 23:47:39 smithi138 bash[430474]: c3f9d2352b7c14581cd3e232ae32d53c8ba444b3e5de419513eb96ddc02ff4e6 2023-09-23T23:47:39.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:39 smithi138 conmon[275015]: 2023-09-23T23:47:39.471+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:39.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:39 smithi138 conmon[275015]: 2023-09-23T23:47:39.471+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:39.943 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:39 smithi138 conmon[275015]: 2023-09-23T23:47:39.471+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:01.605083+0000 front 2023-09-23T23:47:01.605211+0000 (oldest deadline 2023-09-23T23:47:30.405570+0000) 2023-09-23T23:47:39.943 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:39 smithi138 conmon[275015]: 2023-09-23T23:47:39.471+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2023-09-23T23:47:11.506691+0000 front 2023-09-23T23:47:11.506675+0000 (oldest deadline 2023-09-23T23:47:36.706984+0000) 2023-09-23T23:47:40.641 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-23T23:47:40.680 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:47:40.680 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-23T23:47:40.680 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-09-23T23:47:40.680 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T23:47:40.744 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:40 smithi138 conmon[275015]: 2023-09-23T23:47:40.462+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:40.744 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:40 smithi138 conmon[275015]: 2023-09-23T23:47:40.462+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:40.744 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:40 smithi138 conmon[275015]: 2023-09-23T23:47:40.462+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:01.605083+0000 front 2023-09-23T23:47:01.605211+0000 (oldest deadline 2023-09-23T23:47:30.405570+0000) 2023-09-23T23:47:40.744 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:40 smithi138 conmon[275015]: 2023-09-23T23:47:40.462+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2023-09-23T23:47:11.506691+0000 front 2023-09-23T23:47:11.506675+0000 (oldest deadline 2023-09-23T23:47:36.706984+0000) 2023-09-23T23:47:41.000 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:40 smithi138 systemd[1]: Stopping Ceph osd.7 for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:47:41.420 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:40 smithi138 conmon[275015]: 2023-09-23T23:47:40.997+0000 7f0c7ef75700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:47:41.421 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:40 smithi138 conmon[275015]: 2023-09-23T23:47:40.997+0000 7f0c7ef75700 -1 osd.7 773 *** Got signal Terminated *** 2023-09-23T23:47:41.421 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:40 smithi138 conmon[275015]: 2023-09-23T23:47:40.997+0000 7f0c7ef75700 -1 osd.7 773 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:47:41.692 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:41 smithi138 conmon[275015]: 2023-09-23T23:47:41.418+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:41.692 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:41 smithi138 conmon[275015]: 2023-09-23T23:47:41.418+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:41.692 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:41 smithi138 conmon[275015]: 2023-09-23T23:47:41.418+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:01.605083+0000 front 2023-09-23T23:47:01.605211+0000 (oldest deadline 2023-09-23T23:47:30.405570+0000) 2023-09-23T23:47:41.693 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:41 smithi138 conmon[275015]: 2023-09-23T23:47:41.418+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2023-09-23T23:47:11.506691+0000 front 2023-09-23T23:47:11.506675+0000 (oldest deadline 2023-09-23T23:47:36.706984+0000) 2023-09-23T23:47:42.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:42 smithi138 conmon[275015]: 2023-09-23T23:47:42.457+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:42.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:42 smithi138 conmon[275015]: 2023-09-23T23:47:42.457+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:42.943 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:42 smithi138 conmon[275015]: 2023-09-23T23:47:42.457+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:01.605083+0000 front 2023-09-23T23:47:01.605211+0000 (oldest deadline 2023-09-23T23:47:30.405570+0000) 2023-09-23T23:47:42.943 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:42 smithi138 conmon[275015]: 2023-09-23T23:47:42.457+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2023-09-23T23:47:11.506691+0000 front 2023-09-23T23:47:11.506675+0000 (oldest deadline 2023-09-23T23:47:36.706984+0000) 2023-09-23T23:47:43.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:43 smithi138 conmon[275015]: 2023-09-23T23:47:43.452+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:43.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:43 smithi138 conmon[275015]: 2023-09-23T23:47:43.452+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:43.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:43 smithi138 conmon[275015]: 2023-09-23T23:47:43.452+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:01.605083+0000 front 2023-09-23T23:47:01.605211+0000 (oldest deadline 2023-09-23T23:47:30.405570+0000) 2023-09-23T23:47:43.943 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:43 smithi138 conmon[275015]: 2023-09-23T23:47:43.452+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2023-09-23T23:47:11.506691+0000 front 2023-09-23T23:47:11.506675+0000 (oldest deadline 2023-09-23T23:47:36.706984+0000) 2023-09-23T23:47:44.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:44 smithi138 conmon[275015]: 2023-09-23T23:47:44.462+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:44.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:44 smithi138 conmon[275015]: 2023-09-23T23:47:44.462+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:44.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:44 smithi138 conmon[275015]: 2023-09-23T23:47:44.462+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:01.605083+0000 front 2023-09-23T23:47:01.605211+0000 (oldest deadline 2023-09-23T23:47:30.405570+0000) 2023-09-23T23:47:44.943 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:44 smithi138 conmon[275015]: 2023-09-23T23:47:44.462+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2023-09-23T23:47:11.506691+0000 front 2023-09-23T23:47:11.506675+0000 (oldest deadline 2023-09-23T23:47:36.706984+0000) 2023-09-23T23:47:44.943 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:44 smithi138 conmon[275015]: 2023-09-23T23:47:44.462+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.138:6804 osd.4 since back 2023-09-23T23:47:16.707296+0000 front 2023-09-23T23:47:16.707327+0000 (oldest deadline 2023-09-23T23:47:44.307840+0000) 2023-09-23T23:47:45.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:45 smithi138 conmon[275015]: 2023-09-23T23:47:45.449+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6806 osd.0 since back 2023-09-23T23:46:48.302705+0000 front 2023-09-23T23:46:48.302589+0000 (oldest deadline 2023-09-23T23:47:13.002547+0000) 2023-09-23T23:47:45.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:45 smithi138 conmon[275015]: 2023-09-23T23:47:45.449+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6814 osd.1 since back 2023-09-23T23:46:55.704602+0000 front 2023-09-23T23:46:55.704590+0000 (oldest deadline 2023-09-23T23:47:21.604512+0000) 2023-09-23T23:47:45.942 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:45 smithi138 conmon[275015]: 2023-09-23T23:47:45.449+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6822 osd.2 since back 2023-09-23T23:47:01.605083+0000 front 2023-09-23T23:47:01.605211+0000 (oldest deadline 2023-09-23T23:47:30.405570+0000) 2023-09-23T23:47:45.943 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:45 smithi138 conmon[275015]: 2023-09-23T23:47:45.449+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.110:6830 osd.3 since back 2023-09-23T23:47:11.506691+0000 front 2023-09-23T23:47:11.506675+0000 (oldest deadline 2023-09-23T23:47:36.706984+0000) 2023-09-23T23:47:45.943 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:45 smithi138 conmon[275015]: 2023-09-23T23:47:45.449+0000 7f0c7856d700 -1 osd.7 773 heartbeat_check: no reply from 172.21.15.138:6804 osd.4 since back 2023-09-23T23:47:16.707296+0000 front 2023-09-23T23:47:16.707327+0000 (oldest deadline 2023-09-23T23:47:44.307840+0000) 2023-09-23T23:47:47.192 INFO:journalctl@ceph.osd.7.smithi138.stdout:Sep 23 23:47:46 smithi138 bash[431052]: 641f92ed2e0cb2c7a905aa5a77568d8f0649db39dd871f4497f6909ec25f5ca8 2023-09-23T23:47:48.638 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@osd.7.service' 2023-09-23T23:47:48.719 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:47:48.720 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-09-23T23:47:48.720 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-09-23T23:47:48.720 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@prometheus.a 2023-09-23T23:47:48.942 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:47:48 smithi138 systemd[1]: Stopping Ceph prometheus.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:47:49.442 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:47:49 smithi138 conmon[177361]: ts=2023-09-23T23:47:49.018Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T23:47:49.442 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:47:49 smithi138 conmon[177361]: ts=2023-09-23T23:47:49.018Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T23:47:49.442 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:47:49 smithi138 conmon[177361]: ts=2023-09-23T23:47:49.018Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T23:47:49.442 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:47:49 smithi138 conmon[177361]: ts=2023-09-23T23:47:49.018Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T23:47:49.443 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:47:49 smithi138 conmon[177361]: ts=2023-09-23T23:47:49.018Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T23:47:49.443 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:47:49 smithi138 conmon[177361]: ts=2023-09-23T23:47:49.018Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T23:47:49.443 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:47:49 smithi138 conmon[177361]: ts=2023-09-23T23:47:49.018Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T23:47:49.443 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:47:49 smithi138 conmon[177361]: ts=2023-09-23T23:47:49.018Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T23:47:49.443 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:47:49 smithi138 conmon[177361]: ts=2023-09-23T23:47:49.018Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T23:47:49.443 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:47:49 smithi138 conmon[177361]: ts=2023-09-23T23:47:49.020Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T23:47:49.443 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:47:49 smithi138 conmon[177361]: ts=2023-09-23T23:47:49.020Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T23:47:49.444 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Sep 23 23:47:49 smithi138 conmon[177361]: ts=2023-09-23T23:47:49.020Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T23:47:49.724 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@prometheus.a.service' 2023-09-23T23:47:49.760 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:47:49.760 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-09-23T23:47:49.761 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T23:47:51.203 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 23:47:50 smithi110 systemd[1]: Stopping Ceph alertmanager.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:47:51.702 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 23:47:51 smithi110 conmon[196953]: ts=2023-09-23T23:47:51.220Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T23:47:52.046 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 23:47:51 smithi110 bash[733748]: d9641d6ed9caa90f8e0c45a56c037903f7eb3097b342e7adafb892187052668e 2023-09-23T23:47:52.046 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 23:47:51 smithi110 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T23:47:52.046 INFO:journalctl@ceph.alertmanager.a.smithi110.stdout:Sep 23 23:47:51 smithi110 systemd[1]: Stopped Ceph alertmanager.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:47:52.389 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 23:47:52 smithi110 systemd[1]: Stopping Ceph node-exporter.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:47:53.011 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 23:47:52 smithi110 bash[734027]: 39d48d803cf2a37b843b8d37945c5e3d2adf26bef9fe2a8a71fcf24c02defe8d 2023-09-23T23:47:53.011 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 23:47:52 smithi110 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T23:47:53.011 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 23:47:52 smithi110 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T23:47:53.012 INFO:journalctl@ceph.node-exporter.a.smithi110.stdout:Sep 23 23:47:52 smithi110 systemd[1]: Stopped Ceph node-exporter.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:48:04.371 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T23:48:05.692 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 23:48:05 smithi138 systemd[1]: Stopping Ceph grafana.a for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:48:06.581 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 23:48:06 smithi138 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T23:48:06.581 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 23:48:06 smithi138 bash[432238]: 56929ec304ecde569332f0740e5cc8b5e9d3da0e3da108c31c5352f11dfd8dfe 2023-09-23T23:48:06.581 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 23:48:06 smithi138 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-23T23:48:06.581 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Sep 23 23:48:06 smithi138 systemd[1]: Stopped Ceph grafana.a for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:48:07.192 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 23:48:06 smithi138 systemd[1]: Stopping Ceph node-exporter.b for f5f05148-5a62-11ee-8db4-212e2dc638e7... 2023-09-23T23:48:07.549 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 23:48:07 smithi138 bash[432518]: a55c5d37ba4b2f0a4a0560771c481bd95e589f34b0092c51849a47564ce3f258 2023-09-23T23:48:07.549 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 23:48:07 smithi138 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T23:48:07.549 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 23:48:07 smithi138 systemd[1]: ceph-f5f05148-5a62-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T23:48:07.549 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Sep 23 23:48:07 smithi138 systemd[1]: Stopped Ceph node-exporter.b for f5f05148-5a62-11ee-8db4-212e2dc638e7. 2023-09-23T23:48:11.883 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T23:48:11.933 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T23:48:11.964 INFO:tasks.cephadm:Archiving crash dumps... 2023-09-23T23:48:11.966 DEBUG:teuthology.misc:Transferring archived files from smithi110:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401893/remote/smithi110/crash 2023-09-23T23:48:11.966 DEBUG:teuthology.orchestra.run.smithi110:> sudo tar c -f - -C /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T23:48:12.008 INFO:teuthology.orchestra.run.smithi110.stderr:tar: /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T23:48:12.008 INFO:teuthology.orchestra.run.smithi110.stderr:tar: Error is not recoverable: exiting now 2023-09-23T23:48:12.010 DEBUG:teuthology.misc:Transferring archived files from smithi138:/var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401893/remote/smithi138/crash 2023-09-23T23:48:12.011 DEBUG:teuthology.orchestra.run.smithi138:> sudo tar c -f - -C /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T23:48:12.080 INFO:teuthology.orchestra.run.smithi138.stderr:tar: /var/lib/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T23:48:12.080 INFO:teuthology.orchestra.run.smithi138.stderr:tar: Error is not recoverable: exiting now 2023-09-23T23:48:12.081 INFO:tasks.cephadm:Checking cluster log for badness... 2023-09-23T23:48:12.082 DEBUG:teuthology.orchestra.run.smithi110:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-09-23T23:48:12.120 INFO:teuthology.orchestra.run.smithi110.stderr:grep: /var/log/ceph/f5f05148-5a62-11ee-8db4-212e2dc638e7/ceph.log: No such file or directory 2023-09-23T23:48:12.122 INFO:tasks.cephadm:Compressing logs... 2023-09-23T23:48:12.123 DEBUG:teuthology.orchestra.run.smithi110:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T23:48:12.165 DEBUG:teuthology.orchestra.run.smithi138:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T23:48:12.222 INFO:teuthology.orchestra.run.smithi138.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-23T23:48:12.243 INFO:teuthology.orchestra.run.smithi110.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-24T00:03:12.111 INFO:tasks.cephadm:Archiving logs... 2023-09-24T00:03:12.112 DEBUG:teuthology.misc:Transferring archived files from smithi110:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401893/remote/smithi110/log 2023-09-24T00:03:12.112 DEBUG:teuthology.orchestra.run.smithi110:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-24T00:05:00.988 DEBUG:teuthology.misc:Transferring archived files from smithi138:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401893/remote/smithi138/log 2023-09-24T00:05:00.989 DEBUG:teuthology.orchestra.run.smithi138:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-24T00:06:07.125 INFO:tasks.cephadm:Removing cluster... 2023-09-24T00:06:07.126 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 --force 2023-09-24T00:06:08.608 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f5f05148-5a62-11ee-8db4-212e2dc638e7 --force 2023-09-24T00:06:10.006 INFO:tasks.cephadm:Removing cephadm ... 2023-09-24T00:06:10.006 DEBUG:teuthology.orchestra.run.smithi110:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-24T00:06:10.028 DEBUG:teuthology.orchestra.run.smithi138:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-24T00:06:10.055 INFO:tasks.cephadm:Teardown complete 2023-09-24T00:06:10.055 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-09-24T00:06:10.067 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-09-24T00:06:10.067 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-24T00:06:10.071 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-24T00:06:10.184 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-09-24T00:06:10.196 INFO:teuthology.task.clock:Checking final clock skew... 2023-09-24T00:06:10.197 DEBUG:teuthology.orchestra.run.smithi110:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-24T00:06:10.199 DEBUG:teuthology.orchestra.run.smithi138:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-24T00:06:10.215 INFO:teuthology.orchestra.run.smithi138.stderr:bash: ntpq: command not found 2023-09-24T00:06:10.215 INFO:teuthology.orchestra.run.smithi110.stderr:bash: ntpq: command not found 2023-09-24T00:06:10.236 INFO:teuthology.orchestra.run.smithi138.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-24T00:06:10.236 INFO:teuthology.orchestra.run.smithi138.stdout:=============================================================================== 2023-09-24T00:06:10.236 INFO:teuthology.orchestra.run.smithi138.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 15 +819us[ +852us] +/- 89ms 2023-09-24T00:06:10.236 INFO:teuthology.orchestra.run.smithi138.stdout:^* hv02.front.sepia.ceph.com 3 6 377 14 -1715us[-1682us] +/- 68ms 2023-09-24T00:06:10.237 INFO:teuthology.orchestra.run.smithi138.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 15 +1364us[+1397us] +/- 80ms 2023-09-24T00:06:10.237 INFO:teuthology.orchestra.run.smithi138.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-24T00:06:10.243 INFO:teuthology.orchestra.run.smithi110.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-24T00:06:10.243 INFO:teuthology.orchestra.run.smithi110.stdout:=============================================================================== 2023-09-24T00:06:10.243 INFO:teuthology.orchestra.run.smithi110.stdout:^+ hv01.front.sepia.ceph.com 2 7 377 77 +705us[ +724us] +/- 88ms 2023-09-24T00:06:10.243 INFO:teuthology.orchestra.run.smithi110.stdout:^* hv02.front.sepia.ceph.com 3 6 377 13 -1831us[-1812us] +/- 68ms 2023-09-24T00:06:10.244 INFO:teuthology.orchestra.run.smithi110.stdout:^+ hv03.front.sepia.ceph.com 3 7 377 15 +1242us[+1261us] +/- 80ms 2023-09-24T00:06:10.244 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-24T00:06:10.245 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-09-24T00:06:10.266 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-09-24T00:06:10.267 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-09-24T00:06:10.279 DEBUG:teuthology.orchestra.run.smithi110:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-24T00:06:10.432 DEBUG:teuthology.orchestra.run.smithi138:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-24T00:06:10.553 DEBUG:teuthology.orchestra.run.smithi110:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-24T00:06:10.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-24T00:06:10.587 DEBUG:teuthology.orchestra.run.smithi138:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-24T00:06:10.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-24T00:06:10.618 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-09-24T00:06:10.627 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-09-24T00:06:10.638 INFO:teuthology.task.internal:Duration was 5641.418647 seconds 2023-09-24T00:06:10.638 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-09-24T00:06:10.647 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-09-24T00:06:10.647 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-24T00:06:10.650 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-24T00:06:10.716 INFO:teuthology.orchestra.run.smithi110.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-24T00:06:10.738 INFO:teuthology.orchestra.run.smithi138.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-24T00:06:10.979 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-09-24T00:06:10.979 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi110.front.sepia.ceph.com 2023-09-24T00:06:10.979 DEBUG:teuthology.orchestra.run.smithi110:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-24T00:06:11.006 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi138.front.sepia.ceph.com 2023-09-24T00:06:11.007 DEBUG:teuthology.orchestra.run.smithi138:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-24T00:06:11.039 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-09-24T00:06:11.039 DEBUG:teuthology.orchestra.run.smithi110:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-24T00:06:11.049 DEBUG:teuthology.orchestra.run.smithi138:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-24T00:06:11.201 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-09-24T00:06:11.201 DEBUG:teuthology.orchestra.run.smithi110:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-24T00:06:11.203 DEBUG:teuthology.orchestra.run.smithi138:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-24T00:06:11.577 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-09-24T00:06:11.589 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-09-24T00:06:11.589 DEBUG:teuthology.orchestra.run.smithi110:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-24T00:06:11.617 DEBUG:teuthology.orchestra.run.smithi138:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-24T00:06:11.667 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-09-24T00:06:11.677 DEBUG:teuthology.orchestra.run.smithi110:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-24T00:06:11.679 DEBUG:teuthology.orchestra.run.smithi138:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-24T00:06:11.714 INFO:teuthology.orchestra.run.smithi110.stdout:kernel.core_pattern = core 2023-09-24T00:06:11.720 INFO:teuthology.orchestra.run.smithi138.stdout:kernel.core_pattern = core 2023-09-24T00:06:11.735 DEBUG:teuthology.orchestra.run.smithi110:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-24T00:06:11.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-24T00:06:11.792 DEBUG:teuthology.orchestra.run.smithi138:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-24T00:06:11.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-24T00:06:11.808 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-09-24T00:06:11.828 INFO:teuthology.task.internal:Transferring archived files... 2023-09-24T00:06:11.829 DEBUG:teuthology.misc:Transferring archived files from smithi110:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401893/remote/smithi110 2023-09-24T00:06:11.829 DEBUG:teuthology.orchestra.run.smithi110:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-24T00:06:11.896 DEBUG:teuthology.misc:Transferring archived files from smithi138:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401893/remote/smithi138 2023-09-24T00:06:11.896 DEBUG:teuthology.orchestra.run.smithi138:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-24T00:06:11.943 INFO:teuthology.task.internal:Removing archive directory... 2023-09-24T00:06:11.943 DEBUG:teuthology.orchestra.run.smithi110:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-24T00:06:11.945 DEBUG:teuthology.orchestra.run.smithi138:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-24T00:06:11.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-09-24T00:06:12.022 INFO:teuthology.task.internal:Not uploading archives. 2023-09-24T00:06:12.022 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-09-24T00:06:12.035 INFO:teuthology.task.internal:Tidying up after the test... 2023-09-24T00:06:12.035 DEBUG:teuthology.orchestra.run.smithi110:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-24T00:06:12.037 DEBUG:teuthology.orchestra.run.smithi138:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-24T00:06:12.051 INFO:teuthology.orchestra.run.smithi110.stdout: 262307 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 24 00:06 /home/ubuntu/cephtest 2023-09-24T00:06:12.051 INFO:teuthology.orchestra.run.smithi110.stdout: 6291467 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Sep 23 23:44 /home/ubuntu/cephtest/mnt.0 2023-09-24T00:06:12.051 INFO:teuthology.orchestra.run.smithi110.stdout: 6291468 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Sep 23 23:44 /home/ubuntu/cephtest/mnt.0/client.0 2023-09-24T00:06:12.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-24T00:06:12.052 INFO:teuthology.orchestra.run.smithi110.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-09-24T00:06:12.053 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi110 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-24T00:06:12.059 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-09-24T00:06:12.080 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-09-24T00:06:12.092 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi110 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-24T00:06:12.093 INFO:teuthology.nuke:Checking targets against current locks 2023-09-24T00:06:12.099 INFO:teuthology.orchestra.run.smithi138.stdout: 262307 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 24 00:06 /home/ubuntu/cephtest 2023-09-24T00:06:12.125 DEBUG:teuthology.nuke:shortname: smithi110 2023-09-24T00:06:12.126 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-24T00:06:12.146 DEBUG:teuthology.nuke:shortname: smithi138 2023-09-24T00:06:12.147 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-24T00:06:12.150 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi110.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401893', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 22:26:55.528142', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-24T00:06:12.174 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi138.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401893', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 22:26:55.529538', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-24T00:06:12.200 INFO:teuthology.orchestra.console.smithi110:Power off 2023-09-24T00:06:12.200 DEBUG:teuthology.orchestra.console.smithi110:pexpect command: ipmitool -H smithi110.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-24T00:06:12.223 DEBUG:teuthology.orchestra.console.smithi110:power off output: Chassis Power Control: Down/Off 2023-09-24T00:06:12.223 DEBUG:teuthology.orchestra.console.smithi110:pexpect command: ipmitool -H smithi110.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-24T00:06:12.241 INFO:teuthology.orchestra.console.smithi138:Power off 2023-09-24T00:06:12.242 DEBUG:teuthology.orchestra.console.smithi138:pexpect command: ipmitool -H smithi138.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-24T00:06:12.250 DEBUG:teuthology.orchestra.console.smithi110:check power output: Chassis Power is on 2023-09-24T00:06:12.262 DEBUG:teuthology.orchestra.console.smithi138:power off output: Chassis Power Control: Down/Off 2023-09-24T00:06:12.262 DEBUG:teuthology.orchestra.console.smithi138:pexpect command: ipmitool -H smithi138.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-24T00:06:12.284 DEBUG:teuthology.orchestra.console.smithi138:check power output: Chassis Power is on 2023-09-24T00:06:16.252 DEBUG:teuthology.orchestra.console.smithi110:pexpect command: ipmitool -H smithi110.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-24T00:06:16.285 DEBUG:teuthology.orchestra.console.smithi138:pexpect command: ipmitool -H smithi138.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-24T00:06:16.361 DEBUG:teuthology.orchestra.console.smithi110:check power output: Chassis Power is on 2023-09-24T00:06:16.395 DEBUG:teuthology.orchestra.console.smithi138:check power output: Chassis Power is on 2023-09-24T00:06:20.367 DEBUG:teuthology.orchestra.console.smithi110:pexpect command: ipmitool -H smithi110.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-24T00:06:20.397 DEBUG:teuthology.orchestra.console.smithi138:pexpect command: ipmitool -H smithi138.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-24T00:06:20.478 DEBUG:teuthology.orchestra.console.smithi110:check power output: Chassis Power is off 2023-09-24T00:06:20.506 DEBUG:teuthology.orchestra.console.smithi138:check power output: Chassis Power is on 2023-09-24T00:06:20.579 INFO:teuthology.orchestra.console.smithi110:Power off completed 2023-09-24T00:06:24.511 DEBUG:teuthology.orchestra.console.smithi138:pexpect command: ipmitool -H smithi138.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-24T00:06:24.621 DEBUG:teuthology.orchestra.console.smithi138:check power output: Chassis Power is off 2023-09-24T00:06:24.722 INFO:teuthology.orchestra.console.smithi138:Power off completed 2023-09-24T00:06:24.826 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 5641.4186470508575 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=2f2227164bc64a648135d6056d83d7a9 status: fail success: false 2023-09-24T00:06:24.826 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-24T00:06:24.899 INFO:teuthology.run:FAIL